From e9f8db61b756a2cee148eca9e17121f1506ff2f8 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 10:18:17 +0530 Subject: [PATCH 01/11] Add the browser acceptance suite and run it in CI (R24 part 6) Playwright (pinned 1.63.0, approved) drives the built app and worker against the compose Postgres and Keycloak: organizer setup and import, invited and uninvited sign-in, profiles, background matches, search, intro consent, the venue question, isolation from a third attendee, analytics and closing the event; plus phone sign-in, single-use magic links, a tampered cookie, sign-out and phone-width layouts. Login codes come from the app's dev e-mail log. The CI job keeps the report and logs on failure. --- .github/workflows/ci.yml | 75 ++++++++++++++++++++++ ReadMe.md | 1 + package-lock.json | 46 ++++++++++++++ package.json | 2 + playwright.config.ts | 30 +++++++++ tests/e2e/edges.spec.ts | 89 ++++++++++++++++++++++++++ tests/e2e/journey.spec.ts | 129 ++++++++++++++++++++++++++++++++++++++ tests/e2e/support.ts | 101 +++++++++++++++++++++++++++++ 8 files changed, 473 insertions(+) create mode 100644 playwright.config.ts create mode 100644 tests/e2e/edges.spec.ts create mode 100644 tests/e2e/journey.spec.ts create mode 100644 tests/e2e/support.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a282379..27876c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,81 @@ jobs: if: failure() run: docker compose logs --no-color postgres keycloak | tail -n 300 + browser: + name: Browser acceptance + runs-on: ubuntu-latest + needs: [quality] + + # The built app and the worker against the compose Postgres and Keycloak. + # Login codes are logged instead of e-mailed and read from the app's log. + env: + DATABASE_URL: postgres://matchmaker:matchmaker@localhost:5432/matchmaker + DATABASE_SSL: disable + SESSION_SECRET: ci-session-secret-long-enough-for-validation-x + KEYCLOAK_ISSUER: http://localhost:8080/realms/match-maker + KEYCLOAK_PUBLIC_URL: http://localhost:8080 + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + KEYCLOAK_CLIENT_ID: match-maker-web + KEYCLOAK_CLIENT_SECRET: ci-client-secret + APP_BASE_URL: http://localhost:3000 + EMAIL_DEV_FALLBACK: "true" + MATCH_REFRESH_DELAY_SECONDS: "3" + E2E_BASE_URL: http://localhost:3000 + E2E_WEB_LOG: ${{ github.workspace }}/e2e-web.log + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + + - run: npm ci + + - name: Start Postgres and Keycloak + run: docker compose up -d --wait postgres keycloak + + - name: Migrate, seed and configure Keycloak + run: | + npm run db:migrate + npm run db:seed + npm run keycloak:wait + npm run keycloak:sync + + - name: Build + run: npm run build + + - name: Install Chromium + run: npx playwright install --with-deps chromium + + - name: Start app and worker + run: | + (cd apps/web && nohup npx next start -p 3000 > "$E2E_WEB_LOG" 2>&1 &) + nohup npm run worker > e2e-worker.log 2>&1 & + for i in $(seq 1 60); do + curl -fsS http://localhost:3000/api/health/ready && exit 0 + sleep 2 + done + cat "$E2E_WEB_LOG" + exit 1 + + - name: Browser tests + run: npm run test:e2e + + - name: Keep report and logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: browser-acceptance + path: | + playwright-report/ + test-results/ + e2e-web.log + e2e-worker.log + retention-days: 7 + build: name: Build images runs-on: ubuntu-latest diff --git a/ReadMe.md b/ReadMe.md index b51a594..f088f0a 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -103,6 +103,7 @@ The first command imports the global attendee directory so the organizer account | `npm run typecheck` | Type-check all workspaces | | `npm run lint` | Run ESLint | | `npm test` | Run unit and integration suites. Integration suites skip without Postgres/Keycloak; set `REQUIRE_SERVICES=db,keycloak` to fail instead (CI does, and `tools/check-test-report.ts` fails on any skipped test). Stop a locally running worker first, or it claims the pipeline test's jobs | +| `npm run test:e2e` | Browser acceptance suite (Playwright) against a running app and worker started with `EMAIL_DEV_FALLBACK=true`; set `E2E_BASE_URL` and `E2E_WEB_LOG` (the app's output, where login codes are read) | | `npm run db:migrate` | Apply pending PostgreSQL migrations | | `npm run db:seed` | Upsert demo roster, organization, event, and organizer | | `npm run db:import -- [event-slug]` | Import attendee rows and optionally attach them to an event | diff --git a/package-lock.json b/package-lock.json index 7a1db06..44a8a2f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "packages/*" ], "devDependencies": { + "@playwright/test": "1.63.0", "@types/node": "^22.10.2", "@types/pg": "^8.11.10", "@types/react": "^19.0.2", @@ -1512,6 +1513,22 @@ "node": ">=12.4.0" } }, + "node_modules/@playwright/test": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@react-email/render": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@react-email/render/-/render-1.1.2.tgz", @@ -6141,6 +6158,35 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "devOptional": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", diff --git a/package.json b/package.json index 36c262a..c29ef0b 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "typecheck": "turbo run typecheck", "test": "vitest run", "test:watch": "vitest", + "test:e2e": "playwright test", "db:migrate": "tsx packages/db/src/migrate.ts", "db:seed": "tsx packages/db/src/seed.ts", "db:import": "tsx tools/import-attendees.ts", @@ -38,6 +39,7 @@ "models:usage": "tsx tools/model-usage.ts" }, "devDependencies": { + "@playwright/test": "1.63.0", "@types/node": "^22.10.2", "@types/pg": "^8.11.10", "@types/react": "^19.0.2", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..89802d1 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,30 @@ +import { defineConfig } from "@playwright/test"; + +/** + * Browser acceptance suite (tests/e2e). It drives an already running app and + * worker, started with EMAIL_DEV_FALLBACK=true and their output written to + * E2E_WEB_LOG, which is where the tests read sign-in codes: + * + * E2E_BASE_URL=http://localhost:3000 npx playwright test + * + * Set E2E_BROWSER_CHANNEL=chrome to use an installed Chrome instead of the + * bundled Chromium (`npx playwright install chromium`). + */ +export default defineConfig({ + testDir: "tests/e2e", + // The journey is one story told in order; parallel runs would share an event. + workers: 1, + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: 0, + timeout: 60_000, + expect: { timeout: 15_000 }, + reporter: process.env.CI ? [["list"], ["html", { open: "never", outputFolder: "playwright-report" }]] : "list", + outputDir: "test-results/e2e", + use: { + baseURL: process.env.E2E_BASE_URL ?? "http://localhost:3000", + channel: process.env.E2E_BROWSER_CHANNEL || undefined, + screenshot: "only-on-failure", + trace: "retain-on-failure", + }, +}); diff --git a/tests/e2e/edges.spec.ts b/tests/e2e/edges.spec.ts new file mode 100644 index 0000000..908f5f7 --- /dev/null +++ b/tests/e2e/edges.spec.ts @@ -0,0 +1,89 @@ +/** + * Sign-in edges and layout: phone lookup still proves e-mail ownership, + * magic links work once, a tampered cookie leads to a usable login page, + * signing out ends access, and pages fit a phone screen. + */ +import { devices, expect, test, type Page } from "@playwright/test"; +import { ORG, ORGANIZER, RUN, createEvent, enterCode, logOffset, loginMail, phone, rosterFile, sidewaysOverflow, signIn } from "./support"; + +const SLUG = `e2e-edges-${RUN}`; +const dana = { email: `e2e-dana-${RUN}@example.com`, phone: phone() }; +const eli = { email: `e2e-eli-${RUN}@example.com`, phone: phone() }; +const PHONE = { ...devices["Pixel 7"] }; +delete (PHONE as { defaultBrowserType?: string }).defaultBrowserType; + +test.describe.configure({ mode: "serial" }); + +test.describe("sign-in edges and phone layout", () => { + let organizer: Page; + + test.beforeAll(async ({ browser }) => { + organizer = await signIn(browser, ORGANIZER, `/orgs/${ORG}/events/new`, PHONE); + await createEvent(organizer, SLUG); + await organizer.goto(`/orgs/${ORG}/events/${SLUG}/roster`); + await organizer.locator("#field-file").setInputFiles(rosterFile([ + { Name: "Dana Edge", Email: dana.email, Phone: dana.phone }, + { Name: "Eli Edge", Email: eli.email, Phone: eli.phone }, + ])); + await organizer.getByRole("button", { name: "Import roster" }).click(); + await expect(organizer.getByRole("status").first()).toContainText("Added 2 attendee"); + }); + + test("organizer pages fit a phone and offer sign out", async () => { + for (const path of ["", "/roster", "/matching", "/analytics"]) { + await organizer.goto(`/orgs/${ORG}/events/${SLUG}${path}`); + expect(await sidewaysOverflow(organizer), `organizer${path || " dashboard"}`).toBeLessThanOrEqual(1); + await expect(organizer.getByRole("button", { name: "Sign out" })).toHaveCount(1); + } + }); + + test("phone sign-in shows only a masked e-mail, then reaches the event", async ({ browser }) => { + const page = await (await browser.newContext(PHONE)).newPage(); + const returnTo = `/e/${SLUG}/connections`; + await page.goto(`/login?returnTo=${encodeURIComponent(returnTo)}`); + await page.getByRole("button", { name: /Phone Number/ }).click(); + await page.locator('input[type="tel"]').fill(dana.phone); + const since = logOffset(); + await page.getByRole("button", { name: "Send sign-in code" }).click(); + await expect(page.getByText("Check your inbox")).toBeVisible(); + await expect(page.locator("body")).not.toContainText(dana.email); + await enterCode(page, dana.email, since, returnTo); + + for (const path of ["/connections", "/matches"]) { + await page.goto(`/e/${SLUG}${path}`); + expect(await sidewaysOverflow(page), path).toBeLessThanOrEqual(1); + } + + await page.getByRole("button", { name: "Sign out" }).click(); + await page.waitForURL("**/login**"); + await page.goto(`/e/${SLUG}/connections`); + await expect(page).toHaveURL(/\/login/); + }); + + test("a magic link signs in once and cannot be reused", async ({ browser, baseURL }) => { + const page = await (await browser.newContext()).newPage(); + const since = logOffset(); + const response = await page.request.post("/api/auth/otp/request", { + headers: { "content-type": "application/json", origin: baseURL! }, + data: { email: eli.email, returnTo: `/e/${SLUG}/profile` }, + }); + expect(response.ok()).toBe(true); + const { link } = await loginMail(eli.email, since); + const local = link!.replace(/^https?:\/\/[^/]+/, ""); + await page.goto(local); + await expect(page).toHaveURL(new RegExp(`/e/${SLUG}/profile`)); + + const other = await (await browser.newContext()).newPage(); + await other.goto(local); + await expect(other).toHaveURL(/\/login/); + }); + + test("a tampered session cookie leads to a usable login page", async ({ browser, baseURL }) => { + const context = await browser.newContext(); + await context.addCookies([{ name: "mb_session", value: "tampered.value", url: baseURL! }]); + const page = await context.newPage(); + await page.goto(`/e/${SLUG}/matches`); + await expect(page).toHaveURL(/\/login/); + await expect(page.getByRole("button", { name: /Email Address/ })).toBeVisible(); + }); +}); diff --git a/tests/e2e/journey.spec.ts b/tests/e2e/journey.spec.ts new file mode 100644 index 0000000..02dea78 --- /dev/null +++ b/tests/e2e/journey.spec.ts @@ -0,0 +1,129 @@ +/** + * The documented journey, in order: an organizer sets up an event and + * imports the roster; invited attendees sign in and complete profiles; the + * worker matches them; search finds people; an intro is requested and + * accepted; contacts appear only then; a venue question is asked and + * answered; a third attendee sees none of it; closing the event freezes it. + */ +import { expect, test, type Page } from "@playwright/test"; +import { ORG, ORGANIZER, RUN, createEvent, phone, requestCode, rosterFile, signIn } from "./support"; + +const SLUG = `e2e-${RUN}`; +const people = { + alice: { name: "Alice E2E", designation: "ML Engineer", domain: "Artificial Intelligence", about: "I build retrieval systems and LLM evaluation tooling for startups.", interests: "AI, search, startups", intent: "NETWORKING" }, + bob: { name: "Bob E2E", designation: "Product Designer", domain: "Design", about: "Designing AI assistants for climate and health teams.", interests: "AI, design, climate", intent: "NETWORKING" }, + carol: { name: "Carol E2E", designation: "Founder", domain: "Artificial Intelligence", about: "Founder hiring engineers for an AI search startup.", interests: "AI, hiring, search", intent: "HIRING" }, +}; +type Person = keyof typeof people; +const email = (key: Person | "stranger") => `e2e-${key}-${RUN}@example.com`; + +test.describe.configure({ mode: "serial" }); + +test.describe("organizer to meeting", () => { + let organizer: Page; + const pages = {} as Record; + + test("organizer creates the event and imports the roster", async ({ browser }) => { + organizer = await signIn(browser, ORGANIZER, `/orgs/${ORG}/events/new`); + await createEvent(organizer, SLUG); + await organizer.goto(`/orgs/${ORG}/events/${SLUG}/roster`); + const rows = (Object.keys(people) as Person[]).map((key) => ({ Name: people[key].name, Email: email(key), Phone: phone() })); + await organizer.locator("#field-file").setInputFiles(rosterFile([...rows, { Name: "Broken Row", Email: "not-an-email" }])); + await organizer.getByRole("button", { name: "Import roster" }).click(); + const status = organizer.getByRole("status").first(); + await expect(status).toContainText("Added 3 attendee"); + await expect(status).toContainText("Skipped 1 row"); + }); + + test("an uninvited e-mail cannot sign in", async ({ browser }) => { + const page = await (await browser.newContext()).newPage(); + expect((await requestCode(page, email("stranger"), `/e/${SLUG}/profile`)).outcome).toBe("unregistered"); + }); + + test("invited attendees sign in and complete their profiles", async ({ browser }) => { + for (const key of Object.keys(people) as Person[]) { + const page = pages[key] = await signIn(browser, email(key), `/e/${SLUG}/profile`); + const person = people[key]; + await page.getByLabel("Role or designation").fill(person.designation); + await page.getByLabel("Professional domain").fill(person.domain); + await page.getByLabel("About you").fill(person.about); + await page.getByLabel("Years of experience").fill("5"); + await page.getByLabel("Interests").fill(person.interests); + await page.getByLabel("What are you looking for?").selectOption(person.intent); + await page.getByLabel("Where can people find you?").selectOption("Main hall"); + await page.getByRole("button", { name: /Save profile/ }).click(); + await page.waitForURL(/\/matches\?saved=1/); + } + }); + + test("the worker matches the first person to save without a manual refresh", async () => { + const alice = pages.alice; + await expect(async () => { + await alice.goto(`/e/${SLUG}/matches`); + await expect(alice.locator("article", { hasText: people.bob.name }).getByRole("button", { name: "Request intro" })).toBeVisible({ timeout: 1000 }); + }).toPass({ timeout: 45_000, intervals: [1500] }); + }); + + test("search finds others and never the searcher", async () => { + const alice = pages.alice; + await alice.goto(`/e/${SLUG}/search`); + await alice.getByLabel("Describe who you are looking for").fill("designer working on climate"); + await alice.getByRole("button", { name: "Search" }).click(); + const results = alice.locator("article"); + await expect(results.filter({ hasText: people.bob.name })).toHaveCount(1); + await expect(results.filter({ hasText: people.alice.name })).toHaveCount(0); + }); + + test("an intro reveals contacts only after the recipient accepts", async () => { + const { alice, bob } = pages; + await alice.goto(`/e/${SLUG}/matches`); + await alice.locator("article", { hasText: people.bob.name }).getByRole("button", { name: "Request intro" }).click(); + await alice.waitForURL(/connections\?requested=1/); + await expect(alice.locator("main")).not.toContainText(email("bob")); + + await bob.goto(`/e/${SLUG}/connections`); + await bob.getByRole("button", { name: "Accept" }).first().click(); + await bob.waitForLoadState("networkidle"); + await alice.goto(`/e/${SLUG}/connections`); + await expect(alice.locator("main")).toContainText(email("bob")); + }); + + test("a venue question is answered voluntarily and only the asker sees the zone", async () => { + const { alice, bob, carol } = pages; + await alice.getByRole("button", { name: "Ask where they are" }).click(); + await alice.waitForURL(/asked=1/); + await expect(alice.locator("main")).toContainText("Waiting for a reply"); + + await bob.goto(`/e/${SLUG}/connections`); + await expect(bob.locator("main")).toContainText(`${people.alice.name} is asking where you are`); + await bob.getByLabel("Your venue zone").first().selectOption("Cafe"); + await bob.getByRole("button", { name: `Share with ${people.alice.name}` }).click(); + await bob.waitForURL(/replied=1/); + + await alice.goto(`/e/${SLUG}/connections`); + await expect(alice.locator("main")).toContainText("Replied: at Cafe"); + await carol.goto(`/e/${SLUG}/connections`); + const carolView = carol.locator("main"); + for (const hidden of [email("alice"), email("bob"), "Cafe"]) await expect(carolView).not.toContainText(hidden); + }); + + test("analytics counts the accepted intro", async () => { + await organizer.goto(`/orgs/${ORG}/events/${SLUG}/analytics`); + await expect(organizer.locator("main")).toContainText(/Accepted introductions\s*1/); + }); + + test("closing the event keeps matches readable and stops changes", async () => { + await organizer.goto(`/orgs/${ORG}/events/${SLUG}`); + await organizer.getByLabel("Registration status").selectOption("closed"); + await organizer.getByRole("button", { name: "Save event configuration" }).click(); + await expect(organizer.getByText("Event configuration saved")).toBeVisible(); + + const { alice } = pages; + await alice.goto(`/e/${SLUG}/matches`); + await expect(alice.locator("main")).toContainText("has closed"); + await expect(alice.locator("main")).toContainText(people.bob.name); + await expect(alice.getByRole("button", { name: "Request intro" })).toHaveCount(0); + await alice.goto(`/e/${SLUG}/profile`); + await expect(alice).toHaveURL(new RegExp(`/e/${SLUG}$`)); + }); +}); diff --git a/tests/e2e/support.ts b/tests/e2e/support.ts new file mode 100644 index 0000000..e842cb7 --- /dev/null +++ b/tests/e2e/support.ts @@ -0,0 +1,101 @@ +/** + * Helpers for the browser suite: sign in through the real login page with + * the code the app logged (EMAIL_DEV_FALLBACK), build roster sheets, and + * create events. Every run uses its own event and attendees. + */ +import { readFileSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { expect, type Browser, type BrowserContextOptions, type Page } from "@playwright/test"; +import pg from "pg"; +import * as XLSX from "xlsx"; +import "dotenv/config"; + +export const ORG = process.env.E2E_ORGANIZATION ?? "ai-community-lucknow"; +export const ORGANIZER = process.env.E2E_ORGANIZER_EMAIL ?? "organizer@example.com"; +const WEB_LOG = process.env.E2E_WEB_LOG ?? "node_modules/.cache/e2e/web.log"; + +export const RUN = Date.now().toString(36); +export const phone = () => `9${String(Math.floor(Math.random() * 1e9)).padStart(9, "0")}`; + +export async function db(sql: string, params: unknown[] = []): Promise { + const client = new pg.Client({ connectionString: process.env.DATABASE_URL }); + await client.connect(); + try { + return (await client.query(sql, params)).rows; + } finally { + await client.end(); + } +} + +/** Byte offset of the end of the app log; mail logged after it is new. */ +export const logOffset = () => statSync(WEB_LOG).size; + +/** The newest login e-mail the app logged for `email` after byte offset `since`. */ +export async function loginMail(email: string, since: number): Promise<{ code: string; link: string | null }> { + for (let attempt = 0; attempt < 40; attempt++) { + const log = readFileSync(WEB_LOG).subarray(since).toString("utf8"); + const at = log.lastIndexOf(`To: ${email}`); + const code = at >= 0 ? log.slice(at).match(/Code:\s+(\d{6})/)?.[1] : undefined; + if (code) return { code, link: log.slice(at).match(/Link:\s+(\S+)/)?.[1] ?? null }; + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error(`no login e-mail was logged for ${email}; is the app writing to ${WEB_LOG}?`); +} + +/** Sends a code from the login page; resolves with what the page answered. */ +export async function requestCode(page: Page, email: string, returnTo: string): Promise<{ outcome: "sent" | "unregistered"; since: number }> { + await db(`delete from login_challenges where email = $1`, [email]); + await page.goto(`/login?returnTo=${encodeURIComponent(returnTo)}`); + await page.getByRole("button", { name: /Email Address/ }).click(); + await page.locator('input[type="email"]').fill(email); + const since = logOffset(); + await page.getByRole("button", { name: "Send sign-in code" }).click(); + const outcome = await Promise.race([ + page.getByText("Check your inbox").waitFor().then(() => "sent" as const), + page.getByText(/not on the attendee list/i).waitFor().then(() => "unregistered" as const), + ]); + return { outcome, since }; +} + +export async function enterCode(page: Page, email: string, since: number, returnTo: string) { + const { code } = await loginMail(email, since); + await page.getByLabel("Six-digit email verification code").fill(code); + await page.getByRole("button", { name: "Verify & Sign In" }).click(); + await page.waitForURL((url) => url.pathname === returnTo.split("?")[0]); +} + +/** A new browser session signed in as `email`, on `returnTo`. */ +export async function signIn(browser: Browser, email: string, returnTo: string, options: BrowserContextOptions = {}): Promise { + const context = await browser.newContext(options); + const page = await context.newPage(); + const { outcome, since } = await requestCode(page, email, returnTo); + expect(outcome, `${email} should be invited`).toBe("sent"); + await enterCode(page, email, since, returnTo); + return page; +} + +export function rosterFile(rows: Array>): string { + const book = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(book, XLSX.utils.json_to_sheet(rows), "Roster"); + const file = join(tmpdir(), `mr-bridge-roster-${RUN}-${rows.length}.xlsx`); + writeFileSync(file, XLSX.write(book, { type: "buffer", bookType: "xlsx" })); + return file; +} + +export async function createEvent(page: Page, slug: string, zones = "Main hall, Cafe") { + await page.goto(`/orgs/${ORG}/events/new`); + await page.getByLabel("Event name").fill(`E2E ${slug}`); + await page.getByLabel("URL slug").fill(slug); + await page.getByRole("button", { name: "Create event" }).click(); + await page.waitForURL(`**/orgs/${ORG}/events/${slug}`); + await page.getByLabel("Supported intents").fill("NETWORKING, HIRING, MENTORING"); + await page.getByLabel("Venue zones").fill(zones); + await page.getByLabel("Registration status").selectOption("open"); + await page.getByRole("button", { name: "Save event configuration" }).click(); + await expect(page.getByText("Event configuration saved")).toBeVisible(); +} + +/** How far the page scrolls sideways; more than a pixel means a broken phone layout. */ +export const sidewaysOverflow = (page: Page) => + page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); From d9453db706f25bb1a85006b7dfd8074e8eb849eb Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 10:21:50 +0530 Subject: [PATCH 02/11] Rotate the session secret without signing anyone out (R25 part 1) Cookies are read with SESSION_SECRET and then any SESSION_SECRET_PREVIOUS value, and always sealed with the current one. Terraform keeps two secret slots and a session_secret_active switch, so a rotation regenerates the inactive slot and makes it active in one apply. Login codes get their own OTP_PEPPER so a rotation leaves codes in flight valid. --- .env.example | 6 ++++++ infra/terraform/ecs.tf | 2 ++ infra/terraform/secrets.tf | 22 ++++++++++++++++++++-- infra/terraform/variables.tf | 11 +++++++++++ packages/auth/src/config.ts | 11 +++++++++++ packages/auth/src/session.ts | 20 ++++++++++++++++---- tests/unit/session.test.ts | 27 +++++++++++++++++++++++++++ 7 files changed, 93 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 78c0412..a4fe63f 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,12 @@ DEFAULT_EVENT_SLUG=demo # Encrypts the session cookie. REQUIRED, min 32 chars. # openssl rand -base64 48 SESSION_SECRET= +# While rotating: the old secret(s), comma-separated, still accepted for +# reading cookies for one session lifetime so nobody is signed out. +# SESSION_SECRET_PREVIOUS= +# Hashes login codes. Defaults to SESSION_SECRET; set it separately in +# production so rotating the session secret leaves codes in flight working. +# OTP_PEPPER= # ── Postgres ────────────────────────────────────────────────── POSTGRES_USER=matchmaker diff --git a/infra/terraform/ecs.tf b/infra/terraform/ecs.tf index f2b8095..bb30e30 100644 --- a/infra/terraform/ecs.tf +++ b/infra/terraform/ecs.tf @@ -230,6 +230,8 @@ resource "aws_ecs_task_definition" "web" { secrets = [ { name = "DATABASE_URL", valueFrom = local.secret_ref.DATABASE_URL }, { name = "SESSION_SECRET", valueFrom = local.secret_ref.SESSION_SECRET }, + { name = "SESSION_SECRET_PREVIOUS", valueFrom = local.secret_ref.SESSION_SECRET_PREVIOUS }, + { name = "OTP_PEPPER", valueFrom = local.secret_ref.OTP_PEPPER }, { name = "KEYCLOAK_CLIENT_SECRET", valueFrom = local.secret_ref.KEYCLOAK_CLIENT_SECRET }, { name = "RESEND_API_KEY", valueFrom = local.secret_ref.RESEND_API_KEY }, # Only its presence is read: it switches the Google sign-in option on. diff --git a/infra/terraform/secrets.tf b/infra/terraform/secrets.tf index f3b1e51..5261cd1 100644 --- a/infra/terraform/secrets.tf +++ b/infra/terraform/secrets.tf @@ -5,12 +5,28 @@ # ============================================================== # Encrypts the session cookie. Generated here so it never has to be handled by a -# human; rotating it simply signs everyone out. +# human. Two slots make rotation free of sign-outs: the active slot seals new +# cookies, the other is still accepted for reading (SESSION_SECRET_PREVIOUS). +# To rotate, regenerate the inactive slot and make it active in one apply: +# terraform apply -replace=random_password.session_secret_b -var session_secret_active=b +# (next time: -replace=random_password.session_secret -var session_secret_active=a) resource "random_password" "session_secret" { length = 64 special = false } +resource "random_password" "session_secret_b" { + length = 64 + special = false +} + +# Hashes login codes; separate from the session secret so rotating that one +# leaves codes already e-mailed valid. +resource "random_password" "otp_pepper" { + length = 64 + special = false +} + resource "random_password" "keycloak_client_secret" { length = 48 special = false @@ -29,7 +45,9 @@ resource "random_password" "keycloak_admin_password" { locals { secret_values = { DATABASE_URL = local.app_database_url - SESSION_SECRET = random_password.session_secret.result + SESSION_SECRET = var.session_secret_active == "a" ? random_password.session_secret.result : random_password.session_secret_b.result + SESSION_SECRET_PREVIOUS = var.session_secret_active == "a" ? random_password.session_secret_b.result : random_password.session_secret.result + OTP_PEPPER = random_password.otp_pepper.result KEYCLOAK_CLIENT_SECRET = random_password.keycloak_client_secret.result GEMINI_API_KEY = var.gemini_api_key UPSTASH_REDIS_REST_URL = var.upstash_redis_rest_url diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf index 028ca0f..42114e4 100644 --- a/infra/terraform/variables.tf +++ b/infra/terraform/variables.tf @@ -305,3 +305,14 @@ variable "enable_container_insights" { type = bool default = true } + +variable "session_secret_active" { + description = "Which session-secret slot seals new cookies (a or b); the other is still accepted. Flip it, with -replace of the newly active slot, to rotate without signing anyone out." + type = string + default = "a" + + validation { + condition = contains(["a", "b"], var.session_secret_active) + error_message = "session_secret_active must be \"a\" or \"b\"." + } +} diff --git a/packages/auth/src/config.ts b/packages/auth/src/config.ts index 4168037..4f6fb5e 100644 --- a/packages/auth/src/config.ts +++ b/packages/auth/src/config.ts @@ -69,6 +69,12 @@ export interface SessionConfig { cookieName: string; /** Session lifetime in seconds. */ maxAge: number; + /** + * Earlier secrets still accepted for reading cookies (never for writing), + * so rotating SESSION_SECRET does not sign anyone out. Keep the old value in + * SESSION_SECRET_PREVIOUS for at least one session lifetime. + */ + previousSecrets: string[]; secure: boolean; } @@ -80,8 +86,13 @@ export function getSessionConfig(): SessionConfig { "Generate one with: openssl rand -base64 48" ); } + const previousSecrets = (process.env.SESSION_SECRET_PREVIOUS ?? "").split(",").map((value) => value.trim()).filter(Boolean); + if (previousSecrets.some((value) => value.length < 32)) { + throw new Error("Every SESSION_SECRET_PREVIOUS value must be at least 32 characters."); + } return { secret, + previousSecrets, cookieName: optional("SESSION_COOKIE_NAME", "mb_session"), maxAge: Number(optional("SESSION_MAX_AGE", String(60 * 60 * 24 * 7))), secure: process.env.NODE_ENV === "production", diff --git a/packages/auth/src/session.ts b/packages/auth/src/session.ts index 34bb64b..7284099 100644 --- a/packages/auth/src/session.ts +++ b/packages/auth/src/session.ts @@ -34,6 +34,20 @@ function deriveKey(secret: string): Uint8Array { return new Uint8Array(createHash("sha256").update(secret).digest()); } +/** Decrypts with the current secret, then with each previous one (rotation). */ +async function decrypt(value: string, options: { issuer: string; audience: string }) { + const config = getSessionConfig(); + let failure: unknown; + for (const secret of [config.secret, ...config.previousSecrets]) { + try { + return await jwtDecrypt(value, deriveKey(secret), options); + } catch (error) { + failure = error; + } + } + throw failure; +} + /** Encrypt a session into a compact JWE suitable for a cookie value. */ export async function sealSession( session: Omit @@ -52,8 +66,7 @@ export async function sealSession( export async function unsealSession(value: string): Promise { if (!value) return null; try { - const config = getSessionConfig(); - const { payload } = await jwtDecrypt(value, deriveKey(config.secret), { + const { payload } = await decrypt(value, { issuer: ISSUER, audience: AUDIENCE, }); @@ -125,8 +138,7 @@ export async function unsealOAuthState( ): Promise { if (!value) return null; try { - const config = getSessionConfig(); - const { payload } = await jwtDecrypt(value, deriveKey(config.secret), { + const { payload } = await decrypt(value, { issuer: ISSUER, audience: "match-maker-oauth", }); diff --git a/tests/unit/session.test.ts b/tests/unit/session.test.ts index d3c06df..11d1c0a 100644 --- a/tests/unit/session.test.ts +++ b/tests/unit/session.test.ts @@ -39,6 +39,33 @@ describe("session cookie", () => { process.env.SESSION_SECRET = SECRET; }); + it("keeps sessions sealed before a rotation, while new ones use the new secret", async () => { + const before = await sealSession(session); + const rotated = "the-new-session-secret-after-rotation-123456"; + process.env.SESSION_SECRET = rotated; + process.env.SESSION_SECRET_PREVIOUS = SECRET; + try { + expect(await unsealSession(before)).toMatchObject(session); + const after = await sealSession(session); + // Once the old secret is dropped, only cookies sealed after the rotation still work. + delete process.env.SESSION_SECRET_PREVIOUS; + expect(await unsealSession(after)).toMatchObject(session); + expect(await unsealSession(before)).toBeNull(); + } finally { + process.env.SESSION_SECRET = SECRET; + delete process.env.SESSION_SECRET_PREVIOUS; + } + }); + + it("refuses a short previous secret", async () => { + process.env.SESSION_SECRET_PREVIOUS = "too-short"; + try { + await expect(sealSession(session)).rejects.toThrow(/SESSION_SECRET_PREVIOUS/); + } finally { + delete process.env.SESSION_SECRET_PREVIOUS; + } + }); + it("rejects a tampered token", async () => { const sealed = await sealSession(session); const parts = sealed.split("."); From 6385efa1dcffe1532f80427d2beafcc9225a5509 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 10:25:16 +0530 Subject: [PATCH 03/11] Alarm on the matching and notification pipeline (R25 part 2) The worker reports queue backlog, waiting time and permanent failures once a minute with a heartbeat; every search reports its latency and failures; every model call reports its outcome. They are written as CloudWatch embedded metrics (no agent or new service) when METRICS_NAMESPACE is set. Terraform adds alarms for a stopped worker, matching and notification backlogs, failed jobs and notifications, slow or failing search and a sustained model failure rate, plus a dashboard. Each alarm names its runbook section. --- apps/web/src/application/ports/metrics.ts | 6 + .../use-cases/search-participants.ts | 17 +- apps/web/src/composition/container.ts | 5 +- .../observability/emf-metrics.ts | 48 +++++ infra/terraform/ecs.tf | 2 + infra/terraform/monitoring.tf | 178 ++++++++++++++++++ packages/db/src/index.ts | 1 + .../db/src/repositories/pipeline-health.ts | 30 +++ packages/db/workers/metrics.worker.ts | 31 +++ packages/db/workers/worker.ts | 3 + tests/integration/pipeline-health.test.ts | 60 ++++++ tests/unit/metrics.test.ts | 61 ++++++ 12 files changed, 439 insertions(+), 3 deletions(-) create mode 100644 apps/web/src/application/ports/metrics.ts create mode 100644 apps/web/src/infrastructure/observability/emf-metrics.ts create mode 100644 packages/db/src/repositories/pipeline-health.ts create mode 100644 packages/db/workers/metrics.worker.ts create mode 100644 tests/integration/pipeline-health.test.ts create mode 100644 tests/unit/metrics.test.ts diff --git a/apps/web/src/application/ports/metrics.ts b/apps/web/src/application/ports/metrics.ts new file mode 100644 index 0000000..34f8f9d --- /dev/null +++ b/apps/web/src/application/ports/metrics.ts @@ -0,0 +1,6 @@ +export type MetricUnit = "Count" | "Milliseconds" | "Seconds"; + +/** Operational measurements for dashboards and alarms. Never blocks or fails the caller. */ +export interface Metrics { + record(values: Array<{ name: string; value: number; unit: MetricUnit }>, dimensions?: Record): void; +} diff --git a/apps/web/src/application/use-cases/search-participants.ts b/apps/web/src/application/use-cases/search-participants.ts index 3e66fde..504fdc0 100644 --- a/apps/web/src/application/use-cases/search-participants.ts +++ b/apps/web/src/application/use-cases/search-participants.ts @@ -2,11 +2,12 @@ import { MatchingPipeline } from "../../domain/matching/pipeline"; import { normalizeFilters } from "../../domain/matching/search-query"; import type { ParticipantRepository } from "../ports/participant-repository"; import type { RateLimiter } from "../ports/rate-limiter"; +import type { Metrics } from "../ports/metrics"; import { boundedText, LIMITS } from "../../domain/shared/validation"; export async function searchParticipants( input: { eventId: string; participantId: string; query: string; eventIntents: string[]; filters?: { intent?: string | null; domain?: string | null }; limit?: number }, - dependencies: { pipeline: MatchingPipeline; participants: ParticipantRepository; limiter?: RateLimiter } + dependencies: { pipeline: MatchingPipeline; participants: ParticipantRepository; limiter?: RateLimiter; metrics?: Metrics } ) { if (dependencies.limiter && !(await dependencies.limiter.consume(`search:${input.participantId}`, 30, 60))) { throw new Error("Too many searches. Wait a moment and try again."); @@ -16,5 +17,17 @@ export async function searchParticipants( intents: input.eventIntents, domains: input.filters?.domain ? await dependencies.participants.listDomains(input.eventId) : [], }); - return dependencies.pipeline.run({ eventId: input.eventId, participantId: input.participantId, query, filters, limit: input.limit, mode: "search" }); + const started = Date.now(); + let failed = 0; + try { + return await dependencies.pipeline.run({ eventId: input.eventId, participantId: input.participantId, query, filters, limit: input.limit, mode: "search" }); + } catch (error) { + failed = 1; + throw error; + } finally { + dependencies.metrics?.record([ + { name: "SearchLatencyMs", value: Date.now() - started, unit: "Milliseconds" }, + { name: "SearchFailures", value: failed, unit: "Count" }, + ]); + } } diff --git a/apps/web/src/composition/container.ts b/apps/web/src/composition/container.ts index ea938a1..5319bc2 100644 --- a/apps/web/src/composition/container.ts +++ b/apps/web/src/composition/container.ts @@ -14,6 +14,7 @@ import { RealtimeNotifier } from "../infrastructure/notifications/realtime.notif import { ProviderRegistry } from "../infrastructure/providers/provider-registry"; import { GatewayEmbeddingProvider, GatewayLlmProvider } from "../infrastructure/providers/model-gateway"; import { PostgresModelUsageLog } from "../infrastructure/postgres/model-usage.repo"; +import { EmfMetrics, MeteredModelUsageLog } from "../infrastructure/observability/emf-metrics"; import { MatchingPipeline } from "../domain/matching/pipeline"; const globalContainer = globalThis as typeof globalThis & { __matchMakerContainer?: ReturnType }; @@ -23,7 +24,8 @@ function buildContainer() { const matches = new PostgresMatchRepository(); const intros = new PostgresIntroRepository(); const events = new PostgresEventRepository(); - const usage = new PostgresModelUsageLog(); + const metrics = new EmfMetrics(); + const usage = new MeteredModelUsageLog(new PostgresModelUsageLog(), metrics); const embeddings = new GatewayEmbeddingProvider(ProviderRegistry.embedding(), usage); const rawLlm = ProviderRegistry.llm(); const llm = rawLlm ? new GatewayLlmProvider(rawLlm, usage) : null; @@ -41,6 +43,7 @@ function buildContainer() { locations: new PostgresLocationRepository(), notifications: new RealtimeNotifier(), limiter: new PostgresRateLimiter(), + metrics, pipeline: new MatchingPipeline(participants, matches, embeddings, llm, candidateCache, new PostgresExplanationCache()), enqueueMatching: matchingJobs.enqueue, enqueueEmbedding: (eventId: string, participantId: string) => matchingJobs.enqueue(eventId, participantId, "embed"), diff --git a/apps/web/src/infrastructure/observability/emf-metrics.ts b/apps/web/src/infrastructure/observability/emf-metrics.ts new file mode 100644 index 0000000..7f5c552 --- /dev/null +++ b/apps/web/src/infrastructure/observability/emf-metrics.ts @@ -0,0 +1,48 @@ +import type { Metrics } from "../../application/ports/metrics"; +import type { ModelUsageLog } from "../../application/ports/model-usage"; + +/** + * Writes metrics as CloudWatch Embedded Metric Format: one JSON log line that + * CloudWatch Logs turns into metrics, so no agent or extra service is needed. + * Without METRICS_NAMESPACE (local runs) nothing is written. + */ +export class EmfMetrics implements Metrics { + constructor( + private readonly namespace: string | null = process.env.METRICS_NAMESPACE || null, + private readonly write: (line: string) => void = (line) => process.stdout.write(`${line}\n`) + ) {} + + record(values: Parameters[0], dimensions: Record = {}): void { + if (!this.namespace || !values.length) return; + try { + this.write(JSON.stringify({ + _aws: { + Timestamp: Date.now(), + CloudWatchMetrics: [{ + Namespace: this.namespace, + Dimensions: [Object.keys(dimensions)], + Metrics: values.map((metric) => ({ Name: metric.name, Unit: metric.unit })), + }], + }, + ...dimensions, + ...Object.fromEntries(values.map((metric) => [metric.name, metric.value])), + })); + } catch { + // Metrics are best effort. + } + } +} + +/** Adds call counts, failures and latency per operation to an existing usage log. */ +export class MeteredModelUsageLog implements ModelUsageLog { + constructor(private readonly inner: ModelUsageLog, private readonly metrics: Metrics) {} + + record(entry: Parameters[0]): void { + this.inner.record(entry); + this.metrics.record([ + { name: "ModelCalls", value: 1, unit: "Count" }, + { name: "ModelCallFailures", value: entry.outcome === "ok" ? 0 : 1, unit: "Count" }, + { name: "ModelCallLatencyMs", value: entry.latencyMs, unit: "Milliseconds" }, + ], { Operation: entry.operation }); + } +} diff --git a/infra/terraform/ecs.tf b/infra/terraform/ecs.tf index bb30e30..70dbd22 100644 --- a/infra/terraform/ecs.tf +++ b/infra/terraform/ecs.tf @@ -211,6 +211,7 @@ resource "aws_ecs_task_definition" "web" { environment = [ { name = "NODE_ENV", value = "production" }, { name = "PORT", value = "3000" }, + { name = "METRICS_NAMESPACE", value = local.metrics_namespace }, { name = "APP_BASE_URL", value = local.app_base_url }, { name = "DEFAULT_EVENT_SLUG", value = var.default_event_slug }, # One ALB appends the client address to X-Forwarded-For. @@ -479,6 +480,7 @@ resource "aws_ecs_task_definition" "worker" { essential = true environment = [ { name = "NODE_ENV", value = "production" }, + { name = "METRICS_NAMESPACE", value = local.metrics_namespace }, { name = "DATABASE_SSL", value = "verify" }, { name = "DATABASE_CA_CERT_FILE", value = local.database_ca_file }, { name = "GEMINI_LLM_MODEL", value = "gemini-3.8-flash" }, diff --git a/infra/terraform/monitoring.tf b/infra/terraform/monitoring.tf index 2e7c809..3720bab 100644 --- a/infra/terraform/monitoring.tf +++ b/infra/terraform/monitoring.tf @@ -17,6 +17,8 @@ resource "aws_sns_topic_subscription" "alarm_email" { locals { alarm_actions = [aws_sns_topic.alarms.arn] + # The web and worker write metrics here (CloudWatch embedded metric format). + metrics_namespace = "MatchMaker/${var.environment}" } # ── Availability ────────────────────────────────────────────── @@ -214,3 +216,179 @@ resource "aws_cloudwatch_metric_alarm" "login_email_failures" { alarm_actions = local.alarm_actions } + +# ── Pipeline ────────────────────────────────────────────────── +# Written by the worker once a minute and by every search. Each alarm names +# the runbook section in docs/OPERATIONS.md that says what to do. + +locals { + pipeline_alarms = { + worker-down = { + description = "No worker heartbeat for 5 minutes: matching and notifications have stopped. Runbook: Worker down." + metric = "WorkerHeartbeat" + statistic = "SampleCount" + period = 300 + periods = 1 + threshold = 1 + comparison = "LessThanThreshold" + missing = "breaching" + } + matching-backlog = { + description = "A due matching job has waited over 10 minutes. Runbook: Matching backlog." + metric = "OldestDueJobSeconds" + statistic = "Maximum" + period = 300 + periods = 2 + threshold = 600 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + jobs-failed = { + description = "Matching jobs failed after every retry. Runbook: Failed jobs (npm run jobs:failed)." + metric = "JobsFailed" + statistic = "Maximum" + period = 300 + periods = 1 + threshold = 0 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + notification-backlog = { + description = "A due notification has waited over 10 minutes. Runbook: Notifications stuck." + metric = "OldestDueNotificationSeconds" + statistic = "Maximum" + period = 300 + periods = 2 + threshold = 600 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + notifications-failed = { + description = "Notifications failed after every retry. Runbook: Notifications stuck." + metric = "NotificationsFailed" + statistic = "Maximum" + period = 300 + periods = 1 + threshold = 0 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + search-slow = { + description = "Search p95 above its 6 s budget for 15 minutes. Runbook: Slow search." + metric = "SearchLatencyMs" + statistic = "p95" + period = 300 + periods = 3 + threshold = 6000 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + search-failing = { + description = "More than 5 searches failed in 5 minutes. Runbook: Slow search." + metric = "SearchFailures" + statistic = "Sum" + period = 300 + periods = 1 + threshold = 5 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + } +} + +resource "aws_cloudwatch_metric_alarm" "pipeline" { + for_each = local.pipeline_alarms + + alarm_name = "${local.name}-${each.key}" + alarm_description = each.value.description + namespace = local.metrics_namespace + metric_name = each.value.metric + statistic = each.value.statistic == "p95" ? null : each.value.statistic + extended_statistic = each.value.statistic == "p95" ? "p95" : null + period = each.value.period + evaluation_periods = each.value.periods + threshold = each.value.threshold + comparison_operator = each.value.comparison + treat_missing_data = each.value.missing + + alarm_actions = local.alarm_actions + ok_actions = local.alarm_actions +} + +# Model failures are expected occasionally (the gateway falls back); only a +# sustained rate needs a person, e.g. an expired key or an exhausted quota. +resource "aws_cloudwatch_metric_alarm" "model_failures" { + alarm_name = "${local.name}-model-failures" + alarm_description = "Over 20% of model calls failing for 15 minutes; matching runs on fallbacks. Runbook: Model provider failing." + comparison_operator = "GreaterThanThreshold" + evaluation_periods = 3 + threshold = 20 + treat_missing_data = "notBreaching" + + metric_query { + id = "rate" + expression = "IF(calls > 0, 100 * failures / calls, 0)" + label = "Model call failure %" + return_data = true + } + + metric_query { + id = "failures" + expression = join(" + ", [for op in local.model_operations : "f_${replace(op, ":", "_")}"]) + } + + metric_query { + id = "calls" + expression = join(" + ", [for op in local.model_operations : "c_${replace(op, ":", "_")}"]) + } + + dynamic "metric_query" { + for_each = { for pair in setproduct(local.model_operations, ["f", "c"]) : "${pair[1]}_${replace(pair[0], ":", "_")}" => pair } + content { + id = metric_query.key + metric { + namespace = local.metrics_namespace + metric_name = metric_query.value[1] == "f" ? "ModelCallFailures" : "ModelCalls" + dimensions = { Operation = metric_query.value[0] } + period = 300 + stat = "Sum" + } + } + } + + alarm_actions = local.alarm_actions +} + +locals { + model_operations = ["embed:profile", "embed:query", "rerank"] +} + +resource "aws_cloudwatch_dashboard" "main" { + dashboard_name = local.name + dashboard_body = jsonencode({ + widgets = [ + for index, widget in [ + { title = "Search latency (ms)", metrics = [[local.metrics_namespace, "SearchLatencyMs", { stat = "p50" }], ["...", { stat = "p95" }]] }, + { title = "Queue wait (s)", metrics = [[local.metrics_namespace, "OldestDueJobSeconds"], [local.metrics_namespace, "OldestDueNotificationSeconds"]] }, + { title = "Queue size", metrics = [[local.metrics_namespace, "JobsPending"], [local.metrics_namespace, "NotificationsPending"]] }, + { title = "Failed for good", metrics = [[local.metrics_namespace, "JobsFailed"], [local.metrics_namespace, "NotificationsFailed"]] }, + { title = "Model call failures by operation", metrics = [for op in local.model_operations : [local.metrics_namespace, "ModelCallFailures", "Operation", op, { stat = "Sum" }]] }, + { title = "Web response time (s)", metrics = [["AWS/ApplicationELB", "TargetResponseTime", "LoadBalancer", aws_lb.main.arn_suffix, { stat = "p95" }]] }, + ] : { + type = "metric" + x = (index % 2) * 12 + y = floor(index / 2) * 6 + width = 12 + height = 6 + properties = { + title = widget.title + metrics = widget.metrics + region = var.aws_region + stat = "Maximum" + period = 300 + view = "timeSeries" + } + } + ] + }) +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index a13636c..ba2aebe 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -15,3 +15,4 @@ export * as housekeeping from "./repositories/housekeeping"; export * as audit from "./repositories/audit"; export * as modelUsage from "./repositories/model-usage"; export * as outbox from "./repositories/outbox"; +export * as pipelineHealth from "./repositories/pipeline-health"; diff --git a/packages/db/src/repositories/pipeline-health.ts b/packages/db/src/repositories/pipeline-health.ts new file mode 100644 index 0000000..d5a86d2 --- /dev/null +++ b/packages/db/src/repositories/pipeline-health.ts @@ -0,0 +1,30 @@ +import { query } from "../client"; + +/** Backlog and failure counts of the background queues, for metrics and alarms. */ +export interface PipelineHealth { + jobsPending: number; + /** Age of the oldest job that is due but not picked up; 0 when none. */ + oldestDueJobSeconds: number; + /** Jobs that failed after every retry, in the last day. */ + jobsFailed: number; + notificationsPending: number; + oldestDueNotificationSeconds: number; + notificationsFailed: number; +} + +export async function snapshot(): Promise { + const { rows: [row] } = await query>( + `select + (select count(*) from matching_jobs where status = 'pending') as "jobsPending", + (select coalesce(extract(epoch from now() - min(available_at)), 0) from matching_jobs + where status = 'pending' and available_at <= now()) as "oldestDueJobSeconds", + (select count(*) from matching_jobs where status = 'failed' + and coalesce(started_at, created_at) > now() - interval '1 day') as "jobsFailed", + (select count(*) from notification_outbox where status = 'pending') as "notificationsPending", + (select coalesce(extract(epoch from now() - min(available_at)), 0) from notification_outbox + where status = 'pending' and available_at <= now()) as "oldestDueNotificationSeconds", + (select count(*) from notification_outbox where status = 'failed' + and created_at > now() - interval '1 day') as "notificationsFailed"` + ); + return Object.fromEntries(Object.entries(row).map(([key, value]) => [key, Math.round(Number(value))])) as unknown as PipelineHealth; +} diff --git a/packages/db/workers/metrics.worker.ts b/packages/db/workers/metrics.worker.ts new file mode 100644 index 0000000..b015713 --- /dev/null +++ b/packages/db/workers/metrics.worker.ts @@ -0,0 +1,31 @@ +import { pipelineHealth } from "../src/index"; +import type { Metrics } from "../../../apps/web/src/application/ports/metrics"; + +export const REPORT_EVERY_MS = 60 * 1000; + +let lastReport = 0; + +/** + * Reports queue backlog and failures once a minute. WorkerHeartbeat doubles + * as liveness: when it stops arriving, the worker is down. + */ +export async function reportHealthIfDue(metrics: Metrics, now = Date.now()): Promise { + if (now - lastReport < REPORT_EVERY_MS) return false; + lastReport = now; + try { + const health = await pipelineHealth.snapshot(); + metrics.record([ + { name: "WorkerHeartbeat", value: 1, unit: "Count" }, + { name: "JobsPending", value: health.jobsPending, unit: "Count" }, + { name: "OldestDueJobSeconds", value: health.oldestDueJobSeconds, unit: "Seconds" }, + { name: "JobsFailed", value: health.jobsFailed, unit: "Count" }, + { name: "NotificationsPending", value: health.notificationsPending, unit: "Count" }, + { name: "OldestDueNotificationSeconds", value: health.oldestDueNotificationSeconds, unit: "Seconds" }, + { name: "NotificationsFailed", value: health.notificationsFailed, unit: "Count" }, + ]); + } catch (error) { + // A missing heartbeat raises the alarm; the queues keep running. + console.error("[worker:metrics] could not read queue health", error); + } + return true; +} diff --git a/packages/db/workers/worker.ts b/packages/db/workers/worker.ts index 80af71c..28dacc4 100644 --- a/packages/db/workers/worker.ts +++ b/packages/db/workers/worker.ts @@ -5,6 +5,8 @@ import { processEmbedJob } from "./embed.worker"; import { processMatchJob } from "./match.worker"; import { sweepIfDue } from "./housekeeping.worker"; import { processNotification } from "./notification.worker"; +import { reportHealthIfDue } from "./metrics.worker"; +import { getContainer } from "../../../apps/web/src/composition/container"; let stopping = false; process.on("SIGINT", () => { stopping = true; }); @@ -15,6 +17,7 @@ async function main() { console.info(`[worker] embedding and matching queue started (version ${process.env.APP_VERSION || "dev"})`); while (!stopping) { await sweepIfDue(); + await reportHealthIfDue(getContainer().metrics); const embedded = await processEmbedJob(); const matched = await processMatchJob(); const notified = await processNotification(); diff --git a/tests/integration/pipeline-health.test.ts b/tests/integration/pipeline-health.test.ts new file mode 100644 index 0000000..ba3912b --- /dev/null +++ b/tests/integration/pipeline-health.test.ts @@ -0,0 +1,60 @@ +/** + * The queue health snapshot behind the backlog and failure alarms, against a + * real Postgres. Other local rows may exist, so it checks lower bounds from + * rows it creates in its own event. Skipped without DATABASE_URL. + */ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { attendees, closePool, events, pipelineHealth, query } from "@match-maker/db"; +import { migrate } from "../../packages/db/src/migrate"; +import { reportHealthIfDue } from "../../packages/db/workers/metrics.worker"; + +const describeDb = process.env.DATABASE_URL ? describe : describe.skip; +const RUN = Date.now().toString(36); +const ORG = `health-${RUN}`; + +describeDb("pipeline health", () => { + let eventId: string; + let attendeeId: string; + + beforeAll(async () => { + await migrate(); + await attendees.upsertMany([{ name: "Health check", email: `health-${RUN}@example.com` }]); + attendeeId = (await attendees.findByEmail(`health-${RUN}@example.com`))!.id; + const { rows: [org] } = await query<{ id: string }>(`insert into organizations (name, slug) values ($1, $1) returning id`, [ORG]); + eventId = (await query<{ id: string }>(`insert into events (organization_id, name, slug, status) values ($1, $2, $2, 'closed') returning id`, [org.id, ORG])).rows[0].id; + await events.attachRoster(eventId, [attendeeId]); + }); + + afterAll(async () => { + await query(`delete from organizations where slug = $1`, [ORG]); + await query(`delete from registered_attendees where email = $1`, [`health-${RUN}@example.com`]); + await closePool(); + }); + + it("reports how long due work has waited and what failed for good", async () => { + await query(`insert into matching_jobs (event_id, participant_id, job_type, status, available_at, created_at) + values ($1, $2, 'embed', 'pending', now() - interval '20 minutes', now() - interval '20 minutes'), + ($1, $2, 'match', 'failed', now(), now())`, [eventId, attendeeId]); + await query(`insert into notification_outbox (event_id, recipient_id, kind, subject_id, dedupe_key, status, available_at) + values ($1, $2, 'intro.requested', $2, $3, 'pending', now() - interval '15 minutes'), + ($1, $2, 'intro.accepted', $2, $4, 'failed', now())`, + [eventId, attendeeId, `health-a-${RUN}`, `health-b-${RUN}`]); + + const health = await pipelineHealth.snapshot(); + expect(health.jobsPending).toBeGreaterThanOrEqual(1); + expect(health.oldestDueJobSeconds).toBeGreaterThanOrEqual(20 * 60 - 5); + expect(health.jobsFailed).toBeGreaterThanOrEqual(1); + expect(health.oldestDueNotificationSeconds).toBeGreaterThanOrEqual(15 * 60 - 5); + expect(health.notificationsFailed).toBeGreaterThanOrEqual(1); + }); + + it("reports at most once a minute, with a heartbeat", async () => { + const metrics = { record: (values: Array<{ name: string }>) => recorded.push(values.map((value) => value.name)) }; + const recorded: string[][] = []; + const now = Date.now() + 10 * 60_000; + expect(await reportHealthIfDue(metrics, now)).toBe(true); + expect(await reportHealthIfDue(metrics, now + 1000)).toBe(false); + expect(recorded).toHaveLength(1); + expect(recorded[0]).toEqual(expect.arrayContaining(["WorkerHeartbeat", "OldestDueJobSeconds", "NotificationsFailed"])); + }); +}); diff --git a/tests/unit/metrics.test.ts b/tests/unit/metrics.test.ts new file mode 100644 index 0000000..9a40955 --- /dev/null +++ b/tests/unit/metrics.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from "vitest"; +import { EmfMetrics, MeteredModelUsageLog } from "@/src/infrastructure/observability/emf-metrics"; +import { searchParticipants } from "@/src/application/use-cases/search-participants"; +import type { Metrics } from "@/src/application/ports/metrics"; + +describe("embedded metric format", () => { + it("writes one line CloudWatch turns into metrics", () => { + const lines: string[] = []; + new EmfMetrics("MatchMaker/test", (line) => lines.push(line)).record( + [{ name: "SearchLatencyMs", value: 120, unit: "Milliseconds" }, { name: "SearchFailures", value: 0, unit: "Count" }], + { Operation: "rerank" }); + const line = JSON.parse(lines[0]); + expect(line._aws.CloudWatchMetrics).toEqual([{ + Namespace: "MatchMaker/test", + Dimensions: [["Operation"]], + Metrics: [{ Name: "SearchLatencyMs", Unit: "Milliseconds" }, { Name: "SearchFailures", Unit: "Count" }], + }]); + expect(line).toMatchObject({ Operation: "rerank", SearchLatencyMs: 120, SearchFailures: 0 }); + }); + + it("writes nothing without a namespace", () => { + const write = vi.fn(); + new EmfMetrics(null, write).record([{ name: "JobsPending", value: 3, unit: "Count" }]); + expect(write).not.toHaveBeenCalled(); + }); + + it("counts model calls and failures per operation, and still records usage", () => { + const metrics = { record: vi.fn() }; + const inner = { record: vi.fn() }; + const entry = { provider: "gemini", operation: "embed", outcome: "timeout" as const, latencyMs: 900, inputChars: 10 }; + new MeteredModelUsageLog(inner, metrics).record(entry); + expect(inner.record).toHaveBeenCalledWith(entry); + expect(metrics.record).toHaveBeenCalledWith(expect.arrayContaining([ + { name: "ModelCallFailures", value: 1, unit: "Count" }, + { name: "ModelCallLatencyMs", value: 900, unit: "Milliseconds" }, + ]), { Operation: "embed" }); + }); +}); + +describe("search metrics", () => { + const deps = (run: () => Promise) => { + const metrics: Metrics & { record: ReturnType } = { record: vi.fn() }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return { metrics, pipeline: { run } as any, participants: { listDomains: vi.fn(async () => []) } as any }; + }; + const search = (d: ReturnType) => searchParticipants({ eventId: "e", participantId: "p", query: "climate", eventIntents: [] }, d); + const failures = (d: ReturnType) => d.metrics.record.mock.calls[0][0].find((metric: { name: string }) => metric.name === "SearchFailures").value; + + it("records how long a search took", async () => { + const d = deps(async () => []); + await search(d); + expect(failures(d)).toBe(0); + expect(d.metrics.record.mock.calls[0][0][0]).toMatchObject({ name: "SearchLatencyMs", unit: "Milliseconds" }); + }); + + it("records a failed search and still fails it", async () => { + const d = deps(async () => { throw new Error("database down"); }); + await expect(search(d)).rejects.toThrow("database down"); + expect(failures(d)).toBe(1); + }); +}); From 5f895291f21363e663adcb44d5894c33c3739233 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 10:28:58 +0530 Subject: [PATCH 04/11] bug fixx-match-save-deadlock Two workers saving match lists that named each other deadlocked: each locked its own profile row FOR UPDATE, then its inserts took key-share locks on the other's row through the foreign keys. Found by the capacity check (300 attendees, 2 workers). The revision check now locks FOR NO KEY UPDATE, which still blocks a concurrent profile save but not those foreign-key locks. --- .../web/src/infrastructure/postgres/match.repo.ts | 5 ++++- tests/integration/concurrency.test.ts | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/apps/web/src/infrastructure/postgres/match.repo.ts b/apps/web/src/infrastructure/postgres/match.repo.ts index e8a8eea..8942530 100644 --- a/apps/web/src/infrastructure/postgres/match.repo.ts +++ b/apps/web/src/infrastructure/postgres/match.repo.ts @@ -30,8 +30,11 @@ export class PostgresMatchRepository implements MatchRepository { return transaction(async (client) => { if (expectedRevision !== undefined) { // Lock the profile row so a save cannot slip in between check and write. + // NO KEY UPDATE still blocks a profile save, but not the key-share locks + // other people's match inserts take on this row through their foreign + // keys; FOR UPDATE made two lists naming each other deadlock. const { rows } = await client.query<{ profile_revision: string }>( - `select profile_revision from event_participants where event_id = $1 and attendee_id = $2 for update`, [eventId, participantId]); + `select profile_revision from event_participants where event_id = $1 and attendee_id = $2 for no key update`, [eventId, participantId]); if (!rows[0] || Number(rows[0].profile_revision) !== expectedRevision) return false; } await client.query(`delete from event_matches where event_id = $1 and participant_id = $2`, [eventId, participantId]); diff --git a/tests/integration/concurrency.test.ts b/tests/integration/concurrency.test.ts index 55faa30..a473720 100644 --- a/tests/integration/concurrency.test.ts +++ b/tests/integration/concurrency.test.ts @@ -10,6 +10,7 @@ import { migrate } from "../../packages/db/src/migrate"; import { PostgresIntroRepository } from "../../apps/web/src/infrastructure/postgres/intro.repo"; import { PostgresLocationRepository } from "../../apps/web/src/infrastructure/postgres/location.repo"; import { PostgresParticipantRepository } from "../../apps/web/src/infrastructure/postgres/participant.repo"; +import { PostgresMatchRepository } from "../../apps/web/src/infrastructure/postgres/match.repo"; import { requestIntro } from "../../apps/web/src/application/use-cases/request-intro"; import { Intent } from "../../apps/web/src/domain/participant/intent"; @@ -23,6 +24,7 @@ describeDb("concurrent actions", () => { const intros = new PostgresIntroRepository(); const locations = new PostgresLocationRepository(); const participants = new PostgresParticipantRepository(); + const matches = new PostgresMatchRepository(); const ids: Record = {}; let eventId: string; @@ -88,6 +90,19 @@ describeDb("concurrent actions", () => { expect(created.filter(Boolean)).toHaveLength(1); }); + it("saves match lists that name each other at the same time without deadlocking", async () => { + const group = ["p0", "p1", "p2", "p3", "p4", "p5"]; + const profiles = await Promise.all(group.map((key) => participants.findProfile(eventId, ids[key]))); + const listFor = (key: string) => profiles.filter((profile) => profile!.attendeeId !== ids[key]) + .map((profile) => ({ participant: profile!, score: 50, scoreBreakdown: {}, mutualInterest: false, reason: null })); + const revisionOf = (key: string) => profiles.find((profile) => profile!.attendeeId === ids[key])!.revision!; + for (let round = 0; round < 5; round++) { + const results = await settle(group.map((key) => matches.save(eventId, ids[key], listFor(key), "race", revisionOf(key)))); + expect(results.filter((result) => result.status === "rejected")).toEqual([]); + expect(results.map((result) => (result as PromiseFulfilledResult).value)).toEqual(group.map(() => true)); + } + }); + it("imports overlapping roster sheets at once without duplicating anyone", async () => { const sheet = (keys: string[]) => keys.map((key) => ({ name: `Imported ${key}`, email: email(`imp-${key}`), phone: null, about_me: null, designation: null, domain: null, From fed37723b57f6d9f722b21ae69ccb005529f760b Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 10:31:16 +0530 Subject: [PATCH 05/11] bug fixx-retrieval-intent-starvation Retrieval took the 50 nearest people and only then dropped intents the viewer does not welcome, so a mentor whose nearest neighbours were all mentors got no matches even with a mentee in the event. Found by the capacity check (1 of 300 attendees). Retrieval now applies the viewer's welcomed intents in the query, before the limit, for profile matches and search (part of the shared search cache key; retrieval version r3). --- .../ports/participant-repository.ts | 2 + apps/web/src/domain/matching/pipeline.ts | 3 + .../domain/matching/stages/retrieve.stage.ts | 12 +++- apps/web/src/domain/participant/intent.ts | 9 +++ .../postgres/participant.repo.ts | 4 +- tests/integration/retrieval-recall.test.ts | 65 +++++++++++++++++++ 6 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 tests/integration/retrieval-recall.test.ts diff --git a/apps/web/src/application/ports/participant-repository.ts b/apps/web/src/application/ports/participant-repository.ts index c665385..87b5dcd 100644 --- a/apps/web/src/application/ports/participant-repository.ts +++ b/apps/web/src/application/ports/participant-repository.ts @@ -12,6 +12,8 @@ export interface ParticipantRepository { /** Without an embedding, only candidates matching one of these words, best first. */ terms?: string[]; filters?: SearchFilters; + /** Only people with one of these intents (the viewer's welcomed intents); null for all. */ + welcomedIntents?: string[] | null; /** The model behind `embedding`; vectors from other models are not ranked against it. */ embeddingModel?: string | null; limit?: number; diff --git a/apps/web/src/domain/matching/pipeline.ts b/apps/web/src/domain/matching/pipeline.ts index fec3485..8a2c764 100644 --- a/apps/web/src/domain/matching/pipeline.ts +++ b/apps/web/src/domain/matching/pipeline.ts @@ -1,4 +1,5 @@ import type { EmbeddingProvider } from "../../application/ports/embedding-provider"; +import { welcomedIntents } from "../participant/intent"; import type { CandidateCache } from "../../application/ports/candidate-cache"; import type { ExplanationCache } from "../../application/ports/explanation-cache"; import { queryContext } from "./explanation-key"; @@ -107,6 +108,7 @@ export class MatchingPipeline { viewerId: input.participantId, embedding, embeddingSpace: this.embeddings.id, + welcomedIntents: welcomedIntents(viewer.intent), limit: 50, }); const compatible = this.filter.execute(viewer, retrieved); @@ -151,6 +153,7 @@ export class MatchingPipeline { terms, filters: input.filters ?? NO_FILTERS, embeddingSpace: this.embeddings.id, + welcomedIntents: welcomedIntents(viewer.intent), limit: 50, }); const compatible = this.filter.execute(viewer, retrieved); diff --git a/apps/web/src/domain/matching/stages/retrieve.stage.ts b/apps/web/src/domain/matching/stages/retrieve.stage.ts index e7c6598..72d913a 100644 --- a/apps/web/src/domain/matching/stages/retrieve.stage.ts +++ b/apps/web/src/domain/matching/stages/retrieve.stage.ts @@ -7,7 +7,7 @@ import { matchesFilters, NO_FILTERS, type SearchFilters } from "../search-query" import { computeOnce } from "../../../application/shared-computation"; /** Bump when retrieval semantics change, so cached candidate lists are not reused. */ -const RETRIEVAL_VERSION = "r2"; +const RETRIEVAL_VERSION = "r3"; export class RetrieveStage { constructor( @@ -24,6 +24,12 @@ export class RetrieveStage { filters?: SearchFilters; /** Embedding provider/model; vectors from different spaces must not share entries. */ embeddingSpace?: string; + /** + * The viewer's welcomed intents, applied before the limit: filtering + * afterwards left people with no matches when their nearest neighbours + * all had intents they do not welcome. + */ + welcomedIntents?: string[] | null; limit?: number; }): Promise { const limit = input.limit ?? 50; @@ -34,7 +40,7 @@ export class RetrieveStage { // retrieval kind; the ranking that follows is still personal. const corpus = await this.participants.corpusRevision(input.eventId); const key = createHash("sha256").update(JSON.stringify([ - RETRIEVAL_VERSION, input.eventId, corpus, normalizedQuery, filters.intent, filters.domain?.toLowerCase() ?? null, + RETRIEVAL_VERSION, input.eventId, corpus, normalizedQuery, filters.intent, filters.domain?.toLowerCase() ?? null, input.welcomedIntents ?? null, input.embedding ? input.embeddingSpace ?? "semantic" : "lexical", limit, ])).digest("hex"); const ids = await computeOnce({ @@ -49,6 +55,7 @@ export class RetrieveStage { embeddingModel: input.embeddingSpace, terms: input.terms, filters, + welcomedIntents: input.welcomedIntents, limit, })).map((candidate) => candidate.attendeeId), }); @@ -64,6 +71,7 @@ export class RetrieveStage { excludeAttendeeId: input.viewerId, embedding: input.embedding, embeddingModel: input.embeddingSpace, + welcomedIntents: input.welcomedIntents, limit, }); } diff --git a/apps/web/src/domain/participant/intent.ts b/apps/web/src/domain/participant/intent.ts index ec07b33..a64f82e 100644 --- a/apps/web/src/domain/participant/intent.ts +++ b/apps/web/src/domain/participant/intent.ts @@ -23,6 +23,15 @@ export const INTENT_COMPATIBILITY: Record = { [Intent.INVESTING]: [Intent.RAISING_FUNDING, Intent.NETWORKING], }; +/** + * The intents a person welcomes, for narrowing retrieval before any top-K + * cut; null when they welcome everyone. + */ +export function welcomedIntents(intent: Intent): Intent[] | null { + const welcomed = INTENT_COMPATIBILITY[intent] ?? []; + return welcomed.length === Object.values(Intent).length ? null : [...welcomed].sort(); +} + export function isIntentCompatible(from: Intent, to: Intent): boolean { return INTENT_COMPATIBILITY[from]?.includes(to) ?? false; } diff --git a/apps/web/src/infrastructure/postgres/participant.repo.ts b/apps/web/src/infrastructure/postgres/participant.repo.ts index 301c4b2..913e440 100644 --- a/apps/web/src/infrastructure/postgres/participant.repo.ts +++ b/apps/web/src/infrastructure/postgres/participant.repo.ts @@ -97,6 +97,7 @@ export class PostgresParticipantRepository implements ParticipantRepository { embedding?: number[] | null; terms?: string[]; filters?: SearchFilters; + welcomedIntents?: string[] | null; /** The model that produced `embedding`; only candidates embedded by it are ranked by vector. */ embeddingModel?: string | null; limit?: number; @@ -110,13 +111,14 @@ export class PostgresParticipantRepository implements ParticipantRepository { and ($5::text is null or p.intent = $5) and ($6::text is null or lower(p.domain) = lower($6)) and ($7::text is null or ${SEARCH_DOCUMENT} @@ to_tsquery('simple', $7)) + and ($9::text[] is null or p.intent = any($9::text[])) order by case when $3::vector is not null and p.embedding is not null and p.embedding_model is not distinct from $8 then 0 else 1 end, case when p.embedding_model is not distinct from $8 then p.embedding <=> $3::vector end nulls last, case when $7::text is null then 0 else ts_rank(${SEARCH_DOCUMENT}, to_tsquery('simple', $7)) end desc, a.name asc limit $4`, [input.eventId, input.excludeAttendeeId, vector, Math.max(1, Math.min(100, input.limit ?? 50)), - input.filters?.intent ?? null, input.filters?.domain ?? null, tsQuery, input.embeddingModel ?? null] + input.filters?.intent ?? null, input.filters?.domain ?? null, tsQuery, input.embeddingModel ?? null, input.welcomedIntents ?? null] ); return rows.map(mapProfile); } diff --git a/tests/integration/retrieval-recall.test.ts b/tests/integration/retrieval-recall.test.ts new file mode 100644 index 0000000..e7db105 --- /dev/null +++ b/tests/integration/retrieval-recall.test.ts @@ -0,0 +1,65 @@ +/** + * Retrieval must not starve someone whose nearest neighbours all have + * intents they do not welcome: a mentor among many mentors still finds the + * one person seeking a mentor. Real Postgres, fake embeddings, own event. + * Skipped without DATABASE_URL. + */ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { attendees, closePool, events, query } from "@match-maker/db"; +import { migrate } from "../../packages/db/src/migrate"; +import { PostgresParticipantRepository } from "../../apps/web/src/infrastructure/postgres/participant.repo"; +import { PostgresMatchRepository } from "../../apps/web/src/infrastructure/postgres/match.repo"; +import { PostgresCandidateCache } from "../../apps/web/src/infrastructure/cache/postgres-candidate-cache"; +import { MatchingPipeline } from "../../apps/web/src/domain/matching/pipeline"; +import { Intent } from "../../apps/web/src/domain/participant/intent"; + +const describeDb = process.env.DATABASE_URL ? describe : describe.skip; +const RUN = Date.now().toString(36); +const ORG = `recall-${RUN}`; +const MENTORS = 55; +const email = (key: string) => `recall-${key}-${RUN}@example.com`; +const VECTOR = Array.from({ length: 768 }, (_, i) => (i === 0 ? 1 : 0)); +const embeddings = { id: "test:recall", embed: async () => VECTOR }; + +describeDb("retrieval recall", () => { + const participants = new PostgresParticipantRepository(); + const pipeline = new MatchingPipeline(participants, new PostgresMatchRepository(), embeddings, null, new PostgresCandidateCache(), null); + const ids: Record = {}; + let eventId: string; + + beforeAll(async () => { + await migrate(); + const keys = ["viewer", "seeker", ...Array.from({ length: MENTORS }, (_, i) => `mentor${i}`)]; + await attendees.upsertMany(keys.map((key) => ({ name: `Recall ${key}`, email: email(key) }))); + const { rows } = await query<{ id: string; email: string }>(`select id, email from registered_attendees where email like $1`, [`recall-%-${RUN}@example.com`]); + for (const row of rows) ids[row.email.split("-")[1]] = row.id; + const { rows: [org] } = await query<{ id: string }>(`insert into organizations (name, slug) values ($1, $1) returning id`, [ORG]); + eventId = (await query<{ id: string }>(`insert into events (organization_id, name, slug, status) values ($1, $2, $2, 'open') returning id`, [org.id, ORG])).rows[0].id; + await events.attachRoster(eventId, Object.values(ids)); + for (const key of keys) { + await participants.saveProfile({ + eventId, attendeeId: ids[key], designation: "Engineer", domain: "AI", aboutMe: key === "seeker" ? "New to the field" : "Mentors engineers", + selfDescription: null, yearsOfExperience: null, mostImpressiveBuild: null, + intent: key === "seeker" ? Intent.SEEKING_MENTOR : Intent.MENTORING, interests: [], venueZone: null, + }); + // Every mentor sits right next to the viewer; the seeker has no vector at all. + if (key.startsWith("mentor")) await participants.setEmbedding(eventId, ids[key], VECTOR, embeddings.id); + } + }); + + afterAll(async () => { + await query(`delete from organizations where slug = $1`, [ORG]); + await query(`delete from registered_attendees where email like $1`, [`recall-%-${RUN}@example.com`]); + await closePool(); + }); + + it("finds the only compatible person beyond the nearest neighbours", async () => { + const ranked = await pipeline.run({ eventId, participantId: ids.viewer, mode: "profile" }); + expect(ranked.map((candidate) => candidate.participant.attendeeId)).toEqual([ids.seeker]); + }); + + it("does the same for search", async () => { + const ranked = await pipeline.run({ eventId, participantId: ids.viewer, mode: "search", query: "mentors engineers" }); + expect(ranked.map((candidate) => candidate.participant.attendeeId)).toContain(ids.seeker); + }); +}); From d510446c00ef937ebc4070137b2c45607b3a28c9 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 10:33:17 +0530 Subject: [PATCH 06/11] Measure capacity and latency against agreed budgets (R25 part 3) npm run capacity:check creates a throwaway event of the chosen size, saves every profile, runs the real embed and match workers until the queue is quiet, runs concurrent searches through the real use case, and reports time to first match lists, search p50/p95, failures and model calls per attendee and search against the budgets in the runbook, then removes everything. It uses a free deterministic embedding by default (EMBEDDING_PROVIDER=hash, refused in production) or the deployment's real providers with --providers configured. Locally: 300 attendees matched in 6 s with search p95 83 ms; 1000 attendees in 25 s with search p95 169 ms. --- .../providers/hash-embedding.provider.ts | 23 ++ .../providers/provider-registry.ts | 2 + apps/web/src/lib/env.ts | 5 +- package.json | 3 +- tests/integration/capacity-check.test.ts | 44 ++++ tests/unit/capacity-check.test.ts | 49 ++++ tools/capacity-check.ts | 214 ++++++++++++++++++ 7 files changed, 338 insertions(+), 2 deletions(-) create mode 100644 apps/web/src/infrastructure/providers/hash-embedding.provider.ts create mode 100644 tests/integration/capacity-check.test.ts create mode 100644 tests/unit/capacity-check.test.ts create mode 100644 tools/capacity-check.ts diff --git a/apps/web/src/infrastructure/providers/hash-embedding.provider.ts b/apps/web/src/infrastructure/providers/hash-embedding.provider.ts new file mode 100644 index 0000000..902684a --- /dev/null +++ b/apps/web/src/infrastructure/providers/hash-embedding.provider.ts @@ -0,0 +1,23 @@ +import type { EmbeddingProvider } from "../../application/ports/embedding-provider"; + +const DIMENSIONS = 768; + +/** + * Words hashed into a unit vector: similar texts give similar vectors, with + * no network call and no cost. For capacity tests and staging rehearsals + * only (EMBEDDING_PROVIDER=hash); production refuses it. + */ +export class HashEmbeddingProvider implements EmbeddingProvider { + readonly id = "hash:768"; + + async embed(text: string): Promise { + const vector = new Array(DIMENSIONS).fill(0); + for (const word of text.toLowerCase().match(/[a-z0-9]+/g) ?? []) { + let hash = 0; + for (const char of word) hash = (hash * 31 + char.charCodeAt(0)) % DIMENSIONS; + vector[hash] += 1; + } + const norm = Math.hypot(...vector); + return norm ? vector.map((value) => value / norm) : null; + } +} diff --git a/apps/web/src/infrastructure/providers/provider-registry.ts b/apps/web/src/infrastructure/providers/provider-registry.ts index 28f1f5b..1cd4710 100644 --- a/apps/web/src/infrastructure/providers/provider-registry.ts +++ b/apps/web/src/infrastructure/providers/provider-registry.ts @@ -3,9 +3,11 @@ import type { LlmProvider } from "../../application/ports/llm-provider"; import { GeminiEmbeddingProvider } from "./gemini-embedding.provider"; import { GeminiLlmProvider } from "./gemini-llm.provider"; import { LocalBgeProvider } from "./local-bge.provider"; +import { HashEmbeddingProvider } from "./hash-embedding.provider"; export class ProviderRegistry { static embedding(): EmbeddingProvider { + if (process.env.EMBEDDING_PROVIDER === "hash") return new HashEmbeddingProvider(); if (process.env.EMBEDDING_PROVIDER === "local" && process.env.LOCAL_BGE_URL) { return new LocalBgeProvider(); } diff --git a/apps/web/src/lib/env.ts b/apps/web/src/lib/env.ts index 2a9f2b3..8308a4d 100644 --- a/apps/web/src/lib/env.ts +++ b/apps/web/src/lib/env.ts @@ -27,7 +27,7 @@ const POSITIVE_INTEGERS = [ const ALLOWED: Record = { DATABASE_SSL: ["disable", "require", "verify"], - EMBEDDING_PROVIDER: ["gemini", "local"], + EMBEDDING_PROVIDER: ["gemini", "local", "hash"], EMAIL_DEV_FALLBACK: ["true", "false"], TRUSTED_PROXY_HOPS: ["0", "1", "2", "3"], }; @@ -66,6 +66,9 @@ export function checkEnv(role: EnvRole, env: Env = process.env): string[] { // Production must not quietly fall back to unverified or plain transport. // Local hosts are exempt so the docker-compose stack still runs as production. if (value("NODE_ENV") === "production") { + if (value("EMBEDDING_PROVIDER") === "hash") { + problems.push("EMBEDDING_PROVIDER=hash is for capacity tests only and cannot run in production."); + } if (databaseUrl && !isLocalHost(databaseUrl)) { if (value("DATABASE_SSL") !== "verify") { problems.push("DATABASE_SSL must be verify in production, so the database certificate is checked."); diff --git a/package.json b/package.json index c29ef0b..3aa4379 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,8 @@ "account:erase": "tsx tools/manage-account.ts erase", "jobs:failed": "tsx tools/manage-jobs.ts failed", "jobs:retry": "tsx tools/manage-jobs.ts retry", - "models:usage": "tsx tools/model-usage.ts" + "models:usage": "tsx tools/model-usage.ts", + "capacity:check": "tsx tools/capacity-check.ts" }, "devDependencies": { "@playwright/test": "1.63.0", diff --git a/tests/integration/capacity-check.test.ts b/tests/integration/capacity-check.test.ts new file mode 100644 index 0000000..6d1b24b --- /dev/null +++ b/tests/integration/capacity-check.test.ts @@ -0,0 +1,44 @@ +/** + * A small run of the capacity check against a real Postgres with the free + * hash embedding: every attendee gets matches, searches succeed, and the + * throwaway event and people are removed afterwards. Skipped without DATABASE_URL. + */ +import { afterAll, describe, expect, it, vi } from "vitest"; + +const saved = vi.hoisted(() => { + const saved = { EMBEDDING_PROVIDER: process.env.EMBEDDING_PROVIDER, GEMINI_API_KEY: process.env.GEMINI_API_KEY, MATCH_REFRESH_DELAY_SECONDS: process.env.MATCH_REFRESH_DELAY_SECONDS }; + process.env.EMBEDDING_PROVIDER = "hash"; + process.env.GEMINI_API_KEY = ""; + process.env.MATCH_REFRESH_DELAY_SECONDS = "0"; + delete (globalThis as { __matchMakerContainer?: unknown }).__matchMakerContainer; + return saved; +}); + +import { closePool, query } from "@match-maker/db"; +import { migrate } from "../../packages/db/src/migrate"; +import { budgetFailures, runCapacityCheck } from "../../tools/capacity-check"; + +const describeDb = process.env.DATABASE_URL ? describe : describe.skip; + +describeDb("capacity check", () => { + afterAll(async () => { + await closePool(); + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + delete (globalThis as { __matchMakerContainer?: unknown }).__matchMakerContainer; + }); + + it("measures a small event and cleans up after itself", async () => { + await migrate(); + const throwaway = async () => (await query(`select 1 from organizations where slug like 'capacity-%'`)).rows.length; + const before = await throwaway(); + const report = await runCapacityCheck({ attendees: 12, searches: 6, concurrency: 3, workers: 1 }); + expect(report).toMatchObject({ attendees: 12, searches: 6, attendeesWithoutMatches: 0 }); + expect(report.search.failures).toBe(0); + expect(report.search.p95Ms).toBeGreaterThan(0); + expect(budgetFailures(report)).toEqual([]); + expect(await throwaway()).toBe(before); + }); +}); diff --git a/tests/unit/capacity-check.test.ts b/tests/unit/capacity-check.test.ts new file mode 100644 index 0000000..bcaad9c --- /dev/null +++ b/tests/unit/capacity-check.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { budgetFailures, percentile, type CapacityReport } from "../../tools/capacity-check"; +import { HashEmbeddingProvider } from "@/src/infrastructure/providers/hash-embedding.provider"; + +const report = (overrides: Partial = {}): CapacityReport => ({ + attendees: 300, searches: 60, firstListsSeconds: 30, quietSeconds: 60, attendeesWithoutMatches: 0, + search: { p50Ms: 200, p95Ms: 900, maxMs: 1200, failures: 0 }, + modelCalls: { perAttendee: 2, perSearch: 1, approxTokens: 1000, failures: 0 }, + ...overrides, +}); + +describe("capacity budgets", () => { + it("passes a report within every budget", () => { + expect(budgetFailures(report())).toEqual([]); + }); + + it("names each budget a report misses", () => { + const failures = budgetFailures(report({ + search: { p50Ms: 3000, p95Ms: 7000, maxMs: 9000, failures: 2 }, + attendeesWithoutMatches: 1, + modelCalls: { perAttendee: 9, perSearch: 1, approxTokens: 0, failures: 0 }, + })); + expect(failures).toEqual([ + "search p95 7000 ms is over the budget of 6000 ms", + "model calls per attendee 9 is over the budget of 6", + "2 search(es) failed", + "1 attendee(s) have no matches", + ]); + }); + + it("takes nearest-rank percentiles", () => { + const values = Array.from({ length: 100 }, (_, i) => i + 1); + expect(percentile(values, 50)).toBe(50); + expect(percentile(values, 95)).toBe(95); + expect(percentile([], 95)).toBe(0); + }); +}); + +describe("hash embedding", () => { + it("gives unit vectors that are closer for similar texts", async () => { + const provider = new HashEmbeddingProvider(); + const [a, b, c] = await Promise.all(["graph databases in rust", "rust graph databases", "watercolor painting"].map((text) => provider.embed(text))); + const dot = (x: number[], y: number[]) => x.reduce((sum, value, i) => sum + value * y[i], 0); + expect(a).toHaveLength(768); + expect(dot(a!, a!)).toBeCloseTo(1); + expect(dot(a!, b!)).toBeGreaterThan(dot(a!, c!)); + expect(await provider.embed(" ")).toBeNull(); + }); +}); diff --git a/tools/capacity-check.ts b/tools/capacity-check.ts new file mode 100644 index 0000000..4eab32a --- /dev/null +++ b/tools/capacity-check.ts @@ -0,0 +1,214 @@ +/** + * Capacity and latency check for one event of a chosen size, against the + * configured database: + * + * npm run capacity:check -- --attendees 300 --searches 60 --concurrency 15 --workers 2 + * + * It creates a throwaway organization and event with synthetic attendees, + * saves every profile (as people would), runs the real embed and match + * workers until the queue is quiet, then runs concurrent searches through + * the real use case. It reports matching throughput, search latency and + * model calls, compares them with the budgets in docs/OPERATIONS.md, and + * deletes everything it created (unless --keep). Exits 1 when a budget fails. + * + * Models: by default a free deterministic embedding and no rerank, which + * measures the system itself. `--providers configured` uses the deployment's + * real providers and costs money; run it on staging, not in CI. With + * `--workers 0` the deployment's workers do the matching and the check only + * waits; otherwise stop other workers first, or they share this event's jobs. + */ +import "dotenv/config"; +import { parseArgs } from "node:util"; +import { isMainModule } from "../packages/db/src/is-main"; + +export interface CapacityReport { + attendees: number; + searches: number; + /** Seconds until every attendee had a stored match list. */ + firstListsSeconds: number; + /** Seconds until no job was left, refreshes included. */ + quietSeconds: number; + attendeesWithoutMatches: number; + search: { p50Ms: number; p95Ms: number; maxMs: number; failures: number }; + modelCalls: { perAttendee: number; perSearch: number; approxTokens: number; failures: number }; +} + +export interface Budgets { + searchP95Ms: number; + firstListsSeconds: number; + quietSeconds: number; + modelCallsPerAttendee: number; + modelCallsPerSearch: number; +} + +/** Defaults agreed in docs/OPERATIONS.md for a 300-attendee event. */ +export const DEFAULT_BUDGETS: Budgets = { + searchP95Ms: 6000, + firstListsSeconds: 600, + quietSeconds: 1800, + modelCallsPerAttendee: 6, + modelCallsPerSearch: 2, +}; + +/** One message per budget the report misses; empty when all pass. */ +export function budgetFailures(report: CapacityReport, budgets: Budgets = DEFAULT_BUDGETS): string[] { + const failures: string[] = []; + const over = (label: string, actual: number, limit: number, unit: string) => { + if (actual > limit) failures.push(`${label} ${actual}${unit} is over the budget of ${limit}${unit}`); + }; + over("search p95", report.search.p95Ms, budgets.searchP95Ms, " ms"); + over("time to first match lists", report.firstListsSeconds, budgets.firstListsSeconds, " s"); + over("time until the queue is quiet", report.quietSeconds, budgets.quietSeconds, " s"); + over("model calls per attendee", report.modelCalls.perAttendee, budgets.modelCallsPerAttendee, ""); + over("model calls per search", report.modelCalls.perSearch, budgets.modelCallsPerSearch, ""); + if (report.search.failures) failures.push(`${report.search.failures} search(es) failed`); + if (report.attendeesWithoutMatches) failures.push(`${report.attendeesWithoutMatches} attendee(s) have no matches`); + return failures; +} + +export function percentile(values: number[], p: number): number { + if (!values.length) return 0; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)]; +} + +const WORDS = ["retrieval", "search", "climate", "health", "design", "rust", "compilers", "graph", "databases", "fintech", "payments", + "robotics", "vision", "education", "hiring", "mentoring", "startups", "security", "privacy", "mobile", "frontend", "data", "pipelines", "evaluation"]; +const DOMAINS = ["Artificial Intelligence", "Design", "Finance", "Healthcare", "Education", "Security"]; +const pick = (index: number, salt: number) => WORDS[(index * 7 + salt * 13) % WORDS.length]; + +export async function runCapacityCheck(options: { attendees: number; searches: number; concurrency: number; workers: number; keep?: boolean }): Promise { + // Imported late so provider choices made from the arguments apply. + const { attendees, events, query } = await import("@match-maker/db"); + const { processEmbedJob } = await import("../packages/db/workers/embed.worker"); + const { processMatchJob } = await import("../packages/db/workers/match.worker"); + const { getContainer } = await import("../apps/web/src/composition/container"); + const { searchParticipants } = await import("../apps/web/src/application/use-cases/search-participants"); + const { Intent } = await import("../apps/web/src/domain/participant/intent"); + const container = getContainer(); + + const run = `capacity-${Date.now().toString(36)}`; + const emailOf = (i: number) => `${run}-${i}@example.com`; + const started = new Date(); + const { rows: [org] } = await query<{ id: string }>(`insert into organizations (name, slug) values ($1, $1) returning id`, [run]); + const eventId = (await query<{ id: string }>( + `insert into events (organization_id, name, slug, status) values ($1, $2, $2, 'open') returning id`, [org.id, run])).rows[0].id; + try { + const people = Array.from({ length: options.attendees }, (_, i) => ({ name: `Capacity ${i}`, email: emailOf(i) })); + await attendees.upsertMany(people); + const { rows } = await query<{ id: string }>(`select id from registered_attendees where email like $1 order by email`, [`${run}-%`]); + const ids = rows.map((row) => row.id); + await events.attachRoster(eventId, ids); + const intents = [Intent.NETWORKING, Intent.HIRING, Intent.MENTORING]; + + const savedAt = Date.now(); + for (const [i, attendeeId] of ids.entries()) { + await container.participants.saveProfile({ + eventId, attendeeId, designation: "Engineer", domain: DOMAINS[i % DOMAINS.length], + aboutMe: `Works on ${pick(i, 1)} and ${pick(i, 2)} for ${pick(i, 3)} teams`, selfDescription: null, + yearsOfExperience: (i % 15) + 1, mostImpressiveBuild: null, intent: intents[i % intents.length], + interests: [pick(i, 4), pick(i, 5)], venueZone: null, + }); + } + + const listed = async () => Number((await query<{ count: string }>( + `select count(distinct participant_id) from event_matches where event_id = $1`, [eventId])).rows[0].count); + let firstListsSeconds = -1; + // With --workers 0 the deployment's own workers do the matching; this only waits. + const worker = async (processing: boolean) => { + for (;;) { + const worked = processing && ((await processEmbedJob(eventId)) || (await processMatchJob(eventId))); + if (worked) continue; + const open = await query(`select 1 from matching_jobs where event_id = $1 and status in ('pending', 'running') limit 1`, [eventId]); + if (!open.rows.length) return; + await new Promise((resolve) => setTimeout(resolve, 200)); + } + }; + const watcher = setInterval(async () => { + if (firstListsSeconds < 0 && await listed() >= ids.length) firstListsSeconds = Math.round((Date.now() - savedAt) / 1000); + }, 1000); + await Promise.all(options.workers > 0 ? Array.from({ length: options.workers }, () => worker(true)) : [worker(false)]); + clearInterval(watcher); + const quietSeconds = Math.round((Date.now() - savedAt) / 1000); + const withLists = await listed(); + if (firstListsSeconds < 0) firstListsSeconds = quietSeconds; + + const latencies: number[] = []; + let failures = 0; + let next = 0; + const searcher = async () => { + while (next < options.searches) { + const i = next++; + const begin = Date.now(); + try { + await searchParticipants({ eventId, participantId: ids[i % ids.length], query: `${pick(i, 6)} ${pick(i, 7)}`, eventIntents: intents }, + { pipeline: container.pipeline, participants: container.participants }); + } catch { + failures++; + } + latencies.push(Date.now() - begin); + } + }; + await Promise.all(Array.from({ length: options.concurrency }, searcher)); + + const usage = (await query<{ operation: string; calls: string; failures: string; chars: string }>( + `select operation, count(*) as calls, count(*) filter (where outcome <> 'ok') as failures, coalesce(sum(input_chars), 0) as chars + from model_usage where created_at >= $1 group by operation`, [started])).rows; + const callsOf = (prefix: string[]) => usage.filter((row) => prefix.includes(row.operation)).reduce((sum, row) => sum + Number(row.calls), 0); + const round = (value: number) => Math.round(value * 100) / 100; + return { + attendees: ids.length, + searches: options.searches, + firstListsSeconds, + quietSeconds, + attendeesWithoutMatches: ids.length - withLists, + search: { p50Ms: percentile(latencies, 50), p95Ms: percentile(latencies, 95), maxMs: Math.max(0, ...latencies), failures }, + modelCalls: { + perAttendee: round(callsOf(["embed:profile", "rerank"]) / Math.max(1, ids.length)), + perSearch: round(callsOf(["embed:query"]) / Math.max(1, options.searches)), + approxTokens: Math.round(usage.reduce((sum, row) => sum + Number(row.chars), 0) / 4), + failures: usage.reduce((sum, row) => sum + Number(row.failures), 0), + }, + }; + } finally { + if (!options.keep) { + await query(`delete from organizations where id = $1`, [org.id]); + await query(`delete from registered_attendees where email like $1`, [`${run}-%`]); + } + } +} + +if (isMainModule(import.meta.url)) { + const { values } = parseArgs({ + options: { + attendees: { type: "string", default: "300" }, + searches: { type: "string", default: "60" }, + concurrency: { type: "string", default: "15" }, + workers: { type: "string", default: "2" }, + providers: { type: "string", default: "fake" }, + keep: { type: "boolean", default: false }, + }, + }); + if (values.providers === "fake") { + process.env.EMBEDDING_PROVIDER = "hash"; + process.env.GEMINI_API_KEY = ""; + } else if (values.providers !== "configured") { + console.error("--providers must be fake or configured"); + process.exit(2); + } + process.env.MATCH_REFRESH_DELAY_SECONDS ??= "0"; + const { closePool } = await import("@match-maker/db"); + try { + const report = await runCapacityCheck({ + attendees: Number(values.attendees), searches: Number(values.searches), + concurrency: Number(values.concurrency), workers: Number(values.workers), keep: values.keep, + }); + console.info(JSON.stringify({ providers: values.providers, ...report }, null, 2)); + const failures = budgetFailures(report); + for (const failure of failures) console.error(`[capacity] FAIL ${failure}`); + if (!failures.length) console.info("[capacity] every budget met"); + process.exitCode = failures.length ? 1 : 0; + } finally { + await closePool(); + } +} From 96377bbc26c35c5f6086caa2d947f2c5c2a4616c Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 10:36:26 +0530 Subject: [PATCH 07/11] Rehearse restores and keep rollbacks safe (R25 part 4) npm run db:verify checks a database is usable: pgvector, every migration applied from the same file (either line ending), core tables present, and row counts against a source with --compare. npm run db:restore-rehearsal dumps the compose database, restores it into a scratch database, verifies it and removes it, timing each step (locally 1.3 s). CI now refuses a change to a released migration or a new migration that would break the previous image (drop, rename, type change, truncate) unless the line is marked as a deliberate contract step. --- .github/workflows/ci.yml | 9 +++ package.json | 5 +- tests/unit/release-safety.test.ts | 69 +++++++++++++++++++ tools/check-migrations.ts | 78 ++++++++++++++++++++++ tools/restore-rehearsal.ts | 65 ++++++++++++++++++ tools/verify-database.ts | 106 ++++++++++++++++++++++++++++++ 6 files changed, 331 insertions(+), 1 deletion(-) create mode 100644 tests/unit/release-safety.test.ts create mode 100644 tools/check-migrations.ts create mode 100644 tools/restore-rehearsal.ts create mode 100644 tools/verify-database.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27876c7..b1d8b60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + # The migration gate compares with the base commit. + fetch-depth: 0 - uses: actions/setup-node@v4 with: @@ -47,6 +50,12 @@ jobs: - name: Unit tests run: npx vitest run tests/unit + # Migrations run before the new code starts, and a rollback runs the + # previous image on the migrated schema: released migrations never + # change and new ones only expand. + - name: Migrations are safe to roll back from + run: npx tsx tools/check-migrations.ts "${{ github.event.pull_request.base.sha || github.event.before }}" + integration: name: Integration tests runs-on: ubuntu-latest diff --git a/package.json b/package.json index 3aa4379..acb2922 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,10 @@ "jobs:failed": "tsx tools/manage-jobs.ts failed", "jobs:retry": "tsx tools/manage-jobs.ts retry", "models:usage": "tsx tools/model-usage.ts", - "capacity:check": "tsx tools/capacity-check.ts" + "capacity:check": "tsx tools/capacity-check.ts", + "db:verify": "tsx tools/verify-database.ts", + "db:restore-rehearsal": "tsx tools/restore-rehearsal.ts", + "db:check-migrations": "tsx tools/check-migrations.ts" }, "devDependencies": { "@playwright/test": "1.63.0", diff --git a/tests/unit/release-safety.test.ts b/tests/unit/release-safety.test.ts new file mode 100644 index 0000000..5baada9 --- /dev/null +++ b/tests/unit/release-safety.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { migrationProblems } from "../../tools/check-migrations"; +import { compareState, type DatabaseState } from "../../tools/verify-database"; + +const files = (entries: Record) => new Map(Object.entries(entries)); + +describe("migration release gate", () => { + const released = files({ "0001_init.sql": "create table a (id int);\n" }); + + it("accepts expand-only migrations", () => { + expect(migrationProblems(released, files({ + ...Object.fromEntries(released), + "0002_more.sql": "alter table a add column b text;\ncreate index if not exists a_b on a (b);\ndrop index if exists old_idx;\n", + }))).toEqual([]); + }); + + it("refuses changes to a released migration, even when only line endings stay the same", () => { + expect(migrationProblems(released, files({ "0001_init.sql": "create table a (id int);\r\n" }))).toEqual([]); + expect(migrationProblems(released, files({ "0001_init.sql": "create table a (id bigint);\n" }))[0]).toMatch(/must not change/); + expect(migrationProblems(released, files({}))[0]).toMatch(/must not be deleted/); + }); + + it("names each statement that would break the previous image", () => { + const problems = migrationProblems(released, files({ + ...Object.fromEntries(released), + "0002_risky.sql": [ + "alter table a drop column b;", + "alter table a rename column c to d;", + "alter table a alter column e type bigint;", + "drop table old;", + "-- drop table only mentioned in a comment", + "alter table a drop column f; -- contract: unused since release 41", + ].join("\n"), + })); + expect(problems.map((problem) => problem.split(" ")[0])).toEqual(["0002_risky.sql:1", "0002_risky.sql:2", "0002_risky.sql:3", "0002_risky.sql:4"]); + }); + + it("refuses two migrations with the same number", () => { + expect(migrationProblems(new Map(), files({ "0002_a.sql": "", "0002_b.sql": "" }))).toEqual(["0002_b.sql reuses number 0002 of 0002_a.sql"]); + }); +}); + +describe("database verification", () => { + const state = (overrides: Partial = {}): DatabaseState => ({ + vector: true, + applied: files({ "0001_init.sql": "sum1" }), + counts: { events: 3, organizations: 1 }, + ...overrides, + }); + const expected = new Map([["0001_init.sql", ["sum1", "sum1-crlf"]]]); + + it("accepts a complete database", () => { + expect(compareState(state(), expected)).toEqual({ problems: [], differences: [] }); + }); + + it("reports what makes a database unusable", () => { + const { problems } = compareState(state({ vector: false, applied: files({ "0001_init.sql": "other" }), counts: { events: null } }), new Map([...expected, ["0002_next.sql", ["sum2"]]])); + expect(problems).toEqual([ + "the pgvector extension is missing", + "migration 0001_init.sql was applied from a different file", + "migration 0002_next.sql is not applied", + "table events is missing", + ]); + }); + + it("lists row count differences against a source", () => { + expect(compareState(state(), expected, state({ counts: { events: 4, organizations: 1 } })).differences).toEqual(["events: 3 here, 4 in the source"]); + }); +}); diff --git a/tools/check-migrations.ts b/tools/check-migrations.ts new file mode 100644 index 0000000..5d0be4e --- /dev/null +++ b/tools/check-migrations.ts @@ -0,0 +1,78 @@ +/** + * Release gate for migrations, so a rollback to the previous image stays + * safe: migrations run before the new code, and the old code must keep + * working on the migrated schema. + * + * npx tsx tools/check-migrations.ts + * + * - A released migration (present at the base ref) never changes. + * - New migrations only expand the schema. Dropping or renaming a table or + * column, changing a column type or truncating breaks the previous image; + * such a statement needs `-- contract: ` on the same line, and belongs in a release after the code + * stopped using it. + * - Migration numbers are unique. + */ +import { execFileSync } from "node:child_process"; +import { readFileSync, readdirSync } from "node:fs"; +import { isMainModule } from "../packages/db/src/is-main"; + +const DIR = "packages/db/migrations"; +const BREAKING = [ + /\bdrop\s+table\b/i, + /\bdrop\s+column\b/i, + /\brename\s+(column\s+)?\w+\s+to\b/i, + /\brename\s+to\b/i, + /\balter\s+column\s+\w+\s+(set\s+data\s+)?type\b/i, + /\btruncate\b/i, + /\bdrop\s+schema\b/i, +]; + +const normalize = (sql: string) => sql.replace(/\r\n/g, "\n"); + +export function migrationProblems(base: Map, head: Map): string[] { + const problems: string[] = []; + for (const [name, sql] of base) { + if (!head.has(name)) problems.push(`${name} was released and must not be deleted`); + else if (normalize(head.get(name)!) !== normalize(sql)) problems.push(`${name} was released and must not change; add a new migration instead`); + } + for (const [name, sql] of head) { + if (base.has(name)) continue; + normalize(sql).split("\n").forEach((line, index) => { + const code = line.replace(/--.*$/, ""); + if (BREAKING.some((pattern) => pattern.test(code)) && !/--\s*contract:\s*\S/.test(line)) { + problems.push(`${name}:${index + 1} would break the previous release: "${line.trim()}" (mark a deliberate contract step with "-- contract: ")`); + } + }); + } + const numbers = new Map(); + for (const name of head.keys()) { + const number = name.split("_")[0]; + if (numbers.has(number)) problems.push(`${name} reuses number ${number} of ${numbers.get(number)}`); + numbers.set(number, name); + } + return problems; +} + +function filesAt(ref: string): Map { + execFileSync("git", ["rev-parse", "--verify", `${ref}^{commit}`], { stdio: "ignore" }); + // The base may predate the migrations folder: then nothing was released. + const listing = execFileSync("git", ["ls-tree", "-r", "--name-only", ref, "--", DIR], { encoding: "utf8" }); + const names = listing.split("\n").filter((path) => path.endsWith(".sql")).map((path) => path.slice(DIR.length + 1)); + return new Map(names.map((name) => [name, execFileSync("git", ["show", `${ref}:${DIR}/${name}`], { encoding: "utf8" })])); +} + +if (isMainModule(import.meta.url)) { + const ref = process.argv[2]; + const head = new Map(readdirSync(DIR).filter((name) => name.endsWith(".sql")).map((name) => [name, readFileSync(`${DIR}/${name}`, "utf8")])); + let base = new Map(); + if (!ref || /^0+$/.test(ref)) { + console.warn("[migrations] no base ref; checking new-migration rules for every file"); + } else { + base = filesAt(ref); + } + const problems = migrationProblems(base, head); + for (const problem of problems) console.error(`[migrations] ${problem}`); + console.info(problems.length ? `[migrations] ${problems.length} problem(s)` : `[migrations] ${head.size} migrations; ${head.size - base.size} new, all safe to roll back from`); + process.exitCode = problems.length ? 1 : 0; +} diff --git a/tools/restore-rehearsal.ts b/tools/restore-rehearsal.ts new file mode 100644 index 0000000..3176780 --- /dev/null +++ b/tools/restore-rehearsal.ts @@ -0,0 +1,65 @@ +/** + * Backup and restore rehearsal against the local docker compose Postgres: + * dumps the database, restores the dump into a fresh scratch database, + * verifies the copy (migrations, pgvector, core tables, row counts against + * the source), times each step, and removes the scratch database and dump. + * The source database is only read. + * + * npm run db:restore-rehearsal + * + * Production uses RDS snapshots and point-in-time restore instead; see + * "Restore the database" in docs/OPERATIONS.md, which ends with the same + * `npm run db:verify` check. + */ +import "dotenv/config"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, statSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { verifyDatabase } from "./verify-database"; + +const SCRATCH = process.env.RESTORE_REHEARSAL_DATABASE ?? "restore_rehearsal"; +const user = process.env.POSTGRES_USER ?? "matchmaker"; +const database = process.env.POSTGRES_DB ?? "matchmaker"; +const source = process.env.DATABASE_URL; + +function compose(args: string[], input?: Buffer): Buffer { + const result = spawnSync("docker", ["compose", "exec", "-T", "postgres", ...args], { input, maxBuffer: 1024 * 1024 * 1024 }); + if (result.status !== 0) throw new Error(`${args[0]} failed: ${result.stderr?.toString().trim() || result.error?.message}`); + return result.stdout; +} +const psql = (sql: string) => compose(["psql", "-U", user, "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-tAc", sql]).toString().trim(); +async function timed(label: string, step: () => T | Promise): Promise { + const started = Date.now(); + const result = await step(); + console.info(`[restore] ${label}: ${((Date.now() - started) / 1000).toFixed(1)} s`); + return result; +} + +if (!source) { + console.error("[restore] DATABASE_URL must point at the compose Postgres"); + process.exit(2); +} +if (psql(`select 1 from pg_database where datname = '${SCRATCH}'`)) { + console.error(`[restore] database ${SCRATCH} already exists; it is not ours to replace. Remove it or set RESTORE_REHEARSAL_DATABASE.`); + process.exit(2); +} + +const dir = mkdtempSync(join(tmpdir(), "mr-bridge-restore-")); +const dump = join(dir, "backup.dump"); +const started = Date.now(); +let ok = false; +try { + await timed("dump", () => writeFileSync(dump, compose(["pg_dump", "-U", user, "-d", database, "-Fc"]))); + console.info(`[restore] dump size: ${(statSync(dump).size / 1024 / 1024).toFixed(1)} MiB`); + await timed("create scratch database", () => psql(`create database ${SCRATCH}`)); + await timed("restore", () => compose(["pg_restore", "-U", user, "-d", SCRATCH, "--no-owner", "--exit-on-error"], readFileSync(dump))); + const restored = new URL(source); + restored.pathname = `/${SCRATCH}`; + ok = await timed("verify", () => verifyDatabase(restored.toString(), { compare: source })); +} finally { + psql(`drop database if exists ${SCRATCH}`); + rmSync(dir, { recursive: true, force: true }); +} +console.info(`[restore] ${ok ? "rehearsal passed" : "rehearsal FAILED"} in ${((Date.now() - started) / 1000).toFixed(1)} s`); +process.exitCode = ok ? 0 : 1; diff --git a/tools/verify-database.ts b/tools/verify-database.ts new file mode 100644 index 0000000..5fb3e0c --- /dev/null +++ b/tools/verify-database.ts @@ -0,0 +1,106 @@ +/** + * Checks that a database is a usable Mr. Bridge database: reachable, with + * pgvector, every migration in this checkout applied unchanged, and the core + * tables present. With --compare it also compares row counts with another + * database, e.g. a restored copy against its source: + * + * npm run db:verify -- --url --compare [--strict] + * + * Without --url it checks DATABASE_URL. Counts may differ a little after a + * point-in-time restore of a live database; --strict makes any difference fail. + */ +import "dotenv/config"; +import { createHash } from "node:crypto"; +import { readFileSync, readdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import pg from "pg"; +import { resolveSsl } from "../packages/db/src/client"; +import { isMainModule } from "../packages/db/src/is-main"; + +const MIGRATIONS_DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "packages", "db", "migrations"); + +/** The tables the journey cannot work without; their row counts are compared. */ +export const CORE_TABLES = [ + "organizations", "events", "registered_attendees", "event_participants", "event_matches", + "matching_jobs", "intro_requests", "notification_outbox", "location_requests", "audit_events", +]; + +export interface DatabaseState { + vector: boolean; + /** Migration name -> checksum, as recorded in schema_migrations. */ + applied: Map; + counts: Record; +} + +export async function readState(url: string): Promise { + const client = new pg.Client({ connectionString: url, ssl: resolveSsl(url) }); + await client.connect(); + try { + const vector = (await client.query(`select 1 from pg_extension where extname = 'vector'`)).rows.length > 0; + const migrations = await client.query<{ name: string; checksum: string }>(`select name, checksum from schema_migrations`).catch(() => ({ rows: [] })); + const counts: Record = {}; + for (const table of CORE_TABLES) { + const exists = (await client.query(`select to_regclass($1) as name`, [`public.${table}`])).rows[0].name; + counts[table] = exists ? Number((await client.query(`select count(*) from ${table}`)).rows[0].count) : null; + } + return { vector, applied: new Map(migrations.rows.map((row) => [row.name, row.checksum])), counts }; + } finally { + await client.end(); + } +} + +/** + * Migration files in this checkout with their accepted checksums. The runner + * hashes the file as checked out, so a database migrated on Linux (LF) and + * checked from a Windows checkout (CRLF) is the same file: both are accepted. + */ +export function expectedMigrations(dir = MIGRATIONS_DIR): Map { + const sha = (text: string) => createHash("sha256").update(text).digest("hex"); + return new Map(readdirSync(dir).filter((file) => file.endsWith(".sql")).sort().map((file) => { + const lf = readFileSync(join(dir, file), "utf8").replace(/\r\n/g, "\n"); + return [file, [sha(lf), sha(lf.replace(/\n/g, "\r\n"))]]; + })); +} + +/** Problems that make the database unusable, then count differences against `source`. */ +export function compareState(state: DatabaseState, expected: Map, source?: DatabaseState): { problems: string[]; differences: string[] } { + const problems: string[] = []; + if (!state.vector) problems.push("the pgvector extension is missing"); + for (const [name, checksums] of expected) { + const applied = state.applied.get(name); + if (!applied) problems.push(`migration ${name} is not applied`); + else if (!checksums.includes(applied)) problems.push(`migration ${name} was applied from a different file`); + } + for (const [table, count] of Object.entries(state.counts)) { + if (count === null) problems.push(`table ${table} is missing`); + } + const differences = source + ? Object.keys(state.counts).filter((table) => state.counts[table] !== source.counts[table]) + .map((table) => `${table}: ${state.counts[table]} here, ${source.counts[table]} in the source`) + : []; + return { problems, differences }; +} + +export async function verifyDatabase(url: string, options: { compare?: string; strict?: boolean } = {}): Promise { + const state = await readState(url); + const source = options.compare ? await readState(options.compare) : undefined; + const { problems, differences } = compareState(state, expectedMigrations(), source); + console.info(`[db:verify] ${state.applied.size} migrations applied; rows: ${CORE_TABLES.map((table) => `${table}=${state.counts[table] ?? "missing"}`).join(" ")}`); + for (const problem of problems) console.error(`[db:verify] FAIL ${problem}`); + for (const difference of differences) console[options.strict ? "error" : "warn"](`[db:verify] ${options.strict ? "FAIL" : "differs"} ${difference}`); + const ok = problems.length === 0 && (!options.strict || differences.length === 0); + console.info(ok ? "[db:verify] database is usable" : "[db:verify] database check failed"); + return ok; +} + +if (isMainModule(import.meta.url)) { + const { values } = parseArgs({ options: { url: { type: "string" }, compare: { type: "string" }, strict: { type: "boolean", default: false } } }); + const url = values.url ?? process.env.DATABASE_URL; + if (!url) { + console.error("[db:verify] pass --url or set DATABASE_URL"); + process.exit(2); + } + process.exitCode = (await verifyDatabase(url, { compare: values.compare, strict: values.strict })) ? 0 : 1; +} From 46da091d83739f14c73631cc5ba767947cd60e65 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 10:38:41 +0530 Subject: [PATCH 08/11] bug fixx-deploy-role-worker-repo The GitHub deploy role could push to the web, Keycloak and migrate image repositories but not the worker's, while every release pushes a worker image, so a real deploy would stop at "Build & push worker" with access denied. The worker repository is now in the push statement. --- infra/terraform/iam.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/terraform/iam.tf b/infra/terraform/iam.tf index 3061fd9..a83776d 100644 --- a/infra/terraform/iam.tf +++ b/infra/terraform/iam.tf @@ -162,6 +162,7 @@ data "aws_iam_policy_document" "github_deploy" { aws_ecr_repository.web.arn, aws_ecr_repository.keycloak.arn, aws_ecr_repository.migrate.arn, + aws_ecr_repository.worker.arn, ] } From 9f2cfb2cd3f4553348a6a42aa0141ac54120db28 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 10:39:25 +0530 Subject: [PATCH 09/11] Add a staging target and promotion of tested images (R25 part 5) The deploy workflow takes an environment (production or staging). Staging is its own Terraform stack (match-maker-staging-*, example variables and separate state key), its jobs run in the GitHub 'staging' environment and refuse to start unless that environment names the staging stack, so they never fall back to production settings. promote_tag copies the images a staging deploy tested into production unchanged (buildx imagetools, same account) and then migrates and rolls out as usual; production's deploy role may read the staging repositories when image_promotion_source is set. Pushes to main still deploy to production as before. --- .github/workflows/deploy.yml | 113 ++++++++++++++---- .gitignore | 2 + .../environments/staging.tfvars.example | 54 +++++++++ infra/terraform/iam.tf | 11 ++ infra/terraform/variables.tf | 6 + 5 files changed, 163 insertions(+), 23 deletions(-) create mode 100644 infra/terraform/environments/staging.tfvars.example diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 25ad537..6b8dffa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -5,6 +5,15 @@ on: branches: [main] workflow_dispatch: inputs: + environment: + description: "Where to deploy. Staging is a separate Terraform stack (environment = staging)." + type: choice + options: [production, staging] + default: production + promote_tag: + description: "Promote: copy this tag's staging images to production unchanged, then migrate and roll out (production only)" + type: string + default: "" deploy_keycloak: description: "Also rebuild and roll out Keycloak" type: boolean @@ -16,7 +25,7 @@ on: # Never let two deploys race for the same service. concurrency: - group: deploy-production + group: deploy-${{ inputs.environment || 'production' }} cancel-in-progress: false permissions: @@ -24,29 +33,59 @@ permissions: id-token: write contents: read +# Staging names follow the Terraform stack (match-maker-staging-*) and never +# fall back to production's repository variables. Each job that touches AWS +# runs in the GitHub environment of the target, so its role, subnets and URL +# come from that environment's secrets and variables. env: + TARGET: ${{ inputs.environment || 'production' }} + STACK: ${{ inputs.environment == 'staging' && 'match-maker-staging' || 'match-maker-prod' }} AWS_REGION: ${{ vars.AWS_REGION || 'ap-south-1' }} - ECS_CLUSTER: ${{ vars.ECS_CLUSTER || 'match-maker-prod-cluster' }} - ECS_WEB_SERVICE: ${{ vars.ECS_WEB_SERVICE || 'match-maker-prod-web' }} - ECS_WORKER_SERVICE: ${{ vars.ECS_WORKER_SERVICE || 'match-maker-prod-worker' }} - ECS_KEYCLOAK_SERVICE: ${{ vars.ECS_KEYCLOAK_SERVICE || 'match-maker-prod-keycloak' }} - MIGRATE_TASK_FAMILY: ${{ vars.MIGRATE_TASK_FAMILY || 'match-maker-prod-migrate' }} + ECS_CLUSTER: ${{ inputs.environment == 'staging' && 'match-maker-staging-cluster' || vars.ECS_CLUSTER || 'match-maker-prod-cluster' }} + ECS_WEB_SERVICE: ${{ inputs.environment == 'staging' && 'match-maker-staging-web' || vars.ECS_WEB_SERVICE || 'match-maker-prod-web' }} + ECS_WORKER_SERVICE: ${{ inputs.environment == 'staging' && 'match-maker-staging-worker' || vars.ECS_WORKER_SERVICE || 'match-maker-prod-worker' }} + ECS_KEYCLOAK_SERVICE: ${{ inputs.environment == 'staging' && 'match-maker-staging-keycloak' || vars.ECS_KEYCLOAK_SERVICE || 'match-maker-prod-keycloak' }} + MIGRATE_TASK_FAMILY: ${{ inputs.environment == 'staging' && 'match-maker-staging-migrate' || vars.MIGRATE_TASK_FAMILY || 'match-maker-prod-migrate' }} jobs: # Decides what ships: the pushed commit, or an existing tag on rollback. plan: name: Plan release runs-on: ubuntu-latest + environment: ${{ inputs.environment || 'production' }} outputs: tag: ${{ steps.plan.outputs.tag }} rollback: ${{ steps.plan.outputs.rollback }} + promote: ${{ steps.plan.outputs.promote }} steps: + # A staging run must find staging's own settings, never production's. + - name: Check the target environment + env: + STACK_NAME: ${{ vars.STACK_NAME }} + run: | + set -euo pipefail + if [ "$TARGET" = "staging" ] && [ "${STACK_NAME:-}" != "match-maker-staging" ]; then + echo "::error::Set STACK_NAME=match-maker-staging (and staging's AWS secrets) in the GitHub 'staging' environment first." + exit 1 + fi + - id: plan env: ROLLBACK_TAG: ${{ inputs.rollback_tag }} + PROMOTE_TAG: ${{ inputs.promote_tag }} run: | set -euo pipefail - if [ -n "${ROLLBACK_TAG:-}" ]; then + echo "promote=false" >> "$GITHUB_OUTPUT" + if [ -n "${PROMOTE_TAG:-}" ]; then + if [ "$TARGET" != "production" ] || [ -n "${ROLLBACK_TAG:-}" ] || ! [[ "$PROMOTE_TAG" =~ ^[0-9a-f]{7,40}$ ]]; then + echo "::error::promote_tag takes an image tag, targets production, and cannot be combined with rollback_tag." + exit 1 + fi + echo "tag=$PROMOTE_TAG" >> "$GITHUB_OUTPUT" + echo "rollback=false" >> "$GITHUB_OUTPUT" + echo "promote=true" >> "$GITHUB_OUTPUT" + echo "Promoting the staging images tagged $PROMOTE_TAG" + elif [ -n "${ROLLBACK_TAG:-}" ]; then if ! [[ "$ROLLBACK_TAG" =~ ^[0-9a-f]{7,40}$ ]]; then echo "::error::rollback_tag must be an image tag (a commit SHA prefix), got '$ROLLBACK_TAG'." exit 1 @@ -67,7 +106,8 @@ jobs: ci: name: Required checks needs: [plan] - if: needs.plan.outputs.rollback != 'true' + # A promoted tag passed these checks when it was built for staging. + if: needs.plan.outputs.rollback != 'true' && needs.plan.outputs.promote != 'true' uses: ./.github/workflows/ci.yml with: build_images: false @@ -76,6 +116,10 @@ jobs: name: Build & push images runs-on: ubuntu-latest needs: [plan, ci] + if: >- + !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.rollback != 'true' && + (needs.plan.outputs.promote == 'true' || needs.ci.result == 'success') + environment: ${{ inputs.environment || 'production' }} env: TAG: ${{ needs.plan.outputs.tag }} steps: @@ -91,7 +135,24 @@ jobs: - uses: docker/setup-buildx-action@v3 + # The same image digests that passed staging; nothing is rebuilt. Needs + # both stacks' repositories in one AWS account and registry. + - name: Copy the staging images + if: needs.plan.outputs.promote == 'true' + env: + REGISTRY: ${{ steps.ecr.outputs.registry }} + run: | + set -euo pipefail + repos="web migrate worker" + if [ "${{ inputs.deploy_keycloak }}" = "true" ]; then repos="$repos keycloak"; fi + for repo in $repos; do + docker buildx imagetools create \ + --tag "$REGISTRY/$STACK/$repo:$TAG" --tag "$REGISTRY/$STACK/$repo:latest" \ + "$REGISTRY/match-maker-staging/$repo:$TAG" + done + - name: Build & push web + if: needs.plan.outputs.promote != 'true' uses: docker/build-push-action@v6 with: context: . @@ -99,24 +160,26 @@ jobs: push: true build-args: APP_VERSION=${{ env.TAG }} tags: | - ${{ steps.ecr.outputs.registry }}/match-maker-prod/web:${{ env.TAG }} - ${{ steps.ecr.outputs.registry }}/match-maker-prod/web:latest + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/web:${{ env.TAG }} + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/web:latest cache-from: type=gha cache-to: type=gha,mode=max - name: Build & push migrate + if: needs.plan.outputs.promote != 'true' uses: docker/build-push-action@v6 with: context: . file: infra/docker/migrate.Dockerfile push: true tags: | - ${{ steps.ecr.outputs.registry }}/match-maker-prod/migrate:${{ env.TAG }} - ${{ steps.ecr.outputs.registry }}/match-maker-prod/migrate:latest + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/migrate:${{ env.TAG }} + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/migrate:latest cache-from: type=gha cache-to: type=gha,mode=max - name: Build & push worker + if: needs.plan.outputs.promote != 'true' uses: docker/build-push-action@v6 with: context: . @@ -124,21 +187,21 @@ jobs: push: true build-args: APP_VERSION=${{ env.TAG }} tags: | - ${{ steps.ecr.outputs.registry }}/match-maker-prod/worker:${{ env.TAG }} - ${{ steps.ecr.outputs.registry }}/match-maker-prod/worker:latest + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/worker:${{ env.TAG }} + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/worker:latest cache-from: type=gha cache-to: type=gha,mode=max - name: Build & push Keycloak - if: inputs.deploy_keycloak + if: inputs.deploy_keycloak && needs.plan.outputs.promote != 'true' uses: docker/build-push-action@v6 with: context: . file: infra/docker/keycloak.Dockerfile push: true tags: | - ${{ steps.ecr.outputs.registry }}/match-maker-prod/keycloak:${{ env.TAG }} - ${{ steps.ecr.outputs.registry }}/match-maker-prod/keycloak:latest + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/keycloak:${{ env.TAG }} + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/keycloak:latest cache-from: type=gha cache-to: type=gha,mode=max @@ -146,6 +209,9 @@ jobs: name: Run migrations runs-on: ubuntu-latest needs: [plan, build-and-push] + if: >- + !cancelled() && needs.plan.outputs.rollback != 'true' && needs.build-and-push.result == 'success' + environment: ${{ inputs.environment || 'production' }} env: TAG: ${{ needs.plan.outputs.tag }} steps: @@ -164,7 +230,7 @@ jobs: REGISTRY="$(aws sts get-caller-identity --query Account --output text).dkr.ecr.${AWS_REGION}.amazonaws.com" aws ecs describe-task-definition --task-definition "$MIGRATE_TASK_FAMILY" \ --query 'taskDefinition' > migrate-taskdef.json - jq --arg IMAGE "$REGISTRY/match-maker-prod/migrate:$TAG" ' + jq --arg IMAGE "$REGISTRY/$STACK/migrate:$TAG" ' .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt) @@ -198,7 +264,7 @@ jobs: echo "Exit code: $EXIT_CODE ($REASON)" if [ "$EXIT_CODE" != "0" ]; then - echo "::error::Migration failed — aborting deploy. Check /ecs/match-maker-prod/migrate." + echo "::error::Migration failed — aborting deploy. Check /ecs/$STACK/migrate." exit 1 fi @@ -206,6 +272,7 @@ jobs: name: Roll out runs-on: ubuntu-latest needs: [plan, build-and-push, migrate] + environment: ${{ inputs.environment || 'production' }} # A rollback skips checks, build and migrations (the schema only moves # forward); a normal release needs all of them to have succeeded. if: >- @@ -228,7 +295,7 @@ jobs: repos="web worker" if [ "${{ inputs.deploy_keycloak }}" = "true" ]; then repos="$repos keycloak"; fi for repo in $repos; do - aws ecr describe-images --repository-name "match-maker-prod/$repo" \ + aws ecr describe-images --repository-name "$STACK/$repo" \ --image-ids imageTag="$TAG" >/dev/null \ || { echo "::error::No $repo image tagged $TAG in ECR."; exit 1; } done @@ -272,7 +339,7 @@ jobs: # Strip the server-managed fields that RegisterTaskDefinition rejects, # then point the container at the immutable tag. - jq --arg IMAGE "$REGISTRY/match-maker-prod/web:$TAG" ' + jq --arg IMAGE "$REGISTRY/$STACK/web:$TAG" ' .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt) @@ -304,7 +371,7 @@ jobs: aws ecs describe-task-definition --task-definition "$FAMILY" \ --query 'taskDefinition' > worker-taskdef.json - jq --arg IMAGE "$REGISTRY/match-maker-prod/worker:$TAG" ' + jq --arg IMAGE "$REGISTRY/$STACK/worker:$TAG" ' .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt) @@ -335,7 +402,7 @@ jobs: --query 'services[0].taskDefinition' --output text | sed -E 's#^.*task-definition/([^:]+):[0-9]+$#\1#') aws ecs describe-task-definition --task-definition "$FAMILY" \ --query 'taskDefinition' > keycloak-taskdef.json - jq --arg IMAGE "$REGISTRY/match-maker-prod/keycloak:$TAG" ' + jq --arg IMAGE "$REGISTRY/$STACK/keycloak:$TAG" ' .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt) diff --git a/.gitignore b/.gitignore index d1464f3..88b2c56 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,8 @@ infra/terraform/*.tfstate infra/terraform/*.tfstate.* infra/terraform/*.tfvars !infra/terraform/*.tfvars.example +infra/terraform/environments/*.tfvars +!infra/terraform/environments/*.tfvars.example infra/terraform/*.tfplan infra/terraform/.terraform.lock.hcl infra/terraform/override.tf diff --git a/infra/terraform/environments/staging.tfvars.example b/infra/terraform/environments/staging.tfvars.example new file mode 100644 index 0000000..cb6c6b4 --- /dev/null +++ b/infra/terraform/environments/staging.tfvars.example @@ -0,0 +1,54 @@ +# Staging: a separate stack (match-maker-staging-*) with its own state, in the +# same AWS account and region as production so tested images can be promoted +# unchanged. Copy to staging.tfvars (gitignored) and run, from infra/terraform: +# +# terraform init -reconfigure -backend-config="bucket=" \ +# -backend-config="key=match-maker/staging/terraform.tfstate" -backend-config="region=ap-south-1" +# terraform apply -var-file=environments/staging.tfvars +# +# Switch back to production with the same init and key=match-maker/terraform.tfstate. + +aws_region = "ap-south-1" +project_name = "match-maker" +environment = "staging" +default_event_slug = "demo" + +# Staging may be HTTP-only for a first evaluation; give it its own hostname +# and certificate to rehearse the production login flow (Google needs HTTPS). +domain_name = "" +keycloak_domain_name = "" +acm_certificate_arn = "" +route53_zone_id = "" +keycloak_admin_cidrs = [] + +# Production's stack creates the account's GitHub OIDC provider; reuse it. +github_repository = "LucknowAI/Bridge" +github_oidc_provider_arn = "arn:aws:iam:::oidc-provider/token.actions.githubusercontent.com" + +# Small but production-shaped: two web tasks so rolling deploys and the +# shared caches behave as in production. +web_desired_count = 2 +web_min_capacity = 2 +web_max_capacity = 4 +worker_desired_count = 1 +keycloak_min_capacity = 1 +keycloak_max_capacity = 2 + +db_instance_class = "db.t4g.micro" +db_multi_az = false +db_backup_retention_days = 3 +db_deletion_protection = false + +enable_nat_gateway = true + +# Staging gets its own provider keys, so its usage and quota never touch +# production's. Fill them into Secrets Manager after the first apply. +gemini_api_key = "" +upstash_redis_rest_url = "" +upstash_redis_rest_token = "" +resend_api_key = "" +google_client_id = "" +google_client_secret = "" + +email_from = "Mr. Bridge staging " +alarm_email = "" diff --git a/infra/terraform/iam.tf b/infra/terraform/iam.tf index a83776d..a370fc4 100644 --- a/infra/terraform/iam.tf +++ b/infra/terraform/iam.tf @@ -146,6 +146,17 @@ data "aws_iam_policy_document" "github_deploy" { resources = ["*"] } + # Promotion copies another stack's tested images (e.g. staging's) here + # unchanged, so this role may read that stack's repositories. + dynamic "statement" { + for_each = var.image_promotion_source != "" ? [var.image_promotion_source] : [] + content { + sid = "EcrPromoteFrom" + actions = ["ecr:BatchGetImage", "ecr:GetDownloadUrlForLayer", "ecr:BatchCheckLayerAvailability", "ecr:DescribeImages"] + resources = ["arn:aws:ecr:${var.aws_region}:${data.aws_caller_identity.current.account_id}:repository/${statement.value}/*"] + } + } + statement { sid = "EcrPush" actions = [ diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf index 42114e4..2c9ff5c 100644 --- a/infra/terraform/variables.tf +++ b/infra/terraform/variables.tf @@ -316,3 +316,9 @@ variable "session_secret_active" { error_message = "session_secret_active must be \"a\" or \"b\"." } } + +variable "image_promotion_source" { + description = "Stack whose images production may promote unchanged, e.g. \"match-maker-staging\" (same account and region). Empty disables promotion." + type = string + default = "" +} From 4e5e5c4db1b2bb2c857721b4d924104c41e9ce99 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 10:42:40 +0530 Subject: [PATCH 10/11] Write the operations runbook (R25 part 6) docs/OPERATIONS.md: the service targets and measured capacity, the release path through staging and promotion, migration rules and rollback, what to do for every alarm, database restore with verification, rotation of each secret, running tools against the private database, and the launch checklist. The worker image now carries the operator tools so account commands and the Keycloak sync can run as one-off tasks inside the VPC. --- ReadMe.md | 4 +- docs/OPERATIONS.md | 270 +++++++++++++++++++++++++++++++++ infra/docker/worker.Dockerfile | 2 + 3 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 docs/OPERATIONS.md diff --git a/ReadMe.md b/ReadMe.md index f088f0a..6a6b387 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -4,7 +4,7 @@ Mr. Bridge is a multi event networking and matchmaking app. Organizers create ev The implementation follows the architecture described in [docs/PROJECT_STRUCTURE.md](docs/PROJECT_STRUCTURE.md). The [end-to-end flow](docs/END_TO_END_FLOW.md) and [matching pipeline](docs/BRIDGE_ENGINE_PIPELINE.md) docs show the product journeys. -See the [production roadmap](docs/PRODUCTION_ROADMAP.md) for authentication findings, the invited-account signup policy, Google setup, and remaining work ordered by implementation effort. +See the [production roadmap](docs/PRODUCTION_ROADMAP.md) for authentication findings, the invited-account signup policy, Google setup, and remaining work ordered by implementation effort. The [operations runbook](docs/OPERATIONS.md) covers releases, staging, rollback, alarms, database restore, secret rotation and the launch checklist. ## Architecture @@ -105,6 +105,8 @@ The first command imports the global attendee directory so the organizer account | `npm test` | Run unit and integration suites. Integration suites skip without Postgres/Keycloak; set `REQUIRE_SERVICES=db,keycloak` to fail instead (CI does, and `tools/check-test-report.ts` fails on any skipped test). Stop a locally running worker first, or it claims the pipeline test's jobs | | `npm run test:e2e` | Browser acceptance suite (Playwright) against a running app and worker started with `EMAIL_DEV_FALLBACK=true`; set `E2E_BASE_URL` and `E2E_WEB_LOG` (the app's output, where login codes are read) | | `npm run db:migrate` | Apply pending PostgreSQL migrations | +| `npm run capacity:check` | Measure matching throughput, search latency and model calls for an event size against the budgets in the runbook | +| `npm run db:verify` / `db:restore-rehearsal` | Check a database (or a restored copy) is usable; rehearse dump and restore locally | | `npm run db:seed` | Upsert demo roster, organization, event, and organizer | | `npm run db:import -- [event-slug]` | Import attendee rows and optionally attach them to an event | | `npm run keycloak:wait` | Wait for the configured Keycloak realm to respond | diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 0000000..54a2552 --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,270 @@ +# Operations runbook + +How Mr. Bridge is released, watched and recovered in production. Every alarm +names a section of this file. Commands run from the repository root unless a +step says otherwise. AWS names follow the Terraform stack: +`match-maker-prod-*` for production, `match-maker-staging-*` for staging. + +## Service targets + +The launch target is one event of **300 attendees** with up to **15 +concurrent searches** and two worker tasks. + +| Measure | Budget | Checked by | +| --- | --- | --- | +| Search latency, p95 | ≤ 6 s (the search time budget; reranking is skipped when time runs short) | `SearchLatencyMs` alarm, capacity check | +| Every attendee has a match list after the profiles are saved | ≤ 10 min | capacity check | +| Queue quiet (all refreshes done) | ≤ 30 min | capacity check | +| A due matching job or notification waits | ≤ 10 min | `OldestDue*` alarms | +| Model calls per attendee (profile embedding plus reranks) | ≤ 6 | capacity check, `npm run models:usage` | +| Model calls per search (query embedding; cached for a day) | ≤ 2 | capacity check | +| Attendees left without matches | 0 | capacity check | + +Measured on a developer machine (local Postgres, free hash embedding, no +rerank), these are the system's own costs, without model latency: + +| Size | First match lists | Queue quiet | Search p50 / p95 | +| --- | --- | --- | --- | +| 300 attendees, 60 searches, 15 at a time | 6 s | 8 s | 42 ms / 83 ms | +| 1000 attendees, 200 searches, 40 at a time | 25 s | 36 s | 102 ms / 169 ms | + +Repeat the check on staging with the real providers before launch and after +any change to matching (see *Release*). It costs model calls. + +```sh +npm run capacity:check -- --attendees 300 --searches 60 --concurrency 15 --workers 0 --providers configured +``` + +`--workers 0` lets the deployed workers do the matching; the check only +watches. With `--workers N` it runs its own workers, so stop any other +worker on that database first. Without `--providers configured` it uses a +free deterministic embedding, which measures the system and not the models. +The check deletes its throwaway event and people when it finishes. + +## Release + +1. **CI** runs on every pull request and again inside every deploy: + typecheck, lint, unit tests, the migration gate, integration tests + against real Postgres and Keycloak (a skipped test fails the job), the + browser journey against the built app and worker, the image builds and + `terraform validate`. +2. **Staging.** Run the *Deploy to ECS* workflow with `environment: + staging` on the commit to release. It builds, migrates staging, rolls + out and smoke-tests. Then sign in on staging with a real e-mail and + Google account, walk the journey once (create an event, import, two + attendees, match, search, intro, accept, venue question), and run the + capacity check above. +3. **Production.** Merge to `main`, which builds and deploys that commit; + or, to ship exactly the images staging tested, run the workflow with + `environment: production` and `promote_tag: `. Promotion + copies the images unchanged, then migrates production and rolls out. + It needs `image_promotion_source = "match-maker-staging"` in production's + Terraform variables. +4. The deploy waits for the services to settle, checks the new version is + serving and its dependencies are ready, and puts the previous revisions + back by itself if that fails. + +### Migrations and rollback + +Migrations run before the new code and the previous image must keep +working on the migrated schema. CI enforces it (`tools/check-migrations.ts`): +released migrations never change, and a new migration may not drop or +rename a table or column, change a column type or truncate. To remove +something, stop using it in one release and drop it in a later one, with +the line marked `-- contract: `. + +### Roll back a release + +Run *Deploy to ECS* with `rollback_tag: ` (the 12-character +commit prefix shown in earlier runs and in `/api/health`). It skips checks, +build and migrations and redeploys those images. Because migrations only +expand, the old code runs on the current schema. Confirm `/api/health` +reports the old version and watch the alarms for ten minutes. + +Rehearse it on staging before launch: deploy a commit, roll back to the +previous tag, confirm sign-in and search still work, deploy forward again. + +## Alarms + +Alarms go to the SNS topic `match-maker-prod-alarms` (`alarm_email` in the +Terraform variables subscribes an address). The dashboard +`match-maker-prod` shows search latency, queue waits and sizes, failures, +model call failures and web response time. + +### Site down (`web-no-healthy-hosts`, `alb-5xx`, `alb-p95-latency`) + +1. ECS console → `match-maker-prod-web` → *Events* and stopped tasks: a + failing health check, an out-of-memory stop or a configuration error + (the app refuses to start on bad configuration and logs every problem). +2. If it began with a release, roll back (above). +3. If the database is the cause (`/api/health/ready` reports it), see the + database alarms. + +### Keycloak down (`keycloak-no-healthy-hosts`) + +Nobody can sign in (code sign-in also registers the person in Keycloak); +people already signed in carry on. Check the `match-maker-prod-keycloak` +service events and its logs in `/ecs/match-maker-prod/keycloak`; its +database login is in the `identity` secret. + +### Worker down (`worker-down`) + +No heartbeat means matching and notifications have stopped. Check the +`match-maker-prod-worker` service: a crash loop shows in its stopped tasks +and `/ecs/match-maker-prod/worker`. Queued work is durable; when the worker +returns it continues, and jobs held by a dead worker are reclaimed once +their lease ends (5 minutes). + +### Matching backlog (`matching-backlog`) + +Work is arriving faster than one worker handles it, or jobs are slow +because a model provider is slow. Check `ModelCallLatencyMs` and the model +failure alarm. To add capacity, raise `worker_desired_count` and apply, or +scale the service in the ECS console for the event day. Workers share the +queue safely. + +### Failed jobs (`jobs-failed`) + +A job failed five times. Run `npm run jobs:failed` against production (see +*Running tools against production*) to see each error. Fix the cause, then +`npm run jobs:retry -- all`. + +### Notifications stuck (`notification-backlog`, `notifications-failed`) + +Usually e-mail delivery. Check the worker log for `[worker:notify]` errors +and the Resend dashboard (key revoked, domain unverified, quota). Failed +notifications keep their last error in `notification_outbox`; once fixed, +set them back with +`update notification_outbox set status = 'pending', attempts = 0, available_at = now() where status = 'failed'`. +Each is sent at most once per subject, so retrying cannot duplicate. + +### Slow search (`search-slow`, `search-failing`) + +Search waits at most 2.5 s for the query embedding and skips reranking +when time runs short, so sustained slowness is usually the database. Check +RDS CPU and connections, then the model failure alarm, then errors in +`/ecs/match-maker-prod/web`. + +### Model provider failing (`model-failures`) + +Over 20% of Gemini calls fail. Matching keeps working on fallbacks: lexical +search, scored order with generated reasons. `npm run models:usage` shows +failures by operation. Common causes are an expired or revoked key (rotate +it, below) or an exhausted quota (raise it in Google AI Studio). The +gateway's circuit breaker stops calling a failing provider for 30 s at a +time. + +### Database (`rds-cpu`, `rds-free-storage`, `rds-connections`) + +CPU: look for a runaway query in Performance Insights; the worker can be +scaled down to shed load. Storage: raise `db_max_allocated_storage` (storage +grows by itself up to it) and apply. Connections: each task opens at most 10; a climb +without more tasks points at a leak in a recent release, so roll back. + +### Login e-mail failures (`login-email-failures`) + +People cannot sign in by code. Check the Resend dashboard and the +`RESEND_API_KEY` secret; rotate the key if needed (below). + +### Web at maximum capacity (`web-at-max-capacity`) + +Raise `web_max_capacity` and apply; check the database can take the extra +connections (10 per task). + +## Restore the database + +RDS keeps automated backups for `db_backup_retention_days` days and allows +point-in-time restore to any second in that window. + +1. Restore to a **new** instance; never overwrite the live one: + ```sh + aws rds restore-db-instance-to-point-in-time \ + --source-db-instance-identifier match-maker-prod-postgres \ + --target-db-instance-identifier match-maker-prod-restore-$(date +%Y%m%d%H%M) \ + --restore-time 2026-10-19T09:30:00Z \ + --db-subnet-group-name match-maker-prod-db-subnet-group --vpc-security-group-ids \ + --db-parameter-group-name match-maker-prod-pg16 \ + --no-publicly-accessible + aws rds wait db-instance-available --db-instance-identifier + ``` +2. Verify it from a task or host inside the VPC (the database is private): + ```sh + npm run db:verify -- --url "postgres://:@:5432/matchmaker" --compare "$DATABASE_URL" + ``` + It checks pgvector, that every migration of this release is applied + unchanged, the core tables, and lists row count differences against the + live database. +3. To switch over, stop the worker (desired count 0), point the + `DATABASE_URL` values in the `app` and `migrate` secrets at the restored + endpoint, run the migrate task (it also sets the runtime login's + password), force a new deployment of web and worker, and confirm + `/api/health/ready`. Keep the old instance until the restored one has + served a day. +4. For data recovery only (one deleted event, for example), copy the rows + from the restored instance instead of switching over. + +Rehearse before launch: restore staging to a time a few minutes ago and run +steps 1–2. Locally, `npm run db:restore-rehearsal` dumps the compose +database, restores it into a scratch database, verifies it and removes it, +timing each step. + +Targets: data loss at most 5 minutes (RDS uploads transaction logs every 5 +minutes); a restore and switch-over within 1 hour. + +## Rotate secrets + +All runtime secrets live in Secrets Manager (`match-maker-prod/app`, +`/identity`, `/migrate`). ECS reads them when a task starts, so every +rotation ends with a new deployment: + +```sh +aws ecs update-service --cluster match-maker-prod-cluster --service match-maker-prod-web --force-new-deployment +aws ecs update-service --cluster match-maker-prod-cluster --service match-maker-prod-worker --force-new-deployment +``` + +| Secret | How | Effect | +| --- | --- | --- | +| Session secret | `terraform apply -replace=random_password.session_secret_b -var session_secret_active=b`, then redeploy web. Next time replace `random_password.session_secret` and set `a`. | Nobody is signed out: cookies sealed with the old secret are still read (`SESSION_SECRET_PREVIOUS`) until they expire (7 days). Do not rotate again within those 7 days. | +| OTP pepper | `terraform apply -replace=random_password.otp_pepper`, redeploy web. | Codes e-mailed in the last 10 minutes stop working; people request a new one. | +| Database runtime password | `terraform apply -replace=random_password.db_runtime`, run the migrate task (sets the new password), redeploy web and worker. | Old tasks keep their open connections until replaced. | +| Keycloak client secret | `terraform apply -replace=random_password.keycloak_client_secret`, then `npm run keycloak:sync` against production Keycloak from an address in `keycloak_admin_cidrs`, then redeploy web. | Google sign-in fails between the apply and the sync; do it in a quiet hour. | +| Gemini, Resend, Upstash, Google keys | Create the new key at the provider, update the value in `match-maker-prod/app` (and `/identity` for Google), redeploy, then revoke the old key. | None if the old key is revoked last. | + +## Running tools against production + +The database is private, so tools run as one-off ECS tasks in the private +subnets with the command overridden: + +- `jobs:failed`, `jobs:retry`, `models:usage`, `db:verify`: the migrate task + definition (it has the database secrets), e.g. command + `npx tsx tools/manage-jobs.ts failed`. +- `account:*` and `keycloak:sync`: the worker task definition (it also has + the auth package), adding `KEYCLOAK_INTERNAL_ISSUER`, `KEYCLOAK_ADMIN` and + `KEYCLOAK_ADMIN_PASSWORD` (from the `identity` secret) to the overrides. + +## Accounts and data requests + +- Block a person everywhere: `npm run account:disable -- ` + (`account:enable` undoes it); sign out everywhere only: + `npm run account:revoke-sessions -- `. +- Erase a person: `npm run account:erase -- --yes`. This deletes + their profiles, matches, introductions, cached explanations and Keycloak + user, and records it in the audit log. + +## Launch checklist + +- [ ] Production Terraform applied with the domain, certificate, `alarm_email`, + `keycloak_admin_cidrs` and `image_promotion_source`; SNS e-mail confirmed. +- [ ] Real secrets filled in: Gemini, Resend (sending domain verified), Google + OAuth client (redirect URI on the Keycloak domain), Upstash if used. +- [ ] `npm run keycloak:sync` run once against production Keycloak. +- [ ] GitHub environments `production` and `staging` hold their AWS role, + subnets, security group and `APP_BASE_URL`; `staging` has + `STACK_NAME=match-maker-staging`. +- [ ] Staging journey walked with real e-mail and Google sign-in. +- [ ] Capacity check passed on staging with `--providers configured` at the + launch size. +- [ ] Rollback rehearsed on staging. +- [ ] Database restore rehearsed on staging and `db:verify` passed. +- [ ] Every alarm in this file seen once in the console, and the dashboard opened. +- [ ] Organizer accounts created (`npm run org:init`) and the first event set up. diff --git a/infra/docker/worker.Dockerfile b/infra/docker/worker.Dockerfile index fb4e9eb..98b6f14 100644 --- a/infra/docker/worker.Dockerfile +++ b/infra/docker/worker.Dockerfile @@ -15,6 +15,8 @@ COPY apps/web/src apps/web/src COPY packages/db packages/db COPY packages/auth packages/auth COPY packages/email packages/email +# Operator tools (account:*, keycloak:sync) run as one-off tasks from this image. +COPY tools tools # Amazon's RDS CA bundle, so DATABASE_SSL=verify can check the database # certificate (DATABASE_CA_CERT_FILE points here in the task definitions). From 460b61a7172b10cb4d11f8336420c90132950579 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 10:56:48 +0530 Subject: [PATCH 11/11] bug fixx-tsconfig-baseurl-deprecation TypeScript 6 (used by the editor) deprecates baseUrl and reported it on apps/web and packages/auth (inherited from tsconfig.base.json). Path aliases do not need it: the entries are now relative to the file that declares them. Type-checks clean with TypeScript 5.7 and 6.0. --- apps/web/tsconfig.json | 1 - tsconfig.base.json | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index a53f493..d49377e 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -1,7 +1,6 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "baseUrl": ".", "plugins": [{ "name": "next" }], "types": ["node"], "incremental": true, diff --git a/tsconfig.base.json b/tsconfig.base.json index 10c8d14..faa8804 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -6,11 +6,10 @@ "module": "ESNext", "moduleResolution": "Bundler", "resolveJsonModule": true, - "baseUrl": ".", "paths": { - "@match-maker/db": ["packages/db/src/index.ts"], - "@match-maker/auth": ["packages/auth/src/index.ts"], - "@match-maker/email": ["packages/email/src/index.ts"] + "@match-maker/db": ["./packages/db/src/index.ts"], + "@match-maker/auth": ["./packages/auth/src/index.ts"], + "@match-maker/email": ["./packages/email/src/index.ts"] }, "allowJs": true, "checkJs": false,