diff --git a/.env.example b/.env.example index 78c0412..a4fe63f 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,12 @@ DEFAULT_EVENT_SLUG=demo # Encrypts the session cookie. REQUIRED, min 32 chars. # openssl rand -base64 48 SESSION_SECRET= +# While rotating: the old secret(s), comma-separated, still accepted for +# reading cookies for one session lifetime so nobody is signed out. +# SESSION_SECRET_PREVIOUS= +# Hashes login codes. Defaults to SESSION_SECRET; set it separately in +# production so rotating the session secret leaves codes in flight working. +# OTP_PEPPER= # ── Postgres ────────────────────────────────────────────────── POSTGRES_USER=matchmaker diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a282379..b1d8b60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + # The migration gate compares with the base commit. + fetch-depth: 0 - uses: actions/setup-node@v4 with: @@ -47,6 +50,12 @@ jobs: - name: Unit tests run: npx vitest run tests/unit + # Migrations run before the new code starts, and a rollback runs the + # previous image on the migrated schema: released migrations never + # change and new ones only expand. + - name: Migrations are safe to roll back from + run: npx tsx tools/check-migrations.ts "${{ github.event.pull_request.base.sha || github.event.before }}" + integration: name: Integration tests runs-on: ubuntu-latest @@ -101,6 +110,81 @@ jobs: if: failure() run: docker compose logs --no-color postgres keycloak | tail -n 300 + browser: + name: Browser acceptance + runs-on: ubuntu-latest + needs: [quality] + + # The built app and the worker against the compose Postgres and Keycloak. + # Login codes are logged instead of e-mailed and read from the app's log. + env: + DATABASE_URL: postgres://matchmaker:matchmaker@localhost:5432/matchmaker + DATABASE_SSL: disable + SESSION_SECRET: ci-session-secret-long-enough-for-validation-x + KEYCLOAK_ISSUER: http://localhost:8080/realms/match-maker + KEYCLOAK_PUBLIC_URL: http://localhost:8080 + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + KEYCLOAK_CLIENT_ID: match-maker-web + KEYCLOAK_CLIENT_SECRET: ci-client-secret + APP_BASE_URL: http://localhost:3000 + EMAIL_DEV_FALLBACK: "true" + MATCH_REFRESH_DELAY_SECONDS: "3" + E2E_BASE_URL: http://localhost:3000 + E2E_WEB_LOG: ${{ github.workspace }}/e2e-web.log + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + + - run: npm ci + + - name: Start Postgres and Keycloak + run: docker compose up -d --wait postgres keycloak + + - name: Migrate, seed and configure Keycloak + run: | + npm run db:migrate + npm run db:seed + npm run keycloak:wait + npm run keycloak:sync + + - name: Build + run: npm run build + + - name: Install Chromium + run: npx playwright install --with-deps chromium + + - name: Start app and worker + run: | + (cd apps/web && nohup npx next start -p 3000 > "$E2E_WEB_LOG" 2>&1 &) + nohup npm run worker > e2e-worker.log 2>&1 & + for i in $(seq 1 60); do + curl -fsS http://localhost:3000/api/health/ready && exit 0 + sleep 2 + done + cat "$E2E_WEB_LOG" + exit 1 + + - name: Browser tests + run: npm run test:e2e + + - name: Keep report and logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: browser-acceptance + path: | + playwright-report/ + test-results/ + e2e-web.log + e2e-worker.log + retention-days: 7 + build: name: Build images runs-on: ubuntu-latest diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 25ad537..6b8dffa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -5,6 +5,15 @@ on: branches: [main] workflow_dispatch: inputs: + environment: + description: "Where to deploy. Staging is a separate Terraform stack (environment = staging)." + type: choice + options: [production, staging] + default: production + promote_tag: + description: "Promote: copy this tag's staging images to production unchanged, then migrate and roll out (production only)" + type: string + default: "" deploy_keycloak: description: "Also rebuild and roll out Keycloak" type: boolean @@ -16,7 +25,7 @@ on: # Never let two deploys race for the same service. concurrency: - group: deploy-production + group: deploy-${{ inputs.environment || 'production' }} cancel-in-progress: false permissions: @@ -24,29 +33,59 @@ permissions: id-token: write contents: read +# Staging names follow the Terraform stack (match-maker-staging-*) and never +# fall back to production's repository variables. Each job that touches AWS +# runs in the GitHub environment of the target, so its role, subnets and URL +# come from that environment's secrets and variables. env: + TARGET: ${{ inputs.environment || 'production' }} + STACK: ${{ inputs.environment == 'staging' && 'match-maker-staging' || 'match-maker-prod' }} AWS_REGION: ${{ vars.AWS_REGION || 'ap-south-1' }} - ECS_CLUSTER: ${{ vars.ECS_CLUSTER || 'match-maker-prod-cluster' }} - ECS_WEB_SERVICE: ${{ vars.ECS_WEB_SERVICE || 'match-maker-prod-web' }} - ECS_WORKER_SERVICE: ${{ vars.ECS_WORKER_SERVICE || 'match-maker-prod-worker' }} - ECS_KEYCLOAK_SERVICE: ${{ vars.ECS_KEYCLOAK_SERVICE || 'match-maker-prod-keycloak' }} - MIGRATE_TASK_FAMILY: ${{ vars.MIGRATE_TASK_FAMILY || 'match-maker-prod-migrate' }} + ECS_CLUSTER: ${{ inputs.environment == 'staging' && 'match-maker-staging-cluster' || vars.ECS_CLUSTER || 'match-maker-prod-cluster' }} + ECS_WEB_SERVICE: ${{ inputs.environment == 'staging' && 'match-maker-staging-web' || vars.ECS_WEB_SERVICE || 'match-maker-prod-web' }} + ECS_WORKER_SERVICE: ${{ inputs.environment == 'staging' && 'match-maker-staging-worker' || vars.ECS_WORKER_SERVICE || 'match-maker-prod-worker' }} + ECS_KEYCLOAK_SERVICE: ${{ inputs.environment == 'staging' && 'match-maker-staging-keycloak' || vars.ECS_KEYCLOAK_SERVICE || 'match-maker-prod-keycloak' }} + MIGRATE_TASK_FAMILY: ${{ inputs.environment == 'staging' && 'match-maker-staging-migrate' || vars.MIGRATE_TASK_FAMILY || 'match-maker-prod-migrate' }} jobs: # Decides what ships: the pushed commit, or an existing tag on rollback. plan: name: Plan release runs-on: ubuntu-latest + environment: ${{ inputs.environment || 'production' }} outputs: tag: ${{ steps.plan.outputs.tag }} rollback: ${{ steps.plan.outputs.rollback }} + promote: ${{ steps.plan.outputs.promote }} steps: + # A staging run must find staging's own settings, never production's. + - name: Check the target environment + env: + STACK_NAME: ${{ vars.STACK_NAME }} + run: | + set -euo pipefail + if [ "$TARGET" = "staging" ] && [ "${STACK_NAME:-}" != "match-maker-staging" ]; then + echo "::error::Set STACK_NAME=match-maker-staging (and staging's AWS secrets) in the GitHub 'staging' environment first." + exit 1 + fi + - id: plan env: ROLLBACK_TAG: ${{ inputs.rollback_tag }} + PROMOTE_TAG: ${{ inputs.promote_tag }} run: | set -euo pipefail - if [ -n "${ROLLBACK_TAG:-}" ]; then + echo "promote=false" >> "$GITHUB_OUTPUT" + if [ -n "${PROMOTE_TAG:-}" ]; then + if [ "$TARGET" != "production" ] || [ -n "${ROLLBACK_TAG:-}" ] || ! [[ "$PROMOTE_TAG" =~ ^[0-9a-f]{7,40}$ ]]; then + echo "::error::promote_tag takes an image tag, targets production, and cannot be combined with rollback_tag." + exit 1 + fi + echo "tag=$PROMOTE_TAG" >> "$GITHUB_OUTPUT" + echo "rollback=false" >> "$GITHUB_OUTPUT" + echo "promote=true" >> "$GITHUB_OUTPUT" + echo "Promoting the staging images tagged $PROMOTE_TAG" + elif [ -n "${ROLLBACK_TAG:-}" ]; then if ! [[ "$ROLLBACK_TAG" =~ ^[0-9a-f]{7,40}$ ]]; then echo "::error::rollback_tag must be an image tag (a commit SHA prefix), got '$ROLLBACK_TAG'." exit 1 @@ -67,7 +106,8 @@ jobs: ci: name: Required checks needs: [plan] - if: needs.plan.outputs.rollback != 'true' + # A promoted tag passed these checks when it was built for staging. + if: needs.plan.outputs.rollback != 'true' && needs.plan.outputs.promote != 'true' uses: ./.github/workflows/ci.yml with: build_images: false @@ -76,6 +116,10 @@ jobs: name: Build & push images runs-on: ubuntu-latest needs: [plan, ci] + if: >- + !cancelled() && needs.plan.result == 'success' && needs.plan.outputs.rollback != 'true' && + (needs.plan.outputs.promote == 'true' || needs.ci.result == 'success') + environment: ${{ inputs.environment || 'production' }} env: TAG: ${{ needs.plan.outputs.tag }} steps: @@ -91,7 +135,24 @@ jobs: - uses: docker/setup-buildx-action@v3 + # The same image digests that passed staging; nothing is rebuilt. Needs + # both stacks' repositories in one AWS account and registry. + - name: Copy the staging images + if: needs.plan.outputs.promote == 'true' + env: + REGISTRY: ${{ steps.ecr.outputs.registry }} + run: | + set -euo pipefail + repos="web migrate worker" + if [ "${{ inputs.deploy_keycloak }}" = "true" ]; then repos="$repos keycloak"; fi + for repo in $repos; do + docker buildx imagetools create \ + --tag "$REGISTRY/$STACK/$repo:$TAG" --tag "$REGISTRY/$STACK/$repo:latest" \ + "$REGISTRY/match-maker-staging/$repo:$TAG" + done + - name: Build & push web + if: needs.plan.outputs.promote != 'true' uses: docker/build-push-action@v6 with: context: . @@ -99,24 +160,26 @@ jobs: push: true build-args: APP_VERSION=${{ env.TAG }} tags: | - ${{ steps.ecr.outputs.registry }}/match-maker-prod/web:${{ env.TAG }} - ${{ steps.ecr.outputs.registry }}/match-maker-prod/web:latest + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/web:${{ env.TAG }} + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/web:latest cache-from: type=gha cache-to: type=gha,mode=max - name: Build & push migrate + if: needs.plan.outputs.promote != 'true' uses: docker/build-push-action@v6 with: context: . file: infra/docker/migrate.Dockerfile push: true tags: | - ${{ steps.ecr.outputs.registry }}/match-maker-prod/migrate:${{ env.TAG }} - ${{ steps.ecr.outputs.registry }}/match-maker-prod/migrate:latest + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/migrate:${{ env.TAG }} + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/migrate:latest cache-from: type=gha cache-to: type=gha,mode=max - name: Build & push worker + if: needs.plan.outputs.promote != 'true' uses: docker/build-push-action@v6 with: context: . @@ -124,21 +187,21 @@ jobs: push: true build-args: APP_VERSION=${{ env.TAG }} tags: | - ${{ steps.ecr.outputs.registry }}/match-maker-prod/worker:${{ env.TAG }} - ${{ steps.ecr.outputs.registry }}/match-maker-prod/worker:latest + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/worker:${{ env.TAG }} + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/worker:latest cache-from: type=gha cache-to: type=gha,mode=max - name: Build & push Keycloak - if: inputs.deploy_keycloak + if: inputs.deploy_keycloak && needs.plan.outputs.promote != 'true' uses: docker/build-push-action@v6 with: context: . file: infra/docker/keycloak.Dockerfile push: true tags: | - ${{ steps.ecr.outputs.registry }}/match-maker-prod/keycloak:${{ env.TAG }} - ${{ steps.ecr.outputs.registry }}/match-maker-prod/keycloak:latest + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/keycloak:${{ env.TAG }} + ${{ steps.ecr.outputs.registry }}/${{ env.STACK }}/keycloak:latest cache-from: type=gha cache-to: type=gha,mode=max @@ -146,6 +209,9 @@ jobs: name: Run migrations runs-on: ubuntu-latest needs: [plan, build-and-push] + if: >- + !cancelled() && needs.plan.outputs.rollback != 'true' && needs.build-and-push.result == 'success' + environment: ${{ inputs.environment || 'production' }} env: TAG: ${{ needs.plan.outputs.tag }} steps: @@ -164,7 +230,7 @@ jobs: REGISTRY="$(aws sts get-caller-identity --query Account --output text).dkr.ecr.${AWS_REGION}.amazonaws.com" aws ecs describe-task-definition --task-definition "$MIGRATE_TASK_FAMILY" \ --query 'taskDefinition' > migrate-taskdef.json - jq --arg IMAGE "$REGISTRY/match-maker-prod/migrate:$TAG" ' + jq --arg IMAGE "$REGISTRY/$STACK/migrate:$TAG" ' .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt) @@ -198,7 +264,7 @@ jobs: echo "Exit code: $EXIT_CODE ($REASON)" if [ "$EXIT_CODE" != "0" ]; then - echo "::error::Migration failed — aborting deploy. Check /ecs/match-maker-prod/migrate." + echo "::error::Migration failed — aborting deploy. Check /ecs/$STACK/migrate." exit 1 fi @@ -206,6 +272,7 @@ jobs: name: Roll out runs-on: ubuntu-latest needs: [plan, build-and-push, migrate] + environment: ${{ inputs.environment || 'production' }} # A rollback skips checks, build and migrations (the schema only moves # forward); a normal release needs all of them to have succeeded. if: >- @@ -228,7 +295,7 @@ jobs: repos="web worker" if [ "${{ inputs.deploy_keycloak }}" = "true" ]; then repos="$repos keycloak"; fi for repo in $repos; do - aws ecr describe-images --repository-name "match-maker-prod/$repo" \ + aws ecr describe-images --repository-name "$STACK/$repo" \ --image-ids imageTag="$TAG" >/dev/null \ || { echo "::error::No $repo image tagged $TAG in ECR."; exit 1; } done @@ -272,7 +339,7 @@ jobs: # Strip the server-managed fields that RegisterTaskDefinition rejects, # then point the container at the immutable tag. - jq --arg IMAGE "$REGISTRY/match-maker-prod/web:$TAG" ' + jq --arg IMAGE "$REGISTRY/$STACK/web:$TAG" ' .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt) @@ -304,7 +371,7 @@ jobs: aws ecs describe-task-definition --task-definition "$FAMILY" \ --query 'taskDefinition' > worker-taskdef.json - jq --arg IMAGE "$REGISTRY/match-maker-prod/worker:$TAG" ' + jq --arg IMAGE "$REGISTRY/$STACK/worker:$TAG" ' .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt) @@ -335,7 +402,7 @@ jobs: --query 'services[0].taskDefinition' --output text | sed -E 's#^.*task-definition/([^:]+):[0-9]+$#\1#') aws ecs describe-task-definition --task-definition "$FAMILY" \ --query 'taskDefinition' > keycloak-taskdef.json - jq --arg IMAGE "$REGISTRY/match-maker-prod/keycloak:$TAG" ' + jq --arg IMAGE "$REGISTRY/$STACK/keycloak:$TAG" ' .containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt) diff --git a/.gitignore b/.gitignore index d1464f3..88b2c56 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,8 @@ infra/terraform/*.tfstate infra/terraform/*.tfstate.* infra/terraform/*.tfvars !infra/terraform/*.tfvars.example +infra/terraform/environments/*.tfvars +!infra/terraform/environments/*.tfvars.example infra/terraform/*.tfplan infra/terraform/.terraform.lock.hcl infra/terraform/override.tf diff --git a/ReadMe.md b/ReadMe.md index b51a594..6a6b387 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -4,7 +4,7 @@ Mr. Bridge is a multi event networking and matchmaking app. Organizers create ev The implementation follows the architecture described in [docs/PROJECT_STRUCTURE.md](docs/PROJECT_STRUCTURE.md). The [end-to-end flow](docs/END_TO_END_FLOW.md) and [matching pipeline](docs/BRIDGE_ENGINE_PIPELINE.md) docs show the product journeys. -See the [production roadmap](docs/PRODUCTION_ROADMAP.md) for authentication findings, the invited-account signup policy, Google setup, and remaining work ordered by implementation effort. +See the [production roadmap](docs/PRODUCTION_ROADMAP.md) for authentication findings, the invited-account signup policy, Google setup, and remaining work ordered by implementation effort. The [operations runbook](docs/OPERATIONS.md) covers releases, staging, rollback, alarms, database restore, secret rotation and the launch checklist. ## Architecture @@ -103,7 +103,10 @@ The first command imports the global attendee directory so the organizer account | `npm run typecheck` | Type-check all workspaces | | `npm run lint` | Run ESLint | | `npm test` | Run unit and integration suites. Integration suites skip without Postgres/Keycloak; set `REQUIRE_SERVICES=db,keycloak` to fail instead (CI does, and `tools/check-test-report.ts` fails on any skipped test). Stop a locally running worker first, or it claims the pipeline test's jobs | +| `npm run test:e2e` | Browser acceptance suite (Playwright) against a running app and worker started with `EMAIL_DEV_FALLBACK=true`; set `E2E_BASE_URL` and `E2E_WEB_LOG` (the app's output, where login codes are read) | | `npm run db:migrate` | Apply pending PostgreSQL migrations | +| `npm run capacity:check` | Measure matching throughput, search latency and model calls for an event size against the budgets in the runbook | +| `npm run db:verify` / `db:restore-rehearsal` | Check a database (or a restored copy) is usable; rehearse dump and restore locally | | `npm run db:seed` | Upsert demo roster, organization, event, and organizer | | `npm run db:import -- [event-slug]` | Import attendee rows and optionally attach them to an event | | `npm run keycloak:wait` | Wait for the configured Keycloak realm to respond | diff --git a/apps/web/src/application/ports/metrics.ts b/apps/web/src/application/ports/metrics.ts new file mode 100644 index 0000000..34f8f9d --- /dev/null +++ b/apps/web/src/application/ports/metrics.ts @@ -0,0 +1,6 @@ +export type MetricUnit = "Count" | "Milliseconds" | "Seconds"; + +/** Operational measurements for dashboards and alarms. Never blocks or fails the caller. */ +export interface Metrics { + record(values: Array<{ name: string; value: number; unit: MetricUnit }>, dimensions?: Record): void; +} diff --git a/apps/web/src/application/ports/participant-repository.ts b/apps/web/src/application/ports/participant-repository.ts index c665385..87b5dcd 100644 --- a/apps/web/src/application/ports/participant-repository.ts +++ b/apps/web/src/application/ports/participant-repository.ts @@ -12,6 +12,8 @@ export interface ParticipantRepository { /** Without an embedding, only candidates matching one of these words, best first. */ terms?: string[]; filters?: SearchFilters; + /** Only people with one of these intents (the viewer's welcomed intents); null for all. */ + welcomedIntents?: string[] | null; /** The model behind `embedding`; vectors from other models are not ranked against it. */ embeddingModel?: string | null; limit?: number; diff --git a/apps/web/src/application/use-cases/search-participants.ts b/apps/web/src/application/use-cases/search-participants.ts index 3e66fde..504fdc0 100644 --- a/apps/web/src/application/use-cases/search-participants.ts +++ b/apps/web/src/application/use-cases/search-participants.ts @@ -2,11 +2,12 @@ import { MatchingPipeline } from "../../domain/matching/pipeline"; import { normalizeFilters } from "../../domain/matching/search-query"; import type { ParticipantRepository } from "../ports/participant-repository"; import type { RateLimiter } from "../ports/rate-limiter"; +import type { Metrics } from "../ports/metrics"; import { boundedText, LIMITS } from "../../domain/shared/validation"; export async function searchParticipants( input: { eventId: string; participantId: string; query: string; eventIntents: string[]; filters?: { intent?: string | null; domain?: string | null }; limit?: number }, - dependencies: { pipeline: MatchingPipeline; participants: ParticipantRepository; limiter?: RateLimiter } + dependencies: { pipeline: MatchingPipeline; participants: ParticipantRepository; limiter?: RateLimiter; metrics?: Metrics } ) { if (dependencies.limiter && !(await dependencies.limiter.consume(`search:${input.participantId}`, 30, 60))) { throw new Error("Too many searches. Wait a moment and try again."); @@ -16,5 +17,17 @@ export async function searchParticipants( intents: input.eventIntents, domains: input.filters?.domain ? await dependencies.participants.listDomains(input.eventId) : [], }); - return dependencies.pipeline.run({ eventId: input.eventId, participantId: input.participantId, query, filters, limit: input.limit, mode: "search" }); + const started = Date.now(); + let failed = 0; + try { + return await dependencies.pipeline.run({ eventId: input.eventId, participantId: input.participantId, query, filters, limit: input.limit, mode: "search" }); + } catch (error) { + failed = 1; + throw error; + } finally { + dependencies.metrics?.record([ + { name: "SearchLatencyMs", value: Date.now() - started, unit: "Milliseconds" }, + { name: "SearchFailures", value: failed, unit: "Count" }, + ]); + } } diff --git a/apps/web/src/composition/container.ts b/apps/web/src/composition/container.ts index ea938a1..5319bc2 100644 --- a/apps/web/src/composition/container.ts +++ b/apps/web/src/composition/container.ts @@ -14,6 +14,7 @@ import { RealtimeNotifier } from "../infrastructure/notifications/realtime.notif import { ProviderRegistry } from "../infrastructure/providers/provider-registry"; import { GatewayEmbeddingProvider, GatewayLlmProvider } from "../infrastructure/providers/model-gateway"; import { PostgresModelUsageLog } from "../infrastructure/postgres/model-usage.repo"; +import { EmfMetrics, MeteredModelUsageLog } from "../infrastructure/observability/emf-metrics"; import { MatchingPipeline } from "../domain/matching/pipeline"; const globalContainer = globalThis as typeof globalThis & { __matchMakerContainer?: ReturnType }; @@ -23,7 +24,8 @@ function buildContainer() { const matches = new PostgresMatchRepository(); const intros = new PostgresIntroRepository(); const events = new PostgresEventRepository(); - const usage = new PostgresModelUsageLog(); + const metrics = new EmfMetrics(); + const usage = new MeteredModelUsageLog(new PostgresModelUsageLog(), metrics); const embeddings = new GatewayEmbeddingProvider(ProviderRegistry.embedding(), usage); const rawLlm = ProviderRegistry.llm(); const llm = rawLlm ? new GatewayLlmProvider(rawLlm, usage) : null; @@ -41,6 +43,7 @@ function buildContainer() { locations: new PostgresLocationRepository(), notifications: new RealtimeNotifier(), limiter: new PostgresRateLimiter(), + metrics, pipeline: new MatchingPipeline(participants, matches, embeddings, llm, candidateCache, new PostgresExplanationCache()), enqueueMatching: matchingJobs.enqueue, enqueueEmbedding: (eventId: string, participantId: string) => matchingJobs.enqueue(eventId, participantId, "embed"), diff --git a/apps/web/src/domain/matching/pipeline.ts b/apps/web/src/domain/matching/pipeline.ts index fec3485..8a2c764 100644 --- a/apps/web/src/domain/matching/pipeline.ts +++ b/apps/web/src/domain/matching/pipeline.ts @@ -1,4 +1,5 @@ import type { EmbeddingProvider } from "../../application/ports/embedding-provider"; +import { welcomedIntents } from "../participant/intent"; import type { CandidateCache } from "../../application/ports/candidate-cache"; import type { ExplanationCache } from "../../application/ports/explanation-cache"; import { queryContext } from "./explanation-key"; @@ -107,6 +108,7 @@ export class MatchingPipeline { viewerId: input.participantId, embedding, embeddingSpace: this.embeddings.id, + welcomedIntents: welcomedIntents(viewer.intent), limit: 50, }); const compatible = this.filter.execute(viewer, retrieved); @@ -151,6 +153,7 @@ export class MatchingPipeline { terms, filters: input.filters ?? NO_FILTERS, embeddingSpace: this.embeddings.id, + welcomedIntents: welcomedIntents(viewer.intent), limit: 50, }); const compatible = this.filter.execute(viewer, retrieved); diff --git a/apps/web/src/domain/matching/stages/retrieve.stage.ts b/apps/web/src/domain/matching/stages/retrieve.stage.ts index e7c6598..72d913a 100644 --- a/apps/web/src/domain/matching/stages/retrieve.stage.ts +++ b/apps/web/src/domain/matching/stages/retrieve.stage.ts @@ -7,7 +7,7 @@ import { matchesFilters, NO_FILTERS, type SearchFilters } from "../search-query" import { computeOnce } from "../../../application/shared-computation"; /** Bump when retrieval semantics change, so cached candidate lists are not reused. */ -const RETRIEVAL_VERSION = "r2"; +const RETRIEVAL_VERSION = "r3"; export class RetrieveStage { constructor( @@ -24,6 +24,12 @@ export class RetrieveStage { filters?: SearchFilters; /** Embedding provider/model; vectors from different spaces must not share entries. */ embeddingSpace?: string; + /** + * The viewer's welcomed intents, applied before the limit: filtering + * afterwards left people with no matches when their nearest neighbours + * all had intents they do not welcome. + */ + welcomedIntents?: string[] | null; limit?: number; }): Promise { const limit = input.limit ?? 50; @@ -34,7 +40,7 @@ export class RetrieveStage { // retrieval kind; the ranking that follows is still personal. const corpus = await this.participants.corpusRevision(input.eventId); const key = createHash("sha256").update(JSON.stringify([ - RETRIEVAL_VERSION, input.eventId, corpus, normalizedQuery, filters.intent, filters.domain?.toLowerCase() ?? null, + RETRIEVAL_VERSION, input.eventId, corpus, normalizedQuery, filters.intent, filters.domain?.toLowerCase() ?? null, input.welcomedIntents ?? null, input.embedding ? input.embeddingSpace ?? "semantic" : "lexical", limit, ])).digest("hex"); const ids = await computeOnce({ @@ -49,6 +55,7 @@ export class RetrieveStage { embeddingModel: input.embeddingSpace, terms: input.terms, filters, + welcomedIntents: input.welcomedIntents, limit, })).map((candidate) => candidate.attendeeId), }); @@ -64,6 +71,7 @@ export class RetrieveStage { excludeAttendeeId: input.viewerId, embedding: input.embedding, embeddingModel: input.embeddingSpace, + welcomedIntents: input.welcomedIntents, limit, }); } diff --git a/apps/web/src/domain/participant/intent.ts b/apps/web/src/domain/participant/intent.ts index ec07b33..a64f82e 100644 --- a/apps/web/src/domain/participant/intent.ts +++ b/apps/web/src/domain/participant/intent.ts @@ -23,6 +23,15 @@ export const INTENT_COMPATIBILITY: Record = { [Intent.INVESTING]: [Intent.RAISING_FUNDING, Intent.NETWORKING], }; +/** + * The intents a person welcomes, for narrowing retrieval before any top-K + * cut; null when they welcome everyone. + */ +export function welcomedIntents(intent: Intent): Intent[] | null { + const welcomed = INTENT_COMPATIBILITY[intent] ?? []; + return welcomed.length === Object.values(Intent).length ? null : [...welcomed].sort(); +} + export function isIntentCompatible(from: Intent, to: Intent): boolean { return INTENT_COMPATIBILITY[from]?.includes(to) ?? false; } diff --git a/apps/web/src/infrastructure/observability/emf-metrics.ts b/apps/web/src/infrastructure/observability/emf-metrics.ts new file mode 100644 index 0000000..7f5c552 --- /dev/null +++ b/apps/web/src/infrastructure/observability/emf-metrics.ts @@ -0,0 +1,48 @@ +import type { Metrics } from "../../application/ports/metrics"; +import type { ModelUsageLog } from "../../application/ports/model-usage"; + +/** + * Writes metrics as CloudWatch Embedded Metric Format: one JSON log line that + * CloudWatch Logs turns into metrics, so no agent or extra service is needed. + * Without METRICS_NAMESPACE (local runs) nothing is written. + */ +export class EmfMetrics implements Metrics { + constructor( + private readonly namespace: string | null = process.env.METRICS_NAMESPACE || null, + private readonly write: (line: string) => void = (line) => process.stdout.write(`${line}\n`) + ) {} + + record(values: Parameters[0], dimensions: Record = {}): void { + if (!this.namespace || !values.length) return; + try { + this.write(JSON.stringify({ + _aws: { + Timestamp: Date.now(), + CloudWatchMetrics: [{ + Namespace: this.namespace, + Dimensions: [Object.keys(dimensions)], + Metrics: values.map((metric) => ({ Name: metric.name, Unit: metric.unit })), + }], + }, + ...dimensions, + ...Object.fromEntries(values.map((metric) => [metric.name, metric.value])), + })); + } catch { + // Metrics are best effort. + } + } +} + +/** Adds call counts, failures and latency per operation to an existing usage log. */ +export class MeteredModelUsageLog implements ModelUsageLog { + constructor(private readonly inner: ModelUsageLog, private readonly metrics: Metrics) {} + + record(entry: Parameters[0]): void { + this.inner.record(entry); + this.metrics.record([ + { name: "ModelCalls", value: 1, unit: "Count" }, + { name: "ModelCallFailures", value: entry.outcome === "ok" ? 0 : 1, unit: "Count" }, + { name: "ModelCallLatencyMs", value: entry.latencyMs, unit: "Milliseconds" }, + ], { Operation: entry.operation }); + } +} diff --git a/apps/web/src/infrastructure/postgres/match.repo.ts b/apps/web/src/infrastructure/postgres/match.repo.ts index e8a8eea..8942530 100644 --- a/apps/web/src/infrastructure/postgres/match.repo.ts +++ b/apps/web/src/infrastructure/postgres/match.repo.ts @@ -30,8 +30,11 @@ export class PostgresMatchRepository implements MatchRepository { return transaction(async (client) => { if (expectedRevision !== undefined) { // Lock the profile row so a save cannot slip in between check and write. + // NO KEY UPDATE still blocks a profile save, but not the key-share locks + // other people's match inserts take on this row through their foreign + // keys; FOR UPDATE made two lists naming each other deadlock. const { rows } = await client.query<{ profile_revision: string }>( - `select profile_revision from event_participants where event_id = $1 and attendee_id = $2 for update`, [eventId, participantId]); + `select profile_revision from event_participants where event_id = $1 and attendee_id = $2 for no key update`, [eventId, participantId]); if (!rows[0] || Number(rows[0].profile_revision) !== expectedRevision) return false; } await client.query(`delete from event_matches where event_id = $1 and participant_id = $2`, [eventId, participantId]); diff --git a/apps/web/src/infrastructure/postgres/participant.repo.ts b/apps/web/src/infrastructure/postgres/participant.repo.ts index 301c4b2..913e440 100644 --- a/apps/web/src/infrastructure/postgres/participant.repo.ts +++ b/apps/web/src/infrastructure/postgres/participant.repo.ts @@ -97,6 +97,7 @@ export class PostgresParticipantRepository implements ParticipantRepository { embedding?: number[] | null; terms?: string[]; filters?: SearchFilters; + welcomedIntents?: string[] | null; /** The model that produced `embedding`; only candidates embedded by it are ranked by vector. */ embeddingModel?: string | null; limit?: number; @@ -110,13 +111,14 @@ export class PostgresParticipantRepository implements ParticipantRepository { and ($5::text is null or p.intent = $5) and ($6::text is null or lower(p.domain) = lower($6)) and ($7::text is null or ${SEARCH_DOCUMENT} @@ to_tsquery('simple', $7)) + and ($9::text[] is null or p.intent = any($9::text[])) order by case when $3::vector is not null and p.embedding is not null and p.embedding_model is not distinct from $8 then 0 else 1 end, case when p.embedding_model is not distinct from $8 then p.embedding <=> $3::vector end nulls last, case when $7::text is null then 0 else ts_rank(${SEARCH_DOCUMENT}, to_tsquery('simple', $7)) end desc, a.name asc limit $4`, [input.eventId, input.excludeAttendeeId, vector, Math.max(1, Math.min(100, input.limit ?? 50)), - input.filters?.intent ?? null, input.filters?.domain ?? null, tsQuery, input.embeddingModel ?? null] + input.filters?.intent ?? null, input.filters?.domain ?? null, tsQuery, input.embeddingModel ?? null, input.welcomedIntents ?? null] ); return rows.map(mapProfile); } diff --git a/apps/web/src/infrastructure/providers/hash-embedding.provider.ts b/apps/web/src/infrastructure/providers/hash-embedding.provider.ts new file mode 100644 index 0000000..902684a --- /dev/null +++ b/apps/web/src/infrastructure/providers/hash-embedding.provider.ts @@ -0,0 +1,23 @@ +import type { EmbeddingProvider } from "../../application/ports/embedding-provider"; + +const DIMENSIONS = 768; + +/** + * Words hashed into a unit vector: similar texts give similar vectors, with + * no network call and no cost. For capacity tests and staging rehearsals + * only (EMBEDDING_PROVIDER=hash); production refuses it. + */ +export class HashEmbeddingProvider implements EmbeddingProvider { + readonly id = "hash:768"; + + async embed(text: string): Promise { + const vector = new Array(DIMENSIONS).fill(0); + for (const word of text.toLowerCase().match(/[a-z0-9]+/g) ?? []) { + let hash = 0; + for (const char of word) hash = (hash * 31 + char.charCodeAt(0)) % DIMENSIONS; + vector[hash] += 1; + } + const norm = Math.hypot(...vector); + return norm ? vector.map((value) => value / norm) : null; + } +} diff --git a/apps/web/src/infrastructure/providers/provider-registry.ts b/apps/web/src/infrastructure/providers/provider-registry.ts index 28f1f5b..1cd4710 100644 --- a/apps/web/src/infrastructure/providers/provider-registry.ts +++ b/apps/web/src/infrastructure/providers/provider-registry.ts @@ -3,9 +3,11 @@ import type { LlmProvider } from "../../application/ports/llm-provider"; import { GeminiEmbeddingProvider } from "./gemini-embedding.provider"; import { GeminiLlmProvider } from "./gemini-llm.provider"; import { LocalBgeProvider } from "./local-bge.provider"; +import { HashEmbeddingProvider } from "./hash-embedding.provider"; export class ProviderRegistry { static embedding(): EmbeddingProvider { + if (process.env.EMBEDDING_PROVIDER === "hash") return new HashEmbeddingProvider(); if (process.env.EMBEDDING_PROVIDER === "local" && process.env.LOCAL_BGE_URL) { return new LocalBgeProvider(); } diff --git a/apps/web/src/lib/env.ts b/apps/web/src/lib/env.ts index 2a9f2b3..8308a4d 100644 --- a/apps/web/src/lib/env.ts +++ b/apps/web/src/lib/env.ts @@ -27,7 +27,7 @@ const POSITIVE_INTEGERS = [ const ALLOWED: Record = { DATABASE_SSL: ["disable", "require", "verify"], - EMBEDDING_PROVIDER: ["gemini", "local"], + EMBEDDING_PROVIDER: ["gemini", "local", "hash"], EMAIL_DEV_FALLBACK: ["true", "false"], TRUSTED_PROXY_HOPS: ["0", "1", "2", "3"], }; @@ -66,6 +66,9 @@ export function checkEnv(role: EnvRole, env: Env = process.env): string[] { // Production must not quietly fall back to unverified or plain transport. // Local hosts are exempt so the docker-compose stack still runs as production. if (value("NODE_ENV") === "production") { + if (value("EMBEDDING_PROVIDER") === "hash") { + problems.push("EMBEDDING_PROVIDER=hash is for capacity tests only and cannot run in production."); + } if (databaseUrl && !isLocalHost(databaseUrl)) { if (value("DATABASE_SSL") !== "verify") { problems.push("DATABASE_SSL must be verify in production, so the database certificate is checked."); diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json index a53f493..d49377e 100644 --- a/apps/web/tsconfig.json +++ b/apps/web/tsconfig.json @@ -1,7 +1,6 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "baseUrl": ".", "plugins": [{ "name": "next" }], "types": ["node"], "incremental": true, diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 0000000..54a2552 --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,270 @@ +# Operations runbook + +How Mr. Bridge is released, watched and recovered in production. Every alarm +names a section of this file. Commands run from the repository root unless a +step says otherwise. AWS names follow the Terraform stack: +`match-maker-prod-*` for production, `match-maker-staging-*` for staging. + +## Service targets + +The launch target is one event of **300 attendees** with up to **15 +concurrent searches** and two worker tasks. + +| Measure | Budget | Checked by | +| --- | --- | --- | +| Search latency, p95 | ≤ 6 s (the search time budget; reranking is skipped when time runs short) | `SearchLatencyMs` alarm, capacity check | +| Every attendee has a match list after the profiles are saved | ≤ 10 min | capacity check | +| Queue quiet (all refreshes done) | ≤ 30 min | capacity check | +| A due matching job or notification waits | ≤ 10 min | `OldestDue*` alarms | +| Model calls per attendee (profile embedding plus reranks) | ≤ 6 | capacity check, `npm run models:usage` | +| Model calls per search (query embedding; cached for a day) | ≤ 2 | capacity check | +| Attendees left without matches | 0 | capacity check | + +Measured on a developer machine (local Postgres, free hash embedding, no +rerank), these are the system's own costs, without model latency: + +| Size | First match lists | Queue quiet | Search p50 / p95 | +| --- | --- | --- | --- | +| 300 attendees, 60 searches, 15 at a time | 6 s | 8 s | 42 ms / 83 ms | +| 1000 attendees, 200 searches, 40 at a time | 25 s | 36 s | 102 ms / 169 ms | + +Repeat the check on staging with the real providers before launch and after +any change to matching (see *Release*). It costs model calls. + +```sh +npm run capacity:check -- --attendees 300 --searches 60 --concurrency 15 --workers 0 --providers configured +``` + +`--workers 0` lets the deployed workers do the matching; the check only +watches. With `--workers N` it runs its own workers, so stop any other +worker on that database first. Without `--providers configured` it uses a +free deterministic embedding, which measures the system and not the models. +The check deletes its throwaway event and people when it finishes. + +## Release + +1. **CI** runs on every pull request and again inside every deploy: + typecheck, lint, unit tests, the migration gate, integration tests + against real Postgres and Keycloak (a skipped test fails the job), the + browser journey against the built app and worker, the image builds and + `terraform validate`. +2. **Staging.** Run the *Deploy to ECS* workflow with `environment: + staging` on the commit to release. It builds, migrates staging, rolls + out and smoke-tests. Then sign in on staging with a real e-mail and + Google account, walk the journey once (create an event, import, two + attendees, match, search, intro, accept, venue question), and run the + capacity check above. +3. **Production.** Merge to `main`, which builds and deploys that commit; + or, to ship exactly the images staging tested, run the workflow with + `environment: production` and `promote_tag: `. Promotion + copies the images unchanged, then migrates production and rolls out. + It needs `image_promotion_source = "match-maker-staging"` in production's + Terraform variables. +4. The deploy waits for the services to settle, checks the new version is + serving and its dependencies are ready, and puts the previous revisions + back by itself if that fails. + +### Migrations and rollback + +Migrations run before the new code and the previous image must keep +working on the migrated schema. CI enforces it (`tools/check-migrations.ts`): +released migrations never change, and a new migration may not drop or +rename a table or column, change a column type or truncate. To remove +something, stop using it in one release and drop it in a later one, with +the line marked `-- contract: `. + +### Roll back a release + +Run *Deploy to ECS* with `rollback_tag: ` (the 12-character +commit prefix shown in earlier runs and in `/api/health`). It skips checks, +build and migrations and redeploys those images. Because migrations only +expand, the old code runs on the current schema. Confirm `/api/health` +reports the old version and watch the alarms for ten minutes. + +Rehearse it on staging before launch: deploy a commit, roll back to the +previous tag, confirm sign-in and search still work, deploy forward again. + +## Alarms + +Alarms go to the SNS topic `match-maker-prod-alarms` (`alarm_email` in the +Terraform variables subscribes an address). The dashboard +`match-maker-prod` shows search latency, queue waits and sizes, failures, +model call failures and web response time. + +### Site down (`web-no-healthy-hosts`, `alb-5xx`, `alb-p95-latency`) + +1. ECS console → `match-maker-prod-web` → *Events* and stopped tasks: a + failing health check, an out-of-memory stop or a configuration error + (the app refuses to start on bad configuration and logs every problem). +2. If it began with a release, roll back (above). +3. If the database is the cause (`/api/health/ready` reports it), see the + database alarms. + +### Keycloak down (`keycloak-no-healthy-hosts`) + +Nobody can sign in (code sign-in also registers the person in Keycloak); +people already signed in carry on. Check the `match-maker-prod-keycloak` +service events and its logs in `/ecs/match-maker-prod/keycloak`; its +database login is in the `identity` secret. + +### Worker down (`worker-down`) + +No heartbeat means matching and notifications have stopped. Check the +`match-maker-prod-worker` service: a crash loop shows in its stopped tasks +and `/ecs/match-maker-prod/worker`. Queued work is durable; when the worker +returns it continues, and jobs held by a dead worker are reclaimed once +their lease ends (5 minutes). + +### Matching backlog (`matching-backlog`) + +Work is arriving faster than one worker handles it, or jobs are slow +because a model provider is slow. Check `ModelCallLatencyMs` and the model +failure alarm. To add capacity, raise `worker_desired_count` and apply, or +scale the service in the ECS console for the event day. Workers share the +queue safely. + +### Failed jobs (`jobs-failed`) + +A job failed five times. Run `npm run jobs:failed` against production (see +*Running tools against production*) to see each error. Fix the cause, then +`npm run jobs:retry -- all`. + +### Notifications stuck (`notification-backlog`, `notifications-failed`) + +Usually e-mail delivery. Check the worker log for `[worker:notify]` errors +and the Resend dashboard (key revoked, domain unverified, quota). Failed +notifications keep their last error in `notification_outbox`; once fixed, +set them back with +`update notification_outbox set status = 'pending', attempts = 0, available_at = now() where status = 'failed'`. +Each is sent at most once per subject, so retrying cannot duplicate. + +### Slow search (`search-slow`, `search-failing`) + +Search waits at most 2.5 s for the query embedding and skips reranking +when time runs short, so sustained slowness is usually the database. Check +RDS CPU and connections, then the model failure alarm, then errors in +`/ecs/match-maker-prod/web`. + +### Model provider failing (`model-failures`) + +Over 20% of Gemini calls fail. Matching keeps working on fallbacks: lexical +search, scored order with generated reasons. `npm run models:usage` shows +failures by operation. Common causes are an expired or revoked key (rotate +it, below) or an exhausted quota (raise it in Google AI Studio). The +gateway's circuit breaker stops calling a failing provider for 30 s at a +time. + +### Database (`rds-cpu`, `rds-free-storage`, `rds-connections`) + +CPU: look for a runaway query in Performance Insights; the worker can be +scaled down to shed load. Storage: raise `db_max_allocated_storage` (storage +grows by itself up to it) and apply. Connections: each task opens at most 10; a climb +without more tasks points at a leak in a recent release, so roll back. + +### Login e-mail failures (`login-email-failures`) + +People cannot sign in by code. Check the Resend dashboard and the +`RESEND_API_KEY` secret; rotate the key if needed (below). + +### Web at maximum capacity (`web-at-max-capacity`) + +Raise `web_max_capacity` and apply; check the database can take the extra +connections (10 per task). + +## Restore the database + +RDS keeps automated backups for `db_backup_retention_days` days and allows +point-in-time restore to any second in that window. + +1. Restore to a **new** instance; never overwrite the live one: + ```sh + aws rds restore-db-instance-to-point-in-time \ + --source-db-instance-identifier match-maker-prod-postgres \ + --target-db-instance-identifier match-maker-prod-restore-$(date +%Y%m%d%H%M) \ + --restore-time 2026-10-19T09:30:00Z \ + --db-subnet-group-name match-maker-prod-db-subnet-group --vpc-security-group-ids \ + --db-parameter-group-name match-maker-prod-pg16 \ + --no-publicly-accessible + aws rds wait db-instance-available --db-instance-identifier + ``` +2. Verify it from a task or host inside the VPC (the database is private): + ```sh + npm run db:verify -- --url "postgres://:@:5432/matchmaker" --compare "$DATABASE_URL" + ``` + It checks pgvector, that every migration of this release is applied + unchanged, the core tables, and lists row count differences against the + live database. +3. To switch over, stop the worker (desired count 0), point the + `DATABASE_URL` values in the `app` and `migrate` secrets at the restored + endpoint, run the migrate task (it also sets the runtime login's + password), force a new deployment of web and worker, and confirm + `/api/health/ready`. Keep the old instance until the restored one has + served a day. +4. For data recovery only (one deleted event, for example), copy the rows + from the restored instance instead of switching over. + +Rehearse before launch: restore staging to a time a few minutes ago and run +steps 1–2. Locally, `npm run db:restore-rehearsal` dumps the compose +database, restores it into a scratch database, verifies it and removes it, +timing each step. + +Targets: data loss at most 5 minutes (RDS uploads transaction logs every 5 +minutes); a restore and switch-over within 1 hour. + +## Rotate secrets + +All runtime secrets live in Secrets Manager (`match-maker-prod/app`, +`/identity`, `/migrate`). ECS reads them when a task starts, so every +rotation ends with a new deployment: + +```sh +aws ecs update-service --cluster match-maker-prod-cluster --service match-maker-prod-web --force-new-deployment +aws ecs update-service --cluster match-maker-prod-cluster --service match-maker-prod-worker --force-new-deployment +``` + +| Secret | How | Effect | +| --- | --- | --- | +| Session secret | `terraform apply -replace=random_password.session_secret_b -var session_secret_active=b`, then redeploy web. Next time replace `random_password.session_secret` and set `a`. | Nobody is signed out: cookies sealed with the old secret are still read (`SESSION_SECRET_PREVIOUS`) until they expire (7 days). Do not rotate again within those 7 days. | +| OTP pepper | `terraform apply -replace=random_password.otp_pepper`, redeploy web. | Codes e-mailed in the last 10 minutes stop working; people request a new one. | +| Database runtime password | `terraform apply -replace=random_password.db_runtime`, run the migrate task (sets the new password), redeploy web and worker. | Old tasks keep their open connections until replaced. | +| Keycloak client secret | `terraform apply -replace=random_password.keycloak_client_secret`, then `npm run keycloak:sync` against production Keycloak from an address in `keycloak_admin_cidrs`, then redeploy web. | Google sign-in fails between the apply and the sync; do it in a quiet hour. | +| Gemini, Resend, Upstash, Google keys | Create the new key at the provider, update the value in `match-maker-prod/app` (and `/identity` for Google), redeploy, then revoke the old key. | None if the old key is revoked last. | + +## Running tools against production + +The database is private, so tools run as one-off ECS tasks in the private +subnets with the command overridden: + +- `jobs:failed`, `jobs:retry`, `models:usage`, `db:verify`: the migrate task + definition (it has the database secrets), e.g. command + `npx tsx tools/manage-jobs.ts failed`. +- `account:*` and `keycloak:sync`: the worker task definition (it also has + the auth package), adding `KEYCLOAK_INTERNAL_ISSUER`, `KEYCLOAK_ADMIN` and + `KEYCLOAK_ADMIN_PASSWORD` (from the `identity` secret) to the overrides. + +## Accounts and data requests + +- Block a person everywhere: `npm run account:disable -- ` + (`account:enable` undoes it); sign out everywhere only: + `npm run account:revoke-sessions -- `. +- Erase a person: `npm run account:erase -- --yes`. This deletes + their profiles, matches, introductions, cached explanations and Keycloak + user, and records it in the audit log. + +## Launch checklist + +- [ ] Production Terraform applied with the domain, certificate, `alarm_email`, + `keycloak_admin_cidrs` and `image_promotion_source`; SNS e-mail confirmed. +- [ ] Real secrets filled in: Gemini, Resend (sending domain verified), Google + OAuth client (redirect URI on the Keycloak domain), Upstash if used. +- [ ] `npm run keycloak:sync` run once against production Keycloak. +- [ ] GitHub environments `production` and `staging` hold their AWS role, + subnets, security group and `APP_BASE_URL`; `staging` has + `STACK_NAME=match-maker-staging`. +- [ ] Staging journey walked with real e-mail and Google sign-in. +- [ ] Capacity check passed on staging with `--providers configured` at the + launch size. +- [ ] Rollback rehearsed on staging. +- [ ] Database restore rehearsed on staging and `db:verify` passed. +- [ ] Every alarm in this file seen once in the console, and the dashboard opened. +- [ ] Organizer accounts created (`npm run org:init`) and the first event set up. diff --git a/infra/docker/worker.Dockerfile b/infra/docker/worker.Dockerfile index fb4e9eb..98b6f14 100644 --- a/infra/docker/worker.Dockerfile +++ b/infra/docker/worker.Dockerfile @@ -15,6 +15,8 @@ COPY apps/web/src apps/web/src COPY packages/db packages/db COPY packages/auth packages/auth COPY packages/email packages/email +# Operator tools (account:*, keycloak:sync) run as one-off tasks from this image. +COPY tools tools # Amazon's RDS CA bundle, so DATABASE_SSL=verify can check the database # certificate (DATABASE_CA_CERT_FILE points here in the task definitions). diff --git a/infra/terraform/ecs.tf b/infra/terraform/ecs.tf index f2b8095..70dbd22 100644 --- a/infra/terraform/ecs.tf +++ b/infra/terraform/ecs.tf @@ -211,6 +211,7 @@ resource "aws_ecs_task_definition" "web" { environment = [ { name = "NODE_ENV", value = "production" }, { name = "PORT", value = "3000" }, + { name = "METRICS_NAMESPACE", value = local.metrics_namespace }, { name = "APP_BASE_URL", value = local.app_base_url }, { name = "DEFAULT_EVENT_SLUG", value = var.default_event_slug }, # One ALB appends the client address to X-Forwarded-For. @@ -230,6 +231,8 @@ resource "aws_ecs_task_definition" "web" { secrets = [ { name = "DATABASE_URL", valueFrom = local.secret_ref.DATABASE_URL }, { name = "SESSION_SECRET", valueFrom = local.secret_ref.SESSION_SECRET }, + { name = "SESSION_SECRET_PREVIOUS", valueFrom = local.secret_ref.SESSION_SECRET_PREVIOUS }, + { name = "OTP_PEPPER", valueFrom = local.secret_ref.OTP_PEPPER }, { name = "KEYCLOAK_CLIENT_SECRET", valueFrom = local.secret_ref.KEYCLOAK_CLIENT_SECRET }, { name = "RESEND_API_KEY", valueFrom = local.secret_ref.RESEND_API_KEY }, # Only its presence is read: it switches the Google sign-in option on. @@ -477,6 +480,7 @@ resource "aws_ecs_task_definition" "worker" { essential = true environment = [ { name = "NODE_ENV", value = "production" }, + { name = "METRICS_NAMESPACE", value = local.metrics_namespace }, { name = "DATABASE_SSL", value = "verify" }, { name = "DATABASE_CA_CERT_FILE", value = local.database_ca_file }, { name = "GEMINI_LLM_MODEL", value = "gemini-3.8-flash" }, diff --git a/infra/terraform/environments/staging.tfvars.example b/infra/terraform/environments/staging.tfvars.example new file mode 100644 index 0000000..cb6c6b4 --- /dev/null +++ b/infra/terraform/environments/staging.tfvars.example @@ -0,0 +1,54 @@ +# Staging: a separate stack (match-maker-staging-*) with its own state, in the +# same AWS account and region as production so tested images can be promoted +# unchanged. Copy to staging.tfvars (gitignored) and run, from infra/terraform: +# +# terraform init -reconfigure -backend-config="bucket=" \ +# -backend-config="key=match-maker/staging/terraform.tfstate" -backend-config="region=ap-south-1" +# terraform apply -var-file=environments/staging.tfvars +# +# Switch back to production with the same init and key=match-maker/terraform.tfstate. + +aws_region = "ap-south-1" +project_name = "match-maker" +environment = "staging" +default_event_slug = "demo" + +# Staging may be HTTP-only for a first evaluation; give it its own hostname +# and certificate to rehearse the production login flow (Google needs HTTPS). +domain_name = "" +keycloak_domain_name = "" +acm_certificate_arn = "" +route53_zone_id = "" +keycloak_admin_cidrs = [] + +# Production's stack creates the account's GitHub OIDC provider; reuse it. +github_repository = "LucknowAI/Bridge" +github_oidc_provider_arn = "arn:aws:iam:::oidc-provider/token.actions.githubusercontent.com" + +# Small but production-shaped: two web tasks so rolling deploys and the +# shared caches behave as in production. +web_desired_count = 2 +web_min_capacity = 2 +web_max_capacity = 4 +worker_desired_count = 1 +keycloak_min_capacity = 1 +keycloak_max_capacity = 2 + +db_instance_class = "db.t4g.micro" +db_multi_az = false +db_backup_retention_days = 3 +db_deletion_protection = false + +enable_nat_gateway = true + +# Staging gets its own provider keys, so its usage and quota never touch +# production's. Fill them into Secrets Manager after the first apply. +gemini_api_key = "" +upstash_redis_rest_url = "" +upstash_redis_rest_token = "" +resend_api_key = "" +google_client_id = "" +google_client_secret = "" + +email_from = "Mr. Bridge staging " +alarm_email = "" diff --git a/infra/terraform/iam.tf b/infra/terraform/iam.tf index 3061fd9..a370fc4 100644 --- a/infra/terraform/iam.tf +++ b/infra/terraform/iam.tf @@ -146,6 +146,17 @@ data "aws_iam_policy_document" "github_deploy" { resources = ["*"] } + # Promotion copies another stack's tested images (e.g. staging's) here + # unchanged, so this role may read that stack's repositories. + dynamic "statement" { + for_each = var.image_promotion_source != "" ? [var.image_promotion_source] : [] + content { + sid = "EcrPromoteFrom" + actions = ["ecr:BatchGetImage", "ecr:GetDownloadUrlForLayer", "ecr:BatchCheckLayerAvailability", "ecr:DescribeImages"] + resources = ["arn:aws:ecr:${var.aws_region}:${data.aws_caller_identity.current.account_id}:repository/${statement.value}/*"] + } + } + statement { sid = "EcrPush" actions = [ @@ -162,6 +173,7 @@ data "aws_iam_policy_document" "github_deploy" { aws_ecr_repository.web.arn, aws_ecr_repository.keycloak.arn, aws_ecr_repository.migrate.arn, + aws_ecr_repository.worker.arn, ] } diff --git a/infra/terraform/monitoring.tf b/infra/terraform/monitoring.tf index 2e7c809..3720bab 100644 --- a/infra/terraform/monitoring.tf +++ b/infra/terraform/monitoring.tf @@ -17,6 +17,8 @@ resource "aws_sns_topic_subscription" "alarm_email" { locals { alarm_actions = [aws_sns_topic.alarms.arn] + # The web and worker write metrics here (CloudWatch embedded metric format). + metrics_namespace = "MatchMaker/${var.environment}" } # ── Availability ────────────────────────────────────────────── @@ -214,3 +216,179 @@ resource "aws_cloudwatch_metric_alarm" "login_email_failures" { alarm_actions = local.alarm_actions } + +# ── Pipeline ────────────────────────────────────────────────── +# Written by the worker once a minute and by every search. Each alarm names +# the runbook section in docs/OPERATIONS.md that says what to do. + +locals { + pipeline_alarms = { + worker-down = { + description = "No worker heartbeat for 5 minutes: matching and notifications have stopped. Runbook: Worker down." + metric = "WorkerHeartbeat" + statistic = "SampleCount" + period = 300 + periods = 1 + threshold = 1 + comparison = "LessThanThreshold" + missing = "breaching" + } + matching-backlog = { + description = "A due matching job has waited over 10 minutes. Runbook: Matching backlog." + metric = "OldestDueJobSeconds" + statistic = "Maximum" + period = 300 + periods = 2 + threshold = 600 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + jobs-failed = { + description = "Matching jobs failed after every retry. Runbook: Failed jobs (npm run jobs:failed)." + metric = "JobsFailed" + statistic = "Maximum" + period = 300 + periods = 1 + threshold = 0 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + notification-backlog = { + description = "A due notification has waited over 10 minutes. Runbook: Notifications stuck." + metric = "OldestDueNotificationSeconds" + statistic = "Maximum" + period = 300 + periods = 2 + threshold = 600 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + notifications-failed = { + description = "Notifications failed after every retry. Runbook: Notifications stuck." + metric = "NotificationsFailed" + statistic = "Maximum" + period = 300 + periods = 1 + threshold = 0 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + search-slow = { + description = "Search p95 above its 6 s budget for 15 minutes. Runbook: Slow search." + metric = "SearchLatencyMs" + statistic = "p95" + period = 300 + periods = 3 + threshold = 6000 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + search-failing = { + description = "More than 5 searches failed in 5 minutes. Runbook: Slow search." + metric = "SearchFailures" + statistic = "Sum" + period = 300 + periods = 1 + threshold = 5 + comparison = "GreaterThanThreshold" + missing = "notBreaching" + } + } +} + +resource "aws_cloudwatch_metric_alarm" "pipeline" { + for_each = local.pipeline_alarms + + alarm_name = "${local.name}-${each.key}" + alarm_description = each.value.description + namespace = local.metrics_namespace + metric_name = each.value.metric + statistic = each.value.statistic == "p95" ? null : each.value.statistic + extended_statistic = each.value.statistic == "p95" ? "p95" : null + period = each.value.period + evaluation_periods = each.value.periods + threshold = each.value.threshold + comparison_operator = each.value.comparison + treat_missing_data = each.value.missing + + alarm_actions = local.alarm_actions + ok_actions = local.alarm_actions +} + +# Model failures are expected occasionally (the gateway falls back); only a +# sustained rate needs a person, e.g. an expired key or an exhausted quota. +resource "aws_cloudwatch_metric_alarm" "model_failures" { + alarm_name = "${local.name}-model-failures" + alarm_description = "Over 20% of model calls failing for 15 minutes; matching runs on fallbacks. Runbook: Model provider failing." + comparison_operator = "GreaterThanThreshold" + evaluation_periods = 3 + threshold = 20 + treat_missing_data = "notBreaching" + + metric_query { + id = "rate" + expression = "IF(calls > 0, 100 * failures / calls, 0)" + label = "Model call failure %" + return_data = true + } + + metric_query { + id = "failures" + expression = join(" + ", [for op in local.model_operations : "f_${replace(op, ":", "_")}"]) + } + + metric_query { + id = "calls" + expression = join(" + ", [for op in local.model_operations : "c_${replace(op, ":", "_")}"]) + } + + dynamic "metric_query" { + for_each = { for pair in setproduct(local.model_operations, ["f", "c"]) : "${pair[1]}_${replace(pair[0], ":", "_")}" => pair } + content { + id = metric_query.key + metric { + namespace = local.metrics_namespace + metric_name = metric_query.value[1] == "f" ? "ModelCallFailures" : "ModelCalls" + dimensions = { Operation = metric_query.value[0] } + period = 300 + stat = "Sum" + } + } + } + + alarm_actions = local.alarm_actions +} + +locals { + model_operations = ["embed:profile", "embed:query", "rerank"] +} + +resource "aws_cloudwatch_dashboard" "main" { + dashboard_name = local.name + dashboard_body = jsonencode({ + widgets = [ + for index, widget in [ + { title = "Search latency (ms)", metrics = [[local.metrics_namespace, "SearchLatencyMs", { stat = "p50" }], ["...", { stat = "p95" }]] }, + { title = "Queue wait (s)", metrics = [[local.metrics_namespace, "OldestDueJobSeconds"], [local.metrics_namespace, "OldestDueNotificationSeconds"]] }, + { title = "Queue size", metrics = [[local.metrics_namespace, "JobsPending"], [local.metrics_namespace, "NotificationsPending"]] }, + { title = "Failed for good", metrics = [[local.metrics_namespace, "JobsFailed"], [local.metrics_namespace, "NotificationsFailed"]] }, + { title = "Model call failures by operation", metrics = [for op in local.model_operations : [local.metrics_namespace, "ModelCallFailures", "Operation", op, { stat = "Sum" }]] }, + { title = "Web response time (s)", metrics = [["AWS/ApplicationELB", "TargetResponseTime", "LoadBalancer", aws_lb.main.arn_suffix, { stat = "p95" }]] }, + ] : { + type = "metric" + x = (index % 2) * 12 + y = floor(index / 2) * 6 + width = 12 + height = 6 + properties = { + title = widget.title + metrics = widget.metrics + region = var.aws_region + stat = "Maximum" + period = 300 + view = "timeSeries" + } + } + ] + }) +} diff --git a/infra/terraform/secrets.tf b/infra/terraform/secrets.tf index f3b1e51..5261cd1 100644 --- a/infra/terraform/secrets.tf +++ b/infra/terraform/secrets.tf @@ -5,12 +5,28 @@ # ============================================================== # Encrypts the session cookie. Generated here so it never has to be handled by a -# human; rotating it simply signs everyone out. +# human. Two slots make rotation free of sign-outs: the active slot seals new +# cookies, the other is still accepted for reading (SESSION_SECRET_PREVIOUS). +# To rotate, regenerate the inactive slot and make it active in one apply: +# terraform apply -replace=random_password.session_secret_b -var session_secret_active=b +# (next time: -replace=random_password.session_secret -var session_secret_active=a) resource "random_password" "session_secret" { length = 64 special = false } +resource "random_password" "session_secret_b" { + length = 64 + special = false +} + +# Hashes login codes; separate from the session secret so rotating that one +# leaves codes already e-mailed valid. +resource "random_password" "otp_pepper" { + length = 64 + special = false +} + resource "random_password" "keycloak_client_secret" { length = 48 special = false @@ -29,7 +45,9 @@ resource "random_password" "keycloak_admin_password" { locals { secret_values = { DATABASE_URL = local.app_database_url - SESSION_SECRET = random_password.session_secret.result + SESSION_SECRET = var.session_secret_active == "a" ? random_password.session_secret.result : random_password.session_secret_b.result + SESSION_SECRET_PREVIOUS = var.session_secret_active == "a" ? random_password.session_secret_b.result : random_password.session_secret.result + OTP_PEPPER = random_password.otp_pepper.result KEYCLOAK_CLIENT_SECRET = random_password.keycloak_client_secret.result GEMINI_API_KEY = var.gemini_api_key UPSTASH_REDIS_REST_URL = var.upstash_redis_rest_url diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf index 028ca0f..2c9ff5c 100644 --- a/infra/terraform/variables.tf +++ b/infra/terraform/variables.tf @@ -305,3 +305,20 @@ variable "enable_container_insights" { type = bool default = true } + +variable "session_secret_active" { + description = "Which session-secret slot seals new cookies (a or b); the other is still accepted. Flip it, with -replace of the newly active slot, to rotate without signing anyone out." + type = string + default = "a" + + validation { + condition = contains(["a", "b"], var.session_secret_active) + error_message = "session_secret_active must be \"a\" or \"b\"." + } +} + +variable "image_promotion_source" { + description = "Stack whose images production may promote unchanged, e.g. \"match-maker-staging\" (same account and region). Empty disables promotion." + type = string + default = "" +} diff --git a/package-lock.json b/package-lock.json index 7a1db06..44a8a2f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "packages/*" ], "devDependencies": { + "@playwright/test": "1.63.0", "@types/node": "^22.10.2", "@types/pg": "^8.11.10", "@types/react": "^19.0.2", @@ -1512,6 +1513,22 @@ "node": ">=12.4.0" } }, + "node_modules/@playwright/test": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@react-email/render": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@react-email/render/-/render-1.1.2.tgz", @@ -6141,6 +6158,35 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "devOptional": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", diff --git a/package.json b/package.json index 36c262a..acb2922 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "typecheck": "turbo run typecheck", "test": "vitest run", "test:watch": "vitest", + "test:e2e": "playwright test", "db:migrate": "tsx packages/db/src/migrate.ts", "db:seed": "tsx packages/db/src/seed.ts", "db:import": "tsx tools/import-attendees.ts", @@ -35,9 +36,14 @@ "account:erase": "tsx tools/manage-account.ts erase", "jobs:failed": "tsx tools/manage-jobs.ts failed", "jobs:retry": "tsx tools/manage-jobs.ts retry", - "models:usage": "tsx tools/model-usage.ts" + "models:usage": "tsx tools/model-usage.ts", + "capacity:check": "tsx tools/capacity-check.ts", + "db:verify": "tsx tools/verify-database.ts", + "db:restore-rehearsal": "tsx tools/restore-rehearsal.ts", + "db:check-migrations": "tsx tools/check-migrations.ts" }, "devDependencies": { + "@playwright/test": "1.63.0", "@types/node": "^22.10.2", "@types/pg": "^8.11.10", "@types/react": "^19.0.2", diff --git a/packages/auth/src/config.ts b/packages/auth/src/config.ts index 4168037..4f6fb5e 100644 --- a/packages/auth/src/config.ts +++ b/packages/auth/src/config.ts @@ -69,6 +69,12 @@ export interface SessionConfig { cookieName: string; /** Session lifetime in seconds. */ maxAge: number; + /** + * Earlier secrets still accepted for reading cookies (never for writing), + * so rotating SESSION_SECRET does not sign anyone out. Keep the old value in + * SESSION_SECRET_PREVIOUS for at least one session lifetime. + */ + previousSecrets: string[]; secure: boolean; } @@ -80,8 +86,13 @@ export function getSessionConfig(): SessionConfig { "Generate one with: openssl rand -base64 48" ); } + const previousSecrets = (process.env.SESSION_SECRET_PREVIOUS ?? "").split(",").map((value) => value.trim()).filter(Boolean); + if (previousSecrets.some((value) => value.length < 32)) { + throw new Error("Every SESSION_SECRET_PREVIOUS value must be at least 32 characters."); + } return { secret, + previousSecrets, cookieName: optional("SESSION_COOKIE_NAME", "mb_session"), maxAge: Number(optional("SESSION_MAX_AGE", String(60 * 60 * 24 * 7))), secure: process.env.NODE_ENV === "production", diff --git a/packages/auth/src/session.ts b/packages/auth/src/session.ts index 34bb64b..7284099 100644 --- a/packages/auth/src/session.ts +++ b/packages/auth/src/session.ts @@ -34,6 +34,20 @@ function deriveKey(secret: string): Uint8Array { return new Uint8Array(createHash("sha256").update(secret).digest()); } +/** Decrypts with the current secret, then with each previous one (rotation). */ +async function decrypt(value: string, options: { issuer: string; audience: string }) { + const config = getSessionConfig(); + let failure: unknown; + for (const secret of [config.secret, ...config.previousSecrets]) { + try { + return await jwtDecrypt(value, deriveKey(secret), options); + } catch (error) { + failure = error; + } + } + throw failure; +} + /** Encrypt a session into a compact JWE suitable for a cookie value. */ export async function sealSession( session: Omit @@ -52,8 +66,7 @@ export async function sealSession( export async function unsealSession(value: string): Promise { if (!value) return null; try { - const config = getSessionConfig(); - const { payload } = await jwtDecrypt(value, deriveKey(config.secret), { + const { payload } = await decrypt(value, { issuer: ISSUER, audience: AUDIENCE, }); @@ -125,8 +138,7 @@ export async function unsealOAuthState( ): Promise { if (!value) return null; try { - const config = getSessionConfig(); - const { payload } = await jwtDecrypt(value, deriveKey(config.secret), { + const { payload } = await decrypt(value, { issuer: ISSUER, audience: "match-maker-oauth", }); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index a13636c..ba2aebe 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -15,3 +15,4 @@ export * as housekeeping from "./repositories/housekeeping"; export * as audit from "./repositories/audit"; export * as modelUsage from "./repositories/model-usage"; export * as outbox from "./repositories/outbox"; +export * as pipelineHealth from "./repositories/pipeline-health"; diff --git a/packages/db/src/repositories/pipeline-health.ts b/packages/db/src/repositories/pipeline-health.ts new file mode 100644 index 0000000..d5a86d2 --- /dev/null +++ b/packages/db/src/repositories/pipeline-health.ts @@ -0,0 +1,30 @@ +import { query } from "../client"; + +/** Backlog and failure counts of the background queues, for metrics and alarms. */ +export interface PipelineHealth { + jobsPending: number; + /** Age of the oldest job that is due but not picked up; 0 when none. */ + oldestDueJobSeconds: number; + /** Jobs that failed after every retry, in the last day. */ + jobsFailed: number; + notificationsPending: number; + oldestDueNotificationSeconds: number; + notificationsFailed: number; +} + +export async function snapshot(): Promise { + const { rows: [row] } = await query>( + `select + (select count(*) from matching_jobs where status = 'pending') as "jobsPending", + (select coalesce(extract(epoch from now() - min(available_at)), 0) from matching_jobs + where status = 'pending' and available_at <= now()) as "oldestDueJobSeconds", + (select count(*) from matching_jobs where status = 'failed' + and coalesce(started_at, created_at) > now() - interval '1 day') as "jobsFailed", + (select count(*) from notification_outbox where status = 'pending') as "notificationsPending", + (select coalesce(extract(epoch from now() - min(available_at)), 0) from notification_outbox + where status = 'pending' and available_at <= now()) as "oldestDueNotificationSeconds", + (select count(*) from notification_outbox where status = 'failed' + and created_at > now() - interval '1 day') as "notificationsFailed"` + ); + return Object.fromEntries(Object.entries(row).map(([key, value]) => [key, Math.round(Number(value))])) as unknown as PipelineHealth; +} diff --git a/packages/db/workers/metrics.worker.ts b/packages/db/workers/metrics.worker.ts new file mode 100644 index 0000000..b015713 --- /dev/null +++ b/packages/db/workers/metrics.worker.ts @@ -0,0 +1,31 @@ +import { pipelineHealth } from "../src/index"; +import type { Metrics } from "../../../apps/web/src/application/ports/metrics"; + +export const REPORT_EVERY_MS = 60 * 1000; + +let lastReport = 0; + +/** + * Reports queue backlog and failures once a minute. WorkerHeartbeat doubles + * as liveness: when it stops arriving, the worker is down. + */ +export async function reportHealthIfDue(metrics: Metrics, now = Date.now()): Promise { + if (now - lastReport < REPORT_EVERY_MS) return false; + lastReport = now; + try { + const health = await pipelineHealth.snapshot(); + metrics.record([ + { name: "WorkerHeartbeat", value: 1, unit: "Count" }, + { name: "JobsPending", value: health.jobsPending, unit: "Count" }, + { name: "OldestDueJobSeconds", value: health.oldestDueJobSeconds, unit: "Seconds" }, + { name: "JobsFailed", value: health.jobsFailed, unit: "Count" }, + { name: "NotificationsPending", value: health.notificationsPending, unit: "Count" }, + { name: "OldestDueNotificationSeconds", value: health.oldestDueNotificationSeconds, unit: "Seconds" }, + { name: "NotificationsFailed", value: health.notificationsFailed, unit: "Count" }, + ]); + } catch (error) { + // A missing heartbeat raises the alarm; the queues keep running. + console.error("[worker:metrics] could not read queue health", error); + } + return true; +} diff --git a/packages/db/workers/worker.ts b/packages/db/workers/worker.ts index 80af71c..28dacc4 100644 --- a/packages/db/workers/worker.ts +++ b/packages/db/workers/worker.ts @@ -5,6 +5,8 @@ import { processEmbedJob } from "./embed.worker"; import { processMatchJob } from "./match.worker"; import { sweepIfDue } from "./housekeeping.worker"; import { processNotification } from "./notification.worker"; +import { reportHealthIfDue } from "./metrics.worker"; +import { getContainer } from "../../../apps/web/src/composition/container"; let stopping = false; process.on("SIGINT", () => { stopping = true; }); @@ -15,6 +17,7 @@ async function main() { console.info(`[worker] embedding and matching queue started (version ${process.env.APP_VERSION || "dev"})`); while (!stopping) { await sweepIfDue(); + await reportHealthIfDue(getContainer().metrics); const embedded = await processEmbedJob(); const matched = await processMatchJob(); const notified = await processNotification(); diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..89802d1 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,30 @@ +import { defineConfig } from "@playwright/test"; + +/** + * Browser acceptance suite (tests/e2e). It drives an already running app and + * worker, started with EMAIL_DEV_FALLBACK=true and their output written to + * E2E_WEB_LOG, which is where the tests read sign-in codes: + * + * E2E_BASE_URL=http://localhost:3000 npx playwright test + * + * Set E2E_BROWSER_CHANNEL=chrome to use an installed Chrome instead of the + * bundled Chromium (`npx playwright install chromium`). + */ +export default defineConfig({ + testDir: "tests/e2e", + // The journey is one story told in order; parallel runs would share an event. + workers: 1, + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: 0, + timeout: 60_000, + expect: { timeout: 15_000 }, + reporter: process.env.CI ? [["list"], ["html", { open: "never", outputFolder: "playwright-report" }]] : "list", + outputDir: "test-results/e2e", + use: { + baseURL: process.env.E2E_BASE_URL ?? "http://localhost:3000", + channel: process.env.E2E_BROWSER_CHANNEL || undefined, + screenshot: "only-on-failure", + trace: "retain-on-failure", + }, +}); diff --git a/tests/e2e/edges.spec.ts b/tests/e2e/edges.spec.ts new file mode 100644 index 0000000..908f5f7 --- /dev/null +++ b/tests/e2e/edges.spec.ts @@ -0,0 +1,89 @@ +/** + * Sign-in edges and layout: phone lookup still proves e-mail ownership, + * magic links work once, a tampered cookie leads to a usable login page, + * signing out ends access, and pages fit a phone screen. + */ +import { devices, expect, test, type Page } from "@playwright/test"; +import { ORG, ORGANIZER, RUN, createEvent, enterCode, logOffset, loginMail, phone, rosterFile, sidewaysOverflow, signIn } from "./support"; + +const SLUG = `e2e-edges-${RUN}`; +const dana = { email: `e2e-dana-${RUN}@example.com`, phone: phone() }; +const eli = { email: `e2e-eli-${RUN}@example.com`, phone: phone() }; +const PHONE = { ...devices["Pixel 7"] }; +delete (PHONE as { defaultBrowserType?: string }).defaultBrowserType; + +test.describe.configure({ mode: "serial" }); + +test.describe("sign-in edges and phone layout", () => { + let organizer: Page; + + test.beforeAll(async ({ browser }) => { + organizer = await signIn(browser, ORGANIZER, `/orgs/${ORG}/events/new`, PHONE); + await createEvent(organizer, SLUG); + await organizer.goto(`/orgs/${ORG}/events/${SLUG}/roster`); + await organizer.locator("#field-file").setInputFiles(rosterFile([ + { Name: "Dana Edge", Email: dana.email, Phone: dana.phone }, + { Name: "Eli Edge", Email: eli.email, Phone: eli.phone }, + ])); + await organizer.getByRole("button", { name: "Import roster" }).click(); + await expect(organizer.getByRole("status").first()).toContainText("Added 2 attendee"); + }); + + test("organizer pages fit a phone and offer sign out", async () => { + for (const path of ["", "/roster", "/matching", "/analytics"]) { + await organizer.goto(`/orgs/${ORG}/events/${SLUG}${path}`); + expect(await sidewaysOverflow(organizer), `organizer${path || " dashboard"}`).toBeLessThanOrEqual(1); + await expect(organizer.getByRole("button", { name: "Sign out" })).toHaveCount(1); + } + }); + + test("phone sign-in shows only a masked e-mail, then reaches the event", async ({ browser }) => { + const page = await (await browser.newContext(PHONE)).newPage(); + const returnTo = `/e/${SLUG}/connections`; + await page.goto(`/login?returnTo=${encodeURIComponent(returnTo)}`); + await page.getByRole("button", { name: /Phone Number/ }).click(); + await page.locator('input[type="tel"]').fill(dana.phone); + const since = logOffset(); + await page.getByRole("button", { name: "Send sign-in code" }).click(); + await expect(page.getByText("Check your inbox")).toBeVisible(); + await expect(page.locator("body")).not.toContainText(dana.email); + await enterCode(page, dana.email, since, returnTo); + + for (const path of ["/connections", "/matches"]) { + await page.goto(`/e/${SLUG}${path}`); + expect(await sidewaysOverflow(page), path).toBeLessThanOrEqual(1); + } + + await page.getByRole("button", { name: "Sign out" }).click(); + await page.waitForURL("**/login**"); + await page.goto(`/e/${SLUG}/connections`); + await expect(page).toHaveURL(/\/login/); + }); + + test("a magic link signs in once and cannot be reused", async ({ browser, baseURL }) => { + const page = await (await browser.newContext()).newPage(); + const since = logOffset(); + const response = await page.request.post("/api/auth/otp/request", { + headers: { "content-type": "application/json", origin: baseURL! }, + data: { email: eli.email, returnTo: `/e/${SLUG}/profile` }, + }); + expect(response.ok()).toBe(true); + const { link } = await loginMail(eli.email, since); + const local = link!.replace(/^https?:\/\/[^/]+/, ""); + await page.goto(local); + await expect(page).toHaveURL(new RegExp(`/e/${SLUG}/profile`)); + + const other = await (await browser.newContext()).newPage(); + await other.goto(local); + await expect(other).toHaveURL(/\/login/); + }); + + test("a tampered session cookie leads to a usable login page", async ({ browser, baseURL }) => { + const context = await browser.newContext(); + await context.addCookies([{ name: "mb_session", value: "tampered.value", url: baseURL! }]); + const page = await context.newPage(); + await page.goto(`/e/${SLUG}/matches`); + await expect(page).toHaveURL(/\/login/); + await expect(page.getByRole("button", { name: /Email Address/ })).toBeVisible(); + }); +}); diff --git a/tests/e2e/journey.spec.ts b/tests/e2e/journey.spec.ts new file mode 100644 index 0000000..02dea78 --- /dev/null +++ b/tests/e2e/journey.spec.ts @@ -0,0 +1,129 @@ +/** + * The documented journey, in order: an organizer sets up an event and + * imports the roster; invited attendees sign in and complete profiles; the + * worker matches them; search finds people; an intro is requested and + * accepted; contacts appear only then; a venue question is asked and + * answered; a third attendee sees none of it; closing the event freezes it. + */ +import { expect, test, type Page } from "@playwright/test"; +import { ORG, ORGANIZER, RUN, createEvent, phone, requestCode, rosterFile, signIn } from "./support"; + +const SLUG = `e2e-${RUN}`; +const people = { + alice: { name: "Alice E2E", designation: "ML Engineer", domain: "Artificial Intelligence", about: "I build retrieval systems and LLM evaluation tooling for startups.", interests: "AI, search, startups", intent: "NETWORKING" }, + bob: { name: "Bob E2E", designation: "Product Designer", domain: "Design", about: "Designing AI assistants for climate and health teams.", interests: "AI, design, climate", intent: "NETWORKING" }, + carol: { name: "Carol E2E", designation: "Founder", domain: "Artificial Intelligence", about: "Founder hiring engineers for an AI search startup.", interests: "AI, hiring, search", intent: "HIRING" }, +}; +type Person = keyof typeof people; +const email = (key: Person | "stranger") => `e2e-${key}-${RUN}@example.com`; + +test.describe.configure({ mode: "serial" }); + +test.describe("organizer to meeting", () => { + let organizer: Page; + const pages = {} as Record; + + test("organizer creates the event and imports the roster", async ({ browser }) => { + organizer = await signIn(browser, ORGANIZER, `/orgs/${ORG}/events/new`); + await createEvent(organizer, SLUG); + await organizer.goto(`/orgs/${ORG}/events/${SLUG}/roster`); + const rows = (Object.keys(people) as Person[]).map((key) => ({ Name: people[key].name, Email: email(key), Phone: phone() })); + await organizer.locator("#field-file").setInputFiles(rosterFile([...rows, { Name: "Broken Row", Email: "not-an-email" }])); + await organizer.getByRole("button", { name: "Import roster" }).click(); + const status = organizer.getByRole("status").first(); + await expect(status).toContainText("Added 3 attendee"); + await expect(status).toContainText("Skipped 1 row"); + }); + + test("an uninvited e-mail cannot sign in", async ({ browser }) => { + const page = await (await browser.newContext()).newPage(); + expect((await requestCode(page, email("stranger"), `/e/${SLUG}/profile`)).outcome).toBe("unregistered"); + }); + + test("invited attendees sign in and complete their profiles", async ({ browser }) => { + for (const key of Object.keys(people) as Person[]) { + const page = pages[key] = await signIn(browser, email(key), `/e/${SLUG}/profile`); + const person = people[key]; + await page.getByLabel("Role or designation").fill(person.designation); + await page.getByLabel("Professional domain").fill(person.domain); + await page.getByLabel("About you").fill(person.about); + await page.getByLabel("Years of experience").fill("5"); + await page.getByLabel("Interests").fill(person.interests); + await page.getByLabel("What are you looking for?").selectOption(person.intent); + await page.getByLabel("Where can people find you?").selectOption("Main hall"); + await page.getByRole("button", { name: /Save profile/ }).click(); + await page.waitForURL(/\/matches\?saved=1/); + } + }); + + test("the worker matches the first person to save without a manual refresh", async () => { + const alice = pages.alice; + await expect(async () => { + await alice.goto(`/e/${SLUG}/matches`); + await expect(alice.locator("article", { hasText: people.bob.name }).getByRole("button", { name: "Request intro" })).toBeVisible({ timeout: 1000 }); + }).toPass({ timeout: 45_000, intervals: [1500] }); + }); + + test("search finds others and never the searcher", async () => { + const alice = pages.alice; + await alice.goto(`/e/${SLUG}/search`); + await alice.getByLabel("Describe who you are looking for").fill("designer working on climate"); + await alice.getByRole("button", { name: "Search" }).click(); + const results = alice.locator("article"); + await expect(results.filter({ hasText: people.bob.name })).toHaveCount(1); + await expect(results.filter({ hasText: people.alice.name })).toHaveCount(0); + }); + + test("an intro reveals contacts only after the recipient accepts", async () => { + const { alice, bob } = pages; + await alice.goto(`/e/${SLUG}/matches`); + await alice.locator("article", { hasText: people.bob.name }).getByRole("button", { name: "Request intro" }).click(); + await alice.waitForURL(/connections\?requested=1/); + await expect(alice.locator("main")).not.toContainText(email("bob")); + + await bob.goto(`/e/${SLUG}/connections`); + await bob.getByRole("button", { name: "Accept" }).first().click(); + await bob.waitForLoadState("networkidle"); + await alice.goto(`/e/${SLUG}/connections`); + await expect(alice.locator("main")).toContainText(email("bob")); + }); + + test("a venue question is answered voluntarily and only the asker sees the zone", async () => { + const { alice, bob, carol } = pages; + await alice.getByRole("button", { name: "Ask where they are" }).click(); + await alice.waitForURL(/asked=1/); + await expect(alice.locator("main")).toContainText("Waiting for a reply"); + + await bob.goto(`/e/${SLUG}/connections`); + await expect(bob.locator("main")).toContainText(`${people.alice.name} is asking where you are`); + await bob.getByLabel("Your venue zone").first().selectOption("Cafe"); + await bob.getByRole("button", { name: `Share with ${people.alice.name}` }).click(); + await bob.waitForURL(/replied=1/); + + await alice.goto(`/e/${SLUG}/connections`); + await expect(alice.locator("main")).toContainText("Replied: at Cafe"); + await carol.goto(`/e/${SLUG}/connections`); + const carolView = carol.locator("main"); + for (const hidden of [email("alice"), email("bob"), "Cafe"]) await expect(carolView).not.toContainText(hidden); + }); + + test("analytics counts the accepted intro", async () => { + await organizer.goto(`/orgs/${ORG}/events/${SLUG}/analytics`); + await expect(organizer.locator("main")).toContainText(/Accepted introductions\s*1/); + }); + + test("closing the event keeps matches readable and stops changes", async () => { + await organizer.goto(`/orgs/${ORG}/events/${SLUG}`); + await organizer.getByLabel("Registration status").selectOption("closed"); + await organizer.getByRole("button", { name: "Save event configuration" }).click(); + await expect(organizer.getByText("Event configuration saved")).toBeVisible(); + + const { alice } = pages; + await alice.goto(`/e/${SLUG}/matches`); + await expect(alice.locator("main")).toContainText("has closed"); + await expect(alice.locator("main")).toContainText(people.bob.name); + await expect(alice.getByRole("button", { name: "Request intro" })).toHaveCount(0); + await alice.goto(`/e/${SLUG}/profile`); + await expect(alice).toHaveURL(new RegExp(`/e/${SLUG}$`)); + }); +}); diff --git a/tests/e2e/support.ts b/tests/e2e/support.ts new file mode 100644 index 0000000..e842cb7 --- /dev/null +++ b/tests/e2e/support.ts @@ -0,0 +1,101 @@ +/** + * Helpers for the browser suite: sign in through the real login page with + * the code the app logged (EMAIL_DEV_FALLBACK), build roster sheets, and + * create events. Every run uses its own event and attendees. + */ +import { readFileSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { expect, type Browser, type BrowserContextOptions, type Page } from "@playwright/test"; +import pg from "pg"; +import * as XLSX from "xlsx"; +import "dotenv/config"; + +export const ORG = process.env.E2E_ORGANIZATION ?? "ai-community-lucknow"; +export const ORGANIZER = process.env.E2E_ORGANIZER_EMAIL ?? "organizer@example.com"; +const WEB_LOG = process.env.E2E_WEB_LOG ?? "node_modules/.cache/e2e/web.log"; + +export const RUN = Date.now().toString(36); +export const phone = () => `9${String(Math.floor(Math.random() * 1e9)).padStart(9, "0")}`; + +export async function db(sql: string, params: unknown[] = []): Promise { + const client = new pg.Client({ connectionString: process.env.DATABASE_URL }); + await client.connect(); + try { + return (await client.query(sql, params)).rows; + } finally { + await client.end(); + } +} + +/** Byte offset of the end of the app log; mail logged after it is new. */ +export const logOffset = () => statSync(WEB_LOG).size; + +/** The newest login e-mail the app logged for `email` after byte offset `since`. */ +export async function loginMail(email: string, since: number): Promise<{ code: string; link: string | null }> { + for (let attempt = 0; attempt < 40; attempt++) { + const log = readFileSync(WEB_LOG).subarray(since).toString("utf8"); + const at = log.lastIndexOf(`To: ${email}`); + const code = at >= 0 ? log.slice(at).match(/Code:\s+(\d{6})/)?.[1] : undefined; + if (code) return { code, link: log.slice(at).match(/Link:\s+(\S+)/)?.[1] ?? null }; + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error(`no login e-mail was logged for ${email}; is the app writing to ${WEB_LOG}?`); +} + +/** Sends a code from the login page; resolves with what the page answered. */ +export async function requestCode(page: Page, email: string, returnTo: string): Promise<{ outcome: "sent" | "unregistered"; since: number }> { + await db(`delete from login_challenges where email = $1`, [email]); + await page.goto(`/login?returnTo=${encodeURIComponent(returnTo)}`); + await page.getByRole("button", { name: /Email Address/ }).click(); + await page.locator('input[type="email"]').fill(email); + const since = logOffset(); + await page.getByRole("button", { name: "Send sign-in code" }).click(); + const outcome = await Promise.race([ + page.getByText("Check your inbox").waitFor().then(() => "sent" as const), + page.getByText(/not on the attendee list/i).waitFor().then(() => "unregistered" as const), + ]); + return { outcome, since }; +} + +export async function enterCode(page: Page, email: string, since: number, returnTo: string) { + const { code } = await loginMail(email, since); + await page.getByLabel("Six-digit email verification code").fill(code); + await page.getByRole("button", { name: "Verify & Sign In" }).click(); + await page.waitForURL((url) => url.pathname === returnTo.split("?")[0]); +} + +/** A new browser session signed in as `email`, on `returnTo`. */ +export async function signIn(browser: Browser, email: string, returnTo: string, options: BrowserContextOptions = {}): Promise { + const context = await browser.newContext(options); + const page = await context.newPage(); + const { outcome, since } = await requestCode(page, email, returnTo); + expect(outcome, `${email} should be invited`).toBe("sent"); + await enterCode(page, email, since, returnTo); + return page; +} + +export function rosterFile(rows: Array>): string { + const book = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(book, XLSX.utils.json_to_sheet(rows), "Roster"); + const file = join(tmpdir(), `mr-bridge-roster-${RUN}-${rows.length}.xlsx`); + writeFileSync(file, XLSX.write(book, { type: "buffer", bookType: "xlsx" })); + return file; +} + +export async function createEvent(page: Page, slug: string, zones = "Main hall, Cafe") { + await page.goto(`/orgs/${ORG}/events/new`); + await page.getByLabel("Event name").fill(`E2E ${slug}`); + await page.getByLabel("URL slug").fill(slug); + await page.getByRole("button", { name: "Create event" }).click(); + await page.waitForURL(`**/orgs/${ORG}/events/${slug}`); + await page.getByLabel("Supported intents").fill("NETWORKING, HIRING, MENTORING"); + await page.getByLabel("Venue zones").fill(zones); + await page.getByLabel("Registration status").selectOption("open"); + await page.getByRole("button", { name: "Save event configuration" }).click(); + await expect(page.getByText("Event configuration saved")).toBeVisible(); +} + +/** How far the page scrolls sideways; more than a pixel means a broken phone layout. */ +export const sidewaysOverflow = (page: Page) => + page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); diff --git a/tests/integration/capacity-check.test.ts b/tests/integration/capacity-check.test.ts new file mode 100644 index 0000000..6d1b24b --- /dev/null +++ b/tests/integration/capacity-check.test.ts @@ -0,0 +1,44 @@ +/** + * A small run of the capacity check against a real Postgres with the free + * hash embedding: every attendee gets matches, searches succeed, and the + * throwaway event and people are removed afterwards. Skipped without DATABASE_URL. + */ +import { afterAll, describe, expect, it, vi } from "vitest"; + +const saved = vi.hoisted(() => { + const saved = { EMBEDDING_PROVIDER: process.env.EMBEDDING_PROVIDER, GEMINI_API_KEY: process.env.GEMINI_API_KEY, MATCH_REFRESH_DELAY_SECONDS: process.env.MATCH_REFRESH_DELAY_SECONDS }; + process.env.EMBEDDING_PROVIDER = "hash"; + process.env.GEMINI_API_KEY = ""; + process.env.MATCH_REFRESH_DELAY_SECONDS = "0"; + delete (globalThis as { __matchMakerContainer?: unknown }).__matchMakerContainer; + return saved; +}); + +import { closePool, query } from "@match-maker/db"; +import { migrate } from "../../packages/db/src/migrate"; +import { budgetFailures, runCapacityCheck } from "../../tools/capacity-check"; + +const describeDb = process.env.DATABASE_URL ? describe : describe.skip; + +describeDb("capacity check", () => { + afterAll(async () => { + await closePool(); + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + delete (globalThis as { __matchMakerContainer?: unknown }).__matchMakerContainer; + }); + + it("measures a small event and cleans up after itself", async () => { + await migrate(); + const throwaway = async () => (await query(`select 1 from organizations where slug like 'capacity-%'`)).rows.length; + const before = await throwaway(); + const report = await runCapacityCheck({ attendees: 12, searches: 6, concurrency: 3, workers: 1 }); + expect(report).toMatchObject({ attendees: 12, searches: 6, attendeesWithoutMatches: 0 }); + expect(report.search.failures).toBe(0); + expect(report.search.p95Ms).toBeGreaterThan(0); + expect(budgetFailures(report)).toEqual([]); + expect(await throwaway()).toBe(before); + }); +}); diff --git a/tests/integration/concurrency.test.ts b/tests/integration/concurrency.test.ts index 55faa30..a473720 100644 --- a/tests/integration/concurrency.test.ts +++ b/tests/integration/concurrency.test.ts @@ -10,6 +10,7 @@ import { migrate } from "../../packages/db/src/migrate"; import { PostgresIntroRepository } from "../../apps/web/src/infrastructure/postgres/intro.repo"; import { PostgresLocationRepository } from "../../apps/web/src/infrastructure/postgres/location.repo"; import { PostgresParticipantRepository } from "../../apps/web/src/infrastructure/postgres/participant.repo"; +import { PostgresMatchRepository } from "../../apps/web/src/infrastructure/postgres/match.repo"; import { requestIntro } from "../../apps/web/src/application/use-cases/request-intro"; import { Intent } from "../../apps/web/src/domain/participant/intent"; @@ -23,6 +24,7 @@ describeDb("concurrent actions", () => { const intros = new PostgresIntroRepository(); const locations = new PostgresLocationRepository(); const participants = new PostgresParticipantRepository(); + const matches = new PostgresMatchRepository(); const ids: Record = {}; let eventId: string; @@ -88,6 +90,19 @@ describeDb("concurrent actions", () => { expect(created.filter(Boolean)).toHaveLength(1); }); + it("saves match lists that name each other at the same time without deadlocking", async () => { + const group = ["p0", "p1", "p2", "p3", "p4", "p5"]; + const profiles = await Promise.all(group.map((key) => participants.findProfile(eventId, ids[key]))); + const listFor = (key: string) => profiles.filter((profile) => profile!.attendeeId !== ids[key]) + .map((profile) => ({ participant: profile!, score: 50, scoreBreakdown: {}, mutualInterest: false, reason: null })); + const revisionOf = (key: string) => profiles.find((profile) => profile!.attendeeId === ids[key])!.revision!; + for (let round = 0; round < 5; round++) { + const results = await settle(group.map((key) => matches.save(eventId, ids[key], listFor(key), "race", revisionOf(key)))); + expect(results.filter((result) => result.status === "rejected")).toEqual([]); + expect(results.map((result) => (result as PromiseFulfilledResult).value)).toEqual(group.map(() => true)); + } + }); + it("imports overlapping roster sheets at once without duplicating anyone", async () => { const sheet = (keys: string[]) => keys.map((key) => ({ name: `Imported ${key}`, email: email(`imp-${key}`), phone: null, about_me: null, designation: null, domain: null, diff --git a/tests/integration/pipeline-health.test.ts b/tests/integration/pipeline-health.test.ts new file mode 100644 index 0000000..ba3912b --- /dev/null +++ b/tests/integration/pipeline-health.test.ts @@ -0,0 +1,60 @@ +/** + * The queue health snapshot behind the backlog and failure alarms, against a + * real Postgres. Other local rows may exist, so it checks lower bounds from + * rows it creates in its own event. Skipped without DATABASE_URL. + */ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { attendees, closePool, events, pipelineHealth, query } from "@match-maker/db"; +import { migrate } from "../../packages/db/src/migrate"; +import { reportHealthIfDue } from "../../packages/db/workers/metrics.worker"; + +const describeDb = process.env.DATABASE_URL ? describe : describe.skip; +const RUN = Date.now().toString(36); +const ORG = `health-${RUN}`; + +describeDb("pipeline health", () => { + let eventId: string; + let attendeeId: string; + + beforeAll(async () => { + await migrate(); + await attendees.upsertMany([{ name: "Health check", email: `health-${RUN}@example.com` }]); + attendeeId = (await attendees.findByEmail(`health-${RUN}@example.com`))!.id; + const { rows: [org] } = await query<{ id: string }>(`insert into organizations (name, slug) values ($1, $1) returning id`, [ORG]); + eventId = (await query<{ id: string }>(`insert into events (organization_id, name, slug, status) values ($1, $2, $2, 'closed') returning id`, [org.id, ORG])).rows[0].id; + await events.attachRoster(eventId, [attendeeId]); + }); + + afterAll(async () => { + await query(`delete from organizations where slug = $1`, [ORG]); + await query(`delete from registered_attendees where email = $1`, [`health-${RUN}@example.com`]); + await closePool(); + }); + + it("reports how long due work has waited and what failed for good", async () => { + await query(`insert into matching_jobs (event_id, participant_id, job_type, status, available_at, created_at) + values ($1, $2, 'embed', 'pending', now() - interval '20 minutes', now() - interval '20 minutes'), + ($1, $2, 'match', 'failed', now(), now())`, [eventId, attendeeId]); + await query(`insert into notification_outbox (event_id, recipient_id, kind, subject_id, dedupe_key, status, available_at) + values ($1, $2, 'intro.requested', $2, $3, 'pending', now() - interval '15 minutes'), + ($1, $2, 'intro.accepted', $2, $4, 'failed', now())`, + [eventId, attendeeId, `health-a-${RUN}`, `health-b-${RUN}`]); + + const health = await pipelineHealth.snapshot(); + expect(health.jobsPending).toBeGreaterThanOrEqual(1); + expect(health.oldestDueJobSeconds).toBeGreaterThanOrEqual(20 * 60 - 5); + expect(health.jobsFailed).toBeGreaterThanOrEqual(1); + expect(health.oldestDueNotificationSeconds).toBeGreaterThanOrEqual(15 * 60 - 5); + expect(health.notificationsFailed).toBeGreaterThanOrEqual(1); + }); + + it("reports at most once a minute, with a heartbeat", async () => { + const metrics = { record: (values: Array<{ name: string }>) => recorded.push(values.map((value) => value.name)) }; + const recorded: string[][] = []; + const now = Date.now() + 10 * 60_000; + expect(await reportHealthIfDue(metrics, now)).toBe(true); + expect(await reportHealthIfDue(metrics, now + 1000)).toBe(false); + expect(recorded).toHaveLength(1); + expect(recorded[0]).toEqual(expect.arrayContaining(["WorkerHeartbeat", "OldestDueJobSeconds", "NotificationsFailed"])); + }); +}); diff --git a/tests/integration/retrieval-recall.test.ts b/tests/integration/retrieval-recall.test.ts new file mode 100644 index 0000000..e7db105 --- /dev/null +++ b/tests/integration/retrieval-recall.test.ts @@ -0,0 +1,65 @@ +/** + * Retrieval must not starve someone whose nearest neighbours all have + * intents they do not welcome: a mentor among many mentors still finds the + * one person seeking a mentor. Real Postgres, fake embeddings, own event. + * Skipped without DATABASE_URL. + */ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { attendees, closePool, events, query } from "@match-maker/db"; +import { migrate } from "../../packages/db/src/migrate"; +import { PostgresParticipantRepository } from "../../apps/web/src/infrastructure/postgres/participant.repo"; +import { PostgresMatchRepository } from "../../apps/web/src/infrastructure/postgres/match.repo"; +import { PostgresCandidateCache } from "../../apps/web/src/infrastructure/cache/postgres-candidate-cache"; +import { MatchingPipeline } from "../../apps/web/src/domain/matching/pipeline"; +import { Intent } from "../../apps/web/src/domain/participant/intent"; + +const describeDb = process.env.DATABASE_URL ? describe : describe.skip; +const RUN = Date.now().toString(36); +const ORG = `recall-${RUN}`; +const MENTORS = 55; +const email = (key: string) => `recall-${key}-${RUN}@example.com`; +const VECTOR = Array.from({ length: 768 }, (_, i) => (i === 0 ? 1 : 0)); +const embeddings = { id: "test:recall", embed: async () => VECTOR }; + +describeDb("retrieval recall", () => { + const participants = new PostgresParticipantRepository(); + const pipeline = new MatchingPipeline(participants, new PostgresMatchRepository(), embeddings, null, new PostgresCandidateCache(), null); + const ids: Record = {}; + let eventId: string; + + beforeAll(async () => { + await migrate(); + const keys = ["viewer", "seeker", ...Array.from({ length: MENTORS }, (_, i) => `mentor${i}`)]; + await attendees.upsertMany(keys.map((key) => ({ name: `Recall ${key}`, email: email(key) }))); + const { rows } = await query<{ id: string; email: string }>(`select id, email from registered_attendees where email like $1`, [`recall-%-${RUN}@example.com`]); + for (const row of rows) ids[row.email.split("-")[1]] = row.id; + const { rows: [org] } = await query<{ id: string }>(`insert into organizations (name, slug) values ($1, $1) returning id`, [ORG]); + eventId = (await query<{ id: string }>(`insert into events (organization_id, name, slug, status) values ($1, $2, $2, 'open') returning id`, [org.id, ORG])).rows[0].id; + await events.attachRoster(eventId, Object.values(ids)); + for (const key of keys) { + await participants.saveProfile({ + eventId, attendeeId: ids[key], designation: "Engineer", domain: "AI", aboutMe: key === "seeker" ? "New to the field" : "Mentors engineers", + selfDescription: null, yearsOfExperience: null, mostImpressiveBuild: null, + intent: key === "seeker" ? Intent.SEEKING_MENTOR : Intent.MENTORING, interests: [], venueZone: null, + }); + // Every mentor sits right next to the viewer; the seeker has no vector at all. + if (key.startsWith("mentor")) await participants.setEmbedding(eventId, ids[key], VECTOR, embeddings.id); + } + }); + + afterAll(async () => { + await query(`delete from organizations where slug = $1`, [ORG]); + await query(`delete from registered_attendees where email like $1`, [`recall-%-${RUN}@example.com`]); + await closePool(); + }); + + it("finds the only compatible person beyond the nearest neighbours", async () => { + const ranked = await pipeline.run({ eventId, participantId: ids.viewer, mode: "profile" }); + expect(ranked.map((candidate) => candidate.participant.attendeeId)).toEqual([ids.seeker]); + }); + + it("does the same for search", async () => { + const ranked = await pipeline.run({ eventId, participantId: ids.viewer, mode: "search", query: "mentors engineers" }); + expect(ranked.map((candidate) => candidate.participant.attendeeId)).toContain(ids.seeker); + }); +}); diff --git a/tests/unit/capacity-check.test.ts b/tests/unit/capacity-check.test.ts new file mode 100644 index 0000000..bcaad9c --- /dev/null +++ b/tests/unit/capacity-check.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { budgetFailures, percentile, type CapacityReport } from "../../tools/capacity-check"; +import { HashEmbeddingProvider } from "@/src/infrastructure/providers/hash-embedding.provider"; + +const report = (overrides: Partial = {}): CapacityReport => ({ + attendees: 300, searches: 60, firstListsSeconds: 30, quietSeconds: 60, attendeesWithoutMatches: 0, + search: { p50Ms: 200, p95Ms: 900, maxMs: 1200, failures: 0 }, + modelCalls: { perAttendee: 2, perSearch: 1, approxTokens: 1000, failures: 0 }, + ...overrides, +}); + +describe("capacity budgets", () => { + it("passes a report within every budget", () => { + expect(budgetFailures(report())).toEqual([]); + }); + + it("names each budget a report misses", () => { + const failures = budgetFailures(report({ + search: { p50Ms: 3000, p95Ms: 7000, maxMs: 9000, failures: 2 }, + attendeesWithoutMatches: 1, + modelCalls: { perAttendee: 9, perSearch: 1, approxTokens: 0, failures: 0 }, + })); + expect(failures).toEqual([ + "search p95 7000 ms is over the budget of 6000 ms", + "model calls per attendee 9 is over the budget of 6", + "2 search(es) failed", + "1 attendee(s) have no matches", + ]); + }); + + it("takes nearest-rank percentiles", () => { + const values = Array.from({ length: 100 }, (_, i) => i + 1); + expect(percentile(values, 50)).toBe(50); + expect(percentile(values, 95)).toBe(95); + expect(percentile([], 95)).toBe(0); + }); +}); + +describe("hash embedding", () => { + it("gives unit vectors that are closer for similar texts", async () => { + const provider = new HashEmbeddingProvider(); + const [a, b, c] = await Promise.all(["graph databases in rust", "rust graph databases", "watercolor painting"].map((text) => provider.embed(text))); + const dot = (x: number[], y: number[]) => x.reduce((sum, value, i) => sum + value * y[i], 0); + expect(a).toHaveLength(768); + expect(dot(a!, a!)).toBeCloseTo(1); + expect(dot(a!, b!)).toBeGreaterThan(dot(a!, c!)); + expect(await provider.embed(" ")).toBeNull(); + }); +}); diff --git a/tests/unit/metrics.test.ts b/tests/unit/metrics.test.ts new file mode 100644 index 0000000..9a40955 --- /dev/null +++ b/tests/unit/metrics.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from "vitest"; +import { EmfMetrics, MeteredModelUsageLog } from "@/src/infrastructure/observability/emf-metrics"; +import { searchParticipants } from "@/src/application/use-cases/search-participants"; +import type { Metrics } from "@/src/application/ports/metrics"; + +describe("embedded metric format", () => { + it("writes one line CloudWatch turns into metrics", () => { + const lines: string[] = []; + new EmfMetrics("MatchMaker/test", (line) => lines.push(line)).record( + [{ name: "SearchLatencyMs", value: 120, unit: "Milliseconds" }, { name: "SearchFailures", value: 0, unit: "Count" }], + { Operation: "rerank" }); + const line = JSON.parse(lines[0]); + expect(line._aws.CloudWatchMetrics).toEqual([{ + Namespace: "MatchMaker/test", + Dimensions: [["Operation"]], + Metrics: [{ Name: "SearchLatencyMs", Unit: "Milliseconds" }, { Name: "SearchFailures", Unit: "Count" }], + }]); + expect(line).toMatchObject({ Operation: "rerank", SearchLatencyMs: 120, SearchFailures: 0 }); + }); + + it("writes nothing without a namespace", () => { + const write = vi.fn(); + new EmfMetrics(null, write).record([{ name: "JobsPending", value: 3, unit: "Count" }]); + expect(write).not.toHaveBeenCalled(); + }); + + it("counts model calls and failures per operation, and still records usage", () => { + const metrics = { record: vi.fn() }; + const inner = { record: vi.fn() }; + const entry = { provider: "gemini", operation: "embed", outcome: "timeout" as const, latencyMs: 900, inputChars: 10 }; + new MeteredModelUsageLog(inner, metrics).record(entry); + expect(inner.record).toHaveBeenCalledWith(entry); + expect(metrics.record).toHaveBeenCalledWith(expect.arrayContaining([ + { name: "ModelCallFailures", value: 1, unit: "Count" }, + { name: "ModelCallLatencyMs", value: 900, unit: "Milliseconds" }, + ]), { Operation: "embed" }); + }); +}); + +describe("search metrics", () => { + const deps = (run: () => Promise) => { + const metrics: Metrics & { record: ReturnType } = { record: vi.fn() }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return { metrics, pipeline: { run } as any, participants: { listDomains: vi.fn(async () => []) } as any }; + }; + const search = (d: ReturnType) => searchParticipants({ eventId: "e", participantId: "p", query: "climate", eventIntents: [] }, d); + const failures = (d: ReturnType) => d.metrics.record.mock.calls[0][0].find((metric: { name: string }) => metric.name === "SearchFailures").value; + + it("records how long a search took", async () => { + const d = deps(async () => []); + await search(d); + expect(failures(d)).toBe(0); + expect(d.metrics.record.mock.calls[0][0][0]).toMatchObject({ name: "SearchLatencyMs", unit: "Milliseconds" }); + }); + + it("records a failed search and still fails it", async () => { + const d = deps(async () => { throw new Error("database down"); }); + await expect(search(d)).rejects.toThrow("database down"); + expect(failures(d)).toBe(1); + }); +}); diff --git a/tests/unit/release-safety.test.ts b/tests/unit/release-safety.test.ts new file mode 100644 index 0000000..5baada9 --- /dev/null +++ b/tests/unit/release-safety.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { migrationProblems } from "../../tools/check-migrations"; +import { compareState, type DatabaseState } from "../../tools/verify-database"; + +const files = (entries: Record) => new Map(Object.entries(entries)); + +describe("migration release gate", () => { + const released = files({ "0001_init.sql": "create table a (id int);\n" }); + + it("accepts expand-only migrations", () => { + expect(migrationProblems(released, files({ + ...Object.fromEntries(released), + "0002_more.sql": "alter table a add column b text;\ncreate index if not exists a_b on a (b);\ndrop index if exists old_idx;\n", + }))).toEqual([]); + }); + + it("refuses changes to a released migration, even when only line endings stay the same", () => { + expect(migrationProblems(released, files({ "0001_init.sql": "create table a (id int);\r\n" }))).toEqual([]); + expect(migrationProblems(released, files({ "0001_init.sql": "create table a (id bigint);\n" }))[0]).toMatch(/must not change/); + expect(migrationProblems(released, files({}))[0]).toMatch(/must not be deleted/); + }); + + it("names each statement that would break the previous image", () => { + const problems = migrationProblems(released, files({ + ...Object.fromEntries(released), + "0002_risky.sql": [ + "alter table a drop column b;", + "alter table a rename column c to d;", + "alter table a alter column e type bigint;", + "drop table old;", + "-- drop table only mentioned in a comment", + "alter table a drop column f; -- contract: unused since release 41", + ].join("\n"), + })); + expect(problems.map((problem) => problem.split(" ")[0])).toEqual(["0002_risky.sql:1", "0002_risky.sql:2", "0002_risky.sql:3", "0002_risky.sql:4"]); + }); + + it("refuses two migrations with the same number", () => { + expect(migrationProblems(new Map(), files({ "0002_a.sql": "", "0002_b.sql": "" }))).toEqual(["0002_b.sql reuses number 0002 of 0002_a.sql"]); + }); +}); + +describe("database verification", () => { + const state = (overrides: Partial = {}): DatabaseState => ({ + vector: true, + applied: files({ "0001_init.sql": "sum1" }), + counts: { events: 3, organizations: 1 }, + ...overrides, + }); + const expected = new Map([["0001_init.sql", ["sum1", "sum1-crlf"]]]); + + it("accepts a complete database", () => { + expect(compareState(state(), expected)).toEqual({ problems: [], differences: [] }); + }); + + it("reports what makes a database unusable", () => { + const { problems } = compareState(state({ vector: false, applied: files({ "0001_init.sql": "other" }), counts: { events: null } }), new Map([...expected, ["0002_next.sql", ["sum2"]]])); + expect(problems).toEqual([ + "the pgvector extension is missing", + "migration 0001_init.sql was applied from a different file", + "migration 0002_next.sql is not applied", + "table events is missing", + ]); + }); + + it("lists row count differences against a source", () => { + expect(compareState(state(), expected, state({ counts: { events: 4, organizations: 1 } })).differences).toEqual(["events: 3 here, 4 in the source"]); + }); +}); diff --git a/tests/unit/session.test.ts b/tests/unit/session.test.ts index d3c06df..11d1c0a 100644 --- a/tests/unit/session.test.ts +++ b/tests/unit/session.test.ts @@ -39,6 +39,33 @@ describe("session cookie", () => { process.env.SESSION_SECRET = SECRET; }); + it("keeps sessions sealed before a rotation, while new ones use the new secret", async () => { + const before = await sealSession(session); + const rotated = "the-new-session-secret-after-rotation-123456"; + process.env.SESSION_SECRET = rotated; + process.env.SESSION_SECRET_PREVIOUS = SECRET; + try { + expect(await unsealSession(before)).toMatchObject(session); + const after = await sealSession(session); + // Once the old secret is dropped, only cookies sealed after the rotation still work. + delete process.env.SESSION_SECRET_PREVIOUS; + expect(await unsealSession(after)).toMatchObject(session); + expect(await unsealSession(before)).toBeNull(); + } finally { + process.env.SESSION_SECRET = SECRET; + delete process.env.SESSION_SECRET_PREVIOUS; + } + }); + + it("refuses a short previous secret", async () => { + process.env.SESSION_SECRET_PREVIOUS = "too-short"; + try { + await expect(sealSession(session)).rejects.toThrow(/SESSION_SECRET_PREVIOUS/); + } finally { + delete process.env.SESSION_SECRET_PREVIOUS; + } + }); + it("rejects a tampered token", async () => { const sealed = await sealSession(session); const parts = sealed.split("."); diff --git a/tools/capacity-check.ts b/tools/capacity-check.ts new file mode 100644 index 0000000..4eab32a --- /dev/null +++ b/tools/capacity-check.ts @@ -0,0 +1,214 @@ +/** + * Capacity and latency check for one event of a chosen size, against the + * configured database: + * + * npm run capacity:check -- --attendees 300 --searches 60 --concurrency 15 --workers 2 + * + * It creates a throwaway organization and event with synthetic attendees, + * saves every profile (as people would), runs the real embed and match + * workers until the queue is quiet, then runs concurrent searches through + * the real use case. It reports matching throughput, search latency and + * model calls, compares them with the budgets in docs/OPERATIONS.md, and + * deletes everything it created (unless --keep). Exits 1 when a budget fails. + * + * Models: by default a free deterministic embedding and no rerank, which + * measures the system itself. `--providers configured` uses the deployment's + * real providers and costs money; run it on staging, not in CI. With + * `--workers 0` the deployment's workers do the matching and the check only + * waits; otherwise stop other workers first, or they share this event's jobs. + */ +import "dotenv/config"; +import { parseArgs } from "node:util"; +import { isMainModule } from "../packages/db/src/is-main"; + +export interface CapacityReport { + attendees: number; + searches: number; + /** Seconds until every attendee had a stored match list. */ + firstListsSeconds: number; + /** Seconds until no job was left, refreshes included. */ + quietSeconds: number; + attendeesWithoutMatches: number; + search: { p50Ms: number; p95Ms: number; maxMs: number; failures: number }; + modelCalls: { perAttendee: number; perSearch: number; approxTokens: number; failures: number }; +} + +export interface Budgets { + searchP95Ms: number; + firstListsSeconds: number; + quietSeconds: number; + modelCallsPerAttendee: number; + modelCallsPerSearch: number; +} + +/** Defaults agreed in docs/OPERATIONS.md for a 300-attendee event. */ +export const DEFAULT_BUDGETS: Budgets = { + searchP95Ms: 6000, + firstListsSeconds: 600, + quietSeconds: 1800, + modelCallsPerAttendee: 6, + modelCallsPerSearch: 2, +}; + +/** One message per budget the report misses; empty when all pass. */ +export function budgetFailures(report: CapacityReport, budgets: Budgets = DEFAULT_BUDGETS): string[] { + const failures: string[] = []; + const over = (label: string, actual: number, limit: number, unit: string) => { + if (actual > limit) failures.push(`${label} ${actual}${unit} is over the budget of ${limit}${unit}`); + }; + over("search p95", report.search.p95Ms, budgets.searchP95Ms, " ms"); + over("time to first match lists", report.firstListsSeconds, budgets.firstListsSeconds, " s"); + over("time until the queue is quiet", report.quietSeconds, budgets.quietSeconds, " s"); + over("model calls per attendee", report.modelCalls.perAttendee, budgets.modelCallsPerAttendee, ""); + over("model calls per search", report.modelCalls.perSearch, budgets.modelCallsPerSearch, ""); + if (report.search.failures) failures.push(`${report.search.failures} search(es) failed`); + if (report.attendeesWithoutMatches) failures.push(`${report.attendeesWithoutMatches} attendee(s) have no matches`); + return failures; +} + +export function percentile(values: number[], p: number): number { + if (!values.length) return 0; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)]; +} + +const WORDS = ["retrieval", "search", "climate", "health", "design", "rust", "compilers", "graph", "databases", "fintech", "payments", + "robotics", "vision", "education", "hiring", "mentoring", "startups", "security", "privacy", "mobile", "frontend", "data", "pipelines", "evaluation"]; +const DOMAINS = ["Artificial Intelligence", "Design", "Finance", "Healthcare", "Education", "Security"]; +const pick = (index: number, salt: number) => WORDS[(index * 7 + salt * 13) % WORDS.length]; + +export async function runCapacityCheck(options: { attendees: number; searches: number; concurrency: number; workers: number; keep?: boolean }): Promise { + // Imported late so provider choices made from the arguments apply. + const { attendees, events, query } = await import("@match-maker/db"); + const { processEmbedJob } = await import("../packages/db/workers/embed.worker"); + const { processMatchJob } = await import("../packages/db/workers/match.worker"); + const { getContainer } = await import("../apps/web/src/composition/container"); + const { searchParticipants } = await import("../apps/web/src/application/use-cases/search-participants"); + const { Intent } = await import("../apps/web/src/domain/participant/intent"); + const container = getContainer(); + + const run = `capacity-${Date.now().toString(36)}`; + const emailOf = (i: number) => `${run}-${i}@example.com`; + const started = new Date(); + const { rows: [org] } = await query<{ id: string }>(`insert into organizations (name, slug) values ($1, $1) returning id`, [run]); + const eventId = (await query<{ id: string }>( + `insert into events (organization_id, name, slug, status) values ($1, $2, $2, 'open') returning id`, [org.id, run])).rows[0].id; + try { + const people = Array.from({ length: options.attendees }, (_, i) => ({ name: `Capacity ${i}`, email: emailOf(i) })); + await attendees.upsertMany(people); + const { rows } = await query<{ id: string }>(`select id from registered_attendees where email like $1 order by email`, [`${run}-%`]); + const ids = rows.map((row) => row.id); + await events.attachRoster(eventId, ids); + const intents = [Intent.NETWORKING, Intent.HIRING, Intent.MENTORING]; + + const savedAt = Date.now(); + for (const [i, attendeeId] of ids.entries()) { + await container.participants.saveProfile({ + eventId, attendeeId, designation: "Engineer", domain: DOMAINS[i % DOMAINS.length], + aboutMe: `Works on ${pick(i, 1)} and ${pick(i, 2)} for ${pick(i, 3)} teams`, selfDescription: null, + yearsOfExperience: (i % 15) + 1, mostImpressiveBuild: null, intent: intents[i % intents.length], + interests: [pick(i, 4), pick(i, 5)], venueZone: null, + }); + } + + const listed = async () => Number((await query<{ count: string }>( + `select count(distinct participant_id) from event_matches where event_id = $1`, [eventId])).rows[0].count); + let firstListsSeconds = -1; + // With --workers 0 the deployment's own workers do the matching; this only waits. + const worker = async (processing: boolean) => { + for (;;) { + const worked = processing && ((await processEmbedJob(eventId)) || (await processMatchJob(eventId))); + if (worked) continue; + const open = await query(`select 1 from matching_jobs where event_id = $1 and status in ('pending', 'running') limit 1`, [eventId]); + if (!open.rows.length) return; + await new Promise((resolve) => setTimeout(resolve, 200)); + } + }; + const watcher = setInterval(async () => { + if (firstListsSeconds < 0 && await listed() >= ids.length) firstListsSeconds = Math.round((Date.now() - savedAt) / 1000); + }, 1000); + await Promise.all(options.workers > 0 ? Array.from({ length: options.workers }, () => worker(true)) : [worker(false)]); + clearInterval(watcher); + const quietSeconds = Math.round((Date.now() - savedAt) / 1000); + const withLists = await listed(); + if (firstListsSeconds < 0) firstListsSeconds = quietSeconds; + + const latencies: number[] = []; + let failures = 0; + let next = 0; + const searcher = async () => { + while (next < options.searches) { + const i = next++; + const begin = Date.now(); + try { + await searchParticipants({ eventId, participantId: ids[i % ids.length], query: `${pick(i, 6)} ${pick(i, 7)}`, eventIntents: intents }, + { pipeline: container.pipeline, participants: container.participants }); + } catch { + failures++; + } + latencies.push(Date.now() - begin); + } + }; + await Promise.all(Array.from({ length: options.concurrency }, searcher)); + + const usage = (await query<{ operation: string; calls: string; failures: string; chars: string }>( + `select operation, count(*) as calls, count(*) filter (where outcome <> 'ok') as failures, coalesce(sum(input_chars), 0) as chars + from model_usage where created_at >= $1 group by operation`, [started])).rows; + const callsOf = (prefix: string[]) => usage.filter((row) => prefix.includes(row.operation)).reduce((sum, row) => sum + Number(row.calls), 0); + const round = (value: number) => Math.round(value * 100) / 100; + return { + attendees: ids.length, + searches: options.searches, + firstListsSeconds, + quietSeconds, + attendeesWithoutMatches: ids.length - withLists, + search: { p50Ms: percentile(latencies, 50), p95Ms: percentile(latencies, 95), maxMs: Math.max(0, ...latencies), failures }, + modelCalls: { + perAttendee: round(callsOf(["embed:profile", "rerank"]) / Math.max(1, ids.length)), + perSearch: round(callsOf(["embed:query"]) / Math.max(1, options.searches)), + approxTokens: Math.round(usage.reduce((sum, row) => sum + Number(row.chars), 0) / 4), + failures: usage.reduce((sum, row) => sum + Number(row.failures), 0), + }, + }; + } finally { + if (!options.keep) { + await query(`delete from organizations where id = $1`, [org.id]); + await query(`delete from registered_attendees where email like $1`, [`${run}-%`]); + } + } +} + +if (isMainModule(import.meta.url)) { + const { values } = parseArgs({ + options: { + attendees: { type: "string", default: "300" }, + searches: { type: "string", default: "60" }, + concurrency: { type: "string", default: "15" }, + workers: { type: "string", default: "2" }, + providers: { type: "string", default: "fake" }, + keep: { type: "boolean", default: false }, + }, + }); + if (values.providers === "fake") { + process.env.EMBEDDING_PROVIDER = "hash"; + process.env.GEMINI_API_KEY = ""; + } else if (values.providers !== "configured") { + console.error("--providers must be fake or configured"); + process.exit(2); + } + process.env.MATCH_REFRESH_DELAY_SECONDS ??= "0"; + const { closePool } = await import("@match-maker/db"); + try { + const report = await runCapacityCheck({ + attendees: Number(values.attendees), searches: Number(values.searches), + concurrency: Number(values.concurrency), workers: Number(values.workers), keep: values.keep, + }); + console.info(JSON.stringify({ providers: values.providers, ...report }, null, 2)); + const failures = budgetFailures(report); + for (const failure of failures) console.error(`[capacity] FAIL ${failure}`); + if (!failures.length) console.info("[capacity] every budget met"); + process.exitCode = failures.length ? 1 : 0; + } finally { + await closePool(); + } +} diff --git a/tools/check-migrations.ts b/tools/check-migrations.ts new file mode 100644 index 0000000..5d0be4e --- /dev/null +++ b/tools/check-migrations.ts @@ -0,0 +1,78 @@ +/** + * Release gate for migrations, so a rollback to the previous image stays + * safe: migrations run before the new code, and the old code must keep + * working on the migrated schema. + * + * npx tsx tools/check-migrations.ts + * + * - A released migration (present at the base ref) never changes. + * - New migrations only expand the schema. Dropping or renaming a table or + * column, changing a column type or truncating breaks the previous image; + * such a statement needs `-- contract: ` on the same line, and belongs in a release after the code + * stopped using it. + * - Migration numbers are unique. + */ +import { execFileSync } from "node:child_process"; +import { readFileSync, readdirSync } from "node:fs"; +import { isMainModule } from "../packages/db/src/is-main"; + +const DIR = "packages/db/migrations"; +const BREAKING = [ + /\bdrop\s+table\b/i, + /\bdrop\s+column\b/i, + /\brename\s+(column\s+)?\w+\s+to\b/i, + /\brename\s+to\b/i, + /\balter\s+column\s+\w+\s+(set\s+data\s+)?type\b/i, + /\btruncate\b/i, + /\bdrop\s+schema\b/i, +]; + +const normalize = (sql: string) => sql.replace(/\r\n/g, "\n"); + +export function migrationProblems(base: Map, head: Map): string[] { + const problems: string[] = []; + for (const [name, sql] of base) { + if (!head.has(name)) problems.push(`${name} was released and must not be deleted`); + else if (normalize(head.get(name)!) !== normalize(sql)) problems.push(`${name} was released and must not change; add a new migration instead`); + } + for (const [name, sql] of head) { + if (base.has(name)) continue; + normalize(sql).split("\n").forEach((line, index) => { + const code = line.replace(/--.*$/, ""); + if (BREAKING.some((pattern) => pattern.test(code)) && !/--\s*contract:\s*\S/.test(line)) { + problems.push(`${name}:${index + 1} would break the previous release: "${line.trim()}" (mark a deliberate contract step with "-- contract: ")`); + } + }); + } + const numbers = new Map(); + for (const name of head.keys()) { + const number = name.split("_")[0]; + if (numbers.has(number)) problems.push(`${name} reuses number ${number} of ${numbers.get(number)}`); + numbers.set(number, name); + } + return problems; +} + +function filesAt(ref: string): Map { + execFileSync("git", ["rev-parse", "--verify", `${ref}^{commit}`], { stdio: "ignore" }); + // The base may predate the migrations folder: then nothing was released. + const listing = execFileSync("git", ["ls-tree", "-r", "--name-only", ref, "--", DIR], { encoding: "utf8" }); + const names = listing.split("\n").filter((path) => path.endsWith(".sql")).map((path) => path.slice(DIR.length + 1)); + return new Map(names.map((name) => [name, execFileSync("git", ["show", `${ref}:${DIR}/${name}`], { encoding: "utf8" })])); +} + +if (isMainModule(import.meta.url)) { + const ref = process.argv[2]; + const head = new Map(readdirSync(DIR).filter((name) => name.endsWith(".sql")).map((name) => [name, readFileSync(`${DIR}/${name}`, "utf8")])); + let base = new Map(); + if (!ref || /^0+$/.test(ref)) { + console.warn("[migrations] no base ref; checking new-migration rules for every file"); + } else { + base = filesAt(ref); + } + const problems = migrationProblems(base, head); + for (const problem of problems) console.error(`[migrations] ${problem}`); + console.info(problems.length ? `[migrations] ${problems.length} problem(s)` : `[migrations] ${head.size} migrations; ${head.size - base.size} new, all safe to roll back from`); + process.exitCode = problems.length ? 1 : 0; +} diff --git a/tools/restore-rehearsal.ts b/tools/restore-rehearsal.ts new file mode 100644 index 0000000..3176780 --- /dev/null +++ b/tools/restore-rehearsal.ts @@ -0,0 +1,65 @@ +/** + * Backup and restore rehearsal against the local docker compose Postgres: + * dumps the database, restores the dump into a fresh scratch database, + * verifies the copy (migrations, pgvector, core tables, row counts against + * the source), times each step, and removes the scratch database and dump. + * The source database is only read. + * + * npm run db:restore-rehearsal + * + * Production uses RDS snapshots and point-in-time restore instead; see + * "Restore the database" in docs/OPERATIONS.md, which ends with the same + * `npm run db:verify` check. + */ +import "dotenv/config"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, statSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { verifyDatabase } from "./verify-database"; + +const SCRATCH = process.env.RESTORE_REHEARSAL_DATABASE ?? "restore_rehearsal"; +const user = process.env.POSTGRES_USER ?? "matchmaker"; +const database = process.env.POSTGRES_DB ?? "matchmaker"; +const source = process.env.DATABASE_URL; + +function compose(args: string[], input?: Buffer): Buffer { + const result = spawnSync("docker", ["compose", "exec", "-T", "postgres", ...args], { input, maxBuffer: 1024 * 1024 * 1024 }); + if (result.status !== 0) throw new Error(`${args[0]} failed: ${result.stderr?.toString().trim() || result.error?.message}`); + return result.stdout; +} +const psql = (sql: string) => compose(["psql", "-U", user, "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-tAc", sql]).toString().trim(); +async function timed(label: string, step: () => T | Promise): Promise { + const started = Date.now(); + const result = await step(); + console.info(`[restore] ${label}: ${((Date.now() - started) / 1000).toFixed(1)} s`); + return result; +} + +if (!source) { + console.error("[restore] DATABASE_URL must point at the compose Postgres"); + process.exit(2); +} +if (psql(`select 1 from pg_database where datname = '${SCRATCH}'`)) { + console.error(`[restore] database ${SCRATCH} already exists; it is not ours to replace. Remove it or set RESTORE_REHEARSAL_DATABASE.`); + process.exit(2); +} + +const dir = mkdtempSync(join(tmpdir(), "mr-bridge-restore-")); +const dump = join(dir, "backup.dump"); +const started = Date.now(); +let ok = false; +try { + await timed("dump", () => writeFileSync(dump, compose(["pg_dump", "-U", user, "-d", database, "-Fc"]))); + console.info(`[restore] dump size: ${(statSync(dump).size / 1024 / 1024).toFixed(1)} MiB`); + await timed("create scratch database", () => psql(`create database ${SCRATCH}`)); + await timed("restore", () => compose(["pg_restore", "-U", user, "-d", SCRATCH, "--no-owner", "--exit-on-error"], readFileSync(dump))); + const restored = new URL(source); + restored.pathname = `/${SCRATCH}`; + ok = await timed("verify", () => verifyDatabase(restored.toString(), { compare: source })); +} finally { + psql(`drop database if exists ${SCRATCH}`); + rmSync(dir, { recursive: true, force: true }); +} +console.info(`[restore] ${ok ? "rehearsal passed" : "rehearsal FAILED"} in ${((Date.now() - started) / 1000).toFixed(1)} s`); +process.exitCode = ok ? 0 : 1; diff --git a/tools/verify-database.ts b/tools/verify-database.ts new file mode 100644 index 0000000..5fb3e0c --- /dev/null +++ b/tools/verify-database.ts @@ -0,0 +1,106 @@ +/** + * Checks that a database is a usable Mr. Bridge database: reachable, with + * pgvector, every migration in this checkout applied unchanged, and the core + * tables present. With --compare it also compares row counts with another + * database, e.g. a restored copy against its source: + * + * npm run db:verify -- --url --compare [--strict] + * + * Without --url it checks DATABASE_URL. Counts may differ a little after a + * point-in-time restore of a live database; --strict makes any difference fail. + */ +import "dotenv/config"; +import { createHash } from "node:crypto"; +import { readFileSync, readdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import pg from "pg"; +import { resolveSsl } from "../packages/db/src/client"; +import { isMainModule } from "../packages/db/src/is-main"; + +const MIGRATIONS_DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "packages", "db", "migrations"); + +/** The tables the journey cannot work without; their row counts are compared. */ +export const CORE_TABLES = [ + "organizations", "events", "registered_attendees", "event_participants", "event_matches", + "matching_jobs", "intro_requests", "notification_outbox", "location_requests", "audit_events", +]; + +export interface DatabaseState { + vector: boolean; + /** Migration name -> checksum, as recorded in schema_migrations. */ + applied: Map; + counts: Record; +} + +export async function readState(url: string): Promise { + const client = new pg.Client({ connectionString: url, ssl: resolveSsl(url) }); + await client.connect(); + try { + const vector = (await client.query(`select 1 from pg_extension where extname = 'vector'`)).rows.length > 0; + const migrations = await client.query<{ name: string; checksum: string }>(`select name, checksum from schema_migrations`).catch(() => ({ rows: [] })); + const counts: Record = {}; + for (const table of CORE_TABLES) { + const exists = (await client.query(`select to_regclass($1) as name`, [`public.${table}`])).rows[0].name; + counts[table] = exists ? Number((await client.query(`select count(*) from ${table}`)).rows[0].count) : null; + } + return { vector, applied: new Map(migrations.rows.map((row) => [row.name, row.checksum])), counts }; + } finally { + await client.end(); + } +} + +/** + * Migration files in this checkout with their accepted checksums. The runner + * hashes the file as checked out, so a database migrated on Linux (LF) and + * checked from a Windows checkout (CRLF) is the same file: both are accepted. + */ +export function expectedMigrations(dir = MIGRATIONS_DIR): Map { + const sha = (text: string) => createHash("sha256").update(text).digest("hex"); + return new Map(readdirSync(dir).filter((file) => file.endsWith(".sql")).sort().map((file) => { + const lf = readFileSync(join(dir, file), "utf8").replace(/\r\n/g, "\n"); + return [file, [sha(lf), sha(lf.replace(/\n/g, "\r\n"))]]; + })); +} + +/** Problems that make the database unusable, then count differences against `source`. */ +export function compareState(state: DatabaseState, expected: Map, source?: DatabaseState): { problems: string[]; differences: string[] } { + const problems: string[] = []; + if (!state.vector) problems.push("the pgvector extension is missing"); + for (const [name, checksums] of expected) { + const applied = state.applied.get(name); + if (!applied) problems.push(`migration ${name} is not applied`); + else if (!checksums.includes(applied)) problems.push(`migration ${name} was applied from a different file`); + } + for (const [table, count] of Object.entries(state.counts)) { + if (count === null) problems.push(`table ${table} is missing`); + } + const differences = source + ? Object.keys(state.counts).filter((table) => state.counts[table] !== source.counts[table]) + .map((table) => `${table}: ${state.counts[table]} here, ${source.counts[table]} in the source`) + : []; + return { problems, differences }; +} + +export async function verifyDatabase(url: string, options: { compare?: string; strict?: boolean } = {}): Promise { + const state = await readState(url); + const source = options.compare ? await readState(options.compare) : undefined; + const { problems, differences } = compareState(state, expectedMigrations(), source); + console.info(`[db:verify] ${state.applied.size} migrations applied; rows: ${CORE_TABLES.map((table) => `${table}=${state.counts[table] ?? "missing"}`).join(" ")}`); + for (const problem of problems) console.error(`[db:verify] FAIL ${problem}`); + for (const difference of differences) console[options.strict ? "error" : "warn"](`[db:verify] ${options.strict ? "FAIL" : "differs"} ${difference}`); + const ok = problems.length === 0 && (!options.strict || differences.length === 0); + console.info(ok ? "[db:verify] database is usable" : "[db:verify] database check failed"); + return ok; +} + +if (isMainModule(import.meta.url)) { + const { values } = parseArgs({ options: { url: { type: "string" }, compare: { type: "string" }, strict: { type: "boolean", default: false } } }); + const url = values.url ?? process.env.DATABASE_URL; + if (!url) { + console.error("[db:verify] pass --url or set DATABASE_URL"); + process.exit(2); + } + process.exitCode = (await verifyDatabase(url, { compare: values.compare, strict: values.strict })) ? 0 : 1; +} diff --git a/tsconfig.base.json b/tsconfig.base.json index 10c8d14..faa8804 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -6,11 +6,10 @@ "module": "ESNext", "moduleResolution": "Bundler", "resolveJsonModule": true, - "baseUrl": ".", "paths": { - "@match-maker/db": ["packages/db/src/index.ts"], - "@match-maker/auth": ["packages/auth/src/index.ts"], - "@match-maker/email": ["packages/email/src/index.ts"] + "@match-maker/db": ["./packages/db/src/index.ts"], + "@match-maker/auth": ["./packages/auth/src/index.ts"], + "@match-maker/email": ["./packages/email/src/index.ts"] }, "allowJs": true, "checkJs": false,