From 3a8a501b7877273c05f314ad9740580793ea5b5a Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 09:05:30 +0530 Subject: [PATCH 01/26] Coalesce search retrieval across replicas with complete cache keys (R20) Concurrent identical searches each embedded the query and retrieved candidates; the cache key lacked the corpus state, embedding space and limit, and a profile change could serve candidates from the old corpus. - Migration 0011: cache_locks (expiring, owner-only release) and events.corpus_revision, bumped by statement-level triggers whenever an event's participants change (one bump per statement, so large imports stay cheap) - application/shared-computation.ts computeOnce: the first caller takes the lock and computes; others wait up to 2 seconds for its result, then compute themselves. Cache or lock failures fall back to computing directly, so an outage costs work but never fails a search - Retrieval keys: retrieval version, event, corpus revision, normalized query, filters, embedding provider/model (or lexical) and limit; results are re-checked for eligibility and filters after lookup - Query embeddings are cached per provider/model and normalized query for a day and coalesced the same way - CandidateCache gains vectors and locks in both the Postgres and Upstash adapters (Redis SET NX PX and compare-and-delete); embedding providers expose their id; the sweep clears expired locks - Tests: coalescing, waiting, fallbacks; real Postgres locks, vectors, and two pipeline instances coalescing one search while ranking personally, with fresh retrieval after a corpus or filter change --- .../src/application/ports/candidate-cache.ts | 11 ++ .../application/ports/embedding-provider.ts | 2 + .../ports/participant-repository.ts | 2 + .../web/src/application/shared-computation.ts | 64 +++++++++++ apps/web/src/domain/matching/pipeline.ts | 21 +++- .../domain/matching/stages/retrieve.stage.ts | 53 +++++---- .../cache/postgres-candidate-cache.ts | 43 ++++++- .../cache/upstash-candidate-cache.ts | 62 ++++++++-- .../postgres/participant.repo.ts | 5 + .../providers/gemini-embedding.provider.ts | 2 + .../providers/local-bge.provider.ts | 2 + .../db/migrations/0011_shared_retrieval.sql | 42 +++++++ packages/db/src/repositories/housekeeping.ts | 1 + tests/integration/shared-retrieval.test.ts | 107 ++++++++++++++++++ tests/unit/candidate-eligibility.test.ts | 9 +- tests/unit/search.test.ts | 8 +- tests/unit/shared-computation.test.ts | 67 +++++++++++ 17 files changed, 459 insertions(+), 42 deletions(-) create mode 100644 apps/web/src/application/shared-computation.ts create mode 100644 packages/db/migrations/0011_shared_retrieval.sql create mode 100644 tests/integration/shared-retrieval.test.ts create mode 100644 tests/unit/shared-computation.test.ts diff --git a/apps/web/src/application/ports/candidate-cache.ts b/apps/web/src/application/ports/candidate-cache.ts index 5c5a992..1dc1a0f 100644 --- a/apps/web/src/application/ports/candidate-cache.ts +++ b/apps/web/src/application/ports/candidate-cache.ts @@ -1,4 +1,15 @@ +/** + * Shared cache for search retrieval, visible to every web replica: candidate + * id lists, query embeddings, and short ownership locks that let concurrent + * identical searches compute once. + */ export interface CandidateCache { get(key: string): Promise; set(key: string, candidateIds: string[], ttlSeconds: number): Promise; + getVector(key: string): Promise; + setVector(key: string, vector: number[], ttlSeconds: number): Promise; + /** Takes the lock if free or expired; returns an owner token, or null when someone else holds it. */ + acquire(key: string, ttlMs: number): Promise; + /** Releases the lock only if `token` still owns it. */ + release(key: string, token: string): Promise; } diff --git a/apps/web/src/application/ports/embedding-provider.ts b/apps/web/src/application/ports/embedding-provider.ts index 3054a86..4ee3b2a 100644 --- a/apps/web/src/application/ports/embedding-provider.ts +++ b/apps/web/src/application/ports/embedding-provider.ts @@ -1,3 +1,5 @@ export interface EmbeddingProvider { + /** Provider and model; vectors from different ids are different spaces and never mixed. */ + readonly id: string; embed(text: string, task: "profile" | "query"): Promise; } diff --git a/apps/web/src/application/ports/participant-repository.ts b/apps/web/src/application/ports/participant-repository.ts index 0bab47e..7d26773 100644 --- a/apps/web/src/application/ports/participant-repository.ts +++ b/apps/web/src/application/ports/participant-repository.ts @@ -13,6 +13,8 @@ export interface ParticipantRepository { filters?: SearchFilters; limit?: number; }): Promise; + /** Changes whenever the event's participants change; part of retrieval cache keys. */ + corpusRevision(eventId: string): Promise; /** Domains present among eligible participants, for the search filter. */ listDomains(eventId: string): Promise; findCandidatesByIds(eventId: string, ids: string[]): Promise; diff --git a/apps/web/src/application/shared-computation.ts b/apps/web/src/application/shared-computation.ts new file mode 100644 index 0000000..20b782d --- /dev/null +++ b/apps/web/src/application/shared-computation.ts @@ -0,0 +1,64 @@ +import type { CandidateCache } from "./ports/candidate-cache"; + +/** How long a computation may hold its lock, and how long others wait for its result. */ +export const SHARED_LOCK_MS = 10_000; +export const SHARED_WAIT_MS = 2_000; +const POLL_MS = 100; + +/** + * Returns a cached value, or computes it once across replicas: the first + * caller takes the lock and computes; the others wait briefly for its result + * and then compute themselves. Any cache or lock failure falls back to + * computing directly, so a cache outage costs work but never fails a search. + */ +export async function computeOnce(input: { + cache: CandidateCache; + key: string; + read: () => Promise; + write: (value: T) => Promise; + compute: () => Promise; + /** Values not worth sharing (e.g. a missing embedding) are returned but not written. */ + shouldStore?: (value: T) => boolean; + waitMs?: number; +}): Promise { + const read = () => input.read().catch((error) => { + console.warn("[cache] read failed; computing directly", error); + return null; + }); + const store = async (value: T) => { + if (input.shouldStore && !input.shouldStore(value)) return; + await input.write(value).catch((error) => console.warn("[cache] write failed", error)); + }; + + const cached = await read(); + if (cached !== null) return cached; + + const token = await input.cache.acquire(`compute:${input.key}`, SHARED_LOCK_MS).catch((error) => { + console.warn("[cache] lock unavailable; computing directly", error); + return undefined; + }); + if (token === undefined) return input.compute(); + + if (token) { + try { + const raced = await read(); + if (raced !== null) return raced; + const value = await input.compute(); + await store(value); + return value; + } finally { + await input.cache.release(`compute:${input.key}`, token).catch(() => {}); + } + } + + // Someone else is computing: wait for their result, then stop waiting. + const deadline = Date.now() + (input.waitMs ?? SHARED_WAIT_MS); + while (Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, POLL_MS)); + const shared = await read(); + if (shared !== null) return shared; + } + const value = await input.compute(); + await store(value); + return value; +} diff --git a/apps/web/src/domain/matching/pipeline.ts b/apps/web/src/domain/matching/pipeline.ts index ca04f62..11c3c4e 100644 --- a/apps/web/src/domain/matching/pipeline.ts +++ b/apps/web/src/domain/matching/pipeline.ts @@ -2,6 +2,8 @@ import type { EmbeddingProvider } from "../../application/ports/embedding-provid import type { CandidateCache } from "../../application/ports/candidate-cache"; import type { ExplanationCache } from "../../application/ports/explanation-cache"; import { queryContext } from "./explanation-key"; +import { computeOnce } from "../../application/shared-computation"; +import { createHash } from "node:crypto"; import type { LlmProvider } from "../../application/ports/llm-provider"; import type { MatchRepository } from "../../application/ports/match-repository"; import type { ParticipantRepository } from "../../application/ports/participant-repository"; @@ -62,7 +64,7 @@ export class MatchingPipeline { matches: MatchRepository, private readonly embeddings: EmbeddingProvider, llm: LlmProvider | null, - cache: CandidateCache, + private readonly cache: CandidateCache, explanations: ExplanationCache | null = null ) { this.retrieve = new RetrieveStage(participants, cache); @@ -104,10 +106,24 @@ export class MatchingPipeline { * and measures relevance; the searcher's own profile measures compatibility. * Both feed the score, so the same query ranks differently per person. */ + /** Query embeddings are shared across replicas and searches for a day. */ + private embedQuery(query: string): Promise { + const normalized = query.trim().toLowerCase().replace(/\s+/g, " "); + const key = createHash("sha256").update(JSON.stringify(["query-embedding", this.embeddings.id, normalized])).digest("hex"); + return computeOnce({ + cache: this.cache, + key, + read: () => this.cache.getVector(key), + write: (vector) => vector ? this.cache.setVector(key, vector, 24 * 60 * 60) : Promise.resolve(), + compute: () => this.embeddings.embed(normalized, "query"), + shouldStore: (vector) => vector !== null, + }); + } + private async search(input: MatchingInput, viewer: ParticipantProfile) { const query = input.query!.trim(); const terms = queryTerms(query); - const queryEmbedding = await this.embeddings.embed(query, "query"); + const queryEmbedding = await this.embedQuery(query); const retrieved = await this.retrieve.execute({ eventId: input.eventId, viewerId: input.participantId, @@ -115,6 +131,7 @@ export class MatchingPipeline { query, terms, filters: input.filters ?? NO_FILTERS, + embeddingSpace: this.embeddings.id, limit: 50, }); const compatible = this.filter.execute(viewer, retrieved); diff --git a/apps/web/src/domain/matching/stages/retrieve.stage.ts b/apps/web/src/domain/matching/stages/retrieve.stage.ts index f544a85..019b01d 100644 --- a/apps/web/src/domain/matching/stages/retrieve.stage.ts +++ b/apps/web/src/domain/matching/stages/retrieve.stage.ts @@ -4,6 +4,10 @@ import type { MatchRepository } from "../../../application/ports/match-repositor import type { CandidateCache } from "../../../application/ports/candidate-cache"; import type { ParticipantProfile } from "../../participant/participant.entity"; import { matchesFilters, NO_FILTERS, type SearchFilters } from "../search-query"; +import { computeOnce } from "../../../application/shared-computation"; + +/** Bump when retrieval semantics change, so cached candidate lists are not reused. */ +const RETRIEVAL_VERSION = "r2"; export class RetrieveStage { constructor( @@ -18,35 +22,40 @@ export class RetrieveStage { query?: string; terms?: string[]; filters?: SearchFilters; + /** Embedding provider/model; vectors from different spaces must not share entries. */ + embeddingSpace?: string; limit?: number; }): Promise { const limit = input.limit ?? 50; const filters = input.filters ?? NO_FILTERS; const normalizedQuery = input.query?.trim().toLowerCase().replace(/\s+/g, " "); if (normalizedQuery) { - // Shared across viewers for the same event, query, filters and retrieval - // kind; the ranking that follows is still personal. - const key = createHash("sha256") - .update(JSON.stringify([input.eventId, normalizedQuery, filters.intent, filters.domain?.toLowerCase() ?? null, input.embedding ? "semantic" : "lexical"])) - .digest("hex"); - const cachedIds = await this.cache.get(key); - if (cachedIds) { - const cached = await this.participants.findCandidatesByIds(input.eventId, cachedIds); - const byId = new Map(cached.map((profile) => [profile.attendeeId, profile])); - return cachedIds.map((id) => byId.get(id)) - .filter((row): row is ParticipantProfile => row !== undefined && row.attendeeId !== input.viewerId && matchesFilters(row, filters)); - } - - const candidates = await this.participants.listCandidates({ - eventId: input.eventId, - excludeAttendeeId: null, - embedding: input.embedding, - terms: input.terms, - filters, - limit, + // Shared across viewers for the same corpus, query, filters and + // retrieval kind; the ranking that follows is still personal. + const corpus = await this.participants.corpusRevision(input.eventId); + const key = createHash("sha256").update(JSON.stringify([ + RETRIEVAL_VERSION, input.eventId, corpus, normalizedQuery, filters.intent, filters.domain?.toLowerCase() ?? null, + input.embedding ? input.embeddingSpace ?? "semantic" : "lexical", limit, + ])).digest("hex"); + const ids = await computeOnce({ + cache: this.cache, + key, + read: () => this.cache.get(key), + write: (value) => this.cache.set(key, value, 300), + compute: async () => (await this.participants.listCandidates({ + eventId: input.eventId, + excludeAttendeeId: null, + embedding: input.embedding, + terms: input.terms, + filters, + limit, + })).map((candidate) => candidate.attendeeId), }); - await this.cache.set(key, candidates.map((c) => c.attendeeId), 300); - return candidates.filter((candidate) => candidate.attendeeId !== input.viewerId && matchesFilters(candidate, filters)); + // Re-check eligibility and filters: the ids may come from another request. + const profiles = await this.participants.findCandidatesByIds(input.eventId, ids); + const byId = new Map(profiles.map((profile) => [profile.attendeeId, profile])); + return ids.map((id) => byId.get(id)) + .filter((row): row is ParticipantProfile => row !== undefined && row.attendeeId !== input.viewerId && matchesFilters(row, filters)); } return this.participants.listCandidates({ diff --git a/apps/web/src/infrastructure/cache/postgres-candidate-cache.ts b/apps/web/src/infrastructure/cache/postgres-candidate-cache.ts index 90971b8..3478911 100644 --- a/apps/web/src/infrastructure/cache/postgres-candidate-cache.ts +++ b/apps/web/src/infrastructure/cache/postgres-candidate-cache.ts @@ -1,20 +1,55 @@ +import { randomUUID } from "node:crypto"; import { query } from "@match-maker/db"; import type { CandidateCache } from "../../application/ports/candidate-cache"; export class PostgresCandidateCache implements CandidateCache { async get(key: string): Promise { - const { rows } = await query<{ candidate_ids: string[] }>( + const value = await this.read(key); + return Array.isArray(value) && value.every((id) => typeof id === "string") ? value : null; + } + + async set(key: string, candidateIds: string[], ttlSeconds: number): Promise { + await this.write(key, candidateIds, ttlSeconds); + } + + async getVector(key: string): Promise { + const value = await this.read(key); + return Array.isArray(value) && value.length > 0 && value.every((n) => typeof n === "number") ? value : null; + } + + async setVector(key: string, vector: number[], ttlSeconds: number): Promise { + await this.write(key, vector, ttlSeconds); + } + + async acquire(key: string, ttlMs: number): Promise { + const token = randomUUID(); + const { rows } = await query<{ owner: string }>( + `insert into cache_locks (lock_key, owner, expires_at) values ($1, $2, now() + make_interval(secs => $3)) + on conflict (lock_key) do update set owner = excluded.owner, expires_at = excluded.expires_at + where cache_locks.expires_at < now() + returning owner`, + [key, token, ttlMs / 1000] + ); + return rows[0]?.owner === token ? token : null; + } + + async release(key: string, token: string): Promise { + await query(`delete from cache_locks where lock_key = $1 and owner = $2`, [key, token]); + } + + private async read(key: string): Promise { + const { rows } = await query<{ candidate_ids: unknown }>( `select candidate_ids from retrieval_cache where cache_key = $1 and expires_at > now()`, [key] ); return rows[0]?.candidate_ids ?? null; } - async set(key: string, candidateIds: string[], ttlSeconds: number): Promise { + private async write(key: string, value: unknown, ttlSeconds: number): Promise { await query( `insert into retrieval_cache (cache_key, candidate_ids, expires_at) - values ($1, $2::jsonb, now() + ($3 || ' seconds')::interval) + values ($1, $2::jsonb, now() + make_interval(secs => $3)) on conflict (cache_key) do update set candidate_ids = excluded.candidate_ids, expires_at = excluded.expires_at`, - [key, JSON.stringify(candidateIds), String(ttlSeconds)] + [key, JSON.stringify(value), ttlSeconds] ); } } diff --git a/apps/web/src/infrastructure/cache/upstash-candidate-cache.ts b/apps/web/src/infrastructure/cache/upstash-candidate-cache.ts index 3e6ca87..a5ec2ee 100644 --- a/apps/web/src/infrastructure/cache/upstash-candidate-cache.ts +++ b/apps/web/src/infrastructure/cache/upstash-candidate-cache.ts @@ -1,29 +1,71 @@ +import { randomUUID } from "node:crypto"; import type { CandidateCache } from "../../application/ports/candidate-cache"; -/** Redis REST adapter. Falls back to the local PostgreSQL cache when unset. */ +/** Deletes the lock only when it still holds this owner's token. */ +const RELEASE_SCRIPT = `if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) else return 0 end`; + +/** Redis REST adapter. Falls back to the local PostgreSQL cache when unset or unreachable. */ export class UpstashCandidateCache implements CandidateCache { constructor(private readonly fallback: CandidateCache) {} async get(key: string): Promise { - if (!this.configured()) return this.fallback.get(key); + const parsed = await this.readJson(key, () => this.fallback.get(key)); + return Array.isArray(parsed) && parsed.every((id) => typeof id === "string") ? parsed : null; + } + + async set(key: string, candidateIds: string[], ttlSeconds: number): Promise { + await this.writeJson(key, candidateIds, ttlSeconds, () => this.fallback.set(key, candidateIds, ttlSeconds)); + } + + async getVector(key: string): Promise { + const parsed = await this.readJson(key, () => this.fallback.getVector(key)); + return Array.isArray(parsed) && parsed.length > 0 && parsed.every((n) => typeof n === "number") ? parsed : null; + } + + async setVector(key: string, vector: number[], ttlSeconds: number): Promise { + await this.writeJson(key, vector, ttlSeconds, () => this.fallback.setVector(key, vector, ttlSeconds)); + } + + async acquire(key: string, ttlMs: number): Promise { + if (!this.configured()) return this.fallback.acquire(key, ttlMs); + try { + const token = randomUUID(); + return await this.command(["SET", `lock:${key}`, token, "NX", "PX", String(ttlMs)]) === "OK" ? token : null; + } catch (error) { + console.warn("[cache] Redis lock failed; using PostgreSQL lock", error); + return this.fallback.acquire(key, ttlMs); + } + } + + async release(key: string, token: string): Promise { + if (!this.configured()) return this.fallback.release(key, token); + try { + await this.command(["EVAL", RELEASE_SCRIPT, "1", `lock:${key}`, token]); + } catch (error) { + // The lock expires on its own; releasing early is only an optimisation. + console.warn("[cache] Redis unlock failed", error); + await this.fallback.release(key, token).catch(() => {}); + } + } + + private async readJson(key: string, fallback: () => Promise): Promise { + if (!this.configured()) return fallback(); try { const result = await this.command(["GET", key]); - if (typeof result !== "string") return null; - const parsed: unknown = JSON.parse(result); - return Array.isArray(parsed) && parsed.every((id) => typeof id === "string") ? parsed : null; + return typeof result === "string" ? JSON.parse(result) : null; } catch (error) { console.warn("[cache] Redis GET failed; using PostgreSQL cache", error); - return this.fallback.get(key); + return fallback(); } } - async set(key: string, candidateIds: string[], ttlSeconds: number): Promise { - if (!this.configured()) return this.fallback.set(key, candidateIds, ttlSeconds); + private async writeJson(key: string, value: unknown, ttlSeconds: number, fallback: () => Promise): Promise { + if (!this.configured()) return fallback(); try { - await this.command(["SET", key, JSON.stringify(candidateIds), "EX", String(ttlSeconds)]); + await this.command(["SET", key, JSON.stringify(value), "EX", String(ttlSeconds)]); } catch (error) { console.warn("[cache] Redis SET failed; using PostgreSQL cache", error); - await this.fallback.set(key, candidateIds, ttlSeconds); + await fallback(); } } diff --git a/apps/web/src/infrastructure/postgres/participant.repo.ts b/apps/web/src/infrastructure/postgres/participant.repo.ts index 6063788..3b2d48c 100644 --- a/apps/web/src/infrastructure/postgres/participant.repo.ts +++ b/apps/web/src/infrastructure/postgres/participant.repo.ts @@ -108,6 +108,11 @@ export class PostgresParticipantRepository implements ParticipantRepository { return rows.map(mapProfile); } + async corpusRevision(eventId: string): Promise { + const { rows } = await query<{ corpus_revision: string }>(`select corpus_revision from events where id = $1`, [eventId]); + return Number(rows[0]?.corpus_revision ?? 0); + } + async listDomains(eventId: string): Promise { const { rows } = await query<{ domain: string }>( // One entry per domain ignoring case, preferring a capitalised spelling. diff --git a/apps/web/src/infrastructure/providers/gemini-embedding.provider.ts b/apps/web/src/infrastructure/providers/gemini-embedding.provider.ts index 1aa475f..940e965 100644 --- a/apps/web/src/infrastructure/providers/gemini-embedding.provider.ts +++ b/apps/web/src/infrastructure/providers/gemini-embedding.provider.ts @@ -3,6 +3,8 @@ import type { EmbeddingProvider } from "../../application/ports/embedding-provid const DIMENSIONS = 768; export class GeminiEmbeddingProvider implements EmbeddingProvider { + readonly id = `gemini:gemini-embedding-001:${DIMENSIONS}`; + async embed(text: string, task: "profile" | "query"): Promise { const apiKey = process.env.GEMINI_API_KEY; if (!apiKey || !text.trim()) return null; diff --git a/apps/web/src/infrastructure/providers/local-bge.provider.ts b/apps/web/src/infrastructure/providers/local-bge.provider.ts index 55de0a1..ea7fa99 100644 --- a/apps/web/src/infrastructure/providers/local-bge.provider.ts +++ b/apps/web/src/infrastructure/providers/local-bge.provider.ts @@ -2,6 +2,8 @@ import type { EmbeddingProvider } from "../../application/ports/embedding-provid /** Adapter for an internal sentence-transformers inference endpoint. */ export class LocalBgeProvider implements EmbeddingProvider { + readonly id = "local-bge:768"; + async embed(text: string): Promise { const endpoint = process.env.LOCAL_BGE_URL; if (!endpoint) return null; diff --git a/packages/db/migrations/0011_shared_retrieval.sql b/packages/db/migrations/0011_shared_retrieval.sql new file mode 100644 index 0000000..82de180 --- /dev/null +++ b/packages/db/migrations/0011_shared_retrieval.sql @@ -0,0 +1,42 @@ +-- Shared search retrieval across web replicas. +-- +-- cache_locks: short-lived ownership of one expensive computation (a query +-- embedding or a retrieval), so concurrent identical searches on different +-- replicas compute it once. An expired lock can be taken over; only its +-- owner can release it. +-- +-- events.corpus_revision: bumped whenever an event's participants change +-- (added, removed, profile/visibility/access/embedding updated). It is part +-- of every retrieval cache key, so a change sends new searches to a fresh +-- entry instead of serving candidates retrieved from the old corpus. + +create table if not exists cache_locks ( + lock_key text primary key, + owner text not null, + expires_at timestamptz not null +); + +alter table events add column if not exists corpus_revision bigint not null default 0; + +create or replace function bump_corpus_revision() returns trigger language plpgsql as $$ +begin + if tg_op = 'DELETE' then + update events set corpus_revision = corpus_revision + 1 where id in (select distinct event_id from old_rows); + else + update events set corpus_revision = corpus_revision + 1 where id in (select distinct event_id from new_rows); + end if; + return null; +end $$; + +-- One bump per statement and event, so a 10,000-row import is a single update. +drop trigger if exists event_participants_corpus_insert on event_participants; +create trigger event_participants_corpus_insert after insert on event_participants + referencing new table as new_rows for each statement execute function bump_corpus_revision(); + +drop trigger if exists event_participants_corpus_update on event_participants; +create trigger event_participants_corpus_update after update on event_participants + referencing new table as new_rows for each statement execute function bump_corpus_revision(); + +drop trigger if exists event_participants_corpus_delete on event_participants; +create trigger event_participants_corpus_delete after delete on event_participants + referencing old table as old_rows for each statement execute function bump_corpus_revision(); diff --git a/packages/db/src/repositories/housekeeping.ts b/packages/db/src/repositories/housekeeping.ts index 36f995e..7fe1803 100644 --- a/packages/db/src/repositories/housekeeping.ts +++ b/packages/db/src/repositories/housekeeping.ts @@ -27,6 +27,7 @@ export async function sweepExpired(): Promise { ); const cache = await query(`delete from retrieval_cache where expires_at < now()`); const explanations = await query(`delete from pair_explanations where expires_at < now()`); + await query(`delete from cache_locks where expires_at < now()`); const audits = await query(`delete from audit_events where created_at < now() - make_interval(days => $1)`, [AUDIT_RETENTION_DAYS]); return { loginChallenges, rateLimitBuckets: buckets.rowCount ?? 0, retrievalCache: cache.rowCount ?? 0, diff --git a/tests/integration/shared-retrieval.test.ts b/tests/integration/shared-retrieval.test.ts new file mode 100644 index 0000000..656b917 --- /dev/null +++ b/tests/integration/shared-retrieval.test.ts @@ -0,0 +1,107 @@ +/** + * Shared search retrieval against a real Postgres: lock ownership, query + * embedding reuse, and two pipeline instances (two "replicas") coalescing an + * identical search while still ranking personally. Uses its own organization, + * event and attendees. Skipped when DATABASE_URL is unset. + */ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { attendees, closePool, events, query } from "@match-maker/db"; +import { migrate } from "../../packages/db/src/migrate"; +import { PostgresCandidateCache } from "../../apps/web/src/infrastructure/cache/postgres-candidate-cache"; +import { PostgresParticipantRepository } from "../../apps/web/src/infrastructure/postgres/participant.repo"; +import { MatchingPipeline } from "../../apps/web/src/domain/matching/pipeline"; + +const describeDb = process.env.DATABASE_URL ? describe : describe.skip; +const RUN = Date.now().toString(36); +const ORG = `shared-${RUN}`; +const email = (key: string) => `shared-${key}-${RUN}@example.com`; +const PEOPLE: Record = { + ana: { domain: "AI", interests: ["search"] }, + ben: { domain: "Design", interests: ["climate"] }, + cy: { domain: "AI", interests: ["search", "rag"] }, + dee: { domain: "Design", interests: ["climate", "ux"] }, +}; + +describeDb("shared search retrieval", () => { + const cache = new PostgresCandidateCache(); + const ids: Record = {}; + let eventId: string; + + beforeAll(async () => { + await migrate(); + await attendees.upsertMany(Object.keys(PEOPLE).map((key) => ({ name: `Shared ${key}`, email: email(key) }))); + for (const key of Object.keys(PEOPLE)) ids[key] = (await attendees.findByEmail(email(key)))!.id; + const { rows: [org] } = await query<{ id: string }>(`insert into organizations (name, slug) values ($1, $1) returning id`, [ORG]); + eventId = (await query<{ id: string }>(`insert into events (organization_id, name, slug, status) values ($1, $2, $2, 'open') returning id`, [org.id, ORG])).rows[0].id; + await events.attachRoster(eventId, Object.values(ids)); + for (const [key, person] of Object.entries(PEOPLE)) { + await query(`update event_participants set domain = $3, interests = $4, about_me = 'Builds products for people', designation = 'Builder', profile_updated_at = now() + where event_id = $1 and attendee_id = $2`, [eventId, ids[key], person.domain, person.interests]); + } + }); + + afterAll(async () => { + await query(`delete from organizations where slug = $1`, [ORG]); + await query(`delete from registered_attendees where email like $1`, [`shared-%-${RUN}@example.com`]); + await closePool(); + }); + + it("gives a lock to one owner at a time, and only the owner releases it", async () => { + const key = `test:${RUN}:lock`; + const first = await cache.acquire(key, 5000); + expect(first).toBeTruthy(); + expect(await cache.acquire(key, 5000)).toBeNull(); + await cache.release(key, "not-the-owner"); + expect(await cache.acquire(key, 5000)).toBeNull(); + await cache.release(key, first!); + const second = await cache.acquire(key, 5000); + expect(second).toBeTruthy(); + await cache.release(key, second!); + }); + + it("lets an expired lock be taken over", async () => { + const key = `test:${RUN}:expired`; + await cache.acquire(key, 5000); + await query(`update cache_locks set expires_at = now() - interval '1 second' where lock_key = $1`, [key]); + const token = await cache.acquire(key, 5000); + expect(token).toBeTruthy(); + await cache.release(key, token!); + }); + + it("stores and reads query vectors", async () => { + await cache.setVector(`test:${RUN}:vector`, [0.1, 0.2], 60); + expect(await cache.getVector(`test:${RUN}:vector`)).toEqual([0.1, 0.2]); + }); + + /** A pipeline as one web replica would build it, counting retrievals. */ + function replica() { + const participants = new PostgresParticipantRepository(); + const retrievals = vi.spyOn(participants, "listCandidates"); + const embeddings = { id: "none", embed: vi.fn(async () => null) }; + const matches = { save: vi.fn(), listForParticipant: vi.fn() }; + return { pipeline: new MatchingPipeline(participants, matches, embeddings, null, cache), retrievals }; + } + const search = (pipeline: MatchingPipeline, viewer: string, text: string, domain: string | null = null) => + pipeline.run({ eventId, participantId: ids[viewer], mode: "search", query: text, filters: { intent: null, domain } }) + .then((ranked) => ranked.map((match) => match.participant.name)); + + it("coalesces concurrent identical searches across replicas, but ranks personally", async () => { + const one = replica(); + const two = replica(); + const [forAna, forBen] = await Promise.all([search(one.pipeline, "ana", "builds products"), search(two.pipeline, "ben", "builds products")]); + expect(one.retrievals.mock.calls.length + two.retrievals.mock.calls.length).toBe(1); + expect(forAna[0]).toBe("Shared cy"); + expect(forBen[0]).toBe("Shared dee"); + }); + + it("retrieves again after the corpus changes or with a different filter", async () => { + const { pipeline, retrievals } = replica(); + await search(pipeline, "ana", "builds products"); + expect(retrievals).not.toHaveBeenCalled(); + await query(`update event_participants set interests = '{search,agents}' where event_id = $1 and attendee_id = $2`, [eventId, ids.cy]); + await search(pipeline, "ana", "builds products"); + expect(retrievals).toHaveBeenCalledTimes(1); + await search(pipeline, "ana", "builds products", "AI"); + expect(retrievals).toHaveBeenCalledTimes(2); + }); +}); diff --git a/tests/unit/candidate-eligibility.test.ts b/tests/unit/candidate-eligibility.test.ts index eab1454..72a85e2 100644 --- a/tests/unit/candidate-eligibility.test.ts +++ b/tests/unit/candidate-eligibility.test.ts @@ -31,11 +31,16 @@ describe("matching pipeline viewer", () => { const participants = { findProfile: vi.fn(async () => viewer), listCandidates: vi.fn(async () => [eligible, hidden, incomplete]), + findCandidatesByIds: vi.fn(async (_event: string, ids: string[]) => [eligible, hidden, incomplete].filter((p) => ids.includes(p.attendeeId))), + corpusRevision: vi.fn(async () => 1), setEmbedding: vi.fn(), } as unknown as ParticipantRepository; const matches = { save: vi.fn(), listForParticipant: vi.fn() } satisfies MatchRepository; - const embeddings = { embed: vi.fn(async () => null) }; - const cache = { get: vi.fn(), set: vi.fn() }; + const embeddings = { id: "test-embeddings", embed: vi.fn(async () => null) }; + const cache = { + get: vi.fn(async () => null), set: vi.fn(async () => {}), getVector: vi.fn(async () => null), setVector: vi.fn(async () => {}), + acquire: vi.fn(async () => "token"), release: vi.fn(async () => {}), + }; return { pipeline: new MatchingPipeline(participants, matches, embeddings, null, cache), matches }; } const run = (pipeline: MatchingPipeline, mode: "profile" | "search") => diff --git a/tests/unit/search.test.ts b/tests/unit/search.test.ts index 0de3900..f5ddaba 100644 --- a/tests/unit/search.test.ts +++ b/tests/unit/search.test.ts @@ -71,10 +71,14 @@ function searchPipeline(candidates: ParticipantProfile[], viewer: ParticipantPro listCandidates: vi.fn(async () => candidates), findCandidatesByIds: vi.fn(async (_event: string, ids: string[]) => candidates.filter((c) => ids.includes(c.attendeeId))), setEmbedding: vi.fn(), + corpusRevision: vi.fn(async () => 1), } as unknown as ParticipantRepository; - const cache = { get: vi.fn(async () => cached), set: vi.fn() }; + const cache = { + get: vi.fn(async () => cached), set: vi.fn(async () => {}), getVector: vi.fn(async () => null), setVector: vi.fn(async () => {}), + acquire: vi.fn(async () => "token"), release: vi.fn(async () => {}), + }; const matches = { save: vi.fn(), listForParticipant: vi.fn() }; - const pipeline = new MatchingPipeline(participants, matches, { embed: vi.fn(async () => null) }, null, cache); + const pipeline = new MatchingPipeline(participants, matches, { id: "test-embeddings", embed: vi.fn(async () => null) }, null, cache); return { pipeline, participants, cache, matches }; } diff --git a/tests/unit/shared-computation.test.ts b/tests/unit/shared-computation.test.ts new file mode 100644 index 0000000..b1a5b45 --- /dev/null +++ b/tests/unit/shared-computation.test.ts @@ -0,0 +1,67 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { computeOnce } from "@/src/application/shared-computation"; +import type { CandidateCache } from "@/src/application/ports/candidate-cache"; + +/** An in-memory stand-in for the shared cache and its locks. */ +function memoryCache() { + const values = new Map(); + const locks = new Map(); + let next = 0; + const cache: CandidateCache = { + get: vi.fn(async (key) => (values.get(key) as string[]) ?? null), + set: vi.fn(async (key, ids) => { values.set(key, ids); }), + getVector: vi.fn(async (key) => (values.get(key) as number[]) ?? null), + setVector: vi.fn(async (key, vector) => { values.set(key, vector); }), + acquire: vi.fn(async (key) => { + if (locks.has(key)) return null; + const token = `t${next++}`; + locks.set(key, token); + return token; + }), + release: vi.fn(async (key, token) => { if (locks.get(key) === token) locks.delete(key); }), + }; + return { cache, values, locks }; +} + +const slow = (value: T, ms = 50) => new Promise((resolve) => setTimeout(() => resolve(value), ms)); + +describe("computeOnce", () => { + beforeEach(() => vi.spyOn(console, "warn").mockImplementation(() => {})); + + it("computes once for concurrent identical requests and shares the result", async () => { + const { cache, locks } = memoryCache(); + const compute = vi.fn(() => slow(["a", "b"])); + const run = () => computeOnce({ cache, key: "k", read: () => cache.get("k"), write: (v) => cache.set("k", v, 60), compute }); + const results = await Promise.all([run(), run(), run()]); + expect(compute).toHaveBeenCalledTimes(1); + expect(results).toEqual([["a", "b"], ["a", "b"], ["a", "b"]]); + expect(locks.size).toBe(0); + }); + + it("stops waiting for a stuck computation and computes itself", async () => { + const { cache, locks } = memoryCache(); + locks.set("compute:k", "someone-else"); + const compute = vi.fn(async () => ["mine"]); + const result = await computeOnce({ cache, key: "k", read: () => cache.get("k"), write: (v) => cache.set("k", v, 60), compute, waitMs: 150 }); + expect(result).toEqual(["mine"]); + expect(compute).toHaveBeenCalledTimes(1); + }); + + it("computes directly when the cache cannot be read or locked", async () => { + const { cache } = memoryCache(); + vi.mocked(cache.acquire).mockRejectedValue(new Error("down")); + const compute = vi.fn(async () => ["direct"]); + const result = await computeOnce({ cache, key: "k", read: () => Promise.reject(new Error("down")), write: vi.fn(), compute }); + expect(result).toEqual(["direct"]); + }); + + it("returns but does not share values marked not worth storing", async () => { + const { cache, values } = memoryCache(); + const result = await computeOnce({ + cache, key: "q", read: () => cache.getVector("q"), write: (v) => cache.setVector("q", v!, 60), + compute: async () => null, shouldStore: (v) => v !== null, + }); + expect(result).toBeNull(); + expect(values.size).toBe(0); + }); +}); From aaf5f9aafc8ba06197ca5ceaaa9730e29da44fce Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 09:10:27 +0530 Subject: [PATCH 02/26] Make background matching durable and revision-aware (R21 part 1) Profile saves and their jobs were separate writes, a running job made later edits get dropped, a crashed worker's job stayed "running", and a slow job could publish results for an older profile. - Migration 0012: profile_revision per participant; jobs carry the revision they reflect, a lease, the last error and why they exist (own profile change or refresh). One job per person and kind may be pending; a new edit updates it instead of waiting behind a running one - saveProfile bumps the revision and queues the embedding in the same transaction - Embeddings and match lists are written only while the profile is still at the job's revision (the match write locks the row); a stale run completes quietly and the newer job publishes - Claims take a 5-minute lease; an expired lease is reclaimed, or failed for good if it was the last attempt. Failures retry after 30s, 60s, 120s... (capped at 15 minutes) up to 5 attempts, then stay failed with the error kept; a newer pending job supersedes a failing one. Replaces R08's stuck-job workaround in enqueue - listFailed / retryFailed for operators; claim can be scoped to one event (used by tests so they never claim other local jobs) - Real-Postgres tests for each of these --- apps/web/app/actions/participant.actions.ts | 2 +- .../src/application/ports/match-repository.ts | 6 +- .../ports/participant-repository.ts | 3 + .../application/use-cases/compute-matches.ts | 2 +- .../use-cases/upsert-participant.ts | 7 +- apps/web/src/domain/matching/pipeline.ts | 13 +- .../domain/matching/stages/persist.stage.ts | 4 +- .../domain/participant/participant.entity.ts | 2 + .../src/infrastructure/postgres/match.repo.ts | 13 +- .../postgres/participant.repo.ts | 49 +++++-- packages/db/migrations/0012_durable_jobs.sql | 34 +++++ packages/db/src/repositories/matching-jobs.ts | 135 ++++++++++++++---- packages/db/workers/embed.worker.ts | 16 ++- packages/db/workers/match.worker.ts | 10 +- tests/integration/durable-jobs.test.ts | 114 +++++++++++++++ tests/integration/match-queue.test.ts | 17 +-- 16 files changed, 356 insertions(+), 71 deletions(-) create mode 100644 packages/db/migrations/0012_durable_jobs.sql create mode 100644 tests/integration/durable-jobs.test.ts diff --git a/apps/web/app/actions/participant.actions.ts b/apps/web/app/actions/participant.actions.ts index e0f0aa0..b55335b 100644 --- a/apps/web/app/actions/participant.actions.ts +++ b/apps/web/app/actions/participant.actions.ts @@ -46,7 +46,7 @@ async function saveProfile(eventSlug: string, formData: FormData) { yearsOfExperience: parseYears(formData.get("yearsOfExperience")), mostImpressiveBuild: nullable(formData.get("mostImpressiveBuild")), visible: formData.get("visible") === "on", - }, { participants: container.participants, enqueueMatching: container.enqueueEmbedding }, readProfileForm(event.form_config)); + }, { participants: container.participants }, readProfileForm(event.form_config)); } export async function signOutCurrentUser() { diff --git a/apps/web/src/application/ports/match-repository.ts b/apps/web/src/application/ports/match-repository.ts index 26d6ed6..8beaed2 100644 --- a/apps/web/src/application/ports/match-repository.ts +++ b/apps/web/src/application/ports/match-repository.ts @@ -2,6 +2,10 @@ import type { MatchCandidate } from "../../domain/matching/match-strategy"; export interface MatchRepository { /** Replaces the participant's matches; array order is the order shown to them. */ - save(eventId: string, participantId: string, candidates: MatchCandidate[], computationVersion: string): Promise; + /** + * With `expectedRevision`, writes only while the participant's profile is + * still at that revision and returns false otherwise (a newer job will). + */ + save(eventId: string, participantId: string, candidates: MatchCandidate[], computationVersion: string, expectedRevision?: number): Promise; listForParticipant(eventId: string, participantId: string, limit?: number): Promise; } diff --git a/apps/web/src/application/ports/participant-repository.ts b/apps/web/src/application/ports/participant-repository.ts index 7d26773..e03b7e2 100644 --- a/apps/web/src/application/ports/participant-repository.ts +++ b/apps/web/src/application/ports/participant-repository.ts @@ -3,6 +3,7 @@ import type { SearchFilters } from "../../domain/matching/search-query"; export interface ParticipantRepository { findProfile(eventId: string, attendeeId: string): Promise; + /** Saves the profile and durably queues the work that reflects it. */ saveProfile(input: ParticipantProfileInput): Promise; listCandidates(input: { eventId: string; @@ -19,6 +20,8 @@ export interface ParticipantRepository { listDomains(eventId: string): Promise; findCandidatesByIds(eventId: string, ids: string[]): Promise; setEmbedding(eventId: string, attendeeId: string, embedding: number[]): Promise; + /** Writes the embedding only while the profile is still at `revision`; false when it moved on. */ + setEmbeddingAt(eventId: string, attendeeId: string, embedding: number[], revision: number): Promise; shareVenueZone(eventId: string, attendeeId: string, zone: string): Promise; hasEventAccess(eventId: string, attendeeId: string): Promise; } diff --git a/apps/web/src/application/use-cases/compute-matches.ts b/apps/web/src/application/use-cases/compute-matches.ts index a15c0ca..a470fa3 100644 --- a/apps/web/src/application/use-cases/compute-matches.ts +++ b/apps/web/src/application/use-cases/compute-matches.ts @@ -1,5 +1,5 @@ import { MatchingPipeline } from "../../domain/matching/pipeline"; -export function computeMatches(input: { eventId: string; participantId: string; limit?: number }, pipeline: MatchingPipeline) { +export function computeMatches(input: { eventId: string; participantId: string; limit?: number; expectedRevision?: number }, pipeline: MatchingPipeline) { return pipeline.run({ ...input, mode: "profile" }); } diff --git a/apps/web/src/application/use-cases/upsert-participant.ts b/apps/web/src/application/use-cases/upsert-participant.ts index 585d9c5..7539664 100644 --- a/apps/web/src/application/use-cases/upsert-participant.ts +++ b/apps/web/src/application/use-cases/upsert-participant.ts @@ -4,7 +4,7 @@ import type { ParticipantRepository } from "../ports/participant-repository"; export async function upsertParticipant( input: ParticipantProfileInput, - dependencies: { participants: ParticipantRepository; enqueueMatching?: (eventId: string, participantId: string) => Promise }, + dependencies: { participants: ParticipantRepository }, form?: ProfileFormConfig ) { let values = input; @@ -16,7 +16,6 @@ export async function upsertParticipant( if (!form.fields[key].shown) Object.assign(values, { [key]: existing?.[key] ?? (key === "interests" ? [] : null) }); } } - const profile = await dependencies.participants.saveProfile(validateProfile(values, form)); - await dependencies.enqueueMatching?.(profile.eventId, profile.attendeeId); - return profile; + // saveProfile queues the embedding and matching in the same transaction. + return dependencies.participants.saveProfile(validateProfile(values, form)); } diff --git a/apps/web/src/domain/matching/pipeline.ts b/apps/web/src/domain/matching/pipeline.ts index 11c3c4e..2ae079c 100644 --- a/apps/web/src/domain/matching/pipeline.ts +++ b/apps/web/src/domain/matching/pipeline.ts @@ -30,6 +30,8 @@ export interface MatchingInput { query?: string; filters?: SearchFilters; limit?: number; + /** Profile runs from a background job: save only if the profile is still at this revision. */ + expectedRevision?: number; } /** @@ -38,6 +40,14 @@ export interface MatchingInput { */ export const MATCHING_PIPELINE_VERSION = `v2.${SCORING_CONFIG.version}`; +/** The profile changed while matching ran; the newer job will publish instead. */ +export class StaleRevisionError extends Error { + constructor() { + super("Profile changed during matching; the newer job will publish results"); + this.name = "StaleRevisionError"; + } +} + /** The scorers the pipeline combines, weighted by SCORING_CONFIG. */ export function defaultScorers() { return [ @@ -97,7 +107,8 @@ export class MatchingPipeline { const top = this.topK.execute(scored, input.limit ?? 12); const { candidates: ranked, rankedBy } = await this.rerank.execute(viewer, top); - await this.persist.execute(input.eventId, input.participantId, ranked, `${MATCHING_PIPELINE_VERSION}+${rankedBy}`); + const saved = await this.persist.execute(input.eventId, input.participantId, ranked, `${MATCHING_PIPELINE_VERSION}+${rankedBy}`, input.expectedRevision); + if (saved === false) throw new StaleRevisionError(); return ranked; } diff --git a/apps/web/src/domain/matching/stages/persist.stage.ts b/apps/web/src/domain/matching/stages/persist.stage.ts index 1c1c07e..60f94e5 100644 --- a/apps/web/src/domain/matching/stages/persist.stage.ts +++ b/apps/web/src/domain/matching/stages/persist.stage.ts @@ -3,7 +3,7 @@ import type { MatchCandidate } from "../match-strategy"; export class PersistStage { constructor(private readonly matches: MatchRepository) {} - execute(eventId: string, participantId: string, candidates: MatchCandidate[], computationVersion: string): Promise { - return this.matches.save(eventId, participantId, candidates, computationVersion); + execute(eventId: string, participantId: string, candidates: MatchCandidate[], computationVersion: string, expectedRevision?: number): Promise { + return this.matches.save(eventId, participantId, candidates, computationVersion, expectedRevision); } } diff --git a/apps/web/src/domain/participant/participant.entity.ts b/apps/web/src/domain/participant/participant.entity.ts index 53fe075..f27cd4c 100644 --- a/apps/web/src/domain/participant/participant.entity.ts +++ b/apps/web/src/domain/participant/participant.entity.ts @@ -19,6 +19,8 @@ export interface ParticipantProfile { venueZone: string | null; visible: boolean; profileComplete?: boolean; + /** Goes up on every save; background jobs carry the revision they reflect. */ + revision?: number; /** When the profile was last saved; part of cached explanation keys. */ profileRevision?: string | null; /** Version of the event's profile form this profile was saved under. */ diff --git a/apps/web/src/infrastructure/postgres/match.repo.ts b/apps/web/src/infrastructure/postgres/match.repo.ts index f491c75..e8a8eea 100644 --- a/apps/web/src/infrastructure/postgres/match.repo.ts +++ b/apps/web/src/infrastructure/postgres/match.repo.ts @@ -26,10 +26,16 @@ interface MatchRow { }; export class PostgresMatchRepository implements MatchRepository { - async save(eventId: string, participantId: string, candidates: MatchCandidate[], computationVersion: string): Promise { - await transaction(async (client) => { + async save(eventId: string, participantId: string, candidates: MatchCandidate[], computationVersion: string, expectedRevision?: number): Promise { + return transaction(async (client) => { + if (expectedRevision !== undefined) { + // Lock the profile row so a save cannot slip in between check and write. + const { rows } = await client.query<{ profile_revision: string }>( + `select profile_revision from event_participants where event_id = $1 and attendee_id = $2 for update`, [eventId, participantId]); + if (!rows[0] || Number(rows[0].profile_revision) !== expectedRevision) return false; + } await client.query(`delete from event_matches where event_id = $1 and participant_id = $2`, [eventId, participantId]); - if (candidates.length === 0) return; + if (candidates.length === 0) return true; // Array position is the final (possibly reranked) order, stored as rank 1..N. await client.query( `insert into event_matches @@ -45,6 +51,7 @@ export class PostgresMatchRepository implements MatchRepository { candidates.map((candidate) => candidate.reason), candidates.map((candidate) => candidate.mutualInterest)] ); + return true; }); } diff --git a/apps/web/src/infrastructure/postgres/participant.repo.ts b/apps/web/src/infrastructure/postgres/participant.repo.ts index 3b2d48c..962a390 100644 --- a/apps/web/src/infrastructure/postgres/participant.repo.ts +++ b/apps/web/src/infrastructure/postgres/participant.repo.ts @@ -1,4 +1,4 @@ -import { query } from "@match-maker/db"; +import { matchingJobs, query, transaction } from "@match-maker/db"; import { lexicalTsQuery, type SearchFilters } from "../../domain/matching/search-query"; import type { ParticipantRepository } from "../../application/ports/participant-repository"; import { Intent } from "../../domain/participant/intent"; @@ -23,6 +23,7 @@ interface ProfileRow { embedding: string | number[] | null; profile_updated_at?: Date | null; profile_form_version?: number; + profile_revision?: number; } const PROFILE_COLUMNS = `p.event_id, a.id as attendee_id, a.name, @@ -33,7 +34,7 @@ const PROFILE_COLUMNS = `p.event_id, a.id as attendee_id, a.name, coalesce(p.years_of_experience, a.years_of_experience) as years_of_experience, coalesce(p.most_impressive_build, a.most_impressive_build) as most_impressive_build, p.intent, p.interests, - p.venue_zone, p.visible, p.embedding::text as embedding, p.profile_updated_at, p.profile_form_version`; + p.venue_zone, p.visible, p.embedding::text as embedding, p.profile_updated_at, p.profile_form_version, p.profile_revision::int as profile_revision`; const PROFILE_FROM = `from event_participants p join registered_attendees a on a.id = p.attendee_id`; /** @@ -57,20 +58,28 @@ export class PostgresParticipantRepository implements ParticipantRepository { return rows[0] ? mapProfile(rows[0]) : null; } + /** + * Saves the profile and queues its embedding in one transaction, so a crash + * cannot keep the change while losing the work that reflects it. + */ async saveProfile(input: ParticipantProfileInput): Promise { - const { rowCount } = await query( - `update event_participants set - intent = $3, interests = $4, venue_zone = $5, - about_me = $6, designation = $7, domain = $8, - self_description = $9, years_of_experience = $10, - most_impressive_build = $11, profile_form_version = $12, visible = coalesce($13, visible), - embedding = null, profile_updated_at = now(), updated_at = now() - where event_id = $1 and attendee_id = $2`, - [input.eventId, input.attendeeId, input.intent, input.interests, input.venueZone, - input.aboutMe, input.designation, input.domain, input.selfDescription, - input.yearsOfExperience, input.mostImpressiveBuild, input.formVersion ?? 0, input.visible ?? null] - ); - if (!rowCount) throw new Error("This attendee is not on the event roster"); + await transaction(async (client) => { + const { rows: [updated] } = await client.query<{ profile_revision: string }>( + `update event_participants set + intent = $3, interests = $4, venue_zone = $5, + about_me = $6, designation = $7, domain = $8, + self_description = $9, years_of_experience = $10, + most_impressive_build = $11, profile_form_version = $12, visible = coalesce($13, visible), + embedding = null, profile_updated_at = now(), updated_at = now(), profile_revision = profile_revision + 1 + where event_id = $1 and attendee_id = $2 and revoked_at is null + returning profile_revision`, + [input.eventId, input.attendeeId, input.intent, input.interests, input.venueZone, + input.aboutMe, input.designation, input.domain, input.selfDescription, + input.yearsOfExperience, input.mostImpressiveBuild, input.formVersion ?? 0, input.visible ?? null] + ); + if (!updated) throw new Error("This attendee is not on the event roster"); + await matchingJobs.enqueue(input.eventId, input.attendeeId, "embed", { revision: Number(updated.profile_revision) }, client); + }); if (input.visible === false) { // Hidden profiles are not explained to others any more. await query(`delete from pair_explanations where event_id = $1 and candidate_id = $2`, [input.eventId, input.attendeeId]); @@ -135,6 +144,15 @@ export class PostgresParticipantRepository implements ParticipantRepository { return rows.map(mapProfile); } + async setEmbeddingAt(eventId: string, attendeeId: string, embedding: number[], revision: number): Promise { + const { rowCount } = await query( + `update event_participants set embedding = $3::vector, updated_at = now() + where event_id = $1 and attendee_id = $2 and profile_revision = $4`, + [eventId, attendeeId, toVector(embedding), revision] + ); + return (rowCount ?? 0) > 0; + } + async setEmbedding(eventId: string, attendeeId: string, embedding: number[]): Promise { const vector = toVector(embedding); await query( @@ -186,6 +204,7 @@ function mapProfile(row: ProfileRow): ParticipantProfile { visible: row.visible, profileComplete: Boolean(row.profile_updated_at), formVersion: row.profile_form_version ?? 0, + revision: row.profile_revision ?? 0, profileRevision: row.profile_updated_at ? new Date(row.profile_updated_at).toISOString() : null, embedding: parseEmbedding(row.embedding), }; diff --git a/packages/db/migrations/0012_durable_jobs.sql b/packages/db/migrations/0012_durable_jobs.sql new file mode 100644 index 0000000..7d3c5b6 --- /dev/null +++ b/packages/db/migrations/0012_durable_jobs.sql @@ -0,0 +1,34 @@ +-- Durable, revision-aware background matching. +-- +-- profile_revision goes up on every profile save. Jobs carry the revision +-- they must reflect, and embeddings or match lists are written only while the +-- profile is still at that revision, so a slow job never replaces newer work. +-- +-- One job per person and kind may be *pending*; a new edit updates it +-- instead of waiting behind a running one. Running jobs hold a lease; a +-- crashed worker's job is reclaimed when the lease expires. Retryable +-- failures go back to pending with a backoff; after the last attempt a job +-- is 'failed' for good, with its last error kept for inspection. +-- +-- trigger_kind tells a job caused by the person's own profile change +-- ('profile') from a refresh because someone else changed ('refresh'); only +-- the first kind asks for others to be refreshed, so refreshes never cascade. + +alter table event_participants add column if not exists profile_revision bigint not null default 0; + +alter table matching_jobs add column if not exists revision bigint not null default 0; +alter table matching_jobs add column if not exists lease_until timestamptz; +alter table matching_jobs add column if not exists last_error text; +alter table matching_jobs add column if not exists trigger_kind text not null default 'profile'; +alter table matching_jobs drop constraint if exists matching_jobs_trigger_kind_check; +alter table matching_jobs add constraint matching_jobs_trigger_kind_check check (trigger_kind in ('profile', 'refresh')); + +-- Before this migration 'failed' also meant "will retry"; those rows are pending now. +update matching_jobs set status = 'pending' where status = 'failed' and attempts < 5 + and not exists (select 1 from matching_jobs p where p.event_id = matching_jobs.event_id and p.participant_id = matching_jobs.participant_id + and p.job_type = matching_jobs.job_type and p.status = 'pending'); + +drop index if exists idx_matching_jobs_one_active; +create unique index if not exists idx_matching_jobs_one_pending + on matching_jobs (event_id, participant_id, job_type) where status = 'pending'; +create index if not exists idx_matching_jobs_leases on matching_jobs (lease_until) where status = 'running'; diff --git a/packages/db/src/repositories/matching-jobs.ts b/packages/db/src/repositories/matching-jobs.ts index 2f5a4c8..df1a691 100644 --- a/packages/db/src/repositories/matching-jobs.ts +++ b/packages/db/src/repositories/matching-jobs.ts @@ -6,39 +6,69 @@ export interface MatchingJob { participant_id: string; attempts: number; job_type: "embed" | "match"; + /** The profile revision this job must reflect. */ + revision: number; + trigger_kind: "profile" | "refresh"; } -/** Matches STUCK_AFTER_MS in apps/web/src/domain/matching/match-status.ts. */ -const STUCK_AFTER_MINUTES = 10; +/** Attempts before a job fails for good. */ +export const MAX_ATTEMPTS = 5; +/** How long a claimed job may run before another worker may take it over. */ +export const LEASE_SECONDS = 300; -export async function enqueue(eventId: string, participantId: string, jobType: "embed" | "match" = "match"): Promise { - // A job "running" past the stuck limit belongs to a crashed worker and would - // block the one-active-job index forever; release it so this request queues. - await query( - `update matching_jobs set status = 'failed' - where event_id = $1 and participant_id = $2 and job_type = $3 and status = 'running' - and started_at < now() - make_interval(mins => $4)`, - [eventId, participantId, jobType, STUCK_AFTER_MINUTES] - ); - await query( - `insert into matching_jobs (event_id, participant_id, job_type) values ($1, $2, $3) - on conflict (event_id, participant_id, job_type) where status in ('pending', 'running') do nothing`, - [eventId, participantId, jobType] +/** + * Queues work for a participant. At most one job per person and kind is + * pending: a newer request updates it (latest revision, earliest start, and + * "profile" wins over "refresh") instead of being dropped, and a running job + * no longer blocks a new one. Without a revision, the participant's current + * one is used. + */ +export async function enqueue( + eventId: string, + participantId: string, + jobType: "embed" | "match" = "match", + options: { revision?: number; trigger?: "profile" | "refresh"; delaySeconds?: number } = {}, + client: { query: typeof query } = { query } +): Promise { + await client.query( + `insert into matching_jobs (event_id, participant_id, job_type, revision, trigger_kind, available_at) + values ($1, $2, $3, + coalesce($4::bigint, (select profile_revision from event_participants where event_id = $1 and attendee_id = $2), 0), + $5, now() + make_interval(secs => $6)) + on conflict (event_id, participant_id, job_type) where status = 'pending' do update + set revision = greatest(matching_jobs.revision, excluded.revision), + available_at = least(matching_jobs.available_at, excluded.available_at), + trigger_kind = case when matching_jobs.trigger_kind = 'profile' then 'profile' else excluded.trigger_kind end`, + [eventId, participantId, jobType, options.revision ?? null, options.trigger ?? "profile", options.delaySeconds ?? 0] ); } -/** Claim one row safely when multiple worker replicas run at once. */ -export async function claim(jobType: "embed" | "match"): Promise { +/** + * Claims the oldest due job of a kind, safely across worker replicas: a + * pending job whose time has come, or a running one whose lease expired + * (its worker crashed). A crashed job already at its last attempt fails + * for good instead of being retried forever. + */ +export async function claim(jobType: "embed" | "match", onlyEventId: string | null = null): Promise { return transaction(async (client) => { + await client.query( + `update matching_jobs set status = 'failed', lease_until = null, last_error = coalesce(last_error, 'worker stopped responding') + where job_type = $1 and status = 'running' and lease_until < now() and attempts >= $2 + and ($3::uuid is null or event_id = $3)`, + [jobType, MAX_ATTEMPTS, onlyEventId] + ); const { rows } = await client.query( `with next_job as ( select id from matching_jobs - where job_type = $1 and status in ('pending', 'failed') and available_at <= now() and attempts < 5 - order by created_at asc for update skip locked limit 1 + where job_type = $1 and attempts < $2 and ($4::uuid is null or event_id = $4) + and ((status = 'pending' and available_at <= now()) or (status = 'running' and lease_until < now())) + order by available_at asc, created_at asc for update skip locked limit 1 ) - update matching_jobs j set status = 'running', attempts = attempts + 1, started_at = now() + update matching_jobs j set status = 'running', attempts = attempts + 1, started_at = now(), + lease_until = now() + make_interval(secs => $3) from next_job where j.id = next_job.id - returning j.id, j.event_id, j.participant_id, j.attempts, j.job_type`, [jobType] + returning j.id::int as id, j.event_id, j.participant_id, j.attempts, j.job_type, j.revision::int as revision, j.trigger_kind`, + [jobType, MAX_ATTEMPTS, LEASE_SECONDS, onlyEventId] ); return rows[0] ?? null; }); @@ -76,15 +106,64 @@ export async function latestForEvent(eventId: string): Promise { } export async function complete(id: number): Promise { - await query(`update matching_jobs set status = 'done' where id = $1`, [id]); + await query(`update matching_jobs set status = 'done', lease_until = null where id = $1`, [id]); } -export async function fail(id: number, retrySeconds = 30): Promise { +/** Delay before retry `attempt` (1-based): 30s, 60s, 120s, ... capped at 15 minutes. */ +export function retryDelaySeconds(attempt: number): number { + return Math.min(30 * 2 ** Math.max(0, attempt - 1), 15 * 60); +} + +/** + * Records a failed attempt. Before the last attempt the job goes back to + * pending after a backoff; after it, the job is failed for good. If a newer + * job for the same person is already pending, this one is superseded. + */ +export async function fail(id: number, error: string): Promise { await query( - `update matching_jobs - set status = case when attempts >= 5 then 'failed' else 'pending' end, - available_at = now() + ($2 || ' seconds')::interval - where id = $1`, - [id, String(retrySeconds)] + `update matching_jobs j + set status = case + when exists (select 1 from matching_jobs p where p.event_id = j.event_id and p.participant_id = j.participant_id + and p.job_type = j.job_type and p.status = 'pending' and p.id <> j.id) then 'done' + when j.attempts >= $3 then 'failed' + else 'pending' end, + available_at = now() + make_interval(secs => least(30 * power(2, greatest(j.attempts - 1, 0)), 900)), + lease_until = null, + last_error = left($2, 1000) + where j.id = $1`, + [id, error, MAX_ATTEMPTS] + ); +} + +export interface FailedJob { + id: number; + event_slug: string; + email: string | null; + job_type: string; + attempts: number; + last_error: string | null; + started_at: Date | null; +} + +/** Jobs that failed for good, newest first (operator inspection). */ +export async function listFailed(limit = 50): Promise { + const { rows } = await query( + `select j.id::int as id, e.slug as event_slug, a.email, j.job_type, j.attempts, j.last_error, j.started_at + from matching_jobs j join events e on e.id = j.event_id left join registered_attendees a on a.id = j.participant_id + where j.status = 'failed' order by j.started_at desc nulls last limit $1`, + [limit] + ); + return rows; +} + +/** Queues failed jobs again with fresh attempts (one id, or all). Returns how many. */ +export async function retryFailed(id: number | "all"): Promise { + const { rows } = await query<{ event_id: string; participant_id: string; job_type: "embed" | "match" }>( + `update matching_jobs set status = 'done', last_error = coalesce(last_error, '') || ' (retried)' + where status = 'failed' and ($1::bigint is null or id = $1) + returning event_id, participant_id, job_type`, + [id === "all" ? null : id] ); + for (const row of rows) await enqueue(row.event_id, row.participant_id, row.job_type); + return rows.length; } diff --git a/packages/db/workers/embed.worker.ts b/packages/db/workers/embed.worker.ts index 14f0673..00553c1 100644 --- a/packages/db/workers/embed.worker.ts +++ b/packages/db/workers/embed.worker.ts @@ -15,15 +15,21 @@ export async function processEmbedJob(): Promise { } const container = getContainer(); const profile = await container.participants.findProfile(job.event_id, job.participant_id); - if (profile) { - const embedding = await container.embeddings.embed(profileText(profile), "profile"); - if (embedding) await container.participants.setEmbedding(job.event_id, job.participant_id, embedding); + // A newer save queued its own embed job; this one has nothing left to do. + if (!profile || (profile.revision ?? 0) !== job.revision) { + await matchingJobs.complete(job.id); + return true; + } + const embedding = await container.embeddings.embed(profileText(profile), "profile"); + if (embedding && !(await container.participants.setEmbeddingAt(job.event_id, job.participant_id, embedding, job.revision))) { + await matchingJobs.complete(job.id); + return true; } await matchingJobs.complete(job.id); - await matchingJobs.enqueue(job.event_id, job.participant_id, "match"); + await matchingJobs.enqueue(job.event_id, job.participant_id, "match", { revision: job.revision, trigger: job.trigger_kind }); } catch (error) { console.error(`[worker:embed] job ${job.id} failed`, error); - await matchingJobs.fail(job.id); + await matchingJobs.fail(job.id, error instanceof Error ? error.message : String(error)); } return true; } diff --git a/packages/db/workers/match.worker.ts b/packages/db/workers/match.worker.ts index 7eeec83..df7b645 100644 --- a/packages/db/workers/match.worker.ts +++ b/packages/db/workers/match.worker.ts @@ -1,6 +1,7 @@ import { events, matchingJobs } from "../src/index"; import { getContainer } from "../../../apps/web/src/composition/container"; import { computeMatches } from "../../../apps/web/src/application/use-cases/compute-matches"; +import { StaleRevisionError } from "../../../apps/web/src/domain/matching/pipeline"; import { eventAllows } from "../../../apps/web/src/domain/shared/event-lifecycle"; export async function processMatchJob(): Promise { @@ -14,11 +15,16 @@ export async function processMatchJob(): Promise { return true; } const { pipeline } = getContainer(); - await computeMatches({ eventId: job.event_id, participantId: job.participant_id }, pipeline); + await computeMatches({ eventId: job.event_id, participantId: job.participant_id, expectedRevision: job.revision }, pipeline); await matchingJobs.complete(job.id); } catch (error) { + if (error instanceof StaleRevisionError) { + // The profile moved on during this run; the newer job publishes instead. + await matchingJobs.complete(job.id); + return true; + } console.error(`[worker:match] job ${job.id} failed`, error); - await matchingJobs.fail(job.id); + await matchingJobs.fail(job.id, error instanceof Error ? error.message : String(error)); } return true; } diff --git a/tests/integration/durable-jobs.test.ts b/tests/integration/durable-jobs.test.ts new file mode 100644 index 0000000..15acf0d --- /dev/null +++ b/tests/integration/durable-jobs.test.ts @@ -0,0 +1,114 @@ +/** + * Durable, revision-aware matching jobs against a real Postgres. Every claim + * is scoped to this test's own event, so jobs of other local data are never + * touched. Skipped when DATABASE_URL is unset. + */ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { attendees, closePool, events, matchingJobs, query } from "@match-maker/db"; +import { migrate } from "../../packages/db/src/migrate"; +import { PostgresParticipantRepository } from "../../apps/web/src/infrastructure/postgres/participant.repo"; +import { PostgresMatchRepository } from "../../apps/web/src/infrastructure/postgres/match.repo"; +import { Intent } from "../../apps/web/src/domain/participant/intent"; + +const describeDb = process.env.DATABASE_URL ? describe : describe.skip; +const RUN = Date.now().toString(36); +const ORG = `jobs-${RUN}`; +const email = (key: string) => `jobs-${key}-${RUN}@example.com`; + +describeDb("durable matching jobs", () => { + const participants = new PostgresParticipantRepository(); + const matches = new PostgresMatchRepository(); + const ids: Record = {}; + let eventId: string; + + const save = (key: string, aboutMe: string) => participants.saveProfile({ + eventId, attendeeId: ids[key], designation: "Engineer", domain: "AI", aboutMe, selfDescription: null, + yearsOfExperience: null, mostImpressiveBuild: null, intent: Intent.NETWORKING, interests: [], venueZone: null, + }); + const jobsOf = async (key: string) => (await query<{ id: number; job_type: string; status: string; revision: string; attempts: number; last_error: string | null }>( + `select id::int, job_type, status, revision, attempts, last_error from matching_jobs where event_id = $1 and participant_id = $2 order by id`, + [eventId, ids[key]])).rows; + + beforeAll(async () => { + await migrate(); + await attendees.upsertMany(["ann", "bo", "cy"].map((key) => ({ name: `Jobs ${key}`, email: email(key) }))); + for (const key of ["ann", "bo", "cy"]) ids[key] = (await attendees.findByEmail(email(key)))!.id; + const { rows: [org] } = await query<{ id: string }>(`insert into organizations (name, slug) values ($1, $1) returning id`, [ORG]); + eventId = (await query<{ id: string }>(`insert into events (organization_id, name, slug, status) values ($1, $2, $2, 'open') returning id`, [org.id, ORG])).rows[0].id; + await events.attachRoster(eventId, Object.values(ids)); + }); + + afterAll(async () => { + await query(`delete from organizations where slug = $1`, [ORG]); + await query(`delete from registered_attendees where email like $1`, [`jobs-%-${RUN}@example.com`]); + await closePool(); + }); + + it("queues the embedding with the save, and keeps one pending job through rapid edits", async () => { + const first = await save("ann", "First version"); + expect(first.revision).toBe(1); + await save("ann", "Second version"); + const jobs = await jobsOf("ann"); + expect(jobs).toEqual([expect.objectContaining({ job_type: "embed", status: "pending", revision: "2" })]); + }); + + it("queues an edit made while a job runs instead of dropping it", async () => { + const running = await matchingJobs.claim("embed", eventId); + expect(running).toMatchObject({ participant_id: ids.ann, revision: 2 }); + await save("ann", "Third version"); + const jobs = await jobsOf("ann"); + expect(jobs.map((job) => `${job.status}:${job.revision}`)).toEqual(["running:2", "pending:3"]); + // The running job's embedding is now stale and refused. + expect(await participants.setEmbeddingAt(eventId, ids.ann, new Array(768).fill(0.01), 2)).toBe(false); + expect(await participants.setEmbeddingAt(eventId, ids.ann, new Array(768).fill(0.01), 3)).toBe(true); + await matchingJobs.complete(running!.id); + }); + + it("refuses to publish matches computed for an older revision", async () => { + await save("bo", "Bo's profile"); + const candidate = { participant: (await participants.findProfile(eventId, ids.ann))!, score: 70, scoreBreakdown: {}, mutualInterest: false, reason: "r" }; + expect(await matches.save(eventId, ids.bo, [candidate], "test", 0)).toBe(false); + expect(await matches.save(eventId, ids.bo, [candidate], "test", 1)).toBe(true); + expect(await matches.listForParticipant(eventId, ids.bo)).toHaveLength(1); + }); + + it("retries with a growing delay, then fails for good with the error kept", async () => { + await query(`update matching_jobs set status = 'done' where event_id = $1 and status = 'pending'`, [eventId]); + await save("cy", "Cy's profile"); + for (let attempt = 1; attempt <= 5; attempt++) { + await query(`update matching_jobs set available_at = now() where event_id = $1 and participant_id = $2 and status = 'pending'`, [eventId, ids.cy]); + const job = await matchingJobs.claim("embed", eventId); + expect(job).toMatchObject({ participant_id: ids.cy, attempts: attempt }); + await matchingJobs.fail(job!.id, `boom ${attempt}`); + const [row] = await jobsOf("cy"); + expect(row.status).toBe(attempt < 5 ? "pending" : "failed"); + expect(row.last_error).toBe(`boom ${attempt}`); + } + expect(matchingJobs.retryDelaySeconds(1)).toBe(30); + expect(matchingJobs.retryDelaySeconds(3)).toBe(120); + expect(matchingJobs.retryDelaySeconds(10)).toBe(900); + expect((await matchingJobs.listFailed()).some((job) => job.email === email("cy") && job.last_error === "boom 5")).toBe(true); + }); + + it("re-queues failed jobs on request", async () => { + expect(await matchingJobs.retryFailed("all")).toBeGreaterThanOrEqual(1); + expect((await jobsOf("cy")).some((job) => job.status === "pending" && job.attempts === 0)).toBe(true); + }); + + it("marks a failing job superseded when a newer one is already pending", async () => { + await query(`update matching_jobs set available_at = now() where event_id = $1 and participant_id = $2 and status = 'pending'`, [eventId, ids.cy]); + const job = await matchingJobs.claim("embed", eventId); + await save("cy", "Cy edited again"); + await matchingJobs.fail(job!.id, "late failure"); + expect((await jobsOf("cy")).find((row) => row.id === job!.id)?.status).toBe("done"); + }); + + it("fails a crashed job that was already on its last attempt", async () => { + await query(`update matching_jobs set available_at = now() where event_id = $1 and status = 'pending'`, [eventId]); + const job = await matchingJobs.claim("embed", eventId); + await query(`update matching_jobs set attempts = 5, last_error = null, lease_until = now() - interval '1 minute' where id = $1`, [job!.id]); + await matchingJobs.claim("embed", eventId); + expect((await query<{ status: string; last_error: string }>(`select status, last_error from matching_jobs where id = $1`, [job!.id])).rows[0]) + .toEqual({ status: "failed", last_error: "worker stopped responding" }); + }); +}); diff --git a/tests/integration/match-queue.test.ts b/tests/integration/match-queue.test.ts index cdd1e48..3ba4f55 100644 --- a/tests/integration/match-queue.test.ts +++ b/tests/integration/match-queue.test.ts @@ -73,20 +73,21 @@ describeDb("match queue status", () => { expect(await queue.latestJob(otherEventId, ids.c)).toMatchObject({ status: "pending" }); }); - it("re-queues a participant whose job is stuck, leaving fresh running jobs alone", async () => { + it("queues a new edit behind a running job instead of dropping it", async () => { await job(eventId, "c", "embed", "running", 2, 1); await matchingJobs.enqueue(eventId, ids.c, "embed"); - expect(await queue.latestJob(eventId, ids.c)).toMatchObject({ status: "running" }); + expect(await queue.latestJob(eventId, ids.c)).toMatchObject({ type: "embed", status: "pending" }); + }); - await matchingJobs.enqueue(eventId, ids.b, "embed"); - expect(await queue.latestJob(eventId, ids.b)).toMatchObject({ type: "embed", status: "pending" }); - const { rows } = await query<{ status: string }>( - `select status from matching_jobs where event_id = $1 and participant_id = $2 order by created_at`, [eventId, ids.b]); - expect(rows.map((row) => row.status)).toEqual(["failed", "pending"]); + it("reclaims a job whose worker stopped responding", async () => { + await query(`update matching_jobs set lease_until = now() - interval '1 minute' where event_id = $1 and participant_id = $2 and status = 'running'`, [eventId, ids.b]); + const reclaimed = await matchingJobs.claim("embed", eventId); + expect(reclaimed).toMatchObject({ participant_id: ids.b, attempts: 1 }); + await matchingJobs.complete(reclaimed!.id); }); it("lists one latest job per participant in the event", async () => { const jobs = await queue.latestJobsForEvent(eventId); - expect(jobs.map((row) => row.status).sort()).toEqual(["done", "pending", "running"]); + expect(jobs.map((row) => row.status).sort()).toEqual(["done", "done", "pending"]); }); }); From 1b041c34b672c0bda5d8f0cec81e423994f4b6f1 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 09:13:07 +0530 Subject: [PATCH 03/26] Refresh affected participants and add failed-job tooling (R21 part 2) Other people's saved matches never changed when someone joined or edited their profile, so the first participant to save had no matches until they pressed refresh. - After a participant's own profile change is matched, the worker queues refreshes for the people it affects: those whose saved matches include them, and those whose list is not full. Refreshes wait MATCH_REFRESH_DELAY_SECONDS (default 120) so bursts coalesce into one run per person, keep the latest revision, and never trigger further refreshes; a pending profile job stays a profile job - npm run jobs:failed lists jobs that failed after every retry with their last error; npm run jobs:retry -- queues them again - Tests: who gets refreshed (and who does not), coalescing, no cascade; browser check that the first participant receives later arrivals without pressing anything, and that rapid edits end on the latest revision --- .env.example | 4 +++ ReadMe.md | 2 ++ package.json | 4 ++- packages/db/src/repositories/matching-jobs.ts | 23 +++++++++++++ packages/db/workers/match.worker.ts | 11 +++++++ tests/integration/durable-jobs.test.ts | 26 +++++++++++++++ tools/manage-jobs.ts | 33 +++++++++++++++++++ 7 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 tools/manage-jobs.ts diff --git a/.env.example b/.env.example index a64d1d3..78c0412 100644 --- a/.env.example +++ b/.env.example @@ -76,6 +76,10 @@ OTP_TTL_SECONDS=600 OTP_MAX_PER_WINDOW=5 OTP_RATE_WINDOW_SECONDS=900 +# Seconds the worker waits before refreshing other people's matches after a +# profile change, so bursts of changes coalesce (default 120). +# MATCH_REFRESH_DELAY_SECONDS=120 + # Proxies in front of the web app that append to X-Forwarded-For (1 behind # the AWS ALB). 0 trusts no header, so per-IP login limits are skipped locally. TRUSTED_PROXY_HOPS=0 diff --git a/ReadMe.md b/ReadMe.md index 818de3f..6ea993d 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -110,6 +110,8 @@ The first command imports the global attendee directory so the organizer account | `npm run keycloak:sync` | Bring an existing realm up to date: client secret, Google credentials, the account-linking flow (Google joins an account an e-mail code created), and the site login theme. Run after the first Keycloak start and after upgrades | | `npm run account:disable -- ` | Block sign-in and end every session (database and Keycloak); `account:enable` undoes it | | `npm run account:revoke-sessions -- ` | Sign the person out everywhere without disabling the account | +| `npm run jobs:failed` | List background matching jobs that failed after every retry, with their last error | +| `npm run jobs:retry -- ` | Queue failed jobs again with fresh attempts | | `npm run account:erase -- --yes` | Deletion request: delete the person, their event profiles, matches, introductions, cached explanations and Keycloak user (recorded in the audit log) | | `npm run docker:up` | Build images and start the local Compose stack | diff --git a/package.json b/package.json index 3001b6e..1aa9a2d 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,9 @@ "account:disable": "tsx tools/manage-account.ts disable", "account:enable": "tsx tools/manage-account.ts enable", "account:revoke-sessions": "tsx tools/manage-account.ts revoke-sessions", - "account:erase": "tsx tools/manage-account.ts erase" + "account:erase": "tsx tools/manage-account.ts erase", + "jobs:failed": "tsx tools/manage-jobs.ts failed", + "jobs:retry": "tsx tools/manage-jobs.ts retry" }, "devDependencies": { "@types/node": "^22.10.2", diff --git a/packages/db/src/repositories/matching-jobs.ts b/packages/db/src/repositories/matching-jobs.ts index df1a691..749dc83 100644 --- a/packages/db/src/repositories/matching-jobs.ts +++ b/packages/db/src/repositories/matching-jobs.ts @@ -109,6 +109,29 @@ export async function complete(id: number): Promise { await query(`update matching_jobs set status = 'done', lease_until = null where id = $1`, [id]); } +/** + * After someone's own profile change is processed, queues refreshes for the + * people it affects: those whose saved matches include them, and those whose + * list is not full (the person may now belong in it). Refreshes are delayed so + * a burst of changes coalesces into one run per person, and they never ask + * for further refreshes. Returns how many people were queued. + */ +export async function enqueueRefreshes(eventId: string, changedId: string, options: { delaySeconds: number; fullList: number }): Promise { + const { rowCount } = await query( + `insert into matching_jobs (event_id, participant_id, job_type, revision, trigger_kind, available_at) + select $1, p.attendee_id, 'match', p.profile_revision, 'refresh', now() + make_interval(secs => $3) + from event_participants p + where p.event_id = $1 and p.attendee_id <> $2 and p.profile_updated_at is not null and p.revoked_at is null + and (exists (select 1 from event_matches m where m.event_id = $1 and m.participant_id = p.attendee_id and m.candidate_id = $2) + or (select count(*) from event_matches m where m.event_id = $1 and m.participant_id = p.attendee_id) < $4) + on conflict (event_id, participant_id, job_type) where status = 'pending' do update + set revision = greatest(matching_jobs.revision, excluded.revision), + available_at = least(matching_jobs.available_at, excluded.available_at)`, + [eventId, changedId, options.delaySeconds, options.fullList] + ); + return rowCount ?? 0; +} + /** Delay before retry `attempt` (1-based): 30s, 60s, 120s, ... capped at 15 minutes. */ export function retryDelaySeconds(attempt: number): number { return Math.min(30 * 2 ** Math.max(0, attempt - 1), 15 * 60); diff --git a/packages/db/workers/match.worker.ts b/packages/db/workers/match.worker.ts index df7b645..01ceb82 100644 --- a/packages/db/workers/match.worker.ts +++ b/packages/db/workers/match.worker.ts @@ -3,6 +3,10 @@ import { getContainer } from "../../../apps/web/src/composition/container"; import { computeMatches } from "../../../apps/web/src/application/use-cases/compute-matches"; import { StaleRevisionError } from "../../../apps/web/src/domain/matching/pipeline"; import { eventAllows } from "../../../apps/web/src/domain/shared/event-lifecycle"; +import { DEFAULT_MATCH_LIMIT } from "../../../apps/web/src/domain/matching/match-strategy"; + +/** Seconds to wait before refreshing affected people, so bursts coalesce. */ +const REFRESH_DELAY_SECONDS = Number(process.env.MATCH_REFRESH_DELAY_SECONDS ?? 120); export async function processMatchJob(): Promise { const job = await matchingJobs.claim("match"); @@ -17,6 +21,13 @@ export async function processMatchJob(): Promise { const { pipeline } = getContainer(); await computeMatches({ eventId: job.event_id, participantId: job.participant_id, expectedRevision: job.revision }, pipeline); await matchingJobs.complete(job.id); + if (job.trigger_kind === "profile") { + const queued = await matchingJobs.enqueueRefreshes(job.event_id, job.participant_id, { + delaySeconds: Number.isFinite(REFRESH_DELAY_SECONDS) ? REFRESH_DELAY_SECONDS : 120, + fullList: DEFAULT_MATCH_LIMIT, + }); + if (queued) console.info(`[worker:match] job ${job.id}: refreshing ${queued} affected participant(s)`); + } } catch (error) { if (error instanceof StaleRevisionError) { // The profile moved on during this run; the newer job publishes instead. diff --git a/tests/integration/durable-jobs.test.ts b/tests/integration/durable-jobs.test.ts index 15acf0d..6fe5fab 100644 --- a/tests/integration/durable-jobs.test.ts +++ b/tests/integration/durable-jobs.test.ts @@ -103,6 +103,32 @@ describeDb("durable matching jobs", () => { expect((await jobsOf("cy")).find((row) => row.id === job!.id)?.status).toBe("done"); }); + it("refreshes the people a profile change affects, coalesced and without cascading", async () => { + // ann lists bo; cy has no matches yet; dee's (one-slot) list is full without bo; eve never finished a profile. + await attendees.upsertMany(["dee", "eve"].map((key) => ({ name: `Jobs ${key}`, email: email(key) }))); + for (const key of ["dee", "eve"]) ids[key] = (await attendees.findByEmail(email(key)))!.id; + await events.attachRoster(eventId, [ids.dee, ids.eve]); + await save("dee", "Dee's profile"); + await query(`update matching_jobs set status = 'done' where event_id = $1 and status = 'pending'`, [eventId]); + await query(`delete from event_matches where event_id = $1`, [eventId]); + await query(`insert into event_matches (event_id, participant_id, candidate_id, rank, score, computation_version) + values ($1, $2, $3, 1, 50, 't'), ($1, $4, $5, 1, 50, 't')`, [eventId, ids.ann, ids.bo, ids.dee, ids.ann]); + + expect(await matchingJobs.enqueueRefreshes(eventId, ids.bo, { delaySeconds: 60, fullList: 1 })).toBe(2); + await matchingJobs.enqueueRefreshes(eventId, ids.bo, { delaySeconds: 60, fullList: 1 }); + const pending = (await query<{ participant_id: string; trigger_kind: string }>( + `select participant_id, trigger_kind from matching_jobs where event_id = $1 and status = 'pending'`, [eventId])).rows; + expect(pending.map((row) => row.participant_id).sort()).toEqual([ids.ann, ids.cy].sort()); + expect(pending.every((row) => row.trigger_kind === "refresh")).toBe(true); + }); + + it("keeps a pending profile job a profile job when a refresh joins it", async () => { + await save("ann", "Ann edits again"); + await matchingJobs.enqueue(eventId, ids.ann, "embed", { trigger: "refresh" }); + expect((await query<{ trigger_kind: string }>(`select trigger_kind from matching_jobs where event_id = $1 and participant_id = $2 and job_type = 'embed' and status = 'pending'`, + [eventId, ids.ann])).rows[0].trigger_kind).toBe("profile"); + }); + it("fails a crashed job that was already on its last attempt", async () => { await query(`update matching_jobs set available_at = now() where event_id = $1 and status = 'pending'`, [eventId]); const job = await matchingJobs.claim("embed", eventId); diff --git a/tools/manage-jobs.ts b/tools/manage-jobs.ts new file mode 100644 index 0000000..2a9aa2e --- /dev/null +++ b/tools/manage-jobs.ts @@ -0,0 +1,33 @@ +/** + * Inspect and retry background matching jobs that failed for good: + * + * npm run jobs:failed list them with their last error + * npm run jobs:retry -- queue them again with fresh attempts + */ +import "dotenv/config"; +import { closePool, matchingJobs } from "@match-maker/db"; +import { isMainModule } from "../packages/db/src/is-main"; + +if (isMainModule(import.meta.url)) { + const [command, target] = process.argv.slice(2); + try { + if (command === "failed") { + const failed = await matchingJobs.listFailed(); + if (!failed.length) console.info("[jobs] no failed jobs"); + for (const job of failed) { + console.info(`[jobs] #${job.id} ${job.event_slug} ${job.email ?? "(erased)"} ${job.job_type} after ${job.attempts} attempts: ${job.last_error ?? "no error recorded"}`); + } + } else if (command === "retry" && (target === "all" || /^\d+$/.test(target ?? ""))) { + const count = await matchingJobs.retryFailed(target === "all" ? "all" : Number(target)); + console.info(`[jobs] queued ${count} job(s) again`); + } else { + console.error("Usage: tsx tools/manage-jobs.ts failed | retry "); + process.exitCode = 1; + } + } catch (error) { + console.error(`[jobs] ${(error as Error).message}`); + process.exitCode = 1; + } finally { + await closePool(); + } +} From 74149e223cdfbf8b39778e09f0c0808bb5c370c6 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 09:16:31 +0530 Subject: [PATCH 04/26] Put embedding and rerank calls behind a model gateway (R22 part 1) CircuitBreaker and Bulkhead were empty classes, and provider calls only had their own timeouts: no shared budget, concurrency limit, retry policy, fallback or usage accounting. - CircuitBreaker (opens after 5 failures, one trial call after 30s) and Bulkhead (bounded concurrency and queue per provider) - model-gateway.ts wraps both ports: per-operation time budgets that cover retries, abort signals so a timed-out request really stops, one or two jittered retries for 429/5xx/timeouts/network errors only - Fallbacks: a failed query embedding returns null (search matches by words); a failed profile embedding throws (the job's own backoff retries later); a failed rerank keeps the scored order - Search has a 6-second total budget split between query embedding and rerank; a rerank is skipped when under 750 ms remain - Migration 0013: model_usage records every call (provider, operation, outcome, latency, characters), kept 90 days by the sweep --- .../application/ports/embedding-provider.ts | 10 +- .../web/src/application/ports/llm-provider.ts | 3 +- apps/web/src/application/ports/model-usage.ts | 10 ++ apps/web/src/composition/container.ts | 8 +- apps/web/src/domain/matching/pipeline.ts | 13 +- .../matching/stages/llm-rerank.stage.ts | 11 +- .../postgres/model-usage.repo.ts | 9 ++ .../providers/gemini-embedding.provider.ts | 6 +- .../providers/gemini-llm.provider.ts | 5 +- .../providers/local-bge.provider.ts | 6 +- .../infrastructure/providers/model-gateway.ts | 147 ++++++++++++++++++ .../src/infrastructure/resilience/bulkhead.ts | 50 +++++- .../resilience/circuit-breaker.ts | 49 +++++- packages/db/migrations/0013_model_gateway.sql | 22 +++ packages/db/src/index.ts | 1 + packages/db/src/repositories/housekeeping.ts | 2 + packages/db/src/repositories/model-usage.ts | 42 +++++ 17 files changed, 374 insertions(+), 20 deletions(-) create mode 100644 apps/web/src/application/ports/model-usage.ts create mode 100644 apps/web/src/infrastructure/postgres/model-usage.repo.ts create mode 100644 apps/web/src/infrastructure/providers/model-gateway.ts create mode 100644 packages/db/migrations/0013_model_gateway.sql create mode 100644 packages/db/src/repositories/model-usage.ts diff --git a/apps/web/src/application/ports/embedding-provider.ts b/apps/web/src/application/ports/embedding-provider.ts index 4ee3b2a..4c5def3 100644 --- a/apps/web/src/application/ports/embedding-provider.ts +++ b/apps/web/src/application/ports/embedding-provider.ts @@ -1,5 +1,13 @@ +/** Limits a caller can put on one provider call. */ +export interface ProviderCallOptions { + /** The most time this call may take, including retries. */ + timeoutMs?: number; + /** Aborts the underlying request when the caller gives up. */ + signal?: AbortSignal; +} + export interface EmbeddingProvider { /** Provider and model; vectors from different ids are different spaces and never mixed. */ readonly id: string; - embed(text: string, task: "profile" | "query"): Promise; + embed(text: string, task: "profile" | "query", options?: ProviderCallOptions): Promise; } diff --git a/apps/web/src/application/ports/llm-provider.ts b/apps/web/src/application/ports/llm-provider.ts index 8e38241..12fef3f 100644 --- a/apps/web/src/application/ports/llm-provider.ts +++ b/apps/web/src/application/ports/llm-provider.ts @@ -1,10 +1,11 @@ import type { MatchCandidate } from "../../domain/matching/match-strategy"; import type { ParticipantProfile } from "../../domain/participant/participant.entity"; +import type { ProviderCallOptions } from "./embedding-provider"; export interface LlmProvider { /** Identifies the provider and model, recorded with every reranked result. */ readonly id: string; /** Bump when the prompt changes, so cached explanations are not reused across prompts. */ readonly promptVersion: string; - rerank(viewer: ParticipantProfile, candidates: MatchCandidate[]): Promise; + rerank(viewer: ParticipantProfile, candidates: MatchCandidate[], options?: ProviderCallOptions): Promise; } diff --git a/apps/web/src/application/ports/model-usage.ts b/apps/web/src/application/ports/model-usage.ts new file mode 100644 index 0000000..cf123a8 --- /dev/null +++ b/apps/web/src/application/ports/model-usage.ts @@ -0,0 +1,10 @@ +/** Records each model provider call for usage, cost and reliability reports. Never blocks the call. */ +export interface ModelUsageLog { + record(entry: { + provider: string; + operation: string; + outcome: "ok" | "error" | "timeout" | "rejected"; + latencyMs: number; + inputChars: number; + }): void; +} diff --git a/apps/web/src/composition/container.ts b/apps/web/src/composition/container.ts index 05b44b7..b8b986a 100644 --- a/apps/web/src/composition/container.ts +++ b/apps/web/src/composition/container.ts @@ -11,6 +11,8 @@ import { PostgresExplanationCache } from "../infrastructure/cache/postgres-expla import { UpstashCandidateCache } from "../infrastructure/cache/upstash-candidate-cache"; import { RealtimeNotifier } from "../infrastructure/notifications/realtime.notifier"; import { ProviderRegistry } from "../infrastructure/providers/provider-registry"; +import { GatewayEmbeddingProvider, GatewayLlmProvider } from "../infrastructure/providers/model-gateway"; +import { PostgresModelUsageLog } from "../infrastructure/postgres/model-usage.repo"; import { MatchingPipeline } from "../domain/matching/pipeline"; const globalContainer = globalThis as typeof globalThis & { __matchMakerContainer?: ReturnType }; @@ -20,8 +22,10 @@ function buildContainer() { const matches = new PostgresMatchRepository(); const intros = new PostgresIntroRepository(); const events = new PostgresEventRepository(); - const embeddings = ProviderRegistry.embedding(); - const llm = ProviderRegistry.llm(); + const usage = new PostgresModelUsageLog(); + const embeddings = new GatewayEmbeddingProvider(ProviderRegistry.embedding(), usage); + const rawLlm = ProviderRegistry.llm(); + const llm = rawLlm ? new GatewayLlmProvider(rawLlm, usage) : null; const candidateCache = new UpstashCandidateCache(new PostgresCandidateCache()); return { participants, diff --git a/apps/web/src/domain/matching/pipeline.ts b/apps/web/src/domain/matching/pipeline.ts index 2ae079c..d5d661e 100644 --- a/apps/web/src/domain/matching/pipeline.ts +++ b/apps/web/src/domain/matching/pipeline.ts @@ -40,6 +40,10 @@ export interface MatchingInput { */ export const MATCHING_PIPELINE_VERSION = `v2.${SCORING_CONFIG.version}`; +/** Total time a search may spend on model calls; rerank is skipped if too little is left. */ +export const SEARCH_BUDGET_MS = 6_000; +const QUERY_EMBED_BUDGET_MS = 2_500; + /** The profile changed while matching ran; the newer job will publish instead. */ export class StaleRevisionError extends Error { constructor() { @@ -118,7 +122,7 @@ export class MatchingPipeline { * Both feed the score, so the same query ranks differently per person. */ /** Query embeddings are shared across replicas and searches for a day. */ - private embedQuery(query: string): Promise { + private embedQuery(query: string, timeoutMs: number): Promise { const normalized = query.trim().toLowerCase().replace(/\s+/g, " "); const key = createHash("sha256").update(JSON.stringify(["query-embedding", this.embeddings.id, normalized])).digest("hex"); return computeOnce({ @@ -126,15 +130,16 @@ export class MatchingPipeline { key, read: () => this.cache.getVector(key), write: (vector) => vector ? this.cache.setVector(key, vector, 24 * 60 * 60) : Promise.resolve(), - compute: () => this.embeddings.embed(normalized, "query"), + compute: () => this.embeddings.embed(normalized, "query", { timeoutMs }), shouldStore: (vector) => vector !== null, }); } private async search(input: MatchingInput, viewer: ParticipantProfile) { const query = input.query!.trim(); + const deadline = Date.now() + SEARCH_BUDGET_MS; const terms = queryTerms(query); - const queryEmbedding = await this.embedQuery(query); + const queryEmbedding = await this.embedQuery(query, Math.min(QUERY_EMBED_BUDGET_MS, deadline - Date.now())); const retrieved = await this.retrieve.execute({ eventId: input.eventId, viewerId: input.participantId, @@ -155,7 +160,7 @@ export class MatchingPipeline { }; }).sort((a, b) => b.score - a.score); const top = this.topK.execute(scored, input.limit ?? 12); - const { candidates: ranked } = await this.rerank.execute(viewer, top, queryContext(query)); + const { candidates: ranked } = await this.rerank.execute(viewer, top, queryContext(query), { timeoutMs: deadline - Date.now() }); return ranked; } } diff --git a/apps/web/src/domain/matching/stages/llm-rerank.stage.ts b/apps/web/src/domain/matching/stages/llm-rerank.stage.ts index df0e8c2..74cfaef 100644 --- a/apps/web/src/domain/matching/stages/llm-rerank.stage.ts +++ b/apps/web/src/domain/matching/stages/llm-rerank.stage.ts @@ -16,13 +16,16 @@ export interface RerankResult { * the model is not called and the scored order stands. A cached reason never * decides the order of a shortlist. */ +/** Below this much remaining time a rerank call is not worth starting. */ +const MIN_RERANK_MS = 750; + export class LlmRerankStage { constructor( private readonly provider: LlmProvider | null, private readonly cache: ExplanationCache | null = null ) {} - async execute(viewer: ParticipantProfile, candidates: MatchCandidate[], context = "profile"): Promise { + async execute(viewer: ParticipantProfile, candidates: MatchCandidate[], context = "profile", options: { timeoutMs?: number } = {}): Promise { const scored = { candidates: candidates.map(addFallbackReason), rankedBy: "score" }; if (!this.provider || candidates.length === 0) return scored; const provider = this.provider; @@ -34,9 +37,13 @@ export class LlmRerankStage { const cached = await this.readCache(keys); const withCached = candidates.map((candidate, index) => ({ ...candidate, reason: cached.get(keys[index]) ?? candidate.reason })); if (cached.size === candidates.length) return { candidates: withCached, rankedBy: "score" }; + // Too little of the request's budget left for a model call: keep the scored order. + if (options.timeoutMs !== undefined && options.timeoutMs < MIN_RERANK_MS) { + return { candidates: withCached.map(addFallbackReason), rankedBy: "score" }; + } try { - const reranked = await provider.rerank(viewer, candidates); + const reranked = await provider.rerank(viewer, candidates, { timeoutMs: options.timeoutMs }); await this.writeCache(viewer, reranked.filter((candidate) => candidate.reason).map((candidate) => ({ key: keyFor(candidate), candidateId: candidate.participant.attendeeId, reason: candidate.reason!, }))); diff --git a/apps/web/src/infrastructure/postgres/model-usage.repo.ts b/apps/web/src/infrastructure/postgres/model-usage.repo.ts new file mode 100644 index 0000000..a3dc7eb --- /dev/null +++ b/apps/web/src/infrastructure/postgres/model-usage.repo.ts @@ -0,0 +1,9 @@ +import { modelUsage } from "@match-maker/db"; +import type { ModelUsageLog } from "../../application/ports/model-usage"; + +export class PostgresModelUsageLog implements ModelUsageLog { + record(entry: Parameters[0]): void { + // Fire and forget: accounting must never slow down or fail a model call. + modelUsage.record(entry).catch((error) => console.warn("[usage] could not record", error)); + } +} diff --git a/apps/web/src/infrastructure/providers/gemini-embedding.provider.ts b/apps/web/src/infrastructure/providers/gemini-embedding.provider.ts index 940e965..0fc6024 100644 --- a/apps/web/src/infrastructure/providers/gemini-embedding.provider.ts +++ b/apps/web/src/infrastructure/providers/gemini-embedding.provider.ts @@ -1,11 +1,11 @@ -import type { EmbeddingProvider } from "../../application/ports/embedding-provider"; +import type { EmbeddingProvider, ProviderCallOptions } from "../../application/ports/embedding-provider"; const DIMENSIONS = 768; export class GeminiEmbeddingProvider implements EmbeddingProvider { readonly id = `gemini:gemini-embedding-001:${DIMENSIONS}`; - async embed(text: string, task: "profile" | "query"): Promise { + async embed(text: string, task: "profile" | "query", options: ProviderCallOptions = {}): Promise { const apiKey = process.env.GEMINI_API_KEY; if (!apiKey || !text.trim()) return null; const response = await fetch("https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:embedContent", { @@ -16,7 +16,7 @@ export class GeminiEmbeddingProvider implements EmbeddingProvider { taskType: task === "profile" ? "RETRIEVAL_DOCUMENT" : "RETRIEVAL_QUERY", outputDimensionality: DIMENSIONS, }), - signal: AbortSignal.timeout(15_000), + signal: options.signal ? AbortSignal.any([options.signal, AbortSignal.timeout(15_000)]) : AbortSignal.timeout(15_000), }); if (!response.ok) throw new Error(`Gemini embedding failed (${response.status}): ${(await response.text()).slice(0, 300)}`); const payload = await response.json() as { embedding?: { values?: number[] } }; diff --git a/apps/web/src/infrastructure/providers/gemini-llm.provider.ts b/apps/web/src/infrastructure/providers/gemini-llm.provider.ts index d0c9d5d..9a9ae80 100644 --- a/apps/web/src/infrastructure/providers/gemini-llm.provider.ts +++ b/apps/web/src/infrastructure/providers/gemini-llm.provider.ts @@ -1,4 +1,5 @@ import type { LlmProvider } from "../../application/ports/llm-provider"; +import type { ProviderCallOptions } from "../../application/ports/embedding-provider"; import type { MatchCandidate } from "../../domain/matching/match-strategy"; import type { ParticipantProfile } from "../../domain/participant/participant.entity"; @@ -11,7 +12,7 @@ export class GeminiLlmProvider implements LlmProvider { return `gemini:${process.env.GEMINI_LLM_MODEL || DEFAULT_MODEL}`; } - async rerank(viewer: ParticipantProfile, candidates: MatchCandidate[]): Promise { + async rerank(viewer: ParticipantProfile, candidates: MatchCandidate[], options: ProviderCallOptions = {}): Promise { const apiKey = process.env.GEMINI_API_KEY; if (!apiKey) return candidates; const model = process.env.GEMINI_LLM_MODEL || DEFAULT_MODEL; @@ -41,7 +42,7 @@ export class GeminiLlmProvider implements LlmProvider { contents: [{ parts: [{ text: prompt }] }], generationConfig: { temperature: 0.2, maxOutputTokens: 2048, responseMimeType: "application/json" }, }), - signal: AbortSignal.timeout(20_000), + signal: options.signal ? AbortSignal.any([options.signal, AbortSignal.timeout(20_000)]) : AbortSignal.timeout(20_000), }); if (!response.ok) throw new Error(`Gemini rerank failed (${response.status}): ${(await response.text()).slice(0, 300)}`); const payload = await response.json() as { candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }> }; diff --git a/apps/web/src/infrastructure/providers/local-bge.provider.ts b/apps/web/src/infrastructure/providers/local-bge.provider.ts index ea7fa99..e1d16c8 100644 --- a/apps/web/src/infrastructure/providers/local-bge.provider.ts +++ b/apps/web/src/infrastructure/providers/local-bge.provider.ts @@ -1,17 +1,17 @@ -import type { EmbeddingProvider } from "../../application/ports/embedding-provider"; +import type { EmbeddingProvider, ProviderCallOptions } from "../../application/ports/embedding-provider"; /** Adapter for an internal sentence-transformers inference endpoint. */ export class LocalBgeProvider implements EmbeddingProvider { readonly id = "local-bge:768"; - async embed(text: string): Promise { + async embed(text: string, _task: "profile" | "query" = "profile", options: ProviderCallOptions = {}): Promise { const endpoint = process.env.LOCAL_BGE_URL; if (!endpoint) return null; const response = await fetch(`${endpoint.replace(/\/$/, "")}/embed`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text, normalize: true }), - signal: AbortSignal.timeout(20_000), + signal: options.signal ? AbortSignal.any([options.signal, AbortSignal.timeout(20_000)]) : AbortSignal.timeout(20_000), }); if (!response.ok) throw new Error(`Local embedding service failed (${response.status})`); const result = await response.json() as { embedding?: number[] } | number[]; diff --git a/apps/web/src/infrastructure/providers/model-gateway.ts b/apps/web/src/infrastructure/providers/model-gateway.ts new file mode 100644 index 0000000..dec68dc --- /dev/null +++ b/apps/web/src/infrastructure/providers/model-gateway.ts @@ -0,0 +1,147 @@ +import type { EmbeddingProvider, ProviderCallOptions } from "../../application/ports/embedding-provider"; +import type { LlmProvider } from "../../application/ports/llm-provider"; +import type { ModelUsageLog } from "../../application/ports/model-usage"; +import type { MatchCandidate } from "../../domain/matching/match-strategy"; +import type { ParticipantProfile } from "../../domain/participant/participant.entity"; +import { Bulkhead, BulkheadFullError } from "../resilience/bulkhead"; +import { CircuitBreaker, CircuitOpenError } from "../resilience/circuit-breaker"; + +/** Per-operation limits: a time budget for the whole call and how many quick retries fit in it. */ +export interface GatewayPolicy { + timeoutMs: number; + retries: number; +} + +export const POLICIES = { + embedQuery: { timeoutMs: 2_500, retries: 1 }, + embedProfile: { timeoutMs: 15_000, retries: 2 }, + rerank: { timeoutMs: 20_000, retries: 1 }, +} satisfies Record; + +/** Rate limits, server errors, timeouts and network failures are worth one more try. */ +export function isTransient(error: unknown): boolean { + const message = error instanceof Error ? `${error.name} ${error.message}` : String(error); + return /\((429|5\d\d)\)|timeout|aborted|fetch failed|ECONNRESET|ETIMEDOUT|EAI_AGAIN/i.test(message); +} + +interface Guards { + breaker: CircuitBreaker; + bulkhead: Bulkhead; + usage: ModelUsageLog | null; +} + +/** + * Runs one provider call under the gateway's rules: a concurrency limit, a + * circuit breaker, a time budget for the whole call (including retries), and + * a couple of jittered retries for transient errors. Every attempt is + * recorded for usage and cost reports. Provider retries stay separate from + * background job retries, which take over when this gives up. + */ +export async function guardedCall(input: { + provider: string; + operation: string; + inputChars: number; + policy: GatewayPolicy; + budgetMs?: number; + guards: Guards; + call: (signal: AbortSignal) => Promise; +}): Promise { + const deadline = Date.now() + Math.min(input.policy.timeoutMs, input.budgetMs ?? Infinity); + for (let attempt = 0; ; attempt++) { + const remaining = deadline - Date.now(); + if (remaining <= 50) throw new Error(`${input.provider} ${input.operation} timeout: no time left in the budget`); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new Error(`${input.provider} ${input.operation} timeout`)), remaining); + const started = Date.now(); + try { + const result = await input.guards.bulkhead.run(() => input.guards.breaker.execute(() => input.call(controller.signal))); + input.guards.usage?.record({ provider: input.provider, operation: input.operation, outcome: "ok", latencyMs: Date.now() - started, inputChars: input.inputChars }); + return result; + } catch (error) { + const rejected = error instanceof CircuitOpenError || error instanceof BulkheadFullError; + const outcome = rejected ? "rejected" : controller.signal.aborted ? "timeout" : "error"; + input.guards.usage?.record({ provider: input.provider, operation: input.operation, outcome, latencyMs: Date.now() - started, inputChars: input.inputChars }); + const retryable = !rejected && (controller.signal.aborted ? false : isTransient(error)); + if (!retryable || attempt >= input.policy.retries) throw error; + const backoff = Math.min(100 * 2 ** attempt + Math.random() * 100, deadline - Date.now() - 100); + if (backoff <= 0) throw error; + await new Promise((resolve) => setTimeout(resolve, backoff)); + } finally { + clearTimeout(timer); + } + } +} + +/** + * Embedding calls through the gateway. A failed query embedding returns null, + * so search falls back to its word-matching path; a failed profile embedding + * throws, so the background job retries later. + */ +export class GatewayEmbeddingProvider implements EmbeddingProvider { + private readonly guards: Guards; + + constructor(private readonly inner: EmbeddingProvider, usage: ModelUsageLog | null) { + this.guards = { + breaker: new CircuitBreaker(`embeddings:${inner.id}`), + bulkhead: new Bulkhead(`embeddings:${inner.id}`, { maxConcurrent: 8, maxQueue: 32, queueTimeoutMs: 2_000 }), + usage, + }; + } + + get id(): string { + return this.inner.id; + } + + async embed(text: string, task: "profile" | "query", options: ProviderCallOptions = {}): Promise { + try { + return await guardedCall({ + provider: this.inner.id, + operation: `embed:${task}`, + inputChars: text.length, + policy: task === "query" ? POLICIES.embedQuery : POLICIES.embedProfile, + budgetMs: options.timeoutMs, + guards: this.guards, + call: (signal) => this.inner.embed(text, task, { ...options, signal }), + }); + } catch (error) { + if (task === "query") { + console.warn("[gateway] query embedding unavailable; searching by words", (error as Error).message); + return null; + } + throw error; + } + } +} + +/** Rerank calls through the gateway; failures surface so the stage keeps the scored order. */ +export class GatewayLlmProvider implements LlmProvider { + private readonly guards: Guards; + + constructor(private readonly inner: LlmProvider, usage: ModelUsageLog | null) { + this.guards = { + breaker: new CircuitBreaker(`llm:${inner.id}`), + bulkhead: new Bulkhead(`llm:${inner.id}`, { maxConcurrent: 4, maxQueue: 16, queueTimeoutMs: 2_000 }), + usage, + }; + } + + get id(): string { + return this.inner.id; + } + + get promptVersion(): string { + return this.inner.promptVersion; + } + + rerank(viewer: ParticipantProfile, candidates: MatchCandidate[], options: ProviderCallOptions = {}): Promise { + return guardedCall({ + provider: this.inner.id, + operation: "rerank", + inputChars: JSON.stringify(candidates.map((candidate) => candidate.participant.aboutMe ?? "")).length, + policy: POLICIES.rerank, + budgetMs: options.timeoutMs, + guards: this.guards, + call: (signal) => this.inner.rerank(viewer, candidates, { ...options, signal }), + }); + } +} diff --git a/apps/web/src/infrastructure/resilience/bulkhead.ts b/apps/web/src/infrastructure/resilience/bulkhead.ts index 579f032..126bd5a 100644 --- a/apps/web/src/infrastructure/resilience/bulkhead.ts +++ b/apps/web/src/infrastructure/resilience/bulkhead.ts @@ -1 +1,49 @@ -export class Bulkhead {} +/** Thrown when a provider already has as many calls running and waiting as allowed. */ +export class BulkheadFullError extends Error { + constructor(name: string) { + super(`${name} is busy`); + this.name = "BulkheadFullError"; + } +} + +/** + * Limits concurrent calls to one provider in this process. Extra calls wait + * in a bounded queue for at most `queueTimeoutMs`, so a slow provider cannot + * tie up every request or exceed its rate limits. + */ +export class Bulkhead { + private running = 0; + private readonly waiting: Array<() => void> = []; + + constructor( + private readonly name: string, + private readonly options: { maxConcurrent: number; maxQueue: number; queueTimeoutMs: number } = { maxConcurrent: 8, maxQueue: 32, queueTimeoutMs: 2_000 } + ) {} + + async run(call: () => Promise): Promise { + if (this.running >= this.options.maxConcurrent) await this.wait(); + this.running += 1; + try { + return await call(); + } finally { + this.running -= 1; + this.waiting.shift()?.(); + } + } + + private wait(): Promise { + if (this.waiting.length >= this.options.maxQueue) return Promise.reject(new BulkheadFullError(this.name)); + return new Promise((resolve, reject) => { + const turn = () => { + clearTimeout(timer); + resolve(); + }; + const timer = setTimeout(() => { + const index = this.waiting.indexOf(turn); + if (index >= 0) this.waiting.splice(index, 1); + reject(new BulkheadFullError(this.name)); + }, this.options.queueTimeoutMs); + this.waiting.push(turn); + }); + } +} diff --git a/apps/web/src/infrastructure/resilience/circuit-breaker.ts b/apps/web/src/infrastructure/resilience/circuit-breaker.ts index 4068672..8b1a2ae 100644 --- a/apps/web/src/infrastructure/resilience/circuit-breaker.ts +++ b/apps/web/src/infrastructure/resilience/circuit-breaker.ts @@ -1 +1,48 @@ -export class CircuitBreaker {} +/** Thrown instead of calling a provider that keeps failing. */ +export class CircuitOpenError extends Error { + constructor(name: string) { + super(`${name} is temporarily unavailable`); + this.name = "CircuitOpenError"; + } +} + +/** + * Stops calling a provider after `failureThreshold` consecutive failures, for + * `resetAfterMs`. Then one trial call is let through: success closes the + * circuit, failure opens it again. Keeps an outage from adding a timeout to + * every request. + */ +export class CircuitBreaker { + private failures = 0; + private openedAt: number | null = null; + private trialRunning = false; + + constructor( + private readonly name: string, + private readonly options: { failureThreshold: number; resetAfterMs: number } = { failureThreshold: 5, resetAfterMs: 30_000 }, + private readonly now: () => number = Date.now + ) {} + + get state(): "closed" | "open" | "half-open" { + if (this.openedAt === null) return "closed"; + return this.now() - this.openedAt >= this.options.resetAfterMs ? "half-open" : "open"; + } + + async execute(call: () => Promise): Promise { + const state = this.state; + if (state === "open" || (state === "half-open" && this.trialRunning)) throw new CircuitOpenError(this.name); + if (state === "half-open") this.trialRunning = true; + try { + const result = await call(); + this.failures = 0; + this.openedAt = null; + return result; + } catch (error) { + this.failures += 1; + if (state === "half-open" || this.failures >= this.options.failureThreshold) this.openedAt = this.now(); + throw error; + } finally { + if (state === "half-open") this.trialRunning = false; + } + } +} diff --git a/packages/db/migrations/0013_model_gateway.sql b/packages/db/migrations/0013_model_gateway.sql new file mode 100644 index 0000000..91fac31 --- /dev/null +++ b/packages/db/migrations/0013_model_gateway.sql @@ -0,0 +1,22 @@ +-- Model gateway accounting and versioned embeddings. +-- +-- model_usage: one row per provider call (provider and model, operation, +-- outcome, latency, approximate tokens) for cost and reliability reports. +-- Kept 90 days by the housekeeping sweep. +-- +-- event_participants.embedding_model: which provider/model produced the +-- stored vector. Vectors from different models live in different spaces and +-- are never compared; a profile embedded by another model is embedded again. + +create table if not exists model_usage ( + id bigserial primary key, + provider text not null, + operation text not null, + outcome text not null check (outcome in ('ok', 'error', 'timeout', 'rejected')), + latency_ms integer not null, + input_chars integer not null default 0, + created_at timestamptz not null default now() +); +create index if not exists idx_model_usage_created on model_usage (created_at); + +alter table event_participants add column if not exists embedding_model text; diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 596eb49..94bc070 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -13,3 +13,4 @@ export * as events from "./repositories/events"; export * as matchingJobs from "./repositories/matching-jobs"; export * as housekeeping from "./repositories/housekeeping"; export * as audit from "./repositories/audit"; +export * as modelUsage from "./repositories/model-usage"; diff --git a/packages/db/src/repositories/housekeeping.ts b/packages/db/src/repositories/housekeeping.ts index 7fe1803..c6798bb 100644 --- a/packages/db/src/repositories/housekeeping.ts +++ b/packages/db/src/repositories/housekeeping.ts @@ -1,6 +1,7 @@ import { query } from "../client"; import { deleteExpired as deleteExpiredChallenges } from "./login-challenges"; import { AUDIT_RETENTION_DAYS } from "./audit"; +import { USAGE_RETENTION_DAYS } from "./model-usage"; /** How long a dead row is kept before the sweep removes it. */ export const RETENTION = { @@ -28,6 +29,7 @@ export async function sweepExpired(): Promise { const cache = await query(`delete from retrieval_cache where expires_at < now()`); const explanations = await query(`delete from pair_explanations where expires_at < now()`); await query(`delete from cache_locks where expires_at < now()`); + await query(`delete from model_usage where created_at < now() - make_interval(days => $1)`, [USAGE_RETENTION_DAYS]); const audits = await query(`delete from audit_events where created_at < now() - make_interval(days => $1)`, [AUDIT_RETENTION_DAYS]); return { loginChallenges, rateLimitBuckets: buckets.rowCount ?? 0, retrievalCache: cache.rowCount ?? 0, diff --git a/packages/db/src/repositories/model-usage.ts b/packages/db/src/repositories/model-usage.ts new file mode 100644 index 0000000..fe003f8 --- /dev/null +++ b/packages/db/src/repositories/model-usage.ts @@ -0,0 +1,42 @@ +import { query } from "../client"; + +/** Usage rows are kept this long, then swept. */ +export const USAGE_RETENTION_DAYS = 90; + +export async function record(entry: { + provider: string; + operation: string; + outcome: "ok" | "error" | "timeout" | "rejected"; + latencyMs: number; + inputChars: number; +}): Promise { + await query( + `insert into model_usage (provider, operation, outcome, latency_ms, input_chars) values ($1, $2, $3, $4, $5)`, + [entry.provider, entry.operation, entry.outcome, Math.round(entry.latencyMs), entry.inputChars] + ); +} + +export interface UsageSummary { + provider: string; + operation: string; + calls: number; + failures: number; + avg_latency_ms: number; + p95_latency_ms: number; + /** Rough token estimate (characters / 4). */ + approx_tokens: number; +} + +export async function summary(hours = 24): Promise { + const { rows } = await query( + `select provider, operation, count(*)::int as calls, + count(*) filter (where outcome <> 'ok')::int as failures, + round(avg(latency_ms))::int as avg_latency_ms, + round(percentile_cont(0.95) within group (order by latency_ms))::int as p95_latency_ms, + (sum(input_chars) / 4)::int as approx_tokens + from model_usage where created_at > now() - make_interval(hours => $1) + group by provider, operation order by provider, operation`, + [hours] + ); + return rows; +} From b9fc8ee23fffe8f02b74e372676314fadafa9364 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 09:16:31 +0530 Subject: [PATCH 05/26] Test the model gateway and add a usage report (R22 part 2) - Unit tests: circuit breaker open/half-open/close, bulkhead limits and queue timeout, transient-only retries with every attempt recorded, budget timeouts, query vs profile fallbacks, circuit stopping calls, and rerank skipped when the request is nearly out of time - npm run models:usage [-- ] reports calls, failures, average and p95 latency, approximate tokens and, with MODEL_PRICE_PER_1K_TOKENS, estimated cost per provider and operation --- package.json | 3 +- tests/unit/model-gateway.test.ts | 115 +++++++++++++++++++++++++++++++ tools/model-usage.ts | 28 ++++++++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 tests/unit/model-gateway.test.ts create mode 100644 tools/model-usage.ts diff --git a/package.json b/package.json index 1aa9a2d..36c262a 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "account:revoke-sessions": "tsx tools/manage-account.ts revoke-sessions", "account:erase": "tsx tools/manage-account.ts erase", "jobs:failed": "tsx tools/manage-jobs.ts failed", - "jobs:retry": "tsx tools/manage-jobs.ts retry" + "jobs:retry": "tsx tools/manage-jobs.ts retry", + "models:usage": "tsx tools/model-usage.ts" }, "devDependencies": { "@types/node": "^22.10.2", diff --git a/tests/unit/model-gateway.test.ts b/tests/unit/model-gateway.test.ts new file mode 100644 index 0000000..4670b88 --- /dev/null +++ b/tests/unit/model-gateway.test.ts @@ -0,0 +1,115 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { CircuitBreaker, CircuitOpenError } from "@/src/infrastructure/resilience/circuit-breaker"; +import { Bulkhead, BulkheadFullError } from "@/src/infrastructure/resilience/bulkhead"; +import { GatewayEmbeddingProvider, GatewayLlmProvider, guardedCall, isTransient } from "@/src/infrastructure/providers/model-gateway"; +import { LlmRerankStage } from "@/src/domain/matching/stages/llm-rerank.stage"; +import type { ModelUsageLog } from "@/src/application/ports/model-usage"; +import { Intent } from "@/src/domain/participant/intent"; +import type { ParticipantProfile } from "@/src/domain/participant/participant.entity"; + +const usage = (): ModelUsageLog & { entries: Array<{ outcome: string }> } => { + const entries: Array<{ outcome: string }> = []; + return { entries, record: vi.fn((entry) => { entries.push(entry); }) }; +}; +const guards = (log = usage()) => ({ breaker: new CircuitBreaker("t"), bulkhead: new Bulkhead("t"), usage: log }); +const viewer: ParticipantProfile = { + eventId: "e", attendeeId: "v", name: "v", designation: null, domain: null, aboutMe: null, selfDescription: null, + yearsOfExperience: null, mostImpressiveBuild: null, intent: Intent.NETWORKING, interests: [], venueZone: null, visible: true, +}; + +beforeEach(() => { + vi.spyOn(console, "warn").mockImplementation(() => {}); +}); + +describe("circuit breaker", () => { + it("opens after repeated failures, then lets one trial call through", async () => { + let now = 0; + const breaker = new CircuitBreaker("p", { failureThreshold: 2, resetAfterMs: 1000 }, () => now); + const fail = () => breaker.execute(async () => { throw new Error("down"); }); + await expect(fail()).rejects.toThrow("down"); + await expect(fail()).rejects.toThrow("down"); + await expect(breaker.execute(async () => "ok")).rejects.toBeInstanceOf(CircuitOpenError); + now = 1000; + expect(breaker.state).toBe("half-open"); + await expect(breaker.execute(async () => "ok")).resolves.toBe("ok"); + expect(breaker.state).toBe("closed"); + }); +}); + +describe("bulkhead", () => { + it("limits concurrency, queues briefly, and rejects when full", async () => { + const bulkhead = new Bulkhead("p", { maxConcurrent: 1, maxQueue: 1, queueTimeoutMs: 50 }); + let release!: () => void; + const first = bulkhead.run(() => new Promise((resolve) => { release = () => resolve("first"); })); + const second = bulkhead.run(async () => "second"); + await expect(bulkhead.run(async () => "third")).rejects.toBeInstanceOf(BulkheadFullError); + release(); + await expect(Promise.all([first, second])).resolves.toEqual(["first", "second"]); + }); + + it("gives up waiting after the queue timeout", async () => { + const bulkhead = new Bulkhead("p", { maxConcurrent: 1, maxQueue: 5, queueTimeoutMs: 30 }); + const busy = bulkhead.run(() => new Promise((resolve) => setTimeout(resolve, 200))); + await expect(bulkhead.run(async () => "late")).rejects.toBeInstanceOf(BulkheadFullError); + await busy; + }); +}); + +describe("guarded provider calls", () => { + it("retries a transient error and records every attempt", async () => { + const log = usage(); + const call = vi.fn().mockRejectedValueOnce(new Error("Gemini embedding failed (429): slow down")).mockResolvedValue("ok"); + await expect(guardedCall({ provider: "p", operation: "embed:query", inputChars: 4, policy: { timeoutMs: 2000, retries: 1 }, guards: guards(log), call })) + .resolves.toBe("ok"); + expect(call).toHaveBeenCalledTimes(2); + expect(log.entries.map((entry) => entry.outcome)).toEqual(["error", "ok"]); + }); + + it("does not retry a permanent error", async () => { + const call = vi.fn().mockRejectedValue(new Error("Gemini embedding failed (400): bad request")); + await expect(guardedCall({ provider: "p", operation: "x", inputChars: 0, policy: { timeoutMs: 2000, retries: 2 }, guards: guards(), call })).rejects.toThrow("(400)"); + expect(call).toHaveBeenCalledTimes(1); + }); + + it("aborts a call that exceeds its budget", async () => { + const log = usage(); + const call = (signal: AbortSignal) => new Promise((_, reject) => signal.addEventListener("abort", () => reject(signal.reason))); + await expect(guardedCall({ provider: "p", operation: "x", inputChars: 0, policy: { timeoutMs: 10_000, retries: 2 }, budgetMs: 80, guards: guards(log), call })).rejects.toThrow(/timeout/); + expect(log.entries.map((entry) => entry.outcome)).toEqual(["timeout"]); + }); + + it("classifies transient errors", () => { + expect(isTransient(new Error("failed (503)"))).toBe(true); + expect(isTransient(new TypeError("fetch failed"))).toBe(true); + expect(isTransient(new Error("failed (401)"))).toBe(false); + }); +}); + +describe("gateway fallbacks", () => { + it("returns no query embedding on failure (search uses words) but surfaces profile failures", async () => { + const inner = { id: "m", embed: vi.fn().mockRejectedValue(new Error("failed (401)")) }; + const gateway = new GatewayEmbeddingProvider(inner, usage()); + await expect(gateway.embed("climate", "query")).resolves.toBeNull(); + await expect(gateway.embed("profile text", "profile")).rejects.toThrow("(401)"); + expect(gateway.id).toBe("m"); + }); + + it("stops calling a failing model once its circuit opens", async () => { + const inner = { id: "m", embed: vi.fn().mockRejectedValue(new Error("failed (500)")) }; + const gateway = new GatewayEmbeddingProvider(inner, usage()); + for (let i = 0; i < 6; i++) await gateway.embed("q", "query"); + const calls = inner.embed.mock.calls.length; + await gateway.embed("q", "query"); + expect(inner.embed.mock.calls.length).toBe(calls); + }); + + it("skips the rerank when the request has too little time left", async () => { + const rerank = vi.fn(async (_viewer, list) => list); + const llm = new GatewayLlmProvider({ id: "llm", promptVersion: "p1", rerank }, usage()); + const candidate = { participant: { ...viewer, attendeeId: "c" }, score: 50, scoreBreakdown: {}, mutualInterest: false, reason: null }; + const result = await new LlmRerankStage(llm).execute(viewer, [candidate], "profile", { timeoutMs: 200 }); + expect(rerank).not.toHaveBeenCalled(); + expect(result.rankedBy).toBe("score"); + expect(result.candidates[0].reason).toBeTruthy(); + }); +}); diff --git a/tools/model-usage.ts b/tools/model-usage.ts new file mode 100644 index 0000000..8c7400b --- /dev/null +++ b/tools/model-usage.ts @@ -0,0 +1,28 @@ +/** + * Model usage, reliability and approximate cost for the last N hours: + * + * npm run models:usage [-- ] + * + * Set MODEL_PRICE_PER_1K_TOKENS (USD) to include an estimated cost. + */ +import "dotenv/config"; +import { closePool, modelUsage } from "@match-maker/db"; +import { isMainModule } from "../packages/db/src/is-main"; + +if (isMainModule(import.meta.url)) { + const hours = Number(process.argv[2] ?? 24); + const price = Number(process.env.MODEL_PRICE_PER_1K_TOKENS ?? NaN); + try { + const rows = await modelUsage.summary(Number.isFinite(hours) && hours > 0 ? hours : 24); + if (!rows.length) console.info(`[usage] no model calls in the last ${hours} hours`); + for (const row of rows) { + const cost = Number.isFinite(price) ? ` ~$${((row.approx_tokens / 1000) * price).toFixed(4)}` : ""; + console.info(`[usage] ${row.provider} ${row.operation}: ${row.calls} calls, ${row.failures} failed, avg ${row.avg_latency_ms} ms, p95 ${row.p95_latency_ms} ms, ~${row.approx_tokens} tokens${cost}`); + } + } catch (error) { + console.error(`[usage] ${(error as Error).message}`); + process.exitCode = 1; + } finally { + await closePool(); + } +} From 0bb9a41868de825cabc72c60980ce1587c10820b Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 09:19:16 +0530 Subject: [PATCH 06/26] Version stored embeddings by model and never mix spaces (R22 part 3) Vectors from different providers or models live in different spaces, but nothing recorded which model produced a stored embedding, so switching providers would compare unrelated vectors. - event_participants.embedding_model (migration 0013) is written with every embedding (embedding_model of the gateway's provider id) - Retrieval ranks by vector only among candidates embedded by the query's model; others keep their place by text relevance - The semantic scorer and query relevance compare vectors only within one model, falling back to text otherwise - A profile embedded by another model (or before this change) is embedded again with the current one on its next matching run - Tests: scorer and relevance fallbacks, re-embedding, and real-Postgres retrieval that ignores an exact match from another model --- .../ports/participant-repository.ts | 6 ++- apps/web/src/domain/matching/pipeline.ts | 13 ++++-- .../matching/scorers/semantic.scorer.ts | 4 +- apps/web/src/domain/matching/search-query.ts | 5 +- .../domain/matching/stages/retrieve.stage.ts | 2 + .../domain/participant/participant.entity.ts | 2 + .../postgres/participant.repo.ts | 25 +++++----- packages/db/workers/embed.worker.ts | 2 +- tests/integration/search-retrieval.test.ts | 12 +++++ tests/unit/embedding-versions.test.ts | 46 +++++++++++++++++++ 10 files changed, 95 insertions(+), 22 deletions(-) create mode 100644 tests/unit/embedding-versions.test.ts diff --git a/apps/web/src/application/ports/participant-repository.ts b/apps/web/src/application/ports/participant-repository.ts index e03b7e2..c665385 100644 --- a/apps/web/src/application/ports/participant-repository.ts +++ b/apps/web/src/application/ports/participant-repository.ts @@ -12,6 +12,8 @@ export interface ParticipantRepository { /** Without an embedding, only candidates matching one of these words, best first. */ terms?: string[]; filters?: SearchFilters; + /** The model behind `embedding`; vectors from other models are not ranked against it. */ + embeddingModel?: string | null; limit?: number; }): Promise; /** Changes whenever the event's participants change; part of retrieval cache keys. */ @@ -19,9 +21,9 @@ export interface ParticipantRepository { /** Domains present among eligible participants, for the search filter. */ listDomains(eventId: string): Promise; findCandidatesByIds(eventId: string, ids: string[]): Promise; - setEmbedding(eventId: string, attendeeId: string, embedding: number[]): Promise; + setEmbedding(eventId: string, attendeeId: string, embedding: number[], model: string): Promise; /** Writes the embedding only while the profile is still at `revision`; false when it moved on. */ - setEmbeddingAt(eventId: string, attendeeId: string, embedding: number[], revision: number): Promise; + setEmbeddingAt(eventId: string, attendeeId: string, embedding: number[], revision: number, model: string): Promise; shareVenueZone(eventId: string, attendeeId: string, zone: string): Promise; hasEventAccess(eventId: string, attendeeId: string): Promise; } diff --git a/apps/web/src/domain/matching/pipeline.ts b/apps/web/src/domain/matching/pipeline.ts index d5d661e..fec3485 100644 --- a/apps/web/src/domain/matching/pipeline.ts +++ b/apps/web/src/domain/matching/pipeline.ts @@ -94,16 +94,19 @@ export class MatchingPipeline { if (input.mode === "search") return this.search(input, storedProfile); - const embedding = storedProfile.embedding ?? await this.embeddings.embed(profileText(storedProfile), "profile"); - if (embedding && !storedProfile.embedding) { - await this.participants.setEmbedding(input.eventId, input.participantId, embedding); + // A vector from another model is a different space: embed again with the current one. + const current = storedProfile.embeddingModel === this.embeddings.id ? storedProfile.embedding ?? null : null; + const embedding = current ?? await this.embeddings.embed(profileText(storedProfile), "profile"); + if (embedding && !current) { + await this.participants.setEmbedding(input.eventId, input.participantId, embedding, this.embeddings.id); } - const viewer = { ...storedProfile, embedding: embedding ?? storedProfile.embedding }; + const viewer = { ...storedProfile, embedding, embeddingModel: embedding ? this.embeddings.id : null }; const retrieved = await this.retrieve.execute({ eventId: input.eventId, viewerId: input.participantId, embedding, + embeddingSpace: this.embeddings.id, limit: 50, }); const compatible = this.filter.execute(viewer, retrieved); @@ -152,7 +155,7 @@ export class MatchingPipeline { }); const compatible = this.filter.execute(viewer, retrieved); const scored = this.score.execute(viewer, compatible).map((candidate) => { - const relevance = queryRelevance(terms, queryEmbedding, candidate.participant); + const relevance = queryRelevance(terms, queryEmbedding, candidate.participant, this.embeddings.id); return { ...candidate, score: searchScore(relevance, candidate.score), diff --git a/apps/web/src/domain/matching/scorers/semantic.scorer.ts b/apps/web/src/domain/matching/scorers/semantic.scorer.ts index a89f0b7..3b9dd49 100644 --- a/apps/web/src/domain/matching/scorers/semantic.scorer.ts +++ b/apps/web/src/domain/matching/scorers/semantic.scorer.ts @@ -7,7 +7,9 @@ export class SemanticScorer implements Scorer { readonly weight = SCORING_CONFIG.weights.semantic; score(viewer: ParticipantProfile, candidate: ParticipantProfile): number { - const vectorScore = cosineSimilarity(viewer.embedding, candidate.embedding); + // Vectors from different models are not comparable; use the text fallback instead. + const sameSpace = viewer.embeddingModel === undefined || candidate.embeddingModel === undefined || viewer.embeddingModel === candidate.embeddingModel; + const vectorScore = sameSpace ? cosineSimilarity(viewer.embedding, candidate.embedding) : 0; if (vectorScore > 0) return vectorScore; const left = tokenize([viewer.aboutMe, viewer.selfDescription, viewer.mostImpressiveBuild].join(" ")); const right = tokenize([candidate.aboutMe, candidate.selfDescription, candidate.mostImpressiveBuild].join(" ")); diff --git a/apps/web/src/domain/matching/search-query.ts b/apps/web/src/domain/matching/search-query.ts index 31ee5aa..216c7ac 100644 --- a/apps/web/src/domain/matching/search-query.ts +++ b/apps/web/src/domain/matching/search-query.ts @@ -62,8 +62,9 @@ export function lexicalRelevance(terms: string[], profile: ParticipantProfile): } /** How well a candidate answers the query: semantic when both vectors exist, otherwise lexical. */ -export function queryRelevance(terms: string[], queryEmbedding: number[] | null, profile: ParticipantProfile): number { - const semantic = cosineSimilarity(queryEmbedding, profile.embedding); +export function queryRelevance(terms: string[], queryEmbedding: number[] | null, profile: ParticipantProfile, queryModel?: string): number { + const comparable = queryModel === undefined || profile.embeddingModel === undefined || profile.embeddingModel === queryModel; + const semantic = comparable ? cosineSimilarity(queryEmbedding, profile.embedding) : 0; return semantic > 0 ? semantic : lexicalRelevance(terms, profile); } diff --git a/apps/web/src/domain/matching/stages/retrieve.stage.ts b/apps/web/src/domain/matching/stages/retrieve.stage.ts index 019b01d..e7c6598 100644 --- a/apps/web/src/domain/matching/stages/retrieve.stage.ts +++ b/apps/web/src/domain/matching/stages/retrieve.stage.ts @@ -46,6 +46,7 @@ export class RetrieveStage { eventId: input.eventId, excludeAttendeeId: null, embedding: input.embedding, + embeddingModel: input.embeddingSpace, terms: input.terms, filters, limit, @@ -62,6 +63,7 @@ export class RetrieveStage { eventId: input.eventId, excludeAttendeeId: input.viewerId, embedding: input.embedding, + embeddingModel: input.embeddingSpace, limit, }); } diff --git a/apps/web/src/domain/participant/participant.entity.ts b/apps/web/src/domain/participant/participant.entity.ts index f27cd4c..fed1537 100644 --- a/apps/web/src/domain/participant/participant.entity.ts +++ b/apps/web/src/domain/participant/participant.entity.ts @@ -26,6 +26,8 @@ export interface ParticipantProfile { /** Version of the event's profile form this profile was saved under. */ formVersion?: number; embedding?: number[] | null; + /** The provider/model that produced `embedding`; vectors are only compared within one model. */ + embeddingModel?: string | null; } export type ParticipantProfileInput = Pick< diff --git a/apps/web/src/infrastructure/postgres/participant.repo.ts b/apps/web/src/infrastructure/postgres/participant.repo.ts index 962a390..a12541c 100644 --- a/apps/web/src/infrastructure/postgres/participant.repo.ts +++ b/apps/web/src/infrastructure/postgres/participant.repo.ts @@ -24,6 +24,7 @@ interface ProfileRow { profile_updated_at?: Date | null; profile_form_version?: number; profile_revision?: number; + embedding_model?: string | null; } const PROFILE_COLUMNS = `p.event_id, a.id as attendee_id, a.name, @@ -34,7 +35,7 @@ const PROFILE_COLUMNS = `p.event_id, a.id as attendee_id, a.name, coalesce(p.years_of_experience, a.years_of_experience) as years_of_experience, coalesce(p.most_impressive_build, a.most_impressive_build) as most_impressive_build, p.intent, p.interests, - p.venue_zone, p.visible, p.embedding::text as embedding, p.profile_updated_at, p.profile_form_version, p.profile_revision::int as profile_revision`; + p.venue_zone, p.visible, p.embedding::text as embedding, p.profile_updated_at, p.profile_form_version, p.profile_revision::int as profile_revision, p.embedding_model`; const PROFILE_FROM = `from event_participants p join registered_attendees a on a.id = p.attendee_id`; /** @@ -95,6 +96,8 @@ export class PostgresParticipantRepository implements ParticipantRepository { embedding?: number[] | null; terms?: string[]; filters?: SearchFilters; + /** The model that produced `embedding`; only candidates embedded by it are ranked by vector. */ + embeddingModel?: string | null; limit?: number; }): Promise { const vector = toVector(input.embedding ?? null); @@ -106,13 +109,13 @@ export class PostgresParticipantRepository implements ParticipantRepository { and ($5::text is null or p.intent = $5) and ($6::text is null or lower(p.domain) = lower($6)) and ($7::text is null or ${SEARCH_DOCUMENT} @@ to_tsquery('simple', $7)) - order by case when $3::vector is not null and p.embedding is not null then 0 else 1 end, - p.embedding <=> $3::vector nulls last, + order by case when $3::vector is not null and p.embedding is not null and p.embedding_model is not distinct from $8 then 0 else 1 end, + case when p.embedding_model is not distinct from $8 then p.embedding <=> $3::vector end nulls last, case when $7::text is null then 0 else ts_rank(${SEARCH_DOCUMENT}, to_tsquery('simple', $7)) end desc, a.name asc limit $4`, [input.eventId, input.excludeAttendeeId, vector, Math.max(1, Math.min(100, input.limit ?? 50)), - input.filters?.intent ?? null, input.filters?.domain ?? null, tsQuery] + input.filters?.intent ?? null, input.filters?.domain ?? null, tsQuery, input.embeddingModel ?? null] ); return rows.map(mapProfile); } @@ -144,21 +147,20 @@ export class PostgresParticipantRepository implements ParticipantRepository { return rows.map(mapProfile); } - async setEmbeddingAt(eventId: string, attendeeId: string, embedding: number[], revision: number): Promise { + async setEmbeddingAt(eventId: string, attendeeId: string, embedding: number[], revision: number, model: string): Promise { const { rowCount } = await query( - `update event_participants set embedding = $3::vector, updated_at = now() + `update event_participants set embedding = $3::vector, embedding_model = $5, updated_at = now() where event_id = $1 and attendee_id = $2 and profile_revision = $4`, - [eventId, attendeeId, toVector(embedding), revision] + [eventId, attendeeId, toVector(embedding), revision, model] ); return (rowCount ?? 0) > 0; } - async setEmbedding(eventId: string, attendeeId: string, embedding: number[]): Promise { - const vector = toVector(embedding); + async setEmbedding(eventId: string, attendeeId: string, embedding: number[], model: string): Promise { await query( - `update event_participants set embedding = $3::vector, updated_at = now() + `update event_participants set embedding = $3::vector, embedding_model = $4, updated_at = now() where event_id = $1 and attendee_id = $2`, - [eventId, attendeeId, vector] + [eventId, attendeeId, toVector(embedding), model] ); } @@ -205,6 +207,7 @@ function mapProfile(row: ProfileRow): ParticipantProfile { profileComplete: Boolean(row.profile_updated_at), formVersion: row.profile_form_version ?? 0, revision: row.profile_revision ?? 0, + embeddingModel: row.embedding_model ?? null, profileRevision: row.profile_updated_at ? new Date(row.profile_updated_at).toISOString() : null, embedding: parseEmbedding(row.embedding), }; diff --git a/packages/db/workers/embed.worker.ts b/packages/db/workers/embed.worker.ts index 00553c1..4cd250e 100644 --- a/packages/db/workers/embed.worker.ts +++ b/packages/db/workers/embed.worker.ts @@ -21,7 +21,7 @@ export async function processEmbedJob(): Promise { return true; } const embedding = await container.embeddings.embed(profileText(profile), "profile"); - if (embedding && !(await container.participants.setEmbeddingAt(job.event_id, job.participant_id, embedding, job.revision))) { + if (embedding && !(await container.participants.setEmbeddingAt(job.event_id, job.participant_id, embedding, job.revision, container.embeddings.id))) { await matchingJobs.complete(job.id); return true; } diff --git a/tests/integration/search-retrieval.test.ts b/tests/integration/search-retrieval.test.ts index b404d42..66695d8 100644 --- a/tests/integration/search-retrieval.test.ts +++ b/tests/integration/search-retrieval.test.ts @@ -68,6 +68,18 @@ describeDb("search retrieval", () => { expect((await names({ eventId, excludeAttendeeId: null, filters: { intent: null, domain: "AI" } })).sort()).toEqual(["Ben Iyer", "Cy Das"]); }); + it("ranks by vector only among candidates embedded by the query's model", async () => { + const unit = (index: number) => Array.from({ length: 768 }, (_, i) => (i === index ? 1 : 0)); + // Ana's vector matches the query exactly but comes from another model; Ben's is close and from the same one. + await query(`update event_participants set embedding = $3::vector, embedding_model = 'other-model' where event_id = $1 and attendee_id = $2`, + [eventId, ids.designer, `[${unit(0).join(",")}]`]); + await query(`update event_participants set embedding = $3::vector, embedding_model = 'query-model' where event_id = $1 and attendee_id = $2`, + [eventId, ids.engineer, `[${unit(0).map((v, i) => (i === 1 ? 0.5 : v)).join(",")}]`]); + const ranked = await names({ eventId, excludeAttendeeId: null, embedding: unit(0), embeddingModel: "query-model" }); + expect(ranked[0]).toBe("Ben Iyer"); + expect(ranked).toContain("Ana Rao"); + }); + it("lists each domain present once", async () => { expect(await repo.listDomains(eventId)).toEqual(["AI", "Design"]); }); diff --git a/tests/unit/embedding-versions.test.ts b/tests/unit/embedding-versions.test.ts new file mode 100644 index 0000000..e57200b --- /dev/null +++ b/tests/unit/embedding-versions.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it, vi } from "vitest"; +import { SemanticScorer } from "@/src/domain/matching/scorers/semantic.scorer"; +import { queryRelevance } from "@/src/domain/matching/search-query"; +import { MatchingPipeline } from "@/src/domain/matching/pipeline"; +import type { ParticipantRepository } from "@/src/application/ports/participant-repository"; +import { Intent } from "@/src/domain/participant/intent"; +import type { ParticipantProfile } from "@/src/domain/participant/participant.entity"; + +const person = (attendeeId: string, overrides: Partial = {}): ParticipantProfile => ({ + eventId: "event", attendeeId, name: attendeeId, designation: null, domain: "AI", aboutMe: "builds search tools", selfDescription: null, + yearsOfExperience: null, mostImpressiveBuild: null, intent: Intent.NETWORKING, interests: [], venueZone: null, visible: true, + profileComplete: true, revision: 1, ...overrides, +}); + +describe("embeddings from different models", () => { + it("are never compared by the semantic scorer", () => { + const scorer = new SemanticScorer(); + const a = person("a", { embedding: [1, 0], embeddingModel: "model-a" }); + expect(scorer.score(a, person("b", { embedding: [1, 0], embeddingModel: "model-a" }))).toBeCloseTo(1); + // Same numbers, other model: falls back to the text overlap instead of a meaningless cosine. + expect(scorer.score(a, person("c", { embedding: [1, 0], embeddingModel: "model-b", aboutMe: "unrelated words" }))).toBeLessThan(0.5); + }); + + it("make query relevance fall back to words", () => { + const candidate = person("c", { embedding: [1, 0], embeddingModel: "old-model" }); + expect(queryRelevance(["nothing"], [1, 0], candidate, "new-model")).toBe(0); + expect(queryRelevance(["search"], [1, 0], candidate, "new-model")).toBe(1); + expect(queryRelevance(["nothing"], [1, 0], { ...candidate, embeddingModel: "new-model" }, "new-model")).toBeCloseTo(1); + }); + + it("are replaced: a profile embedded by another model is embedded again", async () => { + const viewer = person("viewer", { embedding: [0, 1], embeddingModel: "old-model" }); + const participants = { + findProfile: vi.fn(async () => viewer), + listCandidates: vi.fn(async () => []), + setEmbedding: vi.fn(), + } as unknown as ParticipantRepository; + const embeddings = { id: "new-model", embed: vi.fn(async () => [1, 0]) }; + const matches = { save: vi.fn(async () => true), listForParticipant: vi.fn() }; + const cache = { get: vi.fn(), set: vi.fn(), getVector: vi.fn(), setVector: vi.fn(), acquire: vi.fn(), release: vi.fn() }; + await new MatchingPipeline(participants, matches, embeddings, null, cache).run({ eventId: "event", participantId: "viewer", mode: "profile" }); + expect(embeddings.embed).toHaveBeenCalledWith(expect.any(String), "profile"); + expect(participants.setEmbedding).toHaveBeenCalledWith("event", "viewer", [1, 0], "new-model"); + expect(participants.listCandidates).toHaveBeenCalledWith(expect.objectContaining({ embedding: [1, 0], embeddingModel: "new-model" })); + }); +}); From 74c31db1c9d51a26f3feaf981235bda0f5960d51 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 09:22:08 +0530 Subject: [PATCH 07/26] Deliver intro notifications through a durable outbox (R23 part 1) Notification e-mails were sent inline after the action and a failure was only logged, so a mail outage lost the message. - Migration 0014: notification_outbox (unique dedupe key, attempts, lease, last error) and location_requests (used by part 2); the sweep removes sent notifications after 30 days and location requests after a day - An intro request and its notification, and an acceptance and its notice to the requester, commit in one transaction - The worker delivers queued notifications with a 2-minute lease and backoff retries (30s ... 15 min, 5 attempts); details are read at send time and an interaction that no longer applies is closed without sending; replays never send twice - E-mails carry a deep link to the connection on the Connections page; the notifier gains location-request and reply messages - Real-Postgres tests: queued once with the request, delivered with the link, acceptance notice, and a failed send kept for retry --- .../web/src/application/ports/notification.ts | 16 +++- .../application/use-cases/request-intro.ts | 13 +--- .../application/use-cases/respond-to-intro.ts | 16 +--- .../notifications/realtime.notifier.ts | 32 +++++--- .../src/infrastructure/postgres/intro.repo.ts | 46 ++++++----- .../0014_outbox_location_requests.sql | 48 ++++++++++++ packages/db/src/index.ts | 1 + packages/db/src/repositories/events.ts | 6 ++ packages/db/src/repositories/housekeeping.ts | 4 + packages/db/src/repositories/outbox.ts | 76 +++++++++++++++++++ packages/db/workers/notification.worker.ts | 55 ++++++++++++++ packages/db/workers/worker.ts | 4 +- tests/integration/notification-outbox.test.ts | 74 ++++++++++++++++++ 13 files changed, 333 insertions(+), 58 deletions(-) create mode 100644 packages/db/migrations/0014_outbox_location_requests.sql create mode 100644 packages/db/src/repositories/outbox.ts create mode 100644 packages/db/workers/notification.worker.ts create mode 100644 tests/integration/notification-outbox.test.ts diff --git a/apps/web/src/application/ports/notification.ts b/apps/web/src/application/ports/notification.ts index 7db58ee..8bdd3bf 100644 --- a/apps/web/src/application/ports/notification.ts +++ b/apps/web/src/application/ports/notification.ts @@ -1,5 +1,17 @@ +/** One delivered message: who it goes to and the page it links to. */ +export interface NotificationTarget { + recipientEmail: string; + recipientName: string; + otherName: string; + eventName: string; + /** Deep link to the interaction, e.g. the connection on the Connections page. */ + link: string; +} + export interface NotificationPort { invitation(input: { email: string; name: string | null; eventName: string; organizationName: string; eventUrl: string }): Promise; - introRequested(input: { recipientEmail: string; recipientName: string; requesterName: string; eventName: string; introId: string }): Promise; - introResponded(input: { recipientEmail: string; recipientName: string; requesterName: string; eventName: string; accepted: boolean }): Promise; + introRequested(input: NotificationTarget): Promise; + introAccepted(input: NotificationTarget): Promise; + locationRequested(input: NotificationTarget): Promise; + locationAnswered(input: NotificationTarget & { zone: string | null }): Promise; } diff --git a/apps/web/src/application/use-cases/request-intro.ts b/apps/web/src/application/use-cases/request-intro.ts index ed31eee..7be99a7 100644 --- a/apps/web/src/application/use-cases/request-intro.ts +++ b/apps/web/src/application/use-cases/request-intro.ts @@ -1,13 +1,12 @@ import type { IntroRepository } from "../ports/intro-repository"; import type { ParticipantRepository } from "../ports/participant-repository"; -import type { NotificationPort } from "../ports/notification"; import { boundedText, LIMITS } from "../../domain/shared/validation"; import { INTRO_LIMITS, introRequestRefusal } from "../../domain/connection/connection.rules"; import type { RateLimiter } from "../ports/rate-limiter"; export async function requestIntro( input: { eventId: string; requesterId: string; recipientId: string; message?: string | null; eventName?: string }, - dependencies: { participants: ParticipantRepository; intros: IntroRepository; notifications?: NotificationPort; limiter?: RateLimiter } + dependencies: { participants: ParticipantRepository; intros: IntroRepository; limiter?: RateLimiter } ) { if (input.requesterId === input.recipientId) throw new Error("You cannot request an intro to yourself"); if (!(await dependencies.participants.hasEventAccess(input.eventId, input.requesterId))) throw new Error("You are not registered for this event"); @@ -26,15 +25,7 @@ export async function requestIntro( if (dependencies.limiter && !(await dependencies.limiter.consume(`intro:${input.eventId}:${input.requesterId}`, INTRO_LIMITS.perHour, 3600))) { throw new Error("You have sent many introduction requests this hour. Please try again later."); } + // The notification is queued with the request (outbox) and delivered by the worker. const intro = await dependencies.intros.create({ ...input, message }); - if (recipient.email && dependencies.notifications) { - await dependencies.notifications.introRequested({ - recipientEmail: recipient.email, - recipientName: recipient.name, - requesterName: requester.name, - eventName: input.eventName ?? "your event", - introId: intro.id, - }).catch((error) => console.warn("[intro] notification failed", error)); - } return intro; } diff --git a/apps/web/src/application/use-cases/respond-to-intro.ts b/apps/web/src/application/use-cases/respond-to-intro.ts index fe9492d..5f67aea 100644 --- a/apps/web/src/application/use-cases/respond-to-intro.ts +++ b/apps/web/src/application/use-cases/respond-to-intro.ts @@ -1,11 +1,10 @@ import type { IntroRepository } from "../ports/intro-repository"; -import type { NotificationPort } from "../ports/notification"; import type { ParticipantRepository } from "../ports/participant-repository"; import type { AuditLog } from "../ports/audit-log"; export async function respondToIntro( input: { eventId: string; introId: string; actorId: string; accept: boolean; eventName?: string }, - dependencies: { intros: IntroRepository; participants: ParticipantRepository; notifications?: NotificationPort; audit?: AuditLog } + dependencies: { intros: IntroRepository; participants: ParticipantRepository; audit?: AuditLog } ) { const existing = await dependencies.intros.findById(input.eventId, input.introId); if (!existing) throw new Error("Intro request not found"); @@ -16,18 +15,5 @@ export async function respondToIntro( if (updated.status === "accepted") { await dependencies.audit?.record({ eventId: input.eventId, actorId: input.actorId, action: "contacts.revealed", subjectId: updated.requesterId }); } - if (updated.status === "accepted" && dependencies.notifications) { - const requester = await dependencies.participants.findProfile(input.eventId, updated.requesterId); - const recipient = await dependencies.participants.findProfile(input.eventId, updated.recipientId); - if (requester?.email) { - await dependencies.notifications.introResponded({ - recipientEmail: requester.email, - recipientName: requester.name, - requesterName: recipient?.name ?? "Your connection", - eventName: input.eventName ?? "your event", - accepted: true, - }).catch((error) => console.warn("[intro] notification failed", error)); - } - } return updated; } diff --git a/apps/web/src/infrastructure/notifications/realtime.notifier.ts b/apps/web/src/infrastructure/notifications/realtime.notifier.ts index b5ca8fa..4b61de9 100644 --- a/apps/web/src/infrastructure/notifications/realtime.notifier.ts +++ b/apps/web/src/infrastructure/notifications/realtime.notifier.ts @@ -1,27 +1,35 @@ import { renderInvitationEmail, sendNotificationEmail } from "@match-maker/email"; -import type { NotificationPort } from "../../application/ports/notification"; +import type { NotificationPort, NotificationTarget } from "../../application/ports/notification"; export class RealtimeNotifier implements NotificationPort { async invitation(input: { email: string; name: string | null; eventName: string; organizationName: string; eventUrl: string }): Promise { await sendNotificationEmail({ to: input.email, ...renderInvitationEmail(input) }); } - async introRequested(input: { recipientEmail: string; recipientName: string; requesterName: string; eventName: string; introId: string }): Promise { - const subject = `${input.requesterName} would like to meet at ${input.eventName}`; - const text = `Hi ${input.recipientName},\n\n${input.requesterName} requested an introduction at ${input.eventName}. Sign in to Mr. Bridge to respond.`; - await sendNotificationEmail({ to: input.recipientEmail, subject, text, html: emailHtml(input.recipientName, input.requesterName, input.eventName, "requested an introduction. Sign in to Mr. Bridge to respond.") }); + introRequested(input: NotificationTarget): Promise { + return send(input, `${input.otherName} would like to meet at ${input.eventName}`, `${input.otherName} requested an introduction. Accept to share contact details.`, "Respond"); } - async introResponded(input: { recipientEmail: string; recipientName: string; requesterName: string; eventName: string; accepted: boolean }): Promise { - const status = input.accepted ? "accepted" : "declined"; - const subject = `Your introduction request was ${status}`; - const text = `Hi ${input.recipientName},\n\n${input.requesterName} ${status} your introduction request for ${input.eventName}.`; - await sendNotificationEmail({ to: input.recipientEmail, subject, text, html: emailHtml(input.recipientName, input.requesterName, input.eventName, `${status} your introduction request.`) }); + introAccepted(input: NotificationTarget): Promise { + return send(input, `${input.otherName} accepted your introduction`, `${input.otherName} accepted your introduction request. Their contact details are now on your Connections page.`, "See contact"); + } + + locationRequested(input: NotificationTarget): Promise { + return send(input, `${input.otherName} is asking where you are`, `${input.otherName} would like to find you at ${input.eventName}. Sharing your venue zone is up to you.`, "Reply"); + } + + locationAnswered(input: NotificationTarget & { zone: string | null }): Promise { + const text = input.zone ? `${input.otherName} is at ${input.zone}.` : `${input.otherName} would rather not share their location right now.`; + return send(input, input.zone ? `${input.otherName} is at ${input.zone}` : `${input.otherName} replied to your location request`, text, "Open connections"); } } -function emailHtml(name: string, otherPerson: string, event: string, action: string): string { - return `

Mr. Bridge

Hi ${escapeHtml(name)},

${escapeHtml(otherPerson)} ${escapeHtml(action)}

Event: ${escapeHtml(event)}

Open Mr. Bridge to view your connections.

`; +async function send(input: NotificationTarget, subject: string, line: string, action: string): Promise { + const text = `Hi ${input.recipientName},\n\n${line}\n\n${action}: ${input.link}`; + const html = `

Mr. Bridge

` + + `

Hi ${escapeHtml(input.recipientName)},

${escapeHtml(line)}

Event: ${escapeHtml(input.eventName)}

` + + `

${escapeHtml(action)}

`; + await sendNotificationEmail({ to: input.recipientEmail, subject, text, html }); } function escapeHtml(value: string): string { diff --git a/apps/web/src/infrastructure/postgres/intro.repo.ts b/apps/web/src/infrastructure/postgres/intro.repo.ts index 88595fe..652c159 100644 --- a/apps/web/src/infrastructure/postgres/intro.repo.ts +++ b/apps/web/src/infrastructure/postgres/intro.repo.ts @@ -1,4 +1,4 @@ -import { query } from "@match-maker/db"; +import { outbox, query, transaction } from "@match-maker/db"; import type { IntroRepository } from "../../application/ports/intro-repository"; import type { IntroRequest, IntroStatus } from "../../domain/connection/intro-request.entity"; @@ -20,14 +20,20 @@ export class PostgresIntroRepository implements IntroRepository { async create(input: { eventId: string; requesterId: string; recipientId: string; message?: string | null }): Promise { if (input.requesterId === input.recipientId) throw new Error("You cannot request an intro to yourself"); const message = input.message?.trim().slice(0, 500) || null; - const { rows } = await query( - `insert into intro_requests (event_id, requester_id, recipient_id, message) - values ($1, $2, $3, $4) - on conflict (event_id, requester_id, recipient_id) do nothing - returning ${INTRO_COLUMNS}`, - [input.eventId, input.requesterId, input.recipientId, message] - ); - if (rows[0]) return mapIntro(rows[0]); + // The request and its notification commit together, so neither is lost. + const created = await transaction(async (client) => { + const { rows } = await client.query( + `insert into intro_requests (event_id, requester_id, recipient_id, message) + values ($1, $2, $3, $4) + on conflict (event_id, requester_id, recipient_id) do nothing + returning ${INTRO_COLUMNS}`, + [input.eventId, input.requesterId, input.recipientId, message] + ); + if (!rows[0]) return null; + await outbox.enqueue({ eventId: input.eventId, recipientId: input.recipientId, kind: "intro.requested", subjectId: rows[0].id, dedupeKey: `intro.requested:${rows[0].id}` }, client); + return rows[0]; + }); + if (created) return mapIntro(created); throw new Error("An intro request already exists between these participants"); } @@ -40,14 +46,20 @@ export class PostgresIntroRepository implements IntroRepository { } async respond(eventId: string, introId: string, actorId: string, status: Extract): Promise { - const { rows } = await query( - `update intro_requests set status = $4, responded_at = now() - where event_id = $1 and id = $2 and recipient_id = $3 and status = 'pending' - returning ${INTRO_COLUMNS}`, - [eventId, introId, actorId, status] - ); - if (!rows[0]) throw new Error("Only the recipient can respond to a pending intro request"); - return mapIntro(rows[0]); + const updated = await transaction(async (client) => { + const { rows } = await client.query( + `update intro_requests set status = $4, responded_at = now() + where event_id = $1 and id = $2 and recipient_id = $3 and status = 'pending' + returning ${INTRO_COLUMNS}`, + [eventId, introId, actorId, status] + ); + if (rows[0]?.status === "accepted") { + await outbox.enqueue({ eventId, recipientId: rows[0].requester_id, kind: "intro.accepted", subjectId: rows[0].id, dedupeKey: `intro.accepted:${rows[0].id}` }, client); + } + return rows[0] ?? null; + }); + if (!updated) throw new Error("Only the recipient can respond to a pending intro request"); + return mapIntro(updated); } async findBetween(eventId: string, a: string, b: string): Promise { diff --git a/packages/db/migrations/0014_outbox_location_requests.sql b/packages/db/migrations/0014_outbox_location_requests.sql new file mode 100644 index 0000000..4cbc7e0 --- /dev/null +++ b/packages/db/migrations/0014_outbox_location_requests.sql @@ -0,0 +1,48 @@ +-- Reliable notifications and venue location requests. +-- +-- notification_outbox: written in the same transaction as the interaction it +-- announces (an intro request, an acceptance, a location request or reply), +-- then delivered by the worker with retries. dedupe_key makes each +-- notification unique, so retries and replays never send duplicates. +-- Sent rows are swept after 30 days. +-- +-- location_requests: "where are you?" between two accepted connections. The +-- recipient may reply with a configured venue zone or decline; the reply is +-- visible only to the person who asked, for 15 minutes. Pending requests +-- expire after 30 minutes. Rows are swept after a day. + +create table if not exists notification_outbox ( + id bigserial primary key, + event_id uuid references events(id) on delete cascade, + recipient_id uuid not null, + kind text not null, + subject_id uuid not null, + dedupe_key text not null unique, + status text not null default 'pending' check (status in ('pending', 'running', 'sent', 'failed')), + attempts integer not null default 0, + available_at timestamptz not null default now(), + lease_until timestamptz, + last_error text, + created_at timestamptz not null default now(), + sent_at timestamptz +); +create index if not exists idx_notification_outbox_due on notification_outbox (available_at) where status in ('pending', 'running'); + +create table if not exists location_requests ( + id uuid primary key default gen_random_uuid(), + event_id uuid not null references events(id) on delete cascade, + intro_id uuid not null references intro_requests(id) on delete cascade, + requester_id uuid not null, + recipient_id uuid not null, + status text not null default 'pending' check (status in ('pending', 'answered', 'declined')), + zone text, + created_at timestamptz not null default now(), + expires_at timestamptz not null, + answered_at timestamptz, + zone_expires_at timestamptz, + check (requester_id <> recipient_id) +); +-- One open request per direction of a connection. +create unique index if not exists idx_location_requests_one_pending + on location_requests (intro_id, requester_id) where status = 'pending'; +create index if not exists idx_location_requests_people on location_requests (event_id, recipient_id, created_at desc); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 94bc070..a13636c 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -14,3 +14,4 @@ export * as matchingJobs from "./repositories/matching-jobs"; export * as housekeeping from "./repositories/housekeeping"; export * as audit from "./repositories/audit"; export * as modelUsage from "./repositories/model-usage"; +export * as outbox from "./repositories/outbox"; diff --git a/packages/db/src/repositories/events.ts b/packages/db/src/repositories/events.ts index 979dc01..f848286 100644 --- a/packages/db/src/repositories/events.ts +++ b/packages/db/src/repositories/events.ts @@ -126,6 +126,12 @@ export async function update(input: { return findBySlug(old.organization_slug, old.slug); } +/** Name and public slug, for notification text and links. */ +export async function basics(eventId: string): Promise<{ name: string; slug: string } | null> { + const { rows } = await query<{ name: string; slug: string }>(`select name, slug from events where id = $1`, [eventId]); + return rows[0] ?? null; +} + /** The event's status, or null when the event no longer exists. */ export async function statusOf(eventId: string): Promise { const { rows } = await query<{ status: EventRecord["status"] }>(`select status from events where id = $1`, [eventId]); diff --git a/packages/db/src/repositories/housekeeping.ts b/packages/db/src/repositories/housekeeping.ts index c6798bb..f918561 100644 --- a/packages/db/src/repositories/housekeeping.ts +++ b/packages/db/src/repositories/housekeeping.ts @@ -2,6 +2,7 @@ import { query } from "../client"; import { deleteExpired as deleteExpiredChallenges } from "./login-challenges"; import { AUDIT_RETENTION_DAYS } from "./audit"; import { USAGE_RETENTION_DAYS } from "./model-usage"; +import { SENT_RETENTION_DAYS } from "./outbox"; /** How long a dead row is kept before the sweep removes it. */ export const RETENTION = { @@ -30,6 +31,9 @@ export async function sweepExpired(): Promise { const explanations = await query(`delete from pair_explanations where expires_at < now()`); await query(`delete from cache_locks where expires_at < now()`); await query(`delete from model_usage where created_at < now() - make_interval(days => $1)`, [USAGE_RETENTION_DAYS]); + await query(`delete from notification_outbox where status = 'sent' and sent_at < now() - make_interval(days => $1)`, [SENT_RETENTION_DAYS]); + // Venue locations are only meaningful during the event day. + await query(`delete from location_requests where created_at < now() - interval '1 day'`); const audits = await query(`delete from audit_events where created_at < now() - make_interval(days => $1)`, [AUDIT_RETENTION_DAYS]); return { loginChallenges, rateLimitBuckets: buckets.rowCount ?? 0, retrievalCache: cache.rowCount ?? 0, diff --git a/packages/db/src/repositories/outbox.ts b/packages/db/src/repositories/outbox.ts new file mode 100644 index 0000000..ec6fa35 --- /dev/null +++ b/packages/db/src/repositories/outbox.ts @@ -0,0 +1,76 @@ +import { query, transaction } from "../client"; + +export type NotificationKind = "intro.requested" | "intro.accepted" | "location.requested" | "location.answered"; + +export interface OutboxItem { + id: number; + event_id: string | null; + recipient_id: string; + kind: NotificationKind; + subject_id: string; + attempts: number; +} + +export const MAX_ATTEMPTS = 5; +const LEASE_SECONDS = 120; +/** Sent notifications are kept this long, then swept. */ +export const SENT_RETENTION_DAYS = 30; + +/** + * Queues a notification. Pass the transaction's client so it commits or + * rolls back with the interaction it announces. A repeated dedupe key is + * ignored, so the same notification is never queued twice. + */ +export async function enqueue( + item: { eventId: string; recipientId: string; kind: NotificationKind; subjectId: string; dedupeKey: string }, + client: { query: typeof query } = { query } +): Promise { + await client.query( + `insert into notification_outbox (event_id, recipient_id, kind, subject_id, dedupe_key) + values ($1, $2, $3, $4, $5) on conflict (dedupe_key) do nothing`, + [item.eventId, item.recipientId, item.kind, item.subjectId, item.dedupeKey] + ); +} + +/** Claims the oldest due notification (or one whose sender crashed) for delivery. */ +export async function claim(onlyEventId: string | null = null): Promise { + return transaction(async (client) => { + await client.query( + `update notification_outbox set status = 'failed', last_error = coalesce(last_error, 'sender stopped responding') + where status = 'running' and lease_until < now() and attempts >= $1`, [MAX_ATTEMPTS]); + const { rows } = await client.query( + `with next_item as ( + select id from notification_outbox + where attempts < $1 and ($3::uuid is null or event_id = $3) + and ((status = 'pending' and available_at <= now()) or (status = 'running' and lease_until < now())) + order by available_at, id for update skip locked limit 1 + ) + update notification_outbox o set status = 'running', attempts = attempts + 1, lease_until = now() + make_interval(secs => $2) + from next_item where o.id = next_item.id + returning o.id::int as id, o.event_id, o.recipient_id, o.kind, o.subject_id, o.attempts`, + [MAX_ATTEMPTS, LEASE_SECONDS, onlyEventId] + ); + return rows[0] ?? null; + }); +} + +export async function markSent(id: number): Promise { + await query(`update notification_outbox set status = 'sent', sent_at = now(), lease_until = null where id = $1`, [id]); +} + +/** Retries after 30s, 60s, 120s... (capped at 15 minutes); fails for good after the last attempt. */ +export async function fail(id: number, error: string): Promise { + await query( + `update notification_outbox + set status = case when attempts >= $3 then 'failed' else 'pending' end, + available_at = now() + make_interval(secs => least(30 * power(2, greatest(attempts - 1, 0)), 900)), + lease_until = null, last_error = left($2, 1000) + where id = $1`, + [id, error, MAX_ATTEMPTS] + ); +} + +/** Nothing left to send (e.g. the interaction was deleted): close it without delivery. */ +export async function skip(id: number, reason: string): Promise { + await query(`update notification_outbox set status = 'sent', sent_at = now(), lease_until = null, last_error = $2 where id = $1`, [id, reason]); +} diff --git a/packages/db/workers/notification.worker.ts b/packages/db/workers/notification.worker.ts new file mode 100644 index 0000000..061844b --- /dev/null +++ b/packages/db/workers/notification.worker.ts @@ -0,0 +1,55 @@ +import { events, outbox, query } from "../src/index"; +import { getContainer } from "../../../apps/web/src/composition/container"; + +/** + * Delivers one queued notification. Details are read at send time, so the + * message reflects the current state; an interaction that no longer exists + * is closed without sending. Failures retry with backoff (see outbox.fail). + */ +export async function processNotification(onlyEventId: string | null = null): Promise { + const item = await outbox.claim(onlyEventId); + if (!item) return false; + try { + const delivered = await deliver(item); + if (delivered) await outbox.markSent(item.id); + else await outbox.skip(item.id, "nothing to send any more"); + } catch (error) { + console.error(`[worker:notify] ${item.kind} #${item.id} failed`, error); + await outbox.fail(item.id, error instanceof Error ? error.message : String(error)); + } + return true; +} + +async function deliver(item: outbox.OutboxItem): Promise { + if (!item.event_id) return false; + const { participants, notifications } = getContainer(); + const event = await events.basics(item.event_id); + const recipient = await participants.findProfile(item.event_id, item.recipient_id); + if (!event || !recipient?.email) return false; + + const base = (process.env.APP_BASE_URL || "").replace(/\/$/, ""); + if (item.kind === "intro.requested" || item.kind === "intro.accepted") { + const { rows: [intro] } = await query<{ requester_id: string; recipient_id: string; status: string }>( + `select requester_id, recipient_id, status from intro_requests where id = $1`, [item.subject_id]); + if (!intro || (item.kind === "intro.accepted" && intro.status !== "accepted")) return false; + const otherId = intro.requester_id === item.recipient_id ? intro.recipient_id : intro.requester_id; + const other = await participants.findProfile(item.event_id, otherId); + const target = { recipientEmail: recipient.email, recipientName: recipient.name, otherName: other?.name ?? "A participant", eventName: event.name, link: `${base}/e/${event.slug}/connections#intro-${item.subject_id}` }; + await (item.kind === "intro.requested" ? notifications.introRequested(target) : notifications.introAccepted(target)); + return true; + } + + const { rows: [request] } = await query<{ intro_id: string; requester_id: string; recipient_id: string; status: string; zone: string | null }>( + `select intro_id, requester_id, recipient_id, status, zone from location_requests where id = $1`, [item.subject_id]); + if (!request) return false; + const otherId = request.requester_id === item.recipient_id ? request.recipient_id : request.requester_id; + const other = await participants.findProfile(item.event_id, otherId); + const target = { recipientEmail: recipient.email, recipientName: recipient.name, otherName: other?.name ?? "A participant", eventName: event.name, link: `${base}/e/${event.slug}/connections#intro-${request.intro_id}` }; + if (item.kind === "location.requested") { + if (request.status !== "pending") return false; + await notifications.locationRequested(target); + } else { + await notifications.locationAnswered({ ...target, zone: request.status === "answered" ? request.zone : null }); + } + return true; +} diff --git a/packages/db/workers/worker.ts b/packages/db/workers/worker.ts index 5650e2d..80af71c 100644 --- a/packages/db/workers/worker.ts +++ b/packages/db/workers/worker.ts @@ -4,6 +4,7 @@ import { closePool } from "../src/client"; import { processEmbedJob } from "./embed.worker"; import { processMatchJob } from "./match.worker"; import { sweepIfDue } from "./housekeeping.worker"; +import { processNotification } from "./notification.worker"; let stopping = false; process.on("SIGINT", () => { stopping = true; }); @@ -16,7 +17,8 @@ async function main() { await sweepIfDue(); const embedded = await processEmbedJob(); const matched = await processMatchJob(); - if (!embedded && !matched) await new Promise((resolve) => setTimeout(resolve, 1200)); + const notified = await processNotification(); + if (!embedded && !matched && !notified) await new Promise((resolve) => setTimeout(resolve, 1200)); } await closePool(); console.info("[worker] stopped"); diff --git a/tests/integration/notification-outbox.test.ts b/tests/integration/notification-outbox.test.ts new file mode 100644 index 0000000..d917c70 --- /dev/null +++ b/tests/integration/notification-outbox.test.ts @@ -0,0 +1,74 @@ +/** + * Notifications go through an outbox written with the interaction, and are + * delivered with retries. Real Postgres; delivery uses the e-mail dev + * fallback. Claims are scoped to this test's event. Skipped without DATABASE_URL. + */ +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { attendees, closePool, events, outbox, query } from "@match-maker/db"; +import { migrate } from "../../packages/db/src/migrate"; +import { PostgresIntroRepository } from "../../apps/web/src/infrastructure/postgres/intro.repo"; +import { processNotification } from "../../packages/db/workers/notification.worker"; +import { getContainer } from "../../apps/web/src/composition/container"; + +const describeDb = process.env.DATABASE_URL ? describe : describe.skip; +const RUN = Date.now().toString(36); +const ORG = `outbox-${RUN}`; +const email = (key: string) => `outbox-${key}-${RUN}@example.com`; + +describeDb("notification outbox", () => { + const intros = new PostgresIntroRepository(); + const ids: Record = {}; + let eventId: string; + let introId: string; + const queued = async () => (await query<{ kind: string; recipient_id: string; status: string; attempts: number; last_error: string | null }>( + `select kind, recipient_id, status, attempts, last_error from notification_outbox where event_id = $1 order by id`, [eventId])).rows; + + beforeAll(async () => { + process.env.EMAIL_DEV_FALLBACK = "true"; + await migrate(); + await attendees.upsertMany(["amy", "ben"].map((key) => ({ name: `Outbox ${key}`, email: email(key) }))); + for (const key of ["amy", "ben"]) ids[key] = (await attendees.findByEmail(email(key)))!.id; + const { rows: [org] } = await query<{ id: string }>(`insert into organizations (name, slug) values ($1, $1) returning id`, [ORG]); + eventId = (await query<{ id: string }>(`insert into events (organization_id, name, slug, status) values ($1, $2, $2, 'open') returning id`, [org.id, ORG])).rows[0].id; + await events.attachRoster(eventId, Object.values(ids)); + }); + + afterAll(async () => { + await query(`delete from organizations where slug = $1`, [ORG]); + await query(`delete from registered_attendees where email like $1`, [`outbox-%-${RUN}@example.com`]); + await closePool(); + }); + + it("queues the notification with the request, once", async () => { + introId = (await intros.create({ eventId, requesterId: ids.amy, recipientId: ids.ben })).id; + await outbox.enqueue({ eventId, recipientId: ids.ben, kind: "intro.requested", subjectId: introId, dedupeKey: `intro.requested:${introId}` }); + expect(await queued()).toEqual([expect.objectContaining({ kind: "intro.requested", recipient_id: ids.ben, status: "pending" })]); + }); + + it("delivers it with a link to the connection", async () => { + const info = vi.spyOn(console, "info").mockImplementation(() => {}); + expect(await processNotification(eventId)).toBe(true); + expect((await queued())[0].status).toBe("sent"); + const message = info.mock.calls.map((call) => String(call[0])).find((line) => line.includes(email("ben"))); + expect(message).toContain(`/e/${ORG}/connections#intro-${introId}`); + info.mockRestore(); + }); + + it("queues an acceptance notice for the requester", async () => { + await intros.respond(eventId, introId, ids.ben, "accepted"); + expect((await queued()).at(-1)).toMatchObject({ kind: "intro.accepted", recipient_id: ids.amy, status: "pending" }); + }); + + it("keeps a failed delivery for retry instead of losing it", async () => { + const container = getContainer(); + const failing = vi.spyOn(container.notifications, "introAccepted").mockRejectedValueOnce(new Error("Resend failed (503)")); + vi.spyOn(console, "error").mockImplementation(() => {}); + await processNotification(eventId); + expect((await queued()).at(-1)).toMatchObject({ status: "pending", attempts: 1, last_error: "Resend failed (503)" }); + failing.mockRestore(); + await query(`update notification_outbox set available_at = now() where event_id = $1 and status = 'pending'`, [eventId]); + vi.spyOn(console, "info").mockImplementation(() => {}); + await processNotification(eventId); + expect((await queued()).at(-1)).toMatchObject({ status: "sent", attempts: 2 }); + }); +}); From 0a9deda89b0ecb801b8e5befdc93a6e66c34c641 Mon Sep 17 00:00:00 2001 From: SURYANSH-RAI Date: Mon, 19 Oct 2026 09:25:44 +0530 Subject: [PATCH 08/26] Add "where are you?" location requests with voluntary replies (R23 part 2) "Ping zone" only overwrote the sender's own zone for every connection; there was no way to ask a connection where they are, and no reply. - domain/connection/location.rules.ts: a request is a question, not tracking. One open request per direction of an accepted connection; unanswered requests expire after 30 minutes; a shared zone is visible only to the person who asked, for 15 minutes; 10 requests per person and hour - LocationRepository (Postgres): creating a request and answering it (a configured zone, or "not now") each queue their notification in the same transaction, so the interaction is never lost - Connections page: "Ask where they are", an incoming request with "Share with " / "Not now", the reply or decline for the asker, and the existing broadcast labelled "Share with all my connections"; each connection has an anchor for e-mail deep links - Tests: state and rules, the use cases, real-Postgres visibility, expiry and replacement, and a browser run where the worker e-mails the request, acceptance, location request and reply --- .../e/[eventSlug]/connections/page.tsx | 32 ++++++-- apps/web/app/actions/intro.actions.ts | 37 ++++++++- .../application/ports/location-repository.ts | 14 ++++ .../use-cases/location-requests.ts | 34 ++++++++ apps/web/src/composition/container.ts | 2 + .../src/domain/connection/location.rules.ts | 33 ++++++++ .../infrastructure/postgres/location.repo.ts | 77 +++++++++++++++++++ tests/integration/location-requests.test.ts | 74 ++++++++++++++++++ tests/unit/location-requests.test.ts | 57 ++++++++++++++ 9 files changed, 353 insertions(+), 7 deletions(-) create mode 100644 apps/web/src/application/ports/location-repository.ts create mode 100644 apps/web/src/application/use-cases/location-requests.ts create mode 100644 apps/web/src/domain/connection/location.rules.ts create mode 100644 apps/web/src/infrastructure/postgres/location.repo.ts create mode 100644 tests/integration/location-requests.test.ts create mode 100644 tests/unit/location-requests.test.ts diff --git a/apps/web/app/(participant)/e/[eventSlug]/connections/page.tsx b/apps/web/app/(participant)/e/[eventSlug]/connections/page.tsx index f95bc06..e961833 100644 --- a/apps/web/app/(participant)/e/[eventSlug]/connections/page.tsx +++ b/apps/web/app/(participant)/e/[eventSlug]/connections/page.tsx @@ -1,13 +1,13 @@ import Link from "next/link"; import SignOutButton from "@/components/ui/SignOutButton"; -import { getParticipantConnections, respondToParticipantIntro, shareVenueZoneAction } from "../../../../actions/intro.actions"; +import { answerLocationAction, getParticipantConnections, requestLocationAction, respondToParticipantIntro, shareVenueZoneAction } from "../../../../actions/intro.actions"; import { requireParticipantEvent } from "@/src/lib/auth"; import { ActionForm } from "@/components/ui/form"; import { eventAllows } from "@/src/domain/shared/event-lifecycle"; export default async function ConnectionsPage({ params, searchParams }: { params: Promise<{ eventSlug: string }>; - searchParams: Promise<{ requested?: string; location?: string }>; + searchParams: Promise<{ requested?: string; location?: string; asked?: string; replied?: string; declined?: string }>; }) { const { eventSlug } = await params; const notice = await searchParams; @@ -18,9 +18,12 @@ export default async function ConnectionsPage({ params, searchParams }: { INTRODUCTIONS · {event.name}

Connections

{notice.requested &&

Intro request sent. You will see their contact details once they accept.

} - {notice.location &&

Your venue zone is shared with your accepted connections.

} + {notice.location &&

Your venue zone is shared with all your accepted connections for 15 minutes.

} + {notice.asked &&

Request sent. They decide whether to share where they are.

} + {notice.replied &&

Your zone is shared with them for 15 minutes.

} + {notice.declined &&

Request declined. They have been told you would rather not share right now.

} {!connections.length ?

Your intro requests will show up here. Find someone you would like to meet from your matches or search.

:
- {connections.map((connection) =>
+ {connections.map((connection) =>

{connection.participant.name}

{[connection.participant.designation, connection.participant.domain].filter(Boolean).join(" · ")}

@@ -31,7 +34,25 @@ export default async function ConnectionsPage({ params, searchParams }: { {connection.contact.email && } {connection.contact.phone && } {connection.contact.venueZone &&
At the venue: {connection.contact.venueZone}
} - {event.zones.length > 0 && eventAllows(event.status, "shareZone") && + {connection.locationAsked?.status === "answered" &&
Replied: at {connection.locationAsked.zone}
} + {connection.locationAsked?.status === "pending" &&
Asked where they are. Waiting for a reply.
} + {connection.locationAsked?.status === "declined" &&
They would rather not share where they are right now.
} + {event.zones.length > 0 && eventAllows(event.status, "shareZone") && <> + {connection.locationIncoming &&
+ {connection.participant.name} is asking where you are. Sharing is up to you; only they will see it, for 15 minutes. +
+ + + + + +
+
} + {connection.locationAsked?.status !== "pending" && } + } + {event.zones.length > 0 && eventAllows(event.status, "shareZone") &&