diff --git a/app/(marketing)/trending/page.tsx b/app/(marketing)/trending/page.tsx index 14b7513..9fb8228 100644 --- a/app/(marketing)/trending/page.tsx +++ b/app/(marketing)/trending/page.tsx @@ -42,12 +42,16 @@ export default async function TrendingPage() {

Trending topics

- What English blogs and news sites are writing about more than they were last week, by how many - feeds used the subject in a headline. It comes from{" "} + What people are writing and searching for more than they were last week: subjects in the + headlines of a sample of{" "} RSSAmp - , a sample of its feeds, so read it as movement rather than a census. + 's English blogs and news sites, mixed with searches people voted on at{" "} + + pit.world + + . Read it as movement rather than a census.

{stale && newest && ( @@ -67,7 +71,7 @@ export default async function TrendingPage() { # Subject - Feeds, {TREND_WINDOW_DAYS}d + Mentions, {TREND_WINDOW_DAYS}d Week before Change diff --git a/app/api/cron/ad-trends/route.ts b/app/api/cron/ad-trends/route.ts index 3f0a64c..0c9e531 100644 --- a/app/api/cron/ad-trends/route.ts +++ b/app/api/cron/ad-trends/route.ts @@ -34,6 +34,7 @@ export async function POST(req: Request) { const windowDays = Number(new URL(req.url).searchParams.get("window")) || TREND_WINDOW_DAYS; const result = await ingestTrends(serviceClient(), { url: env.trendUrl, + votedUrl: env.votedSearchesUrl, windowDays, }); if (!result.ok) return NextResponse.json({ ok: false, error: result.error }, { status: result.status }); diff --git a/lib/ads/trends.ts b/lib/ads/trends.ts index 08c5c7d..f575cd5 100644 --- a/lib/ads/trends.ts +++ b/lib/ads/trends.ts @@ -80,6 +80,8 @@ export function parseTrendPayload( export type TrendSourceConfig = { /** Base URL of the trend source, e.g. https://rssamp.com */ url: string; + /** Searches people voted on, mixed into the list. Optional: see votedSignals. */ + votedUrl?: string; windowDays?: number; limit?: number; }; @@ -184,6 +186,47 @@ export async function storeTrends( return { stored: rows.length, removed }; } +/** A voted search outranks a headline: three votes lands it mid-list. */ +export const VOTE_SCORE = 50; + +/** The list pit.world publishes, as signals. Any failure is simply no extra signal. */ +async function votedSignals(url: string, fetchImpl: typeof fetch): Promise { + try { + const response = await fetchImpl(url, { headers: { accept: "application/json" }, signal: AbortSignal.timeout(10_000) }); + if (!response.ok) return []; + const body = (await response.json()) as { topics?: Array<{ phrase?: unknown; votes?: unknown; prior_votes?: unknown }> }; + const topics = (body.topics ?? []).map((row) => ({ + topic: row.phrase, + score: Number(row.votes) * VOTE_SCORE, + count: row.votes, + prior_count: row.prior_votes, + })); + const parsed = parseTrendPayload({ source: TREND_SOURCE, topics }, { limit: 200 }); + return parsed.ok ? parsed.signals : []; + } catch { + return []; + } +} + +/** + * One ranking from what is being written and what is being searched. A subject + * in both adds up, which is the strongest signal either list can give. + */ +export function mixTrends(written: TrendSignal[], searched: TrendSignal[], limit = 50): TrendSignal[] { + const merged = new Map(written.map((signal) => [signal.topic, { ...signal }])); + for (const signal of searched) { + const same = merged.get(signal.topic); + if (!same) { + merged.set(signal.topic, { ...signal, windowDays: written[0]?.windowDays ?? signal.windowDays }); + continue; + } + same.score += signal.score; + same.mentions += signal.mentions; + same.priorMentions += signal.priorMentions; + } + return [...merged.values()].sort((a, b) => b.score - a.score).slice(0, limit); +} + /** Pull and store in one call. What the cron route and the CLI both run. */ export async function ingestTrends( sb: SupabaseClient, @@ -194,7 +237,9 @@ export async function ingestTrends( if (!fetched.ok) return fetched; const windowDays = Math.round(clampNumber(config.windowDays, 1, 90, TREND_WINDOW_DAYS)); try { - const { stored, removed } = await storeTrends(sb, fetched.signals, { windowDays }); + const searched = config.votedUrl ? await votedSignals(config.votedUrl, fetchImpl) : []; + const signals = mixTrends(fetched.signals, searched, config.limit ?? 50); + const { stored, removed } = await storeTrends(sb, signals, { windowDays }); return { ok: true, source: fetched.signals[0]?.source ?? TREND_SOURCE, diff --git a/lib/env.ts b/lib/env.ts index 3d20673..cc179a6 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -63,6 +63,7 @@ export const env = { torSocksUrl: process.env.TOR_SOCKS_URL ?? "", // Where the trending subjects come from. Public, so no key. trendUrl: process.env.TREND_URL ?? "https://rssamp.com", + votedSearchesUrl: process.env.VOTED_SEARCHES_URL ?? "https://pit.world/api/voted-searches", workerUrl: process.env.WORKER_URL ?? "", workerSecret: process.env.WORKER_SHARED_SECRET ?? "", cronSecret: process.env.CRON_SECRET ?? "", diff --git a/tests/ads-trending.test.ts b/tests/ads-trending.test.ts index b657f8c..3470660 100644 --- a/tests/ads-trending.test.ts +++ b/tests/ads-trending.test.ts @@ -17,7 +17,7 @@ import { trendsAreStale, type TrendSignal, } from "@/lib/ads/trending"; -import { parseTrendPayload } from "@/lib/ads/trends"; +import { ingestTrends, mixTrends, parseTrendPayload } from "@/lib/ads/trends"; import { parseCampaignPatch, parseCampaignRequest } from "@/lib/ads/campaign-request"; import { TRENDING_CPC_CENTS, CREDIT_CENTS, trendingCpcCredits } from "@/lib/ads/pricing"; import { campaignBodyFromArgs, promoLine, parseArgs } from "@/cli/index"; @@ -294,3 +294,34 @@ describe("unsafe topics", () => { expect(parsed.ok && parsed.signals.map((s) => s.label)).toEqual(["midterm results"]); }); }); + +describe("mixing written and searched trends", () => { + const signal = (topic: string, score: number, mentions: number) => ({ + source: "rssamp", topic, label: topic, score, mentions, priorMentions: 1, windowDays: 7, generatedAt: null, ingestedAt: null, + }); + + it("adds up a subject in both lists and ranks the result", () => { + const out = mixTrends([signal("blue wave", 100, 60), signal("learjet", 300, 80)], [signal("blue wave", 250, 5), signal("zelda", 150, 3)]); + expect(out.map((s) => s.topic)).toEqual(["blue wave", "learjet", "zelda"]); + expect(out[0]).toMatchObject({ score: 350, mentions: 65, priorMentions: 2 }); + }); + + it("keeps the written list when there are no votes", () => { + expect(mixTrends([signal("a", 1, 1)], []).map((s) => s.topic)).toEqual(["a"]); + }); + + it("survives the vote source being down", async () => { + const stored: unknown[][] = []; + const sb = { + from: () => ({ + upsert: async (rows: unknown[]) => { stored.push(rows); return { error: null }; }, + delete: () => ({ eq: () => ({ eq: () => ({ not: () => ({ select: async () => ({ data: [] }) }) }) }) }), + }), + } as never; + const fetchImpl = (async (url: string) => + String(url).includes("voted") ? new Response("nope", { status: 503 }) : Response.json({ topics: [{ topic: "learjet", score: 5, count: 9 }] })) as typeof fetch; + const out = await ingestTrends(sb, { url: "https://rss.test", votedUrl: "https://pit.test/voted" }, fetchImpl); + expect(out.ok).toBe(true); + expect(stored[0]).toHaveLength(1); + }); +});