From 916d9c9e6a1e2e501fa72d0bf8db327748379bd4 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 7 Nov 2026 03:23:44 +0000 Subject: [PATCH 1/2] Run the DNS audit's AI step on c0mpute first It was the last direct text call that could use the shared backend AI router. It needed ANTHROPIC_API_KEY, so with that key dead the free DNS scan only ever returned the deterministic baseline. Co-Authored-By: Claude Opus 5.5 --- lib/audit/dns-engine.ts | 35 +++++++++++++------------------- tests/dns-engine.test.ts | 43 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 21 deletions(-) create mode 100644 tests/dns-engine.test.ts diff --git a/lib/audit/dns-engine.ts b/lib/audit/dns-engine.ts index a064a3c2..2b360876 100644 --- a/lib/audit/dns-engine.ts +++ b/lib/audit/dns-engine.ts @@ -1,9 +1,9 @@ import Anthropic from "@anthropic-ai/sdk"; -import { zodOutputFormat } from "@anthropic-ai/sdk/helpers/zod"; -// zod/v4 — the SDK's zodOutputFormat calls z.toJSONSchema(), a v4-only API. +// zod/v4: backendAi and the SDK both call z.toJSONSchema(), a v4-only API. // See the note in claude-engine.ts. import { z } from "zod/v4"; import { env } from "../env"; +import { generateStructuredOutput, hasBackendAiTextProvider } from "../lx/backendAi"; import { collectDnsRecords, dnsBaselineFindings, type DnsRecords } from "./dns"; import { scoreFindings } from "./score"; import type { ClaudeAuditResult } from "./claude-engine"; @@ -97,26 +97,19 @@ export async function dnsAudit(targetUrl: string): Promise { // AI step — best effort. Fall back to the deterministic baseline on any // failure so this free scan never hard-fails the worker. - if (env.anthropicApiKey) { + if (hasBackendAiTextProvider({ anthropicApiKey: env.anthropicApiKey, openaiApiKey: env.openaiApiKey })) { try { - const client = new Anthropic({ apiKey: env.anthropicApiKey }); - const stream = client.messages.stream({ - model: "claude-sonnet-4-6", - max_tokens: 16000, - thinking: { type: "disabled" }, - output_config: { - effort: "medium", - // eslint-disable-next-line @typescript-eslint/no-explicit-any - format: zodOutputFormat(ResultSchema as any), - }, - system: [ - { type: "text", text: SYSTEM_PROMPT, cache_control: { type: "ephemeral" } }, - ], - messages: [{ role: "user", content: buildUserPrompt(records) }], - }); - const response = await stream.finalMessage(); - const parsed = response.parsed_output; - if (!parsed) throw new Error(`no parsed output (stop_reason=${response.stop_reason})`); + const { output: parsed } = await generateStructuredOutput({ + name: "dns_audit", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + schema: ResultSchema as any, + system: SYSTEM_PROMPT, + user: buildUserPrompt(records), + maxTokens: 16000, + anthropicModel: "claude-sonnet-4-6", + anthropic: env.anthropicApiKey ? new Anthropic({ apiKey: env.anthropicApiKey }) : null, + anthropicCacheSystemPrompt: true, + }) as { output: z.infer }; const aiFindings: Finding[] = parsed.findings.map( (f: z.infer) => ({ diff --git a/tests/dns-engine.test.ts b/tests/dns-engine.test.ts new file mode 100644 index 00000000..549b7e39 --- /dev/null +++ b/tests/dns-engine.test.ts @@ -0,0 +1,43 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const generate = vi.hoisted(() => vi.fn()); + +vi.mock("@/lib/lx/backendAi", () => ({ + hasBackendAiTextProvider: () => true, + generateStructuredOutput: generate, +})); +vi.mock("@/lib/env", () => ({ env: { anthropicApiKey: "", openaiApiKey: "" } })); +vi.mock("@/lib/audit/dns", () => ({ + collectDnsRecords: async () => ({ domain: "example.com", a: [], aaaa: [], mx: [], ns: [], txt: [], dkim: [] }), + dnsBaselineFindings: () => [ + { section: "DNS", check_key: "dns.inventory", status: "pass", title: "Inventory", priority: 5 }, + ], +})); + +import { dnsAudit } from "@/lib/audit/dns-engine"; + +describe("dnsAudit AI step", () => { + beforeEach(() => generate.mockReset()); + + it("runs with no paid key and takes c0mpute's answer", async () => { + generate.mockResolvedValue({ + provider: "c0mpute", + output: { + score: 80, + findings: [{ check_key: "spf", status: "warn", title: "SPF is soft", priority: 2 }], + markdown: "# DNS", + }, + }); + const result = await dnsAudit("https://example.com"); + expect(generate).toHaveBeenCalledOnce(); + expect(generate.mock.calls[0][0].anthropic).toBeNull(); + expect(result.score).toBe(80); + expect(result.findings.map((f) => f.check_key)).toEqual(["dns.inventory", "dns.spf"]); + }); + + it("keeps the deterministic baseline when every provider fails", async () => { + generate.mockRejectedValue(new Error("Backend AI generation failed")); + const result = await dnsAudit("https://example.com"); + expect(result.findings.map((f) => f.check_key)).toEqual(["dns.inventory"]); + }); +}); From 05ea1e1a894d498c2eed7274a3fe2fcdd80bc24a Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 7 Nov 2026 03:24:40 +0000 Subject: [PATCH 2/2] Fix the DNS audit test's failure case Co-Authored-By: Claude Opus 5.5 --- tests/dns-engine.test.ts | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/dns-engine.test.ts b/tests/dns-engine.test.ts index 549b7e39..5b9e7cb1 100644 --- a/tests/dns-engine.test.ts +++ b/tests/dns-engine.test.ts @@ -1,10 +1,13 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -const generate = vi.hoisted(() => vi.fn()); +const calls = vi.hoisted(() => ({ args: [] as any[], answer: null as null | (() => unknown) })); vi.mock("@/lib/lx/backendAi", () => ({ hasBackendAiTextProvider: () => true, - generateStructuredOutput: generate, + generateStructuredOutput: async (args: unknown) => { + calls.args.push(args); + return calls.answer!(); + }, })); vi.mock("@/lib/env", () => ({ env: { anthropicApiKey: "", openaiApiKey: "" } })); vi.mock("@/lib/audit/dns", () => ({ @@ -17,10 +20,12 @@ vi.mock("@/lib/audit/dns", () => ({ import { dnsAudit } from "@/lib/audit/dns-engine"; describe("dnsAudit AI step", () => { - beforeEach(() => generate.mockReset()); + beforeEach(() => { + calls.args = []; + }); it("runs with no paid key and takes c0mpute's answer", async () => { - generate.mockResolvedValue({ + calls.answer = () => ({ provider: "c0mpute", output: { score: 80, @@ -29,14 +34,16 @@ describe("dnsAudit AI step", () => { }, }); const result = await dnsAudit("https://example.com"); - expect(generate).toHaveBeenCalledOnce(); - expect(generate.mock.calls[0][0].anthropic).toBeNull(); + expect(calls.args).toHaveLength(1); + expect(calls.args[0].anthropic).toBeNull(); expect(result.score).toBe(80); expect(result.findings.map((f) => f.check_key)).toEqual(["dns.inventory", "dns.spf"]); }); it("keeps the deterministic baseline when every provider fails", async () => { - generate.mockRejectedValue(new Error("Backend AI generation failed")); + calls.answer = () => { + throw new Error("Backend AI generation failed"); + }; const result = await dnsAudit("https://example.com"); expect(result.findings.map((f) => f.check_key)).toEqual(["dns.inventory"]); });