From beb647e8ed943751e5a0e5e5f374ffeaa461c8c0 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 16 Nov 2026 16:50:45 +0100 Subject: [PATCH 1/3] fix(integrations): vendor refusals and outages are connection health, not error issues A customer's missing permission, plan limit or a vendor 5xx used to fail a span on every poll and open an error issue. The vendor HTTP attempt span now fails only on an answer it cannot decode; Cloudflare analytics logs tenant and vendor failures as a warning on the rows' health; Supabase reads and Vercel drain creation keep handled refusals off their spans. --- .../CloudflareAnalyticsService.test.ts | 44 ++++++++++- .../CloudflareAnalyticsService.ts | 13 +++- .../integrations/GcpMetricsService.test.ts | 10 +-- .../src/services/integrations/supabase/api.ts | 10 ++- .../integrations/supabase/insights.ts | 10 +-- .../services/integrations/supabase/logs.ts | 6 +- .../services/integrations/vendor-http.test.ts | 76 +++++++++++++++++++ .../src/services/integrations/vendor-http.ts | 46 ++++++++++- .../src/services/integrations/vercel/api.ts | 29 +++---- 9 files changed, 207 insertions(+), 37 deletions(-) diff --git a/packages/backend/src/services/integrations/CloudflareAnalyticsService.test.ts b/packages/backend/src/services/integrations/CloudflareAnalyticsService.test.ts index 0036c6d19e..447122dbcb 100644 --- a/packages/backend/src/services/integrations/CloudflareAnalyticsService.test.ts +++ b/packages/backend/src/services/integrations/CloudflareAnalyticsService.test.ts @@ -288,6 +288,9 @@ const recordSpans = () => { return { tracer, named } } +const spanFailed = (span: Tracer.NativeSpan) => + span.status._tag === "Ended" && span.status.exit._tag === "Failure" + /** Holds the first GraphQL call until `release()`; `reached` resolves once a poll is parked there. */ const gateFirstGraphql = () => { const released = Promise.withResolvers() @@ -2231,9 +2234,10 @@ describe("CloudflareAnalyticsService", () => { }).pipe(Effect.provide(makeLayer(testDb, captured, fetchOptions))) }) - it.effect("pollOrg surfaces a zone authz failure as an observable failure, not just a DB write", () => { + it.effect("a zone authz failure is the connection's health, not an exception event", () => { const testDb = createTestDb(trackedDbs) const captured: CapturedIngest[] = [] + const { tracer, named } = recordSpans() return Effect.gen(function* () { yield* TestClock.setTime(T0) yield* seedConnection() @@ -2245,7 +2249,12 @@ describe("CloudflareAnalyticsService", () => { settingsFetchedAt: T0 - 5 * MIN, }) const service = yield* CloudflareAnalyticsService - const summary = yield* service.pollOrg(ORG) + const summary = yield* service.pollOrg(ORG).pipe(Effect.withTracer(tracer)) + // The token's missing permission repeats every tick until the tenant reconnects, so it + // lands on the summary and the row, never as a failed span. + const observed = named("CloudflareAnalyticsService.datasetPollFailed") + assert.isAbove(observed.length, 0) + assert.deepStrictEqual(Arr.filter(observed, spanFailed), []) // Regression for the invisible outage: a Cloudflare "not authorized" on the zone dataset // used to be written ONLY to cloudflare_analytics_state.lastError with zero telemetry, so a // fully-broken integration was invisible to find_errors/logs for hours. It must now be a @@ -2272,6 +2281,37 @@ describe("CloudflareAnalyticsService", () => { ) }) + it.effect("an unrecognised GraphQL error still fails its span", () => { + const testDb = createTestDb(trackedDbs) + const captured: CapturedIngest[] = [] + const { tracer, named } = recordSpans() + return Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection() + yield* seedStateRow({ + dataset: "http_requests", + zoneId: ZONE_ID, + zoneName: ZONE_NAME, + watermarkAt: T0 - 30 * MIN, + settingsFetchedAt: T0 - 5 * MIN, + }) + const service = yield* CloudflareAnalyticsService + const summary = yield* service.pollOrg(ORG).pipe(Effect.withTracer(tracer)) + assert.include( + Arr.map(summary.failures, (failure) => failure.kind), + "other", + ) + const observed = named("CloudflareAnalyticsService.datasetPollFailed") + assert.isAbove(Arr.filter(observed, spanFailed).length, 0) + }).pipe( + Effect.provide( + makeLayer(testDb, captured, { + graphqlErrors: [{ message: "cannot parse filter argument" }], + }), + ), + ) + }) + it.effect("one dataset's authz failure does not starve its batched siblings", () => { const testDb = createTestDb(trackedDbs) const captured: CapturedIngest[] = [] diff --git a/packages/backend/src/services/integrations/CloudflareAnalyticsService.ts b/packages/backend/src/services/integrations/CloudflareAnalyticsService.ts index 95d958965a..5b146f5630 100644 --- a/packages/backend/src/services/integrations/CloudflareAnalyticsService.ts +++ b/packages/backend/src/services/integrations/CloudflareAnalyticsService.ts @@ -1107,6 +1107,9 @@ type PollOutcome = /** A sibling's error nulled the document's data; this part did not fail and is re-queried alone. */ | { readonly kind: "voided" } +/** Failures the tenant (permissions, plan) or Cloudflare (5xx, timeouts) owns; only `other` may be ours. */ +const isTenantOrVendorFailure = (kind: DatasetPollFailure["kind"]) => kind !== "other" + /** Typed constructor so each failure site returns `PollOutcome` (widens the failure off `as const`). */ const failedOutcome = (failure: DatasetPollFailure): PollOutcome => ({ kind: "failed", failure }) @@ -1230,14 +1233,20 @@ interface IngestRefused { */ const observeDatasetFailure = (orgId: OrgId, failure: DatasetPollFailure) => Effect.gen(function* () { - yield* Effect.logError("cloudflare-analytics dataset poll failed", { + const fields = { orgId, dataset: failure.datasetId, scope: failure.scope, kind: failure.kind, affectedRows: failure.rowIds.length, error: failure.message, - }) + } + // The tenant's grant or Cloudflare's outage, already on the rows and the source's health: + // a warning, not an exception event every tick (one org's token raised ~1.2k a day). + if (isTenantOrVendorFailure(failure.kind)) { + return yield* Effect.logWarning("cloudflare-analytics dataset poll refused", fields) + } + yield* Effect.logError("cloudflare-analytics dataset poll failed", fields) // Failing inside the span below is what records the exception event that error_events / // find_errors unwrap. Caught immediately after the span boundary so a single bad dataset // never fails the org poll. diff --git a/packages/backend/src/services/integrations/GcpMetricsService.test.ts b/packages/backend/src/services/integrations/GcpMetricsService.test.ts index 95f7fccc7f..9a99a72d0d 100644 --- a/packages/backend/src/services/integrations/GcpMetricsService.test.ts +++ b/packages/backend/src/services/integrations/GcpMetricsService.test.ts @@ -896,14 +896,8 @@ describe("GcpMetricsService", () => { 2, ) assert.notInclude(recorder.text(), "must not be stored") - // The request's Client span fails on the 403 (OTel client semantics); Maple's do not. - assert.deepStrictEqual( - Arr.filter( - recorder.failedSpans(), - (span) => !Str.startsWith("VendorHttp.request: ")(span), - ), - [], - ) + // A refused impersonation fails no span, the request's own Client span included. + assert.deepStrictEqual(recorder.failedSpans(), []) }), { recorder }, ) diff --git a/packages/backend/src/services/integrations/supabase/api.ts b/packages/backend/src/services/integrations/supabase/api.ts index 1c7c84d72b..371e552f15 100644 --- a/packages/backend/src/services/integrations/supabase/api.ts +++ b/packages/backend/src/services/integrations/supabase/api.ts @@ -19,6 +19,12 @@ export class SupabaseApiError extends Schema.TaggedError()( }, ) {} +/** A Supabase call's span: a refusal or outage the poller records as health stays off the error list. */ +export const supabaseCallSpan = (name: string) => + VendorHttp.vendorCallSpan(name, (error: { readonly _tag: string }) => + Schema.is(SupabaseApiError)(error) ? error.kind : undefined, + ) + export const DRAINS_PLAN_MESSAGE = "Log drains need a Supabase Pro plan or higher." /** Supabase answered 402: the organization's plan does not include the feature (log drains need Pro). */ @@ -339,7 +345,7 @@ export const createLogDrain = Effect.fn("SupabaseApi.createLogDrain")(function* * `GET /v2/projects/{ref}/analytics/log-drains`; a drain with an undecodable shape is skipped. * None when the plan has no drains: Free answers 403 "does not have access to this API" (live, 2026-10-22). */ -export const listLogDrains = Effect.fn("SupabaseApi.listLogDrains")(function* ( +export const listLogDrains = Effect.fnUntraced(function* ( httpClient: HttpClient.HttpClient, baseUrl: string, accessToken: string, @@ -367,7 +373,7 @@ export const listLogDrains = Effect.fn("SupabaseApi.listLogDrains")(function* ( "maple.supabase.log_drains_skipped": list.data.length - drains.length, }) return Option.some>(drains) -}) +}, supabaseCallSpan("SupabaseApi.listLogDrains")) /** `DELETE /v2/projects/{ref}/analytics/log-drains/{id}`. A 404 means it is already gone. */ export const deleteLogDrain = Effect.fn("SupabaseApi.deleteLogDrain")(function* ( diff --git a/packages/backend/src/services/integrations/supabase/insights.ts b/packages/backend/src/services/integrations/supabase/insights.ts index 5f1e43d3dc..f6de0e9772 100644 --- a/packages/backend/src/services/integrations/supabase/insights.ts +++ b/packages/backend/src/services/integrations/supabase/insights.ts @@ -14,7 +14,7 @@ import type { SupabaseProjectRef } from "@maple/primitives" import { Effect, Option, Schema, Array as Arr } from "effect" import type { HttpClient } from "effect/http" import * as VendorHttp from "../vendor-http" -import { SupabaseApiError, supabaseClient } from "./api" +import { SupabaseApiError, supabaseCallSpan, supabaseClient } from "./api" export const TOP_QUERIES_LIMIT = 10 const QUERY_TEXT_CHARS = 2000 @@ -125,7 +125,7 @@ export const decodeStatements = (raw: ReadonlyArray) => const unexpectedBody = (what: string) => new SupabaseApiError({ message: `Supabase returned an unexpected body while ${what}`, kind: "upstream" }) -export const getAdvisors = Effect.fn("SupabaseApi.getAdvisors")(function* ( +export const getAdvisors = Effect.fnUntraced(function* ( httpClient: HttpClient.HttpClient, apiBaseUrl: string, accessToken: string, @@ -149,10 +149,10 @@ export const getAdvisors = Effect.fn("SupabaseApi.getAdvisors")(function* ( "maple.supabase.lints_skipped": body.value.lints.length - lints.length, }) return lints -}) +}, supabaseCallSpan("SupabaseApi.getAdvisors")) /** The top statements by total execution time. A refused query (e.g. no privilege) fails with its message. */ -export const getTopQueries = Effect.fn("SupabaseApi.getTopQueries")(function* ( +export const getTopQueries = Effect.fnUntraced(function* ( httpClient: HttpClient.HttpClient, apiBaseUrl: string, accessToken: string, @@ -171,4 +171,4 @@ export const getTopQueries = Effect.fn("SupabaseApi.getTopQueries")(function* ( const rows = decodeRowsBody(text) if (Option.isNone(rows)) return yield* unexpectedBody(what) return decodeStatements(rows.value) -}) +}, supabaseCallSpan("SupabaseApi.getTopQueries")) diff --git a/packages/backend/src/services/integrations/supabase/logs.ts b/packages/backend/src/services/integrations/supabase/logs.ts index 6bd1b2b96a..3bf576b68a 100644 --- a/packages/backend/src/services/integrations/supabase/logs.ts +++ b/packages/backend/src/services/integrations/supabase/logs.ts @@ -15,7 +15,7 @@ import type { HttpClient } from "effect/http" import type { SupabaseProjectRef } from "@maple/primitives" import { timestampMs } from "@maple/backend/platform/time" import * as VendorHttp from "../vendor-http" -import { SupabaseApiError, supabaseClient } from "./api" +import { SupabaseApiError, supabaseCallSpan, supabaseClient } from "./api" /** Rows asked for per query; a full page means the window has more. */ export const LOGS_PAGE_SIZE = 1000 @@ -79,7 +79,7 @@ const errorText = (error: unknown) => )(typeof error === "string" ? error : Option.getOrElse(encodeJsonOption(error), () => "unknown error")) /** One page of raw rows; a refused query fails with {@link SupabaseLogsQueryError}. */ -export const queryLogs = Effect.fn("SupabaseApi.queryLogs")(function* ( +export const queryLogs = Effect.fnUntraced(function* ( httpClient: HttpClient.HttpClient, apiBaseUrl: string, accessToken: string, @@ -104,7 +104,7 @@ export const queryLogs = Effect.fn("SupabaseApi.queryLogs")(function* ( } yield* Effect.annotateCurrentSpan("maple.supabase.logs.rows", body.value.result.length) return body.value.result -}) +}, supabaseCallSpan("SupabaseApi.queryLogs")) const LogRowSchema = Schema.Struct({ id: Schema.String, diff --git a/packages/backend/src/services/integrations/vendor-http.test.ts b/packages/backend/src/services/integrations/vendor-http.test.ts index 0cc126ef48..66be12f04b 100644 --- a/packages/backend/src/services/integrations/vendor-http.test.ts +++ b/packages/backend/src/services/integrations/vendor-http.test.ts @@ -394,6 +394,82 @@ describe("VendorHttp", () => { ) }) + it.effect("fails the attempt span only on an undecodable answer, not on a refusal or outage", () => { + const spans: Array = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const fake = VendorFake.make({ + routes: [ + { path: "/v1/forbidden", respond: () => VendorFake.json({}, 403) }, + { path: "/v1/plan", respond: () => VendorFake.json({}, 402) }, + { path: "/v1/down", respond: () => VendorFake.json({}, 503) }, + { path: "/v1/garbled", respond: () => VendorFake.json({ nope: true }) }, + ], + }) + const paths = ["/v1/forbidden", "/v1/plan", "/v1/down", "/v1/garbled"] + return withClient( + fake, + (client) => + Effect.forEach(paths, (path) => Effect.flip(client.json(Page, { path })), { + discard: true, + }).pipe(Effect.withTracer(tracer)), + { retry: false }, + ).pipe( + Effect.map(() => { + const outcome = Arr.map(spans, (span) => [ + span.attributes.get("url.path"), + span.attributes.get("error.type"), + span.status._tag === "Ended" ? span.status.exit._tag : "Open", + ]) + assert.deepStrictEqual(outcome, [ + ["/v1/forbidden", "forbidden", "Success"], + ["/v1/plan", "upstream", "Success"], + ["/v1/down", "upstream", "Success"], + ["/v1/garbled", "invalid_response", "Failure"], + ]) + }), + ) + }) + + it.effect("vendorCallSpan keeps an expected failure off the span and still fails the call", () => { + const spans: Array = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const call = (error: AcmeError) => + Effect.fail(error).pipe( + VendorHttp.vendorCallSpan("Acme.call", (failure: AcmeError) => + failure.kind === "forbidden" ? failure.kind : undefined, + ), + ) + return Effect.gen(function* () { + const refused = yield* Effect.flip(call(new AcmeError({ message: "no", kind: "forbidden" }))) + const broken = yield* Effect.flip( + call(new AcmeError({ message: "bad", kind: "invalid_response" })), + ) + assert.deepStrictEqual([refused.kind, broken.kind], ["forbidden", "invalid_response"]) + assert.deepStrictEqual( + Arr.map(spans, (span) => [ + span.attributes.get("error.type"), + span.status._tag === "Ended" ? span.status.exit._tag : "Open", + ]), + [ + ["forbidden", "Success"], + [undefined, "Failure"], + ], + ) + }).pipe(Effect.withTracer(tracer)) + }) + it.effect("records one Client span per attempt with the path, never the query or token", () => { const spans: Array = [] const tracer = Tracer.make({ diff --git a/packages/backend/src/services/integrations/vendor-http.ts b/packages/backend/src/services/integrations/vendor-http.ts index fdb559db5b..effc8b74bf 100644 --- a/packages/backend/src/services/integrations/vendor-http.ts +++ b/packages/backend/src/services/integrations/vendor-http.ts @@ -3,7 +3,18 @@ * transient retries, `retry-after` on a 429, no redirects, paging, and a Client span that records * only the URL path. Each client maps {@link VendorHttpError} to its own errors through `toError`. */ -import { Clock, Duration, Effect, Option, Redacted, Schedule, Schema, Stream, String as Str } from "effect" +import { + Clock, + Duration, + Effect, + Option, + Redacted, + Result, + Schedule, + Schema, + Stream, + String as Str, +} from "effect" import { FetchHttpClient, HttpClient, HttpClientRequest, type HttpClientResponse } from "effect/http" import { parseDeltaSeconds, parseRetryAfterSeconds, VENDOR_TRANSIENT_RETRY } from "./vendor-retry" @@ -150,6 +161,30 @@ const isTransient = (error: VendorHttpError) => error.kind === "timeout" || (error.kind === "upstream" && (error.status === undefined || error.status >= 500)) +/** Only an answer we could not decode may be our own bug; anything else is the vendor's or the customer's state. */ +const failsAttemptSpan = (error: VendorHttpError) => error.kind === "invalid_response" + +/** + * A vendor client call's own span, for `Effect.fnUntraced(body, vendorCallSpan(name, expectedKind))`. + * A failure `expectedKind` names (a refusal or outage the caller handles) lands on the span as + * `error.type` instead of failing it; anything else fails the span as usual. + */ +export const vendorCallSpan = + (name: string, expectedKind: (error: E) => string | undefined) => + (self: Effect.Effect): Effect.Effect => + self.pipe( + Effect.result, + Effect.flatMap((result): Effect.Effect, E2> => { + if (Result.isSuccess(result)) return Effect.succeed(result) + const kind = expectedKind(result.failure) + return kind === undefined + ? Effect.fail(result.failure) + : Effect.as(Effect.annotateCurrentSpan("error.type", kind), result) + }), + Effect.withSpan(name), + Effect.flatMap(Effect.fromResult), + ) + const decodeJsonText = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown)) export const make = (options: VendorHttpOptions): VendorHttpClient => { @@ -260,6 +295,14 @@ export const make = (options: VendorHttpOptions): VendorHttpClient => { orElse: () => Effect.fail(fail("timeout", "timed out", request)), }), Effect.tapError((error) => Effect.annotateCurrentSpan("error.type", error.kind)), + // A refusal or a vendor outage is the caller's to judge (most record it as the + // connection's health), so only an undecodable answer fails the attempt span. + Effect.map((value): Result.Result => Result.succeed(value)), + Effect.catchIf( + (error) => !failsAttemptSpan(error), + (error): Effect.Effect> => + Effect.succeed(Result.fail(error)), + ), Effect.withSpan("VendorHttp.request", { kind: "client", attributes: { @@ -273,6 +316,7 @@ export const make = (options: VendorHttpOptions): VendorHttpClient => { : { "maple.vendor.operation": request.operation }), }, }), + Effect.flatMap(Effect.fromResult), ), ), ) diff --git a/packages/backend/src/services/integrations/vercel/api.ts b/packages/backend/src/services/integrations/vercel/api.ts index e29f1a7564..99fd1fe97d 100644 --- a/packages/backend/src/services/integrations/vercel/api.ts +++ b/packages/backend/src/services/integrations/vercel/api.ts @@ -338,20 +338,21 @@ const toDrainInfo = (drain: typeof DrainBody.Type): VercelDrainInfo => ({ const drainPath = (drainId?: string) => `/v1/drains${drainId === undefined ? "" : `/${encodeURIComponent(drainId)}`}` -export const createDrain = Effect.fn("VercelApi.createDrain")(function* ( - httpClient: HttpClient.HttpClient, - token: string, - teamId: string, - body: VercelDrainBody, -) { - const drain = yield* client(httpClient, token).json(DrainBody, { - method: "POST", - path: drainPath(), - query: { teamId }, - body, - }) - return toDrainInfo(drain) -}) +export const createDrain = Effect.fnUntraced( + function* (httpClient: HttpClient.HttpClient, token: string, teamId: string, body: VercelDrainBody) { + const drain = yield* client(httpClient, token).json(DrainBody, { + method: "POST", + path: drainPath(), + query: { teamId }, + body, + }) + return toDrainInfo(drain) + }, + // A plan or permission refusal is answered to the person connecting as a 400, not our failure. + VendorHttp.vendorCallSpan("VercelApi.createDrain", (error: VercelApiError) => + error.kind === "invalid_response" ? undefined : error.kind, + ), +) export const updateDrain = Effect.fn("VercelApi.updateDrain")(function* ( httpClient: HttpClient.HttpClient, From 1c197b51b7f4790e412760b1c4e6d1588e362644 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 16 Nov 2026 17:08:06 +0100 Subject: [PATCH 2/3] fix(integrations): address review notes on createDrain span and Cloudflare seam comments --- .../services/integrations/CloudflareAnalyticsService.test.ts | 4 ++-- .../src/services/integrations/CloudflareAnalyticsService.ts | 5 +++-- packages/backend/src/services/integrations/vercel/api.ts | 5 ++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/services/integrations/CloudflareAnalyticsService.test.ts b/packages/backend/src/services/integrations/CloudflareAnalyticsService.test.ts index 447122dbcb..ad5c93de74 100644 --- a/packages/backend/src/services/integrations/CloudflareAnalyticsService.test.ts +++ b/packages/backend/src/services/integrations/CloudflareAnalyticsService.test.ts @@ -2258,8 +2258,8 @@ describe("CloudflareAnalyticsService", () => { // Regression for the invisible outage: a Cloudflare "not authorized" on the zone dataset // used to be written ONLY to cloudflare_analytics_state.lastError with zero telemetry, so a // fully-broken integration was invisible to find_errors/logs for hours. It must now be a - // classified, first-class failure on the summary (which the seam turns into an exception - // event + ERROR log through Maple's own error pipeline). + // classified, first-class failure on the summary (which the seam logs as a warning and + // the source's health shows). const authz = Option.getOrThrow( Arr.findFirst( summary.failures, diff --git a/packages/backend/src/services/integrations/CloudflareAnalyticsService.ts b/packages/backend/src/services/integrations/CloudflareAnalyticsService.ts index 5b146f5630..71eeb4d6c6 100644 --- a/packages/backend/src/services/integrations/CloudflareAnalyticsService.ts +++ b/packages/backend/src/services/integrations/CloudflareAnalyticsService.ts @@ -1225,8 +1225,9 @@ interface IngestRefused { } /** - * The single seam that turns a poll failure into signal: record health to Postgres (as before) AND - * record an OTel exception event + ERROR log so the failure is visible in Maple's own tooling. This + * The single seam that turns a poll failure into signal: health is on the rows and the summary; + * only an `other` failure (possibly ours) adds an OTel exception event + ERROR log, the tenant's or + * Cloudflare's own failures log a warning on an Ok span. This * is the ONLY place that reacts to a `DatasetPollFailure`; every failure path just classifies and * returns one, so a new failure path is observable by construction rather than by remembering to * instrument it. Expected degradations (`quantiles-downgraded`, `disabled`) never reach here. diff --git a/packages/backend/src/services/integrations/vercel/api.ts b/packages/backend/src/services/integrations/vercel/api.ts index 99fd1fe97d..11ef3741db 100644 --- a/packages/backend/src/services/integrations/vercel/api.ts +++ b/packages/backend/src/services/integrations/vercel/api.ts @@ -349,9 +349,8 @@ export const createDrain = Effect.fnUntraced( return toDrainInfo(drain) }, // A plan or permission refusal is answered to the person connecting as a 400, not our failure. - VendorHttp.vendorCallSpan("VercelApi.createDrain", (error: VercelApiError) => - error.kind === "invalid_response" ? undefined : error.kind, - ), + // An undecodable answer still fails the inner `VendorHttp.request` span. + VendorHttp.vendorCallSpan("VercelApi.createDrain", (error: VercelApiError) => error.kind), ) export const updateDrain = Effect.fn("VercelApi.updateDrain")(function* ( From 9fda7467b8c5ef5cbf49d77b476364698a371884 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 16 Nov 2026 17:15:50 +0100 Subject: [PATCH 3/3] fix(integrations): an undecodable Supabase answer fails its span again --- .../src/services/integrations/supabase/api.ts | 23 ++++++++++---- .../integrations/supabase/insights.ts | 5 +++- .../integrations/supabase/logs.test.ts | 30 ++++++++++++++++++- .../services/integrations/supabase/logs.ts | 2 +- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/services/integrations/supabase/api.ts b/packages/backend/src/services/integrations/supabase/api.ts index 371e552f15..2696c1016a 100644 --- a/packages/backend/src/services/integrations/supabase/api.ts +++ b/packages/backend/src/services/integrations/supabase/api.ts @@ -12,8 +12,11 @@ export class SupabaseApiError extends Schema.TaggedError()( "@maple/supabase/SupabaseApiError", { message: Schema.String, - /** `unauthorized` (401) drives refresh-and-retry; `rate_limited` (429) is retryable later. */ - kind: Schema.Literals(["unauthorized", "forbidden", "rate_limited", "upstream"]), + /** + * `unauthorized` (401) drives refresh-and-retry; `rate_limited` (429) is retryable later; + * `invalid_response` is an answer we could not decode, so possibly our bug. + */ + kind: Schema.Literals(["unauthorized", "forbidden", "rate_limited", "upstream", "invalid_response"]), status: Schema.optionalKey(Schema.Finite), retryAfterSeconds: Schema.optionalKey(Schema.Finite), }, @@ -22,7 +25,7 @@ export class SupabaseApiError extends Schema.TaggedError()( /** A Supabase call's span: a refusal or outage the poller records as health stays off the error list. */ export const supabaseCallSpan = (name: string) => VendorHttp.vendorCallSpan(name, (error: { readonly _tag: string }) => - Schema.is(SupabaseApiError)(error) ? error.kind : undefined, + Schema.is(SupabaseApiError)(error) && error.kind !== "invalid_response" ? error.kind : undefined, ) export const DRAINS_PLAN_MESSAGE = "Log drains need a Supabase Pro plan or higher." @@ -104,9 +107,14 @@ const toSupabaseError = (error: VendorHttp.VendorHttpError): SupabaseApiError => }) case "timeout": return new SupabaseApiError({ message: `Supabase timed out while ${what}`, kind: "upstream" }) + case "invalid_response": + return new SupabaseApiError({ + message: `Supabase returned an unexpected body while ${what}`, + kind: "invalid_response", + ...status, + }) case "not_found": case "upstream": - case "invalid_response": return new SupabaseApiError({ message: error.status === undefined @@ -152,7 +160,7 @@ const getArray = ( () => new SupabaseApiError({ message: `Supabase returned an unexpected body while ${what}`, - kind: "upstream", + kind: "invalid_response", }), ), ), @@ -316,7 +324,10 @@ export const otlpLogDrainBody = (spec: OtlpLogDrainSpec) => ({ const drainsPath = (ref: string) => `/v2/projects/${encodeURIComponent(ref)}/analytics/log-drains` const unexpectedBody = (what: string) => - new SupabaseApiError({ message: `Supabase returned an unexpected body while ${what}`, kind: "upstream" }) + new SupabaseApiError({ + message: `Supabase returned an unexpected body while ${what}`, + kind: "invalid_response", + }) /** `POST /v2/projects/{ref}/analytics/log-drains`. A 402 is the plan gate (Pro or higher). */ export const createLogDrain = Effect.fn("SupabaseApi.createLogDrain")(function* ( diff --git a/packages/backend/src/services/integrations/supabase/insights.ts b/packages/backend/src/services/integrations/supabase/insights.ts index f6de0e9772..5b3d85a4cf 100644 --- a/packages/backend/src/services/integrations/supabase/insights.ts +++ b/packages/backend/src/services/integrations/supabase/insights.ts @@ -123,7 +123,10 @@ export const decodeStatements = (raw: ReadonlyArray) => ) const unexpectedBody = (what: string) => - new SupabaseApiError({ message: `Supabase returned an unexpected body while ${what}`, kind: "upstream" }) + new SupabaseApiError({ + message: `Supabase returned an unexpected body while ${what}`, + kind: "invalid_response", + }) export const getAdvisors = Effect.fnUntraced(function* ( httpClient: HttpClient.HttpClient, diff --git a/packages/backend/src/services/integrations/supabase/logs.test.ts b/packages/backend/src/services/integrations/supabase/logs.test.ts index 909427bf2f..06f867d8b7 100644 --- a/packages/backend/src/services/integrations/supabase/logs.test.ts +++ b/packages/backend/src/services/integrations/supabase/logs.test.ts @@ -1,5 +1,5 @@ import { assert, describe, it } from "@effect/vitest" -import { Schema, Array as Arr, Record as Rec, String as Str, Option, Order } from "effect" +import { Effect, Schema, Tracer, Array as Arr, Record as Rec, String as Str, Option, Order } from "effect" // A live logs response (2026-10-21) with the project ref, client IPs, geo, ISP and JWT signature // fields replaced by placeholders (documentation IPs, "Example City", AS64500). import sample from "./__fixtures__/supabase-logs-sample.json" with { type: "json" } @@ -13,6 +13,7 @@ import { parseRowTimestamp, } from "./logs" import { SupabaseProjectRef } from "@maple/primitives" +import { SupabaseApiError, supabaseCallSpan } from "./api" const REF = Schema.decodeSync(SupabaseProjectRef)("abcdefghijklmnopqrst") const minute = 60_000 @@ -245,3 +246,30 @@ describe("advanceWatermark", () => { }) }) }) + +describe("supabaseCallSpan", () => { + it.effect("fails the span for an undecodable answer, keeps a refusal Ok", () => { + const spans: Array = [] + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + }, + }) + const call = (kind: SupabaseApiError["kind"]) => + Effect.flip( + Effect.fail(new SupabaseApiError({ message: kind, kind })).pipe( + supabaseCallSpan("SupabaseApi.test"), + ), + ) + return Effect.gen(function* () { + yield* call("forbidden") + yield* call("invalid_response") + assert.deepStrictEqual( + Arr.map(spans, (span) => (span.status._tag === "Ended" ? span.status.exit._tag : "Open")), + ["Success", "Failure"], + ) + }).pipe(Effect.withTracer(tracer)) + }) +}) diff --git a/packages/backend/src/services/integrations/supabase/logs.ts b/packages/backend/src/services/integrations/supabase/logs.ts index 3bf576b68a..ac7c99add5 100644 --- a/packages/backend/src/services/integrations/supabase/logs.ts +++ b/packages/backend/src/services/integrations/supabase/logs.ts @@ -94,7 +94,7 @@ export const queryLogs = Effect.fnUntraced(function* ( if (Option.isNone(body)) { return yield* new SupabaseApiError({ message: "Supabase returned an unexpected body while reading project logs", - kind: "upstream", + kind: "invalid_response", }) } if ("error" in body.value) {