Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,9 @@ const recordSpans = () => {
return { tracer, named }
}

const spanFailed = (span: Tracer.NativeSpan) =>
span.status._tag === "Ended" && span.status.exit._tag === "Failure"

/** Holds the first GraphQL call until `release()`; `reached` resolves once a poll is parked there. */
const gateFirstGraphql = () => {
const released = Promise.withResolvers<void>()
Expand Down Expand Up @@ -2231,9 +2234,10 @@ describe("CloudflareAnalyticsService", () => {
}).pipe(Effect.provide(makeLayer(testDb, captured, fetchOptions)))
})

it.effect("pollOrg surfaces a zone authz failure as an observable failure, not just a DB write", () => {
it.effect("a zone authz failure is the connection's health, not an exception event", () => {
const testDb = createTestDb(trackedDbs)
const captured: CapturedIngest[] = []
const { tracer, named } = recordSpans()
return Effect.gen(function* () {
yield* TestClock.setTime(T0)
yield* seedConnection()
Expand All @@ -2245,12 +2249,17 @@ describe("CloudflareAnalyticsService", () => {
settingsFetchedAt: T0 - 5 * MIN,
})
const service = yield* CloudflareAnalyticsService
const summary = yield* service.pollOrg(ORG)
const summary = yield* service.pollOrg(ORG).pipe(Effect.withTracer(tracer))
// The token's missing permission repeats every tick until the tenant reconnects, so it
// lands on the summary and the row, never as a failed span.
const observed = named("CloudflareAnalyticsService.datasetPollFailed")
assert.isAbove(observed.length, 0)
assert.deepStrictEqual(Arr.filter(observed, spanFailed), [])
// Regression for the invisible outage: a Cloudflare "not authorized" on the zone dataset
// used to be written ONLY to cloudflare_analytics_state.lastError with zero telemetry, so a
// fully-broken integration was invisible to find_errors/logs for hours. It must now be a
// classified, first-class failure on the summary (which the seam turns into an exception
// event + ERROR log through Maple's own error pipeline).
// classified, first-class failure on the summary (which the seam logs as a warning and
// the source's health shows).
const authz = Option.getOrThrow(
Arr.findFirst(
summary.failures,
Expand All @@ -2272,6 +2281,37 @@ describe("CloudflareAnalyticsService", () => {
)
})

it.effect("an unrecognised GraphQL error still fails its span", () => {
const testDb = createTestDb(trackedDbs)
const captured: CapturedIngest[] = []
const { tracer, named } = recordSpans()
return Effect.gen(function* () {
yield* TestClock.setTime(T0)
yield* seedConnection()
yield* seedStateRow({
dataset: "http_requests",
zoneId: ZONE_ID,
zoneName: ZONE_NAME,
watermarkAt: T0 - 30 * MIN,
settingsFetchedAt: T0 - 5 * MIN,
})
const service = yield* CloudflareAnalyticsService
const summary = yield* service.pollOrg(ORG).pipe(Effect.withTracer(tracer))
assert.include(
Arr.map(summary.failures, (failure) => failure.kind),
"other",
)
const observed = named("CloudflareAnalyticsService.datasetPollFailed")
assert.isAbove(Arr.filter(observed, spanFailed).length, 0)
}).pipe(
Effect.provide(
makeLayer(testDb, captured, {
graphqlErrors: [{ message: "cannot parse filter argument" }],
}),
),
)
})

it.effect("one dataset's authz failure does not starve its batched siblings", () => {
const testDb = createTestDb(trackedDbs)
const captured: CapturedIngest[] = []
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1107,6 +1107,9 @@ type PollOutcome =
/** A sibling's error nulled the document's data; this part did not fail and is re-queried alone. */
| { readonly kind: "voided" }

/** Failures the tenant (permissions, plan) or Cloudflare (5xx, timeouts) owns; only `other` may be ours. */
const isTenantOrVendorFailure = (kind: DatasetPollFailure["kind"]) => kind !== "other"

/** Typed constructor so each failure site returns `PollOutcome` (widens the failure off `as const`). */
const failedOutcome = (failure: DatasetPollFailure): PollOutcome => ({ kind: "failed", failure })

Expand Down Expand Up @@ -1222,22 +1225,29 @@ interface IngestRefused {
}

/**
* The single seam that turns a poll failure into signal: record health to Postgres (as before) AND
* record an OTel exception event + ERROR log so the failure is visible in Maple's own tooling. This
* The single seam that turns a poll failure into signal: health is on the rows and the summary;
* only an `other` failure (possibly ours) adds an OTel exception event + ERROR log, the tenant's or
* Cloudflare's own failures log a warning on an Ok span. This
* is the ONLY place that reacts to a `DatasetPollFailure`; every failure path just classifies and
* returns one, so a new failure path is observable by construction rather than by remembering to
* instrument it. Expected degradations (`quantiles-downgraded`, `disabled`) never reach here.
*/
const observeDatasetFailure = (orgId: OrgId, failure: DatasetPollFailure) =>
Effect.gen(function* () {
yield* Effect.logError("cloudflare-analytics dataset poll failed", {
const fields = {
orgId,
dataset: failure.datasetId,
scope: failure.scope,
kind: failure.kind,
affectedRows: failure.rowIds.length,
error: failure.message,
})
}
// The tenant's grant or Cloudflare's outage, already on the rows and the source's health:
// a warning, not an exception event every tick (one org's token raised ~1.2k a day).
if (isTenantOrVendorFailure(failure.kind)) {
return yield* Effect.logWarning("cloudflare-analytics dataset poll refused", fields)
}
yield* Effect.logError("cloudflare-analytics dataset poll failed", fields)
// Failing inside the span below is what records the exception event that error_events /
// find_errors unwrap. Caught immediately after the span boundary so a single bad dataset
// never fails the org poll.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -896,14 +896,8 @@ describe("GcpMetricsService", () => {
2,
)
assert.notInclude(recorder.text(), "must not be stored")
// The request's Client span fails on the 403 (OTel client semantics); Maple's do not.
assert.deepStrictEqual(
Arr.filter(
recorder.failedSpans(),
(span) => !Str.startsWith("VendorHttp.request: ")(span),
),
[],
)
// A refused impersonation fails no span, the request's own Client span included.
assert.deepStrictEqual(recorder.failedSpans(), [])
}),
{ recorder },
)
Expand Down
31 changes: 24 additions & 7 deletions packages/backend/src/services/integrations/supabase/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,22 @@ export class SupabaseApiError extends Schema.TaggedError<SupabaseApiError>()(
"@maple/supabase/SupabaseApiError",
{
message: Schema.String,
/** `unauthorized` (401) drives refresh-and-retry; `rate_limited` (429) is retryable later. */
kind: Schema.Literals(["unauthorized", "forbidden", "rate_limited", "upstream"]),
/**
* `unauthorized` (401) drives refresh-and-retry; `rate_limited` (429) is retryable later;
* `invalid_response` is an answer we could not decode, so possibly our bug.
*/
kind: Schema.Literals(["unauthorized", "forbidden", "rate_limited", "upstream", "invalid_response"]),
status: Schema.optionalKey(Schema.Finite),
retryAfterSeconds: Schema.optionalKey(Schema.Finite),
},
) {}

/** A Supabase call's span: a refusal or outage the poller records as health stays off the error list. */
export const supabaseCallSpan = (name: string) =>
VendorHttp.vendorCallSpan(name, (error: { readonly _tag: string }) =>
Schema.is(SupabaseApiError)(error) && error.kind !== "invalid_response" ? error.kind : undefined,
)

export const DRAINS_PLAN_MESSAGE = "Log drains need a Supabase Pro plan or higher."

/** Supabase answered 402: the organization's plan does not include the feature (log drains need Pro). */
Expand Down Expand Up @@ -98,9 +107,14 @@ const toSupabaseError = (error: VendorHttp.VendorHttpError): SupabaseApiError =>
})
case "timeout":
return new SupabaseApiError({ message: `Supabase timed out while ${what}`, kind: "upstream" })
case "invalid_response":
return new SupabaseApiError({
message: `Supabase returned an unexpected body while ${what}`,
kind: "invalid_response",
...status,
})
case "not_found":
case "upstream":
case "invalid_response":
return new SupabaseApiError({
message:
error.status === undefined
Expand Down Expand Up @@ -146,7 +160,7 @@ const getArray = (
() =>
new SupabaseApiError({
message: `Supabase returned an unexpected body while ${what}`,
kind: "upstream",
kind: "invalid_response",
}),
),
),
Expand Down Expand Up @@ -310,7 +324,10 @@ export const otlpLogDrainBody = (spec: OtlpLogDrainSpec) => ({
const drainsPath = (ref: string) => `/v2/projects/${encodeURIComponent(ref)}/analytics/log-drains`

const unexpectedBody = (what: string) =>
new SupabaseApiError({ message: `Supabase returned an unexpected body while ${what}`, kind: "upstream" })
new SupabaseApiError({
message: `Supabase returned an unexpected body while ${what}`,
kind: "invalid_response",
})

/** `POST /v2/projects/{ref}/analytics/log-drains`. A 402 is the plan gate (Pro or higher). */
export const createLogDrain = Effect.fn("SupabaseApi.createLogDrain")(function* (
Expand Down Expand Up @@ -339,7 +356,7 @@ export const createLogDrain = Effect.fn("SupabaseApi.createLogDrain")(function*
* `GET /v2/projects/{ref}/analytics/log-drains`; a drain with an undecodable shape is skipped.
* None when the plan has no drains: Free answers 403 "does not have access to this API" (live, 2026-10-22).
*/
export const listLogDrains = Effect.fn("SupabaseApi.listLogDrains")(function* (
export const listLogDrains = Effect.fnUntraced(function* (
httpClient: HttpClient.HttpClient,
baseUrl: string,
accessToken: string,
Expand Down Expand Up @@ -367,7 +384,7 @@ export const listLogDrains = Effect.fn("SupabaseApi.listLogDrains")(function* (
"maple.supabase.log_drains_skipped": list.data.length - drains.length,
})
return Option.some<ReadonlyArray<SupabaseLogDrain>>(drains)
})
}, supabaseCallSpan("SupabaseApi.listLogDrains"))

/** `DELETE /v2/projects/{ref}/analytics/log-drains/{id}`. A 404 means it is already gone. */
export const deleteLogDrain = Effect.fn("SupabaseApi.deleteLogDrain")(function* (
Expand Down
15 changes: 9 additions & 6 deletions packages/backend/src/services/integrations/supabase/insights.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import type { SupabaseProjectRef } from "@maple/primitives"
import { Effect, Option, Schema, Array as Arr } from "effect"
import type { HttpClient } from "effect/http"
import * as VendorHttp from "../vendor-http"
import { SupabaseApiError, supabaseClient } from "./api"
import { SupabaseApiError, supabaseCallSpan, supabaseClient } from "./api"

export const TOP_QUERIES_LIMIT = 10
const QUERY_TEXT_CHARS = 2000
Expand Down Expand Up @@ -123,9 +123,12 @@ export const decodeStatements = (raw: ReadonlyArray<unknown>) =>
)

const unexpectedBody = (what: string) =>
new SupabaseApiError({ message: `Supabase returned an unexpected body while ${what}`, kind: "upstream" })
new SupabaseApiError({
message: `Supabase returned an unexpected body while ${what}`,
kind: "invalid_response",
})

export const getAdvisors = Effect.fn("SupabaseApi.getAdvisors")(function* (
export const getAdvisors = Effect.fnUntraced(function* (
httpClient: HttpClient.HttpClient,
apiBaseUrl: string,
accessToken: string,
Expand All @@ -149,10 +152,10 @@ export const getAdvisors = Effect.fn("SupabaseApi.getAdvisors")(function* (
"maple.supabase.lints_skipped": body.value.lints.length - lints.length,
})
return lints
})
}, supabaseCallSpan("SupabaseApi.getAdvisors"))

/** The top statements by total execution time. A refused query (e.g. no privilege) fails with its message. */
export const getTopQueries = Effect.fn("SupabaseApi.getTopQueries")(function* (
export const getTopQueries = Effect.fnUntraced(function* (
httpClient: HttpClient.HttpClient,
apiBaseUrl: string,
accessToken: string,
Expand All @@ -171,4 +174,4 @@ export const getTopQueries = Effect.fn("SupabaseApi.getTopQueries")(function* (
const rows = decodeRowsBody(text)
if (Option.isNone(rows)) return yield* unexpectedBody(what)
return decodeStatements(rows.value)
})
}, supabaseCallSpan("SupabaseApi.getTopQueries"))
30 changes: 29 additions & 1 deletion packages/backend/src/services/integrations/supabase/logs.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { assert, describe, it } from "@effect/vitest"
import { Schema, Array as Arr, Record as Rec, String as Str, Option, Order } from "effect"
import { Effect, Schema, Tracer, Array as Arr, Record as Rec, String as Str, Option, Order } from "effect"
// A live logs response (2026-10-21) with the project ref, client IPs, geo, ISP and JWT signature
// fields replaced by placeholders (documentation IPs, "Example City", AS64500).
import sample from "./__fixtures__/supabase-logs-sample.json" with { type: "json" }
Expand All @@ -13,6 +13,7 @@ import {
parseRowTimestamp,
} from "./logs"
import { SupabaseProjectRef } from "@maple/primitives"
import { SupabaseApiError, supabaseCallSpan } from "./api"

const REF = Schema.decodeSync(SupabaseProjectRef)("abcdefghijklmnopqrst")
const minute = 60_000
Expand Down Expand Up @@ -245,3 +246,30 @@ describe("advanceWatermark", () => {
})
})
})

describe("supabaseCallSpan", () => {
it.effect("fails the span for an undecodable answer, keeps a refusal Ok", () => {
const spans: Array<Tracer.NativeSpan> = []
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options)
spans.push(span)
return span
},
})
const call = (kind: SupabaseApiError["kind"]) =>
Effect.flip(
Effect.fail(new SupabaseApiError({ message: kind, kind })).pipe(
supabaseCallSpan("SupabaseApi.test"),
),
)
return Effect.gen(function* () {
yield* call("forbidden")
yield* call("invalid_response")
assert.deepStrictEqual(
Arr.map(spans, (span) => (span.status._tag === "Ended" ? span.status.exit._tag : "Open")),
["Success", "Failure"],
)
}).pipe(Effect.withTracer(tracer))
})
})
8 changes: 4 additions & 4 deletions packages/backend/src/services/integrations/supabase/logs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import type { HttpClient } from "effect/http"
import type { SupabaseProjectRef } from "@maple/primitives"
import { timestampMs } from "@maple/backend/platform/time"
import * as VendorHttp from "../vendor-http"
import { SupabaseApiError, supabaseClient } from "./api"
import { SupabaseApiError, supabaseCallSpan, supabaseClient } from "./api"

/** Rows asked for per query; a full page means the window has more. */
export const LOGS_PAGE_SIZE = 1000
Expand Down Expand Up @@ -79,7 +79,7 @@ const errorText = (error: unknown) =>
)(typeof error === "string" ? error : Option.getOrElse(encodeJsonOption(error), () => "unknown error"))

/** One page of raw rows; a refused query fails with {@link SupabaseLogsQueryError}. */
export const queryLogs = Effect.fn("SupabaseApi.queryLogs")(function* (
export const queryLogs = Effect.fnUntraced(function* (
httpClient: HttpClient.HttpClient,
apiBaseUrl: string,
accessToken: string,
Expand All @@ -94,7 +94,7 @@ export const queryLogs = Effect.fn("SupabaseApi.queryLogs")(function* (
if (Option.isNone(body)) {
return yield* new SupabaseApiError({
message: "Supabase returned an unexpected body while reading project logs",
kind: "upstream",
kind: "invalid_response",
})
}
if ("error" in body.value) {
Expand All @@ -104,7 +104,7 @@ export const queryLogs = Effect.fn("SupabaseApi.queryLogs")(function* (
}
yield* Effect.annotateCurrentSpan("maple.supabase.logs.rows", body.value.result.length)
return body.value.result
})
}, supabaseCallSpan("SupabaseApi.queryLogs"))

const LogRowSchema = Schema.Struct({
id: Schema.String,
Expand Down
Loading
Loading