From c5663a1dbd094d42e6c09d4474e77d69d2fd5c02 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 15:29:21 +0100 Subject: [PATCH 01/18] feat(promql): a Prometheus-compatible query API for Grafana Serve the Prometheus HTTP API under /v2/metrics/prometheus/api/v1 so Grafana and other Prometheus clients can add Maple as a Prometheus data source with a metrics:read API key: query and query_range (GET and form POST), series, labels, label values, metadata, status/buildinfo (reports 3.5.0) and an empty query_exemplars. Bodies are Prometheus' own JSON, values formatted like Go's FormatFloat, and errors use its envelope: bad_data 400, execution 422, unavailable 503. The form POSTs are registered as read-only for scopes and open to analysts. PromqlQueryService gains series, labelNames, labelValues and metadata. With match[] they read the selector source with one bucket over the window. Without it they answer from the metric catalog and the metric attribute rollups, so discovery never scans raw metric tables. Adds a docs page on connecting Grafana. --- apps/api/package.json | 1 + .../api/src/routes/v2/prometheus-wire.test.ts | 29 ++ apps/api/src/routes/v2/prometheus-wire.ts | 110 ++++++ .../api/src/routes/v2/prometheus.http.test.ts | 320 ++++++++++++++++++ apps/api/src/routes/v2/prometheus.http.ts | 187 ++++++++++ apps/api/src/runtime/http-graph.ts | 2 + apps/api/src/runtime/service-graph.ts | 2 + .../MapleAPI/Sources/MapleAPI/openapi.json | 4 + .../src/content/docs/reference/prometheus.md | 56 +++ bun.lock | 1 + .../promql/PromqlQueryService.test.ts | 177 +++++++++- .../src/services/promql/PromqlQueryService.ts | 238 ++++++++++++- packages/domain/src/http/v2/api.ts | 2 + packages/domain/src/http/v2/auth.ts | 6 + packages/domain/src/http/v2/index.ts | 1 + packages/domain/src/http/v2/openapi.test.ts | 13 + packages/domain/src/http/v2/prometheus.ts | 210 ++++++++++++ .../query-engine/src/ch/queries/promql.ts | 69 ++++ packages/query-engine/src/promql/discovery.ts | 89 +++++ packages/query-engine/src/promql/index.ts | 6 + 20 files changed, 1520 insertions(+), 3 deletions(-) create mode 100644 apps/api/src/routes/v2/prometheus-wire.test.ts create mode 100644 apps/api/src/routes/v2/prometheus-wire.ts create mode 100644 apps/api/src/routes/v2/prometheus.http.test.ts create mode 100644 apps/api/src/routes/v2/prometheus.http.ts create mode 100644 apps/landing/src/content/docs/reference/prometheus.md create mode 100644 packages/domain/src/http/v2/prometheus.ts create mode 100644 packages/query-engine/src/promql/discovery.ts diff --git a/apps/api/package.json b/apps/api/package.json index 967aa30031..358a1dba4e 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -31,6 +31,7 @@ "@maple/domain": "workspace:*", "@maple/infra": "workspace:*", "@maple/primitives": "workspace:*", + "@maple/promql": "workspace:*", "@maple/query-engine": "workspace:*", "@maple/query-engine-integrations": "workspace:*", "@maple/widgets": "workspace:*", diff --git a/apps/api/src/routes/v2/prometheus-wire.test.ts b/apps/api/src/routes/v2/prometheus-wire.test.ts new file mode 100644 index 0000000000..275b0468b0 --- /dev/null +++ b/apps/api/src/routes/v2/prometheus-wire.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "@effect/vitest" +import { Effect, String as Str } from "effect" +import { formatValue, parseStep, parseTime } from "./prometheus-wire" + +describe("Prometheus wire format", () => { + it("formats floats as Go's FormatFloat(v, 'f', -1, 64)", () => { + expect(formatValue(1)).toBe("1") + expect(formatValue(-0)).toBe("-0") + expect(formatValue(1.5e-7)).toBe("0.00000015") + expect(formatValue(-2.5e22)).toBe("-25000000000000000000000") + expect(formatValue(1.7976931348623157e308)).toMatch(/^179769313486231570+$/) + expect(formatValue(5e-324)).toBe(`0.${Str.repeat(323)("0")}5`) + }) + + it.effect("parses Prometheus timestamps and steps to epoch ms", () => + Effect.gen(function* () { + expect(yield* parseTime("time", "1752580800.123")).toBe(1752580800123) + expect(yield* parseTime("time", "2026-07-15T12:00:00.25+02:00")).toBe( + Date.UTC(2026, 6, 15, 10, 0, 0, 250), + ) + expect(yield* parseStep("1m30s")).toBe(90_000) + expect(yield* parseStep("0.25")).toBe(250) + const invalid = yield* Effect.flip(parseTime("start", "2026-07-15")) + expect(invalid.message).toBe( + 'invalid parameter "start": cannot parse "2026-07-15" to a valid timestamp', + ) + }), + ) +}) diff --git a/apps/api/src/routes/v2/prometheus-wire.ts b/apps/api/src/routes/v2/prometheus-wire.ts new file mode 100644 index 0000000000..11c78ec880 --- /dev/null +++ b/apps/api/src/routes/v2/prometheus-wire.ts @@ -0,0 +1,110 @@ +// Prometheus' HTTP API wire format: request parameter parsing and the exact JSON bodies +// (`[, ""]` samples, Go float formatting, the error envelope). + +import { DateTime, Effect, Option, Array as Arr, String as Str } from "effect" +import { parseDurationMs, type QueryResult } from "@maple/promql" +import { PromqlBadQueryError } from "@maple/backend/services/promql/PromqlQueryService" + +const FLOAT = /^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?$/ +const RFC3339 = /^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$/ + +/** A Prometheus timestamp: Unix seconds with an optional fraction, or RFC3339. Epoch ms. */ +export const parseTime = (name: string, text: string): Effect.Effect => { + if (FLOAT.test(text)) { + const ms = Math.round(Number(text) * 1000) + if (Number.isFinite(ms)) return Effect.succeed(ms) + } else if (RFC3339.test(text)) { + const parsed = DateTime.make(text) + if (Option.isSome(parsed)) return Effect.succeed(DateTime.toEpochMillis(parsed.value)) + } + return Effect.fail( + new PromqlBadQueryError({ + message: `invalid parameter "${name}": cannot parse "${text}" to a valid timestamp`, + }), + ) +} + +/** A Prometheus duration (`30s`, `1m30s`) or float seconds. Epoch ms. */ +export const parseStep = (text: string): Effect.Effect => { + const ms = FLOAT.test(text) ? Math.round(Number(text) * 1000) : parseDurationMs(text) + return ms !== undefined && Number.isFinite(ms) + ? Effect.succeed(ms) + : Effect.fail( + new PromqlBadQueryError({ + message: `invalid parameter "step": cannot parse "${text}" to a valid duration`, + }), + ) +} + +/** Moves the decimal point of `digits` (a plain decimal, no sign) by `exponent` places. */ +const shiftPoint = (digits: string, exponent: number): string => { + const [whole, fraction = ""] = Str.split(digits, ".") + const all = whole + fraction + const point = whole.length + exponent + if (point <= 0) return `0.${Str.repeat(-point)("0")}${all}` + if (point >= all.length) return all + Str.repeat(point - all.length)("0") + return `${Str.slice(0, point)(all)}.${Str.slice(point)(all)}` +} + +/** Go `strconv.FormatFloat(v, 'f', -1, 64)`: shortest round-trip digits, never an exponent. */ +export const formatValue = (value: number): string => { + if (Number.isNaN(value)) return "NaN" + if (value === Infinity) return "+Inf" + if (value === -Infinity) return "-Inf" + if (Object.is(value, -0)) return "-0" + const text = String(value) + const [mantissa, exponent] = Str.split(text, "e") + if (exponent === undefined) return text + const negative = Str.startsWith("-")(mantissa) + const shifted = shiftPoint(negative ? Str.slice(1)(mantissa) : mantissa, Number(exponent)) + return negative ? `-${shifted}` : shifted +} + +type WirePoint = readonly [number, string] + +const point = (t: number, v: number): WirePoint => [t / 1000, formatValue(v)] + +/** The `data` of a query response. */ +export const encodeQueryResult = (result: QueryResult) => { + switch (result.resultType) { + case "matrix": + return { + resultType: result.resultType, + result: Arr.map(result.result, (series) => ({ + metric: series.labels, + values: Arr.map(series.points, (p) => point(p.t, p.v)), + })), + } + case "vector": + return { + resultType: result.resultType, + result: Arr.map(result.result, (sample) => ({ + metric: sample.labels, + value: point(sample.t, sample.value), + })), + } + case "scalar": + return { resultType: result.resultType, result: point(result.result.t, result.result.v) } + case "string": + return { resultType: result.resultType, result: [result.result.t / 1000, result.result.v] } + } +} + +export type PrometheusErrorType = "bad_data" | "execution" | "timeout" | "unavailable" + +const ERROR_STATUS = { + bad_data: 400, + execution: 422, + timeout: 503, + unavailable: 503, +} satisfies Record + +export const errorStatus = (type: PrometheusErrorType): number => ERROR_STATUS[type] + +export const errorBody = (type: PrometheusErrorType, error: string) => ({ + status: "error" as const, + errorType: type, + error, +}) + +export const successBody = (data: A) => ({ status: "success" as const, data }) diff --git a/apps/api/src/routes/v2/prometheus.http.test.ts b/apps/api/src/routes/v2/prometheus.http.test.ts new file mode 100644 index 0000000000..6a7da9f622 --- /dev/null +++ b/apps/api/src/routes/v2/prometheus.http.test.ts @@ -0,0 +1,320 @@ +import { afterEach, describe, expect, it } from "@effect/vitest" +import { OrgId, UserId, WarehouseQueryError } from "@maple/domain/http" +import { MapleApiV2 } from "@maple/domain/http/v2" +import { ConfigProvider, Context, Effect, Layer, ManagedRuntime, Schema, String as Str } from "effect" +import { HttpRouter } from "effect/http" +import { HttpApi, HttpApiBuilder } from "effect/http-api" +import { Env } from "@maple/backend/platform/Env" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/testing/test-pglite" +import { + WarehouseQueryService, + type WarehouseQueryServiceApi, +} from "@maple/backend/services/warehouse/WarehouseQueryService" +import { ApiAuthorizationV2Live } from "@maple/backend/services/auth/ApiAuthorizationV2Live" +import { AuditLogService } from "@maple/backend/services/audit/AuditLogService" +import { ApiKeysService } from "@maple/backend/services/org/ApiKeysService" +import { AuthService } from "@maple/backend/services/auth/AuthService" +import { PromqlQueryService } from "@maple/backend/services/promql/PromqlQueryService" +import { compiledQueryOf } from "@maple/query-engine/execution" +import { V2TransportErrorBoundaryLive } from "./error-envelope" +import { ApiV2RateLimiterAllowAllLayer, makeWarehouseServiceStub } from "./v2-test-support" +import { HttpV2PrometheusLive } from "./prometheus.http" + +const createdDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(createdDbs)) + +const BASE = "/v2/metrics/prometheus/api/v1" + +const testConfig = () => + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3490", + MCP_PORT: "3491", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 1).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + INTERNAL_SERVICE_TOKEN: "test-internal-token", + }), + ) + +/** The metric catalog holds one counter; any other read is empty. Selector reads are size-bounded. */ +const catalogWarehouse = makeWarehouseServiceStub({ + compiledQuery: (_tenant, compiled) => { + const query = compiledQueryOf(compiled) + const catalog = + Str.includes("FROM metric_catalog")(query.sql) && !Str.includes("AS serviceName")(query.sql) + return query.decodeRows( + catalog ? [{ metricName: "http.requests", metricType: "sum", unit: "", isMonotonic: 1 }] : [], + ) + }, + compiledQueryBounded: (_tenant, compiled) => { + const query = compiledQueryOf(compiled) + const catalog = + Str.includes("FROM metric_catalog")(query.sql) && !Str.includes("AS serviceName")(query.sql) + // The bounded read's error union has no decode failure; the fixture rows always decode. + return query + .decodeRows( + catalog ? [{ metricName: "http.requests", metricType: "sum", unit: "", isMonotonic: 1 }] : [], + ) + .pipe(Effect.orDie) + }, +}) + +const failingWarehouse = makeWarehouseServiceStub({ + compiledQuery: () => + Effect.fail(new WarehouseQueryError({ message: "SECRET_DIAGNOSTIC", pipeName: "stub" })), +}) + +const makeHarness = (warehouse: WarehouseQueryServiceApi = catalogWarehouse) => { + const testDb = createTestDb(createdDbs) + const envLive = Env.layer.pipe(Layer.provide(testConfig())) + const servicesLive = Layer.mergeAll(ApiKeysService.layer, AuthService.layer).pipe( + Layer.provideMerge(Layer.mergeAll(envLive, testDb.layer)), + ) + const routes = HttpApiBuilder.layer(HttpApi.make("MapleApiV2").add(MapleApiV2.groups.prometheus)).pipe( + Layer.provide(HttpV2PrometheusLive), + Layer.provide( + Layer.effect(PromqlQueryService, PromqlQueryService.make).pipe( + Layer.provide(Layer.succeed(WarehouseQueryService, warehouse)), + ), + ), + Layer.provide(V2TransportErrorBoundaryLive), + Layer.provideMerge(ApiAuthorizationV2Live), + Layer.provideMerge(AuditLogService.layerMemory), + Layer.provideMerge(ApiV2RateLimiterAllowAllLayer), + Layer.provideMerge(servicesLive), + ) + const { handler, dispose: disposeHandler } = HttpRouter.toWebHandler(routes, { disableLogger: true }) + const runtime = ManagedRuntime.make(servicesLive) + const org = Schema.decodeSync(OrgId)("org_prometheus_e2e") + const user = Schema.decodeSync(UserId)("user_prometheus_e2e") + const bootstrapKey = (scopes?: ReadonlyArray, roles?: ReadonlyArray) => + runtime.runPromise( + Effect.gen(function* () { + const service = yield* ApiKeysService + return yield* service.create(org, user, { + name: "prometheus-test", + scopes, + ...(roles === undefined + ? undefined + : { metadataJson: { source: "maple_cli", roles, deviceName: "laptop" } }), + }) + }), + ) + const send = async (token: string, path: string, form?: URLSearchParams) => { + const response = await handler( + new Request(`http://maple.test${BASE}${path}`, { + method: form === undefined ? "GET" : "POST", + headers: { + authorization: `Bearer ${token}`, + ...(form === undefined + ? undefined + : { "content-type": "application/x-www-form-urlencoded" }), + }, + body: form?.toString(), + }), + Context.empty() as never, + ) + return { status: response.status, body: await response.json() } + } + return { + bootstrapKey, + send, + dispose: async () => { + await disposeHandler() + await runtime.dispose() + }, + } +} + +const T = Date.UTC(2026, 6, 15, 12) / 1000 + +describe("v2 Prometheus query API", () => { + it("answers an instant query at a Unix-seconds time with Go-formatted values", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(["metrics:read"]) + const vector = await harness.send(key.secret, `/query?query=vector(1e21)&time=${T}.5`) + expect(vector).toEqual({ + status: 200, + body: { + status: "success", + data: { + resultType: "vector", + result: [{ metric: {}, value: [T + 0.5, "1000000000000000000000"] }], + }, + }, + }) + for (const [query, value] of [ + ["1/0", "+Inf"], + ["-1/0", "-Inf"], + ["0/0", "NaN"], + ["0.1+0.2", "0.30000000000000004"], + ["1e-7", "0.0000001"], + ] as const) { + const scalar = await harness.send( + key.secret, + `/query?query=${encodeURIComponent(query)}&time=${T}`, + ) + expect(scalar.body).toEqual({ + status: "success", + data: { resultType: "scalar", result: [T, value] }, + }) + } + await harness.dispose() + }) + + it("answers a range query POSTed as a form with RFC3339 bounds and a duration step", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(["metrics:read"]) + const form = new URLSearchParams({ + query: "vector(2)", + start: "2026-07-15T12:00:00Z", + end: "2026-07-15T12:01:00Z", + step: "30s", + }) + const range = await harness.send(key.secret, "/query_range", form) + expect(range).toEqual({ + status: 200, + body: { + status: "success", + data: { + resultType: "matrix", + result: [ + { + metric: {}, + values: [ + [T, "2"], + [T + 30, "2"], + [T + 60, "2"], + ], + }, + ], + }, + }, + }) + const floatStep = await harness.send( + key.secret, + `/query_range?query=vector(1)&start=${T}&end=${T + 1}&step=0.5`, + ) + expect(floatStep.body).toMatchObject({ + data: { + result: [ + { + values: [ + [T, "1"], + [T + 0.5, "1"], + [T + 1, "1"], + ], + }, + ], + }, + }) + await harness.dispose() + }) + + it("reports errors in Prometheus' envelope with its status codes", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(["metrics:read"]) + const syntax = await harness.send(key.secret, `/query?query=${encodeURIComponent("sum(")}&time=${T}`) + expect(syntax.status).toBe(400) + expect(syntax.body).toMatchObject({ status: "error", errorType: "bad_data" }) + const badTime = await harness.send(key.secret, "/query?query=1&time=yesterday") + expect(badTime).toEqual({ + status: 400, + body: { + status: "error", + errorType: "bad_data", + error: 'invalid parameter "time": cannot parse "yesterday" to a valid timestamp', + }, + }) + const badStep = await harness.send( + key.secret, + `/query_range?query=1&start=${T}&end=${T + 60}&step=soon`, + ) + expect(badStep.body).toMatchObject({ + error: 'invalid parameter "step": cannot parse "soon" to a valid duration', + }) + const execution = await harness.send( + key.secret, + `/query?query=${encodeURIComponent("topk(NaN, vector(1))")}&time=${T}`, + ) + expect(execution.status).toBe(422) + expect(execution.body).toMatchObject({ status: "error", errorType: "execution" }) + await harness.dispose() + }) + + it("reports a warehouse failure as unavailable without its diagnostic", async () => { + const harness = makeHarness(failingWarehouse) + const key = await harness.bootstrapKey(["metrics:read"]) + const failed = await harness.send(key.secret, `/query?query=up&time=${T}`) + expect(failed.status).toBe(503) + expect(failed.body).toMatchObject({ status: "error", errorType: "unavailable" }) + expect(JSON.stringify(failed.body)).not.toContain("SECRET_DIAGNOSTIC") + await harness.dispose() + }) +}) + +describe("v2 Prometheus discovery API", () => { + it("lists metric names, label names and build info the way Grafana reads them", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(["metrics:read"]) + const window = `start=${T - 3600}&end=${T}` + expect((await harness.send(key.secret, `/label/__name__/values?${window}`)).body).toEqual({ + status: "success", + data: ["http_requests_total"], + }) + expect((await harness.send(key.secret, "/labels", new URLSearchParams(window))).body).toEqual({ + status: "success", + data: ["__name__", "job"], + }) + const buildInfo = await harness.send(key.secret, "/status/buildinfo") + expect(buildInfo.body).toMatchObject({ data: { version: "3.5.0", vendor: "Maple" } }) + expect( + (await harness.send(key.secret, "/query_exemplars", new URLSearchParams({ query: "up" }))).body, + ).toEqual({ + status: "success", + data: [], + }) + await harness.dispose() + }) + + it("reads repeated match[] parameters and refuses a series call without one", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(["metrics:read"]) + const form = new URLSearchParams([ + ["match[]", "http_requests_total"], + ["match[]", 'up{job="api"}'], + ["start", String(T - 60)], + ["end", String(T)], + ]) + expect((await harness.send(key.secret, "/series", form)).body).toEqual({ + status: "success", + data: [], + }) + const missing = await harness.send(key.secret, "/series") + expect(missing.status).toBe(400) + expect(missing.body).toEqual({ + status: "error", + errorType: "bad_data", + error: "no match[] parameter provided", + }) + await harness.dispose() + }) + + it("refuses a key without metrics:read on GET and form POST, and serves an analyst's POST", async () => { + const harness = makeHarness() + const traces = await harness.bootstrapKey(["traces:read"]) + expect((await harness.send(traces.secret, `/query?query=1&time=${T}`)).status).toBe(403) + const form = new URLSearchParams({ query: "1", time: String(T) }) + expect((await harness.send(traces.secret, "/query", form)).status).toBe(403) + const analyst = await harness.bootstrapKey(undefined, ["org:analyst"]) + const answered = await harness.send(analyst.secret, "/query", form) + expect(answered.status).toBe(200) + expect(answered.body).toEqual({ status: "success", data: { resultType: "scalar", result: [T, "1"] } }) + await harness.dispose() + }) +}) diff --git a/apps/api/src/routes/v2/prometheus.http.ts b/apps/api/src/routes/v2/prometheus.http.ts new file mode 100644 index 0000000000..d8b0887051 --- /dev/null +++ b/apps/api/src/routes/v2/prometheus.http.ts @@ -0,0 +1,187 @@ +// The Prometheus HTTP API over Maple metrics. Handlers answer with raw responses so the body is +// Prometheus' own JSON, errors included; only authentication keeps the v2 envelope. + +import { HttpApiBuilder } from "effect/http-api" +import { HttpServerResponse } from "effect/http" +import { Clock, Effect, Array as Arr, Record as Rec } from "effect" +import type { QueryResult } from "@maple/promql" +import { CurrentTenant } from "@maple/domain/http" +import { MapleApiV2, type PrometheusParams } from "@maple/domain/http/v2" +import { PromqlBadQueryError, PromqlQueryService } from "@maple/backend/services/promql/PromqlQueryService" +import { + encodeQueryResult, + errorBody, + errorStatus, + type PrometheusErrorType, + parseStep, + parseTime, + successBody, +} from "./prometheus-wire" + +type Params = typeof PrometheusParams.Type + +/** The version Grafana reads to enable Prometheus 3 features. */ +export const PROMETHEUS_COMPAT_VERSION = "3.5.0" + +const HOUR_MS = 3_600_000 + +const BUILD_INFO = { + version: PROMETHEUS_COMPAT_VERSION, + revision: "", + branch: "", + buildUser: "", + buildDate: "", + goVersion: "", + vendor: "Maple", +} + +const seriesCount = (result: QueryResult): number => + result.resultType === "matrix" || result.resultType === "vector" ? result.result.length : 1 + +const limited = (values: ReadonlyArray, limit: number | undefined) => + limit === undefined ? values : Arr.take(values, limit) + +/** Prometheus merges URL and form parameters; the body wins, and `match[]` lists add up. */ +const merged = (query: Params, payload: Params): Params => ({ + ...query, + ...payload, + "match[]": [...(query["match[]"] ?? []), ...(payload["match[]"] ?? [])], +}) + +const errorResponse = (type: PrometheusErrorType, message: string) => + Effect.as( + Effect.annotateCurrentSpan({ "maple.promql.error_type": type }), + HttpServerResponse.jsonUnsafe(errorBody(type, message), { status: errorStatus(type) }), + ) + +/** Success as `{status:"success",data}`; query errors as Prometheus' error envelope. */ +const respond = ( + effect: Effect.Effect, +) => + effect.pipe( + Effect.map((data) => HttpServerResponse.jsonUnsafe(successBody(data))), + Effect.catch((error) => { + switch (error._tag) { + case "@maple/backend/promql/PromqlBadQueryError": + return errorResponse("bad_data", error.message) + case "@maple/backend/promql/PromqlExecutionError": + return errorResponse("execution", error.message) + default: + return Effect.andThen( + Effect.logWarning("Prometheus API warehouse read failed", error), + errorResponse("unavailable", "the metric store is unavailable, try again"), + ) + } + }), + ) + +const parseLimit = (text: string | undefined) => { + if (text === undefined || text === "") return Effect.undefined + const limit = Number(text) + return Number.isInteger(limit) && limit >= 0 + ? Effect.succeed(limit === 0 ? undefined : limit) + : Effect.fail( + new PromqlBadQueryError({ message: `invalid parameter "limit": cannot parse "${text}"` }), + ) +} + +/** `[start, end]` of a discovery call; missing bounds mean the last `defaultMs` up to now. */ +const discoveryWindow = (params: Params, defaultMs: number) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis + const endMs = params.end === undefined ? now : yield* parseTime("end", params.end) + const startMs = + params.start === undefined ? endMs - defaultMs : yield* parseTime("start", params.start) + return { startMs, endMs } + }) + +export const HttpV2PrometheusLive = HttpApiBuilder.group(MapleApiV2, "prometheus", (handlers) => + Effect.gen(function* () { + const promql = yield* PromqlQueryService + + const instantQuery = Effect.fn("PrometheusApi.query")(function* (params: Params) { + const tenant = yield* CurrentTenant.Context + const now = yield* Clock.currentTimeMillis + const timeMs = params.time === undefined ? now : yield* parseTime("time", params.time) + const result = yield* promql.query(tenant, { + query: params.query ?? "", + startMs: timeMs, + endMs: timeMs, + stepMs: 0, + }) + yield* Effect.annotateCurrentSpan({ + "maple.promql.result_type": result.resultType, + "maple.promql.result_series": seriesCount(result), + }) + return encodeQueryResult(result) + }) + + const rangeQuery = Effect.fn("PrometheusApi.queryRange")(function* (params: Params) { + const tenant = yield* CurrentTenant.Context + const startMs = yield* parseTime("start", params.start ?? "") + const endMs = yield* parseTime("end", params.end ?? "") + const stepMs = yield* parseStep(params.step ?? "") + const result = yield* promql.query(tenant, { + query: params.query ?? "", + startMs, + endMs, + stepMs, + }) + yield* Effect.annotateCurrentSpan({ + "maple.promql.result_type": result.resultType, + "maple.promql.result_series": seriesCount(result), + }) + return encodeQueryResult(result) + }) + + const discovery = (params: Params) => + Effect.gen(function* () { + const window = yield* discoveryWindow(params, HOUR_MS) + const limit = yield* parseLimit(params.limit) + return { ...window, matches: params["match[]"] ?? [], limit } + }) + + const series = Effect.fn("PrometheusApi.series")(function* (params: Params) { + const tenant = yield* CurrentTenant.Context + const input = yield* discovery(params) + return yield* promql.series(tenant, input) + }) + + const labels = Effect.fn("PrometheusApi.labels")(function* (params: Params) { + const tenant = yield* CurrentTenant.Context + const input = yield* discovery(params) + const names = yield* promql.labelNames(tenant, input) + return limited(names, input.limit) + }) + + const labelValues = Effect.fn("PrometheusApi.labelValues")(function* (name: string, params: Params) { + const tenant = yield* CurrentTenant.Context + const input = yield* discovery(params) + const values = yield* promql.labelValues(tenant, { ...input, name }) + return limited(values, input.limit) + }) + + const metadata = Effect.fn("PrometheusApi.metadata")(function* (params: Params) { + const tenant = yield* CurrentTenant.Context + const window = yield* discoveryWindow(params, 24 * HOUR_MS) + const limit = yield* parseLimit(params.limit) + const found = yield* promql.metadata(tenant, { ...window, metric: params.metric, limit }) + return Rec.fromEntries([...found]) + }) + + return handlers + .handle("query", ({ query }) => respond(instantQuery(query))) + .handle("queryPost", ({ query, payload }) => respond(instantQuery(merged(query, payload)))) + .handle("queryRange", ({ query }) => respond(rangeQuery(query))) + .handle("queryRangePost", ({ query, payload }) => respond(rangeQuery(merged(query, payload)))) + .handle("series", ({ query }) => respond(series(query))) + .handle("seriesPost", ({ query, payload }) => respond(series(merged(query, payload)))) + .handle("labels", ({ query }) => respond(labels(query))) + .handle("labelsPost", ({ query, payload }) => respond(labels(merged(query, payload)))) + .handle("labelValues", ({ params, query }) => respond(labelValues(params.name, query))) + .handle("metadata", ({ query }) => respond(metadata(query))) + .handle("buildInfo", () => respond(Effect.succeed(BUILD_INFO))) + .handle("exemplars", () => respond(Effect.succeed([]))) + .handle("exemplarsPost", () => respond(Effect.succeed([]))) + }), +) diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index 559bc22b98..6d335acbce 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -102,6 +102,7 @@ import { HttpV2TracesLive, } from "@/routes/v2/telemetry.http" import { HttpV2WidgetSummaryLive } from "@/routes/v2/widget-summary.http" +import { HttpV2PrometheusLive } from "@/routes/v2/prometheus.http" import { HttpV2WidgetCredentialsLive } from "@/routes/v2/widget-credentials.http" import { HttpV2DeviceCredentialsLive, HttpV2DeviceTokensLive } from "@/routes/v2/device-credentials.http" import { DeviceCredentialsService } from "@maple/backend/services/auth/DeviceCredentialsService" @@ -249,6 +250,7 @@ const ApiV2Routes = HttpApiBuilder.layer(MapleApiV2).pipe( HttpV2TracesLive, HttpV2LogsLive, HttpV2MetricsLive, + HttpV2PrometheusLive, HttpV2ServicesLive, HttpV2ServiceMapLive, HttpV2EnvironmentsLive, diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index ebbc6eb7ef..ef8e2be5c9 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -138,6 +138,7 @@ import { AuditDetectionsService } from "@maple/backend/services/audit-trail/Audi import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" import { QueryEngineService } from "@maple/backend/services/warehouse/QueryEngineService" import { TelemetryReadService } from "@maple/backend/services/warehouse/TelemetryReadService" +import { PromqlQueryService } from "@maple/backend/services/promql/PromqlQueryService" import { OrgClickHouseSettingsService } from "@maple/backend/services/org/OrgClickHouseSettingsService" import { VcsSourceService } from "@maple/backend/services/integrations/vcs/VcsSourceService" import { CiAnalyticsService } from "@maple/backend/services/pr-review/CiAnalyticsService" @@ -254,6 +255,7 @@ export const HttpServicesLive = Layer.mergeAll( RuntimeAdvisoryCatalog.layer, DependencyScanService.layer, TelemetryReadService.layer, + PromqlQueryService.layer, DashboardWidgetDataService.layer, AlertDestinationsService.layer, AlertReadModelsService.layer, diff --git a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json index 99dfdea4ae..2ca50d523d 100644 --- a/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json +++ b/apps/ios/Packages/MapleAPI/Sources/MapleAPI/openapi.json @@ -14123,6 +14123,10 @@ "description": "Browse and query OpenTelemetry metrics.", "name": "Metrics" }, + { + "description": "The Prometheus HTTP query API over Maple metrics, for Grafana and other Prometheus clients.", + "name": "Prometheus" + }, { "description": "Observed service health summaries.", "name": "Services" diff --git a/apps/landing/src/content/docs/reference/prometheus.md b/apps/landing/src/content/docs/reference/prometheus.md new file mode 100644 index 0000000000..89e3cab4e9 --- /dev/null +++ b/apps/landing/src/content/docs/reference/prometheus.md @@ -0,0 +1,56 @@ +--- +title: "Prometheus API and Grafana" +description: "Query Maple metrics with PromQL through the Prometheus HTTP API, and add Maple to Grafana as a Prometheus data source." +group: "Reference" +order: 7.5 +navLabel: "Prometheus API" +--- + +Maple serves the Prometheus HTTP query API over your metrics. Grafana, and any other tool that speaks to Prometheus, can query Maple with PromQL. + +| | | +| -------------- | ------------------------------------------------ | +| URL (US) | `https://api.maple.dev/v2/metrics/prometheus` | +| URL (EU) | `https://api.eu.maple.dev/v2/metrics/prometheus` | +| Auth | `Authorization: Bearer maple_ak_…` | +| Required scope | `metrics:read` | + +## Connect Grafana + +1. Create an [API key](/docs/reference/authentication) with the `metrics:read` scope. +2. In Grafana, go to **Connections > Data sources > Add data source** and pick **Prometheus**. +3. Set **Prometheus server URL** to `https://api.maple.dev/v2/metrics/prometheus` (or the EU URL). +4. Under **HTTP headers**, add a header named `Authorization` with the value `Bearer `. +5. Leave **HTTP method** on POST, or pick GET. Both work. +6. Click **Save & test**. + +Grafana's metric browser, label filters and query builder read the same API, so metric and label names autocomplete. + +## Metric and label names + +OpenTelemetry metrics get the names the Prometheus OTLP translation gives them: dots become underscores, the unit is added as a suffix, and monotonic sums end in `_total`. A sum `http.server.requests` is `http_server_requests_total`, and a histogram `http.server.duration` in seconds is `http_server_duration_seconds_bucket`, `_count` and `_sum`. The stored OpenTelemetry name also works. + +Attributes become labels the same way (`http.route` is `http_route`). `job` is the service name, prefixed with `service.namespace/` when the service sets one, and `instance` is `service.instance.id`. + +## Endpoints + +All paths are under `/v2/metrics/prometheus/api/v1/` and follow Prometheus' parameters and response format. + +| Endpoint | Methods | +| --------------------- | --------- | +| `query` | GET, POST | +| `query_range` | GET, POST | +| `series` | GET, POST | +| `labels` | GET, POST | +| `label//values` | GET | +| `metadata` | GET | +| `query_exemplars` | GET, POST | +| `status/buildinfo` | GET | + +POST takes an `application/x-www-form-urlencoded` body. Exemplar queries return an empty list. A range query returns at most 11,000 points per series, as in Prometheus. + +```bash +curl -H "Authorization: Bearer $MAPLE_API_KEY" \ + "https://api.maple.dev/v2/metrics/prometheus/api/v1/query" \ + --data-urlencode 'query=sum by (job) (rate(http_server_requests_total[5m]))' +``` diff --git a/bun.lock b/bun.lock index c764a29d90..44c78d663e 100644 --- a/bun.lock +++ b/bun.lock @@ -107,6 +107,7 @@ "@maple/domain": "workspace:*", "@maple/infra": "workspace:*", "@maple/primitives": "workspace:*", + "@maple/promql": "workspace:*", "@maple/query-engine": "workspace:*", "@maple/query-engine-integrations": "workspace:*", "@maple/widgets": "workspace:*", diff --git a/packages/backend/src/services/promql/PromqlQueryService.test.ts b/packages/backend/src/services/promql/PromqlQueryService.test.ts index 1549713ad2..8c1126894c 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.test.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "@effect/vitest" -import { Effect, Layer, Schema } from "effect" +import { Effect, Layer, Option, Schema, Array as Arr, Record as Rec, String as Str } from "effect" +import { compiledQueryOf } from "@maple/query-engine/execution" import type { TenantContext } from "@maple/domain/http" import { OrgId, UserId } from "@maple/primitives" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" @@ -127,3 +128,177 @@ describe("PromqlQueryService", () => { }), ) }) + +const T0 = Date.UTC(2026, 6, 15, 12) + +const selectorRow = (attributes: Record) => ({ + series: JSON.stringify(attributes), + metricName: "http.requests", + serviceName: "api", + attributes, + resourceAttributes: { "service.name": "api" }, + le: "0", + ...Rec.fromEntries( + Arr.map( + ["firstT", "firstV", "lastT", "lastV", "prevT", "prevV", "count", "sum", "min", "max"], + (field) => [field, [1]] as const, + ), + ), + ...Rec.fromEntries( + Arr.map( + ["sumSquares", "increase", "resets", "changes", "sumT", "sumTT", "sumTV"], + (field) => [field, [0]] as const, + ), + ), + end: [T0], + values: [[]], +}) + +/** Answers each warehouse read by the table it reads, recording the SQL that ran. */ +const recordingWarehouse = () => { + const statements: Array = [] + const rowsFor = (sql: string): ReadonlyArray> => { + if (Str.includes("FROM metrics_sum")(sql)) { + return [selectorRow({ "http.route": "/checkout" }), selectorRow({ "http.route": "/cart" })] + } + if (Str.includes("MetricDescription")(sql)) { + return [ + { + metricName: "http.requests", + metricType: "sum", + unit: "1", + description: "Requests", + isMonotonic: 1, + }, + { + metricName: "http.duration", + metricType: "histogram", + unit: "s", + description: "Latency", + isMonotonic: 0, + }, + ] + } + if (Str.includes("AS serviceName")(sql)) return [{ serviceName: "api" }, { serviceName: "worker" }] + if (Str.includes("FROM metric_catalog")(sql)) { + return [{ metricName: "http.requests", metricType: "sum", unit: "", isMonotonic: 1 }] + } + if (Str.includes("FROM attribute_keys_hourly")(sql)) { + return [ + { attributeKey: "http.route", usageCount: 10 }, + { attributeKey: "http_route", usageCount: 1 }, + ] + } + if (Str.includes("FROM attribute_values_hourly")(sql)) return [{ attributeValue: "/checkout" }] + return [] + } + const warehouse = makeWarehouseServiceStub({ + compiledQuery: (_tenant, compiled) => { + const query = compiledQueryOf(compiled) + statements.push(query.sql) + return query.decodeRows(rowsFor(query.sql)) + }, + // Selector reads are size-bounded; record them the same way. + compiledQueryBounded: (_tenant, compiled) => { + const query = compiledQueryOf(compiled) + statements.push(query.sql) + return query.decodeRows(rowsFor(query.sql)) + }, + }) + const layer = Layer.effect(PromqlQueryService, PromqlQueryService.make).pipe( + Layer.provide(Layer.succeed(WarehouseQueryService, warehouse)), + ) + return { statements, layer } +} + +const window = { startMs: T0 - 3_600_000, endMs: T0 } + +const ORG_FILTER = /OrgId = 'org_promql'/ + +describe("PromqlQueryService discovery", () => { + it.effect("returns the label sets a match[] selector selects, read through org-scoped queries", () => { + const { statements, layer } = recordingWarehouse() + return Effect.gen(function* () { + const service = yield* PromqlQueryService + const series = yield* service.series(tenant, { + ...window, + matches: ['http_requests_total{http_route=~"/c.*"}'], + }) + expect(series).toEqual([ + { __name__: "http_requests_total", http_route: "/checkout", job: "api" }, + { __name__: "http_requests_total", http_route: "/cart", job: "api" }, + ]) + expect(statements.length).toBe(2) + for (const sql of statements) expect(sql).toMatch(ORG_FILTER) + }).pipe(Effect.provide(layer)) + }) + + it.effect("refuses a match[] that is not a selector or matches everything", () => + Effect.gen(function* () { + const service = yield* PromqlQueryService + const notSelector = yield* Effect.flip( + service.series(tenant, { ...window, matches: ["sum(up)"] }), + ) + expect(notSelector.message).toMatch(/not a series selector/) + const empty = yield* Effect.flip(service.series(tenant, { ...window, matches: ['{job=~".*"}'] })) + expect(empty.message).toMatch(/at least one non-empty matcher/) + }).pipe(Effect.provide(TestLayer)), + ) + + it.effect("lists label names from the catalog and the attribute rollup without a selector", () => { + const { statements, layer } = recordingWarehouse() + return Effect.gen(function* () { + const service = yield* PromqlQueryService + const names = yield* service.labelNames(tenant, { ...window, matches: [] }) + expect(names).toEqual(["__name__", "http_route", "job"]) + expect(Arr.some(statements, (sql) => Str.includes("metrics_sum")(sql))).toBe(false) + for (const sql of statements) expect(sql).toMatch(ORG_FILTER) + }).pipe(Effect.provide(layer)) + }) + + it.effect("reads label values from the rollup for every raw key the label normalizes from", () => { + const { statements, layer } = recordingWarehouse() + return Effect.gen(function* () { + const service = yield* PromqlQueryService + const values = yield* service.labelValues(tenant, { ...window, matches: [], name: "http_route" }) + expect(values).toEqual(["/checkout"]) + const valuesSql = Arr.findFirst(statements, (sql) => Str.includes("attribute_values_hourly")(sql)) + expect(Option.getOrThrow(valuesSql)).toMatch(/AttributeKey IN \('http\.route', 'http_route'\)/) + expect(Option.getOrThrow(valuesSql)).toMatch(ORG_FILTER) + expect(Option.getOrThrow(valuesSql)).toMatch(/AttributeScope = 'metric'/) + }).pipe(Effect.provide(layer)) + }) + + it.effect("lists metric names and jobs from the catalog", () => { + const { layer } = recordingWarehouse() + return Effect.gen(function* () { + const service = yield* PromqlQueryService + const names = yield* service.labelValues(tenant, { ...window, matches: [], name: "__name__" }) + expect(names).toEqual(["http_requests_total"]) + const jobs = yield* service.labelValues(tenant, { ...window, matches: [], name: "job" }) + expect(jobs).toEqual(["api", "worker"]) + }).pipe(Effect.provide(layer)) + }) + + it.effect("answers metadata keyed by metric family", () => { + const { layer } = recordingWarehouse() + return Effect.gen(function* () { + const service = yield* PromqlQueryService + const metadata = yield* service.metadata(tenant, window) + expect(Rec.fromEntries(metadata)).toEqual({ + http_duration_seconds: [{ type: "histogram", help: "Latency", unit: "seconds" }], + http_requests_total: [{ type: "counter", help: "Requests", unit: "" }], + }) + }).pipe(Effect.provide(layer)) + }) + + it.effect("refuses an invalid label name", () => + Effect.gen(function* () { + const service = yield* PromqlQueryService + const error = yield* Effect.flip( + service.labelValues(tenant, { ...window, matches: [], name: "bad-name" }), + ) + expect(error.message).toMatch(/invalid label name/) + }).pipe(Effect.provide(TestLayer)), + ) +}) diff --git a/packages/backend/src/services/promql/PromqlQueryService.ts b/packages/backend/src/services/promql/PromqlQueryService.ts index 0a8b7779c1..d4d5dec86b 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.ts @@ -7,7 +7,7 @@ // step; every other selector returns pre-aggregated buckets the evaluator // recombines. -import { Context, Effect, Layer, Result, Schema, Array as Arr } from "effect" +import { Context, Effect, Layer, Option, Order, Result, Schema, Array as Arr, Record as Rec } from "effect" import { DEFAULT_LIMITS, evaluate, @@ -28,8 +28,16 @@ import { CH, formatWarehouseDateTime } from "@maple/query-engine" import { buildPushdownQuery, buildSelectorQueries, + canonicalPromNames, catalogFromRows, catalogQuery, + keysForLabel, + labelNamesFrom, + matchesValue, + prometheusMetadata, + promqlAttributeValuesQuery, + promqlMetadataQuery, + promqlServiceNamesQuery, type PushdownPlan, toPushdownData, toSelectorData, @@ -112,6 +120,75 @@ const validate = (query: PromqlQuery): Effect.Effect return parsed.success }) +/** A discovery call (series, labels, label values, metadata) over `[startMs, endMs]`. */ +export interface DiscoveryWindow { + readonly startMs: number + readonly endMs: number +} + +export interface DiscoveryQuery extends DiscoveryWindow { + /** `match[]` series selectors; empty reads the catalog and rollups instead. */ + readonly matches: ReadonlyArray + readonly limit?: number +} + +/** Names and values one discovery call returns at most. */ +export const DISCOVERY_LIMIT = 10_000 +const ATTRIBUTE_KEY_LIMIT = 1_000 +const LABEL_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/ + +const validateWindow = (window: DiscoveryWindow): Effect.Effect => + !Number.isFinite(window.startMs) || !Number.isFinite(window.endMs) || window.endMs < window.startMs + ? Effect.fail(new PromqlBadQueryError({ message: "end timestamp must not be before start time" })) + : Effect.succeed(window) + +/** One `match[]` value: a plain vector selector with at least one matcher that rejects "". */ +const parseMatch = (text: string): Effect.Effect => { + const parsed = parse(text) + if (Result.isFailure(parsed)) { + return Effect.fail( + new PromqlBadQueryError({ + message: `invalid parameter "match[]": ${parsed.failure.message}`, + position: parsed.failure.position, + }), + ) + } + const expr = parsed.success + if (expr.type !== "VectorSelector") { + return Effect.fail( + new PromqlBadQueryError({ + message: `invalid parameter "match[]": ${text} is not a series selector`, + }), + ) + } + if (!Arr.some(expr.matchers, (m) => !matchesValue(m, ""))) { + return Effect.fail( + new PromqlBadQueryError({ + message: "match[] must contain at least one non-empty matcher", + }), + ) + } + return Effect.succeed(expr) +} + +/** One bucket covering `[startMs, endMs]`: the cheapest read that still yields every label set. */ +const discoveryRequest = (selector: VectorSelector, window: DiscoveryWindow): SelectorRequest => ({ + metricName: selector.name, + matchers: selector.matchers, + startMs: window.startMs - 1, + endMs: window.endMs, + originMs: window.startMs - 1, + widthMs: window.endMs - window.startMs + 1, + needsValues: false, +}) + +/** A label set as one string, NUL-separated: label names never hold NUL, and values practically never do. */ +const seriesKey = (labels: Readonly>): string => + Arr.join( + Arr.map(Rec.toEntries(labels), ([name, value]) => `${name}\u0000${value}`), + "\u0000", + ) + const span = (requests: ReadonlyArray) => ({ startMs: Math.min(...Arr.map(requests, (r) => r.startMs)), endMs: Math.max(...Arr.map(requests, (r) => r.endMs)), @@ -253,7 +330,164 @@ const make = Effect.gen(function* () { return result.success satisfies QueryResult }) - return { query } + const windowParams = (tenant: TenantContext, window: DiscoveryWindow) => ({ + orgId: tenant.orgId, + startTime: formatWarehouseDateTime(window.startMs), + endTime: formatWarehouseDateTime(window.endMs), + }) + + const series = Effect.fn("PromqlQueryService.series")(function* ( + tenant: TenantContext, + input: DiscoveryQuery, + ) { + const window = yield* validateWindow(input) + if (input.matches.length === 0) { + return yield* new PromqlBadQueryError({ message: "no match[] parameter provided" }) + } + const selectors = yield* Effect.forEach(input.matches, parseMatch) + const catalog = yield* readCatalog(tenant, window) + const read = yield* Effect.forEach( + selectors, + (selector) => readSelector(tenant, discoveryRequest(selector, window), catalog), + { concurrency: WAREHOUSE_CONCURRENCY }, + ) + // seriesLabels sorts its keys, so the joined pairs are a stable identity. + const unique = new Map( + Arr.flatMap(read, (data) => + Arr.map(data.series, (s) => [seriesKey(s.labels), s.labels] as const), + ), + ) + const limit = Math.min(input.limit ?? DISCOVERY_LIMIT, DISCOVERY_LIMIT) + yield* Effect.annotateCurrentSpan({ + "maple.promql.matches": input.matches.length, + "maple.promql.range_ms": window.endMs - window.startMs, + "maple.promql.series": unique.size, + }) + return Arr.take([...unique.values()], limit) + }) + + const attributeKeys = Effect.fn("PromqlQueryService.attributeKeys")(function* ( + tenant: TenantContext, + window: DiscoveryWindow, + ) { + const compiled = CH.compile( + CH.attributeKeysQuery({ scope: "metric", limit: ATTRIBUTE_KEY_LIMIT }), + windowParams(tenant, window), + ) + const rows = yield* warehouse.compiledQuery(tenant, compiled, { + profile: "discovery", + context: "promqlLabelNames", + }) + return Arr.map(rows, (row) => row.attributeKey) + }) + + const sortedUnique = (values: ReadonlyArray) => + Arr.take(Arr.sort(Arr.dedupe(values), Order.String), DISCOVERY_LIMIT) + + const labelNames = Effect.fn("PromqlQueryService.labelNames")(function* ( + tenant: TenantContext, + input: DiscoveryQuery, + ) { + const window = yield* validateWindow(input) + if (input.matches.length > 0) { + const found = yield* series(tenant, input) + return sortedUnique(Arr.flatMap(found, (labels) => Rec.keys(labels))) + } + const [catalog, keys] = yield* Effect.all( + [readCatalog(tenant, window), attributeKeys(tenant, window)], + { concurrency: 2 }, + ) + return Arr.take(labelNamesFrom(catalog, keys), DISCOVERY_LIMIT) + }) + + const attributeValues = Effect.fn("PromqlQueryService.attributeValues")(function* ( + tenant: TenantContext, + window: DiscoveryWindow, + name: string, + ) { + const allKeys = yield* attributeKeys(tenant, window) + const keys = keysForLabel(allKeys, name) + if (keys.length === 0) return [] + const compiled = CH.compile( + promqlAttributeValuesQuery({ keys, limit: DISCOVERY_LIMIT }), + windowParams(tenant, window), + ) + const rows = yield* warehouse.compiledQuery(tenant, compiled, { + profile: "discovery", + context: "promqlLabelValues", + }) + return Arr.map(rows, (row) => row.attributeValue) + }) + + const serviceNames = Effect.fn("PromqlQueryService.serviceNames")(function* ( + tenant: TenantContext, + window: DiscoveryWindow, + ) { + const compiled = CH.compile( + promqlServiceNamesQuery({ limit: DISCOVERY_LIMIT }), + windowParams(tenant, window), + ) + const rows = yield* warehouse.compiledQuery(tenant, compiled, { + profile: "discovery", + context: "promqlJobs", + }) + return Arr.map(rows, (row) => row.serviceName) + }) + + const labelValues = Effect.fn("PromqlQueryService.labelValues")(function* ( + tenant: TenantContext, + input: DiscoveryQuery & { readonly name: string }, + ) { + if (!LABEL_NAME.test(input.name)) { + return yield* new PromqlBadQueryError({ message: `invalid label name: "${input.name}"` }) + } + const window = yield* validateWindow(input) + yield* Effect.annotateCurrentSpan({ "maple.promql.label": input.name }) + if (input.matches.length > 0) { + const found = yield* series(tenant, input) + return sortedUnique( + Arr.flatMap(found, (labels) => Option.toArray(Option.fromNullishOr(labels[input.name]))), + ) + } + switch (input.name) { + case "__name__": + return yield* Effect.map(readCatalog(tenant, window), (catalog) => + sortedUnique(canonicalPromNames(catalog)), + ) + case "job": + return yield* Effect.map(serviceNames(tenant, window), sortedUnique) + default: + return yield* Effect.map(attributeValues(tenant, window, input.name), sortedUnique) + } + }) + + const metadata = Effect.fn("PromqlQueryService.metadata")(function* ( + tenant: TenantContext, + input: DiscoveryWindow & { readonly metric?: string; readonly limit?: number }, + ) { + const window = yield* validateWindow(input) + const compiled = CH.compile(promqlMetadataQuery({ limit: 50_000 }), windowParams(tenant, window)) + const rows = yield* warehouse.compiledQuery(tenant, compiled, { + profile: "discovery", + context: "promqlMetadata", + }) + const entries = Arr.filter( + [...prometheusMetadata(rows)], + ([name]) => input.metric === undefined || input.metric === name, + ) + const limit = input.limit !== undefined && input.limit > 0 ? input.limit : entries.length + return new Map( + Arr.take( + Arr.sort( + entries, + Order.mapInput(Order.String, (entry: readonly [string, unknown]) => entry[0]), + ), + limit, + ), + ) + }) + + return { query, series, labelNames, labelValues, metadata } }) export class PromqlQueryService extends Context.Service()( diff --git a/packages/domain/src/http/v2/api.ts b/packages/domain/src/http/v2/api.ts index 48b82e168b..d29e798b1f 100644 --- a/packages/domain/src/http/v2/api.ts +++ b/packages/domain/src/http/v2/api.ts @@ -36,6 +36,7 @@ import { V2SlosApiGroup } from "./slos" import { V2UsersApiGroup } from "./users" import { V2StatusPageIncidentsApiGroup, V2StatusPagesApiGroup } from "./status-pages" import { V2InstrumentationAuditApiGroup } from "./setup-audit" +import { V2PrometheusApiGroup } from "./prometheus" import { V2SupportChannelApiGroup } from "./support-channel" import { V2TelemetrySignalsApiGroup } from "./telemetry-signals" import { V2SharePublicApiGroup } from "./share" @@ -146,6 +147,7 @@ export class MapleApiV2 extends HttpApi.make("MapleApiV2") .add(V2TracesApiGroup) .add(V2LogsApiGroup) .add(V2MetricsApiGroup) + .add(V2PrometheusApiGroup) .add(V2ServicesApiGroup) .add(V2ServiceMapApiGroup) .add(V2EnvironmentsApiGroup) diff --git a/packages/domain/src/http/v2/auth.ts b/packages/domain/src/http/v2/auth.ts index 14d085c913..22748a1dd2 100644 --- a/packages/domain/src/http/v2/auth.ts +++ b/packages/domain/src/http/v2/auth.ts @@ -106,6 +106,12 @@ const READ_ONLY_POST_PATHS = new Set([ "/v2/logs/breakdown", "/v2/metrics/timeseries", "/v2/metrics/breakdown", + // The Prometheus query API; Grafana POSTs its reads as forms. + "/v2/metrics/prometheus/api/v1/query", + "/v2/metrics/prometheus/api/v1/query_range", + "/v2/metrics/prometheus/api/v1/series", + "/v2/metrics/prometheus/api/v1/labels", + "/v2/metrics/prometheus/api/v1/query_exemplars", // Live proxies to PlanetScale, POST because the window + filters make a long // key that belongs in a body — reads, not writes. "/v2/integrations/planetscale/query_insights", diff --git a/packages/domain/src/http/v2/index.ts b/packages/domain/src/http/v2/index.ts index 2e8f78fede..e79f10d1a0 100644 --- a/packages/domain/src/http/v2/index.ts +++ b/packages/domain/src/http/v2/index.ts @@ -46,6 +46,7 @@ export * from "./setup-audit" export * from "./support-channel" export * from "./telemetry-signals" export * from "./share" +export * from "./prometheus" export * from "./telemetry" export * from "./widget-credentials" export * from "./device-credentials" diff --git a/packages/domain/src/http/v2/openapi.test.ts b/packages/domain/src/http/v2/openapi.test.ts index adf63eabf0..6d8853ade2 100644 --- a/packages/domain/src/http/v2/openapi.test.ts +++ b/packages/domain/src/http/v2/openapi.test.ts @@ -184,6 +184,14 @@ describe("MapleApiV2 OpenAPI", () => { "GET /v2/logs/{id}", "GET /v2/mcp_connections", "GET /v2/metrics", + "GET /v2/metrics/prometheus/api/v1/label/{name}/values", + "GET /v2/metrics/prometheus/api/v1/labels", + "GET /v2/metrics/prometheus/api/v1/metadata", + "GET /v2/metrics/prometheus/api/v1/query", + "GET /v2/metrics/prometheus/api/v1/query_exemplars", + "GET /v2/metrics/prometheus/api/v1/query_range", + "GET /v2/metrics/prometheus/api/v1/series", + "GET /v2/metrics/prometheus/api/v1/status/buildinfo", "GET /v2/mobile_devices", "GET /v2/onboarding/checklist", "GET /v2/organization", @@ -309,6 +317,11 @@ describe("MapleApiV2 OpenAPI", () => { "POST /v2/logs/search", "POST /v2/logs/timeseries", "POST /v2/metrics/breakdown", + "POST /v2/metrics/prometheus/api/v1/labels", + "POST /v2/metrics/prometheus/api/v1/query", + "POST /v2/metrics/prometheus/api/v1/query_exemplars", + "POST /v2/metrics/prometheus/api/v1/query_range", + "POST /v2/metrics/prometheus/api/v1/series", "POST /v2/metrics/timeseries", "POST /v2/onboarding/checklist/claim", "POST /v2/scrape_targets", diff --git a/packages/domain/src/http/v2/prometheus.ts b/packages/domain/src/http/v2/prometheus.ts new file mode 100644 index 0000000000..31719b7213 --- /dev/null +++ b/packages/domain/src/http/v2/prometheus.ts @@ -0,0 +1,210 @@ +import { HttpApiEndpoint, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/http-api" +import { Schema } from "effect" +import { AuditedRead } from "../audit-log" +import { AnalystAccess } from "../analyst-access" +import { AuthorizationV2 } from "./auth" + +/** + * The Prometheus HTTP API (`/api/v1/*`) over Maple's metrics, so Grafana and other Prometheus + * clients can use Maple as a Prometheus data source. Request parameters and response bodies follow + * Prometheus exactly; errors are `{status:"error",errorType,error}` with 400, 422 or 503. + */ + +const optionalString = (description: string) => Schema.optional(Schema.String).annotate({ description }) + +const matchParam = Schema.optional(Schema.Array(Schema.String)).annotate({ + description: 'Repeated series selector, e.g. `up{job="api"}`.', +}) + +/** Every Prometheus request parameter these endpoints read; each endpoint ignores the rest. */ +export const PrometheusParams = Schema.Struct({ + query: optionalString("PromQL expression."), + time: optionalString("Evaluation time: RFC3339 or Unix seconds. Defaults to now."), + start: optionalString("Range start: RFC3339 or Unix seconds."), + end: optionalString("Range end: RFC3339 or Unix seconds."), + step: optionalString("Resolution: a duration like `30s` or float seconds."), + timeout: optionalString("Accepted and ignored."), + limit: optionalString("Maximum number of returned items."), + metric: optionalString("Metric name to return metadata for."), + "match[]": matchParam, +}).annotate({ identifier: "PrometheusParams", title: "Prometheus parameters" }) + +/** The same parameters as an `application/x-www-form-urlencoded` body, as Grafana POSTs them. */ +export const PrometheusForm = PrometheusParams.pipe(HttpApiSchema.asFormUrlEncoded()) + +const status = Schema.Literal("success") + +export const PrometheusQueryResponse = Schema.Struct({ + status, + data: Schema.Struct({ + resultType: Schema.Literals(["matrix", "vector", "scalar", "string"]), + result: Schema.Unknown, + }), +}).annotate({ + identifier: "PrometheusQueryResponse", + title: "Prometheus query response", + description: 'Values are `[, ""]` pairs, as Prometheus returns them.', +}) + +export const PrometheusStringListResponse = Schema.Struct({ + status, + data: Schema.Array(Schema.String), +}).annotate({ identifier: "PrometheusStringListResponse", title: "Prometheus string list" }) + +export const PrometheusSeriesResponse = Schema.Struct({ + status, + data: Schema.Array(Schema.Record(Schema.String, Schema.String)), +}).annotate({ identifier: "PrometheusSeriesResponse", title: "Prometheus series" }) + +export const PrometheusMetadataResponse = Schema.Struct({ + status, + data: Schema.Record( + Schema.String, + Schema.Array( + Schema.Struct({ + type: Schema.Literals(["counter", "gauge", "histogram"]), + help: Schema.String, + unit: Schema.String, + }), + ), + ), +}).annotate({ identifier: "PrometheusMetadataResponse", title: "Prometheus metadata" }) + +export const PrometheusBuildInfoResponse = Schema.Struct({ + status, + data: Schema.Struct({ + version: Schema.String, + revision: Schema.String, + branch: Schema.String, + buildUser: Schema.String, + buildDate: Schema.String, + goVersion: Schema.String, + vendor: Schema.String, + }), +}).annotate({ identifier: "PrometheusBuildInfoResponse", title: "Prometheus build info" }) + +export const PrometheusExemplarsResponse = Schema.Struct({ + status, + data: Schema.Array(Schema.Unknown), +}).annotate({ identifier: "PrometheusExemplarsResponse", title: "Prometheus exemplars" }) + +const docs = (identifier: string, summary: string, description: string) => + OpenApi.annotations({ identifier, summary, description: `${description} Requires \`metrics:read\`.` }) + +const readGet = ( + name: Name, + path: Path, + success: S, +) => HttpApiEndpoint.get(name, path, { query: PrometheusParams, success }) + +/** The form-POST twin of a GET; Prometheus treats the two the same, and so does the scope check. */ +const readPost = ( + name: Name, + path: Path, + success: S, +) => + HttpApiEndpoint.post(name, path, { query: PrometheusParams, payload: PrometheusForm, success }).annotate( + AnalystAccess, + "read", + ) + +const QUERY = "Evaluates an instant PromQL query at `time`." +const RANGE = "Evaluates a PromQL query over `[start, end]` every `step`." +const SERIES = "Lists the label sets of the series the `match[]` selectors select in `[start, end]`." +const LABELS = "Lists label names, of the `match[]` series when given." +const EXEMPLARS = "Exemplar queries; Maple returns none." + +export class V2PrometheusApiGroup extends HttpApiGroup.make("prometheus") + .add( + readGet("query", "/query", PrometheusQueryResponse).annotateMerge( + docs("prometheusQuery", "PromQL instant query", QUERY), + ), + ) + .add( + readPost("queryPost", "/query", PrometheusQueryResponse).annotateMerge( + docs("prometheusQueryPost", "PromQL instant query (form POST)", QUERY), + ), + ) + .add( + readGet("queryRange", "/query_range", PrometheusQueryResponse).annotateMerge( + docs("prometheusQueryRange", "PromQL range query", RANGE), + ), + ) + .add( + readPost("queryRangePost", "/query_range", PrometheusQueryResponse).annotateMerge( + docs("prometheusQueryRangePost", "PromQL range query (form POST)", RANGE), + ), + ) + .add( + readGet("series", "/series", PrometheusSeriesResponse).annotateMerge( + docs("prometheusSeries", "Find series", SERIES), + ), + ) + .add( + readPost("seriesPost", "/series", PrometheusSeriesResponse).annotateMerge( + docs("prometheusSeriesPost", "Find series (form POST)", SERIES), + ), + ) + .add( + readGet("labels", "/labels", PrometheusStringListResponse).annotateMerge( + docs("prometheusLabels", "List label names", LABELS), + ), + ) + .add( + readPost("labelsPost", "/labels", PrometheusStringListResponse).annotateMerge( + docs("prometheusLabelsPost", "List label names (form POST)", LABELS), + ), + ) + .add( + HttpApiEndpoint.get("labelValues", "/label/:name/values", { + params: { name: Schema.String }, + query: PrometheusParams, + success: PrometheusStringListResponse, + }).annotateMerge( + docs( + "prometheusLabelValues", + "List label values", + "Lists the values of one label, of the `match[]` series when given. `__name__` lists metric names.", + ), + ), + ) + .add( + readGet("metadata", "/metadata", PrometheusMetadataResponse).annotateMerge( + docs( + "prometheusMetadata", + "Metric metadata", + "Type, help and unit of each metric family, from the metric catalog.", + ), + ), + ) + .add( + HttpApiEndpoint.get("buildInfo", "/status/buildinfo", { + success: PrometheusBuildInfoResponse, + }).annotateMerge( + docs( + "prometheusBuildInfo", + "Build information", + "The Prometheus version this API is compatible with, so clients enable the matching features.", + ), + ), + ) + .add( + readGet("exemplars", "/query_exemplars", PrometheusExemplarsResponse).annotateMerge( + docs("prometheusExemplars", "Query exemplars", EXEMPLARS), + ), + ) + .add( + readPost("exemplarsPost", "/query_exemplars", PrometheusExemplarsResponse).annotateMerge( + docs("prometheusExemplarsPost", "Query exemplars (form POST)", EXEMPLARS), + ), + ) + .prefix("/v2/metrics/prometheus/api/v1") + .middleware(AuthorizationV2) + .annotate(AuditedRead, "telemetry.read") + .annotateMerge( + OpenApi.annotations({ + title: "Prometheus", + description: + "The Prometheus HTTP query API over Maple metrics, for Grafana and other Prometheus clients.", + }), + ) {} diff --git a/packages/query-engine/src/ch/queries/promql.ts b/packages/query-engine/src/ch/queries/promql.ts index 385d41cc17..1ad4670089 100644 --- a/packages/query-engine/src/ch/queries/promql.ts +++ b/packages/query-engine/src/ch/queries/promql.ts @@ -730,3 +730,72 @@ export const promqlPushdownRowSchema = Schema.Struct({ values: numbers, }) export type PromqlPushdownRow = typeof promqlPushdownRowSchema.Type + +/** Type, unit and description of every stored metric in the window, for `/api/v1/metadata`. */ +export function promqlMetadataQuery(opts: { readonly limit: number }) { + return from(Datasources.metricCatalog) + .select(($) => ({ + metricName: $.MetricName, + metricType: $.MetricType, + unit: CH.any_($.MetricUnit), + description: CH.any_($.MetricDescription), + isMonotonic: sql(T.uint8)`max(${$.IsMonotonic})`, + })) + .where(($) => [ + $.OrgId.eq(orgIdParam), + $.Hour.gte(CH.toStartOfInterval(CH.toDateTime(utcSecondsParam("startTime")), 3600)), + $.Hour.lte(utcSecondsParam("endTime")), + ]) + .groupBy("metricName", "metricType") + .limit(opts.limit) + .format("JSON") +} + +/** Services that sent metrics in the window: the `job` values a bare label-values call lists. */ +export function promqlServiceNamesQuery(opts: { readonly limit: number }) { + return from(Datasources.metricCatalog) + .select(($) => ({ serviceName: $.ServiceName })) + .where(($) => [ + $.OrgId.eq(orgIdParam), + $.Hour.gte(CH.toStartOfInterval(CH.toDateTime(utcSecondsParam("startTime")), 3600)), + $.Hour.lte(utcSecondsParam("endTime")), + $.ServiceName.neq(""), + ]) + .groupBy("serviceName") + .orderBy(["serviceName", "asc"]) + .limit(opts.limit) + .format("JSON") +} + +/** + * Metric data-point attribute values for the raw keys a label name normalizes from, read from the + * hourly rollup (fed by `metrics_sum` only) so a bare label-values call never scans metric tables. + */ +export function promqlAttributeValuesQuery(opts: { + readonly keys: ReadonlyArray + readonly limit: number +}) { + return from(Datasources.attributeValuesHourly) + .select(($) => ({ attributeValue: $.AttributeValue })) + .where(($) => [ + $.OrgId.eq(orgIdParam), + $.Hour.gte(CH.toStartOfInterval(CH.toDateTime(utcSecondsParam("startTime")), 3600)), + $.Hour.lte(utcSecondsParam("endTime")), + $.AttributeScope.eq("metric"), + CH.inList($.AttributeKey, opts.keys), + $.AttributeValue.neq(""), + ]) + .groupBy("attributeValue") + .orderBy(["attributeValue", "asc"]) + .limit(opts.limit) + .format("JSON") +} + +export const promqlMetadataRowSchema = Schema.Struct({ + metricName: Schema.String, + metricType: Schema.String, + unit: Schema.String, + description: Schema.String, + isMonotonic: CHNumber, +}) +export type PromqlMetadataRow = typeof promqlMetadataRowSchema.Type diff --git a/packages/query-engine/src/promql/discovery.ts b/packages/query-engine/src/promql/discovery.ts new file mode 100644 index 0000000000..5eb2269270 --- /dev/null +++ b/packages/query-engine/src/promql/discovery.ts @@ -0,0 +1,89 @@ +// Prometheus discovery answers (metric names, label names, metadata) derived from the metric +// catalog and the attribute rollups, so they never scan the raw metric tables. Pure. + +import { Array as Arr, Order, String as Str } from "effect" +import type { PromqlMetadataRow } from "../ch/queries/promql" +import { normalizeLabelName } from "./labels" +import { type CatalogMetric, type OtelMetricType, promNamesOf, unitSuffix } from "./names" + +const sortedUnique = (values: ReadonlyArray): ReadonlyArray => + Arr.sort(Arr.dedupe(values), Order.String) + +/** The first name of each component of a stored metric: the one a bare selector reads it by. */ +const canonicalNamesOf = (metric: CatalogMetric): ReadonlyArray => + Arr.map( + Arr.dedupeWith(promNamesOf(metric), (a, b) => a.component === b.component), + (name) => name.promName, + ) + +/** Every metric name the catalog answers to by default, sorted. */ +export const canonicalPromNames = (catalog: ReadonlyArray): ReadonlyArray => + sortedUnique(Arr.flatMap(catalog, canonicalNamesOf)) + +/** The raw attribute keys whose Prometheus label name is `name`. */ +export const keysForLabel = (keys: ReadonlyArray, name: string): ReadonlyArray => + Arr.filter(keys, (key) => normalizeLabelName(key) === name) + +/** Label names when no series selector narrows them: identity labels plus attribute keys. */ +export const labelNamesFrom = ( + catalog: ReadonlyArray, + attributeKeys: ReadonlyArray, +): ReadonlyArray => { + if (catalog.length === 0) return [] + const histogram = Arr.some(catalog, (metric) => metric.metricType === "histogram") + return sortedUnique([ + "__name__", + "job", + ...(histogram ? ["le"] : []), + ...Arr.map(attributeKeys, normalizeLabelName), + ]) +} + +export interface PrometheusMetadata { + readonly type: "counter" | "gauge" | "histogram" + readonly help: string + readonly unit: string +} + +const METRIC_TYPES: ReadonlySet = new Set([ + "sum", + "gauge", + "histogram", + "exponential_histogram", +]) + +const isMetricType = (value: string): value is OtelMetricType => METRIC_TYPES.has(value) + +/** Histograms are keyed by their family name, without the `_bucket`/`_count` series suffix. */ +const familyName = (metric: CatalogMetric): string => { + const [first] = canonicalNamesOf(metric) + if (first === undefined) return metric.metricName + if (metric.metricType === "sum" || metric.metricType === "gauge") return first + return Str.replace(/_(bucket|count|sum)$/, "")(first) +} + +const typeOf = (metric: CatalogMetric): PrometheusMetadata["type"] => { + if (metric.metricType === "sum") return metric.isMonotonic ? "counter" : "gauge" + return metric.metricType === "gauge" ? "gauge" : "histogram" +} + +/** `/api/v1/metadata` data: one entry per metric family, keyed by its Prometheus name. */ +export const prometheusMetadata = ( + rows: ReadonlyArray, +): ReadonlyMap> => { + const out = new Map>() + for (const row of rows) { + if (!isMetricType(row.metricType)) continue + const metric: CatalogMetric = { + metricName: row.metricName, + metricType: row.metricType, + isMonotonic: row.isMonotonic > 0, + unit: row.unit, + } + const name = familyName(metric) + if (out.has(name)) continue + const unit = Arr.join(unitSuffix(row.unit, metric.metricType), "_") + out.set(name, [{ type: typeOf(metric), help: row.description, unit }]) + } + return out +} diff --git a/packages/query-engine/src/promql/index.ts b/packages/query-engine/src/promql/index.ts index 28f2090bae..56b5bebbb9 100644 --- a/packages/query-engine/src/promql/index.ts +++ b/packages/query-engine/src/promql/index.ts @@ -1,3 +1,4 @@ +export * from "./discovery" export * from "./labels" export * from "./names" export * from "./source" @@ -6,6 +7,11 @@ export { type PromqlPushdownRow, promqlPushdownRowSchema, promqlCatalogRowSchema, + type PromqlMetadataRow, + promqlMetadataRowSchema, + promqlAttributeValuesQuery, + promqlMetadataQuery, + promqlServiceNamesQuery, type PromqlSelectorRow, promqlSelectorRowSchema, } from "../ch/queries/promql" From 0fdedfa15c1f72431f350e723a82a3e48f62900d Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 15:53:55 +0100 Subject: [PATCH 02/18] fix(promql): range mode for query_range, query limits, and label limits after dedupe query_range with start equal to end answered like an instant query; it now asks for range mode and returns a one-point matrix. query and query_range apply Prometheus' limit to the series they return and say so in warnings. labels and label values with match[] applied the caller's limit to the series they read, hiding distinct names; it now counts the distinct names or values. Bare label listings no longer advertise le, whose values no rollup can list. --- apps/api/src/routes/v2/prometheus-wire.ts | 27 ++++++- .../api/src/routes/v2/prometheus.http.test.ts | 31 ++++++++ apps/api/src/routes/v2/prometheus.http.ts | 74 ++++++++++++------- .../src/services/promql/PromqlQueryService.ts | 31 +++++--- packages/query-engine/src/promql/discovery.ts | 9 +-- 5 files changed, 129 insertions(+), 43 deletions(-) diff --git a/apps/api/src/routes/v2/prometheus-wire.ts b/apps/api/src/routes/v2/prometheus-wire.ts index 11c78ec880..5de4fbcb12 100644 --- a/apps/api/src/routes/v2/prometheus-wire.ts +++ b/apps/api/src/routes/v2/prometheus-wire.ts @@ -107,4 +107,29 @@ export const errorBody = (type: PrometheusErrorType, error: string) => ({ error, }) -export const successBody = (data: A) => ({ status: "success" as const, data }) +export const successBody = (data: A, warnings: ReadonlyArray = []) => + warnings.length === 0 + ? { status: "success" as const, data } + : { status: "success" as const, data, warnings } + +/** Prometheus' `limit` on a query: at most that many series, with a warning when some were cut. */ +export const limitQueryResult = ( + result: QueryResult, + limit: number | undefined, +): { readonly result: QueryResult; readonly warnings: ReadonlyArray } => { + if (limit === undefined) return { result, warnings: [] } + const cut = [`results truncated due to limit`] + switch (result.resultType) { + case "matrix": + return result.result.length > limit + ? { result: { ...result, result: Arr.take(result.result, limit) }, warnings: cut } + : { result, warnings: [] } + case "vector": + return result.result.length > limit + ? { result: { ...result, result: Arr.take(result.result, limit) }, warnings: cut } + : { result, warnings: [] } + case "scalar": + case "string": + return { result, warnings: [] } + } +} diff --git a/apps/api/src/routes/v2/prometheus.http.test.ts b/apps/api/src/routes/v2/prometheus.http.test.ts index 6a7da9f622..1da8d6eb2d 100644 --- a/apps/api/src/routes/v2/prometheus.http.test.ts +++ b/apps/api/src/routes/v2/prometheus.http.test.ts @@ -167,6 +167,37 @@ describe("v2 Prometheus query API", () => { await harness.dispose() }) + it("answers a one-point matrix for a range whose start equals its end", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(["metrics:read"]) + const range = await harness.send( + key.secret, + `/query_range?query=vector(2)&start=${T}&end=${T}&step=30s`, + ) + expect(range.body).toEqual({ + status: "success", + data: { resultType: "matrix", result: [{ metric: {}, values: [[T, "2"]] }] }, + }) + await harness.dispose() + }) + + it("applies limit to a query's series and says so in warnings", async () => { + const harness = makeHarness() + const key = await harness.bootstrapKey(["metrics:read"]) + const two = encodeURIComponent( + 'label_replace(vector(1), "a", "x", "", "") or label_replace(vector(2), "a", "y", "", "")', + ) + const limited = await harness.send(key.secret, `/query?query=${two}&time=${T}&limit=1`) + expect(limited.body).toEqual({ + status: "success", + data: { resultType: "vector", result: [{ metric: { a: "x" }, value: [T, "1"] }] }, + warnings: ["results truncated due to limit"], + }) + const unlimited = await harness.send(key.secret, `/query?query=${two}&time=${T}&limit=0`) + expect(unlimited.body).toMatchObject({ data: { result: [{}, {}] } }) + await harness.dispose() + }) + it("answers a range query POSTed as a form with RFC3339 bounds and a duration step", async () => { const harness = makeHarness() const key = await harness.bootstrapKey(["metrics:read"]) diff --git a/apps/api/src/routes/v2/prometheus.http.ts b/apps/api/src/routes/v2/prometheus.http.ts index d8b0887051..4536156e53 100644 --- a/apps/api/src/routes/v2/prometheus.http.ts +++ b/apps/api/src/routes/v2/prometheus.http.ts @@ -10,6 +10,7 @@ import { MapleApiV2, type PrometheusParams } from "@maple/domain/http/v2" import { PromqlBadQueryError, PromqlQueryService } from "@maple/backend/services/promql/PromqlQueryService" import { encodeQueryResult, + limitQueryResult, errorBody, errorStatus, type PrometheusErrorType, @@ -54,25 +55,37 @@ const errorResponse = (type: PrometheusErrorType, message: string) => HttpServerResponse.jsonUnsafe(errorBody(type, message), { status: errorStatus(type) }), ) +/** A query error as Prometheus' error envelope; warehouse failures never leak their diagnostics. */ +const toErrorResponse = (error: E) => { + switch (error._tag) { + case "@maple/backend/promql/PromqlBadQueryError": + return errorResponse("bad_data", error.message) + case "@maple/backend/promql/PromqlExecutionError": + return errorResponse("execution", error.message) + default: + return Effect.andThen( + Effect.logWarning("Prometheus API warehouse read failed", error), + errorResponse("unavailable", "the metric store is unavailable, try again"), + ) + } +} + /** Success as `{status:"success",data}`; query errors as Prometheus' error envelope. */ const respond = ( effect: Effect.Effect, ) => effect.pipe( Effect.map((data) => HttpServerResponse.jsonUnsafe(successBody(data))), - Effect.catch((error) => { - switch (error._tag) { - case "@maple/backend/promql/PromqlBadQueryError": - return errorResponse("bad_data", error.message) - case "@maple/backend/promql/PromqlExecutionError": - return errorResponse("execution", error.message) - default: - return Effect.andThen( - Effect.logWarning("Prometheus API warehouse read failed", error), - errorResponse("unavailable", "the metric store is unavailable, try again"), - ) - } - }), + Effect.catch(toErrorResponse), + ) + +/** A query answer with Prometheus' `warnings` beside `data`. */ +const respondWithWarnings = ( + effect: Effect.Effect<{ readonly data: A; readonly warnings: ReadonlyArray }, E, R>, +) => + effect.pipe( + Effect.map(({ data, warnings }) => HttpServerResponse.jsonUnsafe(successBody(data, warnings))), + Effect.catch(toErrorResponse), ) const parseLimit = (text: string | undefined) => { @@ -103,17 +116,20 @@ export const HttpV2PrometheusLive = HttpApiBuilder.group(MapleApiV2, "prometheus const tenant = yield* CurrentTenant.Context const now = yield* Clock.currentTimeMillis const timeMs = params.time === undefined ? now : yield* parseTime("time", params.time) - const result = yield* promql.query(tenant, { + const limit = yield* parseLimit(params.limit) + const answer = yield* promql.query(tenant, { query: params.query ?? "", startMs: timeMs, endMs: timeMs, stepMs: 0, + instant: true, }) yield* Effect.annotateCurrentSpan({ - "maple.promql.result_type": result.resultType, - "maple.promql.result_series": seriesCount(result), + "maple.promql.result_type": answer.resultType, + "maple.promql.result_series": seriesCount(answer), }) - return encodeQueryResult(result) + const { result, warnings } = limitQueryResult(answer, limit) + return { data: encodeQueryResult(result), warnings } }) const rangeQuery = Effect.fn("PrometheusApi.queryRange")(function* (params: Params) { @@ -121,17 +137,21 @@ export const HttpV2PrometheusLive = HttpApiBuilder.group(MapleApiV2, "prometheus const startMs = yield* parseTime("start", params.start ?? "") const endMs = yield* parseTime("end", params.end ?? "") const stepMs = yield* parseStep(params.step ?? "") - const result = yield* promql.query(tenant, { + const limit = yield* parseLimit(params.limit) + // Range mode even when start equals end: Prometheus answers a one-point matrix. + const answer = yield* promql.query(tenant, { query: params.query ?? "", startMs, endMs, stepMs, + instant: false, }) yield* Effect.annotateCurrentSpan({ - "maple.promql.result_type": result.resultType, - "maple.promql.result_series": seriesCount(result), + "maple.promql.result_type": answer.resultType, + "maple.promql.result_series": seriesCount(answer), }) - return encodeQueryResult(result) + const { result, warnings } = limitQueryResult(answer, limit) + return { data: encodeQueryResult(result), warnings } }) const discovery = (params: Params) => @@ -170,10 +190,14 @@ export const HttpV2PrometheusLive = HttpApiBuilder.group(MapleApiV2, "prometheus }) return handlers - .handle("query", ({ query }) => respond(instantQuery(query))) - .handle("queryPost", ({ query, payload }) => respond(instantQuery(merged(query, payload)))) - .handle("queryRange", ({ query }) => respond(rangeQuery(query))) - .handle("queryRangePost", ({ query, payload }) => respond(rangeQuery(merged(query, payload)))) + .handle("query", ({ query }) => respondWithWarnings(instantQuery(query))) + .handle("queryPost", ({ query, payload }) => + respondWithWarnings(instantQuery(merged(query, payload))), + ) + .handle("queryRange", ({ query }) => respondWithWarnings(rangeQuery(query))) + .handle("queryRangePost", ({ query, payload }) => + respondWithWarnings(rangeQuery(merged(query, payload))), + ) .handle("series", ({ query }) => respond(series(query))) .handle("seriesPost", ({ query, payload }) => respond(series(merged(query, payload)))) .handle("labels", ({ query }) => respond(labels(query))) diff --git a/packages/backend/src/services/promql/PromqlQueryService.ts b/packages/backend/src/services/promql/PromqlQueryService.ts index d4d5dec86b..65d5449317 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.ts @@ -97,7 +97,7 @@ const validate = (query: PromqlQuery): Effect.Effect if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs) { return yield* new PromqlBadQueryError({ message: "end timestamp must not be before start time" }) } - if (endMs > startMs) { + if (endMs > startMs || query.instant === false) { if (!(stepMs > 0)) { return yield* new PromqlBadQueryError({ message: @@ -384,20 +384,24 @@ const make = Effect.gen(function* () { const sortedUnique = (values: ReadonlyArray) => Arr.take(Arr.sort(Arr.dedupe(values), Order.String), DISCOVERY_LIMIT) + /** The caller's `limit` counts distinct names or values, so it applies after deduplication. */ + const limited = (values: ReadonlyArray, limit: number | undefined) => + Arr.take(values, Math.min(limit ?? DISCOVERY_LIMIT, DISCOVERY_LIMIT)) + const labelNames = Effect.fn("PromqlQueryService.labelNames")(function* ( tenant: TenantContext, input: DiscoveryQuery, ) { const window = yield* validateWindow(input) if (input.matches.length > 0) { - const found = yield* series(tenant, input) - return sortedUnique(Arr.flatMap(found, (labels) => Rec.keys(labels))) + const found = yield* series(tenant, { ...input, limit: undefined }) + return limited(sortedUnique(Arr.flatMap(found, (labels) => Rec.keys(labels))), input.limit) } const [catalog, keys] = yield* Effect.all( [readCatalog(tenant, window), attributeKeys(tenant, window)], { concurrency: 2 }, ) - return Arr.take(labelNamesFrom(catalog, keys), DISCOVERY_LIMIT) + return limited(labelNamesFrom(catalog, keys), input.limit) }) const attributeValues = Effect.fn("PromqlQueryService.attributeValues")(function* ( @@ -444,20 +448,27 @@ const make = Effect.gen(function* () { const window = yield* validateWindow(input) yield* Effect.annotateCurrentSpan({ "maple.promql.label": input.name }) if (input.matches.length > 0) { - const found = yield* series(tenant, input) - return sortedUnique( - Arr.flatMap(found, (labels) => Option.toArray(Option.fromNullishOr(labels[input.name]))), + const found = yield* series(tenant, { ...input, limit: undefined }) + return limited( + sortedUnique( + Arr.flatMap(found, (labels) => Option.toArray(Option.fromNullishOr(labels[input.name]))), + ), + input.limit, ) } switch (input.name) { case "__name__": return yield* Effect.map(readCatalog(tenant, window), (catalog) => - sortedUnique(canonicalPromNames(catalog)), + limited(sortedUnique(canonicalPromNames(catalog)), input.limit), ) case "job": - return yield* Effect.map(serviceNames(tenant, window), sortedUnique) + return yield* Effect.map(serviceNames(tenant, window), (names) => + limited(sortedUnique(names), input.limit), + ) default: - return yield* Effect.map(attributeValues(tenant, window, input.name), sortedUnique) + return yield* Effect.map(attributeValues(tenant, window, input.name), (values) => + limited(sortedUnique(values), input.limit), + ) } }) diff --git a/packages/query-engine/src/promql/discovery.ts b/packages/query-engine/src/promql/discovery.ts index 5eb2269270..329e84d245 100644 --- a/packages/query-engine/src/promql/discovery.ts +++ b/packages/query-engine/src/promql/discovery.ts @@ -30,13 +30,8 @@ export const labelNamesFrom = ( attributeKeys: ReadonlyArray, ): ReadonlyArray => { if (catalog.length === 0) return [] - const histogram = Arr.some(catalog, (metric) => metric.metricType === "histogram") - return sortedUnique([ - "__name__", - "job", - ...(histogram ? ["le"] : []), - ...Arr.map(attributeKeys, normalizeLabelName), - ]) + // `le` is left out: its values come from bucket bounds, which no rollup lists. + return sortedUnique(["__name__", "job", ...Arr.map(attributeKeys, normalizeLabelName)]) } export interface PrometheusMetadata { From eef4c4e48c005443961d56bf161cb2e00b7de641 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 16:06:15 +0100 Subject: [PATCH 03/18] test(promql): selector rows name their metric with its unit --- .../backend/src/services/promql/PromqlQueryService.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/services/promql/PromqlQueryService.test.ts b/packages/backend/src/services/promql/PromqlQueryService.test.ts index 8c1126894c..d6256b8865 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.test.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.test.ts @@ -133,7 +133,8 @@ const T0 = Date.UTC(2026, 6, 15, 12) const selectorRow = (attributes: Record) => ({ series: JSON.stringify(attributes), - metricName: "http.requests", + // A selector row names its metric with the stored unit after U+001F; this one has none. + metricName: "http.requests\u001f", serviceName: "api", attributes, resourceAttributes: { "service.name": "api" }, From ffa8f2d90c2411cb3e55c73aa48bc1ab78e3b87c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 16:18:09 +0100 Subject: [PATCH 04/18] refactor(promql): floor rollup reads with utcHourFloor The catalog, metadata, service and attribute-value queries spelled out the hour floor the rollups' Hour column has to agree with; they now use the shared helper, so a change to it reaches them. --- packages/query-engine/src/ch/queries/promql.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/query-engine/src/ch/queries/promql.ts b/packages/query-engine/src/ch/queries/promql.ts index ae4b4d45c4..0e6eff9145 100644 --- a/packages/query-engine/src/ch/queries/promql.ts +++ b/packages/query-engine/src/ch/queries/promql.ts @@ -13,6 +13,7 @@ import { CHNumber } from "../schema" import * as Datasources from "@maple/domain/tinybird/datasources" import { orgId, orgIdParam, utcSecondsParam } from "../tables" import { EXCLUDED_KEY } from "../../promql/labels" +import { utcHourFloor } from "./query-helpers" /** Joins a stored metric's name and unit into the identity a selector row reports. */ export const METRIC_UNIT_SEPARATOR = "\u001f" @@ -362,7 +363,7 @@ export function promqlCatalogQuery(opts: { readonly limit: number }) { })) .where(($) => [ $.OrgId.eq(orgIdParam), - $.Hour.gte(CH.toStartOfInterval(CH.toDateTime(utcSecondsParam("startTime")), 3600)), + $.Hour.gte(utcHourFloor("startTime")), $.Hour.lte(utcSecondsParam("endTime")), ]) .groupBy("metricName", "metricType", "unit") @@ -746,7 +747,7 @@ export function promqlMetadataQuery(opts: { readonly limit: number }) { })) .where(($) => [ $.OrgId.eq(orgIdParam), - $.Hour.gte(CH.toStartOfInterval(CH.toDateTime(utcSecondsParam("startTime")), 3600)), + $.Hour.gte(utcHourFloor("startTime")), $.Hour.lte(utcSecondsParam("endTime")), ]) .groupBy("metricName", "metricType") @@ -760,7 +761,7 @@ export function promqlServiceNamesQuery(opts: { readonly limit: number }) { .select(($) => ({ serviceName: $.ServiceName })) .where(($) => [ $.OrgId.eq(orgIdParam), - $.Hour.gte(CH.toStartOfInterval(CH.toDateTime(utcSecondsParam("startTime")), 3600)), + $.Hour.gte(utcHourFloor("startTime")), $.Hour.lte(utcSecondsParam("endTime")), $.ServiceName.neq(""), ]) @@ -782,7 +783,7 @@ export function promqlAttributeValuesQuery(opts: { .select(($) => ({ attributeValue: $.AttributeValue })) .where(($) => [ $.OrgId.eq(orgIdParam), - $.Hour.gte(CH.toStartOfInterval(CH.toDateTime(utcSecondsParam("startTime")), 3600)), + $.Hour.gte(utcHourFloor("startTime")), $.Hour.lte(utcSecondsParam("endTime")), $.AttributeScope.eq("metric"), CH.inList($.AttributeKey, opts.keys), From b7e66085e8e4b47717f5728ef73989a73d9ea6f2 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 16:22:53 +0100 Subject: [PATCH 05/18] fix(promql): cap match[] selectors in discovery calls series, labels and label values read one warehouse query per match[] and accepted any number of them; they now refuse more than a query may read (20), before reading anything. --- .../src/services/promql/PromqlQueryService.test.ts | 11 +++++++++++ .../backend/src/services/promql/PromqlQueryService.ts | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/packages/backend/src/services/promql/PromqlQueryService.test.ts b/packages/backend/src/services/promql/PromqlQueryService.test.ts index d6256b8865..db8c5ec3cd 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.test.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.test.ts @@ -217,6 +217,17 @@ const window = { startMs: T0 - 3_600_000, endMs: T0 } const ORG_FILTER = /OrgId = 'org_promql'/ describe("PromqlQueryService discovery", () => { + it.effect("refuses more match[] selectors than a query may read", () => { + const { statements, layer } = recordingWarehouse() + return Effect.gen(function* () { + const service = yield* PromqlQueryService + const matches = Array.from({ length: 21 }, (_, i) => `up{i="${i}"}`) + const error = yield* Effect.flip(service.series(tenant, { ...window, matches })) + expect(error.message).toMatch(/more than 20 selectors/) + expect(statements).toHaveLength(0) + }).pipe(Effect.provide(layer)) + }) + it.effect("returns the label sets a match[] selector selects, read through org-scoped queries", () => { const { statements, layer } = recordingWarehouse() return Effect.gen(function* () { diff --git a/packages/backend/src/services/promql/PromqlQueryService.ts b/packages/backend/src/services/promql/PromqlQueryService.ts index 1d0adb6320..8e86effb23 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.ts @@ -350,6 +350,12 @@ const make = Effect.gen(function* () { if (input.matches.length === 0) { return yield* new PromqlBadQueryError({ message: "no match[] parameter provided" }) } + // Each match[] is at least one warehouse read, like a selector in a query. + if (input.matches.length > MAX_SELECTORS) { + return yield* new PromqlBadQueryError({ + message: `match[] reads more than ${MAX_SELECTORS} selectors`, + }) + } const selectors = yield* Effect.forEach(input.matches, parseMatch) const catalog = yield* readCatalog(tenant, window) const read = yield* Effect.forEach( From 33f18ff1cbbf72e00c70b5fe709b59063e1bacba Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 16:40:03 +0100 Subject: [PATCH 06/18] test(promql): match[] on a GET's query string, one value and many --- apps/api/src/routes/v2/prometheus.http.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/api/src/routes/v2/prometheus.http.test.ts b/apps/api/src/routes/v2/prometheus.http.test.ts index 1da8d6eb2d..62e09bca5c 100644 --- a/apps/api/src/routes/v2/prometheus.http.test.ts +++ b/apps/api/src/routes/v2/prometheus.http.test.ts @@ -326,6 +326,19 @@ describe("v2 Prometheus discovery API", () => { status: "success", data: [], }) + // The same parameter on a GET's query string: one value, and every repeated value. + const one = new URLSearchParams([["match[]", "http_requests_total"]]) + expect((await harness.send(key.secret, `/series?${one}`)).body).toEqual({ + status: "success", + data: [], + }) + const many = new URLSearchParams(Array.from({ length: 21 }, (_, i) => ["match[]", `up{i="${i}"}`])) + const capped = await harness.send(key.secret, `/series?${many}`) + expect(capped.body).toEqual({ + status: "error", + errorType: "bad_data", + error: "match[] reads more than 20 selectors", + }) const missing = await harness.send(key.secret, "/series") expect(missing.status).toBe(400) expect(missing.body).toEqual({ From a61bd55cd039b346e3257f0808a1ac2e038273aa Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 17:05:33 +0100 Subject: [PATCH 07/18] fix(promql): label discovery lists instance and resource labels /labels and /label//values read the resource-scope rollup (minus excluded keys) beside the datapoint keys, and instance reads service.instance.id, so discovery names every label /series returns. --- .../api/src/routes/v2/prometheus.http.test.ts | 2 +- .../promql/PromqlQueryService.test.ts | 25 +++++++++- .../src/services/promql/PromqlQueryService.ts | 49 +++++++++++++------ .../query-engine/src/ch/queries/promql.ts | 3 +- packages/query-engine/src/promql/discovery.ts | 25 ++++++++-- 5 files changed, 84 insertions(+), 20 deletions(-) diff --git a/apps/api/src/routes/v2/prometheus.http.test.ts b/apps/api/src/routes/v2/prometheus.http.test.ts index 62e09bca5c..34bbb904af 100644 --- a/apps/api/src/routes/v2/prometheus.http.test.ts +++ b/apps/api/src/routes/v2/prometheus.http.test.ts @@ -300,7 +300,7 @@ describe("v2 Prometheus discovery API", () => { }) expect((await harness.send(key.secret, "/labels", new URLSearchParams(window))).body).toEqual({ status: "success", - data: ["__name__", "job"], + data: ["__name__", "instance", "job"], }) const buildInfo = await harness.send(key.secret, "/status/buildinfo") expect(buildInfo.body).toMatchObject({ data: { version: "3.5.0", vendor: "Maple" } }) diff --git a/packages/backend/src/services/promql/PromqlQueryService.test.ts b/packages/backend/src/services/promql/PromqlQueryService.test.ts index db8c5ec3cd..cc94ba845b 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.test.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.test.ts @@ -184,12 +184,24 @@ const recordingWarehouse = () => { if (Str.includes("FROM metric_catalog")(sql)) { return [{ metricName: "http.requests", metricType: "sum", unit: "", isMonotonic: 1 }] } + if ( + Str.includes("FROM attribute_keys_hourly")(sql) && + Str.includes("AttributeScope = 'resource'")(sql) + ) { + return [ + { attributeKey: "host.name", usageCount: 5 }, + { attributeKey: "service.version", usageCount: 5 }, + ] + } if (Str.includes("FROM attribute_keys_hourly")(sql)) { return [ { attributeKey: "http.route", usageCount: 10 }, { attributeKey: "http_route", usageCount: 1 }, ] } + if (Str.includes("FROM attribute_values_hourly")(sql) && Str.includes("service.instance.id")(sql)) { + return [{ attributeValue: "pod-1" }] + } if (Str.includes("FROM attribute_values_hourly")(sql)) return [{ attributeValue: "/checkout" }] return [] } @@ -262,7 +274,7 @@ describe("PromqlQueryService discovery", () => { return Effect.gen(function* () { const service = yield* PromqlQueryService const names = yield* service.labelNames(tenant, { ...window, matches: [] }) - expect(names).toEqual(["__name__", "http_route", "job"]) + expect(names).toEqual(["__name__", "host_name", "http_route", "instance", "job"]) expect(Arr.some(statements, (sql) => Str.includes("metrics_sum")(sql))).toBe(false) for (const sql of statements) expect(sql).toMatch(ORG_FILTER) }).pipe(Effect.provide(layer)) @@ -281,6 +293,17 @@ describe("PromqlQueryService discovery", () => { }).pipe(Effect.provide(layer)) }) + it.effect("reads instance values from service.instance.id in the resource rollup", () => { + const { statements, layer } = recordingWarehouse() + return Effect.gen(function* () { + const service = yield* PromqlQueryService + const values = yield* service.labelValues(tenant, { ...window, matches: [], name: "instance" }) + expect(values).toEqual(["pod-1"]) + const valuesSql = Arr.findFirst(statements, (sql) => Str.includes("attribute_values_hourly")(sql)) + expect(Option.getOrThrow(valuesSql)).toMatch(/AttributeScope = 'resource'/) + }).pipe(Effect.provide(layer)) + }) + it.effect("lists metric names and jobs from the catalog", () => { const { layer } = recordingWarehouse() return Effect.gen(function* () { diff --git a/packages/backend/src/services/promql/PromqlQueryService.ts b/packages/backend/src/services/promql/PromqlQueryService.ts index 8e86effb23..140aeae7ed 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.ts @@ -36,6 +36,7 @@ import { matchesValue, prometheusMetadata, promqlAttributeValuesQuery, + resourceKeysForLabel, promqlMetadataQuery, promqlServiceNamesQuery, type PushdownPlan, @@ -135,6 +136,8 @@ export interface DiscoveryQuery extends DiscoveryWindow { /** Names and values one discovery call returns at most. */ export const DISCOVERY_LIMIT = 10_000 const ATTRIBUTE_KEY_LIMIT = 1_000 + +type AttributeScope = "metric" | "resource" const LABEL_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/ const validateWindow = (window: DiscoveryWindow): Effect.Effect => @@ -381,9 +384,10 @@ const make = Effect.gen(function* () { const attributeKeys = Effect.fn("PromqlQueryService.attributeKeys")(function* ( tenant: TenantContext, window: DiscoveryWindow, + scope: AttributeScope, ) { const compiled = CH.compile( - CH.attributeKeysQuery({ scope: "metric", limit: ATTRIBUTE_KEY_LIMIT }), + CH.attributeKeysQuery({ scope, limit: ATTRIBUTE_KEY_LIMIT }), windowParams(tenant, window), ) const rows = yield* warehouse.compiledQuery(tenant, compiled, { @@ -393,6 +397,12 @@ const make = Effect.gen(function* () { return Arr.map(rows, (row) => row.attributeKey) }) + /** Datapoint keys and resource keys, read concurrently. */ + const bothAttributeKeys = (tenant: TenantContext, window: DiscoveryWindow) => + Effect.all([attributeKeys(tenant, window, "metric"), attributeKeys(tenant, window, "resource")], { + concurrency: 2, + }) + const sortedUnique = (values: ReadonlyArray) => Arr.take(Arr.sort(Arr.dedupe(values), Order.String), DISCOVERY_LIMIT) @@ -410,10 +420,10 @@ const make = Effect.gen(function* () { return limited(sortedUnique(Arr.flatMap(found, (labels) => Rec.keys(labels))), input.limit) } const [catalog, keys] = yield* Effect.all( - [readCatalog(tenant, window), attributeKeys(tenant, window)], + [readCatalog(tenant, window), bothAttributeKeys(tenant, window)], { concurrency: 2 }, ) - return limited(labelNamesFrom(catalog, keys), input.limit) + return limited(labelNamesFrom(catalog, ...keys), input.limit) }) const attributeValues = Effect.fn("PromqlQueryService.attributeValues")(function* ( @@ -421,18 +431,29 @@ const make = Effect.gen(function* () { window: DiscoveryWindow, name: string, ) { - const allKeys = yield* attributeKeys(tenant, window) - const keys = keysForLabel(allKeys, name) - if (keys.length === 0) return [] - const compiled = CH.compile( - promqlAttributeValuesQuery({ keys, limit: DISCOVERY_LIMIT }), - windowParams(tenant, window), + const [metricKeys, resourceKeys] = yield* bothAttributeKeys(tenant, window) + const candidates: ReadonlyArray<{ scope: AttributeScope; keys: ReadonlyArray }> = [ + { scope: "metric", keys: keysForLabel(metricKeys, name) }, + { scope: "resource", keys: resourceKeysForLabel(resourceKeys, name) }, + ] + const reads = Arr.filter(candidates, (read) => read.keys.length > 0) + const values = yield* Effect.forEach( + reads, + (read) => + Effect.map( + warehouse.compiledQuery( + tenant, + CH.compile( + promqlAttributeValuesQuery({ ...read, limit: DISCOVERY_LIMIT }), + windowParams(tenant, window), + ), + { profile: "discovery", context: "promqlLabelValues" }, + ), + (rows) => Arr.map(rows, (row) => row.attributeValue), + ), + { concurrency: 2 }, ) - const rows = yield* warehouse.compiledQuery(tenant, compiled, { - profile: "discovery", - context: "promqlLabelValues", - }) - return Arr.map(rows, (row) => row.attributeValue) + return Arr.flatten(values) }) const serviceNames = Effect.fn("PromqlQueryService.serviceNames")(function* ( diff --git a/packages/query-engine/src/ch/queries/promql.ts b/packages/query-engine/src/ch/queries/promql.ts index 0e6eff9145..71f1224c2d 100644 --- a/packages/query-engine/src/ch/queries/promql.ts +++ b/packages/query-engine/src/ch/queries/promql.ts @@ -776,6 +776,7 @@ export function promqlServiceNamesQuery(opts: { readonly limit: number }) { * hourly rollup (fed by `metrics_sum` only) so a bare label-values call never scans metric tables. */ export function promqlAttributeValuesQuery(opts: { + readonly scope: "metric" | "resource" readonly keys: ReadonlyArray readonly limit: number }) { @@ -785,7 +786,7 @@ export function promqlAttributeValuesQuery(opts: { $.OrgId.eq(orgIdParam), $.Hour.gte(utcHourFloor("startTime")), $.Hour.lte(utcSecondsParam("endTime")), - $.AttributeScope.eq("metric"), + $.AttributeScope.eq(opts.scope), CH.inList($.AttributeKey, opts.keys), $.AttributeValue.neq(""), ]) diff --git a/packages/query-engine/src/promql/discovery.ts b/packages/query-engine/src/promql/discovery.ts index 329e84d245..fed4d35d7b 100644 --- a/packages/query-engine/src/promql/discovery.ts +++ b/packages/query-engine/src/promql/discovery.ts @@ -3,7 +3,7 @@ import { Array as Arr, Order, String as Str } from "effect" import type { PromqlMetadataRow } from "../ch/queries/promql" -import { normalizeLabelName } from "./labels" +import { isExcludedResourceKey, normalizeLabelName } from "./labels" import { type CatalogMetric, type OtelMetricType, promNamesOf, unitSuffix } from "./names" const sortedUnique = (values: ReadonlyArray): ReadonlyArray => @@ -24,14 +24,33 @@ export const canonicalPromNames = (catalog: ReadonlyArray): Reado export const keysForLabel = (keys: ReadonlyArray, name: string): ReadonlyArray => Arr.filter(keys, (key) => normalizeLabelName(key) === name) -/** Label names when no series selector narrows them: identity labels plus attribute keys. */ +/** The raw resource keys whose label name is `name`; `instance` reads `service.instance.id`. */ +export const resourceKeysForLabel = (keys: ReadonlyArray, name: string): ReadonlyArray => + name === "instance" + ? ["service.instance.id"] + : keysForLabel( + Arr.filter(keys, (key) => !isExcludedResourceKey(key)), + name, + ) + +/** + * Label names when no series selector narrows them: identity labels plus datapoint and resource + * keys. The resource rollup is fed by spans, so it can name a label no metric carries, never hide one. + */ export const labelNamesFrom = ( catalog: ReadonlyArray, attributeKeys: ReadonlyArray, + resourceKeys: ReadonlyArray, ): ReadonlyArray => { if (catalog.length === 0) return [] // `le` is left out: its values come from bucket bounds, which no rollup lists. - return sortedUnique(["__name__", "job", ...Arr.map(attributeKeys, normalizeLabelName)]) + const resource = Arr.filter(resourceKeys, (key) => !isExcludedResourceKey(key)) + return sortedUnique([ + "__name__", + "job", + "instance", + ...Arr.map([...attributeKeys, ...resource], normalizeLabelName), + ]) } export interface PrometheusMetadata { From f28129e400bcfaba59caed391022e0cd63ef5b42 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 17:08:23 +0100 Subject: [PATCH 08/18] fix(api): redact PromQL query and match[] values in traced URLs --- apps/api/src/http/api-observability.test.ts | 14 ++++++++++++++ apps/api/src/http/api-observability.ts | 3 +++ 2 files changed, 17 insertions(+) diff --git a/apps/api/src/http/api-observability.test.ts b/apps/api/src/http/api-observability.test.ts index e030ea3d95..0f4f7be905 100644 --- a/apps/api/src/http/api-observability.test.ts +++ b/apps/api/src/http/api-observability.test.ts @@ -32,6 +32,20 @@ describe("ApiObservabilityLive", () => { expect(recorded).not.toContain("abc") }) + it("keeps PromQL text out of the URL a server span records", () => { + const recorded = Effect.gen(function* () { + const names = yield* UrlParams.CurrentRedactedNames + return UrlParams.redactUrl( + "https://api.maple.dev/api/v2/prometheus/api/v1/series?match%5B%5D=up%7Buser%3D%22ada%22%7D&query=rate(x%7Bcustomer%3D%22acme%22%7D%5B5m%5D)&step=60", + names, + ) + }).pipe(Effect.provide(ApiObservabilityLive), Effect.runSync) + + expect(recorded).not.toContain("ada") + expect(recorded).not.toContain("acme") + expect(recorded).toContain("step=60") + }) + it("still traces ordinary API routes", () => { expect(tracedFor("/api/v2/uptime_monitors")).toBe(true) expect(tracedFor("/hbx")).toBe(true) diff --git a/apps/api/src/http/api-observability.ts b/apps/api/src/http/api-observability.ts index 5476cf50bf..43a76e2277 100644 --- a/apps/api/src/http/api-observability.ts +++ b/apps/api/src/http/api-observability.ts @@ -63,8 +63,11 @@ export const ApiObservabilityLive = Layer.mergeAll( // Query values the tracer writes as `` in `url.full` and `url.query`. A custom // list replaces Effect's, so its defaults are kept. `actor` is an audit filter: an admin // types a member's email into it, and a span is read by more people than the audit log is. + // `query` and `match[]` are PromQL text, whose matcher values can name a user or customer. Layer.succeed(UrlParams.CurrentRedactedNames, [ ...UrlParams.CurrentRedactedNames.defaultValue(), "actor", + "query", + "match[]", ]), ) From 40416c14f0d2bbaa9a127c6b3895f7a8bcc49141 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 17:27:28 +0100 Subject: [PATCH 09/18] fix(api): Prometheus timestamps past Date's range are a 400 --- apps/api/src/routes/v2/prometheus-wire.test.ts | 3 +++ apps/api/src/routes/v2/prometheus-wire.ts | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/api/src/routes/v2/prometheus-wire.test.ts b/apps/api/src/routes/v2/prometheus-wire.test.ts index 275b0468b0..60989b91b0 100644 --- a/apps/api/src/routes/v2/prometheus-wire.test.ts +++ b/apps/api/src/routes/v2/prometheus-wire.test.ts @@ -24,6 +24,9 @@ describe("Prometheus wire format", () => { expect(invalid.message).toBe( 'invalid parameter "start": cannot parse "2026-07-15" to a valid timestamp', ) + // Past what a Date renders (nanoseconds sent as seconds): a 400, not a 500 downstream. + const huge = yield* Effect.flip(parseTime("time", "1752580800123456789")) + expect(huge.message).toMatch(/cannot parse/) }), ) }) diff --git a/apps/api/src/routes/v2/prometheus-wire.ts b/apps/api/src/routes/v2/prometheus-wire.ts index 5de4fbcb12..77067e022a 100644 --- a/apps/api/src/routes/v2/prometheus-wire.ts +++ b/apps/api/src/routes/v2/prometheus-wire.ts @@ -8,11 +8,14 @@ import { PromqlBadQueryError } from "@maple/backend/services/promql/PromqlQueryS const FLOAT = /^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?$/ const RFC3339 = /^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$/ +/** The largest epoch ms a JavaScript Date renders. */ +const MAX_DATE_MS = 8.64e15 + /** A Prometheus timestamp: Unix seconds with an optional fraction, or RFC3339. Epoch ms. */ export const parseTime = (name: string, text: string): Effect.Effect => { if (FLOAT.test(text)) { const ms = Math.round(Number(text) * 1000) - if (Number.isFinite(ms)) return Effect.succeed(ms) + if (Math.abs(ms) <= MAX_DATE_MS) return Effect.succeed(ms) } else if (RFC3339.test(text)) { const parsed = DateTime.make(text) if (Option.isSome(parsed)) return Effect.succeed(DateTime.toEpochMillis(parsed.value)) From 2b131e450e5982697a3523213ee840f02a2008d8 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 17:34:46 +0100 Subject: [PATCH 10/18] fix(promql): label values from match[] read own labels only --- .../src/services/promql/PromqlQueryService.test.ts | 13 +++++++++++++ .../src/services/promql/PromqlQueryService.ts | 4 +--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/services/promql/PromqlQueryService.test.ts b/packages/backend/src/services/promql/PromqlQueryService.test.ts index cc94ba845b..63c926399f 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.test.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.test.ts @@ -304,6 +304,19 @@ describe("PromqlQueryService discovery", () => { }).pipe(Effect.provide(layer)) }) + it.effect("answers no values for a label named like an Object member", () => { + const { layer } = recordingWarehouse() + return Effect.gen(function* () { + const service = yield* PromqlQueryService + const values = yield* service.labelValues(tenant, { + ...window, + matches: ['http_requests_total{http_route=~"/c.*"}'], + name: "constructor", + }) + expect(values).toEqual([]) + }).pipe(Effect.provide(layer)) + }) + it.effect("lists metric names and jobs from the catalog", () => { const { layer } = recordingWarehouse() return Effect.gen(function* () { diff --git a/packages/backend/src/services/promql/PromqlQueryService.ts b/packages/backend/src/services/promql/PromqlQueryService.ts index e8fd57f0cd..cad63730bf 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.ts @@ -483,9 +483,7 @@ const make = Effect.gen(function* () { if (input.matches.length > 0) { const found = yield* series(tenant, { ...input, limit: undefined }) return limited( - sortedUnique( - Arr.flatMap(found, (labels) => Option.toArray(Option.fromNullishOr(labels[input.name]))), - ), + sortedUnique(Arr.flatMap(found, (labels) => Option.toArray(Rec.get(labels, input.name)))), input.limit, ) } From f9f786c1bbf6c8976f6f564b7e23034e3f06fe0e Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 17:38:53 +0100 Subject: [PATCH 11/18] fix(promql): instance label values read the metric tables The attribute rollups drop id-like keys, so service.instance.id is never there. /label/instance/values reads distinct values from each metric table, bounded by org, window and the discovery limit. --- apps/cli/test/promql-source.test.ts | 21 +++++++++++- .../promql/PromqlQueryService.test.ts | 15 +++++---- .../src/services/promql/PromqlQueryService.ts | 33 +++++++++++++++++++ .../query-engine/src/ch/queries/promql.ts | 22 +++++++++++++ packages/query-engine/src/promql/discovery.ts | 12 +++---- packages/query-engine/src/promql/index.ts | 1 + 6 files changed, 89 insertions(+), 15 deletions(-) diff --git a/apps/cli/test/promql-source.test.ts b/apps/cli/test/promql-source.test.ts index 2ee7f28851..45b06471ea 100644 --- a/apps/cli/test/promql-source.test.ts +++ b/apps/cli/test/promql-source.test.ts @@ -8,7 +8,7 @@ import { afterAll, beforeAll, expect, test } from "bun:test" import { mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" -import { Array as Arr, Effect, Order, Record as Rec, String as Str } from "effect" +import { Array as Arr, Effect, Order, Record as Rec, Schema, String as Str } from "effect" import schemaSql from "@maple/local-server/schema/local-schema.sql" with { type: "text" } import { Chdb } from "@maple/local-server/chdb" import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" @@ -19,6 +19,7 @@ import { catalogFromRows, catalogQuery, promqlCatalogRowSchema, + promqlInstanceValuesQuery, promqlSelectorRowSchema, toSelectorData, type CatalogMetric, @@ -332,6 +333,7 @@ const expected = (req: Request, series: ReadonlyArray) => const decodeRows = decodeJsonEachRow(promqlSelectorRowSchema) const decodeCatalog = decodeJsonEachRow(promqlCatalogRowSchema) +const decodeInstances = decodeJsonEachRow(Schema.Struct({ instance: Schema.String })) const catalogOf = (db: Chdb, org: string): ReadonlyArray => { const compiled = CH.compileUnsafe(catalogQuery(), { @@ -593,4 +595,21 @@ describeWithChdb("PromQL selector source", () => { ) expect(select(db, ORG_A, request("ordered_total")).series).toHaveLength(1) }) + + test("instance values come from the metric tables, scoped to the org", () => { + const instances = (org: string, table: "metrics_sum" | "metrics_gauge") => { + const compiled = CH.compileUnsafe(promqlInstanceValuesQuery({ table, limit: 100 }), { + orgId: OrgId.make(org), + startTime: Str.slice(0, 19)(chTime(-HOUR_MS)), + endTime: Str.slice(0, 19)(chTime(HOUR_MS)), + }) + return Arr.map( + decodeInstances(db.query(normalizeSqlForClickHouseClient(compiled.sql))), + (row) => row.instance, + ) + } + expect(instances(ORG_A, "metrics_sum")).toEqual(["i1"]) + expect(instances(ORG_A, "metrics_gauge")).toEqual(["i1"]) + expect(instances("org_promql_none", "metrics_sum")).toEqual([]) + }) }) diff --git a/packages/backend/src/services/promql/PromqlQueryService.test.ts b/packages/backend/src/services/promql/PromqlQueryService.test.ts index 63c926399f..2b2709fec6 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.test.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.test.ts @@ -159,6 +159,8 @@ const selectorRow = (attributes: Record) => ({ const recordingWarehouse = () => { const statements: Array = [] const rowsFor = (sql: string): ReadonlyArray> => { + if (Str.includes("AS instance")(sql)) + return Str.includes("FROM metrics_gauge")(sql) ? [{ instance: "pod-2" }] : [{ instance: "pod-1" }] if (Str.includes("FROM metrics_sum")(sql)) { return [selectorRow({ "http.route": "/checkout" }), selectorRow({ "http.route": "/cart" })] } @@ -199,9 +201,6 @@ const recordingWarehouse = () => { { attributeKey: "http_route", usageCount: 1 }, ] } - if (Str.includes("FROM attribute_values_hourly")(sql) && Str.includes("service.instance.id")(sql)) { - return [{ attributeValue: "pod-1" }] - } if (Str.includes("FROM attribute_values_hourly")(sql)) return [{ attributeValue: "/checkout" }] return [] } @@ -293,14 +292,16 @@ describe("PromqlQueryService discovery", () => { }).pipe(Effect.provide(layer)) }) - it.effect("reads instance values from service.instance.id in the resource rollup", () => { + it.effect("reads instance values from the metric tables, which no rollup keeps", () => { const { statements, layer } = recordingWarehouse() return Effect.gen(function* () { const service = yield* PromqlQueryService const values = yield* service.labelValues(tenant, { ...window, matches: [], name: "instance" }) - expect(values).toEqual(["pod-1"]) - const valuesSql = Arr.findFirst(statements, (sql) => Str.includes("attribute_values_hourly")(sql)) - expect(Option.getOrThrow(valuesSql)).toMatch(/AttributeScope = 'resource'/) + expect(values).toEqual(["pod-1", "pod-2"]) + const reads = Arr.filter(statements, (sql) => Str.includes("AS instance")(sql)) + expect(reads).toHaveLength(4) + for (const sql of reads) expect(sql).toMatch(ORG_FILTER) + expect(Arr.some(statements, (sql) => Str.includes("attribute_values_hourly")(sql))).toBe(false) }).pipe(Effect.provide(layer)) }) diff --git a/packages/backend/src/services/promql/PromqlQueryService.ts b/packages/backend/src/services/promql/PromqlQueryService.ts index cad63730bf..4102d34c5a 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.ts @@ -36,6 +36,7 @@ import { matchesValue, prometheusMetadata, promqlAttributeValuesQuery, + promqlInstanceValuesQuery, resourceKeysForLabel, promqlMetadataQuery, promqlServiceNamesQuery, @@ -138,6 +139,13 @@ export const DISCOVERY_LIMIT = 10_000 const ATTRIBUTE_KEY_LIMIT = 1_000 type AttributeScope = "metric" | "resource" + +const PROMQL_METRIC_TABLES = [ + "metrics_sum", + "metrics_gauge", + "metrics_histogram", + "metrics_exponential_histogram", +] as const const LABEL_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/ const validateWindow = (window: DiscoveryWindow): Effect.Effect => @@ -456,6 +464,27 @@ const make = Effect.gen(function* () { return Arr.flatten(values) }) + /** `instance` values: one bounded read per metric table, since no rollup keeps id-like keys. */ + const instanceValues = Effect.fn("PromqlQueryService.instanceValues")(function* ( + tenant: TenantContext, + window: DiscoveryWindow, + ) { + const perTable = yield* Effect.forEach( + PROMQL_METRIC_TABLES, + (table) => + warehouse.compiledQuery( + tenant, + CH.compile( + promqlInstanceValuesQuery({ table, limit: DISCOVERY_LIMIT }), + windowParams(tenant, window), + ), + { profile: "discovery", context: "promqlInstances" }, + ), + { concurrency: "unbounded" }, + ) + return Arr.flatMap(perTable, (rows) => Arr.map(rows, (row) => row.instance)) + }) + const serviceNames = Effect.fn("PromqlQueryService.serviceNames")(function* ( tenant: TenantContext, window: DiscoveryWindow, @@ -496,6 +525,10 @@ const make = Effect.gen(function* () { return yield* Effect.map(serviceNames(tenant, window), (names) => limited(sortedUnique(names), input.limit), ) + case "instance": + return yield* Effect.map(instanceValues(tenant, window), (values) => + limited(sortedUnique(values), input.limit), + ) default: return yield* Effect.map(attributeValues(tenant, window, input.name), (values) => limited(sortedUnique(values), input.limit), diff --git a/packages/query-engine/src/ch/queries/promql.ts b/packages/query-engine/src/ch/queries/promql.ts index 71f1224c2d..5997917d14 100644 --- a/packages/query-engine/src/ch/queries/promql.ts +++ b/packages/query-engine/src/ch/queries/promql.ts @@ -771,6 +771,28 @@ export function promqlServiceNamesQuery(opts: { readonly limit: number }) { .format("JSON") } +/** + * Distinct `service.instance.id` values (the `instance` label) in one metric table. No rollup keeps + * id-like keys, so this reads the table itself, bounded by org, window and limit. + */ +export function promqlInstanceValuesQuery(opts: { + readonly table: PromqlMetricTable + readonly limit: number +}) { + return from(metricTable(opts.table)) + .select(($) => ({ instance: sql(T.string)`${$.ResourceAttributes}['service.instance.id']` })) + .where(($) => [ + $.OrgId.eq(orgIdParam), + $.TimeUnix.gte(utcSecondsParam("startTime")), + $.TimeUnix.lte(utcSecondsParam("endTime")), + sql.cond`${$.ResourceAttributes}['service.instance.id'] != ''`, + ]) + .groupBy("instance") + .orderBy(["instance", "asc"]) + .limit(opts.limit) + .format("JSON") +} + /** * Metric data-point attribute values for the raw keys a label name normalizes from, read from the * hourly rollup (fed by `metrics_sum` only) so a bare label-values call never scans metric tables. diff --git a/packages/query-engine/src/promql/discovery.ts b/packages/query-engine/src/promql/discovery.ts index fed4d35d7b..088f212790 100644 --- a/packages/query-engine/src/promql/discovery.ts +++ b/packages/query-engine/src/promql/discovery.ts @@ -24,14 +24,12 @@ export const canonicalPromNames = (catalog: ReadonlyArray): Reado export const keysForLabel = (keys: ReadonlyArray, name: string): ReadonlyArray => Arr.filter(keys, (key) => normalizeLabelName(key) === name) -/** The raw resource keys whose label name is `name`; `instance` reads `service.instance.id`. */ +/** The raw resource keys, minus the excluded ones, whose label name is `name`. */ export const resourceKeysForLabel = (keys: ReadonlyArray, name: string): ReadonlyArray => - name === "instance" - ? ["service.instance.id"] - : keysForLabel( - Arr.filter(keys, (key) => !isExcludedResourceKey(key)), - name, - ) + keysForLabel( + Arr.filter(keys, (key) => !isExcludedResourceKey(key)), + name, + ) /** * Label names when no series selector narrows them: identity labels plus datapoint and resource diff --git a/packages/query-engine/src/promql/index.ts b/packages/query-engine/src/promql/index.ts index 56b5bebbb9..3e9d38c0d0 100644 --- a/packages/query-engine/src/promql/index.ts +++ b/packages/query-engine/src/promql/index.ts @@ -10,6 +10,7 @@ export { type PromqlMetadataRow, promqlMetadataRowSchema, promqlAttributeValuesQuery, + promqlInstanceValuesQuery, promqlMetadataQuery, promqlServiceNamesQuery, type PromqlSelectorRow, From a87911e089bff620f1e6ef5ccdd262b4dbdd4d1e Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 17:41:57 +0100 Subject: [PATCH 12/18] fix(api): the default discovery start stays inside Date's range --- apps/api/src/routes/v2/prometheus-wire.ts | 2 +- apps/api/src/routes/v2/prometheus.http.test.ts | 3 +++ apps/api/src/routes/v2/prometheus.http.ts | 5 ++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/api/src/routes/v2/prometheus-wire.ts b/apps/api/src/routes/v2/prometheus-wire.ts index 77067e022a..4bfa3456ff 100644 --- a/apps/api/src/routes/v2/prometheus-wire.ts +++ b/apps/api/src/routes/v2/prometheus-wire.ts @@ -9,7 +9,7 @@ const FLOAT = /^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?$/ const RFC3339 = /^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$/ /** The largest epoch ms a JavaScript Date renders. */ -const MAX_DATE_MS = 8.64e15 +export const MAX_DATE_MS = 8.64e15 /** A Prometheus timestamp: Unix seconds with an optional fraction, or RFC3339. Epoch ms. */ export const parseTime = (name: string, text: string): Effect.Effect => { diff --git a/apps/api/src/routes/v2/prometheus.http.test.ts b/apps/api/src/routes/v2/prometheus.http.test.ts index 34bbb904af..386befbafe 100644 --- a/apps/api/src/routes/v2/prometheus.http.test.ts +++ b/apps/api/src/routes/v2/prometheus.http.test.ts @@ -302,6 +302,9 @@ describe("v2 Prometheus discovery API", () => { status: "success", data: ["__name__", "instance", "job"], }) + // The earliest end a Date holds: the default hour before it must not step outside the range. + const edge = await harness.send(key.secret, "/labels", new URLSearchParams({ end: "-8640000000000" })) + expect(edge.status).not.toBe(500) const buildInfo = await harness.send(key.secret, "/status/buildinfo") expect(buildInfo.body).toMatchObject({ data: { version: "3.5.0", vendor: "Maple" } }) expect( diff --git a/apps/api/src/routes/v2/prometheus.http.ts b/apps/api/src/routes/v2/prometheus.http.ts index 4536156e53..640bdd4b9c 100644 --- a/apps/api/src/routes/v2/prometheus.http.ts +++ b/apps/api/src/routes/v2/prometheus.http.ts @@ -14,6 +14,7 @@ import { errorBody, errorStatus, type PrometheusErrorType, + MAX_DATE_MS, parseStep, parseTime, successBody, @@ -104,7 +105,9 @@ const discoveryWindow = (params: Params, defaultMs: number) => const now = yield* Clock.currentTimeMillis const endMs = params.end === undefined ? now : yield* parseTime("end", params.end) const startMs = - params.start === undefined ? endMs - defaultMs : yield* parseTime("start", params.start) + params.start === undefined + ? Math.max(endMs - defaultMs, -MAX_DATE_MS) + : yield* parseTime("start", params.start) return { startMs, endMs } }) From 55bb260b697a98690f6190878804f8191df3ba4c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 18:26:20 +0100 Subject: [PATCH 13/18] test(promql): the service tests refuse reads that are not single-tenant, like the executor --- .../backend/src/services/promql/PromqlQueryService.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/backend/src/services/promql/PromqlQueryService.test.ts b/packages/backend/src/services/promql/PromqlQueryService.test.ts index 2b2709fec6..b4ddddad10 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.test.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.test.ts @@ -208,12 +208,18 @@ const recordingWarehouse = () => { compiledQuery: (_tenant, compiled) => { const query = compiledQueryOf(compiled) statements.push(query.sql) + // The real executor refuses any read that is not single-tenant; so does this stub. + if (query.tenantScope !== "single-tenant") + return Effect.die(`refused ${query.tenantScope} read: ${query.sql}`) return query.decodeRows(rowsFor(query.sql)) }, // Selector reads are size-bounded; record them the same way. compiledQueryBounded: (_tenant, compiled) => { const query = compiledQueryOf(compiled) statements.push(query.sql) + // The real executor refuses any read that is not single-tenant; so does this stub. + if (query.tenantScope !== "single-tenant") + return Effect.die(`refused ${query.tenantScope} read: ${query.sql}`) return query.decodeRows(rowsFor(query.sql)) }, }) From 6eb939803f8d36167845246a8922957817f98a34 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 19:05:11 +0100 Subject: [PATCH 14/18] fix(promql): metadata keeps a metric's two units as two families --- apps/cli/test/promql-source.test.ts | 21 +++++++++++++++++++ .../query-engine/src/ch/queries/promql.ts | 5 +++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/apps/cli/test/promql-source.test.ts b/apps/cli/test/promql-source.test.ts index 45b06471ea..72281502ea 100644 --- a/apps/cli/test/promql-source.test.ts +++ b/apps/cli/test/promql-source.test.ts @@ -20,6 +20,9 @@ import { catalogQuery, promqlCatalogRowSchema, promqlInstanceValuesQuery, + promqlMetadataQuery, + promqlMetadataRowSchema, + prometheusMetadata, promqlSelectorRowSchema, toSelectorData, type CatalogMetric, @@ -333,6 +336,7 @@ const expected = (req: Request, series: ReadonlyArray) => const decodeRows = decodeJsonEachRow(promqlSelectorRowSchema) const decodeCatalog = decodeJsonEachRow(promqlCatalogRowSchema) +const decodeMetadata = decodeJsonEachRow(promqlMetadataRowSchema) const decodeInstances = decodeJsonEachRow(Schema.Struct({ instance: Schema.String })) const catalogOf = (db: Chdb, org: string): ReadonlyArray => { @@ -566,6 +570,23 @@ describeWithChdb("PromQL selector source", () => { const millis = select(db, ORG_A, request("request_duration_milliseconds")).series expect(Arr.map(seconds, (s) => s.labels.job)).toEqual(["seconds-svc"]) expect(Arr.map(millis, (s) => s.labels.job)).toEqual(["millis-svc"]) + const metadata = prometheusMetadata( + decodeMetadata( + db.query( + normalizeSqlForClickHouseClient( + CH.compileUnsafe(promqlMetadataQuery({ limit: 1000 }), { + orgId: OrgId.make(ORG_A), + startTime: Str.slice(0, 19)(chTime(-HOUR_MS)), + endTime: Str.slice(0, 19)(chTime(HOUR_MS)), + }).sql, + ), + ), + ), + ) + expect([ + metadata.has("request_duration_seconds"), + metadata.has("request_duration_milliseconds"), + ]).toEqual([true, true]) }) test("attribute order does not split a series", () => { diff --git a/packages/query-engine/src/ch/queries/promql.ts b/packages/query-engine/src/ch/queries/promql.ts index 5a3af8eece..33a423e9c5 100644 --- a/packages/query-engine/src/ch/queries/promql.ts +++ b/packages/query-engine/src/ch/queries/promql.ts @@ -743,7 +743,8 @@ export function promqlMetadataQuery(opts: { readonly limit: number }) { .select(($) => ({ metricName: $.MetricName, metricType: $.MetricType, - unit: CH.any_($.MetricUnit), + // One name in two units is two metrics (`_seconds`, `_milliseconds`), as in the catalog. + unit: $.MetricUnit, description: CH.any_($.MetricDescription), isMonotonic: sql(T.uint8)`max(${$.IsMonotonic})`, })) @@ -752,7 +753,7 @@ export function promqlMetadataQuery(opts: { readonly limit: number }) { $.Hour.gte(utcHourFloor("startTime")), $.Hour.lte(utcSecondsParam("endTime")), ]) - .groupBy("metricName", "metricType") + .groupBy("metricName", "metricType", "unit") .limit(opts.limit) .format("JSON") } From 1dcf9e23dcc50f05d162af15a631f87b5391a492 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 19:44:49 +0100 Subject: [PATCH 15/18] fix(promql): job label values carry the service namespace A bare /label/job/values listed metric_catalog service names, while a namespaced service's series carry job="namespace/service". Values now come from the metric tables, built as series labels are. --- .../api/src/routes/v2/prometheus.http.test.ts | 6 ++-- apps/cli/test/promql-source.test.ts | 24 ++++++++++++++ .../promql/PromqlQueryService.test.ts | 4 +-- .../src/services/promql/PromqlQueryService.ts | 31 ++++++++++++------- .../query-engine/src/ch/queries/promql.ts | 25 +++++++++------ packages/query-engine/src/promql/index.ts | 2 +- 6 files changed, 65 insertions(+), 27 deletions(-) diff --git a/apps/api/src/routes/v2/prometheus.http.test.ts b/apps/api/src/routes/v2/prometheus.http.test.ts index 386befbafe..407ad12a08 100644 --- a/apps/api/src/routes/v2/prometheus.http.test.ts +++ b/apps/api/src/routes/v2/prometheus.http.test.ts @@ -45,16 +45,14 @@ const testConfig = () => const catalogWarehouse = makeWarehouseServiceStub({ compiledQuery: (_tenant, compiled) => { const query = compiledQueryOf(compiled) - const catalog = - Str.includes("FROM metric_catalog")(query.sql) && !Str.includes("AS serviceName")(query.sql) + const catalog = Str.includes("FROM metric_catalog")(query.sql) return query.decodeRows( catalog ? [{ metricName: "http.requests", metricType: "sum", unit: "", isMonotonic: 1 }] : [], ) }, compiledQueryBounded: (_tenant, compiled) => { const query = compiledQueryOf(compiled) - const catalog = - Str.includes("FROM metric_catalog")(query.sql) && !Str.includes("AS serviceName")(query.sql) + const catalog = Str.includes("FROM metric_catalog")(query.sql) // The bounded read's error union has no decode failure; the fixture rows always decode. return query .decodeRows( diff --git a/apps/cli/test/promql-source.test.ts b/apps/cli/test/promql-source.test.ts index 72281502ea..3d0b4d98ee 100644 --- a/apps/cli/test/promql-source.test.ts +++ b/apps/cli/test/promql-source.test.ts @@ -20,6 +20,7 @@ import { catalogQuery, promqlCatalogRowSchema, promqlInstanceValuesQuery, + promqlJobValuesQuery, promqlMetadataQuery, promqlMetadataRowSchema, prometheusMetadata, @@ -338,6 +339,7 @@ const decodeRows = decodeJsonEachRow(promqlSelectorRowSchema) const decodeCatalog = decodeJsonEachRow(promqlCatalogRowSchema) const decodeMetadata = decodeJsonEachRow(promqlMetadataRowSchema) const decodeInstances = decodeJsonEachRow(Schema.Struct({ instance: Schema.String })) +const decodeJobs = decodeJsonEachRow(Schema.Struct({ job: Schema.String })) const catalogOf = (db: Chdb, org: string): ReadonlyArray => { const compiled = CH.compileUnsafe(catalogQuery(), { @@ -633,4 +635,26 @@ describeWithChdb("PromQL selector source", () => { expect(instances(ORG_A, "metrics_gauge")).toEqual(["i1"]) expect(instances("org_promql_none", "metrics_sum")).toEqual([]) }) + + test("job values carry the namespace, the way series labels do", () => { + insertValues(db, { + ...gauge, + metric: "namespaced.probe", + attributes: {}, + resource: { ...RESOURCE, "service.name": "cart", "service.namespace": "shop" }, + }) + const compiled = CH.compileUnsafe(promqlJobValuesQuery({ table: "metrics_gauge", limit: 100 }), { + orgId: OrgId.make(ORG_A), + startTime: Str.slice(0, 19)(chTime(-HOUR_MS)), + endTime: Str.slice(0, 19)(chTime(HOUR_MS)), + }) + const jobs = Arr.map( + decodeJobs(db.query(normalizeSqlForClickHouseClient(compiled.sql))), + (row) => row.job, + ) + const seriesJobs = Arr.map(select(db, ORG_A, request("namespaced_probe")).series, (s) => s.labels.job) + expect(seriesJobs).toEqual(["shop/cart"]) + expect(Arr.contains(jobs, "shop/cart")).toBe(true) + expect(Arr.contains(jobs, "cart")).toBe(false) + }) }) diff --git a/packages/backend/src/services/promql/PromqlQueryService.test.ts b/packages/backend/src/services/promql/PromqlQueryService.test.ts index b4ddddad10..de7d51d7f8 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.test.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.test.ts @@ -159,6 +159,7 @@ const selectorRow = (attributes: Record) => ({ const recordingWarehouse = () => { const statements: Array = [] const rowsFor = (sql: string): ReadonlyArray> => { + if (Str.includes("AS job")(sql)) return [{ job: "api" }, { job: "worker" }, { job: "" }] if (Str.includes("AS instance")(sql)) return Str.includes("FROM metrics_gauge")(sql) ? [{ instance: "pod-2" }] : [{ instance: "pod-1" }] if (Str.includes("FROM metrics_sum")(sql)) { @@ -182,7 +183,6 @@ const recordingWarehouse = () => { }, ] } - if (Str.includes("AS serviceName")(sql)) return [{ serviceName: "api" }, { serviceName: "worker" }] if (Str.includes("FROM metric_catalog")(sql)) { return [{ metricName: "http.requests", metricType: "sum", unit: "", isMonotonic: 1 }] } @@ -324,7 +324,7 @@ describe("PromqlQueryService discovery", () => { }).pipe(Effect.provide(layer)) }) - it.effect("lists metric names and jobs from the catalog", () => { + it.effect("lists metric names from the catalog and jobs from the metric tables", () => { const { layer } = recordingWarehouse() return Effect.gen(function* () { const service = yield* PromqlQueryService diff --git a/packages/backend/src/services/promql/PromqlQueryService.ts b/packages/backend/src/services/promql/PromqlQueryService.ts index 4102d34c5a..6dd4a8670e 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.ts @@ -39,7 +39,7 @@ import { promqlInstanceValuesQuery, resourceKeysForLabel, promqlMetadataQuery, - promqlServiceNamesQuery, + promqlJobValuesQuery, type PushdownPlan, toPushdownData, toSelectorData, @@ -485,19 +485,28 @@ const make = Effect.gen(function* () { return Arr.flatMap(perTable, (rows) => Arr.map(rows, (row) => row.instance)) }) - const serviceNames = Effect.fn("PromqlQueryService.serviceNames")(function* ( + /** `job` values as series carry them (`namespace/service`): one bounded read per metric table. */ + const jobValues = Effect.fn("PromqlQueryService.jobValues")(function* ( tenant: TenantContext, window: DiscoveryWindow, ) { - const compiled = CH.compile( - promqlServiceNamesQuery({ limit: DISCOVERY_LIMIT }), - windowParams(tenant, window), + const perTable = yield* Effect.forEach( + PROMQL_METRIC_TABLES, + (table) => + warehouse.compiledQuery( + tenant, + CH.compile( + promqlJobValuesQuery({ table, limit: DISCOVERY_LIMIT }), + windowParams(tenant, window), + ), + { profile: "discovery", context: "promqlJobs" }, + ), + { concurrency: "unbounded" }, + ) + return Arr.filter( + Arr.flatMap(perTable, (rows) => Arr.map(rows, (row) => row.job)), + (job) => job !== "", ) - const rows = yield* warehouse.compiledQuery(tenant, compiled, { - profile: "discovery", - context: "promqlJobs", - }) - return Arr.map(rows, (row) => row.serviceName) }) const labelValues = Effect.fn("PromqlQueryService.labelValues")(function* ( @@ -522,7 +531,7 @@ const make = Effect.gen(function* () { limited(sortedUnique(canonicalPromNames(catalog)), input.limit), ) case "job": - return yield* Effect.map(serviceNames(tenant, window), (names) => + return yield* Effect.map(jobValues(tenant, window), (names) => limited(sortedUnique(names), input.limit), ) case "instance": diff --git a/packages/query-engine/src/ch/queries/promql.ts b/packages/query-engine/src/ch/queries/promql.ts index 33a423e9c5..bb49bee6a8 100644 --- a/packages/query-engine/src/ch/queries/promql.ts +++ b/packages/query-engine/src/ch/queries/promql.ts @@ -758,18 +758,25 @@ export function promqlMetadataQuery(opts: { readonly limit: number }) { .format("JSON") } -/** Services that sent metrics in the window: the `job` values a bare label-values call lists. */ -export function promqlServiceNamesQuery(opts: { readonly limit: number }) { - return from(Datasources.metricCatalog) - .select(($) => ({ serviceName: $.ServiceName })) +/** + * Distinct `job` values in one metric table, built as series labels build them: `service.namespace/` + * before the service name when a namespace is set. The catalog keeps no namespace, so this reads the + * table, bounded by org, window and limit. + */ +export function promqlJobValuesQuery(opts: { readonly table: PromqlMetricTable; readonly limit: number }) { + return from(metricTable(opts.table)) + .select(($) => ({ + job: sql( + T.string, + )`multiIf(${$.ResourceAttributes}['service.namespace'] = '', if(${$.ResourceAttributes}['service.name'] = '', ${$.ServiceName}, ${$.ResourceAttributes}['service.name']), concat(${$.ResourceAttributes}['service.namespace'], '/', if(${$.ResourceAttributes}['service.name'] = '', ${$.ServiceName}, ${$.ResourceAttributes}['service.name'])))`, + })) .where(($) => [ $.OrgId.eq(orgIdParam), - $.Hour.gte(utcHourFloor("startTime")), - $.Hour.lte(utcSecondsParam("endTime")), - $.ServiceName.neq(""), + $.TimeUnix.gte(utcSecondsParam("startTime")), + $.TimeUnix.lte(utcSecondsParam("endTime")), ]) - .groupBy("serviceName") - .orderBy(["serviceName", "asc"]) + .groupBy("job") + .orderBy(["job", "asc"]) .limit(opts.limit) .format("JSON") } diff --git a/packages/query-engine/src/promql/index.ts b/packages/query-engine/src/promql/index.ts index 3e9d38c0d0..fd28156169 100644 --- a/packages/query-engine/src/promql/index.ts +++ b/packages/query-engine/src/promql/index.ts @@ -12,7 +12,7 @@ export { promqlAttributeValuesQuery, promqlInstanceValuesQuery, promqlMetadataQuery, - promqlServiceNamesQuery, + promqlJobValuesQuery, type PromqlSelectorRow, promqlSelectorRowSchema, } from "../ch/queries/promql" From 62d0f20925455c938d8337a5035b8c27a127ab53 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 20:06:37 +0100 Subject: [PATCH 16/18] fix(promql): discovery matches series labels exactly - job and instance values are built by the series-label expression (datapoint labels first) and read with millisecond window bounds - /labels lists le when an explicit histogram is in the catalog - RFC3339 times with an impossible clock or calendar field are bad_data, as in Go --- .../api/src/routes/v2/prometheus-wire.test.ts | 10 +++ apps/api/src/routes/v2/prometheus-wire.ts | 39 ++++++++++- apps/cli/test/promql-source.test.ts | 68 ++++++++++--------- .../promql/PromqlQueryService.test.ts | 15 ++-- .../src/services/promql/PromqlQueryService.ts | 49 ++++--------- .../query-engine/src/ch/queries/promql.ts | 46 ++++--------- .../query-engine/src/promql/discovery.test.ts | 20 ++++++ packages/query-engine/src/promql/discovery.ts | 4 +- packages/query-engine/src/promql/index.ts | 3 +- 9 files changed, 144 insertions(+), 110 deletions(-) create mode 100644 packages/query-engine/src/promql/discovery.test.ts diff --git a/apps/api/src/routes/v2/prometheus-wire.test.ts b/apps/api/src/routes/v2/prometheus-wire.test.ts index 60989b91b0..02bbf400ab 100644 --- a/apps/api/src/routes/v2/prometheus-wire.test.ts +++ b/apps/api/src/routes/v2/prometheus-wire.test.ts @@ -27,6 +27,16 @@ describe("Prometheus wire format", () => { // Past what a Date renders (nanoseconds sent as seconds): a 400, not a 500 downstream. const huge = yield* Effect.flip(parseTime("time", "1752580800123456789")) expect(huge.message).toMatch(/cannot parse/) + // A clock or calendar field Date would roll over is refused, as Go's time.Parse refuses it. + for (const text of [ + "2026-07-15T24:00:00Z", + "2026-02-30T00:00:00Z", + "2026-07-15T12:60:00Z", + "2026-13-01T00:00:00Z", + ]) { + expect((yield* Effect.flip(parseTime("time", text))).message).toMatch(/cannot parse/) + } + expect(yield* parseTime("time", "2028-02-29T23:59:59Z")).toBe(Date.UTC(2028, 1, 29, 23, 59, 59)) }), ) }) diff --git a/apps/api/src/routes/v2/prometheus-wire.ts b/apps/api/src/routes/v2/prometheus-wire.ts index 4bfa3456ff..f2b7e69040 100644 --- a/apps/api/src/routes/v2/prometheus-wire.ts +++ b/apps/api/src/routes/v2/prometheus-wire.ts @@ -6,7 +6,39 @@ import { parseDurationMs, type QueryResult } from "@maple/promql" import { PromqlBadQueryError } from "@maple/backend/services/promql/PromqlQueryService" const FLOAT = /^[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?$/ -const RFC3339 = /^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$/ +const RFC3339 = /^(\d{4})-(\d{2})-(\d{2})[Tt](\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:[Zz]|[+-](\d{2}):(\d{2}))$/ + +/** + * Whether an RFC3339 match names a real calendar date and clock time, as Go's `time.Parse` + * checks. JavaScript's Date would roll `24:00` or February 30 into another instant. + */ +const isRealTime = (match: RegExpExecArray): boolean => { + const [ + , + year = 0, + month = 0, + day = 0, + hour = 0, + minute = 0, + second = 0, + , + offsetHour = 0, + offsetMinute = 0, + ] = Arr.map(match, (part) => (part === undefined ? undefined : Number(part))) + // Day 0 of the next month is this month's last day. + const daysInMonth = new Date(Date.UTC(year, month, 0)).getUTCDate() + return ( + month >= 1 && + month <= 12 && + day >= 1 && + day <= daysInMonth && + hour < 24 && + minute < 60 && + second < 60 && + offsetHour < 24 && + offsetMinute < 60 + ) +} /** The largest epoch ms a JavaScript Date renders. */ export const MAX_DATE_MS = 8.64e15 @@ -16,8 +48,9 @@ export const parseTime = (name: string, text: string): Effect.Effect) => const decodeRows = decodeJsonEachRow(promqlSelectorRowSchema) const decodeCatalog = decodeJsonEachRow(promqlCatalogRowSchema) const decodeMetadata = decodeJsonEachRow(promqlMetadataRowSchema) -const decodeInstances = decodeJsonEachRow(Schema.Struct({ instance: Schema.String })) -const decodeJobs = decodeJsonEachRow(Schema.Struct({ job: Schema.String })) +const decodeValues = decodeJsonEachRow(Schema.Struct({ value: Schema.String })) const catalogOf = (db: Chdb, org: string): ReadonlyArray => { const compiled = CH.compileUnsafe(catalogQuery(), { @@ -619,42 +617,50 @@ describeWithChdb("PromQL selector source", () => { expect(select(db, ORG_A, request("ordered_total")).series).toHaveLength(1) }) - test("instance values come from the metric tables, scoped to the org", () => { - const instances = (org: string, table: "metrics_sum" | "metrics_gauge") => { - const compiled = CH.compileUnsafe(promqlInstanceValuesQuery({ table, limit: 100 }), { - orgId: OrgId.make(org), - startTime: Str.slice(0, 19)(chTime(-HOUR_MS)), - endTime: Str.slice(0, 19)(chTime(HOUR_MS)), - }) - return Arr.map( - decodeInstances(db.query(normalizeSqlForClickHouseClient(compiled.sql))), - (row) => row.instance, + test("job and instance values are built like series labels, within the millisecond window", () => { + const identity = ( + org: string, + table: "metrics_sum" | "metrics_gauge", + label: "job" | "instance", + endOffsetMs = HOUR_MS, + ) => + Arr.map( + decodeValues( + db.query( + normalizeSqlForClickHouseClient( + CH.compileUnsafe(promqlIdentityValuesQuery({ table, label, limit: 100 }), { + orgId: OrgId.make(org), + startMs: BASE_MS - HOUR_MS, + endMs: BASE_MS + endOffsetMs, + }).sql, + ), + ), + ), + (row) => row.value, ) - } - expect(instances(ORG_A, "metrics_sum")).toEqual(["i1"]) - expect(instances(ORG_A, "metrics_gauge")).toEqual(["i1"]) - expect(instances("org_promql_none", "metrics_sum")).toEqual([]) - }) + expect(identity(ORG_A, "metrics_sum", "instance")).toEqual(["i1"]) + expect(identity("org_promql_none", "metrics_sum", "instance")).toEqual([]) - test("job values carry the namespace, the way series labels do", () => { insertValues(db, { ...gauge, metric: "namespaced.probe", attributes: {}, resource: { ...RESOURCE, "service.name": "cart", "service.namespace": "shop" }, }) - const compiled = CH.compileUnsafe(promqlJobValuesQuery({ table: "metrics_gauge", limit: 100 }), { - orgId: OrgId.make(ORG_A), - startTime: Str.slice(0, 19)(chTime(-HOUR_MS)), - endTime: Str.slice(0, 19)(chTime(HOUR_MS)), - }) - const jobs = Arr.map( - decodeJobs(db.query(normalizeSqlForClickHouseClient(compiled.sql))), - (row) => row.job, - ) const seriesJobs = Arr.map(select(db, ORG_A, request("namespaced_probe")).series, (s) => s.labels.job) expect(seriesJobs).toEqual(["shop/cart"]) - expect(Arr.contains(jobs, "shop/cart")).toBe(true) - expect(Arr.contains(jobs, "cart")).toBe(false) + expect(Arr.contains(identity(ORG_A, "metrics_gauge", "job"), "shop/cart")).toBe(true) + expect(Arr.contains(identity(ORG_A, "metrics_gauge", "job"), "cart")).toBe(false) + + // Datapoint labels win over the resource, as in series labels; the one sample sits 750ms past + // a whole second, so a seconds-floored end would miss it. + insertValues(db, { + ...gauge, + metric: "scraped.probe", + attributes: { job: "scrape-job", instance: "host:9090" }, + samples: [[2 * HOUR_MS + 750, 1]], + }) + expect(identity(ORG_A, "metrics_gauge", "job", 2 * HOUR_MS + 750)).toContain("scrape-job") + expect(identity(ORG_A, "metrics_gauge", "instance", 2 * HOUR_MS + 750)).toContain("host:9090") }) }) diff --git a/packages/backend/src/services/promql/PromqlQueryService.test.ts b/packages/backend/src/services/promql/PromqlQueryService.test.ts index de7d51d7f8..43e66c76f3 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.test.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.test.ts @@ -159,9 +159,12 @@ const selectorRow = (attributes: Record) => ({ const recordingWarehouse = () => { const statements: Array = [] const rowsFor = (sql: string): ReadonlyArray> => { - if (Str.includes("AS job")(sql)) return [{ job: "api" }, { job: "worker" }, { job: "" }] - if (Str.includes("AS instance")(sql)) - return Str.includes("FROM metrics_gauge")(sql) ? [{ instance: "pod-2" }] : [{ instance: "pod-1" }] + if (Str.includes("HAVING (value != '')")(sql) && Str.includes("service.namespace")(sql)) { + return [{ value: "api" }, { value: "worker" }] + } + if (Str.includes("HAVING (value != '')")(sql)) { + return Str.includes("FROM metrics_gauge")(sql) ? [{ value: "pod-2" }] : [{ value: "pod-1" }] + } if (Str.includes("FROM metrics_sum")(sql)) { return [selectorRow({ "http.route": "/checkout" }), selectorRow({ "http.route": "/cart" })] } @@ -304,7 +307,11 @@ describe("PromqlQueryService discovery", () => { const service = yield* PromqlQueryService const values = yield* service.labelValues(tenant, { ...window, matches: [], name: "instance" }) expect(values).toEqual(["pod-1", "pod-2"]) - const reads = Arr.filter(statements, (sql) => Str.includes("AS instance")(sql)) + const reads = Arr.filter( + statements, + (sql) => + Str.includes("service.instance.id")(sql) && Str.includes("HAVING (value != '')")(sql), + ) expect(reads).toHaveLength(4) for (const sql of reads) expect(sql).toMatch(ORG_FILTER) expect(Arr.some(statements, (sql) => Str.includes("attribute_values_hourly")(sql))).toBe(false) diff --git a/packages/backend/src/services/promql/PromqlQueryService.ts b/packages/backend/src/services/promql/PromqlQueryService.ts index 6dd4a8670e..c24943e1db 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.ts @@ -36,10 +36,9 @@ import { matchesValue, prometheusMetadata, promqlAttributeValuesQuery, - promqlInstanceValuesQuery, resourceKeysForLabel, promqlMetadataQuery, - promqlJobValuesQuery, + promqlIdentityValuesQuery, type PushdownPlan, toPushdownData, toSelectorData, @@ -464,49 +463,27 @@ const make = Effect.gen(function* () { return Arr.flatten(values) }) - /** `instance` values: one bounded read per metric table, since no rollup keeps id-like keys. */ - const instanceValues = Effect.fn("PromqlQueryService.instanceValues")(function* ( + /** `job` or `instance` values as series carry them: one bounded read per metric table. */ + const identityValues = Effect.fn("PromqlQueryService.identityValues")(function* ( tenant: TenantContext, window: DiscoveryWindow, + label: "job" | "instance", ) { const perTable = yield* Effect.forEach( PROMQL_METRIC_TABLES, (table) => warehouse.compiledQuery( tenant, - CH.compile( - promqlInstanceValuesQuery({ table, limit: DISCOVERY_LIMIT }), - windowParams(tenant, window), - ), - { profile: "discovery", context: "promqlInstances" }, - ), - { concurrency: "unbounded" }, - ) - return Arr.flatMap(perTable, (rows) => Arr.map(rows, (row) => row.instance)) - }) - - /** `job` values as series carry them (`namespace/service`): one bounded read per metric table. */ - const jobValues = Effect.fn("PromqlQueryService.jobValues")(function* ( - tenant: TenantContext, - window: DiscoveryWindow, - ) { - const perTable = yield* Effect.forEach( - PROMQL_METRIC_TABLES, - (table) => - warehouse.compiledQuery( - tenant, - CH.compile( - promqlJobValuesQuery({ table, limit: DISCOVERY_LIMIT }), - windowParams(tenant, window), - ), - { profile: "discovery", context: "promqlJobs" }, + CH.compile(promqlIdentityValuesQuery({ table, label, limit: DISCOVERY_LIMIT }), { + orgId: tenant.orgId, + startMs: window.startMs, + endMs: window.endMs, + }), + { profile: "discovery", context: label === "job" ? "promqlJobs" : "promqlInstances" }, ), { concurrency: "unbounded" }, ) - return Arr.filter( - Arr.flatMap(perTable, (rows) => Arr.map(rows, (row) => row.job)), - (job) => job !== "", - ) + return Arr.flatMap(perTable, (rows) => Arr.map(rows, (row) => row.value)) }) const labelValues = Effect.fn("PromqlQueryService.labelValues")(function* ( @@ -531,11 +508,11 @@ const make = Effect.gen(function* () { limited(sortedUnique(canonicalPromNames(catalog)), input.limit), ) case "job": - return yield* Effect.map(jobValues(tenant, window), (names) => + return yield* Effect.map(identityValues(tenant, window, "job"), (names) => limited(sortedUnique(names), input.limit), ) case "instance": - return yield* Effect.map(instanceValues(tenant, window), (values) => + return yield* Effect.map(identityValues(tenant, window, "instance"), (values) => limited(sortedUnique(values), input.limit), ) default: diff --git a/packages/query-engine/src/ch/queries/promql.ts b/packages/query-engine/src/ch/queries/promql.ts index bb49bee6a8..c6d75eae31 100644 --- a/packages/query-engine/src/ch/queries/promql.ts +++ b/packages/query-engine/src/ch/queries/promql.ts @@ -12,7 +12,7 @@ import * as CH from "@maple-dev/effect-orm/expr" import { CHNumber } from "../schema" import * as Datasources from "@maple/domain/tinybird/datasources" import { orgId, orgIdParam, utcSecondsParam } from "../tables" -import { EXCLUDED_KEY } from "../../promql/labels" +import { EXCLUDED_KEY, labelValueExpr } from "../../promql/labels" import { utcHourFloor } from "./query-helpers" /** Joins a stored metric's name and unit into the identity a selector row reports. */ @@ -759,46 +759,26 @@ export function promqlMetadataQuery(opts: { readonly limit: number }) { } /** - * Distinct `job` values in one metric table, built as series labels build them: `service.namespace/` - * before the service name when a namespace is set. The catalog keeps no namespace, so this reads the - * table, bounded by org, window and limit. + * Distinct `job` or `instance` values in one metric table, built by the same expression as series + * labels (a datapoint label first, then the resource: `service.namespace/service`, + * `service.instance.id`). No rollup keeps these, so this reads the table, bounded by org, the + * millisecond window and the limit. */ -export function promqlJobValuesQuery(opts: { readonly table: PromqlMetricTable; readonly limit: number }) { - return from(metricTable(opts.table)) - .select(($) => ({ - job: sql( - T.string, - )`multiIf(${$.ResourceAttributes}['service.namespace'] = '', if(${$.ResourceAttributes}['service.name'] = '', ${$.ServiceName}, ${$.ResourceAttributes}['service.name']), concat(${$.ResourceAttributes}['service.namespace'], '/', if(${$.ResourceAttributes}['service.name'] = '', ${$.ServiceName}, ${$.ResourceAttributes}['service.name'])))`, - })) - .where(($) => [ - $.OrgId.eq(orgIdParam), - $.TimeUnix.gte(utcSecondsParam("startTime")), - $.TimeUnix.lte(utcSecondsParam("endTime")), - ]) - .groupBy("job") - .orderBy(["job", "asc"]) - .limit(opts.limit) - .format("JSON") -} - -/** - * Distinct `service.instance.id` values (the `instance` label) in one metric table. No rollup keeps - * id-like keys, so this reads the table itself, bounded by org, window and limit. - */ -export function promqlInstanceValuesQuery(opts: { +export function promqlIdentityValuesQuery(opts: { readonly table: PromqlMetricTable + readonly label: "job" | "instance" readonly limit: number }) { return from(metricTable(opts.table)) - .select(($) => ({ instance: sql(T.string)`${$.ResourceAttributes}['service.instance.id']` })) + .select(($) => ({ value: labelValueExpr($, opts.label) ?? sql(T.string)`''` })) .where(($) => [ $.OrgId.eq(orgIdParam), - $.TimeUnix.gte(utcSecondsParam("startTime")), - $.TimeUnix.lte(utcSecondsParam("endTime")), - sql.cond`${$.ResourceAttributes}['service.instance.id'] != ''`, + sql.cond`${$.TimeUnix} >= fromUnixTimestamp64Milli(toInt64(${param.int("startMs")}))`, + sql.cond`${$.TimeUnix} <= fromUnixTimestamp64Milli(toInt64(${param.int("endMs")}))`, ]) - .groupBy("instance") - .orderBy(["instance", "asc"]) + .groupBy("value") + .having(() => [sql.cond`value != ''`]) + .orderBy(["value", "asc"]) .limit(opts.limit) .format("JSON") } diff --git a/packages/query-engine/src/promql/discovery.test.ts b/packages/query-engine/src/promql/discovery.test.ts new file mode 100644 index 0000000000..4189f88756 --- /dev/null +++ b/packages/query-engine/src/promql/discovery.test.ts @@ -0,0 +1,20 @@ +import { Array as Arr } from "effect" +import { describe, expect, it } from "vitest" +import { labelNamesFrom } from "./discovery" +import type { CatalogMetric } from "./names" + +const metric = (metricType: CatalogMetric["metricType"]): CatalogMetric => ({ + metricName: "http.server.request.duration", + metricType, + isMonotonic: metricType === "sum", + unit: "s", +}) + +describe("labelNamesFrom", () => { + it("lists le when an explicit histogram is in the catalog, whose bucket series carry it", () => { + expect(Arr.contains(labelNamesFrom([metric("histogram")], [], []), "le")).toBe(true) + expect( + Arr.contains(labelNamesFrom([metric("sum"), metric("exponential_histogram")], [], []), "le"), + ).toBe(false) + }) +}) diff --git a/packages/query-engine/src/promql/discovery.ts b/packages/query-engine/src/promql/discovery.ts index 088f212790..9cf0c6c02b 100644 --- a/packages/query-engine/src/promql/discovery.ts +++ b/packages/query-engine/src/promql/discovery.ts @@ -41,12 +41,14 @@ export const labelNamesFrom = ( resourceKeys: ReadonlyArray, ): ReadonlyArray => { if (catalog.length === 0) return [] - // `le` is left out: its values come from bucket bounds, which no rollup lists. const resource = Arr.filter(resourceKeys, (key) => !isExcludedResourceKey(key)) + // Only explicit histograms expose `_bucket` series, which carry `le`. + const hasBuckets = Arr.some(catalog, (metric) => metric.metricType === "histogram") return sortedUnique([ "__name__", "job", "instance", + ...(hasBuckets ? ["le"] : []), ...Arr.map([...attributeKeys, ...resource], normalizeLabelName), ]) } diff --git a/packages/query-engine/src/promql/index.ts b/packages/query-engine/src/promql/index.ts index fd28156169..5017f4c22c 100644 --- a/packages/query-engine/src/promql/index.ts +++ b/packages/query-engine/src/promql/index.ts @@ -10,9 +10,8 @@ export { type PromqlMetadataRow, promqlMetadataRowSchema, promqlAttributeValuesQuery, - promqlInstanceValuesQuery, + promqlIdentityValuesQuery, promqlMetadataQuery, - promqlJobValuesQuery, type PromqlSelectorRow, promqlSelectorRowSchema, } from "../ch/queries/promql" From da29622a8532b2de207221b3b2c1e1617e763678 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 20:11:16 +0100 Subject: [PATCH 17/18] fix(api): RFC3339 month lengths are computed, so year 0000 keeps its leap day --- apps/api/src/routes/v2/prometheus-wire.test.ts | 5 +++++ apps/api/src/routes/v2/prometheus-wire.ts | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/api/src/routes/v2/prometheus-wire.test.ts b/apps/api/src/routes/v2/prometheus-wire.test.ts index 02bbf400ab..32223b3c8a 100644 --- a/apps/api/src/routes/v2/prometheus-wire.test.ts +++ b/apps/api/src/routes/v2/prometheus-wire.test.ts @@ -37,6 +37,11 @@ describe("Prometheus wire format", () => { expect((yield* Effect.flip(parseTime("time", text))).message).toMatch(/cannot parse/) } expect(yield* parseTime("time", "2028-02-29T23:59:59Z")).toBe(Date.UTC(2028, 1, 29, 23, 59, 59)) + // Year 0000 is a leap year; Date.UTC would read it as 1900. + expect(Number.isFinite(yield* parseTime("time", "0000-02-29T00:00:00Z"))).toBe(true) + expect((yield* Effect.flip(parseTime("time", "1900-02-29T00:00:00Z"))).message).toMatch( + /cannot parse/, + ) }), ) }) diff --git a/apps/api/src/routes/v2/prometheus-wire.ts b/apps/api/src/routes/v2/prometheus-wire.ts index f2b7e69040..a51373ae59 100644 --- a/apps/api/src/routes/v2/prometheus-wire.ts +++ b/apps/api/src/routes/v2/prometheus-wire.ts @@ -25,8 +25,9 @@ const isRealTime = (match: RegExpExecArray): boolean => { offsetHour = 0, offsetMinute = 0, ] = Arr.map(match, (part) => (part === undefined ? undefined : Number(part))) - // Day 0 of the next month is this month's last day. - const daysInMonth = new Date(Date.UTC(year, month, 0)).getUTCDate() + // Arithmetic, not Date.UTC, which reads years 0 to 99 as 1900 to 1999. + const isLeap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) + const daysInMonth = month === 2 ? (isLeap ? 29 : 28) : [4, 6, 9, 11].includes(month) ? 30 : 31 return ( month >= 1 && month <= 12 && From 97602053c0765ae1b5e1c1f8439bd22c6dd624f5 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 20:18:11 +0100 Subject: [PATCH 18/18] test(api): GET /metadata answers family-keyed metadata the way Grafana reads it --- .../api/src/routes/v2/prometheus.http.test.ts | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/apps/api/src/routes/v2/prometheus.http.test.ts b/apps/api/src/routes/v2/prometheus.http.test.ts index 407ad12a08..88be6aa585 100644 --- a/apps/api/src/routes/v2/prometheus.http.test.ts +++ b/apps/api/src/routes/v2/prometheus.http.test.ts @@ -47,7 +47,17 @@ const catalogWarehouse = makeWarehouseServiceStub({ const query = compiledQueryOf(compiled) const catalog = Str.includes("FROM metric_catalog")(query.sql) return query.decodeRows( - catalog ? [{ metricName: "http.requests", metricType: "sum", unit: "", isMonotonic: 1 }] : [], + catalog + ? [ + { + metricName: "http.requests", + metricType: "sum", + unit: "", + isMonotonic: 1, + description: "Requests", + }, + ] + : [], ) }, compiledQueryBounded: (_tenant, compiled) => { @@ -56,7 +66,17 @@ const catalogWarehouse = makeWarehouseServiceStub({ // The bounded read's error union has no decode failure; the fixture rows always decode. return query .decodeRows( - catalog ? [{ metricName: "http.requests", metricType: "sum", unit: "", isMonotonic: 1 }] : [], + catalog + ? [ + { + metricName: "http.requests", + metricType: "sum", + unit: "", + isMonotonic: 1, + description: "Requests", + }, + ] + : [], ) .pipe(Effect.orDie) }, @@ -303,6 +323,11 @@ describe("v2 Prometheus discovery API", () => { // The earliest end a Date holds: the default hour before it must not step outside the range. const edge = await harness.send(key.secret, "/labels", new URLSearchParams({ end: "-8640000000000" })) expect(edge.status).not.toBe(500) + // Metadata is keyed by the Prometheus family name, with the OTel description as help. + expect((await harness.send(key.secret, `/metadata?${window}`)).body).toEqual({ + status: "success", + data: { http_requests_total: [{ type: "counter", help: "Requests", unit: "" }] }, + }) const buildInfo = await harness.send(key.secret, "/status/buildinfo") expect(buildInfo.body).toMatchObject({ data: { version: "3.5.0", vendor: "Maple" } }) expect(