From 5aef3bcc34945c6c3848904b2b4a70607e8f14e9 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 15:28:41 +0100 Subject: [PATCH 1/4] perf(promql): evaluate range functions and aggregations in ClickHouse A selector the evaluator reads returns every bucket of every series, about 140 bytes of JSON each. A 24 h Grafana panel over 1,000 series at a 60 s step is 193 MB of JSON and 350 MB of heap, far past a Worker's 128 MB. The common panel shapes now run in ClickHouse whole: a range function over a selector (rate, increase, delta, irate, idelta, resets, changes, the *_over_time family) and sum, avg, min, max, count or group by labels over one or over a bare selector. Each bucket fans out to the steps whose window holds it and the window is recombined with the evaluator's exact rules, so one value per series or group and step comes back. The source declines a pushdown it cannot make exact (more than one stored metric, an `le` matcher) and the evaluator reads the selector as before. On 12.7 M rows in chDB (scripts/bench-promql.ts): the 24 h, 60 s-step `sum by (status) (rate(...[4m]))` went from 193 MB of JSON, 351 MB of heap and 1,062 ms of evaluation to 0.1 MB, 30 MB and 2 ms; SQL time is unchanged at about 1.5 s. Every corpus query still equals the reference with pushdown on and off (apps/cli/test/promql-engine.test.ts), and a selector read past 24 MB now fails with a clear execution error instead of exhausting the isolate. --- apps/cli/scripts/bench-promql.ts | 204 ++++++++++ apps/cli/test/promql-engine.test.ts | 52 ++- knip.json | 1 + lib/promql/src/engine/evaluate.ts | 35 +- lib/promql/src/engine/index.ts | 9 +- lib/promql/src/engine/plan.ts | 116 +++++- lib/promql/src/engine/types.ts | 53 +++ .../promql/PromqlQueryService.test.ts | 24 ++ .../src/services/promql/PromqlQueryService.ts | 125 ++++-- .../query-engine/src/ch/queries/promql.ts | 363 ++++++++++++++++++ packages/query-engine/src/promql/index.ts | 2 + packages/query-engine/src/promql/source.ts | 127 +++++- 12 files changed, 1066 insertions(+), 45 deletions(-) create mode 100644 apps/cli/scripts/bench-promql.ts diff --git a/apps/cli/scripts/bench-promql.ts b/apps/cli/scripts/bench-promql.ts new file mode 100644 index 0000000000..1fa4d96ee5 --- /dev/null +++ b/apps/cli/scripts/bench-promql.ts @@ -0,0 +1,204 @@ +/** + * PromQL on the warehouse at volume, on chDB. + * + * MAPLE_TELEMETRY=off bun run scripts/bench-promql.ts [--series 1000] [--hours 24] + * + * Loads a counter with `--series` series and a 12-bucket histogram with a fifth of + * that, scraped every 15 s for `--hours`, plus an unrelated metric of the same size + * the scan must skip. Times each query's SQL (per selector) and evaluation. + */ +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Effect, Result } from "effect" +import schemaSql from "@maple/local-server/schema/local-schema.sql" with { type: "text" } +import { Chdb } from "@maple/local-server/chdb" +import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" +import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" +import * as CH from "@maple/query-engine/ch" +import { + buildPushdownQuery, + buildSelectorQueries, + catalogFromRows, + catalogQuery, + promqlCatalogRowSchema, + promqlPushdownRowSchema, + promqlSelectorRowSchema, + toPushdownData, + toSelectorData, +} from "@maple/query-engine/promql" +import { + evaluate, + type Expr, + parse, + planPushdowns, + planRequests, + type PushdownData, + type SelectorData, + type VectorSelector, +} from "@maple/promql" +import { OrgId } from "@maple/domain" + +const arg = (name: string, fallback: number): number => { + const index = process.argv.indexOf(`--${name}`) + return index === -1 ? fallback : Number(process.argv[index + 1]) +} +const SERIES = arg("series", 1000) +const HOURS = arg("hours", 24) +const ORG = "org_bench" +const HOUR_MS = 3_600_000 +const END = Math.floor(Date.now() / HOUR_MS) * HOUR_MS - HOUR_MS +const START = END - HOURS * HOUR_MS +const POINTS = (HOURS * HOUR_MS) / 15_000 + +const root = mkdtempSync(join(tmpdir(), "maple-promql-bench-")) +const db = Chdb.open({ dataDir: join(root, "data"), schemaSql }) + +const time = (f: () => A): [A, number] => { + const t0 = performance.now() + const out = f() + return [out, performance.now() - t0] +} + +const load = () => { + const ts = `fromUnixTimestamp64Milli(toInt64(${START} + (number % ${POINTS}) * 15000))` + const counter = (name: string) => ` + INSERT INTO metrics_sum (OrgId, ResourceAttributes, ServiceName, MetricName, Attributes, StartTimeUnix, TimeUnix, Value, AggregationTemporality, IsMonotonic) + SELECT '${ORG}', map('service.name', 'svc-' || toString(intDiv(number, ${POINTS}) % 10), 'k8s.pod.name', 'pod-' || toString(intDiv(number, ${POINTS}))), + 'svc-' || toString(intDiv(number, ${POINTS}) % 10), '${name}', + map('route', '/r' || toString(intDiv(number, ${POINTS}) % 50), 'status', toString(200 + (intDiv(number, ${POINTS}) % 4) * 100)), + fromUnixTimestamp64Milli(toInt64(${START - HOUR_MS})), ${ts}, + toFloat64((number % ${POINTS}) * (1 + intDiv(number, ${POINTS}) % 7)), 2, true + FROM numbers(${SERIES * POINTS})` + db.exec(counter("http_requests_total")) + db.exec(counter("other_requests_total")) + db.exec(` + INSERT INTO metrics_histogram (OrgId, ResourceAttributes, ServiceName, MetricName, MetricUnit, Attributes, StartTimeUnix, TimeUnix, Count, Sum, BucketCounts, ExplicitBounds, AggregationTemporality) + SELECT '${ORG}', map('service.name', 'svc-' || toString(intDiv(number, ${POINTS}) % 10)), 'svc-' || toString(intDiv(number, ${POINTS}) % 10), + 'http.server.request.duration', 's', map('http.route', '/r' || toString(intDiv(number, ${POINTS}) % 50)), + fromUnixTimestamp64Milli(toInt64(${START - HOUR_MS})), ${ts}, + (number % ${POINTS}) * 12, toFloat64(number % ${POINTS}) * 3.5, + arrayMap(i -> toUInt64(number % ${POINTS}), range(12)), + [0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5], 2 + FROM numbers(${Math.max(1, Math.floor(SERIES / 5)) * POINTS})`) +} + +const [, loadMs] = time(load) +console.log( + `loaded ${(SERIES * POINTS * 2 + Math.floor(SERIES / 5) * POINTS).toLocaleString()} rows in ${(loadMs / 1000).toFixed(1)}s`, +) + +const catalogRows = decodeJsonEachRow(promqlCatalogRowSchema)( + db.query( + normalizeSqlForClickHouseClient( + CH.compileUnsafe(catalogQuery(), { + orgId: OrgId.make(ORG), + startTime: new Date(START - 2 * HOUR_MS).toISOString().slice(0, 19).replace("T", " "), + endTime: new Date(END + HOUR_MS).toISOString().slice(0, 19).replace("T", " "), + }).sql, + ), + ), +) +const catalog = catalogFromRows(catalogRows) +const decodeRows = decodeJsonEachRow(promqlSelectorRowSchema) + +const decodePushdown = decodeJsonEachRow(promqlPushdownRowSchema) +const pushdownEnabled = !process.argv.includes("--no-pushdown") + +const run = (query: string, params: { startMs: number; endMs: number; stepMs: number }) => { + const parsed = parse(query) + if (Result.isFailure(parsed)) throw new Error(parsed.failure.message) + const data = new Map() + const pushed = new Map() + const skipped = new Set() + let sqlMs = 0 + let rows = 0 + let buckets = 0 + let jsonBytes = 0 + for (const [node, candidate] of pushdownEnabled ? planPushdowns(parsed.success, params) : []) { + const plan = buildPushdownQuery(candidate.request, catalog, { + orgId: OrgId.make(ORG), + maxSeries: 10_000, + }) + if (plan === undefined) continue + const sql = normalizeSqlForClickHouseClient(Effect.runSync(plan.compiled).sql) + const [raw, ms] = time(() => db.query(sql)) + sqlMs += ms + jsonBytes += raw.length + const decoded = decodePushdown(raw) + rows += decoded.length + for (const r of decoded) buckets += r.values.length + pushed.set(node, toPushdownData(candidate.request, plan, decoded)) + skipped.add(candidate.selector) + } + for (const [selector, request] of planRequests(parsed.success, params)) { + if (skipped.has(selector)) continue + const results = buildSelectorQueries(request, catalog, { + orgId: OrgId.make(ORG), + maxSeries: 10_000, + }).map((plan) => { + const sql = normalizeSqlForClickHouseClient(Effect.runSync(plan.compiled).sql) + const [raw, ms] = time(() => db.query(sql)) + sqlMs += ms + jsonBytes += typeof raw === "string" ? raw.length : JSON.stringify(raw).length + const decoded = decodeRows(raw) + rows += decoded.length + return { plan, rows: decoded } + }) + const selectorData = toSelectorData(request, results) + for (const s of selectorData.series) buckets += s.buckets.length + data.set(selector, selectorData) + } + Bun.gc(true) + const heapMb = process.memoryUsage().heapUsed / 1e6 + const [result, evalMs] = time(() => evaluate({ expr: parsed.success, params, data, pushed })) + if (Result.isFailure(result)) throw new Error(result.failure.message) + const out = result.success + const outSeries = out.resultType === "matrix" || out.resultType === "vector" ? out.result.length : 1 + Bun.gc(true) + const afterMb = process.memoryUsage().heapUsed / 1e6 + return { sqlMs, evalMs, rows, buckets, outSeries, heapMb, afterMb, jsonMb: jsonBytes / 1e6 } +} + +const cases: Array<[string, string, { startMs: number; endMs: number; stepMs: number }]> = [ + [ + "24h step 5m", + "sum by (route) (rate(http_requests_total[5m]))", + { startMs: START + HOUR_MS, endMs: END, stepMs: 300_000 }, + ], + [ + "24h step 60s", + "sum by (status) (rate(http_requests_total[$r]))".replace("$r", "4m"), + { startMs: START + HOUR_MS, endMs: END, stepMs: 60_000 }, + ], + [ + "1h step 15s", + 'sum by (route) (rate(http_requests_total{status="200"}[1m]))', + { startMs: END - HOUR_MS, endMs: END, stepMs: 15_000 }, + ], + [ + "6h p95", + "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket[5m])))", + { startMs: END - 6 * HOUR_MS, endMs: END, stepMs: 60_000 }, + ], + [ + "instant", + "topk(5, sum by (k8s_pod_name) (increase(http_requests_total[1h])))", + { startMs: END, endMs: END, stepMs: 0 }, + ], + [ + "regex 6h", + 'sum(rate(http_requests_total{route=~"/r1.*"}[5m]))', + { startMs: END - 6 * HOUR_MS, endMs: END, stepMs: 60_000 }, + ], +] + +for (const [name, query, params] of cases) { + run(query, params) + const r = run(query, params) + console.log( + `${name.padEnd(12)} sql ${r.sqlMs.toFixed(0).padStart(6)}ms eval ${r.evalMs.toFixed(0).padStart(5)}ms rows ${String(r.rows).padStart(5)} buckets ${String(r.buckets).padStart(8)} out ${r.outSeries} json ${r.jsonMb.toFixed(0)}MB heap ${r.heapMb.toFixed(0)}->${r.afterMb.toFixed(0)}MB ${query}`, + ) +} +db.close() +rmSync(root, { recursive: true, force: true }) diff --git a/apps/cli/test/promql-engine.test.ts b/apps/cli/test/promql-engine.test.ts index 5066c0fb24..97b3c808da 100644 --- a/apps/cli/test/promql-engine.test.ts +++ b/apps/cli/test/promql-engine.test.ts @@ -15,19 +15,25 @@ import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" import * as CH from "@maple/query-engine/ch" import { + buildPushdownQuery, buildSelectorQueries, catalogFromRows, catalogQuery, promqlCatalogRowSchema, + promqlPushdownRowSchema, promqlSelectorRowSchema, + toPushdownData, toSelectorData, type CatalogMetric, } from "@maple/query-engine/promql" import { evaluate, + type Expr, memorySource, parse, + planPushdowns, planRequests, + type PushdownData, type QueryParams, type QueryResult, type RawSeries, @@ -110,13 +116,35 @@ const insertHistogram = (db: Chdb, service: string): void => { } const decodeRows = decodeJsonEachRow(promqlSelectorRowSchema) +const decodePushdown = decodeJsonEachRow(promqlPushdownRowSchema) +const options = { orgId: OrgId.make(ORG), maxSeries: 10_000 } +let pushedCount = 0 const decodeCatalog = decodeJsonEachRow(promqlCatalogRowSchema) -const sqlRun = (db: Chdb, catalog: ReadonlyArray, query: string, params: QueryParams) => { +const sqlRun = ( + db: Chdb, + catalog: ReadonlyArray, + query: string, + params: QueryParams, + pushdown: boolean, +) => { const parsed = parse(query) if (Result.isFailure(parsed)) return Result.fail(parsed.failure.message) + const pushed = new Map() + const skipped = new Set() + for (const [node, candidate] of pushdown ? planPushdowns(parsed.success, params) : []) { + const plan = buildPushdownQuery(candidate.request, catalog, options) + if (plan === undefined) continue + const rows = decodePushdown( + db.query(normalizeSqlForClickHouseClient(Effect.runSync(plan.compiled).sql)), + ) + pushed.set(node, toPushdownData(candidate.request, plan, rows)) + skipped.add(candidate.selector) + pushedCount += 1 + } const data = new Map() for (const [selector, request] of planRequests(parsed.success, params)) { + if (skipped.has(selector)) continue const results = Arr.map( buildSelectorQueries(request, catalog, { orgId: OrgId.make(ORG), maxSeries: 10_000 }), (plan) => ({ @@ -128,7 +156,7 @@ const sqlRun = (db: Chdb, catalog: ReadonlyArray, query: string, ) data.set(selector, toSelectorData(request, results)) } - return Result.mapError(evaluate({ expr: parsed.success, params, data }), (e) => e.message) + return Result.mapError(evaluate({ expr: parsed.success, params, data, pushed }), (e) => e.message) } const flatten = (result: QueryResult): ReadonlyArray => { @@ -217,14 +245,22 @@ describeWithChdb("PromQL on the warehouse matches the reference", () => { test(query, () => { for (const { name, params } of cases) { const want = runQuery(query, params, reference) - const got = sqlRun(db, catalog, query, params) - if (Result.isFailure(want)) { - expect(Result.isFailure(got)).toBe(true) - continue + for (const pushdown of [false, true]) { + const label = `${name}${pushdown ? " pushdown" : ""}` + const got = sqlRun(db, catalog, query, params, pushdown) + if (Result.isFailure(want)) { + expect(Result.isFailure(got)).toBe(true) + continue + } + if (Result.isFailure(got)) throw new Error(`${label}: ${got.failure}`) + expect(`${label}: ${mismatch(got.success, want.success) ?? "ok"}`).toBe(`${label}: ok`) } - if (Result.isFailure(got)) throw new Error(`${name}: ${got.failure}`) - expect(`${name}: ${mismatch(got.success, want.success) ?? "ok"}`).toBe(`${name}: ok`) } }) } + + test("pushes most of the corpus down", () => { + // Each corpus query runs four cases; dozens of them have a pushable subtree. + expect(pushedCount).toBeGreaterThan(100) + }) }) diff --git a/knip.json b/knip.json index 4e308e6b95..4a7b30c964 100644 --- a/knip.json +++ b/knip.json @@ -92,6 +92,7 @@ }, "apps/cli": { "entry": [ + "scripts/*.ts", "test/probes/*.ts", "test/archive-probe-helpers.ts", "test/merge-injection-probe.ts", diff --git a/lib/promql/src/engine/evaluate.ts b/lib/promql/src/engine/evaluate.ts index 47ca6ad81d..3baece542a 100644 --- a/lib/promql/src/engine/evaluate.ts +++ b/lib/promql/src/engine/evaluate.ts @@ -20,6 +20,7 @@ import { type Labels, LOOKBACK_MS, type Point, + type PushdownData, type QueryResult, type RangeSeries, type Sample, @@ -204,6 +205,8 @@ export interface EvaluationInput { readonly params: QueryParams /** What the source returned for each selector of `planRequests`. */ readonly data: ReadonlyMap + /** Subtrees the source evaluated itself (see `planPushdowns`). */ + readonly pushed?: ReadonlyMap readonly limits?: EvalLimits } @@ -215,6 +218,7 @@ class Evaluator { private readonly params: QueryParams, private readonly data: ReadonlyMap, private readonly limits: EvalLimits, + private readonly pushed: ReadonlyMap, ) { this.lookbackMs = params.lookbackMs ?? LOOKBACK_MS } @@ -510,7 +514,31 @@ class Evaluator { }) } + /** A pushed subtree's answer as steps; a bare range function drops the metric name. */ + fromPushed(expr: Expr, data: PushdownData, grid: TimeGrid): R { + const times = gridTimes(grid) + const vectors: Sample[][] = times.map(() => []) + const keepName = expr.type === "Aggregate" || (expr.type === "Call" && expr.func === "last_over_time") + // Label order, as the selector would have read them; ties in topk depend on it. + const ordered = [...data.series].sort((x, y) => compareLabels(x.labels, y.labels)) + for (const series of ordered) { + const labels = keepName ? series.labels : dropMetricName(series.labels) + for (const [i, step] of series.steps.entries()) { + const t = times[step] + const value = series.values[i] + if (t !== undefined && value !== undefined) vectors[step]?.push({ labels, value, t }) + } + } + for (const vector of vectors) { + const unique = checkUnique(vector) + if (Result.isFailure(unique)) return Result.fail(unique.failure) + } + return this.steps(times, vectors) + } + eval(expr: Expr, grid: TimeGrid): R { + const pushed = this.pushed.get(expr) + if (pushed !== undefined) return this.fromPushed(expr, pushed, grid) return Result.gen(this, function* (): Generator, Value> { switch (expr.type) { case "NumberLiteral": @@ -624,7 +652,12 @@ class Evaluator { /** Evaluate a planned query once the source has answered every request of `planRequests`. */ export const evaluate = (input: EvaluationInput): R => { const { expr, params } = input - const evaluator = new Evaluator(params, input.data, input.limits ?? DEFAULT_LIMITS) + const evaluator = new Evaluator( + params, + input.data, + input.limits ?? DEFAULT_LIMITS, + input.pushed ?? new Map(), + ) const instant = params.startMs === params.endMs const grid: TimeGrid = instant ? { startMs: params.startMs, endMs: params.startMs, stepMs: 0 } diff --git a/lib/promql/src/engine/index.ts b/lib/promql/src/engine/index.ts index 9f47e8f448..b5ff97215b 100644 --- a/lib/promql/src/engine/index.ts +++ b/lib/promql/src/engine/index.ts @@ -9,7 +9,14 @@ import type { EvalLimits, QueryResult, SelectorData, SelectorRequest } from "./t export { evaluate, type EvaluationInput } from "./evaluate" export { bucketSamples, matchesLabels, memorySource, type RawSeries } from "./memory-source" export type { EvalError } from "./ops" -export { gridTimes, planRequests, type QueryParams, type RequestPlan } from "./plan" +export { + gridTimes, + planPushdowns, + planRequests, + type PushdownCandidate, + type QueryParams, + type RequestPlan, +} from "./plan" export * from "./types" /** Parse, plan and evaluate against a synchronous source. Tests and the compliance runner use it. */ diff --git a/lib/promql/src/engine/plan.ts b/lib/promql/src/engine/plan.ts index 26837afa01..1d5566df07 100644 --- a/lib/promql/src/engine/plan.ts +++ b/lib/promql/src/engine/plan.ts @@ -1,5 +1,13 @@ import type { AtModifier, Expr, MatrixSelector, SubqueryExpr, VectorSelector } from "../ast" -import { LOOKBACK_MS, type SelectorRequest } from "./types" +import { + LOOKBACK_MS, + PUSHDOWN_AGGREGATIONS, + PUSHDOWN_FUNCTIONS, + type PushdownAggregation, + type PushdownFunction, + type PushdownRequest, + type SelectorRequest, +} from "./types" /** Query parameters. An instant query has `startMs === endMs` and any step. */ export interface QueryParams { @@ -131,3 +139,109 @@ export const planRequests = (expr: Expr, params: QueryParams): RequestPlan => { walk(expr, top, expr.type === "MatrixSelector") return plan } + +/** A subtree a source may evaluate whole, and the one selector it reads. */ +export interface PushdownCandidate { + readonly request: PushdownRequest + readonly selector: VectorSelector +} + +const isPushdownFunction = (func: string): func is PushdownFunction => + func !== "last" && PUSHDOWN_FUNCTIONS.some((f) => f === func) + +const isPushdownAggregation = (op: string): op is PushdownAggregation => + PUSHDOWN_AGGREGATIONS.some((a) => a === op) + +const unparen = (expr: Expr): Expr => (expr.type === "Paren" ? unparen(expr.expr) : expr) + +/** `func(selector[range])`, or a bare selector read as its lookback (`last`). */ +const pushableInner = ( + expr: Expr, + lookbackMs: number, +): { selector: VectorSelector; func: PushdownFunction; rangeMs: number } | undefined => { + const node = unparen(expr) + if (node.type === "VectorSelector" && node.at === undefined) + return { selector: node, func: "last", rangeMs: lookbackMs } + if (node.type !== "Call" || !isPushdownFunction(node.func) || node.args.length !== 1) return undefined + const arg = node.args[0] + if (arg?.type !== "MatrixSelector" || arg.vector.at !== undefined) return undefined + return { selector: arg.vector, func: node.func, rangeMs: arg.rangeMs } +} + +/** + * Subtrees of the top-level grid a source may evaluate whole: a range function + * over a selector, or `sum|avg|min|max|count|group [by (...)]` over one or over a + * bare selector. The source may decline any of them; the evaluator then reads + * the selector as usual. Nothing inside a subquery is offered. + */ +export const planPushdowns = (expr: Expr, params: QueryParams): Map => { + const out = new Map() + const lookback = params.lookbackMs ?? LOOKBACK_MS + const instant = params.startMs === params.endMs || params.stepMs <= 0 + const steps = instant ? 1 : Math.floor((params.endMs - params.startMs) / params.stepMs) + 1 + const offer = ( + node: Expr, + inner: { selector: VectorSelector; func: PushdownFunction; rangeMs: number }, + aggregation: PushdownRequest["aggregation"], + ): boolean => { + const widthMs = instant ? inner.rangeMs : gcd(params.stepMs, inner.rangeMs) + if (widthMs < MIN_BUCKET_MS) return false + out.set(node, { + selector: inner.selector, + request: { + metricName: inner.selector.name, + matchers: inner.selector.matchers, + offsetMs: inner.selector.offsetMs, + func: inner.func, + rangeMs: inner.rangeMs, + startMs: params.startMs, + stepMs: instant ? 0 : params.stepMs, + steps, + widthMs, + aggregation, + }, + }) + return true + } + const walk = (node: Expr): void => { + switch (node.type) { + case "Aggregate": { + const inner = pushableInner(node.expr, lookback) + if ( + inner !== undefined && + isPushdownAggregation(node.op) && + !node.without && + node.param === undefined && + offer(node, inner, { op: node.op, by: node.grouping }) + ) { + return + } + if (node.param !== undefined) walk(node.param) + walk(node.expr) + return + } + case "Call": { + const inner = pushableInner(node, lookback) + if (inner !== undefined && inner.func !== "last" && offer(node, inner, undefined)) return + for (const arg of node.args) walk(arg) + return + } + case "Paren": + case "Unary": + walk(node.expr) + return + case "Binary": + walk(node.lhs) + walk(node.rhs) + return + case "NumberLiteral": + case "StringLiteral": + case "VectorSelector": + case "MatrixSelector": + case "Subquery": + return + } + } + walk(expr) + return out +} diff --git a/lib/promql/src/engine/types.ts b/lib/promql/src/engine/types.ts index a1cd56fbf6..e40d77e45b 100644 --- a/lib/promql/src/engine/types.ts +++ b/lib/promql/src/engine/types.ts @@ -109,3 +109,56 @@ export const DEFAULT_LIMITS: EvalLimits = { maxSeries: 10_000, maxPoints: 5_000_ /** Prometheus' default lookback for instant selectors. */ export const LOOKBACK_MS = 300_000 + +/** Range functions a source may evaluate itself; `last` is an instant selector's lookback. */ +export const PUSHDOWN_FUNCTIONS = [ + "rate", + "increase", + "delta", + "irate", + "idelta", + "resets", + "changes", + "avg_over_time", + "sum_over_time", + "count_over_time", + "min_over_time", + "max_over_time", + "last_over_time", + "present_over_time", + "stddev_over_time", + "stdvar_over_time", + "last", +] as const +export type PushdownFunction = (typeof PUSHDOWN_FUNCTIONS)[number] + +export const PUSHDOWN_AGGREGATIONS = ["sum", "avg", "min", "max", "count", "group"] as const +export type PushdownAggregation = (typeof PUSHDOWN_AGGREGATIONS)[number] + +/** + * A subtree the source may evaluate whole: `func(selector[range])` at every step, + * optionally combined `op by (labels)`. Steps are `startMs + j * stepMs`, the + * window of step j is `(t - offset - range, t - offset]`, and `widthMs` divides + * both the step and the range so windows are whole buckets. + */ +export interface PushdownRequest { + readonly metricName: string | undefined + readonly matchers: ReadonlyArray + readonly offsetMs: number + readonly func: PushdownFunction + readonly rangeMs: number + readonly startMs: number + readonly stepMs: number + readonly steps: number + readonly widthMs: number + readonly aggregation: { readonly op: PushdownAggregation; readonly by: ReadonlyArray } | undefined +} + +/** A source's answer: per series (or per group), the steps that have a value. */ +export interface PushdownData { + readonly series: ReadonlyArray<{ + readonly labels: Labels + readonly steps: ReadonlyArray + readonly values: ReadonlyArray + }> +} diff --git a/packages/backend/src/services/promql/PromqlQueryService.test.ts b/packages/backend/src/services/promql/PromqlQueryService.test.ts index 0dfe6eb675..a1feb43cbc 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.test.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.test.ts @@ -4,6 +4,7 @@ import type { TenantContext } from "@maple/domain/http" import { OrgId, UserId } from "@maple/primitives" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support" +import { WarehouseResponseLimitError } from "@maple/query-engine/execution" import { PromqlQueryService } from "./PromqlQueryService" const tenant: TenantContext = { @@ -68,4 +69,27 @@ describe("PromqlQueryService", () => { expect(error._tag).toBe("@maple/backend/promql/PromqlExecutionError") }), ) + + it.effect("fails cleanly when a selector's buckets would not fit in memory", () => + Effect.gen(function* () { + // The catalog knows `up`; every bounded read reports the response ceiling. + const warehouse = makeWarehouseServiceStub({ + compiledQuery: () => + Effect.succeed([{ metricName: "up", metricType: "gauge", unit: "", isMonotonic: 0 }]), + compiledQueryBounded: () => + Effect.fail(new WarehouseResponseLimitError({ kind: "bytes", message: "too big" })), + }) + const layer = Layer.effect(PromqlQueryService, PromqlQueryService.make).pipe( + Layer.provide(Layer.succeed(WarehouseQueryService, warehouse)), + ) + const error = yield* Effect.gen(function* () { + const service = yield* PromqlQueryService + return yield* Effect.flip( + service.query(tenant, { query: "up", startMs: 0, endMs: 3_600_000, stepMs: 60_000 }), + ) + }).pipe(Effect.provide(layer)) + expect(error._tag).toBe("@maple/backend/promql/PromqlExecutionError") + expect(error.message).toMatch(/too many samples/) + }), + ) }) diff --git a/packages/backend/src/services/promql/PromqlQueryService.ts b/packages/backend/src/services/promql/PromqlQueryService.ts index 2756176cfe..2b9d37af86 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.ts @@ -2,8 +2,10 @@ // // The engine (`@maple/promql`) parses, plans and evaluates; this service is the // Maple source behind it: one catalog read resolves Prometheus names to stored -// metrics, then every selector's warehouse queries run concurrently and the -// evaluator recombines their pre-aggregated buckets. +// metrics. Subtrees ClickHouse can evaluate whole (a range function over a +// selector, optionally summed by labels) run there and return one value per +// step; every other selector returns pre-aggregated buckets the evaluator +// recombines. import { Context, Effect, Layer, Result, Schema, Array as Arr } from "effect" import { @@ -11,7 +13,10 @@ import { evaluate, type Expr, parse, + planPushdowns, planRequests, + type PushdownCandidate, + type PushdownData, type QueryParams, type QueryResult, type SelectorData, @@ -20,9 +25,12 @@ import { } from "@maple/promql" import { CH, formatWarehouseDateTime } from "@maple/query-engine" import { + buildPushdownQuery, buildSelectorQueries, catalogFromRows, catalogQuery, + type PushdownPlan, + toPushdownData, toSelectorData, } from "@maple/query-engine/promql" import type { CatalogMetric } from "@maple/query-engine/promql" @@ -49,6 +57,21 @@ const MAX_SELECTORS = 20 const WAREHOUSE_CONCURRENCY = 4 +/** + * The most one warehouse response may hold. The API runs in a 128 MB isolate; + * a selector the source cannot evaluate whole returns its buckets, and past + * this the query fails cleanly instead of exhausting memory. + */ +const MAX_RESPONSE_BYTES = 24 * 1024 * 1024 + +const responseLimits = { maxRows: DEFAULT_LIMITS.maxSeries + 1, maxBytes: MAX_RESPONSE_BYTES } + +const tooLarge = () => + new PromqlExecutionError({ + message: + "query processing would load too many samples into memory; aggregate with sum by (...), narrow the selector, or increase the step", + }) + export interface PromqlQuery extends QueryParams { readonly query: string } @@ -106,34 +129,56 @@ const make = Effect.gen(function* () { return catalogFromRows(rows) }) - const readSelector = Effect.fn("PromqlQueryService.readSelector")(function* ( - tenant: TenantContext, - request: SelectorRequest, - catalog: ReadonlyArray, - ) { - const plans = buildSelectorQueries(request, catalog, { - orgId: tenant.orgId, - maxSeries: DEFAULT_LIMITS.maxSeries, - }) - yield* Effect.annotateCurrentSpan({ - "maple.promql.metric": request.metricName ?? "", - "maple.promql.bucket_ms": request.widthMs, - "maple.promql.queries": plans.length, - }) - const results = yield* Effect.forEach( - plans, - (plan) => - Effect.map( - warehouse.compiledQuery(tenant, plan.compiled, { - profile: "aggregation", - context: "promqlSelector", - }), - (rows) => ({ plan, rows }), - ), - { concurrency: WAREHOUSE_CONCURRENCY }, - ) - return toSelectorData(request, results) - }) + const readSelector = Effect.fn("PromqlQueryService.readSelector")( + function* (tenant: TenantContext, request: SelectorRequest, catalog: ReadonlyArray) { + const plans = buildSelectorQueries(request, catalog, { + orgId: tenant.orgId, + maxSeries: DEFAULT_LIMITS.maxSeries, + }) + yield* Effect.annotateCurrentSpan({ + "maple.promql.metric": request.metricName ?? "", + "maple.promql.bucket_ms": request.widthMs, + "maple.promql.queries": plans.length, + }) + const results = yield* Effect.forEach( + plans, + (plan) => + Effect.map( + warehouse.compiledQueryBounded(tenant, plan.compiled, { + profile: "aggregation", + context: "promqlSelector", + responseLimits, + }), + (rows) => ({ plan, rows }), + ), + { concurrency: WAREHOUSE_CONCURRENCY }, + ) + return toSelectorData(request, results) + }, + Effect.catchTag("@maple/query-engine/execution/WarehouseResponseLimitError", () => + Effect.fail(tooLarge()), + ), + ) + + const readPushdown = Effect.fn("PromqlQueryService.readPushdown")( + function* (tenant: TenantContext, candidate: PushdownCandidate, plan: PushdownPlan) { + yield* Effect.annotateCurrentSpan({ + "maple.promql.metric": plan.promName, + "maple.promql.function": candidate.request.func, + "maple.promql.aggregation": candidate.request.aggregation?.op ?? "", + "maple.promql.bucket_ms": candidate.request.widthMs, + }) + const rows = yield* warehouse.compiledQueryBounded(tenant, plan.compiled, { + profile: "aggregation", + context: "promqlPushdown", + responseLimits, + }) + return toPushdownData(candidate.request, plan, rows) + }, + Effect.catchTag("@maple/query-engine/execution/WarehouseResponseLimitError", () => + Effect.fail(tooLarge()), + ), + ) const query = Effect.fn("PromqlQueryService.query")(function* ( tenant: TenantContext, @@ -156,8 +201,24 @@ const make = Effect.gen(function* () { entries.length === 0 ? [] : yield* readCatalog(tenant, span(Arr.map(entries, ([, request]) => request))) + const options = { orgId: tenant.orgId, maxSeries: DEFAULT_LIMITS.maxSeries } + const pushdowns = Arr.flatMap([...planPushdowns(expr, input)], ([node, candidate]) => { + const plan = buildPushdownQuery(candidate.request, catalog, options) + return plan === undefined ? [] : [{ node, candidate, plan }] + }) + const skipped = new Set(Arr.map(pushdowns, ({ candidate }) => candidate.selector)) + yield* Effect.annotateCurrentSpan("maple.promql.pushdowns", pushdowns.length) + const pushed = yield* Effect.forEach( + pushdowns, + ({ node, candidate, plan }) => + Effect.map( + readPushdown(tenant, candidate, plan), + (pushdownData): readonly [Expr, PushdownData] => [node, pushdownData], + ), + { concurrency: WAREHOUSE_CONCURRENCY }, + ) const read = yield* Effect.forEach( - entries, + Arr.filter(entries, ([selector]) => !skipped.has(selector)), ([selector, request]) => Effect.map( readSelector(tenant, request, catalog), @@ -165,7 +226,7 @@ const make = Effect.gen(function* () { ), { concurrency: WAREHOUSE_CONCURRENCY }, ) - const result = evaluate({ expr, params: input, data: new Map(read) }) + const result = evaluate({ expr, params: input, data: new Map(read), pushed: new Map(pushed) }) if (Result.isFailure(result)) return yield* new PromqlExecutionError({ message: result.failure.message }) return result.success satisfies QueryResult diff --git a/packages/query-engine/src/ch/queries/promql.ts b/packages/query-engine/src/ch/queries/promql.ts index efea4edd0d..a5ef45b20a 100644 --- a/packages/query-engine/src/ch/queries/promql.ts +++ b/packages/query-engine/src/ch/queries/promql.ts @@ -362,3 +362,366 @@ export const promqlCatalogRowSchema = Schema.Struct({ isMonotonic: CHNumber, }) export type PromqlCatalogRow = typeof promqlCatalogRowSchema.Type + +// Pushdown: the source evaluates `func(selector[range])` per (series, step), and optionally +// `op by (labels)` per (group, step), so one value per step leaves ClickHouse instead of every +// bucket. The window of step j holds buckets k in (j*s - m, j*s] (s = step/width, m = +// range/width); each bucket fans out to the steps whose window holds it, and the window is +// recombined exactly as the evaluator does (lib/promql/src/engine/window.ts). + +export type PromqlPushdownFunction = + | "rate" + | "increase" + | "delta" + | "irate" + | "idelta" + | "resets" + | "changes" + | "avg_over_time" + | "sum_over_time" + | "count_over_time" + | "min_over_time" + | "max_over_time" + | "last_over_time" + | "present_over_time" + | "stddev_over_time" + | "stdvar_over_time" + | "last" + +export interface PromqlPushdownOpts extends Omit { + readonly func: PromqlPushdownFunction + readonly rangeMs: number + /** Evaluation steps; step j is `originMs + j * stepBuckets * widthMs` (offset already applied). */ + readonly steps: number + readonly stepBuckets: number + readonly rangeBuckets: number + readonly limit: number +} + +const linkedQuery = (opts: PromqlPushdownOpts) => + fromQuery(bucketQuery({ ...opts, needsValues: false, limit: opts.limit }), "buckets").select(($) => { + const prev = CH.windowSpec({ + partitionBy: [$.sk], + orderBy: [[$.end, "asc"]], + frame: CH.rowsBetween(CH.preceding(1), CH.currentRow), + }) + return { + sk: $.sk, + MetricName: $.MetricName, + ServiceName: $.ServiceName, + Attributes: $.Attributes, + ResourceAttributes: $.ResourceAttributes, + le: $.le, + k: sql(T.int64)`intDiv(${$.end} - ${param.int("originMs")}, ${opts.widthMs})`, + pk: sql( + T.int64, + )`intDiv(${CH.over(CH.lagInFrame($.end, 1, $.end), prev)} - ${param.int("originMs")}, ${opts.widthMs})`, + hasPrev: sql(T.uint8)`${CH.over(CH.count(), prev)} = 2`, + pLastT: CH.over(CH.lagInFrame($.lastT, 1, $.lastT), prev), + pLastV: CH.over(CH.lagInFrame($.lastV, 1, $.lastV), prev), + firstT: $.firstT, + firstV: $.firstV, + lastT: $.lastT, + lastV: $.lastV, + prevT: $.prevT, + prevV: $.prevV, + count: $.count, + sum: $.sum, + min: $.min, + max: $.max, + sumSquares: $.sumSquares, + increase: $.increase, + resets: $.resets, + changes: $.changes, + } + }) + +/** Each bucket once per step whose window holds it, with whether its predecessor is in that window too. */ +const expandedQuery = (opts: PromqlPushdownOpts) => + fromQuery(linkedQuery(opts), "linked").select(($) => { + const s = opts.stepBuckets + const m = opts.rangeBuckets + const j = sql( + T.int64, + )`arrayJoin(range(toInt64(greatest(0, ceil(${$.k} / ${s}))), toInt64(least(${opts.steps - 1}, floor((${$.k} + ${m - 1}) / ${s}))) + 1))` + return { + sk: $.sk, + MetricName: $.MetricName, + ServiceName: $.ServiceName, + Attributes: $.Attributes, + ResourceAttributes: $.ResourceAttributes, + le: $.le, + k: $.k, + j, + // The pair (predecessor's last sample, this bucket's first) is in the window of step j. + prevIn: sql(T.uint8)`${$.hasPrev} AND ${$.pk} > ${j} * ${s} - ${m}`, + pLastT: $.pLastT, + pLastV: $.pLastV, + firstT: $.firstT, + firstV: $.firstV, + lastT: $.lastT, + lastV: $.lastV, + prevT: $.prevT, + prevV: $.prevV, + count: $.count, + sum: $.sum, + min: $.min, + max: $.max, + sumSquares: $.sumSquares, + increase: $.increase, + resets: $.resets, + changes: $.changes, + } + }) + +/** The window of every (series, step), recombined from its buckets. */ +const windowQuery = (opts: PromqlPushdownOpts) => + fromQuery(expandedQuery(opts), "expanded") + .select(($) => { + const crossIncrease = sql( + T.float64, + )`sumIf(if(${$.firstV} < ${$.pLastV}, ${$.firstV}, ${$.firstV} - ${$.pLastV}), ${$.prevIn} = 1)` + return { + sk: $.sk, + j: $.j, + MetricName: CH.any_($.MetricName), + ServiceName: CH.any_($.ServiceName), + Attributes: CH.any_($.Attributes), + ResourceAttributes: CH.any_($.ResourceAttributes), + le: CH.any_($.le), + n: sql(T.float64)`sum(${$.count})`, + firstT: CH.argMin($.firstT, $.k), + firstV: CH.argMin($.firstV, $.k), + lastT: CH.argMax($.lastT, $.k), + lastV: CH.argMax($.lastV, $.k), + lastCount: CH.argMax($.count, $.k), + lastPrevT: CH.argMax($.prevT, $.k), + lastPrevV: CH.argMax($.prevV, $.k), + lastPrevIn: CH.argMax($.prevIn, $.k), + lastPLastT: CH.argMax($.pLastT, $.k), + lastPLastV: CH.argMax($.pLastV, $.k), + sum: CH.sum($.sum), + min: CH.min_($.min), + max: CH.max_($.max), + sumSquares: CH.sum($.sumSquares), + increase: sql(T.float64)`sum(${$.increase}) + ${crossIncrease}`, + resets: sql( + T.float64, + )`sum(${$.resets}) + countIf(${$.prevIn} = 1 AND ${$.firstV} < ${$.pLastV})`, + changes: sql( + T.float64, + )`sum(${$.changes}) + countIf(${$.prevIn} = 1 AND ${$.firstV} != ${$.pLastV})`, + } + }) + .groupBy("sk", "j") + +type WindowRow = { + readonly j: CH.Expr + readonly n: CH.Expr + readonly firstT: CH.Expr + readonly firstV: CH.Expr + readonly lastT: CH.Expr + readonly lastV: CH.Expr + readonly lastCount: CH.Expr + readonly lastPrevT: CH.Expr + readonly lastPrevV: CH.Expr + readonly lastPrevIn: CH.Expr + readonly lastPLastT: CH.Expr + readonly lastPLastV: CH.Expr + readonly sum: CH.Expr + readonly min: CH.Expr + readonly max: CH.Expr + readonly sumSquares: CH.Expr + readonly increase: CH.Expr + readonly resets: CH.Expr + readonly changes: CH.Expr +} + +/** Prometheus' extrapolatedRate (rate, increase, delta) over the window ending at step j. */ +const extrapolated = (opts: PromqlPushdownOpts, $: WindowRow) => { + const isCounter = opts.func !== "delta" + const stepEnd = sql(T.float64)`(${param.int("originMs")} + ${$.j} * ${opts.stepBuckets * opts.widthMs})` + const result = isCounter ? sql(T.float64)`${$.increase}` : sql(T.float64)`(${$.lastV} - ${$.firstV})` + const sampled = sql(T.float64)`((${$.lastT} - ${$.firstT}) / 1000)` + const between = sql(T.float64)`(${sampled} / (${$.n} - 1))` + const toStart0 = sql(T.float64)`((${$.firstT} - (${stepEnd} - ${opts.rangeMs})) / 1000)` + const toStart1 = sql(T.float64)`if(${toStart0} >= ${between} * 1.1, ${between} / 2, ${toStart0})` + const toZero = sql(T.float64)`(${sampled} * (${$.firstV} / ${result}))` + const toStart = isCounter + ? sql( + T.float64, + )`if(${result} > 0 AND ${$.firstV} >= 0 AND ${toZero} < ${toStart1}, ${toZero}, ${toStart1})` + : toStart1 + const toEnd0 = sql(T.float64)`((${stepEnd} - ${$.lastT}) / 1000)` + const toEnd = sql(T.float64)`if(${toEnd0} >= ${between} * 1.1, ${between} / 2, ${toEnd0})` + const factor = sql(T.float64)`((${sampled} + ${toStart} + ${toEnd}) / ${sampled})` + const value = + opts.func === "rate" + ? sql(T.float64)`${result} * ${factor} / ${opts.rangeMs / 1000}` + : sql(T.float64)`${result} * ${factor}` + return { value, valid: sql(T.uint8)`${$.n} >= 2 AND ${$.lastT} > ${$.firstT}` } +} + +/** irate and idelta: the window's last two samples. */ +const instantPair = (opts: PromqlPushdownOpts, $: WindowRow) => { + const pT = sql(T.float64)`if(${$.lastCount} >= 2, ${$.lastPrevT}, ${$.lastPLastT})` + const pV = sql(T.float64)`if(${$.lastCount} >= 2, ${$.lastPrevV}, ${$.lastPLastV})` + const hasPair = sql(T.uint8)`(${$.lastCount} >= 2 OR ${$.lastPrevIn} = 1)` + const value = + opts.func === "irate" + ? sql( + T.float64, + )`if(${$.lastV} < ${pV}, ${$.lastV}, ${$.lastV} - ${pV}) / ((${$.lastT} - ${pT}) / 1000)` + : sql(T.float64)`${$.lastV} - ${pV}` + return { value, valid: sql(T.uint8)`${hasPair} AND ${$.lastT} > ${pT}` } +} + +/** The function's value for one (series, step) window, and whether it has one. */ +const windowValue = (opts: PromqlPushdownOpts, $: WindowRow) => { + const any = sql(T.uint8)`${$.n} >= 1` + const variance = sql(T.float64)`greatest(0, ${$.sumSquares} / ${$.n} - pow(${$.sum} / ${$.n}, 2))` + switch (opts.func) { + case "rate": + case "increase": + case "delta": + return extrapolated(opts, $) + case "irate": + case "idelta": + return instantPair(opts, $) + case "resets": + return { value: $.resets, valid: any } + case "changes": + return { value: $.changes, valid: any } + case "avg_over_time": + return { value: sql(T.float64)`${$.sum} / ${$.n}`, valid: any } + case "sum_over_time": + return { value: $.sum, valid: any } + case "count_over_time": + return { value: $.n, valid: any } + case "min_over_time": + return { value: $.min, valid: any } + case "max_over_time": + return { value: $.max, valid: any } + case "last_over_time": + case "last": + return { value: $.lastV, valid: any } + case "present_over_time": + return { value: sql(T.float64)`1`, valid: any } + case "stddev_over_time": + return { value: sql(T.float64)`sqrt(${variance})`, valid: any } + case "stdvar_over_time": + return { value: variance, valid: any } + } +} + +const valuedQuery = (opts: PromqlPushdownOpts) => + fromQuery(windowQuery(opts), "windows").select(($) => { + const { value, valid } = windowValue(opts, $) + return { + sk: $.sk, + j: $.j, + MetricName: $.MetricName, + ServiceName: $.ServiceName, + Attributes: $.Attributes, + ResourceAttributes: $.ResourceAttributes, + le: $.le, + value: sql(T.float64)`toFloat64(${value})`, + valid: sql(T.uint8)`${valid}`, + } + }) + +/** One row per series: its labels' sources and the steps that have a value. */ +const pushdownSeriesQuery = (opts: PromqlPushdownOpts) => + fromQuery(valuedQuery(opts), "valued") + .select(($) => ({ + series: sql(T.string)`toString(${$.sk})`, + metricName: CH.any_($.MetricName), + serviceName: CH.any_($.ServiceName), + attributes: CH.any_($.Attributes), + resourceAttributes: CH.any_($.ResourceAttributes), + le: sql(T.string)`toString(any(${$.le}))`, + groups: sql(T.array(T.string))`emptyArrayString()`, + steps: sql(T.array(T.float64))`groupArray(toFloat64(${$.j}))`, + values: sql(T.array(T.float64))`groupArray(${$.value})`, + })) + .where(($) => [$.valid.eq(1)]) + .groupBy("series") + .limit(opts.limit) + .format("JSON") + +/** Combine the series of each group; `labels` are label expressions in `by` order, `le` the bucket bound. */ +export interface PromqlPushdownAggregation { + readonly op: "sum" | "avg" | "min" | "max" | "count" | "group" + readonly labels: ($: PromqlLabelColumns) => ReadonlyArray> + readonly byLe: boolean +} + +const aggregate = (op: PromqlPushdownAggregation["op"], value: CH.Expr) => { + switch (op) { + case "sum": + return sql(T.float64)`sum(${value})` + case "avg": + return sql(T.float64)`avg(${value})` + case "min": + return sql(T.float64)`min(${value})` + case "max": + return sql(T.float64)`max(${value})` + case "count": + return sql(T.float64)`toFloat64(count())` + case "group": + return sql(T.float64)`1` + } +} + +/** One row per group: the group's label values (in `by` order) and its value per step. */ +const pushdownGroupQuery = (opts: PromqlPushdownOpts, aggregation: PromqlPushdownAggregation) => { + const perStep = fromQuery(valuedQuery(opts), "valued") + .select(($) => { + const labels = aggregation.labels($) + return { + j: $.j, + groups: + labels.length === 0 + ? sql(T.array(T.string))`emptyArrayString()` + : sql(T.array(T.string))`[${sql.join(labels, ", ")}]`, + gle: aggregation.byLe ? $.le : sql(T.float64)`0`, + value: aggregate(aggregation.op, $.value), + } + }) + .where(($) => [$.valid.eq(1)]) + .groupBy("j", "groups", "gle") + return fromQuery(perStep, "per_step") + .select(($) => ({ + series: sql(T.string)`''`, + metricName: sql(T.string)`''`, + serviceName: sql(T.string)`''`, + attributes: sql(stringMap)`map()`, + resourceAttributes: sql(stringMap)`map()`, + le: sql(T.string)`toString(${$.gle})`, + groups: $.groups, + steps: sql(T.array(T.float64))`groupArray(toFloat64(${$.j}))`, + values: sql(T.array(T.float64))`groupArray(${$.value})`, + })) + .groupBy("groups", "le") + .limit(opts.limit) + .format("JSON") +} + +/** Per series. Params: `orgId`, epoch ms `startMs`/`endMs` (scan), `originMs` (step 0's window end), `timeOriginMs`. */ +export const promqlPushdownSeriesQuery = pushdownSeriesQuery + +/** Per `by` group; same params and row shape as `promqlPushdownSeriesQuery`. */ +export const promqlPushdownGroupQuery = pushdownGroupQuery + +export const promqlPushdownRowSchema = Schema.Struct({ + series: Schema.String, + metricName: Schema.String, + serviceName: Schema.String, + attributes: Schema.Record(Schema.String, Schema.String), + resourceAttributes: Schema.Record(Schema.String, Schema.String), + le: Schema.String, + groups: Schema.Array(Schema.String), + steps: numbers, + values: numbers, +}) +export type PromqlPushdownRow = typeof promqlPushdownRowSchema.Type diff --git a/packages/query-engine/src/promql/index.ts b/packages/query-engine/src/promql/index.ts index 43a35be15b..28f2090bae 100644 --- a/packages/query-engine/src/promql/index.ts +++ b/packages/query-engine/src/promql/index.ts @@ -3,6 +3,8 @@ export * from "./names" export * from "./source" export { type PromqlCatalogRow, + type PromqlPushdownRow, + promqlPushdownRowSchema, promqlCatalogRowSchema, type PromqlSelectorRow, promqlSelectorRowSchema, diff --git a/packages/query-engine/src/promql/source.ts b/packages/query-engine/src/promql/source.ts index a6e1a0df20..470bddae60 100644 --- a/packages/query-engine/src/promql/source.ts +++ b/packages/query-engine/src/promql/source.ts @@ -2,17 +2,29 @@ // (table, component) of the stored metrics its name resolves to, and their rows become SelectorData. // Pure: the caller runs the compiled queries. -import type { LabelMatcher, SampleBucket, SelectorData, SelectorRequest, SeriesData } from "@maple/promql" +import type { + LabelMatcher, + PushdownData, + PushdownRequest, + SampleBucket, + SelectorData, + SelectorRequest, + SeriesData, +} from "@maple/promql" import { Array as Arr, Order } from "effect" import { compile } from "@maple-dev/effect-orm/clickhouse" import type { OrgId } from "@maple/primitives" import { type PromqlCatalogRow, type PromqlSelectorRow, + type PromqlPushdownRow, promqlCatalogQuery, + type PromqlPushdownOpts, + promqlPushdownGroupQuery, + promqlPushdownSeriesQuery, promqlSelectorQuery, } from "../ch/queries/promql" -import { matchesAll, matcherCondition, seriesLabels } from "./labels" +import { formatLe, labelValueExpr, matchesAll, matcherCondition, seriesLabels } from "./labels" import { type CatalogMetric, type MetricComponent, @@ -197,3 +209,114 @@ export const catalogFromRows = (rows: ReadonlyArray): Readonly ] : [], ) + +export interface PushdownPlan { + readonly promName: string + readonly component: MetricComponent + readonly compiled: ReturnType +} + +const compilePushdown = ( + request: PushdownRequest, + target: PromNameTarget, + labelMatchers: ReadonlyArray, + byLabels: ReadonlyArray, + options: SelectorPlanOptions, +) => { + const originMs = request.startMs - request.offsetMs + const lastEnd = originMs + (request.steps - 1) * request.stepMs + const params = { + orgId: options.orgId, + startMs: originMs - request.rangeMs, + endMs: lastEnd, + originMs, + timeOriginMs: originMs, + } + const opts: PromqlPushdownOpts = { + table: target.table, + component: target.component, + metricNames: [target.metricName], + widthMs: request.widthMs, + func: request.func, + rangeMs: request.rangeMs, + steps: request.steps, + stepBuckets: request.steps === 1 ? 1 : request.stepMs / request.widthMs, + rangeBuckets: request.rangeMs / request.widthMs, + limit: options.maxSeries + 1, + conditions: ($) => Arr.map(labelMatchers, (m) => matcherCondition($, m)), + } + const aggregation = request.aggregation + if (aggregation === undefined) return compile(promqlPushdownSeriesQuery(opts), params) + return compile( + promqlPushdownGroupQuery(opts, { + op: aggregation.op, + labels: ($) => + Arr.flatMap(byLabels, (name) => { + const expr = labelValueExpr($, name) + return expr === undefined ? [] : [expr] + }), + byLe: aggregation.by.includes("le"), + }), + params, + ) +} + +/** + * The warehouse query for a pushdown, or undefined when it cannot be exact: the selector must + * resolve to one stored metric, and every label matcher must run in SQL (an `le` matcher is + * only checked after the fetch). The evaluator then reads the selector instead. + */ +export const buildPushdownQuery = ( + request: PushdownRequest, + catalog: ReadonlyArray, + options: SelectorPlanOptions, +): PushdownPlan | undefined => { + const targets = resolveSelectorName( + Arr.filter(request.matchers, (m) => m.name === "__name__"), + catalog, + ) + const target = targets[0] + if (targets.length !== 1 || target === undefined) return undefined + const labelMatchers = Arr.filter(request.matchers, (m) => m.name !== "__name__") + if (Arr.some(labelMatchers, (m) => m.name === "le")) return undefined + if (request.widthMs <= 0 || request.rangeMs % request.widthMs !== 0) return undefined + if (request.steps > 1 && request.stepMs % request.widthMs !== 0) return undefined + const byLabels = Arr.filter(request.aggregation?.by ?? [], (name) => name !== "le" && name !== "__name__") + return { + promName: target.promName, + component: target.component, + compiled: compilePushdown(request, target, labelMatchers, byLabels, options), + } +} + +/** The pushdown's rows as the evaluator reads them. */ +export const toPushdownData = ( + request: PushdownRequest, + plan: PushdownPlan, + rows: ReadonlyArray, +): PushdownData => { + const by = request.aggregation?.by + const byLabels = Arr.filter(by ?? [], (name) => name !== "le" && name !== "__name__") + return { + series: Arr.map(rows, (row) => { + if (by === undefined) { + const labels = seriesLabels({ + promName: plan.promName, + serviceName: row.serviceName, + attributes: row.attributes, + resourceAttributes: row.resourceAttributes, + le: plan.component === "bucket" ? parseLe(row.le) : undefined, + }) + return { labels, steps: row.steps, values: row.values } + } + const labels: Record = {} + Arr.forEach(byLabels, (name, i) => { + const value = row.groups[i] ?? "" + if (value !== "") labels[name] = value + }) + if (by.includes("le") && plan.component === "bucket") labels.le = formatLe(parseLe(row.le)) + if (by.includes("__name__")) labels.__name__ = plan.promName + return { labels, steps: row.steps, values: row.values } + }), + } +} From 9486e3c58c387b9b5abd33fa0bfbfa00283c87cd Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 15:51:24 +0100 Subject: [PATCH 2/4] fix(promql): pushdown overflow, metric names and le grouping A pushed-down subtree past the series cap now fails like a selector read does instead of returning a truncated answer. sum by (__name__) over a range function no longer carries the metric name, which every pushed function but last_over_time drops. Grouping by le on data that is not a histogram bucket is left to the evaluator. Both name cases join the compliance corpus: 420 of 420 match Prometheus 3.5. --- lib/promql/scripts/compliance/queries.ts | 2 ++ lib/promql/src/engine/plan.ts | 2 +- packages/query-engine/src/promql/source.ts | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/promql/scripts/compliance/queries.ts b/lib/promql/scripts/compliance/queries.ts index c43762c41e..b97211ca57 100644 --- a/lib/promql/scripts/compliance/queries.ts +++ b/lib/promql/scripts/compliance/queries.ts @@ -49,6 +49,8 @@ export const QUERIES: ReadonlyArray = [ "quantile(0.5, node_memory_free_bytes)", 'count_values("value", http_requests_total > 0)', "group by (job) (http_requests_total)", + "sum by (__name__) (rate(http_requests_total[5m]))", + "sum by (__name__, job) (last_over_time(node_memory_free_bytes[5m]))", // histograms "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "histogram_quantile(0.5, sum by (le, service) (rate(http_request_duration_seconds_bucket[5m])))", diff --git a/lib/promql/src/engine/plan.ts b/lib/promql/src/engine/plan.ts index 325b416205..a9a30f6775 100644 --- a/lib/promql/src/engine/plan.ts +++ b/lib/promql/src/engine/plan.ts @@ -181,7 +181,7 @@ const pushableInner = ( export const planPushdowns = (expr: Expr, params: QueryParams): Map => { const out = new Map() const lookback = params.lookbackMs ?? LOOKBACK_MS - const instant = params.startMs === params.endMs || params.stepMs <= 0 + const instant = isInstant(params) || params.stepMs <= 0 const steps = instant ? 1 : Math.floor((params.endMs - params.startMs) / params.stepMs) + 1 const offer = ( node: Expr, diff --git a/packages/query-engine/src/promql/source.ts b/packages/query-engine/src/promql/source.ts index 470bddae60..ef9a8c2597 100644 --- a/packages/query-engine/src/promql/source.ts +++ b/packages/query-engine/src/promql/source.ts @@ -279,6 +279,8 @@ export const buildPushdownQuery = ( if (targets.length !== 1 || target === undefined) return undefined const labelMatchers = Arr.filter(request.matchers, (m) => m.name !== "__name__") if (Arr.some(labelMatchers, (m) => m.name === "le")) return undefined + // Grouping reads `le` from the bucket bound; an `le` attribute on other data is not supported. + if (request.aggregation?.by.includes("le") === true && target.component !== "bucket") return undefined if (request.widthMs <= 0 || request.rangeMs % request.widthMs !== 0) return undefined if (request.steps > 1 && request.stepMs % request.widthMs !== 0) return undefined const byLabels = Arr.filter(request.aggregation?.by ?? [], (name) => name !== "le" && name !== "__name__") @@ -315,7 +317,10 @@ export const toPushdownData = ( if (value !== "") labels[name] = value }) if (by.includes("le") && plan.component === "bucket") labels.le = formatLe(parseLe(row.le)) - if (by.includes("__name__")) labels.__name__ = plan.promName + // Every pushed function but last_over_time drops the name before aggregating. + if (by.includes("__name__") && (request.func === "last" || request.func === "last_over_time")) { + labels.__name__ = plan.promName + } return { labels, steps: row.steps, values: row.values } }), } From 586096c183c5e0d886413404165e28a483ba15ba Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 17:34:06 +0100 Subject: [PATCH 3/4] fix(promql): grouped pushdown labels are built without prototype writes --- packages/query-engine/src/promql/source.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/query-engine/src/promql/source.ts b/packages/query-engine/src/promql/source.ts index 46286417d1..5d7ea7a8fe 100644 --- a/packages/query-engine/src/promql/source.ts +++ b/packages/query-engine/src/promql/source.ts @@ -11,7 +11,7 @@ import type { SelectorRequest, SeriesData, } from "@maple/promql" -import { Array as Arr, Order } from "effect" +import { Array as Arr, Order, Record as Rec } from "effect" import { compile } from "@maple-dev/effect-orm/clickhouse" import type { OrgId } from "@maple/primitives" import { @@ -316,16 +316,18 @@ export const toPushdownData = ( }) return { labels, steps: row.steps, values: row.values } } - const labels: Record = {} - Arr.forEach(byLabels, (name, i) => { - const value = row.groups[i] ?? "" - if (value !== "") labels[name] = value - }) - if (by.includes("le") && plan.component === "bucket") labels.le = formatLe(parseLe(row.le)) + const entries: Array = Arr.map(byLabels, (name, i) => [ + name, + row.groups[i] ?? "", + ]) + if (by.includes("le") && plan.component === "bucket") + entries.push(["le", formatLe(parseLe(row.le))]) // Every pushed function but last_over_time drops the name before aggregating. if (by.includes("__name__") && (request.func === "last" || request.func === "last_over_time")) { - labels.__name__ = plan.promName + entries.push(["__name__", plan.promName]) } + // fromEntries defines own properties, so `by (__proto__)` stays a label. + const labels = Rec.fromEntries(Arr.filter(entries, ([, value]) => value !== "")) return { labels, steps: row.steps, values: row.values } }), } From 0fe27ec9af11e8e20dfe2c34d6764a806e92d5b3 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 15 Nov 2026 18:25:49 +0100 Subject: [PATCH 4/4] test(promql): pushdown reads are held to single-tenant too --- apps/cli/test/promql-engine.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/cli/test/promql-engine.test.ts b/apps/cli/test/promql-engine.test.ts index 2617ef8daa..9c8ad06098 100644 --- a/apps/cli/test/promql-engine.test.ts +++ b/apps/cli/test/promql-engine.test.ts @@ -142,9 +142,7 @@ const sqlRun = ( for (const [node, candidate] of pushdown ? planPushdowns(parsed.success, params) : []) { const plan = buildPushdownQuery(candidate.request, catalog, options) if (plan === undefined) continue - const rows = decodePushdown( - db.query(normalizeSqlForClickHouseClient(Effect.runSync(plan.compiled).sql)), - ) + const rows = decodePushdown(db.query(sqlOf(Effect.runSync(plan.compiled)))) pushed.set(node, toPushdownData(candidate.request, plan, rows)) skipped.add(candidate.selector) pushedCount += 1