diff --git a/apps/cli/scripts/bench-promql.ts b/apps/cli/scripts/bench-promql.ts new file mode 100644 index 0000000000..1fa4d96ee5 --- /dev/null +++ b/apps/cli/scripts/bench-promql.ts @@ -0,0 +1,204 @@ +/** + * PromQL on the warehouse at volume, on chDB. + * + * MAPLE_TELEMETRY=off bun run scripts/bench-promql.ts [--series 1000] [--hours 24] + * + * Loads a counter with `--series` series and a 12-bucket histogram with a fifth of + * that, scraped every 15 s for `--hours`, plus an unrelated metric of the same size + * the scan must skip. Times each query's SQL (per selector) and evaluation. + */ +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Effect, Result } from "effect" +import schemaSql from "@maple/local-server/schema/local-schema.sql" with { type: "text" } +import { Chdb } from "@maple/local-server/chdb" +import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" +import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" +import * as CH from "@maple/query-engine/ch" +import { + buildPushdownQuery, + buildSelectorQueries, + catalogFromRows, + catalogQuery, + promqlCatalogRowSchema, + promqlPushdownRowSchema, + promqlSelectorRowSchema, + toPushdownData, + toSelectorData, +} from "@maple/query-engine/promql" +import { + evaluate, + type Expr, + parse, + planPushdowns, + planRequests, + type PushdownData, + type SelectorData, + type VectorSelector, +} from "@maple/promql" +import { OrgId } from "@maple/domain" + +const arg = (name: string, fallback: number): number => { + const index = process.argv.indexOf(`--${name}`) + return index === -1 ? fallback : Number(process.argv[index + 1]) +} +const SERIES = arg("series", 1000) +const HOURS = arg("hours", 24) +const ORG = "org_bench" +const HOUR_MS = 3_600_000 +const END = Math.floor(Date.now() / HOUR_MS) * HOUR_MS - HOUR_MS +const START = END - HOURS * HOUR_MS +const POINTS = (HOURS * HOUR_MS) / 15_000 + +const root = mkdtempSync(join(tmpdir(), "maple-promql-bench-")) +const db = Chdb.open({ dataDir: join(root, "data"), schemaSql }) + +const time = (f: () => A): [A, number] => { + const t0 = performance.now() + const out = f() + return [out, performance.now() - t0] +} + +const load = () => { + const ts = `fromUnixTimestamp64Milli(toInt64(${START} + (number % ${POINTS}) * 15000))` + const counter = (name: string) => ` + INSERT INTO metrics_sum (OrgId, ResourceAttributes, ServiceName, MetricName, Attributes, StartTimeUnix, TimeUnix, Value, AggregationTemporality, IsMonotonic) + SELECT '${ORG}', map('service.name', 'svc-' || toString(intDiv(number, ${POINTS}) % 10), 'k8s.pod.name', 'pod-' || toString(intDiv(number, ${POINTS}))), + 'svc-' || toString(intDiv(number, ${POINTS}) % 10), '${name}', + map('route', '/r' || toString(intDiv(number, ${POINTS}) % 50), 'status', toString(200 + (intDiv(number, ${POINTS}) % 4) * 100)), + fromUnixTimestamp64Milli(toInt64(${START - HOUR_MS})), ${ts}, + toFloat64((number % ${POINTS}) * (1 + intDiv(number, ${POINTS}) % 7)), 2, true + FROM numbers(${SERIES * POINTS})` + db.exec(counter("http_requests_total")) + db.exec(counter("other_requests_total")) + db.exec(` + INSERT INTO metrics_histogram (OrgId, ResourceAttributes, ServiceName, MetricName, MetricUnit, Attributes, StartTimeUnix, TimeUnix, Count, Sum, BucketCounts, ExplicitBounds, AggregationTemporality) + SELECT '${ORG}', map('service.name', 'svc-' || toString(intDiv(number, ${POINTS}) % 10)), 'svc-' || toString(intDiv(number, ${POINTS}) % 10), + 'http.server.request.duration', 's', map('http.route', '/r' || toString(intDiv(number, ${POINTS}) % 50)), + fromUnixTimestamp64Milli(toInt64(${START - HOUR_MS})), ${ts}, + (number % ${POINTS}) * 12, toFloat64(number % ${POINTS}) * 3.5, + arrayMap(i -> toUInt64(number % ${POINTS}), range(12)), + [0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5], 2 + FROM numbers(${Math.max(1, Math.floor(SERIES / 5)) * POINTS})`) +} + +const [, loadMs] = time(load) +console.log( + `loaded ${(SERIES * POINTS * 2 + Math.floor(SERIES / 5) * POINTS).toLocaleString()} rows in ${(loadMs / 1000).toFixed(1)}s`, +) + +const catalogRows = decodeJsonEachRow(promqlCatalogRowSchema)( + db.query( + normalizeSqlForClickHouseClient( + CH.compileUnsafe(catalogQuery(), { + orgId: OrgId.make(ORG), + startTime: new Date(START - 2 * HOUR_MS).toISOString().slice(0, 19).replace("T", " "), + endTime: new Date(END + HOUR_MS).toISOString().slice(0, 19).replace("T", " "), + }).sql, + ), + ), +) +const catalog = catalogFromRows(catalogRows) +const decodeRows = decodeJsonEachRow(promqlSelectorRowSchema) + +const decodePushdown = decodeJsonEachRow(promqlPushdownRowSchema) +const pushdownEnabled = !process.argv.includes("--no-pushdown") + +const run = (query: string, params: { startMs: number; endMs: number; stepMs: number }) => { + const parsed = parse(query) + if (Result.isFailure(parsed)) throw new Error(parsed.failure.message) + const data = new Map() + const pushed = new Map() + const skipped = new Set() + let sqlMs = 0 + let rows = 0 + let buckets = 0 + let jsonBytes = 0 + for (const [node, candidate] of pushdownEnabled ? planPushdowns(parsed.success, params) : []) { + const plan = buildPushdownQuery(candidate.request, catalog, { + orgId: OrgId.make(ORG), + maxSeries: 10_000, + }) + if (plan === undefined) continue + const sql = normalizeSqlForClickHouseClient(Effect.runSync(plan.compiled).sql) + const [raw, ms] = time(() => db.query(sql)) + sqlMs += ms + jsonBytes += raw.length + const decoded = decodePushdown(raw) + rows += decoded.length + for (const r of decoded) buckets += r.values.length + pushed.set(node, toPushdownData(candidate.request, plan, decoded)) + skipped.add(candidate.selector) + } + for (const [selector, request] of planRequests(parsed.success, params)) { + if (skipped.has(selector)) continue + const results = buildSelectorQueries(request, catalog, { + orgId: OrgId.make(ORG), + maxSeries: 10_000, + }).map((plan) => { + const sql = normalizeSqlForClickHouseClient(Effect.runSync(plan.compiled).sql) + const [raw, ms] = time(() => db.query(sql)) + sqlMs += ms + jsonBytes += typeof raw === "string" ? raw.length : JSON.stringify(raw).length + const decoded = decodeRows(raw) + rows += decoded.length + return { plan, rows: decoded } + }) + const selectorData = toSelectorData(request, results) + for (const s of selectorData.series) buckets += s.buckets.length + data.set(selector, selectorData) + } + Bun.gc(true) + const heapMb = process.memoryUsage().heapUsed / 1e6 + const [result, evalMs] = time(() => evaluate({ expr: parsed.success, params, data, pushed })) + if (Result.isFailure(result)) throw new Error(result.failure.message) + const out = result.success + const outSeries = out.resultType === "matrix" || out.resultType === "vector" ? out.result.length : 1 + Bun.gc(true) + const afterMb = process.memoryUsage().heapUsed / 1e6 + return { sqlMs, evalMs, rows, buckets, outSeries, heapMb, afterMb, jsonMb: jsonBytes / 1e6 } +} + +const cases: Array<[string, string, { startMs: number; endMs: number; stepMs: number }]> = [ + [ + "24h step 5m", + "sum by (route) (rate(http_requests_total[5m]))", + { startMs: START + HOUR_MS, endMs: END, stepMs: 300_000 }, + ], + [ + "24h step 60s", + "sum by (status) (rate(http_requests_total[$r]))".replace("$r", "4m"), + { startMs: START + HOUR_MS, endMs: END, stepMs: 60_000 }, + ], + [ + "1h step 15s", + 'sum by (route) (rate(http_requests_total{status="200"}[1m]))', + { startMs: END - HOUR_MS, endMs: END, stepMs: 15_000 }, + ], + [ + "6h p95", + "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket[5m])))", + { startMs: END - 6 * HOUR_MS, endMs: END, stepMs: 60_000 }, + ], + [ + "instant", + "topk(5, sum by (k8s_pod_name) (increase(http_requests_total[1h])))", + { startMs: END, endMs: END, stepMs: 0 }, + ], + [ + "regex 6h", + 'sum(rate(http_requests_total{route=~"/r1.*"}[5m]))', + { startMs: END - 6 * HOUR_MS, endMs: END, stepMs: 60_000 }, + ], +] + +for (const [name, query, params] of cases) { + run(query, params) + const r = run(query, params) + console.log( + `${name.padEnd(12)} sql ${r.sqlMs.toFixed(0).padStart(6)}ms eval ${r.evalMs.toFixed(0).padStart(5)}ms rows ${String(r.rows).padStart(5)} buckets ${String(r.buckets).padStart(8)} out ${r.outSeries} json ${r.jsonMb.toFixed(0)}MB heap ${r.heapMb.toFixed(0)}->${r.afterMb.toFixed(0)}MB ${query}`, + ) +} +db.close() +rmSync(root, { recursive: true, force: true }) diff --git a/apps/cli/test/promql-engine.test.ts b/apps/cli/test/promql-engine.test.ts index 73d6765ac6..9c8ad06098 100644 --- a/apps/cli/test/promql-engine.test.ts +++ b/apps/cli/test/promql-engine.test.ts @@ -15,19 +15,25 @@ import { decodeJsonEachRow } from "@maple/local-server/chdb-rows" import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" import * as CH from "@maple/query-engine/ch" import { + buildPushdownQuery, buildSelectorQueries, catalogFromRows, catalogQuery, promqlCatalogRowSchema, + promqlPushdownRowSchema, promqlSelectorRowSchema, + toPushdownData, toSelectorData, type CatalogMetric, } from "@maple/query-engine/promql" import { evaluate, + type Expr, memorySource, parse, + planPushdowns, planRequests, + type PushdownData, type QueryParams, type QueryResult, type RawSeries, @@ -110,6 +116,9 @@ const insertHistogram = (db: Chdb, service: string): void => { } const decodeRows = decodeJsonEachRow(promqlSelectorRowSchema) +const decodePushdown = decodeJsonEachRow(promqlPushdownRowSchema) +const options = { orgId: OrgId.make(ORG), maxSeries: 10_000 } +let pushedCount = 0 const decodeCatalog = decodeJsonEachRow(promqlCatalogRowSchema) /** The tenant scope of every compiled read; the warehouse refuses anything but single-tenant. */ @@ -119,11 +128,28 @@ const sqlOf = (compiled: { readonly sql: string; readonly tenantScope: string }) return normalizeSqlForClickHouseClient(compiled.sql) } -const sqlRun = (db: Chdb, catalog: ReadonlyArray, query: string, params: QueryParams) => { +const sqlRun = ( + db: Chdb, + catalog: ReadonlyArray, + query: string, + params: QueryParams, + pushdown: boolean, +) => { const parsed = parse(query) if (Result.isFailure(parsed)) return Result.fail(parsed.failure.message) + const pushed = new Map() + const skipped = new Set() + for (const [node, candidate] of pushdown ? planPushdowns(parsed.success, params) : []) { + const plan = buildPushdownQuery(candidate.request, catalog, options) + if (plan === undefined) continue + const rows = decodePushdown(db.query(sqlOf(Effect.runSync(plan.compiled)))) + pushed.set(node, toPushdownData(candidate.request, plan, rows)) + skipped.add(candidate.selector) + pushedCount += 1 + } const data = new Map() for (const [selector, request] of planRequests(parsed.success, params)) { + if (skipped.has(selector)) continue const results = Arr.map( buildSelectorQueries(request, catalog, { orgId: OrgId.make(ORG), maxSeries: 10_000 }), (plan) => ({ @@ -133,7 +159,7 @@ const sqlRun = (db: Chdb, catalog: ReadonlyArray, query: string, ) data.set(selector, toSelectorData(request, results)) } - return Result.mapError(evaluate({ expr: parsed.success, params, data }), (e) => e.message) + return Result.mapError(evaluate({ expr: parsed.success, params, data, pushed }), (e) => e.message) } const flatten = (result: QueryResult): ReadonlyArray => { @@ -222,17 +248,25 @@ describeWithChdb("PromQL on the warehouse matches the reference", () => { test(query, () => { for (const { name, params } of cases) { const want = runQuery(query, params, reference) - const got = sqlRun(db, catalog, query, params) - if (Result.isFailure(want)) { - expect(Result.isFailure(got)).toBe(true) - continue + for (const pushdown of [false, true]) { + const label = `${name}${pushdown ? " pushdown" : ""}` + const got = sqlRun(db, catalog, query, params, pushdown) + if (Result.isFailure(want)) { + expect(Result.isFailure(got)).toBe(true) + continue + } + if (Result.isFailure(got)) throw new Error(`${label}: ${got.failure}`) + expect(`${label}: ${mismatch(got.success, want.success) ?? "ok"}`).toBe(`${label}: ok`) } - if (Result.isFailure(got)) throw new Error(`${name}: ${got.failure}`) - expect(`${name}: ${mismatch(got.success, want.success) ?? "ok"}`).toBe(`${name}: ok`) } }) } + test("pushes most of the corpus down", () => { + // Each corpus query runs four cases; dozens of them have a pushable subtree. + expect(pushedCount).toBeGreaterThan(100) + }) + test("every compiled read is single-tenant, as the warehouse requires", () => { expect([...scopes]).toEqual(["single-tenant"]) }) diff --git a/knip.json b/knip.json index 2918db0dfa..c5d3d9cc9b 100644 --- a/knip.json +++ b/knip.json @@ -94,6 +94,7 @@ }, "apps/cli": { "entry": [ + "scripts/*.ts", "test/probes/*.ts", "test/archive-probe-helpers.ts", "test/merge-injection-probe.ts", diff --git a/lib/promql/scripts/compliance/queries.ts b/lib/promql/scripts/compliance/queries.ts index c4065a0102..70fc604b27 100644 --- a/lib/promql/scripts/compliance/queries.ts +++ b/lib/promql/scripts/compliance/queries.ts @@ -54,6 +54,8 @@ export const QUERIES: ReadonlyArray = [ "quantile(0.5, node_memory_free_bytes)", 'count_values("value", http_requests_total > 0)', "group by (job) (http_requests_total)", + "sum by (__name__) (rate(http_requests_total[5m]))", + "sum by (__name__, job) (last_over_time(node_memory_free_bytes[5m]))", // histograms "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "histogram_quantile(0.5, sum by (le, service) (rate(http_request_duration_seconds_bucket[5m])))", diff --git a/lib/promql/src/engine/evaluate.ts b/lib/promql/src/engine/evaluate.ts index 27c753e6ee..4089565560 100644 --- a/lib/promql/src/engine/evaluate.ts +++ b/lib/promql/src/engine/evaluate.ts @@ -34,6 +34,7 @@ import { type Labels, LOOKBACK_MS, type Point, + type PushdownData, type QueryResult, type RangeSeries, type Sample, @@ -197,6 +198,8 @@ export interface EvaluationInput { readonly params: QueryParams /** What the source returned for each selector of `planRequests`. */ readonly data: ReadonlyMap + /** Subtrees the source evaluated itself (see `planPushdowns`). */ + readonly pushed?: ReadonlyMap readonly limits?: EvalLimits } @@ -208,6 +211,7 @@ class Evaluator { private readonly params: QueryParams, private readonly data: ReadonlyMap, private readonly limits: EvalLimits, + private readonly pushed: ReadonlyMap, ) { this.lookbackMs = params.lookbackMs ?? LOOKBACK_MS } @@ -598,7 +602,37 @@ class Evaluator { }) } + /** A pushed subtree's answer as steps; a bare range function drops the metric name. */ + fromPushed(expr: Expr, data: PushdownData, grid: TimeGrid): R { + // The source returns one row past the cap, so overflow shows here as it does for selectors. + if (data.series.length > this.limits.maxSeries) { + return err( + `query returns more than ${this.limits.maxSeries} series; narrow the selector or aggregate`, + ) + } + const times = gridTimes(grid) + const vectors: Sample[][] = times.map(() => []) + const keepName = expr.type === "Aggregate" || (expr.type === "Call" && expr.func === "last_over_time") + // Label order, as the selector would have read them; ties in topk depend on it. + const ordered = [...data.series].sort((x, y) => compareLabels(x.labels, y.labels)) + for (const series of ordered) { + const labels = keepName ? series.labels : dropMetricName(series.labels) + for (const [i, step] of series.steps.entries()) { + const t = times[step] + const value = series.values[i] + if (t !== undefined && value !== undefined) vectors[step]?.push({ labels, value, t }) + } + } + for (const vector of vectors) { + const unique = checkUnique(vector) + if (Result.isFailure(unique)) return Result.fail(unique.failure) + } + return this.steps(times, vectors) + } + eval(expr: Expr, grid: TimeGrid): R { + const pushed = this.pushed.get(expr) + if (pushed !== undefined) return this.fromPushed(expr, pushed, grid) return Result.gen(this, function* (): Generator, Value> { switch (expr.type) { case "NumberLiteral": @@ -728,7 +762,12 @@ export const evaluate = (input: EvaluationInput): R => { // Callers may skip `runQuery`; refuse a grid that would be too large before building it. const resolution = resolutionError(expr, params) if (resolution !== undefined) return err(resolution) - const evaluator = new Evaluator(params, input.data, input.limits ?? DEFAULT_LIMITS) + const evaluator = new Evaluator( + params, + input.data, + input.limits ?? DEFAULT_LIMITS, + input.pushed ?? new Map(), + ) const instant = isInstant(params) const grid: TimeGrid = instant ? { startMs: params.startMs, endMs: params.startMs, stepMs: 0 } diff --git a/lib/promql/src/engine/index.ts b/lib/promql/src/engine/index.ts index 9d06975e18..c665631ab4 100644 --- a/lib/promql/src/engine/index.ts +++ b/lib/promql/src/engine/index.ts @@ -13,7 +13,9 @@ export { gridTimes, isInstant, MAX_POINTS_PER_SERIES, + planPushdowns, planRequests, + type PushdownCandidate, type QueryParams, type RequestPlan, resolutionError, diff --git a/lib/promql/src/engine/plan.ts b/lib/promql/src/engine/plan.ts index d86659234e..c4f30ed7fe 100644 --- a/lib/promql/src/engine/plan.ts +++ b/lib/promql/src/engine/plan.ts @@ -1,5 +1,13 @@ import type { AtModifier, Expr, MatrixSelector, SubqueryExpr, VectorSelector } from "../ast" -import { LOOKBACK_MS, type SelectorRequest } from "./types" +import { + LOOKBACK_MS, + PUSHDOWN_AGGREGATIONS, + PUSHDOWN_FUNCTIONS, + type PushdownAggregation, + type PushdownFunction, + type PushdownRequest, + type SelectorRequest, +} from "./types" /** Query parameters. An instant query has `startMs === endMs` and any step. */ export interface QueryParams { @@ -142,6 +150,110 @@ export const planRequests = (expr: Expr, params: QueryParams): RequestPlan => { return plan } +/** A subtree a source may evaluate whole, and the one selector it reads. */ +export interface PushdownCandidate { + readonly request: PushdownRequest + readonly selector: VectorSelector +} + +const isPushdownFunction = (func: string): func is PushdownFunction => + func !== "last" && PUSHDOWN_FUNCTIONS.some((f) => f === func) + +const isPushdownAggregation = (op: string): op is PushdownAggregation => + PUSHDOWN_AGGREGATIONS.some((a) => a === op) + +/** `func(selector[range])`, or a bare selector read as its lookback (`last`). */ +const pushableInner = ( + expr: Expr, + lookbackMs: number, +): { selector: VectorSelector; func: PushdownFunction; rangeMs: number } | undefined => { + const node = unparen(expr) + if (node.type === "VectorSelector" && node.at === undefined) + return { selector: node, func: "last", rangeMs: lookbackMs } + if (node.type !== "Call" || !isPushdownFunction(node.func) || node.args.length !== 1) return undefined + const arg = node.args[0] + if (arg?.type !== "MatrixSelector" || arg.vector.at !== undefined) return undefined + return { selector: arg.vector, func: node.func, rangeMs: arg.rangeMs } +} + +/** + * Subtrees of the top-level grid a source may evaluate whole: a range function + * over a selector, or `sum|avg|min|max|count|group [by (...)]` over one or over a + * bare selector. The source may decline any of them; the evaluator then reads + * the selector as usual. Nothing inside a subquery is offered. + */ +export const planPushdowns = (expr: Expr, params: QueryParams): Map => { + const out = new Map() + const lookback = params.lookbackMs ?? LOOKBACK_MS + const instant = isInstant(params) || params.stepMs <= 0 + const steps = instant ? 1 : Math.floor((params.endMs - params.startMs) / params.stepMs) + 1 + const offer = ( + node: Expr, + inner: { selector: VectorSelector; func: PushdownFunction; rangeMs: number }, + aggregation: PushdownRequest["aggregation"], + ): boolean => { + const widthMs = instant ? inner.rangeMs : gcd(params.stepMs, inner.rangeMs) + if (widthMs < MIN_BUCKET_MS) return false + out.set(node, { + selector: inner.selector, + request: { + metricName: inner.selector.name, + matchers: inner.selector.matchers, + offsetMs: inner.selector.offsetMs, + func: inner.func, + rangeMs: inner.rangeMs, + startMs: params.startMs, + stepMs: instant ? 0 : params.stepMs, + steps, + widthMs, + aggregation, + }, + }) + return true + } + const walk = (node: Expr): void => { + switch (node.type) { + case "Aggregate": { + const inner = pushableInner(node.expr, lookback) + if ( + inner !== undefined && + isPushdownAggregation(node.op) && + !node.without && + node.param === undefined && + offer(node, inner, { op: node.op, by: node.grouping }) + ) { + return + } + if (node.param !== undefined) walk(node.param) + walk(node.expr) + return + } + case "Call": { + const inner = pushableInner(node, lookback) + if (inner !== undefined && inner.func !== "last" && offer(node, inner, undefined)) return + for (const arg of node.args) walk(arg) + return + } + case "Paren": + case "Unary": + walk(node.expr) + return + case "Binary": + walk(node.lhs) + walk(node.rhs) + return + case "NumberLiteral": + case "StringLiteral": + case "VectorSelector": + case "MatrixSelector": + case "Subquery": + return + } + } + walk(expr) + return out +} + /** Prometheus' cap on points per series: steps of a range query, and points of every subquery. */ export const MAX_POINTS_PER_SERIES = 11_000 diff --git a/lib/promql/src/engine/types.ts b/lib/promql/src/engine/types.ts index a1cd56fbf6..d47bda6c63 100644 --- a/lib/promql/src/engine/types.ts +++ b/lib/promql/src/engine/types.ts @@ -109,3 +109,54 @@ export const DEFAULT_LIMITS: EvalLimits = { maxSeries: 10_000, maxPoints: 5_000_ /** Prometheus' default lookback for instant selectors. */ export const LOOKBACK_MS = 300_000 + +/** Range functions a source may evaluate itself; `last` is an instant selector's lookback. */ +export const PUSHDOWN_FUNCTIONS = [ + "rate", + "increase", + "delta", + "irate", + "idelta", + "resets", + "changes", + "avg_over_time", + "sum_over_time", + "count_over_time", + "min_over_time", + "max_over_time", + "last_over_time", + "present_over_time", + "last", +] as const +export type PushdownFunction = (typeof PUSHDOWN_FUNCTIONS)[number] + +export const PUSHDOWN_AGGREGATIONS = ["sum", "avg", "min", "max", "count", "group"] as const +export type PushdownAggregation = (typeof PUSHDOWN_AGGREGATIONS)[number] + +/** + * A subtree the source may evaluate whole: `func(selector[range])` at every step, + * optionally combined `op by (labels)`. Steps are `startMs + j * stepMs`, the + * window of step j is `(t - offset - range, t - offset]`, and `widthMs` divides + * both the step and the range so windows are whole buckets. + */ +export interface PushdownRequest { + readonly metricName: string | undefined + readonly matchers: ReadonlyArray + readonly offsetMs: number + readonly func: PushdownFunction + readonly rangeMs: number + readonly startMs: number + readonly stepMs: number + readonly steps: number + readonly widthMs: number + readonly aggregation: { readonly op: PushdownAggregation; readonly by: ReadonlyArray } | undefined +} + +/** A source's answer: per series (or per group), the steps that have a value. */ +export interface PushdownData { + readonly series: ReadonlyArray<{ + readonly labels: Labels + readonly steps: ReadonlyArray + readonly values: ReadonlyArray + }> +} diff --git a/packages/backend/src/services/promql/PromqlQueryService.test.ts b/packages/backend/src/services/promql/PromqlQueryService.test.ts index ade64ce4e0..1549713ad2 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.test.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.test.ts @@ -4,6 +4,7 @@ import type { TenantContext } from "@maple/domain/http" import { OrgId, UserId } from "@maple/primitives" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support" +import { WarehouseResponseLimitError } from "@maple/query-engine/execution" import { PromqlQueryService } from "./PromqlQueryService" const tenant: TenantContext = { @@ -69,6 +70,29 @@ describe("PromqlQueryService", () => { }), ) + it.effect("fails cleanly when a selector's buckets would not fit in memory", () => + Effect.gen(function* () { + // The catalog knows `up`; every bounded read reports the response ceiling. + const warehouse = makeWarehouseServiceStub({ + compiledQuery: () => + Effect.succeed([{ metricName: "up", metricType: "gauge", unit: "", isMonotonic: 0 }]), + compiledQueryBounded: () => + Effect.fail(new WarehouseResponseLimitError({ kind: "bytes", message: "too big" })), + }) + const layer = Layer.effect(PromqlQueryService, PromqlQueryService.make).pipe( + Layer.provide(Layer.succeed(WarehouseQueryService, warehouse)), + ) + const error = yield* Effect.gen(function* () { + const service = yield* PromqlQueryService + return yield* Effect.flip( + service.query(tenant, { query: "up", startMs: 0, endMs: 3_600_000, stepMs: 60_000 }), + ) + }).pipe(Effect.provide(layer)) + expect(error._tag).toBe("@maple/backend/promql/PromqlExecutionError") + expect(error.message).toMatch(/too many samples/) + }), + ) + const withCatalog = (rows: ReadonlyArray>) => Layer.effect(PromqlQueryService, PromqlQueryService.make).pipe( Layer.provide( diff --git a/packages/backend/src/services/promql/PromqlQueryService.ts b/packages/backend/src/services/promql/PromqlQueryService.ts index 8af99b2a9c..f5ef43ea65 100644 --- a/packages/backend/src/services/promql/PromqlQueryService.ts +++ b/packages/backend/src/services/promql/PromqlQueryService.ts @@ -2,8 +2,10 @@ // // The engine (`@maple/promql`) parses, plans and evaluates; this service is the // Maple source behind it: one catalog read resolves Prometheus names to stored -// metrics, then every selector's warehouse queries run concurrently and the -// evaluator recombines their pre-aggregated buckets. +// metrics. Subtrees ClickHouse can evaluate whole (a range function over a +// selector, optionally summed by labels) run there and return one value per +// step; every other selector returns pre-aggregated buckets the evaluator +// recombines. import { Context, Effect, Layer, Result, Schema, Array as Arr } from "effect" import { @@ -11,7 +13,10 @@ import { evaluate, type Expr, parse, + planPushdowns, planRequests, + type PushdownCandidate, + type PushdownData, type QueryParams, resolutionError, type QueryResult, @@ -21,9 +26,12 @@ import { } from "@maple/promql" import { CH, formatWarehouseDateTime } from "@maple/query-engine" import { + buildPushdownQuery, buildSelectorQueries, catalogFromRows, catalogQuery, + type PushdownPlan, + toPushdownData, toSelectorData, } from "@maple/query-engine/promql" import type { CatalogMetric } from "@maple/query-engine/promql" @@ -50,12 +58,27 @@ const MAX_SELECTORS = 20 const WAREHOUSE_CONCURRENCY = 4 +/** + * The most one warehouse response may hold. The API runs in a 128 MB isolate; + * a selector the source cannot evaluate whole returns its buckets, and past + * this the query fails cleanly instead of exhausting memory. + */ +const MAX_RESPONSE_BYTES = 24 * 1024 * 1024 + /** Stored metrics one selector may read; `{__name__=~".+"}` would otherwise read them all. */ const MAX_METRICS_PER_SELECTOR = 100 /** Catalog rows read to resolve names; one more comes back when the org has more. */ const CATALOG_LIMIT = 50_000 +const responseLimits = { maxRows: DEFAULT_LIMITS.maxSeries + 1, maxBytes: MAX_RESPONSE_BYTES } + +const tooLarge = () => + new PromqlExecutionError({ + message: + "query processing would load too many samples into memory; aggregate with sum by (...), narrow the selector, or increase the step", + }) + export interface PromqlQuery extends QueryParams { readonly query: string } @@ -119,46 +142,68 @@ const make = Effect.gen(function* () { return catalogFromRows(rows) }) - const readSelector = Effect.fn("PromqlQueryService.readSelector")(function* ( - tenant: TenantContext, - request: SelectorRequest, - catalog: ReadonlyArray, - ) { - const plans = buildSelectorQueries(request, catalog, { - orgId: tenant.orgId, - maxSeries: DEFAULT_LIMITS.maxSeries, - }) - const metrics = Arr.reduce(plans, 0, (n, plan) => n + plan.promNames.size) - if (metrics > MAX_METRICS_PER_SELECTOR) { - return yield* new PromqlExecutionError({ - message: `a selector matches ${metrics} metrics, at most ${MAX_METRICS_PER_SELECTOR}; narrow its __name__ matcher`, + const readSelector = Effect.fn("PromqlQueryService.readSelector")( + function* (tenant: TenantContext, request: SelectorRequest, catalog: ReadonlyArray) { + const plans = buildSelectorQueries(request, catalog, { + orgId: tenant.orgId, + maxSeries: DEFAULT_LIMITS.maxSeries, }) - } - yield* Effect.annotateCurrentSpan({ - "maple.promql.metric": request.metricName ?? "", - "maple.promql.bucket_ms": request.widthMs, - "maple.promql.queries": plans.length, - }) - const results = yield* Effect.forEach( - plans, - (plan) => - Effect.map( - warehouse.compiledQuery(tenant, plan.compiled, { - profile: "aggregation", - context: "promqlSelector", - }), - (rows) => ({ plan, rows }), - ), - { concurrency: WAREHOUSE_CONCURRENCY }, - ) - // A query that hit its LIMIT was cut before the `le` check after it; refuse a partial answer. - if (Arr.some(results, ({ rows }) => rows.length > DEFAULT_LIMITS.maxSeries)) { - return yield* new PromqlExecutionError({ - message: `query selects more than ${DEFAULT_LIMITS.maxSeries} series; narrow the selector`, + const metrics = Arr.reduce(plans, 0, (n, plan) => n + plan.promNames.size) + if (metrics > MAX_METRICS_PER_SELECTOR) { + return yield* new PromqlExecutionError({ + message: `a selector matches ${metrics} metrics, at most ${MAX_METRICS_PER_SELECTOR}; narrow its __name__ matcher`, + }) + } + yield* Effect.annotateCurrentSpan({ + "maple.promql.metric": request.metricName ?? "", + "maple.promql.bucket_ms": request.widthMs, + "maple.promql.queries": plans.length, }) - } - return toSelectorData(request, results) - }) + const results = yield* Effect.forEach( + plans, + (plan) => + Effect.map( + warehouse.compiledQueryBounded(tenant, plan.compiled, { + profile: "aggregation", + context: "promqlSelector", + responseLimits, + }), + (rows) => ({ plan, rows }), + ), + { concurrency: WAREHOUSE_CONCURRENCY }, + ) + // A query that hit its LIMIT was cut before the `le` check after it; refuse a partial answer. + if (Arr.some(results, ({ rows }) => rows.length > DEFAULT_LIMITS.maxSeries)) { + return yield* new PromqlExecutionError({ + message: `query selects more than ${DEFAULT_LIMITS.maxSeries} series; narrow the selector`, + }) + } + return toSelectorData(request, results) + }, + Effect.catchTag("@maple/query-engine/execution/WarehouseResponseLimitError", () => + Effect.fail(tooLarge()), + ), + ) + + const readPushdown = Effect.fn("PromqlQueryService.readPushdown")( + function* (tenant: TenantContext, candidate: PushdownCandidate, plan: PushdownPlan) { + yield* Effect.annotateCurrentSpan({ + "maple.promql.metric": plan.promName, + "maple.promql.function": candidate.request.func, + "maple.promql.aggregation": candidate.request.aggregation?.op ?? "", + "maple.promql.bucket_ms": candidate.request.widthMs, + }) + const rows = yield* warehouse.compiledQueryBounded(tenant, plan.compiled, { + profile: "aggregation", + context: "promqlPushdown", + responseLimits, + }) + return toPushdownData(candidate.request, plan, rows) + }, + Effect.catchTag("@maple/query-engine/execution/WarehouseResponseLimitError", () => + Effect.fail(tooLarge()), + ), + ) const query = Effect.fn("PromqlQueryService.query")(function* ( tenant: TenantContext, @@ -183,8 +228,24 @@ const make = Effect.gen(function* () { entries.length === 0 ? [] : yield* readCatalog(tenant, span(Arr.map(entries, ([, request]) => request))) + const options = { orgId: tenant.orgId, maxSeries: DEFAULT_LIMITS.maxSeries } + const pushdowns = Arr.flatMap([...planPushdowns(expr, input)], ([node, candidate]) => { + const plan = buildPushdownQuery(candidate.request, catalog, options) + return plan === undefined ? [] : [{ node, candidate, plan }] + }) + const skipped = new Set(Arr.map(pushdowns, ({ candidate }) => candidate.selector)) + yield* Effect.annotateCurrentSpan("maple.promql.pushdowns", pushdowns.length) + const pushed = yield* Effect.forEach( + pushdowns, + ({ node, candidate, plan }) => + Effect.map( + readPushdown(tenant, candidate, plan), + (pushdownData): readonly [Expr, PushdownData] => [node, pushdownData], + ), + { concurrency: WAREHOUSE_CONCURRENCY }, + ) const read = yield* Effect.forEach( - entries, + Arr.filter(entries, ([selector]) => !skipped.has(selector)), ([selector, request]) => Effect.map( readSelector(tenant, request, catalog), @@ -192,7 +253,7 @@ const make = Effect.gen(function* () { ), { concurrency: WAREHOUSE_CONCURRENCY }, ) - const result = evaluate({ expr, params: input, data: new Map(read) }) + const result = evaluate({ expr, params: input, data: new Map(read), pushed: new Map(pushed) }) if (Result.isFailure(result)) return yield* new PromqlExecutionError({ message: result.failure.message }) return result.success satisfies QueryResult diff --git a/packages/query-engine/src/ch/queries/promql.ts b/packages/query-engine/src/ch/queries/promql.ts index b48461669d..b2a9131851 100644 --- a/packages/query-engine/src/ch/queries/promql.ts +++ b/packages/query-engine/src/ch/queries/promql.ts @@ -379,3 +379,359 @@ export const promqlCatalogRowSchema = Schema.Struct({ isMonotonic: CHNumber, }) export type PromqlCatalogRow = typeof promqlCatalogRowSchema.Type + +// Pushdown: the source evaluates `func(selector[range])` per (series, step), and optionally +// `op by (labels)` per (group, step), so one value per step leaves ClickHouse instead of every +// bucket. The window of step j holds buckets k in (j*s - m, j*s] (s = step/width, m = +// range/width); each bucket fans out to the steps whose window holds it, and the window is +// recombined exactly as the evaluator does (lib/promql/src/engine/window.ts). + +export type PromqlPushdownFunction = + | "rate" + | "increase" + | "delta" + | "irate" + | "idelta" + | "resets" + | "changes" + | "avg_over_time" + | "sum_over_time" + | "count_over_time" + | "min_over_time" + | "max_over_time" + | "last_over_time" + | "present_over_time" + | "last" + +export interface PromqlPushdownOpts extends Omit { + readonly func: PromqlPushdownFunction + readonly rangeMs: number + /** Evaluation steps; step j is `originMs + j * stepBuckets * widthMs` (offset already applied). */ + readonly steps: number + readonly stepBuckets: number + readonly rangeBuckets: number + readonly limit: number +} + +const linkedQuery = (opts: PromqlPushdownOpts) => + fromQuery(bucketQuery({ ...opts, needsValues: false, limit: opts.limit }), "buckets").select(($) => { + const prev = CH.windowSpec({ + partitionBy: [$.sk], + orderBy: [[$.end, "asc"]], + frame: CH.rowsBetween(CH.preceding(1), CH.currentRow), + }) + return { + sk: $.sk, + MetricName: $.MetricName, + ServiceName: $.ServiceName, + Attributes: $.Attributes, + ResourceAttributes: $.ResourceAttributes, + le: $.le, + k: sql(T.int64)`intDiv(${$.end} - ${param.int("originMs")}, ${opts.widthMs})`, + pk: sql( + T.int64, + )`intDiv(${CH.over(CH.lagInFrame($.end, 1, $.end), prev)} - ${param.int("originMs")}, ${opts.widthMs})`, + hasPrev: sql(T.uint8)`${CH.over(CH.count(), prev)} = 2`, + pLastT: CH.over(CH.lagInFrame($.lastT, 1, $.lastT), prev), + pLastV: CH.over(CH.lagInFrame($.lastV, 1, $.lastV), prev), + firstT: $.firstT, + firstV: $.firstV, + lastT: $.lastT, + lastV: $.lastV, + prevT: $.prevT, + prevV: $.prevV, + count: $.count, + sum: $.sum, + min: $.min, + max: $.max, + sumSquares: $.sumSquares, + increase: $.increase, + resets: $.resets, + changes: $.changes, + } + }) + +/** Each bucket once per step whose window holds it, with whether its predecessor is in that window too. */ +const expandedQuery = (opts: PromqlPushdownOpts) => + fromQuery(linkedQuery(opts), "linked").select(($) => { + const s = opts.stepBuckets + const m = opts.rangeBuckets + const j = sql( + T.int64, + )`arrayJoin(range(toInt64(greatest(0, ceil(${$.k} / ${s}))), toInt64(least(${opts.steps - 1}, floor((${$.k} + ${m - 1}) / ${s}))) + 1))` + return { + sk: $.sk, + MetricName: $.MetricName, + ServiceName: $.ServiceName, + Attributes: $.Attributes, + ResourceAttributes: $.ResourceAttributes, + le: $.le, + k: $.k, + j, + // The pair (predecessor's last sample, this bucket's first) is in the window of step j. + prevIn: sql(T.uint8)`${$.hasPrev} AND ${$.pk} > ${j} * ${s} - ${m}`, + pLastT: $.pLastT, + pLastV: $.pLastV, + firstT: $.firstT, + firstV: $.firstV, + lastT: $.lastT, + lastV: $.lastV, + prevT: $.prevT, + prevV: $.prevV, + count: $.count, + sum: $.sum, + min: $.min, + max: $.max, + sumSquares: $.sumSquares, + increase: $.increase, + resets: $.resets, + changes: $.changes, + } + }) + +/** The window of every (series, step), recombined from its buckets. */ +const windowQuery = (opts: PromqlPushdownOpts) => + fromQuery(expandedQuery(opts), "expanded") + .select(($) => { + const crossIncrease = sql( + T.float64, + )`sumIf(if(${$.firstV} < ${$.pLastV}, ${$.firstV}, ${$.firstV} - ${$.pLastV}), ${$.prevIn} = 1)` + return { + sk: $.sk, + j: $.j, + MetricName: CH.any_($.MetricName), + ServiceName: CH.any_($.ServiceName), + Attributes: CH.any_($.Attributes), + ResourceAttributes: CH.any_($.ResourceAttributes), + le: CH.any_($.le), + n: sql(T.float64)`sum(${$.count})`, + firstT: CH.argMin($.firstT, $.k), + firstV: CH.argMin($.firstV, $.k), + lastT: CH.argMax($.lastT, $.k), + lastV: CH.argMax($.lastV, $.k), + lastCount: CH.argMax($.count, $.k), + lastPrevT: CH.argMax($.prevT, $.k), + lastPrevV: CH.argMax($.prevV, $.k), + lastPrevIn: CH.argMax($.prevIn, $.k), + lastPLastT: CH.argMax($.pLastT, $.k), + lastPLastV: CH.argMax($.pLastV, $.k), + sum: CH.sum($.sum), + min: CH.min_($.min), + max: CH.max_($.max), + sumSquares: CH.sum($.sumSquares), + increase: sql(T.float64)`sum(${$.increase}) + ${crossIncrease}`, + resets: sql( + T.float64, + )`sum(${$.resets}) + countIf(${$.prevIn} = 1 AND ${$.firstV} < ${$.pLastV})`, + changes: sql( + T.float64, + )`sum(${$.changes}) + countIf(${$.prevIn} = 1 AND ${$.firstV} != ${$.pLastV})`, + } + }) + .groupBy("sk", "j") + +type WindowRow = { + readonly j: CH.Expr + readonly n: CH.Expr + readonly firstT: CH.Expr + readonly firstV: CH.Expr + readonly lastT: CH.Expr + readonly lastV: CH.Expr + readonly lastCount: CH.Expr + readonly lastPrevT: CH.Expr + readonly lastPrevV: CH.Expr + readonly lastPrevIn: CH.Expr + readonly lastPLastT: CH.Expr + readonly lastPLastV: CH.Expr + readonly sum: CH.Expr + readonly min: CH.Expr + readonly max: CH.Expr + readonly sumSquares: CH.Expr + readonly increase: CH.Expr + readonly resets: CH.Expr + readonly changes: CH.Expr +} + +/** Prometheus' extrapolatedRate (rate, increase, delta) over the window ending at step j. */ +const extrapolated = (opts: PromqlPushdownOpts, $: WindowRow) => { + const isCounter = opts.func !== "delta" + const stepEnd = sql(T.float64)`(${param.int("originMs")} + ${$.j} * ${opts.stepBuckets * opts.widthMs})` + const result = isCounter ? sql(T.float64)`${$.increase}` : sql(T.float64)`(${$.lastV} - ${$.firstV})` + const sampled = sql(T.float64)`((${$.lastT} - ${$.firstT}) / 1000)` + const between = sql(T.float64)`(${sampled} / (${$.n} - 1))` + const toStart0 = sql(T.float64)`((${$.firstT} - (${stepEnd} - ${opts.rangeMs})) / 1000)` + const toStart1 = sql(T.float64)`if(${toStart0} >= ${between} * 1.1, ${between} / 2, ${toStart0})` + const toZero = sql(T.float64)`(${sampled} * (${$.firstV} / ${result}))` + const toStart = isCounter + ? sql( + T.float64, + )`if(${result} > 0 AND ${$.firstV} >= 0 AND ${toZero} < ${toStart1}, ${toZero}, ${toStart1})` + : toStart1 + const toEnd0 = sql(T.float64)`((${stepEnd} - ${$.lastT}) / 1000)` + const toEnd = sql(T.float64)`if(${toEnd0} >= ${between} * 1.1, ${between} / 2, ${toEnd0})` + const factor = sql(T.float64)`((${sampled} + ${toStart} + ${toEnd}) / ${sampled})` + const value = + opts.func === "rate" + ? sql(T.float64)`${result} * ${factor} / ${opts.rangeMs / 1000}` + : sql(T.float64)`${result} * ${factor}` + return { value, valid: sql(T.uint8)`${$.n} >= 2 AND ${$.lastT} > ${$.firstT}` } +} + +/** irate and idelta: the window's last two samples. */ +const instantPair = (opts: PromqlPushdownOpts, $: WindowRow) => { + const pT = sql(T.float64)`if(${$.lastCount} >= 2, ${$.lastPrevT}, ${$.lastPLastT})` + const pV = sql(T.float64)`if(${$.lastCount} >= 2, ${$.lastPrevV}, ${$.lastPLastV})` + const hasPair = sql(T.uint8)`(${$.lastCount} >= 2 OR ${$.lastPrevIn} = 1)` + const value = + opts.func === "irate" + ? sql( + T.float64, + )`if(${$.lastV} < ${pV}, ${$.lastV}, ${$.lastV} - ${pV}) / ((${$.lastT} - ${pT}) / 1000)` + : sql(T.float64)`${$.lastV} - ${pV}` + return { value, valid: sql(T.uint8)`${hasPair} AND ${$.lastT} > ${pT}` } +} + +/** The function's value for one (series, step) window, and whether it has one. */ +const windowValue = (opts: PromqlPushdownOpts, $: WindowRow) => { + const any = sql(T.uint8)`${$.n} >= 1` + switch (opts.func) { + case "rate": + case "increase": + case "delta": + return extrapolated(opts, $) + case "irate": + case "idelta": + return instantPair(opts, $) + case "resets": + return { value: $.resets, valid: any } + case "changes": + return { value: $.changes, valid: any } + case "avg_over_time": + return { value: sql(T.float64)`${$.sum} / ${$.n}`, valid: any } + case "sum_over_time": + return { value: $.sum, valid: any } + case "count_over_time": + return { value: $.n, valid: any } + case "min_over_time": + return { value: $.min, valid: any } + case "max_over_time": + return { value: $.max, valid: any } + case "last_over_time": + case "last": + return { value: $.lastV, valid: any } + case "present_over_time": + return { value: sql(T.float64)`1`, valid: any } + } +} + +const valuedQuery = (opts: PromqlPushdownOpts) => + fromQuery(windowQuery(opts), "windows").select(($) => { + const { value, valid } = windowValue(opts, $) + return { + sk: $.sk, + j: $.j, + MetricName: $.MetricName, + ServiceName: $.ServiceName, + Attributes: $.Attributes, + ResourceAttributes: $.ResourceAttributes, + le: $.le, + value: sql(T.float64)`toFloat64(${value})`, + valid: sql(T.uint8)`${valid}`, + } + }) + +/** One row per series: its labels' sources and the steps that have a value. */ +const pushdownSeriesQuery = (opts: PromqlPushdownOpts) => + fromQuery(valuedQuery(opts), "valued") + .select(($) => ({ + series: sql(T.string)`toString(${$.sk})`, + metricName: CH.any_($.MetricName), + serviceName: CH.any_($.ServiceName), + attributes: CH.any_($.Attributes), + resourceAttributes: CH.any_($.ResourceAttributes), + le: sql(T.string)`toString(any(${$.le}))`, + groups: sql(T.array(T.string))`emptyArrayString()`, + steps: sql(T.array(T.float64))`groupArray(toFloat64(${$.j}))`, + values: sql(T.array(T.float64))`groupArray(${$.value})`, + })) + .where(($) => [$.valid.eq(1)]) + .groupBy("series") + .limit(opts.limit) + .format("JSON") + +/** Combine the series of each group; `labels` are label expressions in `by` order, `le` the bucket bound. */ +export interface PromqlPushdownAggregation { + readonly op: "sum" | "avg" | "min" | "max" | "count" | "group" + readonly labels: ($: PromqlLabelColumns) => ReadonlyArray> + readonly byLe: boolean +} + +const aggregate = (op: PromqlPushdownAggregation["op"], value: CH.Expr) => { + switch (op) { + case "sum": + return sql(T.float64)`sum(${value})` + case "avg": + return sql(T.float64)`avg(${value})` + case "min": + return sql(T.float64)`min(${value})` + case "max": + return sql(T.float64)`max(${value})` + case "count": + return sql(T.float64)`toFloat64(count())` + case "group": + return sql(T.float64)`1` + } +} + +/** One row per group: the group's label values (in `by` order) and its value per step. */ +const pushdownGroupQuery = (opts: PromqlPushdownOpts, aggregation: PromqlPushdownAggregation) => { + const perStep = fromQuery(valuedQuery(opts), "valued") + .select(($) => { + const labels = aggregation.labels($) + return { + j: $.j, + groups: + labels.length === 0 + ? sql(T.array(T.string))`emptyArrayString()` + : sql(T.array(T.string))`[${sql.join(labels, ", ")}]`, + gle: aggregation.byLe ? $.le : sql(T.float64)`0`, + value: aggregate(aggregation.op, $.value), + } + }) + .where(($) => [$.valid.eq(1)]) + .groupBy("j", "groups", "gle") + return fromQuery(perStep, "per_step") + .select(($) => ({ + series: sql(T.string)`''`, + metricName: sql(T.string)`''`, + serviceName: sql(T.string)`''`, + attributes: sql(stringMap)`map()`, + resourceAttributes: sql(stringMap)`map()`, + le: sql(T.string)`toString(${$.gle})`, + groups: $.groups, + steps: sql(T.array(T.float64))`groupArray(toFloat64(${$.j}))`, + values: sql(T.array(T.float64))`groupArray(${$.value})`, + })) + .groupBy("groups", "le") + .limit(opts.limit) + .format("JSON") +} + +/** Per series. Params: `orgId`, epoch ms `startMs`/`endMs` (scan), `originMs` (step 0's window end), `timeOriginMs`. */ +export const promqlPushdownSeriesQuery = pushdownSeriesQuery + +/** Per `by` group; same params and row shape as `promqlPushdownSeriesQuery`. */ +export const promqlPushdownGroupQuery = pushdownGroupQuery + +export const promqlPushdownRowSchema = Schema.Struct({ + series: Schema.String, + metricName: Schema.String, + serviceName: Schema.String, + attributes: Schema.Record(Schema.String, Schema.String), + resourceAttributes: Schema.Record(Schema.String, Schema.String), + le: Schema.String, + groups: Schema.Array(Schema.String), + steps: numbers, + values: numbers, +}) +export type PromqlPushdownRow = typeof promqlPushdownRowSchema.Type diff --git a/packages/query-engine/src/promql/index.ts b/packages/query-engine/src/promql/index.ts index 43a35be15b..28f2090bae 100644 --- a/packages/query-engine/src/promql/index.ts +++ b/packages/query-engine/src/promql/index.ts @@ -3,6 +3,8 @@ export * from "./names" export * from "./source" export { type PromqlCatalogRow, + type PromqlPushdownRow, + promqlPushdownRowSchema, promqlCatalogRowSchema, type PromqlSelectorRow, promqlSelectorRowSchema, diff --git a/packages/query-engine/src/promql/source.ts b/packages/query-engine/src/promql/source.ts index ec8669c52a..5d7ea7a8fe 100644 --- a/packages/query-engine/src/promql/source.ts +++ b/packages/query-engine/src/promql/source.ts @@ -2,18 +2,30 @@ // (table, component) of the stored metrics its name resolves to, and their rows become SelectorData. // Pure: the caller runs the compiled queries. -import type { LabelMatcher, SampleBucket, SelectorData, SelectorRequest, SeriesData } from "@maple/promql" -import { Array as Arr, Order } from "effect" +import type { + LabelMatcher, + PushdownData, + PushdownRequest, + SampleBucket, + SelectorData, + SelectorRequest, + SeriesData, +} from "@maple/promql" +import { Array as Arr, Order, Record as Rec } from "effect" import { compile } from "@maple-dev/effect-orm/clickhouse" import type { OrgId } from "@maple/primitives" import { METRIC_UNIT_SEPARATOR, type PromqlCatalogRow, type PromqlSelectorRow, + type PromqlPushdownRow, promqlCatalogQuery, + type PromqlPushdownOpts, + promqlPushdownGroupQuery, + promqlPushdownSeriesQuery, promqlSelectorQuery, } from "../ch/queries/promql" -import { matchesAll, matcherCondition, seriesLabels } from "./labels" +import { formatLe, labelValueExpr, matchesAll, matcherCondition, seriesLabels } from "./labels" import { type CatalogMetric, type MetricComponent, @@ -202,3 +214,121 @@ export const catalogFromRows = (rows: ReadonlyArray): Readonly ] : [], ) + +export interface PushdownPlan { + readonly promName: string + readonly component: MetricComponent + readonly compiled: ReturnType +} + +const compilePushdown = ( + request: PushdownRequest, + target: PromNameTarget, + labelMatchers: ReadonlyArray, + byLabels: ReadonlyArray, + options: SelectorPlanOptions, +) => { + const originMs = request.startMs - request.offsetMs + const lastEnd = originMs + (request.steps - 1) * request.stepMs + const params = { + orgId: options.orgId, + startMs: originMs - request.rangeMs, + endMs: lastEnd, + originMs, + timeOriginMs: originMs, + } + const opts: PromqlPushdownOpts = { + table: target.table, + component: target.component, + targets: [target], + widthMs: request.widthMs, + func: request.func, + rangeMs: request.rangeMs, + steps: request.steps, + stepBuckets: request.steps === 1 ? 1 : request.stepMs / request.widthMs, + rangeBuckets: request.rangeMs / request.widthMs, + limit: options.maxSeries + 1, + conditions: ($) => Arr.map(labelMatchers, (m) => matcherCondition($, m)), + } + const aggregation = request.aggregation + if (aggregation === undefined) return compile(promqlPushdownSeriesQuery(opts), params) + return compile( + promqlPushdownGroupQuery(opts, { + op: aggregation.op, + labels: ($) => + Arr.flatMap(byLabels, (name) => { + const expr = labelValueExpr($, name) + return expr === undefined ? [] : [expr] + }), + byLe: aggregation.by.includes("le"), + }), + params, + ) +} + +/** + * The warehouse query for a pushdown, or undefined when it cannot be exact: the selector must + * resolve to one stored metric, and every label matcher must run in SQL (an `le` matcher is + * only checked after the fetch). The evaluator then reads the selector instead. + */ +export const buildPushdownQuery = ( + request: PushdownRequest, + catalog: ReadonlyArray, + options: SelectorPlanOptions, +): PushdownPlan | undefined => { + const targets = resolveSelectorName( + Arr.filter(request.matchers, (m) => m.name === "__name__"), + catalog, + ) + const target = targets[0] + if (targets.length !== 1 || target === undefined) return undefined + const labelMatchers = Arr.filter(request.matchers, (m) => m.name !== "__name__") + if (Arr.some(labelMatchers, (m) => m.name === "le")) return undefined + // Grouping reads `le` from the bucket bound; an `le` attribute on other data is not supported. + if (request.aggregation?.by.includes("le") === true && target.component !== "bucket") return undefined + if (request.widthMs <= 0 || request.rangeMs % request.widthMs !== 0) return undefined + if (request.steps > 1 && request.stepMs % request.widthMs !== 0) return undefined + const byLabels = Arr.filter(request.aggregation?.by ?? [], (name) => name !== "le" && name !== "__name__") + return { + promName: target.promName, + component: target.component, + compiled: compilePushdown(request, target, labelMatchers, byLabels, options), + } +} + +/** The pushdown's rows as the evaluator reads them. */ +export const toPushdownData = ( + request: PushdownRequest, + plan: PushdownPlan, + rows: ReadonlyArray, +): PushdownData => { + const by = request.aggregation?.by + const byLabels = Arr.filter(by ?? [], (name) => name !== "le" && name !== "__name__") + return { + series: Arr.map(rows, (row) => { + if (by === undefined) { + const labels = seriesLabels({ + promName: plan.promName, + serviceName: row.serviceName, + attributes: row.attributes, + resourceAttributes: row.resourceAttributes, + le: plan.component === "bucket" ? parseLe(row.le) : undefined, + }) + return { labels, steps: row.steps, values: row.values } + } + const entries: Array = Arr.map(byLabels, (name, i) => [ + name, + row.groups[i] ?? "", + ]) + if (by.includes("le") && plan.component === "bucket") + entries.push(["le", formatLe(parseLe(row.le))]) + // Every pushed function but last_over_time drops the name before aggregating. + if (by.includes("__name__") && (request.func === "last" || request.func === "last_over_time")) { + entries.push(["__name__", plan.promName]) + } + // fromEntries defines own properties, so `by (__proto__)` stays a label. + const labels = Rec.fromEntries(Arr.filter(entries, ([, value]) => value !== "")) + return { labels, steps: row.steps, values: row.values } + }), + } +}