From 2482e6ad5e27f30a976c67030a85ce3506dbf156 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 16 Nov 2026 16:53:21 +0100 Subject: [PATCH 1/2] fix(web): raw SQL tiles without a display type load, trace paging stops at the API's offset cap A raw SQL widget may be saved without displayType (the stored source and the share API both allow it), but the browser's raw_sql_chart fetch required one, so those tiles failed with "Missing key at displayType". It now returns the rows as sent, like the share API. The traces list asks for (pages + 1) * 100 rows consumed. With noise traces hidden, fewer than 2000 rows can show while that offset passes the query's 2000 cap, and the request constructor threw, crashing the page. The list now stops at MAX_TRACES_LIST_OFFSET, and listTraces rejects a deeper offset as a typed decode error. --- .../src/page/traces/list/traces.scene.test.ts | 47 +++++++++++- apps/web/src/page/traces/list/update.ts | 8 +- packages/domain/src/query-engine.ts | 13 ++-- .../src/warehouse/raw-sql-chart.test.ts | 73 +++++++++++++++++++ .../web-data/src/warehouse/raw-sql-chart.ts | 16 ++-- .../web-data/src/warehouse/traces.test.ts | 18 +++++ packages/web-data/src/warehouse/traces.ts | 9 ++- 7 files changed, 169 insertions(+), 15 deletions(-) create mode 100644 packages/web-data/src/warehouse/raw-sql-chart.test.ts diff --git a/apps/web/src/page/traces/list/traces.scene.test.ts b/apps/web/src/page/traces/list/traces.scene.test.ts index 9363c7cd5a..f8f15536c2 100644 --- a/apps/web/src/page/traces/list/traces.scene.test.ts +++ b/apps/web/src/page/traces/list/traces.scene.test.ts @@ -1,5 +1,6 @@ import { OrgId } from "@maple/domain" -import { Result, Schema, Array as Arr, String as Str } from "effect" +import { MAX_TRACES_LIST_OFFSET } from "@maple/domain/query-engine" +import { Option, Result, Schema, Array as Arr, String as Str } from "effect" import { click, Command, @@ -192,6 +193,50 @@ describe("traces list", () => { expect(retried.model.pagination).toMatchObject({ isFailed: false, isFetching: true }) }) + test("paging stops at the API's deepest offset even when hidden noise keeps the shown rows few", () => { + // Ten traces shown from each hundred read: the offset reaches the cap long before the rows do. + const sparse = (pageIndex: number): TracesPage => ({ + rows: Array.from({ length: 10 }, (_, i) => trace(`p${pageIndex}-${i}`)), + scannedCount: 100, + hiddenCount: 90, + }) + type Asked = ReturnType["commands"] + const nextOffsetOf = (commands: Asked): number | null => { + const [command] = commands ?? [] + return command?.name === FetchNextPage.name && + command.args !== undefined && + "offset" in command.args + ? Number(command.args.offset) + : null + } + type State = { model: Model; asked: Asked; offsets: ReadonlyArray } + // An appended page that leaves the view near the end asks for the next one itself. + const step = (state: State, pageIndex: number): State => { + const near = + nextOffsetOf(state.asked) === null + ? pageUpdate(state.model, windowOver(0, allRows(state.model).length)) + : { model: state.model, commands: state.asked } + const offset = nextOffsetOf(near.commands) + if (offset === null) return { ...state, model: near.model, asked: [] } + const answered = pageUpdate( + near.model, + Message.CompletedFetchNextPage({ + key: near.model.pagination.key ?? "", + generation: near.model.pagination.generation, + offset, + result: Result.succeed(sparse(pageIndex)), + }), + ) + return { model: answered.model, asked: answered.commands, offsets: [...state.offsets, offset] } + } + const initial: State = { model: loaded(sparse(0)), asked: [], offsets: [] } + const { model, offsets } = Arr.reduce(Arr.range(1, 25), initial, step) + expect(Arr.last(offsets)).toEqual(Option.some(MAX_TRACES_LIST_OFFSET)) + expect(Arr.every(offsets, (offset) => offset <= MAX_TRACES_LIST_OFFSET)).toBe(true) + expect(allRows(model)).toHaveLength(10 + offsets.length * 10) + expect(nextOffsetOf(pageUpdate(model, windowOver(0, allRows(model).length)).commands)).toBeNull() + }) + test("a next page answers only its own request: filter A, B, then A again appends page 2 once", () => { const resolved = (model: Model) => pageUpdate( diff --git a/apps/web/src/page/traces/list/update.ts b/apps/web/src/page/traces/list/update.ts index ad856f58b4..579bdd43d1 100644 --- a/apps/web/src/page/traces/list/update.ts +++ b/apps/web/src/page/traces/list/update.ts @@ -1,5 +1,6 @@ import { pickerStep } from "../../kit/page-time" import type { RequestMode } from "@maple/views/data/keyed" +import { MAX_TRACES_LIST_OFFSET } from "@maple/domain/query-engine" import { Array as Arr, Option, Result } from "effect" import { AsyncData, Command } from "foldkit" import * as Keyed from "@maple/views/data/keyed" @@ -67,9 +68,14 @@ export const hiddenCount = (model: Model): number => first.hiddenCount + Arr.reduce(model.pagination.pages, 0, (sum, page) => sum + page.hiddenCount), }) +/** The warehouse rows the pages so far consumed: the next page starts there. */ +const nextOffset = (pagination: Pagination): number => (pagination.pages.length + 1) * PAGE_SIZE + /** More pages exist when the last warehouse page came back full before the noise filter. */ export const hasNextPage = (model: Model): boolean => { if (allRows(model).length >= MAX_RETAINED_TRACES || model.pagination.isStopped) return false + // Hidden noise traces keep the shown rows under the cap while the offset passes the API's. + if (nextOffset(model.pagination) > MAX_TRACES_LIST_OFFSET) return false const first = AsyncData.getData(model.firstPage.data) if (Option.isNone(first) || AsyncData.isFailure(model.firstPage.data)) return false const last = Arr.last(model.pagination.pages) @@ -145,8 +151,6 @@ const requestAll = (model: Model, context: PageContext, window: TimeWindow, mode } } -const nextOffset = (pagination: Pagination): number => (pagination.pages.length + 1) * PAGE_SIZE - /** Asks for the next page, one at a time; also the retry of a page that failed. */ const fetchNext = (model: Model, context: PageContext): Return => { const { window, pagination } = model diff --git a/packages/domain/src/query-engine.ts b/packages/domain/src/query-engine.ts index a8b5da1eeb..7c184cb95b 100644 --- a/packages/domain/src/query-engine.ts +++ b/packages/domain/src/query-engine.ts @@ -421,6 +421,13 @@ export const MetricsSparklinesQuery = Schema.Struct({ }) export type MetricsSparklinesQuery = Schema.Schema.Type +/** + * The deepest offset the trace list reads. The client pages by rows consumed (noise + * traces are dropped after the read), so it must stop on this, not on rows shown. + * Stage 1 only reads the three light sort columns, so a deep offset stays cheap. + */ +export const MAX_TRACES_LIST_OFFSET = 2_000 + export const TracesListQuery = Schema.Struct({ kind: Schema.Literal("list"), source: Schema.Literal("traces"), @@ -436,15 +443,11 @@ export const TracesListQuery = Schema.Struct({ limit: Schema.optional( Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(200)), ), - // Bound matches the web list's MAX_RETAINED_TRACES (2000): with server-side - // noise filtering the client pages by rows CONSUMED, so a 1000 cap would end - // pagination while far fewer rows are visible. Stage 1 only reads the three - // light sort columns, so a deep offset stays cheap. offset: Schema.optional( Schema.Number.check( Schema.isInt(), Schema.isGreaterThanOrEqualTo(0), - Schema.isLessThanOrEqualTo(2000), + Schema.isLessThanOrEqualTo(MAX_TRACES_LIST_OFFSET), ), ), cursor: Schema.optional(Schema.String), diff --git a/packages/web-data/src/warehouse/raw-sql-chart.test.ts b/packages/web-data/src/warehouse/raw-sql-chart.test.ts new file mode 100644 index 0000000000..07edd63069 --- /dev/null +++ b/packages/web-data/src/warehouse/raw-sql-chart.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "@effect/vitest" +import { MapleInternalApi } from "@maple/domain/http" +import { Effect, Layer } from "effect" +import { FetchHttpClient } from "effect/http" +import { HttpApiClient } from "effect/http-api" +import { getRawSqlChart } from "./raw-sql-chart" +import { MapleInternalHttp, WarehouseExecutor } from "./warehouse-services" + +const rows = [ + { bucket: "2026-11-16 10:00:00", requests: 3 }, + { bucket: "2026-11-16 10:01:00", requests: 5 }, +] + +const answerRawSql = (): Promise => + Promise.resolve( + Response.json({ + data: rows, + meta: { rowCount: rows.length, columns: ["bucket", "requests"], granularitySeconds: 60 }, + }), + ) + +const ApiLayer = Layer.mergeAll( + Layer.mock(WarehouseExecutor, {}), + Layer.effect( + MapleInternalHttp, + HttpApiClient.make(MapleInternalApi, { baseUrl: "https://api.test" }), + ).pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(Layer.succeed(FetchHttpClient.Fetch, answerRawSql)), + ), +) + +const withApi = (effect: Effect.Effect) => + effect.pipe(Effect.provide(ApiLayer)) + +const window = { startTime: "2026-11-16 10:00:00", endTime: "2026-11-16 11:00:00" } + +describe("getRawSqlChart", () => { + it.effect("runs a widget saved without a display type and returns its rows as the API sent them", () => + withApi( + Effect.gen(function* () { + const result = yield* getRawSqlChart({ data: { sql: "SELECT 1", ...window } }) + expect(result.data).toEqual(rows) + expect(result.meta.displayType).toBeUndefined() + }), + ), + ) + + it.effect("treats a display type this build does not know like a missing one", () => + withApi( + Effect.gen(function* () { + const result = yield* getRawSqlChart({ + data: { sql: "SELECT 1", displayType: "sankey", ...window }, + }) + expect(result.data).toEqual(rows) + expect(result.meta.displayType).toBeUndefined() + }), + ), + ) + + it.effect("still reshapes a line chart's rows into series", () => + withApi( + Effect.gen(function* () { + const result = yield* getRawSqlChart({ + data: { sql: "SELECT 1", displayType: "line", ...window }, + }) + expect(result.meta.displayType).toBe("line") + expect(result.data).toHaveLength(2) + expect(result.data[0]).toMatchObject({ bucket: expect.any(String), requests: 3 }) + }), + ), + ) +}) diff --git a/packages/web-data/src/warehouse/raw-sql-chart.ts b/packages/web-data/src/warehouse/raw-sql-chart.ts index 504aa4976e..798ee754df 100644 --- a/packages/web-data/src/warehouse/raw-sql-chart.ts +++ b/packages/web-data/src/warehouse/raw-sql-chart.ts @@ -7,7 +7,7 @@ import { rawSqlRowsForDisplay } from "@maple/query-engine" // Raw SQL chart server function (widget data source `raw_sql_chart`). // // Widget params shape: -// { sql, displayType, granularitySeconds?, startTime, endTime, ... } +// { sql, displayType?, granularitySeconds?, startTime, endTime, ... } // displayType ∈ "line" | "area" | "bar" | "table" | "stat" | "pie" | "histogram" | "heatmap" // // Returns rows in a renderer-friendly shape: @@ -25,9 +25,11 @@ import { rawSqlRowsForDisplay } from "@maple/query-engine" // - heatmap → raw rows; chart accepts `{ x, y, value }` or wide // `{ name, …numeric }` formats. +// Optional and open, like the stored source (`RawSqlWidgetDataSource`) and the share API: a widget +// saved without one, or with a type this build does not know, gets its rows as returned. const GetRawSqlChartInputSchema = Schema.Struct({ sql: Schema.String, - displayType: RawSqlDisplayType, + displayType: Schema.optional(Schema.String), startTime: WarehouseDateTimeString, endTime: WarehouseDateTimeString, granularitySeconds: Schema.optional(Schema.Number), @@ -41,7 +43,7 @@ interface RawSqlChartResponse { rowCount: number columns: ReadonlyArray granularitySeconds: number - displayType: Schema.Schema.Type + displayType: RawSqlDisplayType | undefined } } @@ -51,6 +53,7 @@ export const getRawSqlChart = Effect.fn("QueryEngine.getRawSqlChart")(function* data: GetRawSqlChartInput }) { const input = yield* decodeInput(GetRawSqlChartInputSchema, data, "getRawSqlChart") + const displayType = Schema.is(RawSqlDisplayType)(input.displayType) ? input.displayType : undefined const result = yield* runWarehouseQuery("rawSqlChart", () => Effect.gen(function* () { @@ -58,7 +61,8 @@ export const getRawSqlChart = Effect.fn("QueryEngine.getRawSqlChart")(function* return yield* client.queryEngine.executeRawSql({ payload: new RawSqlExecuteRequest({ sql: input.sql, - displayType: input.displayType, + // The API does not read it; "table" names rows as returned. + displayType: displayType ?? "table", startTime: input.startTime, endTime: input.endTime, granularitySeconds: input.granularitySeconds, @@ -71,7 +75,7 @@ export const getRawSqlChart = Effect.fn("QueryEngine.getRawSqlChart")(function* // The same reshaping the share API applies server-side, so a raw-SQL line // chart draws identically on a board and on its share link. - const chartRows = rawSqlRowsForDisplay(rows, input.displayType) + const chartRows = rawSqlRowsForDisplay(rows, displayType) return { data: chartRows, @@ -79,7 +83,7 @@ export const getRawSqlChart = Effect.fn("QueryEngine.getRawSqlChart")(function* rowCount: result.meta.rowCount, columns: result.meta.columns, granularitySeconds: result.meta.granularitySeconds, - displayType: input.displayType, + displayType, }, } satisfies RawSqlChartResponse }) diff --git a/packages/web-data/src/warehouse/traces.test.ts b/packages/web-data/src/warehouse/traces.test.ts index 4c4b80ece4..fb9db7da0f 100644 --- a/packages/web-data/src/warehouse/traces.test.ts +++ b/packages/web-data/src/warehouse/traces.test.ts @@ -2,6 +2,7 @@ import { describe, it } from "@effect/vitest" import { Deferred, Effect, Array as Arr, String as Str } from "effect" import { beforeEach, expect, vi } from "vitest" +import { MAX_TRACES_LIST_OFFSET } from "@maple/domain/query-engine" import { mockedWarehouse } from "./testing" const executeQueryEngineMock = vi.fn() @@ -231,4 +232,21 @@ describe("tinybird traces attribute filter params", () => { expect(isWarehouseApiError(error) && error._tag).toBe("@maple/web/errors/WarehouseTransformError") }).pipe(mockedWarehouse), ) + + it.effect("refuses an offset past the list's cap as invalid input, without reading", () => + Effect.gen(function* () { + const error = yield* Effect.flip( + listTraces({ + data: { + startTime: "2026-02-01 00:00:00", + endTime: "2026-02-01 01:00:00", + offset: MAX_TRACES_LIST_OFFSET + 100, + }, + }), + ) + + expect(isWarehouseApiError(error) && error._tag).toBe("@maple/web/errors/WarehouseDecodeError") + expect(executeQueryEngineMock).not.toHaveBeenCalled() + }).pipe(mockedWarehouse), + ) }) diff --git a/packages/web-data/src/warehouse/traces.ts b/packages/web-data/src/warehouse/traces.ts index dc6181174d..89edb631f3 100644 --- a/packages/web-data/src/warehouse/traces.ts +++ b/packages/web-data/src/warehouse/traces.ts @@ -1,6 +1,7 @@ import { Clock, Effect, Option, Schema, Array as Arr, Record as Rec } from "effect" import { formatWarehouseDateTime, CH } from "@maple/query-engine" import { + MAX_TRACES_LIST_OFFSET, QueryEngineExecuteRequest, TracesFacetDimension, type AttributeFilter, @@ -63,7 +64,13 @@ const ListTracesInputSchema = Schema.Struct({ limit: Schema.optional( Schema.Int.check(Schema.isGreaterThanOrEqualTo(1), Schema.isLessThanOrEqualTo(1000)), ), - offset: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))), + // Bounded here so a deeper offset is a typed decode error, not a throw from the request constructor. + offset: Schema.optional( + Schema.Int.check( + Schema.isGreaterThanOrEqualTo(0), + Schema.isLessThanOrEqualTo(MAX_TRACES_LIST_OFFSET), + ), + ), sortBy: Schema.optional(Schema.Literals(["timestamp", "durationMs"])), sortDir: Schema.optional(Schema.Literals(["asc", "desc"])), startTime: Schema.optional(WarehouseDateTimeString), From bfc7a252ede54b6d7c749ae25cab01187f632e74 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 16 Nov 2026 17:22:23 +0100 Subject: [PATCH 2/2] fix(web): the traces footer says the list was cut short when the offset cap ends paging Review note: with noise traces hidden, paging can stop at the offset cap while few rows show, and the footer read like a complete list. isCapped now counts both caps, and hasNextPage reads it. --- .../src/page/traces/list/traces.scene.test.ts | 9 +++++++++ apps/web/src/page/traces/list/update.ts | 19 ++++++++++++++----- apps/web/src/page/traces/list/view-table.ts | 4 ++-- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/apps/web/src/page/traces/list/traces.scene.test.ts b/apps/web/src/page/traces/list/traces.scene.test.ts index f8f15536c2..7e3ca1fb59 100644 --- a/apps/web/src/page/traces/list/traces.scene.test.ts +++ b/apps/web/src/page/traces/list/traces.scene.test.ts @@ -235,6 +235,15 @@ describe("traces list", () => { expect(Arr.every(offsets, (offset) => offset <= MAX_TRACES_LIST_OFFSET)).toBe(true) expect(allRows(model)).toHaveLength(10 + offsets.length * 10) expect(nextOffsetOf(pageUpdate(model, windowOver(0, allRows(model).length)).commands)).toBeNull() + // The footer says the list was cut short, not that these are all the traces. + const shown = allRows(model).length + scene( + { update: pageUpdate, view }, + given(model), + Mount.resolve(VirtualizeTraces, windowOver(0, shown)), + Mount.resolve(MountPeekRows, Message.ClickedPeekRow({ index: 999 })), + expectView(text(`Showing first ${shown} traces, narrow the filters to see more`)).toExist(), + ) }) test("a next page answers only its own request: filter A, B, then A again appends page 2 once", () => { diff --git a/apps/web/src/page/traces/list/update.ts b/apps/web/src/page/traces/list/update.ts index 579bdd43d1..1e88aab343 100644 --- a/apps/web/src/page/traces/list/update.ts +++ b/apps/web/src/page/traces/list/update.ts @@ -71,11 +71,8 @@ export const hiddenCount = (model: Model): number => /** The warehouse rows the pages so far consumed: the next page starts there. */ const nextOffset = (pagination: Pagination): number => (pagination.pages.length + 1) * PAGE_SIZE -/** More pages exist when the last warehouse page came back full before the noise filter. */ -export const hasNextPage = (model: Model): boolean => { - if (allRows(model).length >= MAX_RETAINED_TRACES || model.pagination.isStopped) return false - // Hidden noise traces keep the shown rows under the cap while the offset passes the API's. - if (nextOffset(model.pagination) > MAX_TRACES_LIST_OFFSET) return false +/** The last warehouse page came back full before the noise filter, so more rows may follow. */ +const isLastPageFull = (model: Model): boolean => { const first = AsyncData.getData(model.firstPage.data) if (Option.isNone(first) || AsyncData.isFailure(model.firstPage.data)) return false const last = Arr.last(model.pagination.pages) @@ -87,6 +84,18 @@ export const hasNextPage = (model: Model): boolean => { ) } +/** + * The list hit a limit rather than its end: the rows it keeps, or the deepest offset the API reads + * (hidden noise traces keep the shown rows under the first while the offset passes the second). + */ +export const isCapped = (model: Model): boolean => + allRows(model).length >= MAX_RETAINED_TRACES || + (nextOffset(model.pagination) > MAX_TRACES_LIST_OFFSET && isLastPageFull(model)) + +/** More pages exist when the last page came back full and no cap stops the list. */ +export const hasNextPage = (model: Model): boolean => + !model.pagination.isStopped && !isCapped(model) && isLastPageFull(model) + const resolveCommand = (model: Model, context: PageContext, mode: RequestMode): Cmd => ResolveWindow({ mode, diff --git a/apps/web/src/page/traces/list/view-table.ts b/apps/web/src/page/traces/list/view-table.ts index 196e035f02..97703ecb80 100644 --- a/apps/web/src/page/traces/list/view-table.ts +++ b/apps/web/src/page/traces/list/view-table.ts @@ -24,7 +24,7 @@ import { presenceOf } from "@maple/views/data/signal-presence" import { isFiltered, type TraceFilterChip, traceFilterChips } from "./derive" import { Message, type SortKey } from "./message" import type { Model } from "./model" -import { allRows, DEFAULT_PRESET, hasNextPage, hiddenCount, MAX_RETAINED_TRACES } from "./update" +import { allRows, DEFAULT_PRESET, hasNextPage, hiddenCount, isCapped } from "./update" import { noiseNote, traceRow } from "./view-row" import { COUNT_ATTRIBUTE, VirtualizeTraces } from "./virtualize" import { Array as Arr } from "effect" @@ -185,7 +185,7 @@ const loadedTable = ( const tally = { shown: count, noun: "traces", - isCapped: count >= MAX_RETAINED_TRACES, + isCapped: isCapped(model), hasMore: hasNextPage(model), align: "start", className: "p-0",