diff --git a/apps/web/src/page/traces/list/traces.scene.test.ts b/apps/web/src/page/traces/list/traces.scene.test.ts index 9363c7cd5a..7e3ca1fb59 100644 --- a/apps/web/src/page/traces/list/traces.scene.test.ts +++ b/apps/web/src/page/traces/list/traces.scene.test.ts @@ -1,5 +1,6 @@ import { OrgId } from "@maple/domain" -import { Result, Schema, Array as Arr, String as Str } from "effect" +import { MAX_TRACES_LIST_OFFSET } from "@maple/domain/query-engine" +import { Option, Result, Schema, Array as Arr, String as Str } from "effect" import { click, Command, @@ -192,6 +193,59 @@ describe("traces list", () => { expect(retried.model.pagination).toMatchObject({ isFailed: false, isFetching: true }) }) + test("paging stops at the API's deepest offset even when hidden noise keeps the shown rows few", () => { + // Ten traces shown from each hundred read: the offset reaches the cap long before the rows do. + const sparse = (pageIndex: number): TracesPage => ({ + rows: Array.from({ length: 10 }, (_, i) => trace(`p${pageIndex}-${i}`)), + scannedCount: 100, + hiddenCount: 90, + }) + type Asked = ReturnType["commands"] + const nextOffsetOf = (commands: Asked): number | null => { + const [command] = commands ?? [] + return command?.name === FetchNextPage.name && + command.args !== undefined && + "offset" in command.args + ? Number(command.args.offset) + : null + } + type State = { model: Model; asked: Asked; offsets: ReadonlyArray } + // An appended page that leaves the view near the end asks for the next one itself. + const step = (state: State, pageIndex: number): State => { + const near = + nextOffsetOf(state.asked) === null + ? pageUpdate(state.model, windowOver(0, allRows(state.model).length)) + : { model: state.model, commands: state.asked } + const offset = nextOffsetOf(near.commands) + if (offset === null) return { ...state, model: near.model, asked: [] } + const answered = pageUpdate( + near.model, + Message.CompletedFetchNextPage({ + key: near.model.pagination.key ?? "", + generation: near.model.pagination.generation, + offset, + result: Result.succeed(sparse(pageIndex)), + }), + ) + return { model: answered.model, asked: answered.commands, offsets: [...state.offsets, offset] } + } + const initial: State = { model: loaded(sparse(0)), asked: [], offsets: [] } + const { model, offsets } = Arr.reduce(Arr.range(1, 25), initial, step) + expect(Arr.last(offsets)).toEqual(Option.some(MAX_TRACES_LIST_OFFSET)) + expect(Arr.every(offsets, (offset) => offset <= MAX_TRACES_LIST_OFFSET)).toBe(true) + expect(allRows(model)).toHaveLength(10 + offsets.length * 10) + expect(nextOffsetOf(pageUpdate(model, windowOver(0, allRows(model).length)).commands)).toBeNull() + // The footer says the list was cut short, not that these are all the traces. + const shown = allRows(model).length + scene( + { update: pageUpdate, view }, + given(model), + Mount.resolve(VirtualizeTraces, windowOver(0, shown)), + Mount.resolve(MountPeekRows, Message.ClickedPeekRow({ index: 999 })), + expectView(text(`Showing first ${shown} traces, narrow the filters to see more`)).toExist(), + ) + }) + test("a next page answers only its own request: filter A, B, then A again appends page 2 once", () => { const resolved = (model: Model) => pageUpdate( diff --git a/apps/web/src/page/traces/list/update.ts b/apps/web/src/page/traces/list/update.ts index ad856f58b4..1e88aab343 100644 --- a/apps/web/src/page/traces/list/update.ts +++ b/apps/web/src/page/traces/list/update.ts @@ -1,5 +1,6 @@ import { pickerStep } from "../../kit/page-time" import type { RequestMode } from "@maple/views/data/keyed" +import { MAX_TRACES_LIST_OFFSET } from "@maple/domain/query-engine" import { Array as Arr, Option, Result } from "effect" import { AsyncData, Command } from "foldkit" import * as Keyed from "@maple/views/data/keyed" @@ -67,9 +68,11 @@ export const hiddenCount = (model: Model): number => first.hiddenCount + Arr.reduce(model.pagination.pages, 0, (sum, page) => sum + page.hiddenCount), }) -/** More pages exist when the last warehouse page came back full before the noise filter. */ -export const hasNextPage = (model: Model): boolean => { - if (allRows(model).length >= MAX_RETAINED_TRACES || model.pagination.isStopped) return false +/** The warehouse rows the pages so far consumed: the next page starts there. */ +const nextOffset = (pagination: Pagination): number => (pagination.pages.length + 1) * PAGE_SIZE + +/** The last warehouse page came back full before the noise filter, so more rows may follow. */ +const isLastPageFull = (model: Model): boolean => { const first = AsyncData.getData(model.firstPage.data) if (Option.isNone(first) || AsyncData.isFailure(model.firstPage.data)) return false const last = Arr.last(model.pagination.pages) @@ -81,6 +84,18 @@ export const hasNextPage = (model: Model): boolean => { ) } +/** + * The list hit a limit rather than its end: the rows it keeps, or the deepest offset the API reads + * (hidden noise traces keep the shown rows under the first while the offset passes the second). + */ +export const isCapped = (model: Model): boolean => + allRows(model).length >= MAX_RETAINED_TRACES || + (nextOffset(model.pagination) > MAX_TRACES_LIST_OFFSET && isLastPageFull(model)) + +/** More pages exist when the last page came back full and no cap stops the list. */ +export const hasNextPage = (model: Model): boolean => + !model.pagination.isStopped && !isCapped(model) && isLastPageFull(model) + const resolveCommand = (model: Model, context: PageContext, mode: RequestMode): Cmd => ResolveWindow({ mode, @@ -145,8 +160,6 @@ const requestAll = (model: Model, context: PageContext, window: TimeWindow, mode } } -const nextOffset = (pagination: Pagination): number => (pagination.pages.length + 1) * PAGE_SIZE - /** Asks for the next page, one at a time; also the retry of a page that failed. */ const fetchNext = (model: Model, context: PageContext): Return => { const { window, pagination } = model diff --git a/apps/web/src/page/traces/list/view-table.ts b/apps/web/src/page/traces/list/view-table.ts index 196e035f02..97703ecb80 100644 --- a/apps/web/src/page/traces/list/view-table.ts +++ b/apps/web/src/page/traces/list/view-table.ts @@ -24,7 +24,7 @@ import { presenceOf } from "@maple/views/data/signal-presence" import { isFiltered, type TraceFilterChip, traceFilterChips } from "./derive" import { Message, type SortKey } from "./message" import type { Model } from "./model" -import { allRows, DEFAULT_PRESET, hasNextPage, hiddenCount, MAX_RETAINED_TRACES } from "./update" +import { allRows, DEFAULT_PRESET, hasNextPage, hiddenCount, isCapped } from "./update" import { noiseNote, traceRow } from "./view-row" import { COUNT_ATTRIBUTE, VirtualizeTraces } from "./virtualize" import { Array as Arr } from "effect" @@ -185,7 +185,7 @@ const loadedTable = ( const tally = { shown: count, noun: "traces", - isCapped: count >= MAX_RETAINED_TRACES, + isCapped: isCapped(model), hasMore: hasNextPage(model), align: "start", className: "p-0", diff --git a/packages/domain/src/query-engine.ts b/packages/domain/src/query-engine.ts index a8b5da1eeb..7c184cb95b 100644 --- a/packages/domain/src/query-engine.ts +++ b/packages/domain/src/query-engine.ts @@ -421,6 +421,13 @@ export const MetricsSparklinesQuery = Schema.Struct({ }) export type MetricsSparklinesQuery = Schema.Schema.Type +/** + * The deepest offset the trace list reads. The client pages by rows consumed (noise + * traces are dropped after the read), so it must stop on this, not on rows shown. + * Stage 1 only reads the three light sort columns, so a deep offset stays cheap. + */ +export const MAX_TRACES_LIST_OFFSET = 2_000 + export const TracesListQuery = Schema.Struct({ kind: Schema.Literal("list"), source: Schema.Literal("traces"), @@ -436,15 +443,11 @@ export const TracesListQuery = Schema.Struct({ limit: Schema.optional( Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(200)), ), - // Bound matches the web list's MAX_RETAINED_TRACES (2000): with server-side - // noise filtering the client pages by rows CONSUMED, so a 1000 cap would end - // pagination while far fewer rows are visible. Stage 1 only reads the three - // light sort columns, so a deep offset stays cheap. offset: Schema.optional( Schema.Number.check( Schema.isInt(), Schema.isGreaterThanOrEqualTo(0), - Schema.isLessThanOrEqualTo(2000), + Schema.isLessThanOrEqualTo(MAX_TRACES_LIST_OFFSET), ), ), cursor: Schema.optional(Schema.String), diff --git a/packages/web-data/src/warehouse/raw-sql-chart.test.ts b/packages/web-data/src/warehouse/raw-sql-chart.test.ts new file mode 100644 index 0000000000..07edd63069 --- /dev/null +++ b/packages/web-data/src/warehouse/raw-sql-chart.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "@effect/vitest" +import { MapleInternalApi } from "@maple/domain/http" +import { Effect, Layer } from "effect" +import { FetchHttpClient } from "effect/http" +import { HttpApiClient } from "effect/http-api" +import { getRawSqlChart } from "./raw-sql-chart" +import { MapleInternalHttp, WarehouseExecutor } from "./warehouse-services" + +const rows = [ + { bucket: "2026-11-16 10:00:00", requests: 3 }, + { bucket: "2026-11-16 10:01:00", requests: 5 }, +] + +const answerRawSql = (): Promise => + Promise.resolve( + Response.json({ + data: rows, + meta: { rowCount: rows.length, columns: ["bucket", "requests"], granularitySeconds: 60 }, + }), + ) + +const ApiLayer = Layer.mergeAll( + Layer.mock(WarehouseExecutor, {}), + Layer.effect( + MapleInternalHttp, + HttpApiClient.make(MapleInternalApi, { baseUrl: "https://api.test" }), + ).pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(Layer.succeed(FetchHttpClient.Fetch, answerRawSql)), + ), +) + +const withApi = (effect: Effect.Effect) => + effect.pipe(Effect.provide(ApiLayer)) + +const window = { startTime: "2026-11-16 10:00:00", endTime: "2026-11-16 11:00:00" } + +describe("getRawSqlChart", () => { + it.effect("runs a widget saved without a display type and returns its rows as the API sent them", () => + withApi( + Effect.gen(function* () { + const result = yield* getRawSqlChart({ data: { sql: "SELECT 1", ...window } }) + expect(result.data).toEqual(rows) + expect(result.meta.displayType).toBeUndefined() + }), + ), + ) + + it.effect("treats a display type this build does not know like a missing one", () => + withApi( + Effect.gen(function* () { + const result = yield* getRawSqlChart({ + data: { sql: "SELECT 1", displayType: "sankey", ...window }, + }) + expect(result.data).toEqual(rows) + expect(result.meta.displayType).toBeUndefined() + }), + ), + ) + + it.effect("still reshapes a line chart's rows into series", () => + withApi( + Effect.gen(function* () { + const result = yield* getRawSqlChart({ + data: { sql: "SELECT 1", displayType: "line", ...window }, + }) + expect(result.meta.displayType).toBe("line") + expect(result.data).toHaveLength(2) + expect(result.data[0]).toMatchObject({ bucket: expect.any(String), requests: 3 }) + }), + ), + ) +}) diff --git a/packages/web-data/src/warehouse/raw-sql-chart.ts b/packages/web-data/src/warehouse/raw-sql-chart.ts index 504aa4976e..798ee754df 100644 --- a/packages/web-data/src/warehouse/raw-sql-chart.ts +++ b/packages/web-data/src/warehouse/raw-sql-chart.ts @@ -7,7 +7,7 @@ import { rawSqlRowsForDisplay } from "@maple/query-engine" // Raw SQL chart server function (widget data source `raw_sql_chart`). // // Widget params shape: -// { sql, displayType, granularitySeconds?, startTime, endTime, ... } +// { sql, displayType?, granularitySeconds?, startTime, endTime, ... } // displayType ∈ "line" | "area" | "bar" | "table" | "stat" | "pie" | "histogram" | "heatmap" // // Returns rows in a renderer-friendly shape: @@ -25,9 +25,11 @@ import { rawSqlRowsForDisplay } from "@maple/query-engine" // - heatmap → raw rows; chart accepts `{ x, y, value }` or wide // `{ name, …numeric }` formats. +// Optional and open, like the stored source (`RawSqlWidgetDataSource`) and the share API: a widget +// saved without one, or with a type this build does not know, gets its rows as returned. const GetRawSqlChartInputSchema = Schema.Struct({ sql: Schema.String, - displayType: RawSqlDisplayType, + displayType: Schema.optional(Schema.String), startTime: WarehouseDateTimeString, endTime: WarehouseDateTimeString, granularitySeconds: Schema.optional(Schema.Number), @@ -41,7 +43,7 @@ interface RawSqlChartResponse { rowCount: number columns: ReadonlyArray granularitySeconds: number - displayType: Schema.Schema.Type + displayType: RawSqlDisplayType | undefined } } @@ -51,6 +53,7 @@ export const getRawSqlChart = Effect.fn("QueryEngine.getRawSqlChart")(function* data: GetRawSqlChartInput }) { const input = yield* decodeInput(GetRawSqlChartInputSchema, data, "getRawSqlChart") + const displayType = Schema.is(RawSqlDisplayType)(input.displayType) ? input.displayType : undefined const result = yield* runWarehouseQuery("rawSqlChart", () => Effect.gen(function* () { @@ -58,7 +61,8 @@ export const getRawSqlChart = Effect.fn("QueryEngine.getRawSqlChart")(function* return yield* client.queryEngine.executeRawSql({ payload: new RawSqlExecuteRequest({ sql: input.sql, - displayType: input.displayType, + // The API does not read it; "table" names rows as returned. + displayType: displayType ?? "table", startTime: input.startTime, endTime: input.endTime, granularitySeconds: input.granularitySeconds, @@ -71,7 +75,7 @@ export const getRawSqlChart = Effect.fn("QueryEngine.getRawSqlChart")(function* // The same reshaping the share API applies server-side, so a raw-SQL line // chart draws identically on a board and on its share link. - const chartRows = rawSqlRowsForDisplay(rows, input.displayType) + const chartRows = rawSqlRowsForDisplay(rows, displayType) return { data: chartRows, @@ -79,7 +83,7 @@ export const getRawSqlChart = Effect.fn("QueryEngine.getRawSqlChart")(function* rowCount: result.meta.rowCount, columns: result.meta.columns, granularitySeconds: result.meta.granularitySeconds, - displayType: input.displayType, + displayType, }, } satisfies RawSqlChartResponse }) diff --git a/packages/web-data/src/warehouse/traces.test.ts b/packages/web-data/src/warehouse/traces.test.ts index 4c4b80ece4..fb9db7da0f 100644 --- a/packages/web-data/src/warehouse/traces.test.ts +++ b/packages/web-data/src/warehouse/traces.test.ts @@ -2,6 +2,7 @@ import { describe, it } from "@effect/vitest" import { Deferred, Effect, Array as Arr, String as Str } from "effect" import { beforeEach, expect, vi } from "vitest" +import { MAX_TRACES_LIST_OFFSET } from "@maple/domain/query-engine" import { mockedWarehouse } from "./testing" const executeQueryEngineMock = vi.fn() @@ -231,4 +232,21 @@ describe("tinybird traces attribute filter params", () => { expect(isWarehouseApiError(error) && error._tag).toBe("@maple/web/errors/WarehouseTransformError") }).pipe(mockedWarehouse), ) + + it.effect("refuses an offset past the list's cap as invalid input, without reading", () => + Effect.gen(function* () { + const error = yield* Effect.flip( + listTraces({ + data: { + startTime: "2026-02-01 00:00:00", + endTime: "2026-02-01 01:00:00", + offset: MAX_TRACES_LIST_OFFSET + 100, + }, + }), + ) + + expect(isWarehouseApiError(error) && error._tag).toBe("@maple/web/errors/WarehouseDecodeError") + expect(executeQueryEngineMock).not.toHaveBeenCalled() + }).pipe(mockedWarehouse), + ) }) diff --git a/packages/web-data/src/warehouse/traces.ts b/packages/web-data/src/warehouse/traces.ts index dc6181174d..89edb631f3 100644 --- a/packages/web-data/src/warehouse/traces.ts +++ b/packages/web-data/src/warehouse/traces.ts @@ -1,6 +1,7 @@ import { Clock, Effect, Option, Schema, Array as Arr, Record as Rec } from "effect" import { formatWarehouseDateTime, CH } from "@maple/query-engine" import { + MAX_TRACES_LIST_OFFSET, QueryEngineExecuteRequest, TracesFacetDimension, type AttributeFilter, @@ -63,7 +64,13 @@ const ListTracesInputSchema = Schema.Struct({ limit: Schema.optional( Schema.Int.check(Schema.isGreaterThanOrEqualTo(1), Schema.isLessThanOrEqualTo(1000)), ), - offset: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))), + // Bounded here so a deeper offset is a typed decode error, not a throw from the request constructor. + offset: Schema.optional( + Schema.Int.check( + Schema.isGreaterThanOrEqualTo(0), + Schema.isLessThanOrEqualTo(MAX_TRACES_LIST_OFFSET), + ), + ), sortBy: Schema.optional(Schema.Literals(["timestamp", "durationMs"])), sortDir: Schema.optional(Schema.Literals(["asc", "desc"])), startTime: Schema.optional(WarehouseDateTimeString),