Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 55 additions & 1 deletion apps/web/src/page/traces/list/traces.scene.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { OrgId } from "@maple/domain"
import { Result, Schema, Array as Arr, String as Str } from "effect"
import { MAX_TRACES_LIST_OFFSET } from "@maple/domain/query-engine"
import { Option, Result, Schema, Array as Arr, String as Str } from "effect"
import {
click,
Command,
Expand Down Expand Up @@ -192,6 +193,59 @@ describe("traces list", () => {
expect(retried.model.pagination).toMatchObject({ isFailed: false, isFetching: true })
})

test("paging stops at the API's deepest offset even when hidden noise keeps the shown rows few", () => {
// Ten traces shown from each hundred read: the offset reaches the cap long before the rows do.
const sparse = (pageIndex: number): TracesPage => ({
rows: Array.from({ length: 10 }, (_, i) => trace(`p${pageIndex}-${i}`)),
scannedCount: 100,
hiddenCount: 90,
})
type Asked = ReturnType<typeof pageUpdate>["commands"]
const nextOffsetOf = (commands: Asked): number | null => {
const [command] = commands ?? []
return command?.name === FetchNextPage.name &&
command.args !== undefined &&
"offset" in command.args
? Number(command.args.offset)
: null
}
type State = { model: Model; asked: Asked; offsets: ReadonlyArray<number> }
// An appended page that leaves the view near the end asks for the next one itself.
const step = (state: State, pageIndex: number): State => {
const near =
nextOffsetOf(state.asked) === null
? pageUpdate(state.model, windowOver(0, allRows(state.model).length))
: { model: state.model, commands: state.asked }
const offset = nextOffsetOf(near.commands)
if (offset === null) return { ...state, model: near.model, asked: [] }
const answered = pageUpdate(
near.model,
Message.CompletedFetchNextPage({
key: near.model.pagination.key ?? "",
generation: near.model.pagination.generation,
offset,
result: Result.succeed(sparse(pageIndex)),
}),
)
return { model: answered.model, asked: answered.commands, offsets: [...state.offsets, offset] }
}
const initial: State = { model: loaded(sparse(0)), asked: [], offsets: [] }
const { model, offsets } = Arr.reduce(Arr.range(1, 25), initial, step)
expect(Arr.last(offsets)).toEqual(Option.some(MAX_TRACES_LIST_OFFSET))
expect(Arr.every(offsets, (offset) => offset <= MAX_TRACES_LIST_OFFSET)).toBe(true)
expect(allRows(model)).toHaveLength(10 + offsets.length * 10)
expect(nextOffsetOf(pageUpdate(model, windowOver(0, allRows(model).length)).commands)).toBeNull()
// The footer says the list was cut short, not that these are all the traces.
const shown = allRows(model).length
scene(
{ update: pageUpdate, view },
given(model),
Mount.resolve(VirtualizeTraces, windowOver(0, shown)),
Mount.resolve(MountPeekRows, Message.ClickedPeekRow({ index: 999 })),
expectView(text(`Showing first ${shown} traces, narrow the filters to see more`)).toExist(),
)
})

test("a next page answers only its own request: filter A, B, then A again appends page 2 once", () => {
const resolved = (model: Model) =>
pageUpdate(
Expand Down
23 changes: 18 additions & 5 deletions apps/web/src/page/traces/list/update.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { pickerStep } from "../../kit/page-time"
import type { RequestMode } from "@maple/views/data/keyed"
import { MAX_TRACES_LIST_OFFSET } from "@maple/domain/query-engine"
import { Array as Arr, Option, Result } from "effect"
import { AsyncData, Command } from "foldkit"
import * as Keyed from "@maple/views/data/keyed"
Expand Down Expand Up @@ -67,9 +68,11 @@ export const hiddenCount = (model: Model): number =>
first.hiddenCount + Arr.reduce(model.pagination.pages, 0, (sum, page) => sum + page.hiddenCount),
})

/** More pages exist when the last warehouse page came back full before the noise filter. */
export const hasNextPage = (model: Model): boolean => {
if (allRows(model).length >= MAX_RETAINED_TRACES || model.pagination.isStopped) return false
/** The warehouse rows the pages so far consumed: the next page starts there. */
const nextOffset = (pagination: Pagination): number => (pagination.pages.length + 1) * PAGE_SIZE

/** The last warehouse page came back full before the noise filter, so more rows may follow. */
const isLastPageFull = (model: Model): boolean => {
const first = AsyncData.getData(model.firstPage.data)
if (Option.isNone(first) || AsyncData.isFailure(model.firstPage.data)) return false
const last = Arr.last(model.pagination.pages)
Expand All @@ -81,6 +84,18 @@ export const hasNextPage = (model: Model): boolean => {
)
}

/**
* The list hit a limit rather than its end: the rows it keeps, or the deepest offset the API reads
* (hidden noise traces keep the shown rows under the first while the offset passes the second).
*/
export const isCapped = (model: Model): boolean =>
allRows(model).length >= MAX_RETAINED_TRACES ||
(nextOffset(model.pagination) > MAX_TRACES_LIST_OFFSET && isLastPageFull(model))

/** More pages exist when the last page came back full and no cap stops the list. */
export const hasNextPage = (model: Model): boolean =>
!model.pagination.isStopped && !isCapped(model) && isLastPageFull(model)

const resolveCommand = (model: Model, context: PageContext, mode: RequestMode): Cmd =>
ResolveWindow({
mode,
Expand Down Expand Up @@ -145,8 +160,6 @@ const requestAll = (model: Model, context: PageContext, window: TimeWindow, mode
}
}

const nextOffset = (pagination: Pagination): number => (pagination.pages.length + 1) * PAGE_SIZE

/** Asks for the next page, one at a time; also the retry of a page that failed. */
const fetchNext = (model: Model, context: PageContext): Return => {
const { window, pagination } = model
Expand Down
4 changes: 2 additions & 2 deletions apps/web/src/page/traces/list/view-table.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import { presenceOf } from "@maple/views/data/signal-presence"
import { isFiltered, type TraceFilterChip, traceFilterChips } from "./derive"
import { Message, type SortKey } from "./message"
import type { Model } from "./model"
import { allRows, DEFAULT_PRESET, hasNextPage, hiddenCount, MAX_RETAINED_TRACES } from "./update"
import { allRows, DEFAULT_PRESET, hasNextPage, hiddenCount, isCapped } from "./update"
import { noiseNote, traceRow } from "./view-row"
import { COUNT_ATTRIBUTE, VirtualizeTraces } from "./virtualize"
import { Array as Arr } from "effect"
Expand Down Expand Up @@ -185,7 +185,7 @@ const loadedTable = (
const tally = {
shown: count,
noun: "traces",
isCapped: count >= MAX_RETAINED_TRACES,
isCapped: isCapped(model),
hasMore: hasNextPage(model),
align: "start",
className: "p-0",
Expand Down
13 changes: 8 additions & 5 deletions packages/domain/src/query-engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,13 @@ export const MetricsSparklinesQuery = Schema.Struct({
})
export type MetricsSparklinesQuery = Schema.Schema.Type<typeof MetricsSparklinesQuery>

/**
* The deepest offset the trace list reads. The client pages by rows consumed (noise
* traces are dropped after the read), so it must stop on this, not on rows shown.
* Stage 1 only reads the three light sort columns, so a deep offset stays cheap.
*/
export const MAX_TRACES_LIST_OFFSET = 2_000

export const TracesListQuery = Schema.Struct({
kind: Schema.Literal("list"),
source: Schema.Literal("traces"),
Expand All @@ -436,15 +443,11 @@ export const TracesListQuery = Schema.Struct({
limit: Schema.optional(
Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(200)),
),
// Bound matches the web list's MAX_RETAINED_TRACES (2000): with server-side
// noise filtering the client pages by rows CONSUMED, so a 1000 cap would end
// pagination while far fewer rows are visible. Stage 1 only reads the three
// light sort columns, so a deep offset stays cheap.
offset: Schema.optional(
Schema.Number.check(
Schema.isInt(),
Schema.isGreaterThanOrEqualTo(0),
Schema.isLessThanOrEqualTo(2000),
Schema.isLessThanOrEqualTo(MAX_TRACES_LIST_OFFSET),
),
),
cursor: Schema.optional(Schema.String),
Expand Down
73 changes: 73 additions & 0 deletions packages/web-data/src/warehouse/raw-sql-chart.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import { describe, expect, it } from "@effect/vitest"
import { MapleInternalApi } from "@maple/domain/http"
import { Effect, Layer } from "effect"
import { FetchHttpClient } from "effect/http"
import { HttpApiClient } from "effect/http-api"
import { getRawSqlChart } from "./raw-sql-chart"
import { MapleInternalHttp, WarehouseExecutor } from "./warehouse-services"

const rows = [
{ bucket: "2026-11-16 10:00:00", requests: 3 },
{ bucket: "2026-11-16 10:01:00", requests: 5 },
]

const answerRawSql = (): Promise<Response> =>
Promise.resolve(
Response.json({
data: rows,
meta: { rowCount: rows.length, columns: ["bucket", "requests"], granularitySeconds: 60 },
}),
)

const ApiLayer = Layer.mergeAll(
Layer.mock(WarehouseExecutor, {}),
Layer.effect(
MapleInternalHttp,
HttpApiClient.make(MapleInternalApi, { baseUrl: "https://api.test" }),
).pipe(
Layer.provide(FetchHttpClient.layer),
Layer.provide(Layer.succeed(FetchHttpClient.Fetch, answerRawSql)),
),
)

const withApi = <A, E>(effect: Effect.Effect<A, E, MapleInternalHttp | WarehouseExecutor>) =>
effect.pipe(Effect.provide(ApiLayer))

const window = { startTime: "2026-11-16 10:00:00", endTime: "2026-11-16 11:00:00" }

describe("getRawSqlChart", () => {
it.effect("runs a widget saved without a display type and returns its rows as the API sent them", () =>
withApi(
Effect.gen(function* () {
const result = yield* getRawSqlChart({ data: { sql: "SELECT 1", ...window } })
expect(result.data).toEqual(rows)
expect(result.meta.displayType).toBeUndefined()
}),
),
)

it.effect("treats a display type this build does not know like a missing one", () =>
withApi(
Effect.gen(function* () {
const result = yield* getRawSqlChart({
data: { sql: "SELECT 1", displayType: "sankey", ...window },
})
expect(result.data).toEqual(rows)
expect(result.meta.displayType).toBeUndefined()
}),
),
)

it.effect("still reshapes a line chart's rows into series", () =>
withApi(
Effect.gen(function* () {
const result = yield* getRawSqlChart({
data: { sql: "SELECT 1", displayType: "line", ...window },
})
expect(result.meta.displayType).toBe("line")
expect(result.data).toHaveLength(2)
expect(result.data[0]).toMatchObject({ bucket: expect.any(String), requests: 3 })
}),
),
)
})
16 changes: 10 additions & 6 deletions packages/web-data/src/warehouse/raw-sql-chart.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { rawSqlRowsForDisplay } from "@maple/query-engine"
// Raw SQL chart server function (widget data source `raw_sql_chart`).
//
// Widget params shape:
// { sql, displayType, granularitySeconds?, startTime, endTime, ... }
// { sql, displayType?, granularitySeconds?, startTime, endTime, ... }
// displayType ∈ "line" | "area" | "bar" | "table" | "stat" | "pie" | "histogram" | "heatmap"
//
// Returns rows in a renderer-friendly shape:
Expand All @@ -25,9 +25,11 @@ import { rawSqlRowsForDisplay } from "@maple/query-engine"
// - heatmap → raw rows; chart accepts `{ x, y, value }` or wide
// `{ name, …numeric }` formats.

// Optional and open, like the stored source (`RawSqlWidgetDataSource`) and the share API: a widget
// saved without one, or with a type this build does not know, gets its rows as returned.
const GetRawSqlChartInputSchema = Schema.Struct({
sql: Schema.String,
displayType: RawSqlDisplayType,
displayType: Schema.optional(Schema.String),
startTime: WarehouseDateTimeString,
endTime: WarehouseDateTimeString,
granularitySeconds: Schema.optional(Schema.Number),
Expand All @@ -41,7 +43,7 @@ interface RawSqlChartResponse {
rowCount: number
columns: ReadonlyArray<string>
granularitySeconds: number
displayType: Schema.Schema.Type<typeof RawSqlDisplayType>
displayType: RawSqlDisplayType | undefined
}
}

Expand All @@ -51,14 +53,16 @@ export const getRawSqlChart = Effect.fn("QueryEngine.getRawSqlChart")(function*
data: GetRawSqlChartInput
}) {
const input = yield* decodeInput(GetRawSqlChartInputSchema, data, "getRawSqlChart")
const displayType = Schema.is(RawSqlDisplayType)(input.displayType) ? input.displayType : undefined

const result = yield* runWarehouseQuery("rawSqlChart", () =>
Effect.gen(function* () {
const client = yield* MapleInternalHttp
return yield* client.queryEngine.executeRawSql({
payload: new RawSqlExecuteRequest({
sql: input.sql,
displayType: input.displayType,
// The API does not read it; "table" names rows as returned.
displayType: displayType ?? "table",
startTime: input.startTime,
endTime: input.endTime,
granularitySeconds: input.granularitySeconds,
Expand All @@ -71,15 +75,15 @@ export const getRawSqlChart = Effect.fn("QueryEngine.getRawSqlChart")(function*

// The same reshaping the share API applies server-side, so a raw-SQL line
// chart draws identically on a board and on its share link.
const chartRows = rawSqlRowsForDisplay(rows, input.displayType)
const chartRows = rawSqlRowsForDisplay(rows, displayType)

return {
data: chartRows,
meta: {
rowCount: result.meta.rowCount,
columns: result.meta.columns,
granularitySeconds: result.meta.granularitySeconds,
displayType: input.displayType,
displayType,
},
} satisfies RawSqlChartResponse
})
18 changes: 18 additions & 0 deletions packages/web-data/src/warehouse/traces.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import { describe, it } from "@effect/vitest"
import { Deferred, Effect, Array as Arr, String as Str } from "effect"
import { beforeEach, expect, vi } from "vitest"
import { MAX_TRACES_LIST_OFFSET } from "@maple/domain/query-engine"
import { mockedWarehouse } from "./testing"

const executeQueryEngineMock = vi.fn()
Expand Down Expand Up @@ -231,4 +232,21 @@ describe("tinybird traces attribute filter params", () => {
expect(isWarehouseApiError(error) && error._tag).toBe("@maple/web/errors/WarehouseTransformError")
}).pipe(mockedWarehouse),
)

it.effect("refuses an offset past the list's cap as invalid input, without reading", () =>
Effect.gen(function* () {
const error = yield* Effect.flip(
listTraces({
data: {
startTime: "2026-02-01 00:00:00",
endTime: "2026-02-01 01:00:00",
offset: MAX_TRACES_LIST_OFFSET + 100,
},
}),
)

expect(isWarehouseApiError(error) && error._tag).toBe("@maple/web/errors/WarehouseDecodeError")
expect(executeQueryEngineMock).not.toHaveBeenCalled()
}).pipe(mockedWarehouse),
)
})
9 changes: 8 additions & 1 deletion packages/web-data/src/warehouse/traces.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { Clock, Effect, Option, Schema, Array as Arr, Record as Rec } from "effect"
import { formatWarehouseDateTime, CH } from "@maple/query-engine"
import {
MAX_TRACES_LIST_OFFSET,
QueryEngineExecuteRequest,
TracesFacetDimension,
type AttributeFilter,
Expand Down Expand Up @@ -63,7 +64,13 @@ const ListTracesInputSchema = Schema.Struct({
limit: Schema.optional(
Schema.Int.check(Schema.isGreaterThanOrEqualTo(1), Schema.isLessThanOrEqualTo(1000)),
),
offset: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))),
// Bounded here so a deeper offset is a typed decode error, not a throw from the request constructor.
offset: Schema.optional(
Schema.Int.check(
Schema.isGreaterThanOrEqualTo(0),
Schema.isLessThanOrEqualTo(MAX_TRACES_LIST_OFFSET),
),
),
sortBy: Schema.optional(Schema.Literals(["timestamp", "durationMs"])),
sortDir: Schema.optional(Schema.Literals(["asc", "desc"])),
startTime: Schema.optional(WarehouseDateTimeString),
Expand Down
Loading