diff --git a/contributing/DATABASE.md b/contributing/DATABASE.md index f70dffa08..853fa8ab6 100644 --- a/contributing/DATABASE.md +++ b/contributing/DATABASE.md @@ -84,3 +84,29 @@ Keep locked regions short and bounded. See `LOCKING.md` for how to take advisory `DSTACK_DB_COMMAND_TIMEOUT` (default 300 seconds) bounds every operation on a Postgres connection, so a coroutine waiting on a connection whose socket died silently gets an error and releases the slot instead of holding it forever. It applies to migrations too, since they run on the same engine, so it must leave room for index builds and backfills. `DSTACK_DB_POOL_SIZE` and `DSTACK_DB_MAX_OVERFLOW` size the pool per process. Raising them lets more coroutines hold connections at once, which hides a long hold rather than fixing it, and each replica needs `pool_size + max_overflow` connections from the Postgres `max_connections` budget. + +## Queries + +### Cursor pagination + +List APIs paginate with a cursor: the timestamp and id of the last returned row, e.g. `prev_created_at` and `prev_id`. Rows with the same timestamp are ordered by id in the opposite direction, so the cursor filter is an OR: + +```python +or_( + Model.created_at > prev_created_at, + and_(Model.created_at == prev_created_at, Model.id < prev_id), +) +``` + +Always add the timestamp bound next to it, `>=` for ascending and `<=` for descending: + +```python +filters.append(Model.created_at >= prev_created_at) +filters.append(or_(...)) +``` + +The bound doesn't change the results, but without it a request can read the whole table. Postgres can use the OR with an index only by looking up each branch separately and sorting the combined result. When it expects many rows after the cursor, it reads the timestamp index in order instead, and it can start that scan only from a single condition such as `created_at >= value`, which it doesn't derive from the OR. Then the scan starts at the oldest row and filters out everything before the cursor. Postgres expects many rows after the cursor when the cursor is old, and always with a generic plan of a prepared statement, which is planned without the cursor value. See [#4465](https://github.com/dstackai/dstack/issues/4465). + +A row comparison such as `(created_at, id) > (prev_created_at, prev_id)` would avoid the OR, but it requires both columns to be ordered in the same direction. + +The bound only removes reading rows before the cursor. A request still reads rows after the cursor until it finds `limit` rows that match its other filters, so on a large table a filter that matches few rows, such as one project's rows, also needs an index that starts with the filter column. diff --git a/src/dstack/_internal/server/services/events.py b/src/dstack/_internal/server/services/events.py index 61cbf5bb5..8ad07ad28 100644 --- a/src/dstack/_internal/server/services/events.py +++ b/src/dstack/_internal/server/services/events.py @@ -418,6 +418,9 @@ async def list_events( if prev_id is None: event_filters.append(EventModel.recorded_at > prev_recorded_at) else: + # Redundant with the OR below, but unlike the OR it lets the database + # start the index scan at the cursor instead of scanning from the start. + event_filters.append(EventModel.recorded_at >= prev_recorded_at) event_filters.append( or_( EventModel.recorded_at > prev_recorded_at, @@ -428,6 +431,7 @@ async def list_events( if prev_id is None: event_filters.append(EventModel.recorded_at < prev_recorded_at) else: + event_filters.append(EventModel.recorded_at <= prev_recorded_at) event_filters.append( or_( EventModel.recorded_at < prev_recorded_at, diff --git a/src/dstack/_internal/server/services/fleets.py b/src/dstack/_internal/server/services/fleets.py index 7be4bded4..c959adc33 100644 --- a/src/dstack/_internal/server/services/fleets.py +++ b/src/dstack/_internal/server/services/fleets.py @@ -277,6 +277,9 @@ async def list_projects_fleet_models( if prev_id is None: filters.append(FleetModel.created_at > prev_created_at) else: + # Redundant with the OR below, but unlike the OR it lets the database + # start the index scan at the cursor instead of scanning from the start. + filters.append(FleetModel.created_at >= prev_created_at) filters.append( or_( FleetModel.created_at > prev_created_at, @@ -287,6 +290,7 @@ async def list_projects_fleet_models( if prev_id is None: filters.append(FleetModel.created_at < prev_created_at) else: + filters.append(FleetModel.created_at <= prev_created_at) filters.append( or_( FleetModel.created_at < prev_created_at, diff --git a/src/dstack/_internal/server/services/instances.py b/src/dstack/_internal/server/services/instances.py index ef3f15efb..b30e2e342 100644 --- a/src/dstack/_internal/server/services/instances.py +++ b/src/dstack/_internal/server/services/instances.py @@ -796,6 +796,9 @@ async def list_projects_instance_models( if prev_id is None: filters.append(InstanceModel.created_at > prev_created_at) else: + # Redundant with the OR below, but unlike the OR it lets the database + # start the index scan at the cursor instead of scanning from the start. + filters.append(InstanceModel.created_at >= prev_created_at) filters.append( or_( InstanceModel.created_at > prev_created_at, @@ -809,6 +812,7 @@ async def list_projects_instance_models( if prev_id is None: filters.append(InstanceModel.created_at < prev_created_at) else: + filters.append(InstanceModel.created_at <= prev_created_at) filters.append( or_( InstanceModel.created_at < prev_created_at, diff --git a/src/dstack/_internal/server/services/projects.py b/src/dstack/_internal/server/services/projects.py index 452ff93c2..282e078cc 100644 --- a/src/dstack/_internal/server/services/projects.py +++ b/src/dstack/_internal/server/services/projects.py @@ -137,6 +137,9 @@ async def list_user_accessible_projects( if prev_id is None: pagination_filters.append(ProjectModel.created_at > prev_created_at) else: + # Redundant with the OR below, but unlike the OR it lets the database + # start the index scan at the cursor instead of scanning from the start. + pagination_filters.append(ProjectModel.created_at >= prev_created_at) pagination_filters.append( or_( ProjectModel.created_at > prev_created_at, @@ -149,6 +152,7 @@ async def list_user_accessible_projects( if prev_id is None: pagination_filters.append(ProjectModel.created_at < prev_created_at) else: + pagination_filters.append(ProjectModel.created_at <= prev_created_at) pagination_filters.append( or_( ProjectModel.created_at < prev_created_at, diff --git a/src/dstack/_internal/server/services/runs/__init__.py b/src/dstack/_internal/server/services/runs/__init__.py index 436b74409..a132c15d8 100644 --- a/src/dstack/_internal/server/services/runs/__init__.py +++ b/src/dstack/_internal/server/services/runs/__init__.py @@ -302,6 +302,9 @@ async def list_projects_run_models( if prev_run_id is None: filters.append(RunModel.submitted_at > prev_submitted_at) else: + # Redundant with the OR below, but unlike the OR it lets the database + # start the index scan at the cursor instead of scanning from the start. + filters.append(RunModel.submitted_at >= prev_submitted_at) filters.append( or_( RunModel.submitted_at > prev_submitted_at, @@ -314,6 +317,7 @@ async def list_projects_run_models( if prev_run_id is None: filters.append(RunModel.submitted_at < prev_submitted_at) else: + filters.append(RunModel.submitted_at <= prev_submitted_at) filters.append( or_( RunModel.submitted_at < prev_submitted_at, diff --git a/src/dstack/_internal/server/services/users.py b/src/dstack/_internal/server/services/users.py index 35c1fe5b3..b5efb7973 100644 --- a/src/dstack/_internal/server/services/users.py +++ b/src/dstack/_internal/server/services/users.py @@ -108,6 +108,9 @@ async def list_all_users( if prev_id is None: pagination_filters.append(UserModel.created_at > prev_created_at) else: + # Redundant with the OR below, but unlike the OR it lets the database + # start the index scan at the cursor instead of scanning from the start. + pagination_filters.append(UserModel.created_at >= prev_created_at) pagination_filters.append( or_( UserModel.created_at > prev_created_at, @@ -121,6 +124,7 @@ async def list_all_users( if prev_id is None: pagination_filters.append(UserModel.created_at < prev_created_at) else: + pagination_filters.append(UserModel.created_at <= prev_created_at) pagination_filters.append( or_( UserModel.created_at < prev_created_at, diff --git a/src/dstack/_internal/server/services/volumes.py b/src/dstack/_internal/server/services/volumes.py index d77d6f5ea..5d90fa1d6 100644 --- a/src/dstack/_internal/server/services/volumes.py +++ b/src/dstack/_internal/server/services/volumes.py @@ -150,6 +150,9 @@ async def list_projects_volume_models( if prev_id is None: filters.append(VolumeModel.created_at > prev_created_at) else: + # Redundant with the OR below, but unlike the OR it lets the database + # start the index scan at the cursor instead of scanning from the start. + filters.append(VolumeModel.created_at >= prev_created_at) filters.append( or_( VolumeModel.created_at > prev_created_at, @@ -160,6 +163,7 @@ async def list_projects_volume_models( if prev_id is None: filters.append(VolumeModel.created_at < prev_created_at) else: + filters.append(VolumeModel.created_at <= prev_created_at) filters.append( or_( VolumeModel.created_at < prev_created_at, diff --git a/src/tests/_internal/server/routers/test_events.py b/src/tests/_internal/server/routers/test_events.py index 1c9149791..d27cfb9cb 100644 --- a/src/tests/_internal/server/routers/test_events.py +++ b/src/tests/_internal/server/routers/test_events.py @@ -1476,6 +1476,42 @@ async def test_pagination( resp.raise_for_status() assert len(resp.json()) == 0 + @pytest.mark.parametrize("ascending", [True, False]) + async def test_pagination_with_same_recorded_at( + self, session: AsyncSession, client: AsyncClient, ascending: bool + ) -> None: + users = [await create_user(session=session, name=f"user_{i}") for i in range(3)] + with time_machine.travel(datetime(2026, 1, 1, 12, 0, 0), tick=False): + for user in users: + events.emit( + session, + "User created", + actor=events.UserActor.from_user(user), + targets=[events.Target.from_model(user)], + ) + await session.commit() + + listed = [] + pagination = {} + for _ in range(len(users) + 1): + resp = await client.post( + "/api/events/list", + json={"limit": 1, "ascending": ascending, **pagination}, + headers=get_auth_headers(users[0].token), + ) + resp.raise_for_status() + if not resp.json(): + break + listed.extend(resp.json()) + pagination = { + "prev_id": resp.json()[-1]["id"], + "prev_recorded_at": resp.json()[-1]["recorded_at"], + } + ids = [event["id"] for event in listed] + # Events with the same `recorded_at` are ordered by `id`, descending when ascending. + assert ids == sorted(ids, reverse=ascending) + assert len(set(ids)) == len(users) + async def test_limits_events_regardless_number_of_targets( self, session: AsyncSession, client: AsyncClient ) -> None: