Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions contributing/DATABASE.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,3 +84,29 @@ Keep locked regions short and bounded. See `LOCKING.md` for how to take advisory
`DSTACK_DB_COMMAND_TIMEOUT` (default 300 seconds) bounds every operation on a Postgres connection, so a coroutine waiting on a connection whose socket died silently gets an error and releases the slot instead of holding it forever. It applies to migrations too, since they run on the same engine, so it must leave room for index builds and backfills.

`DSTACK_DB_POOL_SIZE` and `DSTACK_DB_MAX_OVERFLOW` size the pool per process. Raising them lets more coroutines hold connections at once, which hides a long hold rather than fixing it, and each replica needs `pool_size + max_overflow` connections from the Postgres `max_connections` budget.

## Queries

### Cursor pagination

List APIs paginate with a cursor: the timestamp and id of the last returned row, e.g. `prev_created_at` and `prev_id`. Rows with the same timestamp are ordered by id in the opposite direction, so the cursor filter is an OR:

```python
or_(
Model.created_at > prev_created_at,
and_(Model.created_at == prev_created_at, Model.id < prev_id),
)
```

Always add the timestamp bound next to it, `>=` for ascending and `<=` for descending:

```python
filters.append(Model.created_at >= prev_created_at)
filters.append(or_(...))
```

The bound doesn't change the results, but without it a request can read the whole table. Postgres can use the OR with an index only by looking up each branch separately and sorting the combined result. When it expects many rows after the cursor, it reads the timestamp index in order instead, and it can start that scan only from a single condition such as `created_at >= value`, which it doesn't derive from the OR. Then the scan starts at the oldest row and filters out everything before the cursor. Postgres expects many rows after the cursor when the cursor is old, and always with a generic plan of a prepared statement, which is planned without the cursor value. See [#4465](https://github.com/dstackai/dstack/issues/4465).

A row comparison such as `(created_at, id) > (prev_created_at, prev_id)` would avoid the OR, but it requires both columns to be ordered in the same direction.

The bound only removes reading rows before the cursor. A request still reads rows after the cursor until it finds `limit` rows that match its other filters, so on a large table a filter that matches few rows, such as one project's rows, also needs an index that starts with the filter column.
4 changes: 4 additions & 0 deletions src/dstack/_internal/server/services/events.py
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,9 @@ async def list_events(
if prev_id is None:
event_filters.append(EventModel.recorded_at > prev_recorded_at)
else:
# Redundant with the OR below, but unlike the OR it lets the database
# start the index scan at the cursor instead of scanning from the start.
event_filters.append(EventModel.recorded_at >= prev_recorded_at)
event_filters.append(
or_(
EventModel.recorded_at > prev_recorded_at,
Expand All @@ -428,6 +431,7 @@ async def list_events(
if prev_id is None:
event_filters.append(EventModel.recorded_at < prev_recorded_at)
else:
event_filters.append(EventModel.recorded_at <= prev_recorded_at)
event_filters.append(
or_(
EventModel.recorded_at < prev_recorded_at,
Expand Down
4 changes: 4 additions & 0 deletions src/dstack/_internal/server/services/fleets.py
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,9 @@ async def list_projects_fleet_models(
if prev_id is None:
filters.append(FleetModel.created_at > prev_created_at)
else:
# Redundant with the OR below, but unlike the OR it lets the database
# start the index scan at the cursor instead of scanning from the start.
filters.append(FleetModel.created_at >= prev_created_at)
filters.append(
or_(
FleetModel.created_at > prev_created_at,
Expand All @@ -287,6 +290,7 @@ async def list_projects_fleet_models(
if prev_id is None:
filters.append(FleetModel.created_at < prev_created_at)
else:
filters.append(FleetModel.created_at <= prev_created_at)
filters.append(
or_(
FleetModel.created_at < prev_created_at,
Expand Down
4 changes: 4 additions & 0 deletions src/dstack/_internal/server/services/instances.py
Original file line number Diff line number Diff line change
Expand Up @@ -796,6 +796,9 @@ async def list_projects_instance_models(
if prev_id is None:
filters.append(InstanceModel.created_at > prev_created_at)
else:
# Redundant with the OR below, but unlike the OR it lets the database
# start the index scan at the cursor instead of scanning from the start.
filters.append(InstanceModel.created_at >= prev_created_at)
filters.append(
or_(
InstanceModel.created_at > prev_created_at,
Expand All @@ -809,6 +812,7 @@ async def list_projects_instance_models(
if prev_id is None:
filters.append(InstanceModel.created_at < prev_created_at)
else:
filters.append(InstanceModel.created_at <= prev_created_at)
filters.append(
or_(
InstanceModel.created_at < prev_created_at,
Expand Down
4 changes: 4 additions & 0 deletions src/dstack/_internal/server/services/projects.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,9 @@ async def list_user_accessible_projects(
if prev_id is None:
pagination_filters.append(ProjectModel.created_at > prev_created_at)
else:
# Redundant with the OR below, but unlike the OR it lets the database
# start the index scan at the cursor instead of scanning from the start.
pagination_filters.append(ProjectModel.created_at >= prev_created_at)
pagination_filters.append(
or_(
ProjectModel.created_at > prev_created_at,
Expand All @@ -149,6 +152,7 @@ async def list_user_accessible_projects(
if prev_id is None:
pagination_filters.append(ProjectModel.created_at < prev_created_at)
else:
pagination_filters.append(ProjectModel.created_at <= prev_created_at)
pagination_filters.append(
or_(
ProjectModel.created_at < prev_created_at,
Expand Down
4 changes: 4 additions & 0 deletions src/dstack/_internal/server/services/runs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,9 @@ async def list_projects_run_models(
if prev_run_id is None:
filters.append(RunModel.submitted_at > prev_submitted_at)
else:
# Redundant with the OR below, but unlike the OR it lets the database
# start the index scan at the cursor instead of scanning from the start.
filters.append(RunModel.submitted_at >= prev_submitted_at)
filters.append(
or_(
RunModel.submitted_at > prev_submitted_at,
Expand All @@ -314,6 +317,7 @@ async def list_projects_run_models(
if prev_run_id is None:
filters.append(RunModel.submitted_at < prev_submitted_at)
else:
filters.append(RunModel.submitted_at <= prev_submitted_at)
filters.append(
or_(
RunModel.submitted_at < prev_submitted_at,
Expand Down
4 changes: 4 additions & 0 deletions src/dstack/_internal/server/services/users.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,9 @@ async def list_all_users(
if prev_id is None:
pagination_filters.append(UserModel.created_at > prev_created_at)
else:
# Redundant with the OR below, but unlike the OR it lets the database
# start the index scan at the cursor instead of scanning from the start.
pagination_filters.append(UserModel.created_at >= prev_created_at)
pagination_filters.append(
or_(
UserModel.created_at > prev_created_at,
Expand All @@ -121,6 +124,7 @@ async def list_all_users(
if prev_id is None:
pagination_filters.append(UserModel.created_at < prev_created_at)
else:
pagination_filters.append(UserModel.created_at <= prev_created_at)
pagination_filters.append(
or_(
UserModel.created_at < prev_created_at,
Expand Down
4 changes: 4 additions & 0 deletions src/dstack/_internal/server/services/volumes.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ async def list_projects_volume_models(
if prev_id is None:
filters.append(VolumeModel.created_at > prev_created_at)
else:
# Redundant with the OR below, but unlike the OR it lets the database
# start the index scan at the cursor instead of scanning from the start.
filters.append(VolumeModel.created_at >= prev_created_at)
filters.append(
or_(
VolumeModel.created_at > prev_created_at,
Expand All @@ -160,6 +163,7 @@ async def list_projects_volume_models(
if prev_id is None:
filters.append(VolumeModel.created_at < prev_created_at)
else:
filters.append(VolumeModel.created_at <= prev_created_at)
filters.append(
or_(
VolumeModel.created_at < prev_created_at,
Expand Down
36 changes: 36 additions & 0 deletions src/tests/_internal/server/routers/test_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -1476,6 +1476,42 @@ async def test_pagination(
resp.raise_for_status()
assert len(resp.json()) == 0

@pytest.mark.parametrize("ascending", [True, False])
async def test_pagination_with_same_recorded_at(
self, session: AsyncSession, client: AsyncClient, ascending: bool
) -> None:
users = [await create_user(session=session, name=f"user_{i}") for i in range(3)]
with time_machine.travel(datetime(2026, 1, 1, 12, 0, 0), tick=False):
for user in users:
events.emit(
session,
"User created",
actor=events.UserActor.from_user(user),
targets=[events.Target.from_model(user)],
)
await session.commit()

listed = []
pagination = {}
for _ in range(len(users) + 1):
resp = await client.post(
"/api/events/list",
json={"limit": 1, "ascending": ascending, **pagination},
headers=get_auth_headers(users[0].token),
)
resp.raise_for_status()
if not resp.json():
break
listed.extend(resp.json())
pagination = {
"prev_id": resp.json()[-1]["id"],
"prev_recorded_at": resp.json()[-1]["recorded_at"],
}
ids = [event["id"] for event in listed]
# Events with the same `recorded_at` are ordered by `id`, descending when ascending.
assert ids == sorted(ids, reverse=ascending)
assert len(set(ids)) == len(users)

async def test_limits_events_regardless_number_of_targets(
self, session: AsyncSession, client: AsyncClient
) -> None:
Expand Down
Loading