Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
"""Replace ix_jobs_run_id with ix_jobs_run_id_replica_num_job_num_submission_num

Revision ID: 6b5610ff36c1
Revises: ca5fb194134e
Create Date: 2026-11-06 10:12:47.823527+00:00

"""

from alembic import op

# revision identifiers, used by Alembic.
revision = "6b5610ff36c1"
down_revision = "ca5fb194134e"
branch_labels = None
depends_on = None

TABLE_NAME = "jobs"
OLD_INDEX_NAME = "ix_jobs_run_id"
NEW_INDEX_NAME = "ix_jobs_run_id_replica_num_job_num_submission_num"
NEW_INDEX_COLUMNS = ["run_id", "replica_num", "job_num", "submission_num"]


def _drop_index(name: str) -> None:
op.drop_index(
name,
table_name=TABLE_NAME,
if_exists=True,
postgresql_concurrently=True,
)


def upgrade() -> None:
with op.get_context().autocommit_block():
_drop_index(NEW_INDEX_NAME)
op.create_index(
NEW_INDEX_NAME,
TABLE_NAME,
NEW_INDEX_COLUMNS,
unique=False,
postgresql_concurrently=True,
)
# The new index starts with `run_id`, so it serves every lookup the old one did.
_drop_index(OLD_INDEX_NAME)


def downgrade() -> None:
with op.get_context().autocommit_block():
_drop_index(OLD_INDEX_NAME)
op.create_index(
OLD_INDEX_NAME,
TABLE_NAME,
["run_id"],
unique=False,
postgresql_concurrently=True,
)
_drop_index(NEW_INDEX_NAME)
12 changes: 9 additions & 3 deletions src/dstack/_internal/server/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -522,9 +522,7 @@ class JobModel(PipelineModelMixin, BaseModel):
)
project: Mapped["ProjectModel"] = relationship()

run_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("runs.id", ondelete="CASCADE"), index=True
)
run_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("runs.id", ondelete="CASCADE"))
run: Mapped["RunModel"] = relationship()

fleet_id: Mapped[Optional[uuid.UUID]] = mapped_column(ForeignKey("fleets.id"))
Expand Down Expand Up @@ -620,6 +618,14 @@ class JobModel(PipelineModelMixin, BaseModel):
postgresql_where=status.not_in(JobStatus.finished_statuses()),
sqlite_where=status.not_in(JobStatus.finished_statuses()),
),
# Serves lookups by `run_id` and finding the latest submissions of each job.
Index(
"ix_jobs_run_id_replica_num_job_num_submission_num",
run_id,
replica_num,
job_num,
submission_num,
),
)


Expand Down
131 changes: 43 additions & 88 deletions src/dstack/_internal/server/services/runs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@

import pydantic
from apscheduler.triggers.cron import CronTrigger
from sqlalchemy import and_, func, or_, select, update
from sqlalchemy import and_, case, func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import aliased, joinedload, raiseload, selectinload
from sqlalchemy.orm import joinedload, raiseload, selectinload

import dstack._internal.utils.common as common_utils
from dstack._internal.core.errors import (
Expand Down Expand Up @@ -186,7 +186,7 @@ async def list_user_runs(
username: Optional[str],
only_active: bool,
include_jobs: bool,
job_submissions_limit: Optional[int],
job_submissions_limit: int,
prev_submitted_at: Optional[datetime],
prev_run_id: Optional[uuid.UUID],
limit: int,
Expand Down Expand Up @@ -348,7 +348,7 @@ async def _list_job_models_by_run_id(
session: AsyncSession,
run_models: List[RunModel],
include_jobs: bool,
job_submissions_limit: Optional[int],
job_submissions_limit: int,
return_in_api: bool,
) -> dict[uuid.UUID, List[JobModel]]:
"""
Expand All @@ -375,106 +375,61 @@ async def _list_job_models_by_run_id(
async def _list_job_models(
session: AsyncSession,
run_ids: List[uuid.UUID],
job_submissions_limit: Optional[int],
job_submissions_limit: int,
include_probes: bool,
) -> List[JobModel]:
"""
List job models for runs list responses.

When job_submissions_limit is set, include up to job_submissions_limit latest
submissions per job plus the latest terminated submission per job. This gives
run_model_to_run enough data without loading every historical submission.
Return at most job_submissions_limit latest submissions of each job: those whose
submission_num is within job_submissions_limit of the job's latest submission_num.
A job gets fewer if its submission numbers have gaps, which happens when submissions
are deleted. Also return the latest submission with a termination reason, which
run_model_to_run needs for status messages such as `retrying`.
"""
options = []
if include_probes:
options.append(joinedload(JobModel.probes))
if job_submissions_limit is None:
# With no job_submissions_limit, return full submission history. This
# can be slow. UI/CLI list views pass a limit, but API callers may omit it.
res = await session.execute(
select(JobModel)
.where(JobModel.run_id.in_(run_ids))
.options(*options)
.order_by(
JobModel.run_id,
JobModel.replica_num,
JobModel.job_num,
JobModel.submission_num,
)
)
return list(res.unique().scalars().all())

requested_jobs = await _list_latest_job_models_per_job(
session=session,
run_ids=run_ids,
limit_per_job=max(job_submissions_limit, 1),
include_probes=include_probes,
)
# Also load rows needed to preserve run.status_message, e.g. `retrying`.
status_message_jobs = await _list_latest_job_models_per_job(
session=session,
run_ids=run_ids,
limit_per_job=1,
include_probes=False,
only_with_termination_reason=True,
)

# Merge the two job lists by ID because the same row may appear in both.
jobs_by_id = {job.id: job for job in requested_jobs}
for job in status_message_jobs:
jobs_by_id.setdefault(job.id, job)
return sorted(
jobs_by_id.values(),
key=lambda j: (j.run_id, j.replica_num, j.job_num, j.submission_num),
)


async def _list_latest_job_models_per_job(
session: AsyncSession,
run_ids: List[uuid.UUID],
limit_per_job: int,
include_probes: bool,
only_with_termination_reason: bool = False,
) -> List[JobModel]:
"""
List up to N newest submissions for each job.

A newer submission has a higher submission_num. The SQL window applies the
per-job limit in the database instead of loading all retries and slicing them
in Python.
"""
row_number = (
func.row_number()
.over(
partition_by=(JobModel.run_id, JobModel.replica_num, JobModel.job_num),
order_by=JobModel.submission_num.desc(),
)
.label("row_number")
)
filters = [JobModel.run_id.in_(run_ids)]
if only_with_termination_reason:
filters.append(JobModel.termination_reason.isnot(None))
jobs_sq = (
limit = max(job_submissions_limit, 1)
latest = (
select(
JobModel,
row_number,
JobModel.run_id,
JobModel.replica_num,
JobModel.job_num,
func.max(JobModel.submission_num).label("submission_num"),
func.max(
case((JobModel.termination_reason.isnot(None), JobModel.submission_num))
).label("terminated_submission_num"),
)
.where(*filters)
.where(JobModel.run_id.in_(run_ids))
.group_by(JobModel.run_id, JobModel.replica_num, JobModel.job_num)
.subquery()
)
job_alias = aliased(JobModel, jobs_sq)
options = []
if include_probes:
options.append(joinedload(job_alias.probes))
res = await session.execute(
select(job_alias)
.where(jobs_sq.c.row_number <= limit_per_job)
select(JobModel)
.join(
latest,
and_(
JobModel.run_id == latest.c.run_id,
JobModel.replica_num == latest.c.replica_num,
JobModel.job_num == latest.c.job_num,
),
)
.where(
# Redundant with the join, but lets Postgres hash-join the jobs with `latest`
# instead of looking up each job's run separately.
JobModel.run_id.in_(run_ids),
or_(
JobModel.submission_num > latest.c.submission_num - limit,
JobModel.submission_num == latest.c.terminated_submission_num,
),
)
.options(*options)
.order_by(
job_alias.run_id,
job_alias.replica_num,
job_alias.job_num,
job_alias.submission_num,
JobModel.run_id,
JobModel.replica_num,
JobModel.job_num,
JobModel.submission_num,
)
)
return list(res.unique().scalars().all())
Expand Down
56 changes: 53 additions & 3 deletions src/tests/_internal/server/services/runs/test_runs.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,10 @@ async def list_jobs_wrapper(*args, **kwargs):
@event.listens_for(test_db.engine.sync_engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
normalized_statement = statement.lower()
if "from jobs" in normalized_statement and "row_number" not in normalized_statement:
if (
"from jobs" in normalized_statement
and "max(jobs.submission_num)" not in normalized_statement
):
unbounded_job_selects.append(statement)

try:
Expand Down Expand Up @@ -136,7 +139,7 @@ def before_cursor_execute(conn, cursor, statement, parameters, context, executem
[
(True, 1),
(True, 0),
(True, None),
(True, 10),
(False, 1),
],
)
Expand All @@ -145,7 +148,7 @@ async def test_limited_list_preserves_status_message_matrix(
test_db,
session: AsyncSession,
include_jobs: bool,
job_submissions_limit: int | None,
job_submissions_limit: int,
) -> None:
user = await create_user(session=session, global_role=GlobalRole.USER)
project = await create_project(session=session, owner=user)
Expand Down Expand Up @@ -326,6 +329,53 @@ async def create_listed_run(
if not include_jobs:
assert all(run.jobs == [] for run in runs)

@pytest.mark.asyncio
@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
async def test_returns_submissions_among_last_submission_nums(
self, test_db, session: AsyncSession
) -> None:
user = await create_user(session=session)
project = await create_project(session=session, owner=user)
repo = await create_repo(session=session, project_id=project.id)
run = await create_run(session=session, project=project, repo=repo, user=user)
# Replica 1 has a gap where submissions were deleted.
submission_nums_by_replica = {0: [0, 1, 2, 3, 4], 1: [0, 1, 7, 8]}
for replica_num, submission_nums in submission_nums_by_replica.items():
for submission_num in submission_nums:
await create_job(
session=session,
run=run,
replica_num=replica_num,
submission_num=submission_num,
status=(
JobStatus.SUBMITTED
if submission_num == submission_nums[-1]
else JobStatus.TERMINATED
),
)

runs = await runs_services.list_user_runs(
session=session,
user=user,
project_name=project.name,
repo_id=None,
username=None,
only_active=False,
include_jobs=True,
job_submissions_limit=3,
prev_submitted_at=None,
prev_run_id=None,
limit=100,
ascending=False,
)

assert len(runs) == 1
submission_nums = {
job.job_spec.replica_num: [s.submission_num for s in job.job_submissions]
for job in runs[0].jobs
}
assert submission_nums == {0: [2, 3, 4], 1: [7, 8]}


class TestCanAttachRunVolumes:
@pytest.mark.asyncio
Expand Down
Loading