diff --git a/src/dstack/_internal/server/migrations/versions/2026/11_06_1012_6b5610ff36c1_replace_ix_jobs_run_id_with_ix_jobs_run_.py b/src/dstack/_internal/server/migrations/versions/2026/11_06_1012_6b5610ff36c1_replace_ix_jobs_run_id_with_ix_jobs_run_.py new file mode 100644 index 000000000..771ddcb04 --- /dev/null +++ b/src/dstack/_internal/server/migrations/versions/2026/11_06_1012_6b5610ff36c1_replace_ix_jobs_run_id_with_ix_jobs_run_.py @@ -0,0 +1,56 @@ +"""Replace ix_jobs_run_id with ix_jobs_run_id_replica_num_job_num_submission_num + +Revision ID: 6b5610ff36c1 +Revises: ca5fb194134e +Create Date: 2026-11-06 10:12:47.823527+00:00 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "6b5610ff36c1" +down_revision = "ca5fb194134e" +branch_labels = None +depends_on = None + +TABLE_NAME = "jobs" +OLD_INDEX_NAME = "ix_jobs_run_id" +NEW_INDEX_NAME = "ix_jobs_run_id_replica_num_job_num_submission_num" +NEW_INDEX_COLUMNS = ["run_id", "replica_num", "job_num", "submission_num"] + + +def _drop_index(name: str) -> None: + op.drop_index( + name, + table_name=TABLE_NAME, + if_exists=True, + postgresql_concurrently=True, + ) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + _drop_index(NEW_INDEX_NAME) + op.create_index( + NEW_INDEX_NAME, + TABLE_NAME, + NEW_INDEX_COLUMNS, + unique=False, + postgresql_concurrently=True, + ) + # The new index starts with `run_id`, so it serves every lookup the old one did. + _drop_index(OLD_INDEX_NAME) + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + _drop_index(OLD_INDEX_NAME) + op.create_index( + OLD_INDEX_NAME, + TABLE_NAME, + ["run_id"], + unique=False, + postgresql_concurrently=True, + ) + _drop_index(NEW_INDEX_NAME) diff --git a/src/dstack/_internal/server/models.py b/src/dstack/_internal/server/models.py index 71db4aeab..880176808 100644 --- a/src/dstack/_internal/server/models.py +++ b/src/dstack/_internal/server/models.py @@ -522,9 +522,7 @@ class JobModel(PipelineModelMixin, BaseModel): ) project: Mapped["ProjectModel"] = relationship() - run_id: Mapped[uuid.UUID] = mapped_column( - ForeignKey("runs.id", ondelete="CASCADE"), index=True - ) + run_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("runs.id", ondelete="CASCADE")) run: Mapped["RunModel"] = relationship() fleet_id: Mapped[Optional[uuid.UUID]] = mapped_column(ForeignKey("fleets.id")) @@ -620,6 +618,14 @@ class JobModel(PipelineModelMixin, BaseModel): postgresql_where=status.not_in(JobStatus.finished_statuses()), sqlite_where=status.not_in(JobStatus.finished_statuses()), ), + # Serves lookups by `run_id` and finding the latest submissions of each job. + Index( + "ix_jobs_run_id_replica_num_job_num_submission_num", + run_id, + replica_num, + job_num, + submission_num, + ), ) diff --git a/src/dstack/_internal/server/services/runs/__init__.py b/src/dstack/_internal/server/services/runs/__init__.py index a132c15d8..f9033c1f5 100644 --- a/src/dstack/_internal/server/services/runs/__init__.py +++ b/src/dstack/_internal/server/services/runs/__init__.py @@ -8,9 +8,9 @@ import pydantic from apscheduler.triggers.cron import CronTrigger -from sqlalchemy import and_, func, or_, select, update +from sqlalchemy import and_, case, func, or_, select, update from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import aliased, joinedload, raiseload, selectinload +from sqlalchemy.orm import joinedload, raiseload, selectinload import dstack._internal.utils.common as common_utils from dstack._internal.core.errors import ( @@ -186,7 +186,7 @@ async def list_user_runs( username: Optional[str], only_active: bool, include_jobs: bool, - job_submissions_limit: Optional[int], + job_submissions_limit: int, prev_submitted_at: Optional[datetime], prev_run_id: Optional[uuid.UUID], limit: int, @@ -348,7 +348,7 @@ async def _list_job_models_by_run_id( session: AsyncSession, run_models: List[RunModel], include_jobs: bool, - job_submissions_limit: Optional[int], + job_submissions_limit: int, return_in_api: bool, ) -> dict[uuid.UUID, List[JobModel]]: """ @@ -375,106 +375,61 @@ async def _list_job_models_by_run_id( async def _list_job_models( session: AsyncSession, run_ids: List[uuid.UUID], - job_submissions_limit: Optional[int], + job_submissions_limit: int, include_probes: bool, ) -> List[JobModel]: """ List job models for runs list responses. - When job_submissions_limit is set, include up to job_submissions_limit latest - submissions per job plus the latest terminated submission per job. This gives - run_model_to_run enough data without loading every historical submission. + Return at most job_submissions_limit latest submissions of each job: those whose + submission_num is within job_submissions_limit of the job's latest submission_num. + A job gets fewer if its submission numbers have gaps, which happens when submissions + are deleted. Also return the latest submission with a termination reason, which + run_model_to_run needs for status messages such as `retrying`. """ options = [] if include_probes: options.append(joinedload(JobModel.probes)) - if job_submissions_limit is None: - # With no job_submissions_limit, return full submission history. This - # can be slow. UI/CLI list views pass a limit, but API callers may omit it. - res = await session.execute( - select(JobModel) - .where(JobModel.run_id.in_(run_ids)) - .options(*options) - .order_by( - JobModel.run_id, - JobModel.replica_num, - JobModel.job_num, - JobModel.submission_num, - ) - ) - return list(res.unique().scalars().all()) - - requested_jobs = await _list_latest_job_models_per_job( - session=session, - run_ids=run_ids, - limit_per_job=max(job_submissions_limit, 1), - include_probes=include_probes, - ) - # Also load rows needed to preserve run.status_message, e.g. `retrying`. - status_message_jobs = await _list_latest_job_models_per_job( - session=session, - run_ids=run_ids, - limit_per_job=1, - include_probes=False, - only_with_termination_reason=True, - ) - - # Merge the two job lists by ID because the same row may appear in both. - jobs_by_id = {job.id: job for job in requested_jobs} - for job in status_message_jobs: - jobs_by_id.setdefault(job.id, job) - return sorted( - jobs_by_id.values(), - key=lambda j: (j.run_id, j.replica_num, j.job_num, j.submission_num), - ) - - -async def _list_latest_job_models_per_job( - session: AsyncSession, - run_ids: List[uuid.UUID], - limit_per_job: int, - include_probes: bool, - only_with_termination_reason: bool = False, -) -> List[JobModel]: - """ - List up to N newest submissions for each job. - - A newer submission has a higher submission_num. The SQL window applies the - per-job limit in the database instead of loading all retries and slicing them - in Python. - """ - row_number = ( - func.row_number() - .over( - partition_by=(JobModel.run_id, JobModel.replica_num, JobModel.job_num), - order_by=JobModel.submission_num.desc(), - ) - .label("row_number") - ) - filters = [JobModel.run_id.in_(run_ids)] - if only_with_termination_reason: - filters.append(JobModel.termination_reason.isnot(None)) - jobs_sq = ( + limit = max(job_submissions_limit, 1) + latest = ( select( - JobModel, - row_number, + JobModel.run_id, + JobModel.replica_num, + JobModel.job_num, + func.max(JobModel.submission_num).label("submission_num"), + func.max( + case((JobModel.termination_reason.isnot(None), JobModel.submission_num)) + ).label("terminated_submission_num"), ) - .where(*filters) + .where(JobModel.run_id.in_(run_ids)) + .group_by(JobModel.run_id, JobModel.replica_num, JobModel.job_num) .subquery() ) - job_alias = aliased(JobModel, jobs_sq) - options = [] - if include_probes: - options.append(joinedload(job_alias.probes)) res = await session.execute( - select(job_alias) - .where(jobs_sq.c.row_number <= limit_per_job) + select(JobModel) + .join( + latest, + and_( + JobModel.run_id == latest.c.run_id, + JobModel.replica_num == latest.c.replica_num, + JobModel.job_num == latest.c.job_num, + ), + ) + .where( + # Redundant with the join, but lets Postgres hash-join the jobs with `latest` + # instead of looking up each job's run separately. + JobModel.run_id.in_(run_ids), + or_( + JobModel.submission_num > latest.c.submission_num - limit, + JobModel.submission_num == latest.c.terminated_submission_num, + ), + ) .options(*options) .order_by( - job_alias.run_id, - job_alias.replica_num, - job_alias.job_num, - job_alias.submission_num, + JobModel.run_id, + JobModel.replica_num, + JobModel.job_num, + JobModel.submission_num, ) ) return list(res.unique().scalars().all()) diff --git a/src/tests/_internal/server/services/runs/test_runs.py b/src/tests/_internal/server/services/runs/test_runs.py index e9087daa6..1a982c74c 100644 --- a/src/tests/_internal/server/services/runs/test_runs.py +++ b/src/tests/_internal/server/services/runs/test_runs.py @@ -92,7 +92,10 @@ async def list_jobs_wrapper(*args, **kwargs): @event.listens_for(test_db.engine.sync_engine, "before_cursor_execute") def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): normalized_statement = statement.lower() - if "from jobs" in normalized_statement and "row_number" not in normalized_statement: + if ( + "from jobs" in normalized_statement + and "max(jobs.submission_num)" not in normalized_statement + ): unbounded_job_selects.append(statement) try: @@ -136,7 +139,7 @@ def before_cursor_execute(conn, cursor, statement, parameters, context, executem [ (True, 1), (True, 0), - (True, None), + (True, 10), (False, 1), ], ) @@ -145,7 +148,7 @@ async def test_limited_list_preserves_status_message_matrix( test_db, session: AsyncSession, include_jobs: bool, - job_submissions_limit: int | None, + job_submissions_limit: int, ) -> None: user = await create_user(session=session, global_role=GlobalRole.USER) project = await create_project(session=session, owner=user) @@ -326,6 +329,53 @@ async def create_listed_run( if not include_jobs: assert all(run.jobs == [] for run in runs) + @pytest.mark.asyncio + @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) + async def test_returns_submissions_among_last_submission_nums( + self, test_db, session: AsyncSession + ) -> None: + user = await create_user(session=session) + project = await create_project(session=session, owner=user) + repo = await create_repo(session=session, project_id=project.id) + run = await create_run(session=session, project=project, repo=repo, user=user) + # Replica 1 has a gap where submissions were deleted. + submission_nums_by_replica = {0: [0, 1, 2, 3, 4], 1: [0, 1, 7, 8]} + for replica_num, submission_nums in submission_nums_by_replica.items(): + for submission_num in submission_nums: + await create_job( + session=session, + run=run, + replica_num=replica_num, + submission_num=submission_num, + status=( + JobStatus.SUBMITTED + if submission_num == submission_nums[-1] + else JobStatus.TERMINATED + ), + ) + + runs = await runs_services.list_user_runs( + session=session, + user=user, + project_name=project.name, + repo_id=None, + username=None, + only_active=False, + include_jobs=True, + job_submissions_limit=3, + prev_submitted_at=None, + prev_run_id=None, + limit=100, + ascending=False, + ) + + assert len(runs) == 1 + submission_nums = { + job.job_spec.replica_num: [s.submission_num for s in job.job_submissions] + for job in runs[0].jobs + } + assert submission_nums == {0: [2, 3, 4], 1: [7, 8]} + class TestCanAttachRunVolumes: @pytest.mark.asyncio