From 49ac2331664baab58cddb86a7a7bf678b2e960e1 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Fri, 13 Nov 2026 11:57:08 +0000 Subject: [PATCH] Report signals in job exit status When a signal killed the job's process, the runner reported exit status -1, the value of os.ProcessState.ExitCode() for any signal, and the job was shown as `exited (-1)`. The runner now reports -N for signal N, as Python's subprocess does. The server shows it as 128+N, which is what a forking shell or Docker reports, so a kill looks the same whichever shell the job runs in. -1 is shown as is, since older runners report it for any signal. A failed job without an exit status is now shown as `error` instead of `exited (None)`. Fixes: https://github.com/dstackai/dstack/issues/4487 Co-Authored-By: Claude Opus 5.5 --- runner/internal/runner/executor/executor.go | 29 ++++++++++++-- .../internal/runner/executor/executor_test.go | 39 +++++++++++++------ src/dstack/_internal/core/models/runs.py | 5 +++ src/dstack/_internal/server/schemas/runner.py | 3 ++ .../server/services/jobs/__init__.py | 19 ++++++++- src/tests/_internal/cli/utils/test_run.py | 24 ++++++++++++ 6 files changed, 102 insertions(+), 17 deletions(-) diff --git a/runner/internal/runner/executor/executor.go b/runner/internal/runner/executor/executor.go index b1917913cc..04343ffc72 100644 --- a/runner/internal/runner/executor/executor.go +++ b/runner/internal/runner/executor/executor.go @@ -315,9 +315,8 @@ func (ex *RunExecutor) Run(ctx context.Context) (err error) { // todo fail reason? log.Error(ctx, "Exec failed", "err", err) - var exitError *exec.ExitError - if errors.As(err, &exitError) { - ex.SetJobStateWithExitStatus(ctx, schemas.JobStateFailed, exitError.ExitCode()) + if exitStatus, ok := jobExitStatus(err); ok { + ex.SetJobStateWithExitStatus(ctx, schemas.JobStateFailed, exitStatus) } else { ex.SetJobState(ctx, schemas.JobStateFailed) } @@ -717,6 +716,30 @@ func watchLogQuota(stopJob context.CancelFunc, quotaExceeded <-chan struct{}) (s return func() { close(done) } } +// jobExitStatus extracts the job's exit status from the error execJob returned: the status +// itself if the job process exited, or the negated signal number if a signal killed it, as in +// Python's subprocess. The stops the runner initiates never get here, Run handles them first. +// +// The job process is the shell running the job's commands, so the signal reaches the runner only +// if the shell exec'd the command that got it. If the shell forked, it exits with 128+N instead. +func jobExitStatus(err error) (int, bool) { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + return 0, false + } + ws, ok := exitErr.Sys().(syscall.WaitStatus) + if !ok { + return 0, false + } + switch { + case ws.Exited(): + return ws.ExitStatus(), true + case ws.Signaled(): + return -int(ws.Signal()), true + } + return 0, false +} + // setupGitCredentials must be called from Run after setJobUser func (ex *RunExecutor) setupGitCredentials(ctx context.Context) (func(), error) { if ex.repoCredentials == nil { diff --git a/runner/internal/runner/executor/executor_test.go b/runner/internal/runner/executor/executor_test.go index bd3c3f35e8..6290f5696c 100644 --- a/runner/internal/runner/executor/executor_test.go +++ b/runner/internal/runner/executor/executor_test.go @@ -71,18 +71,33 @@ func TestExecutor_HomeDir(t *testing.T) { assert.Equal(t, ex.currentUser.HomeDir+"\n", strings.ReplaceAll(b.String(), "\r\n", "\n")) } -func TestExecutor_NonZeroExit(t *testing.T) { - ex := makeTestExecutor(t) - ex.jobSpec.Commands = append(ex.jobSpec.Commands, "exit 100") - makeCodeTar(t, ex) - - setUpTestExecutor(t, ex) - err := ex.Run(t.Context()) - assert.Error(t, err) - assert.NotEmpty(t, ex.jobStateHistory) - exitStatus := ex.jobStateHistory[len(ex.jobStateHistory)-1].ExitStatus - assert.NotNil(t, exitStatus) - assert.Equal(t, 100, *exitStatus) +func TestExecutor_ExitStatus(t *testing.T) { + testCases := []struct { + name string + command string + expected int + }{ + {"exited", "exit 100", 100}, + // $$ is the shell itself, the runner's direct child, so no shell is left to turn the + // signal into 128+N + {"killed_by_signal", "kill -KILL $$", -9}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ex := makeTestExecutor(t) + ex.jobSpec.Commands = append(ex.jobSpec.Commands, tc.command) + makeCodeTar(t, ex) + + setUpTestExecutor(t, ex) + err := ex.Run(t.Context()) + assert.Error(t, err) + require.NotEmpty(t, ex.jobStateHistory) + lastState := ex.jobStateHistory[len(ex.jobStateHistory)-1] + assert.Equal(t, schemas.JobStateFailed, lastState.State) + require.NotNil(t, lastState.ExitStatus) + assert.Equal(t, tc.expected, *lastState.ExitStatus) + }) + } } func TestExecutor_SSHCredentials(t *testing.T) { diff --git a/src/dstack/_internal/core/models/runs.py b/src/dstack/_internal/core/models/runs.py index a888c99074..a63ead8c68 100644 --- a/src/dstack/_internal/core/models/runs.py +++ b/src/dstack/_internal/core/models/runs.py @@ -448,6 +448,11 @@ class JobSubmission(CoreModel): """ termination_reason_message: Optional[str] = None exit_status: Optional[int] = None + """`exit_status` is `-N` if the job's process was killed by signal `N`, e.g. `-9` for + `SIGKILL`, as in Python's `subprocess`. The process is the shell running the job's commands, + so this happens only if the shell has replaced itself with the command that got the signal. + Otherwise, the shell exits with `128 + N`. Older runners report `-1` for any signal. + """ job_provisioning_data: Optional[JobProvisioningData] = None job_runtime_data: Optional[JobRuntimeData] = None error: Optional[str] = None diff --git a/src/dstack/_internal/server/schemas/runner.py b/src/dstack/_internal/server/schemas/runner.py index a9aa404921..3a3aac4a0a 100644 --- a/src/dstack/_internal/server/schemas/runner.py +++ b/src/dstack/_internal/server/schemas/runner.py @@ -25,6 +25,9 @@ class JobStateEvent(CoreModel): termination_reason: Optional[str] = None termination_message: Optional[str] = None exit_status: Optional[int] = None + """`exit_status` is `-N` if the job's process was killed by signal `N`. + See `JobSubmission.exit_status`. + """ class LogEvent(CoreModel): diff --git a/src/dstack/_internal/server/services/jobs/__init__.py b/src/dstack/_internal/server/services/jobs/__init__.py index 158358d2ae..a9dfc2dfe1 100644 --- a/src/dstack/_internal/server/services/jobs/__init__.py +++ b/src/dstack/_internal/server/services/jobs/__init__.py @@ -618,8 +618,12 @@ def _get_job_status_message(job_model: JobModel) -> str: if job_model.status == JobStatus.DONE: return "exited (0)" elif job_model.status == JobStatus.FAILED: - if job_model.termination_reason == JobTerminationReason.CONTAINER_EXITED_WITH_ERROR: - return f"exited ({job_model.exit_status})" + if ( + job_model.termination_reason == JobTerminationReason.CONTAINER_EXITED_WITH_ERROR + # Not set if the job failed outside of its process, e.g. the repo setup failed + and job_model.exit_status is not None + ): + return f"exited ({_get_shell_exit_status(job_model.exit_status)})" elif ( job_model.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY ): @@ -641,6 +645,17 @@ def _get_job_status_message(job_model: JobModel) -> str: return job_model.status.value +def _get_shell_exit_status(exit_status: int) -> int: + """ + Converts a signal `N`, stored as `-N`, to `128 + N`, the way shells and Docker report it. + A forking shell already reports the signal as `128 + N`, so this makes a kill look the same + whichever shell the job runs in. `-1` is kept as is: older runners report it for any signal. + """ + if exit_status >= 0 or exit_status == -1: + return exit_status + return 128 - exit_status + + def _get_job_error(job_model: JobModel) -> Optional[str]: if job_model.termination_reason is None: return None diff --git a/src/tests/_internal/cli/utils/test_run.py b/src/tests/_internal/cli/utils/test_run.py index dffde505e8..7b37c3270e 100644 --- a/src/tests/_internal/cli/utils/test_run.py +++ b/src/tests/_internal/cli/utils/test_run.py @@ -250,6 +250,30 @@ async def test_simple_run(self, session: AsyncSession): "exited (42)", "indian_red1", ), + ( + JobStatus.FAILED, + JobTerminationReason.CONTAINER_EXITED_WITH_ERROR, + -9, + None, + "exited (137)", + "indian_red1", + ), + ( + JobStatus.FAILED, + JobTerminationReason.CONTAINER_EXITED_WITH_ERROR, + -1, + None, + "exited (-1)", + "indian_red1", + ), + ( + JobStatus.FAILED, + JobTerminationReason.CONTAINER_EXITED_WITH_ERROR, + None, + None, + "error", + "indian_red1", + ), ( JobStatus.FAILED, JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY,