From 59f6b7e4c97eb0fcbb524b349aedc48d8fa9a3d2 Mon Sep 17 00:00:00 2001 From: Dmitry Meyer Date: Tue, 27 Oct 2026 10:16:52 +0000 Subject: [PATCH] [runner] Keep sshd alive until the runner stops it sshd was started on the runner's signal context, so a shutdown signal killed it immediately, while the runner went on waiting for the server to pull the job's final state. On backends where the server reaches the runner through the container's sshd, the pull could never happen: the server marked the job INSTANCE_UNREACHABLE after two minutes. Detach sshd's lifetime from the caller's context and let Stop cancel it, keeping SIGTERM with the SIGKILL backstop as the only way down. Stop now checks the exit status itself, as Wait reports the cancellation in its place. Fixes: https://github.com/dstackai/dstack/issues/4415 Co-Authored-By: Claude Opus 5.5 (1M context) --- runner/internal/runner/ssh/sshd.go | 28 ++++++++++----- runner/internal/runner/ssh/sshd_test.go | 47 +++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 runner/internal/runner/ssh/sshd_test.go diff --git a/runner/internal/runner/ssh/sshd.go b/runner/internal/runner/ssh/sshd.go index 07aa1ec82d..411c83d54a 100644 --- a/runner/internal/runner/ssh/sshd.go +++ b/runner/internal/runner/ssh/sshd.go @@ -45,7 +45,8 @@ type Sshd struct { akMu sync.Mutex - cmd *exec.Cmd + cmd *exec.Cmd + cancel context.CancelFunc } func NewSshd(binPath string) *Sshd { @@ -129,27 +130,36 @@ func (d *Sshd) Start(ctx context.Context) error { if d.confPath == "" { return errors.New("not configured") } + // The server pulls the job's final state through sshd, so sshd must outlive the shutdown + // signal that cancels the caller's context. Only Stop ends it. + ctx, d.cancel = context.WithCancel(context.WithoutCancel(ctx)) cmd := exec.CommandContext(ctx, d.binPath, "-D", "-f", d.confPath, "-E", d.logPath) cmd.Cancel = func() error { - return d.sendSigterm() + return cmd.Process.Signal(syscall.SIGTERM) } cmd.WaitDelay = time.Second * 10 d.cmd = cmd - return cmd.Start() + if err := cmd.Start(); err != nil { + d.cancel() + return err + } + return nil } func (d *Sshd) Stop(ctx context.Context) error { if d.cmd == nil { return errors.New("not started") } - if err := d.sendSigterm(); err != nil { + d.cancel() + if err := d.cmd.Wait(); !errors.Is(err, context.Canceled) { return err } - return d.cmd.Wait() -} - -func (d *Sshd) sendSigterm() error { - return d.cmd.Process.Signal(syscall.SIGTERM) + // Wait reports the cancellation in place of the exit status. sshd exits with 0 on SIGTERM, + // so any other status means it had already failed by the time it was stopped. + if state := d.cmd.ProcessState; !state.Success() { + return &exec.ExitError{ProcessState: state} + } + return nil } func generateHostKeys(ctx context.Context, confDir string) error { diff --git a/runner/internal/runner/ssh/sshd_test.go b/runner/internal/runner/ssh/sshd_test.go new file mode 100644 index 0000000000..399fc0bd6e --- /dev/null +++ b/runner/internal/runner/ssh/sshd_test.go @@ -0,0 +1,47 @@ +package ssh + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestSshdStartIgnoresCallerCancellation(t *testing.T) { + readyPath := filepath.Join(t.TempDir(), "ready") + require.NoError(t, unix.Mkfifo(readyPath, 0o600)) + // Exits with 0 on SIGTERM, as sshd does. Writing to the FIFO blocks until the test reads + // it, so the test cannot stop the script before the trap is installed. + d := newFakeSshd(t, fmt.Sprintf("trap 'kill $!; exit 0' TERM; sleep 60 & echo > %s; wait", readyPath)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + // exec.CommandContext refuses to start on a context that is already done, so Start + // only succeeds if sshd's lifetime is detached from the caller's context + require.NoError(t, d.Start(ctx)) + _, err := os.ReadFile(readyPath) + require.NoError(t, err) + require.NoError(t, d.Stop(context.Background())) +} + +func TestSshdStopReportsFailure(t *testing.T) { + d := newFakeSshd(t, "exit 1") + require.NoError(t, d.Start(context.Background())) + + var exitErr *exec.ExitError + require.ErrorAs(t, d.Stop(context.Background()), &exitErr) +} + +// newFakeSshd returns an Sshd that runs script in place of the sshd binary. Spawning a shell +// costs a few milliseconds, and it is what exercises the real os/exec cancellation. +func newFakeSshd(t *testing.T, script string) *Sshd { + t.Helper() + binPath := filepath.Join(t.TempDir(), "sshd") + require.NoError(t, os.WriteFile(binPath, []byte("#!/bin/sh\n"+script+"\n"), 0o755)) + return &Sshd{binPath: binPath, confPath: "sshd_config", logPath: "sshd.log"} +}