diff --git a/runner/internal/runner/ssh/sshd.go b/runner/internal/runner/ssh/sshd.go index 07aa1ec82d..411c83d54a 100644 --- a/runner/internal/runner/ssh/sshd.go +++ b/runner/internal/runner/ssh/sshd.go @@ -45,7 +45,8 @@ type Sshd struct { akMu sync.Mutex - cmd *exec.Cmd + cmd *exec.Cmd + cancel context.CancelFunc } func NewSshd(binPath string) *Sshd { @@ -129,27 +130,36 @@ func (d *Sshd) Start(ctx context.Context) error { if d.confPath == "" { return errors.New("not configured") } + // The server pulls the job's final state through sshd, so sshd must outlive the shutdown + // signal that cancels the caller's context. Only Stop ends it. + ctx, d.cancel = context.WithCancel(context.WithoutCancel(ctx)) cmd := exec.CommandContext(ctx, d.binPath, "-D", "-f", d.confPath, "-E", d.logPath) cmd.Cancel = func() error { - return d.sendSigterm() + return cmd.Process.Signal(syscall.SIGTERM) } cmd.WaitDelay = time.Second * 10 d.cmd = cmd - return cmd.Start() + if err := cmd.Start(); err != nil { + d.cancel() + return err + } + return nil } func (d *Sshd) Stop(ctx context.Context) error { if d.cmd == nil { return errors.New("not started") } - if err := d.sendSigterm(); err != nil { + d.cancel() + if err := d.cmd.Wait(); !errors.Is(err, context.Canceled) { return err } - return d.cmd.Wait() -} - -func (d *Sshd) sendSigterm() error { - return d.cmd.Process.Signal(syscall.SIGTERM) + // Wait reports the cancellation in place of the exit status. sshd exits with 0 on SIGTERM, + // so any other status means it had already failed by the time it was stopped. + if state := d.cmd.ProcessState; !state.Success() { + return &exec.ExitError{ProcessState: state} + } + return nil } func generateHostKeys(ctx context.Context, confDir string) error { diff --git a/runner/internal/runner/ssh/sshd_test.go b/runner/internal/runner/ssh/sshd_test.go new file mode 100644 index 0000000000..399fc0bd6e --- /dev/null +++ b/runner/internal/runner/ssh/sshd_test.go @@ -0,0 +1,47 @@ +package ssh + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestSshdStartIgnoresCallerCancellation(t *testing.T) { + readyPath := filepath.Join(t.TempDir(), "ready") + require.NoError(t, unix.Mkfifo(readyPath, 0o600)) + // Exits with 0 on SIGTERM, as sshd does. Writing to the FIFO blocks until the test reads + // it, so the test cannot stop the script before the trap is installed. + d := newFakeSshd(t, fmt.Sprintf("trap 'kill $!; exit 0' TERM; sleep 60 & echo > %s; wait", readyPath)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + // exec.CommandContext refuses to start on a context that is already done, so Start + // only succeeds if sshd's lifetime is detached from the caller's context + require.NoError(t, d.Start(ctx)) + _, err := os.ReadFile(readyPath) + require.NoError(t, err) + require.NoError(t, d.Stop(context.Background())) +} + +func TestSshdStopReportsFailure(t *testing.T) { + d := newFakeSshd(t, "exit 1") + require.NoError(t, d.Start(context.Background())) + + var exitErr *exec.ExitError + require.ErrorAs(t, d.Stop(context.Background()), &exitErr) +} + +// newFakeSshd returns an Sshd that runs script in place of the sshd binary. Spawning a shell +// costs a few milliseconds, and it is what exercises the real os/exec cancellation. +func newFakeSshd(t *testing.T, script string) *Sshd { + t.Helper() + binPath := filepath.Join(t.TempDir(), "sshd") + require.NoError(t, os.WriteFile(binPath, []byte("#!/bin/sh\n"+script+"\n"), 0o755)) + return &Sshd{binPath: binPath, confPath: "sshd_config", logPath: "sshd.log"} +}