Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 19 additions & 9 deletions runner/internal/runner/ssh/sshd.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ type Sshd struct {

akMu sync.Mutex

cmd *exec.Cmd
cmd *exec.Cmd
cancel context.CancelFunc
}

func NewSshd(binPath string) *Sshd {
Expand Down Expand Up @@ -129,27 +130,36 @@ func (d *Sshd) Start(ctx context.Context) error {
if d.confPath == "" {
return errors.New("not configured")
}
// The server pulls the job's final state through sshd, so sshd must outlive the shutdown
// signal that cancels the caller's context. Only Stop ends it.
ctx, d.cancel = context.WithCancel(context.WithoutCancel(ctx))
cmd := exec.CommandContext(ctx, d.binPath, "-D", "-f", d.confPath, "-E", d.logPath)
cmd.Cancel = func() error {
return d.sendSigterm()
return cmd.Process.Signal(syscall.SIGTERM)
}
cmd.WaitDelay = time.Second * 10
d.cmd = cmd
return cmd.Start()
if err := cmd.Start(); err != nil {
d.cancel()
return err
}
return nil
}

func (d *Sshd) Stop(ctx context.Context) error {
if d.cmd == nil {
return errors.New("not started")
}
if err := d.sendSigterm(); err != nil {
d.cancel()
if err := d.cmd.Wait(); !errors.Is(err, context.Canceled) {
return err
}
return d.cmd.Wait()
}

func (d *Sshd) sendSigterm() error {
return d.cmd.Process.Signal(syscall.SIGTERM)
// Wait reports the cancellation in place of the exit status. sshd exits with 0 on SIGTERM,
// so any other status means it had already failed by the time it was stopped.
if state := d.cmd.ProcessState; !state.Success() {
return &exec.ExitError{ProcessState: state}
}
return nil
}

func generateHostKeys(ctx context.Context, confDir string) error {
Expand Down
47 changes: 47 additions & 0 deletions runner/internal/runner/ssh/sshd_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package ssh

import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"testing"

"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)

func TestSshdStartIgnoresCallerCancellation(t *testing.T) {
readyPath := filepath.Join(t.TempDir(), "ready")
require.NoError(t, unix.Mkfifo(readyPath, 0o600))
// Exits with 0 on SIGTERM, as sshd does. Writing to the FIFO blocks until the test reads
// it, so the test cannot stop the script before the trap is installed.
d := newFakeSshd(t, fmt.Sprintf("trap 'kill $!; exit 0' TERM; sleep 60 & echo > %s; wait", readyPath))
ctx, cancel := context.WithCancel(context.Background())
cancel()

// exec.CommandContext refuses to start on a context that is already done, so Start
// only succeeds if sshd's lifetime is detached from the caller's context
require.NoError(t, d.Start(ctx))
_, err := os.ReadFile(readyPath)
require.NoError(t, err)
require.NoError(t, d.Stop(context.Background()))
}

func TestSshdStopReportsFailure(t *testing.T) {
d := newFakeSshd(t, "exit 1")
require.NoError(t, d.Start(context.Background()))

var exitErr *exec.ExitError
require.ErrorAs(t, d.Stop(context.Background()), &exitErr)
}

// newFakeSshd returns an Sshd that runs script in place of the sshd binary. Spawning a shell
// costs a few milliseconds, and it is what exercises the real os/exec cancellation.
func newFakeSshd(t *testing.T, script string) *Sshd {
t.Helper()
binPath := filepath.Join(t.TempDir(), "sshd")
require.NoError(t, os.WriteFile(binPath, []byte("#!/bin/sh\n"+script+"\n"), 0o755))
return &Sshd{binPath: binPath, confPath: "sshd_config", logPath: "sshd.log"}
}
Loading