Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 19 additions & 12 deletions manager/orchestrator/taskinit/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,26 +25,33 @@ type InitHandler interface {
// not have finished processing their updates and left them in an inconsistent state.
func CheckTasks(ctx context.Context, s *store.MemoryStore, readTx store.ReadTx, initHandler InitHandler, startSupervisor restart.SupervisorInterface) error {
instances := make(map[orchestrator.SlotTuple][]*api.Task)
err := s.Batch(func(batch *store.Batch) error {
tasks, err := store.FindTasks(readTx, store.All)
if err != nil {
return err
tasks, err := store.FindTasks(readTx, store.All)
if err != nil {
return err
}

// Delete orphaned tasks in separate transactions. Their full task objects
// are included in the replicated delete actions, so several large tasks can
// exceed the Raft transaction limit when removed in a single batch.
for _, t := range tasks {
if t.ServiceID == "" || store.GetService(readTx, t.ServiceID) != nil {
continue
}
if err := s.Update(func(tx store.Tx) error {
return store.DeleteTask(tx, t.ID)
}); err != nil {
log.G(ctx).WithError(err).WithField("task.id", t.ID).Error("failed to delete task")
}
}

err = s.Batch(func(batch *store.Batch) error {
for _, t := range tasks {
if t.ServiceID == "" {
continue
}

// TODO(aluzzardi): We should NOT retrieve the service here.
service := store.GetService(readTx, t.ServiceID)
if service == nil {
// Service was deleted
err := batch.Update(func(tx store.Tx) error {
return store.DeleteTask(tx, t.ID)
})
if err != nil {
log.G(ctx).WithError(err).Error("failed to delete task")
}
continue
}
if !initHandler.IsRelatedService(service) {
Expand Down
80 changes: 80 additions & 0 deletions manager/orchestrator/taskinit/init_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package taskinit

import (
"context"
"fmt"
"strings"
"testing"

"github.com/moby/swarmkit/v2/api"
"github.com/moby/swarmkit/v2/manager/orchestrator"
"github.com/moby/swarmkit/v2/manager/state/store"
"github.com/moby/swarmkit/v2/manager/state/testutils"
"github.com/stretchr/testify/require"
)

// sizeLimitedProposer enforces the serialized request limit used by Raft.
type sizeLimitedProposer struct {
testutils.MockProposer
}

func (p *sizeLimitedProposer) ProposeValue(ctx context.Context, actions []api.StoreAction, cb func()) error {
request := api.InternalRaftRequest{Action: actions}
if size := request.Size(); size > store.MaxTransactionBytes {
return fmt.Errorf("raft request size %d exceeds limit %d", size, int(store.MaxTransactionBytes))
}
return p.MockProposer.ProposeValue(ctx, actions, cb)
}

// unrelatedInitHandler leaves tasks belonging to another orchestrator alone.
type unrelatedInitHandler struct{}

func (unrelatedInitHandler) IsRelatedService(*api.Service) bool { return false }
func (unrelatedInitHandler) FixTask(context.Context, *store.Batch, *api.Task) {}
func (unrelatedInitHandler) SlotTuple(*api.Task) orchestrator.SlotTuple {
return orchestrator.SlotTuple{}
}

func TestCheckTasksLargeOrphanTasks(t *testing.T) {
s := store.NewMemoryStore(&sizeLimitedProposer{})
defer s.Close()

// Each task can be created and deleted independently, but two deletion
// actions together exceed Raft's limit before Batch checks its threshold.
for _, id := range []string{"orphan-1", "orphan-2", "orphan-3"} {
task := &api.Task{
ID: id,
ServiceID: "deleted-service",
Spec: api.TaskSpec{Runtime: &api.TaskSpec_Container{
Container: &api.ContainerSpec{
Image: "busybox:latest",
Env: []string{"PAYLOAD=" + strings.Repeat("x", 850*1024)},
},
}},
Status: api.TaskStatus{State: api.TaskStateFailed},
}
require.NoError(t, s.Update(func(tx store.Tx) error {
return store.CreateTask(tx, task)
}))
}
require.NoError(t, s.Update(func(tx store.Tx) error {
if err := store.CreateService(tx, &api.Service{ID: "existing-service"}); err != nil {
return err
}
if err := store.CreateTask(tx, &api.Task{ID: "retained", ServiceID: "existing-service"}); err != nil {
return err
}
return store.CreateTask(tx, &api.Task{ID: "standalone"})
}))

s.View(func(readTx store.ReadTx) {
require.NoError(t, CheckTasks(context.Background(), s, readTx, unrelatedInitHandler{}, nil))
})
s.View(func(tx store.ReadTx) {
for _, id := range []string{"orphan-1", "orphan-2", "orphan-3"} {
require.Nil(t, store.GetTask(tx, id))
}
require.NotNil(t, store.GetTask(tx, "retained"))
require.NotNil(t, store.GetTask(tx, "standalone"))
})
}