diff --git a/manager/orchestrator/taskinit/init.go b/manager/orchestrator/taskinit/init.go index 34331e6548..224b523f3e 100644 --- a/manager/orchestrator/taskinit/init.go +++ b/manager/orchestrator/taskinit/init.go @@ -25,26 +25,33 @@ type InitHandler interface { // not have finished processing their updates and left them in an inconsistent state. func CheckTasks(ctx context.Context, s *store.MemoryStore, readTx store.ReadTx, initHandler InitHandler, startSupervisor restart.SupervisorInterface) error { instances := make(map[orchestrator.SlotTuple][]*api.Task) - err := s.Batch(func(batch *store.Batch) error { - tasks, err := store.FindTasks(readTx, store.All) - if err != nil { - return err + tasks, err := store.FindTasks(readTx, store.All) + if err != nil { + return err + } + + // Delete orphaned tasks in separate transactions. Their full task objects + // are included in the replicated delete actions, so several large tasks can + // exceed the Raft transaction limit when removed in a single batch. + for _, t := range tasks { + if t.ServiceID == "" || store.GetService(readTx, t.ServiceID) != nil { + continue + } + if err := s.Update(func(tx store.Tx) error { + return store.DeleteTask(tx, t.ID) + }); err != nil { + log.G(ctx).WithError(err).WithField("task.id", t.ID).Error("failed to delete task") } + } + + err = s.Batch(func(batch *store.Batch) error { for _, t := range tasks { if t.ServiceID == "" { continue } - // TODO(aluzzardi): We should NOT retrieve the service here. service := store.GetService(readTx, t.ServiceID) if service == nil { - // Service was deleted - err := batch.Update(func(tx store.Tx) error { - return store.DeleteTask(tx, t.ID) - }) - if err != nil { - log.G(ctx).WithError(err).Error("failed to delete task") - } continue } if !initHandler.IsRelatedService(service) { diff --git a/manager/orchestrator/taskinit/init_test.go b/manager/orchestrator/taskinit/init_test.go new file mode 100644 index 0000000000..f25dbf3b8b --- /dev/null +++ b/manager/orchestrator/taskinit/init_test.go @@ -0,0 +1,80 @@ +package taskinit + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/moby/swarmkit/v2/api" + "github.com/moby/swarmkit/v2/manager/orchestrator" + "github.com/moby/swarmkit/v2/manager/state/store" + "github.com/moby/swarmkit/v2/manager/state/testutils" + "github.com/stretchr/testify/require" +) + +// sizeLimitedProposer enforces the serialized request limit used by Raft. +type sizeLimitedProposer struct { + testutils.MockProposer +} + +func (p *sizeLimitedProposer) ProposeValue(ctx context.Context, actions []api.StoreAction, cb func()) error { + request := api.InternalRaftRequest{Action: actions} + if size := request.Size(); size > store.MaxTransactionBytes { + return fmt.Errorf("raft request size %d exceeds limit %d", size, int(store.MaxTransactionBytes)) + } + return p.MockProposer.ProposeValue(ctx, actions, cb) +} + +// unrelatedInitHandler leaves tasks belonging to another orchestrator alone. +type unrelatedInitHandler struct{} + +func (unrelatedInitHandler) IsRelatedService(*api.Service) bool { return false } +func (unrelatedInitHandler) FixTask(context.Context, *store.Batch, *api.Task) {} +func (unrelatedInitHandler) SlotTuple(*api.Task) orchestrator.SlotTuple { + return orchestrator.SlotTuple{} +} + +func TestCheckTasksLargeOrphanTasks(t *testing.T) { + s := store.NewMemoryStore(&sizeLimitedProposer{}) + defer s.Close() + + // Each task can be created and deleted independently, but two deletion + // actions together exceed Raft's limit before Batch checks its threshold. + for _, id := range []string{"orphan-1", "orphan-2", "orphan-3"} { + task := &api.Task{ + ID: id, + ServiceID: "deleted-service", + Spec: api.TaskSpec{Runtime: &api.TaskSpec_Container{ + Container: &api.ContainerSpec{ + Image: "busybox:latest", + Env: []string{"PAYLOAD=" + strings.Repeat("x", 850*1024)}, + }, + }}, + Status: api.TaskStatus{State: api.TaskStateFailed}, + } + require.NoError(t, s.Update(func(tx store.Tx) error { + return store.CreateTask(tx, task) + })) + } + require.NoError(t, s.Update(func(tx store.Tx) error { + if err := store.CreateService(tx, &api.Service{ID: "existing-service"}); err != nil { + return err + } + if err := store.CreateTask(tx, &api.Task{ID: "retained", ServiceID: "existing-service"}); err != nil { + return err + } + return store.CreateTask(tx, &api.Task{ID: "standalone"}) + })) + + s.View(func(readTx store.ReadTx) { + require.NoError(t, CheckTasks(context.Background(), s, readTx, unrelatedInitHandler{}, nil)) + }) + s.View(func(tx store.ReadTx) { + for _, id := range []string{"orphan-1", "orphan-2", "orphan-3"} { + require.Nil(t, store.GetTask(tx, id)) + } + require.NotNil(t, store.GetTask(tx, "retained")) + require.NotNil(t, store.GetTask(tx, "standalone")) + }) +}