Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 35 additions & 6 deletions eth/payment_observer.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,16 +83,30 @@ type ObserverStatus struct {
ObservedAt, ContactAt, HeadProgressAt, L1ProgressAt time.Time
Healthy, RecipientActive bool
RecipientTracked, RecipientFailed bool
RecipientRejection string
PayerEntries, PayerReads, PayerFailures int
}

func (o *PaymentObserver) Status() ObserverStatus {
o.mu.RLock()
defer o.mu.RUnlock()
healthy := o.validate(o.current) == nil
reason := o.viewRejection(o.current, o.l1Progress)
healthy := reason == ""
tracked, failed := o.recipient != (common.Address{}), o.current.recipientErr != nil
if healthy {
switch {
case !tracked:
reason = "recipient_untracked"
case failed:
reason = "recipient_activity_check_failed"
case !o.current.recipientActive:
reason = "recipient_inactive"
}
}
status := ObserverStatus{
ObservedAt: o.current.observedAt, ContactAt: o.contact, HeadProgressAt: o.progress, L1ProgressAt: o.l1Progress,
RecipientActive: healthy && o.recipient != (common.Address{}) && o.current.recipientErr == nil && o.current.recipientActive, PayerEntries: len(o.payers), PayerReads: o.activeReads, Healthy: healthy, RecipientTracked: o.recipient != (common.Address{}), RecipientFailed: o.current.recipientErr != nil,
RecipientActive: reason == "", RecipientTracked: tracked, RecipientFailed: failed, RecipientRejection: reason,
PayerEntries: len(o.payers), PayerReads: o.activeReads, Healthy: healthy,
}
for _, entry := range o.payers {
if !entry.valid && entry.failure != nil && entry.failedAt == o.current.BlockHash && entry.failedGeneration == o.current.generation {
Expand Down Expand Up @@ -140,15 +154,30 @@ func (s ChainSnapshot) Validate() error {
}

func (o *PaymentObserver) validate(s ChainSnapshot) error {
if (o.lifecycle != nil && o.lifecycle.Err() != nil) || s.generation != o.generation || o.expired(s, o.l1Progress) {
if o.viewRejection(s, o.l1Progress) != "" {
return ErrObservationUnavailable
}
return nil
}

func (o *PaymentObserver) expired(s ChainSnapshot, l1Progress time.Time) bool {
func (o *PaymentObserver) viewRejection(s ChainSnapshot, l1Progress time.Time) string {
now := o.now()
return s.observedAt.IsZero() || now.Sub(s.observedAt) >= o.maxAge || now.Sub(s.blockTime) >= o.maxAge || now.Sub(l1Progress) >= o.maxAge
switch {
case o.lifecycle != nil && o.lifecycle.Err() != nil:
return "chain_observer_stopped"
case s.generation != o.generation:
return "chain_view_invalidated"
case s.observedAt.IsZero():
return "chain_view_missing"
case now.Sub(s.observedAt) >= o.maxAge:
return "chain_view_stale"
case now.Sub(s.blockTime) >= o.maxAge:
return "chain_head_stale"
case now.Sub(l1Progress) >= o.maxAge:
return "chain_l1_progress_stale"
default:
return ""
}
}

// Only Run calls refresh.
Expand Down Expand Up @@ -246,7 +275,7 @@ func (o *PaymentObserver) refresh(ctx context.Context) error {
l1Progress, highWater = at, s.Block
}
s.observer, s.generation, s.observedAt, s.blockTime = o, o.generation, at, blockTime
if (o.lifecycle != nil && o.lifecycle.Err() != nil) || o.expired(s, l1Progress) {
if o.viewRejection(s, l1Progress) != "" {
return ErrObservationUnavailable
}
for key, entry := range o.payers {
Expand Down
10 changes: 10 additions & 0 deletions eth/payment_observer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ func TestObserverLocalImmutableSnapshot(t *testing.T) {
f, o, _ := newTestObserver(t)
_, err := o.Snapshot(t.Context())
require.ErrorIs(t, err, ErrObservationUnavailable)
require.Equal(t, "chain_view_missing", o.Status().RecipientRejection)
f.Update(func(s *test.ChainState) { s.CurrentRound = 6 })
require.NoError(t, o.refresh(t.Context()))
before := f.Stats()
Expand Down Expand Up @@ -95,13 +96,15 @@ func TestObserverContactDoesNotRenewOldHead(t *testing.T) {
original := f.State()
f.Update(func(s *test.ChainState) { s.Number-- })
require.Error(t, o.refresh(t.Context()))
require.Equal(t, "chain_view_invalidated", o.Status().RecipientRejection)
f.Update(func(s *test.ChainState) { *s = original })
require.NoError(t, o.refresh(t.Context()))
require.True(t, o.Status().Healthy)
require.ErrorIs(t, old.Validate(), ErrObservationUnavailable)
require.Equal(t, initial.ObservedAt, o.Status().ObservedAt)
*now = now.Add(20 * time.Second)
require.Error(t, o.refresh(t.Context()))
require.Equal(t, "chain_view_stale", o.Status().RecipientRejection)
_, err = o.Snapshot(t.Context())
require.ErrorIs(t, err, ErrObservationUnavailable)
advanceObserver(f, *now)
Expand Down Expand Up @@ -185,6 +188,7 @@ func TestObserverL1ProgressDeadline(t *testing.T) {
require.Equal(t, progress, o.Status().L1ProgressAt)
}
require.False(t, o.Status().Healthy)
require.Equal(t, "chain_l1_progress_stale", o.Status().RecipientRejection)
advanceObserver(f, *now)
f.Update(func(s *test.ChainState) { s.L1Block = 51 })
require.NoError(t, o.refresh(t.Context()))
Expand Down Expand Up @@ -345,6 +349,11 @@ func TestObserverRecipientActivity(t *testing.T) {
active, err := c.IsActiveAt(t.Context(), o.recipient, snapshot)
require.NoError(t, err)
require.Equal(t, step.active, active)
if step.active {
require.Empty(t, o.Status().RecipientRejection)
} else {
require.Equal(t, "recipient_inactive", o.Status().RecipientRejection)
}
_, err = c.IsActiveAt(t.Context(), common.HexToAddress("0x9999"), snapshot)
require.ErrorIs(t, err, ErrObservationUnavailable)
unowned := snapshot
Expand All @@ -369,6 +378,7 @@ func TestObserverRecipientActivity(t *testing.T) {
f.Update(func(s *test.ChainState) { s.Fail = "BondingManager.isActiveTranscoder" })
require.NoError(t, o.refresh(t.Context()), "a recipient failure must not reject unrelated chain state")
require.True(t, o.Status().Healthy)
require.Equal(t, "recipient_activity_check_failed", o.Status().RecipientRejection)
_, err = c.PayerFunds(t.Context(), common.HexToAddress("0x2000"), o.recipient)
require.NoError(t, err)
_, err = c.IsActive(t.Context(), o.recipient)
Expand Down
49 changes: 41 additions & 8 deletions orchestrator/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,21 +127,54 @@ type paidReadiness struct {
snapshot eth.ChainSnapshot
interval time.Duration
observer *eth.PaymentObserver
probeMu sync.Mutex
lastRejection string
}

func (p *paidReadiness) ready(registry *Registry) bool {
p.probeMu.Lock()
defer p.probeMu.Unlock()

p.mu.Lock()
limit := max(time.Minute, 3*p.interval)
chainReady := p.chain && p.active
if p.observer != nil {
chainReady = p.observer.Status().RecipientActive
var reason string
switch {
case p.observer != nil:
reason = p.observer.Status().RecipientRejection
case !p.chain:
reason = "chain_check_failed"
case !p.active:
reason = "recipient_inactive"
}

switch {
case reason != "":
case p.completed.IsZero():
reason = "payment_check_pending"
case !p.storage:
reason = "payment_storage_unavailable"
case time.Since(p.completed) > limit:
reason = "payment_check_stale"
case p.workerCompleted.IsZero():
reason = "redemption_worker_pending"
case time.Since(p.workerCompleted) > limit:
reason = "redemption_worker_stale"
}
ok := chainReady && p.storage && !p.completed.IsZero() && time.Since(p.completed) <= limit && !p.workerCompleted.IsZero() && time.Since(p.workerCompleted) <= limit
p.mu.Unlock()
registry.mu.Lock()
fresh := registry.rateUntil.IsZero() || time.Now().Before(registry.rateUntil)
registry.mu.Unlock()
return ok && fresh

if reason == "" {
registry.mu.Lock()
if !registry.rateUntil.IsZero() && !time.Now().Before(registry.rateUntil) {
reason = "price_feed_stale"
}
registry.mu.Unlock()
}

if reason != "" && reason != p.lastRejection && registry.logger != nil {
registry.logger.Warn("payment readiness rejected", "event", "payment_readiness", "operation", "readyz", "outcome", "unavailable", "reason", reason)
}
p.lastRejection = reason
return reason == ""
}

func (p *paidReadiness) check(ctx context.Context, store *RedeemerDB, chain eth.PaymentChain, engine *PaymentEngine, metrics *observability.Metrics, logger *slog.Logger) {
Expand Down
31 changes: 31 additions & 0 deletions orchestrator/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,10 @@ func TestPaidReadinessUsesCurrentPublication(t *testing.T) {
o := chain.Observer
snapshot, err := chain.Snapshot(t.Context())
require.NoError(t, err)
var logs bytes.Buffer
p := &paidReadiness{chain: true, active: true, storage: true, completed: time.Now(), workerCompleted: time.Now(), observer: o, snapshot: snapshot}
registry := NewRegistry("", "http://localhost", time.Second, time.Minute)
registry.logger = slog.New(slog.NewTextHandler(&logs, nil))
f.Update(func(s *test.ChainState) {
s.L1Block++
s.Hash = common.HexToHash("0xeeee")
Expand Down Expand Up @@ -115,6 +117,35 @@ func TestPaidReadinessUsesCurrentPublication(t *testing.T) {
p.check(t.Context(), store, direct, &PaymentEngine{recipient: ticket.Recipient}, observability.New("orchestrator"), slog.New(slog.NewTextHandler(io.Discard, nil)))
require.True(t, p.storage, "chain unavailability must not mark healthy storage unavailable")
require.False(t, p.ready(registry))
require.Contains(t, logs.String(), "reason=chain_observer_stopped")
}

func TestPaidReadinessLogsRejectionTransitions(t *testing.T) {
var logs bytes.Buffer
p := &paidReadiness{chain: true, active: true}
registry := NewRegistry("", "http://localhost", time.Second, time.Minute)
registry.logger = slog.New(slog.NewJSONHandler(&logs, nil))
require.False(t, p.ready(registry))
p.completed, p.storage = time.Now(), true
require.False(t, p.ready(registry))
p.workerCompleted = time.Now()
for _, check := range []struct{ active, storage bool }{
{false, true}, // Reject, then suppress the repeat.
{false, true},
{true, true}, // Recovery clears suppression.
{false, true}, // The same failure is logged again.
{true, false}, // A different failure is logged without recovery.
} {
p.active, p.storage = check.active, check.storage
require.Equal(t, check.active && check.storage, p.ready(registry))
}
dec := json.NewDecoder(&logs)
for _, expected := range []string{"payment_check_pending", "redemption_worker_pending", "recipient_inactive", "recipient_inactive", "payment_storage_unavailable"} {
var event struct{ Reason string }
require.NoError(t, dec.Decode(&event))
require.Equal(t, expected, event.Reason)
}
require.ErrorIs(t, dec.Decode(new(any)), io.EOF)
}

func TestRunnerCapacityOutcomes(t *testing.T) {
Expand Down
Loading