internal/mtprotoedge/rpc_delivery_coordinator_test.go

Rama main · réplica de lectura del repositorio original.

package mtprotoedge

import (
	"context"
	"errors"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

func TestRPCDeliveryCoordinatorWaitsFromClaimedThroughDone(t *testing.T) {
	coordinator := newRPCResultDelivery(1).coordinator
	started := make(chan struct{})
	release := make(chan struct{})
	coordinator.setHook(func() {
		close(started)
		<-release
	})

	claim, err := coordinator.claimReplayDeliveredHook(context.Background(), false)
	if err != nil || claim == nil {
		t.Fatalf("initial hook claim = %p, err=%v", claim, err)
	}
	if got := coordinator.hookState(); got != rpcResultDeliveryHookClaimed {
		t.Fatalf("hook state after claim = %d, want claimed", got)
	}

	type claimResult struct {
		claim *rpcResultDeliveryHookClaim
		err   error
	}
	waited := make(chan claimResult, 1)
	go func() {
		other, waitErr := coordinator.claimReplayDeliveredHook(context.Background(), false)
		waited <- claimResult{claim: other, err: waitErr}
	}()
	select {
	case result := <-waited:
		t.Fatalf("second replay passed a claimed hook: claim=%p err=%v", result.claim, result.err)
	case <-time.After(20 * time.Millisecond):
	}

	go claim.run()
	select {
	case <-started:
	case <-time.After(time.Second):
		t.Fatal("claimed hook did not enter in-progress")
	}
	if got := coordinator.hookState(); got != rpcResultDeliveryHookInProgress {
		t.Fatalf("hook state while callback blocked = %d, want in-progress", got)
	}
	select {
	case result := <-waited:
		t.Fatalf("second replay passed an in-progress hook: claim=%p err=%v", result.claim, result.err)
	case <-time.After(20 * time.Millisecond):
	}

	close(release)
	select {
	case result := <-waited:
		if result.err != nil || result.claim != nil {
			t.Fatalf("wait after done = claim:%p err:%v", result.claim, result.err)
		}
	case <-time.After(time.Second):
		t.Fatal("completion did not notify the waiting replay")
	}
	if got := coordinator.hookState(); got != rpcResultDeliveryHookDone {
		t.Fatalf("terminal hook state = %d, want done", got)
	}
}

func TestRPCDeliveryCoordinatorAbandonedClaimCannotRunAfterReacquire(t *testing.T) {
	coordinator := newRPCResultDelivery(1).coordinator
	var calls atomic.Int32
	coordinator.setHook(func() { calls.Add(1) })

	stale, err := coordinator.claimReplayDeliveredHook(context.Background(), false)
	if err != nil || stale == nil || !stale.abandon() {
		t.Fatalf("abandon initial claim = %p err=%v", stale, err)
	}
	current, err := coordinator.claimReplayDeliveredHook(context.Background(), false)
	if err != nil || current == nil {
		t.Fatalf("reacquire hook = %p err=%v", current, err)
	}
	stale.run()
	if got := calls.Load(); got != 0 {
		t.Fatalf("stale claim calls = %d, want 0", got)
	}
	if got := coordinator.hookState(); got != rpcResultDeliveryHookClaimed {
		t.Fatalf("stale claim changed current state to %d", got)
	}
	current.run()
	if got := calls.Load(); got != 1 {
		t.Fatalf("current claim calls = %d, want 1", got)
	}
	if got := coordinator.hookState(); got != rpcResultDeliveryHookDone {
		t.Fatalf("terminal hook state = %d, want done", got)
	}
}

func TestBoundedRPCReplayRestoreDoesNotRetainWorkerOrBarrier(t *testing.T) {
	s := New(Options{})
	c := &Conn{metrics: NopMetrics{}}
	finishBarrier := c.beginRPCReplayRestore()

	coordinator := newRPCResultDelivery(1).coordinator
	var hookCalls atomic.Int32
	coordinator.setHook(func() { hookCalls.Add(1) })
	claim, err := coordinator.claimReplayDeliveredHook(context.Background(), false)
	if err != nil || claim == nil {
		t.Fatalf("claim hook = %p err=%v", claim, err)
	}

	replacementStarted := make(chan struct{})
	releaseReplacement := make(chan struct{})
	t.Cleanup(func() {
		select {
		case <-releaseReplacement:
		default:
			close(releaseReplacement)
		}
	})
	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
	defer cancel()
	started := time.Now()
	restoreResult := make(chan error, 1)
	go func() {
		restoreResult <- s.runBoundedRPCReplayRestore(ctx, c, "non-cooperative replacement", claim, func() error {
			close(replacementStarted)
			<-releaseReplacement // Deliberately ignores ctx.
			return nil
		})
	}()
	select {
	case <-replacementStarted:
	case <-time.After(time.Second):
		t.Fatal("replacement callback never started")
	}
	select {
	case err = <-restoreResult:
	case <-time.After(time.Second):
		t.Fatal("bounded restore retained its caller past the watchdog")
	}
	if !errors.Is(err, context.DeadlineExceeded) {
		t.Fatalf("bounded restore error = %v, want deadline", err)
	}
	if elapsed := time.Since(started); elapsed > 500*time.Millisecond {
		t.Fatalf("non-cooperative restore retained caller for %v", elapsed)
	}
	finishBarrier()
	if !c.isRetired() {
		t.Fatal("timed-out restore did not fence its physical generation")
	}
	c.rpcMu.Lock()
	pendingBarriers := c.rpcReplayRestores
	c.rpcMu.Unlock()
	if pendingBarriers != 0 {
		t.Fatalf("timed-out restore retained %d scheduler barriers", pendingBarriers)
	}
	if got := coordinator.hookState(); got != rpcResultDeliveryHookPending {
		t.Fatalf("not-yet-started timed-out hook state = %d, want pending", got)
	}

	// A replacement replay can now acquire the logical hook. The old blocked
	// goroutine holds a stale token and must not execute it when it eventually
	// returns.
	retry, err := coordinator.claimReplayDeliveredHook(context.Background(), false)
	if err != nil || retry == nil {
		t.Fatalf("retry hook claim = %p err=%v", retry, err)
	}
	retry.run()
	close(releaseReplacement)
	deadline := time.Now().Add(time.Second)
	for len(rpcReplayRestoreSlots) != 0 && time.Now().Before(deadline) {
		time.Sleep(time.Millisecond)
	}
	if got := hookCalls.Load(); got != 1 {
		t.Fatalf("logical hook calls after stale restore return = %d, want 1", got)
	}
}

func TestBoundedRPCReplayRestoreKeepsInProgressHookAtMostOnce(t *testing.T) {
	s := New(Options{})
	c := &Conn{metrics: NopMetrics{}}
	finishBarrier := c.beginRPCReplayRestore()

	coordinator := newRPCResultDelivery(1).coordinator
	hookStarted := make(chan struct{})
	releaseHook := make(chan struct{})
	t.Cleanup(func() {
		select {
		case <-releaseHook:
		default:
			close(releaseHook)
		}
	})
	coordinator.setHook(func() {
		close(hookStarted)
		<-releaseHook // Deliberately ignores the restore deadline.
	})
	claim, err := coordinator.claimReplayDeliveredHook(context.Background(), false)
	if err != nil || claim == nil {
		t.Fatalf("claim hook = %p err=%v", claim, err)
	}
	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
	defer cancel()
	restoreResult := make(chan error, 1)
	go func() {
		restoreResult <- s.runBoundedRPCReplayRestore(ctx, c, "non-cooperative logical hook", claim, nil)
	}()
	select {
	case <-hookStarted:
	case <-time.After(time.Second):
		t.Fatal("logical hook never entered in-progress")
	}
	select {
	case err = <-restoreResult:
	case <-time.After(time.Second):
		t.Fatal("non-cooperative logical hook retained its caller past the watchdog")
	}
	if !errors.Is(err, context.DeadlineExceeded) {
		t.Fatalf("bounded logical-hook error = %v, want deadline", err)
	}
	if got := coordinator.hookState(); got != rpcResultDeliveryHookInProgress {
		t.Fatalf("timed-out logical hook state = %d, want in-progress", got)
	}
	finishBarrier()
	if !c.isRetired() {
		t.Fatal("timed-out logical hook did not fence its physical generation")
	}
	c.rpcMu.Lock()
	pendingBarriers := c.rpcReplayRestores
	c.rpcMu.Unlock()
	if pendingBarriers != 0 {
		t.Fatalf("timed-out logical hook retained %d scheduler barriers", pendingBarriers)
	}

	waitCtx, waitCancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
	defer waitCancel()
	if retry, retryErr := coordinator.claimReplayDeliveredHook(waitCtx, false); retry != nil ||
		!errors.Is(retryErr, context.DeadlineExceeded) {
		t.Fatalf("retry during in-progress hook = claim:%p err:%v", retry, retryErr)
	}
	close(releaseHook)
	deadline := time.Now().Add(time.Second)
	for coordinator.hookState() != rpcResultDeliveryHookDone && time.Now().Before(deadline) {
		time.Sleep(time.Millisecond)
	}
	if got := coordinator.hookState(); got != rpcResultDeliveryHookDone {
		t.Fatalf("released logical hook state = %d, want done", got)
	}
}

func TestCachedReplacementReplayBarrierWaitsForLogicalHookDone(t *testing.T) {
	s := New(Options{})
	const reqMsgID = int64(71001)
	encoded := encodedRPCResultForPriorityTest(reqMsgID, 0)
	encoded.delivery = newRPCResultDelivery(reqMsgID)

	var order atomic.Int32
	hookStarted := make(chan struct{})
	releaseHook := make(chan struct{})
	t.Cleanup(func() {
		select {
		case <-releaseHook:
		default:
			close(releaseHook)
		}
	})
	encoded.setDeliveryHook(func() {
		close(hookStarted)
		<-releaseHook
		if !order.CompareAndSwap(1, 2) {
			panic("logical hook did not follow first replacement restore")
		}
	})

	firstConn := newOutboundTestConn(t, &collectingSessionTransport{}, newOutboundTrackedBudget(1<<20))
	firstResult := make(chan error, 1)
	go func() {
		firstResult <- s.sendReplayedRPCResultWithHook(context.Background(), firstConn, encoded, func() error {
			if !order.CompareAndSwap(0, 1) {
				return errors.New("first replacement restore ran out of order")
			}
			return nil
		})
	}()
	select {
	case <-hookStarted:
	case <-time.After(time.Second):
		t.Fatal("first replacement did not enter logical hook")
	}
	if got := encoded.delivery.coordinator.hookState(); got != rpcResultDeliveryHookInProgress {
		t.Fatalf("shared hook state = %d, want in-progress", got)
	}

	secondTransport := &collectingSessionTransport{}
	secondConn := newOutboundTestConn(t, secondTransport, newOutboundTrackedBudget(1<<20))
	secondReplacement := make(chan struct{})
	secondResult := make(chan error, 1)
	go func() {
		secondResult <- s.sendReplayedRPCResultWithHook(context.Background(), secondConn, encoded, func() error {
			if !order.CompareAndSwap(2, 3) {
				return errors.New("second replacement restore passed logical hook completion")
			}
			close(secondReplacement)
			return nil
		})
	}()
	deadline := time.Now().Add(time.Second)
	for len(secondTransport.snapshot()) == 0 && time.Now().Before(deadline) {
		time.Sleep(time.Millisecond)
	}
	if len(secondTransport.snapshot()) == 0 {
		t.Fatal("second replacement did not physically deliver cached result")
	}
	secondConn.rpcMu.Lock()
	secondBarriers := secondConn.rpcReplayRestores
	secondConn.rpcMu.Unlock()
	if secondBarriers != 1 {
		t.Fatalf("second replacement barriers while hook in progress = %d, want 1", secondBarriers)
	}
	select {
	case err := <-secondResult:
		t.Fatalf("second replacement returned before hook done: %v", err)
	case <-time.After(20 * time.Millisecond):
	}
	select {
	case <-secondReplacement:
		t.Fatal("second replacement metadata ran before shared hook done")
	default:
	}

	close(releaseHook)
	for name, result := range map[string]<-chan error{
		"first": firstResult, "second": secondResult,
	} {
		select {
		case err := <-result:
			if err != nil {
				t.Fatalf("%s replacement replay: %v", name, err)
			}
		case <-time.After(time.Second):
			t.Fatalf("%s replacement replay did not finish", name)
		}
	}
	if got := order.Load(); got != 3 {
		t.Fatalf("replacement/logical terminal order = %d, want 3", got)
	}
	secondConn.rpcMu.Lock()
	secondBarriers = secondConn.rpcReplayRestores
	secondConn.rpcMu.Unlock()
	if secondBarriers != 0 {
		t.Fatalf("second replacement retained %d barriers after hook done", secondBarriers)
	}
}

func TestRPCRewrapWatchdogCoversCommittedRestoreWithoutBlockingTimerOrWorker(t *testing.T) {
	s := New(Options{})
	c := &Conn{metrics: NopMetrics{}}
	const reqMsgID = int64(72001)
	encoded := encodedRPCResultForPriorityTest(reqMsgID, 0)
	encoded.delivery = newRPCResultDelivery(reqMsgID)
	var logicalCalls atomic.Int32
	encoded.setDeliveryHook(func() { logicalCalls.Add(1) })

	replacementStarted := make(chan struct{})
	releaseReplacement := make(chan struct{})
	t.Cleanup(func() {
		select {
		case <-releaseReplacement:
		default:
			close(releaseReplacement)
		}
	})
	alias := &rpcRewrapAlias{
		conn: c, newReqID: reqMsgID, method: "help.getConfig",
		afterSuccessfulDelivery: func() error {
			close(replacementStarted)
			<-releaseReplacement // Deliberately ignores every deadline.
			return nil
		},
	}
	alias.executionOK.Store(true)
	alias.beginReplayRestore()

	var (
		once sync.Once
		jobs chan rpcRewrapDeliveryJob
	)
	workerDone := make(chan error, 1)
	watchdogDone := make(chan struct{})
	job := rpcRewrapDeliveryJob{
		deadline: time.Now().Add(80 * time.Millisecond),
		run: func(control *rpcRewrapDeliveryControl, _ time.Time) {
			if !control.commit() {
				workerDone <- errors.New("commit lost before restore")
				return
			}
			restoreCtx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
			defer cancel()
			workerDone <- s.completeDeliveredRPCRewrapResult(
				restoreCtx, alias, encoded, "committed restore watchdog regression",
			)
		},
		fail: func(error) {
			c.fenceUndeliveredRPCResult()
			alias.releaseReplayRestoreBarrier()
			close(watchdogDone)
		},
	}
	if !scheduleRPCRewrapJob(job, &once, &jobs, 1, 2) {
		t.Fatal("schedule committed restore job")
	}
	select {
	case <-replacementStarted:
	case <-time.After(time.Second):
		t.Fatal("committed replacement restore never started")
	}
	select {
	case <-watchdogDone:
	case <-time.After(500 * time.Millisecond):
		t.Fatal("committed-state watchdog did not return promptly")
	}
	if !c.isRetired() {
		t.Fatal("committed-state watchdog did not fence connection")
	}
	c.rpcMu.Lock()
	pendingBarriers := c.rpcReplayRestores
	c.rpcMu.Unlock()
	if pendingBarriers != 0 {
		t.Fatalf("watchdog retained %d scheduler barriers", pendingBarriers)
	}
	select {
	case err := <-workerDone:
		if !errors.Is(err, context.DeadlineExceeded) {
			t.Fatalf("bounded committed restore error = %v, want deadline", err)
		}
	case <-time.After(time.Second):
		t.Fatal("non-cooperative restore permanently retained rewrap worker")
	}

	followingRan := make(chan struct{})
	if !scheduleRPCRewrapJob(rpcRewrapDeliveryJob{
		deadline: time.Now().Add(time.Second),
		run:      func(*rpcRewrapDeliveryControl, time.Time) { close(followingRan) },
	}, &once, &jobs, 1, 2) {
		t.Fatal("schedule following rewrap job")
	}
	select {
	case <-followingRan:
	case <-time.After(time.Second):
		t.Fatal("rewrap worker did not accept following job after restore timeout")
	}
	if got := encoded.delivery.coordinator.hookState(); got != rpcResultDeliveryHookPending {
		t.Fatalf("timed-out pre-hook restore state = %d, want pending", got)
	}
	if got := logicalCalls.Load(); got != 0 {
		t.Fatalf("logical hook ran behind non-cooperative replacement = %d", got)
	}
	close(releaseReplacement)
	deadline := time.Now().Add(time.Second)
	for len(rpcReplayRestoreSlots) != 0 && time.Now().Before(deadline) {
		time.Sleep(time.Millisecond)
	}
}