internal/mtprotoedge/rpc_result_delivery_test.go

Rama main · réplica de lectura del repositorio original.

package mtprotoedge

import (
	"context"
	"errors"
	"io"
	"sync"
	"testing"
	"time"

	"github.com/iamxvbaba/td/bin"
	"github.com/iamxvbaba/td/clock"
	"github.com/iamxvbaba/td/proto"
	"github.com/iamxvbaba/td/tg"
	"go.uber.org/zap/zaptest"

	appupdates "telesrv/internal/app/updates"
	"telesrv/internal/domain"
	rpchandler "telesrv/internal/rpc"
	"telesrv/internal/store/memory"
)

type blockingCloseRPCResultTransport struct {
	started chan struct{}
	release chan struct{}
	once    sync.Once
}

type identifiedRouterHandler struct {
	router *rpchandler.Router
	userID int64
}

type failingRPCResultEncoder struct {
	err error
}

func (e failingRPCResultEncoder) Encode(*bin.Buffer) error { return e.err }

type encodeFailingIdentifiedRouterHandler struct {
	identifiedRouterHandler
	err error
}

func (h encodeFailingIdentifiedRouterHandler) Dispatch(ctx context.Context, authKeyID [8]byte, sessionID int64, b *bin.Buffer) (bin.Encoder, error) {
	result, err := h.identifiedRouterHandler.Dispatch(ctx, authKeyID, sessionID, b)
	if err != nil {
		return nil, err
	}
	if result == nil {
		return nil, nil
	}
	return failingRPCResultEncoder{err: h.err}, nil
}

func (h encodeFailingIdentifiedRouterHandler) DispatchWithMethod(ctx context.Context, authKeyID [8]byte, sessionID int64, b *bin.Buffer) (bin.Encoder, string, error) {
	result, method, err := h.identifiedRouterHandler.DispatchWithMethod(ctx, authKeyID, sessionID, b)
	if err != nil {
		return nil, method, err
	}
	if result == nil {
		return nil, method, nil
	}
	return failingRPCResultEncoder{err: h.err}, method, nil
}

func (h identifiedRouterHandler) requestContext(ctx context.Context) context.Context {
	return rpchandler.WithClientInfo(rpchandler.WithUserID(ctx, h.userID), rpchandler.ClientInfo{Type: rpchandler.ClientTypeTDesktop})
}

func (h identifiedRouterHandler) Dispatch(ctx context.Context, authKeyID [8]byte, sessionID int64, b *bin.Buffer) (bin.Encoder, error) {
	return h.router.Dispatch(h.requestContext(ctx), authKeyID, sessionID, b)
}

func (h identifiedRouterHandler) DispatchWithMethod(ctx context.Context, authKeyID [8]byte, sessionID int64, b *bin.Buffer) (bin.Encoder, string, error) {
	return h.router.DispatchWithMethod(h.requestContext(ctx), authKeyID, sessionID, b)
}

func (h identifiedRouterHandler) NegotiatedLayer(authKeyID [8]byte, sessionID int64) (int, bool) {
	return h.router.NegotiatedLayer(authKeyID, sessionID)
}

func newBlockingCloseRPCResultTransport() *blockingCloseRPCResultTransport {
	return &blockingCloseRPCResultTransport{started: make(chan struct{}), release: make(chan struct{})}
}

func (*blockingCloseRPCResultTransport) Send(context.Context, *bin.Buffer) error { return nil }
func (*blockingCloseRPCResultTransport) Recv(context.Context, *bin.Buffer) error { return io.EOF }
func (t *blockingCloseRPCResultTransport) Close() error {
	t.once.Do(func() { close(t.started) })
	<-t.release
	return nil
}

func TestRPCExecutionLedgerPublishesOnlyAfterPhysicalWrite(t *testing.T) {
	tr := newGatedRequiredControlTransport(nil)
	s := New(Options{WriteTimeout: time.Second})
	key := newTestAuthKey(t)
	c := s.newConn(tr, key, 74001, 1)
	t.Cleanup(c.ForceClose)
	reqMsgID := proto.NewMessageIDGen(time.Now).New(proto.MessageFromClient)

	owner, err := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
	if err != nil || owner.state != rpcResultAcquireOwner {
		t.Fatalf("initial flight owner = %+v err=%v", owner, err)
	}
	done := make(chan error, 1)
	go func() {
		done <- s.sendResult(context.Background(), c, reqMsgID, exactTestRPCResult(&tg.Config{ThisDC: 2}))
	}()
	select {
	case <-tr.started:
	case <-time.After(time.Second):
		t.Fatal("rpc_result did not reach physical writer")
	}
	if _, ok := s.rpcResults.Replay(key.ID, c.sessionID, reqMsgID); ok {
		t.Fatal("rpc_result became completed before physical write")
	}
	pending, err := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
	if err != nil || pending.state != rpcResultAcquirePending || pending.waiter == nil {
		t.Fatalf("flight while write blocked = %+v err=%v", pending, err)
	}

	tr.unblock()
	select {
	case err := <-done:
		if err != nil {
			t.Fatalf("sendResult after write: %v", err)
		}
	case <-time.After(time.Second):
		t.Fatal("sendResult did not finish")
	}
	if encoded, ok, err := pending.waiter.Wait(context.Background()); err != nil || !ok || encoded == nil {
		t.Fatalf("pending waiter after write = encoded:%p ok:%v err:%v", encoded, ok, err)
	}
	completed, err := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
	if err != nil || completed.state != rpcResultAcquireCompleted || completed.encoded == nil {
		t.Fatalf("flight after write = %+v err=%v", completed, err)
	}
}

func TestRPCResultPostResponseHookWaitsForPhysicalWrite(t *testing.T) {
	tr := newGatedRequiredControlTransport(nil)
	s := New(Options{WriteTimeout: time.Second})
	key := newTestAuthKey(t)
	c := s.newConn(tr, key, 74006, 1)
	legacyCanonicalTestConn(t, c)
	t.Cleanup(c.ForceClose)
	reqMsgID := proto.NewMessageIDGen(time.Now).New(proto.MessageFromClient)
	claim, err := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
	if err != nil || claim.state != rpcResultAcquireOwner {
		t.Fatalf("initial flight owner = %+v err=%v", claim, err)
	}

	delivered := make(chan struct{})
	if err := s.publishRPCResult(c, reqMsgID, "updates.getState", claim.owner, exactTestRPCResult(&tg.UpdatesState{}), func() {
		close(delivered)
	}); err != nil {
		t.Fatalf("publish rpc_result: %v", err)
	}
	select {
	case <-tr.started:
	case <-time.After(time.Second):
		t.Fatal("rpc_result did not reach physical writer")
	}
	select {
	case <-delivered:
		t.Fatal("post-response hook ran while physical writer was blocked")
	case <-time.After(20 * time.Millisecond):
	}

	tr.unblock()
	select {
	case <-delivered:
	case <-time.After(time.Second):
		t.Fatal("post-response hook did not run after physical write")
	}
}

func TestRouterUpdateCursorCommitsOnlyAfterHandleRPCPhysicalWrite(t *testing.T) {
	const userID int64 = 1000000301
	events := memory.NewUpdateEventStore()
	states := memory.NewUpdateStateStore()
	if err := events.Append(context.Background(), userID, domain.UpdateEvent{
		UserID: userID, Type: domain.UpdateEventNewMessage,
		Pts: 1, PtsCount: 1, Date: 1700000301,
		Message: domain.Message{ID: 1, OwnerUserID: userID},
	}); err != nil {
		t.Fatalf("seed update event: %v", err)
	}
	router := rpchandler.New(rpchandler.Config{}, rpchandler.Deps{
		Updates: appupdates.NewService(states, events),
	}, zaptest.NewLogger(t), clock.System)
	handler := identifiedRouterHandler{router: router, userID: userID}
	tr := newGatedRequiredControlTransport(nil)
	s := New(Options{legacyRPC: handler, WriteTimeout: time.Second})
	key := newTestAuthKey(t)
	c := s.newConn(tr, key, 74007, 1)
	t.Cleanup(c.ForceClose)
	reqMsgID := proto.NewMessageIDGen(time.Now).New(proto.MessageFromClient)
	claim, err := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
	if err != nil || claim.state != rpcResultAcquireOwner {
		t.Fatalf("acquire flight = %+v err=%v", claim, err)
	}
	var request bin.Buffer
	if err := (&tg.UpdatesGetStateRequest{}).Encode(&request); err != nil {
		t.Fatalf("encode getState: %v", err)
	}
	handled := make(chan error, 1)
	go func() {
		handled <- s.handleRPC(context.Background(), c, reqMsgID, "updates.getState", &request, claim.owner)
	}()
	select {
	case <-tr.started:
	case <-time.After(time.Second):
		t.Fatal("getState rpc_result did not reach physical writer")
	}
	select {
	case err := <-handled:
		if err != nil {
			t.Fatalf("handleRPC: %v", err)
		}
	case <-time.After(time.Second):
		t.Fatal("handleRPC remained coupled to physical write")
	}
	if state, found, err := states.Get(context.Background(), key.ID, userID); err != nil || found {
		t.Fatalf("confirmed before physical write = %+v/%v err=%v", state, found, err)
	}
	if state, found := states.ObservedClientState(key.ID, userID); found {
		t.Fatalf("observed before physical write = %+v/%v", state, found)
	}

	tr.unblock()
	deadline := time.Now().Add(time.Second)
	for time.Now().Before(deadline) {
		confirmed, found, err := states.Get(context.Background(), key.ID, userID)
		observed, observedFound := states.ObservedClientState(key.ID, userID)
		if err == nil && found && confirmed.Pts == 1 && observedFound && observed.Pts == 1 {
			return
		}
		time.Sleep(time.Millisecond)
	}
	confirmed, found, err := states.Get(context.Background(), key.ID, userID)
	observed, observedFound := states.ObservedClientState(key.ID, userID)
	t.Fatalf("post-write cursor = confirmed:%+v/%v err=%v observed:%+v/%v", confirmed, found, err, observed, observedFound)
}

func TestRouterUpdateCursorDoesNotCommitAfterPhysicalWriteFailure(t *testing.T) {
	const userID int64 = 1000000302
	events := memory.NewUpdateEventStore()
	states := memory.NewUpdateStateStore()
	if err := events.Append(context.Background(), userID, domain.UpdateEvent{
		UserID: userID, Type: domain.UpdateEventNewMessage,
		Pts: 1, PtsCount: 1, Date: 1700000302,
		Message: domain.Message{ID: 1, OwnerUserID: userID},
	}); err != nil {
		t.Fatalf("seed update event: %v", err)
	}
	router := rpchandler.New(rpchandler.Config{}, rpchandler.Deps{
		Updates: appupdates.NewService(states, events),
	}, zaptest.NewLogger(t), clock.System)
	tr := &failAfterTransport{}
	tr.failAt.Store(1)
	s := New(Options{legacyRPC: identifiedRouterHandler{router: router, userID: userID}, WriteTimeout: time.Second})
	key := newTestAuthKey(t)
	c := s.newConn(tr, key, 74008, 1)
	t.Cleanup(c.ForceClose)
	reqMsgID := proto.NewMessageIDGen(time.Now).New(proto.MessageFromClient)
	claim, err := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
	if err != nil || claim.state != rpcResultAcquireOwner {
		t.Fatalf("acquire flight = %+v err=%v", claim, err)
	}
	var request bin.Buffer
	if err := (&tg.UpdatesGetStateRequest{}).Encode(&request); err != nil {
		t.Fatalf("encode getState: %v", err)
	}
	if err := s.handleRPC(context.Background(), c, reqMsgID, "updates.getState", &request, claim.owner); err != nil {
		t.Fatalf("handleRPC admission: %v", err)
	}
	deadline := time.Now().Add(time.Second)
	replayable := false
	for time.Now().Before(deadline) {
		if cached, ok := s.rpcResults.Replay(key.ID, c.sessionID, reqMsgID); ok && cached.deliveryState() == rpcResultDeliveryReplayable {
			replayable = true
			break
		}
		time.Sleep(time.Millisecond)
	}
	if !replayable {
		t.Fatal("failed physical result did not become replayable")
	}
	if state, found, err := states.Get(context.Background(), key.ID, userID); err != nil || found {
		t.Fatalf("failed write advanced confirmed = %+v/%v err=%v", state, found, err)
	}
	if state, found := states.ObservedClientState(key.ID, userID); found {
		t.Fatalf("failed write advanced observed = %+v/%v", state, found)
	}
}

func TestRouterUpdateCursorDoesNotCommitWhenResultEncodingFails(t *testing.T) {
	const userID int64 = 1000000303
	events := memory.NewUpdateEventStore()
	states := memory.NewUpdateStateStore()
	if err := events.Append(context.Background(), userID, domain.UpdateEvent{
		UserID: userID, Type: domain.UpdateEventNewMessage,
		Pts: 1, PtsCount: 1, Date: 1700000303,
		Message: domain.Message{ID: 1, OwnerUserID: userID},
	}); err != nil {
		t.Fatalf("seed update event: %v", err)
	}
	router := rpchandler.New(rpchandler.Config{}, rpchandler.Deps{
		Updates: appupdates.NewService(states, events),
	}, zaptest.NewLogger(t), clock.System)
	handler := encodeFailingIdentifiedRouterHandler{
		identifiedRouterHandler: identifiedRouterHandler{router: router, userID: userID},
		err:                     errors.New("encode result"),
	}
	tr := &collectingSessionTransport{}
	s := New(Options{legacyRPC: handler, WriteTimeout: time.Second})
	key := newTestAuthKey(t)
	c := s.newConn(tr, key, 74009, 1)
	t.Cleanup(c.ForceClose)
	reqMsgID := proto.NewMessageIDGen(time.Now).New(proto.MessageFromClient)
	claim, err := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
	if err != nil || claim.state != rpcResultAcquireOwner {
		t.Fatalf("acquire flight = %+v err=%v", claim, err)
	}
	var request bin.Buffer
	if err := (&tg.UpdatesGetStateRequest{}).Encode(&request); err != nil {
		t.Fatalf("encode getState: %v", err)
	}
	if err := s.handleRPC(context.Background(), c, reqMsgID, "updates.getState", &request, claim.owner); err != nil {
		t.Fatalf("handleRPC encoding fallback: %v", err)
	}
	deadline := time.Now().Add(time.Second)
	for time.Now().Before(deadline) {
		if cached, ok := s.rpcResults.Replay(key.ID, c.sessionID, reqMsgID); ok && cached.deliveryState() == rpcResultDeliveryDelivered {
			break
		}
		time.Sleep(time.Millisecond)
	}
	if len(tr.snapshot()) == 0 {
		t.Fatal("INTERNAL fallback was not physically delivered")
	}
	if state, found, err := states.Get(context.Background(), key.ID, userID); err != nil || found {
		t.Fatalf("encoding failure advanced confirmed = %+v/%v err=%v", state, found, err)
	}
	if state, found := states.ObservedClientState(key.ID, userID); found {
		t.Fatalf("encoding failure advanced observed = %+v/%v", state, found)
	}
}

func TestRPCResultPrewriteFailureFencesConnBeforeCachePublication(t *testing.T) {
	tr := &collectingSessionTransport{}
	s := New(Options{WriteTimeout: 20 * time.Millisecond})
	// No rpc_result can reserve its conservative 3x wire scratch from one byte.
	// The actor therefore fails before touching the socket, which used to leave a
	// live Conn with a prematurely completed cache entry.
	s.outboundScratchPool = newOutboundScratchPool(1)
	key := newTestAuthKey(t)
	c := s.newConn(tr, key, 74002, 1)
	reqMsgID := proto.NewMessageIDGen(time.Now).New(proto.MessageFromClient)
	owner, err := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
	if err != nil || owner.state != rpcResultAcquireOwner {
		t.Fatalf("initial flight owner = %+v err=%v", owner, err)
	}

	err = s.sendResult(context.Background(), c, reqMsgID, exactTestRPCResult(&tg.Config{ThisDC: 2}))
	if err == nil || (!errors.Is(err, context.DeadlineExceeded) &&
		!errors.Is(err, ErrConnClosed) && !errors.Is(err, ErrOutboundTrackedBudget)) {
		t.Fatalf("prewrite sendResult error = %v", err)
	}
	closeDeadline := time.Now().Add(time.Second)
	for !tr.closed.Load() && time.Now().Before(closeDeadline) {
		time.Sleep(time.Millisecond)
	}
	if !c.isRetired() || !tr.closed.Load() || c.isPhysicalTransportCurrentOpen() {
		t.Fatalf("failed delivery did not fence Conn: retired=%v closed=%v current_open=%v", c.isRetired(), tr.closed.Load(), c.isPhysicalTransportCurrentOpen())
	}
	completed, acquireErr := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
	if acquireErr != nil || completed.state != rpcResultAcquireCompleted || completed.encoded == nil {
		t.Fatalf("terminal replay cache = %+v err=%v", completed, acquireErr)
	}
	if got := s.rpcResults.flightLimit.snapshot(); got != 0 {
		t.Fatalf("failed delivery leaked flight slot: %d", got)
	}
	c.Close()
}

func TestRPCResultFailureAfterIntentionalTerminalDoesNotCloseTransferLease(t *testing.T) {
	tr := &collectingSessionTransport{}
	s := New(Options{WriteTimeout: time.Second})
	s.outboundTrackedBudget = newOutboundTrackedBudget(1)
	key := newTestAuthKey(t)
	oldConn := s.newConn(tr, key, 74003, 1)
	reqMsgID := proto.NewMessageIDGen(time.Now).New(proto.MessageFromClient)
	owner, err := s.rpcResults.Acquire(key.ID, oldConn.sessionID, reqMsgID)
	if err != nil || owner.state != rpcResultAcquireOwner {
		t.Fatalf("initial flight owner = %+v err=%v", owner, err)
	}

	// Session replacement publishes terminal before transferring the physical
	// lease. A late old-generation result sees ErrConnClosed at producer admission;
	// it publishes a metadata-only capacity tombstone, but must not upgrade that
	// intentional fence into a physical close that makes Transfer fail.
	oldConn.beginTerminalShutdown()
	err = s.sendResult(context.Background(), oldConn, reqMsgID, exactTestRPCResult(&tg.Config{ThisDC: 2}))
	if !errors.Is(err, ErrOutboundTrackedBudget) {
		t.Fatalf("late result error = %v, want ErrOutboundTrackedBudget", err)
	}
	if tr.closed.Load() {
		t.Fatal("late result closed intentionally transferable transport")
	}
	if !oldConn.waitOutboundShutdownUntil(time.Second) {
		t.Fatal("old outbound actor did not stop")
	}
	nextLease, ok := oldConn.transferTransportOwnership()
	if !ok || nextLease == nil {
		t.Fatal("late result prevented physical transfer")
	}
	newConn := s.newConnWithLease(nextLease, key, 74004, 1)
	oldConn.ForceClose()
	if tr.closed.Load() || !newConn.isPhysicalTransportCurrentOpen() {
		t.Fatalf("stale close after transfer: raw_closed=%v current_open=%v", tr.closed.Load(), newConn.isPhysicalTransportCurrentOpen())
	}
	if _, acquireErr := s.rpcResults.Acquire(key.ID, oldConn.sessionID, reqMsgID); !errors.Is(acquireErr, ErrRPCResultFlightCapacity) {
		t.Fatalf("late result tombstone err=%v, want %v", acquireErr, ErrRPCResultFlightCapacity)
	}
	newConn.ForceClose()
}

func TestRPCResultPublishesBeforePathologicalPhysicalCloseReturns(t *testing.T) {
	tr := newBlockingCloseRPCResultTransport()
	s := New(Options{WriteTimeout: time.Second})
	s.outboundTrackedBudget = newOutboundTrackedBudget(1)
	key := newTestAuthKey(t)
	c := s.newConn(tr, key, 74005, 1)
	reqMsgID := proto.NewMessageIDGen(time.Now).New(proto.MessageFromClient)
	owner, err := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID)
	if err != nil || owner.state != rpcResultAcquireOwner {
		t.Fatalf("initial flight owner = %+v err=%v", owner, err)
	}

	done := make(chan error, 1)
	go func() {
		done <- s.sendResult(context.Background(), c, reqMsgID, exactTestRPCResult(&tg.Config{ThisDC: 2}))
	}()
	select {
	case <-tr.started:
	case <-time.After(time.Second):
		t.Fatal("physical Close did not start")
	}
	select {
	case err := <-done:
		if !errors.Is(err, ErrOutboundTrackedBudget) {
			t.Fatalf("sendResult error = %v, want budget error", err)
		}
	case <-time.After(time.Second):
		t.Fatal("pathological physical Close blocked result publication")
	}
	if _, acquireErr := s.rpcResults.Acquire(key.ID, c.sessionID, reqMsgID); !errors.Is(acquireErr, ErrRPCResultFlightCapacity) {
		t.Fatalf("result tombstone before raw Close return err=%v, want %v", acquireErr, ErrRPCResultFlightCapacity)
	}
	close(tr.release)
	if c.transportLease != nil {
		if err := c.transportLease.owner.waitClosed(); err != nil {
			t.Fatalf("physical Close: %v", err)
		}
	}
	c.Close()
}