main_reconnect_test.go
Rama main · réplica de lectura del repositorio original.
package main
import (
"context"
"io"
"log/slog"
"sync"
"testing"
"time"
"github.com/pingular/pingularity/internal/config"
"github.com/pingular/pingularity/internal/speedtest"
"github.com/pingular/pingularity/internal/store"
)
// clockBase is an arbitrary fixed instant the gate tests advance by hand. Nothing
// in reconnectGate reads a clock, so the tests are wall-clock-free and can step
// hours in a nanosecond.
var clockBase = time.Date(2026, 7, 25, 12, 0, 0, 0, time.UTC)
// A flapping link confirms a reconnect every few seconds; m.OnReconnect used to
// dispatch a speedtest and a netinfo refresh for every one of them, so the whole
// value of reconnectGate is what it does on the SECOND and later call. Pin the
// three properties the trigger depends on: the first call is never suppressed
// (the trigger exists to measure the link right after it recovers), a call inside
// the window is, and a suppressed call does not slide the window forward - a link
// flapping faster than the window must still get a run once the window expires.
func TestReconnectGateSpacing(t *testing.T) {
const min = 15 * time.Minute
var g reconnectGate
// Zero value is open: no startup priming, no "hot" gate.
if !g.allow(clockBase, min) {
t.Fatal("zero-value gate suppressed the first reconnect; the first one must always run")
}
// Inside the window: suppressed, at both ends of it.
if g.allow(clockBase.Add(time.Second), min) {
t.Error("a reconnect 1s after a fire was allowed; the flap storm is exactly this case")
}
if g.allow(clockBase.Add(min-time.Nanosecond), min) {
t.Error("a reconnect just short of the window was allowed")
}
// Exactly at the window: allowed (the spacing is a minimum, not a strict gap).
if !g.allow(clockBase.Add(min), min) {
t.Fatal("a reconnect exactly one window after the fire was suppressed")
}
// The window now runs from that fire, not from the suppressed attempts in
// between: one more window from clockBase+min, not from clockBase.
if g.allow(clockBase.Add(min).Add(min-time.Nanosecond), min) {
t.Error("window measured from the wrong fire: a call inside the second window was allowed")
}
if !g.allow(clockBase.Add(2*min), min) {
t.Error("a reconnect two windows in was suppressed; suppressed calls must not slide the window")
}
}
// A sustained flap must not starve the trigger: every call being suppressed would
// slide the window forward forever if allow recorded suppressed attempts too. Ten
// minutes of reconnects every 15s (the shipped-defaults cadence from the bug
// report) must still produce runs on the window, and only on the window.
func TestReconnectGateFlapStormStillFires(t *testing.T) {
const min = 5 * time.Minute
var g reconnectGate
fires := 0
for i := 0; i <= 40; i++ { // 41 reconnects over 10 minutes
if g.allow(clockBase.Add(time.Duration(i)*15*time.Second), min) {
fires++
}
}
// t=0, t=5m, t=10m: the first plus one per elapsed window.
if fires != 3 {
t.Errorf("10min of 15s flapping fired %d times, want 3 (one per %v window)", fires, min)
}
}
// The two jobs the reconnect callback dispatches are metered separately, and the
// cheap one must not inherit the expensive one's floor: at a point where the
// netinfo gate has reopened the speedtest gate must still be shut, or a flap
// storm gets a speedtest every few minutes after all. This is the check that
// fails if someone later "simplifies" the two gates into one shared instance or
// one shared constant.
func TestReconnectGatesAreIndependent(t *testing.T) {
if reconnectNetinfoGap >= minReconnectSpeedGap {
t.Fatalf("netinfo floor %v >= speedtest floor %v; the cheap lookup must reopen sooner than a 30-60s bulk transfer",
reconnectNetinfoGap, minReconnectSpeedGap)
}
var netinfoGate, speedGate reconnectGate
speedMin := reconnectSpeedGap(time.Hour) // stock schedule -> the hard floor
if !netinfoGate.allow(clockBase, reconnectNetinfoGap) || !speedGate.allow(clockBase, speedMin) {
t.Fatal("first reconnect suppressed on one of the gates")
}
// One netinfo window later: the lookup is due again, the speedtest is not.
at := clockBase.Add(reconnectNetinfoGap)
if !netinfoGate.allow(at, reconnectNetinfoGap) {
t.Errorf("netinfo refresh suppressed %v after the last one, its own floor", reconnectNetinfoGap)
}
if speedGate.allow(at, speedMin) {
t.Errorf("speedtest allowed %v after the last one; its floor is %v", reconnectNetinfoGap, speedMin)
}
// One speedtest window later it is due, and the netinfo gate it shares nothing
// with is unaffected by either decision.
if !speedGate.allow(clockBase.Add(speedMin), speedMin) {
t.Errorf("speedtest suppressed %v after the last one, its own floor", speedMin)
}
}
// reconnectSpeedGap reads the configured speedtest interval as a data budget: the
// spacing is the larger of the hard floor and the user's own cadence, so a 6h
// schedule does not get reconnect tests every 15 minutes. Pin both halves plus the
// floor's own derivation, since a floor below 5x the smallest schedule the
// settings layer accepts (config.MinSpeedInterval) would put us back in
// back-to-back-transfer territory on a flapping link.
func TestReconnectSpeedGap(t *testing.T) {
if minReconnectSpeedGap < 5*config.MinSpeedInterval {
t.Errorf("speedtest floor %v is under 5x the %v interval floor", minReconnectSpeedGap, config.MinSpeedInterval)
}
cases := []struct {
name string
interval time.Duration
want time.Duration
}{
{"unset interval falls back to the floor", 0, minReconnectSpeedGap},
{"minimum schedule takes the floor", config.MinSpeedInterval, minReconnectSpeedGap},
{"schedule just under the floor takes the floor", minReconnectSpeedGap - time.Minute, minReconnectSpeedGap},
{"schedule at the floor takes the floor", minReconnectSpeedGap, minReconnectSpeedGap},
{"stock hourly schedule wins over the floor", time.Hour, time.Hour},
{"6h schedule is not tested every 15m", 6 * time.Hour, 6 * time.Hour},
{"maximum schedule wins", config.MaxSpeedInterval, config.MaxSpeedInterval},
}
for _, c := range cases {
if got := reconnectSpeedGap(c.interval); got != c.want {
t.Errorf("%s: reconnectSpeedGap(%v) = %v, want %v", c.name, c.interval, got, c.want)
}
}
}
// The gate is called from the monitor goroutine today, but its correctness must
// not depend on that: concurrent callers at the same instant must still produce
// exactly one fire. Run under -race, where an unguarded last would report.
func TestReconnectGateConcurrentFiresOnce(t *testing.T) {
var g reconnectGate
if !g.allow(clockBase, time.Hour) { // close the gate first; the zero value is open
t.Fatal("first call suppressed")
}
const callers = 16
var wg sync.WaitGroup
var mu sync.Mutex
allowed := 0
for i := 0; i < callers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
// All well past the window, all at the same instant: exactly one wins.
if g.allow(clockBase.Add(2*time.Hour), time.Hour) {
mu.Lock()
allowed++
mu.Unlock()
}
}()
}
wg.Wait()
if allowed != 1 {
t.Errorf("%d of %d concurrent callers were allowed, want exactly 1", allowed, callers)
}
}
// A dispatch that collides with a run already in progress has measured nothing,
// so it must not consume the reconnect window.
//
// The window was reserved at dispatch and kept regardless, on the reasoning that
// the run we collided with is measuring this same recovery. That holds only if it
// started AFTER the link came back. A scheduled run already in flight when the
// reconnect lands is measuring the link while it was still broken - and keeping
// the window then suppresses the real recovery test for the whole gap, which is
// up to 24h once the interval is read as a data budget.
func TestABusyReconnectDispatchGivesTheWindowBack(t *testing.T) {
var g reconnectGate
now := time.Now()
gap := reconnectSpeedGap(time.Hour)
if !g.allow(now, gap) {
t.Fatal("the first reconnect must be allowed through an open gate")
}
// That dispatch bounced: nothing was measured.
g.release(now)
// The next reconnect, moments later, must still get its test.
if !g.allow(now.Add(time.Second), gap) {
t.Error("a reconnect was suppressed by a dispatch that measured nothing; the run it " +
"collided with may predate the reconnect entirely")
}
}
// ...but a release must never undo a window some LATER dispatch has since taken,
// or a flap storm could hand out an unbounded number of real tests.
func TestReleaseCannotUndoALaterFire(t *testing.T) {
var g reconnectGate
first := time.Now()
gap := reconnectSpeedGap(time.Hour)
if !g.allow(first, gap) {
t.Fatal("first dispatch should pass")
}
// A second reconnect an hour later legitimately claims the gate.
second := first.Add(gap + time.Minute)
if !g.allow(second, gap) {
t.Fatal("second dispatch should pass after the gap")
}
// The FIRST dispatch now reports it bounced. It must not reopen the gate the
// second one is holding.
g.release(first)
if g.allow(second.Add(time.Second), gap) {
t.Error("a stale release reopened the gate, so a flap storm could dispatch a real " +
"speedtest per flap")
}
}
// A run that actually happened keeps the window, which is the whole point of the
// gate.
func TestASuccessfulReconnectRunKeepsTheWindow(t *testing.T) {
var g reconnectGate
now := time.Now()
gap := reconnectSpeedGap(time.Hour)
if !g.allow(now, gap) {
t.Fatal("first dispatch should pass")
}
// No release: the run measured something.
if g.allow(now.Add(time.Minute), gap) {
t.Error("a second reconnect ran a minute after a completed one; the gate is not spacing")
}
}
// A reconnect test after a blip starts at once; after a real outage - ten
// minutes or more - it waits for the link to settle first. The boundary is the
// line between the outages on record that came back whole and the one that did
// not (see reconnectSettleAfter).
func TestReconnectSettleFor(t *testing.T) {
for _, c := range []struct {
down time.Duration
want time.Duration
}{
{0, 0},
{90 * time.Second, 0},
{reconnectSettleAfter - time.Second, 0},
{reconnectSettleAfter, reconnectSettle},
{4 * time.Hour, reconnectSettle},
} {
if got := reconnectSettleFor(c.down); got != c.want {
t.Errorf("reconnectSettleFor(%v) = %v, want %v", c.down, got, c.want)
}
}
}
// The wait is real, and a link that is still up at the end of it gets its test.
func TestASettledReconnectRunWaitsOutTheSettle(t *testing.T) {
start := time.Now()
if !settleBeforeReconnectRun(context.Background(), 60*time.Millisecond, func() bool { return true }) {
t.Fatal("a link still up after the settle was not run")
}
if took := time.Since(start); took < 60*time.Millisecond {
t.Errorf("returned after %v, before the settle had passed", took)
}
}
// A link that went down again while the run was waiting is not measured: the
// run would fail and spend the reconnect window on it, and the next up event
// brings its own reconnect.
func TestASettledReconnectRunSkipsALinkThatDroppedAgain(t *testing.T) {
if settleBeforeReconnectRun(context.Background(), time.Millisecond, func() bool { return false }) {
t.Fatal("a link that dropped again during the settle was run anyway")
}
}
// A daemon stopping mid-settle does not wait the settle out.
func TestASettledReconnectRunStopsWithTheDaemon(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
time.AfterFunc(20*time.Millisecond, cancel)
start := time.Now()
if settleBeforeReconnectRun(ctx, 10*time.Second, func() bool { return true }) {
t.Fatal("a run was started on a stopping daemon")
}
if took := time.Since(start); took > time.Second {
t.Errorf("returned after %v; the cancel should have ended the wait at once", took)
}
}
// With nothing to wait for the decision is the link's state alone, taken now.
func TestAnUnsettledReconnectRunStartsAtOnce(t *testing.T) {
start := time.Now()
if !settleBeforeReconnectRun(context.Background(), 0, func() bool { return true }) {
t.Fatal("an up link with no settle was not run")
}
if took := time.Since(start); took > 50*time.Millisecond {
t.Errorf("a zero settle took %v", took)
}
}
// The settle itself is a real wait, not a placeholder: a minute, enough for a
// modem to re-provision and a lease to renew.
func TestReconnectSettleIsAMinute(t *testing.T) {
if reconnectSettle != time.Minute {
t.Fatalf("reconnectSettle = %v, want 1m", reconnectSettle)
}
if reconnectSettleAfter != 10*time.Minute {
t.Fatalf("reconnectSettleAfter = %v, want 10m", reconnectSettleAfter)
}
}
// reconnectDispatch through its outcomes. A run that happened keeps the window;
// every way a run does not happen hands it back, so the next reconnect gets its
// test instead of waiting out a window spent on nothing.
func TestReconnectDispatchKeepsTheWindowOnlyForARunThatHappened(t *testing.T) {
quiet := slog.New(slog.NewTextHandler(io.Discard, nil))
gap := reconnectSpeedGap(time.Hour)
ran := func() func(context.Context, string) (store.SpeedSample, error) {
return func(context.Context, string) (store.SpeedSample, error) { return store.SpeedSample{}, nil }
}
busy := func(context.Context, string) (store.SpeedSample, error) {
return store.SpeedSample{}, speedtest.ErrBusy
}
up, down := func() bool { return true }, func() bool { return false }
cases := []struct {
name string
settle time.Duration
online func() bool
run func(context.Context, string) (store.SpeedSample, error)
cancel bool
wantRun bool
reopen bool // the window is handed back
}{
{"blip, link up, run happens", 0, up, ran(), false, true, false},
{"real outage, link still up after the settle", time.Millisecond, up, ran(), false, true, false},
{"real outage, link down again after the settle", time.Millisecond, down, ran(), false, false, true},
{"daemon stopping during the settle", 10 * time.Second, up, ran(), true, false, true},
{"run bounced off a run already in flight", 0, up, busy, false, true, true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
var g reconnectGate
now := time.Now()
if !g.allow(now, gap) {
t.Fatal("an open gate refused the first dispatch")
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if c.cancel {
time.AfterFunc(10*time.Millisecond, cancel)
}
calls := 0
run := func(ctx context.Context, reason string) (store.SpeedSample, error) {
calls++
if reason != "reconnect" {
t.Errorf("run reason = %q, want reconnect", reason)
}
return c.run(ctx, reason)
}
reconnectDispatch(ctx, &g, now, c.settle, 4*3600, c.online, run, quiet)
if (calls > 0) != c.wantRun {
t.Errorf("run called %d times, want called=%v", calls, c.wantRun)
}
if got := g.allow(now.Add(time.Second), gap); got != c.reopen {
t.Errorf("a reconnect a second later allowed=%v, want %v", got, c.reopen)
}
})
}
}