utils/ctx_hangup_unix_test.go
Rama main · réplica de lectura del repositorio original.
//go:build unix
package utils
import (
"bytes"
"errors"
"os"
"os/exec"
"os/signal"
"path/filepath"
"strconv"
"strings"
"syscall"
"testing"
"time"
)
// hangupHelperDir, set, makes TestHangupHelper the keploy run the tests below
// start and signal, with its markers in that directory.
const hangupHelperDir = "KEPLOY_TEST_HANGUP_HELPER"
// TestHangupHelper is not a test of its own. With hangupHelperDir set it is a
// keploy run: NewCtx -- then DieOnHangup, as `keploy agent` calls it, if
// KEPLOY_TEST_HANGUP_AGENT is set -- and a command that runs until the root
// context ends, and then a shutdown that takes a second, as stopping the
// test command and the agent does. It leaves in its directory:
//
// ready once it waits, whether SIGHUP is ignored
// holding with KEPLOY_TEST_HANGUP_HOLD, once its signal hold waits:
// whether hang-ups were absorbed by then
// hooks-ran once the pre-cancel hooks have run
// stopping once its shutdown has begun
// absorbed as an agent, at the cancel: whether hang-ups were absorbed by then
// stopped once its shutdown is done, whether it was interrupted
func TestHangupHelper(t *testing.T) {
dir := os.Getenv(hangupHelperDir)
if dir == "" {
return
}
// Written whole, then renamed into place: the test reads a marker as
// soon as it exists.
mark := func(name, content string) {
tmp := filepath.Join(dir, name+".tmp")
if os.WriteFile(tmp, []byte(content), 0o600) != nil || os.Rename(tmp, filepath.Join(dir, name)) != nil {
os.Exit(3)
}
}
ctx := NewCtx()
agent := os.Getenv("KEPLOY_TEST_HANGUP_AGENT") != ""
if agent {
// As cli/agent.go does.
DieOnHangup()
IgnoreHangupOnceStopping(ctx)
}
RegisterPreCancelHook(func() { mark("hooks-ran", "") })
if agent {
// What the cancel finds, on both ways it is reached: the signal
// handler runs the pre-cancel hooks just before it, and Stop runs
// the cancel SetCancel installed.
RegisterPreCancelHook(func() { mark("absorbed", strconv.FormatBool(hangupsAbsorbed.Load())) })
orig := cancel
SetCancel(func() { mark("absorbed", strconv.FormatBool(hangupsAbsorbed.Load())); orig() })
}
if os.Getenv("KEPLOY_TEST_HANGUP_HOLD") != "" {
// As cli/agent.go holds the first stop signal while a recording's
// client still drains: here until the test says.
SetSignalHold(func() {
mark("holding", strconv.FormatBool(hangupsAbsorbed.Load()))
for {
if _, err := os.Stat(filepath.Join(dir, "release-hold")); err == nil {
return
}
time.Sleep(5 * time.Millisecond)
}
})
}
mark("ready", strconv.FormatBool(signal.Ignored(syscall.SIGHUP)))
if os.Getenv("KEPLOY_TEST_HANGUP_STOP_API") != "" {
// Stopped as the agent API's /stop does (Stop), once the test says.
go func() {
for {
if _, err := os.Stat(filepath.Join(dir, "stop")); err == nil {
ExecCancel()
return
}
time.Sleep(5 * time.Millisecond)
}
}()
}
<-ctx.Done()
// No wait for anything: a hang-up from here on must not kill an agent.
mark("stopping", "")
if os.Getenv("KEPLOY_TEST_HANGUP_RELEASE") != "" {
// The shutdown lasts until the test says: through its hang-up,
// whatever the test's own scheduling.
for {
if _, err := os.Stat(filepath.Join(dir, "release")); err == nil {
break
}
time.Sleep(5 * time.Millisecond)
}
} else {
time.Sleep(time.Second)
}
mark("stopped", strconv.FormatBool(Interrupted()))
os.Exit(0)
}
// hangupRun is a TestHangupHelper process.
type hangupRun struct {
t *testing.T
dir string
cmd *exec.Cmd
out bytes.Buffer
exited chan error
}
// startHangupRun starts a TestHangupHelper process through launcher (nohup,
// or nothing) and waits until it is ready to be signalled.
func startHangupRun(t *testing.T, agent bool, launcher ...string) *hangupRun {
t.Helper()
r := &hangupRun{t: t, dir: t.TempDir(), exited: make(chan error, 1)}
argv := append(append([]string{}, launcher...), os.Args[0], "-test.run=^TestHangupHelper$")
r.cmd = exec.Command(argv[0], argv[1:]...)
r.cmd.Env = append(os.Environ(), hangupHelperDir+"="+r.dir)
if agent {
r.cmd.Env = append(r.cmd.Env, "KEPLOY_TEST_HANGUP_AGENT=1")
}
r.cmd.Stdout, r.cmd.Stderr = &r.out, &r.out
if err := r.cmd.Start(); err != nil {
t.Fatal(err)
}
go func() { r.exited <- r.cmd.Wait() }()
deadline := time.After(30 * time.Second)
for !r.has("ready") {
select {
case err := <-r.exited:
t.Fatalf("the run ended before it was ready: %v\n%s", err, r.out.String())
case <-deadline:
t.Fatalf("the run was never ready\n%s", r.kill())
case <-time.After(10 * time.Millisecond):
}
}
return r
}
func (r *hangupRun) has(name string) bool {
_, err := os.Stat(filepath.Join(r.dir, name))
return err == nil
}
func (r *hangupRun) marker(name string) string {
b, err := os.ReadFile(filepath.Join(r.dir, name))
if err != nil {
return "<none: " + err.Error() + ">"
}
return string(b)
}
func (r *hangupRun) signal(sig syscall.Signal) {
if err := r.cmd.Process.Signal(sig); err != nil {
r.t.Fatalf("%v: %v\n%s", sig, err, r.kill())
}
}
// kill ends the run and returns what it printed.
func (r *hangupRun) kill() string {
_ = r.cmd.Process.Kill()
<-r.exited
return r.out.String()
}
// wait returns how the run ended, failing the test if it runs on for 30s.
func (r *hangupRun) wait() syscall.WaitStatus {
var err error
select {
case err = <-r.exited:
case <-time.After(30 * time.Second):
r.t.Fatalf("the run was still running 30s after it was signalled\n%s", r.kill())
}
var exit *exec.ExitError
if err != nil && !errors.As(err, &exit) {
r.t.Fatalf("the run: %v\n%s", err, r.out.String())
}
return r.cmd.ProcessState.Sys().(syscall.WaitStatus)
}
// catchHangups starts every run this test starts with SIGHUP's default
// action, as a terminal's task gets it, even when `go test` itself runs under
// nohup, whose SIG_IGN a run would otherwise inherit and, rightly, keep: Go
// starts a child with the default action for a signal it catches.
func catchHangups(t *testing.T) {
caught := make(chan os.Signal, 1)
signal.Notify(caught, syscall.SIGHUP)
t.Cleanup(func() { signal.Stop(caught) })
}
// A hang-up stops keploy as SIGTERM does -- the run is interrupted, the
// pre-cancel hooks run, the root context ends and the shutdown finishes -- and
// every SIGHUP after the first changes nothing. The VS Code extension's Stop
// ends a run with a hang-up, and SIGHUP killed keploy where it stood: the test
// command it had started ran on, and none of the cleanup keploy defers ran.
// One hang-up can deliver two SIGHUPs, the second while keploy is shutting
// down, and that one must not kill it either. 0ms apart the two may be merged
// into one on the way; 30ms and 100ms apart they arrive one by one.
func TestAHangUpStopsKeployAsSIGTERMDoes(t *testing.T) {
catchHangups(t)
type step struct {
sig syscall.Signal
after time.Duration // since the step before
}
for _, tc := range []struct {
name string
steps []step
// what the handler says of the second signal, if anything
says string
}{
{name: "SIGTERM", steps: []step{{sig: syscall.SIGTERM}}},
{name: "SIGHUP", steps: []step{{sig: syscall.SIGHUP}}},
{name: "two SIGHUPs 0ms apart", steps: []step{{sig: syscall.SIGHUP}, {sig: syscall.SIGHUP}}},
{name: "two SIGHUPs 30ms apart", steps: []step{{sig: syscall.SIGHUP}, {sig: syscall.SIGHUP, after: 30 * time.Millisecond}}},
{name: "two SIGHUPs 100ms apart", steps: []step{{sig: syscall.SIGHUP}, {sig: syscall.SIGHUP, after: 100 * time.Millisecond}}},
// A hang-up while an interrupt is already being handled is no second
// stop, and must not be reported as one.
{
name: "SIGINT, then SIGHUP 100ms later",
steps: []step{{sig: syscall.SIGINT}, {sig: syscall.SIGHUP, after: 100 * time.Millisecond}},
says: "Signal received: hangup, ignored: Keploy is already handling interrupt\n",
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
r := startHangupRun(t, false)
for _, s := range tc.steps {
time.Sleep(s.after)
r.signal(s.sig)
}
st := r.wait()
out := r.out.String()
if st.Signaled() {
t.Fatalf("the run died of %v where it stood, want it stopped as SIGTERM stops it\n%s", st.Signal(), out)
}
if st.ExitStatus() != 0 {
t.Fatalf("the run exited %d, want 0\n%s", st.ExitStatus(), out)
}
if got := r.marker("stopped"); got != "true" {
t.Errorf("stopped = %q, want the shutdown finished and the run marked interrupted\n%s", got, out)
}
if !r.has("hooks-ran") {
t.Errorf("the pre-cancel hooks never ran\n%s", out)
}
first := "Signal received: " + tc.steps[0].sig.String() + ", canceling context...\n"
if !strings.HasPrefix(out, first) {
t.Errorf("the run's output does not start %q\n%s", first, out)
}
if n := strings.Count(out, "canceling context"); n != 1 {
t.Errorf("the run said %d times that it was stopping, want once\n%s", n, out)
}
if tc.says != "" && !strings.Contains(out, tc.says) {
t.Errorf("the run did not say %q\n%s", tc.says, out)
}
})
}
}
// A run started with SIGHUP ignored -- `nohup keploy record ...` -- asked to
// outlive its terminal, and does: NewCtx does not start listening for it, so
// the process stays deaf to it and SIGTERM is still what stops it. The same
// for `keploy agent` under nohup, whose DieOnHangup must give back the SIG_IGN
// it started with, not death.
func TestARunStartedUnderNohupIgnoresHangUps(t *testing.T) {
catchHangups(t)
nohup, err := exec.LookPath("nohup")
if err != nil {
t.Fatalf("nohup: %v", err)
}
for _, agent := range []bool{false, true} {
t.Run("agent="+strconv.FormatBool(agent), func(t *testing.T) {
t.Parallel()
r := startHangupRun(t, agent, nohup)
if got := r.marker("ready"); got != "true" {
t.Fatalf("SIGHUP ignored = %q once the run started under nohup, want true\n%s", got, r.kill())
}
r.signal(syscall.SIGHUP)
time.Sleep(30 * time.Millisecond)
r.signal(syscall.SIGHUP)
time.Sleep(300 * time.Millisecond)
r.signal(syscall.SIGTERM)
st := r.wait()
out := r.out.String()
if st.Signaled() || st.ExitStatus() != 0 {
t.Fatalf("the run ended with %v, want exit 0 once SIGTERM stopped it\n%s", r.cmd.ProcessState, out)
}
if first := "Signal received: terminated, canceling context...\n"; !strings.HasPrefix(out, first) {
t.Errorf("the run's output does not start %q: a run under nohup heard a hang-up\n%s", first, out)
}
})
}
}
// A `keploy agent` no keploy client started opts out: DieOnHangup gives
// SIGHUP back its default action, so a hang-up kills the agent, as its
// owners count on (see DieOnHangup).
func TestDieOnHangupLetsAHangUpKill(t *testing.T) {
catchHangups(t)
r := startHangupRun(t, true)
if got := r.marker("ready"); got != "false" {
t.Fatalf("SIGHUP ignored = %q in a run not started under nohup, want false\n%s", got, r.kill())
}
r.signal(syscall.SIGHUP)
st := r.wait()
if !st.Signaled() || st.Signal() != syscall.SIGHUP {
t.Fatalf("the run ended with %v, want it dead of SIGHUP\n%s", r.cmd.ProcessState, r.out.String())
}
}
// Once an agent is shutting down, a hang-up no longer kills it: it finishes
// its shutdown, which removes what it attached to the kernel -- a hang-up that
// killed it part way through removing a TLS uprobe leaked the breakpoint
// (IgnoreHangupOnceStopping). The CLI that started it on a PTY can exit while
// it shuts down, and that exit hangs it up. The hang-up is sent the moment the
// shutdown has begun, on both ways a root context is cancelled: a signal, and
// the agent API's stop. A signal the agent holds for its recording client
// (SetSignalHold) has begun nothing while it waits, so hang-ups are not
// absorbed then; they are once the hold ends, still before the cancel.
func TestAnAgentShuttingDownSurvivesAHangUp(t *testing.T) {
for _, how := range []string{"signal", "signal-held", "stop-api"} {
t.Run(how, func(t *testing.T) {
catchHangups(t)
t.Setenv("KEPLOY_TEST_HANGUP_RELEASE", "1")
if how == "stop-api" {
t.Setenv("KEPLOY_TEST_HANGUP_STOP_API", "1")
}
if how == "signal-held" {
t.Setenv("KEPLOY_TEST_HANGUP_HOLD", "1")
}
r := startHangupRun(t, true)
if how != "stop-api" {
r.signal(syscall.SIGTERM)
} else if err := os.WriteFile(filepath.Join(r.dir, "stop"), nil, 0o600); err != nil {
t.Fatal(err)
}
if how == "signal-held" {
for !r.has("holding") {
select {
case err := <-r.exited:
t.Fatalf("the run ended before its signal hold: %v\n%s", err, r.out.String())
case <-time.After(5 * time.Millisecond):
}
}
if got := r.marker("holding"); got != "false" {
t.Errorf("hang-ups were absorbed while the agent only held the signal for its client: %q", got)
}
if r.has("stopping") {
t.Fatalf("the root context was cancelled during the signal hold\n%s", r.out.String())
}
if err := os.WriteFile(filepath.Join(r.dir, "release-hold"), nil, 0o600); err != nil {
t.Fatal(err)
}
}
for !r.has("stopping") {
select {
case err := <-r.exited:
t.Fatalf("the run ended before its shutdown began: %v\n%s", err, r.out.String())
case <-time.After(5 * time.Millisecond):
}
}
// Absorbed before the cancel, not merely before this hang-up
// arrived: anything that runs on the cancellation is covered.
if got := r.marker("absorbed"); got != "true" {
t.Errorf("hang-ups were not absorbed by the time the root context was cancelled: %q", got)
}
r.signal(syscall.SIGHUP)
if err := os.WriteFile(filepath.Join(r.dir, "release"), nil, 0o600); err != nil {
t.Fatal(err)
}
st := r.wait()
if st.Signaled() {
t.Fatalf("a hang-up killed the agent during its shutdown (%v)\n%s", r.cmd.ProcessState, r.out.String())
}
if got := r.marker("stopped"); got != strconv.FormatBool(how != "stop-api") {
t.Fatalf("the agent's shutdown did not finish as it should: stopped = %q\n%s", got, r.out.String())
}
})
}
}