agent/agentscripts/script_order_internal_test.go
Rama main · réplica de lectura del repositorio original.
package agentscripts
import (
"context"
"fmt"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/xerrors"
"github.com/coder/coder/v2/agent/proto"
"github.com/coder/coder/v2/agent/unit"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/codersdk/agentsdk"
"github.com/coder/coder/v2/testutil"
"github.com/coder/quartz"
)
// goleak runs from TestMain in agentscripts_test.go, which shares this
// test binary.
const (
requireSuccess = codersdk.WorkspaceAgentScriptDependencyRequirementSuccess
requireCompletion = codersdk.WorkspaceAgentScriptDependencyRequirementCompletion
)
func TestLifecycleExecutorOrdersScripts(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.hold("coder_script.clone")
executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
clone := scriptOrderTestScript("coder_script.clone")
install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
done := runExecute(ctx, executor, clone, install)
runs.waitStarted(ctx, t, "coder_script.clone")
runs.requireNotStarted(t, "coder_script.install")
runs.release("coder_script.clone", nil)
runs.waitStarted(ctx, t, "coder_script.install")
require.NoError(t, testutil.RequireReceive(ctx, t, done))
}
func TestLifecycleExecutorWaitsForEveryPrerequisite(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.hold("coder_script.a")
runs.hold("coder_script.b")
executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
a := scriptOrderTestScript("coder_script.a")
b := scriptOrderTestScript("coder_script.b")
c := scriptOrderTestScript("coder_script.c",
scriptOrderTestDependency("coder_script.a", requireSuccess),
scriptOrderTestDependency("coder_script.b", requireCompletion),
)
done := runExecute(ctx, executor, a, b, c)
runs.waitStarted(ctx, t, "coder_script.a")
runs.waitStarted(ctx, t, "coder_script.b")
runs.release("coder_script.a", nil)
runs.requireNotStarted(t, "coder_script.c")
runs.release("coder_script.b", nil)
runs.waitStarted(ctx, t, "coder_script.c")
require.NoError(t, testutil.RequireReceive(ctx, t, done))
}
// The executor hands run a context with no deadline of its own, so the
// timeout context created inside run cannot have started during the wait.
// TestExecuteOrderedTimeoutStartsAfterWaiting covers the same with a real
// process.
func TestLifecycleExecutorTimeoutStartsAfterWaiting(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
executeCtx, cancel := context.WithCancel(context.Background())
defer cancel()
runs := newScriptOrderRuns()
runs.hold("coder_script.clone")
executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
deadlines := make(chan bool, 2)
executor.run = func(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
_, hasDeadline := ctx.Deadline()
deadlines <- hasDeadline
return runs.run(ctx, script, option)
}
clone := scriptOrderTestScript("coder_script.clone")
install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
install.Timeout = 10 * time.Millisecond
done := runExecute(executeCtx, executor, clone, install)
runs.waitStarted(ctx, t, "coder_script.clone")
require.False(t, testutil.RequireReceive(ctx, t, deadlines))
runs.release("coder_script.clone", nil)
runs.waitStarted(ctx, t, "coder_script.install")
require.False(t, testutil.RequireReceive(ctx, t, deadlines))
require.NoError(t, testutil.RequireReceive(ctx, t, done))
}
func TestScriptOutcome(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name string
runErr error
ctxErr error
want unit.Outcome
}{
{name: "Succeeded", want: unit.OutcomeSucceeded},
{name: "TimedOut", runErr: ErrTimeout, want: unit.OutcomeTimedOut},
{name: "TimedOutDuringCancel", runErr: ErrTimeout, ctxErr: context.Canceled, want: unit.OutcomeTimedOut},
{name: "PipesLeftOpen", runErr: ErrOutputPipesOpen, want: unit.OutcomeFailed},
{name: "ExitCode", runErr: xerrors.New("exit status 1"), want: unit.OutcomeFailed},
{name: "Canceled", runErr: context.Canceled, ctxErr: context.Canceled, want: unit.OutcomeCanceled},
{name: "OtherErrorDuringCancel", runErr: xerrors.New("signal: killed"), ctxErr: context.Canceled, want: unit.OutcomeCanceled},
{name: "WrappedTimeout", runErr: xerrors.Errorf("run agent script: %w", ErrTimeout), want: unit.OutcomeTimedOut},
} {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
require.Equal(t, test.want, scriptOutcome(test.runErr, test.ctxErr))
})
}
}
func TestLifecycleExecutorRequirements(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name string
requirement codersdk.WorkspaceAgentScriptDependencyRequirement
cloneErr error
wantRun bool
wantLogs []string
}{
{name: "SuccessAfterFailure", requirement: requireSuccess, cloneErr: xerrors.New("exit status 128"), wantLogs: []string{`Skipping script: dependency "Clone" failed.`}},
{name: "SuccessAfterTimeout", requirement: requireSuccess, cloneErr: ErrTimeout, wantLogs: []string{`Skipping script: dependency "Clone" timed out.`}},
{name: "SuccessAfterSuccess", requirement: requireSuccess, wantRun: true},
{name: "CompletionAfterFailure", requirement: requireCompletion, cloneErr: xerrors.New("exit status 128"), wantRun: true},
{name: "CompletionAfterTimeout", requirement: requireCompletion, cloneErr: ErrTimeout, wantRun: true},
{name: "CompletionAfterSuccess", requirement: requireCompletion, wantRun: true},
} {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.result("coder_script.clone", test.cloneErr)
logs := newScriptOrderTestLogs()
executor := newTestExecutor(t, runs, logs)
clone := scriptOrderTestScript("coder_script.clone")
clone.DisplayName = "Clone"
install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", test.requirement))
err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{clone, install}, ExecuteStartScripts)
if test.cloneErr != nil {
require.ErrorIs(t, err, test.cloneErr)
} else {
require.NoError(t, err)
}
require.Equal(t, test.wantRun, runs.started("coder_script.install"))
require.Equal(t, test.wantLogs, logs.outputs(install.LogSourceID))
})
}
}
func TestLifecycleExecutorPropagatesSkippedOutcomes(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.result("coder_script.a", xerrors.New("exit status 1"))
logs := newScriptOrderTestLogs()
executor := newTestExecutor(t, runs, logs)
a := scriptOrderTestScript("coder_script.a")
b := scriptOrderTestScript("coder_script.b", scriptOrderTestDependency("coder_script.a", requireSuccess))
c := scriptOrderTestScript("coder_script.c", scriptOrderTestDependency("coder_script.b", requireCompletion))
d := scriptOrderTestScript("coder_script.d", scriptOrderTestDependency("coder_script.b", requireSuccess))
err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{a, b, c, d}, ExecuteStartScripts)
require.Error(t, err)
require.ElementsMatch(t, []string{"coder_script.a", "coder_script.c"}, runs.allStarted())
require.Equal(t, []string{`Skipping script: dependency "coder_script.a" failed.`}, logs.outputs(b.LogSourceID))
require.Empty(t, logs.outputs(c.LogSourceID))
require.Equal(t, []string{`Skipping script: dependency "coder_script.b" was skipped.`}, logs.outputs(d.LogSourceID))
}
func TestLifecycleExecutorReportsSkippedWithoutRunning(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.result("coder_script.clone", xerrors.New("exit status 128"))
logs := newScriptOrderTestLogs()
reports := newScriptOrderReports()
executor := newTestExecutor(t, runs, logs)
executor.reportUnstarted = func(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption, status proto.Timing_Status) {
assert.NotEmpty(t, logs.outputs(script.LogSourceID), "the skip line is written before the timing is sent")
reports.record(ctx, script, option, status)
}
clone := scriptOrderTestScript("coder_script.clone")
clone.DisplayName = "Clone"
install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{clone, install}, ExecuteStartScripts)
require.Error(t, err)
require.Equal(t, []string{"coder_script.clone"}, runs.allStarted())
require.Equal(t, []scriptOrderReport{
{address: "coder_script.install", option: ExecuteStartScripts, status: proto.Timing_SKIPPED},
}, reports.all())
require.Equal(t, []string{`Skipping script: dependency "Clone" failed.`}, logs.outputs(install.LogSourceID))
}
func TestLifecycleExecutorNilReporter(t *testing.T) {
t.Parallel()
t.Run("Skipped", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.result("coder_script.clone", xerrors.New("exit status 128"))
executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
clone := scriptOrderTestScript("coder_script.clone")
install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
require.Error(t, executor.execute(ctx, []codersdk.WorkspaceAgentScript{clone, install}, ExecuteStartScripts))
require.False(t, runs.started("coder_script.install"))
})
t.Run("Canceled", func(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(testutil.Context(t, testutil.WaitShort))
defer cancel()
runs := newScriptOrderRuns()
runs.hold("coder_script.clone")
executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
clone := scriptOrderTestScript("coder_script.clone")
install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
done := runExecute(ctx, executor, clone, install)
runs.waitStarted(ctx, t, "coder_script.clone")
cancel()
waitCtx := testutil.Context(t, testutil.WaitShort)
require.ErrorIs(t, testutil.RequireReceive(waitCtx, t, done), context.Canceled)
require.False(t, runs.started("coder_script.install"))
})
}
func TestLifecycleExecutorMixedRequirements(t *testing.T) {
t.Parallel()
scripts := func() (clone, warm, build codersdk.WorkspaceAgentScript) {
clone = scriptOrderTestScript("coder_script.clone_repo")
clone.DisplayName = "Clone repo"
warm = scriptOrderTestScript("coder_script.warm_cache")
warm.DisplayName = "Warm cache"
build = scriptOrderTestScript("coder_script.build",
scriptOrderTestDependency("coder_script.clone_repo", requireSuccess),
scriptOrderTestDependency("coder_script.warm_cache", requireCompletion),
)
return clone, warm, build
}
t.Run("CompletionPrerequisiteFails", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.result("coder_script.warm_cache", xerrors.New("exit status 1"))
logs := newScriptOrderTestLogs()
executor := newTestExecutor(t, runs, logs)
clone, warm, build := scripts()
err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{clone, warm, build}, ExecuteStartScripts)
require.Error(t, err)
require.True(t, runs.started("coder_script.build"))
require.Empty(t, logs.outputs(build.LogSourceID))
})
t.Run("SuccessPrerequisiteFails", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.result("coder_script.clone_repo", xerrors.New("exit status 128"))
logs := newScriptOrderTestLogs()
executor := newTestExecutor(t, runs, logs)
clone, warm, build := scripts()
err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{clone, warm, build}, ExecuteStartScripts)
require.Error(t, err)
require.False(t, runs.started("coder_script.build"))
require.Equal(t, []string{`Skipping script: dependency "Clone repo" failed.`}, logs.outputs(build.LogSourceID))
})
}
func TestLifecycleExecutorSkipsOnFirstImpossibleEdge(t *testing.T) {
t.Parallel()
scripts := func() (clone, secrets, build codersdk.WorkspaceAgentScript) {
clone = scriptOrderTestScript("coder_script.clone_repo")
clone.DisplayName = "Clone repo"
secrets = scriptOrderTestScript("coder_script.fetch_secrets")
secrets.DisplayName = "Fetch secrets"
build = scriptOrderTestScript("coder_script.build",
scriptOrderTestDependency("coder_script.clone_repo", requireSuccess),
scriptOrderTestDependency("coder_script.fetch_secrets", requireSuccess),
)
return clone, secrets, build
}
t.Run("OnePrerequisiteStillRunning", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.hold("coder_script.clone_repo")
runs.result("coder_script.fetch_secrets", xerrors.New("exit status 1"))
logs := newScriptOrderTestLogs()
executor := newTestExecutor(t, runs, logs)
clone, secrets, build := scripts()
done := runExecute(ctx, executor, clone, secrets, build)
runs.waitStarted(ctx, t, "coder_script.clone_repo")
// The skip line arrives while clone_repo is still held.
require.Equal(t, `Skipping script: dependency "Fetch secrets" failed.`, logs.waitOutput(ctx, t, build.LogSourceID))
runs.release("coder_script.clone_repo", nil)
require.Error(t, testutil.RequireReceive(ctx, t, done))
require.False(t, runs.started("coder_script.build"))
})
t.Run("BothPrerequisitesFinishedBadly", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.result("coder_script.clone_repo", xerrors.New("exit status 128"))
runs.result("coder_script.fetch_secrets", ErrTimeout)
logs := newScriptOrderTestLogs()
executor := newTestExecutor(t, runs, logs)
clone, secrets, build := scripts()
// Both prerequisites finish before build is evaluated, so the
// manager lists both as impossible.
manager, byAddress, err := lifecycleManager([]codersdk.WorkspaceAgentScript{clone, secrets, build})
require.NoError(t, err)
require.Error(t, executor.executeWith(ctx, manager, byAddress, []codersdk.WorkspaceAgentScript{clone, secrets}, ExecuteStartScripts))
require.NoError(t, executor.executeWith(ctx, manager, byAddress, []codersdk.WorkspaceAgentScript{build}, ExecuteStartScripts))
require.False(t, runs.started("coder_script.build"))
require.Equal(t, []string{
`Skipping script: dependency "Clone repo" failed, dependency "Fetch secrets" timed out.`,
}, logs.outputs(build.LogSourceID))
})
}
func TestLifecycleExecutorWaitingLogsAreCombinedAndSorted(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
clock := quartz.NewMock(t)
tickerTrap := clock.Trap().NewTicker("agent_scripts", "dependency_wait")
defer tickerTrap.Close()
runs := newScriptOrderRuns()
runs.hold("coder_script.z_clone")
runs.hold("coder_script.a_auth")
logs := newScriptOrderTestLogs()
executor := newTestExecutor(t, runs, logs)
executor.clock = clock
zClone := scriptOrderTestScript("coder_script.z_clone")
aAuth := scriptOrderTestScript("coder_script.a_auth")
install := scriptOrderTestScript("coder_script.install",
scriptOrderTestDependency("coder_script.z_clone", requireSuccess),
scriptOrderTestDependency("coder_script.a_auth", requireCompletion),
)
scripts := []codersdk.WorkspaceAgentScript{zClone, aAuth, install}
manager, byAddress, err := lifecycleManager(scripts)
require.NoError(t, err)
done := make(chan error, 1)
go func() { done <- executor.executeWith(ctx, manager, byAddress, scripts, ExecuteStartScripts) }()
tickerTrap.MustWait(ctx).MustRelease(ctx)
clock.Advance(dependencyLogInterval).MustWait(ctx)
require.Equal(t, `Waiting for dependencies: "coder_script.a_auth", "coder_script.z_clone" (30s)`, logs.waitOutput(ctx, t, install.LogSourceID))
clock.Advance(dependencyLogInterval).MustWait(ctx)
require.Equal(t, `Waiting for dependencies: "coder_script.a_auth", "coder_script.z_clone" (60s)`, logs.waitOutput(ctx, t, install.LogSourceID))
runs.release("coder_script.a_auth", nil)
// The outcome is recorded after run returns, so wait for the record
// itself rather than for run.
require.Eventually(t, func() bool {
u, err := manager.Unit("coder_script.a_auth")
return err == nil && u.Outcome() == unit.OutcomeSucceeded
}, testutil.WaitShort, testutil.IntervalFast)
clock.Advance(dependencyLogInterval).MustWait(ctx)
require.Equal(t, `Waiting for dependencies: "coder_script.z_clone" (90s)`, logs.waitOutput(ctx, t, install.LogSourceID))
runs.release("coder_script.z_clone", nil)
runs.waitStarted(ctx, t, "coder_script.install")
require.NoError(t, testutil.RequireReceive(ctx, t, done))
require.Len(t, logs.outputs(install.LogSourceID), 3)
}
func TestLifecycleExecutorNoWaitingLineWhenDecidedEarly(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
clock := quartz.NewMock(t)
tickerTrap := clock.Trap().NewTicker("agent_scripts", "dependency_wait")
defer tickerTrap.Close()
runs := newScriptOrderRuns()
runs.hold("coder_script.clone")
logs := newScriptOrderTestLogs()
executor := newTestExecutor(t, runs, logs)
executor.clock = clock
clone := scriptOrderTestScript("coder_script.clone")
install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
done := runExecute(ctx, executor, clone, install)
// Only install waits, so exactly one ticker is created.
tickerTrap.MustWait(ctx).MustRelease(ctx)
runs.release("coder_script.clone", nil)
runs.waitStarted(ctx, t, "coder_script.install")
require.NoError(t, testutil.RequireReceive(ctx, t, done))
require.Empty(t, logs.outputs(clone.LogSourceID))
require.Empty(t, logs.outputs(install.LogSourceID))
}
func TestLifecycleExecutorCancellation(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(testutil.Context(t, testutil.WaitShort))
defer cancel()
runs := newScriptOrderRuns()
runs.hold("coder_script.a")
logs := newScriptOrderTestLogs()
reports := newScriptOrderReports()
executor := newTestExecutor(t, runs, logs)
executor.reportUnstarted = reports.record
a := scriptOrderTestScript("coder_script.a")
b := scriptOrderTestScript("coder_script.b", scriptOrderTestDependency("coder_script.a", requireSuccess))
c := scriptOrderTestScript("coder_script.c", scriptOrderTestDependency("coder_script.a", requireCompletion))
d := scriptOrderTestScript("coder_script.d")
scripts := []codersdk.WorkspaceAgentScript{a, b, c, d}
manager, byAddress, err := lifecycleManager(scripts)
require.NoError(t, err)
done := make(chan error, 1)
go func() { done <- executor.executeWith(ctx, manager, byAddress, scripts, ExecuteStartScripts) }()
runs.waitStarted(ctx, t, "coder_script.a")
runs.waitFinished(ctx, t, "coder_script.d")
cancel()
waitCtx := testutil.Context(t, testutil.WaitShort)
require.ErrorIs(t, testutil.RequireReceive(waitCtx, t, done), context.Canceled)
require.ElementsMatch(t, []string{"coder_script.a", "coder_script.d"}, runs.allStarted())
// a's own timing comes from run, which the fake does not model.
require.ElementsMatch(t, []scriptOrderReport{
{address: "coder_script.b", option: ExecuteStartScripts, status: proto.Timing_CANCELED},
{address: "coder_script.c", option: ExecuteStartScripts, status: proto.Timing_CANCELED},
}, reports.all(), "only the scripts that never started report from the executor")
require.Equal(t, []string{canceledRunningLine}, logs.outputs(a.LogSourceID))
require.Equal(t, []string{canceledBeforeStartLine}, logs.outputs(b.LogSourceID))
require.Equal(t, []string{canceledBeforeStartLine}, logs.outputs(c.LogSourceID))
require.Empty(t, logs.outputs(d.LogSourceID), "a finished script gets no line")
for address, want := range map[string]unit.Outcome{
"coder_script.a": unit.OutcomeCanceled,
"coder_script.b": unit.OutcomeCanceled,
"coder_script.c": unit.OutcomeCanceled,
"coder_script.d": unit.OutcomeSucceeded,
} {
u, err := manager.Unit(unit.ID(address))
require.NoError(t, err)
require.Equal(t, want, u.Outcome(), address)
}
}
func TestLifecycleExecutorRootScriptCanceledBeforeStarting(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(testutil.Context(t, testutil.WaitShort))
cancel()
runs := newScriptOrderRuns()
logs := newScriptOrderTestLogs()
reports := newScriptOrderReports()
executor := newTestExecutor(t, runs, logs)
executor.reportUnstarted = reports.record
root := scriptOrderTestScript("coder_script.root")
scripts := []codersdk.WorkspaceAgentScript{root}
manager, byAddress, err := lifecycleManager(scripts)
require.NoError(t, err)
require.ErrorIs(t, executor.executeWith(ctx, manager, byAddress, scripts, ExecuteStartScripts), context.Canceled)
require.Empty(t, runs.allStarted())
require.Equal(t, []scriptOrderReport{
{address: "coder_script.root", option: ExecuteStartScripts, status: proto.Timing_CANCELED},
}, reports.all())
require.Equal(t, []string{canceledBeforeStartLine}, logs.outputs(root.LogSourceID))
u, err := manager.Unit(unit.ID("coder_script.root"))
require.NoError(t, err)
require.Equal(t, unit.OutcomeCanceled, u.Outcome())
}
func TestLifecycleExecutorStopScripts(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
saveErr := xerrors.New("exit status 1")
runs := newScriptOrderRuns()
runs.result("coder_script.save_state", saveErr)
logs := newScriptOrderTestLogs()
reports := newScriptOrderReports()
executor := newTestExecutor(t, runs, logs)
executor.reportUnstarted = reports.record
save := scriptOrderTestScript("coder_script.save_state")
save.DisplayName = "Save state"
stopDB := scriptOrderTestScript("coder_script.stop_db", scriptOrderTestDependency("coder_script.save_state", requireSuccess))
report := scriptOrderTestScript("coder_script.report_stop", scriptOrderTestDependency("coder_script.save_state", requireCompletion))
err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{save, stopDB, report}, ExecuteStopScripts)
require.ErrorIs(t, err, saveErr)
require.ElementsMatch(t, []string{"coder_script.save_state", "coder_script.report_stop"}, runs.allStarted())
require.Equal(t, []string{`Skipping script: dependency "Save state" failed.`}, logs.outputs(stopDB.LogSourceID))
require.Equal(t, ExecuteStopScripts, runs.option("coder_script.report_stop"))
// The executor passes the option through; Runner.reportUnstarted is
// what drops stop reports (TestRunnerReportsUnstartedTiming).
require.Equal(t, []scriptOrderReport{
{address: "coder_script.stop_db", option: ExecuteStopScripts, status: proto.Timing_SKIPPED},
}, reports.all())
}
func TestLifecycleExecutorFreshManagerPerExecution(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
clone := scriptOrderTestScript("coder_script.clone")
install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
scripts := []codersdk.WorkspaceAgentScript{clone, install}
first := newScriptOrderRuns()
first.result("coder_script.clone", xerrors.New("exit status 128"))
executor := newTestExecutor(t, first, newScriptOrderTestLogs())
require.Error(t, executor.execute(ctx, scripts, ExecuteStartScripts))
require.False(t, first.started("coder_script.install"))
second := newScriptOrderRuns()
executor.run = second.run
require.NoError(t, executor.execute(ctx, scripts, ExecuteStartScripts))
require.True(t, second.started("coder_script.install"))
}
func TestLifecycleExecutorRecordsFailureOnUnrecordedExit(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
logs := newScriptOrderTestLogs()
executor := newTestExecutor(t, runs, logs)
clone := scriptOrderTestScript("coder_script.clone")
install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
scripts := []codersdk.WorkspaceAgentScript{clone, install}
manager, byAddress, err := lifecycleManager(scripts)
require.NoError(t, err)
// Pre-setting the outcome makes the executor's own update fail.
require.NoError(t, manager.UpdateOutcome("coder_script.clone", unit.OutcomeRunning))
err = executor.executeWith(ctx, manager, byAddress, scripts, ExecuteStartScripts)
require.ErrorIs(t, err, unit.ErrSameOutcomeAlreadySet)
require.NoError(t, ctx.Err())
require.Empty(t, runs.allStarted())
u, err := manager.Unit("coder_script.clone")
require.NoError(t, err)
require.Equal(t, unit.OutcomeFailed, u.Outcome())
require.Equal(t, []string{`Skipping script: dependency "coder_script.clone" failed.`}, logs.outputs(install.LogSourceID))
}
func TestLifecycleExecutorAddresslessScriptsRunAtOnce(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.hold("coder_script.clone")
executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
clone := scriptOrderTestScript("coder_script.clone")
install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
legacy := scriptOrderTestScript("")
devcontainer := scriptOrderTestScript("")
done := runExecute(ctx, executor, clone, install, legacy, devcontainer)
runs.waitStarted(ctx, t, "coder_script.clone")
runs.waitFinished(ctx, t, legacy.LogSourceID.String())
runs.waitFinished(ctx, t, devcontainer.LogSourceID.String())
runs.requireNotStarted(t, "coder_script.install")
runs.release("coder_script.clone", nil)
require.NoError(t, testutil.RequireReceive(ctx, t, done))
}
func TestLifecycleExecutorSkipIsNotAnError(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name string
cloneErr error
}{
{name: "Failure", cloneErr: xerrors.New("exit status 128")},
{name: "Timeout", cloneErr: ErrTimeout},
} {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.result("coder_script.clone", test.cloneErr)
executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
clone := scriptOrderTestScript("coder_script.clone")
install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
post := scriptOrderTestScript("coder_script.post", scriptOrderTestDependency("coder_script.clone", requireCompletion))
err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{clone, install, post}, ExecuteStartScripts)
require.ErrorIs(t, err, test.cloneErr)
// errgroup keeps the first error only, so a skip that returned
// an error could only show up as the whole result.
require.Equal(t, test.cloneErr.Error(), err.Error())
require.ElementsMatch(t, []string{"coder_script.clone", "coder_script.post"}, runs.allStarted())
})
}
}
func TestLifecycleExecutorRejectsInvalidDependenciesBeforeRunning(t *testing.T) {
t.Parallel()
noAddress := scriptOrderTestScript("", scriptOrderTestDependency("coder_script.clone", requireSuccess))
noAddress.DisplayName = "Install tools"
for _, test := range []struct {
name string
scripts []codersdk.WorkspaceAgentScript
wantErr string
}{
{
name: "DependenciesWithoutAddress",
scripts: []codersdk.WorkspaceAgentScript{
scriptOrderTestScript("coder_script.clone"),
noAddress,
},
wantErr: `script "Install tools" has dependencies but no Terraform resource address`,
},
{
name: "DuplicateAddress",
scripts: []codersdk.WorkspaceAgentScript{
scriptOrderTestScript("coder_script.setup"),
scriptOrderTestScript("coder_script.setup", scriptOrderTestDependency("coder_script.setup", requireSuccess)),
},
wantErr: `duplicate script resource address "coder_script.setup"`,
},
{
name: "MissingPrerequisite",
scripts: []codersdk.WorkspaceAgentScript{
scriptOrderTestScript("coder_script.install_tools", scriptOrderTestDependency("coder_script.clone", requireSuccess)),
},
wantErr: `script "coder_script.install_tools" depends on "coder_script.clone", which is not part of this lifecycle execution`,
},
{
name: "UnsupportedRequirement",
scripts: []codersdk.WorkspaceAgentScript{
scriptOrderTestScript("coder_script.clone_repo"),
scriptOrderTestScript("coder_script.install_tools", scriptOrderTestDependency("coder_script.clone_repo", "started")),
},
wantErr: `script "coder_script.install_tools" dependency on "coder_script.clone_repo": unsupported requirement "started"`,
},
{
name: "Cycle",
scripts: []codersdk.WorkspaceAgentScript{
scriptOrderTestScript("coder_script.a", scriptOrderTestDependency("coder_script.b", requireSuccess)),
scriptOrderTestScript("coder_script.b", scriptOrderTestDependency("coder_script.a", requireSuccess)),
},
wantErr: `add script dependency "coder_script.b" after "coder_script.a"`,
},
{
name: "ConflictingRequirement",
scripts: []codersdk.WorkspaceAgentScript{
scriptOrderTestScript("coder_script.clone_repo"),
scriptOrderTestScript("coder_script.build",
scriptOrderTestDependency("coder_script.clone_repo", requireSuccess),
scriptOrderTestDependency("coder_script.clone_repo", requireCompletion),
),
},
wantErr: `add script dependency "coder_script.build" after "coder_script.clone_repo"`,
},
} {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
logs := newScriptOrderTestLogs()
executor := newTestExecutor(t, runs, logs)
err := executor.execute(ctx, test.scripts, ExecuteStartScripts)
require.ErrorContains(t, err, test.wantErr)
switch test.name {
case "Cycle":
require.ErrorIs(t, err, unit.ErrCycleDetected)
case "ConflictingRequirement":
require.ErrorIs(t, err, unit.ErrConflictingRequirement)
}
require.Empty(t, runs.allStarted())
for _, script := range test.scripts {
require.Equal(t, []string{"Lifecycle scripts did not run: " + err.Error()}, logs.outputs(script.LogSourceID))
}
})
}
}
func TestLifecycleExecutorManyWaiters(t *testing.T) {
t.Parallel()
const waiters = 50
scripts := func() []codersdk.WorkspaceAgentScript {
all := []codersdk.WorkspaceAgentScript{scriptOrderTestScript("coder_script.clone")}
for i := range waiters {
all = append(all, scriptOrderTestScript(
fmt.Sprintf("coder_script.dependent_%02d", i),
scriptOrderTestDependency("coder_script.clone", requireSuccess),
))
}
return all
}
t.Run("AllRun", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
runs := newScriptOrderRuns()
runs.hold("coder_script.clone")
executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
done := runExecute(ctx, executor, scripts()...)
runs.waitStarted(ctx, t, "coder_script.clone")
require.Len(t, runs.allStarted(), 1)
runs.release("coder_script.clone", nil)
require.NoError(t, testutil.RequireReceive(ctx, t, done))
require.Len(t, runs.allStarted(), waiters+1)
})
t.Run("AllCanceled", func(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(testutil.Context(t, testutil.WaitShort))
defer cancel()
runs := newScriptOrderRuns()
runs.hold("coder_script.clone")
reports := newScriptOrderReports()
executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
executor.reportUnstarted = reports.record
all := scripts()
manager, byAddress, err := lifecycleManager(all)
require.NoError(t, err)
done := make(chan error, 1)
go func() { done <- executor.executeWith(ctx, manager, byAddress, all, ExecuteStartScripts) }()
runs.waitStarted(ctx, t, "coder_script.clone")
cancel()
waitCtx := testutil.Context(t, testutil.WaitShort)
require.ErrorIs(t, testutil.RequireReceive(waitCtx, t, done), context.Canceled)
require.Len(t, runs.allStarted(), 1)
for _, script := range all {
u, err := manager.Unit(unit.ID(script.ResourceAddress))
require.NoError(t, err)
require.Equal(t, unit.OutcomeCanceled, u.Outcome(), script.ResourceAddress)
}
var want []scriptOrderReport
for _, script := range all[1:] {
want = append(want, scriptOrderReport{address: script.ResourceAddress, option: ExecuteStartScripts, status: proto.Timing_CANCELED})
}
require.ElementsMatch(t, want, reports.all(), "one report per waiting script, none for the running one")
})
}
func newTestExecutor(t *testing.T, runs *scriptOrderRuns, logs *scriptOrderTestLogs) lifecycleExecutor {
t.Helper()
return lifecycleExecutor{
clock: quartz.NewReal(),
logger: testutil.Logger(t),
getScriptLogger: logs.logger,
run: runs.run,
}
}
func runExecute(ctx context.Context, executor lifecycleExecutor, scripts ...codersdk.WorkspaceAgentScript) <-chan error {
done := make(chan error, 1)
go func() { done <- executor.execute(ctx, scripts, ExecuteStartScripts) }()
return done
}
func scriptOrderTestScript(address string, dependencies ...codersdk.WorkspaceAgentScriptDependency) codersdk.WorkspaceAgentScript {
return codersdk.WorkspaceAgentScript{
ID: uuid.New(),
LogSourceID: uuid.New(),
ResourceAddress: address,
RunOnStart: true,
Dependencies: dependencies,
}
}
func scriptOrderTestDependency(address string, requirement codersdk.WorkspaceAgentScriptDependencyRequirement) codersdk.WorkspaceAgentScriptDependency {
return codersdk.WorkspaceAgentScriptDependency{
PrerequisiteResourceAddress: address,
Requirement: requirement,
}
}
// scriptOrderRuns is a fake run. A script's key is its resource address,
// or its log source ID when it has no address. A held script blocks until
// released; any other script returns its configured result at once.
type scriptOrderRuns struct {
mu sync.Mutex
held map[string]chan error
results map[string]error
startedCh map[string]chan struct{}
finishedCh map[string]chan struct{}
options map[string]ExecuteOption
order []string
}
func newScriptOrderRuns() *scriptOrderRuns {
return &scriptOrderRuns{
held: make(map[string]chan error),
results: make(map[string]error),
startedCh: make(map[string]chan struct{}),
finishedCh: make(map[string]chan struct{}),
options: make(map[string]ExecuteOption),
}
}
func scriptOrderKey(script codersdk.WorkspaceAgentScript) string {
if script.ResourceAddress != "" {
return script.ResourceAddress
}
return script.LogSourceID.String()
}
func (r *scriptOrderRuns) hold(key string) {
r.mu.Lock()
defer r.mu.Unlock()
r.held[key] = make(chan error, 1)
}
func (r *scriptOrderRuns) result(key string, err error) {
r.mu.Lock()
defer r.mu.Unlock()
r.results[key] = err
}
func (r *scriptOrderRuns) release(key string, err error) {
r.mu.Lock()
release := r.held[key]
r.mu.Unlock()
release <- err
}
func signal(m map[string]chan struct{}, key string) chan struct{} {
if _, ok := m[key]; !ok {
m[key] = make(chan struct{})
}
return m[key]
}
func (r *scriptOrderRuns) run(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
key := scriptOrderKey(script)
r.mu.Lock()
r.order = append(r.order, key)
r.options[key] = option
close(signal(r.startedCh, key))
release, held := r.held[key]
result := r.results[key]
finished := signal(r.finishedCh, key)
r.mu.Unlock()
defer close(finished)
if !held {
return result
}
select {
case err := <-release:
return err
case <-ctx.Done():
return ctx.Err()
}
}
func (r *scriptOrderRuns) waitStarted(ctx context.Context, t *testing.T, key string) {
t.Helper()
r.mu.Lock()
started := signal(r.startedCh, key)
r.mu.Unlock()
select {
case <-started:
case <-ctx.Done():
t.Fatalf("script %q did not start", key)
}
}
func (r *scriptOrderRuns) waitFinished(ctx context.Context, t *testing.T, key string) {
t.Helper()
r.mu.Lock()
finished := signal(r.finishedCh, key)
r.mu.Unlock()
select {
case <-finished:
case <-ctx.Done():
t.Fatalf("script %q did not finish", key)
}
}
func (r *scriptOrderRuns) requireNotStarted(t *testing.T, key string) {
t.Helper()
require.False(t, r.started(key), "script %q started too early", key)
}
func (r *scriptOrderRuns) started(key string) bool {
r.mu.Lock()
defer r.mu.Unlock()
started, ok := r.startedCh[key]
if !ok {
return false
}
select {
case <-started:
return true
default:
return false
}
}
func (r *scriptOrderRuns) allStarted() []string {
r.mu.Lock()
defer r.mu.Unlock()
return append([]string(nil), r.order...)
}
func (r *scriptOrderRuns) option(key string) ExecuteOption {
r.mu.Lock()
defer r.mu.Unlock()
return r.options[key]
}
// scriptOrderReports is a fake reportUnstarted that records each call.
type scriptOrderReports struct {
mu sync.Mutex
records []scriptOrderReport
}
type scriptOrderReport struct {
address string
option ExecuteOption
status proto.Timing_Status
}
func newScriptOrderReports() *scriptOrderReports {
return &scriptOrderReports{}
}
func (r *scriptOrderReports) record(_ context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption, status proto.Timing_Status) {
r.mu.Lock()
defer r.mu.Unlock()
r.records = append(r.records, scriptOrderReport{address: script.ResourceAddress, option: option, status: status})
}
func (r *scriptOrderReports) all() []scriptOrderReport {
r.mu.Lock()
defer r.mu.Unlock()
return append([]scriptOrderReport(nil), r.records...)
}
// scriptOrderTestLogs records every line sent to each script's log source.
type scriptOrderTestLogs struct {
mu sync.Mutex
byID map[uuid.UUID][]string
sent chan scriptOrderLog
}
type scriptOrderLog struct {
logSourceID uuid.UUID
output string
}
func newScriptOrderTestLogs() *scriptOrderTestLogs {
return &scriptOrderTestLogs{
byID: make(map[uuid.UUID][]string),
sent: make(chan scriptOrderLog, 256),
}
}
func (l *scriptOrderTestLogs) logger(logSourceID uuid.UUID) ScriptLogger {
return scriptOrderTestLogger{logs: l, logSourceID: logSourceID}
}
func (l *scriptOrderTestLogs) outputs(logSourceID uuid.UUID) []string {
l.mu.Lock()
defer l.mu.Unlock()
return append([]string(nil), l.byID[logSourceID]...)
}
// waitOutput blocks until the next line for the given log source arrives.
func (l *scriptOrderTestLogs) waitOutput(ctx context.Context, t *testing.T, logSourceID uuid.UUID) string {
t.Helper()
for {
log := testutil.RequireReceive(ctx, t, l.sent)
if log.logSourceID == logSourceID {
return log.output
}
}
}
type scriptOrderTestLogger struct {
logs *scriptOrderTestLogs
logSourceID uuid.UUID
}
func (l scriptOrderTestLogger) Send(_ context.Context, logs ...agentsdk.Log) error {
l.logs.mu.Lock()
defer l.logs.mu.Unlock()
for _, log := range logs {
l.logs.byID[l.logSourceID] = append(l.logs.byID[l.logSourceID], log.Output)
l.logs.sent <- scriptOrderLog{logSourceID: l.logSourceID, output: log.Output}
}
return nil
}
func (scriptOrderTestLogger) Flush(context.Context) error {
return nil
}