tests/Feature/DeploymentSensitiveCommandFailureTest.php
Rama main · réplica de lectura del repositorio original.
<?php
use App\Exceptions\DeploymentException;
use App\Jobs\ApplicationDeploymentJob;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\Environment;
use App\Models\EnvironmentVariable;
use App\Models\IntegrationToken;
use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Process\PendingProcess;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;
uses(RefreshDatabase::class);
const SENSITIVE_FAILURE_SECRET = 'harmless-secret-marker-7431';
class SensitiveCommandFailureDeploymentJob extends ApplicationDeploymentJob
{
public function __construct() {}
}
beforeEach(function () {
config()->set('constants.ssh.mux_enabled', false);
config()->set('constants.ssh.max_retries', 1);
});
/**
* @return array{0: SensitiveCommandFailureDeploymentJob, 1: ReflectionClass, 2: ApplicationDeploymentQueue, 3: Application}
*/
function makeSensitiveCommandFailureJob(array $environmentVariables = []): array
{
$user = User::factory()->create();
$team = $user->teams()->first();
$privateKeyContent = "-----BEGIN OPENSSH PRIVATE KEY-----\n"
."b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\n"
."QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk\n"
."hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA\n"
."AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV\n"
."uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==\n"
.'-----END OPENSSH PRIVATE KEY-----';
$privateKey = PrivateKey::create([
'name' => 'sensitive-failure-key-'.uniqid(),
'private_key' => $privateKeyContent,
'team_id' => $team->id,
]);
Storage::fake('ssh-keys');
Storage::disk('ssh-keys')->put("ssh_key@{$privateKey->uuid}", $privateKeyContent);
$server = Server::factory()->create([
'team_id' => $team->id,
'private_key_id' => $privateKey->id,
'user' => 'root',
]);
$project = Project::create(['name' => 'Sensitive Failure Project', 'team_id' => $team->id]);
$environment = Environment::where('project_id', $project->id)->firstOrFail();
$destination = $server->standaloneDockers()->firstOrFail();
$application = Application::factory()->create([
'environment_id' => $environment->id,
'destination_id' => $destination->id,
'destination_type' => $destination->getMorphClass(),
'build_pack' => 'dockerfile',
]);
$application->settings()->update([
'include_source_commit_in_build' => false,
'is_env_sorting_enabled' => false,
]);
foreach ($environmentVariables as $attributes) {
EnvironmentVariable::create([
'resourceable_type' => Application::class,
'resourceable_id' => $application->id,
'is_preview' => false,
'is_runtime' => true,
'is_buildtime' => true,
'is_multiline' => false,
'is_literal' => false,
...$attributes,
]);
}
$queue = ApplicationDeploymentQueue::create([
'deployment_uuid' => 'sensitive-failure-deployment',
'application_id' => $application->id,
'server_id' => $server->id,
'status' => 'in_progress',
]);
$job = new SensitiveCommandFailureDeploymentJob;
$reflection = new ReflectionClass(ApplicationDeploymentJob::class);
foreach ([
'application' => $application->fresh(),
'application_deployment_queue' => $queue,
'server' => $server,
'mainServer' => $server,
'build_pack' => 'dockerfile',
'pull_request_id' => 0,
'commit' => 'HEAD',
'basedir' => '/artifacts/sensitive-failure',
'workdir' => '/artifacts/sensitive-failure',
'configuration_dir' => '/data/coolify/applications/sensitive-failure',
'deployment_uuid' => 'sensitive-failure-deployment',
'dockerfile_location' => '/Dockerfile',
'container_name' => 'sensitive-failure-app',
'coolify_variables' => null,
'dockerSecretsSupported' => false,
'saved_outputs' => new Collection,
] as $property => $value) {
$reflectionProperty = $reflection->getProperty($property);
$reflectionProperty->setAccessible(true);
$reflectionProperty->setValue($job, $value);
}
return [$job, $reflection, $queue, $application];
}
function invokeSensitiveFailureJobMethod(object $job, ReflectionClass $reflection, string $method, mixed ...$arguments): mixed
{
$reflectionMethod = $reflection->getMethod($method);
$reflectionMethod->setAccessible(true);
return $reflectionMethod->invoke($job, ...$arguments);
}
function captureDeploymentException(callable $callback): DeploymentException
{
try {
$callback();
} catch (DeploymentException $exception) {
return $exception;
}
throw new RuntimeException('Expected a DeploymentException.');
}
/**
* @return list<string>
*/
function base64PayloadsInCommand(string $command): array
{
preg_match_all('~[A-Za-z0-9+/]{16,}={0,2}~', $command, $matches);
return array_values(array_filter(
$matches[0],
static fn (string $candidate): bool => str_contains((string) base64_decode($candidate, true), SENSITIVE_FAILURE_SECRET)
));
}
it('hides the .env write command and its base64 payload when the write fails', function () {
[$job, $reflection, $queue] = makeSensitiveCommandFailureJob([
['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET],
]);
$payloads = [];
Process::fake(function (PendingProcess $process) use (&$payloads) {
$payloads = array_merge($payloads, base64PayloadsInCommand(remoteProcessScript($process)));
return Process::result(errorOutput: 'tee: /artifacts/sensitive-failure/.env: No space left on device', exitCode: 1);
});
$exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables'));
expect($payloads)->not->toBeEmpty();
expect($exception->getMessage())
->toContain('Command execution failed (exit code 1): [command hidden because it contains sensitive data]')
->toContain('No space left on device')
->not->toContain(SENSITIVE_FAILURE_SECRET)
->not->toContain('base64 -d');
foreach ($payloads as $payload) {
expect($exception->getMessage())->not->toContain($payload);
}
$storedLogs = (string) $queue->fresh()->logs;
expect($storedLogs)
->toContain('No space left on device')
->not->toContain(SENSITIVE_FAILURE_SECRET);
foreach ($payloads as $payload) {
expect($storedLogs)->not->toContain($payload);
}
});
it('redacts the sensitive payload when the error output echoes the command', function () {
[$job, $reflection, $queue] = makeSensitiveCommandFailureJob([
['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET],
]);
$payloads = [];
Process::fake(function (PendingProcess $process) use (&$payloads) {
$commandPayloads = base64PayloadsInCommand(remoteProcessScript($process));
$payloads = array_merge($payloads, $commandPayloads);
return Process::result(errorOutput: "bash: line 1: echo '".($commandPayloads[0] ?? '')."': unexpected failure", exitCode: 2);
});
$exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables'));
expect($payloads)->not->toBeEmpty();
$storedLogs = (string) $queue->fresh()->logs;
foreach ($payloads as $payload) {
expect($exception->getMessage())->not->toContain($payload);
expect($storedLogs)->not->toContain($payload);
}
expect($exception->getMessage())->toContain('unexpected failure');
});
it('hides the build-time env write command when the write fails', function () {
[$job, $reflection] = makeSensitiveCommandFailureJob([
['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false],
]);
$payloads = [];
Process::fake(function (PendingProcess $process) use (&$payloads) {
$payloads = array_merge($payloads, base64PayloadsInCommand(remoteProcessScript($process)));
return Process::result(errorOutput: 'tee: /artifacts/build-time.env: No such file or directory', exitCode: 1);
});
$exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'save_buildtime_environment_variables'));
expect($payloads)->not->toBeEmpty();
expect($exception->getMessage())
->toContain('[command hidden because it contains sensitive data]')
->toContain('No such file or directory')
->not->toContain(SENSITIVE_FAILURE_SECRET);
foreach ($payloads as $payload) {
expect($exception->getMessage())->not->toContain($payload);
}
});
it('keeps the command in the error message for failed commands that are not sensitive', function () {
[$job] = makeSensitiveCommandFailureJob();
Process::fake(['*' => Process::result(errorOutput: 'ls: cannot access: No such file or directory', exitCode: 2)]);
$exception = captureDeploymentException(fn () => $job->execute_remote_command([
'command' => 'ls /artifacts/harmless-missing-directory',
'hidden' => true,
]));
expect($exception->getMessage())
->toContain('Command execution failed (exit code 2): ls /artifacts/harmless-missing-directory')
->toContain('No such file or directory')
->not->toContain('[command hidden because it contains sensitive data]');
});
it('explains how to fix a Docker network that Docker Engine reports with an IPv6 gateway prefix', function () {
[$job] = makeSensitiveCommandFailureJob();
Process::fake(['*' => Process::result(errorOutput: 'ParseAddr("fde4:75c5:b9d4::1/64"): unexpected character, want colon (at "/64")', exitCode: 1)]);
$exception = captureDeploymentException(fn () => $job->execute_remote_command([
'command' => 'docker compose up -d',
]));
expect($exception->getMessage())
->toContain('ParseAddr("fde4:75c5:b9d4::1/64")')
->toContain('systemctl restart docker')
->toContain('28.0.1');
});
it('does not add the Docker network hint to other failures', function () {
[$job] = makeSensitiveCommandFailureJob();
Process::fake(['*' => Process::result(errorOutput: 'ParseAddr("fde4:75c5:b9d4::1 "): unexpected character', exitCode: 1)]);
$exception = captureDeploymentException(fn () => $job->execute_remote_command([
'command' => 'docker compose up -d',
]));
expect($exception->getMessage())->not->toContain('systemctl restart docker');
});
it('reports the exit code and explains a lost SSH connection when a command ends with exit code 255', function () {
[$job] = makeSensitiveCommandFailureJob();
Process::fake(['*' => Process::result(errorOutput: '#26 exporting layers', exitCode: 255)]);
$exception = captureDeploymentException(fn () => $job->execute_remote_command([
'command' => 'docker buildx build .',
]));
expect($exception->getCode())->toBe(255)
->and($exception->getMessage())
->toContain('exit code 255')
->toContain('SSH connection to the server closed');
});
it('does not add the SSH connection hint to failures of the remote command', function () {
[$job] = makeSensitiveCommandFailureJob();
Process::fake(['*' => Process::result(errorOutput: 'build failed', exitCode: 1)]);
$exception = captureDeploymentException(fn () => $job->execute_remote_command([
'command' => 'docker buildx build .',
]));
expect($exception->getMessage())->not->toContain('SSH connection');
});
it('redacts locked values from the error output of failed commands', function () {
[$job] = makeSensitiveCommandFailureJob([
['key' => 'LOCKED_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_shown_once' => true],
]);
Process::fake(['*' => Process::result(errorOutput: 'failed with value '.SENSITIVE_FAILURE_SECRET, exitCode: 1)]);
$exception = captureDeploymentException(fn () => $job->execute_remote_command([
'command' => 'harmless-command --flag',
]));
expect($exception->getMessage())
->toContain('harmless-command --flag')
->toContain('failed with value '.REDACTED)
->not->toContain(SENSITIVE_FAILURE_SECRET);
});
it('logs only the variable names of the .env files in development mode', function () {
config()->set('app.env', 'local');
[$job, $reflection, $queue] = makeSensitiveCommandFailureJob([
['key' => 'APP_SECRET', 'value' => SENSITIVE_FAILURE_SECRET],
]);
$ranCommands = [];
Process::fake(function (PendingProcess $process) use (&$ranCommands) {
$ranCommands[] = remoteProcessScript($process);
return Process::result(output: str_contains(remoteProcessScript($process), 'cat ') ? 'APP_SECRET='.SENSITIVE_FAILURE_SECRET : '');
});
invokeSensitiveFailureJobMethod($job, $reflection, 'save_runtime_environment_variables');
invokeSensitiveFailureJobMethod($job, $reflection, 'save_buildtime_environment_variables');
$logEntries = collect(json_decode((string) $queue->fresh()->logs, true));
$keyListEntries = $logEntries->filter(fn (array $entry): bool => str_contains((string) $entry['output'], '[DEBUG] Variable names in'));
expect($keyListEntries)->toHaveCount(2)
->each(fn ($entry) => $entry->hidden->toBeTrue()->output->toContain('APP_SECRET'));
expect($logEntries->pluck('output')->merge($logEntries->pluck('command'))->filter()->implode("\n"))
->not->toContain('APP_SECRET='.SENSITIVE_FAILURE_SECRET)
->not->toContain('APP_SECRET="'.SENSITIVE_FAILURE_SECRET);
expect(collect($ranCommands)->filter(fn (string $command): bool => str_contains($command, 'cat /artifacts/sensitive-failure/.env') || str_contains($command, 'cat '.ApplicationDeploymentJob::BUILD_TIME_ENV_PATH)))
->toBeEmpty();
});
function setSensitiveFailureJobProperties(object $job, ReflectionClass $reflection, array $properties): void
{
foreach ($properties as $property => $value) {
$reflectionProperty = $reflection->getProperty($property);
$reflectionProperty->setAccessible(true);
$reflectionProperty->setValue($job, $value);
}
}
it('hides the helper container command when build secrets are passed as environment flags', function () {
[$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([
['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false],
]);
$application->settings()->update(['use_build_secrets' => true]);
$server = readSensitiveFailureJobProperty($job, $reflection, 'server');
setSensitiveFailureJobProperties($job, $reflection, [
'application' => $application->fresh(),
'destination' => StandaloneDocker::forceCreate([
'name' => 'sensitive-failure-network',
'network' => 'sensitive-failure-network',
'server_id' => $server->id,
]),
]);
Process::fake(function (PendingProcess $process) {
if (str_contains(remoteProcessScript($process), 'docker run -d')) {
return Process::result(errorOutput: 'docker: Error response from daemon: network not found.', exitCode: 125);
}
return Process::result(output: str_contains(remoteProcessScript($process), 'echo $HOME') ? '/root' : 'NOK');
});
$exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image'));
expect($exception->getMessage())
->toContain('[command hidden because it contains sensitive data]')
->toContain('network not found')
->not->toContain(SENSITIVE_FAILURE_SECRET);
expect((string) $queue->fresh()->logs)->not->toContain(SENSITIVE_FAILURE_SECRET);
});
it('starts the helper with its own DOCKER_CONFIG after build secret HOME and DOCKER_CONFIG variables', function () {
[$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([
['key' => 'HOME', 'value' => '/app', 'is_runtime' => false],
['key' => 'DOCKER_CONFIG', 'value' => '/app/.docker', 'is_runtime' => false],
]);
$application->settings()->update(['use_build_secrets' => true]);
$server = readSensitiveFailureJobProperty($job, $reflection, 'server');
setSensitiveFailureJobProperties($job, $reflection, [
'application' => $application->fresh(),
'destination' => $server->standaloneDockers()->firstOrFail(),
]);
$helperCommands = [];
Process::fake(function (PendingProcess $process) use (&$helperCommands) {
if (str_contains(remoteProcessScript($process), 'docker run -d')) {
$helperCommands[] = remoteProcessScript($process);
}
return Process::result(output: str_contains(remoteProcessScript($process), 'echo $HOME') ? '/root' : 'NOK');
});
invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image');
// docker run uses the last -e value of a variable, so the helper Docker CLI keeps /root/.docker/cli-plugins.
expect($helperCommands)->toHaveCount(1)
->and($helperCommands[0])->toContain('/app/.docker')
->and(strrpos($helperCommands[0], '-e DOCKER_CONFIG=/root/.docker'))->toBeGreaterThan(strpos($helperCommands[0], '/app/.docker'));
});
it('mounts the server CA bundle into the helper so the Docker CLI can fetch registry tokens', function (string $detectedBundle, ?string $expectedMount) {
[$job, $reflection] = makeSensitiveCommandFailureJob();
$server = readSensitiveFailureJobProperty($job, $reflection, 'server');
setSensitiveFailureJobProperties($job, $reflection, ['destination' => $server->standaloneDockers()->firstOrFail()]);
$helperCommands = [];
Process::fake(function (PendingProcess $process) use (&$helperCommands, $detectedBundle) {
$script = remoteProcessScript($process);
if (str_contains($script, 'docker run -d')) {
$helperCommands[] = $script;
}
return Process::result(output: match (true) {
str_contains($script, 'echo $HOME') => '/root',
str_contains($script, 'for f in /etc/ssl/certs/ca-certificates.crt') => $detectedBundle,
default => 'NOK',
});
});
invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image');
expect($helperCommands)->toHaveCount(1);
if ($expectedMount === null) {
expect($helperCommands[0])->not->toContain('coolify-host-ca-certificates.crt');
} else {
expect($helperCommands[0])->toContain($expectedMount);
}
})->with([
'Debian and Ubuntu bundle' => ['/etc/ssl/certs/ca-certificates.crt', '--mount type=bind,source=/etc/ssl/certs/ca-certificates.crt,target=/etc/ssl/certs/coolify-host-ca-certificates.crt,readonly'],
'RHEL bundle' => ['/etc/pki/tls/certs/ca-bundle.crt', '--mount type=bind,source=/etc/pki/tls/certs/ca-bundle.crt,target=/etc/ssl/certs/coolify-host-ca-certificates.crt,readonly'],
'no bundle' => ['', null],
'unknown output' => ['/root:/x -v /', null],
]);
it('shows the helper container command when no build secrets are passed', function () {
[$job, $reflection] = makeSensitiveCommandFailureJob();
$server = readSensitiveFailureJobProperty($job, $reflection, 'server');
setSensitiveFailureJobProperties($job, $reflection, ['destination' => $server->standaloneDockers()->firstOrFail()]);
Process::fake(function (PendingProcess $process) {
if (str_contains(remoteProcessScript($process), 'docker run -d')) {
return Process::result(errorOutput: 'docker: Error response from daemon: network not found.', exitCode: 125);
}
return Process::result(output: str_contains(remoteProcessScript($process), 'echo $HOME') ? '/root' : 'NOK');
});
$exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image'));
expect($exception->getMessage())
->toContain('docker run -d')
->not->toContain('-e DOCKER_CONFIG=')
->not->toContain('[command hidden because it contains sensitive data]');
});
it('hides the Railpack prepare command that passes build-time values', function () {
[$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([
['key' => 'BUILD_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_runtime' => false],
]);
$application->update(['build_pack' => 'railpack']);
setSensitiveFailureJobProperties($job, $reflection, [
'application' => $application->fresh(),
'build_pack' => 'railpack',
'dockerBuildxAvailable' => true,
]);
Process::fake(function (PendingProcess $process) {
if (str_contains(remoteProcessScript($process), 'railpack prepare')) {
return Process::result(errorOutput: 'railpack: failed to detect a provider', exitCode: 1);
}
return Process::result(output: str_contains(remoteProcessScript($process), 'buildx version') ? 'available' : 'missing');
});
$exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'build_railpack_image'));
expect($exception->getMessage())
->toContain('[command hidden because it contains sensitive data]')
->toContain('failed to detect a provider')
->not->toContain(SENSITIVE_FAILURE_SECRET);
expect((string) $queue->fresh()->logs)->not->toContain(SENSITIVE_FAILURE_SECRET);
});
it('hides the Nixpacks plan write command that contains build-time values', function (bool $isStatic) {
[$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob();
$application->update(['build_pack' => 'nixpacks']);
$application->settings()->update(['is_static' => $isStatic]);
setSensitiveFailureJobProperties($job, $reflection, [
'application' => $application->fresh(),
'build_pack' => 'nixpacks',
'build_args' => collect(),
'disableBuildCache' => false,
'force_rebuild' => false,
'nixpacks_plan' => json_encode(['variables' => ['BUILD_SECRET' => SENSITIVE_FAILURE_SECRET]]),
]);
$payloads = [];
Process::fake(function (PendingProcess $process) use (&$payloads) {
$commandPayloads = base64PayloadsInCommand(remoteProcessScript($process));
if ($commandPayloads !== []) {
$payloads = array_merge($payloads, $commandPayloads);
return Process::result(errorOutput: 'tee: /artifacts/thegameplan.json: No space left on device', exitCode: 1);
}
return Process::result();
});
$exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'build_image'));
expect($payloads)->not->toBeEmpty();
expect($exception->getMessage())
->toContain('[command hidden because it contains sensitive data]')
->not->toContain(SENSITIVE_FAILURE_SECRET);
$storedLogs = (string) $queue->fresh()->logs;
foreach ($payloads as $payload) {
expect($exception->getMessage())->not->toContain($payload);
expect($storedLogs)->not->toContain($payload);
}
})->with([
'static' => [true],
'not static' => [false],
]);
it('streams the compose file over stdin so its content stays out of commands, errors, and logs', function () {
[$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob();
$server = readSensitiveFailureJobProperty($job, $reflection, 'server');
setSensitiveFailureJobProperties($job, $reflection, [
'destination' => $server->standaloneDockers()->firstOrFail(),
'production_image_name' => 'example/app:latest',
]);
$composeWrite = null;
Process::fake(function (PendingProcess $process) use (&$composeWrite) {
if (str_contains((string) $process->input, 'services:')) {
$composeWrite = $process;
return Process::result(errorOutput: 'tee: docker-compose.yaml: No space left on device', exitCode: 1);
}
return Process::result();
});
invokeSensitiveFailureJobMethod($job, $reflection, 'generate_compose_file');
$exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'write_compose_file_to_workdir'));
$compose = (string) readSensitiveFailureJobProperty($job, $reflection, 'docker_compose');
expect($composeWrite)->not->toBeNull()
->and($composeWrite->input)->toBe($compose)
->and($composeWrite->command)->toEndWith(escapeshellarg('docker exec -i sensitive-failure-deployment tee /artifacts/sensitive-failure/docker-compose.yaml > /dev/null'))
->and($composeWrite->command)->not->toContain('services:')
->and($exception->getMessage())->toContain('No space left on device')
->and($exception->getMessage())->not->toContain('services:')
->and((string) $queue->fresh()->logs)->not->toContain('services:');
});
function readSensitiveFailureJobProperty(object $job, ReflectionClass $reflection, string $property): mixed
{
$reflectionProperty = $reflection->getProperty($property);
$reflectionProperty->setAccessible(true);
return $reflectionProperty->getValue($job);
}
it('redacts shell-escaped remote secrets in command logs and later deployment log entries', function () {
[$job, $reflection, $queue] = makeSensitiveCommandFailureJob();
$remoteSecrets = $reflection->getProperty('remote_secrets_cache');
$remoteSecrets->setValue($job, ['DB_PASS' => "s3cr'et-value"]);
$escapedSecret = "'s3cr'\\''et-value'";
Process::fake(fn () => Process::result(output: "DB_PASS={$escapedSecret}"));
invokeSensitiveFailureJobMethod($job, $reflection, 'execute_remote_command', ["echo DB_PASS={$escapedSecret}"]);
$queue->addLogEntry("Build argument DB_PASS={$escapedSecret}");
$storedLogs = collect(json_decode((string) $queue->fresh()->logs, true));
expect($storedLogs)->toHaveCount(2)
->and($storedLogs->toJson())->not->toContain('s3cr')
->and($storedLogs->pluck('output')->all())->each->toContain('DB_PASS='.REDACTED);
});
function linkSensitiveFailureSecretManager(Application $application): void
{
$token = IntegrationToken::query()->create([
'team_id' => $application->team()->id,
'provider' => 'doppler',
'name' => 'Doppler',
'token' => 'the-secret-token',
'capabilities' => ['secrets'],
]);
$application->secretManagerLink()->create([
'integration_token_id' => $token->id,
'settings' => ['project' => 'app', 'config' => 'prd'],
]);
}
it('redacts fetched remote secrets in log entries the deployment adds before its next remote command', function () {
[$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob();
linkSensitiveFailureSecretManager($application);
Http::fake(['https://api.doppler.com/*' => Http::response(['DB_PASS' => 'fetched-remote-secret-5521'])]);
invokeSensitiveFailureJobMethod($job, $reflection, 'remote_secrets');
$queue->addLogEntry('Resolved DB_PASS=fetched-remote-secret-5521');
$storedLogs = (string) $queue->fresh()->logs;
expect($storedLogs)->not->toContain('fetched-remote-secret-5521')
->toContain('DB_PASS='.REDACTED);
});
it('keeps a literal vault reference as text when the application has no secret manager source', function () {
[$job, $reflection, $queue] = makeSensitiveCommandFailureJob([
['key' => 'LITERAL_REF', 'value' => '{{vault.FOO}}', 'is_literal' => true],
]);
Http::fake();
$runtime = invokeSensitiveFailureJobMethod($job, $reflection, 'generate_runtime_environment_variables');
$buildtime = invokeSensitiveFailureJobMethod($job, $reflection, 'generate_buildtime_environment_variables');
invokeSensitiveFailureJobMethod($job, $reflection, 'generate_env_variables');
expect($runtime->first(fn (string $line): bool => str_starts_with($line, 'LITERAL_REF=')))->toBe('LITERAL_REF='.escapeComposeEnvFileValue('{{vault.FOO}}'))
->and($buildtime->first(fn (string $line): bool => str_starts_with($line, 'LITERAL_REF=')))->toBe("LITERAL_REF='{{vault.FOO}}'")
->and(readSensitiveFailureJobProperty($job, $reflection, 'env_args')->get('LITERAL_REF'))->toBe("'{{vault.FOO}}'")
->and(invokeSensitiveFailureJobMethod($job, $reflection, 'has_remote_buildtime_secret_references'))->toBeFalse();
Http::assertNothingSent();
});
it('still fails a deployment when a non-literal vault reference has no secret manager source', function () {
[$job, $reflection] = makeSensitiveCommandFailureJob([
['key' => 'REMOTE_REF', 'value' => '{{vault.FOO}}', 'is_literal' => false],
]);
$exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'generate_runtime_environment_variables'));
expect($exception->getMessage())->toContain('no secret manager source is configured');
});
it('resolves a literal vault reference when the application has a secret manager source', function () {
[$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob([
['key' => 'LITERAL_REF', 'value' => '{{vault.FOO}}', 'is_literal' => true],
]);
linkSensitiveFailureSecretManager($application);
Http::fake(['https://api.doppler.com/*' => Http::response(['FOO' => 'resolved-remote-value'])]);
$runtime = invokeSensitiveFailureJobMethod($job, $reflection, 'generate_runtime_environment_variables');
expect($runtime->first(fn (string $line): bool => str_starts_with($line, 'LITERAL_REF=')))->toBe('LITERAL_REF='.escapeComposeEnvFileValue('resolved-remote-value'));
});
it('tells how to log in when the build server has no Docker config for the SSH user', function () {
[$job, $reflection] = makeSensitiveCommandFailureJob();
$server = readSensitiveFailureJobProperty($job, $reflection, 'server');
$server->forceFill(['user' => 'coolify', 'name' => 'build-1'])->save();
setSensitiveFailureJobProperties($job, $reflection, ['server' => $server->fresh(), 'use_build_server' => true]);
Process::fake(fn (PendingProcess $process) => Process::result(output: str_contains(remoteProcessScript($process), 'echo $HOME') ? '/home/coolify' : 'NOK'));
$exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image'));
expect($exception->getMessage())
->toContain('of the SSH user (coolify) not found on the build server (build-1)')
->toContain('Server → Docker Registries')
->toContain('"sudo docker login" is saved for root');
Process::assertNotRan(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), 'docker run -d'));
});
it('mounts the seccomp profile from custom docker options into the helper container', function () {
[$job, $reflection, $queue, $application] = makeSensitiveCommandFailureJob();
$application->update(['custom_docker_run_options' => '--security-opt seccomp=/etc/docker/seccomp-profile.json']);
$server = readSensitiveFailureJobProperty($job, $reflection, 'server');
setSensitiveFailureJobProperties($job, $reflection, [
'application' => $application->fresh(),
'destination' => StandaloneDocker::forceCreate([
'name' => 'seccomp-network',
'network' => 'seccomp-network',
'server_id' => $server->id,
]),
]);
$helperCommands = [];
Process::fake(function (PendingProcess $process) use (&$helperCommands) {
if (str_contains(remoteProcessScript($process), 'docker run -d')) {
$helperCommands[] = remoteProcessScript($process);
return Process::result(errorOutput: 'docker: Error response from daemon: invalid mount config for type "bind": bind source path does not exist: /etc/docker/seccomp-profile.json', exitCode: 125);
}
return Process::result(output: str_contains(remoteProcessScript($process), 'echo $HOME') ? '/root' : 'NOK');
});
$exception = captureDeploymentException(fn () => invokeSensitiveFailureJobMethod($job, $reflection, 'prepare_builder_image'));
expect($helperCommands)->toHaveCount(1)
->and($helperCommands[0])->toContain('type=bind,source=/etc/docker/seccomp-profile.json,target=/etc/docker/seccomp-profile.json,readonly')
->and($exception->getMessage())->toContain('bind source path does not exist: /etc/docker/seccomp-profile.json');
});
it('removes the old helper synchronously before it starts the helper again with the same name', function () {
[$job, $reflection] = makeSensitiveCommandFailureJob();
$server = readSensitiveFailureJobProperty($job, $reflection, 'server');
setSensitiveFailureJobProperties($job, $reflection, ['destination' => $server->standaloneDockers()->firstOrFail()]);
$commands = [];
Process::fake(function (PendingProcess $process) use (&$commands) {
$commands[] = remoteProcessScript($process);
return Process::result(output: str_contains(remoteProcessScript($process), 'echo $HOME') ? '/root' : 'NOK');
});
invokeSensitiveFailureJobMethod($job, $reflection, 'restart_builder_container_with_actual_commit');
$runIndex = collect($commands)->search(fn (string $command) => str_contains($command, 'docker run -d'));
$beforeRun = collect($commands)->take($runIndex);
// "docker stop" lets the --rm auto-removal hold the name after it returns; "docker rm -f" frees the name before it returns.
expect($runIndex)->toBeInt()
->and($beforeRun->contains(fn (string $command) => str_contains($command, 'docker rm -f') && str_contains($command, 'sensitive-failure-deployment')))->toBeTrue()
->and($beforeRun->contains(fn (string $command) => str_contains($command, 'docker stop')))->toBeFalse();
});