docker-compose.byo-postgres.yml
Rama main · réplica de lectura del repositorio original.
# ─────────────────────────────────────────────────────────────────────────────
# Bring-your-own PostgreSQL — drop the bundled metadata database.
#
# docker compose -f docker-compose.quickstart.yml -f docker-compose.byo-postgres.yml up -d
#
# Requires the connection details in .env, e.g.:
# POSTGRES_HOST=my-instance.abc123.us-east-1.rds.amazonaws.com
# POSTGRES_PORT=5432
# POSTGRES_USER=rsync
# POSTGRES_PASSWORD=...
# POSTGRES_DB=pipeline_db
# POSTGRES_SSLMODE=require
# POSTGRES_TLS_ENABLED=true # Temporal's own TLS switch, see below
#
# This is the metadata database: pipeline definitions, run history, and the
# encrypted connection credentials. Pipeline *rows* never land here. Combine
# freely with docker-compose.byo-kafka.yml — the two overlays are independent.
#
# ── What you must create first ───────────────────────────────────────────────
# One thing: the ROLE named by POSTGRES_USER, with a password matching
# POSTGRES_PASSWORD and the right to create databases.
#
# CREATE ROLE rsync LOGIN PASSWORD '…' CREATEDB;
#
# Nothing in this file can create that role, because everything in it
# authenticates AS it. The databases are no longer yours to create: `db-init`
# below makes POSTGRES_DB and Temporal's two, plus the two extensions the
# migrations need, before any service that reads them starts. It is idempotent,
# so bringing the stack up again is a no-op.
#
# If your instance's application role cannot hold CREATEDB, create those three
# databases as an admin and db-init will find them and move on. What it will
# NOT do is treat "permission denied" and "already exists" as the same outcome:
# it fails loudly and prints the grant to run. Per-provider caveats (Azure's
# azure.extensions allow-list, Cloud SQL's cloudsqlsuperuser):
# docs/deployment/self-hosting.md § Bring Your Own PostgreSQL.
#
# ── TLS is not one switch ────────────────────────────────────────────────────
# POSTGRES_SSLMODE covers `db-init` below and the three Go services
# (api-gateway's DSN, orchestrator's DB_SSLMODE, temporal-adapter's
# POSTGRES_SSLMODE). Temporal ignores it, and Temporal is really two programs:
# auto-setup runs temporal-sql-tool to create the schemas, then starts the
# server, and they read DIFFERENT env names for the same fact. The base file
# feeds both from one POSTGRES_TLS_* key each, so you set these once:
#
# POSTGRES_TLS_ENABLED -> SQL_TLS_ENABLED (server) + SQL_TLS (schema tool)
# POSTGRES_TLS_CA_FILE -> SQL_CA + SQL_TLS_CA_FILE
# POSTGRES_TLS_SERVER_NAME -> SQL_HOST_NAME + SQL_TLS_SERVER_NAME
# POSTGRES_TLS_VERIFY_HOST -> SQL_HOST_VERIFICATION
# POSTGRES_TLS_SKIP_HOST_VERIFY -> SQL_TLS_DISABLE_HOST_VERIFICATION (inverse!)
#
# Set sslmode alone against a database that mandates TLS and the schema step
# fails before the server ever binds — no workflow engine, every pipeline hangs,
# and no TLS error in any log to point at the cause.
#
# ── How the removal works, and why it is spelled this way ────────────────────
# Compose overlays merge; they cannot delete a service. Assigning a profile that
# is never activated is the supported way to exclude one, because a service in a
# non-active profile is absent from the rendered project entirely.
#
# That half alone is a trap. A service excluded by profile is still named by
# `depends_on` in the base file, and Compose rejects the whole project with
# "depends on undefined service postgres" — nothing starts, which is strictly
# worse than the waste this overlay removes. The pairing fix lives in the base
# file: every `depends_on: postgres` there carries `required: false`. Both
# halves are needed; either one alone takes the stack down.
#
# Do not copy that flag onto the three `depends_on: db-init` entries below. It
# reads like the house style here, and it would quietly undo this whole file:
# measured on Compose 2.x, a FAILED `service_completed_successfully` dependency
# marked `required: false` degrades to the warning `optional dependency "…"
# didn't complete successfully: exit 7`, and the dependent starts anyway with
# `up` still exiting 0. That is the silent start on an empty schema db-init
# exists to prevent. The flag is for a service that may be absent from the
# project; db-init is defined right here and never is.
#
# ── What still runs ──────────────────────────────────────────────────────────
# `demo-warehouse` is a different service and is deliberately NOT excluded. It
# is the sample destination behind "Start with sample data" — a throwaway
# Postgres holding demo rows, nothing to do with the metadata database. Drop it
# by removing the demo profile, not with this overlay.
# ─────────────────────────────────────────────────────────────────────────────
services:
postgres:
# Never activated => the bundled database is excluded from the project.
profiles: ["bundled-db-disabled"]
# ─── DATABASE BOOTSTRAP ──────────────────────────────────────────────────────
#
# The bundled path creates three databases without anyone noticing, through two
# upstream images: the postgres image's initdb makes POSTGRES_DB, and Temporal's
# auto-setup makes its own two. Excluding `postgres` above removes the first and
# SKIP_DB_CREATE below removes the second, so on this path NOTHING creates them
# — and until this service existed, that gap was fifty lines of instructions in
# this header, enforced by nothing.
#
# It is worth being exact about why the gap had to be closed with code rather
# than prose: two of the three failures are silent. Temporal's is loud (its
# entrypoint is `auto-setup.sh && start-temporal.sh` under `set -e`, so the
# container crash-loops). POSTGRES_DB's is not. A failed `db.Init` is a warning
# and not an exit (main.go:306-308), and the migration runner sits inside that
# call's success branch — so no schema is created and the gateway serves
# regardless. Missing extensions fail the same way one step later: the runner
# returns on the first failing file (migrate.go:126-129) and 001_init_schema.sql
# is the first, needing uuid-ossp.
#
# Nothing in this project observes either one. api-gateway and orchestrator
# declare no healthcheck at all — of the services here only kafka, postgres,
# redis, minio, temporal, llm-service, kafka-connect and demo-warehouse do — so
# `docker compose ps` reports both as plain `running` over an empty database.
# `/ready` is the endpoint that knows (ready.go:15-25 returns `db_not_connected`
# then `schema_not_migrated`), and install.sh polls it for exactly this reason
# (install.sh:744-754). A hand-run `docker compose up` polls nothing.
#
# Mirrors the chart's templates/jobs/db-init.yaml, which does the same work as
# a pre-install hook, on the two paths where the chart renders it: external
# Postgres with dbInit left on and IAM auth off.
# The two are hand-kept in step for the same reason kafka-init's script is
# inline: install.sh downloads compose files alone, with no repo checkout, so
# there is no shared script on the host to call.
db-init:
image: postgres:16-alpine
container_name: rsync-ai-db-init
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
# Run-once job: one psql client, a handful of statements.
mem_limit: 256m
environment:
PGHOST: ${POSTGRES_HOST:-postgres}
PGPORT: ${POSTGRES_PORT:-5432}
PGUSER: ${POSTGRES_USER:-rsync}
PGPASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
# The database this connects TO in order to run CREATE DATABASE — never one
# of the databases it creates. `postgres` exists on RDS, Cloud SQL and Azure
# Database alike; override it only for an instance that dropped it.
PGDATABASE: ${POSTGRES_MAINTENANCE_DB:-postgres}
PGSSLMODE: ${POSTGRES_SSLMODE:-disable}
# Not PGSSLROOTCERT directly: libpq reads an EMPTY PGSSLROOTCERT as a
# filename and fails to open it, so under verify-ca an unset CA would break
# the connection rather than fall back to the system store. The script
# exports it only when non-empty. Note that nothing here mounts the file —
# if you set this, give this service the same bind mount you gave `temporal`.
POSTGRES_TLS_CA_FILE: ${POSTGRES_TLS_CA_FILE:-}
# Fail fast per attempt; the retry loop below supplies the patience.
PGCONNECT_TIMEOUT: "10"
APP_DATABASE: ${POSTGRES_DB:-pipeline_db}
# auto-setup's own defaults, and the SAME two names `temporal` is given
# explicitly below. A name that the creator and the consumer each hardcode
# separately is a name that can drift.
TEMPORAL_DATABASE: ${TEMPORAL_DB:-temporal}
TEMPORAL_VISIBILITY_DATABASE: ${TEMPORAL_VISIBILITY_DB:-temporal_visibility}
# /bin/sh, not /bin/bash: the image is Alpine and has no bash.
entrypoint: ["/bin/sh", "-c"]
# Single-element list, not a bare block scalar: Compose splits a string
# command into argv, which would leave `sh -c` running only the first word.
# `docker compose config` must show ONE item.
command:
- |
set -u;
if [ -n "$${POSTGRES_TLS_CA_FILE:-}" ]; then
export PGSSLROOTCERT="$$POSTGRES_TLS_CA_FILE";
fi;
# A managed instance that is still provisioning answers the port before
# it accepts connections. The error is captured rather than discarded
# because the failures that land here read identically from outside — a
# wrong password and an unreachable host both just end this loop — and
# the operator needs the server's own words to tell them apart.
ERR="";
n=0;
until ERR="$$(psql -w -Atc 'SELECT 1' 2>&1)"; do
n=$$((n + 1));
if [ "$$n" -ge 30 ]; then
echo "FATAL: cannot reach PostgreSQL at $$PGHOST:$$PGPORT as \"$$PGUSER\" after 60s." >&2;
echo " Last error from the server: $$ERR" >&2;
echo " Checked in this order:" >&2;
echo " * POSTGRES_HOST / POSTGRES_PORT reach the instance" >&2;
echo " * the role \"$$PGUSER\" exists and its password matches" >&2;
echo " POSTGRES_PASSWORD (nothing here creates the role it logs in as)" >&2;
echo " * POSTGRES_MAINTENANCE_DB=\"$$PGDATABASE\" exists and \"$$PGUSER\"" >&2;
echo " may connect to it" >&2;
echo " * POSTGRES_SSLMODE=\"$$PGSSLMODE\" matches what the server requires" >&2;
exit 1;
fi;
echo "waiting for postgres at $$PGHOST:$$PGPORT ($$n/30): $$ERR";
sleep 2;
done;
echo "connected to $$PGHOST:$$PGPORT as \"$$PGUSER\" (maintenance database \"$$PGDATABASE\")";
# CREATE DATABASE has no IF NOT EXISTS and cannot run inside a
# transaction, so idempotency is an existence check against pg_database
# rather than a swallowed error. Swallowing would be worse than it looks:
# "permission denied to create database" and "already exists" would then
# be the same outcome, and the first has to stay fatal.
#
# No OWNER clause. The creating role owns what it creates, and naming an
# owner is how this would break on a managed instance whose admin role is
# not POSTGRES_USER.
mkdb() {
if [ "$$(psql -w -Atc "SELECT 1 FROM pg_database WHERE datname = '$$1'")" = "1" ]; then
echo " database \"$$1\" already exists";
return 0;
fi;
if ! OUT="$$(psql -w -v ON_ERROR_STOP=1 -c "CREATE DATABASE \"$$1\"" 2>&1)"; then
if [ "$$(psql -w -Atc "SELECT 1 FROM pg_database WHERE datname = '$$1'")" = "1" ]; then
echo " database \"$$1\" was created concurrently";
return 0;
fi;
echo "FATAL: could not create database \"$$1\"." >&2;
echo " $$OUT" >&2;
echo " The role \"$$PGUSER\" needs CREATEDB:" >&2;
echo " ALTER ROLE \"$$PGUSER\" CREATEDB;" >&2;
echo " On a managed instance grant its admin role instead (RDS:" >&2;
echo " rds_superuser; Azure: azure_pg_admin; Cloud SQL:" >&2;
echo " cloudsqlsuperuser). Or create the three databases yourself —" >&2;
echo " this finds them and moves on." >&2;
exit 1;
fi;
echo " database \"$$1\" created";
};
mkdb "$$APP_DATABASE";
mkdb "$$TEMPORAL_DATABASE";
mkdb "$$TEMPORAL_VISIBILITY_DATABASE";
# In the APPLICATION database only. Temporal's two need no extensions,
# and creating one inside a database whose schema another product owns is
# not ours to do.
#
# Fatal, not best-effort. Both are trusted extensions on PostgreSQL 13+,
# so a database owner can create them without superuser — but a managed
# instance can still refuse (Azure Flexible Server permits only what
# `azure.extensions` lists). Continuing past that refusal hands it to the
# migration runner, which stops at the first failing file and logs one
# line while the process keeps serving: the whole schema is then missing
# and nothing says why.
mkext() {
if ! OUT="$$(psql -w -v ON_ERROR_STOP=1 -d "$$APP_DATABASE" -c "CREATE EXTENSION IF NOT EXISTS \"$$1\"" 2>&1)"; then
echo "FATAL: could not create extension \"$$1\" in \"$$APP_DATABASE\"." >&2;
echo " $$OUT" >&2;
echo " $$2" >&2;
echo " Both are trusted extensions on PostgreSQL 13+, so the database" >&2;
echo " owner may create them — but a managed instance can still refuse." >&2;
echo " Azure Database for PostgreSQL needs the extension allowlisted in" >&2;
echo " the azure.extensions server parameter before this succeeds." >&2;
exit 1;
fi;
echo " extension \"$$1\" ready";
};
mkext "uuid-ossp" "001_init_schema.sql calls uuid_generate_v4() and is the FIRST migration, so without it NO table is created.";
mkext "pg_trgm" "020b_workspace_connection_refs.sql builds the gin_trgm_ops indexes.";
echo "✅ Databases and extensions ready"
restart: "no"
temporal:
depends_on:
db-init:
condition: service_completed_successfully
environment:
# db-init above creates the two databases, so auto-setup must not try.
# Its create is unconditional: the only guard is the name test
# `[[ ${DBNAME} != ${POSTGRES_USER} && ${SKIP_DB_CREATE} != true ]]`, so
# pre-existence does not stop it, and it runs as POSTGRES_USER, which on a
# managed instance normally lacks CREATEDB. The bundled path does not need
# this (there POSTGRES_USER is the superuser), which is why it lives in
# this overlay: it must not change the bundled-database behaviour.
SKIP_DB_CREATE: "true"
# auto-setup's own defaults, set explicitly so that the creator above and
# the consumer here read ONE pair of names instead of two hardcoded copies.
DBNAME: ${TEMPORAL_DB:-temporal}
VISIBILITY_DBNAME: ${TEMPORAL_VISIBILITY_DB:-temporal_visibility}
orchestrator:
depends_on:
db-init:
condition: service_completed_successfully
api-gateway:
depends_on:
db-init:
condition: service_completed_successfully