docker-compose.quickstart.yml
Rama main · réplica de lectura del repositorio original.
#
# rsync.ai — Quickstart (single-command install)
#
# Usage:
# docker compose -f docker-compose.quickstart.yml --env-file .env up -d
#
# All images are pre-built and pulled from GHCR — no source code required.
# Requires: Docker 24+, docker compose v2, and an .env file (see install.sh).
#
# Every first-party image is `${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/<name>:
# ${RSYNC_VERSION:-latest}`: both halves are one variable each, so a mirror, an
# air-gapped registry or a pre-release build moves every image together. The
# same variable name is the Kubernetes installer's (install-k8s.sh).
#
# ── Sizing ───────────────────────────────────────────────────────────────────
# Targets a 4 vCPU / 16 GiB host. Every service carries an explicit `mem_limit`,
# which is doing two jobs: it stops one runaway container from taking the host
# down via the OOM killer (which picks the largest RSS on the box — Kafka or
# Postgres — not the container at fault), and it is what makes the orchestrator's
# own memory-aware sizing engage at all. resolveTableConcurrencyWithReason() reads the
# CGROUP limit (backend-orchestrator/internal/agents/executor/table_concurrency.go
# detectMemoryLimitBytes); with no limit set, that read returns "unlimited" and
# the memory guard is dead code, so a 4-core box fans out to 8 concurrent table
# loads budgeted at ~384 MiB each.
#
# These are CEILINGS, not reservations: nothing is pre-allocated, and the sum
# deliberately exceeds host RAM (~13.8 GiB default, ~16.8 GiB with the cdc
# profile) because no two services peak together. Steady state for the default
# profile is far lower. The number that is NOT slack is the orchestrator's 2 GiB
# — that one is a real buffer budget, since concurrent table loads are sized
# against it. Shrink it and throughput drops; the stack will not fall over.
#
# ── Optional profiles ────────────────────────────────────────────────────────
# The default `up` starts the batch data plane only. Two opt-in profiles:
#
# --profile cdc change-data-capture: kafka-connect + debezium-mcp.
# ~2 GiB, and REQUIRED for any CDC pipeline — the
# orchestrator's preflight hard-requires both plus
# kafka-mcp-sink (internal/workers/infra_preflight.go
# requiredServices), so without this profile a CDC
# pipeline fails preflight rather than degrading.
# kafka-mcp-sink is NOT in the profile: batch runs
# write through it too, so it always starts.
# --profile generate context7-mcp, API-doc research used only while GENERATING
# a new connector. Existing connectors run without it.
# --profile connectors sample-data-mcp pre-started. Off by default: the first
# connection that needs it (or the first "Try the demo")
# deploys it on demand through connector-deployer, as it
# already does postgresql/mysql/aws-s3 and every other
# catalog connector. Turn it on only to skip that first wait.
#
# docker compose -f docker-compose.quickstart.yml --profile cdc up -d
#
name: rsync-ai
# ─────────────────────────────────────────────────────────────────────────────
# Kafka client security — BYO / managed broker (MSK, Confluent Cloud, Aiven, …)
# ─────────────────────────────────────────────────────────────────────────────
# Merged into every service that opens a Kafka client. It exists because Compose
# injects ONLY what a service declares: a KAFKA_SASL_PASSWORD sitting in the
# root .env reaches interpolation but NOT the container, so the operator sets
# credentials, the app connects anonymously, and an ACL-enforcing broker rejects
# it with an error naming the broker rather than the missing setting.
#
# Every value defaults to empty, and empty is read as unset
# (shared/go/kafkaclient/config.go:240 protocolWasSet) — so with none of these in
# your .env the rendered config is byte-identical to the bundled-broker default.
#
# Minimum for SASL/SCRAM against a managed cluster:
# KAFKA_BROKERS=b-1.example.amazonaws.com:9096,b-2.example.amazonaws.com:9096
# KAFKA_SECURITY_PROTOCOL=SASL_SSL
# KAFKA_SASL_MECHANISM=SCRAM-SHA-512
# KAFKA_SASL_USERNAME=... / KAFKA_SASL_PASSWORD=...
#
# Setting the credentials WITHOUT KAFKA_SECURITY_PROTOCOL is the common slip; the
# client logs "SASL settings are configured … they are IGNORED" once and connects
# anonymously (config.go:281-297). Read that warning as a hard failure.
x-kafka-security: &kafka-security
KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-}
KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-}
KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-}
KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-}
# OIDC client-credentials, for KAFKA_SASL_MECHANISM=OAUTHBEARER.
KAFKA_SASL_OAUTHBEARER_CLIENT_ID: ${KAFKA_SASL_OAUTHBEARER_CLIENT_ID:-}
KAFKA_SASL_OAUTHBEARER_CLIENT_SECRET: ${KAFKA_SASL_OAUTHBEARER_CLIENT_SECRET:-}
KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT: ${KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT:-}
KAFKA_SASL_OAUTHBEARER_SCOPE: ${KAFKA_SASL_OAUTHBEARER_SCOPE:-}
KAFKA_SASL_OAUTHBEARER_EXTENSIONS: ${KAFKA_SASL_OAUTHBEARER_EXTENSIONS:-}
# Re-permits a plain-http token endpoint off loopback, which every runtime
# otherwise refuses at start-up (the client-credentials grant POSTs
# CLIENT_SECRET on every fetch). Carried here for ONE reason: kafka-connect
# and the kafka-init job read their whole security profile from this anchor,
# so without a passthrough a rig with a throwaway http IdP can configure the
# Go and Python services but not the JVM ones -- the same image behaving
# differently under compose than under the chart, which does make it
# reachable. The default is empty, i.e. unset, i.e. the refusal stays armed;
# a passthrough is not a setting. Never give it a value in this file --
# llm-service/tests/test_insecure_token_endpoint_rule_is_one_rule.py fails if
# any file but the Kafka security matrix harness sets it to true.
KAFKA_SASL_OAUTHBEARER_ALLOW_INSECURE_TOKEN_ENDPOINT: ${KAFKA_SASL_OAUTHBEARER_ALLOW_INSECURE_TOKEN_ENDPOINT:-}
# JVM-only, and only the CDC profile has a JVM in it (kafka-connect). Carried
# here anyway because Debezium's schema-history client is configured from the
# connector container's environment, so the name has to survive the trip.
KAFKA_SASL_OAUTHBEARER_LOGIN_CALLBACK_HANDLER: ${KAFKA_SASL_OAUTHBEARER_LOGIN_CALLBACK_HANDLER:-}
# AWS_MSK_IAM. The Go services implement it (kafkaclient/tokenauth/msk.go); the
# Python tier and the JVM Connect image do not, so an IAM-only cluster runs the
# Go data plane and leaves the CDC profile unable to authenticate.
KAFKA_AWS_REGION: ${KAFKA_AWS_REGION:-}
# Paths INSIDE the container. Mount the files yourself — this file cannot guess
# a host path.
KAFKA_SSL_CA_LOCATION: ${KAFKA_SSL_CA_LOCATION:-}
# mTLS, the two-path shape the Go and Python clients read.
KAFKA_SSL_CERT_LOCATION: ${KAFKA_SSL_CERT_LOCATION:-}
KAFKA_SSL_KEY_LOCATION: ${KAFKA_SSL_KEY_LOCATION:-}
# The SAME keypair in the shape a JVM can load: ONE file holding chain + key.
# Optional -- kafka-init and kafka-connect build that file from the pair above
# when this is unset (the connect image's connect-entrypoint.sh). Set it only
# to use a combined file you already have; it then wins over the pair. Inert
# in the Go and Python services.
KAFKA_SSL_KEYSTORE_LOCATION: ${KAFKA_SSL_KEYSTORE_LOCATION:-}
# Disables certificate verification, and is logged unconditionally when set: a
# self-signed lab, not prod.
#
# TWO SPELLINGS FOR ONE SETTING, and that is not tidy-uppable here. Python
# reads only KAFKA_SSL_SKIP_VERIFY (kafka_security.py:54); Go reads both
# (config.go:80 and the alias at :115). Shipping the Go spelling alone leaves
# verification silently ON for the Python half -- the exact split the Go alias
# was added to end. So both names are emitted, each defaulting to the other,
# and an operator who sets either one gets both.
KAFKA_SSL_SKIP_VERIFY: ${KAFKA_SSL_SKIP_VERIFY:-${KAFKA_SSL_INSECURE_SKIP_VERIFY:-}}
KAFKA_SSL_INSECURE_SKIP_VERIFY: ${KAFKA_SSL_INSECURE_SKIP_VERIFY:-${KAFKA_SSL_SKIP_VERIFY:-}}
services:
# ─── INFRASTRUCTURE ──────────────────────────────────────────────────────────
postgres:
image: postgres:16-alpine
container_name: rsync-ai-postgres
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
# Metadata DB only — pipeline rows never land here. Default shared_buffers
# is 128 MiB; 1 GiB leaves room for work_mem across the connection pool.
mem_limit: 1g
environment:
POSTGRES_USER: ${POSTGRES_USER:-rsync}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
POSTGRES_DB: ${POSTGRES_DB:-pipeline_db}
command:
- postgres
- -c
- wal_level=logical
- -c
- max_wal_senders=1
- -c
- max_replication_slots=1
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-rsync} -d ${POSTGRES_DB:-pipeline_db}"]
interval: 5s
timeout: 5s
retries: 10
# ─── DEMO WAREHOUSE ──────────────────────────────────────────────────────────
#
# The destination half of the zero-credential try-it path. `sample-data` gives a
# fresh install a source it can connect to with no credentials, but a pipeline
# needs TWO steps: intent.go:417 returns "plan has insufficient steps" for
# len(steps) < 2. Every other destination in the catalog requires a credential
# the new user does not have yet, so without this container the demo ends at a
# successful connection test and never produces a row.
#
# This is deliberately NOT the `postgres` service above. That one is sized and
# tuned as a metadata DB ("pipeline rows never land here") and runs with
# max_wal_senders=1 / max_replication_slots=1 -- exactly one CDC slot. Landing
# demo rows there would put pipeline data in the control-plane DB and contend
# for that single slot.
#
# It gets its OWN credential rather than reusing POSTGRES_PASSWORD. Sharing the
# control-plane password with a user-facing destination is a real, if
# second-order, escalation path: any workspace member can run SQL against this
# database through the pipeline/explorer, a superuser there can read
# pg_authid.rolpassword, and a SCRAM verifier is offline-crackable. The two
# credentials have no reason to be the same string, so they are not.
#
# The default below is a literal, on purpose, and is not a secret. A new
# REQUIRED var -- one written with the `:?` interpolation guard -- would break a
# fresh host and ONLY a fresh host, because a single unsatisfied `:?` aborts
# interpolation for the whole merged config while prod, staging and CI stay
# green. So the choice was a default or nothing. This database is never
# published off the internal network and holds nothing but the sample rows the
# demo writes into it. Override RSYNC_DEMO_WAREHOUSE_PASSWORD if you disagree.
demo-warehouse:
image: postgres:16-alpine
container_name: rsync-ai-demo-warehouse
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 256m
environment:
POSTGRES_USER: rsync_demo
POSTGRES_PASSWORD: ${RSYNC_DEMO_WAREHOUSE_PASSWORD:-rsync-demo-not-a-secret}
POSTGRES_DB: demo_warehouse
volumes:
- demo_warehouse_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U rsync_demo -d demo_warehouse"]
interval: 5s
timeout: 5s
retries: 10
# The demo seed hands this host to the JIT-deployed postgresql connector, which
# joins ONLY rsync-ai-mcp (SEC-M-06), so on `default` alone the connector cannot
# resolve "demo-warehouse" and the seed 502s. Bridging it is safe where bridging
# postgres would not be: it has its own role and holds only the sample rows.
# Guarded by test_a_host_handed_to_a_connector_is_on_the_connector_network.
networks:
- default
- rsync-ai-mcp
redis:
image: redis:7-alpine
container_name: rsync-ai-redis
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 768m
environment:
# Consumed by the command + healthcheck below; keeps the secret off ARGV.
REDIS_PASSWORD: ${REDIS_PASSWORD:?REDIS_PASSWORD is required}
command:
- sh
- -c
# The password is written to a 0600 file and never appears on the
# redis-server ARGV, so it stays out of `ps` on the host and out of
# the rendered compose spec (`docker inspect` / `compose config`).
- |
umask 077
: > /tmp/redis.conf
printf 'requirepass %s\n' "$$REDIS_PASSWORD" > /tmp/redis.conf
exec redis-server /tmp/redis.conf --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru
volumes:
- redis_data:/data
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli --no-auth-warning ping"]
interval: 5s
timeout: 3s
retries: 10
kafka:
image: apache/kafka:3.7.0
hostname: kafka
container_name: rsync-ai-kafka
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
# KAFKA_HEAP_OPTS below pins Xmx at 512m. Only PART of the rest is reclaimable:
# log-segment page cache does shrink under pressure, but direct/network buffers
# and metaspace are anonymous memory and count against the cap permanently.
#
# 1g gave that anonymous remainder ~512 MiB. The same broker config on the GCP
# host, capped at 1536M, measures 644.7 MiB resident at REST (2026-09-23) — i.e.
# already over what 1g would leave once heap is fully used, before any load. The
# cap here now matches docker-compose.prod.yml rather than sitting 512 MiB under
# it for an identical JVM; self-host is the deployment most likely to meet a
# large first sync with no operator watching.
mem_limit: 1536m
environment:
KAFKA_NODE_ID: 1
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 3000
KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS: 180000
KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 6000
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_PROCESS_ROLES: broker,controller
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29092,CONTROLLER://0.0.0.0:29093,PLAINTEXT_HOST://0.0.0.0:9092
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
KAFKA_LOG_DIRS: /var/lib/kafka/data
# Broker cap on a single record. The 1 MiB default rejects CDC change events for
# rows with large JSON/text/blob columns (RecordTooLargeException -> Debezium task
# FAILED -> restart loop). LOCKSTEP with CONNECT_PRODUCER_MAX_REQUEST_SIZE on
# kafka-connect below and with the sink worker's ReaderConfig.MaxBytes, which must
# exceed it. docker-compose.yml has carried this since the bug was first hit; the
# quickstart shipped the producer half alone.
KAFKA_MESSAGE_MAX_BYTES: 15728640
# Retention for topics rsync.ai does not create itself — Debezium auto-creates
# rsync.cdc-* and those inherit the BROKER default, not kafka/topology.go.
# 168h/delete matches Apache's own defaults, so nothing changes today; it
# pins the 7-day policy so a future image default cannot silently restore
# unbounded growth. See docker-compose.yml for the full note. Per-topic
# retention.ms=-1 on the Debezium schema-history topic wins over this,
# which is what that one exception relies on.
KAFKA_LOG_RETENTION_HOURS: 168
KAFKA_LOG_CLEANUP_POLICY: delete
CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
KAFKA_HEAP_OPTS: "-Xms256m -Xmx512m -XX:+ExitOnOutOfMemoryError"
# Kafka's two internal topics default to 50 partitions EACH. On a single
# broker that is 100 partition directories, 100 index/timeindex file pairs
# and 100 sets of in-memory metadata, bought for a stack whose real traffic
# is a handful of 3-partition topics. 8 keeps consumer-group parallelism far
# above anything this deployment uses.
#
# Both are read ONLY when the topic is first created. On an existing cluster
# this is a harmless no-op — the topic keeps whatever partition count it was
# born with — so it cannot be applied retroactively by editing this file.
KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 8
KAFKA_TRANSACTION_STATE_LOG_NUM_PARTITIONS: 8
volumes:
- kafka_data:/var/lib/kafka/data
healthcheck:
test: /opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1
interval: 10s
timeout: 5s
retries: 10
start_period: 30s
kafka-init:
image: apache/kafka:3.7.0
container_name: rsync-ai-kafka-init
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
# Run-once job, but it starts a JVM AdminClient. Capped so the topic
# creation pass cannot compete with the broker it just waited for.
mem_limit: 512m
depends_on:
kafka:
condition: service_healthy
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
entrypoint: ["/bin/bash", "-c"]
environment:
# Bare `-`, matching kafkaclient.TopicPrefix(): an explicitly empty prefix
# is the migration lever for a deployment with live unprefixed topics, and
# `:-` would silently overwrite that choice with the default.
# The bundled broker by default; a BYO cluster when KAFKA_BROKERS is set.
KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:29092}
KAFKA_TOPIC_PREFIX: ${KAFKA_TOPIC_PREFIX-rsync.}
# Same two names the Go side reads (backend-orchestrator/internal/kafka/
# replication.go EnvReplicationFactor / EnvMinInsyncReplicas), so ONE operator
# setting covers the bootstrapper below and the runtime topic-creation path.
# The bundled single-broker Kafka can only serve RF=1, but this file is also
# the starting point for a self-host pointed at a real cluster via
# KAFKA_BROKERS, and there RF=1 topics go unavailable during routine rolling
# maintenance. Empty means "operator stated nothing" -> the default below.
KAFKA_REPLICATION_FACTOR: ${KAFKA_REPLICATION_FACTOR:-}
KAFKA_MIN_INSYNC_REPLICAS: ${KAFKA_MIN_INSYNC_REPLICAS:-}
# The topic bootstrapper is a Kafka CLIENT too. Without these it speaks
# PLAINTEXT to a SASL listener, which does not error -- it BLOCKS in the
# handshake until the CLI's 5-minute default gives up, so the install
# appears to hang rather than to be misconfigured.
<<: *kafka-security
# Single-element list, not a bare block scalar: Compose splits a string
# command into argv, which left `bash -c` running only "export" and exiting 0
# without creating a single topic. `docker compose config` must show ONE item.
command:
- |
export PATH="/opt/kafka/bin:$$PATH";
# Was hardcoded to kafka:29092 in all four calls below, so the RF /
# min-insync knobs above could never reach the real cluster they were
# written for. Honour KAFKA_BROKERS like every other service does.
B="$${KAFKA_BROKERS:-kafka:29092}";
P="$${KAFKA_TOPIC_PREFIX-rsync.}";
P="$$(printf '%s' "$$P" | tr -cd 'a-zA-Z0-9._-')";
case "$$P" in ""|*[._-]) ;; *) P="$$P." ;; esac;
echo "Topic prefix: $${P:-<none>}";
RF="$${KAFKA_REPLICATION_FACTOR:-1}";
case "$$RF" in ''|*[!0-9]*) RF=1 ;; esac;
if [ "$$RF" -lt 1 ]; then RF=1; fi;
MISR="$${KAFKA_MIN_INSYNC_REPLICAS:-}";
case "$$MISR" in *[!0-9]*) MISR='' ;; esac;
if [ -n "$$MISR" ] && [ "$$MISR" -lt 1 ]; then MISR='' ; fi;
MISR_ARG="";
if [ -n "$$MISR" ]; then
if [ "$$MISR" -gt "$$RF" ]; then
echo "WARN: KAFKA_MIN_INSYNC_REPLICAS=$$MISR exceeds replication factor $$RF - clamping to $$RF (min.insync.replicas > RF creates a topic that every acks=all produce rejects with NOT_ENOUGH_REPLICAS on brokers older than 3.7)";
MISR="$$RF";
fi;
MISR_ARG="--config min.insync.replicas=$$MISR";
fi;
echo "Replication factor: $$RF (min ISR: $${MISR:-<broker default>})";
# ── AdminClient config ────────────────────────────────────────────────
# Written even for PLAINTEXT, because the timeouts are not optional. A
# client whose security config does not match the listener does NOT get
# an error -- it BLOCKS. Mirrors deploy/helm/rsync-ai/templates/jobs/
# kafka-init.yaml; the two are hand-kept in lockstep because compose can
# ship no file but this one (install.sh downloads only this compose file),
# so the script cannot be factored out.
CFG=/tmp/kafka-client.properties;
umask 077;
{ echo "request.timeout.ms=10000";
echo "default.api.timeout.ms=15000";
echo "security.protocol=$${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}"; } > "$$CFG";
# type=PEM is the whole point (KIP-651, Kafka 2.7+): without it the JVM
# defaults to JKS, fails to parse the PEM, and names neither the file nor
# the setting. The Go and Python clients read the same file unconverted.
if [ -n "$${KAFKA_SSL_CA_LOCATION:-}" ]; then
{ echo "ssl.truststore.type=PEM";
echo "ssl.truststore.location=$$KAFKA_SSL_CA_LOCATION"; } >> "$$CFG";
fi;
# mTLS. A JVM PEM keystore is ONE file holding chain + key; it cannot
# take the two paths the Go and Python clients read, and pointing it at
# the cert half alone yields "Failed to load PEM keystore" -- a message
# naming the file, so it reads like a bad certificate, not a missing key.
# An explicit KAFKA_SSL_KEYSTORE_LOCATION wins; otherwise the file is
# built here from the pair, with the same checks the kafka-connect
# image's entrypoint (connect-entrypoint.sh) makes on the same input.
KS="$${KAFKA_SSL_KEYSTORE_LOCATION:-}";
CRT="$${KAFKA_SSL_CERT_LOCATION:-}";
KEY="$${KAFKA_SSL_KEY_LOCATION:-}";
if [ -z "$$KS" ] && [ -n "$$CRT$$KEY" ]; then
# Half a pair is a config error, not "no mTLS": the broker would
# reject the handshake anyway, with an alert naming neither variable.
if [ -z "$$CRT" ] || [ -z "$$KEY" ]; then
echo "FATAL: KAFKA_SSL_CERT_LOCATION and KAFKA_SSL_KEY_LOCATION must be set together (mTLS needs both halves of the keypair)." >&2;
exit 1;
fi;
for f in "$$CRT" "$$KEY"; do
[ -r "$$f" ] || { echo "FATAL: $$f (from KAFKA_SSL_CERT_LOCATION / KAFKA_SSL_KEY_LOCATION) is not a readable file in this container." >&2; exit 1; };
done;
grep -q -- '-----BEGIN CERTIFICATE-----' "$$CRT" || {
echo "FATAL: KAFKA_SSL_CERT_LOCATION=$$CRT holds no PEM certificate (-----BEGIN CERTIFICATE-----)." >&2; exit 1; };
grep -q -- '-----BEGIN PRIVATE KEY' "$$KEY" || {
echo "FATAL: KAFKA_SSL_KEY_LOCATION=$$KEY must be an unencrypted PKCS#8 key (BEGIN PRIVATE KEY)." >&2;
echo " The Kafka CLI is a JVM and cannot load PKCS#1, SEC1 or encrypted keys. Convert once:" >&2;
echo " openssl pkcs8 -topk8 -nocrypt -in client.key -out client.pk8.key" >&2;
echo " The Go and Python services read the PKCS#8 file unchanged." >&2;
exit 1; };
# umask 077 above: owner-only, like the properties file beside it.
KS=/tmp/kafka-client.pem;
{ cat "$$CRT"; echo; cat "$$KEY"; } > "$$KS";
fi;
if [ -n "$$KS" ]; then
{ echo "ssl.keystore.type=PEM";
echo "ssl.keystore.location=$$KS"; } >> "$$CFG";
fi;
# KAFKA_SSL_SKIP_VERIFY is the spelling every service reads; the anchor
# at the top of this file fills it in from either name.
if [ "$${KAFKA_SSL_SKIP_VERIFY:-}" = "true" ]; then
# As close as a JVM gets: drops the HOSTNAME check only, the chain is
# still validated. The Go client skips both, so this job can fail on a
# certificate the data plane accepted. Deliberately not papered over.
echo "ssl.endpoint.identification.algorithm=" >> "$$CFG";
fi;
if [ -n "$${KAFKA_SASL_MECHANISM:-}" ]; then
# The login module is a FUNCTION of the mechanism; hardcoding one turns
# a mechanism swap into a handshake failure naming the module.
case "$$KAFKA_SASL_MECHANISM" in
SCRAM-SHA-256|SCRAM-SHA-512) MOD=org.apache.kafka.common.security.scram.ScramLoginModule ;;
PLAIN) MOD=org.apache.kafka.common.security.plain.PlainLoginModule ;;
OAUTHBEARER) MOD=org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule ;;
*) echo "FATAL: kafka-init cannot build a JAAS config for sasl.mechanism=$$KAFKA_SASL_MECHANISM" >&2;
echo " Supported here: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, OAUTHBEARER." >&2;
echo " AWS_MSK_IAM is NOT one of them: the Go data plane implements it" >&2;
echo " (shared/go/kafkaclient/tokenauth/msk.go) but this JVM CLI image does" >&2;
echo " not. Create the topics out of band and drop kafka-init -- see" >&2;
echo " docs/deployment/kafka-acls.md." >&2;
exit 1 ;;
esac;
# The password crosses TWO grammars (a Java .properties file, then the
# StreamTokenizer-backed sasl.jaas.config value) and each eats a level
# of backslash. Escaping once corrupts identically to not escaping:
# C:\Users\svc arrives as C:Userssvc and the broker reports an ordinary
# auth failure, so the password looks wrong rather than mangled.
esc() { printf '%s' "$$1" | sed 's/\\/\\\\/g; s/"/\\"/g' | sed 's/\\/\\\\/g'; };
# True when fetching a token from $$1 would put the client secret on
# a wire. Loopback is exempt because the request never leaves the
# host. Mirrors _insecure_token_endpoint in
# scripts/kafka-init-new-topics.sh -- change both.
_ins_tok() {
case "$$1" in https://*|HTTPS://*) return 1 ;; esac;
H=$${1#*://}; H=$${H%%/*}; H=$${H%%\?*}; H=$${H##*@};
case "$$H" in
\[*) H=$${H#\[}; H=$${H%%]*} ;;
*) H=$${H%%:*} ;;
esac;
H=$$(printf '%s' "$$H" | tr '[:upper:]' '[:lower:]'); H=$${H%.};
[ -n "$$H" ] || return 1;
case "$$H" in
localhost|*.localhost|::1|0:0:0:0:0:0:0:1) return 1 ;;
127.*) case "$$H" in *[!0-9.]*) return 0 ;; *) return 1 ;; esac ;;
esac;
return 0;
};
if [ "$$KAFKA_SASL_MECHANISM" = "OAUTHBEARER" ]; then
for v in KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT KAFKA_SASL_OAUTHBEARER_CLIENT_ID KAFKA_SASL_OAUTHBEARER_CLIENT_SECRET; do
eval "set_to=\$${$$v:-}";
if [ -z "$$set_to" ]; then
echo "FATAL: sasl.mechanism=OAUTHBEARER but $$v is empty." >&2; exit 1;
fi;
done;
# The client-credentials grant POSTs the client secret to the token
# endpoint on every fetch, so http off loopback hands a credential
# that never expires to anyone on the path. Same rule and opt-out
# as the Go, Python and Kafka Connect halves.
if [ "$${KAFKA_SASL_OAUTHBEARER_ALLOW_INSECURE_TOKEN_ENDPOINT:-}" != "true" ] && _ins_tok "$$KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT"; then
echo "FATAL: KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT=$$KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT is http and its host is not loopback." >&2;
echo " KAFKA_SASL_OAUTHBEARER_CLIENT_SECRET would be sent in the clear on every token fetch." >&2;
echo " Use https, a loopback address, or set KAFKA_SASL_OAUTHBEARER_ALLOW_INSECURE_TOKEN_ENDPOINT=true for a disposable test rig." >&2;
exit 1;
fi;
# kafka-clients 3.9.2 gates the token endpoint on a JVM system
# property, and UNSET MEANS ALLOW ANY URL.
case "$${KAFKA_OPTS:-}" in
*org.apache.kafka.sasl.oauthbearer.allowed.urls*) ;;
*) export KAFKA_OPTS="$${KAFKA_OPTS:+$$KAFKA_OPTS }-Dorg.apache.kafka.sasl.oauthbearer.allowed.urls=$$KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT" ;;
esac;
OPTS=$$(printf ' clientId="%s" clientSecret="%s"' "$$(esc "$$KAFKA_SASL_OAUTHBEARER_CLIENT_ID")" "$$(esc "$$KAFKA_SASL_OAUTHBEARER_CLIENT_SECRET")");
if [ -n "$${KAFKA_SASL_OAUTHBEARER_EXTENSIONS:-}" ]; then
OLDIFS=$$IFS; IFS=,;
for kv in $$KAFKA_SASL_OAUTHBEARER_EXTENSIONS; do
IFS=$$OLDIFS;
case "$$kv" in *=*) ;; *) echo "FATAL: extensions entry '$$kv' has no '=' separator." >&2; exit 1 ;; esac;
EK=$${kv%%=*}; EV=$${kv#*=};
# 'auth' is the OAUTHBEARER frame's own key; sending it as an
# extension makes a malformed frame the broker rejects without
# naming the extension.
if [ -z "$$EK" ] || [ "$$EK" = "auth" ]; then
echo "FATAL: extensions has an empty or reserved name in '$$kv'." >&2; exit 1;
fi;
OPTS=$$OPTS$$(printf ' extension_%s="%s"' "$$EK" "$$(esc "$$EV")");
IFS=,;
done;
IFS=$$OLDIFS;
fi;
if [ -n "$${KAFKA_SASL_OAUTHBEARER_SCOPE:-}" ]; then
OPTS=$$OPTS$$(printf ' scope="%s"' "$$(esc "$$KAFKA_SASL_OAUTHBEARER_SCOPE")");
fi;
{ echo "sasl.mechanism=$$KAFKA_SASL_MECHANISM";
printf 'sasl.jaas.config=%s required%s;\n' "$$MOD" "$$OPTS";
# Without the handler class OAuthBearerLoginModule falls back to
# its UNSECURED default and mints a self-signed JWS; the broker
# then rejects a token it cannot validate, and the error describes
# the token rather than the missing handler.
echo "sasl.login.callback.handler.class=org.apache.kafka.common.security.oauthbearer.secured.OAuthBearerLoginCallbackHandler";
echo "sasl.oauthbearer.token.endpoint.url=$$KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT"; } >> "$$CFG";
else
if [ -z "$${KAFKA_SASL_USERNAME:-}" ] || [ -z "$${KAFKA_SASL_PASSWORD:-}" ]; then
echo "FATAL: sasl.mechanism=$$KAFKA_SASL_MECHANISM but KAFKA_SASL_USERNAME or KAFKA_SASL_PASSWORD is empty." >&2;
exit 1;
fi;
{ echo "sasl.mechanism=$$KAFKA_SASL_MECHANISM";
printf 'sasl.jaas.config=%s required username="%s" password="%s";\n' "$$MOD" "$$(esc "$$KAFKA_SASL_USERNAME")" "$$(esc "$$KAFKA_SASL_PASSWORD")"; } >> "$$CFG";
fi;
fi;
CC="--command-config $$CFG";
# Wait for the broker. On a BYO cluster this is the DNS/auth check too --
# and it must run BEFORE --if-not-exists, whose "already exists" success
# would otherwise mask an auth failure as a no-op.
READY="";
i=1;
while [ "$$i" -le 30 ]; do
if kafka-topics.sh --bootstrap-server "$$B" $$CC --list >/dev/null 2>&1; then READY=1; break; fi;
echo "waiting for kafka at $$B ($$i/30)...";
sleep 5;
i=$$((i + 1));
done;
if [ -z "$$READY" ]; then
echo "FATAL: could not list topics on $$B" >&2;
echo " security.protocol=$${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} sasl.mechanism=$${KAFKA_SASL_MECHANISM:-<none>} sasl username=$${KAFKA_SASL_USERNAME:-<none>}" >&2;
echo " If the credentials are set but security.protocol is not, the client" >&2;
echo " connected ANONYMOUSLY and the broker rejected it: set KAFKA_SECURITY_PROTOCOL." >&2;
exit 1;
fi;
# One config for all three, the one the orchestrator's topology
# provisioner (backend-orchestrator/internal/kafka/topology.go) gives the
# same names. No creator alters a topic that already exists, so whichever
# runs first fixes the config for good; they must agree. Retention is
# bounded at 7 days, pipeline.domain.events included. Same list as
# deploy/helm/rsync-ai/templates/jobs/kafka-init.yaml.
TC="--config cleanup.policy=delete --config retention.ms=604800000 --config compression.type=snappy";
for t in pipeline.domain.events pii.scan.request pii.scan.response; do
kafka-topics.sh --create --if-not-exists --bootstrap-server "$$B" $$CC --topic "$${P}$$t" --partitions 3 --replication-factor "$$RF" $$MISR_ARG $$TC || exit 1;
done &&
echo "✅ Kafka topics ready"
restart: "no"
minio:
# Pinned: this is the one-command install. An unpinned tag means two operators running
# the same install command on different days get different object stores. Pinned by
# digest because cgr.dev/chainguard serves only `latest` (and `latest-dev`)
# anonymously, so a digest is the only pin there is.
# Chainguard's build, not MinIO's. MinIO revoked anonymous pulls on
# docker.io/minio/* on 2026-09-12 and then on quay.io/minio/* -- every tag, server
# and `mc` client alike, 401 to an unauthenticated client. This file is the
# one-command install: `docker compose pull` died here before a single container
# started, and the error Docker reports for a walled repository is "repository
# does not exist or may require 'docker login'", which reads like a typo rather
# than a vendor policy change. This image carries both `minio` (its entrypoint)
# and `mc`, which the healthcheck below uses; it has no curl.
image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
# Root, like the image this replaces. A volume written by v0.1.5 is root-owned
# 0755, and this image's default user (65532) cannot write it: the server exits
# "Unable to write to the backend". Nothing here waits on MinIO's health, so
# without this the upgrade would report success with a dead object store.
user: "0:0"
container_name: rsync-ai-minio
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 768m
environment:
MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required}
MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required}
command: server /data --console-address ":9001"
volumes:
- minio_data:/data
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 10s
timeout: 5s
retries: 5
# One-shot: create the `pipeline-data` bucket every batch stage and claim-check
# writes into, and put the staging-prefix expiry backstop on it. Nothing else on
# this install creates it: the minio connector only head_bucket()s and
# put_object()s, and the orchestrator has no MakeBucket. docker-compose.yml and
# the Helm chart each carry this step (minio-lifecycle-init / the `<release>-minio-mb` Job);
# this file, being standalone, has to repeat it or a fresh install stages into a
# bucket that does not exist. Same image and digest as `minio` above, which ships
# `mc` and a shell. The 7-day expiry must stay >= the batch-topic retention, for
# the reason spelled out on minio-lifecycle-init in docker-compose.yml.
minio-init:
image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
container_name: rsync-ai-minio-init
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
depends_on:
minio:
condition: service_healthy
environment:
MINIO_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required}
MINIO_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required}
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -e
until mc alias set local http://minio:9000 "$$MINIO_ACCESS_KEY_ID" "$$MINIO_SECRET_ACCESS_KEY"; do
echo "waiting for minio..."; sleep 2;
done
mc mb --ignore-existing local/pipeline-data
mc ilm rule add local/pipeline-data --expire-days 7 --prefix staging/ \
|| echo "WARN: could not set the staging expiry rule; the sink's own delete is the primary cleanup"
echo "minio-init: pipeline-data bucket ready"
restart: "no"
# The blob lane's staging store: a second MinIO, on the connector network ONLY.
# Blob passthrough (copy raw files between object stores) has the SOURCE
# connector stage each object here and the DESTINATION connector fetch it, and
# connectors join only rsync-ai-mcp (SEC-M-06). `minio` above is on `default`
# only, so a connector could not resolve it; bridging it would hand every
# connector the store that holds every pipeline's staged claim-check rows. This
# one holds only staged blobs, under its own login, and nothing on `default`
# can reach it (the orchestrator only passes its URL along). Same image and
# digest as `minio`. The name must keep ":9000": the connectors' staging client
# uses path-style addressing only for an endpoint with "minio", "localhost" or
# ":9000" in it (base_connector.py _get_staging_client).
blob-staging:
image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
container_name: rsync-ai-blob-staging
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 512m
environment:
MINIO_ROOT_USER: ${BLOB_STAGING_ACCESS_KEY:?BLOB_STAGING_ACCESS_KEY is required}
MINIO_ROOT_PASSWORD: ${BLOB_STAGING_SECRET_KEY:?BLOB_STAGING_SECRET_KEY is required}
command: server /data
volumes:
- blob_staging_data:/data
networks:
- rsync-ai-mcp
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 10s
timeout: 5s
retries: 5
# Bucket + expiry for blob-staging. Nothing deletes a staged blob per message:
# blobs are content-addressed and shared between messages, so this rule is the
# only cleanup. 7 days, like minio-init's, and for the same reason.
blob-staging-init:
image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
container_name: rsync-ai-blob-staging-init
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
depends_on:
blob-staging:
condition: service_healthy
environment:
BLOB_STAGING_ACCESS_KEY: ${BLOB_STAGING_ACCESS_KEY:?BLOB_STAGING_ACCESS_KEY is required}
BLOB_STAGING_SECRET_KEY: ${BLOB_STAGING_SECRET_KEY:?BLOB_STAGING_SECRET_KEY is required}
networks:
- rsync-ai-mcp
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -e
until mc alias set local http://blob-staging:9000 "$$BLOB_STAGING_ACCESS_KEY" "$$BLOB_STAGING_SECRET_KEY"; do
echo "waiting for blob-staging..."; sleep 2;
done
mc mb --ignore-existing local/pipeline-data
mc ilm rule add local/pipeline-data --expire-days 7 --prefix staging/ \
|| echo "WARN: could not set the blob staging expiry rule"
echo "blob-staging-init: pipeline-data bucket ready"
restart: "no"
# ─── TEMPORAL ────────────────────────────────────────────────────────────────
temporal:
image: temporalio/auto-setup:1.22.7
container_name: rsync-ai-temporal
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 1g
depends_on:
postgres:
condition: service_healthy
# Other half of docker-compose.byo-postgres.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
environment:
DB: postgresql
DB_PORT: ${POSTGRES_PORT:-5432}
POSTGRES_USER: ${POSTGRES_USER:-rsync}
POSTGRES_PWD: ${POSTGRES_PASSWORD}
POSTGRES_SEEDS: ${POSTGRES_HOST:-postgres}
# TLS to the database. Temporal does NOT read POSTGRES_SSLMODE -- it has
# its own names, verified against the pinned image's
# /etc/temporal/config/config_template.yaml (1.22.4; byte-identical in
# 1.22.7). SQL_TLS_ENABLED gates all four sql blocks (default store +
# visibility) at once.
# Managed Postgres that mandates TLS (Azure Database, RDS with
# rds.force_ssl=1) needs this true or auto-setup cannot create its
# schemas -- and then no workflow engine starts and every pipeline fails.
SQL_TLS_ENABLED: ${POSTGRES_TLS_ENABLED:-false}
SQL_HOST_VERIFICATION: ${POSTGRES_TLS_VERIFY_HOST:-false}
SQL_HOST_NAME: ${POSTGRES_TLS_SERVER_NAME:-}
SQL_CA: ${POSTGRES_TLS_CA_FILE:-}
# The SECOND TLS family. auto-setup runs temporal-sql-tool to create the
# schemas BEFORE starting the server, and that is a different binary with
# its own flags -- verified from `temporal-sql-tool --help` on this pinned
# image, not from the server's config template. Set only the four names
# above and the schema step still connects in plaintext, so a database
# that refuses plaintext kills auto-setup before the server ever binds:
# no workflow engine, every pipeline hangs, and no TLS error in any log.
SQL_TLS: ${POSTGRES_TLS_ENABLED:-false}
SQL_TLS_CA_FILE: ${POSTGRES_TLS_CA_FILE:-}
SQL_TLS_SERVER_NAME: ${POSTGRES_TLS_SERVER_NAME:-}
# Stated as "disable" -- the INVERSE of SQL_HOST_VERIFICATION above.
# Compose interpolation cannot negate, so this is its own key rather than
# a derivation. Turning on host verification means setting BOTH:
# POSTGRES_TLS_VERIFY_HOST=true and POSTGRES_TLS_SKIP_HOST_VERIFY=false.
SQL_TLS_DISABLE_HOST_VERIFICATION: ${POSTGRES_TLS_SKIP_HOST_VERIFY:-true}
# config/dynamicconfig/docker.yaml -- NOT development-sql.yaml. Both
# docker-compose.yml and the Helm chart name development-sql.yaml, but both
# also mount it (deploy/temporal, and a ConfigMap). This file cannot:
# install.sh downloads this compose file ALONE, with no repo checkout, so
# there is nothing on the host to bind-mount. Naming a file the image does
# not ship makes the server exit at boot -- "unable to validate dynamic
# config: no such file or directory" -- and `restart: unless-stopped` turns
# that into a permanent crash loop with no workflow engine, i.e. every
# pipeline hangs. docker.yaml ships in the image (empty) and is the image's
# own default. The only setting the mounted file carried is
# forceSearchAttributesCacheRefreshOnRead, which is inert here: this
# product registers no custom search attributes. Add one and this needs a
# real file again -- via `configs:` content, not a bind mount.
DYNAMIC_CONFIG_FILE_PATH: config/dynamicconfig/docker.yaml
healthcheck:
test: ["CMD", "tctl", "--address", "temporal:7233", "workflow", "list"]
interval: 10s
timeout: 5s
retries: 10
# ─── APPLICATION SERVICES ────────────────────────────────────────────────────
# The COMMUNITY image, not the private ./llm-service/Dockerfile one. Built from
# Dockerfile.community: an allowlist COPY cut against llm-service/oss-strip-list.txt,
# so the connector-generation moat (tool_generator/config + the agents that read it)
# is absent from every layer. It carries the /chat gateway — the only
# pipeline-creation path in the product — plus the Data Explorer, suggestions and
# the PII scanner. See .github/workflows/docker-publish.yml.
llm-service:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/llm-service-oss:${RSYNC_VERSION:-latest}
container_name: rsync-ai-llm-service
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 1536m
# The rate limiter degrades to per-process in-memory counters the first time
# it cannot reach Redis, and stays degraded for the life of the process. Wait
# for the server so that decision is never made against a Redis that was
# merely still booting. Same edge the orchestrator, adapter and gateway
# already declare.
depends_on:
redis:
condition: service_healthy
# OLLAMA_URL below defaults to host.docker.internal, which Docker Desktop
# (macOS/Windows) resolves for free and Linux Docker does NOT -- so without
# this line the offline provider fails to resolve its own default on exactly
# the platform every server install runs, and an operator who sets
# LLM_PROVIDER=ollama for an Ollama on the host is on exactly that path.
#
# Note this only makes the NAME resolve; Ollama must also be listening off
# loopback (OLLAMA_HOST=0.0.0.0), since host-gateway is the bridge address,
# not 127.0.0.1.
extra_hosts:
- "host.docker.internal:host-gateway"
environment:
# Kafka client security (BYO/managed broker). See x-kafka-security at the
# top of this file; every key defaults to empty and empty means unset.
<<: *kafka-security
PORT: 5000
# LLM cost logging (src/utils/llm_cost.py) writes llm_usage_events into the
# database api-gateway migrates. Unset, every LLM call goes unmetered.
DATABASE_URL: postgres://${POSTGRES_USER:-rsync}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-pipeline_db}?sslmode=${POSTGRES_SSLMODE:-disable}
# Sent as X-Internal-Secret on service-to-service calls to the orchestrator
# topology API (requirePrincipal). Empty = topology calls 401 in production.
INTERNAL_SERVICE_SECRET: ${INTERNAL_SERVICE_SECRET:-}
# Soft (not :?) so a fully-offline Ollama install (no OpenAI key) can boot.
# Provider selection is by LLM_PROVIDER; the client falls back to Ollama when
# no cloud key is present (see llm-service openai_client.py).
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
# LLM_PROVIDER=openai means "speaks the OpenAI wire protocol", not "is
# OpenAI": Vertex AI, Azure, Groq, OpenRouter, Together and vLLM all serve
# that protocol, and OPENAI_BASE_URL is the only var that points the client
# at one of them. There is no env_file on this service, so without this line
# an operator who sets OPENAI_BASE_URL in .env watches it be silently
# ignored and the stack keeps talking to api.openai.com — the one endpoint
# their key is not for. docker-compose.prod.yml:585 already passes it, which
# is what makes the omission here an oversight rather than a policy.
OPENAI_BASE_URL: ${OPENAI_BASE_URL:-}
# gcp-metadata = take the bearer from this VM's service account and refresh it
# before its hour is up, which is what Vertex AI needs. Empty = OPENAI_API_KEY
# as written.
OPENAI_API_KEY_SOURCE: ${OPENAI_API_KEY_SOURCE:-}
# LLM_PROVIDER=groq and LLM_PROVIDER=azure read these. Until they were listed
# here neither provider could be set up from .env: the key never reached the
# container and the service quietly fell back to Ollama.
GROQ_API_KEY: ${GROQ_API_KEY:-}
AZURE_OPENAI_ENDPOINT: ${AZURE_OPENAI_ENDPOINT:-}
AZURE_OPENAI_API_KEY: ${AZURE_OPENAI_API_KEY:-}
AZURE_OPENAI_API_VERSION: ${AZURE_OPENAI_API_VERSION:-}
AZURE_OPENAI_DEPLOYMENT: ${AZURE_OPENAI_DEPLOYMENT:-}
LLM_PROVIDER: ${LLM_PROVIDER:-openai}
# Empty, not gpt-4o. Empty lets each provider use its own default model
# (openai_client.py get_default_model); gpt-4o is a name only OpenAI's catalog
# has, so a Groq, Azure or Ollama stack with no LLM_MODEL got 404 on every call.
LLM_MODEL: ${LLM_MODEL:-}
# Empty = 4096 (openai_client.py with_reasoning_headroom): room a thinking model
# reasons in before it answers. 0 for a model with a small output limit.
LLM_REASONING_TOKEN_HEADROOM: ${LLM_REASONING_TOKEN_HEADROOM:-}
OLLAMA_URL: ${OLLAMA_URL:-http://host.docker.internal:11434}
# OLLAMA_BASE_URL wins over OLLAMA_URL (openai_client.py _ollama_base_url).
# docs/deployment/env-vars.md tells operators to set this one, so it needs a
# delivery path here or the value they set is silently ignored.
OLLAMA_BASE_URL: ${OLLAMA_BASE_URL:-}
# Air-gapped / on-prem: pins ONLY the Data Explorer to Ollama so schema
# metadata never leaves the deployment. There is no env_file on this service,
# so every var the operator is allowed to set must be listed explicitly —
# without this line EXPLORER_OFFLINE_ONLY=true in .env never reaches the
# container and the privacy guarantee silently does not hold.
EXPLORER_OFFLINE_ONLY: ${EXPLORER_OFFLINE_ONLY:-false}
EXPLORER_LLM_PROVIDER: ${EXPLORER_LLM_PROVIDER:-}
EXPLORER_TABLE_LINK_MODEL: ${EXPLORER_TABLE_LINK_MODEL:-}
EXPLORER_COLUMN_LINK_MODEL: ${EXPLORER_COLUMN_LINK_MODEL:-}
EXPLORER_NEXT_STEPS_MODEL: ${EXPLORER_NEXT_STEPS_MODEL:-}
EXPLORER_QUERY_SPEC_MODEL: ${EXPLORER_QUERY_SPEC_MODEL:-}
EXPLORER_SQL_MODEL: ${EXPLORER_SQL_MODEL:-}
EXPLORER_SQL_MODEL_MYSQL: ${EXPLORER_SQL_MODEL_MYSQL:-}
# /agents/rank-tables (table recommendation during pipeline setup). Unset
# inherits EXPLORER_LLM_PROVIDER then LLM_PROVIDER; EXPLORER_OFFLINE_ONLY
# overrides both.
RANK_TABLES_LLM_PROVIDER: ${RANK_TABLES_LLM_PROVIDER:-}
RANK_TABLES_MODEL: ${RANK_TABLES_MODEL:-}
EXPLORER_SQL_PROVIDER: ${EXPLORER_SQL_PROVIDER:-}
EXPLORER_SQL_ALLOW_ONLINE: ${EXPLORER_SQL_ALLOW_ONLINE:-}
EXPLORER_SQL_OPENAI_MODEL: ${EXPLORER_SQL_OPENAI_MODEL:-}
EXPLORER_SQL_FALLBACK_MODELS: ${EXPLORER_SQL_FALLBACK_MODELS:-}
OLLAMA_MODEL: ${OLLAMA_MODEL:-}
# The async PII-scan consumer reads these; without them it falls back to
# localhost:9092 and its startup failure is swallowed into a log warning.
KAFKA_BROKERS: "${KAFKA_BROKERS:-kafka:29092}"
KAFKA_TOPIC_PREFIX: ${KAFKA_TOPIC_PREFIX-rsync.}
USE_MOCK_LLM: "false"
# The gateway's rate limiter (src/utils/ratelimit.py get_limiter) builds its
# Redis URL from these four. With none of them delivered it still resolved
# the host "redis" by luck of the default, then failed AUTH against a server
# this file always starts with requirepass — so every limited route ran on a
# per-replica in-memory counter while the logs said Redis. Names and values
# match the orchestrator block below; REDIS_PASSWORD is the same one the
# redis service is started with.
REDIS_HOST: redis
REDIS_PORT: 6379
REDIS_DB: 0
REDIS_PASSWORD: ${REDIS_PASSWORD}
# This bundle ships no OTLP collector. The gRPC exporter connects lazily, so an absent collector is not a
# startup error: the BatchSpanProcessor just queues and retries every batch
# at localhost:4317 for the life of the process, blocking a worker for the
# full export timeout each time. The code default is true (cloud runs
# a collector); only this file turns it off, per the OSS/cloud split rule in
# CLAUDE.md. Spans are still created, they are simply not shipped —
# observability here is `docker logs`.
OTEL_ENABLED: "false"
LOG_FORMAT: json
LOG_LEVEL: info
ENVIRONMENT: production
command: ["python", "-m", "src.gateway.main"]
healthcheck:
test: ["CMD", "python", "-c", "import httpx; httpx.get('http://localhost:5000/health', timeout=3)"]
interval: 15s
timeout: 5s
retries: 5
start_period: 20s
# One-shot init: seed the mcp_connectors volume so a COLD self-host boots WITH
# connectors. Without it the volume is created empty and every pipeline run
# resolves zero connectors — a stack that comes up green and cannot move a row.
# Runs the image's seed-connectors.sh on every boot: a no-op on the same
# release, a refresh of the shipped connectors after an upgrade (generated
# connectors kept), and the tree handed to tool-generator's uid. No `command:`
# here -- an inline copy of the logic is how an upgrade once kept the old
# connector code. Same definition as docker-compose.oss.yml so layering the
# two merges cleanly.
connector-seed:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/connector-seed:${RSYNC_VERSION:-latest}
container_name: rsync-ai-connector-seed
restart: "no"
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
volumes:
- mcp_connectors:/target
# Connector Deployer (SEC-H-02): THE trusted socket-holder. tool-generator now
# drops root + the docker socket and delegates all JIT connector build/run to
# this small no-LLM service over an authenticated internal RPC. The security
# property is allow-by-construction (the deployer builds the Docker create
# payload itself with a hardcoded-safe HostConfig); because the HOST daemon
# creates the container it still joins rsync-ai-mcp by name/alias so the sink
# resolves JIT destinations. Same topology on Docker Desktop as in cloud — the
# deployer holds the socket in every stack. See
# docs/internal/jit-connector-sandboxing.md + connector-deployer/CONTRACT.md.
connector-deployer:
# An IMAGE, not a `build:`. This file is downloaded on its own by install.sh
# into an otherwise empty directory, so a build context is a path that does
# not exist on the customer's box -- `docker compose up` aborts the whole
# stack with `unable to prepare context: path "./connector-deployer" not
# found`. Published by docker-publish.yml alongside the other core services.
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/connector-deployer:${RSYNC_VERSION:-latest}
container_name: rsync-ai-connector-deployer
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 512m
environment:
# Same app-level fail-closed posture the tool-generator carried: ENVIRONMENT
# production => the deployer's S2S gate 503s when the secret is unset. The
# compose-level :? guard lives in prod.yml ONLY (OSS/cloud split rule), so
# here the secret defaults to empty for local Docker Desktop runs.
ENVIRONMENT: production
LOG_FORMAT: json
LOG_LEVEL: info
# Rotation for the connector containers THIS service creates -- compose's
# `logging:` blocks only reach the services compose itself starts.
RSYNC_LOG_MAX_SIZE: ${RSYNC_LOG_MAX_SIZE:-10m}
RSYNC_LOG_MAX_FILE: ${RSYNC_LOG_MAX_FILE:-3}
INTERNAL_SERVICE_SECRET: ${INTERNAL_SERVICE_SECRET:-}
DEPLOYER_DOCKER_NETWORK: rsync-ai-mcp
MCP_SHARED_NETWORK: rsync-ai-mcp
OAUTH_TOKENS_VOLUME_NAME: rsync-ai-oauth-tokens
# Root + RW docker.sock live HERE now, not on tool-generator.
user: "0:0"
networks:
- default
- rsync-ai-mcp
volumes:
# Connector artifacts, READ-ONLY (build contexts only).
- mcp_connectors:/app/shared/mcp-connectors:ro
# RW docker socket for build/run of JIT connector containers.
- /var/run/docker.sock:/var/run/docker.sock
# Connector LIFECYCLE, not connector generation. This image (Dockerfile.oss)
# serves GET /health + POST /v1/deploy and nothing else — it carries none of the
# LLM/NLP dependency tree and no generation code, which is why it is publishable.
# /v1/deploy is the one endpoint the Go data plane calls
# (backend-orchestrator server_manager.go -> TOOL_GENERATOR_URL) for connector
# self-heal and pinned-version JIT builds, so the service name stays
# `tool-generator` — every peer resolves it by that hostname.
tool-generator:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/connector-lifecycle:${RSYNC_VERSION:-latest}
container_name: rsync-ai-tool-generator
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 1g
# No llm-service edge: the lifecycle image never calls the gateway. It waits for
# the connector volume to be seeded so /v1/deploy can resolve connectors on disk,
# and for the deployer it delegates every build/run to.
depends_on:
connector-seed:
condition: service_completed_successfully
connector-deployer:
condition: service_started
environment:
PORT: 5010
TOOLS_DIR: /app/shared/mcp-connectors
# Universal default: managed mode on. Tool-generator dynamically
# builds images and starts user MCP containers when the
# orchestrator's Preflight calls /v1/deploy. Override to "false"
# if you'd rather manage user MCPs via a separate compose stack.
RSYNC_MANAGED_CONNECTORS: ${RSYNC_MANAGED_CONNECTORS:-true}
DOCKER_NETWORK: rsync-ai-mcp
RSYNC_EDITION: community
LOG_FORMAT: json
LOG_LEVEL: info
ENVIRONMENT: production
# SEC-H-03: S2S secret for /v1/deploy (X-Internal-Secret). Unset => no-op.
INTERNAL_SERVICE_SECRET: ${INTERNAL_SERVICE_SECRET:-}
# SEC-H-02: managed-mode JIT build/run is delegated to the connector-deployer
# over this internal RPC; tool-generator no longer touches the docker socket.
DEPLOYER_URL: http://connector-deployer:5011
command: ["python", "-m", "src.lifecycle.main"]
# SEC-H-02: runs the image's default unprivileged appuser — no root, no
# docker.sock. The connector-deployer is the sole trusted socket holder.
networks:
- default
- rsync-ai-mcp
volumes:
- mcp_connectors:/app/shared/mcp-connectors
planner:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/llm-service-oss:${RSYNC_VERSION:-latest}
container_name: rsync-ai-planner
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 768m
depends_on:
llm-service:
condition: service_healthy
# Same sticky-degrade reason as llm-service above.
redis:
condition: service_healthy
# Same reason as llm-service: OLLAMA_URL below defaults to a name Linux
# Docker does not resolve on its own.
extra_hosts:
- "host.docker.internal:host-gateway"
environment:
# Kafka client security (BYO/managed broker). See x-kafka-security at the
# top of this file; every key defaults to empty and empty means unset.
<<: *kafka-security
PORT: 5011
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
# Same OpenAI-compatible-endpoint story as llm-service above: without this
# the planner talks to api.openai.com no matter what the operator set, so a
# stack pointed at Vertex/Azure/vLLM would answer chat and fail planning.
OPENAI_BASE_URL: ${OPENAI_BASE_URL:-}
# The rest of the provider settings, for the same reason as on llm-service:
# the planner builds its client with the same helper, and a variable not
# listed here never reaches it.
OPENAI_API_KEY_SOURCE: ${OPENAI_API_KEY_SOURCE:-}
GROQ_API_KEY: ${GROQ_API_KEY:-}
AZURE_OPENAI_ENDPOINT: ${AZURE_OPENAI_ENDPOINT:-}
AZURE_OPENAI_API_KEY: ${AZURE_OPENAI_API_KEY:-}
AZURE_OPENAI_API_VERSION: ${AZURE_OPENAI_API_VERSION:-}
AZURE_OPENAI_DEPLOYMENT: ${AZURE_OPENAI_DEPLOYMENT:-}
LLM_PROVIDER: ${LLM_PROVIDER:-openai}
LLM_MODEL: ${LLM_MODEL:-}
LLM_REASONING_TOKEN_HEADROOM: ${LLM_REASONING_TOKEN_HEADROOM:-}
OLLAMA_URL: ${OLLAMA_URL:-http://host.docker.internal:11434}
OLLAMA_BASE_URL: ${OLLAMA_BASE_URL:-}
OLLAMA_MODEL: ${OLLAMA_MODEL:-}
TOOL_GENERATOR_URL: http://tool-generator:5010
API_GATEWAY_URL: http://api-gateway:8080
LLM_GATEWAY_URL: http://llm-service:5000
KAFKA_BOOTSTRAP_SERVERS: "${KAFKA_BROKERS:-kafka:29092}"
# Namespace for every Kafka topic this product creates, so an operator can
# tell them apart from other applications' topics on a shared (BYO-Kafka)
# cluster. Every service that touches Kafka must agree on this value: a
# producer and a consumer resolving different names do not error, the
# consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
# an explicit empty value means "disable" (keep pre-prefix topic names) and
# `:-` would silently turn that back into the default.
KAFKA_TOPIC_PREFIX: ${KAFKA_TOPIC_PREFIX-rsync.}
CONNECTORS_DIR: /app/shared/mcp-connectors
# src/agents/planner/service.py imports the same get_limiter as the gateway,
# so it needs the same four vars or its limits are per-replica too.
REDIS_HOST: redis
REDIS_PORT: 6379
REDIS_DB: 0
REDIS_PASSWORD: ${REDIS_PASSWORD}
# No collector in this bundle — see the llm-service block above.
OTEL_ENABLED: "false"
LOG_FORMAT: json
LOG_LEVEL: info
ENVIRONMENT: production
# Plan-time topic sizing posts to the orchestrator's /topology group, which
# sits behind requirePrincipal (strategies.py _create_topic_via_topology_agent).
INTERNAL_SERVICE_SECRET: ${INTERNAL_SERVICE_SECRET:-}
command: ["python", "-m", "src.agents.planner.service"]
volumes:
- mcp_connectors:/app/shared/mcp-connectors:ro
orchestrator:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/orchestrator:${RSYNC_VERSION:-latest}
container_name: rsync-ai-orchestrator
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
# Repeated from docker-compose.yml, which says why: this file is standalone.
stop_grace_period: 45s
mem_limit: 2g
depends_on:
postgres:
condition: service_healthy
# Other half of docker-compose.byo-postgres.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
kafka:
condition: service_healthy
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
redis:
condition: service_healthy
environment:
# Kafka client security (BYO/managed broker). See x-kafka-security at the
# top of this file; every key defaults to empty and empty means unset.
<<: *kafka-security
DB_HOST: ${POSTGRES_HOST:-postgres}
DB_PORT: ${POSTGRES_PORT:-5432}
DB_USER: ${POSTGRES_USER:-rsync}
DB_PASSWORD: ${POSTGRES_PASSWORD}
DB_NAME: ${POSTGRES_DB:-pipeline_db}
# internal/config/config.go:168 reads DB_SSLMODE (default "disable").
# Naming it explicitly matters: a DSN that omits sslmode negotiates
# plaintext without complaining, so a BYO database would look connected
# while shipping credentials in the clear.
DB_SSLMODE: ${POSTGRES_SSLMODE:-disable}
KAFKA_BROKERS: "${KAFKA_BROKERS:-kafka:29092}"
# Durability for every topic the orchestrator creates at runtime. Read by
# internal/kafka/replication.go (EnvReplicationFactor / EnvMinInsyncReplicas).
# Empty means "operator stated nothing" -- not zero -- so the built-in
# derivation applies (1 on a single broker, min(3, brokers) beyond).
#
# The pair travels together: min.insync.replicas > replication factor yields a
# topic that is created, listed and subscribable, and then rejects every
# acks=all produce with NOT_ENOUGH_REPLICAS -- a pipeline that reports running
# and moves zero rows. The Go side clamps misr down to the topic's final RF,
# so setting only one of these cannot wedge a cluster. Brokers from 3.7 on cap
# the effective value at the replica count and are immune; the older managed
# clusters a self-host points at are not. See docker-compose.yml for the
# measurement.
KAFKA_REPLICATION_FACTOR: ${KAFKA_REPLICATION_FACTOR:-}
KAFKA_MIN_INSYNC_REPLICAS: ${KAFKA_MIN_INSYNC_REPLICAS:-}
# Partition count for every CDC data topic (backend-orchestrator/internal/kafka
# partitions.go EnvCDCTopicPartitions). Empty means "operator stated nothing", so
# the derivation applies: 1 on a single broker, min(3, brokers) beyond that. Set
# it only to go WIDER than 3 on a large cluster.
#
# It is a birth setting, not a live one. Kafka cannot reduce a partition count at
# all, and RAISING one on a topic that is already streaming rehashes every key
# onto a different partition and breaks per-key ordering, so rsync.ai never resizes
# an existing topic -- a change here reaches only topics created after it.
KAFKA_CDC_TOPIC_PARTITIONS: ${KAFKA_CDC_TOPIC_PARTITIONS:-}
# Namespace for every Kafka topic this product creates, so an operator can
# tell them apart from other applications' topics on a shared (BYO-Kafka)
# cluster. Every service that touches Kafka must agree on this value: a
# producer and a consumer resolving different names do not error, the
# consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
# an explicit empty value means "disable" (keep pre-prefix topic names) and
# `:-` would silently turn that back into the default.
KAFKA_TOPIC_PREFIX: ${KAFKA_TOPIC_PREFIX-rsync.}
# Avro requires a Schema Registry; this compose ships none. The code default is
# true, so leaving it unset points the producer at a host that does not exist here.
KAFKA_USE_AVRO: "false"
JWT_SECRET: ${JWT_SECRET:?JWT_SECRET is required}
ENCRYPTION_KEY: ${ENCRYPTION_KEY:?ENCRYPTION_KEY is required}
INTERNAL_SERVICE_SECRET: ${INTERNAL_SERVICE_SECRET:-}
REDIS_HOST: redis
REDIS_PORT: 6379
REDIS_DB: 0
REDIS_PASSWORD: ${REDIS_PASSWORD}
TOOL_GENERATOR_URL: http://tool-generator:5010
# Sentinel core-service probes (sentinel/service_probes.go). Each is probed only
# when its address is set; after 3 failed 30s checks the service is recorded
# unhealthy and every admin gets an INFRASTRUCTURE_DOWN alert. Redis is always
# probed, at the address the correlation client uses. TEMPORAL_ADDRESS is also the
# (parked) native Temporal worker's dial address, whose default is this same value.
TEMPORAL_ADDRESS: temporal:7233
SENTINEL_PROBE_TEMPORAL_ADAPTER_URL: http://temporal-adapter:8082
SENTINEL_PROBE_CONNECTOR_DEPLOYER_URL: http://connector-deployer:5011
SENTINEL_PROBE_LLM_SERVICE_URL: http://llm-service:5000
# Object-storage-to-object-storage runs (blob passthrough) stage bytes in
# blob-staging, and the orchestrator only hands these to the two connectors
# (executor/blob_lane.go buildStagingConfig). NOT `minio`: connectors join
# only rsync-ai-mcp and cannot resolve it. Unset, the lane falls back to
# MINIO_ENDPOINT_URL and then to rsync-ai-minio, neither of which a
# connector can reach here. The orchestrator reads no other MINIO_* value,
# so it carries no claim-check credentials at all.
BLOB_STAGING_ENDPOINT_URL: http://blob-staging:9000
BLOB_STAGING_ACCESS_KEY_ID: ${BLOB_STAGING_ACCESS_KEY:?BLOB_STAGING_ACCESS_KEY is required}
BLOB_STAGING_SECRET_ACCESS_KEY: ${BLOB_STAGING_SECRET_KEY:?BLOB_STAGING_SECRET_KEY is required}
CORRELATION_CLAIM_TTL: 1h
ENABLE_SCHEDULER: "false"
ENABLE_CDC_TABLE_STATS: "true"
# Batch data-plane sizing. Both empty by default = the code's own values,
# which are now correct rather than merely default: with this container's
# mem_limit set above, resolveTableConcurrencyWithReason() can finally read a real
# cgroup limit and cap the pool at what the box can buffer (~384 MiB per
# concurrent table load). Pin EXECUTOR_TABLE_CONCURRENCY only to override
# that derivation; lower EXECUTOR_TABLE_BATCH_ROWS (default 10000) if a
# source has unusually wide rows, since one page is held in memory.
EXECUTOR_TABLE_CONCURRENCY: ${EXECUTOR_TABLE_CONCURRENCY:-}
EXECUTOR_TABLE_BATCH_ROWS: ${EXECUTOR_TABLE_BATCH_ROWS:-}
DOCKER_NETWORK: rsync-ai-mcp
# No collector in this bundle — same reason as OTEL_ENABLED below.
# internal/config/config.go:231 defaults this to true for cloud; the OSS
# compose is the only place it is turned off.
OTEL_ENABLED: "false"
LOG_FORMAT: json
LOG_LEVEL: info
ENVIRONMENT: production
networks:
- default
- rsync-ai-mcp
volumes:
- mcp_connectors:/app/shared/mcp-connectors
temporal-adapter:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/temporal-adapter:${RSYNC_VERSION:-latest}
container_name: rsync-ai-temporal-adapter
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 768m
depends_on:
temporal:
condition: service_healthy
kafka:
condition: service_healthy
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
redis:
condition: service_healthy
environment:
# Kafka client security (BYO/managed broker). See x-kafka-security at the
# top of this file; every key defaults to empty and empty means unset.
<<: *kafka-security
TEMPORAL_ADDRESS: temporal:7233
KAFKA_BOOTSTRAP_SERVERS: "${KAFKA_BROKERS:-kafka:29092}"
# Namespace for every Kafka topic this product creates, so an operator can
# tell them apart from other applications' topics on a shared (BYO-Kafka)
# cluster. Every service that touches Kafka must agree on this value: a
# producer and a consumer resolving different names do not error, the
# consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
# an explicit empty value means "disable" (keep pre-prefix topic names) and
# `:-` would silently turn that back into the default.
KAFKA_TOPIC_PREFIX: ${KAFKA_TOPIC_PREFIX-rsync.}
# Avro requires a Schema Registry; this compose ships none. The code default is
# true, so leaving it unset points the producer at a host that does not exist here.
KAFKA_USE_AVRO: "false"
ENCRYPTION_KEY: ${ENCRYPTION_KEY}
POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
POSTGRES_PORT: ${POSTGRES_PORT:-5432}
POSTGRES_USER: ${POSTGRES_USER:-rsync}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB:-pipeline_db}
# internal/db/db.go:23 reads POSTGRES_SSLMODE (default "disable").
POSTGRES_SSLMODE: ${POSTGRES_SSLMODE:-disable}
REDIS_ADDRESS: redis:6379
# The `redis` service above always starts with `requirepass` (REDIS_PASSWORD
# is `:?`-required at line 209), so this client must authenticate or it does
# not start at all: cmd/adapter/main.go:222 reads exactly this name and
# main.go:223-226 `log.Fatalf`s when the correlation store cannot init. Its
# own comment at main.go:217-220 names this failure "Redis auth split-brain".
REDIS_PASSWORD: ${REDIS_PASSWORD}
LOG_LEVEL: info
ENVIRONMENT: production
# Scheduled saved queries and the model-freshness sweep call the gateway's
# internal endpoints and return "INTERNAL_SERVICE_SECRET is not configured"
# without this (scheduled_model_workflow.go:123-128,
# model_freshness_workflow.go:209-214). The Schedules page still shows
# them active; they just never run. docker-compose.yml has always passed
# both; this standalone file never did.
API_GATEWAY_URL: http://api-gateway:8080
INTERNAL_SERVICE_SECRET: ${INTERNAL_SERVICE_SECRET:-}
volumes:
# REQUIRED, and this file is the only one that ever missed it: this is the
# container that runs ConnectorAvailabilityActivityV2, and that activity
# resolves connectors off the LOCAL FILESYSTEM — connector_check_activity.go:93-97
# stats /app/shared/mcp-connectors/{public,internal,} and /app/tools. Without
# the mount every root Stat fails, so checkConnectorExists returns false for
# EVERY connector, the workflow escalates to GenerateConnectorActivityV2, and
# the user sees "Connector generation failed for <x>" for a connector that is
# sitting right there in the volume. Nothing about it is connector-specific;
# no pipeline of any kind can be created.
# docker-compose.yml (base, which prod/staging/e2e/ci-isolate all overlay)
# has carried this mount from the start — `./shared:/app/shared`, commented
# "Required for connector filesystem checks". This quickstart is the one
# standalone compose, so it is the only file that had to repeat it, and the
# only deployment path that broke: every self-host install.
# Read-only on purpose: the adapter only ever reads this tree (ReadDir/Stat/
# ReadFile/WalkDir). Generation output is written by tool-generator, which
# mounts the same volume read-write.
- mcp_connectors:/app/shared/mcp-connectors:ro
api-gateway:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/api-gateway:${RSYNC_VERSION:-latest}
container_name: rsync-ai-api-gateway
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 1g
ports:
# Loopback by default -- a laptop install must not publish an unauthenticated
# port to the LAN. A server install has the opposite problem: bound here, the
# only two published ports in this file are unreachable from anywhere but the
# server itself, so install.sh could ask for a domain, print a URL for it, and
# be believed, because its health check polls localhost and always passes.
# install.sh sets RSYNC_BIND_ADDR=0.0.0.0 when the operator names a real host
# and is not putting a TLS proxy in front. Leave it at the default when a
# reverse proxy on the host terminates TLS -- it reaches loopback fine, and
# 0.0.0.0 would additionally expose the plaintext port beside it.
- "${RSYNC_BIND_ADDR:-127.0.0.1}:5001:8080"
depends_on:
postgres:
condition: service_healthy
# Other half of docker-compose.byo-postgres.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
kafka:
condition: service_healthy
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
redis:
condition: service_healthy
# Wait for Temporal so the startup client.Dial wins the cold-boot race.
temporal:
condition: service_healthy
orchestrator:
condition: service_started
environment:
# Kafka client security (BYO/managed broker). See x-kafka-security at the
# top of this file; every key defaults to empty and empty means unset.
<<: *kafka-security
DATABASE_URL: postgres://${POSTGRES_USER:-rsync}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-pipeline_db}?sslmode=${POSTGRES_SSLMODE:-disable}
KAFKA_BROKERS: "${KAFKA_BROKERS:-kafka:29092}"
# Namespace for every Kafka topic this product creates, so an operator can
# tell them apart from other applications' topics on a shared (BYO-Kafka)
# cluster. Every service that touches Kafka must agree on this value: a
# producer and a consumer resolving different names do not error, the
# consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
# an explicit empty value means "disable" (keep pre-prefix topic names) and
# `:-` would silently turn that back into the default.
KAFKA_TOPIC_PREFIX: ${KAFKA_TOPIC_PREFIX-rsync.}
# Avro requires a Schema Registry; this compose ships none. The code default is
# true, so leaving it unset points the producer at a host that does not exist here.
KAFKA_USE_AVRO: "false"
TEMPORAL_ADDRESS: temporal:7233
ORCHESTRATOR_URL: http://orchestrator:8080
AGENT_ORCHESTRATOR_URL: http://orchestrator:8080
TOOL_GENERATOR_URL: http://tool-generator:5010
# Deadline for the llm-service calls in one chat turn (llmServiceTimeout()
# in internal/handlers/chat_nl_pipeline.go). The code default is 60s, sized
# for a hosted API; this bundle runs inference on CPU Ollama, where a first
# token takes 60-100s. When it was 10s, every plain-English message
# returned the canned fallback while the regex fast path kept answering
# canonical phrasings instantly, so the stack looked healthy.
#
# Under 300 on purpose: cmd/server/main.go caps the whole response at a
# 300s WriteTimeout, so a longer upstream deadline would just move the cut
# to a worse place. Raise both together if you ever need more.
LLM_SERVICE_TIMEOUT_SECONDS: ${LLM_SERVICE_TIMEOUT_SECONDS:-240}
JWT_SECRET: ${JWT_SECRET}
REDIS_ADDRESS: redis:6379
# Same requirepass as the adapter above. api-gateway fails SOFTER -- it
# retries 5x then logs "proceeding without caching" (cmd/server/main.go:412-427)
# -- so the omission does not crash the stack, it silently disables the schema
# cache and the Data Explorer cache built on the same client. Read at
# cmd/server/main.go:408 and internal/handlers/admin_health.go:80, which is why
# /admin/health would otherwise report redis "down" on a healthy Redis.
REDIS_PASSWORD: ${REDIS_PASSWORD}
ENCRYPTION_KEY: ${ENCRYPTION_KEY}
INTERNAL_SERVICE_SECRET: ${INTERNAL_SERVICE_SECRET:-}
RSYNC_CSRF_ENFORCE: "true"
RSYNC_ALLOW_INTERNAL_CONNECTORS: "false"
# Self-host has no Stripe / cloud plans: disable plan-quota enforcement so a
# fresh install is never trial-gated (migration 060 would otherwise seed a
# 14-day, 1-pipeline trial with no upgrade path). Cloud leaves this unset ⇒
# enforced. See resolvePlanQuota / billingEnforced() in plan_quota.go.
RSYNC_BILLING_ENFORCED: "false"
# Zero-credential try-it path. Unset ⇒ the demo endpoints report unavailable
# and refuse to seed, which is the CLOUD behavior (app.rsync.ai must never
# offer a bundled warehouse). Only this compose file sets it, so the split
# stays "which compose runs", not a runtime edition gate.
# Parsed by demoDestinationConfig() in api-gateway/internal/handlers/demo.go;
# host/port/database/user/password are handed to the `postgresql` connector,
# whose required_config is exactly those five keys. The value never reaches
# the browser — the seed endpoint builds the connection server-side.
RSYNC_DEMO_DESTINATION_DSN: postgres://rsync_demo:${RSYNC_DEMO_WAREHOUSE_PASSWORD:-rsync-demo-not-a-secret}@demo-warehouse:5432/demo_warehouse
# SEC-M-07: self-host is NOT behind Cloudflare, so CF-Connecting-IP is
# attacker-controllable. Disable Cloudflare client-IP trust here; cloud
# leaves this unset => enabled. See trustCloudflareClientIP() in api-gateway.
RSYNC_TRUST_CLOUDFLARE: "false"
# This stack ships no OTLP collector to export spans to.
# LoadTelemetryConfig defaults this to true (cloud runs
# one), and until now InitTracerWithConfig ignored the flag entirely, so
# every self-host retried gRPC exports at localhost:4317 forever. Only this
# file turns it off, per the OSS/cloud split rule in CLAUDE.md; the tracer
# falls back to a no-op provider, so instrumented paths keep working.
OTEL_ENABLED: "false"
RSYNC_ADMIN_EMAILS: ${RSYNC_ADMIN_EMAILS:-}
OAUTH_CALLBACK_URL: ${OAUTH_CALLBACK_URL:-}
# Left empty, OAUTH_CALLBACK_URL is derived from this (oauth_byo.go
# oauthCallbackBase); without it a server install redirected every OAuth
# consent back to localhost:5001.
PUBLIC_URL: ${PUBLIC_URL:-}
GITHUB_CLIENT_ID: ${GITHUB_CLIENT_ID:-}
GITHUB_CLIENT_SECRET: ${GITHUB_CLIENT_SECRET:-}
GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID:-}
GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET:-}
SLACK_CLIENT_ID: ${SLACK_CLIENT_ID:-}
SLACK_CLIENT_SECRET: ${SLACK_CLIENT_SECRET:-}
HUBSPOT_CLIENT_ID: ${HUBSPOT_CLIENT_ID:-}
HUBSPOT_CLIENT_SECRET: ${HUBSPOT_CLIENT_SECRET:-}
SALESFORCE_CLIENT_ID: ${SALESFORCE_CLIENT_ID:-}
SALESFORCE_CLIENT_SECRET: ${SALESFORCE_CLIENT_SECRET:-}
# Notifier delivery. api-gateway ALWAYS runs the notifier consumer
# (cmd/server/main.go) and always persists to pipeline_notifications, but
# Slack/email fan-out is enabled purely by the presence of these values:
# slackEnabled = NOTIFIER_SLACK_WEBHOOK_URL != "", and emailEnabled needs
# BOTH SMTP_HOST and SMTP_FROM (notifier.go:199-200). Until this block
# existed the vars were simply not passed through here, so a self-host got
# persist-only alerting -- rows in the DB, nothing in Slack or email, and
# no error at any layer to say so. Every one defaults empty: unset stays
# persist-only, which is the safe default for a fresh install.
NOTIFIER_SLACK_WEBHOOK_URL: ${NOTIFIER_SLACK_WEBHOOK_URL:-}
SMTP_HOST: ${SMTP_HOST:-}
SMTP_PORT: ${SMTP_PORT:-587}
SMTP_USER: ${SMTP_USER:-}
SMTP_PASSWORD: ${SMTP_PASSWORD:-}
# Required for email delivery, not optional: SMTP_HOST alone leaves
# emailEnabled false and the omission looks identical to "email off".
SMTP_FROM: ${SMTP_FROM:-}
# The absolute base every alert's "Open pipeline" link is built on
# (notifier.go:173). It falls back to http://localhost:3000 in code, which
# is a dead link in every alert on a server install -- so it follows
# NEXTAUTH_URL, the value install.sh actually persists and the same one
# RSYNC_CORS_ORIGINS below already keys on.
APP_BASE_URL: ${APP_BASE_URL:-${NEXTAUTH_URL:-http://localhost:3000}}
FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3000}
# CORS allowlist: must match the frontend origin so browser requests succeed
RSYNC_CORS_ORIGINS: ${NEXTAUTH_URL:-http://localhost:3000}
# Secure flag on the session and CSRF cookies. install.sh derives it from
# the scheme it published; left empty the gateway falls back to
# ENVIRONMENT, i.e. Secure. Fail-closed on purpose -- the failure this
# guards against (a Secure cookie dropped by the browser on plain http) is
# invisible, while the opposite mistake is at least visible as a warning.
RSYNC_COOKIE_SECURE: ${RSYNC_COOKIE_SECURE:-}
LOG_FORMAT: json
LOG_LEVEL: info
ENVIRONMENT: production
networks:
- default
- rsync-ai-mcp
volumes:
- mcp_connectors:/app/shared/mcp-connectors:ro
frontend:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/frontend:${RSYNC_VERSION:-latest}
container_name: rsync-ai-frontend
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 768m
ports:
# Same knob as the api-gateway port above; see the comment there.
- "${RSYNC_BIND_ADDR:-127.0.0.1}:3000:3000"
depends_on:
- api-gateway
environment:
API_GATEWAY_INTERNAL_URL: http://api-gateway:8080
# The dashboard's server-side CDC fetch (app/(dashboard)/page.tsx) falls back
# to localhost:8081 without this, which inside this container is nothing.
ORCHESTRATOR_INTERNAL_URL: http://orchestrator:8080
# The NEXT_PUBLIC_* pair below is inert on a published image -- the Next.js
# compiler substituted those names at `docker build` time, so nothing set
# here can reach the browser through them. They stay for a locally built
# image, where build and run share an environment.
#
# These two are what actually reach the browser, served per request by
# /runtime-config.js. Without them the UI published on :3000 resolves the
# api-gateway to its own origin and every request 404s against Next.js,
# while the installer's health probe on :5001 still reports success.
PUBLIC_URL: ${PUBLIC_URL:-}
PUBLIC_WS_URL: ${PUBLIC_WS_URL:-}
NEXT_PUBLIC_API_URL: ${PUBLIC_URL:-http://localhost:5001}
NEXT_PUBLIC_WS_URL: ${PUBLIC_WS_URL:-ws://localhost:5001/ws}
NEXTAUTH_URL: ${NEXTAUTH_URL:-http://localhost:3000}
INTERNAL_SERVICE_SECRET: ${INTERNAL_SERVICE_SECRET:-}
# ─── INTERNAL MCP CONNECTORS ─────────────────────────────────────────────────
# ── Container names are load-bearing ────────────────────────────────────────
# The orchestrator does not look these services up by compose service name. It
# builds the hostname itself, as `<STACK_PREFIX>-<connector>-v1-0-0-mcp`
# (backend-orchestrator/internal/workers/infra_preflight.go requiredServices,
# via mcp.StackPrefix() which defaults to "rsync-ai"). The short names used here
# before — rsync-minio-mcp, rsync-debezium-mcp, rsync-kafka-mcp-sink — matched
# neither that pattern nor anything else the orchestrator asks DNS for.
#
# Nothing errored. The preflight health poll simply NXDOMAIN'd for its full
# 24 x 5s budget (preflightKafkaConnectRetries) and then fell through to its
# "start it via tool-generator" last resort, which deployed a SECOND, correctly
# named container beside the compose one. So every batch pipeline on a
# quickstart box paid 120 seconds at preflight and then ran with a duplicate
# minio-mcp resident, while the compose-started container sat unused forever.
# Renaming is the whole fix; do not shorten these again.
#
# Pre-started, unlike sample-data-mcp below, because on-demand is not an
# option for it yet: the JIT deploy path (connector-deployer, tool-generator
# deployment, infra_preflight's last resort) passes no MINIO_* credentials, so
# a container it started would come up unable to reach the bucket. The env
# block below is the only place those credentials reach this connector.
minio-mcp:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/mcp-minio:${RSYNC_VERSION:-latest}
container_name: rsync-ai-minio-v1-0-0-mcp
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
mem_limit: 512m
environment:
PORT: 8000
MCP_HTTP_MODE: "true"
MINIO_ENDPOINT_URL: http://minio:9000
MINIO_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required}
MINIO_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required}
MINIO_BUCKET: pipeline-data
ENVIRONMENT: production
# The zero-credential demo source, and the reason it is worth a service block
# of its own: it is the ONLY connector in the tree whose auth_type is "none"
# (every other one of the 21 public connectors needs a credential before it
# will do anything). It carries its customers/orders/products rows inside its
# own image and imports nothing outside the Python standard library, so it is
# the one source a fresh install can connect to before the operator has
# produced a single secret.
#
# It was absent from THIS file while being present in the catalog the
# connector-seed image bakes -- Dockerfile.seed allowlist-COPYs public/
# wholesale, so sample-data has always been listed as installable. That is the
# worst of the two possible states: the connector was offered in the UI and
# selecting it failed at connection test with a DNS miss for a container no
# compose file here ever started. Absent-and-unlisted would at least have been
# honest.
#
# container_name is load-bearing and must keep EXACTLY this shape --
# <StackPrefix()>-<id>-vX-Y-Z-mcp, built by MCPContainerName()
# (backend-orchestrator/internal/mcp/server_manager.go:310). Docker's embedded
# DNS resolves it, so it doubles as the hostname the orchestrator health-checks
# at http://<name>:8000/health. The minio-mcp note above is the record of what
# shortening one of these costs: 120s of preflight per pipeline plus a
# duplicate container. Do not shorten it.
#
# Behind the opt-in "connectors" profile: an install starts no catalog
# connector. Creating the sample-data connection deploys this same container
# name on demand (connector-deployer builds it from the seeded source), and
# POST /demo/seed retries its 503 connector_deploying while that runs
# (api-gateway demo.go demoDeployRetryWindow). minio-mcp above and the cdc
# pair stay pre-started for reasons given at each.
sample-data-mcp:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/mcp-sample-data:${RSYNC_VERSION:-latest}
container_name: rsync-ai-sample-data-v1-0-0-mcp
profiles: ["connectors"]
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
# Stdlib-only, and it serves three in-memory lists. It does not need the
# 512m the credentialed connectors take.
mem_limit: 128m
environment:
PORT: 8000
MCP_PORT: 8000
MCP_HTTP_MODE: "true"
ENVIRONMENT: production
# ── CDC data plane (profile: cdc) ───────────────────────────────────────────
# kafka-connect and debezium-mcp are a set. The orchestrator's preflight
# requires both (plus kafka-mcp-sink, which is unprofiled because batch needs
# it too) for any CDC pipeline, so shipping a
# subset is strictly worse than shipping none: the two MCPs used to start
# unconditionally here while kafka-connect — the engine debezium-mcp is a bridge
# to — had no service at all, so a CDC pipeline failed preflight on a box that
# was already paying to run half of it.
#
# The two MCPs are not deployed on demand the way catalog connectors are:
# preflight's JIT last resort starts them without this file's Kafka client and
# security env (x-kafka-client-env), and only after polling 24 x 5s first. So
# the profile, not the first CDC pipeline, is what starts them.
kafka-connect:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/kafka-connect:${RSYNC_VERSION:-latest}
container_name: kafka-connect
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
profiles: ["cdc"]
restart: unless-stopped
# Parity with docker-compose.prod.yml, which was raised to 2048M after this
# container was OOM-killed repeatedly on the GCP host. The overrun is NATIVE,
# not heap: Debezium's BufferingChangeStreamCursor allocates outside the JVM
# heap, so KAFKA_HEAP_OPTS (-Xmx768M, below) does not bound it and raising
# Xmx would make the kill arrive SOONER, since the cgroup counts total RSS.
# Kernel evidence, 2026-09-23 — three separate kills at the old 1024M cap:
# Memory cgroup out of memory: Killed process (java) anon-rss:1042068kB
# 1536m left only 768 MiB of non-heap headroom; 2048m leaves 1280 MiB, which
# is what prod now runs. This file is the one-command self-host path, so it is
# where a first CDC snapshot of a large collection actually lands.
mem_limit: 2048m
depends_on:
kafka:
condition: service_healthy
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
environment:
# Unprefixed names on purpose: the debezium/connect entrypoint hand-exports
# this fixed list. See docker-compose.yml for the full annotation — anything
# else unprefixed is discarded in silence.
BOOTSTRAP_SERVERS: "${KAFKA_BROKERS:-kafka:29092}"
GROUP_ID: "rsync-connect-cluster"
CONFIG_STORAGE_TOPIC: "_rsync-connect-configs"
OFFSET_STORAGE_TOPIC: "_rsync-connect-offsets"
STATUS_STORAGE_TOPIC: "_rsync-connect-status"
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "${KAFKA_REPLICATION_FACTOR:-1}"
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "${KAFKA_REPLICATION_FACTOR:-1}"
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "${KAFKA_REPLICATION_FACTOR:-1}"
KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
OFFSET_FLUSH_INTERVAL_MS: 10000
ADVERTISED_HOST_NAME: "kafka-connect"
CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false"
CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
CONNECT_PRODUCER_ACKS: "all"
CONNECT_PRODUCER_RETRIES: 10
# Lockstep with the broker's message.max.bytes and the sink worker's
# ReaderConfig.MaxBytes — a row with a large JSON/text/blob column fails the
# task with RecordTooLargeException if these three disagree.
CONNECT_PRODUCER_MAX_REQUEST_SIZE: 15728640
CONNECT_CONSUMER_AUTO_OFFSET_RESET: "earliest"
# FileConfigProvider: source DB passwords resolve at task start from the
# shared connect_secrets volume, so they never enter the Kafka config topic
# nor the GET /connectors/<name>/config response.
CONNECT_CONFIG_PROVIDERS: "file"
CONNECT_CONFIG_PROVIDERS_FILE_CLASS: "org.apache.kafka.common.config.provider.FileConfigProvider"
CONNECT_CONFIG_PROVIDERS_FILE_PARAM_ALLOWED_PATHS: "/connect-secrets"
KAFKA_HEAP_OPTS: "-Xms256M -Xmx768M -XX:+ExitOnOutOfMemoryError"
# ── BYO-Kafka security ───────────────────────────────────────────────
# The base image's entrypoint turns every ^CONNECT_ variable into one
# connect-distributed.properties line (strip CONNECT_, lowercase, '_' ->
# '.') and DISCARDS everything else in silence. The KAFKA_* names the Go
# and Python services read still work here, because this image runs its
# own entrypoint first (connect-entrypoint.sh, the same code Helm runs)
# and it derives the CONNECT_* lines from them:
# * KAFKA_SASL_USERNAME/PASSWORD, or the KAFKA_SASL_OAUTHBEARER_* set,
# become the JAAS line for the mechanism;
# * KAFKA_SSL_CERT_LOCATION + KAFKA_SSL_KEY_LOCATION become the one-file
# PEM keystore a JVM needs (KAFKA_SSL_KEYSTORE_LOCATION, if set, wins).
# Mount the certificate files into this container too, at the same paths.
<<: *kafka-security
#
# Connect needs the profile in THREE places. The worker's own admin client
# is a different client from the producer each source task writes with and
# the consumer each sink task reads with, and the task-level ones do not
# inherit reliably. Configure only the worker and the failure is
# silent-partial: the worker authenticates, joins the group, answers the
# REST API and reports connectors RUNNING while every task fails its own
# handshake and not one row moves.
#
# Unset, these render exactly the bundled-broker behaviour: PLAINTEXT,
# with sasl.mechanism inert because no handshake is attempted.
CONNECT_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}"
CONNECT_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-PLAIN}"
CONNECT_PRODUCER_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}"
CONNECT_PRODUCER_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-PLAIN}"
CONNECT_CONSUMER_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}"
CONNECT_CONSUMER_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-PLAIN}"
#
# An OVERRIDE, normally left unset: the entrypoint builds the JAAS line
# from KAFKA_SASL_USERNAME/PASSWORD (or the OAuth set), choosing the login
# module the mechanism needs and escaping the values for the two grammars
# they cross. Set KAFKA_SASL_JAAS_CONFIG only for a login module it has no
# arm for; a non-empty value is used as-is and nothing is built. It then
# needs BOTH escapes -- a properties grammar, then a JAAS StreamTokenizer
# -- so `p"w` is written `p\\"w`; escaping once corrupts it identically
# to not escaping at all.
CONNECT_SASL_JAAS_CONFIG: "${KAFKA_SASL_JAAS_CONFIG:-}"
CONNECT_PRODUCER_SASL_JAAS_CONFIG: "${KAFKA_SASL_JAAS_CONFIG:-}"
CONNECT_CONSUMER_SASL_JAAS_CONFIG: "${KAFKA_SASL_JAAS_CONFIG:-}"
#
# Truststore PATH and PASSWORD are BARE pass-through keys -- no `${...:-}`
# default, deliberately. A key with no value is OMITTED from the container
# environment when the host variable is unset; `${VAR:-}` renders it
# present-but-EMPTY, and this entrypoint writes every CONNECT_ var it sees,
# empty included. Kafka then reads an empty ssl.truststore.location as a
# real path and the worker exits 2 with "Failed to load PEM SSL keystore".
# Worse for the password: Kafka REJECTS a truststore password outright for
# PEM stores, and an empty string counts as specified. Both would break the
# commonest BYO case -- MSK and Confluent Cloud chain to a public root and
# need no truststore setting at all.
#
# Set all THREE of each; the worker-only half is the classic partial.
# export CONNECT_SSL_TRUSTSTORE_LOCATION=/connect-certs/ca.pem
# export CONNECT_PRODUCER_SSL_TRUSTSTORE_LOCATION="$CONNECT_SSL_TRUSTSTORE_LOCATION"
# export CONNECT_CONSUMER_SSL_TRUSTSTORE_LOCATION="$CONNECT_SSL_TRUSTSTORE_LOCATION"
CONNECT_SSL_TRUSTSTORE_LOCATION:
CONNECT_PRODUCER_SSL_TRUSTSTORE_LOCATION:
CONNECT_CONSUMER_SSL_TRUSTSTORE_LOCATION:
CONNECT_SSL_TRUSTSTORE_PASSWORD:
CONNECT_PRODUCER_SSL_TRUSTSTORE_PASSWORD:
CONNECT_CONSUMER_SSL_TRUSTSTORE_PASSWORD:
# TYPE is safe to default: it is consulted only when a location is set, and
# JKS is Kafka's own default. Set KAFKA_SSL_TRUSTSTORE_TYPE=PEM for a PEM
# CA bundle -- one variable, all three clients.
CONNECT_SSL_TRUSTSTORE_TYPE: "${KAFKA_SSL_TRUSTSTORE_TYPE:-JKS}"
CONNECT_PRODUCER_SSL_TRUSTSTORE_TYPE: "${KAFKA_SSL_TRUSTSTORE_TYPE:-JKS}"
CONNECT_CONSUMER_SSL_TRUSTSTORE_TYPE: "${KAFKA_SSL_TRUSTSTORE_TYPE:-JKS}"
# Bare `-`, not `:-`: unset means Kafka's default ("https", verify), while
# an explicitly EMPTY value is Kafka's documented off switch and has to
# survive as empty -- `:-` would substitute the default back over it.
CONNECT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "${KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM-https}"
CONNECT_PRODUCER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "${KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM-https}"
CONNECT_CONSUMER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "${KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM-https}"
volumes:
# Read-only: this worker only RESOLVES the secrets debezium-mcp writes.
- connect_secrets:/connect-secrets:ro
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8083/"]
interval: 30s
timeout: 10s
retries: 5
start_period: 60s
debezium-mcp:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/mcp-debezium:${RSYNC_VERSION:-latest}
container_name: rsync-ai-debezium-v1-0-0-mcp
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
profiles: ["cdc"]
restart: unless-stopped
mem_limit: 512m
environment:
PORT: 8000
MCP_HTTP_MODE: "true"
# Where debezium-mcp WRITES the per-connector secret file that
# kafka-connect's FileConfigProvider reads back. Unset, it had nowhere to
# put them and the ${file:...} references in a CDC connector config would
# resolve to nothing — so the volume and this variable travel together.
DEBEZIUM_SECRETS_DIR: /connect-secrets
ENVIRONMENT: production
# BYO-Kafka. This service authors the Debezium connector config, and the
# connector's own producer plus its schema-history client are configured
# FROM here -- so an unsecured debezium-mcp writes a connector that dials
# a secured broker anonymously, which surfaces as a task that never leaves
# RUNNING and a snapshot that never starts.
#
# The ADDRESS travels with the credentials, and must: no caller passes
# kafka_bootstrap_servers, so _bootstrap_from_args() falls through to
# connector.py:533's `_env("KAFKA_BROKERS") or ... "kafka:29092"`. Without
# this line a BYO operator gets a connector whose schema history is
# SASL-credentialed for their cluster and ADDRESSED to the bundled broker
# -- so it authenticates against the wrong Kafka, or (under SASL_SSL) hangs
# handshaking with a PLAINTEXT one. The schema-history client is separate
# from the task producer, so the connector starts fine and only breaks on
# the restart that replays it.
KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:29092}
<<: *kafka-security
volumes:
- connect_secrets:/connect-secrets
kafka-mcp-sink-mcp:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/mcp-kafka-sink:${RSYNC_VERSION:-latest}
container_name: rsync-ai-kafka-mcp-sink-v1-0-0-mcp
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
# Deliberately NOT behind the cdc profile. The sink is the Kafka →
# destination writer for BATCH runs too (the executor starts a sink worker
# for pipeline.<id>.data before producing), and the orchestrator's preflight
# requires it for every pipeline with a destination
# (internal/workers/infra_preflight.go requiredServices). Profiled out, a
# batch-only install fails every batch run at preflight.
restart: unless-stopped
mem_limit: 768m
environment:
PORT: 8000
MCP_HTTP_MODE: "true"
MINIO_ENDPOINT_URL: http://minio:9000
MINIO_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required}
MINIO_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required}
MINIO_BUCKET: pipeline-data
ENVIRONMENT: production
# The sink owns the durable ACK ledger (pipeline_batch_acks) and fails
# CLOSED without it: main.go:3153-3156 reads POSTGRES_URL then DATABASE_URL,
# and main.go:3186-3196 logs "FATAL: ACK ledger (POSTGRES_URL / DATABASE_URL)
# is unavailable" and os.Exit(1)s unless RSYNC_SINK_ALLOW_NO_LEDGER=true.
# With neither name set the worker exits on every spawn and the Python
# supervisor respawns it forever, so nothing drains the data topic and the
# pipeline reports `completed` into an empty destination.
#
# sslmode is ${POSTGRES_SSLMODE:-disable}, NOT a hardcoded `disable`: the
# literal is what broke staging and prod against a TLS-requiring managed
# Postgres (see the two archived write-ups), and it is the one URL part no
# other env var could override. Same form as api-gateway's DATABASE_URL above.
POSTGRES_URL: postgres://${POSTGRES_USER:-rsync}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-pipeline_db}?sslmode=${POSTGRES_SSLMODE:-disable}
# BYO-Kafka. The Go sink worker reads exactly these names
# (kafka-mcp-sink/worker-src/.../kafka_security.go initKafkaSecurity) and
# is fatal on an INVALID profile -- but silent on an ABSENT one, where it
# defaults to PLAINTEXT and consumes anonymously. That is the failure this
# block exists to prevent: it is the last hop of the CDC path, so the
# symptom is a pipeline that reports `completed` with an empty destination.
#
# The address travels with the credentials. Today every caller passes it
# explicitly (kafka.SinkBootstrapServers()), so this changes no behaviour --
# it is here so the rule "anything merging x-kafka-security also declares a
# broker" holds without an exception, because the exception is what makes
# the next service's omission look deliberate.
# How many destination flushes this sink runs at once. Unset/empty = 4.
#
# A CDC data topic is born with min(3, brokers) partitions and Debezium gives
# every source table a topic of its own, so one worker is assigned tens of
# partitions. It consumed all of them already, but drained them one at a time
# because the destination write ran inline on the consume goroutine — one slow
# table stalled every other partition. Lanes are sharded on (topic, partition),
# so per-row order is untouched.
#
# Set to 0 or 1 to restore the old inline behaviour exactly; this is the kill
# switch. Raising it opens more concurrent connections to the DESTINATION, which
# is somebody else's database — 4 is deliberately modest.
RSYNC_SINK_FLUSH_LANES: ${RSYNC_SINK_FLUSH_LANES:-}
KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:29092}
<<: *kafka-security
# The sink worker writes to destination MCPs (e.g. rsync-ai-postgresql-v1-0-0-mcp)
# by container name. JIT-deployed connector MCPs live on the rsync-ai-mcp network
# (tool-generator sets DOCKER_NETWORK=rsync-ai-mcp), so without bridging the sink
# into it the sink resolves NOTHING and every destination write fails with
# "no such host" — the silent DLQ mode that lets a pipeline report `completed`
# while the destination stays empty. Mirrors docker-compose.yml (the regular stack).
networks:
- default
- rsync-ai-mcp
# Generation-time only (profile: generate). Every reference to context7 in the
# product is under llm-service/src/agents/tool_generator/, and the one call site
# is wrapped in try/except with a 3s timeout, so its absence is tolerated by
# design — a stack that only RUNS existing connectors never needs it.
context7-mcp:
image: ${RSYNC_IMAGE_REGISTRY:-ghcr.io/rsync-ai}/mcp-context7:${RSYNC_VERSION:-latest}
# Not `rsync-ai-<x>-mcp`: api-gateway reads that shape as a connector (tools.go
# getMCPDockerContainers), and this is the doc-lookup helper generation uses.
container_name: rsync-ai-context7
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
profiles: ["generate"]
restart: unless-stopped
mem_limit: 256m
environment:
PORT: 8080
CONTEXT7_API_URL: https://context7.com/api
CONTEXT7_API_KEY: ${CONTEXT7_API_KEY:-}
ENVIRONMENT: production
volumes:
postgres_data:
demo_warehouse_data:
redis_data:
kafka_data:
minio_data:
blob_staging_data:
# Shared between debezium-mcp (writes) and kafka-connect (reads, ro) so source
# DB credentials never transit the Kafka config topic.
connect_secrets:
mcp_connectors:
name: rsync-ai-mcp-connectors
networks:
rsync-ai-mcp:
name: rsync-ai-mcp