docker-compose.yml
Rama main · réplica de lectura del repositorio original.
name: rsync-ai
# ─────────────────────────────────────────────────────────────────────────────
# Kafka client security, for a broker this file did not start.
#
# Point KAFKA_BROKERS at a managed cluster (MSK, Confluent Cloud, Aiven) and the
# clients below still default to PLAINTEXT unless these names reach them. Compose
# injects only what a service DECLARES, so a credential sitting in .env reaches
# interpolation but not the container: the operator sets a password, the client
# connects anonymously, and an ACL-enforcing broker rejects it with an error
# naming the BROKER rather than the missing setting.
#
# Every key defaults to empty and empty means "unset" -- kafkaclient reads
# `protocolWasSet := c.SecurityProtocol != ""` (shared/go/kafkaclient/config.go),
# so the bundled-broker default renders exactly as it did before these keys
# existed. Kept key-for-key in lockstep with the same anchor in
# docker-compose.quickstart.yml; tests/test_compose_kafka_security_env.py fails
# if the two drift.
# ─────────────────────────────────────────────────────────────────────────────
x-kafka-security: &kafka-security
KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-}
KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-}
KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-}
KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-}
# OIDC client-credentials, for KAFKA_SASL_MECHANISM=OAUTHBEARER.
KAFKA_SASL_OAUTHBEARER_CLIENT_ID: ${KAFKA_SASL_OAUTHBEARER_CLIENT_ID:-}
KAFKA_SASL_OAUTHBEARER_CLIENT_SECRET: ${KAFKA_SASL_OAUTHBEARER_CLIENT_SECRET:-}
KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT: ${KAFKA_SASL_OAUTHBEARER_TOKEN_ENDPOINT:-}
KAFKA_SASL_OAUTHBEARER_SCOPE: ${KAFKA_SASL_OAUTHBEARER_SCOPE:-}
KAFKA_SASL_OAUTHBEARER_EXTENSIONS: ${KAFKA_SASL_OAUTHBEARER_EXTENSIONS:-}
# Re-permits a plain-http token endpoint off loopback, which every runtime
# otherwise refuses at start-up (the client-credentials grant POSTs
# CLIENT_SECRET on every fetch). Carried here for ONE reason: kafka-connect
# and the kafka-init job read their whole security profile from this anchor,
# so without a passthrough a rig with a throwaway http IdP can configure the
# Go and Python services but not the JVM ones -- the same image behaving
# differently under compose than under the chart, which does make it
# reachable. The default is empty, i.e. unset, i.e. the refusal stays armed;
# a passthrough is not a setting. Never give it a value in this file or in
# docker-compose.prod.yml --
# llm-service/tests/test_insecure_token_endpoint_rule_is_one_rule.py fails if
# any file but the Kafka security matrix harness sets it to true.
KAFKA_SASL_OAUTHBEARER_ALLOW_INSECURE_TOKEN_ENDPOINT: ${KAFKA_SASL_OAUTHBEARER_ALLOW_INSECURE_TOKEN_ENDPOINT:-}
# JVM-only; in this file only kafka-connect has a JVM. Carried anyway because
# Debezium's schema-history client is configured from the connector
# container's environment, so the name has to survive the trip.
KAFKA_SASL_OAUTHBEARER_LOGIN_CALLBACK_HANDLER: ${KAFKA_SASL_OAUTHBEARER_LOGIN_CALLBACK_HANDLER:-}
# AWS_MSK_IAM. The Go services implement it (kafkaclient/tokenauth/msk.go); the
# Python tier and the JVM Connect image do not, so an IAM-only cluster runs the
# Go data plane and leaves the CDC profile unable to authenticate.
KAFKA_AWS_REGION: ${KAFKA_AWS_REGION:-}
# Paths INSIDE the container. Mount the files yourself — this file cannot guess
# a host path.
KAFKA_SSL_CA_LOCATION: ${KAFKA_SSL_CA_LOCATION:-}
# mTLS, the two-path shape the Go and Python clients read.
KAFKA_SSL_CERT_LOCATION: ${KAFKA_SSL_CERT_LOCATION:-}
KAFKA_SSL_KEY_LOCATION: ${KAFKA_SSL_KEY_LOCATION:-}
# The SAME keypair in the shape a JVM can load: ONE file holding chain + key.
# Optional -- kafka-init and kafka-connect build that file from the pair above
# when this is unset (the connect image's connect-entrypoint.sh). Set it only
# to use a combined file you already have; it then wins over the pair. Inert
# in the Go and Python services.
KAFKA_SSL_KEYSTORE_LOCATION: ${KAFKA_SSL_KEYSTORE_LOCATION:-}
# Disables certificate verification, and is logged unconditionally when set: a
# self-signed lab, not prod.
#
# TWO SPELLINGS FOR ONE SETTING, and that is not tidy-uppable here. Python
# reads only KAFKA_SSL_SKIP_VERIFY (kafka_security.py:54); Go reads both
# (config.go:80 and the alias at :115). Shipping the Go spelling alone leaves
# verification silently ON for the Python half -- the exact split the Go alias
# was added to end. So both names are emitted, each defaulting to the other,
# and an operator who sets either one gets both.
# TWO SPELLINGS FOR ONE SETTING, and that is not tidy-uppable here. Python
# reads only KAFKA_SSL_SKIP_VERIFY (kafka_security.py:54); Go reads both
# (config.go:80 and the alias at :115). Shipping the Go spelling alone leaves
# verification silently ON for the Python half -- the exact split the Go alias
# was added to end. So both names are emitted, each defaulting to the other,
# and an operator who sets either one gets both. Identical to the pair in
# docker-compose.quickstart.yml; the key SETS of the two anchors must match,
# which llm-service/tests/test_compose_kafka_security_env.py asserts.
KAFKA_SSL_SKIP_VERIFY: ${KAFKA_SSL_SKIP_VERIFY:-${KAFKA_SSL_INSECURE_SKIP_VERIFY:-}}
KAFKA_SSL_INSECURE_SKIP_VERIFY: ${KAFKA_SSL_INSECURE_SKIP_VERIFY:-${KAFKA_SSL_SKIP_VERIFY:-}}
# ─────────────────────────────────────────────────────────────────────────────
# Build info baked into the api-gateway, orchestrator, temporal-adapter and
# llm-service images: each serves it on GET /version, and the admin drift check
# (GET /api/v1/admin/drift) compares the commits. Compose cannot run git, so the
# caller exports these -- scripts/deploy-service.sh does, and so do the upgrade
# steps in docs/deployment/self-hosting.md. A build without them bakes in
# "dev"/"unknown", which the drift check reports as "commit unknown".
# Published images get the release sha from .github/workflows/docker-publish.yml.
# ─────────────────────────────────────────────────────────────────────────────
x-build-info: &build-info
GIT_COMMIT: ${GIT_COMMIT:-dev}
BUILD_TIME: ${BUILD_TIME:-unknown}
services:
# ============================================
# INFRASTRUCTURE
# ============================================
# NOTE: this bundle ships an OTel Collector but no telemetry backend.
# Services export to otel-collector:4317; the collector forwards to whatever
# OTLP endpoint OTLP_BACKEND_ENDPOINT names. Leave it at the default and
# nothing is stored — the supported story here is `docker compose logs`.
# Read-only Docker socket proxy. Lets api-gateway surface LIVE MCP container
# status (docker_status=running, port), and lets the orchestrator's sentinel tell
# a connector this host never deployed from one that is deployed and broken —
# both without mounting the raw socket or running as root, since it can only GET
# /containers/json (POST disabled). Wired via MCP_DOCKER_API_URL on both. Prod
# defines its own copy for Traefik in docker-compose.prod.yml; this base copy
# also serves api-gateway and the orchestrator locally.
docker-socket-proxy:
# Pinned, not `latest`. This container IS the docker-socket boundary, and `latest`
# moved v0.4.2 -> v0.5.0 on 2026-07-27 with no change on our side; deploy-service.sh
# runs `--force-recreate`, so that arrives on the next deploy of this service.
# v0.5.0 is the digest `latest` resolves to today, so this pin froze it, not changed it.
image: tecnativa/docker-socket-proxy:v0.5.0
container_name: rsync-ai-docker-socket-proxy
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
environment:
CONTAINERS: 1
NETWORKS: 1
SERVICES: 1
TASKS: 1
VERSION: 1
POST: 0
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
networks:
- default
# PostgreSQL Database (with CDC/Logical Replication enabled)
postgres:
image: postgres:16-alpine
container_name: postgres
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
ports:
# Every host port in this file binds 127.0.0.1. A port with no host IP binds
# 0.0.0.0, and Docker writes its own iptables rules AHEAD of ufw/firewalld,
# so "the firewall is on" does not close it -- on a laptop that is the coffee
# shop's Wi-Fi reaching an unauthenticated Postgres, Kafka, Redis and the
# orchestrator's internal routes. The two user-facing ports (frontend,
# api-gateway) read RSYNC_BIND_ADDR like docker-compose.quickstart.yml does,
# so a dev who wants the UI on the LAN sets that one knob and exposes only
# those two. Nothing else here is meant to be reachable off the box.
- "127.0.0.1:${RSYNC_HP_POSTGRES:-5432}:5432"
environment:
- POSTGRES_USER=user
- POSTGRES_PASSWORD=password
- POSTGRES_DB=pipeline_db
command:
- "postgres"
- "-c"
- "wal_level=logical"
- "-c"
- "max_wal_senders=1"
- "-c"
- "max_replication_slots=1"
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U user -d pipeline_db"]
interval: 5s
timeout: 5s
retries: 5
# Kafka (KRaft Mode - No ZooKeeper)
# Uses apache/kafka — Apache 2.0 licensed (replaces Confluent cp-kafka which is CCL licensed)
# apache/kafka uses the same KAFKA_* env var convention as cp-kafka — no rename needed.
kafka:
image: apache/kafka:3.7.0
hostname: kafka
container_name: rsync-ai-kafka
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
ports:
- "127.0.0.1:${RSYNC_HP_KAFKA:-9092}:9092"
environment:
KAFKA_NODE_ID: 1
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
# Partition counts for the two INTERNAL topics. Kafka defaults both to 50,
# and nothing in this repo overrode them, so a single bundled broker was
# carrying 100 internal partitions -- each one a log dir with its own
# segment plus offset/time index, against KAFKA_HEAP_OPTS of -Xmx512m.
# This broker serves one platform with a handful of consumer groups; 8 is
# ample coordinator spread and cuts the internal partition count by ~84%.
#
# Takes effect on a FRESH cluster ONLY. __consumer_offsets and
# __transaction_state are created once, on first use, with whatever this
# said at the time; an existing volume keeps its 50 and ignores the
# setting. So this buys nothing on a running deployment and everything on
# a new self-host install -- do NOT delete the topics to "apply" it, that
# discards every committed offset.
KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 8
KAFKA_TRANSACTION_STATE_LOG_NUM_PARTITIONS: 8
# Consumer group rebalance settings (CRITICAL for multi-worker stability)
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 3000
KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS: 180000
KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 6000
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_PROCESS_ROLES: 'broker,controller'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
# Bind to 0.0.0.0 (canonical fix for i/o timeouts)
KAFKA_LISTENERS: 'PLAINTEXT://0.0.0.0:29092,CONTROLLER://0.0.0.0:29093,PLAINTEXT_HOST://0.0.0.0:9092'
KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
KAFKA_LOG_DIRS: '/var/lib/kafka/data'
# Broker cap on a single record. The 1 MB default rejects CDC change events for
# rows with large JSON/text/blob columns (RecordTooLargeException → Debezium task
# FAILED → restart loop). Kept in LOCKSTEP with CONNECT_PRODUCER_MAX_REQUEST_SIZE
# (kafka-connect, producer side) and the kafka-sink-worker ReaderConfig.MaxBytes
# (consumer side, which must exceed this). Topics inherit this broker default.
KAFKA_MESSAGE_MAX_BYTES: 15728640
# Retention policy for topics rsync.ai does NOT create itself.
#
# This is not redundant with kafka/topology.go: Debezium auto-creates the
# rsync.cdc-* topics before the orchestrator's topology manager would, so
# those topics carry NO per-topic overrides and take whatever the broker
# says. Verified on the GCP host 2026-09-23 — `kafka-configs.sh --describe`
# returns an empty dynamic config set for the live CDC topic, so the code's
# values had never applied to it. This line is therefore the only place the
# 7-day policy actually reaches them.
#
# 168h/delete matches Apache's own defaults, so this changes no behaviour
# today. It pins it: an image whose defaults differ can no longer silently
# re-open the unbounded-growth hole that fills the disk and stops BOTH data
# planes. Topics rsync.ai creates explicitly still carry their own config, and
# the one deliberate never-expire exception (the Debezium schema-history
# topic) sets retention.ms=-1 per topic and overrides this.
KAFKA_LOG_RETENTION_HOURS: 168
KAFKA_LOG_CLEANUP_POLICY: 'delete'
CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk'
KAFKA_HEAP_OPTS: "-Xms256m -Xmx512m -XX:+ExitOnOutOfMemoryError"
volumes:
- kafka_data:/var/lib/kafka/data
healthcheck:
test: /opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1
interval: 30s
timeout: 15s
retries: 5
start_period: 60s
# One-shot bootstrapper for the platform topics (pipeline.domain.events,
# pii.scan.request, pii.scan.response); see scripts/kafka-init-new-topics.sh.
# Ensures test suites that expect topics to exist are deterministic.
kafka-init:
image: apache/kafka:3.7.0
container_name: rsync-ai-kafka-init
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
depends_on:
kafka:
condition: service_healthy
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
entrypoint: ["/bin/bash", "-lc"]
# A bare `command: |` block is split into argv by Compose, so `bash -lc` only
# ever received the first token ("export") and the container exited 0 without
# running the script -- a green init that created nothing. The single-element
# list keeps the whole script as one argument. Verified with
# `docker compose config`: `command:` must render as ONE list item.
command:
- |
export PATH="/opt/kafka/bin:$$PATH";
# Same broker the orchestrator and kafka-connect use. Hardcoding
# kafka:29092 here gave KAFKA_REPLICATION_FACTOR two meanings in one
# file -- "RF on the customer's BYO cluster" for every other service,
# "RF on the bundled single broker" for this one -- so the BYO operator
# who correctly sets RF=3 is the one whose bootstrap dies with
# InvalidReplicationFactorException against the bundled broker.
# A BYO cluster with SASL/TLS also needs a client.properties, which
# kafka-init-new-topics.sh now builds from the x-kafka-security env
# merged in below and passes as --command-config.
export KAFKA_BROKER="${KAFKA_BROKERS:-kafka:29092}";
export PARTITIONS="3";
bash /scripts/kafka-init-new-topics.sh
environment:
# The topic bootstrapper is a Kafka CLIENT too. Without these it speaks
# PLAINTEXT to a SASL listener, which does not error -- it BLOCKS in the
# handshake until the CLI's default timeout gives up, so the run appears
# to hang rather than to be misconfigured. See x-kafka-security at the top
# of this file; scripts/kafka-init-new-topics.sh turns them into the
# --command-config the Kafka CLIs need.
<<: *kafka-security
# The bootstrapper must land on the same names the services subscribe to.
# Bare `-`, matching kafkaclient.TopicPrefix(): an explicitly empty prefix
# is the migration lever and must survive, which `:-` would overwrite.
KAFKA_TOPIC_PREFIX: ${KAFKA_TOPIC_PREFIX-rsync.}
# Same durability knobs the orchestrator reads, so topics created by the
# bootstrapper and topics created at runtime agree. Passed as env rather
# than exported in `command:` above so an operator override actually
# reaches the script -- a hardcoded `export REPLICATION_FACTOR="1"` there
# would win over anything set here.
KAFKA_REPLICATION_FACTOR: ${KAFKA_REPLICATION_FACTOR:-}
KAFKA_MIN_INSYNC_REPLICAS: ${KAFKA_MIN_INSYNC_REPLICAS:-}
volumes:
- ./scripts:/scripts:ro
restart: "no"
# Schema Registry — OPTIONAL, Apache 2.0 (Apicurio replaces Confluent cp-schema-registry)
# Enable with: docker compose --profile schema-registry up -d
schema-registry:
image: apicurio/apicurio-registry:3.0.6
hostname: schema-registry
container_name: schema-registry
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
profiles:
- schema-registry
depends_on:
kafka:
condition: service_healthy
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
ports:
- "127.0.0.1:${RSYNC_HP_SCHEMA_REGISTRY:-8085}:8080"
environment:
# No KAFKA_*/REGISTRY_* keys here, on purpose. Apicurio 3 reads apicurio.*-prefixed
# config; KAFKA_BOOTSTRAP_SERVERS and REGISTRY_KAFKASQL_TOPIC (the Apicurio-2 spelling)
# occur ZERO times in the runner jar, in all 356 shipped lib jars, and in the
# entrypoint -- they were inert. Storage here is the image default sql/H2, NOT kafkasql,
# so there is no broker to name.
#
# THE x-kafka-security ANCHOR ABOVE CANNOT SECURE THIS IMAGE, and an earlier version of
# this comment wrongly said a kafkasql switch would need it. All 17 of its keys are as
# inert here as the two deleted above: in both the env spelling and SmallRye's dotted
# equivalent (34 spellings) they occur 0 times in the runner jar (2,419 entries) and
# 0 times across all 356 lib jars (57,339 entries) -- against live controls measured in
# the same scan (security.protocol 3 runner / 6 lib entries, sasl.mechanism 7 / 3,
# bootstrap.servers 8 / 5). Merging it would read as secured and speak PLAINTEXT.
#
# What the image DOES read for a kafkasql client is declared by
# io/apicurio/registry/storage/impl/kafkasql/KafkaSqlFactory.class in the runner jar:
# APICURIO_KAFKASQL_SECURITY_PROTOCOL plus APICURIO_KAFKASQL_SECURITY_SASL_{ENABLED,
# MECHANISM,CLIENT_ID,CLIENT_SECRET,TOKEN_ENDPOINT,LOGIN_CALLBACK_HANDLER_CLASS}. Image
# defaults (application.properties): sasl.enabled=false (:22), mechanism=OAUTHBEARER
# (:196), client-id/client-secret=sa (:157/:6), token.endpoint=http://localhost:8090
# (:87), login.callback.handler.class=Strimzi's JaasClientOauthLoginCallbackHandler
# (:162). Two of those properties are spelled with a HYPHEN -- `...sasl.client-id`,
# `...sasl.client-secret` -- and the env names above are SmallRye's documented mapping
# of dots AND dashes to `_`, so the underscore spelling is correct despite the hyphen.
#
# A real switch is therefore APICURIO_STORAGE_KIND=kafkasql +
# APICURIO_KAFKASQL_BOOTSTRAP_SERVERS + that family, NOT the anchor.
# test_compose_kafka_security_env.py requires exactly that family (and refuses to let
# the anchor stand in for it) for any service naming the Apicurio broker key.
#
# NOTE: H2 here is in-memory and this service declares no volumes: -- schemas do NOT
# survive a restart.
QUARKUS_HTTP_PORT: 8080
healthcheck:
test: curl -f http://localhost:8080/health/ready || exit 1
interval: 10s
timeout: 5s
retries: 5
# ============================================
# TEMPORAL - Workflow Orchestration
# ============================================
# Temporal Server
temporal:
image: temporalio/auto-setup:1.22.7
container_name: temporal
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
depends_on:
postgres:
condition: service_healthy
# Other half of docker-compose.byo-postgres.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
environment:
- DB=postgresql
- DB_PORT=5432
- POSTGRES_USER=user
- POSTGRES_PWD=password
- POSTGRES_SEEDS=postgres
- DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml
ports:
- "127.0.0.1:${RSYNC_HP_TEMPORAL:-7233}:7233"
healthcheck:
test: ["CMD", "tctl", "--address", "temporal:7233", "workflow", "list"]
interval: 10s
timeout: 5s
retries: 5
volumes:
- ./deploy/temporal:/etc/temporal/config/dynamicconfig
# Temporal UI
temporal-ui:
image: temporalio/ui:2.21.3
container_name: temporal-ui
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
depends_on:
- temporal
environment:
- TEMPORAL_ADDRESS=temporal:7233
- TEMPORAL_CORS_ORIGINS=http://localhost:3000
ports:
- "127.0.0.1:${RSYNC_HP_TEMPORAL_UI:-8233}:8080"
# Temporal Admin Tools
temporal-admin-tools:
image: temporalio/admin-tools:1.22.7
container_name: temporal-admin-tools
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
depends_on:
- temporal
environment:
- TEMPORAL_ADDRESS=temporal:7233
stdin_open: true
tty: true
# Temporal Adapter (Bridges Temporal ↔ Kafka)
temporal-adapter:
build:
# Root context: the adapter `replace`s ../shared/go/kafkaclient, which a
# ./backend-temporal-adapter context cannot reach. Matches api-gateway
# and orchestrator, which already build from the repo root.
context: .
dockerfile: backend-temporal-adapter/Dockerfile
args: *build-info
container_name: rsync-ai-temporal-adapter
depends_on:
temporal:
condition: service_healthy
kafka:
condition: service_healthy
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
environment:
# Kafka client security for a BYO/managed broker. See x-kafka-security at
# the top of this file; every key defaults to empty and empty means unset,
# so the bundled broker is unaffected.
<<: *kafka-security
TEMPORAL_ADDRESS: "temporal:7233"
KAFKA_BOOTSTRAP_SERVERS: "${KAFKA_BROKERS:-kafka:29092}"
# Namespace for every Kafka topic this product creates, so an operator can
# tell them apart from other applications' topics on a shared (BYO-Kafka)
# cluster. Every service that touches Kafka must agree on this value: a
# producer and a consumer resolving different names do not error, the
# consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
# an explicit empty value means "disable" (keep pre-prefix topic names) and
# `:-` would silently turn that back into the default.
KAFKA_TOPIC_PREFIX: "${KAFKA_TOPIC_PREFIX-rsync.}"
LOG_FORMAT: "json"
LOG_LEVEL: "info"
ENVIRONMENT: "development"
# SECURITY: Must match api-gateway/orchestrator ENCRYPTION_KEY for decrypting connection configs in healing activities
ENCRYPTION_KEY: "${ENCRYPTION_KEY:-dev-encryption-key-32-bytes-long!!}"
# Database (StateUpdateActivity / connection validation lookups)
POSTGRES_HOST: "postgres"
POSTGRES_PORT: "5432"
POSTGRES_USER: "user"
POSTGRES_PASSWORD: "password"
POSTGRES_DB: "pipeline_db"
# V2 Correlation Store (Phase D)
REDIS_ADDRESS: "redis:6379"
REDIS_PASSWORD: "${REDIS_PASSWORD:-}"
# Scheduled saved-query model runs (RunModelActivity) call the gateway's internal
# endpoint over authenticated S2S. Without both of these a fired schedule fails
# closed instead of rebuilding its table.
API_GATEWAY_URL: "http://api-gateway:8080"
INTERNAL_SERVICE_SECRET: "${INTERNAL_SERVICE_SECRET:-}"
# OpenTelemetry
OTEL_EXPORTER_OTLP_ENDPOINT: "otel-collector:4317"
OTEL_SERVICE_NAME: "temporal-adapter"
OTEL_ENABLED: "${OTEL_ENABLED:-false}"
ports:
- "127.0.0.1:${RSYNC_HP_TEMPORAL_ADAPTER:-8082}:8082"
volumes:
# Required for connector filesystem checks + (optional) connector generation output
- ./shared:/app/shared
extra_hosts:
- "host.docker.internal:host-gateway"
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
# ============================================
# CDC & DATA MOVEMENT
# ============================================
# Kafka Connect (Debezium) — required for CDC pipelines
kafka-connect:
build:
context: ./shared/internal/infra/kafka-connect
dockerfile: Dockerfile
container_name: kafka-connect
# Without this the container defaults to RestartPolicy=no, and a single OOM
# kill stops CDC capture PERMANENTLY: Debezium is the only producer into the
# rsync.cdc-* topics, so the sink drains what is already there, every topic
# reaches lag 0, and the pipeline still reads `running` in the control plane.
# A dead producer and a caught-up one are indistinguishable from lag alone.
# Measured on the demo VM 2026-09-23: 877k source rows silently uncaptured
# until the container was started by hand. kafka-mcp-sink-mcp, the consuming
# half of the same path, has carried this policy all along.
restart: unless-stopped
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
depends_on:
kafka:
condition: service_healthy
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
ports:
- "127.0.0.1:${RSYNC_HP_CONNECT:-8083}:8083"
environment:
# ── debezium/connect env-var convention (NOT Confluent cp-kafka-connect) ──
# The Debezium image's entrypoint reads these UNPREFIXED names to build
# connect-distributed.properties. The old CONNECT_* names were silently
# ignored → "CONFIG_STORAGE_TOPIC must be set" crash.
BOOTSTRAP_SERVERS: "${KAFKA_BROKERS:-kafka:29092}"
GROUP_ID: "rsync-connect-cluster"
CONFIG_STORAGE_TOPIC: "_rsync-connect-configs"
OFFSET_STORAGE_TOPIC: "_rsync-connect-offsets"
STATUS_STORAGE_TOPIC: "_rsync-connect-status"
# Replication factor for Connect's three internal topics.
#
# These MUST carry the CONNECT_ prefix. The unprefixed spelling used here
# before (CONFIG_STORAGE_REPLICATION_FACTOR: 1) was silently ignored: the
# entrypoint hand-exports a FIXED list of unprefixed names (BOOTSTRAP_SERVERS,
# GROUP_ID, the three *_STORAGE_TOPICs, the converters, the flush timers) and
# the string "REPLICATION" appears nowhere in it, so anything else unprefixed
# is dropped without a word. It read as working only because the value being
# ignored (1) happened to equal the image's shipped default. Verified against
# quay.io/debezium/connect:3.1.3.Final: with the unprefixed name set to 7 the
# generated connect-distributed.properties still said 1.
#
# min.insync.replicas has no Connect worker property, so RF is the only lever
# here -- which makes it the one that matters. On a BYO cluster whose broker
# default is min.insync.replicas=2 (MSK, most managed clusters running a
# pre-3.7 broker), RF=1 internal topics are created successfully and are then
# unwritable: the worker starts, the REST API answers, and storing a connector
# config hangs.
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "${KAFKA_REPLICATION_FACTOR:-1}"
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "${KAFKA_REPLICATION_FACTOR:-1}"
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "${KAFKA_REPLICATION_FACTOR:-1}"
KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
OFFSET_FLUSH_INTERVAL_MS: 10000
ADVERTISED_HOST_NAME: "kafka-connect"
# JSON converters without embedded schemas (Debezium honors CONNECT_-prefixed
# extras as additional worker properties).
CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false"
CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
CONNECT_PRODUCER_ACKS: "all"
CONNECT_PRODUCER_RETRIES: 10
# Worker-level producer cap so EVERY connector (incl. Debezium CDC) may emit records
# for rows with large JSON/text/blob columns instead of failing the task with
# RecordTooLargeException. Worker-level (not per-connector producer.override.*) so it
# needs no client-config override policy. LOCKSTEP with KAFKA_MESSAGE_MAX_BYTES (broker)
# and the kafka-sink-worker ReaderConfig.MaxBytes (consumer).
CONNECT_PRODUCER_MAX_REQUEST_SIZE: 15728640
CONNECT_CONSUMER_AUTO_OFFSET_RESET: "earliest"
# ── BYO-Kafka security: SASL / TLS ───────────────────────────────────────
# Mapping convention, read out of the pinned base image's
# /docker-entrypoint.sh (quay.io/debezium/connect:3.1.3.Final, the
# "Process all environment variables that start with 'CONNECT_'" loop):
# every env var matching ^CONNECT_ becomes one connect-distributed.properties
# line -- strip CONNECT_, lowercase, translate '_' to '.'. So
# CONNECT_SASL_JAAS_CONFIG -> sasl.jaas.config. A variable that does NOT
# match ^CONNECT_ is discarded in silence, which is why these carry the
# prefix while BOOTSTRAP_SERVERS/GROUP_ID/*_STORAGE_TOPIC above deliberately
# do not (those few are re-exported as CONNECT_* by the entrypoint itself).
#
# The KAFKA_* contract every other service reads (x-kafka-security at the
# top of this file) still arrives here, because this image runs its own
# entrypoint first (shared/internal/infra/kafka-connect/connect-entrypoint.sh,
# the same code Helm runs) and it derives the CONNECT_* lines from it:
# * KAFKA_SASL_USERNAME/PASSWORD, or the KAFKA_SASL_OAUTHBEARER_* set,
# become the JAAS line for the mechanism;
# * KAFKA_SSL_CERT_LOCATION + KAFKA_SSL_KEY_LOCATION become the one-file
# PEM keystore a JVM needs (KAFKA_SSL_KEYSTORE_LOCATION, if set, wins).
# Mount the certificate files into this container too, at the same paths.
<<: *kafka-security
#
# Connect needs this in THREE places. The worker's own coordination/admin
# client is a different client from the producer each source task writes
# with and the consumer each sink task reads with, and the task-level ones
# do NOT inherit reliably. Configure only the worker and the failure is
# silent-partial: the worker authenticates, joins the group, serves the REST
# API and reports connectors RUNNING, while every task fails its own
# handshake and not one row moves.
#
# Unset, these render the pre-existing local behaviour exactly:
# security.protocol=PLAINTEXT, and sasl.mechanism is inert because no SASL
# handshake is attempted.
CONNECT_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}"
CONNECT_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-PLAIN}"
CONNECT_PRODUCER_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}"
CONNECT_PRODUCER_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-PLAIN}"
CONNECT_CONSUMER_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}"
CONNECT_CONSUMER_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-PLAIN}"
#
# An OVERRIDE, normally left unset: the entrypoint builds the JAAS line from
# KAFKA_SASL_USERNAME/PASSWORD (or the OAuth set), choosing the login module
# the mechanism needs and escaping the values for the two grammars they
# cross. Set KAFKA_SASL_JAAS_CONFIG only for a login module it has no arm
# for; a non-empty value is used as-is and nothing is built.
CONNECT_SASL_JAAS_CONFIG: "${KAFKA_SASL_JAAS_CONFIG:-}"
CONNECT_PRODUCER_SASL_JAAS_CONFIG: "${KAFKA_SASL_JAAS_CONFIG:-}"
CONNECT_CONSUMER_SASL_JAAS_CONFIG: "${KAFKA_SASL_JAAS_CONFIG:-}"
#
# Truststore PATH -- and only the path -- is a bare pass-through key with no
# `${...:-}` default. A key with no value is OMITTED from the container
# environment when the host variable is unset, whereas `${VAR:-}` renders it
# present-but-empty, and the debezium entrypoint writes every CONNECT_ var it
# sees, empty included. Kafka then reads an empty ssl.truststore.location as a
# real path: "Failed to load PEM SSL keystore" / NoSuchFileException, worker
# exit 2 (reproduced against this image). An empty default would therefore
# break the commonest BYO case outright -- Confluent Cloud and MSK present
# certificates chaining to a public CA that is already in the JVM default
# truststore, so they need NO truststore setting at all.
#
# Compose cannot omit a key conditionally on a DIFFERENTLY named variable, so
# the path keeps the CONNECT_ names themselves, and all THREE must be set.
# Setting only the worker one is the classic Connect partial failure: the
# worker starts and the REST API answers, while source-task producers and
# sink-task consumers fall back to the default truststore and fail their own
# handshake later, far from the config that caused it.
#
# export CONNECT_SSL_TRUSTSTORE_LOCATION=/connect-certs/ca.pem
# export CONNECT_PRODUCER_SSL_TRUSTSTORE_LOCATION="$CONNECT_SSL_TRUSTSTORE_LOCATION"
# export CONNECT_CONSUMER_SSL_TRUSTSTORE_LOCATION="$CONNECT_SSL_TRUSTSTORE_LOCATION"
# export KAFKA_SSL_TRUSTSTORE_TYPE=PEM # one variable, reaches all three
#
CONNECT_SSL_TRUSTSTORE_LOCATION:
CONNECT_PRODUCER_SSL_TRUSTSTORE_LOCATION:
CONNECT_CONSUMER_SSL_TRUSTSTORE_LOCATION:
#
# The PASSWORD is pass-through for the same reason, and the reason is sharper
# than it looks. Kafka rejects a truststore password outright for PEM stores:
#
# InvalidConfigurationException: SSL trust store password cannot be
# specified for PEM format.
#
# and an EMPTY string counts as specified (ConfigDef turns "" into a
# non-null Password). So `${KAFKA_SSL_TRUSTSTORE_PASSWORD:-}` would kill the
# worker at boot for every operator using a PEM CA bundle -- the modern,
# password-less form -- while blaming a password they never set. Verified
# against this image: PEM + empty password exits 2, PEM + absent password
# starts and reaches the broker.
CONNECT_SSL_TRUSTSTORE_PASSWORD:
CONNECT_PRODUCER_SSL_TRUSTSTORE_PASSWORD:
CONNECT_CONSUMER_SSL_TRUSTSTORE_PASSWORD:
# The TYPE is safe to default and so takes one variable for all three
# clients: it is consulted only when a location is set, and JKS is Kafka's
# own default, so an always-present ssl.truststore.type=JKS changes nothing.
CONNECT_SSL_TRUSTSTORE_TYPE: "${KAFKA_SSL_TRUSTSTORE_TYPE:-JKS}"
CONNECT_PRODUCER_SSL_TRUSTSTORE_TYPE: "${KAFKA_SSL_TRUSTSTORE_TYPE:-JKS}"
CONNECT_CONSUMER_SSL_TRUSTSTORE_TYPE: "${KAFKA_SSL_TRUSTSTORE_TYPE:-JKS}"
# Hostname verification. Bare `-`, not `:-`: unset means Kafka's default
# ("https", verify), while an explicitly EMPTY value is Kafka's documented off
# switch and has to survive as empty -- `:-` would substitute the default back
# over that deliberate choice. Same idiom as KAFKA_TOPIC_PREFIX.
CONNECT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "${KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM-https}"
CONNECT_PRODUCER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "${KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM-https}"
CONNECT_CONSUMER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "${KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM-https}"
# ── FileConfigProvider ──
# Resolve ${file:/connect-secrets/<name>.properties:key} references at task
# start so source DB passwords never enter the Kafka config topic nor the
# GET /connectors/<name>/config REST response. debezium-mcp writes the
# per-connector secret file to the shared connect_secrets volume; this worker
# reads it (ro). allowed.paths confines the provider to the secrets dir
# (Kafka >= 3.5; debezium/connect:3.0 ships Kafka 3.8).
CONNECT_CONFIG_PROVIDERS: "file"
CONNECT_CONFIG_PROVIDERS_FILE_CLASS: "org.apache.kafka.common.config.provider.FileConfigProvider"
CONNECT_CONFIG_PROVIDERS_FILE_PARAM_ALLOWED_PATHS: "/connect-secrets"
# JVM heap (avoid OOM exit 137 on dev machines)
KAFKA_HEAP_OPTS: "-Xms256M -Xmx768M -XX:+ExitOnOutOfMemoryError"
volumes:
# Read-only: the worker only RESOLVES secrets that debezium-mcp writes.
- connect_secrets:/connect-secrets:ro
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8083/"]
interval: 30s
timeout: 10s
retries: 5
start_period: 60s
# ============================================
# INTERNAL MCP SERVICES (CORE / SELF-HOSTED)
# ============================================
# These are platform/internal connectors that run in the core `rsync-ai` project.
# Only user/external MCP connectors should run in the separate `rsync-ai-mcp` project.
debezium-mcp:
build:
context: ./shared/mcp-connectors/internal/debezium/versions/v1.0.0
dockerfile: Dockerfile
container_name: rsync-ai-debezium-v1-0-0-mcp
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
labels:
- rsync-ai.mcp=true
- rsync-ai.connector=debezium
environment:
# Kafka client credentials. This connector configures Debezium's *schema
# history* producer AND consumer, which are separate Kafka clients from
# the Connect worker's own — it reads KAFKA_SECURITY_PROTOCOL /
# KAFKA_SASL_* / KAFKA_SSL_* straight out of this env
# (versions/v1.0.0/test_schema_history_security.py proves each name).
# The Dockerfile's KAFKA_BOOTSTRAP_SERVERS default supplies only an
# ADDRESS, so without the anchor a BYO broker is dialled anonymously over
# PLAINTEXT and the failure reads as an ordinary broker error rather than
# a missing-config one. Address and credentials must travel together: a
# correct key aimed at the bundled broker is the same outage with a more
# confusing message, which is why KAFKA_BROKERS is declared right here
# and not left to the image default.
<<: *kafka-security
KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:29092}
PORT: "8000"
MCP_PORT: "8000"
MCP_HTTP_MODE: "true"
DOCKER_CONTAINER: "true"
LOG_FORMAT: json
LOG_LEVEL: info
ENVIRONMENT: development
# Externalize per-connector secrets to this shared-with-kafka-connect volume so
# the source password stays OUT of the Kafka config topic. Unset ⇒ inline config
# (backward compatible). See connector.py externalize_secrets().
DEBEZIUM_SECRETS_DIR: /connect-secrets
restart: unless-stopped
volumes:
# Read-write: debezium-mcp WRITES the secret .properties the worker resolves.
- connect_secrets:/connect-secrets
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
interval: 30s
timeout: 10s
start_period: 10s
retries: 3
# MinIO S3-compatible object storage backend.
# Required for claim-check staging when batch payloads exceed the inline Kafka size cap.
# Without this, minio-mcp and kafka-mcp-sink fall back to chunked inline Kafka writes.
minio:
# Pinned: an unpinned object store silently changes under a re-pull, and this one
# holds staged pipeline data. Pinned by digest because cgr.dev/chainguard serves
# only `latest` (and `latest-dev`) anonymously, so a digest is the only pin there is.
# Chainguard's build, not MinIO's: MinIO withdrew anonymous pulls from
# docker.io/minio/* (2026-09-12) and then from quay.io/minio/* (401). One multi-arch
# image carries both `minio` (its entrypoint) and `mc`, so the healthcheck below
# and minio-lifecycle-init use it too. It has no curl: health checks go through mc.
image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
# Root, like the image this replaces. A volume written by v0.1.5 is root-owned
# 0755, and this image's default user (65532) cannot write it: the server exits
# "Unable to write to the backend" and restarts forever, and everything that
# waits on its health never starts.
user: "0:0"
container_name: rsync-ai-minio
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
environment:
MINIO_ROOT_USER: ${MINIO_ACCESS_KEY_ID:-minioadmin}
MINIO_ROOT_PASSWORD: ${MINIO_SECRET_ACCESS_KEY:-minioadmin}
command: server /data --console-address ":9001"
volumes:
- minio_data:/data
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 10s
timeout: 5s
retries: 5
# One-shot: ensure the claim-check bucket exists and apply an expiration
# lifecycle rule on the staging prefix. This is the BACKSTOP for MinIO disk
# growth — the sink deletes each claim-check object after a successful write,
# but if a delete is missed (worker crash, etc.) this rule reclaims the object
# after MINIO_STAGING_EXPIRE_DAYS.
#
# TTL MUST be >= the Kafka batch-topic retention (7 days, see
# backend-orchestrator/internal/kafka/topology.go retention.ms). For a >256KB
# batch only a pointer goes to Kafka while the rows live as a MinIO object; if
# the staged object expired BEFORE its Kafka pointer, a lagging sink would read
# NoSuchKey and lose those rows. With uncapped chunked transfers a sink can lag
# for hours/days on a multi-million-row backfill, so the old 2-day default was
# SHORTER than retention and opened a loss window — default raised to 7 days.
minio-lifecycle-init:
# The server image above, same digest, for the same reasons; mc writes the
# lifecycle rule. Chainguard's mc-only image is distroless (no /bin/sh for the
# entrypoint below), so the server image, which ships both, serves here too.
# No `user:` needed: nothing here writes the data volume.
image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
container_name: rsync-ai-minio-lifecycle-init
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
depends_on:
minio:
condition: service_healthy
environment:
- MINIO_BUCKET=${MINIO_BUCKET:-pipeline-data}
- MINIO_PREFIX=${MINIO_PREFIX:-staging}
- MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin}
- MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin}
- MINIO_STAGING_EXPIRE_DAYS=${MINIO_STAGING_EXPIRE_DAYS:-7}
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -e
until mc alias set local http://minio:9000 "$$MINIO_ACCESS_KEY_ID" "$$MINIO_SECRET_ACCESS_KEY"; do
echo "waiting for minio..."; sleep 2;
done
mc mb --ignore-existing "local/$$MINIO_BUCKET"
# Apply expiration on the staging prefix (newer mc: `ilm rule add`, older: `ilm add`)
mc ilm rule add "local/$$MINIO_BUCKET" --expire-days "$$MINIO_STAGING_EXPIRE_DAYS" --prefix "$$MINIO_PREFIX/" \
|| mc ilm add --expiry-days "$$MINIO_STAGING_EXPIRE_DAYS" --prefix "$$MINIO_PREFIX/" "local/$$MINIO_BUCKET" \
|| echo "WARN: could not set ILM rule (mc version?); sink-side delete is primary cleanup"
echo "✅ minio-init: bucket + lifecycle ready"
restart: "no"
# The blob lane's staging store: a second MinIO on the connector network ONLY.
# Blob passthrough has the SOURCE connector stage each object here and the
# DESTINATION connector fetch it, and connectors join only rsync-ai-mcp
# (SEC-M-06). `minio` above is on `default` only, so a connector cannot resolve
# it, and bridging it would hand every connector the store that holds every
# pipeline's staged claim-check rows. This one holds only staged blobs, under
# its own login. Keep ":9000" in its URL: the connectors' staging client uses
# path-style addressing only for an endpoint with "minio", "localhost" or
# ":9000" in it (base_connector.py _get_staging_client).
# KI-BLOB-LANE-STAGING-UNREACHABLE-FROM-CONNECTORS.
blob-staging:
image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
container_name: ${STACK_PREFIX:-rsync-ai}-blob-staging
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
environment:
MINIO_ROOT_USER: ${BLOB_STAGING_ACCESS_KEY:-${MINIO_ACCESS_KEY_ID:-minioadmin}}
MINIO_ROOT_PASSWORD: ${BLOB_STAGING_SECRET_KEY:-${MINIO_SECRET_ACCESS_KEY:-minioadmin}}
command: server /data
volumes:
- blob_staging_data:/data
networks:
- rsync-ai-mcp
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 10s
timeout: 5s
retries: 5
# Bucket + expiry for blob-staging. Nothing deletes a staged blob per message:
# blobs are content-addressed and shared between messages, so this rule is the
# only cleanup. Same TTL floor as minio-lifecycle-init, for the same reason.
blob-staging-init:
image: cgr.dev/chainguard/minio@sha256:bd014394a80898e68c149f2311fdf8d5a2c2f3bb2c33b9327ae6d02b4b065ae1
container_name: ${STACK_PREFIX:-rsync-ai}-blob-staging-init
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
depends_on:
blob-staging:
condition: service_healthy
environment:
- MINIO_BUCKET=${MINIO_BUCKET:-pipeline-data}
- MINIO_PREFIX=${MINIO_PREFIX:-staging}
- BLOB_STAGING_ACCESS_KEY=${BLOB_STAGING_ACCESS_KEY:-${MINIO_ACCESS_KEY_ID:-minioadmin}}
- BLOB_STAGING_SECRET_KEY=${BLOB_STAGING_SECRET_KEY:-${MINIO_SECRET_ACCESS_KEY:-minioadmin}}
- MINIO_STAGING_EXPIRE_DAYS=${MINIO_STAGING_EXPIRE_DAYS:-7}
networks:
- rsync-ai-mcp
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -e
until mc alias set local http://blob-staging:9000 "$$BLOB_STAGING_ACCESS_KEY" "$$BLOB_STAGING_SECRET_KEY"; do
echo "waiting for blob-staging..."; sleep 2;
done
mc mb --ignore-existing "local/$$MINIO_BUCKET"
mc ilm rule add "local/$$MINIO_BUCKET" --expire-days "$$MINIO_STAGING_EXPIRE_DAYS" --prefix "$$MINIO_PREFIX/" \
|| mc ilm add --expiry-days "$$MINIO_STAGING_EXPIRE_DAYS" --prefix "$$MINIO_PREFIX/" "local/$$MINIO_BUCKET" \
|| echo "WARN: could not set the blob staging expiry rule"
echo "blob-staging-init: bucket + lifecycle ready"
restart: "no"
# Internal: MinIO staging MCP (claim-check upload/read/delete)
minio-mcp:
build:
context: ./shared/mcp-connectors/internal/minio/versions/v1.0.0
dockerfile: Dockerfile
container_name: rsync-ai-minio-v1-0-0-mcp
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
labels:
- rsync-ai.mcp=true
- rsync-ai.connector=minio
- rsync-ai.connector_version=v1.0.0
environment:
- PORT=8000
- MCP_PORT=8000
- MCP_HTTP_MODE=true
- DOCKER_CONTAINER=true
- LOG_FORMAT=json
- LOG_LEVEL=info
- ENVIRONMENT=development
# Optional defaults (can be overridden per request via params.config)
# Pin to the unambiguous container_name, NOT the bare `minio` alias: the
# e2e overlay (docker-compose.e2e.dbs.yml) ships a second container that
# also aliases `minio` on the shared network, so `http://minio:9000`
# DNS-round-robins between two MinIO stores. Writes landing on one and
# reads hitting the other → NoSuchKey → DLQ → silent claim-check drop.
# `rsync-ai-minio` resolves to exactly one container. Override via env for
# external object stores.
- MINIO_ENDPOINT_URL=${MINIO_ENDPOINT_URL:-http://rsync-ai-minio:9000}
- MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin}
- MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin}
- MINIO_REGION=${MINIO_REGION:-us-east-1}
- MINIO_BUCKET=${MINIO_BUCKET:-pipeline-data}
- MINIO_PREFIX=${MINIO_PREFIX:-staging}
# Same class of silent disagreement as RSYNC_LLM_DIAGNOSER_ENABLED above:
# connector.py:36 reads RSYNC_CLAIM_CHECK_GZIP, and no compose file passed
# it through, so an operator who set it in .env got plain JSON and no error
# said why. Default stays empty (= off), so wiring it changes nothing until
# someone sets it deliberately. The reader is flag-INDEPENDENT magic-byte
# autodetect, so turning it on mid-flight still reads back legacy plain
# objects -- it is safe to flip without draining staging.
- RSYNC_CLAIM_CHECK_GZIP=${RSYNC_CLAIM_CHECK_GZIP:-}
depends_on:
minio:
condition: service_healthy
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
interval: 30s
timeout: 10s
start_period: 10s
retries: 3
kafka-mcp-sink-mcp:
build:
# Use repo root as build context so the worker can import shared/go/transforms via replace.
context: .
dockerfile: shared/mcp-connectors/internal/kafka-mcp-sink/Dockerfile
container_name: rsync-ai-kafka-mcp-sink-v1-0-0-mcp
labels:
- rsync-ai.mcp=true
- rsync-ai.connector=kafka-mcp-sink
environment:
# Kafka client credentials. The sink worker is a consumer group member on
# every pipeline topic — initKafkaSecurity() in
# worker-src/cmd/kafka-sink-worker/kafka_security.go reads these names,
# and kafka_security_test.go pins them. Without the anchor a BYO broker is
# dialled anonymously over PLAINTEXT. KAFKA_BROKERS is declared alongside
# so credentials can never be aimed at a cluster other than the one they
# belong to.
<<: *kafka-security
KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:29092}
PORT: "8000"
# How many destination flushes this sink runs at once. Unset/empty = 4.
#
# A CDC data topic is born with min(3, brokers) partitions and Debezium gives
# every source table a topic of its own, so one worker is assigned tens of
# partitions. It consumed all of them already, but drained them one at a time
# because the destination write ran inline on the consume goroutine — one slow
# table stalled every other partition. Lanes are sharded on (topic, partition),
# so per-row order is untouched.
#
# Set to 0 or 1 to restore the old inline behaviour exactly; this is the kill
# switch. Raising it opens more concurrent connections to the DESTINATION, which
# is somebody else's database — 4 is deliberately modest.
RSYNC_SINK_FLUSH_LANES: ${RSYNC_SINK_FLUSH_LANES:-}
# Mirror of the orchestrator's schema-drift switch. The sink worker reports the
# columns CDC adds on its own to rsync.healer.schema-changes only when this is
# "true" (newSchemaDriftWriter); off, it writes nothing there, because the
# orchestrator then neither creates nor consumes that topic. Must stay
# byte-identical to the orchestrator line; a census test pins it.
RSYNC_SCHEMA_DRIFT_ENABLED: "${RSYNC_SCHEMA_DRIFT_ENABLED:-false}"
MCP_PORT: "8000"
MCP_HTTP_MODE: "true"
DOCKER_CONTAINER: "true"
LOG_FORMAT: json
LOG_LEVEL: info
ENVIRONMENT: development
# Allow kafka-mcp-sink to fetch claim-check objects from MinIO (S3-compatible)
# Pin to the unambiguous container_name, NOT the bare `minio` alias: the
# e2e overlay (docker-compose.e2e.dbs.yml) ships a second container that
# also aliases `minio` on the shared network, so `http://minio:9000`
# DNS-round-robins between two MinIO stores. Writes landing on one and
# reads hitting the other → NoSuchKey → DLQ → silent claim-check drop.
# `rsync-ai-minio` resolves to exactly one container. Override via env for
# external object stores.
MINIO_ENDPOINT_URL: ${MINIO_ENDPOINT_URL:-http://rsync-ai-minio:9000}
MINIO_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY_ID:-minioadmin}
MINIO_SECRET_ACCESS_KEY: ${MINIO_SECRET_ACCESS_KEY:-minioadmin}
MINIO_REGION: ${MINIO_REGION:-us-east-1}
MINIO_BUCKET: ${MINIO_BUCKET:-pipeline-data}
MINIO_PREFIX: ${MINIO_PREFIX:-staging}
# Sink writes the durable ACK ledger (pipeline_batch_acks). Without
# this URL pgDB was nil and the ledger stayed empty — the postflight
# silent-drop invariant then had to fall back to sink-emitted stats
# events, which is a weaker signal. With it, Postgres is the single
# source of truth for "has this Kafka offset been written to dest?".
# URL is built from the same env vars the `postgres` service uses, so
# changing creds in one place updates every consumer.
POSTGRES_USER: ${POSTGRES_USER:-user}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
POSTGRES_DB: ${POSTGRES_DB:-pipeline_db}
POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
POSTGRES_PORT: "${POSTGRES_PORT:-5432}"
POSTGRES_URL: "postgres://${POSTGRES_USER:-user}:${POSTGRES_PASSWORD:-password}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-pipeline_db}?sslmode=${DB_SSLMODE:-disable}"
# Redis for the ACK ledger / offset checkpointing. Must carry the password
# when Redis runs with --requirepass (staging and prod) — otherwise the
# sink worker crashes on every start with "NOAUTH Authentication required".
REDIS_URL: "redis://:${REDIS_PASSWORD:-}@redis:6379/0"
# Sink worker calls destination MCPs (e.g. rsync-ai-postgresql-v1-0-0-mcp)
# by container name. JIT-deployed MCPs live on rsync-ai-mcp; without
# bridging into that network the sink resolves nothing and every batch
# write fails with "no such host". This is the silent failure mode that
# let pipelines report `completed` while leaving the destination empty.
networks:
- default
- rsync-ai-mcp
restart: unless-stopped
# Ship the sink worker's (now structured-JSON) stdout/stderr to fluent-bit so
# the destination-write leg is visible + trace-correlated downstream. Without
# this driver the sink is the one data-plane stage absent from the flow view. F-Obs-2.
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
interval: 30s
timeout: 10s
start_period: 10s
retries: 3
# Context7 MCP Service (Documentation Lookup)
context7-mcp:
build:
context: ./shared/mcp-connectors/internal/context7
dockerfile: Dockerfile
container_name: rsync-ai-context7-v1-0-0-mcp
environment:
- PORT=8080
# Use context7.com base URL - api. subdomain doesn't resolve
- CONTEXT7_API_URL=https://context7.com/api
# Provide via your shell environment or env file (do not commit keys).
- CONTEXT7_API_KEY=${CONTEXT7_API_KEY:-}
- CONTEXT7_DEFAULT_TOKENS=10000
- CONTEXT7_TIMEOUT=30
- LOG_FORMAT=json
- LOG_LEVEL=info
- ENVIRONMENT=development
# OpenTelemetry Configuration
- OTEL_EXPORTER_OTLP_ENDPOINT=otel-collector:4317
- OTEL_SERVICE_NAME=context7-mcp
- OTEL_ENABLED=true
ports:
- "127.0.0.1:${RSYNC_HP_CONTEXT7:-8087}:8080"
# Add DNS servers to resolve external hostnames
dns:
- 8.8.8.8
- 8.8.4.4
- 1.1.1.1
healthcheck:
test: ["CMD", "python", "-c", "import httpx; httpx.get('http://localhost:8080/health', timeout=2)"]
interval: 10s
timeout: 5s
retries: 3
extra_hosts:
- "host.docker.internal:host-gateway"
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
restart: unless-stopped
# Redis (State Store for Orchestrator & Caching)
redis:
image: redis:7-alpine
container_name: redis
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
ports:
- "127.0.0.1:${RSYNC_HP_REDIS:-6379}:6379"
command:
- sh
- -c
# The password is written to a 0600 file and never appears on the
# redis-server ARGV, so it stays out of `ps` on the host and out of
# the rendered compose spec (`docker inspect` / `compose config`).
- |
umask 077
: > /tmp/redis.conf
if [ -n "$$REDIS_PASSWORD" ]; then
printf 'requirepass %s\n' "$$REDIS_PASSWORD" > /tmp/redis.conf
fi
exec redis-server /tmp/redis.conf --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru
environment:
- REDIS_PASSWORD=${REDIS_PASSWORD:-}
volumes:
- redis_data:/data
healthcheck:
# REDISCLI_AUTH is read from the environment by redis-cli, so the password
# never reaches ARGV (a healthcheck re-runs every 5s, once per interval).
test: ["CMD-SHELL", "if [ -n \"$$REDIS_PASSWORD\" ]; then REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli --no-auth-warning ping; else redis-cli ping; fi"]
interval: 5s
timeout: 3s
retries: 5
# Backend Orchestrator (Go)
orchestrator:
build:
context: .
dockerfile: backend-orchestrator/Dockerfile
args: *build-info
container_name: rsync-ai-orchestrator
restart: unless-stopped
# How long `docker compose stop/down/restart` waits after SIGTERM before it
# SIGKILLs, which skips every deferred Stop/Close in main.go. Left unset,
# the daemon decides: 10 s on Docker Engine, 1 s on the Docker Desktop
# (engine 28.5.1) this was measured on. Shutdown drains HTTP for up to
# 10 s, then gives the Kafka consumer groups up to 15 s to leave
# (consumerCloseTimeout in backend-orchestrator/internal/kafka/manager.go),
# on top of the agents' own Stop calls; an idle dev stack takes 3-9 s.
# backend-orchestrator/internal/kafka/stop_grace_test.go pins this value
# and its copies in docker-compose.quickstart.yml and the Helm chart.
stop_grace_period: 45s
deploy:
# SCALING:
# One replica. Static membership is OPT-IN via
# KAFKA_STATIC_MEMBERSHIP=true and derives instance_id from $HOSTNAME
# (unique per pod); it only reduces rebalance churn on restarts.
replicas: 1
ports:
- "127.0.0.1:${RSYNC_HP_ORCHESTRATOR:-8081}:8080" # Internal port for debugging
environment:
# Kafka client security for a BYO/managed broker. See x-kafka-security at
# the top of this file; every key defaults to empty and empty means unset,
# so the bundled broker is unaffected.
<<: *kafka-security
DB_HOST: "postgres"
DB_PORT: "5432"
DB_USER: "user"
DB_PASSWORD: "password"
DB_NAME: "pipeline_db"
KAFKA_BROKERS: "${KAFKA_BROKERS:-kafka:29092}"
# Durability for every topic the orchestrator creates. Read by
# internal/kafka/replication.go (EnvReplicationFactor / EnvMinInsyncReplicas);
# empty means "operator stated nothing", which is not the same as zero -- the
# built-in derivation (1 on a single broker, min(3, brokers) beyond) applies.
#
# These two travel together on purpose. min.insync.replicas > replication
# factor produces a topic that is CREATED, listed and subscribable, and then
# rejects every acks=all produce with NOT_ENOUGH_REPLICAS: a pipeline that
# reports running and streams zero rows. The Go side clamps misr down to the
# topic's final RF for exactly that reason, so setting only one of these
# cannot wedge a cluster.
#
# Version boundary, measured rather than assumed: on the bundled
# apache/kafka:3.7.0 broker an RF=1 topic carrying min.insync.replicas=2
# still ACCEPTS acks=all -- 3.7 caps the effective value at the replica count
# (kafka.cluster.Partition#effectiveMinIsr, present in that broker's
# kafka_2.13-3.7.0.jar). The trap is therefore real for the OLDER brokers a
# BYO/self-host deployment actually points at (MSK and most managed clusters
# still run 2.8-3.6), which is exactly the case these variables exist for.
# Clamp regardless: on a 3.7+ broker the clamp is a no-op.
KAFKA_REPLICATION_FACTOR: "${KAFKA_REPLICATION_FACTOR:-}"
KAFKA_MIN_INSYNC_REPLICAS: "${KAFKA_MIN_INSYNC_REPLICAS:-}"
# Partition count for every CDC data topic (backend-orchestrator/internal/kafka
# partitions.go EnvCDCTopicPartitions). Empty means "operator stated nothing", so
# the derivation applies: 1 on a single broker, min(3, brokers) beyond that. Set
# it only to go WIDER than 3 on a large cluster.
#
# It is a birth setting, not a live one. Kafka cannot reduce a partition count at
# all, and RAISING one on a topic that is already streaming rehashes every key
# onto a different partition and breaks per-key ordering, so rsync.ai never resizes
# an existing topic -- a change here reaches only topics created after it.
KAFKA_CDC_TOPIC_PARTITIONS: "${KAFKA_CDC_TOPIC_PARTITIONS:-}"
# Namespace for every Kafka topic this product creates, so an operator can
# tell them apart from other applications' topics on a shared (BYO-Kafka)
# cluster. Every service that touches Kafka must agree on this value: a
# producer and a consumer resolving different names do not error, the
# consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
# an explicit empty value means "disable" (keep pre-prefix topic names) and
# `:-` would silently turn that back into the default.
KAFKA_TOPIC_PREFIX: "${KAFKA_TOPIC_PREFIX-rsync.}"
# Blob-passthrough staging (universal-blob-passthrough plan §3): the
# executor's blob lane hands these to the source connector (to stage object
# bytes) and, via the sink message, to the destination connector (to fetch
# them). blob-staging, NOT minio: connectors join only rsync-ai-mcp and
# cannot resolve minio (KI-BLOB-LANE-STAGING-UNREACHABLE-FROM-CONNECTORS).
# The lane reads no other MinIO credential, so the orchestrator holds none.
# MINIO_BUCKET/PREFIX stay: the staged blob's bucket must be in the sink's
# allow-list and its prefix under blob-staging-init's expiry rule.
BLOB_STAGING_ENDPOINT_URL: "${BLOB_STAGING_ENDPOINT_URL:-http://blob-staging:9000}"
BLOB_STAGING_ACCESS_KEY_ID: "${BLOB_STAGING_ACCESS_KEY:-${MINIO_ACCESS_KEY_ID:-minioadmin}}"
BLOB_STAGING_SECRET_ACCESS_KEY: "${BLOB_STAGING_SECRET_KEY:-${MINIO_SECRET_ACCESS_KEY:-minioadmin}}"
MINIO_REGION: "${MINIO_REGION:-us-east-1}"
MINIO_BUCKET: "${MINIO_BUCKET:-pipeline-data}"
MINIO_PREFIX: "${MINIO_PREFIX:-staging}"
JWT_SECRET: "dev_secret_key_change_in_prod"
# S2S secret the gateway's internal routes check (namespace lock, heal,
# OAuth refresh) and requirePrincipal accepts. Same value as api-gateway, or
# every one of those calls is refused 401/503 and only logged as a warning.
INTERNAL_SERVICE_SECRET: "${INTERNAL_SERVICE_SECRET:-}"
# Guarded operator endpoint for recovering FAILED Debezium connectors.
# Default off for safety. Enable explicitly with CDC_RECOVERY_ENABLED=true.
CDC_RECOVERY_ENABLED: "${CDC_RECOVERY_ENABLED:-false}"
# Autonomous CDC sink self-heal (Bug B, #678): bounded per-pipeline sink-worker
# auto-restart on a GENUINE 3-signal wedge (corrected lag + stale + flat). Default
# OFF everywhere (operational-safety gate mirroring CDC_RECOVERY_ENABLED, NOT an
# edition gate — preserves OSS/cloud parity). Arm by setting the env var to true.
CDC_SINK_AUTORESTART_ENABLED: "${CDC_SINK_AUTORESTART_ENABLED:-false}"
# Orphan sink-worker reaper (KI-CDC-SINK-WORKER-NO-REAPER): off | dry_run | enforce.
# Default OFF, same op-safety gate; stops only workers whose pipeline is deleted or stopped.
SINK_WORKER_REAPER_MODE: "${SINK_WORKER_REAPER_MODE:-off}"
# Self-healing schema-drift detector (P0/P1): the drift-detect -> approve loop.
# Default OFF. Flip to true to activate the executor's baseline+diff+emit
# (schema_drift.go) AND the healer's schema-change consumer (StartSchemaOnly in
# workers/executor.go). When false, both paths are inert no-ops -> zero behavior change.
RSYNC_SCHEMA_DRIFT_ENABLED: "${RSYNC_SCHEMA_DRIFT_ENABLED:-false}"
# Additive auto-apply, a SEPARATE opt-in from the line above (healer.go
# autoApplyEnabled()). It was read by the orchestrator and documented in
# .env.prod.example + docs/design/self-healing-drift-approve.md, but no compose
# file passed it through — so setting it in an env file did nothing and the
# operator had no way to tell. That silent disagreement is the bug; the default
# stays `false`, so wiring it changes nothing until someone sets it deliberately.
# Destructive classes ignore it entirely (DROP/TRUNCATE always need approval).
RSYNC_SCHEMA_DRIFT_AUTOAPPLY: "${RSYNC_SCHEMA_DRIFT_AUTOAPPLY:-false}"
# LLM error diagnoser, chained BEHIND the rules (pkg/diagnose/llm.go
# NewFromEnv). The RuleBasedDiagnoser always runs first; the LLM is asked
# only about a CategoryUnknown result, and its confidence is clamped below
# heal's auto-execute band, so it can suggest but never auto-act.
#
# Exactly the same silent disagreement as the line above: the code read
# RSYNC_LLM_DIAGNOSER_ENABLED and NO compose file passed it through, so an
# operator who set it in .env got nothing and no error said why. The
# default stays `false`, so wiring it changes nothing until someone sets
# it deliberately — but turning it ON sends llmscrub-scrubbed failure text
# over the network to the LLM service, which is a privacy-surface change,
# not a config tidy-up.
RSYNC_LLM_DIAGNOSER_ENABLED: "${RSYNC_LLM_DIAGNOSER_ENABLED:-false}"
# Optional overrides, both empty by default (empty = fall back in code).
# Model: RSYNC_LLM_DIAGNOSER_MODEL -> LLM_MODEL -> gpt-4o-mini.
# URL is diagnoser-specific ON PURPOSE: LLM_SERVICE_URL is also read by
# internal/workers/intent.go with a DIFFERENT fallback (planner:5011), so
# passing that name through here would silently repoint the intent worker.
RSYNC_LLM_DIAGNOSER_MODEL: "${RSYNC_LLM_DIAGNOSER_MODEL:-}"
RSYNC_LLM_DIAGNOSER_URL: "${RSYNC_LLM_DIAGNOSER_URL:-}"
# SECURITY: ENCRYPTION_KEY must be 32+ characters. Generate with: openssl rand -base64 32
# WARNING: Changing this key will make existing encrypted connection configs unreadable!
# For production: Override via .env file or environment variable
ENCRYPTION_KEY: "${ENCRYPTION_KEY:-dev-encryption-key-32-bytes-long!!}"
TOOLS_DIR: "/app/shared/mcp-connectors"
# Self-healing MCP deployment (tool-generator has Docker access via docker.sock)
TOOL_GENERATOR_URL: "http://tool-generator:5010"
# Sentinel core-service probes (sentinel/service_probes.go). Each is probed only
# when its address is set; after 3 failed 30s checks the service is recorded
# unhealthy and every admin gets an INFRASTRUCTURE_DOWN alert. Redis is always
# probed, at the address the correlation client uses. TEMPORAL_ADDRESS is also the
# (parked) native Temporal worker's dial address, whose default is this same value.
TEMPORAL_ADDRESS: "temporal:7233"
SENTINEL_PROBE_TEMPORAL_ADAPTER_URL: "http://temporal-adapter:8082"
SENTINEL_PROBE_CONNECTOR_DEPLOYER_URL: "http://connector-deployer:5011"
SENTINEL_PROBE_LLM_SERVICE_URL: "http://llm-service:5000"
# Redis Configuration (State Store)
REDIS_HOST: "redis"
REDIS_PORT: "6379"
REDIS_DB: "0"
REDIS_PASSWORD: "${REDIS_PASSWORD:-}"
# Correlation Store (V2 Request/Reply Pattern)
# Must be >= longest expected request processing time to avoid duplicate claims.
CORRELATION_CLAIM_TTL: "1h"
# Serialization (Demo-2 default: JSON; Schema Registry is optional behind a profile)
KAFKA_USE_AVRO: "false"
SCHEMA_REGISTRY_URL: "http://schema-registry:8080"
AVRO_AUTO_REGISTER_SCHEMAS: "false"
# Logging Configuration (JSON for log collection)
LOG_FORMAT: "json"
LOG_LEVEL: "info"
ENVIRONMENT: "development"
# Scheduler is deprecated (schedules are managed via API Gateway + Temporal).
# Disable to avoid legacy DB queries against removed columns.
ENABLE_SCHEDULER: "false"
# OpenTelemetry Configuration (exports to the bundled OTel Collector)
OTEL_EXPORTER_OTLP_ENDPOINT: "otel-collector:4317"
OTEL_SERVICE_NAME: "orchestrator"
OTEL_ENABLED: "${OTEL_ENABLED:-false}"
OTEL_TRACES_EXPORTER: "otlp"
OTEL_METRICS_EXPORTER: "otlp"
# Enable Debezium topic consumer that emits "captured" CDC table stats
# (incoming inserts/updates/deletes), so UI can show incoming vs applied like DMS.
ENABLE_CDC_TABLE_STATS: "true"
# User MCP connectors run on the rsync-ai-mcp network (separate from internal services)
DOCKER_NETWORK: "${STACK_PREFIX:-rsync-ai}-mcp"
# Drives isolated-stack MCP container-name discovery in the orchestrator
# (infra_preflight health URLs + mcp.containerNameCandidates). Default
# rsync-ai; an isolated CI stack sets rsync-ci.
STACK_PREFIX: "${STACK_PREFIX:-rsync-ai}"
# Phase 3: Temporal-native executor dispatch — ⛔ PARKED, keep these OFF.
# The native path is dead scaffolding: nothing populates the workflow's
# ExecutorDispatch field, so all pipelines use the correlation path and the
# native worker (if started) only idles. EXECUTOR_DISPATCH is not even read in
# code. Leave both at their defaults below.
EXECUTOR_NATIVE_WORKER: "${EXECUTOR_NATIVE_WORKER:-false}"
EXECUTOR_DISPATCH: "${EXECUTOR_DISPATCH:-correlation}"
# Export page size (rows/batch). Empty = code default (10000). Raise to cut
# staging round-trips for large tables; opt-in per CAPABILITIES (batch-size lever).
EXECUTOR_TABLE_BATCH_ROWS: "${EXECUTOR_TABLE_BATCH_ROWS:-}"
# Batches exported per dispatch before signaling needs_continuation (chunk
# boundary = this × EXECUTOR_TABLE_BATCH_ROWS rows). Empty = code default (200).
# Lower it to force the chunked-continuation resume path on small tables.
EXECUTOR_TABLE_CHUNK_BATCHES: "${EXECUTOR_TABLE_CHUNK_BATCHES:-}"
# Hard per-table runaway ceiling (rows). Empty/0 = unlimited; the chunk loop
# runs to natural EOF. Set >0 only to bound a suspected pagination bug.
EXECUTOR_TABLE_MAX_ROWS: "${EXECUTOR_TABLE_MAX_ROWS:-}"
# Which MCP connector containers exist on THIS host, for sentinel's connector
# health sweep (see the docker-socket-proxy service above). The sweep used to
# infer "not deployed here" from a name failing to resolve, which is only sound
# on a resolver that answers an unknown name with a clean NXDOMAIN. On a GCP VM
# it is not: /etc/resolv.conf carries `search ....internal` with ndots:0, Docker's
# resolver forwards those variants upstream, and one non-NXDOMAIN answer left
# every undeployed connector looking like a fault — 18 of them on prod. The
# Docker API needs no resolver semantics, and `all=true` additionally tells a
# stopped container (a real fault) from one that was never here, which DNS
# cannot. Unset (quickstart/self-host runs no proxy) → the resolver fallback,
# which now records nothing rather than guessing.
MCP_DOCKER_API_URL: "http://docker-socket-proxy:2375"
depends_on:
postgres:
condition: service_healthy
# Other half of docker-compose.byo-postgres.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
kafka:
condition: service_healthy
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
redis:
condition: service_healthy
# Read-only Docker socket proxy for the connector-deployment census above.
docker-socket-proxy:
condition: service_started
networks:
- default
- rsync-ai-mcp
volumes:
- ./shared/mcp-connectors:/app/shared/mcp-connectors
extra_hosts:
- "host.docker.internal:host-gateway"
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
# API Gateway (Go)
api-gateway:
build:
context: .
dockerfile: api-gateway/Dockerfile
args: *build-info
container_name: rsync-ai-api-gateway
# main() exits when the database is unreachable at boot (db.Init has already
# retried for DB_CONNECT_TIMEOUT). The restart is what turns that exit into a
# recovery once Postgres is up; without it the container just stays down.
restart: unless-stopped
ports:
- "${RSYNC_BIND_ADDR:-127.0.0.1}:${RSYNC_HP_API_GATEWAY:-5001}:8080" # Public API port (5001 to avoid macOS AirPlay on 5000)
environment:
# Kafka client security for a BYO/managed broker. See x-kafka-security at
# the top of this file; every key defaults to empty and empty means unset,
# so the bundled broker is unaffected.
<<: *kafka-security
DATABASE_URL: "postgres://user:password@postgres:5432/pipeline_db?sslmode=disable"
KAFKA_BROKERS: "${KAFKA_BROKERS:-kafka:29092}"
# Namespace for every Kafka topic this product creates, so an operator can
# tell them apart from other applications' topics on a shared (BYO-Kafka)
# cluster. Every service that touches Kafka must agree on this value: a
# producer and a consumer resolving different names do not error, the
# consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
# an explicit empty value means "disable" (keep pre-prefix topic names) and
# `:-` would silently turn that back into the default.
KAFKA_TOPIC_PREFIX: "${KAFKA_TOPIC_PREFIX-rsync.}"
# Dev/E2E: allow internal-only connectors (e.g. MinIO) to be used from UI/tests.
RSYNC_ALLOW_INTERNAL_CONNECTORS: "true"
# Live MCP container status via the read-only docker-socket-proxy (see the
# docker-socket-proxy service above). Without this, api-gateway can't read
# /var/run/docker.sock as non-root, so the catalog reports every connector
# as "auto_deploys_on_use" and the UI shows "Running (0)". Unset → falls
# back to dialing the local Unix socket directly.
MCP_DOCKER_API_URL: "http://docker-socket-proxy:2375"
# Admin panel access (beta). Comma-separated allowlist of emails.
# Example: RSYNC_ADMIN_EMAILS=alice@company.com,bob@company.com
RSYNC_ADMIN_EMAILS: "${RSYNC_ADMIN_EMAILS:-}"
# CDC auto-pickup: a pipeline selected as "this whole database/schema"
# keeps that meaning — the api-gateway re-expands the selection rule on
# this interval and adds source tables created since. Default ON; set
# false to freeze every CDC pipeline at its current table list. The
# interval is floored at 60s (discovery runs once per CDC pipeline
# per tick).
CDC_TABLE_AUTOPICKUP_ENABLED: "${CDC_TABLE_AUTOPICKUP_ENABLED:-true}"
CDC_TABLE_WATCH_INTERVAL_SECONDS: "${CDC_TABLE_WATCH_INTERVAL_SECONDS:-300}"
# Temporal is required for the agentic pipeline stage-by-stage execution + progress events.
# Without this, the gateway falls back to legacy direct-Kafka which doesn't drive the V2 UI timeline.
TEMPORAL_ADDRESS: "temporal:7233"
ORCHESTRATOR_URL: "http://orchestrator:8080"
AGENT_ORCHESTRATOR_URL: "http://orchestrator:8080"
TOOL_GENERATOR_URL: "http://tool-generator:5010"
JWT_SECRET: "dev_secret_key_change_in_prod"
# V2 is the ONLY workflow (V1 deleted)
REDIS_ADDRESS: "redis:6379"
REDIS_PASSWORD: "${REDIS_PASSWORD:-}"
# SECURITY: Must match api-gateway ENCRYPTION_KEY for shared credential access
ENCRYPTION_KEY: "${ENCRYPTION_KEY:-dev-encryption-key-32-bytes-long!!}"
# CSRF protection (double-submit). Default off in dev; enable in staging with RSYNC_CSRF_ENFORCE=true.
RSYNC_CSRF_ENFORCE: "${RSYNC_CSRF_ENFORCE:-false}"
# Verifies X-Internal-Secret on internal S2S endpoints (e.g. the scheduled
# model-run endpoint the temporal-adapter calls). Unset → those endpoints 503.
INTERNAL_SERVICE_SECRET: "${INTERNAL_SERVICE_SECRET:-}"
# Read-only mirror of the orchestrator's schema-drift switch, so the pipeline's
# "Schema change alerts" card can say when detection is off for the whole
# installation (schema_evolution.go schemaDriftDetectorEnabled). Must stay
# byte-identical to the orchestrator line; a census test pins it.
RSYNC_SCHEMA_DRIFT_ENABLED: "${RSYNC_SCHEMA_DRIFT_ENABLED:-false}"
# Serialization (Demo-2 default: JSON; Schema Registry is optional behind a profile)
KAFKA_USE_AVRO: "false"
SCHEMA_REGISTRY_URL: "http://schema-registry:8080"
AVRO_AUTO_REGISTER_SCHEMAS: "false"
# Logging Configuration (JSON for log collection)
LOG_FORMAT: "json"
LOG_LEVEL: "info"
ENVIRONMENT: "development"
# OAuth Configuration
OAUTH_CALLBACK_URL: "http://localhost:5001/oauth/callback"
GITHUB_CLIENT_ID: "${GITHUB_CLIENT_ID}"
GITHUB_CLIENT_SECRET: "${GITHUB_CLIENT_SECRET}"
GOOGLE_CLIENT_ID: "${GOOGLE_CLIENT_ID}"
GOOGLE_CLIENT_SECRET: "${GOOGLE_CLIENT_SECRET}"
HUBSPOT_CLIENT_ID: "${HUBSPOT_CLIENT_ID}"
HUBSPOT_CLIENT_SECRET: "${HUBSPOT_CLIENT_SECRET}"
SALESFORCE_CLIENT_ID: "${SALESFORCE_CLIENT_ID}"
SALESFORCE_CLIENT_SECRET: "${SALESFORCE_CLIENT_SECRET}"
PIPEDRIVE_CLIENT_ID: "${PIPEDRIVE_CLIENT_ID}"
PIPEDRIVE_CLIENT_SECRET: "${PIPEDRIVE_CLIENT_SECRET}"
SLACK_CLIENT_ID: "${SLACK_CLIENT_ID}"
SLACK_CLIENT_SECRET: "${SLACK_CLIENT_SECRET}"
SHOPIFY_CLIENT_ID: "${SHOPIFY_CLIENT_ID:-}"
SHOPIFY_CLIENT_SECRET: "${SHOPIFY_CLIENT_SECRET:-}"
# Notifier (Pillar 6) — delivers pipeline failure alerts to Slack + email.
# Leave empty in dev to keep notifications in "persist-only" mode
# (queryable via UI, no external delivery).
NOTIFIER_SLACK_WEBHOOK_URL: "${NOTIFIER_SLACK_WEBHOOK_URL:-}"
# Slack one-click drift approvals (inbound interactivity). SIGNING_SECRET
# verifies Slack request signatures; BOT_TOKEN (users:read.email) maps the
# clicking Slack user to their rsync.ai account by email. Both unset ⇒ the
# inbound receiver is inert and drift alerts keep the plain link button.
SLACK_SIGNING_SECRET: "${SLACK_SIGNING_SECRET:-}"
SLACK_BOT_TOKEN: "${SLACK_BOT_TOKEN:-}"
SMTP_HOST: "${SMTP_HOST:-}"
SMTP_PORT: "${SMTP_PORT:-587}"
SMTP_USER: "${SMTP_USER:-}"
SMTP_PASSWORD: "${SMTP_PASSWORD:-}"
SMTP_FROM: "${SMTP_FROM:-}"
# Email verification (Resend). Unset → signups auto-verify (dev default).
# To TEST email verification locally: set RESEND_API_KEY + RESEND_FROM_ADDR
# in the root .env (must be a Resend-verified domain), then recreate api-gateway.
RESEND_API_KEY: "${RESEND_API_KEY:-}"
RESEND_FROM_ADDR: "${RESEND_FROM_ADDR:-}"
APP_BASE_URL: "${APP_BASE_URL:-http://localhost:3000}"
# OpenTelemetry Configuration (exports to the bundled OTel Collector)
OTEL_EXPORTER_OTLP_ENDPOINT: "otel-collector:4317"
OTEL_SERVICE_NAME: "api-gateway"
OTEL_ENABLED: "${OTEL_ENABLED:-false}"
OTEL_TRACES_EXPORTER: "otlp"
OTEL_METRICS_EXPORTER: "otlp"
# Monitoring UI feature flags (V1)
FEATURE_MONITORING_OVERVIEW: "true"
FEATURE_MONITORING_INFRA: "true"
FEATURE_MONITORING_TRACES: "true"
# JIT Integration: Path to generated tools
TOOLS_PATH: "/app/shared/mcp-connectors"
depends_on:
postgres:
condition: service_healthy
# Other half of docker-compose.byo-postgres.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
kafka:
condition: service_healthy
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
redis:
condition: service_healthy
# Wait for Temporal to accept connections so the startup client.Dial
# succeeds instead of losing the cold-boot race (workflows disabled).
# Belt-and-suspenders: the gateway also retries + lazily reconnects.
temporal:
condition: service_healthy
orchestrator:
condition: service_started
# Read-only Docker socket proxy for live MCP container status.
docker-socket-proxy:
condition: service_started
networks:
- default
- rsync-ai-mcp
volumes:
# JIT Integration: Read access to generated tools
- ./shared/mcp-connectors:/app/shared/mcp-connectors:ro
# NOTE: live docker container status (docker_status=running, port,
# container_name) is NOT reported by the catalog endpoint because
# api-gateway runs as a non-root user that can't read
# /var/run/docker.sock. Mounting the socket here returns
# "permission denied" without also setting `user: root` or doing
# GID mapping — and we don't want to elevate api-gateway just for
# an informational field. The catalog instead reports
# docker_status="auto_deploys_on_use" for connectors whose
# containers will be started on demand by the orchestrator. The
# full live-status feature is tracked as a follow-up that would
# route the docker query through tool-generator (which already
# has its own RW socket mount with the appropriate privileges).
extra_hosts:
- "host.docker.internal:host-gateway"
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
# LLM Gateway (Python) - Internal service, no host port needed
llm-service:
build:
context: ./llm-service
dockerfile: Dockerfile
args: *build-info
container_name: rsync-ai-llm-service
env_file:
# Gitignored local override. `required: false` so `docker compose config`
# (e.g. the e2e/staging wiring guards) doesn't hard-fail on a runner that
# has no local .env; the file is still loaded wherever it exists.
- path: ./llm-service/.env
required: false
environment:
# Kafka client credentials. llm-service runs the PII-scanner consumer and
# its producer (src/agents/pii_scanner/kafka_consumer.py via
# src/utils/kafka_security.py), started as a background task at
# src/gateway/main.py:770 whose failure is swallowed into a warning — so
# an unauthenticated or unreachable broker here produces NO error surface
# at all, just a scanner that never runs. Without the anchor a BYO broker
# is dialled anonymously over PLAINTEXT.
<<: *kafka-security
# The address belongs here for a second reason: docker-compose.byo-kafka.yml
# REMOVES the bundled `kafka` service, so an operator who sets
# KAFKA_BROKERS=my-msk:9098 redirected every other service while this one
# kept the code default `kafka:29092` — a host that no longer exists.
KAFKA_BROKERS: ${KAFKA_BROKERS:-kafka:29092}
# Namespace for every Kafka topic this product creates. A producer and a
# consumer resolving different names do not error, the consumer just blocks
# on a topic nobody writes — so this must agree with the five services that
# already declare it. `-` not `:-` on purpose: an explicit empty value means
# "disable" (keep pre-prefix topic names) and `:-` would silently turn that
# back into the default.
KAFKA_TOPIC_PREFIX: ${KAFKA_TOPIC_PREFIX-rsync.}
PORT: "5000"
LOG_FORMAT: json
LOG_LEVEL: info
ENVIRONMENT: development
# Sent as X-Internal-Secret on service-to-service calls to the orchestrator
# topology API (requirePrincipal). Empty = topology calls 401 in production.
INTERNAL_SERVICE_SECRET: ${INTERNAL_SERVICE_SECRET:-}
# Control-plane DB for best-effort LLM cost/usage logging (llm_usage_events +
# check_and_charge_user, migration 057). Without this the cost writer falls
# back to an empty-password DSN and fails every write with "fe_sendauth: no
# password supplied". Mirrors the api-gateway/orchestrator default; staging &
# prod override DATABASE_URL via .env to point at Azure PostgreSQL.
DATABASE_URL: ${DATABASE_URL:-postgres://user:password@postgres:5432/pipeline_db?sslmode=disable}
# OpenTelemetry Configuration (exports to the bundled OTel Collector)
OTEL_EXPORTER_OTLP_ENDPOINT: otel-collector:4317
OTEL_SERVICE_NAME: llm-service
OTEL_ENABLED: "${OTEL_ENABLED:-false}"
OTEL_TRACES_EXPORTER: otlp
OTEL_METRICS_EXPORTER: otlp
# No host port exposed - accessed internally by planner/tool-generator via Docker network
command: ["python", "-m", "src.gateway.main"]
extra_hosts:
- "host.docker.internal:host-gateway"
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
# Connector Deployer (SEC-H-02): the small, trusted, no-LLM microservice that
# holds /var/run/docker.sock. tool-generator (which runs LLM/user-influenced
# connector code) now DROPS root + the socket and calls this service over an
# authenticated internal RPC (DEPLOYER_URL) to build/run JIT connector
# containers. The security property is allow-by-construction: the deployer
# builds the Docker `create` payload itself with a hardcoded-safe HostConfig,
# so untrusted input is DATA — never a Docker API call — and no
# Privileged/bind/cap field can be smuggled through. Because the HOST daemon
# creates the container, it joins rsync-ai-mcp by name/alias and kafka-mcp-sink
# still resolves JIT destination connectors there (flat-name reachability, the
# decisive constraint). See docs/internal/jit-connector-sandboxing.md and
# connector-deployer/CONTRACT.md.
connector-deployer:
build:
context: ./connector-deployer
args:
# SEC-H-02 increment 2: lock the deployer's buildctl (client) to the buildkitd
# sidecar's image tag in docker-compose.prod.yml. Bump both via one env knob.
BUILDKIT_VERSION: ${BUILDKIT_VERSION:-v0.31.2}
container_name: rsync-ai-connector-deployer
environment:
- ENVIRONMENT=development
- LOG_FORMAT=json
- LOG_LEVEL=info
# Rotation for the connector containers THIS service creates -- compose's
# `logging:` blocks only reach the services compose itself starts.
- RSYNC_LOG_MAX_SIZE=${RSYNC_LOG_MAX_SIZE:-10m}
- RSYNC_LOG_MAX_FILE=${RSYNC_LOG_MAX_FILE:-3}
# S2S shared secret — callers send it as X-Internal-Secret. Unset in dev =>
# the deployer allows with a warning (CONTRACT auth table); the prod overlay
# fail-closes it with :? (mirrors the MinIO C3 pattern).
- INTERNAL_SERVICE_SECRET=${INTERNAL_SERVICE_SECRET:-}
# Pinned connector network used as NetworkMode — the SEC-M-06 isolation
# boundary. Same value tool-generator passed as DOCKER_NETWORK.
- DEPLOYER_DOCKER_NETWORK=${STACK_PREFIX:-rsync-ai}-mcp
# "Also-join" network for kafka-mcp-sink reachability; == the primary net
# here, so the also-join is a no-op (matches tool-generator's prior
# MCP_SHARED_NETWORK default of rsync-ai-mcp).
- MCP_SHARED_NETWORK=${STACK_PREFIX:-rsync-ai}-mcp
# Connector container-name prefix (<STACK_PREFIX>-<id>-vX-Y-Z-mcp). Without it
# an isolated stack's compose-managed connectors are not recognised as
# protected, so a stopped one can never be restarted.
- STACK_PREFIX=${STACK_PREFIX:-rsync-ai}
# Named volume for the OAuth TokenManager, mounted at /root/.rsync-ai below.
# SAME value tool-generator passed to connectors (docker_builder.py default),
# so persisted connector OAuth tokens survive the tool-generator → deployer move.
- OAUTH_TOKENS_VOLUME_NAME=rsync-ai-oauth-tokens
# Passphrase the TokenManager derives its AES-GCM key from (PBKDF2, random
# per-record salt). Without it every token write raised and was swallowed,
# so the volume above stayed empty while store_token() logged success —
# OAUTH_ENCRYPTION_KEY, which takes precedence, is set by no compose file
# and no chart. Reusing ENCRYPTION_KEY keeps this to one secret per
# deployment instead of a second one nothing remembers to set.
- ENCRYPTION_KEY=${ENCRYPTION_KEY:-dev-encryption-key-32-bytes-long!!}
# THE trusted socket holder: runs as root (0:0) and mounts the RW docker.sock.
# This is the one small, audited surface that speaks the Docker API — moved
# OFF tool-generator, which runs untrusted connector code.
user: "0:0"
networks:
- default
- rsync-ai-mcp
volumes:
# RW docker socket — build images + create/start/stop/rm JIT connector containers.
- /var/run/docker.sock:/var/run/docker.sock
# Connector artifacts, READ-ONLY: the deployer only reads build contexts,
# never writes them.
- ./shared/mcp-connectors:/app/shared/mcp-connectors:ro
# docker-compose.mcp.yml, READ-ONLY: protected-compose-container detection
# (never rebuild/tear down a compose-managed connector serving live CDC).
- ./docker-compose.mcp.yml:/app/docker-compose.mcp.yml:ro
# The SAME OAuth TokenManager volume tool-generator used, for token persistence.
- tool_generator_oauth_tokens:/root/.rsync-ai
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
# Connector-tree ownership init (one-shot). SEC-H-02 (#643) correctly dropped
# tool-generator to the image's unprivileged appuser (uid 1000) — but that same
# service still WRITES generated connector source into the RW-mounted
# ./shared/mcp-connectors tree (public/<slug>/versions/… and oauth/providers.json).
# On a root-owned checkout (prod lives at /root/rsync-ai) that tree is root:… 0755,
# so uid 1000 can't create public/<slug> and connector GENERATION fails with
# PermissionError (regression found on prod 2026-07-20; the #643 staging E2E only
# exercised DEPLOYING existing connectors, never generating a new one). This root
# one-shot chowns the tree to uid 1000 before tool-generator starts, so a fresh
# checkout / re-provision can't silently re-break generation.
#
# NOT a SEC-H-02 regression: this container holds NO docker.sock and touches ONLY
# the connector tree. The host-root surface #643 removed is the docker socket,
# which lives solely on connector-deployer — this init never goes near it.
# (OSS/quickstart are unaffected: they mount the tree :ro from a named volume
# seeded by connector-seed, so they never take this base service.)
connector-fs-init:
image: busybox:1.37.0
container_name: rsync-ai-connector-fs-init
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
user: "0:0"
command: ["chown", "-R", "1000:1000", "/app/shared/mcp-connectors"]
volumes:
- ./shared/mcp-connectors:/app/shared/mcp-connectors
restart: "no"
# Tool Generator Agent
tool-generator:
build:
context: ./llm-service
dockerfile: Dockerfile
container_name: rsync-ai-tool-generator
env_file:
# Gitignored local override. `required: false` so `docker compose config`
# (e.g. the e2e/staging wiring guards) doesn't hard-fail on a runner that
# has no local .env; the file is still loaded wherever it exists.
- path: ./llm-service/.env
required: false
environment:
- PORT=5010
- TOOLS_DIR=/app/shared/mcp-connectors
# Default: enabled for local dev. In managed mode tool-generator asks the
# connector-deployer (DEPLOYER_URL) to build images and start containers on
# the rsync-ai-mcp network in response to /v1/deploy (called by the
# orchestrator's Preflight agent when a required MCP container isn't already
# running). tool-generator no longer touches docker.sock itself (SEC-H-02):
# the connector-deployer is the sole trusted socket holder. Override to
# "false" for environments where MCPs are managed by an external
# docker-compose.mcp.yml stack (docker-compose.prod.yml ships that pattern).
- RSYNC_MANAGED_CONNECTORS=${RSYNC_MANAGED_CONNECTORS:-true}
# SEC-H-02: JIT build/run is delegated to the connector-deployer over an
# authenticated internal RPC. INTERNAL_SERVICE_SECRET is sent as
# X-Internal-Secret; unset in dev => no-op (deployer allows with a warning).
- DEPLOYER_URL=http://connector-deployer:5011
- INTERNAL_SERVICE_SECRET=${INTERNAL_SERVICE_SECRET:-}
- LOG_FORMAT=json
- LOG_LEVEL=info
- ENVIRONMENT=development
# LLM Service URL (internal Docker network)
- LLM_GATEWAY_URL=http://llm-service:5000
# Context7 MCP Service URL (internal Docker network)
- CONTEXT7_MCP_URL=http://context7-mcp:8080
# Guardrail: require a strong match between connector name and resolved Context7 docs
# to avoid false positives for random strings (tunable in prod).
- CONTEXT7_VALIDATE_MIN_SCORE=0.90
- CONTEXT7_RESOLVE_MIN_SCORE=0.85
# LLM Provider Configuration
# OPENAI_API_KEY must be provided via env_file (./llm-service/.env) or your shell environment.
- OLLAMA_BASE_URL=http://host.docker.internal:11434/v1
- OLLAMA_MODEL=${OLLAMA_MODEL:-qwen2.5:7b}
- CONTEXT7_MCP_URL=http://context7-mcp:8080
# JIT-deployed (user/LLM-generated) connectors run on the DEDICATED rsync-ai-mcp
# network — isolated from the internal services that sit on rsync-ai_default
# (kafka, temporal, redis, postgres) so untrusted connector code cannot reach
# them by L3 (SEC-M-06). orchestrator, api-gateway, tool-generator, AND
# kafka-mcp-sink all bridge into rsync-ai-mcp (see each service's networks:
# block), so they still resolve JIT + compose-MCP connectors by container name
# / alias there. This matches the compose-MCP topology (the generator script
# scripts/mcp_generate_compose.py joins rsync-ai-mcp) and the OSS/quickstart
# stacks (which already set DOCKER_NETWORK=rsync-ai-mcp). An earlier revision
# put JIT containers on rsync-ai_default to fix a sink "no such host" bug; that
# workaround is obsolete now that kafka-mcp-sink bridges into rsync-ai-mcp
# (see its networks: block above).
- DOCKER_NETWORK=${STACK_PREFIX:-rsync-ai}-mcp
# Connector container-name prefix: the orchestrator looks for
# <STACK_PREFIX>-<id>-vX-Y-Z-mcp, so /v1/deploy must start that name.
- STACK_PREFIX=${STACK_PREFIX:-rsync-ai}
# OpenTelemetry Configuration (off by default; set OTEL_ENABLED=true to export to your own collector)
- OTEL_EXPORTER_OTLP_ENDPOINT=otel-collector:4317
- OTEL_SERVICE_NAME=tool-generator
- OTEL_ENABLED=${OTEL_ENABLED:-false}
- OTEL_TRACES_EXPORTER=otlp
- OTEL_METRICS_EXPORTER=otlp
# No host port — tool-generator is internal only, accessed via api-gateway
# Entrypoint is src.lifecycle.main, NOT src.agents.tool_generator.service: the
# latter is the connector-GENERATION service and is not part of this repo, so
# its module would be missing and the container would die on import. This one
# serves the connector-lifecycle surface the data plane calls (/v1/deploy).
command: ["python", "-m", "src.lifecycle.main"]
# SEC-H-02: tool-generator runs the image's default unprivileged appuser
# (uid 1000) — it no longer needs root, because it no longer holds the
# docker socket. All JIT build/run goes through the connector-deployer
# (DEPLOYER_URL), the sole trusted socket holder. This is the security win:
# the one service that runs LLM/user-influenced connector code is no longer
# host-root-equivalent.
networks:
- default
- rsync-ai-mcp
volumes:
- ./shared/mcp-connectors:/app/shared/mcp-connectors
- ./docker-compose.mcp.yml:/app/docker-compose.mcp.yml
# Persist OAuth TokenManager storage across container restarts. The
# connector-deployer mounts this SAME named volume (SEC-H-02), so OAuth
# tokens stay readable by the connectors it spawns.
# NOTE (SEC-H-02): no docker.sock mount here anymore — managed-mode JIT
# build/run is delegated to the connector-deployer over its internal RPC.
- tool_generator_oauth_tokens:/root/.rsync-ai
# Map form (not the short list) so connector-fs-init can gate on completion:
# tool-generator must not start until the connector tree is chowned to uid 1000,
# or its first generation write races the init and fails with PermissionError.
depends_on:
llm-service:
condition: service_started
context7-mcp:
condition: service_started
connector-deployer:
condition: service_started
connector-fs-init:
condition: service_completed_successfully
extra_hosts:
- "host.docker.internal:host-gateway"
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
# Planner Agent
planner:
build:
context: ./llm-service
dockerfile: Dockerfile
container_name: rsync-ai-planner
env_file:
# Gitignored local override. `required: false` so `docker compose config`
# (e.g. the e2e/staging wiring guards) doesn't hard-fail on a runner that
# has no local .env; the file is still loaded wherever it exists.
- path: ./llm-service/.env
required: false
environment:
# Kafka client security for a BYO/managed broker. See x-kafka-security at
# the top of this file; every key defaults to empty and empty means unset,
# so the bundled broker is unaffected.
<<: *kafka-security
PORT: "5011"
LOG_FORMAT: "json"
LOG_LEVEL: "info"
ENVIRONMENT: "development"
TOOL_GENERATOR_URL: "http://tool-generator:5010"
API_GATEWAY_URL: "http://api-gateway:8080"
LLM_GATEWAY_URL: "http://llm-service:5000"
KAFKA_BOOTSTRAP_SERVERS: "${KAFKA_BROKERS:-kafka:29092}"
# Namespace for every Kafka topic this product creates, so an operator can
# tell them apart from other applications' topics on a shared (BYO-Kafka)
# cluster. Every service that touches Kafka must agree on this value: a
# producer and a consumer resolving different names do not error, the
# consumer just blocks on a topic nobody writes. `-` not `:-` on purpose --
# an explicit empty value means "disable" (keep pre-prefix topic names) and
# `:-` would silently turn that back into the default.
KAFKA_TOPIC_PREFIX: "${KAFKA_TOPIC_PREFIX-rsync.}"
# JIT Integration: Planner needs access to check connector existence
CONNECTORS_DIR: "/app/shared/mcp-connectors"
# OpenTelemetry Configuration (off by default; set OTEL_ENABLED=true to export to your own collector)
OTEL_EXPORTER_OTLP_ENDPOINT: "otel-collector:4317"
OTEL_SERVICE_NAME: "planner"
OTEL_ENABLED: "${OTEL_ENABLED:-false}"
OTEL_TRACES_EXPORTER: "otlp"
OTEL_METRICS_EXPORTER: "otlp"
# Sent to the orchestrator's /api/v1/topology (strategies.py), which
# requirePrincipal gates; without it plan-time topic creation gets 401.
INTERNAL_SERVICE_SECRET: "${INTERNAL_SERVICE_SECRET:-}"
volumes:
# JIT Integration: Shared volume for connector existence checks
- ./shared/mcp-connectors:/app/shared/mcp-connectors:ro
ports:
- "127.0.0.1:${RSYNC_HP_PLANNER:-8001}:5011"
command: ["python", "-m", "src.agents.planner.service"]
# Long-form depends_on: the short list form cannot carry `required: false`,
# and the `kafka` entry below needs it — see docker-compose.byo-kafka.yml.
# `service_started` on every entry is exactly what the short form meant.
depends_on:
llm-service:
condition: service_started
tool-generator:
condition: service_started
api-gateway:
condition: service_started
kafka:
condition: service_started
# Other half of docker-compose.byo-kafka.yml — see that file's header:
# profiles: without required:false = "depends on undefined service".
required: false
extra_hosts:
- "host.docker.internal:host-gateway"
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
# Sentinel Agent (Infrastructure Watchdog)
# Frontend (Next.js)
frontend:
container_name: rsync-ai-frontend
build:
context: ./frontend
dockerfile: Dockerfile
env_file:
# Gitignored local override — optional so `docker compose config` works
# on a runner without a local .env (loaded normally where it exists).
- path: ./frontend/.env
required: false
ports:
- "${RSYNC_BIND_ADDR:-127.0.0.1}:${RSYNC_HP_FRONTEND:-3000}:3000"
environment:
# Internal URLs for server-side fetches (within Docker network)
- API_GATEWAY_INTERNAL_URL=http://api-gateway:8080
- ORCHESTRATOR_INTERNAL_URL=http://orchestrator:8080
# Server-only (NOT NEXT_PUBLIC): the RSC dashboard authenticates its
# server-side call to the orchestrator with it (instrumentation.ts warns if unset).
- INTERNAL_SERVICE_SECRET=${INTERNAL_SERVICE_SECRET:-}
# Public URLs for client-side fetches (from browser via host)
- NEXT_PUBLIC_API_URL=http://localhost:5001
- NEXT_PUBLIC_BACKEND_ORCHESTRATOR_URL=http://localhost:8080
- NEXT_PUBLIC_WS_URL=ws://localhost:5001/ws
# Planner (agent) service exposed on host port 8001
- NEXT_PUBLIC_LLM_SERVICE_URL=http://localhost:8001
- NEXTAUTH_URL=http://localhost:3000
# OpenTelemetry Configuration (for browser tracing)
- NEXT_PUBLIC_OTEL_ENDPOINT=http://localhost:4318
- NEXT_PUBLIC_OTEL_SERVICE_NAME=frontend
# Monitoring UI feature flags (V1)
- NEXT_PUBLIC_FEATURE_MONITORING_OVERVIEW=true
- NEXT_PUBLIC_FEATURE_MONITORING_INFRA=true
- NEXT_PUBLIC_FEATURE_MONITORING_TRACES=true
depends_on:
- api-gateway
extra_hosts:
- "host.docker.internal:host-gateway"
logging:
driver: json-file
options:
max-size: ${RSYNC_LOG_MAX_SIZE:-10m}
max-file: "${RSYNC_LOG_MAX_FILE:-3}"
volumes:
postgres_data:
kafka_data:
redis_data:
minio_data:
blob_staging_data:
# Debezium source secrets: written by debezium-mcp, read-only-mounted by
# kafka-connect, so credentials resolve via FileConfigProvider instead of being
# stored plaintext in the Kafka config topic.
connect_secrets:
tool_generator_oauth_tokens:
name: rsync-ai-oauth-tokens
networks:
# rsync-ai-mcp: dedicated network for USER MCP connector containers — both the
# compose-generated connectors (docker-compose.mcp.yml) and the JIT-deployed ones
# (tool-generator docker_builder.py). orchestrator, tool-generator, api-gateway,
# AND kafka-mcp-sink bridge into it so they can reach those connectors by container
# name / alias. Internal services (postgres, redis, kafka, temporal) do NOT join
# this network, so user connectors on it cannot reach them by L3 — this is the
# SEC-M-06 isolation boundary (JIT connectors are pinned here via DOCKER_NETWORK).
rsync-ai-mcp:
name: rsync-ai-mcp