docker-compose.yml
Rama main · réplica de lectura del repositorio original.
# Run with DOCKER_BUILD_TARGET=dev to build Rust inside Docker
include:
- scripts/compose/resources.yml
- scripts/compose/edgeshark.yml
- scripts/compose/portal.yml
x-erlang-cluster: &erlang-cluster
ERLANG_CLUSTER_ADAPTER: "Elixir.Cluster.Strategy.Epmd"
ERLANG_CLUSTER_ADAPTER_CONFIG: '{"hosts":["portal@portal.cluster.local"]}'
x-portal-urls: &portal-urls
WEB_EXTERNAL_URL: http://localhost:8080/
API_EXTERNAL_URL: http://localhost:8081/
x-health-check: &health-check
interval: 1s
retries: 15
timeout: 1s
# Shared by the pool-member container (FIREZONE_ID) and the seeds (POOL_MEMBER_FIREZONE_ID).
x-pool-member-firezone-id: &pool-member-firezone-id "68511a32d29b4c81591877beea18d3c95e4d12c52314b1aa2f5a568eccc0e6ad"
services:
portal:
extends:
file: scripts/compose/portal.yml
service: common
build:
context: elixir
args:
APPLICATION_NAME: portal
image: ${PORTAL_IMAGE:-ghcr.io/firezone/portal}:${PORTAL_TAG:-main}
hostname: portal.cluster.local
ports:
- 8080:8080/tcp
- 8081:8081/tcp
environment:
<<: [*portal-urls, *erlang-cluster]
PHOENIX_SECURE_COOKIES: "false"
RELEASE_HOSTNAME: "portal.cluster.local"
RELEASE_NAME: "portal"
LOG_LEVEL: "debug"
user: root # Needed to run `ip route` commands
cap_add:
- NET_ADMIN # Needed to run `tc` commands to add simulated delay
command:
- sh
- -c
- |
set -e
# Add static route to internet subnet via router
ip -4 route add 203.0.113.0/24 via 172.28.0.254
ip -6 route add 203:0:113::/64 via 172:28:0::254
exec su default -c "bin/server"
depends_on:
postgres:
condition: "service_healthy"
portal-router:
condition: "service_healthy"
healthcheck:
test: ["CMD-SHELL", "curl -f localhost:8080/readyz"]
<<: *health-check
networks:
app-internal:
ipv4_address: 172.28.0.100
ipv6_address: 172:28:0::100
portal-router:
extends:
file: scripts/compose/router.yml
service: router
environment:
PORT_FORWARDS: |
8081 172.28.0.100 tcp
8081 172:28:0::100 tcp
MASQUERADE_TYPE: ""
NETWORK_LATENCY_MS: "${PORTAL_ROUTER_LATENCY_MS:-}"
networks:
app-internal:
ipv4_address: 172.28.0.254
ipv6_address: 172:28:0::254
interface_name: internal
internet:
ipv4_address: 203.0.113.10
ipv6_address: 203:0:113::10
interface_name: internet
# This is a service container which allows to run mix tasks for local development
# without having to install Elixir and Erlang on the host machine.
elixir:
extends:
file: scripts/compose/portal.yml
service: common
build:
context: elixir
target: compiler
args:
APPLICATION_NAME: portal
image: ${ELIXIR_IMAGE:-ghcr.io/firezone/elixir}:${ELIXIR_TAG:-main}
hostname: elixir
environment:
<<: *portal-urls
RELEASE_HOSTNAME: "mix.cluster.local"
RELEASE_NAME: "mix"
# Higher log level not to make seeds output too verbose
LOG_LEVEL: "info"
# Mix env should be set to prod to use secrets declared above,
# otherwise seeds will generate invalid tokens
MIX_ENV: "prod"
POOL_MEMBER_FIREZONE_ID: *pool-member-firezone-id
# Toggles the account-wide ICE-less feature flag in `mix ecto.seed`, so
# integration tests can run with or without ICE-less. Defaults to off.
FEATURE_ICELESS_ENABLED: ${FEATURE_ICELESS_ENABLED:-false}
depends_on:
postgres:
condition: "service_healthy"
# Run with DOCKER_BUILD_TARGET=dev to build Rust inside Docker
primary-client:
extends:
file: scripts/compose/client.yml
service: client
environment:
FIREZONE_TOKEN: "n.SFMyNTY.g2gDaANtAAAAJGM4OWJjYzhjLTkzOTItNGRhZS1hNDBkLTg4OGFlZjZkMjhlMG0AAAAkN2RhN2QxY2QtMTExYy00NGE3LWI1YWMtNDAyN2I5ZDIzMGU1bQAAACtBaUl5XzZwQmstV0xlUkFQenprQ0ZYTnFJWktXQnMyRGR3XzJ2Z0lRdkZnbgYAR_ywiZQBYgABUYA.PLNlzyqMSgZlbQb1QX5EzZgYNuY9oeOddP0qDkTwtGg"
FIREZONE_ID: 2606a54a327020a1285cc141aeceb5737a91625e5e1a85c234682266f45820bf
command:
- sh
- -c
- |
set -e
# Add static route to internet subnet via router
ip -4 route add 203.0.113.0/24 via 172.30.0.254
ip -6 route add 203:0:113::/64 via 172:30:0::254
ip link set dev eth0 qlen 100000
exec firezone-headless-client
depends_on:
primary-client-router:
condition: "service_healthy"
networks:
primary-client-internal:
ipv4_address: 172.30.0.100
ipv6_address: 172:30:0::100
primary-client-router:
extends:
file: scripts/compose/router.yml
service: router
environment:
MASQUERADE_TYPE: ${CLIENT_MASQUERADE:-}
NETWORK_LATENCY_MS: "${EDGE_ROUTER_LATENCY_MS:-}"
networks:
primary-client-internal:
ipv4_address: 172.30.0.254
ipv6_address: 172:30:0::254
interface_name: internal
internet:
interface_name: internet
# One-shot secnetperf client for QUIC performance tests. Shares the network
# namespace of `primary-client` so its traffic is routed through the tunnel.
# Run with `docker compose run --rm secnetperf-client secnetperf -target:...`.
secnetperf-client:
image: ghcr.io/firezone/secnetperf:v2.6.0
network_mode: service:primary-client
depends_on:
primary-client:
condition: "service_healthy"
pool-member:
extends:
file: scripts/compose/client.yml
service: client
environment:
FIREZONE_TOKEN: "n.SFMyNTY.g2gDaANtAAAAJGM4OWJjYzhjLTkzOTItNGRhZS1hNDBkLTg4OGFlZjZkMjhlMG0AAAAkZmUyYmYyMmQtMDk4Ni00MzNhLTg1ZWQtYThmMzdjOTBhMzRibQAAADg5SE00QTZSRE03UUxEVjhOMUMwRFFEMThKVEpOVUpJQ0Y0QThWUklKODM0SkhDTjA5NFIwPT09PW4GACphrxmeAWIAAVGA.LHufO50MaDTncVMV1CBGr3pnpemR-__n8RSbh9xwB3Y"
FIREZONE_ID: *pool-member-firezone-id
command:
- sh
- -c
- |
set -e
# Add static route to internet subnet via router
ip -4 route add 203.0.113.0/24 via 172.32.0.254
ip -6 route add 203:0:113::/64 via 172:32:0::254
ip link set dev eth0 qlen 100000
exec firezone-headless-client
depends_on:
pool-member-router:
condition: "service_healthy"
networks:
pool-member-internal:
ipv4_address: 172.32.0.100
ipv6_address: 172:32:0::100
pool-member-router:
extends:
file: scripts/compose/router.yml
service: router
environment:
MASQUERADE_TYPE: ${POOL_MEMBER_MASQUERADE:-}
NETWORK_LATENCY_MS: "${EDGE_ROUTER_LATENCY_MS:-}"
networks:
pool-member-internal:
ipv4_address: 172.32.0.254
ipv6_address: 172:32:0::254
interface_name: internal
internet:
interface_name: internet
gateway:
healthcheck:
test: ["CMD-SHELL", "ip link | grep tun-firezone"]
<<: *health-check
environment:
FIREZONE_TOKEN: ".SFMyNTY.g2gDaANtAAAAJGM4OWJjYzhjLTkzOTItNGRhZS1hNDBkLTg4OGFlZjZkMjhlMG0AAAAkMjI3NDU2MGItZTk3Yi00NWU0LThiMzQtNjc5Yzc2MTdlOThkbQAAADhPMDJMN1VTMkozVklOT01QUjlKNklMODhRSVFQNlVPOEFRVk82VTVJUEwwVkpDMjJKR0gwPT09PW4GAAH8sImUAWIAAVGA.tAm2O9FcyF67VAF3rZdwQpeADrYOIs3S2l2K51G26OM"
RUST_LOG: ${RUST_LOG:-wire=trace,debug,flow_logs=trace}
FIREZONE_FLOW_LOGS: true
FIREZONE_API_URL: ws://portal:8081
FIREZONE_ID: c9cc2a8a01f9d3d6b2c26719965709196ae2c79214a837e28f0a751af1e2f2ca
command:
- sh
- -c
- |
set -e
# Add static route to internet subnet via router
ip -4 route add 203.0.113.0/24 via 172.31.0.254
ip -6 route add 203:0:113::/64 via 172:31:0::254 || true
ip link set dev eth0 qlen 100000
ip link set dev eth1 qlen 100000
ip link set dev eth2 qlen 100000
exec firezone-gateway
init: true
build:
target: ${DOCKER_BUILD_TARGET:-debug}
context: rust
dockerfile: Dockerfile
args:
PACKAGE: firezone-gateway
image: ${GATEWAY_IMAGE:-ghcr.io/firezone/debug/gateway}:${GATEWAY_TAG:-main}
cap_add:
- NET_ADMIN
sysctls:
- net.ipv4.ip_forward=1
- net.ipv6.conf.all.disable_ipv6=${GATEWAY_DISABLE_IPV6:-0}
- net.ipv6.conf.default.disable_ipv6=${GATEWAY_DISABLE_IPV6:-0}
- net.ipv6.conf.all.forwarding=1
- net.ipv6.conf.default.forwarding=1
devices:
- "/dev/net/tun:/dev/net/tun"
depends_on:
gateway-router:
condition: "service_healthy"
portal:
condition: "service_healthy"
networks:
gateway-internal:
ipv4_address: 172.31.0.100
ipv6_address: 172:31:0::100
dns_resources:
resources:
extra_hosts:
- "portal:203.0.113.10"
- "portal:203:0:113::10"
gateway-router:
extends:
file: scripts/compose/router.yml
service: router
environment:
MASQUERADE_TYPE: ${GATEWAY_MASQUERADE:-}
NETWORK_LATENCY_MS: "${EDGE_ROUTER_LATENCY_MS:-}"
networks:
gateway-internal:
ipv4_address: 172.31.0.254
ipv6_address: 172:31:0::254
interface_name: internal
internet:
interface_name: internet
relay-1:
extends:
file: scripts/compose/relay.yml
service: relay
environment:
PUBLIC_IP4_ADDR: 203.0.113.101
PUBLIC_IP6_ADDR: 203:0:113::101
command:
- sh
- -c
- |
set -e
# Add static route to internet subnet via router
ip -4 route add 203.0.113.0/24 via 172.29.1.254
ip -6 route add 203:0:113::/64 via 172:29:1::254
# Our own packets cross the xdp_pass program on the host-side veth peer,
# which requires segmented frames with software-computed checksums.
apk add --no-cache ethtool
ethtool -K eth0 tx off
firezone-relay
depends_on:
relay-1-router:
condition: "service_healthy"
networks:
relay-1-internal:
ipv4_address: 172.29.1.100
ipv6_address: 172:29:1::100
relay-1-router:
extends:
file: scripts/compose/router.yml
service: router
environment:
PORT_FORWARDS: |
3478 172.29.1.100 udp
49152-65535 172.29.1.100 udp
3478 172:29:1::100 udp
49152-65535 172:29:1::100 udp
NETWORK_LATENCY_MS: "${RELAY_ROUTER_LATENCY_MS:-}"
networks:
relay-1-internal:
ipv4_address: 172.29.1.254
ipv6_address: 172:29:1::254
interface_name: internal
internet:
ipv4_address: 203.0.113.101
ipv6_address: 203:0:113::101
interface_name: internet
relay-2:
extends:
file: scripts/compose/relay.yml
service: relay
environment:
PUBLIC_IP4_ADDR: 203.0.113.102
PUBLIC_IP6_ADDR: 203:0:113::102
command:
- sh
- -c
- |
set -e
# Add static route to internet subnet via router
ip -4 route add 203.0.113.0/24 via 172.29.2.254
ip -6 route add 203:0:113::/64 via 172:29:2::254
# Our own packets cross the xdp_pass program on the host-side veth peer,
# which requires segmented frames with software-computed checksums.
apk add --no-cache ethtool
ethtool -K eth0 tx off
firezone-relay
depends_on:
relay-2-router:
condition: "service_healthy"
networks:
relay-2-internal:
ipv4_address: 172.29.2.100
ipv6_address: 172:29:2::100
relay-2-router:
extends:
file: scripts/compose/router.yml
service: router
environment:
PORT_FORWARDS: |
3478 172.29.2.100 udp
49152-65535 172.29.2.100 udp
3478 172:29:2::100 udp
49152-65535 172:29:2::100 udp
NETWORK_LATENCY_MS: "${RELAY_ROUTER_LATENCY_MS:-}"
networks:
relay-2-internal:
ipv4_address: 172.29.2.254
ipv6_address: 172:29:2::254
interface_name: internal
internet:
ipv4_address: 203.0.113.102
ipv6_address: 203:0:113::102
interface_name: internet
# The veth driver uses a pair of interfaces to connect the docker bridge to the container namespace.
# For containers that have an eBPF program attached and do XDP_TX, we need to attach a dummy program
# to the corresponding veth interface on the host to be able to receive the XDP_TX traffic and pass
# it up to the docker bridge successfully.
#
# The "recommended" way to do this is to set both veth interfaces' GRO to on, or attach an XDP program
# that does XDP_PASS to the host side veth interface. The GRO method is not reliable and was shown to
# only pass packets in large bursts every 15-20 seconds which breaks ICE setup, so we use the XDP method.
#
# For correct behaviour, we also disable tx offload (checksumming and, by dependency, TSO/USO)
# on the relay-internal veth and bridge devices. This forces the kernel to segment packets and
# calculate their checksums in software before they reach the XDP programs. All other devices
# keep their default offloads so packets traverse the rest of the compose networks coalesced.
network-config:
image: ghcr.io/firezone/xdp-pass
pid: host
network_mode: host
privileged: true
restart: on-failure
command:
- sh
- -c
- |
set -e
# The relays do XDP_TX on eth0, so the host-side veth peering each relay must
# run an XDP program to receive those frames and pass them up to the bridge.
# Those peers are exactly the veths enslaved to the relay-internal bridges
# (the only docker networks in 172.29.0.0/16). Scoping xdp_pass to them keeps
# it off the client<->gateway path, where attaching to every veth would cost
# ~18% of the forwarding softirq (veth_xdp_rcv). This service depends on both
# relays being healthy, so their veths already exist on those bridges.
#
# XDP sees raw frames, so packets crossing these devices must be segmented and
# carry software-computed checksums. Disabling tx offload on the relay-internal
# bridges and their veths forces both at this last hop into the relay.
for br in $$(ip -json link show type bridge | jq -r '.[].ifname'); do
ip -json addr show dev $$br | jq -e '.[].addr_info[]? | select((.local // "") | startswith("172.29."))' >/dev/null || continue
ethtool -K $$br tx off
for dev in $$(ip -json link show master $$br | jq -r '.[].ifname'); do
ethtool -K $$dev tx off
ip link set dev $$dev xdpdrv off 2>/dev/null || true
ip link set dev $$dev xdpdrv obj /xdp/xdp_pass.o sec xdp
echo "Disabled tx offload and attached xdp_pass to $$dev (relay-internal bridge $$br)"
done
done
depends_on:
relay-1:
condition: "service_healthy"
relay-2:
condition: "service_healthy"
relay-1-router:
condition: "service_healthy"
relay-2-router:
condition: "service_healthy"
otel:
image: otel/opentelemetry-collector:latest
ports:
- 127.0.0.1:4317:4317 # OTLP gRPC receiver
- 127.0.0.1:4318:4318 # OTLP http receiver
networks:
app-internal:
networks:
# Internet network - where all public IPs live
internet:
enable_ipv6: true
ipam:
config:
- subnet: 203.0.113.0/24
- subnet: 203:0:113::/64
app-internal:
enable_ipv6: true
ipam:
config:
- subnet: 172.28.0.0/24
- subnet: 172:28:0::/64
relay-1-internal:
enable_ipv6: true
ipam:
config:
- subnet: 172.29.1.0/24
- subnet: 172:29:1::/64
relay-2-internal:
enable_ipv6: true
ipam:
config:
- subnet: 172.29.2.0/24
- subnet: 172:29:2::/64
primary-client-internal:
enable_ipv6: true
ipam:
config:
- subnet: 172.30.0.0/24
- subnet: 172:30:0::/64
pool-member-internal:
enable_ipv6: true
ipam:
config:
- subnet: 172.32.0.0/24
- subnet: 172:32:0::/64
gateway-internal:
enable_ipv6: true
ipam:
config:
- subnet: 172.31.0.0/24
- subnet: 172:31:0::/64