Makefile
Rama dev · réplica de lectura del repositorio original.
# Makefile for Bifrost
# Variables
HOST ?= localhost
PORT ?= 8080
APP_DIR ?=
PROMETHEUS_LABELS ?=
LOG_STYLE ?= json
LOG_LEVEL ?= info
TEST_REPORTS_DIR ?= test-reports
GOTESTSUM_FORMAT ?= standard-verbose
FLOW ?=
VERSION ?= dev-build
LOCAL ?=
DEBUG ?=
COMPAT ?=
# Colors for output
RED=\033[0;31m
GREEN=\033[0;32m
YELLOW=\033[1;33m
BLUE=\033[0;34m
CYAN=\033[0;36m
NC=\033[0m # No Color
ECHO := printf '%b\n'
# nvm requires bash-compatible shell semantics; /bin/sh is dash on some Linux distros.
SHELL := /usr/bin/env bash
# Ensures the Node version pinned in .nvmrc is active before any npm/node call.
# nvm is a shell function, so each recipe that needs it must inline this snippet
# via `$(USE_NODE); <your command>`.
USE_NODE = NVM_SH="$${NVM_DIR:-$$HOME/.nvm}/nvm.sh"; \
[ -s "$$NVM_SH" ] || NVM_SH="$$(brew --prefix nvm 2>/dev/null)/nvm.sh"; \
if [ -s "$$NVM_SH" ]; then . "$$NVM_SH" >/dev/null && nvm install >/dev/null 2>&1 && nvm use >/dev/null 2>&1; fi
# Loads secrets into the current recipe shell. Reads USE_INFISICAL env var:
# USE_INFISICAL=1|y|Y|yes|YES|true|TRUE -> source secrets from Infisical (`infisical export --path <p>`)
# anything else -> source ./.env
# Honors INFISICAL_PATH (default /local) when sourcing from Infisical.
# After invoking `$(EXPOSE_ENV);`, all subsequent commands inherit the secrets
# - no per-command prefix needed.
# Use as: `$(EXPOSE_ENV); <your command>`
define EXPOSE_ENV
case "$$USE_INFISICAL" in \
1|y|Y|yes|YES|true|TRUE) USE_INFISICAL_RESOLVED=1 ;; \
*) USE_INFISICAL_RESOLVED=0 ;; \
esac; \
if [ "$$USE_INFISICAL_RESOLVED" = "1" ]; then \
if ! which infisical > /dev/null 2>&1; then \
$(ECHO) "$(RED)infisical CLI not found. Install: https://infisical.com/docs/cli/overview$(NC)"; \
exit 1; \
fi; \
INFISICAL_PATH_VAL="$${INFISICAL_PATH:-/local}"; \
$(ECHO) "$(GREEN)Sourcing secrets from Infisical (path=$$INFISICAL_PATH_VAL)$(NC)"; \
if ! infisical export --path "$$INFISICAL_PATH_VAL" --format dotenv > /dev/null 2>&1; then \
$(ECHO) "$(RED)Failed to export secrets from Infisical (path=$$INFISICAL_PATH_VAL)$(NC)"; \
infisical export --path "$$INFISICAL_PATH_VAL" --format dotenv 2>&1 >/dev/null; \
exit 1; \
fi; \
set -a; . <(infisical export --path "$$INFISICAL_PATH_VAL" --format dotenv); set +a; \
else \
if [ -f .env ]; then \
$(ECHO) "$(YELLOW)Loading environment variables from .env...$(NC)"; \
set -a; . ./.env; set +a; \
fi; \
fi
endef
.PHONY: test-memory all help dev dev-pulse build-ui build build-cli run run-cli install-air install-pulse clean test test-cli install-ui setup-workspace work-init work-clean docs docker-image docker-run cleanup-enterprise mod-tidy test-integrations test-integrations-py test-integrations-ts install-playwright run-e2e run-a11y-audit run-e2e-ui run-e2e-headed e2e-ui-down run-e2e-api run-mcp-codemode-test run-warp-test format ui install-newman run-provider-harness-test smoke-provider-harness-test run-cli-harness-test cli-harness-report test-harness-runner-lib run-video-costing-test list-video-costing-cases test-semantic-cache test-semantic-cache-complete _test-semantic-cache-complete-inner helm-index install-microsocks socks5-proxy install-tinyproxy http-proxy install-squid install-stunnel run-e2e-api-proxy run-mcp-harness-test run-log-store-matrix
all: help
# Include deployment recipes
include recipes/fly.mk
include recipes/ecs.mk
include recipes/local-k8s.mk
# Default target
help: ## Show this help message
@$(ECHO) "$(BLUE)Bifrost Development - Available Commands:$(NC)"
@$(ECHO) ""
@grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " $(GREEN)%-15s$(NC) %s\n", $$1, $$2}'
@$(ECHO) ""
@$(ECHO) "$(YELLOW)Environment Variables:$(NC)"
@$(ECHO) " HOST Server host (default: localhost)"
@$(ECHO) " PORT Server port (default: 8080)"
@$(ECHO) " PROMETHEUS_LABELS Labels for Prometheus metrics"
@$(ECHO) " LOG_STYLE Logger output format: json|pretty (default: json)"
@$(ECHO) " LOG_LEVEL Logger level: debug|info|warn|error (default: info)"
@$(ECHO) " APP_DIR App data directory inside container (default: /app/data)"
@$(ECHO) " LOCAL Use local go.work for builds (e.g., make build LOCAL=1)"
@$(ECHO) " DEBUG Enable delve debugger on port 2345 (e.g., make dev DEBUG=1, make test-core DEBUG=1, make test-governance DEBUG=1)"
@$(ECHO) ""
@$(ECHO) "$(YELLOW)Test Configuration:$(NC)"
@$(ECHO) " TEST_REPORTS_DIR Directory for HTML test reports (default: test-reports)"
@$(ECHO) " GOTESTSUM_FORMAT Test output format: testname|dots|pkgname|standard-verbose (default: standard-verbose)"
@$(ECHO) " TESTCASE Exact test name to run (e.g., TestVirtualKeyTokenRateLimit)"
@$(ECHO) " PATTERN Substring pattern to filter tests (alternative to TESTCASE)"
@$(ECHO) " FLOW E2E test flow to run: providers|virtual-keys (default: all)"
cleanup-enterprise: ## Clean up enterprise directories if present
@$(ECHO) "$(GREEN)Cleaning up enterprise...$(NC)"
@if [ -d "ui/app/enterprise" ]; then rm -rf ui/app/enterprise; fi
@$(ECHO) "$(GREEN)Enterprise cleaned up$(NC)"
install-ui: cleanup-enterprise
@$(USE_NODE); \
which node > /dev/null || ($(ECHO) "$(RED)Error: Node.js is not installed. Please install Node.js first.$(NC)" && exit 1); \
which npm > /dev/null || ($(ECHO) "$(RED)Error: npm is not installed. Please install npm first.$(NC)" && exit 1); \
$(ECHO) "$(GREEN)Node.js $$(node -v) and npm $$(npm -v) are installed$(NC)"; \
if [ ! -d "ui/node_modules" ] || [ "ui/package.json" -nt "ui/node_modules/.package-lock.json" ] || [ "ui/package-lock.json" -nt "ui/node_modules/.package-lock.json" ]; then \
$(ECHO) "$(YELLOW)Dependencies changed, running npm ci...$(NC)"; \
cd ui && npm ci; \
else \
$(ECHO) "$(GREEN)UI dependencies up to date, skipping install$(NC)"; \
fi
@$(ECHO) "$(GREEN)UI deps are in sync$(NC)"
install-air: ## Install air for hot reloading (if not already installed)
@which air > /dev/null || ($(ECHO) "$(YELLOW)Installing air for hot reloading...$(NC)" && go install github.com/air-verse/air@latest)
@$(ECHO) "$(GREEN)Air is ready$(NC)"
install-pulse: ## Install pulse for hot reloading (if not already installed)
@which pulse > /dev/null || ($(ECHO) "$(YELLOW)Installing pulse for hot reloading...$(NC)" && go install github.com/Pratham-Mishra04/pulse@latest)
@$(ECHO) "$(GREEN)Pulse is ready$(NC)"
install-delve: ## Install delve for debugging (if not already installed)
@which dlv > /dev/null || ($(ECHO) "$(YELLOW)Installing delve for debugging...$(NC)" && go install github.com/go-delve/delve/cmd/dlv@latest)
@$(ECHO) "$(GREEN)Delve is ready$(NC)"
install-gotestsum: ## Install gotestsum for test reporting (if not already installed)
@which gotestsum > /dev/null || ($(ECHO) "$(YELLOW)Installing gotestsum for test reporting...$(NC)" && go install gotest.tools/gotestsum@latest)
@$(ECHO) "$(GREEN)gotestsum is ready$(NC)"
install-junit-viewer: ## Install junit-viewer for HTML report generation (if not already installed)
@if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(GREEN)junit-viewer is already installed$(NC)"; \
else \
$(ECHO) "$(YELLOW)Installing junit-viewer for HTML reports...$(NC)"; \
$(USE_NODE); \
if npm install -g junit-viewer 2>&1; then \
$(ECHO) "$(GREEN)junit-viewer installed successfully$(NC)"; \
else \
$(ECHO) "$(RED)Failed to install junit-viewer. HTML reports will be skipped.$(NC)"; \
$(ECHO) "$(YELLOW)You can install it manually: npm install -g junit-viewer$(NC)"; \
exit 0; \
fi; \
fi; \
else \
$(ECHO) "$(YELLOW)CI environment detected, skipping junit-viewer installation$(NC)"; \
fi
install-microsocks: ## Install microsocks SOCKS5 proxy for local testing (if not already installed)
@which microsocks > /dev/null || (command -v brew > /dev/null && $(ECHO) "$(YELLOW)Installing microsocks via Homebrew...$(NC)" && brew install microsocks) || ($(ECHO) "$(RED)Error: microsocks not found and Homebrew is unavailable. Install manually: https://github.com/rofl0r/microsocks$(NC)" && exit 1)
@$(ECHO) "$(GREEN)microsocks is ready$(NC)"
socks5-proxy: install-microsocks ## Run a local SOCKS5 proxy for testing provider proxy_config (Usage: make socks5-proxy [PORT=1080] [HOST=127.0.0.1])
@PROXY_PORT=$${PORT:-1080}; \
PROXY_HOST=$${HOST:-127.0.0.1}; \
$(ECHO) "$(GREEN)Starting SOCKS5 proxy on $$PROXY_HOST:$$PROXY_PORT (no auth, logs each connection, Ctrl+C to stop)...$(NC)"; \
$(ECHO) "$(YELLOW)Point a provider's proxy_config at socks5://$$PROXY_HOST:$$PROXY_PORT to test$(NC)"; \
microsocks -i $$PROXY_HOST -p $$PROXY_PORT
install-tinyproxy: ## Install tinyproxy HTTP proxy for local testing (if not already installed)
@which tinyproxy > /dev/null || (command -v brew > /dev/null && $(ECHO) "$(YELLOW)Installing tinyproxy via Homebrew...$(NC)" && brew install tinyproxy) || ($(ECHO) "$(RED)Error: tinyproxy not found and Homebrew is unavailable. Install manually: https://github.com/tinyproxy/tinyproxy$(NC)" && exit 1)
@$(ECHO) "$(GREEN)tinyproxy is ready$(NC)"
http-proxy: install-tinyproxy ## Run a local HTTP proxy for testing provider proxy_config (Usage: make http-proxy [PORT=8888] [HOST=127.0.0.1])
@PROXY_PORT=$${PORT:-8888}; \
PROXY_HOST=$${HOST:-127.0.0.1}; \
CONF=$$(mktemp -t bifrost-tinyproxy); \
trap 'rm -f "$$CONF"' EXIT INT TERM; \
printf 'Port %s\nListen %s\nTimeout 600\nAllow 127.0.0.1\nAllow ::1\nLogLevel Info\n' "$$PROXY_PORT" "$$PROXY_HOST" > "$$CONF"; \
$(ECHO) "$(GREEN)Starting HTTP proxy on $$PROXY_HOST:$$PROXY_PORT (no auth, logs each connection, Ctrl+C to stop)...$(NC)"; \
$(ECHO) "$(YELLOW)Point a provider's proxy_config at http://$$PROXY_HOST:$$PROXY_PORT to test$(NC)"; \
tinyproxy -d -c "$$CONF"
install-squid: ## Install squid HTTP proxy for the proxy e2e flow (if not already installed)
@command -v squid > /dev/null || [ -x /usr/sbin/squid ] || (command -v brew > /dev/null && $(ECHO) "$(YELLOW)Installing squid via Homebrew...$(NC)" && brew install squid) || ($(ECHO) "$(RED)Error: squid not found and Homebrew is unavailable. Install it with your package manager (apt install squid)$(NC)" && exit 1)
@$(ECHO) "$(GREEN)squid is ready$(NC)"
install-stunnel: ## Install stunnel (TLS in front of squid for https:// proxies) for the proxy e2e flow (if not already installed)
@command -v stunnel > /dev/null || command -v stunnel4 > /dev/null || (command -v brew > /dev/null && $(ECHO) "$(YELLOW)Installing stunnel via Homebrew...$(NC)" && brew install stunnel) || ($(ECHO) "$(RED)Error: stunnel not found and Homebrew is unavailable. Install it with your package manager (apt install stunnel4)$(NC)" && exit 1)
@$(ECHO) "$(GREEN)stunnel is ready$(NC)"
dev: install-ui install-air setup-workspace $(if $(DEBUG),install-delve) ## Start complete development environment (UI + API with proxy)
@$(EXPOSE_ENV); \
set +m; \
ui_pid=""; \
api_pid=""; \
cleanup() { \
$(ECHO) "$(YELLOW)[make dev] cleanup started; ui_pid=$$ui_pid api_pid=$$api_pid$(NC)"; \
trap - EXIT INT TERM HUP; \
for pid in "$$ui_pid" "$$api_pid"; do \
if [ -n "$$pid" ]; then \
children="$$(pgrep -P "$$pid" 2>/dev/null || true)"; \
$(ECHO) "$(YELLOW)[make dev] sending TERM to pid $$pid and children: $${children:-none}$(NC)"; \
kill -TERM $$children "$$pid" 2>/dev/null || true; \
fi; \
done; \
sleep 1; \
for pid in "$$ui_pid" "$$api_pid"; do \
if [ -n "$$pid" ]; then \
children="$$(pgrep -P "$$pid" 2>/dev/null || true)"; \
$(ECHO) "$(YELLOW)[make dev] sending KILL to pid $$pid and remaining children: $${children:-none}$(NC)"; \
kill -KILL $$children "$$pid" 2>/dev/null || true; \
fi; \
done; \
$(ECHO) "$(YELLOW)[make dev] waiting for background jobs to exit...$(NC)"; \
wait 2>/dev/null || true; \
$(ECHO) "$(GREEN)[make dev] cleanup completed.$(NC)"; \
}; \
stop_dev() { \
$(ECHO) "$(YELLOW)[make dev] received shutdown signal; starting cleanup...$(NC)"; \
cleanup; \
exit 130; \
}; \
trap cleanup EXIT; \
trap stop_dev INT TERM HUP; \
$(ECHO) "$(GREEN)Starting Bifrost complete development environment...$(NC)"; \
$(ECHO) "$(YELLOW)This will start:$(NC)"; \
$(ECHO) " 1. UI development server (localhost:3000)"; \
$(ECHO) " 2. API server with UI proxy (localhost:$(PORT))"; \
$(ECHO) "$(CYAN)Access everything at: http://localhost:$(PORT)$(NC)"; \
if [ -n "$(DEBUG)" ]; then \
$(ECHO) "$(CYAN) 3. Debugger (delve) listening on port 2345$(NC)"; \
fi; \
if [ ! -d "transports/bifrost-http/ui" ]; then \
$(ECHO) "$(YELLOW)Creating transports/bifrost-http/ui directory...$(NC)"; \
mkdir -p transports/bifrost-http/ui; \
touch transports/bifrost-http/ui/.tmp; \
fi; \
$(ECHO) ""; \
$(ECHO) "$(YELLOW)Starting UI development server...$(NC)"; \
$(USE_NODE); if [ -n "$(DISABLE_PROFILER)" ]; then \
$(ECHO) "$(CYAN)DevProfiler disabled for testing$(NC)"; \
(cd ui && BIFROST_PORT=$(PORT) BIFROST_DISABLE_PROFILER=1 npm run dev) & \
else \
(cd ui && BIFROST_PORT=$(PORT) npm run dev) & \
fi; \
ui_pid="$$!"; \
$(ECHO) "$(YELLOW)[make dev] UI dev server started with pid $$ui_pid$(NC)"; \
sleep 3; \
$(ECHO) "$(YELLOW)Starting API server with UI proxy...$(NC)"; \
$(MAKE) setup-workspace >/dev/null; \
if [ -n "$(DEBUG)" ]; then \
$(ECHO) "$(CYAN)Starting with air + delve debugger on port 2345...$(NC)"; \
$(ECHO) "$(YELLOW)Attach your debugger to localhost:2345$(NC)"; \
(cd transports/bifrost-http && BIFROST_UI_DEV=true air -c .air.debug.toml -- \
-host "$(HOST)" \
-port "$(PORT)" \
-log-style "$(LOG_STYLE)" \
-log-level "$(LOG_LEVEL)" \
$(if $(PROMETHEUS_LABELS),-prometheus-labels "$(PROMETHEUS_LABELS)") \
$(if $(BASE_PATH),-base-path "$(BASE_PATH)") \
$(if $(APP_DIR),-app-dir "$(abspath $(APP_DIR))")) & \
else \
(cd transports/bifrost-http && BIFROST_UI_DEV=true air -c .air.toml -- \
-host "$(HOST)" \
-port "$(PORT)" \
-log-style "$(LOG_STYLE)" \
-log-level "$(LOG_LEVEL)" \
$(if $(PROMETHEUS_LABELS),-prometheus-labels "$(PROMETHEUS_LABELS)") \
$(if $(BASE_PATH),-base-path "$(BASE_PATH)") \
$(if $(APP_DIR),-app-dir "$(abspath $(APP_DIR))")) & \
fi; \
api_pid="$$!"; \
$(ECHO) "$(YELLOW)[make dev] API dev server started with pid $$api_pid$(NC)"; \
while kill -0 "$$ui_pid" 2>/dev/null && kill -0 "$$api_pid" 2>/dev/null; do sleep 1; done; \
$(ECHO) "$(YELLOW)[make dev] one of the dev processes exited; running cleanup...$(NC)"; \
cleanup; \
exit 1
dev-pulse: install-ui install-pulse setup-workspace $(if $(DEBUG),install-delve) ## Start complete development environment using pulse for hot reloading
@$(EXPOSE_ENV); \
set +m; \
ui_pid=""; \
pulse_pid=""; \
cleanup() { \
$(ECHO) "$(YELLOW)[make dev-pulse] cleanup started; ui_pid=$$ui_pid pulse_pid=$$pulse_pid$(NC)"; \
trap - EXIT INT TERM HUP; \
for pid in "$$ui_pid" "$$pulse_pid"; do \
if [ -n "$$pid" ]; then \
children="$$(pgrep -P "$$pid" 2>/dev/null || true)"; \
$(ECHO) "$(YELLOW)[make dev-pulse] sending TERM to pid $$pid and children: $${children:-none}$(NC)"; \
kill -TERM $$children "$$pid" 2>/dev/null || true; \
fi; \
done; \
sleep 1; \
for pid in "$$ui_pid" "$$pulse_pid"; do \
if [ -n "$$pid" ]; then \
children="$$(pgrep -P "$$pid" 2>/dev/null || true)"; \
$(ECHO) "$(YELLOW)[make dev-pulse] sending KILL to pid $$pid and remaining children: $${children:-none}$(NC)"; \
kill -KILL $$children "$$pid" 2>/dev/null || true; \
fi; \
done; \
$(ECHO) "$(YELLOW)[make dev-pulse] waiting for background jobs to exit...$(NC)"; \
wait 2>/dev/null || true; \
$(ECHO) "$(GREEN)[make dev-pulse] cleanup completed.$(NC)"; \
}; \
stop_dev() { \
$(ECHO) "$(YELLOW)[make dev-pulse] received shutdown signal; starting cleanup...$(NC)"; \
cleanup; \
exit 130; \
}; \
trap cleanup EXIT; \
trap stop_dev INT TERM HUP; \
$(ECHO) "$(GREEN)Starting Bifrost complete development environment (pulse)...$(NC)"; \
$(ECHO) "$(YELLOW)This will start:$(NC)"; \
$(ECHO) " 1. UI development server (localhost:3000)"; \
$(ECHO) " 2. API server with UI proxy (localhost:$(PORT))"; \
$(ECHO) "$(CYAN)Access everything at: http://localhost:$(PORT)$(NC)"; \
if [ -n "$(DEBUG)" ]; then \
$(ECHO) "$(CYAN) 3. Debugger (delve) listening on port 2345$(NC)"; \
fi; \
if [ ! -d "transports/bifrost-http/ui" ]; then \
$(ECHO) "$(YELLOW)Creating transports/bifrost-http/ui directory...$(NC)"; \
mkdir -p transports/bifrost-http/ui; \
touch transports/bifrost-http/ui/.tmp; \
fi; \
$(ECHO) ""; \
$(ECHO) "$(YELLOW)Starting UI development server...$(NC)"; \
$(USE_NODE); if [ -n "$(DISABLE_PROFILER)" ]; then \
$(ECHO) "$(CYAN)DevProfiler disabled for testing$(NC)"; \
(cd ui && BIFROST_PORT=$(PORT) BIFROST_DISABLE_PROFILER=1 npm run dev) & \
else \
(cd ui && BIFROST_PORT=$(PORT) npm run dev) & \
fi; \
ui_pid="$$!"; \
$(ECHO) "$(YELLOW)[make dev-pulse] UI dev server started with pid $$ui_pid$(NC)"; \
sleep 3; \
$(ECHO) "$(YELLOW)Starting API server with UI proxy...$(NC)"; \
$(MAKE) setup-workspace >/dev/null; \
if [ -n "$(DEBUG)" ]; then \
$(ECHO) "$(CYAN)Starting with pulse + delve debugger on port 2345...$(NC)"; \
$(ECHO) "$(YELLOW)Attach your debugger to localhost:2345$(NC)"; \
PORT="$(PORT)" HOST="$(HOST)" LOG_STYLE="$(LOG_STYLE)" LOG_LEVEL="$(LOG_LEVEL)" BIFROST_UI_DEV=true \
$(if $(APP_DIR),APP_DIR="$(abspath $(APP_DIR))") pulse & \
else \
PORT="$(PORT)" HOST="$(HOST)" LOG_STYLE="$(LOG_STYLE)" LOG_LEVEL="$(LOG_LEVEL)" BIFROST_UI_DEV=true \
$(if $(APP_DIR),APP_DIR="$(abspath $(APP_DIR))") pulse & \
fi; \
pulse_pid="$$!"; \
$(ECHO) "$(YELLOW)[make dev-pulse] pulse started with pid $$pulse_pid$(NC)"; \
while kill -0 "$$ui_pid" 2>/dev/null && kill -0 "$$pulse_pid" 2>/dev/null; do sleep 1; done; \
$(ECHO) "$(YELLOW)[make dev-pulse] one of the dev processes exited; running cleanup...$(NC)"; \
cleanup; \
exit 1
build-ui: install-ui ## Build ui
@$(ECHO) "$(GREEN)Building ui...$(NC)"
@rm -rf ui/.next
@$(USE_NODE); cd ui && npm run build && npm run copy-build
build: build-ui ## Build bifrost-http binary (BUILD_TAGS adds Go build tags; the MCP harness passes harness)
@if [ -n "$(LOCAL)" ]; then \
$(ECHO) "$(GREEN)╔═══════════════════════════════════════════════╗$(NC)"; \
$(ECHO) "$(GREEN)║ Building bifrost-http with local go.work... ║$(NC)"; \
$(ECHO) "$(GREEN)╚═══════════════════════════════════════════════╝$(NC)"; \
else \
$(ECHO) "$(GREEN)╔═══════════════════════════════════════╗$(NC)"; \
$(ECHO) "$(GREEN)║ Building bifrost-http... ║$(NC)"; \
$(ECHO) "$(GREEN)╚═══════════════════════════════════════╝$(NC)"; \
fi
@if [ -n "$(DYNAMIC)" ]; then \
$(ECHO) "$(YELLOW)Note: This will create a dynamically linked build.$(NC)"; \
else \
$(ECHO) "$(YELLOW)Note: This will create a statically linked build.$(NC)"; \
fi
@mkdir -p ./tmp
@# set -e: this recipe is ONE shell, and its branches chain with `;`, so a
@# failed `go build` fell through to the success echo below and left the
@# recipe exiting 0 - make reported a green build for a binary that was
@# never produced. Only the caller's own "binary not found" guard caught
@# it, several steps later and with a misleading message.
@set -e; \
TARGET_OS="$(GOOS)"; \
TARGET_ARCH="$(GOARCH)"; \
ACTUAL_OS=$$(uname -s | tr '[:upper:]' '[:lower:]' | sed 's/darwin/darwin/;s/linux/linux/;s/mingw.*/windows/'); \
ACTUAL_ARCH=$$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/;s/arm64/arm64/'); \
if [ -z "$$TARGET_OS" ]; then \
TARGET_OS=$$ACTUAL_OS; \
fi; \
if [ -z "$$TARGET_ARCH" ]; then \
TARGET_ARCH=$$ACTUAL_ARCH; \
fi; \
HOST_OS=$$ACTUAL_OS; \
HOST_ARCH=$$ACTUAL_ARCH; \
$(ECHO) "$(CYAN)Host: $$HOST_OS/$$HOST_ARCH | Target: $$TARGET_OS/$$TARGET_ARCH$(NC)"; \
if [ "$$TARGET_OS" = "linux" ] && [ "$$HOST_OS" = "linux" ]; then \
if [ -n "$(DYNAMIC)" ]; then \
$(ECHO) "$(CYAN)Building for $$TARGET_OS/$$TARGET_ARCH with dynamic linking...$(NC)"; \
cd transports/bifrost-http && CGO_ENABLED=1 GOOS=$$TARGET_OS GOARCH=$$TARGET_ARCH $(if $(LOCAL),,GOWORK=off) go build \
-ldflags="-w -s -X main.Version=v$(VERSION)" \
-a -trimpath \
$(if $(BUILD_TAGS),-tags "$(BUILD_TAGS)") \
-o ../../tmp/bifrost-http \
.; \
else \
$(ECHO) "$(CYAN)Building for $$TARGET_OS/$$TARGET_ARCH with static linking...$(NC)"; \
cd transports/bifrost-http && CGO_ENABLED=1 GOOS=$$TARGET_OS GOARCH=$$TARGET_ARCH $(if $(LOCAL),,GOWORK=off) go build \
-ldflags="-w -s -extldflags "-static" -X main.Version=v$(VERSION)" \
-a -trimpath \
-tags "sqlite_static $(BUILD_TAGS)" \
-o ../../tmp/bifrost-http \
.; \
fi; \
$(ECHO) "$(GREEN)Built: tmp/bifrost-http (version: v$(VERSION))$(NC)"; \
elif [ "$$TARGET_OS" = "$$HOST_OS" ] && [ "$$TARGET_ARCH" = "$$HOST_ARCH" ]; then \
$(ECHO) "$(CYAN)Building for $$TARGET_OS/$$TARGET_ARCH (native build with CGO)...$(NC)"; \
cd transports/bifrost-http && CGO_ENABLED=1 GOOS=$$TARGET_OS GOARCH=$$TARGET_ARCH $(if $(LOCAL),,GOWORK=off) go build \
-ldflags="-w -s -X main.Version=v$(VERSION)" \
-a -trimpath \
-tags "sqlite_static $(BUILD_TAGS)" \
-o ../../tmp/bifrost-http \
.; \
$(ECHO) "$(GREEN)Built: tmp/bifrost-http (version: v$(VERSION))$(NC)"; \
else \
$(ECHO) "$(YELLOW)Cross-compilation detected: $$HOST_OS/$$HOST_ARCH -> $$TARGET_OS/$$TARGET_ARCH$(NC)"; \
$(ECHO) "$(CYAN)Using Docker for cross-compilation...$(NC)"; \
$(MAKE) _build-with-docker TARGET_OS=$$TARGET_OS TARGET_ARCH=$$TARGET_ARCH $(if $(DYNAMIC),DYNAMIC=$(DYNAMIC)); \
fi
build-cli: ## Build bifrost CLI binary
@$(ECHO) "$(GREEN)Building bifrost CLI...$(NC)"
@mkdir -p ./tmp
@cd cli && $(if $(LOCAL),,GOWORK=off) go build -ldflags "-X main.version=v0.1.1-dev" -o ../tmp/bifrost .
@$(ECHO) "$(GREEN)Built: tmp/bifrost$(NC)"
_build-with-docker: # Internal target for Docker-based cross-compilation
@# set -e for the same reason as build-http above: without it a failed
@# `docker run` still reaches the "Built:" echo and returns 0.
@set -e; \
$(ECHO) "$(CYAN)Using Docker for cross-compilation...$(NC)"; \
if [ "$(TARGET_OS)" = "linux" ]; then \
if [ -n "$(DYNAMIC)" ]; then \
$(ECHO) "$(CYAN)Building for $(TARGET_OS)/$(TARGET_ARCH) in Docker container with dynamic linking...$(NC)"; \
docker run --rm \
--platform linux/$(TARGET_ARCH) \
-v "$(shell pwd):/workspace" \
-w /workspace/transports/bifrost-http \
-e CGO_ENABLED=1 \
-e GOOS=$(TARGET_OS) \
-e GOARCH=$(TARGET_ARCH) \
$(if $(LOCAL),,-e GOWORK=off) \
golang:1.27.3-alpine3.24@sha256:f2c872bcee949b7be9c490efd02b48a8a2fbf947cdf533f1c58525ec9d8170fd \
sh -c "apk add --no-cache gcc musl-dev && \
go build \
-ldflags='-w -s -X main.Version=v$(VERSION)' \
-a -trimpath \
$(if $(BUILD_TAGS),-tags \"$(BUILD_TAGS)\") \
-o ../../tmp/bifrost-http \
."; \
else \
$(ECHO) "$(CYAN)Building for $(TARGET_OS)/$(TARGET_ARCH) in Docker container...$(NC)"; \
docker run --rm \
--platform linux/$(TARGET_ARCH) \
-v "$(shell pwd):/workspace" \
-w /workspace/transports/bifrost-http \
-e CGO_ENABLED=1 \
-e GOOS=$(TARGET_OS) \
-e GOARCH=$(TARGET_ARCH) \
$(if $(LOCAL),,-e GOWORK=off) \
golang:1.27.3-alpine3.24@sha256:f2c872bcee949b7be9c490efd02b48a8a2fbf947cdf533f1c58525ec9d8170fd \
sh -c "apk add --no-cache gcc musl-dev && \
go build \
-ldflags='-w -s -extldflags "-static" -X main.Version=v$(VERSION)' \
-a -trimpath \
-tags \"sqlite_static $(BUILD_TAGS)\" \
-o ../../tmp/bifrost-http \
."; \
fi; \
$(ECHO) "$(GREEN)Built: tmp/bifrost-http ($(TARGET_OS)/$(TARGET_ARCH), version: v$(VERSION))$(NC)"; \
else \
$(ECHO) "$(RED)Error: Docker cross-compilation only supports Linux targets$(NC)"; \
$(ECHO) "$(YELLOW)For $(TARGET_OS), please build on a native $(TARGET_OS) machine$(NC)"; \
exit 1; \
fi
docker-image: build-ui ## Build Docker image (LOCAL=1 to use Dockerfile.local)
@$(ECHO) "$(GREEN)Building Docker image...$(NC)"
$(eval GIT_SHA=$(shell git rev-parse --short HEAD))
$(eval DOCKERFILE=$(if $(LOCAL),transports/Dockerfile.local,transports/Dockerfile))
@docker build -f $(DOCKERFILE) -t bifrost -t bifrost:$(GIT_SHA) -t bifrost:latest .
@$(ECHO) "$(GREEN)Docker image built: bifrost, bifrost:$(GIT_SHA), bifrost:latest (using $(DOCKERFILE))$(NC)"
docker-run: ## Run Docker container (Usage: make docker-run [CONFIG=path/to/config.json or path/to/dir/])
@$(ECHO) "$(GREEN)Running Docker container...$(NC)"
@CONFIG_PATH="$(abspath $(CONFIG))"; \
if [ -n "$(CONFIG)" ]; then \
if [ -d "$$CONFIG_PATH" ]; then \
CONFIG_PATH="$$CONFIG_PATH/config.json"; \
fi; \
CONFIG_MOUNT="-v $$CONFIG_PATH:/app/data/config.json"; \
else \
CONFIG_MOUNT=""; \
fi; \
docker run -e APP_PORT=$(PORT) -e APP_HOST=0.0.0.0 -p $(PORT):$(PORT) -e LOG_LEVEL=$(LOG_LEVEL) -e LOG_STYLE=$(LOG_STYLE) -v $(shell pwd):/app/data $$CONFIG_MOUNT bifrost
docs: ## Prepare local docs (bundles OpenAPI spec then starts Mintlify dev server)
@$(ECHO) "$(GREEN)Bundling OpenAPI spec...$(NC)"
@cd docs/openapi && python3 bundle.py
@$(ECHO) "$(GREEN)Preparing local docs...$(NC)"
@cd docs && npx --yes mintlify@latest dev
run: build ## Build and run bifrost-http (no hot reload)
@$(ECHO) "$(GREEN)Running bifrost-http...$(NC)"
@./tmp/bifrost-http \
-host "$(HOST)" \
-port "$(PORT)" \
-log-style "$(LOG_STYLE)" \
-log-level "$(LOG_LEVEL)" \
$(if $(PROMETHEUS_LABELS),-prometheus-labels "$(PROMETHEUS_LABELS)") \
$(if $(BASE_PATH),-base-path "$(BASE_PATH)") \
$(if $(APP_DIR),-app-dir "$(abspath $(APP_DIR))")
run-cli: build-cli ## Run bifrost CLI (Usage: make run-cli [ARGS="--config ~/.bifrost/config.json"])
@$(ECHO) "$(GREEN)Running bifrost CLI...$(NC)"
@./tmp/bifrost $(ARGS)
clean: ## Clean build artifacts and temporary files
@$(ECHO) "$(YELLOW)Cleaning build artifacts...$(NC)"
@rm -rf tmp/
@rm -f transports/bifrost-http/build-errors.log
@rm -rf transports/bifrost-http/tmp/
@rm -rf $(TEST_REPORTS_DIR)/
@$(ECHO) "$(GREEN)Clean complete$(NC)"
clean-test-reports: ## Clean test reports only
@$(ECHO) "$(YELLOW)Cleaning test reports...$(NC)"
@rm -rf $(TEST_REPORTS_DIR)/
@$(ECHO) "$(GREEN)Test reports cleaned$(NC)"
helm-index: ## Repackage helm chart, regenerate index.yaml digest, then remove the .tgz
@if ! which helm > /dev/null 2>&1; then \
$(ECHO) "$(RED)Error: helm not installed$(NC)"; \
exit 1; \
fi
@CHART_VERSION=$$(grep '^version:' helm-charts/bifrost/Chart.yaml | awk '{print $$2}'); \
$(ECHO) "$(YELLOW)Packaging helm chart v$$CHART_VERSION...$(NC)"; \
cd helm-charts && \
helm package bifrost && \
$(ECHO) "$(YELLOW)Regenerating index.yaml digest...$(NC)" && \
if [ -f index.yaml ]; then \
helm repo index . --url https://github.com/maximhq/bifrost/releases/download/helm-chart-v$$CHART_VERSION --merge index.yaml; \
else \
helm repo index . --url https://github.com/maximhq/bifrost/releases/download/helm-chart-v$$CHART_VERSION; \
fi && \
$(ECHO) "$(YELLOW)Removing bifrost-$$CHART_VERSION.tgz...$(NC)" && \
rm -f bifrost-$$CHART_VERSION.tgz && \
$(ECHO) "$(GREEN)Helm index updated$(NC)"
generate-html-reports: ## Convert existing XML reports to HTML
@if ! which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(RED)Error: junit-viewer not installed$(NC)"; \
$(ECHO) "$(YELLOW)Install with: make install-junit-viewer$(NC)"; \
exit 1; \
fi
@$(ECHO) "$(GREEN)Converting XML reports to HTML...$(NC)"
@if [ ! -d "$(TEST_REPORTS_DIR)" ] || [ -z "$$(ls -A $(TEST_REPORTS_DIR)/*.xml 2>/dev/null)" ]; then \
$(ECHO) "$(YELLOW)No XML reports found in $(TEST_REPORTS_DIR)$(NC)"; \
$(ECHO) "$(YELLOW)Run tests first: make test-all$(NC)"; \
exit 0; \
fi
@for xml in $(TEST_REPORTS_DIR)/*.xml; do \
html=$${xml%.xml}.html; \
$(ECHO) " Converting $$(basename $$xml) → $$(basename $$html)"; \
junit-viewer --results=$$xml --save=$$html 2>/dev/null || true; \
done
@$(ECHO) ""
@$(ECHO) "$(GREEN)✓ HTML reports generated$(NC)"
@$(ECHO) "$(CYAN)View reports:$(NC)"
@ls -1 $(TEST_REPORTS_DIR)/*.html 2>/dev/null | sed 's|$(TEST_REPORTS_DIR)/| open $(TEST_REPORTS_DIR)/|' || true
test: install-gotestsum ## Run tests for bifrost-http
@$(ECHO) "$(GREEN)Running bifrost-http tests...$(NC)"
@mkdir -p $(TEST_REPORTS_DIR)
@cd transports/bifrost-http && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../$(TEST_REPORTS_DIR)/bifrost-http.xml \
-- -v ./...
@if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report...$(NC)"; \
if junit-viewer --results=$(TEST_REPORTS_DIR)/bifrost-http.xml --save=$(TEST_REPORTS_DIR)/bifrost-http.html 2>/dev/null; then \
$(ECHO) ""; \
$(ECHO) "$(CYAN)HTML report: $(TEST_REPORTS_DIR)/bifrost-http.html$(NC)"; \
$(ECHO) "$(CYAN)Open with: open $(TEST_REPORTS_DIR)/bifrost-http.html$(NC)"; \
else \
$(ECHO) "$(YELLOW)HTML generation failed. JUnit XML report available.$(NC)"; \
$(ECHO) "$(CYAN)JUnit XML report: $(TEST_REPORTS_DIR)/bifrost-http.xml$(NC)"; \
fi; \
else \
$(ECHO) ""; \
$(ECHO) "$(YELLOW)junit-viewer not installed. Install with: make install-junit-viewer$(NC)"; \
$(ECHO) "$(CYAN)JUnit XML report: $(TEST_REPORTS_DIR)/bifrost-http.xml$(NC)"; \
fi; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $(TEST_REPORTS_DIR)/bifrost-http.xml$(NC)"; \
fi
test-core: install-gotestsum $(if $(DEBUG),install-delve) ## Run core tests (Usage: make test-core PROVIDER=openai TESTCASE=TestName or PATTERN=substring, DEBUG=1 for debugger)
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running core tests...$(NC)"; \
mkdir -p $(TEST_REPORTS_DIR); \
if [ -n "$(PATTERN)" ] && [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(RED)Error: PATTERN and TESTCASE are mutually exclusive$(NC)"; \
$(ECHO) "$(YELLOW)Use PATTERN for substring matching or TESTCASE for exact match$(NC)"; \
exit 1; \
fi; \
TEST_FAILED=0; \
REPORT_FILE=""; \
if [ -n "$(PROVIDER)" ]; then \
$(ECHO) "$(CYAN)Running tests for provider: $(PROVIDER)$(NC)"; \
if [ ! -f "core/providers/$(PROVIDER)/$(PROVIDER)_test.go" ]; then \
$(ECHO) "$(RED)Error: Provider test file '$(PROVIDER)_test.go' not found in core/providers/$(PROVIDER)/$(NC)"; \
$(ECHO) "$(YELLOW)Available providers:$(NC)"; \
find core/providers -name "*_test.go" -type f 2>/dev/null | sed 's|core/providers/\([^/]*\)/.*|\1|' | sort -u | sed 's/^/ - /'; \
exit 1; \
fi; \
fi; \
if [ -n "$(DEBUG)" ]; then \
$(ECHO) "$(CYAN)Debug mode enabled - delve debugger will listen on port 2345$(NC)"; \
$(ECHO) "$(YELLOW)Attach your debugger to localhost:2345$(NC)"; \
fi; \
if [ -n "$(PROVIDER)" ]; then \
PROVIDER_TEST_NAME=$$($(ECHO) "$(PROVIDER)" | awk '{print toupper(substr($$0,1,1)) tolower(substr($$0,2))}' | sed 's/openai/OpenAI/i; s/openrouter/OpenRouter/i; s/sgl/SGL/i; s/xai/XAI/i; s/vllm/VLLM/i; s/githubcopilot/GithubCopilot/i; s/sagemaker/SageMaker/i'); \
if [ -n "$(TESTCASE)" ]; then \
CLEAN_TESTCASE="$(TESTCASE)"; \
CLEAN_TESTCASE=$${CLEAN_TESTCASE#Test$${PROVIDER_TEST_NAME}/}; \
CLEAN_TESTCASE=$${CLEAN_TESTCASE#$${PROVIDER_TEST_NAME}Tests/}; \
CLEAN_TESTCASE=$$($(ECHO) "$$CLEAN_TESTCASE" | sed 's|^Test[A-Z][A-Za-z]*/[A-Z][A-Za-z]*Tests/||'); \
$(ECHO) "$(CYAN)Running Test$${PROVIDER_TEST_NAME}/$${PROVIDER_TEST_NAME}Tests/$$CLEAN_TESTCASE...$(NC)"; \
REPORT_FILE="$(TEST_REPORTS_DIR)/core-$(PROVIDER)-$$(echo $$CLEAN_TESTCASE | sed 's|/|_|g').xml"; \
if [ -n "$(DEBUG)" ]; then \
cd core/providers/$(PROVIDER) && GOWORK=off dlv test --headless --listen=:2345 --api-version=2 -- -test.v -test.run "^Test$${PROVIDER_TEST_NAME}$$/.*Tests/$$CLEAN_TESTCASE$$" || TEST_FAILED=1; \
else \
cd core/providers/$(PROVIDER) && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../../$$REPORT_FILE \
-- -v -timeout 20m -run "^Test$${PROVIDER_TEST_NAME}$$/.*Tests/$$CLEAN_TESTCASE$$" || TEST_FAILED=1; \
fi; \
cd ../../..; \
$(MAKE) cleanup-junit-xml REPORT_FILE=$$REPORT_FILE; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report...$(NC)"; \
junit-viewer --results=$$REPORT_FILE --save=$${REPORT_FILE%.xml}.html 2>/dev/null || true; \
$(ECHO) ""; \
$(ECHO) "$(CYAN)HTML report: $${REPORT_FILE%.xml}.html$(NC)"; \
$(ECHO) "$(CYAN)Open with: open $${REPORT_FILE%.xml}.html$(NC)"; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
elif [ -n "$(PATTERN)" ]; then \
$(ECHO) "$(CYAN)Running tests matching '$(PATTERN)' for $${PROVIDER_TEST_NAME}...$(NC)"; \
REPORT_FILE="$(TEST_REPORTS_DIR)/core-$(PROVIDER)-$(PATTERN).xml"; \
if [ -n "$(DEBUG)" ]; then \
cd core/providers/$(PROVIDER) && GOWORK=off dlv test --headless --listen=:2345 --api-version=2 -- -test.v -test.run ".*$(PATTERN).*" || TEST_FAILED=1; \
else \
cd core/providers/$(PROVIDER) && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../../$$REPORT_FILE \
-- -v -timeout 20m -run ".*$(PATTERN).*" || TEST_FAILED=1; \
fi; \
cd ../../..; \
$(MAKE) cleanup-junit-xml REPORT_FILE=$$REPORT_FILE; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report...$(NC)"; \
junit-viewer --results=$$REPORT_FILE --save=$${REPORT_FILE%.xml}.html 2>/dev/null || true; \
$(ECHO) ""; \
$(ECHO) "$(CYAN)HTML report: $${REPORT_FILE%.xml}.html$(NC)"; \
$(ECHO) "$(CYAN)Open with: open $${REPORT_FILE%.xml}.html$(NC)"; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
$(ECHO) "$(CYAN)Running Test$${PROVIDER_TEST_NAME}...$(NC)"; \
REPORT_FILE="$(TEST_REPORTS_DIR)/core-$(PROVIDER).xml"; \
if [ -n "$(DEBUG)" ]; then \
cd core/providers/$(PROVIDER) && GOWORK=off dlv test --headless --listen=:2345 --api-version=2 -- -test.v -test.run "^Test$${PROVIDER_TEST_NAME}$$" || TEST_FAILED=1; \
else \
cd core/providers/$(PROVIDER) && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../../$$REPORT_FILE \
-- -v -timeout 20m -run "^Test$${PROVIDER_TEST_NAME}$$" || TEST_FAILED=1; \
fi; \
cd ../../..; \
$(MAKE) cleanup-junit-xml REPORT_FILE=$$REPORT_FILE; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report...$(NC)"; \
junit-viewer --results=$$REPORT_FILE --save=$${REPORT_FILE%.xml}.html 2>/dev/null || true; \
$(ECHO) ""; \
$(ECHO) "$(CYAN)HTML report: $${REPORT_FILE%.xml}.html$(NC)"; \
$(ECHO) "$(CYAN)Open with: open $${REPORT_FILE%.xml}.html$(NC)"; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
fi; \
else \
if [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(RED)Error: TESTCASE requires PROVIDER to be specified$(NC)"; \
$(ECHO) "$(YELLOW)Usage: make test-core PROVIDER=openai TESTCASE=SpeechSynthesisStreamAdvanced/MultipleVoices_Streaming/StreamingVoice_echo$(NC)"; \
exit 1; \
fi; \
if [ -n "$(PATTERN)" ]; then \
$(ECHO) "$(CYAN)Running tests matching '$(PATTERN)' across core and all providers...$(NC)"; \
REPORT_FILE="$(TEST_REPORTS_DIR)/core-all-$(PATTERN).xml"; \
if [ -n "$(DEBUG)" ]; then \
cd core && GOWORK=off dlv test --headless --listen=:2345 --api-version=2 . ./providers/... -- -test.v -test.run ".*$(PATTERN).*" || TEST_FAILED=1; \
else \
cd core && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../$$REPORT_FILE \
-- -v -timeout 20m -run ".*$(PATTERN).*" . ./providers/... || TEST_FAILED=1; \
fi; \
else \
REPORT_FILE="$(TEST_REPORTS_DIR)/core-all.xml"; \
if [ -n "$(DEBUG)" ]; then \
cd core && GOWORK=off dlv test --headless --listen=:2345 --api-version=2 ./providers/... -- -test.v || TEST_FAILED=1; \
else \
cd core && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../$$REPORT_FILE \
-- -v ./providers/... || TEST_FAILED=1; \
fi; \
fi; \
cd ..; \
$(MAKE) cleanup-junit-xml REPORT_FILE=$$REPORT_FILE; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report...$(NC)"; \
junit-viewer --results=$$REPORT_FILE --save=$${REPORT_FILE%.xml}.html 2>/dev/null || true; \
$(ECHO) ""; \
$(ECHO) "$(CYAN)HTML report: $${REPORT_FILE%.xml}.html$(NC)"; \
$(ECHO) "$(CYAN)Open with: open $${REPORT_FILE%.xml}.html$(NC)"; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
fi; \
if [ -f "$$REPORT_FILE" ]; then \
ALL_FAILED=$$(grep -B 1 '<failure' "$$REPORT_FILE" 2>/dev/null | \
grep '<testcase' | \
sed 's/.*name="\([^"]*\)".*/\1/' | \
sort -u); \
MAX_DEPTH=$$($(ECHO) "$$ALL_FAILED" | awk -F'/' '{print NF}' | sort -n | tail -1); \
FAILED_TESTS=$$($(ECHO) "$$ALL_FAILED" | awk -F'/' -v max="$$MAX_DEPTH" 'NF == max'); \
FAILURES=$$($(ECHO) "$$FAILED_TESTS" | grep -v '^$$' | wc -l | tr -d ' '); \
if [ "$$FAILURES" -gt 0 ]; then \
$(ECHO) ""; \
$(ECHO) "$(RED)═══════════════════════════════════════════════════════════$(NC)"; \
$(ECHO) "$(RED) FAILED TEST CASES $(NC)"; \
$(ECHO) "$(RED)═══════════════════════════════════════════════════════════$(NC)"; \
$(ECHO) ""; \
printf "$(YELLOW)%-60s %-20s$(NC)\n" "Test Name" "Status"; \
printf "$(YELLOW)%-60s %-20s$(NC)\n" "─────────────────────────────────────────────────────────────" "────────────────────"; \
$(ECHO) "$$FAILED_TESTS" | while read -r testname; do \
if [ -n "$$testname" ]; then \
printf "$(RED)%-60s %-20s$(NC)\n" "$$testname" "FAILED"; \
fi; \
done; \
$(ECHO) ""; \
$(ECHO) "$(RED)Total Failures: $$FAILURES$(NC)"; \
$(ECHO) ""; \
else \
$(ECHO) ""; \
$(ECHO) "$(GREEN)═══════════════════════════════════════════════════════════$(NC)"; \
$(ECHO) "$(GREEN) ALL TESTS PASSED ✓ $(NC)"; \
$(ECHO) "$(GREEN)═══════════════════════════════════════════════════════════$(NC)"; \
$(ECHO) ""; \
fi; \
fi; \
if [ -n "$$REPORT_FILE" ] && [ -f "$$REPORT_FILE" ]; then \
SUMMARY_PREFIX=$$(basename "$$REPORT_FILE" .xml | sed 's/-.*//'); \
SUMMARY_TITLE=$$($(ECHO) "$$SUMMARY_PREFIX" | awk '{print toupper(substr($$0,1,1)) substr($$0,2)}'); \
$(MAKE) --no-print-directory print-test-summary \
SUMMARY_LABEL="$$SUMMARY_TITLE" \
SUMMARY_STRIP="$$SUMMARY_PREFIX-" \
SUMMARY_FILES="$$REPORT_FILE"; \
fi; \
if [ $$TEST_FAILED -eq 1 ]; then \
exit 1; \
fi
cleanup-junit-xml: ## Internal: Clean up JUnit XML to remove parent test cases with child failures
@if [ -z "$(REPORT_FILE)" ]; then \
$(ECHO) "$(RED)Error: REPORT_FILE not specified$(NC)"; \
exit 1; \
fi
@if [ ! -f "$(REPORT_FILE)" ]; then \
exit 0; \
fi
@ALL_FAILED=$$(grep -B 1 '<failure' "$(REPORT_FILE)" 2>/dev/null | \
grep '<testcase' | \
sed 's/.*name="\([^"]*\)".*/\1/' | \
sort -u); \
if [ -n "$$ALL_FAILED" ]; then \
MAX_DEPTH=$$($(ECHO) "$$ALL_FAILED" | awk -F'/' '{print NF}' | sort -n | tail -1); \
PARENT_TESTS=$$($(ECHO) "$$ALL_FAILED" | awk -F'/' -v max="$$MAX_DEPTH" 'NF < max'); \
if [ -n "$$PARENT_TESTS" ]; then \
cp "$(REPORT_FILE)" "$(REPORT_FILE).tmp"; \
$(ECHO) "$$PARENT_TESTS" | while IFS= read -r parent; do \
if [ -n "$$parent" ]; then \
ESCAPED=$$($(ECHO) "$$parent" | sed 's/[\/&]/\\&/g'); \
perl -i -pe 'BEGIN{undef $$/;} s/<testcase[^>]*name="'"$$ESCAPED"'"[^>]*>.*?<failure.*?<\/testcase>//gs' "$(REPORT_FILE).tmp" 2>/dev/null || true; \
fi; \
done; \
if [ -f "$(REPORT_FILE).tmp" ]; then \
mv "$(REPORT_FILE).tmp" "$(REPORT_FILE)"; \
fi; \
fi; \
fi
test-plugins: install-gotestsum ## Run plugin tests
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running plugin tests...$(NC)"; \
mkdir -p $(TEST_REPORTS_DIR); \
cd plugins && find . -name "*.go" -path "*/tests/*" -o -name "*_test.go" | head -1 > /dev/null && \
for dir in $$(find . -name "*_test.go" -exec dirname {} \; | sort -u); do \
plugin_name=$$(echo $$dir | sed 's|^\./||' | sed 's|/|-|g'); \
$(ECHO) "Testing $$dir..."; \
cd $$dir && gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../$(TEST_REPORTS_DIR)/plugin-$$plugin_name.xml \
-- -v ./... && cd - > /dev/null; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report for $$plugin_name...$(NC)"; \
junit-viewer --results=../$(TEST_REPORTS_DIR)/plugin-$$plugin_name.xml --save=../$(TEST_REPORTS_DIR)/plugin-$$plugin_name.html 2>/dev/null || true; \
fi; \
fi; \
done || $(ECHO) "No plugin tests found"
@$(ECHO) ""
@if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
$(ECHO) "$(CYAN)HTML reports saved to $(TEST_REPORTS_DIR)/plugin-*.html$(NC)"; \
else \
$(ECHO) "$(CYAN)JUnit XML reports saved to $(TEST_REPORTS_DIR)/plugin-*.xml$(NC)"; \
fi
@$(MAKE) --no-print-directory print-test-summary \
SUMMARY_LABEL="Plugin" \
SUMMARY_STRIP="plugin-" \
SUMMARY_FILES="$(TEST_REPORTS_DIR)/plugin-*.xml"
test-memory: install-gotestsum ## Run memory-regression tests (allocation scaling + coverage gate)
@$(ECHO) "$(GREEN)Running memory regression tests...$(NC)"
@$(ECHO) "$(CYAN)Asserts allocation grows with input size, not its square, and that every$(NC)"
@$(ECHO) "$(CYAN)loop-bound raw-JSON mutation in core/providers has been classified.$(NC)"
@mkdir -p $(TEST_REPORTS_DIR)
@rc=0; \
for mod in core transports; do \
pkgs=$$(cd $$mod && grep -rl "_AllocationScaling\|_Retention" --include="*_test.go" . 2>/dev/null | xargs -n1 dirname | sort -u); \
if [ "$$mod" = "core" ]; then pkgs="./internal/memtest $$(echo "$$pkgs" | grep -v '^./internal/memtest$$')"; fi; \
[ -z "$$(echo $$pkgs)" ] && continue; \
$(ECHO) "$(CYAN)$$mod:$(NC)"; \
for p in $$pkgs; do $(ECHO) " $$p"; done; \
( cd $$mod && gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=$(CURDIR)/$(TEST_REPORTS_DIR)/memory-$$mod.xml \
-- -timeout 15m -count=1 \
-run '_AllocationScaling|_Retention|TestEveryLoopBoundJSONMutationIsReviewed|TestCoveredSitesHaveARealTest' \
$$pkgs ) || rc=1; \
done; \
exit $$rc
@$(ECHO) ""
@$(ECHO) "$(CYAN)JUnit XML report saved to $(TEST_REPORTS_DIR)/memory-*.xml$(NC)"
test-framework: install-gotestsum ## Run framework tests
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running framework tests...$(NC)"; \
mkdir -p $(TEST_REPORTS_DIR); \
rm -f $(TEST_REPORTS_DIR)/.framework-failed; \
cd framework && find . -name "*.go" -path "*/tests/*" -o -name "*_test.go" | head -1 > /dev/null && \
for dir in $$(find . -name "*_test.go" -exec dirname {} \; | sort -u); do \
pkg_name=$$(echo $$dir | sed 's|^\./||' | sed 's|/|-|g'); \
$(ECHO) "Testing $$dir..."; \
( cd $$dir && gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=$(CURDIR)/$(TEST_REPORTS_DIR)/framework-$$pkg_name.xml \
-- -v -timeout 30m ./... ) || touch $(CURDIR)/$(TEST_REPORTS_DIR)/.framework-failed; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report for $$pkg_name...$(NC)"; \
junit-viewer --results=$(CURDIR)/$(TEST_REPORTS_DIR)/framework-$$pkg_name.xml --save=$(CURDIR)/$(TEST_REPORTS_DIR)/framework-$$pkg_name.html 2>/dev/null || true; \
fi; \
fi; \
done || $(ECHO) "No framework tests found"
@$(ECHO) ""
@if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
$(ECHO) "$(CYAN)HTML reports saved to $(TEST_REPORTS_DIR)/framework-*.html$(NC)"; \
else \
$(ECHO) "$(CYAN)JUnit XML reports saved to $(TEST_REPORTS_DIR)/framework-*.xml$(NC)"; \
fi
@$(MAKE) --no-print-directory print-test-summary \
SUMMARY_LABEL="Framework" \
SUMMARY_STRIP="framework-" \
SUMMARY_FILES="$(TEST_REPORTS_DIR)/framework-*.xml"
@if [ -f $(TEST_REPORTS_DIR)/.framework-failed ]; then \
rm -f $(TEST_REPORTS_DIR)/.framework-failed; \
exit 1; \
fi
# Internal: render a table of test reports + a final pass/fail scenario.
# Usage: $(MAKE) print-test-summary SUMMARY_LABEL="Framework" SUMMARY_STRIP="framework-" SUMMARY_FILES="<glob or files>"
# Each report becomes one row: tests/failures/errors come from the <testsuites> aggregate,
# while skipped is summed from the per-<testsuite> attrs (the aggregate omits skipped).
SUMMARY_SEP := --------------------------------------------------
print-test-summary:
@$(ECHO) ""; \
$(ECHO) "$(CYAN)============================================================================$(NC)"; \
$(ECHO) "$(CYAN)$(SUMMARY_LABEL) Test Report Summary$(NC)"; \
$(ECHO) "$(CYAN)============================================================================$(NC)"; \
total_tests=0; total_pass=0; total_fail=0; total_err=0; total_skip=0; reports=0; \
printf "%-50s %7s %7s %7s %7s %7s\n" "REPORT" "TESTS" "PASS" "FAIL" "ERR" "SKIP"; \
printf "%-50s %7s %7s %7s %7s %7s\n" "$(SUMMARY_SEP)" "-------" "-------" "-------" "-------" "-------"; \
for xml in $(SUMMARY_FILES); do \
[ -e "$$xml" ] || continue; \
line=$$(grep -o '<testsuites[^>]*>' "$$xml" | head -1); \
t=$$(printf '%s' "$$line" | sed -n 's/.*[^a-z]tests="\([0-9]*\)".*/\1/p'); \
f=$$(printf '%s' "$$line" | sed -n 's/.*failures="\([0-9]*\)".*/\1/p'); \
e=$$(printf '%s' "$$line" | sed -n 's/.*errors="\([0-9]*\)".*/\1/p'); \
s=$$(grep -o '<testsuite [^>]*>' "$$xml" | grep -o 'skipped="[0-9]*"' | grep -o '[0-9]*' | awk '{x+=$$1} END{print x+0}'); \
t=$${t:-0}; f=$${f:-0}; e=$${e:-0}; s=$${s:-0}; \
p=$$((t - f - e - s)); \
name=$$(basename "$$xml" .xml | sed 's/^$(SUMMARY_STRIP)//'); \
reports=$$((reports + 1)); \
total_tests=$$((total_tests + t)); total_pass=$$((total_pass + p)); \
total_fail=$$((total_fail + f)); total_err=$$((total_err + e)); total_skip=$$((total_skip + s)); \
if [ $$((f + e)) -gt 0 ]; then color="$(RED)"; else color="$(GREEN)"; fi; \
printf "%b%-50s %7s %7s %7s %7s %7s%b\n" "$$color" "$$name" "$$t" "$$p" "$$f" "$$e" "$$s" "$(NC)"; \
done; \
printf "%-50s %7s %7s %7s %7s %7s\n" "$(SUMMARY_SEP)" "-------" "-------" "-------" "-------" "-------"; \
printf "%-50s %7s %7s %7s %7s %7s\n" "TOTAL ($$reports reports)" "$$total_tests" "$$total_pass" "$$total_fail" "$$total_err" "$$total_skip"; \
$(ECHO) "$(CYAN)============================================================================$(NC)"; \
if [ "$$reports" -eq 0 ]; then \
$(ECHO) "$(YELLOW)No $(SUMMARY_LABEL) test reports found$(NC)"; \
elif [ $$((total_fail + total_err)) -eq 0 ]; then \
$(ECHO) "$(GREEN)✓ ALL $(SUMMARY_LABEL) TESTS PASSED ($$total_pass/$$total_tests passed, $$total_skip skipped)$(NC)"; \
else \
$(ECHO) "$(RED)✗ $(SUMMARY_LABEL) TESTS FAILED ($$total_fail failures, $$total_err errors out of $$total_tests tests)$(NC)"; \
fi
test-http-transport: install-gotestsum ## Run HTTP transport tests
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running HTTP transport tests...$(NC)"; \
mkdir -p $(TEST_REPORTS_DIR); \
cd transports/bifrost-http && find . -name "*.go" -path "*/tests/*" -o -name "*_test.go" | head -1 > /dev/null && \
for dir in $$(find . -name "*_test.go" -exec dirname {} \; | sort -u); do \
pkg_name=$$(echo $$dir | sed 's|^\./||' | sed 's|/|-|g'); \
$(ECHO) "Testing $$dir..."; \
cd $$dir && gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../../$(TEST_REPORTS_DIR)/http-transport-$$pkg_name.xml \
-- -v ./... && cd - > /dev/null; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report for $$pkg_name...$(NC)"; \
junit-viewer --results=../../$(TEST_REPORTS_DIR)/http-transport-$$pkg_name.xml --save=../../$(TEST_REPORTS_DIR)/http-transport-$$pkg_name.html 2>/dev/null || true; \
fi; \
fi; \
done || $(ECHO) "No HTTP transport tests found"
@$(ECHO) ""
@if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
$(ECHO) "$(CYAN)HTML reports saved to $(TEST_REPORTS_DIR)/http-transport-*.html$(NC)"; \
else \
$(ECHO) "$(CYAN)JUnit XML reports saved to $(TEST_REPORTS_DIR)/http-transport-*.xml$(NC)"; \
fi
test-governance: install-gotestsum $(if $(DEBUG),install-delve) ## Run governance tests (Usage: make test-governance TESTCASE=TestName or PATTERN=substring, DEBUG=1 for debugger)
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running governance tests...$(NC)"; \
mkdir -p $(TEST_REPORTS_DIR); \
if [ -n "$(PATTERN)" ] && [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(RED)Error: PATTERN and TESTCASE are mutually exclusive$(NC)"; \
$(ECHO) "$(YELLOW)Use PATTERN for substring matching or TESTCASE for exact match$(NC)"; \
exit 1; \
fi; \
if [ ! -d "tests/governance" ]; then \
$(ECHO) "$(RED)Error: Governance tests directory not found$(NC)"; \
exit 1; \
fi; \
TEST_FAILED=0; \
REPORT_FILE=""; \
if [ -n "$(DEBUG)" ]; then \
$(ECHO) "$(CYAN)Debug mode enabled - delve debugger will listen on port 2345$(NC)"; \
$(ECHO) "$(YELLOW)Attach your debugger to localhost:2345$(NC)"; \
fi; \
if [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(CYAN)Running test case: $(TESTCASE)$(NC)"; \
REPORT_FILE="$(TEST_REPORTS_DIR)/governance-$$(echo $(TESTCASE) | sed 's|/|_|g').xml"; \
if [ -n "$(DEBUG)" ]; then \
cd tests/governance && GOWORK=off dlv test --headless --listen=:2345 --api-version=2 -- -test.v -test.run "^$(TESTCASE)$$" || TEST_FAILED=1; \
else \
cd tests/governance && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../$$REPORT_FILE \
-- -v -run "^$(TESTCASE)$$" || TEST_FAILED=1; \
fi; \
cd ../..; \
$(MAKE) cleanup-junit-xml REPORT_FILE=$$REPORT_FILE; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report...$(NC)"; \
junit-viewer --results=$$REPORT_FILE --save=$${REPORT_FILE%.xml}.html 2>/dev/null || true; \
$(ECHO) ""; \
$(ECHO) "$(CYAN)HTML report: $${REPORT_FILE%.xml}.html$(NC)"; \
$(ECHO) "$(CYAN)Open with: open $${REPORT_FILE%.xml}.html$(NC)"; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
elif [ -n "$(PATTERN)" ]; then \
$(ECHO) "$(CYAN)Running tests matching '$(PATTERN)'...$(NC)"; \
REPORT_FILE="$(TEST_REPORTS_DIR)/governance-$(PATTERN).xml"; \
if [ -n "$(DEBUG)" ]; then \
cd tests/governance && GOWORK=off dlv test --headless --listen=:2345 --api-version=2 -- -test.v -test.run ".*$(PATTERN).*" || TEST_FAILED=1; \
else \
cd tests/governance && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../$$REPORT_FILE \
-- -v -run ".*$(PATTERN).*" || TEST_FAILED=1; \
fi; \
cd ../..; \
$(MAKE) cleanup-junit-xml REPORT_FILE=$$REPORT_FILE; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report...$(NC)"; \
junit-viewer --results=$$REPORT_FILE --save=$${REPORT_FILE%.xml}.html 2>/dev/null || true; \
$(ECHO) ""; \
$(ECHO) "$(CYAN)HTML report: $${REPORT_FILE%.xml}.html$(NC)"; \
$(ECHO) "$(CYAN)Open with: open $${REPORT_FILE%.xml}.html$(NC)"; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
$(ECHO) "$(CYAN)Running all governance tests...$(NC)"; \
REPORT_FILE="$(TEST_REPORTS_DIR)/governance-all.xml"; \
if [ -n "$(DEBUG)" ]; then \
cd tests/governance && GOWORK=off dlv test --headless --listen=:2345 --api-version=2 -- -test.v || TEST_FAILED=1; \
else \
cd tests/governance && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../$$REPORT_FILE \
-- -v || TEST_FAILED=1; \
fi; \
cd ../..; \
$(MAKE) cleanup-junit-xml REPORT_FILE=$$REPORT_FILE; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report...$(NC)"; \
junit-viewer --results=$$REPORT_FILE --save=$${REPORT_FILE%.xml}.html 2>/dev/null || true; \
$(ECHO) ""; \
$(ECHO) "$(CYAN)HTML report: $${REPORT_FILE%.xml}.html$(NC)"; \
$(ECHO) "$(CYAN)Open with: open $${REPORT_FILE%.xml}.html$(NC)"; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
fi; \
if [ -f "$$REPORT_FILE" ]; then \
ALL_FAILED=$$(grep -B 1 '<failure' "$$REPORT_FILE" 2>/dev/null | \
grep '<testcase' | \
sed 's/.*name="\([^"]*\)".*/\1/' | \
sort -u); \
MAX_DEPTH=$$($(ECHO) "$$ALL_FAILED" | awk -F'/' '{print NF}' | sort -n | tail -1); \
FAILED_TESTS=$$($(ECHO) "$$ALL_FAILED" | awk -F'/' -v max="$$MAX_DEPTH" 'NF == max'); \
FAILURES=$$($(ECHO) "$$FAILED_TESTS" | grep -v '^$$' | wc -l | tr -d ' '); \
if [ "$$FAILURES" -gt 0 ]; then \
$(ECHO) ""; \
$(ECHO) "$(RED)═══════════════════════════════════════════════════════════$(NC)"; \
$(ECHO) "$(RED) FAILED TEST CASES $(NC)"; \
$(ECHO) "$(RED)═══════════════════════════════════════════════════════════$(NC)"; \
$(ECHO) ""; \
printf "$(YELLOW)%-60s %-20s$(NC)\n" "Test Name" "Status"; \
printf "$(YELLOW)%-60s %-20s$(NC)\n" "─────────────────────────────────────────────────────────────" "────────────────────"; \
$(ECHO) "$$FAILED_TESTS" | while read -r testname; do \
if [ -n "$$testname" ]; then \
printf "$(RED)%-60s %-20s$(NC)\n" "$$testname" "FAILED"; \
fi; \
done; \
$(ECHO) ""; \
$(ECHO) "$(RED)Total Failures: $$FAILURES$(NC)"; \
$(ECHO) ""; \
else \
$(ECHO) ""; \
$(ECHO) "$(GREEN)═══════════════════════════════════════════════════════════$(NC)"; \
$(ECHO) "$(GREEN) ALL TESTS PASSED ✓ $(NC)"; \
$(ECHO) "$(GREEN)═══════════════════════════════════════════════════════════$(NC)"; \
$(ECHO) ""; \
fi; \
fi; \
if [ -n "$$REPORT_FILE" ] && [ -f "$$REPORT_FILE" ]; then \
SUMMARY_PREFIX=$$(basename "$$REPORT_FILE" .xml | sed 's/-.*//'); \
SUMMARY_TITLE=$$($(ECHO) "$$SUMMARY_PREFIX" | awk '{print toupper(substr($$0,1,1)) substr($$0,2)}'); \
$(MAKE) --no-print-directory print-test-summary \
SUMMARY_LABEL="$$SUMMARY_TITLE" \
SUMMARY_STRIP="$$SUMMARY_PREFIX-" \
SUMMARY_FILES="$$REPORT_FILE"; \
fi; \
if [ $$TEST_FAILED -eq 1 ]; then \
exit 1; \
fi
setup-mcp-tests: ## Build all MCP test servers in examples/mcps/ (Go and TypeScript)
@$(ECHO) "$(GREEN)Building MCP test servers...$(NC)"
@$(USE_NODE); \
FAILED=0; \
for mcp_dir in examples/mcps/*/; do \
if [ -d "$$mcp_dir" ]; then \
mcp_name=$$(basename $$mcp_dir); \
if [ -f "$$mcp_dir/go.mod" ]; then \
$(ECHO) "$(CYAN)Building $$mcp_name (Go)...$(NC)"; \
mkdir -p "$$mcp_dir/bin"; \
if cd "$$mcp_dir" && GOWORK=off go build -o bin/$$mcp_name . && cd - > /dev/null; then \
$(ECHO) "$(GREEN) ✓ $$mcp_name$(NC)"; \
else \
$(ECHO) "$(RED) ✗ $$mcp_name failed$(NC)"; \
FAILED=1; \
cd - > /dev/null 2>&1 || true; \
fi; \
elif [ -f "$$mcp_dir/package.json" ]; then \
$(ECHO) "$(CYAN)Building $$mcp_name (TypeScript)...$(NC)"; \
if cd "$$mcp_dir" && npm install --silent && npm run build && cd - > /dev/null; then \
$(ECHO) "$(GREEN) ✓ $$mcp_name$(NC)"; \
else \
$(ECHO) "$(RED) ✗ $$mcp_name failed$(NC)"; \
FAILED=1; \
cd - > /dev/null 2>&1 || true; \
fi; \
fi; \
fi; \
done; \
if [ $$FAILED -eq 1 ]; then \
$(ECHO) "$(RED)Some MCP test servers failed to build$(NC)"; \
exit 1; \
fi
@$(ECHO) ""
@$(ECHO) "$(GREEN)✓ All MCP test servers built$(NC)"
test-mcp: install-gotestsum setup-mcp-tests ## Run MCP tests (Usage: make test-mcp [TYPE=connection] [TESTCASE=TestName] [PATTERN=substring])
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running MCP tests...$(NC)"; \
mkdir -p $(TEST_REPORTS_DIR); \
if [ -n "$(PATTERN)" ] && [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(RED)Error: PATTERN and TESTCASE are mutually exclusive$(NC)"; \
$(ECHO) "$(YELLOW)Use PATTERN for substring matching or TESTCASE for exact match$(NC)"; \
exit 1; \
fi; \
if [ ! -d "core/internal/mcptests" ]; then \
$(ECHO) "$(RED)Error: MCP tests directory not found$(NC)"; \
exit 1; \
fi; \
TEST_FAILED=0; \
REPORT_FILE=""; \
if [ -n "$(TYPE)" ]; then \
TYPE_CLEAN=$$(echo $(TYPE) | sed 's/_test\.go$$//'); \
TEST_FILE="core/internal/mcptests/$${TYPE_CLEAN}_test.go"; \
if [ ! -f "$$TEST_FILE" ]; then \
$(ECHO) "$(RED)Error: Test file '$$TEST_FILE' not found$(NC)"; \
$(ECHO) "$(YELLOW)Available test types:$(NC)"; \
ls -1 core/internal/mcptests/*_test.go 2>/dev/null | sed 's|core/internal/mcptests/||' | sed 's|_test\.go$$||' | sed 's/^/ - /'; \
exit 1; \
fi; \
TEST_PATTERN=$$(grep -h "^func Test" $$TEST_FILE 2>/dev/null | sed 's/func \(Test[^(]*\).*/\1/' | paste -sd '|' - || $(ECHO) "^Test"); \
if [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(CYAN)Running $(TYPE) test: $(TESTCASE)...$(NC)"; \
SAFE_TESTCASE=$$($(ECHO) "$(TESTCASE)" | sed 's|/|_|g'); \
REPORT_FILE="$(TEST_REPORTS_DIR)/mcp-$(TYPE)-$$SAFE_TESTCASE.xml"; \
cd core/internal/mcptests && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../../$$REPORT_FILE \
-- -v -race -run "^$(TESTCASE)$$" . || TEST_FAILED=1; \
elif [ -n "$(PATTERN)" ]; then \
$(ECHO) "$(CYAN)Running $(TYPE) tests matching '$(PATTERN)'...$(NC)"; \
SAFE_PATTERN=$$($(ECHO) "$(PATTERN)" | sed 's|/|_|g'); \
REPORT_FILE="$(TEST_REPORTS_DIR)/mcp-$(TYPE)-$$SAFE_PATTERN.xml"; \
cd core/internal/mcptests && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../../$$REPORT_FILE \
-- -v -race -run ".*$(PATTERN).*" . || TEST_FAILED=1; \
else \
$(ECHO) "$(CYAN)Running all $(TYPE) tests (pattern: $$TEST_PATTERN)...$(NC)"; \
REPORT_FILE="$(TEST_REPORTS_DIR)/mcp-$(TYPE).xml"; \
cd core/internal/mcptests && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../../$$REPORT_FILE \
-- -v -race -run "$$TEST_PATTERN" . || TEST_FAILED=1; \
fi; \
cd ../../..; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report...$(NC)"; \
junit-viewer --results=$$REPORT_FILE --save=$${REPORT_FILE%.xml}.html 2>/dev/null || true; \
$(ECHO) ""; \
$(ECHO) "$(CYAN)HTML report: $${REPORT_FILE%.xml}.html$(NC)"; \
$(ECHO) "$(CYAN)Open with: open $${REPORT_FILE%.xml}.html$(NC)"; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
if [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(CYAN)Running test case: $(TESTCASE) across all MCP tests...$(NC)"; \
REPORT_FILE="$(TEST_REPORTS_DIR)/mcp-all-$(TESTCASE).xml"; \
cd core/internal/mcptests && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../../$$REPORT_FILE \
-- -v -race -run "^$(TESTCASE)$$" || TEST_FAILED=1; \
elif [ -n "$(PATTERN)" ]; then \
$(ECHO) "$(CYAN)Running tests matching '$(PATTERN)' across all MCP tests...$(NC)"; \
REPORT_FILE="$(TEST_REPORTS_DIR)/mcp-all-$(PATTERN).xml"; \
cd core/internal/mcptests && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../../$$REPORT_FILE \
-- -v -race -run ".*$(PATTERN).*" || TEST_FAILED=1; \
else \
$(ECHO) "$(CYAN)Running all MCP tests...$(NC)"; \
REPORT_FILE="$(TEST_REPORTS_DIR)/mcp-all.xml"; \
cd core/internal/mcptests && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../../$$REPORT_FILE \
-- -v -race || TEST_FAILED=1; \
fi; \
cd ../../..; \
if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
if which junit-viewer > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Generating HTML report...$(NC)"; \
junit-viewer --results=$$REPORT_FILE --save=$${REPORT_FILE%.xml}.html 2>/dev/null || true; \
$(ECHO) ""; \
$(ECHO) "$(CYAN)HTML report: $${REPORT_FILE%.xml}.html$(NC)"; \
$(ECHO) "$(CYAN)Open with: open $${REPORT_FILE%.xml}.html$(NC)"; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
else \
$(ECHO) ""; \
$(ECHO) "$(CYAN)JUnit XML report: $$REPORT_FILE$(NC)"; \
fi; \
fi; \
if [ $$TEST_FAILED -eq 1 ]; then \
exit 1; \
fi
test-all: test-core test-framework test-plugins test-http-transport test test-cli ## Run all tests
@$(ECHO) ""
@$(ECHO) "$(GREEN)═══════════════════════════════════════════════════════════$(NC)"
@$(ECHO) "$(GREEN) All Tests Complete - Summary $(NC)"
@$(ECHO) "$(GREEN)═══════════════════════════════════════════════════════════$(NC)"
@$(ECHO) ""
@if [ -z "$$CI" ] && [ -z "$$GITHUB_ACTIONS" ] && [ -z "$$GITLAB_CI" ] && [ -z "$$CIRCLECI" ] && [ -z "$$JENKINS_HOME" ]; then \
$(ECHO) "$(YELLOW)Generating combined HTML report...$(NC)"; \
junit-viewer --results=$(TEST_REPORTS_DIR) --save=$(TEST_REPORTS_DIR)/index.html 2>/dev/null || true; \
$(ECHO) ""; \
$(ECHO) "$(CYAN)HTML reports available in $(TEST_REPORTS_DIR)/:$(NC)"; \
ls -1 $(TEST_REPORTS_DIR)/*.html 2>/dev/null | sed 's/^/ ✓ /' || $(ECHO) " No reports found"; \
$(ECHO) ""; \
$(ECHO) "$(YELLOW)📊 View all test results:$(NC)"; \
$(ECHO) "$(CYAN) open $(TEST_REPORTS_DIR)/index.html$(NC)"; \
$(ECHO) ""; \
$(ECHO) "$(YELLOW)Or view individual reports:$(NC)"; \
ls -1 $(TEST_REPORTS_DIR)/*.html 2>/dev/null | grep -v index.html | sed 's|$(TEST_REPORTS_DIR)/| open $(TEST_REPORTS_DIR)/|' || true; \
$(ECHO) ""; \
else \
$(ECHO) "$(CYAN)JUnit XML reports available in $(TEST_REPORTS_DIR)/:$(NC)"; \
ls -1 $(TEST_REPORTS_DIR)/*.xml 2>/dev/null | sed 's/^/ ✓ /' || $(ECHO) " No reports found"; \
$(ECHO) ""; \
fi
test-semantic-cache: ## Run semantic_cache e2e tests (Usage: [CACHE_TYPE=direct|semantic] [RUN_FORCE=0] make test-semantic-cache). RUN_FORCE defaults to 1. Auto-detects trail CLI and wraps the run when present.
@cd tests/semanticcache && \
case "$$CACHE_TYPE" in \
direct) \
filter='^(TestPreconditions|TestDirect|TestLifecycle)$$'; \
$(ECHO) "$(CYAN)CACHE_TYPE=direct → running preconditions + direct + lifecycle$(NC)"; \
;; \
semantic) \
filter='^(TestPreconditions|TestParaphraseFixtures|TestSemantic|TestLifecycle)$$'; \
$(ECHO) "$(CYAN)CACHE_TYPE=semantic → running preconditions + fixtures + semantic + lifecycle$(NC)"; \
;; \
'') \
filter=''; \
$(ECHO) "$(CYAN)CACHE_TYPE unset → running all phases$(NC)"; \
;; \
*) \
$(ECHO) "$(RED)CACHE_TYPE=$$CACHE_TYPE invalid; expected 'direct', 'semantic', or unset$(NC)"; \
exit 1; \
;; \
esac; \
if command -v trail >/dev/null 2>&1; then \
$(ECHO) "$(GREEN)trail detected — wrapping run in 'trail run' (session id will be printed by trail)$(NC)"; \
if [ -n "$$filter" ]; then \
exec trail run -- env RUN_FORCE=$${RUN_FORCE:-1} GOWORK=off go test -v -run "$$filter" ./...; \
else \
exec trail run -- env RUN_FORCE=$${RUN_FORCE:-1} GOWORK=off go test -v ./...; \
fi; \
else \
$(ECHO) "$(YELLOW)trail not on PATH — falling back to direct go test (install 'trail' for capture-based debugging)$(NC)"; \
if [ -n "$$filter" ]; then \
exec env RUN_FORCE=$${RUN_FORCE:-1} GOWORK=off go test -v -run "$$filter" ./...; \
else \
exec env RUN_FORCE=$${RUN_FORCE:-1} GOWORK=off go test -v ./...; \
fi; \
fi
test-semantic-cache-complete: ## Run BOTH plugin unit tests + e2e tests for semantic_cache. RUN_FORCE defaults to 1. Wraps everything in trail if available.
@if command -v trail >/dev/null 2>&1; then \
$(ECHO) "$(GREEN)trail detected — wrapping unit + e2e tests in a single trail session (id printed by trail)$(NC)"; \
exec trail run -- $(MAKE) _test-semantic-cache-complete-inner; \
else \
$(ECHO) "$(YELLOW)trail not on PATH — running tests directly (install 'trail' for capture-based debugging)$(NC)"; \
$(MAKE) _test-semantic-cache-complete-inner; \
fi
_test-semantic-cache-complete-inner:
@$(ECHO) ""
@$(ECHO) "$(CYAN)═══════════════════════════════════════════════════════════$(NC)"
@$(ECHO) "$(CYAN) Running semantic_cache plugin UNIT tests $(NC)"
@$(ECHO) "$(CYAN)═══════════════════════════════════════════════════════════$(NC)"
@cd plugins/semanticcache && go test -v ./...
@$(ECHO) ""
@$(ECHO) "$(GREEN)═══════════════════════════════════════════════════════════$(NC)"
@$(ECHO) "$(GREEN) Unit tests completed $(NC)"
@$(ECHO) "$(GREEN)═══════════════════════════════════════════════════════════$(NC)"
@$(ECHO) ""
@$(ECHO) "$(CYAN)═══════════════════════════════════════════════════════════$(NC)"
@$(ECHO) "$(CYAN) Running semantic_cache E2E tests $(NC)"
@$(ECHO) "$(CYAN)═══════════════════════════════════════════════════════════$(NC)"
@cd tests/semanticcache && RUN_FORCE=$${RUN_FORCE:-1} GOWORK=off go test -v ./...
@$(ECHO) ""
@$(ECHO) "$(GREEN)═══════════════════════════════════════════════════════════$(NC)"
@$(ECHO) "$(GREEN) E2E tests completed $(NC)"
@$(ECHO) "$(GREEN)═══════════════════════════════════════════════════════════$(NC)"
test-chatbot: ## Run interactive chatbot integration test (Usage: RUN_CHATBOT_TEST=1 make test-chatbot)
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running interactive chatbot integration test...$(NC)"; \
if [ -z "$(RUN_CHATBOT_TEST)" ]; then \
$(ECHO) "$(YELLOW)⚠️ This is an interactive test. Set RUN_CHATBOT_TEST=1 to run it.$(NC)"; \
$(ECHO) "$(CYAN)Usage: RUN_CHATBOT_TEST=1 make test-chatbot$(NC)"; \
$(ECHO) ""; \
$(ECHO) "$(YELLOW)Required environment variables:$(NC)"; \
$(ECHO) " - OPENAI_API_KEY (required)"; \
$(ECHO) " - ANTHROPIC_API_KEY (optional)"; \
$(ECHO) " - Additional provider keys as needed"; \
exit 0; \
fi; \
cd core && RUN_CHATBOT_TEST=1 go test -v -run TestChatbot
test-integrations-py: ## Run Python integration tests (Usage: make test-integrations-py [INTEGRATION=openai] [TESTCASE=test_name] [PATTERN=substring] [VERBOSE=1])
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running Python integration tests...$(NC)"; \
if [ ! -d "tests/integrations/python" ]; then \
$(ECHO) "$(RED)Error: tests/integrations/python directory not found$(NC)"; \
exit 1; \
fi; \
if [ -n "$(PATTERN)" ] && [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(RED)Error: PATTERN and TESTCASE are mutually exclusive$(NC)"; \
$(ECHO) "$(YELLOW)Use PATTERN for substring matching or TESTCASE for exact match$(NC)"; \
exit 1; \
fi; \
if [ -n "$(TESTCASE)" ] && [ -z "$(INTEGRATION)" ]; then \
$(ECHO) "$(RED)Error: TESTCASE requires INTEGRATION to be specified$(NC)"; \
$(ECHO) "$(YELLOW)Usage: make test-integrations-py INTEGRATION=anthropic TESTCASE=test_05_end2end_tool_calling$(NC)"; \
exit 1; \
fi; \
BIFROST_STARTED=0; \
BIFROST_PID=""; \
TAIL_PID=""; \
TEST_PORT=$${PORT:-8080}; \
TEST_HOST=$${HOST:-localhost}; \
$(ECHO) "$(CYAN)Checking if Bifrost is running on $$TEST_HOST:$$TEST_PORT...$(NC)"; \
if curl -s -o /dev/null -w "%{http_code}" http://$$TEST_HOST:$$TEST_PORT/health 2>/dev/null | grep -q "200\|404"; then \
$(ECHO) "$(GREEN)✓ Bifrost is already running$(NC)"; \
else \
$(ECHO) "$(YELLOW)Bifrost not running, starting it...$(NC)"; \
./tmp/bifrost-http -host "$$TEST_HOST" -port "$$TEST_PORT" -log-style "$(LOG_STYLE)" -log-level "$(LOG_LEVEL)" -app-dir tests/integrations/python > /tmp/bifrost-test.log 2>&1 & \
BIFROST_PID=$$!; \
BIFROST_STARTED=1; \
$(ECHO) "$(YELLOW)Waiting for Bifrost to be ready...$(NC)"; \
$(ECHO) "$(CYAN)Bifrost logs: /tmp/bifrost-test.log$(NC)"; \
(tail -f /tmp/bifrost-test.log 2>/dev/null | grep -E "error|panic|Error|ERRO|fatal|Fatal|FATAL" --line-buffered &) & \
TAIL_PID=$$!; \
for i in $$(seq 1 30); do \
if curl -s -o /dev/null http://$$TEST_HOST:$$TEST_PORT/health 2>/dev/null; then \
$(ECHO) "$(GREEN)✓ Bifrost is ready (PID: $$BIFROST_PID)$(NC)"; \
break; \
fi; \
if [ $$i -eq 10 ]; then \
$(ECHO) "$(RED)Failed to start Bifrost$(NC)"; \
$(ECHO) "$(YELLOW)Bifrost logs:$(NC)"; \
cat /tmp/bifrost-test.log 2>/dev/null || $(ECHO) "No log file found"; \
[ -n "$$BIFROST_PID" ] && kill $$BIFROST_PID 2>/dev/null; \
[ -n "$$TAIL_PID" ] && kill $$TAIL_PID 2>/dev/null; \
exit 1; \
fi; \
sleep 1; \
done; \
fi; \
TEST_FAILED=0; \
if ! which uv > /dev/null 2>&1; then \
$(ECHO) "$(YELLOW)uv not found, checking for pytest...$(NC)"; \
if ! which pytest > /dev/null 2>&1; then \
$(ECHO) "$(RED)Error: Neither uv nor pytest found$(NC)"; \
$(ECHO) "$(YELLOW)Install uv: curl -LsSf https://astral.sh/uv/install.sh | sh$(NC)"; \
$(ECHO) "$(YELLOW)Or install pytest: pip install pytest$(NC)"; \
[ $$BIFROST_STARTED -eq 1 ] && [ -n "$$BIFROST_PID" ] && kill $$BIFROST_PID 2>/dev/null; \
[ -n "$$TAIL_PID" ] && kill $$TAIL_PID 2>/dev/null; \
exit 1; \
fi; \
$(ECHO) "$(CYAN)Using pytest directly$(NC)"; \
if [ -n "$(INTEGRATION)" ]; then \
if [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(CYAN)Running $(INTEGRATION) integration test: $(TESTCASE)...$(NC)"; \
cd tests/integrations/python && \
if [[ "$(TESTCASE)" == *::* ]]; then \
pytest tests/test_$(INTEGRATION).py::$(TESTCASE) $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
else \
pytest tests/test_$(INTEGRATION).py -k "$(TESTCASE)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
fi; \
elif [ -n "$(PATTERN)" ]; then \
$(ECHO) "$(CYAN)Running $(INTEGRATION) integration tests matching '$(PATTERN)'...$(NC)"; \
cd tests/integrations/python && pytest tests/test_$(INTEGRATION).py -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
else \
$(ECHO) "$(CYAN)Running $(INTEGRATION) integration tests...$(NC)"; \
cd tests/integrations/python && pytest tests/test_$(INTEGRATION).py $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
fi; \
else \
if [ -n "$(PATTERN)" ]; then \
$(ECHO) "$(CYAN)Running all integration tests matching '$(PATTERN)'...$(NC)"; \
cd tests/integrations/python && pytest -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
else \
$(ECHO) "$(CYAN)Running all integration tests...$(NC)"; \
cd tests/integrations/python && pytest $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
fi; \
fi; \
else \
$(ECHO) "$(CYAN)Using uv (fast mode)$(NC)"; \
cd tests/integrations/python && \
if [ -n "$(INTEGRATION)" ]; then \
if [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(CYAN)Running $(INTEGRATION) integration test: $(TESTCASE)...$(NC)"; \
if [[ "$(TESTCASE)" == *::* ]]; then \
uv run --frozen pytest tests/test_$(INTEGRATION).py::$(TESTCASE) $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
else \
uv run --frozen pytest tests/test_$(INTEGRATION).py -k "$(TESTCASE)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
fi; \
elif [ -n "$(PATTERN)" ]; then \
$(ECHO) "$(CYAN)Running $(INTEGRATION) integration tests matching '$(PATTERN)'...$(NC)"; \
uv run --frozen pytest tests/test_$(INTEGRATION).py -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
else \
$(ECHO) "$(CYAN)Running $(INTEGRATION) integration tests...$(NC)"; \
uv run --frozen pytest tests/test_$(INTEGRATION).py $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
fi; \
else \
if [ -n "$(PATTERN)" ]; then \
$(ECHO) "$(CYAN)Running all integration tests matching '$(PATTERN)'...$(NC)"; \
uv run --frozen pytest -k "$(PATTERN)" $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
else \
$(ECHO) "$(CYAN)Running all integration tests...$(NC)"; \
uv run --frozen pytest $(if $(VERBOSE),-v,-q) || TEST_FAILED=1; \
fi; \
fi; \
fi; \
if [ $$BIFROST_STARTED -eq 1 ] && [ -n "$$BIFROST_PID" ]; then \
$(ECHO) "$(YELLOW)Stopping Bifrost (PID: $$BIFROST_PID)...$(NC)"; \
kill $$BIFROST_PID 2>/dev/null || true; \
[ -n "$$TAIL_PID" ] && kill $$TAIL_PID 2>/dev/null || true; \
wait $$BIFROST_PID 2>/dev/null || true; \
$(ECHO) "$(GREEN)✓ Bifrost stopped$(NC)"; \
if [ $$TEST_FAILED -eq 1 ]; then \
$(ECHO) ""; \
$(ECHO) "$(YELLOW)Last 50 lines of Bifrost logs:$(NC)"; \
tail -50 /tmp/bifrost-test.log 2>/dev/null || $(ECHO) "No log file found"; \
fi; \
fi; \
$(ECHO) ""; \
if [ $$TEST_FAILED -eq 1 ]; then \
$(ECHO) "$(RED)✗ Integration tests failed$(NC)"; \
$(ECHO) "$(CYAN)Full Bifrost logs: /tmp/bifrost-test.log$(NC)"; \
exit 1; \
else \
$(ECHO) "$(GREEN)✓ Integration tests complete$(NC)"; \
fi
test-integrations-ts: ## Run TypeScript integration tests (Usage: make test-integrations-ts [INTEGRATION=openai] [TESTCASE=test_name] [PATTERN=substring] [VERBOSE=1])
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running TypeScript integration tests...$(NC)"; \
if [ ! -d "tests/integrations/typescript" ]; then \
$(ECHO) "$(RED)Error: tests/integrations/typescript directory not found$(NC)"; \
exit 1; \
fi; \
if [ -n "$(PATTERN)" ] && [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(RED)Error: PATTERN and TESTCASE are mutually exclusive$(NC)"; \
$(ECHO) "$(YELLOW)Use PATTERN for substring matching or TESTCASE for exact match$(NC)"; \
exit 1; \
fi; \
if [ -n "$(TESTCASE)" ] && [ -z "$(INTEGRATION)" ]; then \
$(ECHO) "$(RED)Error: TESTCASE requires INTEGRATION to be specified$(NC)"; \
$(ECHO) "$(YELLOW)Usage: make test-integrations-ts INTEGRATION=openai TESTCASE=test_simple_chat$(NC)"; \
exit 1; \
fi; \
BIFROST_STARTED=0; \
BIFROST_PID=""; \
TAIL_PID=""; \
TEST_PORT=$${PORT:-8080}; \
TEST_HOST=$${HOST:-localhost}; \
$(ECHO) "$(CYAN)Checking if Bifrost is running on $$TEST_HOST:$$TEST_PORT...$(NC)"; \
if curl -s -o /dev/null -w "%{http_code}" http://$$TEST_HOST:$$TEST_PORT/health 2>/dev/null | grep -q "200\|404"; then \
$(ECHO) "$(GREEN)✓ Bifrost is already running$(NC)"; \
else \
$(ECHO) "$(YELLOW)Bifrost not running, starting it...$(NC)"; \
./tmp/bifrost-http -host "$$TEST_HOST" -port "$$TEST_PORT" -log-style "$(LOG_STYLE)" -log-level "$(LOG_LEVEL)" -app-dir tests/integrations/typescript > /tmp/bifrost-test.log 2>&1 & \
BIFROST_PID=$$!; \
BIFROST_STARTED=1; \
$(ECHO) "$(YELLOW)Waiting for Bifrost to be ready...$(NC)"; \
$(ECHO) "$(CYAN)Bifrost logs: /tmp/bifrost-test.log$(NC)"; \
(tail -f /tmp/bifrost-test.log 2>/dev/null | grep -E "error|panic|Error|ERRO|fatal|Fatal|FATAL" --line-buffered &) & \
TAIL_PID=$$!; \
for i in $$(seq 1 30); do \
if curl -s -o /dev/null http://$$TEST_HOST:$$TEST_PORT/health 2>/dev/null; then \
$(ECHO) "$(GREEN)✓ Bifrost is ready (PID: $$BIFROST_PID)$(NC)"; \
break; \
fi; \
if [ $$i -eq 10 ]; then \
$(ECHO) "$(RED)Failed to start Bifrost$(NC)"; \
$(ECHO) "$(YELLOW)Bifrost logs:$(NC)"; \
cat /tmp/bifrost-test.log 2>/dev/null || $(ECHO) "No log file found"; \
[ -n "$$BIFROST_PID" ] && kill $$BIFROST_PID 2>/dev/null; \
[ -n "$$TAIL_PID" ] && kill $$TAIL_PID 2>/dev/null; \
exit 1; \
fi; \
sleep 1; \
done; \
fi; \
TEST_FAILED=0; \
$(USE_NODE); \
if ! which npm > /dev/null 2>&1; then \
$(ECHO) "$(RED)Error: npm not found$(NC)"; \
$(ECHO) "$(YELLOW)Install Node.js: https://nodejs.org/$(NC)"; \
[ $$BIFROST_STARTED -eq 1 ] && [ -n "$$BIFROST_PID" ] && kill $$BIFROST_PID 2>/dev/null; \
[ -n "$$TAIL_PID" ] && kill $$TAIL_PID 2>/dev/null; \
exit 1; \
fi; \
$(ECHO) "$(CYAN)Using npm$(NC)"; \
cd tests/integrations/typescript && \
if [ ! -d "node_modules" ]; then \
$(ECHO) "$(YELLOW)Installing dependencies...$(NC)"; \
npm install; \
fi; \
if [ -n "$(INTEGRATION)" ]; then \
if [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(CYAN)Running $(INTEGRATION) integration test: $(TESTCASE)...$(NC)"; \
npm test -- tests/test-$(INTEGRATION).test.ts -t "$(TESTCASE)" $(if $(VERBOSE),--reporter=verbose,) || TEST_FAILED=1; \
elif [ -n "$(PATTERN)" ]; then \
$(ECHO) "$(CYAN)Running $(INTEGRATION) integration tests matching '$(PATTERN)'...$(NC)"; \
npm test -- tests/test-$(INTEGRATION).test.ts -t "$(PATTERN)" $(if $(VERBOSE),--reporter=verbose,) || TEST_FAILED=1; \
else \
$(ECHO) "$(CYAN)Running $(INTEGRATION) integration tests...$(NC)"; \
npm test -- tests/test-$(INTEGRATION).test.ts $(if $(VERBOSE),--reporter=verbose,) || TEST_FAILED=1; \
fi; \
else \
if [ -n "$(PATTERN)" ]; then \
$(ECHO) "$(CYAN)Running all integration tests matching '$(PATTERN)'...$(NC)"; \
npm test -- -t "$(PATTERN)" $(if $(VERBOSE),--reporter=verbose,) || TEST_FAILED=1; \
else \
$(ECHO) "$(CYAN)Running all integration tests...$(NC)"; \
npm test $(if $(VERBOSE),-- --reporter=verbose,) || TEST_FAILED=1; \
fi; \
fi; \
if [ $$BIFROST_STARTED -eq 1 ] && [ -n "$$BIFROST_PID" ]; then \
$(ECHO) "$(YELLOW)Stopping Bifrost (PID: $$BIFROST_PID)...$(NC)"; \
kill $$BIFROST_PID 2>/dev/null || true; \
[ -n "$$TAIL_PID" ] && kill $$TAIL_PID 2>/dev/null || true; \
wait $$BIFROST_PID 2>/dev/null || true; \
$(ECHO) "$(GREEN)✓ Bifrost stopped$(NC)"; \
if [ $$TEST_FAILED -eq 1 ]; then \
$(ECHO) ""; \
$(ECHO) "$(YELLOW)Last 50 lines of Bifrost logs:$(NC)"; \
tail -50 /tmp/bifrost-test.log 2>/dev/null || $(ECHO) "No log file found"; \
fi; \
fi; \
$(ECHO) ""; \
if [ $$TEST_FAILED -eq 1 ]; then \
$(ECHO) "$(RED)✗ TypeScript integration tests failed$(NC)"; \
$(ECHO) "$(CYAN)Full Bifrost logs: /tmp/bifrost-test.log$(NC)"; \
exit 1; \
else \
$(ECHO) "$(GREEN)✓ TypeScript integration tests complete$(NC)"; \
fi
test-integrations: ## Run Python + TypeScript SDK integration tests in parallel, one process per test file, against one shared gateway (Usage: make test-integrations [JOBS=30] [INTEGRATION=openai] [BUILD=1] [PORT=8080])
@: "Delegates to the CI script rather than reimplementing the fan-out. That script already"; \
: "throttles both suites against one gateway, group-kills descendants on cancellation, and"; \
: "replays each file's buffered log in order - see .github/workflows/scripts/test-integrations.sh."; \
: ""; \
: "Bash check first, before EXPOSE_ENV: an unusable shell should cost milliseconds rather"; \
: "than an Infisical round trip. This recipe shell is the same /usr/bin/env bash the"; \
: "script's shebang resolves, so BASH_VERSINFO here is what the script will see. The script"; \
: "rejects old shells too; this only adds the remedy and names the sequential fallback."; \
if [ "$${BASH_VERSINFO[0]}" -lt 5 ] || { [ "$${BASH_VERSINFO[0]}" -eq 5 ] && [ "$${BASH_VERSINFO[1]}" -lt 1 ]; }; then \
$(ECHO) "$(RED)Error: bash 5.1+ required (the fan-out uses 'wait -f -n -p'), found $$BASH_VERSION$(NC)"; \
$(ECHO) "$(YELLOW)macOS ships bash 3.2. Install a newer one and put it ahead of /bin in PATH:$(NC)"; \
$(ECHO) "$(CYAN) brew install bash$(NC)"; \
$(ECHO) "$(YELLOW)Or run the suites sequentially: make test-integrations-py && make test-integrations-ts$(NC)"; \
exit 1; \
fi; \
SCRIPT=.github/workflows/scripts/test-integrations.sh; \
if [ ! -x "$$SCRIPT" ]; then \
$(ECHO) "$(RED)Error: $$SCRIPT is missing or not executable$(NC)"; \
exit 1; \
fi; \
$(EXPOSE_ENV); \
$(USE_NODE); \
TEST_HOST="$${HOST:-localhost}"; \
TEST_PORT="$${PORT:-8080}"; \
: "Gateway: reuse one that is already serving this port, matching test-integrations-py"; \
: "and -ts. The script cannot work this out for itself - it starts its own gateway"; \
: "unconditionally, and its readiness loop curls /health before checking whether its own"; \
: "child survived, so an existing server would answer for it while the real child died"; \
: "unable to bind. SKIP_GATEWAY_START=1 makes that explicit instead of accidental, and"; \
: "leaves the reused server running at the end - we did not start it."; \
: ""; \
: "Probed after EXPOSE_ENV because .env may set PORT."; \
REUSING=0; \
if curl -sf --connect-timeout 2 --max-time 5 "http://$$TEST_HOST:$$TEST_PORT/health" > /dev/null 2>&1; then \
REUSING=1; \
export SKIP_GATEWAY_START=1; \
$(ECHO) "$(GREEN)Reusing the gateway already serving http://$$TEST_HOST:$$TEST_PORT$(NC)"; \
$(ECHO) "$(YELLOW) Its config is whatever that server was started with. These suites expect$(NC)"; \
$(ECHO) "$(YELLOW) -app-dir tests/integrations/python (35 providers plus the vk-test virtual$(NC)"; \
$(ECHO) "$(YELLOW) key); started elsewhere, expect config-shaped failures.$(NC)"; \
elif command -v nc > /dev/null 2>&1 && nc -z -w 1 "$$TEST_HOST" "$$TEST_PORT" 2>/dev/null; then \
: "Listening but not answering /health - not a gateway, and we cannot bind either."; \
$(ECHO) "$(RED)Error: $$TEST_HOST:$$TEST_PORT is in use but does not answer /health$(NC)"; \
$(ECHO) "$(YELLOW)Something that is not a Bifrost gateway holds the port, and this target$(NC)"; \
$(ECHO) "$(YELLOW)cannot start its own there.$(NC)"; \
$(ECHO) "$(CYAN)Stop it, or: make test-integrations PORT=8081$(NC)"; \
exit 1; \
else \
$(ECHO) "$(CYAN)No gateway on $$TEST_HOST:$$TEST_PORT - starting one (-app-dir tests/integrations/python)$(NC)"; \
fi; \
JOBS_VAL="$${JOBS:-$(JOBS)}"; JOBS_VAL="$${JOBS_VAL:-30}"; \
: "Default 30 against 15 test files total (9 Python + 6 TypeScript) means every file"; \
: "launches at once - the cap is a ceiling for the day the suites grow, not a throttle"; \
: "today. What it stops protecting is provider quota: every test spends a real billed"; \
: "call, and conftest.py marks every test flaky(reruns=3), so one failure costs up to"; \
: "four. Lower it (JOBS=4, what release-pipeline.yml pins) if a run starts drawing 429s."; \
: "The script validates this too; checked here so the message names JOBS, which is what"; \
: "the caller actually typed, rather than INTEGRATION_TEST_MAX_PARALLEL."; \
case "$$JOBS_VAL" in \
''|*[!0-9]*|0) \
$(ECHO) "$(RED)Error: JOBS must be a positive integer, got '$$JOBS_VAL'$(NC)"; \
exit 1 ;; \
esac; \
: "Build: reuse tmp/bifrost-http when present, BUILD=1 forces the script's own"; \
: "'make LOCAL=1 build'. Not a prerequisite on 'build': that pulls in build-ui and is"; \
: ".PHONY, so every run would rebuild the whole Next.js UI before any test could start,"; \
: "and no SDK integration test touches the UI. The mtime is printed because silently"; \
: "reusing a weeks-old binary is the obvious hazard of defaulting to skip."; \
if [ "$$REUSING" = "1" ]; then \
: "No binary needed: we are attaching to a gateway, not starting one. The script"; \
: "skips its own build under SKIP_GATEWAY_START and never looks for tmp/bifrost-http."; \
$(ECHO) "$(CYAN)Skipping build - attaching to the running gateway$(NC)"; \
elif [ -n "$(BUILD)" ]; then \
$(ECHO) "$(YELLOW)BUILD=1: rebuilding UI and bifrost-http first (slow)$(NC)"; \
elif [ -x tmp/bifrost-http ]; then \
export SKIP_GATEWAY_BUILD=1; \
$(ECHO) "$(GREEN)Reusing tmp/bifrost-http (built $$(date -r tmp/bifrost-http '+%Y-%m-%d %H:%M' 2>/dev/null || echo unknown))$(NC)"; \
$(ECHO) "$(YELLOW)Stale? Rebuild with: make test-integrations BUILD=1$(NC)"; \
else \
$(ECHO) "$(YELLOW)No tmp/bifrost-http yet - building it, UI included (slow)$(NC)"; \
fi; \
: "One process per test FILE, deliberately, not pytest-xdist. tests/test_langchain.py,"; \
: "test_litellm.py and test_pydanticai.py each carry an autouse fixture that overwrites"; \
: "OPENAI_API_KEY and friends in os.environ with dummy values and restores only the base"; \
: "URLs, never the keys. Any xdist mode lets one worker take several files in sequence, so"; \
: "the dummies leak forward - and tests/test_google.py:753 builds a genai.Client that talks"; \
: "to Google directly rather than through the gateway, where a dummy key is a hard auth"; \
: "failure. Process-per-file is a correctness requirement here, which is why pytest-xdist"; \
: "stays unused despite being a declared dependency."; \
if [ -n "$(INTEGRATION)" ]; then \
export INTEGRATION_TEST_FILTER="$(INTEGRATION)"; \
$(ECHO) "$(CYAN)Integration filter: $(INTEGRATION)$(NC)"; \
fi; \
export INTEGRATION_TEST_MAX_PARALLEL="$$JOBS_VAL"; \
: "So the script's markdown failure report lands wherever the Go targets put theirs."; \
export TEST_REPORTS_DIR="$(TEST_REPORTS_DIR)"; \
export PORT="$$TEST_PORT"; \
export HOST="$$TEST_HOST"; \
: "The TS suite runs against the Python app-dir on purpose - one shared gateway, and"; \
: "python/config.json's providers are a superset of typescript/config.json's. CI has run"; \
: "it this way since release-pipeline.yml adopted --parallel-files."; \
$(ECHO) "$(GREEN)Running Python + TypeScript integration tests$(NC)"; \
$(ECHO) "$(CYAN) Gateway: http://$$TEST_HOST:$$TEST_PORT (app-dir tests/integrations/python)$(NC)"; \
$(ECHO) "$(CYAN) Parallel: up to $$JOBS_VAL concurrent test files across both suites$(NC)"; \
"$$SCRIPT" --parallel-files
install-playwright: ## Install Playwright test dependencies
@$(ECHO) "$(GREEN)Installing Playwright dependencies...$(NC)"
@which node > /dev/null || ($(ECHO) "$(RED)Error: Node.js is not installed. Please install Node.js first.$(NC)" && exit 1)
@which npm > /dev/null || ($(ECHO) "$(RED)Error: npm is not installed. Please install npm first.$(NC)" && exit 1)
@$(USE_NODE); cd tests/e2e && npm ci
@cd tests/e2e && if npx playwright install --list 2>/dev/null | grep -q "chromium"; then \
$(ECHO) "$(CYAN)Chromium is already installed, skipping download$(NC)"; \
else \
$(ECHO) "$(CYAN)Installing Chromium...$(NC)"; \
npx playwright install --with-deps chromium; \
fi
@$(ECHO) "$(GREEN)Playwright is ready$(NC)"
build-test-plugin: ## Build test plugin for E2E tests (copies to tmp/bifrost-test-plugin.so)
@$(ECHO) "$(GREEN)Building test plugin for E2E tests...$(NC)"
@cd examples/plugins/hello-world && make dev
@mkdir -p tmp
@cp examples/plugins/hello-world/build/hello-world.so tmp/bifrost-test-plugin.so
@$(ECHO) "$(GREEN)✓ Test plugin ready at tmp/bifrost-test-plugin.so$(NC)"
run-e2e: run-e2e-ui ## Alias of run-e2e-ui
run-a11y-audit: install-playwright ## Scan every UI route with axe-core and report accessibility coverage (Usage: make run-a11y-audit [ROUTES=logs,providers] [MIN_SCORE=80])
@$(ECHO) "$(GREEN)Running accessibility audit...$(NC)"
@cd tests/e2e && A11Y_ROUTES="$(ROUTES)" A11Y_MIN_SCORE="$(MIN_SCORE)" npx playwright test --config=playwright.a11y.config.ts
@$(ECHO) "$(CYAN)Report: tests/e2e/a11y-report/summary.md$(NC)"
@$(ECHO) "$(CYAN)Static JSX checks: cd ui && npm run lint:a11y$(NC)"
# Flags for tests/e2e/scripts/run-e2e.mjs, built from the make variables below.
E2E_RUNNER_ARGS = $(if $(WORKERS),--workers $(WORKERS)) $(if $(FLOW),--features $(FLOW)) $(if $(RERUN),--rerun $(RERUN)) $(if $(SKIP_BUILD),--skip-build) $(if $(KEEP),--keep) $(if $(INTERACTIVE),--ui)
run-e2e-ui: install-playwright ## Run the UI E2E suite on isolated Bifrost workers (Usage: make run-e2e-ui [WORKERS=4] [FLOW=providers,logs] [RERUN=tests/e2e/reports/e2e-results.json] [SKIP_BUILD=1] [KEEP=1] [INTERACTIVE=1])
@$(USE_NODE); $(EXPOSE_ENV); \
. .github/workflows/scripts/setup-go-workspace.sh; \
node tests/e2e/scripts/run-e2e.mjs $(E2E_RUNNER_ARGS)
run-e2e-headed: install-playwright ## Run the UI E2E suite with visible browsers (same variables as run-e2e-ui)
@$(USE_NODE); $(EXPOSE_ENV); \
. .github/workflows/scripts/setup-go-workspace.sh; \
node tests/e2e/scripts/run-e2e.mjs $(E2E_RUNNER_ARGS) --headed
e2e-ui-down: ## Stop servers left running by run-e2e-ui KEEP=1 and remove the e2e Postgres
@$(USE_NODE); node tests/e2e/scripts/run-e2e.mjs --down
run-e2e-api: install-newman ## Run E2E API management tests (/api/* and /health). PROXY=1 runs the proxy e2e flow instead: real provider calls through squid/stunnel/microsocks for every proxy mode (Usage: make run-e2e-api PROXY=1 [PROXY_CELLS="provider-http env-socks5"] [PORT=8080])
@if [ -n "$(PROXY)" ]; then $(ECHO) "$(GREEN)Running proxy e2e (tests/e2e/api/runners/proxy/run-proxy-matrix.sh)...$(NC)"; else $(ECHO) "$(GREEN)Running E2E API management tests...$(NC)"; fi
@BASH4="$${BIFROST_BASH:-}"; \
if [ -z "$$BASH4" ]; then \
for candidate in \
"$$(brew --prefix bash 2>/dev/null)/bin/bash" \
/opt/homebrew/bin/bash \
/usr/local/bin/bash \
"$$(command -v bash)"; do \
if [ -n "$$candidate" ] && [ -x "$$candidate" ] && "$$candidate" -c 'test "$${BASH_VERSINFO[0]}" -ge 4' >/dev/null 2>&1; then \
BASH4="$$candidate"; \
break; \
fi; \
done; \
fi; \
if [ -z "$$BASH4" ]; then \
$(ECHO) "$(RED)Error: run-e2e-api requires Bash 4.0+ for the API runner.$(NC)"; \
$(ECHO) "$(YELLOW)Install a newer Bash with 'brew install bash', or pass BIFROST_BASH=/path/to/bash.$(NC)"; \
exit 1; \
fi; \
if [ -n "$(PROXY)" ]; then \
$(MAKE) --no-print-directory install-squid install-stunnel install-microsocks || exit 1; \
$(EXPOSE_ENV); \
exec "$$BASH4" ./tests/e2e/api/runners/proxy/run-proxy-matrix.sh; \
fi; \
cd tests/e2e/api && "$$BASH4" ./runners/run-newman-api-tests.sh --all-reports
run-e2e-api-proxy: ## Proxy e2e flow, same as make run-e2e-api PROXY=1
@$(MAKE) run-e2e-api PROXY=1
run-mcp-codemode-test: install-newman ## Run the hermetic MCP Code Mode E2E suite (no API keys, no paid calls). Builds tmp/bifrost-http from local code unless BINARY is given (Usage: make run-mcp-codemode-test [BINARY=path/to/bifrost-http])
@BINARY="$(BINARY)"; \
if [ -z "$$BINARY" ]; then \
$(MAKE) build LOCAL=1 || exit 1; \
BINARY=tmp/bifrost-http; \
fi; \
if [ ! -x "$$BINARY" ]; then \
$(ECHO) "$(RED)Error: bifrost-http binary not found or not executable: $$BINARY$(NC)"; \
exit 1; \
fi; \
BINARY="$$(cd "$$(dirname "$$BINARY")" && pwd)/$$(basename "$$BINARY")"; \
$(ECHO) "$(GREEN)Running MCP Code Mode E2E tests against $$BINARY...$(NC)"; \
./tests/e2e/api/runners/individual/run-newman-mcp-codemode-tests.sh --binary "$$BINARY"
run-mcp-harness-test: install-newman ## Run the MCP harness (generated by tests/e2e/api/runners/build-mcp-harness.mjs), one folder per test case. Hermetic: boots its own Bifrost and an mcpchaos fleet. Builds tmp/bifrost-http from local code with the harness build tag (OAS-45 needs it) unless BINARY is given (Usage: make run-mcp-harness-test [BINARY=path/to/bifrost-http] [FOLDER="ADM"] [HARNESS_PORT=8093] [CHAOS_PORT=8494] [INPUT_PORT=8496] [APPS_PORT=8497])
@BINARY="$(BINARY)"; \
if [ -z "$$BINARY" ]; then \
$(MAKE) build LOCAL=1 BUILD_TAGS=harness || exit 1; \
BINARY=tmp/bifrost-http; \
fi; \
if [ ! -x "$$BINARY" ]; then \
$(ECHO) "$(RED)Error: bifrost-http binary not found or not executable: $$BINARY$(NC)"; \
exit 1; \
fi; \
BINARY="$$(cd "$$(dirname "$$BINARY")" && pwd)/$$(basename "$$BINARY")"; \
$(ECHO) "$(GREEN)Running MCP harness against $$BINARY...$(NC)"; \
./tests/e2e/api/runners/individual/run-newman-mcp-harness-tests.sh --binary "$$BINARY" \
$(if $(FOLDER),--folder "$(FOLDER)") $(if $(HARNESS_PORT),--port "$(HARNESS_PORT)") $(if $(CHAOS_PORT),--chaos-port "$(CHAOS_PORT)") $(if $(INPUT_PORT),--input-port "$(INPUT_PORT)") $(if $(APPS_PORT),--apps-port "$(APPS_PORT)")
run-log-store-matrix: ## Relay and LLM log permutation matrix on every log store: sqlite, postgres, and hybrid over a local fake S3 (with and without offload exclusions). Hermetic: local upstreams, no API keys, no paid calls. Postgres cells need POSTGRES_URL. Builds tmp/bifrost-http unless BINARY is given (Usage: make run-log-store-matrix [BINARY=path/to/bifrost-http] [CELLS=sqlite,sqlite+s3] [SUITES=pricing,proxy] [PARALLEL=3] [POSTGRES_URL=postgresql://bifrost:bifrost_password@localhost:5432/postgres] [SKIP_CONTENT_MATRIX=1])
@$(USE_NODE); BINARY="$(BINARY)"; \
if [ -z "$$BINARY" ]; then \
$(MAKE) build LOCAL=1 || exit 1; \
BINARY=tmp/bifrost-http; \
fi; \
if [ ! -x "$$BINARY" ]; then \
$(ECHO) "$(RED)Error: bifrost-http binary not found or not executable: $$BINARY$(NC)"; \
exit 1; \
fi; \
if ! npm --prefix tests/e2e/api ls --depth=0 >/dev/null 2>&1; then \
(cd tests/e2e/api && npm install --silent) || exit 1; \
fi; \
BIFROST_E2E_POSTGRES_URL="$(POSTGRES_URL)" node tests/e2e/api/runners/run-log-store-matrix.mjs --binary "$$BINARY" \
$(if $(CELLS),--cells "$(CELLS)") $(if $(SUITES),--suites "$(SUITES)") $(if $(PARALLEL),--parallel "$(PARALLEL)") $(if $(SKIP_CONTENT_MATRIX),--skip-content-matrix)
run-log-store-matrix: ## Relay and LLM log permutation matrix on every log store: sqlite, postgres, and hybrid over a local fake S3 (with and without offload exclusions). Hermetic: local upstreams, no API keys, no paid calls. Postgres cells need POSTGRES_URL. Builds tmp/bifrost-http unless BINARY is given (Usage: make run-log-store-matrix [BINARY=path/to/bifrost-http] [CELLS=sqlite,sqlite+s3] [SUITES=pricing,proxy] [PARALLEL=3] [POSTGRES_URL=postgresql://bifrost:bifrost_password@localhost:5432/postgres] [SKIP_CONTENT_MATRIX=1])
@$(USE_NODE); BINARY="$(BINARY)"; \
if [ -z "$$BINARY" ]; then \
$(MAKE) build LOCAL=1 || exit 1; \
BINARY=tmp/bifrost-http; \
fi; \
if [ ! -x "$$BINARY" ]; then \
$(ECHO) "$(RED)Error: bifrost-http binary not found or not executable: $$BINARY$(NC)"; \
exit 1; \
fi; \
if ! npm --prefix tests/e2e/api ls --depth=0 >/dev/null 2>&1; then \
(cd tests/e2e/api && npm install --silent) || exit 1; \
fi; \
BIFROST_E2E_POSTGRES_URL="$(POSTGRES_URL)" node tests/e2e/api/runners/run-log-store-matrix.mjs --binary "$$BINARY" \
$(if $(CELLS),--cells "$(CELLS)") $(if $(SUITES),--suites "$(SUITES)") $(if $(PARALLEL),--parallel "$(PARALLEL)") $(if $(SKIP_CONTENT_MATRIX),--skip-content-matrix)
run-warp-test: install-newman ## Run the Warp E2E suite: boots Bifrost on a throwaway Postgres DB, seeds it (tests/cmd/seed/warpseed) and asks Warp questions with a live model. Needs OPENAI_API_KEY (or WARP_UPSTREAM_BIFROST=<url> to use a running Bifrost's key), Postgres and Weaviate (tests/docker-compose.yml). Builds tmp/bifrost-http unless BINARY is given (Usage: make run-warp-test [BINARY=path/to/bifrost-http] [FOLDER="Guardrails"] [WARP_UPSTREAM_BIFROST=http://localhost:8080] [WEAVIATE_HOST=localhost:9000] [POSTGRES_PORT=5432] [USE_INFISICAL=1])
@$(EXPOSE_ENV); \
BINARY="$(BINARY)"; \
if [ -z "$$BINARY" ]; then \
$(MAKE) build LOCAL=1 || exit 1; \
BINARY=tmp/bifrost-http; \
fi; \
if [ ! -x "$$BINARY" ]; then \
$(ECHO) "$(RED)Error: bifrost-http binary not found or not executable: $$BINARY$(NC)"; \
exit 1; \
fi; \
BINARY="$$(cd "$$(dirname "$$BINARY")" && pwd)/$$(basename "$$BINARY")"; \
$(ECHO) "$(GREEN)Running Warp E2E tests against $$BINARY...$(NC)"; \
./tests/e2e/api/runners/individual/run-newman-warp-tests.sh --binary "$$BINARY" $(if $(FOLDER),--folder "$(FOLDER)",)
# Quick start with example config
quick-start: ## Quick start with example config and maxim plugin
@$(ECHO) "$(GREEN)Quick starting Bifrost with example configuration...$(NC)"
@$(MAKE) dev
# Linting and formatting
lint: ## Run linter for Go code
@$(ECHO) "$(GREEN)Running golangci-lint...$(NC)"
@golangci-lint run ./...
fmt: ## Format Go code
@$(ECHO) "$(GREEN)Formatting Go code...$(NC)"
@gofmt -s -w .
@goimports -w .
format: ## Format code (Usage: make format ui)
ifeq (ui,$(filter ui,$(MAKECMDGOALS)))
@$(ECHO) "$(GREEN)Formatting UI code...$(NC)"
@cd ui && $(USE_NODE); npm run format
else
@$(ECHO) "$(YELLOW)Usage: make format ui$(NC)"
endif
ui:
@:
# Workspace helpers
setup-workspace: ## Set up Go workspace with all local modules for development
@$(ECHO) "$(GREEN)Setting up Go workspace for local development...$(NC)"
@$(ECHO) "$(YELLOW)Cleaning existing workspace...$(NC)"
@rm -f go.work go.work.sum || true
@$(ECHO) "$(YELLOW)Initializing new workspace...$(NC)"
@go work init ./cli ./core ./framework ./transports
@$(ECHO) "$(YELLOW)Adding plugin modules...$(NC)"
@for plugin_dir in ./plugins/*/; do \
if [ -d "$$plugin_dir" ] && [ -f "$$plugin_dir/go.mod" ]; then \
$(ECHO) " Adding plugin: $$(basename $$plugin_dir)"; \
go work use "$$plugin_dir"; \
fi; \
done
@$(ECHO) "$(YELLOW)Adding test command modules...$(NC)"
@for cmd_dir in ./tests/cmd/*/; do \
if [ -d "$$cmd_dir" ] && [ -f "$$cmd_dir/go.mod" ]; then \
$(ECHO) " Adding test cmd: $$(basename $$cmd_dir)"; \
go work use "$$cmd_dir"; \
fi; \
done
@$(ECHO) "$(YELLOW)Syncing workspace...$(NC)"
@go work sync
@$(ECHO) "$(GREEN)✓ Go workspace ready with all local modules$(NC)"
@$(ECHO) ""
@$(ECHO) "$(CYAN)Local modules in workspace:$(NC)"
@go list -m all | grep "github.com/maximhq/bifrost" | grep -v " v" | sed 's/^/ ✓ /'
@$(ECHO) ""
@$(ECHO) "$(CYAN)Remote modules (no local version):$(NC)"
@go list -m all | grep "github.com/maximhq/bifrost" | grep " v" | sed 's/^/ → /'
@$(ECHO) ""
@$(ECHO) "$(YELLOW)Note: go.work files are not committed to version control$(NC)"
work-init: ## Create local go.work to use local modules for development (legacy)
@$(ECHO) "$(YELLOW)⚠️ work-init is deprecated, use 'make setup-workspace' instead$(NC)"
@$(MAKE) setup-workspace
work-clean: ## Remove local go.work
@rm -f go.work go.work.sum || true
@$(ECHO) "$(GREEN)Removed local go.work files$(NC)"
# Module parameter for mod-tidy (all/core/plugins/framework/transport/tests)
MODULE ?= all
mod-tidy: ## Run go mod tidy on modules (Usage: make mod-tidy [MODULE=all|cli|core|plugins|framework|transport|tests])
@$(ECHO) "$(GREEN)Running go mod tidy...$(NC)"
@if [ "$(MODULE)" = "all" ] || [ "$(MODULE)" = "cli" ]; then \
$(ECHO) "$(CYAN)Tidying cli...$(NC)"; \
cd cli && $(if $(LOCAL),,GOWORK=off) go mod tidy && $(ECHO) "$(GREEN) ✓ cli$(NC)"; \
fi
@if [ "$(MODULE)" = "all" ] || [ "$(MODULE)" = "core" ]; then \
$(ECHO) "$(CYAN)Tidying core...$(NC)"; \
cd core && go mod tidy && $(ECHO) "$(GREEN) ✓ core$(NC)"; \
fi
@if [ "$(MODULE)" = "all" ] || [ "$(MODULE)" = "framework" ]; then \
$(ECHO) "$(CYAN)Tidying framework...$(NC)"; \
cd framework && go mod tidy && $(ECHO) "$(GREEN) ✓ framework$(NC)"; \
fi
@if [ "$(MODULE)" = "all" ] || [ "$(MODULE)" = "transport" ]; then \
$(ECHO) "$(CYAN)Tidying transports...$(NC)"; \
cd transports && go mod tidy && $(ECHO) "$(GREEN) ✓ transports$(NC)"; \
fi
@if [ "$(MODULE)" = "all" ] || [ "$(MODULE)" = "plugins" ]; then \
$(ECHO) "$(CYAN)Tidying plugins...$(NC)"; \
for plugin_dir in ./plugins/*/; do \
if [ -d "$$plugin_dir" ] && [ -f "$$plugin_dir/go.mod" ]; then \
plugin_name=$$(basename $$plugin_dir); \
cd $$plugin_dir && go mod tidy && cd ../.. && $(ECHO) "$(GREEN) ✓ plugins/$$plugin_name$(NC)"; \
fi; \
done; \
fi
@if [ "$(MODULE)" = "all" ] || [ "$(MODULE)" = "tests" ]; then \
$(ECHO) "$(CYAN)Tidying test command modules...$(NC)"; \
for cmd_dir in ./tests/cmd/*/; do \
if [ -d "$$cmd_dir" ] && [ -f "$$cmd_dir/go.mod" ]; then \
cmd_name=$$(basename $$cmd_dir); \
cd $$cmd_dir && go mod tidy && cd ../../.. && $(ECHO) "$(GREEN) ✓ tests/cmd/$$cmd_name$(NC)"; \
fi; \
done; \
fi
@$(ECHO) ""
@$(ECHO) "$(GREEN)✓ go mod tidy complete$(NC)"
.PHONY: test-cli-integration
test-cli-integration: ## Build the CLI and test configuration and management commands (Python 3.11+)
@cd cli && python3 scripts/testconfigure.py -v
test-cli: install-gotestsum ## Run CLI tests
@$(ECHO) "$(GREEN)Running CLI tests...$(NC)"
@mkdir -p $(TEST_REPORTS_DIR)
@cd cli && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../$(TEST_REPORTS_DIR)/cli.xml \
-- ./...
# The CLI harness always runs `go test -v`, because without it `go test` buffers
# the whole package's stdout AND stderr and discards it when the package passes -
# which would swallow the harness's own per-cell RUNNING/PASS/FAIL lines and its
# progress table, leaving a multi-minute run with no output at all.
#
# -v then emits its own per-subtest scaffolding, which for this matrix is three
# lines (=== RUN / PAUSE / CONT) per cell plus a skip line per gated scenario -
# far more volume than the results themselves. This strips exactly that, leaving
# every diagnostic (t.Fatal bodies, panics, build errors) intact. The two skip
# messages matched here are the harness's own strings from clis_test.go, not
# go test's, so this cannot drift out from under a Go release.
#
# awk rather than grep: awk exits 0 even when it emits nothing (grep exits 1,
# which `set -o pipefail` would turn into a spurious failure), and fflush keeps
# the stream line-buffered so results appear as cells finish rather than in one
# dump at the end. VERBOSE=1 bypasses the filter entirely.
# The `--- PASS/FAIL/SKIP` summary is emitted once per nesting level and indented
# to match, so the pattern allows leading whitespace - anchoring at column 0
# would strip only the outermost line and leave the rest of the tree.
#
# The package-result line is `FAIL<TAB>pkg<TAB>0.13s`, matched on the TAB
# specifically: the harness's own status column is space-padded ("FAIL" then
# spaces), so a `^FAIL[ \t]` pattern would swallow every failed cell's result -
# the single most important line in the output.
CLI_HARNESS_FILTER = awk '/^=== /||/^[ \t]*--- (PASS|FAIL|SKIP)[: ]/||/^(PASS|FAIL)$$/||/^ok[ \t]/||/^FAIL\t/||/unsupported for /||/not configured in bifrost/{next} {print; fflush()}'
test-live: install-gotestsum ## Run GPT Live e2e tests against a running gateway (Usage: make test-live [TESTCASE=TestName|PATTERN=substring] [LIVE_UPSTREAM=fake|real] [BIFROST_BASE_URL=http://localhost:8080]; fake mode: start the gateway with APP_DIR=$$(pwd)/tests/live)
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running GPT Live e2e tests ($${LIVE_UPSTREAM:-fake} upstream)...$(NC)"; \
mkdir -p $(TEST_REPORTS_DIR); \
if [ -n "$(PATTERN)" ] && [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(RED)Error: PATTERN and TESTCASE are mutually exclusive$(NC)"; exit 1; \
fi; \
RUN_FLAG=""; \
if [ -n "$(TESTCASE)" ]; then RUN_FLAG="-run ^$(TESTCASE)$$"; elif [ -n "$(PATTERN)" ]; then RUN_FLAG="-run .*$(PATTERN).*"; \
elif [ "$${LIVE_UPSTREAM:-fake}" = "real" ]; then RUN_FLAG="-run ^Test(Real|SDK)"; fi; \
REPORT_FILE="$(TEST_REPORTS_DIR)/live.xml"; \
cd tests/live && GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../$$REPORT_FILE \
-- -v -timeout 900s $$RUN_FLAG; \
STATUS=$$?; \
cd ../..; \
$(MAKE) cleanup-junit-xml REPORT_FILE=$$REPORT_FILE; \
exit $$STATUS
test-live-long: install-gotestsum ## Run the long GPT Live sessions: 17 min on the fake, 15 min per transport on the real upstream (Usage: make test-live-long [TESTCASE=TestName] [LIVE_UPSTREAM=fake|real] [BIFROST_BASE_URL=http://localhost:8080])
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running long GPT Live sessions ($${LIVE_UPSTREAM:-fake} upstream)...$(NC)"; \
mkdir -p $(TEST_REPORTS_DIR); \
RUN_FLAG="-run Long"; \
if [ -n "$(TESTCASE)" ]; then RUN_FLAG="-run ^$(TESTCASE)$$"; fi; \
REPORT_FILE="$(TEST_REPORTS_DIR)/live-long.xml"; \
cd tests/live && LIVE_LONG=1 GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../$$REPORT_FILE \
-- -v -timeout 90m $$RUN_FLAG; \
STATUS=$$?; \
cd ../..; \
$(MAKE) cleanup-junit-xml REPORT_FILE=$$REPORT_FILE; \
exit $$STATUS
test-realtime: install-gotestsum
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running Realtime API e2e tests...$(NC)"; \
mkdir -p $(TEST_REPORTS_DIR); \
if [ -n "$(PATTERN)" ] && [ -n "$(TESTCASE)" ]; then \
$(ECHO) "$(RED)Error: PATTERN and TESTCASE are mutually exclusive$(NC)"; exit 1; \
fi; \
RUN_FLAG=""; \
if [ -n "$(TESTCASE)" ]; then RUN_FLAG="-run ^$(TESTCASE)$$"; elif [ -n "$(PATTERN)" ]; then RUN_FLAG="-run .*$(PATTERN).*"; fi; \
REPORT_FILE="$(TEST_REPORTS_DIR)/realtime.xml"; \
cd tests/realtime && REALTIME_REQUIRE_GATEWAY=1 GOWORK=off gotestsum \
--format=$(GOTESTSUM_FORMAT) \
--junitfile=../../$$REPORT_FILE \
-- -v -count 1 -timeout 600s $$RUN_FLAG; \
STATUS=$$?; \
cd ../..; \
$(MAKE) cleanup-junit-xml REPORT_FILE=$$REPORT_FILE; \
exit $$STATUS
run-cli-harness-test: ## Run the Claude Code + Codex + OpenCode E2E harness (non-interactive, multi-turn JSON streams). Prints one line per cell plus a progress table; MIRROR=1 adds the raw CLI stream, VERBOSE=1 adds go test -v. Usage: make run-cli-harness-test [TESTCASE='TestCLIs/...'] [CLI=claude|codex|opencode] [PROVIDER=openai|anthropic|azure|gemini|bedrock|vertex] [MODEL=<id-substring>] [SCENARIO=simple-chat|conversation-memory|...] [PARALLEL=4] [BASE_URL=http://localhost:8080] [API_KEY=...] [TIMEOUT=60m] [MIRROR=1] [VERBOSE=1] [QUIET=1]
@$(EXPOSE_ENV); \
$(ECHO) "$(GREEN)Running CLI harness E2E tests...$(NC)"; \
BASE_URL_VAL="$${BASE_URL:-$(BASE_URL)}"; BASE_URL_VAL="$${BASE_URL_VAL:-http://localhost:8080}"; \
PARALLEL_VAL="$${PARALLEL:-$(PARALLEL)}"; PARALLEL_VAL="$${PARALLEL_VAL:-4}"; \
$(ECHO) "$(CYAN) Bifrost: $$BASE_URL_VAL$(NC)"; \
$(ECHO) "$(CYAN) Parallel: $$PARALLEL_VAL$(NC)"; \
if [ -n "$(PROVIDER)" ]; then \
case "$(PROVIDER)" in openai|anthropic|azure|gemini|bedrock|vertex) ;; \
*) $(ECHO) "$(RED)Error: invalid PROVIDER=$(PROVIDER). Use one of: openai, anthropic, azure, gemini, bedrock, vertex$(NC)"; exit 1 ;; \
esac; \
$(ECHO) "$(CYAN) Provider: $(PROVIDER)$(NC)"; \
fi; \
if ! curl -s -o /dev/null -w "%{http_code}" "$$BASE_URL_VAL/api/providers" | grep -qE '^[2-4]'; then \
$(ECHO) "$(RED)Error: Bifrost not reachable at $$BASE_URL_VAL$(NC)"; \
$(ECHO) "$(YELLOW)Start Bifrost first (e.g. make dev) or pass BASE_URL=...$(NC)"; \
exit 1; \
fi; \
for bin in claude codex opencode; do \
if [ "$(CLI)" = "" ] || [ "$(CLI)" = "$$bin" ] || { [ "$$bin" = opencode ] && [ "$(CLI)" = opencode-responses ]; }; then \
if ! command -v $$bin >/dev/null 2>&1; then \
$(ECHO) "$(YELLOW)Warning: $$bin not on PATH; matrix cells for $$bin will fail.$(NC)"; \
$(ECHO) "$(YELLOW) Install: npm i -g $$( [ $$bin = claude ] && echo @anthropic-ai/claude-code || { [ $$bin = codex ] && echo @openai/codex || echo opencode-ai; } )$(NC)"; \
fi; \
fi; \
done; \
if [ -n "$(TESTCASE)" ]; then \
RUN_PARTS="$(TESTCASE)"; \
else \
RUN_PARTS="TestCLIs"; \
if [ -n "$(CLI)" ]; then RUN_PARTS="$$RUN_PARTS/$(CLI)"; else RUN_PARTS="$$RUN_PARTS/[^/]+"; fi; \
if [ -n "$(PROVIDER)" ]; then RUN_PARTS="$$RUN_PARTS/$(PROVIDER)"; else RUN_PARTS="$$RUN_PARTS/[^/]+"; fi; \
if [ -n "$(MODEL)" ]; then RUN_PARTS="$$RUN_PARTS/[^/]*$(MODEL)[^/]*"; else RUN_PARTS="$$RUN_PARTS/[^/]+"; fi; \
if [ -n "$(SCENARIO)" ]; then RUN_PARTS="$$RUN_PARTS/$(SCENARIO)"; fi; \
fi; \
$(ECHO) "$(CYAN) Filter: $$RUN_PARTS$(NC)"; \
if [ -n "$(MODEL)" ]; then $(ECHO) "$(CYAN) Model: $(MODEL) (substring filter)$(NC)"; fi; \
set -o pipefail; \
cd tests/e2e/clis && \
BIFROST_BASE_URL="$$BASE_URL_VAL" \
BIFROST_API_KEY="$${API_KEY:-$(API_KEY)}" \
BIFROST_E2E_CLIS_QUIET="$(QUIET)" \
BIFROST_E2E_CLIS_MIRROR="$(MIRROR)" \
MODEL="$(MODEL)" \
GOWORK=off go test \
-count=1 \
-timeout=$${TIMEOUT:-$(if $(TIMEOUT),$(TIMEOUT),60m)} \
-parallel="$$PARALLEL_VAL" \
-run "^$$RUN_PARTS$$" \
-v \
./... $(if $(VERBOSE),,| $(CLI_HARNESS_FILTER))
cli-harness-report: ## Regenerate tests/e2e/clis/reports/index.html from existing reports/*.json, without running any tests (free, instant). Usage: make cli-harness-report
@$(ECHO) "$(GREEN)Rendering CLI harness report from existing reports/*.json...$(NC)"
@cd tests/e2e/clis && GOWORK=off go test -run "^TestRenderReport$$" -v ./...
# The harness runner unit tests are plain node scripts with no framework (the
# tests/e2e/api dir configures no test runner), so nothing discovers them
# automatically. Without a target they are only ever run by hand, which is how
# they rot - and they cover the parsing that produces the status table's numbers.
test-harness-runner-lib: ## Run the provider-harness runner unit tests (tests/e2e/api/runners/lib/*.test.mjs). No network, no Bifrost, no credentials.
@$(ECHO) "$(GREEN)Running provider-harness runner lib tests...$(NC)"
@$(USE_NODE); RC=0; \
for t in tests/e2e/api/runners/lib/*.test.mjs; do \
printf '%s\n' "$(CYAN)-> $$t$(NC)"; \
node "$$t" || RC=1; \
done; \
if [ "$$RC" -ne 0 ]; then $(ECHO) "$(RED)harness runner lib tests failed$(NC)"; else $(ECHO) "$(GREEN)harness runner lib tests passed$(NC)"; fi; \
exit $$RC
# Video bills at SETTLEMENT, minutes after the POST returns, so neither newman
# (no way to wait for an out-of-band row) nor a Go test (no provider) can check
# the figure. This runner does the whole submit -> poll -> settle -> assert loop.
run-video-costing-test: ## Verify async video jobs bill correctly against real providers (Usage: make run-video-costing-test ARGS="--group Runware --seed-pricing"). Needs a running Bifrost with video provider keys.
@$(ECHO) "$(GREEN)Running video costing checks...$(NC)"
@$(USE_NODE); cd tests/e2e/api && node runners/run-video-costing.mjs $(ARGS)
list-video-costing-cases: ## List the video costing cases and which checklist line each covers. No network, no Bifrost.
@$(USE_NODE); cd tests/e2e/api && node runners/run-video-costing.mjs --list
# Named target rather than documentation telling people to type SMOKE=1: a smoke
# set nobody can invoke in one word does not get used before a release.
smoke-provider-harness-test: ## Run the curated ~100-request provider-harness smoke set (tests/e2e/api/collections/smoke-manifest.json). Same flags as run-provider-harness-test; equivalent to SMOKE=1.
@$(MAKE) run-provider-harness-test SMOKE=1
# Versions pinned to match the CI installs in .github/workflows/release-pipeline.yml
# (test-core, test-api-integrations, test-docker-image-*). Keep them in sync.
NEWMAN_VERSION ?= 6.2.1
NEWMAN_HTMLEXTRA_VERSION ?= 1.23.1
# Every provider fork the harness knows how to run. Also the provider set the
# status table lists, including the deferred cache-parity pass, so it lives in
# one place rather than being restated per newman invocation.
HARNESS_PROVIDERS := openai anthropic bedrock bedrock_mantle gemini vertex azure passthrough openrouter huggingface groq xai mistral cohere deepseek baseten
# Second parallelism axis. Each provider fork is sharded again by modality class so the run is not
# bound by one provider's whole sequential item list: openai alone is ~1264 requests, and its
# largest class is ~268. filter-collection.mjs --class assigns every item to exactly one of these
# (first match wins, "other" is the catch-all), so the shards partition the fork rather than
# overlapping it. Order here is only the launch order; the priority order lives in CLASS_ORDER.
#
# Listed SLOWEST-FIRST, measured rather than guessed: in a full sweep the last six shards to finish
# were reasoning (x4 providers), tools and chat, with reasoning trailing the median shard by ~20
# minutes. Launch order only matters once HARNESS_JOBS actually binds, which it now does - the
# sub-shard axis below pushes the grid past the cap - and when it binds, a slow shard that starts
# last sets the wall clock all by itself.
HARNESS_CLASSES := reasoning tools chat streaming json vision other audio embeddings image-gen
# Third parallelism axis: how many sub-shards each modality class is split into, via
# filter-collection.mjs --shard <k>/<n>. The class axis alone cannot flatten the tail, because the
# expensive classes are expensive PER REQUEST rather than per row count: a reasoning row costs ~8s
# against a chat row's ~1s, so "openai reasoning" is a ~21 minute serial run at 161 rows while
# "anthropic tools" clears 226 rows in a fraction of that. Splitting the slow classes cuts their
# serial length by n while the fast ones stay single shards, so the grid grows only where it buys
# wall clock. Counts are set from that same measured finish order. A class not listed here is 1.
# Set SUBSHARDS=0 to collapse this axis and get the previous one-shard-per-class behaviour.
#
# FALLBACK ONLY as of the shard planner. When a timing table exists (HARNESS_TIMINGS, written at
# the end of every run by build-harness-timings.mjs), filter-collection.mjs --plan sizes each
# (provider, class) cell from measured cost and this table is not consulted. It still runs a fresh
# checkout's first sweep, a run with a corrupt table, and SUBSHARDS=0.
#
# It is a fallback rather than the mechanism because one number per CLASS cannot fit eight
# providers: "reasoning" is ~30 minutes for openai and seconds for embeddings-heavy providers, so
# any single count is badly wrong for one of them. Measured: a full sweep spent 77% of its 10.7
# minute wall clock on the last 16% of its requests, dropping from 98 concurrent shards to 1 while
# openai--reasoning-s4 finished alone. Sizing from cost puts the same work in 2.8 minutes.
HARNESS_SUBSHARDS := reasoning=4 tools=3 chat=3 streaming=3 json=2 vision=2
# Wall clock a single sub-shard aims for, in seconds. The planner splits each cell into
# ceil(cellCost / target) shards, capped by its row count. Simulated against a measured sweep at
# 120/150/180/240s: 150 is the smallest target that still reaches the floor set by the slowest
# single request (2.8 min worst shard) without buying shards that cannot help - 120s costs 21 more
# node processes for the same 2.8 min, 180s saves 16 processes for +0.2 min.
#
# Lower it to trade processes for wall clock, raise it to trade the other way. It cannot push the
# sweep below its slowest single request (~170s today), because a request is not divisible.
HARNESS_SHARD_TARGET ?= 150
# Where the measured per-request timing table lives. Refreshed at the end of every run from that
# run's newman reports, and merged onto whatever was there so a scoped run refines its own rows
# instead of erasing the seven providers it did not touch. The committed baseline at
# tests/e2e/api/collections/harness-timings.json is consulted when this file is absent, which is
# what lets a fresh checkout and CI balance on their first sweep rather than their second.
HARNESS_TIMINGS ?= tmp/harness-timings.json
# Cap on concurrently running newman shards. The provider x class x sub-shard grid is ~168 cells
# now that HARNESS_SUBSHARDS splits the slow classes, so unlike before the cap genuinely binds and
# the launcher blocks - which is exactly why HARNESS_CLASSES is ordered slowest-first, so the long
# shards hold slots from the start instead of queueing behind a hundred cheap ones. The cap is kept
# rather than removed because the grid grows with HARNESS_PROVIDERS, HARNESS_CLASSES and
# HARNESS_SUBSHARDS together, and an unbounded loop would fork whatever their product becomes.
# Lower it if a provider starts returning 429s - analyze-failures.mjs does file those as
# rate_limit rather than as defects, but they still fail the run.
HARNESS_JOBS ?= 100
# Echoes are suppressed under CI so run-provider-harness-test, which takes this
# as a prerequisite, really does emit nothing but its status table. Install
# failures still surface: npm's own stderr is untouched and the recipe still fails.
install-newman: ## Install newman + htmlextra reporter if not already installed (pinned via NEWMAN_VERSION / NEWMAN_HTMLEXTRA_VERSION)
@$(USE_NODE); which newman > /dev/null 2>&1 || ([ -n "$$CI" ] || $(ECHO) "$(YELLOW)Installing newman@$(NEWMAN_VERSION)...$(NC)"; npm install -g newman@$(NEWMAN_VERSION))
@$(USE_NODE); npm list -g newman-reporter-htmlextra > /dev/null 2>&1 || ([ -n "$$CI" ] || $(ECHO) "$(YELLOW)Installing newman-reporter-htmlextra@$(NEWMAN_HTMLEXTRA_VERSION)...$(NC)"; npm install -g newman-reporter-htmlextra@$(NEWMAN_HTMLEXTRA_VERSION))
@[ -n "$$CI" ] || $(ECHO) "$(GREEN)Newman + htmlextra are ready$(NC)"
run-provider-harness-test: $(if $(HELP),,install-newman) ## Run the Bifrost provider-harness Postman collection. HELP=1 prints full parameter docs. Filter via PROVIDER=openai|anthropic|bedrock|gemini|vertex|azure|passthrough|openrouter|huggingface, FEATURE="<kw>" or FEATURE="<kw1>,<kw2>" (AND across substrings; matches request name/URL/body), RERUN_FAILED=1 (re-run only items that failed last run). INCLUDE_PREVIEW=1 to run [PREVIEW]-tagged account/region-scoped cases. SKIP_STREAM_CANCEL=1 skips stream cancellation probes. USE_INFISICAL=1 to source from Infisical (Usage: make run-provider-harness-test [HELP=1] [PROVIDER=anthropic] [FEATURE="web search"] [FEATURE="cross-cut,structured output"] [RERUN_FAILED=1] [INCLUDE_PREVIEW=1] [BASE_URL=...] [FOLDER="..."] [ENV_FILE=...] [VIEWER_PORT=8090] [CI=1])
@if [ -n "$(HELP)" ]; then \
printf '\n%s\n' "$(CYAN)run-provider-harness-test - Bifrost provider harness runner$(NC)"; \
printf '%s\n\n' "Runs the Bifrost provider-harness Postman collection through newman, with optional filtering."; \
printf '%s\n\n' "Includes §8 Criss-Cross: endpoint-shape × model-provider × modality matrix (chat, streaming, embeddings, audio, image gen, tools, vision, JSON, reasoning)."; \
printf '%s\n' "$(YELLOW)PARAMETERS$(NC)"; \
printf ' %-18s %s\n' "HELP=1" "Print this help and exit (no Bifrost or network activity)."; \
printf ' %-18s %s\n' "PROVIDER=<name>" "Filter requests by provider. One of: $(HARNESS_PROVIDERS)."; \
printf ' %-18s %s\n' "" " Matches via PROVIDER_KEYWORDS in tests/e2e/api/runners/filter-collection.mjs (loose name/body substring)."; \
printf ' %-18s %s\n' "FEATURE=\"<kw>\"" "Filter by case-insensitive keyword(s) against the full request JSON (name + URL + body + ancestor folder names)."; \
printf ' %-18s %s\n' "" " Single: FEATURE=\"web search\". Multi-keyword AND (comma-separated): FEATURE=\"cross-cut,structured output\"."; \
printf ' %-18s %s\n' "" " \"cross-cut\" is a structural keyword - matches any row routed through unified /v1/chat/completions with a provider/model body, regardless of name."; \
printf ' %-18s %s\n' "RERUN_FAILED=1" "Re-run only requests that failed in the prior run (reads tmp/newman-report.json)."; \
printf ' %-18s %s\n' "" " Composes with PROVIDER and FEATURE (predicates AND together)."; \
printf ' %-18s %s\n' "BASE_URL=<url>" "Bifrost gateway URL (default: http://localhost:8080). Skips auto-start if /health responds."; \
printf ' %-18s %s\n' "APP_DIR=<dir>" "Config dir passed to 'make dev' if Bifrost isn't already running (default: tests/integrations/python)."; \
printf ' %-18s %s\n' "FOLDER=\"<name>\"" "Scope to a single Postman folder (substring match, e.g. \"8. Cross-Model\"). Pre-filters (like PROVIDER/FEATURE) AND is"; \
printf ' %-18s %s\n' "" " passed to newman's own --folder as the final scope. In parallel mode (default), provider forks with zero items"; \
printf ' %-18s %s\n' "" " in that folder are skipped cleanly instead of forked-then-failed - use with PROVIDER=<one> to run a single fork."; \
printf ' %-18s %s\n' "ENV_FILE=<path>" "Postman environment JSON with real keys (kept out of git)."; \
printf ' %-18s %s\n' "VIEWER_PORT=N" "Port for the interactive HTML viewer (default: 8090). Ignored if CI=1."; \
printf ' %-18s %s\n' "CI=1" "CI mode: skip the interactive viewer, emit artifacts only. Prints a one-line heartbeat every"; \
printf ' %-18s %s\n' "" " MONITOR_INTERVAL seconds and the provider status table (provider x total/pass/failed) exactly ONCE,"; \
printf ' %-18s %s\n' "" " at the end - an Actions log is append-only, so reprinting the table left hundreds of stale copies."; \
printf ' %-18s %s\n' "" " Everything the run would"; \
printf ' %-18s %s\n' "" " otherwise have echoed goes to tmp/harness-quiet.log; newman's own output goes to tmp/newman-cli*.log."; \
printf ' %-18s %s\n' "MONITOR_INTERVAL=N" "Seconds between heartbeat lines in CI mode (default 5, clamped to 5..2700)."; \
printf ' %-18s %s\n' "MONITOR_TABLE_REPRINT=1" "Restore the old CI behaviour of reprinting the whole table on every interval. Debugging only."; \
printf ' %-18s %s\n' "INCLUDE_PREVIEW=1" "Run [PREVIEW]-tagged requests (account/region-scoped: vector stores, cached content, MCP servers, preview-model deployments). Off by default."; \
printf ' %-18s %s\n' "INCLUDE_SKIP=1" "Run [SKIP]-tagged criss-cross cells (provider+modality pairs that return NewUnsupportedOperationError by design, e.g., anthropic embeddings, bedrock audio). Off by default."; \
printf ' %-18s %s\n' "SMOKE=1" "Run only the curated smoke set in tests/e2e/api/collections/smoke-manifest.json (~100 requests"; \
printf ' %-18s %s\n' "" " across all $(words $(HARNESS_PROVIDERS)) providers) instead of the full ~1900-request sweep. SMOKE=<path> uses a different manifest."; \
printf ' %-18s %s\n' "" " Rows are matched by (folder, name) against the AUGMENTED collection, which is the only way to reach the"; \
printf ' %-18s %s\n' "" " generated cache-parity rows - they do not exist in provider-harness.json and cannot be tagged in place."; \
printf ' %-18s %s\n' "" " Forces the deferred cache-parity pass ON: a third of the smoke set is cache parity, and the default"; \
printf ' %-18s %s\n' "" " deferral only fires on an unfiltered run. Composes with PROVIDER; RERUN_FAILED wins and skips the defer."; \
printf ' %-18s %s\n' "PARALLEL=0" "Disable per-provider parallelism (default: ON). When ON, forks one newman per provider ($(HARNESS_PROVIDERS)) concurrently; reports merged into tmp/newman-report.json. The htmlextra report is only emitted in sequential mode (PARALLEL=0)."; \
printf ' %-18s %s\n' "CLASS_SHARDS=0" "Collapse the second parallelism axis. By default each provider fork is sharded again by modality"; \
printf ' %-18s %s\n' "" " class (streaming, tools, chat, reasoning, json, vision, other, audio, embeddings, image-gen) so the"; \
printf ' %-18s %s\n' "" " run is not bound by one provider's whole sequential list - openai alone is ~1264 requests and its"; \
printf ' %-18s %s\n' "" " largest class is ~268. filter-collection.mjs --class puts every request in exactly one class, so the"; \
printf ' %-18s %s\n' "" " shards partition the fork instead of overlapping it. Shard artifacts are named <provider>--<class>."; \
printf ' %-18s %s\n' "SUBSHARDS=0" "Collapse the third parallelism axis. By default each (provider, class) cell is split again via"; \
printf ' %-18s %s\n' "" " filter-collection.mjs --shard <k>/<n>, because the slow classes are slow PER REQUEST (~8s for a"; \
printf ' %-18s %s\n' "" " reasoning row vs ~1s for chat), so the class axis alone leaves a long tail. How many sub-shards"; \
printf ' %-18s %s\n' "" " and which rows go in each are both MEASURED, from HARNESS_TIMINGS - see the two knobs below."; \
printf ' %-18s %s\n' "" " Sub-shard artifacts are named <provider>--<class>-s<k>."; \
printf ' %-18s %s\n' "HARNESS_SHARD_TARGET=N" ""; \
printf ' %-18s %s\n' "" " Wall clock a single sub-shard aims for, in seconds (default 150). Each cell is split into"; \
printf ' %-18s %s\n' "" " ceil(cellCost / target) shards, capped by its row count, and filled by greedy longest-"; \
printf ' %-18s %s\n' "" " processing-time rather than by row position. Lower it to trade node processes for wall clock."; \
printf ' %-18s %s\n' "" " It cannot push a sweep below its slowest single request (~170s today) - a request is not"; \
printf ' %-18s %s\n' "" " divisible. Measured effect: worst shard 9.4min -> 2.8min, using fewer shards than the old table."; \
printf ' %-18s %s\n' "HARNESS_TIMINGS=path" ""; \
printf ' %-18s %s\n' "" " Per-request timing table (default tmp/harness-timings.json), refreshed at the end of every run"; \
printf ' %-18s %s\n' "" " and MERGED onto what was there, so a scoped run refines its own rows without erasing the"; \
printf ' %-18s %s\n' "" " providers it never ran. Falls back to tests/e2e/api/collections/harness-timings.json, which is"; \
printf ' %-18s %s\n' "" " what lets a fresh checkout and CI balance on their FIRST sweep. With no table at all the run"; \
printf ' %-18s %s\n' "" " falls back to the static HARNESS_SUBSHARDS counts and positional slicing - slower, never wrong."; \
printf ' %-18s %s\n' "HARNESS_JOBS=N" "Cap on concurrently running newman shards (default 100). The grid is ~168 live cells with sub-shards"; \
printf ' %-18s %s\n' "" " on, so the cap does block - which is why HARNESS_CLASSES is ordered slowest-first, to keep the long"; \
printf ' %-18s %s\n' "" " shards holding slots from the start. Lower it if a provider starts returning 429s."; \
printf ' %-18s %s\n' "HARNESS_MAX_REQUESTS=N" ""; \
printf ' %-18s %s\n' "" " Approved ceiling on paid requests for this run. Checked with each newman launch's EXACT filtered"; \
printf ' %-18s %s\n' "" " request count (main shards, 429 replays, the deferred cache-parity pass, sequential mode) before it"; \
printf ' %-18s %s\n' "" " starts: a launch that would cross the cap is refused, so the live total never exceeds N. Running"; \
printf ' %-18s %s\n' "" " shards drain and the report merges as usual; the target exits 3. Stream-cancellation probes are"; \
printf ' %-18s %s\n' "" " never sent under a cap (their count is not known up front). Unset = no cap (default)."; \
printf ' %-18s %s\n' "RETRY_429=N" "Max transient-failure retry attempts per shard (default 3; 0 disables). Covers 429 plus the two"; \
printf ' %-18s %s\n' "" " overload codes - 503 (OpenAI 'engine is currently overloaded') and 529 (Anthropic"; \
printf ' %-18s %s\n' "" " overloaded_error, which is its whole equivalent of 503; its error table has no 503). Other 5xx"; \
printf ' %-18s %s\n' "" " are NOT retried: a 500/502 is ambiguous between an upstream blip and a Bifrost defect, and"; \
printf ' %-18s %s\n' "" " finding the second kind is what this sweep is for. After the main pass, any shard that failed"; \
printf ' %-18s %s\n' "" " on one of those replays ONLY its affected rows, after waiting max(retry-after) across them"; \
printf ' %-18s %s\n' "" " or an exponential 5/10/20s when the provider sent no header (capped at 120s). A shard with any"; \
printf ' %-18s %s\n' "" " non-retryable failure stays failed however well the retry went, so a real defect is never masked."; \
printf ' %-18s %s\n' "" " Retry reports merge LAST, so a successful attempt supersedes its own failure in tmp/newman-report.json."; \
printf ' %-18s %s\n' "SHARD_LINES=0" "Drop the per-shard completion lines (<shard> N total/pass/fail) and show only the provider table."; \
printf ' %-18s %s\n' "SKIP_STREAM_CANCEL=1" "Skip the post-Newman stream-abort probes that verify server-side cancellation on client disconnect."; \
printf ' %-18s %s\n' "NONSTREAM_TRIALS=6" "Non-streaming abort trials per provider in those probes; every trial must leave a terminal log row (#6972). Default 6."; \
printf ' %-18s %s\n' "HARNESS_SERVER_CWD" "Server working directory for relative logs_store SQLite paths (default: transports/bifrost-http, matching make dev). BIFROST_LOGS_DB_URL overrides config resolution."; \
printf ' %-18s %s\n' "DB_VERIFY=0" "Disable the dbverify reporter (ON by default). When on, [Costing]/[Accounting] requests assert the logs DB cost matches the getbifrost.ai/datasheet-computed cost (resolves DB from APP_DIR/config.json or BIFROST_LOGS_DB_URL); skips gracefully if no logs DB is reachable."; \
printf ' %-18s %s\n' "USE_INFISICAL=1" "Source secrets from Infisical CLI ('infisical export --path /local --format dotenv') instead of .env."; \
printf ' %-18s %s\n' "VERTEX_GCS_BUCKET" "Env-sourced (.env/Infisical): GCS bucket for Vertex file ops (forwarded to Newman as vertexGcsBucket)."; \
printf ' %-18s %s\n' "VERTEX_GCS_PREFIX" "Env-sourced: GCS object prefix for Vertex file ops (forwarded as vertexGcsPrefix)."; \
printf ' %-18s %s\n' "CHATGPT_ACCESS_TOKEN" "Env-sourced: ChatGPT sign-in access token for the Codex passthrough cases (forwarded as chatgptAccessToken; they skip when unset)."; \
printf '\n%s\n' "$(CYAN)Token Parity Matrix$(NC) ('Cross-Cut Round 33', generated - FOLDER or FEATURE=\"token parity\" to target it)"; \
printf ' %s\n' "Runs the same 3-round conversation directly against each provider AND through Bifrost, asserts usage lands in the same"; \
printf ' %s\n' "range, and writes tmp/harness-token-parity.md. Reuses the SAME env vars tests/integrations/python/config.json already reads"; \
printf ' %s\n' "for Bifrost's own provider config (sourced via .env/Infisical, same as everything else in this target - no separate setup):"; \
printf ' %s\n' " openai/anthropic/gemini: OPENAI_API_KEY / ANTHROPIC_API_KEY / GEMINI_API_KEY -> {{openaiKey}}/{{anthropicKey}}/{{genaiKey}}."; \
printf ' %s\n' " bedrock: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_REGION (same creds Bifrost's own bedrock provider uses), plus {{bedrockModel}} via ENV_FILE."; \
printf ' %s\n' " vertex: VERTEX_PROJECT_ID / GOOGLE_LOCATION for project/region, plus gcloud CLI on PATH + authenticated ('gcloud auth login') -"; \
printf ' %s\n' " the Makefile mints a fresh OAuth access token per run (VERTEX_CREDENTIALS in config.json is a service-account key,"; \
printf ' %s\n' " not a bearer token Postman can use directly, so this is the one leg that still needs its own auth step)."; \
printf ' %s\n' " ENV_FILE still overrides any of the above if set - these are just sane defaults from what's already injected."; \
printf ' %s\n' " Skipped cells (provider genuinely can't do it, e.g. OpenAI+PDF, Anthropic+audio/video) are listed in the folder description."; \
printf '\n%s\n' "$(CYAN)Cache Parity Matrices$(NC) ('Cross-Cut Rounds 34 + 35', generated - pick 'cache-parity' in the menu or FEATURE=\"cache\")"; \
printf ' %s\n' "Round 34 (Mid-Conversation System Cache-Anchor Parity) runs the same conversation on three legs - direct Bedrock Converse"; \
printf ' %s\n' " over SigV4, Bifrost /anthropic/v1/messages, Bifrost /openai/v1/responses - and diffs the cache read/write counts. The direct"; \
printf ' %s\n' " leg is the baseline: a Bifrost leg reading materially less on identical bytes means a dropped cache breakpoint."; \
printf ' %s\n' "Round 35 (Cross-Provider Prompt-Cache Matrix) sweeps provider x model x explicit/implicit mechanism through Bifrost."; \
printf ' %s\n' "Both write tmp/harness-cache-parity.md. Credentials reuse the same AWS_*/provider keys as the Round 33 token-parity matrix."; \
printf ' %s\n' "Selecting 'cache-parity' in the interactive menu does NOT drop the run to PARALLEL=0: those rows are pulled out of the main"; \
printf ' %s\n' " pass (which keeps its per-provider parallelism) and replayed afterwards in one sequential newman, because they match every"; \
printf ' %s\n' " provider fork and would otherwise run up to six times each. A FEATURE whose comma-separated keywords include"; \
printf ' %s\n' " 'cache-parity' (matched case-insensitively, like FEATURE itself) gets the same treatment, and the whole FEATURE and"; \
printf ' %s\n' " FOLDER predicate is forwarded to that pass so the scope still applies. RERUN_FAILED=1 does not defer, because the"; \
printf ' %s\n' " failed-row selection comes from the main pass it would skip. A keyword that merely contains the key, such as"; \
printf ' %s\n' " FEATURE=\"cache\", is not the key and has no such protection - add PARALLEL=0 yourself in that case."; \
printf '\n%s\n' "$(YELLOW)EXAMPLES$(NC)"; \
printf ' %s\n' "make run-provider-harness-test HELP=1"; \
printf ' %s\n' "make run-provider-harness-test # full provider sweep"; \
printf ' %s\n' "make smoke-provider-harness-test # curated ~100-request smoke set (all providers)"; \
printf ' %s\n' "make run-provider-harness-test SMOKE=1 PROVIDER=bedrock # the smoke set, bedrock rows only"; \
printf ' %s\n' "make run-provider-harness-test PROVIDER=bedrock # bedrock-only"; \
printf ' %s\n' "make run-provider-harness-test FEATURE=\"web search\" # all providers, web-search entries"; \
printf ' %s\n' "make run-provider-harness-test FEATURE=\"cross-cut,structured output\" # AND of substrings"; \
printf ' %s\n' "make run-provider-harness-test RERUN_FAILED=1 # triage iteration loop"; \
printf ' %s\n' "make run-provider-harness-test PROVIDER=anthropic RERUN_FAILED=1 # anthropic failures only"; \
printf ' %s\n' "make run-provider-harness-test PROVIDER=passthrough # passthrough sweep (incl. Bedrock SigV4)"; \
printf ' %s\n' "make run-provider-harness-test CI=1 USE_INFISICAL=1 # CI run with Infisical secrets"; \
printf '\n%s\n' "$(YELLOW)ARTIFACTS$(NC)"; \
printf ' %-30s %s\n' "tmp/newman-report.json" "Machine-readable run report (used by RERUN_FAILED and the analyzer)."; \
printf ' %-30s %s\n' "tmp/newman-cli.log" "Captured newman CLI output (stdout+stderr)."; \
printf ' %-30s %s\n' "tmp/newman-cli-cache-parity.log" "Newman CLI output of the deferred sequential cache-parity pass (also appended to tmp/newman-cli.log)."; \
printf ' %-30s %s\n' "tmp/harness-quiet.log" "Everything the target would have echoed, when CI=1 or a live status table suppressed it."; \
printf ' %-30s %s\n' "tmp/harness-monitor-passes.jsonl" "Pass manifest the single run-wide status table follows (one record per newman invocation)."; \
printf ' %-30s %s\n' "tmp/harness-failures.md" "Categorized failure analyzer output + coverage matrices."; \
printf ' %-30s %s\n' "tmp/bifrost-dev.log" "Bifrost runtime log (only if we auto-started it)."; \
printf ' %-30s %s\n' "tmp/harness-mcp-fixture.log" "MCP test server log (sse_mcp on 3012; only if we started it)."; \
printf ' %-30s %s\n' "tmp/harness-relay-fixture.log" "Relay upstream fixture log (relays-e2e rows, 127.0.0.1:8783; only if we started it)."; \
printf ' %-30s %s\n' "tmp/harness-augmented.json" "Provider harness plus generated streaming/thinking rows."; \
printf ' %-30s %s\n' "tmp/harness-filtered.json" "Filtered collection (only if PROVIDER/FEATURE/RERUN_FAILED set)."; \
printf ' %-30s %s\n' "tmp/harness-timings.json" "Measured median ms per request name, merged across runs. Drives both the shard sizing and the"; \
printf ' %-30s %s\n' "" " slice fill. Safe to delete: the next run falls back to the static table and rewrites it."; \
printf ' %-30s %s\n' "tmp/harness-shard-plan.txt" "The sized grid this run launched: '<provider> <class> <shards> <rows> <seconds>' per cell."; \
printf ' %-30s %s\n' "" " Absent means no timing table was available and the static HARNESS_SUBSHARDS counts were used."; \
printf ' %-30s %s\n' "tmp/newman-report-<shard>.json" "Per-shard newman report (parallel mode only). <shard> is \"<provider>--<class>\", \"<provider>--<class>-s<k>\" for a sub-sharded class, or plain \"<provider>\" under CLASS_SHARDS=0."; \
printf ' %-30s %s\n' "tmp/parallel-exit-<shard>" "Exit code of each shard's newman process. Read instead of 'wait <pid>' because the HARNESS_JOBS cap reaps pids as slots free up."; \
printf ' %-30s %s\n' "tmp/newman-cli-<p>.log" "Per-provider newman stdout/stderr (parallel mode only)."; \
printf ' %-30s %s\n' "tmp/parallel-status" "Per-provider pass/fail summary (parallel mode only)."; \
printf ' %-30s %s\n' "tmp/newman-report.html" "htmlextra report (sequential mode only — PARALLEL=0)."; \
printf ' %-30s %s\n' "tmp/stream-cancel-report.json" "Server-side stream cancellation probe report."; \
printf ' %-30s %s\n' "tmp/harness-token-parity-*.json" "Per-newman-process token parity fragments (one per provider fork in parallel mode)."; \
printf ' %-30s %s\n' "tmp/harness-token-parity.md" "Direct-vs-Bifrost token usage table (prompt/completion/cached/total per backend+modality) - see Token Parity Matrix above."; \
printf ' %-30s %s\n' "" " Same table is also injected into tmp/newman-report.html when present (sequential mode / PARALLEL=0 only)."; \
printf ' %-30s %s\n' "tmp/harness-cache-parity-*.json" "Per-newman-process cache parity fragments (CACHE_ANCHOR_REPORT + CACHE_MATRIX_REPORT blobs)."; \
printf ' %-30s %s\n' "tmp/harness-cache-parity.md" "Round 34 direct-vs-Bifrost cache hit-rate table + Round 35 cross-provider matrix - see Cache Parity Matrices above."; \
printf ' %-30s %s\n' "" " Both parity .md files are DELETED at run start alongside their fragments, because rendering is skipped"; \
printf ' %-30s %s\n' "" " when a run produces no fragments (scope excluded those rows, or the reporter deps failed to install)."; \
printf ' %-30s %s\n' "" " Absent means 'this run measured none'; a leftover from an earlier run reads as current and contradicts"; \
printf ' %-30s %s\n' "" " the freshly written tmp/harness-failures.md, which is rewritten unconditionally on every run."; \
printf ' %-30s %s\n' "tmp/newman-report-cache-parity.json" "Newman report for the deferred sequential cache pass (merged into tmp/newman-report.json)."; \
printf ' %-30s %s\n' "tmp/newman-report-serial.json" "Newman report for the serial pass: [SERIAL] folders run after the parallel sweep, alone against the gateway (merged into tmp/newman-report.json)."; \
printf ' %-30s %s\n' "tmp/newman-cli-serial.log" "Newman CLI output of the serial pass (also appended to tmp/newman-cli.log)."; \
printf ' %-30s %s\n' "tmp/newman-merge.jq" "jq program used to merge per-provider + cache-pass reports into tmp/newman-report.json."; \
printf '\n'; \
exit 0; \
fi
@if [ -n "$(HELP)" ]; then exit 0; fi; \
HARNESS_QUIET=0; \
if [ -n "$$CI" ] || [ -n "$(CI)" ]; then HARNESS_QUIET=1; fi; \
HARNESS_MONITORED=0; \
mkdir -p tmp; \
QUIET_LOG="$(CURDIR)/tmp/harness-quiet.log"; \
: > "$$QUIET_LOG"; \
MONITOR_ROSTER="$(or $(PROVIDER),$(HARNESS_PROVIDERS))"; \
MONITOR_PASSES="tmp/harness-monitor-passes.jsonl"; \
MONITOR_LIVE=0; \
: "Per-shard totals, reported as each shard finishes. On by default: with the provider x class"; \
: "grid running up to 80 shards, the provider row says how much is failing but not which slice,"; \
: "and that is otherwise only recoverable from 80 separate tmp/newman-cli-*.log files."; \
: "SHARD_LINES=0 drops back to the plain table."; \
STREAM_FLAG=""; \
if [ "$(SHARD_LINES)" = "0" ]; then STREAM_FLAG="--shard-lines 0"; fi; \
: > "$$MONITOR_PASSES"; \
if [ -f tmp/harness-monitor.pid ]; then \
kill $$(cat tmp/harness-monitor.pid) 2>/dev/null || true; \
rm -f tmp/harness-monitor.pid; \
fi; \
say() { \
if [ "$$HARNESS_QUIET" = "1" ] || [ "$$MONITOR_LIVE" = "1" ]; then \
printf '%b\n' "$$@" >> "$$QUIET_LOG"; \
else printf '%b\n' "$$@"; fi; \
}; \
run_quiet() { \
if [ "$$HARNESS_QUIET" = "1" ] || [ "$$MONITOR_LIVE" = "1" ]; then "$$@" >> "$$QUIET_LOG" 2>&1; \
else "$$@"; fi; \
}; \
: "HARNESS_MAX_REQUESTS is the approved paid-request ceiling for this run. Every newman"; \
: "launch below passes its exact filtered request count here first; a launch that would"; \
: "cross the cap is refused and later launches are refused too, so the live total is"; \
: "bounded by construction rather than estimated from the preflight. Shards already"; \
: "running drain normally and the report still merges; the target then exits 3."; \
BUDGET_USED=0; BUDGET_EXCEEDED=0; \
: "tmp/harness-budget-used mirrors BUDGET_USED so a COMPAT=both parent can hand the"; \
: "remaining budget to its second sub-run instead of letting both spend the full cap."; \
mkdir -p tmp; printf '0' > tmp/harness-budget-used; \
budget_ok() { \
if [ -z "$(HARNESS_MAX_REQUESTS)" ]; then return 0; fi; \
if [ "$$BUDGET_EXCEEDED" = "1" ] || [ $$((BUDGET_USED + $$1)) -gt "$(HARNESS_MAX_REQUESTS)" ]; then \
BUDGET_EXCEEDED=1; \
say "$(RED)[$$2] not launched: $$1 request(s) would push the total to $$((BUDGET_USED + $$1)), past HARNESS_MAX_REQUESTS=$(HARNESS_MAX_REQUESTS) (launched so far: $$BUDGET_USED)$(NC)"; \
return 1; \
fi; \
BUDGET_USED=$$((BUDGET_USED + $$1)); \
printf '%s' "$$BUDGET_USED" > tmp/harness-budget-used; \
return 0; \
}; \
start_monitor() { \
if [ -f tmp/harness-monitor.pid ]; then return 0; fi; \
if [ "$$HARNESS_QUIET" != "1" ] && [ ! -t 1 ]; then return 0; fi; \
$(USE_NODE); \
if [ "$$HARNESS_QUIET" = "1" ]; then \
node tests/e2e/api/runners/harness-monitor.mjs \
--providers "$$MONITOR_ROSTER" --tmp-dir tmp --passes "$$MONITOR_PASSES" \
--ci --ci-interval "$(or $(MONITOR_INTERVAL),5)" \
$$STREAM_FLAG \
$(if $(MONITOR_TABLE_REPRINT),--ci-reprint-table,) < /dev/null & \
else \
node tests/e2e/api/runners/harness-monitor.mjs \
--providers "$$MONITOR_ROSTER" --tmp-dir tmp --passes "$$MONITOR_PASSES" \
$$STREAM_FLAG \
< /dev/null > /dev/tty 2>&1 & \
fi; \
echo $$! > tmp/harness-monitor.pid; \
HARNESS_MONITORED=1; MONITOR_LIVE=1; \
}; \
add_pass() { \
printf '%s\n' "$$1" >> "$$MONITOR_PASSES"; \
start_monitor; \
}; \
end_pass() { \
[ -f tmp/harness-monitor.pid ] || return 0; \
printf '{"t":"pass-end","id":"%s"}\n' "$$1" >> "$$MONITOR_PASSES"; \
}; \
monitor_note() { \
[ -f tmp/harness-monitor.pid ] || return 0; \
printf '{"t":"note","text":"%s"}\n' "$$1" >> "$$MONITOR_PASSES"; \
}; \
stop_monitor() { \
if [ -f tmp/harness-monitor.pid ]; then \
MPID=$$(cat tmp/harness-monitor.pid); \
kill -TERM $$MPID 2>/dev/null; \
wait $$MPID 2>/dev/null || true; \
rm -f tmp/harness-monitor.pid; \
fi; \
MONITOR_LIVE=0; \
}; \
if [ "$(COMPAT)" = "both" ]; then \
mkdir -p tmp; \
say "$(CYAN)COMPAT=both: running harness with compat OFF then ON (sub-runs forced CI=1 to skip the interactive viewer)...$(NC)"; \
: "Both sub-runs share one HARNESS_MAX_REQUESTS: the second gets what the first left."; \
BOTH_REMAINING="$(HARNESS_MAX_REQUESTS)"; BOTH_USED=0; \
for mode in off on; do \
if [ -n "$$BOTH_REMAINING" ] && [ "$$BOTH_REMAINING" -le 0 ]; then \
say "$(RED)compat $$mode not run: HARNESS_MAX_REQUESTS=$(HARNESS_MAX_REQUESTS) already spent by the earlier sub-run ($$BOTH_USED launched)$(NC)"; \
BOTH_RC=3; continue; \
fi; \
say "$(CYAN)=== Harness run: compat $$mode ===$(NC)"; \
$(MAKE) run-provider-harness-test COMPAT=$$mode CI=1 HARNESS_MAX_REQUESTS="$$BOTH_REMAINING"; \
RC=$$?; \
if [ -n "$$BOTH_REMAINING" ]; then \
SUB_USED="$$(cat tmp/harness-budget-used 2>/dev/null || echo 0)"; \
BOTH_USED=$$((BOTH_USED + SUB_USED)); BOTH_REMAINING=$$((BOTH_REMAINING - SUB_USED)); \
fi; \
mv -f tmp/newman-report.json "tmp/newman-report-compat-$$mode.json" 2>/dev/null || true; \
mv -f tmp/newman-report.html "tmp/newman-report-compat-$$mode.html" 2>/dev/null || true; \
mv -f tmp/harness-failures.md "tmp/harness-failures-compat-$$mode.md" 2>/dev/null || true; \
if [ "$$RC" -ne 0 ]; then say "$(RED)compat $$mode run failed (exit $$RC)$(NC)"; BOTH_RC=$$RC; fi; \
done; \
say "$(GREEN)COMPAT=both complete. Reports: tmp/newman-report-compat-{off,on}.{json,html}, tmp/harness-failures-compat-{off,on}.md$(NC)"; \
exit $${BOTH_RC:-0}; \
fi; \
$(EXPOSE_ENV); \
mkdir -p tmp; \
BASE_URL_VAL="$(or $(BASE_URL),http://localhost:8080)"; \
APP_DIR_VAL="$(or $(APP_DIR),tests/integrations/python)"; \
VIEWER_PORT_VAL="$(or $(VIEWER_PORT),8090)"; \
: "Measured per-request cost, used to size the shard grid and to fill each shard (see"; \
: "lib/shard-cost.mjs). tmp/ first because it is refreshed by every run and so tracks the"; \
: "collection as it changes; the committed baseline is the fallback that makes a fresh"; \
: "checkout and CI balanced on their FIRST sweep rather than on their second."; \
TIMINGS_FILE=""; \
for cand in "$(or $(HARNESS_TIMINGS),tmp/harness-timings.json)" tests/e2e/api/collections/harness-timings.json; do \
if [ -s "$$cand" ]; then TIMINGS_FILE="$$cand"; break; fi; \
done; \
DBVERIFY_REPORTER=""; DBVERIFY_ARGS=""; DBVERIFY_READY=0; E2E_DEPS_READY=0; \
if [ -d tests/e2e/api/node_modules ] && npm --prefix tests/e2e/api ls --depth=0 >/dev/null 2>&1; then \
E2E_DEPS_READY=1; \
else \
say "$(YELLOW)Installing e2e reporter deps (dbverify, token-parity)...$(NC)"; \
if (cd tests/e2e/api && run_quiet npm install --silent); then \
E2E_DEPS_READY=1; \
else \
say "$(YELLOW)e2e reporter dep install failed; dbverify cost checks and the token-parity reporter are disabled for this run$(NC)"; \
fi; \
fi; \
if [ "$$E2E_DEPS_READY" = "1" ]; then \
export NODE_PATH="$(CURDIR)/tests/e2e/api/node_modules$${NODE_PATH:+:$$NODE_PATH}"; \
fi; \
if [ "$(DB_VERIFY)" != "0" ] && [ "$$E2E_DEPS_READY" = "1" ]; then \
DBVERIFY_READY=1; \
DBVERIFY_REPORTER=",dbverify"; \
LOGS_DB_VAL="$$(node tests/e2e/api/lib/logs-db-url.js "$$APP_DIR_VAL/config.json" "$(or $(HARNESS_SERVER_CWD),$(CURDIR)/transports/bifrost-http)")"; \
export BIFROST_LOGS_DB_URL="$$LOGS_DB_VAL"; \
DBVERIFY_ARGS="--reporter-dbverify-config $$APP_DIR_VAL/config.json"; \
say "$(CYAN)dbverify reporter enabled (logs DB resolved from configuration or BIFROST_LOGS_DB_URL). Set DB_VERIFY=0 to disable.$(NC)"; \
fi; \
TOKEN_PARITY_REPORTER=""; \
CACHE_PARITY_REPORTER=""; \
if [ "$$E2E_DEPS_READY" = "1" ]; then \
TOKEN_PARITY_REPORTER=",token-parity"; \
CACHE_PARITY_REPORTER=",cache-parity"; \
fi; \
rm -f tmp/harness-token-parity-*.json tmp/harness-cache-parity-*.json tmp/harness-token-parity.md tmp/harness-cache-parity.md; \
cp tests/e2e/api/runners/lib/newman-merge.jq tmp/newman-merge.jq; \
if command -v gcloud > /dev/null 2>&1; then \
VERTEX_ACCESS_TOKEN_VAL="$$(gcloud auth print-access-token 2>/dev/null)"; \
if [ -n "$$VERTEX_ACCESS_TOKEN_VAL" ]; then \
say "$(CYAN)Vertex direct-call access token minted via gcloud (expires in ~1h).$(NC)"; \
else \
say "$(YELLOW)gcloud present but 'gcloud auth print-access-token' failed - Vertex direct-provider parity cells will 401 (run 'gcloud auth login' to fix).$(NC)"; \
fi; \
else \
VERTEX_ACCESS_TOKEN_VAL=""; \
say "$(YELLOW)gcloud not found - Vertex direct-provider parity cells will 401 (install the gcloud CLI and auth to enable them).$(NC)"; \
fi; \
STARTED_BY_US=0; \
cleanup() { \
if [ -f tmp/harness-monitor.pid ]; then \
MPID=$$(cat tmp/harness-monitor.pid); \
kill $$MPID 2>/dev/null; \
rm -f tmp/harness-monitor.pid; \
fi; \
if [ -f tmp/harness-viewer.pid ]; then \
VPID=$$(cat tmp/harness-viewer.pid); \
kill $$VPID 2>/dev/null; \
rm -f tmp/harness-viewer.pid; \
fi; \
if [ "$$STARTED_BY_US" = "1" ] && [ -f tmp/bifrost-dev.pid ]; then \
BPID=$$(cat tmp/bifrost-dev.pid); \
say "$(YELLOW)Stopping Bifrost (pid $$BPID) - we started it...$(NC)"; \
kill $$BPID 2>/dev/null; \
pkill -P $$BPID 2>/dev/null; \
rm -f tmp/bifrost-dev.pid; \
fi; \
if [ -f tmp/harness-mcp-fixture.pid ]; then \
kill $$(cat tmp/harness-mcp-fixture.pid) 2>/dev/null; \
rm -f tmp/harness-mcp-fixture.pid; \
fi; \
if [ -f tmp/harness-relay-fixture.pid ]; then \
kill $$(cat tmp/harness-relay-fixture.pid) 2>/dev/null; \
rm -f tmp/harness-relay-fixture.pid; \
fi; \
}; \
preempt_viewer_port() { \
if [ -f tmp/harness-viewer.pid ]; then \
OLD=$$(cat tmp/harness-viewer.pid); \
if kill -0 $$OLD 2>/dev/null; then \
say "$(YELLOW)Killing orphaned viewer pid $$OLD from a prior run...$(NC)"; \
kill $$OLD 2>/dev/null; sleep 1; \
fi; \
rm -f tmp/harness-viewer.pid; \
fi; \
pkill -f "tests/e2e/api/runners/harness-viewer.mjs" 2>/dev/null || true; \
if command -v lsof > /dev/null 2>&1 && lsof -ti tcp:$$VIEWER_PORT_VAL > /dev/null 2>&1; then \
say "$(YELLOW)Port $$VIEWER_PORT_VAL still in use - freeing it...$(NC)"; \
lsof -ti tcp:$$VIEWER_PORT_VAL | xargs kill 2>/dev/null || true; \
sleep 1; \
fi; \
}; \
trap cleanup EXIT INT TERM HUP; \
PICKED_FEATURES=""; \
if [ -t 0 ] && [ -t 1 ] && [ -z "$$CI" ] && [ -z "$(CI)" ] \
&& [ -z "$(PROVIDER)" ] && [ -z "$(FEATURE)" ] && [ -z "$(FOLDER)" ] \
&& [ -z "$(RERUN_FAILED)" ] && [ -z "$(SMOKE)" ]; then \
$(USE_NODE); \
PICKED_FEATURES=$$(node tests/e2e/api/runners/pick-features.mjs); \
PICK_RC=$$?; \
case $$PICK_RC in \
0) ;; \
1) say "$(YELLOW)Cancelled.$(NC)"; exit 1 ;; \
2) ;; \
*) exit $$PICK_RC ;; \
esac; \
if [ -n "$$PICKED_FEATURES" ]; then \
say "$(GREEN)Modalities: $$PICKED_FEATURES$(NC)"; \
else \
say "$(GREEN)Modalities: all (no filter)$(NC)"; \
fi; \
fi; \
: "The profile's sse_mcp client (localhost:3012) serves the injected web_search tool."; \
: "Without it injection fails open and every injected-web-search row fails, so start the"; \
: "examples/mcps/remote-test-server fixture before the gateway resolves its MCP clients."; \
MCP_FIXTURE_STARTED=0; \
if nc -z localhost 3012 2>/dev/null; then \
say "$(GREEN)MCP test server already listening on 3012$(NC)"; \
else \
MCP_FIXTURE_BIN="$(CURDIR)/examples/mcps/remote-test-server/bin/remote-test-server"; \
if [ ! -x "$$MCP_FIXTURE_BIN" ] || [ examples/mcps/remote-test-server/main.go -nt "$$MCP_FIXTURE_BIN" ]; then \
say "$(CYAN)Building MCP test server (examples/mcps/remote-test-server)...$(NC)"; \
(cd examples/mcps/remote-test-server && GOWORK=off go build -o bin/remote-test-server .) || { \
say "$(RED)Could not build the MCP test server (examples/mcps/remote-test-server); not running the harness against a stale or missing fixture$(NC)"; \
exit 1; \
}; \
fi; \
if [ -x "$$MCP_FIXTURE_BIN" ]; then \
MCP_HTTP_PORT=3011 MCP_SSE_PORT=3012 "$$MCP_FIXTURE_BIN" > tmp/harness-mcp-fixture.log 2>&1 & \
echo $$! > tmp/harness-mcp-fixture.pid; \
for i in $$(seq 1 40); do nc -z localhost 3012 2>/dev/null && break; sleep 0.25; done; \
if nc -z localhost 3012 2>/dev/null; then \
say "$(GREEN)MCP test server up on 3012 (log: tmp/harness-mcp-fixture.log)$(NC)"; \
MCP_FIXTURE_STARTED=1; \
else \
say "$(RED)MCP test server did not open 3012. See tmp/harness-mcp-fixture.log$(NC)"; \
fi; \
fi; \
fi; \
: "The profile's relays (harness-relay-*) point at a local upstream on 127.0.0.1:8783, so the"; \
: "relays-e2e rows never reach a third party. The gateway dials it per request, not at boot."; \
if nc -z 127.0.0.1 8783 2>/dev/null; then \
if curl -fsS --max-time 2 http://127.0.0.1:8783/__relay/health 2>/dev/null | grep -q '"fixture":"relay-upstream"'; then \
say "$(GREEN)Relay upstream fixture already listening on 8783$(NC)"; \
else \
say "$(RED)Port 8783 is taken by something other than the relay upstream fixture (no /__relay/health match); relays-e2e rows will fail until it is freed.$(NC)"; \
case "$(FEATURE)" in *relay*) exit 1;; esac; \
fi; \
else \
$(USE_NODE); RELAY_UPSTREAM_FIXTURE_PORT=8783 node tests/e2e/api/runners/relay-upstream-fixture.mjs > tmp/harness-relay-fixture.log 2>&1 & \
echo $$! > tmp/harness-relay-fixture.pid; \
for i in $$(seq 1 40); do nc -z 127.0.0.1 8783 2>/dev/null && break; sleep 0.25; done; \
if nc -z 127.0.0.1 8783 2>/dev/null; then \
say "$(GREEN)Relay upstream fixture up on 8783 (log: tmp/harness-relay-fixture.log)$(NC)"; \
else \
say "$(RED)Relay upstream fixture did not open 8783; relays-e2e rows will fail. See tmp/harness-relay-fixture.log$(NC)"; \
case "$(FEATURE)" in *relay*) exit 1;; esac; \
fi; \
fi; \
if curl -fsS --max-time 2 "$$BASE_URL_VAL/health" > /dev/null 2>&1; then \
say "$(GREEN)Bifrost already running at $$BASE_URL_VAL$(NC)"; \
if [ "$$MCP_FIXTURE_STARTED" = "1" ]; then \
say "$(YELLOW)The MCP test server started after this gateway booted, so its sse_mcp client may still be disconnected; restart the gateway if injected-web-search rows fail.$(NC)"; \
fi; \
else \
say "$(YELLOW)Bifrost not running - launching 'make dev' (APP_DIR=$$APP_DIR_VAL) in background...$(NC)"; \
$(MAKE) dev APP_DIR="$$APP_DIR_VAL" > tmp/bifrost-dev.log 2>&1 & \
echo $$! > tmp/bifrost-dev.pid; \
STARTED_BY_US=1; \
say "$(CYAN)Waiting for Bifrost /health to respond (up to 60s)...$(NC)"; \
for i in $$(seq 1 30); do \
if curl -fsS --max-time 2 "$$BASE_URL_VAL/health" > /dev/null 2>&1; then \
say "$(GREEN)Bifrost is up$(NC)"; break; \
fi; \
sleep 2; \
done; \
if ! curl -fsS --max-time 2 "$$BASE_URL_VAL/health" > /dev/null 2>&1; then \
say "$(RED)Bifrost did not become healthy. See tmp/bifrost-dev.log$(NC)"; \
exit 1; \
fi; \
fi; \
: "Rows that assert on extra_fields.raw_request send x-bf-send-back-raw-*; core ignores those"; \
: "headers unless client_config.allow_per_request_raw_override is on. The harness config.json"; \
: "already enables it for a gateway we start ourselves; an already-running one may not have it."; \
say "$(CYAN)Ensuring allow_per_request_raw_override is on (x-bf-send-back-raw-* headers)...$(NC)"; \
BIFROST_BASE_URL="$$BASE_URL_VAL" node tests/e2e/api/runners/set-raw-override-config.mjs enable || { \
say "$(RED)Could not enable allow_per_request_raw_override; raw_request assertions would fail. Set BIFROST_E2E_AUTH_HEADER if auth is on.$(NC)"; \
exit 1; \
}; \
say "$(CYAN)Augmenting provider harness with generated streaming/thinking cases...$(NC)"; \
: "VERTEX_ACCESS_TOKEN_VAL is exported so the token-parity matrix can skip the Vertex"; \
: "direct legs when gcloud could not mint a token, instead of emitting cells that post an"; \
: "unresolved {{vertexAccessToken}} placeholder and 401 en masse."; \
export VERTEX_ACCESS_TOKEN_VAL; \
$(USE_NODE); run_quiet node tests/e2e/api/runners/augment-provider-harness.mjs \
--source tests/e2e/api/collections/provider-harness.json \
--out tmp/harness-augmented.json || { say "$(RED)Harness augmentation failed$(NC)"; exit 1; }; \
COLLECTION_FILE="tmp/harness-augmented.json"; \
: "SMOKE_MANIFEST alone carries smoke mode: it is both the truth test and the flag value."; \
: "Packing --smoke and the path into one scalar would leave field splitting as the only"; \
: "thing separating them, so a manifest path containing a space would arrive truncated."; \
SMOKE_MANIFEST=""; \
if [ -n "$(SMOKE)" ]; then \
case "$(SMOKE)" in \
1|true|TRUE|yes|YES|y|Y) SMOKE_MANIFEST="tests/e2e/api/collections/smoke-manifest.json" ;; \
*) SMOKE_MANIFEST="$(SMOKE)" ;; \
esac; \
if [ ! -f "$$SMOKE_MANIFEST" ]; then \
say "$(RED)SMOKE manifest not found: $$SMOKE_MANIFEST$(NC)"; exit 1; \
fi; \
say "$(CYAN)Smoke mode: selecting the curated set in $$SMOKE_MANIFEST.$(NC)"; \
fi; \
DEFERRED_KEYS="cache-parity"; \
MAIN_FEATURES=""; CACHE_PASS=0; SKIP_MAIN=0; PICK_SAW_DEFERRED=0; \
for k in $$(printf '%s' "$$PICKED_FEATURES" | tr ',' ' '); do \
IS_DEFERRED=0; \
for d in $$DEFERRED_KEYS; do if [ "$$k" = "$$d" ]; then IS_DEFERRED=1; fi; done; \
if [ "$$IS_DEFERRED" = "1" ]; then PICK_SAW_DEFERRED=1; \
else MAIN_FEATURES="$${MAIN_FEATURES:+$$MAIN_FEATURES,}$$k"; fi; \
done; \
: "Two separate decisions. CACHE_PASS: do the cache-parity rows run at all (in their own"; \
: "sequential newman). CARVE_OUT: are they kept out of the parallel forks. They used to be one"; \
: "flag, and that is how a menu pick without cache-parity ran every cache anchor anyway: the"; \
: "pick is an OR of modality aliases, 'chat' matches any request carrying messages, so the"; \
: "anchors rode in as chat rows - once per fork, and cost-sliced away from their write rounds."; \
: "Those rows are per-process state twice over (the collection-level pre-request script mints"; \
: "{{pcNonce}} once per newman process, and the rounds hand their counters to each other through"; \
: "collectionVariables), so a read round in a different process than its write round starts"; \
: "cold, writes again, and fails on read=0 for a reason that has nothing to do with the cache."; \
if [ -n "$$PICKED_FEATURES" ]; then \
: "The menu is the one place a deselected cache-parity means: do not run them anywhere."; \
CACHE_PASS=$$PICK_SAW_DEFERRED; \
elif [ -z "$(RERUN_FAILED)" ] && [ -z "$(FOLDER)" ] && [ -z "$(FEATURE)" ]; then \
CACHE_PASS=1; \
elif [ "$(or $(PARALLEL),1)" != "0" ]; then \
: "FEATURE/FOLDER/RERUN_FAILED are predicates that may well select cache rows. In parallel"; \
: "mode those rows must not be forked, so the cache pass runs them - ANDed with the same"; \
: "predicate, so it replays only what the scope selected, and skips itself when that is none."; \
: "PARALLEL=0 is one process in collection order, where a cell's rounds already share a"; \
: "nonce, so it keeps the row-level selection in the main pass."; \
CACHE_PASS=1; \
fi; \
if [ "$$CACHE_PASS" = "1" ] && [ -n "$$PICKED_FEATURES" ] && [ -z "$$MAIN_FEATURES" ]; then SKIP_MAIN=1; fi; \
FEATURE_TOKENS="$$(printf '%s' "$(FEATURE)" | tr 'A-Z,' 'a-z ')"; \
for d in $$DEFERRED_KEYS; do \
for t in $$FEATURE_TOKENS; do \
if [ "$$t" = "$$d" ] && [ -z "$(RERUN_FAILED)" ]; then CACHE_PASS=1; SKIP_MAIN=1; fi; \
done; \
done; \
: "SMOKE forces the deferred cache pass on. The default above turns CACHE_PASS on only when nothing"; \
: "is filtering the run, and SMOKE is a filter - so without this line a smoke run would carve the"; \
: "cache-parity rows out of the main pass via EXCLUDE_FLAG and then never replay them, silently"; \
: "dropping the third of the smoke set that exists to catch a dropped cache breakpoint."; \
if [ -n "$${SMOKE_MANIFEST:-}" ] && [ -z "$(RERUN_FAILED)" ]; then CACHE_PASS=1; SKIP_MAIN=0; fi; \
: "Computed last, after SMOKE has had its say: a parallel run always carves the cache rows out"; \
: "of the forks, whether or not a cache pass follows (a deselected menu pick: neither)."; \
CARVE_OUT=$$CACHE_PASS; \
if [ "$(or $(PARALLEL),1)" != "0" ]; then CARVE_OUT=1; fi; \
EXCLUDE_FLAG=""; \
if [ "$$CARVE_OUT" = "1" ]; then \
EXCLUDE_FLAG="--exclude-feature-any cache-parity"; \
if [ "$$CACHE_PASS" = "1" ]; then \
say "$(CYAN)cache-parity deferred to a sequential pass after the main run (its rows are per-process state, so the parallel forks would slice a cell's rounds apart and repeat each request once per fork).$(NC)"; \
else \
say "$(CYAN)cache-parity not selected - its rows are kept out of the parallel forks and do not run.$(NC)"; \
fi; \
fi; \
FEATURE_ANY_FLAG=""; \
if [ -n "$$MAIN_FEATURES" ]; then FEATURE_ANY_FLAG="--feature-any $$MAIN_FEATURES"; fi; \
: "[SERIAL] folders change gateway-wide state (a feature flag, client config) around the request"; \
: "under test. The parallel forks would run other rows against the gateway while one of them has"; \
: "it changed, so a parallel run carves them out of the forks and runs them afterwards in a pass"; \
: "of their own, alone against the gateway. A sequential run already sends one request at a time."; \
SERIAL_PASS=0; SERIAL_EXCLUDE_FLAG=""; SERIAL_RAN=0; MAIN_EMPTY=0; \
if [ "$(or $(PARALLEL),1)" != "0" ] && [ "$$SKIP_MAIN" != "1" ]; then SERIAL_PASS=1; SERIAL_EXCLUDE_FLAG="--exclude-serial"; fi; \
: "The main pass merges its shards over tmp/newman-report.json before the cache pass runs, so"; \
: "a rerun's selection for that pass is read from a snapshot taken now. Dot-prefixed so the"; \
: "tmp/newman-report-*.json merge glob never sweeps it in."; \
rm -f tmp/.newman-report-prior.json; \
if [ -n "$(RERUN_FAILED)" ] && [ -f tmp/newman-report.json ]; then cp -f tmp/newman-report.json tmp/.newman-report-prior.json; fi; \
if [ "$$SKIP_MAIN" != "1" ] && { [ -n "$(PROVIDER)" ] || [ -n "$(FEATURE)" ] || [ -n "$(FOLDER)" ] || [ -n "$(RERUN_FAILED)" ] || [ -n "$$MAIN_FEATURES" ] || [ -n "$$EXCLUDE_FLAG" ] || [ -n "$$SMOKE_MANIFEST" ]; }; then \
say "$(CYAN)Filtering collection (provider=$(PROVIDER), feature=$(FEATURE), folder=$(FOLDER), feature-any=$$MAIN_FEATURES, exclude=$${EXCLUDE_FLAG:+cache-parity}, smoke=$${SMOKE_MANIFEST:--}, rerun-failed=$(RERUN_FAILED))...$(NC)"; \
$(USE_NODE); run_quiet node tests/e2e/api/runners/filter-collection.mjs \
--source "$$COLLECTION_FILE" \
--out tmp/harness-filtered.json \
$(if $(PROVIDER),--provider $(PROVIDER),) \
$(if $(FEATURE),--feature "$(FEATURE)",) \
$(if $(FOLDER),--folder "$(FOLDER)",) \
$$FEATURE_ANY_FLAG \
$$EXCLUDE_FLAG \
$$SERIAL_EXCLUDE_FLAG \
$${SMOKE_MANIFEST:+--smoke "$$SMOKE_MANIFEST"} \
$(if $(RERUN_FAILED),--rerun-failed --report tmp/newman-report.json,) || { say "$(RED)Filter step failed$(NC)"; exit 1; }; \
COLLECTION_FILE="tmp/harness-filtered.json"; \
fi; \
say "$(YELLOW)Running newman against $$BASE_URL_VAL using $$COLLECTION_FILE...$(NC)"; \
set -o pipefail; \
$(USE_NODE); \
PARALLEL_VAL="$(or $(PARALLEL),1)"; \
if [ "$$SKIP_MAIN" = "1" ]; then \
say "$(CYAN)cache-parity was the only selection - skipping the main pass and going straight to the sequential cache pass.$(NC)"; \
printf '%s' '{"collection":{},"environment":{},"run":{"executions":[],"failures":[],"stats":{"iterations":{"total":1,"pending":0,"failed":0},"items":{"total":0},"requests":{"total":0,"failed":0}},"timings":{}}}' > tmp/newman-report.json; \
: > tmp/newman-cli.log; \
NEWMAN_EXIT=0; \
elif [ "$$PARALLEL_VAL" != "0" ] && [ -n "$$PARALLEL_VAL" ]; then \
say "$(CYAN)Parallel mode (default): forking one newman per provider ($(HARNESS_PROVIDERS)) x modality class x sub-shard, slowest class first. Set PARALLEL=0 to disable, SUBSHARDS=0 to drop the sub-shard axis, CLASS_SHARDS=0 for one fork per provider.$(NC)"; \
: "harness-filtered-*.json is cleaned here too, not just the reports. The monitor derives"; \
: "each provider's denominator by summing the leaves of every shard collection it finds in"; \
: "tmp/, so a previous run with a wider FEATURE/FOLDER scope leaves shard files this run"; \
: "never launches: their rows join the total, the sweep reads larger than it is, and the ETA"; \
: "never converges. The 'filesRead < launched' latch cannot catch it - extra files raise"; \
: "filesRead rather than lower it. The unsharded tmp/harness-filtered.json is deliberately"; \
: "NOT matched by this glob (it has no '-' after 'filtered'): it was written above as"; \
: "COLLECTION_FILE and every shard below filters FROM it."; \
rm -f tmp/newman-report-*.json tmp/newman-cli-*.log tmp/parallel-pids tmp/parallel-status tmp/parallel-exit-* tmp/harness-filtered-*.json; \
: > tmp/parallel-pids; \
: > tmp/parallel-status; \
PROVIDERS="$(HARNESS_PROVIDERS)"; \
if [ -n "$(PROVIDER)" ]; then PROVIDERS="$(PROVIDER)"; fi; \
: "CLASS_SHARDS=0 collapses the class axis back to a single fork per provider. The empty"; \
: "string is the sentinel for 'do not pass --class', so the loop body stays one code path."; \
CLASSES="$(HARNESS_CLASSES)"; \
if [ "$(CLASS_SHARDS)" = "0" ]; then CLASSES="-"; fi; \
: "Sub-shard count for a (provider, class) cell. Prefers the measured plan in $$SHARD_PLAN,"; \
: "and falls back to the hand-written HARNESS_SUBSHARDS 'class=n' list; 1 when unlisted."; \
: "Returns 1 for the '-' class too - CLASS_SHARDS=0 collapses both axes, not just the first."; \
: ""; \
: "The plan is per PROVIDER as well as per class, which the static table cannot express and"; \
: "which is where most of the imbalance lived: 'reasoning' is one number, but openai spends"; \
: "~30 minutes there against embeddings' seconds, so any single count is wrong for one of"; \
: "them. An absent or empty plan file means no timings were available, and the static table"; \
: "stands - see filter-collection.mjs --plan, which writes nothing rather than a grid of 1s."; \
subshards_for() { \
SS_P="$$1"; SS_C="$$2"; SS_N=1; \
if [ "$(SUBSHARDS)" != "0" ] && [ "$$SS_C" != "-" ]; then \
SS_N=""; \
: "Braced default so the function is safe under 'set -u' both here (SHARD_PLAN is"; \
: "set just above the launch loop) and in shard-split.test.mjs, which extracts this"; \
: "block and runs it standalone to prove the test cannot pass against stale logic."; \
if [ -s "$${SHARD_PLAN:-}" ]; then \
SS_N="$$(awk -v p="$$SS_P" -v c="$$SS_C" '$$1==p && $$2==c {print $$3; exit}' "$$SHARD_PLAN")"; \
fi; \
if [ -z "$$SS_N" ]; then \
SS_N=1; \
for kv in $(HARNESS_SUBSHARDS); do \
case "$$kv" in "$$SS_C="*) SS_N="$${kv#*=}" ;; esac; \
done; \
fi; \
fi; \
printf '%s' "$$SS_N"; \
}; \
JOBS_CAP="$(or $(HARNESS_JOBS),100)"; \
say "$(CYAN)Shard concurrency cap: $$JOBS_CAP (HARNESS_JOBS).$(NC)"; \
: "Size the provider x class grid from measured per-request cost before launching it."; \
: "One planner invocation for the whole grid rather than one per cell: sizing a cell means"; \
: "running the same predicate stack that selects it, and doing that ~80 times would parse"; \
: "the 20MB collection ~80 times. Writes nothing when no timing table exists, which is the"; \
: "signal subshards_for reads as 'keep the static HARNESS_SUBSHARDS table'."; \
SHARD_PLAN="tmp/harness-shard-plan.txt"; \
rm -f "$$SHARD_PLAN"; \
if [ -n "$$TIMINGS_FILE" ] && [ "$(SUBSHARDS)" != "0" ]; then \
if node tests/e2e/api/runners/filter-collection.mjs \
--source "$$COLLECTION_FILE" --plan \
--providers "$$PROVIDERS" --classes "$$CLASSES" \
--timings "$$TIMINGS_FILE" \
--target "$(or $(HARNESS_SHARD_TARGET),150)" \
$(if $(FEATURE),--feature "$(FEATURE)",) \
$(if $(FOLDER),--folder "$(FOLDER)",) \
> "$$SHARD_PLAN" 2>> "$$QUIET_LOG"; then \
say "$(CYAN)Shard plan sized from $$TIMINGS_FILE -> $$SHARD_PLAN ($$(awk '{s+=$$3} END{print s+0}' "$$SHARD_PLAN") shards across $$(awk 'END{print NR+0}' "$$SHARD_PLAN") cells, $(or $(HARNESS_SHARD_TARGET),150)s target).$(NC)"; \
else \
: "A planner that failed leaves a partial file, and a partial plan is worse than"; \
: "none: the cells it did emit would be measured while the rest silently fell back."; \
rm -f "$$SHARD_PLAN"; \
say "$(YELLOW)Shard planner failed - falling back to the static HARNESS_SUBSHARDS table (see $$QUIET_LOG)$(NC)"; \
fi; \
else \
say "$(YELLOW)No timing table yet - using the static HARNESS_SUBSHARDS table. It is written at the end of this run, so the next sweep balances by measured cost.$(NC)"; \
fi; \
: "One newman invocation, called from two places: the main launch loop and the 429 retry"; \
: "pass. Factored out so the retry cannot drift from the main run - the two must send the"; \
: "same env vars and reporters or the retry would exercise a different configuration than"; \
: "the failure it is meant to clear. Args: <shard> <collection> <report-out> <provider>."; \
: "Runs in the foreground and returns the PIPELINE status (pipefail is set above), so the"; \
: "caller can record it; backgrounding stays at the call site so pids remain trackable."; \
: "Running SHARD jobs, excluding every long-lived helper this shell backgrounded: the"; \
: "monitor, the MCP and relay fixtures and a gateway started by 'make dev'. A plain"; \
: "'jobs -pr | wc -l' counts them as shards, and with a low HARNESS_JOBS the count would"; \
: "sit at the cap while 'wait -n' blocks on a process that never exits."; \
shard_jobs() { \
SJ_SKIP="$$(cat tmp/harness-monitor.pid tmp/harness-mcp-fixture.pid tmp/harness-relay-fixture.pid tmp/bifrost-dev.pid 2>/dev/null)"; \
jobs -pr | grep -v -x -F "$${SJ_SKIP:-__none__}" | wc -l | tr -d ' '; \
}; \
newman_shard() { \
NS_SHARD="$$1"; NS_COLLECTION="$$2"; NS_REPORT="$$3"; NS_PROV="$$4"; \
newman run "$$NS_COLLECTION" \
--env-var "baseUrl=$$BASE_URL_VAL" \
$(if $(filter on true 1 yes YES y Y,$(COMPAT)),--env-var "compat=true",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_PREVIEW)),--env-var "include_preview=1",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_SKIP)),--env-var "include_skip=1",) \
$${BEDROCK_GUARDRAIL_IDENTIFIER:+--env-var "bedrockGuardrailIdentifier=$$BEDROCK_GUARDRAIL_IDENTIFIER"} \
$${BEDROCK_GUARDRAIL_VERSION:+--env-var "bedrockGuardrailVersion=$$BEDROCK_GUARDRAIL_VERSION"} \
$${VERTEX_GCS_BUCKET:+--env-var "vertexGcsBucket=$$VERTEX_GCS_BUCKET"} \
$${VERTEX_GCS_PREFIX:+--env-var "vertexGcsPrefix=$$VERTEX_GCS_PREFIX"} \
$${AWS_S3_BUCKET:+--env-var "awsS3Bucket=$$AWS_S3_BUCKET"} \
$${OPENAI_API_KEY:+--env-var "openaiKey=$$OPENAI_API_KEY"} \
$${CHATGPT_ACCESS_TOKEN:+--env-var "chatgptAccessToken=$$CHATGPT_ACCESS_TOKEN"} \
$${ANTHROPIC_API_KEY:+--env-var "anthropicKey=$$ANTHROPIC_API_KEY"} \
$${GEMINI_API_KEY:+--env-var "genaiKey=$$GEMINI_API_KEY"} \
$${AWS_ACCESS_KEY_ID:+--env-var "bedrockDirectAccessKeyId=$$AWS_ACCESS_KEY_ID"} \
$${AWS_SECRET_ACCESS_KEY:+--env-var "bedrockDirectSecretAccessKey=$$AWS_SECRET_ACCESS_KEY"} \
$${AWS_REGION:+--env-var "bedrockDirectRegion=$$AWS_REGION"} \
$${VERTEX_PROJECT_ID:+--env-var "vertexProject=$$VERTEX_PROJECT_ID"} \
$${GOOGLE_LOCATION:+--env-var "vertexLocation=$$GOOGLE_LOCATION"} \
$${VERTEX_ACCESS_TOKEN_VAL:+--env-var "vertexAccessToken=$$VERTEX_ACCESS_TOKEN_VAL"} \
$(if $(ENV_FILE),--environment $(ENV_FILE),) \
$(if $(FOLDER),--folder "$(FOLDER)",) \
--reporters cli,json$$DBVERIFY_REPORTER$$TOKEN_PARITY_REPORTER $$DBVERIFY_ARGS \
$${TOKEN_PARITY_REPORTER:+--reporter-token-parity-out "tmp/harness-token-parity-$$NS_SHARD.json"} \
--reporter-json-export "$$NS_REPORT" 2>&1 | sed "s/^/[$$NS_PROV] /"; \
NS_RC=$$?; \
: "Assertions have already run against the full response, so the stored copy of large bodies"; \
: "(image/audio Buffers, ~4 JSON chars per byte) is dead weight: image-gen shards alone were 1.3GB."; \
: "Slim the report in place with the merge program (idempotent; same trimming the final merge did)."; \
: "Serialized with flock because slimming a big report takes several GB and shards finish together."; \
: "Only reports over 20MB are touched, and a failed slim leaves the original untouched."; \
if command -v jq >/dev/null 2>&1 && command -v flock >/dev/null 2>&1 && [ -f tmp/newman-merge.jq ] && [ "$$(stat -c %s "$$NS_REPORT" 2>/dev/null || echo 0)" -gt 20971520 ]; then \
flock tmp/.slim.lock sh -c 'jq -c -s -f tmp/newman-merge.jq "$$1" > "$$1.slim" && [ -s "$$1.slim" ] && mv -f "$$1.slim" "$$1" || rm -f "$$1.slim"' _ "$$NS_REPORT"; \
fi; \
return $$NS_RC; \
}; \
LAUNCHED=0; \
: "A shard whose filter step fails never reaches tmp/parallel-pids, so the verdict loop"; \
: "below cannot see it and PFAILED alone would report the run green with that slice never"; \
: "run. Counted separately and folded into NEWMAN_EXIT so a skipped shard fails the run."; \
FILTER_FAILED=0; \
for p in $$PROVIDERS; do \
for c in $$CLASSES; do \
SUB_N="$$(subshards_for "$$p" "$$c")"; \
SUB_K=0; \
: "SUB_K is bumped at the TOP of the body, not the bottom: the body below uses 'continue'"; \
: "for a failed filter and for an empty shard, and a bottom increment would be skipped by"; \
: "both - leaving SUB_K stuck and this while loop spinning on the same sub-shard forever."; \
while [ "$$SUB_K" -lt "$$SUB_N" ]; do \
SUB_K=$$((SUB_K+1)); \
if [ "$$c" = "-" ]; then SHARD="$$p"; CLASS_FLAG=""; else SHARD="$$p--$$c"; CLASS_FLAG="--class $$c"; fi; \
: "The -s<k> suffix stays inside the '<provider>--<rest>' shape every consumer parses:"; \
: "the retry pass takes the provider as $${rs%%--*}, and harness-monitor.mjs matches"; \
: "shard files by the 'harness-filtered-<provider>--' prefix and detects retry logs by a"; \
: "trailing -retry<n>. A single-sub-shard class keeps its old unsuffixed name, so the"; \
: "common case produces byte-identical filenames to before this axis existed."; \
SHARD_FLAG=""; \
if [ "$$SUB_N" -gt 1 ]; then SHARD="$$SHARD-s$$SUB_K"; SHARD_FLAG="--shard $$SUB_K/$$SUB_N"; fi; \
: "--timings makes the slice balance measured cost instead of row count. Every"; \
: "sub-shard of a cell recomputes the whole assignment from the same table and keeps"; \
: "only its own slice, so they must all be passed the same file or the slices stop"; \
: "being a partition - rows would land in two shards and in none."; \
if ! node tests/e2e/api/runners/filter-collection.mjs \
--source "$$COLLECTION_FILE" \
--out "tmp/harness-filtered-$$SHARD.json" \
--provider "$$p" \
$$CLASS_FLAG \
$$SHARD_FLAG \
$${TIMINGS_FILE:+--timings "$$TIMINGS_FILE"} \
$(if $(FEATURE),--feature "$(FEATURE)",) \
$(if $(FOLDER),--folder "$(FOLDER)",) >> "$$QUIET_LOG" 2>&1; then \
say "$(RED)[$$SHARD] filter step failed - skipping (see $$QUIET_LOG)$(NC)"; \
FILTER_FAILED=$$((FILTER_FAILED+1)); \
continue; \
fi; \
P_ITEM_COUNT=$$(grep -c '"request":' "tmp/harness-filtered-$$SHARD.json" 2>/dev/null); \
P_ITEM_COUNT=$${P_ITEM_COUNT:-0}; \
if [ "$$P_ITEM_COUNT" -eq 0 ]; then \
rm -f "tmp/harness-filtered-$$SHARD.json"; \
continue; \
fi; \
: "A refused shard must not leave its filtered file behind: the status table counts"; \
: "every tmp/harness-filtered-*.json as planned work, so it would show rows that never ran."; \
budget_ok "$$P_ITEM_COUNT" "$$SHARD" || { rm -f "tmp/harness-filtered-$$SHARD.json"; continue; }; \
: "Block until a slot frees. 'wait -n' reaps one arbitrary child, which is why shard"; \
: "exit codes are recorded by the subshell into tmp/parallel-exit-<shard> instead of"; \
: "being collected later with 'wait <pid>' - that pid may already have been reaped here."; \
while [ "$$(shard_jobs)" -ge "$$JOBS_CAP" ]; do wait -n 2>/dev/null || true; done; \
( \
newman_shard "$$SHARD" "tmp/harness-filtered-$$SHARD.json" "tmp/newman-report-$$SHARD.json" "$$p"; \
echo "$$?" > "tmp/parallel-exit-$$SHARD"; \
) > "tmp/newman-cli-$$SHARD.log" 2>&1 & \
BG_PID=$$!; \
LAUNCHED=$$((LAUNCHED+1)); \
echo "$$BG_PID:$$SHARD" >> tmp/parallel-pids; \
say "$(GREEN)[$$SHARD] launched (pid $$BG_PID, $$P_ITEM_COUNT requests)$(NC)"; \
done; \
done; \
done; \
MAIN_EMPTY=0; \
if [ "$$LAUNCHED" -eq 0 ]; then \
if [ "$$BUDGET_EXCEEDED" = "1" ]; then \
say "$(RED)Aborted before any launch: every shard would exceed HARNESS_MAX_REQUESTS=$(HARNESS_MAX_REQUESTS). Raise the cap or narrow PROVIDER/FEATURE.$(NC)"; \
exit 3; \
fi; \
: "A filter that matches only [SERIAL] rows launches no fork: those rows were carved out"; \
: "for the serial pass below, which decides after it runs whether anything ran at all."; \
if [ "$$SERIAL_PASS" != "1" ]; then \
say "$(RED)No provider runs were launched. Check PROVIDER/FEATURE/FOLDER filters.$(NC)"; \
exit 1; \
fi; \
MAIN_EMPTY=1; \
say "$(CYAN)No parallel rows match the filters; going straight to the serial pass.$(NC)"; \
fi; \
if [ "$$MAIN_EMPTY" = "1" ]; then \
: "Nothing to drain, retry or merge. The serial pass merges its report over this empty one,"; \
: "so the final report is the serial report (the same skeleton a cache-only run starts from)."; \
printf '%s' '{"collection":{},"environment":{},"run":{"executions":[],"failures":[],"stats":{"iterations":{"total":1,"pending":0,"failed":0},"items":{"total":0},"requests":{"total":0,"failed":0}},"timings":{}}}' > tmp/newman-report.json; \
: > tmp/newman-cli.log; \
NEWMAN_EXIT=$$FILTER_FAILED; \
else \
add_pass "$$(printf '{"t":"pass","id":"main","mode":"parallel","statusFile":"tmp/parallel-status","launched":%s}' "$$LAUNCHED")"; \
: "Drain the shards, then read verdicts from the exit files rather than from wait's status:"; \
: "the cap loop above already reaped an unknown subset with 'wait -n', so a status here is"; \
: "not trustworthy - hence '|| true' and the exit files."; \
: ""; \
: "MUST wait on the recorded pids, never a bare 'wait'. start_monitor backgrounds the"; \
: "monitor into THIS shell, so a bare wait blocks on it too - and the monitor only exits"; \
: "once tmp/parallel-status has 'launched' lines, which the loop below writes after this"; \
: "point. That is a deadlock: every shard finishes, the status file stays empty, and the"; \
: "run hangs forever reprinting a frozen heartbeat."; \
while read pidp; do wait "$${pidp%%:*}" 2>/dev/null || true; done < tmp/parallel-pids; \
: "Transient-failure retry pass, covering 429 plus the overload codes 503 and 529 - see"; \
: "RETRYABLE_CODES in lib/rate-limit-retry.mjs for why those three and not the other 5xx."; \
: "Running ~80 shards at once against one set of keys makes rate limiting an"; \
: "expected outcome rather than an exotic one, and analyze-failures.mjs already classifies a"; \
: "429 as 'Backoff/retry; not a harness bug' - which is the tell that the run should do the"; \
: "backoff itself. Only the 429 ITEMS are replayed (--rerun-rate-limited), never the whole"; \
: "shard: replaying an assertion failure would burn quota re-confirming a real defect."; \
: "RETRY_429=0 disables. Reports land as newman-report-<shard>-retry<n>.json and are merged"; \
: "LAST so the successful attempt supersedes its own 429 (see lib/newman-merge.jq)."; \
RETRY_MAX="$(or $(RETRY_429),3)"; \
RETRY_ATTEMPT=1; \
while [ "$$RETRY_MAX" != "0" ] && [ "$$RETRY_ATTEMPT" -le "$$RETRY_MAX" ]; do \
RETRY_SHARDS=""; RETRY_WAIT=0; \
while read pidp; do \
rs="$${pidp#*:}"; \
[ "$$(cat "tmp/parallel-exit-$$rs" 2>/dev/null || echo 1)" = "0" ] && continue; \
: "Judge from the shard's LATEST attempt, not its original report. The original keeps"; \
: "its 429s forever, so reading it would re-queue a shard whose rows already recovered"; \
: "- a shard that also holds a real defect can never clear its verdict, and would"; \
: "otherwise replay the same recovered rows once per allowed attempt for nothing."; \
RS_LAST="$$(ls -1 tmp/newman-report-$$rs-retry*.json 2>/dev/null | sort -V | tail -1)"; \
RS_SRC="$${RS_LAST:-tmp/newman-report-$$rs.json}"; \
W="$$(node tests/e2e/api/runners/rate-limit-backoff.mjs --report "$$RS_SRC" --attempt "$$RETRY_ATTEMPT" 2>/dev/null || echo 0)"; \
[ "$${W:-0}" -gt 0 ] 2>/dev/null || continue; \
RETRY_SHARDS="$$RETRY_SHARDS $$rs"; \
[ "$$W" -gt "$$RETRY_WAIT" ] && RETRY_WAIT="$$W"; \
done < tmp/parallel-pids; \
[ -z "$$RETRY_SHARDS" ] && break; \
say "$(YELLOW)429 retry $$RETRY_ATTEMPT/$$RETRY_MAX: waiting $${RETRY_WAIT}s, then replaying rate-limited rows in:$$RETRY_SHARDS$(NC)"; \
monitor_note "429 retry $$RETRY_ATTEMPT: sleeping $${RETRY_WAIT}s"; \
sleep "$$RETRY_WAIT"; \
RETRY_PIDS=""; \
for rs in $$RETRY_SHARDS; do \
rp="$${rs%%--*}"; \
RETRY_COLL="tmp/harness-filtered-$$rs-retry$$RETRY_ATTEMPT.json"; \
RS_LAST="$$(ls -1 tmp/newman-report-$$rs-retry*.json 2>/dev/null | sort -V | tail -1)"; \
RS_SRC="$${RS_LAST:-tmp/newman-report-$$rs.json}"; \
: "Selects from the latest attempt too, so attempt N replays only the rows STILL"; \
: "rate-limited rather than every row that was ever throttled."; \
if ! node tests/e2e/api/runners/filter-collection.mjs \
--source "tmp/harness-filtered-$$rs.json" \
--out "$$RETRY_COLL" \
--rerun-rate-limited --report "$$RS_SRC" > /dev/null 2>&1; then \
say "$(YELLOW)[$$rs] retry filter failed - keeping the original verdict$(NC)"; \
continue; \
fi; \
RETRY_COUNT="$$(grep -c '"request":' "$$RETRY_COLL" 2>/dev/null || true)"; RETRY_COUNT="$${RETRY_COUNT:-0}"; \
[ "$$RETRY_COUNT" -eq 0 ] && continue; \
budget_ok "$$RETRY_COUNT" "$$rs-retry$$RETRY_ATTEMPT" || { rm -f "$$RETRY_COLL"; continue; }; \
while [ "$$(shard_jobs)" -ge "$$JOBS_CAP" ]; do wait -n 2>/dev/null || true; done; \
( \
newman_shard "$$rs-retry$$RETRY_ATTEMPT" "$$RETRY_COLL" "tmp/newman-report-$$rs-retry$$RETRY_ATTEMPT.json" "$$rp"; \
RC=$$?; \
: "Clear the shard only when the retry passed AND every original failure was retryable."; \
: "A shard that also failed an assertion stays failed however well the retry went,"; \
: "otherwise a real defect sitting beside a 429 would be erased by the backoff."; \
if [ "$$RC" = "0" ] && [ "$$(node tests/e2e/api/runners/rate-limit-backoff.mjs --report "tmp/newman-report-$$rs.json" --query only-rate-limited 2>/dev/null || echo 0)" = "1" ]; then \
echo 0 > "tmp/parallel-exit-$$rs"; \
fi; \
) > "tmp/newman-cli-$$rs-retry$$RETRY_ATTEMPT.log" 2>&1 & \
RETRY_PIDS="$$RETRY_PIDS $$!"; \
done; \
: "Same reason as the main drain: a bare wait would block on the backgrounded monitor."; \
for rpid in $$RETRY_PIDS; do wait "$$rpid" 2>/dev/null || true; done; \
RETRY_ATTEMPT=$$((RETRY_ATTEMPT+1)); \
done; \
PFAILED=0; \
while read pidp; do \
p="$${pidp#*:}"; \
SHARD_RC="$$(cat "tmp/parallel-exit-$$p" 2>/dev/null || echo 1)"; \
if [ "$$SHARD_RC" = "0" ]; then \
echo "$$p:pass" >> tmp/parallel-status; \
PVERDICT="$(GREEN)[$$p] passed$(NC)"; \
else \
echo "$$p:fail" >> tmp/parallel-status; \
PVERDICT="$(RED)[$$p] failed$(NC)"; \
PFAILED=$$((PFAILED+1)); \
fi; \
if [ ! -f tmp/harness-monitor.pid ]; then \
say "$$PVERDICT"; \
tail -n 20 "tmp/newman-cli-$$p.log" 2>/dev/null; \
fi; \
done < tmp/parallel-pids; \
end_pass main; \
monitor_note "Merging per-provider reports"; \
say "$(CYAN)Merging per-provider reports into tmp/newman-report.json...$(NC)"; \
if command -v jq >/dev/null 2>&1 && ls tmp/newman-report-*.json >/dev/null 2>&1; then \
: "Order is load-bearing, so the files are listed explicitly instead of by glob: the merge"; \
: "keeps the LAST occurrence of each item id, and a glob sorts newman-report-<s>-retry1.json"; \
: "BEFORE newman-report-<s>.json ('-' is 0x2D, '.' is 0x2E) - the stale 429 would overwrite"; \
: "its own successful retry. Main reports first, then retries in attempt order."; \
MERGE_MAIN="$$(ls tmp/newman-report-*.json 2>/dev/null | grep -v -e '-retry[0-9]*\.json$$' | sort)"; \
MERGE_RETRY="$$(ls tmp/newman-report-*-retry*.json 2>/dev/null | sort -V)"; \
: "Two stages: one jq -s over every shard (~3.6GB) gets OOM-killed, and the > redirect had"; \
: "already truncated newman-report.json to 0 bytes, so every reader died on 'Unexpected end of"; \
: "JSON input'. The merge program is idempotent, so slim each shard alone first (peak = the"; \
: "largest shard) and slurp only the slimmed copies, order preserved. Output lands in a temp"; \
: "file and is moved into place only on success, so a failed merge never leaves an empty report."; \
rm -rf tmp/.merge-slim; mkdir -p tmp/.merge-slim; \
MERGE_SLIM=""; MERGE_OK=1; MERGE_N=0; \
for f in $$MERGE_MAIN $$MERGE_RETRY; do \
MERGE_N=$$((MERGE_N + 1)); \
o="$$(printf 'tmp/.merge-slim/%04d.json' $$MERGE_N)"; \
jq -c -s -f tmp/newman-merge.jq "$$f" > "$$o" || { MERGE_OK=0; say "$(YELLOW)Slimming $$f failed$(NC)"; break; }; \
MERGE_SLIM="$$MERGE_SLIM $$o"; \
done; \
if [ "$$MERGE_OK" = "1" ] && jq -c -s -f tmp/newman-merge.jq $$MERGE_SLIM > tmp/.newman-report-merged.json; then \
mv -f tmp/.newman-report-merged.json tmp/newman-report.json; \
else \
rm -f tmp/.newman-report-merged.json; \
say "$(YELLOW)Report merge failed; per-provider reports remain at tmp/newman-report-*.json$(NC)"; \
fi; \
rm -rf tmp/.merge-slim; \
rm -f tmp/.newman-report.slim.json; \
cat tmp/newman-cli-*.log > tmp/newman-cli.log 2>/dev/null || true; \
else \
say "$(YELLOW)jq not found or no reports produced; skipping merge. See tmp/newman-report-*.json$(NC)"; \
fi; \
if [ "$$HARNESS_MONITORED" != "1" ]; then \
say "$(CYAN)Parallel summary:$(NC)"; \
while read sp; do \
pname="$${sp%%:*}"; \
pstat="$${sp#*:}"; \
if [ "$$pstat" = "pass" ]; then \
say " $(GREEN)✓ $$pname$(NC)"; \
else \
say " $(RED)✗ $$pname$(NC)"; \
fi; \
done < tmp/parallel-status; \
fi; \
if [ "$$FILTER_FAILED" -gt 0 ]; then \
say "$(RED)$$FILTER_FAILED shard(s) skipped - their filter step failed, so those requests never ran. See $$QUIET_LOG$(NC)"; \
fi; \
NEWMAN_EXIT=$$((PFAILED+FILTER_FAILED)); \
fi; \
else \
SEQ_PROVIDERS="$(or $(PROVIDER),$(HARNESS_PROVIDERS))"; \
: > tmp/newman-cli.log; \
SEQ_COUNT="$$(grep -c '"request":' "$$COLLECTION_FILE" 2>/dev/null || true)"; SEQ_COUNT="$${SEQ_COUNT:-0}"; \
if budget_ok "$$SEQ_COUNT" main; then \
add_pass "$$(printf '{"t":"pass","id":"main","mode":"sequential","log":"tmp/newman-cli.log","collection":"%s"}' "$$COLLECTION_FILE")"; \
newman run "$$COLLECTION_FILE" \
--env-var "baseUrl=$$BASE_URL_VAL" \
$(if $(filter on true 1 yes YES y Y,$(COMPAT)),--env-var "compat=true",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_PREVIEW)),--env-var "include_preview=1",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_SKIP)),--env-var "include_skip=1",) \
$${BEDROCK_GUARDRAIL_IDENTIFIER:+--env-var "bedrockGuardrailIdentifier=$$BEDROCK_GUARDRAIL_IDENTIFIER"} \
$${BEDROCK_GUARDRAIL_VERSION:+--env-var "bedrockGuardrailVersion=$$BEDROCK_GUARDRAIL_VERSION"} \
$${VERTEX_GCS_BUCKET:+--env-var "vertexGcsBucket=$$VERTEX_GCS_BUCKET"} \
$${VERTEX_GCS_PREFIX:+--env-var "vertexGcsPrefix=$$VERTEX_GCS_PREFIX"} \
$${AWS_S3_BUCKET:+--env-var "awsS3Bucket=$$AWS_S3_BUCKET"} \
$${OPENAI_API_KEY:+--env-var "openaiKey=$$OPENAI_API_KEY"} \
$${CHATGPT_ACCESS_TOKEN:+--env-var "chatgptAccessToken=$$CHATGPT_ACCESS_TOKEN"} \
$${ANTHROPIC_API_KEY:+--env-var "anthropicKey=$$ANTHROPIC_API_KEY"} \
$${GEMINI_API_KEY:+--env-var "genaiKey=$$GEMINI_API_KEY"} \
$${AWS_ACCESS_KEY_ID:+--env-var "bedrockDirectAccessKeyId=$$AWS_ACCESS_KEY_ID"} \
$${AWS_SECRET_ACCESS_KEY:+--env-var "bedrockDirectSecretAccessKey=$$AWS_SECRET_ACCESS_KEY"} \
$${AWS_REGION:+--env-var "bedrockDirectRegion=$$AWS_REGION"} \
$${VERTEX_PROJECT_ID:+--env-var "vertexProject=$$VERTEX_PROJECT_ID"} \
$${GOOGLE_LOCATION:+--env-var "vertexLocation=$$GOOGLE_LOCATION"} \
$${VERTEX_ACCESS_TOKEN_VAL:+--env-var "vertexAccessToken=$$VERTEX_ACCESS_TOKEN_VAL"} \
$(if $(ENV_FILE),--environment $(ENV_FILE),) \
$(if $(FOLDER),--folder "$(FOLDER)",) \
--reporters cli,json,htmlextra$$DBVERIFY_REPORTER$$TOKEN_PARITY_REPORTER $$DBVERIFY_ARGS \
$${TOKEN_PARITY_REPORTER:+--reporter-token-parity-out "tmp/harness-token-parity-sequential.json"} \
--reporter-json-export tmp/newman-report.json \
--reporter-htmlextra-export tmp/newman-report.html \
--reporter-htmlextra-title "Bifrost Provider Harness" \
--reporter-htmlextra-darkTheme > tmp/newman-cli.log 2>&1; \
NEWMAN_EXIT=$$?; \
end_pass main; \
else NEWMAN_EXIT=0; fi; \
if [ "$$HARNESS_MONITORED" != "1" ] && [ "$$HARNESS_QUIET" != "1" ]; then cat tmp/newman-cli.log; fi; \
if command -v jq >/dev/null 2>&1 && [ -f tmp/newman-report.json ]; then \
say "$(CYAN)Sanitizing tmp/newman-report.json (newman embeds the whole parent folder in every failure)...$(NC)"; \
jq -c -s -f tmp/newman-merge.jq tmp/newman-report.json > tmp/.newman-report-sanitized.json \
&& mv -f tmp/.newman-report-sanitized.json tmp/newman-report.json \
&& rm -f tmp/.newman-report.slim.json \
|| { rm -f tmp/.newman-report-sanitized.json; say "$(YELLOW)Report sanitize failed; tmp/newman-report.json left as-is (may exceed the viewer's 512MB parse limit).$(NC)"; }; \
fi; \
fi; \
if [ "$$CACHE_PASS" = "1" ]; then \
say "$(CYAN)Cache parity pass (sequential, single newman - these rows match every provider fork)...$(NC)"; \
rm -f tmp/harness-cache-filtered.json; \
: "cache-parity is the AND selector, so a FEATURE predicate narrows it: FEATURE is appended,"; \
: "not replaced, and 'cache-parity,gemini' stays narrower than 'cache-parity'. A menu pick"; \
: "is deliberately NOT forwarded - this pass only runs on the menu path when cache-parity"; \
: "itself was picked, and then every cache row is wanted, not the ones that also look like"; \
: "'streaming'."; \
CACHE_FEATURE="cache-parity"; \
if [ -n "$(FEATURE)" ]; then CACHE_FEATURE="cache-parity,$(FEATURE)"; fi; \
$(USE_NODE); run_quiet node tests/e2e/api/runners/filter-collection.mjs \
--source tmp/harness-augmented.json \
--out tmp/harness-cache-filtered.json \
--feature "$$CACHE_FEATURE" \
$(if $(FOLDER),--folder "$(FOLDER)",) \
$${SMOKE_MANIFEST:+--smoke "$$SMOKE_MANIFEST"} \
$(if $(RERUN_FAILED),--rerun-failed --report tmp/.newman-report-prior.json,) \
$(if $(PROVIDER),--provider $(PROVIDER),) || { say "$(RED)Cache parity filter step failed$(NC)"; }; \
CACHE_COUNT="$$(grep -c '"request":' tmp/harness-cache-filtered.json 2>/dev/null || true)"; CACHE_COUNT="$${CACHE_COUNT:-0}"; \
if [ "$$CACHE_COUNT" -eq 0 ]; then \
say "$(CYAN)Cache parity pass: the scope selects no cache-parity row - nothing to replay.$(NC)"; \
elif [ -f tmp/harness-cache-filtered.json ] && budget_ok "$$CACHE_COUNT" cache-parity; then \
CACHE_PROVIDERS="$(or $(PROVIDER),$(HARNESS_PROVIDERS))"; \
: > tmp/newman-cli-cache-parity.log; \
add_pass '{"t":"pass","id":"cache-parity","mode":"sequential","log":"tmp/newman-cli-cache-parity.log","collection":"tmp/harness-cache-filtered.json"}'; \
newman run tmp/harness-cache-filtered.json \
--env-var "baseUrl=$$BASE_URL_VAL" \
$(if $(filter on true 1 yes YES y Y,$(COMPAT)),--env-var "compat=true",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_PREVIEW)),--env-var "include_preview=1",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_SKIP)),--env-var "include_skip=1",) \
$${OPENAI_API_KEY:+--env-var "openaiKey=$$OPENAI_API_KEY"} \
$${CHATGPT_ACCESS_TOKEN:+--env-var "chatgptAccessToken=$$CHATGPT_ACCESS_TOKEN"} \
$${ANTHROPIC_API_KEY:+--env-var "anthropicKey=$$ANTHROPIC_API_KEY"} \
$${GEMINI_API_KEY:+--env-var "genaiKey=$$GEMINI_API_KEY"} \
$${AWS_ACCESS_KEY_ID:+--env-var "bedrockDirectAccessKeyId=$$AWS_ACCESS_KEY_ID"} \
$${AWS_SECRET_ACCESS_KEY:+--env-var "bedrockDirectSecretAccessKey=$$AWS_SECRET_ACCESS_KEY"} \
$${AWS_REGION:+--env-var "bedrockDirectRegion=$$AWS_REGION"} \
$${VERTEX_PROJECT_ID:+--env-var "vertexProject=$$VERTEX_PROJECT_ID"} \
$${GOOGLE_LOCATION:+--env-var "vertexLocation=$$GOOGLE_LOCATION"} \
$${VERTEX_ACCESS_TOKEN_VAL:+--env-var "vertexAccessToken=$$VERTEX_ACCESS_TOKEN_VAL"} \
$(if $(ENV_FILE),--environment $(ENV_FILE),) \
--reporters cli,json$$CACHE_PARITY_REPORTER \
$${CACHE_PARITY_REPORTER:+--reporter-cache-parity-out "tmp/harness-cache-parity-pass.json"} \
--reporter-json-export tmp/newman-report-cache-parity.json > tmp/newman-cli-cache-parity.log 2>&1; \
CACHE_EXIT=$$?; \
end_pass cache-parity; \
: "end_pass above is load-bearing, not tidiness: under PARALLEL=0 the main"; \
: "pass tails tmp/newman-cli.log, and the append below would otherwise be"; \
: "replayed into the counters - every cache row counted a second time."; \
cat tmp/newman-cli-cache-parity.log >> tmp/newman-cli.log 2>/dev/null || true; \
if [ "$$HARNESS_MONITORED" != "1" ] && [ "$$HARNESS_QUIET" != "1" ]; then cat tmp/newman-cli-cache-parity.log; fi; \
if [ "$$CACHE_EXIT" -ne 0 ]; then NEWMAN_EXIT=$$((NEWMAN_EXIT+1)); fi; \
if command -v jq >/dev/null 2>&1 && [ -f tmp/newman-report-cache-parity.json ]; then \
jq -c -s -f tmp/newman-merge.jq tmp/newman-report.json tmp/newman-report-cache-parity.json > tmp/newman-report-combined.json \
&& mv tmp/newman-report-combined.json tmp/newman-report.json \
|| say "$(YELLOW)Cache pass report merge failed; it remains at tmp/newman-report-cache-parity.json$(NC)"; \
fi; \
fi; \
fi; \
if [ "$$SERIAL_PASS" = "1" ]; then \
say "$(CYAN)Serial pass ([SERIAL] folders, one newman, alone against the gateway)...$(NC)"; \
rm -f tmp/harness-serial-filtered.json; SERIAL_FILTER_FAILED=0; \
$(USE_NODE); run_quiet node tests/e2e/api/runners/filter-collection.mjs \
--source tmp/harness-augmented.json \
--out tmp/harness-serial-filtered.json \
--only-serial \
$(if $(PROVIDER),--provider $(PROVIDER),) \
$(if $(FEATURE),--feature "$(FEATURE)",) \
$(if $(FOLDER),--folder "$(FOLDER)",) \
$$FEATURE_ANY_FLAG \
$$EXCLUDE_FLAG \
$${SMOKE_MANIFEST:+--smoke "$$SMOKE_MANIFEST"} \
$(if $(RERUN_FAILED),--rerun-failed --report tmp/.newman-report-prior.json,) || { say "$(RED)Serial pass filter step failed - the [SERIAL] folders did not run$(NC)"; NEWMAN_EXIT=$$((NEWMAN_EXIT+1)); SERIAL_FILTER_FAILED=1; }; \
SERIAL_COUNT="$$(grep -c '"request":' tmp/harness-serial-filtered.json 2>/dev/null || true)"; SERIAL_COUNT="$${SERIAL_COUNT:-0}"; \
if [ "$$SERIAL_FILTER_FAILED" = "1" ]; then \
:; \
elif [ "$$SERIAL_COUNT" -eq 0 ]; then \
say "$(CYAN)Serial pass: the scope selects no [SERIAL] folder - nothing to run.$(NC)"; \
elif [ -f tmp/harness-serial-filtered.json ] && budget_ok "$$SERIAL_COUNT" serial; then \
: > tmp/newman-cli-serial.log; \
add_pass '{"t":"pass","id":"serial","mode":"sequential","log":"tmp/newman-cli-serial.log","collection":"tmp/harness-serial-filtered.json"}'; \
newman run tmp/harness-serial-filtered.json \
--env-var "baseUrl=$$BASE_URL_VAL" \
$(if $(filter on true 1 yes YES y Y,$(COMPAT)),--env-var "compat=true",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_PREVIEW)),--env-var "include_preview=1",) \
$(if $(filter 1 true TRUE yes YES y Y,$(INCLUDE_SKIP)),--env-var "include_skip=1",) \
$${OPENAI_API_KEY:+--env-var "openaiKey=$$OPENAI_API_KEY"} \
$${CHATGPT_ACCESS_TOKEN:+--env-var "chatgptAccessToken=$$CHATGPT_ACCESS_TOKEN"} \
$${ANTHROPIC_API_KEY:+--env-var "anthropicKey=$$ANTHROPIC_API_KEY"} \
$${GEMINI_API_KEY:+--env-var "genaiKey=$$GEMINI_API_KEY"} \
$(if $(ENV_FILE),--environment $(ENV_FILE),) \
--reporters cli,json \
--reporter-json-export tmp/newman-report-serial.json > tmp/newman-cli-serial.log 2>&1; \
SERIAL_EXIT=$$?; \
SERIAL_RAN=1; \
end_pass serial; \
cat tmp/newman-cli-serial.log >> tmp/newman-cli.log 2>/dev/null || true; \
if [ "$$HARNESS_MONITORED" != "1" ] && [ "$$HARNESS_QUIET" != "1" ]; then cat tmp/newman-cli-serial.log; fi; \
if [ "$$SERIAL_EXIT" -ne 0 ]; then NEWMAN_EXIT=$$((NEWMAN_EXIT+1)); fi; \
if command -v jq >/dev/null 2>&1 && [ -f tmp/newman-report-serial.json ]; then \
jq -c -s -f tmp/newman-merge.jq tmp/newman-report.json tmp/newman-report-serial.json > tmp/newman-report-combined.json \
&& mv tmp/newman-report-combined.json tmp/newman-report.json \
|| say "$(YELLOW)Serial pass report merge failed; it remains at tmp/newman-report-serial.json$(NC)"; \
fi; \
elif [ "$${MAIN_EMPTY:-0}" = "1" ] && [ "$${BUDGET_EXCEEDED:-0}" = "1" ]; then \
: "budget_ok refused the serial pass, and the main pass had nothing: nothing ran at all."; \
stop_monitor; \
say "$(RED)Aborted before any launch: the serial pass would exceed HARNESS_MAX_REQUESTS=$(HARNESS_MAX_REQUESTS). Raise the cap or narrow PROVIDER/FEATURE.$(NC)"; \
exit 3; \
fi; \
fi; \
if [ "$${MAIN_EMPTY:-0}" = "1" ] && [ "$${SERIAL_RAN:-0}" != "1" ]; then \
stop_monitor; \
say "$(RED)No provider runs were launched. Check PROVIDER/FEATURE/FOLDER filters.$(NC)"; \
exit 1; \
fi; \
stop_monitor; \
: "The single teardown for the whole run: one alt-screen exit, one persistent"; \
: "table snapshot, one \$$GITHUB_STEP_SUMMARY block. Everything after this point"; \
: "prints normally to a restored main screen."; \
say "$(GREEN)Newman finished. Reports: tmp/newman-report.{json,html} + tmp/newman-cli.log$(NC)"; \
STREAM_CANCEL_EXIT=0; \
if [ -n "$(HARNESS_MAX_REQUESTS)" ]; then \
say "$(YELLOW)Skipping stream cancellation probes: their request count is not known before they run, so they are never sent under HARNESS_MAX_REQUESTS.$(NC)"; \
elif [ -z "$(SKIP_STREAM_CANCEL)" ] && [ -z "$(RERUN_FAILED)" ] && [ "$(PROVIDER)" != "passthrough" ] && { [ -z "$(FOLDER)" ] || printf '%s' "$(FOLDER)" | grep -qi 'stream'; }; then \
say "$(CYAN)Running stream cancellation probes...$(NC)"; \
$(USE_NODE); node tests/e2e/api/runners/run-stream-cancellation.mjs \
--base-url "$$BASE_URL_VAL" \
--config "$$APP_DIR_VAL/config.json" \
--server-working-dir "$(or $(HARNESS_SERVER_CWD),$(CURDIR)/transports/bifrost-http)" \
$(if $(PROVIDER),--provider "$(PROVIDER)",) \
$(if $(NONSTREAM_TRIALS),--nonstream-trials "$(NONSTREAM_TRIALS)",) \
--out tmp/stream-cancel-report.json > tmp/stream-cancel-cli.log 2>&1; \
STREAM_CANCEL_EXIT=$$?; \
if [ "$$HARNESS_QUIET" != "1" ]; then cat tmp/stream-cancel-cli.log; fi; \
else \
say "$(YELLOW)Skipping stream cancellation probes (SKIP_STREAM_CANCEL/RERUN_FAILED/FOLDER filter).$(NC)"; \
fi; \
: "Refresh the timing table from this run's reports so the NEXT sweep sizes and fills its"; \
: "shards from what actually happened. Merges onto the existing table rather than replacing"; \
: "it, so a scoped run (PROVIDER=..., FEATURE=...) refines its own rows without erasing the"; \
: "providers it never ran. Never fails the target: a sweep that already produced its results"; \
: "must not go red because a cache file could not be refreshed."; \
$(USE_NODE); run_quiet node tests/e2e/api/runners/build-harness-timings.mjs \
--dir tmp --out "$(or $(HARNESS_TIMINGS),tmp/harness-timings.json)" || true; \
say "$(CYAN)Analyzing failures...$(NC)"; \
$(USE_NODE); run_quiet node tests/e2e/api/runners/analyze-failures.mjs \
--report tmp/newman-report.json \
--bifrost-log tmp/bifrost-dev.log \
--out tmp/harness-failures.md || true; \
say "$(GREEN)Failure breakdown: tmp/harness-failures.md$(NC)"; \
if ls tmp/harness-token-parity-*.json >/dev/null 2>&1; then \
say "$(CYAN)Rendering token parity report...$(NC)"; \
$(USE_NODE); run_quiet node tests/e2e/api/runners/render-token-parity-report.mjs \
--glob "tmp/harness-token-parity-*.json" \
--out tmp/harness-token-parity.md \
--html tmp/newman-report.html || true; \
say "$(GREEN)Token parity report: tmp/harness-token-parity.md (also injected into tmp/newman-report.html when present - sequential mode / PARALLEL=0 only)$(NC)"; \
fi; \
if ls tmp/harness-cache-parity-*.json >/dev/null 2>&1; then \
say "$(CYAN)Rendering cache parity report...$(NC)"; \
$(USE_NODE); run_quiet node tests/e2e/api/runners/render-cache-parity-report.mjs \
--glob "tmp/harness-cache-parity-*.json" \
--out tmp/harness-cache-parity.md \
--html tmp/newman-report.html || true; \
say "$(GREEN)Cache parity report: tmp/harness-cache-parity.md$(NC)"; \
fi; \
if [ -n "$(CI)" ] || [ -n "$$CI" ]; then \
say "$(CYAN)CI mode - skipping interactive viewer. Upload tmp/newman-report.html, tmp/harness-failures.md, and tmp/bifrost-dev.log as workflow artifacts.$(NC)"; \
else \
preempt_viewer_port; \
say "$(CYAN)Launching interactive viewer on http://localhost:$$VIEWER_PORT_VAL (Bifrost stays up for resend)...$(NC)"; \
$(USE_NODE); node tests/e2e/api/runners/harness-viewer.mjs --report tmp/newman-report.json --port $$VIEWER_PORT_VAL & \
VIEWER_PID=$$!; \
echo $$VIEWER_PID > tmp/harness-viewer.pid; \
wait $$VIEWER_PID; \
VIEWER_EXIT=$$?; \
rm -f tmp/harness-viewer.pid; \
if [ $$VIEWER_EXIT -ne 0 ]; then \
say "$(RED)Viewer exited with code $$VIEWER_EXIT (see message above).$(NC)"; \
else \
say "$(GREEN)Viewer closed.$(NC)"; \
fi; \
fi; \
if [ "$$BUDGET_EXCEEDED" = "1" ]; then \
say "$(RED)Aborted: HARNESS_MAX_REQUESTS=$(HARNESS_MAX_REQUESTS) reached; $$BUDGET_USED request(s) were launched. The provider table above is the actual total.$(NC)"; \
exit 3; \
fi; \
if [ "$$NEWMAN_EXIT" -ne 0 ]; then exit $$NEWMAN_EXIT; fi; \
exit $$STREAM_CANCEL_EXIT