localai.conf

Rama main · réplica de lectura del repositorio original.

# LocalAI shared configuration.
#
# This file is sourced by the installer and helper scripts. Values can still be
# overridden for one command by exporting the same variable before running a
# script, for example:
#
#   LOCALAI_DIR=~/my-ai LLAMA_CPP_BACKEND=cpu ./install-local-ai.sh

# Default install directory used when LOCALAI_DIR is not set.
LOCALAI_VERSION="${LOCALAI_VERSION:-1.8.0}"
LOCALAI_DEFAULT_DIR="${LOCALAI_DEFAULT_DIR:-$HOME/ai}"
LOCALAI_DEFAULT_DIR_DISPLAY="${LOCALAI_DEFAULT_DIR_DISPLAY:-~/ai}"

# Source used by localai update when refreshing installed helper scripts.
LOCALAI_REPO_URL="${LOCALAI_REPO_URL:-https://github.com/hossbit/local-ai-server.git}"
LOCALAI_TARBALL_BASE="${LOCALAI_TARBALL_BASE:-https://github.com/hossbit/local-ai-server/archive}"
LOCALAI_REF="${LOCALAI_REF:-main}"
# When set to a local checkout path, `localai update` uses it instead of
# fetching LOCALAI_REPO_URL from GitHub -- useful while developing/testing
# local changes (e.g. to lib/*.sh) that aren't published yet. Empty (default)
# means always fetch from LOCALAI_REPO_URL, matching every other install.
LOCALAI_LOCAL_SOURCE_DIR="${LOCALAI_LOCAL_SOURCE_DIR:-}"

# Installed directory names and state files.
LOCALAI_USER_BIN_DIR="${LOCALAI_USER_BIN_DIR:-$HOME/.local/bin}"
LOCALAI_CLI_NAME="${LOCALAI_CLI_NAME:-localai}"
LOCALAI_BIN_SUBDIR="${LOCALAI_BIN_SUBDIR:-bin}"
LOCALAI_LIB_SUBDIR="${LOCALAI_LIB_SUBDIR:-lib}"
LOCALAI_CONF_SUBDIR="${LOCALAI_CONF_SUBDIR:-conf}"
LOCALAI_MODELS_SUBDIR="${LOCALAI_MODELS_SUBDIR:-models}"
LOCALAI_LOGS_SUBDIR="${LOCALAI_LOGS_SUBDIR:-logs}"
LOCALAI_CONFIG_FILE="${LOCALAI_CONFIG_FILE:-config.yaml}"
LOCALAI_PORT_FILE="${LOCALAI_PORT_FILE:-port}"
LOCALAI_PID_FILE="${LOCALAI_PID_FILE:-llama-swap.pid}"
LOCALAI_BACKEND_FILE="${LOCALAI_BACKEND_FILE:-llama-cpp-backend}"
# Generated record of the versions/artifacts actually installed (JSON). It is
# rewritten by every install/update; it never pins future versions.
LOCALAI_MANIFEST_FILE="${LOCALAI_MANIFEST_FILE:-installed.json}"

# Update safety. `localai update` keeps the llama.cpp backend and llama-swap
# it replaces until the restarted service passes its readiness check, and
# restores them automatically if it does not. With LOCALAI_KEEP_PREVIOUS=1
# the replaced versions are then kept in $BIN_DIR/$LOCALAI_PREVIOUS_SUBDIR so
# `localai rollback` can switch back later (one previous version is kept; it
# costs the disk space of the replaced backend). Set 0 to delete them once an
# update succeeds.
LOCALAI_KEEP_PREVIOUS="${LOCALAI_KEEP_PREVIOUS:-1}"
LOCALAI_PREVIOUS_SUBDIR="${LOCALAI_PREVIOUS_SUBDIR:-previous}"

# Local HTTP endpoint defaults.
LOCALAI_LISTEN_HOST="${LOCALAI_LISTEN_HOST:-127.0.0.1}"
LOCALAI_DEFAULT_PORT="${LOCALAI_DEFAULT_PORT:-11435}"

# systemd user service.
LOCALAI_SERVICE_NAME="${LOCALAI_SERVICE_NAME:-localai.service}"
LOCALAI_SYSTEMD_USER_DIR="${LOCALAI_SYSTEMD_USER_DIR:-$HOME/.config/systemd/user}"

# llama.cpp release and backend selection.
LLAMA_CPP_VERSION="${LLAMA_CPP_VERSION:-latest}"
LOCALAI_DEFAULT_BACKEND="${LOCALAI_DEFAULT_BACKEND:-vulkan}"
LLAMA_CPP_RELEASE_API="${LLAMA_CPP_RELEASE_API:-https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/${LLAMA_CPP_VERSION}}"
LLAMA_CPP_LATEST_API="${LLAMA_CPP_LATEST_API:-https://api.github.com/repos/ggml-org/llama.cpp/releases/latest}"

# Upstream llama.cpp now publishes two tag families (see
# https://github.com/ggml-org/ggml/discussions/1579): b[NUM] "bleeding-edge"
# builds cut on nearly every commit to master (always marked prerelease on
# GitHub), and slower vX.Y.Z "stable" builds recommended for downstream
# distribution (GitHub's actual /releases/latest). LLAMA_CPP_VERSION=latest
# resolves through whichever of these two LLAMA_CPP_CHANNEL selects; pin
# LLAMA_CPP_VERSION to an exact tag (either family) to bypass channel
# resolution entirely. bleeding-edge matches what this project has always
# tracked.
LLAMA_CPP_CHANNEL="${LLAMA_CPP_CHANNEL:-bleeding-edge}"
LLAMA_CPP_RELEASES_API="${LLAMA_CPP_RELEASES_API:-https://api.github.com/repos/ggml-org/llama.cpp/releases}"
LLAMA_CPP_RELEASES_PAGE_SIZE="${LLAMA_CPP_RELEASES_PAGE_SIZE:-20}"
# A stable vX.Y.Z release ships no backend binaries of its own -- only this
# asset, naming the bleeding-edge release actually built from that point.
LLAMA_CPP_NIGHTLY_TAG_ASSET="${LLAMA_CPP_NIGHTLY_TAG_ASSET:-nightly-tag.txt}"

# Release asset patterns for each supported llama.cpp backend.
LLAMA_CPP_CPU_ASSET_RE="${LLAMA_CPP_CPU_ASSET_RE:-ubuntu-x64\\.tar\\.gz$}"
LLAMA_CPP_VULKAN_ASSET_RE="${LLAMA_CPP_VULKAN_ASSET_RE:-ubuntu-vulkan-x64\\.tar\\.gz$}"
LLAMA_CPP_ROCM_ASSET_RE="${LLAMA_CPP_ROCM_ASSET_RE:-ubuntu-rocm-[0-9.]+-x64\\.tar\\.gz$}"
LLAMA_CPP_OPENVINO_ASSET_RE="${LLAMA_CPP_OPENVINO_ASSET_RE:-ubuntu-openvino-[0-9.]+-x64\\.tar\\.gz$}"
LLAMA_CPP_SYCL_FP16_ASSET_RE="${LLAMA_CPP_SYCL_FP16_ASSET_RE:-ubuntu-sycl-fp16-x64\\.tar\\.gz$}"
LLAMA_CPP_SYCL_FP32_ASSET_RE="${LLAMA_CPP_SYCL_FP32_ASSET_RE:-ubuntu-sycl-fp32-x64\\.tar\\.gz$}"
LOCALAI_CPU_FALLBACK="${LOCALAI_CPU_FALLBACK:-1}"

# CUDA backend (LLAMA_CPP_BACKEND=cuda, or auto when a working NVIDIA driver +
# nvcc are both detected). Unlike the other backends this is a source build,
# to match the local toolkit and GPU. Upstream also publishes Ubuntu CUDA
# archives tied to specific runtime versions; LocalAI keeps source builds.
# See README.md and .claude/skills/localai-cuda-backend/ for details.
#
# Explicit cuda is strict by default (this must stay 0): an explicit backend
# request should install that exact backend or fail with an actionable
# error, never silently substitute a different one -- only `auto` picks a
# fallback backend on its own. Set to 1 to opt into the old silent-fallback
# behavior instead.
LOCALAI_CUDA_FALLBACK="${LOCALAI_CUDA_FALLBACK:-0}"
LOCALAI_NVCC="${LOCALAI_NVCC:-}"
LOCALAI_CUDA_ARCHITECTURES="${LOCALAI_CUDA_ARCHITECTURES:-auto}"
LOCALAI_BUILD_JOBS="${LOCALAI_BUILD_JOBS:-}"
LLAMA_CPP_SOURCE_URL="${LLAMA_CPP_SOURCE_URL:-https://github.com/ggml-org/llama.cpp.git}"
LOCALAI_CUDA_META_FILE="${LOCALAI_CUDA_META_FILE:-llama-cpp-cuda-meta}"
# Ordinary build tools needed only for a CUDA source build (installed in
# addition to LOCALAI_*_PACKAGES, only when the resolved backend is cuda).
# Never includes the CUDA Toolkit itself or NVIDIA driver/repo packages --
# those are the user's own, deliberate install (see README.md).
LOCALAI_CUDA_APT_PACKAGES="${LOCALAI_CUDA_APT_PACKAGES:-build-essential cmake ninja-build pkg-config git}"
LOCALAI_CUDA_DNF_PACKAGES="${LOCALAI_CUDA_DNF_PACKAGES:-gcc gcc-c++ make cmake ninja-build pkgconfig git}"
LOCALAI_CUDA_YUM_PACKAGES="${LOCALAI_CUDA_YUM_PACKAGES:-gcc gcc-c++ make cmake ninja-build pkgconfig git}"

# llama-swap release.
LLAMA_SWAP_VERSION="${LLAMA_SWAP_VERSION:-latest}"
LLAMA_SWAP_RELEASE_API="${LLAMA_SWAP_RELEASE_API:-https://api.github.com/repos/mostlygeek/llama-swap/releases/tags/${LLAMA_SWAP_VERSION}}"
LLAMA_SWAP_LATEST_API="${LLAMA_SWAP_LATEST_API:-https://api.github.com/repos/mostlygeek/llama-swap/releases/latest}"
LLAMA_SWAP_ASSET_RE="${LLAMA_SWAP_ASSET_RE:-linux_amd64\\.tar\\.gz$}"
LLAMA_SWAP_URL="${LLAMA_SWAP_URL:-}"
LLAMA_SWAP_INSTALL_PATH="${LLAMA_SWAP_INSTALL_PATH:-}"
LLAMA_SWAP_BIN="${LLAMA_SWAP_BIN:-$LLAMA_SWAP_INSTALL_PATH}"
LOCALAI_SKIP_DIGEST="${LOCALAI_SKIP_DIGEST:-0}"

# Package names used by install-local-ai.sh.
LOCALAI_APT_PACKAGES="${LOCALAI_APT_PACKAGES:-ca-certificates curl iproute2 jq libvulkan1 mesa-vulkan-drivers tar}"
LOCALAI_DNF_PACKAGES="${LOCALAI_DNF_PACKAGES:-ca-certificates curl iproute jq mesa-vulkan-drivers tar vulkan-loader}"
LOCALAI_YUM_PACKAGES="${LOCALAI_YUM_PACKAGES:-ca-certificates curl iproute jq mesa-vulkan-drivers tar vulkan-loader}"

# Runtime defaults used when rebuild-config.sh generates llama-swap config.
# CPU backend uses smaller, CPU-safe defaults unless explicitly overridden.
if [ "${LLAMA_CPP_BACKEND:-$LOCALAI_DEFAULT_BACKEND}" = "cpu" ]; then
  LOCALAI_CTX_SIZE="${LOCALAI_CTX_SIZE:-4096}"
  LOCALAI_N_GPU_LAYERS="${LOCALAI_N_GPU_LAYERS:-0}"
  LOCALAI_FLASH_ATTN="${LOCALAI_FLASH_ATTN:-0}"
  LOCALAI_PARALLEL="${LOCALAI_PARALLEL:-1}"
  LOCALAI_BATCH_SIZE="${LOCALAI_BATCH_SIZE:-}"
  LOCALAI_UBATCH_SIZE="${LOCALAI_UBATCH_SIZE:-}"
  # Self-speculative n-gram decoding needs no extra VRAM, but the CPU backend
  # is already the slow-path fallback, so leave it off unless requested.
  LOCALAI_SPEC_TYPE="${LOCALAI_SPEC_TYPE:-}"
else
  LOCALAI_CTX_SIZE="${LOCALAI_CTX_SIZE:-16384}"
  LOCALAI_N_GPU_LAYERS="${LOCALAI_N_GPU_LAYERS:-8}"
  LOCALAI_FLASH_ATTN="${LOCALAI_FLASH_ATTN:-1}"
  LOCALAI_PARALLEL="${LOCALAI_PARALLEL:-2}"
  LOCALAI_BATCH_SIZE="${LOCALAI_BATCH_SIZE:-2048}"
  LOCALAI_UBATCH_SIZE="${LOCALAI_UBATCH_SIZE:-512}"
  # ngram-simple drafts tokens from patterns already seen in the response
  # (e.g. repeated code) and is verified against the target model every step,
  # so it never changes output quality, only speed. No extra model/VRAM needed.
  LOCALAI_SPEC_TYPE="${LOCALAI_SPEC_TYPE:-ngram-simple}"
fi
# Optional on newer llama.cpp: greedy or probabilistic (draft-simple/draft-mtp only).
# Empty preserves upstream defaults and compatibility with older pinned builds.
LOCALAI_SPEC_DRAFT_SAMPLING="${LOCALAI_SPEC_DRAFT_SAMPLING:-}"
LOCALAI_SPEC_DRAFT_N_MAX="${LOCALAI_SPEC_DRAFT_N_MAX:-16}"
# Speculative-draft model KV cache types (--cache-type-k-draft/-v-draft).
# Independent of the main model's CACHE_TYPE_K/V so a draft model can use a
# cheaper cache (e.g. q8_0) without changing the main model's cache. Empty by
# default (llama-server falls back to its own defaults); set per model in
# models.d when SPEC_DRAFT_MODEL is used.
LOCALAI_SPEC_DRAFT_CACHE_TYPE_K="${LOCALAI_SPEC_DRAFT_CACHE_TYPE_K:-}"
LOCALAI_SPEC_DRAFT_CACHE_TYPE_V="${LOCALAI_SPEC_DRAFT_CACHE_TYPE_V:-}"
# Draft-model MoE CPU offload (--cpu-moe-draft/--n-cpu-moe-draft): same
# behavior as LOCALAI_CPU_MOE/LOCALAI_N_CPU_MOE but applied to the speculative
# draft model only, for MoE draft models too big for VRAM. SPEC_DRAFT_N_CPU_MOE
# wins over SPEC_DRAFT_CPU_MOE when both are set, matching llama-server. Off by
# default; set per model in models.d when SPEC_DRAFT_MODEL is used.
LOCALAI_SPEC_DRAFT_CPU_MOE="${LOCALAI_SPEC_DRAFT_CPU_MOE:-0}"
LOCALAI_SPEC_DRAFT_N_CPU_MOE="${LOCALAI_SPEC_DRAFT_N_CPU_MOE:-}"
# MoE (Mixture-of-Experts) CPU offload. Lets very large MoE models (DeepSeek,
# GLM, Qwen3-MoE, Kimi K2, gpt-oss, ...) run on a GPU too small to hold every
# expert by keeping some/all expert weights in system RAM instead of VRAM.
# LOCALAI_CPU_MOE=1 keeps every expert layer on CPU (--cpu-moe, simplest
# "make it fit" option, GPU still holds attention/shared layers). LOCALAI_N_CPU_MOE=N
# keeps only the first N expert layers on CPU (--n-cpu-moe N, finer control --
# lower N leaves more experts on GPU for speed); N_CPU_MOE wins over CPU_MOE
# when both are set, matching llama-server's own precedence. Both off/empty by
# default so dense-model installs are unaffected; set per model in models.d
# since only some installed models are MoE.
LOCALAI_CPU_MOE="${LOCALAI_CPU_MOE:-0}"
LOCALAI_N_CPU_MOE="${LOCALAI_N_CPU_MOE:-}"
# Optional GPU cache for CPU-offloaded MoE experts, in MiB. Disabled by default.
# Requires a llama-server with --moe-cache-mib; choose per model, leaving VRAM
# for attention and the context cache. This does not benefit dense models.
LOCALAI_MOE_CACHE_MIB="${LOCALAI_MOE_CACHE_MIB:-0}"
# Work around intermittent streaming-related 502 / proxy error: EOF failures.
# Requires llama-swap v263+. Use DISABLE_KEEP_ALIVES=1 in one model's override
# only when needed; each request then opens a fresh backend connection.
LOCALAI_DISABLE_KEEP_ALIVES="${LOCALAI_DISABLE_KEEP_ALIVES:-0}"
# Reasoning/thinking-model controls (DeepSeek-R1, GLM, Qwen3 thinking,
# gpt-oss, ...). Blank/0 by default -- llama-server already auto-detects
# reasoning behavior from the model's own chat template; only set these to
# override it. REASONING is on/off/auto; REASONING_BUDGET is a token budget
# (-1 unrestricted, 0 = answer immediately with no thinking); REASONING_FORMAT
# controls thought-tag handling (none/deepseek/deepseek-legacy);
# REASONING_PRESERVE keeps the reasoning trace across turns instead of
# stripping it from history.
LOCALAI_REASONING="${LOCALAI_REASONING:-}"
LOCALAI_REASONING_BUDGET="${LOCALAI_REASONING_BUDGET:-}"
LOCALAI_REASONING_FORMAT="${LOCALAI_REASONING_FORMAT:-}"
LOCALAI_REASONING_PRESERVE="${LOCALAI_REASONING_PRESERVE:-0}"
LOCALAI_THREADS="${LOCALAI_THREADS:-$(nproc 2>/dev/null || printf '6')}"
LOCALAI_CACHE_TYPE_K="${LOCALAI_CACHE_TYPE_K:-f16}"
LOCALAI_CACHE_TYPE_V="${LOCALAI_CACHE_TYPE_V:-f16}"
# AUTO_TUNE computes per-model ctx-size/n-gpu-layers/cache-type/flash-attn from
# actual GGUF file size and detected RAM/VRAM (see
# lib/common.sh:compute_model_runtime_defaults), overriding the flat
# LOCALAI_CTX_SIZE/LOCALAI_N_GPU_LAYERS/CACHE_TYPE_* values above per model.
# Set to 0 to force the flat global values onto every model instead.
LOCALAI_AUTO_TUNE="${LOCALAI_AUTO_TUNE:-1}"
# Per-model override files: conf/models.d/<model-id>.conf, sourced (bash) after
# auto-tuning so you can hand-set CTX_SIZE, N_GPU_LAYERS, CACHE_TYPE_K/V,
# FLASH_ATTN, SPEC_TYPE, SPEC_DRAFT_N_MAX, TTL, ALIASES, MMPROJ,
# SET_TEMPERATURE, SET_TOP_P, EXTRA_ARGS, SPLIT_MODE, TENSOR_SPLIT, MAIN_GPU,
# DEVICE, CPU_MOE, N_CPU_MOE, MOE_CACHE_MIB, DISABLE_KEEP_ALIVES, OVERRIDE_TENSOR, REASONING, REASONING_BUDGET,
# REASONING_FORMAT, REASONING_PRESERVE, SPEC_DRAFT_MODEL, SPEC_DRAFT_N_MIN,
# SPEC_DRAFT_DEVICE, SPEC_DRAFT_NGL, SPEC_DRAFT_CACHE_TYPE_K,
# SPEC_DRAFT_CACHE_TYPE_V, SPEC_DRAFT_CPU_MOE, SPEC_DRAFT_N_CPU_MOE,
# MMPROJ_URL, MMPROJ_OFFLOAD, IMAGE_MIN_TOKENS, IMAGE_MAX_TOKENS, LORA, or
# LORA_SCALED for one model. See
# README.md.
# EXTRA_ARGS can be a plain string ("--foo bar") for simple flags, or a bash
# array (EXTRA_ARGS=(--foo 'bar baz')) when a value needs embedded spaces or
# quotes -- e.g. EXTRA_ARGS=(--chat-template-kwargs '{"enable_thinking":false}')
# -- array elements are safely quoted for you, no manual escaping needed.
LOCALAI_MODELS_OVERRIDE_SUBDIR="${LOCALAI_MODELS_OVERRIDE_SUBDIR:-models.d}"
# Shorter default TTL for embedding models (cheap/fast to reload) so they
# don't hold GPU memory as long as chat models between requests.
LOCALAI_EMBEDDING_TTL="${LOCALAI_EMBEDDING_TTL:-120}"
# Optional request-body limit in MB (llama-swap v266+). Blank preserves the
# upstream default, currently 250 MB, and works with older versions too.
# Set globally in conf/localai.conf, for example LOCALAI_MAX_REQUEST_BODY_MB=500.
LOCALAI_MAX_REQUEST_BODY_MB="${LOCALAI_MAX_REQUEST_BODY_MB:-}"
# Comma or space separated model IDs to warm on `localai start`/`restart`
# via llama-swap's hooks.on_startup.preload.
LOCALAI_PRELOAD_MODELS="${LOCALAI_PRELOAD_MODELS:-}"
# Exposes llama-swap's built-in /metrics endpoint (system + GPU stats for
# Prometheus/Grafana). No llama-server rebuild or extra flags needed. When 0,
# the generated config sets llama-swap's performance.disabled: true so the
# stats really stop being collected (upstream collects by default).
LOCALAI_METRICS_ENABLED="${LOCALAI_METRICS_ENABLED:-1}"
# `localai key` API-key registry, stored under conf/ (outside Git) with mode
# 0600. See README.md for the create/list/revoke/rotate workflow.
LOCALAI_API_KEY_FILE="${LOCALAI_API_KEY_FILE:-api-keys.tsv}"
# When 1, config generation refuses to produce a config with zero active
# keys, so the API can never silently fall back to unauthenticated. Leave at
# 0 (default) so existing installs keep working until you create a key.
LOCALAI_REQUIRE_API_KEY="${LOCALAI_REQUIRE_API_KEY:-0}"
# Active keys are rendered into their own file under conf/, loaded via
# llama-swap's --config-dir, instead of inline in config.yaml -- keeps
# config.yaml (models, ports, tuning) and plaintext secrets in separate
# files. Both directory and filename are just names inside conf/.
LOCALAI_KEYS_SUBDIR="${LOCALAI_KEYS_SUBDIR:-keys.d}"
LOCALAI_KEYS_FILE="${LOCALAI_KEYS_FILE:-keys.yaml}"
LOCALAI_JINJA="${LOCALAI_JINJA:-1}"
LOCALAI_MLOCK="${LOCALAI_MLOCK:-0}"
LOCALAI_NO_MMAP="${LOCALAI_NO_MMAP:-0}"
LOCALAI_EXTRA_LLAMA_ARGS="${LOCALAI_EXTRA_LLAMA_ARGS:-}"
# MTP (multi-token prediction) assistant models directory. When set, llama-server
# auto-loads MTP draft models from this directory (upstream --models-dir, added
# in llama.cpp #24431). Empty (default) disables it. Path is quoted for you.
LOCALAI_MTP_MODELS_DIR="${LOCALAI_MTP_MODELS_DIR:-}"

# Multi-GPU placement. All blank by default, which leaves llama-server on its
# own defaults (split-mode layer across every visible device). Override per
# model in models.d when different models should land on different GPUs.
# See docs/multi-gpu.md in the llama.cpp repo.
LOCALAI_SPLIT_MODE="${LOCALAI_SPLIT_MODE:-}"
LOCALAI_TENSOR_SPLIT="${LOCALAI_TENSOR_SPLIT:-}"
LOCALAI_MAIN_GPU="${LOCALAI_MAIN_GPU:-}"
LOCALAI_DEVICE="${LOCALAI_DEVICE:-}"
LOCALAI_HEALTH_CHECK_TIMEOUT="${LOCALAI_HEALTH_CHECK_TIMEOUT:-300}"
LOCALAI_GLOBAL_TTL="${LOCALAI_GLOBAL_TTL:-900}"
# Seconds stop.sh waits for llama-swap to finish unloading models before it
# reports a failure. The old 5-second hard-coded wait was shorter than many
# model unloads.
LOCALAI_STOP_TIMEOUT="${LOCALAI_STOP_TIMEOUT:-60}"