internal/autogen/vllm.go
Rama main · réplica de lectura del repositorio original.
package autogen
import (
"fmt"
"math"
"strings"
"sync"
)
// vllmDefaultGpuUtil is the fallback fraction of the GPU vllm may fill (weights
// + KV + activations) when the card's size is unknown, so a utilization can't be
// derived from the VRAM budget. vllm self-manages placement inside this budget —
// there is no -ngl / --n-cpu-moe layer offload like llama.cpp, so we don't place
// layers per-model; we hand vllm a cap and let its allocator fit inside it. Tune
// per-model via Override.VllmGpuUtil.
const vllmDefaultGpuUtil = 0.90
// vllm utilization bounds. The floor keeps a tiny budget on a big card from
// producing a utilization vllm can't even load weights into; the ceiling leaves
// the display/compositor room, since vllm measures utilization against TOTAL
// card memory and refuses to start if that much isn't actually free.
const (
vllmMinGpuUtil = 0.10
vllmMaxGpuUtil = 0.95
)
// vllmOverheadGB is the non-weights, non-KV VRAM vllm needs on top of the model:
// activations, CUDA graphs, and the profiling run's peak. A flat figure, unlike
// the llama path's modelled compute buffer — vllm's allocator is opaque to us, so
// this is a reserve, not an estimate.
const vllmOverheadGB = 1.5
// vllmFallbackCtx caps --max-model-len when the KV cost model can't be built
// from the gguf header (missing attention dims). Better a conservative window
// than the model's full trained context, which is what vllm would otherwise try
// to allocate KV for at startup.
const vllmFallbackCtx = 32768
// kindClasses maps a backend kind to every model class it can serve, most
// representative first, so auto-pick can match a model to a compatible backend.
// llama and vllm both serve LLMs.
//
// Nearly every engine serves exactly one class, and the singular kindClass below
// is what the UI labels a backend with. audio.cpp is the exception: one binary
// that does speech synthesis AND transcription (and, later, music), so a single
// registry row has to be resolvable from more than one class. Use
// kindServesClass for "can this backend run that model", never an equality test
// against kindClass.
func kindClasses(kind string) []string {
switch strings.ToLower(strings.TrimSpace(kind)) {
case "llama", "llama.cpp", "server", "vllm":
return []string{"llm"}
case "sd", "sd-server", "image":
return []string{"image"}
case "tts", "tts-server", "speech", "ttscpp", "tts.cpp", "kokoro":
return []string{"tts"}
case "asr", "parakeet", "parakeet-server", "transcribe":
return []string{"asr"}
case "audiocpp", "audio.cpp", "audiocpp-server":
return []string{"tts", "asr"}
case "sam", "sam3", "segment":
return []string{"segment"}
case "upscale", "realesrgan", "esrgan":
return []string{"upscale"}
case "trellis2", "trellis", "3d":
return []string{"3d"}
}
return nil
}
// kindClass is the class a backend is LABELLED with: the first one it serves.
// Matching a model against a backend is kindServesClass's job.
func kindClass(kind string) string {
if cs := kindClasses(kind); len(cs) > 0 {
return cs[0]
}
return ""
}
// kindServesClass reports whether a backend of this kind can run models of the
// given class.
func kindServesClass(kind, class string) bool {
for _, c := range kindClasses(kind) {
if c == class {
return true
}
}
return false
}
// KindClass exposes the kind→class mapping to callers outside this package (the
// managed-backend installer needs it to decide whether a class already has an
// entry before marking a freshly installed backend as the class default).
func KindClass(kind string) string { return kindClass(kind) }
// KindClasses exposes the full class set of a kind outside this package.
func KindClasses(kind string) []string { return kindClasses(kind) }
// KindServesClass exposes the class-membership test outside this package. It is
// what callers deciding "is there already a backend for this class" must use:
// an equality test against KindClass silently ignores every class a multi-class
// engine serves beyond its first.
func KindServesClass(kind, class string) bool { return kindServesClass(kind, class) }
// ClassTaken reports whether the registry already holds an entry serving any
// class the given kind would serve. It is the "an install never steals ★" test,
// asked by all three registration paths (install, activate, first-run setup).
// Shared because the multi-class case has to consider EVERY class the new
// backend would serve: audio.cpp arriving on a box that already has parakeet
// must not take the auto-pick just because nothing claimed the tts class yet.
func ClassTaken(list []BackendEntry, kind string) bool {
for _, c := range kindClasses(kind) {
for _, e := range list {
if kindServesClass(e.Kind, c) {
return true
}
}
}
return false
}
// resolvedBackend is the backend a model resolves to: its kind (drives which
// emitter runs) and exe (the launcher). A zero value => no registry entry
// matched; the caller falls back to the legacy derived exe (ServerExe etc.).
type resolvedBackend struct {
Kind string
Exe string
ID string
}
// resolveBackend picks the backend entry for a model of the given class.
// Precedence: an explicit Override.Backend (registry entry id) wins; else the
// class-default entry (Default=true); else the first entry of that class; else a
// zero value so the caller keeps the legacy behaviour. Auto-pick is thus:
// "the ★-default backend for this class, or the first one installed."
func resolveBackend(s Settings, ov *Override, class string) resolvedBackend {
if ov != nil {
if id := strings.TrimSpace(ov.Backend); id != "" {
for _, e := range s.Backends {
if e.ID == id {
return resolvedBackend{Kind: e.Kind, Exe: strings.TrimSpace(e.Path), ID: e.ID}
}
}
// A stale id (backend removed from the registry) falls through to
// auto-pick rather than emitting a broken command.
}
}
var first *BackendEntry
for i := range s.Backends {
e := &s.Backends[i]
if !kindServesClass(e.Kind, class) {
continue
}
if e.Default {
return resolvedBackend{Kind: e.Kind, Exe: strings.TrimSpace(e.Path), ID: e.ID}
}
if first == nil {
first = e
}
}
if first != nil {
return resolvedBackend{Kind: first.Kind, Exe: strings.TrimSpace(first.Path), ID: first.ID}
}
return resolvedBackend{}
}
// resolveBackendPreferring is resolveBackend with a family hint. Some classes
// hold engines that are NOT interchangeable: llama and vllm both serve any LLM
// gguf, but qwentts.cpp and TTS.cpp each read their own weights format, so the
// ★-default of the "tts" class is the wrong pick for a model of the other
// family. preferKind therefore beats Default — an explicit per-model
// Override.Backend still wins over both, since that is the user overruling the
// auto-pick on purpose. Falls back to plain resolveBackend when the class holds
// no entry of the preferred kind.
func resolveBackendPreferring(s Settings, ov *Override, class, preferKind string) resolvedBackend {
if ov != nil {
if id := strings.TrimSpace(ov.Backend); id != "" {
for _, e := range s.Backends {
if e.ID == id {
return resolvedBackend{Kind: e.Kind, Exe: strings.TrimSpace(e.Path), ID: e.ID}
}
}
}
}
if preferKind != "" {
var first *BackendEntry
for i := range s.Backends {
e := &s.Backends[i]
if !kindServesClass(e.Kind, class) || !strings.EqualFold(strings.TrimSpace(e.Kind), preferKind) {
continue
}
if e.Default {
return resolvedBackend{Kind: e.Kind, Exe: strings.TrimSpace(e.Path), ID: e.ID}
}
if first == nil {
first = e
}
}
if first != nil {
return resolvedBackend{Kind: first.Kind, Exe: strings.TrimSpace(first.Path), ID: first.ID}
}
}
return resolveBackend(s, ov, class)
}
// vllmCmdLines builds the vllm argv (exe first) for a gguf or an HF folder
// served through vllm. Shared by emitVllmModel (YAML emit) and RenderSoloCmd
// (editor preview) so the launch-parameters box matches a save. vllm loads the
// SAME file QM discovered (--quantization gguf for a gguf; an HF folder names its
// own quantization in config.json); no per-model VRAM sizing — vllm's allocator fits inside
// --gpu-memory-utilization. Ctx maps to --max-model-len (caps the KV window).
func vllmCmdLines(s Settings, row GgufRow, ov *Override, name string, be resolvedBackend, meta Metadata) []string {
modelPath := strings.ReplaceAll(row.FullPath, "\\", "/")
exe := be.Exe
if exe == "" {
exe = "vllm"
}
ctx, _ := vllmMaxModelLen(s, ov, row, meta)
util, _ := vllmGpuUtil(s, ov, vllmFootprintGB(s, ov, row, meta, ctx))
lines := []string{
exe,
"serve " + imageArg(modelPath),
"--host 127.0.0.1",
"--port ${PORT}",
"--served-model-name " + imageArg(name),
}
// An HF folder carries its own quantization_config (or none), which vllm reads
// itself; forcing gguf there makes it look for a .gguf that is not in it.
if !row.IsHF {
lines = append(lines, "--quantization gguf")
}
if !vllmExtraSetsMemory(ov) {
lines = append(lines, fmt.Sprintf("--gpu-memory-utilization %g", round2(util)))
if kv, ok := vllmKvCacheGB(ov, meta, ctx); ok {
lines = append(lines, fmt.Sprintf("--kv-cache-memory-bytes %d", int64(math.Ceil(kv*gib))))
}
}
if ctx > 0 {
lines = append(lines, fmt.Sprintf("--max-model-len %d", ctx))
}
// Upstream: "We recommend using the tokenizer from base model instead of GGUF
// model. Because the tokenizer conversion from GGUF is time-consuming and
// unstable." It is not emitted automatically: GgufRow.Repo is the local folder
// name, not a verified Hugging Face id, so deriving one would bake a guessed
// remote reference into a launch command. Set it explicitly per model.
if ov != nil {
if tok := strings.TrimSpace(ov.VllmTokenizer); tok != "" {
lines = append(lines, "--tokenizer "+imageArg(tok))
}
}
if parser, _ := vllmToolParser(ov, meta, name); parser != "" && !vllmExtraSetsToolParser(ov) {
lines = append(lines, "--enable-auto-tool-choice", "--tool-call-parser "+parser)
}
if ov != nil && ov.VllmTensorParallel > 1 {
lines = append(lines, fmt.Sprintf("--tensor-parallel-size %d", ov.VllmTensorParallel))
}
if ov != nil {
if extra := strings.TrimSpace(ov.ExtraArgs); extra != "" {
lines = append(lines, extra)
}
}
return lines
}
// emitVllmModel writes a vllm YAML entry for a gguf LLM. vllm speaks the OpenAI
// API natively (/v1/chat/completions, /health) so it needs no capabilities block
// (default text chat) and no explicit checkEndpoint (llama parity — the config
// default /health probe fits vllm too). Named/ctx-tier variants are NOT emitted
// for vllm: the llama profile/KV sizing that produces them doesn't apply here.
func emitVllmModel(b *strings.Builder, s Settings, row GgufRow, ov *Override, name string, be resolvedBackend, meta Metadata, emitted *[]string) {
if !row.IsHF && isSplitGguf(row) {
// Discovery represents a split set by shard 1 alone, which is all
// llama.cpp needs — it opens the sibling shards itself. vllm does not:
// it would load a fifth of the weights and fail somewhere downstream.
// Emit the reason as a comment rather than a command that cannot work,
// and leave the model out of `emitted` so nothing routes to it.
fmt.Fprintf(b, "\n # %s: skipped (vllm cannot load split gguf shards; merge with llama-gguf-split --merge)\n", name)
return
}
lines := vllmCmdLines(s, row, ov, name, be, meta)
ctx, ctxNote := vllmMaxModelLen(s, ov, row, meta)
footprint := vllmFootprintGB(s, ov, row, meta, ctx)
util, utilNote := vllmGpuUtil(s, ov, footprint)
format := "gguf"
if row.IsHF {
format = "safetensors"
}
kvNote := "kv=vllm-sized"
if kv, ok := vllmKvCacheGB(ov, meta, ctx); ok {
kvNote = fmt.Sprintf("kv=%.2fGB pinned", kv)
}
fmt.Fprintf(b, "\n # arch=%s size=%gGB (vllm, %s, gpu-util=%g [%s], %s, max-model-len=%d [%s])\n",
meta.Architecture, row.SizeGB, format, round2(util), utilNote, kvNote, ctx, ctxNote)
fmt.Fprintf(b, " %q:\n", name)
b.WriteString(" cmd: >\n")
for _, line := range lines {
fmt.Fprintf(b, " %s\n", line)
}
fmt.Fprintf(b, " ttl: %d\n", s.TtlSec)
// vllm allocates its whole pool (weights + KV) up front from
// --gpu-memory-utilization, so its footprint is whatever that flag grants,
// not a weights+KV sum: the router must charge what vllm will really hold.
writeEstVram(b, vllmReservedGB(util, footprint))
if ov != nil && ov.Unlisted {
b.WriteString(" unlisted: true\n")
}
writeDisplayName(b, s, name)
// Advertised in /v1/models as function_calling, so a client can tell a
// model that will take `tools` from one vLLM will 400 on.
if parser, parserNote := vllmToolParser(ov, meta, name); parser != "" || vllmExtraSetsToolParser(ov) {
b.WriteString(" capabilities:\n tools: true\n")
} else {
fmt.Fprintf(b, " # no tool-call parser (%s): vllm rejects requests carrying tools\n", parserNote)
}
*emitted = append(*emitted, name)
}
// vllmToolParsers maps an architecture (HF config.json model_type, or a gguf's
// general.architecture) to the vLLM --tool-call-parser for its family's
// tool-call format. Deliberately short: a wrong parser does not fail, it
// silently leaves tool calls unparsed in the text, which is worse than the 400
// an unmapped model gets. Anything else is picked per model in the editor.
var vllmToolParsers = map[string]string{
"qwen2": "hermes", "qwen2_moe": "hermes", "qwen2moe": "hermes",
"qwen3": "hermes", "qwen3_moe": "hermes", "qwen3moe": "hermes",
"qwen3_next": "hermes", "qwen3next": "hermes",
// Qwen3.5 moved to the XML tool format Qwen3-Coder introduced.
"qwen3_5": "qwen3_coder", "qwen3_5_moe": "qwen3_coder", "qwen35": "qwen3_coder", "qwen35moe": "qwen3_coder",
"llama": "llama3_json",
"mistral": "mistral", "mistral3": "mistral",
"gemma4": "gemma4",
"glm4_moe": "glm45", "glm4moe": "glm45",
"deepseek_v3": "deepseek_v3",
"granite": "granite", "granitemoe": "granite",
"minimax_m2": "minimax_m2",
"olmo3": "olmo3",
}
// vllmToolParser is the --tool-call-parser for a model and why: the override
// when set ("none" turns it off), else the architecture's entry. Qwen3 coder
// finetunes share model_type with the chat models but emit the XML format, so
// the served name breaks that tie.
func vllmToolParser(ov *Override, meta Metadata, name string) (parser, note string) {
if ov != nil {
switch p := strings.TrimSpace(ov.VllmToolParser); {
case strings.EqualFold(p, "none"):
return "", "turned off"
case p != "":
return p, "pinned"
}
}
arch := strings.ToLower(strings.TrimSpace(meta.Architecture))
p, ok := vllmToolParsers[arch]
if !ok {
return "", fmt.Sprintf("no parser known for arch %q; pick one in the model editor", meta.Architecture)
}
if p == "hermes" && strings.HasPrefix(arch, "qwen3") && strings.Contains(strings.ToLower(name), "coder") {
p = "qwen3_coder"
}
return p, "auto"
}
// vllmExtraSetsToolParser reports whether the model's hand-written extra args
// already name a parser: the workaround before this knob existed. Their pick
// stands, and a second copy of the flag in the launch preview reads as a bug.
func vllmExtraSetsToolParser(ov *Override) bool {
return ov != nil && strings.Contains(ov.ExtraArgs, "--tool-call-parser")
}
// isSplitGguf reports whether the row's file is one shard of a split set.
// Discovery collapses a set to its first shard, so this is the only trace left
// that the weights continue in sibling files.
func isSplitGguf(row GgufRow) bool {
return shardRe.MatchString(strings.ReplaceAll(row.FullPath, "\\", "/"))
}
// vllmBudgetGB is the VRAM this model may occupy: the per-model cap when set,
// else the fleet budget. Same precedence the llama sizer uses.
func vllmBudgetGB(s Settings, ov *Override) float64 {
if ov != nil && ov.VramTargetGB > 0 {
return ov.VramTargetGB
}
return s.TargetVramGB
}
// vllmMaxModelLen sizes --max-model-len against the VRAM budget instead of
// handing vllm the model's full trained context. vllm allocates its KV pool up
// front from --max-model-len, so a 262144-token trained window on a card that
// can hold a fraction of it is not a large context — it is a refused or OOMing
// startup. An explicit Override.Ctx always wins: a pinned window is the user
// stating the requirement, and vllm failing loudly beats silently serving a
// smaller one.
//
// The KV cost model is llama's, evaluated at f16 (vllm's KV is fp16 and paged;
// paging changes fragmentation, not the per-token cost), so this is an
// approximation of a different engine's allocator — deliberately conservative
// rather than exact. note explains the choice for the YAML comment.
func vllmMaxModelLen(s Settings, ov *Override, row GgufRow, meta Metadata) (ctx int, note string) {
trained := int(meta.ContextLength)
if ov != nil && ov.Ctx > 0 {
return ov.Ctx, "pinned"
}
avail := vllmBudgetGB(s, ov) - row.SizeGB - vllmOverheadGB
if avail <= 0 {
// Weights alone overrun the budget. vllm has no CPU offload to fall back
// on, so this model needs a bigger budget or a smaller quant; emit the
// floor and say so rather than emitting a window that implies it fits.
return RoundedCtx(0), "weights exceed the VRAM budget"
}
m := GetKvCostModel(meta, "f16", "f16")
if !m.OK {
ctx = vllmFallbackCtx
note = "no KV cost model; conservative default"
} else {
ctx = RoundedCtx(float64(MaxCtxForBudget(avail, m.SlopeGB, m.ConstGB)))
note = fmt.Sprintf("sized to %.1fGB free of budget", avail)
}
if trained > 0 && ctx > trained {
ctx, note = trained, "trained context"
}
return ctx, note
}
// vllmFootprintGB is the VRAM this model should be granted: weights + the KV
// pool for the chosen --max-model-len + the flat overhead, capped at the budget.
//
// vllm preallocates its whole --gpu-memory-utilization share and fills whatever
// is left after the weights with KV blocks, so handing it the full budget makes
// a small model hold the full budget. A 0.5B model with a 32k trained window
// needs ~3GB but would have taken 22.8GB of a 24GB card, and the router (which
// charges estVramGB) would evict everything else to admit it. When the window
// was sized TO the budget this returns the budget anyway, since that is what the
// window was computed to fill. No KV cost model => the budget: without one there
// is nothing to size a smaller share from.
//
// With the KV pool pinned (vllmKvCacheGB) the cap is dropped: vllm will hold the
// pinned pool however the window was sized, so the honest figure is the sum, and
// a share rounded down to the budget would fail vllm's startup free-memory check
// less often but charge the router less than the process takes.
func vllmFootprintGB(s Settings, ov *Override, row GgufRow, meta Metadata, ctx int) float64 {
if kv, ok := vllmKvCacheGB(ov, meta, ctx); ok {
return row.SizeGB + kv + vllmOverheadGB
}
budget := vllmBudgetGB(s, ov)
m := GetKvCostModel(meta, "f16", "f16")
if !m.OK || ctx <= 0 {
return budget
}
need := row.SizeGB + KvReserveGB(ctx, m.SlopeGB, m.ConstGB) + vllmOverheadGB
return math.Min(need, budget)
}
// vllmKvMarginFrac and vllmKvMarginGB pad the KV pool handed to vllm over what
// the llama KV cost model predicts for --max-model-len. vllm refuses to start
// when the pool cannot hold one full-length sequence, and its paged layout
// (16-token blocks, per-group page sizes on hybrid models) does not match the
// llama model byte for byte. A few hundred MB of slack is cheap next to a model
// that will not start.
const (
vllmKvMarginFrac = 0.10
vllmKvMarginGB = 0.25
)
// vllmKvCacheGB is the KV pool to pin with --kv-cache-memory-bytes: what one
// --max-model-len sequence needs, plus the margin above.
//
// Why pin it at all: without the flag vllm sizes KV as utilization x card minus
// everything else it measured (weights, the profiling run's activation peak,
// CUDA graphs, non-torch allocations). We can only guess that "everything else"
// (vllmOverheadGB), and every GB guessed low comes straight out of KV until the
// pool cannot hold one sequence and vllm refuses to start (issue #93: MiniCPM
// needed a hand-raised utilization). With the pool pinned, vllm skips that
// subtraction and allocates exactly this much KV; the overhead is whatever it
// really is. --gpu-memory-utilization is still emitted beside it because vllm
// checks free memory >= utilization x card at startup regardless of this flag.
//
// Not pinned (ok=false) when the user pinned a utilization (they asked vllm to
// fill that share with KV, the pre-#93 behaviour and the escape hatch for a vllm
// older than 0.11, which lacks the flag) or when there is no KV cost model or
// window to size from.
func vllmKvCacheGB(ov *Override, meta Metadata, ctx int) (gb float64, ok bool) {
if ov != nil && ov.VllmGpuUtil > 0 {
return 0, false
}
m := GetKvCostModel(meta, "f16", "f16")
if !m.OK || ctx <= 0 {
return 0, false
}
kv := KvReserveGB(ctx, m.SlopeGB, m.ConstGB)
return kv*(1+vllmKvMarginFrac) + vllmKvMarginGB, true
}
// vllmExtraSetsMemory reports whether the hand-written extra args already size
// vllm's memory, the workaround before the KV pin existed. Their numbers stand:
// emitting ours too would let --kv-cache-memory-bytes silently override the
// utilization the user raised to get the model to start.
func vllmExtraSetsMemory(ov *Override) bool {
return ov != nil && (strings.Contains(ov.ExtraArgs, "--gpu-memory-utilization") ||
strings.Contains(ov.ExtraArgs, "--kv-cache-memory"))
}
// vllmGpuUtil derives --gpu-memory-utilization from the model's footprint (see
// vllmFootprintGB) and the card's real size. The old flat 0.90 was a fraction of
// TOTAL memory regardless of what the budget said or what the desktop already
// holds, so it both ignored a deliberately small budget and could exceed what is
// actually free. Falls back to the flat default when the card's size can't be
// read (headless, no GPU telemetry) - there is nothing to take a fraction of.
//
// Rounded UP to the 2 decimals the flag is printed with: rounding a derived
// share to nearest can land under the footprint, and a model sized to fit
// exactly then refuses to start for lack of KV blocks.
func vllmGpuUtil(s Settings, ov *Override, footprintGB float64) (util float64, note string) {
if ov != nil && ov.VllmGpuUtil > 0 {
return ov.VllmGpuUtil, "pinned"
}
total, ok := cachedTotalVramGB()
if !ok || total <= 0 {
return vllmDefaultGpuUtil, "no GPU reading"
}
util = math.Ceil(footprintGB/total*100-1e-9) / 100
if util < vllmMinGpuUtil {
util = vllmMinGpuUtil
}
if util > vllmMaxGpuUtil {
util = vllmMaxGpuUtil
}
if budget := vllmBudgetGB(s, ov); footprintGB < budget {
return util, fmt.Sprintf("%.1fGB needed (weights+KV+overhead) of %.1fGB card, under the %.1fGB budget", footprintGB, total, budget)
}
return util, fmt.Sprintf("%.1fGB budget of %.1fGB card", footprintGB, total)
}
// vllmReservedGB is what vllm will actually hold: the granted share of the
// card when its size is known (so the floor clamp and a pinned util are
// charged too), else the computed footprint.
func vllmReservedGB(util, footprintGB float64) float64 {
if total, ok := cachedTotalVramGB(); ok && total > 0 {
return round2(util * total)
}
return footprintGB
}
// cachedTotalVramGB probes the card once per process. Sizing runs per model and
// a GPU query is not free, and the card does not change size while we run. The
// function variable is the seam tests use to supply a card without a GPU.
var cachedTotalVramGB = sync.OnceValues(func() (float64, bool) {
// currentProbePolicy, not a Settings: this probe is memoised for the process
// and filled lazily during sizing, but ResolveGpuSet has always run by then
// (both EnsureConfig and the estimate preview resolve the device set first).
return SampleTotalVramGB(autoVramSampleTimeout, currentProbePolicy())
})
// round2 trims float noise out of a derived utilization so the emitted flag
// reads 0.72, not 0.7200000000000001.
func round2(f float64) float64 { return math.Round(f*100) / 100 }