internal/autogen/generate.go
Rama main · réplica de lectura del repositorio original.
package autogen
import (
"fmt"
"slices"
"sort"
"strings"
)
// genMoeShare is Generate-Config.ps1's own expert-weight share table (line 190).
// It differs from the planner's table (plan.go) by including qwen35moe; the
// generation-side ctx sizing and the force-low-active-MoE recompute use this one.
var genMoeShare = map[string]float64{
"gemma4": 0.88,
"qwen3": 0.90,
"qwen3moe": 0.90,
"qwen35moe": 0.90,
"llama": 0.80,
"lfm2": 0.78,
"lfm2moe": 0.78,
}
// genMoeShareFor mirrors Generate-Config's Get-MoeShare (default 0.85).
func genMoeShareFor(arch string) float64 {
if s, ok := genMoeShare[strings.ToLower(arch)]; ok {
return s
}
return 0.85
}
// profile is one emitted quartermaster entry for a model: the solo variant or an
// optional ctx-tier variant. Target/Overhead are the VRAM budget the sizing math
// uses; the flags drive ub/spec/reasoning.
type profile struct {
Name string
Target float64
Overhead float64
Unlisted bool
Ctx int // 0 = auto-size; >0 forces that ctx via the manual-cap path
// CtxExact marks a Ctx that came from a custom launch pin (-c): it is the
// window the process will run with, so the sizer must not round it to a
// 4096 multiple (RoundedCtx) the way it rounds its own picks.
CtxExact bool
// IsLong marks a profile whose context is pinned at or above
// longCtxThreshold (a ctx tier, a long named variant, or a long Ctx on the
// model itself). It tops the VRAM budget's safety slack up to
// longCtxHeadroomGB (see longCtxTarget) and drops -ub to 512 on a
// small-VRAM card.
IsLong bool
// Per-variant overrides. Empty/zero => inherit the model-wide override. Set
// only by named custom variants (Override.Variants); emitProfile and the
// kv-cost sizing prefer these over the model-wide values.
KvK, KvV string
Spec string
ReasoningFmt string
Ub int // physical batch size override (0 => default)
CpuOffload int // >0 pins layers offloaded to CPU, overriding the sizer
// CpuOffloadSet distinguishes a pinned CpuOffload of 0 (every layer on GPU:
// `-ngl <blocks>` on a dense model) from "no pin". The field's zero value is
// otherwise indistinguishable from auto.
CpuOffloadSet bool
// CtxCheckpoints, when non-nil, emits --ctx-checkpoints N (0 disables the KV
// prompt-prefix checkpoint cache). nil => inherit the model-wide value, else
// the llama-server default (32). See effectiveCtxCheckpoints.
CtxCheckpoints *int
// DraftSlopeGB is the per-token VRAM (GB) the baked-in MTP drafter's own KV
// cache adds on top of the main model's, at this profile's draft KV quant.
// See mtpDraftSlopeGB: the drafter is a second llama_context over the same
// model at the SAME n_ctx, so its cost scales with the window and has to be
// part of the slope the sizer solves ctx against, not a flat overhead.
DraftSlopeGB float64
// CheckpointMinStep, when > 0, is the resolved -cms (checkpoint spacing in
// prompt tokens) for this profile. 0 => the arch default from
// defaultCheckpointMinStep. Both the emitted flag and the VRAM reserve read
// it, so they can't drift apart.
CheckpointMinStep int
// Variant, when non-nil, is the named-variant source this profile was built
// from. Its engine knobs (kvInRam/flash/mmap/mlock/threads/parallel/extraArgs)
// layer over the model-wide override at emit so a variant carries the full
// launch shape. Solo/ctx-tier profiles leave this nil and use the override.
Variant *VariantSpec
// Vision marks a profile that loads the CLIP projector: emits --mmproj
// <projector> and an image-input capabilities block so a client can send
// images. EVERY profile of a model with a projector carries this, not just
// the "-vision" twin — a served id that silently rejects images is a trap,
// since llama-server can only load a projector at spawn time and answers an
// image with "image input is not supported" for the life of the process.
// The twin remains as the id whose projector is GPU-resident.
Vision bool
// MmprojGB is the projector footprint (weights + CLIP compute reserve) this
// profile charges as VRAM overhead. Zero when CpuMmproj puts it in RAM.
MmprojGB float64
// MmprojPin is the resolved projector placement for this profile, "gpu" or
// "ram". Defaults differ by profile: the "-vision" twin is GPU-resident (it
// exists to encode images fast), every other profile keeps it in RAM (vision
// is the exception there, and the text side must not pay for it). The Default
// tab's mmproj override sets the former, the "vision" variant's the latter.
MmprojPin string
// TensorSplit is the resolved per-device layer ratio for a multi-GPU plan,
// one entry per eligible device in Index order, or nil for a single-GPU box
// (and for every non-llama class, none of which can split). Derived by the
// SIZER rather than the renderer because it depends on Overhead, which is
// only final after the compute buffer is charged: see gpuset.go.
TensorSplit []float64
// MainGpu is the device index --main-gpu pins the non-splittable buffers to.
// -1 when there is nothing to pin (single GPU / no telemetry).
MainGpu int
// CpuMmproj emits --no-mmproj-offload: the CLIP projector runs on the CPU, so
// it costs no VRAM and the profile keeps the ctx/layer placement it would
// have had without vision, at the price of a slow (host-side) image encode.
// That trade is why it is the default everywhere except the "-vision" twin:
// the encode is a one-off per image, while VRAM spent on a projector is paid
// by every token of every request, image or not.
CpuMmproj bool
}
// Generate discovers models under gf.Settings.ModelsRoot and returns a complete
// quartermaster config YAML. Port of Generate-Config.ps1. nowRFC is stamped into
// the header comment (passed in so the function stays deterministic/testable).
func Generate(gf GenerateFile, nowRFC string) (string, error) {
s := gf.Settings
rows, err := DiscoverGgufModelsMulti(s.RootList())
if err != nil {
return "", err
}
sort.Slice(rows, func(i, j int) bool {
if rows[i].ID != rows[j].ID {
return rows[i].ID < rows[j].ID
}
return rows[i].Publisher < rows[j].Publisher
})
// Name-based prompt-enhancer detection, over the same rows the emit loop is
// about to walk: it pre-fills an image model that names no enhancer of its
// own (see resolveEnhancerIDs). The other half of the same classifier keeps
// enhancers OUT of the text-encoder pool, where a same-arch, bigger-file PE
// otherwise outranks the real conditioner (see encoderpool.go).
s.autoEnhancers = detectEnhancers(rows)
// Qwen3-ASR's forced aligner is a companion found anywhere in the scanned
// roots, not beside the ASR (see audiocppaligner.go).
s.qwen3Aligner = detectQwen3Aligner(rows)
var b strings.Builder
fmt.Fprintf(&b, "# Quartermaster config - generated %s\n", nowRFC)
fmt.Fprintf(&b, "# TargetVramGB=%g MaxRamGB=%g Threads=%d\n", s.TargetVramGB, s.MaxRamGB, s.Threads)
b.WriteString("# Regen: Quartermaster startup (hash-gated)\n\n")
fmt.Fprintf(&b, "healthCheckTimeout: %d\n\n", s.HealthCheckTimeout)
emitVramBudget(&b, s)
emitRamBudget(&b, s)
emitSlotCache(&b, s.SlotCache)
emitAPIKeys(&b, s.APIKeys)
emitPromptEnhancers(&b, s.PromptEnhancers)
b.WriteString("models:\n")
var emitted []string
var coexist coexistSets
seen := map[string]bool{}
for _, row := range rows {
ov := ResolveOverride(row, gf.Overrides)
if ov != nil && ov.Skip {
continue
}
name := uniqueModelName(row, func(n string) bool { return seen[n] || slices.Contains(emitted, n) })
seen[name] = true
// A single unparseable/misdetected gguf must not nuke the whole config
// (and with it startup): note it in-band and skip, keeping every other
// model servable.
if err := emitModel(&b, s, gf, row, ov, name, &emitted, &coexist); err != nil {
fmt.Fprintf(&b, "\n # SKIPPED %q: %v\n", name, err)
continue
}
if row.IsSam {
coexist.Sam = append(coexist.Sam, name)
}
}
emitExtraImageModels(&b, s, gf.Overrides, seen, &emitted)
emitExtraModels(&b, s, seen, &emitted)
emitGroupsAndListeners(&b, s, emitted, coexist)
return b.String(), nil
}
// uniqueModelName picks the served id for a discovered row: its own id, else
// qualified by publisher, then by repo folder, then numbered. Every candidate is
// re-checked, since copies of one quant under one publisher derive the same
// publisher tag too, and a reused key makes the loader keep one block and drop
// the other. Rows arrive sorted, so the same tree always yields the same names.
func uniqueModelName(row GgufRow, taken func(string) bool) string {
name := row.ID
for _, tag := range []string{row.Publisher, row.Publisher + "-" + row.Repo} {
if !taken(name) {
return name
}
if t := slugify(tag); t != "" {
name = row.ID + "-" + t
}
}
base := name
for n := 2; taken(name); n++ {
name = fmt.Sprintf("%s-%d", base, n)
}
return name
}
// emitModel reads metadata once and emits every profile (solo, ctx tiers, game)
// for one discovered gguf.
func emitModel(b *strings.Builder, s Settings, gf GenerateFile, row GgufRow, ov *Override, name string, emitted *[]string, coexist *coexistSets) error {
// SAM models are raw *.ggml with no gguf header — route them before the
// metadata read (ReadGgufMetadata would fail). Served by sam3_server.
if row.IsSam {
emitSamModel(b, s, row, ov, name, emitted)
return nil
}
// TRELLIS.2 packages are directories served by trellis2-server, so they route
// before the metadata read for the same reason: there is no gguf to open.
if row.IsTrellis {
return emitTrellisModel(b, s, row, ov, name, emitted)
}
// HF folders (config.json + safetensors) are a directory with no gguf header,
// and vllm is the only backend here that can load one.
if row.IsHF {
return emitHFModel(b, s, row, ov, name, emitted)
}
meta, err := ReadGgufMetadataCached(row.FullPath)
if err != nil {
return fmt.Errorf("%s: %w", name, err)
}
// Video-generation GGUFs go to sd-server as their own class, and are tested
// BEFORE the image path: MiniMax-H3 ships with no metadata whatsoever, and
// the only arch string that would route a video model here ("wan") belongs
// to ERNIE-Image-Turbo, an image model. Detection is therefore structural
// (see video.go), and it has to win over isImageArch.
if vid := videoInfoFrom(meta); vid.is() {
emitVideoModel(b, s, row, ov, name, effectiveImageArch(meta), vid, meta.CondHidden, emitted)
if ov != nil {
for _, v := range ov.Variants {
if strings.TrimSpace(v.Name) == "" {
continue
}
vov := mergeImageVariant(*ov, v)
emitVideoModel(b, s, row, &vov, name+"-"+v.Name, effectiveImageArch(meta), vid, meta.CondHidden, emitted)
}
}
return nil
}
// Diffusion GGUFs go to sd-server, not llama-server: no KV cache / -ngl
// sizing applies. Detect by arch and emit a separate block. Unknown image
// archs fall through to the llama path, where the YAML "# arch=..." comment
// reveals the real arch name to add to imageArchs.
imgArch := effectiveImageArch(meta)
if isImageArch(imgArch) {
emitImageModel(b, s, row, ov, name, imgArch, meta.HasBakedEncoders, meta.CondHidden, emitted)
// Named variants are emitted as separate "<name>-<variant>" sd-server
// entries. Each inherits the model's component paths/placement and overrides
// only its own generation preset. Fleet-wide defaultVariants are llama-shaped
// (game/judge) and intentionally skipped for image models.
if ov != nil {
for _, v := range ov.Variants {
if strings.TrimSpace(v.Name) == "" {
continue
}
vov := mergeImageVariant(*ov, v)
emitImageModel(b, s, row, &vov, name+"-"+v.Name, imgArch, meta.HasBakedEncoders, meta.CondHidden, emitted)
}
}
return nil
}
// Decision models go to llama-server's /v1/systemone. Tested ahead of the
// embedder check: one built on an encoder can bake a pooling_type, and the
// --embeddings launch it would get there serves the wrong API. See decision.go.
if IsDecisionModel(meta) {
emitDecisionModel(b, s, row, ov, name, meta, emitted)
if ov.InCpuPool() {
coexist.CPU = append(coexist.CPU, name)
}
return nil
}
// Text embedders (BERT-family or any model baking a pooling_type) go to
// llama-server with --embeddings: no KV/ctx-tier/spec sizing, and the
// embedding capability buckets them out of the chat-able LLM catalog.
if IsEmbeddingModel(meta) {
emitEmbeddingModel(b, s, row, ov, name, meta, emitted)
if ov.InCpuPool() {
coexist.CPU = append(coexist.CPU, name)
}
return nil
}
// audio.cpp GGUFs go to audiocpp_server, and are tested ahead of both legacy
// speech engines: the header states the family outright (general.architecture
// = audiocpp), so there is nothing to disambiguate, and a qwentts or TTS.cpp
// command built for one would fail at load. The emitter can decline (a music
// family, or a conversion carrying no family id), in which case the model is
// deliberately dropped from the config with a comment saying why - no other
// engine here could serve it either.
if IsAudioCppModel(meta) {
emitAudioCppModel(b, s, row, ov, name, meta, emitted)
return nil
}
// Speech GGUFs go to a TTS server (OpenAI /v1/audio/speech), not llama-server:
// a Qwen3-TTS "talker" emits audio-codec tokens and loads with a paired codec
// gguf (qwentts.cpp), a TTS.cpp export is self-contained. Detect ahead of the
// LLM path — a talker is a small qwen3 LM that would otherwise route to chat.
// emitTTSModel picks the engine from the registry (see ttsBackend).
if IsTTSModel(meta, row.FileName) {
emitTTSModel(b, s, row, ov, name, meta, emitted)
// TTS.cpp runs on the CPU and is charged no VRAM, so it belongs in the
// coexistence group rather than the exclusive one: speaking a reply must
// not evict the chat model that produced it. qwentts is on the GPU and
// stays exclusive.
if kind, _ := ttsBackend(s, row, ov, meta); kind == ttsKindTTSCpp {
coexist.TTS = append(coexist.TTS, name)
}
return nil
}
// Parakeet-family ASR GGUFs go to parakeet.cpp's parakeet-server (OpenAI
// /v1/audio/transcriptions), not llama-server: transducer encoder+decoder, no
// KV/ctx/spec sizing, and the audio->text capability buckets them out of the
// chat-able LLM catalog.
if IsASRModel(meta, row.FileName) {
emitASRModel(b, s, row, ov, name, meta.Architecture, emitted)
// Same reasoning as TTS.cpp above: parakeet runs on the CPU (20-36x
// realtime) and is charged no VRAM, so dictating must not evict the chat
// model the transcript is headed for. A GPU opt-in via extraArgs keeps
// coexisting — the same accepted under-charge emitASRModel already makes
// for estVramGB.
coexist.ASR = append(coexist.ASR, name)
return nil
}
// Everything left is launched as a chat LLM, which a non-causal encoder or a
// classifier-headed file can never be: it attends both ways and ends in a
// label head, so it has nothing to generate with. The decision and embedder
// checks above already took the ones llama-server knows how to serve, so
// this is a classifier converted without decision.type. Drop it with the
// reason rather than list a chat model that answers garbage.
if meta.NonCausal || meta.ClassifierHead {
return fmt.Errorf("non-causal classifier (arch=%s) with no decision.type: llama-server cannot serve it as chat or on /v1/systemone; re-convert it as a decision model", meta.Architecture)
}
// Resolve which backend serves this LLM. A "vllm" (or other non-llama) kind
// emits through its own path — different engine, different args, no KV/-ngl
// sizing. A chosen "llama" build just swaps the exe threaded through the llama
// path below. No registry match => keep the legacy ServerExe (zero behaviour
// change for single-backend setups).
be := resolveBackend(s, ov, "llm")
if strings.EqualFold(be.Kind, "vllm") {
emitVllmModel(b, s, row, ov, name, be, meta, emitted)
return nil
}
if be.Exe != "" {
s.ServerExe = be.Exe // this model's chosen llama build wins (local copy)
}
// Every profile this model emits (solo, ctx tiers, variants, vision twin)
// runs on the same device, so a CPU-pool pin files them all in the cpu group.
if ov.InCpuPool() {
start := len(*emitted)
defer func() { coexist.CPU = append(coexist.CPU, (*emitted)[start:]...) }()
}
// Custom launch arguments are sizing constraints, not only argv overrides: a
// pinned -c/-ctk/-ub/... plans the launch (pins.go), so the emitted flags and
// the baked estVramGB describe the process that will actually run. The text
// is still appended verbatim by ComposeCmd and still wins at spawn; ov here
// is a local copy, the caller's override is untouched.
if ov != nil {
if pins := PinsFromArgs(ov.CustomArgsText()); !pins.Empty() {
pinned := pins.ApplyToOverride(*ov)
ov = &pinned
}
}
// Charge the draft sidecar only when this spec chain actually attaches it as
// -md (matchedDraftSizeGB mirrors the emitter's kind gate). A dir pairing a
// DFlash drafter to a model that runs on its baked-in MTP head must fall back
// to the flat baked-in charge, not the drafter's weights.
modelSpec := effectiveSpec(meta, ov, row.DraftKind)
specOh := draftOverheadGB(modelSpec, matchedDraftSizeGB(modelSpec, row.DraftKind, row.DraftSizeGB))
// KV quant: f16 unless it can't buy denseMinCtx in this model's budget, in
// which case q8_0 (see defaultKvQuant); settings.kvQuant pins it outright.
// Per-model override still wins. Fast flash-attention needs matched K/V and
// never iq4_nl.
kvTarget := s.TargetVramGB
if ov != nil && ov.VramTargetGB > 0 {
kvTarget = ov.VramTargetGB
}
kvDef := defaultKvQuant(s, meta, kvTarget, s.VramOverheadGB+specOh)
ovKvK, ovKvV := "", ""
if ov != nil {
ovKvK, ovKvV = ov.KvK, ov.KvV
}
kvK, kvV := resolveKvPair(ovKvK, ovKvV, kvDef, kvDef)
kvModel := GetKvCostModel(meta, kvK, kvV)
perTokGB := 0.0
kvConstGB := 0.0
if kvModel.OK {
perTokGB = kvModel.SlopeGB
kvConstGB = kvModel.ConstGB
}
modelMax := nativeCtx(meta)
if ov != nil {
// Rope scaling is the one knob that lifts the trained-length ceiling.
modelMax = ropeCeiling(meta, ov.RopeScaling, ov.Ctx)
}
kvInRam := ov != nil && ov.KvInRam
var ctxVariants []int
override := Override{}
if ov != nil {
override = *ov
ctxVariants = ov.CtxVariants
}
// A prompt enhancer is emitted as ONE profile shaped for its job (see
// enhancerCtx): none of the fleet's default variants, no vision twin.
peDir, _, isPE := promptEnhancerName(row.ID)
// Build the profile set: solo + optional ctx tiers.
// Per-model VRAM budget: an override caps it below the fleet default.
soloTarget := s.TargetVramGB
if override.VramTargetGB > 0 {
soloTarget = override.VramTargetGB
}
profiles := []profile{{
Name: name,
Target: soloTarget,
Overhead: s.VramOverheadGB + specOh,
Unlisted: override.Unlisted,
Ctx: override.Ctx,
// A model pinned to a long ctx is a long profile like any tier — it earns
// the same budget headroom (longCtxTarget) and the same -ub treatment. An
// auto ctx (0) is not: the sizer picks the window against the full budget.
IsLong: override.Ctx >= longCtxThreshold,
CpuOffload: override.CpuOffload,
CtxCheckpoints: override.CtxCheckpoints,
CheckpointMinStep: override.CheckpointMinStep,
}}
if isPE && profiles[0].Ctx == 0 {
profiles[0].Ctx = enhancerCtx
}
for _, cv := range ctxVariants {
profiles = append(profiles, profile{
Name: fmt.Sprintf("%s-%s", name, formatCtxTag(cv)),
// Budget headroom for the long rungs is charged by longCtxTarget off
// IsLong, so every long profile (and the editor preview) gets it.
Target: s.TargetVramGB,
Overhead: s.VramOverheadGB + specOh,
Ctx: cv,
IsLong: cv >= longCtxThreshold,
CtxCheckpoints: override.CtxCheckpoints,
CheckpointMinStep: override.CheckpointMinStep,
})
}
// Named custom variants: per-override ones plus the fleet-wide
// settings.DefaultVariants, each emitting "<model>-<slug>" with its own
// ctx/VRAM/kv/spec; zero fields inherit. Spec affects the VRAM overhead, so
// bake the per-variant overhead here.
fleetVariants := s.DefaultVariants
if isPE {
// The fleet's tiers (32k/64k/game) are sized for chat models; on a
// rewriter they are three more ids to pick wrong. Per-model variants are
// the user asking, and stay.
fleetVariants = nil
}
variantSpecs := append(append([]VariantSpec{}, override.Variants...), fleetVariants...)
var visionSpec *VariantSpec
for i := range variantSpecs {
v := variantSpecs[i]
if strings.TrimSpace(v.Name) == "" {
continue
}
// "vision" is reserved: it tunes the auto-generated vision twin below in
// place (ctx/vram/unlisted/kv/spec/…) instead of spawning its own profile.
if strings.EqualFold(v.Name, "vision") {
visionSpec = &variantSpecs[i]
continue
}
vTarget := s.TargetVramGB
if v.VramTargetGB > 0 {
vTarget = v.VramTargetGB
}
// Variants inherit the model-wide spec chain unless they set their own,
// so charge the draft overhead off the effective spec.
vEffSpec := modelSpec
if v.Spec != "" {
vEffSpec = v.Spec
}
vSpecOh := draftOverheadGB(vEffSpec, matchedDraftSizeGB(vEffSpec, row.DraftKind, row.DraftSizeGB))
// Standalone: a blank ctx-checkpoints uses the generator default, not the
// model-wide override value.
vCheckpoints := v.CtxCheckpoints
// Spacing DOES inherit the model-wide value (matching the effOv merge at
// emit): a blank variant -cms must not diverge from the emitted flag.
vMinStep := v.CheckpointMinStep
if vMinStep == 0 {
vMinStep = override.CheckpointMinStep
}
profiles = append(profiles, profile{
Name: fmt.Sprintf("%s-%s", name, slugify(v.Name)),
Target: vTarget,
Overhead: s.VramOverheadGB + vSpecOh,
Unlisted: v.Unlisted,
Ctx: v.Ctx,
IsLong: v.Ctx >= longCtxThreshold,
KvK: v.KvK,
KvV: v.KvV,
Spec: v.Spec,
ReasoningFmt: v.ReasoningFmt,
Ub: v.Ub,
CpuOffload: v.CpuOffload,
CtxCheckpoints: vCheckpoints,
CheckpointMinStep: vMinStep,
Variant: &variantSpecs[i],
})
}
// Vision twin: a model shipping a sibling mmproj projector gets an extra
// "-vision" profile that loads it (--mmproj) and declares image input. Listed
// by default (a distinct served id, scopeable by API keys); the config editor's
// reserved "vision" variant can still mark it unlisted. Charged the projector's
// file size as flat VRAM overhead.
// "none" drops the twin outright: a projector the user does not want wired
// (a family-inherited one they judge wrong for this finetune, or vision they
// simply never use) should cost no served id at all.
// The projector is whatever the override names, falling back to discovery.
// An explicit path is also what CREATES the twin: a model that paired with
// nothing has no twin at all today, and that is the whole point of the field
// (one shared mmproj in its own folder, pointed at by several models).
// The reserved "vision" variant may name its own, with the usual sentinel.
mmprojFile := override.MmprojFile
if v := visionSpec; v != nil {
mmprojFile = inheritStr(v.MmprojFile, mmprojFile)
}
mmprojPath, mmprojSizeGB := mmprojFor(row, mmprojFile)
modelPin := strings.ToLower(strings.TrimSpace(override.Mmproj))
// A t2i rewriter is never shown an image, so by default it wires no
// projector at all; an explicit pin still wins.
if isPE && peDir == PEDirText && modelPin == "" {
modelPin = "none"
}
if mmprojPath != "" && modelPin != "none" {
mmprojOh := MmprojVramGB(mmprojPath, mmprojSizeGB, s)
// Every OTHER profile of this model loads the projector too, in RAM
// (--no-mmproj-offload). A projector on the CPU costs no VRAM and so
// changes neither the context window nor the layer placement these
// profiles would have had without it — the only cost is a one-off encode
// per image, on the requests that actually carry one. That makes "wire it
// everywhere" strictly better than the alternative, which was a default id
// that 500s on an image for the life of the process. The Default tab's
// mmproj pin overrides the placement ("gpu" to pay VRAM for a fast encode
// here too); "none" above opts the model out of vision entirely.
defPin := modelPin
if defPin == "" {
defPin = "ram"
// An i2i rewriter is handed an image on EVERY call, so the RAM-side
// projector never banks its saving and pays a host encode each time
// (measured: minutes per reference image). Its one profile holds the
// projector in VRAM instead of leaving that to a twin.
if isPE {
defPin = "gpu"
}
}
for i := range profiles {
p := &profiles[i]
p.Vision = true
p.MmprojPin = defPin
if defPin == "ram" {
p.CpuMmproj = true
} else {
p.MmprojGB = mmprojOh
p.Overhead += mmprojOh
}
}
// The twin is the id that pays VRAM for the projector, so its default is
// the opposite: GPU-resident, for an image encode an order of magnitude
// faster than the host-side one. The reserved "vision" variant re-pins it.
vp := profile{
Name: fmt.Sprintf("%s-vision", name),
Target: soloTarget,
Overhead: s.VramOverheadGB + specOh + mmprojOh,
MmprojGB: mmprojOh,
MmprojPin: "gpu",
Ctx: override.Ctx,
CheckpointMinStep: override.CheckpointMinStep,
Vision: true,
}
// Small text window only for dedicated VL models (Qwen-VL, InternVL, …):
// their whole purpose is image chat, so the maxed 32k ctx (~2.5 GB KV) is
// wasted. A plain LLM that merely ships an mmproj sidecar keeps full context
// on its vision twin (ctx 0 → sizer picks the budget max, same as solo).
// A model or vision-variant Ctx override still wins below.
if vp.Ctx == 0 && isVLModel(name, meta.Architecture) {
vp.Ctx = s.VisionCtx
}
// A reserved "vision" variant (from the config editor) tunes the twin in
// place; blank fields keep the auto defaults, so an untouched seed emits
// the same profile as before. Engine knobs layer over the model override
// at emit via Variant.
if v := visionSpec; v != nil {
if v.VramTargetGB > 0 {
vp.Target = v.VramTargetGB
}
if v.Ctx > 0 {
vp.Ctx = v.Ctx
vp.IsLong = v.Ctx >= longCtxThreshold
}
// Spec changes the draft overhead; recharge off the variant's spec.
if v.Spec != "" {
vp.Overhead = s.VramOverheadGB + draftOverheadGB(v.Spec, matchedDraftSizeGB(v.Spec, row.DraftKind, row.DraftSizeGB)) + mmprojOh
}
vp.Unlisted = v.Unlisted
vp.KvK = v.KvK
vp.KvV = v.KvV
vp.Spec = v.Spec
vp.ReasoningFmt = v.ReasoningFmt
vp.Ub = v.Ub
vp.CpuOffload = v.CpuOffload
vp.CtxCheckpoints = v.CtxCheckpoints
if v.CheckpointMinStep > 0 {
vp.CheckpointMinStep = v.CheckpointMinStep
}
vp.Variant = v
// The vision variant IS the twin, so it is the only thing that repins
// the twin's projector; blank keeps the GPU default above. The Default
// tab's pin governs the other profiles and never reaches here.
if p := strings.ToLower(strings.TrimSpace(v.Mmproj)); p != "" {
vp.MmprojPin = p
}
}
// A "ram" pin is applied here rather than at construction so the variant
// re-charge above (which rebuilds Overhead from scratch for a vision
// variant with its own spec) can't hand the projector back.
if vp.MmprojPin == "ram" {
vp.Overhead -= mmprojOh
vp.MmprojGB = 0
vp.CpuMmproj = true
}
// "none" from the vision variant lands here rather than at the gate above,
// which only sees the model-wide value; either way no twin is emitted.
if vp.MmprojPin != "none" && !isPE {
profiles = append(profiles, vp)
}
}
// Fold the pins of the text that applies to each profile into its sizing
// inputs: a named variant's own text sizes that variant, and a ctx tier's own
// window sizes that tier. Runs before the compute-buffer pass below, which
// reads the profile's -ub (and before sizing, which reads ctx/placement).
for i := range profiles {
pinsForProfile(&profiles[i], override, meta)
}
// Charge the GPU compute buffer (logits + activations + the GPU runtime
// constant) per profile; it scales with the physical batch and lives on the
// GPU regardless of CPU expert offload, so it's flat VRAM overhead. Replaces
// the old flat 0.17 GB ubSoloOh fudge. A baked-in MTP drafter runs a second
// llama_context and is charged its own graph on top, at the same ub.
gpus := s.GpuSetOrEmpty()
for i := range profiles {
ub := effectiveUb(meta, profiles[i], ov, profiles[i].Ctx, s.TargetVramGB)
pspec := modelSpec
if profiles[i].Spec != "" {
pspec = profiles[i].Spec
}
pDraftGB := matchedDraftSizeGB(pspec, row.DraftKind, row.DraftSizeGB)
profiles[i].Overhead += computeBufferGB(meta, ub, s.ComputeBufFactor) +
mtpDraftComputeGB(meta, pspec, pDraftGB, ub, s.ComputeBufFactor)
// Every device past the main one pays its own runtime context, and none
// of it splits, so it is overhead against the POOLED budget, charged
// before the split ratio is derived from what is left. Order matters:
// PlanTensorSplit reads the FINISHED Overhead as the main device's fixed
// cost, which is what makes the pooled budget reachable without any one
// card going over (gpuset.go), so it has to run after the compute-buffer
// and drafter-graph charges above.
profiles[i].MainGpu = -1
if gpus.Multi() {
// The main device's fixed cost is the overhead as it stands HERE.
// ExtraDeviceOverheadGB is the OTHER devices' runtime contexts, which
// splitBy already charges to each of them individually; folding it in
// as well billed the secondary card's runtime twice, once to the main
// card's side of the ratio, and pushed layers off the main GPU onto a
// card that had no room for them.
mainFixedGB := profiles[i].Overhead
profiles[i].Overhead += gpus.ExtraDeviceOverheadGB()
profiles[i].TensorSplit = gpus.PlanTensorSplit(mainFixedGB)
if profiles[i].TensorSplit != nil {
profiles[i].MainGpu = gpus.PlanMainIndex()
}
}
}
for _, prof := range profiles {
// Per-variant kv quant changes the KV cost model, so resolve effective
// kv and re-derive the cost slope/const for this profile (defaults to the
// model-wide values when the variant doesn't override kv).
// Variants inherit the model-wide kv quant; a variant's own kv wins.
// A variant that pins only one side mirrors it; only a genuine mismatch
// falls back to the model-wide quant.
ekvK, ekvV := resolveKvPair(prof.KvK, prof.KvV, kvK, kvV)
ptg, kcg := perTokGB, kvConstGB
if ekvK != kvK || ekvV != kvV {
if m := GetKvCostModel(meta, ekvK, ekvV); m.OK {
ptg, kcg = m.SlopeGB, m.ConstGB
}
}
// A variant may force KV in/out of RAM; otherwise inherit the model-wide
// setting. Affects sizing, so resolve before sizeProfile.
pkvInRam := kvInRam
if prof.Variant != nil && prof.Variant.KvInRam {
pkvInRam = true
}
// A variant can turn rope scaling on (or ask for a bigger ctx) by itself, so
// the trained-length ceiling is resolved per profile, not once per model.
pModelMax := modelMax
if v := prof.Variant; v != nil {
rs := v.RopeScaling
if rs == "" {
rs = override.RopeScaling
}
if c := ropeCeiling(meta, rs, prof.Ctx); c > pModelMax {
pModelMax = c
}
}
// Multi-slot: --kv-unified gives every slot one shared KV pool, so N slots
// each holding a full-size conversation cost N x the KV. Charge that here —
// sizing then yields a PER-SLOT ctx that N of can actually fit (buildCmdLines
// multiplies it back out for -c), and every downstream number (kvReserve,
// checkpoint reserve, estVramGB) describes the whole pool. Sizing against the
// unscaled cost instead would hand slot 1 the whole card and let slots 2..N
// evict it mid-conversation.
pSlots := profileParallel(prof, override)
if pSlots > 1 {
ptg *= float64(pSlots)
kcg *= float64(pSlots)
}
// A baked-in MTP head runs as a second context over the same model at the
// same window, so its KV is a per-token cost on top of ptg, not a flat
// overhead (draftOverheadGB now only charges its compute pad). Resolve it
// per profile: a variant can carry its own spec chain and its own kv quant,
// and -ctkd follows the effective main quant.
pspec := modelSpec
if prof.Spec != "" {
pspec = prof.Spec
}
pdKvK, pdKvV := draftKvPair(override.KvKDraft, override.KvVDraft, ekvK, ekvV)
prof.DraftSlopeGB = mtpDraftSlopeFor(meta, pspec, pdKvK, pdKvV, matchedDraftSizeGB(pspec, row.DraftKind, row.DraftSizeGB))
if pSlots > 1 {
prof.DraftSlopeGB *= float64(pSlots)
}
ctx, plan, kvReserve, planCkptGB, err := sizeProfile(meta, s, prof, ptg, kcg, pModelMax, pkvInRam)
if err != nil {
return err
}
ngl, ncpuMoe := forceLowActiveMoE(meta, plan, prof, kvReserve)
if prof.CpuOffload > 0 || prof.CpuOffloadSet {
ngl, ncpuMoe = applyForcedOffload(meta, prof.CpuOffload)
}
// A CPU-device pin: nothing on the GPU, so the whole footprint the
// offload model prices for ngl 0 (its VRAM half is the compute buffer
// and overhead, which now live in RAM too) is charged to RAM.
cpuDev := override.CpuDevice()
if cpuDev {
ngl, ncpuMoe = 0, 0
}
// Re-price whenever the emitted placement is not the one GetLoadPlan
// costed, so the baked estVramGB/estRamGB describe the flags we actually
// write (forceLowActiveMoE rewrites them without touching the estimate).
if ngl != plan.Ngl || ncpuMoe != plan.NCpuMoe || cpuDev {
plan.EstVramGB, plan.EstRamGB = estForOffload(meta, prof, kvReserve, planCkptGB, ngl, ncpuMoe)
if cpuDev {
plan.EstVramGB, plan.EstRamGB = 0, plan.EstVramGB+plan.EstRamGB
}
plan.RamExceeded = s.MaxRamGB > 0 && plan.EstRamGB > s.MaxRamGB
}
// Per-variant spec/reasoning override the model-wide values for emit.
// Variants layer their engine knobs OVER the model-wide override (inherit
// at generate time: spec/draft chain, kv quant, reasoning budget, etc. flow
// down); a variant's own non-blank field still wins, and sidecar edits keep
// drifting per-variant freely.
effOv := override
if prof.Spec != "" {
effOv.Spec = prof.Spec
}
if prof.ReasoningFmt != "" {
effOv.ReasoningFmt = prof.ReasoningFmt
}
// A named variant carries the full launch shape: layer its engine knobs
// over the model-wide override (zero/empty => inherit the override value).
if v := prof.Variant; v != nil {
// Inherit the model's preserve-thinking; an explicit variant value wins.
// Both are *bool with the same nil = on meaning, so this is a pointer
// assignment, not a deref: a nil variant value must leave the model's
// own setting (which may be an explicit false) alone.
if v.PreserveThinking != nil {
effOv.PreserveThinking = v.PreserveThinking
}
if v.SlotCache != nil {
effOv.SlotCache = v.SlotCache
}
if v.FlashAttn != "" {
effOv.FlashAttn = v.FlashAttn
}
if v.Mmap != "" {
effOv.Mmap = v.Mmap
}
if v.Mlock {
effOv.Mlock = true
}
if v.Threads > 0 {
effOv.Threads = v.Threads
}
if v.Parallel > 0 {
effOv.Parallel = v.Parallel
}
// Free-form string knobs (chat template, extra args, tensor
// placement, draft KV) resolve through the shared sentinel rule:
// empty inherits, "none" forces the knob off. See inherit.go.
mergeInheritStrings(&effOv, v)
// Sampler / speculative sub-knobs: non-zero/non-nil variant value wins.
if v.Dry != nil {
effOv.Dry = v.Dry
}
if v.DryMultiplier != 0 {
effOv.DryMultiplier = v.DryMultiplier
}
if v.DryBase != 0 {
effOv.DryBase = v.DryBase
}
if v.DryAllowedLength != 0 {
effOv.DryAllowedLength = v.DryAllowedLength
}
// Sampler defaults: nil => inherit, non-nil wins INCLUDING an explicit
// 0 (greedy temp / min-p off are real settings, not "unset").
if v.Temp != nil {
effOv.Temp = v.Temp
}
if v.TopK != nil {
effOv.TopK = v.TopK
}
if v.TopP != nil {
effOv.TopP = v.TopP
}
if v.MinP != nil {
effOv.MinP = v.MinP
}
if v.PresencePenalty != nil {
effOv.PresencePenalty = v.PresencePenalty
}
if v.SpecDraftNMax != 0 {
effOv.SpecDraftNMax = v.SpecDraftNMax
}
if v.SpecDefault {
effOv.SpecDefault = true
}
if v.SpecNgramSizeN != 0 {
effOv.SpecNgramSizeN = v.SpecNgramSizeN
}
if v.SpecNgramSizeM != 0 {
effOv.SpecNgramSizeM = v.SpecNgramSizeM
}
if v.SpecNgramMinHits != 0 {
effOv.SpecNgramMinHits = v.SpecNgramMinHits
}
// Advanced knobs: variant's own non-zero/non-empty value wins.
if v.ThreadsBatch != 0 {
effOv.ThreadsBatch = v.ThreadsBatch
}
if v.Prio != 0 {
effOv.Prio = v.Prio
}
if v.DirectIo {
effOv.DirectIo = true
}
if v.NoOpOffload {
effOv.NoOpOffload = true
}
if v.NoRepack {
effOv.NoRepack = true
}
if v.CacheReuse != 0 {
effOv.CacheReuse = v.CacheReuse
}
if v.CacheRamMB != 0 {
effOv.CacheRamMB = v.CacheRamMB
}
if v.CacheIdleSlots != "" {
effOv.CacheIdleSlots = v.CacheIdleSlots
}
if v.SwaFull {
effOv.SwaFull = true
}
if v.CheckpointMinStep != 0 {
effOv.CheckpointMinStep = v.CheckpointMinStep
}
if v.ContextShift != "" {
effOv.ContextShift = v.ContextShift
}
if v.SpecDraftNMin != 0 {
effOv.SpecDraftNMin = v.SpecDraftNMin
}
if v.SlotPromptSimilarity != 0 {
effOv.SlotPromptSimilarity = v.SlotPromptSimilarity
}
if v.RopeScaling != "" {
effOv.RopeScaling = v.RopeScaling
}
if v.RopeScale != 0 {
effOv.RopeScale = v.RopeScale
}
if v.RopeFreqBase != 0 {
effOv.RopeFreqBase = v.RopeFreqBase
}
if v.YarnOrigCtx != 0 {
effOv.YarnOrigCtx = v.YarnOrigCtx
}
if v.SplitMode != "" {
effOv.SplitMode = v.SplitMode
}
if v.MainGpu != 0 {
effOv.MainGpu = v.MainGpu
}
}
emitProfile(b, s, meta, row, prof, ctx, ngl, ncpuMoe, plan, ekvK, ekvV, pkvInRam, &effOv)
*emitted = append(*emitted, prof.Name)
}
return nil
}