internal/autogen/video.go
Rama main · réplica de lectura del repositorio original.
package autogen
import (
"fmt"
"strings"
)
// Video generation is served by the SAME binary as image generation
// (stable-diffusion.cpp's sd-server), so this file is deliberately thin: it owns
// detection, the component role an image model does not have (--audio-vae), the
// per-family generation profile, and the YAML block. Everything else (budget,
// offload, placement, threads, LoRA dir) comes from imageCmdLines, which takes a
// videoInfo and branches only where video actually differs. Forking that builder
// would have duplicated ~95% identical argv and let the two drift.
//
// DETECTION IS STRUCTURAL, and that is not a style choice:
//
// - MiniMax-H3's gguf carries ZERO metadata KVs. No general.architecture, no
// general.type, nothing. The tensor table is the only thing in the file that
// says what it is.
// - The one arch string in circulation that reads "wan" belongs to
// ERNIE-Image-Turbo, which is an IMAGE model (it takes the flux.2 arm in
// resolveComponents). Routing arch=wan to video would have broken a model
// that works today.
//
// See readTensorScan: video_patch_proj / final_layer.video_out mark H3, a 5-dim
// (conv3d) patch embedding marks the Wan2.x layout, and audio_patch_proj marks a
// model that also denoises a soundtrack latent.
// Video families, as named by tensorScan.videoKind.
const (
VideoFamilyMinimaxH3 = "minimax_h3"
VideoFamilyWan = "wan"
// VideoFamilyLtxAV is Lightricks LTX-2.x: ONE flat DiT over a joint
// video+audio token stream, so the soundtrack is not a bolted-on second
// tower but half the sequence. It patchifies with a LINEAR layer
// (patchify_proj.weight) rather than H3's conv stem or Wan's conv3d, which
// is why it needs its own marker rather than falling out of either.
VideoFamilyLtxAV = "ltxav"
)
// videoComputeOverheadGB approximates the non-weight VRAM a video generation
// needs on top of the resident diffusion weights, AT THE FAMILY'S OWN DEFAULT
// PROFILE. It is imageComputeOverheadGB's sibling and is deliberately much
// larger: the sampler holds a latent for every frame at once, and the 3D VAE
// decode at the end is the real peak (it reconstructs a whole temporal tile,
// not one picture).
//
// This is a REFERENCE point, not the number the sizer uses: videoComputeOverhead
// scales it by how far the resolved profile sits from that default. Raise it (or
// pin vramTargetGB) if a clip OOMs at decode on DEFAULT framing.
const videoComputeOverheadGB = 4.0
// videoTemporalCompression is how many source frames one latent frame covers.
// Only the RATIO of two profiles in the SAME family is ever computed from it
// (see videoComputeOverhead), so it has to be right per family but never has to
// agree across them.
//
// LTX compresses 8x along time, the same 8 its frames%8==1 grid exposes. The
// others land on 4 (Wan's 4n+1 grid). H3 aligns to 17k+5 and is not a clean
// power of two, but 4 is the closest honest stand-in and the +1 slack below
// keeps the difference sub-percent at any length worth pricing.
func videoTemporalCompression(v videoInfo) int {
if v.Kind == VideoFamilyLtxAV {
return 8
}
return 4
}
// latentFrames is how many latent frames a source frame count occupies. Every
// family's grid is k*c+1: one keyframe plus k compressed windows.
func latentFrames(frames, comp int) int {
if frames < 1 {
frames = 1
}
if comp < 1 {
comp = 1
}
return (frames-1)/comp + 1
}
// videoComputeOverhead prices the non-weight peak for the profile a model will
// ACTUALLY launch with, rather than the flat constant that used to stand here.
//
// The flat number was wrong in a way that only bites big models. A 22B LTX at
// Q4 is 14.6GB, and 14.6+4.0 fits inside a 24GB card's ~22.3GB budget, so the
// sizer concluded "resident", set offload=false, and graphBudget then handed
// --max-vram whatever was LEFT - 5.9GB - to run a 14k-token joint audio+video
// graph in. sd.cpp graph-cuts to fit, fails, and the driver backs the
// allocation with host memory: a silent spill into shared memory rather than an
// error. It gets worse the longer the clip, and LTX-2.5 is specified to 20s, so
// the flat constant is off by ~3x at the top of the model's own range.
//
// Scaling is RELATIVE to the family's own default profile, deliberately:
//
// - a model left on its defaults prices EXACTLY as it did before, so this
// cannot regress an H3/Wan placement that works today;
// - per-family spatial compression cancels out of a same-family ratio
// entirely, so no absolute calibration is needed - only
// videoTemporalCompression, and only for the +1 slack.
//
// Tokens go as (W/c)*(H/c)*latentFrames, so area and latent length both enter
// linearly. Floored at the image overhead: a video model asked for one small
// still still owns a 3D VAE.
func videoComputeOverhead(v videoInfo, name string, def genDefaults) float64 {
base := videoDefaultsFor(v, name)
if base.width <= 0 || base.height <= 0 || base.frames <= 0 {
return videoComputeOverheadGB
}
w, h, f := def.width, def.height, def.frames
if w <= 0 {
w = base.width
}
if h <= 0 {
h = base.height
}
if f <= 0 {
f = base.frames
}
comp := videoTemporalCompression(v)
area := (float64(w) * float64(h)) / (float64(base.width) * float64(base.height))
length := float64(latentFrames(f, comp)) / float64(latentFrames(base.frames, comp))
if got := videoComputeOverheadGB * area * length; got > imageComputeOverheadGB {
return got
}
return imageComputeOverheadGB
}
// videoInfo is what the tensor scan concluded about a diffusion gguf's temporal
// nature. The zero value means "not a video model", which is what every image
// path passes.
type videoInfo struct {
Kind string // VideoFamily*, "" for an image model
AudioOut bool // also denoises an audio latent => needs --audio-vae
}
func (v videoInfo) is() bool { return strings.TrimSpace(v.Kind) != "" }
// videoInfoFrom lifts the scan result off a parsed gguf header.
func videoInfoFrom(meta Metadata) videoInfo {
return videoInfo{Kind: meta.VideoKind, AudioOut: meta.HasAudioOut}
}
// IsVideoModel reports whether a gguf is a video-generation DiT. Checked BEFORE
// the image path in emitModel, because a video model's arch (when it declares
// one at all) is an image arch as far as isImageArch is concerned.
func IsVideoModel(meta Metadata) bool { return videoInfoFrom(meta).is() }
// genDefaults is the generation profile baked into a model's launch command,
// applied by sd-server whenever a request omits the field. A zero field means
// "emit nothing, keep sd-server's own default".
type genDefaults struct {
steps int
cfg float64
sampler string
width int
height int
frames int
fps int
// flowShift is --flow-shift, the flow-matching timestep shift. No override
// field: a hand-written --flow-shift suppresses it through sdFlagTable.
flowShift float64
}
// videoDefaultsFor returns the family profile. An image model gets the zero
// profile here; its own (much shorter) table is imageDefaultsFor.
//
// The H3 numbers are from leejet's own reference command, and two of them are
// load-bearing rather than taste:
//
// - cfg 1.0: H3 conditions at guidance 1.0. sd-server's built-in default is
// 7.0, so a model launched without this pin renders every clip wrong.
// - frames 56: --video-frames defaults to 1, which is a single still. H3
// aligns the count UP to the 17k+5 grid (5, 22, 39, 56, 73...), NOT to the
// 4n+1 grid every other video family uses, so 56 is exact and the old 25
// would have silently become 39. sd.cpp: SDVersion-gated align_video_frames.
//
// steps is deliberately LEFT UNSET for H3. The base model is NOT distilled: it
// samples at sd-server's default 20, and the 4-step figure everyone quotes
// belongs to the turbo LoRAs (lightx2v et al), which arrive per REQUEST as
// `lora: [{path, multiplier}]` or as <lora:name:1.0> in the prompt. A launch
// flag cannot know whether a given request carries one, and pinning 4 here made
// every LoRA-less render mush.
//
// LTX is the one family whose profile cannot be read off the weights. Its
// distilled and dev checkpoints are TENSOR-IDENTICAL: same arch, same shapes,
// same names, and they differ only in the schedule they were trained to run
// (distilled carries 8 predefined sigmas and is guidance-free, dev wants ~20
// euler steps at cfg 3.0). Nothing in the file distinguishes them, so the name
// is the only signal there is, the same compromise chroma/klein already make on
// the image side. A misread here is not subtle: 8 steps on dev is noise, and
// cfg 3.0 on distilled is a scorched, over-saturated clip.
func videoDefaultsFor(v videoInfo, name string) genDefaults {
switch v.Kind {
case VideoFamilyLtxAV:
// 1280x704 at 121 frames is ~14k latent tokens by LTX's own ratios (it
// compresses 32x spatially and 8x temporally, against the 8x/4x every
// other family here uses), so it is LIGHTER per pixel than H3 at
// 640x384 despite being four times the canvas. 121 is on the 8k+1 grid
// the checkpoint requires (frames % 8 == 1).
//
// 121 is a CONSERVATIVE default, NOT a ceiling. LTX-2.5 is specified for
// 6-20 seconds, and past 10s it wants 720p/1080p at 24/25fps, which
// 1280x704 @24 already is. An earlier version of this comment claimed a
// 153-frame ceiling, read off positional_embedding_max_pos[0]=20 as "20
// latent frames". That was wrong twice over:
//
// - LTX's positional embedding is rope over fractional coordinates
// NORMALIZED by max_pos: get_fractional_positions divides the index
// grid BY it. max_pos is a divisor, not the length of a learned
// table, so there is no array to run off the end of. Past it you are
// extrapolating, which costs coherence, not a hard failure.
// - the 20 is almost certainly SECONDS. The same config carries
// audio_positional_embedding_max_pos [20], and the audio and video
// latents do not share a frame count, so an identical 20 on both
// only makes sense on a shared time axis - which is exactly what a
// joint AV transformer with use_audio_video_cross_attention needs.
// It matches the specified 20s maximum exactly.
//
// So clip length here is a VRAM question (see videoComputeOverhead), not
// a model-capability one. Override.DefaultFrames 241 / 361 / 481 buys
// 10 / 15 / 20 seconds at 24fps.
d := genDefaults{width: 1280, height: 704, frames: 121, fps: 24}
if isDistilledName(name) {
d.steps, d.cfg = 8, 1.0
} else {
d.cfg = 3.0
d.sampler = "euler"
}
return d
case VideoFamilyMinimaxH3:
return genDefaults{cfg: 1.0, width: 640, height: 384, frames: 56, fps: 24}
case VideoFamilyWan:
// Wan2.x is not distilled: leave steps/cfg at sd-server's defaults and
// pin only what is unusable by default (one frame) or wrong for the
// family's training resolution.
return genDefaults{width: 832, height: 480, frames: 81, fps: 16}
}
return genDefaults{}
}
// resolveGenDefaults is the family profile (video) or the arch profile (image)
// with the per-model Override applied, i.e. the generation settings the model will actually launch with.
//
// Split out of imageCmdLines because it now has TWO readers inside that one
// function and the order between them matters: the video compute overhead is
// priced off this result (so it must resolve BEFORE the offload decision), while
// the --steps/--width/--video-frames argv is emitted at the very end. Resolving
// it twice would let a DefaultFrames override change the emitted clip length
// without changing the placement that clip was sized for, which is precisely the
// mismatch that makes a render spill into shared memory.
func resolveGenDefaults(v videoInfo, arch, name string, condHidden int64, ov *Override) genDefaults {
def := videoDefaultsFor(v, name)
if !v.is() {
def = imageDefaultsFor(arch, name, condHidden)
}
if ov == nil {
return def
}
if ov.DefaultSteps > 0 {
def.steps = ov.DefaultSteps
}
if ov.DefaultCfg > 0 {
def.cfg = ov.DefaultCfg
}
if ov.DefaultSampler != "" {
def.sampler = ov.DefaultSampler
}
if ov.DefaultWidth > 0 {
def.width = ov.DefaultWidth
}
if ov.DefaultHeight > 0 {
def.height = ov.DefaultHeight
}
if ov.DefaultFrames > 0 {
def.frames = ov.DefaultFrames
}
if ov.DefaultFps > 0 {
def.fps = ov.DefaultFps
}
return def
}
// vaeTemporalTiling reports whether this family's VAE can actually decode in
// windows along TIME, which is a property of the VAE implementation and not of
// the flag.
//
// sd-server accepts --temporal-tiling for any model and silently falls back:
// "%s does not support temporal tiling for %s; processing the full temporal
// dimension". So an ungated flag is not an error, it is worse than one, a launch
// line that claims a VRAM lever the decode never applies. Emitting it only where
// it bites keeps the config honest about what is running.
//
// Only two implementations exist in the shipped binary, one per family:
// WanVAERunner::_compute_temporal_tiled (stateful, carrying causal conv state
// across windows) and LTXVideoVAE::decode_temporal_tiled_streaming. H3's VAE is
// a transformer autoencoder with no tiled decode path at all, which is the same
// structural difference that gives it its own family in the encoder pool.
func vaeTemporalTiling(v videoInfo) bool {
switch v.Kind {
case VideoFamilyWan, VideoFamilyLtxAV:
return true
}
return false
}
// isDistilledName reports whether a checkpoint's id names it as the distilled
// (few-step, guidance-free) sibling of a family that ships both. Name matching
// is a last resort everywhere else in this package, and it is used here only
// because the two files are byte-for-byte the same SHAPE - see videoDefaultsFor.
func isDistilledName(name string) bool {
l := strings.ToLower(name)
return strings.Contains(l, "distill") || strings.Contains(l, "turbo")
}
// videoComponents wires the component files a video DiT needs, drawing from the
// same pool the image arms use. It returns the paths and appends to missing any
// role the family REQUIRES that nothing supplied (surfaced as a YAML WARNING, so
// a half-wired model is visible rather than silently broken).
//
// llmDefault is the width-matched text encoder resolveComponents already picked:
// H3 states condHidden 5120 on condition_proj.weight, which is Qwen3-VL-32B.
func videoComponents(v videoInfo, enc EncoderSet, pool *EncoderPool, llmDefault string, missing *[]string) (vae, audioVae, t5, llm string) {
req := func(role, path string) string {
if path == "" {
*missing = append(*missing, role)
}
return path
}
switch v.Kind {
case VideoFamilyMinimaxH3:
// H3's video VAE is a transformer autoencoder, its own family in the
// pool (VaeFamilyVideo3D) precisely because it shares no tensor shape
// with the 2D AEs or with Wan's 3D causal VAE.
vae = req("vae", firstNonEmpty(enc.VideoVae, pool.Vae(VaeFamilyVideo3D)))
llm = req("llm", llmDefault)
case VideoFamilyLtxAV:
// Two autoencoders, always both: the video half decodes the picture
// latents, the audio half decodes the mel latents its joint token
// stream also produced. They are picked by FAMILY rather than by the
// enc.VideoVae/enc.AudioVae pins, which are one slot each and already
// mean H3's pair on a box that has one - see fillEncoderSet. Override
// .VaePath / .AudioVaePath remain the per-model escape hatch.
//
// The "-conv" hint is load-bearing, not cosmetic. LTX-2.5 publishes TWO
// video decoders of the same family and latent width, and with no hint
// pickByHint falls back to the first path in sorted order, which is the
// SLOW one ("...-vae-bf16" sorts before "...-vae-conv-bf16"). Measured on
// a 24GB card, same prompts, only the decoder swapped:
//
// 0.4MP @ 7s 76s -> 32s
// 1.0MP @ 5s 376s -> 53s
// 0.3MP @ 13s 437s -> 45s
//
// The ratio is 2.4x on the one case that was NOT already spilling and up
// to 9.7x on the worst, which is the shape of a memory cliff being
// removed rather than a faster kernel: the conv decoder's peak is flat in
// the latent grid, so the decode stops overflowing into host memory.
// A box holding only one of the two still gets it, hint or no hint.
vae = req("vae", pool.Vae(VaeFamilyLtx, "-conv"))
// The text encoder is a Gemma-4-12B REPUBLISHED with LTX's caption
// projection grafted on, so a stock Gemma of the same width would load
// and then condition on nothing. The path hint is what separates them;
// resolveComponents feeds it in as llmDefault.
llm = req("llm", llmDefault)
// Deliberately NOT emitting --embeddings-connectors: the 2.5 checkpoint
// declares use_embeddings_connector and carries the connector layers
// itself (video_embeddings_connector.* / audio_embeddings_connector.*),
// so pointing the flag at an external file would load a second copy.
case VideoFamilyWan:
// Wan2.x decodes through the 3D causal VAE and conditions on umT5-XXL.
// The Wan-2.1 and Qwen-Image VAEs are structurally identical, so the
// path is the only signal available (the same hint the image arm uses).
vae = req("vae", pool.Vae(VaeFamilyWan3D, "wan"))
t5 = req("t5xxl", enc.T5)
}
if v.AudioOut {
switch v.Kind {
case VideoFamilyLtxAV:
audioVae = req("audio_vae", pool.AudioVae(VaeFamilyLtx))
default:
audioVae = req("audio_vae", firstNonEmpty(enc.AudioVae, pool.AudioVae(VaeFamilyVideo3D)))
}
}
return vae, audioVae, t5, llm
}
// emitVideoModel writes an sd-server YAML entry for a video-generation GGUF.
// Same server, same budget math and placement as an image model: what differs is
// the component set (an --audio-vae), the generation profile, the larger compute
// overhead, and the declared capabilities, which is what routes the playground to
// the Video tab and the async job API rather than to /sdapi txt2img.
func emitVideoModel(b *strings.Builder, s Settings, row GgufRow, ov *Override, name, arch string, vid videoInfo, condHidden int64, emitted *[]string) {
cmd, budget, graph, offload, missing := imageCmdLines(s, row, ov, arch, name, condHidden, vid)
lines := cmd.Lines
archNote := strings.TrimSpace(arch)
if archNote == "" {
// H3 declares no arch at all; say so rather than emitting "arch=" and
// leaving the reader of the config to guess what happened.
archNote = "(none declared)"
}
fmt.Fprintf(b, "\n # arch=%s family=%s size=%gGB (video model, sd-server, budget=%gGB, max-vram=%gGB, offload=%t)\n", archNote, vid.Kind, row.SizeGB, budget, graph, offload)
if len(missing) > 0 {
fmt.Fprintf(b, " # WARNING: %s needs component(s) [%s] that were not found on disk - generation will fail until they are downloaded or declared\n", name, strings.Join(missing, ", "))
}
fmt.Fprintf(b, " %q:\n", name)
b.WriteString(" cmd: >\n")
for _, line := range lines {
fmt.Fprintf(b, " %s\n", line)
}
fmt.Fprintf(b, " ttl: %d\n", s.TtlSec)
writeMissingComponents(b, missing)
writeSingleDeviceEnv(b, s, imageExe(s, ov))
// Admission estimate = the budget this model was sized against, not the
// --max-vram it launches with (that is only the graph headroom left once the
// resident weights are paid for),
// the same accepted under-charge the image path makes: the true peak during
// 3D VAE decode runs above it, and the scheduler's in-flight guard is what
// actually keeps a second process from spawning underneath a render.
writeEstVram(b, budget)
b.WriteString(" checkEndpoint: /\n")
// Same wiring as the image path, and it rides the model entry for the same
// reason: the rewrite is not a launch flag (sd-server has no such concept), so
// the playground reads it off /api/models and offers the rewrite BEFORE the job
// is posted, which is the only place a rewrite is reviewable.
//
// The two directions read t2v / i2v on this path: the Video tab's edit
// direction is a FIRST-FRAME reference, not an img2img base. Video wants the
// rewrite more than images do - LTX is trained on single-paragraph audio-visual
// captions of 150-220 words and degrades on a short prompt, so a bare prompt is
// out of distribution rather than merely vague - and pays less for it: the
// enhancer swaps in once, ahead of a render measured in minutes.
writeEnhancerBlock(b, s, row.ID, ovPromptEnhancer(ov), ovPromptEnhancerEdit(ov), ovEnhancerPrompt(ov), ovEnhancerEditPrompt(ov))
if ov != nil && ov.Unlisted {
b.WriteString(" unlisted: true\n")
}
b.WriteString(" capabilities:\n")
b.WriteString(" in: [text]\n")
if vid.AudioOut {
b.WriteString(" out: [video, audio]\n")
} else {
b.WriteString(" out: [video]\n")
}
writeDisplayName(b, s, name)
*emitted = append(*emitted, name)
}