internal/autogen/liveoffload_test.go

Rama main · réplica de lectura del repositorio original.

package autogen

import (
	"reflect"
	"testing"
)

func TestLiveOffload_Parsing(t *testing.T) {
	args := []string{
		"-m", "/models/qwen.gguf", "-ngl", "99", "-c", "8192",
		"-ctk", "q8_0", "-ctv", "q8_0", "--no-kv-offload",
		"--spec-type", "draft-mtp", "--ctx-checkpoints", "0",
		"--n-cpu-moe", "12",
	}

	if v, _ := argVal(args, "-m", "--model"); v != "/models/qwen.gguf" {
		t.Fatalf("model = %q", v)
	}
	if got := atoiFlag(args, "-c", "--ctx-size"); got != 8192 {
		t.Fatalf("ctx = %d", got)
	}
	if got := flagStr(args, "-ctk", "--cache-type-k"); got != "q8_0" {
		t.Fatalf("ctk = %q", got)
	}
	if !hasFlag(args, "--no-kv-offload") {
		t.Fatal("expected --no-kv-offload detected")
	}
	if got := specTypes(args); got != "draft-mtp" {
		t.Fatalf("spec = %q", got)
	}
	if n, ok := atoiFlagOK(args, "--ctx-checkpoints"); !ok || n != 0 {
		t.Fatalf("ctx-checkpoints = %d ok=%v", n, ok)
	}
	if _, idx := argVal(args, "--missing"); idx != -1 {
		t.Fatalf("absent flag idx = %d", idx)
	}
}

func TestLiveOffload_Rewrite(t *testing.T) {
	// MoE: --n-cpu-moe present -> bumped in place, -ngl untouched value rewritten.
	moe := []string{"-m", "x.gguf", "-ngl", "99", "--n-cpu-moe", "12"}
	_, nglIdx := argVal(moe, "-ngl")
	_, ncIdx := argVal(moe, "--n-cpu-moe")
	got := rewriteOffload(moe, nglIdx, ncIdx, 99, 40)
	want := []string{"-m", "x.gguf", "-ngl", "99", "--n-cpu-moe", "40"}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("moe bump: got %v want %v", got, want)
	}

	// Dense: no --n-cpu-moe -> -ngl reduced, no flag appended (newNcpu 0).
	dense := []string{"-m", "x.gguf", "-ngl", "32"}
	_, nglIdx = argVal(dense, "-ngl")
	got = rewriteOffload(dense, nglIdx, -1, 10, 0)
	want = []string{"-m", "x.gguf", "-ngl", "10"}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("dense reduce: got %v want %v", got, want)
	}

	// MoE without baked flag -> append --n-cpu-moe.
	bare := []string{"-m", "x.gguf", "-ngl", "99"}
	_, nglIdx = argVal(bare, "-ngl")
	got = rewriteOffload(bare, nglIdx, -1, 99, 8)
	want = []string{"-m", "x.gguf", "-ngl", "99", "--n-cpu-moe", "8"}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("moe append: got %v want %v", got, want)
	}

	// rewriteOffload must not mutate the input slice.
	if !reflect.DeepEqual(bare, []string{"-m", "x.gguf", "-ngl", "99"}) {
		t.Fatalf("input mutated: %v", bare)
	}
}

func TestLiveOffload_NoGpuReadingIsNoop(t *testing.T) {
	args := []string{"-m", "x.gguf", "-ngl", "99", "--n-cpu-moe", "12"}
	out, err := LiveOffloadArgs(Settings{}, args, 0, false, nil)
	if err != nil {
		t.Fatalf("err = %v", err)
	}
	if !reflect.DeepEqual(out, args) {
		t.Fatalf("expected unchanged, got %v", out)
	}
}

func TestLiveOffload_NonLlamaCmdIsNoop(t *testing.T) {
	// A custom cmd with no --max-vram and no .gguf model -> left alone.
	args := []string{"--model", "/models/flux.safetensors", "-ngl", "10"}
	out, err := LiveOffloadArgs(Settings{}, args, 4, true, nil)
	if err != nil {
		t.Fatalf("err = %v", err)
	}
	if !reflect.DeepEqual(out, args) {
		t.Fatalf("expected unchanged, got %v", out)
	}
}

func TestLiveOffload_SdMaxVram(t *testing.T) {
	base := []string{"sd-server", "--diffusion-model", "flux.gguf", "--max-vram", "6.5", "--vae-tiling"}

	// Tight live VRAM: 5.5 free - 1.0 headroom = 4.5 < 6.5 baked -> tighten.
	out, err := LiveOffloadArgs(Settings{}, base, 5.5, true, nil)
	if err != nil {
		t.Fatalf("err = %v", err)
	}
	if v, _ := argVal(out, "--max-vram"); v != "4.5" {
		t.Errorf("--max-vram = %q, want 4.5", v)
	}

	// Ample live VRAM: 8.0 - 1.0 = 7.0 >= 6.5 baked -> leave the baked budget.
	out, err = LiveOffloadArgs(Settings{}, base, 8.0, true, nil)
	if err != nil {
		t.Fatalf("err = %v", err)
	}
	if !reflect.DeepEqual(out, base) {
		t.Errorf("ample VRAM should not loosen baked budget, got %v", out)
	}

	// Barely any free VRAM: clamps to the floor, never negative, never refused.
	out, _ = LiveOffloadArgs(Settings{}, base, 1.2, true, nil)
	if v, _ := argVal(out, "--max-vram"); v != "0.5" {
		t.Errorf("--max-vram = %q, want floor 0.5", v)
	}

	// No GPU reading -> baked plan trusted.
	out, _ = LiveOffloadArgs(Settings{}, base, 0, false, nil)
	if !reflect.DeepEqual(out, base) {
		t.Errorf("no reading should be a noop, got %v", out)
	}
}

func TestLiveOffload_BudgetCap(t *testing.T) {
	// Target below live free binds: the spawn snapshot is not a safe budget on a
	// desktop, where dwm/Discord/a VR runtime grow into VRAM after the load.
	if got := liveBudgetGB(Settings{TargetVramGB: 20.5}, 22.5); got != 20.5 {
		t.Fatalf("target should cap free: got %v want 20.5", got)
	}
	// Target above what the card actually has left must not talk us into an OOM.
	if got := liveBudgetGB(Settings{TargetVramGB: 22.8}, 18.0); got != 18.0 {
		t.Fatalf("free should win when tighter: got %v want 18.0", got)
	}
	// Unset target leaves live free as the only bound (hand-written config).
	if got := liveBudgetGB(Settings{}, 12.0); got != 12.0 {
		t.Fatalf("unset target: got %v want 12.0", got)
	}
}

// The whole point of the ctx trim: when the live budget lands a hair under what
// the baked plan assumed, the sizer's answer is to spill a layer (a dense
// model's weights AND its KV to RAM, crossed every token) to buy back a few
// tenths of a GB. Giving up a block of context buys the same VRAM and costs
// only window, so trimCtxForPlacement has to find a smaller ctx that keeps the
// baked placement rather than letting the placement rewrite happen.
func TestLiveOffload_TrimCtxKeepsBakedPlacement(t *testing.T) {
	meta := Metadata{
		Architecture: "llama", BlockCount: 32,
		HeadCountKv: 8, KeyLength: 128, ValueLength: 128,
		ContextLength: 131072, EmbeddingLength: 4096,
		FileSizeGB: 12,
	}
	s := Settings{
		TargetVramGB: 24, VramOverheadGB: 0.5, MaxRamGB: 64,
		DenseCtxLadder: []int{131072, 65536, 32768}, DenseMinCtx: 4096,
	}
	in := EstimateInput{Ctx: 32768, KvK: "q8_0", KvV: "q8_0", TargetVramGB: 24}

	baked, err := EstimatePlan(s, meta, in)
	if err != nil {
		t.Fatal(err)
	}
	if baked.Ngl < int(meta.BlockCount) {
		t.Fatalf("setup: expected a whole-model fit at 24GB, got ngl=%d", baked.Ngl)
	}

	// Walk the budget down until the sizer first gives up a layer. That is the
	// "64/65" the user reported, reproduced without guessing a magic number.
	var short EstimateResult
	tight := in
	for b := 23.9; b > 12; b -= 0.1 {
		tight.TargetVramGB = b
		short, err = EstimatePlan(s, meta, tight)
		if err != nil {
			t.Fatal(err)
		}
		if short.Ngl < baked.Ngl {
			break
		}
	}
	if short.Ngl >= baked.Ngl {
		t.Fatalf("setup: never lost a layer down to 12GB (ngl=%d)", short.Ngl)
	}

	ctx, trimmed, ok := trimCtxForPlacement(s, meta, tight, 1, baked.Ngl, baked.NCpuMoe)
	if !ok {
		t.Fatalf("no trim found at %.1fGB (ngl would drop %d->%d)", tight.TargetVramGB, baked.Ngl, short.Ngl)
	}
	if ctx >= in.Ctx || ctx%4096 != 0 {
		t.Fatalf("trimmed ctx = %d, want a smaller multiple of 4096", ctx)
	}
	if ctx < int(float64(in.Ctx)*ctxTrimFloorFrac) {
		t.Fatalf("trimmed ctx %d is below the %.0f%% floor", ctx, ctxTrimFloorFrac*100)
	}
	if trimmed.Ngl < baked.Ngl || trimmed.NCpuMoe > baked.NCpuMoe {
		t.Fatalf("trim did not restore the baked placement: ngl=%d ncpumoe=%d", trimmed.Ngl, trimmed.NCpuMoe)
	}
}

// The step is per-slot: with --parallel 4 the total -c has to move in 4x4096
// blocks, or each slot ends up holding a non-multiple of the sizer's block.
func TestLiveOffload_TrimStepsWholeSlots(t *testing.T) {
	if got := argSlots([]string{"-m", "x.gguf", "-np", "4"}); got != 4 {
		t.Fatalf("slots = %d want 4", got)
	}
	if got := argSlots([]string{"-m", "x.gguf"}); got != 1 {
		t.Fatalf("slots default = %d want 1", got)
	}

	args := []string{"-m", "x.gguf", "-c", "32768", "-ngl", "99"}
	_, idx := argVal(args, "-c", "--ctx-size")
	out := rewriteCtx(args, idx, 24576)
	if !reflect.DeepEqual(out, []string{"-m", "x.gguf", "-c", "24576", "-ngl", "99"}) {
		t.Fatalf("rewriteCtx: %v", out)
	}
	if args[3] != "32768" {
		t.Fatalf("rewriteCtx mutated its input: %v", args)
	}
}