internal/autogen/kvdefault_test.go

Rama main · réplica de lectura del repositorio original.

package autogen

import "testing"

// A plain-attention model with room to spare keeps f16; the same model on a card
// that can't buy denseMinCtx at f16 steps down to q8_0.
func TestDefaultKvQuant_StepsDownOnlyUnderPressure(t *testing.T) {
	// 32 layers x 8 kv heads x 128 k/v dims => ~0.5 MB/token at f16.
	meta := Metadata{
		BlockCount: 32, HeadCountKv: 8, KeyLength: 128, ValueLength: 128,
		ContextLength: 131072, FileSizeGB: 8,
	}
	s := Settings{DenseMinCtx: 32768}

	f16 := GetKvCostModel(meta, "f16", "f16")
	roomy := 8 + 1 + f16.SlopeGB*32768 + 0.5 // weights + overhead + 32k of f16 KV
	if got := defaultKvQuant(s, meta, roomy, 1.0); got != "f16" {
		t.Errorf("roomy budget: got %q, want f16", got)
	}
	// Half the KV budget: f16 can't reach 32k, q8_0 (~0.53x) can.
	tight := 8 + 1 + f16.SlopeGB*16384
	if got := defaultKvQuant(s, meta, tight, 1.0); got != "q8_0" {
		t.Errorf("tight budget: got %q, want q8_0", got)
	}
}

// settings.kvQuant pins the type outright; an unknown value falls back to auto.
func TestDefaultKvQuant_SettingPin(t *testing.T) {
	meta := Metadata{BlockCount: 32, HeadCountKv: 8, KeyLength: 128, ValueLength: 128, FileSizeGB: 8}
	if got := defaultKvQuant(Settings{KvQuant: "q5_1", DenseMinCtx: 32768}, meta, 100, 1.0); got != "q5_1" {
		t.Errorf("pinned: got %q, want q5_1", got)
	}
	// Never emit a type llama would reject, even if a config asks for it.
	if got := defaultKvQuant(Settings{KvQuant: "iq4_nl", DenseMinCtx: 32768}, meta, 100, 1.0); got != "f16" {
		t.Errorf("iq4_nl pin: got %q, want the auto pick f16", got)
	}
}

// A one-sided pin mirrors rather than collapsing to the default. Dropping it
// used to size the fleet default f16 for a model pinned to q8_0, which on a
// tight budget costs a whole layer of offload in the editor's estimate.
func TestResolveKvPair(t *testing.T) {
	cases := []struct {
		name         string
		k, v         string
		wantK, wantV string
	}{
		{"both blank => default", "", "", "f16", "f16"},
		{"both set", "q8_0", "q8_0", "q8_0", "q8_0"},
		{"K only mirrors to V", "q8_0", "", "q8_0", "q8_0"},
		{"V only mirrors to K", "", "q8_0", "q8_0", "q8_0"},
		{"genuine mismatch => default", "q8_0", "q5_0", "f16", "f16"},
		{"unsupported type => default", "iq4_nl", "", "f16", "f16"},
	}
	for _, c := range cases {
		gotK, gotV := resolveKvPair(c.k, c.v, "f16", "f16")
		if gotK != c.wantK || gotV != c.wantV {
			t.Errorf("%s: resolveKvPair(%q,%q) = %q,%q; want %q,%q",
				c.name, c.k, c.v, gotK, gotV, c.wantK, c.wantV)
		}
	}
}

func TestValidKvPair(t *testing.T) {
	for _, q := range []string{"f32", "f16", "bf16", "q8_0", "q5_1", "q5_0", "q4_1", "q4_0"} {
		if !ValidKvPair(q, q) {
			t.Errorf("%s/%s should be valid", q, q)
		}
	}
	// Mismatched K/V breaks flash-attention; iq4_nl has no FA kernel at all.
	if ValidKvPair("q8_0", "f16") {
		t.Error("mismatched K/V should be rejected")
	}
	if ValidKvPair("iq4_nl", "iq4_nl") {
		t.Error("iq4_nl should be rejected")
	}
	if ValidKvPair("q3_k", "q3_k") {
		t.Error("unknown quant should be rejected")
	}
}