internal/autogen/hf_test.go
Rama main · réplica de lectura del repositorio original.
package autogen
import (
"os"
"path/filepath"
"strings"
"testing"
)
// writeHFDir lays down an HF model folder: config.json with the given body and
// one stub file per weight name. Discovery reads config.json and stats the
// weights, and never opens a safetensors, so the stubs need no real header.
func writeHFDir(t *testing.T, dir, config string, files ...string) string {
t.Helper()
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
if config != "" {
if err := os.WriteFile(filepath.Join(dir, hfConfigFile), []byte(config), 0o644); err != nil {
t.Fatal(err)
}
}
for _, f := range files {
if err := os.WriteFile(filepath.Join(dir, f), make([]byte, 2048), 0o644); err != nil {
t.Fatal(err)
}
}
return dir
}
const (
hfLlamaConfig = `{"architectures":["LlamaForCausalLM"],"model_type":"llama",
"num_hidden_layers":32,"num_attention_heads":32,"num_key_value_heads":8,
"hidden_size":4096,"max_position_embeddings":131072,"torch_dtype":"bfloat16"}`
hfGemmaConfig = `{"architectures":["Gemma3ForConditionalGeneration"],"model_type":"gemma3",
"text_config":{"model_type":"gemma3_text","num_hidden_layers":34,"num_attention_heads":8,
"num_key_value_heads":4,"head_dim":256,"hidden_size":2560,"max_position_embeddings":131072},
"vision_config":{"model_type":"siglip_vision_model"},"torch_dtype":"bfloat16"}`
)
func hfRowsByPath(rows []GgufRow) map[string]GgufRow {
m := map[string]GgufRow{}
for _, r := range rows {
m[filepath.Clean(r.FullPath)] = r
}
return m
}
// The folders a real models tree holds that LOOK like an HF model: only the
// generative ones with their own weights, and no conversion beside them, may
// become rows.
func TestDiscover_HFFolders(t *testing.T) {
root := t.TempDir()
at := func(rel string) string { return filepath.Join(root, filepath.FromSlash(rel)) }
sharded := writeHFDir(t, at("meta/Llama-3-8B-Instruct"), hfLlamaConfig,
"model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors", "model.safetensors.index.json")
multimodal := writeHFDir(t, at("google/gemma-3-4b-it"), hfGemmaConfig, "model.safetensors")
awq := writeHFDir(t, at("q/Qwen2.5-7B-Instruct"),
`{"architectures":["Qwen2ForCausalLM"],"num_hidden_layers":28,"quantization_config":{"quant_method":"awq"}}`,
"model.safetensors")
negatives := map[string]string{
// A classifier head and a vision backbone: safetensors + config.json, not chat.
"answerdotai/ModernBERT": `{"architectures":["ModernBertForMaskedLM"]}`,
"facebook/dinov3": `{"architectures":["DINOv3ViTModel"]}`,
// Seq2seq models call themselves *ForConditionalGeneration too.
"google/t5-base": `{"architectures":["T5ForConditionalGeneration"]}`,
"openai/whisper-small": `{"architectures":["WhisperForConditionalGeneration"]}`,
"broken/json": `{"architectures":[`,
"x/no-config": ``,
}
for rel, cfg := range negatives {
writeHFDir(t, at(rel), cfg, "model.safetensors")
}
noWeights := writeHFDir(t, at("x/config-only"), hfLlamaConfig)
// The source folder a gguf was converted from: the gguf wins, the folder is
// not a second copy of the model.
converted := writeHFDir(t, at("conv/Llama-3-8B"), hfLlamaConfig, "model.safetensors")
convGguf := seededGguf(t, root, "conv/Llama-3-8B/Llama-3-8B-Q4_K_M.gguf")
// A projector in the PARENT folder of an HF model belongs to whatever gguf
// sits there; the dir-local pairing must not hand it to the folder.
// Pairing reads the header arch, so the stub is seeded as a clip projector.
proj := at("meta/mmproj-F16.gguf")
if err := os.WriteFile(proj, make([]byte, 1024), 0o644); err != nil {
t.Fatal(err)
}
seedMetaCache(t, proj, Metadata{Architecture: "clip"})
rows, err := DiscoverGgufModels(root)
if err != nil {
t.Fatal(err)
}
byPath := hfRowsByPath(rows)
for _, want := range []struct {
dir, id, quant, publisher string
}{
{sharded, "llama-3-8b-instruct-bf16", "BF16", "meta"},
{multimodal, "gemma-3-4b-it-bf16", "BF16", "google"},
{awq, "qwen2.5-7b-instruct-awq", "AWQ", "q"},
} {
r, ok := byPath[filepath.Clean(want.dir)]
if !ok {
t.Errorf("%s: no row", want.dir)
continue
}
if !r.IsHF || r.ID != want.id || r.Quant != want.quant || r.Publisher != want.publisher {
t.Errorf("%s: row = {IsHF:%v ID:%q Quant:%q Publisher:%q}, want {true %q %q %q}",
want.dir, r.IsHF, r.ID, r.Quant, r.Publisher, want.id, want.quant, want.publisher)
}
if r.MmprojPath != "" || r.DraftPath != "" {
t.Errorf("%s: HF row picked up a sidecar (mmproj %q, draft %q)", want.dir, r.MmprojPath, r.DraftPath)
}
}
if r := byPath[filepath.Clean(sharded)]; r.SizeGB != round(float64(2*2048)/gib, 2) {
t.Errorf("sharded size = %g, want both shards summed", r.SizeGB)
}
for rel := range negatives {
if _, ok := byPath[filepath.Clean(at(rel))]; ok {
t.Errorf("%s: became a row", rel)
}
}
for _, dir := range []string{noWeights, converted} {
if _, ok := byPath[filepath.Clean(dir)]; ok {
t.Errorf("%s: became a row", dir)
}
}
if r, ok := byPath[filepath.Clean(convGguf)]; !ok || r.IsHF {
t.Errorf("the conversion's gguf should still be an ordinary row, got %+v (found %v)", r, ok)
}
}
// The HF cache layout names the model only in a grandparent folder; the folder
// holding the weights is a commit hash, which would make a meaningless id.
func TestHFRow_CacheLayout(t *testing.T) {
root := t.TempDir()
dir := writeHFDir(t, filepath.Join(root, "models--Qwen--Qwen3-4B--Instruct", "snapshots", "0123abcd"),
hfLlamaConfig, "model.safetensors")
r, ok := HFRowFor(dir)
if !ok {
t.Fatal("snapshot folder not recognized")
}
if r.Publisher != "Qwen" || r.Repo != "Qwen3-4B--Instruct" || r.ID != "qwen3-4b--instruct-bf16" {
t.Errorf("cache row = {Publisher:%q Repo:%q ID:%q}", r.Publisher, r.Repo, r.ID)
}
}
func TestReadHFMetadata(t *testing.T) {
root := t.TempDir()
t.Run("dense", func(t *testing.T) {
m, err := ReadHFMetadata(writeHFDir(t, filepath.Join(root, "llama"), hfLlamaConfig, "model.safetensors"))
if err != nil {
t.Fatal(err)
}
// head_dim absent: hidden/heads, the transformers default.
if m.Architecture != "llama" || m.BlockCount != 32 || m.HeadCount != 32 || m.HeadCountKv != 8 ||
m.KeyLength != 128 || m.ValueLength != 128 || m.ContextLength != 131072 || m.QuantLabel != "BF16" {
t.Errorf("dense meta = %+v", m)
}
})
t.Run("multimodal text_config", func(t *testing.T) {
m, err := ReadHFMetadata(writeHFDir(t, filepath.Join(root, "gemma"), hfGemmaConfig, "model.safetensors"))
if err != nil {
t.Fatal(err)
}
if m.BlockCount != 34 || m.HeadCountKv != 4 || m.KeyLength != 256 || m.ContextLength != 131072 {
t.Errorf("text_config not read: %+v", m)
}
})
t.Run("MHA omits kv heads", func(t *testing.T) {
m, err := ReadHFMetadata(writeHFDir(t, filepath.Join(root, "mha"),
`{"architectures":["GPT2LMHeadModel","XForCausalLM"],"num_hidden_layers":4,"num_attention_heads":16,"hidden_size":1024}`,
"model.safetensors"))
if err != nil {
t.Fatal(err)
}
if m.HeadCountKv != 16 || m.KeyLength != 64 {
t.Errorf("MHA meta = %+v, want kv heads = heads and hidden/heads head dim", m)
}
})
t.Run("hybrid layer_types", func(t *testing.T) {
m, err := ReadHFMetadata(writeHFDir(t, filepath.Join(root, "hybrid"),
`{"architectures":["Qwen3NextForCausalLM"],"num_hidden_layers":4,"num_attention_heads":16,
"num_key_value_heads":2,"head_dim":256,"hidden_size":2048,
"layer_types":["linear_attention","linear_attention","linear_attention","full_attention"]}`,
"model.safetensors"))
if err != nil {
t.Fatal(err)
}
if m.AttnLayerCount != 1 || len(m.HeadCountKvArr) != 4 || m.HeadCountKvArr[3] != 2 || m.HeadCountKvArr[0] != 0 {
t.Errorf("hybrid meta = AttnLayerCount %d, HeadCountKvArr %v", m.AttnLayerCount, m.HeadCountKvArr)
}
})
t.Run("layer_types length mismatch is not trusted", func(t *testing.T) {
m, err := ReadHFMetadata(writeHFDir(t, filepath.Join(root, "mismatch"),
`{"architectures":["XForCausalLM"],"num_hidden_layers":4,"num_attention_heads":8,"hidden_size":512,
"layer_types":["linear_attention","full_attention"]}`,
"model.safetensors"))
if err != nil {
t.Fatal(err)
}
if m.HeadCountKvArr != nil || m.AttnLayerCount != 0 {
t.Errorf("mismatched layer_types was used: %+v", m)
}
})
}
// The same model described by config.json and by its gguf header must size to
// the same vllm window: the HF path reuses the gguf KV model, so any gap here is
// a mapping bug in ReadHFMetadata.
func TestVllmMaxModelLen_HFParityWithGguf(t *testing.T) {
dir := writeHFDir(t, filepath.Join(t.TempDir(), "llama"), hfLlamaConfig, "model.safetensors")
hfMeta, err := ReadHFMetadata(dir)
if err != nil {
t.Fatal(err)
}
ggufMeta := Metadata{
Architecture: "llama", BlockCount: 32, ContextLength: 131072, EmbeddingLength: 4096,
HeadCount: 32, HeadCountKv: 8, KeyLength: 128, ValueLength: 128,
}
s := Settings{TargetVramGB: 24}
row := GgufRow{SizeGB: 15}
hfCtx, _ := vllmMaxModelLen(s, nil, row, hfMeta)
ggCtx, _ := vllmMaxModelLen(s, nil, row, ggufMeta)
if hfCtx != ggCtx || hfCtx <= 0 {
t.Errorf("config.json window %d != gguf window %d", hfCtx, ggCtx)
}
}
// servesBlocks is blocksFor for vllm, which names its model positionally.
func servesBlocks(out, dir string) []string {
var got []string
for _, cmd := range modelBlocks(out) {
if strings.Contains(cmd, "serve "+dir+"\n") {
got = append(got, cmd)
}
}
return got
}
func TestGenerate_HFFolderServedByVllm(t *testing.T) {
withCard(t, 24, true)
root := t.TempDir()
dir := writeHFDir(t, filepath.Join(root, "meta", "Llama-3-8B-Instruct"), hfLlamaConfig, "model.safetensors")
slashed := filepath.ToSlash(dir)
t.Run("vllm registered", func(t *testing.T) {
// llama is the class default; the HF folder must still go to vllm,
// because llama.cpp cannot read it at all.
s := Settings{ModelsRoot: root, TargetVramGB: 16, Backends: []BackendEntry{
{ID: "llama", Kind: "llama", Path: "llama-server", Default: true},
{ID: "v", Kind: "vllm", Path: "vllm"},
}}
out, err := Generate(GenerateFile{Settings: s}, "test")
if err != nil {
t.Fatal(err)
}
blocks := servesBlocks(out, slashed)
if len(blocks) != 1 {
t.Fatalf("want one block serving %s, got %d in:\n%s", slashed, len(blocks), out)
}
for _, want := range []string{"serve " + slashed, "--served-model-name llama-3-8b-instruct-bf16", "--max-model-len"} {
if !strings.Contains(blocks[0], want) {
t.Errorf("block missing %q:\n%s", want, blocks[0])
}
}
if strings.Contains(blocks[0], "--quantization gguf") {
t.Errorf("HF folder emitted the gguf quantization flag:\n%s", blocks[0])
}
if !strings.Contains(out, "(vllm, safetensors,") {
t.Errorf("comment should name the safetensors format:\n%s", out)
}
})
t.Run("no vllm backend", func(t *testing.T) {
s := Settings{ModelsRoot: root, TargetVramGB: 16}
out, err := Generate(GenerateFile{Settings: s}, "test")
if err != nil {
t.Fatal(err)
}
if len(servesBlocks(out, slashed)) != 0 {
t.Errorf("HF folder emitted without vllm:\n%s", out)
}
if !strings.Contains(out, `# SKIPPED "llama-3-8b-instruct-bf16": safetensors model folder needs a vllm backend`) {
t.Errorf("want an in-band skip reason:\n%s", out)
}
})
}
func TestRenderSoloCmdLayers_HFFolder(t *testing.T) {
withCard(t, 24, true)
dir := writeHFDir(t, filepath.Join(t.TempDir(), "meta", "Llama-3-8B"), hfLlamaConfig, "model.safetensors")
row, ok := HFRowFor(dir)
if !ok {
t.Fatal("not an HF folder")
}
meta, err := ReadHFMetadata(dir)
if err != nil {
t.Fatal(err)
}
s := Settings{TargetVramGB: 16, Backends: []BackendEntry{{ID: "v", Kind: "vllm", Path: "vllm"}}}
cmd, err := RenderSoloCmdLayers(s, meta, row, Override{})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(cmd.Effective, "serve "+filepath.ToSlash(dir)) || strings.Contains(cmd.Effective, "--quantization gguf") {
t.Errorf("preview = %s", cmd.Effective)
}
if _, err := RenderSoloCmdLayers(Settings{TargetVramGB: 16}, meta, row, Override{}); err == nil {
t.Error("preview with no vllm backend should refuse, not render a llama command")
}
}
// A folder dropped into the models tree must change the inputs hash, or the
// config is never regenerated to serve it.
func TestInputsHash_SeesHFFolders(t *testing.T) {
root := t.TempDir()
before, err := InputsHash(root, nil)
if err != nil {
t.Fatal(err)
}
writeHFDir(t, filepath.Join(root, "meta", "Llama"), hfLlamaConfig, "model.safetensors")
after, err := InputsHash(root, nil)
if err != nil {
t.Fatal(err)
}
if before == after {
t.Error("adding an HF folder left the inputs hash unchanged")
}
}
func TestHFSkipHint_WindowsPointsAtGguf(t *testing.T) {
if h := hfSkipHint("windows"); !strings.Contains(h, "GGUF") || strings.Contains(h, "Settings > Backends") {
t.Errorf("windows hint must point at a gguf, not an uninstallable backend: %q", h)
}
if h := hfSkipHint("linux"); !strings.Contains(h, "Settings > Backends") {
t.Errorf("linux hint must point at the backends page: %q", h)
}
}