internal/hub/hf.go
Rama main · réplica de lectura del repositorio original.
package hub
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strings"
"sync"
"time"
)
// HF is the Hugging Face adapter.
//
// Only the read-only public API is used: /api/models for search, /api/models/{id}
// for the file list, and the raw README for the model page. Downloads go through
// /resolve/, which answers with a CDN redirect — hence CheckURL allowing the
// CDN hosts as well as huggingface.co itself.
type HF struct {
// Token, when set, is sent as a bearer for gated and private repos. It
// never reaches the browser: every hub call is proxied server-side.
Token func() string
hc *http.Client
once sync.Once
mu sync.Mutex
cache map[string]hfCached
}
type hfCached struct {
at time.Time
val any
}
const (
hfAPI = "https://huggingface.co"
hfCacheTTL = 10 * time.Minute
hfTimeout = 20 * time.Second
hfUserAgent = "quartermaster"
)
func NewHF() *HF { return &HF{} }
func (h *HF) ID() string { return "hf" }
func (h *HF) Name() string { return "Hugging Face" }
func (h *HF) client() *http.Client {
h.once.Do(func() {
h.hc = &http.Client{Timeout: hfTimeout}
h.cache = map[string]hfCached{}
})
return h.hc
}
// Authorize attaches the stored token. Go's http client drops the Authorization
// header on a cross-domain redirect, so the CDN leg is unauthenticated by
// construction — which is what HF's signed URLs expect.
func (h *HF) Authorize(req *http.Request) {
if h.Token == nil {
return
}
if t := strings.TrimSpace(h.Token()); t != "" {
req.Header.Set("Authorization", "Bearer "+t)
}
}
// CheckURL pins downloads to hosts Hugging Face controls. The file path comes
// from the model list (and ultimately from the user), so the host must be ours
// no matter what the API or a redirect claims.
func (h *HF) CheckURL(raw string) error {
u, err := url.Parse(raw)
if err != nil {
return err
}
host := strings.ToLower(u.Hostname())
ok := host == "huggingface.co" ||
strings.HasSuffix(host, ".huggingface.co") ||
host == "hf.co" ||
strings.HasSuffix(host, ".hf.co")
if u.Scheme != "https" || !ok {
return fmt.Errorf("refusing non-Hugging-Face download URL: %s", raw)
}
return nil
}
func (h *HF) FileURL(repoID, path string) (string, error) {
if err := validRepoID(repoID); err != nil {
return "", err
}
if err := validRepoPath(path); err != nil {
return "", err
}
segs := strings.Split(path, "/")
for i, s := range segs {
segs[i] = url.PathEscape(s)
}
return hfAPI + "/" + repoID + "/resolve/main/" + strings.Join(segs, "/"), nil
}
// --- search ---
type hfModelJSON struct {
ID string `json:"id"`
Author string `json:"author"`
Downloads int64 `json:"downloads"`
Likes int64 `json:"likes"`
LastModified string `json:"lastModified"`
CreatedAt string `json:"createdAt"`
Pipeline string `json:"pipeline_tag"`
Tags []string `json:"tags"`
Private bool `json:"private"`
// Gated is `false` for open repos and a string ("auto"/"manual") for gated
// ones, so it cannot be decoded straight into a bool.
Gated json.RawMessage `json:"gated"`
Sibs []hfSibling `json:"siblings"`
}
type hfSibling struct {
Path string `json:"rfilename"`
Size int64 `json:"size"`
// Oid is the git blob sha of a plain file. Non-LFS in a GGUF repo means a
// README or a config, so in practice the LFS oid below is the one that
// matters — but taking both costs nothing and keeps the identity defined
// for every file we might download.
Oid string `json:"oid"`
LFS *struct {
Size int64 `json:"size"`
Oid string `json:"oid"`
} `json:"lfs"`
}
func (m hfModelJSON) toModel() Model {
author, name := m.Author, m.ID
if i := strings.Index(m.ID, "/"); i >= 0 {
if author == "" {
author = m.ID[:i]
}
name = m.ID[i+1:]
}
out := Model{
ID: m.ID,
Source: "hf",
Author: author,
Name: name,
Downloads: m.Downloads,
Likes: m.Likes,
Pipeline: m.Pipeline,
Tags: m.Tags,
Private: m.Private,
Gated: len(m.Gated) > 0 && string(m.Gated) != "false" && string(m.Gated) != "null",
ParamsB: ParamsB(name),
}
if t, err := time.Parse(time.RFC3339, m.LastModified); err == nil {
out.Updated = t
}
if t, err := time.Parse(time.RFC3339, m.CreatedAt); err == nil {
out.Created = t
}
return out
}
// searchFilters maps a Query.Kind onto the hub-side filter tags, ANDed by the
// API. Filtering at the hub beats filtering locally: an unfiltered "qwen" search
// is thousands of safetensors repos quartermaster cannot load, and a category
// tab that filtered a 30-row page client-side would mostly show nothing.
//
// The kinds are the UI's own model categories (`MODEL_CATEGORIES` in
// ui-svelte/src/lib/modelUtils.ts) so browsing and the local catalog agree on
// what "Image" or "TTS" means; each pairs `gguf` with the hub's pipeline tag,
// since a repo we cannot load is not worth listing under any category.
// searchFilterSets is what Search actually asks the hub: a UNION of filter
// sets, each set ANDed by the API as usual. It exists for `video`, the one
// category no single HF query describes.
//
// The exact pipeline tags are split across the catalog — Wan's T2V repos carry
// `text-to-video`, its I2V repos `image-to-video`, and a repo carries the one
// it is — so either alone hides half the models. The broad `video` tag is not
// the answer either: it is a capability tag that every video-UNDERSTANDING VLM
// carries (GLM Flash, MiniCPM-V) while plenty of generation repos do NOT carry
// it at all (lightx2v/Minimax-h3-Turbo is tagged image-to-video and nothing
// else). Two queries, merged and deduped, is the only shape that both finds
// everything and keeps the VLMs out.
//
// `gguf` is deliberately absent here, unlike every other category: video is
// where the useful repos are LoRAs, fp8 weights and component sets in
// safetensors, and requiring a GGUF hid all of them. Same reasoning as `3d`.
//
// Decision models (llama.cpp /v1/systemone) have no pipeline tag of their own:
// their repos say text-classification, text-generation or nothing. The tag
// they share is `decision-model` (413 gguf repos in Oct 2026); `typed-decisions`
// and `system-one` each catch a few dozen more that skip it. The plain
// `decision` and `decision-making` tags were measured and left out: they add
// LoRAs and ordinary chat models, not decision heads.
func searchFilterSets(kind string) [][]string {
switch strings.ToLower(strings.TrimSpace(kind)) {
case "video":
return [][]string{{"text-to-video"}, {"image-to-video"}}
case "music":
// Generators are text-to-audio, separators audio-to-audio.
return [][]string{{"gguf", "text-to-audio"}, {"gguf", "audio-to-audio"}}
case "decision":
return [][]string{{"gguf", "decision-model"}, {"gguf", "typed-decisions"}, {"gguf", "system-one"}}
}
return [][]string{searchFilters(kind)}
}
func searchFilters(kind string) []string {
switch strings.ToLower(strings.TrimSpace(kind)) {
case "", "llm", "text", "gguf":
return []string{"gguf"}
case "any":
return nil
case "image":
return []string{"gguf", "text-to-image"}
case "tts":
return []string{"gguf", "text-to-speech"}
case "music":
// The primary of searchFilterSets' two music queries.
return []string{"gguf", "text-to-audio"}
case "transcribe":
return []string{"gguf", "automatic-speech-recognition"}
case "embed":
return []string{"gguf", "feature-extraction"}
case "decision":
// The primary of searchFilterSets' three decision queries.
return []string{"gguf", "decision-model"}
case "segment":
// mask-generation, not image-segmentation: it is what the SAM/BiRefNet
// GGUF repos quartermaster's segment backend loads are tagged with.
return []string{"gguf", "mask-generation"}
case "3d":
// Deliberately WITHOUT gguf: the 3D models this category exists for are
// component sets (safetensors plus json manifests), and adding the gguf
// tag would hide every one of them behind repos that mostly convert
// other people's work. image-to-3d is the real pipeline tag on
// microsoft/TRELLIS.2-4B and TRELLIS-image-large.
return []string{"image-to-3d"}
default:
return []string{strings.ToLower(strings.TrimSpace(kind))}
}
}
// hfExpand is the field set every search asks for.
//
// `expand[]` is opt-in per field — asking for one drops all the others — so the
// list has to name everything Model carries. It exists for `createdAt`, which
// the plain listing does not return and which is the only honest input to the
// "trendy" filter: `lastModified` moves for a README fix.
var hfExpand = []string{
"author", "downloads", "likes", "lastModified", "createdAt",
"pipeline_tag", "tags", "gated", "private",
}
func (h *HF) Search(ctx context.Context, q Query) (Page, error) {
limit := q.Limit
if limit <= 0 || limit > 100 {
limit = 30
}
sortBy := q.Sort
switch sortBy {
case "likes", "lastModified", "downloads":
case "modified":
sortBy = "lastModified"
default:
sortBy = "downloads"
}
// Neither the size cap nor the age gate is a hub filter — HF has no
// parameter-count filter and no created-after one — so ask for more rows
// than a page needs or a filtered page comes back nearly empty.
fetch := limit
if q.MaxParamsB > 0 || q.MaxAgeDays > 0 {
fetch = min(limit*3, 100)
}
sets := searchFilterSets(q.Kind)
seen := make(map[string]bool)
merged := []Model{}
// The hub offset advances by the LONGEST sub-query, not by the merged
// length: every set is asked at the same skip, so that is where the next
// page of each begins. Advancing by the deduped total would skip rows in
// whichever set returned fewer.
longest := 0
var firstErr error
for _, filters := range sets {
got, err := h.searchOnce(ctx, q, filters, sortBy, fetch)
if err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
longest = max(longest, len(got))
for _, m := range got {
if !seen[m.ID] {
seen[m.ID] = true
merged = append(merged, m)
}
}
}
// One set failing out of several is a thinner page, not a failed search.
// Nothing at all is the error the caller has to see.
if firstErr != nil && len(merged) == 0 {
return Page{}, firstErr
}
if len(sets) > 1 {
sortModels(merged, sortBy)
}
return hfPage(merged, q, fetch, longest), nil
}
// searchOnce is one hub query: one filter set, its own cache entry.
func (h *HF) searchOnce(ctx context.Context, q Query, filters []string, sortBy string, fetch int) ([]Model, error) {
v := url.Values{}
if s := strings.TrimSpace(q.Text); s != "" {
v.Set("search", s)
}
for _, f := range filters {
v.Add("filter", f)
}
for _, f := range hfExpand {
v.Add("expand[]", f)
}
v.Set("sort", sortBy)
v.Set("direction", "-1")
v.Set("limit", fmt.Sprint(fetch))
if q.Skip > 0 {
v.Set("skip", fmt.Sprint(q.Skip))
}
key := "search:" + v.Encode()
if hit, ok := cacheGet[[]Model](h, key); ok {
return hit, nil
}
var raw []hfModelJSON
if err := h.getJSON(ctx, hfAPI+"/api/models?"+v.Encode(), &raw); err != nil {
return nil, err
}
out := make([]Model, 0, len(raw))
for _, m := range raw {
out = append(out, m.toModel())
}
// Cache what the hub said, filter after: the caps are a per-request view of
// the same page, so a capped search must not poison an uncapped one.
cachePut(h, key, out)
return out, nil
}
// sortModels re-imposes the requested order on a MERGED result. Each sub-query
// came back sorted, but concatenating two sorted lists does not.
func sortModels(in []Model, sortBy string) {
sort.SliceStable(in, func(i, j int) bool {
switch sortBy {
case "likes":
return in[i].Likes > in[j].Likes
case "lastModified":
return in[i].Updated.After(in[j].Updated)
default:
return in[i].Downloads > in[j].Downloads
}
})
}
// hfPage applies the response-side filters and reports where the NEXT page
// starts. The offset advances by what the hub returned, never by what survived
// — the filters are a view of that page, not a re-ordering of the hub's list.
//
// Nothing is trimmed to Limit: Limit sizes the fetch, and trimming the survivors
// would throw away rows the caller has already paid a round trip for and would
// then have to re-request under a different offset.
func hfPage(raw []Model, q Query, fetch, advance int) Page {
out := capParams(raw, q.MaxParamsB, 0)
out = capAge(out, q.MaxAgeDays)
if isTextKind(q.Kind) {
out = keepText(out)
}
return Page{
Models: out,
NextSkip: q.Skip + advance,
// A short page means the hub has nothing more under this query.
HasMore: advance >= fetch && fetch > 0,
}
}
// capAge drops repos created longer than maxDays ago. A repo stating no
// creation date is KEPT, the same posture as an unreadable parameter count:
// this filter narrows a listing, and it should never be the reason a repo is
// invisible for a fact about its metadata rather than about the model.
func capAge(in []Model, maxDays int) []Model {
if maxDays <= 0 {
return in
}
cutoff := time.Now().AddDate(0, 0, -maxDays)
out := make([]Model, 0, len(in))
for _, m := range in {
if m.Created.IsZero() || m.Created.After(cutoff) {
out = append(out, m)
}
}
return out
}
// isTextKind reports whether a Query.Kind is the LLM tab. "gguf" is not: it
// asks for every GGUF repo, which is exactly what this tab must no longer be.
func isTextKind(kind string) bool {
switch strings.ToLower(strings.TrimSpace(kind)) {
case "", "llm", "text":
return true
}
return false
}
// textTasks are the HF pipeline tags of a model llama-server runs as a chat
// model. VLMs are `image-text-to-text` and Gemma's omni builds `any-to-any`, so
// `text-generation` alone would lose most of today's catalog.
var textTasks = map[string]bool{
"text-generation": true,
"text2text-generation": true,
"conversational": true,
"image-text-to-text": true,
"any-to-any": true,
"audio-text-to-text": true,
"video-text-to-text": true,
"visual-question-answering": true,
"image-to-text": true,
}
// otherTasks are the pipeline tags that name a DIFFERENT job. Only consulted
// for a repo with no pipeline_tag, where a task in its plain tag list is the
// only statement it makes about itself.
var otherTasks = map[string]bool{
"text-to-speech": true,
"text-to-audio": true,
"audio-to-audio": true,
"automatic-speech-recognition": true,
"audio-classification": true,
"voice-activity-detection": true,
"text-to-image": true,
"image-to-image": true,
"text-to-video": true,
"image-to-video": true,
"image-text-to-video": true,
"video-to-video": true,
"image-to-3d": true,
"text-to-3d": true,
"feature-extraction": true,
"sentence-similarity": true,
"text-ranking": true,
"mask-generation": true,
"image-segmentation": true,
"object-detection": true,
"depth-estimation": true,
"image-classification": true,
"token-classification": true,
"text-classification": true,
}
// keepText is the LLM tab's filter. The hub query is plain `gguf`, since no
// hub-side tag describes "an LLM": half the popular quant repos (unsloth's and
// lmstudio-community's among them) carry no pipeline_tag at all, so filtering
// on one at the hub would hide them. It is therefore an EXCLUSION, applied
// here: a repo is dropped only when it says it is something else. Otherwise
// every GGUF-shipping speech, video and embedding repo (audio.cpp,
// transcribe.cpp, Wan) was listed as an LLM.
func keepText(in []Model) []Model {
out := make([]Model, 0, len(in))
for _, m := range in {
if isText(m) {
out = append(out, m)
}
}
return out
}
// decisionTags mark a llama.cpp decision model (searchFilterSets). Checked
// before the pipeline tag because many of those repos declare text-generation
// and would otherwise be offered as chat models.
var decisionTags = map[string]bool{"decision-model": true, "typed-decisions": true, "system-one": true}
func isText(m Model) bool {
for _, t := range m.Tags {
if decisionTags[strings.ToLower(t)] {
return false
}
}
if p := strings.ToLower(m.Pipeline); p != "" {
return textTasks[p]
}
other := false
for _, t := range m.Tags {
t = strings.ToLower(t)
if textTasks[t] {
return true
}
other = other || otherTasks[t]
}
return !other
}
// capParams applies the size cap and trims to the requested page size.
func capParams(in []Model, maxB float64, limit int) []Model {
out := in
if maxB > 0 {
out = make([]Model, 0, len(in))
for _, m := range in {
// Judge by the field the browser is also shown, not by a second
// parse of the id: the owner segment is part of the id and could
// carry a number of its own, and a filter that disagrees with the
// badge next to it is unexplainable.
if m.ParamsB == 0 || m.ParamsB < maxB {
out = append(out, m)
}
}
}
if limit > 0 && len(out) > limit {
out = out[:limit]
}
return out
}
// --- author avatars ---
// Avatar resolves one author's avatar image URL, or "" when they have none.
//
// HF has two account kinds behind one namespace and no endpoint that covers
// both, so an org lookup is tried first and a user lookup second — `unsloth` is
// an org, `bartowski` is a person, and a repo id doesn't say which. The "" for
// a miss is CACHED like any hit: a namespace that is neither answers 404 twice,
// and the browser asks again for every row of every search.
func (h *HF) Avatar(ctx context.Context, author string) (string, error) {
if err := validSegment(author); err != nil {
return "", err
}
key := "avatar:" + author
if hit, ok := cacheGet[string](h, key); ok {
return hit, nil
}
var body struct {
AvatarURL string `json:"avatarUrl"`
}
for _, kind := range []string{"organizations", "users"} {
var got struct {
AvatarURL string `json:"avatarUrl"`
}
if err := h.getJSON(ctx, hfAPI+"/api/"+kind+"/"+url.PathEscape(author)+"/avatar", &got); err == nil && got.AvatarURL != "" {
body = got
break
}
}
cachePut(h, key, body.AvatarURL)
return body.AvatarURL, nil
}
// --- detail ---
func (h *HF) Detail(ctx context.Context, repoID string) (ModelDetail, error) {
if err := validRepoID(repoID); err != nil {
return ModelDetail{}, err
}
key := "detail:" + repoID
if hit, ok := cacheGet[ModelDetail](h, key); ok {
return hit, nil
}
var raw hfModelJSON
// blobs=true is what makes siblings carry a size; without it the file list
// is names only and the file picker has no sizes to show.
if err := h.getJSON(ctx, hfAPI+"/api/models/"+repoID+"?blobs=true", &raw); err != nil {
return ModelDetail{}, err
}
det := ModelDetail{Model: raw.toModel()}
if det.ID == "" {
det.ID = repoID
}
all := make([]File, 0, len(raw.Sibs))
for _, s := range raw.Sibs {
f := File{Path: s.Path, SizeBytes: s.Size, OID: s.Oid}
if s.LFS != nil {
if f.SizeBytes == 0 {
f.SizeBytes = s.LFS.Size
}
// The LFS oid identifies the CONTENT; the plain oid identifies the
// pointer file that stands in for it, which changes for reasons the
// bytes don't. Prefer the content id wherever there is one.
if s.LFS.Oid != "" {
f.OID = s.LFS.Oid
}
}
classify(&f)
all = append(all, f)
}
// The listing policy needs the whole repo: a component set is decided by the
// ABSENCE of any quant file, and every file in one shares a group key.
det.Files = MarkSelection(det.ID, all)
sort.Slice(det.Files, func(i, j int) bool { return det.Files[i].Path < det.Files[j].Path })
det.Readme = h.readme(ctx, repoID)
cachePut(h, key, det)
return det, nil
}
// readme fetches the model card. A missing or oversized card is not an error —
// the file list is the useful half of the page.
func (h *HF) readme(ctx context.Context, repoID string) string {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, hfAPI+"/"+repoID+"/raw/main/README.md", nil)
if err != nil {
return ""
}
req.Header.Set("User-Agent", hfUserAgent)
h.Authorize(req)
resp, err := h.client().Do(req)
if err != nil {
return ""
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return ""
}
b, err := io.ReadAll(io.LimitReader(resp.Body, 256<<10))
if err != nil {
return ""
}
return string(b)
}
// --- plumbing ---
func (h *HF) getJSON(ctx context.Context, rawURL string, out any) error {
if err := h.CheckURL(rawURL); err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", hfUserAgent)
req.Header.Set("Accept", "application/json")
h.Authorize(req)
resp, err := h.client().Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return hubHTTPError(resp)
}
return json.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(out)
}
// hubHTTPError turns a hub response into an error the UI can act on. 401/403 on
// a gated repo means "accept the license", which is a link the user follows —
// not a retry.
func hubHTTPError(resp *http.Response) error {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
msg := strings.TrimSpace(string(body))
if len(msg) > 300 {
msg = msg[:300]
}
switch resp.StatusCode {
case http.StatusUnauthorized, http.StatusForbidden:
return &AuthError{Status: resp.StatusCode, Detail: msg}
case http.StatusNotFound:
return fmt.Errorf("not found on the hub (404)")
}
if msg == "" {
return fmt.Errorf("hub request failed: %s", resp.Status)
}
return fmt.Errorf("hub request failed: %s: %s", resp.Status, msg)
}
// AuthError marks the gated/private case so the HTTP layer can tell the user to
// accept the license or add a token, rather than showing a generic failure.
type AuthError struct {
Status int
Detail string
Repo string
}
func (e *AuthError) Error() string {
if e.Repo != "" {
return fmt.Sprintf("access denied for %s (%d): accept the license on the model page, or add a Hugging Face token", e.Repo, e.Status)
}
return fmt.Sprintf("access denied by the hub (%d): the repo is gated or private", e.Status)
}
func cacheGet[T any](h *HF, key string) (T, bool) {
var zero T
h.client()
h.mu.Lock()
defer h.mu.Unlock()
c, ok := h.cache[key]
if !ok || time.Since(c.at) > hfCacheTTL {
return zero, false
}
v, ok := c.val.(T)
return v, ok
}
func cachePut(h *HF, key string, val any) {
h.client()
h.mu.Lock()
defer h.mu.Unlock()
// Bound the map: a session that browses a hundred repos should not pin
// their file lists and READMEs for the process lifetime.
if len(h.cache) > 64 {
for k, c := range h.cache {
if time.Since(c.at) > hfCacheTTL {
delete(h.cache, k)
}
}
if len(h.cache) > 128 {
h.cache = map[string]hfCached{}
}
}
h.cache[key] = hfCached{at: time.Now(), val: val}
}
// validRepoID accepts "owner/name" with hub-legal characters only, so a repo id
// can never inject path segments or query strings into an API URL.
func validRepoID(id string) error {
owner, name, ok := strings.Cut(id, "/")
if !ok || owner == "" || name == "" || strings.Contains(name, "/") {
return fmt.Errorf("invalid repo id %q, want owner/name", id)
}
for _, s := range []string{owner, name} {
if err := validSegment(s); err != nil {
return fmt.Errorf("invalid repo id %q", id)
}
}
return nil
}
// validSegment checks one hub-legal name part — an owner, a repo name, or an
// author looked up on its own. Runs before the value becomes a URL path.
func validSegment(s string) error {
if s == "" || s == "." || s == ".." {
return fmt.Errorf("invalid name %q", s)
}
for _, r := range s {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
case r == '-', r == '_', r == '.':
default:
return fmt.Errorf("invalid name %q", s)
}
}
return nil
}
// validRepoPath rejects anything that could escape the destination folder once
// joined, before it is ever used as a URL or a filename.
func validRepoPath(p string) error {
if p == "" || strings.HasPrefix(p, "/") || strings.Contains(p, "\\") {
return fmt.Errorf("invalid file path %q", p)
}
for _, seg := range strings.Split(p, "/") {
if seg == "" || seg == "." || seg == ".." {
return fmt.Errorf("invalid file path %q", p)
}
}
return nil
}