internal/ingestion/pipeline/pipeline_params_test.go
Rama main · réplica de lectura del repositorio original.
package pipeline
import (
"encoding/json"
"reflect"
"strings"
"testing"
"ragflow/internal/entity"
)
func buildTestParserConfig(t *testing.T, dsl []byte, overrides map[string]any) entity.JSONMap {
t.Helper()
config, err := BuildParserConfig(dsl, overrides)
if err != nil {
t.Fatal(err)
}
return config
}
// generalDSL returns a minimal DSL resembling the "general" template's component
// structure, with one Parser and one Chunker component.
func generalDSL(t *testing.T) []byte {
t.Helper()
dsl := map[string]any{
"components": map[string]any{
"Parser:HipSignsRhyme": map[string]any{
"obj": map[string]any{
"component_name": "Parser",
"params": map[string]any{
"outputs": map[string]any{},
"pdf": map[string]any{"parse_method": "DeepDOC", "lang": "en"},
"docx": map[string]any{"lang": "en"},
},
},
},
"GeneralChunker:LegalReadersDecide": map[string]any{
"obj": map[string]any{
"component_name": "GeneralChunker",
"params": map[string]any{
"outputs": map[string]any{},
"chunk_size": float64(512),
"chunk_overlap": float64(128),
},
},
},
},
}
raw, err := json.Marshal(dsl)
if err != nil {
t.Fatalf("marshal dsl fixture: %v", err)
}
return raw
}
func TestBuildParserConfigNormalizesGeneralLegacyDelimiter(t *testing.T) {
dsl := map[string]any{
"components": map[string]any{
"GeneralChunker:Legacy": map[string]any{
"obj": map[string]any{
"component_name": "GeneralChunker",
"params": map[string]any{
"delimiters": []any{"\n"},
},
},
},
},
}
dslJSON, err := json.Marshal(dsl)
if err != nil {
t.Fatalf("marshal dsl: %v", err)
}
result := buildTestParserConfig(t, dslJSON, map[string]any{
"GeneralChunker:Legacy": map[string]any{
"delimiter": "\n!?;。；！？",
},
})
params, ok := result["GeneralChunker:Legacy"].(map[string]any)
if !ok {
t.Fatalf("normalized component params = %#v", result)
}
want := []string{"\n", "!", "?", ";", "。", "；", "！", "？"}
if !reflect.DeepEqual(params["delimiters"], want) {
t.Fatalf("delimiters = %#v, want %#v", params["delimiters"], want)
}
if _, ok := params["delimiter"]; ok {
t.Fatal("legacy delimiter key survived normalization")
}
}
func TestCleanComponentParamsNormalizesDelimiterWhenDSLDeclaresOnlyLegacyKey(t *testing.T) {
dsl := map[string]any{
"components": map[string]any{
"GeneralChunker:Custom": map[string]any{
"obj": map[string]any{
"component_name": "GeneralChunker",
"params": map[string]any{
"delimiter": "\n",
},
},
},
},
}
dslJSON, err := json.Marshal(dsl)
if err != nil {
t.Fatalf("marshal DSL: %v", err)
}
schemas, err := ExtractAllComponentParams(dslJSON)
if err != nil {
t.Fatal(err)
}
result := cleanComponentParams(schemas, map[string]any{
"GeneralChunker:Custom": map[string]any{
"delimiter": "\n!?",
},
})
params, ok := result["GeneralChunker:Custom"].(map[string]any)
if !ok {
t.Fatalf("normalized component params = %#v", result)
}
want := []string{"\n", "!", "?"}
if !reflect.DeepEqual(params["delimiters"], want) {
t.Fatalf("delimiters = %#v, want %#v", params["delimiters"], want)
}
if _, ok := params["delimiter"]; ok {
t.Fatal("legacy delimiter key survived normalization")
}
}
func TestBuildParserConfig_DropsLegacyFlatFields(t *testing.T) {
dslJSON := generalDSL(t)
raw := map[string]any{
"chunk_token_num": 256,
"image_context_size": 10,
}
result := buildTestParserConfig(t, dslJSON, raw)
for key := range raw {
if _, exists := result[key]; exists {
t.Errorf("flat field %q survived config building: %v", key, result)
}
}
}
func TestBuildParserConfig_DropsUnknownParamKey(t *testing.T) {
dslJSON := generalDSL(t)
raw := map[string]any{
"Parser:HipSignsRhyme": map[string]any{
"no_such_param": 1,
"pdf": map[string]any{"parse_method": "deepdoc"},
},
}
result := buildTestParserConfig(t, dslJSON, raw)
params := result["Parser:HipSignsRhyme"].(map[string]any)
if _, ok := params["no_such_param"]; ok {
t.Error("expected unknown param key to be dropped")
}
if _, ok := params["pdf"]; !ok {
t.Error("expected known param key 'pdf' to be kept")
}
}
func TestBuildParserConfig_DropsRetiredParserFamilySuffix(t *testing.T) {
dslJSON := generalDSL(t)
raw := map[string]any{
"Parser:HipSignsRhyme": map[string]any{
"pdf": map[string]any{
"parse_method": "deepdoc",
"suffix": []any{"pdf"},
},
"docx": map[string]any{"suffix": []any{"docx"}},
},
}
result := buildTestParserConfig(t, dslJSON, raw)
params := result["Parser:HipSignsRhyme"].(map[string]any)
pdf, ok := params["pdf"].(map[string]any)
if !ok {
t.Fatalf("expected pdf family, got %T", params["pdf"])
}
if _, found := pdf["suffix"]; found {
t.Errorf("pdf suffix must not survive: %#v", pdf)
}
if pdf["parse_method"] != "deepdoc" {
t.Errorf("pdf parse_method = %v, want deepdoc", pdf["parse_method"])
}
docx, ok := params["docx"].(map[string]any)
if !ok {
t.Fatalf("expected docx family, got %T", params["docx"])
}
if _, found := docx["suffix"]; found {
t.Errorf("docx suffix must not survive: %#v", docx)
}
// The cleanup must not mutate the caller's request map.
originalPDF := raw["Parser:HipSignsRhyme"].(map[string]any)["pdf"].(map[string]any)
if _, found := originalPDF["suffix"]; !found {
t.Error("BuildParserConfig mutated the caller's parser config")
}
}
// --- BuildParserConfig ---
func TestBuildParserConfig_ShallowMerge_NestedParam(t *testing.T) {
// A component with a nested-map param "chunk" that has sub-keys.
dsl := map[string]any{
"components": map[string]any{
"GeneralChunker:xyz": map[string]any{
"obj": map[string]any{
"component_name": "GeneralChunker",
"params": map[string]any{
"outputs": map[string]any{},
"chunk": map[string]any{
"size": float64(512),
"overlap": float64(128),
},
},
},
},
},
}
dslJSON, err := json.Marshal(dsl)
if err != nil {
t.Fatalf("marshal dsl: %v", err)
}
// User only overrides one sub-key of "chunk".
overrides := map[string]any{
"GeneralChunker:xyz": map[string]any{
"chunk": map[string]any{"size": float64(1024)},
},
}
result := buildTestParserConfig(t, dslJSON, overrides)
chunker, ok := result["GeneralChunker:xyz"].(map[string]any)
if !ok {
t.Fatal("expected GeneralChunker:xyz in result")
}
chunk, ok := chunker["chunk"].(map[string]any)
if !ok {
t.Fatal("expected chunk key in result")
}
// After shallow merge: size is overridden, overlap is GONE.
if chunk["size"] != float64(1024) {
t.Errorf("expected size=1024 from override, got %v", chunk["size"])
}
if _, ok := chunk["overlap"]; ok {
t.Error("shallow merge: overlap from defaults should NOT be preserved when chunk is fully replaced")
}
}
// A canvas saved by the Python frontend — and parser_config rows derived
// from it — nest the Parser's per-family setups under a "setups" key. Both
// the DSL defaults and the incoming overrides must be flattened so the
// stored parser_config carries file families at the top level.
func TestBuildParserConfig_FlattensNestedParserSetups(t *testing.T) {
dsl := map[string]any{
"components": map[string]any{
"Parser:HipSignsRhyme": map[string]any{
"obj": map[string]any{
"component_name": "Parser",
"params": map[string]any{
"outputs": map[string]any{"html": map[string]any{"type": "string"}},
"setups": map[string]any{
"pdf": map[string]any{"parse_method": "deepdoc", "lang": "Chinese"},
"spreadsheet": map[string]any{"parse_method": "deepdoc"},
},
},
},
},
},
}
dslJSON, err := json.Marshal(dsl)
if err != nil {
t.Fatalf("marshal dsl: %v", err)
}
// Legacy nested override, as stored in an upgraded document's parser_config.
overrides := map[string]any{
"Parser:HipSignsRhyme": map[string]any{
"setups": map[string]any{
"pdf": map[string]any{"parse_method": "vision"},
},
},
}
result := buildTestParserConfig(t, dslJSON, overrides)
parser, ok := result["Parser:HipSignsRhyme"].(map[string]any)
if !ok {
t.Fatal("expected Parser:HipSignsRhyme in result")
}
if _, ok := parser["setups"]; ok {
t.Error("nested \"setups\" key must be flattened away")
}
pdf, ok := parser["pdf"].(map[string]any)
if !ok {
t.Fatal("expected flattened pdf family in result")
}
if pdf["parse_method"] != "vision" {
t.Errorf("pdf parse_method = %v, want vision from override", pdf["parse_method"])
}
// BuildParserConfig shallow-merges per top-level key: the override's pdf
// family replaces the defaults' pdf family outright (the semantics locked
// by TestBuildParserConfig_ShallowMerge_NestedParam).
if _, ok := pdf["lang"]; ok {
t.Error("pdf lang should not survive: the override family replaces the default family")
}
// Families the override does not touch keep their DSL defaults.
spreadsheet, ok := parser["spreadsheet"].(map[string]any)
if !ok {
t.Fatal("spreadsheet family from DSL defaults missing")
}
if spreadsheet["parse_method"] != "deepdoc" {
t.Errorf("spreadsheet parse_method = %v, want deepdoc from DSL defaults", spreadsheet["parse_method"])
}
}
// The runtime passes Doc.ParserConfig as override_params, where top-level
// keys from the override replace the DSL-baked ones. A legacy nested Parser
// entry must be lifted BEFORE that merge — otherwise the DSL's flat families
// win over the user's override once FlattenLegacyParserSetups applies its
// top-level-wins rule inside NewParserComponent.
func TestNormalizeParserConfigSetups(t *testing.T) {
raw := map[string]any{
"Parser:HipSignsRhyme": map[string]any{
"setups": map[string]any{
"pdf": map[string]any{"parse_method": "vision"},
},
},
"Extractor:AutoExtractDefault": map[string]any{"llm_id": "llm-a"},
"metadata": map[string]any{"enabled": true},
}
out := NormalizeParserConfigSetups(raw)
parser, ok := out["Parser:HipSignsRhyme"].(map[string]any)
if !ok {
t.Fatal("expected Parser:HipSignsRhyme in result")
}
if _, ok := parser["setups"]; ok {
t.Error("nested \"setups\" key must be flattened away")
}
pdf, ok := parser["pdf"].(map[string]any)
if !ok {
t.Fatalf("expected flattened pdf family, got %T", parser["pdf"])
}
if pdf["parse_method"] != "vision" {
t.Errorf("pdf parse_method = %v, want vision", pdf["parse_method"])
}
// Non-Parser entries and dataset-level keys pass through untouched.
if got := out["Extractor:AutoExtractDefault"].(map[string]any)["llm_id"]; got != "llm-a" {
t.Errorf("extractor llm_id = %v, want llm-a (must not be touched)", got)
}
if _, ok := out["metadata"].(map[string]any)["enabled"]; !ok {
t.Error("dataset-level metadata entry lost")
}
// Flat entries and nil maps are idempotent and nil-safe.
flat := NormalizeParserConfigSetups(map[string]any{
"Parser:Flat": map[string]any{"pdf": map[string]any{"parse_method": "deepdoc"}},
})
if _, ok := flat["Parser:Flat"].(map[string]any)["pdf"]; !ok {
t.Error("flat Parser entry must pass through unchanged")
}
if got := NormalizeParserConfigSetups(nil); got != nil {
t.Errorf("nil input must return nil, got %v", got)
}
}
// An empty nested "setups" map must still be lifted away: leaving the key in
// place would create a bogus "setups" file family downstream.
func TestNormalizeParserConfigSetups_EmptyNestedMap(t *testing.T) {
out := NormalizeParserConfigSetups(map[string]any{
"Parser:HipSignsRhyme": map[string]any{
"setups": map[string]any{},
},
})
parser, ok := out["Parser:HipSignsRhyme"].(map[string]any)
if !ok {
t.Fatal("expected Parser:HipSignsRhyme in result")
}
if _, ok := parser["setups"]; ok {
t.Error("empty nested \"setups\" key must be removed")
}
}
func TestBuildParserConfig_ScalarOverridePreservesOtherDefaults(t *testing.T) {
dslJSON := generalDSL(t)
overrides := map[string]any{
"GeneralChunker:LegalReadersDecide": map[string]any{
"chunk_size": float64(1024),
},
}
result := buildTestParserConfig(t, dslJSON, overrides)
chunker, ok := result["GeneralChunker:LegalReadersDecide"].(map[string]any)
if !ok {
t.Fatal("expected GeneralChunker:LegalReadersDecide in result")
}
if chunker["chunk_size"] != float64(1024) {
t.Errorf("expected chunk_size=1024, got %v", chunker["chunk_size"])
}
// chunk_overlap should be preserved from DSL defaults since it wasn't overridden.
if chunker["chunk_overlap"] != float64(128) {
t.Errorf("expected chunk_overlap=128 preserved from defaults, got %v", chunker["chunk_overlap"])
}
}
func TestBuildParserConfig_UnknownCPNIDNotPresentInResult(t *testing.T) {
dslJSON := generalDSL(t)
overrides := map[string]any{
"Parser:Unknown": map[string]any{"chunk_size": float64(256)},
}
result := buildTestParserConfig(t, dslJSON, overrides)
// Unknown cpnID should be dropped by BuildParserConfig; the result should
// still contain the DSL-defined components with their defaults.
if _, ok := result["Parser:Unknown"]; ok {
t.Error("expected unknown cpnID to be absent from result")
}
if _, ok := result["Parser:HipSignsRhyme"]; !ok {
t.Error("expected valid component from DSL to be present")
}
}
func TestBuildParserConfigRejectsInvalidDSL(t *testing.T) {
for _, dsl := range []string{"not json", `{}`, `{"components":{"Extractor:first":{},"Extractor:second":{}}}`} {
result, err := BuildParserConfig([]byte(dsl), map[string]any{"key": "val"})
if err == nil || result != nil {
t.Fatalf("invalid DSL must return an error without config: config=%#v err=%v", result, err)
}
}
}
func TestBuildParserConfig_AllComponentsPresent(t *testing.T) {
dslJSON := generalDSL(t)
result := buildTestParserConfig(t, dslJSON, nil)
// Both components from the DSL fixture should be present.
if _, ok := result["Parser:HipSignsRhyme"]; !ok {
t.Error("expected Parser:HipSignsRhyme")
}
if _, ok := result["GeneralChunker:LegalReadersDecide"]; !ok {
t.Error("expected GeneralChunker:LegalReadersDecide")
}
}
func TestBuildParserConfig_BuiltinExtractorKeepsTagFileID(t *testing.T) {
registry, err := DefaultRegistry()
if err != nil {
t.Fatalf("DefaultRegistry: %v", err)
}
checked := 0
for _, ref := range registry.Refs() {
tpl, ok := registry.Get(ref)
if !ok {
t.Fatalf("registry.Get(%q) failed", ref)
}
dslJSON, err := json.Marshal(tpl.DSL)
if err != nil {
t.Fatalf("marshal DSL %q: %v", ref, err)
}
schemas, err := ExtractAllComponentParams(dslJSON)
if err != nil {
t.Fatalf("ExtractAllComponentParams %q: %v", ref, err)
}
for _, s := range schemas {
if s.ComponentName != "Extractor" {
continue
}
checked++
overrides := map[string]any{
s.CpnID: map[string]any{
"tags": map[string]any{"top_n": 5, "tag_file_id": "file-123"},
},
}
result := buildTestParserConfig(t, dslJSON, overrides)
params, ok := result[s.CpnID].(map[string]any)
if !ok {
t.Fatalf("template %q: expected component %q in result", ref, s.CpnID)
}
var tagFileID any
if tags, ok := params["tags"].(map[string]any); ok {
tagFileID = tags["tag_file_id"]
}
if tagFileID != "file-123" {
t.Errorf("template %q: expected tag_file_id to survive BuildParserConfig, got %v", ref, tagFileID)
}
}
}
if checked == 0 {
t.Fatal("expected at least one builtin template with an Extractor component")
}
}
// --- ComponentParamsDefaults ---
func TestComponentParamsDefaults_Basic(t *testing.T) {
dslJSON := generalDSL(t)
result, err := ComponentParamsDefaults(dslJSON)
if err != nil {
t.Fatalf("ComponentParamsDefaults: %v", err)
}
if len(result) == 0 {
t.Fatal("expected non-empty result")
}
// outputs should be stripped.
parser := result["Parser:HipSignsRhyme"]
if _, ok := parser["outputs"]; ok {
t.Error("expected outputs to be stripped")
}
if _, ok := parser["pdf"]; !ok {
t.Error("expected pdf to be present")
}
chunker := result["GeneralChunker:LegalReadersDecide"]
if chunker["chunk_size"] != float64(512) {
t.Errorf("expected chunk_size=512, got %v", chunker["chunk_size"])
}
}
func TestComponentParamsDefaults_InvalidJSON(t *testing.T) {
_, err := ComponentParamsDefaults([]byte("not json"))
if err == nil {
t.Error("expected error for invalid JSON")
}
}
func TestComponentParamsDefaults_ResultIsMutable(t *testing.T) {
// Verify the returned map is a deep copy, not a reference to shared state.
dslJSON := generalDSL(t)
result, err := ComponentParamsDefaults(dslJSON)
if err != nil {
t.Fatalf("ComponentParamsDefaults: %v", err)
}
// Mutate the result.
parser := result["Parser:HipSignsRhyme"]
delete(parser, "pdf")
// Re-read: the second call should return a fresh copy unaffected by the mutation.
result2, _ := ComponentParamsDefaults(dslJSON)
parser2 := result2["Parser:HipSignsRhyme"]
if _, ok := parser2["pdf"]; !ok {
t.Error("expected result to be independent copy (pdf preserved)")
}
}
// ensure entity.JSONMap is used (import used).
var _ entity.JSONMap
func TestBuildParserConfig_BuiltinExtractorKeepsBuiltInMetadata(t *testing.T) {
registry, err := DefaultRegistry()
if err != nil {
t.Fatalf("DefaultRegistry: %v", err)
}
checked := 0
for _, ref := range registry.Refs() {
tpl, ok := registry.Get(ref)
if !ok {
t.Fatalf("registry.Get(%q) failed", ref)
}
dslJSON, err := json.Marshal(tpl.DSL)
if err != nil {
t.Fatalf("marshal DSL %q: %v", ref, err)
}
schemas, err := ExtractAllComponentParams(dslJSON)
if err != nil {
t.Fatalf("ExtractAllComponentParams %q: %v", ref, err)
}
for _, s := range schemas {
if s.ComponentName != "Extractor" {
continue
}
checked++
overrides := map[string]any{
s.CpnID: map[string]any{
"metadata": map[string]any{
"enabled": true,
"built_in_metadata": []any{
map[string]any{"key": "update_time", "type": "time"},
},
},
},
}
result := buildTestParserConfig(t, dslJSON, overrides)
params, ok := result[s.CpnID].(map[string]any)
if !ok {
t.Fatalf("template %q: expected component %q in result", ref, s.CpnID)
}
if _, ok := params["metadata"]; !ok {
t.Errorf("template %q: metadata dropped by BuildParserConfig", ref)
}
}
}
if checked == 0 {
t.Fatal("expected at least one builtin template with an Extractor component")
}
}
func TestApplyParentChildChunkerConfig(t *testing.T) {
config := entity.JSONMap{
"GeneralChunker:one": map[string]interface{}{"chunk_token_num": 256, "children_delimiters": []string{"wrong"}, "parent_child": map[string]interface{}{"use_parent_child": true, "children_delimiter": "|"}},
"TokenChunker:two": map[string]interface{}{"chunk_token_num": 128, "children_delimiters": []string{"wrong"}, "parent_child": map[string]interface{}{"use_parent_child": true, "children_delimiter": "|"}},
"Extractor:three": map[string]interface{}{"llm_id": "llm-1"},
}
ApplyParentChildChunkerConfig(config, map[string]interface{}{})
for _, componentID := range []string{"GeneralChunker:one", "TokenChunker:two"} {
params := config[componentID].(map[string]interface{})
got, ok := params["children_delimiters"].([]string)
if !ok || len(got) != 1 || got[0] != "|" {
t.Fatalf("%s children_delimiters = %#v, want [|]", componentID, params["children_delimiters"])
}
}
if _, ok := config["Extractor:three"].(map[string]interface{})["children_delimiters"]; ok {
t.Fatal("non-chunker component was modified")
}
}
func TestBuildParserConfig_KeepsModularExtractorParams(t *testing.T) {
// Builtin template with empty Extractor params: {}
dsl := map[string]any{
"components": map[string]any{
"Extractor:AutoExtractDefault": map[string]any{
"obj": map[string]any{
"component_name": "Extractor",
"params": map[string]any{},
},
},
},
}
dslJSON, err := json.Marshal(dsl)
if err != nil {
t.Fatalf("marshal dsl: %v", err)
}
overrides := map[string]any{
"Extractor:AutoExtractDefault": map[string]any{
"summary": map[string]any{
"enabled": true,
"system_prompt": "summary prompt",
},
"metadata": map[string]any{
"enabled": true,
"metadata": []any{
map[string]any{"key": "category", "type": "string"},
},
},
"keywords": map[string]any{
"top_n": 5,
},
"llm_id": "gpt-4",
"temperature": 0.7,
"enable_summary": 1, // legacy flat param should be dropped
},
}
result := buildTestParserConfig(t, dslJSON, overrides)
ext, ok := result["Extractor:AutoExtractDefault"].(map[string]any)
if !ok {
t.Fatalf("expected Extractor:AutoExtractDefault in result, got: %v", result)
}
if sum, ok := ext["summary"].(map[string]any); !ok || sum["enabled"] != true {
t.Errorf("expected summary.enabled == true, got: %v", ext["summary"])
}
if meta, ok := ext["metadata"].(map[string]any); !ok || meta["enabled"] != true {
t.Errorf("expected metadata.enabled == true, got: %v", ext["metadata"])
}
if kw, ok := ext["keywords"].(map[string]any); !ok || kw["top_n"] != 5 {
t.Errorf("expected keywords.top_n == 5, got: %v", ext["keywords"])
}
if ext["llm_id"] != "gpt-4" {
t.Errorf("expected llm_id == gpt-4, got: %v", ext["llm_id"])
}
if _, ok := ext["enable_summary"]; ok {
t.Errorf("expected legacy flat field enable_summary to be dropped, got: %v", ext["enable_summary"])
}
}
func TestBuildParserConfig_ExtractorBuiltInMetadataRetained(t *testing.T) {
dslJSON := []byte(`{
"components": {
"Extractor:AutoExtractDefault": {
"obj": {
"component_name": "Extractor",
"params": {
"outputs": {},
"metadata": {}
}
}
}
}
}`)
rawConfig := map[string]any{
"Extractor:AutoExtractDefault": map[string]any{
"metadata": map[string]any{
"enabled": true,
"metadata": []map[string]any{
{"key": "custom_field", "type": "string"},
},
"built_in_metadata": []map[string]any{
{"key": "doc_name", "type": "string"},
},
},
},
}
result := buildTestParserConfig(t, dslJSON, rawConfig)
ext, ok := result["Extractor:AutoExtractDefault"].(map[string]any)
if !ok {
t.Fatalf("expected Extractor:AutoExtractDefault in result, got: %v", result)
}
meta, ok := ext["metadata"].(map[string]any)
if !ok {
t.Fatalf("expected metadata object, got: %#v", ext["metadata"])
}
if meta["enabled"] != true {
t.Errorf("expected enabled == true, got: %#v", meta["enabled"])
}
mFields, ok := meta["metadata"].([]map[string]any)
if !ok || len(mFields) != 1 {
t.Errorf("expected 1 metadata field, got: %#v", meta["metadata"])
}
bFields, ok := meta["built_in_metadata"].([]map[string]any)
if !ok || len(bFields) != 1 {
t.Errorf("expected 1 built_in_metadata field, got: %#v", meta["built_in_metadata"])
}
}
func TestNormalizeExtractorParams_TableDriven(t *testing.T) {
tests := []struct {
name string
input map[string]any
validate func(t *testing.T, out map[string]any)
}{
{
name: "nil input returns empty map",
input: nil,
validate: func(t *testing.T, out map[string]any) {
if len(out) != 0 {
t.Errorf("expected empty map for nil input, got %#v", out)
}
},
},
{
name: "modular keywords preserved",
input: map[string]any{
"keywords": map[string]any{
"top_n": 5,
"system_prompt": "custom kw prompt",
},
},
validate: func(t *testing.T, out map[string]any) {
kw, ok := out["keywords"].(map[string]any)
if !ok || kw["top_n"] != 5 || kw["system_prompt"] != "custom kw prompt" {
t.Errorf("keywords mismatch: %#v", out["keywords"])
}
},
},
{
name: "modular questions preserved",
input: map[string]any{
"questions": map[string]any{
"top_n": 3,
"system_prompt": "custom q prompt",
},
},
validate: func(t *testing.T, out map[string]any) {
q, ok := out["questions"].(map[string]any)
if !ok || q["top_n"] != 3 || q["system_prompt"] != "custom q prompt" {
t.Errorf("questions mismatch: %#v", out["questions"])
}
},
},
{
name: "modular tags preserved",
input: map[string]any{
"tags": map[string]any{
"top_n": 7,
"tag_file_id": "file-123",
},
},
validate: func(t *testing.T, out map[string]any) {
tag, ok := out["tags"].(map[string]any)
if !ok || tag["top_n"] != 7 || tag["tag_file_id"] != "file-123" {
t.Errorf("tags mismatch: %#v", out["tags"])
}
},
},
{
name: "modular summary preserved",
input: map[string]any{
"summary": map[string]any{
"enabled": true,
"system_prompt": "summary prompt",
},
},
validate: func(t *testing.T, out map[string]any) {
sum, ok := out["summary"].(map[string]any)
if !ok || sum["enabled"] != true || sum["system_prompt"] != "summary prompt" {
t.Errorf("summary mismatch: %#v", out["summary"])
}
},
},
{
name: "modular metadata preserved",
input: map[string]any{
"metadata": map[string]any{
"enabled": true,
"metadata": []map[string]any{
{"key": "cat", "type": "string"},
},
"built_in_metadata": []map[string]any{
{"key": "file_name", "type": "string"},
},
},
},
validate: func(t *testing.T, out map[string]any) {
meta, ok := out["metadata"].(map[string]any)
if !ok || meta["enabled"] != true {
t.Fatalf("expected enabled == true, got %#v", out["metadata"])
}
mList, ok := meta["metadata"].([]map[string]any)
if !ok || len(mList) != 1 {
t.Errorf("metadata slice mismatch: %#v", meta["metadata"])
}
bList, ok := meta["built_in_metadata"].([]map[string]any)
if !ok || len(bList) != 1 {
t.Errorf("built_in_metadata slice mismatch: %#v", meta["built_in_metadata"])
}
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
out := NormalizeExtractorParams(tc.input)
tc.validate(t, out)
})
}
}
// compilerDSL mirrors the compiler pipeline template's component structure.
func compilerDSL(t *testing.T) []byte {
t.Helper()
dsl := map[string]any{
"components": map[string]any{
"Compiler:NewBoxesLove": map[string]any{
"obj": map[string]any{
"component_name": "Compiler",
"params": map[string]any{
"outputs": map[string]any{},
"compilation_template_group_id": "",
"llm_id": "",
"plan": false,
},
},
},
},
}
raw, err := json.Marshal(dsl)
if err != nil {
t.Fatalf("marshal dsl fixture: %v", err)
}
return raw
}
func TestBuildParserConfig_CompilerKeepsLLMRuntimeParams(t *testing.T) {
dslJSON := compilerDSL(t)
raw := map[string]any{
"Compiler:NewBoxesLove": map[string]any{
"compilation_template_group_id": "group-1",
"llm_id": "model-1",
"temperature": 0.7,
"temperatureEnabled": true,
"top_p": 0.9,
"topPEnabled": true,
"presence_penalty": 0.2,
"presencePenaltyEnabled": true,
"frequency_penalty": 0.3,
"frequencyPenaltyEnabled": true,
"max_tokens": 1024.0,
"maxTokensEnabled": true,
"parameter": "Custom",
"thinking": "enabled",
"mode": "structure",
},
}
result := buildTestParserConfig(t, dslJSON, raw)
params, ok := result["Compiler:NewBoxesLove"].(map[string]any)
if !ok {
t.Fatalf("expected Compiler params to survive, got %#v", result)
}
for _, key := range []string{
"compilation_template_group_id", "llm_id",
"temperature", "temperatureEnabled",
"top_p", "topPEnabled",
"presence_penalty", "presencePenaltyEnabled",
"frequency_penalty", "frequencyPenaltyEnabled",
"max_tokens", "maxTokensEnabled",
"parameter", "thinking",
} {
if _, ok := params[key]; !ok {
t.Errorf("expected LLM runtime param %q to be kept, got %#v", key, params)
}
}
if _, ok := params["mode"]; ok {
t.Error("expected unknown param key 'mode' to be dropped")
}
for _, key := range []string{"keywords", "questions", "tags", "summary", "metadata"} {
if _, ok := params[key]; ok {
t.Errorf("expected no extractor group %q on Compiler params", key)
}
}
}
func TestBuildParserConfig_CompilerRuntimeParamsSurvive(t *testing.T) {
dslJSON := compilerDSL(t)
raw := map[string]any{
"Compiler:NewBoxesLove": map[string]any{
"compilation_template_group_id": "group-1",
"llm_id": "model-1",
"temperature": 0.7,
"temperatureEnabled": true,
"thinking": "disabled",
},
}
result := buildTestParserConfig(t, dslJSON, raw)
params, ok := result["Compiler:NewBoxesLove"].(map[string]any)
if !ok {
t.Fatalf("expected Compiler entry in built parser_config, got %#v", result)
}
if params["compilation_template_group_id"] != "group-1" || params["llm_id"] != "model-1" {
t.Errorf("compiler overrides mismatch: %#v", params)
}
if params["temperature"] != 0.7 || params["temperatureEnabled"] != true {
t.Errorf("expected LLM runtime overrides to survive, got %#v", params)
}
if params["thinking"] != "disabled" {
t.Errorf("expected thinking override to survive, got %#v", params)
}
if plan, ok := params["plan"].(bool); !ok || plan {
t.Errorf("expected DSL default plan=false to be baked in, got %#v", params["plan"])
}
}
// --- componentParamSchemaKeys (schema-derived component param whitelist) ---
// titleFamilyDSL mirrors a canvas snapshot saved before chunk_token_cap /
// root_chunk_as_heading existed: the title-chunker node bakes only the older
// params.
func titleFamilyDSL(t *testing.T) []byte {
t.Helper()
dsl := map[string]any{
"components": map[string]any{
"TitleChunker:StaleSnapshot": map[string]any{
"obj": map[string]any{
"component_name": "TitleChunker",
"params": map[string]any{
"outputs": map[string]any{},
"method": "hierarchy",
"hierarchy": float64(3),
"levels": []any{[]any{"^#[^#]"}},
},
},
},
},
}
raw, err := json.Marshal(dsl)
if err != nil {
t.Fatalf("marshal dsl fixture: %v", err)
}
return raw
}
func TestBuildParserConfig_TitleFamilyKeepsSchemaParamsWithoutBakedKeys(t *testing.T) {
dslJSON := titleFamilyDSL(t)
raw := map[string]any{
"TitleChunker:StaleSnapshot": map[string]any{
"chunk_token_cap": 256,
"root_chunk_as_heading": true,
},
}
result := buildTestParserConfig(t, dslJSON, raw)
params, ok := result["TitleChunker:StaleSnapshot"].(map[string]any)
if !ok {
t.Fatalf("expected TitleChunker params to survive, got %#v", result)
}
if _, ok := params["chunk_token_cap"]; !ok {
t.Errorf("chunk_token_cap dropped although the component schema declares it: %#v", params)
}
if _, ok := params["root_chunk_as_heading"]; !ok {
t.Errorf("root_chunk_as_heading dropped although the component schema declares it: %#v", params)
}
}
// TestBuildParserConfig_KeepsTitleChunkerPerMethodKeys locks the persistence
// contract for the title-chunker family: the frontend stores each method's
// rules, hierarchy value and token cap under its own key so the dataset
// config keeps Hierarchy and Group independent; BuildParserConfig must not
// drop them.
func TestBuildParserConfig_KeepsTitleChunkerPerMethodKeys(t *testing.T) {
dslJSON := titleFamilyDSL(t)
overrides := map[string]any{
"TitleChunker:StaleSnapshot": map[string]any{
"method": "group",
"levels": []any{[]any{"第[0-9]+章"}},
"hierarchy": 0,
"hierarchyRules": []any{map[string]any{"levels": []any{map[string]any{"expression": "^#[^#]"}}}},
"groupRules": []any{map[string]any{"levels": []any{map[string]any{"expression": "第[0-9]+章"}}}},
"hierarchyHierarchy": "3",
"hierarchyGroup": "2",
"hierarchyChunkTokenCap": 1024,
"groupChunkTokenCap": 2048,
},
}
params, ok := buildTestParserConfig(t, dslJSON, overrides)["TitleChunker:StaleSnapshot"].(map[string]any)
if !ok {
t.Fatalf("expected TitleChunker params to survive, got %#v", params)
}
for _, key := range []string{
"hierarchyRules", "groupRules", "hierarchyHierarchy", "hierarchyGroup",
"hierarchyChunkTokenCap", "groupChunkTokenCap",
} {
if _, ok := params[key]; !ok {
t.Errorf("per-method persistence key %q dropped by BuildParserConfig: %#v", key, params)
}
}
}
// TestBuildParserConfig_TitleFamilyParamsSurviveBuiltinTemplates locks the
// dataset/document save path for the title-chunker family: both params must
// survive BuildParserConfig for every builtin template that carries a
// TitleChunker or ManualChunker node, none of which bake the two keys in their
// DSL params.
func TestBuildParserConfig_TitleFamilyParamsSurviveBuiltinTemplates(t *testing.T) {
registry, err := DefaultRegistry()
if err != nil {
t.Fatalf("DefaultRegistry: %v", err)
}
tested := 0
for _, ref := range registry.Refs() {
tpl, ok := registry.Get(ref)
if !ok {
continue
}
dslJSON, err := json.Marshal(tpl.DSL)
if err != nil {
t.Fatalf("marshal DSL %q: %v", ref, err)
}
schemas, err := ExtractAllComponentParams(dslJSON)
if err != nil {
t.Fatalf("ExtractAllComponentParams %q: %v", ref, err)
}
for _, s := range schemas {
if s.ComponentName != "TitleChunker" && s.ComponentName != "ManualChunker" {
continue
}
tested++
overrides := map[string]interface{}{
s.CpnID: map[string]any{
"chunk_token_cap": 256,
"root_chunk_as_heading": true,
},
}
params, ok := buildTestParserConfig(t, dslJSON, overrides)[s.CpnID].(map[string]any)
if !ok {
t.Fatalf("template %q: component %q missing from built parser_config", ref, s.CpnID)
}
if !reflect.DeepEqual(params["chunk_token_cap"], 256) {
t.Errorf("template %q %s: chunk_token_cap = %#v, want 256", ref, s.CpnID, params["chunk_token_cap"])
}
if !reflect.DeepEqual(params["root_chunk_as_heading"], true) {
t.Errorf("template %q %s: root_chunk_as_heading = %#v, want true", ref, s.CpnID, params["root_chunk_as_heading"])
}
}
}
if tested == 0 {
t.Fatal("expected at least one builtin template with a TitleChunker/ManualChunker component")
}
}
// operatorFormParamKeys mirrors the params the dataset/document pipeline
// operator forms submit (transformFormConfigToApi in
// web/src/utils/pipeline-operator.ts). Every key listed for a component must be
// accepted by BuildParserConfig for that component in every builtin
// template; otherwise a user edit is silently dropped.
var operatorFormParamKeys = map[string][]string{
"TitleChunker": {
"method", "levels", "hierarchy", "include_heading_content",
"root_chunk_as_heading", "chunk_token_cap", "table_context_size", "image_context_size",
// Per-method persistence keys: each method's rules, hierarchy value
// and token cap are stored under their own key so the dataset config
// keeps them independent of the active method (the runtime reads only
// levels/hierarchy/chunk_token_cap).
"hierarchyRules", "groupRules", "hierarchyHierarchy", "hierarchyGroup",
"hierarchyChunkTokenCap", "groupChunkTokenCap",
},
"ManualChunker": {
"method", "levels", "hierarchy", "root_chunk_as_heading",
"table_context_size", "image_context_size",
"hierarchyRules", "groupRules", "hierarchyHierarchy", "hierarchyGroup",
},
"TokenChunker": {
"chunk_token_size", "overlapped_percent", "delimiters", "delimiter_mode",
"children_delimiters", "enable_children", "table_context_size", "image_context_size",
},
"GeneralChunker": {
"chunk_token_size", "overlapped_percent", "delimiters", "children_delimiters",
"enable_children", "table_context_size", "image_context_size",
},
"Tokenizer": {"search_method", "filename_embd_weight", "fields"},
"Extractor": {"llm_id", "keywords", "questions", "tags", "summary", "metadata"},
}
func TestBuildParserConfig_AcceptsOperatorFormParams(t *testing.T) {
registry, err := DefaultRegistry()
if err != nil {
t.Fatalf("DefaultRegistry: %v", err)
}
checked := 0
for _, ref := range registry.Refs() {
tpl, ok := registry.Get(ref)
if !ok {
continue
}
dslJSON, err := json.Marshal(tpl.DSL)
if err != nil {
t.Fatalf("marshal DSL %q: %v", ref, err)
}
schemas, err := ExtractAllComponentParams(dslJSON)
if err != nil {
t.Fatalf("ExtractAllComponentParams %q: %v", ref, err)
}
for _, s := range schemas {
expected, ok := operatorFormParamKeys[s.ComponentName]
if !ok {
continue
}
checked++
params := make(map[string]any, len(expected))
for _, key := range expected {
params[key] = "probe"
}
cleaned := buildTestParserConfig(t, dslJSON, map[string]interface{}{s.CpnID: params})
got, ok := cleaned[s.CpnID].(map[string]any)
if !ok {
t.Fatalf("template %q: component %q missing from cleaned result", ref, s.CpnID)
}
for _, key := range expected {
if _, ok := got[key]; !ok {
t.Errorf("template %q component %s: operator-form param %q is silently dropped", ref, s.CpnID, key)
}
}
}
}
if checked == 0 {
t.Fatal("expected at least one builtin template component matched by operatorFormParamKeys")
}
}
// componentParamWhitelistExemptions lists the builtin template component names
// that intentionally have no entry in componentParamSchemaKeys, with the reason.
// The inventory test below fails on any component that is neither derived nor
// exempt, so a renamed or newly added component cannot silently lose its
// whitelist coverage.
var componentParamWhitelistExemptions = map[string]string{
"File": "no user-editable params",
"Parser": "DSL file families plus the global vision parameter whitelist",
"GeneralChunker": "param struct lives in the chunker package without json tags; template keys + the delimiters exception cover it",
"QAChunker": "no operator form; param struct is not in the schema package",
"OneChunker": "no user-editable params",
"PageChunker": "no user-editable params",
"TableChunker": "no user-editable params",
"Extractor": "covered by the cpnID-prefixed dynamic whitelist",
"Compiler": "covered by the cpnID-prefixed dynamic whitelist",
}
// TestBuiltinTemplateComponentsHaveWhitelistCoverage pins the component-name
// inventory of the builtin templates: every name must either derive its param
// whitelist from a schema (componentParamSchemaKeys) or be exempted above. It
// guards the operator-form test against becoming a no-op: that test matches by
// component name, so a rename would otherwise silently skip it.
func TestBuiltinTemplateComponentsHaveWhitelistCoverage(t *testing.T) {
registry, err := DefaultRegistry()
if err != nil {
t.Fatalf("DefaultRegistry: %v", err)
}
seen := make(map[string]string)
for _, ref := range registry.Refs() {
tpl, ok := registry.Get(ref)
if !ok {
continue
}
dslJSON, err := json.Marshal(tpl.DSL)
if err != nil {
t.Fatalf("marshal DSL %q: %v", ref, err)
}
schemas, err := ExtractAllComponentParams(dslJSON)
if err != nil {
t.Fatalf("ExtractAllComponentParams %q: %v", ref, err)
}
for _, s := range schemas {
seen[s.ComponentName] = ref
}
}
if len(seen) == 0 {
t.Fatal("expected builtin templates to contain components")
}
for name, ref := range seen {
if _, ok := componentParamSchemaKeys[strings.ToLower(name)]; ok {
continue
}
if _, ok := componentParamWhitelistExemptions[name]; ok {
continue
}
t.Errorf("component %q (template %q) is neither schema-derived nor exempt: add it to componentParamSchemaKeys or exempt it with a reason", name, ref)
}
}
// TestBuildParserConfig_TitleFamilyStillDropsUnknownKeys keeps the union
// honest: a key the component schema does not declare is still dropped.
func TestBuildParserConfig_TitleFamilyStillDropsUnknownKeys(t *testing.T) {
dslJSON := titleFamilyDSL(t)
raw := map[string]any{
"TitleChunker:StaleSnapshot": map[string]any{
"chunk_token_cap_typo": 256,
"method": "hierarchy",
},
}
result := buildTestParserConfig(t, dslJSON, raw)
params, ok := result["TitleChunker:StaleSnapshot"].(map[string]any)
if !ok {
t.Fatalf("expected TitleChunker params to survive, got %#v", result)
}
if _, ok := params["chunk_token_cap_typo"]; ok {
t.Errorf("unknown key must still be dropped, got %#v", params)
}
if _, ok := params["method"]; !ok {
t.Errorf("declared key must survive, got %#v", params)
}
}