tests/integration/config_orc_fc_deep_research.py

Rama main · réplica de lectura del repositorio original.

import dotenv

dotenv.load_dotenv()


# hyperbolic/meta-llama/Llama-3.3-70B-Instruct
# hyperbolic/deepseek-ai/DeepSeek-V3
# hyperbolic/Qwen/Qwen3-235B-A22B
# hyperbolic/Qwen/Qwen2.5-VL-7B-Instruct
# hyperbolic/openai/gpt-oss-120b
# together-ai/deepcogito/cogito-v2-1-671b
# together-ai/ServiceNow-AI/Apriel-1.6-15b-Thinker


# together-ai/deepcogito/cogito-v2-preview-llama-109B-MoE <-- Model not available
# together-ai/deepcogito/cogito-v2-preview-llama-405B <-- model not available
# together-ai/deepcogito/cogito-v2-preview-llama-70B < -- model not available

# together-ai/deepseek-ai/DeepSeek-R1-0528-tput < -- model not available
# together-ai/deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free < -- model not available

# together-ai/google/gemma-2b-it-Ishan <---Input validation error: `inputs` tokens + `max_new_tokens` must be <= 8193

# together-ai/meta-llama/Llama-3-70b-hf < -- Not entities appropved list
# together-ai/meta-llama/Llama-3.1-405B-Instruct < -- Not in approved list
# together-ai/meta-llama/Llama-3.2-1B-Instruct < -- Not in approved list

DEEP_RESEARCH_TEST_SUITE = {
    "test_1_deep_comparative": {
        "name": "Deep Comparative (Multi-Metric)",
        "query": (
            "Compare NVIDIA and AMD's 2024 performance across:\n"
            "1. Annual revenue\n"
            "2. Data center segment revenue\n"
            "3. Gaming segment revenue\n"
            "4. Year-over-year growth rates\n\n"
            "Cite official investor relations sources."
        ),
        "expected_sources": 4,  # 2 per company minimum
        "complexity": "TIER_2",
        "validates": ["multi_entity_research", "segment_breakdown", "source_citation"],
    },
    "test_2_trend_analysis": {
        "name": "Trend Analysis (Temporal Depth)",
        "query": (
            "How has NVIDIA's data center revenue grown from 2021 to 2024? "
            "Include quarterly breakdowns for 2024 and identify key inflection points."
        ),
        "expected_sources": 5,  # Multiple years + quarterly data
        "complexity": "TIER_3",
        "validates": [
            "temporal_research",
            "quarterly_granularity",
            "trend_identification",
        ],
    },
    "test_3_multi_source_verification": {
        "name": "Multi-Source Verification",
        "query": (
            "What was SpaceX's valuation in its most recent funding round? "
            "Verify the answer from at least 3 different sources and note any discrepancies."
        ),
        "expected_sources": 3,  # Explicitly required
        "complexity": "TIER_2",
        "validates": ["source_diversity", "conflict_detection", "verification_depth"],
    },
    "test_4_complex_synthesis": {
        "name": "Complex Synthesis (Multi-Vendor)",
        "query": (
            "What are the key differences between AWS, Azure, and Google Cloud's "
            "AI infrastructure offerings as of Q4 2024? Focus on:\n"
            "- GPU availability (H100, A100)\n"
            "- Pricing per compute hour\n"
            "- Integration with major ML frameworks"
        ),
        "expected_sources": 6,  # 2 per cloud provider
        "complexity": "TIER_3",
        "validates": [
            "multi_entity_research",
            "technical_comparison",
            "synthesis_quality",
        ],
    },
    "test_5_edge_case": {
        "name": "Edge Case (Requires Scrolling/Search)",
        "query": (
            "Find the exact date and location of Anthropic's Series C funding announcement. "
            "Include the lead investors and round size."
        ),
        "expected_sources": 2,
        "complexity": "TIER_1",
        "validates": [
            "specific_fact_retrieval",
            "search_web_page_usage",
            "detail_accuracy",
        ],
    },
    # BONUS: The original failing test for regression checking
    "test_0_baseline": {
        "name": "Baseline (Original Bug Test)",
        "query": (
            "Find the 2024 revenue of NVIDIA and compare it to AMD's 2024 revenue. "
            "Save your findings to the scratchpad before answering."
        ),
        "expected_sources": 2,  # 1 per company minimum
        "complexity": "TIER_1",
        "validates": [
            "both_entities_researched",
            "scratchpad_usage",
            "basic_comparison",
        ],
    },
}


PROMPTS = {
    "RESEARCH": (
        "Find the 2024 revenue of NVIDIA and compare it to AMD's 2024 revenue. "
        "Save your findings to the scratchpad before answering."
    ),
    "EXTERNAL": ("Check the flight times from LAX to JFK."),
}

config = {
    "together_api_key": "",
    "entities_api_key": "",
    "entities_user_id": "",
    "base_url": "http://localhost:9000",
    "url": "https://api.together.xyz/v1/chat/completions",
    "model": "together-ai/Qwen/Qwen3-Next-80B-A3B-Instruct",
    "provider": "together",
    "assistant_id": "asst_13HyDgBnZxVwh5XexYu74F",
    "test_prompt": DEEP_RESEARCH_TEST_SUITE["test_0_baseline"][
        "query"
    ],  # The baseline/original test
    # or
    # "test_prompt": DEEP_RESEARCH_TEST_SUITE["test_1_deep_comparative"]["query"],
    # or
    # "test_prompt": DEEP_RESEARCH_TEST_SUITE["test_3_multi_source_verification"]["query"],
}