tests/test_config_defaults.py
Rama main · réplica de lectura del repositorio original.
"""Tests for BrowserConfig.set_defaults / CrawlerRunConfig.set_defaults."""
import warnings
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig, Provenance
from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
@pytest.fixture(autouse=True)
def _reset_defaults():
"""Ensure every test starts and ends with a clean slate."""
BrowserConfig.reset_defaults()
CrawlerRunConfig.reset_defaults()
yield
BrowserConfig.reset_defaults()
CrawlerRunConfig.reset_defaults()
# ── Basic API ──────────────────────────────────────────────────────────
class TestBasicAPI:
def test_set_and_get_defaults(self):
BrowserConfig.set_defaults(headless=False, viewport_width=1920)
d = BrowserConfig.get_defaults()
assert d == {"headless": False, "viewport_width": 1920}
def test_get_defaults_returns_copy(self):
BrowserConfig.set_defaults(headers={"X-Foo": "bar"})
d = BrowserConfig.get_defaults()
d["headers"]["X-Foo"] = "changed"
assert BrowserConfig.get_defaults()["headers"]["X-Foo"] == "bar"
def test_reset_all(self):
BrowserConfig.set_defaults(headless=False)
BrowserConfig.reset_defaults()
assert BrowserConfig.get_defaults() == {}
def test_reset_selective(self):
BrowserConfig.set_defaults(headless=False, viewport_width=1920)
BrowserConfig.reset_defaults("headless")
assert BrowserConfig.get_defaults() == {"viewport_width": 1920}
def test_invalid_param_raises(self):
with pytest.raises(ValueError, match="Invalid parameter"):
BrowserConfig.set_defaults(not_a_real_param=42)
def test_invalid_param_among_valid(self):
with pytest.raises(ValueError):
BrowserConfig.set_defaults(headless=False, bogus=True)
# Nothing should have been stored
assert BrowserConfig.get_defaults() == {}
def test_set_defaults_overwrites(self):
BrowserConfig.set_defaults(headless=False)
BrowserConfig.set_defaults(headless=True)
assert BrowserConfig.get_defaults()["headless"] is True
def test_crawler_run_config_basic(self):
CrawlerRunConfig.set_defaults(verbose=False, scan_full_page=True)
d = CrawlerRunConfig.get_defaults()
assert d == {"verbose": False, "scan_full_page": True}
# ── Default injection ──────────────────────────────────────────────────
class TestDefaultInjection:
def test_browser_config_defaults_applied(self):
BrowserConfig.set_defaults(
headless=False,
cache_cdp_connection=True,
cdp_close_delay=0,
)
cfg = BrowserConfig()
assert cfg.headless is False
assert cfg.cache_cdp_connection is True
assert cfg.cdp_close_delay == 0
def test_crawler_run_config_defaults_applied(self):
CrawlerRunConfig.set_defaults(verbose=False, scan_full_page=True)
cfg = CrawlerRunConfig()
assert cfg.verbose is False
assert cfg.scan_full_page is True
def test_partial_defaults(self):
BrowserConfig.set_defaults(headless=False)
cfg = BrowserConfig()
assert cfg.headless is False
# Other params keep their hardcoded defaults
assert cfg.browser_type == "chromium"
assert cfg.viewport_width == 1080
def test_multiple_instances_get_defaults(self):
BrowserConfig.set_defaults(headless=False)
c1 = BrowserConfig()
c2 = BrowserConfig()
assert c1.headless is False
assert c2.headless is False
# ── Explicit override ──────────────────────────────────────────────────
class TestExplicitOverride:
def test_explicit_kwarg_wins(self):
BrowserConfig.set_defaults(headless=False)
cfg = BrowserConfig(headless=True)
assert cfg.headless is True
def test_explicit_same_as_default_still_wins(self):
"""Even if user passes the same value as user-default, it should be treated as explicit."""
BrowserConfig.set_defaults(headless=False)
cfg = BrowserConfig(headless=False)
assert cfg.headless is False
def test_explicit_none_wins(self):
BrowserConfig.set_defaults(cdp_url="ws://localhost:9222")
cfg = BrowserConfig(cdp_url=None)
assert cfg.cdp_url is None
def test_mixed_explicit_and_default(self):
BrowserConfig.set_defaults(headless=False, viewport_width=1920)
cfg = BrowserConfig(headless=True)
assert cfg.headless is True # explicit
assert cfg.viewport_width == 1920 # from user default
# ── Mutable isolation ──────────────────────────────────────────────────
class TestMutableIsolation:
def test_list_default_not_shared(self):
BrowserConfig.set_defaults(cookies=[{"name": "a", "value": "1"}])
c1 = BrowserConfig()
c2 = BrowserConfig()
c1.cookies.append({"name": "b", "value": "2"})
assert len(c2.cookies) == 1 # c2 should be unaffected
def test_dict_default_not_shared(self):
BrowserConfig.set_defaults(headers={"X-Foo": "bar"})
c1 = BrowserConfig()
c2 = BrowserConfig()
c1.headers["X-New"] = "val"
assert "X-New" not in c2.headers
def test_set_defaults_input_not_mutated(self):
original = {"X-Foo": "bar"}
BrowserConfig.set_defaults(headers=original)
cfg = BrowserConfig()
cfg.headers["X-Added"] = "val"
assert "X-Added" not in original
assert "X-Added" not in BrowserConfig.get_defaults()["headers"]
# ── Special processing ─────────────────────────────────────────────────
class TestSpecialProcessing:
def test_browser_mode_builtin_sets_managed(self):
BrowserConfig.set_defaults(browser_mode="builtin")
cfg = BrowserConfig()
assert cfg.use_managed_browser is True
def test_viewport_dict_overrides_dimensions(self):
BrowserConfig.set_defaults(viewport={"width": 1920, "height": 1080})
cfg = BrowserConfig()
assert cfg.viewport_width == 1920
assert cfg.viewport_height == 1080
def test_proxy_string_converted_to_proxy_config(self):
BrowserConfig.set_defaults(proxy="http://user:pass@proxy:8080")
cfg = BrowserConfig()
assert cfg.proxy_config is not None
assert cfg.proxy_config.server == "http://proxy:8080"
def test_crawler_run_config_proxy_dict_converted(self):
CrawlerRunConfig.set_defaults(
proxy_config={"server": "http://proxy:8080"}
)
cfg = CrawlerRunConfig()
from crawl4ai.async_configs import ProxyConfig
assert isinstance(cfg.proxy_config, ProxyConfig)
# ── Clone / from_kwargs ────────────────────────────────────────────────
class TestCloneAndFromKwargs:
def test_clone_preserves_user_default_values(self):
BrowserConfig.set_defaults(headless=False, viewport_width=1920)
cfg = BrowserConfig()
cloned = cfg.clone()
assert cloned.headless is False
assert cloned.viewport_width == 1920
def test_clone_with_override(self):
BrowserConfig.set_defaults(headless=False)
cfg = BrowserConfig()
cloned = cfg.clone(headless=True)
assert cloned.headless is True
def test_from_kwargs_explicit_values(self):
BrowserConfig.set_defaults(headless=False)
cfg = BrowserConfig.from_kwargs({"headless": True})
assert cfg.headless is True
# ── Dump / Load round-trip ─────────────────────────────────────────────
class TestDumpLoad:
def test_dump_load_preserves_user_defaults(self):
BrowserConfig.set_defaults(headless=False, viewport_width=1920)
cfg = BrowserConfig()
data = cfg.dump()
loaded = BrowserConfig.load(data)
assert loaded.headless is False
assert loaded.viewport_width == 1920
def test_dump_load_survives_reset(self):
"""Values should be baked into serialized data, independent of class defaults."""
BrowserConfig.set_defaults(headless=False)
cfg = BrowserConfig()
data = cfg.dump()
BrowserConfig.reset_defaults()
loaded = BrowserConfig.load(data)
assert loaded.headless is False
def test_crawler_run_config_dump_load(self):
assert CrawlerRunConfig().body_visibility_timeout == 30000
CrawlerRunConfig.set_defaults(
verbose=False, scan_full_page=True, body_visibility_timeout=2000
)
cfg = CrawlerRunConfig()
data = cfg.dump()
CrawlerRunConfig.reset_defaults()
loaded = CrawlerRunConfig.load(data)
assert loaded.verbose is False
assert loaded.scan_full_page is True
assert loaded.body_visibility_timeout == 2000
@pytest.mark.parametrize("timeout", [None, 0, -1, "1000", True])
def test_body_visibility_timeout_must_be_positive_number(self, timeout):
with pytest.raises(ValueError, match="must be a positive number"):
CrawlerRunConfig(body_visibility_timeout=timeout)
def test_untrusted_body_visibility_timeout_is_clamped(self):
from crawl4ai.async_configs import Provenance
config = CrawlerRunConfig.load(
{"body_visibility_timeout": 500_000}, provenance=Provenance.UNTRUSTED
)
assert config.body_visibility_timeout == 60_000
@pytest.mark.asyncio
async def test_body_visibility_timeout_reaches_wait(self):
page = MagicMock()
page.evaluate = AsyncMock()
page.set_content = AsyncMock()
page.wait_for_selector = AsyncMock()
page.content = AsyncMock(return_value="<body>visible</body>")
strategy = AsyncPlaywrightCrawlerStrategy.__new__(
AsyncPlaywrightCrawlerStrategy
)
strategy.browser_config = SimpleNamespace(
use_persistent_context=False, accept_downloads=False, text_mode=True
)
strategy.browser_manager = SimpleNamespace(
get_page=AsyncMock(return_value=(page, MagicMock()))
)
strategy.execute_hook = AsyncMock()
strategy.csp_compliant_wait = AsyncMock(return_value=True)
config = CrawlerRunConfig(
session_id="body-timeout-test", body_visibility_timeout=1234
)
await strategy._crawl_web("raw:<body>visible</body>", config)
assert strategy.csp_compliant_wait.await_args.kwargs["timeout"] == 1234
def test_to_dict_includes_user_default_values(self):
BrowserConfig.set_defaults(headless=False)
cfg = BrowserConfig()
d = cfg.to_dict()
assert d["headless"] is False
# ── Class isolation ────────────────────────────────────────────────────
class TestClassIsolation:
def test_browser_defaults_dont_leak_to_crawler(self):
BrowserConfig.set_defaults(verbose=False)
cfg = CrawlerRunConfig()
assert cfg.verbose is True # CrawlerRunConfig hardcoded default
def test_crawler_defaults_dont_leak_to_browser(self):
CrawlerRunConfig.set_defaults(verbose=False)
cfg = BrowserConfig()
assert cfg.verbose is True # BrowserConfig hardcoded default
def test_independent_reset(self):
BrowserConfig.set_defaults(headless=False)
CrawlerRunConfig.set_defaults(verbose=False)
BrowserConfig.reset_defaults()
assert BrowserConfig.get_defaults() == {}
assert CrawlerRunConfig.get_defaults() == {"verbose": False}
# ── Untrusted timeout ceiling ──────────────────────────────────────────
class TestMaxTimeoutCeiling:
"""CRAWL4AI_MAX_TIMEOUT_MS raises (or lowers) the untrusted clamp."""
TIMEOUT_FIELDS = ("page_timeout", "wait_for_timeout", "body_visibility_timeout")
@pytest.mark.parametrize("field", TIMEOUT_FIELDS)
def test_defaults_to_60s_when_unset(self, monkeypatch, field):
monkeypatch.delenv("CRAWL4AI_MAX_TIMEOUT_MS", raising=False)
config = CrawlerRunConfig.load(
{field: 500_000}, provenance=Provenance.UNTRUSTED
)
assert getattr(config, field) == 60_000
@pytest.mark.parametrize("field", TIMEOUT_FIELDS)
def test_env_raises_the_ceiling(self, monkeypatch, field):
monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "300000")
config = CrawlerRunConfig.load(
{field: 300_000}, provenance=Provenance.UNTRUSTED
)
assert getattr(config, field) == 300_000
def test_a_request_over_the_raised_ceiling_is_still_clamped(self, monkeypatch):
monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "300000")
config = CrawlerRunConfig.load(
{"page_timeout": 900_000}, provenance=Provenance.UNTRUSTED
)
assert config.page_timeout == 300_000
def test_env_can_tighten_the_ceiling(self, monkeypatch):
monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "5000")
config = CrawlerRunConfig.load(
{"page_timeout": 30_000}, provenance=Provenance.UNTRUSTED
)
assert config.page_timeout == 5_000
# A typo must not silently widen a DoS bound, so the default is kept and
# the operator is told rather than left to find out under load.
@pytest.mark.parametrize("value", ["", "abc", "0", "-1", "1e5"])
def test_a_non_positive_integer_keeps_the_default(self, monkeypatch, value):
monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", value)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
config = CrawlerRunConfig.load(
{"page_timeout": 500_000}, provenance=Provenance.UNTRUSTED
)
assert config.page_timeout == 60_000
def test_an_underscored_integer_is_accepted(self, monkeypatch):
# int("60_000") == 60000 in Python, so this is a valid ceiling, not a typo.
monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "120_000")
config = CrawlerRunConfig.load(
{"page_timeout": 500_000}, provenance=Provenance.UNTRUSTED
)
assert config.page_timeout == 120_000
def test_malformed_timeout_falls_back_to_the_default_not_the_ceiling(
self, monkeypatch
):
# The one field an untrusted caller gets for free must not inherit a
# raised ceiling just by being junk.
monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "300000")
config = CrawlerRunConfig.load(
{"page_timeout": 0}, provenance=Provenance.UNTRUSTED
)
assert config.page_timeout == 60_000
def test_malformed_timeout_respects_a_tightened_ceiling(self, monkeypatch):
monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "5000")
config = CrawlerRunConfig.load(
{"page_timeout": "abc"}, provenance=Provenance.UNTRUSTED
)
assert config.page_timeout == 5_000
@pytest.mark.parametrize("value", ["abc", "0", "-1"])
def test_a_bad_value_warns(self, monkeypatch, value):
monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", value)
with pytest.warns(UserWarning, match="CRAWL4AI_MAX_TIMEOUT_MS"):
CrawlerRunConfig.load(
{"page_timeout": 1_000}, provenance=Provenance.UNTRUSTED
)
def test_trusted_config_is_never_clamped(self, monkeypatch):
monkeypatch.delenv("CRAWL4AI_MAX_TIMEOUT_MS", raising=False)
config = CrawlerRunConfig(page_timeout=900_000)
assert config.page_timeout == 900_000