Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion crawl4ai/async_configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ class UntrustedConfigError(ValueError):
"fetch_ssl_certificate",
# timing / waiting
"wait_until", "page_timeout", "wait_for", "wait_for_timeout",
"body_visibility_timeout",
"wait_for_images", "delay_before_return_html", "mean_delay", "max_range",
# scrolling / rendering
"ignore_body_visibility", "scan_full_page", "scroll_delay",
Expand Down Expand Up @@ -299,7 +300,7 @@ def _cap_timeout(v):
return min(int(v), _MAX_TIMEOUT_MS)

if type_name == "CrawlerRunConfig":
for f in ("page_timeout", "wait_for_timeout"):
for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):
if f in params:
params[f] = _cap_timeout(params[f])
if isinstance(params.get("max_scroll_steps"), int):
Expand Down Expand Up @@ -1463,6 +1464,8 @@ class CrawlerRunConfig():
Default: False.
ignore_body_visibility (bool): If True, ignore whether the body is visible before proceeding.
Default: True.
body_visibility_timeout (int): Maximum time in ms to wait for the body to become visible.
Default: 30000.
scan_full_page (bool): If True, scroll through the entire page to load all content.
Default: False.
scroll_delay (float): Delay in seconds between scroll steps if scan_full_page is True.
Expand Down Expand Up @@ -1640,6 +1643,7 @@ def __init__(
c4a_script: Union[str, List[str]] = None,
js_only: bool = False,
ignore_body_visibility: bool = True,
body_visibility_timeout: int = 30000,
scan_full_page: bool = False,
scroll_delay: float = 0.2,
max_scroll_steps: Optional[int] = None,
Expand Down Expand Up @@ -1770,6 +1774,13 @@ def __init__(
self.c4a_script = c4a_script
self.js_only = js_only
self.ignore_body_visibility = ignore_body_visibility
if (
not isinstance(body_visibility_timeout, (int, float))
or isinstance(body_visibility_timeout, bool)
or body_visibility_timeout <= 0
):
raise ValueError("body_visibility_timeout must be a positive number")
self.body_visibility_timeout = body_visibility_timeout
self.scan_full_page = scan_full_page
self.scroll_delay = scroll_delay
self.max_scroll_steps = max_scroll_steps
Expand Down Expand Up @@ -2137,6 +2148,7 @@ def to_dict(self):
"js_code_before_wait": self.js_code_before_wait,
"js_only": self.js_only,
"ignore_body_visibility": self.ignore_body_visibility,
"body_visibility_timeout": self.body_visibility_timeout,
"scan_full_page": self.scan_full_page,
"scroll_delay": self.scroll_delay,
"max_scroll_steps": self.max_scroll_steps,
Expand Down
2 changes: 1 addition & 1 deletion crawl4ai/async_crawler_strategy.py
Original file line number Diff line number Diff line change
Expand Up @@ -823,7 +823,7 @@ async def handle_request_failed_capture(request):
style.opacity !== '0';
return isVisible;
}""",
timeout=30000,
timeout=config.body_visibility_timeout,
)

if not is_visible and not config.ignore_body_visibility:
Expand Down
1 change: 1 addition & 0 deletions docs/md_v2/api/parameters.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@ Use these for controlling whether you read or write from a local content cache.
| **`c4a_script`** | `str or list[str]` (None) | C4A script that compiles to JavaScript. Alternative to writing raw JS. |
| **`js_only`** | `bool` (False) | If `True`, indicates we're reusing an existing session and only applying JS. No full reload. |
| **`ignore_body_visibility`** | `bool` (True) | Skip checking if `<body>` is visible. Usually best to keep `True`. |
| **`body_visibility_timeout`** | `int` (30000) | Maximum time in milliseconds to wait for `<body>` to become visible. Must be positive. |
| **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). |
| **`scroll_delay`** | `float` (0.2) | Delay between scroll steps when scanning the full page (`scan_full_page=True`) or capturing full-page screenshots. |
| **`max_scroll_steps`** | `int or None` (None) | Maximum number of scroll steps during full page scan. If None, scrolls until entire page is loaded. |
Expand Down
1 change: 1 addition & 0 deletions docs/md_v2/complete-sdk-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -1791,6 +1791,7 @@ run_cfg = CrawlerRunConfig(
| **`js_code_before_wait`** | `str or list[str]` (None) | JavaScript to run **before** `wait_for`. Use for triggering loading that `wait_for` then checks. |
| **`js_only`** | `bool` (False) | If `True`, indicates we're reusing an existing session and only applying JS. No full reload. |
| **`ignore_body_visibility`** | `bool` (True) | Skip checking if `<body>` is visible. Usually best to keep `True`. |
| **`body_visibility_timeout`** | `int` (30000) | Maximum time in milliseconds to wait for `<body>` to become visible. Must be positive. |
| **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). |
| **`scroll_delay`** | `float` (0.2) | Delay between scroll steps when scanning the full page (`scan_full_page=True`) or capturing full-page screenshots. |
| **`process_iframes`** | `bool` (False) | Inlines iframe content for single-page extraction. |
Expand Down
51 changes: 50 additions & 1 deletion tests/test_config_defaults.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
"""Tests for BrowserConfig.set_defaults / CrawlerRunConfig.set_defaults."""

from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock

import pytest

from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy


@pytest.fixture(autouse=True)
Expand Down Expand Up @@ -226,13 +231,57 @@ def test_dump_load_survives_reset(self):
assert loaded.headless is False

def test_crawler_run_config_dump_load(self):
CrawlerRunConfig.set_defaults(verbose=False, scan_full_page=True)
assert CrawlerRunConfig().body_visibility_timeout == 30000
CrawlerRunConfig.set_defaults(
verbose=False, scan_full_page=True, body_visibility_timeout=2000
)
cfg = CrawlerRunConfig()
data = cfg.dump()
CrawlerRunConfig.reset_defaults()
loaded = CrawlerRunConfig.load(data)
assert loaded.verbose is False
assert loaded.scan_full_page is True
assert loaded.body_visibility_timeout == 2000

@pytest.mark.parametrize("timeout", [None, 0, -1, "1000", True])
def test_body_visibility_timeout_must_be_positive_number(self, timeout):
with pytest.raises(ValueError, match="must be a positive number"):
CrawlerRunConfig(body_visibility_timeout=timeout)

def test_untrusted_body_visibility_timeout_is_clamped(self):
from crawl4ai.async_configs import Provenance

config = CrawlerRunConfig.load(
{"body_visibility_timeout": 500_000}, provenance=Provenance.UNTRUSTED
)
assert config.body_visibility_timeout == 60_000

@pytest.mark.asyncio
async def test_body_visibility_timeout_reaches_wait(self):
page = MagicMock()
page.evaluate = AsyncMock()
page.set_content = AsyncMock()
page.wait_for_selector = AsyncMock()
page.content = AsyncMock(return_value="<body>visible</body>")

strategy = AsyncPlaywrightCrawlerStrategy.__new__(
AsyncPlaywrightCrawlerStrategy
)
strategy.browser_config = SimpleNamespace(
use_persistent_context=False, accept_downloads=False, text_mode=True
)
strategy.browser_manager = SimpleNamespace(
get_page=AsyncMock(return_value=(page, MagicMock()))
)
strategy.execute_hook = AsyncMock()
strategy.csp_compliant_wait = AsyncMock(return_value=True)

config = CrawlerRunConfig(
session_id="body-timeout-test", body_visibility_timeout=1234
)
await strategy._crawl_web("raw:<body>visible</body>", config)

assert strategy.csp_compliant_wait.await_args.kwargs["timeout"] == 1234

def test_to_dict_includes_user_default_values(self):
BrowserConfig.set_defaults(headless=False)
Expand Down
Loading