Crawl4AI 0.9.2__tar.gz → 0.9.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/PKG-INFO +41 -3
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/SOURCES.txt +8 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/PKG-INFO +41 -3
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/README.md +40 -2
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/__init__.py +2 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/__version__.py +1 -1
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_configs.py +149 -19
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_crawler_strategy.py +32 -2
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_url_seeder.py +7 -2
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_webcrawler.py +3 -2
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/browser_manager.py +36 -5
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/cli.py +2 -1
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/content_filter_strategy.py +28 -0
- crawl4ai-0.9.4/crawl4ai/content_filter_strategy_lxml.py +391 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/bff_strategy.py +7 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/bfs_strategy.py +6 -2
- crawl4ai-0.9.4/crawl4ai/egress_policy.py +35 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/remove_consent_popups.js +7 -2
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/remove_overlay_elements.js +9 -2
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/models.py +1 -0
- crawl4ai-0.9.4/crawl4ai/processors/pdf/__init__.py +360 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/processors/pdf/processor.py +23 -3
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/processors/pdf/utils.py +8 -4
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/table_extraction.py +58 -11
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/types.py +4 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/utils.py +67 -24
- crawl4ai-0.9.4/tests/test_body_visibility_warning.py +128 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_config_defaults.py +161 -2
- crawl4ai-0.9.4/tests/test_docker_pdf_crawler_pairing.py +273 -0
- crawl4ai-0.9.4/tests/test_issue_2127_docker_pdf.py +71 -0
- crawl4ai-0.9.4/tests/test_issue_2161_overlay_html_body.py +81 -0
- crawl4ai-0.9.4/tests/test_placeholder_html_antibot.py +134 -0
- crawl4ai-0.9.4/tests/test_table_rowspan_grid.py +162 -0
- crawl4ai-0.9.2/crawl4ai/processors/pdf/__init__.py +0 -196
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/dependency_links.txt +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/entry_points.txt +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/requires.txt +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/top_level.txt +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/LICENSE +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/MANIFEST.in +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/adaptive_crawler.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/antibot_detector.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_database.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_dispatcher.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_logger.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/browser_adapter.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/browser_profiler.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/cache_context.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/cache_validator.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/chunking_strategy.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/cloud/__init__.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/cloud/cli.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/components/crawler_monitor.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/config.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/content_scraping_strategy.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/__init__.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/amazon_product/__init__.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/amazon_product/crawler.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/__init__.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/crawler.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/script.js +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/__init__.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/base_strategy.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/crazy.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/dfs_strategy.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/filters.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/scorers.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/docker_client.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/domain_mapper.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/extraction_strategy.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/__init__.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/__main__.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/_typing.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/cli.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/config.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/elements.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/utils.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/hub.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/install.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/__init__.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/flatten_shadow_dom.js +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/navigator_overrider.js +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/update_image_dimensions.js +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/__init__.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/cli.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/crawler_strategy.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/database.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/docs_manager.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/llmtxt.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/version_manager.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/web_crawler.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/link_preview.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/markdown_generation_strategy.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/migrations.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/model_loader.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/prompts.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/proxy_strategy.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/script/__init__.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/script/c4a_compile.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/script/c4a_result.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/script/c4ai_script.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/ssl_certificate.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/user_agent_generator.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/pyproject.toml +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/requirements.txt +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/setup.cfg +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/setup.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_arun_many.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_async_logger_stderr.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_bug_batch_1622_1786_1796.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_cdp_changes.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_cli_docs.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_cloud_bugs_batch.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_config_matching_only.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_config_selection.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_docker.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_docker_api_with_llm_provider.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_eval_security_adversarial.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_http_timeout_unit_1894.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1043_mermaid_svg.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1213_bm25_dedup.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1370_1818_1762_1509.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1484_css_selector.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1594_mcp_sse.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1611_llm_provider.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1748_screenshot_scroll_delay.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1750_screenshot_scan_full_page.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1837_config_list.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1842_browser_none.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1848_logger_serialize.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1850_mcp_sse.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_link_extractor.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_llm_extraction_parallel_issue_1055.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_llm_simple_url.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_llmtxt.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_main.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_markdown_generator_validation_1880.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_memory_macos.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_merge_head_data_scoring.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_multi_config.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_normalize_url.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pr_1290_1668.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pr_1435_redirected_status_code.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pr_1463_device_scale_factor.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pr_1795_1798_1734.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_prefetch_integration.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_prefetch_mode.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_prefetch_regression.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_preserve_https_for_internal_links.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pruning_preserve_whitelist_1900.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pyopenssl_security_fix.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pyopenssl_update.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_raw_html_browser.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_raw_html_edge_cases.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_raw_html_redirected_url.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_scraping_strategy.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_source_sibling_selector.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_table_gfm_compliance.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_type_annotations.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_virtual_scroll.py +0 -0
- {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_web_crawler.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: Crawl4AI
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.4
|
|
4
4
|
Summary: 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper
|
|
5
5
|
Home-page: https://github.com/unclecode/crawl4ai
|
|
6
6
|
Author: Unclecode
|
|
@@ -120,9 +120,13 @@ Limited slots._
|
|
|
120
120
|
|
|
121
121
|
Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
|
|
122
122
|
|
|
123
|
-
[✨ Check out latest update v0.9.
|
|
123
|
+
[✨ Check out latest update v0.9.4](#-recent-updates)
|
|
124
124
|
|
|
125
|
-
✨ **New in v0.9.
|
|
125
|
+
✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
126
|
+
|
|
127
|
+
✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
|
|
128
|
+
|
|
129
|
+
✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
|
|
126
130
|
|
|
127
131
|
✨ Recent v0.9.0: Major secure-by-default release of the Docker API server. Auth is on by default, the server binds loopback unless given a token, and the request body is now an untrusted trust boundary. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.0.md)
|
|
128
132
|
|
|
@@ -648,6 +652,40 @@ async def test_news_crawl():
|
|
|
648
652
|
## ✨ Recent Updates
|
|
649
653
|
|
|
650
654
|
<details open>
|
|
655
|
+
<summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
|
|
656
|
+
|
|
657
|
+
A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
|
|
658
|
+
|
|
659
|
+
It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
|
|
660
|
+
|
|
661
|
+
No breaking changes.
|
|
662
|
+
|
|
663
|
+
```bash
|
|
664
|
+
pip install -U crawl4ai
|
|
665
|
+
```
|
|
666
|
+
|
|
667
|
+
[Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
668
|
+
|
|
669
|
+
</details>
|
|
670
|
+
|
|
671
|
+
<details>
|
|
672
|
+
<summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
|
|
673
|
+
|
|
674
|
+
A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
|
|
675
|
+
|
|
676
|
+
It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
|
|
677
|
+
|
|
678
|
+
No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
|
|
679
|
+
|
|
680
|
+
```bash
|
|
681
|
+
pip install -U crawl4ai
|
|
682
|
+
```
|
|
683
|
+
|
|
684
|
+
[Full v0.9.3 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
|
|
685
|
+
|
|
686
|
+
</details>
|
|
687
|
+
|
|
688
|
+
<details>
|
|
651
689
|
<summary><strong>Version 0.9.2 Release Highlights - Maintenance Bug Fixes</strong></summary>
|
|
652
690
|
|
|
653
691
|
A maintenance patch release with bug fixes across the dispatcher, Docker, and GPU builds. `MemoryAdaptiveDispatcher` no longer leaks crawl tasks and browser pages when a streaming crawl is closed. Docker fixes cover the Playground "Advanced Config" 400, the Monitor WebSocket 500 under JWT auth, and Playwright headless-shell packaging. `ENABLE_GPU=true` Docker builds no longer fail on the CUDA toolkit.
|
|
@@ -31,9 +31,11 @@ crawl4ai/chunking_strategy.py
|
|
|
31
31
|
crawl4ai/cli.py
|
|
32
32
|
crawl4ai/config.py
|
|
33
33
|
crawl4ai/content_filter_strategy.py
|
|
34
|
+
crawl4ai/content_filter_strategy_lxml.py
|
|
34
35
|
crawl4ai/content_scraping_strategy.py
|
|
35
36
|
crawl4ai/docker_client.py
|
|
36
37
|
crawl4ai/domain_mapper.py
|
|
38
|
+
crawl4ai/egress_policy.py
|
|
37
39
|
crawl4ai/extraction_strategy.py
|
|
38
40
|
crawl4ai/hub.py
|
|
39
41
|
crawl4ai/install.py
|
|
@@ -96,6 +98,7 @@ crawl4ai/script/c4a_result.py
|
|
|
96
98
|
crawl4ai/script/c4ai_script.py
|
|
97
99
|
tests/test_arun_many.py
|
|
98
100
|
tests/test_async_logger_stderr.py
|
|
101
|
+
tests/test_body_visibility_warning.py
|
|
99
102
|
tests/test_bug_batch_1622_1786_1796.py
|
|
100
103
|
tests/test_cdp_changes.py
|
|
101
104
|
tests/test_cli_docs.py
|
|
@@ -105,6 +108,7 @@ tests/test_config_matching_only.py
|
|
|
105
108
|
tests/test_config_selection.py
|
|
106
109
|
tests/test_docker.py
|
|
107
110
|
tests/test_docker_api_with_llm_provider.py
|
|
111
|
+
tests/test_docker_pdf_crawler_pairing.py
|
|
108
112
|
tests/test_eval_security_adversarial.py
|
|
109
113
|
tests/test_http_timeout_unit_1894.py
|
|
110
114
|
tests/test_issue_1043_mermaid_svg.py
|
|
@@ -119,6 +123,8 @@ tests/test_issue_1837_config_list.py
|
|
|
119
123
|
tests/test_issue_1842_browser_none.py
|
|
120
124
|
tests/test_issue_1848_logger_serialize.py
|
|
121
125
|
tests/test_issue_1850_mcp_sse.py
|
|
126
|
+
tests/test_issue_2127_docker_pdf.py
|
|
127
|
+
tests/test_issue_2161_overlay_html_body.py
|
|
122
128
|
tests/test_link_extractor.py
|
|
123
129
|
tests/test_llm_extraction_parallel_issue_1055.py
|
|
124
130
|
tests/test_llm_simple_url.py
|
|
@@ -129,6 +135,7 @@ tests/test_memory_macos.py
|
|
|
129
135
|
tests/test_merge_head_data_scoring.py
|
|
130
136
|
tests/test_multi_config.py
|
|
131
137
|
tests/test_normalize_url.py
|
|
138
|
+
tests/test_placeholder_html_antibot.py
|
|
132
139
|
tests/test_pr_1290_1668.py
|
|
133
140
|
tests/test_pr_1435_redirected_status_code.py
|
|
134
141
|
tests/test_pr_1463_device_scale_factor.py
|
|
@@ -146,6 +153,7 @@ tests/test_raw_html_redirected_url.py
|
|
|
146
153
|
tests/test_scraping_strategy.py
|
|
147
154
|
tests/test_source_sibling_selector.py
|
|
148
155
|
tests/test_table_gfm_compliance.py
|
|
156
|
+
tests/test_table_rowspan_grid.py
|
|
149
157
|
tests/test_type_annotations.py
|
|
150
158
|
tests/test_virtual_scroll.py
|
|
151
159
|
tests/test_web_crawler.py
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: Crawl4AI
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.4
|
|
4
4
|
Summary: 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper
|
|
5
5
|
Home-page: https://github.com/unclecode/crawl4ai
|
|
6
6
|
Author: Unclecode
|
|
@@ -120,9 +120,13 @@ Limited slots._
|
|
|
120
120
|
|
|
121
121
|
Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
|
|
122
122
|
|
|
123
|
-
[✨ Check out latest update v0.9.
|
|
123
|
+
[✨ Check out latest update v0.9.4](#-recent-updates)
|
|
124
124
|
|
|
125
|
-
✨ **New in v0.9.
|
|
125
|
+
✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
126
|
+
|
|
127
|
+
✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
|
|
128
|
+
|
|
129
|
+
✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
|
|
126
130
|
|
|
127
131
|
✨ Recent v0.9.0: Major secure-by-default release of the Docker API server. Auth is on by default, the server binds loopback unless given a token, and the request body is now an untrusted trust boundary. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.0.md)
|
|
128
132
|
|
|
@@ -648,6 +652,40 @@ async def test_news_crawl():
|
|
|
648
652
|
## ✨ Recent Updates
|
|
649
653
|
|
|
650
654
|
<details open>
|
|
655
|
+
<summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
|
|
656
|
+
|
|
657
|
+
A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
|
|
658
|
+
|
|
659
|
+
It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
|
|
660
|
+
|
|
661
|
+
No breaking changes.
|
|
662
|
+
|
|
663
|
+
```bash
|
|
664
|
+
pip install -U crawl4ai
|
|
665
|
+
```
|
|
666
|
+
|
|
667
|
+
[Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
668
|
+
|
|
669
|
+
</details>
|
|
670
|
+
|
|
671
|
+
<details>
|
|
672
|
+
<summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
|
|
673
|
+
|
|
674
|
+
A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
|
|
675
|
+
|
|
676
|
+
It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
|
|
677
|
+
|
|
678
|
+
No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
|
|
679
|
+
|
|
680
|
+
```bash
|
|
681
|
+
pip install -U crawl4ai
|
|
682
|
+
```
|
|
683
|
+
|
|
684
|
+
[Full v0.9.3 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
|
|
685
|
+
|
|
686
|
+
</details>
|
|
687
|
+
|
|
688
|
+
<details>
|
|
651
689
|
<summary><strong>Version 0.9.2 Release Highlights - Maintenance Bug Fixes</strong></summary>
|
|
652
690
|
|
|
653
691
|
A maintenance patch release with bug fixes across the dispatcher, Docker, and GPU builds. `MemoryAdaptiveDispatcher` no longer leaks crawl tasks and browser pages when a streaming crawl is closed. Docker fixes cover the Playground "Advanced Config" 400, the Monitor WebSocket 500 under JWT auth, and Playwright headless-shell packaging. `ENABLE_GPU=true` Docker builds no longer fail on the CUDA toolkit.
|
|
@@ -37,9 +37,13 @@ Limited slots._
|
|
|
37
37
|
|
|
38
38
|
Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
|
|
39
39
|
|
|
40
|
-
[✨ Check out latest update v0.9.
|
|
40
|
+
[✨ Check out latest update v0.9.4](#-recent-updates)
|
|
41
41
|
|
|
42
|
-
✨ **New in v0.9.
|
|
42
|
+
✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
43
|
+
|
|
44
|
+
✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
|
|
45
|
+
|
|
46
|
+
✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
|
|
43
47
|
|
|
44
48
|
✨ Recent v0.9.0: Major secure-by-default release of the Docker API server. Auth is on by default, the server binds loopback unless given a token, and the request body is now an untrusted trust boundary. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.0.md)
|
|
45
49
|
|
|
@@ -565,6 +569,40 @@ async def test_news_crawl():
|
|
|
565
569
|
## ✨ Recent Updates
|
|
566
570
|
|
|
567
571
|
<details open>
|
|
572
|
+
<summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
|
|
573
|
+
|
|
574
|
+
A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
|
|
575
|
+
|
|
576
|
+
It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
|
|
577
|
+
|
|
578
|
+
No breaking changes.
|
|
579
|
+
|
|
580
|
+
```bash
|
|
581
|
+
pip install -U crawl4ai
|
|
582
|
+
```
|
|
583
|
+
|
|
584
|
+
[Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
585
|
+
|
|
586
|
+
</details>
|
|
587
|
+
|
|
588
|
+
<details>
|
|
589
|
+
<summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
|
|
590
|
+
|
|
591
|
+
A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
|
|
592
|
+
|
|
593
|
+
It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
|
|
594
|
+
|
|
595
|
+
No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
|
|
596
|
+
|
|
597
|
+
```bash
|
|
598
|
+
pip install -U crawl4ai
|
|
599
|
+
```
|
|
600
|
+
|
|
601
|
+
[Full v0.9.3 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
|
|
602
|
+
|
|
603
|
+
</details>
|
|
604
|
+
|
|
605
|
+
<details>
|
|
568
606
|
<summary><strong>Version 0.9.2 Release Highlights - Maintenance Bug Fixes</strong></summary>
|
|
569
607
|
|
|
570
608
|
A maintenance patch release with bug fixes across the dispatcher, Docker, and GPU builds. `MemoryAdaptiveDispatcher` no longer leaks crawl tasks and browser pages when a streaming crawl is closed. Docker fixes cover the Playground "Advanced Config" 400, the Monitor WebSocket 500 under JWT auth, and Playwright headless-shell packaging. `ENABLE_GPU=true` Docker builds no longer fail on the CUDA toolkit.
|
|
@@ -42,6 +42,7 @@ from .content_filter_strategy import (
|
|
|
42
42
|
LLMContentFilter,
|
|
43
43
|
RelevantContentFilter,
|
|
44
44
|
)
|
|
45
|
+
from .content_filter_strategy_lxml import PruningContentFilterLXML
|
|
45
46
|
from .models import CrawlResult, MarkdownGenerationResult, DisplayMode
|
|
46
47
|
from .components.crawler_monitor import CrawlerMonitor
|
|
47
48
|
from .link_preview import LinkPreview
|
|
@@ -176,6 +177,7 @@ __all__ = [
|
|
|
176
177
|
"LLMTableExtraction",
|
|
177
178
|
"RelevantContentFilter",
|
|
178
179
|
"PruningContentFilter",
|
|
180
|
+
"PruningContentFilterLXML",
|
|
179
181
|
"BM25ContentFilter",
|
|
180
182
|
"LLMContentFilter",
|
|
181
183
|
"BaseDispatcher",
|
|
@@ -133,7 +133,7 @@ ALLOWED_DESERIALIZE_TYPES = {
|
|
|
133
133
|
"CosineStrategy", "RegexExtractionStrategy",
|
|
134
134
|
# Markdown / content
|
|
135
135
|
"DefaultMarkdownGenerator",
|
|
136
|
-
"PruningContentFilter", "BM25ContentFilter", "LLMContentFilter",
|
|
136
|
+
"PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter", "LLMContentFilter",
|
|
137
137
|
# Scraping
|
|
138
138
|
"LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
|
|
139
139
|
# Chunking
|
|
@@ -190,7 +190,7 @@ UNTRUSTED_ALLOWED_TYPES = {
|
|
|
190
190
|
# non-LLM extraction / markdown / scraping / chunking strategies
|
|
191
191
|
"JsonCssExtractionStrategy", "JsonXPathExtractionStrategy",
|
|
192
192
|
"JsonLxmlExtractionStrategy", "RegexExtractionStrategy", "CosineStrategy",
|
|
193
|
-
"DefaultMarkdownGenerator", "PruningContentFilter", "BM25ContentFilter",
|
|
193
|
+
"DefaultMarkdownGenerator", "PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter",
|
|
194
194
|
"LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
|
|
195
195
|
"RegexChunking",
|
|
196
196
|
"DefaultTableExtraction", "NoTableExtraction",
|
|
@@ -214,6 +214,29 @@ UNTRUSTED_FORBIDDEN_FIELDS = {
|
|
|
214
214
|
"override_navigator", "magic", "process_in_browser", "shared_data",
|
|
215
215
|
"session_id",
|
|
216
216
|
},
|
|
217
|
+
# PDFContentScrapingStrategy has no scalar allowlist (it is kept for its
|
|
218
|
+
# value fields), so its filesystem-write knobs must be blocked explicitly:
|
|
219
|
+
# image_save_dir is an unconfined write destination and save_images_locally
|
|
220
|
+
# turns the write on. extract_images stays allowed - it only returns the
|
|
221
|
+
# image bytes base64-inline in the response, no disk write.
|
|
222
|
+
"PDFContentScrapingStrategy": {
|
|
223
|
+
"image_save_dir", "save_images_locally",
|
|
224
|
+
},
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
# Field names that must NEVER be set from an untrusted body on ANY allowed
|
|
228
|
+
# type, checked regardless of per-type allowlist. This is the fail-closed
|
|
229
|
+
# backstop: the per-type maps above only cover BrowserConfig/CrawlerRunConfig,
|
|
230
|
+
# so a strategy that (now or in future) exposes a filesystem-write, code, or
|
|
231
|
+
# routing constructor arg is caught here even with no explicit allowlist.
|
|
232
|
+
# Presence => 400 (loud), matching js_code/extra_args behavior.
|
|
233
|
+
UNTRUSTED_GLOBAL_FORBIDDEN_FIELDS = {
|
|
234
|
+
# filesystem write / read sinks
|
|
235
|
+
"image_save_dir", "save_images_locally", "downloads_path", "user_data_dir",
|
|
236
|
+
"output_path", "save_path", "file_path", "local_path", "storage_state",
|
|
237
|
+
# code / command execution
|
|
238
|
+
"js_code", "js_code_before_wait", "c4a_script", "init_scripts",
|
|
239
|
+
"code", "command", "hook", "hooks",
|
|
217
240
|
}
|
|
218
241
|
|
|
219
242
|
# Scalar knobs an untrusted body MAY set, per class. A field not listed here is
|
|
@@ -244,6 +267,7 @@ UNTRUSTED_FIELD_ALLOWLIST = {
|
|
|
244
267
|
"fetch_ssl_certificate",
|
|
245
268
|
# timing / waiting
|
|
246
269
|
"wait_until", "page_timeout", "wait_for", "wait_for_timeout",
|
|
270
|
+
"body_visibility_timeout",
|
|
247
271
|
"wait_for_images", "delay_before_return_html", "mean_delay", "max_range",
|
|
248
272
|
# scrolling / rendering
|
|
249
273
|
"ignore_body_visibility", "scan_full_page", "scroll_delay",
|
|
@@ -269,9 +293,49 @@ UNTRUSTED_FIELD_ALLOWLIST = {
|
|
|
269
293
|
}
|
|
270
294
|
|
|
271
295
|
# Upper bounds applied to attacker-influenced quantities after filtering.
|
|
272
|
-
|
|
296
|
+
_DEFAULT_MAX_TIMEOUT_MS = 60_000
|
|
273
297
|
_MAX_SCROLL_STEPS = 1000
|
|
274
298
|
_MAX_VIEWPORT = 4000
|
|
299
|
+
_MAX_PDF_BYTES = 100 * 1024 * 1024
|
|
300
|
+
_MAX_PDF_PAGES = 2000
|
|
301
|
+
_MAX_LINK_PREVIEW_LINKS = 100 # the class default
|
|
302
|
+
_MAX_LINK_PREVIEW_CONCURRENCY = 10 # the class default
|
|
303
|
+
_MAX_LINK_PREVIEW_TIMEOUT_S = 10 # seconds here, not ms
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _max_timeout_ms() -> int:
|
|
307
|
+
"""Ceiling for the untrusted timeout fields, in milliseconds.
|
|
308
|
+
|
|
309
|
+
60s is the right bound for a server reachable by untrusted callers, and
|
|
310
|
+
stays the default. An operator whose deployment is not public — a crawler
|
|
311
|
+
on a private network fetching pages that legitimately take minutes — can
|
|
312
|
+
raise it with CRAWL4AI_MAX_TIMEOUT_MS, or lower it to tighten the bound.
|
|
313
|
+
|
|
314
|
+
Read per call rather than captured at import so the setting applies
|
|
315
|
+
wherever the process picked its environment up, and so a test can set it
|
|
316
|
+
without reloading the module. A value that is not a positive integer is
|
|
317
|
+
refused loudly and the default kept: a typo here would silently widen a
|
|
318
|
+
DoS bound, which is the one outcome worse than the timeout being fixed.
|
|
319
|
+
"""
|
|
320
|
+
raw = os.getenv("CRAWL4AI_MAX_TIMEOUT_MS")
|
|
321
|
+
|
|
322
|
+
if raw is None or raw == "":
|
|
323
|
+
return _DEFAULT_MAX_TIMEOUT_MS
|
|
324
|
+
|
|
325
|
+
try:
|
|
326
|
+
ceiling = int(raw)
|
|
327
|
+
except ValueError:
|
|
328
|
+
ceiling = 0
|
|
329
|
+
|
|
330
|
+
if ceiling <= 0:
|
|
331
|
+
warnings.warn(
|
|
332
|
+
f"CRAWL4AI_MAX_TIMEOUT_MS={raw!r} is not a positive integer; "
|
|
333
|
+
f"keeping the {_DEFAULT_MAX_TIMEOUT_MS}ms default.",
|
|
334
|
+
stacklevel=2,
|
|
335
|
+
)
|
|
336
|
+
return _DEFAULT_MAX_TIMEOUT_MS
|
|
337
|
+
|
|
338
|
+
return ceiling
|
|
275
339
|
|
|
276
340
|
|
|
277
341
|
def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
|
|
@@ -280,7 +344,7 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
|
|
|
280
344
|
allowlist = UNTRUSTED_FIELD_ALLOWLIST.get(type_name) # None => keep all non-forbidden
|
|
281
345
|
out = {}
|
|
282
346
|
for key, value in params.items():
|
|
283
|
-
if key in forbidden:
|
|
347
|
+
if key in UNTRUSTED_GLOBAL_FORBIDDEN_FIELDS or key in forbidden:
|
|
284
348
|
raise UntrustedConfigError(
|
|
285
349
|
f"field '{key}' is not permitted on {type_name} from an untrusted request"
|
|
286
350
|
)
|
|
@@ -292,14 +356,18 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
|
|
|
292
356
|
|
|
293
357
|
def _clamp_untrusted(type_name: str, params: dict) -> dict:
|
|
294
358
|
"""Clamp attacker-influenced quantities to safe upper bounds."""
|
|
359
|
+
ceiling = _max_timeout_ms()
|
|
360
|
+
|
|
295
361
|
def _cap_timeout(v):
|
|
296
362
|
# 0 historically meant "no timeout"; treat as the cap, never unbounded.
|
|
363
|
+
# Malformed input is the one value an attacker gets for free, so it
|
|
364
|
+
# falls back to the 60s default rather than to a raised ceiling.
|
|
297
365
|
if not isinstance(v, (int, float)) or v <= 0:
|
|
298
|
-
return
|
|
299
|
-
return min(int(v),
|
|
366
|
+
return min(_DEFAULT_MAX_TIMEOUT_MS, ceiling)
|
|
367
|
+
return min(int(v), ceiling)
|
|
300
368
|
|
|
301
369
|
if type_name == "CrawlerRunConfig":
|
|
302
|
-
for f in ("page_timeout", "wait_for_timeout"):
|
|
370
|
+
for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):
|
|
303
371
|
if f in params:
|
|
304
372
|
params[f] = _cap_timeout(params[f])
|
|
305
373
|
if isinstance(params.get("max_scroll_steps"), int):
|
|
@@ -308,6 +376,29 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:
|
|
|
308
376
|
for f in ("viewport_width", "viewport_height"):
|
|
309
377
|
if isinstance(params.get(f), int):
|
|
310
378
|
params[f] = max(1, min(params[f], _MAX_VIEWPORT))
|
|
379
|
+
elif type_name == "PDFContentScrapingStrategy":
|
|
380
|
+
# This class has no field allowlist, so without clamping a body could
|
|
381
|
+
# simply raise its own caps back to unbounded and re-open the DoS.
|
|
382
|
+
for f, cap in (("max_pdf_bytes", _MAX_PDF_BYTES), ("max_pdf_pages", _MAX_PDF_PAGES)):
|
|
383
|
+
if f in params:
|
|
384
|
+
v = params[f]
|
|
385
|
+
# <=0 or non-int would read as "no limit"; pin to the cap.
|
|
386
|
+
params[f] = cap if not isinstance(v, int) or v <= 0 else min(v, cap)
|
|
387
|
+
# Rasterizing every page is the most expensive thing this strategy can
|
|
388
|
+
# do, and nothing about untrusted crawling needs it.
|
|
389
|
+
params["extract_images"] = False
|
|
390
|
+
elif type_name == "LinkPreviewConfig":
|
|
391
|
+
# Also no field allowlist, and each link is a separate outbound fetch:
|
|
392
|
+
# unclamped, one request body can order millions of them at any
|
|
393
|
+
# concurrency it likes.
|
|
394
|
+
for f, cap in (
|
|
395
|
+
("max_links", _MAX_LINK_PREVIEW_LINKS),
|
|
396
|
+
("concurrency", _MAX_LINK_PREVIEW_CONCURRENCY),
|
|
397
|
+
("timeout", _MAX_LINK_PREVIEW_TIMEOUT_S),
|
|
398
|
+
):
|
|
399
|
+
if f in params:
|
|
400
|
+
v = params[f]
|
|
401
|
+
params[f] = cap if not isinstance(v, int) or v <= 0 else min(v, cap)
|
|
311
402
|
return params
|
|
312
403
|
|
|
313
404
|
|
|
@@ -397,6 +488,21 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False):
|
|
|
397
488
|
return str(obj)
|
|
398
489
|
|
|
399
490
|
|
|
491
|
+
def _is_typed_shape(data: Any) -> bool:
|
|
492
|
+
"""True for the dict shapes to_serializable_dict() emits:
|
|
493
|
+
{"type": "<ClassName>", "params": {...}} or {"type": "dict", "value": {...}}.
|
|
494
|
+
|
|
495
|
+
Plain business dicts that happen to carry a "type" key (JSON-Schema
|
|
496
|
+
fragments, JsonCss field specs like {"type": "text", "name": "..."}) have
|
|
497
|
+
neither "params" nor "value" and are not typed shapes.
|
|
498
|
+
"""
|
|
499
|
+
return (
|
|
500
|
+
isinstance(data, dict)
|
|
501
|
+
and "type" in data
|
|
502
|
+
and ("params" in data or (data["type"] == "dict" and "value" in data))
|
|
503
|
+
)
|
|
504
|
+
|
|
505
|
+
|
|
400
506
|
def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
|
|
401
507
|
"""
|
|
402
508
|
Recursively convert a serializable dictionary back to an object instance.
|
|
@@ -422,14 +528,23 @@ def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
|
|
|
422
528
|
# carry a "type" key (e.g. JSON-Schema fragments, JsonCss field specs like
|
|
423
529
|
# {"type": "text", "name": "..."}) have neither "params" nor "value" and
|
|
424
530
|
# must fall through to the raw-dict path below so they are passed as data.
|
|
425
|
-
if (
|
|
426
|
-
isinstance(data, dict)
|
|
427
|
-
and "type" in data
|
|
428
|
-
and ("params" in data or (data["type"] == "dict" and "value" in data))
|
|
429
|
-
):
|
|
531
|
+
if _is_typed_shape(data):
|
|
430
532
|
# Handle plain dictionaries
|
|
431
533
|
if data["type"] == "dict" and "value" in data:
|
|
432
|
-
|
|
534
|
+
unwrapped = {
|
|
535
|
+
k: from_serializable_dict(v, provenance) for k, v in data["value"].items()
|
|
536
|
+
}
|
|
537
|
+
# The unwrap above recurses over .items(), so data["value"] is never
|
|
538
|
+
# seen as a whole typed object and the type gate below never fires
|
|
539
|
+
# for it. Without this check an untrusted body launders a forbidden
|
|
540
|
+
# type past the gate by wrapping it:
|
|
541
|
+
# {"type": "dict", "value": {"type": "LLMConfig", "params": {...}}}
|
|
542
|
+
if provenance == Provenance.UNTRUSTED and _is_typed_shape(unwrapped):
|
|
543
|
+
raise UntrustedConfigError(
|
|
544
|
+
"an untrusted request may not wrap a typed object in "
|
|
545
|
+
"{'type': 'dict', 'value': ...}"
|
|
546
|
+
)
|
|
547
|
+
return unwrapped
|
|
433
548
|
|
|
434
549
|
# Security: only allow known-safe types to be deserialized.
|
|
435
550
|
# Unknown types (e.g. logging.Logger serialized by older clients) are
|
|
@@ -936,11 +1051,13 @@ class BrowserConfig:
|
|
|
936
1051
|
)
|
|
937
1052
|
|
|
938
1053
|
@staticmethod
|
|
939
|
-
def from_kwargs(kwargs: dict) -> "BrowserConfig":
|
|
1054
|
+
def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "BrowserConfig":
|
|
940
1055
|
# Auto-deserialize any dict values that use the {"type": ..., "params": ...}
|
|
941
1056
|
# serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
|
|
942
1057
|
kwargs = {
|
|
943
|
-
k: from_serializable_dict(v
|
|
1058
|
+
k: from_serializable_dict(v, provenance)
|
|
1059
|
+
if isinstance(v, dict) and "type" in v
|
|
1060
|
+
else v
|
|
944
1061
|
for k, v in kwargs.items()
|
|
945
1062
|
}
|
|
946
1063
|
# Only pass keys present in kwargs so that __init__ defaults (and
|
|
@@ -1026,7 +1143,7 @@ class BrowserConfig:
|
|
|
1026
1143
|
# so a body that sends bare kwargs (no {type,params}) cannot bypass it.
|
|
1027
1144
|
if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
|
|
1028
1145
|
config = _enforce_untrusted("BrowserConfig", config)
|
|
1029
|
-
return BrowserConfig.from_kwargs(config)
|
|
1146
|
+
return BrowserConfig.from_kwargs(config, provenance)
|
|
1030
1147
|
|
|
1031
1148
|
def set_nstproxy(
|
|
1032
1149
|
self,
|
|
@@ -1463,6 +1580,8 @@ class CrawlerRunConfig():
|
|
|
1463
1580
|
Default: False.
|
|
1464
1581
|
ignore_body_visibility (bool): If True, ignore whether the body is visible before proceeding.
|
|
1465
1582
|
Default: True.
|
|
1583
|
+
body_visibility_timeout (int): Maximum time in ms to wait for the body to become visible.
|
|
1584
|
+
Default: 30000.
|
|
1466
1585
|
scan_full_page (bool): If True, scroll through the entire page to load all content.
|
|
1467
1586
|
Default: False.
|
|
1468
1587
|
scroll_delay (float): Delay in seconds between scroll steps if scan_full_page is True.
|
|
@@ -1640,6 +1759,7 @@ class CrawlerRunConfig():
|
|
|
1640
1759
|
c4a_script: Union[str, List[str]] = None,
|
|
1641
1760
|
js_only: bool = False,
|
|
1642
1761
|
ignore_body_visibility: bool = True,
|
|
1762
|
+
body_visibility_timeout: int = 30000,
|
|
1643
1763
|
scan_full_page: bool = False,
|
|
1644
1764
|
scroll_delay: float = 0.2,
|
|
1645
1765
|
max_scroll_steps: Optional[int] = None,
|
|
@@ -1770,6 +1890,13 @@ class CrawlerRunConfig():
|
|
|
1770
1890
|
self.c4a_script = c4a_script
|
|
1771
1891
|
self.js_only = js_only
|
|
1772
1892
|
self.ignore_body_visibility = ignore_body_visibility
|
|
1893
|
+
if (
|
|
1894
|
+
not isinstance(body_visibility_timeout, (int, float))
|
|
1895
|
+
or isinstance(body_visibility_timeout, bool)
|
|
1896
|
+
or body_visibility_timeout <= 0
|
|
1897
|
+
):
|
|
1898
|
+
raise ValueError("body_visibility_timeout must be a positive number")
|
|
1899
|
+
self.body_visibility_timeout = body_visibility_timeout
|
|
1773
1900
|
self.scan_full_page = scan_full_page
|
|
1774
1901
|
self.scroll_delay = scroll_delay
|
|
1775
1902
|
self.max_scroll_steps = max_scroll_steps
|
|
@@ -2057,12 +2184,14 @@ class CrawlerRunConfig():
|
|
|
2057
2184
|
super().__setattr__(name, value)
|
|
2058
2185
|
|
|
2059
2186
|
@staticmethod
|
|
2060
|
-
def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
|
|
2187
|
+
def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "CrawlerRunConfig":
|
|
2061
2188
|
# Auto-deserialize any dict values that use the {"type": ..., "params": ...}
|
|
2062
2189
|
# serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
|
|
2063
2190
|
# This covers markdown_generator, extraction_strategy, content_filter, etc.
|
|
2064
2191
|
kwargs = {
|
|
2065
|
-
k: from_serializable_dict(v
|
|
2192
|
+
k: from_serializable_dict(v, provenance)
|
|
2193
|
+
if isinstance(v, dict) and "type" in v
|
|
2194
|
+
else v
|
|
2066
2195
|
for k, v in kwargs.items()
|
|
2067
2196
|
}
|
|
2068
2197
|
# Only pass keys present in kwargs so that __init__ defaults (and
|
|
@@ -2085,7 +2214,7 @@ class CrawlerRunConfig():
|
|
|
2085
2214
|
return config
|
|
2086
2215
|
if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
|
|
2087
2216
|
config = _enforce_untrusted("CrawlerRunConfig", config)
|
|
2088
|
-
return CrawlerRunConfig.from_kwargs(config)
|
|
2217
|
+
return CrawlerRunConfig.from_kwargs(config, provenance)
|
|
2089
2218
|
|
|
2090
2219
|
def to_dict(self):
|
|
2091
2220
|
return {
|
|
@@ -2137,6 +2266,7 @@ class CrawlerRunConfig():
|
|
|
2137
2266
|
"js_code_before_wait": self.js_code_before_wait,
|
|
2138
2267
|
"js_only": self.js_only,
|
|
2139
2268
|
"ignore_body_visibility": self.ignore_body_visibility,
|
|
2269
|
+
"body_visibility_timeout": self.body_visibility_timeout,
|
|
2140
2270
|
"scan_full_page": self.scan_full_page,
|
|
2141
2271
|
"scroll_delay": self.scroll_delay,
|
|
2142
2272
|
"max_scroll_steps": self.max_scroll_steps,
|
|
@@ -812,6 +812,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
|
|
|
812
812
|
await page.wait_for_selector("body", state="attached", timeout=30000)
|
|
813
813
|
|
|
814
814
|
# Use the new check_visibility function with csp_compliant_wait
|
|
815
|
+
_visibility_wait_start = time.perf_counter()
|
|
815
816
|
is_visible = await self.csp_compliant_wait(
|
|
816
817
|
page,
|
|
817
818
|
"""() => {
|
|
@@ -823,9 +824,38 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
|
|
|
823
824
|
style.opacity !== '0';
|
|
824
825
|
return isVisible;
|
|
825
826
|
}""",
|
|
826
|
-
timeout=
|
|
827
|
+
timeout=config.body_visibility_timeout,
|
|
827
828
|
)
|
|
828
829
|
|
|
830
|
+
_visibility_wait_ms = (
|
|
831
|
+
time.perf_counter() - _visibility_wait_start
|
|
832
|
+
) * 1000
|
|
833
|
+
|
|
834
|
+
# csp_compliant_wait also returns False when the evaluation itself
|
|
835
|
+
# fails (page closed, context destroyed by a redirect), which costs
|
|
836
|
+
# no time — only warn about a wait that actually burned its budget.
|
|
837
|
+
if (
|
|
838
|
+
not is_visible
|
|
839
|
+
and config.ignore_body_visibility
|
|
840
|
+
and _visibility_wait_ms >= config.body_visibility_timeout * 0.9
|
|
841
|
+
):
|
|
842
|
+
# The wait timed out and its result is about to be discarded, so
|
|
843
|
+
# the crawl still succeeds — just this much slower, on every crawl
|
|
844
|
+
# of this page. force_verbose because the whole point is that the
|
|
845
|
+
# delay is otherwise invisible, and the servers and batch jobs that
|
|
846
|
+
# most need to see it run with verbose off (see #2144).
|
|
847
|
+
self.logger.warning(
|
|
848
|
+
message=(
|
|
849
|
+
"Body never became visible after {elapsed}ms — the page may "
|
|
850
|
+
"use ng-cloak/v-cloak. This delay is added to every crawl of "
|
|
851
|
+
"this page; lower CrawlerRunConfig.body_visibility_timeout "
|
|
852
|
+
"to shorten it."
|
|
853
|
+
),
|
|
854
|
+
tag="WARNING",
|
|
855
|
+
params={"elapsed": round(_visibility_wait_ms)},
|
|
856
|
+
force_verbose=True,
|
|
857
|
+
)
|
|
858
|
+
|
|
829
859
|
if not is_visible and not config.ignore_body_visibility:
|
|
830
860
|
visibility_info = await self.check_visibility(page)
|
|
831
861
|
raise Error(f"Body element is hidden: {visibility_info}")
|
|
@@ -1534,7 +1564,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
|
|
|
1534
1564
|
}})()
|
|
1535
1565
|
"""
|
|
1536
1566
|
)
|
|
1537
|
-
await page.wait_for_timeout(
|
|
1567
|
+
await page.wait_for_timeout(600) # Wait for any animations to complete
|
|
1538
1568
|
except Exception as e:
|
|
1539
1569
|
self.logger.warning(
|
|
1540
1570
|
message="Failed to remove overlay elements: {error}",
|