Crawl4AI 0.9.3__tar.gz → 0.9.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/PKG-INFO +24 -3
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/SOURCES.txt +3 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/PKG-INFO +24 -3
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/README.md +23 -2
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/__init__.py +2 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/__version__.py +1 -1
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_configs.py +99 -17
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_url_seeder.py +7 -2
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/browser_manager.py +8 -4
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/cli.py +2 -1
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/content_filter_strategy.py +28 -0
- crawl4ai-0.9.4/crawl4ai/content_filter_strategy_lxml.py +391 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/bff_strategy.py +7 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/bfs_strategy.py +6 -2
- crawl4ai-0.9.4/crawl4ai/egress_policy.py +35 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/table_extraction.py +58 -11
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/types.py +4 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/utils.py +67 -24
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_config_defaults.py +111 -1
- crawl4ai-0.9.4/tests/test_table_rowspan_grid.py +162 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/dependency_links.txt +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/entry_points.txt +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/requires.txt +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/top_level.txt +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/LICENSE +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/MANIFEST.in +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/adaptive_crawler.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/antibot_detector.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_crawler_strategy.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_database.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_dispatcher.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_logger.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_webcrawler.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/browser_adapter.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/browser_profiler.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/cache_context.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/cache_validator.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/chunking_strategy.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/cloud/__init__.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/cloud/cli.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/components/crawler_monitor.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/config.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/content_scraping_strategy.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/__init__.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/amazon_product/__init__.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/amazon_product/crawler.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/__init__.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/crawler.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/script.js +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/__init__.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/base_strategy.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/crazy.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/dfs_strategy.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/filters.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/scorers.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/docker_client.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/domain_mapper.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/extraction_strategy.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/__init__.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/__main__.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/_typing.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/cli.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/config.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/elements.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/utils.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/hub.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/install.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/__init__.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/flatten_shadow_dom.js +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/navigator_overrider.js +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/remove_consent_popups.js +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/remove_overlay_elements.js +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/update_image_dimensions.js +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/__init__.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/cli.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/crawler_strategy.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/database.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/docs_manager.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/llmtxt.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/version_manager.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/web_crawler.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/link_preview.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/markdown_generation_strategy.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/migrations.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/model_loader.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/models.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/processors/pdf/__init__.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/processors/pdf/processor.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/processors/pdf/utils.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/prompts.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/proxy_strategy.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/script/__init__.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/script/c4a_compile.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/script/c4a_result.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/script/c4ai_script.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/ssl_certificate.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/user_agent_generator.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/pyproject.toml +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/requirements.txt +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/setup.cfg +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/setup.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_arun_many.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_async_logger_stderr.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_body_visibility_warning.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_bug_batch_1622_1786_1796.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_cdp_changes.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_cli_docs.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_cloud_bugs_batch.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_config_matching_only.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_config_selection.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_docker.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_docker_api_with_llm_provider.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_docker_pdf_crawler_pairing.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_eval_security_adversarial.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_http_timeout_unit_1894.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1043_mermaid_svg.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1213_bm25_dedup.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1370_1818_1762_1509.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1484_css_selector.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1594_mcp_sse.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1611_llm_provider.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1748_screenshot_scroll_delay.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1750_screenshot_scan_full_page.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1837_config_list.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1842_browser_none.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1848_logger_serialize.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1850_mcp_sse.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_2127_docker_pdf.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_2161_overlay_html_body.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_link_extractor.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_llm_extraction_parallel_issue_1055.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_llm_simple_url.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_llmtxt.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_main.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_markdown_generator_validation_1880.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_memory_macos.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_merge_head_data_scoring.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_multi_config.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_normalize_url.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_placeholder_html_antibot.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pr_1290_1668.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pr_1435_redirected_status_code.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pr_1463_device_scale_factor.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pr_1795_1798_1734.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_prefetch_integration.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_prefetch_mode.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_prefetch_regression.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_preserve_https_for_internal_links.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pruning_preserve_whitelist_1900.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pyopenssl_security_fix.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pyopenssl_update.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_raw_html_browser.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_raw_html_edge_cases.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_raw_html_redirected_url.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_scraping_strategy.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_source_sibling_selector.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_table_gfm_compliance.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_type_annotations.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_virtual_scroll.py +0 -0
- {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_web_crawler.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: Crawl4AI
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.4
|
|
4
4
|
Summary: 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper
|
|
5
5
|
Home-page: https://github.com/unclecode/crawl4ai
|
|
6
6
|
Author: Unclecode
|
|
@@ -120,9 +120,11 @@ Limited slots._
|
|
|
120
120
|
|
|
121
121
|
Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
|
|
122
122
|
|
|
123
|
-
[✨ Check out latest update v0.9.
|
|
123
|
+
[✨ Check out latest update v0.9.4](#-recent-updates)
|
|
124
124
|
|
|
125
|
-
✨ **New in v0.9.
|
|
125
|
+
✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
126
|
+
|
|
127
|
+
✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
|
|
126
128
|
|
|
127
129
|
✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
|
|
128
130
|
|
|
@@ -650,10 +652,29 @@ async def test_news_crawl():
|
|
|
650
652
|
## ✨ Recent Updates
|
|
651
653
|
|
|
652
654
|
<details open>
|
|
655
|
+
<summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
|
|
656
|
+
|
|
657
|
+
A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
|
|
658
|
+
|
|
659
|
+
It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
|
|
660
|
+
|
|
661
|
+
No breaking changes.
|
|
662
|
+
|
|
663
|
+
```bash
|
|
664
|
+
pip install -U crawl4ai
|
|
665
|
+
```
|
|
666
|
+
|
|
667
|
+
[Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
668
|
+
|
|
669
|
+
</details>
|
|
670
|
+
|
|
671
|
+
<details>
|
|
653
672
|
<summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
|
|
654
673
|
|
|
655
674
|
A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
|
|
656
675
|
|
|
676
|
+
It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
|
|
677
|
+
|
|
657
678
|
No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
|
|
658
679
|
|
|
659
680
|
```bash
|
|
@@ -31,9 +31,11 @@ crawl4ai/chunking_strategy.py
|
|
|
31
31
|
crawl4ai/cli.py
|
|
32
32
|
crawl4ai/config.py
|
|
33
33
|
crawl4ai/content_filter_strategy.py
|
|
34
|
+
crawl4ai/content_filter_strategy_lxml.py
|
|
34
35
|
crawl4ai/content_scraping_strategy.py
|
|
35
36
|
crawl4ai/docker_client.py
|
|
36
37
|
crawl4ai/domain_mapper.py
|
|
38
|
+
crawl4ai/egress_policy.py
|
|
37
39
|
crawl4ai/extraction_strategy.py
|
|
38
40
|
crawl4ai/hub.py
|
|
39
41
|
crawl4ai/install.py
|
|
@@ -151,6 +153,7 @@ tests/test_raw_html_redirected_url.py
|
|
|
151
153
|
tests/test_scraping_strategy.py
|
|
152
154
|
tests/test_source_sibling_selector.py
|
|
153
155
|
tests/test_table_gfm_compliance.py
|
|
156
|
+
tests/test_table_rowspan_grid.py
|
|
154
157
|
tests/test_type_annotations.py
|
|
155
158
|
tests/test_virtual_scroll.py
|
|
156
159
|
tests/test_web_crawler.py
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: Crawl4AI
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.4
|
|
4
4
|
Summary: 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper
|
|
5
5
|
Home-page: https://github.com/unclecode/crawl4ai
|
|
6
6
|
Author: Unclecode
|
|
@@ -120,9 +120,11 @@ Limited slots._
|
|
|
120
120
|
|
|
121
121
|
Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
|
|
122
122
|
|
|
123
|
-
[✨ Check out latest update v0.9.
|
|
123
|
+
[✨ Check out latest update v0.9.4](#-recent-updates)
|
|
124
124
|
|
|
125
|
-
✨ **New in v0.9.
|
|
125
|
+
✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
126
|
+
|
|
127
|
+
✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
|
|
126
128
|
|
|
127
129
|
✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
|
|
128
130
|
|
|
@@ -650,10 +652,29 @@ async def test_news_crawl():
|
|
|
650
652
|
## ✨ Recent Updates
|
|
651
653
|
|
|
652
654
|
<details open>
|
|
655
|
+
<summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
|
|
656
|
+
|
|
657
|
+
A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
|
|
658
|
+
|
|
659
|
+
It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
|
|
660
|
+
|
|
661
|
+
No breaking changes.
|
|
662
|
+
|
|
663
|
+
```bash
|
|
664
|
+
pip install -U crawl4ai
|
|
665
|
+
```
|
|
666
|
+
|
|
667
|
+
[Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
668
|
+
|
|
669
|
+
</details>
|
|
670
|
+
|
|
671
|
+
<details>
|
|
653
672
|
<summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
|
|
654
673
|
|
|
655
674
|
A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
|
|
656
675
|
|
|
676
|
+
It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
|
|
677
|
+
|
|
657
678
|
No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
|
|
658
679
|
|
|
659
680
|
```bash
|
|
@@ -37,9 +37,11 @@ Limited slots._
|
|
|
37
37
|
|
|
38
38
|
Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
|
|
39
39
|
|
|
40
|
-
[✨ Check out latest update v0.9.
|
|
40
|
+
[✨ Check out latest update v0.9.4](#-recent-updates)
|
|
41
41
|
|
|
42
|
-
✨ **New in v0.9.
|
|
42
|
+
✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
43
|
+
|
|
44
|
+
✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
|
|
43
45
|
|
|
44
46
|
✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
|
|
45
47
|
|
|
@@ -567,10 +569,29 @@ async def test_news_crawl():
|
|
|
567
569
|
## ✨ Recent Updates
|
|
568
570
|
|
|
569
571
|
<details open>
|
|
572
|
+
<summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
|
|
573
|
+
|
|
574
|
+
A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
|
|
575
|
+
|
|
576
|
+
It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
|
|
577
|
+
|
|
578
|
+
No breaking changes.
|
|
579
|
+
|
|
580
|
+
```bash
|
|
581
|
+
pip install -U crawl4ai
|
|
582
|
+
```
|
|
583
|
+
|
|
584
|
+
[Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
|
|
585
|
+
|
|
586
|
+
</details>
|
|
587
|
+
|
|
588
|
+
<details>
|
|
570
589
|
<summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
|
|
571
590
|
|
|
572
591
|
A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
|
|
573
592
|
|
|
593
|
+
It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
|
|
594
|
+
|
|
574
595
|
No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
|
|
575
596
|
|
|
576
597
|
```bash
|
|
@@ -42,6 +42,7 @@ from .content_filter_strategy import (
|
|
|
42
42
|
LLMContentFilter,
|
|
43
43
|
RelevantContentFilter,
|
|
44
44
|
)
|
|
45
|
+
from .content_filter_strategy_lxml import PruningContentFilterLXML
|
|
45
46
|
from .models import CrawlResult, MarkdownGenerationResult, DisplayMode
|
|
46
47
|
from .components.crawler_monitor import CrawlerMonitor
|
|
47
48
|
from .link_preview import LinkPreview
|
|
@@ -176,6 +177,7 @@ __all__ = [
|
|
|
176
177
|
"LLMTableExtraction",
|
|
177
178
|
"RelevantContentFilter",
|
|
178
179
|
"PruningContentFilter",
|
|
180
|
+
"PruningContentFilterLXML",
|
|
179
181
|
"BM25ContentFilter",
|
|
180
182
|
"LLMContentFilter",
|
|
181
183
|
"BaseDispatcher",
|
|
@@ -133,7 +133,7 @@ ALLOWED_DESERIALIZE_TYPES = {
|
|
|
133
133
|
"CosineStrategy", "RegexExtractionStrategy",
|
|
134
134
|
# Markdown / content
|
|
135
135
|
"DefaultMarkdownGenerator",
|
|
136
|
-
"PruningContentFilter", "BM25ContentFilter", "LLMContentFilter",
|
|
136
|
+
"PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter", "LLMContentFilter",
|
|
137
137
|
# Scraping
|
|
138
138
|
"LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
|
|
139
139
|
# Chunking
|
|
@@ -190,7 +190,7 @@ UNTRUSTED_ALLOWED_TYPES = {
|
|
|
190
190
|
# non-LLM extraction / markdown / scraping / chunking strategies
|
|
191
191
|
"JsonCssExtractionStrategy", "JsonXPathExtractionStrategy",
|
|
192
192
|
"JsonLxmlExtractionStrategy", "RegexExtractionStrategy", "CosineStrategy",
|
|
193
|
-
"DefaultMarkdownGenerator", "PruningContentFilter", "BM25ContentFilter",
|
|
193
|
+
"DefaultMarkdownGenerator", "PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter",
|
|
194
194
|
"LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
|
|
195
195
|
"RegexChunking",
|
|
196
196
|
"DefaultTableExtraction", "NoTableExtraction",
|
|
@@ -293,11 +293,49 @@ UNTRUSTED_FIELD_ALLOWLIST = {
|
|
|
293
293
|
}
|
|
294
294
|
|
|
295
295
|
# Upper bounds applied to attacker-influenced quantities after filtering.
|
|
296
|
-
|
|
296
|
+
_DEFAULT_MAX_TIMEOUT_MS = 60_000
|
|
297
297
|
_MAX_SCROLL_STEPS = 1000
|
|
298
298
|
_MAX_VIEWPORT = 4000
|
|
299
299
|
_MAX_PDF_BYTES = 100 * 1024 * 1024
|
|
300
300
|
_MAX_PDF_PAGES = 2000
|
|
301
|
+
_MAX_LINK_PREVIEW_LINKS = 100 # the class default
|
|
302
|
+
_MAX_LINK_PREVIEW_CONCURRENCY = 10 # the class default
|
|
303
|
+
_MAX_LINK_PREVIEW_TIMEOUT_S = 10 # seconds here, not ms
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _max_timeout_ms() -> int:
|
|
307
|
+
"""Ceiling for the untrusted timeout fields, in milliseconds.
|
|
308
|
+
|
|
309
|
+
60s is the right bound for a server reachable by untrusted callers, and
|
|
310
|
+
stays the default. An operator whose deployment is not public — a crawler
|
|
311
|
+
on a private network fetching pages that legitimately take minutes — can
|
|
312
|
+
raise it with CRAWL4AI_MAX_TIMEOUT_MS, or lower it to tighten the bound.
|
|
313
|
+
|
|
314
|
+
Read per call rather than captured at import so the setting applies
|
|
315
|
+
wherever the process picked its environment up, and so a test can set it
|
|
316
|
+
without reloading the module. A value that is not a positive integer is
|
|
317
|
+
refused loudly and the default kept: a typo here would silently widen a
|
|
318
|
+
DoS bound, which is the one outcome worse than the timeout being fixed.
|
|
319
|
+
"""
|
|
320
|
+
raw = os.getenv("CRAWL4AI_MAX_TIMEOUT_MS")
|
|
321
|
+
|
|
322
|
+
if raw is None or raw == "":
|
|
323
|
+
return _DEFAULT_MAX_TIMEOUT_MS
|
|
324
|
+
|
|
325
|
+
try:
|
|
326
|
+
ceiling = int(raw)
|
|
327
|
+
except ValueError:
|
|
328
|
+
ceiling = 0
|
|
329
|
+
|
|
330
|
+
if ceiling <= 0:
|
|
331
|
+
warnings.warn(
|
|
332
|
+
f"CRAWL4AI_MAX_TIMEOUT_MS={raw!r} is not a positive integer; "
|
|
333
|
+
f"keeping the {_DEFAULT_MAX_TIMEOUT_MS}ms default.",
|
|
334
|
+
stacklevel=2,
|
|
335
|
+
)
|
|
336
|
+
return _DEFAULT_MAX_TIMEOUT_MS
|
|
337
|
+
|
|
338
|
+
return ceiling
|
|
301
339
|
|
|
302
340
|
|
|
303
341
|
def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
|
|
@@ -318,11 +356,15 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
|
|
|
318
356
|
|
|
319
357
|
def _clamp_untrusted(type_name: str, params: dict) -> dict:
|
|
320
358
|
"""Clamp attacker-influenced quantities to safe upper bounds."""
|
|
359
|
+
ceiling = _max_timeout_ms()
|
|
360
|
+
|
|
321
361
|
def _cap_timeout(v):
|
|
322
362
|
# 0 historically meant "no timeout"; treat as the cap, never unbounded.
|
|
363
|
+
# Malformed input is the one value an attacker gets for free, so it
|
|
364
|
+
# falls back to the 60s default rather than to a raised ceiling.
|
|
323
365
|
if not isinstance(v, (int, float)) or v <= 0:
|
|
324
|
-
return
|
|
325
|
-
return min(int(v),
|
|
366
|
+
return min(_DEFAULT_MAX_TIMEOUT_MS, ceiling)
|
|
367
|
+
return min(int(v), ceiling)
|
|
326
368
|
|
|
327
369
|
if type_name == "CrawlerRunConfig":
|
|
328
370
|
for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):
|
|
@@ -345,6 +387,18 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:
|
|
|
345
387
|
# Rasterizing every page is the most expensive thing this strategy can
|
|
346
388
|
# do, and nothing about untrusted crawling needs it.
|
|
347
389
|
params["extract_images"] = False
|
|
390
|
+
elif type_name == "LinkPreviewConfig":
|
|
391
|
+
# Also no field allowlist, and each link is a separate outbound fetch:
|
|
392
|
+
# unclamped, one request body can order millions of them at any
|
|
393
|
+
# concurrency it likes.
|
|
394
|
+
for f, cap in (
|
|
395
|
+
("max_links", _MAX_LINK_PREVIEW_LINKS),
|
|
396
|
+
("concurrency", _MAX_LINK_PREVIEW_CONCURRENCY),
|
|
397
|
+
("timeout", _MAX_LINK_PREVIEW_TIMEOUT_S),
|
|
398
|
+
):
|
|
399
|
+
if f in params:
|
|
400
|
+
v = params[f]
|
|
401
|
+
params[f] = cap if not isinstance(v, int) or v <= 0 else min(v, cap)
|
|
348
402
|
return params
|
|
349
403
|
|
|
350
404
|
|
|
@@ -434,6 +488,21 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False):
|
|
|
434
488
|
return str(obj)
|
|
435
489
|
|
|
436
490
|
|
|
491
|
+
def _is_typed_shape(data: Any) -> bool:
|
|
492
|
+
"""True for the dict shapes to_serializable_dict() emits:
|
|
493
|
+
{"type": "<ClassName>", "params": {...}} or {"type": "dict", "value": {...}}.
|
|
494
|
+
|
|
495
|
+
Plain business dicts that happen to carry a "type" key (JSON-Schema
|
|
496
|
+
fragments, JsonCss field specs like {"type": "text", "name": "..."}) have
|
|
497
|
+
neither "params" nor "value" and are not typed shapes.
|
|
498
|
+
"""
|
|
499
|
+
return (
|
|
500
|
+
isinstance(data, dict)
|
|
501
|
+
and "type" in data
|
|
502
|
+
and ("params" in data or (data["type"] == "dict" and "value" in data))
|
|
503
|
+
)
|
|
504
|
+
|
|
505
|
+
|
|
437
506
|
def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
|
|
438
507
|
"""
|
|
439
508
|
Recursively convert a serializable dictionary back to an object instance.
|
|
@@ -459,14 +528,23 @@ def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
|
|
|
459
528
|
# carry a "type" key (e.g. JSON-Schema fragments, JsonCss field specs like
|
|
460
529
|
# {"type": "text", "name": "..."}) have neither "params" nor "value" and
|
|
461
530
|
# must fall through to the raw-dict path below so they are passed as data.
|
|
462
|
-
if (
|
|
463
|
-
isinstance(data, dict)
|
|
464
|
-
and "type" in data
|
|
465
|
-
and ("params" in data or (data["type"] == "dict" and "value" in data))
|
|
466
|
-
):
|
|
531
|
+
if _is_typed_shape(data):
|
|
467
532
|
# Handle plain dictionaries
|
|
468
533
|
if data["type"] == "dict" and "value" in data:
|
|
469
|
-
|
|
534
|
+
unwrapped = {
|
|
535
|
+
k: from_serializable_dict(v, provenance) for k, v in data["value"].items()
|
|
536
|
+
}
|
|
537
|
+
# The unwrap above recurses over .items(), so data["value"] is never
|
|
538
|
+
# seen as a whole typed object and the type gate below never fires
|
|
539
|
+
# for it. Without this check an untrusted body launders a forbidden
|
|
540
|
+
# type past the gate by wrapping it:
|
|
541
|
+
# {"type": "dict", "value": {"type": "LLMConfig", "params": {...}}}
|
|
542
|
+
if provenance == Provenance.UNTRUSTED and _is_typed_shape(unwrapped):
|
|
543
|
+
raise UntrustedConfigError(
|
|
544
|
+
"an untrusted request may not wrap a typed object in "
|
|
545
|
+
"{'type': 'dict', 'value': ...}"
|
|
546
|
+
)
|
|
547
|
+
return unwrapped
|
|
470
548
|
|
|
471
549
|
# Security: only allow known-safe types to be deserialized.
|
|
472
550
|
# Unknown types (e.g. logging.Logger serialized by older clients) are
|
|
@@ -973,11 +1051,13 @@ class BrowserConfig:
|
|
|
973
1051
|
)
|
|
974
1052
|
|
|
975
1053
|
@staticmethod
|
|
976
|
-
def from_kwargs(kwargs: dict) -> "BrowserConfig":
|
|
1054
|
+
def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "BrowserConfig":
|
|
977
1055
|
# Auto-deserialize any dict values that use the {"type": ..., "params": ...}
|
|
978
1056
|
# serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
|
|
979
1057
|
kwargs = {
|
|
980
|
-
k: from_serializable_dict(v
|
|
1058
|
+
k: from_serializable_dict(v, provenance)
|
|
1059
|
+
if isinstance(v, dict) and "type" in v
|
|
1060
|
+
else v
|
|
981
1061
|
for k, v in kwargs.items()
|
|
982
1062
|
}
|
|
983
1063
|
# Only pass keys present in kwargs so that __init__ defaults (and
|
|
@@ -1063,7 +1143,7 @@ class BrowserConfig:
|
|
|
1063
1143
|
# so a body that sends bare kwargs (no {type,params}) cannot bypass it.
|
|
1064
1144
|
if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
|
|
1065
1145
|
config = _enforce_untrusted("BrowserConfig", config)
|
|
1066
|
-
return BrowserConfig.from_kwargs(config)
|
|
1146
|
+
return BrowserConfig.from_kwargs(config, provenance)
|
|
1067
1147
|
|
|
1068
1148
|
def set_nstproxy(
|
|
1069
1149
|
self,
|
|
@@ -2104,12 +2184,14 @@ class CrawlerRunConfig():
|
|
|
2104
2184
|
super().__setattr__(name, value)
|
|
2105
2185
|
|
|
2106
2186
|
@staticmethod
|
|
2107
|
-
def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
|
|
2187
|
+
def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "CrawlerRunConfig":
|
|
2108
2188
|
# Auto-deserialize any dict values that use the {"type": ..., "params": ...}
|
|
2109
2189
|
# serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
|
|
2110
2190
|
# This covers markdown_generator, extraction_strategy, content_filter, etc.
|
|
2111
2191
|
kwargs = {
|
|
2112
|
-
k: from_serializable_dict(v
|
|
2192
|
+
k: from_serializable_dict(v, provenance)
|
|
2193
|
+
if isinstance(v, dict) and "type" in v
|
|
2194
|
+
else v
|
|
2113
2195
|
for k, v in kwargs.items()
|
|
2114
2196
|
}
|
|
2115
2197
|
# Only pass keys present in kwargs so that __init__ defaults (and
|
|
@@ -2132,7 +2214,7 @@ class CrawlerRunConfig():
|
|
|
2132
2214
|
return config
|
|
2133
2215
|
if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
|
|
2134
2216
|
config = _enforce_untrusted("CrawlerRunConfig", config)
|
|
2135
|
-
return CrawlerRunConfig.from_kwargs(config)
|
|
2217
|
+
return CrawlerRunConfig.from_kwargs(config, provenance)
|
|
2136
2218
|
|
|
2137
2219
|
def to_dict(self):
|
|
2138
2220
|
return {
|
|
@@ -53,6 +53,7 @@ except ImportError:
|
|
|
53
53
|
# You might need to adjust this import based on your exact file structure
|
|
54
54
|
# Import AsyncLogger for default if needed
|
|
55
55
|
from .async_logger import AsyncLoggerBase, AsyncLogger
|
|
56
|
+
from .egress_policy import proxy_url
|
|
56
57
|
|
|
57
58
|
# Import SeedingConfig for type hints
|
|
58
59
|
from typing import TYPE_CHECKING
|
|
@@ -303,7 +304,11 @@ class AsyncUrlSeeder:
|
|
|
303
304
|
):
|
|
304
305
|
self.ttl = ttl
|
|
305
306
|
self._owns_client = client is None # Track if we created the client
|
|
306
|
-
|
|
307
|
+
# proxy=None unless an embedder installed one (the Docker server does).
|
|
308
|
+
# Every request in this class goes through self.client, so this single
|
|
309
|
+
# kwarg puts the seeder's whole fan-out -- sitemaps, robots, link heads,
|
|
310
|
+
# and each redirect hop -- behind the egress policy.
|
|
311
|
+
self.client = client or httpx.AsyncClient(http2=True, timeout=20, proxy=proxy_url(), headers={
|
|
307
312
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) +AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
|
|
308
313
|
})
|
|
309
314
|
self.logger = logger # Store the logger instance
|
|
@@ -1772,7 +1777,7 @@ class AsyncUrlSeeder:
|
|
|
1772
1777
|
self._log("info", "Fetching latest Common Crawl index from {url}",
|
|
1773
1778
|
params={"url": COLLINFO_URL}, tag="URL_SEED")
|
|
1774
1779
|
try:
|
|
1775
|
-
async with httpx.AsyncClient() as c:
|
|
1780
|
+
async with httpx.AsyncClient(proxy=proxy_url()) as c:
|
|
1776
1781
|
j = await c.get(COLLINFO_URL, timeout=10)
|
|
1777
1782
|
j.raise_for_status() # Raise an exception for bad status codes
|
|
1778
1783
|
idx = j.json()[0]["id"]
|
|
@@ -85,8 +85,10 @@ class ManagedBrowser:
|
|
|
85
85
|
"--force-color-profile=srgb",
|
|
86
86
|
"--mute-audio",
|
|
87
87
|
"--disable-background-timer-throttling",
|
|
88
|
-
# Memory-saving flags: disable unused Chrome features
|
|
89
|
-
|
|
88
|
+
# Memory-saving flags: disable unused Chrome features.
|
|
89
|
+
# Do not disable OptimizationHints — it SEGV_ACCERRs Chrome-for-Testing
|
|
90
|
+
# under --headless=new on macOS arm64 (issue #2239).
|
|
91
|
+
"--disable-features=MediaRouter,DialMediaRouteProvider",
|
|
90
92
|
"--disable-component-update",
|
|
91
93
|
"--disable-domain-reliability",
|
|
92
94
|
]
|
|
@@ -1102,8 +1104,10 @@ class BrowserManager:
|
|
|
1102
1104
|
"--force-color-profile=srgb",
|
|
1103
1105
|
"--mute-audio",
|
|
1104
1106
|
"--disable-background-timer-throttling",
|
|
1105
|
-
# Memory-saving flags: disable unused Chrome features
|
|
1106
|
-
|
|
1107
|
+
# Memory-saving flags: disable unused Chrome features.
|
|
1108
|
+
# Do not disable OptimizationHints — it SEGV_ACCERRs Chrome-for-Testing
|
|
1109
|
+
# under --headless=new on macOS arm64 (issue #2239).
|
|
1110
|
+
"--disable-features=MediaRouter,DialMediaRouteProvider",
|
|
1107
1111
|
"--disable-component-update",
|
|
1108
1112
|
"--disable-domain-reliability",
|
|
1109
1113
|
# "--single-process",
|
|
@@ -25,6 +25,7 @@ from crawl4ai import (
|
|
|
25
25
|
JsonXPathExtractionStrategy,
|
|
26
26
|
BM25ContentFilter,
|
|
27
27
|
PruningContentFilter,
|
|
28
|
+
PruningContentFilterLXML,
|
|
28
29
|
BrowserProfiler,
|
|
29
30
|
DefaultMarkdownGenerator,
|
|
30
31
|
LLMConfig,
|
|
@@ -1094,7 +1095,7 @@ def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config:
|
|
|
1094
1095
|
)
|
|
1095
1096
|
elif filter_conf["type"] == "pruning":
|
|
1096
1097
|
crawler_cfg.markdown_generator = DefaultMarkdownGenerator(
|
|
1097
|
-
content_filter =
|
|
1098
|
+
content_filter = PruningContentFilterLXML(
|
|
1098
1099
|
user_query=filter_conf.get("query"),
|
|
1099
1100
|
threshold=filter_conf.get("threshold", 0.48)
|
|
1100
1101
|
)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
import re
|
|
3
3
|
import time
|
|
4
|
+
import warnings
|
|
4
5
|
from bs4 import BeautifulSoup, Tag
|
|
5
6
|
from typing import List, Tuple, Dict, Optional
|
|
6
7
|
from rank_bm25 import BM25Okapi
|
|
@@ -584,6 +585,19 @@ class PruningContentFilter(RelevantContentFilter):
|
|
|
584
585
|
preserve_tags (list): HTML tag names to always keep regardless of score (optional).
|
|
585
586
|
"""
|
|
586
587
|
super().__init__(None)
|
|
588
|
+
# Deprecation: the BeautifulSoup engine is superseded by the lxml one.
|
|
589
|
+
# Only warn for direct use of this class, not when PruningContentFilterLXML
|
|
590
|
+
# (a subclass) calls super().__init__.
|
|
591
|
+
if type(self) is PruningContentFilter:
|
|
592
|
+
warnings.warn(
|
|
593
|
+
"PruningContentFilter (BeautifulSoup-based) is deprecated in favor of "
|
|
594
|
+
"PruningContentFilterLXML, an lxml reimplementation that is ~10x faster "
|
|
595
|
+
"with identical output. Switch to PruningContentFilterLXML now. In an "
|
|
596
|
+
"upcoming release PruningContentFilter will become an alias for the lxml "
|
|
597
|
+
"implementation, and PruningContentFilterLXML will be kept as a legacy alias.",
|
|
598
|
+
DeprecationWarning,
|
|
599
|
+
stacklevel=2,
|
|
600
|
+
)
|
|
587
601
|
self.min_word_threshold = min_word_threshold
|
|
588
602
|
self.threshold_type = threshold_type
|
|
589
603
|
self.threshold = threshold
|
|
@@ -1108,3 +1122,17 @@ class LLMContentFilter(RelevantContentFilter):
|
|
|
1108
1122
|
f"{i:<10} {usage.completion_tokens:>12,} "
|
|
1109
1123
|
f"{usage.prompt_tokens:>12,} {usage.total_tokens:>12,}"
|
|
1110
1124
|
)
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
def __getattr__(name):
|
|
1128
|
+
"""Lazily re-export PruningContentFilterLXML from this module.
|
|
1129
|
+
|
|
1130
|
+
The lxml engine lives in ``content_filter_strategy_lxml`` (which imports
|
|
1131
|
+
from this module). Exposing it here via PEP 562 lazy attribute access lets
|
|
1132
|
+
``from crawl4ai.content_filter_strategy import PruningContentFilterLXML``
|
|
1133
|
+
work without creating an import-time circular dependency.
|
|
1134
|
+
"""
|
|
1135
|
+
if name == "PruningContentFilterLXML":
|
|
1136
|
+
from .content_filter_strategy_lxml import PruningContentFilterLXML
|
|
1137
|
+
return PruningContentFilterLXML
|
|
1138
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|