Crawl4AI 0.9.3__tar.gz → 0.9.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/PKG-INFO +24 -3
  2. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/SOURCES.txt +3 -0
  3. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/PKG-INFO +24 -3
  4. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/README.md +23 -2
  5. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/__init__.py +2 -0
  6. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/__version__.py +1 -1
  7. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_configs.py +99 -17
  8. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_url_seeder.py +7 -2
  9. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/browser_manager.py +8 -4
  10. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/cli.py +2 -1
  11. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/content_filter_strategy.py +28 -0
  12. crawl4ai-0.9.4/crawl4ai/content_filter_strategy_lxml.py +391 -0
  13. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/bff_strategy.py +7 -0
  14. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/bfs_strategy.py +6 -2
  15. crawl4ai-0.9.4/crawl4ai/egress_policy.py +35 -0
  16. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/table_extraction.py +58 -11
  17. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/types.py +4 -0
  18. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/utils.py +67 -24
  19. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_config_defaults.py +111 -1
  20. crawl4ai-0.9.4/tests/test_table_rowspan_grid.py +162 -0
  21. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/dependency_links.txt +0 -0
  22. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/entry_points.txt +0 -0
  23. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/requires.txt +0 -0
  24. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/Crawl4AI.egg-info/top_level.txt +0 -0
  25. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/LICENSE +0 -0
  26. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/MANIFEST.in +0 -0
  27. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/adaptive_crawler.py +0 -0
  28. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/antibot_detector.py +0 -0
  29. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_crawler_strategy.py +0 -0
  30. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_database.py +0 -0
  31. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_dispatcher.py +0 -0
  32. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_logger.py +0 -0
  33. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/async_webcrawler.py +0 -0
  34. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/browser_adapter.py +0 -0
  35. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/browser_profiler.py +0 -0
  36. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/cache_context.py +0 -0
  37. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/cache_validator.py +0 -0
  38. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/chunking_strategy.py +0 -0
  39. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/cloud/__init__.py +0 -0
  40. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/cloud/cli.py +0 -0
  41. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/components/crawler_monitor.py +0 -0
  42. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/config.py +0 -0
  43. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/content_scraping_strategy.py +0 -0
  44. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/__init__.py +0 -0
  45. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/amazon_product/__init__.py +0 -0
  46. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/amazon_product/crawler.py +0 -0
  47. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/__init__.py +0 -0
  48. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/crawler.py +0 -0
  49. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/script.js +0 -0
  50. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/__init__.py +0 -0
  51. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/base_strategy.py +0 -0
  52. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/crazy.py +0 -0
  53. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/dfs_strategy.py +0 -0
  54. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/filters.py +0 -0
  55. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/scorers.py +0 -0
  56. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/docker_client.py +0 -0
  57. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/domain_mapper.py +0 -0
  58. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/extraction_strategy.py +0 -0
  59. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/__init__.py +0 -0
  60. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/__main__.py +0 -0
  61. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/_typing.py +0 -0
  62. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/cli.py +0 -0
  63. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/config.py +0 -0
  64. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/elements.py +0 -0
  65. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/html2text/utils.py +0 -0
  66. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/hub.py +0 -0
  67. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/install.py +0 -0
  68. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/__init__.py +0 -0
  69. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/flatten_shadow_dom.js +0 -0
  70. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/navigator_overrider.js +0 -0
  71. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/remove_consent_popups.js +0 -0
  72. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/remove_overlay_elements.js +0 -0
  73. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/js_snippet/update_image_dimensions.js +0 -0
  74. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/__init__.py +0 -0
  75. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/cli.py +0 -0
  76. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/crawler_strategy.py +0 -0
  77. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/database.py +0 -0
  78. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/docs_manager.py +0 -0
  79. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/llmtxt.py +0 -0
  80. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/version_manager.py +0 -0
  81. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/legacy/web_crawler.py +0 -0
  82. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/link_preview.py +0 -0
  83. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/markdown_generation_strategy.py +0 -0
  84. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/migrations.py +0 -0
  85. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/model_loader.py +0 -0
  86. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/models.py +0 -0
  87. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/processors/pdf/__init__.py +0 -0
  88. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/processors/pdf/processor.py +0 -0
  89. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/processors/pdf/utils.py +0 -0
  90. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/prompts.py +0 -0
  91. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/proxy_strategy.py +0 -0
  92. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/script/__init__.py +0 -0
  93. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/script/c4a_compile.py +0 -0
  94. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/script/c4a_result.py +0 -0
  95. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/script/c4ai_script.py +0 -0
  96. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/ssl_certificate.py +0 -0
  97. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/crawl4ai/user_agent_generator.py +0 -0
  98. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/pyproject.toml +0 -0
  99. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/requirements.txt +0 -0
  100. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/setup.cfg +0 -0
  101. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/setup.py +0 -0
  102. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_arun_many.py +0 -0
  103. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_async_logger_stderr.py +0 -0
  104. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_body_visibility_warning.py +0 -0
  105. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_bug_batch_1622_1786_1796.py +0 -0
  106. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_cdp_changes.py +0 -0
  107. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_cli_docs.py +0 -0
  108. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_cloud_bugs_batch.py +0 -0
  109. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_config_matching_only.py +0 -0
  110. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_config_selection.py +0 -0
  111. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_docker.py +0 -0
  112. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_docker_api_with_llm_provider.py +0 -0
  113. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_docker_pdf_crawler_pairing.py +0 -0
  114. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_eval_security_adversarial.py +0 -0
  115. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_http_timeout_unit_1894.py +0 -0
  116. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1043_mermaid_svg.py +0 -0
  117. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1213_bm25_dedup.py +0 -0
  118. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1370_1818_1762_1509.py +0 -0
  119. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1484_css_selector.py +0 -0
  120. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1594_mcp_sse.py +0 -0
  121. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1611_llm_provider.py +0 -0
  122. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1748_screenshot_scroll_delay.py +0 -0
  123. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1750_screenshot_scan_full_page.py +0 -0
  124. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1837_config_list.py +0 -0
  125. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1842_browser_none.py +0 -0
  126. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1848_logger_serialize.py +0 -0
  127. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_1850_mcp_sse.py +0 -0
  128. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_2127_docker_pdf.py +0 -0
  129. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_issue_2161_overlay_html_body.py +0 -0
  130. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_link_extractor.py +0 -0
  131. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_llm_extraction_parallel_issue_1055.py +0 -0
  132. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_llm_simple_url.py +0 -0
  133. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_llmtxt.py +0 -0
  134. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_main.py +0 -0
  135. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_markdown_generator_validation_1880.py +0 -0
  136. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_memory_macos.py +0 -0
  137. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_merge_head_data_scoring.py +0 -0
  138. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_multi_config.py +0 -0
  139. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_normalize_url.py +0 -0
  140. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_placeholder_html_antibot.py +0 -0
  141. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pr_1290_1668.py +0 -0
  142. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pr_1435_redirected_status_code.py +0 -0
  143. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pr_1463_device_scale_factor.py +0 -0
  144. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pr_1795_1798_1734.py +0 -0
  145. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_prefetch_integration.py +0 -0
  146. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_prefetch_mode.py +0 -0
  147. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_prefetch_regression.py +0 -0
  148. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_preserve_https_for_internal_links.py +0 -0
  149. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pruning_preserve_whitelist_1900.py +0 -0
  150. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pyopenssl_security_fix.py +0 -0
  151. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_pyopenssl_update.py +0 -0
  152. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_raw_html_browser.py +0 -0
  153. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_raw_html_edge_cases.py +0 -0
  154. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_raw_html_redirected_url.py +0 -0
  155. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_scraping_strategy.py +0 -0
  156. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_source_sibling_selector.py +0 -0
  157. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_table_gfm_compliance.py +0 -0
  158. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_type_annotations.py +0 -0
  159. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_virtual_scroll.py +0 -0
  160. {crawl4ai-0.9.3 → crawl4ai-0.9.4}/tests/test_web_crawler.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: Crawl4AI
3
- Version: 0.9.3
3
+ Version: 0.9.4
4
4
  Summary: 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper
5
5
  Home-page: https://github.com/unclecode/crawl4ai
6
6
  Author: Unclecode
@@ -120,9 +120,11 @@ Limited slots._
120
120
 
121
121
  Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
122
122
 
123
- [✨ Check out latest update v0.9.3](#-recent-updates)
123
+ [✨ Check out latest update v0.9.4](#-recent-updates)
124
124
 
125
- ✨ **New in v0.9.3**: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
125
+ ✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
126
+
127
+ ✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
126
128
 
127
129
  ✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
128
130
 
@@ -650,10 +652,29 @@ async def test_news_crawl():
650
652
  ## ✨ Recent Updates
651
653
 
652
654
  <details open>
655
+ <summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
656
+
657
+ A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
658
+
659
+ It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
660
+
661
+ No breaking changes.
662
+
663
+ ```bash
664
+ pip install -U crawl4ai
665
+ ```
666
+
667
+ [Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
668
+
669
+ </details>
670
+
671
+ <details>
653
672
  <summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
654
673
 
655
674
  A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
656
675
 
676
+ It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
677
+
657
678
  No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
658
679
 
659
680
  ```bash
@@ -31,9 +31,11 @@ crawl4ai/chunking_strategy.py
31
31
  crawl4ai/cli.py
32
32
  crawl4ai/config.py
33
33
  crawl4ai/content_filter_strategy.py
34
+ crawl4ai/content_filter_strategy_lxml.py
34
35
  crawl4ai/content_scraping_strategy.py
35
36
  crawl4ai/docker_client.py
36
37
  crawl4ai/domain_mapper.py
38
+ crawl4ai/egress_policy.py
37
39
  crawl4ai/extraction_strategy.py
38
40
  crawl4ai/hub.py
39
41
  crawl4ai/install.py
@@ -151,6 +153,7 @@ tests/test_raw_html_redirected_url.py
151
153
  tests/test_scraping_strategy.py
152
154
  tests/test_source_sibling_selector.py
153
155
  tests/test_table_gfm_compliance.py
156
+ tests/test_table_rowspan_grid.py
154
157
  tests/test_type_annotations.py
155
158
  tests/test_virtual_scroll.py
156
159
  tests/test_web_crawler.py
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: Crawl4AI
3
- Version: 0.9.3
3
+ Version: 0.9.4
4
4
  Summary: 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper
5
5
  Home-page: https://github.com/unclecode/crawl4ai
6
6
  Author: Unclecode
@@ -120,9 +120,11 @@ Limited slots._
120
120
 
121
121
  Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
122
122
 
123
- [✨ Check out latest update v0.9.3](#-recent-updates)
123
+ [✨ Check out latest update v0.9.4](#-recent-updates)
124
124
 
125
- ✨ **New in v0.9.3**: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
125
+ ✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
126
+
127
+ ✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
126
128
 
127
129
  ✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
128
130
 
@@ -650,10 +652,29 @@ async def test_news_crawl():
650
652
  ## ✨ Recent Updates
651
653
 
652
654
  <details open>
655
+ <summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
656
+
657
+ A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
658
+
659
+ It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
660
+
661
+ No breaking changes.
662
+
663
+ ```bash
664
+ pip install -U crawl4ai
665
+ ```
666
+
667
+ [Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
668
+
669
+ </details>
670
+
671
+ <details>
653
672
  <summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
654
673
 
655
674
  A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
656
675
 
676
+ It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
677
+
657
678
  No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
658
679
 
659
680
  ```bash
@@ -37,9 +37,11 @@ Limited slots._
37
37
 
38
38
  Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
39
39
 
40
- [✨ Check out latest update v0.9.3](#-recent-updates)
40
+ [✨ Check out latest update v0.9.4](#-recent-updates)
41
41
 
42
- ✨ **New in v0.9.3**: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
42
+ ✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
43
+
44
+ ✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
43
45
 
44
46
  ✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
45
47
 
@@ -567,10 +569,29 @@ async def test_news_crawl():
567
569
  ## ✨ Recent Updates
568
570
 
569
571
  <details open>
572
+ <summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
573
+
574
+ A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
575
+
576
+ It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
577
+
578
+ No breaking changes.
579
+
580
+ ```bash
581
+ pip install -U crawl4ai
582
+ ```
583
+
584
+ [Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
585
+
586
+ </details>
587
+
588
+ <details>
570
589
  <summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
571
590
 
572
591
  A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
573
592
 
593
+ It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
594
+
574
595
  No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
575
596
 
576
597
  ```bash
@@ -42,6 +42,7 @@ from .content_filter_strategy import (
42
42
  LLMContentFilter,
43
43
  RelevantContentFilter,
44
44
  )
45
+ from .content_filter_strategy_lxml import PruningContentFilterLXML
45
46
  from .models import CrawlResult, MarkdownGenerationResult, DisplayMode
46
47
  from .components.crawler_monitor import CrawlerMonitor
47
48
  from .link_preview import LinkPreview
@@ -176,6 +177,7 @@ __all__ = [
176
177
  "LLMTableExtraction",
177
178
  "RelevantContentFilter",
178
179
  "PruningContentFilter",
180
+ "PruningContentFilterLXML",
179
181
  "BM25ContentFilter",
180
182
  "LLMContentFilter",
181
183
  "BaseDispatcher",
@@ -1,7 +1,7 @@
1
1
  # crawl4ai/__version__.py
2
2
 
3
3
  # This is the version that will be used for stable releases
4
- __version__ = "0.9.3"
4
+ __version__ = "0.9.4"
5
5
 
6
6
  # For nightly builds, this gets set during build process
7
7
  __nightly_version__ = None
@@ -133,7 +133,7 @@ ALLOWED_DESERIALIZE_TYPES = {
133
133
  "CosineStrategy", "RegexExtractionStrategy",
134
134
  # Markdown / content
135
135
  "DefaultMarkdownGenerator",
136
- "PruningContentFilter", "BM25ContentFilter", "LLMContentFilter",
136
+ "PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter", "LLMContentFilter",
137
137
  # Scraping
138
138
  "LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
139
139
  # Chunking
@@ -190,7 +190,7 @@ UNTRUSTED_ALLOWED_TYPES = {
190
190
  # non-LLM extraction / markdown / scraping / chunking strategies
191
191
  "JsonCssExtractionStrategy", "JsonXPathExtractionStrategy",
192
192
  "JsonLxmlExtractionStrategy", "RegexExtractionStrategy", "CosineStrategy",
193
- "DefaultMarkdownGenerator", "PruningContentFilter", "BM25ContentFilter",
193
+ "DefaultMarkdownGenerator", "PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter",
194
194
  "LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
195
195
  "RegexChunking",
196
196
  "DefaultTableExtraction", "NoTableExtraction",
@@ -293,11 +293,49 @@ UNTRUSTED_FIELD_ALLOWLIST = {
293
293
  }
294
294
 
295
295
  # Upper bounds applied to attacker-influenced quantities after filtering.
296
- _MAX_TIMEOUT_MS = 60_000
296
+ _DEFAULT_MAX_TIMEOUT_MS = 60_000
297
297
  _MAX_SCROLL_STEPS = 1000
298
298
  _MAX_VIEWPORT = 4000
299
299
  _MAX_PDF_BYTES = 100 * 1024 * 1024
300
300
  _MAX_PDF_PAGES = 2000
301
+ _MAX_LINK_PREVIEW_LINKS = 100 # the class default
302
+ _MAX_LINK_PREVIEW_CONCURRENCY = 10 # the class default
303
+ _MAX_LINK_PREVIEW_TIMEOUT_S = 10 # seconds here, not ms
304
+
305
+
306
+ def _max_timeout_ms() -> int:
307
+ """Ceiling for the untrusted timeout fields, in milliseconds.
308
+
309
+ 60s is the right bound for a server reachable by untrusted callers, and
310
+ stays the default. An operator whose deployment is not public — a crawler
311
+ on a private network fetching pages that legitimately take minutes — can
312
+ raise it with CRAWL4AI_MAX_TIMEOUT_MS, or lower it to tighten the bound.
313
+
314
+ Read per call rather than captured at import so the setting applies
315
+ wherever the process picked its environment up, and so a test can set it
316
+ without reloading the module. A value that is not a positive integer is
317
+ refused loudly and the default kept: a typo here would silently widen a
318
+ DoS bound, which is the one outcome worse than the timeout being fixed.
319
+ """
320
+ raw = os.getenv("CRAWL4AI_MAX_TIMEOUT_MS")
321
+
322
+ if raw is None or raw == "":
323
+ return _DEFAULT_MAX_TIMEOUT_MS
324
+
325
+ try:
326
+ ceiling = int(raw)
327
+ except ValueError:
328
+ ceiling = 0
329
+
330
+ if ceiling <= 0:
331
+ warnings.warn(
332
+ f"CRAWL4AI_MAX_TIMEOUT_MS={raw!r} is not a positive integer; "
333
+ f"keeping the {_DEFAULT_MAX_TIMEOUT_MS}ms default.",
334
+ stacklevel=2,
335
+ )
336
+ return _DEFAULT_MAX_TIMEOUT_MS
337
+
338
+ return ceiling
301
339
 
302
340
 
303
341
  def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
@@ -318,11 +356,15 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
318
356
 
319
357
  def _clamp_untrusted(type_name: str, params: dict) -> dict:
320
358
  """Clamp attacker-influenced quantities to safe upper bounds."""
359
+ ceiling = _max_timeout_ms()
360
+
321
361
  def _cap_timeout(v):
322
362
  # 0 historically meant "no timeout"; treat as the cap, never unbounded.
363
+ # Malformed input is the one value an attacker gets for free, so it
364
+ # falls back to the 60s default rather than to a raised ceiling.
323
365
  if not isinstance(v, (int, float)) or v <= 0:
324
- return _MAX_TIMEOUT_MS
325
- return min(int(v), _MAX_TIMEOUT_MS)
366
+ return min(_DEFAULT_MAX_TIMEOUT_MS, ceiling)
367
+ return min(int(v), ceiling)
326
368
 
327
369
  if type_name == "CrawlerRunConfig":
328
370
  for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):
@@ -345,6 +387,18 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:
345
387
  # Rasterizing every page is the most expensive thing this strategy can
346
388
  # do, and nothing about untrusted crawling needs it.
347
389
  params["extract_images"] = False
390
+ elif type_name == "LinkPreviewConfig":
391
+ # Also no field allowlist, and each link is a separate outbound fetch:
392
+ # unclamped, one request body can order millions of them at any
393
+ # concurrency it likes.
394
+ for f, cap in (
395
+ ("max_links", _MAX_LINK_PREVIEW_LINKS),
396
+ ("concurrency", _MAX_LINK_PREVIEW_CONCURRENCY),
397
+ ("timeout", _MAX_LINK_PREVIEW_TIMEOUT_S),
398
+ ):
399
+ if f in params:
400
+ v = params[f]
401
+ params[f] = cap if not isinstance(v, int) or v <= 0 else min(v, cap)
348
402
  return params
349
403
 
350
404
 
@@ -434,6 +488,21 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False):
434
488
  return str(obj)
435
489
 
436
490
 
491
+ def _is_typed_shape(data: Any) -> bool:
492
+ """True for the dict shapes to_serializable_dict() emits:
493
+ {"type": "<ClassName>", "params": {...}} or {"type": "dict", "value": {...}}.
494
+
495
+ Plain business dicts that happen to carry a "type" key (JSON-Schema
496
+ fragments, JsonCss field specs like {"type": "text", "name": "..."}) have
497
+ neither "params" nor "value" and are not typed shapes.
498
+ """
499
+ return (
500
+ isinstance(data, dict)
501
+ and "type" in data
502
+ and ("params" in data or (data["type"] == "dict" and "value" in data))
503
+ )
504
+
505
+
437
506
  def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
438
507
  """
439
508
  Recursively convert a serializable dictionary back to an object instance.
@@ -459,14 +528,23 @@ def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
459
528
  # carry a "type" key (e.g. JSON-Schema fragments, JsonCss field specs like
460
529
  # {"type": "text", "name": "..."}) have neither "params" nor "value" and
461
530
  # must fall through to the raw-dict path below so they are passed as data.
462
- if (
463
- isinstance(data, dict)
464
- and "type" in data
465
- and ("params" in data or (data["type"] == "dict" and "value" in data))
466
- ):
531
+ if _is_typed_shape(data):
467
532
  # Handle plain dictionaries
468
533
  if data["type"] == "dict" and "value" in data:
469
- return {k: from_serializable_dict(v, provenance) for k, v in data["value"].items()}
534
+ unwrapped = {
535
+ k: from_serializable_dict(v, provenance) for k, v in data["value"].items()
536
+ }
537
+ # The unwrap above recurses over .items(), so data["value"] is never
538
+ # seen as a whole typed object and the type gate below never fires
539
+ # for it. Without this check an untrusted body launders a forbidden
540
+ # type past the gate by wrapping it:
541
+ # {"type": "dict", "value": {"type": "LLMConfig", "params": {...}}}
542
+ if provenance == Provenance.UNTRUSTED and _is_typed_shape(unwrapped):
543
+ raise UntrustedConfigError(
544
+ "an untrusted request may not wrap a typed object in "
545
+ "{'type': 'dict', 'value': ...}"
546
+ )
547
+ return unwrapped
470
548
 
471
549
  # Security: only allow known-safe types to be deserialized.
472
550
  # Unknown types (e.g. logging.Logger serialized by older clients) are
@@ -973,11 +1051,13 @@ class BrowserConfig:
973
1051
  )
974
1052
 
975
1053
  @staticmethod
976
- def from_kwargs(kwargs: dict) -> "BrowserConfig":
1054
+ def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "BrowserConfig":
977
1055
  # Auto-deserialize any dict values that use the {"type": ..., "params": ...}
978
1056
  # serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
979
1057
  kwargs = {
980
- k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v
1058
+ k: from_serializable_dict(v, provenance)
1059
+ if isinstance(v, dict) and "type" in v
1060
+ else v
981
1061
  for k, v in kwargs.items()
982
1062
  }
983
1063
  # Only pass keys present in kwargs so that __init__ defaults (and
@@ -1063,7 +1143,7 @@ class BrowserConfig:
1063
1143
  # so a body that sends bare kwargs (no {type,params}) cannot bypass it.
1064
1144
  if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
1065
1145
  config = _enforce_untrusted("BrowserConfig", config)
1066
- return BrowserConfig.from_kwargs(config)
1146
+ return BrowserConfig.from_kwargs(config, provenance)
1067
1147
 
1068
1148
  def set_nstproxy(
1069
1149
  self,
@@ -2104,12 +2184,14 @@ class CrawlerRunConfig():
2104
2184
  super().__setattr__(name, value)
2105
2185
 
2106
2186
  @staticmethod
2107
- def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
2187
+ def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "CrawlerRunConfig":
2108
2188
  # Auto-deserialize any dict values that use the {"type": ..., "params": ...}
2109
2189
  # serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
2110
2190
  # This covers markdown_generator, extraction_strategy, content_filter, etc.
2111
2191
  kwargs = {
2112
- k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v
2192
+ k: from_serializable_dict(v, provenance)
2193
+ if isinstance(v, dict) and "type" in v
2194
+ else v
2113
2195
  for k, v in kwargs.items()
2114
2196
  }
2115
2197
  # Only pass keys present in kwargs so that __init__ defaults (and
@@ -2132,7 +2214,7 @@ class CrawlerRunConfig():
2132
2214
  return config
2133
2215
  if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
2134
2216
  config = _enforce_untrusted("CrawlerRunConfig", config)
2135
- return CrawlerRunConfig.from_kwargs(config)
2217
+ return CrawlerRunConfig.from_kwargs(config, provenance)
2136
2218
 
2137
2219
  def to_dict(self):
2138
2220
  return {
@@ -53,6 +53,7 @@ except ImportError:
53
53
  # You might need to adjust this import based on your exact file structure
54
54
  # Import AsyncLogger for default if needed
55
55
  from .async_logger import AsyncLoggerBase, AsyncLogger
56
+ from .egress_policy import proxy_url
56
57
 
57
58
  # Import SeedingConfig for type hints
58
59
  from typing import TYPE_CHECKING
@@ -303,7 +304,11 @@ class AsyncUrlSeeder:
303
304
  ):
304
305
  self.ttl = ttl
305
306
  self._owns_client = client is None # Track if we created the client
306
- self.client = client or httpx.AsyncClient(http2=True, timeout=20, headers={
307
+ # proxy=None unless an embedder installed one (the Docker server does).
308
+ # Every request in this class goes through self.client, so this single
309
+ # kwarg puts the seeder's whole fan-out -- sitemaps, robots, link heads,
310
+ # and each redirect hop -- behind the egress policy.
311
+ self.client = client or httpx.AsyncClient(http2=True, timeout=20, proxy=proxy_url(), headers={
307
312
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) +AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
308
313
  })
309
314
  self.logger = logger # Store the logger instance
@@ -1772,7 +1777,7 @@ class AsyncUrlSeeder:
1772
1777
  self._log("info", "Fetching latest Common Crawl index from {url}",
1773
1778
  params={"url": COLLINFO_URL}, tag="URL_SEED")
1774
1779
  try:
1775
- async with httpx.AsyncClient() as c:
1780
+ async with httpx.AsyncClient(proxy=proxy_url()) as c:
1776
1781
  j = await c.get(COLLINFO_URL, timeout=10)
1777
1782
  j.raise_for_status() # Raise an exception for bad status codes
1778
1783
  idx = j.json()[0]["id"]
@@ -85,8 +85,10 @@ class ManagedBrowser:
85
85
  "--force-color-profile=srgb",
86
86
  "--mute-audio",
87
87
  "--disable-background-timer-throttling",
88
- # Memory-saving flags: disable unused Chrome features
89
- "--disable-features=OptimizationHints,MediaRouter,DialMediaRouteProvider",
88
+ # Memory-saving flags: disable unused Chrome features.
89
+ # Do not disable OptimizationHints — it SEGV_ACCERRs Chrome-for-Testing
90
+ # under --headless=new on macOS arm64 (issue #2239).
91
+ "--disable-features=MediaRouter,DialMediaRouteProvider",
90
92
  "--disable-component-update",
91
93
  "--disable-domain-reliability",
92
94
  ]
@@ -1102,8 +1104,10 @@ class BrowserManager:
1102
1104
  "--force-color-profile=srgb",
1103
1105
  "--mute-audio",
1104
1106
  "--disable-background-timer-throttling",
1105
- # Memory-saving flags: disable unused Chrome features
1106
- "--disable-features=OptimizationHints,MediaRouter,DialMediaRouteProvider",
1107
+ # Memory-saving flags: disable unused Chrome features.
1108
+ # Do not disable OptimizationHints — it SEGV_ACCERRs Chrome-for-Testing
1109
+ # under --headless=new on macOS arm64 (issue #2239).
1110
+ "--disable-features=MediaRouter,DialMediaRouteProvider",
1107
1111
  "--disable-component-update",
1108
1112
  "--disable-domain-reliability",
1109
1113
  # "--single-process",
@@ -25,6 +25,7 @@ from crawl4ai import (
25
25
  JsonXPathExtractionStrategy,
26
26
  BM25ContentFilter,
27
27
  PruningContentFilter,
28
+ PruningContentFilterLXML,
28
29
  BrowserProfiler,
29
30
  DefaultMarkdownGenerator,
30
31
  LLMConfig,
@@ -1094,7 +1095,7 @@ def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config:
1094
1095
  )
1095
1096
  elif filter_conf["type"] == "pruning":
1096
1097
  crawler_cfg.markdown_generator = DefaultMarkdownGenerator(
1097
- content_filter = PruningContentFilter(
1098
+ content_filter = PruningContentFilterLXML(
1098
1099
  user_query=filter_conf.get("query"),
1099
1100
  threshold=filter_conf.get("threshold", 0.48)
1100
1101
  )
@@ -1,6 +1,7 @@
1
1
  import inspect
2
2
  import re
3
3
  import time
4
+ import warnings
4
5
  from bs4 import BeautifulSoup, Tag
5
6
  from typing import List, Tuple, Dict, Optional
6
7
  from rank_bm25 import BM25Okapi
@@ -584,6 +585,19 @@ class PruningContentFilter(RelevantContentFilter):
584
585
  preserve_tags (list): HTML tag names to always keep regardless of score (optional).
585
586
  """
586
587
  super().__init__(None)
588
+ # Deprecation: the BeautifulSoup engine is superseded by the lxml one.
589
+ # Only warn for direct use of this class, not when PruningContentFilterLXML
590
+ # (a subclass) calls super().__init__.
591
+ if type(self) is PruningContentFilter:
592
+ warnings.warn(
593
+ "PruningContentFilter (BeautifulSoup-based) is deprecated in favor of "
594
+ "PruningContentFilterLXML, an lxml reimplementation that is ~10x faster "
595
+ "with identical output. Switch to PruningContentFilterLXML now. In an "
596
+ "upcoming release PruningContentFilter will become an alias for the lxml "
597
+ "implementation, and PruningContentFilterLXML will be kept as a legacy alias.",
598
+ DeprecationWarning,
599
+ stacklevel=2,
600
+ )
587
601
  self.min_word_threshold = min_word_threshold
588
602
  self.threshold_type = threshold_type
589
603
  self.threshold = threshold
@@ -1108,3 +1122,17 @@ class LLMContentFilter(RelevantContentFilter):
1108
1122
  f"{i:<10} {usage.completion_tokens:>12,} "
1109
1123
  f"{usage.prompt_tokens:>12,} {usage.total_tokens:>12,}"
1110
1124
  )
1125
+
1126
+
1127
+ def __getattr__(name):
1128
+ """Lazily re-export PruningContentFilterLXML from this module.
1129
+
1130
+ The lxml engine lives in ``content_filter_strategy_lxml`` (which imports
1131
+ from this module). Exposing it here via PEP 562 lazy attribute access lets
1132
+ ``from crawl4ai.content_filter_strategy import PruningContentFilterLXML``
1133
+ work without creating an import-time circular dependency.
1134
+ """
1135
+ if name == "PruningContentFilterLXML":
1136
+ from .content_filter_strategy_lxml import PruningContentFilterLXML
1137
+ return PruningContentFilterLXML
1138
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")