Crawl4AI 0.9.2__tar.gz → 0.9.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/PKG-INFO +41 -3
  2. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/SOURCES.txt +8 -0
  3. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/PKG-INFO +41 -3
  4. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/README.md +40 -2
  5. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/__init__.py +2 -0
  6. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/__version__.py +1 -1
  7. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_configs.py +149 -19
  8. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_crawler_strategy.py +32 -2
  9. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_url_seeder.py +7 -2
  10. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_webcrawler.py +3 -2
  11. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/browser_manager.py +36 -5
  12. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/cli.py +2 -1
  13. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/content_filter_strategy.py +28 -0
  14. crawl4ai-0.9.4/crawl4ai/content_filter_strategy_lxml.py +391 -0
  15. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/bff_strategy.py +7 -0
  16. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/bfs_strategy.py +6 -2
  17. crawl4ai-0.9.4/crawl4ai/egress_policy.py +35 -0
  18. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/remove_consent_popups.js +7 -2
  19. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/remove_overlay_elements.js +9 -2
  20. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/models.py +1 -0
  21. crawl4ai-0.9.4/crawl4ai/processors/pdf/__init__.py +360 -0
  22. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/processors/pdf/processor.py +23 -3
  23. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/processors/pdf/utils.py +8 -4
  24. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/table_extraction.py +58 -11
  25. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/types.py +4 -0
  26. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/utils.py +67 -24
  27. crawl4ai-0.9.4/tests/test_body_visibility_warning.py +128 -0
  28. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_config_defaults.py +161 -2
  29. crawl4ai-0.9.4/tests/test_docker_pdf_crawler_pairing.py +273 -0
  30. crawl4ai-0.9.4/tests/test_issue_2127_docker_pdf.py +71 -0
  31. crawl4ai-0.9.4/tests/test_issue_2161_overlay_html_body.py +81 -0
  32. crawl4ai-0.9.4/tests/test_placeholder_html_antibot.py +134 -0
  33. crawl4ai-0.9.4/tests/test_table_rowspan_grid.py +162 -0
  34. crawl4ai-0.9.2/crawl4ai/processors/pdf/__init__.py +0 -196
  35. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/dependency_links.txt +0 -0
  36. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/entry_points.txt +0 -0
  37. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/requires.txt +0 -0
  38. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/Crawl4AI.egg-info/top_level.txt +0 -0
  39. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/LICENSE +0 -0
  40. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/MANIFEST.in +0 -0
  41. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/adaptive_crawler.py +0 -0
  42. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/antibot_detector.py +0 -0
  43. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_database.py +0 -0
  44. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_dispatcher.py +0 -0
  45. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/async_logger.py +0 -0
  46. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/browser_adapter.py +0 -0
  47. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/browser_profiler.py +0 -0
  48. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/cache_context.py +0 -0
  49. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/cache_validator.py +0 -0
  50. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/chunking_strategy.py +0 -0
  51. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/cloud/__init__.py +0 -0
  52. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/cloud/cli.py +0 -0
  53. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/components/crawler_monitor.py +0 -0
  54. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/config.py +0 -0
  55. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/content_scraping_strategy.py +0 -0
  56. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/__init__.py +0 -0
  57. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/amazon_product/__init__.py +0 -0
  58. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/amazon_product/crawler.py +0 -0
  59. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/__init__.py +0 -0
  60. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/crawler.py +0 -0
  61. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/crawlers/google_search/script.js +0 -0
  62. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/__init__.py +0 -0
  63. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/base_strategy.py +0 -0
  64. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/crazy.py +0 -0
  65. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/dfs_strategy.py +0 -0
  66. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/filters.py +0 -0
  67. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/deep_crawling/scorers.py +0 -0
  68. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/docker_client.py +0 -0
  69. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/domain_mapper.py +0 -0
  70. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/extraction_strategy.py +0 -0
  71. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/__init__.py +0 -0
  72. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/__main__.py +0 -0
  73. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/_typing.py +0 -0
  74. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/cli.py +0 -0
  75. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/config.py +0 -0
  76. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/elements.py +0 -0
  77. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/html2text/utils.py +0 -0
  78. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/hub.py +0 -0
  79. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/install.py +0 -0
  80. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/__init__.py +0 -0
  81. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/flatten_shadow_dom.js +0 -0
  82. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/navigator_overrider.js +0 -0
  83. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/js_snippet/update_image_dimensions.js +0 -0
  84. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/__init__.py +0 -0
  85. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/cli.py +0 -0
  86. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/crawler_strategy.py +0 -0
  87. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/database.py +0 -0
  88. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/docs_manager.py +0 -0
  89. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/llmtxt.py +0 -0
  90. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/version_manager.py +0 -0
  91. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/legacy/web_crawler.py +0 -0
  92. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/link_preview.py +0 -0
  93. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/markdown_generation_strategy.py +0 -0
  94. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/migrations.py +0 -0
  95. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/model_loader.py +0 -0
  96. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/prompts.py +0 -0
  97. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/proxy_strategy.py +0 -0
  98. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/script/__init__.py +0 -0
  99. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/script/c4a_compile.py +0 -0
  100. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/script/c4a_result.py +0 -0
  101. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/script/c4ai_script.py +0 -0
  102. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/ssl_certificate.py +0 -0
  103. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/crawl4ai/user_agent_generator.py +0 -0
  104. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/pyproject.toml +0 -0
  105. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/requirements.txt +0 -0
  106. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/setup.cfg +0 -0
  107. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/setup.py +0 -0
  108. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_arun_many.py +0 -0
  109. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_async_logger_stderr.py +0 -0
  110. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_bug_batch_1622_1786_1796.py +0 -0
  111. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_cdp_changes.py +0 -0
  112. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_cli_docs.py +0 -0
  113. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_cloud_bugs_batch.py +0 -0
  114. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_config_matching_only.py +0 -0
  115. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_config_selection.py +0 -0
  116. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_docker.py +0 -0
  117. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_docker_api_with_llm_provider.py +0 -0
  118. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_eval_security_adversarial.py +0 -0
  119. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_http_timeout_unit_1894.py +0 -0
  120. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1043_mermaid_svg.py +0 -0
  121. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1213_bm25_dedup.py +0 -0
  122. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1370_1818_1762_1509.py +0 -0
  123. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1484_css_selector.py +0 -0
  124. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1594_mcp_sse.py +0 -0
  125. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1611_llm_provider.py +0 -0
  126. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1748_screenshot_scroll_delay.py +0 -0
  127. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1750_screenshot_scan_full_page.py +0 -0
  128. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1837_config_list.py +0 -0
  129. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1842_browser_none.py +0 -0
  130. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1848_logger_serialize.py +0 -0
  131. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_issue_1850_mcp_sse.py +0 -0
  132. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_link_extractor.py +0 -0
  133. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_llm_extraction_parallel_issue_1055.py +0 -0
  134. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_llm_simple_url.py +0 -0
  135. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_llmtxt.py +0 -0
  136. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_main.py +0 -0
  137. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_markdown_generator_validation_1880.py +0 -0
  138. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_memory_macos.py +0 -0
  139. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_merge_head_data_scoring.py +0 -0
  140. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_multi_config.py +0 -0
  141. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_normalize_url.py +0 -0
  142. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pr_1290_1668.py +0 -0
  143. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pr_1435_redirected_status_code.py +0 -0
  144. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pr_1463_device_scale_factor.py +0 -0
  145. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pr_1795_1798_1734.py +0 -0
  146. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_prefetch_integration.py +0 -0
  147. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_prefetch_mode.py +0 -0
  148. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_prefetch_regression.py +0 -0
  149. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_preserve_https_for_internal_links.py +0 -0
  150. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pruning_preserve_whitelist_1900.py +0 -0
  151. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pyopenssl_security_fix.py +0 -0
  152. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_pyopenssl_update.py +0 -0
  153. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_raw_html_browser.py +0 -0
  154. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_raw_html_edge_cases.py +0 -0
  155. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_raw_html_redirected_url.py +0 -0
  156. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_scraping_strategy.py +0 -0
  157. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_source_sibling_selector.py +0 -0
  158. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_table_gfm_compliance.py +0 -0
  159. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_type_annotations.py +0 -0
  160. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_virtual_scroll.py +0 -0
  161. {crawl4ai-0.9.2 → crawl4ai-0.9.4}/tests/test_web_crawler.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: Crawl4AI
3
- Version: 0.9.2
3
+ Version: 0.9.4
4
4
  Summary: 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper
5
5
  Home-page: https://github.com/unclecode/crawl4ai
6
6
  Author: Unclecode
@@ -120,9 +120,13 @@ Limited slots._
120
120
 
121
121
  Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
122
122
 
123
- [✨ Check out latest update v0.9.2](#-recent-updates)
123
+ [✨ Check out latest update v0.9.4](#-recent-updates)
124
124
 
125
- ✨ **New in v0.9.2**: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
125
+ ✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
126
+
127
+ ✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
128
+
129
+ ✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
126
130
 
127
131
  ✨ Recent v0.9.0: Major secure-by-default release of the Docker API server. Auth is on by default, the server binds loopback unless given a token, and the request body is now an untrusted trust boundary. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.0.md)
128
132
 
@@ -648,6 +652,40 @@ async def test_news_crawl():
648
652
  ## ✨ Recent Updates
649
653
 
650
654
  <details open>
655
+ <summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
656
+
657
+ A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
658
+
659
+ It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
660
+
661
+ No breaking changes.
662
+
663
+ ```bash
664
+ pip install -U crawl4ai
665
+ ```
666
+
667
+ [Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
668
+
669
+ </details>
670
+
671
+ <details>
672
+ <summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
673
+
674
+ A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
675
+
676
+ It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
677
+
678
+ No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
679
+
680
+ ```bash
681
+ pip install -U crawl4ai
682
+ ```
683
+
684
+ [Full v0.9.3 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
685
+
686
+ </details>
687
+
688
+ <details>
651
689
  <summary><strong>Version 0.9.2 Release Highlights - Maintenance Bug Fixes</strong></summary>
652
690
 
653
691
  A maintenance patch release with bug fixes across the dispatcher, Docker, and GPU builds. `MemoryAdaptiveDispatcher` no longer leaks crawl tasks and browser pages when a streaming crawl is closed. Docker fixes cover the Playground "Advanced Config" 400, the Monitor WebSocket 500 under JWT auth, and Playwright headless-shell packaging. `ENABLE_GPU=true` Docker builds no longer fail on the CUDA toolkit.
@@ -31,9 +31,11 @@ crawl4ai/chunking_strategy.py
31
31
  crawl4ai/cli.py
32
32
  crawl4ai/config.py
33
33
  crawl4ai/content_filter_strategy.py
34
+ crawl4ai/content_filter_strategy_lxml.py
34
35
  crawl4ai/content_scraping_strategy.py
35
36
  crawl4ai/docker_client.py
36
37
  crawl4ai/domain_mapper.py
38
+ crawl4ai/egress_policy.py
37
39
  crawl4ai/extraction_strategy.py
38
40
  crawl4ai/hub.py
39
41
  crawl4ai/install.py
@@ -96,6 +98,7 @@ crawl4ai/script/c4a_result.py
96
98
  crawl4ai/script/c4ai_script.py
97
99
  tests/test_arun_many.py
98
100
  tests/test_async_logger_stderr.py
101
+ tests/test_body_visibility_warning.py
99
102
  tests/test_bug_batch_1622_1786_1796.py
100
103
  tests/test_cdp_changes.py
101
104
  tests/test_cli_docs.py
@@ -105,6 +108,7 @@ tests/test_config_matching_only.py
105
108
  tests/test_config_selection.py
106
109
  tests/test_docker.py
107
110
  tests/test_docker_api_with_llm_provider.py
111
+ tests/test_docker_pdf_crawler_pairing.py
108
112
  tests/test_eval_security_adversarial.py
109
113
  tests/test_http_timeout_unit_1894.py
110
114
  tests/test_issue_1043_mermaid_svg.py
@@ -119,6 +123,8 @@ tests/test_issue_1837_config_list.py
119
123
  tests/test_issue_1842_browser_none.py
120
124
  tests/test_issue_1848_logger_serialize.py
121
125
  tests/test_issue_1850_mcp_sse.py
126
+ tests/test_issue_2127_docker_pdf.py
127
+ tests/test_issue_2161_overlay_html_body.py
122
128
  tests/test_link_extractor.py
123
129
  tests/test_llm_extraction_parallel_issue_1055.py
124
130
  tests/test_llm_simple_url.py
@@ -129,6 +135,7 @@ tests/test_memory_macos.py
129
135
  tests/test_merge_head_data_scoring.py
130
136
  tests/test_multi_config.py
131
137
  tests/test_normalize_url.py
138
+ tests/test_placeholder_html_antibot.py
132
139
  tests/test_pr_1290_1668.py
133
140
  tests/test_pr_1435_redirected_status_code.py
134
141
  tests/test_pr_1463_device_scale_factor.py
@@ -146,6 +153,7 @@ tests/test_raw_html_redirected_url.py
146
153
  tests/test_scraping_strategy.py
147
154
  tests/test_source_sibling_selector.py
148
155
  tests/test_table_gfm_compliance.py
156
+ tests/test_table_rowspan_grid.py
149
157
  tests/test_type_annotations.py
150
158
  tests/test_virtual_scroll.py
151
159
  tests/test_web_crawler.py
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: Crawl4AI
3
- Version: 0.9.2
3
+ Version: 0.9.4
4
4
  Summary: 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper
5
5
  Home-page: https://github.com/unclecode/crawl4ai
6
6
  Author: Unclecode
@@ -120,9 +120,13 @@ Limited slots._
120
120
 
121
121
  Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
122
122
 
123
- [✨ Check out latest update v0.9.2](#-recent-updates)
123
+ [✨ Check out latest update v0.9.4](#-recent-updates)
124
124
 
125
- ✨ **New in v0.9.2**: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
125
+ ✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
126
+
127
+ ✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
128
+
129
+ ✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
126
130
 
127
131
  ✨ Recent v0.9.0: Major secure-by-default release of the Docker API server. Auth is on by default, the server binds loopback unless given a token, and the request body is now an untrusted trust boundary. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.0.md)
128
132
 
@@ -648,6 +652,40 @@ async def test_news_crawl():
648
652
  ## ✨ Recent Updates
649
653
 
650
654
  <details open>
655
+ <summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
656
+
657
+ A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
658
+
659
+ It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
660
+
661
+ No breaking changes.
662
+
663
+ ```bash
664
+ pip install -U crawl4ai
665
+ ```
666
+
667
+ [Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
668
+
669
+ </details>
670
+
671
+ <details>
672
+ <summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
673
+
674
+ A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
675
+
676
+ It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
677
+
678
+ No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
679
+
680
+ ```bash
681
+ pip install -U crawl4ai
682
+ ```
683
+
684
+ [Full v0.9.3 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
685
+
686
+ </details>
687
+
688
+ <details>
651
689
  <summary><strong>Version 0.9.2 Release Highlights - Maintenance Bug Fixes</strong></summary>
652
690
 
653
691
  A maintenance patch release with bug fixes across the dispatcher, Docker, and GPU builds. `MemoryAdaptiveDispatcher` no longer leaks crawl tasks and browser pages when a streaming crawl is closed. Docker fixes cover the Playground "Advanced Config" 400, the Monitor WebSocket 500 under JWT auth, and Playwright headless-shell packaging. `ENABLE_GPU=true` Docker builds no longer fail on the CUDA toolkit.
@@ -37,9 +37,13 @@ Limited slots._
37
37
 
38
38
  Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
39
39
 
40
- [✨ Check out latest update v0.9.2](#-recent-updates)
40
+ [✨ Check out latest update v0.9.4](#-recent-updates)
41
41
 
42
- ✨ **New in v0.9.2**: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
42
+ ✨ **New in v0.9.4**: Security release. Closes three coordinated-disclosure advisories: two SSRF paths (robots.txt and link preview) that bypassed the Docker server's egress controls, and a config trust-boundary bypass that leaked server environment variables. Also adds `PruningContentFilterLXML`, about 10x faster pruning, now the default. No breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
43
+
44
+ ✨ Recent v0.9.3: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
45
+
46
+ ✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md)
43
47
 
44
48
  ✨ Recent v0.9.0: Major secure-by-default release of the Docker API server. Auth is on by default, the server binds loopback unless given a token, and the request body is now an untrusted trust boundary. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.0.md)
45
49
 
@@ -565,6 +569,40 @@ async def test_news_crawl():
565
569
  ## ✨ Recent Updates
566
570
 
567
571
  <details open>
572
+ <summary><strong>Version 0.9.4 Release Highlights - Security Release and Faster Pruning</strong></summary>
573
+
574
+ A security release closing three coordinated-disclosure advisories. Two are SSRF paths that did not go through the Docker server's egress rule: the robots.txt fetch behind `check_robots_txt`, and the URL seeder behind `link_preview_config`, which also returned the fetched `<head>` to the caller. Both now go through the server's pinning egress proxy. The third is a bypass of the untrusted-config gate: a `{"type": "dict"}` wrapper let a forbidden `LLMConfig` through, so a non-admin client could read server environment variables.
575
+
576
+ It also adds `PruningContentFilterLXML`, an lxml-native pruning filter that is about 10x faster and gives byte-identical output. It is now the default, and `PruningContentFilter` is deprecated. Bug fixes cover deep-crawl speed, tables with `rowspan`/`colspan`, robots.txt rules, pooled browser recycling, and the Docker Playground.
577
+
578
+ No breaking changes.
579
+
580
+ ```bash
581
+ pip install -U crawl4ai
582
+ ```
583
+
584
+ [Full v0.9.4 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.4.md)
585
+
586
+ </details>
587
+
588
+ <details>
589
+ <summary><strong>Version 0.9.3 Release Highlights - Security Release</strong></summary>
590
+
591
+ A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token.
592
+
593
+ It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks.
594
+
595
+ No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0.
596
+
597
+ ```bash
598
+ pip install -U crawl4ai
599
+ ```
600
+
601
+ [Full v0.9.3 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md)
602
+
603
+ </details>
604
+
605
+ <details>
568
606
  <summary><strong>Version 0.9.2 Release Highlights - Maintenance Bug Fixes</strong></summary>
569
607
 
570
608
  A maintenance patch release with bug fixes across the dispatcher, Docker, and GPU builds. `MemoryAdaptiveDispatcher` no longer leaks crawl tasks and browser pages when a streaming crawl is closed. Docker fixes cover the Playground "Advanced Config" 400, the Monitor WebSocket 500 under JWT auth, and Playwright headless-shell packaging. `ENABLE_GPU=true` Docker builds no longer fail on the CUDA toolkit.
@@ -42,6 +42,7 @@ from .content_filter_strategy import (
42
42
  LLMContentFilter,
43
43
  RelevantContentFilter,
44
44
  )
45
+ from .content_filter_strategy_lxml import PruningContentFilterLXML
45
46
  from .models import CrawlResult, MarkdownGenerationResult, DisplayMode
46
47
  from .components.crawler_monitor import CrawlerMonitor
47
48
  from .link_preview import LinkPreview
@@ -176,6 +177,7 @@ __all__ = [
176
177
  "LLMTableExtraction",
177
178
  "RelevantContentFilter",
178
179
  "PruningContentFilter",
180
+ "PruningContentFilterLXML",
179
181
  "BM25ContentFilter",
180
182
  "LLMContentFilter",
181
183
  "BaseDispatcher",
@@ -1,7 +1,7 @@
1
1
  # crawl4ai/__version__.py
2
2
 
3
3
  # This is the version that will be used for stable releases
4
- __version__ = "0.9.2"
4
+ __version__ = "0.9.4"
5
5
 
6
6
  # For nightly builds, this gets set during build process
7
7
  __nightly_version__ = None
@@ -133,7 +133,7 @@ ALLOWED_DESERIALIZE_TYPES = {
133
133
  "CosineStrategy", "RegexExtractionStrategy",
134
134
  # Markdown / content
135
135
  "DefaultMarkdownGenerator",
136
- "PruningContentFilter", "BM25ContentFilter", "LLMContentFilter",
136
+ "PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter", "LLMContentFilter",
137
137
  # Scraping
138
138
  "LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
139
139
  # Chunking
@@ -190,7 +190,7 @@ UNTRUSTED_ALLOWED_TYPES = {
190
190
  # non-LLM extraction / markdown / scraping / chunking strategies
191
191
  "JsonCssExtractionStrategy", "JsonXPathExtractionStrategy",
192
192
  "JsonLxmlExtractionStrategy", "RegexExtractionStrategy", "CosineStrategy",
193
- "DefaultMarkdownGenerator", "PruningContentFilter", "BM25ContentFilter",
193
+ "DefaultMarkdownGenerator", "PruningContentFilter", "PruningContentFilterLXML", "BM25ContentFilter",
194
194
  "LXMLWebScrapingStrategy", "PDFContentScrapingStrategy",
195
195
  "RegexChunking",
196
196
  "DefaultTableExtraction", "NoTableExtraction",
@@ -214,6 +214,29 @@ UNTRUSTED_FORBIDDEN_FIELDS = {
214
214
  "override_navigator", "magic", "process_in_browser", "shared_data",
215
215
  "session_id",
216
216
  },
217
+ # PDFContentScrapingStrategy has no scalar allowlist (it is kept for its
218
+ # value fields), so its filesystem-write knobs must be blocked explicitly:
219
+ # image_save_dir is an unconfined write destination and save_images_locally
220
+ # turns the write on. extract_images stays allowed - it only returns the
221
+ # image bytes base64-inline in the response, no disk write.
222
+ "PDFContentScrapingStrategy": {
223
+ "image_save_dir", "save_images_locally",
224
+ },
225
+ }
226
+
227
+ # Field names that must NEVER be set from an untrusted body on ANY allowed
228
+ # type, checked regardless of per-type allowlist. This is the fail-closed
229
+ # backstop: the per-type maps above only cover BrowserConfig/CrawlerRunConfig,
230
+ # so a strategy that (now or in future) exposes a filesystem-write, code, or
231
+ # routing constructor arg is caught here even with no explicit allowlist.
232
+ # Presence => 400 (loud), matching js_code/extra_args behavior.
233
+ UNTRUSTED_GLOBAL_FORBIDDEN_FIELDS = {
234
+ # filesystem write / read sinks
235
+ "image_save_dir", "save_images_locally", "downloads_path", "user_data_dir",
236
+ "output_path", "save_path", "file_path", "local_path", "storage_state",
237
+ # code / command execution
238
+ "js_code", "js_code_before_wait", "c4a_script", "init_scripts",
239
+ "code", "command", "hook", "hooks",
217
240
  }
218
241
 
219
242
  # Scalar knobs an untrusted body MAY set, per class. A field not listed here is
@@ -244,6 +267,7 @@ UNTRUSTED_FIELD_ALLOWLIST = {
244
267
  "fetch_ssl_certificate",
245
268
  # timing / waiting
246
269
  "wait_until", "page_timeout", "wait_for", "wait_for_timeout",
270
+ "body_visibility_timeout",
247
271
  "wait_for_images", "delay_before_return_html", "mean_delay", "max_range",
248
272
  # scrolling / rendering
249
273
  "ignore_body_visibility", "scan_full_page", "scroll_delay",
@@ -269,9 +293,49 @@ UNTRUSTED_FIELD_ALLOWLIST = {
269
293
  }
270
294
 
271
295
  # Upper bounds applied to attacker-influenced quantities after filtering.
272
- _MAX_TIMEOUT_MS = 60_000
296
+ _DEFAULT_MAX_TIMEOUT_MS = 60_000
273
297
  _MAX_SCROLL_STEPS = 1000
274
298
  _MAX_VIEWPORT = 4000
299
+ _MAX_PDF_BYTES = 100 * 1024 * 1024
300
+ _MAX_PDF_PAGES = 2000
301
+ _MAX_LINK_PREVIEW_LINKS = 100 # the class default
302
+ _MAX_LINK_PREVIEW_CONCURRENCY = 10 # the class default
303
+ _MAX_LINK_PREVIEW_TIMEOUT_S = 10 # seconds here, not ms
304
+
305
+
306
+ def _max_timeout_ms() -> int:
307
+ """Ceiling for the untrusted timeout fields, in milliseconds.
308
+
309
+ 60s is the right bound for a server reachable by untrusted callers, and
310
+ stays the default. An operator whose deployment is not public — a crawler
311
+ on a private network fetching pages that legitimately take minutes — can
312
+ raise it with CRAWL4AI_MAX_TIMEOUT_MS, or lower it to tighten the bound.
313
+
314
+ Read per call rather than captured at import so the setting applies
315
+ wherever the process picked its environment up, and so a test can set it
316
+ without reloading the module. A value that is not a positive integer is
317
+ refused loudly and the default kept: a typo here would silently widen a
318
+ DoS bound, which is the one outcome worse than the timeout being fixed.
319
+ """
320
+ raw = os.getenv("CRAWL4AI_MAX_TIMEOUT_MS")
321
+
322
+ if raw is None or raw == "":
323
+ return _DEFAULT_MAX_TIMEOUT_MS
324
+
325
+ try:
326
+ ceiling = int(raw)
327
+ except ValueError:
328
+ ceiling = 0
329
+
330
+ if ceiling <= 0:
331
+ warnings.warn(
332
+ f"CRAWL4AI_MAX_TIMEOUT_MS={raw!r} is not a positive integer; "
333
+ f"keeping the {_DEFAULT_MAX_TIMEOUT_MS}ms default.",
334
+ stacklevel=2,
335
+ )
336
+ return _DEFAULT_MAX_TIMEOUT_MS
337
+
338
+ return ceiling
275
339
 
276
340
 
277
341
  def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
@@ -280,7 +344,7 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
280
344
  allowlist = UNTRUSTED_FIELD_ALLOWLIST.get(type_name) # None => keep all non-forbidden
281
345
  out = {}
282
346
  for key, value in params.items():
283
- if key in forbidden:
347
+ if key in UNTRUSTED_GLOBAL_FORBIDDEN_FIELDS or key in forbidden:
284
348
  raise UntrustedConfigError(
285
349
  f"field '{key}' is not permitted on {type_name} from an untrusted request"
286
350
  )
@@ -292,14 +356,18 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
292
356
 
293
357
  def _clamp_untrusted(type_name: str, params: dict) -> dict:
294
358
  """Clamp attacker-influenced quantities to safe upper bounds."""
359
+ ceiling = _max_timeout_ms()
360
+
295
361
  def _cap_timeout(v):
296
362
  # 0 historically meant "no timeout"; treat as the cap, never unbounded.
363
+ # Malformed input is the one value an attacker gets for free, so it
364
+ # falls back to the 60s default rather than to a raised ceiling.
297
365
  if not isinstance(v, (int, float)) or v <= 0:
298
- return _MAX_TIMEOUT_MS
299
- return min(int(v), _MAX_TIMEOUT_MS)
366
+ return min(_DEFAULT_MAX_TIMEOUT_MS, ceiling)
367
+ return min(int(v), ceiling)
300
368
 
301
369
  if type_name == "CrawlerRunConfig":
302
- for f in ("page_timeout", "wait_for_timeout"):
370
+ for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):
303
371
  if f in params:
304
372
  params[f] = _cap_timeout(params[f])
305
373
  if isinstance(params.get("max_scroll_steps"), int):
@@ -308,6 +376,29 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:
308
376
  for f in ("viewport_width", "viewport_height"):
309
377
  if isinstance(params.get(f), int):
310
378
  params[f] = max(1, min(params[f], _MAX_VIEWPORT))
379
+ elif type_name == "PDFContentScrapingStrategy":
380
+ # This class has no field allowlist, so without clamping a body could
381
+ # simply raise its own caps back to unbounded and re-open the DoS.
382
+ for f, cap in (("max_pdf_bytes", _MAX_PDF_BYTES), ("max_pdf_pages", _MAX_PDF_PAGES)):
383
+ if f in params:
384
+ v = params[f]
385
+ # <=0 or non-int would read as "no limit"; pin to the cap.
386
+ params[f] = cap if not isinstance(v, int) or v <= 0 else min(v, cap)
387
+ # Rasterizing every page is the most expensive thing this strategy can
388
+ # do, and nothing about untrusted crawling needs it.
389
+ params["extract_images"] = False
390
+ elif type_name == "LinkPreviewConfig":
391
+ # Also no field allowlist, and each link is a separate outbound fetch:
392
+ # unclamped, one request body can order millions of them at any
393
+ # concurrency it likes.
394
+ for f, cap in (
395
+ ("max_links", _MAX_LINK_PREVIEW_LINKS),
396
+ ("concurrency", _MAX_LINK_PREVIEW_CONCURRENCY),
397
+ ("timeout", _MAX_LINK_PREVIEW_TIMEOUT_S),
398
+ ):
399
+ if f in params:
400
+ v = params[f]
401
+ params[f] = cap if not isinstance(v, int) or v <= 0 else min(v, cap)
311
402
  return params
312
403
 
313
404
 
@@ -397,6 +488,21 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False):
397
488
  return str(obj)
398
489
 
399
490
 
491
+ def _is_typed_shape(data: Any) -> bool:
492
+ """True for the dict shapes to_serializable_dict() emits:
493
+ {"type": "<ClassName>", "params": {...}} or {"type": "dict", "value": {...}}.
494
+
495
+ Plain business dicts that happen to carry a "type" key (JSON-Schema
496
+ fragments, JsonCss field specs like {"type": "text", "name": "..."}) have
497
+ neither "params" nor "value" and are not typed shapes.
498
+ """
499
+ return (
500
+ isinstance(data, dict)
501
+ and "type" in data
502
+ and ("params" in data or (data["type"] == "dict" and "value" in data))
503
+ )
504
+
505
+
400
506
  def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
401
507
  """
402
508
  Recursively convert a serializable dictionary back to an object instance.
@@ -422,14 +528,23 @@ def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
422
528
  # carry a "type" key (e.g. JSON-Schema fragments, JsonCss field specs like
423
529
  # {"type": "text", "name": "..."}) have neither "params" nor "value" and
424
530
  # must fall through to the raw-dict path below so they are passed as data.
425
- if (
426
- isinstance(data, dict)
427
- and "type" in data
428
- and ("params" in data or (data["type"] == "dict" and "value" in data))
429
- ):
531
+ if _is_typed_shape(data):
430
532
  # Handle plain dictionaries
431
533
  if data["type"] == "dict" and "value" in data:
432
- return {k: from_serializable_dict(v, provenance) for k, v in data["value"].items()}
534
+ unwrapped = {
535
+ k: from_serializable_dict(v, provenance) for k, v in data["value"].items()
536
+ }
537
+ # The unwrap above recurses over .items(), so data["value"] is never
538
+ # seen as a whole typed object and the type gate below never fires
539
+ # for it. Without this check an untrusted body launders a forbidden
540
+ # type past the gate by wrapping it:
541
+ # {"type": "dict", "value": {"type": "LLMConfig", "params": {...}}}
542
+ if provenance == Provenance.UNTRUSTED and _is_typed_shape(unwrapped):
543
+ raise UntrustedConfigError(
544
+ "an untrusted request may not wrap a typed object in "
545
+ "{'type': 'dict', 'value': ...}"
546
+ )
547
+ return unwrapped
433
548
 
434
549
  # Security: only allow known-safe types to be deserialized.
435
550
  # Unknown types (e.g. logging.Logger serialized by older clients) are
@@ -936,11 +1051,13 @@ class BrowserConfig:
936
1051
  )
937
1052
 
938
1053
  @staticmethod
939
- def from_kwargs(kwargs: dict) -> "BrowserConfig":
1054
+ def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "BrowserConfig":
940
1055
  # Auto-deserialize any dict values that use the {"type": ..., "params": ...}
941
1056
  # serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
942
1057
  kwargs = {
943
- k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v
1058
+ k: from_serializable_dict(v, provenance)
1059
+ if isinstance(v, dict) and "type" in v
1060
+ else v
944
1061
  for k, v in kwargs.items()
945
1062
  }
946
1063
  # Only pass keys present in kwargs so that __init__ defaults (and
@@ -1026,7 +1143,7 @@ class BrowserConfig:
1026
1143
  # so a body that sends bare kwargs (no {type,params}) cannot bypass it.
1027
1144
  if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
1028
1145
  config = _enforce_untrusted("BrowserConfig", config)
1029
- return BrowserConfig.from_kwargs(config)
1146
+ return BrowserConfig.from_kwargs(config, provenance)
1030
1147
 
1031
1148
  def set_nstproxy(
1032
1149
  self,
@@ -1463,6 +1580,8 @@ class CrawlerRunConfig():
1463
1580
  Default: False.
1464
1581
  ignore_body_visibility (bool): If True, ignore whether the body is visible before proceeding.
1465
1582
  Default: True.
1583
+ body_visibility_timeout (int): Maximum time in ms to wait for the body to become visible.
1584
+ Default: 30000.
1466
1585
  scan_full_page (bool): If True, scroll through the entire page to load all content.
1467
1586
  Default: False.
1468
1587
  scroll_delay (float): Delay in seconds between scroll steps if scan_full_page is True.
@@ -1640,6 +1759,7 @@ class CrawlerRunConfig():
1640
1759
  c4a_script: Union[str, List[str]] = None,
1641
1760
  js_only: bool = False,
1642
1761
  ignore_body_visibility: bool = True,
1762
+ body_visibility_timeout: int = 30000,
1643
1763
  scan_full_page: bool = False,
1644
1764
  scroll_delay: float = 0.2,
1645
1765
  max_scroll_steps: Optional[int] = None,
@@ -1770,6 +1890,13 @@ class CrawlerRunConfig():
1770
1890
  self.c4a_script = c4a_script
1771
1891
  self.js_only = js_only
1772
1892
  self.ignore_body_visibility = ignore_body_visibility
1893
+ if (
1894
+ not isinstance(body_visibility_timeout, (int, float))
1895
+ or isinstance(body_visibility_timeout, bool)
1896
+ or body_visibility_timeout <= 0
1897
+ ):
1898
+ raise ValueError("body_visibility_timeout must be a positive number")
1899
+ self.body_visibility_timeout = body_visibility_timeout
1773
1900
  self.scan_full_page = scan_full_page
1774
1901
  self.scroll_delay = scroll_delay
1775
1902
  self.max_scroll_steps = max_scroll_steps
@@ -2057,12 +2184,14 @@ class CrawlerRunConfig():
2057
2184
  super().__setattr__(name, value)
2058
2185
 
2059
2186
  @staticmethod
2060
- def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
2187
+ def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "CrawlerRunConfig":
2061
2188
  # Auto-deserialize any dict values that use the {"type": ..., "params": ...}
2062
2189
  # serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
2063
2190
  # This covers markdown_generator, extraction_strategy, content_filter, etc.
2064
2191
  kwargs = {
2065
- k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v
2192
+ k: from_serializable_dict(v, provenance)
2193
+ if isinstance(v, dict) and "type" in v
2194
+ else v
2066
2195
  for k, v in kwargs.items()
2067
2196
  }
2068
2197
  # Only pass keys present in kwargs so that __init__ defaults (and
@@ -2085,7 +2214,7 @@ class CrawlerRunConfig():
2085
2214
  return config
2086
2215
  if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
2087
2216
  config = _enforce_untrusted("CrawlerRunConfig", config)
2088
- return CrawlerRunConfig.from_kwargs(config)
2217
+ return CrawlerRunConfig.from_kwargs(config, provenance)
2089
2218
 
2090
2219
  def to_dict(self):
2091
2220
  return {
@@ -2137,6 +2266,7 @@ class CrawlerRunConfig():
2137
2266
  "js_code_before_wait": self.js_code_before_wait,
2138
2267
  "js_only": self.js_only,
2139
2268
  "ignore_body_visibility": self.ignore_body_visibility,
2269
+ "body_visibility_timeout": self.body_visibility_timeout,
2140
2270
  "scan_full_page": self.scan_full_page,
2141
2271
  "scroll_delay": self.scroll_delay,
2142
2272
  "max_scroll_steps": self.max_scroll_steps,
@@ -812,6 +812,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
812
812
  await page.wait_for_selector("body", state="attached", timeout=30000)
813
813
 
814
814
  # Use the new check_visibility function with csp_compliant_wait
815
+ _visibility_wait_start = time.perf_counter()
815
816
  is_visible = await self.csp_compliant_wait(
816
817
  page,
817
818
  """() => {
@@ -823,9 +824,38 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
823
824
  style.opacity !== '0';
824
825
  return isVisible;
825
826
  }""",
826
- timeout=30000,
827
+ timeout=config.body_visibility_timeout,
827
828
  )
828
829
 
830
+ _visibility_wait_ms = (
831
+ time.perf_counter() - _visibility_wait_start
832
+ ) * 1000
833
+
834
+ # csp_compliant_wait also returns False when the evaluation itself
835
+ # fails (page closed, context destroyed by a redirect), which costs
836
+ # no time — only warn about a wait that actually burned its budget.
837
+ if (
838
+ not is_visible
839
+ and config.ignore_body_visibility
840
+ and _visibility_wait_ms >= config.body_visibility_timeout * 0.9
841
+ ):
842
+ # The wait timed out and its result is about to be discarded, so
843
+ # the crawl still succeeds — just this much slower, on every crawl
844
+ # of this page. force_verbose because the whole point is that the
845
+ # delay is otherwise invisible, and the servers and batch jobs that
846
+ # most need to see it run with verbose off (see #2144).
847
+ self.logger.warning(
848
+ message=(
849
+ "Body never became visible after {elapsed}ms — the page may "
850
+ "use ng-cloak/v-cloak. This delay is added to every crawl of "
851
+ "this page; lower CrawlerRunConfig.body_visibility_timeout "
852
+ "to shorten it."
853
+ ),
854
+ tag="WARNING",
855
+ params={"elapsed": round(_visibility_wait_ms)},
856
+ force_verbose=True,
857
+ )
858
+
829
859
  if not is_visible and not config.ignore_body_visibility:
830
860
  visibility_info = await self.check_visibility(page)
831
861
  raise Error(f"Body element is hidden: {visibility_info}")
@@ -1534,7 +1564,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
1534
1564
  }})()
1535
1565
  """
1536
1566
  )
1537
- await page.wait_for_timeout(500) # Wait for any animations to complete
1567
+ await page.wait_for_timeout(600) # Wait for any animations to complete
1538
1568
  except Exception as e:
1539
1569
  self.logger.warning(
1540
1570
  message="Failed to remove overlay elements: {error}",