Crawl4AI 0.9.3__tar.gz → 0.9.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. crawl4ai-0.9.5/Crawl4AI.egg-info/PKG-INFO +620 -0
  2. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/Crawl4AI.egg-info/SOURCES.txt +8 -0
  3. crawl4ai-0.9.5/PKG-INFO +620 -0
  4. crawl4ai-0.9.5/README.md +537 -0
  5. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/__init__.py +2 -0
  6. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/__version__.py +1 -1
  7. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/async_configs.py +249 -21
  8. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/async_crawler_strategy.py +51 -8
  9. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/async_url_seeder.py +7 -2
  10. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/async_webcrawler.py +13 -0
  11. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/browser_manager.py +35 -11
  12. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/cache_validator.py +5 -0
  13. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/cli.py +22 -8
  14. crawl4ai-0.9.5/crawl4ai/cloud_notice.py +149 -0
  15. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/content_filter_strategy.py +28 -0
  16. crawl4ai-0.9.5/crawl4ai/content_filter_strategy_lxml.py +391 -0
  17. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/content_scraping_strategy.py +35 -1
  18. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/deep_crawling/bff_strategy.py +8 -0
  19. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/deep_crawling/bfs_strategy.py +24 -3
  20. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/deep_crawling/dfs_strategy.py +2 -0
  21. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/deep_crawling/filters.py +17 -11
  22. crawl4ai-0.9.5/crawl4ai/egress_policy.py +35 -0
  23. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/install.py +19 -1
  24. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/models.py +9 -2
  25. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/processors/pdf/processor.py +9 -12
  26. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/table_extraction.py +58 -11
  27. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/types.py +4 -0
  28. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/utils.py +71 -25
  29. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_config_defaults.py +111 -1
  30. crawl4ai-0.9.5/tests/test_http_proxy_error_redaction_2326.py +93 -0
  31. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1837_config_list.py +8 -6
  32. crawl4ai-0.9.5/tests/test_issue_2233_mcp_sse.py +96 -0
  33. crawl4ai-0.9.5/tests/test_issue_2328_inline_whitespace.py +38 -0
  34. crawl4ai-0.9.5/tests/test_table_rowspan_grid.py +162 -0
  35. crawl4ai-0.9.5/tests/test_unclosed_noscript.py +113 -0
  36. crawl4ai-0.9.3/Crawl4AI.egg-info/PKG-INFO +0 -1395
  37. crawl4ai-0.9.3/PKG-INFO +0 -1395
  38. crawl4ai-0.9.3/README.md +0 -1312
  39. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/Crawl4AI.egg-info/dependency_links.txt +0 -0
  40. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/Crawl4AI.egg-info/entry_points.txt +0 -0
  41. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/Crawl4AI.egg-info/requires.txt +0 -0
  42. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/Crawl4AI.egg-info/top_level.txt +0 -0
  43. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/LICENSE +0 -0
  44. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/MANIFEST.in +0 -0
  45. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/adaptive_crawler.py +0 -0
  46. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/antibot_detector.py +0 -0
  47. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/async_database.py +0 -0
  48. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/async_dispatcher.py +0 -0
  49. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/async_logger.py +0 -0
  50. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/browser_adapter.py +0 -0
  51. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/browser_profiler.py +0 -0
  52. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/cache_context.py +0 -0
  53. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/chunking_strategy.py +0 -0
  54. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/cloud/__init__.py +0 -0
  55. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/cloud/cli.py +0 -0
  56. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/components/crawler_monitor.py +0 -0
  57. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/config.py +0 -0
  58. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/crawlers/__init__.py +0 -0
  59. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/crawlers/amazon_product/__init__.py +0 -0
  60. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/crawlers/amazon_product/crawler.py +0 -0
  61. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/crawlers/google_search/__init__.py +0 -0
  62. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/crawlers/google_search/crawler.py +0 -0
  63. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/crawlers/google_search/script.js +0 -0
  64. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/deep_crawling/__init__.py +0 -0
  65. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/deep_crawling/base_strategy.py +0 -0
  66. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/deep_crawling/crazy.py +0 -0
  67. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/deep_crawling/scorers.py +0 -0
  68. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/docker_client.py +0 -0
  69. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/domain_mapper.py +0 -0
  70. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/extraction_strategy.py +0 -0
  71. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/html2text/__init__.py +0 -0
  72. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/html2text/__main__.py +0 -0
  73. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/html2text/_typing.py +0 -0
  74. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/html2text/cli.py +0 -0
  75. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/html2text/config.py +0 -0
  76. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/html2text/elements.py +0 -0
  77. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/html2text/utils.py +0 -0
  78. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/hub.py +0 -0
  79. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/js_snippet/__init__.py +0 -0
  80. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/js_snippet/flatten_shadow_dom.js +0 -0
  81. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/js_snippet/navigator_overrider.js +0 -0
  82. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/js_snippet/remove_consent_popups.js +0 -0
  83. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/js_snippet/remove_overlay_elements.js +0 -0
  84. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/js_snippet/update_image_dimensions.js +0 -0
  85. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/legacy/__init__.py +0 -0
  86. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/legacy/cli.py +0 -0
  87. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/legacy/crawler_strategy.py +0 -0
  88. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/legacy/database.py +0 -0
  89. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/legacy/docs_manager.py +0 -0
  90. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/legacy/llmtxt.py +0 -0
  91. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/legacy/version_manager.py +0 -0
  92. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/legacy/web_crawler.py +0 -0
  93. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/link_preview.py +0 -0
  94. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/markdown_generation_strategy.py +0 -0
  95. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/migrations.py +0 -0
  96. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/model_loader.py +0 -0
  97. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/processors/pdf/__init__.py +0 -0
  98. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/processors/pdf/utils.py +0 -0
  99. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/prompts.py +0 -0
  100. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/proxy_strategy.py +0 -0
  101. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/script/__init__.py +0 -0
  102. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/script/c4a_compile.py +0 -0
  103. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/script/c4a_result.py +0 -0
  104. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/script/c4ai_script.py +0 -0
  105. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/ssl_certificate.py +0 -0
  106. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/crawl4ai/user_agent_generator.py +0 -0
  107. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/pyproject.toml +0 -0
  108. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/requirements.txt +0 -0
  109. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/setup.cfg +0 -0
  110. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/setup.py +0 -0
  111. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_arun_many.py +0 -0
  112. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_async_logger_stderr.py +0 -0
  113. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_body_visibility_warning.py +0 -0
  114. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_bug_batch_1622_1786_1796.py +0 -0
  115. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_cdp_changes.py +0 -0
  116. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_cli_docs.py +0 -0
  117. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_cloud_bugs_batch.py +0 -0
  118. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_config_matching_only.py +0 -0
  119. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_config_selection.py +0 -0
  120. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_docker.py +0 -0
  121. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_docker_api_with_llm_provider.py +0 -0
  122. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_docker_pdf_crawler_pairing.py +0 -0
  123. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_eval_security_adversarial.py +0 -0
  124. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_http_timeout_unit_1894.py +0 -0
  125. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1043_mermaid_svg.py +0 -0
  126. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1213_bm25_dedup.py +0 -0
  127. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1370_1818_1762_1509.py +0 -0
  128. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1484_css_selector.py +0 -0
  129. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1594_mcp_sse.py +0 -0
  130. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1611_llm_provider.py +0 -0
  131. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1748_screenshot_scroll_delay.py +0 -0
  132. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1750_screenshot_scan_full_page.py +0 -0
  133. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1842_browser_none.py +0 -0
  134. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1848_logger_serialize.py +0 -0
  135. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_1850_mcp_sse.py +0 -0
  136. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_2127_docker_pdf.py +0 -0
  137. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_issue_2161_overlay_html_body.py +0 -0
  138. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_link_extractor.py +0 -0
  139. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_llm_extraction_parallel_issue_1055.py +0 -0
  140. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_llm_simple_url.py +0 -0
  141. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_llmtxt.py +0 -0
  142. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_main.py +0 -0
  143. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_markdown_generator_validation_1880.py +0 -0
  144. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_memory_macos.py +0 -0
  145. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_merge_head_data_scoring.py +0 -0
  146. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_multi_config.py +0 -0
  147. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_normalize_url.py +0 -0
  148. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_placeholder_html_antibot.py +0 -0
  149. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_pr_1290_1668.py +0 -0
  150. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_pr_1435_redirected_status_code.py +0 -0
  151. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_pr_1463_device_scale_factor.py +0 -0
  152. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_pr_1795_1798_1734.py +0 -0
  153. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_prefetch_integration.py +0 -0
  154. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_prefetch_mode.py +0 -0
  155. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_prefetch_regression.py +0 -0
  156. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_preserve_https_for_internal_links.py +0 -0
  157. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_pruning_preserve_whitelist_1900.py +0 -0
  158. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_pyopenssl_security_fix.py +0 -0
  159. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_pyopenssl_update.py +0 -0
  160. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_raw_html_browser.py +0 -0
  161. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_raw_html_edge_cases.py +0 -0
  162. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_raw_html_redirected_url.py +0 -0
  163. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_scraping_strategy.py +0 -0
  164. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_source_sibling_selector.py +0 -0
  165. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_table_gfm_compliance.py +0 -0
  166. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_type_annotations.py +0 -0
  167. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_virtual_scroll.py +0 -0
  168. {crawl4ai-0.9.3 → crawl4ai-0.9.5}/tests/test_web_crawler.py +0 -0
@@ -0,0 +1,620 @@
1
+ Metadata-Version: 2.4
2
+ Name: Crawl4AI
3
+ Version: 0.9.5
4
+ Summary: 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper
5
+ Home-page: https://github.com/unclecode/crawl4ai
6
+ Author: Unclecode
7
+ Author-email: Unclecode <unclecode@kidocode.com>
8
+ License-Expression: Apache-2.0
9
+ Classifier: Development Status :: 4 - Beta
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Requires-Python: >=3.10
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: aiofiles>=24.1.0
20
+ Requires-Dist: aiohttp>=3.11.11
21
+ Requires-Dist: aiosqlite~=0.20
22
+ Requires-Dist: anyio>=4.0.0
23
+ Requires-Dist: lxml<7,>=5.3
24
+ Requires-Dist: unclecode-litellm==1.81.13
25
+ Requires-Dist: numpy<3,>=1.26.0
26
+ Requires-Dist: pillow>=10.4
27
+ Requires-Dist: playwright>=1.49.0
28
+ Requires-Dist: patchright>=1.49.0
29
+ Requires-Dist: python-dotenv~=1.0
30
+ Requires-Dist: requests~=2.26
31
+ Requires-Dist: beautifulsoup4~=4.12
32
+ Requires-Dist: playwright-stealth>=2.0.0
33
+ Requires-Dist: xxhash~=3.4
34
+ Requires-Dist: rank-bm25~=0.2
35
+ Requires-Dist: snowballstemmer~=2.2
36
+ Requires-Dist: pydantic>=2.10
37
+ Requires-Dist: pyOpenSSL>=25.3.0
38
+ Requires-Dist: psutil>=6.1.1
39
+ Requires-Dist: PyYAML>=6.0
40
+ Requires-Dist: nltk>=3.9.1
41
+ Requires-Dist: rich>=13.9.4
42
+ Requires-Dist: cssselect>=1.2.0
43
+ Requires-Dist: httpx>=0.27.2
44
+ Requires-Dist: httpx[http2]>=0.27.2
45
+ Requires-Dist: fake-useragent>=2.0.3
46
+ Requires-Dist: click>=8.1.7
47
+ Requires-Dist: chardet>=5.2.0
48
+ Requires-Dist: brotli>=1.1.0
49
+ Requires-Dist: humanize>=4.10.0
50
+ Requires-Dist: lark>=1.2.2
51
+ Requires-Dist: alphashape>=1.3.1
52
+ Requires-Dist: shapely>=2.0.0
53
+ Provides-Extra: pdf
54
+ Requires-Dist: pypdf; extra == "pdf"
55
+ Provides-Extra: torch
56
+ Requires-Dist: torch; extra == "torch"
57
+ Requires-Dist: nltk; extra == "torch"
58
+ Requires-Dist: scikit-learn; extra == "torch"
59
+ Provides-Extra: transformer
60
+ Requires-Dist: transformers; extra == "transformer"
61
+ Requires-Dist: tokenizers; extra == "transformer"
62
+ Requires-Dist: sentence-transformers; extra == "transformer"
63
+ Provides-Extra: cosine
64
+ Requires-Dist: torch; extra == "cosine"
65
+ Requires-Dist: transformers; extra == "cosine"
66
+ Requires-Dist: nltk; extra == "cosine"
67
+ Requires-Dist: sentence-transformers; extra == "cosine"
68
+ Provides-Extra: sync
69
+ Requires-Dist: selenium; extra == "sync"
70
+ Provides-Extra: all
71
+ Requires-Dist: pypdf; extra == "all"
72
+ Requires-Dist: torch; extra == "all"
73
+ Requires-Dist: nltk; extra == "all"
74
+ Requires-Dist: scikit-learn; extra == "all"
75
+ Requires-Dist: transformers; extra == "all"
76
+ Requires-Dist: tokenizers; extra == "all"
77
+ Requires-Dist: sentence-transformers; extra == "all"
78
+ Requires-Dist: selenium; extra == "all"
79
+ Dynamic: author
80
+ Dynamic: home-page
81
+ Dynamic: license-file
82
+ Dynamic: requires-python
83
+
84
+ # 🚀🤖 Crawl4AI: the open-source web crawler for LLMs and AI agents
85
+
86
+ <div align="center">
87
+
88
+ <a href="https://trendshift.io/repositories/11716" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11716" alt="unclecode%2Fcrawl4ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
89
+
90
+ [![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers)
91
+ [![PyPI version](https://badge.fury.io/py/crawl4ai.svg)](https://badge.fury.io/py/crawl4ai)
92
+ [![Downloads](https://static.pepy.tech/badge/crawl4ai/month)](https://pepy.tech/project/crawl4ai)
93
+ [![Discord](https://img.shields.io/badge/Discord-join%20us-5865F2?logo=discord&logoColor=white)](https://discord.gg/jP8KfhDhyN)
94
+ [![Crawl4AI Cloud](https://img.shields.io/badge/Crawl4AI_Cloud-try_it_free-f5a300?style=flat&labelColor=0d0d10)](https://crawl4ai.com/?ref=readme-badge)
95
+
96
+ **Latest: [v0.9.5](https://github.com/unclecode/crawl4ai/releases/tag/v0.9.5) (14 Oct 2026)** · [all releases →](https://github.com/unclecode/crawl4ai/releases)
97
+
98
+ <a href="https://crawl4ai.com/?ref=readme-banner">
99
+ <picture>
100
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/cloud-launch-banner-dark.svg">
101
+ <img alt="Crawl4AI Cloud is live. Soft launch: free credit to start, no card. Get your key." src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/cloud-launch-banner-light.svg" width="960">
102
+ </picture>
103
+ </a>
104
+
105
+ </div>
106
+
107
+ Crawl4AI turns any website into clean, LLM-ready Markdown for RAG, AI agents and data pipelines. Run the open-source web crawler and scraper yourself, free forever, or use it hosted with one key: `/scrape` a page to Markdown, `/extract` it to JSON, `/search` the web, `/serp` for Google's results page as data, `/answer` a question with sources; MCP for your agent. Free: 10 a minute, 2 at a time, 2,000 credits to start. Paid: 120 a minute, 10 at a time, from $10.
108
+
109
+ ## Two ways to use Crawl4AI
110
+
111
+ ### 🐍 Run it yourself: open source, forever
112
+
113
+ ```bash
114
+ pip install -U crawl4ai
115
+ crawl4ai-setup # installs the browser, once
116
+ ```
117
+
118
+ ```python
119
+ import asyncio
120
+ from crawl4ai import AsyncWebCrawler
121
+
122
+ async def main():
123
+ async with AsyncWebCrawler() as crawler:
124
+ result = await crawler.arun(url="https://news.ycombinator.com")
125
+ print(result.markdown)
126
+
127
+ asyncio.run(main())
128
+ ```
129
+
130
+ Docker server, CLI and every option: [Installation](#installation) · [docs.crawl4ai.com](https://docs.crawl4ai.com)
131
+
132
+ ### ☁️ Or use the cloud: no browsers, no proxies
133
+
134
+ 1. [![Get a key in 10 seconds](https://img.shields.io/badge/Get_a_key_in_10_seconds-20_free_credits%2C_no_signup-f5a300?style=for-the-badge&labelColor=0d0d10)](https://crawl4ai.com/?ref=readme)
135
+ Verify your email and free credit to start is yours. No card. Soft launch: prices can change, what you buy stays yours.
136
+ 2. Get any page as Markdown:
137
+
138
+ ```bash
139
+ curl -s https://api.crawl4ai.com/scrape \
140
+ -H "Authorization: Bearer $CRAWL4AI_KEY" \
141
+ -H "Content-Type: application/json" \
142
+ -d '{"url": "https://news.ycombinator.com"}' | jq -r .markdown
143
+ ```
144
+
145
+ The same key works for `/extract`, `/search`, `/serp`, `/answer` and many URLs at once (`/scrape/batch`, `/scrape/jobs`). Pay as you go: [live prices](https://crawl4ai.com/docs?ref=readme#pricing).
146
+ 3. Give it to your AI agent. Claude Code shown; [Codex, Cursor and OpenCode →](https://crawl4ai.com/docs?ref=readme#mcp)
147
+
148
+ ```bash
149
+ claude mcp add --transport http crawl4ai https://api.crawl4ai.com/mcp \
150
+ --header "Authorization: Bearer $CRAWL4AI_KEY"
151
+ ```
152
+
153
+ ### Which one?
154
+
155
+ | | 🐍 Library | 🐳 Your own server | ☁️ Crawl4AI Cloud |
156
+ |---|---|---|---|
157
+ | **Runs the browsers** | you, in your Python process | you, in Docker on your machine | we do |
158
+ | **JS-heavy pages and bot walls** | your settings, your proxies | your settings, your proxies | handled for you, automatically |
159
+ | **Web search** | – | – | `/search`, `/serp` and `/answer` |
160
+ | **Price** | free, forever | free (your hosting) | pay as you go; free credit to start |
161
+
162
+ <details>
163
+ <summary>🤓 <strong>My Personal Story</strong></summary>
164
+
165
+ I grew up on an Amstrad, thanks to my dad, and never stopped building. In grad school I specialized in NLP and built crawlers for research. That’s where I learned how much extraction matters.
166
+
167
+ In 2023, I needed web-to-Markdown. The “open source” option wanted an account, API token, and $16, and still under-delivered. I went turbo anger mode, built Crawl4AI in days, and it went viral. Now it’s the most-starred crawler on GitHub.
168
+
169
+ I made it open source for **availability**, anyone can use it without a gate. Now I’m building the platform for **affordability**, anyone can run serious crawls without breaking the bank. If that resonates, join in, send feedback, or just crawl something amazing.
170
+
171
+ That platform is live now: [Crawl4AI Cloud](https://crawl4ai.com/?ref=readme).
172
+ </details>
173
+
174
+ <details>
175
+ <summary>Why developers pick Crawl4AI</summary>
176
+
177
+ - **LLM-ready output**: smart Markdown with headings, tables, code and citation hints
178
+ - **Fast in practice**: async browser pool, caching, minimal hops
179
+ - **Full control**: sessions, proxies, cookies, user scripts, hooks
180
+ - **Adaptive intelligence**: learns site patterns, explores only what matters
181
+ - **Deploy anywhere**: no keys needed, CLI and Docker, or the hosted cloud
182
+ </details>
183
+
184
+ ## ✨ Features
185
+
186
+ <details>
187
+ <summary>📝 <strong>Markdown generation</strong></summary>
188
+
189
+ - 🧹 **Clean Markdown**: headings, lists, tables and code blocks, in a structure an LLM reads well.
190
+ - 🎯 **Fit Markdown**: filters remove menus, footers and boilerplate: `PruningContentFilterLXML`, `BM25ContentFilter` (for a query) and `LLMContentFilter`.
191
+ - 🔗 **Citations**: page links become a numbered reference list.
192
+ - 🛠️ **Your own strategy**: plug in a custom Markdown generator.
193
+
194
+ ☁️ Same in the cloud: `POST /scrape` returns this Markdown, with no browser to run. [Docs →](https://crawl4ai.com/docs?ref=readme#scrape)
195
+ </details>
196
+
197
+ <details>
198
+ <summary>📊 <strong>Structured data extraction</strong></summary>
199
+
200
+ - 🔎 **CSS and XPath schemas**: fast extraction with no LLM (`JsonCssExtractionStrategy`, `JsonXPathExtractionStrategy`, `RegexExtractionStrategy`).
201
+ - 🪄 **Schema generator**: describe what you want once; `generate_schema` writes a reusable schema.
202
+ - 🤖 **LLM extraction**: any LLM provider, open-source or hosted, into a typed JSON schema (`LLMExtractionStrategy`).
203
+ - 🧱 **Chunking**: topic, regex and sentence chunking for long pages.
204
+ - 🌌 **Cosine similarity**: find the chunks that match a query (`CosineStrategy`).
205
+
206
+ ☁️ Same in the cloud: `POST /extract`, with no LLM key of your own. [Docs →](https://crawl4ai.com/docs?ref=readme#extract)
207
+ </details>
208
+
209
+ <details>
210
+ <summary>🌐 <strong>Browser control</strong></summary>
211
+
212
+ - 🖥️ **Your own browser**: persistent profiles with saved logins, cookies and settings.
213
+ - 🔄 **Remote browsers**: connect over the Chrome DevTools Protocol (CDP).
214
+ - 🔒 **Sessions**: keep a browser state across multi-step crawls.
215
+ - 🧩 **Proxies**: with authentication and rotation.
216
+ - 🕶️ **Stealth mode**: `enable_stealth`, and an undetected-browser adapter for sites that detect automation.
217
+ - ⚙️ **Full control**: headers, cookies, user agents, viewport.
218
+ - 🌍 **Chromium, Firefox and WebKit**.
219
+ </details>
220
+
221
+ <details>
222
+ <summary>🔎 <strong>Crawling and scraping</strong></summary>
223
+
224
+ - 🕸️ **Deep crawl**: BFS, DFS and best-first strategies, with crash recovery (`resume_state`) for long crawls.
225
+ - 🧠 **Adaptive crawling**: `AdaptiveCrawler` stops when it has learned enough to answer your query.
226
+ - 🌱 **URL discovery**: `AsyncUrlSeeder` (sitemaps, Common Crawl) and `DomainMapper`; `prefetch=True` finds URLs 5 to 10 times faster.
227
+ - 🚀 **Dynamic pages**: run JavaScript, wait for elements, scroll the full page (`scan_full_page`) for infinite scroll and lazy images.
228
+ - 📸 **Screenshots and PDFs** of any page.
229
+ - 🖼️ **Media and links**: images, audio, video, `srcset`, internal and external links, iframes, metadata.
230
+ - 📂 **Raw HTML and local files**: `raw:` and `file://`.
231
+ - 🛠️ **Hooks** at every step of a crawl.
232
+ - 💾 **Caching** to skip repeated fetches.
233
+ - ⚡ **Many URLs at once**: `arun_many` with a memory-adaptive dispatcher.
234
+
235
+ ☁️ Same in the cloud: up to 50 URLs in one streamed call, or 10,000 in a background job. [Docs →](https://crawl4ai.com/docs?ref=readme#batch)
236
+ </details>
237
+
238
+ <details>
239
+ <summary>🐳 <strong>Self-hosting (Docker)</strong></summary>
240
+
241
+ - 🔐 **Secure by default**: every endpoint needs your `CRAWL4AI_API_TOKEN`.
242
+ - 🧰 **REST API**: `/md`, `/html`, `/crawl`, `/crawl/stream`, `/screenshot`, `/pdf`, `/execute_js`.
243
+ - 🤖 **MCP**: connect Claude Code and other agents to your own server.
244
+ - 📊 **Monitoring dashboard and playground**, a browser pool with pre-warmed pages.
245
+ - 🏗️ **AMD64 and ARM64** images.
246
+
247
+ ☁️ Rather not run a server? The cloud is the same idea, hosted. [Get a key →](https://crawl4ai.com/?ref=readme)
248
+ </details>
249
+
250
+ <details>
251
+ <summary>☁️ <strong>What the cloud adds</strong></summary>
252
+
253
+ - 🔍 **Web search API**: `GET /search`, browser-free, ranked and cleaned. [Docs →](https://crawl4ai.com/docs?ref=readme#search)
254
+ - 📊 **SERP API**: `GET /serp`, Google's results page as data: true ranks, every block, localised to a place. Beta. [Docs →](https://crawl4ai.com/docs?ref=readme#serp)
255
+ - 💬 **Answers**: `GET /answer` gives a direct answer to a question (experimental). [Docs →](https://crawl4ai.com/docs?ref=readme#answer)
256
+ - 🧪 **Extraction without your own LLM key**: `POST /extract`. [Docs →](https://crawl4ai.com/docs?ref=readme#extract)
257
+ - 🧗 **JS-heavy pages and bot walls**: handled automatically; you never pick an engine. [Docs →](https://crawl4ai.com/docs?ref=readme#scrape)
258
+ - 🤝 **MCP for your agent**: one line in Claude Code, Codex, Cursor or OpenCode. [Docs →](https://crawl4ai.com/docs?ref=readme#mcp)
259
+ </details>
260
+
261
+ <a id="installation"></a>
262
+
263
+ ## 🛠️ Installation
264
+
265
+ <details>
266
+ <summary>🐍 <strong>pip</strong></summary>
267
+
268
+ ```bash
269
+ pip install -U crawl4ai
270
+ crawl4ai-setup # installs and sets up the browser
271
+ crawl4ai-doctor # checks the installation
272
+ ```
273
+
274
+ If the browser setup fails, install it by hand:
275
+
276
+ ```bash
277
+ python -m playwright install --with-deps chromium
278
+ ```
279
+
280
+ Pre-release versions: `pip install crawl4ai --pre`
281
+
282
+ **Development install**, for contributors:
283
+
284
+ ```bash
285
+ git clone https://github.com/unclecode/crawl4ai.git
286
+ cd crawl4ai
287
+ pip install -e ".[all]" # or: pip install -e . (the core only)
288
+ ```
289
+ </details>
290
+
291
+ <details>
292
+ <summary>🐳 <strong>Docker server</strong></summary>
293
+
294
+ The server needs a token. Without one it answers only inside its container.
295
+
296
+ ```bash
297
+ export CRAWL4AI_API_TOKEN="$(openssl rand -hex 32)"
298
+ docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g \
299
+ -e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN" \
300
+ unclecode/crawl4ai:latest
301
+ ```
302
+
303
+ Test it (allow about 10 seconds for the start):
304
+
305
+ ```bash
306
+ curl -s http://localhost:11235/md \
307
+ -H "Authorization: Bearer $CRAWL4AI_API_TOKEN" \
308
+ -H "Content-Type: application/json" \
309
+ -d '{"url": "https://news.ycombinator.com"}' | jq -r .markdown
310
+ ```
311
+
312
+ The dashboard is at `http://localhost:11235/dashboard`, the playground at `http://localhost:11235/playground`. LLM keys, MCP and every setting: [Self-hosting guide](https://docs.crawl4ai.com/core/self-hosting/).
313
+ </details>
314
+
315
+ <details>
316
+ <summary>⌨️ <strong>Command line (`crwl`)</strong></summary>
317
+
318
+ ```bash
319
+ # A page as Markdown
320
+ crwl https://news.ycombinator.com -o markdown
321
+
322
+ # Deep crawl, breadth first, at most 10 pages
323
+ crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10
324
+
325
+ # Ask a question about a page (needs an LLM key: crwl config)
326
+ crwl https://www.example.com/products -q "Extract all product prices"
327
+ ```
328
+ </details>
329
+
330
+ ## 🔬 Advanced usage examples
331
+
332
+ More in [docs/examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples).
333
+
334
+ <details>
335
+ <summary>📝 <strong>Clean and fit Markdown</strong></summary>
336
+
337
+ ```python
338
+ import asyncio
339
+ from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
340
+ from crawl4ai.content_filter_strategy import PruningContentFilterLXML
341
+ from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
342
+
343
+ async def main():
344
+ run_config = CrawlerRunConfig(
345
+ cache_mode=CacheMode.BYPASS,
346
+ markdown_generator=DefaultMarkdownGenerator(
347
+ content_filter=PruningContentFilterLXML(threshold=0.48, threshold_type="fixed", min_word_threshold=0)
348
+ ),
349
+ )
350
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
351
+ result = await crawler.arun(url="https://en.wikipedia.org/wiki/Web_crawler", config=run_config)
352
+ print(len(result.markdown.raw_markdown), "characters of raw Markdown")
353
+ print(len(result.markdown.fit_markdown), "characters after the filter")
354
+
355
+ asyncio.run(main())
356
+ ```
357
+ </details>
358
+
359
+ <details>
360
+ <summary>🖥️ <strong>A JavaScript page and structured data, without an LLM</strong></summary>
361
+
362
+ ```python
363
+ import asyncio, json
364
+ from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, JsonCssExtractionStrategy
365
+
366
+ schema = {
367
+ "name": "Quotes",
368
+ "baseSelector": "div.quote",
369
+ "fields": [
370
+ {"name": "text", "selector": "span.text", "type": "text"},
371
+ {"name": "author", "selector": "small.author", "type": "text"},
372
+ {"name": "tags", "selector": "a.tag", "type": "list", "fields": [{"name": "tag", "type": "text"}]},
373
+ ],
374
+ }
375
+
376
+ async def main():
377
+ run_config = CrawlerRunConfig(
378
+ extraction_strategy=JsonCssExtractionStrategy(schema),
379
+ scan_full_page=True, # scroll to the end, so the page loads every quote
380
+ scroll_delay=0.5,
381
+ cache_mode=CacheMode.BYPASS,
382
+ )
383
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
384
+ result = await crawler.arun(url="https://quotes.toscrape.com/scroll", config=run_config)
385
+ quotes = json.loads(result.extracted_content)
386
+ print(f"Extracted {len(quotes)} quotes")
387
+ print(json.dumps(quotes[0], indent=2))
388
+
389
+ asyncio.run(main())
390
+ ```
391
+ </details>
392
+
393
+ <details>
394
+ <summary>📚 <strong>Structured data with an LLM</strong></summary>
395
+
396
+ ```python
397
+ import os, asyncio
398
+ from pydantic import BaseModel, Field
399
+ from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode, LLMConfig, LLMExtractionStrategy
400
+
401
+ class ModelFee(BaseModel):
402
+ model_name: str = Field(..., description="Name of the model.")
403
+ input_fee: str = Field(..., description="Fee for input tokens.")
404
+ output_fee: str = Field(..., description="Fee for output tokens.")
405
+
406
+ async def main():
407
+ run_config = CrawlerRunConfig(
408
+ cache_mode=CacheMode.BYPASS,
409
+ extraction_strategy=LLMExtractionStrategy(
410
+ # any provider LiteLLM supports, e.g. "ollama/llama3.3" with api_token="no-token"
411
+ llm_config=LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY")),
412
+ schema=ModelFee.model_json_schema(),
413
+ extraction_type="schema",
414
+ instruction="Extract every model name with its input and output token fee.",
415
+ ),
416
+ )
417
+ async with AsyncWebCrawler() as crawler:
418
+ result = await crawler.arun(url="https://openai.com/api/pricing/", config=run_config)
419
+ print(result.extracted_content)
420
+
421
+ asyncio.run(main())
422
+ ```
423
+ </details>
424
+
425
+ <details>
426
+ <summary>🤖 <strong>Your own browser with a saved profile</strong></summary>
427
+
428
+ ```python
429
+ import os, asyncio
430
+ from pathlib import Path
431
+ from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
432
+
433
+ async def main():
434
+ user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile")
435
+ os.makedirs(user_data_dir, exist_ok=True)
436
+ browser_config = BrowserConfig(headless=True, user_data_dir=user_data_dir, use_persistent_context=True)
437
+ run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, magic=True)
438
+ async with AsyncWebCrawler(config=browser_config) as crawler:
439
+ result = await crawler.arun(url="ADDRESS_OF_A_CHALLENGING_WEBSITE", config=run_config)
440
+ print(result.success, len(result.markdown))
441
+
442
+ asyncio.run(main())
443
+ ```
444
+ </details>
445
+
446
+ ## 📖 Documentation
447
+
448
+ - Library docs, guides and API reference: [docs.crawl4ai.com](https://docs.crawl4ai.com/)
449
+ - Cloud docs: [crawl4ai.com/docs](https://crawl4ai.com/docs?ref=readme)
450
+ - Release notes: [releases](https://github.com/unclecode/crawl4ai/releases) · Roadmap: [ROADMAP.md](https://github.com/unclecode/crawl4ai/blob/main/ROADMAP.md)
451
+
452
+ ## 🤝 Contributing
453
+
454
+ We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTORS.md) for more information.
455
+
456
+ ## 📄 License & Attribution
457
+
458
+ This project is licensed under the Apache License 2.0, attribution is recommended via the badges below. See the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) file for details.
459
+
460
+ ### Attribution Requirements
461
+ When using Crawl4AI, you must include one of the following attribution methods:
462
+
463
+ <details>
464
+ <summary>📈 <strong>1. Badge Attribution (Recommended)</strong></summary>
465
+ Add one of these badges to your README, documentation, or website:
466
+
467
+ | Theme | Badge |
468
+ |-------|-------|
469
+ | **Disco Theme (Animated)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/></a> |
470
+ | **Night Theme (Dark with Neon)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/></a> |
471
+ | **Dark Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/></a> |
472
+ | **Light Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/></a> |
473
+
474
+
475
+ HTML code for adding the badges:
476
+ ```html
477
+ <!-- Disco Theme (Animated) -->
478
+ <a href="https://github.com/unclecode/crawl4ai">
479
+ <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/>
480
+ </a>
481
+
482
+ <!-- Night Theme (Dark with Neon) -->
483
+ <a href="https://github.com/unclecode/crawl4ai">
484
+ <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/>
485
+ </a>
486
+
487
+ <!-- Dark Theme (Classic) -->
488
+ <a href="https://github.com/unclecode/crawl4ai">
489
+ <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/>
490
+ </a>
491
+
492
+ <!-- Light Theme (Classic) -->
493
+ <a href="https://github.com/unclecode/crawl4ai">
494
+ <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/>
495
+ </a>
496
+
497
+ <!-- Simple Shield Badge -->
498
+ <a href="https://github.com/unclecode/crawl4ai">
499
+ <img src="https://img.shields.io/badge/Powered%20by-Crawl4AI-blue?style=flat-square" alt="Powered by Crawl4AI"/>
500
+ </a>
501
+ ```
502
+
503
+ </details>
504
+
505
+ <details>
506
+ <summary>📖 <strong>2. Text Attribution</strong></summary>
507
+ Add this line to your documentation:
508
+ ```
509
+ This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.
510
+ ```
511
+ </details>
512
+
513
+ ## 📚 Citation
514
+
515
+ If you use Crawl4AI in your research or project, please cite:
516
+
517
+ ```bibtex
518
+ @software{crawl4ai2024,
519
+ author = {UncleCode},
520
+ title = {Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper},
521
+ year = {2024},
522
+ publisher = {GitHub},
523
+ journal = {GitHub Repository},
524
+ howpublished = {\url{https://github.com/unclecode/crawl4ai}},
525
+ commit = {Please use the commit hash you're working with}
526
+ }
527
+ ```
528
+
529
+ Text citation format:
530
+ ```
531
+ UncleCode. (2024). Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper [Computer software].
532
+ GitHub. https://github.com/unclecode/crawl4ai
533
+ ```
534
+
535
+ ## 🗾 Mission
536
+
537
+ Our mission is to unlock the value of personal and enterprise data by turning digital footprints into structured, useful assets. Crawl4AI gives individuals and organizations open-source tools to extract and structure data, and a fair way to benefit from it. [Full mission statement →](./MISSION.md)
538
+
539
+ ## 💖 Support Crawl4AI
540
+
541
+ 1. ⭐ **Star the repo**: it helps more people find it.
542
+ 2. ☁️ **Use the cloud**: [crawl4ai.com](https://crawl4ai.com/?ref=readme). It funds the library.
543
+ 3. 💝 **Sponsor on GitHub**: [github.com/sponsors/unclecode](https://github.com/sponsors/unclecode)
544
+ 4. 🏢 **Companies**: the sponsor tiers and benefits are in [SPONSORS.md](SPONSORS.md).
545
+
546
+ ## 🌟 Current Sponsors
547
+
548
+ ### 🤝 Strategic Partners
549
+
550
+ These companies provide core infrastructure and technology that power Crawl4AI’s capabilities — from web access and proxy networks to AI tooling and data pipelines.
551
+
552
+ | Company | About |
553
+ |------|------|
554
+ | <a href="https://www.joinmassive.com/" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="docs/assets/sponsors/massive_light.svg"><source media="(prefers-color-scheme: light)" srcset="docs/assets/sponsors/massive.svg"><img alt="Massive" src="docs/assets/sponsors/massive.svg" height="40"/></picture></a> | Massive is a web access API backed by millions of volunteer devices in 195+ countries. AI agents, models, and data pipelines use it to reach any site on the internet, reliably, in real time, and at scale. |
555
+
556
+ ### 🏢 Enterprise Sponsors
557
+
558
+ Our enterprise sponsors support Crawl4AI and help scale it to power production-grade data pipelines.
559
+
560
+ | Company | About | Sponsorship Tier |
561
+ |------|------|----------------------------|
562
+ | <a href="https://kipo.ai" target="_blank"><img src="https://docs.crawl4ai.com/uploads/sponsors/20251013045751_2d54f57f117c651e.png" alt="DataSync" height="40"/></a> | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold |
563
+ | <a href="https://www.kidocode.com/" target="_blank"><img src="https://docs.crawl4ai.com/uploads/sponsors/20251013045045_bb8dace3f0440d65.svg" alt="Kidocode" height="40"/></a> | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold |
564
+ | <a href="https://www.alephnull.sg/" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="docs/assets/sponsors/aleph_null_light.svg"><source media="(prefers-color-scheme: light)" srcset="docs/assets/sponsors/aleph_null.svg"><img alt="Aleph null" src="docs/assets/sponsors/aleph_null.svg" height="40"/></picture></a> | Singapore-based Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold |
565
+
566
+ ---
567
+
568
+ ### 💼 Become a Strategic Partner or Sponsor
569
+
570
+ Interested in partnering with Crawl4AI?
571
+
572
+ Whether you’re a proxy provider, AI infrastructure company, cloud platform, or an organization looking to support the Crawl4AI ecosystem, we’d love to hear from you.
573
+
574
+ 📩 Contact: hello@crawl4ai.com
575
+
576
+
577
+
578
+ ### 🧑‍🤝 Individual Sponsors
579
+
580
+ A heartfelt thanks to our individual supporters! Every contribution helps us keep our opensource mission alive and thriving!
581
+
582
+ <p align="left">
583
+ <a href="https://github.com/hafezparast"><img src="https://avatars.githubusercontent.com/u/14273305?s=60&v=4" style="border-radius:50%;" width="64px;"/></a>
584
+ <a href="https://github.com/ntohidi"><img src="https://avatars.githubusercontent.com/u/17140097?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
585
+ <a href="https://github.com/Sjoeborg"><img src="https://avatars.githubusercontent.com/u/17451310?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
586
+ <a href="https://github.com/romek-rozen"><img src="https://avatars.githubusercontent.com/u/30595969?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
587
+ <a href="https://github.com/Kourosh-Kiyani"><img src="https://avatars.githubusercontent.com/u/34105600?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
588
+ <a href="https://github.com/Etherdrake"><img src="https://avatars.githubusercontent.com/u/67021215?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
589
+ <a href="https://github.com/shaman247"><img src="https://avatars.githubusercontent.com/u/211010067?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
590
+ <a href="https://github.com/work-flow-manager"><img src="https://avatars.githubusercontent.com/u/217665461?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
591
+ </p>
592
+
593
+ > Want to join them? [Sponsor Crawl4AI →](https://github.com/sponsors/unclecode)
594
+
595
+ ## 📧 Contact
596
+
597
+ [Discord](https://discord.gg/jP8KfhDhyN) · [X @unclecode](https://x.com/unclecode) · [GitHub @unclecode](https://github.com/unclecode) · hello@crawl4ai.com
598
+
599
+ - **Building crawlers or AI agents for a living?** DM me on X. I want to work with people like you, and we are hiring.
600
+ - **From a company?** We have an enterprise offer and we tailor it to your business. SOC 2 Type I is done, Type II is in progress. Write to hello@crawl4ai.com.
601
+
602
+ <p align="center">
603
+ <a href="https://github.com/unclecode/crawl4ai/network/members"><img src="https://img.shields.io/github/forks/unclecode/crawl4ai?style=social" alt="GitHub Forks"/></a>
604
+ <a href="https://pypi.org/project/crawl4ai/"><img src="https://img.shields.io/pypi/pyversions/crawl4ai" alt="Python Version"/></a>
605
+ <a href="https://github.com/sponsors/unclecode"><img src="https://img.shields.io/github/sponsors/unclecode?style=flat&logo=GitHub-Sponsors&label=Sponsors&color=pink" alt="GitHub Sponsors"/></a>
606
+ <a href="https://x.com/crawl4ai"><img src="https://img.shields.io/badge/Follow%20on%20X-000000?style=flat&logo=x&logoColor=white" alt="Follow on X"/></a>
607
+ <a href="https://www.linkedin.com/company/crawl4ai"><img src="https://img.shields.io/badge/LinkedIn-0077B5?style=flat&logo=linkedin&logoColor=white" alt="Follow on LinkedIn"/></a>
608
+ </p>
609
+
610
+ Happy crawling! 🕸️🚀
611
+
612
+ ## Star History
613
+
614
+ <a href="https://www.star-history.com/?type=date&repos=unclecode%2Fcrawl4ai">
615
+ <picture>
616
+ <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=unclecode/crawl4ai&type=date&theme=dark&legend=top-left&sealed_token=KuajrA7ScH8VT4KagC7nm1xbazTVaNs6rdok4At2dV6tDl91YR_dxmHhmsffjhFiWdLYlzdACxZ-cWLwp8tZHCYxSDMjITf3Vnu4mPns7YdLetyQBPHMQ2f_KakXdbvbVP6PofI82GNqGCVEXtPnZWHC8WM6CzZe6s6cJb6_ga_kn-jh-BeHdyuRRdVR" />
617
+ <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=unclecode/crawl4ai&type=date&legend=top-left&sealed_token=KuajrA7ScH8VT4KagC7nm1xbazTVaNs6rdok4At2dV6tDl91YR_dxmHhmsffjhFiWdLYlzdACxZ-cWLwp8tZHCYxSDMjITf3Vnu4mPns7YdLetyQBPHMQ2f_KakXdbvbVP6PofI82GNqGCVEXtPnZWHC8WM6CzZe6s6cJb6_ga_kn-jh-BeHdyuRRdVR" />
618
+ <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=unclecode/crawl4ai&type=date&legend=top-left&sealed_token=KuajrA7ScH8VT4KagC7nm1xbazTVaNs6rdok4At2dV6tDl91YR_dxmHhmsffjhFiWdLYlzdACxZ-cWLwp8tZHCYxSDMjITf3Vnu4mPns7YdLetyQBPHMQ2f_KakXdbvbVP6PofI82GNqGCVEXtPnZWHC8WM6CzZe6s6cJb6_ga_kn-jh-BeHdyuRRdVR" />
619
+ </picture>
620
+ </a>
@@ -29,11 +29,14 @@ crawl4ai/cache_context.py
29
29
  crawl4ai/cache_validator.py
30
30
  crawl4ai/chunking_strategy.py
31
31
  crawl4ai/cli.py
32
+ crawl4ai/cloud_notice.py
32
33
  crawl4ai/config.py
33
34
  crawl4ai/content_filter_strategy.py
35
+ crawl4ai/content_filter_strategy_lxml.py
34
36
  crawl4ai/content_scraping_strategy.py
35
37
  crawl4ai/docker_client.py
36
38
  crawl4ai/domain_mapper.py
39
+ crawl4ai/egress_policy.py
37
40
  crawl4ai/extraction_strategy.py
38
41
  crawl4ai/hub.py
39
42
  crawl4ai/install.py
@@ -108,6 +111,7 @@ tests/test_docker.py
108
111
  tests/test_docker_api_with_llm_provider.py
109
112
  tests/test_docker_pdf_crawler_pairing.py
110
113
  tests/test_eval_security_adversarial.py
114
+ tests/test_http_proxy_error_redaction_2326.py
111
115
  tests/test_http_timeout_unit_1894.py
112
116
  tests/test_issue_1043_mermaid_svg.py
113
117
  tests/test_issue_1213_bm25_dedup.py
@@ -123,6 +127,8 @@ tests/test_issue_1848_logger_serialize.py
123
127
  tests/test_issue_1850_mcp_sse.py
124
128
  tests/test_issue_2127_docker_pdf.py
125
129
  tests/test_issue_2161_overlay_html_body.py
130
+ tests/test_issue_2233_mcp_sse.py
131
+ tests/test_issue_2328_inline_whitespace.py
126
132
  tests/test_link_extractor.py
127
133
  tests/test_llm_extraction_parallel_issue_1055.py
128
134
  tests/test_llm_simple_url.py
@@ -151,6 +157,8 @@ tests/test_raw_html_redirected_url.py
151
157
  tests/test_scraping_strategy.py
152
158
  tests/test_source_sibling_selector.py
153
159
  tests/test_table_gfm_compliance.py
160
+ tests/test_table_rowspan_grid.py
154
161
  tests/test_type_annotations.py
162
+ tests/test_unclosed_noscript.py
155
163
  tests/test_virtual_scroll.py
156
164
  tests/test_web_crawler.py