Crawl4AI 0.9.4__tar.gz → 0.9.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. crawl4ai-0.9.6/Crawl4AI.egg-info/PKG-INFO +690 -0
  2. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/Crawl4AI.egg-info/SOURCES.txt +17 -0
  3. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/Crawl4AI.egg-info/requires.txt +3 -3
  4. crawl4ai-0.9.6/PKG-INFO +690 -0
  5. crawl4ai-0.9.6/README.md +607 -0
  6. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/__version__.py +1 -1
  7. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/adaptive_crawler.py +3 -2
  8. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/async_configs.py +234 -11
  9. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/async_crawler_strategy.py +79 -16
  10. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/async_database.py +5 -8
  11. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/async_url_seeder.py +81 -17
  12. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/async_webcrawler.py +41 -6
  13. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/browser_manager.py +91 -23
  14. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/cache_validator.py +5 -0
  15. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/cli.py +21 -8
  16. crawl4ai-0.9.6/crawl4ai/cloud_notice.py +139 -0
  17. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/config.py +4 -7
  18. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/content_filter_strategy.py +38 -0
  19. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/content_filter_strategy_lxml.py +5 -0
  20. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/content_scraping_strategy.py +61 -43
  21. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/deep_crawling/base_strategy.py +28 -10
  22. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/deep_crawling/bff_strategy.py +28 -8
  23. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/deep_crawling/bfs_strategy.py +39 -6
  24. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/deep_crawling/dfs_strategy.py +14 -2
  25. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/deep_crawling/filters.py +21 -11
  26. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/docker_client.py +56 -25
  27. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/domain_mapper.py +2 -1
  28. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/extraction_strategy.py +15 -42
  29. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/html2text/__init__.py +3 -7
  30. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/install.py +82 -58
  31. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/markdown_generation_strategy.py +10 -2
  32. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/models.py +11 -2
  33. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/processors/pdf/__init__.py +90 -25
  34. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/processors/pdf/processor.py +16 -14
  35. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/ssl_certificate.py +34 -4
  36. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/table_extraction.py +2 -1
  37. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/user_agent_generator.py +56 -6
  38. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/utils.py +71 -20
  39. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/pyproject.toml +3 -3
  40. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/requirements.txt +3 -3
  41. crawl4ai-0.9.6/tests/test_base_href_markdown.py +53 -0
  42. crawl4ai-0.9.6/tests/test_base_href_relative_links.py +53 -0
  43. crawl4ai-0.9.6/tests/test_cache_processing_errors.py +264 -0
  44. crawl4ai-0.9.6/tests/test_cdp_cache_stale_release.py +149 -0
  45. crawl4ai-0.9.6/tests/test_content_type_filter_query.py +44 -0
  46. crawl4ai-0.9.6/tests/test_default_llm_provider.py +54 -0
  47. crawl4ai-0.9.6/tests/test_format_html_parser.py +30 -0
  48. crawl4ai-0.9.6/tests/test_http_proxy_error_redaction_2326.py +93 -0
  49. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1837_config_list.py +8 -6
  50. crawl4ai-0.9.6/tests/test_issue_2233_mcp_sse.py +96 -0
  51. crawl4ai-0.9.6/tests/test_issue_2328_inline_whitespace.py +38 -0
  52. crawl4ai-0.9.6/tests/test_issue_2416_tail_text.py +105 -0
  53. crawl4ai-0.9.6/tests/test_issue_2490_cache_failed_crawl.py +186 -0
  54. crawl4ai-0.9.6/tests/test_issue_2493_user_agent.py +159 -0
  55. crawl4ai-0.9.6/tests/test_normalize_url_userinfo.py +62 -0
  56. crawl4ai-0.9.6/tests/test_unclosed_noscript.py +113 -0
  57. crawl4ai-0.9.6/tests/test_user_agent_browser_type.py +59 -0
  58. crawl4ai-0.9.4/Crawl4AI.egg-info/PKG-INFO +0 -1416
  59. crawl4ai-0.9.4/PKG-INFO +0 -1416
  60. crawl4ai-0.9.4/README.md +0 -1333
  61. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/Crawl4AI.egg-info/dependency_links.txt +0 -0
  62. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/Crawl4AI.egg-info/entry_points.txt +0 -0
  63. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/Crawl4AI.egg-info/top_level.txt +0 -0
  64. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/LICENSE +0 -0
  65. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/MANIFEST.in +0 -0
  66. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/__init__.py +0 -0
  67. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/antibot_detector.py +0 -0
  68. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/async_dispatcher.py +0 -0
  69. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/async_logger.py +0 -0
  70. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/browser_adapter.py +0 -0
  71. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/browser_profiler.py +0 -0
  72. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/cache_context.py +0 -0
  73. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/chunking_strategy.py +0 -0
  74. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/cloud/__init__.py +0 -0
  75. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/cloud/cli.py +0 -0
  76. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/components/crawler_monitor.py +0 -0
  77. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/crawlers/__init__.py +0 -0
  78. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/crawlers/amazon_product/__init__.py +0 -0
  79. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/crawlers/amazon_product/crawler.py +0 -0
  80. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/crawlers/google_search/__init__.py +0 -0
  81. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/crawlers/google_search/crawler.py +0 -0
  82. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/crawlers/google_search/script.js +0 -0
  83. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/deep_crawling/__init__.py +0 -0
  84. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/deep_crawling/crazy.py +0 -0
  85. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/deep_crawling/scorers.py +0 -0
  86. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/egress_policy.py +0 -0
  87. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/html2text/__main__.py +0 -0
  88. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/html2text/_typing.py +0 -0
  89. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/html2text/cli.py +0 -0
  90. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/html2text/config.py +0 -0
  91. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/html2text/elements.py +0 -0
  92. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/html2text/utils.py +0 -0
  93. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/hub.py +0 -0
  94. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/js_snippet/__init__.py +0 -0
  95. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/js_snippet/flatten_shadow_dom.js +0 -0
  96. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/js_snippet/navigator_overrider.js +0 -0
  97. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/js_snippet/remove_consent_popups.js +0 -0
  98. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/js_snippet/remove_overlay_elements.js +0 -0
  99. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/js_snippet/update_image_dimensions.js +0 -0
  100. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/legacy/__init__.py +0 -0
  101. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/legacy/cli.py +0 -0
  102. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/legacy/crawler_strategy.py +0 -0
  103. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/legacy/database.py +0 -0
  104. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/legacy/docs_manager.py +0 -0
  105. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/legacy/llmtxt.py +0 -0
  106. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/legacy/version_manager.py +0 -0
  107. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/legacy/web_crawler.py +0 -0
  108. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/link_preview.py +0 -0
  109. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/migrations.py +0 -0
  110. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/model_loader.py +0 -0
  111. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/processors/pdf/utils.py +0 -0
  112. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/prompts.py +0 -0
  113. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/proxy_strategy.py +0 -0
  114. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/script/__init__.py +0 -0
  115. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/script/c4a_compile.py +0 -0
  116. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/script/c4a_result.py +0 -0
  117. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/script/c4ai_script.py +0 -0
  118. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/crawl4ai/types.py +0 -0
  119. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/setup.cfg +0 -0
  120. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/setup.py +0 -0
  121. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_arun_many.py +0 -0
  122. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_async_logger_stderr.py +0 -0
  123. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_body_visibility_warning.py +0 -0
  124. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_bug_batch_1622_1786_1796.py +0 -0
  125. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_cdp_changes.py +0 -0
  126. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_cli_docs.py +0 -0
  127. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_cloud_bugs_batch.py +0 -0
  128. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_config_defaults.py +0 -0
  129. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_config_matching_only.py +0 -0
  130. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_config_selection.py +0 -0
  131. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_docker.py +0 -0
  132. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_docker_api_with_llm_provider.py +0 -0
  133. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_docker_pdf_crawler_pairing.py +0 -0
  134. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_eval_security_adversarial.py +0 -0
  135. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_http_timeout_unit_1894.py +0 -0
  136. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1043_mermaid_svg.py +0 -0
  137. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1213_bm25_dedup.py +0 -0
  138. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1370_1818_1762_1509.py +0 -0
  139. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1484_css_selector.py +0 -0
  140. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1594_mcp_sse.py +0 -0
  141. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1611_llm_provider.py +0 -0
  142. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1748_screenshot_scroll_delay.py +0 -0
  143. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1750_screenshot_scan_full_page.py +0 -0
  144. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1842_browser_none.py +0 -0
  145. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1848_logger_serialize.py +0 -0
  146. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_1850_mcp_sse.py +0 -0
  147. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_2127_docker_pdf.py +0 -0
  148. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_issue_2161_overlay_html_body.py +0 -0
  149. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_link_extractor.py +0 -0
  150. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_llm_extraction_parallel_issue_1055.py +0 -0
  151. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_llm_simple_url.py +0 -0
  152. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_llmtxt.py +0 -0
  153. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_main.py +0 -0
  154. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_markdown_generator_validation_1880.py +0 -0
  155. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_memory_macos.py +0 -0
  156. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_merge_head_data_scoring.py +0 -0
  157. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_multi_config.py +0 -0
  158. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_normalize_url.py +0 -0
  159. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_placeholder_html_antibot.py +0 -0
  160. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_pr_1290_1668.py +0 -0
  161. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_pr_1435_redirected_status_code.py +0 -0
  162. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_pr_1463_device_scale_factor.py +0 -0
  163. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_pr_1795_1798_1734.py +0 -0
  164. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_prefetch_integration.py +0 -0
  165. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_prefetch_mode.py +0 -0
  166. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_prefetch_regression.py +0 -0
  167. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_preserve_https_for_internal_links.py +0 -0
  168. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_pruning_preserve_whitelist_1900.py +0 -0
  169. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_pyopenssl_security_fix.py +0 -0
  170. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_pyopenssl_update.py +0 -0
  171. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_raw_html_browser.py +0 -0
  172. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_raw_html_edge_cases.py +0 -0
  173. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_raw_html_redirected_url.py +0 -0
  174. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_scraping_strategy.py +0 -0
  175. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_source_sibling_selector.py +0 -0
  176. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_table_gfm_compliance.py +0 -0
  177. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_table_rowspan_grid.py +0 -0
  178. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_type_annotations.py +0 -0
  179. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_virtual_scroll.py +0 -0
  180. {crawl4ai-0.9.4 → crawl4ai-0.9.6}/tests/test_web_crawler.py +0 -0
@@ -0,0 +1,690 @@
1
+ Metadata-Version: 2.4
2
+ Name: Crawl4AI
3
+ Version: 0.9.6
4
+ Summary: 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper
5
+ Home-page: https://github.com/unclecode/crawl4ai
6
+ Author: Unclecode
7
+ Author-email: Unclecode <unclecode@kidocode.com>
8
+ License-Expression: Apache-2.0
9
+ Classifier: Development Status :: 4 - Beta
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Requires-Python: >=3.10
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: aiofiles>=24.1.0
20
+ Requires-Dist: aiohttp>=3.11.11
21
+ Requires-Dist: aiosqlite~=0.20
22
+ Requires-Dist: anyio>=4.0.0
23
+ Requires-Dist: lxml<7,>=5.3
24
+ Requires-Dist: unclecode-litellm==1.81.13
25
+ Requires-Dist: numpy<3,>=1.26.0
26
+ Requires-Dist: pillow>=10.4
27
+ Requires-Dist: playwright>=1.64.1
28
+ Requires-Dist: patchright>=1.64.2
29
+ Requires-Dist: python-dotenv~=1.0
30
+ Requires-Dist: requests~=2.26
31
+ Requires-Dist: beautifulsoup4~=4.12
32
+ Requires-Dist: playwright-stealth>=2.0.0
33
+ Requires-Dist: xxhash~=3.4
34
+ Requires-Dist: rank-bm25~=0.2
35
+ Requires-Dist: snowballstemmer~=2.2
36
+ Requires-Dist: pydantic>=2.10
37
+ Requires-Dist: pyOpenSSL>=25.3.0
38
+ Requires-Dist: psutil>=6.1.1
39
+ Requires-Dist: PyYAML>=6.0
40
+ Requires-Dist: nltk>=3.9.1
41
+ Requires-Dist: rich>=13.9.4
42
+ Requires-Dist: cssselect>=1.2.0
43
+ Requires-Dist: httpx>=0.27.2
44
+ Requires-Dist: httpx[http2]>=0.27.2
45
+ Requires-Dist: fake-useragent>=2.0.3
46
+ Requires-Dist: click>=8.1.7
47
+ Requires-Dist: chardet>=5.2.0
48
+ Requires-Dist: brotli>=1.2.1
49
+ Requires-Dist: humanize>=4.10.0
50
+ Requires-Dist: lark>=1.2.2
51
+ Requires-Dist: alphashape>=1.3.1
52
+ Requires-Dist: shapely>=2.0.0
53
+ Provides-Extra: pdf
54
+ Requires-Dist: pypdf; extra == "pdf"
55
+ Provides-Extra: torch
56
+ Requires-Dist: torch; extra == "torch"
57
+ Requires-Dist: nltk; extra == "torch"
58
+ Requires-Dist: scikit-learn; extra == "torch"
59
+ Provides-Extra: transformer
60
+ Requires-Dist: transformers; extra == "transformer"
61
+ Requires-Dist: tokenizers; extra == "transformer"
62
+ Requires-Dist: sentence-transformers; extra == "transformer"
63
+ Provides-Extra: cosine
64
+ Requires-Dist: torch; extra == "cosine"
65
+ Requires-Dist: transformers; extra == "cosine"
66
+ Requires-Dist: nltk; extra == "cosine"
67
+ Requires-Dist: sentence-transformers; extra == "cosine"
68
+ Provides-Extra: sync
69
+ Requires-Dist: selenium; extra == "sync"
70
+ Provides-Extra: all
71
+ Requires-Dist: pypdf; extra == "all"
72
+ Requires-Dist: torch; extra == "all"
73
+ Requires-Dist: nltk; extra == "all"
74
+ Requires-Dist: scikit-learn; extra == "all"
75
+ Requires-Dist: transformers; extra == "all"
76
+ Requires-Dist: tokenizers; extra == "all"
77
+ Requires-Dist: sentence-transformers; extra == "all"
78
+ Requires-Dist: selenium; extra == "all"
79
+ Dynamic: author
80
+ Dynamic: home-page
81
+ Dynamic: license-file
82
+ Dynamic: requires-python
83
+
84
+ <h1><a href="https://trendshift.io/repositories/11716" target="_blank"><img align="right" src="https://trendshift.io/api/badge/repositories/11716" alt="unclecode%2Fcrawl4ai | Trendshift" width="130"/></a>Crawl4AI</h1>
85
+
86
+ **Crawl4AI Cloud is a hosted web scraping, extraction, and search API for AI agents and data pipelines.** Turn pages into clean Markdown for RAG, extract structured JSON, or search the live web with one API key. Crawl4AI runs the browsers and manages the proxies.
87
+
88
+ **[Get a free API key](https://crawl4ai.com/?ref=readme)** · 1,000 free credits every month. No credit card required.
89
+
90
+ [Try a URL in the playground](https://crawl4ai.com/?ref=readme-playground#playground) · [Set up Crawl4AI Cloud in Claude Code, Cursor, or Codex](https://crawl4ai.com/llms.txt?ref=readme-llms)
91
+
92
+ This repository is home to the Apache-2.0 crawler that powers Crawl4AI Cloud. [Run it yourself](#run-crawl4ai-yourself) with Python or Docker.
93
+
94
+ <a href="https://crawl4ai.com/?ref=readme-banner">
95
+ <picture>
96
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/cloud-launch-banner-dark.svg">
97
+ <img alt="Crawl4AI Cloud is live. 1,000 free credits every month, no card. Get your key." src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/cloud-launch-banner-light.svg" width="960">
98
+ </picture>
99
+ </a>
100
+
101
+ <div align="center">
102
+
103
+ [![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers)
104
+ [![PyPI version](https://badge.fury.io/py/crawl4ai.svg)](https://badge.fury.io/py/crawl4ai)
105
+ [![Downloads](https://static.pepy.tech/badge/crawl4ai/month)](https://pepy.tech/project/crawl4ai)
106
+ [![Discord](https://img.shields.io/badge/Discord-join%20us-5865F2?logo=discord&logoColor=white)](https://discord.gg/jP8KfhDhyN)
107
+
108
+ </div>
109
+
110
+ ## Cloud quickstart
111
+
112
+ Hosted API base URL: `https://api.crawl4ai.com`.
113
+
114
+ [Cloud API reference](https://api.crawl4ai.com/docs) · [Agent setup and MCP](https://crawl4ai.com/llms.txt?ref=readme-llms)
115
+
116
+ Set `CRAWL4AI_KEY` to your API key, then scrape a page as Markdown:
117
+
118
+ ```bash
119
+ curl -s https://api.crawl4ai.com/scrape \
120
+ -H "Authorization: Bearer $CRAWL4AI_KEY" \
121
+ -H "Content-Type: application/json" \
122
+ -d '{"url": "https://news.ycombinator.com"}' | jq -r .markdown
123
+ ```
124
+
125
+ <details>
126
+ <summary>Python 3</summary>
127
+
128
+ ```python
129
+ import json
130
+ import os
131
+ from urllib.request import Request, urlopen
132
+
133
+ request = Request(
134
+ "https://api.crawl4ai.com/scrape",
135
+ data=json.dumps({"url": "https://news.ycombinator.com"}).encode("utf-8"),
136
+ headers={
137
+ "Authorization": f"Bearer {os.environ['CRAWL4AI_KEY']}",
138
+ "Content-Type": "application/json",
139
+ },
140
+ method="POST",
141
+ )
142
+
143
+ with urlopen(request) as response:
144
+ print(json.load(response)["markdown"])
145
+ ```
146
+
147
+ </details>
148
+
149
+ <details>
150
+ <summary>Node.js (18+)</summary>
151
+
152
+ ```javascript
153
+ async function main() {
154
+ const key = process.env.CRAWL4AI_KEY;
155
+ if (!key) throw new Error("Set CRAWL4AI_KEY first.");
156
+
157
+ const response = await fetch("https://api.crawl4ai.com/scrape", {
158
+ method: "POST",
159
+ headers: {
160
+ Authorization: `Bearer ${key}`,
161
+ "Content-Type": "application/json",
162
+ },
163
+ body: JSON.stringify({ url: "https://news.ycombinator.com" }),
164
+ });
165
+
166
+ if (!response.ok) {
167
+ throw new Error(`Scrape failed (HTTP ${response.status}): ${await response.text()}`);
168
+ }
169
+
170
+ const { markdown } = await response.json();
171
+ console.log(markdown);
172
+ }
173
+
174
+ main().catch((error) => {
175
+ console.error(error.message);
176
+ process.exitCode = 1;
177
+ });
178
+ ```
179
+
180
+ </details>
181
+
182
+ Example Markdown output (illustrative excerpt):
183
+
184
+ ```markdown
185
+ # Hacker News
186
+
187
+ 1. [Example story title](https://example.com/story)
188
+ 2. [Another story title](https://example.com/another-story)
189
+ ```
190
+
191
+ The same key works for `/extract`, `/search`, `/serp`, `/answer` and many URLs at once (`/scrape/batch`, `/scrape/jobs`). Pay as you go: [live prices](https://crawl4ai.com/docs?ref=readme#pricing).
192
+ For Claude Code, connect to the hosted MCP server:
193
+
194
+ ```bash
195
+ claude mcp add --transport http crawl4ai https://api.crawl4ai.com/mcp \
196
+ --header "Authorization: Bearer $CRAWL4AI_KEY"
197
+ ```
198
+
199
+ ## Run Crawl4AI yourself
200
+
201
+ **Latest: [v0.9.6](https://github.com/unclecode/crawl4ai/releases/tag/v0.9.6) (20 Nov 2026)** · [all releases →](https://github.com/unclecode/crawl4ai/releases)
202
+
203
+
204
+ ```bash
205
+ pip install -U crawl4ai
206
+ crawl4ai-setup # installs the browser, once
207
+ ```
208
+
209
+ ```python
210
+ import asyncio
211
+ from crawl4ai import AsyncWebCrawler
212
+
213
+ async def main():
214
+ async with AsyncWebCrawler() as crawler:
215
+ result = await crawler.arun(url="https://news.ycombinator.com")
216
+ print(result.markdown)
217
+
218
+ asyncio.run(main())
219
+ ```
220
+
221
+ Docker server, CLI and every option: [Installation](#installation) · [Self-hosted docs](https://docs.crawl4ai.com)
222
+
223
+ ### Which one?
224
+
225
+ | | ☁️ Crawl4AI Cloud | 🐍 Library | 🐳 Your own server |
226
+ |---|---|---|---|
227
+ | **Runs the browsers** | we do | you, in your Python process | you, in Docker on your machine |
228
+ | **JS-heavy pages and bot walls** | handled for you, automatically | your settings, your proxies | your settings, your proxies |
229
+ | **Web search** | `/search`, `/serp` and `/answer` | – | – |
230
+ | **Price** | 1,000 free credits a month; plans from $10 | Free, Apache-2.0. You cover infrastructure. | Free, Apache-2.0. You cover hosting. |
231
+
232
+ <details>
233
+ <summary>🤓 <strong>My Personal Story</strong></summary>
234
+
235
+ I grew up on an Amstrad, thanks to my dad, and never stopped building. In grad school I specialized in NLP and built crawlers for research. That’s where I learned how much extraction matters.
236
+
237
+ In 2023, I needed web-to-Markdown. The “open source” option wanted an account, API token, and $16, and still under-delivered. I went turbo anger mode, built Crawl4AI in days, and it went viral.
238
+
239
+ I made it open source for **availability**, anyone can use it without a gate. Now I’m building the platform for **affordability**, anyone can run serious crawls without breaking the bank. If that resonates, join in, send feedback, or just crawl something amazing.
240
+
241
+ That platform is live now: [Crawl4AI Cloud](https://crawl4ai.com/?ref=readme).
242
+ </details>
243
+
244
+ <details>
245
+ <summary>Why developers pick Crawl4AI</summary>
246
+
247
+ - **LLM-ready output**: smart Markdown with headings, tables, code and citation hints
248
+ - **Fast in practice**: async browser pool, caching, minimal hops
249
+ - **Full control**: sessions, proxies, cookies, user scripts, hooks
250
+ - **Adaptive intelligence**: learns site patterns, explores only what matters
251
+ - **Deploy anywhere**: run locally with Python or the CLI, secure your Docker server with a self-managed token, or use a Crawl4AI Cloud API key.
252
+ </details>
253
+
254
+ ## ✨ Features
255
+
256
+ <details>
257
+ <summary>📝 <strong>Markdown generation</strong></summary>
258
+
259
+ - 🧹 **Clean Markdown**: headings, lists, tables and code blocks, in a structure an LLM reads well.
260
+ - 🎯 **Fit Markdown**: filters remove menus, footers and boilerplate: `PruningContentFilterLXML`, `BM25ContentFilter` (for a query) and `LLMContentFilter`.
261
+ - 🔗 **Citations**: page links become a numbered reference list.
262
+ - 🛠️ **Your own strategy**: plug in a custom Markdown generator.
263
+
264
+ ☁️ Same in the cloud: `POST /scrape` returns this Markdown, with no browser to run. [Docs →](https://crawl4ai.com/docs?ref=readme#scrape)
265
+ </details>
266
+
267
+ <details>
268
+ <summary>📊 <strong>Structured data extraction</strong></summary>
269
+
270
+ - 🔎 **CSS and XPath schemas**: fast extraction with no LLM (`JsonCssExtractionStrategy`, `JsonXPathExtractionStrategy`, `RegexExtractionStrategy`).
271
+ - 🪄 **Schema generator**: describe what you want once; `generate_schema` writes a reusable schema.
272
+ - 🤖 **LLM extraction**: any LLM provider, open-source or hosted, into a typed JSON schema (`LLMExtractionStrategy`).
273
+ - 🧱 **Chunking**: topic, regex and sentence chunking for long pages.
274
+ - 🌌 **Cosine similarity**: find the chunks that match a query (`CosineStrategy`).
275
+
276
+ ☁️ Same in the cloud: `POST /extract`, with no LLM key of your own. [Docs →](https://crawl4ai.com/docs?ref=readme#extract)
277
+ </details>
278
+
279
+ <details>
280
+ <summary>🌐 <strong>Browser control</strong></summary>
281
+
282
+ - 🖥️ **Your own browser**: persistent profiles with saved logins, cookies and settings.
283
+ - 🔄 **Remote browsers**: connect over the Chrome DevTools Protocol (CDP).
284
+ - 🔒 **Sessions**: keep a browser state across multi-step crawls.
285
+ - 🧩 **Proxies**: with authentication and rotation.
286
+ - 🕶️ **Stealth mode**: `enable_stealth`, and an undetected-browser adapter for sites that detect automation.
287
+ - ⚙️ **Full control**: headers, cookies, user agents, viewport.
288
+ - 🌍 **Chromium, Firefox and WebKit**.
289
+ </details>
290
+
291
+ <details>
292
+ <summary>🔎 <strong>Crawling and scraping</strong></summary>
293
+
294
+ - 🕸️ **Deep crawl**: BFS, DFS and best-first strategies, with crash recovery (`resume_state`) for long crawls.
295
+ - 🧠 **Adaptive crawling**: `AdaptiveCrawler` stops when it has learned enough to answer your query.
296
+ - 🌱 **URL discovery**: `AsyncUrlSeeder` (sitemaps, Common Crawl) and `DomainMapper`; `prefetch=True` finds URLs 5 to 10 times faster.
297
+ - 🚀 **Dynamic pages**: run JavaScript, wait for elements, scroll the full page (`scan_full_page`) for infinite scroll and lazy images.
298
+ - 📸 **Screenshots and PDFs** of any page.
299
+ - 🖼️ **Media and links**: images, audio, video, `srcset`, internal and external links, iframes, metadata.
300
+ - 📂 **Raw HTML and local files**: `raw:` and `file://`.
301
+ - 🛠️ **Hooks** at every step of a crawl.
302
+ - 💾 **Caching** to skip repeated fetches.
303
+ - ⚡ **Many URLs at once**: `arun_many` with a memory-adaptive dispatcher.
304
+
305
+ ☁️ Same in the cloud: up to 50 URLs in one streamed call, or 10,000 in a background job. [Docs →](https://crawl4ai.com/docs?ref=readme#batch)
306
+ </details>
307
+
308
+ <details>
309
+ <summary>🐳 <strong>Self-hosting (Docker)</strong></summary>
310
+
311
+ - 🔐 **Secure by default**: every endpoint needs your `CRAWL4AI_API_TOKEN`.
312
+ - 🧰 **REST API**: `/md`, `/html`, `/crawl`, `/crawl/stream`, `/screenshot`, `/pdf`, `/execute_js`.
313
+ - 🤖 **MCP**: connect Claude Code and other agents to your own server.
314
+ - 📊 **Monitoring dashboard and playground**, a browser pool with pre-warmed pages.
315
+ - 🏗️ **AMD64 and ARM64** images.
316
+
317
+ ☁️ Rather not run a server? The cloud is the same idea, hosted. [Get a key →](https://crawl4ai.com/?ref=readme)
318
+ </details>
319
+
320
+ <details>
321
+ <summary>☁️ <strong>What the cloud adds</strong></summary>
322
+
323
+ - 🔍 **Web search API**: `GET /search`, browser-free, ranked and cleaned. [Docs →](https://crawl4ai.com/docs?ref=readme#search)
324
+ - 📊 **SERP API**: `GET /serp`, Google's results page as data: true ranks, every block, localised to a place. Beta. [Docs →](https://crawl4ai.com/docs?ref=readme#serp)
325
+ - 💬 **Answers**: `GET /answer` gives a direct answer to a question (experimental). [Docs →](https://crawl4ai.com/docs?ref=readme#answer)
326
+ - 🧪 **Extraction without your own LLM key**: `POST /extract`. [Docs →](https://crawl4ai.com/docs?ref=readme#extract)
327
+ - 🧗 **JS-heavy pages and bot walls**: handled automatically; you never pick an engine. [Docs →](https://crawl4ai.com/docs?ref=readme#scrape)
328
+ - 🤝 **MCP for your agent**: one line in Claude Code, Codex, Cursor or OpenCode. [Docs →](https://crawl4ai.com/docs?ref=readme#mcp)
329
+ </details>
330
+
331
+ <a id="installation"></a>
332
+
333
+ ## 🛠️ Installation
334
+
335
+ <details>
336
+ <summary>🐍 <strong>pip</strong></summary>
337
+
338
+ ```bash
339
+ pip install -U crawl4ai
340
+ crawl4ai-setup # installs and sets up the browser
341
+ crawl4ai-doctor # checks the installation
342
+ ```
343
+
344
+ If the browser setup fails, install it by hand:
345
+
346
+ ```bash
347
+ python -m playwright install --with-deps chromium
348
+ ```
349
+
350
+ Pre-release versions: `pip install crawl4ai --pre`
351
+
352
+ **Development install**, for contributors:
353
+
354
+ ```bash
355
+ git clone https://github.com/unclecode/crawl4ai.git
356
+ cd crawl4ai
357
+ pip install -e ".[all]" # or: pip install -e . (the core only)
358
+ ```
359
+ </details>
360
+
361
+ <details>
362
+ <summary>🐳 <strong>Docker server</strong></summary>
363
+
364
+ The server needs a token. Without one it answers only inside its container.
365
+
366
+ ```bash
367
+ export CRAWL4AI_API_TOKEN="$(openssl rand -hex 32)"
368
+ docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g \
369
+ -e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN" \
370
+ unclecode/crawl4ai:latest
371
+ ```
372
+
373
+ Test it (allow about 10 seconds for the start):
374
+
375
+ ```bash
376
+ curl -s http://localhost:11235/md \
377
+ -H "Authorization: Bearer $CRAWL4AI_API_TOKEN" \
378
+ -H "Content-Type: application/json" \
379
+ -d '{"url": "https://news.ycombinator.com"}' | jq -r .markdown
380
+ ```
381
+
382
+ The dashboard is at `http://localhost:11235/dashboard`, the playground at `http://localhost:11235/playground`. LLM keys, MCP and every setting: [Self-hosting guide](https://docs.crawl4ai.com/core/self-hosting/).
383
+ </details>
384
+
385
+ <details>
386
+ <summary>⌨️ <strong>Command line (`crwl`)</strong></summary>
387
+
388
+ ```bash
389
+ # A page as Markdown
390
+ crwl https://news.ycombinator.com -o markdown
391
+
392
+ # Deep crawl, breadth first, at most 10 pages
393
+ crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10
394
+
395
+ # Ask a question about a page (needs an LLM key: crwl config)
396
+ crwl https://www.example.com/products -q "Extract all product prices"
397
+ ```
398
+ </details>
399
+
400
+ ## 🔬 Advanced usage examples
401
+
402
+ More in [docs/examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples).
403
+
404
+ <details>
405
+ <summary>📝 <strong>Clean and fit Markdown</strong></summary>
406
+
407
+ ```python
408
+ import asyncio
409
+ from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
410
+ from crawl4ai.content_filter_strategy import PruningContentFilterLXML
411
+ from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
412
+
413
+ async def main():
414
+ run_config = CrawlerRunConfig(
415
+ cache_mode=CacheMode.BYPASS,
416
+ markdown_generator=DefaultMarkdownGenerator(
417
+ content_filter=PruningContentFilterLXML(threshold=0.48, threshold_type="fixed", min_word_threshold=0)
418
+ ),
419
+ )
420
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
421
+ result = await crawler.arun(url="https://en.wikipedia.org/wiki/Web_crawler", config=run_config)
422
+ print(len(result.markdown.raw_markdown), "characters of raw Markdown")
423
+ print(len(result.markdown.fit_markdown), "characters after the filter")
424
+
425
+ asyncio.run(main())
426
+ ```
427
+ </details>
428
+
429
+ <details>
430
+ <summary>🖥️ <strong>A JavaScript page and structured data, without an LLM</strong></summary>
431
+
432
+ ```python
433
+ import asyncio, json
434
+ from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, JsonCssExtractionStrategy
435
+
436
+ schema = {
437
+ "name": "Quotes",
438
+ "baseSelector": "div.quote",
439
+ "fields": [
440
+ {"name": "text", "selector": "span.text", "type": "text"},
441
+ {"name": "author", "selector": "small.author", "type": "text"},
442
+ {"name": "tags", "selector": "a.tag", "type": "list", "fields": [{"name": "tag", "type": "text"}]},
443
+ ],
444
+ }
445
+
446
+ async def main():
447
+ run_config = CrawlerRunConfig(
448
+ extraction_strategy=JsonCssExtractionStrategy(schema),
449
+ scan_full_page=True, # scroll to the end, so the page loads every quote
450
+ scroll_delay=0.5,
451
+ cache_mode=CacheMode.BYPASS,
452
+ )
453
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
454
+ result = await crawler.arun(url="https://quotes.toscrape.com/scroll", config=run_config)
455
+ quotes = json.loads(result.extracted_content)
456
+ print(f"Extracted {len(quotes)} quotes")
457
+ print(json.dumps(quotes[0], indent=2))
458
+
459
+ asyncio.run(main())
460
+ ```
461
+ </details>
462
+
463
+ <details>
464
+ <summary>📚 <strong>Structured data with an LLM</strong></summary>
465
+
466
+ ```python
467
+ import os, asyncio
468
+ from pydantic import BaseModel, Field
469
+ from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode, LLMConfig, LLMExtractionStrategy
470
+
471
+ class ModelFee(BaseModel):
472
+ model_name: str = Field(..., description="Name of the model.")
473
+ input_fee: str = Field(..., description="Fee for input tokens.")
474
+ output_fee: str = Field(..., description="Fee for output tokens.")
475
+
476
+ async def main():
477
+ run_config = CrawlerRunConfig(
478
+ cache_mode=CacheMode.BYPASS,
479
+ extraction_strategy=LLMExtractionStrategy(
480
+ # any provider LiteLLM supports, e.g. "ollama/llama3.3" with api_token="no-token"
481
+ llm_config=LLMConfig(provider="openai/gpt-6-luna", api_token=os.getenv("OPENAI_API_KEY")),
482
+ schema=ModelFee.model_json_schema(),
483
+ extraction_type="schema",
484
+ instruction="Extract every model name with its input and output token fee.",
485
+ ),
486
+ )
487
+ async with AsyncWebCrawler() as crawler:
488
+ result = await crawler.arun(url="https://openai.com/api/pricing/", config=run_config)
489
+ print(result.extracted_content)
490
+
491
+ asyncio.run(main())
492
+ ```
493
+ </details>
494
+
495
+ <details>
496
+ <summary>🤖 <strong>Your own browser with a saved profile</strong></summary>
497
+
498
+ ```python
499
+ import os, asyncio
500
+ from pathlib import Path
501
+ from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
502
+
503
+ async def main():
504
+ user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile")
505
+ os.makedirs(user_data_dir, exist_ok=True)
506
+ browser_config = BrowserConfig(headless=True, user_data_dir=user_data_dir, use_persistent_context=True)
507
+ run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, magic=True)
508
+ async with AsyncWebCrawler(config=browser_config) as crawler:
509
+ result = await crawler.arun(url="ADDRESS_OF_A_CHALLENGING_WEBSITE", config=run_config)
510
+ print(result.success, len(result.markdown))
511
+
512
+ asyncio.run(main())
513
+ ```
514
+ </details>
515
+
516
+ ## 📖 Documentation
517
+
518
+ - Library docs, guides and API reference: [docs.crawl4ai.com](https://docs.crawl4ai.com/)
519
+ - Cloud docs: [crawl4ai.com/docs](https://crawl4ai.com/docs?ref=readme)
520
+ - Release notes: [releases](https://github.com/unclecode/crawl4ai/releases) · Roadmap: [ROADMAP.md](https://github.com/unclecode/crawl4ai/blob/main/ROADMAP.md)
521
+
522
+ ## 🤝 Contributing
523
+
524
+ We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTORS.md) for more information.
525
+
526
+ ## 📄 License & Attribution
527
+
528
+ This project is licensed under the Apache License 2.0, attribution is recommended via the badges below. See the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) file for details.
529
+
530
+ ### Attribution Requirements
531
+ When using Crawl4AI, you must include one of the following attribution methods:
532
+
533
+ <details>
534
+ <summary>📈 <strong>1. Badge Attribution (Recommended)</strong></summary>
535
+ Add one of these badges to your README, documentation, or website:
536
+
537
+ | Theme | Badge |
538
+ |-------|-------|
539
+ | **Disco Theme (Animated)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/></a> |
540
+ | **Night Theme (Dark with Neon)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/></a> |
541
+ | **Dark Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/></a> |
542
+ | **Light Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/></a> |
543
+
544
+
545
+ HTML code for adding the badges:
546
+ ```html
547
+ <!-- Disco Theme (Animated) -->
548
+ <a href="https://github.com/unclecode/crawl4ai">
549
+ <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/>
550
+ </a>
551
+
552
+ <!-- Night Theme (Dark with Neon) -->
553
+ <a href="https://github.com/unclecode/crawl4ai">
554
+ <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/>
555
+ </a>
556
+
557
+ <!-- Dark Theme (Classic) -->
558
+ <a href="https://github.com/unclecode/crawl4ai">
559
+ <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/>
560
+ </a>
561
+
562
+ <!-- Light Theme (Classic) -->
563
+ <a href="https://github.com/unclecode/crawl4ai">
564
+ <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/>
565
+ </a>
566
+
567
+ <!-- Simple Shield Badge -->
568
+ <a href="https://github.com/unclecode/crawl4ai">
569
+ <img src="https://img.shields.io/badge/Powered%20by-Crawl4AI-blue?style=flat-square" alt="Powered by Crawl4AI"/>
570
+ </a>
571
+ ```
572
+
573
+ </details>
574
+
575
+ <details>
576
+ <summary>📖 <strong>2. Text Attribution</strong></summary>
577
+ Add this line to your documentation:
578
+ ```
579
+ This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.
580
+ ```
581
+ </details>
582
+
583
+ ## 📚 Citation
584
+
585
+ If you use Crawl4AI in your research or project, please cite:
586
+
587
+ ```bibtex
588
+ @software{crawl4ai2024,
589
+ author = {UncleCode},
590
+ title = {Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper},
591
+ year = {2024},
592
+ publisher = {GitHub},
593
+ journal = {GitHub Repository},
594
+ howpublished = {\url{https://github.com/unclecode/crawl4ai}},
595
+ commit = {Please use the commit hash you're working with}
596
+ }
597
+ ```
598
+
599
+ Text citation format:
600
+ ```
601
+ UncleCode. (2024). Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper [Computer software].
602
+ GitHub. https://github.com/unclecode/crawl4ai
603
+ ```
604
+
605
+ ## 🗾 Mission
606
+
607
+ Our mission is to unlock the value of personal and enterprise data by turning digital footprints into structured, useful assets. Crawl4AI gives individuals and organizations open-source tools to extract and structure data, and a fair way to benefit from it. [Full mission statement →](./MISSION.md)
608
+
609
+ ## 💖 Support Crawl4AI
610
+
611
+ 1. ⭐ **Star the repo**: it helps more people find it.
612
+ 2. ☁️ **Use the cloud**: [crawl4ai.com](https://crawl4ai.com/?ref=readme). It funds the library.
613
+ 3. 💝 **Sponsor on GitHub**: [github.com/sponsors/unclecode](https://github.com/sponsors/unclecode)
614
+ 4. 🏢 **Companies**: the sponsor tiers and benefits are in [SPONSORS.md](SPONSORS.md).
615
+
616
+ ## 🌟 Current Sponsors
617
+
618
+ ### 🤝 Strategic Partners
619
+
620
+ These companies provide core infrastructure and technology that power Crawl4AI’s capabilities — from web access and proxy networks to AI tooling and data pipelines.
621
+
622
+ | Company | About |
623
+ |------|------|
624
+ | <a href="https://www.joinmassive.com/" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="docs/assets/sponsors/massive_light.svg"><source media="(prefers-color-scheme: light)" srcset="docs/assets/sponsors/massive.svg"><img alt="Massive" src="docs/assets/sponsors/massive.svg" height="40"/></picture></a> | Massive provides a web access API for AI agents and data pipelines. [Partnership details](https://www.joinmassive.com/blog/crawl4ai-partner-spotlight). |
625
+
626
+ ### 🏢 Enterprise Sponsors
627
+
628
+ Our enterprise sponsors support Crawl4AI and help scale it to power production-grade data pipelines.
629
+
630
+ | Company | About | Sponsorship Tier |
631
+ |------|------|----------------------------|
632
+ | <a href="https://kipo.ai" target="_blank"><img src="https://docs.crawl4ai.com/uploads/sponsors/20251013045751_2d54f57f117c651e.png" alt="DataSync" height="40"/></a> | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold |
633
+ | <a href="https://www.kidocode.com/" target="_blank"><img src="https://docs.crawl4ai.com/uploads/sponsors/20251013045045_bb8dace3f0440d65.svg" alt="Kidocode" height="40"/></a> | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold |
634
+ | <a href="https://www.alephnull.sg/" target="_blank"><picture><source media="(prefers-color-scheme: dark)" srcset="docs/assets/sponsors/aleph_null_light.svg"><source media="(prefers-color-scheme: light)" srcset="docs/assets/sponsors/aleph_null.svg"><img alt="Aleph null" src="docs/assets/sponsors/aleph_null.svg" height="40"/></picture></a> | Singapore-based Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold |
635
+
636
+ ---
637
+
638
+ ### 💼 Become a Strategic Partner or Sponsor
639
+
640
+ Interested in partnering with Crawl4AI?
641
+
642
+ Whether you’re a proxy provider, AI infrastructure company, cloud platform, or an organization looking to support the Crawl4AI ecosystem, we’d love to hear from you.
643
+
644
+ 📩 Contact: hello@crawl4ai.com
645
+
646
+
647
+
648
+ ### 🧑‍🤝 Individual Sponsors
649
+
650
+ A heartfelt thanks to our individual supporters! Every contribution helps us keep our opensource mission alive and thriving!
651
+
652
+ <p align="left">
653
+ <a href="https://github.com/hafezparast"><img src="https://avatars.githubusercontent.com/u/14273305?s=60&v=4" style="border-radius:50%;" width="64px;"/></a>
654
+ <a href="https://github.com/ntohidi"><img src="https://avatars.githubusercontent.com/u/17140097?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
655
+ <a href="https://github.com/Sjoeborg"><img src="https://avatars.githubusercontent.com/u/17451310?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
656
+ <a href="https://github.com/romek-rozen"><img src="https://avatars.githubusercontent.com/u/30595969?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
657
+ <a href="https://github.com/Kourosh-Kiyani"><img src="https://avatars.githubusercontent.com/u/34105600?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
658
+ <a href="https://github.com/Etherdrake"><img src="https://avatars.githubusercontent.com/u/67021215?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
659
+ <a href="https://github.com/shaman247"><img src="https://avatars.githubusercontent.com/u/211010067?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
660
+ <a href="https://github.com/work-flow-manager"><img src="https://avatars.githubusercontent.com/u/217665461?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
661
+ </p>
662
+
663
+ > Want to join them? [Sponsor Crawl4AI →](https://github.com/sponsors/unclecode)
664
+
665
+ ## 📧 Contact
666
+
667
+ [Discord](https://discord.gg/jP8KfhDhyN) · [X @unclecode](https://x.com/unclecode) · [GitHub @unclecode](https://github.com/unclecode) · hello@crawl4ai.com
668
+
669
+ - **Building crawlers or AI agents for a living?** DM me on X. I want to work with people like you, and we are hiring.
670
+ - **From a company?** We have an enterprise offer and we tailor it to your business. See our [security and compliance page](https://crawl4ai.com/security) for current details. Write to hello@crawl4ai.com.
671
+
672
+ <p align="center">
673
+ <a href="https://github.com/unclecode/crawl4ai/network/members"><img src="https://img.shields.io/github/forks/unclecode/crawl4ai?style=social" alt="GitHub Forks"/></a>
674
+ <a href="https://pypi.org/project/crawl4ai/"><img src="https://img.shields.io/pypi/pyversions/crawl4ai" alt="Python Version"/></a>
675
+ <a href="https://github.com/sponsors/unclecode"><img src="https://img.shields.io/github/sponsors/unclecode?style=flat&logo=GitHub-Sponsors&label=Sponsors&color=pink" alt="GitHub Sponsors"/></a>
676
+ <a href="https://x.com/crawl4ai"><img src="https://img.shields.io/badge/Follow%20on%20X-000000?style=flat&logo=x&logoColor=white" alt="Follow on X"/></a>
677
+ <a href="https://www.linkedin.com/company/crawl4ai"><img src="https://img.shields.io/badge/LinkedIn-0077B5?style=flat&logo=linkedin&logoColor=white" alt="Follow on LinkedIn"/></a>
678
+ </p>
679
+
680
+ Happy crawling! 🕸️🚀
681
+
682
+ ## Star History
683
+
684
+ <a href="https://www.star-history.com/?type=date&repos=unclecode%2Fcrawl4ai">
685
+ <picture>
686
+ <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=unclecode/crawl4ai&type=date&theme=dark&legend=top-left&sealed_token=KuajrA7ScH8VT4KagC7nm1xbazTVaNs6rdok4At2dV6tDl91YR_dxmHhmsffjhFiWdLYlzdACxZ-cWLwp8tZHCYxSDMjITf3Vnu4mPns7YdLetyQBPHMQ2f_KakXdbvbVP6PofI82GNqGCVEXtPnZWHC8WM6CzZe6s6cJb6_ga_kn-jh-BeHdyuRRdVR" />
687
+ <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=unclecode/crawl4ai&type=date&legend=top-left&sealed_token=KuajrA7ScH8VT4KagC7nm1xbazTVaNs6rdok4At2dV6tDl91YR_dxmHhmsffjhFiWdLYlzdACxZ-cWLwp8tZHCYxSDMjITf3Vnu4mPns7YdLetyQBPHMQ2f_KakXdbvbVP6PofI82GNqGCVEXtPnZWHC8WM6CzZe6s6cJb6_ga_kn-jh-BeHdyuRRdVR" />
688
+ <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=unclecode/crawl4ai&type=date&legend=top-left&sealed_token=KuajrA7ScH8VT4KagC7nm1xbazTVaNs6rdok4At2dV6tDl91YR_dxmHhmsffjhFiWdLYlzdACxZ-cWLwp8tZHCYxSDMjITf3Vnu4mPns7YdLetyQBPHMQ2f_KakXdbvbVP6PofI82GNqGCVEXtPnZWHC8WM6CzZe6s6cJb6_ga_kn-jh-BeHdyuRRdVR" />
689
+ </picture>
690
+ </a>