omnicrawl-core 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. omnicrawl/__init__.py +82 -0
  2. omnicrawl/anti_detect/__init__.py +8 -0
  3. omnicrawl/anti_detect/captcha_solver.py +755 -0
  4. omnicrawl/anti_detect/fingerprint_consistency.py +576 -0
  5. omnicrawl/anti_detect/rate_limiter.py +81 -0
  6. omnicrawl/anti_detect/waf_bypass.py +103 -0
  7. omnicrawl/cli.py +341 -0
  8. omnicrawl/client.py +466 -0
  9. omnicrawl/config.py +213 -0
  10. omnicrawl/fetchers/__init__.py +12 -0
  11. omnicrawl/fetchers/base.py +87 -0
  12. omnicrawl/fetchers/browser_fetcher.py +123 -0
  13. omnicrawl/fetchers/camoufox_fetcher.py +222 -0
  14. omnicrawl/fetchers/http_fetcher.py +116 -0
  15. omnicrawl/fetchers/stealth_fetcher.py +137 -0
  16. omnicrawl/fingerprint/__init__.py +5 -0
  17. omnicrawl/fingerprint/tls.py +94 -0
  18. omnicrawl/parser/__init__.py +6 -0
  19. omnicrawl/parser/html_parser.py +87 -0
  20. omnicrawl/parser/interactive_state.py +326 -0
  21. omnicrawl/parser/markdown.py +311 -0
  22. omnicrawl/plugins/__init__.py +18 -0
  23. omnicrawl/plugins/base.py +185 -0
  24. omnicrawl/plugins/builtin.py +136 -0
  25. omnicrawl/proxy/__init__.py +6 -0
  26. omnicrawl/proxy/rotator.py +110 -0
  27. omnicrawl/proxy/scorer.py +211 -0
  28. omnicrawl/proxy/validator.py +127 -0
  29. omnicrawl/session/__init__.py +3 -0
  30. omnicrawl/session/manager.py +322 -0
  31. omnicrawl/spider/__init__.py +17 -0
  32. omnicrawl/spider/base.py +509 -0
  33. omnicrawl/spider/link_extractor.py +137 -0
  34. omnicrawl/spider/pipeline.py +260 -0
  35. omnicrawl/spider/smart_spider.py +698 -0
  36. omnicrawl/storage/__init__.py +23 -0
  37. omnicrawl/storage/base.py +75 -0
  38. omnicrawl/storage/memory.py +77 -0
  39. omnicrawl/storage/redis_store.py +105 -0
  40. omnicrawl/utils/__init__.py +0 -0
  41. omnicrawl/utils/logger.py +18 -0
  42. omnicrawl_core-0.2.0.dist-info/METADATA +721 -0
  43. omnicrawl_core-0.2.0.dist-info/RECORD +45 -0
  44. omnicrawl_core-0.2.0.dist-info/WHEEL +4 -0
  45. omnicrawl_core-0.2.0.dist-info/entry_points.txt +2 -0
omnicrawl/__init__.py ADDED
@@ -0,0 +1,82 @@
1
+ """OmniCrawl — 无所不能的爬虫框架
2
+
3
+ 整合 Scrapling + curl_cffi + Camoufox + Playwright,绕过 WAF,LLM 友好输出。
4
+
5
+ 快速开始:
6
+ from omnicrawl import OmniClient
7
+
8
+ client = OmniClient()
9
+ response = await client.get("https://example.com")
10
+ print(response.markdown)
11
+ """
12
+
13
+ __version__ = "0.2.0"
14
+
15
+ # 延迟导入:首次访问时才 import,避免顶层加载全部模块
16
+ _LAZY_IMPORTS: dict[str, tuple[str, str]] = {
17
+ # Core
18
+ "OmniClient": ("omnicrawl.client", "OmniClient"),
19
+ "FetchMode": ("omnicrawl.fetchers.base", "FetchMode"),
20
+ "FetchResult": ("omnicrawl.fetchers.base", "FetchResult"),
21
+ # Proxy
22
+ "ProxyRotator": ("omnicrawl.proxy.rotator", "ProxyRotator"),
23
+ "ProxyValidator": ("omnicrawl.proxy.validator", "ProxyValidator"),
24
+ "ProxyScorer": ("omnicrawl.proxy.scorer", "ProxyScorer"),
25
+ "ProxyStats": ("omnicrawl.proxy.scorer", "ProxyStats"),
26
+ # Fingerprint
27
+ "TLSFingerprint": ("omnicrawl.fingerprint.tls", "TLSFingerprint"),
28
+ "FingerprintConsistency": ("omnicrawl.anti_detect.fingerprint_consistency", "FingerprintConsistency"),
29
+ "BrowserIdentity": ("omnicrawl.anti_detect.fingerprint_consistency", "BrowserIdentity"),
30
+ # Anti-detect
31
+ "WAFBypass": ("omnicrawl.anti_detect.waf_bypass", "WAFBypass"),
32
+ "RateLimiter": ("omnicrawl.anti_detect.rate_limiter", "RateLimiter"),
33
+ "CaptchaSolver": ("omnicrawl.anti_detect.captcha_solver", "CaptchaSolver"),
34
+ "CaptchaDetector": ("omnicrawl.anti_detect.captcha_solver", "CaptchaDetector"),
35
+ "CaptchaType": ("omnicrawl.anti_detect.captcha_solver", "CaptchaType"),
36
+ # Session
37
+ "SessionManager": ("omnicrawl.session.manager", "SessionManager"),
38
+ "BrowserHandle": ("omnicrawl.session.manager", "BrowserHandle"),
39
+ "Session": ("omnicrawl.session.manager", "Session"),
40
+ # Parser
41
+ "HTMLParser": ("omnicrawl.parser.html_parser", "HTMLParser"),
42
+ "MarkdownConverter": ("omnicrawl.parser.markdown", "MarkdownConverter"),
43
+ "InteractiveStateExtractor": ("omnicrawl.parser.interactive_state", "InteractiveStateExtractor"),
44
+ "PageState": ("omnicrawl.parser.interactive_state", "PageState"),
45
+ "InteractiveElement": ("omnicrawl.parser.interactive_state", "InteractiveElement"),
46
+ # Spider
47
+ "Spider": ("omnicrawl.spider.base", "Spider"),
48
+ "CrawlSpider": ("omnicrawl.spider.base", "CrawlSpider"),
49
+ "SpiderItem": ("omnicrawl.spider.base", "SpiderItem"),
50
+ "SpiderStats": ("omnicrawl.spider.base", "SpiderStats"),
51
+ "SmartSpider": ("omnicrawl.spider.smart_spider", "SmartSpider"),
52
+ "ApiEndpoint": ("omnicrawl.spider.smart_spider", "ApiEndpoint"),
53
+ "NetworkCapture": ("omnicrawl.spider.smart_spider", "NetworkCapture"),
54
+ "LinkExtractor": ("omnicrawl.spider.link_extractor", "LinkExtractor"),
55
+ "Pipeline": ("omnicrawl.spider.pipeline", "Pipeline"),
56
+ "CleanPipeline": ("omnicrawl.spider.pipeline", "CleanPipeline"),
57
+ "ValidatePipeline": ("omnicrawl.spider.pipeline", "ValidatePipeline"),
58
+ "DedupPipeline": ("omnicrawl.spider.pipeline", "DedupPipeline"),
59
+ "RedisDedupPipeline": ("omnicrawl.spider.pipeline", "RedisDedupPipeline"),
60
+ "JsonFilePipeline": ("omnicrawl.spider.pipeline", "JsonFilePipeline"),
61
+ # Storage
62
+ "StateStore": ("omnicrawl.storage", "StateStore"),
63
+ "MemoryStore": ("omnicrawl.storage", "MemoryStore"),
64
+ # Plugins
65
+ "Plugin": ("omnicrawl.plugins", "Plugin"),
66
+ "PluginManager": ("omnicrawl.plugins", "PluginManager"),
67
+ "LoggingPlugin": ("omnicrawl.plugins", "LoggingPlugin"),
68
+ "StatsPlugin": ("omnicrawl.plugins", "StatsPlugin"),
69
+ "FilterPlugin": ("omnicrawl.plugins", "FilterPlugin"),
70
+ "TransformPlugin": ("omnicrawl.plugins", "TransformPlugin"),
71
+ }
72
+
73
+ __all__ = list(_LAZY_IMPORTS.keys())
74
+
75
+
76
+ def __getattr__(name: str):
77
+ if name in _LAZY_IMPORTS:
78
+ module_path, attr_name = _LAZY_IMPORTS[name]
79
+ import importlib
80
+ module = importlib.import_module(module_path)
81
+ return getattr(module, attr_name)
82
+ raise AttributeError(f"module 'omnicrawl' has no attribute {name!r}")
@@ -0,0 +1,8 @@
1
+ """反检测模块"""
2
+
3
+ from omnicrawl.anti_detect.captcha_solver import CaptchaSolver, CaptchaType, CaptchaDetector
4
+ from omnicrawl.anti_detect.fingerprint_consistency import FingerprintConsistency
5
+ from omnicrawl.anti_detect.rate_limiter import RateLimiter
6
+ from omnicrawl.anti_detect.waf_bypass import WAFBypass
7
+
8
+ __all__ = ["CaptchaSolver", "CaptchaType", "CaptchaDetector", "FingerprintConsistency", "RateLimiter", "WAFBypass"]