omnicrawl-core 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omnicrawl/__init__.py +82 -0
- omnicrawl/anti_detect/__init__.py +8 -0
- omnicrawl/anti_detect/captcha_solver.py +755 -0
- omnicrawl/anti_detect/fingerprint_consistency.py +576 -0
- omnicrawl/anti_detect/rate_limiter.py +81 -0
- omnicrawl/anti_detect/waf_bypass.py +103 -0
- omnicrawl/cli.py +341 -0
- omnicrawl/client.py +466 -0
- omnicrawl/config.py +213 -0
- omnicrawl/fetchers/__init__.py +12 -0
- omnicrawl/fetchers/base.py +87 -0
- omnicrawl/fetchers/browser_fetcher.py +123 -0
- omnicrawl/fetchers/camoufox_fetcher.py +222 -0
- omnicrawl/fetchers/http_fetcher.py +116 -0
- omnicrawl/fetchers/stealth_fetcher.py +137 -0
- omnicrawl/fingerprint/__init__.py +5 -0
- omnicrawl/fingerprint/tls.py +94 -0
- omnicrawl/parser/__init__.py +6 -0
- omnicrawl/parser/html_parser.py +87 -0
- omnicrawl/parser/interactive_state.py +326 -0
- omnicrawl/parser/markdown.py +311 -0
- omnicrawl/plugins/__init__.py +18 -0
- omnicrawl/plugins/base.py +185 -0
- omnicrawl/plugins/builtin.py +136 -0
- omnicrawl/proxy/__init__.py +6 -0
- omnicrawl/proxy/rotator.py +110 -0
- omnicrawl/proxy/scorer.py +211 -0
- omnicrawl/proxy/validator.py +127 -0
- omnicrawl/session/__init__.py +3 -0
- omnicrawl/session/manager.py +322 -0
- omnicrawl/spider/__init__.py +17 -0
- omnicrawl/spider/base.py +509 -0
- omnicrawl/spider/link_extractor.py +137 -0
- omnicrawl/spider/pipeline.py +260 -0
- omnicrawl/spider/smart_spider.py +698 -0
- omnicrawl/storage/__init__.py +23 -0
- omnicrawl/storage/base.py +75 -0
- omnicrawl/storage/memory.py +77 -0
- omnicrawl/storage/redis_store.py +105 -0
- omnicrawl/utils/__init__.py +0 -0
- omnicrawl/utils/logger.py +18 -0
- omnicrawl_core-0.2.0.dist-info/METADATA +721 -0
- omnicrawl_core-0.2.0.dist-info/RECORD +45 -0
- omnicrawl_core-0.2.0.dist-info/WHEEL +4 -0
- omnicrawl_core-0.2.0.dist-info/entry_points.txt +2 -0
omnicrawl/__init__.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""OmniCrawl — 无所不能的爬虫框架
|
|
2
|
+
|
|
3
|
+
整合 Scrapling + curl_cffi + Camoufox + Playwright,绕过 WAF,LLM 友好输出。
|
|
4
|
+
|
|
5
|
+
快速开始:
|
|
6
|
+
from omnicrawl import OmniClient
|
|
7
|
+
|
|
8
|
+
client = OmniClient()
|
|
9
|
+
response = await client.get("https://example.com")
|
|
10
|
+
print(response.markdown)
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
__version__ = "0.2.0"
|
|
14
|
+
|
|
15
|
+
# 延迟导入:首次访问时才 import,避免顶层加载全部模块
|
|
16
|
+
_LAZY_IMPORTS: dict[str, tuple[str, str]] = {
|
|
17
|
+
# Core
|
|
18
|
+
"OmniClient": ("omnicrawl.client", "OmniClient"),
|
|
19
|
+
"FetchMode": ("omnicrawl.fetchers.base", "FetchMode"),
|
|
20
|
+
"FetchResult": ("omnicrawl.fetchers.base", "FetchResult"),
|
|
21
|
+
# Proxy
|
|
22
|
+
"ProxyRotator": ("omnicrawl.proxy.rotator", "ProxyRotator"),
|
|
23
|
+
"ProxyValidator": ("omnicrawl.proxy.validator", "ProxyValidator"),
|
|
24
|
+
"ProxyScorer": ("omnicrawl.proxy.scorer", "ProxyScorer"),
|
|
25
|
+
"ProxyStats": ("omnicrawl.proxy.scorer", "ProxyStats"),
|
|
26
|
+
# Fingerprint
|
|
27
|
+
"TLSFingerprint": ("omnicrawl.fingerprint.tls", "TLSFingerprint"),
|
|
28
|
+
"FingerprintConsistency": ("omnicrawl.anti_detect.fingerprint_consistency", "FingerprintConsistency"),
|
|
29
|
+
"BrowserIdentity": ("omnicrawl.anti_detect.fingerprint_consistency", "BrowserIdentity"),
|
|
30
|
+
# Anti-detect
|
|
31
|
+
"WAFBypass": ("omnicrawl.anti_detect.waf_bypass", "WAFBypass"),
|
|
32
|
+
"RateLimiter": ("omnicrawl.anti_detect.rate_limiter", "RateLimiter"),
|
|
33
|
+
"CaptchaSolver": ("omnicrawl.anti_detect.captcha_solver", "CaptchaSolver"),
|
|
34
|
+
"CaptchaDetector": ("omnicrawl.anti_detect.captcha_solver", "CaptchaDetector"),
|
|
35
|
+
"CaptchaType": ("omnicrawl.anti_detect.captcha_solver", "CaptchaType"),
|
|
36
|
+
# Session
|
|
37
|
+
"SessionManager": ("omnicrawl.session.manager", "SessionManager"),
|
|
38
|
+
"BrowserHandle": ("omnicrawl.session.manager", "BrowserHandle"),
|
|
39
|
+
"Session": ("omnicrawl.session.manager", "Session"),
|
|
40
|
+
# Parser
|
|
41
|
+
"HTMLParser": ("omnicrawl.parser.html_parser", "HTMLParser"),
|
|
42
|
+
"MarkdownConverter": ("omnicrawl.parser.markdown", "MarkdownConverter"),
|
|
43
|
+
"InteractiveStateExtractor": ("omnicrawl.parser.interactive_state", "InteractiveStateExtractor"),
|
|
44
|
+
"PageState": ("omnicrawl.parser.interactive_state", "PageState"),
|
|
45
|
+
"InteractiveElement": ("omnicrawl.parser.interactive_state", "InteractiveElement"),
|
|
46
|
+
# Spider
|
|
47
|
+
"Spider": ("omnicrawl.spider.base", "Spider"),
|
|
48
|
+
"CrawlSpider": ("omnicrawl.spider.base", "CrawlSpider"),
|
|
49
|
+
"SpiderItem": ("omnicrawl.spider.base", "SpiderItem"),
|
|
50
|
+
"SpiderStats": ("omnicrawl.spider.base", "SpiderStats"),
|
|
51
|
+
"SmartSpider": ("omnicrawl.spider.smart_spider", "SmartSpider"),
|
|
52
|
+
"ApiEndpoint": ("omnicrawl.spider.smart_spider", "ApiEndpoint"),
|
|
53
|
+
"NetworkCapture": ("omnicrawl.spider.smart_spider", "NetworkCapture"),
|
|
54
|
+
"LinkExtractor": ("omnicrawl.spider.link_extractor", "LinkExtractor"),
|
|
55
|
+
"Pipeline": ("omnicrawl.spider.pipeline", "Pipeline"),
|
|
56
|
+
"CleanPipeline": ("omnicrawl.spider.pipeline", "CleanPipeline"),
|
|
57
|
+
"ValidatePipeline": ("omnicrawl.spider.pipeline", "ValidatePipeline"),
|
|
58
|
+
"DedupPipeline": ("omnicrawl.spider.pipeline", "DedupPipeline"),
|
|
59
|
+
"RedisDedupPipeline": ("omnicrawl.spider.pipeline", "RedisDedupPipeline"),
|
|
60
|
+
"JsonFilePipeline": ("omnicrawl.spider.pipeline", "JsonFilePipeline"),
|
|
61
|
+
# Storage
|
|
62
|
+
"StateStore": ("omnicrawl.storage", "StateStore"),
|
|
63
|
+
"MemoryStore": ("omnicrawl.storage", "MemoryStore"),
|
|
64
|
+
# Plugins
|
|
65
|
+
"Plugin": ("omnicrawl.plugins", "Plugin"),
|
|
66
|
+
"PluginManager": ("omnicrawl.plugins", "PluginManager"),
|
|
67
|
+
"LoggingPlugin": ("omnicrawl.plugins", "LoggingPlugin"),
|
|
68
|
+
"StatsPlugin": ("omnicrawl.plugins", "StatsPlugin"),
|
|
69
|
+
"FilterPlugin": ("omnicrawl.plugins", "FilterPlugin"),
|
|
70
|
+
"TransformPlugin": ("omnicrawl.plugins", "TransformPlugin"),
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
__all__ = list(_LAZY_IMPORTS.keys())
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def __getattr__(name: str):
|
|
77
|
+
if name in _LAZY_IMPORTS:
|
|
78
|
+
module_path, attr_name = _LAZY_IMPORTS[name]
|
|
79
|
+
import importlib
|
|
80
|
+
module = importlib.import_module(module_path)
|
|
81
|
+
return getattr(module, attr_name)
|
|
82
|
+
raise AttributeError(f"module 'omnicrawl' has no attribute {name!r}")
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""反检测模块"""
|
|
2
|
+
|
|
3
|
+
from omnicrawl.anti_detect.captcha_solver import CaptchaSolver, CaptchaType, CaptchaDetector
|
|
4
|
+
from omnicrawl.anti_detect.fingerprint_consistency import FingerprintConsistency
|
|
5
|
+
from omnicrawl.anti_detect.rate_limiter import RateLimiter
|
|
6
|
+
from omnicrawl.anti_detect.waf_bypass import WAFBypass
|
|
7
|
+
|
|
8
|
+
__all__ = ["CaptchaSolver", "CaptchaType", "CaptchaDetector", "FingerprintConsistency", "RateLimiter", "WAFBypass"]
|