crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

crawlo/utils/url.py

@@ -1,40 +1,40 @@
-from urllib.parse import urldefrag
-from w3lib.url import add_or_replace_parameter
-
-
-def escape_ajax(url: str) -> str:
-    """
-    根据Google AJAX爬取规范转换URL（处理哈希片段#!）：
-    https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
-
-    规则说明：
-    1. 仅当URL包含 `#!` 时才转换（表示这是AJAX可爬取页面）
-    2. 将 `#!key=value` 转换为 `?_escaped_fragment_=key%3Dvalue`
-    3. 保留原始查询参数（如果有）
-
-    示例：
-    >>> escape_ajax("www.example.com/ajax.html#!key=value")
-    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
-    >>> escape_ajax("www.example.com/ajax.html?k1=v1#!key=value")
-    'www.example.com/ajax.html?k1=v1&_escaped_fragment_=key%3Dvalue'
-    >>> escape_ajax("www.example.com/ajax.html#!")
-    'www.example.com/ajax.html?_escaped_fragment_='
-
-    非AJAX可爬取的URL（无#!）原样返回：
-    >>> escape_ajax("www.example.com/ajax.html#normal")
-    'www.example.com/ajax.html#normal'
-    """
-    # 分离URL的基础部分和哈希片段
-    de_frag, frag = urldefrag(url)
-
-    # 仅处理以"!"开头的哈希片段（Google规范）
-    if not frag.startswith("!"):
-        return url  # 不符合规则则原样返回
-
-    # 调用辅助函数添加 `_escaped_fragment_` 参数
-    return add_or_replace_parameter(de_frag, "_escaped_fragment_", frag[1:])
-
-
-if __name__ == '__main__':
-    f = escape_ajax('http://example.com/page#!')
+from urllib.parse import urldefrag
+from w3lib.url import add_or_replace_parameter
+
+
+def escape_ajax(url: str) -> str:
+    """
+    根据Google AJAX爬取规范转换URL（处理哈希片段#!）：
+    https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+
+    规则说明：
+    1. 仅当URL包含 `#!` 时才转换（表示这是AJAX可爬取页面）
+    2. 将 `#!key=value` 转换为 `?_escaped_fragment_=key%3Dvalue`
+    3. 保留原始查询参数（如果有）
+
+    示例：
+    >>> escape_ajax("www.example.com/ajax.html#!key=value")
+    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html?k1=v1#!key=value")
+    'www.example.com/ajax.html?k1=v1&_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html#!")
+    'www.example.com/ajax.html?_escaped_fragment_='
+
+    非AJAX可爬取的URL（无#!）原样返回：
+    >>> escape_ajax("www.example.com/ajax.html#normal")
+    'www.example.com/ajax.html#normal'
+    """
+    # 分离URL的基础部分和哈希片段
+    de_frag, frag = urldefrag(url)
+
+    # 仅处理以"!"开头的哈希片段（Google规范）
+    if not frag.startswith("!"):
+        return url  # 不符合规则则原样返回
+
+    # 调用辅助函数添加 `_escaped_fragment_` 参数
+    return add_or_replace_parameter(de_frag, "_escaped_fragment_", frag[1:])
+
+
+if __name__ == '__main__':
+    f = escape_ajax('http://example.com/page#!')
     print(f)

crawlo-1.4.3.dist-info/METADATA

@@ -0,0 +1,190 @@
+Metadata-Version: 2.4
+Name: crawlo
+Version: 1.4.3
+Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
+Home-page: https://github.com/crawl-coder/Crawlo.git
+Author: crawl-coder
+Author-email: crawlo@qq.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp>=3.12.14
+Requires-Dist: aiomysql>=0.2.0
+Requires-Dist: aioredis>=2.0.1
+Requires-Dist: asyncmy>=0.2.10
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: dateparser>=1.2.2
+Requires-Dist: httpx[http2]>=0.27.0
+Requires-Dist: curl-cffi>=0.13.0
+Requires-Dist: lxml>=5.2.1
+Requires-Dist: motor>=3.7.0
+Requires-Dist: parsel>=1.9.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: pymongo>=4.11
+Requires-Dist: PyMySQL>=1.1.1
+Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: redis>=6.2.0
+Requires-Dist: requests>=2.32.4
+Requires-Dist: six>=1.17.0
+Requires-Dist: ujson>=5.9.0
+Requires-Dist: urllib3>=2.5.0
+Requires-Dist: w3lib>=2.1.2
+Requires-Dist: rich>=14.1.0
+Requires-Dist: astor>=0.8.1
+Requires-Dist: watchdog>=6.0.0
+Provides-Extra: render
+Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
+Requires-Dist: playwright; extra == "render"
+Requires-Dist: selenium>=3.141.0; extra == "render"
+Provides-Extra: all
+Requires-Dist: bitarray>=1.5.3; extra == "all"
+Requires-Dist: PyExecJS>=1.5.1; extra == "all"
+Requires-Dist: pymongo>=3.10.1; extra == "all"
+Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
+Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: selenium>=3.141.0; extra == "all"
+
+# Crawlo 爬虫框架
+
+Crawlo 是一个高性能、可扩展的 Python 爬虫框架，支持单机和分布式部署。
+
+## 特性
+
+- 高性能异步爬取
+- 支持多种下载器 (aiohttp, httpx, curl-cffi)
+- 内置数据清洗和验证
+- 分布式爬取支持
+- 灵活的中间件系统
+- 强大的配置管理系统
+- 详细的日志记录和监控
+- Windows 和 Linux 兼容
+
+## 安装
+
+```bash
+pip install crawlo
+```
+
+或者从源码安装：
+
+```bash
+git clone https://github.com/your-username/crawlo.git
+cd crawlo
+pip install -r requirements.txt
+pip install .
+```
+
+## 快速开始
+
+```python
+from crawlo import Spider
+
+class MySpider(Spider):
+    name = 'example'
+    
+    def parse(self, response):
+        # 解析逻辑
+        pass
+
+# 运行爬虫
+# crawlo run example
+```
+
+## 日志系统
+
+Crawlo 拥有一个功能强大的日志系统，支持多种配置选项：
+
+### 基本配置
+
+```python
+from crawlo.logging import configure_logging, get_logger
+
+# 配置日志系统
+configure_logging(
+    LOG_LEVEL='INFO',
+    LOG_FILE='logs/app.log',
+    LOG_MAX_BYTES=10*1024*1024,  # 10MB
+    LOG_BACKUP_COUNT=5
+)
+
+# 获取logger
+logger = get_logger('my_module')
+logger.info('这是一条日志消息')
+```
+
+### 高级配置
+
+```python
+# 分别配置控制台和文件日志级别
+configure_logging(
+    LOG_LEVEL='INFO',
+    LOG_CONSOLE_LEVEL='WARNING',  # 控制台只显示WARNING及以上级别
+    LOG_FILE_LEVEL='DEBUG',       # 文件记录DEBUG及以上级别
+    LOG_FILE='logs/app.log',
+    LOG_INCLUDE_THREAD_ID=True,   # 包含线程ID
+    LOG_INCLUDE_PROCESS_ID=True   # 包含进程ID
+)
+
+# 模块特定日志级别
+configure_logging(
+    LOG_LEVEL='WARNING',
+    LOG_LEVELS={
+        'my_module.debug': 'DEBUG',
+        'my_module.info': 'INFO'
+    }
+)
+```
+
+### 性能监控
+
+```python
+from crawlo.logging import get_monitor
+
+# 启用日志性能监控
+monitor = get_monitor()
+monitor.enable_monitoring()
+
+# 获取性能报告
+report = monitor.get_performance_report()
+print(report)
+```
+
+### 日志采样
+
+```python
+from crawlo.logging import get_sampler
+
+# 设置采样率（只记录30%的日志）
+sampler = get_sampler()
+sampler.set_sample_rate('my_module', 0.3)
+
+# 设置速率限制（每秒最多100条日志）
+sampler.set_rate_limit('my_module', 100)
+```
+
+## Windows 兼容性说明
+
+在 Windows 系统上使用日志轮转功能时，可能会遇到文件锁定问题。为了解决这个问题，建议安装 `concurrent-log-handler` 库：
+
+```bash
+pip install concurrent-log-handler
+```
+
+Crawlo 框架会自动检测并使用这个库来提供更好的 Windows 兼容性。
+
+如果未安装 `concurrent-log-handler`，在 Windows 上运行时可能会出现以下错误：
+```
+PermissionError: [WinError 32] 另一个程序正在使用此文件，进程无法访问。
+```
+
+## 文档
+
+请查看 [文档](https://your-docs-url.com) 获取更多信息。
+
+## 许可证
+
+MIT

crawlo-1.4.3.dist-info/RECORD

@@ -0,0 +1,326 @@
+crawlo/__init__.py,sha256=2Io5P9qJghOAjjD3YWdaiIq5laPLyLWVkEqgiVfUa3o,2381
+crawlo/__version__.py,sha256=adPldP4cMp2T8pbTFGYTXV50hu_smS3hxWkk5kLXpZE,23
+crawlo/cli.py,sha256=AQnAB5NMI-Ic1VPw_Jjng8L4AI4-wMozOwzE6CfXkZU,2402
+crawlo/config.py,sha256=EQIT7WpkXAlr2ocd5SYJYOKTSWUlQx2AkTHX7ErEWxw,9798
+crawlo/config_validator.py,sha256=oY4-2bwXUlwHAnGgkI-EznviDfML_dcxbWSGXNSxC2k,11516
+crawlo/crawler.py,sha256=i4rc9beEOilKGK633nRh5UxCNgciil9Lyfj38xgIauU,26998
+crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
+crawlo/exceptions.py,sha256=YVIDnC1bKSMv3fXH_6tinWMuD9HmKHIaUfO4_fkX5sY,1247
+crawlo/framework.py,sha256=9gP6VN4MHqutGXaxnwpNMSULfVYbNp906UdZiJGywlQ,9458
+crawlo/mode_manager.py,sha256=S4dUoeVZ4fMnd4pXWutcHwk5Zv68ZBTgo9taR9OkQiM,7768
+crawlo/project.py,sha256=nVRc0CIdd9g863NGfuItvajl8zlO5mEta4FQCx9_vZ8,14060
+crawlo/stats_collector.py,sha256=hIjlnX750jU4Oncyand1jBccfaX4Tu7egd2DBYu2N7A,2379
+crawlo/subscriber.py,sha256=h8fx69NJZeWem0ZkCmfHAi2kgfDGFObHpwN0aGNUM6Y,5115
+crawlo/task_manager.py,sha256=I9h3Rl0VRAfwqp24CHT3TuEAapNdTbVghkmuJhtM7jg,5966
+crawlo/commands/__init__.py,sha256=orvY6wLOBwGUEJKeF3h_T1fxj8AaQLjngBDd-3xKOE4,392
+crawlo/commands/check.py,sha256=TKDhI_sj7kErgiJpt2vCZ9QL-g6yWjrrPWKbgh8pgEU,23199
+crawlo/commands/genspider.py,sha256=7YGZdv12G341SWmkGbyDeMde2RgqGYxYXRExFy7KKNc,5088
+crawlo/commands/help.py,sha256=8xPC0iNCg1rRBoK2bb6noAEANc1JwrdM35eF-j6yeZM,5111
+crawlo/commands/list.py,sha256=trzcd3kG6DhkOqYZADcl3yR7M8iJBgRw5fE-g9e0gVM,5877
+crawlo/commands/run.py,sha256=EjpIilgCTkXGVSV4rEISbJubdhqrok9nNe5-xDfDK5E,13169
+crawlo/commands/startproject.py,sha256=-Bo8vvDfIhqzGmWyhxMatBlPLhYpRwJC7l4fpbN8vVk,16506
+crawlo/commands/stats.py,sha256=vlGJLyiXZtY0ASdzCK59JNereSsAel4W9JCGaOzCr-8,6201
+crawlo/commands/utils.py,sha256=YVNEEzlm_qNY3SVvU8h6o2lQMkVgypvoB4ZFrP4gln0,5578
+crawlo/core/__init__.py,sha256=BWkj3AqZwp2Bk73UzUlC_qsqv_MH_HNrzy4DY1hosj4,1330
+crawlo/core/engine.py,sha256=y9mj0nKHb3Ki4scXkxsMO6XoTIqxmbsD0WuryR_6iHg,19385
+crawlo/core/processor.py,sha256=hR5MrbeZvDUx0ShKntr4qwkeVZzjlPJ8EAKgIFkNVts,1555
+crawlo/core/scheduler.py,sha256=-6DBz7gUg8WwUl39DAbi6Ng2AJSswNBCDr_mV3sUZFs,14088
+crawlo/data/__init__.py,sha256=UPqgioMdu3imSUmpLWzVlpvoBnEfaPSAT-crCcWd7iw,121
+crawlo/data/user_agents.py,sha256=zjjFkldQkqtrn45j0WZplaZLannPxZDeAU0JofxQcBc,9891
+crawlo/downloader/__init__.py,sha256=VZG5HiSHOmimiH9okQN3MBwgXsCzxr2awflVz5UiboY,8897
+crawlo/downloader/aiohttp_downloader.py,sha256=-dIFucMOQhiiEmtgEpG2Lqh1vF-PvDddbIrZ8Hge0Ig,9556
+crawlo/downloader/cffi_downloader.py,sha256=QxoeocCE2DsQCnhZla6-BjhplaTZDWMbEJmNrghWSDA,10488
+crawlo/downloader/httpx_downloader.py,sha256=MpgDeIdGqNsiSKLOEDBnr5Z0eUbhHnqVEmAuoIfJmFU,12296
+crawlo/downloader/hybrid_downloader.py,sha256=dNnFeegRnyLaOxTWI6XrWKqqVPx80AZBZNgmrcKRVBM,8240
+crawlo/downloader/playwright_downloader.py,sha256=L-TVzG7cYfuBlqW0XSZuz5C_r9fpJrmYNcoQ-cDEna4,16663
+crawlo/downloader/selenium_downloader.py,sha256=P8GuhEw6OYVeN3oeksuBLpUJCELXiu0mAR23X6IIOAA,21508
+crawlo/extension/__init__.py,sha256=-R4P9fklpgSB8cGEduMsjkbJZ7ReYSrZaYjApgYUm9U,2986
+crawlo/extension/health_check.py,sha256=stDpyP4gOzAdbBlPbSf0rge0QounAhF8CtrGq5fa_7c,5657
+crawlo/extension/log_interval.py,sha256=N25aNjFkjh9br6g3ViFqRrz06C2geAdfGas-OT2oZh8,4497
+crawlo/extension/log_stats.py,sha256=CWjMb_V1o8j8uwGFvh9SZ7EYLl_OYzmuIsOT5V-_BE4,2452
+crawlo/extension/logging_extension.py,sha256=WnHVoC4aLHYLapAN0ylt3k5aanP_T1GOyJrAVy-6ePE,2415
+crawlo/extension/memory_monitor.py,sha256=fClPchpCkVjcIiU0AJHCKDd7HEiz5B4KqNqKTRZ2hcU,4394
+crawlo/extension/performance_profiler.py,sha256=BjWD3LOb4VwjQJQvQtWNg7GluEwFquI1CztNfgMzy3c,5032
+crawlo/extension/request_recorder.py,sha256=KA_RmcfscDxP5wPdolO76yKfRj-1jmHhG3jkVGO1pbc,4181
+crawlo/factories/__init__.py,sha256=24dH70p05pZerO9-9gaKpTawRGeGvQYw7j5brvq8GUg,714
+crawlo/factories/base.py,sha256=loB_vyc0CsQK0BgwRoSOFS8gPcmv-b9irtjC9UaBGA4,1832
+crawlo/factories/crawler.py,sha256=e9zl4qytByzsYbz66klY3cZTvQei0-9GjdFK4XCyXcg,3198
+crawlo/factories/registry.py,sha256=YU87CdsntOz609M0aQbGcCG9glPinUJxOn-_CaM4f-M,2595
+crawlo/filters/__init__.py,sha256=noSe07tp2Ip_zXwAbS021BojrqNRaObDO-2YV6DOQfc,4381
+crawlo/filters/aioredis_filter.py,sha256=unms0WaRsxbCL6VaAQMT-SsBHKyxR6-o118pf-3ErK0,9512
+crawlo/filters/memory_filter.py,sha256=ZojFhZ6gE76aQBC-rfImxSkSMwQtiotenx0pIcQOaFg,9561
+crawlo/initialization/__init__.py,sha256=uNRMm9GccMYZi51scpvo-CPx_3ayp3Y81psBHlUoDfw,1132
+crawlo/initialization/built_in.py,sha256=1uEEtYCTQlfx5uRW-s9oumlmIEJrjZa0QpOLAbZbZqI,15758
+crawlo/initialization/context.py,sha256=wG9t-M-Qttj7TN6gDumPX5Q5GHaPDUpLTZZDne2r3TE,4863
+crawlo/initialization/core.py,sha256=sMiSBueoaWoDohQJ50IFC_DSvj0EeYSB7G1MORlDtMc,6872
+crawlo/initialization/phases.py,sha256=iWhGITh9eudfSmzf2G3DLPAIJkCDrv9TVBtnAoS1_3c,4176
+crawlo/initialization/registry.py,sha256=kKVegqWxtPCaZ1mTyVHN4yFecAGDOPFJfebkP-SoobE,4919
+crawlo/items/__init__.py,sha256=rFpx1qFBo0Ik7bSdnXC8EVTJUOQdoJYGVdhYjaH00nk,409
+crawlo/items/base.py,sha256=q0YTJlqUtizsqXwfWlk0ndcINV9dDyUckwMx8_JrkeY,602
+crawlo/items/fields.py,sha256=l-DIwK6CCpdzNvf6ELz7Ckc7YCghZD9UCXA8vhNn2UE,1852
+crawlo/items/items.py,sha256=OmVEvMmgofMU95GkaiWkfNQ2fjsH2fY9sw3SKcmUhLs,3478
+crawlo/logging/__init__.py,sha256=NlvL0sc9NZqy-Poplwpd2wsUleGyg33MyRb1wxyG-zs,1184
+crawlo/logging/async_handler.py,sha256=cMBVi9Ue1y5yZ7r0Uzwr7j_4nyxwmcGXwSE1J7_yIOw,5259
+crawlo/logging/config.py,sha256=0N7w542vlKyE5tSLAbDJYm9U4_1lQro_TQKVIJE_pA4,7167
+crawlo/logging/factory.py,sha256=b4Z0fBmP00GpvpJ7k4QxqYP32n_EqG5KD3ouUWU7L4U,6656
+crawlo/logging/manager.py,sha256=aem7yla0q83rf2CpwQEyg5YMbey4TzkquBVWiKtcqdQ,3182
+crawlo/logging/monitor.py,sha256=mzZWm3rQ2mGUoAmkEJPUkBmR0VWK66l14aqqhQ0zwE8,4935
+crawlo/logging/sampler.py,sha256=1BoRMpusP3wbXRnet5xl9_Yb_3_-AUq9WJhK9gYg7v4,5292
+crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
+crawlo/middleware/default_header.py,sha256=Pw-ev8ffi16GeCh84R5L3hAZgp3G1QXS-H5kV3JEp4Q,5164
+crawlo/middleware/download_delay.py,sha256=2iWnJFtWDlqDy5MsAob8TPiJQoiz9v21yatkBI0eptg,3542
+crawlo/middleware/middleware_manager.py,sha256=_Kgd6Ir4cRUiPCEHJELZPOkKNtmu-WAE59dRWKPpAU8,6415
+crawlo/middleware/offsite.py,sha256=4tUkPqXMMXsi1WwYnJ_e7wMd6sRgK19QHRCYq8-w8jk,4682
+crawlo/middleware/proxy.py,sha256=uKk5OSLIs7jv9bBgkZwsi1rIpthooxhMrGBC2BPRDCc,16022
+crawlo/middleware/request_ignore.py,sha256=7qdX4zAimjSGwdod_aWUbOTfzLBWZ5KzLVFchGMCxCI,2663
+crawlo/middleware/response_code.py,sha256=d5t0hmP8QliuvvtFOqW-ogCBtZxg2eyjsOtlQAEUxM8,4533
+crawlo/middleware/response_filter.py,sha256=tVGr06bfJBR3xAHI2G5c3WimFsGHu8qoJtDcsVuCATU,4384
+crawlo/middleware/retry.py,sha256=Acfo95B9wF8fQTCQIqluZOS2hHdnknQu_FOHvpGKJp0,4248
+crawlo/middleware/simple_proxy.py,sha256=rQ4RkqewGvDRCw021nGrg8ngkBzg3wqrEVqvSmBgQ6M,2256
+crawlo/network/__init__.py,sha256=bvEnpEUBZJ79URfNZbsHhsBKna54hM2-x_BV8eotTA4,418
+crawlo/network/request.py,sha256=e6-YLgK7SU8D19n21mQwqt_b_aeRVJFOgWPIBPal2ys,14178
+crawlo/network/response.py,sha256=QwJhL3xJfPVy_gwtGrg61oAgaqCoCmjyj1Ug7Zju7Pg,13060
+crawlo/pipelines/__init__.py,sha256=FDe2Pr5tiHtV8hFlheElRO_O1aVKvSWlkTcAl9BXAKA,637
+crawlo/pipelines/bloom_dedup_pipeline.py,sha256=vIF_6noJAdpotrJpnCmrVXCi59gRmHHn28mYW6VukbM,5465
+crawlo/pipelines/console_pipeline.py,sha256=bwe5hZgaVSWmh3R8XpOaaeAjJme-Ttrpo6G6f1cnLIg,1287
+crawlo/pipelines/csv_pipeline.py,sha256=qbXZoqq4FIR9QkUGpC0ryWzmqGJSrM2bxmWLM4I1nXM,12490
+crawlo/pipelines/database_dedup_pipeline.py,sha256=IxahtD_mhni-Y21_idOMX58_Htf46A7n52enG9VR2PI,7296
+crawlo/pipelines/json_pipeline.py,sha256=wrCsh8YInmcPLAkhPrHObMx89VZfhf-c7qRrYsTixPE,8585
+crawlo/pipelines/memory_dedup_pipeline.py,sha256=lKkYPu6vkpPjfQ1-xOLvPFT4VdTI8QVx0yjqtVR0ZB0,3598
+crawlo/pipelines/mongo_pipeline.py,sha256=PohTKTGw3QRvuP-T6SrquwW3FAHSno8jQ2D2cH_d75U,5837
+crawlo/pipelines/mysql_pipeline.py,sha256=Kjgu6cks1KD4FPXwlTnFaos2LG-N8LLaBDyKZ_MEcsI,14196
+crawlo/pipelines/pipeline_manager.py,sha256=R6MRb5d-caOit7PZoglJLHa3qQ68U5YAQlwt8KcjRxo,4393
+crawlo/pipelines/redis_dedup_pipeline.py,sha256=RB1kXLr8ZuWNrgZKYwt--tlmnWsQTbuwTsSt3pafol8,6077
+crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlo/queue/pqueue.py,sha256=Q4H2ePag7-pBjzJ0a4S1P4z_IT_G08T6l1uZBkuxO5A,1262
+crawlo/queue/queue_manager.py,sha256=Y2K4P-il5ACdP72X5jgTT-65Fu94_kQZ7H4c7AdRt-c,21547
+crawlo/queue/redis_priority_queue.py,sha256=vLvg2toKaRrXD1QyEdu1ZjTmANv7clFaBF7mCtstBmI,15995
+crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
+crawlo/settings/default_settings.py,sha256=DdoMvB7M_OyGKlBOr0lY-T5TN4V1bo6PlPvu4ZcQXnY,12836
+crawlo/settings/setting_manager.py,sha256=yI1tGaludevxKGGZO3Pn4aYofrg2cwYwvMZCFC5PPZw,8595
+crawlo/spider/__init__.py,sha256=QGhe_yNsnfnCF3G9nSoWEw23b8SkP5oSFU5W79C5DzI,21881
+crawlo/templates/crawlo.cfg.tmpl,sha256=lwiUVe5sFixJgHFEjn1OtbAeyWsECOrz37uheuVtulk,240
+crawlo/templates/run.py.tmpl,sha256=Mt4gcw7RJu9ri4eBRnsAzf0EDt2giFWzpX36OTqEUpQ,961
+crawlo/templates/spiders_init.py.tmpl,sha256=p6UK8KWr8FDydNxiAh6Iz29MY5WmgXIkf2z-buOGhOM,354
+crawlo/templates/project/__init__.py.tmpl,sha256=aQnHaOjMSkTviOC8COUX0fKymuyf8lx2tGduxkMkXEE,61
+crawlo/templates/project/items.py.tmpl,sha256=8_3DBA8HrS2XbfHzsMZNJiZbFY6fDJUUMFoFti_obJk,314
+crawlo/templates/project/middlewares.py.tmpl,sha256=fxHqi-Sjec5GiHJciprOU-6SAUTzM728NlZckIqf9hM,4278
+crawlo/templates/project/pipelines.py.tmpl,sha256=j9oqEhCezmmHlBhMWgYtlgup4jhWnMlv6AEiAOHODkg,2704
+crawlo/templates/project/settings.py.tmpl,sha256=gGYwPf-RCuw1pWyBZRUvL1BHzgoLYcIQ1sXUmWDGp1k,7050
+crawlo/templates/project/settings_distributed.py.tmpl,sha256=YSN7YcA05MT2yO5mg5MpryVyeEDhcaJEp9TU2NAUPS8,7147
+crawlo/templates/project/settings_gentle.py.tmpl,sha256=HS1qumqDciXFBTZYm-RS__ldphDVbuo6Poz8pTCHNEg,6797
+crawlo/templates/project/settings_high_performance.py.tmpl,sha256=u6n7S0KzSgJU23pkRifXnsS7AdQplJXefRyeG51v_QI,6887
+crawlo/templates/project/settings_minimal.py.tmpl,sha256=GJG44CJlop0_kDwtqhbKJlG_42VThHfICGVUNijkkRs,2550
+crawlo/templates/project/settings_simple.py.tmpl,sha256=fmy5PgrS_uVlXnIwXQFQ1Q28Ls121lbVg21Dbxrn6B4,6649
+crawlo/templates/project/spiders/__init__.py.tmpl,sha256=llhcIItXpm0TlEeumeLwp4fcYv2NHl8Iru7tLhDhxiE,216
+crawlo/templates/spider/spider.py.tmpl,sha256=KvU-9YpN6MifDE7XzejjyyQS7RUjLDLZ8zqJcLwSsu0,5198
+crawlo/tools/__init__.py,sha256=tOYfYPvZlrO8cmvnMWBjTma6UTLTFZN3qdC8pJwHrzI,4142
+crawlo/tools/anti_crawler.py,sha256=LwLC6BkxDSkxc5H1hQ6kY9j7O0PZGAMPZECr7gbqw2M,9431
+crawlo/tools/authenticated_proxy.py,sha256=ULCK0Cc9F2rGhRqu6kzKBdxzK9v2n1CsatSQ_PMxpAg,7272
+crawlo/tools/data_formatter.py,sha256=iBDHpZBZvn9O7pLkTQilE1TzYJQEc3z3f6HXoVus0f0,7808
+crawlo/tools/data_validator.py,sha256=bLWnkpFdclJuqjtSAgMI5nznN4vAuPwE1YaiFWKWenM,5490
+crawlo/tools/date_tools.py,sha256=QOT3W5MqcEQhVM3cTZYxu1MRfgX-TI4aF1RI9s0QbdE,9195
+crawlo/tools/distributed_coordinator.py,sha256=kkRbRoxz7iXKI3AQElyTptDpYl352ErbSkM3wjSHVwU,12574
+crawlo/tools/encoding_converter.py,sha256=CqHAsR2rwxuzsyR-TeQNb79HX5mH4KAUixEY-sX7204,4170
+crawlo/tools/network_diagnostic.py,sha256=X1hSbUthIVbMHCU7ti43Zpu8XTaDJd5Oxr2zAkEuSB0,13013
+crawlo/tools/request_tools.py,sha256=oXrk4yWMACVa65fDQCQgzsg6a94FH4_lS7qNR53FHYU,2420
+crawlo/tools/retry_mechanism.py,sha256=4AQ_HLuYt4hYMI9XHoKFk2GQKEiDJB5pAnsMCfjc6Bk,7777
+crawlo/tools/scenario_adapter.py,sha256=pzysL1B2uQ1ZSEncVHd9Hv2viHNgaxP44YAUcDcppfw,9660
+crawlo/tools/text_cleaner.py,sha256=UrMGcgRnJaufjmDKIDsRYKMA8znCAArHDgouttWPygk,6690
+crawlo/utils/__init__.py,sha256=8kMbOZf9bzOUjtvh2QvqXZmiZh3pYzxXH9YQhYcwcoY,597
+crawlo/utils/batch_processor.py,sha256=8LNy-K2SrQVUxmGEWxQyYw_j9M-erN4Ie7O4d3zpBvM,9142
+crawlo/utils/class_loader.py,sha256=kZRGfyA3OPAH2QsQ-beOKjw3JKKBs6OyJyJyXkvWDrc,675
+crawlo/utils/controlled_spider_mixin.py,sha256=8CuM3Cr2wQLHbaO_ohbCsPcImJnyfZHpERbSeMgQ-AQ,16936
+crawlo/utils/db_helper.py,sha256=ZqOt1d3mErVv4TOvoWlov0niUxORB9aHByTmMoNFIDw,10902
+crawlo/utils/enhanced_error_handler.py,sha256=fJC__rnYNKTNUHNbgjZtT846HoE31qyGbPft9bwyYLU,14214
+crawlo/utils/env_config.py,sha256=W-VD_WF63DHxsyJysvp1eJwRh3L_pBRl_PitQAY3nQY,4079
+crawlo/utils/error_handler.py,sha256=vJ_4EVkuVn_TrM16VgN4doyhe_Pg7xWwVUWIENgJQAg,5455
+crawlo/utils/fingerprint.py,sha256=70Me5avs40HYbz6LQ9La56EVP2qRapYjX-zF7WQDGkM,3687
+crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
+crawlo/utils/large_scale_config.py,sha256=NZMsDj4qbVx06Fu0aHqNKX1yzo6WFT7CgrhVnvw1ZFs,8372
+crawlo/utils/large_scale_helper.py,sha256=4ORkZcIrwJ0SlKOUh7l7WIuERORuRhNBgHCM71Rz0n0,12452
+crawlo/utils/log.py,sha256=KmUWVYq8t6fSGOC88nnYCDxwBUdoPWvaBmpOSHn2oWI,2914
+crawlo/utils/performance_monitor.py,sha256=32KspSo7RWvCX_fl0ZFn4ScWWOqbVVwEhPRd921Ez6I,9832
+crawlo/utils/queue_helper.py,sha256=gFmkh1jKlIcN1rmo2Jl6vYcLP5ByUWlfHO9eNlZPBLs,4918
+crawlo/utils/redis_connection_pool.py,sha256=EsPZkmQctWkoYU2wcrqkgwnIWnE6nG4XCXECKn216JA,12575
+crawlo/utils/redis_key_validator.py,sha256=-UTTx0Ul184pzwSply8hVdH0lp-gkXXOc_gEHR_7VlU,5809
+crawlo/utils/request.py,sha256=ejdKpTwc-HE04HQybafhOVywzz57IV3pY0YMkSLyGUo,9065
+crawlo/utils/request_serializer.py,sha256=b5abcgjJk4IU6Wfg46AmOAU2wmzu_WqcpEbuAncRMGQ,8931
+crawlo/utils/spider_loader.py,sha256=xNzQb7qhQ7TqZsfFtCLpuVcsGi-USriZosU0YSBr9II,2233
+crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
+crawlo/utils/tools.py,sha256=uy7qw5Z1BIhyEgiHENvtM7WoGCJxlS8EX3PmOA7ouCo,275
+crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
+examples/__init__.py,sha256=NkRbV8_S1tb8S2AW6BE2U6P2-eGOPwMR1k0YQAwQpSE,130
+examples/test_project/__init__.py,sha256=BQ6FVVDjB00-Vyib8h2I3P-LV5tjsFzTSLC2rFDe7Gw,136
+examples/test_project/run.py,sha256=HHMFVY8D9ouWytp-xNvFDKB8NphrZGuFbVrah6-afG8,953
+examples/test_project/test_project/__init__.py,sha256=RU5IUwU1oeDABAj3ZsLdMo53r3XG4saYIYgO1SXo13g,57
+examples/test_project/test_project/items.py,sha256=L_bnTuJLW7xtuja4AaqYuSsQvVdaqMunZWdh1OTz4HY,310
+examples/test_project/test_project/middlewares.py,sha256=fE7AUI2Yb9_ZwA4vC4ngn-s231FtEYeYWjXOwd5mkzE,4270
+examples/test_project/test_project/pipelines.py,sha256=7EDGrvuNH6JY0XINRwzgPmB-X4Ax6J790jtxbkL2O_U,2696
+examples/test_project/test_project/settings.py,sha256=imfrkrwSJ1V3jINe2ZhynlT4_w5bUBkeI9DFK7DS2g0,7115
+examples/test_project/test_project/spiders/__init__.py,sha256=Lx_To88ShpAR6Pyd9PUMjct690MZH0gETxz6knRqPRY,212
+examples/test_project/test_project/spiders/of_week_dis.py,sha256=-gDLaKGoF5birxCoLL_CX82bYopXjo4QmOV6a7I-Ci0,5178
+tests/__init__.py,sha256=409aRX8hsPffiZCVjOogtxwhACzBp8G2UTJyUQSxhK0,136
+tests/advanced_tools_example.py,sha256=1_iitECKCuWUYMNNGo61l3lmwMRrWdA8F_Xw56UaGZY,9340
+tests/authenticated_proxy_example.py,sha256=fKmHXXxIxCJXjEplttCWRh7PZhbxkBSxJF91Bx-qOME,3019
+tests/baidu_performance_test.py,sha256=wxdaI7UwKboMYH_qcaqZLxAStvndH60bvKGzD8F-jaI,3974
+tests/baidu_test.py,sha256=NKYnwDbPJX3tmKtRn7uQ_QWzUXiLTQC-Gdr1cQkJzEo,1874
+tests/cleaners_example.py,sha256=blVqSJ7SeWUNd17JjHZJgVTzWH65XKevLyaMB_Wg8qA,5324
+tests/comprehensive_framework_test.py,sha256=_1N-OGbKvBTNachNvIjkL_izr4uv6OUybUkhxxz5MAk,5977
+tests/comprehensive_test.py,sha256=wypCaB56IV8w8nd5VA5LSXUQ3IgLf0AKKUiCci6yEJQ,2969
+tests/comprehensive_testing_summary.md,sha256=S01gclEas5oDhSLjxT_mOoeMl8FE4NMA-gveQOrIS4g,6513
+tests/config_validation_demo.py,sha256=jbZ7h-HGsJmuqBb1euB_AhmKjllkvPmItRF1K0MQrVM,4171
+tests/controlled_spider_example.py,sha256=2SAQKoREGHe-OzVaSkGpopCcrou6QXmeW7rLdmsyopw,7981
+tests/date_tools_example.py,sha256=XI3iFEzeo7Nb5YepK8WHytIaBegtxWVSISpqQgpV6M8,5042
+tests/debug_configure.py,sha256=9VZRJtA0u_SbCC_ZMdsiT9raseRE-C2spZpZuXZNysI,2213
+tests/debug_framework_logger.py,sha256=p-dfOjEipjXQd6yk75S-SMoepzwujiwuv3rNwNSyjow,3413
+tests/debug_log_config.py,sha256=cPS6qOLnynYTFOxpjcy9OUgIqrkasWb9f2c_PASc2_E,3714
+tests/debug_log_levels.py,sha256=CZWG3KGDq-hYJ5TPhoZTyjKFKkkM-AoK3oP1w-JC1sc,2168
+tests/debug_pipelines.py,sha256=FMb36bH9lQxBLb-nM579hBRK1S16Vxu1t_BC3Dj8O2w,2164
+tests/detailed_log_test.py,sha256=oTCFF_Un7Jq2gV4rpRDFOxlHJSthnQhvEf0CSItfB7I,7501
+tests/distributed_test.py,sha256=u6cEiymZzCItaTClKTxwVjNmOj9_PZii4_eGNAVMDW8,1825
+tests/distributed_test_debug.py,sha256=pUv6ZKEJ5pK9xOA7lgVk6WW3cBAtnb1bsuZzJ8oGLvY,2181
+tests/dynamic_loading_example.py,sha256=7LdeQZFevrb-U1_dgr4oX3aYo2Da4HvE_0KIf1fw4Ew,18786
+tests/dynamic_loading_test.py,sha256=dzDW7b66HeDsIYsYgvNRihE3V6b6gEbUGQpp-eJbcIM,3413
+tests/env_config_example.py,sha256=_ZRDh_LR23ZKpy9E--y_KM0QIOiZF5vRT98QTn52TY8,4951
+tests/error_handling_example.py,sha256=grTeo1X17rFz4lhgASb0g5yu4NWbmNz5neyuonnNR40,5294
+tests/final_command_test_report.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/final_comprehensive_test.py,sha256=szTNbtwKfYNmE0kzDPCsE_kvnTG7FNKl2JERakGhKIk,4314
+tests/final_log_test.py,sha256=CpZ4ZvvuvFiBvz1a50qN599XIU086ett_I0bSX42BLU,9367
+tests/final_validation_test.py,sha256=4cuTr58i46JI6M4Tz54e7vrVFrOr3R7HSWgyQPKmM9M,5244
+tests/fix_log_test.py,sha256=hcRy0j3j0CT0oLN7KNA0VL-_o4M-uE1amR6GziBflfU,4440
+tests/framework_performance_test.py,sha256=Qp47VrsCK0ylEhDkFOm7lnD8rVkaJ7u1MopsEhAomrE,6985
+tests/log_buffering_test.py,sha256=0B5UY1yQuxnBU1pEyz3IBYweN__4fOkPXly-kYfOpNU,3226
+tests/log_generation_timing_test.py,sha256=zHb_m2FqlpRCYw-wqFWFn8cbVH8UR3VvXKSM6nNnbgo,4681
+tests/optimized_performance_test.py,sha256=bA0dN4j7ViyTSSiCJEjlkJ9Y7jspTFKs2xX7UXHE8Gs,7379
+tests/performance_comparison.py,sha256=CKGFbcwqsF3CAWIaOWF3Ca6o_OdEdZCgIfYb_m6CzIw,9302
+tests/queue_blocking_test.py,sha256=hp-6hmTOO64oOAWVtlN8cFJ95GjbK3t9fj-4q_TKowk,3955
+tests/queue_test.py,sha256=HeBiBXqAgIAbUkLVQ3McS6NdRselA30m3lnuxNBvZbk,2689
+tests/redis_key_validation_demo.py,sha256=WD2jvuBwHhLYIb3lVFtvYSSnmXWn1EW4EPCEwFhfi6M,4467
+tests/request_params_example.py,sha256=J50NdsnK1sDrqG-5m3oA-mu1_wHwVwHIfsWxGeQpz7o,4250
+tests/response_improvements_example.py,sha256=t1cbG3nesp82bqog4_ku1GvQzNbhRyWa5EaKTmOPrSk,5402
+tests/simple_command_test.py,sha256=8TowzW45ukKTPeaNC5uij3RR7rqPULiBr2PguSSMdP8,3688
+tests/simple_crawlo_test.py,sha256=gfL910xt56HhrERAB9QhO_oxKWKw2j2IsifQkgYtwvE,4848
+tests/simple_log_test.py,sha256=xOEGH5UzRLsCpwqgi1VEoZ7NY3DoWckwy6Wy9lKQ6ZE,1757
+tests/simple_log_test2.py,sha256=Rn3XerVlkT0M-vbQmrQL7bVIZG3REnJNmMvUvKr6C20,3944
+tests/simple_optimization_test.py,sha256=hflvaC81ra1ZrPOp-Z7rQrH95OnSADvAjy95BLulD6o,3678
+tests/simple_queue_type_test.py,sha256=wAf4XLKl9oS5BlfrRJ1SLY-kYmNq4YY0LdIC7HmW-yg,1193
+tests/simple_spider_test.py,sha256=RzziJg-fbIVJ6_CgbismfkwrLwpJp4WWp2RLgG7Tpws,1168
+tests/simple_test.py,sha256=Pyxgg0YnBG_3_NRFla0HgJ21CYlfO4K-js3x6_-6ZEk,1258
+tests/spider_log_timing_test.py,sha256=pvYpKZemClr4mCR76xywhsiWbT5sPdzD_taZKFjlgvM,5573
+tests/test_advanced_tools.py,sha256=HT_TcwfFzli-CavIJSqQqnCxnBn5FDMX09zL7AJ5tNY,5398
+tests/test_all_commands.py,sha256=VgVa9SzU5Irvn5igHpC2W4E_6ZDWDt7jc-T4UPK_PFE,7718
+tests/test_all_pipeline_fingerprints.py,sha256=NDrBYr0f9CAhjmSezTS4NUrAdcotrSX3ElJTWqjXXbU,5308
+tests/test_all_redis_key_configs.py,sha256=dWc4Dsr07_vuSpb4hwkMpyy6XO8SI7vglVjGuGvXoa4,5710
+tests/test_authenticated_proxy.py,sha256=lnvmQwuf0zaZP_E05EzcNFR2VJbwTkLjOmZGNoJKaC4,4339
+tests/test_batch_processor.py,sha256=4_nYlu9R1JkDCFHq0bYc9LUNqsg41r7sQ879hkrhEts,7212
+tests/test_cleaners.py,sha256=HDK8_YU7GUj_3hGU415cxEeUR74mnDSk0yroLlgDI0I,1816
+tests/test_component_factory.py,sha256=V3hO5pJHSDtViLAykXSUqkeH4g-GB4GczwutrTatS2U,5809
+tests/test_comprehensive.py,sha256=dvRJeeVYc1cgXK9Y171hH9Y847zZpWSAFFH-EI3UepQ,5182
+tests/test_config_consistency.py,sha256=RgSxyaypMpysltsGSh1vFMeOShiZZG0rmUKzEhNLpYw,2001
+tests/test_config_merge.py,sha256=ts1j-TIKkFS0EO5q1I4O7f4YUKR5MLTmRSqOpOlv094,5606
+tests/test_config_validator.py,sha256=Z4gBHkI0_fEx-xgiiG4T33F4BAuePuF81obpNTXfseY,6202
+tests/test_controlled_spider_mixin.py,sha256=AQ493ic6AxZAKd7QCgnUES92BBWCMNteTd5DjoQlhwo,2864
+tests/test_crawlo_proxy_integration.py,sha256=81DVwosMoiSMxj4V_jLzcL7aqvSv_8ucggkQyXsvzT0,2733
+tests/test_date_tools.py,sha256=pcLDyhLrZ_jh-PhPm4CvLZEgNeH9kLMPKN5zacHwuWM,4053
+tests/test_dedup_fix.py,sha256=UFdm8lIi0ZIdp40W8ruxRD69bxzijuFUfNyJmB4Fwl0,8788
+tests/test_dedup_pipeline_consistency.py,sha256=dn5EAZSU5gQOV5EQwreHp76i5aQZ9tEdltSGO7dif5M,5176
+tests/test_default_header_middleware.py,sha256=UDjEPIUCre1M6ndjV_uXLVCfY7WJwyN-1Xn15hzbKMo,13126
+tests/test_distributed.py,sha256=78Pn4HPLIaO8t1IiaSkckBmuEVTcnC8IDw7znf9_Zcw,1790
+tests/test_double_crawlo_fix.py,sha256=lZwrT5ij6Jbh0EzZswhw05FXwgKaEZsSHekLTrJJajg,7856
+tests/test_double_crawlo_fix_simple.py,sha256=NDmCEeyvpf_D1tGQMA66iLPPKlAnSZcEg71e7GHYcjg,4768
+tests/test_download_delay_middleware.py,sha256=Idc6KzhL3hY3aDKgn1j_v5-mLIHz7dTnV5c4tJVZh5Q,9107
+tests/test_downloader_proxy_compatibility.py,sha256=0hgIzWXIqd92YXEB5sNneyp4Sk7PaG76up2cd6N9QQY,8903
+tests/test_dynamic_downloaders_proxy.py,sha256=t_aWpxOHi4h3_fg2ImtIq7IIJ0r3PTHtnXiopPe2ZlM,4450
+tests/test_dynamic_proxy.py,sha256=zi7Ocbhc9GL1zCs0XhmG2NvBBeIZ2d2hPJVh18lH4Y0,3172
+tests/test_dynamic_proxy_config.py,sha256=C_9CEjCJtrr0SxIXCyLDhSIi88ujF7UAT1F-FAphd0w,5853
+tests/test_dynamic_proxy_real.py,sha256=krWnbFIH26mWNPhOfPMmx3ZxJfOreZxMZFGwVb_8-K8,3511
+tests/test_edge_cases.py,sha256=1RnFaCebYTDNNz_LK8M0MepiSwPvJUk_FBK4nQTCUbg,10729
+tests/test_enhanced_error_handler.py,sha256=Ku_86jv7iDe25v8ZxalcXxJJjIiIvQXWH8ZldbwdVm8,8581
+tests/test_enhanced_error_handler_comprehensive.py,sha256=j_cxyIPGks9A3untKhAdj5HU0hrLbbzOLu0uAtGUlJo,9369
+tests/test_env_config.py,sha256=Qu1sDeADs69dSr1x0QmEe8nJrMHneE_4JClt-N901e8,4867
+tests/test_error_handler_compatibility.py,sha256=xJ43cmCwfBGh-qBwCGiMDPPlfNDLw4ZrmlrHN9IojkU,4241
+tests/test_factories.py,sha256=wKFfr8YBXPs-AQ8YOFgDhINn5uivKqPBZQPUe5GL9Ig,8865
+tests/test_final_validation.py,sha256=OuZI01O0E68Pao--bD-BFDTRZFPc_Mt4W-OXUzlt6ZA,4966
+tests/test_fingerprint_consistency.py,sha256=68V5u_2hNABI5pNWzXUrA1PJ08Xh9x3-JsMSNNjORMo,4956
+tests/test_fingerprint_simple.py,sha256=qiSba8gF3Zl91QO_ijJO7KstLdjATs30V_GZCNHShig,1626
+tests/test_framework_env_usage.py,sha256=bFb_ptdLeX2obdJWEqEHPWweiWR-wR2BpvEaJMQK7h4,4201
+tests/test_framework_logger.py,sha256=nAtL_N49L7OurthY329vZK_jBjJIcPcETEIiV0HGqt0,2560
+tests/test_framework_startup.py,sha256=I_ij7J6NO3DTBuHlh1Z4CJUcGxLEjRaIB6EdykiISEc,2267
+tests/test_get_component_logger.py,sha256=UKj5uT1F3L3atoJFmpk7QSDO2fZHgw-7Y84vMFbHRkM,2285
+tests/test_hash_performance.py,sha256=4eVPwbu66Oun0LVyTTNd9d2cj2V1xq0YZkRg8Z0TO-Q,3211
+tests/test_integration.py,sha256=lVEzKNAjFzFZHRNZAyJmXxa_5Ogf_qqL4APqs620o58,4839
+tests/test_item_dedup_redis_key.py,sha256=dp_H59exJLaZHh5oMtmMEOWh-DNZwbnwIFYDjOpHgd0,3842
+tests/test_large_scale_config.py,sha256=Ik32ilAOQXsyw2sHR53gDPNNjY0AXybQ9ya2JY-EeqM,4296
+tests/test_large_scale_helper.py,sha256=0L6EKHcKgh7XHvoW4wRSkxmw8GolUwSOCgZ_-ZmCyDo,8371
+tests/test_logging_enhancements.py,sha256=YHcYWC8PG_AP5wZnmOr6H7QuU7m-3xzxEhppM0Jubvg,12731
+tests/test_logging_final.py,sha256=K9vxyODslXza05hElVEcvzbXgzthYKK5CRj4UJTftIw,6376
+tests/test_logging_integration.py,sha256=5WpExyt6BmYBZwrjqtQIGOw1Id64opJBAIahDk70Mlc,11131
+tests/test_logging_system.py,sha256=LGfK14ZEWzRtl3_VkBGz-AaVa_dDtuC5zu40m8FvmMo,9206
+tests/test_middleware_debug.py,sha256=gtiaWCxBSTcaNkdqXirM7CsThr_HfiCueBdQCpp7rqg,4572
+tests/test_mode_change.py,sha256=GT53CBdxcG3-evcKz_OOfH4PBiq_oqQyuDjRXrvv1UU,2665
+tests/test_mode_consistency.py,sha256=t72WX0etC_AayaL2AT6e2lIgbfP-zxTgYAiTARSN2Jk,1276
+tests/test_offsite_middleware.py,sha256=njpXTdngOqBs60Wj6xgo5EEXlJnMHd7vtYGi9dVauW0,10602
+tests/test_offsite_middleware_simple.py,sha256=4MfDKSXGHcoFLYnnxCH2rmnzztWyN0xByYLoHtepyiA,7918
+tests/test_parsel.py,sha256=wuZqRFIm9xx1tt6o3Xi_OjvwhT_MPmHiUEj2ax06zlo,701
+tests/test_performance.py,sha256=Lqs2iu3dmWipZkBPARcwIjDLXsqe42ntz1M4RzqqXKo,11457
+tests/test_performance_monitor.py,sha256=paW3HGg6ReHb9lwnOivGCrI8STwbwp_mbuhgfds1h3I,4187
+tests/test_pipeline_fingerprint_consistency.py,sha256=LL55oGSDGy0K8LxoyKa6ogNHXhJlZHe509vCFbibLkk,2847
+tests/test_priority_behavior.py,sha256=p04M0HIgBaXyuVHmp-ImITA9jGaKI_RPwZ3DPY_Trt4,9134
+tests/test_priority_consistency.py,sha256=rVX7nku5N_QpB_ffDu3xqREkCWPX5aNNiXy112o9wpA,5756
+tests/test_priority_consistency_fixed.py,sha256=MlYi5PIr5wxunC3Ku4ilnxOatKyRu2qIvhV7pjadkjg,10765
+tests/test_proxy_api.py,sha256=XnmklS-xU4ke_560gV6AIlBsRmG8YLQTGFAZrTUZuhc,11013
+tests/test_proxy_health_check.py,sha256=_tDlxa_6TdL3M5RLkHF82roXJ8WIuG5hELBp2GADyKQ,1123
+tests/test_proxy_middleware.py,sha256=EdQAfwwAJIBxw9JmUFTDEu_pdxapaTlcJr7KcrY6-AY,4021
+tests/test_proxy_middleware_enhanced.py,sha256=QR-p26F63N7MxNjZ2QJUeerh_xdnCDejkrGPIh7Fh4U,7035
+tests/test_proxy_middleware_integration.py,sha256=mTPK_XvbmLCV_QoVZzA3ybWOOX61493Ew78WfTp-bYQ,4441
+tests/test_proxy_middleware_refactored.py,sha256=VbkTWkmmomcyswobA_gf3p_bERl_eexY2e6ohJQS_A8,6960
+tests/test_proxy_providers.py,sha256=u_R2fhab90vqvQEaOAztpAOe9tJXvUMIdoDxmStmXJ4,1749
+tests/test_proxy_stats.py,sha256=ES00CEoDITYPFBGPk8pecFzD3ItYIv6NSpcqNd8-kvo,526
+tests/test_proxy_strategies.py,sha256=9Z1pXmTNyw-eIhGXlf2abZbJx6igLohYq-_3hldQ5uE,1868
+tests/test_queue_empty_check.py,sha256=ZJC6jOgZq0Wb0-ubrB1ZNcCaUiWeCxoNZmjkd6PY6t0,1182
+tests/test_queue_manager_double_crawlo.py,sha256=MijZ3JuyHMuqGbRC-8kclFr-4O7m_T8CqezP4qiWk-E,6957
+tests/test_queue_manager_redis_key.py,sha256=txHLq5XUZZN7h9HUlqlUCEVCTe2IXdf9r7F_P2zNVdY,7117
+tests/test_queue_naming.py,sha256=GzPqxvATF9A5iQMgn2wQM68qTHTbsQ9dPhktAkyzDG4,4786
+tests/test_queue_type.py,sha256=N8y3s0Cwco-XphRcIiR4bQuNAEOgmxvlRoR1jzghTtY,3304
+tests/test_queue_type_redis_config_consistency.py,sha256=1ew7Zp9CxH1DQ0RUmsZMV-n7s8B5dCly6u3FkJbR1qE,5259
+tests/test_random_headers_default.py,sha256=ulDb3_kRpnTCN1-TO3m6wVM-eMkZS_ezsSbd1ur8Xpg,12772
+tests/test_random_headers_necessity.py,sha256=SSbNQIE347oCQvuG6yaAambFU-3MyQzTV5jN1kArRGY,11741
+tests/test_random_user_agent.py,sha256=6HjU4iUcMk-J6bR2N5FhIkWDfnaFKAPNVyRzxmQQ14k,2302
+tests/test_real_scenario_proxy.py,sha256=clmLvBfap5OpsaCE08MAWap-78jhVrxYfVfDNyoa4Hg,8454
+tests/test_redis_config.py,sha256=51_Fy1PqIhS0MMO2nR4q6oQjBFxfqcUPK_4NNf5s83g,903
+tests/test_redis_connection_pool.py,sha256=pKfXdE3Cm_L_fNqI9zqFmqiidCwR0t7hiM_Fu_V1cNI,9328
+tests/test_redis_key_naming.py,sha256=MTFk656JhiGVTsMctBDhBNOMFcBDZrsQA3UfPZ-Dgj4,6911
+tests/test_redis_key_validator.py,sha256=GszSzGADgk3uN6Bye1d8pS-AtMVgB8jwqW-22gPNM6M,4418
+tests/test_redis_queue.py,sha256=WQV3MtGg8rJzHgC2kRfXM6lSMXpwXJVQZfqn2dVrhg0,6758
+tests/test_redis_queue_name_fix.py,sha256=9V-qDmUz97wOdPv8pKcVnQRxeZ126WSdB5JBnow4ho8,5770
+tests/test_redis_queue_type_fallback.py,sha256=Bg6b5bqt-TKotE_wWdkaZNQr2WXT8gaDfolv8785NyE,4692
+tests/test_request_ignore_middleware.py,sha256=QN81wgG_W_XfXCF9LvJNxCNwbOH6_tZnLIwLDTK2K5Q,6229
+tests/test_request_params.py,sha256=l2etiDebqylPBym1e9DSDn4wxwTHv8DQHKq9AzlzlG0,4287
+tests/test_request_serialization.py,sha256=Ikgec8tt_sPCK6jcZyK8vRw84zRNE6nxQy9rba1WKmE,2332
+tests/test_response_code_middleware.py,sha256=wSe525bm-bk_iWMjPDzUu1LfOQrwJY8_MLKAspq2dzk,12193
+tests/test_response_filter_middleware.py,sha256=YWrGzJ7wmftTjJXcNTtJl3b3EdJsO4oR22ZLWwgErhg,16327
+tests/test_response_improvements.py,sha256=vNqHKyoEoYeEGAUiRzdsff2V6yvJ9QnDwGg7gmN38Ow,6028
+tests/test_retry_middleware.py,sha256=mi7s4HDAqmmd9nvyxs3ZgxdEKOYkCgDu3rDvU_9o8vQ,11133
+tests/test_retry_middleware_realistic.py,sha256=Sam5y4jCN8oeElU4xxeS5zjAyzS-P8siPV7OaifgsyU,9679
+tests/test_scheduler.py,sha256=1fCu35QgK5gzgrhD0aUZj5lxL0QbokzPav-yEJxz9Ig,8182
+tests/test_scheduler_config_update.py,sha256=LuxjEbt20QrPyVkjSFxvTnFtUxwMaHB6TcqjFyo8bow,4261
+tests/test_simple_response.py,sha256=_ui2PuVZvJcAuLY7HZ8xcsy_tDBimgBqX0ukj3kE5J0,1549
+tests/test_telecom_spider_redis_key.py,sha256=c-gfixPul2VlYMQJGf0H5ZgYJ461fQgSKbCPrbAU45M,7625
+tests/test_template_content.py,sha256=2RgCdOA3pMUSOqC_JbTGeW7KonbTqJ0ySYJNWegU-v0,2903
+tests/test_template_redis_key.py,sha256=99-s0_-8MFJbIvGG_X__sH0qkXWTtJv8fdTdlftsq4I,4876
+tests/test_tools.py,sha256=z50Bvq_q8FwpyxNkmh00_A3sXkSv2l1Q_EbK02FDYgk,5504
+tests/test_user_agent_randomness.py,sha256=tE8_zh-BjMAQ9CTgScxZze6JarNher6COkdoLU68YfA,5681
+tests/test_user_agents.py,sha256=e4haX-o8Janl-PawGJ9MemZyMqTX33_tBF_WnYSVoUw,3327
+tests/tools_example.py,sha256=Rxu5vVKnj3CZ3mCx-EEotBWPtZs2S7ktyqq-SYeclxU,7999
+tests/untested_features_report.md,sha256=31aUlsw_1OKe0_ijAjeH85kJ7HJ8qzKLJdOHDjWtYdY,4169
+tests/verify_debug.py,sha256=iQ4Efwg9bQTHscr73VYAAZ8rBIe1u6mQfeaEK5YgneY,1564
+tests/verify_distributed.py,sha256=0IolM4ymuPOz_uTfHSWFO3Vxzp7Lo6i0zhSbzJhHFtI,4045
+tests/verify_log_fix.py,sha256=7reyVl3MXTDASyChgU5BAYuzuxvFjSLG9HywAHso0qg,4336
+tests/scrapy_comparison/ofweek_scrapy.py,sha256=rhVds_WjYum1bLuWWe90HtXE51fZXEqhhPSc822ZasQ,5790
+tests/scrapy_comparison/scrapy_test.py,sha256=-IsGUHPBgEL0TmXjeLZl-TUA01B7Dsc2nRo4JZbFwZA,5599
+crawlo-1.4.3.dist-info/METADATA,sha256=PTbW7GP9xgBjFEX1adwjxeJHqM_x9VRU_ZLZsezaydU,4848
+crawlo-1.4.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.4.3.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.4.3.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.4.3.dist-info/RECORD,,

examples/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 12:36
-# @Author  :   oscar
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 12:36
+# @Author  :   oscar
+# @Desc    :   None
+"""

examples/test_project/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-09-27 23:36
-# @Author  :   crawl-coder
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-09-27 23:36
+# @Author  :   crawl-coder
+# @Desc    :   None
+"""