crawlo 1.1.3__py3-none-any.whl → 1.1.4__py3-none-any.whl

crawlo/utils/url.py

@@ -1,40 +1,40 @@
-from urllib.parse import urldefrag
-from w3lib.url import add_or_replace_parameter
-
-
-def escape_ajax(url: str) -> str:
-    """
-    根据Google AJAX爬取规范转换URL（处理哈希片段#!）：
-    https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
-
-    规则说明：
-    1. 仅当URL包含 `#!` 时才转换（表示这是AJAX可爬取页面）
-    2. 将 `#!key=value` 转换为 `?_escaped_fragment_=key%3Dvalue`
-    3. 保留原始查询参数（如果有）
-
-    示例：
-    >>> escape_ajax("www.example.com/ajax.html#!key=value")
-    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
-    >>> escape_ajax("www.example.com/ajax.html?k1=v1#!key=value")
-    'www.example.com/ajax.html?k1=v1&_escaped_fragment_=key%3Dvalue'
-    >>> escape_ajax("www.example.com/ajax.html#!")
-    'www.example.com/ajax.html?_escaped_fragment_='
-
-    非AJAX可爬取的URL（无#!）原样返回：
-    >>> escape_ajax("www.example.com/ajax.html#normal")
-    'www.example.com/ajax.html#normal'
-    """
-    # 分离URL的基础部分和哈希片段
-    de_frag, frag = urldefrag(url)
-
-    # 仅处理以"!"开头的哈希片段（Google规范）
-    if not frag.startswith("!"):
-        return url  # 不符合规则则原样返回
-
-    # 调用辅助函数添加 `_escaped_fragment_` 参数
-    return add_or_replace_parameter(de_frag, "_escaped_fragment_", frag[1:])
-
-
-if __name__ == '__main__':
-    f = escape_ajax('http://example.com/page#!')
+from urllib.parse import urldefrag
+from w3lib.url import add_or_replace_parameter
+
+
+def escape_ajax(url: str) -> str:
+    """
+    根据Google AJAX爬取规范转换URL（处理哈希片段#!）：
+    https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+
+    规则说明：
+    1. 仅当URL包含 `#!` 时才转换（表示这是AJAX可爬取页面）
+    2. 将 `#!key=value` 转换为 `?_escaped_fragment_=key%3Dvalue`
+    3. 保留原始查询参数（如果有）
+
+    示例：
+    >>> escape_ajax("www.example.com/ajax.html#!key=value")
+    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html?k1=v1#!key=value")
+    'www.example.com/ajax.html?k1=v1&_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html#!")
+    'www.example.com/ajax.html?_escaped_fragment_='
+
+    非AJAX可爬取的URL（无#!）原样返回：
+    >>> escape_ajax("www.example.com/ajax.html#normal")
+    'www.example.com/ajax.html#normal'
+    """
+    # 分离URL的基础部分和哈希片段
+    de_frag, frag = urldefrag(url)
+
+    # 仅处理以"!"开头的哈希片段（Google规范）
+    if not frag.startswith("!"):
+        return url  # 不符合规则则原样返回
+
+    # 调用辅助函数添加 `_escaped_fragment_` 参数
+    return add_or_replace_parameter(de_frag, "_escaped_fragment_", frag[1:])
+
+
+if __name__ == '__main__':
+    f = escape_ajax('http://example.com/page#!')
     print(f)

crawlo-1.1.4.dist-info/METADATA

@@ -0,0 +1,403 @@
+Metadata-Version: 2.4
+Name: crawlo
+Version: 1.1.4
+Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
+Home-page: https://github.com/crawl-coder/Crawlo.git
+Author: crawl-coder
+Author-email: crawlo@qq.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp>=3.12.14
+Requires-Dist: aiomysql>=0.2.0
+Requires-Dist: aioredis>=2.0.1
+Requires-Dist: asyncmy>=0.2.10
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: dateparser>=1.2.2
+Requires-Dist: httpx[http2]>=0.27.0
+Requires-Dist: curl-cffi>=0.13.0
+Requires-Dist: lxml>=5.2.1
+Requires-Dist: motor>=3.7.0
+Requires-Dist: parsel>=1.9.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: pymongo>=4.11
+Requires-Dist: PyMySQL>=1.1.1
+Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: redis>=6.2.0
+Requires-Dist: requests>=2.32.4
+Requires-Dist: six>=1.17.0
+Requires-Dist: ujson>=5.9.0
+Requires-Dist: urllib3>=2.5.0
+Requires-Dist: w3lib>=2.1.2
+Requires-Dist: rich>=14.1.0
+Requires-Dist: astor>=0.8.1
+Requires-Dist: watchdog>=6.0.0
+Provides-Extra: render
+Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
+Requires-Dist: playwright; extra == "render"
+Requires-Dist: selenium>=3.141.0; extra == "render"
+Provides-Extra: all
+Requires-Dist: bitarray>=1.5.3; extra == "all"
+Requires-Dist: PyExecJS>=1.5.1; extra == "all"
+Requires-Dist: pymongo>=3.10.1; extra == "all"
+Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
+Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: selenium>=3.141.0; extra == "all"
+
+# Crawlo - 异步分布式爬虫框架
+
+<div align="center">
+
+[![Python Version](https://img.shields.io/badge/python-3.8%2B-blue)](https://www.python.org/downloads/)
+[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+[![Documentation](https://img.shields.io/badge/docs-latest-brightgreen)](https://crawlo.readthedocs.io/)
+
+一个基于 asyncio 的高性能异步分布式爬虫框架，支持单机和分布式部署。
+
+</div>
+
+## 🌟 特性
+
+- **异步高性能**: 基于 asyncio 实现，充分利用现代 CPU 多核性能
+- **分布式支持**: 内置 Redis 队列，轻松实现分布式部署
+- **模块化设计**: 中间件、管道、扩展组件系统，易于定制和扩展
+- **智能去重**: 多种去重策略（内存、Redis、Bloom Filter）
+- **灵活配置**: 支持多种配置方式，适应不同场景需求
+- **丰富文档**: 完整的中英文双语文档和示例项目
+
+## 🚀 快速开始
+
+### 安装
+
+```bash
+pip install crawlo
+```
+
+### 创建项目
+
+```bash
+crawlo startproject myproject
+cd myproject
+```
+
+### 编写爬虫
+
+```python
+from crawlo import Spider, Request, Item
+
+class MyItem(Item):
+    title = ''
+    url = ''
+
+class MySpider(Spider):
+    name = 'myspider'
+    
+    async def start_requests(self):
+        yield Request('https://httpbin.org/get', callback=self.parse)
+    
+    async def parse(self, response):
+        yield MyItem(
+            title='Example Title',
+            url=response.url
+        )
+```
+
+### 运行爬虫
+
+```bash
+crawlo crawl myspider
+```
+
+## 🏗️ 架构设计
+
+### 组件交互图
+
+```
+┌─────────────────────────────────────────────────────────────────────┐
+│                            Crawler                                  │
+│  ┌──────────────┐  ┌──────────────┐  ┌──────────────────────────┐  │
+│  │   Spider     │  │   Engine     │  │      ExtensionManager     │  │
+│  │              │  │              │  │                          │  │
+│  │ start_urls   │  │  Scheduler ◄─┼──┼──► StatsCollector         │  │
+│  │ parse()      │  │              │  │                          │  │
+│  │              │  │ Downloader ◄─┼──┼──► MiddlewareManager     │  │
+│  │              │  │              │  │                          │  │
+│  │              │  │ Processor  ◄─┼──┼──► PipelineManager       │  │
+│  └──────────────┘  └──────┬───────┘  └──────────────────────────┘  │
+└──────────────────────────┼─────────────────────────────────────────┘
+                           │
+        ┌──────────────────▼──────────────────┐
+        │         Scheduler                   │
+        │  ┌──────────────────────────────┐   │
+        │  │       QueueManager           │   │
+        │  │  ┌─────────┐  ┌────────────┐ │   │
+        │  │  │ Memory  │  │   Redis    │ │   │
+        │  │  │ Queue   │  │  Queue     │ │   │
+        │  │  └─────────┘  └────────────┘ │   │
+        │  └──────────────────────────────┘   │
+        │  ┌──────────────────────────────┐   │
+        │  │        Filter                │   │
+        │  │  ┌─────────┐  ┌────────────┐ │   │
+        │  │  │ Memory  │  │   Redis    │ │   │
+        │  │  │ Filter  │  │  Filter    │ │   │
+        │  │  └─────────┘  └────────────┘ │   │
+        │  └──────────────────────────────┘   │
+        └─────────────────────────────────────┘
+                           │
+        ┌──────────────────▼──────────────────┐
+        │         Downloader                  │
+        │  ┌──────────────────────────────┐   │
+        │  │    MiddlewareManager         │   │
+        │  │                              │   │
+        │  │ RequestMiddleware ◄────────┐ │   │
+        │  │ ResponseMiddleware        │ │   │
+        │  │ ExceptionMiddleware       │ │   │
+        │  │                          ╱  │   │
+        │  └─────────────────────────╱───┘   │
+        │                           ╱        │
+        │  ┌───────────────────────▼──┐      │
+        │  │  Download Implementations │      │
+        │  │  - AioHttpDownloader   │      │
+        │  │  - HttpXDownloader     │      │
+        │  │  - CurlCffiDownloader  │      │
+        │  └──────────────────────────┘      │
+        └─────────────────────────────────────┘
+                           │
+        ┌──────────────────▼──────────────────┐
+        │          Processor                  │
+        │  ┌──────────────────────────────┐   │
+        │  │    PipelineManager           │   │
+        │  │  ┌─────────────────────────┐ │   │
+        │  │  │   Pipeline Stages       │ │   │
+        │  │  │ - ValidationPipeline    │ │   │
+        │  │  │ - ProcessingPipeline    │ │   │
+        │  │  │ - StoragePipeline       │ │   │
+        │  │  └─────────────────────────┘ │   │
+        │  └──────────────────────────────┘   │
+        └─────────────────────────────────────┘
+```
+
+### 运行模式切换图
+
+```
+                    ┌─────────────────────┐
+                    │   ModeManager       │
+                    │  (运行模式管理器)    │
+                    └─────────┬───────────┘
+                              │
+        ┌─────────────────────┼─────────────────────┐
+        │                     │                     │
+        ▼                     ▼                     ▼
+┌───────────────┐    ┌─────────────────┐   ┌─────────────────┐
+│  Standalone   │    │   Distributed   │   │      Auto       │
+│   (单机模式)   │    │   (分布式模式)   │   │   (自动检测模式)  │
+└───────┬───────┘    └─────────┬───────┘   └─────────┬───────┘
+        │                      │                     │
+        ▼                      ▼                     ▼
+┌───────────────┐    ┌─────────────────┐   ┌─────────────────┐
+│ Memory Queue  │    │   Redis Queue   │   │  Auto Select    │
+│ Memory Filter │    │  Redis Filter   │   │ Memory/Redis    │
+└───────────────┘    └─────────────────┘   └─────────────────┘
+```
+
+### 数据流向图
+
+```
+┌─────────────┐    1.生成初始请求     ┌──────────────┐
+│   Spider    ├─────────────────────►│  Scheduler   │
+└─────────────┘                      └──────┬───────┘
+                                            │ 2.去重检查
+                                            ▼
+                                  ┌─────────────────┐
+                                  │     Filter      │
+                                  └─────────┬───────┘
+                                            │ 3.入队
+                                            ▼
+                                  ┌─────────────────┐
+                                  │      Queue      │
+                                  └─────────┬───────┘
+                                            │ 4.获取请求
+                                            ▼
+                                  ┌─────────────────┐    5.下载请求    
+                                  │   Downloader    ├──────────────────┐
+                                  └─────────────────┘                  │
+                                            │ 6.解析响应              │
+                                            ▼                         ▼
+                                  ┌─────────────────┐    7.生成数据    ┌─────────────┐
+                                  │   Processor     ├────────────────►│   Pipeline  │
+                                  └─────────────────┘                 └─────────────┘
+                                            │ 8.存储数据
+                                            ▼
+                                  ┌─────────────────┐
+                                  │     Items       │
+                                  └─────────────────┘
+```
+
+### 模块层次结构图
+
+```
+crawlo/
+├── cli.py                          # 命令行接口
+├── crawler.py                      # 爬虫运行实例
+├── project.py                      # 项目管理
+├── config.py                       # 配置管理
+├── mode_manager.py                 # 运行模式管理器
+├── stats_collector.py              # 统计收集器
+├── subscriber.py                   # 事件订阅器
+├── task_manager.py                 # 任务管理器
+├── event.py                        # 事件定义
+├── exceptions.py                   # 异常定义
+├──
+├── core/                           # 核心组件
+│   ├── engine.py                   # 引擎
+│   ├── scheduler.py                # 调度器
+│   ├── processor.py                # 处理器
+│
+├── spider/                         # 爬虫基类
+│   └── __init__.py                 # 爬虫元类和基类
+│
+├── network/                        # 网络相关
+│   ├── request.py                  # 请求对象
+│   └── response.py                 # 响应对象
+│
+├── downloader/                     # 下载器
+│   ├── __init__.py                 # 下载器基类
+│   ├── aiohttp_downloader.py      # AioHttp实现
+│   ├── httpx_downloader.py        # HttpX实现
+│   └── cffi_downloader.py         # CurlCffi实现
+│
+├── queue/                          # 队列管理
+│   ├── __init__.py                 
+│   ├── queue_manager.py           # 队列管理器
+│   ├── pqueue.py                  # 内存优先队列
+│   └── redis_priority_queue.py    # Redis优先队列
+│
+├── filters/                        # 过滤器
+│   ├── __init__.py                 
+│   ├── base_filter.py             # 过滤器基类
+│   ├── memory_filter.py           # 内存过滤器
+│   └── aioredis_filter.py         # Redis过滤器
+│
+├── middleware/                     # 中间件
+│   ├── __init__.py                 
+│   ├── middleware_manager.py      # 中间件管理器
+│   ├── default_header.py          # 默认请求头
+│   ├── download_delay.py          # 下载延迟
+│   ├── proxy.py                   # 代理支持
+│   ├── request_ignore.py          # 请求忽略
+│   ├── response_code.py           # 响应码处理
+│   ├── response_filter.py         # 响应过滤
+│   └── retry.py                   # 重试机制
+│
+├── pipelines/                      # 数据管道
+│   ├── __init__.py                 
+│   ├── pipeline_manager.py        # 管道管理器
+│   ├── base_pipeline.py           # 管道基类
+│   ├── console_pipeline.py        # 控制台输出管道
+│   └── mysql_pipeline.py          # MySQL存储管道
+│
+├── extension/                      # 扩展组件
+│   ├── __init__.py                 
+│   ├── log_interval.py            # 定时日志
+│   ├── log_stats.py               # 统计日志
+│   ├── logging_extension.py       # 日志扩展
+│   ├── memory_monitor.py          # 内存监控
+│   └── performance_profiler.py    # 性能分析
+│
+├── settings/                       # 配置系统
+│   ├── __init__.py                 
+│   ├── default_settings.py        # 默认配置
+│   └── setting_manager.py         # 配置管理器
+│
+├── utils/                          # 工具库
+│   ├── __init__.py                 
+│   ├── log.py                     # 日志工具
+│   ├── request.py                 # 请求工具
+│   ├── request_serializer.py      # 请求序列化
+│   └── func_tools.py              # 函数工具
+│
+└── templates/                      # 模板文件
+    ├── project/                   
+    └── spider/
+```
+
+### 组件说明
+
+- **Crawler**: 爬虫运行实例，管理Spider与引擎的生命周期
+- **Engine**: 引擎组件，协调Scheduler、Downloader、Processor
+- **Scheduler**: 调度器，管理请求队列和去重过滤
+- **Downloader**: 下载器，负责网络请求，支持多种实现(aiohttp, httpx, curl-cffi)
+- **Processor**: 处理器，处理响应数据和管道
+- **QueueManager**: 统一的队列管理器，支持内存队列和Redis队列的自动切换
+- **Filter**: 请求去重过滤器，支持内存和Redis两种实现
+- **Middleware**: 中间件系统，处理请求/响应的预处理和后处理
+- **Pipeline**: 数据处理管道，支持多种存储方式(控制台、数据库等)
+- **Spider**: 爬虫基类，定义爬取逻辑
+
+### 运行模式
+
+Crawlo支持三种运行模式：
+- **standalone**: 单机模式，使用内存队列和内存过滤器
+- **distributed**: 分布式模式，使用Redis队列和Redis过滤器
+- **auto**: 自动检测模式，根据环境自动选择最佳运行方式
+
+## 🎛️ 配置系统
+
+### 传统配置方式
+
+```
+# settings.py
+PROJECT_NAME = 'myproject'
+CONCURRENCY = 16
+DOWNLOAD_DELAY = 1.0
+QUEUE_TYPE = 'memory'  # 单机模式
+# QUEUE_TYPE = 'redis'   # 分布式模式
+```
+
+### 命令行配置
+
+```
+crawlo crawl myspider --concurrency=32 --delay=0.5
+```
+
+## 🧩 核心组件
+
+### 中间件系统
+灵活的中间件系统，支持请求预处理、响应处理和异常处理。
+
+### 管道系统
+可扩展的数据处理管道，支持多种存储方式（控制台、数据库等）。
+
+### 扩展组件
+功能增强扩展，包括日志、监控、性能分析等。
+
+### 过滤系统
+智能去重过滤，支持多种去重策略（内存、Redis、Bloom Filter）。
+
+## 📦 示例项目
+
+- [API数据采集](examples/api_data_collection/) - 简单的API数据采集示例
+- [电信设备许可证](examples/telecom_licenses_distributed/) - 分布式爬取示例
+
+## 📚 文档
+
+完整的文档请访问 [Crawlo Documentation](https://crawlo.readthedocs.io/)
+
+- [快速开始指南](docs/quick_start.md)
+- [框架文档](docs/crawlo_framework_documentation.md)
+- [API参考](docs/api_reference.md)
+- [分布式爬取教程](docs/distributed_crawling_tutorial.md)
+- [配置最佳实践](docs/configuration_best_practices.md)
+- [扩展组件](docs/extensions.md)
+
+## 🤝 贡献
+
+欢迎提交 Issue 和 Pull Request 来帮助改进 Crawlo！
+
+## 📄 许可证
+
+本项目采用 MIT 许可证，详情请见 [LICENSE](LICENSE) 文件。

crawlo-1.1.4.dist-info/RECORD

@@ -0,0 +1,117 @@
+crawlo/__init__.py,sha256=esOolburYDjtF43D5N9Kh6TSQW2yKcz888ilhBSinBc,825
+crawlo/__version__.py,sha256=XxXhu8-QnuD9hA8Ah0WX5rgpt_DwOQmAwcK-FtpngyQ,22
+crawlo/cli.py,sha256=CtR2Pfa7SyRxEKPaXqt-6E6K5Vq5z3rfdAI95UO4cbU,1166
+crawlo/config.py,sha256=i0Amz6wNPgv_aVcdCBRRlcwuZLSa87cH9OEmTQvB97Q,8329
+crawlo/crawler.py,sha256=v6i5tjgSOtbMoqiw1qdgKx1cY4kcVcd5l5bUTWtJNNU,36461
+crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
+crawlo/exceptions.py,sha256=sMay0wnWLfc_FXWslqxm60qz6b66LXs3EdN_w8ygE9k,1166
+crawlo/mode_manager.py,sha256=WIxrq9S3EAH0D71LH1AxvcqXomeABqoXgtUN4A--DKY,6702
+crawlo/project.py,sha256=xWN2eTAjf_Pza-wWvvV4JjScQRWxe9hXlztX81ccUMc,5182
+crawlo/stats_collector.py,sha256=NkO09CB-220qz5rxFcD_dedGfr2VPFrDo4hya0Zh8Qc,1577
+crawlo/subscriber.py,sha256=gioTIqRdEwVG-bwIiQonbk1vWWAqTh9hzVkrqZ1AfP0,5006
+crawlo/task_manager.py,sha256=19InAxS9oJ9EMj20Aw2urN-v6BeC22dkgPuW-B9-4UI,819
+crawlo/commands/__init__.py,sha256=AMYjXG7ulE8dPVmgWVo0uqXsaCYUUZYmmu2-7kFzH1M,342
+crawlo/commands/check.py,sha256=172OiAxnX5wwSlszUsyPgMZwAoIbGDTdfhtRz309ilc,22843
+crawlo/commands/genspider.py,sha256=-jGJdfXLsefX_H1ydQ2wirdu6p6wmhClzVXY_0L-1aE,5050
+crawlo/commands/list.py,sha256=yByqQeZBgvjewOKxpnOobpeJ7Hnbs-CWsoyITqZu2ZY,5781
+crawlo/commands/run.py,sha256=8Qngjsl8Q4RBdO39a__wKGsheY2PFuPit2hds_jwEbM,10524
+crawlo/commands/startproject.py,sha256=bzNgpkKzUEggY2m7Iw810mSPe8wOPFBqSCO0jZX3z_g,7138
+crawlo/commands/stats.py,sha256=6pAgkEi8MBnCer2rWmKpaTYr1jaM6HeMG9owAvEzJyY,6064
+crawlo/commands/utils.py,sha256=nohMvUU2zLvX0XzXk6KeCNxP0EvSWj9DiVLxM_7tD5o,5106
+crawlo/core/__init__.py,sha256=PnFyJdVNHBoPmV1sW0AHQXijeoSTQ8cMYrbNM1JK8kA,41
+crawlo/core/engine.py,sha256=8Dcew1XyxChW5Fz1wFEWpJlPrQb2hKDWKul8e61S-Q0,6662
+crawlo/core/enhanced_engine.py,sha256=9I9Uxdy2oAz8zDGTzEiytuKu__VDVmIN8zwZKfrD8bw,6254
+crawlo/core/processor.py,sha256=qmCqAeqhwYu-UE86evYesaGt9qpuSIfH-ZIZKcXFCZc,1140
+crawlo/core/scheduler.py,sha256=fiU-Q-lzyC3B6ih8NSWqjP1Xw_ryNVb_4dLUARtWRBE,5804
+crawlo/downloader/__init__.py,sha256=tl0mE54reR-PuJYSsXsKP2VY5uzvq4lITxZwKKjNzPs,7663
+crawlo/downloader/aiohttp_downloader.py,sha256=UKupGYPOWrscAVsjhFgKYElTa9tbEeltqV7nuWqjIeE,8005
+crawlo/downloader/cffi_downloader.py,sha256=-GVfSIhi1Ip56suSiGf8jnUE2EBF1P56vw0uxLh_T6I,10440
+crawlo/downloader/httpx_downloader.py,sha256=7jfQfvAtfk8yD_mvwUbWLhYOxMM7r1nudiU7m_Jl9wc,12037
+crawlo/extension/__init__.py,sha256=Sg588p6UhyrwFNTiD2wqGW-i3xgLX6HlLuQPKT7mayE,1526
+crawlo/extension/health_check.py,sha256=IVaaVo_0CcZtf1LoCAYXIBvs3wZ7hdmT6U4-NYWAgP0,5527
+crawlo/extension/log_interval.py,sha256=VCIeNqXcWDnxj4m6l77cjqgRzV8LfsPMb22X0Xc1Vwc,2417
+crawlo/extension/log_stats.py,sha256=Ssxz6R1YpWIj5WJvQ2cJ9F5oR7FUFdj-ITc9lV92SSU,2908
+crawlo/extension/logging_extension.py,sha256=ET6VAu1J2qNMz4NnG1G3zQLRhbsvV7l6xRIuQLE6DaE,1626
+crawlo/extension/memory_monitor.py,sha256=gg-GK5RD9XhnrN_zp3KTmPKyWDmKLMv_OTY-HxSxBNI,3664
+crawlo/extension/performance_profiler.py,sha256=NvQuuvE83dXJ-1URpN8OF9za9J1l7xhVbV22JynPQpA,4235
+crawlo/extension/request_recorder.py,sha256=RC23yzXClnVv9j2ljQvjBkUfWznfnDHsrQejKhE9y5E,4074
+crawlo/filters/__init__.py,sha256=XC_Q4ykZtSNYizYlAcehVwBBNO3lZ2zuWwafzXiuWyQ,4241
+crawlo/filters/aioredis_filter.py,sha256=WhkFZcVAym9wLSUa8WTVctYfEibjxG42umtmacO1IY0,8370
+crawlo/filters/memory_filter.py,sha256=VJO0UFRYGxmV8dj4G1subsQ-FtvPcGLbvd7IVtqXnOs,9260
+crawlo/items/__init__.py,sha256=bqekZrRlDhxfWie0UbCs656TptYseoe9QJ67I4E7Elk,386
+crawlo/items/base.py,sha256=tAYrPJgblp3ZEihDXvappdYc6pGdim6x2_9QSmMKI2o,577
+crawlo/items/fields.py,sha256=wMlakQTsEwyrlLzMt1gI4pScLQZMqd3E1xcfH4dbSqk,1801
+crawlo/items/items.py,sha256=e-3nXI9ckD64vcDxxQiAU6ufbtJMs09gbZQcYjxgwHY,3374
+crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0,549
+crawlo/middleware/default_header.py,sha256=i_Uj07JObyeZFxL7ZAZmvZsHvA1HGtkNab1sA0d-nWI,1067
+crawlo/middleware/download_delay.py,sha256=2M-TchDA7MwyTfYy0Hzh_bW9wlHlpiP-oQlys7crTj0,966
+crawlo/middleware/middleware_manager.py,sha256=j1hkWRFB5rnC5SnB7oXWE5eUNv8blS9krDIDM5fIDs8,6213
+crawlo/middleware/proxy.py,sha256=m2ZZ50En9hUtgrqSqA6hItGT74xMqccHFPhZshutIco,9811
+crawlo/middleware/request_ignore.py,sha256=QI2z4fUnJ-4xvPTZAmsL-GqR4RFHS1xq9iDr5KFrMco,997
+crawlo/middleware/response_code.py,sha256=tmef2QVl3JCiTMii6VQkASlOY2OyqmOPoOfNxIK1eF8,659
+crawlo/middleware/response_filter.py,sha256=ep8ZxDlfIefi9YqK8dPASEp5TTDRo9QEY_jMceC411s,837
+crawlo/middleware/retry.py,sha256=-7zpRURugiTTm4QYUSUlbnURD5mcT2Ji0yHvCgY1wGc,4124
+crawlo/network/__init__.py,sha256=BLPERYPo22g1BXrW--wUnlolrdFUmOPjgOB8XQQJlck,397
+crawlo/network/request.py,sha256=tPAiOVJyF3Kk-midqydTGXgv5M5tsYJRtwUUJTrUsrE,11075
+crawlo/network/response.py,sha256=cUvdjsB2cQ-qWEKHNGIkwWGgCg-EnQ81xTrjrUOVno0,9738
+crawlo/pipelines/__init__.py,sha256=lrdVDjeHLNkA4_MAwI1auk_I9xfeU1SlBWXiammb6lc,616
+crawlo/pipelines/bloom_dedup_pipeline.py,sha256=QQxGFGEoMHN4Vx2kq7G_i1o9pmuXp8clZebilOar3fk,5642
+crawlo/pipelines/console_pipeline.py,sha256=KABkR3J-rqO0Awox7lizxKR2XuHfVhWPiVRgIybwwu4,1248
+crawlo/pipelines/csv_pipeline.py,sha256=6FBT2AoU6iNU-5NfgWRq7-JpF9dK2nBokjxx-y4jIas,12174
+crawlo/pipelines/database_dedup_pipeline.py,sha256=wVBXEGArFR3uxoN7yfJSOarBmtGrJpOqowAqa7OUs98,8000
+crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
+crawlo/pipelines/memory_dedup_pipeline.py,sha256=5jeL2jEq7sioYmXlzfkx-LNSbWyChrXeWx8d15YEZOA,3839
+crawlo/pipelines/mongo_pipeline.py,sha256=k7gNqAO-g2MtIfArphC6z5ZzkKVRkBKcv-2ImziPFA0,5706
+crawlo/pipelines/mysql_pipeline.py,sha256=cwgJvRORTRea_Eep2coBaMf3G8PQVTQA1qrnIlDZApc,13480
+crawlo/pipelines/pipeline_manager.py,sha256=VrbebOYiqrobtKhp5II18w-odCICdWkmRg5WPK0Emz4,2112
+crawlo/pipelines/redis_dedup_pipeline.py,sha256=TaokJ4wP5-Cxf-ueFJdh4SX58hchT0QzZ5RBDXHDN64,6003
+crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlo/queue/pqueue.py,sha256=yzF900ko2RReVNZtfk_VV3PzSXEUALI6SHf97geuu6s,1058
+crawlo/queue/queue_manager.py,sha256=NMR0Fo8XFBg6_g7htq4D80cS6Ilo0EKt5QtyF-KxNuM,11467
+crawlo/queue/redis_priority_queue.py,sha256=boJCKqcKxRw9XCCzaHy5qmrX9DvzPiQBzOkBHI5odfE,8116
+crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
+crawlo/settings/default_settings.py,sha256=B4_61tNJvqzVvyqt9AtRV7Iq5q8G4pJOExcN0ve7S_A,11559
+crawlo/settings/setting_manager.py,sha256=SxKB1aCWh4OySM_bH9cYng9I3PAmrSP-Q8XOZEWEwbI,2899
+crawlo/spider/__init__.py,sha256=Z_rK23l5yt-DuwJPg8bcqodM_FIs4-iHLaKOimGumcE,20452
+crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
+crawlo/templates/project/__init__.py.tmpl,sha256=f3ETIXw_O6K-lkL6lXM5znMPJW1FZYGFrwDs2BnHcnQ,58
+crawlo/templates/project/items.py.tmpl,sha256=mt1Mm--H2Ouos3r7JPkYh0r33rgYJf1YOMz0OZy8TYs,297
+crawlo/templates/project/middlewares.py.tmpl,sha256=jpmj7b7Zb7d3nVyxcaVNdp4KqSts6l2cPSqn_oJUSrM,3775
+crawlo/templates/project/pipelines.py.tmpl,sha256=k_4MJnwZ6GPqVwJSEDURUlTxWybmts4vHrF0de2vgAk,2620
+crawlo/templates/project/run.py.tmpl,sha256=ktkYOslcCh9mpklg6yE5VqfATx3Frj_jNT5z5gHjQ4o,8177
+crawlo/templates/project/settings.py.tmpl,sha256=O_teIARjzRD3aMvPnuIgjaDHdjwW-3beyzfo1QH-Hr8,9580
+crawlo/templates/project/spiders/__init__.py.tmpl,sha256=j_YKsw6HQMJyqlk3WUouP3bsr-XVxshRoSNakHBc00g,106
+crawlo/templates/spider/spider.py.tmpl,sha256=a8S9j43z5gE4auMhf_OnnuVHSZN3JbMDu8Bczu8zIZY,4944
+crawlo/utils/__init__.py,sha256=BDORpyjMN7VGPKImnCDKSkprS-petgD7ezc9rMlBvb0,123
+crawlo/utils/controlled_spider_mixin.py,sha256=VjT30pNW_YIgmTD0nb7DDl2D3HvpnAYFzgSgV3fxFN0,16475
+crawlo/utils/date_tools.py,sha256=0yG0tzGb1VFgWDJJ_cow2LJfz3kj_w2MqSjmfKKESl8,6961
+crawlo/utils/db_helper.py,sha256=3ib5-agrlwf2t5S_QtLRYH75wvJDlYbRqRmDEbpH5Bo,10559
+crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
+crawlo/utils/large_scale_config.py,sha256=j7wQ5ty7pQlBRygw2vhRJ7OI19RYBZKPfYMP3WeF2WI,8154
+crawlo/utils/large_scale_helper.py,sha256=JJqcGSI6VaVe3MSL6IWjmCp8XQIu6T4U-BvBLSttr_s,12157
+crawlo/utils/log.py,sha256=A3lPyhD8kD88cV23KOL-_eT8g69xGQ5L1toDB2AO0mc,4005
+crawlo/utils/queue_helper.py,sha256=xpUUTOqlU1xz2Pb9NKAVGo3AfAO-7Xvx8Lm1q65Dgck,4743
+crawlo/utils/request.py,sha256=yoLB2rY8d78vgPjIWpdhY5SalIKjyLIvTG_UH6EMdVI,8798
+crawlo/utils/request_serializer.py,sha256=bPoSQqE2ksiMyP3WiPB3w3UqZs4f_LgkAw4Pj0qyBDo,8565
+crawlo/utils/spider_loader.py,sha256=pEDUsYOTGjszA6KgjiMlYN4GS5fP4uakkhcp3JTFFQY,2187
+crawlo/utils/system.py,sha256=HvWV1acxou0Rn0L7pNq4CnV_GWFeU0Tmjy3_nLD8M64,237
+crawlo/utils/tools.py,sha256=5Uv25Wy4m_ndZY0-n-eX-t3PxvaZ6wR3-Wvx-o7_Vrs,271
+crawlo/utils/url.py,sha256=rlgX2VlJv6JvLmCDTsbxzMSXE6R5ZL_0dLEqprsA-JU,1482
+examples/__init__.py,sha256=6i631BPnS_TR_BWUjtjB5CBO-zv9kRkwQTQvSya2wHE,123
+examples/controlled_spider_example.py,sha256=SP_k4mdKPvD1JCPs9UCm68jcy2Frg84vvXv9-14RC6I,7776
+tests/__init__.py,sha256=scL1IPVT1iucth7v8ffrjRdeW7QrC_Y7AMmFVMdTY1Y,129
+tests/test_final_validation.py,sha256=fBxf_6YcAEa_HyV_oGAXmmVHY4i6FdA4J6klCmc36hQ,4925
+tests/test_proxy_health_check.py,sha256=xo3QMP1YNw9hu7JDHZOYCUZmFFKLJpHSh4SbxXhCRPQ,1091
+tests/test_proxy_middleware_integration.py,sha256=zcl7fR9Toc-I-stSUTzKZPwcfh3kgrpjI5SbkZ6AVmE,4305
+tests/test_proxy_providers.py,sha256=XwWZCywTYguSsUxSm6fsbaoH1p9dKjqSIx9-sqKZehA,1693
+tests/test_proxy_stats.py,sha256=Til_yksrRz2yBVw-yJi5-36LhNW3vTwpXTm4BdR9PUM,507
+tests/test_proxy_strategies.py,sha256=ZkziozkvZd3KWOQnpHQ8Upd3WpyoX7gN0qFGluNm348,1809
+tests/test_redis_config.py,sha256=TqzFRojc6esGXjGhUCvSLYQDUTAgEJsty9vRVuNraMU,893
+tests/test_redis_queue.py,sha256=o6xViXxJcdx-1eMcG3vhAQEIm8h346HnZb7JXs7ZjwM,6622
+tests/test_request_serialization.py,sha256=8sVdppAsohJ5u-m1WvablCndwL-M_36YPLdGKwgeznM,2289
+tests/test_scheduler.py,sha256=-FOkTWzaMdr6yfO1Msu74hI_GgSfD7iRxO-cFA-9Iyk,7442
+crawlo-1.1.4.dist-info/METADATA,sha256=2I2NA0BR-MWoPZmRUkWrUQYMjuPiUi9mrogIYPWpASU,19781
+crawlo-1.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.1.4.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.1.4.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.1.4.dist-info/RECORD,,

examples/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 12:36
-# @Author  :   oscar
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 12:36
+# @Author  :   oscar
+# @Desc    :   None
+"""