crawlo 1.0.9__tar.gz → 1.1.1__tar.gz

{crawlo-1.0.9 → crawlo-1.1.1}/LICENSE

@@ -1,23 +1,23 @@
-MIT License
-
-Modifications:
-
-Copyright (c) 2020 crawl-coder <2251018029@qq.com>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+MIT License
+
+Modifications:
+
+Copyright (c) 2020 crawl-coder <2251018029@qq.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.

{crawlo-1.0.9 → crawlo-1.1.1}/MANIFEST.in

@@ -1,17 +1,17 @@
-include README.md
-include LICENSE
-include requirements.txt  # 如果根目录有全局requirements.txt
-include VERSION           # 如果根目录有全局VERSION文件
-
-# 包内文件包含
-recursive-include crawlo/utils/js *
-recursive-include crawlo/templates *
-
-# 测试文件（如果需要在分发包中包含测试）
-recursive-include tests *
-
-# 排除项
-global-exclude __pycache__ *.py[cod] .DS_Store *.so
-global-exclude *.bak *.swp *.orig *.rej
-prune samples              # 排除示例目录
+include README.md
+include LICENSE
+include requirements.txt  # 如果根目录有全局requirements.txt
+include VERSION           # 如果根目录有全局VERSION文件
+
+# 包内文件包含
+recursive-include crawlo/utils/js *
+recursive-include crawlo/templates *
+
+# 测试文件（如果需要在分发包中包含测试）
+recursive-include tests *
+
+# 排除项
+global-exclude __pycache__ *.py[cod] .DS_Store *.so
+global-exclude *.bak *.swp *.orig *.rej
+prune samples              # 排除示例目录
 prune docs                 # 排除文档目录

crawlo-1.1.1/PKG-INFO

@@ -0,0 +1,220 @@
+Metadata-Version: 2.4
+Name: crawlo
+Version: 1.1.1
+Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
+Home-page: https://github.com/crawl-coder/Crawlo.git
+Author: crawl-coder
+Author-email: crawlo@qq.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp>=3.12.14
+Requires-Dist: aiomysql>=0.2.0
+Requires-Dist: aioredis>=2.0.1
+Requires-Dist: asyncmy>=0.2.10
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: dateparser>=1.2.2
+Requires-Dist: httpx[http2]>=0.27.0
+Requires-Dist: curl-cffi>=0.13.0
+Requires-Dist: lxml>=5.2.1
+Requires-Dist: motor>=3.7.0
+Requires-Dist: parsel>=1.9.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: pymongo>=4.11
+Requires-Dist: PyMySQL>=1.1.1
+Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: redis>=6.2.0
+Requires-Dist: requests>=2.32.4
+Requires-Dist: six>=1.17.0
+Requires-Dist: ujson>=5.9.0
+Requires-Dist: urllib3>=2.5.0
+Requires-Dist: w3lib>=2.1.2
+Requires-Dist: rich>=14.1.0
+Requires-Dist: astor>=0.8.1
+Requires-Dist: watchdog>=6.0.0
+Provides-Extra: render
+Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
+Requires-Dist: playwright; extra == "render"
+Requires-Dist: selenium>=3.141.0; extra == "render"
+Provides-Extra: all
+Requires-Dist: bitarray>=1.5.3; extra == "all"
+Requires-Dist: PyExecJS>=1.5.1; extra == "all"
+Requires-Dist: pymongo>=3.10.1; extra == "all"
+Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
+Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: selenium>=3.141.0; extra == "all"
+
+# 🕷️ Crawlo - 轻量级异步爬虫框架
+
+> 一个简洁、易用、可扩展的 Python 异步爬虫框架，灵感源自 Scrapy，但更轻量、更易上手。
+
+🚀 支持命令行操作、爬虫生成、合规检查、运行监控与统计分析，适合快速开发中小型爬虫项目。
+
+---
+
+## 📦 特性
+
+- ✅ **命令行驱动**：`crawlo startproject`, `crawlo genspider` 等
+- ✅ **自动发现爬虫**：无需手动注册，自动加载 `spiders/` 模块
+- ✅ **异步核心**：基于 `asyncio` 实现高并发抓取
+- ✅ **灵活配置**：通过 `crawlo.cfg` 和 `settings.py` 管理项目
+- ✅ **爬虫检查**：`crawlo check` 验证爬虫定义是否合规
+- ✅ **运行统计**：`crawlo stats` 查看历史运行指标（持久化存储）
+- ✅ **批量运行**：支持 `crawlo run all` 启动所有爬虫
+- ✅ **日志与调试**：结构化日志输出，便于排查问题
+
+---
+
+## 🚀 快速开始
+
+### 1. 安装 Crawlo
+
+```bash
+pip install crawlo
+```
+
+> ⚠️ 当前为开发阶段，建议使用源码安装：
+>
+> ```bash
+> git clone https://github.com/yourname/crawlo.git
+> pip install -e crawlo
+> ```
+
+### 2. 创建项目
+
+```bash
+crawlo startproject myproject
+cd myproject
+```
+
+生成项目结构：
+
+```
+myproject/
+├── crawlo.cfg
+├── myproject/
+│   ├── __init__.py
+│   ├── settings.py
+│   └── spiders/
+│       ├── __init__.py
+│       └── (你的爬虫将在这里)
+```
+
+### 3. 生成爬虫
+
+```bash
+crawlo genspider example example.com
+```
+
+生成 `spiders/example.py`：
+
+```python
+class ExampleSpider(Spider):
+    name = "example"
+    start_urls = ["https://example.com"]
+    
+    def parse(self, response):
+        # 解析逻辑
+        pass
+```
+
+### 4. 检查爬虫合规性
+
+```bash
+crawlo check
+```
+
+输出示例：
+
+```
+🔍 Checking 1 spider(s)...
+✅ example              ExampleSpider (OK)
+🎉 All spiders are compliant!
+```
+
+### 5. 运行爬虫
+
+```bash
+# 运行单个爬虫
+crawlo run example
+
+# 运行所有爬虫
+crawlo run all
+```
+
+### 6. 查看运行统计
+
+```bash
+crawlo stats
+```
+
+查看最近一次运行的请求、响应、项目数等指标：
+
+```
+📊 Recent Spider Statistics (last run):
+🕷️  example
+    downloader/request_count           1
+    item_scraped_count                 1
+    log_count/INFO                     7
+```
+
+---
+
+## 🛠️ 命令列表
+
+| 命令 | 说明 |
+|------|------|
+| `crawlo startproject <name>` | 创建新项目 |
+| `crawlo genspider <name> <domain>` | 生成爬虫模板 |
+| `crawlo list` | 列出所有已注册的爬虫 |
+| `crawlo check` | 检查爬虫定义是否合规 |
+| `crawlo run <spider_name>` | 运行指定爬虫 |
+| `crawlo run all` | 运行所有爬虫 |
+| `crawlo stats` | 查看最近运行的统计信息 |
+| `crawlo stats <spider_name>` | 查看指定爬虫的统计 |
+
+---
+
+## 📁 项目结构说明
+
+```ini
+# crawlo.cfg
+[settings]
+default = myproject.settings
+```
+
+```python
+# settings.py
+BOT_NAME = "myproject"
+LOG_LEVEL = "DEBUG"
+CONCURRENT_REQUESTS = 3
+DOWNLOAD_DELAY = 1.0
+# 其他配置...
+```
+
+---
+
+## 📊 统计持久化
+
+每次爬虫运行结束后，统计信息会自动保存到：
+
+```
+logs/stats/<spider_name>_YYYYMMDD_HHMMSS.json
+```
+
+可通过 `crawlo stats` 命令读取，支持跨进程查看。
+
+---
+
+## 🧪 开发者提示
+
+- 确保 `spiders/__init__.py` 中导入了你的爬虫类，否则无法被发现
+- 使用 `get_project_root()` 自动定位项目根目录（通过查找 `crawlo.cfg`）
+- 所有命令行工具均支持直接运行：`python -m crawlo.commands.list`
+
+---
+

crawlo-1.1.1/README.md

@@ -0,0 +1,170 @@
+# 🕷️ Crawlo - 轻量级异步爬虫框架
+
+> 一个简洁、易用、可扩展的 Python 异步爬虫框架，灵感源自 Scrapy，但更轻量、更易上手。
+
+🚀 支持命令行操作、爬虫生成、合规检查、运行监控与统计分析，适合快速开发中小型爬虫项目。
+
+---
+
+## 📦 特性
+
+- ✅ **命令行驱动**：`crawlo startproject`, `crawlo genspider` 等
+- ✅ **自动发现爬虫**：无需手动注册，自动加载 `spiders/` 模块
+- ✅ **异步核心**：基于 `asyncio` 实现高并发抓取
+- ✅ **灵活配置**：通过 `crawlo.cfg` 和 `settings.py` 管理项目
+- ✅ **爬虫检查**：`crawlo check` 验证爬虫定义是否合规
+- ✅ **运行统计**：`crawlo stats` 查看历史运行指标（持久化存储）
+- ✅ **批量运行**：支持 `crawlo run all` 启动所有爬虫
+- ✅ **日志与调试**：结构化日志输出，便于排查问题
+
+---
+
+## 🚀 快速开始
+
+### 1. 安装 Crawlo
+
+```bash
+pip install crawlo
+```
+
+> ⚠️ 当前为开发阶段，建议使用源码安装：
+>
+> ```bash
+> git clone https://github.com/yourname/crawlo.git
+> pip install -e crawlo
+> ```
+
+### 2. 创建项目
+
+```bash
+crawlo startproject myproject
+cd myproject
+```
+
+生成项目结构：
+
+```
+myproject/
+├── crawlo.cfg
+├── myproject/
+│   ├── __init__.py
+│   ├── settings.py
+│   └── spiders/
+│       ├── __init__.py
+│       └── (你的爬虫将在这里)
+```
+
+### 3. 生成爬虫
+
+```bash
+crawlo genspider example example.com
+```
+
+生成 `spiders/example.py`：
+
+```python
+class ExampleSpider(Spider):
+    name = "example"
+    start_urls = ["https://example.com"]
+    
+    def parse(self, response):
+        # 解析逻辑
+        pass
+```
+
+### 4. 检查爬虫合规性
+
+```bash
+crawlo check
+```
+
+输出示例：
+
+```
+🔍 Checking 1 spider(s)...
+✅ example              ExampleSpider (OK)
+🎉 All spiders are compliant!
+```
+
+### 5. 运行爬虫
+
+```bash
+# 运行单个爬虫
+crawlo run example
+
+# 运行所有爬虫
+crawlo run all
+```
+
+### 6. 查看运行统计
+
+```bash
+crawlo stats
+```
+
+查看最近一次运行的请求、响应、项目数等指标：
+
+```
+📊 Recent Spider Statistics (last run):
+🕷️  example
+    downloader/request_count           1
+    item_scraped_count                 1
+    log_count/INFO                     7
+```
+
+---
+
+## 🛠️ 命令列表
+
+| 命令 | 说明 |
+|------|------|
+| `crawlo startproject <name>` | 创建新项目 |
+| `crawlo genspider <name> <domain>` | 生成爬虫模板 |
+| `crawlo list` | 列出所有已注册的爬虫 |
+| `crawlo check` | 检查爬虫定义是否合规 |
+| `crawlo run <spider_name>` | 运行指定爬虫 |
+| `crawlo run all` | 运行所有爬虫 |
+| `crawlo stats` | 查看最近运行的统计信息 |
+| `crawlo stats <spider_name>` | 查看指定爬虫的统计 |
+
+---
+
+## 📁 项目结构说明
+
+```ini
+# crawlo.cfg
+[settings]
+default = myproject.settings
+```
+
+```python
+# settings.py
+BOT_NAME = "myproject"
+LOG_LEVEL = "DEBUG"
+CONCURRENT_REQUESTS = 3
+DOWNLOAD_DELAY = 1.0
+# 其他配置...
+```
+
+---
+
+## 📊 统计持久化
+
+每次爬虫运行结束后，统计信息会自动保存到：
+
+```
+logs/stats/<spider_name>_YYYYMMDD_HHMMSS.json
+```
+
+可通过 `crawlo stats` 命令读取，支持跨进程查看。
+
+---
+
+## 🧪 开发者提示
+
+- 确保 `spiders/__init__.py` 中导入了你的爬虫类，否则无法被发现
+- 使用 `get_project_root()` 自动定位项目根目录（通过查找 `crawlo.cfg`）
+- 所有命令行工具均支持直接运行：`python -m crawlo.commands.list`
+
+---
+

{crawlo-1.0.9 → crawlo-1.1.1}/crawlo/__init__.py

@@ -1,25 +1,34 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Crawlo - 一个异步爬虫框架
-"""
-from crawlo.spider import Spider
-from crawlo.items.items import Item
-from crawlo.network.request import Request
-from crawlo.network.response import Response
-from crawlo.downloader import DownloaderBase
-from crawlo.middleware import BaseMiddleware
-
-# 版本号
-from crawlo.__version__ import __version__
-
-# 可选：定义对外暴露的接口
-__all__ = [
-    'Spider',
-    'Item',
-    'Request',
-    'Response',
-    'DownloaderBase',
-    'BaseMiddleware',
-    '__version__',
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo - 一个异步爬虫框架
+"""
+from crawlo.spider import Spider
+from crawlo.items.items import Item
+from crawlo.network.request import Request
+from crawlo.network.response import Response
+from crawlo.downloader import DownloaderBase
+from crawlo.middleware import BaseMiddleware
+
+
+# 版本号：优先从元数据读取
+try:
+    from importlib.metadata import version
+    __version__ = version("crawlo")
+except Exception:
+    # 开发模式下可能未安装，回退到 __version__.py 或 dev
+    try:
+        from crawlo.__version__ import __version__
+    except ImportError:
+        __version__ = "dev"
+
+# 定义对外 API
+__all__ = [
+    'Spider',
+    'Item',
+    'Request',
+    'Response',
+    'DownloaderBase',
+    'BaseMiddleware',
+    '__version__',
 ]

crawlo-1.1.1/crawlo/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.1.1"

{crawlo-1.0.9 → crawlo-1.1.1}/crawlo/cli.py

@@ -1,41 +1,41 @@
-# crawlo/cli.py
-# !/usr/bin/python
-# -*- coding: UTF-8 -*-
-import sys
-import argparse
-from crawlo.commands import get_commands
-
-
-def main():
-    # 获取所有可用命令
-    commands = get_commands()
-
-    parser = argparse.ArgumentParser(
-        description="Crawlo: A lightweight web crawler framework.",
-        usage="crawlo <command> [options]"
-    )
-    parser.add_argument('command', help='Available commands: ' + ', '.join(commands.keys()))
-    # 注意：这里不添加具体参数，由子命令解析
-
-    # 只解析命令
-    args, unknown = parser.parse_known_args()
-
-    if args.command not in commands:
-        print(f"Unknown command: {args.command}")
-        print(f"Available commands: {', '.join(commands.keys())}")
-        sys.exit(1)
-
-    # 动态导入并执行命令
-    try:
-        module = __import__(commands[args.command], fromlist=['main'])
-        sys.exit(module.main(unknown))
-    except ImportError as e:
-        print(f"Failed to load command '{args.command}': {e}")
-        sys.exit(1)
-    except Exception as e:
-        print(f"Command '{args.command}' failed: {e}")
-        sys.exit(1)
-
-
-if __name__ == '__main__':
+# crawlo/cli.py
+# !/usr/bin/python
+# -*- coding: UTF-8 -*-
+import sys
+import argparse
+from crawlo.commands import get_commands
+
+
+def main():
+    # 获取所有可用命令
+    commands = get_commands()
+
+    parser = argparse.ArgumentParser(
+        description="Crawlo: A lightweight web crawler framework.",
+        usage="crawlo <command> [options]"
+    )
+    parser.add_argument('command', help='Available commands: ' + ', '.join(commands.keys()))
+    # 注意：这里不添加具体参数，由子命令解析
+
+    # 只解析命令
+    args, unknown = parser.parse_known_args()
+
+    if args.command not in commands:
+        print(f"Unknown command: {args.command}")
+        print(f"Available commands: {', '.join(commands.keys())}")
+        sys.exit(1)
+
+    # 动态导入并执行命令
+    try:
+        module = __import__(commands[args.command], fromlist=['main'])
+        sys.exit(module.main(unknown))
+    except ImportError as e:
+        print(f"Failed to load command '{args.command}': {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Command '{args.command}' failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
     main()

{crawlo-1.0.9 → crawlo-1.1.1}/crawlo/commands/__init__.py

@@ -1,14 +1,14 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-
-_commands = {
-    'startproject': 'crawlo.commands.startproject',
-    'genspider': 'crawlo.commands.genspider',
-    'run': 'crawlo.commands.run',
-    'check': 'crawlo.commands.check',
-    'list': 'crawlo.commands.list',
-    'stats': 'crawlo.commands.stats'
-}
-
-def get_commands():
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+
+_commands = {
+    'startproject': 'crawlo.commands.startproject',
+    'genspider': 'crawlo.commands.genspider',
+    'run': 'crawlo.commands.run',
+    'check': 'crawlo.commands.check',
+    'list': 'crawlo.commands.list',
+    'stats': 'crawlo.commands.stats'
+}
+
+def get_commands():
     return _commands