crawlo 1.4.2__tar.gz → 1.4.3__tar.gz

{crawlo-1.4.2 → crawlo-1.4.3}/LICENSE

@@ -1,23 +1,23 @@
-MIT License
-
-Modifications:
-
-Copyright (c) 2020 crawl-coder <2251018029@qq.com>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+MIT License
+
+Modifications:
+
+Copyright (c) 2020 crawl-coder <2251018029@qq.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.

{crawlo-1.4.2 → crawlo-1.4.3}/MANIFEST.in

@@ -1,17 +1,17 @@
-include README.md
-include LICENSE
-include requirements.txt  # 如果根目录有全局requirements.txt
-include VERSION           # 如果根目录有全局VERSION文件
-
-# 包内文件包含
-recursive-include crawlo/utils/js *
-recursive-include crawlo/templates *
-
-# 测试文件（如果需要在分发包中包含测试）
-recursive-include tests *
-
-# 排除项
-global-exclude __pycache__ *.py[cod] .DS_Store *.so
-global-exclude *.bak *.swp *.orig *.rej
-prune samples              # 排除示例目录
+include README.md
+include LICENSE
+include requirements.txt  # 如果根目录有全局requirements.txt
+include VERSION           # 如果根目录有全局VERSION文件
+
+# 包内文件包含
+recursive-include crawlo/utils/js *
+recursive-include crawlo/templates *
+
+# 测试文件（如果需要在分发包中包含测试）
+recursive-include tests *
+
+# 排除项
+global-exclude __pycache__ *.py[cod] .DS_Store *.so
+global-exclude *.bak *.swp *.orig *.rej
+prune samples              # 排除示例目录
 prune docs                 # 排除文档目录

crawlo-1.4.3/PKG-INFO

@@ -0,0 +1,190 @@
+Metadata-Version: 2.4
+Name: crawlo
+Version: 1.4.3
+Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
+Home-page: https://github.com/crawl-coder/Crawlo.git
+Author: crawl-coder
+Author-email: crawlo@qq.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp>=3.12.14
+Requires-Dist: aiomysql>=0.2.0
+Requires-Dist: aioredis>=2.0.1
+Requires-Dist: asyncmy>=0.2.10
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: dateparser>=1.2.2
+Requires-Dist: httpx[http2]>=0.27.0
+Requires-Dist: curl-cffi>=0.13.0
+Requires-Dist: lxml>=5.2.1
+Requires-Dist: motor>=3.7.0
+Requires-Dist: parsel>=1.9.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: pymongo>=4.11
+Requires-Dist: PyMySQL>=1.1.1
+Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: redis>=6.2.0
+Requires-Dist: requests>=2.32.4
+Requires-Dist: six>=1.17.0
+Requires-Dist: ujson>=5.9.0
+Requires-Dist: urllib3>=2.5.0
+Requires-Dist: w3lib>=2.1.2
+Requires-Dist: rich>=14.1.0
+Requires-Dist: astor>=0.8.1
+Requires-Dist: watchdog>=6.0.0
+Provides-Extra: render
+Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
+Requires-Dist: playwright; extra == "render"
+Requires-Dist: selenium>=3.141.0; extra == "render"
+Provides-Extra: all
+Requires-Dist: bitarray>=1.5.3; extra == "all"
+Requires-Dist: PyExecJS>=1.5.1; extra == "all"
+Requires-Dist: pymongo>=3.10.1; extra == "all"
+Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
+Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: selenium>=3.141.0; extra == "all"
+
+# Crawlo 爬虫框架
+
+Crawlo 是一个高性能、可扩展的 Python 爬虫框架，支持单机和分布式部署。
+
+## 特性
+
+- 高性能异步爬取
+- 支持多种下载器 (aiohttp, httpx, curl-cffi)
+- 内置数据清洗和验证
+- 分布式爬取支持
+- 灵活的中间件系统
+- 强大的配置管理系统
+- 详细的日志记录和监控
+- Windows 和 Linux 兼容
+
+## 安装
+
+```bash
+pip install crawlo
+```
+
+或者从源码安装：
+
+```bash
+git clone https://github.com/your-username/crawlo.git
+cd crawlo
+pip install -r requirements.txt
+pip install .
+```
+
+## 快速开始
+
+```python
+from crawlo import Spider
+
+class MySpider(Spider):
+    name = 'example'
+    
+    def parse(self, response):
+        # 解析逻辑
+        pass
+
+# 运行爬虫
+# crawlo run example
+```
+
+## 日志系统
+
+Crawlo 拥有一个功能强大的日志系统，支持多种配置选项：
+
+### 基本配置
+
+```python
+from crawlo.logging import configure_logging, get_logger
+
+# 配置日志系统
+configure_logging(
+    LOG_LEVEL='INFO',
+    LOG_FILE='logs/app.log',
+    LOG_MAX_BYTES=10*1024*1024,  # 10MB
+    LOG_BACKUP_COUNT=5
+)
+
+# 获取logger
+logger = get_logger('my_module')
+logger.info('这是一条日志消息')
+```
+
+### 高级配置
+
+```python
+# 分别配置控制台和文件日志级别
+configure_logging(
+    LOG_LEVEL='INFO',
+    LOG_CONSOLE_LEVEL='WARNING',  # 控制台只显示WARNING及以上级别
+    LOG_FILE_LEVEL='DEBUG',       # 文件记录DEBUG及以上级别
+    LOG_FILE='logs/app.log',
+    LOG_INCLUDE_THREAD_ID=True,   # 包含线程ID
+    LOG_INCLUDE_PROCESS_ID=True   # 包含进程ID
+)
+
+# 模块特定日志级别
+configure_logging(
+    LOG_LEVEL='WARNING',
+    LOG_LEVELS={
+        'my_module.debug': 'DEBUG',
+        'my_module.info': 'INFO'
+    }
+)
+```
+
+### 性能监控
+
+```python
+from crawlo.logging import get_monitor
+
+# 启用日志性能监控
+monitor = get_monitor()
+monitor.enable_monitoring()
+
+# 获取性能报告
+report = monitor.get_performance_report()
+print(report)
+```
+
+### 日志采样
+
+```python
+from crawlo.logging import get_sampler
+
+# 设置采样率（只记录30%的日志）
+sampler = get_sampler()
+sampler.set_sample_rate('my_module', 0.3)
+
+# 设置速率限制（每秒最多100条日志）
+sampler.set_rate_limit('my_module', 100)
+```
+
+## Windows 兼容性说明
+
+在 Windows 系统上使用日志轮转功能时，可能会遇到文件锁定问题。为了解决这个问题，建议安装 `concurrent-log-handler` 库：
+
+```bash
+pip install concurrent-log-handler
+```
+
+Crawlo 框架会自动检测并使用这个库来提供更好的 Windows 兼容性。
+
+如果未安装 `concurrent-log-handler`，在 Windows 上运行时可能会出现以下错误：
+```
+PermissionError: [WinError 32] 另一个程序正在使用此文件，进程无法访问。
+```
+
+## 文档
+
+请查看 [文档](https://your-docs-url.com) 获取更多信息。
+
+## 许可证
+
+MIT

crawlo-1.4.3/README.md

@@ -0,0 +1,140 @@
+# Crawlo 爬虫框架
+
+Crawlo 是一个高性能、可扩展的 Python 爬虫框架，支持单机和分布式部署。
+
+## 特性
+
+- 高性能异步爬取
+- 支持多种下载器 (aiohttp, httpx, curl-cffi)
+- 内置数据清洗和验证
+- 分布式爬取支持
+- 灵活的中间件系统
+- 强大的配置管理系统
+- 详细的日志记录和监控
+- Windows 和 Linux 兼容
+
+## 安装
+
+```bash
+pip install crawlo
+```
+
+或者从源码安装：
+
+```bash
+git clone https://github.com/your-username/crawlo.git
+cd crawlo
+pip install -r requirements.txt
+pip install .
+```
+
+## 快速开始
+
+```python
+from crawlo import Spider
+
+class MySpider(Spider):
+    name = 'example'
+    
+    def parse(self, response):
+        # 解析逻辑
+        pass
+
+# 运行爬虫
+# crawlo run example
+```
+
+## 日志系统
+
+Crawlo 拥有一个功能强大的日志系统，支持多种配置选项：
+
+### 基本配置
+
+```python
+from crawlo.logging import configure_logging, get_logger
+
+# 配置日志系统
+configure_logging(
+    LOG_LEVEL='INFO',
+    LOG_FILE='logs/app.log',
+    LOG_MAX_BYTES=10*1024*1024,  # 10MB
+    LOG_BACKUP_COUNT=5
+)
+
+# 获取logger
+logger = get_logger('my_module')
+logger.info('这是一条日志消息')
+```
+
+### 高级配置
+
+```python
+# 分别配置控制台和文件日志级别
+configure_logging(
+    LOG_LEVEL='INFO',
+    LOG_CONSOLE_LEVEL='WARNING',  # 控制台只显示WARNING及以上级别
+    LOG_FILE_LEVEL='DEBUG',       # 文件记录DEBUG及以上级别
+    LOG_FILE='logs/app.log',
+    LOG_INCLUDE_THREAD_ID=True,   # 包含线程ID
+    LOG_INCLUDE_PROCESS_ID=True   # 包含进程ID
+)
+
+# 模块特定日志级别
+configure_logging(
+    LOG_LEVEL='WARNING',
+    LOG_LEVELS={
+        'my_module.debug': 'DEBUG',
+        'my_module.info': 'INFO'
+    }
+)
+```
+
+### 性能监控
+
+```python
+from crawlo.logging import get_monitor
+
+# 启用日志性能监控
+monitor = get_monitor()
+monitor.enable_monitoring()
+
+# 获取性能报告
+report = monitor.get_performance_report()
+print(report)
+```
+
+### 日志采样
+
+```python
+from crawlo.logging import get_sampler
+
+# 设置采样率（只记录30%的日志）
+sampler = get_sampler()
+sampler.set_sample_rate('my_module', 0.3)
+
+# 设置速率限制（每秒最多100条日志）
+sampler.set_rate_limit('my_module', 100)
+```
+
+## Windows 兼容性说明
+
+在 Windows 系统上使用日志轮转功能时，可能会遇到文件锁定问题。为了解决这个问题，建议安装 `concurrent-log-handler` 库：
+
+```bash
+pip install concurrent-log-handler
+```
+
+Crawlo 框架会自动检测并使用这个库来提供更好的 Windows 兼容性。
+
+如果未安装 `concurrent-log-handler`，在 Windows 上运行时可能会出现以下错误：
+```
+PermissionError: [WinError 32] 另一个程序正在使用此文件，进程无法访问。
+```
+
+## 文档
+
+请查看 [文档](https://your-docs-url.com) 获取更多信息。
+
+## 许可证
+
+MIT

{crawlo-1.4.2 → crawlo-1.4.3}/crawlo/__init__.py

@@ -1,93 +1,93 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Crawlo - 一个异步爬虫框架
-"""
-from typing import TYPE_CHECKING
-
-from crawlo.spider import Spider
-from crawlo.items import Item, Field
-from crawlo.network.request import Request
-from crawlo.network.response import Response
-from crawlo.downloader import DownloaderBase
-from crawlo.middleware import BaseMiddleware
-from crawlo.utils import (
-    TimeUtils,
-    parse_time,
-    format_time,
-    time_diff,
-    to_timestamp,
-    to_datetime,
-    now,
-    to_timezone,
-    to_utc,
-    to_local,
-    from_timestamp_with_tz
-)
-from crawlo import tools
-
-# 框架核心模块 - 使用TYPE_CHECKING避免循环导入
-if TYPE_CHECKING:
-    from crawlo.initialization import get_framework_initializer, initialize_framework
-
-# 为了向后兼容，从tools中导入cleaners相关的功能
-import crawlo.tools as cleaners
-
-
-# 延迟导入的辅助函数
-def get_framework_initializer():
-    """延迟导入get_framework_initializer以避免循环依赖"""
-    from crawlo.initialization import get_framework_initializer as _get_framework_initializer
-    return _get_framework_initializer()
-
-
-def initialize_framework(custom_settings=None):
-    """延迟导入initialize_framework以避免循环依赖"""
-    from crawlo.initialization import initialize_framework as _initialize_framework
-    return _initialize_framework(custom_settings)
-
-
-# 向后兼容的别名
-def get_bootstrap_manager():
-    """向后兼容的别名"""
-    return get_framework_initializer()
-
-
-# 版本号：优先从元数据读取
-try:
-    from importlib.metadata import version
-
-    __version__ = version("crawlo")
-except Exception:
-    # 开发模式下可能未安装，回退到 __version__.py 或 dev
-    try:
-        from crawlo.__version__ import __version__
-    except ImportError:
-        __version__ = "dev"
-
-# 定义对外 API
-__all__ = [
-    'Spider',
-    'Item',
-    'Field',
-    'Request',
-    'Response',
-    'DownloaderBase',
-    'BaseMiddleware',
-    'TimeUtils',
-    'parse_time',
-    'format_time',
-    'time_diff',
-    'to_timestamp',
-    'to_datetime',
-    'now',
-    'to_timezone',
-    'to_utc',
-    'to_local',
-    'from_timestamp_with_tz',
-    'cleaners',
-    'tools',
-    'get_framework_initializer',
-    'get_bootstrap_manager',
-    '__version__',
-]
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo - 一个异步爬虫框架
+"""
+from typing import TYPE_CHECKING
+
+from crawlo.spider import Spider
+from crawlo.items import Item, Field
+from crawlo.network.request import Request
+from crawlo.network.response import Response
+from crawlo.downloader import DownloaderBase
+from crawlo.middleware import BaseMiddleware
+from crawlo.utils import (
+    TimeUtils,
+    parse_time,
+    format_time,
+    time_diff,
+    to_timestamp,
+    to_datetime,
+    now,
+    to_timezone,
+    to_utc,
+    to_local,
+    from_timestamp_with_tz
+)
+from crawlo import tools
+
+# 框架核心模块 - 使用TYPE_CHECKING避免循环导入
+if TYPE_CHECKING:
+    from crawlo.initialization import get_framework_initializer, initialize_framework
+
+# 为了向后兼容，从tools中导入cleaners相关的功能
+import crawlo.tools as cleaners
+
+
+# 延迟导入的辅助函数
+def get_framework_initializer():
+    """延迟导入get_framework_initializer以避免循环依赖"""
+    from crawlo.initialization import get_framework_initializer as _get_framework_initializer
+    return _get_framework_initializer()
+
+
+def initialize_framework(custom_settings=None):
+    """延迟导入initialize_framework以避免循环依赖"""
+    from crawlo.initialization import initialize_framework as _initialize_framework
+    return _initialize_framework(custom_settings)
+
+
+# 向后兼容的别名
+def get_bootstrap_manager():
+    """向后兼容的别名"""
+    return get_framework_initializer()
+
+
+# 版本号：优先从元数据读取
+try:
+    from importlib.metadata import version
+
+    __version__ = version("crawlo")
+except Exception:
+    # 开发模式下可能未安装，回退到 __version__.py 或 dev
+    try:
+        from crawlo.__version__ import __version__
+    except ImportError:
+        __version__ = "dev"
+
+# 定义对外 API
+__all__ = [
+    'Spider',
+    'Item',
+    'Field',
+    'Request',
+    'Response',
+    'DownloaderBase',
+    'BaseMiddleware',
+    'TimeUtils',
+    'parse_time',
+    'format_time',
+    'time_diff',
+    'to_timestamp',
+    'to_datetime',
+    'now',
+    'to_timezone',
+    'to_utc',
+    'to_local',
+    'from_timestamp_with_tz',
+    'cleaners',
+    'tools',
+    'get_framework_initializer',
+    'get_bootstrap_manager',
+    '__version__',
+]

crawlo-1.4.3/crawlo/__version__.py

@@ -0,0 +1 @@
+__version__ = '1.4.3'