crawlo 1.1.2__py3-none-any.whl → 1.1.4__py3-none-any.whl

crawlo/extension/performance_profiler.py

@@ -0,0 +1,118 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import io
+import os
+import pstats
+import asyncio
+import cProfile
+from typing import Any, Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed
+
+
+class PerformanceProfilerExtension:
+    """
+    性能分析扩展
+    在爬虫运行期间进行性能分析，帮助优化爬虫性能
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('PERFORMANCE_PROFILER_ENABLED', False)
+        self.output_dir = self.settings.get('PERFORMANCE_PROFILER_OUTPUT_DIR', 'profiling')
+        self.interval = self.settings.get_int('PERFORMANCE_PROFILER_INTERVAL', 300)  # 默认5分钟
+        
+        self.profiler: Optional[cProfile.Profile] = None
+        self.task: Optional[asyncio.Task] = None
+        
+        # 创建输出目录
+        if self.enabled:
+            os.makedirs(self.output_dir, exist_ok=True)
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'PerformanceProfilerExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('PERFORMANCE_PROFILER_ENABLED', False):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("PerformanceProfilerExtension: PERFORMANCE_PROFILER_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self) -> None:
+        """爬虫启动时开始性能分析"""
+        if not self.enabled:
+            return
+            
+        self.profiler = cProfile.Profile()
+        self.profiler.enable()
+        
+        # 启动定期保存分析结果的任务
+        self.task = asyncio.create_task(self._periodic_save())
+        
+        self.logger.info("Performance profiler started.")
+
+    async def spider_closed(self) -> None:
+        """爬虫关闭时停止性能分析并保存结果"""
+        if not self.enabled or not self.profiler:
+            return
+            
+        # 停止定期保存任务
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+        
+        # 停止分析器并保存最终结果
+        self.profiler.disable()
+        
+        # 保存分析结果
+        await self._save_profile("final")
+        self.logger.info("Performance profiler stopped and results saved.")
+
+    async def _periodic_save(self) -> None:
+        """定期保存分析结果"""
+        counter = 1
+        while True:
+            try:
+                await asyncio.sleep(self.interval)
+                if self.profiler:
+                    # 临时禁用分析器以保存结果
+                    self.profiler.disable()
+                    await self._save_profile(f"periodic_{counter}")
+                    counter += 1
+                    # 重新启用分析器
+                    self.profiler.enable()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error(f"Error in periodic profiling save: {e}")
+
+    async def _save_profile(self, name: str) -> None:
+        """保存分析结果到文件"""
+        try:
+            # 创建内存中的字符串流
+            s = io.StringIO()
+            ps = pstats.Stats(self.profiler, stream=s)
+            
+            # 排序并打印统计信息
+            ps.sort_stats('cumulative')
+            ps.print_stats()
+            
+            # 保存到文件
+            filename = os.path.join(self.output_dir, f'profile_{name}.txt')
+            with open(filename, 'w', encoding='utf-8') as f:
+                f.write(s.getvalue())
+            
+            self.logger.info(f"Performance profile saved to {filename}")
+        except Exception as e:
+            self.logger.error(f"Error saving performance profile: {e}")

crawlo/extension/request_recorder.py

@@ -0,0 +1,108 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import os
+import json
+from typing import Any
+from datetime import datetime
+
+from crawlo import event
+from crawlo.utils.log import get_logger
+
+
+class RequestRecorderExtension:
+    """
+    请求记录扩展
+    记录所有发送的请求信息到文件，便于调试和分析
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('REQUEST_RECORDER_ENABLED', False)
+        self.output_dir = self.settings.get('REQUEST_RECORDER_OUTPUT_DIR', 'requests_log')
+        self.max_file_size = self.settings.get_int('REQUEST_RECORDER_MAX_FILE_SIZE', 10 * 1024 * 1024)  # 默认10MB
+        
+        # 创建输出目录
+        if self.enabled:
+            os.makedirs(self.output_dir, exist_ok=True)
+            
+        self.current_file = None
+        self.current_file_size = 0
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'RequestRecorderExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('REQUEST_RECORDER_ENABLED', False):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("RequestRecorderExtension: REQUEST_RECORDER_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
+            crawler.subscriber.subscribe(o.response_received, event=event.response_received)
+            crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
+        return o
+
+    async def request_scheduled(self, request: Any, spider: Any) -> None:
+        """记录调度的请求"""
+        if not self.enabled:
+            return
+            
+        try:
+            request_info = {
+                'timestamp': datetime.now().isoformat(),
+                'type': 'request',
+                'url': request.url,
+                'method': request.method,
+                'headers': dict(request.headers),
+                'meta': getattr(request, 'meta', {}),
+            }
+            
+            await self._write_record(request_info)
+        except Exception as e:
+            self.logger.error(f"Error recording request: {e}")
+
+    async def response_received(self, response: Any, spider: Any) -> None:
+        """记录接收到的响应"""
+        if not self.enabled:
+            return
+            
+        try:
+            response_info = {
+                'timestamp': datetime.now().isoformat(),
+                'type': 'response',
+                'url': response.url,
+                'status_code': response.status_code,
+                'headers': dict(response.headers),
+            }
+            
+            await self._write_record(response_info)
+        except Exception as e:
+            self.logger.error(f"Error recording response: {e}")
+
+    async def spider_closed(self, spider: Any) -> None:
+        """爬虫关闭时清理资源"""
+        if self.current_file:
+            self.current_file.close()
+            self.current_file = None
+        self.logger.info("Request recorder closed.")
+
+    async def _write_record(self, record: dict) -> None:
+        """写入记录到文件"""
+        # 检查是否需要创建新文件
+        if not self.current_file or self.current_file_size > self.max_file_size:
+            if self.current_file:
+                self.current_file.close()
+            
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            filename = os.path.join(self.output_dir, f'requests_{timestamp}.jsonl')
+            self.current_file = open(filename, 'a', encoding='utf-8')
+            self.current_file_size = 0
+        
+        # 写入记录
+        line = json.dumps(record, ensure_ascii=False) + '\n'
+        self.current_file.write(line)
+        self.current_file.flush()
+        self.current_file_size += len(line.encode('utf-8'))

crawlo/filters/aioredis_filter.py

@@ -11,8 +11,8 @@ Redis 过滤器实现
 - 高性能: 使用 Redis pipeline 优化
 - 容错设计: 网络异常自动重试
 """
+import redis.asyncio as aioredis
 from typing import Optional
-from redis import asyncio as aioredis
 from crawlo.filters import BaseFilter
 from crawlo.utils.log import get_logger
 from crawlo.utils.request import request_fingerprint
@@ -239,4 +239,4 @@ class AioRedisFilter(BaseFilter):
                 await self.redis.close()
                 self.logger.debug("Redis连接已关闭")
         except Exception as e:
-            self.logger.warning(f"Redis关闭时出错：{e}")
+            self.logger.warning(f"Redis关闭时出错：{e}")

crawlo/middleware/retry.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 from typing import List
-from asyncio.exceptions import TimeoutError
+import asyncio
 
 try:
     from anyio import EndOfStream
@@ -49,7 +49,7 @@ from crawlo.stats_collector import StatsCollector
 _retry_exceptions = [
     EndOfStream,
     ReadError,
-    TimeoutError,
+    asyncio.TimeoutError,
     ConnectError,
     ReadTimeout,
     ClientConnectorError,
@@ -122,4 +122,4 @@ class RetryMiddleware(object):
             return request
         else:
             self.logger.warning(f"{spider} {request} {reason} retry max {self.max_retry_times} times, give up.")
-            return None
+            return None

crawlo/network/request.py

@@ -16,7 +16,6 @@ from w3lib.url import safe_url_string
 from typing import Dict, Optional, Callable, Union, Any, TypeVar, List
 
 from crawlo.utils.url import escape_ajax
-from crawlo.utils.log import get_logger
 
 
 _Request = TypeVar("_Request", bound="Request")
@@ -176,7 +175,8 @@ class Request:
         self.dont_filter = dont_filter
         self._set_url(url)
 
-    def _safe_deepcopy_meta(self, meta: Dict[str, Any]) -> Dict[str, Any]:
+    @staticmethod
+    def _safe_deepcopy_meta(meta: Dict[str, Any]) -> Dict[str, Any]:
         """安全地 deepcopy meta，移除 logger 后再复制"""
         import logging
         

crawlo/network/response.py

@@ -32,14 +32,14 @@ class Response:
     """
 
     def __init__(
-        self,
-        url: str,
-        *,
-        headers: Dict[str, Any] = None,
-        body: bytes = b"",
-        method: str = 'GET',
-        request: 'Request' = None,  # 使用字符串注解避免循环导入
-        status_code: int = 200,
+            self,
+            url: str,
+            *,
+            headers: Dict[str, Any] = None,
+            body: bytes = b"",
+            method: str = 'GET',
+            request: 'Request' = None,  # 使用字符串注解避免循环导入
+            status_code: int = 200,
     ):
         # 基本属性
         self.url = url
@@ -48,15 +48,15 @@ class Response:
         self.method = method.upper()
         self.request = request
         self.status_code = status_code
-        
+
         # 编码处理
         self.encoding = self._determine_encoding()
-        
+
         # 缓存属性
         self._text_cache = None
         self._json_cache = None
         self._selector_instance = None
-        
+
         # 状态标记
         self._is_success = 200 <= status_code < 300
         self._is_redirect = 300 <= status_code < 400
@@ -68,14 +68,14 @@ class Response:
         # 1. 优先使用 request 的编码
         if self.request and self.request.encoding:
             return self.request.encoding
-        
+
         # 2. 从 Content-Type 头中检测
         content_type = self.headers.get("content-type", "") or self.headers.get("Content-Type", "")
         if content_type:
             charset_match = re.search(r"charset=([w-]+)", content_type, re.I)
             if charset_match:
                 return charset_match.group(1).lower()
-        
+
         # 3. 从 HTML meta 标签中检测(仅对HTML内容)
         if b'<html' in self.body[:1024].lower():
             # 查找 <meta charset="xxx"> 或 <meta http-equiv="Content-Type" content="...charset=xxx">
@@ -86,16 +86,17 @@ class Response:
                 charset_match = re.search(r'<meta[^>]+charset=["\']?([\w-]+)', html_text, re.I)
                 if charset_match:
                     return charset_match.group(1).lower()
-                
+
                 # <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
                 content_match = re.search(r'<meta[^>]+content=["\'][^"\'>]*charset=([\w-]+)', html_text, re.I)
                 if content_match:
                     return content_match.group(1).lower()
             except Exception:
                 pass
-        
+
         # 4. 默认使用 utf-8
         return 'utf-8'
+
     @property
     def text(self) -> str:
         """将响应体(body)以正确的编码解码为字符串，并缓存结果。"""
@@ -115,7 +116,7 @@ class Response:
         if 'gb2312' not in encodings_to_try:
             encodings_to_try.append('gb2312')
         encodings_to_try.append('latin1')  # 最后的回退选项
-        
+
         for encoding in encodings_to_try:
             if not encoding:
                 continue
@@ -124,7 +125,7 @@ class Response:
                 return self._text_cache
             except (UnicodeDecodeError, LookupError):
                 continue
-        
+
         # 所有编码都失败，使用容错解码
         try:
             self._text_cache = self.body.decode('utf-8', errors='replace')
@@ -136,37 +137,38 @@ class Response:
     def is_success(self) -> bool:
         """检查响应是否成功 (2xx)"""
         return self._is_success
-    
+
     @property
     def is_redirect(self) -> bool:
         """检查响应是否为重定向 (3xx)"""
         return self._is_redirect
-    
+
     @property
     def is_client_error(self) -> bool:
         """检查响应是否为客户端错误 (4xx)"""
         return self._is_client_error
-    
+
     @property
     def is_server_error(self) -> bool:
         """检查响应是否为服务器错误 (5xx)"""
         return self._is_server_error
-    
+
     @property
     def content_type(self) -> str:
         """获取响应的 Content-Type"""
         return self.headers.get('content-type', '') or self.headers.get('Content-Type', '')
-    
+
     @property
     def content_length(self) -> Optional[int]:
         """获取响应的 Content-Length"""
         length = self.headers.get('content-length') or self.headers.get('Content-Length')
         return int(length) if length else None
+
     def json(self, default: Any = None) -> Any:
         """将响应文本解析为 JSON 对象。"""
         if self._json_cache is not None:
             return self._json_cache
-        
+
         try:
             self._json_cache = ujson.loads(self.text)
             return self._json_cache

crawlo/pipelines/__init__.py

@@ -11,3 +11,12 @@ class BasePipeline:
     @classmethod
     def create_instance(cls, crawler):
         return cls()
+
+
+# 导出去重管道
+from .memory_dedup_pipeline import MemoryDedupPipeline
+from .redis_dedup_pipeline import RedisDedupPipeline
+from .bloom_dedup_pipeline import BloomDedupPipeline
+from .database_dedup_pipeline import DatabaseDedupPipeline
+
+__all__ = ['BasePipeline', 'MemoryDedupPipeline', 'RedisDedupPipeline', 'BloomDedupPipeline', 'DatabaseDedupPipeline']

crawlo/pipelines/bloom_dedup_pipeline.py

@@ -0,0 +1,157 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+基于 Bloom Filter 的数据项去重管道
+=============================
+提供大规模数据采集场景下的高效去重功能，使用概率性数据结构节省内存。
+
+特点:
+- 内存效率高: 相比传统集合节省大量内存
+- 高性能: 快速的插入和查找操作
+- 可扩展: 支持自定义容量和误判率
+- 适用性广: 特别适合大规模数据采集
+
+注意: Bloom Filter 有误判率，可能会错误地丢弃一些未见过的数据项。
+"""
+
+import hashlib
+try:
+    from pybloom_live import BloomFilter
+    BLOOM_FILTER_AVAILABLE = True
+except ImportError:
+    # 如果没有安装 pybloom_live，使用简单的替代方案
+    BLOOM_FILTER_AVAILABLE = False
+    
+    class BloomFilter:
+        def __init__(self, capacity, error_rate):
+            self._data = set()
+        
+        def add(self, item):
+            if item in self._data:
+                return False
+            else:
+                self._data.add(item)
+                return True
+        
+        def __contains__(self, item):
+            return item in self._data
+
+from crawlo import Item
+from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import DropItem
+
+
+class BloomDedupPipeline:
+    """基于 Bloom Filter 的数据项去重管道"""
+
+    def __init__(
+            self,
+            capacity: int = 1000000,
+            error_rate: float = 0.001,
+            log_level: str = "INFO"
+    ):
+        """
+        初始化 Bloom Filter 去重管道
+        
+        :param capacity: 预期存储的元素数量
+        :param error_rate: 误判率 (例如 0.001 表示 0.1%)
+        :param log_level: 日志级别
+        """
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        
+        # 初始化 Bloom Filter
+        try:
+            self.bloom_filter = BloomFilter(capacity=capacity, error_rate=error_rate)
+            self.logger.info(f"Bloom Filter 去重管道初始化完成 (容量: {capacity}, 误判率: {error_rate})")
+        except Exception as e:
+            self.logger.error(f"Bloom Filter 初始化失败: {e}")
+            raise RuntimeError(f"Bloom Filter 初始化失败: {e}")
+
+        self.capacity = capacity
+        self.error_rate = error_rate
+        self.dropped_count = 0
+        self.added_count = 0
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """从爬虫配置创建管道实例"""
+        settings = crawler.settings
+        
+        return cls(
+            capacity=settings.getint('BLOOM_FILTER_CAPACITY', 1000000),
+            error_rate=settings.getfloat('BLOOM_FILTER_ERROR_RATE', 0.001),
+            log_level=settings.get('LOG_LEVEL', 'INFO')
+        )
+
+    def process_item(self, item: Item, spider: Spider) -> Item:
+        """
+        处理数据项，进行去重检查
+        
+        :param item: 要处理的数据项
+        :param spider: 爬虫实例
+        :return: 处理后的数据项或抛出 DropItem 异常
+        """
+        try:
+            # 生成数据项指纹
+            fingerprint = self._generate_item_fingerprint(item)
+            
+            # 检查指纹是否已存在
+            if fingerprint in self.bloom_filter:
+                # 如果可能已存在（Bloom Filter 可能有误判），丢弃这个数据项
+                self.dropped_count += 1
+                self.logger.debug(f"可能丢弃重复数据项: {fingerprint[:20]}...")
+                raise DropItem(f"可能重复的数据项: {fingerprint}")
+            else:
+                # 添加指纹到 Bloom Filter
+                self.bloom_filter.add(fingerprint)
+                self.added_count += 1
+                self.logger.debug(f"处理新数据项: {fingerprint[:20]}...")
+                return item
+                
+        except Exception as e:
+            self.logger.error(f"处理数据项时出错: {e}")
+            # 在错误时继续处理，避免丢失数据
+            return item
+
+    def _generate_item_fingerprint(self, item: Item) -> str:
+        """
+        生成数据项指纹
+        
+        基于数据项的所有字段生成唯一指纹，用于去重判断。
+        
+        :param item: 数据项
+        :return: 指纹字符串
+        """
+        # 将数据项转换为可序列化的字典
+        try:
+            item_dict = item.to_dict()
+        except AttributeError:
+            # 兼容没有to_dict方法的Item实现
+            item_dict = dict(item)
+        
+        # 对字典进行排序以确保一致性
+        sorted_items = sorted(item_dict.items())
+        
+        # 生成指纹字符串
+        fingerprint_string = '|'.join([f"{k}={v}" for k, v in sorted_items if v is not None])
+        
+        # 使用 SHA256 生成固定长度的指纹
+        return hashlib.sha256(fingerprint_string.encode('utf-8')).hexdigest()
+
+    def close_spider(self, spider: Spider) -> None:
+        """
+        爬虫关闭时的清理工作
+        
+        :param spider: 爬虫实例
+        """
+        self.logger.info(f"爬虫 {spider.name} 关闭:")
+        self.logger.info(f"  - 处理的数据项数: {self.added_count}")
+        self.logger.info(f"  - 可能丢弃的重复数据项: {self.dropped_count}")
+        
+        if BLOOM_FILTER_AVAILABLE:
+            # 注意：Bloom Filter 无法准确统计元素数量
+            self.logger.info(f"  - Bloom Filter 容量: {self.capacity}")
+            self.logger.info(f"  - Bloom Filter 误判率: {self.error_rate}")
+        else:
+            self.logger.warning("  - 未安装 pybloom_live，使用内存集合作为替代")