crawlo 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl

crawlo/extension/request_recorder.py

@@ -1,108 +1,108 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import os
-import json
-from typing import Any
-from datetime import datetime
-
-from crawlo import event
-from crawlo.utils.log import get_logger
-
-
-class RequestRecorderExtension:
-    """
-    请求记录扩展
-    记录所有发送的请求信息到文件，便于调试和分析
-    """
-
-    def __init__(self, crawler: Any):
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        
-        # 获取配置参数
-        self.enabled = self.settings.get_bool('REQUEST_RECORDER_ENABLED', False)
-        self.output_dir = self.settings.get('REQUEST_RECORDER_OUTPUT_DIR', 'requests_log')
-        self.max_file_size = self.settings.get_int('REQUEST_RECORDER_MAX_FILE_SIZE', 10 * 1024 * 1024)  # 默认10MB
-        
-        # 创建输出目录
-        if self.enabled:
-            os.makedirs(self.output_dir, exist_ok=True)
-            
-        self.current_file = None
-        self.current_file_size = 0
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'RequestRecorderExtension':
-        # 只有当配置启用时才创建实例
-        if not crawler.settings.get_bool('REQUEST_RECORDER_ENABLED', False):
-            from crawlo.exceptions import NotConfigured
-            raise NotConfigured("RequestRecorderExtension: REQUEST_RECORDER_ENABLED is False")
-        
-        o = cls(crawler)
-        if o.enabled:
-            crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
-            crawler.subscriber.subscribe(o.response_received, event=event.response_received)
-            crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
-        return o
-
-    async def request_scheduled(self, request: Any, spider: Any) -> None:
-        """记录调度的请求"""
-        if not self.enabled:
-            return
-            
-        try:
-            request_info = {
-                'timestamp': datetime.now().isoformat(),
-                'type': 'request',
-                'url': request.url,
-                'method': request.method,
-                'headers': dict(request.headers),
-                'meta': getattr(request, 'meta', {}),
-            }
-            
-            await self._write_record(request_info)
-        except Exception as e:
-            self.logger.error(f"Error recording request: {e}")
-
-    async def response_received(self, response: Any, spider: Any) -> None:
-        """记录接收到的响应"""
-        if not self.enabled:
-            return
-            
-        try:
-            response_info = {
-                'timestamp': datetime.now().isoformat(),
-                'type': 'response',
-                'url': response.url,
-                'status_code': response.status_code,
-                'headers': dict(response.headers),
-            }
-            
-            await self._write_record(response_info)
-        except Exception as e:
-            self.logger.error(f"Error recording response: {e}")
-
-    async def spider_closed(self, spider: Any) -> None:
-        """爬虫关闭时清理资源"""
-        if self.current_file:
-            self.current_file.close()
-            self.current_file = None
-        self.logger.info("Request recorder closed.")
-
-    async def _write_record(self, record: dict) -> None:
-        """写入记录到文件"""
-        # 检查是否需要创建新文件
-        if not self.current_file or self.current_file_size > self.max_file_size:
-            if self.current_file:
-                self.current_file.close()
-            
-            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-            filename = os.path.join(self.output_dir, f'requests_{timestamp}.jsonl')
-            self.current_file = open(filename, 'a', encoding='utf-8')
-            self.current_file_size = 0
-        
-        # 写入记录
-        line = json.dumps(record, ensure_ascii=False) + '\n'
-        self.current_file.write(line)
-        self.current_file.flush()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import os
+import json
+from typing import Any
+from datetime import datetime
+
+from crawlo import event
+from crawlo.utils.log import get_logger
+
+
+class RequestRecorderExtension:
+    """
+    请求记录扩展
+    记录所有发送的请求信息到文件，便于调试和分析
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('REQUEST_RECORDER_ENABLED', False)
+        self.output_dir = self.settings.get('REQUEST_RECORDER_OUTPUT_DIR', 'requests_log')
+        self.max_file_size = self.settings.get_int('REQUEST_RECORDER_MAX_FILE_SIZE', 10 * 1024 * 1024)  # 默认10MB
+        
+        # 创建输出目录
+        if self.enabled:
+            os.makedirs(self.output_dir, exist_ok=True)
+            
+        self.current_file = None
+        self.current_file_size = 0
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'RequestRecorderExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('REQUEST_RECORDER_ENABLED', False):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("RequestRecorderExtension: REQUEST_RECORDER_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
+            crawler.subscriber.subscribe(o.response_received, event=event.response_received)
+            crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
+        return o
+
+    async def request_scheduled(self, request: Any, spider: Any) -> None:
+        """记录调度的请求"""
+        if not self.enabled:
+            return
+            
+        try:
+            request_info = {
+                'timestamp': datetime.now().isoformat(),
+                'type': 'request',
+                'url': request.url,
+                'method': request.method,
+                'headers': dict(request.headers),
+                'meta': getattr(request, 'meta', {}),
+            }
+            
+            await self._write_record(request_info)
+        except Exception as e:
+            self.logger.error(f"Error recording request: {e}")
+
+    async def response_received(self, response: Any, spider: Any) -> None:
+        """记录接收到的响应"""
+        if not self.enabled:
+            return
+            
+        try:
+            response_info = {
+                'timestamp': datetime.now().isoformat(),
+                'type': 'response',
+                'url': response.url,
+                'status_code': response.status_code,
+                'headers': dict(response.headers),
+            }
+            
+            await self._write_record(response_info)
+        except Exception as e:
+            self.logger.error(f"Error recording response: {e}")
+
+    async def spider_closed(self, spider: Any) -> None:
+        """爬虫关闭时清理资源"""
+        if self.current_file:
+            self.current_file.close()
+            self.current_file = None
+        self.logger.info("Request recorder closed.")
+
+    async def _write_record(self, record: dict) -> None:
+        """写入记录到文件"""
+        # 检查是否需要创建新文件
+        if not self.current_file or self.current_file_size > self.max_file_size:
+            if self.current_file:
+                self.current_file.close()
+            
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            filename = os.path.join(self.output_dir, f'requests_{timestamp}.jsonl')
+            self.current_file = open(filename, 'a', encoding='utf-8')
+            self.current_file_size = 0
+        
+        # 写入记录
+        line = json.dumps(record, ensure_ascii=False) + '\n'
+        self.current_file.write(line)
+        self.current_file.flush()
         self.current_file_size += len(line.encode('utf-8'))

crawlo/filters/__init__.py

@@ -1,154 +1,154 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-Crawlo Filters Module
-====================
-提供多种请求去重过滤器实现。
-
-过滤器类型:
-- MemoryFilter: 基于内存的高效去重，适合单机模式
-- AioRedisFilter: 基于Redis的分布式去重，适合分布式模式
-- MemoryFileFilter: 内存+文件持久化，适合需要重启恢复的场景
-
-核心接口:
-- BaseFilter: 所有过滤器的基类
-- requested(): 检查请求是否重复的主要方法
-"""
-from abc import ABC, abstractmethod
-from typing import Optional
-
-from crawlo.utils.request import request_fingerprint
-
-
-class BaseFilter(ABC):
-    """
-    请求去重过滤器基类
-    
-    提供统一的去重接口和统计功能。
-    所有过滤器实现都应该继承此类。
-    """
-
-    def __init__(self, logger, stats, debug: bool = False):
-        """
-        初始化过滤器
-        
-        :param logger: 日志器实例
-        :param stats: 统计信息存储
-        :param debug: 是否启用调试日志
-        """
-        self.logger = logger
-        self.stats = stats
-        self.debug = debug
-        self._request_count = 0
-        self._duplicate_count = 0
-
-    @classmethod
-    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
-        return cls(*args, **kwargs)
-
-    def requested(self, request) -> bool:
-        """
-        检查请求是否重复（主要接口）
-        
-        :param request: 请求对象
-        :return: True 表示重复，False 表示新请求
-        """
-        self._request_count += 1
-        fp = request_fingerprint(request)
-        
-        if fp in self:
-            self._duplicate_count += 1
-            self.log_stats(request)
-            return True
-            
-        self.add_fingerprint(fp)
-        return False
-
-    @abstractmethod
-    def add_fingerprint(self, fp: str) -> None:
-        """
-        添加请求指纹（子类必须实现）
-        
-        :param fp: 请求指纹字符串
-        """
-        pass
-    
-    @abstractmethod
-    def __contains__(self, item: str) -> bool:
-        """
-        检查指纹是否存在（支持 in 操作符）
-        
-        :param item: 要检查的指纹
-        :return: 是否已存在
-        """
-        pass
-
-    def log_stats(self, request) -> None:
-        """
-        记录统计信息
-        
-        :param request: 重复的请求对象
-        """
-        if self.debug:
-            self.logger.debug(f'Filtered duplicate request: {request}')
-        self.stats.inc_value(f'{self}/filtered_count')
-    
-    def get_stats(self) -> dict:
-        """
-        获取过滤器统计信息
-        
-        :return: 统计信息字典
-        """
-        return {
-            'total_requests': self._request_count,
-            'duplicate_requests': self._duplicate_count,
-            'unique_requests': self._request_count - self._duplicate_count,
-            'duplicate_rate': f"{self._duplicate_count / max(1, self._request_count) * 100:.2f}%"
-        }
-    
-    def reset_stats(self) -> None:
-        """重置统计信息"""
-        self._request_count = 0
-        self._duplicate_count = 0
-    
-    def close(self) -> None:
-        """关闭过滤器并清理资源"""
-        pass
-
-    def __str__(self) -> str:
-        return f'{self.__class__.__name__}'
-
-
-# 导出所有可用的过滤器
-__all__ = ['BaseFilter']
-
-# 动态导入具体实现
-try:
-    from .memory_filter import MemoryFilter, MemoryFileFilter
-    __all__.extend(['MemoryFilter', 'MemoryFileFilter'])
-except ImportError:
-    MemoryFilter = None
-    MemoryFileFilter = None
-
-try:
-    from .aioredis_filter import AioRedisFilter
-    __all__.append('AioRedisFilter')
-except ImportError:
-    AioRedisFilter = None
-
-# 提供便捷的过滤器映射
-FILTER_MAP = {
-    'memory': MemoryFilter,
-    'memory_file': MemoryFileFilter,
-    'redis': AioRedisFilter,
-    'aioredis': AioRedisFilter,  # 别名
-}
-
-# 过滤掉不可用的过滤器
-FILTER_MAP = {k: v for k, v in FILTER_MAP.items() if v is not None}
-
-def get_filter_class(name: str):
-    """根据名称获取过滤器类"""
-    if name in FILTER_MAP:
-        return FILTER_MAP[name]
-    raise ValueError(f"未知的过滤器类型: {name}。可用类型: {list(FILTER_MAP.keys())}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Crawlo Filters Module
+====================
+提供多种请求去重过滤器实现。
+
+过滤器类型:
+- MemoryFilter: 基于内存的高效去重，适合单机模式
+- AioRedisFilter: 基于Redis的分布式去重，适合分布式模式
+- MemoryFileFilter: 内存+文件持久化，适合需要重启恢复的场景
+
+核心接口:
+- BaseFilter: 所有过滤器的基类
+- requested(): 检查请求是否重复的主要方法
+"""
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from crawlo.utils.request import request_fingerprint
+
+
+class BaseFilter(ABC):
+    """
+    请求去重过滤器基类
+    
+    提供统一的去重接口和统计功能。
+    所有过滤器实现都应该继承此类。
+    """
+
+    def __init__(self, logger, stats, debug: bool = False):
+        """
+        初始化过滤器
+        
+        :param logger: 日志器实例
+        :param stats: 统计信息存储
+        :param debug: 是否启用调试日志
+        """
+        self.logger = logger
+        self.stats = stats
+        self.debug = debug
+        self._request_count = 0
+        self._duplicate_count = 0
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
+        return cls(*args, **kwargs)
+
+    def requested(self, request) -> bool:
+        """
+        检查请求是否重复（主要接口）
+        
+        :param request: 请求对象
+        :return: True 表示重复，False 表示新请求
+        """
+        self._request_count += 1
+        fp = request_fingerprint(request)
+        
+        if fp in self:
+            self._duplicate_count += 1
+            self.log_stats(request)
+            return True
+            
+        self.add_fingerprint(fp)
+        return False
+
+    @abstractmethod
+    def add_fingerprint(self, fp: str) -> None:
+        """
+        添加请求指纹（子类必须实现）
+        
+        :param fp: 请求指纹字符串
+        """
+        pass
+    
+    @abstractmethod
+    def __contains__(self, item: str) -> bool:
+        """
+        检查指纹是否存在（支持 in 操作符）
+        
+        :param item: 要检查的指纹
+        :return: 是否已存在
+        """
+        pass
+
+    def log_stats(self, request) -> None:
+        """
+        记录统计信息
+        
+        :param request: 重复的请求对象
+        """
+        if self.debug:
+            self.logger.debug(f'Filtered duplicate request: {request}')
+        self.stats.inc_value(f'{self}/filtered_count')
+    
+    def get_stats(self) -> dict:
+        """
+        获取过滤器统计信息
+        
+        :return: 统计信息字典
+        """
+        return {
+            'total_requests': self._request_count,
+            'duplicate_requests': self._duplicate_count,
+            'unique_requests': self._request_count - self._duplicate_count,
+            'duplicate_rate': f"{self._duplicate_count / max(1, self._request_count) * 100:.2f}%"
+        }
+    
+    def reset_stats(self) -> None:
+        """重置统计信息"""
+        self._request_count = 0
+        self._duplicate_count = 0
+    
+    def close(self) -> None:
+        """关闭过滤器并清理资源"""
+        pass
+
+    def __str__(self) -> str:
+        return f'{self.__class__.__name__}'
+
+
+# 导出所有可用的过滤器
+__all__ = ['BaseFilter']
+
+# 动态导入具体实现
+try:
+    from .memory_filter import MemoryFilter, MemoryFileFilter
+    __all__.extend(['MemoryFilter', 'MemoryFileFilter'])
+except ImportError:
+    MemoryFilter = None
+    MemoryFileFilter = None
+
+try:
+    from .aioredis_filter import AioRedisFilter
+    __all__.append('AioRedisFilter')
+except ImportError:
+    AioRedisFilter = None
+
+# 提供便捷的过滤器映射
+FILTER_MAP = {
+    'memory': MemoryFilter,
+    'memory_file': MemoryFileFilter,
+    'redis': AioRedisFilter,
+    'aioredis': AioRedisFilter,  # 别名
+}
+
+# 过滤掉不可用的过滤器
+FILTER_MAP = {k: v for k, v in FILTER_MAP.items() if v is not None}
+
+def get_filter_class(name: str):
+    """根据名称获取过滤器类"""
+    if name in FILTER_MAP:
+        return FILTER_MAP[name]
+    raise ValueError(f"未知的过滤器类型: {name}。可用类型: {list(FILTER_MAP.keys())}")