crawlo 1.1.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

crawlo/downloader/__init__.py

@@ -1,35 +1,84 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+"""
+Crawlo Downloader Module
+========================
+提供多种高性能异步下载器实现。
+
+下载器类型:
+- AioHttpDownloader: 基于aiohttp的高性能下载器
+- CurlCffiDownloader: 支持浏览器指纹模拟的curl-cffi下载器  
+- HttpXDownloader: 支持HTTP/2的httpx下载器
+
+核心类:
+- DownloaderBase: 下载器基类
+- ActivateRequestManager: 活跃请求管理器
+"""
 from abc import abstractmethod, ABCMeta
-from typing_extensions import Self
 from typing import Final, Set, Optional
 from contextlib import asynccontextmanager
 
-from crawlo import Response, Request
 from crawlo.utils.log import get_logger
 from crawlo.middleware.middleware_manager import MiddlewareManager
 
 
 class ActivateRequestManager:
+    """活跃请求管理器 - 跟踪和管理正在处理的请求"""
 
     def __init__(self):
         self._active: Final[Set] = set()
+        self._total_requests: int = 0
+        self._completed_requests: int = 0
+        self._failed_requests: int = 0
 
     def add(self, request):
+        """添加活跃请求"""
         self._active.add(request)
+        self._total_requests += 1
+        return request
 
-    def remove(self, request):
-        self._active.remove(request)
+    def remove(self, request, success: bool = True):
+        """移除活跃请求并更新统计"""
+        self._active.discard(request)  # 使用discard避免KeyError
+        if success:
+            self._completed_requests += 1
+        else:
+            self._failed_requests += 1
 
     @asynccontextmanager
     async def __call__(self, request):
+        """上下文管理器用法"""
+        self.add(request)
+        success = False
         try:
-            yield self.add(request)
+            yield request
+            success = True
+        except Exception:
+            success = False
+            raise
         finally:
-            self.remove(request)
+            self.remove(request, success)
 
     def __len__(self):
+        """返回当前活跃请求数"""
         return len(self._active)
+    
+    def get_stats(self) -> dict:
+        """获取请求统计信息"""
+        return {
+            'active_requests': len(self._active),
+            'total_requests': self._total_requests,
+            'completed_requests': self._completed_requests,
+            'failed_requests': self._failed_requests,
+            'success_rate': self._completed_requests / max(1, self._total_requests - len(self._active))
+        }
+    
+    def reset_stats(self):
+        """重置统计信息"""
+        self._total_requests = 0
+        self._completed_requests = 0
+        self._failed_requests = 0
+        # 注意：不清空 _active，因为可能有正在进行的请求
 
 
 class DownloaderMeta(ABCMeta):
@@ -42,37 +91,152 @@ class DownloaderMeta(ABCMeta):
 
 
 class DownloaderBase(metaclass=DownloaderMeta):
+    """
+    下载器基类 - 提供通用的下载器功能和接口
+    
+    所有下载器实现都应该继承此基类。
+    """
+    
     def __init__(self, crawler):
         self.crawler = crawler
         self._active = ActivateRequestManager()
         self.middleware: Optional[MiddlewareManager] = None
         self.logger = get_logger(self.__class__.__name__, crawler.settings.get("LOG_LEVEL"))
+        self._closed = False
+        self._stats_enabled = crawler.settings.get_bool("DOWNLOADER_STATS", True)
 
     @classmethod
-    def create_instance(cls, *args, **kwargs) -> Self:
+    def create_instance(cls, *args, **kwargs):
+        """创建下载器实例"""
         return cls(*args, **kwargs)
 
     def open(self) -> None:
+        """初始化下载器"""
+        if self._closed:
+            raise RuntimeError(f"{self.__class__.__name__} 已关闭，无法重新打开")
+            
         self.logger.info(
-            f"{self.crawler.spider} <downloader class：{type(self).__name__}>"
-            f"<concurrency：{self.crawler.settings.get_int('CONCURRENCY')}>"
+            f"{self.crawler.spider} <下载器类：{type(self).__name__}> "
+            f"<并发数：{self.crawler.settings.get_int('CONCURRENCY')}>"
         )
-        self.middleware = MiddlewareManager.create_instance(self.crawler)
-
-    async def fetch(self, request) -> Optional[Response]:
+        
+        try:
+            self.middleware = MiddlewareManager.create_instance(self.crawler)
+            self.logger.debug(f"{self.__class__.__name__} 中间件初始化完成")
+        except Exception as e:
+            self.logger.error(f"中间件初始化失败: {e}")
+            raise
+
+    async def fetch(self, request) -> Optional['Response']:
+        """获取请求响应（经过中间件处理）"""
+        if self._closed:
+            raise RuntimeError(f"{self.__class__.__name__} 已关闭")
+            
+        if not self.middleware:
+            raise RuntimeError("中间件未初始化")
+            
         async with self._active(request):
-            response = await self.middleware.download(request)
-            return response
+            try:
+                response = await self.middleware.download(request)
+                return response
+            except Exception as e:
+                self.logger.error(f"下载请求 {request.url} 失败: {e}")
+                raise
 
     @abstractmethod
-    async def download(self, request: Request) -> Response:
+    async def download(self, request) -> 'Response':
+        """子类必须实现的下载方法"""
         pass
 
     async def close(self) -> None:
-        pass
+        """关闭下载器并清理资源"""
+        if not self._closed:
+            self._closed = True
+            if self._stats_enabled:
+                stats = self.get_stats()
+                self.logger.info(f"{self.__class__.__name__} 统计: {stats}")
+            self.logger.debug(f"{self.__class__.__name__} 已关闭")
 
     def idle(self) -> bool:
-        return len(self) == 0
+        """检查是否空闲（无活跃请求）"""
+        return len(self._active) == 0
 
     def __len__(self) -> int:
+        """返回活跃请求数"""
         return len(self._active)
+    
+    def get_stats(self) -> dict:
+        """获取下载器统计信息"""
+        base_stats = {
+            'downloader_class': self.__class__.__name__,
+            'is_idle': self.idle(),
+            'is_closed': self._closed
+        }
+        
+        if self._stats_enabled:
+            base_stats.update(self._active.get_stats())
+            
+        return base_stats
+    
+    def reset_stats(self):
+        """重置统计信息"""
+        if self._stats_enabled:
+            self._active.reset_stats()
+    
+    def health_check(self) -> dict:
+        """健康检查"""
+        return {
+            'status': 'healthy' if not self._closed and self.middleware else 'unhealthy',
+            'active_requests': len(self._active),
+            'middleware_ready': self.middleware is not None,
+            'closed': self._closed
+        }
+
+
+# 导入具体的下载器实现
+try:
+    from .aiohttp_downloader import AioHttpDownloader
+except ImportError:
+    AioHttpDownloader = None
+
+try:
+    from .cffi_downloader import CurlCffiDownloader
+except ImportError:
+    CurlCffiDownloader = None
+
+try:
+    from .httpx_downloader import HttpXDownloader
+except ImportError:
+    HttpXDownloader = None
+
+# 导出所有可用的类
+__all__ = [
+    'DownloaderBase',
+    'DownloaderMeta', 
+    'ActivateRequestManager',
+]
+
+# 添加可用的下载器
+if AioHttpDownloader:
+    __all__.append('AioHttpDownloader')
+if CurlCffiDownloader:
+    __all__.append('CurlCffiDownloader')
+if HttpXDownloader:
+    __all__.append('HttpXDownloader')
+
+# 提供便捷的下载器映射
+DOWNLOADER_MAP = {
+    'aiohttp': AioHttpDownloader,
+    'httpx': HttpXDownloader, 
+    'curl_cffi': CurlCffiDownloader,
+    'cffi': CurlCffiDownloader,  # 别名
+}
+
+# 过滤掉不可用的下载器
+DOWNLOADER_MAP = {k: v for k, v in DOWNLOADER_MAP.items() if v is not None}
+
+def get_downloader_class(name: str):
+    """根据名称获取下载器类"""
+    if name in DOWNLOADER_MAP:
+        return DOWNLOADER_MAP[name]
+    raise ValueError(f"未知的下载器类型: {name}。可用类型: {list(DOWNLOADER_MAP.keys())}")

crawlo/downloader/aiohttp_downloader.py

@@ -12,7 +12,7 @@ from aiohttp import (
     BasicAuth,
 )
 
-from crawlo import Response
+from crawlo.network.response import Response
 from crawlo.utils.log import get_logger
 from crawlo.downloader import DownloaderBase
 
@@ -74,9 +74,15 @@ class AioHttpDownloader(DownloaderBase):
         self.logger.debug("AioHttpDownloader initialized.")
 
     async def download(self, request) -> Optional[Response]:
+        """下载请求并返回响应"""
         if not self.session or self.session.closed:
             raise RuntimeError("AioHttpDownloader session is not open.")
 
+        start_time = None
+        if self.crawler.settings.get_bool("DOWNLOAD_STATS", True):
+            import time
+            start_time = time.time()
+
         try:
             # 使用通用发送逻辑（支持所有 HTTP 方法）
             async with await self._send_request(self.session, request) as resp:
@@ -86,7 +92,14 @@ class AioHttpDownloader(DownloaderBase):
                     raise OverflowError(f"Response too large: {content_length} > {self.max_download_size}")
 
                 body = await resp.read()
-                return self._structure_response(request, resp, body)
+                response = self._structure_response(request, resp, body)
+                
+                # 记录下载统计
+                if start_time:
+                    download_time = time.time() - start_time
+                    self.logger.debug(f"Downloaded {request.url} in {download_time:.3f}s, size: {len(body)} bytes")
+                
+                return response
 
         except ClientError as e:
             self.logger.error(f"Client error for {request.url}: {e}")

crawlo/downloader/cffi_downloader.py

@@ -7,7 +7,7 @@ from typing import Optional, Dict, Any
 from curl_cffi import CurlError
 from curl_cffi.requests import AsyncSession
 
-from crawlo import Response
+from crawlo.network.response import Response
 from crawlo.downloader import DownloaderBase
 
 
@@ -141,6 +141,11 @@ class CurlCffiDownloader(DownloaderBase):
         if not self.session:
             raise RuntimeError("会话未初始化")
 
+        start_time = None
+        if self.crawler.settings.get_bool("DOWNLOAD_STATS", True):
+            import time
+            start_time = time.time()
+
         kwargs = self._build_request_kwargs(request)
         method = request.method.lower()
 
@@ -173,6 +178,11 @@ class CurlCffiDownloader(DownloaderBase):
         if actual_size > self.download_warn_size:
             self.logger.warning(f"响应体较大: {actual_size} 字节，来自 {request.url}")
 
+        # 记录下载统计
+        if start_time:
+            download_time = time.time() - start_time
+            self.logger.debug(f"Downloaded {request.url} in {download_time:.3f}s, size: {actual_size} bytes")
+
         return self._structure_response(request, response, body)
 
     def _build_request_kwargs(self, request) -> Dict[str, Any]:

crawlo/downloader/httpx_downloader.py

@@ -4,7 +4,7 @@ import httpx
 from typing import Optional
 from httpx import AsyncClient, Timeout, Limits
 
-from crawlo import Response
+from crawlo.network.response import Response
 from crawlo.downloader import DownloaderBase
 from crawlo.utils.log import get_logger
 
@@ -90,9 +90,15 @@ class HttpXDownloader(DownloaderBase):
         self.logger.debug("HttpXDownloader initialized.")
 
     async def download(self, request) -> Optional[Response]:
+        """下载请求并返回响应，支持代理失败后的优雅降级"""
         if not self._client:
             raise RuntimeError("HttpXDownloader client is not available.")
 
+        start_time = None
+        if self.crawler.settings.get_bool("DOWNLOAD_STATS", True):
+            import time
+            start_time = time.time()
+
         # --- 1. 确定要使用的 client 实例 ---
         effective_client = self._client  # 默认使用共享的主 client
         temp_client = None  # 用于可能创建的临时 client
@@ -194,7 +200,12 @@ class HttpXDownloader(DownloaderBase):
             # --- 7. 读取响应体 ---
             body = await httpx_response.aread()
 
-            # --- 8. 构造并返回 Response ---
+            # --- 8. 记录下载统计 ---
+            if start_time:
+                download_time = time.time() - start_time
+                self.logger.debug(f"Downloaded {request.url} in {download_time:.3f}s, size: {len(body)} bytes")
+
+            # --- 9. 构造并返回 Response ---
             return self.structure_response(request=request, response=httpx_response, body=body)
 
         except httpx.TimeoutException as e:
@@ -219,7 +230,7 @@ class HttpXDownloader(DownloaderBase):
             raise
 
         finally:
-            # --- 9. 清理：关闭临时 client ---
+            # --- 10. 清理：关闭临时 client ---
             # 如果创建了临时 client，则关闭它
             if temp_client:
                 try:

crawlo/filters/__init__.py

@@ -1,37 +1,154 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+"""
+Crawlo Filters Module
+====================
+提供多种请求去重过滤器实现。
+
+过滤器类型:
+- MemoryFilter: 基于内存的高效去重，适合单机模式
+- AioRedisFilter: 基于Redis的分布式去重，适合分布式模式
+- MemoryFileFilter: 内存+文件持久化，适合需要重启恢复的场景
+
+核心接口:
+- BaseFilter: 所有过滤器的基类
+- requested(): 检查请求是否重复的主要方法
+"""
 from abc import ABC, abstractmethod
+from typing import Optional
 
-from crawlo import Request
 from crawlo.utils.request import request_fingerprint
 
 
 class BaseFilter(ABC):
+    """
+    请求去重过滤器基类
+    
+    提供统一的去重接口和统计功能。
+    所有过滤器实现都应该继承此类。
+    """
 
-    def __init__(self, logger, stats, debug: bool):
+    def __init__(self, logger, stats, debug: bool = False):
+        """
+        初始化过滤器
+        
+        :param logger: 日志器实例
+        :param stats: 统计信息存储
+        :param debug: 是否启用调试日志
+        """
         self.logger = logger
         self.stats = stats
         self.debug = debug
+        self._request_count = 0
+        self._duplicate_count = 0
 
     @classmethod
     def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
         return cls(*args, **kwargs)
 
-    def requested(self, request: Request):
+    def requested(self, request) -> bool:
+        """
+        检查请求是否重复（主要接口）
+        
+        :param request: 请求对象
+        :return: True 表示重复，False 表示新请求
+        """
+        self._request_count += 1
         fp = request_fingerprint(request)
+        
         if fp in self:
+            self._duplicate_count += 1
+            self.log_stats(request)
             return True
+            
         self.add_fingerprint(fp)
         return False
 
     @abstractmethod
-    def add_fingerprint(self, fp) -> None:
+    def add_fingerprint(self, fp: str) -> None:
+        """
+        添加请求指纹（子类必须实现）
+        
+        :param fp: 请求指纹字符串
+        """
+        pass
+    
+    @abstractmethod
+    def __contains__(self, item: str) -> bool:
+        """
+        检查指纹是否存在（支持 in 操作符）
+        
+        :param item: 要检查的指纹
+        :return: 是否已存在
+        """
         pass
 
-    def log_stats(self, request: Request) -> None:
+    def log_stats(self, request) -> None:
+        """
+        记录统计信息
+        
+        :param request: 重复的请求对象
+        """
         if self.debug:
             self.logger.debug(f'Filtered duplicate request: {request}')
         self.stats.inc_value(f'{self}/filtered_count')
+    
+    def get_stats(self) -> dict:
+        """
+        获取过滤器统计信息
+        
+        :return: 统计信息字典
+        """
+        return {
+            'total_requests': self._request_count,
+            'duplicate_requests': self._duplicate_count,
+            'unique_requests': self._request_count - self._duplicate_count,
+            'duplicate_rate': f"{self._duplicate_count / max(1, self._request_count) * 100:.2f}%"
+        }
+    
+    def reset_stats(self) -> None:
+        """重置统计信息"""
+        self._request_count = 0
+        self._duplicate_count = 0
+    
+    def close(self) -> None:
+        """关闭过滤器并清理资源"""
+        pass
 
     def __str__(self) -> str:
         return f'{self.__class__.__name__}'
+
+
+# 导出所有可用的过滤器
+__all__ = ['BaseFilter']
+
+# 动态导入具体实现
+try:
+    from .memory_filter import MemoryFilter, MemoryFileFilter
+    __all__.extend(['MemoryFilter', 'MemoryFileFilter'])
+except ImportError:
+    MemoryFilter = None
+    MemoryFileFilter = None
+
+try:
+    from .aioredis_filter import AioRedisFilter
+    __all__.append('AioRedisFilter')
+except ImportError:
+    AioRedisFilter = None
+
+# 提供便捷的过滤器映射
+FILTER_MAP = {
+    'memory': MemoryFilter,
+    'memory_file': MemoryFileFilter,
+    'redis': AioRedisFilter,
+    'aioredis': AioRedisFilter,  # 别名
+}
+
+# 过滤掉不可用的过滤器
+FILTER_MAP = {k: v for k, v in FILTER_MAP.items() if v is not None}
+
+def get_filter_class(name: str):
+    """根据名称获取过滤器类"""
+    if name in FILTER_MAP:
+        return FILTER_MAP[name]
+    raise ValueError(f"未知的过滤器类型: {name}。可用类型: {list(FILTER_MAP.keys())}")