crawlo 1.2.2__py3-none-any.whl → 1.2.4__py3-none-any.whl

crawlo/data/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Crawlo 数据模块
+包含用户代理、字典等数据文件
+"""

crawlo/data/user_agents.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+User-Agent列表
+包含各种设备和浏览器的User-Agent字符串，用于爬虫伪装
+"""
+
+# 桌面浏览器User-Agent
+DESKTOP_USER_AGENTS = [
+    # Chrome
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    
+    # Firefox
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:125.0) Gecko/20100101 Firefox/125.0",
+    "Mozilla/5.0 (X11; Linux i686; rv:125.0) Gecko/20100101 Firefox/125.0",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0",
+    
+    # Safari
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.15",
+    
+    # Edge
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.2478.80",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/123.0.2420.40",
+    
+    # Internet Explorer
+    "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
+]
+
+# 移动设备User-Agent
+MOBILE_USER_AGENTS = [
+    # iPhone
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1",
+    
+    # iPad
+    "Mozilla/5.0 (iPad; CPU OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (iPad; CPU OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1",
+    
+    # Android
+    "Mozilla/5.0 (Linux; Android 14; SM-S911B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 13; SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.118 Mobile Safari/537.36",
+    
+    # Android平板
+    "Mozilla/5.0 (Linux; Android 13; SM-X906C Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/124.0.6367.118 Safari/537.36",
+]
+
+# 爬虫/机器人User-Agent (用于测试)
+BOT_USER_AGENTS = [
+    "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+    "Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+    "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
+    "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
+]
+
+# 所有User-Agent的组合列表
+ALL_USER_AGENTS = DESKTOP_USER_AGENTS + MOBILE_USER_AGENTS
+
+# 按设备类型分类的User-Agent字典
+USER_AGENTS_BY_TYPE = {
+    "desktop": DESKTOP_USER_AGENTS,
+    "mobile": MOBILE_USER_AGENTS,
+    "bot": BOT_USER_AGENTS,
+    "all": ALL_USER_AGENTS,
+}
+
+def get_user_agents(device_type="all"):
+    """
+    获取指定类型的User-Agent列表
+    
+    Args:
+        device_type (str): 设备类型，可选值: "desktop", "mobile", "bot", "all"
+        
+    Returns:
+        list: User-Agent字符串列表
+    """
+    return USER_AGENTS_BY_TYPE.get(device_type, ALL_USER_AGENTS)
+
+def get_random_user_agent(device_type="all"):
+    """
+    获取随机User-Agent
+    
+    Args:
+        device_type (str): 设备类型，可选值: "desktop", "mobile", "bot", "all"
+        
+    Returns:
+        str: 随机User-Agent字符串
+    """
+    import random
+    user_agents = get_user_agents(device_type)
+    return random.choice(user_agents) if user_agents else ""
+
+# 导出常用的User-Agent列表
+__all__ = [
+    "DESKTOP_USER_AGENTS",
+    "MOBILE_USER_AGENTS", 
+    "BOT_USER_AGENTS",
+    "ALL_USER_AGENTS",
+    "USER_AGENTS_BY_TYPE",
+    "get_user_agents",
+    "get_random_user_agent"
+]

crawlo/downloader/__init__.py

@@ -1,266 +1,266 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-Crawlo Downloader Module
-========================
-提供多种高性能异步下载器实现。
-
-下载器类型:
-- AioHttpDownloader: 基于aiohttp的高性能下载器
-- CurlCffiDownloader: 支持浏览器指纹模拟的curl-cffi下载器  
-- HttpXDownloader: 支持HTTP/2的httpx下载器
-
-核心类:
-- DownloaderBase: 下载器基类
-- ActivateRequestManager: 活跃请求管理器
-"""
-from abc import abstractmethod, ABCMeta
-from typing import Final, Set, Optional
-from contextlib import asynccontextmanager
-
-from crawlo.utils.log import get_logger
-from crawlo.middleware.middleware_manager import MiddlewareManager
-
-
-class ActivateRequestManager:
-    """活跃请求管理器 - 跟踪和管理正在处理的请求"""
-
-    def __init__(self):
-        self._active: Final[Set] = set()
-        self._total_requests: int = 0
-        self._completed_requests: int = 0
-        self._failed_requests: int = 0
-
-    def add(self, request):
-        """添加活跃请求"""
-        self._active.add(request)
-        self._total_requests += 1
-        return request
-
-    def remove(self, request, success: bool = True):
-        """移除活跃请求并更新统计"""
-        self._active.discard(request)  # 使用discard避免KeyError
-        if success:
-            self._completed_requests += 1
-        else:
-            self._failed_requests += 1
-
-    @asynccontextmanager
-    async def __call__(self, request):
-        """上下文管理器用法"""
-        self.add(request)
-        success = False
-        try:
-            yield request
-            success = True
-        except Exception:
-            success = False
-            raise
-        finally:
-            self.remove(request, success)
-
-    def __len__(self):
-        """返回当前活跃请求数"""
-        return len(self._active)
-    
-    def get_stats(self) -> dict:
-        """获取请求统计信息"""
-        return {
-            'active_requests': len(self._active),
-            'total_requests': self._total_requests,
-            'completed_requests': self._completed_requests,
-            'failed_requests': self._failed_requests,
-            'success_rate': self._completed_requests / max(1, self._total_requests - len(self._active))
-        }
-    
-    def reset_stats(self):
-        """重置统计信息"""
-        self._total_requests = 0
-        self._completed_requests = 0
-        self._failed_requests = 0
-        # 注意：不清空 _active，因为可能有正在进行的请求
-
-
-class DownloaderMeta(ABCMeta):
-    def __subclasscheck__(self, subclass):
-        required_methods = ('fetch', 'download', 'create_instance', 'close')
-        is_subclass = all(
-            hasattr(subclass, method) and callable(getattr(subclass, method, None)) for method in required_methods
-        )
-        return is_subclass
-
-
-class DownloaderBase(metaclass=DownloaderMeta):
-    """
-    下载器基类 - 提供通用的下载器功能和接口
-    
-    所有下载器实现都应该继承此基类。
-    """
-    
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self._active = ActivateRequestManager()
-        self.middleware: Optional[MiddlewareManager] = None
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get("LOG_LEVEL"))
-        self._closed = False
-        self._stats_enabled = crawler.settings.get_bool("DOWNLOADER_STATS", True)
-
-    @classmethod
-    def create_instance(cls, *args, **kwargs):
-        """创建下载器实例"""
-        return cls(*args, **kwargs)
-
-    def open(self) -> None:
-        """初始化下载器"""
-        if self._closed:
-            raise RuntimeError(f"{self.__class__.__name__} 已关闭，无法重新打开")
-            
-        self.logger.info(
-            f"{self.crawler.spider} <下载器类：{type(self).__name__}> "
-            f"<并发数：{self.crawler.settings.get_int('CONCURRENCY')}>"
-        )
-        
-        try:
-            self.middleware = MiddlewareManager.create_instance(self.crawler)
-            self.logger.debug(f"{self.__class__.__name__} 中间件初始化完成")
-        except Exception as e:
-            self.logger.error(f"中间件初始化失败: {e}")
-            raise
-
-    async def fetch(self, request) -> Optional['Response']:
-        """获取请求响应（经过中间件处理）"""
-        if self._closed:
-            raise RuntimeError(f"{self.__class__.__name__} 已关闭")
-            
-        if not self.middleware:
-            raise RuntimeError("中间件未初始化")
-            
-        async with self._active(request):
-            try:
-                response = await self.middleware.download(request)
-                return response
-            except Exception as e:
-                self.logger.error(f"下载请求 {request.url} 失败: {e}")
-                raise
-
-    @abstractmethod
-    async def download(self, request) -> 'Response':
-        """子类必须实现的下载方法"""
-        pass
-
-    async def close(self) -> None:
-        """关闭下载器并清理资源"""
-        if not self._closed:
-            self._closed = True
-            if self._stats_enabled:
-                stats = self.get_stats()
-                self.logger.info(f"{self.__class__.__name__} 统计: {stats}")
-            self.logger.debug(f"{self.__class__.__name__} 已关闭")
-
-    def idle(self) -> bool:
-        """检查是否空闲（无活跃请求）"""
-        return len(self._active) == 0
-
-    def __len__(self) -> int:
-        """返回活跃请求数"""
-        return len(self._active)
-    
-    def get_stats(self) -> dict:
-        """获取下载器统计信息"""
-        base_stats = {
-            'downloader_class': self.__class__.__name__,
-            'is_idle': self.idle(),
-            'is_closed': self._closed
-        }
-        
-        if self._stats_enabled:
-            base_stats.update(self._active.get_stats())
-            
-        return base_stats
-    
-    def reset_stats(self):
-        """重置统计信息"""
-        if self._stats_enabled:
-            self._active.reset_stats()
-    
-    def health_check(self) -> dict:
-        """健康检查"""
-        return {
-            'status': 'healthy' if not self._closed and self.middleware else 'unhealthy',
-            'active_requests': len(self._active),
-            'middleware_ready': self.middleware is not None,
-            'closed': self._closed
-        }
-
-
-# 导入具体的下载器实现
-try:
-    from .aiohttp_downloader import AioHttpDownloader
-except ImportError:
-    AioHttpDownloader = None
-
-try:
-    from .cffi_downloader import CurlCffiDownloader
-except ImportError:
-    CurlCffiDownloader = None
-
-try:
-    from .httpx_downloader import HttpXDownloader
-except ImportError:
-    HttpXDownloader = None
-
-try:
-    from .selenium_downloader import SeleniumDownloader
-except ImportError:
-    SeleniumDownloader = None
-
-try:
-    from .playwright_downloader import PlaywrightDownloader
-except ImportError:
-    PlaywrightDownloader = None
-
-try:
-    from .hybrid_downloader import HybridDownloader
-except ImportError:
-    HybridDownloader = None
-
-# 导出所有可用的类
-__all__ = [
-    'DownloaderBase',
-    'DownloaderMeta', 
-    'ActivateRequestManager',
-]
-
-# 添加可用的下载器
-if AioHttpDownloader:
-    __all__.append('AioHttpDownloader')
-if CurlCffiDownloader:
-    __all__.append('CurlCffiDownloader')
-if HttpXDownloader:
-    __all__.append('HttpXDownloader')
-if SeleniumDownloader:
-    __all__.append('SeleniumDownloader')
-if PlaywrightDownloader:
-    __all__.append('PlaywrightDownloader')
-if HybridDownloader:
-    __all__.append('HybridDownloader')
-
-# 提供便捷的下载器映射
-DOWNLOADER_MAP = {
-    'aiohttp': AioHttpDownloader,
-    'httpx': HttpXDownloader, 
-    'curl_cffi': CurlCffiDownloader,
-    'cffi': CurlCffiDownloader,  # 别名
-    'selenium': SeleniumDownloader,
-    'playwright': PlaywrightDownloader,
-    'hybrid': HybridDownloader,
-}
-
-# 过滤掉不可用的下载器
-DOWNLOADER_MAP = {k: v for k, v in DOWNLOADER_MAP.items() if v is not None}
-
-def get_downloader_class(name: str):
-    """根据名称获取下载器类"""
-    if name in DOWNLOADER_MAP:
-        return DOWNLOADER_MAP[name]
-    raise ValueError(f"未知的下载器类型: {name}。可用类型: {list(DOWNLOADER_MAP.keys())}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Crawlo Downloader Module
+========================
+提供多种高性能异步下载器实现。
+
+下载器类型:
+- AioHttpDownloader: 基于aiohttp的高性能下载器
+- CurlCffiDownloader: 支持浏览器指纹模拟的curl-cffi下载器  
+- HttpXDownloader: 支持HTTP/2的httpx下载器
+
+核心类:
+- DownloaderBase: 下载器基类
+- ActivateRequestManager: 活跃请求管理器
+"""
+from abc import abstractmethod, ABCMeta
+from typing import Final, Set, Optional
+from contextlib import asynccontextmanager
+
+from crawlo.utils.log import get_logger
+from crawlo.middleware.middleware_manager import MiddlewareManager
+
+
+class ActivateRequestManager:
+    """活跃请求管理器 - 跟踪和管理正在处理的请求"""
+
+    def __init__(self):
+        self._active: Final[Set] = set()
+        self._total_requests: int = 0
+        self._completed_requests: int = 0
+        self._failed_requests: int = 0
+
+    def add(self, request):
+        """添加活跃请求"""
+        self._active.add(request)
+        self._total_requests += 1
+        return request
+
+    def remove(self, request, success: bool = True):
+        """移除活跃请求并更新统计"""
+        self._active.discard(request)  # 使用discard避免KeyError
+        if success:
+            self._completed_requests += 1
+        else:
+            self._failed_requests += 1
+
+    @asynccontextmanager
+    async def __call__(self, request):
+        """上下文管理器用法"""
+        self.add(request)
+        success = False
+        try:
+            yield request
+            success = True
+        except Exception:
+            success = False
+            raise
+        finally:
+            self.remove(request, success)
+
+    def __len__(self):
+        """返回当前活跃请求数"""
+        return len(self._active)
+    
+    def get_stats(self) -> dict:
+        """获取请求统计信息"""
+        return {
+            'active_requests': len(self._active),
+            'total_requests': self._total_requests,
+            'completed_requests': self._completed_requests,
+            'failed_requests': self._failed_requests,
+            'success_rate': self._completed_requests / max(1, self._total_requests - len(self._active))
+        }
+    
+    def reset_stats(self):
+        """重置统计信息"""
+        self._total_requests = 0
+        self._completed_requests = 0
+        self._failed_requests = 0
+        # 注意：不清空 _active，因为可能有正在进行的请求
+
+
+class DownloaderMeta(ABCMeta):
+    def __subclasscheck__(self, subclass):
+        required_methods = ('fetch', 'download', 'create_instance', 'close')
+        is_subclass = all(
+            hasattr(subclass, method) and callable(getattr(subclass, method, None)) for method in required_methods
+        )
+        return is_subclass
+
+
+class DownloaderBase(metaclass=DownloaderMeta):
+    """
+    下载器基类 - 提供通用的下载器功能和接口
+    
+    所有下载器实现都应该继承此基类。
+    """
+    
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self._active = ActivateRequestManager()
+        self.middleware: Optional[MiddlewareManager] = None
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get("LOG_LEVEL"))
+        self._closed = False
+        self._stats_enabled = crawler.settings.get_bool("DOWNLOADER_STATS", True)
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs):
+        """创建下载器实例"""
+        return cls(*args, **kwargs)
+
+    def open(self) -> None:
+        """初始化下载器"""
+        if self._closed:
+            raise RuntimeError(f"{self.__class__.__name__} 已关闭，无法重新打开")
+            
+        self.logger.info(
+            f"{self.crawler.spider} <下载器类：{type(self).__name__}> "
+            f"<并发数：{self.crawler.settings.get_int('CONCURRENCY')}>"
+        )
+        
+        try:
+            self.middleware = MiddlewareManager.create_instance(self.crawler)
+            self.logger.debug(f"{self.__class__.__name__} 中间件初始化完成")
+        except Exception as e:
+            self.logger.error(f"中间件初始化失败: {e}")
+            raise
+
+    async def fetch(self, request) -> Optional['Response']:
+        """获取请求响应（经过中间件处理）"""
+        if self._closed:
+            raise RuntimeError(f"{self.__class__.__name__} 已关闭")
+            
+        if not self.middleware:
+            raise RuntimeError("中间件未初始化")
+            
+        async with self._active(request):
+            try:
+                response = await self.middleware.download(request)
+                return response
+            except Exception as e:
+                self.logger.error(f"下载请求 {request.url} 失败: {e}")
+                raise
+
+    @abstractmethod
+    async def download(self, request) -> 'Response':
+        """子类必须实现的下载方法"""
+        pass
+
+    async def close(self) -> None:
+        """关闭下载器并清理资源"""
+        if not self._closed:
+            self._closed = True
+            if self._stats_enabled:
+                stats = self.get_stats()
+                self.logger.info(f"{self.__class__.__name__} 统计: {stats}")
+            self.logger.debug(f"{self.__class__.__name__} 已关闭")
+
+    def idle(self) -> bool:
+        """检查是否空闲（无活跃请求）"""
+        return len(self._active) == 0
+
+    def __len__(self) -> int:
+        """返回活跃请求数"""
+        return len(self._active)
+    
+    def get_stats(self) -> dict:
+        """获取下载器统计信息"""
+        base_stats = {
+            'downloader_class': self.__class__.__name__,
+            'is_idle': self.idle(),
+            'is_closed': self._closed
+        }
+        
+        if self._stats_enabled:
+            base_stats.update(self._active.get_stats())
+            
+        return base_stats
+    
+    def reset_stats(self):
+        """重置统计信息"""
+        if self._stats_enabled:
+            self._active.reset_stats()
+    
+    def health_check(self) -> dict:
+        """健康检查"""
+        return {
+            'status': 'healthy' if not self._closed and self.middleware else 'unhealthy',
+            'active_requests': len(self._active),
+            'middleware_ready': self.middleware is not None,
+            'closed': self._closed
+        }
+
+
+# 导入具体的下载器实现
+try:
+    from .aiohttp_downloader import AioHttpDownloader
+except ImportError:
+    AioHttpDownloader = None
+
+try:
+    from .cffi_downloader import CurlCffiDownloader
+except ImportError:
+    CurlCffiDownloader = None
+
+try:
+    from .httpx_downloader import HttpXDownloader
+except ImportError:
+    HttpXDownloader = None
+
+try:
+    from .selenium_downloader import SeleniumDownloader
+except ImportError:
+    SeleniumDownloader = None
+
+try:
+    from .playwright_downloader import PlaywrightDownloader
+except ImportError:
+    PlaywrightDownloader = None
+
+try:
+    from .hybrid_downloader import HybridDownloader
+except ImportError:
+    HybridDownloader = None
+
+# 导出所有可用的类
+__all__ = [
+    'DownloaderBase',
+    'DownloaderMeta', 
+    'ActivateRequestManager',
+]
+
+# 添加可用的下载器
+if AioHttpDownloader:
+    __all__.append('AioHttpDownloader')
+if CurlCffiDownloader:
+    __all__.append('CurlCffiDownloader')
+if HttpXDownloader:
+    __all__.append('HttpXDownloader')
+if SeleniumDownloader:
+    __all__.append('SeleniumDownloader')
+if PlaywrightDownloader:
+    __all__.append('PlaywrightDownloader')
+if HybridDownloader:
+    __all__.append('HybridDownloader')
+
+# 提供便捷的下载器映射
+DOWNLOADER_MAP = {
+    'aiohttp': AioHttpDownloader,
+    'httpx': HttpXDownloader, 
+    'curl_cffi': CurlCffiDownloader,
+    'cffi': CurlCffiDownloader,  # 别名
+    'selenium': SeleniumDownloader,
+    'playwright': PlaywrightDownloader,
+    'hybrid': HybridDownloader,
+}
+
+# 过滤掉不可用的下载器
+DOWNLOADER_MAP = {k: v for k, v in DOWNLOADER_MAP.items() if v is not None}
+
+def get_downloader_class(name: str):
+    """根据名称获取下载器类"""
+    if name in DOWNLOADER_MAP:
+        return DOWNLOADER_MAP[name]
+    raise ValueError(f"未知的下载器类型: {name}。可用类型: {list(DOWNLOADER_MAP.keys())}")