crawlo 1.1.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

crawlo/mode_manager.py

@@ -0,0 +1,201 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+运行模式管理器
+==============
+管理 Crawlo 框架的不同运行模式，提供优雅的配置方式。
+
+支持的运行模式：
+1. standalone - 单机模式（默认）
+2. distributed - 分布式模式  
+3. auto - 自动检测模式
+"""
+
+from typing import Dict, Any, Optional
+from enum import Enum
+import os
+from crawlo.utils.log import get_logger
+
+
+class RunMode(Enum):
+    """运行模式枚举"""
+    STANDALONE = "standalone"    # 单机模式
+    DISTRIBUTED = "distributed"  # 分布式模式
+    AUTO = "auto"               # 自动检测模式
+
+
+class ModeManager:
+    """运行模式管理器"""
+    
+    def __init__(self):
+        self.logger = get_logger(self.__class__.__name__)
+    
+    @staticmethod
+    def get_standalone_settings() -> Dict[str, Any]:
+        """获取单机模式配置"""
+        return {
+            'QUEUE_TYPE': 'memory',
+            'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',
+            'CONCURRENCY': 8,
+            'MAX_RUNNING_SPIDERS': 1,
+            'DOWNLOAD_DELAY': 1.0,
+            'LOG_LEVEL': 'INFO',
+        }
+    
+    @staticmethod
+    def get_distributed_settings(
+        redis_host: str = '127.0.0.1',
+        redis_port: int = 6379,
+        redis_password: Optional[str] = None,
+        project_name: str = 'crawlo'
+    ) -> Dict[str, Any]:
+        """获取分布式模式配置"""
+        # 构建 Redis URL
+        if redis_password:
+            redis_url = f'redis://:{redis_password}@{redis_host}:{redis_port}/0'
+        else:
+            redis_url = f'redis://{redis_host}:{redis_port}/0'
+        
+        return {
+            'QUEUE_TYPE': 'redis',
+            'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
+            'REDIS_HOST': redis_host,
+            'REDIS_PORT': redis_port,
+            'REDIS_PASSWORD': redis_password,
+            'REDIS_URL': redis_url,
+            'SCHEDULER_QUEUE_NAME': f'{project_name}:requests',
+            'REDIS_KEY': f'{project_name}:fingerprint',
+            'CONCURRENCY': 16,
+            'MAX_RUNNING_SPIDERS': 1,
+            'DOWNLOAD_DELAY': 1.0,
+            'LOG_LEVEL': 'INFO',
+        }
+    
+    @staticmethod
+    def get_auto_settings() -> Dict[str, Any]:
+        """获取自动检测模式配置"""
+        return {
+            'QUEUE_TYPE': 'auto',
+            'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',  # 默认内存过滤器
+            'CONCURRENCY': 12,
+            'MAX_RUNNING_SPIDERS': 1,
+            'DOWNLOAD_DELAY': 1.0,
+            'LOG_LEVEL': 'INFO',
+        }
+    
+    def resolve_mode_settings(
+        self, 
+        mode: str = 'standalone',
+        **kwargs
+    ) -> Dict[str, Any]:
+        """
+        解析运行模式并返回对应配置
+        
+        Args:
+            mode: 运行模式 ('standalone', 'distributed', 'auto')
+            **kwargs: 额外配置参数
+            
+        Returns:
+            Dict[str, Any]: 配置字典
+        """
+        mode = RunMode(mode.lower())
+        
+        if mode == RunMode.STANDALONE:
+            self.logger.info("🏠 使用单机模式 - 简单快速，适合开发和中小规模爬取")
+            settings = self.get_standalone_settings()
+            
+        elif mode == RunMode.DISTRIBUTED:
+            self.logger.info("🌐 使用分布式模式 - 支持多节点扩展，适合大规模爬取")
+            settings = self.get_distributed_settings(
+                redis_host=kwargs.get('redis_host', '127.0.0.1'),
+                redis_port=kwargs.get('redis_port', 6379),
+                redis_password=kwargs.get('redis_password'),
+                project_name=kwargs.get('project_name', 'crawlo')
+            )
+            
+        elif mode == RunMode.AUTO:
+            self.logger.info("🤖 使用自动检测模式 - 智能选择最佳运行方式")
+            settings = self.get_auto_settings()
+            
+        else:
+            raise ValueError(f"不支持的运行模式: {mode}")
+        
+        # 合并用户自定义配置
+        user_settings = {k: v for k, v in kwargs.items() 
+                        if k not in ['redis_host', 'redis_port', 'redis_password', 'project_name']}
+        settings.update(user_settings)
+        
+        return settings
+    
+    def from_environment(self) -> Dict[str, Any]:
+        """从环境变量构建配置"""
+        config = {}
+        
+        # 扫描 CRAWLO_ 前缀的环境变量
+        for key, value in os.environ.items():
+            if key.startswith('CRAWLO_'):
+                config_key = key[7:]  # 去掉 'CRAWLO_' 前缀
+                # 简单的类型转换
+                if value.lower() in ('true', 'false'):
+                    config[config_key] = value.lower() == 'true'
+                elif value.isdigit():
+                    config[config_key] = int(value)
+                else:
+                    try:
+                        config[config_key] = float(value)
+                    except ValueError:
+                        config[config_key] = value
+        
+        return config
+
+
+# 便利函数
+def standalone_mode(**kwargs) -> Dict[str, Any]:
+    """快速创建单机模式配置"""
+    return ModeManager().resolve_mode_settings('standalone', **kwargs)
+
+
+def distributed_mode(
+    redis_host: str = '127.0.0.1',
+    redis_port: int = 6379,
+    redis_password: Optional[str] = None,
+    project_name: str = 'crawlo',
+    **kwargs
+) -> Dict[str, Any]:
+    """快速创建分布式模式配置"""
+    return ModeManager().resolve_mode_settings(
+        'distributed',
+        redis_host=redis_host,
+        redis_port=redis_port,
+        redis_password=redis_password,
+        project_name=project_name,
+        **kwargs
+    )
+
+
+def auto_mode(**kwargs) -> Dict[str, Any]:
+    """快速创建自动检测模式配置"""
+    return ModeManager().resolve_mode_settings('auto', **kwargs)
+
+
+# 环境变量支持
+def from_env(default_mode: str = 'standalone') -> Dict[str, Any]:
+    """从环境变量创建配置"""
+    mode = os.getenv('CRAWLO_MODE', default_mode).lower()
+    
+    if mode == 'distributed':
+        return distributed_mode(
+            redis_host=os.getenv('REDIS_HOST', '127.0.0.1'),
+            redis_port=int(os.getenv('REDIS_PORT', 6379)),
+            redis_password=os.getenv('REDIS_PASSWORD'),
+            project_name=os.getenv('PROJECT_NAME', 'crawlo'),
+            CONCURRENCY=int(os.getenv('CONCURRENCY', 16)),
+        )
+    elif mode == 'auto':
+        return auto_mode(
+            CONCURRENCY=int(os.getenv('CONCURRENCY', 12)),
+        )
+    else:  # standalone
+        return standalone_mode(
+            CONCURRENCY=int(os.getenv('CONCURRENCY', 8)),
+        )

crawlo/network/__init__.py

@@ -1,7 +1,21 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 """
-# @Time    :    2025-02-05 14:07
-# @Author  :   oscar
-# @Desc    :   None
+Crawlo Network Module
+====================
+提供HTTP请求和响应对象的封装。
+
+主要组件:
+- Request: HTTP请求封装
+- Response: HTTP响应封装
+- RequestPriority: 请求优先级常量
 """
+
+from .request import Request, RequestPriority
+from .response import Response
+
+__all__ = [
+    'Request',
+    'RequestPriority', 
+    'Response',
+]

crawlo/network/request.py

@@ -1,5 +1,14 @@
 #!/usr/bin/python
 # -*- coding: UTF-8 -*-
+"""
+HTTP Request 封装模块
+====================
+提供功能完善的HTTP请求封装，支持:
+- JSON/表单数据自动处理
+- 优先级排序机制
+- 安全的深拷贝操作
+- 灵活的请求配置
+"""
 import json
 from copy import deepcopy
 from urllib.parse import urlencode
@@ -7,16 +16,38 @@ from w3lib.url import safe_url_string
 from typing import Dict, Optional, Callable, Union, Any, TypeVar, List
 
 from crawlo.utils.url import escape_ajax
+from crawlo.utils.log import get_logger
 
 
 _Request = TypeVar("_Request", bound="Request")
 
 
 class RequestPriority:
-    """请求优先级常量"""
-    HIGH = -100
-    NORMAL = 0
-    LOW = 100
+    """请求优先级常量和工具类"""
+    URGENT = -200      # 紧急任务
+    HIGH = -100        # 高优先级  
+    NORMAL = 0         # 正常优先级(默认)
+    LOW = 100          # 低优先级
+    BACKGROUND = 200   # 后台任务
+    
+    @classmethod
+    def get_all_priorities(cls) -> Dict[str, int]:
+        """获取所有优先级常量"""
+        return {
+            'URGENT': cls.URGENT,
+            'HIGH': cls.HIGH,
+            'NORMAL': cls.NORMAL,
+            'LOW': cls.LOW,
+            'BACKGROUND': cls.BACKGROUND
+        }
+    
+    @classmethod
+    def from_string(cls, priority_str: str) -> int:
+        """从字符串获取优先级值"""
+        priorities = cls.get_all_priorities()
+        if priority_str.upper() not in priorities:
+            raise ValueError(f"不支持的优先级: {priority_str}, 支持: {list(priorities.keys())}")
+        return priorities[priority_str.upper()]
 
 
 class Request:
@@ -99,7 +130,10 @@ class Request:
         self.headers = headers or {}
         self.cookies = cookies or {}
         self.priority = -priority  # 用于排序：值越小优先级越高
-        self._meta = deepcopy(meta) if meta is not None else {}
+        
+        # 🔧 安全处理 meta，移除 logger 后再 deepcopy
+        self._meta = self._safe_deepcopy_meta(meta) if meta is not None else {}
+        
         self.timeout = self._meta.get('download_timeout', timeout)
         self.proxy = proxy
         self.allow_redirects = allow_redirects
@@ -142,6 +176,34 @@ class Request:
         self.dont_filter = dont_filter
         self._set_url(url)
 
+    def _safe_deepcopy_meta(self, meta: Dict[str, Any]) -> Dict[str, Any]:
+        """安全地 deepcopy meta，移除 logger 后再复制"""
+        import logging
+        
+        def clean_logger_recursive(obj):
+            """递归移除 logger 对象"""
+            if isinstance(obj, logging.Logger):
+                return None
+            elif isinstance(obj, dict):
+                cleaned = {}
+                for k, v in obj.items():
+                    if not (k == 'logger' or isinstance(v, logging.Logger)):
+                        cleaned[k] = clean_logger_recursive(v)
+                return cleaned
+            elif isinstance(obj, (list, tuple)):
+                cleaned_list = []
+                for item in obj:
+                    cleaned_item = clean_logger_recursive(item)
+                    if cleaned_item is not None:
+                        cleaned_list.append(cleaned_item)
+                return type(obj)(cleaned_list)
+            else:
+                return obj
+        
+        # 先清理 logger，再 deepcopy
+        cleaned_meta = clean_logger_recursive(meta)
+        return deepcopy(cleaned_meta)
+
     def copy(self: _Request) -> _Request:
         """
         创建当前请求的副本，保留所有高层语义（json_body/form_data）。
@@ -169,22 +231,68 @@ class Request:
             encoding=self.encoding
         )
 
-    def set_meta(self, key: str, value: Any) -> None:
-        """设置 meta 中的某个键值。"""
+    def set_meta(self, key: str, value: Any) -> 'Request':
+        """设置 meta 中的某个键值，支持链式调用。"""
         self._meta[key] = value
+        return self
+    
+    def add_header(self, key: str, value: str) -> 'Request':
+        """添加请求头，支持链式调用。"""
+        self.headers[key] = value
+        return self
+    
+    def add_headers(self, headers: Dict[str, str]) -> 'Request':
+        """批量添加请求头，支持链式调用。"""
+        self.headers.update(headers)
+        return self
+    
+    def set_proxy(self, proxy: str) -> 'Request':
+        """设置代理，支持链式调用。"""
+        self.proxy = proxy
+        return self
+    
+    def set_timeout(self, timeout: float) -> 'Request':
+        """设置超时时间，支持链式调用。"""
+        self.timeout = timeout
+        return self
+    
+    def add_flag(self, flag: str) -> 'Request':
+        """添加标记，支持链式调用。"""
+        if flag not in self.flags:
+            self.flags.append(flag)
+        return self
+    
+    def remove_flag(self, flag: str) -> 'Request':
+        """移除标记，支持链式调用。"""
+        if flag in self.flags:
+            self.flags.remove(flag)
+        return self
 
     def _set_url(self, url: str) -> None:
         """安全设置 URL，确保格式正确。"""
         if not isinstance(url, str):
             raise TypeError(f"Request url 必须为字符串，当前类型: {type(url).__name__}")
+        
+        if not url.strip():
+            raise ValueError("URL 不能为空")
+        
+        # 检查危险的 URL scheme
+        dangerous_schemes = ['file://', 'ftp://', 'javascript:', 'data:']
+        if any(url.lower().startswith(scheme) for scheme in dangerous_schemes):
+            raise ValueError(f"URL scheme 不安全: {url[:20]}...")
 
         s = safe_url_string(url, self.encoding)
         escaped_url = escape_ajax(s)
+        
+        if not escaped_url.startswith(('http://', 'https://')):
+            raise ValueError(f"URL 缺少 HTTP(S) scheme: {escaped_url[:50]}...")
+        
+        # 检查 URL 长度
+        if len(escaped_url) > 8192:  # 大多数服务器支持的最大 URL 长度
+            raise ValueError(f"URL 过长 (超过 8192 字符): {len(escaped_url)} 字符")
+        
         self._url = escaped_url
 
-        if not self._url.startswith(('http://', 'https://')):
-            raise ValueError(f"URL 缺少 scheme: {self._url}")
-
     @property
     def url(self) -> str:
         return self._url

crawlo/network/response.py

@@ -1,76 +1,179 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+"""
+HTTP Response 封装模块
+=====================
+提供功能丰富的HTTP响应封装，支持:
+- 智能编码检测和解码
+- XPath/CSS 选择器
+- JSON 解析和缓存
+- 正则表达式支持
+- Cookie 处理
+"""
 import re
 import ujson
 from http.cookies import SimpleCookie
 from parsel import Selector, SelectorList
-from typing import Dict, Any, List, Optional
+from typing import Dict, Any, List, Optional, Union
 from urllib.parse import urljoin as _urljoin
 
-from crawlo import Request
 from crawlo.exceptions import DecodeError
 
 
 class Response:
     """
     HTTP响应的封装，提供数据解析的便捷方法。
+    
+    功能特性:
+    - 智能编码检测和缓存
+    - 懒加载 Selector 实例
+    - JSON 解析和缓存
+    - 多类型数据提取
     """
 
     def __init__(
         self,
         url: str,
         *,
-        headers: Dict[str, Any],
+        headers: Dict[str, Any] = None,
         body: bytes = b"",
         method: str = 'GET',
-        request: Request = None,
+        request: 'Request' = None,  # 使用字符串注解避免循环导入
         status_code: int = 200,
     ):
+        # 基本属性
         self.url = url
-        self.headers = headers
+        self.headers = headers or {}
         self.body = body
-        self.method = method
+        self.method = method.upper()
         self.request = request
         self.status_code = status_code
-        self.encoding = self.request.encoding if self.request else None
+        
+        # 编码处理
+        self.encoding = self._determine_encoding()
+        
+        # 缓存属性
         self._text_cache = None
         self._json_cache = None
-        self._selector_instance = None  # 修改变量名，避免与 @property 冲突
+        self._selector_instance = None
+        
+        # 状态标记
+        self._is_success = 200 <= status_code < 300
+        self._is_redirect = 300 <= status_code < 400
+        self._is_client_error = 400 <= status_code < 500
+        self._is_server_error = status_code >= 500
 
+    def _determine_encoding(self) -> Optional[str]:
+        """智能检测响应编码"""
+        # 1. 优先使用 request 的编码
+        if self.request and self.request.encoding:
+            return self.request.encoding
+        
+        # 2. 从 Content-Type 头中检测
+        content_type = self.headers.get("content-type", "") or self.headers.get("Content-Type", "")
+        if content_type:
+            charset_match = re.search(r"charset=([w-]+)", content_type, re.I)
+            if charset_match:
+                return charset_match.group(1).lower()
+        
+        # 3. 从 HTML meta 标签中检测(仅对HTML内容)
+        if b'<html' in self.body[:1024].lower():
+            # 查找 <meta charset="xxx"> 或 <meta http-equiv="Content-Type" content="...charset=xxx">
+            html_start = self.body[:4096]  # 只检查前4KB
+            try:
+                html_text = html_start.decode('ascii', errors='ignore')
+                # <meta charset="utf-8">
+                charset_match = re.search(r'<meta[^>]+charset=["\']?([\w-]+)', html_text, re.I)
+                if charset_match:
+                    return charset_match.group(1).lower()
+                
+                # <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+                content_match = re.search(r'<meta[^>]+content=["\'][^"\'>]*charset=([\w-]+)', html_text, re.I)
+                if content_match:
+                    return content_match.group(1).lower()
+            except Exception:
+                pass
+        
+        # 4. 默认使用 utf-8
+        return 'utf-8'
     @property
     def text(self) -> str:
         """将响应体(body)以正确的编码解码为字符串，并缓存结果。"""
         if self._text_cache is not None:
             return self._text_cache
 
-        encoding = self.encoding
-        try:
-            # 优先使用 request 提供的编码
-            if encoding:
-                self._text_cache = self.body.decode(encoding)
-                return self._text_cache
+        if not self.body:
+            self._text_cache = ""
+            return self._text_cache
 
-            # 从 Content-Type 中提取编码
-            content_type = self.headers.get("Content-Type", "")
-            charset_match = re.search(r"charset=([\w-]+)", content_type, re.I)
-            if charset_match:
-                encoding = charset_match.group(1)
+        # 尝试多种编码
+        encodings_to_try = [self.encoding]
+        if self.encoding != 'utf-8':
+            encodings_to_try.append('utf-8')
+        if 'gbk' not in encodings_to_try:
+            encodings_to_try.append('gbk')
+        if 'gb2312' not in encodings_to_try:
+            encodings_to_try.append('gb2312')
+        encodings_to_try.append('latin1')  # 最后的回退选项
+        
+        for encoding in encodings_to_try:
+            if not encoding:
+                continue
+            try:
                 self._text_cache = self.body.decode(encoding)
                 return self._text_cache
-
-            # 默认尝试 UTF-8
-            self._text_cache = self.body.decode("utf-8")
+            except (UnicodeDecodeError, LookupError):
+                continue
+        
+        # 所有编码都失败，使用容错解码
+        try:
+            self._text_cache = self.body.decode('utf-8', errors='replace')
             return self._text_cache
-
-        except UnicodeDecodeError as e:
+        except Exception as e:
             raise DecodeError(f"Failed to decode response from {self.url}: {e}")
 
-    def json(self) -> Any:
+    @property
+    def is_success(self) -> bool:
+        """检查响应是否成功 (2xx)"""
+        return self._is_success
+    
+    @property
+    def is_redirect(self) -> bool:
+        """检查响应是否为重定向 (3xx)"""
+        return self._is_redirect
+    
+    @property
+    def is_client_error(self) -> bool:
+        """检查响应是否为客户端错误 (4xx)"""
+        return self._is_client_error
+    
+    @property
+    def is_server_error(self) -> bool:
+        """检查响应是否为服务器错误 (5xx)"""
+        return self._is_server_error
+    
+    @property
+    def content_type(self) -> str:
+        """获取响应的 Content-Type"""
+        return self.headers.get('content-type', '') or self.headers.get('Content-Type', '')
+    
+    @property
+    def content_length(self) -> Optional[int]:
+        """获取响应的 Content-Length"""
+        length = self.headers.get('content-length') or self.headers.get('Content-Length')
+        return int(length) if length else None
+    def json(self, default: Any = None) -> Any:
         """将响应文本解析为 JSON 对象。"""
-        if self._json_cache:
+        if self._json_cache is not None:
+            return self._json_cache
+        
+        try:
+            self._json_cache = ujson.loads(self.text)
             return self._json_cache
-        self._json_cache = ujson.loads(self.text)
-        return self._json_cache
+        except (ujson.JSONDecodeError, ValueError) as e:
+            if default is not None:
+                return default
+            raise DecodeError(f"Failed to parse JSON from {self.url}: {e}")
 
     def urljoin(self, url: str) -> str:
         """拼接 URL，自动处理相对路径。"""

crawlo/pipelines/__init__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-from crawlo.items.items import Item
+from crawlo.items import Item
 
 
 class BasePipeline: