crawlo 1.4.6__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/middleware/__init__.py

@@ -1,24 +1,87 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional, Union
 
 if TYPE_CHECKING:
     from crawlo import Request, Response
 
 
-class BaseMiddleware(object):
-    def process_request(self, request, spider) -> 'None | Request | Response':
-        # 请求预处理
+class BaseMiddleware:
+    """中间件基类
+    
+    定义了中间件的标准接口，所有自定义中间件都应该继承此类。
+    
+    中间件处理流程：
+    1. process_request: 请求发送前处理
+    2. process_response: 响应接收后处理
+    3. process_exception: 异常发生时处理
+    """
+    
+    def process_request(
+        self, 
+        request: 'Request', 
+        spider
+    ) -> Optional[Union['Request', 'Response']]:
+        """处理请求
+        
+        Args:
+            request: 待处理的请求对象
+            spider: 当前爬虫实例
+            
+        Returns:
+            None: 继续处理
+            Request: 替换原请求
+            Response: 跳过下载，直接返回响应
+        """
         pass
 
-    def process_response(self, request, response, spider) -> 'Request | Response':
-        # 响应预处理
-        pass
+    def process_response(
+        self, 
+        request: 'Request', 
+        response: 'Response', 
+        spider
+    ) -> Union['Request', 'Response']:
+        """处理响应
+        
+        Args:
+            request: 原始请求对象
+            response: 接收到的响应对象
+            spider: 当前爬虫实例
+            
+        Returns:
+            Request: 重新发起请求
+            Response: 返回响应（可能是修改后的）
+        """
+        return response
 
-    def process_exception(self, request, exp, spider) -> 'None | Request | Response':
-        # 异常预处理
+    def process_exception(
+        self, 
+        request: 'Request', 
+        exp: Exception, 
+        spider
+    ) -> Optional[Union['Request', 'Response']]:
+        """处理异常
+        
+        Args:
+            request: 发生异常的请求
+            exp: 捕获的异常对象
+            spider: 当前爬虫实例
+            
+        Returns:
+            None: 继续传递异常
+            Request: 重新发起请求
+            Response: 返回响应
+        """
         pass
 
     @classmethod
     def create_instance(cls, crawler):
+        """创建中间件实例
+        
+        Args:
+            crawler: Crawler实例，包含settings等配置
+            
+        Returns:
+            中间件实例
+        """
         return cls()

crawlo/middleware/default_header.py

@@ -6,7 +6,7 @@ DefaultHeaderMiddleware 中间件
 """
 
 import random
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.exceptions import NotConfiguredError
 # 导入User-Agent数据
 from crawlo.data.user_agents import get_user_agents
@@ -22,7 +22,7 @@ class DefaultHeaderMiddleware(object):
         """
         初始化中间件
         """
-        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.logger = get_logger(self.__class__.__name__)
 
         # 获取默认请求头配置
         self.headers = settings.get_dict('DEFAULT_REQUEST_HEADERS', {})

crawlo/middleware/download_delay.py

@@ -7,7 +7,7 @@ DownloadDelayMiddleware 中间件
 
 from asyncio import sleep
 from random import uniform
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.exceptions import NotConfiguredError
 
 
@@ -51,7 +51,7 @@ class DownloadDelayMiddleware(object):
             # 如果配置不完整，使用默认值
             self.floor, self.upper = 0.5, 1.5
             
-        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.logger = get_logger(self.__class__.__name__)
         self.stats = stats
 
     @classmethod

crawlo/middleware/middleware_manager.py

@@ -14,11 +14,11 @@ else:
     # 为 isinstance 检查导入实际的类
     from crawlo.network.request import Request
     from crawlo.network.response import Response
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.utils.misc import load_object
 from crawlo.middleware import BaseMiddleware
 from crawlo.project import common_call
-from crawlo.event import ignore_request, response_received
+from crawlo.event import CrawlerEvent
 from crawlo.exceptions import MiddlewareInitError, InvalidOutputError, RequestMethodError, IgnoreRequestError, \
     NotConfiguredError
 
@@ -27,7 +27,7 @@ class MiddlewareManager:
 
     def __init__(self, crawler):
         self.crawler = crawler
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
         self.middlewares: List = []
         self.methods: Dict[str, List[MethodType]] = defaultdict(list)
         middlewares = self.crawler.settings.get_list('MIDDLEWARES')
@@ -54,7 +54,7 @@ class MiddlewareManager:
             try:
                 response = await common_call(method, request, response, self.crawler.spider)
             except IgnoreRequestError as exp:
-                create_task(self.crawler.subscriber.notify(ignore_request, exp, request, self.crawler.spider))
+                create_task(self.crawler.subscriber.notify(CrawlerEvent.IGNORE_REQUEST, exp, request, self.crawler.spider))
             if isinstance(response, Request):
                 return response
             if isinstance(response, Response):
@@ -86,13 +86,13 @@ class MiddlewareManager:
         except KeyError:
             raise RequestMethodError(f"{request.method.lower()} is not supported")
         except IgnoreRequestError as exp:
-            create_task(self.crawler.subscriber.notify(ignore_request, exp, request, self.crawler.spider))
+            create_task(self.crawler.subscriber.notify(CrawlerEvent.IGNORE_REQUEST, exp, request, self.crawler.spider))
             response = await self._process_exception(request, exp)
         except Exception as exp:
             self._stats.inc_value(f'download_error/{exp.__class__.__name__}')
             response = await self._process_exception(request, exp)
         else:
-            create_task(self.crawler.subscriber.notify(response_received, response, self.crawler.spider))
+            create_task(self.crawler.subscriber.notify(CrawlerEvent.RESPONSE_RECEIVED, response, self.crawler.spider))
             self._stats.inc_value('response_received_count')
         if isinstance(response, Response):
             response = await self._process_response(request, response)

crawlo/middleware/offsite.py

@@ -7,7 +7,7 @@ OffsiteMiddleware 中间件
 import re
 from urllib.parse import urlparse
 
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.exceptions import IgnoreRequestError
 
 
@@ -18,7 +18,7 @@ class OffsiteMiddleware:
     """
 
     def __init__(self, stats, log_level, allowed_domains=None):
-        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.logger = get_logger(self.__class__.__name__)
         self.stats = stats
         self.allowed_domains = allowed_domains or []
 

crawlo/middleware/proxy.py

@@ -9,14 +9,14 @@ from urllib.parse import urlparse
 from typing import Optional, List
 
 from crawlo.network import Request, Response
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
 
 class ProxyMiddleware:
     """通用代理中间件"""
 
     def __init__(self, settings, log_level):
-        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.logger = get_logger(self.__class__.__name__)
 
         # 获取代理列表和API URL
         self.proxies: List[str] = settings.get("PROXY_LIST", [])

crawlo/middleware/request_ignore.py

@@ -4,9 +4,9 @@
 RequestIgnoreMiddleware 中间件
 用于处理和记录被忽略的请求
 """
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.exceptions import IgnoreRequestError
-from crawlo.event import ignore_request
+from crawlo.event import CrawlerEvent
 
 
 class RequestIgnoreMiddleware(object):
@@ -23,7 +23,7 @@ class RequestIgnoreMiddleware(object):
             stats: 统计信息收集器
             log_level: 日志级别
         """
-        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.logger = get_logger(self.__class__.__name__)
         self.stats = stats
 
     @classmethod
@@ -38,7 +38,7 @@ class RequestIgnoreMiddleware(object):
             RequestIgnoreMiddleware: 中间件实例
         """
         o = cls(stats=crawler.stats, log_level=crawler.settings.get('LOG_LEVEL'))
-        crawler.subscriber.subscribe(o.request_ignore, event=ignore_request)
+        crawler.subscriber.subscribe(o.request_ignore, event=CrawlerEvent.IGNORE_REQUEST)
         return o
 
     async def request_ignore(self, exc, request, _spider):

crawlo/middleware/response_code.py

@@ -4,7 +4,7 @@
 ResponseCodeMiddleware 中间件
 用于处理HTTP响应状态码，记录统计信息并支持特殊状态码处理
 """
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
 
 class ResponseCodeMiddleware(object):
@@ -27,7 +27,7 @@ class ResponseCodeMiddleware(object):
             stats: 统计信息收集器
             log_level: 日志级别
         """
-        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.logger = get_logger(self.__class__.__name__)
         self.stats = stats
 
     @classmethod

crawlo/middleware/response_filter.py

@@ -4,7 +4,7 @@
 ResponseFilterMiddleware 中间件
 用于过滤不符合要求的HTTP响应，支持自定义允许的状态码
 """
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.exceptions import IgnoreRequestError
 
 
@@ -47,7 +47,7 @@ class ResponseFilterMiddleware:
                 except (ValueError, TypeError):
                     pass  # 忽略无效的状态码
                     
-        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.logger = get_logger(self.__class__.__name__)
 
     @classmethod
     def create_instance(cls, crawler):

crawlo/middleware/retry.py

@@ -43,7 +43,7 @@ except ImportError:
     class ClientResponseError(Exception):
         pass
 
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.stats_collector import StatsCollector
 
 _retry_exceptions = [

crawlo/mode_manager.py

@@ -34,7 +34,7 @@ class ModeManager:
         """延迟获取logger实例"""
         if self._logger is None:
             try:
-                from crawlo.utils.log import get_logger
+                from crawlo.logging import get_logger
                 self._logger = get_logger(__name__)
             except Exception:
                 # 如果日志系统尚未初始化，返回None
@@ -248,6 +248,40 @@ def auto_mode(
 
 # 环境变量支持
 def from_env(default_mode: str = 'standalone') -> Dict[str, Any]:
-    """从环境变量创建配置"""
-    # 移除直接使用 os.getenv()，要求通过 settings 配置
-    raise RuntimeError("环境变量配置已移除，请在 settings 中配置相关参数")
+    """从环境变量创建配置
+    
+    支持的环境变量：
+    - CRAWLO_MODE: 运行模式 (standalone/distributed/auto)
+    - CRAWLO_REDIS_HOST: Redis主机地址
+    - CRAWLO_REDIS_PORT: Redis端口
+    - CRAWLO_REDIS_PASSWORD: Redis密码
+    - CRAWLO_REDIS_DB: Redis数据库编号
+    - CRAWLO_PROJECT_NAME: 项目名称
+    - CRAWLO_CONCURRENCY: 并发数
+    
+    Args:
+        default_mode: 默认运行模式（当未设置环境变量时使用）
+    
+    Returns:
+        配置字典
+    """
+    mode = os.getenv('CRAWLO_MODE', default_mode).lower()
+    
+    kwargs = {}
+    
+    # 分布式模式特有配置
+    if mode == 'distributed':
+        kwargs['redis_host'] = os.getenv('CRAWLO_REDIS_HOST', '127.0.0.1')
+        kwargs['redis_port'] = int(os.getenv('CRAWLO_REDIS_PORT', '6379'))
+        if password := os.getenv('CRAWLO_REDIS_PASSWORD'):
+            kwargs['redis_password'] = password
+        kwargs['redis_db'] = int(os.getenv('CRAWLO_REDIS_DB', '0'))
+    
+    # 通用配置
+    if project_name := os.getenv('CRAWLO_PROJECT_NAME'):
+        kwargs['project_name'] = project_name
+    
+    if concurrency := os.getenv('CRAWLO_CONCURRENCY'):
+        kwargs['CONCURRENCY'] = int(concurrency)
+    
+    return ModeManager().resolve_mode_settings(mode, **kwargs)

crawlo/network/request.py

@@ -12,42 +12,32 @@ HTTP Request 封装模块
 """
 import json
 from copy import deepcopy
-from urllib.parse import urlencode, urlparse, urlunparse, parse_qsl
-from w3lib.url import safe_url_string
+from enum import IntEnum
+from urllib.parse import urldefrag, urlencode, urlparse, urlunparse, parse_qsl
+from w3lib.url import safe_url_string, add_or_replace_parameter
 from typing import Dict, Optional, Callable, Union, Any, TypeVar, List
 
-from crawlo.utils.url import escape_ajax
+
 
 
 _Request = TypeVar("_Request", bound="Request")
 
 
-class RequestPriority:
-    """请求优先级常量和工具类"""
+class RequestPriority(IntEnum):
+    """
+    请求优先级枚举。
+    
+    数值越小，优先级越高。使用 IntEnum 确保可以直接当作整数使用。
+    
+    Examples:
+        >>> request = Request(url, priority=RequestPriority.HIGH)
+        >>> request.priority = RequestPriority.URGENT
+    """
     URGENT = -200      # 紧急任务
     HIGH = -100        # 高优先级  
     NORMAL = 0         # 正常优先级(默认)
     LOW = 100          # 低优先级
     BACKGROUND = 200   # 后台任务
-    
-    @classmethod
-    def get_all_priorities(cls) -> Dict[str, int]:
-        """获取所有优先级常量"""
-        return {
-            'URGENT': cls.URGENT,
-            'HIGH': cls.HIGH,
-            'NORMAL': cls.NORMAL,
-            'LOW': cls.LOW,
-            'BACKGROUND': cls.BACKGROUND
-        }
-    
-    @classmethod
-    def from_string(cls, priority_str: str) -> int:
-        """从字符串获取优先级值"""
-        priorities = cls.get_all_priorities()
-        if priority_str.upper() not in priorities:
-            raise ValueError(f"不支持的优先级: {priority_str}, 支持: {list(priorities.keys())}")
-        return priorities[priority_str.upper()]
 
 
 class Request:
@@ -87,6 +77,7 @@ class Request:
         self,
         url: str,
         callback: Optional[Callable] = None,
+        err_back: Optional[Callable] = None,
         method: Optional[str] = 'GET',
         headers: Optional[Dict[str, str]] = None,
         body: Optional[Union[bytes, str, Dict[Any, Any]]] = None,
@@ -114,6 +105,7 @@ class Request:
 
         :param url: 请求 URL（必须）
         :param callback: 成功回调函数
+        :param err_back: 错误回调函数
         :param method: HTTP 方法，默认 GET
         :param headers: 请求头
         :param body: 原始请求体（bytes/str），若为 dict 且未使用 json_body/form_data，则自动转为 JSON
@@ -134,6 +126,7 @@ class Request:
         :param encoding: 字符编码，默认 utf-8
         """
         self.callback = callback
+        self.err_back = err_back
         self.method = str(method).upper()
         self.headers = headers or {}
         self.cookies = cookies or {}
@@ -229,7 +222,7 @@ class Request:
         """安全地 deepcopy meta，移除 logger 后再复制"""
         import logging
         
-        def clean_logger_recursive(obj):
+        def clean_logger_recursive(obj: Any) -> Any:
             """递归移除 logger 对象"""
             if isinstance(obj, logging.Logger):
                 return None
@@ -251,6 +244,9 @@ class Request:
         
         # 先清理 logger，再 deepcopy
         cleaned_meta = clean_logger_recursive(meta)
+        # 确保返回字典类型
+        if not isinstance(cleaned_meta, dict):
+            return {}
         return deepcopy(cleaned_meta)
 
     def copy(self: _Request) -> _Request:
@@ -377,4 +373,36 @@ class Request:
 
     def __lt__(self, other: _Request) -> bool:
         """用于按优先级排序"""
-        return self.priority < other.priority
+        return self.priority < other.priority
+
+def escape_ajax(url: str) -> str:
+    """
+    根据Google AJAX爬取规范转换URL（处理哈希片段#!）：
+    https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+
+    规则说明：
+    1. 仅当URL包含 `#!` 时才转换（表示这是AJAX可爬取页面）
+    2. 将 `#!key=value` 转换为 `?_escaped_fragment_=key%3Dvalue`
+    3. 保留原始查询参数（如果有）
+
+    示例：
+    >>> escape_ajax("www.example.com/ajax.html#!key=value")
+    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html?k1=v1#!key=value")
+    'www.example.com/ajax.html?k1=v1&_escaped_fragment_=key%3Dvalue'
+    >>> escape_ajax("www.example.com/ajax.html#!")
+    'www.example.com/ajax.html?_escaped_fragment_='
+
+    非AJAX可爬取的URL（无#!）原样返回：
+    >>> escape_ajax("www.example.com/ajax.html#normal")
+    'www.example.com/ajax.html#normal'
+    """
+    # 分离URL的基础部分和哈希片段
+    de_frag, frag = urldefrag(url)
+
+    # 仅处理以"!"开头的哈希片段（Google规范）
+    if not frag.startswith("!"):
+        return url  # 不符合规则则原样返回
+
+    # 调用辅助函数添加 `_escaped_fragment_` 参数
+    return add_or_replace_parameter(de_frag, "_escaped_fragment_", frag[1:])