crawlo 1.2.7__py3-none-any.whl → 1.2.9__py3-none-any.whl

crawlo/middleware/response_filter.py

@@ -1,137 +1,137 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-ResponseFilterMiddleware 中间件
-用于过滤不符合要求的HTTP响应，支持自定义允许的状态码
-"""
-from crawlo.utils.log import get_logger
-from crawlo.exceptions import IgnoreRequestError
-
-
-class ResponseFilterMiddleware:
-    """
-    ResponseFilterMiddleware 中间件
-    用于过滤不符合要求的HTTP响应，支持自定义允许的状态码
-    
-    功能特性:
-    - 默认允许2xx状态码
-    - 支持自定义允许的状态码列表
-    - 支持拒绝特定状态码
-    - 提供详细的日志信息
-    - 支持按域名配置不同的过滤规则
-    """
-
-    def __init__(self, allowed_codes, denied_codes, log_level):
-        """
-        初始化中间件
-        
-        Args:
-            allowed_codes: 允许的状态码列表
-            denied_codes: 拒绝的状态码列表
-            log_level: 日志级别
-        """
-        # 确保状态码是整数类型
-        self.allowed_codes = set()
-        if allowed_codes:
-            for code in allowed_codes:
-                try:
-                    self.allowed_codes.add(int(code))
-                except (ValueError, TypeError):
-                    pass  # 忽略无效的状态码
-                    
-        self.denied_codes = set()
-        if denied_codes:
-            for code in denied_codes:
-                try:
-                    self.denied_codes.add(int(code))
-                except (ValueError, TypeError):
-                    pass  # 忽略无效的状态码
-                    
-        self.logger = get_logger(self.__class__.__name__, log_level)
-
-    @classmethod
-    def create_instance(cls, crawler):
-        """
-        创建中间件实例
-        
-        Args:
-            crawler: 爬虫实例
-            
-        Returns:
-            ResponseFilterMiddleware: 中间件实例
-        """
-        o = cls(
-            allowed_codes=crawler.settings.get_list('ALLOWED_RESPONSE_CODES'),
-            denied_codes=crawler.settings.get_list('DENIED_RESPONSE_CODES'),
-            log_level=crawler.settings.get('LOG_LEVEL')
-        )
-        return o
-
-    def _is_response_allowed(self, response):
-        """
-        判断响应是否被允许
-        
-        Args:
-            response: 响应对象
-            
-        Returns:
-            bool: 是否被允许
-        """
-        status_code = response.status_code
-        
-        # 首先检查是否被明确拒绝
-        if status_code in self.denied_codes:
-            return False
-            
-        # 检查是否被明确允许
-        if status_code in self.allowed_codes:
-            return True
-            
-        # 默认允许2xx状态码
-        if 200 <= status_code < 300:
-            return True
-            
-        # 默认拒绝其他状态码
-        return False
-
-    def _get_filter_reason(self, status_code):
-        """
-        获取过滤原因描述
-        
-        Args:
-            status_code (int): HTTP状态码
-            
-        Returns:
-            str: 过滤原因描述
-        """
-        if status_code in self.denied_codes:
-            return f"状态码 {status_code} 被明确拒绝"
-        elif status_code not in self.allowed_codes and not (200 <= status_code < 300):
-            return f"状态码 {status_code} 不在允许列表中"
-        else:
-            return f"状态码 {status_code} 被过滤"
-
-    def process_response(self, request, response, spider):
-        """
-        处理响应，过滤不符合要求的响应
-        
-        Args:
-            request: 请求对象
-            response: 响应对象
-            spider: 爬虫实例
-            
-        Returns:
-            response: 响应对象（如果被允许）
-            
-        Raises:
-            IgnoreRequestError: 如果响应被过滤
-        """
-        if self._is_response_allowed(response):
-            return response
-            
-        # 响应被过滤
-        reason = self._get_filter_reason(response.status_code)
-        self.logger.debug(f"过滤响应: {response.status_code} {response.url} - {reason}")
-        
-        # 抛出异常以忽略该响应
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+ResponseFilterMiddleware 中间件
+用于过滤不符合要求的HTTP响应，支持自定义允许的状态码
+"""
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import IgnoreRequestError
+
+
+class ResponseFilterMiddleware:
+    """
+    ResponseFilterMiddleware 中间件
+    用于过滤不符合要求的HTTP响应，支持自定义允许的状态码
+    
+    功能特性:
+    - 默认允许2xx状态码
+    - 支持自定义允许的状态码列表
+    - 支持拒绝特定状态码
+    - 提供详细的日志信息
+    - 支持按域名配置不同的过滤规则
+    """
+
+    def __init__(self, allowed_codes, denied_codes, log_level):
+        """
+        初始化中间件
+        
+        Args:
+            allowed_codes: 允许的状态码列表
+            denied_codes: 拒绝的状态码列表
+            log_level: 日志级别
+        """
+        # 确保状态码是整数类型
+        self.allowed_codes = set()
+        if allowed_codes:
+            for code in allowed_codes:
+                try:
+                    self.allowed_codes.add(int(code))
+                except (ValueError, TypeError):
+                    pass  # 忽略无效的状态码
+                    
+        self.denied_codes = set()
+        if denied_codes:
+            for code in denied_codes:
+                try:
+                    self.denied_codes.add(int(code))
+                except (ValueError, TypeError):
+                    pass  # 忽略无效的状态码
+                    
+        self.logger = get_logger(self.__class__.__name__, log_level)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        """
+        创建中间件实例
+        
+        Args:
+            crawler: 爬虫实例
+            
+        Returns:
+            ResponseFilterMiddleware: 中间件实例
+        """
+        o = cls(
+            allowed_codes=crawler.settings.get_list('ALLOWED_RESPONSE_CODES'),
+            denied_codes=crawler.settings.get_list('DENIED_RESPONSE_CODES'),
+            log_level=crawler.settings.get('LOG_LEVEL')
+        )
+        return o
+
+    def _is_response_allowed(self, response):
+        """
+        判断响应是否被允许
+        
+        Args:
+            response: 响应对象
+            
+        Returns:
+            bool: 是否被允许
+        """
+        status_code = response.status_code
+        
+        # 首先检查是否被明确拒绝
+        if status_code in self.denied_codes:
+            return False
+            
+        # 检查是否被明确允许
+        if status_code in self.allowed_codes:
+            return True
+            
+        # 默认允许2xx状态码
+        if 200 <= status_code < 300:
+            return True
+            
+        # 默认拒绝其他状态码
+        return False
+
+    def _get_filter_reason(self, status_code):
+        """
+        获取过滤原因描述
+        
+        Args:
+            status_code (int): HTTP状态码
+            
+        Returns:
+            str: 过滤原因描述
+        """
+        if status_code in self.denied_codes:
+            return f"状态码 {status_code} 被明确拒绝"
+        elif status_code not in self.allowed_codes and not (200 <= status_code < 300):
+            return f"状态码 {status_code} 不在允许列表中"
+        else:
+            return f"状态码 {status_code} 被过滤"
+
+    def process_response(self, request, response, spider):
+        """
+        处理响应，过滤不符合要求的响应
+        
+        Args:
+            request: 请求对象
+            response: 响应对象
+            spider: 爬虫实例
+            
+        Returns:
+            response: 响应对象（如果被允许）
+            
+        Raises:
+            IgnoreRequestError: 如果响应被过滤
+        """
+        if self._is_response_allowed(response):
+            return response
+            
+        # 响应被过滤
+        reason = self._get_filter_reason(response.status_code)
+        self.logger.debug(f"过滤响应: {response.status_code} {response.url} - {reason}")
+        
+        # 抛出异常以忽略该响应
         raise IgnoreRequestError(f"response filtered: {reason} - {response.status_code} {response.url}")

crawlo/middleware/retry.py

@@ -1,125 +1,125 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from typing import List
-
-try:
-    from anyio import EndOfStream
-except ImportError:
-    # 如果 anyio 不可用或者 EndOfStream 不存在，创建一个占位符
-    class EndOfStream(Exception):
-        pass
-
-try:
-    from httpcore import ReadError
-except ImportError:
-    class ReadError(Exception):
-        pass
-
-try:
-    from httpx import RemoteProtocolError, ConnectError, ReadTimeout
-except ImportError:
-    class RemoteProtocolError(Exception):
-        pass
-    class ConnectError(Exception):
-        pass
-    class ReadTimeout(Exception):
-        pass
-
-try:
-    from aiohttp.client_exceptions import ClientConnectionError, ClientPayloadError
-    from aiohttp import ClientConnectorError, ClientTimeout, ClientConnectorSSLError, ClientResponseError
-except ImportError:
-    class ClientConnectionError(Exception):
-        pass
-    class ClientPayloadError(Exception):
-        pass
-    class ClientConnectorError(Exception):
-        pass
-    class ClientTimeout(Exception):
-        pass
-    class ClientConnectorSSLError(Exception):
-        pass
-    class ClientResponseError(Exception):
-        pass
-
-from crawlo.utils.log import get_logger
-from crawlo.stats_collector import StatsCollector
-
-_retry_exceptions = [
-    EndOfStream,
-    ReadError,
-    asyncio.TimeoutError,
-    ConnectError,
-    ReadTimeout,
-    ClientConnectorError,
-    ClientResponseError,
-    RemoteProtocolError,
-    ClientTimeout,
-    ClientConnectorSSLError,
-    ClientPayloadError,
-    ClientConnectionError
-]
-
-
-class RetryMiddleware(object):
-
-    def __init__(
-            self,
-            *,
-            retry_http_codes: List,
-            ignore_http_codes: List,
-            max_retry_times: int,
-            retry_exceptions: List,
-            stats: StatsCollector,
-            retry_priority: int
-    ):
-        self.retry_http_codes = retry_http_codes
-        self.ignore_http_codes = ignore_http_codes
-        self.max_retry_times = max_retry_times
-        self.retry_exceptions = tuple(retry_exceptions + _retry_exceptions)
-        self.retry_priority = retry_priority
-        self.stats = stats
-        self.logger = get_logger(self.__class__.__name__)
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(
-            retry_http_codes=crawler.settings.get_list('RETRY_HTTP_CODES'),
-            ignore_http_codes=crawler.settings.get_list('IGNORE_HTTP_CODES'),
-            max_retry_times=crawler.settings.get_int('MAX_RETRY_TIMES'),
-            retry_exceptions=crawler.settings.get_list('RETRY_EXCEPTIONS'),
-            stats=crawler.stats,
-            retry_priority=crawler.settings.get_int('RETRY_PRIORITY')
-        )
-        return o
-
-    def process_response(self, request, response, spider):
-        if request.meta.get('dont_retry', False):
-            return response
-        if response.status_code in self.ignore_http_codes:
-            return response
-        if response.status_code in self.retry_http_codes:
-            # 重试逻辑
-            reason = f"response code {response.status_code}"
-            return self._retry(request, reason, spider) or response
-        return response
-
-    def process_exception(self, request, exc, spider):
-        if isinstance(exc, self.retry_exceptions) and not request.meta.get('dont_retry', False):
-            return self._retry(request=request, reason=type(exc).__name__, spider=spider)
-
-    def _retry(self, request, reason, spider):
-        retry_times = request.meta.get('retry_times', 0)
-        if retry_times < self.max_retry_times:
-            retry_times += 1
-            self.logger.info(f"{spider} {request} {reason} retrying {retry_times} time...")
-            request.meta['retry_times'] = retry_times
-            # request.dont_retry = True
-            request.meta['dont_retry'] = True
-            request.priority = request.priority + self.retry_priority
-            self.stats.inc_value("retry_count")
-            return request
-        else:
-            self.logger.warning(f"{spider} {request} {reason} retry max {self.max_retry_times} times, give up.")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import List
+
+try:
+    from anyio import EndOfStream
+except ImportError:
+    # 如果 anyio 不可用或者 EndOfStream 不存在，创建一个占位符
+    class EndOfStream(Exception):
+        pass
+
+try:
+    from httpcore import ReadError
+except ImportError:
+    class ReadError(Exception):
+        pass
+
+try:
+    from httpx import RemoteProtocolError, ConnectError, ReadTimeout
+except ImportError:
+    class RemoteProtocolError(Exception):
+        pass
+    class ConnectError(Exception):
+        pass
+    class ReadTimeout(Exception):
+        pass
+
+try:
+    from aiohttp.client_exceptions import ClientConnectionError, ClientPayloadError
+    from aiohttp import ClientConnectorError, ClientTimeout, ClientConnectorSSLError, ClientResponseError
+except ImportError:
+    class ClientConnectionError(Exception):
+        pass
+    class ClientPayloadError(Exception):
+        pass
+    class ClientConnectorError(Exception):
+        pass
+    class ClientTimeout(Exception):
+        pass
+    class ClientConnectorSSLError(Exception):
+        pass
+    class ClientResponseError(Exception):
+        pass
+
+from crawlo.utils.log import get_logger
+from crawlo.stats_collector import StatsCollector
+
+_retry_exceptions = [
+    EndOfStream,
+    ReadError,
+    asyncio.TimeoutError,
+    ConnectError,
+    ReadTimeout,
+    ClientConnectorError,
+    ClientResponseError,
+    RemoteProtocolError,
+    ClientTimeout,
+    ClientConnectorSSLError,
+    ClientPayloadError,
+    ClientConnectionError
+]
+
+
+class RetryMiddleware(object):
+
+    def __init__(
+            self,
+            *,
+            retry_http_codes: List,
+            ignore_http_codes: List,
+            max_retry_times: int,
+            retry_exceptions: List,
+            stats: StatsCollector,
+            retry_priority: int
+    ):
+        self.retry_http_codes = retry_http_codes
+        self.ignore_http_codes = ignore_http_codes
+        self.max_retry_times = max_retry_times
+        self.retry_exceptions = tuple(retry_exceptions + _retry_exceptions)
+        self.retry_priority = retry_priority
+        self.stats = stats
+        self.logger = get_logger(self.__class__.__name__)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(
+            retry_http_codes=crawler.settings.get_list('RETRY_HTTP_CODES'),
+            ignore_http_codes=crawler.settings.get_list('IGNORE_HTTP_CODES'),
+            max_retry_times=crawler.settings.get_int('MAX_RETRY_TIMES'),
+            retry_exceptions=crawler.settings.get_list('RETRY_EXCEPTIONS'),
+            stats=crawler.stats,
+            retry_priority=crawler.settings.get_int('RETRY_PRIORITY')
+        )
+        return o
+
+    def process_response(self, request, response, spider):
+        if request.meta.get('dont_retry', False):
+            return response
+        if response.status_code in self.ignore_http_codes:
+            return response
+        if response.status_code in self.retry_http_codes:
+            # 重试逻辑
+            reason = f"response code {response.status_code}"
+            return self._retry(request, reason, spider) or response
+        return response
+
+    def process_exception(self, request, exc, spider):
+        if isinstance(exc, self.retry_exceptions) and not request.meta.get('dont_retry', False):
+            return self._retry(request=request, reason=type(exc).__name__, spider=spider)
+
+    def _retry(self, request, reason, spider):
+        retry_times = request.meta.get('retry_times', 0)
+        if retry_times < self.max_retry_times:
+            retry_times += 1
+            self.logger.info(f"{spider} {request} {reason} retrying {retry_times} time...")
+            request.meta['retry_times'] = retry_times
+            # request.dont_retry = True
+            request.meta['dont_retry'] = True
+            request.priority = request.priority + self.retry_priority
+            self.stats.inc_value("retry_count")
+            return request
+        else:
+            self.logger.warning(f"{spider} {request} {reason} retry max {self.max_retry_times} times, give up.")
             return None

crawlo/middleware/simple_proxy.py

@@ -0,0 +1,65 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+简化版代理中间件
+提供基础的代理功能，避免过度复杂的实现
+"""
+import random
+from typing import Optional, List
+
+from crawlo import Request, Response
+from crawlo.exceptions import NotConfiguredError
+from crawlo.utils.log import get_logger
+
+
+class SimpleProxyMiddleware:
+    """简化版代理中间件"""
+
+    def __init__(self, settings, log_level):
+        self.logger = get_logger(self.__class__.__name__, log_level)
+
+        # 获取代理列表
+        self.proxies: List[str] = settings.get("PROXY_LIST", [])
+        self.enabled = settings.get_bool("PROXY_ENABLED", False)
+
+        if not self.enabled:
+            self.logger.info("SimpleProxyMiddleware disabled")
+            return
+
+        if not self.proxies:
+            raise NotConfiguredError("PROXY_LIST not configured, SimpleProxyMiddleware disabled")
+
+        self.logger.info(f"SimpleProxyMiddleware enabled with {len(self.proxies)} proxies")
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls(settings=crawler.settings, log_level=crawler.settings.get("LOG_LEVEL"))
+
+    async def process_request(self, request: Request, spider) -> Optional[Request]:
+        """为请求分配代理"""
+        if not self.enabled:
+            return None
+
+        if request.proxy:
+            # 请求已指定代理，不覆盖
+            return None
+
+        if self.proxies:
+            # 随机选择一个代理
+            proxy = random.choice(self.proxies)
+            request.proxy = proxy
+            self.logger.debug(f"Assigned proxy {proxy} to {request.url}")
+
+        return None
+
+    def process_response(self, request: Request, response: Response, spider) -> Response:
+        """处理响应"""
+        if request.proxy:
+            self.logger.debug(f"Proxy request successful: {request.proxy} | {request.url}")
+        return response
+
+    def process_exception(self, request: Request, exception: Exception, spider) -> Optional[Request]:
+        """处理异常"""
+        if request.proxy:
+            self.logger.warning(f"Proxy request failed: {request.proxy} | {request.url} | {repr(exception)}")
+        return None