crawlo 1.4.7__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/middleware/response_code.py

@@ -1,151 +1,151 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-ResponseCodeMiddleware 中间件
-用于处理HTTP响应状态码，记录统计信息并支持特殊状态码处理
-"""
-from crawlo.logging import get_logger
-
-
-class ResponseCodeMiddleware(object):
-    """
-    ResponseCodeMiddleware 中间件
-    用于处理HTTP响应状态码，记录统计信息并支持特殊状态码处理
-    
-    功能特性:
-    - 记录各种HTTP状态码的出现次数
-    - 支持特殊状态码的详细处理
-    - 提供详细的日志信息
-    - 支持状态码分类统计(2xx, 3xx, 4xx, 5xx)
-    """
-
-    def __init__(self, stats, log_level):
-        """
-        初始化中间件
-        
-        Args:
-            stats: 统计信息收集器
-            log_level: 日志级别
-        """
-        self.logger = get_logger(self.__class__.__name__)
-        self.stats = stats
-
-    @classmethod
-    def create_instance(cls, crawler):
-        """
-        创建中间件实例
-        
-        Args:
-            crawler: 爬虫实例
-            
-        Returns:
-            ResponseCodeMiddleware: 中间件实例
-        """
-        o = cls(stats=crawler.stats, log_level=crawler.settings.get('LOG_LEVEL'))
-        return o
-
-    def _get_status_category(self, status_code):
-        """
-        获取状态码分类
-        
-        Args:
-            status_code (int): HTTP状态码
-            
-        Returns:
-            str: 状态码分类 (2xx, 3xx, 4xx, 5xx, other)
-        """
-        if 200 <= status_code < 300:
-            return "2xx"
-        elif 300 <= status_code < 400:
-            return "3xx"
-        elif 400 <= status_code < 500:
-            return "4xx"
-        elif 500 <= status_code < 600:
-            return "5xx"
-        else:
-            return "other"
-
-    def _is_success_response(self, status_code):
-        """
-        判断是否为成功响应
-        
-        Args:
-            status_code (int): HTTP状态码
-            
-        Returns:
-            bool: 是否为成功响应
-        """
-        return 200 <= status_code < 300
-
-    def _is_redirect_response(self, status_code):
-        """
-        判断是否为重定向响应
-        
-        Args:
-            status_code (int): HTTP状态码
-            
-        Returns:
-            bool: 是否为重定向响应
-        """
-        return 300 <= status_code < 400
-
-    def _is_client_error(self, status_code):
-        """
-        判断是否为客户端错误
-        
-        Args:
-            status_code (int): HTTP状态码
-            
-        Returns:
-            bool: 是否为客户端错误
-        """
-        return 400 <= status_code < 500
-
-    def _is_server_error(self, status_code):
-        """
-        判断是否为服务器错误
-        
-        Args:
-            status_code (int): HTTP状态码
-            
-        Returns:
-            bool: 是否为服务器错误
-        """
-        return 500 <= status_code < 600
-
-    def process_response(self, request, response, spider):
-        """
-        处理响应，记录状态码统计信息
-        
-        Args:
-            request: 请求对象
-            response: 响应对象
-            spider: 爬虫实例
-            
-        Returns:
-            response: 响应对象
-        """
-        status_code = response.status_code
-        
-        # 只记录总的统计信息，不记录每个域名和每个状态码的详细信息
-        # 记录状态码分类统计
-        category = self._get_status_category(status_code)
-        self.stats.inc_value(f'response_status_code/category/{category}')
-        
-        # 记录成功/失败统计
-        if self._is_success_response(status_code):
-            self.stats.inc_value('response_status_code/success_count')
-        elif self._is_client_error(status_code) or self._is_server_error(status_code):
-            self.stats.inc_value('response_status_code/error_count')
-            
-        # 记录响应大小统计
-        if hasattr(response, 'content_length') and response.content_length:
-            self.stats.inc_value('response_total_bytes', response.content_length)
-        
-        # 详细日志记录
-        self.logger.debug(
-            f'收到响应: {status_code} {response.url} '
-            f'(分类: {category}, 大小: {getattr(response, "content_length", "unknown")} bytes)'
-        )
-        
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+ResponseCodeMiddleware 中间件
+用于处理HTTP响应状态码，记录统计信息并支持特殊状态码处理
+"""
+from crawlo.logging import get_logger
+
+
+class ResponseCodeMiddleware(object):
+    """
+    ResponseCodeMiddleware 中间件
+    用于处理HTTP响应状态码，记录统计信息并支持特殊状态码处理
+    
+    功能特性:
+    - 记录各种HTTP状态码的出现次数
+    - 支持特殊状态码的详细处理
+    - 提供详细的日志信息
+    - 支持状态码分类统计(2xx, 3xx, 4xx, 5xx)
+    """
+
+    def __init__(self, stats, log_level):
+        """
+        初始化中间件
+        
+        Args:
+            stats: 统计信息收集器
+            log_level: 日志级别
+        """
+        self.logger = get_logger(self.__class__.__name__)
+        self.stats = stats
+
+    @classmethod
+    def create_instance(cls, crawler):
+        """
+        创建中间件实例
+        
+        Args:
+            crawler: 爬虫实例
+            
+        Returns:
+            ResponseCodeMiddleware: 中间件实例
+        """
+        o = cls(stats=crawler.stats, log_level=crawler.settings.get('LOG_LEVEL'))
+        return o
+
+    def _get_status_category(self, status_code):
+        """
+        获取状态码分类
+        
+        Args:
+            status_code (int): HTTP状态码
+            
+        Returns:
+            str: 状态码分类 (2xx, 3xx, 4xx, 5xx, other)
+        """
+        if 200 <= status_code < 300:
+            return "2xx"
+        elif 300 <= status_code < 400:
+            return "3xx"
+        elif 400 <= status_code < 500:
+            return "4xx"
+        elif 500 <= status_code < 600:
+            return "5xx"
+        else:
+            return "other"
+
+    def _is_success_response(self, status_code):
+        """
+        判断是否为成功响应
+        
+        Args:
+            status_code (int): HTTP状态码
+            
+        Returns:
+            bool: 是否为成功响应
+        """
+        return 200 <= status_code < 300
+
+    def _is_redirect_response(self, status_code):
+        """
+        判断是否为重定向响应
+        
+        Args:
+            status_code (int): HTTP状态码
+            
+        Returns:
+            bool: 是否为重定向响应
+        """
+        return 300 <= status_code < 400
+
+    def _is_client_error(self, status_code):
+        """
+        判断是否为客户端错误
+        
+        Args:
+            status_code (int): HTTP状态码
+            
+        Returns:
+            bool: 是否为客户端错误
+        """
+        return 400 <= status_code < 500
+
+    def _is_server_error(self, status_code):
+        """
+        判断是否为服务器错误
+        
+        Args:
+            status_code (int): HTTP状态码
+            
+        Returns:
+            bool: 是否为服务器错误
+        """
+        return 500 <= status_code < 600
+
+    def process_response(self, request, response, spider):
+        """
+        处理响应，记录状态码统计信息
+        
+        Args:
+            request: 请求对象
+            response: 响应对象
+            spider: 爬虫实例
+            
+        Returns:
+            response: 响应对象
+        """
+        status_code = response.status_code
+        
+        # 只记录总的统计信息，不记录每个域名和每个状态码的详细信息
+        # 记录状态码分类统计
+        category = self._get_status_category(status_code)
+        self.stats.inc_value(f'response_status_code/category/{category}')
+        
+        # 记录成功/失败统计
+        if self._is_success_response(status_code):
+            self.stats.inc_value('response_status_code/success_count')
+        elif self._is_client_error(status_code) or self._is_server_error(status_code):
+            self.stats.inc_value('response_status_code/error_count')
+            
+        # 记录响应大小统计
+        if hasattr(response, 'content_length') and response.content_length:
+            self.stats.inc_value('response_total_bytes', response.content_length)
+        
+        # 详细日志记录
+        self.logger.debug(
+            f'收到响应: {status_code} {response.url} '
+            f'(分类: {category}, 大小: {getattr(response, "content_length", "unknown")} bytes)'
+        )
+        
         return response

crawlo/middleware/response_filter.py

@@ -1,137 +1,137 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-ResponseFilterMiddleware 中间件
-用于过滤不符合要求的HTTP响应，支持自定义允许的状态码
-"""
-from crawlo.logging import get_logger
-from crawlo.exceptions import IgnoreRequestError
-
-
-class ResponseFilterMiddleware:
-    """
-    ResponseFilterMiddleware 中间件
-    用于过滤不符合要求的HTTP响应，支持自定义允许的状态码
-    
-    功能特性:
-    - 默认允许2xx状态码
-    - 支持自定义允许的状态码列表
-    - 支持拒绝特定状态码
-    - 提供详细的日志信息
-    - 支持按域名配置不同的过滤规则
-    """
-
-    def __init__(self, allowed_codes, denied_codes, log_level):
-        """
-        初始化中间件
-        
-        Args:
-            allowed_codes: 允许的状态码列表
-            denied_codes: 拒绝的状态码列表
-            log_level: 日志级别
-        """
-        # 确保状态码是整数类型
-        self.allowed_codes = set()
-        if allowed_codes:
-            for code in allowed_codes:
-                try:
-                    self.allowed_codes.add(int(code))
-                except (ValueError, TypeError):
-                    pass  # 忽略无效的状态码
-                    
-        self.denied_codes = set()
-        if denied_codes:
-            for code in denied_codes:
-                try:
-                    self.denied_codes.add(int(code))
-                except (ValueError, TypeError):
-                    pass  # 忽略无效的状态码
-                    
-        self.logger = get_logger(self.__class__.__name__)
-
-    @classmethod
-    def create_instance(cls, crawler):
-        """
-        创建中间件实例
-        
-        Args:
-            crawler: 爬虫实例
-            
-        Returns:
-            ResponseFilterMiddleware: 中间件实例
-        """
-        o = cls(
-            allowed_codes=crawler.settings.get_list('ALLOWED_RESPONSE_CODES'),
-            denied_codes=crawler.settings.get_list('DENIED_RESPONSE_CODES'),
-            log_level=crawler.settings.get('LOG_LEVEL')
-        )
-        return o
-
-    def _is_response_allowed(self, response):
-        """
-        判断响应是否被允许
-        
-        Args:
-            response: 响应对象
-            
-        Returns:
-            bool: 是否被允许
-        """
-        status_code = response.status_code
-        
-        # 首先检查是否被明确拒绝
-        if status_code in self.denied_codes:
-            return False
-            
-        # 检查是否被明确允许
-        if status_code in self.allowed_codes:
-            return True
-            
-        # 默认允许2xx状态码
-        if 200 <= status_code < 300:
-            return True
-            
-        # 默认拒绝其他状态码
-        return False
-
-    def _get_filter_reason(self, status_code):
-        """
-        获取过滤原因描述
-        
-        Args:
-            status_code (int): HTTP状态码
-            
-        Returns:
-            str: 过滤原因描述
-        """
-        if status_code in self.denied_codes:
-            return f"状态码 {status_code} 被明确拒绝"
-        elif status_code not in self.allowed_codes and not (200 <= status_code < 300):
-            return f"状态码 {status_code} 不在允许列表中"
-        else:
-            return f"状态码 {status_code} 被过滤"
-
-    def process_response(self, request, response, spider):
-        """
-        处理响应，过滤不符合要求的响应
-        
-        Args:
-            request: 请求对象
-            response: 响应对象
-            spider: 爬虫实例
-            
-        Returns:
-            response: 响应对象（如果被允许）
-            
-        Raises:
-            IgnoreRequestError: 如果响应被过滤
-        """
-        if self._is_response_allowed(response):
-            return response
-            
-        # 响应被过滤
-        reason = self._get_filter_reason(response.status_code)
-        self.logger.debug(f"过滤响应: {response.status_code} {response.url} - {reason}")
-        
-        # 抛出异常以忽略该响应
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+ResponseFilterMiddleware 中间件
+用于过滤不符合要求的HTTP响应，支持自定义允许的状态码
+"""
+from crawlo.logging import get_logger
+from crawlo.exceptions import IgnoreRequestError
+
+
+class ResponseFilterMiddleware:
+    """
+    ResponseFilterMiddleware 中间件
+    用于过滤不符合要求的HTTP响应，支持自定义允许的状态码
+    
+    功能特性:
+    - 默认允许2xx状态码
+    - 支持自定义允许的状态码列表
+    - 支持拒绝特定状态码
+    - 提供详细的日志信息
+    - 支持按域名配置不同的过滤规则
+    """
+
+    def __init__(self, allowed_codes, denied_codes, log_level):
+        """
+        初始化中间件
+        
+        Args:
+            allowed_codes: 允许的状态码列表
+            denied_codes: 拒绝的状态码列表
+            log_level: 日志级别
+        """
+        # 确保状态码是整数类型
+        self.allowed_codes = set()
+        if allowed_codes:
+            for code in allowed_codes:
+                try:
+                    self.allowed_codes.add(int(code))
+                except (ValueError, TypeError):
+                    pass  # 忽略无效的状态码
+                    
+        self.denied_codes = set()
+        if denied_codes:
+            for code in denied_codes:
+                try:
+                    self.denied_codes.add(int(code))
+                except (ValueError, TypeError):
+                    pass  # 忽略无效的状态码
+                    
+        self.logger = get_logger(self.__class__.__name__)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        """
+        创建中间件实例
+        
+        Args:
+            crawler: 爬虫实例
+            
+        Returns:
+            ResponseFilterMiddleware: 中间件实例
+        """
+        o = cls(
+            allowed_codes=crawler.settings.get_list('ALLOWED_RESPONSE_CODES'),
+            denied_codes=crawler.settings.get_list('DENIED_RESPONSE_CODES'),
+            log_level=crawler.settings.get('LOG_LEVEL')
+        )
+        return o
+
+    def _is_response_allowed(self, response):
+        """
+        判断响应是否被允许
+        
+        Args:
+            response: 响应对象
+            
+        Returns:
+            bool: 是否被允许
+        """
+        status_code = response.status_code
+        
+        # 首先检查是否被明确拒绝
+        if status_code in self.denied_codes:
+            return False
+            
+        # 检查是否被明确允许
+        if status_code in self.allowed_codes:
+            return True
+            
+        # 默认允许2xx状态码
+        if 200 <= status_code < 300:
+            return True
+            
+        # 默认拒绝其他状态码
+        return False
+
+    def _get_filter_reason(self, status_code):
+        """
+        获取过滤原因描述
+        
+        Args:
+            status_code (int): HTTP状态码
+            
+        Returns:
+            str: 过滤原因描述
+        """
+        if status_code in self.denied_codes:
+            return f"状态码 {status_code} 被明确拒绝"
+        elif status_code not in self.allowed_codes and not (200 <= status_code < 300):
+            return f"状态码 {status_code} 不在允许列表中"
+        else:
+            return f"状态码 {status_code} 被过滤"
+
+    def process_response(self, request, response, spider):
+        """
+        处理响应，过滤不符合要求的响应
+        
+        Args:
+            request: 请求对象
+            response: 响应对象
+            spider: 爬虫实例
+            
+        Returns:
+            response: 响应对象（如果被允许）
+            
+        Raises:
+            IgnoreRequestError: 如果响应被过滤
+        """
+        if self._is_response_allowed(response):
+            return response
+            
+        # 响应被过滤
+        reason = self._get_filter_reason(response.status_code)
+        self.logger.debug(f"过滤响应: {response.status_code} {response.url} - {reason}")
+        
+        # 抛出异常以忽略该响应
         raise IgnoreRequestError(f"response filtered: {reason} - {response.status_code} {response.url}")