crawlo 1.4.6__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/network/response.py

@@ -11,12 +11,9 @@ HTTP Response 封装模块
 - Cookie 处理
 """
 import re
-from http.cookies import SimpleCookie
-from typing import Dict, Any, List, Optional, Tuple
-from urllib.parse import urljoin as _urljoin, urlparse as _urlparse, urlsplit as _urlsplit, parse_qs as _parse_qs, \
-    urlencode as _urlencode, quote as _quote, unquote as _unquote, urldefrag as _urldefrag
-
 import ujson
+from typing import Dict, Any, List, Optional
+from urllib.parse import urljoin as _urljoin
 from parsel import Selector, SelectorList
 
 # 尝试导入 w3lib 编码检测函数
@@ -31,6 +28,14 @@ try:
     W3LIB_AVAILABLE = True
 except ImportError:
     W3LIB_AVAILABLE = False
+    # 当 w3lib 不可用时，从 utils 导入替代函数
+    from crawlo.utils import (
+        html_body_declared_encoding,
+        html_to_unicode,
+        http_content_type_encoding,
+        read_bom,
+        resolve_encoding,
+    )
 
 from crawlo.exceptions import DecodeError
 from crawlo.utils import (
@@ -38,7 +43,11 @@ from crawlo.utils import (
     extract_texts,
     extract_attr,
     extract_attrs,
-    is_xpath
+    is_xpath,
+    parse_cookies,
+    regex_search,
+    regex_findall,
+    get_header_value
 )
 
 
@@ -74,10 +83,10 @@ class Response:
             self,
             url: str,
             *,
-            headers: Dict[str, Any] = None,
+            headers: Optional[Dict[str, Any]] = None,
             body: bytes = b"",
             method: str = 'GET',
-            request: 'Request' = None,  # 使用字符串注解避免循环导入
+            request: Optional['Request'] = None,  # 使用字符串注解避免循环导入
             status_code: int = 200,
     ):
         # 基本属性
@@ -106,7 +115,7 @@ class Response:
     
     def _determine_encoding(self) -> str:
         """
-        智能检测响应编码（参考 Scrapy 设计）
+        智能检测响应编码
         
         编码检测优先级：
         1. Request 中指定的编码
@@ -184,15 +193,19 @@ class Response:
     def _bom_encoding(self) -> Optional[str]:
         """BOM 字节顺序标记编码检测"""
         if not W3LIB_AVAILABLE:
-            return None
+            # 使用替代函数
+            encoding, _ = read_bom(self.body)
+            return encoding
         return read_bom(self.body)[0]
 
     @memoize_method_noargs
     def _headers_encoding(self) -> Optional[str]:
         """HTTP Content-Type 头部编码检测"""
         if not W3LIB_AVAILABLE:
-            return None
-        content_type = self.headers.get(b"Content-Type", b"") or self.headers.get("content-type", b"")
+            # 使用替代函数
+            content_type = self.headers.get("content-type", "") or self.headers.get("Content-Type", "")
+            return http_content_type_encoding(content_type)
+        content_type = self.headers.get("Content-Type", b"") or self.headers.get("content-type", b"")
         if isinstance(content_type, bytes):
             content_type = content_type.decode('latin-1')
         return http_content_type_encoding(content_type)
@@ -201,23 +214,26 @@ class Response:
     def _body_declared_encoding(self) -> Optional[str]:
         """HTML meta 标签声明编码检测"""
         if not W3LIB_AVAILABLE:
-            return None
+            # 使用替代函数
+            return html_body_declared_encoding(self.body)
         return html_body_declared_encoding(self.body)
 
     @memoize_method_noargs
     def _body_inferred_encoding(self) -> str:
         """内容自动检测编码"""
         if not W3LIB_AVAILABLE:
-            # 回退到简单的自动检测
-            for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"):
-                try:
-                    self.body.decode(enc)
-                except UnicodeError:
-                    continue
-                return enc
-            return self._DEFAULT_ENCODING
-
-        content_type = self.headers.get(b"Content-Type", b"") or self.headers.get("content-type", b"")
+            # 使用替代函数
+            content_type = self.headers.get("content-type", "") or self.headers.get("Content-Type", "")
+            # 使用 html_to_unicode 函数进行编码检测
+            encoding, _ = html_to_unicode(
+                content_type,
+                self.body,
+                auto_detect_fun=self._auto_detect_fun,
+                default_encoding=self._DEFAULT_ENCODING,
+            )
+            return encoding
+
+        content_type = self.headers.get("Content-Type", b"") or self.headers.get("content-type", b"")
         if isinstance(content_type, bytes):
             content_type = content_type.decode('latin-1')
         
@@ -233,6 +249,13 @@ class Response:
     def _auto_detect_fun(self, text: bytes) -> Optional[str]:
         """自动检测编码的回调函数"""
         if not W3LIB_AVAILABLE:
+            # 使用替代函数
+            for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"):
+                try:
+                    text.decode(enc)
+                except UnicodeError:
+                    continue
+                return resolve_encoding(enc)
             return None
         for enc in (self._DEFAULT_ENCODING, "utf-8", "cp1252"):
             try:
@@ -255,7 +278,7 @@ class Response:
         # 如果可用，使用 w3lib 进行更准确的解码
         if W3LIB_AVAILABLE:
             try:
-                content_type = self.headers.get(b"Content-Type", b"") or self.headers.get("content-type", b"")
+                content_type = self.headers.get("Content-Type", b"") or self.headers.get("content-type", b"")
                 if isinstance(content_type, bytes):
                     content_type = content_type.decode('latin-1')
                 
@@ -269,6 +292,20 @@ class Response:
             except Exception:
                 # 如果 w3lib 解码失败，回退到原有方法
                 pass
+        else:
+            # 使用替代函数
+            try:
+                content_type = self.headers.get("content-type", "") or self.headers.get("Content-Type", "")
+                # 使用 html_to_unicode 函数
+                _, self._text_cache = html_to_unicode(
+                    content_type,
+                    self.body,
+                    default_encoding=self.encoding
+                )
+                return self._text_cache
+            except Exception:
+                # 如果解码失败，回退到原有方法
+                pass
 
         # 尝试多种编码
         encodings_to_try = [self.encoding]
@@ -323,12 +360,12 @@ class Response:
     @property
     def content_type(self) -> str:
         """获取响应的 Content-Type"""
-        return self.headers.get('content-type', '') or self.headers.get('Content-Type', '')
+        return get_header_value(self.headers, 'content-type', '')
 
     @property
     def content_length(self) -> Optional[int]:
         """获取响应的 Content-Length"""
-        length = self.headers.get('content-length') or self.headers.get('Content-Length')
+        length = get_header_value(self.headers, 'content-length')
         return int(length) if length else None
 
     # ==================== JSON处理方法 ====================
@@ -352,103 +389,8 @@ class Response:
         """拼接 URL，自动处理相对路径。"""
         return _urljoin(self.url, url)
 
-    def urlparse(self, url: str = None) -> Tuple:
-        """
-        解析 URL 为组件元组 (scheme, netloc, path, params, query, fragment)
-        
-        Args:
-            url (str, optional): 要解析的URL，默认为响应的URL
-            
-        Returns:
-            tuple: URL组件元组
-        """
-        target_url = url if url is not None else self.url
-        return _urlparse(target_url)
-
-    def urlsplit(self, url: str = None) -> Tuple:
-        """
-        解析 URL 为组件元组 (scheme, netloc, path, query, fragment)
-        
-        Args:
-            url (str, optional): 要解析的URL，默认为响应的URL
-            
-        Returns:
-            tuple: URL组件元组（不包含params）
-        """
-        target_url = url if url is not None else self.url
-        return _urlsplit(target_url)
-
-    def parse_qs(self, query_string: str = None, keep_blank_values: bool = False) -> Dict[str, List[str]]:
-        """
-        解析查询字符串为字典
-        
-        Args:
-            query_string (str, optional): 查询字符串，默认从URL中提取
-            keep_blank_values (bool): 是否保留空值
-            
-        Returns:
-            dict: 查询参数字典
-        """
-        if query_string is None:
-            # 从URL中提取查询字符串
-            parsed = _urlparse(self.url)
-            query_string = parsed.query
-            
-        return _parse_qs(query_string, keep_blank_values=keep_blank_values)
-
-    def urlencode(self, query: Dict[str, Any]) -> str:
-        """
-        将字典编码为查询字符串
-        
-        Args:
-            query (dict): 要编码的查询参数字典
-            
-        Returns:
-            str: 编码后的查询字符串
-        """
-        return _urlencode(query)
-
-    def quote(self, string: str, safe: str = '/') -> str:
-        """
-        URL 编码
-        
-        Args:
-            string (str): 要编码的字符串
-            safe (str): 不编码的字符，默认为 '/'
-            
-        Returns:
-            str: URL编码后的字符串
-        """
-        return _quote(string, safe=safe)
-
-    def unquote(self, string: str) -> str:
-        """
-        URL 解码
-        
-        Args:
-            string (str): 要解码的字符串
-            
-        Returns:
-            str: URL解码后的字符串
-        """
-        return _unquote(string)
-
-    def urldefrag(self, url: str = None) -> Tuple[str, str]:
-        """
-        移除 URL 中的片段标识符
-        
-        Args:
-            url (str, optional): 要处理的URL，默认为响应的URL
-            
-        Returns:
-            tuple: (去除片段的URL, 片段)
-        """
-        target_url = url if url is not None else self.url
-        defrag_result = _urldefrag(target_url)
-        return (defrag_result.url, defrag_result.fragment)
-
     # ==================== 选择器相关方法 ====================
-    
+
     @property
     def _selector(self) -> Selector:
         """懒加载 Selector 实例"""
@@ -507,7 +449,7 @@ class Response:
         except Exception:
             return default
 
-    def extract_texts(self, xpath_or_css: str, join_str: str = " ", default: List[str] = None) -> List[str]:
+    def extract_texts(self, xpath_or_css: str, join_str: str = " ", default: Optional[List[str]] = None) -> List[str]:
         """
         提取多个元素的文本内容列表，支持CSS和XPath选择器
 
@@ -570,7 +512,7 @@ class Response:
         except Exception:
             return default
 
-    def extract_attrs(self, xpath_or_css: str, attr_name: str, default: List[Any] = None) -> List[Any]:
+    def extract_attrs(self, xpath_or_css: str, attr_name: str, default: Optional[List[Any]] = None) -> List[Any]:
         """
         提取多个元素的属性值列表，支持CSS和XPath选择器
 
@@ -608,26 +550,18 @@ class Response:
     
     def re_search(self, pattern: str, flags: int = re.DOTALL) -> Optional[re.Match]:
         """在响应文本上执行正则表达式搜索。"""
-        if not isinstance(pattern, str):
-            raise TypeError("Pattern must be a string")
-        return re.search(pattern, self.text, flags=flags)
+        return regex_search(pattern, self.text, flags)
 
     def re_findall(self, pattern: str, flags: int = re.DOTALL) -> List[Any]:
         """在响应文本上执行正则表达式查找。"""
-        if not isinstance(pattern, str):
-            raise TypeError("Pattern must be a string")
-        return re.findall(pattern, self.text, flags=flags)
+        return regex_findall(pattern, self.text, flags)
 
     # ==================== Cookie处理方法 ====================
     
     def get_cookies(self) -> Dict[str, str]:
         """从响应头中解析并返回Cookies。"""
         cookie_header = self.headers.get("Set-Cookie", "")
-        if isinstance(cookie_header, list):
-            cookie_header = ", ".join(cookie_header)
-        cookies = SimpleCookie()
-        cookies.load(cookie_header)
-        return {key: morsel.value for key, morsel in cookies.items()}
+        return parse_cookies(cookie_header)
 
     # ==================== 请求相关方法 ====================
     

crawlo/pipelines/__init__.py

@@ -1,17 +1,36 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-from crawlo.items import Item
+"""
+Pipeline 模块
+=============
 
+Pipeline体系：
+- BasePipeline: 基础抽象类，定义Pipeline接口规范
+- ResourceManagedPipeline: 提供资源管理功能（推荐使用）
+- FileBasedPipeline/DatabasePipeline/CacheBasedPipeline: 特定场景的专用基类
 
-class BasePipeline:
+内置去重Pipeline：
+- MemoryDedupPipeline: 基于内存的去重
+- RedisDedupPipeline: 基于Redis的分布式去重
+- BloomDedupPipeline: 基于Bloom Filter的高效去重
+- DatabaseDedupPipeline: 基于数据库的去重
 
-    def process_item(self, item: Item, spider):
-        raise NotImplementedError
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls()
+使用示例：
+    # 在settings.py中配置
+    PIPELINES = [
+        'crawlo.pipelines.RedisDedupPipeline',
+        'your_project.pipelines.MongoPipeline',
+    ]
+"""
 
+# 导入所有基类（从base_pipeline.py）
+from .base_pipeline import (
+    BasePipeline,
+    ResourceManagedPipeline,
+    FileBasedPipeline,
+    DatabasePipeline,
+    CacheBasedPipeline
+)
 
 # 导出去重管道
 from .memory_dedup_pipeline import MemoryDedupPipeline
@@ -19,4 +38,16 @@ from .redis_dedup_pipeline import RedisDedupPipeline
 from .bloom_dedup_pipeline import BloomDedupPipeline
 from .database_dedup_pipeline import DatabaseDedupPipeline
 
-__all__ = ['BasePipeline', 'MemoryDedupPipeline', 'RedisDedupPipeline', 'BloomDedupPipeline', 'DatabaseDedupPipeline']
+__all__ = [
+    # 基类
+    'BasePipeline',
+    'ResourceManagedPipeline',
+    'FileBasedPipeline', 
+    'DatabasePipeline',
+    'CacheBasedPipeline',
+    # 去重管道
+    'MemoryDedupPipeline',
+    'RedisDedupPipeline', 
+    'BloomDedupPipeline',
+    'DatabaseDedupPipeline'
+]