crawlo 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

crawlo/network/response.py

@@ -1,166 +1,271 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import re
-import ujson
-from http.cookies import SimpleCookie
-from parsel import Selector, SelectorList
-from typing import Dict, Any, List, Optional
-from urllib.parse import urljoin as _urljoin
-
-from crawlo import Request
-from crawlo.exceptions import DecodeError
-
-
-class Response:
-    """
-    HTTP响应的封装，提供数据解析的便捷方法。
-    """
-
-    def __init__(
-        self,
-        url: str,
-        *,
-        headers: Dict[str, Any],
-        body: bytes = b"",
-        method: str = 'GET',
-        request: Request = None,
-        status_code: int = 200,
-    ):
-        self.url = url
-        self.headers = headers
-        self.body = body
-        self.method = method
-        self.request = request
-        self.status_code = status_code
-        self.encoding = self.request.encoding if self.request else None
-        self._text_cache = None
-        self._json_cache = None
-        self._selector_instance = None  # 修改变量名，避免与 @property 冲突
-
-    @property
-    def text(self) -> str:
-        """将响应体(body)以正确的编码解码为字符串，并缓存结果。"""
-        if self._text_cache is not None:
-            return self._text_cache
-
-        encoding = self.encoding
-        try:
-            # 优先使用 request 提供的编码
-            if encoding:
-                self._text_cache = self.body.decode(encoding)
-                return self._text_cache
-
-            # 从 Content-Type 中提取编码
-            content_type = self.headers.get("Content-Type", "")
-            charset_match = re.search(r"charset=([\w-]+)", content_type, re.I)
-            if charset_match:
-                encoding = charset_match.group(1)
-                self._text_cache = self.body.decode(encoding)
-                return self._text_cache
-
-            # 默认尝试 UTF-8
-            self._text_cache = self.body.decode("utf-8")
-            return self._text_cache
-
-        except UnicodeDecodeError as e:
-            raise DecodeError(f"Failed to decode response from {self.url}: {e}")
-
-    def json(self) -> Any:
-        """将响应文本解析为 JSON 对象。"""
-        if self._json_cache:
-            return self._json_cache
-        self._json_cache = ujson.loads(self.text)
-        return self._json_cache
-
-    def urljoin(self, url: str) -> str:
-        """拼接 URL，自动处理相对路径。"""
-        return _urljoin(self.url, url)
-
-    @property
-    def _selector(self) -> Selector:
-        """懒加载 Selector 实例"""
-        if self._selector_instance is None:
-            self._selector_instance = Selector(self.text)
-        return self._selector_instance
-
-    def xpath(self, query: str) -> SelectorList:
-        """使用 XPath 选择器查询文档。"""
-        return self._selector.xpath(query)
-
-    def css(self, query: str) -> SelectorList:
-        """使用 CSS 选择器查询文档。"""
-        return self._selector.css(query)
-
-    def xpath_text(self, query: str) -> str:
-        """使用 XPath 提取并返回纯文本。"""
-        fragments = self.xpath(f"{query}//text()").getall()
-        return " ".join(text.strip() for text in fragments if text.strip())
-
-    def css_text(self, query: str) -> str:
-        """使用 CSS 选择器提取并返回纯文本。"""
-        fragments = self.css(f"{query} ::text").getall()
-        return " ".join(text.strip() for text in fragments if text.strip())
-
-    def get_text(self, xpath_or_css: str, join_str: str = " ") -> str:
-        """
-        获取指定节点的纯文本(自动拼接子节点文本)
-
-        参数:
-            xpath_or_css: XPath或CSS选择器
-            join_str: 文本拼接分隔符(默认为空格)
-
-        返回:
-            拼接后的纯文本字符串
-        """
-        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
-        texts = elements.xpath('.//text()').getall()
-        return join_str.join(text.strip() for text in texts if text.strip())
-
-    def get_all_text(self, xpath_or_css: str, join_str: str = " ") -> List[str]:
-        """
-        获取多个节点的纯文本列表
-
-        参数:
-            xpath_or_css: XPath或CSS选择器
-            join_str: 单个节点内文本拼接分隔符
-
-        返回:
-            纯文本列表(每个元素对应一个节点的文本)
-        """
-        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
-        result = []
-        for element in elements:
-            texts = element.xpath('.//text()').getall()
-            clean_text = join_str.join(text.strip() for text in texts if text.strip())
-            if clean_text:
-                result.append(clean_text)
-        return result
-
-    def re_search(self, pattern: str, flags: int = re.DOTALL) -> Optional[re.Match]:
-        """在响应文本上执行正则表达式搜索。"""
-        if not isinstance(pattern, str):
-            raise TypeError("Pattern must be a string")
-        return re.search(pattern, self.text, flags=flags)
-
-    def re_findall(self, pattern: str, flags: int = re.DOTALL) -> List[Any]:
-        """在响应文本上执行正则表达式查找。"""
-        if not isinstance(pattern, str):
-            raise TypeError("Pattern must be a string")
-        return re.findall(pattern, self.text, flags=flags)
-
-    def get_cookies(self) -> Dict[str, str]:
-        """从响应头中解析并返回Cookies。"""
-        cookie_header = self.headers.get("Set-Cookie", "")
-        if isinstance(cookie_header, list):
-            cookie_header = ", ".join(cookie_header)
-        cookies = SimpleCookie()
-        cookies.load(cookie_header)
-        return {key: morsel.value for key, morsel in cookies.items()}
-
-    @property
-    def meta(self) -> Dict:
-        """获取关联的 Request 对象的 meta 字典。"""
-        return self.request.meta if self.request else {}
-
-    def __str__(self):
-        return f"<{self.status_code} {self.url}>"
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+HTTP Response 封装模块
+=====================
+提供功能丰富的HTTP响应封装，支持:
+- 智能编码检测和解码
+- XPath/CSS 选择器
+- JSON 解析和缓存
+- 正则表达式支持
+- Cookie 处理
+"""
+import re
+import ujson
+from http.cookies import SimpleCookie
+from parsel import Selector, SelectorList
+from typing import Dict, Any, List, Optional, Union
+from urllib.parse import urljoin as _urljoin
+
+from crawlo.exceptions import DecodeError
+
+
+class Response:
+    """
+    HTTP响应的封装，提供数据解析的便捷方法。
+    
+    功能特性:
+    - 智能编码检测和缓存
+    - 懒加载 Selector 实例
+    - JSON 解析和缓存
+    - 多类型数据提取
+    """
+
+    def __init__(
+            self,
+            url: str,
+            *,
+            headers: Dict[str, Any] = None,
+            body: bytes = b"",
+            method: str = 'GET',
+            request: 'Request' = None,  # 使用字符串注解避免循环导入
+            status_code: int = 200,
+    ):
+        # 基本属性
+        self.url = url
+        self.headers = headers or {}
+        self.body = body
+        self.method = method.upper()
+        self.request = request
+        self.status_code = status_code
+
+        # 编码处理
+        self.encoding = self._determine_encoding()
+
+        # 缓存属性
+        self._text_cache = None
+        self._json_cache = None
+        self._selector_instance = None
+
+        # 状态标记
+        self._is_success = 200 <= status_code < 300
+        self._is_redirect = 300 <= status_code < 400
+        self._is_client_error = 400 <= status_code < 500
+        self._is_server_error = status_code >= 500
+
+    def _determine_encoding(self) -> Optional[str]:
+        """智能检测响应编码"""
+        # 1. 优先使用 request 的编码
+        if self.request and self.request.encoding:
+            return self.request.encoding
+
+        # 2. 从 Content-Type 头中检测
+        content_type = self.headers.get("content-type", "") or self.headers.get("Content-Type", "")
+        if content_type:
+            charset_match = re.search(r"charset=([w-]+)", content_type, re.I)
+            if charset_match:
+                return charset_match.group(1).lower()
+
+        # 3. 从 HTML meta 标签中检测(仅对HTML内容)
+        if b'<html' in self.body[:1024].lower():
+            # 查找 <meta charset="xxx"> 或 <meta http-equiv="Content-Type" content="...charset=xxx">
+            html_start = self.body[:4096]  # 只检查前4KB
+            try:
+                html_text = html_start.decode('ascii', errors='ignore')
+                # <meta charset="utf-8">
+                charset_match = re.search(r'<meta[^>]+charset=["\']?([\w-]+)', html_text, re.I)
+                if charset_match:
+                    return charset_match.group(1).lower()
+
+                # <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+                content_match = re.search(r'<meta[^>]+content=["\'][^"\'>]*charset=([\w-]+)', html_text, re.I)
+                if content_match:
+                    return content_match.group(1).lower()
+            except Exception:
+                pass
+
+        # 4. 默认使用 utf-8
+        return 'utf-8'
+
+    @property
+    def text(self) -> str:
+        """将响应体(body)以正确的编码解码为字符串，并缓存结果。"""
+        if self._text_cache is not None:
+            return self._text_cache
+
+        if not self.body:
+            self._text_cache = ""
+            return self._text_cache
+
+        # 尝试多种编码
+        encodings_to_try = [self.encoding]
+        if self.encoding != 'utf-8':
+            encodings_to_try.append('utf-8')
+        if 'gbk' not in encodings_to_try:
+            encodings_to_try.append('gbk')
+        if 'gb2312' not in encodings_to_try:
+            encodings_to_try.append('gb2312')
+        encodings_to_try.append('latin1')  # 最后的回退选项
+
+        for encoding in encodings_to_try:
+            if not encoding:
+                continue
+            try:
+                self._text_cache = self.body.decode(encoding)
+                return self._text_cache
+            except (UnicodeDecodeError, LookupError):
+                continue
+
+        # 所有编码都失败，使用容错解码
+        try:
+            self._text_cache = self.body.decode('utf-8', errors='replace')
+            return self._text_cache
+        except Exception as e:
+            raise DecodeError(f"Failed to decode response from {self.url}: {e}")
+
+    @property
+    def is_success(self) -> bool:
+        """检查响应是否成功 (2xx)"""
+        return self._is_success
+
+    @property
+    def is_redirect(self) -> bool:
+        """检查响应是否为重定向 (3xx)"""
+        return self._is_redirect
+
+    @property
+    def is_client_error(self) -> bool:
+        """检查响应是否为客户端错误 (4xx)"""
+        return self._is_client_error
+
+    @property
+    def is_server_error(self) -> bool:
+        """检查响应是否为服务器错误 (5xx)"""
+        return self._is_server_error
+
+    @property
+    def content_type(self) -> str:
+        """获取响应的 Content-Type"""
+        return self.headers.get('content-type', '') or self.headers.get('Content-Type', '')
+
+    @property
+    def content_length(self) -> Optional[int]:
+        """获取响应的 Content-Length"""
+        length = self.headers.get('content-length') or self.headers.get('Content-Length')
+        return int(length) if length else None
+
+    def json(self, default: Any = None) -> Any:
+        """将响应文本解析为 JSON 对象。"""
+        if self._json_cache is not None:
+            return self._json_cache
+
+        try:
+            self._json_cache = ujson.loads(self.text)
+            return self._json_cache
+        except (ujson.JSONDecodeError, ValueError) as e:
+            if default is not None:
+                return default
+            raise DecodeError(f"Failed to parse JSON from {self.url}: {e}")
+
+    def urljoin(self, url: str) -> str:
+        """拼接 URL，自动处理相对路径。"""
+        return _urljoin(self.url, url)
+
+    @property
+    def _selector(self) -> Selector:
+        """懒加载 Selector 实例"""
+        if self._selector_instance is None:
+            self._selector_instance = Selector(self.text)
+        return self._selector_instance
+
+    def xpath(self, query: str) -> SelectorList:
+        """使用 XPath 选择器查询文档。"""
+        return self._selector.xpath(query)
+
+    def css(self, query: str) -> SelectorList:
+        """使用 CSS 选择器查询文档。"""
+        return self._selector.css(query)
+
+    def xpath_text(self, query: str) -> str:
+        """使用 XPath 提取并返回纯文本。"""
+        fragments = self.xpath(f"{query}//text()").getall()
+        return " ".join(text.strip() for text in fragments if text.strip())
+
+    def css_text(self, query: str) -> str:
+        """使用 CSS 选择器提取并返回纯文本。"""
+        fragments = self.css(f"{query} ::text").getall()
+        return " ".join(text.strip() for text in fragments if text.strip())
+
+    def get_text(self, xpath_or_css: str, join_str: str = " ") -> str:
+        """
+        获取指定节点的纯文本(自动拼接子节点文本)
+
+        参数:
+            xpath_or_css: XPath或CSS选择器
+            join_str: 文本拼接分隔符(默认为空格)
+
+        返回:
+            拼接后的纯文本字符串
+        """
+        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
+        texts = elements.xpath('.//text()').getall()
+        return join_str.join(text.strip() for text in texts if text.strip())
+
+    def get_all_text(self, xpath_or_css: str, join_str: str = " ") -> List[str]:
+        """
+        获取多个节点的纯文本列表
+
+        参数:
+            xpath_or_css: XPath或CSS选择器
+            join_str: 单个节点内文本拼接分隔符
+
+        返回:
+            纯文本列表(每个元素对应一个节点的文本)
+        """
+        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
+        result = []
+        for element in elements:
+            texts = element.xpath('.//text()').getall()
+            clean_text = join_str.join(text.strip() for text in texts if text.strip())
+            if clean_text:
+                result.append(clean_text)
+        return result
+
+    def re_search(self, pattern: str, flags: int = re.DOTALL) -> Optional[re.Match]:
+        """在响应文本上执行正则表达式搜索。"""
+        if not isinstance(pattern, str):
+            raise TypeError("Pattern must be a string")
+        return re.search(pattern, self.text, flags=flags)
+
+    def re_findall(self, pattern: str, flags: int = re.DOTALL) -> List[Any]:
+        """在响应文本上执行正则表达式查找。"""
+        if not isinstance(pattern, str):
+            raise TypeError("Pattern must be a string")
+        return re.findall(pattern, self.text, flags=flags)
+
+    def get_cookies(self) -> Dict[str, str]:
+        """从响应头中解析并返回Cookies。"""
+        cookie_header = self.headers.get("Set-Cookie", "")
+        if isinstance(cookie_header, list):
+            cookie_header = ", ".join(cookie_header)
+        cookies = SimpleCookie()
+        cookies.load(cookie_header)
+        return {key: morsel.value for key, morsel in cookies.items()}
+
+    @property
+    def meta(self) -> Dict:
+        """获取关联的 Request 对象的 meta 字典。"""
+        return self.request.meta if self.request else {}
+
+    def __str__(self):
+        return f"<{self.status_code} {self.url}>"

crawlo/pipelines/__init__.py

@@ -1,13 +1,22 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo.items.items import Item
-
-
-class BasePipeline:
-
-    def process_item(self, item: Item, spider):
-        raise NotImplementedError
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.items import Item
+
+
+class BasePipeline:
+
+    def process_item(self, item: Item, spider):
+        raise NotImplementedError
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls()
+
+
+# 导出去重管道
+from .memory_dedup_pipeline import MemoryDedupPipeline
+from .redis_dedup_pipeline import RedisDedupPipeline
+from .bloom_dedup_pipeline import BloomDedupPipeline
+from .database_dedup_pipeline import DatabaseDedupPipeline
+
+__all__ = ['BasePipeline', 'MemoryDedupPipeline', 'RedisDedupPipeline', 'BloomDedupPipeline', 'DatabaseDedupPipeline']

crawlo/pipelines/bloom_dedup_pipeline.py

@@ -0,0 +1,157 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+基于 Bloom Filter 的数据项去重管道
+=============================
+提供大规模数据采集场景下的高效去重功能，使用概率性数据结构节省内存。
+
+特点:
+- 内存效率高: 相比传统集合节省大量内存
+- 高性能: 快速的插入和查找操作
+- 可扩展: 支持自定义容量和误判率
+- 适用性广: 特别适合大规模数据采集
+
+注意: Bloom Filter 有误判率，可能会错误地丢弃一些未见过的数据项。
+"""
+
+import hashlib
+try:
+    from pybloom_live import BloomFilter
+    BLOOM_FILTER_AVAILABLE = True
+except ImportError:
+    # 如果没有安装 pybloom_live，使用简单的替代方案
+    BLOOM_FILTER_AVAILABLE = False
+    
+    class BloomFilter:
+        def __init__(self, capacity, error_rate):
+            self._data = set()
+        
+        def add(self, item):
+            if item in self._data:
+                return False
+            else:
+                self._data.add(item)
+                return True
+        
+        def __contains__(self, item):
+            return item in self._data
+
+from crawlo import Item
+from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import DropItem
+
+
+class BloomDedupPipeline:
+    """基于 Bloom Filter 的数据项去重管道"""
+
+    def __init__(
+            self,
+            capacity: int = 1000000,
+            error_rate: float = 0.001,
+            log_level: str = "INFO"
+    ):
+        """
+        初始化 Bloom Filter 去重管道
+        
+        :param capacity: 预期存储的元素数量
+        :param error_rate: 误判率 (例如 0.001 表示 0.1%)
+        :param log_level: 日志级别
+        """
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        
+        # 初始化 Bloom Filter
+        try:
+            self.bloom_filter = BloomFilter(capacity=capacity, error_rate=error_rate)
+            self.logger.info(f"Bloom Filter 去重管道初始化完成 (容量: {capacity}, 误判率: {error_rate})")
+        except Exception as e:
+            self.logger.error(f"Bloom Filter 初始化失败: {e}")
+            raise RuntimeError(f"Bloom Filter 初始化失败: {e}")
+
+        self.capacity = capacity
+        self.error_rate = error_rate
+        self.dropped_count = 0
+        self.added_count = 0
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """从爬虫配置创建管道实例"""
+        settings = crawler.settings
+        
+        return cls(
+            capacity=settings.getint('BLOOM_FILTER_CAPACITY', 1000000),
+            error_rate=settings.getfloat('BLOOM_FILTER_ERROR_RATE', 0.001),
+            log_level=settings.get('LOG_LEVEL', 'INFO')
+        )
+
+    def process_item(self, item: Item, spider: Spider) -> Item:
+        """
+        处理数据项，进行去重检查
+        
+        :param item: 要处理的数据项
+        :param spider: 爬虫实例
+        :return: 处理后的数据项或抛出 DropItem 异常
+        """
+        try:
+            # 生成数据项指纹
+            fingerprint = self._generate_item_fingerprint(item)
+            
+            # 检查指纹是否已存在
+            if fingerprint in self.bloom_filter:
+                # 如果可能已存在（Bloom Filter 可能有误判），丢弃这个数据项
+                self.dropped_count += 1
+                self.logger.debug(f"可能丢弃重复数据项: {fingerprint[:20]}...")
+                raise DropItem(f"可能重复的数据项: {fingerprint}")
+            else:
+                # 添加指纹到 Bloom Filter
+                self.bloom_filter.add(fingerprint)
+                self.added_count += 1
+                self.logger.debug(f"处理新数据项: {fingerprint[:20]}...")
+                return item
+                
+        except Exception as e:
+            self.logger.error(f"处理数据项时出错: {e}")
+            # 在错误时继续处理，避免丢失数据
+            return item
+
+    def _generate_item_fingerprint(self, item: Item) -> str:
+        """
+        生成数据项指纹
+        
+        基于数据项的所有字段生成唯一指纹，用于去重判断。
+        
+        :param item: 数据项
+        :return: 指纹字符串
+        """
+        # 将数据项转换为可序列化的字典
+        try:
+            item_dict = item.to_dict()
+        except AttributeError:
+            # 兼容没有to_dict方法的Item实现
+            item_dict = dict(item)
+        
+        # 对字典进行排序以确保一致性
+        sorted_items = sorted(item_dict.items())
+        
+        # 生成指纹字符串
+        fingerprint_string = '|'.join([f"{k}={v}" for k, v in sorted_items if v is not None])
+        
+        # 使用 SHA256 生成固定长度的指纹
+        return hashlib.sha256(fingerprint_string.encode('utf-8')).hexdigest()
+
+    def close_spider(self, spider: Spider) -> None:
+        """
+        爬虫关闭时的清理工作
+        
+        :param spider: 爬虫实例
+        """
+        self.logger.info(f"爬虫 {spider.name} 关闭:")
+        self.logger.info(f"  - 处理的数据项数: {self.added_count}")
+        self.logger.info(f"  - 可能丢弃的重复数据项: {self.dropped_count}")
+        
+        if BLOOM_FILTER_AVAILABLE:
+            # 注意：Bloom Filter 无法准确统计元素数量
+            self.logger.info(f"  - Bloom Filter 容量: {self.capacity}")
+            self.logger.info(f"  - Bloom Filter 误判率: {self.error_rate}")
+        else:
+            self.logger.warning("  - 未安装 pybloom_live，使用内存集合作为替代")