crawlo 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

crawlo/filters/memory_filter.py

@@ -0,0 +1,203 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import os
+import threading
+from weakref import WeakSet
+from typing import Set, TextIO, Optional
+
+from crawlo import Request
+from crawlo.filters import BaseFilter
+from crawlo.utils.log import get_logger
+from crawlo.utils.request import request_fingerprint
+
+
+class MemoryFilter(BaseFilter):
+    """基于内存的高效请求去重过滤器，适用于单机爬虫"""
+
+    def __init__(self, crawler):
+        """
+        初始化内存过滤器
+
+        :param crawler: 爬虫实例，用于获取配置
+        """
+        self.fingerprints: Set[str] = set()  # 主指纹存储
+        self._temp_weak_refs = WeakSet()  # 弱引用临时存储（可选）
+
+        debug = crawler.settings.get_bool('FILTER_DEBUG', False)
+        logger = get_logger(
+            self.__class__.__name__,  # 使用类名替代字符串
+            crawler.settings.get('LOG_LEVEL', 'INFO')
+        )
+        super().__init__(logger, crawler.stats, debug)
+
+        # 性能计数器
+        self._dupe_count = 0
+        self._unique_count = 0
+
+    def add_fingerprint(self, fp: str) -> None:
+        """
+        添加请求指纹
+
+        :param fp: 请求指纹字符串
+        :raises TypeError: 如果指纹不是字符串类型
+        """
+        if not isinstance(fp, str):
+            raise TypeError(f"指纹必须是字符串类型，得到 {type(fp)}")
+
+        self.fingerprints.add(fp)
+        self._unique_count += 1
+        # self.logger.debug(f"添加指纹: {fp[:10]}...")  # 日志截断防止过长
+
+    def requested(self, request: Request) -> bool:
+        """
+        检查请求是否重复（主要接口）
+
+        :param request: 请求对象
+        :return: 是否重复
+        """
+        fp = request_fingerprint(request)
+        if fp in self:
+            self._dupe_count += 1
+            # self.logger.debug(f"发现重复请求: {fp[:10]}...")
+            return True
+
+        self.add_fingerprint(fp)
+        return False
+
+    def __contains__(self, item: str) -> bool:
+        """
+        支持 in 操作符检查
+
+        :param item: 要检查的指纹
+        :return: 是否已存在
+        """
+        return item in self.fingerprints
+
+    @property
+    def stats_summary(self) -> dict:
+        """获取过滤器统计信息"""
+        return {
+            'capacity': len(self.fingerprints),
+            'duplicates': self._dupe_count,
+            'uniques': self._unique_count,
+            'memory_usage': self._estimate_memory()
+        }
+
+    def _estimate_memory(self) -> str:
+        """估算内存使用量（近似值）"""
+        avg_item_size = sum(len(x) for x in self.fingerprints) / max(1, len(self.fingerprints))
+        total = len(self.fingerprints) * (avg_item_size + 50)  # 50字节额外开销
+        return f"{total / (1024 * 1024):.2f} MB"
+
+    def clear(self) -> None:
+        """清空所有指纹数据"""
+        self.fingerprints.clear()
+        self._dupe_count = 0
+        self._unique_count = 0
+        self.logger.info("已清空内存过滤器")
+
+    def close(self) -> None:
+        """关闭过滤器（清理资源）"""
+        self.clear()
+
+    # 兼容旧版异步接口
+    async def closed(self):
+        """兼容异步接口"""
+        self.close()
+
+
+class MemoryFileFilter(BaseFilter):
+    """基于内存的请求指纹过滤器，支持原子化文件持久化"""
+
+    def __init__(self, crawler):
+        """
+        初始化过滤器
+        :param crawler: Scrapy Crawler对象，用于获取配置
+        """
+        self.fingerprints: Set[str] = set()  # 主存储集合
+        self._lock = threading.RLock()  # 线程安全锁
+        self._file: Optional[TextIO] = None  # 文件句柄
+
+        debug = crawler.settings.get_bool("FILTER_DEBUG", False)
+        logger = get_logger(
+            self.__class__.__name__,  # 使用类名作为日志标识
+            crawler.settings.get("LOG_LEVEL", "INFO")
+        )
+        super().__init__(logger, crawler.stats, debug)
+
+        # 初始化文件存储
+        request_dir = crawler.settings.get("REQUEST_DIR")
+        if request_dir:
+            self._init_file_store(request_dir)
+
+    def _init_file_store(self, request_dir: str) -> None:
+        """原子化初始化文件存储"""
+        with self._lock:
+            try:
+                os.makedirs(request_dir, exist_ok=True)
+                file_path = os.path.join(request_dir, 'request_fingerprints.txt')
+
+                # 原子化操作：读取现有指纹
+                if os.path.exists(file_path):
+                    with open(file_path, 'r', encoding='utf-8') as f:
+                        self.fingerprints.update(
+                            line.strip() for line in f
+                            if line.strip()
+                        )
+
+                # 以追加模式打开文件
+                self._file = open(file_path, 'a+', encoding='utf-8')
+                self.logger.info(f"Initialized fingerprint file: {file_path}")
+
+            except Exception as e:
+                self.logger.error(f"Failed to init file store: {str(e)}")
+                raise
+
+    def add_fingerprint(self, fp: str) -> None:
+        """
+        线程安全的指纹添加操作
+        :param fp: 请求指纹字符串
+        """
+        with self._lock:
+            if fp not in self.fingerprints:
+                self.fingerprints.add(fp)
+                self._persist_fp(fp)
+
+    def _persist_fp(self, fp: str) -> None:
+        """持久化指纹到文件（需在锁保护下调用）"""
+        if self._file:
+            try:
+                self._file.write(f"{fp}\n")
+                self._file.flush()
+                os.fsync(self._file.fileno())  # 确保写入磁盘
+            except IOError as e:
+                self.logger.error(f"Failed to persist fingerprint: {str(e)}")
+
+    def __contains__(self, item: str) -> bool:
+        """
+        线程安全的指纹检查
+        :param item: 要检查的指纹
+        :return: 是否已存在
+        """
+        with self._lock:
+            return item in self.fingerprints
+
+    def close(self) -> None:
+        """安全关闭资源（同步方法）"""
+        with self._lock:
+            if self._file and not self._file.closed:
+                try:
+                    self._file.flush()
+                    os.fsync(self._file.fileno())
+                finally:
+                    self._file.close()
+                self.logger.info(f"Closed fingerprint file: {self._file.name}")
+
+    def __del__(self):
+        """析构函数双保险"""
+        self.close()
+
+    # 兼容异步接口
+    async def closed(self):
+        """标准的关闭入口"""
+        self.close()

crawlo/filters/redis_filter.py

@@ -0,0 +1,120 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import redis
+
+from crawlo import Request
+from crawlo.filters import BaseFilter
+from crawlo.utils.log import get_logger
+from crawlo.utils.request import request_fingerprint
+
+
+class RedisFilter(BaseFilter):
+    """使用Redis集合实现的同步请求去重过滤器"""
+
+    def __init__(
+            self,
+            redis_key: str,
+            client: redis.Redis,
+            stats: dict,
+            debug: bool,
+            log_level: str,
+            save_fp: bool
+    ):
+        """
+        初始化过滤器
+
+        :param redis_key: Redis存储键名
+        :param client: redis客户端实例
+        :param stats: 统计字典
+        :param debug: 是否启用调试模式
+        :param log_level: 日志级别
+        :param save_fp: 是否保留指纹数据
+        """
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        super().__init__(self.logger, stats, debug)
+
+        self.redis_key = redis_key
+        self.redis = client
+        self.save_fp = save_fp
+
+    @classmethod
+    def create_instance(cls, crawler) -> 'BaseFilter':
+        """工厂方法创建实例"""
+        redis_url = crawler.settings.get('REDIS_URL', 'redis://localhost:6379')
+        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', True)
+
+        try:
+            # 添加连接池配置
+            redis_client = redis.from_url(
+                redis_url,
+                decode_responses=decode_responses,
+                socket_timeout=5,  # 超时设置
+                socket_connect_timeout=5,
+                max_connections=20  # 连接池大小
+            )
+            # 测试连接是否有效
+            redis_client.ping()
+        except redis.RedisError as e:
+            raise RuntimeError(f"Redis连接失败: {str(e)}")
+
+        return cls(
+            redis_key=f"{crawler.settings.get('PROJECT_NAME')}:{crawler.settings.get('REDIS_KEY', 'request_fingerprints')}",
+            client=redis_client,
+            stats=crawler.stats,
+            save_fp=crawler.settings.get_bool('SAVE_FP', False),
+            debug=crawler.settings.get_bool('FILTER_DEBUG', False),
+            log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
+        )
+
+    def requested(self, request: Request) -> bool:
+        """
+        检查请求是否已存在
+
+        :param request: 请求对象
+        :return: 是否重复
+        """
+        fp = request_fingerprint(request)
+        try:
+            if self.redis.sismember(self.redis_key, fp):
+                self.logger.debug(f"重复请求: {fp}")
+                return True
+
+            self.add_fingerprint(fp)
+            return False
+        except redis.RedisError as e:
+            self.logger.error(f"Redis操作失败: {str(e)}")
+            raise
+
+    def add_fingerprint(self, fp: str) -> None:
+        """添加指纹到Redis集合"""
+        try:
+            self.redis.sadd(self.redis_key, fp)
+            self.logger.debug(f"新增指纹: {fp}")
+        except redis.RedisError as e:
+            self.logger.error(f"指纹添加失败: {str(e)}")
+            raise
+
+    def __contains__(self, item) -> bool:
+        """支持 in 操作符检查 (必须返回bool类型)"""
+        try:
+            # 显式将redis返回的0/1转换为bool
+            return bool(self.redis.sismember(self.redis_key, item))
+        except redis.RedisError as e:
+            self.logger.error(f"Redis查询失败: {str(e)}")
+            raise
+
+    def close(self) -> None:
+        """同步清理方法（注意不是异步的closed）"""
+        if not self.save_fp:
+            try:
+                count = self.redis.delete(self.redis_key)
+                self.logger.info(f"已清理Redis键 {self.redis_key}, 删除数量: {count}")
+            except redis.RedisError as e:
+                self.logger.error(f"清理失败: {str(e)}")
+            finally:
+                # 同步客户端需要手动关闭连接池
+                self.redis.close()
+
+    async def closed(self):
+        """兼容异步接口的同步实现"""
+        self.close()

crawlo/items/__init__.py

@@ -1,10 +1,48 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 from abc import ABCMeta
+from typing import Any, Optional, Type
 
 
-class Field(dict):
-    pass
+class Field:
+    def __init__(
+        self,
+        nullable: bool = True,
+        *,
+        default: Any = None,
+        field_type: Optional[Type] = None,
+        max_length: Optional[int] = None,
+        description: str = ""
+    ):
+        self.nullable = nullable
+        self.default = default
+        self.field_type = field_type
+        self.max_length = max_length
+        self.description = description
+
+    def validate(self, value: Any, field_name: str = "") -> Any:
+        if value is None or (isinstance(value, str) and value.strip() == ""):
+            if self.default is not None:
+                return self.default
+            elif not self.nullable:
+                raise ValueError(
+                    f"字段 '{field_name}' 不允许为空。"
+                )
+
+        if value is not None and not (isinstance(value, str) and value.strip() == ""):
+            if self.field_type and not isinstance(value, self.field_type):
+                raise TypeError(
+                    f"字段 '{field_name}' 类型错误：期望类型 {self.field_type}, 得到 {type(value)}，值：{value!r}"
+                )
+            if self.max_length and len(str(value)) > self.max_length:
+                raise ValueError(
+                    f"字段 '{field_name}' 长度超限：最大长度 {self.max_length}，当前长度 {len(str(value))}，值：{value!r}"
+                )
+
+        return value
+
+    def __repr__(self):
+        return f"<Field required={self.nullable} type={self.field_type} default={self.default}>"
 
 
 class ItemMeta(ABCMeta):

crawlo/items/items.py

@@ -21,13 +21,44 @@ class Item(MutableMapping, metaclass=ItemMeta):
 
         self._values: Dict[str, Any] = {}
 
+        # 初始化字段，默认值填充
+        for field_name, field_obj in self.FIELDS.items():
+            if field_obj.default is not None:
+                self._values[field_name] = field_obj.default
+
+        # 覆盖默认值或设置新值
+        for key, value in kwargs.items():
+            self[key] = value
+
     def __getitem__(self, item: str) -> Any:
         return self._values[item]
 
+    # def __setitem__(self, key: str, value: Any) -> None:
+    #     if key not in self.FIELDS:
+    #         raise KeyError(f"{self.__class__.__name__} 不包含字段：{key}")
+    #     self._values[key] = value
+
     def __setitem__(self, key: str, value: Any) -> None:
         if key not in self.FIELDS:
             raise KeyError(f"{self.__class__.__name__} 不包含字段：{key}")
-        self._values[key] = value
+
+        field = self.FIELDS[key]
+        try:
+            validated_value = field.validate(value, field_name=key)
+            self._values[key] = validated_value
+        except Exception as e:
+            error_lines = [
+                "",
+                "【字段校验失败】",
+                f"字段名称: {key}",
+                f"数据类型: {type(value)}",
+                f"原始值:   {repr(value)}",
+                f"是否允许空值: {field.nullable}",
+                f"错误原因: {str(e)}",
+                ""
+            ]
+            detailed_error = "\n".join(error_lines)
+            raise type(e)(detailed_error) from e
 
     def __delitem__(self, key: str) -> None:
         del self._values[key]
@@ -78,11 +109,11 @@ class Item(MutableMapping, metaclass=ItemMeta):
 
 if __name__ == '__main__':
     class TestItem(Item):
-        url = Field()
-        title = Field()
+        url = Field(nullable=False, field_type=str, max_length=100)
+        title = Field(default="无标题", field_type=str)
 
     test_item = TestItem()
     test_item['title'] = '百度首页'
-    test_item['url'] = 'http://example.com'
+    test_item['url'] = 'hhh'
     # test_item.title = 'fffff'
-    print(test_item.title)
+    print(test_item)

crawlo/middleware/retry.py

@@ -36,12 +36,14 @@ class RetryMiddleware(object):
             ignore_http_codes: List,
             max_retry_times: int,
             retry_exceptions: List,
-            stats: StatsCollector
+            stats: StatsCollector,
+            retry_priority: int
     ):
         self.retry_http_codes = retry_http_codes
         self.ignore_http_codes = ignore_http_codes
         self.max_retry_times = max_retry_times
         self.retry_exceptions = tuple(retry_exceptions + _retry_exceptions)
+        self.retry_priority = retry_priority
         self.stats = stats
         self.logger = get_logger(self.__class__.__name__)
 
@@ -52,7 +54,8 @@ class RetryMiddleware(object):
             ignore_http_codes=crawler.settings.get_list('IGNORE_HTTP_CODES'),
             max_retry_times=crawler.settings.get_int('MAX_RETRY_TIMES'),
             retry_exceptions=crawler.settings.get_list('RETRY_EXCEPTIONS'),
-            stats=crawler.stats
+            stats=crawler.stats,
+            retry_priority=crawler.settings.get_int('RETRY_PRIORITY')
         )
         return o
 
@@ -77,6 +80,8 @@ class RetryMiddleware(object):
             retry_times += 1
             self.logger.info(f"{spider} {request} {reason} retrying {retry_times} time...")
             request.meta['retry_times'] = retry_times
+            request.dont_retry = True
+            request.retry_priority = request.priority + self.retry_priority
             self.stats.inc_value("retry_count")
             return request
         else:

crawlo/network/request.py

@@ -2,7 +2,10 @@
 # -*- coding:UTF-8 -*-
 import hashlib
 from copy import deepcopy
-from typing import Dict, Optional, Callable
+from w3lib.url import safe_url_string
+from typing import Dict, Optional, Callable, Union, Any
+
+from crawlo.utils.url import escape_ajax
 
 
 class Request(object):
@@ -13,40 +16,140 @@ class Request(object):
             *,
             callback: Optional[Callable] = None,
             headers: Optional[Dict[str, str]] = None,
-            body: Optional[bytes] = None,
+            body: Optional[Union[Dict, bytes, str]] = None,
             method: Optional[str] = 'GET',
             cookies: Optional[Dict[str, str]] = None,
             priority: int = 0,
-            encoding: Optional[str] = 'UTF-8',
-            meta: Optional[Dict[str, str]] = None
-
+            encoding: Optional[str] = 'utf-8',
+            meta: Optional[Dict[str, Any]] = None,
+            dont_filter: bool = False,
+            timeout: Optional[float] = None,
+            proxy: Optional[str] = None,
+            allow_redirects: bool = True,
+            auth: Optional[tuple] = None,
+            verify: bool = True
     ):
-        self.url = url
+        # 先初始化基本属性
         self.callback = callback
         self.headers = headers if headers else {}
         self.body = body
-        self.method = method
-        self.cookies = cookies
-        self.priority = priority
+        self.method = str(method).upper()
+        self.cookies = cookies if cookies else {}
+        self.priority = -priority
         self.encoding = encoding
+        self.dont_filter = dont_filter
         self._meta = meta if meta is not None else {}
+        self.timeout = timeout
+        self.proxy = proxy
+        self.allow_redirects = allow_redirects
+        self.auth = auth
+        self.verify = verify
+
+        # 最后处理URL，确保encoding等依赖属性已初始化
+        self._set_url(url)
 
     def copy(self):
         return deepcopy(self)
 
-    def fingerprint(self) -> str:
-        data = f"{self.url}{self.method}{self.body or b''}".encode()
-        return hashlib.sha256(data).hexdigest()
-
-    def set_meta(self, key: str, value: str):
+    def set_meta(self, key: str, value: Any) -> None:
         self._meta[key] = value
 
+    def _set_url(self, url: str) -> None:
+        if not isinstance(url, str):
+            raise TypeError(f"Request url must be str, got {type(url).__name__}")
+
+        s = safe_url_string(url, self.encoding)
+        self._url = escape_ajax(s)
+
+        if (
+                "://" not in self._url
+                and not self._url.startswith("about:")
+                and not self._url.startswith("data:")
+        ):
+            raise ValueError(f"Missing scheme in request url: {self._url}")
+
+    @property
+    def url(self) -> str:
+        return self._url
+
     @property
-    def meta(self):
+    def meta(self) -> Dict[str, Any]:
         return self._meta
 
-    def __str__(self):
-        return f'<Request url={self.url}> method={self.method}>'
+    def __str__(self) -> str:
+        return f'<Request url={self.url} method={self.method}>'
 
-    def __lt__(self, other):
+    def __repr__(self) -> str:
+        return self.__str__()
+
+    def __lt__(self, other) -> bool:
         return self.priority < other.priority
+
+
+# #!/usr/bin/python
+# # -*- coding:UTF-8 -*-
+# import hashlib
+# from copy import deepcopy
+# from w3lib.url import safe_url_string
+# from typing import Dict, Optional, Callable, Union
+#
+# from crawlo.utils.url import escape_ajax
+#
+#
+# class Request(object):
+#
+#     def __init__(
+#             self,
+#             url: str,
+#             *,
+#             callback: Optional[Callable] = None,
+#             headers: Optional[Dict[str, str]] = None,
+#             body: Optional[Dict] = None,
+#             method: Optional[str] = 'GET',
+#             cookies: Optional[Dict[str, str]] = None,
+#             priority: int = 0,
+#             encoding: Optional[str] = 'UTF-8',
+#             meta: Optional[Dict[str, str]] = None,
+#             dont_filter: bool = False
+#
+#     ):
+#         self.url = url
+#         self.callback = callback
+#         self.headers = headers if headers else {}
+#         self.body = body
+#         self.method = str(method).upper()
+#         self.cookies = cookies
+#         self.priority = -priority
+#         self.encoding = encoding
+#         self.dont_filter = dont_filter
+#         self._meta = meta if meta is not None else {}
+#
+#     def copy(self):
+#         return deepcopy(self)
+#
+#     def set_meta(self, key: str, value: str):
+#         self._meta[key] = value
+#
+#     def _set_url(self, url: str) -> None:
+#         if not isinstance(url, str):
+#             raise TypeError(f"Request url must be str, got {type(url).__name__}")
+#
+#         s = safe_url_string(url, self.encoding)
+#         self._url = escape_ajax(s)
+#
+#         if (
+#             "://" not in self._url
+#             and not self._url.startswith("about:")
+#             and not self._url.startswith("data:")
+#         ):
+#             raise ValueError(f"Missing scheme in request url: {self._url}")
+#
+#     @property
+#     def meta(self):
+#         return self._meta
+#
+#     def __str__(self):
+#         return f'<Request url={self.url}> method={self.method}>'
+#
+#     def __lt__(self, other):
+#         return self.priority < other.priority

crawlo/pipelines/console_pipeline.py

@@ -1,20 +1,40 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+from typing import Dict, Any
+
 from crawlo import Item
 from crawlo.spider import Spider
 from crawlo.utils.log import get_logger
 
 
-class DebugPipeline:
+class ConsolePipeline:
+    """将Item内容输出到控制台的管道"""
 
-    def __init__(self, logger):
-        self.logger = logger
+    def __init__(self, log_level: str = "DEBUG"):
+        self.logger = get_logger(self.__class__.__name__, log_level)
 
     @classmethod
-    def create_instance(cls, crawler):
-        logger = get_logger(cls.__name__, crawler.settings.get('LOG_LEVEL'))
-        return cls(logger)
+    def from_crawler(cls, crawler):
+        """从crawler实例创建管道"""
+        return cls(
+            log_level=crawler.settings.get('LOG_LEVEL', 'DEBUG')
+        )
 
     async def process_item(self, item: Item, spider: Spider) -> Item:
-        self.logger.debug(item.to_dict())
-        return item
+        """处理Item并输出到日志"""
+        try:
+            item_dict = self._convert_to_serializable(item)
+            self.logger.info(f"Item processed: {item_dict}")
+            return item
+        except Exception as e:
+            self.logger.error(f"Error processing item: {e}", exc_info=True)
+            raise
+
+    @staticmethod
+    def _convert_to_serializable(item: Item) -> Dict[str, Any]:
+        """将Item转换为可序列化的字典"""
+        try:
+            return item.to_dict()
+        except AttributeError:
+            # 兼容没有to_dict方法的Item实现
+            return dict(item)