crawlo 1.1.0__py3-none-any.whl → 1.1.2__py3-none-any.whl

crawlo/exceptions.py

@@ -1,78 +1,78 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-class TransformTypeError(TypeError):
-    pass
-
-
-class OutputError(Exception):
-    pass
-
-
-class SpiderTypeError(TypeError):
-    pass
-
-
-class ItemInitError(Exception):
-    pass
-
-
-class ItemAttributeError(Exception):
-    pass
-
-
-class DecodeError(Exception):
-    pass
-
-
-class MiddlewareInitError(Exception):
-    pass
-
-
-class PipelineInitError(Exception):
-    pass
-
-
-class InvalidOutputError(Exception):
-    pass
-
-
-class RequestMethodError(Exception):
-    pass
-
-
-class IgnoreRequestError(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-        super(IgnoreRequestError, self).__init__(msg)
-
-
-class ItemDiscard(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-        super(ItemDiscard, self).__init__(msg)
-
-
-class NotConfigured(Exception):
-    pass
-
-
-class NotConfiguredError(Exception):
-    pass
-
-
-class ExtensionInitError(Exception):
-    pass
-
-
-class ReceiverTypeError(Exception):
-    pass
-
-
-class SpiderCreationError(Exception):
-    """爬虫实例化失败异常"""
-    pass
-
-
-class ItemValidationError(Exception):
-    """Item 字段验证错误"""
-    pass
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+class TransformTypeError(TypeError):
+    pass
+
+
+class OutputError(Exception):
+    pass
+
+
+class SpiderTypeError(TypeError):
+    pass
+
+
+class ItemInitError(Exception):
+    pass
+
+
+class ItemAttributeError(Exception):
+    pass
+
+
+class DecodeError(Exception):
+    pass
+
+
+class MiddlewareInitError(Exception):
+    pass
+
+
+class PipelineInitError(Exception):
+    pass
+
+
+class InvalidOutputError(Exception):
+    pass
+
+
+class RequestMethodError(Exception):
+    pass
+
+
+class IgnoreRequestError(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+        super(IgnoreRequestError, self).__init__(msg)
+
+
+class ItemDiscard(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+        super(ItemDiscard, self).__init__(msg)
+
+
+class NotConfigured(Exception):
+    pass
+
+
+class NotConfiguredError(Exception):
+    pass
+
+
+class ExtensionInitError(Exception):
+    pass
+
+
+class ReceiverTypeError(Exception):
+    pass
+
+
+class SpiderCreationError(Exception):
+    """爬虫实例化失败异常"""
+    pass
+
+
+class ItemValidationError(Exception):
+    """Item 字段验证错误"""
+    pass

crawlo/extension/__init__.py

@@ -1,31 +1,31 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import List
-from pprint import pformat
-
-from crawlo.utils.log import get_logger
-from crawlo.utils.project import load_class
-from crawlo.exceptions import ExtensionInitError
-
-
-class ExtensionManager(object):
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.extensions: List = []
-        extensions = self.crawler.settings.get_list('EXTENSIONS')
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        self._add_extensions(extensions)
-
-    @classmethod
-    def create_instance(cls, *args, **kwargs):
-        return cls(*args, **kwargs)
-
-    def _add_extensions(self, extensions):
-        for extension in extensions:
-            extension_cls = load_class(extension)
-            if not hasattr(extension_cls, 'create_instance'):
-                raise ExtensionInitError(f"extension init failed, Must have method 'create_instance()")
-            self.extensions.append(extension_cls.create_instance(self.crawler))
-        if extensions:
-            self.logger.info(f"enabled extensions: \n {pformat(extensions)}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List
+from pprint import pformat
+
+from crawlo.utils.log import get_logger
+from crawlo.project import load_class
+from crawlo.exceptions import ExtensionInitError
+
+
+class ExtensionManager(object):
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.extensions: List = []
+        extensions = self.crawler.settings.get_list('EXTENSIONS')
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self._add_extensions(extensions)
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs):
+        return cls(*args, **kwargs)
+
+    def _add_extensions(self, extensions):
+        for extension in extensions:
+            extension_cls = load_class(extension)
+            if not hasattr(extension_cls, 'create_instance'):
+                raise ExtensionInitError(f"extension init failed, Must have method 'create_instance()")
+            self.extensions.append(extension_cls.create_instance(self.crawler))
+        if extensions:
+            self.logger.info(f"enabled extensions: \n {pformat(extensions)}")

crawlo/extension/log_interval.py

@@ -1,49 +1,49 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-
-from crawlo.utils.log import get_logger
-from crawlo.event import spider_opened, spider_closed
-
-
-class LogIntervalExtension(object):
-
-    def __init__(self, crawler):
-        self.task = None
-        self.stats = crawler.stats
-        self.item_count = 0
-        self.response_count = 0
-        self.seconds = crawler.settings.get('INTERVAL')
-        self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
-        self.interval = "" if self.interval == 1 else self.interval
-        self.unit = 'min' if self.seconds % 60 == 0 else 's'
-
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(crawler)
-        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-        return o
-
-    async def spider_opened(self):
-        self.task = asyncio.create_task(self.interval_log())
-        await self.task
-
-    async def spider_closed(self):
-        if self.task:
-            self.task.cancel()
-
-    async def interval_log(self):
-        while True:
-            last_item_count = self.stats.get_value('item_successful_count', default=0)
-            last_response_count = self.stats.get_value('response_received_count', default=0)
-            item_rate = last_item_count - self.item_count
-            response_rate = last_response_count - self.response_count
-            self.item_count, self.response_count = last_item_count, last_response_count
-            self.logger.info(
-                f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
-                f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
-            )
-            await asyncio.sleep(self.seconds)
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed
+
+
+class LogIntervalExtension(object):
+
+    def __init__(self, crawler):
+        self.task = None
+        self.stats = crawler.stats
+        self.item_count = 0
+        self.response_count = 0
+        self.seconds = crawler.settings.get('INTERVAL')
+        self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
+        self.interval = "" if self.interval == 1 else self.interval
+        self.unit = 'min' if self.seconds % 60 == 0 else 's'
+
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self):
+        self.task = asyncio.create_task(self.interval_log())
+        await self.task
+
+    async def spider_closed(self):
+        if self.task:
+            self.task.cancel()
+
+    async def interval_log(self):
+        while True:
+            last_item_count = self.stats.get_value('item_successful_count', default=0)
+            last_response_count = self.stats.get_value('response_received_count', default=0)
+            item_rate = last_item_count - self.item_count
+            response_rate = last_response_count - self.response_count
+            self.item_count, self.response_count = last_item_count, last_response_count
+            self.logger.info(
+                f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
+                f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
+            )
+            await asyncio.sleep(self.seconds)

crawlo/extension/log_stats.py

@@ -1,44 +1,44 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo import event
-from crawlo.utils.date_tools import now, time_diff
-
-
-class LogStats(object):
-
-    def __init__(self, stats):
-        self._stats = stats
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(crawler.stats)
-        crawler.subscriber.subscribe(o.spider_opened, event=event.spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
-        crawler.subscriber.subscribe(o.item_successful, event=event.item_successful)
-        crawler.subscriber.subscribe(o.item_discard, event=event.item_discard)
-        crawler.subscriber.subscribe(o.response_received, event=event.response_received)
-        crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
-
-        return o
-
-    async def spider_opened(self):
-        self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-
-    async def spider_closed(self):
-        self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-        self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
-
-    async def item_successful(self, _item, _spider):
-        self._stats.inc_value('item_successful_count')
-
-    async def item_discard(self, _item, exc, _spider):
-        self._stats.inc_value('item_discard_count')
-        reason = exc.msg
-        if reason:
-            self._stats.inc_value(f"item_discard/{reason}")
-
-    async def response_received(self, _response, _spider):
-        self._stats.inc_value('response_received_count')
-
-    async def request_scheduled(self, _request, _spider):
-        self._stats.inc_value('request_scheduler_count')
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo import event
+from crawlo.utils.date_tools import now, time_diff
+
+
+class LogStats(object):
+
+    def __init__(self, stats):
+        self._stats = stats
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(crawler.stats)
+        crawler.subscriber.subscribe(o.spider_opened, event=event.spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
+        crawler.subscriber.subscribe(o.item_successful, event=event.item_successful)
+        crawler.subscriber.subscribe(o.item_discard, event=event.item_discard)
+        crawler.subscriber.subscribe(o.response_received, event=event.response_received)
+        crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
+
+        return o
+
+    async def spider_opened(self):
+        self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+
+    async def spider_closed(self):
+        self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+        self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+
+    async def item_successful(self, _item, _spider):
+        self._stats.inc_value('item_successful_count')
+
+    async def item_discard(self, _item, exc, _spider):
+        self._stats.inc_value('item_discard_count')
+        reason = exc.msg
+        if reason:
+            self._stats.inc_value(f"item_discard/{reason}")
+
+    async def response_received(self, _response, _spider):
+        self._stats.inc_value('response_received_count')
+
+    async def request_scheduled(self, _request, _spider):
+        self._stats.inc_value('request_scheduler_count')

crawlo/extension/logging_extension.py

@@ -1,35 +1,35 @@
-from crawlo.exceptions import NotConfigured
-from crawlo.utils.log import get_logger
-from crawlo.utils.log import LoggerManager
-
-
-class CustomLoggerExtension:
-    """
-    日志系统初始化扩展
-    遵循与 ExtensionManager 一致的接口规范：使用 create_instance
-    """
-
-    def __init__(self, settings):
-        self.settings = settings
-        # 初始化全局日志配置
-        LoggerManager.configure(settings)
-
-    @classmethod
-    def create_instance(cls, crawler, *args, **kwargs):
-        """
-        工厂方法：兼容 ExtensionManager 的创建方式
-        被 ExtensionManager 调用
-        """
-        # 可以通过 settings 控制是否启用
-        if not crawler.settings.get('LOG_FILE') and not crawler.settings.get('LOG_ENABLE_CUSTOM'):
-            raise NotConfigured("CustomLoggerExtension: LOG_FILE not set and LOG_ENABLE_CUSTOM=False")
-
-        return cls(crawler.settings)
-
-    def spider_opened(self, spider):
-        logger = get_logger(__name__)
-        logger.info(
-            f"CustomLoggerExtension: Logging initialized. "
-            f"LOG_FILE={self.settings.get('LOG_FILE')}, "
-            f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"
+from crawlo.exceptions import NotConfigured
+from crawlo.utils.log import get_logger
+from crawlo.utils.log import LoggerManager
+
+
+class CustomLoggerExtension:
+    """
+    日志系统初始化扩展
+    遵循与 ExtensionManager 一致的接口规范：使用 create_instance
+    """
+
+    def __init__(self, settings):
+        self.settings = settings
+        # 初始化全局日志配置
+        LoggerManager.configure(settings)
+
+    @classmethod
+    def create_instance(cls, crawler, *args, **kwargs):
+        """
+        工厂方法：兼容 ExtensionManager 的创建方式
+        被 ExtensionManager 调用
+        """
+        # 可以通过 settings 控制是否启用
+        if not crawler.settings.get('LOG_FILE') and not crawler.settings.get('LOG_ENABLE_CUSTOM'):
+            raise NotConfigured("CustomLoggerExtension: LOG_FILE not set and LOG_ENABLE_CUSTOM=False")
+
+        return cls(crawler.settings)
+
+    def spider_opened(self, spider):
+        logger = get_logger(__name__)
+        logger.info(
+            f"CustomLoggerExtension: Logging initialized. "
+            f"LOG_FILE={self.settings.get('LOG_FILE')}, "
+            f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"
         )

crawlo/filters/__init__.py

@@ -1,37 +1,154 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from abc import ABC, abstractmethod
-
-from crawlo import Request
-from crawlo.utils.request import request_fingerprint
-
-
-class BaseFilter(ABC):
-
-    def __init__(self, logger, stats, debug: bool):
-        self.logger = logger
-        self.stats = stats
-        self.debug = debug
-
-    @classmethod
-    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
-        return cls(*args, **kwargs)
-
-    def requested(self, request: Request):
-        fp = request_fingerprint(request)
-        if fp in self:
-            return True
-        self.add_fingerprint(fp)
-        return False
-
-    @abstractmethod
-    def add_fingerprint(self, fp) -> None:
-        pass
-
-    def log_stats(self, request: Request) -> None:
-        if self.debug:
-            self.logger.debug(f'Filtered duplicate request: {request}')
-        self.stats.inc_value(f'{self}/filtered_count')
-
-    def __str__(self) -> str:
-        return f'{self.__class__.__name__}'
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Crawlo Filters Module
+====================
+提供多种请求去重过滤器实现。
+
+过滤器类型:
+- MemoryFilter: 基于内存的高效去重，适合单机模式
+- AioRedisFilter: 基于Redis的分布式去重，适合分布式模式
+- MemoryFileFilter: 内存+文件持久化，适合需要重启恢复的场景
+
+核心接口:
+- BaseFilter: 所有过滤器的基类
+- requested(): 检查请求是否重复的主要方法
+"""
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from crawlo.utils.request import request_fingerprint
+
+
+class BaseFilter(ABC):
+    """
+    请求去重过滤器基类
+    
+    提供统一的去重接口和统计功能。
+    所有过滤器实现都应该继承此类。
+    """
+
+    def __init__(self, logger, stats, debug: bool = False):
+        """
+        初始化过滤器
+        
+        :param logger: 日志器实例
+        :param stats: 统计信息存储
+        :param debug: 是否启用调试日志
+        """
+        self.logger = logger
+        self.stats = stats
+        self.debug = debug
+        self._request_count = 0
+        self._duplicate_count = 0
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
+        return cls(*args, **kwargs)
+
+    def requested(self, request) -> bool:
+        """
+        检查请求是否重复（主要接口）
+        
+        :param request: 请求对象
+        :return: True 表示重复，False 表示新请求
+        """
+        self._request_count += 1
+        fp = request_fingerprint(request)
+        
+        if fp in self:
+            self._duplicate_count += 1
+            self.log_stats(request)
+            return True
+            
+        self.add_fingerprint(fp)
+        return False
+
+    @abstractmethod
+    def add_fingerprint(self, fp: str) -> None:
+        """
+        添加请求指纹（子类必须实现）
+        
+        :param fp: 请求指纹字符串
+        """
+        pass
+    
+    @abstractmethod
+    def __contains__(self, item: str) -> bool:
+        """
+        检查指纹是否存在（支持 in 操作符）
+        
+        :param item: 要检查的指纹
+        :return: 是否已存在
+        """
+        pass
+
+    def log_stats(self, request) -> None:
+        """
+        记录统计信息
+        
+        :param request: 重复的请求对象
+        """
+        if self.debug:
+            self.logger.debug(f'Filtered duplicate request: {request}')
+        self.stats.inc_value(f'{self}/filtered_count')
+    
+    def get_stats(self) -> dict:
+        """
+        获取过滤器统计信息
+        
+        :return: 统计信息字典
+        """
+        return {
+            'total_requests': self._request_count,
+            'duplicate_requests': self._duplicate_count,
+            'unique_requests': self._request_count - self._duplicate_count,
+            'duplicate_rate': f"{self._duplicate_count / max(1, self._request_count) * 100:.2f}%"
+        }
+    
+    def reset_stats(self) -> None:
+        """重置统计信息"""
+        self._request_count = 0
+        self._duplicate_count = 0
+    
+    def close(self) -> None:
+        """关闭过滤器并清理资源"""
+        pass
+
+    def __str__(self) -> str:
+        return f'{self.__class__.__name__}'
+
+
+# 导出所有可用的过滤器
+__all__ = ['BaseFilter']
+
+# 动态导入具体实现
+try:
+    from .memory_filter import MemoryFilter, MemoryFileFilter
+    __all__.extend(['MemoryFilter', 'MemoryFileFilter'])
+except ImportError:
+    MemoryFilter = None
+    MemoryFileFilter = None
+
+try:
+    from .aioredis_filter import AioRedisFilter
+    __all__.append('AioRedisFilter')
+except ImportError:
+    AioRedisFilter = None
+
+# 提供便捷的过滤器映射
+FILTER_MAP = {
+    'memory': MemoryFilter,
+    'memory_file': MemoryFileFilter,
+    'redis': AioRedisFilter,
+    'aioredis': AioRedisFilter,  # 别名
+}
+
+# 过滤掉不可用的过滤器
+FILTER_MAP = {k: v for k, v in FILTER_MAP.items() if v is not None}
+
+def get_filter_class(name: str):
+    """根据名称获取过滤器类"""
+    if name in FILTER_MAP:
+        return FILTER_MAP[name]
+    raise ValueError(f"未知的过滤器类型: {name}。可用类型: {list(FILTER_MAP.keys())}")