crawlo 1.4.5__py3-none-any.whl → 1.4.7__py3-none-any.whl

crawlo/event.py

@@ -1,11 +1,45 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-
-spider_error = "spider_error"
-spider_opened = "spider_open"
-spider_closed = "spider_closed"
-ignore_request = "ignore_request"
-request_scheduled = "request_scheduled"
-response_received = "request_received"
-item_successful = "item_successful"
-item_discard = "item_discard"
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Crawlo 事件系统
+==============
+定义框架中的所有事件类型，支持类型安全和IDE自动补全。
+"""
+from enum import Enum
+
+
+class CrawlerEvent(str, Enum):
+    """
+    爬虫事件枚举
+    
+    所有事件都应该使用此枚举类型，以获得：
+    - 类型安全：避免拼写错误
+    - IDE支持：自动补全和提示
+    - 文档化：集中管理所有事件
+    
+    使用示例：
+        >>> from crawlo.event import CrawlerEvent
+        >>> await subscriber.notify(CrawlerEvent.SPIDER_OPENED, spider)
+    """
+    
+    # 爬虫生命周期事件
+    SPIDER_OPENED = "spider_opened"      # 爬虫启动
+    SPIDER_CLOSED = "spider_closed"      # 爬虫关闭
+    SPIDER_ERROR = "spider_error"        # 爬虫错误
+    
+    # 请求相关事件
+    REQUEST_SCHEDULED = "request_scheduled"  # 请求已调度
+    IGNORE_REQUEST = "ignore_request"        # 请求被忽略
+    
+    # 响应相关事件
+    RESPONSE_RECEIVED = "response_received"  # 响应已接收
+    
+    # Item相关事件
+    ITEM_SUCCESSFUL = "item_successful"  # Item处理成功
+    ITEM_DISCARD = "item_discard"        # Item被丢弃
+
+
+# 导出所有公共API
+__all__ = [
+    'CrawlerEvent',
+]

crawlo/exceptions.py

@@ -1,82 +1,215 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-class TransformTypeError(TypeError):
-    pass
-
-
-class OutputError(Exception):
-    pass
-
-
-class SpiderTypeError(TypeError):
-    pass
-
-
-class ItemInitError(Exception):
-    pass
-
-
-class ItemAttributeError(Exception):
-    pass
-
-
-class DecodeError(Exception):
-    pass
-
-
-class MiddlewareInitError(Exception):
-    pass
-
-
-class PipelineInitError(Exception):
-    pass
-
-
-class InvalidOutputError(Exception):
-    pass
-
-
-class RequestMethodError(Exception):
-    pass
-
-
-class IgnoreRequestError(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-        super(IgnoreRequestError, self).__init__(msg)
-
-
-class ItemDiscard(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-        super(ItemDiscard, self).__init__(msg)
-
-
-class NotConfigured(Exception):
-    pass
-
-
-class NotConfiguredError(Exception):
-    pass
-
-
-class ExtensionInitError(Exception):
-    pass
-
-
-class ReceiverTypeError(Exception):
-    pass
-
-
-class SpiderCreationError(Exception):
-    """爬虫实例化失败异常"""
-    pass
-
-
-class ItemValidationError(Exception):
-    """Item 字段验证错误"""
-    pass
-
-
-class DropItem(Exception):
-    pass
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Crawlo 框架异常定义
+===================
+提供层次化的异常体系，便于统一处理和类型安全。
+
+异常层次：
+    CrawloException (基础异常)
+    ├── SpiderException (爬虫相关)
+    ├── ComponentInitException (组件初始化)
+    ├── DataException (数据处理)
+    ├── RequestException (请求/响应)
+    ├── OutputException (输出)
+    └── ConfigException (配置)
+
+使用示例：
+    >>> try:
+    ...     # 代码
+    ... except CrawloException as e:
+    ...     # 捕获所有框架异常
+    ... except Exception as e:
+    ...     # 其他异常
+"""
+
+
+# ============= 基础异常 =============
+class CrawloException(Exception):
+    """Crawlo框架基础异常。所有框架异常都应继承此类。"""
+    pass
+
+
+# ============= 爬虫相关异常 =============
+class SpiderException(CrawloException):
+    """爬虫相关异常基类。"""
+    pass
+
+
+class SpiderTypeError(SpiderException, TypeError):
+    """爬虫类型错误。当爬虫类型不符合预期时抛出。"""
+    pass
+
+
+class SpiderCreationError(SpiderException):
+    """爬虫实例化失败异常。当无法创建爬虫实例时抛出。"""
+    pass
+
+
+# ============= 组件初始化异常 =============
+class ComponentInitException(CrawloException):
+    """组件初始化异常基类。"""
+    pass
+
+
+class MiddlewareInitError(ComponentInitException):
+    """中间件初始化失败异常。"""
+    pass
+
+
+class PipelineInitError(ComponentInitException):
+    """管道初始化失败异常。"""
+    pass
+
+
+class ExtensionInitError(ComponentInitException):
+    """扩展初始化失败异常。"""
+    pass
+
+
+# ============= 数据处理异常 =============
+class DataException(CrawloException):
+    """数据处理异常基类。"""
+    pass
+
+
+class ItemInitError(DataException):
+    """Item初始化错误。当Item实例创建失败时抛出。"""
+    pass
+
+
+class ItemAttributeError(DataException, AttributeError):
+    """Item属性错误。当访问不存在的Item属性时抛出。"""
+    pass
+
+
+class ItemValidationError(DataException):
+    """Item字段验证错误。当Item字段值不符合验证规则时抛出。"""
+    pass
+
+
+class ItemDiscard(DataException):
+    """
+    Item被丢弃异常。
+    
+    注意：这不是一个真正的错误，而是用于流程控制，
+    表示Item应该被管道丢弃（例如重复数据）。
+    """
+    def __init__(self, msg):
+        self.msg = msg
+        super().__init__(msg)
+
+
+# ============= 请求/响应异常 =============
+class RequestException(CrawloException):
+    """请求异常基类。"""
+    pass
+
+
+class RequestMethodError(RequestException):
+    """请求方法错误。当使用不支持的HTTP方法时抛出。"""
+    pass
+
+
+class IgnoreRequestError(RequestException):
+    """
+    请求被忽略异常。
+    
+    用于流程控制，表示请求应该被跳过处理。
+    """
+    def __init__(self, msg):
+        self.msg = msg
+        super().__init__(msg)
+
+
+class DecodeError(RequestException):
+    """响应解码错误。当无法解码响应内容时抛出。"""
+    pass
+
+
+# ============= 输出异常 =============
+class OutputException(CrawloException):
+    """输出异常基类。"""
+    pass
+
+
+class OutputError(OutputException):
+    """输出错误。当输出处理失败时抛出。"""
+    pass
+
+
+class InvalidOutputError(OutputException):
+    """无效的输出错误。当输出类型或格式不符合预期时抛出。"""
+    pass
+
+
+# ============= 配置异常 =============
+class ConfigException(CrawloException):
+    """配置异常基类。"""
+    pass
+
+
+class NotConfigured(ConfigException):
+    """组件未配置异常。当必需的配置缺失时抛出。"""
+    pass
+
+
+class NotConfiguredError(ConfigException):
+    """配置错误异常。当配置值无效时抛出。"""
+    pass
+
+
+# ============= 类型异常 =============
+class TransformTypeError(CrawloException, TypeError):
+    """转换类型错误。当数据转换类型不匹配时抛出。"""
+    pass
+
+
+class ReceiverTypeError(CrawloException, TypeError):
+    """接收者类型错误。当事件接收者类型不符合预期时抛出。"""
+    pass
+
+
+# ============= 导出所有异常 =============
+__all__ = [
+    # 基础异常
+    'CrawloException',
+    
+    # 爬虫相关
+    'SpiderException',
+    'SpiderTypeError',
+    'SpiderCreationError',
+    
+    # 组件初始化
+    'ComponentInitException',
+    'MiddlewareInitError',
+    'PipelineInitError',
+    'ExtensionInitError',
+    
+    # 数据处理
+    'DataException',
+    'ItemInitError',
+    'ItemAttributeError',
+    'ItemValidationError',
+    'ItemDiscard',
+    
+    # 请求/响应
+    'RequestException',
+    'RequestMethodError',
+    'IgnoreRequestError',
+    'DecodeError',
+    
+    # 输出
+    'OutputException',
+    'OutputError',
+    'InvalidOutputError',
+    
+    # 配置
+    'ConfigException',
+    'NotConfigured',
+    'NotConfiguredError',
+    
+    # 类型
+    'TransformTypeError',
+    'ReceiverTypeError',
+]

crawlo/extension/__init__.py

@@ -1,64 +1,65 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import List, Any
-from pprint import pformat
-
-from crawlo.utils.log import get_logger
-from crawlo.utils.misc import load_object
-from crawlo.exceptions import ExtensionInitError
-
-
-class ExtensionManager(object):
-
-    def __init__(self, crawler: Any):
-        self.crawler = crawler
-        self.extensions: List = []
-        extensions = self.crawler.settings.get_list('EXTENSIONS')
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        self._add_extensions(extensions)
-        self._subscribe_extensions()
-
-    @classmethod
-    def create_instance(cls, *args: Any, **kwargs: Any) -> 'ExtensionManager':
-        return cls(*args, **kwargs)
-
-    def _add_extensions(self, extensions: List[str]) -> None:
-        for extension_path in extensions:
-            try:
-                extension_cls = load_object(extension_path)
-                if not hasattr(extension_cls, 'create_instance'):
-                    raise ExtensionInitError(
-                        f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"
-                    )
-                self.extensions.append(extension_cls.create_instance(self.crawler))
-            except Exception as e:
-                self.logger.error(f"Failed to load extension '{extension_path}': {e}")
-                raise ExtensionInitError(f"Failed to load extension '{extension_path}': {e}")
-        
-        if extensions:
-            # 恢复INFO级别日志，保留关键的启用信息
-            self.logger.info(f"Enabled extensions: \n{pformat(extensions)}")
-
-    def _subscribe_extensions(self) -> None:
-        """订阅扩展方法到相应的事件"""
-        for extension in self.extensions:
-            # 订阅 spider_closed 方法
-            if hasattr(extension, 'spider_closed'):
-                self.crawler.subscriber.subscribe(extension.spider_closed, event="spider_closed")
-            
-            # 订阅 item_successful 方法
-            if hasattr(extension, 'item_successful'):
-                self.crawler.subscriber.subscribe(extension.item_successful, event="item_successful")
-            
-            # 订阅 item_discard 方法
-            if hasattr(extension, 'item_discard'):
-                self.crawler.subscriber.subscribe(extension.item_discard, event="item_discard")
-            
-            # 订阅 response_received 方法
-            if hasattr(extension, 'response_received'):
-                # 修复：将事件名称从 "request_received" 更正为 "response_received"
-                self.crawler.subscriber.subscribe(extension.response_received, event="response_received")
-            
-            # 订阅 request_scheduled 方法
-            if hasattr(extension, 'request_scheduled'):
-                self.crawler.subscriber.subscribe(extension.request_scheduled, event="request_scheduled")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List, Any
+from pprint import pformat
+
+from crawlo.logging import get_logger
+from crawlo.utils.misc import load_object
+from crawlo.exceptions import ExtensionInitError
+
+
+class ExtensionManager:
+
+    def __init__(self, crawler: Any):
+        self.crawler = crawler
+        self.extensions: List = []
+        extensions = self.crawler.settings.get_list('EXTENSIONS')
+        self.logger = get_logger(self.__class__.__name__)
+        self._add_extensions(extensions)
+        self._subscribe_extensions()
+
+    @classmethod
+    def create_instance(cls, *args: Any, **kwargs: Any) -> 'ExtensionManager':
+        return cls(*args, **kwargs)
+
+    def _add_extensions(self, extensions: List[str]) -> None:
+        for extension_path in extensions:
+            try:
+                extension_cls = load_object(extension_path)
+                if not hasattr(extension_cls, 'create_instance'):
+                    raise ExtensionInitError(
+                        f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"
+                    )
+                self.extensions.append(extension_cls.create_instance(self.crawler))
+            except Exception as e:
+                self.logger.error(f"Failed to load extension '{extension_path}': {e}")
+                raise ExtensionInitError(f"Failed to load extension '{extension_path}': {e}")
+        
+        if extensions:
+            # 恢复INFO级别日志，保留关键的启用信息
+            self.logger.info(f"Enabled extensions: \n{pformat(extensions)}")
+
+    def _subscribe_extensions(self) -> None:
+        """订阅扩展方法到相应的事件"""
+        from crawlo.event import CrawlerEvent
+        
+        for extension in self.extensions:
+            # 订阅 spider_closed 方法
+            if hasattr(extension, 'spider_closed'):
+                self.crawler.subscriber.subscribe(extension.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
+            
+            # 订阅 item_successful 方法
+            if hasattr(extension, 'item_successful'):
+                self.crawler.subscriber.subscribe(extension.item_successful, event=CrawlerEvent.ITEM_SUCCESSFUL)
+            
+            # 订阅 item_discard 方法
+            if hasattr(extension, 'item_discard'):
+                self.crawler.subscriber.subscribe(extension.item_discard, event=CrawlerEvent.ITEM_DISCARD)
+            
+            # 订阅 response_received 方法
+            if hasattr(extension, 'response_received'):
+                self.crawler.subscriber.subscribe(extension.response_received, event=CrawlerEvent.RESPONSE_RECEIVED)
+            
+            # 订阅 request_scheduled 方法
+            if hasattr(extension, 'request_scheduled'):
+                self.crawler.subscriber.subscribe(extension.request_scheduled, event=CrawlerEvent.REQUEST_SCHEDULED)