crawlo 1.4.6__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/downloader/cffi_downloader.py

@@ -9,7 +9,7 @@ from curl_cffi.requests import AsyncSession
 
 from crawlo.network.response import Response
 from crawlo.downloader import DownloaderBase
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
 class CurlCffiDownloader(DownloaderBase):
     """
@@ -25,7 +25,7 @@ class CurlCffiDownloader(DownloaderBase):
         # 调用父类初始化方法，确保 _closed 等属性被正确初始化
         super().__init__(crawler)
         
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
         self._active_requests = set()
 
         # --- 基础配置 ---
@@ -237,6 +237,9 @@ class CurlCffiDownloader(DownloaderBase):
                 self.logger.warning(f"关闭 curl-cffi 会话时出错: {e}")
             finally:
                 self.session = None
+                # 清空活跃请求跟踪
+                self._active_requests.clear()
+        
         self.logger.debug("CurlCffiDownloader 已关闭")
 
     def idle(self) -> bool:

crawlo/downloader/httpx_downloader.py

@@ -6,7 +6,7 @@ from httpx import AsyncClient, Timeout, Limits
 
 from crawlo.network.response import Response
 from crawlo.downloader import DownloaderBase
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
 # 尝试导入 httpx 异常，用于更精确地捕获
 try:
@@ -48,7 +48,7 @@ class HttpXDownloader(DownloaderBase):
         self._timeout: Optional[Timeout] = None
         self._limits: Optional[Limits] = None
         # --- 获取 logger 实例 ---
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get("LOG_LEVEL"))
+        self.logger = get_logger(self.__class__.__name__)
 
     def open(self):
         super().open()
@@ -255,5 +255,11 @@ class HttpXDownloader(DownloaderBase):
         """关闭主客户端"""
         if self._client:
             self.logger.info("Closing HttpXDownloader client...")
-            await self._client.aclose()
+            try:
+                await self._client.aclose()
+            except Exception as e:
+                self.logger.warning(f"Error during client close: {e}")
+            finally:
+                self._client = None
+        
         self.logger.debug("HttpXDownloader closed.")

crawlo/downloader/hybrid_downloader.py

@@ -23,7 +23,7 @@ from urllib.parse import urlparse
 from crawlo.downloader import DownloaderBase
 from crawlo.network.request import Request
 from crawlo.network.response import Response
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
 # 动态导入下载器（避免循环导入）
 try:
@@ -59,7 +59,7 @@ class HybridDownloader(DownloaderBase):
 
     def __init__(self, crawler):
         super().__init__(crawler)
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get("LOG_LEVEL"))
+        self.logger = get_logger(self.__class__.__name__)
         
         # 下载器实例缓存
         self._downloaders: Dict[str, DownloaderBase] = {}

crawlo/downloader/playwright_downloader.py

@@ -23,7 +23,7 @@ from playwright.async_api import async_playwright, Playwright, Browser, Page, Br
 
 from crawlo.downloader import DownloaderBase
 from crawlo.network.response import Response
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
 
 class PlaywrightDownloader(DownloaderBase):
@@ -37,7 +37,7 @@ class PlaywrightDownloader(DownloaderBase):
         self.playwright: Optional[Playwright] = None
         self.browser: Optional[Browser] = None
         self.context: Optional[BrowserContext] = None
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get("LOG_LEVEL"))
+        self.logger = get_logger(self.__class__.__name__)
         self.default_timeout = crawler.settings.get_int("PLAYWRIGHT_TIMEOUT", 30000)  # 毫秒
         self.load_timeout = crawler.settings.get_int("PLAYWRIGHT_LOAD_TIMEOUT", 10000)  # 毫秒
         self.browser_type = crawler.settings.get("PLAYWRIGHT_BROWSER_TYPE", "chromium").lower()
@@ -319,25 +319,48 @@ class PlaywrightDownloader(DownloaderBase):
         """关闭 Playwright 资源"""
         try:
             # 关闭所有页面
-            for page in self._page_pool:
-                try:
-                    await page.close()
-                except:
-                    pass
-            self._page_pool.clear()
-            self._used_pages.clear()
+            if self._page_pool:
+                self.logger.debug(f"Closing {len(self._page_pool)} page(s)...")
+                for page in self._page_pool:
+                    try:
+                        await page.close()
+                    except Exception as e:
+                        self.logger.warning(f"Error closing page: {e}")
+                
+                self._page_pool.clear()
+                self._used_pages.clear()
             
+            # 关闭上下文
             if self.context:
-                await self.context.close()
+                try:
+                    await self.context.close()
+                except Exception as e:
+                    self.logger.warning(f"Error closing context: {e}")
+                finally:
+                    self.context = None
+            
+            # 关闭浏览器
             if self.browser:
-                await self.browser.close()
+                try:
+                    await self.browser.close()
+                except Exception as e:
+                    self.logger.warning(f"Error closing browser: {e}")
+                finally:
+                    self.browser = None
+            
+            # 停止 Playwright
             if self.playwright:
-                await self.playwright.stop()
-                
+                try:
+                    await self.playwright.stop()
+                except Exception as e:
+                    self.logger.warning(f"Error stopping playwright: {e}")
+                finally:
+                    self.playwright = None
+                    
             self.logger.info("PlaywrightDownloader closed.")
         except Exception as e:
-            self.logger.warning(f"Error closing Playwright resources: {e}")
-        finally:
+            self.logger.error(f"Error during Playwright cleanup: {e}", exc_info=True)
+            # 确保资源被清空
             self.context = None
             self.browser = None
             self.playwright = None

crawlo/downloader/selenium_downloader.py

@@ -31,7 +31,7 @@ from selenium.webdriver.support.ui import WebDriverWait
 
 from crawlo.downloader import DownloaderBase
 from crawlo.network.response import Response
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
 
 class SeleniumDownloader(DownloaderBase):
@@ -43,7 +43,7 @@ class SeleniumDownloader(DownloaderBase):
     def __init__(self, crawler):
         super().__init__(crawler)
         self.driver: Optional[webdriver.Chrome] = None
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get("LOG_LEVEL"))
+        self.logger = get_logger(self.__class__.__name__)
         self.default_timeout = crawler.settings.get_int("SELENIUM_TIMEOUT", 30)
         self.load_timeout = crawler.settings.get_int("SELENIUM_LOAD_TIMEOUT", 10)
         self.window_width = crawler.settings.get_int("SELENIUM_WINDOW_WIDTH", 1920)
@@ -465,9 +465,23 @@ class SeleniumDownloader(DownloaderBase):
         if self.driver:
             self.logger.info("Closing SeleniumDownloader driver...")
             try:
+                # 关闭所有标签页
+                if self._window_handles:
+                    self.logger.debug(f"Closing {len(self._window_handles)} tab(s)...")
+                    for handle in self._window_handles[1:]:  # 保留第一个，其他关闭
+                        try:
+                            self.driver.switch_to.window(handle)
+                            self.driver.close()
+                        except Exception as e:
+                            self.logger.warning(f"Error closing tab {handle}: {e}")
+                    
+                    self._window_handles.clear()
+                
+                # 退出浏览器
                 self.driver.quit()
             except Exception as e:
                 self.logger.warning(f"Error closing Selenium driver: {e}")
             finally:
                 self.driver = None
+        
         self.logger.debug("SeleniumDownloader closed.")

crawlo/event.py

@@ -1,11 +1,45 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+"""
+Crawlo 事件系统
+==============
+定义框架中的所有事件类型，支持类型安全和IDE自动补全。
+"""
+from enum import Enum
 
-spider_error = "spider_error"
-spider_opened = "spider_open"
-spider_closed = "spider_closed"
-ignore_request = "ignore_request"
-request_scheduled = "request_scheduled"
-response_received = "request_received"
-item_successful = "item_successful"
-item_discard = "item_discard"
+
+class CrawlerEvent(str, Enum):
+    """
+    爬虫事件枚举
+    
+    所有事件都应该使用此枚举类型，以获得：
+    - 类型安全：避免拼写错误
+    - IDE支持：自动补全和提示
+    - 文档化：集中管理所有事件
+    
+    使用示例：
+        >>> from crawlo.event import CrawlerEvent
+        >>> await subscriber.notify(CrawlerEvent.SPIDER_OPENED, spider)
+    """
+    
+    # 爬虫生命周期事件
+    SPIDER_OPENED = "spider_opened"      # 爬虫启动
+    SPIDER_CLOSED = "spider_closed"      # 爬虫关闭
+    SPIDER_ERROR = "spider_error"        # 爬虫错误
+    
+    # 请求相关事件
+    REQUEST_SCHEDULED = "request_scheduled"  # 请求已调度
+    IGNORE_REQUEST = "ignore_request"        # 请求被忽略
+    
+    # 响应相关事件
+    RESPONSE_RECEIVED = "response_received"  # 响应已接收
+    
+    # Item相关事件
+    ITEM_SUCCESSFUL = "item_successful"  # Item处理成功
+    ITEM_DISCARD = "item_discard"        # Item被丢弃
+
+
+# 导出所有公共API
+__all__ = [
+    'CrawlerEvent',
+]

crawlo/exceptions.py

@@ -1,82 +1,215 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-class TransformTypeError(TypeError):
+"""
+Crawlo 框架异常定义
+===================
+提供层次化的异常体系，便于统一处理和类型安全。
+
+异常层次：
+    CrawloException (基础异常)
+    ├── SpiderException (爬虫相关)
+    ├── ComponentInitException (组件初始化)
+    ├── DataException (数据处理)
+    ├── RequestException (请求/响应)
+    ├── OutputException (输出)
+    └── ConfigException (配置)
+
+使用示例：
+    >>> try:
+    ...     # 代码
+    ... except CrawloException as e:
+    ...     # 捕获所有框架异常
+    ... except Exception as e:
+    ...     # 其他异常
+"""
+
+
+# ============= 基础异常 =============
+class CrawloException(Exception):
+    """Crawlo框架基础异常。所有框架异常都应继承此类。"""
+    pass
+
+
+# ============= 爬虫相关异常 =============
+class SpiderException(CrawloException):
+    """爬虫相关异常基类。"""
     pass
 
 
-class OutputError(Exception):
+class SpiderTypeError(SpiderException, TypeError):
+    """爬虫类型错误。当爬虫类型不符合预期时抛出。"""
     pass
 
 
-class SpiderTypeError(TypeError):
+class SpiderCreationError(SpiderException):
+    """爬虫实例化失败异常。当无法创建爬虫实例时抛出。"""
     pass
 
 
-class ItemInitError(Exception):
+# ============= 组件初始化异常 =============
+class ComponentInitException(CrawloException):
+    """组件初始化异常基类。"""
     pass
 
 
-class ItemAttributeError(Exception):
+class MiddlewareInitError(ComponentInitException):
+    """中间件初始化失败异常。"""
     pass
 
 
-class DecodeError(Exception):
+class PipelineInitError(ComponentInitException):
+    """管道初始化失败异常。"""
     pass
 
 
-class MiddlewareInitError(Exception):
+class ExtensionInitError(ComponentInitException):
+    """扩展初始化失败异常。"""
     pass
 
 
-class PipelineInitError(Exception):
+# ============= 数据处理异常 =============
+class DataException(CrawloException):
+    """数据处理异常基类。"""
     pass
 
 
-class InvalidOutputError(Exception):
+class ItemInitError(DataException):
+    """Item初始化错误。当Item实例创建失败时抛出。"""
     pass
 
 
-class RequestMethodError(Exception):
+class ItemAttributeError(DataException, AttributeError):
+    """Item属性错误。当访问不存在的Item属性时抛出。"""
     pass
 
 
-class IgnoreRequestError(Exception):
+class ItemValidationError(DataException):
+    """Item字段验证错误。当Item字段值不符合验证规则时抛出。"""
+    pass
+
+
+class ItemDiscard(DataException):
+    """
+    Item被丢弃异常。
+    
+    注意：这不是一个真正的错误，而是用于流程控制，
+    表示Item应该被管道丢弃（例如重复数据）。
+    """
     def __init__(self, msg):
         self.msg = msg
-        super(IgnoreRequestError, self).__init__(msg)
+        super().__init__(msg)
+
+
+# ============= 请求/响应异常 =============
+class RequestException(CrawloException):
+    """请求异常基类。"""
+    pass
 
 
-class ItemDiscard(Exception):
+class RequestMethodError(RequestException):
+    """请求方法错误。当使用不支持的HTTP方法时抛出。"""
+    pass
+
+
+class IgnoreRequestError(RequestException):
+    """
+    请求被忽略异常。
+    
+    用于流程控制，表示请求应该被跳过处理。
+    """
     def __init__(self, msg):
         self.msg = msg
-        super(ItemDiscard, self).__init__(msg)
+        super().__init__(msg)
+
+
+class DecodeError(RequestException):
+    """响应解码错误。当无法解码响应内容时抛出。"""
+    pass
+
+
+# ============= 输出异常 =============
+class OutputException(CrawloException):
+    """输出异常基类。"""
+    pass
+
+
+class OutputError(OutputException):
+    """输出错误。当输出处理失败时抛出。"""
+    pass
 
 
-class NotConfigured(Exception):
+class InvalidOutputError(OutputException):
+    """无效的输出错误。当输出类型或格式不符合预期时抛出。"""
     pass
 
 
-class NotConfiguredError(Exception):
+# ============= 配置异常 =============
+class ConfigException(CrawloException):
+    """配置异常基类。"""
     pass
 
 
-class ExtensionInitError(Exception):
+class NotConfigured(ConfigException):
+    """组件未配置异常。当必需的配置缺失时抛出。"""
     pass
 
 
-class ReceiverTypeError(Exception):
+class NotConfiguredError(ConfigException):
+    """配置错误异常。当配置值无效时抛出。"""
     pass
 
 
-class SpiderCreationError(Exception):
-    """爬虫实例化失败异常"""
+# ============= 类型异常 =============
+class TransformTypeError(CrawloException, TypeError):
+    """转换类型错误。当数据转换类型不匹配时抛出。"""
     pass
 
 
-class ItemValidationError(Exception):
-    """Item 字段验证错误"""
+class ReceiverTypeError(CrawloException, TypeError):
+    """接收者类型错误。当事件接收者类型不符合预期时抛出。"""
     pass
 
 
-class DropItem(Exception):
-    pass
+# ============= 导出所有异常 =============
+__all__ = [
+    # 基础异常
+    'CrawloException',
+    
+    # 爬虫相关
+    'SpiderException',
+    'SpiderTypeError',
+    'SpiderCreationError',
+    
+    # 组件初始化
+    'ComponentInitException',
+    'MiddlewareInitError',
+    'PipelineInitError',
+    'ExtensionInitError',
+    
+    # 数据处理
+    'DataException',
+    'ItemInitError',
+    'ItemAttributeError',
+    'ItemValidationError',
+    'ItemDiscard',
+    
+    # 请求/响应
+    'RequestException',
+    'RequestMethodError',
+    'IgnoreRequestError',
+    'DecodeError',
+    
+    # 输出
+    'OutputException',
+    'OutputError',
+    'InvalidOutputError',
+    
+    # 配置
+    'ConfigException',
+    'NotConfigured',
+    'NotConfiguredError',
+    
+    # 类型
+    'TransformTypeError',
+    'ReceiverTypeError',
+]

crawlo/extension/__init__.py

@@ -3,18 +3,18 @@
 from typing import List, Any
 from pprint import pformat
 
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.utils.misc import load_object
 from crawlo.exceptions import ExtensionInitError
 
 
-class ExtensionManager(object):
+class ExtensionManager:
 
     def __init__(self, crawler: Any):
         self.crawler = crawler
         self.extensions: List = []
         extensions = self.crawler.settings.get_list('EXTENSIONS')
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
         self._add_extensions(extensions)
         self._subscribe_extensions()
 
@@ -41,24 +41,25 @@ class ExtensionManager(object):
 
     def _subscribe_extensions(self) -> None:
         """订阅扩展方法到相应的事件"""
+        from crawlo.event import CrawlerEvent
+        
         for extension in self.extensions:
             # 订阅 spider_closed 方法
             if hasattr(extension, 'spider_closed'):
-                self.crawler.subscriber.subscribe(extension.spider_closed, event="spider_closed")
+                self.crawler.subscriber.subscribe(extension.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
             
             # 订阅 item_successful 方法
             if hasattr(extension, 'item_successful'):
-                self.crawler.subscriber.subscribe(extension.item_successful, event="item_successful")
+                self.crawler.subscriber.subscribe(extension.item_successful, event=CrawlerEvent.ITEM_SUCCESSFUL)
             
             # 订阅 item_discard 方法
             if hasattr(extension, 'item_discard'):
-                self.crawler.subscriber.subscribe(extension.item_discard, event="item_discard")
+                self.crawler.subscriber.subscribe(extension.item_discard, event=CrawlerEvent.ITEM_DISCARD)
             
             # 订阅 response_received 方法
             if hasattr(extension, 'response_received'):
-                # 修复：将事件名称从 "request_received" 更正为 "response_received"
-                self.crawler.subscriber.subscribe(extension.response_received, event="response_received")
+                self.crawler.subscriber.subscribe(extension.response_received, event=CrawlerEvent.RESPONSE_RECEIVED)
             
             # 订阅 request_scheduled 方法
             if hasattr(extension, 'request_scheduled'):
-                self.crawler.subscriber.subscribe(extension.request_scheduled, event="request_scheduled")
+                self.crawler.subscriber.subscribe(extension.request_scheduled, event=CrawlerEvent.REQUEST_SCHEDULED)

crawlo/extension/health_check.py

@@ -4,8 +4,8 @@ import asyncio
 from datetime import datetime
 from typing import Any, Optional, Dict
 
-from crawlo.event import spider_opened, spider_closed, response_received, request_scheduled
-from crawlo.utils.log import get_logger
+from crawlo.event import CrawlerEvent
+from crawlo.logging import get_logger
 
 
 class HealthCheckExtension:
@@ -16,7 +16,7 @@ class HealthCheckExtension:
 
     def __init__(self, crawler: Any):
         self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
         
         # 获取配置参数
         self.enabled = self.settings.get_bool('HEALTH_CHECK_ENABLED', True)
@@ -43,10 +43,10 @@ class HealthCheckExtension:
         
         o = cls(crawler)
         if o.enabled:
-            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-            crawler.subscriber.subscribe(o.response_received, event=response_received)
-            crawler.subscriber.subscribe(o.request_scheduled, event=request_scheduled)
+            crawler.subscriber.subscribe(o.spider_opened, event=CrawlerEvent.SPIDER_OPENED)
+            crawler.subscriber.subscribe(o.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
+            crawler.subscriber.subscribe(o.response_received, event=CrawlerEvent.RESPONSE_RECEIVED)
+            crawler.subscriber.subscribe(o.request_scheduled, event=CrawlerEvent.REQUEST_SCHEDULED)
         return o
 
     async def spider_opened(self) -> None:

crawlo/extension/log_interval.py

@@ -3,11 +3,11 @@
 import asyncio
 from typing import Any, Optional
 
-from crawlo.utils.log import get_logger
-from crawlo.event import spider_opened, spider_closed
+from crawlo.logging import get_logger
+from crawlo.event import CrawlerEvent
 
 
-class LogIntervalExtension(object):
+class LogIntervalExtension:
 
     def __init__(self, crawler: Any):
         self.task: Optional[asyncio.Task] = None
@@ -30,14 +30,14 @@ class LogIntervalExtension(object):
         else:
             self.interval_display = str(self.interval)
 
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
         self.logger.info(f"LogIntervalExtension initialized with interval: {self.seconds} seconds")
 
     @classmethod
     def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
         o = cls(crawler)
-        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        crawler.subscriber.subscribe(o.spider_opened, event=CrawlerEvent.SPIDER_OPENED)
+        crawler.subscriber.subscribe(o.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
         return o
 
     async def spider_opened(self) -> None:

crawlo/extension/log_stats.py

@@ -7,7 +7,7 @@
 import asyncio
 from typing import Any
 
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.utils import now, time_diff
 
 
@@ -18,7 +18,7 @@ class LogStats:
 
     def __init__(self, crawler):
         self.crawler = crawler
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
         self._stats = crawler.stats
         self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
 

crawlo/extension/logging_extension.py

@@ -1,16 +1,9 @@
 from typing import Any
 from crawlo.exceptions import NotConfigured
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
-# 延迟获取logger，确保在日志系统配置之后获取
-_logger = None
-
-def logger():
-    """延迟获取logger实例，确保在日志系统配置之后获取"""
-    global _logger
-    if _logger is None:
-        _logger = get_logger(__name__)
-    return _logger
+# 获取logger实例
+_logger = get_logger(__name__)
 
 
 class CustomLoggerExtension:
@@ -50,9 +43,8 @@ class CustomLoggerExtension:
         return cls(crawler.settings)
 
     def spider_opened(self, spider: Any) -> None:
-        logger_instance = logger()
         try:
-            logger_instance.info(
+            _logger.info(
                 f"CustomLoggerExtension: Logging initialized. "
                 f"LOG_FILE={self.settings.get('LOG_FILE')}, "
                 f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"