crawlo 1.3.9__tar.gz → 1.4.1__tar.gz

{crawlo-1.3.9/crawlo.egg-info → crawlo-1.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.3.9
+Version: 1.4.1
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/__init__.py

@@ -28,30 +28,35 @@ from crawlo import tools
 
 # 框架核心模块 - 使用TYPE_CHECKING避免循环导入
 if TYPE_CHECKING:
-    from crawlo.core.framework_initializer import get_framework_initializer, initialize_framework
+    from crawlo.initialization import get_framework_initializer, initialize_framework
 
 # 为了向后兼容，从tools中导入cleaners相关的功能
 import crawlo.tools as cleaners
 
+
 # 延迟导入的辅助函数
 def get_framework_initializer():
     """延迟导入get_framework_initializer以避免循环依赖"""
-    from crawlo.core.framework_initializer import get_framework_initializer as _get_framework_initializer
+    from crawlo.initialization import get_framework_initializer as _get_framework_initializer
     return _get_framework_initializer()
 
+
 def initialize_framework(custom_settings=None):
     """延迟导入initialize_framework以避免循环依赖"""
-    from crawlo.core.framework_initializer import initialize_framework as _initialize_framework
+    from crawlo.initialization import initialize_framework as _initialize_framework
     return _initialize_framework(custom_settings)
 
+
 # 向后兼容的别名
 def get_bootstrap_manager():
     """向后兼容的别名"""
     return get_framework_initializer()
 
+
 # 版本号：优先从元数据读取
 try:
     from importlib.metadata import version
+
     __version__ = version("crawlo")
 except Exception:
     # 开发模式下可能未安装，回退到 __version__.py 或 dev
@@ -85,4 +90,4 @@ __all__ = [
     'get_framework_initializer',
     'get_bootstrap_manager',
     '__version__',
-]
+]

crawlo-1.4.1/crawlo/__version__.py

@@ -0,0 +1 @@
+__version__ = '1.4.1'

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/commands/run.py

@@ -23,7 +23,7 @@ from crawlo.crawler import CrawlerProcess
 from crawlo.project import get_settings, _find_project_root
 # 使用新的统一初始化系统
 from crawlo.initialization import initialize_framework
-from crawlo.core import get_framework_initializer
+from crawlo.initialization import get_framework_initializer
 from crawlo.utils.log import get_logger
 
 # 延迟获取logger，确保在日志系统配置之后获取

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/core/__init__.py

@@ -10,37 +10,43 @@ from ..initialization import (
     is_framework_ready
 )
 
+
 # 向后兼容的别名
 def async_initialize_framework(*args, **kwargs):
     """Async wrapper for framework initialization"""
     return initialize_framework(*args, **kwargs)
 
+
 def get_framework_initializer():
     """Get framework initializer - compatibility function"""
     from ..initialization.core import CoreInitializer
     return CoreInitializer()
 
+
 def get_framework_logger(name='crawlo.core'):
     """Get framework logger - compatibility function"""
     from ..logging import get_logger
     return get_logger(name)
 
+
 # 向后兼容
 def bootstrap_framework(*args, **kwargs):
     """Bootstrap framework - compatibility function"""
     return initialize_framework(*args, **kwargs)
 
+
 def get_bootstrap_manager():
     """Get bootstrap manager - compatibility function"""
     return get_framework_initializer()
 
+
 __all__ = [
     'initialize_framework',
-    'async_initialize_framework', 
+    'async_initialize_framework',
     'get_framework_initializer',
     'is_framework_ready',
     'get_framework_logger',
     # 向后兼容
     'bootstrap_framework',
     'get_bootstrap_manager'
-]
+]

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/core/processor.py

@@ -1,10 +1,12 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-from asyncio import Queue
+from asyncio import Queue, create_task
 from typing import Union, Optional
 
 from crawlo import Request, Item
 from crawlo.pipelines.pipeline_manager import PipelineManager
+from crawlo.exceptions import ItemDiscard
+from crawlo.event import item_discard
 
 
 class Processor(object):
@@ -27,7 +29,13 @@ class Processor(object):
                 await self._process_item(result)
 
     async def _process_item(self, item):
-        await self.pipelines.process_item(item=item)
+        try:
+            await self.pipelines.process_item(item=item)
+        except ItemDiscard as exc:
+            # Item was discarded by a pipeline (e.g., deduplication pipeline)
+            # We simply ignore this item and don't pass it to subsequent pipelines
+            # The statistics system has already been notified in PipelineManager, so we don't need to notify again
+            pass
 
     async def enqueue(self, output: Union[Request, Item]):
         await self.queue.put(output)
@@ -37,4 +45,4 @@ class Processor(object):
         return len(self) == 0
 
     def __len__(self):
-        return self.queue.qsize()
+        return self.queue.qsize()

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/core/scheduler.py

@@ -77,8 +77,8 @@ class Scheduler:
             # 只有在确实需要更新配置时才重新创建过滤器实例
             # 检查是否真的进行了配置更新
             filter_updated = (
-                (self.queue_manager._queue_type == QueueType.REDIS and 'memory_filter' in self.crawler.settings.get('FILTER_CLASS', '')) or
-                (self.queue_manager._queue_type == QueueType.MEMORY and ('aioredis_filter' in self.crawler.settings.get('FILTER_CLASS', '') or 'redis_filter' in self.crawler.settings.get('FILTER_CLASS', '')))
+                (self.queue_manager._queue_type == QueueType.REDIS and 'aioredis_filter' in self.crawler.settings.get('FILTER_CLASS', '')) or
+                (self.queue_manager._queue_type == QueueType.MEMORY and 'memory_filter' in self.crawler.settings.get('FILTER_CLASS', ''))
             )
             
             if needs_config_update or filter_updated:

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/crawler.py

@@ -308,6 +308,18 @@ class ModernCrawler:
                 except Exception as e:
                     self._logger.warning(f"Spider cleanup failed: {e}")
             
+            # 调用StatsCollector的close_spider方法，设置reason和spider_name
+            if self._stats and hasattr(self._stats, 'close_spider'):
+                try:
+                    # 使用默认的'finished'作为reason
+                    self._stats.close_spider(self._spider, reason='finished')
+                except Exception as e:
+                    self._logger.warning(f"Stats close_spider failed: {e}")
+            
+            # 触发spider_closed事件，通知所有订阅者（包括扩展）
+            # 传递reason参数，这里使用默认的'finished'作为reason
+            await self.subscriber.notify("spider_closed", reason='finished')
+            
             if self._stats and hasattr(self._stats, 'close'):
                 try:
                     close_result = self._stats.close()

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/extension/__init__.py

@@ -16,6 +16,7 @@ class ExtensionManager(object):
         extensions = self.crawler.settings.get_list('EXTENSIONS')
         self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
         self._add_extensions(extensions)
+        self._subscribe_extensions()
 
     @classmethod
     def create_instance(cls, *args: Any, **kwargs: Any) -> 'ExtensionManager':
@@ -37,3 +38,27 @@ class ExtensionManager(object):
         if extensions:
             # 恢复INFO级别日志，保留关键的启用信息
             self.logger.info(f"Enabled extensions: \n{pformat(extensions)}")
+
+    def _subscribe_extensions(self) -> None:
+        """订阅扩展方法到相应的事件"""
+        for extension in self.extensions:
+            # 订阅 spider_closed 方法
+            if hasattr(extension, 'spider_closed'):
+                self.crawler.subscriber.subscribe(extension.spider_closed, event="spider_closed")
+            
+            # 订阅 item_successful 方法
+            if hasattr(extension, 'item_successful'):
+                self.crawler.subscriber.subscribe(extension.item_successful, event="item_successful")
+            
+            # 订阅 item_discard 方法
+            if hasattr(extension, 'item_discard'):
+                self.crawler.subscriber.subscribe(extension.item_discard, event="item_discard")
+            
+            # 订阅 response_received 方法
+            if hasattr(extension, 'response_received'):
+                # 修复：将事件名称从 "request_received" 更正为 "response_received"
+                self.crawler.subscriber.subscribe(extension.response_received, event="response_received")
+            
+            # 订阅 request_scheduled 方法
+            if hasattr(extension, 'request_scheduled'):
+                self.crawler.subscriber.subscribe(extension.request_scheduled, event="request_scheduled")

crawlo-1.4.1/crawlo/extension/log_interval.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Any, Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed
+
+
+class LogIntervalExtension(object):
+
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
+        self.stats = crawler.stats
+        self.item_count = 0
+        self.response_count = 0
+        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
+        
+        # 修复时间单位计算逻辑
+        if self.seconds % 60 == 0:
+            self.interval = int(self.seconds / 60)
+            self.unit = 'min'
+        else:
+            self.interval = self.seconds
+            self.unit = 's'
+        
+        # 处理单数情况
+        if self.interval == 1 and self.unit == 'min':
+            self.interval_display = ""
+        else:
+            self.interval_display = str(self.interval)
+
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger.info(f"LogIntervalExtension initialized with interval: {self.seconds} seconds")
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self) -> None:
+        self.logger.info("Spider opened, starting interval logging task")
+        self.task = asyncio.create_task(self.interval_log())
+        self.logger.info("Interval logging task started")
+
+    async def spider_closed(self) -> None:
+        self.logger.info("Spider closed, stopping interval logging task")
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+            self.task = None
+
+    async def interval_log(self) -> None:
+        iteration = 0
+        while True:
+            try:
+                iteration += 1
+                self.logger.debug(f"Interval log iteration {iteration} starting")
+                last_item_count = self.stats.get_value('item_successful_count', default=0)
+                last_response_count = self.stats.get_value('response_received_count', default=0)
+                item_rate = last_item_count - self.item_count
+                response_rate = last_response_count - self.response_count
+                
+                # 添加调试信息
+                self.logger.debug(f"Debug info - Iteration: {iteration}, Last item count: {last_item_count}, Last response count: {last_response_count}")
+                self.logger.debug(f"Debug info - Previous item count: {self.item_count}, Previous response count: {self.response_count}")
+                self.logger.debug(f"Debug info - Item rate: {item_rate}, Response rate: {response_rate}")
+                
+                self.item_count, self.response_count = last_item_count, last_response_count
+                
+                # 修复效率计算，确保使用正确的单位
+                if self.unit == 'min' and self.seconds > 0:
+                    # 转换为每分钟速率
+                    pages_per_min = response_rate * 60 / self.seconds if self.seconds > 0 else 0
+                    items_per_min = item_rate * 60 / self.seconds if self.seconds > 0 else 0
+                    self.logger.info(
+                        f'Crawled {last_response_count} pages (at {pages_per_min:.0f} pages/min),'
+                        f' Got {last_item_count} items (at {items_per_min:.0f} items/min).'
+                    )
+                else:
+                    # 使用原始单位
+                    self.logger.info(
+                        f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval_display}{self.unit}),'
+                        f' Got {last_item_count} items (at {item_rate} items/{self.interval_display}{self.unit}).'
+                    )
+                self.logger.debug(f"Interval log iteration {iteration} completed, sleeping for {self.seconds} seconds")
+                await asyncio.sleep(self.seconds)
+            except Exception as e:
+                self.logger.error(f"Error in interval logging: {e}")
+                await asyncio.sleep(self.seconds)  # 即使出错也继续执行

crawlo-1.4.1/crawlo/extension/log_stats.py

@@ -0,0 +1,71 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+日志统计扩展
+提供详细的爬虫运行统计信息
+"""
+import asyncio
+from typing import Any
+
+from crawlo.utils.log import get_logger
+from crawlo.utils import now, time_diff
+
+
+class LogStats:
+    """
+    日志统计扩展，记录和输出爬虫运行过程中的各种统计信息
+    """
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self._stats = crawler.stats
+        self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls.from_crawler(crawler)
+
+    async def spider_closed(self, reason: str = 'finished') -> None:
+        try:
+            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+            self._stats['reason'] = reason
+        except Exception as e:
+            # 添加日志以便调试
+            self.logger.error(f"Error in spider_closed: {e}")
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_successful(self, _item: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_successful_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
+        try:
+            # 只增加总的丢弃计数，不记录每个丢弃项目的原因详情
+            self._stats.inc_value('item_discard_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def response_received(self, _response: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('response_received_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('request_scheduler_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/initialization/__init__.py

@@ -16,25 +16,29 @@ from .context import InitializationContext
 from .core import CoreInitializer
 from .phases import InitializationPhase
 
+
 # 公共接口
 def initialize_framework(settings=None, **kwargs):
     """初始化框架的主要入口"""
     return CoreInitializer().initialize(settings, **kwargs)
 
+
 def is_framework_ready():
     """检查框架是否已准备就绪"""
     return CoreInitializer().is_ready
 
+
 def get_framework_context():
     """获取框架初始化上下文"""
     return CoreInitializer().context
 
+
 __all__ = [
     'InitializerRegistry',
-    'InitializationContext', 
+    'InitializationContext',
     'CoreInitializer',
     'InitializationPhase',
     'initialize_framework',
     'is_framework_ready',
     'get_framework_context'
-]
+]

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/middleware/middleware_manager.py

@@ -86,7 +86,7 @@ class MiddlewareManager:
             response = await self._process_exception(request, exp)
         else:
             create_task(self.crawler.subscriber.notify(response_received, response, self.crawler.spider))
-            # self.crawler.stats.inc_value('response_received_count')
+            self._stats.inc_value('response_received_count')
         if isinstance(response, Response):
             response = await self._process_response(request, response)
         if isinstance(response, Request):

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/middleware/response_code.py

@@ -127,9 +127,7 @@ class ResponseCodeMiddleware(object):
         """
         status_code = response.status_code
         
-        # 记录具体状态码统计
-        self.stats.inc_value(f'response_status_code/count/{status_code}')
-        
+        # 只记录总的统计信息，不记录每个域名和每个状态码的详细信息
         # 记录状态码分类统计
         category = self._get_status_category(status_code)
         self.stats.inc_value(f'response_status_code/category/{category}')
@@ -144,17 +142,6 @@ class ResponseCodeMiddleware(object):
         if hasattr(response, 'content_length') and response.content_length:
             self.stats.inc_value('response_total_bytes', response.content_length)
         
-        # 记录域名统计
-        try:
-            from urllib.parse import urlparse
-            parsed_url = urlparse(response.url)
-            domain = parsed_url.netloc
-            if domain:
-                self.stats.inc_value(f'response_status_code/domain/{domain}/count/{status_code}')
-                self.stats.inc_value(f'response_status_code/domain/{domain}/category/{category}')
-        except Exception:
-            self.stats.inc_value('response_status_code/domain/invalid_url/count/{status_code}')
-        
         # 详细日志记录
         self.logger.debug(
             f'收到响应: {status_code} {response.url} '

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/mode_manager.py

@@ -7,7 +7,7 @@
 
 支持的运行模式：
 1. standalone - 单机模式（默认）
-2. distributed - 分布式模式  
+2. distributed - 分布式模式
 3. auto - 自动检测模式
 """
 import os
@@ -29,7 +29,7 @@ class ModeManager:
         # 延迟初始化logger，避免循环依赖
         self._logger = None
         self._debug("运行模式管理器初始化完成")
-    
+
     def _get_logger(self):
         """延迟获取logger实例"""
         if self._logger is None:
@@ -40,7 +40,7 @@ class ModeManager:
                 # 如果日志系统尚未初始化，返回None
                 pass
         return self._logger
-    
+
     def _debug(self, message: str):
         """调试日志"""
         logger = self._get_logger()
@@ -73,7 +73,7 @@ class ModeManager:
             redis_url = f'redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}'
         else:
             redis_url = f'redis://{redis_host}:{redis_port}/{redis_db}'
-            
+
         return {
             'QUEUE_TYPE': 'redis',
             'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
@@ -119,6 +119,7 @@ class ModeManager:
 
         if mode == RunMode.STANDALONE:
             mode_info = "使用单机模式 - 简单快速，适合开发和中小规模爬取"
+            # 对于单机模式，如果用户设置了QUEUE_TYPE为'auto'，应该保留用户的设置
             settings = self.get_standalone_settings()
             self._debug("应用单机模式配置")
 
@@ -142,8 +143,13 @@ class ModeManager:
             raise ValueError(f"不支持的运行模式: {mode}")
 
         # 合并用户自定义配置
-        user_settings = {k: v for k, v in kwargs.items()
-                         if k not in ['redis_host', 'redis_port', 'redis_password', 'project_name']}
+        user_settings = {
+            k: v for k,
+            v in kwargs.items() if k not in [
+                'redis_host',
+                'redis_port',
+                'redis_password',
+                'project_name']}
         settings.update(user_settings)
         self._debug(f"合并用户自定义配置: {list(user_settings.keys())}")
 
@@ -210,4 +216,4 @@ def auto_mode(**kwargs) -> Dict[str, Any]:
 def from_env(default_mode: str = 'standalone') -> Dict[str, Any]:
     """从环境变量创建配置"""
     # 移除直接使用 os.getenv()，要求通过 settings 配置
-    raise RuntimeError("环境变量配置已移除，请在 settings 中配置相关参数")
+    raise RuntimeError("环境变量配置已移除，请在 settings 中配置相关参数")

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/pipelines/bloom_dedup_pipeline.py

@@ -38,6 +38,7 @@ except ImportError:
 
 from crawlo import Item
 from crawlo.spider import Spider
+from crawlo.utils.fingerprint import FingerprintGenerator
 from crawlo.utils.log import get_logger
 from crawlo.exceptions import DropItem, ItemDiscard
 
@@ -109,6 +110,9 @@ class BloomDedupPipeline:
                 self.logger.debug(f"Processing new item: {fingerprint[:20]}...")
                 return item
                 
+        except ItemDiscard:
+            # 重新抛出ItemDiscard异常，确保管道管理器能正确处理
+            raise
         except Exception as e:
             self.logger.error(f"Error processing item: {e}")
             # 在错误时继续处理，避免丢失数据
@@ -123,21 +127,7 @@ class BloomDedupPipeline:
         :param item: 数据项
         :return: 指纹字符串
         """
-        # 将数据项转换为可序列化的字典
-        try:
-            item_dict = item.to_dict()
-        except AttributeError:
-            # 兼容没有to_dict方法的Item实现
-            item_dict = dict(item)
-        
-        # 对字典进行排序以确保一致性
-        sorted_items = sorted(item_dict.items())
-        
-        # 生成指纹字符串
-        fingerprint_string = '|'.join([f"{k}={v}" for k, v in sorted_items if v is not None])
-        
-        # 使用 SHA256 生成固定长度的指纹
-        return hashlib.sha256(fingerprint_string.encode('utf-8')).hexdigest()
+        return FingerprintGenerator.item_fingerprint(item)
 
     def close_spider(self, spider: Spider) -> None:
         """

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/pipelines/database_dedup_pipeline.py

@@ -17,6 +17,7 @@ import aiomysql
 from crawlo import Item
 from crawlo.exceptions import DropItem, ItemDiscard
 from crawlo.spider import Spider
+from crawlo.utils.fingerprint import FingerprintGenerator
 from crawlo.utils.log import get_logger
 
 
@@ -140,6 +141,9 @@ class DatabaseDedupPipeline:
                 self.logger.debug(f"Processing new item: {fingerprint[:20]}...")
                 return item
                 
+        except ItemDiscard:
+            # 重新抛出ItemDiscard异常，确保管道管理器能正确处理
+            raise
         except Exception as e:
             self.logger.error(f"Error processing item: {e}")
             # 在错误时继续处理，避免丢失数据
@@ -190,11 +194,4 @@ class DatabaseDedupPipeline:
         :param item: 数据项
         :return: 指纹字符串
         """
-        # 将数据项转换为可序列化的字典
-        try:
-            item_dict = item.to_dict()
-        except AttributeError:
-            # 兼容没有to_dict方法的Item实现
-            item_dict = dict(item)
-        
-        # 对字典进行排序以确保一致性
+        return FingerprintGenerator.item_fingerprint(item)

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/pipelines/memory_dedup_pipeline.py

@@ -18,6 +18,7 @@ from typing import Set
 from crawlo import Item
 from crawlo.exceptions import DropItem, ItemDiscard
 from crawlo.spider import Spider
+from crawlo.utils.fingerprint import FingerprintGenerator
 from crawlo.utils.log import get_logger
 
 
@@ -71,6 +72,9 @@ class MemoryDedupPipeline:
                 self.logger.debug(f"Processing new item: {fingerprint[:20]}...")
                 return item
                 
+        except ItemDiscard:
+            # 重新抛出ItemDiscard异常，确保管道管理器能正确处理
+            raise
         except Exception as e:
             self.logger.error(f"Error processing item: {e}")
             # 在错误时继续处理，避免丢失数据
@@ -85,21 +89,7 @@ class MemoryDedupPipeline:
         :param item: 数据项
         :return: 指纹字符串
         """
-        # 将数据项转换为可序列化的字典
-        try:
-            item_dict = item.to_dict()
-        except AttributeError:
-            # 兼容没有to_dict方法的Item实现
-            item_dict = dict(item)
-        
-        # 对字典进行排序以确保一致性
-        sorted_items = sorted(item_dict.items())
-        
-        # 生成指纹字符串
-        fingerprint_string = '|'.join([f"{k}={v}" for k, v in sorted_items if v is not None])
-        
-        # 使用 SHA256 生成固定长度的指纹
-        return hashlib.sha256(fingerprint_string.encode('utf-8')).hexdigest()
+        return FingerprintGenerator.item_fingerprint(item)
 
     def close_spider(self, spider: Spider) -> None:
         """

{crawlo-1.3.9 → crawlo-1.4.1}/crawlo/pipelines/pipeline_manager.py

@@ -66,11 +66,19 @@ class PipelineManager:
 
     async def process_item(self, item):
         try:
-            for method in self.methods:
-                item = await common_call(method, item, self.crawler.spider)
-                if item is None:
-                    raise InvalidOutputError(f"{method.__qualname__} return None is not supported.")
-        except (ItemDiscard, DropItem) as exc:  # 同时捕获两种异常类型
-            create_task(self.crawler.subscriber.notify(item_discard, item, exc, self.crawler.spider))
+            for i, method in enumerate(self.methods):
+                self.logger.debug(f"Processing item with pipeline method {i}: {method.__qualname__}")
+                try:
+                    item = await common_call(method, item, self.crawler.spider)
+                    if item is None:
+                        raise InvalidOutputError(f"{method.__qualname__} return None is not supported.")
+                except (ItemDiscard, DropItem) as exc:  # 同时捕获两种异常类型
+                    self.logger.debug(f"Item discarded by pipeline: {exc}")
+                    create_task(self.crawler.subscriber.notify(item_discard, item, exc, self.crawler.spider))
+                    # 重新抛出异常，确保上层调用者也能捕获到，并停止执行后续管道
+                    raise
+        except (ItemDiscard, DropItem):
+            # 异常已经被处理和通知，这里只需要重新抛出
+            raise
         else:
-            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))
+            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))