crawlo 1.4.0__tar.gz → 1.4.2__tar.gz

{crawlo-1.4.0/crawlo.egg-info → crawlo-1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.4.0
+Version: 1.4.2
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/__init__.py

@@ -28,30 +28,35 @@ from crawlo import tools
 
 # 框架核心模块 - 使用TYPE_CHECKING避免循环导入
 if TYPE_CHECKING:
-    from crawlo.core.framework_initializer import get_framework_initializer, initialize_framework
+    from crawlo.initialization import get_framework_initializer, initialize_framework
 
 # 为了向后兼容，从tools中导入cleaners相关的功能
 import crawlo.tools as cleaners
 
+
 # 延迟导入的辅助函数
 def get_framework_initializer():
     """延迟导入get_framework_initializer以避免循环依赖"""
-    from crawlo.core.framework_initializer import get_framework_initializer as _get_framework_initializer
+    from crawlo.initialization import get_framework_initializer as _get_framework_initializer
     return _get_framework_initializer()
 
+
 def initialize_framework(custom_settings=None):
     """延迟导入initialize_framework以避免循环依赖"""
-    from crawlo.core.framework_initializer import initialize_framework as _initialize_framework
+    from crawlo.initialization import initialize_framework as _initialize_framework
     return _initialize_framework(custom_settings)
 
+
 # 向后兼容的别名
 def get_bootstrap_manager():
     """向后兼容的别名"""
     return get_framework_initializer()
 
+
 # 版本号：优先从元数据读取
 try:
     from importlib.metadata import version
+
     __version__ = version("crawlo")
 except Exception:
     # 开发模式下可能未安装，回退到 __version__.py 或 dev
@@ -85,4 +90,4 @@ __all__ = [
     'get_framework_initializer',
     'get_bootstrap_manager',
     '__version__',
-]
+]

crawlo-1.4.2/crawlo/__version__.py

@@ -0,0 +1 @@
+__version__ = '1.4.2'

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/core/__init__.py

@@ -10,37 +10,43 @@ from ..initialization import (
     is_framework_ready
 )
 
+
 # 向后兼容的别名
 def async_initialize_framework(*args, **kwargs):
     """Async wrapper for framework initialization"""
     return initialize_framework(*args, **kwargs)
 
+
 def get_framework_initializer():
     """Get framework initializer - compatibility function"""
     from ..initialization.core import CoreInitializer
     return CoreInitializer()
 
+
 def get_framework_logger(name='crawlo.core'):
     """Get framework logger - compatibility function"""
     from ..logging import get_logger
     return get_logger(name)
 
+
 # 向后兼容
 def bootstrap_framework(*args, **kwargs):
     """Bootstrap framework - compatibility function"""
     return initialize_framework(*args, **kwargs)
 
+
 def get_bootstrap_manager():
     """Get bootstrap manager - compatibility function"""
     return get_framework_initializer()
 
+
 __all__ = [
     'initialize_framework',
-    'async_initialize_framework', 
+    'async_initialize_framework',
     'get_framework_initializer',
     'is_framework_ready',
     'get_framework_logger',
     # 向后兼容
     'bootstrap_framework',
     'get_bootstrap_manager'
-]
+]

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/core/scheduler.py

@@ -77,8 +77,8 @@ class Scheduler:
             # 只有在确实需要更新配置时才重新创建过滤器实例
             # 检查是否真的进行了配置更新
             filter_updated = (
-                (self.queue_manager._queue_type == QueueType.REDIS and 'memory_filter' in self.crawler.settings.get('FILTER_CLASS', '')) or
-                (self.queue_manager._queue_type == QueueType.MEMORY and ('aioredis_filter' in self.crawler.settings.get('FILTER_CLASS', '') or 'redis_filter' in self.crawler.settings.get('FILTER_CLASS', '')))
+                (self.queue_manager._queue_type == QueueType.REDIS and 'aioredis_filter' in self.crawler.settings.get('FILTER_CLASS', '')) or
+                (self.queue_manager._queue_type == QueueType.MEMORY and 'memory_filter' in self.crawler.settings.get('FILTER_CLASS', ''))
             )
             
             if needs_config_update or filter_updated:

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/downloader/aiohttp_downloader.py

@@ -55,8 +55,13 @@ class AioHttpDownloader(DownloaderBase):
             family=0,  # 允许IPv4和IPv6
         )
 
-        # 超时控制
-        timeout = ClientTimeout(total=timeout_secs)
+        # 超时控制 - 增加更多超时设置
+        timeout = ClientTimeout(
+            total=timeout_secs,
+            connect=timeout_secs/2,  # 连接超时
+            sock_read=timeout_secs,  # 读取超时
+            sock_connect=timeout_secs/2  # socket连接超时
+        )
 
         # 请求追踪
         trace_config = TraceConfig()

crawlo-1.4.2/crawlo/extension/log_interval.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Any, Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed
+
+
+class LogIntervalExtension(object):
+
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
+        self.stats = crawler.stats
+        self.item_count = 0
+        self.response_count = 0
+        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
+        
+        # 修复时间单位计算逻辑
+        if self.seconds % 60 == 0:
+            self.interval = int(self.seconds / 60)
+            self.unit = 'min'
+        else:
+            self.interval = self.seconds
+            self.unit = 's'
+        
+        # 处理单数情况
+        if self.interval == 1 and self.unit == 'min':
+            self.interval_display = ""
+        else:
+            self.interval_display = str(self.interval)
+
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger.info(f"LogIntervalExtension initialized with interval: {self.seconds} seconds")
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self) -> None:
+        self.logger.info("Spider opened, starting interval logging task")
+        self.task = asyncio.create_task(self.interval_log())
+        self.logger.info("Interval logging task started")
+
+    async def spider_closed(self) -> None:
+        self.logger.info("Spider closed, stopping interval logging task")
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+            self.task = None
+
+    async def interval_log(self) -> None:
+        iteration = 0
+        while True:
+            try:
+                iteration += 1
+                self.logger.debug(f"Interval log iteration {iteration} starting")
+                last_item_count = self.stats.get_value('item_successful_count', default=0)
+                last_response_count = self.stats.get_value('response_received_count', default=0)
+                item_rate = last_item_count - self.item_count
+                response_rate = last_response_count - self.response_count
+                
+                # 添加调试信息
+                self.logger.debug(f"Debug info - Iteration: {iteration}, Last item count: {last_item_count}, Last response count: {last_response_count}")
+                self.logger.debug(f"Debug info - Previous item count: {self.item_count}, Previous response count: {self.response_count}")
+                self.logger.debug(f"Debug info - Item rate: {item_rate}, Response rate: {response_rate}")
+                
+                self.item_count, self.response_count = last_item_count, last_response_count
+                
+                # 修复效率计算，确保使用正确的单位
+                if self.unit == 'min' and self.seconds > 0:
+                    # 转换为每分钟速率
+                    pages_per_min = response_rate * 60 / self.seconds if self.seconds > 0 else 0
+                    items_per_min = item_rate * 60 / self.seconds if self.seconds > 0 else 0
+                    self.logger.info(
+                        f'Crawled {last_response_count} pages (at {pages_per_min:.0f} pages/min),'
+                        f' Got {last_item_count} items (at {items_per_min:.0f} items/min).'
+                    )
+                else:
+                    # 使用原始单位
+                    self.logger.info(
+                        f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval_display}{self.unit}),'
+                        f' Got {last_item_count} items (at {item_rate} items/{self.interval_display}{self.unit}).'
+                    )
+                self.logger.debug(f"Interval log iteration {iteration} completed, sleeping for {self.seconds} seconds")
+                await asyncio.sleep(self.seconds)
+            except Exception as e:
+                self.logger.error(f"Error in interval logging: {e}")
+                await asyncio.sleep(self.seconds)  # 即使出错也继续执行

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/initialization/__init__.py

@@ -16,25 +16,29 @@ from .context import InitializationContext
 from .core import CoreInitializer
 from .phases import InitializationPhase
 
+
 # 公共接口
 def initialize_framework(settings=None, **kwargs):
     """初始化框架的主要入口"""
     return CoreInitializer().initialize(settings, **kwargs)
 
+
 def is_framework_ready():
     """检查框架是否已准备就绪"""
     return CoreInitializer().is_ready
 
+
 def get_framework_context():
     """获取框架初始化上下文"""
     return CoreInitializer().context
 
+
 __all__ = [
     'InitializerRegistry',
-    'InitializationContext', 
+    'InitializationContext',
     'CoreInitializer',
     'InitializationPhase',
     'initialize_framework',
     'is_framework_ready',
     'get_framework_context'
-]
+]

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/middleware/middleware_manager.py

@@ -86,7 +86,7 @@ class MiddlewareManager:
             response = await self._process_exception(request, exp)
         else:
             create_task(self.crawler.subscriber.notify(response_received, response, self.crawler.spider))
-            # self.crawler.stats.inc_value('response_received_count')
+            self._stats.inc_value('response_received_count')
         if isinstance(response, Response):
             response = await self._process_response(request, response)
         if isinstance(response, Request):

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/mode_manager.py

@@ -7,7 +7,7 @@
 
 支持的运行模式：
 1. standalone - 单机模式（默认）
-2. distributed - 分布式模式  
+2. distributed - 分布式模式
 3. auto - 自动检测模式
 """
 import os
@@ -29,7 +29,7 @@ class ModeManager:
         # 延迟初始化logger，避免循环依赖
         self._logger = None
         self._debug("运行模式管理器初始化完成")
-    
+
     def _get_logger(self):
         """延迟获取logger实例"""
         if self._logger is None:
@@ -40,7 +40,7 @@ class ModeManager:
                 # 如果日志系统尚未初始化，返回None
                 pass
         return self._logger
-    
+
     def _debug(self, message: str):
         """调试日志"""
         logger = self._get_logger()
@@ -73,7 +73,7 @@ class ModeManager:
             redis_url = f'redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}'
         else:
             redis_url = f'redis://{redis_host}:{redis_port}/{redis_db}'
-            
+
         return {
             'QUEUE_TYPE': 'redis',
             'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
@@ -119,6 +119,7 @@ class ModeManager:
 
         if mode == RunMode.STANDALONE:
             mode_info = "使用单机模式 - 简单快速，适合开发和中小规模爬取"
+            # 对于单机模式，如果用户设置了QUEUE_TYPE为'auto'，应该保留用户的设置
             settings = self.get_standalone_settings()
             self._debug("应用单机模式配置")
 
@@ -142,8 +143,13 @@ class ModeManager:
             raise ValueError(f"不支持的运行模式: {mode}")
 
         # 合并用户自定义配置
-        user_settings = {k: v for k, v in kwargs.items()
-                         if k not in ['redis_host', 'redis_port', 'redis_password', 'project_name']}
+        user_settings = {
+            k: v for k,
+            v in kwargs.items() if k not in [
+                'redis_host',
+                'redis_port',
+                'redis_password',
+                'project_name']}
         settings.update(user_settings)
         self._debug(f"合并用户自定义配置: {list(user_settings.keys())}")
 
@@ -210,4 +216,4 @@ def auto_mode(**kwargs) -> Dict[str, Any]:
 def from_env(default_mode: str = 'standalone') -> Dict[str, Any]:
     """从环境变量创建配置"""
     # 移除直接使用 os.getenv()，要求通过 settings 配置
-    raise RuntimeError("环境变量配置已移除，请在 settings 中配置相关参数")
+    raise RuntimeError("环境变量配置已移除，请在 settings 中配置相关参数")

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/pipelines/bloom_dedup_pipeline.py

@@ -38,6 +38,7 @@ except ImportError:
 
 from crawlo import Item
 from crawlo.spider import Spider
+from crawlo.utils.fingerprint import FingerprintGenerator
 from crawlo.utils.log import get_logger
 from crawlo.exceptions import DropItem, ItemDiscard
 
@@ -109,6 +110,9 @@ class BloomDedupPipeline:
                 self.logger.debug(f"Processing new item: {fingerprint[:20]}...")
                 return item
                 
+        except ItemDiscard:
+            # 重新抛出ItemDiscard异常，确保管道管理器能正确处理
+            raise
         except Exception as e:
             self.logger.error(f"Error processing item: {e}")
             # 在错误时继续处理，避免丢失数据
@@ -123,21 +127,7 @@ class BloomDedupPipeline:
         :param item: 数据项
         :return: 指纹字符串
         """
-        # 将数据项转换为可序列化的字典
-        try:
-            item_dict = item.to_dict()
-        except AttributeError:
-            # 兼容没有to_dict方法的Item实现
-            item_dict = dict(item)
-        
-        # 对字典进行排序以确保一致性
-        sorted_items = sorted(item_dict.items())
-        
-        # 生成指纹字符串
-        fingerprint_string = '|'.join([f"{k}={v}" for k, v in sorted_items if v is not None])
-        
-        # 使用 SHA256 生成固定长度的指纹
-        return hashlib.sha256(fingerprint_string.encode('utf-8')).hexdigest()
+        return FingerprintGenerator.item_fingerprint(item)
 
     def close_spider(self, spider: Spider) -> None:
         """

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/pipelines/database_dedup_pipeline.py

@@ -17,6 +17,7 @@ import aiomysql
 from crawlo import Item
 from crawlo.exceptions import DropItem, ItemDiscard
 from crawlo.spider import Spider
+from crawlo.utils.fingerprint import FingerprintGenerator
 from crawlo.utils.log import get_logger
 
 
@@ -140,6 +141,9 @@ class DatabaseDedupPipeline:
                 self.logger.debug(f"Processing new item: {fingerprint[:20]}...")
                 return item
                 
+        except ItemDiscard:
+            # 重新抛出ItemDiscard异常，确保管道管理器能正确处理
+            raise
         except Exception as e:
             self.logger.error(f"Error processing item: {e}")
             # 在错误时继续处理，避免丢失数据
@@ -190,11 +194,4 @@ class DatabaseDedupPipeline:
         :param item: 数据项
         :return: 指纹字符串
         """
-        # 将数据项转换为可序列化的字典
-        try:
-            item_dict = item.to_dict()
-        except AttributeError:
-            # 兼容没有to_dict方法的Item实现
-            item_dict = dict(item)
-        
-        # 对字典进行排序以确保一致性
+        return FingerprintGenerator.item_fingerprint(item)

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/pipelines/memory_dedup_pipeline.py

@@ -18,6 +18,7 @@ from typing import Set
 from crawlo import Item
 from crawlo.exceptions import DropItem, ItemDiscard
 from crawlo.spider import Spider
+from crawlo.utils.fingerprint import FingerprintGenerator
 from crawlo.utils.log import get_logger
 
 
@@ -71,6 +72,9 @@ class MemoryDedupPipeline:
                 self.logger.debug(f"Processing new item: {fingerprint[:20]}...")
                 return item
                 
+        except ItemDiscard:
+            # 重新抛出ItemDiscard异常，确保管道管理器能正确处理
+            raise
         except Exception as e:
             self.logger.error(f"Error processing item: {e}")
             # 在错误时继续处理，避免丢失数据
@@ -85,21 +89,7 @@ class MemoryDedupPipeline:
         :param item: 数据项
         :return: 指纹字符串
         """
-        # 将数据项转换为可序列化的字典
-        try:
-            item_dict = item.to_dict()
-        except AttributeError:
-            # 兼容没有to_dict方法的Item实现
-            item_dict = dict(item)
-        
-        # 对字典进行排序以确保一致性
-        sorted_items = sorted(item_dict.items())
-        
-        # 生成指纹字符串
-        fingerprint_string = '|'.join([f"{k}={v}" for k, v in sorted_items if v is not None])
-        
-        # 使用 SHA256 生成固定长度的指纹
-        return hashlib.sha256(fingerprint_string.encode('utf-8')).hexdigest()
+        return FingerprintGenerator.item_fingerprint(item)
 
     def close_spider(self, spider: Spider) -> None:
         """

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/pipelines/redis_dedup_pipeline.py

@@ -18,6 +18,7 @@ from typing import Optional
 from crawlo import Item
 from crawlo.spider import Spider
 from crawlo.exceptions import DropItem, ItemDiscard
+from crawlo.utils.fingerprint import FingerprintGenerator
 from crawlo.utils.log import get_logger
 
 
@@ -132,21 +133,7 @@ class RedisDedupPipeline:
         :param item: 数据项
         :return: 指纹字符串
         """
-        # 将数据项转换为可序列化的字典
-        try:
-            item_dict = item.to_dict()
-        except AttributeError:
-            # 兼容没有to_dict方法的Item实现
-            item_dict = dict(item)
-        
-        # 对字典进行排序以确保一致性
-        sorted_items = sorted(item_dict.items())
-        
-        # 生成指纹字符串
-        fingerprint_string = '|'.join([f"{k}={v}" for k, v in sorted_items if v is not None])
-        
-        # 使用 SHA256 生成固定长度的指纹
-        return hashlib.sha256(fingerprint_string.encode('utf-8')).hexdigest()
+        return FingerprintGenerator.item_fingerprint(item)
 
     def close_spider(self, spider: Spider) -> None:
         """

{crawlo-1.4.0 → crawlo-1.4.2}/crawlo/project.py

@@ -289,11 +289,22 @@ def _load_project_settings(custom_settings: Optional[dict] = None) -> SettingMan
     if run_mode:
         from crawlo.mode_manager import ModeManager
         mode_manager = ModeManager()
-        mode_settings = mode_manager.resolve_mode_settings(run_mode)
-        # 合并模式配置，但不覆盖用户已设置的配置
+        # 获取项目名称并传递给模式配置
+        project_name = settings.get('PROJECT_NAME', 'crawlo')
+        mode_settings = mode_manager.resolve_mode_settings(run_mode, project_name=project_name)
+        
+        # 特殊处理：如果用户在settings.py中明确设置了QUEUE_TYPE为'auto'，
+        # 即使在单机模式下也应该保留这个设置
+        user_queue_type = settings.get('QUEUE_TYPE')
+        if user_queue_type == 'auto' and run_mode == 'standalone':
+            mode_settings['QUEUE_TYPE'] = 'auto'
+        
+        # 合并模式配置
         for key, value in mode_settings.items():
-            # 只有当用户没有设置该配置项时才应用模式配置
-            if key not in settings.attributes:
+            # 对于特定的配置项，模式配置应该优先于用户配置
+            # 特别是与运行模式密切相关的配置项
+            priority_keys = ['QUEUE_TYPE', 'FILTER_CLASS', 'DEFAULT_DEDUP_PIPELINE']
+            if key in priority_keys or key not in settings.attributes:
                 settings.set(key, value)
         _temp_debug(f"🔧 已应用 {run_mode} 模式配置")
 
@@ -311,9 +322,9 @@ def get_settings(custom_settings: Optional[dict] = None) -> SettingManager:
     获取配置管理器实例（主入口函数）
     
     注意：这个函数现在作为向后兼容的入口，实际的初始化逻辑已经移到
-    crawlo.core.framework_initializer 模块中。建议使用新的初始化方式：
+    crawlo.initialization 模块中。建议使用新的初始化方式：
     
-    >>> from crawlo.core.framework_initializer import initialize_framework
+    >>> from crawlo.initialization import initialize_framework
     >>> settings = initialize_framework(custom_settings)
 
     Args:
@@ -323,5 +334,5 @@ def get_settings(custom_settings: Optional[dict] = None) -> SettingManager:
         SettingManager: 已加载配置的实例
     """
     # 使用新的统一初始化管理器
-    from crawlo.core.framework_initializer import initialize_framework
+    from crawlo.initialization import initialize_framework
     return initialize_framework(custom_settings)