crawlo 1.1.5__tar.gz → 1.1.6__tar.gz

{crawlo-1.1.5/crawlo.egg-info → crawlo-1.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.1.5
+Version: 1.1.6
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

crawlo-1.1.6/crawlo/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.1.6"

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/commands/run.py

@@ -8,6 +8,7 @@
 import sys
 import asyncio
 import configparser
+import os
 from pathlib import Path
 from importlib import import_module
 
@@ -32,6 +33,12 @@ def get_project_root():
     向上查找 crawlo.cfg 来确定项目根目录
     """
     current = Path.cwd()
+    # 首先检查当前目录及其子目录
+    for root, dirs, files in os.walk(current):
+        if "crawlo.cfg" in files:
+            return Path(root)
+    
+    # 如果在子目录中没找到，再向上查找
     for _ in range(10):
         cfg = current / "crawlo.cfg"
         if cfg.exists():

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/commands/startproject.py

@@ -10,7 +10,7 @@ import re
 import sys
 import os
 from pathlib import Path
-from typing import Optional
+from typing import Optional, List
 
 # 添加项目根目录到路径，以便能够导入utils模块
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
@@ -56,6 +56,19 @@ TEMPLATE_TYPES = {
     'gentle': '温和模板 - 低负载配置，对目标网站友好'
 }
 
+# 可选的模块组件
+OPTIONAL_MODULES = {
+    'mysql': 'MySQL数据库支持',
+    'mongodb': 'MongoDB数据库支持',
+    'redis': 'Redis支持（分布式队列和去重）',
+    'proxy': '代理支持',
+    'monitoring': '监控和性能分析',
+    'dedup': '去重功能',
+    'httpx': 'HttpX下载器',
+    'aiohttp': 'AioHttp下载器',
+    'curl': 'CurlCffi下载器'
+}
+
 
 def show_error_panel(title, content):
     """显示错误面板的简单实现"""
@@ -84,9 +97,10 @@ def _render_template(tmpl_path, context):
     return content
 
 
-def _copytree_with_templates(src, dst, context, template_type='default'):
+def _copytree_with_templates(src, dst, context, template_type='default', modules: List[str] = None):
     """
     递归复制目录，将 .tmpl 文件渲染后复制（去除 .tmpl 后缀），其他文件直接复制。
+    支持选择性模块复制。
     """
     src_path = Path(src)
     dst_path = Path(dst)
@@ -96,24 +110,43 @@ def _copytree_with_templates(src, dst, context, template_type='default'):
         rel_path = item.relative_to(src_path)
         dst_item = dst_path / rel_path
 
+        # 检查是否应该包含此文件（基于模块选择）
+        if not _should_include_file(rel_path, modules):
+            continue
+
         if item.is_dir():
             dst_item.mkdir(parents=True, exist_ok=True)
         else:
             if item.suffix == '.tmpl':
+                rendered_content = None
                 # 处理特定模板类型的设置文件
-                if item.name == 'settings.py.tmpl' and template_type != 'default':
-                    # 使用特定模板类型的设置文件
-                    template_file_name = f'settings_{template_type}.py.tmpl'
-                    template_file_path = src_path / template_file_name
-                    if template_file_path.exists():
-                        rendered_content = _render_template(template_file_path, context)
+                if item.name == 'settings.py.tmpl':
+                    # 对于设置文件，根据模板类型选择相应的内容模板
+                    if template_type != 'default':
+                        # 使用特定模板类型的设置文件
+                        template_file_name = f'settings_{template_type}.py.tmpl'
+                        template_file_path = src_path / template_file_name
+                        if template_file_path.exists():
+                            rendered_content = _render_template(template_file_path, context)
+                        else:
+                            # 如果特定模板不存在，使用默认模板
+                            rendered_content = _render_template(item, context)
                     else:
-                        # 如果特定模板不存在，使用默认模板
+                        # 使用默认模板
                         rendered_content = _render_template(item, context)
+                # 跳过其他以 settings_ 开头的模板文件，避免重复处理
+                elif item.name.startswith('settings_') and item.name.endswith('.py.tmpl'):
+                    continue
                 else:
                     rendered_content = _render_template(item, context)
                 
-                final_dst = dst_item.with_suffix('')
+                # 确保设置文件始终命名为 settings.py
+                if item.name == 'settings.py.tmpl':
+                    # 特殊处理设置模板文件，统一生成为 settings.py
+                    final_dst = dst_item.parent / 'settings.py'
+                else:
+                    final_dst = dst_item.with_suffix('')
+                    
                 final_dst.parent.mkdir(parents=True, exist_ok=True)
                 with open(final_dst, 'w', encoding='utf-8') as f:
                     f.write(rendered_content)
@@ -121,6 +154,54 @@ def _copytree_with_templates(src, dst, context, template_type='default'):
                 shutil.copy2(item, dst_item)
 
 
+def _should_include_file(rel_path, modules: List[str]) -> bool:
+    """
+    根据选择的模块决定是否包含文件
+    """
+    if modules is None:
+        # 如果没有指定模块，则包含所有文件
+        return True
+    
+    # 基础文件始终包含
+    basic_files = [
+        '__init__.py.tmpl',
+        'settings.py.tmpl',
+        'spiders/__init__.py.tmpl',
+        'items.py.tmpl',
+        'middlewares.py.tmpl',
+        'run.py.tmpl'
+    ]
+    
+    path_str = str(rel_path).replace('\\', '/')
+    
+    # 始终包含基础文件
+    if path_str in basic_files:
+        return True
+    
+    # 根据模块选择包含特定文件
+    if 'mysql' in modules and 'mysql' in path_str:
+        return True
+    if 'mongodb' in modules and 'mongo' in path_str:
+        return True
+    if 'redis' in modules and 'redis' in path_str:
+        return True
+    if 'proxy' in modules and 'proxy' in path_str:
+        return True
+    if 'monitoring' in modules and ('monitor' in path_str or 'stats' in path_str):
+        return True
+    if 'dedup' in modules and 'dedup' in path_str:
+        return True
+    if 'httpx' in modules and 'httpx' in path_str:
+        return True
+    if 'aiohttp' in modules and 'aiohttp' in path_str:
+        return True
+    if 'curl' in modules and 'cffi' in path_str:
+        return True
+    
+    # 默认不包含特定模块文件
+    return False
+
+
 def validate_project_name(project_name: str) -> tuple[bool, str]:
     """
     验证项目名称是否有效
@@ -184,18 +265,52 @@ def show_template_options():
             print(f"  {template_type}: {description}")
 
 
+def show_module_options():
+    """显示可用的模块选项"""
+    if RICH_AVAILABLE:
+        table = Table(title="可选模块组件", show_header=True, header_style="bold magenta")
+        table.add_column("模块", style="cyan", no_wrap=True)
+        table.add_column("描述", style="green")
+        
+        for module, description in OPTIONAL_MODULES.items():
+            table.add_row(module, description)
+        
+        console.print(table)
+    else:
+        print("可选模块组件:")
+        for module, description in OPTIONAL_MODULES.items():
+            print(f"  {module}: {description}")
+
+
 def main(args):
-    if len(args) < 1 or len(args) > 2:
-        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo startproject[/blue] <project_name> [template_type]")
+    if len(args) < 1:
+        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo startproject[/blue] <project_name> [template_type] [--modules module1,module2]")
         console.print("💡 Examples:")
         console.print("   [blue]crawlo startproject[/blue] my_spider_project")
         console.print("   [blue]crawlo startproject[/blue] news_crawler simple")
-        console.print("   [blue]crawlo startproject[/blue] ecommerce_spider distributed")
+        console.print("   [blue]crawlo startproject[/blue] ecommerce_spider distributed --modules mysql,proxy")
         show_template_options()
+        show_module_options()
         return 1
 
+    # 解析参数
     project_name = args[0]
-    template_type = args[1] if len(args) > 1 else 'default'
+    template_type = 'default'
+    modules = None
+    
+    # 解析可选参数
+    if len(args) > 1:
+        for i, arg in enumerate(args[1:], 1):
+            if arg.startswith('--modules='):
+                modules_str = arg.split('=', 1)[1]
+                modules = [m.strip() for m in modules_str.split(',') if m.strip()]
+            elif arg.startswith('--modules'):
+                # 处理 --modules module1,module2 格式
+                if i + 1 < len(args):
+                    modules_str = args[i + 1]
+                    modules = [m.strip() for m in modules_str.split(',') if m.strip()]
+            elif not arg.startswith('--') and arg in TEMPLATE_TYPES:
+                template_type = arg
     
     # 验证模板类型
     if template_type not in TEMPLATE_TYPES:
@@ -249,7 +364,7 @@ def main(args):
 
         # 3. 复制并渲染项目包内容
         package_dir = project_dir / project_name
-        _copytree_with_templates(template_dir, package_dir, context, template_type)
+        _copytree_with_templates(template_dir, package_dir, context, template_type, modules)
         console.print(f":white_check_mark: Created project package: [green]{package_dir}[/green]")
 
         # 4. 创建 logs 目录
@@ -267,6 +382,10 @@ def main(args):
         # 显示使用的模板类型
         if template_type != 'default':
             console.print(f":information: 使用模板类型: [bold blue]{template_type}[/bold blue] - {TEMPLATE_TYPES[template_type]}")
+        
+        # 显示选择的模块
+        if modules:
+            console.print(f":information: 选择的模块: [bold blue]{', '.join(modules)}[/bold blue]")
 
         # 下一步操作提示（对齐美观 + 语法高亮）
         next_steps = f"""

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/config.py

@@ -158,6 +158,7 @@ class CrawloConfig:
         redis_host: str = '127.0.0.1',
         redis_port: int = 6379,
         redis_password: Optional[str] = None,
+        redis_db: int = 0,  # 添加 redis_db 参数
         project_name: str = 'crawlo',
         concurrency: int = 16,
         download_delay: float = 1.0,
@@ -170,6 +171,7 @@ class CrawloConfig:
             redis_host: Redis 服务器地址
             redis_port: Redis 端口
             redis_password: Redis 密码
+            redis_db: Redis 数据库编号
             project_name: 项目名称（用于命名空间）
             concurrency: 并发数
             download_delay: 下载延迟
@@ -179,6 +181,7 @@ class CrawloConfig:
             redis_host=redis_host,
             redis_port=redis_port,
             redis_password=redis_password,
+            redis_db=redis_db,  # 传递 redis_db 参数
             project_name=project_name,
             CONCURRENCY=concurrency,
             DOWNLOAD_DELAY=download_delay,

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/core/engine.py

@@ -70,7 +70,7 @@ class Engine(object):
     def engine_start(self):
         self.running = True
         self.logger.info(
-            f"Crawlo (version {self.settings.get_float('VERSION')}) started. "
+            f"Crawlo (version {self.settings.get('VERSION')}) started. "
             f"(project name : {self.settings.get('PROJECT_NAME')})"
         )
 

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/filters/aioredis_filter.py

@@ -75,7 +75,8 @@ class AioRedisFilter(BaseFilter):
     def create_instance(cls, crawler) -> 'BaseFilter':
         """从爬虫配置创建过滤器实例"""
         redis_url = crawler.settings.get('REDIS_URL', 'redis://localhost:6379')
-        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', False)
+        # 确保 decode_responses=False 以避免编码问题
+        decode_responses = False  # crawler.settings.get_bool('DECODE_RESPONSES', False)
         ttl_setting = crawler.settings.get_int('REDIS_TTL')
 
         # 处理TTL设置
@@ -84,7 +85,7 @@ class AioRedisFilter(BaseFilter):
             ttl = max(0, int(ttl_setting)) if ttl_setting > 0 else None
 
         try:
-            # 使用优化的连接池
+            # 使用优化的连接池，确保 decode_responses=False
             redis_pool = get_redis_pool(
                 redis_url,
                 max_connections=20,
@@ -92,7 +93,7 @@ class AioRedisFilter(BaseFilter):
                 socket_timeout=30,
                 health_check_interval=30,
                 retry_on_timeout=True,
-                decode_responses=decode_responses,
+                decode_responses=decode_responses,  # 确保不自动解码响应
                 encoding='utf-8'
             )
             

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/mode_manager.py

@@ -47,21 +47,24 @@ class ModeManager:
         redis_host: str = '127.0.0.1',
         redis_port: int = 6379,
         redis_password: Optional[str] = None,
+        redis_db: int = 0,  # 添加 redis_db 参数
         project_name: str = 'crawlo'
     ) -> Dict[str, Any]:
         """获取分布式模式配置"""
-        # 构建 Redis URL
+        # 构建 Redis URL，使用传入的 redis_db 参数
         if redis_password:
-            redis_url = f'redis://:{redis_password}@{redis_host}:{redis_port}/0'
+            redis_url = f'redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}'
         else:
-            redis_url = f'redis://{redis_host}:{redis_port}/0'
+            redis_url = f'redis://{redis_host}:{redis_port}/{redis_db}'
         
         return {
+            'PROJECT_NAME': project_name,  # 添加项目名称到配置中
             'QUEUE_TYPE': 'redis',
             'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
             'REDIS_HOST': redis_host,
             'REDIS_PORT': redis_port,
             'REDIS_PASSWORD': redis_password,
+            'REDIS_DB': redis_db,  # 添加 Redis 数据库编号到配置中
             'REDIS_URL': redis_url,
             'SCHEDULER_QUEUE_NAME': f'crawlo:{project_name}:queue:requests',  # 使用统一命名规范
             # Redis key配置已移至各组件中，使用统一的命名规范
@@ -111,6 +114,7 @@ class ModeManager:
                 redis_host=kwargs.get('redis_host', '127.0.0.1'),
                 redis_port=kwargs.get('redis_port', 6379),
                 redis_password=kwargs.get('redis_password'),
+                redis_db=kwargs.get('redis_db', 0),  # 添加 redis_db 参数
                 project_name=kwargs.get('project_name', 'crawlo')
             )
             
@@ -160,6 +164,7 @@ def distributed_mode(
     redis_host: str = '127.0.0.1',
     redis_port: int = 6379,
     redis_password: Optional[str] = None,
+    redis_db: int = 0,  # 添加 redis_db 参数
     project_name: str = 'crawlo',
     **kwargs
 ) -> Dict[str, Any]:
@@ -169,6 +174,7 @@ def distributed_mode(
         redis_host=redis_host,
         redis_port=redis_port,
         redis_password=redis_password,
+        redis_db=redis_db,  # 传递 redis_db 参数
         project_name=project_name,
         **kwargs
     )

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/pipelines/pipeline_manager.py

@@ -30,12 +30,17 @@ class PipelineManager:
 
     def _add_pipelines(self, pipelines):
         for pipeline in pipelines:
-            pipeline_cls = load_class(pipeline)
-            if not hasattr(pipeline_cls, 'from_crawler'):
-                raise PipelineInitError(
-                    f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
-                )
-            self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
+            try:
+                pipeline_cls = load_class(pipeline)
+                if not hasattr(pipeline_cls, 'from_crawler'):
+                    raise PipelineInitError(
+                        f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
+                    )
+                self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
+            except Exception as e:
+                self.logger.error(f"Failed to load pipeline {pipeline}: {e}")
+                # 可以选择继续加载其他管道或抛出异常
+                raise
         if pipelines:
             self.logger.info(f"enabled pipelines: \n {pformat(pipelines)}")
 

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/pipelines/redis_dedup_pipeline.py

@@ -13,13 +13,14 @@
 """
 
 import hashlib
-from typing import Dict, Any, Optional
+from typing import Optional
+
 import redis
 
 from crawlo import Item
+from crawlo.exceptions import DropItem
 from crawlo.spider import Spider
 from crawlo.utils.log import get_logger
-from crawlo.exceptions import DropItem
 
 
 class RedisDedupPipeline:

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/project.py

@@ -30,6 +30,15 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
         2. 存在 '__init__.py' 和 'settings.py'（即 Python 包）
     """
     path = os.path.abspath(start_path)
+    
+    # 首先检查当前目录及其子目录
+    for root, dirs, files in os.walk(path):
+        if "crawlo.cfg" in files:
+            cfg_path = os.path.join(root, "crawlo.cfg")
+            logger.info(f"✅ 找到项目配置文件: {cfg_path}")
+            return root
+    
+    # 如果在子目录中没找到，再向上查找
     while True:
         cfg_file = os.path.join(path, "crawlo.cfg")
         if os.path.isfile(cfg_file):
@@ -128,17 +137,43 @@ def load_class(_path):
             raise TypeError(f"args expect str or object, got {_path}")
 
     module_name, class_name = _path.rsplit('.', 1)
-    module = import_module(module_name)
+    
+    try:
+        module = import_module(module_name)
+    except ImportError as e:
+        # 尝试不同的导入方式
+        try:
+            # 尝试直接导入完整路径
+            module = import_module(_path)
+            return module
+        except ImportError:
+            pass
+        raise ImportError(f"Cannot import module {module_name}: {e}")
 
     try:
         cls = getattr(module, class_name)
     except AttributeError:
-        raise NameError(f"Module {module_name!r} has no class named {class_name!r}")
+        # 提供更详细的错误信息
+        available_attrs = [attr for attr in dir(module) if not attr.startswith('_')]
+        raise NameError(f"Module {module_name!r} has no class named {class_name!r}. Available attributes: {available_attrs}")
     return cls
 
 
 def merge_settings(spider, settings):
     spider_name = getattr(spider, 'name', 'UnknownSpider')
+    # 检查 settings 是否为 SettingManager 实例
+    if not hasattr(settings, 'update_attributes'):
+        logger.error(f"merge_settings 接收到的 settings 不是 SettingManager 实例: {type(settings)}")
+        # 如果是字典，创建一个新的 SettingManager 实例
+        if isinstance(settings, dict):
+            from crawlo.settings.setting_manager import SettingManager
+            new_settings = SettingManager()
+            new_settings.update_attributes(settings)
+            settings = new_settings
+        else:
+            logger.error("无法处理的 settings 类型")
+            return
+            
     if hasattr(spider, 'custom_settings'):
         custom_settings = getattr(spider, 'custom_settings')
         settings.update_attributes(custom_settings)
@@ -150,4 +185,4 @@ async def common_call(func: Callable, *args, **kwargs):
     if iscoroutinefunction(func):
         return await func(*args, **kwargs)
     else:
-        return func(*args, **kwargs)
+        return func(*args, **kwargs)

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/queue/queue_manager.py

@@ -267,7 +267,21 @@ class QueueManager:
             if ':' in self.config.queue_name:
                 parts = self.config.queue_name.split(':')
                 if len(parts) >= 2:
-                    project_name = parts[1]  # 取第二个部分作为项目名称
+                    # 处理可能的双重 crawlo 前缀
+                    if parts[0] == "crawlo" and parts[1] == "crawlo":
+                        # 双重 crawlo 前缀，取第三个部分作为项目名称
+                        if len(parts) >= 3:
+                            project_name = parts[2]
+                        else:
+                            project_name = "default"
+                    elif parts[0] == "crawlo":
+                        # 正常的 crawlo 前缀，取第二个部分作为项目名称
+                        project_name = parts[1]
+                    else:
+                        # 没有 crawlo 前缀，使用第一个部分作为项目名称
+                        project_name = parts[0]
+                else:
+                    project_name = self.config.queue_name or "default"
             else:
                 project_name = self.config.queue_name or "default"
             

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/queue/redis_priority_queue.py

@@ -1,17 +1,16 @@
+import asyncio
 import pickle
 import time
-import asyncio
+import traceback
 from typing import Optional
+
 import redis.asyncio as aioredis
-import traceback
-import os
 
 from crawlo import Request
-from crawlo.utils.log import get_logger
-from crawlo.utils.request_serializer import RequestSerializer
 from crawlo.utils.error_handler import ErrorHandler
+from crawlo.utils.log import get_logger
 from crawlo.utils.redis_connection_pool import get_redis_pool, OptimizedRedisConnectionPool
-
+from crawlo.utils.request_serializer import RequestSerializer
 
 logger = get_logger(__name__)
 error_handler = ErrorHandler(__name__)
@@ -45,11 +44,8 @@ class RedisPriorityQueue:
         if queue_name is None:
             self.queue_name = f"crawlo:{module_name}:queue:requests"
         else:
-            # 如果提供了 queue_name，但不符合规范格式，则转换为规范格式
-            if not queue_name.startswith("crawlo:"):
-                self.queue_name = f"crawlo:{module_name}:queue:requests"
-            else:
-                self.queue_name = queue_name
+            # 保持用户提供的队列名称不变，不做修改
+            self.queue_name = queue_name
         
         # 如果未提供 processing_queue，则根据 queue_name 自动生成
         if processing_queue is None:
@@ -85,14 +81,16 @@ class RedisPriorityQueue:
 
             for attempt in range(max_retries):
                 try:
-                    # 使用优化的连接池
+                    # 使用优化的连接池，确保 decode_responses=False 以避免编码问题
                     self._redis_pool = get_redis_pool(
                         self.redis_url,
                         max_connections=self.max_connections,
                         socket_connect_timeout=5,
                         socket_timeout=30,
                         health_check_interval=30,
-                        retry_on_timeout=True
+                        retry_on_timeout=True,
+                        decode_responses=False,  # 确保不自动解码响应
+                        encoding='utf-8'
                     )
                     
                     self._redis = await self._redis_pool.get_connection()
@@ -131,7 +129,15 @@ class RedisPriorityQueue:
             # 🔥 使用专用的序列化工具清理 Request
             clean_request = self.request_serializer.prepare_for_serialization(request)
             
-            serialized = pickle.dumps(clean_request)
+            # 确保序列化后的数据可以被正确反序列化
+            try:
+                serialized = pickle.dumps(clean_request)
+                # 验证序列化数据可以被反序列化
+                pickle.loads(serialized)
+            except Exception as serialize_error:
+                logger.error(f"❌ 请求序列化验证失败 (Module: {self.module_name}): {serialize_error}")
+                return False
+            
             pipe = self._redis.pipeline()
             pipe.zadd(self.queue_name, {key: score})
             pipe.hset(f"{self.queue_name}:data", key, serialized)
@@ -174,7 +180,23 @@ class RedisPriorityQueue:
                     pipe.hdel(f"{self.queue_name}:data", key)
                     await pipe.execute()
 
-                    return pickle.loads(serialized)
+                    # 更安全的反序列化方式
+                    try:
+                        # 首先尝试标准的 pickle 反序列化
+                        request = pickle.loads(serialized)
+                        return request
+                    except UnicodeDecodeError:
+                        # 如果出现编码错误，尝试使用 latin1 解码
+                        request = pickle.loads(serialized, encoding='latin1')
+                        return request
+                    except Exception as pickle_error:
+                        # 如果pickle反序列化失败，记录错误并跳过这个任务
+                        logger.error(f"❌ 无法反序列化请求数据 (Module: {self.module_name}): {pickle_error}")
+                        # 从processing队列中移除这个无效的任务
+                        await self._redis.zrem(self.processing_queue, processing_key)
+                        await self._redis.hdel(f"{self.processing_queue}:data", processing_key)
+                        # 继续尝试下一个任务
+                        continue
 
                 # 检查是否超时
                 if asyncio.get_event_loop().time() - start_time > timeout:

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/settings/default_settings.py

@@ -25,6 +25,9 @@ CONCURRENCY = get_runtime_config()['CONCURRENCY']
 
 # ============================== 爬虫核心配置 ==============================
 
+# 默认下载器
+DOWNLOADER = 'crawlo.downloader.aiohttp_downloader.AioHttpDownloader'
+
 # 请求延迟（秒）
 DOWNLOAD_DELAY = 1
 

{crawlo-1.1.5 → crawlo-1.1.6}/crawlo/settings/setting_manager.py

@@ -97,4 +97,27 @@ class SettingManager(MutableMapping):
                 self.set(key, value)
 
     def copy(self):
-        return deepcopy(self)
+        return deepcopy(self)
+
+    def __deepcopy__(self, memo):
+        """
+        自定义深度复制方法，避免复制logger等不可pickle的对象
+        """
+        # 创建一个新的实例
+        cls = self.__class__
+        new_instance = cls.__new__(cls)
+        
+        # 复制attributes字典，但排除不可pickle的对象
+        new_attributes = {}
+        for key, value in self.attributes.items():
+            try:
+                # 尝试深度复制值
+                new_attributes[key] = deepcopy(value, memo)
+            except Exception:
+                # 如果复制失败，保留原始引用（对于logger等对象）
+                new_attributes[key] = value
+        
+        # 设置新实例的attributes
+        new_instance.attributes = new_attributes
+        
+        return new_instance

{crawlo-1.1.5 → crawlo-1.1.6/crawlo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.1.5
+Version: 1.1.6
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder