crawlo 1.1.5__py3-none-any.whl → 1.1.8__py3-none-any.whl

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = "1.1.5"
+__version__ = "1.1.8"

crawlo/cli.py

@@ -10,24 +10,49 @@ def main():
     # 获取所有可用命令
     commands = get_commands()
 
+    # 创建主解析器
     parser = argparse.ArgumentParser(
         description="Crawlo: A lightweight web crawler framework.",
-        usage="crawlo <command> [options]"
+        usage="crawlo <command> [options]",
+        add_help=False  # 禁用默认帮助，我们自己处理
     )
-    parser.add_argument('command', help='Available commands: ' + ', '.join(commands.keys()))
-    # 注意：这里不添加具体参数，由子命令解析
-
-    # 只解析命令
+    
+    # 添加帮助参数
+    parser.add_argument('-h', '--help', action='store_true', help='显示帮助信息')
+    parser.add_argument('command', nargs='?', help='可用命令: ' + ', '.join(commands.keys()))
+    
+    # 解析已知参数
     args, unknown = parser.parse_known_args()
 
+    # 处理帮助参数
+    if args.help or (args.command is None and not unknown):
+        # 导入并运行帮助命令
+        try:
+            module = __import__(commands['help'], fromlist=['main'])
+            sys.exit(module.main([]))
+        except ImportError as e:
+            print(f"Failed to load help command: {e}")
+            sys.exit(1)
+        except Exception as e:
+            print(f"Help command failed: {e}")
+            sys.exit(1)
+
+    # 检查命令是否存在
     if args.command not in commands:
         print(f"Unknown command: {args.command}")
         print(f"Available commands: {', '.join(commands.keys())}")
+        # 显示帮助信息
+        try:
+            module = __import__(commands['help'], fromlist=['main'])
+            module.main([])
+        except:
+            pass
         sys.exit(1)
 
     # 动态导入并执行命令
     try:
         module = __import__(commands[args.command], fromlist=['main'])
+        # 将未知参数传递给子命令
         sys.exit(module.main(unknown))
     except ImportError as e:
         print(f"Failed to load command '{args.command}': {e}")

crawlo/commands/__init__.py

@@ -7,7 +7,8 @@ _commands = {
     'run': 'crawlo.commands.run',
     'check': 'crawlo.commands.check',
     'list': 'crawlo.commands.list',
-    'stats': 'crawlo.commands.stats'
+    'stats': 'crawlo.commands.stats',
+    'help': 'crawlo.commands.help'
 }
 
 def get_commands():

crawlo/commands/help.py

@@ -0,0 +1,133 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-12
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo -h|--help，显示帮助信息。
+"""
+import sys
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich.text import Text
+from rich import box
+
+console = Console()
+
+
+def main(args):
+    """
+    主函数：显示帮助信息
+    用法:
+        crawlo -h|--help
+    """
+    # 检查是否有无效参数
+    if args and args[0] not in ['-h', '--help', 'help']:
+        console.print("[bold red]❌ 无效参数:[/bold red] [yellow]{}[/yellow]".format(args[0]))
+        console.print("[bold blue]💡 提示:[/bold blue] 使用 [green]crawlo -h[/green] 或 [green]crawlo --help[/green] 查看帮助信息")
+        return 1
+
+    # 显示帮助信息
+    show_help()
+    return 0
+
+
+def show_help():
+    """显示完整的帮助信息"""
+    # 显示框架标题和版本
+    console.print(Panel(
+        Text.from_markup(":spider_web: [bold blue]Crawlo[/bold blue] [bold white]v1.1.7[/bold white] - 异步爬虫框架"),
+        expand=False,
+        border_style="blue"
+    ))
+    
+    # 显示基本用法
+    console.print("[bold green]基本用法:[/bold green]")
+    console.print("  [blue]crawlo[/blue] [cyan]<command>[/cyan] [options]")
+    console.print()
+    
+    # 显示可用命令
+    console.print("[bold green]可用命令:[/bold green]")
+    table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta")
+    table.add_column("命令", style="cyan", width=15)
+    table.add_column("描述", style="white")
+    table.add_column("用法", style="yellow")
+    
+    table.add_row("startproject", "创建新项目", "crawlo startproject <project_name>")
+    table.add_row("genspider", "生成爬虫模板", "crawlo genspider <spider_name> [domain]")
+    table.add_row("run", "运行爬虫", "crawlo run <spider_name>|all [options]")
+    table.add_row("check", "检查爬虫代码", "crawlo check [options]")
+    table.add_row("list", "列出所有爬虫", "crawlo list")
+    table.add_row("stats", "查看统计信息", "crawlo stats [spider_name]")
+    table.add_row("help", "显示帮助信息", "crawlo -h|--help")
+    
+    console.print(table)
+    console.print()
+    
+    # 显示全局选项
+    console.print("[bold green]全局选项:[/bold green]")
+    table = Table(box=box.SIMPLE, show_header=False)
+    table.add_column("选项", style="cyan", width=15)
+    table.add_column("描述", style="white")
+    
+    table.add_row("-h, --help", "显示帮助信息")
+    table.add_row("-v, --version", "显示版本信息")
+    
+    console.print(table)
+    console.print()
+    
+    # 显示各命令的详细用法
+    console.print("[bold green]命令详细用法:[/bold green]")
+    
+    # run 命令
+    console.print("[bold cyan]run[/bold cyan] - 运行爬虫")
+    console.print("  用法: crawlo run <spider_name>|all [--json] [--no-stats]")
+    console.print("  示例:")
+    console.print("    crawlo run myspider")
+    console.print("    crawlo run all")
+    console.print("    crawlo run all --json --no-stats")
+    console.print()
+    
+    # check 命令
+    console.print("[bold cyan]check[/bold cyan] - 检查爬虫代码")
+    console.print("  用法: crawlo check [--fix] [--ci] [--json] [--watch]")
+    console.print("  示例:")
+    console.print("    crawlo check")
+    console.print("    crawlo check --fix")
+    console.print("    crawlo check --ci --json")
+    console.print()
+    
+    # startproject 命令
+    console.print("[bold cyan]startproject[/bold cyan] - 创建新项目")
+    console.print("  用法: crawlo startproject <project_name>")
+    console.print("  示例:")
+    console.print("    crawlo startproject myproject")
+    console.print()
+    
+    # genspider 命令
+    console.print("[bold cyan]genspider[/bold cyan] - 生成爬虫模板")
+    console.print("  用法: crawlo genspider <spider_name> [domain]")
+    console.print("  示例:")
+    console.print("    crawlo genspider myspider example.com")
+    console.print()
+    
+    # list 命令
+    console.print("[bold cyan]list[/bold cyan] - 列出所有爬虫")
+    console.print("  用法: crawlo list")
+    console.print("  示例:")
+    console.print("    crawlo list")
+    console.print()
+    
+    # stats 命令
+    console.print("[bold cyan]stats[/bold cyan] - 查看统计信息")
+    console.print("  用法: crawlo stats [spider_name]")
+    console.print("  示例:")
+    console.print("    crawlo stats")
+    console.print("    crawlo stats myspider")
+    console.print()
+    
+    # 显示更多信息
+    console.print("[bold green]更多信息:[/bold green]")
+    console.print("  文档: https://crawlo.readthedocs.io/")
+    console.print("  源码: https://github.com/crawl-coder/Crawlo")
+    console.print("  问题: https://github.com/crawl-coder/Crawlo/issues")

crawlo/commands/run.py

@@ -8,6 +8,7 @@
 import sys
 import asyncio
 import configparser
+import os
 from pathlib import Path
 from importlib import import_module
 
@@ -32,6 +33,12 @@ def get_project_root():
     向上查找 crawlo.cfg 来确定项目根目录
     """
     current = Path.cwd()
+    # 首先检查当前目录及其子目录
+    for root, dirs, files in os.walk(current):
+        if "crawlo.cfg" in files:
+            return Path(root)
+    
+    # 如果在子目录中没找到，再向上查找
     for _ in range(10):
         cfg = current / "crawlo.cfg"
         if cfg.exists():

crawlo/commands/startproject.py

@@ -10,7 +10,7 @@ import re
 import sys
 import os
 from pathlib import Path
-from typing import Optional
+from typing import Optional, List
 
 # 添加项目根目录到路径，以便能够导入utils模块
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
@@ -56,6 +56,19 @@ TEMPLATE_TYPES = {
     'gentle': '温和模板 - 低负载配置，对目标网站友好'
 }
 
+# 可选的模块组件
+OPTIONAL_MODULES = {
+    'mysql': 'MySQL数据库支持',
+    'mongodb': 'MongoDB数据库支持',
+    'redis': 'Redis支持（分布式队列和去重）',
+    'proxy': '代理支持',
+    'monitoring': '监控和性能分析',
+    'dedup': '去重功能',
+    'httpx': 'HttpX下载器',
+    'aiohttp': 'AioHttp下载器',
+    'curl': 'CurlCffi下载器'
+}
+
 
 def show_error_panel(title, content):
     """显示错误面板的简单实现"""
@@ -84,9 +97,10 @@ def _render_template(tmpl_path, context):
     return content
 
 
-def _copytree_with_templates(src, dst, context, template_type='default'):
+def _copytree_with_templates(src, dst, context, template_type='default', modules: List[str] = None):
     """
     递归复制目录，将 .tmpl 文件渲染后复制（去除 .tmpl 后缀），其他文件直接复制。
+    支持选择性模块复制。
     """
     src_path = Path(src)
     dst_path = Path(dst)
@@ -96,24 +110,43 @@ def _copytree_with_templates(src, dst, context, template_type='default'):
         rel_path = item.relative_to(src_path)
         dst_item = dst_path / rel_path
 
+        # 检查是否应该包含此文件（基于模块选择）
+        if not _should_include_file(rel_path, modules):
+            continue
+
         if item.is_dir():
             dst_item.mkdir(parents=True, exist_ok=True)
         else:
             if item.suffix == '.tmpl':
+                rendered_content = None
                 # 处理特定模板类型的设置文件
-                if item.name == 'settings.py.tmpl' and template_type != 'default':
-                    # 使用特定模板类型的设置文件
-                    template_file_name = f'settings_{template_type}.py.tmpl'
-                    template_file_path = src_path / template_file_name
-                    if template_file_path.exists():
-                        rendered_content = _render_template(template_file_path, context)
+                if item.name == 'settings.py.tmpl':
+                    # 对于设置文件，根据模板类型选择相应的内容模板
+                    if template_type != 'default':
+                        # 使用特定模板类型的设置文件
+                        template_file_name = f'settings_{template_type}.py.tmpl'
+                        template_file_path = src_path / template_file_name
+                        if template_file_path.exists():
+                            rendered_content = _render_template(template_file_path, context)
+                        else:
+                            # 如果特定模板不存在，使用默认模板
+                            rendered_content = _render_template(item, context)
                     else:
-                        # 如果特定模板不存在，使用默认模板
+                        # 使用默认模板
                         rendered_content = _render_template(item, context)
+                # 跳过其他以 settings_ 开头的模板文件，避免重复处理
+                elif item.name.startswith('settings_') and item.name.endswith('.py.tmpl'):
+                    continue
                 else:
                     rendered_content = _render_template(item, context)
                 
-                final_dst = dst_item.with_suffix('')
+                # 确保设置文件始终命名为 settings.py
+                if item.name == 'settings.py.tmpl':
+                    # 特殊处理设置模板文件，统一生成为 settings.py
+                    final_dst = dst_item.parent / 'settings.py'
+                else:
+                    final_dst = dst_item.with_suffix('')
+                    
                 final_dst.parent.mkdir(parents=True, exist_ok=True)
                 with open(final_dst, 'w', encoding='utf-8') as f:
                     f.write(rendered_content)
@@ -121,6 +154,54 @@ def _copytree_with_templates(src, dst, context, template_type='default'):
                 shutil.copy2(item, dst_item)
 
 
+def _should_include_file(rel_path, modules: List[str]) -> bool:
+    """
+    根据选择的模块决定是否包含文件
+    """
+    if modules is None:
+        # 如果没有指定模块，则包含所有文件
+        return True
+    
+    # 基础文件始终包含
+    basic_files = [
+        '__init__.py.tmpl',
+        'settings.py.tmpl',
+        'spiders/__init__.py.tmpl',
+        'items.py.tmpl',
+        'middlewares.py.tmpl',
+        'run.py.tmpl'
+    ]
+    
+    path_str = str(rel_path).replace('\\', '/')
+    
+    # 始终包含基础文件
+    if path_str in basic_files:
+        return True
+    
+    # 根据模块选择包含特定文件
+    if 'mysql' in modules and 'mysql' in path_str:
+        return True
+    if 'mongodb' in modules and 'mongo' in path_str:
+        return True
+    if 'redis' in modules and 'redis' in path_str:
+        return True
+    if 'proxy' in modules and 'proxy' in path_str:
+        return True
+    if 'monitoring' in modules and ('monitor' in path_str or 'stats' in path_str):
+        return True
+    if 'dedup' in modules and 'dedup' in path_str:
+        return True
+    if 'httpx' in modules and 'httpx' in path_str:
+        return True
+    if 'aiohttp' in modules and 'aiohttp' in path_str:
+        return True
+    if 'curl' in modules and 'cffi' in path_str:
+        return True
+    
+    # 默认不包含特定模块文件
+    return False
+
+
 def validate_project_name(project_name: str) -> tuple[bool, str]:
     """
     验证项目名称是否有效
@@ -184,18 +265,52 @@ def show_template_options():
             print(f"  {template_type}: {description}")
 
 
+def show_module_options():
+    """显示可用的模块选项"""
+    if RICH_AVAILABLE:
+        table = Table(title="可选模块组件", show_header=True, header_style="bold magenta")
+        table.add_column("模块", style="cyan", no_wrap=True)
+        table.add_column("描述", style="green")
+        
+        for module, description in OPTIONAL_MODULES.items():
+            table.add_row(module, description)
+        
+        console.print(table)
+    else:
+        print("可选模块组件:")
+        for module, description in OPTIONAL_MODULES.items():
+            print(f"  {module}: {description}")
+
+
 def main(args):
-    if len(args) < 1 or len(args) > 2:
-        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo startproject[/blue] <project_name> [template_type]")
+    if len(args) < 1:
+        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo startproject[/blue] <project_name> [template_type] [--modules module1,module2]")
         console.print("💡 Examples:")
         console.print("   [blue]crawlo startproject[/blue] my_spider_project")
         console.print("   [blue]crawlo startproject[/blue] news_crawler simple")
-        console.print("   [blue]crawlo startproject[/blue] ecommerce_spider distributed")
+        console.print("   [blue]crawlo startproject[/blue] ecommerce_spider distributed --modules mysql,proxy")
         show_template_options()
+        show_module_options()
         return 1
 
+    # 解析参数
     project_name = args[0]
-    template_type = args[1] if len(args) > 1 else 'default'
+    template_type = 'default'
+    modules = None
+    
+    # 解析可选参数
+    if len(args) > 1:
+        for i, arg in enumerate(args[1:], 1):
+            if arg.startswith('--modules='):
+                modules_str = arg.split('=', 1)[1]
+                modules = [m.strip() for m in modules_str.split(',') if m.strip()]
+            elif arg.startswith('--modules'):
+                # 处理 --modules module1,module2 格式
+                if i + 1 < len(args):
+                    modules_str = args[i + 1]
+                    modules = [m.strip() for m in modules_str.split(',') if m.strip()]
+            elif not arg.startswith('--') and arg in TEMPLATE_TYPES:
+                template_type = arg
     
     # 验证模板类型
     if template_type not in TEMPLATE_TYPES:
@@ -249,7 +364,7 @@ def main(args):
 
         # 3. 复制并渲染项目包内容
         package_dir = project_dir / project_name
-        _copytree_with_templates(template_dir, package_dir, context, template_type)
+        _copytree_with_templates(template_dir, package_dir, context, template_type, modules)
         console.print(f":white_check_mark: Created project package: [green]{package_dir}[/green]")
 
         # 4. 创建 logs 目录
@@ -267,6 +382,10 @@ def main(args):
         # 显示使用的模板类型
         if template_type != 'default':
             console.print(f":information: 使用模板类型: [bold blue]{template_type}[/bold blue] - {TEMPLATE_TYPES[template_type]}")
+        
+        # 显示选择的模块
+        if modules:
+            console.print(f":information: 选择的模块: [bold blue]{', '.join(modules)}[/bold blue]")
 
         # 下一步操作提示（对齐美观 + 语法高亮）
         next_steps = f"""

crawlo/config.py

@@ -158,6 +158,7 @@ class CrawloConfig:
         redis_host: str = '127.0.0.1',
         redis_port: int = 6379,
         redis_password: Optional[str] = None,
+        redis_db: int = 0,  # 添加 redis_db 参数
         project_name: str = 'crawlo',
         concurrency: int = 16,
         download_delay: float = 1.0,
@@ -170,6 +171,7 @@ class CrawloConfig:
             redis_host: Redis 服务器地址
             redis_port: Redis 端口
             redis_password: Redis 密码
+            redis_db: Redis 数据库编号
             project_name: 项目名称（用于命名空间）
             concurrency: 并发数
             download_delay: 下载延迟
@@ -179,6 +181,7 @@ class CrawloConfig:
             redis_host=redis_host,
             redis_port=redis_port,
             redis_password=redis_password,
+            redis_db=redis_db,  # 传递 redis_db 参数
             project_name=project_name,
             CONCURRENCY=concurrency,
             DOWNLOAD_DELAY=download_delay,

crawlo/core/engine.py

@@ -70,7 +70,7 @@ class Engine(object):
     def engine_start(self):
         self.running = True
         self.logger.info(
-            f"Crawlo (version {self.settings.get_float('VERSION')}) started. "
+            f"Crawlo (version {self.settings.get('VERSION')}) started. "
             f"(project name : {self.settings.get('PROJECT_NAME')})"
         )
 

crawlo/filters/aioredis_filter.py

@@ -75,7 +75,8 @@ class AioRedisFilter(BaseFilter):
     def create_instance(cls, crawler) -> 'BaseFilter':
         """从爬虫配置创建过滤器实例"""
         redis_url = crawler.settings.get('REDIS_URL', 'redis://localhost:6379')
-        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', False)
+        # 确保 decode_responses=False 以避免编码问题
+        decode_responses = False  # crawler.settings.get_bool('DECODE_RESPONSES', False)
         ttl_setting = crawler.settings.get_int('REDIS_TTL')
 
         # 处理TTL设置
@@ -84,7 +85,7 @@ class AioRedisFilter(BaseFilter):
             ttl = max(0, int(ttl_setting)) if ttl_setting > 0 else None
 
         try:
-            # 使用优化的连接池
+            # 使用优化的连接池，确保 decode_responses=False
             redis_pool = get_redis_pool(
                 redis_url,
                 max_connections=20,
@@ -92,7 +93,7 @@ class AioRedisFilter(BaseFilter):
                 socket_timeout=30,
                 health_check_interval=30,
                 retry_on_timeout=True,
-                decode_responses=decode_responses,
+                decode_responses=decode_responses,  # 确保不自动解码响应
                 encoding='utf-8'
             )
             

crawlo/mode_manager.py

@@ -47,21 +47,24 @@ class ModeManager:
         redis_host: str = '127.0.0.1',
         redis_port: int = 6379,
         redis_password: Optional[str] = None,
+        redis_db: int = 0,  # 添加 redis_db 参数
         project_name: str = 'crawlo'
     ) -> Dict[str, Any]:
         """获取分布式模式配置"""
-        # 构建 Redis URL
+        # 构建 Redis URL，使用传入的 redis_db 参数
         if redis_password:
-            redis_url = f'redis://:{redis_password}@{redis_host}:{redis_port}/0'
+            redis_url = f'redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}'
         else:
-            redis_url = f'redis://{redis_host}:{redis_port}/0'
+            redis_url = f'redis://{redis_host}:{redis_port}/{redis_db}'
         
         return {
+            'PROJECT_NAME': project_name,  # 添加项目名称到配置中
             'QUEUE_TYPE': 'redis',
             'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
             'REDIS_HOST': redis_host,
             'REDIS_PORT': redis_port,
             'REDIS_PASSWORD': redis_password,
+            'REDIS_DB': redis_db,  # 添加 Redis 数据库编号到配置中
             'REDIS_URL': redis_url,
             'SCHEDULER_QUEUE_NAME': f'crawlo:{project_name}:queue:requests',  # 使用统一命名规范
             # Redis key配置已移至各组件中，使用统一的命名规范
@@ -111,6 +114,7 @@ class ModeManager:
                 redis_host=kwargs.get('redis_host', '127.0.0.1'),
                 redis_port=kwargs.get('redis_port', 6379),
                 redis_password=kwargs.get('redis_password'),
+                redis_db=kwargs.get('redis_db', 0),  # 添加 redis_db 参数
                 project_name=kwargs.get('project_name', 'crawlo')
             )
             
@@ -160,6 +164,7 @@ def distributed_mode(
     redis_host: str = '127.0.0.1',
     redis_port: int = 6379,
     redis_password: Optional[str] = None,
+    redis_db: int = 0,  # 添加 redis_db 参数
     project_name: str = 'crawlo',
     **kwargs
 ) -> Dict[str, Any]:
@@ -169,6 +174,7 @@ def distributed_mode(
         redis_host=redis_host,
         redis_port=redis_port,
         redis_password=redis_password,
+        redis_db=redis_db,  # 传递 redis_db 参数
         project_name=project_name,
         **kwargs
     )

crawlo/pipelines/pipeline_manager.py

@@ -30,12 +30,17 @@ class PipelineManager:
 
     def _add_pipelines(self, pipelines):
         for pipeline in pipelines:
-            pipeline_cls = load_class(pipeline)
-            if not hasattr(pipeline_cls, 'from_crawler'):
-                raise PipelineInitError(
-                    f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
-                )
-            self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
+            try:
+                pipeline_cls = load_class(pipeline)
+                if not hasattr(pipeline_cls, 'from_crawler'):
+                    raise PipelineInitError(
+                        f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
+                    )
+                self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
+            except Exception as e:
+                self.logger.error(f"Failed to load pipeline {pipeline}: {e}")
+                # 可以选择继续加载其他管道或抛出异常
+                raise
         if pipelines:
             self.logger.info(f"enabled pipelines: \n {pformat(pipelines)}")
 

crawlo/pipelines/redis_dedup_pipeline.py

@@ -13,13 +13,14 @@
 """
 
 import hashlib
-from typing import Dict, Any, Optional
+from typing import Optional
+
 import redis
 
 from crawlo import Item
+from crawlo.exceptions import DropItem
 from crawlo.spider import Spider
 from crawlo.utils.log import get_logger
-from crawlo.exceptions import DropItem
 
 
 class RedisDedupPipeline:

crawlo/project.py

@@ -30,6 +30,15 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
         2. 存在 '__init__.py' 和 'settings.py'（即 Python 包）
     """
     path = os.path.abspath(start_path)
+    
+    # 首先检查当前目录及其子目录
+    for root, dirs, files in os.walk(path):
+        if "crawlo.cfg" in files:
+            cfg_path = os.path.join(root, "crawlo.cfg")
+            logger.info(f"✅ 找到项目配置文件: {cfg_path}")
+            return root
+    
+    # 如果在子目录中没找到，再向上查找
     while True:
         cfg_file = os.path.join(path, "crawlo.cfg")
         if os.path.isfile(cfg_file):
@@ -128,17 +137,43 @@ def load_class(_path):
             raise TypeError(f"args expect str or object, got {_path}")
 
     module_name, class_name = _path.rsplit('.', 1)
-    module = import_module(module_name)
+    
+    try:
+        module = import_module(module_name)
+    except ImportError as e:
+        # 尝试不同的导入方式
+        try:
+            # 尝试直接导入完整路径
+            module = import_module(_path)
+            return module
+        except ImportError:
+            pass
+        raise ImportError(f"Cannot import module {module_name}: {e}")
 
     try:
         cls = getattr(module, class_name)
     except AttributeError:
-        raise NameError(f"Module {module_name!r} has no class named {class_name!r}")
+        # 提供更详细的错误信息
+        available_attrs = [attr for attr in dir(module) if not attr.startswith('_')]
+        raise NameError(f"Module {module_name!r} has no class named {class_name!r}. Available attributes: {available_attrs}")
     return cls
 
 
 def merge_settings(spider, settings):
     spider_name = getattr(spider, 'name', 'UnknownSpider')
+    # 检查 settings 是否为 SettingManager 实例
+    if not hasattr(settings, 'update_attributes'):
+        logger.error(f"merge_settings 接收到的 settings 不是 SettingManager 实例: {type(settings)}")
+        # 如果是字典，创建一个新的 SettingManager 实例
+        if isinstance(settings, dict):
+            from crawlo.settings.setting_manager import SettingManager
+            new_settings = SettingManager()
+            new_settings.update_attributes(settings)
+            settings = new_settings
+        else:
+            logger.error("无法处理的 settings 类型")
+            return
+            
     if hasattr(spider, 'custom_settings'):
         custom_settings = getattr(spider, 'custom_settings')
         settings.update_attributes(custom_settings)
@@ -150,4 +185,4 @@ async def common_call(func: Callable, *args, **kwargs):
     if iscoroutinefunction(func):
         return await func(*args, **kwargs)
     else:
-        return func(*args, **kwargs)
+        return func(*args, **kwargs)