crawlo 1.4.3__py3-none-any.whl → 1.4.5__py3-none-any.whl

crawlo/__init__.py

@@ -3,14 +3,17 @@
 """
 Crawlo - 一个异步爬虫框架
 """
-from typing import TYPE_CHECKING
 
-from crawlo.spider import Spider
+# 为了向后兼容，从tools中导入cleaners相关的功能
+import crawlo.tools as cleaners
+from crawlo import tools
+from crawlo.crawler import CrawlerProcess
+from crawlo.downloader import DownloaderBase
 from crawlo.items import Item, Field
+from crawlo.middleware import BaseMiddleware
 from crawlo.network.request import Request
 from crawlo.network.response import Response
-from crawlo.downloader import DownloaderBase
-from crawlo.middleware import BaseMiddleware
+from crawlo.spider import Spider
 from crawlo.utils import (
     TimeUtils,
     parse_time,
@@ -24,21 +27,13 @@ from crawlo.utils import (
     to_local,
     from_timestamp_with_tz
 )
-from crawlo import tools
-
-# 框架核心模块 - 使用TYPE_CHECKING避免循环导入
-if TYPE_CHECKING:
-    from crawlo.initialization import get_framework_initializer, initialize_framework
-
-# 为了向后兼容，从tools中导入cleaners相关的功能
-import crawlo.tools as cleaners
 
 
 # 延迟导入的辅助函数
 def get_framework_initializer():
-    """延迟导入get_framework_initializer以避免循环依赖"""
-    from crawlo.initialization import get_framework_initializer as _get_framework_initializer
-    return _get_framework_initializer()
+    """延迟导入CoreInitializer以避免循环依赖"""
+    from crawlo.initialization import CoreInitializer
+    return CoreInitializer()
 
 
 def initialize_framework(custom_settings=None):
@@ -87,6 +82,7 @@ __all__ = [
     'from_timestamp_with_tz',
     'cleaners',
     'tools',
+    'CrawlerProcess',
     'get_framework_initializer',
     'get_bootstrap_manager',
     '__version__',

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = '1.4.3'
+__version__ = '1.4.5'

crawlo/commands/genspider.py

@@ -6,15 +6,16 @@
 # @Desc    : 命令行入口：crawlo genspider baidu，创建爬虫。
 """
 import sys
+import re
 from pathlib import Path
 import configparser
 import importlib
 from rich.console import Console
 
 from .utils import (
-    get_project_root, 
-    validate_project_environment, 
-    show_error_panel, 
+    get_project_root,
+    validate_project_environment,
+    show_error_panel,
     show_success_panel,
     validate_spider_name,
     is_valid_domain
@@ -35,6 +36,39 @@ def _render_template(tmpl_path, context):
     return content
 
 
+def generate_class_name(spider_name):
+    """
+    根据爬虫名称生成类名
+    规则：蛇形命名 → 大驼峰命名 + 'Spider'
+    示例：
+        'news_spider' → 'NewsSpider'
+        'ofweek_standalone' → 'OfweekStandaloneSpider'
+        'baidu' → 'BaiduSpider'
+    """
+    # 如果名称已包含 'spider' 后缀，先去除
+    name_clean = spider_name
+    
+    # 定义要移除的后缀列表
+    spider_suffixes = ['_spider', 'spider']
+    
+    # 检查并移除后缀
+    for suffix in spider_suffixes:
+        if spider_name.endswith(suffix):
+            name_clean = spider_name[:-len(suffix)]
+            break
+
+    # 按分隔符拆分单词
+    words = re.split(r'[_-]', name_clean)
+
+    # 将每个单词首字母大写
+    capitalized_words = [word.capitalize() for word in words if word]
+
+    # 组合成类名
+    class_name = ''.join(capitalized_words) + 'Spider'
+
+    return class_name
+
+
 def main(args):
     if len(args) < 2:
         console.print("[bold red]错误:[/bold red] 用法: [blue]crawlo genspider[/blue] <爬虫名称> <域名>")
@@ -45,11 +79,11 @@ def main(args):
 
     spider_name = args[0]
     domain = args[1]
-    
+
     # 验证爬虫名称
     if not validate_spider_name(spider_name):
         show_error_panel(
-            "无效的爬虫名称", 
+            "无效的爬虫名称",
             f"爬虫名称 '[cyan]{spider_name}[/cyan]' 无效。\n"
             "爬虫名称应:\n"
             "  • 以小写字母开头\n"
@@ -57,11 +91,11 @@ def main(args):
             "  • 是有效的Python标识符"
         )
         return 1
-    
+
     # 验证域名格式
     if not is_valid_domain(domain):
         show_error_panel(
-            "无效的域名", 
+            "无效的域名",
             f"域名 '[cyan]{domain}[/cyan]' 格式无效。\n"
             "请提供有效的域名，如 'example.com'"
         )
@@ -72,7 +106,7 @@ def main(args):
     if not is_valid:
         show_error_panel("非Crawlo项目", error_msg)
         return 1
-    
+
     project_root = get_project_root()
 
     # 确定 items 模块的路径
@@ -91,7 +125,8 @@ def main(args):
         if item_classes:
             default_item_class = item_classes[0].__name__
         else:
-            console.print("[yellow]警告:[/yellow] 在 [cyan]items.py[/cyan] 中未找到项目类，使用 [green]ExampleItem[/green]。")
+            console.print(
+                "[yellow]警告:[/yellow] 在 [cyan]items.py[/cyan] 中未找到项目类，使用 [green]ExampleItem[/green]。")
 
     except ImportError as e:
         console.print(f"[yellow]警告:[/yellow] 导入 [cyan]{items_module_path}[/cyan] 失败: {e}")
@@ -104,7 +139,7 @@ def main(args):
     spider_file = spiders_dir / f'{spider_name}.py'
     if spider_file.exists():
         show_error_panel(
-            "爬虫已存在", 
+            "爬虫已存在",
             f"爬虫 '[cyan]{spider_name}[/cyan]' 已存在于\n[green]{spider_file}[/green]"
         )
         return 1
@@ -113,13 +148,13 @@ def main(args):
     tmpl_path = TEMPLATES_DIR / 'spider' / 'spider.py.tmpl'
     if not tmpl_path.exists():
         show_error_panel(
-            "模板未找到", 
+            "模板未找到",
             f"模板文件未找到于 [cyan]{tmpl_path}[/cyan]"
         )
         return 1
 
-    # 生成类名
-    class_name = f"{spider_name.replace('_', '').capitalize()}Spider"
+    # 生成类名（使用新的转换函数）
+    class_name = generate_class_name(spider_name)
 
     context = {
         'spider_name': spider_name,
@@ -133,7 +168,7 @@ def main(args):
         content = _render_template(tmpl_path, context)
         with open(spider_file, 'w', encoding='utf-8') as f:
             f.write(content)
-        
+
         console.print(f"[green]爬虫 '[bold]{spider_name}[/bold]' 创建成功！[/green]")
         console.print(f"  → 位置: [cyan]{spider_file}[/cyan]")
         console.print(f"  → 类名: [yellow]{class_name}[/yellow]")
@@ -141,12 +176,12 @@ def main(args):
         console.print("\n[bold]下一步操作:[/bold]")
         console.print(f"  [blue]crawlo run[/blue] {spider_name}")
         console.print(f"  [blue]crawlo check[/blue] {spider_name}")
-        
+
         return 0
-        
+
     except Exception as e:
         show_error_panel(
-            "创建失败", 
+            "创建失败",
             f"创建爬虫失败: {e}"
         )
         return 1

crawlo/commands/startproject.py

@@ -92,8 +92,32 @@ def _render_template(tmpl_path, context):
     """读取模板文件，替换 {{key}} 为 context 中的值"""
     with open(tmpl_path, 'r', encoding='utf-8') as f:
         content = f.read()
+    
+    # 处理简单的过滤器语法 {{key|filter}}
+    import re
+    
+    def apply_filter(value, filter_name):
+        if filter_name == 'title':
+            # 将 snake_case 转换为 TitleCase
+            words = value.replace('_', ' ').split()
+            return ''.join(word.capitalize() for word in words)
+        return value
+    
+    # 查找并替换 {{key|filter}} 格式的占位符
+    pattern = r'\{\{([^}|]+)\|([^}]+)\}\}'
+    def replace_filter_match(match):
+        key = match.group(1).strip()
+        filter_name = match.group(2).strip()
+        if key in context:
+            return str(apply_filter(context[key], filter_name))
+        return match.group(0)  # 如果找不到key，保持原样
+    
+    content = re.sub(pattern, replace_filter_match, content)
+    
+    # 处理普通的 {{key}} 占位符
     for key, value in context.items():
         content = content.replace(f'{{{{{key}}}}}', str(value))
+    
     return content
 
 

crawlo/core/engine.py

@@ -11,7 +11,7 @@ from crawlo.core.scheduler import Scheduler
 from crawlo.downloader import DownloaderBase
 from crawlo.event import spider_opened, spider_error, request_scheduled
 from crawlo.exceptions import OutputError
-from crawlo.utils.class_loader import load_class
+from crawlo.utils.misc import load_object
 from crawlo.spider import Spider
 from crawlo.task_manager import TaskManager
 from crawlo.utils.func_tools import transform
@@ -62,7 +62,7 @@ class Engine(object):
                 self.logger.warning(f"无法使用下载器类型 '{downloader_type}': {e}，回退到默认配置")
         
         # 方式2: 使用 DOWNLOADER 完整类路径（兼容旧版本）
-        downloader_cls = load_class(self.settings.get('DOWNLOADER'))
+        downloader_cls = load_object(self.settings.get('DOWNLOADER'))
         if not issubclass(downloader_cls, DownloaderBase):
             raise TypeError(f'下载器 {downloader_cls.__name__} 不是 DownloaderBase 的子类。')
         return downloader_cls

crawlo/core/scheduler.py

@@ -6,7 +6,7 @@ from typing import Optional, Callable
 from crawlo.utils.log import get_logger
 from crawlo.utils.request import set_request
 from crawlo.utils.error_handler import ErrorHandler
-from crawlo.utils.class_loader import load_class
+from crawlo.utils.misc import load_object
 from crawlo.project import common_call
 from crawlo.utils.request_serializer import RequestSerializer
 from crawlo.queue.queue_manager import QueueManager, QueueConfig, QueueType
@@ -26,7 +26,7 @@ class Scheduler:
 
     @classmethod
     def create_instance(cls, crawler):
-        filter_cls = load_class(crawler.settings.get('FILTER_CLASS'))
+        filter_cls = load_object(crawler.settings.get('FILTER_CLASS'))
         o = cls(
             crawler=crawler,
             dupe_filter=filter_cls.create_instance(crawler),
@@ -120,7 +120,7 @@ class Scheduler:
             # 如果需要更新配置，则执行更新
             if needs_config_update:
                 # 重新创建过滤器实例，确保使用更新后的配置
-                filter_cls = load_class(self.crawler.settings.get('FILTER_CLASS'))
+                filter_cls = load_object(self.crawler.settings.get('FILTER_CLASS'))
                 self.dupe_filter = filter_cls.create_instance(self.crawler)
                 
                 # 记录警告信息
@@ -136,7 +136,7 @@ class Scheduler:
                     self._switch_to_memory_config()
                 
                 # 重新创建过滤器实例
-                filter_cls = load_class(self.crawler.settings.get('FILTER_CLASS'))
+                filter_cls = load_object(self.crawler.settings.get('FILTER_CLASS'))
                 self.dupe_filter = filter_cls.create_instance(self.crawler)
     
     def _is_filter_matching_queue_type(self, current_filter_class):

crawlo/crawler.py

@@ -13,14 +13,14 @@
 
 import asyncio
 import time
-from contextlib import asynccontextmanager
-from dataclasses import dataclass
 from enum import Enum
+from dataclasses import dataclass
+from contextlib import asynccontextmanager
 from typing import Optional, Type, Dict, Any, List
 
+from crawlo.logging import get_logger
 from crawlo.factories import get_component_registry
 from crawlo.initialization import initialize_framework, is_framework_ready
-from crawlo.logging import get_logger
 
 
 class CrawlerState(Enum):
@@ -345,16 +345,23 @@ class CrawlerProcess:
     """
     
     def __init__(self, settings=None, max_concurrency: int = 3, spider_modules=None):
+        # 初始化框架配置
         self._settings = settings or initialize_framework()
         self._max_concurrency = max_concurrency
         self._crawlers: List[ModernCrawler] = []
         self._semaphore = asyncio.Semaphore(max_concurrency)
         self._logger = get_logger('crawler.process')
-        self._spider_modules = spider_modules  # 保存spider_modules
+        
+        # 如果没有显式提供spider_modules，则从settings中获取
+        if spider_modules is None and self._settings:
+            spider_modules = self._settings.get('SPIDER_MODULES', [])
+            self._logger.debug(f"从settings中获取SPIDER_MODULES: {spider_modules}")
+        
+        self._spider_modules = spider_modules or []  # 保存spider_modules
         
         # 如果提供了spider_modules，自动注册这些模块中的爬虫
-        if spider_modules:
-            self._register_spider_modules(spider_modules)
+        if self._spider_modules:
+            self._register_spider_modules(self._spider_modules)
         
         # 指标
         self._start_time: Optional[float] = None

crawlo/downloader/__init__.py

@@ -15,12 +15,15 @@ Crawlo Downloader Module
 - ActivateRequestManager: 活跃请求管理器
 """
 from abc import abstractmethod, ABCMeta
-from typing import Final, Set, Optional
+from typing import Final, Set, Optional, TYPE_CHECKING
 from contextlib import asynccontextmanager
 
 from crawlo.utils.log import get_logger
 from crawlo.middleware.middleware_manager import MiddlewareManager
 
+if TYPE_CHECKING:
+    from crawlo import Response
+
 
 class ActivateRequestManager:
     """活跃请求管理器 - 跟踪和管理正在处理的请求"""
@@ -134,7 +137,7 @@ class DownloaderBase(metaclass=DownloaderMeta):
             self.logger.error(f"中间件初始化失败: {e}")
             raise
 
-    async def fetch(self, request) -> Optional['Response']:
+    async def fetch(self, request) -> 'Optional[Response]':
         """获取请求响应（经过中间件处理）"""
         if self._closed:
             raise RuntimeError(f"{self.__class__.__name__} 已关闭")

crawlo/extension/__init__.py

@@ -4,7 +4,7 @@ from typing import List, Any
 from pprint import pformat
 
 from crawlo.utils.log import get_logger
-from crawlo.utils.class_loader import load_class
+from crawlo.utils.misc import load_object
 from crawlo.exceptions import ExtensionInitError
 
 
@@ -25,7 +25,7 @@ class ExtensionManager(object):
     def _add_extensions(self, extensions: List[str]) -> None:
         for extension_path in extensions:
             try:
-                extension_cls = load_class(extension_path)
+                extension_cls = load_object(extension_path)
                 if not hasattr(extension_cls, 'create_instance'):
                     raise ExtensionInitError(
                         f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"

crawlo/filters/aioredis_filter.py

@@ -142,7 +142,14 @@ class AioRedisFilter(BaseFilter):
             if redis_client is None:
                 return False
             
-            fp = str(request_fingerprint(request))
+            # 使用统一的指纹生成器
+            from crawlo.utils.fingerprint import FingerprintGenerator
+            fp = str(FingerprintGenerator.request_fingerprint(
+                request.method, 
+                request.url, 
+                request.body or b'', 
+                dict(request.headers) if hasattr(request, 'headers') else None
+            ))
             self._redis_operations += 1
 
             # 使用 pipeline 优化性能

crawlo/filters/memory_filter.py

@@ -102,7 +102,14 @@ class MemoryFilter(BaseFilter):
         :return: 是否重复
         """
         with self._lock:
-            fp = request_fingerprint(request)
+            # 使用统一的指纹生成器
+            from crawlo.utils.fingerprint import FingerprintGenerator
+            fp = FingerprintGenerator.request_fingerprint(
+                request.method, 
+                request.url, 
+                request.body or b'', 
+                dict(request.headers) if hasattr(request, 'headers') else None
+            )
             if fp in self.fingerprints:
                 self._dupe_count += 1
                 # if self.debug:

crawlo/initialization/built_in.py

@@ -210,8 +210,17 @@ class SettingsInitializer(BaseInitializer):
             from crawlo.settings.setting_manager import SettingManager
             from crawlo.project import _load_project_settings
             
-            # 创建配置管理器并加载项目配置
-            settings = _load_project_settings(context.custom_settings)
+            # 如果上下文中已有设置，则使用它作为基础配置
+            if context.settings:
+                # 使用用户传递的设置作为基础配置
+                settings = context.settings
+                # 加载项目配置并合并
+                project_settings = _load_project_settings(context.custom_settings)
+                # 合并配置，用户配置优先
+                settings.update_attributes(project_settings.attributes)
+            else:
+                # 创建配置管理器并加载项目配置
+                settings = _load_project_settings(context.custom_settings)
             
             # 存储到上下文
             context.settings = settings
@@ -346,8 +355,8 @@ class ExtensionsInitializer(BaseInitializer):
             initialized_extensions = []
             for extension_path in extensions:
                 try:
-                    from crawlo.utils.class_loader import load_class
-                    extension_class = load_class(extension_path)
+                    from crawlo.utils.misc import load_object
+                    extension_class = load_object(extension_path)
                     extension_instance = extension_class()
                     initialized_extensions.append(extension_instance)
                 except Exception as e:

crawlo/initialization/core.py

@@ -4,14 +4,14 @@
 核心初始化器 - 协调整个初始化过程
 """
 
-import time
 import threading
+import time
 from typing import Optional, Any
 
-from .context import InitializationContext  
-from .phases import InitializationPhase, PhaseResult, get_execution_order, get_phase_definition
-from .registry import get_global_registry, BaseInitializer, register_initializer
 from .built_in import register_built_in_initializers
+from .context import InitializationContext
+from .phases import InitializationPhase, PhaseResult, get_execution_order, get_phase_definition
+from .registry import get_global_registry
 
 
 class CoreInitializer:
@@ -78,6 +78,7 @@ class CoreInitializer:
             # 创建初始化上下文
             context = InitializationContext()
             context.custom_settings = kwargs
+            context.settings = settings
             self._context = context
             
             try:

crawlo/interfaces.py

@@ -0,0 +1,24 @@
+from abc import ABC, abstractmethod
+from typing import List, Type, Protocol
+
+from crawlo.spider import Spider
+from crawlo.network.request import Request
+
+
+class ISpiderLoader(Protocol):
+    """Spider loader interface"""
+    
+    @abstractmethod
+    def load(self, spider_name: str) -> Type[Spider]:
+        """Load a spider by name"""
+        pass
+    
+    @abstractmethod
+    def list(self) -> List[str]:
+        """List all available spider names"""
+        pass
+    
+    @abstractmethod
+    def find_by_request(self, request: Request) -> List[str]:
+        """Find spider names that can handle the given request"""
+        pass

crawlo/middleware/__init__.py

@@ -1,18 +1,21 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-from crawlo import Request, Response
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from crawlo import Request, Response
 
 
 class BaseMiddleware(object):
-    def process_request(self, request, spider) -> None | Request | Response:
+    def process_request(self, request, spider) -> 'None | Request | Response':
         # 请求预处理
         pass
 
-    def process_response(self, request, response, spider) -> Request | Response:
+    def process_response(self, request, response, spider) -> 'Request | Response':
         # 响应预处理
         pass
 
-    def process_exception(self, request, exp, spider) -> None | Request | Response:
+    def process_exception(self, request, exp, spider) -> 'None | Request | Response':
         # 异常预处理
         pass
 

crawlo/middleware/middleware_manager.py

@@ -4,11 +4,18 @@ from pprint import pformat
 from types import MethodType
 from asyncio import create_task
 from collections import defaultdict
-from typing import List, Dict, Callable, Optional
+from typing import List, Dict, Callable, Optional, TYPE_CHECKING
 
-from crawlo import Request, Response
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from crawlo import Request, Response
+else:
+    # 为 isinstance 检查导入实际的类
+    from crawlo.network.request import Request
+    from crawlo.network.response import Response
 from crawlo.utils.log import get_logger
-from crawlo.utils.class_loader import load_class
+from crawlo.utils.misc import load_object
 from crawlo.middleware import BaseMiddleware
 from crawlo.project import common_call
 from crawlo.event import ignore_request, response_received
@@ -30,7 +37,7 @@ class MiddlewareManager:
         self.download_method: Callable = crawler.engine.downloader.download
         self._stats = crawler.stats
 
-    async def _process_request(self, request: Request):
+    async def _process_request(self, request: 'Request'):
         for method in self.methods['process_request']:
             result = await common_call(method, request, self.crawler.spider)
             if result is None:
@@ -42,7 +49,7 @@ class MiddlewareManager:
             )
         return await self.download_method(request)
 
-    async def _process_response(self, request: Request, response: Response):
+    async def _process_response(self, request: 'Request', response: 'Response'):
         for method in reversed(self.methods['process_response']):
             try:
                 response = await common_call(method, request, response, self.crawler.spider)
@@ -57,7 +64,7 @@ class MiddlewareManager:
             )
         return response
 
-    async def _process_exception(self, request: Request, exp: Exception):
+    async def _process_exception(self, request: 'Request', exp: Exception):
         for method in self.methods['process_exception']:
             response = await common_call(method, request, exp, self.crawler.spider)
             if response is None:
@@ -72,7 +79,7 @@ class MiddlewareManager:
         else:
             raise exp
 
-    async def download(self, request) -> Optional[Response]:
+    async def download(self, request) -> 'Optional[Response]':
         """ called in the download method. """
         try:
             response = await self._process_request(request)
@@ -105,7 +112,7 @@ class MiddlewareManager:
             self.logger.info(f'Enabled middlewares:\n {pformat(enabled_middlewares)}')
 
     def _validate_middleware(self, middleware):
-        middleware_cls = load_class(middleware)
+        middleware_cls = load_object(middleware)
         if not hasattr(middleware_cls, 'create_instance'):
             raise MiddlewareInitError(
                 f"Middleware init failed, must inherit from `BaseMiddleware` or have a `create_instance` method"