crawlo 1.4.4__py3-none-any.whl → 1.4.5__py3-none-any.whl

crawlo/__init__.py

@@ -3,14 +3,17 @@
 """
 Crawlo - 一个异步爬虫框架
 """
-from typing import TYPE_CHECKING
 
-from crawlo.spider import Spider
+# 为了向后兼容，从tools中导入cleaners相关的功能
+import crawlo.tools as cleaners
+from crawlo import tools
+from crawlo.crawler import CrawlerProcess
+from crawlo.downloader import DownloaderBase
 from crawlo.items import Item, Field
+from crawlo.middleware import BaseMiddleware
 from crawlo.network.request import Request
 from crawlo.network.response import Response
-from crawlo.downloader import DownloaderBase
-from crawlo.middleware import BaseMiddleware
+from crawlo.spider import Spider
 from crawlo.utils import (
     TimeUtils,
     parse_time,
@@ -24,21 +27,13 @@ from crawlo.utils import (
     to_local,
     from_timestamp_with_tz
 )
-from crawlo import tools
-
-# 框架核心模块 - 使用TYPE_CHECKING避免循环导入
-if TYPE_CHECKING:
-    from crawlo.initialization import get_framework_initializer, initialize_framework
-
-# 为了向后兼容，从tools中导入cleaners相关的功能
-import crawlo.tools as cleaners
 
 
 # 延迟导入的辅助函数
 def get_framework_initializer():
-    """延迟导入get_framework_initializer以避免循环依赖"""
-    from crawlo.initialization import get_framework_initializer as _get_framework_initializer
-    return _get_framework_initializer()
+    """延迟导入CoreInitializer以避免循环依赖"""
+    from crawlo.initialization import CoreInitializer
+    return CoreInitializer()
 
 
 def initialize_framework(custom_settings=None):
@@ -87,6 +82,7 @@ __all__ = [
     'from_timestamp_with_tz',
     'cleaners',
     'tools',
+    'CrawlerProcess',
     'get_framework_initializer',
     'get_bootstrap_manager',
     '__version__',

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = '1.4.4'
+__version__ = '1.4.5'

crawlo/commands/startproject.py

@@ -92,8 +92,32 @@ def _render_template(tmpl_path, context):
     """读取模板文件，替换 {{key}} 为 context 中的值"""
     with open(tmpl_path, 'r', encoding='utf-8') as f:
         content = f.read()
+    
+    # 处理简单的过滤器语法 {{key|filter}}
+    import re
+    
+    def apply_filter(value, filter_name):
+        if filter_name == 'title':
+            # 将 snake_case 转换为 TitleCase
+            words = value.replace('_', ' ').split()
+            return ''.join(word.capitalize() for word in words)
+        return value
+    
+    # 查找并替换 {{key|filter}} 格式的占位符
+    pattern = r'\{\{([^}|]+)\|([^}]+)\}\}'
+    def replace_filter_match(match):
+        key = match.group(1).strip()
+        filter_name = match.group(2).strip()
+        if key in context:
+            return str(apply_filter(context[key], filter_name))
+        return match.group(0)  # 如果找不到key，保持原样
+    
+    content = re.sub(pattern, replace_filter_match, content)
+    
+    # 处理普通的 {{key}} 占位符
     for key, value in context.items():
         content = content.replace(f'{{{{{key}}}}}', str(value))
+    
     return content
 
 

crawlo/core/engine.py

@@ -11,7 +11,7 @@ from crawlo.core.scheduler import Scheduler
 from crawlo.downloader import DownloaderBase
 from crawlo.event import spider_opened, spider_error, request_scheduled
 from crawlo.exceptions import OutputError
-from crawlo.utils.class_loader import load_class
+from crawlo.utils.misc import load_object
 from crawlo.spider import Spider
 from crawlo.task_manager import TaskManager
 from crawlo.utils.func_tools import transform
@@ -62,7 +62,7 @@ class Engine(object):
                 self.logger.warning(f"无法使用下载器类型 '{downloader_type}': {e}，回退到默认配置")
         
         # 方式2: 使用 DOWNLOADER 完整类路径（兼容旧版本）
-        downloader_cls = load_class(self.settings.get('DOWNLOADER'))
+        downloader_cls = load_object(self.settings.get('DOWNLOADER'))
         if not issubclass(downloader_cls, DownloaderBase):
             raise TypeError(f'下载器 {downloader_cls.__name__} 不是 DownloaderBase 的子类。')
         return downloader_cls

crawlo/core/scheduler.py

@@ -6,7 +6,7 @@ from typing import Optional, Callable
 from crawlo.utils.log import get_logger
 from crawlo.utils.request import set_request
 from crawlo.utils.error_handler import ErrorHandler
-from crawlo.utils.class_loader import load_class
+from crawlo.utils.misc import load_object
 from crawlo.project import common_call
 from crawlo.utils.request_serializer import RequestSerializer
 from crawlo.queue.queue_manager import QueueManager, QueueConfig, QueueType
@@ -26,7 +26,7 @@ class Scheduler:
 
     @classmethod
     def create_instance(cls, crawler):
-        filter_cls = load_class(crawler.settings.get('FILTER_CLASS'))
+        filter_cls = load_object(crawler.settings.get('FILTER_CLASS'))
         o = cls(
             crawler=crawler,
             dupe_filter=filter_cls.create_instance(crawler),
@@ -120,7 +120,7 @@ class Scheduler:
             # 如果需要更新配置，则执行更新
             if needs_config_update:
                 # 重新创建过滤器实例，确保使用更新后的配置
-                filter_cls = load_class(self.crawler.settings.get('FILTER_CLASS'))
+                filter_cls = load_object(self.crawler.settings.get('FILTER_CLASS'))
                 self.dupe_filter = filter_cls.create_instance(self.crawler)
                 
                 # 记录警告信息
@@ -136,7 +136,7 @@ class Scheduler:
                     self._switch_to_memory_config()
                 
                 # 重新创建过滤器实例
-                filter_cls = load_class(self.crawler.settings.get('FILTER_CLASS'))
+                filter_cls = load_object(self.crawler.settings.get('FILTER_CLASS'))
                 self.dupe_filter = filter_cls.create_instance(self.crawler)
     
     def _is_filter_matching_queue_type(self, current_filter_class):

crawlo/crawler.py

@@ -13,14 +13,14 @@
 
 import asyncio
 import time
-from contextlib import asynccontextmanager
-from dataclasses import dataclass
 from enum import Enum
+from dataclasses import dataclass
+from contextlib import asynccontextmanager
 from typing import Optional, Type, Dict, Any, List
 
+from crawlo.logging import get_logger
 from crawlo.factories import get_component_registry
 from crawlo.initialization import initialize_framework, is_framework_ready
-from crawlo.logging import get_logger
 
 
 class CrawlerState(Enum):
@@ -345,6 +345,7 @@ class CrawlerProcess:
     """
     
     def __init__(self, settings=None, max_concurrency: int = 3, spider_modules=None):
+        # 初始化框架配置
         self._settings = settings or initialize_framework()
         self._max_concurrency = max_concurrency
         self._crawlers: List[ModernCrawler] = []
@@ -353,14 +354,14 @@ class CrawlerProcess:
         
         # 如果没有显式提供spider_modules，则从settings中获取
         if spider_modules is None and self._settings:
-            spider_modules = self._settings.get('SPIDER_MODULES')
+            spider_modules = self._settings.get('SPIDER_MODULES', [])
             self._logger.debug(f"从settings中获取SPIDER_MODULES: {spider_modules}")
         
-        self._spider_modules = spider_modules  # 保存spider_modules
+        self._spider_modules = spider_modules or []  # 保存spider_modules
         
         # 如果提供了spider_modules，自动注册这些模块中的爬虫
-        if spider_modules:
-            self._register_spider_modules(spider_modules)
+        if self._spider_modules:
+            self._register_spider_modules(self._spider_modules)
         
         # 指标
         self._start_time: Optional[float] = None

crawlo/downloader/__init__.py

@@ -15,12 +15,15 @@ Crawlo Downloader Module
 - ActivateRequestManager: 活跃请求管理器
 """
 from abc import abstractmethod, ABCMeta
-from typing import Final, Set, Optional
+from typing import Final, Set, Optional, TYPE_CHECKING
 from contextlib import asynccontextmanager
 
 from crawlo.utils.log import get_logger
 from crawlo.middleware.middleware_manager import MiddlewareManager
 
+if TYPE_CHECKING:
+    from crawlo import Response
+
 
 class ActivateRequestManager:
     """活跃请求管理器 - 跟踪和管理正在处理的请求"""
@@ -134,7 +137,7 @@ class DownloaderBase(metaclass=DownloaderMeta):
             self.logger.error(f"中间件初始化失败: {e}")
             raise
 
-    async def fetch(self, request) -> Optional['Response']:
+    async def fetch(self, request) -> 'Optional[Response]':
         """获取请求响应（经过中间件处理）"""
         if self._closed:
             raise RuntimeError(f"{self.__class__.__name__} 已关闭")

crawlo/extension/__init__.py

@@ -4,7 +4,7 @@ from typing import List, Any
 from pprint import pformat
 
 from crawlo.utils.log import get_logger
-from crawlo.utils.class_loader import load_class
+from crawlo.utils.misc import load_object
 from crawlo.exceptions import ExtensionInitError
 
 
@@ -25,7 +25,7 @@ class ExtensionManager(object):
     def _add_extensions(self, extensions: List[str]) -> None:
         for extension_path in extensions:
             try:
-                extension_cls = load_class(extension_path)
+                extension_cls = load_object(extension_path)
                 if not hasattr(extension_cls, 'create_instance'):
                     raise ExtensionInitError(
                         f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"

crawlo/filters/aioredis_filter.py

@@ -142,7 +142,14 @@ class AioRedisFilter(BaseFilter):
             if redis_client is None:
                 return False
             
-            fp = str(request_fingerprint(request))
+            # 使用统一的指纹生成器
+            from crawlo.utils.fingerprint import FingerprintGenerator
+            fp = str(FingerprintGenerator.request_fingerprint(
+                request.method, 
+                request.url, 
+                request.body or b'', 
+                dict(request.headers) if hasattr(request, 'headers') else None
+            ))
             self._redis_operations += 1
 
             # 使用 pipeline 优化性能

crawlo/filters/memory_filter.py

@@ -102,7 +102,14 @@ class MemoryFilter(BaseFilter):
         :return: 是否重复
         """
         with self._lock:
-            fp = request_fingerprint(request)
+            # 使用统一的指纹生成器
+            from crawlo.utils.fingerprint import FingerprintGenerator
+            fp = FingerprintGenerator.request_fingerprint(
+                request.method, 
+                request.url, 
+                request.body or b'', 
+                dict(request.headers) if hasattr(request, 'headers') else None
+            )
             if fp in self.fingerprints:
                 self._dupe_count += 1
                 # if self.debug:

crawlo/initialization/built_in.py

@@ -210,8 +210,17 @@ class SettingsInitializer(BaseInitializer):
             from crawlo.settings.setting_manager import SettingManager
             from crawlo.project import _load_project_settings
             
-            # 创建配置管理器并加载项目配置
-            settings = _load_project_settings(context.custom_settings)
+            # 如果上下文中已有设置，则使用它作为基础配置
+            if context.settings:
+                # 使用用户传递的设置作为基础配置
+                settings = context.settings
+                # 加载项目配置并合并
+                project_settings = _load_project_settings(context.custom_settings)
+                # 合并配置，用户配置优先
+                settings.update_attributes(project_settings.attributes)
+            else:
+                # 创建配置管理器并加载项目配置
+                settings = _load_project_settings(context.custom_settings)
             
             # 存储到上下文
             context.settings = settings
@@ -346,8 +355,8 @@ class ExtensionsInitializer(BaseInitializer):
             initialized_extensions = []
             for extension_path in extensions:
                 try:
-                    from crawlo.utils.class_loader import load_class
-                    extension_class = load_class(extension_path)
+                    from crawlo.utils.misc import load_object
+                    extension_class = load_object(extension_path)
                     extension_instance = extension_class()
                     initialized_extensions.append(extension_instance)
                 except Exception as e:

crawlo/initialization/core.py

@@ -4,14 +4,14 @@
 核心初始化器 - 协调整个初始化过程
 """
 
-import time
 import threading
+import time
 from typing import Optional, Any
 
-from .context import InitializationContext  
-from .phases import InitializationPhase, PhaseResult, get_execution_order, get_phase_definition
-from .registry import get_global_registry, BaseInitializer, register_initializer
 from .built_in import register_built_in_initializers
+from .context import InitializationContext
+from .phases import InitializationPhase, PhaseResult, get_execution_order, get_phase_definition
+from .registry import get_global_registry
 
 
 class CoreInitializer:
@@ -78,6 +78,7 @@ class CoreInitializer:
             # 创建初始化上下文
             context = InitializationContext()
             context.custom_settings = kwargs
+            context.settings = settings
             self._context = context
             
             try:

crawlo/interfaces.py

@@ -0,0 +1,24 @@
+from abc import ABC, abstractmethod
+from typing import List, Type, Protocol
+
+from crawlo.spider import Spider
+from crawlo.network.request import Request
+
+
+class ISpiderLoader(Protocol):
+    """Spider loader interface"""
+    
+    @abstractmethod
+    def load(self, spider_name: str) -> Type[Spider]:
+        """Load a spider by name"""
+        pass
+    
+    @abstractmethod
+    def list(self) -> List[str]:
+        """List all available spider names"""
+        pass
+    
+    @abstractmethod
+    def find_by_request(self, request: Request) -> List[str]:
+        """Find spider names that can handle the given request"""
+        pass

crawlo/middleware/__init__.py

@@ -1,18 +1,21 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-from crawlo import Request, Response
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from crawlo import Request, Response
 
 
 class BaseMiddleware(object):
-    def process_request(self, request, spider) -> None | Request | Response:
+    def process_request(self, request, spider) -> 'None | Request | Response':
         # 请求预处理
         pass
 
-    def process_response(self, request, response, spider) -> Request | Response:
+    def process_response(self, request, response, spider) -> 'Request | Response':
         # 响应预处理
         pass
 
-    def process_exception(self, request, exp, spider) -> None | Request | Response:
+    def process_exception(self, request, exp, spider) -> 'None | Request | Response':
         # 异常预处理
         pass
 

crawlo/middleware/middleware_manager.py

@@ -4,11 +4,18 @@ from pprint import pformat
 from types import MethodType
 from asyncio import create_task
 from collections import defaultdict
-from typing import List, Dict, Callable, Optional
+from typing import List, Dict, Callable, Optional, TYPE_CHECKING
 
-from crawlo import Request, Response
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from crawlo import Request, Response
+else:
+    # 为 isinstance 检查导入实际的类
+    from crawlo.network.request import Request
+    from crawlo.network.response import Response
 from crawlo.utils.log import get_logger
-from crawlo.utils.class_loader import load_class
+from crawlo.utils.misc import load_object
 from crawlo.middleware import BaseMiddleware
 from crawlo.project import common_call
 from crawlo.event import ignore_request, response_received
@@ -30,7 +37,7 @@ class MiddlewareManager:
         self.download_method: Callable = crawler.engine.downloader.download
         self._stats = crawler.stats
 
-    async def _process_request(self, request: Request):
+    async def _process_request(self, request: 'Request'):
         for method in self.methods['process_request']:
             result = await common_call(method, request, self.crawler.spider)
             if result is None:
@@ -42,7 +49,7 @@ class MiddlewareManager:
             )
         return await self.download_method(request)
 
-    async def _process_response(self, request: Request, response: Response):
+    async def _process_response(self, request: 'Request', response: 'Response'):
         for method in reversed(self.methods['process_response']):
             try:
                 response = await common_call(method, request, response, self.crawler.spider)
@@ -57,7 +64,7 @@ class MiddlewareManager:
             )
         return response
 
-    async def _process_exception(self, request: Request, exp: Exception):
+    async def _process_exception(self, request: 'Request', exp: Exception):
         for method in self.methods['process_exception']:
             response = await common_call(method, request, exp, self.crawler.spider)
             if response is None:
@@ -72,7 +79,7 @@ class MiddlewareManager:
         else:
             raise exp
 
-    async def download(self, request) -> Optional[Response]:
+    async def download(self, request) -> 'Optional[Response]':
         """ called in the download method. """
         try:
             response = await self._process_request(request)
@@ -105,7 +112,7 @@ class MiddlewareManager:
             self.logger.info(f'Enabled middlewares:\n {pformat(enabled_middlewares)}')
 
     def _validate_middleware(self, middleware):
-        middleware_cls = load_class(middleware)
+        middleware_cls = load_object(middleware)
         if not hasattr(middleware_cls, 'create_instance'):
             raise MiddlewareInitError(
                 f"Middleware init failed, must inherit from `BaseMiddleware` or have a `create_instance` method"

crawlo/mode_manager.py

@@ -51,9 +51,11 @@ class ModeManager:
     def get_standalone_settings() -> Dict[str, Any]:
         """获取单机模式配置"""
         return {
+            'RUN_MODE': 'standalone',
             'QUEUE_TYPE': 'memory',
             'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',
             'DEFAULT_DEDUP_PIPELINE': 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline',
+            'PROJECT_NAME': 'crawlo',
             'CONCURRENCY': 8,
             'MAX_RUNNING_SPIDERS': 1,
             'DOWNLOAD_DELAY': 1.0,
@@ -75,6 +77,7 @@ class ModeManager:
             redis_url = f'redis://{redis_host}:{redis_port}/{redis_db}'
 
         return {
+            'RUN_MODE': 'distributed',
             'QUEUE_TYPE': 'redis',
             'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
             'DEFAULT_DEDUP_PIPELINE': 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline',
@@ -95,6 +98,7 @@ class ModeManager:
         """获取自动检测模式配置"""
         # 默认使用内存队列和过滤器
         settings = ModeManager.get_standalone_settings()
+        settings['RUN_MODE'] = 'auto'
         settings['QUEUE_TYPE'] = 'auto'
         return settings
 
@@ -143,13 +147,29 @@ class ModeManager:
             raise ValueError(f"不支持的运行模式: {mode}")
 
         # 合并用户自定义配置
-        user_settings = {
-            k: v for k,
-            v in kwargs.items() if k not in [
-                'redis_host',
-                'redis_port',
-                'redis_password',
-                'project_name']}
+        # 对于分布式模式，过滤掉特定参数
+        if mode == RunMode.DISTRIBUTED:
+            user_settings = {
+                k.upper(): v for k,
+                v in kwargs.items() if k not in [
+                    'redis_host',
+                    'redis_port',
+                    'redis_password',
+                    'project_name']}
+            # 特别处理project_name
+            if 'project_name' in kwargs:
+                settings['PROJECT_NAME'] = kwargs['project_name']
+        else:
+            # 对于单机模式和自动模式，只过滤Redis相关参数
+            user_settings = {
+                k.upper(): v for k,
+                v in kwargs.items() if k not in [
+                    'redis_host',
+                    'redis_port',
+                    'redis_password']}
+            # 特别处理project_name
+            if 'project_name' in kwargs:
+                settings['PROJECT_NAME'] = kwargs['project_name']
         settings.update(user_settings)
         self._debug(f"合并用户自定义配置: {list(user_settings.keys())}")
 
@@ -182,9 +202,16 @@ class ModeManager:
 
 
 # 便利函数
-def standalone_mode(**kwargs) -> Dict[str, Any]:
+def standalone_mode(
+        project_name: str = 'crawlo',
+        **kwargs
+) -> Dict[str, Any]:
     """快速创建单机模式配置"""
-    return ModeManager().resolve_mode_settings('standalone', **kwargs)
+    return ModeManager().resolve_mode_settings(
+        'standalone',
+        project_name=project_name,
+        **kwargs
+    )
 
 
 def distributed_mode(
@@ -207,9 +234,16 @@ def distributed_mode(
     )
 
 
-def auto_mode(**kwargs) -> Dict[str, Any]:
+def auto_mode(
+        project_name: str = 'crawlo',
+        **kwargs
+) -> Dict[str, Any]:
     """快速创建自动检测模式配置"""
-    return ModeManager().resolve_mode_settings('auto', **kwargs)
+    return ModeManager().resolve_mode_settings(
+        'auto',
+        project_name=project_name,
+        **kwargs
+    )
 
 
 # 环境变量支持