crawlo 1.0.7__py3-none-any.whl → 1.0.8__py3-none-any.whl

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.7"
+__version__ = "1.0.8"

crawlo/commands/check.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-08-31 22:35
+# @Author  :   crawl-coder
+# @Desc    :   命令行入口：crawlo check， 检查所有爬虫定义是否合规。
+"""
+import sys
+import configparser
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.utils.project import get_settings
+from crawlo.utils.log import get_logger
+
+
+logger = get_logger(__name__)
+
+
+def main(args):
+    if args:
+        print("Usage: crawlo check")
+        return 1
+
+    try:
+        project_root = get_settings().get('PROJECT_ROOT')
+        if not project_root:
+            print("❌ Error: Cannot determine project root.")
+            return 1
+
+        if str(project_root) not in sys.path:
+            sys.path.insert(0, str(project_root))
+
+        cfg_file = project_root / 'crawlo.cfg'
+        if not cfg_file.exists():
+            print(f"❌ Error: crawlo.cfg not found in {project_root}")
+            return 1
+
+        config = configparser.ConfigParser()
+        config.read(cfg_file, encoding='utf-8')
+
+        if not config.has_section('settings') or not config.has_option('settings', 'default'):
+            print("❌ Error: Missing [settings] section or 'default' option in crawlo.cfg")
+            return 1
+
+        settings_module = config.get('settings', 'default')
+        project_package = settings_module.split('.')[0]
+
+        # 创建 CrawlerProcess 并发现爬虫
+        process = CrawlerProcess(spider_modules=[f"{project_package}.spiders"])
+        spider_names = process.get_spider_names()
+
+        if not spider_names:
+            print("📭 No spiders found.")
+            return 1
+
+        print(f"🔍 Checking {len(spider_names)} spider(s)...")
+        print("-" * 60)
+
+        issues_found = False
+        for name in sorted(spider_names):
+            cls = process.get_spider_class(name)
+            issues = []
+
+            if not hasattr(cls, 'name') or not cls.name:
+                issues.append("missing or empty 'name' attribute")
+            elif not isinstance(cls.name, str):
+                issues.append("'name' is not a string")
+
+            if not callable(getattr(cls, 'start_requests', None)):
+                issues.append("missing or non-callable 'start_requests' method")
+
+            if hasattr(cls, 'start_urls') and isinstance(cls.start_urls, str):
+                issues.append("'start_urls' is a string, should be list/tuple")
+
+            # 实例化检查（轻量）
+            try:
+                spider = cls.create_instance(None)
+                if not callable(getattr(spider, 'parse', None)):
+                    issues.append("no 'parse' method defined (optional but recommended)")
+            except Exception as e:
+                issues.append(f"failed to create instance: {e}")
+
+            if issues:
+                print(f"❌ {name:<20} {cls.__name__}")
+                for issue in issues:
+                    print(f"     • {issue}")
+                issues_found = True
+            else:
+                print(f"✅ {name:<20} {cls.__name__} (OK)")
+
+        print("-" * 60)
+        if issues_found:
+            print("⚠️  Some spiders have issues. Please fix them.")
+            return 1
+        else:
+            print("🎉 All spiders are compliant!")
+            return 0
+
+    except Exception as e:
+        print(f"❌ Error during check: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

crawlo/commands/list.py

@@ -0,0 +1,92 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-08-31 22:33
+# @Author  :   crawl-coder
+# @Desc    :   命令行入口：crawlo list，用于列出所有已注册的爬虫
+"""
+import sys
+import configparser
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.utils.project import get_settings
+from crawlo.utils.log import get_logger
+
+
+logger = get_logger(__name__)
+
+
+def main(args):
+    """
+    列出所有可用爬虫
+    用法: crawlo list
+    """
+    if args:
+        print("Usage: crawlo list")
+        return 1
+
+    try:
+        # 1. 获取项目根目录
+        project_root = get_settings().get('PROJECT_ROOT')
+        if not project_root:
+            print("❌ Error: Cannot determine project root.")
+            return 1
+
+        # 将项目根目录加入 sys.path
+        project_root_str = str(project_root)
+        if project_root_str not in sys.path:
+            sys.path.insert(0, project_root_str)
+
+        # 2. 读取 crawlo.cfg 获取项目包名
+        cfg_file = project_root / 'crawlo.cfg'
+        if not cfg_file.exists():
+            print(f"❌ Error: crawlo.cfg not found in {project_root}")
+            return 1
+
+        config = configparser.ConfigParser()
+        config.read(cfg_file, encoding='utf-8')
+
+        if not config.has_section('settings') or not config.has_option('settings', 'default'):
+            print("❌ Error: Missing [settings] section or 'default' option in crawlo.cfg")
+            return 1
+
+        settings_module = config.get('settings', 'default')
+        project_package = settings_module.split('.')[0]
+
+        # 3. 创建 CrawlerProcess 并自动发现爬虫
+        spider_modules = [f"{project_package}.spiders"]
+        process = CrawlerProcess(spider_modules=spider_modules)
+
+        # 4. 获取所有爬虫信息
+        spider_names = process.get_spider_names()
+        if not spider_names:
+            print("📭 No spiders found.")
+            print("💡 Make sure:")
+            print("   - Your spider classes inherit from `Spider`")
+            print("   - They define a `name` attribute")
+            print("   - The modules are imported (e.g. via __init__.py)")
+            return 1
+
+        # 5. 输出爬虫列表
+        print(f"📋 Found {len(spider_names)} spider(s):")
+        print("-" * 50)
+        for name in sorted(spider_names):
+            cls = process.get_spider_class(name)
+            module = cls.__module__.replace(project_package + ".", "")  # 简化模块名
+            print(f"🕷️  {name:<20} {cls.__name__:<25} ({module})")
+        print("-" * 50)
+        return 0
+
+    except Exception as e:
+        print(f"❌ Error listing spiders: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+
+
+if __name__ == '__main__':
+    """
+    允许直接运行：
+        python -m crawlo.commands.list
+    """
+    sys.exit(main(sys.argv[1:]))

crawlo/commands/run.py

@@ -1,14 +1,15 @@
-# crawlo/commands/run.py
+"""
+命令行入口：crawlo run <spider_name>
+用于运行指定名称的爬虫。
+"""
+
 import asyncio
-import importlib
-import sys
 from pathlib import Path
 import configparser
 
 from crawlo.crawler import CrawlerProcess
 from crawlo.utils.project import get_settings
 from crawlo.utils.log import get_logger
-from crawlo.utils.spider_loader import SpiderLoader
 
 logger = get_logger(__name__)
 
@@ -16,24 +17,30 @@ logger = get_logger(__name__)
 def main(args):
     """
     运行指定爬虫的主函数
-    用法: crawlo run <spider_name>
+    用法:
+        crawlo run <spider_name>
+        crawlo run all
     """
     if len(args) < 1:
-        print("Usage: crawlo run <spider_name>")
-        print("Example: crawlo run baidu")
+        print("Usage: crawlo run <spider_name>|all")
+        print("Examples:")
+        print("  crawlo run baidu")
+        print("  crawlo run all")
         return 1
 
-    spider_name = args[0]
+    spider_arg = args[0]
 
     try:
         # 1. 获取项目根目录
-        project_root = get_settings()
+        project_root = get_settings().get('PROJECT_ROOT')
+        if not project_root:
+            print("❌ Error: Cannot determine project root.")
+            return 1
 
-        # 将项目根目录添加到 Python 路径
         if str(project_root) not in sys.path:
             sys.path.insert(0, str(project_root))
 
-        # 2. 读取配置文件获取项目包名
+        # 2. 读取 crawlo.cfg 获取项目包名
         cfg_file = project_root / 'crawlo.cfg'
         if not cfg_file.exists():
             print(f"❌ Error: crawlo.cfg not found in {project_root}")
@@ -49,27 +56,60 @@ def main(args):
         settings_module = config.get('settings', 'default')
         project_package = settings_module.split('.')[0]
 
-        # 3. 查找并加载指定名称的 Spider
-        spider_class = find_spider_by_name(project_package, spider_name)
-        if spider_class is None:
+        # 3. 创建 CrawlerProcess 并自动发现爬虫模块
+        spider_modules = [f"{project_package}.spiders"]
+        settings = get_settings()
+        process = CrawlerProcess(settings=settings, spider_modules=spider_modules)
+
+        # === 新增：支持 'all' ===
+        if spider_arg.lower() == "all":
+            spider_names = process.get_spider_names()
+            if not spider_names:
+                print("❌ No spiders found. Make sure spiders are defined and imported.")
+                return 1
+
+            print(f"🚀 Starting ALL {len(spider_names)} spiders:")
+            for name in sorted(spider_names):
+                cls = process.get_spider_class(name)
+                print(f"   🕷️  {name} ({cls.__name__})")
+            print("-" * 50)
+
+            # 启动所有爬虫
+            asyncio.run(process.crawl(spider_names))
+            return 0
+
+        # === 原有：启动单个爬虫 ===
+        spider_name = spider_arg
+        if not process.is_spider_registered(spider_name):
+            print(f"❌ Error: Spider with name '{spider_name}' not found.")
+            available_names = process.get_spider_names()
+            if available_names:
+                print("💡 Available spiders:")
+                for name in sorted(available_names):
+                    cls = process.get_spider_class(name)
+                    print(f"   - {name} (class: {cls.__name__})")
+            else:
+                print("💡 No spiders found. Make sure your spider classes are defined and imported.")
             return 1
 
-        # 4. 创建 CrawlerProcess 并运行单个爬虫
-        settings = get_settings()
-        process = CrawlerProcess(settings)
+        spider_class = process.get_spider_class(spider_name)
 
-        print(f"🚀 Starting spider: {spider_class.name}")
+        # 打印启动信息
+        print(f"🚀 Starting spider: {spider_name}")
         print(f"📁 Project: {project_package}")
         print(f"🕷️  Class: {spider_class.__name__}")
         print("-" * 50)
 
-        # 运行单个爬虫
-        asyncio.run(process.crawl(spider_class))
+        # 启动爬虫
+        asyncio.run(process.crawl(spider_name))
 
         print("-" * 50)
         print("✅ Spider completed successfully!")
         return 0
 
+    except KeyboardInterrupt:
+        print("\n⚠️  Spider interrupted by user.")
+        return 1
     except Exception as e:
         print(f"❌ Error running spider: {e}")
         import traceback
@@ -77,73 +117,65 @@ def main(args):
         return 1
 
 
-def find_spider_by_name(project_package: str, target_spider_name: str):
-    """使用 SpiderLoader 查找爬虫"""
-    loader = SpiderLoader(project_package)
-    spider_class = loader.load(target_spider_name)
-
-    if spider_class is None:
-        print(f"❌ Error: Spider with name '{target_spider_name}' not found")
-        print("💡 Available spiders:")
-        available_spiders = loader.list()
-        for spider_name in available_spiders:
-            print(f"   - {spider_name}")
-        return None
-
-    return spider_class
-
-
 def list_available_spiders(project_package: str):
     """
-    列出所有可用的爬虫
+    列出指定项目包中所有可用的爬虫（用于调试或命令行扩展）
     """
-    spiders_dir = Path.cwd() / project_package / 'spiders'
-    if not spiders_dir.exists():
-        print("   No spiders directory found")
-        return
-
-    spider_count = 0
-    for py_file in spiders_dir.glob("*.py"):
-        if py_file.name.startswith('_'):
-            continue
-
-        module_name = py_file.stem
-        spider_module_path = f"{project_package}.spiders.{module_name}"
-
-        try:
-            module = importlib.import_module(spider_module_path)
-        except ImportError:
-            continue
-
-        # 查找模块中所有 Spider 子类
-        from crawlo.spider import Spider
-        for attr_name in dir(module):
-            attr_value = getattr(module, attr_name)
-            if (isinstance(attr_value, type) and
-                    issubclass(attr_value, Spider) and
-                    attr_value != Spider and
-                    hasattr(attr_value, 'name')):
-                print(f"   - {attr_value.name} (class: {attr_value.__name__}, module: {module_name})")
-                spider_count += 1
-
-    if spider_count == 0:
-        print("   No spiders found")
-
-
-def run_spider_by_name(spider_name: str, project_root: Path = None):
+    try:
+        # 临时创建一个 CrawlerProcess 来发现爬虫
+        process = CrawlerProcess(spider_modules=[f"{project_package}.spiders"])
+        available_names = process.get_spider_names()
+
+        if not available_names:
+            print("   No spiders found. Make sure:")
+            print("   - spiders/ 目录存在")
+            print("   - 爬虫类继承 Spider 且定义了 name")
+            print("   - 模块被导入（可通过 __init__.py 触发）")
+            return
+
+        print(f"Found {len(available_names)} spider(s):")
+        for name in sorted(available_names):
+            cls = process.get_spider_class(name)
+            module = cls.__module__.replace(project_package + ".", "")
+            print(f"   - {name} ({cls.__name__} @ {module})")
+    except Exception as e:
+        print(f"❌ Failed to list spiders: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+def run_spider_by_name(spider_name: str, project_package: str = None):
     """
-    直接在代码中通过 spider name 运行爬虫
+    在代码中直接运行某个爬虫（需提供 project_package）
     """
-    if project_root:
-        if str(project_root) not in sys.path:
-            sys.path.insert(0, str(project_root))
+    if project_package is None:
+        # 尝试从配置读取
+        cfg_file = Path('crawlo.cfg')
+        if cfg_file.exists():
+            config = configparser.ConfigParser()
+            config.read(cfg_file, encoding='utf-8')
+            if config.has_option('settings', 'default'):
+                project_package = config.get('settings', 'default').split('.')[0]
+
+    if not project_package:
+        print("❌ Error: project_package is required.")
+        return 1
+
+    # 添加项目路径
+    project_root = get_settings().get('PROJECT_ROOT')
+    if project_root and str(project_root) not in sys.path:
+        sys.path.insert(0, str(project_root))
 
+    # 复用 main 函数逻辑
     args = [spider_name]
     return main(args)
 
 
 if __name__ == '__main__':
-    # 允许直接运行: python -m crawlo.commands.run <spider_name>
+    """
+    允许直接运行：
+        python -m crawlo.commands.run <spider_name>
+    """
     import sys
 
-    sys.exit(main(sys.argv[1:]))
+    sys.exit(main(sys.argv[1:]))

crawlo/commands/stats.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-08-31 22:36
+# @Author  :   crawl-coder
+# @Desc    :   命令行入口：crawlo stats，查看最近运行的爬虫统计信息。
+"""
+import sys
+from crawlo.utils.log import get_logger
+
+
+logger = get_logger(__name__)
+
+# 保存最近运行的爬虫的统计（示例）
+_LAST_RUN_STATS = {}
+
+
+def record_stats(crawler):
+    """在爬虫关闭后记录统计（需在 close 中调用）"""
+    if crawler.stats and crawler.spider:
+        _LAST_RUN_STATS[crawler.spider.name] = crawler.stats.get_stats()
+
+
+def main(args):
+    if len(args) == 0:
+        # 显示所有历史统计
+        if not _LAST_RUN_STATS:
+            print("📊 No stats available. Run a spider first.")
+            return 0
+
+        print("📊 Recent Spider Statistics:")
+        print("-" * 60)
+        for spider_name, stats in _LAST_RUN_STATS.items():
+            print(f"🕷️  {spider_name}")
+            for k, v in stats.items():
+                print(f"    {k:<30} {v}")
+            print()
+        return 0
+
+    elif len(args) == 1:
+        spider_name = args[0]
+        if spider_name not in _LAST_RUN_STATS:
+            print(f"📊 No stats found for spider '{spider_name}'")
+            return 1
+
+        stats = _LAST_RUN_STATS[spider_name]
+        print(f"📊 Stats for '{spider_name}':")
+        print("-" * 60)
+        for k, v in stats.items():
+            print(f"    {k:<30} {v}")
+        return 0
+
+    else:
+        print("Usage: crawlo stats [spider_name]")
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

crawlo/crawler.py

@@ -1,19 +1,18 @@
 #!/usr/bin/python
 # -*- coding: UTF-8 -*-
+from __future__ import annotations
 import asyncio
 import signal
-from typing import Type, Optional, Set, List
-
-from crawlo.spider import Spider
-from crawlo.core.engine import Engine
-from crawlo.utils.log import get_logger
-from crawlo.subscriber import Subscriber
-from crawlo.extension import ExtensionManager
-from crawlo.exceptions import SpiderTypeError
-from crawlo.stats_collector import StatsCollector
-from crawlo.event import spider_opened, spider_closed
-from crawlo.settings.setting_manager import SettingManager
-from crawlo.utils.project import merge_settings, get_settings
+from typing import Type, Optional, Set, List, Union, Dict
+from .spider import Spider, get_global_spider_registry
+from .core.engine import Engine
+from .utils.log import get_logger
+from .subscriber import Subscriber
+from .extension import ExtensionManager
+from .stats_collector import StatsCollector
+from .event import spider_opened, spider_closed
+from .settings.setting_manager import SettingManager
+from .utils.project import merge_settings, get_settings
 
 
 logger = get_logger(__name__)
@@ -30,7 +29,7 @@ class Crawler:
         self.subscriber: Optional[Subscriber] = None
         self.extension: Optional[ExtensionManager] = None
         self.settings: SettingManager = settings.copy()
-        self._closed = False  # 新增状态
+        self._closed = False
         self._close_lock = asyncio.Lock()
 
     async def crawl(self):
@@ -40,7 +39,6 @@ class Crawler:
         self.engine = self._create_engine()
         self.stats = self._create_stats()
         self.extension = self._create_extension()
-
         await self.engine.start_spider(self.spider)
 
     @staticmethod
@@ -50,7 +48,6 @@ class Crawler:
     def _create_spider(self) -> Spider:
         spider = self.spider_cls.create_instance(self)
 
-        # --- 关键属性检查 ---
         if not getattr(spider, 'name', None):
             raise AttributeError(f"爬虫类 '{self.spider_cls.__name__}' 必须定义 'name' 属性。")
 
@@ -92,19 +89,36 @@ class Crawler:
             await self.subscriber.notify(spider_closed)
             if self.stats and self.spider:
                 self.stats.close_spider(spider=self.spider, reason=reason)
+                from crawlo.commands.stats import record_stats
+                record_stats(self)
 
 
 class CrawlerProcess:
     """
-    爬虫进程管理器，支持多爬虫并发调度、信号量控制、实时日志与优雅关闭
+    爬虫进程管理器，支持：
+    - 自动发现爬虫模块
+    - 通过 name 或类启动爬虫
+    - 并发控制
+    - 优雅关闭
     """
 
-    def __init__(self, settings: Optional[SettingManager] = None, max_concurrency: Optional[int] = None):
+    def __init__(
+        self,
+        settings: Optional[SettingManager] = None,
+        max_concurrency: Optional[int] = None,
+        spider_modules: Optional[List[str]] = None
+    ):
         self.settings: SettingManager = settings or self._get_default_settings()
         self.crawlers: Set[Crawler] = set()
         self._active_tasks: Set[asyncio.Task] = set()
 
-        # 使用专用配置，降级使用 CONCURRENCY
+        # 自动发现并导入爬虫模块
+        if spider_modules:
+            self.auto_discover(spider_modules)
+
+        # 使用全局注册表的快照（避免后续导入影响）
+        self._spider_registry: Dict[str, Type[Spider]] = get_global_spider_registry()
+
         self.max_concurrency: int = (
             max_concurrency
             or self.settings.get('MAX_RUNNING_SPIDERS')
@@ -117,80 +131,120 @@ class CrawlerProcess:
         signal.signal(signal.SIGTERM, self._shutdown)
         logger.info(f"CrawlerProcess 初始化完成，最大并行爬虫数: {self.max_concurrency}")
 
-    async def crawl(self, spiders):
-        """
-        启动一个或多个爬虫，流式调度，支持实时进度反馈
-        """
-        spider_classes = self._normalize_spiders(spiders)
-        total = len(spider_classes)
+    def auto_discover(self, modules: List[str]):
+        """自动导入模块，触发 Spider 类定义和注册"""
+        import importlib
+        import pkgutil
+        for module_name in modules:
+            try:
+                module = importlib.import_module(module_name)
+                if hasattr(module, '__path__'):
+                    for _, name, _ in pkgutil.walk_packages(module.__path__, module.__name__ + "."):
+                        importlib.import_module(name)
+                else:
+                    importlib.import_module(module_name)
+                logger.debug(f"已扫描模块: {module_name}")
+            except Exception as e:
+                logger.error(f"扫描模块 {module_name} 失败: {e}", exc_info=True)
+
+    # === 公共只读接口：避免直接访问 _spider_registry ===
+
+    def get_spider_names(self) -> List[str]:
+        """获取所有已注册的爬虫名称"""
+        return list(self._spider_registry.keys())
+
+    def get_spider_class(self, name: str) -> Optional[Type[Spider]]:
+        """根据 name 获取爬虫类"""
+        return self._spider_registry.get(name)
+
+    def is_spider_registered(self, name: str) -> bool:
+        """检查某个 name 是否已注册"""
+        return name in self._spider_registry
+
+    async def crawl(self, spiders: Union[Type[Spider], str, List[Union[Type[Spider], str]]]):
+        """启动一个或多个爬虫"""
+        spider_classes_to_run = self._resolve_spiders_to_run(spiders)
+        total = len(spider_classes_to_run)
 
         if total == 0:
-            raise ValueError("至少需要提供一个爬虫类")
-
-        # 按名称排序
-        spider_classes.sort(key=lambda cls: cls.__name__.lower())
+            raise ValueError("至少需要提供一个爬虫类或名称")
 
+        # 按类名排序，保证启动顺序可预测
+        spider_classes_to_run.sort(key=lambda cls: cls.__name__.lower())
         logger.info(f"启动 {total} 个爬虫.")
 
-        # 流式启动所有爬虫任务
+        # 流式启动
         tasks = [
             asyncio.create_task(self._run_spider_with_limit(spider_cls, index + 1, total))
-            for index, spider_cls in enumerate(spider_classes)
+            for index, spider_cls in enumerate(spider_classes_to_run)
         ]
 
-        # 等待所有任务完成（失败不中断）
+        # 等待完成（失败不中断）
         results = await asyncio.gather(*tasks, return_exceptions=True)
-
-        # 统计异常
         failed = [i for i, r in enumerate(results) if isinstance(r, Exception)]
         if failed:
-            logger.error(f"共 {len(failed)} 个爬虫执行异常: {[spider_classes[i].__name__ for i in failed]}")
-
-    @staticmethod
-    def _normalize_spiders(spiders) -> List[Type[Spider]]:
-        """标准化输入为爬虫类列表"""
-        if isinstance(spiders, type) and issubclass(spiders, Spider):
-            return [spiders]
-        elif isinstance(spiders, (list, tuple)):
-            return list(spiders)
+            logger.error(f"共 {len(failed)} 个爬虫执行异常: {[spider_classes_to_run[i].__name__ for i in failed]}")
+
+    def _resolve_spiders_to_run(
+        self,
+        spiders_input: Union[Type[Spider], str, List[Union[Type[Spider], str]]]
+    ) -> List[Type[Spider]]:
+        """解析输入为爬虫类列表"""
+        inputs = self._normalize_inputs(spiders_input)
+        seen_spider_names: Set[str] = set()
+        spider_classes: List[Type[Spider]] = []
+
+        for item in inputs:
+            spider_cls = self._resolve_spider_class(item)
+            spider_name = spider_cls.name
+
+            if spider_name in seen_spider_names:
+                raise ValueError(f"本次运行中爬虫名称 '{spider_name}' 重复。")
+
+            seen_spider_names.add(spider_name)
+            spider_classes.append(spider_cls)
+
+        return spider_classes
+
+    def _normalize_inputs(self, spiders_input) -> List[Union[Type[Spider], str]]:
+        """标准化输入为列表"""
+        if isinstance(spiders_input, (type, str)):
+            return [spiders_input]
+        elif isinstance(spiders_input, (list, tuple)):
+            return list(spiders_input)
         else:
-            raise TypeError("spiders 必须是爬虫类或爬虫类列表/元组")
+            raise TypeError("spiders 必须是爬虫类、name 字符串，或它们的列表/元组")
+
+    def _resolve_spider_class(self, item: Union[Type[Spider], str]) -> Type[Spider]:
+        """解析单个输入项为爬虫类"""
+        if isinstance(item, type) and issubclass(item, Spider):
+            return item
+        elif isinstance(item, str):
+            spider_cls = self._spider_registry.get(item)
+            if not spider_cls:
+                raise ValueError(f"未找到名为 '{item}' 的爬虫。")
+            return spider_cls
+        else:
+            raise TypeError(f"无效类型 {type(item)}。必须是 Spider 类或字符串 name。")
 
     async def _run_spider_with_limit(self, spider_cls: Type[Spider], seq: int, total: int):
-        """
-        受信号量限制的爬虫运行函数，带进度日志
-        """
+        """受信号量限制的爬虫运行函数"""
         task = asyncio.current_task()
         self._active_tasks.add(task)
-
         try:
-            # 获取并发许可
             await self.semaphore.acquire()
-
-            start_msg = f"[{seq}/{total}] 启动爬虫: {spider_cls.__name__}"
-            logger.info(start_msg)
-
-            # 创建并运行爬虫
-            crawler = self._create_crawler(spider_cls)
+            logger.info(f"[{seq}/{total}] 启动爬虫: {spider_cls.__name__}")
+            crawler = Crawler(spider_cls, self.settings)
             self.crawlers.add(crawler)
             await crawler.crawl()
-
-            end_msg = f"[{seq}/{total}] 爬虫完成: {spider_cls.__name__}"
-            logger.info(end_msg)
-
+            logger.info(f"[{seq}/{total}] 爬虫完成: {spider_cls.__name__}")
         except Exception as e:
             logger.error(f"爬虫 {spider_cls.__name__} 执行失败: {e}", exc_info=True)
             raise
         finally:
             if task in self._active_tasks:
                 self._active_tasks.remove(task)
-            self.semaphore.release()  # 必须释放
-
-    def _create_crawler(self, spider_cls: Type[Spider]) -> Crawler:
-        """创建爬虫实例"""
-        if isinstance(spider_cls, str):
-            raise SpiderTypeError(f"不支持字符串形式的爬虫: {spider_cls}")
-        return Crawler(spider_cls, self.settings)
+            self.semaphore.release()
 
     def _shutdown(self, _signum, _frame):
         """优雅关闭信号处理"""
@@ -216,4 +270,224 @@ class CrawlerProcess:
             return get_settings()
         except Exception as e:
             logger.warning(f"无法加载默认配置: {e}")
-            return SettingManager()
+            return SettingManager()
+
+# #!/usr/bin/python
+# # -*- coding: UTF-8 -*-
+# import asyncio
+# import signal
+# from typing import Type, Optional, Set, List
+#
+# from crawlo.spider import Spider
+# from crawlo.core.engine import Engine
+# from crawlo.utils.log import get_logger
+# from crawlo.subscriber import Subscriber
+# from crawlo.extension import ExtensionManager
+# from crawlo.exceptions import SpiderTypeError
+# from crawlo.stats_collector import StatsCollector
+# from crawlo.event import spider_opened, spider_closed
+# from crawlo.settings.setting_manager import SettingManager
+# from crawlo.utils.project import merge_settings, get_settings
+#
+#
+# logger = get_logger(__name__)
+#
+#
+# class Crawler:
+#     """单个爬虫运行实例，绑定 Spider 与引擎"""
+#
+#     def __init__(self, spider_cls: Type[Spider], settings: SettingManager):
+#         self.spider_cls = spider_cls
+#         self.spider: Optional[Spider] = None
+#         self.engine: Optional[Engine] = None
+#         self.stats: Optional[StatsCollector] = None
+#         self.subscriber: Optional[Subscriber] = None
+#         self.extension: Optional[ExtensionManager] = None
+#         self.settings: SettingManager = settings.copy()
+#         self._closed = False  # 新增状态
+#         self._close_lock = asyncio.Lock()
+#
+#     async def crawl(self):
+#         """启动爬虫核心流程"""
+#         self.subscriber = self._create_subscriber()
+#         self.spider = self._create_spider()
+#         self.engine = self._create_engine()
+#         self.stats = self._create_stats()
+#         self.extension = self._create_extension()
+#
+#         await self.engine.start_spider(self.spider)
+#
+#     @staticmethod
+#     def _create_subscriber() -> Subscriber:
+#         return Subscriber()
+#
+#     def _create_spider(self) -> Spider:
+#         spider = self.spider_cls.create_instance(self)
+#
+#         # --- 关键属性检查 ---
+#         if not getattr(spider, 'name', None):
+#             raise AttributeError(f"爬虫类 '{self.spider_cls.__name__}' 必须定义 'name' 属性。")
+#
+#         if not callable(getattr(spider, 'start_requests', None)):
+#             raise AttributeError(f"爬虫 '{spider.name}' 必须实现可调用的 'start_requests' 方法。")
+#
+#         start_urls = getattr(spider, 'start_urls', [])
+#         if isinstance(start_urls, str):
+#             raise TypeError(f"爬虫 '{spider.name}' 的 'start_urls' 必须是列表或元组，不能是字符串。")
+#
+#         if not callable(getattr(spider, 'parse', None)):
+#             logger.warning(
+#                 f"爬虫 '{spider.name}' 未定义 'parse' 方法。请确保所有 Request 都指定了回调函数，否则响应将被忽略。")
+#
+#         self._set_spider(spider)
+#         return spider
+#
+#     def _create_engine(self) -> Engine:
+#         engine = Engine(self)
+#         engine.engine_start()
+#         return engine
+#
+#     def _create_stats(self) -> StatsCollector:
+#         return StatsCollector(self)
+#
+#     def _create_extension(self) -> ExtensionManager:
+#         return ExtensionManager.create_instance(self)
+#
+#     def _set_spider(self, spider: Spider):
+#         self.subscriber.subscribe(spider.spider_opened, event=spider_opened)
+#         self.subscriber.subscribe(spider.spider_closed, event=spider_closed)
+#         merge_settings(spider, self.settings)
+#
+#     async def close(self, reason='finished') -> None:
+#         async with self._close_lock:
+#             if self._closed:
+#                 return
+#             self._closed = True
+#             await self.subscriber.notify(spider_closed)
+#             if self.stats and self.spider:
+#                 self.stats.close_spider(spider=self.spider, reason=reason)
+#
+#
+# class CrawlerProcess:
+#     """
+#     爬虫进程管理器，支持多爬虫并发调度、信号量控制、实时日志与优雅关闭
+#     """
+#
+#     def __init__(self, settings: Optional[SettingManager] = None, max_concurrency: Optional[int] = None):
+#         self.settings: SettingManager = settings or self._get_default_settings()
+#         self.crawlers: Set[Crawler] = set()
+#         self._active_tasks: Set[asyncio.Task] = set()
+#
+#         # 使用专用配置，降级使用 CONCURRENCY
+#         self.max_concurrency: int = (
+#             max_concurrency
+#             or self.settings.get('MAX_RUNNING_SPIDERS')
+#             or self.settings.get('CONCURRENCY', 3)
+#         )
+#         self.semaphore = asyncio.Semaphore(self.max_concurrency)
+#
+#         # 注册信号量
+#         signal.signal(signal.SIGINT, self._shutdown)
+#         signal.signal(signal.SIGTERM, self._shutdown)
+#         logger.info(f"CrawlerProcess 初始化完成，最大并行爬虫数: {self.max_concurrency}")
+#
+#     async def crawl(self, spiders):
+#         """
+#         启动一个或多个爬虫，流式调度，支持实时进度反馈
+#         """
+#         spider_classes = self._normalize_spiders(spiders)
+#         total = len(spider_classes)
+#
+#         if total == 0:
+#             raise ValueError("至少需要提供一个爬虫类")
+#
+#         # 按名称排序
+#         spider_classes.sort(key=lambda cls: cls.__name__.lower())
+#
+#         logger.info(f"启动 {total} 个爬虫.")
+#
+#         # 流式启动所有爬虫任务
+#         tasks = [
+#             asyncio.create_task(self._run_spider_with_limit(spider_cls, index + 1, total))
+#             for index, spider_cls in enumerate(spider_classes)
+#         ]
+#
+#         # 等待所有任务完成（失败不中断）
+#         results = await asyncio.gather(*tasks, return_exceptions=True)
+#
+#         # 统计异常
+#         failed = [i for i, r in enumerate(results) if isinstance(r, Exception)]
+#         if failed:
+#             logger.error(f"共 {len(failed)} 个爬虫执行异常: {[spider_classes[i].__name__ for i in failed]}")
+#
+#     @staticmethod
+#     def _normalize_spiders(spiders) -> List[Type[Spider]]:
+#         """标准化输入为爬虫类列表"""
+#         if isinstance(spiders, type) and issubclass(spiders, Spider):
+#             return [spiders]
+#         elif isinstance(spiders, (list, tuple)):
+#             return list(spiders)
+#         else:
+#             raise TypeError("spiders 必须是爬虫类或爬虫类列表/元组")
+#
+#     async def _run_spider_with_limit(self, spider_cls: Type[Spider], seq: int, total: int):
+#         """
+#         受信号量限制的爬虫运行函数，带进度日志
+#         """
+#         task = asyncio.current_task()
+#         self._active_tasks.add(task)
+#
+#         try:
+#             # 获取并发许可
+#             await self.semaphore.acquire()
+#
+#             start_msg = f"[{seq}/{total}] 启动爬虫: {spider_cls.__name__}"
+#             logger.info(start_msg)
+#
+#             # 创建并运行爬虫
+#             crawler = self._create_crawler(spider_cls)
+#             self.crawlers.add(crawler)
+#             await crawler.crawl()
+#
+#             end_msg = f"[{seq}/{total}] 爬虫完成: {spider_cls.__name__}"
+#             logger.info(end_msg)
+#
+#         except Exception as e:
+#             logger.error(f"爬虫 {spider_cls.__name__} 执行失败: {e}", exc_info=True)
+#             raise
+#         finally:
+#             if task in self._active_tasks:
+#                 self._active_tasks.remove(task)
+#             self.semaphore.release()  # 必须释放
+#
+#     def _create_crawler(self, spider_cls: Type[Spider]) -> Crawler:
+#         """创建爬虫实例"""
+#         if isinstance(spider_cls, str):
+#             raise SpiderTypeError(f"不支持字符串形式的爬虫: {spider_cls}")
+#         return Crawler(spider_cls, self.settings)
+#
+#     def _shutdown(self, _signum, _frame):
+#         """优雅关闭信号处理"""
+#         logger.warning("收到关闭信号，正在停止所有爬虫...")
+#         for crawler in list(self.crawlers):
+#             if crawler.engine:
+#                 crawler.engine.running = False
+#                 crawler.engine.normal = False
+#         asyncio.create_task(self._wait_for_shutdown())
+#
+#     async def _wait_for_shutdown(self):
+#         """等待所有活跃任务完成"""
+#         pending = [t for t in self._active_tasks if not t.done()]
+#         if pending:
+#             logger.info(f"等待 {len(pending)} 个活跃任务完成...")
+#             await asyncio.gather(*pending, return_exceptions=True)
+#         logger.info("所有爬虫已优雅关闭")
+#
+#     @classmethod
+#     def _get_default_settings(cls) -> SettingManager:
+#         """加载默认配置"""
+#         try:
+#             return get_settings()
+#         except Exception as e:
+#             logger.warning(f"无法加载默认配置: {e}")
+#             return SettingManager()

crawlo/spider/__init__.py

@@ -1,11 +1,44 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+from __future__ import annotations
+from typing import Type, Any, Optional, List, Dict
 from ..network.request import Request
 from ..utils.log import get_logger
 
 
-class Spider(object):
-    name = None
+# 全局注册表
+_DEFAULT_SPIDER_REGISTRY: dict[str, Type[Spider]] = {}
+
+
+class SpiderMeta(type):
+    def __new__(mcs, name: str, bases: tuple[type], namespace: dict[str, Any], **kwargs):
+        cls = super().__new__(mcs, name, bases, namespace)
+
+        is_spider_subclass = any(
+            base is Spider or (isinstance(base, type) and issubclass(base, Spider))
+            for base in bases
+        )
+        if not is_spider_subclass:
+            return cls
+
+        spider_name = namespace.get('name')
+        if not isinstance(spider_name, str):
+            raise AttributeError(f"爬虫类 '{cls.__name__}' 必须定义字符串类型的 'name' 属性。")
+
+        if spider_name in _DEFAULT_SPIDER_REGISTRY:
+            raise ValueError(
+                f"爬虫名称 '{spider_name}' 已被 {_DEFAULT_SPIDER_REGISTRY[spider_name].__name__} 占用。"
+                f"请确保每个爬虫的 name 属性全局唯一。"
+            )
+
+        _DEFAULT_SPIDER_REGISTRY[spider_name] = cls
+        get_logger(__name__).debug(f"自动注册爬虫: {spider_name} -> {cls.__name__}")
+
+        return cls
+
+
+class Spider(metaclass=SpiderMeta):
+    name: str = None
 
     def __init__(self, name=None, **kwargs):
         if not hasattr(self, 'start_urls'):
@@ -15,7 +48,7 @@ class Spider(object):
         self.logger = get_logger(self.name or self.__class__.__name__)
 
     @classmethod
-    def create_instance(cls, crawler):
+    def create_instance(cls, crawler) -> Spider:
         o = cls()
         o.crawler = crawler
         return o
@@ -39,3 +72,58 @@ class Spider(object):
 
     def __str__(self):
         return self.__class__.__name__
+
+
+# === 公共只读接口 ===
+def get_global_spider_registry() -> dict[str, Type[Spider]]:
+    return _DEFAULT_SPIDER_REGISTRY.copy()
+
+
+def get_spider_by_name(name: str) -> Optional[Type[Spider]]:
+    return _DEFAULT_SPIDER_REGISTRY.get(name)
+
+
+def get_all_spider_classes() -> list[Type[Spider]]:
+    return list(set(_DEFAULT_SPIDER_REGISTRY.values()))
+
+# #!/usr/bin/python
+# # -*- coding:UTF-8 -*-
+# from ..network.request import Request
+# from ..utils.log import get_logger
+#
+#
+# class Spider(object):
+#     name = None
+#
+#     def __init__(self, name=None, **kwargs):
+#         if not hasattr(self, 'start_urls'):
+#             self.start_urls = []
+#         self.crawler = None
+#         self.name = name or self.name
+#         self.logger = get_logger(self.name or self.__class__.__name__)
+#
+#     @classmethod
+#     def create_instance(cls, crawler):
+#         o = cls()
+#         o.crawler = crawler
+#         return o
+#
+#     def start_requests(self):
+#         if self.start_urls:
+#             for url in self.start_urls:
+#                 yield Request(url=url, dont_filter=True)
+#         else:
+#             if hasattr(self, 'start_url') and isinstance(getattr(self, 'start_url'), str):
+#                 yield Request(getattr(self, 'start_url'), dont_filter=True)
+#
+#     def parse(self, response):
+#         raise NotImplementedError
+#
+#     async def spider_opened(self):
+#         pass
+#
+#     async def spider_closed(self):
+#         pass
+#
+#     def __str__(self):
+#         return self.__class__.__name__

crawlo/utils/project.py

@@ -19,8 +19,7 @@ from typing import Callable, Optional
 from crawlo.utils.log import get_logger
 from crawlo.settings.setting_manager import SettingManager
 
-
-logger =get_logger(__name__)
+logger = get_logger(__name__)
 
 
 def _find_project_root(start_path: str = '.') -> Optional[str]:
@@ -37,13 +36,11 @@ def _find_project_root(start_path: str = '.') -> Optional[str]:
         Optional[str]: 找到的项目根目录的绝对路径，如果未找到则返回 None。
     """
     path = os.path.abspath(start_path)
-    logger.info(f"开始向上搜索项目根目录，起始路径: {path}")
 
     while True:
         # 1. 检查是否存在 crawlo.cfg 文件
         cfg_file = os.path.join(path, 'crawlo.cfg')
         if os.path.isfile(cfg_file):
-            logger.info(f"在路径 {path} 找到 'crawlo.cfg' 文件，确定为项目根目录。")
             return path
 
         # 2. 检查是否存在 settings.py 文件，并且它位于一个 Python 包中
@@ -51,7 +48,6 @@ def _find_project_root(start_path: str = '.') -> Optional[str]:
         if os.path.isfile(settings_file):
             init_file = os.path.join(path, '__init__.py')
             if os.path.isfile(init_file):
-                logger.info(f"在路径 {path} 找到 'settings.py' 文件，确定为项目根目录。")
                 return path
             else:
                 logger.debug(f"在路径 {path} 找到 'settings.py'，但缺少 '__init__.py'，忽略。")
@@ -86,7 +82,7 @@ def _get_settings_module_from_cfg(cfg_path: str) -> str:
         config.read(cfg_path, encoding='utf-8')
         if config.has_section('settings') and config.has_option('settings', 'default'):
             module_path = config.get('settings', 'default')
-            logger.info(f"从 'crawlo.cfg' 中读取到 settings 模块路径: {module_path}")
+            logger.debug(f"从 'crawlo.cfg' 中读取到 settings 模块路径: {module_path}")
             return module_path
         else:
             error_msg = f"配置文件 '{cfg_path}' 缺少 '[settings]' 或 'default' 配置项。"
@@ -113,7 +109,7 @@ def get_settings(custom_settings=None):
         RuntimeError: 当无法找到项目或配置文件时。
         ImportError: 当无法导入指定的 settings 模块时。
     """
-    logger.info("正在初始化配置管理器...")
+    logger.debug("正在初始化配置管理器...")
 
     # 1. 发现项目根目录
     project_root = _find_project_root()
@@ -122,7 +118,7 @@ def get_settings(custom_settings=None):
         logger.error(error_msg)
         raise RuntimeError(error_msg)
 
-    logger.info(f"项目根目录已确定: {project_root}")
+    logger.debug(f"项目根目录已确定: {project_root}")
 
     # 2. 确定 settings 模块的导入路径
     settings_module_path = None
@@ -132,27 +128,27 @@ def get_settings(custom_settings=None):
     if os.path.isfile(cfg_file):
         settings_module_path = _get_settings_module_from_cfg(cfg_file)
     else:
-        logger.info("未找到 'crawlo.cfg'，尝试推断 settings 模块路径...")
+        logger.debug("未找到 'crawlo.cfg'，尝试推断 settings 模块路径...")
         # 推断：项目目录名.settings
         project_name = os.path.basename(project_root)
         settings_module_path = f"{project_name}.settings"
-        logger.info(f"推断 settings 模块路径为: {settings_module_path}")
+        logger.debug(f"推断 settings 模块路径为: {settings_module_path}")
 
     # 3. 将项目根目录添加到 Python 路径，确保可以成功导入
     if project_root not in sys.path:
         sys.path.insert(0, project_root)
-        logger.info(f"已将项目根目录 '{project_root}' 添加到 Python 路径。")
+        logger.debug(f"已将项目根目录 '{project_root}' 添加到 Python 路径。")
     else:
         logger.debug(f"项目根目录 '{project_root}' 已在 Python 路径中。")
 
     # 4. 创建 SettingManager 并加载配置
-    logger.info(f"正在加载 settings 模块: {settings_module_path}")
+    logger.debug(f"正在加载 settings 模块: {settings_module_path}")
     settings = SettingManager()
 
     try:
         # 这会触发 SettingManager.set_settings()，从模块中加载所有大写常量
         settings.set_settings(settings_module_path)
-        logger.info("settings 模块加载成功。")
+        logger.debug("settings 模块加载成功。")
     except Exception as e:
         error_msg = f"加载 settings 模块 '{settings_module_path}' 失败: {e}"
         logger.error(error_msg)
@@ -160,13 +156,14 @@ def get_settings(custom_settings=None):
 
     # 5. 应用运行时自定义设置
     if custom_settings:
-        logger.info(f"正在应用运行时自定义设置: {custom_settings}")
+        logger.debug(f"正在应用运行时自定义设置: {custom_settings}")
         settings.update_attributes(custom_settings)
         logger.info("运行时自定义设置已应用。")
 
-    logger.info("配置管理器初始化完成。")
+    logger.debug("配置管理器初始化完成。")
     return settings
 
+
 def load_class(_path):
     if not isinstance(_path, str):
         if callable(_path):
@@ -183,13 +180,14 @@ def load_class(_path):
         raise NameError(f"Module {module_name!r} has no class named {class_name!r}")
     return cls
 
+
 def merge_settings(spider, settings):
     spider_name = getattr(spider, 'name', 'UnknownSpider')
     if hasattr(spider, 'custom_settings'):
         custom_settings = getattr(spider, 'custom_settings')
         settings.update_attributes(custom_settings)
     else:
-        logger.debug(f"爬虫 '{spider_name}' 无 custom_settings，跳过合并") # 添加日志
+        logger.debug(f"爬虫 '{spider_name}' 无 custom_settings，跳过合并")  # 添加日志
 
 
 async def common_call(func: Callable, *args, **kwargs):

{crawlo-1.0.7.dist-info → crawlo-1.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.0.7
+Version: 1.0.8
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

{crawlo-1.0.7.dist-info → crawlo-1.0.8.dist-info}/RECORD

@@ -1,16 +1,19 @@
 crawlo/__init__.py,sha256=xpiIAZbSG3CzneJuDLPCbwfRcvw2wyHYl2kJjaNfNGY,584
-crawlo/__version__.py,sha256=xYbb67T0bE-s3OjgVTmJzJVvjxFP4YEjxAaBB2nj3zA,23
+crawlo/__version__.py,sha256=uyL3a6o1xccXPZ2OS65zqIN_lbEMT7PcCxErq7cuWwA,23
 crawlo/cli.py,sha256=hjAJKx9pba375sATvvcy-dtZyBIgXj8fRBq9RFIZHA4,1206
-crawlo/crawler.py,sha256=nfuA_f8QnuIp2F4ZbaJv8Fceo_QPwqV1jYdD_edkMjg,8527
+crawlo/crawler.py,sha256=AyKxUyJvCwb1u4d3Zn3vFmjH28ExWKIygfTICps-3yY,20026
 crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
 crawlo/exceptions.py,sha256=xdyZkvVcLEJ-19sWMHvn9IJsu30-hAY2jJhA2kYIims,1207
 crawlo/stats_collector.py,sha256=v4jC9BAe-23w93hWzbeMCCgQ9VuFPyxw5JV9ItbGH8w,1636
 crawlo/subscriber.py,sha256=udlHeTR0ymGQhCDxVUGwUzeeeR4TYCEJrJwFnkgr0cU,3836
 crawlo/task_manager.py,sha256=D9m-nqnGj-FZPtGk4CdwZX3Gw7IWyYvTS7CHpRGWc_w,748
 crawlo/commands/__init__.py,sha256=dRu3ipuhDM7M1eTb6zJtQZ_u7N_tZumGfH5_I92xno8,252
+crawlo/commands/check.py,sha256=Q8wFjIo43XW0wP93TTlM7HSShgytJsbSWHIlmkcNxz0,3585
 crawlo/commands/genspider.py,sha256=kSHYsAGHRoxU6Qf_MGpR_VS-Ua5NUGY2KGm_Wapn0sw,3529
-crawlo/commands/run.py,sha256=Upv8K4sM0c0I1fIwTFK18VDcSHF7xabqfXtQ82fk56g,4628
+crawlo/commands/list.py,sha256=itR05muZlZs8FbRh88kOhcRbZc77OXiR6A86UnVhSMY,2974
+crawlo/commands/run.py,sha256=s6JJC8HNa-tBgPDB2BPUmj26D7PMckhlx4AOEz57ESY,6197
 crawlo/commands/startproject.py,sha256=1KOq_CALy01oklr0dAUYhGFzu4f7w45q2H0O3qafLX4,3494
+crawlo/commands/stats.py,sha256=rH0TlD0o-xUr9RxtvNYgnSjHHoRyma3rvx9Q9nIGDNg,1659
 crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
 crawlo/core/engine.py,sha256=JFHooPp-5cfHSyxEh87nOOR5NMaPLVDfNSqAsbtx4PM,6030
 crawlo/core/processor.py,sha256=oHLs-cno0bJGTNc9NGD2S7_2-grI3ruvggO0SY2mf3Q,1180
@@ -51,7 +54,7 @@ crawlo/pipelines/pipeline_manager.py,sha256=k-Rg0os0Havrov99D-Jn3ROpnz154K30tf7a
 crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
 crawlo/settings/default_settings.py,sha256=urj4XJ--ZpVRbbo3fWUT71bYQLmElx43AC9KeHtqHBs,7310
 crawlo/settings/setting_manager.py,sha256=4xXOzKwZCgAp8ybwvVcs2R--CsOD7c6dBIkj6DJHB3c,2998
-crawlo/spider/__init__.py,sha256=lWi0bCR7HLT5bnj7_e9UIgFJjuqoeWtbwADfNkaajug,1139
+crawlo/spider/__init__.py,sha256=IyQd4ufbAIhA_cvWrsNReRv3tj76CHc5Aef9c8KR-9s,3983
 crawlo/templates/crawlo.cfg.tmpl,sha256=lwiUVe5sFixJgHFEjn1OtbAeyWsECOrz37uheuVtulk,240
 crawlo/templates/project/__init__.py.tmpl,sha256=aQnHaOjMSkTviOC8COUX0fKymuyf8lx2tGduxkMkXEE,61
 crawlo/templates/project/items.py.tmpl,sha256=bXx-oCldMr2EgBKUAH9LH5gMnbyLiWX-EySAaMzcu2g,318
@@ -67,7 +70,7 @@ crawlo/utils/db_helper.py,sha256=ZqOt1d3mErVv4TOvoWlov0niUxORB9aHByTmMoNFIDw,109
 crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
 crawlo/utils/log.py,sha256=YD2FfXuuE2MC9ZdQQZ0H7KysE7l_LHZqQepaTPlcApo,4133
 crawlo/utils/pqueue.py,sha256=HDgX4HAkc7RqYUtX6q51tzI1ZRTACf8P_4jLqC4-uC0,5559
-crawlo/utils/project.py,sha256=qAiCmpIxiB7RxCLG-U5lGV6k4UCa21uRdykTfnAF834,7669
+crawlo/utils/project.py,sha256=hXSKV55OBUFjJi7TXekB4X3MmAgsqAeVTj5wPUWOizc,7394
 crawlo/utils/request.py,sha256=ejdKpTwc-HE04HQybafhOVywzz57IV3pY0YMkSLyGUo,9065
 crawlo/utils/spider_loader.py,sha256=V0CBTicJBYBZafhwLfDEfuEc_hJ2mSoiptT6qKufI9U,2249
 crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
@@ -87,8 +90,8 @@ tests/test_proxy_middleware_integration.py,sha256=mTPK_XvbmLCV_QoVZzA3ybWOOX6149
 tests/test_proxy_providers.py,sha256=u_R2fhab90vqvQEaOAztpAOe9tJXvUMIdoDxmStmXJ4,1749
 tests/test_proxy_stats.py,sha256=ES00CEoDITYPFBGPk8pecFzD3ItYIv6NSpcqNd8-kvo,526
 tests/test_proxy_strategies.py,sha256=9Z1pXmTNyw-eIhGXlf2abZbJx6igLohYq-_3hldQ5uE,1868
-crawlo-1.0.7.dist-info/METADATA,sha256=ZjDus84i4MbVr_-7BEStA2OmZHEGcdw-b1MGUosylOI,1825
-crawlo-1.0.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.0.7.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.0.7.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.0.7.dist-info/RECORD,,
+crawlo-1.0.8.dist-info/METADATA,sha256=ia-nA0g0Rl76iHFIlvaRbvUnjd88KEKoxIrJKcjtCyw,1825
+crawlo-1.0.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.0.8.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.0.8.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.0.8.dist-info/RECORD,,