crawlo 1.4.6__py3-none-any.whl → 1.4.7__py3-none-any.whl

crawlo/__init__.py

@@ -1,89 +1,90 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Crawlo - 一个异步爬虫框架
-"""
-
-# 为了向后兼容，从tools中导入cleaners相关的功能
-import crawlo.tools as cleaners
-from crawlo import tools
-from crawlo.crawler import CrawlerProcess
-from crawlo.downloader import DownloaderBase
-from crawlo.items import Item, Field
-from crawlo.middleware import BaseMiddleware
-from crawlo.network.request import Request
-from crawlo.network.response import Response
-from crawlo.spider import Spider
-from crawlo.utils import (
-    TimeUtils,
-    parse_time,
-    format_time,
-    time_diff,
-    to_timestamp,
-    to_datetime,
-    now,
-    to_timezone,
-    to_utc,
-    to_local,
-    from_timestamp_with_tz
-)
-
-
-# 延迟导入的辅助函数
-def get_framework_initializer():
-    """延迟导入CoreInitializer以避免循环依赖"""
-    from crawlo.initialization import CoreInitializer
-    return CoreInitializer()
-
-
-def initialize_framework(custom_settings=None):
-    """延迟导入initialize_framework以避免循环依赖"""
-    from crawlo.initialization import initialize_framework as _initialize_framework
-    return _initialize_framework(custom_settings)
-
-
-# 向后兼容的别名
-def get_bootstrap_manager():
-    """向后兼容的别名"""
-    return get_framework_initializer()
-
-
-# 版本号：优先从元数据读取
-try:
-    from importlib.metadata import version
-
-    __version__ = version("crawlo")
-except Exception:
-    # 开发模式下可能未安装，回退到 __version__.py 或 dev
-    try:
-        from crawlo.__version__ import __version__
-    except ImportError:
-        __version__ = "dev"
-
-# 定义对外 API
-__all__ = [
-    'Spider',
-    'Item',
-    'Field',
-    'Request',
-    'Response',
-    'DownloaderBase',
-    'BaseMiddleware',
-    'TimeUtils',
-    'parse_time',
-    'format_time',
-    'time_diff',
-    'to_timestamp',
-    'to_datetime',
-    'now',
-    'to_timezone',
-    'to_utc',
-    'to_local',
-    'from_timestamp_with_tz',
-    'cleaners',
-    'tools',
-    'CrawlerProcess',
-    'get_framework_initializer',
-    'get_bootstrap_manager',
-    '__version__',
-]
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo - 一个异步爬虫框架
+"""
+
+# 为了向后兼容，从tools中导入cleaners相关的功能
+import crawlo.tools as cleaners
+from crawlo import tools
+from crawlo.crawler import Crawler, CrawlerProcess
+from crawlo.downloader import DownloaderBase
+from crawlo.items import Item, Field
+from crawlo.middleware import BaseMiddleware
+from crawlo.network.request import Request
+from crawlo.network.response import Response
+from crawlo.spider import Spider
+from crawlo.utils import (
+    TimeUtils,
+    parse_time,
+    format_time,
+    time_diff,
+    to_timestamp,
+    to_datetime,
+    now,
+    to_timezone,
+    to_utc,
+    to_local,
+    from_timestamp_with_tz
+)
+
+
+# 延迟导入的辅助函数
+def get_framework_initializer():
+    """延迟导入CoreInitializer以避免循环依赖"""
+    from crawlo.initialization import CoreInitializer
+    return CoreInitializer()
+
+
+def initialize_framework(custom_settings=None):
+    """延迟导入initialize_framework以避免循环依赖"""
+    from crawlo.initialization import initialize_framework as _initialize_framework
+    return _initialize_framework(custom_settings)
+
+
+# 向后兼容的别名
+def get_bootstrap_manager():
+    """向后兼容的别名"""
+    return get_framework_initializer()
+
+
+# 版本号：优先从元数据读取
+try:
+    from importlib.metadata import version
+
+    __version__ = version("crawlo")
+except Exception:
+    # 开发模式下可能未安装，回退到 __version__.py 或 dev
+    try:
+        from crawlo.__version__ import __version__
+    except ImportError:
+        __version__ = "dev"
+
+# 定义对外 API
+__all__ = [
+    'Spider',
+    'Item',
+    'Field',
+    'Request',
+    'Response',
+    'DownloaderBase',
+    'BaseMiddleware',
+    'TimeUtils',
+    'parse_time',
+    'format_time',
+    'time_diff',
+    'to_timestamp',
+    'to_datetime',
+    'now',
+    'to_timezone',
+    'to_utc',
+    'to_local',
+    'from_timestamp_with_tz',
+    'cleaners',
+    'tools',
+    'Crawler',
+    'CrawlerProcess',
+    'get_framework_initializer',
+    'get_bootstrap_manager',
+    '__version__',
+]

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = '1.4.6'
+__version__ = '1.4.7'

crawlo/cli.py

@@ -1,76 +1,76 @@
-# crawlo/cli.py
-# !/usr/bin/python
-# -*- coding: UTF-8 -*-
-import sys
-import argparse
-from crawlo.commands import get_commands
-from crawlo.utils.env_config import get_version
-
-
-def main():
-    # 获取框架版本号
-    VERSION = get_version()
-
-    # 获取所有可用命令
-    commands = get_commands()
-
-    # 创建主解析器
-    parser = argparse.ArgumentParser(
-        description="Crawlo: A lightweight web crawler framework.",
-        usage="crawlo <command> [options]",
-        add_help=False  # 禁用默认帮助，我们自己处理
-    )
-    
-    # 添加帮助参数
-    parser.add_argument('-h', '--help', action='store_true', help='显示帮助信息')
-    parser.add_argument('-v', '--version', action='store_true', help='显示版本信息')
-    parser.add_argument('command', nargs='?', help='可用命令: ' + ', '.join(commands.keys()))
-    
-    # 解析已知参数
-    args, unknown = parser.parse_known_args()
-
-    # 处理版本参数
-    if args.version:
-        print(f"Crawlo {VERSION}")
-        sys.exit(0)
-
-    # 处理帮助参数
-    if args.help or (args.command is None and not unknown):
-        # 导入并运行帮助命令
-        try:
-            module = __import__(commands['help'], fromlist=['main'])
-            sys.exit(module.main([]))
-        except ImportError as e:
-            print(f"Failed to load help command: {e}")
-            sys.exit(1)
-        except Exception as e:
-            print(f"Help command failed: {e}")
-            sys.exit(1)
-
-    # 检查命令是否存在
-    if args.command not in commands:
-        print(f"Unknown command: {args.command}")
-        print(f"Available commands: {', '.join(commands.keys())}")
-        # 显示帮助信息
-        try:
-            module = __import__(commands['help'], fromlist=['main'])
-            module.main([])
-        except:
-            pass
-        sys.exit(1)
-
-    # 动态导入并执行命令
-    try:
-        module = __import__(commands[args.command], fromlist=['main'])
-        # 将未知参数传递给子命令
-        sys.exit(module.main(unknown))
-    except ImportError as e:
-        print(f"Failed to load command '{args.command}': {e}")
-        sys.exit(1)
-    except Exception as e:
-        print(f"Command '{args.command}' failed: {e}")
-        sys.exit(1)
-
-
-if __name__ == '__main__':
+# crawlo/cli.py
+# !/usr/bin/python
+# -*- coding: UTF-8 -*-
+import sys
+import argparse
+from crawlo.commands import get_commands
+from crawlo.utils.config_manager import EnvConfigManager
+
+
+def main():
+    # 获取框架版本号
+    VERSION = EnvConfigManager.get_version()
+
+    # 获取所有可用命令
+    commands = get_commands()
+
+    # 创建主解析器
+    parser = argparse.ArgumentParser(
+        description="Crawlo: A lightweight web crawler framework.",
+        usage="crawlo <command> [options]",
+        add_help=False  # 禁用默认帮助，我们自己处理
+    )
+    
+    # 添加帮助参数
+    parser.add_argument('-h', '--help', action='store_true', help='显示帮助信息')
+    parser.add_argument('-v', '--version', action='store_true', help='显示版本信息')
+    parser.add_argument('command', nargs='?', help='可用命令: ' + ', '.join(commands.keys()))
+    
+    # 解析已知参数
+    args, unknown = parser.parse_known_args()
+
+    # 处理版本参数
+    if args.version:
+        print(f"Crawlo {VERSION}")
+        sys.exit(0)
+
+    # 处理帮助参数
+    if args.help or (args.command is None and not unknown):
+        # 导入并运行帮助命令
+        try:
+            module = __import__(commands['help'], fromlist=['main'])
+            sys.exit(module.main([]))
+        except ImportError as e:
+            print(f"Failed to load help command: {e}")
+            sys.exit(1)
+        except Exception as e:
+            print(f"Help command failed: {e}")
+            sys.exit(1)
+
+    # 检查命令是否存在
+    if args.command not in commands:
+        print(f"Unknown command: {args.command}")
+        print(f"Available commands: {', '.join(commands.keys())}")
+        # 显示帮助信息
+        try:
+            module = __import__(commands['help'], fromlist=['main'])
+            module.main([])
+        except:
+            pass
+        sys.exit(1)
+
+    # 动态导入并执行命令
+    try:
+        module = __import__(commands[args.command], fromlist=['main'])
+        # 将未知参数传递给子命令
+        sys.exit(module.main(unknown))
+    except ImportError as e:
+        print(f"Failed to load command '{args.command}': {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Command '{args.command}' failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
     main()

crawlo/commands/__init__.py

@@ -1,15 +1,15 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-
-_commands = {
-    'startproject': 'crawlo.commands.startproject',
-    'genspider': 'crawlo.commands.genspider',
-    'run': 'crawlo.commands.run',
-    'check': 'crawlo.commands.check',
-    'list': 'crawlo.commands.list',
-    'stats': 'crawlo.commands.stats',
-    'help': 'crawlo.commands.help'
-}
-
-def get_commands():
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+
+_commands = {
+    'startproject': 'crawlo.commands.startproject',
+    'genspider': 'crawlo.commands.genspider',
+    'run': 'crawlo.commands.run',
+    'check': 'crawlo.commands.check',
+    'list': 'crawlo.commands.list',
+    'stats': 'crawlo.commands.stats',
+    'help': 'crawlo.commands.help'
+}
+
+def get_commands():
     return _commands