crawlo 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

tests/date_tools_example.py

@@ -1,181 +1,181 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Crawlo框架日期工具使用示例
-"""
-from crawlo.tools import (
-    TimeUtils,
-    parse_time,
-    format_time,
-    time_diff,
-    to_timestamp,
-    to_datetime,
-    now,
-    to_timezone,
-    to_utc,
-    to_local
-)
-
-
-def demo_date_parsing():
-    """演示日期解析功能"""
-    print("=== 日期解析演示 ===\n")
-    
-    # 解析各种格式的日期字符串
-    date_strings = [
-        "2025-09-10 14:30:00",
-        "September 10, 2025 2:30 PM",
-        "2025/09/10 14:30:00",
-        "10-09-2025 14:30:00",
-        "2025年9月10日 14时30分00秒"
-    ]
-    
-    for date_str in date_strings:
-        parsed = parse_time(date_str)
-        print(f"解析 '{date_str}' -> {parsed}")
-    
-    print()
-
-
-def demo_date_formatting():
-    """演示日期格式化功能"""
-    print("=== 日期格式化演示 ===\n")
-    
-    # 获取当前时间
-    current_time = now()
-    print(f"当前时间: {current_time}")
-    
-    # 使用不同格式进行格式化
-    formats = [
-        "%Y-%m-%d",
-        "%Y年%m月%d日",
-        "%Y-%m-%d %H:%M:%S",
-        "%B %d, %Y at %I:%M %p",
-        "%A, %B %d, %Y"
-    ]
-    
-    for fmt in formats:
-        formatted = format_time(current_time, fmt)
-        print(f"格式化为 '{fmt}' -> {formatted}")
-    
-    print()
-
-
-def demo_time_calculations():
-    """演示时间计算功能"""
-    print("=== 时间计算演示 ===\n")
-    
-    # 计算时间差
-    start_time = "2025-09-10 10:00:00"
-    end_time = "2025-09-10 14:30:45"
-    
-    diff_seconds = time_diff(start_time, end_time, "seconds")
-    diff_minutes = time_diff(start_time, end_time, "minutes")
-    diff_hours = time_diff(start_time, end_time, "hours")
-    
-    print(f"起始时间: {start_time}")
-    print(f"结束时间: {end_time}")
-    print(f"时间差 - 秒: {diff_seconds}")
-    print(f"时间差 - 分钟: {diff_minutes}")
-    print(f"时间差 - 小时: {diff_hours}")
-    
-    print()
-    
-    # 日期加减
-    base_date = "2025-09-10"
-    plus_30_days = TimeUtils.add_days(base_date, 30)
-    minus_15_days = TimeUtils.add_days(base_date, -15)
-    plus_3_months = TimeUtils.add_months(base_date, 3)
-    
-    print(f"基础日期: {base_date}")
-    print(f"加30天: {plus_30_days}")
-    print(f"减15天: {minus_15_days}")
-    print(f"加3个月: {plus_3_months}")
-    
-    print()
-
-
-def demo_timezone_conversion():
-    """演示时区转换功能"""
-    print("=== 时区转换演示 ===\n")
-    
-    # 创建一个时间
-    dt = parse_time("2025-09-10 14:30:00")
-    print(f"原始时间: {dt}")
-    
-    # 转换为UTC时区
-    utc_time = to_utc(dt)
-    print(f"UTC时间: {utc_time}")
-    
-    # 转换为本地时区
-    local_time = to_local(dt)
-    print(f"本地时间: {local_time}")
-    
-    # 转换为纽约时区
-    ny_time = to_timezone(dt, "America/New_York")
-    print(f"纽约时间: {ny_time}")
-    
-    # 转换为伦敦时区
-    london_time = to_timezone(dt, "Europe/London")
-    print(f"伦敦时间: {london_time}")
-    
-    print()
-
-
-def demo_timestamp_conversion():
-    """演示时间戳转换功能"""
-    print("=== 时间戳转换演示 ===\n")
-    
-    # 获取当前时间戳
-    current_timestamp = to_timestamp(now())
-    print(f"当前时间戳: {current_timestamp}")
-    
-    # 从时间戳转换为datetime
-    dt_from_timestamp = to_datetime(current_timestamp)
-    print(f"从时间戳转换: {dt_from_timestamp}")
-    
-    # 从时间戳创建带时区的datetime
-    dt_with_tz = to_timezone(to_datetime(current_timestamp), "Asia/Shanghai")
-    print(f"带时区的时间: {dt_with_tz}")
-    
-    print()
-
-
-def demo_in_spider():
-    """演示在爬虫中使用日期工具"""
-    print("=== 在爬虫中使用日期工具 ===\n")
-    print("在爬虫项目中，您可以这样使用日期工具:")
-    print("""
-from crawlo import Spider
-from crawlo.tools import parse_time, format_time
-
-class NewsSpider(Spider):
-    def parse(self, response):
-        # 提取文章发布时间
-        publish_time_str = response.css('.publish-time::text').get()
-        
-        # 解析发布时间
-        publish_time = parse_time(publish_time_str)
-        
-        # 格式化时间为标准格式
-        formatted_time = format_time(publish_time, "%Y-%m-%d %H:%M:%S")
-        
-        # 计算文章发布多久了
-        current_time = self.tools.now()
-        hours_since_publish = self.tools.time_diff(publish_time, current_time, "hours")
-        
-        # 根据发布时间决定是否继续处理
-        if hours_since_publish < 24:  # 只处理24小时内的文章
-            # 处理文章...
-            pass
-    """)
-
-
-if __name__ == '__main__':
-    # 运行演示
-    demo_date_parsing()
-    demo_date_formatting()
-    demo_time_calculations()
-    demo_timezone_conversion()
-    demo_timestamp_conversion()
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo框架日期工具使用示例
+"""
+from crawlo.tools import (
+    TimeUtils,
+    parse_time,
+    format_time,
+    time_diff,
+    to_timestamp,
+    to_datetime,
+    now,
+    to_timezone,
+    to_utc,
+    to_local
+)
+
+
+def demo_date_parsing():
+    """演示日期解析功能"""
+    print("=== 日期解析演示 ===\n")
+    
+    # 解析各种格式的日期字符串
+    date_strings = [
+        "2025-09-10 14:30:00",
+        "September 10, 2025 2:30 PM",
+        "2025/09/10 14:30:00",
+        "10-09-2025 14:30:00",
+        "2025年9月10日 14时30分00秒"
+    ]
+    
+    for date_str in date_strings:
+        parsed = parse_time(date_str)
+        print(f"解析 '{date_str}' -> {parsed}")
+    
+    print()
+
+
+def demo_date_formatting():
+    """演示日期格式化功能"""
+    print("=== 日期格式化演示 ===\n")
+    
+    # 获取当前时间
+    current_time = now()
+    print(f"当前时间: {current_time}")
+    
+    # 使用不同格式进行格式化
+    formats = [
+        "%Y-%m-%d",
+        "%Y年%m月%d日",
+        "%Y-%m-%d %H:%M:%S",
+        "%B %d, %Y at %I:%M %p",
+        "%A, %B %d, %Y"
+    ]
+    
+    for fmt in formats:
+        formatted = format_time(current_time, fmt)
+        print(f"格式化为 '{fmt}' -> {formatted}")
+    
+    print()
+
+
+def demo_time_calculations():
+    """演示时间计算功能"""
+    print("=== 时间计算演示 ===\n")
+    
+    # 计算时间差
+    start_time = "2025-09-10 10:00:00"
+    end_time = "2025-09-10 14:30:45"
+    
+    diff_seconds = time_diff(start_time, end_time, "seconds")
+    diff_minutes = time_diff(start_time, end_time, "minutes")
+    diff_hours = time_diff(start_time, end_time, "hours")
+    
+    print(f"起始时间: {start_time}")
+    print(f"结束时间: {end_time}")
+    print(f"时间差 - 秒: {diff_seconds}")
+    print(f"时间差 - 分钟: {diff_minutes}")
+    print(f"时间差 - 小时: {diff_hours}")
+    
+    print()
+    
+    # 日期加减
+    base_date = "2025-09-10"
+    plus_30_days = TimeUtils.add_days(base_date, 30)
+    minus_15_days = TimeUtils.add_days(base_date, -15)
+    plus_3_months = TimeUtils.add_months(base_date, 3)
+    
+    print(f"基础日期: {base_date}")
+    print(f"加30天: {plus_30_days}")
+    print(f"减15天: {minus_15_days}")
+    print(f"加3个月: {plus_3_months}")
+    
+    print()
+
+
+def demo_timezone_conversion():
+    """演示时区转换功能"""
+    print("=== 时区转换演示 ===\n")
+    
+    # 创建一个时间
+    dt = parse_time("2025-09-10 14:30:00")
+    print(f"原始时间: {dt}")
+    
+    # 转换为UTC时区
+    utc_time = to_utc(dt)
+    print(f"UTC时间: {utc_time}")
+    
+    # 转换为本地时区
+    local_time = to_local(dt)
+    print(f"本地时间: {local_time}")
+    
+    # 转换为纽约时区
+    ny_time = to_timezone(dt, "America/New_York")
+    print(f"纽约时间: {ny_time}")
+    
+    # 转换为伦敦时区
+    london_time = to_timezone(dt, "Europe/London")
+    print(f"伦敦时间: {london_time}")
+    
+    print()
+
+
+def demo_timestamp_conversion():
+    """演示时间戳转换功能"""
+    print("=== 时间戳转换演示 ===\n")
+    
+    # 获取当前时间戳
+    current_timestamp = to_timestamp(now())
+    print(f"当前时间戳: {current_timestamp}")
+    
+    # 从时间戳转换为datetime
+    dt_from_timestamp = to_datetime(current_timestamp)
+    print(f"从时间戳转换: {dt_from_timestamp}")
+    
+    # 从时间戳创建带时区的datetime
+    dt_with_tz = to_timezone(to_datetime(current_timestamp), "Asia/Shanghai")
+    print(f"带时区的时间: {dt_with_tz}")
+    
+    print()
+
+
+def demo_in_spider():
+    """演示在爬虫中使用日期工具"""
+    print("=== 在爬虫中使用日期工具 ===\n")
+    print("在爬虫项目中，您可以这样使用日期工具:")
+    print("""
+from crawlo import Spider
+from crawlo.tools import parse_time, format_time
+
+class NewsSpider(Spider):
+    def parse(self, response):
+        # 提取文章发布时间
+        publish_time_str = response.css('.publish-time::text').get()
+        
+        # 解析发布时间
+        publish_time = parse_time(publish_time_str)
+        
+        # 格式化时间为标准格式
+        formatted_time = format_time(publish_time, "%Y-%m-%d %H:%M:%S")
+        
+        # 计算文章发布多久了
+        current_time = self.tools.now()
+        hours_since_publish = self.tools.time_diff(publish_time, current_time, "hours")
+        
+        # 根据发布时间决定是否继续处理
+        if hours_since_publish < 24:  # 只处理24小时内的文章
+            # 处理文章...
+            pass
+    """)
+
+
+if __name__ == '__main__':
+    # 运行演示
+    demo_date_parsing()
+    demo_date_formatting()
+    demo_time_calculations()
+    demo_timezone_conversion()
+    demo_timestamp_conversion()
     demo_in_spider()

tests/debug_configure.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+逐步调试LoggerManager.configure
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+from crawlo.utils.log import LoggerManager
+
+print("=== 逐步调试LoggerManager.configure ===")
+
+print("1. 检查初始状态")
+print(f"   _early_initialized: {LoggerManager._early_initialized}")
+print(f"   _configured: {LoggerManager._configured}")
+
+print("2. 调用early_initialize")
+LoggerManager.early_initialize()
+print(f"   _early_initialized: {LoggerManager._early_initialized}")
+
+print("3. 进入configure - 开始")
+# 模拟configure方法的各个步骤
+with LoggerManager._config_lock:
+    print("   获得锁")
+    
+    if LoggerManager._configured:
+        print("   已配置，直接返回")
+    else:
+        print("   开始配置...")
+        
+        # 更新状态
+        print("   设置_log_state")
+        from crawlo.utils.log import _log_state
+        _log_state['current_step'] = 'basic_setup'
+        
+        print("   处理参数")
+        kwargs = {'LOG_LEVEL': 'INFO', 'LOG_FILE': 'test.log'}
+        get_val = lambda k, d=None: kwargs.get(k, d)
+        
+        filename = get_val('LOG_FILE')
+        level = get_val('LOG_LEVEL', None)
+        if level is None:
+            level = 'INFO'
+        
+        print(f"     filename: {filename}")
+        print(f"     level: {level}")
+        
+        print("   设置默认值")
+        LoggerManager._default_filename = filename
+        LoggerManager._default_level = LoggerManager._to_level(level)
+        LoggerManager._default_file_level = LoggerManager._to_level(level)
+        LoggerManager._default_console_level = LoggerManager._default_level
+        
+        print("   清空缓存")
+        LoggerManager.logger_cache.clear()
+        
+        print("   设置已配置状态")
+        LoggerManager._configured = True
+        _log_state['current_step'] = 'full_config'
+        
+        print("   配置完成")
+
+print("4. 测试创建logger")
+from crawlo.utils.log import get_logger
+logger = get_logger('test')
+print(f"   Logger: {logger}")
+print(f"   Handlers: {len(logger.handlers)}")
+
+print("=== 调试完成 ===")

tests/debug_framework_logger.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+调试框架logger配置
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+from crawlo.core.framework_initializer import initialize_framework, get_framework_initializer
+from crawlo.utils.log import get_logger, LoggerManager
+import logging
+
+def debug_framework_logger():
+    print("=== 调试框架logger配置 ===")
+    
+    # 1. 初始化框架，模拟ofweek_standalone的配置
+    print("1. 初始化框架...")
+    custom_settings = {
+        'LOG_LEVEL': 'INFO',
+        'LOG_FILE': 'logs/debug_framework.log',
+        'PROJECT_NAME': 'debug_test',
+        'RUN_MODE': 'standalone'
+    }
+    
+    # 确保日志目录存在
+    os.makedirs('../logs', exist_ok=True)
+    
+    settings = initialize_framework(custom_settings)
+    print(f"   LOG_LEVEL: {settings.get('LOG_LEVEL')}")
+    print(f"   LOG_FILE: {settings.get('LOG_FILE')}")
+    
+    # 2. 获取框架初始化管理器
+    init_manager = get_framework_initializer()
+    print(f"   框架是否就绪: {init_manager.is_ready}")
+    print(f"   初始化阶段: {init_manager.phase}")
+    
+    # 3. 测试框架logger
+    framework_logger = init_manager.logger
+    print(f"   框架logger: {framework_logger}")
+    if framework_logger:
+        print(f"     名称: {framework_logger.name}")
+        print(f"     级别: {framework_logger.level} ({logging.getLevelName(framework_logger.level)})")
+        print(f"     处理器数量: {len(framework_logger.handlers)}")
+        
+        for i, handler in enumerate(framework_logger.handlers):
+            handler_type = type(handler).__name__
+            handler_level = handler.level
+            print(f"       处理器{i}: {handler_type}, 级别: {handler_level} ({logging.getLevelName(handler_level)})")
+            if hasattr(handler, 'baseFilename'):
+                print(f"         文件: {handler.baseFilename}")
+    
+    # 4. 手动创建一个crawlo.framework logger对比
+    manual_logger = get_logger('crawlo.framework')
+    print(f"   手动创建的logger: {manual_logger}")
+    if manual_logger:
+        print(f"     名称: {manual_logger.name}")
+        print(f"     级别: {manual_logger.level} ({logging.getLevelName(manual_logger.level)})")
+        print(f"     处理器数量: {len(manual_logger.handlers)}")
+        
+        for i, handler in enumerate(manual_logger.handlers):
+            handler_type = type(handler).__name__
+            handler_level = handler.level
+            print(f"       处理器{i}: {handler_type}, 级别: {handler_level} ({logging.getLevelName(handler_level)})")
+            if hasattr(handler, 'baseFilename'):
+                print(f"         文件: {handler.baseFilename}")
+    
+    # 5. 测试日志输出
+    print("2. 测试日志输出...")
+    
+    if framework_logger:
+        framework_logger.info("这是框架logger测试消息 - INFO级别")
+        framework_logger.debug("这是框架logger测试消息 - DEBUG级别")
+    
+    if manual_logger:
+        manual_logger.info("这是手动logger测试消息 - INFO级别")
+        manual_logger.debug("这是手动logger测试消息 - DEBUG级别")
+    
+    # 6. 检查是否同一个实例
+    print(f"3. 是否同一实例: {framework_logger is manual_logger}")
+    
+    print("=== 调试完成 ===")
+
+if __name__ == "__main__":
+    debug_framework_logger()

tests/debug_log_levels.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+调试日志级别配置脚本
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+from crawlo.core.framework_initializer import initialize_framework
+from crawlo.utils.log import LoggerManager, get_logger
+import logging
+
+def main():
+    print("=== 开始调试日志级别配置 ===")
+    
+    # 初始化框架
+    print("1. 初始化框架...")
+    settings = initialize_framework()
+    
+    # 打印配置信息
+    print(f"2. 配置信息:")
+    print(f"   LOG_LEVEL: {settings.get('LOG_LEVEL')}")
+    print(f"   LOG_FILE: {settings.get('LOG_FILE')}")
+    print(f"   LoggerManager._default_level: {LoggerManager._default_level}")
+    print(f"   LoggerManager._default_console_level: {LoggerManager._default_console_level}")
+    print(f"   LoggerManager._default_file_level: {LoggerManager._default_file_level}")
+    
+    # 测试不同组件的日志级别
+    components = [
+        'crawlo.framework',
+        'crawlo.crawler', 
+        'QueueManager',
+        'Scheduler',
+        'AioHttpDownloader',
+        'MiddlewareManager',
+        'PipelineManager',
+        'ExtensionManager',
+        'of_week_standalone'
+    ]
+    
+    print("3. 组件日志级别测试:")
+    for component_name in components:
+        logger = get_logger(component_name)
+        print(f"   {component_name}:")
+        print(f"     logger.level: {logger.level} ({logging.getLevelName(logger.level)})")
+        
+        for handler in logger.handlers:
+            handler_type = type(handler).__name__
+            print(f"     {handler_type}.level: {handler.level} ({logging.getLevelName(handler.level)})")
+    
+    # 实际测试日志输出
+    print("4. 测试日志输出:")
+    test_logger = get_logger('TestLogger')
+    
+    print("   控制台应该看到以下日志:")
+    test_logger.debug("这是DEBUG级别日志 - 控制台应该看不到")
+    test_logger.info("这是INFO级别日志 - 控制台应该能看到")
+    test_logger.warning("这是WARNING级别日志 - 控制台应该能看到")
+    
+    print("=== 调试完成 ===")
+
+if __name__ == '__main__':
+    main()