crawlo 1.3.3__py3-none-any.whl → 1.3.5__py3-none-any.whl

tests/verify_debug.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+验证运行模式日志级别修改的简单测试
+"""
+import os
+
+# 删除旧日志文件
+log_file = 'verify_debug.log'
+if os.path.exists(log_file):
+    os.remove(log_file)
+
+# 简单测试日志级别
+from crawlo.utils.log import LoggerManager
+
+# 配置日志系统
+LoggerManager.configure(
+    LOG_LEVEL='INFO',
+    LOG_FILE=log_file
+)
+
+from crawlo.utils.log import get_logger
+
+# 创建测试logger
+test_logger = get_logger('crawlo.framework')
+
+# 测试输出
+test_logger.info("这是INFO级别的测试信息")
+test_logger.debug("这是DEBUG级别的测试信息（不应该在INFO级别的日志中出现）")
+test_logger.debug("使用单机模式 - 简单快速，适合开发和中小规模爬取")
+
+print("测试完成")
+
+# 检查日志文件
+if os.path.exists(log_file):
+    with open(log_file, 'r', encoding='utf-8') as f:
+        content = f.read()
+        print(f"日志文件内容（{len(content)} 字符）:")
+        print(content)
+        
+        # 检查是否包含不应该出现的DEBUG信息
+        if "DEBUG" in content:
+            print("❌ 发现DEBUG级别信息（不应该出现）")
+        else:
+            print("✅ 没有发现DEBUG级别信息（正确）")
+            
+        if "使用单机模式" in content:
+            print("❌ 发现运行模式信息（不应该出现在INFO级别）")
+        else:
+            print("✅ 没有发现运行模式信息（正确）")
+else:
+    print("❌ 日志文件未创建")

tests/verify_distributed.py

@@ -1,117 +1,117 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-"""
-分布式采集功能验证脚本
-验证Crawlo框架的分布式采集功能是否正常工作
-"""
-
-import redis
-import json
-import os
-import sys
-
-# 添加项目根目录到 Python 路径
-project_root = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, project_root)
-
-
-def verify_distributed_functionality():
-    """验证分布式采集功能"""
-    print("=== Crawlo分布式采集功能验证 ===\n")
-
-    # 1. 连接Redis
-    try:
-        r = redis.Redis(host='localhost', port=6379, db=2, decode_responses=False)
-        r.ping()
-        print("✓ Redis连接成功")
-    except Exception as e:
-        print(f"✗ Redis连接失败: {e}")
-        return False
-
-    # 2. 检查项目配置
-    try:
-        with open('../examples/ofweek_distributed/crawlo.cfg', 'r') as f:
-            config_content = f.read()
-            if 'ofweek_distributed.settings' in config_content:
-                print("✓ 项目配置文件正确")
-            else:
-                print("✗ 项目配置文件不正确")
-                return False
-    except Exception as e:
-        print(f"✗ 无法读取配置文件: {e}")
-        return False
-
-    # 3. 检查设置文件
-    try:
-        with open('../examples/ofweek_distributed/ofweek_distributed/settings.py', 'r') as f:
-            settings_content = f.read()
-            checks = [
-                ('RUN_MODE = \'distributed\'', '运行模式设置为分布式'),
-                ('QUEUE_TYPE = \'redis\'', '队列类型设置为Redis'),
-                ('FILTER_CLASS = \'crawlo.filters.aioredis_filter.AioRedisFilter\'', '过滤器设置为Redis过滤器'),
-                ('REDIS_HOST = \'127.0.0.1\'', 'Redis主机配置正确'),
-            ]
-
-            all_passed = True
-            for check, description in checks:
-                if check in settings_content:
-                    print(f"✓ {description}")
-                else:
-                    print(f"✗ {description}")
-                    all_passed = False
-
-            if not all_passed:
-                return False
-    except Exception as e:
-        print(f"✗ 无法读取设置文件: {e}")
-        return False
-
-    # 4. 检查Redis中的数据
-    try:
-        # 检查请求去重指纹
-        request_fingerprints = r.scard("crawlo:ofweek_distributed:filter:fingerprint")
-        print(f"✓ 请求去重指纹数量: {request_fingerprints}")
-
-        # 检查数据项去重指纹
-        item_fingerprints = r.scard("crawlo:ofweek_distributed:item:fingerprint")
-        print(f"✓ 数据项去重指纹数量: {item_fingerprints}")
-
-        # 检查请求队列
-        queue_size = r.zcard("crawlo:ofweek_distributed:queue:requests")
-        print(f"✓ 请求队列大小: {queue_size}")
-
-        # 验证数据是否存在
-        if request_fingerprints > 0 and item_fingerprints > 0:
-            print("✓ Redis中存在分布式采集数据")
-        else:
-            print("⚠ Redis中分布式采集数据为空")
-
-    except Exception as e:
-        print(f"✗ Redis数据检查失败: {e}")
-        return False
-
-    # 5. 检查输出文件
-    try:
-        import glob
-        json_files = glob.glob("output/*.json")
-        if json_files:
-            latest_file = max(json_files, key=os.path.getctime)
-            file_size = os.path.getsize(latest_file)
-            print(f"✓ 输出文件存在: {latest_file} ({file_size} bytes)")
-        else:
-            print("⚠ 未找到输出文件")
-    except Exception as e:
-        print(f"✗ 输出文件检查失败: {e}")
-
-    print("\n=== 验证结果 ===")
-    print("✓ Crawlo分布式采集功能正常工作!")
-    print("  - Redis连接正常")
-    print("  - 分布式配置正确")
-    print("  - Redis数据存储正常")
-    print("  - 采集任务执行正常")
-
-    return True
-
-
-if __name__ == '__main__':
-    verify_distributed_functionality()
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+分布式采集功能验证脚本
+验证Crawlo框架的分布式采集功能是否正常工作
+"""
+
+import redis
+import json
+import os
+import sys
+
+# 添加项目根目录到 Python 路径
+project_root = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, project_root)
+
+
+def verify_distributed_functionality():
+    """验证分布式采集功能"""
+    print("=== Crawlo分布式采集功能验证 ===\n")
+
+    # 1. 连接Redis
+    try:
+        r = redis.Redis(host='localhost', port=6379, db=2, decode_responses=False)
+        r.ping()
+        print("✓ Redis连接成功")
+    except Exception as e:
+        print(f"✗ Redis连接失败: {e}")
+        return False
+
+    # 2. 检查项目配置
+    try:
+        with open('../examples/ofweek_distributed/crawlo.cfg', 'r') as f:
+            config_content = f.read()
+            if 'ofweek_distributed.settings' in config_content:
+                print("✓ 项目配置文件正确")
+            else:
+                print("✗ 项目配置文件不正确")
+                return False
+    except Exception as e:
+        print(f"✗ 无法读取配置文件: {e}")
+        return False
+
+    # 3. 检查设置文件
+    try:
+        with open('../examples/ofweek_distributed/ofweek_distributed/settings.py', 'r') as f:
+            settings_content = f.read()
+            checks = [
+                ('RUN_MODE = \'distributed\'', '运行模式设置为分布式'),
+                ('QUEUE_TYPE = \'redis\'', '队列类型设置为Redis'),
+                ('FILTER_CLASS = \'crawlo.filters.aioredis_filter.AioRedisFilter\'', '过滤器设置为Redis过滤器'),
+                ('REDIS_HOST = \'127.0.0.1\'', 'Redis主机配置正确'),
+            ]
+
+            all_passed = True
+            for check, description in checks:
+                if check in settings_content:
+                    print(f"✓ {description}")
+                else:
+                    print(f"✗ {description}")
+                    all_passed = False
+
+            if not all_passed:
+                return False
+    except Exception as e:
+        print(f"✗ 无法读取设置文件: {e}")
+        return False
+
+    # 4. 检查Redis中的数据
+    try:
+        # 检查请求去重指纹
+        request_fingerprints = r.scard("crawlo:ofweek_distributed:filter:fingerprint")
+        print(f"✓ 请求去重指纹数量: {request_fingerprints}")
+
+        # 检查数据项去重指纹
+        item_fingerprints = r.scard("crawlo:ofweek_distributed:item:fingerprint")
+        print(f"✓ 数据项去重指纹数量: {item_fingerprints}")
+
+        # 检查请求队列
+        queue_size = r.zcard("crawlo:ofweek_distributed:queue:requests")
+        print(f"✓ 请求队列大小: {queue_size}")
+
+        # 验证数据是否存在
+        if request_fingerprints > 0 and item_fingerprints > 0:
+            print("✓ Redis中存在分布式采集数据")
+        else:
+            print("⚠ Redis中分布式采集数据为空")
+
+    except Exception as e:
+        print(f"✗ Redis数据检查失败: {e}")
+        return False
+
+    # 5. 检查输出文件
+    try:
+        import glob
+        json_files = glob.glob("output/*.json")
+        if json_files:
+            latest_file = max(json_files, key=os.path.getctime)
+            file_size = os.path.getsize(latest_file)
+            print(f"✓ 输出文件存在: {latest_file} ({file_size} bytes)")
+        else:
+            print("⚠ 未找到输出文件")
+    except Exception as e:
+        print(f"✗ 输出文件检查失败: {e}")
+
+    print("\n=== 验证结果 ===")
+    print("✓ Crawlo分布式采集功能正常工作!")
+    print("  - Redis连接正常")
+    print("  - 分布式配置正确")
+    print("  - Redis数据存储正常")
+    print("  - 采集任务执行正常")
+
+    return True
+
+
+if __name__ == '__main__':
+    verify_distributed_functionality()

tests/verify_log_fix.py

@@ -0,0 +1,112 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+验证日志级别修复效果
+创建一个简化的测试来验证控制台和日志文件级别的一致性
+"""
+import sys
+import os
+import tempfile
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, '/')
+
+from crawlo.utils.log import LoggerManager, get_logger
+
+
+def main():
+    """验证日志级别修复效果"""
+    print("🔧 验证日志级别修复效果")
+    print("=" * 50)
+    
+    # 创建临时日志文件
+    temp_log = tempfile.NamedTemporaryFile(mode='w+', suffix='.log', delete=False)
+    temp_log_path = temp_log.name
+    temp_log.close()
+    
+    try:
+        # 重置LoggerManager状态
+        LoggerManager.reset()
+        
+        # 使用INFO级别配置
+        LoggerManager.configure(
+            LOG_LEVEL='INFO',
+            LOG_FILE=temp_log_path,
+            LOG_FORMAT='%(asctime)s - [%(name)s] - %(levelname)s: %(message)s'
+        )
+        
+        print(f"✅ 配置完成:")
+        print(f"   默认级别: {LoggerManager._default_level}")
+        print(f"   控制台级别: {LoggerManager._default_console_level}")
+        print(f"   文件级别: {LoggerManager._default_file_level}")
+        print(f"   日志文件: {temp_log_path}")
+        
+        # 创建测试logger
+        test_logger = get_logger('crawlo.test')
+        
+        # 检查handler配置
+        print(f"\n📋 Handler配置:")
+        for i, handler in enumerate(test_logger.handlers):
+            handler_type = type(handler).__name__
+            handler_level = handler.level
+            print(f"   Handler {i} ({handler_type}): 级别 {handler_level}")
+        
+        # 测试日志输出
+        print(f"\n📝 测试日志输出（控制台）:")
+        test_logger.debug("这是DEBUG级别日志 - 不应该显示")
+        test_logger.info("这是INFO级别日志 - 应该显示")
+        test_logger.warning("这是WARNING级别日志 - 应该显示")
+        test_logger.error("这是ERROR级别日志 - 应该显示")
+        
+        # 检查日志文件内容
+        print(f"\n📄 检查日志文件内容:")
+        with open(temp_log_path, 'r', encoding='utf-8') as f:
+            log_content = f.read()
+            if log_content:
+                print("日志文件内容:")
+                print(log_content)
+            else:
+                print("❌ 日志文件为空")
+        
+        # 分析结果
+        lines = log_content.strip().split('\n') if log_content.strip() else []
+        debug_lines = [line for line in lines if '- DEBUG:' in line]
+        info_lines = [line for line in lines if '- INFO:' in line]
+        warning_lines = [line for line in lines if '- WARNING:' in line]
+        error_lines = [line for line in lines if '- ERROR:' in line]
+        
+        print(f"\n📊 分析结果:")
+        print(f"   DEBUG级别日志: {len(debug_lines)}条 {'✅ 正确' if len(debug_lines) == 0 else '❌ 错误'}")
+        print(f"   INFO级别日志: {len(info_lines)}条 {'✅ 正确' if len(info_lines) >= 1 else '❌ 错误'}")
+        print(f"   WARNING级别日志: {len(warning_lines)}条 {'✅ 正确' if len(warning_lines) >= 1 else '❌ 错误'}")
+        print(f"   ERROR级别日志: {len(error_lines)}条 {'✅ 正确' if len(error_lines) >= 1 else '❌ 错误'}")
+        
+        # 判断修复是否成功
+        success = (len(debug_lines) == 0 and len(info_lines) >= 1 and 
+                  len(warning_lines) >= 1 and len(error_lines) >= 1)
+        
+        print(f"\n🎯 修复结果: {'✅ 成功' if success else '❌ 失败'}")
+        
+        if success:
+            print("📋 控制台和日志文件现在使用相同的INFO级别")
+            print("🎉 日志级别一致性问题已解决")
+        else:
+            print("❌ 仍存在日志级别不一致问题，需要进一步调试")
+            
+    except Exception as e:
+        print(f"❌ 验证过程中发生错误: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    finally:
+        # 清理临时文件
+        try:
+            os.unlink(temp_log_path)
+        except:
+            pass
+            
+    return 0 if success else 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

crawlo-1.3.3.dist-info/RECORD

@@ -1,219 +0,0 @@
-crawlo/__init__.py,sha256=2H7llH-yRV3N5_DomJ02JYsd5wNJdUNZI3VowiTQvOc,1444
-crawlo/__version__.py,sha256=9ap_Mho2n-5Wj2kAxLL8yqq57pG-v6Z_7an7VCKga44,23
-crawlo/cli.py,sha256=AQnAB5NMI-Ic1VPw_Jjng8L4AI4-wMozOwzE6CfXkZU,2402
-crawlo/config.py,sha256=EQIT7WpkXAlr2ocd5SYJYOKTSWUlQx2AkTHX7ErEWxw,9798
-crawlo/config_validator.py,sha256=oY4-2bwXUlwHAnGgkI-EznviDfML_dcxbWSGXNSxC2k,11516
-crawlo/crawler.py,sha256=rixy3qIy7e0vg1Ns4u1NC3S3Lbi-Mbqe_edPkXgV0yc,43600
-crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
-crawlo/exceptions.py,sha256=YVIDnC1bKSMv3fXH_6tinWMuD9HmKHIaUfO4_fkX5sY,1247
-crawlo/mode_manager.py,sha256=h6ZWOK9U9WZXCLk1MXwBkpzMOw6l5royxfrst4vCtJg,6573
-crawlo/project.py,sha256=Qw_Z8-ppYdv-aynBlzxH9CqABbrMIyGjtfLsM0werqc,12550
-crawlo/stats_collector.py,sha256=v4jC9BAe-23w93hWzbeMCCgQ9VuFPyxw5JV9ItbGH8w,1636
-crawlo/subscriber.py,sha256=h8fx69NJZeWem0ZkCmfHAi2kgfDGFObHpwN0aGNUM6Y,5115
-crawlo/task_manager.py,sha256=PScfEB03306Txa0l38AeQ_0WVhKzeWOFyT3bnrkbHW0,849
-crawlo/commands/__init__.py,sha256=orvY6wLOBwGUEJKeF3h_T1fxj8AaQLjngBDd-3xKOE4,392
-crawlo/commands/check.py,sha256=TKDhI_sj7kErgiJpt2vCZ9QL-g6yWjrrPWKbgh8pgEU,23199
-crawlo/commands/genspider.py,sha256=7YGZdv12G341SWmkGbyDeMde2RgqGYxYXRExFy7KKNc,5088
-crawlo/commands/help.py,sha256=8xPC0iNCg1rRBoK2bb6noAEANc1JwrdM35eF-j6yeZM,5111
-crawlo/commands/list.py,sha256=trzcd3kG6DhkOqYZADcl3yR7M8iJBgRw5fE-g9e0gVM,5877
-crawlo/commands/run.py,sha256=ybATvNXjXdr6GczW6gujkaTK05dhfk1tUTA0qXCO_rg,12360
-crawlo/commands/startproject.py,sha256=-Bo8vvDfIhqzGmWyhxMatBlPLhYpRwJC7l4fpbN8vVk,16506
-crawlo/commands/stats.py,sha256=vlGJLyiXZtY0ASdzCK59JNereSsAel4W9JCGaOzCr-8,6201
-crawlo/commands/utils.py,sha256=YVNEEzlm_qNY3SVvU8h6o2lQMkVgypvoB4ZFrP4gln0,5578
-crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
-crawlo/core/engine.py,sha256=d6L4Xwwjc1UQJY9QutqC_Uk88ZzBCyN9T_7z3lMSuIQ,14861
-crawlo/core/processor.py,sha256=oHLs-cno0bJGTNc9NGD2S7_2-grI3ruvggO0SY2mf3Q,1180
-crawlo/core/scheduler.py,sha256=AaZID01ovAbFzf1Urp55SPyUd7AOtHetX6R_GEYMTfA,12849
-crawlo/data/__init__.py,sha256=UPqgioMdu3imSUmpLWzVlpvoBnEfaPSAT-crCcWd7iw,121
-crawlo/data/user_agents.py,sha256=zjjFkldQkqtrn45j0WZplaZLannPxZDeAU0JofxQcBc,9891
-crawlo/downloader/__init__.py,sha256=VZG5HiSHOmimiH9okQN3MBwgXsCzxr2awflVz5UiboY,8897
-crawlo/downloader/aiohttp_downloader.py,sha256=GaUgR5WwG7VvMDQnL9tdwnLgu2bt8btdMuecWKyP2Uk,9195
-crawlo/downloader/cffi_downloader.py,sha256=QxoeocCE2DsQCnhZla6-BjhplaTZDWMbEJmNrghWSDA,10488
-crawlo/downloader/httpx_downloader.py,sha256=MpgDeIdGqNsiSKLOEDBnr5Z0eUbhHnqVEmAuoIfJmFU,12296
-crawlo/downloader/hybrid_downloader.py,sha256=dNnFeegRnyLaOxTWI6XrWKqqVPx80AZBZNgmrcKRVBM,8240
-crawlo/downloader/playwright_downloader.py,sha256=L-TVzG7cYfuBlqW0XSZuz5C_r9fpJrmYNcoQ-cDEna4,16663
-crawlo/downloader/selenium_downloader.py,sha256=P8GuhEw6OYVeN3oeksuBLpUJCELXiu0mAR23X6IIOAA,21508
-crawlo/extension/__init__.py,sha256=jOdyLjtf-JqEKN67x2haIhtMhy_5bGSMbdFIdsERU7o,1633
-crawlo/extension/health_check.py,sha256=stDpyP4gOzAdbBlPbSf0rge0QounAhF8CtrGq5fa_7c,5657
-crawlo/extension/log_interval.py,sha256=2R3XVdM1grDN8wh9TTHRB_WmQypCr5YSGvESNDnS16s,2474
-crawlo/extension/log_stats.py,sha256=6Hoq0ASU8evjT5AsUuc0b018-vkzeeO6CyJrU9ZabWk,2989
-crawlo/extension/logging_extension.py,sha256=hAi3hUbrVMRcE7b0tqybSRgnDYrgZYIDTsF-wxmezI0,1940
-crawlo/extension/memory_monitor.py,sha256=fClPchpCkVjcIiU0AJHCKDd7HEiz5B4KqNqKTRZ2hcU,4394
-crawlo/extension/performance_profiler.py,sha256=BjWD3LOb4VwjQJQvQtWNg7GluEwFquI1CztNfgMzy3c,5032
-crawlo/extension/request_recorder.py,sha256=KA_RmcfscDxP5wPdolO76yKfRj-1jmHhG3jkVGO1pbc,4181
-crawlo/filters/__init__.py,sha256=lX-QOCDTiTRFoiK1qrZ5HABo7LgZfcxScx_lELYEvJk,4395
-crawlo/filters/aioredis_filter.py,sha256=aB1GPCALikvPUWdoACaGsvmnkzseKXxpR7l3gh1glsY,8479
-crawlo/filters/memory_filter.py,sha256=ZojFhZ6gE76aQBC-rfImxSkSMwQtiotenx0pIcQOaFg,9561
-crawlo/items/__init__.py,sha256=rFpx1qFBo0Ik7bSdnXC8EVTJUOQdoJYGVdhYjaH00nk,409
-crawlo/items/base.py,sha256=hwGJEdFWOdaZfalFX8umRkh_HUWLEbCjvq4j70fplMQ,598
-crawlo/items/fields.py,sha256=l-DIwK6CCpdzNvf6ELz7Ckc7YCghZD9UCXA8vhNn2UE,1852
-crawlo/items/items.py,sha256=OmVEvMmgofMU95GkaiWkfNQ2fjsH2fY9sw3SKcmUhLs,3478
-crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
-crawlo/middleware/default_header.py,sha256=Pw-ev8ffi16GeCh84R5L3hAZgp3G1QXS-H5kV3JEp4Q,5164
-crawlo/middleware/download_delay.py,sha256=2iWnJFtWDlqDy5MsAob8TPiJQoiz9v21yatkBI0eptg,3542
-crawlo/middleware/middleware_manager.py,sha256=69l0QS6HJA2TmhdEHgyXMMhJ1nZlVUjODUFo3xhSth4,6413
-crawlo/middleware/offsite.py,sha256=4tUkPqXMMXsi1WwYnJ_e7wMd6sRgK19QHRCYq8-w8jk,4682
-crawlo/middleware/proxy.py,sha256=uKk5OSLIs7jv9bBgkZwsi1rIpthooxhMrGBC2BPRDCc,16022
-crawlo/middleware/request_ignore.py,sha256=7qdX4zAimjSGwdod_aWUbOTfzLBWZ5KzLVFchGMCxCI,2663
-crawlo/middleware/response_code.py,sha256=0_NbiCzLgJmSuSSF2_jqpypWYy0ES4GV-0iWQPLfYLc,5097
-crawlo/middleware/response_filter.py,sha256=tVGr06bfJBR3xAHI2G5c3WimFsGHu8qoJtDcsVuCATU,4384
-crawlo/middleware/retry.py,sha256=Acfo95B9wF8fQTCQIqluZOS2hHdnknQu_FOHvpGKJp0,4248
-crawlo/middleware/simple_proxy.py,sha256=rQ4RkqewGvDRCw021nGrg8ngkBzg3wqrEVqvSmBgQ6M,2256
-crawlo/network/__init__.py,sha256=bvEnpEUBZJ79URfNZbsHhsBKna54hM2-x_BV8eotTA4,418
-crawlo/network/request.py,sha256=e6-YLgK7SU8D19n21mQwqt_b_aeRVJFOgWPIBPal2ys,14178
-crawlo/network/response.py,sha256=QwJhL3xJfPVy_gwtGrg61oAgaqCoCmjyj1Ug7Zju7Pg,13060
-crawlo/pipelines/__init__.py,sha256=FDe2Pr5tiHtV8hFlheElRO_O1aVKvSWlkTcAl9BXAKA,637
-crawlo/pipelines/bloom_dedup_pipeline.py,sha256=NoqU0pCS8clRvdmR-7EsJEzBGn_RJvI5-Fz-iVpO5mc,5817
-crawlo/pipelines/console_pipeline.py,sha256=bwe5hZgaVSWmh3R8XpOaaeAjJme-Ttrpo6G6f1cnLIg,1287
-crawlo/pipelines/csv_pipeline.py,sha256=qbXZoqq4FIR9QkUGpC0ryWzmqGJSrM2bxmWLM4I1nXM,12490
-crawlo/pipelines/database_dedup_pipeline.py,sha256=L9lc6k62kUzwcDPgUJ0wT3KHhnC_lls_L5XMb08i_H8,8200
-crawlo/pipelines/json_pipeline.py,sha256=wrCsh8YInmcPLAkhPrHObMx89VZfhf-c7qRrYsTixPE,8585
-crawlo/pipelines/memory_dedup_pipeline.py,sha256=Wf_M7-FFmqXvcr3_Rpz97q0KcKebx8Ii2iRHv2A3orc,3952
-crawlo/pipelines/mongo_pipeline.py,sha256=PohTKTGw3QRvuP-T6SrquwW3FAHSno8jQ2D2cH_d75U,5837
-crawlo/pipelines/mysql_pipeline.py,sha256=fESKJ6qBcW3NZ9Gz1ACASL-PILvYAW3YagIZMg7H1h0,13818
-crawlo/pipelines/pipeline_manager.py,sha256=wr79XeVDH-v7meSwB10W1qU3AZDh9IopxGWg5rWFerw,3154
-crawlo/pipelines/redis_dedup_pipeline.py,sha256=kexmobW_JNSkaVRTQ4uhsKW5hGTXeqjGjOFmOP_wflw,6508
-crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/queue/pqueue.py,sha256=qTFOuvEXsYEZbm0ULjsOeZo0XtSsZ-SHpx7nFEtmluE,1095
-crawlo/queue/queue_manager.py,sha256=xjodzF8Yjb1wJ3ut_Mu3eRFrqeCMo5O0RXW5tdw9o1M,15320
-crawlo/queue/redis_priority_queue.py,sha256=J30HcC16E3FjxfUCbL_9HbFoaszBy7prFvG8xRvWd3M,13432
-crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
-crawlo/settings/default_settings.py,sha256=Wu1iUdhdgsFihCSuJJiXYE8DAHeV0HNuR663Hqsmg0U,9436
-crawlo/settings/setting_manager.py,sha256=LTs4NQ_CXvfhnDmmdKGlvosIjHtZk_48v7BEa_O0ghQ,7710
-crawlo/spider/__init__.py,sha256=I2_eb6NtgTQ-dckhQXZyDFQORUTx1OHqcn-9yleumkg,21074
-crawlo/templates/crawlo.cfg.tmpl,sha256=lwiUVe5sFixJgHFEjn1OtbAeyWsECOrz37uheuVtulk,240
-crawlo/templates/run.py.tmpl,sha256=vYCRPWpG2LxK3UvTxIyCDJh7qy43eoaU1CrJgBF-I6Y,1071
-crawlo/templates/project/__init__.py.tmpl,sha256=aQnHaOjMSkTviOC8COUX0fKymuyf8lx2tGduxkMkXEE,61
-crawlo/templates/project/items.py.tmpl,sha256=8_3DBA8HrS2XbfHzsMZNJiZbFY6fDJUUMFoFti_obJk,314
-crawlo/templates/project/middlewares.py.tmpl,sha256=fxHqi-Sjec5GiHJciprOU-6SAUTzM728NlZckIqf9hM,4278
-crawlo/templates/project/pipelines.py.tmpl,sha256=j9oqEhCezmmHlBhMWgYtlgup4jhWnMlv6AEiAOHODkg,2704
-crawlo/templates/project/settings.py.tmpl,sha256=5J__P6nTNr-3-ziSPdbBe-aklP4G6OFglI7UgEEF1zE,10562
-crawlo/templates/project/settings_distributed.py.tmpl,sha256=ci_vMOTwVwUeoqKGTa5tA6ygeruuKGt677liMYcp4Tw,7181
-crawlo/templates/project/settings_gentle.py.tmpl,sha256=CJMn4gvTg8xNUJLgObh8OmBCyGGyeCcLrfEdBAw8I4I,2770
-crawlo/templates/project/settings_high_performance.py.tmpl,sha256=rqPqIeChbfjEvQmMjAbuRml7pMhxTv2WBIkn21CZ6ew,5134
-crawlo/templates/project/settings_minimal.py.tmpl,sha256=TIKOnEbE2A6P52oe_aVtVYpR6zN-4-ECwl-zjNSFGV4,1241
-crawlo/templates/project/settings_simple.py.tmpl,sha256=8a0c1KTtnA4JzszhifIzG8wf9P7yEMjMhCMjoO0Qz2s,4500
-crawlo/templates/project/spiders/__init__.py.tmpl,sha256=zMbePipgLsctQUEnda4WkHz8rDLUX--rc8ruI6zkpWc,111
-crawlo/templates/spider/spider.py.tmpl,sha256=KvU-9YpN6MifDE7XzejjyyQS7RUjLDLZ8zqJcLwSsu0,5198
-crawlo/tools/__init__.py,sha256=tOYfYPvZlrO8cmvnMWBjTma6UTLTFZN3qdC8pJwHrzI,4142
-crawlo/tools/anti_crawler.py,sha256=LwLC6BkxDSkxc5H1hQ6kY9j7O0PZGAMPZECr7gbqw2M,9431
-crawlo/tools/authenticated_proxy.py,sha256=ULCK0Cc9F2rGhRqu6kzKBdxzK9v2n1CsatSQ_PMxpAg,7272
-crawlo/tools/data_formatter.py,sha256=iBDHpZBZvn9O7pLkTQilE1TzYJQEc3z3f6HXoVus0f0,7808
-crawlo/tools/data_validator.py,sha256=bLWnkpFdclJuqjtSAgMI5nznN4vAuPwE1YaiFWKWenM,5490
-crawlo/tools/date_tools.py,sha256=QOT3W5MqcEQhVM3cTZYxu1MRfgX-TI4aF1RI9s0QbdE,9195
-crawlo/tools/distributed_coordinator.py,sha256=0Ej8hv5GA0UmUI7EXNpCNdgh-D-DC7Eapm_3O2POV0U,12711
-crawlo/tools/encoding_converter.py,sha256=CqHAsR2rwxuzsyR-TeQNb79HX5mH4KAUixEY-sX7204,4170
-crawlo/tools/request_tools.py,sha256=oXrk4yWMACVa65fDQCQgzsg6a94FH4_lS7qNR53FHYU,2420
-crawlo/tools/retry_mechanism.py,sha256=4AQ_HLuYt4hYMI9XHoKFk2GQKEiDJB5pAnsMCfjc6Bk,7777
-crawlo/tools/scenario_adapter.py,sha256=pzysL1B2uQ1ZSEncVHd9Hv2viHNgaxP44YAUcDcppfw,9660
-crawlo/tools/text_cleaner.py,sha256=UrMGcgRnJaufjmDKIDsRYKMA8znCAArHDgouttWPygk,6690
-crawlo/utils/__init__.py,sha256=8kMbOZf9bzOUjtvh2QvqXZmiZh3pYzxXH9YQhYcwcoY,597
-crawlo/utils/batch_processor.py,sha256=8LNy-K2SrQVUxmGEWxQyYw_j9M-erN4Ie7O4d3zpBvM,9142
-crawlo/utils/controlled_spider_mixin.py,sha256=8CuM3Cr2wQLHbaO_ohbCsPcImJnyfZHpERbSeMgQ-AQ,16936
-crawlo/utils/db_helper.py,sha256=ZqOt1d3mErVv4TOvoWlov0niUxORB9aHByTmMoNFIDw,10902
-crawlo/utils/enhanced_error_handler.py,sha256=fJC__rnYNKTNUHNbgjZtT846HoE31qyGbPft9bwyYLU,14214
-crawlo/utils/env_config.py,sha256=W-VD_WF63DHxsyJysvp1eJwRh3L_pBRl_PitQAY3nQY,4079
-crawlo/utils/error_handler.py,sha256=N6suB8Utcn7tp6WRJ8gKECr0RIAG86dcOXdwOr998OE,4367
-crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
-crawlo/utils/large_scale_config.py,sha256=lsraHTAQx3sMPjTnCBY_SVIpkuIBUb3zD9eFvmccOOM,8440
-crawlo/utils/large_scale_helper.py,sha256=4ORkZcIrwJ0SlKOUh7l7WIuERORuRhNBgHCM71Rz0n0,12452
-crawlo/utils/log.py,sha256=05-OrQW-qNsAJHoVytICss6oHVSIHVCMnfivg2qI6tQ,7829
-crawlo/utils/performance_monitor.py,sha256=32KspSo7RWvCX_fl0ZFn4ScWWOqbVVwEhPRd921Ez6I,9832
-crawlo/utils/queue_helper.py,sha256=gFmkh1jKlIcN1rmo2Jl6vYcLP5ByUWlfHO9eNlZPBLs,4918
-crawlo/utils/redis_connection_pool.py,sha256=Czm0RoYmgJ5E5xIuVbBnm0IgSWH2AfeaFTGnYVWneYk,11401
-crawlo/utils/redis_key_validator.py,sha256=-UTTx0Ul184pzwSply8hVdH0lp-gkXXOc_gEHR_7VlU,5809
-crawlo/utils/request.py,sha256=ejdKpTwc-HE04HQybafhOVywzz57IV3pY0YMkSLyGUo,9065
-crawlo/utils/request_serializer.py,sha256=KIQBbQWCb5Ne3jFPMtqD96TNs5dTD85Ex3xr16vBrUM,8739
-crawlo/utils/spider_loader.py,sha256=xNzQb7qhQ7TqZsfFtCLpuVcsGi-USriZosU0YSBr9II,2233
-crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
-crawlo/utils/tools.py,sha256=uy7qw5Z1BIhyEgiHENvtM7WoGCJxlS8EX3PmOA7ouCo,275
-crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
-examples/__init__.py,sha256=NkRbV8_S1tb8S2AW6BE2U6P2-eGOPwMR1k0YQAwQpSE,130
-tests/DOUBLE_CRAWLO_PREFIX_FIX_REPORT.md,sha256=4W6HlT9Uc3cyu77T9pfbkrMxpAZ-xq_L9MU-GbukLV0,3427
-tests/__init__.py,sha256=409aRX8hsPffiZCVjOogtxwhACzBp8G2UTJyUQSxhK0,136
-tests/advanced_tools_example.py,sha256=1_iitECKCuWUYMNNGo61l3lmwMRrWdA8F_Xw56UaGZY,9340
-tests/authenticated_proxy_example.py,sha256=GhmbISta3lDnNRl6Cr1P2ZEy2jXasz942cOeXiSbVXk,2971
-tests/cleaners_example.py,sha256=blVqSJ7SeWUNd17JjHZJgVTzWH65XKevLyaMB_Wg8qA,5324
-tests/config_validation_demo.py,sha256=jbZ7h-HGsJmuqBb1euB_AhmKjllkvPmItRF1K0MQrVM,4171
-tests/controlled_spider_example.py,sha256=2SAQKoREGHe-OzVaSkGpopCcrou6QXmeW7rLdmsyopw,7981
-tests/date_tools_example.py,sha256=XI3iFEzeo7Nb5YepK8WHytIaBegtxWVSISpqQgpV6M8,5042
-tests/debug_pipelines.py,sha256=FMb36bH9lQxBLb-nM579hBRK1S16Vxu1t_BC3Dj8O2w,2164
-tests/dynamic_loading_example.py,sha256=7LdeQZFevrb-U1_dgr4oX3aYo2Da4HvE_0KIf1fw4Ew,18786
-tests/dynamic_loading_test.py,sha256=dzDW7b66HeDsIYsYgvNRihE3V6b6gEbUGQpp-eJbcIM,3413
-tests/env_config_example.py,sha256=_ZRDh_LR23ZKpy9E--y_KM0QIOiZF5vRT98QTn52TY8,4951
-tests/error_handling_example.py,sha256=grTeo1X17rFz4lhgASb0g5yu4NWbmNz5neyuonnNR40,5294
-tests/redis_key_validation_demo.py,sha256=WD2jvuBwHhLYIb3lVFtvYSSnmXWn1EW4EPCEwFhfi6M,4467
-tests/request_params_example.py,sha256=J50NdsnK1sDrqG-5m3oA-mu1_wHwVwHIfsWxGeQpz7o,4250
-tests/response_improvements_example.py,sha256=t1cbG3nesp82bqog4_ku1GvQzNbhRyWa5EaKTmOPrSk,5402
-tests/test_advanced_tools.py,sha256=HT_TcwfFzli-CavIJSqQqnCxnBn5FDMX09zL7AJ5tNY,5398
-tests/test_all_redis_key_configs.py,sha256=dWc4Dsr07_vuSpb4hwkMpyy6XO8SI7vglVjGuGvXoa4,5710
-tests/test_authenticated_proxy.py,sha256=lnvmQwuf0zaZP_E05EzcNFR2VJbwTkLjOmZGNoJKaC4,4339
-tests/test_cleaners.py,sha256=HDK8_YU7GUj_3hGU415cxEeUR74mnDSk0yroLlgDI0I,1816
-tests/test_comprehensive.py,sha256=dvRJeeVYc1cgXK9Y171hH9Y847zZpWSAFFH-EI3UepQ,5182
-tests/test_config_consistency.py,sha256=RgSxyaypMpysltsGSh1vFMeOShiZZG0rmUKzEhNLpYw,2001
-tests/test_config_merge.py,sha256=ts1j-TIKkFS0EO5q1I4O7f4YUKR5MLTmRSqOpOlv094,5606
-tests/test_config_validator.py,sha256=Z4gBHkI0_fEx-xgiiG4T33F4BAuePuF81obpNTXfseY,6202
-tests/test_crawlo_proxy_integration.py,sha256=miag_gufreZodFBwYGtyZNN-wfGyxUztg4w5-HEiBCY,2728
-tests/test_date_tools.py,sha256=pcLDyhLrZ_jh-PhPm4CvLZEgNeH9kLMPKN5zacHwuWM,4053
-tests/test_default_header_middleware.py,sha256=v-ei_1EY7cvFSsySrQPXF5-DmyGsq2yzjYhhrwFMOXs,6003
-tests/test_distributed.py,sha256=78Pn4HPLIaO8t1IiaSkckBmuEVTcnC8IDw7znf9_Zcw,1790
-tests/test_double_crawlo_fix.py,sha256=uT-PJLxGS4psOvVkJhurffV19hxhhlX5zHMPEyi59og,7977
-tests/test_double_crawlo_fix_simple.py,sha256=NDmCEeyvpf_D1tGQMA66iLPPKlAnSZcEg71e7GHYcjg,4768
-tests/test_download_delay_middleware.py,sha256=Idc6KzhL3hY3aDKgn1j_v5-mLIHz7dTnV5c4tJVZh5Q,9107
-tests/test_downloader_proxy_compatibility.py,sha256=0hgIzWXIqd92YXEB5sNneyp4Sk7PaG76up2cd6N9QQY,8903
-tests/test_dynamic_downloaders_proxy.py,sha256=t_aWpxOHi4h3_fg2ImtIq7IIJ0r3PTHtnXiopPe2ZlM,4450
-tests/test_dynamic_proxy.py,sha256=zi7Ocbhc9GL1zCs0XhmG2NvBBeIZ2d2hPJVh18lH4Y0,3172
-tests/test_dynamic_proxy_config.py,sha256=C_9CEjCJtrr0SxIXCyLDhSIi88ujF7UAT1F-FAphd0w,5853
-tests/test_dynamic_proxy_real.py,sha256=krWnbFIH26mWNPhOfPMmx3ZxJfOreZxMZFGwVb_8-K8,3511
-tests/test_edge_cases.py,sha256=1RnFaCebYTDNNz_LK8M0MepiSwPvJUk_FBK4nQTCUbg,10729
-tests/test_enhanced_error_handler.py,sha256=Ku_86jv7iDe25v8ZxalcXxJJjIiIvQXWH8ZldbwdVm8,8581
-tests/test_env_config.py,sha256=Qu1sDeADs69dSr1x0QmEe8nJrMHneE_4JClt-N901e8,4867
-tests/test_error_handler_compatibility.py,sha256=xJ43cmCwfBGh-qBwCGiMDPPlfNDLw4ZrmlrHN9IojkU,4241
-tests/test_final_validation.py,sha256=OuZI01O0E68Pao--bD-BFDTRZFPc_Mt4W-OXUzlt6ZA,4966
-tests/test_framework_env_usage.py,sha256=bFb_ptdLeX2obdJWEqEHPWweiWR-wR2BpvEaJMQK7h4,4201
-tests/test_integration.py,sha256=lVEzKNAjFzFZHRNZAyJmXxa_5Ogf_qqL4APqs620o58,4839
-tests/test_item_dedup_redis_key.py,sha256=dp_H59exJLaZHh5oMtmMEOWh-DNZwbnwIFYDjOpHgd0,3842
-tests/test_mode_consistency.py,sha256=hS9JwawnBvNwSu1l3DfArlGQGWPyVYXGuXCQtMuDHKs,1226
-tests/test_offsite_middleware.py,sha256=1DYktO_D-hiLEB6dBnc0iOvnWimqOdE6kimnS8aof_s,7764
-tests/test_parsel.py,sha256=wuZqRFIm9xx1tt6o3Xi_OjvwhT_MPmHiUEj2ax06zlo,701
-tests/test_performance.py,sha256=Lqs2iu3dmWipZkBPARcwIjDLXsqe42ntz1M4RzqqXKo,11457
-tests/test_proxy_api.py,sha256=XnmklS-xU4ke_560gV6AIlBsRmG8YLQTGFAZrTUZuhc,11013
-tests/test_proxy_health_check.py,sha256=_tDlxa_6TdL3M5RLkHF82roXJ8WIuG5hELBp2GADyKQ,1123
-tests/test_proxy_middleware.py,sha256=EdQAfwwAJIBxw9JmUFTDEu_pdxapaTlcJr7KcrY6-AY,4021
-tests/test_proxy_middleware_enhanced.py,sha256=QR-p26F63N7MxNjZ2QJUeerh_xdnCDejkrGPIh7Fh4U,7035
-tests/test_proxy_middleware_integration.py,sha256=mTPK_XvbmLCV_QoVZzA3ybWOOX61493Ew78WfTp-bYQ,4441
-tests/test_proxy_middleware_refactored.py,sha256=VbkTWkmmomcyswobA_gf3p_bERl_eexY2e6ohJQS_A8,6960
-tests/test_proxy_providers.py,sha256=u_R2fhab90vqvQEaOAztpAOe9tJXvUMIdoDxmStmXJ4,1749
-tests/test_proxy_stats.py,sha256=ES00CEoDITYPFBGPk8pecFzD3ItYIv6NSpcqNd8-kvo,526
-tests/test_proxy_strategies.py,sha256=9Z1pXmTNyw-eIhGXlf2abZbJx6igLohYq-_3hldQ5uE,1868
-tests/test_queue_manager_double_crawlo.py,sha256=MijZ3JuyHMuqGbRC-8kclFr-4O7m_T8CqezP4qiWk-E,6957
-tests/test_queue_manager_redis_key.py,sha256=B6JqScm_8FcriPb0UtBDLoEmLCGCI2z3NuqC0UMJsWo,6407
-tests/test_random_user_agent.py,sha256=6HjU4iUcMk-J6bR2N5FhIkWDfnaFKAPNVyRzxmQQ14k,2302
-tests/test_real_scenario_proxy.py,sha256=clmLvBfap5OpsaCE08MAWap-78jhVrxYfVfDNyoa4Hg,8454
-tests/test_redis_config.py,sha256=51_Fy1PqIhS0MMO2nR4q6oQjBFxfqcUPK_4NNf5s83g,903
-tests/test_redis_connection_pool.py,sha256=pKfXdE3Cm_L_fNqI9zqFmqiidCwR0t7hiM_Fu_V1cNI,9328
-tests/test_redis_key_naming.py,sha256=MTFk656JhiGVTsMctBDhBNOMFcBDZrsQA3UfPZ-Dgj4,6911
-tests/test_redis_key_validator.py,sha256=GszSzGADgk3uN6Bye1d8pS-AtMVgB8jwqW-22gPNM6M,4418
-tests/test_redis_queue.py,sha256=WQV3MtGg8rJzHgC2kRfXM6lSMXpwXJVQZfqn2dVrhg0,6758
-tests/test_request_ignore_middleware.py,sha256=QN81wgG_W_XfXCF9LvJNxCNwbOH6_tZnLIwLDTK2K5Q,6229
-tests/test_request_params.py,sha256=l2etiDebqylPBym1e9DSDn4wxwTHv8DQHKq9AzlzlG0,4287
-tests/test_request_serialization.py,sha256=Ikgec8tt_sPCK6jcZyK8vRw84zRNE6nxQy9rba1WKmE,2332
-tests/test_response_code_middleware.py,sha256=wSe525bm-bk_iWMjPDzUu1LfOQrwJY8_MLKAspq2dzk,12193
-tests/test_response_filter_middleware.py,sha256=YWrGzJ7wmftTjJXcNTtJl3b3EdJsO4oR22ZLWwgErhg,16327
-tests/test_response_improvements.py,sha256=vNqHKyoEoYeEGAUiRzdsff2V6yvJ9QnDwGg7gmN38Ow,6028
-tests/test_retry_middleware.py,sha256=RmSYSf0GagGPGAVi5TXJWc0bZlmAI_hwFr2FYhvuKrk,8097
-tests/test_scheduler.py,sha256=1fCu35QgK5gzgrhD0aUZj5lxL0QbokzPav-yEJxz9Ig,8182
-tests/test_scheduler_config_update.py,sha256=LuxjEbt20QrPyVkjSFxvTnFtUxwMaHB6TcqjFyo8bow,4261
-tests/test_simple_response.py,sha256=_ui2PuVZvJcAuLY7HZ8xcsy_tDBimgBqX0ukj3kE5J0,1549
-tests/test_telecom_spider_redis_key.py,sha256=c-gfixPul2VlYMQJGf0H5ZgYJ461fQgSKbCPrbAU45M,7625
-tests/test_template_content.py,sha256=2RgCdOA3pMUSOqC_JbTGeW7KonbTqJ0ySYJNWegU-v0,2903
-tests/test_template_redis_key.py,sha256=99-s0_-8MFJbIvGG_X__sH0qkXWTtJv8fdTdlftsq4I,4876
-tests/test_tools.py,sha256=z50Bvq_q8FwpyxNkmh00_A3sXkSv2l1Q_EbK02FDYgk,5504
-tests/test_user_agents.py,sha256=e4haX-o8Janl-PawGJ9MemZyMqTX33_tBF_WnYSVoUw,3327
-tests/tools_example.py,sha256=Rxu5vVKnj3CZ3mCx-EEotBWPtZs2S7ktyqq-SYeclxU,7999
-tests/verify_distributed.py,sha256=0IolM4ymuPOz_uTfHSWFO3Vxzp7Lo6i0zhSbzJhHFtI,4045
-crawlo-1.3.3.dist-info/METADATA,sha256=I5cs60iVGIq6oms6duJsGrlzs-WGdlVtUYbMkzYBXvE,27833
-crawlo-1.3.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.3.3.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.3.3.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.3.3.dist-info/RECORD,,