crawlo 1.3.1__py3-none-any.whl → 1.3.3__py3-none-any.whl

tests/verify_distributed.py

@@ -1,117 +1,117 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-"""
-分布式采集功能验证脚本
-验证Crawlo框架的分布式采集功能是否正常工作
-"""
-
-import redis
-import json
-import os
-import sys
-
-# 添加项目根目录到 Python 路径
-project_root = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, project_root)
-
-
-def verify_distributed_functionality():
-    """验证分布式采集功能"""
-    print("=== Crawlo分布式采集功能验证 ===\n")
-
-    # 1. 连接Redis
-    try:
-        r = redis.Redis(host='localhost', port=6379, db=2, decode_responses=False)
-        r.ping()
-        print("✓ Redis连接成功")
-    except Exception as e:
-        print(f"✗ Redis连接失败: {e}")
-        return False
-
-    # 2. 检查项目配置
-    try:
-        with open('../examples/ofweek_distributed/crawlo.cfg', 'r') as f:
-            config_content = f.read()
-            if 'ofweek_distributed.settings' in config_content:
-                print("✓ 项目配置文件正确")
-            else:
-                print("✗ 项目配置文件不正确")
-                return False
-    except Exception as e:
-        print(f"✗ 无法读取配置文件: {e}")
-        return False
-
-    # 3. 检查设置文件
-    try:
-        with open('../examples/ofweek_distributed/ofweek_distributed/settings.py', 'r') as f:
-            settings_content = f.read()
-            checks = [
-                ('RUN_MODE = \'distributed\'', '运行模式设置为分布式'),
-                ('QUEUE_TYPE = \'redis\'', '队列类型设置为Redis'),
-                ('FILTER_CLASS = \'crawlo.filters.aioredis_filter.AioRedisFilter\'', '过滤器设置为Redis过滤器'),
-                ('REDIS_HOST = \'127.0.0.1\'', 'Redis主机配置正确'),
-            ]
-
-            all_passed = True
-            for check, description in checks:
-                if check in settings_content:
-                    print(f"✓ {description}")
-                else:
-                    print(f"✗ {description}")
-                    all_passed = False
-
-            if not all_passed:
-                return False
-    except Exception as e:
-        print(f"✗ 无法读取设置文件: {e}")
-        return False
-
-    # 4. 检查Redis中的数据
-    try:
-        # 检查请求去重指纹
-        request_fingerprints = r.scard("crawlo:ofweek_distributed:filter:fingerprint")
-        print(f"✓ 请求去重指纹数量: {request_fingerprints}")
-
-        # 检查数据项去重指纹
-        item_fingerprints = r.scard("crawlo:ofweek_distributed:item:fingerprint")
-        print(f"✓ 数据项去重指纹数量: {item_fingerprints}")
-
-        # 检查请求队列
-        queue_size = r.zcard("crawlo:ofweek_distributed:queue:requests")
-        print(f"✓ 请求队列大小: {queue_size}")
-
-        # 验证数据是否存在
-        if request_fingerprints > 0 and item_fingerprints > 0:
-            print("✓ Redis中存在分布式采集数据")
-        else:
-            print("⚠ Redis中分布式采集数据为空")
-
-    except Exception as e:
-        print(f"✗ Redis数据检查失败: {e}")
-        return False
-
-    # 5. 检查输出文件
-    try:
-        import glob
-        json_files = glob.glob("output/*.json")
-        if json_files:
-            latest_file = max(json_files, key=os.path.getctime)
-            file_size = os.path.getsize(latest_file)
-            print(f"✓ 输出文件存在: {latest_file} ({file_size} bytes)")
-        else:
-            print("⚠ 未找到输出文件")
-    except Exception as e:
-        print(f"✗ 输出文件检查失败: {e}")
-
-    print("\n=== 验证结果 ===")
-    print("✓ Crawlo分布式采集功能正常工作!")
-    print("  - Redis连接正常")
-    print("  - 分布式配置正确")
-    print("  - Redis数据存储正常")
-    print("  - 采集任务执行正常")
-
-    return True
-
-
-if __name__ == '__main__':
-    verify_distributed_functionality()
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+分布式采集功能验证脚本
+验证Crawlo框架的分布式采集功能是否正常工作
+"""
+
+import redis
+import json
+import os
+import sys
+
+# 添加项目根目录到 Python 路径
+project_root = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, project_root)
+
+
+def verify_distributed_functionality():
+    """验证分布式采集功能"""
+    print("=== Crawlo分布式采集功能验证 ===\n")
+
+    # 1. 连接Redis
+    try:
+        r = redis.Redis(host='localhost', port=6379, db=2, decode_responses=False)
+        r.ping()
+        print("✓ Redis连接成功")
+    except Exception as e:
+        print(f"✗ Redis连接失败: {e}")
+        return False
+
+    # 2. 检查项目配置
+    try:
+        with open('../examples/ofweek_distributed/crawlo.cfg', 'r') as f:
+            config_content = f.read()
+            if 'ofweek_distributed.settings' in config_content:
+                print("✓ 项目配置文件正确")
+            else:
+                print("✗ 项目配置文件不正确")
+                return False
+    except Exception as e:
+        print(f"✗ 无法读取配置文件: {e}")
+        return False
+
+    # 3. 检查设置文件
+    try:
+        with open('../examples/ofweek_distributed/ofweek_distributed/settings.py', 'r') as f:
+            settings_content = f.read()
+            checks = [
+                ('RUN_MODE = \'distributed\'', '运行模式设置为分布式'),
+                ('QUEUE_TYPE = \'redis\'', '队列类型设置为Redis'),
+                ('FILTER_CLASS = \'crawlo.filters.aioredis_filter.AioRedisFilter\'', '过滤器设置为Redis过滤器'),
+                ('REDIS_HOST = \'127.0.0.1\'', 'Redis主机配置正确'),
+            ]
+
+            all_passed = True
+            for check, description in checks:
+                if check in settings_content:
+                    print(f"✓ {description}")
+                else:
+                    print(f"✗ {description}")
+                    all_passed = False
+
+            if not all_passed:
+                return False
+    except Exception as e:
+        print(f"✗ 无法读取设置文件: {e}")
+        return False
+
+    # 4. 检查Redis中的数据
+    try:
+        # 检查请求去重指纹
+        request_fingerprints = r.scard("crawlo:ofweek_distributed:filter:fingerprint")
+        print(f"✓ 请求去重指纹数量: {request_fingerprints}")
+
+        # 检查数据项去重指纹
+        item_fingerprints = r.scard("crawlo:ofweek_distributed:item:fingerprint")
+        print(f"✓ 数据项去重指纹数量: {item_fingerprints}")
+
+        # 检查请求队列
+        queue_size = r.zcard("crawlo:ofweek_distributed:queue:requests")
+        print(f"✓ 请求队列大小: {queue_size}")
+
+        # 验证数据是否存在
+        if request_fingerprints > 0 and item_fingerprints > 0:
+            print("✓ Redis中存在分布式采集数据")
+        else:
+            print("⚠ Redis中分布式采集数据为空")
+
+    except Exception as e:
+        print(f"✗ Redis数据检查失败: {e}")
+        return False
+
+    # 5. 检查输出文件
+    try:
+        import glob
+        json_files = glob.glob("output/*.json")
+        if json_files:
+            latest_file = max(json_files, key=os.path.getctime)
+            file_size = os.path.getsize(latest_file)
+            print(f"✓ 输出文件存在: {latest_file} ({file_size} bytes)")
+        else:
+            print("⚠ 未找到输出文件")
+    except Exception as e:
+        print(f"✗ 输出文件检查失败: {e}")
+
+    print("\n=== 验证结果 ===")
+    print("✓ Crawlo分布式采集功能正常工作!")
+    print("  - Redis连接正常")
+    print("  - 分布式配置正确")
+    print("  - Redis数据存储正常")
+    print("  - 采集任务执行正常")
+
+    return True
+
+
+if __name__ == '__main__':
+    verify_distributed_functionality()

crawlo-1.3.1.dist-info/RECORD

@@ -1,219 +0,0 @@
-crawlo/__init__.py,sha256=qZzTmb7hw5h_qcP2EYGUZcoSScxlKZFJ76CjSeS7UfA,1381
-crawlo/__version__.py,sha256=P2DFKJQEJRlJhF0IW0Lwt4G4uMYyJJ5ymhv-XrCcPGo,22
-crawlo/cli.py,sha256=OXprmcTUbFK02ptw_Gq8Gk4-ZCU-WEMJgzU1ztgP6Bk,2327
-crawlo/config.py,sha256=dNoNyTkXLe2msQ7bZx3YTQItk1m49nIg5-g89FQDNwE,9486
-crawlo/config_validator.py,sha256=gsiLqf5swWd9ISDvoLqCdG7iSXr-ZdBPD4iT6ug1ua4,11239
-crawlo/crawler.py,sha256=Fiu9O_eFHKCfzgzFe0O9gpzWGyneY-imI8-9O4hiWqU,42608
-crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
-crawlo/exceptions.py,sha256=sMay0wnWLfc_FXWslqxm60qz6b66LXs3EdN_w8ygE9k,1166
-crawlo/mode_manager.py,sha256=IN3CsWW1pzKnzHxvULxJTTx3Vw2kxJilfeLuDWfxm4Q,4890
-crawlo/project.py,sha256=swSTcan4Ky7sYfCatpNLKsVxztmPkIVwjdo3u6dgcpI,11128
-crawlo/stats_collector.py,sha256=NkO09CB-220qz5rxFcD_dedGfr2VPFrDo4hya0Zh8Qc,1577
-crawlo/subscriber.py,sha256=D3hzE7Pc_zJjc-zR7lct5pt32bz6LsDYeC8uHlS4Hso,4986
-crawlo/task_manager.py,sha256=19InAxS9oJ9EMj20Aw2urN-v6BeC22dkgPuW-B9-4UI,819
-crawlo/commands/__init__.py,sha256=QbhGAmItiwVrtlTr9UUbEJMegLJo-SdzaKX2PUhBgfI,378
-crawlo/commands/check.py,sha256=7pD43s97DD-fSLO9OEOuNcNr7o-2g94rJULL8fUzdaI,22605
-crawlo/commands/genspider.py,sha256=HhtvBLkIuhYtJUzom6PquItiC22vU9LNpOkjDUiqdM4,4937
-crawlo/commands/help.py,sha256=gwfHibRpdYDmZO6waUMOEn8SMJ_ubdjL-prD5fiuVY8,4973
-crawlo/commands/list.py,sha256=BqlPjBa5FLotjAlyZ3-nGmXg5cWcCNbHi8U5znb2_D8,5722
-crawlo/commands/run.py,sha256=KcJ4h4D7lavB6qQDpYMrbgJMgY5vCSLHaLckos5EUNY,11793
-crawlo/commands/startproject.py,sha256=aqKRJarKqTf5XjJnGXwjRpp0uYF16LreFbwwQLGpK-0,16070
-crawlo/commands/stats.py,sha256=8wTubR1RQ1JPTlpOKArcGcQ39bM-0cuH27lDpndnwPQ,6014
-crawlo/commands/utils.py,sha256=Psfu2tKrmDloMq0WnfXLaxx0lJFitMZ-FWS3HAIrziQ,5382
-crawlo/core/__init__.py,sha256=PnFyJdVNHBoPmV1sW0AHQXijeoSTQ8cMYrbNM1JK8kA,41
-crawlo/core/engine.py,sha256=Hy0K_g9My6aQ3CPkxAcCiPsumdwh4O8qRhmFlNoErd4,14496
-crawlo/core/processor.py,sha256=qmCqAeqhwYu-UE86evYesaGt9qpuSIfH-ZIZKcXFCZc,1140
-crawlo/core/scheduler.py,sha256=D-YzXVvnP6DEkovmz9hThhzIe2UgRrQLNt9pJCPEPwY,12593
-crawlo/data/__init__.py,sha256=8MgDxcMhx-emFARcLAw_ODOZNz0neYBcx7kEbzothd8,116
-crawlo/data/user_agents.py,sha256=6V34lYHREWV5ZR5wH-1pCnr1Y3ZYC7iMLfC6vZHyhZQ,9697
-crawlo/downloader/__init__.py,sha256=PB8oluLFMX2PBmeb3NBKkM6GaceX0ujFId8t2URy1ks,8624
-crawlo/downloader/aiohttp_downloader.py,sha256=KZY8xJ8jubrlfZNQugf8lpSeJ_Axk5-klpPSSfb4j1w,8969
-crawlo/downloader/cffi_downloader.py,sha256=BpA1q6Udz7sSXJ0gX94xGnzy8cdgK-vlr_Q6YA4QIxE,10243
-crawlo/downloader/httpx_downloader.py,sha256=7jfQfvAtfk8yD_mvwUbWLhYOxMM7r1nudiU7m_Jl9wc,12037
-crawlo/downloader/hybrid_downloader.py,sha256=4SzOPEwBlSZVzUAWR3DyxMx2Tsx15YrpBvQS4it4Vps,8028
-crawlo/downloader/playwright_downloader.py,sha256=Lnc7k5cXhVnURXSxgZFCYMJkBxLg5F_OE67rtf3G7Ig,16261
-crawlo/downloader/selenium_downloader.py,sha256=B_0muNi-GQ_hgoYHcf7wgu01V68q7xKnSh-0kzlUiio,21036
-crawlo/extension/__init__.py,sha256=FbOwJ4jh60xCbSh7P9CUGJsGAC-VH4MyOtCftRMlxbk,1594
-crawlo/extension/health_check.py,sha256=0GveZgUtFwjYEKlm3qbwIvCmb4FR0qrIKc8cEF1yQV8,5516
-crawlo/extension/log_interval.py,sha256=VCIeNqXcWDnxj4m6l77cjqgRzV8LfsPMb22X0Xc1Vwc,2417
-crawlo/extension/log_stats.py,sha256=vrChs3bj_Dvay3kxxkBOp4-w0K-IG-2XZ0PoSUahTPs,2908
-crawlo/extension/logging_extension.py,sha256=RfL1wI4nz-1Xtg420Ktp7RXnOPnZSHwO0Zpg1w4fO4M,1726
-crawlo/extension/memory_monitor.py,sha256=4aszl3C0GMQbqFhGZjZq5iQuXQR1sOz06VHjjEHgkyE,4290
-crawlo/extension/performance_profiler.py,sha256=EPiNuXuPPDU0Jtgy8arYHpr_8ASK13cCI2BytdJnu_I,4899
-crawlo/extension/request_recorder.py,sha256=RC23yzXClnVv9j2ljQvjBkUfWznfnDHsrQejKhE9y5E,4074
-crawlo/filters/__init__.py,sha256=XC_Q4ykZtSNYizYlAcehVwBBNO3lZ2zuWwafzXiuWyQ,4241
-crawlo/filters/aioredis_filter.py,sha256=XixK3DD5QbCLOw3Me2YdtMkxQpXOT75FE-GiVr_PUGc,8245
-crawlo/filters/memory_filter.py,sha256=mO4oBPV5_uAiBQF3a16tU5tcD8244dOjKoNX_MU8cEw,9292
-crawlo/items/__init__.py,sha256=bqekZrRlDhxfWie0UbCs656TptYseoe9QJ67I4E7Elk,386
-crawlo/items/base.py,sha256=tAYrPJgblp3ZEihDXvappdYc6pGdim6x2_9QSmMKI2o,577
-crawlo/items/fields.py,sha256=jCG0-PS8mVO48lP_ioTZCQCa0vjP5Sfv-sAyvYQqr-s,1800
-crawlo/items/items.py,sha256=e-3nXI9ckD64vcDxxQiAU6ufbtJMs09gbZQcYjxgwHY,3374
-crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0,549
-crawlo/middleware/default_header.py,sha256=wQ7BrUHd-hRosFoKsReV9hwNNr_jwK6V0ZfxL6MOGrk,5032
-crawlo/middleware/download_delay.py,sha256=zt9R5g2HWErWA_MAOnGcw_D8l6HD769Kyaw-Hv-vcTc,3438
-crawlo/middleware/middleware_manager.py,sha256=9Sj9rrWK6R9NZq9eT38sWRGuBKLKfjSgEAxu-5NCWgU,6278
-crawlo/middleware/offsite.py,sha256=cR0nVAygxAGbkmyI8yqiRtrZWTRLWddUiToMYGmhrfs,4084
-crawlo/middleware/proxy.py,sha256=NquB6tqHAgHs3-2_1_5220kJYfjNG5JyHRJyo_2j4wo,15636
-crawlo/middleware/request_ignore.py,sha256=xcyZ1c7r_HhbzR3r9pfjsLGW7L7FBVeYvlNt8cpP2wY,2577
-crawlo/middleware/response_code.py,sha256=-Aa9Mm9nJN-WdddN7iTanJRMA83_LYYgSEz3XLQGvMo,4934
-crawlo/middleware/response_filter.py,sha256=6VBUe04mu8C7XxmOak6XyhGMWZPYEm3AMo5Kt_r1SXY,4248
-crawlo/middleware/retry.py,sha256=HxeIf7DibeLCpZ_y4rNARWMyzlrsdq5UR2CaFZInA3s,4124
-crawlo/middleware/simple_proxy.py,sha256=V_v28L-faiMJtt8vi-u5O4za-aU77_JTqNTCYSfWzCE,2191
-crawlo/network/__init__.py,sha256=BLPERYPo22g1BXrW--wUnlolrdFUmOPjgOB8XQQJlck,397
-crawlo/network/request.py,sha256=9kV-gqb_d6aCsSBAwyzxnP9a70cAViwX8qvpyYV7Ym4,13799
-crawlo/network/response.py,sha256=EZiG4LjuIb7PxdGou4H-oSOQhec1ZdBRTkO-5fl8JTo,12701
-crawlo/pipelines/__init__.py,sha256=lrdVDjeHLNkA4_MAwI1auk_I9xfeU1SlBWXiammb6lc,616
-crawlo/pipelines/bloom_dedup_pipeline.py,sha256=omB_gHtoacbco0sn_c6HO6PHCh6xylSecK7UbJIeLq8,5661
-crawlo/pipelines/console_pipeline.py,sha256=KABkR3J-rqO0Awox7lizxKR2XuHfVhWPiVRgIybwwu4,1248
-crawlo/pipelines/csv_pipeline.py,sha256=6FBT2AoU6iNU-5NfgWRq7-JpF9dK2nBokjxx-y4jIas,12174
-crawlo/pipelines/database_dedup_pipeline.py,sha256=Ao_5jvVPl5QikxXhPeIrcB7_3tinR9bPNRV5Fu5zfDU,7978
-crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
-crawlo/pipelines/memory_dedup_pipeline.py,sha256=oIksbIrmSw9s9jMh6JJMfVbv6hzseVMV_g9S8UHQUP4,3837
-crawlo/pipelines/mongo_pipeline.py,sha256=k7gNqAO-g2MtIfArphC6z5ZzkKVRkBKcv-2ImziPFA0,5706
-crawlo/pipelines/mysql_pipeline.py,sha256=CKll3rNFXc0-BGQ_0A6QOSm2-ymHtdjdybX6bSB8i2g,13500
-crawlo/pipelines/pipeline_manager.py,sha256=R4zXYaMUseZsv5d7vbsNPFuXGjO_KWyDve0bHpqsL7Y,3079
-crawlo/pipelines/redis_dedup_pipeline.py,sha256=POYRiWAOp1pqDW9iTPJ8h3VcpLALeLrpw74MvJJqPiM,6342
-crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/queue/pqueue.py,sha256=yzF900ko2RReVNZtfk_VV3PzSXEUALI6SHf97geuu6s,1058
-crawlo/queue/queue_manager.py,sha256=XqS_oVbNQJWdtokOuDDPK-FzMrVdnZ3UKp1MF_DMJww,14941
-crawlo/queue/redis_priority_queue.py,sha256=k1OChSMRovSMkbbJ9388axfhpYeMevuJTe-3N1oYhbA,13126
-crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
-crawlo/settings/default_settings.py,sha256=VKaCb8JnHx_B-Zi3hN6Mt0QIdv0YRhFlhVEZ2fhx84o,9193
-crawlo/settings/setting_manager.py,sha256=4uuMpGVYzxjmQjvlGqfZ8hDaoSh34OAoL0LCATsMCkI,7512
-crawlo/spider/__init__.py,sha256=ZnSAL9PXLZSIH-Jdv-P6RuWmQUdukr8KPLQK6SXZZaU,20435
-crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
-crawlo/templates/run.py.tmpl,sha256=v_g-LQMYJ6pC8TZgyWj0yB2yTTKrwy9lEJufAYCXyxY,1228
-crawlo/templates/project/__init__.py.tmpl,sha256=f3ETIXw_O6K-lkL6lXM5znMPJW1FZYGFrwDs2BnHcnQ,58
-crawlo/templates/project/items.py.tmpl,sha256=mt1Mm--H2Ouos3r7JPkYh0r33rgYJf1YOMz0OZy8TYs,297
-crawlo/templates/project/middlewares.py.tmpl,sha256=T67p8j0laL4NJJ_3xzPM9yivgZRjTEMiEtEWLPwbkmw,4160
-crawlo/templates/project/pipelines.py.tmpl,sha256=GBHYU0Jx8sKDCdGJp44FMSH7u2slxoFg6a-R9Uwg_-I,2608
-crawlo/templates/project/settings.py.tmpl,sha256=K0WOyCJsiykbZjoZRhzmTVssoahETkYS2zb2q3Ai5Ts,9998
-crawlo/templates/project/settings_distributed.py.tmpl,sha256=BsSqtYl69NtFUTq-lDkXTzU7cwhqZhkna3x__pQx7oc,6692
-crawlo/templates/project/settings_gentle.py.tmpl,sha256=9S6l-v0yJOPDy3oxsCIpSDieXjxtHNPHhBqfGP28CG4,3975
-crawlo/templates/project/settings_high_performance.py.tmpl,sha256=zjlGESHvt3m_vsvAYDc5oE2Eui3eI9QEn3-uKFyHXpc,4706
-crawlo/templates/project/settings_minimal.py.tmpl,sha256=dFoz39BGkSzLDTwBN-mQh242SLzcP6g8MhI8Zk49jvw,909
-crawlo/templates/project/settings_simple.py.tmpl,sha256=iWQFaw1WxTJA2QF-kXH4nspDGuXvnGEAuqpGQHCfuew,4101
-crawlo/templates/project/spiders/__init__.py.tmpl,sha256=j_YKsw6HQMJyqlk3WUouP3bsr-XVxshRoSNakHBc00g,106
-crawlo/templates/spider/spider.py.tmpl,sha256=jMhzyxpIpV_KigB-pmN-5mGMiYtu4mfQIOvpZcCGGJI,5055
-crawlo/tools/__init__.py,sha256=8igeUXLD0vJ5ta2X91QyTvna6dOioKCn0z7EF4oHvHI,3942
-crawlo/tools/anti_crawler.py,sha256=MU6KEPT0q85e_-Px8Rmw1fxdwlmOdpXfo0KYVpPlivU,9163
-crawlo/tools/authenticated_proxy.py,sha256=L93WeXajIZ3si8xNcE7bBPv34FvqSyTvVfD78fJAKQE,7032
-crawlo/tools/data_formatter.py,sha256=NEj3NqPiyG67V6qDgn2KNj9VNHWOLNwX-7p_nad0znc,7583
-crawlo/tools/data_validator.py,sha256=hxPN28YtJDFFLjBBYhDjHmR8ShNTEjgIsv-cmcDKIu8,5310
-crawlo/tools/date_tools.py,sha256=jjP5xA0-aDgm9UIK1RG2qaNagBzHFQ-BBDMo_YzSlLQ,8906
-crawlo/tools/distributed_coordinator.py,sha256=Au20nZ4qUiAZUD2A1yfwD3soaHADpkEZt1hRyegp6M4,12323
-crawlo/tools/encoding_converter.py,sha256=7P9Z7J1ALw_PPNApmjFsHZDpRxgxzduiViluenlSLEU,4043
-crawlo/tools/request_tools.py,sha256=CjyFBtRQf_vFjQhaVwgHSGai4ZaWS8IIaF1flSfJxDs,2338
-crawlo/tools/retry_mechanism.py,sha256=aT5hEs5O7B09K1IaNFZEOWR9e_mX52Dtq4gx-onsyRI,7553
-crawlo/tools/scenario_adapter.py,sha256=JouFxI3513PRe1ObwHWc72vBvptNpNv0Ew3pRaEKjQQ,9398
-crawlo/tools/text_cleaner.py,sha256=SOgT9frD6Cg-2D7ZIzrixrxFYfYisLPU48ir9U2ZbA0,6458
-crawlo/utils/__init__.py,sha256=to1N8t0rNoczU9pteGt_RxhNrvfjtDxQidRwsTKcIjI,563
-crawlo/utils/batch_processor.py,sha256=_J-dKj98csB9LdhTBHh_dKvV4OzHiP22-5OWxavDglQ,8883
-crawlo/utils/controlled_spider_mixin.py,sha256=RVRAf9Wbi7z9NAlog4763xhHUEjl5r33aVMk7Oj4HCA,16497
-crawlo/utils/db_helper.py,sha256=3ib5-agrlwf2t5S_QtLRYH75wvJDlYbRqRmDEbpH5Bo,10559
-crawlo/utils/enhanced_error_handler.py,sha256=hj5AElt3ajfqnP4csQnEfEnzkbIep9k65DNQiCbmTFo,13858
-crawlo/utils/env_config.py,sha256=HbZOEKkeQ0FMdZYJu9SgmSNEmfPJrmAzA7lHu5Du1DA,3937
-crawlo/utils/error_handler.py,sha256=q6NqHxjYrKdswfmhshMYMmfBIr0M2YWPYxts4ScHl4Y,4244
-crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
-crawlo/utils/large_scale_config.py,sha256=j7wQ5ty7pQlBRygw2vhRJ7OI19RYBZKPfYMP3WeF2WI,8154
-crawlo/utils/large_scale_helper.py,sha256=Kxdy3WMuqjzQTyCc6z4xEYxXDi4xnYKJzsVwaBYZrrg,12108
-crawlo/utils/log.py,sha256=vAMACdX8N3kTIRegmKDE1oVImESufeGEskoRmNQQkJo,5281
-crawlo/utils/performance_monitor.py,sha256=Q9xxuXBIfFoig_U-FQPOUuPAh1axO3MzYgpielDyku0,9547
-crawlo/utils/queue_helper.py,sha256=xpUUTOqlU1xz2Pb9NKAVGo3AfAO-7Xvx8Lm1q65Dgck,4743
-crawlo/utils/redis_connection_pool.py,sha256=amGjhaKpodMrw9X56qxZ6f3OTZhjrI89sSVGqgwAQGU,11050
-crawlo/utils/redis_key_validator.py,sha256=M461uMU5mRZfYRSwf-fXJUi4UITNKUAZmLe-cvytm9c,5611
-crawlo/utils/request.py,sha256=yoLB2rY8d78vgPjIWpdhY5SalIKjyLIvTG_UH6EMdVI,8798
-crawlo/utils/request_serializer.py,sha256=k7PQG_Wa1S1k9qTvcKDeLOlX1aaa_0jo9sFUCQZBKBk,8521
-crawlo/utils/spider_loader.py,sha256=WK9gL99sOeIrFC-a0Y10lygtryQR7-wfdGks-uwMYTM,2172
-crawlo/utils/system.py,sha256=HvWV1acxou0Rn0L7pNq4CnV_GWFeU0Tmjy3_nLD8M64,237
-crawlo/utils/tools.py,sha256=5Uv25Wy4m_ndZY0-n-eX-t3PxvaZ6wR3-Wvx-o7_Vrs,271
-crawlo/utils/url.py,sha256=rlgX2VlJv6JvLmCDTsbxzMSXE6R5ZL_0dLEqprsA-JU,1482
-examples/__init__.py,sha256=6i631BPnS_TR_BWUjtjB5CBO-zv9kRkwQTQvSya2wHE,123
-tests/DOUBLE_CRAWLO_PREFIX_FIX_REPORT.md,sha256=HjMZv6RjN1o5D1mfgEydP8Mcc9T_4ScR6lG3xVxs8P8,3346
-tests/__init__.py,sha256=scL1IPVT1iucth7v8ffrjRdeW7QrC_Y7AMmFVMdTY1Y,129
-tests/advanced_tools_example.py,sha256=7nlFLRVMVYzDz_CAdgQa9fJu7o0M6jBMo7PTvUsRbo0,9065
-tests/authenticated_proxy_example.py,sha256=gebJn8x_haztbFbizAL5CUosEAlNRsQhnmD-jV0glDk,2864
-tests/cleaners_example.py,sha256=J6rT4rTbNzeN2YWf7IfLVwCGm3-UcSxE4LhH5AV-CE0,5164
-tests/config_validation_demo.py,sha256=5MzW5P7ZX6xoMW_zC6XmIA50KWMTu0iB5H2hTe42Sb8,4029
-tests/controlled_spider_example.py,sha256=SP_k4mdKPvD1JCPs9UCm68jcy2Frg84vvXv9-14RC6I,7776
-tests/date_tools_example.py,sha256=x_-duqnVZ-Hrk-SaNplIfcIV6W3c6u6MTxW35u1i0F0,4862
-tests/debug_pipelines.py,sha256=VpUmoYlt6Ci7foIGuQIotUu42xp6TzoA1cBDeagBzDk,2098
-tests/dynamic_loading_example.py,sha256=NI0SCg4lPME0RCcNpDDw1HjErjmCgJntCN0ahAEw61g,18263
-tests/dynamic_loading_test.py,sha256=DYbMrEewerx0VGXixci3p9VYgDDQvCPevA92CNjq1Jo,3309
-tests/env_config_example.py,sha256=sKE8DvMBhM3uy439LpgLHd4wF7MGUrUc-X6E7g9qsz0,4818
-tests/error_handling_example.py,sha256=goF8fnTXxU3CgHcX4ALEcidVPd-zACn2tDqqQislRPA,5123
-tests/redis_key_validation_demo.py,sha256=FxqEXRgJllkgjyIyEuegQrLDuXAvi9N-dfMlvFotRZ4,4337
-tests/request_params_example.py,sha256=bjHxK_ca6UO7kBff88nmoxXY1odiLQCGC36okjEi7gM,4100
-tests/response_improvements_example.py,sha256=wnYGJO6MKj5_jbwKLDlbXu_Dli5XC7vlWdzByi82_5Y,5258
-tests/test_advanced_tools.py,sha256=3R8EfKVyBHEb6FA5TP3ieaWeHZhobVgSx8t3phipCrE,5250
-tests/test_all_redis_key_configs.py,sha256=SGoip8M7oB2LNWC_31aJ4ECcDRmx0psr7i7DGzuaH7c,5565
-tests/test_authenticated_proxy.py,sha256=s4pr5JyBTHYQgRq_IymiVKE08vyW1MwR27pSwrrVLVk,4198
-tests/test_cleaners.py,sha256=UD-X_eLnQic6GYbtFzYnAKqG4XKOSGIDd1X2fAl7Jso,1762
-tests/test_comprehensive.py,sha256=kGNcJ9UkQxysYqvsBu0YxAaPleOvN9_hztLy7ljkfc4,5036
-tests/test_config_consistency.py,sha256=DJaAQxGL7RXHs-DWF_B4yhHFGSGHWHUoDmLFiMi4aJg,1921
-tests/test_config_merge.py,sha256=d8i8sU1XKS3egNKEYPZ2a6CBnJRx2M3p6q04wYufAcw,5454
-tests/test_config_validator.py,sha256=5ivB71KstHGNi2BPzcclf9hBukXEgt_B8N4l1HRjBFc,6020
-tests/test_crawlo_proxy_integration.py,sha256=_L62_soaHRYy_0fShjiZSmv-RtGICw7_kzhTNRoyFfc,2620
-tests/test_date_tools.py,sha256=CQdAmIS6bpAdwQH9ETDH__06l2gGL7EHUQuh7mdTF-A,3930
-tests/test_default_header_middleware.py,sha256=7kpONSsGMsmWgTX2pCpseme54_-82Baak0xVz6gclJk,5845
-tests/test_distributed.py,sha256=RQHUpDfRNG2x_1Cdr9DLk25IBcgapm_u0xSBMObE0Xc,1725
-tests/test_double_crawlo_fix.py,sha256=ZNkRDgWW2WN-QRNZhvIgTHonY-T_U_R_MOIBLuyJd_I,7770
-tests/test_double_crawlo_fix_simple.py,sha256=MlWUqo51kOQ7Gu6Neoler8FVyRs0jpmQWoORHMBENz0,4644
-tests/test_download_delay_middleware.py,sha256=Va79gsH_8BVrVVLA8gSwFEbrRJ7qwJMCC1cDJN6il_0,8886
-tests/test_downloader_proxy_compatibility.py,sha256=3Jn7RJd1R2ywuitHp2Jju1yYNg57R4QmKwjuHGojDUE,8635
-tests/test_dynamic_downloaders_proxy.py,sha256=PtEW-pnVijeX2yX34UcoXYEY23yTBxb-kyNYh-WDljQ,4326
-tests/test_dynamic_proxy.py,sha256=YL2sghNKG7k27-SaHMh4boNLVBHhfSttUwUqiSsOEX4,3080
-tests/test_dynamic_proxy_config.py,sha256=uYXZ804ULI9qYMF-uNjMbi3L_NGzoMqLJcEZAl7aZ2I,5707
-tests/test_dynamic_proxy_real.py,sha256=DTjP8JnSwBnNZ3Ls1BjDAmt6xSuye_6CxwZ4LBisPTM,3402
-tests/test_edge_cases.py,sha256=4XZIUPOtNM9WCoAV1dJYAK8T6NiWp18rcwLLwnpxILE,10426
-tests/test_enhanced_error_handler.py,sha256=YYKyjT9ARcIcyKDOObaQTws18HfsHN923BOTAzaxYF8,8311
-tests/test_env_config.py,sha256=nfP4nCG9ZHeJUfxo1JKUmiihYbhSeWx_oNW5mMfDHfQ,4746
-tests/test_error_handler_compatibility.py,sha256=o5JLLLdo25Sl_3hpMx6I2fqSgZFAcnI4E6Ci-KxAxwA,4129
-tests/test_final_validation.py,sha256=aAiWLzhDCcv-GEXg9sauaVIfq5rz3s2vm67Gk2_lmBI,4813
-tests/test_framework_env_usage.py,sha256=HYpTwORXeaJHMffCYAGHGvc_a6ax4lo28xP8BYOaKxk,4098
-tests/test_integration.py,sha256=OCkjyv76Wop7CrXEko6rfoDsIK6SESA18KgCaTwL7Q4,4670
-tests/test_item_dedup_redis_key.py,sha256=QxLuXHUx0xqT6y7lQzOWcrLkRui7Qs7C6NgRvjzIypA,3720
-tests/test_mode_consistency.py,sha256=X12X4496OoepOkRLz5OkJcJfFUeChnP9TiRWcR2J5p4,1175
-tests/test_offsite_middleware.py,sha256=L5YT9ZqcQwBunUv0Ddj-sLZcW4IMlAlgaJCwICHFWxI,7543
-tests/test_parsel.py,sha256=KYskaN_4HBc1XDTltjVo12v1i7JAThB2UIwcWZ-mwbY,672
-tests/test_performance.py,sha256=gOJ1EpU9uGynIxETLAroe98OA4QPcX1wchCDJoO41Kc,11130
-tests/test_proxy_api.py,sha256=dVqGElyL3K0_9IqkXzn7Ka2jSuhvYfR1BfZgyVukNM0,10749
-tests/test_proxy_health_check.py,sha256=xo3QMP1YNw9hu7JDHZOYCUZmFFKLJpHSh4SbxXhCRPQ,1091
-tests/test_proxy_middleware.py,sha256=qm2B0lepBZqzUpXNi4t1gjrQxUV4MQ2wvpmcaYV6O5A,3900
-tests/test_proxy_middleware_enhanced.py,sha256=YQRZs4bniU2CR-eKRc-sS4zZ6cjdSHwijUWL0T4Tq1w,6819
-tests/test_proxy_middleware_integration.py,sha256=zcl7fR9Toc-I-stSUTzKZPwcfh3kgrpjI5SbkZ6AVmE,4305
-tests/test_proxy_middleware_refactored.py,sha256=QiV9OodRb6hUcPnjDs-jraV8hlBBVLsUJE04geWHoD8,6776
-tests/test_proxy_providers.py,sha256=XwWZCywTYguSsUxSm6fsbaoH1p9dKjqSIx9-sqKZehA,1693
-tests/test_proxy_stats.py,sha256=Til_yksrRz2yBVw-yJi5-36LhNW3vTwpXTm4BdR9PUM,507
-tests/test_proxy_strategies.py,sha256=ZkziozkvZd3KWOQnpHQ8Upd3WpyoX7gN0qFGluNm348,1809
-tests/test_queue_manager_double_crawlo.py,sha256=YzM6PnoyRSST-f2NVyI97bpPcoYWL06HUwf08Fyx3Qg,6784
-tests/test_queue_manager_redis_key.py,sha256=nCCMnpKPNP5fyd4zb4LG2kmJAUcLoa8ODhBGcz4GcCU,6231
-tests/test_random_user_agent.py,sha256=LuyR8WaKfqOap9WBQl4WEBcZDmKxhW80T-_wXbuo2Qw,2230
-tests/test_real_scenario_proxy.py,sha256=LGtxEvCiTgn6aTPGd7ZuqaCjApsjosD2DunJrd8-jFE,8259
-tests/test_redis_config.py,sha256=DBrqURBQt517Rt1h1l2iIKrKDfbkJzQSRUEYYbapcy4,875
-tests/test_redis_connection_pool.py,sha256=WIUQlI6K3IINan14vknI4oFf9a8wpHCWi87KSfoB7_E,9034
-tests/test_redis_key_naming.py,sha256=7_X_PSzFQn5m0n_7qLlCjFvY4ZKScC36cqWFu1PAFRw,6730
-tests/test_redis_key_validator.py,sha256=VFuawmaA0G7VSHueCvZEQNKY-L2IdDGlEcyuJ9nZu7Q,4295
-tests/test_redis_queue.py,sha256=2OZJHn5fN9b6XEgEs4Ht1AL6TOJ_H-IR9JxPzzvqMpg,6534
-tests/test_request_ignore_middleware.py,sha256=8_2E6JU27eOWI3iHeh3YscLnp3SIHaubWdA477Ki6PE,6047
-tests/test_request_params.py,sha256=9vNksaOrFbuSb0UffruPxUHhJXZxVYyjQw9J69FSzH8,4176
-tests/test_request_serialization.py,sha256=TPBIzjaifcAjFWCFSFZ5ewRn814jSGPL28MGTwvrr_w,2262
-tests/test_response_code_middleware.py,sha256=EAPHsNN3J4ShJ5UfpzcZpOPjDvYgYJgcpNxpRgpQBaE,11844
-tests/test_response_filter_middleware.py,sha256=yVXl7LYxKxziGQg-YpdbVc6CVhmfmNOAZ86SaEVfMyI,15900
-tests/test_response_improvements.py,sha256=zvbkTkWhgdlFYtRu_ckgq6wGDGwpe_PTECYqpLDM3BU,5876
-tests/test_retry_middleware.py,sha256=g7QPDDLaX7hu3jqlWbDsjfuCbQXisTvs7YrcMLmn1Hw,7856
-tests/test_scheduler.py,sha256=j1t-ItXkCHZpoLI4T_2SFzl0Wbcg81LYeOOfnjsYKuk,7930
-tests/test_scheduler_config_update.py,sha256=eVE9lr7pTHlPCra9rtDge5rp4csiY3veYlG43qc34Fg,4128
-tests/test_simple_response.py,sha256=6RYOBRzAtyNvJ9a5JVTNubM-rvxnuX8jQOvq3sUZxwo,1488
-tests/test_telecom_spider_redis_key.py,sha256=hL_gjLJQbv6Tnli6Id7jXJcP-bgIQzhuWLUTBok6iv0,7420
-tests/test_template_content.py,sha256=q3OYI26e-qpmCYb1qmg5qg0kl7A8BhYbK2QjvKGaS0o,2816
-tests/test_template_redis_key.py,sha256=wJGAgWGO3hpSWoAUHHpBexXF7J2UP_tM6Z_PBjJl96Q,4742
-tests/test_tools.py,sha256=9t9FXZ61MfdB70nck9NYzCq97yd3SLVlLiMybEAlClk,5345
-tests/test_user_agents.py,sha256=rUotyuE2iJDi2LQBrUh980U-dAMTs4ARPMJxICOoQFY,3231
-tests/tools_example.py,sha256=MtIypR-OFiWwi-skurwmq4fM0cGTt-GUX4hSekYs7BY,7739
-tests/verify_distributed.py,sha256=krnYYA5Qx9xXDMWc9YF5DxPSplGvawDg2n0l-3CAqoM,3928
-crawlo-1.3.1.dist-info/METADATA,sha256=6m0OV78Iso8DwQFiIB0_mWkYSDymuhjw69F9aVKwtCo,26813
-crawlo-1.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.3.1.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.3.1.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.3.1.dist-info/RECORD,,