crawlo 1.1.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

{crawlo-1.1.1.dist-info → crawlo-1.1.2.dist-info}/RECORD

@@ -1,35 +1,39 @@
-crawlo/__init__.py,sha256=zOGI9hnWawIl0QA0Hnjmqo7vfd-WXNevOP9nLCq04XA,811
-crawlo/__version__.py,sha256=q8_5C0f-8mHWNb6mMw02zlYPnEGXBqvOmP3z0CEwZKM,22
+crawlo/__init__.py,sha256=esOolburYDjtF43D5N9Kh6TSQW2yKcz888ilhBSinBc,825
+crawlo/__version__.py,sha256=5SgGjThsHu_ITn8V83BvCziqCwxdXxTQqcC3KQMHPfM,22
 crawlo/cli.py,sha256=CtR2Pfa7SyRxEKPaXqt-6E6K5Vq5z3rfdAI95UO4cbU,1166
-crawlo/crawler.py,sha256=xwViGsJutKjAvfrYlMUd0NQKQtBX2r5qNMvpWkujxTs,19558
+crawlo/config.py,sha256=i0Amz6wNPgv_aVcdCBRRlcwuZLSa87cH9OEmTQvB97Q,8329
+crawlo/crawler.py,sha256=v6i5tjgSOtbMoqiw1qdgKx1cY4kcVcd5l5bUTWtJNNU,36461
 crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
 crawlo/exceptions.py,sha256=pthF1lJlJHyRZm-mE6NAo5WzK3GYJqmRqIuIlK1Odx8,1129
+crawlo/mode_manager.py,sha256=WIxrq9S3EAH0D71LH1AxvcqXomeABqoXgtUN4A--DKY,6702
 crawlo/project.py,sha256=xWN2eTAjf_Pza-wWvvV4JjScQRWxe9hXlztX81ccUMc,5182
 crawlo/stats_collector.py,sha256=NkO09CB-220qz5rxFcD_dedGfr2VPFrDo4hya0Zh8Qc,1577
 crawlo/subscriber.py,sha256=3d4eYtkSgPj-18-mTZM6RQLSil-ux13FUcmfFxr3sMk,3730
-crawlo/task_manager.py,sha256=AS7Xu_8Q_eb3jg9QSkK_wv6W1rRXaI6WjDp8p6h9ltU,721
+crawlo/task_manager.py,sha256=19InAxS9oJ9EMj20Aw2urN-v6BeC22dkgPuW-B9-4UI,819
 crawlo/commands/__init__.py,sha256=AMYjXG7ulE8dPVmgWVo0uqXsaCYUUZYmmu2-7kFzH1M,342
 crawlo/commands/check.py,sha256=172OiAxnX5wwSlszUsyPgMZwAoIbGDTdfhtRz309ilc,22843
-crawlo/commands/genspider.py,sha256=In7X463vFCDhow73Netb4S1Vug_E0VC-1sevDc5nmSA,4267
-crawlo/commands/list.py,sha256=P4O9zma8RA0061B2t8hRgz6FtRzINJRPbeWH7m6TZRg,5091
+crawlo/commands/genspider.py,sha256=-jGJdfXLsefX_H1ydQ2wirdu6p6wmhClzVXY_0L-1aE,5050
+crawlo/commands/list.py,sha256=yByqQeZBgvjewOKxpnOobpeJ7Hnbs-CWsoyITqZu2ZY,5781
 crawlo/commands/run.py,sha256=8Qngjsl8Q4RBdO39a__wKGsheY2PFuPit2hds_jwEbM,10524
-crawlo/commands/startproject.py,sha256=lu7_E_ygnM-S5LsViuHoaCdJubFGIY5ecWLL-g3R8A8,3953
+crawlo/commands/startproject.py,sha256=bzNgpkKzUEggY2m7Iw810mSPe8wOPFBqSCO0jZX3z_g,7138
 crawlo/commands/stats.py,sha256=6pAgkEi8MBnCer2rWmKpaTYr1jaM6HeMG9owAvEzJyY,6064
+crawlo/commands/utils.py,sha256=nohMvUU2zLvX0XzXk6KeCNxP0EvSWj9DiVLxM_7tD5o,5106
 crawlo/core/__init__.py,sha256=PnFyJdVNHBoPmV1sW0AHQXijeoSTQ8cMYrbNM1JK8kA,41
-crawlo/core/engine.py,sha256=xBYi-V1O3IfZU9Qo1TJynOpoMjdP_h8kmHC4iDbVfwE,5868
+crawlo/core/engine.py,sha256=8Dcew1XyxChW5Fz1wFEWpJlPrQb2hKDWKul8e61S-Q0,6662
+crawlo/core/enhanced_engine.py,sha256=9I9Uxdy2oAz8zDGTzEiytuKu__VDVmIN8zwZKfrD8bw,6254
 crawlo/core/processor.py,sha256=qmCqAeqhwYu-UE86evYesaGt9qpuSIfH-ZIZKcXFCZc,1140
-crawlo/core/scheduler.py,sha256=oxVzdXvU0CDRfUFkv2MP0NvTRfTGbcvoz-__dNy_rzU,1885
-crawlo/downloader/__init__.py,sha256=ukrDBULCaoDWoMLCO3XcQhDoasF0oUzj0PHnJ_ACJaE,2306
-crawlo/downloader/aiohttp_downloader.py,sha256=ZxeKvWyAYzKCwu_yRfduX0hSwU3TEqvhBQaB9UBpzNE,7476
-crawlo/downloader/cffi_downloader.py,sha256=j_LHvGxrQ9Ynod9RYp79vPgHlwKPPNc6NH8cN5hk-pk,10052
-crawlo/downloader/httpx_downloader.py,sha256=fuienQkkn2yQ9R4RlvrxxWqX7OlChJSBEP11-5wNboY,11508
+crawlo/core/scheduler.py,sha256=iNZl47PvurnLLXRZN8Jfj1un6YP2q9m5n56X3Eb7_A8,5615
+crawlo/downloader/__init__.py,sha256=tl0mE54reR-PuJYSsXsKP2VY5uzvq4lITxZwKKjNzPs,7663
+crawlo/downloader/aiohttp_downloader.py,sha256=UKupGYPOWrscAVsjhFgKYElTa9tbEeltqV7nuWqjIeE,8005
+crawlo/downloader/cffi_downloader.py,sha256=-GVfSIhi1Ip56suSiGf8jnUE2EBF1P56vw0uxLh_T6I,10440
+crawlo/downloader/httpx_downloader.py,sha256=FJcdE3BoxUYFQDPm3quaveSjweAKfAQRaeJm1hQViLg,12003
 crawlo/extension/__init__.py,sha256=PEBbMxi6ULBZ9ivEJ4T7IH_R376BQJ6Rz_D9Ce-Cqbs,1133
 crawlo/extension/log_interval.py,sha256=S-hSoiz9GdmgHrac4vDQ52fleoBcH-kzdPUD8YRAons,1922
 crawlo/extension/log_stats.py,sha256=VinABjzuFa-JmNXSX86gQ_3Oyx1y3vUhpWTEu9HRRqg,1677
 crawlo/extension/logging_extension.py,sha256=Jce2HHc9ejsIWQitkHZuhnLbT_1MIfcvywfzYkkJ5eY,1202
-crawlo/filters/__init__.py,sha256=BCZl86BHiTfDGRe_b1TlNSr6pfNbMKTu0Uq0j4gX_1Q,977
-crawlo/filters/aioredis_filter.py,sha256=tlkKe7pdXxFEjFBl8UN07JHn_QbOJNY3y3BRj6uFXpg,5471
-crawlo/filters/memory_filter.py,sha256=bs2WUe7CdHiXgr344vzDqMfBv1b3RwXJMnwxpDb64Pw,6639
+crawlo/filters/__init__.py,sha256=XC_Q4ykZtSNYizYlAcehVwBBNO3lZ2zuWwafzXiuWyQ,4241
+crawlo/filters/aioredis_filter.py,sha256=3vbPOY3L_H7TUgL8K6AtaP6QDtoIg5xpE70Kxpil6dM,8376
+crawlo/filters/memory_filter.py,sha256=VJO0UFRYGxmV8dj4G1subsQ-FtvPcGLbvd7IVtqXnOs,9260
 crawlo/items/__init__.py,sha256=bqekZrRlDhxfWie0UbCs656TptYseoe9QJ67I4E7Elk,386
 crawlo/items/base.py,sha256=tAYrPJgblp3ZEihDXvappdYc6pGdim6x2_9QSmMKI2o,577
 crawlo/items/fields.py,sha256=wMlakQTsEwyrlLzMt1gI4pScLQZMqd3E1xcfH4dbSqk,1801
@@ -38,63 +42,67 @@ crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0
 crawlo/middleware/default_header.py,sha256=i_Uj07JObyeZFxL7ZAZmvZsHvA1HGtkNab1sA0d-nWI,1067
 crawlo/middleware/download_delay.py,sha256=2M-TchDA7MwyTfYy0Hzh_bW9wlHlpiP-oQlys7crTj0,966
 crawlo/middleware/middleware_manager.py,sha256=j1hkWRFB5rnC5SnB7oXWE5eUNv8blS9krDIDM5fIDs8,6213
-crawlo/middleware/proxy.py,sha256=3msCHiPgaaKt6vRjnNswkwSzUFrO4a8OgL9BeV42ySg,9555
+crawlo/middleware/proxy.py,sha256=m2ZZ50En9hUtgrqSqA6hItGT74xMqccHFPhZshutIco,9811
 crawlo/middleware/request_ignore.py,sha256=QI2z4fUnJ-4xvPTZAmsL-GqR4RFHS1xq9iDr5KFrMco,997
 crawlo/middleware/response_code.py,sha256=tmef2QVl3JCiTMii6VQkASlOY2OyqmOPoOfNxIK1eF8,659
 crawlo/middleware/response_filter.py,sha256=ep8ZxDlfIefi9YqK8dPASEp5TTDRo9QEY_jMceC411s,837
-crawlo/middleware/retry.py,sha256=6I0FXJ4zCnJ-pMBxoc8Cg4OVCcH4Buyfoy6ivSu9bro,3369
-crawlo/network/__init__.py,sha256=VaD0GmsgDYJ8UMgrtjeOc1Wc7lDGee1uAF3neRpyug0,123
-crawlo/network/request.py,sha256=JD4p9e0osjAFb40Ux57NVJliUe7NQ6_IxoMwj4faczs,7044
-crawlo/network/response.py,sha256=_oaElq7_Y-5gz2k8lhfZIss7hxFh6dJ77OpuXZXr9oI,6034
-crawlo/pipelines/__init__.py,sha256=Hk-M6X0VCGLp6OEdgnhXGhGhKS5TjKf6dkg8bU9pvUE,260
+crawlo/middleware/retry.py,sha256=pH4fdW71DwQzMY7APutX7yqtvSTAoznYeM3ux2EOT7E,4146
+crawlo/network/__init__.py,sha256=BLPERYPo22g1BXrW--wUnlolrdFUmOPjgOB8XQQJlck,397
+crawlo/network/request.py,sha256=5bzXJJWcAgGngNos-k4eIh9QaJdnGw0_SLgdrNt_0-4,11103
+crawlo/network/response.py,sha256=M0iG8ggcEpFzN7aAinKwVbxHAhANLYj8Zp4-FwaTpYo,9812
+crawlo/pipelines/__init__.py,sha256=Tvs9FQ_2qz8GcmoEqLrz_YFZrRpL3gGIssiI5Ce9b5I,254
 crawlo/pipelines/console_pipeline.py,sha256=KABkR3J-rqO0Awox7lizxKR2XuHfVhWPiVRgIybwwu4,1248
+crawlo/pipelines/csv_pipeline.py,sha256=6FBT2AoU6iNU-5NfgWRq7-JpF9dK2nBokjxx-y4jIas,12174
+crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
 crawlo/pipelines/mongo_pipeline.py,sha256=lv-Zn_mWdE_jVy7Nh30Lzqm3YhtLRV5rMy-m4rBWYe0,4442
-crawlo/pipelines/mysql_batch_pipline.py,sha256=C7EPzAluY-Kplc6MB2UYLLDdahYsnZMZf3cxQ5vloeQ,10339
 crawlo/pipelines/mysql_pipeline.py,sha256=6g6PGTwAyzH5VStlPAg2SdG5t1lPw1Zu-cY7x6Mz16s,7861
 crawlo/pipelines/pipeline_manager.py,sha256=VrbebOYiqrobtKhp5II18w-odCICdWkmRg5WPK0Emz4,2112
+crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlo/queue/pqueue.py,sha256=yzF900ko2RReVNZtfk_VV3PzSXEUALI6SHf97geuu6s,1058
+crawlo/queue/queue_manager.py,sha256=OQ9YBbc9Y4uE_5WbQhe233FFdhaeiOhDVbqWF7ev42U,11303
+crawlo/queue/redis_priority_queue.py,sha256=hf7hiIiRS-2DIq1bIJbs1yLEZjuMnHwc-n2_I7PlE9A,7294
 crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
-crawlo/settings/default_settings.py,sha256=BDpjZkPYQhuutxZomWYS6WvBHomKfHdNVSvTmyz1bKY,7100
+crawlo/settings/default_settings.py,sha256=jRM_Cqt3tQ7V4mgqBTAIQY3BcfGsLkKdvW1TcbWHv48,9627
 crawlo/settings/setting_manager.py,sha256=SxKB1aCWh4OySM_bH9cYng9I3PAmrSP-Q8XOZEWEwbI,2899
-crawlo/spider/__init__.py,sha256=kNBJ02QB0EZcGvzMH6GYh5UssI1HzOREIHz3FVo8LOA,3854
+crawlo/spider/__init__.py,sha256=Z_rK23l5yt-DuwJPg8bcqodM_FIs4-iHLaKOimGumcE,20452
 crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
 crawlo/templates/project/__init__.py.tmpl,sha256=f3ETIXw_O6K-lkL6lXM5znMPJW1FZYGFrwDs2BnHcnQ,58
 crawlo/templates/project/items.py.tmpl,sha256=3h-4nuneUoCAGUzSLyLNsdgeAAUXzz4rV0QctzCkIJA,301
-crawlo/templates/project/middlewares.py.tmpl,sha256=oy5RCkRqLeBpvtYA1NGVlbYfl8rWuS3VDgQSx5opnHQ,2102
-crawlo/templates/project/pipelines.py.tmpl,sha256=unEag7_qMN5DiffIzTKRmTVUCiTkHGCEad-jRQ6eH4I,1817
-crawlo/templates/project/settings.py.tmpl,sha256=By0YaZPxAP0pha_oY7J-vm9ntuvEfJO1jTiUhNddGic,1716
+crawlo/templates/project/middlewares.py.tmpl,sha256=Ua2vG3WXliMBSmwsbYaSwzx2lZPw9vLrG2dLqtZWGRg,3157
+crawlo/templates/project/pipelines.py.tmpl,sha256=-dxK7T396AVTbh0FhivLDZq4lKxw3p1-5Y6Ij4jgImE,12041
+crawlo/templates/project/run.py.tmpl,sha256=RQ_k_B-Pp1R853-N_58yUGjIouXa6ipBtUPQenLjswE,7519
+crawlo/templates/project/settings.py.tmpl,sha256=NxPvXGnomTV5NkczPIC7Tscn2vjmHUE4CYmbQB2SmDQ,7884
 crawlo/templates/project/spiders/__init__.py.tmpl,sha256=j_YKsw6HQMJyqlk3WUouP3bsr-XVxshRoSNakHBc00g,106
-crawlo/templates/spider/spider.py.tmpl,sha256=nyFMfk6qsmwfGuuNEcu6N6VnrOtq1GHVcoJOYRHrZFA,838
+crawlo/templates/spider/spider.py.tmpl,sha256=Wx1yMBsqDQvJGb9HoEVecX6XZcVrJiBj9blXYrernTo,6167
 crawlo/utils/__init__.py,sha256=BDORpyjMN7VGPKImnCDKSkprS-petgD7ezc9rMlBvb0,123
+crawlo/utils/controlled_spider_mixin.py,sha256=BpH76XxBJtYQ0rmCX6vrG-8gQXT9-LP65ibetzfqncU,12500
 crawlo/utils/date_tools.py,sha256=0yG0tzGb1VFgWDJJ_cow2LJfz3kj_w2MqSjmfKKESl8,6961
 crawlo/utils/db_helper.py,sha256=3ib5-agrlwf2t5S_QtLRYH75wvJDlYbRqRmDEbpH5Bo,10559
 crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
+crawlo/utils/large_scale_config.py,sha256=j7wQ5ty7pQlBRygw2vhRJ7OI19RYBZKPfYMP3WeF2WI,8154
+crawlo/utils/large_scale_helper.py,sha256=JJqcGSI6VaVe3MSL6IWjmCp8XQIu6T4U-BvBLSttr_s,12157
 crawlo/utils/log.py,sha256=A3lPyhD8kD88cV23KOL-_eT8g69xGQ5L1toDB2AO0mc,4005
-crawlo/utils/pqueue.py,sha256=4Ymkm38fRFqEcSJeD_ULkuBaCk6QdYvJdnYvtJjh-Tk,5386
+crawlo/utils/queue_helper.py,sha256=xpUUTOqlU1xz2Pb9NKAVGo3AfAO-7Xvx8Lm1q65Dgck,4743
 crawlo/utils/request.py,sha256=yoLB2rY8d78vgPjIWpdhY5SalIKjyLIvTG_UH6EMdVI,8798
+crawlo/utils/request_serializer.py,sha256=bPoSQqE2ksiMyP3WiPB3w3UqZs4f_LgkAw4Pj0qyBDo,8565
 crawlo/utils/spider_loader.py,sha256=pEDUsYOTGjszA6KgjiMlYN4GS5fP4uakkhcp3JTFFQY,2187
 crawlo/utils/system.py,sha256=HvWV1acxou0Rn0L7pNq4CnV_GWFeU0Tmjy3_nLD8M64,237
 crawlo/utils/tools.py,sha256=5Uv25Wy4m_ndZY0-n-eX-t3PxvaZ6wR3-Wvx-o7_Vrs,271
 crawlo/utils/url.py,sha256=rlgX2VlJv6JvLmCDTsbxzMSXE6R5ZL_0dLEqprsA-JU,1482
 examples/__init__.py,sha256=6i631BPnS_TR_BWUjtjB5CBO-zv9kRkwQTQvSya2wHE,123
-examples/baidu_spider/__init__.py,sha256=xlj0-TBQBhcKglllla_bQbufNiv10UFE0KsWMLvzFz4,123
-examples/baidu_spider/demo.py,sha256=MTEHkm7U4Kyx5QULCgR6to391xn4XPay6fmuV1c1uRc,24278
-examples/baidu_spider/items.py,sha256=e_LNs13SFiKkqwP2sdKhX72XkiX-E-ythMQOsY5P8IA,1264
-examples/baidu_spider/middleware.py,sha256=I71ZMmWTiDBFq4t2zfTE7IIXCqwaaeQ1DvKGW70q2Yg,1397
-examples/baidu_spider/pipeline.py,sha256=TUK_LnrU818UYmCn2_gKeNaTZjaj9qjrlndRLsR4wf0,1437
-examples/baidu_spider/run.py,sha256=7ifJVryShJ41rnlcdzU6rfex8L0Av1XpeSo4DhRRH6w,564
-examples/baidu_spider/settings.py,sha256=GOnHyYCSyOacCTn-sFAptNR3oSBpWxrGf1XkVHLaeng,3925
-examples/baidu_spider/spiders/__init__.py,sha256=eJ_ih4GiGfwQzPILeouy1Hnc4BrPz0KNPYlLHYvrvoc,123
-examples/baidu_spider/spiders/bai_du.py,sha256=pw4WccbmBR07CuSqCgm_7x9SH63FDJS_sXSaN5Ew5Tw,1589
-examples/baidu_spider/spiders/miit.py,sha256=RStlK5GdrILhqTRNpeWlP_jCijWPUpmmN7Sq6ua_K3s,6338
-examples/baidu_spider/spiders/sina.py,sha256=Q1TwvwutfGdFZzMIDyppOlwQIdr4bQSd2m42iNP8T5o,2418
 tests/__init__.py,sha256=scL1IPVT1iucth7v8ffrjRdeW7QrC_Y7AMmFVMdTY1Y,129
+tests/test_final_validation.py,sha256=fBxf_6YcAEa_HyV_oGAXmmVHY4i6FdA4J6klCmc36hQ,4925
 tests/test_proxy_health_check.py,sha256=xo3QMP1YNw9hu7JDHZOYCUZmFFKLJpHSh4SbxXhCRPQ,1091
 tests/test_proxy_middleware_integration.py,sha256=zcl7fR9Toc-I-stSUTzKZPwcfh3kgrpjI5SbkZ6AVmE,4305
 tests/test_proxy_providers.py,sha256=XwWZCywTYguSsUxSm6fsbaoH1p9dKjqSIx9-sqKZehA,1693
 tests/test_proxy_stats.py,sha256=Til_yksrRz2yBVw-yJi5-36LhNW3vTwpXTm4BdR9PUM,507
 tests/test_proxy_strategies.py,sha256=ZkziozkvZd3KWOQnpHQ8Upd3WpyoX7gN0qFGluNm348,1809
-crawlo-1.1.1.dist-info/METADATA,sha256=t26VSUHs-38nd7L3JdSmD4zCICOUCv8aHFUSsvvLgIs,5347
-crawlo-1.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.1.1.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.1.1.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.1.1.dist-info/RECORD,,
+tests/test_redis_config.py,sha256=TqzFRojc6esGXjGhUCvSLYQDUTAgEJsty9vRVuNraMU,893
+tests/test_redis_queue.py,sha256=o6xViXxJcdx-1eMcG3vhAQEIm8h346HnZb7JXs7ZjwM,6622
+tests/test_request_serialization.py,sha256=8sVdppAsohJ5u-m1WvablCndwL-M_36YPLdGKwgeznM,2289
+tests/test_scheduler.py,sha256=-FOkTWzaMdr6yfO1Msu74hI_GgSfD7iRxO-cFA-9Iyk,7442
+crawlo-1.1.2.dist-info/METADATA,sha256=fIXPyz2xk3GN82z5693BePRT1OWvul312NOh41CBRXA,17061
+crawlo-1.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.1.2.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.1.2.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.1.2.dist-info/RECORD,,

tests/test_final_validation.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+最终验证测试：确认分布式队列的 logger 序列化问题已完全解决
+"""
+import asyncio
+import pickle
+import sys
+sys.path.insert(0, "..")
+
+from crawlo.network.request import Request
+from crawlo.spider import Spider
+from crawlo.core.scheduler import Scheduler
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+from crawlo.utils.log import get_logger
+from unittest.mock import Mock
+
+
+class TestSpider(Spider):
+    """测试爬虫"""
+    name = "validation_spider"
+    
+    def __init__(self):
+        super().__init__()
+        # 故意添加多个 logger 来测试清理
+        self.custom_logger = get_logger("custom")
+        self.debug_logger = get_logger("debug")
+        self.nested_data = {
+            'logger': get_logger("nested"),
+            'sub': {
+                'logger_ref': get_logger("sub_logger")
+            }
+        }
+    
+    def parse(self, response):
+        # 验证主 logger 还在
+        self.logger.info(f"✅ 主 logger 工作正常: {response.url}")
+        return {"url": response.url, "status": "success"}
+
+
+def test_scheduler_cleaning():
+    """测试调度器的 logger 清理"""
+    print("🔍 测试调度器 logger 清理...")
+    
+    spider = TestSpider()
+    request = Request(
+        url="https://scheduler-test.com",
+        callback=spider.parse,
+        meta={"logger": get_logger("meta_logger")}
+    )
+    
+    # Mock crawler 和 scheduler
+    class MockCrawler:
+        def __init__(self):
+            self.spider = spider
+    
+    class MockScheduler(Scheduler):
+        def __init__(self):
+            self.crawler = MockCrawler()
+            self.logger = get_logger("MockScheduler")
+    
+    scheduler = MockScheduler()
+    
+    # 清理前检查
+    print(f"   🔧 清理前 - spider.logger: {spider.logger is not None}")
+    print(f"   🔧 清理前 - spider.custom_logger: {spider.custom_logger is not None}")
+    print(f"   🔧 清理前 - request.callback: {request.callback is not None}")
+    
+    # 执行清理
+    cleaned_request = scheduler._deep_clean_loggers(request)
+    
+    # 清理后检查
+    print(f"   ✅ 清理后 - spider.logger: {spider.logger is not None}")
+    print(f"   ✅ 清理后 - spider.custom_logger: {spider.custom_logger is None}")
+    print(f"   ✅ 清理后 - request.callback: {cleaned_request.callback is None}")
+    
+    # 序列化测试
+    try:
+        serialized = pickle.dumps(cleaned_request)
+        print(f"   ✅ 调度器清理后序列化成功，大小: {len(serialized)} bytes")
+        return True
+    except Exception as e:
+        print(f"   ❌ 调度器清理后序列化失败: {e}")
+        return False
+
+
+async def test_redis_queue_cleaning():
+    """测试 Redis 队列的 logger 清理"""
+    print("\\n🔍 测试 Redis 队列 logger 清理...")
+    
+    spider = TestSpider()
+    request = Request(
+        url="https://redis-test.com",
+        callback=spider.parse,
+        meta={"logger": get_logger("meta_logger")}
+    )
+    
+    try:
+        queue = RedisPriorityQueue(redis_url="redis://127.0.0.1:6379/0")
+        await queue.connect()
+        
+        # 入队测试
+        success = await queue.put(request, priority=0)
+        print(f"   ✅ Redis 队列入队成功: {success}")
+        
+        if success:
+            # 出队测试
+            retrieved = await queue.get(timeout=2.0)
+            if retrieved:
+                print(f"   ✅ Redis 队列出队成功: {retrieved.url}")
+                print(f"   ✅ callback 信息保存: {'_callback_info' in retrieved.meta}")
+                await queue.close()
+                return True
+            else:
+                print("   ❌ 出队失败")
+                await queue.close()
+                return False
+        else:
+            await queue.close()
+            return False
+            
+    except Exception as e:
+        print(f"   ❌ Redis 队列测试失败: {e}")
+        return False
+
+
+async def main():
+    """主测试函数"""
+    print("🚀 开始最终验证测试...")
+    print("=" * 60)
+    
+    # 测试 1: 调度器清理
+    scheduler_ok = test_scheduler_cleaning()
+    
+    # 测试 2: Redis 队列清理
+    redis_ok = await test_redis_queue_cleaning()
+    
+    print("\\n" + "=" * 60)
+    print("📊 测试结果汇总:")
+    print(f"   调度器 logger 清理: {'✅ 通过' if scheduler_ok else '❌ 失败'}")
+    print(f"   Redis 队列清理: {'✅ 通过' if redis_ok else '❌ 失败'}")
+    
+    if scheduler_ok and redis_ok:
+        print("\\n🎉 所有测试通过！")
+        print("✅ 分布式队列的 logger 序列化问题已完全修复！")
+        print("✅ Crawlo 现在可以正常使用 Redis 分布式队列了！")
+        return True
+    else:
+        print("\\n❌ 部分测试失败，需要进一步修复")
+        return False
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/test_redis_config.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+快速测试 Redis 连接配置修复
+"""
+import asyncio
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+from crawlo.settings.default_settings import REDIS_URL
+
+async def test_redis_config():
+    """测试修复后的 Redis 配置"""
+    print(f"🔍 测试 Redis 配置: {REDIS_URL}")
+    
+    try:
+        queue = RedisPriorityQueue(redis_url=REDIS_URL)
+        await queue.connect()
+        print("✅ Redis 连接成功！")
+        await queue.close()
+        return True
+    except Exception as e:
+        print(f"❌ Redis 连接失败: {e}")
+        return False
+
+if __name__ == "__main__":
+    success = asyncio.run(test_redis_config())
+    if success:
+        print("🎉 配置修复成功！现在可以运行你的爬虫了。")
+    else:
+        print("❌ 配置仍有问题，请检查 Redis 服务状态。")

tests/test_redis_queue.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Redis 分布式队列测试脚本
+用于诊断和修复分布式队列问题
+"""
+import asyncio
+import sys
+import traceback
+import time
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+from crawlo.network.request import Request
+
+
+async def test_redis_connection():
+    """测试 Redis 连接"""
+    print("🔍 1. 测试 Redis 连接...")
+    
+    # 测试不同的 Redis URL 格式
+    test_urls = [
+        "redis://localhost:6379/0",
+        "redis://:oscar&0503@127.0.0.1:6379/0",  # 带密码
+        "redis://127.0.0.1:6379/0",               # 无密码
+    ]
+    
+    for redis_url in test_urls:
+        try:
+            print(f"   尝试连接: {redis_url}")
+            queue = RedisPriorityQueue(redis_url=redis_url)
+            await queue.connect()
+            print(f"   ✅ 连接成功: {redis_url}")
+            await queue.close()
+            return redis_url
+        except Exception as e:
+            print(f"   ❌ 连接失败: {redis_url} - {e}")
+    
+    raise ConnectionError("所有 Redis URL 都连接失败")
+
+
+async def test_queue_operations(redis_url):
+    """测试队列基本操作"""
+    print("🔍 2. 测试队列基本操作...")
+    
+    queue = RedisPriorityQueue(
+        redis_url=redis_url,
+        queue_name="test:crawlo:requests",
+        max_retries=2
+    )
+    
+    try:
+        await queue.connect()
+        
+        # 测试 put 操作
+        test_request = Request(url="https://example.com", priority=5)
+        print(f"   📤 插入请求: {test_request.url}")
+        
+        success = await queue.put(test_request, priority=5)
+        if success:
+            print("   ✅ 插入成功")
+        else:
+            print("   ❌ 插入失败")
+            return False
+            
+        # 测试队列大小
+        size = await queue.qsize()
+        print(f"   📊 队列大小: {size}")
+        
+        # 测试 get 操作
+        print("   📥 获取请求...")
+        retrieved_request = await queue.get(timeout=2.0)
+        
+        if retrieved_request:
+            print(f"   ✅ 获取成功: {retrieved_request.url}")
+            # 测试 ack
+            await queue.ack(retrieved_request)
+            print("   ✅ ACK 成功")
+        else:
+            print("   ❌ 获取失败（超时）")
+            return False
+            
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 队列操作失败: {e}")
+        traceback.print_exc()
+        return False
+    finally:
+        await queue.close()
+
+
+async def test_serialization():
+    """测试序列化问题"""
+    print("🔍 3. 测试 Request 序列化...")
+    
+    try:
+        import pickle
+        from crawlo.network.request import Request
+        
+        # 创建测试请求
+        request = Request(
+            url="https://example.com",
+            method="GET",
+            headers={"User-Agent": "Test"},
+            meta={"test": "data"},
+            priority=5
+        )
+        
+        # 测试序列化
+        serialized = pickle.dumps(request)
+        print(f"   ✅ 序列化成功，大小: {len(serialized)} bytes")
+        
+        # 测试反序列化
+        deserialized = pickle.loads(serialized)
+        print(f"   ✅ 反序列化成功: {deserialized.url}")
+        
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 序列化失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_concurrent_operations(redis_url):
+    """测试并发操作"""
+    print("🔍 4. 测试并发操作...")
+    
+    async def producer(queue, start_id):
+        """生产者"""
+        try:
+            for i in range(5):
+                request = Request(url=f"https://example{start_id + i}.com", priority=i)
+                await queue.put(request, priority=i)
+                await asyncio.sleep(0.1)
+            print(f"   ✅ 生产者 {start_id} 完成")
+        except Exception as e:
+            print(f"   ❌ 生产者 {start_id} 失败: {e}")
+    
+    async def consumer(queue, consumer_id):
+        """消费者"""
+        consumed = 0
+        try:
+            for _ in range(3):  # 每个消费者处理3个请求
+                request = await queue.get(timeout=5.0)
+                if request:
+                    await queue.ack(request)
+                    consumed += 1
+                    await asyncio.sleep(0.05)
+                else:
+                    break
+            print(f"   ✅ 消费者 {consumer_id} 处理了 {consumed} 个请求")
+        except Exception as e:
+            print(f"   ❌ 消费者 {consumer_id} 失败: {e}")
+    
+    queue = RedisPriorityQueue(
+        redis_url=redis_url,
+        queue_name="test:concurrent:requests"
+    )
+    
+    try:
+        await queue.connect()
+        
+        # 并发运行生产者和消费者
+        tasks = [
+            producer(queue, 0),
+            producer(queue, 10),
+            consumer(queue, 1),
+            consumer(queue, 2),
+        ]
+        
+        await asyncio.gather(*tasks, return_exceptions=True)
+        
+        # 检查剩余队列大小
+        final_size = await queue.qsize()
+        print(f"   📊 最终队列大小: {final_size}")
+        
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 并发测试失败: {e}")
+        return False
+    finally:
+        await queue.close()
+
+
+async def main():
+    """主测试函数"""
+    print("🚀 开始 Redis 分布式队列诊断...")
+    print("=" * 50)
+    
+    try:
+        # 1. 测试连接
+        redis_url = await test_redis_connection()
+        
+        # 2. 测试序列化
+        if not await test_serialization():
+            return
+            
+        # 3. 测试基本操作
+        if not await test_queue_operations(redis_url):
+            return
+            
+        # 4. 测试并发操作
+        if not await test_concurrent_operations(redis_url):
+            return
+            
+        print("=" * 50)
+        print("🎉 所有测试通过！Redis 队列工作正常")
+        
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 诊断失败: {e}")
+        traceback.print_exc()
+        
+        # 提供解决建议
+        print("\n🔧 可能的解决方案:")
+        print("1. 检查 Redis 服务是否启动: redis-server")
+        print("2. 检查 Redis 密码配置")
+        print("3. 检查防火墙和端口 6379")
+        print("4. 安装 Redis: pip install redis")
+        print("5. 检查 Redis 配置文件中的 bind 设置")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/test_request_serialization.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试 Request 序列化问题修复
+"""
+import pickle
+import sys
+sys.path.insert(0, "..")
+
+from crawlo.network.request import Request
+from crawlo.core.scheduler import Scheduler
+from unittest.mock import Mock
+
+# 模拟一个带 logger 的 Request
+class TestRequest(Request):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # 添加一个 logger 属性模拟问题
+        from crawlo.utils.log import get_logger
+        self.logger = get_logger("test_request")
+        self.meta['spider_logger'] = get_logger("spider_logger")
+
+def test_request_serialization():
+    """测试 Request 序列化"""
+    print("🔍 测试 Request 序列化修复...")
+    
+    # 创建一个带 logger 的请求
+    request = TestRequest(
+        url="https://example.com",
+        meta={"test": "data"}  # 移除 Mock 对象
+    )
+    
+    print(f"   📦 原始请求: {request}")
+    print(f"   🔧 请求有 logger: {hasattr(request, 'logger')}")
+    print(f"   🔧 meta 有 logger: {'spider_logger' in request.meta}")
+    
+    # 创建一个 mock scheduler 来测试清理
+    class MockScheduler:
+        def _deep_clean_loggers(self, request):
+            return Scheduler._deep_clean_loggers(self, request)
+        def _remove_logger_from_dict(self, d):
+            return Scheduler._remove_logger_from_dict(self, d)
+    
+    scheduler = MockScheduler()
+    
+    # 执行清理
+    scheduler._deep_clean_loggers(request)
+    
+    print(f"   🧹 清理后有 logger: {hasattr(request, 'logger')}")
+    print(f"   🧹 清理后 meta 有 logger: {'spider_logger' in request.meta}")
+    
+    # 测试序列化
+    try:
+        serialized = pickle.dumps(request)
+        print(f"   ✅ 序列化成功，大小: {len(serialized)} bytes")
+        
+        # 测试反序列化
+        deserialized = pickle.loads(serialized)
+        print(f"   ✅ 反序列化成功: {deserialized}")
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 序列化失败: {e}")
+        return False
+
+if __name__ == "__main__":
+    success = test_request_serialization()
+    if success:
+        print("🎉 Request 序列化修复成功！")
+    else:
+        print("❌ 序列化问题仍未解决")