crawlo 1.4.0__py3-none-any.whl → 1.4.2__py3-none-any.whl

{crawlo-1.4.0.dist-info → crawlo-1.4.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
-crawlo/__init__.py,sha256=rCeDq1OoX6mmcBxuK60eUpEp1cIg5T8Zgic3FUQAOkA,2318
-crawlo/__version__.py,sha256=EyMGX1ADFzN6XVXHWbJUtKPONYKeFkvWoKIFPDDB2I8,22
+crawlo/__init__.py,sha256=WD-F_HxUb9ad2v7J_yipTK4wwENCzIYwa7AF14eL4sg,2288
+crawlo/__version__.py,sha256=Gom1_ABKKkZoIghfDiYvSntLG2OKXC64sJD2Xm8bUKc,22
 crawlo/cli.py,sha256=OXprmcTUbFK02ptw_Gq8Gk4-ZCU-WEMJgzU1ztgP6Bk,2327
 crawlo/config.py,sha256=dNoNyTkXLe2msQ7bZx3YTQItk1m49nIg5-g89FQDNwE,9486
 crawlo/config_validator.py,sha256=gsiLqf5swWd9ISDvoLqCdG7iSXr-ZdBPD4iT6ug1ua4,11239
@@ -7,8 +7,8 @@ crawlo/crawler.py,sha256=E83JhClOe58XVX1ma0f-HAF1BJ7Ej9Zs0w51ERs3fgA,26348
 crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
 crawlo/exceptions.py,sha256=sMay0wnWLfc_FXWslqxm60qz6b66LXs3EdN_w8ygE9k,1166
 crawlo/framework.py,sha256=1RVBwj_VBzfJiMB3lq6XcfFHCjRBHyT4D_T2X4fU_6g,9166
-crawlo/mode_manager.py,sha256=JP8_jkH2p9LMg1-g1e05PhSggSvt4jO_oO2h51pLVYQ,7399
-crawlo/project.py,sha256=DooXmO0nmcHPVRsnDBTE0dOrX-KOqnJe6A0s_-qOxRI,12147
+crawlo/mode_manager.py,sha256=j5Kez-KwZQWbEWeYlyUZov18kk8ZRQFYEBUzbkIc-DY,7549
+crawlo/project.py,sha256=LofpNMP1D5AHMe987jfgkSFTjRmrfF5zUBkXl6Cvp3U,12730
 crawlo/stats_collector.py,sha256=copzmfWTArYZCkMeZJsJfJcdC36s7_LM88hxAYttoeE,2306
 crawlo/subscriber.py,sha256=D3hzE7Pc_zJjc-zR7lct5pt32bz6LsDYeC8uHlS4Hso,4986
 crawlo/task_manager.py,sha256=a9JWpqiozFEhReH4PwD9HsDs050HERwi9X9LNGdOp0E,5828
@@ -21,14 +21,14 @@ crawlo/commands/run.py,sha256=gQ14PN3ZxsRNapRsyGZ4qdhbqzh70EnuS2YPaIUA8q0,12828
 crawlo/commands/startproject.py,sha256=aqKRJarKqTf5XjJnGXwjRpp0uYF16LreFbwwQLGpK-0,16070
 crawlo/commands/stats.py,sha256=8wTubR1RQ1JPTlpOKArcGcQ39bM-0cuH27lDpndnwPQ,6014
 crawlo/commands/utils.py,sha256=Psfu2tKrmDloMq0WnfXLaxx0lJFitMZ-FWS3HAIrziQ,5382
-crawlo/core/__init__.py,sha256=nikMDqFwnDfE8ugqwAIfycBtIqIVZpeprjEYW-H5Dkw,1272
+crawlo/core/__init__.py,sha256=Mcf4BKnjz-AP72eLCSh5ryLoZchxJnr7Jqv0W_OpkiI,1278
 crawlo/core/engine.py,sha256=0l7TVNf2R8EHJAZ4ktj71j-qysrq84cYqf_7LEzzYJM,19096
 crawlo/core/processor.py,sha256=wO6DMU-Azr0yaMLJw8LSTG19a6ZAvPuT3J7wNLfbpu4,1577
-crawlo/core/scheduler.py,sha256=By1JB0iukcss5j0nrj1rq1Lk-VmmUHIiGl0RLCH9YUs,12630
+crawlo/core/scheduler.py,sha256=C2V23SIm1VGdi3A41TThB3zE99QFxTnsJp4SDOvSaBE,12561
 crawlo/data/__init__.py,sha256=8MgDxcMhx-emFARcLAw_ODOZNz0neYBcx7kEbzothd8,116
 crawlo/data/user_agents.py,sha256=6V34lYHREWV5ZR5wH-1pCnr1Y3ZYC7iMLfC6vZHyhZQ,9697
 crawlo/downloader/__init__.py,sha256=PB8oluLFMX2PBmeb3NBKkM6GaceX0ujFId8t2URy1ks,8624
-crawlo/downloader/aiohttp_downloader.py,sha256=rkCgEfX_s7w-cRK2ZoX43Unt9C7pPPYP64q22ShJMso,9107
+crawlo/downloader/aiohttp_downloader.py,sha256=nlsq2CE-NcmT7nsNEENWC66YlWDx_JzS-eA-P2cvw_Q,9323
 crawlo/downloader/cffi_downloader.py,sha256=BpA1q6Udz7sSXJ0gX94xGnzy8cdgK-vlr_Q6YA4QIxE,10243
 crawlo/downloader/httpx_downloader.py,sha256=7jfQfvAtfk8yD_mvwUbWLhYOxMM7r1nudiU7m_Jl9wc,12037
 crawlo/downloader/hybrid_downloader.py,sha256=4SzOPEwBlSZVzUAWR3DyxMx2Tsx15YrpBvQS4it4Vps,8028
@@ -36,7 +36,7 @@ crawlo/downloader/playwright_downloader.py,sha256=Lnc7k5cXhVnURXSxgZFCYMJkBxLg5F
 crawlo/downloader/selenium_downloader.py,sha256=B_0muNi-GQ_hgoYHcf7wgu01V68q7xKnSh-0kzlUiio,21036
 crawlo/extension/__init__.py,sha256=7HxWQKBuiVphZUBLIBVCtIjgFIbzTa5QDOQp6WH4HhU,2923
 crawlo/extension/health_check.py,sha256=0GveZgUtFwjYEKlm3qbwIvCmb4FR0qrIKc8cEF1yQV8,5516
-crawlo/extension/log_interval.py,sha256=VCIeNqXcWDnxj4m6l77cjqgRzV8LfsPMb22X0Xc1Vwc,2417
+crawlo/extension/log_interval.py,sha256=gWCcUCvMzpmx1IE_HvUGAvhfUIqd-UjARNNQLqvjyow,4403
 crawlo/extension/log_stats.py,sha256=X90Y_E6USAdm55yvRN5t59HNLmyN9QMKglhbPPxtehA,2382
 crawlo/extension/logging_extension.py,sha256=8KT-WJRK5tocS2kBOiSquree53L11qD1vLg-P8ob40U,2354
 crawlo/extension/memory_monitor.py,sha256=4aszl3C0GMQbqFhGZjZq5iQuXQR1sOz06VHjjEHgkyE,4290
@@ -49,7 +49,7 @@ crawlo/factories/registry.py,sha256=LrtH7wMGQ2ZrswxnHDM9s43ckJ1isJKL7R8uyMQ8hCc,
 crawlo/filters/__init__.py,sha256=XC_Q4ykZtSNYizYlAcehVwBBNO3lZ2zuWwafzXiuWyQ,4241
 crawlo/filters/aioredis_filter.py,sha256=ZEApX23-S-7ruO_TSTKI0Noh8SEwjdZznf8TySeEtbQ,9255
 crawlo/filters/memory_filter.py,sha256=mO4oBPV5_uAiBQF3a16tU5tcD8244dOjKoNX_MU8cEw,9292
-crawlo/initialization/__init__.py,sha256=Y7pLI5MDiAgG1NuRvkIhwzaE6438_2ZP5jK317utgUQ,1084
+crawlo/initialization/__init__.py,sha256=ML9fKhttJC3TRAtp5n9MTevLBH-vdPUXwKihQdxEUlE,1088
 crawlo/initialization/built_in.py,sha256=EkZIPBrqsvFf0CuBL6POk2IJiDFf8q30eRGMqcL2N8M,15333
 crawlo/initialization/context.py,sha256=SL2ge47EmyLHzB5yldISA-xr__ZOV1xnQP_-1RF5v0Y,4722
 crawlo/initialization/core.py,sha256=ZR6veRDkKU5erQGXGKzgX6TU2_i6YkWsuFUeWnOEVjo,6679
@@ -66,7 +66,7 @@ crawlo/logging/manager.py,sha256=hbdwyFGnnyRFrVDXoqzHs8oERx72NHrf7KqwCf4oPc4,307
 crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0,549
 crawlo/middleware/default_header.py,sha256=wQ7BrUHd-hRosFoKsReV9hwNNr_jwK6V0ZfxL6MOGrk,5032
 crawlo/middleware/download_delay.py,sha256=zt9R5g2HWErWA_MAOnGcw_D8l6HD769Kyaw-Hv-vcTc,3438
-crawlo/middleware/middleware_manager.py,sha256=bQuIxn-i2oud-0hDkv890sa3YvNMbuJIR8zuAmIdLKA,6289
+crawlo/middleware/middleware_manager.py,sha256=1-qeqlyjBDvLhojt8aOT-EASe9gCcwzJ2q53iRagOHI,6280
 crawlo/middleware/offsite.py,sha256=FIWZvkkzlDJfvQc7Ud7BdfDZ78Sa85qlEEwAR76hSBk,4559
 crawlo/middleware/proxy.py,sha256=NquB6tqHAgHs3-2_1_5220kJYfjNG5JyHRJyo_2j4wo,15636
 crawlo/middleware/request_ignore.py,sha256=xcyZ1c7r_HhbzR3r9pfjsLGW7L7FBVeYvlNt8cpP2wY,2577
@@ -78,23 +78,23 @@ crawlo/network/__init__.py,sha256=BLPERYPo22g1BXrW--wUnlolrdFUmOPjgOB8XQQJlck,39
 crawlo/network/request.py,sha256=9kV-gqb_d6aCsSBAwyzxnP9a70cAViwX8qvpyYV7Ym4,13799
 crawlo/network/response.py,sha256=EZiG4LjuIb7PxdGou4H-oSOQhec1ZdBRTkO-5fl8JTo,12701
 crawlo/pipelines/__init__.py,sha256=lrdVDjeHLNkA4_MAwI1auk_I9xfeU1SlBWXiammb6lc,616
-crawlo/pipelines/bloom_dedup_pipeline.py,sha256=AIejBxWpRWg2SSYuMPmPy2VrG_FhFmjXYaAcRhuF7h0,5677
+crawlo/pipelines/bloom_dedup_pipeline.py,sha256=yNs7ON4icJK0nDGpDyZzghbPyf350bMEi1gOn4r_ya8,5317
 crawlo/pipelines/console_pipeline.py,sha256=KABkR3J-rqO0Awox7lizxKR2XuHfVhWPiVRgIybwwu4,1248
 crawlo/pipelines/csv_pipeline.py,sha256=6FBT2AoU6iNU-5NfgWRq7-JpF9dK2nBokjxx-y4jIas,12174
-crawlo/pipelines/database_dedup_pipeline.py,sha256=qQ6w0eoGcUB26VO1lgI8GFhVjERG-GZhg5UCTYKyovg,7135
+crawlo/pipelines/database_dedup_pipeline.py,sha256=EfX_y9JiqD5EissBZNWfDYJavMcCYQ72swzfe7RsNZc,7099
 crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
-crawlo/pipelines/memory_dedup_pipeline.py,sha256=9KuUA1S0uHWSB3qJntPdg9ifPdRXwc8ju4j9tWe8qTo,3853
+crawlo/pipelines/memory_dedup_pipeline.py,sha256=sHoihpQiMZ7teiH4Ctl8OIPJs1x1bRRgzdy89JXFm7U,3493
 crawlo/pipelines/mongo_pipeline.py,sha256=k7gNqAO-g2MtIfArphC6z5ZzkKVRkBKcv-2ImziPFA0,5706
 crawlo/pipelines/mysql_pipeline.py,sha256=_oRfIvlEiOsTKkr4v-yPTcL8nG9O9coRmke2ZSkkKII,13871
 crawlo/pipelines/pipeline_manager.py,sha256=BX17CU9JK2xJeIdzQ4FeK7kwpwew1k-BEVMk9oviqTQ,3682
-crawlo/pipelines/redis_dedup_pipeline.py,sha256=6fkHt7O-R2TTMlULgxyqPdyKBjsRzYh_GL-Juye4ZQ0,6410
+crawlo/pipelines/redis_dedup_pipeline.py,sha256=zNk7WJxtgWNfRzKnyxOM2OyfsKhPNsOmy_8IGT1iZi0,5921
 crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 crawlo/queue/pqueue.py,sha256=j2ISmyays5t1tuI36xM6EcELwSpq2xIjAScSBWSRZms,1220
 crawlo/queue/queue_manager.py,sha256=JfkjtOD04e_OZZvEEvp3O_W3lfGXhHslZHrCgw90amY,20693
 crawlo/queue/redis_priority_queue.py,sha256=Evmo514OFL0a7Xu2SdCiz6klFUGH1gmjlxCc01vX1tQ,15400
 crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
-crawlo/settings/default_settings.py,sha256=LReJ11Wm8wh1z2GHgGRBYH6Tpq6_CkRd8QZ4dKRCqI0,13220
-crawlo/settings/setting_manager.py,sha256=AWPvvhOGo04Yv_q3jqEMyzhEpbxOX_Wr8tSHmI2sUnA,8109
+crawlo/settings/default_settings.py,sha256=BBJkTYe2ViIuGzr6KtWnJRDe0Yj6dkcBo56VxDeJIkE,12552
+crawlo/settings/setting_manager.py,sha256=hfmZfRcCQjTCSFWOelrzsIjLBIGDPmy_vlWDm6rhHeM,8376
 crawlo/spider/__init__.py,sha256=oi9LEYq9xaCSjktAIRUgjpGQQI7rTtN61ESdHeWb1x4,21224
 crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
 crawlo/templates/run.py.tmpl,sha256=j8xRI4IEI_YiDFvMyfYG0XD2GgBM9eXgSqpvBniferU,927
@@ -117,7 +117,7 @@ crawlo/tools/authenticated_proxy.py,sha256=L93WeXajIZ3si8xNcE7bBPv34FvqSyTvVfD78
 crawlo/tools/data_formatter.py,sha256=NEj3NqPiyG67V6qDgn2KNj9VNHWOLNwX-7p_nad0znc,7583
 crawlo/tools/data_validator.py,sha256=hxPN28YtJDFFLjBBYhDjHmR8ShNTEjgIsv-cmcDKIu8,5310
 crawlo/tools/date_tools.py,sha256=jjP5xA0-aDgm9UIK1RG2qaNagBzHFQ-BBDMo_YzSlLQ,8906
-crawlo/tools/distributed_coordinator.py,sha256=Au20nZ4qUiAZUD2A1yfwD3soaHADpkEZt1hRyegp6M4,12323
+crawlo/tools/distributed_coordinator.py,sha256=eUAQudIaz96WGIPVJxF_W0BOKDpRBX3ZmPLsDmiBKXM,12190
 crawlo/tools/encoding_converter.py,sha256=7P9Z7J1ALw_PPNApmjFsHZDpRxgxzduiViluenlSLEU,4043
 crawlo/tools/network_diagnostic.py,sha256=92diB7Ppo_TKGDYCRLzy7uvQMGApgGLwv7P5w4OpCms,12649
 crawlo/tools/request_tools.py,sha256=CjyFBtRQf_vFjQhaVwgHSGai4ZaWS8IIaF1flSfJxDs,2338
@@ -132,6 +132,7 @@ crawlo/utils/db_helper.py,sha256=3ib5-agrlwf2t5S_QtLRYH75wvJDlYbRqRmDEbpH5Bo,105
 crawlo/utils/enhanced_error_handler.py,sha256=hj5AElt3ajfqnP4csQnEfEnzkbIep9k65DNQiCbmTFo,13858
 crawlo/utils/env_config.py,sha256=HbZOEKkeQ0FMdZYJu9SgmSNEmfPJrmAzA7lHu5Du1DA,3937
 crawlo/utils/error_handler.py,sha256=nDfDA99q2sirE2pe7OT2bcA54GqUiAYgtdAh38uFEX4,5290
+crawlo/utils/fingerprint.py,sha256=5XTH8ma0HSHc8arvre_sVv2g0Unxte1KLScmf3bHPDA,3565
 crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
 crawlo/utils/large_scale_config.py,sha256=j7wQ5ty7pQlBRygw2vhRJ7OI19RYBZKPfYMP3WeF2WI,8154
 crawlo/utils/large_scale_helper.py,sha256=Kxdy3WMuqjzQTyCc6z4xEYxXDi4xnYKJzsVwaBYZrrg,12108
@@ -147,6 +148,15 @@ crawlo/utils/system.py,sha256=HvWV1acxou0Rn0L7pNq4CnV_GWFeU0Tmjy3_nLD8M64,237
 crawlo/utils/tools.py,sha256=5Uv25Wy4m_ndZY0-n-eX-t3PxvaZ6wR3-Wvx-o7_Vrs,271
 crawlo/utils/url.py,sha256=rlgX2VlJv6JvLmCDTsbxzMSXE6R5ZL_0dLEqprsA-JU,1482
 examples/__init__.py,sha256=6i631BPnS_TR_BWUjtjB5CBO-zv9kRkwQTQvSya2wHE,123
+examples/test_project/__init__.py,sha256=sBO0-NPkGiKKO44U9OYknIdkPGKgJGn7ne2M7v_YOSU,129
+examples/test_project/run.py,sha256=5kADlYJYVp2seoxM5KPCX8qwSB-YzyLf-SGm66FcvyQ,919
+examples/test_project/test_project/__init__.py,sha256=-8hHXHKxttgUB58mL95faujZIsubNXuA_0-VzU-Mmy8,54
+examples/test_project/test_project/items.py,sha256=f4GN_2OChAdDI0zunbWIA6dpP7ou6ySt4mY98GGZzos,293
+examples/test_project/test_project/middlewares.py,sha256=qh_UvYaxH_mK1h8oNL405NQ2SYLEWAKs7r9Fyw-Jw5M,4152
+examples/test_project/test_project/pipelines.py,sha256=k1Z4M5StwGF4eHW8JoLCNLSVi57FiAxRRO-ABFynJ1k,2600
+examples/test_project/test_project/settings.py,sha256=Aidvz43WZYu7LAJ3N9gXdpddWCP_656kIJGJMe7J4NA,6946
+examples/test_project/test_project/spiders/__init__.py,sha256=tWZ6jEMrDw1KoRBw12F54srI1-YX7RNYe0dtU4OP0aQ,203
+examples/test_project/test_project/spiders/of_week_dis.py,sha256=zKulqu_lc7NS0ewDh_MCttwpXMBp0P2fPostkxxM3Xc,5035
 tests/__init__.py,sha256=scL1IPVT1iucth7v8ffrjRdeW7QrC_Y7AMmFVMdTY1Y,129
 tests/advanced_tools_example.py,sha256=7nlFLRVMVYzDz_CAdgQa9fJu7o0M6jBMo7PTvUsRbo0,9065
 tests/authenticated_proxy_example.py,sha256=rsLmILsrf9PpR77ekGi8KpB1dAYZdF26hlxkBjm4rSQ,2913
@@ -160,9 +170,9 @@ tests/config_validation_demo.py,sha256=5MzW5P7ZX6xoMW_zC6XmIA50KWMTu0iB5H2hTe42S
 tests/controlled_spider_example.py,sha256=SP_k4mdKPvD1JCPs9UCm68jcy2Frg84vvXv9-14RC6I,7776
 tests/date_tools_example.py,sha256=x_-duqnVZ-Hrk-SaNplIfcIV6W3c6u6MTxW35u1i0F0,4862
 tests/debug_configure.py,sha256=E-6Djz8kk7tf2pzEqrGdekW2W20vrJeZN7iNm9ArWKk,2144
-tests/debug_framework_logger.py,sha256=l2OX6igGu-pCUGrlwdWqcenqSSK9wMDheZ47XhEUqPg,3341
+tests/debug_framework_logger.py,sha256=lAxT1esheIyNtgjXf6VJj8Lv0TNw5KUHdjaKHTYrF7Q,3329
 tests/debug_log_config.py,sha256=F5MrUmpkaD9Dc1eSbajaOrw3nKo9Sp55BTnNxmQkUWc,3588
-tests/debug_log_levels.py,sha256=yPyKRNwz9kNWU1QMVLRD989Wh2sb6CrH4GAsMO0PHW8,2117
+tests/debug_log_levels.py,sha256=39ugttR13gfotsS3I4XVoWRRjTXPpLtuJOyU8V_ijEs,2105
 tests/debug_pipelines.py,sha256=VpUmoYlt6Ci7foIGuQIotUu42xp6TzoA1cBDeagBzDk,2098
 tests/detailed_log_test.py,sha256=M6yXjQypKmIjihgTSBffkgOzC7Nn0_4ZdQLrBN-L8i0,7268
 tests/distributed_test.py,sha256=qZpFAMQTFcg0KUEdp2RUpkuYauSCf4C3lbbosyIDqgw,1759
@@ -197,6 +207,7 @@ tests/simple_test.py,sha256=kzMspCmfJxdnAIXXJv9tmDW1gpodkD9pznW5vA_gL84,1211
 tests/spider_log_timing_test.py,sha256=ngZQ_v3o9oHYcs_BtZgxH1N-N2tZUDPu-cnTnsHEpP8,5396
 tests/test_advanced_tools.py,sha256=3R8EfKVyBHEb6FA5TP3ieaWeHZhobVgSx8t3phipCrE,5250
 tests/test_all_commands.py,sha256=yGPw8zMrB5Z5w5LkaymSzKRLOcZsBPBXLvllCkgEY4I,7488
+tests/test_all_pipeline_fingerprints.py,sha256=4dtn-APwK2JIg-c4J-7u9J1PF9L_pdMh4CmIyVP266I,5175
 tests/test_all_redis_key_configs.py,sha256=SGoip8M7oB2LNWC_31aJ4ECcDRmx0psr7i7DGzuaH7c,5565
 tests/test_authenticated_proxy.py,sha256=s4pr5JyBTHYQgRq_IymiVKE08vyW1MwR27pSwrrVLVk,4198
 tests/test_batch_processor.py,sha256=gMPoQcnUMm2-G_d7Zt9QnrRjCx1urzT31tYqoFNEklc,7034
@@ -210,7 +221,7 @@ tests/test_controlled_spider_mixin.py,sha256=7t6VGWr6Hxw0xtIFyToLH8_deSagUtsdqSJ
 tests/test_crawlo_proxy_integration.py,sha256=SvdBuZjS6N2vuvFkTnc59U5n3dHV3E4dmFayxtmjCm4,2625
 tests/test_date_tools.py,sha256=CQdAmIS6bpAdwQH9ETDH__06l2gGL7EHUQuh7mdTF-A,3930
 tests/test_dedup_fix.py,sha256=9rFzzsDJKQbFaohzKJAlqo3Mm4wFy8-wAm3fWmw8jb4,8568
-tests/test_default_header_middleware.py,sha256=7kpONSsGMsmWgTX2pCpseme54_-82Baak0xVz6gclJk,5845
+tests/test_default_header_middleware.py,sha256=5zYSZmUR_02UChFZfoCqurabNdAZLy7t4yNZ8I7Bw8E,12813
 tests/test_distributed.py,sha256=RQHUpDfRNG2x_1Cdr9DLk25IBcgapm_u0xSBMObE0Xc,1725
 tests/test_double_crawlo_fix.py,sha256=E5NxWHnQkwRTIrJGoag8G29fZqVMnsN6eCPuv17gGq0,7652
 tests/test_double_crawlo_fix_simple.py,sha256=MlWUqo51kOQ7Gu6Neoler8FVyRs0jpmQWoORHMBENz0,4644
@@ -227,21 +238,26 @@ tests/test_env_config.py,sha256=nfP4nCG9ZHeJUfxo1JKUmiihYbhSeWx_oNW5mMfDHfQ,4746
 tests/test_error_handler_compatibility.py,sha256=o5JLLLdo25Sl_3hpMx6I2fqSgZFAcnI4E6Ci-KxAxwA,4129
 tests/test_factories.py,sha256=vXI8tx42iuBivCKQoY2kH7G6c0i_QCmCq77krEgQiGU,8613
 tests/test_final_validation.py,sha256=aAiWLzhDCcv-GEXg9sauaVIfq5rz3s2vm67Gk2_lmBI,4813
+tests/test_fingerprint_consistency.py,sha256=LI0hDokBtW5OJ6kwn9-x1WiiS_euBqXGNsD1iwngBhQ,4821
+tests/test_fingerprint_simple.py,sha256=H-8C-ae8TDx29BY9xQGQzZXbpk6szj-kWK8zknR1jwk,1575
 tests/test_framework_env_usage.py,sha256=HYpTwORXeaJHMffCYAGHGvc_a6ax4lo28xP8BYOaKxk,4098
-tests/test_framework_logger.py,sha256=HNkOlyA-dQKEdE6H4VaUHfF3aeVkKRoISSr53Hw90qQ,2506
-tests/test_framework_startup.py,sha256=I0zUfJUjkM7JgUBChO2w9cIL-tDJwUHdzKm3QjuEEJM,2215
+tests/test_framework_logger.py,sha256=RBwf0aOV1TibMqOajg-KYiq_Oe7gxLfvRAurpyY4rnI,2494
+tests/test_framework_startup.py,sha256=zvpzWwozMxGaCfdEER9A2qJuMYytW5mg3q4T86mJ6to,2203
 tests/test_get_component_logger.py,sha256=f7nmmSGxoD1i3d17FlSicOmMGLTcyJxcujoS5eJFbAI,2202
+tests/test_hash_performance.py,sha256=pHJpxF09JCCxk_32hRGXx221ySQROCx2OK7Yk6-B1bk,3112
 tests/test_integration.py,sha256=OCkjyv76Wop7CrXEko6rfoDsIK6SESA18KgCaTwL7Q4,4670
 tests/test_item_dedup_redis_key.py,sha256=QxLuXHUx0xqT6y7lQzOWcrLkRui7Qs7C6NgRvjzIypA,3720
 tests/test_large_scale_config.py,sha256=wyeMOMjGYhbZ6mrcnLH3Eh6GfspJwhavwWoyOy1y90c,4184
 tests/test_large_scale_helper.py,sha256=spvL0MPyXMAUDpzI2fY6-OQdSxOHtgJ1yuSUIbydyHY,8136
 tests/test_logging_system.py,sha256=_LRdgprZFrChA26JJgkjVyf6S6qRIyi6BRajK13l_Q8,8924
-tests/test_mode_change.py,sha256=kh5C4ut7T5dZ8b2dDot4RbLWMXJidv4FHzuTIgDxMBI,2605
+tests/test_mode_change.py,sha256=zNVdmvFOeKD6LMVnCwjtrIyhTw95n1ZQJPjNhOanUHw,2593
 tests/test_mode_consistency.py,sha256=YJXf0SqAYVnFXy8eeBLC-zGTFAyO2fnsR4qLB76gZts,1225
-tests/test_offsite_middleware.py,sha256=L5YT9ZqcQwBunUv0Ddj-sLZcW4IMlAlgaJCwICHFWxI,7543
+tests/test_offsite_middleware.py,sha256=K5NREsWwpJda4OWgf9J4NXIqpbG9jNlgpbALevuGLzc,10358
+tests/test_offsite_middleware_simple.py,sha256=CVBS7dvjpJ_fWBae7t9GtAQF_etn0U_cZ4UQFdu8eh8,7715
 tests/test_parsel.py,sha256=KYskaN_4HBc1XDTltjVo12v1i7JAThB2UIwcWZ-mwbY,672
 tests/test_performance.py,sha256=gOJ1EpU9uGynIxETLAroe98OA4QPcX1wchCDJoO41Kc,11130
 tests/test_performance_monitor.py,sha256=5oEHPJfjZXdtDK2nW_2MuGbOFgTTZyEhLapV9Ug1iHY,4072
+tests/test_pipeline_fingerprint_consistency.py,sha256=WV-1UC6PSXb43B5ATyRDSjW9FcHSBMmHDCFFJwQqE_s,2761
 tests/test_proxy_api.py,sha256=dVqGElyL3K0_9IqkXzn7Ka2jSuhvYfR1BfZgyVukNM0,10749
 tests/test_proxy_health_check.py,sha256=xo3QMP1YNw9hu7JDHZOYCUZmFFKLJpHSh4SbxXhCRPQ,1091
 tests/test_proxy_middleware.py,sha256=qm2B0lepBZqzUpXNi4t1gjrQxUV4MQ2wvpmcaYV6O5A,3900
@@ -286,8 +302,8 @@ tests/verify_distributed.py,sha256=krnYYA5Qx9xXDMWc9YF5DxPSplGvawDg2n0l-3CAqoM,3
 tests/verify_log_fix.py,sha256=TD7M1R22NxLqQPufvgE-H33u9tUjyz-rSR2ayIXozRU,4225
 tests/scrapy_comparison/ofweek_scrapy.py,sha256=2Hvpi6DRTubUxBy6RyJApQxMQONPLc1zWjKTQO_i5U4,5652
 tests/scrapy_comparison/scrapy_test.py,sha256=5sw7jOHhaTmQ8bsUd1TiolAUTRQYQOe-f49HPfysqbI,5466
-crawlo-1.4.0.dist-info/METADATA,sha256=WIambmjeZxudgM_Ej7lv8CKd1EyoQ4f-Z4CugfsHTkY,33235
-crawlo-1.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.4.0.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.4.0.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.4.0.dist-info/RECORD,,
+crawlo-1.4.2.dist-info/METADATA,sha256=IwT9XfhtTQOUvMyKss9cne9PjX5RuWpiOkwMVu6zV6c,33235
+crawlo-1.4.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.4.2.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.4.2.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.4.2.dist-info/RECORD,,

examples/test_project/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-09-27 23:36
+# @Author  :   crawl-coder
+# @Desc    :   None
+"""

examples/test_project/run.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# -*- coding: UTF-8 -*-
+"""
+test_project 项目运行脚本
+============================
+基于 Crawlo 框架的简化爬虫启动器。
+
+框架会自动处理爬虫模块的导入和注册，用户无需手动导入。
+只需指定spider_modules参数，框架会自动扫描并导入所有爬虫。
+"""
+import sys
+import asyncio
+
+from crawlo.crawler import CrawlerProcess
+
+
+def main():
+    """主函数：运行爬虫"""
+    try:
+        # 指定爬虫模块路径，框架会自动导入并注册所有爬虫
+        spider_modules = ['test_project.spiders']
+        process = CrawlerProcess(spider_modules=spider_modules)
+
+        # TODO 运行指定的爬虫
+        asyncio.run(process.crawl('of_week_dis'))
+
+    except Exception as e:
+        print(f"❌ 运行失败: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

examples/test_project/test_project/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: UTF-8 -*-
+"""
+test_project 项目包
+"""

examples/test_project/test_project/items.py

@@ -0,0 +1,18 @@
+# -*- coding: UTF-8 -*-
+"""
+test_project.items
+======================
+定义你抓取的数据结构。
+"""
+
+from crawlo.items import Item, Field
+
+
+class ExampleItem(Item):
+    """
+    一个示例数据项。
+    """
+    id = Field()
+    # price = Field()
+    # description = Field()
+    pass

examples/test_project/test_project/middlewares.py

@@ -0,0 +1,119 @@
+# -*- coding: UTF-8 -*-
+"""
+test_project.middlewares
+============================
+自定义中间件，用于在请求/响应/异常处理过程中插入自定义逻辑。
+
+这是一个简单的示例中间件，您可以根据需要添加更多中间件。
+"""
+
+import random
+from crawlo import Request, Response
+from crawlo.utils.log import get_logger
+
+
+class ExampleMiddleware:
+    """
+    示例中间件，演示如何处理请求、响应和异常。
+    
+    此中间件会：
+    1. 为请求添加随机 User-Agent
+    2. 记录请求和响应信息
+    3. 处理异常情况
+    """
+    
+    def __init__(self):
+        self.logger = get_logger(self.__class__.__name__)
+        self.user_agents = [
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36',
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0',
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:135.0) Gecko/20100101 Firefox/135.0',
+        ]
+
+    def process_request(self, request, spider):
+        """
+        在请求被下载器执行前调用。
+        
+        Args:
+            request: 请求对象
+            spider: 爬虫实例
+            
+        Returns:
+            None: 继续处理请求
+            Response: 返回响应对象（短路处理）
+            Request: 返回新请求对象（替换原请求）
+        """
+        # 为请求添加随机 User-Agent
+        if 'User-Agent' not in request.headers:
+            ua = random.choice(self.user_agents)
+            request.headers['User-Agent'] = ua
+            self.logger.debug(f"为请求 {request.url} 设置 User-Agent: {ua[:50]}...")
+        
+        return None
+
+    def process_response(self, request, response, spider):
+        """
+        在响应被 Spider 处理前调用。
+        
+        Args:
+            request: 原始请求对象
+            response: 响应对象
+            spider: 爬虫实例
+            
+        Returns:
+            Response: 处理后的响应对象
+        """
+        # 记录响应信息
+        self.logger.info(f"收到响应: {request.url} - 状态码: {response.status_code}")
+        
+        # 可以在这里处理特殊状态码
+        if response.status_code == 403:
+            self.logger.warning(f"访问被拒绝: {request.url}")
+        
+        return response
+
+    def process_exception(self, request, exception, spider):
+        """
+        在下载或处理过程中发生异常时调用。
+        
+        Args:
+            request: 请求对象
+            exception: 异常对象
+            spider: 爬虫实例
+            
+        Returns:
+            None: 异常将继续传播
+            Response: 返回响应对象（处理异常）
+            Request: 返回新请求对象（重试请求）
+        """
+        self.logger.error(f"请求异常: {request.url} - {exception}")
+        return None
+
+
+# ======================== 使用说明 ========================
+# 
+# 在 settings.py 中启用中间件：
+# MIDDLEWARES = [
+#     'test_project.middlewares.ExampleMiddleware',
+# ]
+# 
+# 您可以根据需要添加更多中间件，例如：
+# 1. 请求处理中间件（修改请求头、设置代理等）
+# 2. 响应处理中间件（解析、过滤等）
+# 3. 异常处理中间件（重试、记录等）
+# 
+# 每个中间件可以实现以下方法：
+# - process_request: 处理请求
+# - process_response: 处理响应
+# - process_exception: 处理异常
+# 
+# 注意：Crawlo框架提供了许多内置中间件，您可以直接使用：
+# - DownloadDelayMiddleware: 控制请求延迟
+# - ResponseCodeMiddleware: 处理HTTP状态码并记录统计信息
+# - ResponseFilterMiddleware: 过滤特定状态码的响应
+# - DefaultHeaderMiddleware: 添加默认请求头
+# - ProxyMiddleware: 设置代理
+# - RetryMiddleware: 处理重试逻辑
+# - OffsiteMiddleware: 过滤站外请求
+# ======================== 使用说明 ========================

examples/test_project/test_project/pipelines.py

@@ -0,0 +1,97 @@
+# -*- coding: UTF-8 -*-
+"""
+test_project.pipelines
+==========================
+数据管道，用于处理 Spider 返回的 Item。
+例如：清理、验证、去重、保存到数据库等。
+
+这是一个简单的示例管道，您可以根据需要添加更多管道。
+"""
+
+from datetime import datetime
+from crawlo.exceptions import DropItem
+from crawlo.utils.log import get_logger
+
+
+class ExamplePipeline:
+    """
+    示例管道，演示如何处理数据项。
+    
+    此管道会：
+    1. 验证必要字段
+    2. 清理数据
+    3. 添加时间戳
+    4. 记录处理日志
+    """
+    
+    def __init__(self):
+        self.logger = get_logger(self.__class__.__name__)
+        self.item_count = 0
+
+    def process_item(self, item, spider):
+        """
+        处理数据项。
+        
+        Args:
+            item: 要处理的数据项
+            spider: 爬虫实例
+            
+        Returns:
+            处理后的数据项
+            
+        Raises:
+            DropItem: 如果数据项无效则抛出此异常
+        """
+        # 验证必要字段
+        if not item.get('title') or not item.get('url'):
+            raise DropItem("缺少必要字段: title 或 url")
+        
+        # 数据清理
+        item['title'] = str(item['title']).strip()
+        
+        # 添加处理时间戳
+        item['processed_at'] = datetime.now().isoformat()
+        
+        # 计数器
+        self.item_count += 1
+        
+        # 记录日志
+        self.logger.info(f"处理第 {self.item_count} 个数据项: {item['title']}")
+        
+        return item
+
+    def open_spider(self, spider):
+        """
+        爬虫启动时调用。
+        
+        Args:
+            spider: 爬虫实例
+        """
+        self.logger.info(f"管道已启动，准备处理爬虫 '{spider.name}' 的数据")
+
+    def close_spider(self, spider):
+        """
+        爬虫关闭时调用。
+        
+        Args:
+            spider: 爬虫实例
+        """
+        self.logger.info(f"管道已关闭，共处理了 {self.item_count} 个数据项")
+
+
+# ======================== 使用说明 ========================
+# 
+# 在 settings.py 中启用管道：
+# PIPELINES = [
+#     'test_project.pipelines.ExamplePipeline',
+# ]
+# 
+# 您可以根据需要添加更多管道，例如：
+# 1. 数据验证管道
+# 2. 去重管道
+# 3. 数据存储管道（数据库、文件等）
+# 4. 数据转换管道
+# 
+# 每个管道都应该实现 process_item 方法，
+# 可选实现 open_spider 和 close_spider 方法。
+# ======================== 使用说明 ========================

examples/test_project/test_project/settings.py

@@ -0,0 +1,170 @@
+# -*- coding: UTF-8 -*-
+"""
+test_project 项目配置文件（分布式版）
+=============================
+基于 Crawlo 框架的分布式爬虫项目配置。
+适合大规模数据采集和多节点部署。
+"""
+
+import os
+
+# ============================== 项目基本信息 ==============================
+PROJECT_NAME = 'test_project'
+
+# ============================== 运行模式 ==============================
+RUN_MODE = 'distributed'
+
+# ============================== 并发配置 ==============================
+CONCURRENCY = 16
+MAX_RUNNING_SPIDERS = 5
+DOWNLOAD_DELAY = 1.0
+
+# ============================== 下载器配置 ==============================
+# 可选下载器:
+# DOWNLOADER = 'crawlo.downloader.aiohttp_downloader.AioHttpDownloader'
+# DOWNLOADER = 'crawlo.downloader.httpx_downloader.HttpXDownloader'
+# DOWNLOADER = 'crawlo.downloader.cffi_downloader.CurlCffiDownloader'
+DOWNLOADER = 'crawlo.downloader.aiohttp_downloader.AioHttpDownloader'
+
+# ============================== 队列配置 ==============================
+QUEUE_TYPE = 'redis'
+# 当使用Redis队列时，可自定义队列名称
+# 队列名称遵循统一命名规范: crawlo:{PROJECT_NAME}:queue:requests
+# SCHEDULER_QUEUE_NAME = f'crawlo:{PROJECT_NAME}:queue:requests'
+
+# ============================== 去重过滤器 ==============================
+FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'
+
+# ============================== 默认去重管道 ==============================
+DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline'
+
+# ============================== 爬虫模块配置 ==============================
+SPIDER_MODULES = ['test_project.spiders']
+
+# ============================== 中间件 ==============================
+# MIDDLEWARES = [
+#     'crawlo.middleware.simple_proxy.SimpleProxyMiddleware',
+# ]
+
+# ============================== 默认请求头配置 ==============================
+# 为DefaultHeaderMiddleware配置默认请求头
+DEFAULT_REQUEST_HEADERS = {
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+    'Accept-Encoding': 'gzip, deflate, br',
+}
+
+# ============================== 允许的域名 ==============================
+# 为OffsiteMiddleware配置允许的域名
+# ALLOWED_DOMAINS = ['example.com']
+
+# ============================== 数据管道 ==============================
+# PIPELINES = [
+#     'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',     # MySQL 存储（使用asyncmy异步库）
+# ]
+
+# ============================== 扩展组件 ==============================
+# EXTENSIONS = [
+#     'crawlo.extension.log_interval.LogIntervalExtension',
+#     'crawlo.extension.log_stats.LogStats',
+#     'crawlo.extension.logging_extension.CustomLoggerExtension',
+# ]
+
+# ============================== 日志配置 ==============================
+LOG_LEVEL = 'INFO'
+LOG_FILE = 'logs/test_project.log'
+LOG_ENCODING = 'utf-8'  # 明确指定日志文件编码
+STATS_DUMP = True
+
+# ============================== 输出配置 ==============================
+OUTPUT_DIR = 'output'
+
+# ============================== Redis配置 ==============================
+REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')
+REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
+REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
+REDIS_DB = int(os.getenv('REDIS_DB', 0))
+
+# 根据是否有密码生成 URL
+if REDIS_PASSWORD:
+    REDIS_URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+else:
+    REDIS_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+
+# ============================== MySQL配置 ==============================
+MYSQL_HOST = os.getenv('MYSQL_HOST', '127.0.0.1')
+MYSQL_PORT = int(os.getenv('MYSQL_PORT', 3306))
+MYSQL_USER = os.getenv('MYSQL_USER', 'root')
+MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', '123456')
+MYSQL_DB = os.getenv('MYSQL_DB', 'test_project')
+MYSQL_TABLE = 'test_project_data'
+MYSQL_BATCH_SIZE = 100
+MYSQL_USE_BATCH = True  # 是否启用批量插入
+
+# ============================== MongoDB配置 ==============================
+MONGO_URI = os.getenv('MONGO_URI', 'mongodb://localhost:27017')
+MONGO_DATABASE = 'test_project_db'
+MONGO_COLLECTION = 'test_project_items'
+MONGO_MAX_POOL_SIZE = 200
+MONGO_MIN_POOL_SIZE = 20
+MONGO_BATCH_SIZE = 100  # 批量插入条数
+MONGO_USE_BATCH = True  # 是否启用批量插入
+
+# ============================== 代理配置 ==============================
+# 代理功能默认不启用，如需使用请在项目配置文件中启用并配置相关参数
+PROXY_ENABLED = False  # 是否启用代理
+
+# 简化版代理配置（适用于SimpleProxyMiddleware）
+PROXY_LIST = []  # 代理列表，例如: ["http://proxy1:8080", "http://proxy2:8080"]
+
+# 高级代理配置（适用于ProxyMiddleware）
+PROXY_API_URL = ""  # 代理获取接口（请替换为真实地址）
+
+# 代理提取方式（支持字段路径或函数）
+# 示例: "proxy" 适用于 {"proxy": "http://1.1.1.1:8080"}
+# 示例: "data.proxy" 适用于 {"data": {"proxy": "http://1.1.1.1:8080"}}
+PROXY_EXTRACTOR = "proxy"
+
+# 代理刷新控制
+PROXY_REFRESH_INTERVAL = 60  # 代理刷新间隔（秒）
+PROXY_API_TIMEOUT = 10  # 请求代理 API 超时时间
+
+# ============================== Curl-Cffi 特有配置 ==============================
+# 浏览器指纹模拟（仅 CurlCffi 下载器有效）
+CURL_BROWSER_TYPE = "chrome"  # 可选: chrome, edge, safari, firefox 或版本如 chrome136
+
+# 自定义浏览器版本映射（可覆盖默认行为）
+CURL_BROWSER_VERSION_MAP = {
+    "chrome": "chrome136",
+    "edge": "edge101",
+    "safari": "safari184",
+    "firefox": "firefox135",
+}
+
+# ============================== 下载器优化配置 ==============================
+# 下载器健康检查
+DOWNLOADER_HEALTH_CHECK = True  # 是否启用下载器健康检查
+HEALTH_CHECK_INTERVAL = 60  # 健康检查间隔（秒）
+
+# 请求统计配置
+REQUEST_STATS_ENABLED = True  # 是否启用请求统计
+STATS_RESET_ON_START = False  # 启动时是否重置统计
+
+# HttpX 下载器专用配置
+HTTPX_HTTP2 = True  # 是否启用HTTP/2支持
+HTTPX_FOLLOW_REDIRECTS = True  # 是否自动跟随重定向
+
+# AioHttp 下载器专用配置
+AIOHTTP_AUTO_DECOMPRESS = True  # 是否自动解压响应
+AIOHTTP_FORCE_CLOSE = False  # 是否强制关闭连接
+
+# 通用优化配置
+CONNECTION_TTL_DNS_CACHE = 300  # DNS缓存TTL（秒）
+CONNECTION_KEEPALIVE_TIMEOUT = 15  # Keep-Alive超时（秒）
+
+# ============================== 内存监控配置 ==============================
+# 内存监控扩展默认不启用，如需使用请在项目配置文件中启用
+MEMORY_MONITOR_ENABLED = False  # 是否启用内存监控
+MEMORY_MONITOR_INTERVAL = 60  # 内存监控检查间隔（秒）
+MEMORY_WARNING_THRESHOLD = 80.0  # 内存使用率警告阈值（百分比）
+MEMORY_CRITICAL_THRESHOLD = 90.0  # 内存使用率严重阈值（百分比）