crawlo 1.1.8__py3-none-any.whl → 1.1.9__py3-none-any.whl

crawlo-1.1.9.dist-info/RECORD

@@ -0,0 +1,190 @@
+crawlo/__init__.py,sha256=1tc6uUDF1yRNU7K_k-Dl6h9FGy7Jp8fdhRsXu9PctFI,1312
+crawlo/__version__.py,sha256=lnHAM_YSOt5TvtAm4ZVs32T0TpsCQ5kOOznnQlur1YE,22
+crawlo/cli.py,sha256=pbd9wR6evB4aHWNrTRG4WW2ScCN5p4kz44eAuohSdR0,2029
+crawlo/config.py,sha256=cQTDYn2VCdlIs3Jb8mGwF6IWqj85BMr6HQaGS3XjZ7g,9535
+crawlo/config_validator.py,sha256=Q2j9rGW2lZiaA1ka5cJWaabPr1W0fwYHzY_gv-qpPyY,9903
+crawlo/crawler.py,sha256=Mjv4WrNduSTXv3hJuSynwEu-MyUhgx2TOMJ913LxkmY,36371
+crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
+crawlo/exceptions.py,sha256=sMay0wnWLfc_FXWslqxm60qz6b66LXs3EdN_w8ygE9k,1166
+crawlo/mode_manager.py,sha256=cLHwKW_iupFQ-uTgyl90SFhrYtbpVIdqevMXgPOJpWE,7517
+crawlo/project.py,sha256=FDAwiucebeck5Ftuc7M8mFCRunAyUd49FtKcCAE7R4c,6644
+crawlo/stats_collector.py,sha256=NkO09CB-220qz5rxFcD_dedGfr2VPFrDo4hya0Zh8Qc,1577
+crawlo/subscriber.py,sha256=gioTIqRdEwVG-bwIiQonbk1vWWAqTh9hzVkrqZ1AfP0,5006
+crawlo/task_manager.py,sha256=19InAxS9oJ9EMj20Aw2urN-v6BeC22dkgPuW-B9-4UI,819
+crawlo/cleaners/__init__.py,sha256=DVD4W09aJF-ih8vYx3DPZaXIE18MTEM36uo1SnqLMLk,1201
+crawlo/cleaners/data_formatter.py,sha256=NEj3NqPiyG67V6qDgn2KNj9VNHWOLNwX-7p_nad0znc,7583
+crawlo/cleaners/encoding_converter.py,sha256=uWyZMHJxC2y5cBRg8LnrBAYV3X4ESrkmKVE2ho8s1FY,4113
+crawlo/cleaners/text_cleaner.py,sha256=WDcQdvoyaY3tSxYa7EdS6BGvZNggoJbfyE662Y8TiL8,6475
+crawlo/commands/__init__.py,sha256=QbhGAmItiwVrtlTr9UUbEJMegLJo-SdzaKX2PUhBgfI,378
+crawlo/commands/check.py,sha256=172OiAxnX5wwSlszUsyPgMZwAoIbGDTdfhtRz309ilc,22843
+crawlo/commands/genspider.py,sha256=-jGJdfXLsefX_H1ydQ2wirdu6p6wmhClzVXY_0L-1aE,5050
+crawlo/commands/help.py,sha256=EaU2v4A5cFFaUUG_HTp6Hby1p36777oK_OogGnrzBfI,4879
+crawlo/commands/list.py,sha256=yByqQeZBgvjewOKxpnOobpeJ7Hnbs-CWsoyITqZu2ZY,5781
+crawlo/commands/run.py,sha256=LL6P0eGgwCgMWLkGsTWjXDu6Vh74N8vq4m7-PUiNKWc,10751
+crawlo/commands/startproject.py,sha256=aBBR5dNb7R-yEMVMisoKiEA86HUFB_yHa0VgPftDXko,15354
+crawlo/commands/stats.py,sha256=6pAgkEi8MBnCer2rWmKpaTYr1jaM6HeMG9owAvEzJyY,6064
+crawlo/commands/utils.py,sha256=nohMvUU2zLvX0XzXk6KeCNxP0EvSWj9DiVLxM_7tD5o,5106
+crawlo/core/__init__.py,sha256=PnFyJdVNHBoPmV1sW0AHQXijeoSTQ8cMYrbNM1JK8kA,41
+crawlo/core/engine.py,sha256=tWiQGAxK3bhwmq6xgg03gz-hyflGi5jPSI1ML2vrBgE,13139
+crawlo/core/processor.py,sha256=qmCqAeqhwYu-UE86evYesaGt9qpuSIfH-ZIZKcXFCZc,1140
+crawlo/core/scheduler.py,sha256=4RsGl9P0hmXB6crbOwfNr2RLiIUET1h5JFIVKtrqyD0,4953
+crawlo/downloader/__init__.py,sha256=MlstaKfW-WLXNuZs7tb7cG_wG2sQLw2hdWmUjZEIH7c,8299
+crawlo/downloader/aiohttp_downloader.py,sha256=n2qP3Q13lOnvwB7cb3YxNyNKYVHKqofNNg7j9tV9h-E,8400
+crawlo/downloader/cffi_downloader.py,sha256=IpQUqvls4mEYs_UwPvtN2L4uUIujqn-rf03NuZZkMl0,10710
+crawlo/downloader/httpx_downloader.py,sha256=7jfQfvAtfk8yD_mvwUbWLhYOxMM7r1nudiU7m_Jl9wc,12037
+crawlo/downloader/hybrid_downloader.py,sha256=otiIQBJYmzDeY9dB4Qrlu9LSSdJOpZVVae-HnNKBXik,8043
+crawlo/downloader/playwright_downloader.py,sha256=Lnc7k5cXhVnURXSxgZFCYMJkBxLg5F_OE67rtf3G7Ig,16261
+crawlo/downloader/selenium_downloader.py,sha256=vykWifXoSSs61k8BB6PtcTDDBKp5uZ9-rYYFR4_pgSQ,21036
+crawlo/extension/__init__.py,sha256=Sg588p6UhyrwFNTiD2wqGW-i3xgLX6HlLuQPKT7mayE,1526
+crawlo/extension/health_check.py,sha256=IVaaVo_0CcZtf1LoCAYXIBvs3wZ7hdmT6U4-NYWAgP0,5527
+crawlo/extension/log_interval.py,sha256=VCIeNqXcWDnxj4m6l77cjqgRzV8LfsPMb22X0Xc1Vwc,2417
+crawlo/extension/log_stats.py,sha256=Ssxz6R1YpWIj5WJvQ2cJ9F5oR7FUFdj-ITc9lV92SSU,2908
+crawlo/extension/logging_extension.py,sha256=ET6VAu1J2qNMz4NnG1G3zQLRhbsvV7l6xRIuQLE6DaE,1626
+crawlo/extension/memory_monitor.py,sha256=4aszl3C0GMQbqFhGZjZq5iQuXQR1sOz06VHjjEHgkyE,4290
+crawlo/extension/performance_profiler.py,sha256=EPiNuXuPPDU0Jtgy8arYHpr_8ASK13cCI2BytdJnu_I,4899
+crawlo/extension/request_recorder.py,sha256=RC23yzXClnVv9j2ljQvjBkUfWznfnDHsrQejKhE9y5E,4074
+crawlo/filters/__init__.py,sha256=XC_Q4ykZtSNYizYlAcehVwBBNO3lZ2zuWwafzXiuWyQ,4241
+crawlo/filters/aioredis_filter.py,sha256=E8BBvzGJfiLuwEZrlfI3O-jANudizT9xcFWrxxFHpgk,10028
+crawlo/filters/memory_filter.py,sha256=VJO0UFRYGxmV8dj4G1subsQ-FtvPcGLbvd7IVtqXnOs,9260
+crawlo/items/__init__.py,sha256=bqekZrRlDhxfWie0UbCs656TptYseoe9QJ67I4E7Elk,386
+crawlo/items/base.py,sha256=tAYrPJgblp3ZEihDXvappdYc6pGdim6x2_9QSmMKI2o,577
+crawlo/items/fields.py,sha256=wMlakQTsEwyrlLzMt1gI4pScLQZMqd3E1xcfH4dbSqk,1801
+crawlo/items/items.py,sha256=e-3nXI9ckD64vcDxxQiAU6ufbtJMs09gbZQcYjxgwHY,3374
+crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0,549
+crawlo/middleware/default_header.py,sha256=i_Uj07JObyeZFxL7ZAZmvZsHvA1HGtkNab1sA0d-nWI,1067
+crawlo/middleware/download_delay.py,sha256=2M-TchDA7MwyTfYy0Hzh_bW9wlHlpiP-oQlys7crTj0,966
+crawlo/middleware/middleware_manager.py,sha256=j1hkWRFB5rnC5SnB7oXWE5eUNv8blS9krDIDM5fIDs8,6213
+crawlo/middleware/proxy.py,sha256=qLPA9_iualWBdZhIRJJLlSZVMbwOophLEEpzS8yBG5M,11089
+crawlo/middleware/request_ignore.py,sha256=QI2z4fUnJ-4xvPTZAmsL-GqR4RFHS1xq9iDr5KFrMco,997
+crawlo/middleware/response_code.py,sha256=tmef2QVl3JCiTMii6VQkASlOY2OyqmOPoOfNxIK1eF8,659
+crawlo/middleware/response_filter.py,sha256=ep8ZxDlfIefi9YqK8dPASEp5TTDRo9QEY_jMceC411s,837
+crawlo/middleware/retry.py,sha256=-7zpRURugiTTm4QYUSUlbnURD5mcT2Ji0yHvCgY1wGc,4124
+crawlo/network/__init__.py,sha256=BLPERYPo22g1BXrW--wUnlolrdFUmOPjgOB8XQQJlck,397
+crawlo/network/request.py,sha256=63aBgH6H3ZZakBt6b_c-HUKE-c70AHxhggNOzmxjA2g,12295
+crawlo/network/response.py,sha256=EZiG4LjuIb7PxdGou4H-oSOQhec1ZdBRTkO-5fl8JTo,12701
+crawlo/pipelines/__init__.py,sha256=lrdVDjeHLNkA4_MAwI1auk_I9xfeU1SlBWXiammb6lc,616
+crawlo/pipelines/bloom_dedup_pipeline.py,sha256=QQxGFGEoMHN4Vx2kq7G_i1o9pmuXp8clZebilOar3fk,5642
+crawlo/pipelines/console_pipeline.py,sha256=KABkR3J-rqO0Awox7lizxKR2XuHfVhWPiVRgIybwwu4,1248
+crawlo/pipelines/csv_pipeline.py,sha256=6FBT2AoU6iNU-5NfgWRq7-JpF9dK2nBokjxx-y4jIas,12174
+crawlo/pipelines/database_dedup_pipeline.py,sha256=wVBXEGArFR3uxoN7yfJSOarBmtGrJpOqowAqa7OUs98,8000
+crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
+crawlo/pipelines/memory_dedup_pipeline.py,sha256=5jeL2jEq7sioYmXlzfkx-LNSbWyChrXeWx8d15YEZOA,3839
+crawlo/pipelines/mongo_pipeline.py,sha256=k7gNqAO-g2MtIfArphC6z5ZzkKVRkBKcv-2ImziPFA0,5706
+crawlo/pipelines/mysql_pipeline.py,sha256=cwgJvRORTRea_Eep2coBaMf3G8PQVTQA1qrnIlDZApc,13480
+crawlo/pipelines/pipeline_manager.py,sha256=vK87pAEmpGR24yl6Cr7ovCKag2oB5mruijfYT8nnG5o,2358
+crawlo/pipelines/redis_dedup_pipeline.py,sha256=sgrBSVdxPWgh8HQxvGsazz1MSyBERJF5jd1yoeYo0lE,6166
+crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlo/queue/pqueue.py,sha256=yzF900ko2RReVNZtfk_VV3PzSXEUALI6SHf97geuu6s,1058
+crawlo/queue/queue_manager.py,sha256=UxIhyLzVM0HNNzwQ6LvOoCJ-zOcE95gl0ZKpzTz1ybA,12930
+crawlo/queue/redis_priority_queue.py,sha256=RbwKsVxzk31B1VRvyve4vHKe2DesL7K37IZAA31kdd0,12783
+crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
+crawlo/settings/default_settings.py,sha256=cwEIVL294hs7iWVOSfnwA4nuRWjHXJk_Bqgbd37Vmcw,8783
+crawlo/settings/setting_manager.py,sha256=0RYAk07qoJ5WTw_mvV4ECWGS2QNpCnGmBZVTHjqOVIg,3707
+crawlo/spider/__init__.py,sha256=Z_rK23l5yt-DuwJPg8bcqodM_FIs4-iHLaKOimGumcE,20452
+crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
+crawlo/templates/project/__init__.py.tmpl,sha256=f3ETIXw_O6K-lkL6lXM5znMPJW1FZYGFrwDs2BnHcnQ,58
+crawlo/templates/project/items.py.tmpl,sha256=mt1Mm--H2Ouos3r7JPkYh0r33rgYJf1YOMz0OZy8TYs,297
+crawlo/templates/project/middlewares.py.tmpl,sha256=Dyp_tyNgF9U4w1oMAvOAOfSepWsjXWOvH__qQ3BOkSQ,3731
+crawlo/templates/project/pipelines.py.tmpl,sha256=GBHYU0Jx8sKDCdGJp44FMSH7u2slxoFg6a-R9Uwg_-I,2608
+crawlo/templates/project/run.py.tmpl,sha256=KaLhqngKEzKtvDI2KCHe00yFRXnxMwbdq7YoXPR4kEg,1226
+crawlo/templates/project/settings.py.tmpl,sha256=JpK0Zq9TdxwucizUliOSLNYs9KgnFwJuai8GJEGlt6o,11710
+crawlo/templates/project/settings_distributed.py.tmpl,sha256=6xG7ehu86qAXscWOAFbrtXL5mzGNarCgYpbW5LXzExg,4202
+crawlo/templates/project/settings_gentle.py.tmpl,sha256=IPE9H9WqRQWVdj9supFutt9gvypYQRVhRFOwZyYxem0,3226
+crawlo/templates/project/settings_high_performance.py.tmpl,sha256=UNDx9f76kF2je8hE4ZE73wQcLcApjrEzpNLHWp9DvaI,5283
+crawlo/templates/project/settings_simple.py.tmpl,sha256=qk0EnX4xncVBPVlFmqS1yDn56Cz6TBEQhgDdY8TABJ8,2297
+crawlo/templates/project/spiders/__init__.py.tmpl,sha256=j_YKsw6HQMJyqlk3WUouP3bsr-XVxshRoSNakHBc00g,106
+crawlo/templates/spider/spider.py.tmpl,sha256=CzZr8SNzkuEQGA8F11CfMxYvOpwnHOWCG-1MiLqN6D0,4896
+crawlo/tools/__init__.py,sha256=5H6rAhjfNSqRMjjlLDVq-vEJWRFyCO-J6HN2kexnXJU,3671
+crawlo/tools/anti_crawler.py,sha256=MU6KEPT0q85e_-Px8Rmw1fxdwlmOdpXfo0KYVpPlivU,9163
+crawlo/tools/authenticated_proxy.py,sha256=L93WeXajIZ3si8xNcE7bBPv34FvqSyTvVfD78fJAKQE,7032
+crawlo/tools/data_validator.py,sha256=hxPN28YtJDFFLjBBYhDjHmR8ShNTEjgIsv-cmcDKIu8,5310
+crawlo/tools/date_tools.py,sha256=icbx6x08t-Aj8j0jAB-93hDdSpBV9Rd-aaNt4UpyjNA,600
+crawlo/tools/distributed_coordinator.py,sha256=ZN-zYsDPw0JWSBF7oHqL2vpUDgRElkDTVW7TcY1ArcI,12477
+crawlo/tools/retry_mechanism.py,sha256=27iIuLCg1ZHjog6Gcr7dz0APkeTZ5cIlcnZ0pF18ZVA,7801
+crawlo/tools/scenario_adapter.py,sha256=JouFxI3513PRe1ObwHWc72vBvptNpNv0Ew3pRaEKjQQ,9398
+crawlo/utils/__init__.py,sha256=OVjs7qjyd7sn1vrh4MWykO4sMsAPLIWkq-74dQ2Pzak,557
+crawlo/utils/batch_processor.py,sha256=Bim6iSinjFKuKI2c1Ss6PQ1R-zJEvLdVxPiBDTa75z4,8895
+crawlo/utils/controlled_spider_mixin.py,sha256=VjT30pNW_YIgmTD0nb7DDl2D3HvpnAYFzgSgV3fxFN0,16475
+crawlo/utils/date_tools.py,sha256=6zNU23vQ8p3-xHknlIp5airCLOREETaqw4YElyMO_lQ,8907
+crawlo/utils/db_helper.py,sha256=3ib5-agrlwf2t5S_QtLRYH75wvJDlYbRqRmDEbpH5Bo,10559
+crawlo/utils/enhanced_error_handler.py,sha256=qcFR3o2HyEyBQf0n1l6-hSOiFGytYGmhAzseqeidwHA,13906
+crawlo/utils/env_config.py,sha256=Rn5DOUoshtIfCDqWh5CNpTfiXo1bF_zEb0tLo09DLMk,2869
+crawlo/utils/error_handler.py,sha256=v-UVJfIp9k-lOXq5DBukJfE_nb-kv83tlr28RXrIUYQ,4286
+crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
+crawlo/utils/large_scale_config.py,sha256=j7wQ5ty7pQlBRygw2vhRJ7OI19RYBZKPfYMP3WeF2WI,8154
+crawlo/utils/large_scale_helper.py,sha256=JJqcGSI6VaVe3MSL6IWjmCp8XQIu6T4U-BvBLSttr_s,12157
+crawlo/utils/log.py,sha256=A3lPyhD8kD88cV23KOL-_eT8g69xGQ5L1toDB2AO0mc,4005
+crawlo/utils/performance_monitor.py,sha256=mygXJ0FNxHyunb_aQSgmcBv0bkT4ISG5QqlOXANtvlk,9595
+crawlo/utils/queue_helper.py,sha256=xpUUTOqlU1xz2Pb9NKAVGo3AfAO-7Xvx8Lm1q65Dgck,4743
+crawlo/utils/redis_connection_pool.py,sha256=9WprN-HbtYGLhAu0AXjXeHcVbJ9jd3dS3U4e_stdCMY,10331
+crawlo/utils/redis_key_validator.py,sha256=mDqZROAlrnMvKX7ToXdaSjQGfu2J5tpCqo1tQ75n5RA,5637
+crawlo/utils/request.py,sha256=yoLB2rY8d78vgPjIWpdhY5SalIKjyLIvTG_UH6EMdVI,8798
+crawlo/utils/request_serializer.py,sha256=bPoSQqE2ksiMyP3WiPB3w3UqZs4f_LgkAw4Pj0qyBDo,8565
+crawlo/utils/spider_loader.py,sha256=pEDUsYOTGjszA6KgjiMlYN4GS5fP4uakkhcp3JTFFQY,2187
+crawlo/utils/system.py,sha256=HvWV1acxou0Rn0L7pNq4CnV_GWFeU0Tmjy3_nLD8M64,237
+crawlo/utils/tools.py,sha256=5Uv25Wy4m_ndZY0-n-eX-t3PxvaZ6wR3-Wvx-o7_Vrs,271
+crawlo/utils/url.py,sha256=rlgX2VlJv6JvLmCDTsbxzMSXE6R5ZL_0dLEqprsA-JU,1482
+examples/__init__.py,sha256=6i631BPnS_TR_BWUjtjB5CBO-zv9kRkwQTQvSya2wHE,123
+tests/DOUBLE_CRAWLO_PREFIX_FIX_REPORT.md,sha256=HjMZv6RjN1o5D1mfgEydP8Mcc9T_4ScR6lG3xVxs8P8,3346
+tests/__init__.py,sha256=scL1IPVT1iucth7v8ffrjRdeW7QrC_Y7AMmFVMdTY1Y,129
+tests/advanced_tools_example.py,sha256=7nlFLRVMVYzDz_CAdgQa9fJu7o0M6jBMo7PTvUsRbo0,9065
+tests/authenticated_proxy_example.py,sha256=8I8bxAIeYDdsh5JoGIK1nxaKEpIdOFnCtufVLjpuq_o,8241
+tests/cleaners_example.py,sha256=i90lrmno16qSq_qjc6WOVOw1Mx4_vzoBgVeqJYC7Vro,5170
+tests/config_validation_demo.py,sha256=iPe_BWBTa-gRvRrnYYkkRDqyKqLzAhSCoWB_yFSkhrg,3116
+tests/controlled_spider_example.py,sha256=SP_k4mdKPvD1JCPs9UCm68jcy2Frg84vvXv9-14RC6I,7776
+tests/date_tools_example.py,sha256=x_-duqnVZ-Hrk-SaNplIfcIV6W3c6u6MTxW35u1i0F0,4862
+tests/dynamic_loading_example.py,sha256=NI0SCg4lPME0RCcNpDDw1HjErjmCgJntCN0ahAEw61g,18263
+tests/dynamic_loading_test.py,sha256=DYbMrEewerx0VGXixci3p9VYgDDQvCPevA92CNjq1Jo,3309
+tests/env_config_example.py,sha256=sKE8DvMBhM3uy439LpgLHd4wF7MGUrUc-X6E7g9qsz0,4818
+tests/error_handling_example.py,sha256=219H1dOm6TZaQ0tmd1zXZlo0ODcwQSBL7m4jZmCFoCo,5161
+tests/redis_key_validation_demo.py,sha256=z1R7ZW6oiG3T0mSZJfP6FRqp4IsOjgbCPxvPuEL7z0I,4345
+tests/response_improvements_example.py,sha256=wnYGJO6MKj5_jbwKLDlbXu_Dli5XC7vlWdzByi82_5Y,5258
+tests/test_advanced_tools.py,sha256=3R8EfKVyBHEb6FA5TP3ieaWeHZhobVgSx8t3phipCrE,5250
+tests/test_all_redis_key_configs.py,sha256=uMLlgtJ7tglzFO_Xbnwol723Lu6bjyIeHaLHtemphaQ,5660
+tests/test_authenticated_proxy.py,sha256=s4pr5JyBTHYQgRq_IymiVKE08vyW1MwR27pSwrrVLVk,4198
+tests/test_cleaners.py,sha256=7KRAjqXF7lWXe-9Hj_5CSFlbRJ6SB7XyMNuMXx36Jmg,1765
+tests/test_comprehensive.py,sha256=kGNcJ9UkQxysYqvsBu0YxAaPleOvN9_hztLy7ljkfc4,5036
+tests/test_config_validator.py,sha256=eZCnZyh34rpbYbzXMWvQkLNl7gu7_Js0fE7iFatXyB4,6632
+tests/test_date_tools.py,sha256=CQdAmIS6bpAdwQH9ETDH__06l2gGL7EHUQuh7mdTF-A,3930
+tests/test_double_crawlo_fix.py,sha256=t7EZsKIfE49ZL3FM0bu1AZ1hOMygwALUN5UtqDGtLUs,7826
+tests/test_double_crawlo_fix_simple.py,sha256=0OPbW8c71n_oB7pt10rRZi-W9eFCO_JOKTquWdIgTUU,4683
+tests/test_dynamic_downloaders_proxy.py,sha256=PtEW-pnVijeX2yX34UcoXYEY23yTBxb-kyNYh-WDljQ,4326
+tests/test_dynamic_proxy.py,sha256=YL2sghNKG7k27-SaHMh4boNLVBHhfSttUwUqiSsOEX4,3080
+tests/test_dynamic_proxy_config.py,sha256=uYXZ804ULI9qYMF-uNjMbi3L_NGzoMqLJcEZAl7aZ2I,5707
+tests/test_dynamic_proxy_real.py,sha256=DTjP8JnSwBnNZ3Ls1BjDAmt6xSuye_6CxwZ4LBisPTM,3402
+tests/test_edge_cases.py,sha256=fFRL3-ChBylg4e-73L9ZSc3u72dPmSfg3jpAg_6GOGA,10553
+tests/test_enhanced_error_handler.py,sha256=E6HHzqGA6U9frzQvy0iVqFHi6LvRJobRwkFWk37tA10,8435
+tests/test_env_config.py,sha256=nfP4nCG9ZHeJUfxo1JKUmiihYbhSeWx_oNW5mMfDHfQ,4746
+tests/test_error_handler_compatibility.py,sha256=o5JLLLdo25Sl_3hpMx6I2fqSgZFAcnI4E6Ci-KxAxwA,4129
+tests/test_final_validation.py,sha256=fBxf_6YcAEa_HyV_oGAXmmVHY4i6FdA4J6klCmc36hQ,4925
+tests/test_framework_env_usage.py,sha256=HYpTwORXeaJHMffCYAGHGvc_a6ax4lo28xP8BYOaKxk,4098
+tests/test_integration.py,sha256=8Bky_n5fbJDbb2pbKqc6IwJgpV0B9iau84C-PwcbsLA,11321
+tests/test_item_dedup_redis_key.py,sha256=GOmmJ7hdTV_2XifRa_PQDrQpBt7Rnt3ViYNjNpflGgw,3755
+tests/test_parsel.py,sha256=KYskaN_4HBc1XDTltjVo12v1i7JAThB2UIwcWZ-mwbY,672
+tests/test_performance.py,sha256=heeFoS3R9n_t7ri0MyUL8kE0FY9sYJiC7BSQNI2C5HQ,11286
+tests/test_proxy_health_check.py,sha256=xo3QMP1YNw9hu7JDHZOYCUZmFFKLJpHSh4SbxXhCRPQ,1091
+tests/test_proxy_middleware_integration.py,sha256=zcl7fR9Toc-I-stSUTzKZPwcfh3kgrpjI5SbkZ6AVmE,4305
+tests/test_proxy_providers.py,sha256=XwWZCywTYguSsUxSm6fsbaoH1p9dKjqSIx9-sqKZehA,1693
+tests/test_proxy_stats.py,sha256=Til_yksrRz2yBVw-yJi5-36LhNW3vTwpXTm4BdR9PUM,507
+tests/test_proxy_strategies.py,sha256=ZkziozkvZd3KWOQnpHQ8Upd3WpyoX7gN0qFGluNm348,1809
+tests/test_queue_manager_double_crawlo.py,sha256=LYduecvJ93HAU6Ub-tIrGM2pwOUGoCTkpmzBxjpY5tM,9116
+tests/test_queue_manager_redis_key.py,sha256=-dnqs2p50zpf09-Z7vo74s8hLQ6cxXd8WCdHM0l17qM,6278
+tests/test_redis_config.py,sha256=TqzFRojc6esGXjGhUCvSLYQDUTAgEJsty9vRVuNraMU,893
+tests/test_redis_connection_pool.py,sha256=ORBU0k-6htQ2VVfyQkqTKVLGQWX04XRxdEGsZKeerQA,9177
+tests/test_redis_key_naming.py,sha256=ojHhBED2DEf0VzqQgtzefygSn6hZiBhPFyVUBs_4rdI,6776
+tests/test_redis_key_validator.py,sha256=kud0AhnS_6NP9s8G5HB5L9rob5rv2W6WLy6ypHYQDbk,4304
+tests/test_redis_queue.py,sha256=o6xViXxJcdx-1eMcG3vhAQEIm8h346HnZb7JXs7ZjwM,6622
+tests/test_request_serialization.py,sha256=8sVdppAsohJ5u-m1WvablCndwL-M_36YPLdGKwgeznM,2289
+tests/test_response_improvements.py,sha256=zvbkTkWhgdlFYtRu_ckgq6wGDGwpe_PTECYqpLDM3BU,5876
+tests/test_scheduler.py,sha256=-FOkTWzaMdr6yfO1Msu74hI_GgSfD7iRxO-cFA-9Iyk,7442
+tests/test_simple_response.py,sha256=6RYOBRzAtyNvJ9a5JVTNubM-rvxnuX8jQOvq3sUZxwo,1488
+tests/test_telecom_spider_redis_key.py,sha256=R581J-kvjh4CpDCCIZKPnV0vY6MvtkvTzmSIbwhXSXs,7470
+tests/test_template_content.py,sha256=5QAnhKZFDKg-_uFryllLMpCk3a1nCS44hMmYfXm8gWk,2878
+tests/test_template_redis_key.py,sha256=U6L5HtnDyGp3s6-O4F_yG2Q2nNIGTqB_Q-ESv2EMeOU,4812
+tests/test_tools.py,sha256=IWiu9JGV-5Ow0ivFtiDw62REht-8Hn7NfyR9rnYSlbU,5113
+tests/tools_example.py,sha256=MfVBYemKvHs6MUbydgrJfhiGnygp5dRoIE-eIXCOR7M,7669
+crawlo-1.1.9.dist-info/METADATA,sha256=i3EFNboqsUyzh_CSajaNfJajxlwq-uiP7iDkjMCTQjg,25548
+crawlo-1.1.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.1.9.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.1.9.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.1.9.dist-info/RECORD,,

examples/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 12:36
-# @Author  :   oscar
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 12:36
+# @Author  :   oscar
+# @Desc    :   None
+"""

tests/DOUBLE_CRAWLO_PREFIX_FIX_REPORT.md

@@ -1,82 +1,82 @@
-# 双重 crawlo 前缀问题修复报告
-
-## 问题描述
-用户在使用分布式爬虫时发现Redis key中出现了双重`crawlo`前缀，例如`crawlo:crawlo:queue:processing:data`。这导致了Redis key命名不一致和潜在的混淆问题。
-
-## 问题分析
-经过代码分析，发现问题出在以下两个方面：
-1. RedisPriorityQueue类在处理队列名称时会自动修改用户提供的队列名称
-2. QueueManager类在提取项目名称时没有正确处理双重`crawlo`前缀的情况
-
-## 修复方案
-
-### 1. RedisPriorityQueue类修复
-文件：`crawlo/queue/redis_priority_queue.py`
-
-**修复前**：
-```python
-# 如果提供了 queue_name，确保符合命名规范
-# 处理可能的重复前缀问题
-if queue_name.startswith("crawlo:crawlo:"):
-    # 修复双重 crawlo 前缀
-    self.queue_name = queue_name.replace("crawlo:crawlo:", "crawlo:", 1)
-elif not queue_name.startswith("crawlo:"):
-    # 如果没有 crawlo 前缀，添加它
-    self.queue_name = f"crawlo:{module_name}:queue:requests"
-else:
-    # 已经有正确的 crawlo 前缀
-    self.queue_name = queue_name
-```
-
-**修复后**：
-```python
-# 保持用户提供的队列名称不变，不做修改
-self.queue_name = queue_name
-```
-
-### 2. QueueManager类修复
-文件：`crawlo/queue/queue_manager.py`
-
-**修复后**：
-```python
-# 处理可能的双重 crawlo 前缀
-if parts[0] == "crawlo" and parts[1] == "crawlo":
-    # 双重 crawlo 前缀，取第三个部分作为项目名称
-    if len(parts) >= 3:
-        project_name = parts[2]
-    else:
-        project_name = "default"
-elif parts[0] == "crawlo":
-    # 正常的 crawlo 前缀，取第二个部分作为项目名称
-    project_name = parts[1]
-else:
-    # 没有 crawlo 前缀，使用第一个部分作为项目名称
-    project_name = parts[0]
-```
-
-## 测试验证
-
-### 测试1：Redis队列命名修复测试
-验证RedisPriorityQueue正确处理各种队列名称格式：
-- 正常命名：`crawlo:test_project:queue:requests` → `crawlo:test_project:queue:requests`
-- 双重 crawlo 前缀：`crawlo:crawlo:queue:requests` → `crawlo:crawlo:queue:requests`
-- 三重 crawlo 前缀：`crawlo:crawlo:crawlo:queue:requests` → `crawlo:crawlo:crawlo:queue:requests`
-
-### 测试2：队列管理器项目名称提取测试
-验证QueueManager正确提取项目名称：
-- 正常命名：`crawlo:test_project:queue:requests` → `test_project`
-- 双重 crawlo 前缀：`crawlo:crawlo:queue:requests` → [queue](file://d:\dowell\projects\Crawlo\crawlo\core\processor.py#L13-L13)
-- 三重 crawlo 前缀：`crawlo:crawlo:crawlo:queue:requests` → `crawlo`
-
-### 测试3：队列管理器创建队列测试
-验证整个流程的正确性，确保队列名称在传递过程中保持一致。
-
-所有测试均已通过，表明双重`crawlo`前缀问题已得到解决。
-
-## 结论
-通过以上修复，我们成功解决了Redis key中出现双重`crawlo`前缀的问题。现在Redis队列名称将保持用户配置的一致性，processing和failed队列也会相应地保持相同的前缀结构。
-
-## 建议
-1. 建议用户在项目配置中使用标准的队列名称格式，如`crawlo:{project_name}:queue:requests`
-2. 可以使用Redis key验证工具定期检查和规范Redis key命名
+# 双重 crawlo 前缀问题修复报告
+
+## 问题描述
+用户在使用分布式爬虫时发现Redis key中出现了双重`crawlo`前缀，例如`crawlo:crawlo:queue:processing:data`。这导致了Redis key命名不一致和潜在的混淆问题。
+
+## 问题分析
+经过代码分析，发现问题出在以下两个方面：
+1. RedisPriorityQueue类在处理队列名称时会自动修改用户提供的队列名称
+2. QueueManager类在提取项目名称时没有正确处理双重`crawlo`前缀的情况
+
+## 修复方案
+
+### 1. RedisPriorityQueue类修复
+文件：`crawlo/queue/redis_priority_queue.py`
+
+**修复前**：
+```python
+# 如果提供了 queue_name，确保符合命名规范
+# 处理可能的重复前缀问题
+if queue_name.startswith("crawlo:crawlo:"):
+    # 修复双重 crawlo 前缀
+    self.queue_name = queue_name.replace("crawlo:crawlo:", "crawlo:", 1)
+elif not queue_name.startswith("crawlo:"):
+    # 如果没有 crawlo 前缀，添加它
+    self.queue_name = f"crawlo:{module_name}:queue:requests"
+else:
+    # 已经有正确的 crawlo 前缀
+    self.queue_name = queue_name
+```
+
+**修复后**：
+```python
+# 保持用户提供的队列名称不变，不做修改
+self.queue_name = queue_name
+```
+
+### 2. QueueManager类修复
+文件：`crawlo/queue/queue_manager.py`
+
+**修复后**：
+```python
+# 处理可能的双重 crawlo 前缀
+if parts[0] == "crawlo" and parts[1] == "crawlo":
+    # 双重 crawlo 前缀，取第三个部分作为项目名称
+    if len(parts) >= 3:
+        project_name = parts[2]
+    else:
+        project_name = "default"
+elif parts[0] == "crawlo":
+    # 正常的 crawlo 前缀，取第二个部分作为项目名称
+    project_name = parts[1]
+else:
+    # 没有 crawlo 前缀，使用第一个部分作为项目名称
+    project_name = parts[0]
+```
+
+## 测试验证
+
+### 测试1：Redis队列命名修复测试
+验证RedisPriorityQueue正确处理各种队列名称格式：
+- 正常命名：`crawlo:test_project:queue:requests` → `crawlo:test_project:queue:requests`
+- 双重 crawlo 前缀：`crawlo:crawlo:queue:requests` → `crawlo:crawlo:queue:requests`
+- 三重 crawlo 前缀：`crawlo:crawlo:crawlo:queue:requests` → `crawlo:crawlo:crawlo:queue:requests`
+
+### 测试2：队列管理器项目名称提取测试
+验证QueueManager正确提取项目名称：
+- 正常命名：`crawlo:test_project:queue:requests` → `test_project`
+- 双重 crawlo 前缀：`crawlo:crawlo:queue:requests` → [queue](file://d:\dowell\projects\Crawlo\crawlo\core\processor.py#L13-L13)
+- 三重 crawlo 前缀：`crawlo:crawlo:crawlo:queue:requests` → `crawlo`
+
+### 测试3：队列管理器创建队列测试
+验证整个流程的正确性，确保队列名称在传递过程中保持一致。
+
+所有测试均已通过，表明双重`crawlo`前缀问题已得到解决。
+
+## 结论
+通过以上修复，我们成功解决了Redis key中出现双重`crawlo`前缀的问题。现在Redis队列名称将保持用户配置的一致性，processing和failed队列也会相应地保持相同的前缀结构。
+
+## 建议
+1. 建议用户在项目配置中使用标准的队列名称格式，如`crawlo:{project_name}:queue:requests`
+2. 可以使用Redis key验证工具定期检查和规范Redis key命名
 3. 如果需要统一的命名规范，可以在项目初始化时明确指定队列名称

tests/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-08-24 12:36
-# @Author  :   crawl-coder
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-08-24 12:36
+# @Author  :   crawl-coder
+# @Desc    :   None
+"""