crawlo 1.2.2__py3-none-any.whl → 1.2.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of crawlo might be problematic. Click here for more details.

Files changed (47) hide show
  1. crawlo/__version__.py +1 -1
  2. crawlo/commands/check.py +69 -69
  3. crawlo/commands/genspider.py +25 -25
  4. crawlo/commands/list.py +23 -23
  5. crawlo/commands/run.py +34 -34
  6. crawlo/commands/startproject.py +35 -36
  7. crawlo/commands/stats.py +21 -21
  8. crawlo/commands/utils.py +4 -4
  9. crawlo/config.py +4 -4
  10. crawlo/config_validator.py +1 -2
  11. crawlo/data/__init__.py +6 -0
  12. crawlo/data/user_agents.py +108 -0
  13. crawlo/downloader/hybrid_downloader.py +0 -1
  14. crawlo/downloader/selenium_downloader.py +1 -1
  15. crawlo/extension/health_check.py +2 -2
  16. crawlo/items/fields.py +0 -1
  17. crawlo/middleware/offsite.py +0 -1
  18. crawlo/middleware/proxy.py +3 -2
  19. crawlo/middleware/request_ignore.py +0 -1
  20. crawlo/middleware/response_code.py +0 -1
  21. crawlo/middleware/response_filter.py +0 -1
  22. crawlo/middleware/retry.py +1 -1
  23. crawlo/mode_manager.py +3 -3
  24. crawlo/pipelines/database_dedup_pipeline.py +1 -3
  25. crawlo/pipelines/memory_dedup_pipeline.py +2 -2
  26. crawlo/pipelines/mysql_pipeline.py +4 -3
  27. crawlo/pipelines/redis_dedup_pipeline.py +2 -4
  28. crawlo/project.py +2 -2
  29. crawlo/subscriber.py +1 -2
  30. crawlo/templates/project/settings.py.tmpl +1 -3
  31. crawlo/templates/project/settings_distributed.py.tmpl +2 -0
  32. crawlo/utils/batch_processor.py +2 -3
  33. crawlo/utils/controlled_spider_mixin.py +1 -1
  34. crawlo/utils/enhanced_error_handler.py +3 -6
  35. crawlo/utils/env_config.py +1 -1
  36. crawlo/utils/error_handler.py +2 -4
  37. crawlo/utils/large_scale_helper.py +2 -1
  38. crawlo/utils/performance_monitor.py +5 -4
  39. crawlo/utils/redis_connection_pool.py +4 -4
  40. crawlo/utils/redis_key_validator.py +1 -2
  41. crawlo/utils/request_serializer.py +1 -2
  42. crawlo/utils/spider_loader.py +0 -1
  43. {crawlo-1.2.2.dist-info → crawlo-1.2.3.dist-info}/METADATA +1 -1
  44. {crawlo-1.2.2.dist-info → crawlo-1.2.3.dist-info}/RECORD +47 -45
  45. {crawlo-1.2.2.dist-info → crawlo-1.2.3.dist-info}/WHEEL +0 -0
  46. {crawlo-1.2.2.dist-info → crawlo-1.2.3.dist-info}/entry_points.txt +0 -0
  47. {crawlo-1.2.2.dist-info → crawlo-1.2.3.dist-info}/top_level.txt +0 -0
{crawlo-1.2.2.dist-info → crawlo-1.2.3.dist-info}/RECORD RENAMED
@@ -1,42 +1,44 @@
1
1
  crawlo/__init__.py,sha256=jSOsZbDJ_Q5wZV8onSXx5LgNM7Z1q3zCROGdImBDr2I,1373
2
- crawlo/__version__.py,sha256=ZQHRX6XTb8PHUxt2yUXlITCSopFOjlI8X1fch8t4uA8,23
2
+ crawlo/__version__.py,sha256=dFcQl_qO2lt-i4L2cxednHdShJmSoHYFLa05bQQXA4E,23
3
3
  crawlo/cli.py,sha256=DQIS3qIqLvQk_RVBc1j52to4QULfkq3DKK0fpNdiQ9k,2617
4
- crawlo/config.py,sha256=_pORwcVEgjEhrqVaApu51X0Il3TBK6w9aezGnnqYu8Y,9847
5
- crawlo/config_validator.py,sha256=M118EATR-tITzRSe2oSinV5oh2QsooMCkEJ5WS8ma_0,10155
4
+ crawlo/config.py,sha256=pQm5q4hFFW8DMpQzvlhrHKvZIzhbrDVa0PMQPbk6ybg,9818
5
+ crawlo/config_validator.py,sha256=GsIgHM8LagVN_l5zWv8twXt7yiDyE6anH81Zjeaa7rQ,10134
6
6
  crawlo/crawler.py,sha256=24EE7zFPByeYLJnf1K_R9fhJMqaFUjBSa6TuUhlY4TI,37398
7
7
  crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
8
8
  crawlo/exceptions.py,sha256=YVIDnC1bKSMv3fXH_6tinWMuD9HmKHIaUfO4_fkX5sY,1247
9
- crawlo/mode_manager.py,sha256=q34nX-bMCLCKFfeYy_34VrBT86sL_K030LlLX2H3m0I,7728
10
- crawlo/project.py,sha256=MH2OYXIZdc9EJolUpUBXTcAgS7qvPP366LkOy62iP_8,6831
9
+ crawlo/mode_manager.py,sha256=T34jxDwNSM3Zmpa-IB-nOxb8HzRROTTYpkmnzxEYrrk,7728
10
+ crawlo/project.py,sha256=H_OWtYYQg1UllqoQiJQ0eezDoVmZbtLUmJCHmXq3bjY,6824
11
11
  crawlo/stats_collector.py,sha256=v4jC9BAe-23w93hWzbeMCCgQ9VuFPyxw5JV9ItbGH8w,1636
12
- crawlo/subscriber.py,sha256=Aj0kPpbBYlzOb1uViDFraoaThsQEVlqOSYUaFT3jSDs,5136
12
+ crawlo/subscriber.py,sha256=h8fx69NJZeWem0ZkCmfHAi2kgfDGFObHpwN0aGNUM6Y,5115
13
13
  crawlo/task_manager.py,sha256=PScfEB03306Txa0l38AeQ_0WVhKzeWOFyT3bnrkbHW0,849
14
14
  crawlo/cleaners/__init__.py,sha256=lxL-ZWDKW-DdobdgKUQ27wNmBiUhGnD0CVG6HWkX3_o,1261
15
15
  crawlo/cleaners/data_formatter.py,sha256=iBDHpZBZvn9O7pLkTQilE1TzYJQEc3z3f6HXoVus0f0,7808
16
16
  crawlo/cleaners/encoding_converter.py,sha256=G3khLlk0uBeTwIutsWxVUeSuyc1GMC1BDNJDwsU9ryg,4238
17
17
  crawlo/cleaners/text_cleaner.py,sha256=16e6WqIIb9qANMiK-vCEl4TvgkId19Aa2W1NMLU-jFQ,6707
18
18
  crawlo/commands/__init__.py,sha256=orvY6wLOBwGUEJKeF3h_T1fxj8AaQLjngBDd-3xKOE4,392
19
- crawlo/commands/check.py,sha256=jW8SgfkOS35j4VS7nRZBZdFCBX9CVFez5LR2sfP_H1U,23437
20
- crawlo/commands/genspider.py,sha256=_3GwFMYK79BuKk__5L0ljuwWwOzN80MeuhRkL4Ql11A,5201
19
+ crawlo/commands/check.py,sha256=c2_paopV6S8PXSQoyZWyecYpKjuNe_-QXRs6jSouIo0,23443
20
+ crawlo/commands/genspider.py,sha256=MccEP-JGeY4oafpN9yfKa87hZQIK0cee75QgK3a-kxM,5142
21
21
  crawlo/commands/help.py,sha256=-bQRaGiV0tw4FKURDY2OXeQQuHGGlq6uzYJQ0tHB20w,5322
22
- crawlo/commands/list.py,sha256=octTk0QZhapiyM7WgCPersP2v3MesthbJeG9vMqVFOs,5936
23
- crawlo/commands/run.py,sha256=Go9hAEUMuG3GphBgemG5S5W4MF39XOxp7-E06rX-pTU,11043
24
- crawlo/commands/startproject.py,sha256=UYGelGY4dM6Zu3U4G5m8snKqbsfgszhvfpAJLl5b5tM,15772
25
- crawlo/commands/stats.py,sha256=iEKdxHoqsJuTkn8zAF9ekBVO1--8__BeD7xohYG5NwE,6252
26
- crawlo/commands/utils.py,sha256=b7yW6UlOLprR3gN9oOdhcl3fsCwWRE3-_gDxWz5xhMo,5292
22
+ crawlo/commands/list.py,sha256=8gzQGbSzZ3HVciQZNUHEcJSRZ0ldoPTtXD4YPLMZluQ,5927
23
+ crawlo/commands/run.py,sha256=nGy1SODgSMCudSlUCbNFL2qCHp6SHy0FfWTF9FvyFYA,11003
24
+ crawlo/commands/startproject.py,sha256=KjEPI3_BwafOK99lx1S9u9B_Tiez2mwx8EnIvQx7Cgo,15650
25
+ crawlo/commands/stats.py,sha256=_V5dtoYnTY76Zrt3-zolcfwgJv3v7yjfJuQ0B7t4Aj4,6315
26
+ crawlo/commands/utils.py,sha256=6OVdbP8u7e3DF8daNs7KDA9zBEz6vX0MgGWOiBIZoW8,5273
27
27
  crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
28
28
  crawlo/core/engine.py,sha256=kkNHQt1Dycb1hrEmG1m3seyX9C5qQp4GhOF-0H97t-A,14078
29
29
  crawlo/core/processor.py,sha256=oHLs-cno0bJGTNc9NGD2S7_2-grI3ruvggO0SY2mf3Q,1180
30
30
  crawlo/core/scheduler.py,sha256=LGeytsc04itdX7ulsl46MXBTyRFhBUu7UCSQzbtKZsk,5362
31
+ crawlo/data/__init__.py,sha256=UPqgioMdu3imSUmpLWzVlpvoBnEfaPSAT-crCcWd7iw,121
32
+ crawlo/data/user_agents.py,sha256=uTASHTJJ-EZJ_A0W3WKo0KRZcRklOH9codkGTr8Qiiw,4805
31
33
  crawlo/downloader/__init__.py,sha256=8-r4_Wc_X64FJtKzNQamwsZsc428creKeFo19VxF33o,8565
32
34
  crawlo/downloader/aiohttp_downloader.py,sha256=Ck4ybwpox3sC7v4IGc8RIcvmIBf9v6jAdyNGHByY1m4,8620
33
35
  crawlo/downloader/cffi_downloader.py,sha256=U2CMYQ8ZyKMQNNPvyI2USqI-Dclq2d7ZRgp3LxLjTFs,10966
34
36
  crawlo/downloader/httpx_downloader.py,sha256=MpgDeIdGqNsiSKLOEDBnr5Z0eUbhHnqVEmAuoIfJmFU,12296
35
- crawlo/downloader/hybrid_downloader.py,sha256=nn_cD4SOKj0_TE2zuLfXfJhQppeftZDL5u8DZuITwVQ,8256
37
+ crawlo/downloader/hybrid_downloader.py,sha256=dNnFeegRnyLaOxTWI6XrWKqqVPx80AZBZNgmrcKRVBM,8240
36
38
  crawlo/downloader/playwright_downloader.py,sha256=L-TVzG7cYfuBlqW0XSZuz5C_r9fpJrmYNcoQ-cDEna4,16663
37
- crawlo/downloader/selenium_downloader.py,sha256=ho0ovIVU7e99xMT2_Clgj6SIH3GjxPx7yPN85C0_w9o,21508
39
+ crawlo/downloader/selenium_downloader.py,sha256=P8GuhEw6OYVeN3oeksuBLpUJCELXiu0mAR23X6IIOAA,21508
38
40
  crawlo/extension/__init__.py,sha256=g7Nj1vm8L8n9ZmIbTzOXMYTkMSAJj4pw8TefaDyd52k,1563
39
- crawlo/extension/health_check.py,sha256=Yiu50Dkn_rIS6NnzurIoQmwDM2d-SmEetO1d5wor2Xc,5668
41
+ crawlo/extension/health_check.py,sha256=stDpyP4gOzAdbBlPbSf0rge0QounAhF8CtrGq5fa_7c,5657
40
42
  crawlo/extension/log_interval.py,sha256=2R3XVdM1grDN8wh9TTHRB_WmQypCr5YSGvESNDnS16s,2474
41
43
  crawlo/extension/log_stats.py,sha256=5CpcTHj0lCtFvhBTI3pG41gkiL_hPtv5FWY87L6EVjM,2989
42
44
  crawlo/extension/logging_extension.py,sha256=euecjvj71aK6dElMzYCuNXiNb9jO8qQmL5QspQ1xoz8,1669
@@ -48,18 +50,18 @@ crawlo/filters/aioredis_filter.py,sha256=oveE9qFRtK8C9FZGRSZusRakXcAF-B2vkwbpsc0
48
50
  crawlo/filters/memory_filter.py,sha256=FzGJPhVKfZ8P23kP6de-VSfE8oVMjjpfWzKJIdiMtZU,9529
49
51
  crawlo/items/__init__.py,sha256=rFpx1qFBo0Ik7bSdnXC8EVTJUOQdoJYGVdhYjaH00nk,409
50
52
  crawlo/items/base.py,sha256=hwGJEdFWOdaZfalFX8umRkh_HUWLEbCjvq4j70fplMQ,598
51
- crawlo/items/fields.py,sha256=fpS0vlRPpZYjTaMDgI9Q8z_YQqruwf6fi4Dgm6R2oEk,1854
53
+ crawlo/items/fields.py,sha256=l-DIwK6CCpdzNvf6ELz7Ckc7YCghZD9UCXA8vhNn2UE,1852
52
54
  crawlo/items/items.py,sha256=OmVEvMmgofMU95GkaiWkfNQ2fjsH2fY9sw3SKcmUhLs,3478
53
55
  crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
54
56
  crawlo/middleware/default_header.py,sha256=PHz_VWRZga1gqkpTGY_lDoG2c_ktt03dDacVKUzNu_I,5348
55
57
  crawlo/middleware/download_delay.py,sha256=2iWnJFtWDlqDy5MsAob8TPiJQoiz9v21yatkBI0eptg,3542
56
58
  crawlo/middleware/middleware_manager.py,sha256=G9R9SnDVey_zS2k1zm_358EPHDxRWfP953I4QavL7P0,6348
57
- crawlo/middleware/offsite.py,sha256=Y1SQaNjZOmWPNCHy884X0hfwVR31sIdJ1ofv83n0NDU,4142
58
- crawlo/middleware/proxy.py,sha256=eHeEDaCm72cpipXM7b-KO3i8ScYMsPuu5byLZUFuHhY,15604
59
- crawlo/middleware/request_ignore.py,sha256=cjjwtACdbH4Me0NF7RwYn6cJQNlrJktO_ZaRE5OjiF8,2665
60
- crawlo/middleware/response_code.py,sha256=Nk5fQtgyzZVOfFvFcgF442rk9hXBm9ixzTh4nH8BhSM,5099
61
- crawlo/middleware/response_filter.py,sha256=tUe5Hc2N8p5_4kO8pdDa6sChUBkP2Mk7XAo762ScZOk,4386
62
- crawlo/middleware/retry.py,sha256=D_v4-8grofGsLkGJIhLKklMbdHO0gtij7DIS4G52NJc,4248
59
+ crawlo/middleware/offsite.py,sha256=FZJwsWuAHfgojLfbuXmnyedGwEbo2BAcGQbmL2jOHxU,4140
60
+ crawlo/middleware/proxy.py,sha256=FUwcXhUm-rH_zDVsIeTwC-qnoHZDogLkc9V7uSYwMvo,15606
61
+ crawlo/middleware/request_ignore.py,sha256=7qdX4zAimjSGwdod_aWUbOTfzLBWZ5KzLVFchGMCxCI,2663
62
+ crawlo/middleware/response_code.py,sha256=0_NbiCzLgJmSuSSF2_jqpypWYy0ES4GV-0iWQPLfYLc,5097
63
+ crawlo/middleware/response_filter.py,sha256=tVGr06bfJBR3xAHI2G5c3WimFsGHu8qoJtDcsVuCATU,4384
64
+ crawlo/middleware/retry.py,sha256=Acfo95B9wF8fQTCQIqluZOS2hHdnknQu_FOHvpGKJp0,4248
63
65
  crawlo/network/__init__.py,sha256=bvEnpEUBZJ79URfNZbsHhsBKna54hM2-x_BV8eotTA4,418
64
66
  crawlo/network/request.py,sha256=amZdRvDJvFpx82thtWwv6uo5uv9vtCwbvw3jAc3sCrY,12633
65
67
  crawlo/network/response.py,sha256=QwJhL3xJfPVy_gwtGrg61oAgaqCoCmjyj1Ug7Zju7Pg,13060
@@ -67,13 +69,13 @@ crawlo/pipelines/__init__.py,sha256=FDe2Pr5tiHtV8hFlheElRO_O1aVKvSWlkTcAl9BXAKA,
67
69
  crawlo/pipelines/bloom_dedup_pipeline.py,sha256=n0Ay7MtIEJ8L4Otiha4zRvI9toFUSNFugTNubi-Q3aw,5798
68
70
  crawlo/pipelines/console_pipeline.py,sha256=bwe5hZgaVSWmh3R8XpOaaeAjJme-Ttrpo6G6f1cnLIg,1287
69
71
  crawlo/pipelines/csv_pipeline.py,sha256=qbXZoqq4FIR9QkUGpC0ryWzmqGJSrM2bxmWLM4I1nXM,12490
70
- crawlo/pipelines/database_dedup_pipeline.py,sha256=6_zKtCNgFBPJyTI3Mk4l75fMQ2UQQGKFfNTpZqGs_zI,8224
72
+ crawlo/pipelines/database_dedup_pipeline.py,sha256=GTfnSk-l_Uuultsaf8QH-vkTLgWgXTLYgafpLG7baKY,8182
71
73
  crawlo/pipelines/json_pipeline.py,sha256=wrCsh8YInmcPLAkhPrHObMx89VZfhf-c7qRrYsTixPE,8585
72
- crawlo/pipelines/memory_dedup_pipeline.py,sha256=oQcBODO-I2p6B7Nm_klXvuhzSMIHP-JWwC4_o6Gkgcc,3954
74
+ crawlo/pipelines/memory_dedup_pipeline.py,sha256=Buh_j5cup_DSI8Lh9RM4nS3YRTIXE9-_Xsnnlah7dbk,3943
73
75
  crawlo/pipelines/mongo_pipeline.py,sha256=PohTKTGw3QRvuP-T6SrquwW3FAHSno8jQ2D2cH_d75U,5837
74
- crawlo/pipelines/mysql_pipeline.py,sha256=RRKMuO-7BTomyRFYCmDpfvjBTcU4SGdjGDV4wBeKWck,13796
76
+ crawlo/pipelines/mysql_pipeline.py,sha256=Zb8PuJf8PK42dz1aeLjOlLZg26SLr-a_NtQY0hBaBfw,13790
75
77
  crawlo/pipelines/pipeline_manager.py,sha256=g7lQBllYpKLymDWShjsR4SsXG41XnzD5mUR0UDBt0ZQ,2419
76
- crawlo/pipelines/redis_dedup_pipeline.py,sha256=U2mms0XBkTgC8_DywIigGboChjUH_l0-cfo_ioYQNsI,6333
78
+ crawlo/pipelines/redis_dedup_pipeline.py,sha256=8HZJppVYPrUkNoeybUKgdrFCQ36GK9r95fYGYTvjlxU,6329
77
79
  crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
78
80
  crawlo/queue/pqueue.py,sha256=qTFOuvEXsYEZbm0ULjsOeZo0XtSsZ-SHpx7nFEtmluE,1095
79
81
  crawlo/queue/queue_manager.py,sha256=wdQq0UYJcmyw9JPXiro2GP3q58vtjeIV3ouWyFmuhBc,13211
@@ -88,8 +90,8 @@ crawlo/templates/project/items.py.tmpl,sha256=8_3DBA8HrS2XbfHzsMZNJiZbFY6fDJUUMF
88
90
  crawlo/templates/project/middlewares.py.tmpl,sha256=fxHqi-Sjec5GiHJciprOU-6SAUTzM728NlZckIqf9hM,4278
89
91
  crawlo/templates/project/pipelines.py.tmpl,sha256=j9oqEhCezmmHlBhMWgYtlgup4jhWnMlv6AEiAOHODkg,2704
90
92
  crawlo/templates/project/run.py.tmpl,sha256=yOxpPkyffXLwa7NDx2Y96c8U-QN81_3mqZSuB526DNs,1271
91
- crawlo/templates/project/settings.py.tmpl,sha256=1481CdB6VdDzNXyYe0a5Qn5z-UJ0ELIELuSVbExnrOg,12167
92
- crawlo/templates/project/settings_distributed.py.tmpl,sha256=Nli-qR-UB4TXAq4mXjx17y-yAv46NwwpcVidjGeM00A,4321
93
+ crawlo/templates/project/settings.py.tmpl,sha256=B5iVv623K9ub4reJzznFEQb-RMFhM3nsIEcqP32wYyE,12041
94
+ crawlo/templates/project/settings_distributed.py.tmpl,sha256=N2sK8JjzwEEVEz_Tjk3idELmxTRhaVFcTO6I90Y7B-U,4380
93
95
  crawlo/templates/project/settings_gentle.py.tmpl,sha256=ljXf9vKV2c-cN8yvf5U4UWsPWrMVmJUfbXladIdS2mg,3320
94
96
  crawlo/templates/project/settings_high_performance.py.tmpl,sha256=3wf8fFYZ5EVE2742JlcwwrPF794vEIEmbxFSbqyGnJQ,5434
95
97
  crawlo/templates/project/settings_simple.py.tmpl,sha256=UKfSEZFAXDywswwws9SvR5vBMng2maJ37DJMfK0nAmI,2365
@@ -104,24 +106,24 @@ crawlo/tools/distributed_coordinator.py,sha256=QM4ECyhTba9jtV7-AkOuT_lDJObeKVeBs
104
106
  crawlo/tools/retry_mechanism.py,sha256=VbHo9-pT8ADbwyPuKcRfeUo4BtVZQy6yvrSvugoiZRA,8021
105
107
  crawlo/tools/scenario_adapter.py,sha256=pzysL1B2uQ1ZSEncVHd9Hv2viHNgaxP44YAUcDcppfw,9660
106
108
  crawlo/utils/__init__.py,sha256=5unISyKR7pZdj7EwP5tadGYKvyb_W1uoC52YzqSbxis,592
107
- crawlo/utils/batch_processor.py,sha256=wnpWL41Su6gSvbzNgRWv7sRxKzBFV0J27V9gC3WsLJc,9155
108
- crawlo/utils/controlled_spider_mixin.py,sha256=os5v8eFX08Nbqm48W-xWdY04LboqDgOgOL-93IX9zvg,16914
109
+ crawlo/utils/batch_processor.py,sha256=8LNy-K2SrQVUxmGEWxQyYw_j9M-erN4Ie7O4d3zpBvM,9142
110
+ crawlo/utils/controlled_spider_mixin.py,sha256=tv-fyy5OLQr4FKlYSUWisVKmdQq5IR1GflTSwoDRq7g,16909
109
111
  crawlo/utils/date_tools.py,sha256=gsgHweZFOzeW-K91wRlBEg_XLKBCXl7UZJfnjI_W-J0,9197
110
112
  crawlo/utils/db_helper.py,sha256=ZqOt1d3mErVv4TOvoWlov0niUxORB9aHByTmMoNFIDw,10902
111
- crawlo/utils/enhanced_error_handler.py,sha256=o6oxCtWFcAGslON3FZNRgxvHutd9z0tqbc0R5kmF_lk,14265
112
- crawlo/utils/env_config.py,sha256=9hJ4T-ufAzWPZugytgLMZStZxD17JaE900S8xpGoyn0,2974
113
- crawlo/utils/error_handler.py,sha256=7AHwZHJD-CTq0DNOwbDjUvzdbushDI_xWJiwe4KzBeQ,4411
113
+ crawlo/utils/enhanced_error_handler.py,sha256=fJC__rnYNKTNUHNbgjZtT846HoE31qyGbPft9bwyYLU,14214
114
+ crawlo/utils/env_config.py,sha256=4ZhDMoAEWREDNOj0mzdghSE8ScfrsI_nNJe6aidTw4Q,2957
115
+ crawlo/utils/error_handler.py,sha256=N6suB8Utcn7tp6WRJ8gKECr0RIAG86dcOXdwOr998OE,4367
114
116
  crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
115
117
  crawlo/utils/large_scale_config.py,sha256=lsraHTAQx3sMPjTnCBY_SVIpkuIBUb3zD9eFvmccOOM,8440
116
- crawlo/utils/large_scale_helper.py,sha256=ZazAI7KV3V-3hzc4a3BWxTXfEO2XIEBMzxTLM9S1l_Q,12500
118
+ crawlo/utils/large_scale_helper.py,sha256=_0VgLvO10G2GeTQeNlL0K5vYjIMlTiX3geigM5Egtls,12477
117
119
  crawlo/utils/log.py,sha256=YD2FfXuuE2MC9ZdQQZ0H7KysE7l_LHZqQepaTPlcApo,4133
118
- crawlo/utils/performance_monitor.py,sha256=ku_QAx3MX8U-Dfd5fqMr-YV5j1srYJbtF_5jdVJx6tY,9879
120
+ crawlo/utils/performance_monitor.py,sha256=2a3hFLQJnxgfbv_iMhsqPYOuX9BC4xQcicu_e3IVcrI,9861
119
121
  crawlo/utils/queue_helper.py,sha256=gFmkh1jKlIcN1rmo2Jl6vYcLP5ByUWlfHO9eNlZPBLs,4918
120
- crawlo/utils/redis_connection_pool.py,sha256=rGaKcn2YVeP_bA38svkl3A-lU3DpfX6x0TzdaxWvIlo,10665
121
- crawlo/utils/redis_key_validator.py,sha256=5lft8SPbHkBf9NdrQaW13R-fXLJoMBkJlc1w38wuPmU,5836
122
+ crawlo/utils/redis_connection_pool.py,sha256=yKB_CXWaLEc5srkD55n48ukzVpRHUCtHWFAdUkM4v0o,10644
123
+ crawlo/utils/redis_key_validator.py,sha256=dOL_LVZJhzMZUZ3CmU57Yy0skWABeepI1GCecSzXstw,5825
122
124
  crawlo/utils/request.py,sha256=ejdKpTwc-HE04HQybafhOVywzz57IV3pY0YMkSLyGUo,9065
123
- crawlo/utils/request_serializer.py,sha256=bZhuonZV2AdB_X0aje7sDljqWAIrEzUYwEaxXytaWsg,8784
124
- crawlo/utils/spider_loader.py,sha256=V0CBTicJBYBZafhwLfDEfuEc_hJ2mSoiptT6qKufI9U,2249
125
+ crawlo/utils/request_serializer.py,sha256=2Wd55pQDtBWnkhtxHxjzROQ_-IjqaqbT4qaTMtqO6R8,8754
126
+ crawlo/utils/spider_loader.py,sha256=xNzQb7qhQ7TqZsfFtCLpuVcsGi-USriZosU0YSBr9II,2233
125
127
  crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
126
128
  crawlo/utils/tools.py,sha256=uy7qw5Z1BIhyEgiHENvtM7WoGCJxlS8EX3PmOA7ouCo,275
127
129
  crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
@@ -213,8 +215,8 @@ tests/test_template_content.py,sha256=URwjlAzMCdUN0sW_OupUcuSNMxp1OKgW79JOpkLPXn
213
215
  tests/test_template_redis_key.py,sha256=dOFutic8CL3tOzGbYhWbMrYiXZ8R3fhNoF5VKax5Iy0,4946
214
216
  tests/test_tools.py,sha256=fgzXL2L7eBV_nGjeMxH8IMhfc0dviQ80XgzZkJp_4dA,5266
215
217
  tests/tools_example.py,sha256=uXNS4xXJ-OD_xInAn2zjKLG_nlbgVGXZLoJtfhaG9lI,7926
216
- crawlo-1.2.2.dist-info/METADATA,sha256=Kd8W53gp0eac7V4kT-IGgdpQ68iHd22tmiV4En9v8xs,20484
217
- crawlo-1.2.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
218
- crawlo-1.2.2.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
219
- crawlo-1.2.2.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
220
- crawlo-1.2.2.dist-info/RECORD,,
218
+ crawlo-1.2.3.dist-info/METADATA,sha256=nodmfHvQysGLjeFZLZN860ZJnBs53DMJDCl7ypcnjtA,20484
219
+ crawlo-1.2.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
220
+ crawlo-1.2.3.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
221
+ crawlo-1.2.3.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
222
+ crawlo-1.2.3.dist-info/RECORD,,
{crawlo-1.2.2.dist-info → crawlo-1.2.3.dist-info}/WHEEL RENAMED
File without changes