crawlo 1.2.5__py3-none-any.whl → 1.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawlo/__version__.py +1 -1
- crawlo/core/engine.py +3 -1
- crawlo/core/scheduler.py +102 -6
- crawlo/filters/aioredis_filter.py +44 -91
- crawlo/queue/queue_manager.py +47 -8
- crawlo/queue/redis_priority_queue.py +9 -2
- crawlo/settings/default_settings.py +5 -7
- crawlo/templates/project/settings.py.tmpl +3 -39
- crawlo/templates/project/settings_distributed.py.tmpl +4 -1
- crawlo/templates/project/settings_gentle.py.tmpl +59 -86
- crawlo/templates/project/settings_high_performance.py.tmpl +84 -99
- crawlo/templates/project/settings_simple.py.tmpl +72 -76
- crawlo/templates/run.py.tmpl +1 -3
- crawlo/utils/redis_connection_pool.py +19 -2
- {crawlo-1.2.5.dist-info → crawlo-1.2.6.dist-info}/METADATA +1 -1
- {crawlo-1.2.5.dist-info → crawlo-1.2.6.dist-info}/RECORD +19 -19
- {crawlo-1.2.5.dist-info → crawlo-1.2.6.dist-info}/WHEEL +0 -0
- {crawlo-1.2.5.dist-info → crawlo-1.2.6.dist-info}/entry_points.txt +0 -0
- {crawlo-1.2.5.dist-info → crawlo-1.2.6.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
crawlo/__init__.py,sha256=1tc6uUDF1yRNU7K_k-Dl6h9FGy7Jp8fdhRsXu9PctFI,1312
|
|
2
|
-
crawlo/__version__.py,sha256=
|
|
2
|
+
crawlo/__version__.py,sha256=vMQK58X8_YZGKzRm0ThvPAKFtpfyejGmUnDrY9RQ13w,22
|
|
3
3
|
crawlo/cli.py,sha256=8OD2us3YYG55AMTm_3YmHiZ8_Plokkx6p2DVR7w0JGs,2872
|
|
4
4
|
crawlo/config.py,sha256=zbFn2HfcbDcD1QOSgNOB9kjw12UDb7zo4Y7RLcJJirQ,9506
|
|
5
5
|
crawlo/config_validator.py,sha256=eQWrSY4xNnbDgwAmPOENyMb0wb3rdzOL55yBCKmpIak,9883
|
|
@@ -25,9 +25,9 @@ crawlo/commands/startproject.py,sha256=8tMokyCBN4nEpWjAca3NQorU5t8WR5ffBQZ7eyvUW
|
|
|
25
25
|
crawlo/commands/stats.py,sha256=fCNk7ix6v9PrscC-wKQl6ddA4xaQLogrN_1SNEj66tk,6128
|
|
26
26
|
crawlo/commands/utils.py,sha256=3nP8bMgPH36ELqg1ilzPhHS873ncNvZNxOEDYY4uDm4,5087
|
|
27
27
|
crawlo/core/__init__.py,sha256=PnFyJdVNHBoPmV1sW0AHQXijeoSTQ8cMYrbNM1JK8kA,41
|
|
28
|
-
crawlo/core/engine.py,sha256=
|
|
28
|
+
crawlo/core/engine.py,sha256=yCpwwxfKKYQ8r1ehjOuwKpPQ8YHpK8DqbFCatrHeCSw,13854
|
|
29
29
|
crawlo/core/processor.py,sha256=qmCqAeqhwYu-UE86evYesaGt9qpuSIfH-ZIZKcXFCZc,1140
|
|
30
|
-
crawlo/core/scheduler.py,sha256=
|
|
30
|
+
crawlo/core/scheduler.py,sha256=MDItA0ZXAaHup18NTSBBeeCKg8cqEBGIYCt0YUceIZU,11845
|
|
31
31
|
crawlo/data/__init__.py,sha256=8MgDxcMhx-emFARcLAw_ODOZNz0neYBcx7kEbzothd8,116
|
|
32
32
|
crawlo/data/user_agents.py,sha256=Zpm9nhP1Q3CVLtRGRBmpMFYW8qP-GxlDBeAOWF_2ad0,4698
|
|
33
33
|
crawlo/downloader/__init__.py,sha256=MlstaKfW-WLXNuZs7tb7cG_wG2sQLw2hdWmUjZEIH7c,8299
|
|
@@ -46,7 +46,7 @@ crawlo/extension/memory_monitor.py,sha256=4aszl3C0GMQbqFhGZjZq5iQuXQR1sOz06VHjjE
|
|
|
46
46
|
crawlo/extension/performance_profiler.py,sha256=EPiNuXuPPDU0Jtgy8arYHpr_8ASK13cCI2BytdJnu_I,4899
|
|
47
47
|
crawlo/extension/request_recorder.py,sha256=RC23yzXClnVv9j2ljQvjBkUfWznfnDHsrQejKhE9y5E,4074
|
|
48
48
|
crawlo/filters/__init__.py,sha256=XC_Q4ykZtSNYizYlAcehVwBBNO3lZ2zuWwafzXiuWyQ,4241
|
|
49
|
-
crawlo/filters/aioredis_filter.py,sha256=
|
|
49
|
+
crawlo/filters/aioredis_filter.py,sha256=XixK3DD5QbCLOw3Me2YdtMkxQpXOT75FE-GiVr_PUGc,8245
|
|
50
50
|
crawlo/filters/memory_filter.py,sha256=VJO0UFRYGxmV8dj4G1subsQ-FtvPcGLbvd7IVtqXnOs,9260
|
|
51
51
|
crawlo/items/__init__.py,sha256=bqekZrRlDhxfWie0UbCs656TptYseoe9QJ67I4E7Elk,386
|
|
52
52
|
crawlo/items/base.py,sha256=tAYrPJgblp3ZEihDXvappdYc6pGdim6x2_9QSmMKI2o,577
|
|
@@ -78,23 +78,23 @@ crawlo/pipelines/pipeline_manager.py,sha256=vK87pAEmpGR24yl6Cr7ovCKag2oB5mruijfY
|
|
|
78
78
|
crawlo/pipelines/redis_dedup_pipeline.py,sha256=QCHybXm7E-AoqeIaI1ZoFJIoeOB9B6DDlShxpgTyaFg,6164
|
|
79
79
|
crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
80
80
|
crawlo/queue/pqueue.py,sha256=yzF900ko2RReVNZtfk_VV3PzSXEUALI6SHf97geuu6s,1058
|
|
81
|
-
crawlo/queue/queue_manager.py,sha256=
|
|
82
|
-
crawlo/queue/redis_priority_queue.py,sha256=
|
|
81
|
+
crawlo/queue/queue_manager.py,sha256=megRgpuEQ1cOWJAHaaSfqu2h3oM5j9sIb9H5uJ4DbfE,15002
|
|
82
|
+
crawlo/queue/redis_priority_queue.py,sha256=_0rROJk8VHMtxFXpCENsEQb_oOfn8DTPhcLQueD1uOM,13115
|
|
83
83
|
crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
|
|
84
|
-
crawlo/settings/default_settings.py,sha256=
|
|
84
|
+
crawlo/settings/default_settings.py,sha256=wBol4uHlSZtFXyi9bs4a12sr50_Mltpbl8KrRqPngNY,8962
|
|
85
85
|
crawlo/settings/setting_manager.py,sha256=0RYAk07qoJ5WTw_mvV4ECWGS2QNpCnGmBZVTHjqOVIg,3707
|
|
86
86
|
crawlo/spider/__init__.py,sha256=Z_rK23l5yt-DuwJPg8bcqodM_FIs4-iHLaKOimGumcE,20452
|
|
87
87
|
crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
|
|
88
|
-
crawlo/templates/run.py.tmpl,sha256=
|
|
88
|
+
crawlo/templates/run.py.tmpl,sha256=bk5TbuzxQEaF3wF3i754MT7Z4BqOk_nzSRDnJElpg14,1099
|
|
89
89
|
crawlo/templates/project/__init__.py.tmpl,sha256=f3ETIXw_O6K-lkL6lXM5znMPJW1FZYGFrwDs2BnHcnQ,58
|
|
90
90
|
crawlo/templates/project/items.py.tmpl,sha256=mt1Mm--H2Ouos3r7JPkYh0r33rgYJf1YOMz0OZy8TYs,297
|
|
91
91
|
crawlo/templates/project/middlewares.py.tmpl,sha256=T67p8j0laL4NJJ_3xzPM9yivgZRjTEMiEtEWLPwbkmw,4160
|
|
92
92
|
crawlo/templates/project/pipelines.py.tmpl,sha256=GBHYU0Jx8sKDCdGJp44FMSH7u2slxoFg6a-R9Uwg_-I,2608
|
|
93
|
-
crawlo/templates/project/settings.py.tmpl,sha256=
|
|
94
|
-
crawlo/templates/project/settings_distributed.py.tmpl,sha256=
|
|
95
|
-
crawlo/templates/project/settings_gentle.py.tmpl,sha256=
|
|
96
|
-
crawlo/templates/project/settings_high_performance.py.tmpl,sha256=
|
|
97
|
-
crawlo/templates/project/settings_simple.py.tmpl,sha256=
|
|
93
|
+
crawlo/templates/project/settings.py.tmpl,sha256=TbeOEFoGBct6MU2jbne20cRo7tPD0BRCbZ1DCvbqOEs,10718
|
|
94
|
+
crawlo/templates/project/settings_distributed.py.tmpl,sha256=5DdiSmEuAzqMuxQ4zAu3vLAvJk0bj-fxCuzofubrF48,5887
|
|
95
|
+
crawlo/templates/project/settings_gentle.py.tmpl,sha256=fhm4f8BArNN6d2eHpiUOblQOagLTyLYSzZ89jVBRJQQ,3839
|
|
96
|
+
crawlo/templates/project/settings_high_performance.py.tmpl,sha256=3BF_eSdxOl0KknwFipN9vh_gi-ZkEQsw1ZX1HvMxiww,4902
|
|
97
|
+
crawlo/templates/project/settings_simple.py.tmpl,sha256=cpm7VAAC3tA2RR2Znab3Ukf-2FpcdkY_IAEPK1m8MlQ,3752
|
|
98
98
|
crawlo/templates/project/spiders/__init__.py.tmpl,sha256=j_YKsw6HQMJyqlk3WUouP3bsr-XVxshRoSNakHBc00g,106
|
|
99
99
|
crawlo/templates/spider/spider.py.tmpl,sha256=jMhzyxpIpV_KigB-pmN-5mGMiYtu4mfQIOvpZcCGGJI,5055
|
|
100
100
|
crawlo/tools/__init__.py,sha256=5H6rAhjfNSqRMjjlLDVq-vEJWRFyCO-J6HN2kexnXJU,3671
|
|
@@ -119,7 +119,7 @@ crawlo/utils/large_scale_helper.py,sha256=HpcoyBTu8SZnQnS_F3dAL8WpQX5Xahe50H0ORt
|
|
|
119
119
|
crawlo/utils/log.py,sha256=A3lPyhD8kD88cV23KOL-_eT8g69xGQ5L1toDB2AO0mc,4005
|
|
120
120
|
crawlo/utils/performance_monitor.py,sha256=5B8d0Pjy8K2z_Gh6zhBdC-Hl6ldy55MDro30-wnE2mw,9576
|
|
121
121
|
crawlo/utils/queue_helper.py,sha256=xpUUTOqlU1xz2Pb9NKAVGo3AfAO-7Xvx8Lm1q65Dgck,4743
|
|
122
|
-
crawlo/utils/redis_connection_pool.py,sha256=
|
|
122
|
+
crawlo/utils/redis_connection_pool.py,sha256=Ng03XkCOQlWtnGfrMXBNLumJ6Iz9C8rZvMYEHdbmhHc,11066
|
|
123
123
|
crawlo/utils/redis_key_validator.py,sha256=-efQ44FaxrQdhbN3xT23M7O7a6L3IayumObNH3A_Nuc,5627
|
|
124
124
|
crawlo/utils/request.py,sha256=yoLB2rY8d78vgPjIWpdhY5SalIKjyLIvTG_UH6EMdVI,8798
|
|
125
125
|
crawlo/utils/request_serializer.py,sha256=oCGsLC_-Do7EuIREMQE01gINRtM0cC9X2JbW6ykkV74,8536
|
|
@@ -199,8 +199,8 @@ tests/test_template_content.py,sha256=5QAnhKZFDKg-_uFryllLMpCk3a1nCS44hMmYfXm8gW
|
|
|
199
199
|
tests/test_template_redis_key.py,sha256=U6L5HtnDyGp3s6-O4F_yG2Q2nNIGTqB_Q-ESv2EMeOU,4812
|
|
200
200
|
tests/test_tools.py,sha256=IWiu9JGV-5Ow0ivFtiDw62REht-8Hn7NfyR9rnYSlbU,5113
|
|
201
201
|
tests/tools_example.py,sha256=MfVBYemKvHs6MUbydgrJfhiGnygp5dRoIE-eIXCOR7M,7669
|
|
202
|
-
crawlo-1.2.
|
|
203
|
-
crawlo-1.2.
|
|
204
|
-
crawlo-1.2.
|
|
205
|
-
crawlo-1.2.
|
|
206
|
-
crawlo-1.2.
|
|
202
|
+
crawlo-1.2.6.dist-info/METADATA,sha256=Pe5YqJrdYaCh6nNn-Npxw5a3pHpIwq-BiR8FvxgyV5U,19185
|
|
203
|
+
crawlo-1.2.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
204
|
+
crawlo-1.2.6.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
|
|
205
|
+
crawlo-1.2.6.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
|
|
206
|
+
crawlo-1.2.6.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|