crawlo 1.1.3__py3-none-any.whl → 1.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of crawlo might be problematic. Click here for more details.

Files changed (118) hide show
  1. crawlo/__init__.py +34 -34
  2. crawlo/__version__.py +1 -1
  3. crawlo/cli.py +40 -40
  4. crawlo/commands/__init__.py +13 -13
  5. crawlo/commands/check.py +594 -594
  6. crawlo/commands/genspider.py +151 -151
  7. crawlo/commands/list.py +155 -155
  8. crawlo/commands/run.py +285 -285
  9. crawlo/commands/startproject.py +196 -196
  10. crawlo/commands/stats.py +188 -188
  11. crawlo/commands/utils.py +186 -186
  12. crawlo/config.py +279 -279
  13. crawlo/core/__init__.py +2 -2
  14. crawlo/core/engine.py +171 -171
  15. crawlo/core/enhanced_engine.py +189 -189
  16. crawlo/core/processor.py +40 -40
  17. crawlo/core/scheduler.py +165 -165
  18. crawlo/crawler.py +1027 -1027
  19. crawlo/downloader/__init__.py +242 -242
  20. crawlo/downloader/aiohttp_downloader.py +212 -212
  21. crawlo/downloader/cffi_downloader.py +251 -251
  22. crawlo/downloader/httpx_downloader.py +259 -259
  23. crawlo/event.py +11 -11
  24. crawlo/exceptions.py +81 -81
  25. crawlo/extension/__init__.py +38 -31
  26. crawlo/extension/health_check.py +142 -0
  27. crawlo/extension/log_interval.py +58 -49
  28. crawlo/extension/log_stats.py +82 -44
  29. crawlo/extension/logging_extension.py +44 -35
  30. crawlo/extension/memory_monitor.py +89 -0
  31. crawlo/extension/performance_profiler.py +118 -0
  32. crawlo/extension/request_recorder.py +108 -0
  33. crawlo/filters/__init__.py +154 -154
  34. crawlo/filters/aioredis_filter.py +241 -241
  35. crawlo/filters/memory_filter.py +269 -269
  36. crawlo/items/__init__.py +23 -23
  37. crawlo/items/base.py +21 -21
  38. crawlo/items/fields.py +53 -53
  39. crawlo/items/items.py +104 -104
  40. crawlo/middleware/__init__.py +21 -21
  41. crawlo/middleware/default_header.py +32 -32
  42. crawlo/middleware/download_delay.py +28 -28
  43. crawlo/middleware/middleware_manager.py +135 -135
  44. crawlo/middleware/proxy.py +248 -248
  45. crawlo/middleware/request_ignore.py +30 -30
  46. crawlo/middleware/response_code.py +18 -18
  47. crawlo/middleware/response_filter.py +26 -26
  48. crawlo/middleware/retry.py +124 -124
  49. crawlo/mode_manager.py +200 -200
  50. crawlo/network/__init__.py +21 -21
  51. crawlo/network/request.py +311 -311
  52. crawlo/network/response.py +271 -271
  53. crawlo/pipelines/__init__.py +21 -21
  54. crawlo/pipelines/bloom_dedup_pipeline.py +156 -156
  55. crawlo/pipelines/console_pipeline.py +39 -39
  56. crawlo/pipelines/csv_pipeline.py +316 -316
  57. crawlo/pipelines/database_dedup_pipeline.py +224 -224
  58. crawlo/pipelines/json_pipeline.py +218 -218
  59. crawlo/pipelines/memory_dedup_pipeline.py +115 -115
  60. crawlo/pipelines/mongo_pipeline.py +132 -117
  61. crawlo/pipelines/mysql_pipeline.py +317 -195
  62. crawlo/pipelines/pipeline_manager.py +56 -56
  63. crawlo/pipelines/redis_dedup_pipeline.py +162 -162
  64. crawlo/project.py +153 -153
  65. crawlo/queue/pqueue.py +37 -37
  66. crawlo/queue/queue_manager.py +307 -307
  67. crawlo/queue/redis_priority_queue.py +208 -208
  68. crawlo/settings/__init__.py +7 -7
  69. crawlo/settings/default_settings.py +278 -244
  70. crawlo/settings/setting_manager.py +99 -99
  71. crawlo/spider/__init__.py +639 -639
  72. crawlo/stats_collector.py +59 -59
  73. crawlo/subscriber.py +131 -106
  74. crawlo/task_manager.py +30 -30
  75. crawlo/templates/crawlo.cfg.tmpl +10 -10
  76. crawlo/templates/project/__init__.py.tmpl +3 -3
  77. crawlo/templates/project/items.py.tmpl +17 -17
  78. crawlo/templates/project/middlewares.py.tmpl +111 -87
  79. crawlo/templates/project/pipelines.py.tmpl +97 -341
  80. crawlo/templates/project/run.py.tmpl +251 -251
  81. crawlo/templates/project/settings.py.tmpl +279 -250
  82. crawlo/templates/project/spiders/__init__.py.tmpl +5 -5
  83. crawlo/templates/spider/spider.py.tmpl +142 -178
  84. crawlo/utils/__init__.py +7 -7
  85. crawlo/utils/controlled_spider_mixin.py +439 -439
  86. crawlo/utils/date_tools.py +233 -233
  87. crawlo/utils/db_helper.py +343 -343
  88. crawlo/utils/func_tools.py +82 -82
  89. crawlo/utils/large_scale_config.py +286 -286
  90. crawlo/utils/large_scale_helper.py +343 -343
  91. crawlo/utils/log.py +128 -128
  92. crawlo/utils/queue_helper.py +175 -175
  93. crawlo/utils/request.py +267 -267
  94. crawlo/utils/request_serializer.py +219 -219
  95. crawlo/utils/spider_loader.py +62 -62
  96. crawlo/utils/system.py +11 -11
  97. crawlo/utils/tools.py +4 -4
  98. crawlo/utils/url.py +39 -39
  99. crawlo-1.1.4.dist-info/METADATA +403 -0
  100. crawlo-1.1.4.dist-info/RECORD +117 -0
  101. examples/__init__.py +7 -7
  102. examples/controlled_spider_example.py +205 -205
  103. tests/__init__.py +7 -7
  104. tests/test_final_validation.py +153 -153
  105. tests/test_proxy_health_check.py +32 -32
  106. tests/test_proxy_middleware_integration.py +136 -136
  107. tests/test_proxy_providers.py +56 -56
  108. tests/test_proxy_stats.py +19 -19
  109. tests/test_proxy_strategies.py +59 -59
  110. tests/test_redis_config.py +28 -28
  111. tests/test_redis_queue.py +224 -224
  112. tests/test_request_serialization.py +70 -70
  113. tests/test_scheduler.py +241 -241
  114. crawlo-1.1.3.dist-info/METADATA +0 -635
  115. crawlo-1.1.3.dist-info/RECORD +0 -113
  116. {crawlo-1.1.3.dist-info → crawlo-1.1.4.dist-info}/WHEEL +0 -0
  117. {crawlo-1.1.3.dist-info → crawlo-1.1.4.dist-info}/entry_points.txt +0 -0
  118. {crawlo-1.1.3.dist-info → crawlo-1.1.4.dist-info}/top_level.txt +0 -0
@@ -1,100 +1,100 @@
1
- #!/usr/bin/python
2
- # -*- coding: UTF-8 -*-
3
- import json
4
- from copy import deepcopy
5
- from importlib import import_module
6
- from collections.abc import MutableMapping
7
-
8
- from crawlo.settings import default_settings
9
-
10
-
11
- class SettingManager(MutableMapping):
12
-
13
- def __init__(self, values=None):
14
- self.attributes = {}
15
- self.set_settings(default_settings)
16
- self.update_attributes(values)
17
-
18
- def get(self, key, default=None):
19
- """安全获取值,不触发递归"""
20
- value = self.attributes.get(key, default)
21
- return value if value is not None else default
22
-
23
- def get_int(self, key, default=0):
24
- return int(self.get(key, default=default))
25
-
26
- def get_float(self, key, default=0.0):
27
- return float(self.get(key, default=default))
28
-
29
- def get_bool(self, key, default=False):
30
- got = self.get(key, default=default)
31
- if isinstance(got, bool):
32
- return got
33
- if isinstance(got, (int, float)):
34
- return bool(got)
35
- got_lower = str(got).strip().lower()
36
- if got_lower in ('1', 'true'):
37
- return True
38
- if got_lower in ('0', 'false'):
39
- return False
40
- raise ValueError(
41
- f"Unsupported value for boolean setting: {got}. "
42
- "Supported values are: 0/1, True/False, '0'/'1', 'True'/'False' (case-insensitive)."
43
- )
44
-
45
- def get_list(self, key, default=None):
46
- values = self.get(key, default or [])
47
- if isinstance(values, str):
48
- return [v.strip() for v in values.split(',') if v.strip()]
49
- try:
50
- return list(values)
51
- except TypeError:
52
- return [values]
53
-
54
- def get_dict(self, key, default=None):
55
- value = self.get(key, default or {})
56
- if isinstance(value, str):
57
- value = json.loads(value)
58
- try:
59
- return dict(value)
60
- except TypeError:
61
- return value
62
-
63
- def set(self, key, value):
64
- self.attributes[key] = value
65
-
66
- def set_settings(self, module):
67
- if isinstance(module, str):
68
- module = import_module(module)
69
- for key in dir(module):
70
- if key.isupper():
71
- self.set(key, getattr(module, key))
72
-
73
- # 实现 MutableMapping 必须的方法
74
- def __getitem__(self, item):
75
- return self.attributes[item]
76
-
77
- def __setitem__(self, key, value):
78
- self.set(key, value)
79
-
80
- def __delitem__(self, key):
81
- del self.attributes[key]
82
-
83
- def __iter__(self):
84
- return iter(self.attributes)
85
-
86
- def __len__(self):
87
- return len(self.attributes)
88
-
89
- def __str__(self):
90
- return f'<Settings: {self.attributes}>'
91
-
92
- __repr__ = __str__
93
-
94
- def update_attributes(self, attributes):
95
- if attributes is not None:
96
- for key, value in attributes.items():
97
- self.set(key, value)
98
-
99
- def copy(self):
1
+ #!/usr/bin/python
2
+ # -*- coding: UTF-8 -*-
3
+ import json
4
+ from copy import deepcopy
5
+ from importlib import import_module
6
+ from collections.abc import MutableMapping
7
+
8
+ from crawlo.settings import default_settings
9
+
10
+
11
+ class SettingManager(MutableMapping):
12
+
13
+ def __init__(self, values=None):
14
+ self.attributes = {}
15
+ self.set_settings(default_settings)
16
+ self.update_attributes(values)
17
+
18
+ def get(self, key, default=None):
19
+ """安全获取值,不触发递归"""
20
+ value = self.attributes.get(key, default)
21
+ return value if value is not None else default
22
+
23
+ def get_int(self, key, default=0):
24
+ return int(self.get(key, default=default))
25
+
26
+ def get_float(self, key, default=0.0):
27
+ return float(self.get(key, default=default))
28
+
29
+ def get_bool(self, key, default=False):
30
+ got = self.get(key, default=default)
31
+ if isinstance(got, bool):
32
+ return got
33
+ if isinstance(got, (int, float)):
34
+ return bool(got)
35
+ got_lower = str(got).strip().lower()
36
+ if got_lower in ('1', 'true'):
37
+ return True
38
+ if got_lower in ('0', 'false'):
39
+ return False
40
+ raise ValueError(
41
+ f"Unsupported value for boolean setting: {got}. "
42
+ "Supported values are: 0/1, True/False, '0'/'1', 'True'/'False' (case-insensitive)."
43
+ )
44
+
45
+ def get_list(self, key, default=None):
46
+ values = self.get(key, default or [])
47
+ if isinstance(values, str):
48
+ return [v.strip() for v in values.split(',') if v.strip()]
49
+ try:
50
+ return list(values)
51
+ except TypeError:
52
+ return [values]
53
+
54
+ def get_dict(self, key, default=None):
55
+ value = self.get(key, default or {})
56
+ if isinstance(value, str):
57
+ value = json.loads(value)
58
+ try:
59
+ return dict(value)
60
+ except TypeError:
61
+ return value
62
+
63
+ def set(self, key, value):
64
+ self.attributes[key] = value
65
+
66
+ def set_settings(self, module):
67
+ if isinstance(module, str):
68
+ module = import_module(module)
69
+ for key in dir(module):
70
+ if key.isupper():
71
+ self.set(key, getattr(module, key))
72
+
73
+ # 实现 MutableMapping 必须的方法
74
+ def __getitem__(self, item):
75
+ return self.attributes[item]
76
+
77
+ def __setitem__(self, key, value):
78
+ self.set(key, value)
79
+
80
+ def __delitem__(self, key):
81
+ del self.attributes[key]
82
+
83
+ def __iter__(self):
84
+ return iter(self.attributes)
85
+
86
+ def __len__(self):
87
+ return len(self.attributes)
88
+
89
+ def __str__(self):
90
+ return f'<Settings: {self.attributes}>'
91
+
92
+ __repr__ = __str__
93
+
94
+ def update_attributes(self, attributes):
95
+ if attributes is not None:
96
+ for key, value in attributes.items():
97
+ self.set(key, value)
98
+
99
+ def copy(self):
100
100
  return deepcopy(self)