aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.7.dist-info}/LICENSE +1 -1
  2. aio_scrapy-2.1.7.dist-info/METADATA +147 -0
  3. aio_scrapy-2.1.7.dist-info/RECORD +134 -0
  4. {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.7.dist-info}/WHEEL +1 -1
  5. aioscrapy/VERSION +1 -1
  6. aioscrapy/cmdline.py +438 -5
  7. aioscrapy/core/downloader/__init__.py +522 -17
  8. aioscrapy/core/downloader/handlers/__init__.py +187 -5
  9. aioscrapy/core/downloader/handlers/aiohttp.py +190 -6
  10. aioscrapy/core/downloader/handlers/curl_cffi.py +126 -5
  11. aioscrapy/core/downloader/handlers/httpx.py +135 -5
  12. aioscrapy/core/downloader/handlers/pyhttpx.py +137 -5
  13. aioscrapy/core/downloader/handlers/requests.py +120 -2
  14. aioscrapy/core/downloader/handlers/webdriver/__init__.py +2 -0
  15. aioscrapy/core/downloader/handlers/webdriver/drissionpage.py +493 -0
  16. aioscrapy/core/downloader/handlers/webdriver/driverpool.py +234 -0
  17. aioscrapy/core/downloader/handlers/webdriver/playwright.py +498 -0
  18. aioscrapy/core/engine.py +381 -20
  19. aioscrapy/core/scheduler.py +350 -36
  20. aioscrapy/core/scraper.py +509 -33
  21. aioscrapy/crawler.py +392 -10
  22. aioscrapy/db/__init__.py +149 -0
  23. aioscrapy/db/absmanager.py +212 -6
  24. aioscrapy/db/aiomongo.py +292 -10
  25. aioscrapy/db/aiomysql.py +363 -10
  26. aioscrapy/db/aiopg.py +299 -2
  27. aioscrapy/db/aiorabbitmq.py +444 -4
  28. aioscrapy/db/aioredis.py +260 -11
  29. aioscrapy/dupefilters/__init__.py +110 -5
  30. aioscrapy/dupefilters/disk.py +124 -2
  31. aioscrapy/dupefilters/redis.py +598 -32
  32. aioscrapy/exceptions.py +151 -13
  33. aioscrapy/http/__init__.py +1 -1
  34. aioscrapy/http/headers.py +237 -3
  35. aioscrapy/http/request/__init__.py +257 -11
  36. aioscrapy/http/request/form.py +83 -3
  37. aioscrapy/http/request/json_request.py +121 -9
  38. aioscrapy/http/response/__init__.py +306 -33
  39. aioscrapy/http/response/html.py +42 -3
  40. aioscrapy/http/response/text.py +496 -49
  41. aioscrapy/http/response/web_driver.py +144 -0
  42. aioscrapy/http/response/xml.py +45 -3
  43. aioscrapy/libs/downloader/defaultheaders.py +66 -2
  44. aioscrapy/libs/downloader/downloadtimeout.py +91 -2
  45. aioscrapy/libs/downloader/ja3fingerprint.py +95 -2
  46. aioscrapy/libs/downloader/retry.py +192 -6
  47. aioscrapy/libs/downloader/stats.py +142 -0
  48. aioscrapy/libs/downloader/useragent.py +93 -2
  49. aioscrapy/libs/extensions/closespider.py +166 -4
  50. aioscrapy/libs/extensions/corestats.py +151 -1
  51. aioscrapy/libs/extensions/logstats.py +145 -1
  52. aioscrapy/libs/extensions/metric.py +370 -1
  53. aioscrapy/libs/extensions/throttle.py +235 -1
  54. aioscrapy/libs/pipelines/__init__.py +345 -2
  55. aioscrapy/libs/pipelines/csv.py +242 -0
  56. aioscrapy/libs/pipelines/excel.py +545 -0
  57. aioscrapy/libs/pipelines/mongo.py +132 -0
  58. aioscrapy/libs/pipelines/mysql.py +67 -0
  59. aioscrapy/libs/pipelines/pg.py +67 -0
  60. aioscrapy/libs/spider/depth.py +141 -3
  61. aioscrapy/libs/spider/httperror.py +144 -4
  62. aioscrapy/libs/spider/offsite.py +202 -2
  63. aioscrapy/libs/spider/referer.py +396 -21
  64. aioscrapy/libs/spider/urllength.py +97 -1
  65. aioscrapy/link.py +115 -8
  66. aioscrapy/logformatter.py +199 -8
  67. aioscrapy/middleware/absmanager.py +328 -2
  68. aioscrapy/middleware/downloader.py +218 -0
  69. aioscrapy/middleware/extension.py +50 -1
  70. aioscrapy/middleware/itempipeline.py +96 -0
  71. aioscrapy/middleware/spider.py +360 -7
  72. aioscrapy/process.py +200 -0
  73. aioscrapy/proxy/__init__.py +142 -3
  74. aioscrapy/proxy/redis.py +136 -2
  75. aioscrapy/queue/__init__.py +168 -16
  76. aioscrapy/scrapyd/runner.py +124 -3
  77. aioscrapy/serializer.py +182 -2
  78. aioscrapy/settings/__init__.py +610 -128
  79. aioscrapy/settings/default_settings.py +314 -14
  80. aioscrapy/signalmanager.py +151 -20
  81. aioscrapy/signals.py +183 -1
  82. aioscrapy/spiderloader.py +165 -12
  83. aioscrapy/spiders/__init__.py +233 -6
  84. aioscrapy/statscollectors.py +312 -1
  85. aioscrapy/utils/conf.py +345 -17
  86. aioscrapy/utils/curl.py +168 -16
  87. aioscrapy/utils/decorators.py +76 -6
  88. aioscrapy/utils/deprecate.py +212 -19
  89. aioscrapy/utils/httpobj.py +55 -3
  90. aioscrapy/utils/log.py +79 -0
  91. aioscrapy/utils/misc.py +189 -21
  92. aioscrapy/utils/ossignal.py +67 -5
  93. aioscrapy/utils/project.py +165 -3
  94. aioscrapy/utils/python.py +254 -44
  95. aioscrapy/utils/reqser.py +75 -1
  96. aioscrapy/utils/request.py +173 -12
  97. aioscrapy/utils/response.py +91 -6
  98. aioscrapy/utils/signal.py +196 -14
  99. aioscrapy/utils/spider.py +51 -4
  100. aioscrapy/utils/template.py +93 -6
  101. aioscrapy/utils/tools.py +191 -17
  102. aioscrapy/utils/trackref.py +198 -12
  103. aioscrapy/utils/url.py +341 -36
  104. aio_scrapy-2.1.4.dist-info/METADATA +0 -239
  105. aio_scrapy-2.1.4.dist-info/RECORD +0 -133
  106. aioscrapy/core/downloader/handlers/playwright/__init__.py +0 -115
  107. aioscrapy/core/downloader/handlers/playwright/driverpool.py +0 -59
  108. aioscrapy/core/downloader/handlers/playwright/webdriver.py +0 -96
  109. aioscrapy/http/response/playwright.py +0 -36
  110. aioscrapy/libs/pipelines/execl.py +0 -169
  111. {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.7.dist-info}/entry_points.txt +0 -0
  112. {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.7.dist-info}/top_level.txt +0 -0
@@ -1,3 +1,14 @@
1
+ """
2
+ Settings module for aioscrapy.
3
+ aioscrapy的设置模块。
4
+
5
+ This module provides classes for managing settings in aioscrapy.
6
+ It includes a settings system that supports priorities, type conversion,
7
+ and immutable settings.
8
+ 此模块提供了在aioscrapy中管理设置的类。
9
+ 它包括一个支持优先级、类型转换和不可变设置的设置系统。
10
+ """
11
+
1
12
  import json
2
13
  import copy
3
14
  from collections.abc import MutableMapping
@@ -18,9 +29,22 @@ SETTINGS_PRIORITIES = {
18
29
 
19
30
  def get_settings_priority(priority):
20
31
  """
21
- Small helper function that looks up a given string priority in the
22
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
23
- numerical value, or directly returns a given numerical priority.
32
+ Convert a priority string to its numerical value.
33
+ 将优先级字符串转换为其数值。
34
+
35
+ This helper function looks up a given string priority in the
36
+ SETTINGS_PRIORITIES dictionary and returns its numerical value,
37
+ or directly returns a given numerical priority.
38
+ 此辅助函数在SETTINGS_PRIORITIES字典中查找给定的字符串优先级并返回其数值,
39
+ 或直接返回给定的数值优先级。
40
+
41
+ Args:
42
+ priority: The priority as a string (e.g., 'default', 'project') or a number.
43
+ 作为字符串(例如,'default','project')或数字的优先级。
44
+
45
+ Returns:
46
+ int: The numerical priority value.
47
+ 数值优先级值。
24
48
  """
25
49
  if isinstance(priority, str):
26
50
  return SETTINGS_PRIORITIES[priority]
@@ -29,22 +53,51 @@ def get_settings_priority(priority):
29
53
 
30
54
 
31
55
  class SettingsAttribute:
32
-
33
- """Class for storing data related to settings attributes.
34
-
35
- This class is intended for internal usage, you should try Settings class
36
- for settings configuration, not this one.
56
+ """
57
+ Class for storing data related to settings attributes.
58
+ 用于存储与设置属性相关的数据的类。
59
+
60
+ This class is intended for internal usage. You should use the Settings class
61
+ for settings configuration, not this one. It stores both the value of a setting
62
+ and its priority.
63
+ 此类仅供内部使用。您应该使用Settings类进行设置配置,而不是这个类。
64
+ 它存储设置的值和其优先级。
65
+
66
+ Attributes:
67
+ value: The value of the setting. 设置的值。
68
+ priority: The priority of the setting. 设置的优先级。
37
69
  """
38
70
 
39
71
  def __init__(self, value, priority):
72
+ """
73
+ Initialize a SettingsAttribute.
74
+ 初始化SettingsAttribute。
75
+
76
+ Args:
77
+ value: The value of the setting.
78
+ 设置的值。
79
+ priority: The priority of the setting.
80
+ 设置的优先级。
81
+ """
40
82
  self.value = value
41
83
  if isinstance(self.value, BaseSettings):
84
+ # If the value is a BaseSettings, use the maximum priority
85
+ # 如果值是BaseSettings,使用最大优先级
42
86
  self.priority = max(self.value.maxpriority(), priority)
43
87
  else:
44
88
  self.priority = priority
45
89
 
46
90
  def set(self, value, priority):
47
- """Sets value if priority is higher or equal than current priority."""
91
+ """
92
+ Set the value if the priority is higher or equal than current priority.
93
+ 如果优先级高于或等于当前优先级,则设置值。
94
+
95
+ Args:
96
+ value: The new value to set.
97
+ 要设置的新值。
98
+ priority: The priority of the new value.
99
+ 新值的优先级。
100
+ """
48
101
  if priority >= self.priority:
49
102
  if isinstance(self.value, BaseSettings):
50
103
  value = BaseSettings(value, priority=priority)
@@ -52,79 +105,167 @@ class SettingsAttribute:
52
105
  self.priority = priority
53
106
 
54
107
  def __str__(self):
108
+ """
109
+ Return a string representation of the SettingsAttribute.
110
+ 返回SettingsAttribute的字符串表示。
111
+
112
+ Returns:
113
+ str: A string representation of the SettingsAttribute.
114
+ SettingsAttribute的字符串表示。
115
+ """
55
116
  return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"
56
117
 
118
+ # Make __repr__ use the same implementation as __str__
119
+ # 使__repr__使用与__str__相同的实现
57
120
  __repr__ = __str__
58
121
 
59
122
 
60
123
  class BaseSettings(MutableMapping):
61
124
  """
125
+ Dictionary-like settings container with priority support.
126
+ 具有优先级支持的类字典设置容器。
127
+
62
128
  Instances of this class behave like dictionaries, but store priorities
63
- along with their ``(key, value)`` pairs, and can be frozen (i.e. marked
129
+ along with their (key, value) pairs, and can be frozen (i.e., marked
64
130
  immutable).
131
+ 此类的实例行为类似于字典,但存储(键, 值)对及其优先级,
132
+ 并且可以被冻结(即标记为不可变)。
65
133
 
66
134
  Key-value entries can be passed on initialization with the ``values``
67
135
  argument, and they would take the ``priority`` level (unless ``values`` is
68
- already an instance of :class:`~scrapy.settings.BaseSettings`, in which
69
- case the existing priority levels will be kept). If the ``priority``
70
- argument is a string, the priority name will be looked up in
71
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES`. Otherwise, a specific integer
136
+ already an instance of BaseSettings, in which case the existing priority
137
+ levels will be kept). If the ``priority`` argument is a string, the priority
138
+ name will be looked up in SETTINGS_PRIORITIES. Otherwise, a specific integer
72
139
  should be provided.
140
+ 可以在初始化时通过``values``参数传递键值条目,它们将采用``priority``级别
141
+ (除非``values``已经是BaseSettings的实例,在这种情况下,将保留现有的优先级)。
142
+ 如果``priority``参数是字符串,则将在SETTINGS_PRIORITIES中查找优先级名称。
143
+ 否则,应提供特定的整数。
73
144
 
74
145
  Once the object is created, new settings can be loaded or updated with the
75
- :meth:`~scrapy.settings.BaseSettings.set` method, and can be accessed with
76
- the square bracket notation of dictionaries, or with the
77
- :meth:`~scrapy.settings.BaseSettings.get` method of the instance and its
78
- value conversion variants. When requesting a stored key, the value with the
146
+ ``set`` method, and can be accessed with the square bracket notation of
147
+ dictionaries, or with the ``get`` method of the instance and its value
148
+ conversion variants. When requesting a stored key, the value with the
79
149
  highest priority will be retrieved.
150
+ 创建对象后,可以使用``set``方法加载或更新新设置,并可以使用字典的方括号表示法
151
+ 或实例的``get``方法及其值转换变体进行访问。请求存储的键时,将检索具有最高优先级的值。
152
+
153
+ Attributes:
154
+ frozen (bool): Whether the settings are frozen (immutable).
155
+ 设置是否被冻结(不可变)。
156
+ attributes (dict): Dictionary storing the settings as SettingsAttribute objects.
157
+ 存储设置为SettingsAttribute对象的字典。
80
158
  """
81
159
 
82
160
  def __init__(self, values=None, priority='project'):
161
+ """
162
+ Initialize a BaseSettings instance.
163
+ 初始化BaseSettings实例。
164
+
165
+ Args:
166
+ values: Initial settings values as a dict or BaseSettings instance.
167
+ 作为字典或BaseSettings实例的初始设置值。
168
+ priority: Priority level for the initial values.
169
+ 初始值的优先级。
170
+ """
171
+ # Start with unfrozen settings
172
+ # 从未冻结的设置开始
83
173
  self.frozen = False
174
+
175
+ # Initialize empty attributes dictionary
176
+ # 初始化空属性字典
84
177
  self.attributes = {}
178
+
179
+ # Load initial values if provided
180
+ # 如果提供了初始值,则加载它们
85
181
  if values:
86
182
  self.update(values, priority)
87
183
 
88
184
  def __getitem__(self, opt_name):
185
+ """
186
+ Get a setting value by name.
187
+ 通过名称获取设置值。
188
+
189
+ Args:
190
+ opt_name: The name of the setting to retrieve.
191
+ 要检索的设置的名称。
192
+
193
+ Returns:
194
+ The value of the setting, or None if it doesn't exist.
195
+ 设置的值,如果不存在则为None。
196
+ """
89
197
  if opt_name not in self:
90
198
  return None
91
199
  return self.attributes[opt_name].value
92
200
 
93
201
  def __contains__(self, name):
202
+ """
203
+ Check if a setting exists.
204
+ 检查设置是否存在。
205
+
206
+ Args:
207
+ name: The name of the setting to check.
208
+ 要检查的设置的名称。
209
+
210
+ Returns:
211
+ bool: True if the setting exists, False otherwise.
212
+ 如果设置存在则为True,否则为False。
213
+ """
94
214
  return name in self.attributes
95
215
 
96
216
  def get(self, name, default=None):
97
217
  """
98
218
  Get a setting value without affecting its original type.
219
+ 获取设置值而不影响其原始类型。
99
220
 
100
- :param name: the setting name
101
- :type name: str
221
+ Args:
222
+ name: The setting name.
223
+ 设置名称。
224
+ default: The value to return if no setting is found.
225
+ 如果未找到设置,则返回的值。
102
226
 
103
- :param default: the value to return if no setting is found
104
- :type default: object
227
+ Returns:
228
+ The value of the setting, or the default value if not found.
229
+ 设置的值,如果未找到则为默认值。
105
230
  """
106
231
  return self[name] if self[name] is not None else default
107
232
 
108
233
  def getbool(self, name, default=False):
109
234
  """
110
235
  Get a setting value as a boolean.
236
+ 将设置值作为布尔值获取。
111
237
 
112
- ``1``, ``'1'``, `True`` and ``'True'`` return ``True``,
238
+ ``1``, ``'1'``, ``True`` and ``'True'`` return ``True``,
113
239
  while ``0``, ``'0'``, ``False``, ``'False'`` and ``None`` return ``False``.
240
+ ``1``、``'1'``、``True``和``'True'``返回``True``,
241
+ 而``0``、``'0'``、``False``、``'False'``和``None``返回``False``。
114
242
 
115
243
  For example, settings populated through environment variables set to
116
244
  ``'0'`` will return ``False`` when using this method.
245
+ 例如,通过环境变量设置为``'0'``的设置在使用此方法时将返回``False``。
117
246
 
118
- :param name: the setting name
119
- :type name: str
247
+ Args:
248
+ name: The setting name.
249
+ 设置名称。
250
+ default: The value to return if no setting is found.
251
+ 如果未找到设置,则返回的值。
120
252
 
121
- :param default: the value to return if no setting is found
122
- :type default: object
253
+ Returns:
254
+ bool: The boolean value of the setting.
255
+ 设置的布尔值。
256
+
257
+ Raises:
258
+ ValueError: If the value cannot be converted to a boolean.
259
+ 如果值无法转换为布尔值。
123
260
  """
124
261
  got = self.get(name, default)
125
262
  try:
263
+ # Try to convert to int first, then to bool
264
+ # 首先尝试转换为int,然后转换为bool
126
265
  return bool(int(got))
127
266
  except ValueError:
267
+ # Handle string boolean values
268
+ # 处理字符串布尔值
128
269
  if got in ("True", "true"):
129
270
  return True
130
271
  if got in ("False", "false"):
@@ -136,40 +277,68 @@ class BaseSettings(MutableMapping):
136
277
  def getint(self, name, default=0):
137
278
  """
138
279
  Get a setting value as an int.
280
+ 将设置值作为整数获取。
281
+
282
+ Args:
283
+ name: The setting name.
284
+ 设置名称。
285
+ default: The value to return if no setting is found.
286
+ 如果未找到设置,则返回的值。
139
287
 
140
- :param name: the setting name
141
- :type name: str
288
+ Returns:
289
+ int: The integer value of the setting.
290
+ 设置的整数值。
142
291
 
143
- :param default: the value to return if no setting is found
144
- :type default: object
292
+ Raises:
293
+ ValueError: If the value cannot be converted to an int.
294
+ 如果值无法转换为整数。
145
295
  """
146
296
  return int(self.get(name, default))
147
297
 
148
298
  def getfloat(self, name, default=0.0):
149
299
  """
150
300
  Get a setting value as a float.
301
+ 将设置值作为浮点数获取。
302
+
303
+ Args:
304
+ name: The setting name.
305
+ 设置名称。
306
+ default: The value to return if no setting is found.
307
+ 如果未找到设置,则返回的值。
151
308
 
152
- :param name: the setting name
153
- :type name: str
309
+ Returns:
310
+ float: The float value of the setting.
311
+ 设置的浮点值。
154
312
 
155
- :param default: the value to return if no setting is found
156
- :type default: object
313
+ Raises:
314
+ ValueError: If the value cannot be converted to a float.
315
+ 如果值无法转换为浮点数。
157
316
  """
158
317
  return float(self.get(name, default))
159
318
 
160
319
  def getlist(self, name, default=None):
161
320
  """
162
- Get a setting value as a list. If the setting original type is a list, a
163
- copy of it will be returned. If it's a string it will be split by ",".
321
+ Get a setting value as a list.
322
+ 将设置值作为列表获取。
323
+
324
+ If the setting original type is a list, a copy of it will be returned.
325
+ If it's a string it will be split by ",".
326
+ 如果设置的原始类型是列表,将返回其副本。
327
+ 如果是字符串,将按","拆分。
164
328
 
165
329
  For example, settings populated through environment variables set to
166
330
  ``'one,two'`` will return a list ['one', 'two'] when using this method.
331
+ 例如,通过环境变量设置为``'one,two'``的设置在使用此方法时将返回列表['one', 'two']。
167
332
 
168
- :param name: the setting name
169
- :type name: str
333
+ Args:
334
+ name: The setting name.
335
+ 设置名称。
336
+ default: The value to return if no setting is found.
337
+ 如果未找到设置,则返回的值。
170
338
 
171
- :param default: the value to return if no setting is found
172
- :type default: object
339
+ Returns:
340
+ list: The list value of the setting.
341
+ 设置的列表值。
173
342
  """
174
343
  value = self.get(name, default or [])
175
344
  if isinstance(value, str):
@@ -178,19 +347,33 @@ class BaseSettings(MutableMapping):
178
347
 
179
348
  def getdict(self, name, default=None):
180
349
  """
181
- Get a setting value as a dictionary. If the setting original type is a
182
- dictionary, a copy of it will be returned. If it is a string it will be
183
- evaluated as a JSON dictionary. In the case that it is a
184
- :class:`~scrapy.settings.BaseSettings` instance itself, it will be
185
- converted to a dictionary, containing all its current settings values
186
- as they would be returned by :meth:`~scrapy.settings.BaseSettings.get`,
187
- and losing all information about priority and mutability.
350
+ Get a setting value as a dictionary.
351
+ 将设置值作为字典获取。
188
352
 
189
- :param name: the setting name
190
- :type name: str
191
-
192
- :param default: the value to return if no setting is found
193
- :type default: object
353
+ If the setting original type is a dictionary, a copy of it will be returned.
354
+ If it is a string it will be evaluated as a JSON dictionary.
355
+ In the case that it is a BaseSettings instance itself, it will be
356
+ converted to a dictionary, containing all its current settings values
357
+ as they would be returned by get, and losing all information about
358
+ priority and mutability.
359
+ 如果设置的原始类型是字典,将返回其副本。
360
+ 如果是字符串,将被评估为JSON字典。
361
+ 如果它本身是BaseSettings实例,它将被转换为字典,包含所有当前设置值,
362
+ 就像它们由get返回一样,并丢失有关优先级和可变性的所有信息。
363
+
364
+ Args:
365
+ name: The setting name.
366
+ 设置名称。
367
+ default: The value to return if no setting is found.
368
+ 如果未找到设置,则返回的值。
369
+
370
+ Returns:
371
+ dict: The dictionary value of the setting.
372
+ 设置的字典值。
373
+
374
+ Raises:
375
+ ValueError: If the string value cannot be parsed as JSON.
376
+ 如果字符串值无法解析为JSON。
194
377
  """
195
378
  value = self.get(name, default or {})
196
379
  if isinstance(value, str):
@@ -198,11 +381,26 @@ class BaseSettings(MutableMapping):
198
381
  return dict(value)
199
382
 
200
383
  def getwithbase(self, name):
201
- """Get a composition of a dictionary-like setting and its `_BASE`
202
- counterpart.
384
+ """
385
+ Get a composition of a dictionary-like setting and its base variants.
386
+ 获取字典类设置及其基础变体的组合。
387
+
388
+ This method creates a new BaseSettings instance that combines three settings:
389
+ name + '_BASE', name, and name + '_SPIDER', with increasing priority.
390
+ 此方法创建一个新的BaseSettings实例,它结合了三个设置:
391
+ name + '_BASE'、name和name + '_SPIDER',优先级依次增加。
392
+
393
+ This is useful for settings that have base values and spider-specific overrides,
394
+ such as middlewares and extensions.
395
+ 这对于具有基本值和爬虫特定覆盖的设置很有用,例如中间件和扩展。
203
396
 
204
- :param name: name of the dictionary-like setting
205
- :type name: str
397
+ Args:
398
+ name: Name of the dictionary-like setting without the '_BASE' or '_SPIDER' suffix.
399
+ 不带'_BASE'或'_SPIDER'后缀的字典类设置的名称。
400
+
401
+ Returns:
402
+ BaseSettings: A new BaseSettings instance with the combined settings.
403
+ 包含组合设置的新BaseSettings实例。
206
404
  """
207
405
  compbs = BaseSettings()
208
406
  compbs.update(self[name + '_BASE'])
@@ -212,11 +410,20 @@ class BaseSettings(MutableMapping):
212
410
 
213
411
  def getpriority(self, name):
214
412
  """
215
- Return the current numerical priority value of a setting, or ``None`` if
216
- the given ``name`` does not exist.
413
+ Return the current numerical priority value of a setting.
414
+ 返回设置的当前数值优先级。
415
+
416
+ This method returns the priority of a setting, which determines whether
417
+ the setting can be overridden by other settings with different priorities.
418
+ 此方法返回设置的优先级,它决定了该设置是否可以被具有不同优先级的其他设置覆盖。
217
419
 
218
- :param name: the setting name
219
- :type name: str
420
+ Args:
421
+ name: The name of the setting.
422
+ 设置的名称。
423
+
424
+ Returns:
425
+ int or None: The numerical priority value of the setting, or None if the setting doesn't exist.
426
+ 设置的数值优先级,如果设置不存在则为None。
220
427
  """
221
428
  if name not in self:
222
429
  return None
@@ -224,10 +431,18 @@ class BaseSettings(MutableMapping):
224
431
 
225
432
  def maxpriority(self):
226
433
  """
227
- Return the numerical value of the highest priority present throughout
228
- all settings, or the numerical value for ``default`` from
229
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES` if there are no settings
230
- stored.
434
+ Return the highest priority value among all settings.
435
+ 返回所有设置中的最高优先级值。
436
+
437
+ This method scans through all settings and returns the highest priority value.
438
+ If there are no settings stored, it returns the priority value for 'default'.
439
+ 此方法扫描所有设置并返回最高优先级值。
440
+ 如果没有存储设置,则返回'default'的优先级值。
441
+
442
+ Returns:
443
+ int: The highest priority value among all settings, or the priority value for 'default'
444
+ if there are no settings.
445
+ 所有设置中的最高优先级值,如果没有设置则为'default'的优先级值。
231
446
  """
232
447
  if len(self) > 0:
233
448
  return max(self.getpriority(name) for name in self)
@@ -240,29 +455,43 @@ class BaseSettings(MutableMapping):
240
455
  def set(self, name, value, priority='project'):
241
456
  """
242
457
  Store a key/value attribute with a given priority.
458
+ 存储具有给定优先级的键/值属性。
243
459
 
244
- Settings should be populated *before* configuring the Crawler object
245
- (through the :meth:`~scrapy.crawler.Crawler.configure` method),
246
- otherwise they won't have any effect.
460
+ This method sets a setting with the specified name, value, and priority.
461
+ If a setting with the same name already exists, it will be updated only
462
+ if the new priority is equal to or higher than the existing priority.
463
+ 此方法设置具有指定名称、值和优先级的设置。
464
+ 如果已存在同名设置,则仅当新优先级等于或高于现有优先级时才会更新。
247
465
 
248
- :param name: the setting name
249
- :type name: str
250
-
251
- :param value: the value to associate with the setting
252
- :type value: object
253
-
254
- :param priority: the priority of the setting. Should be a key of
255
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
256
- :type priority: str or int
466
+ Settings should be populated *before* configuring the Crawler object,
467
+ otherwise they won't have any effect.
468
+ 应在配置Crawler对象之前填充设置,否则它们将不会产生任何效果。
469
+
470
+ Args:
471
+ name: The setting name.
472
+ 设置名称。
473
+ value: The value to associate with the setting.
474
+ 与设置关联的值。
475
+ priority: The priority of the setting. Should be a key of
476
+ SETTINGS_PRIORITIES or an integer.
477
+ 设置的优先级。应该是SETTINGS_PRIORITIES的键或整数。
478
+
479
+ Raises:
480
+ TypeError: If the settings object is frozen (immutable).
481
+ 如果设置对象被冻结(不可变)。
257
482
  """
258
483
  self._assert_mutability()
259
484
  priority = get_settings_priority(priority)
260
485
  if name not in self:
486
+ # If the setting doesn't exist, create a new SettingsAttribute
487
+ # 如果设置不存在,创建一个新的SettingsAttribute
261
488
  if isinstance(value, SettingsAttribute):
262
489
  self.attributes[name] = value
263
490
  else:
264
491
  self.attributes[name] = SettingsAttribute(value, priority)
265
492
  else:
493
+ # If the setting exists, update it with the new value and priority
494
+ # 如果设置存在,使用新的值和优先级更新它
266
495
  self.attributes[name].set(value, priority)
267
496
 
268
497
  def setdict(self, values, priority='project'):
@@ -271,21 +500,36 @@ class BaseSettings(MutableMapping):
271
500
  def setmodule(self, module, priority='project'):
272
501
  """
273
502
  Store settings from a module with a given priority.
274
-
275
- This is a helper function that calls
276
- :meth:`~scrapy.settings.BaseSettings.set` for every globally declared
277
- uppercase variable of ``module`` with the provided ``priority``.
278
-
279
- :param module: the module or the path of the module
280
- :type module: types.ModuleType or str
281
-
282
- :param priority: the priority of the settings. Should be a key of
283
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
284
- :type priority: str or int
503
+ 以给定优先级存储来自模块的设置。
504
+
505
+ This is a helper function that calls set() for every globally declared
506
+ uppercase variable of the module with the provided priority.
507
+ 这是一个辅助函数,它为模块的每个全局声明的大写变量调用set(),并使用提供的优先级。
508
+
509
+ This is commonly used to load settings from a settings module.
510
+ 这通常用于从设置模块加载设置。
511
+
512
+ Args:
513
+ module: The module or the path of the module.
514
+ 模块或模块的路径。
515
+ priority: The priority of the settings. Should be a key of
516
+ SETTINGS_PRIORITIES or an integer.
517
+ 设置的优先级。应该是SETTINGS_PRIORITIES的键或整数。
518
+
519
+ Raises:
520
+ TypeError: If the settings object is frozen (immutable).
521
+ 如果设置对象被冻结(不可变)。
522
+ ImportError: If the module path cannot be imported.
523
+ 如果无法导入模块路径。
285
524
  """
286
525
  self._assert_mutability()
287
526
  if isinstance(module, str):
527
+ # If module is a string, import it
528
+ # 如果模块是字符串,导入它
288
529
  module = import_module(module)
530
+
531
+ # Set all uppercase variables from the module
532
+ # 设置模块中的所有大写变量
289
533
  for key in dir(module):
290
534
  if key.isupper():
291
535
  self.set(key, getattr(module, key), priority)
@@ -293,37 +537,73 @@ class BaseSettings(MutableMapping):
293
537
  def update(self, values, priority='project'):
294
538
  """
295
539
  Store key/value pairs with a given priority.
296
-
297
- This is a helper function that calls
298
- :meth:`~scrapy.settings.BaseSettings.set` for every item of ``values``
299
- with the provided ``priority``.
300
-
301
- If ``values`` is a string, it is assumed to be JSON-encoded and parsed
302
- into a dict with ``json.loads()`` first. If it is a
303
- :class:`~scrapy.settings.BaseSettings` instance, the per-key priorities
304
- will be used and the ``priority`` parameter ignored. This allows
305
- inserting/updating settings with different priorities with a single
306
- command.
307
-
308
- :param values: the settings names and values
309
- :type values: dict or string or :class:`~scrapy.settings.BaseSettings`
310
-
311
- :param priority: the priority of the settings. Should be a key of
312
- :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
313
- :type priority: str or int
540
+ 以给定优先级存储键/值对。
541
+
542
+ This is a helper function that calls set() for every item of values
543
+ with the provided priority.
544
+ 这是一个辅助函数,它为values的每个项目调用set(),并使用提供的优先级。
545
+
546
+ If values is a string, it is assumed to be JSON-encoded and parsed
547
+ into a dict with json.loads() first. If it is a BaseSettings instance,
548
+ the per-key priorities will be used and the priority parameter ignored.
549
+ This allows inserting/updating settings with different priorities with
550
+ a single command.
551
+ 如果values是字符串,则假定它是JSON编码的,并首先使用json.loads()解析为字典。
552
+ 如果它是BaseSettings实例,则将使用每个键的优先级,并忽略priority参数。
553
+ 这允许使用单个命令插入/更新具有不同优先级的设置。
554
+
555
+ Args:
556
+ values: The settings names and values.
557
+ 设置名称和值。
558
+ Can be a dict, a JSON string, or a BaseSettings instance.
559
+ 可以是字典、JSON字符串或BaseSettings实例。
560
+ priority: The priority of the settings. Should be a key of
561
+ SETTINGS_PRIORITIES or an integer.
562
+ 设置的优先级。应该是SETTINGS_PRIORITIES的键或整数。
563
+
564
+ Raises:
565
+ TypeError: If the settings object is frozen (immutable).
566
+ 如果设置对象被冻结(不可变)。
567
+ ValueError: If values is a string that cannot be parsed as JSON.
568
+ 如果values是无法解析为JSON的字符串。
314
569
  """
315
570
  self._assert_mutability()
316
571
  if isinstance(values, str):
572
+ # Parse JSON string into a dict
573
+ # 将JSON字符串解析为字典
317
574
  values = json.loads(values)
575
+
318
576
  if values is not None:
319
577
  if isinstance(values, BaseSettings):
578
+ # If values is a BaseSettings, use its per-key priorities
579
+ # 如果values是BaseSettings,使用其每个键的优先级
320
580
  for name, value in values.items():
321
581
  self.set(name, value, values.getpriority(name))
322
582
  else:
583
+ # Otherwise, use the provided priority for all items
584
+ # 否则,对所有项目使用提供的优先级
323
585
  for name, value in values.items():
324
586
  self.set(name, value, priority)
325
587
 
326
588
  def delete(self, name, priority='project'):
589
+ """
590
+ Delete a setting if the given priority is higher or equal than its priority.
591
+ 如果给定优先级高于或等于设置的优先级,则删除该设置。
592
+
593
+ This method deletes a setting with the specified name if the provided
594
+ priority is higher than or equal to the setting's priority.
595
+ 此方法删除具有指定名称的设置,如果提供的优先级高于或等于设置的优先级。
596
+
597
+ Args:
598
+ name: The name of the setting to delete.
599
+ 要删除的设置的名称。
600
+ priority: The priority level. Should be a key of SETTINGS_PRIORITIES or an integer.
601
+ 优先级级别。应该是SETTINGS_PRIORITIES的键或整数。
602
+
603
+ Raises:
604
+ TypeError: If the settings object is frozen (immutable).
605
+ 如果设置对象被冻结(不可变)。
606
+ """
327
607
  self._assert_mutability()
328
608
  priority = get_settings_priority(priority)
329
609
  if priority >= self.getpriority(name):
@@ -334,40 +614,75 @@ class BaseSettings(MutableMapping):
334
614
  del self.attributes[name]
335
615
 
336
616
  def _assert_mutability(self):
617
+ """
618
+ Assert that the settings object is mutable.
619
+ 断言设置对象是可变的。
620
+
621
+ This internal method checks if the settings object is frozen (immutable)
622
+ and raises a TypeError if it is.
623
+ 此内部方法检查设置对象是否被冻结(不可变),如果是则引发TypeError。
624
+
625
+ Raises:
626
+ TypeError: If the settings object is frozen (immutable).
627
+ 如果设置对象被冻结(不可变)。
628
+ """
337
629
  if self.frozen:
338
630
  raise TypeError("Trying to modify an immutable Settings object")
339
631
 
340
632
  def copy(self):
341
633
  """
342
634
  Make a deep copy of current settings.
635
+ 创建当前设置的深拷贝。
343
636
 
344
- This method returns a new instance of the :class:`Settings` class,
637
+ This method returns a new instance of the BaseSettings class,
345
638
  populated with the same values and their priorities.
639
+ 此方法返回BaseSettings类的新实例,填充相同的值及其优先级。
346
640
 
347
641
  Modifications to the new object won't be reflected on the original
348
- settings.
642
+ settings, and vice versa.
643
+ 对新对象的修改不会反映在原始设置上,反之亦然。
644
+
645
+ Returns:
646
+ BaseSettings: A new instance with the same settings and priorities.
647
+ 具有相同设置和优先级的新实例。
349
648
  """
350
649
  return copy.deepcopy(self)
351
650
 
352
651
  def freeze(self):
353
652
  """
354
653
  Disable further changes to the current settings.
654
+ 禁止对当前设置进行进一步更改。
355
655
 
356
656
  After calling this method, the present state of the settings will become
357
- immutable. Trying to change values through the :meth:`~set` method and
358
- its variants won't be possible and will be alerted.
657
+ immutable. Trying to change values through the set() method and
658
+ its variants won't be possible and will raise a TypeError.
659
+ 调用此方法后,设置的当前状态将变为不可变。
660
+ 尝试通过set()方法及其变体更改值将不可能,并将引发TypeError。
661
+
662
+ Returns:
663
+ None
359
664
  """
360
665
  self.frozen = True
361
666
 
362
667
  def frozencopy(self):
363
668
  """
364
669
  Return an immutable copy of the current settings.
670
+ 返回当前设置的不可变副本。
365
671
 
366
- Alias for a :meth:`~freeze` call in the object returned by :meth:`copy`.
672
+ This method creates a copy of the current settings and then freezes it,
673
+ making it immutable.
674
+ 此方法创建当前设置的副本,然后冻结它,使其不可变。
675
+
676
+ It's equivalent to calling copy() followed by freeze().
677
+ 它相当于调用copy()后跟freeze()。
678
+
679
+ Returns:
680
+ BaseSettings: An immutable copy of the current settings.
681
+ 当前设置的不可变副本。
367
682
  """
368
- copy = self.copy()
369
- copy.freeze()
370
- return copy
683
+ settings_copy = self.copy()
684
+ settings_copy.freeze()
685
+ return settings_copy
371
686
 
372
687
  def __iter__(self):
373
688
  return iter(self.attributes)
@@ -376,90 +691,257 @@ class BaseSettings(MutableMapping):
376
691
  return len(self.attributes)
377
692
 
378
693
  def _to_dict(self):
694
+ """
695
+ Internal method to convert settings to a dictionary.
696
+ 将设置转换为字典的内部方法。
697
+
698
+ This method recursively converts BaseSettings instances to dictionaries.
699
+ 此方法递归地将BaseSettings实例转换为字典。
700
+
701
+ Returns:
702
+ dict: A dictionary representation of the settings.
703
+ 设置的字典表示。
704
+ """
379
705
  return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
380
706
  for k, v in self.items()}
381
707
 
382
708
  def copy_to_dict(self):
383
709
  """
384
710
  Make a copy of current settings and convert to a dict.
711
+ 创建当前设置的副本并转换为字典。
385
712
 
386
713
  This method returns a new dict populated with the same values
387
- and their priorities as the current settings.
714
+ as the current settings, with nested BaseSettings also converted to dicts.
715
+ 此方法返回一个新字典,填充与当前设置相同的值,嵌套的BaseSettings也转换为字典。
388
716
 
389
717
  Modifications to the returned dict won't be reflected on the original
390
- settings.
718
+ settings, and vice versa.
719
+ 对返回的字典的修改不会反映在原始设置上,反之亦然。
391
720
 
392
721
  This method can be useful for example for printing settings
393
- in Scrapy shell.
722
+ or for serialization.
723
+ 此方法例如对于打印设置或序列化很有用。
724
+
725
+ Returns:
726
+ dict: A dictionary representation of the settings.
727
+ 设置的字典表示。
394
728
  """
395
729
  settings = self.copy()
396
730
  return settings._to_dict()
397
731
 
398
732
  def _repr_pretty_(self, p, cycle):
733
+ """
734
+ Pretty-printing support for IPython/Jupyter.
735
+ IPython/Jupyter的美观打印支持。
736
+
737
+ This method is called by IPython/Jupyter to format the object for display.
738
+ 此方法由IPython/Jupyter调用,用于格式化对象以供显示。
739
+
740
+ Args:
741
+ p: The pretty printer instance.
742
+ 美观打印器实例。
743
+ cycle: Whether a cycle was detected in the object graph.
744
+ 是否在对象图中检测到循环。
745
+
746
+ Returns:
747
+ None
748
+ """
399
749
  if cycle:
750
+ # If a cycle is detected, use the standard repr
751
+ # 如果检测到循环,使用标准的repr
400
752
  p.text(repr(self))
401
753
  else:
754
+ # Otherwise, format the settings as a pretty-printed dict
755
+ # 否则,将设置格式化为美观打印的字典
402
756
  p.text(pformat(self.copy_to_dict()))
403
757
 
404
758
 
405
759
  class _DictProxy(MutableMapping):
760
+ """
761
+ Dictionary proxy that updates settings when modified.
762
+ 修改时更新设置的字典代理。
763
+
764
+ This class is used internally to provide a dictionary-like interface
765
+ that updates settings with a specific priority when modified.
766
+ 此类在内部用于提供类字典接口,在修改时以特定优先级更新设置。
767
+ """
406
768
 
407
769
  def __init__(self, settings, priority):
770
+ """
771
+ Initialize a _DictProxy.
772
+ 初始化_DictProxy。
773
+
774
+ Args:
775
+ settings: The settings object to update.
776
+ 要更新的设置对象。
777
+ priority: The priority to use when updating settings.
778
+ 更新设置时使用的优先级。
779
+ """
408
780
  self.o = {}
409
781
  self.settings = settings
410
782
  self.priority = priority
411
783
 
412
784
  def __len__(self):
785
+ """
786
+ Return the number of items in the dictionary.
787
+ 返回字典中的项目数。
788
+
789
+ Returns:
790
+ int: The number of items in the dictionary.
791
+ 字典中的项目数。
792
+ """
413
793
  return len(self.o)
414
794
 
415
795
  def __getitem__(self, k):
796
+ """
797
+ Get an item from the dictionary.
798
+ 从字典中获取项目。
799
+
800
+ Args:
801
+ k: The key to get.
802
+ 要获取的键。
803
+
804
+ Returns:
805
+ The value for the key.
806
+ 键的值。
807
+
808
+ Raises:
809
+ KeyError: If the key is not found.
810
+ 如果未找到键。
811
+ """
416
812
  return self.o[k]
417
813
 
418
814
  def __setitem__(self, k, v):
815
+ """
816
+ Set an item in the dictionary and update the settings.
817
+ 在字典中设置项目并更新设置。
818
+
819
+ Args:
820
+ k: The key to set.
821
+ 要设置的键。
822
+ v: The value to set.
823
+ 要设置的值。
824
+ """
419
825
  self.settings.set(k, v, priority=self.priority)
420
826
  self.o[k] = v
421
827
 
422
828
  def __delitem__(self, k):
829
+ """
830
+ Delete an item from the dictionary.
831
+ 从字典中删除项目。
832
+
833
+ Args:
834
+ k: The key to delete.
835
+ 要删除的键。
836
+
837
+ Raises:
838
+ KeyError: If the key is not found.
839
+ 如果未找到键。
840
+ """
423
841
  del self.o[k]
424
842
 
425
- def __iter__(self, k, v):
843
+ def __iter__(self):
844
+ """
845
+ Return an iterator over the dictionary keys.
846
+ 返回字典键的迭代器。
847
+
848
+ Returns:
849
+ iterator: An iterator over the dictionary keys.
850
+ 字典键的迭代器。
851
+ """
426
852
  return iter(self.o)
427
853
 
428
854
 
429
855
  class Settings(BaseSettings):
430
856
  """
431
- This object stores Scrapy settings for the configuration of internal
432
- components, and can be used for any further customization.
857
+ Settings container for aioscrapy with default settings.
858
+ 具有默认设置的aioscrapy设置容器。
433
859
 
434
- It is a direct subclass and supports all methods of
435
- :class:`~scrapy.settings.BaseSettings`. Additionally, after instantiation
436
- of this class, the new object will have the global default settings
437
- described on :ref:`topics-settings-ref` already populated.
860
+ This object stores aioscrapy settings for the configuration of internal
861
+ components, and can be used for any further customization.
862
+ 此对象存储用于配置内部组件的aioscrapy设置,可用于任何进一步的自定义。
863
+
864
+ It is a direct subclass and supports all methods of BaseSettings.
865
+ Additionally, after instantiation of this class, the new object will
866
+ have the global default settings from default_settings already populated.
867
+ 它是BaseSettings的直接子类,支持BaseSettings的所有方法。
868
+ 此外,在实例化此类后,新对象将已经填充了来自default_settings的全局默认设置。
869
+
870
+ Attributes:
871
+ frozen (bool): Whether the settings are frozen (immutable).
872
+ 设置是否被冻结(不可变)。
873
+ attributes (dict): Dictionary storing the settings as SettingsAttribute objects.
874
+ 存储设置为SettingsAttribute对象的字典。
438
875
  """
439
876
 
440
877
  def __init__(self, values=None, priority='project'):
878
+ """
879
+ Initialize a Settings instance with default settings.
880
+ 使用默认设置初始化Settings实例。
881
+
882
+ Args:
883
+ values: Initial settings values as a dict or BaseSettings instance.
884
+ 作为字典或BaseSettings实例的初始设置值。
885
+ priority: Priority level for the initial values.
886
+ 初始值的优先级。
887
+ """
441
888
  # Do not pass kwarg values here. We don't want to promote user-defined
442
889
  # dicts, and we want to update, not replace, default dicts with the
443
890
  # values given by the user
891
+ # 不要在这里传递kwarg值。我们不想提升用户定义的字典,
892
+ # 我们想用用户给定的值更新默认字典,而不是替换它们
444
893
  super().__init__()
894
+
895
+ # Load default settings with 'default' priority
896
+ # 加载具有'default'优先级的默认设置
445
897
  self.setmodule(default_settings, 'default')
446
- # Promote default dictionaries to BaseSettings instances for per-key
447
- # priorities
898
+
899
+ # Promote default dictionaries to BaseSettings instances for per-key priorities
900
+ # 将默认字典提升为BaseSettings实例以实现每个键的优先级
448
901
  for name, val in self.items():
449
902
  if isinstance(val, dict):
450
903
  self.set(name, BaseSettings(val, 'default'), 'default')
904
+
905
+ # Update with user-provided values
906
+ # 使用用户提供的值更新
451
907
  self.update(values, priority)
452
908
 
453
909
 
454
910
  def iter_default_settings():
455
- """Return the default settings as an iterator of (name, value) tuples"""
911
+ """
912
+ Return the default settings as an iterator.
913
+ 将默认设置作为迭代器返回。
914
+
915
+ This function iterates through all uppercase attributes in the default_settings
916
+ module and yields them as (name, value) tuples.
917
+ 此函数遍历default_settings模块中的所有大写属性,并将它们作为(名称, 值)元组生成。
918
+
919
+ Returns:
920
+ iterator: An iterator of (name, value) tuples of default settings.
921
+ 默认设置的(名称, 值)元组的迭代器。
922
+ """
456
923
  for name in dir(default_settings):
457
924
  if name.isupper():
458
925
  yield name, getattr(default_settings, name)
459
926
 
460
927
 
461
928
  def overridden_settings(settings):
462
- """Return a dict of the settings that have been overridden"""
929
+ """
930
+ Return a dict of the settings that have been overridden.
931
+ 返回已被覆盖的设置的字典。
932
+
933
+ This function compares the values in the provided settings object with
934
+ the default values and yields the ones that have been changed.
935
+ 此函数将提供的设置对象中的值与默认值进行比较,并生成已更改的值。
936
+
937
+ Args:
938
+ settings: The settings object to check.
939
+ 要检查的设置对象。
940
+
941
+ Returns:
942
+ iterator: An iterator of (name, value) tuples of overridden settings.
943
+ 已覆盖设置的(名称, 值)元组的迭代器。
944
+ """
463
945
  for name, defvalue in iter_default_settings():
464
946
  value = settings[name]
465
947
  if not isinstance(defvalue, dict) and value != defvalue: