aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.7.dist-info}/LICENSE +1 -1
- aio_scrapy-2.1.7.dist-info/METADATA +147 -0
- aio_scrapy-2.1.7.dist-info/RECORD +134 -0
- {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.7.dist-info}/WHEEL +1 -1
- aioscrapy/VERSION +1 -1
- aioscrapy/cmdline.py +438 -5
- aioscrapy/core/downloader/__init__.py +522 -17
- aioscrapy/core/downloader/handlers/__init__.py +187 -5
- aioscrapy/core/downloader/handlers/aiohttp.py +190 -6
- aioscrapy/core/downloader/handlers/curl_cffi.py +126 -5
- aioscrapy/core/downloader/handlers/httpx.py +135 -5
- aioscrapy/core/downloader/handlers/pyhttpx.py +137 -5
- aioscrapy/core/downloader/handlers/requests.py +120 -2
- aioscrapy/core/downloader/handlers/webdriver/__init__.py +2 -0
- aioscrapy/core/downloader/handlers/webdriver/drissionpage.py +493 -0
- aioscrapy/core/downloader/handlers/webdriver/driverpool.py +234 -0
- aioscrapy/core/downloader/handlers/webdriver/playwright.py +498 -0
- aioscrapy/core/engine.py +381 -20
- aioscrapy/core/scheduler.py +350 -36
- aioscrapy/core/scraper.py +509 -33
- aioscrapy/crawler.py +392 -10
- aioscrapy/db/__init__.py +149 -0
- aioscrapy/db/absmanager.py +212 -6
- aioscrapy/db/aiomongo.py +292 -10
- aioscrapy/db/aiomysql.py +363 -10
- aioscrapy/db/aiopg.py +299 -2
- aioscrapy/db/aiorabbitmq.py +444 -4
- aioscrapy/db/aioredis.py +260 -11
- aioscrapy/dupefilters/__init__.py +110 -5
- aioscrapy/dupefilters/disk.py +124 -2
- aioscrapy/dupefilters/redis.py +598 -32
- aioscrapy/exceptions.py +151 -13
- aioscrapy/http/__init__.py +1 -1
- aioscrapy/http/headers.py +237 -3
- aioscrapy/http/request/__init__.py +257 -11
- aioscrapy/http/request/form.py +83 -3
- aioscrapy/http/request/json_request.py +121 -9
- aioscrapy/http/response/__init__.py +306 -33
- aioscrapy/http/response/html.py +42 -3
- aioscrapy/http/response/text.py +496 -49
- aioscrapy/http/response/web_driver.py +144 -0
- aioscrapy/http/response/xml.py +45 -3
- aioscrapy/libs/downloader/defaultheaders.py +66 -2
- aioscrapy/libs/downloader/downloadtimeout.py +91 -2
- aioscrapy/libs/downloader/ja3fingerprint.py +95 -2
- aioscrapy/libs/downloader/retry.py +192 -6
- aioscrapy/libs/downloader/stats.py +142 -0
- aioscrapy/libs/downloader/useragent.py +93 -2
- aioscrapy/libs/extensions/closespider.py +166 -4
- aioscrapy/libs/extensions/corestats.py +151 -1
- aioscrapy/libs/extensions/logstats.py +145 -1
- aioscrapy/libs/extensions/metric.py +370 -1
- aioscrapy/libs/extensions/throttle.py +235 -1
- aioscrapy/libs/pipelines/__init__.py +345 -2
- aioscrapy/libs/pipelines/csv.py +242 -0
- aioscrapy/libs/pipelines/excel.py +545 -0
- aioscrapy/libs/pipelines/mongo.py +132 -0
- aioscrapy/libs/pipelines/mysql.py +67 -0
- aioscrapy/libs/pipelines/pg.py +67 -0
- aioscrapy/libs/spider/depth.py +141 -3
- aioscrapy/libs/spider/httperror.py +144 -4
- aioscrapy/libs/spider/offsite.py +202 -2
- aioscrapy/libs/spider/referer.py +396 -21
- aioscrapy/libs/spider/urllength.py +97 -1
- aioscrapy/link.py +115 -8
- aioscrapy/logformatter.py +199 -8
- aioscrapy/middleware/absmanager.py +328 -2
- aioscrapy/middleware/downloader.py +218 -0
- aioscrapy/middleware/extension.py +50 -1
- aioscrapy/middleware/itempipeline.py +96 -0
- aioscrapy/middleware/spider.py +360 -7
- aioscrapy/process.py +200 -0
- aioscrapy/proxy/__init__.py +142 -3
- aioscrapy/proxy/redis.py +136 -2
- aioscrapy/queue/__init__.py +168 -16
- aioscrapy/scrapyd/runner.py +124 -3
- aioscrapy/serializer.py +182 -2
- aioscrapy/settings/__init__.py +610 -128
- aioscrapy/settings/default_settings.py +314 -14
- aioscrapy/signalmanager.py +151 -20
- aioscrapy/signals.py +183 -1
- aioscrapy/spiderloader.py +165 -12
- aioscrapy/spiders/__init__.py +233 -6
- aioscrapy/statscollectors.py +312 -1
- aioscrapy/utils/conf.py +345 -17
- aioscrapy/utils/curl.py +168 -16
- aioscrapy/utils/decorators.py +76 -6
- aioscrapy/utils/deprecate.py +212 -19
- aioscrapy/utils/httpobj.py +55 -3
- aioscrapy/utils/log.py +79 -0
- aioscrapy/utils/misc.py +189 -21
- aioscrapy/utils/ossignal.py +67 -5
- aioscrapy/utils/project.py +165 -3
- aioscrapy/utils/python.py +254 -44
- aioscrapy/utils/reqser.py +75 -1
- aioscrapy/utils/request.py +173 -12
- aioscrapy/utils/response.py +91 -6
- aioscrapy/utils/signal.py +196 -14
- aioscrapy/utils/spider.py +51 -4
- aioscrapy/utils/template.py +93 -6
- aioscrapy/utils/tools.py +191 -17
- aioscrapy/utils/trackref.py +198 -12
- aioscrapy/utils/url.py +341 -36
- aio_scrapy-2.1.4.dist-info/METADATA +0 -239
- aio_scrapy-2.1.4.dist-info/RECORD +0 -133
- aioscrapy/core/downloader/handlers/playwright/__init__.py +0 -115
- aioscrapy/core/downloader/handlers/playwright/driverpool.py +0 -59
- aioscrapy/core/downloader/handlers/playwright/webdriver.py +0 -96
- aioscrapy/http/response/playwright.py +0 -36
- aioscrapy/libs/pipelines/execl.py +0 -169
- {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.7.dist-info}/entry_points.txt +0 -0
- {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.7.dist-info}/top_level.txt +0 -0
aioscrapy/settings/__init__.py
CHANGED
|
@@ -1,3 +1,14 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Settings module for aioscrapy.
|
|
3
|
+
aioscrapy的设置模块。
|
|
4
|
+
|
|
5
|
+
This module provides classes for managing settings in aioscrapy.
|
|
6
|
+
It includes a settings system that supports priorities, type conversion,
|
|
7
|
+
and immutable settings.
|
|
8
|
+
此模块提供了在aioscrapy中管理设置的类。
|
|
9
|
+
它包括一个支持优先级、类型转换和不可变设置的设置系统。
|
|
10
|
+
"""
|
|
11
|
+
|
|
1
12
|
import json
|
|
2
13
|
import copy
|
|
3
14
|
from collections.abc import MutableMapping
|
|
@@ -18,9 +29,22 @@ SETTINGS_PRIORITIES = {
|
|
|
18
29
|
|
|
19
30
|
def get_settings_priority(priority):
|
|
20
31
|
"""
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
32
|
+
Convert a priority string to its numerical value.
|
|
33
|
+
将优先级字符串转换为其数值。
|
|
34
|
+
|
|
35
|
+
This helper function looks up a given string priority in the
|
|
36
|
+
SETTINGS_PRIORITIES dictionary and returns its numerical value,
|
|
37
|
+
or directly returns a given numerical priority.
|
|
38
|
+
此辅助函数在SETTINGS_PRIORITIES字典中查找给定的字符串优先级并返回其数值,
|
|
39
|
+
或直接返回给定的数值优先级。
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
priority: The priority as a string (e.g., 'default', 'project') or a number.
|
|
43
|
+
作为字符串(例如,'default','project')或数字的优先级。
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
int: The numerical priority value.
|
|
47
|
+
数值优先级值。
|
|
24
48
|
"""
|
|
25
49
|
if isinstance(priority, str):
|
|
26
50
|
return SETTINGS_PRIORITIES[priority]
|
|
@@ -29,22 +53,51 @@ def get_settings_priority(priority):
|
|
|
29
53
|
|
|
30
54
|
|
|
31
55
|
class SettingsAttribute:
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
for
|
|
56
|
+
"""
|
|
57
|
+
Class for storing data related to settings attributes.
|
|
58
|
+
用于存储与设置属性相关的数据的类。
|
|
59
|
+
|
|
60
|
+
This class is intended for internal usage. You should use the Settings class
|
|
61
|
+
for settings configuration, not this one. It stores both the value of a setting
|
|
62
|
+
and its priority.
|
|
63
|
+
此类仅供内部使用。您应该使用Settings类进行设置配置,而不是这个类。
|
|
64
|
+
它存储设置的值和其优先级。
|
|
65
|
+
|
|
66
|
+
Attributes:
|
|
67
|
+
value: The value of the setting. 设置的值。
|
|
68
|
+
priority: The priority of the setting. 设置的优先级。
|
|
37
69
|
"""
|
|
38
70
|
|
|
39
71
|
def __init__(self, value, priority):
|
|
72
|
+
"""
|
|
73
|
+
Initialize a SettingsAttribute.
|
|
74
|
+
初始化SettingsAttribute。
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
value: The value of the setting.
|
|
78
|
+
设置的值。
|
|
79
|
+
priority: The priority of the setting.
|
|
80
|
+
设置的优先级。
|
|
81
|
+
"""
|
|
40
82
|
self.value = value
|
|
41
83
|
if isinstance(self.value, BaseSettings):
|
|
84
|
+
# If the value is a BaseSettings, use the maximum priority
|
|
85
|
+
# 如果值是BaseSettings,使用最大优先级
|
|
42
86
|
self.priority = max(self.value.maxpriority(), priority)
|
|
43
87
|
else:
|
|
44
88
|
self.priority = priority
|
|
45
89
|
|
|
46
90
|
def set(self, value, priority):
|
|
47
|
-
"""
|
|
91
|
+
"""
|
|
92
|
+
Set the value if the priority is higher or equal than current priority.
|
|
93
|
+
如果优先级高于或等于当前优先级,则设置值。
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
value: The new value to set.
|
|
97
|
+
要设置的新值。
|
|
98
|
+
priority: The priority of the new value.
|
|
99
|
+
新值的优先级。
|
|
100
|
+
"""
|
|
48
101
|
if priority >= self.priority:
|
|
49
102
|
if isinstance(self.value, BaseSettings):
|
|
50
103
|
value = BaseSettings(value, priority=priority)
|
|
@@ -52,79 +105,167 @@ class SettingsAttribute:
|
|
|
52
105
|
self.priority = priority
|
|
53
106
|
|
|
54
107
|
def __str__(self):
|
|
108
|
+
"""
|
|
109
|
+
Return a string representation of the SettingsAttribute.
|
|
110
|
+
返回SettingsAttribute的字符串表示。
|
|
111
|
+
|
|
112
|
+
Returns:
|
|
113
|
+
str: A string representation of the SettingsAttribute.
|
|
114
|
+
SettingsAttribute的字符串表示。
|
|
115
|
+
"""
|
|
55
116
|
return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"
|
|
56
117
|
|
|
118
|
+
# Make __repr__ use the same implementation as __str__
|
|
119
|
+
# 使__repr__使用与__str__相同的实现
|
|
57
120
|
__repr__ = __str__
|
|
58
121
|
|
|
59
122
|
|
|
60
123
|
class BaseSettings(MutableMapping):
|
|
61
124
|
"""
|
|
125
|
+
Dictionary-like settings container with priority support.
|
|
126
|
+
具有优先级支持的类字典设置容器。
|
|
127
|
+
|
|
62
128
|
Instances of this class behave like dictionaries, but store priorities
|
|
63
|
-
along with their
|
|
129
|
+
along with their (key, value) pairs, and can be frozen (i.e., marked
|
|
64
130
|
immutable).
|
|
131
|
+
此类的实例行为类似于字典,但存储(键, 值)对及其优先级,
|
|
132
|
+
并且可以被冻结(即标记为不可变)。
|
|
65
133
|
|
|
66
134
|
Key-value entries can be passed on initialization with the ``values``
|
|
67
135
|
argument, and they would take the ``priority`` level (unless ``values`` is
|
|
68
|
-
already an instance of
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
:attr:`~scrapy.settings.SETTINGS_PRIORITIES`. Otherwise, a specific integer
|
|
136
|
+
already an instance of BaseSettings, in which case the existing priority
|
|
137
|
+
levels will be kept). If the ``priority`` argument is a string, the priority
|
|
138
|
+
name will be looked up in SETTINGS_PRIORITIES. Otherwise, a specific integer
|
|
72
139
|
should be provided.
|
|
140
|
+
可以在初始化时通过``values``参数传递键值条目,它们将采用``priority``级别
|
|
141
|
+
(除非``values``已经是BaseSettings的实例,在这种情况下,将保留现有的优先级)。
|
|
142
|
+
如果``priority``参数是字符串,则将在SETTINGS_PRIORITIES中查找优先级名称。
|
|
143
|
+
否则,应提供特定的整数。
|
|
73
144
|
|
|
74
145
|
Once the object is created, new settings can be loaded or updated with the
|
|
75
|
-
|
|
76
|
-
the
|
|
77
|
-
|
|
78
|
-
value conversion variants. When requesting a stored key, the value with the
|
|
146
|
+
``set`` method, and can be accessed with the square bracket notation of
|
|
147
|
+
dictionaries, or with the ``get`` method of the instance and its value
|
|
148
|
+
conversion variants. When requesting a stored key, the value with the
|
|
79
149
|
highest priority will be retrieved.
|
|
150
|
+
创建对象后,可以使用``set``方法加载或更新新设置,并可以使用字典的方括号表示法
|
|
151
|
+
或实例的``get``方法及其值转换变体进行访问。请求存储的键时,将检索具有最高优先级的值。
|
|
152
|
+
|
|
153
|
+
Attributes:
|
|
154
|
+
frozen (bool): Whether the settings are frozen (immutable).
|
|
155
|
+
设置是否被冻结(不可变)。
|
|
156
|
+
attributes (dict): Dictionary storing the settings as SettingsAttribute objects.
|
|
157
|
+
存储设置为SettingsAttribute对象的字典。
|
|
80
158
|
"""
|
|
81
159
|
|
|
82
160
|
def __init__(self, values=None, priority='project'):
|
|
161
|
+
"""
|
|
162
|
+
Initialize a BaseSettings instance.
|
|
163
|
+
初始化BaseSettings实例。
|
|
164
|
+
|
|
165
|
+
Args:
|
|
166
|
+
values: Initial settings values as a dict or BaseSettings instance.
|
|
167
|
+
作为字典或BaseSettings实例的初始设置值。
|
|
168
|
+
priority: Priority level for the initial values.
|
|
169
|
+
初始值的优先级。
|
|
170
|
+
"""
|
|
171
|
+
# Start with unfrozen settings
|
|
172
|
+
# 从未冻结的设置开始
|
|
83
173
|
self.frozen = False
|
|
174
|
+
|
|
175
|
+
# Initialize empty attributes dictionary
|
|
176
|
+
# 初始化空属性字典
|
|
84
177
|
self.attributes = {}
|
|
178
|
+
|
|
179
|
+
# Load initial values if provided
|
|
180
|
+
# 如果提供了初始值,则加载它们
|
|
85
181
|
if values:
|
|
86
182
|
self.update(values, priority)
|
|
87
183
|
|
|
88
184
|
def __getitem__(self, opt_name):
|
|
185
|
+
"""
|
|
186
|
+
Get a setting value by name.
|
|
187
|
+
通过名称获取设置值。
|
|
188
|
+
|
|
189
|
+
Args:
|
|
190
|
+
opt_name: The name of the setting to retrieve.
|
|
191
|
+
要检索的设置的名称。
|
|
192
|
+
|
|
193
|
+
Returns:
|
|
194
|
+
The value of the setting, or None if it doesn't exist.
|
|
195
|
+
设置的值,如果不存在则为None。
|
|
196
|
+
"""
|
|
89
197
|
if opt_name not in self:
|
|
90
198
|
return None
|
|
91
199
|
return self.attributes[opt_name].value
|
|
92
200
|
|
|
93
201
|
def __contains__(self, name):
|
|
202
|
+
"""
|
|
203
|
+
Check if a setting exists.
|
|
204
|
+
检查设置是否存在。
|
|
205
|
+
|
|
206
|
+
Args:
|
|
207
|
+
name: The name of the setting to check.
|
|
208
|
+
要检查的设置的名称。
|
|
209
|
+
|
|
210
|
+
Returns:
|
|
211
|
+
bool: True if the setting exists, False otherwise.
|
|
212
|
+
如果设置存在则为True,否则为False。
|
|
213
|
+
"""
|
|
94
214
|
return name in self.attributes
|
|
95
215
|
|
|
96
216
|
def get(self, name, default=None):
|
|
97
217
|
"""
|
|
98
218
|
Get a setting value without affecting its original type.
|
|
219
|
+
获取设置值而不影响其原始类型。
|
|
99
220
|
|
|
100
|
-
:
|
|
101
|
-
|
|
221
|
+
Args:
|
|
222
|
+
name: The setting name.
|
|
223
|
+
设置名称。
|
|
224
|
+
default: The value to return if no setting is found.
|
|
225
|
+
如果未找到设置,则返回的值。
|
|
102
226
|
|
|
103
|
-
:
|
|
104
|
-
|
|
227
|
+
Returns:
|
|
228
|
+
The value of the setting, or the default value if not found.
|
|
229
|
+
设置的值,如果未找到则为默认值。
|
|
105
230
|
"""
|
|
106
231
|
return self[name] if self[name] is not None else default
|
|
107
232
|
|
|
108
233
|
def getbool(self, name, default=False):
|
|
109
234
|
"""
|
|
110
235
|
Get a setting value as a boolean.
|
|
236
|
+
将设置值作为布尔值获取。
|
|
111
237
|
|
|
112
|
-
``1``, ``'1'``,
|
|
238
|
+
``1``, ``'1'``, ``True`` and ``'True'`` return ``True``,
|
|
113
239
|
while ``0``, ``'0'``, ``False``, ``'False'`` and ``None`` return ``False``.
|
|
240
|
+
``1``、``'1'``、``True``和``'True'``返回``True``,
|
|
241
|
+
而``0``、``'0'``、``False``、``'False'``和``None``返回``False``。
|
|
114
242
|
|
|
115
243
|
For example, settings populated through environment variables set to
|
|
116
244
|
``'0'`` will return ``False`` when using this method.
|
|
245
|
+
例如,通过环境变量设置为``'0'``的设置在使用此方法时将返回``False``。
|
|
117
246
|
|
|
118
|
-
:
|
|
119
|
-
|
|
247
|
+
Args:
|
|
248
|
+
name: The setting name.
|
|
249
|
+
设置名称。
|
|
250
|
+
default: The value to return if no setting is found.
|
|
251
|
+
如果未找到设置,则返回的值。
|
|
120
252
|
|
|
121
|
-
:
|
|
122
|
-
|
|
253
|
+
Returns:
|
|
254
|
+
bool: The boolean value of the setting.
|
|
255
|
+
设置的布尔值。
|
|
256
|
+
|
|
257
|
+
Raises:
|
|
258
|
+
ValueError: If the value cannot be converted to a boolean.
|
|
259
|
+
如果值无法转换为布尔值。
|
|
123
260
|
"""
|
|
124
261
|
got = self.get(name, default)
|
|
125
262
|
try:
|
|
263
|
+
# Try to convert to int first, then to bool
|
|
264
|
+
# 首先尝试转换为int,然后转换为bool
|
|
126
265
|
return bool(int(got))
|
|
127
266
|
except ValueError:
|
|
267
|
+
# Handle string boolean values
|
|
268
|
+
# 处理字符串布尔值
|
|
128
269
|
if got in ("True", "true"):
|
|
129
270
|
return True
|
|
130
271
|
if got in ("False", "false"):
|
|
@@ -136,40 +277,68 @@ class BaseSettings(MutableMapping):
|
|
|
136
277
|
def getint(self, name, default=0):
|
|
137
278
|
"""
|
|
138
279
|
Get a setting value as an int.
|
|
280
|
+
将设置值作为整数获取。
|
|
281
|
+
|
|
282
|
+
Args:
|
|
283
|
+
name: The setting name.
|
|
284
|
+
设置名称。
|
|
285
|
+
default: The value to return if no setting is found.
|
|
286
|
+
如果未找到设置,则返回的值。
|
|
139
287
|
|
|
140
|
-
:
|
|
141
|
-
|
|
288
|
+
Returns:
|
|
289
|
+
int: The integer value of the setting.
|
|
290
|
+
设置的整数值。
|
|
142
291
|
|
|
143
|
-
:
|
|
144
|
-
|
|
292
|
+
Raises:
|
|
293
|
+
ValueError: If the value cannot be converted to an int.
|
|
294
|
+
如果值无法转换为整数。
|
|
145
295
|
"""
|
|
146
296
|
return int(self.get(name, default))
|
|
147
297
|
|
|
148
298
|
def getfloat(self, name, default=0.0):
|
|
149
299
|
"""
|
|
150
300
|
Get a setting value as a float.
|
|
301
|
+
将设置值作为浮点数获取。
|
|
302
|
+
|
|
303
|
+
Args:
|
|
304
|
+
name: The setting name.
|
|
305
|
+
设置名称。
|
|
306
|
+
default: The value to return if no setting is found.
|
|
307
|
+
如果未找到设置,则返回的值。
|
|
151
308
|
|
|
152
|
-
:
|
|
153
|
-
|
|
309
|
+
Returns:
|
|
310
|
+
float: The float value of the setting.
|
|
311
|
+
设置的浮点值。
|
|
154
312
|
|
|
155
|
-
:
|
|
156
|
-
|
|
313
|
+
Raises:
|
|
314
|
+
ValueError: If the value cannot be converted to a float.
|
|
315
|
+
如果值无法转换为浮点数。
|
|
157
316
|
"""
|
|
158
317
|
return float(self.get(name, default))
|
|
159
318
|
|
|
160
319
|
def getlist(self, name, default=None):
|
|
161
320
|
"""
|
|
162
|
-
Get a setting value as a list.
|
|
163
|
-
|
|
321
|
+
Get a setting value as a list.
|
|
322
|
+
将设置值作为列表获取。
|
|
323
|
+
|
|
324
|
+
If the setting original type is a list, a copy of it will be returned.
|
|
325
|
+
If it's a string it will be split by ",".
|
|
326
|
+
如果设置的原始类型是列表,将返回其副本。
|
|
327
|
+
如果是字符串,将按","拆分。
|
|
164
328
|
|
|
165
329
|
For example, settings populated through environment variables set to
|
|
166
330
|
``'one,two'`` will return a list ['one', 'two'] when using this method.
|
|
331
|
+
例如,通过环境变量设置为``'one,two'``的设置在使用此方法时将返回列表['one', 'two']。
|
|
167
332
|
|
|
168
|
-
:
|
|
169
|
-
|
|
333
|
+
Args:
|
|
334
|
+
name: The setting name.
|
|
335
|
+
设置名称。
|
|
336
|
+
default: The value to return if no setting is found.
|
|
337
|
+
如果未找到设置,则返回的值。
|
|
170
338
|
|
|
171
|
-
:
|
|
172
|
-
|
|
339
|
+
Returns:
|
|
340
|
+
list: The list value of the setting.
|
|
341
|
+
设置的列表值。
|
|
173
342
|
"""
|
|
174
343
|
value = self.get(name, default or [])
|
|
175
344
|
if isinstance(value, str):
|
|
@@ -178,19 +347,33 @@ class BaseSettings(MutableMapping):
|
|
|
178
347
|
|
|
179
348
|
def getdict(self, name, default=None):
|
|
180
349
|
"""
|
|
181
|
-
Get a setting value as a dictionary.
|
|
182
|
-
|
|
183
|
-
evaluated as a JSON dictionary. In the case that it is a
|
|
184
|
-
:class:`~scrapy.settings.BaseSettings` instance itself, it will be
|
|
185
|
-
converted to a dictionary, containing all its current settings values
|
|
186
|
-
as they would be returned by :meth:`~scrapy.settings.BaseSettings.get`,
|
|
187
|
-
and losing all information about priority and mutability.
|
|
350
|
+
Get a setting value as a dictionary.
|
|
351
|
+
将设置值作为字典获取。
|
|
188
352
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
353
|
+
If the setting original type is a dictionary, a copy of it will be returned.
|
|
354
|
+
If it is a string it will be evaluated as a JSON dictionary.
|
|
355
|
+
In the case that it is a BaseSettings instance itself, it will be
|
|
356
|
+
converted to a dictionary, containing all its current settings values
|
|
357
|
+
as they would be returned by get, and losing all information about
|
|
358
|
+
priority and mutability.
|
|
359
|
+
如果设置的原始类型是字典,将返回其副本。
|
|
360
|
+
如果是字符串,将被评估为JSON字典。
|
|
361
|
+
如果它本身是BaseSettings实例,它将被转换为字典,包含所有当前设置值,
|
|
362
|
+
就像它们由get返回一样,并丢失有关优先级和可变性的所有信息。
|
|
363
|
+
|
|
364
|
+
Args:
|
|
365
|
+
name: The setting name.
|
|
366
|
+
设置名称。
|
|
367
|
+
default: The value to return if no setting is found.
|
|
368
|
+
如果未找到设置,则返回的值。
|
|
369
|
+
|
|
370
|
+
Returns:
|
|
371
|
+
dict: The dictionary value of the setting.
|
|
372
|
+
设置的字典值。
|
|
373
|
+
|
|
374
|
+
Raises:
|
|
375
|
+
ValueError: If the string value cannot be parsed as JSON.
|
|
376
|
+
如果字符串值无法解析为JSON。
|
|
194
377
|
"""
|
|
195
378
|
value = self.get(name, default or {})
|
|
196
379
|
if isinstance(value, str):
|
|
@@ -198,11 +381,26 @@ class BaseSettings(MutableMapping):
|
|
|
198
381
|
return dict(value)
|
|
199
382
|
|
|
200
383
|
def getwithbase(self, name):
|
|
201
|
-
"""
|
|
202
|
-
|
|
384
|
+
"""
|
|
385
|
+
Get a composition of a dictionary-like setting and its base variants.
|
|
386
|
+
获取字典类设置及其基础变体的组合。
|
|
387
|
+
|
|
388
|
+
This method creates a new BaseSettings instance that combines three settings:
|
|
389
|
+
name + '_BASE', name, and name + '_SPIDER', with increasing priority.
|
|
390
|
+
此方法创建一个新的BaseSettings实例,它结合了三个设置:
|
|
391
|
+
name + '_BASE'、name和name + '_SPIDER',优先级依次增加。
|
|
392
|
+
|
|
393
|
+
This is useful for settings that have base values and spider-specific overrides,
|
|
394
|
+
such as middlewares and extensions.
|
|
395
|
+
这对于具有基本值和爬虫特定覆盖的设置很有用,例如中间件和扩展。
|
|
203
396
|
|
|
204
|
-
:
|
|
205
|
-
|
|
397
|
+
Args:
|
|
398
|
+
name: Name of the dictionary-like setting without the '_BASE' or '_SPIDER' suffix.
|
|
399
|
+
不带'_BASE'或'_SPIDER'后缀的字典类设置的名称。
|
|
400
|
+
|
|
401
|
+
Returns:
|
|
402
|
+
BaseSettings: A new BaseSettings instance with the combined settings.
|
|
403
|
+
包含组合设置的新BaseSettings实例。
|
|
206
404
|
"""
|
|
207
405
|
compbs = BaseSettings()
|
|
208
406
|
compbs.update(self[name + '_BASE'])
|
|
@@ -212,11 +410,20 @@ class BaseSettings(MutableMapping):
|
|
|
212
410
|
|
|
213
411
|
def getpriority(self, name):
|
|
214
412
|
"""
|
|
215
|
-
Return the current numerical priority value of a setting
|
|
216
|
-
|
|
413
|
+
Return the current numerical priority value of a setting.
|
|
414
|
+
返回设置的当前数值优先级。
|
|
415
|
+
|
|
416
|
+
This method returns the priority of a setting, which determines whether
|
|
417
|
+
the setting can be overridden by other settings with different priorities.
|
|
418
|
+
此方法返回设置的优先级,它决定了该设置是否可以被具有不同优先级的其他设置覆盖。
|
|
217
419
|
|
|
218
|
-
:
|
|
219
|
-
|
|
420
|
+
Args:
|
|
421
|
+
name: The name of the setting.
|
|
422
|
+
设置的名称。
|
|
423
|
+
|
|
424
|
+
Returns:
|
|
425
|
+
int or None: The numerical priority value of the setting, or None if the setting doesn't exist.
|
|
426
|
+
设置的数值优先级,如果设置不存在则为None。
|
|
220
427
|
"""
|
|
221
428
|
if name not in self:
|
|
222
429
|
return None
|
|
@@ -224,10 +431,18 @@ class BaseSettings(MutableMapping):
|
|
|
224
431
|
|
|
225
432
|
def maxpriority(self):
|
|
226
433
|
"""
|
|
227
|
-
Return the
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
434
|
+
Return the highest priority value among all settings.
|
|
435
|
+
返回所有设置中的最高优先级值。
|
|
436
|
+
|
|
437
|
+
This method scans through all settings and returns the highest priority value.
|
|
438
|
+
If there are no settings stored, it returns the priority value for 'default'.
|
|
439
|
+
此方法扫描所有设置并返回最高优先级值。
|
|
440
|
+
如果没有存储设置,则返回'default'的优先级值。
|
|
441
|
+
|
|
442
|
+
Returns:
|
|
443
|
+
int: The highest priority value among all settings, or the priority value for 'default'
|
|
444
|
+
if there are no settings.
|
|
445
|
+
所有设置中的最高优先级值,如果没有设置则为'default'的优先级值。
|
|
231
446
|
"""
|
|
232
447
|
if len(self) > 0:
|
|
233
448
|
return max(self.getpriority(name) for name in self)
|
|
@@ -240,29 +455,43 @@ class BaseSettings(MutableMapping):
|
|
|
240
455
|
def set(self, name, value, priority='project'):
|
|
241
456
|
"""
|
|
242
457
|
Store a key/value attribute with a given priority.
|
|
458
|
+
存储具有给定优先级的键/值属性。
|
|
243
459
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
460
|
+
This method sets a setting with the specified name, value, and priority.
|
|
461
|
+
If a setting with the same name already exists, it will be updated only
|
|
462
|
+
if the new priority is equal to or higher than the existing priority.
|
|
463
|
+
此方法设置具有指定名称、值和优先级的设置。
|
|
464
|
+
如果已存在同名设置,则仅当新优先级等于或高于现有优先级时才会更新。
|
|
247
465
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
:
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
:
|
|
256
|
-
|
|
466
|
+
Settings should be populated *before* configuring the Crawler object,
|
|
467
|
+
otherwise they won't have any effect.
|
|
468
|
+
应在配置Crawler对象之前填充设置,否则它们将不会产生任何效果。
|
|
469
|
+
|
|
470
|
+
Args:
|
|
471
|
+
name: The setting name.
|
|
472
|
+
设置名称。
|
|
473
|
+
value: The value to associate with the setting.
|
|
474
|
+
与设置关联的值。
|
|
475
|
+
priority: The priority of the setting. Should be a key of
|
|
476
|
+
SETTINGS_PRIORITIES or an integer.
|
|
477
|
+
设置的优先级。应该是SETTINGS_PRIORITIES的键或整数。
|
|
478
|
+
|
|
479
|
+
Raises:
|
|
480
|
+
TypeError: If the settings object is frozen (immutable).
|
|
481
|
+
如果设置对象被冻结(不可变)。
|
|
257
482
|
"""
|
|
258
483
|
self._assert_mutability()
|
|
259
484
|
priority = get_settings_priority(priority)
|
|
260
485
|
if name not in self:
|
|
486
|
+
# If the setting doesn't exist, create a new SettingsAttribute
|
|
487
|
+
# 如果设置不存在,创建一个新的SettingsAttribute
|
|
261
488
|
if isinstance(value, SettingsAttribute):
|
|
262
489
|
self.attributes[name] = value
|
|
263
490
|
else:
|
|
264
491
|
self.attributes[name] = SettingsAttribute(value, priority)
|
|
265
492
|
else:
|
|
493
|
+
# If the setting exists, update it with the new value and priority
|
|
494
|
+
# 如果设置存在,使用新的值和优先级更新它
|
|
266
495
|
self.attributes[name].set(value, priority)
|
|
267
496
|
|
|
268
497
|
def setdict(self, values, priority='project'):
|
|
@@ -271,21 +500,36 @@ class BaseSettings(MutableMapping):
|
|
|
271
500
|
def setmodule(self, module, priority='project'):
|
|
272
501
|
"""
|
|
273
502
|
Store settings from a module with a given priority.
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
uppercase variable of
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
503
|
+
以给定优先级存储来自模块的设置。
|
|
504
|
+
|
|
505
|
+
This is a helper function that calls set() for every globally declared
|
|
506
|
+
uppercase variable of the module with the provided priority.
|
|
507
|
+
这是一个辅助函数,它为模块的每个全局声明的大写变量调用set(),并使用提供的优先级。
|
|
508
|
+
|
|
509
|
+
This is commonly used to load settings from a settings module.
|
|
510
|
+
这通常用于从设置模块加载设置。
|
|
511
|
+
|
|
512
|
+
Args:
|
|
513
|
+
module: The module or the path of the module.
|
|
514
|
+
模块或模块的路径。
|
|
515
|
+
priority: The priority of the settings. Should be a key of
|
|
516
|
+
SETTINGS_PRIORITIES or an integer.
|
|
517
|
+
设置的优先级。应该是SETTINGS_PRIORITIES的键或整数。
|
|
518
|
+
|
|
519
|
+
Raises:
|
|
520
|
+
TypeError: If the settings object is frozen (immutable).
|
|
521
|
+
如果设置对象被冻结(不可变)。
|
|
522
|
+
ImportError: If the module path cannot be imported.
|
|
523
|
+
如果无法导入模块路径。
|
|
285
524
|
"""
|
|
286
525
|
self._assert_mutability()
|
|
287
526
|
if isinstance(module, str):
|
|
527
|
+
# If module is a string, import it
|
|
528
|
+
# 如果模块是字符串,导入它
|
|
288
529
|
module = import_module(module)
|
|
530
|
+
|
|
531
|
+
# Set all uppercase variables from the module
|
|
532
|
+
# 设置模块中的所有大写变量
|
|
289
533
|
for key in dir(module):
|
|
290
534
|
if key.isupper():
|
|
291
535
|
self.set(key, getattr(module, key), priority)
|
|
@@ -293,37 +537,73 @@ class BaseSettings(MutableMapping):
|
|
|
293
537
|
def update(self, values, priority='project'):
|
|
294
538
|
"""
|
|
295
539
|
Store key/value pairs with a given priority.
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
with the provided
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
will be used and the
|
|
305
|
-
inserting/updating settings with different priorities with
|
|
306
|
-
command.
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
:
|
|
312
|
-
:
|
|
313
|
-
|
|
540
|
+
以给定优先级存储键/值对。
|
|
541
|
+
|
|
542
|
+
This is a helper function that calls set() for every item of values
|
|
543
|
+
with the provided priority.
|
|
544
|
+
这是一个辅助函数,它为values的每个项目调用set(),并使用提供的优先级。
|
|
545
|
+
|
|
546
|
+
If values is a string, it is assumed to be JSON-encoded and parsed
|
|
547
|
+
into a dict with json.loads() first. If it is a BaseSettings instance,
|
|
548
|
+
the per-key priorities will be used and the priority parameter ignored.
|
|
549
|
+
This allows inserting/updating settings with different priorities with
|
|
550
|
+
a single command.
|
|
551
|
+
如果values是字符串,则假定它是JSON编码的,并首先使用json.loads()解析为字典。
|
|
552
|
+
如果它是BaseSettings实例,则将使用每个键的优先级,并忽略priority参数。
|
|
553
|
+
这允许使用单个命令插入/更新具有不同优先级的设置。
|
|
554
|
+
|
|
555
|
+
Args:
|
|
556
|
+
values: The settings names and values.
|
|
557
|
+
设置名称和值。
|
|
558
|
+
Can be a dict, a JSON string, or a BaseSettings instance.
|
|
559
|
+
可以是字典、JSON字符串或BaseSettings实例。
|
|
560
|
+
priority: The priority of the settings. Should be a key of
|
|
561
|
+
SETTINGS_PRIORITIES or an integer.
|
|
562
|
+
设置的优先级。应该是SETTINGS_PRIORITIES的键或整数。
|
|
563
|
+
|
|
564
|
+
Raises:
|
|
565
|
+
TypeError: If the settings object is frozen (immutable).
|
|
566
|
+
如果设置对象被冻结(不可变)。
|
|
567
|
+
ValueError: If values is a string that cannot be parsed as JSON.
|
|
568
|
+
如果values是无法解析为JSON的字符串。
|
|
314
569
|
"""
|
|
315
570
|
self._assert_mutability()
|
|
316
571
|
if isinstance(values, str):
|
|
572
|
+
# Parse JSON string into a dict
|
|
573
|
+
# 将JSON字符串解析为字典
|
|
317
574
|
values = json.loads(values)
|
|
575
|
+
|
|
318
576
|
if values is not None:
|
|
319
577
|
if isinstance(values, BaseSettings):
|
|
578
|
+
# If values is a BaseSettings, use its per-key priorities
|
|
579
|
+
# 如果values是BaseSettings,使用其每个键的优先级
|
|
320
580
|
for name, value in values.items():
|
|
321
581
|
self.set(name, value, values.getpriority(name))
|
|
322
582
|
else:
|
|
583
|
+
# Otherwise, use the provided priority for all items
|
|
584
|
+
# 否则,对所有项目使用提供的优先级
|
|
323
585
|
for name, value in values.items():
|
|
324
586
|
self.set(name, value, priority)
|
|
325
587
|
|
|
326
588
|
def delete(self, name, priority='project'):
|
|
589
|
+
"""
|
|
590
|
+
Delete a setting if the given priority is higher or equal than its priority.
|
|
591
|
+
如果给定优先级高于或等于设置的优先级,则删除该设置。
|
|
592
|
+
|
|
593
|
+
This method deletes a setting with the specified name if the provided
|
|
594
|
+
priority is higher than or equal to the setting's priority.
|
|
595
|
+
此方法删除具有指定名称的设置,如果提供的优先级高于或等于设置的优先级。
|
|
596
|
+
|
|
597
|
+
Args:
|
|
598
|
+
name: The name of the setting to delete.
|
|
599
|
+
要删除的设置的名称。
|
|
600
|
+
priority: The priority level. Should be a key of SETTINGS_PRIORITIES or an integer.
|
|
601
|
+
优先级级别。应该是SETTINGS_PRIORITIES的键或整数。
|
|
602
|
+
|
|
603
|
+
Raises:
|
|
604
|
+
TypeError: If the settings object is frozen (immutable).
|
|
605
|
+
如果设置对象被冻结(不可变)。
|
|
606
|
+
"""
|
|
327
607
|
self._assert_mutability()
|
|
328
608
|
priority = get_settings_priority(priority)
|
|
329
609
|
if priority >= self.getpriority(name):
|
|
@@ -334,40 +614,75 @@ class BaseSettings(MutableMapping):
|
|
|
334
614
|
del self.attributes[name]
|
|
335
615
|
|
|
336
616
|
def _assert_mutability(self):
|
|
617
|
+
"""
|
|
618
|
+
Assert that the settings object is mutable.
|
|
619
|
+
断言设置对象是可变的。
|
|
620
|
+
|
|
621
|
+
This internal method checks if the settings object is frozen (immutable)
|
|
622
|
+
and raises a TypeError if it is.
|
|
623
|
+
此内部方法检查设置对象是否被冻结(不可变),如果是则引发TypeError。
|
|
624
|
+
|
|
625
|
+
Raises:
|
|
626
|
+
TypeError: If the settings object is frozen (immutable).
|
|
627
|
+
如果设置对象被冻结(不可变)。
|
|
628
|
+
"""
|
|
337
629
|
if self.frozen:
|
|
338
630
|
raise TypeError("Trying to modify an immutable Settings object")
|
|
339
631
|
|
|
340
632
|
def copy(self):
|
|
341
633
|
"""
|
|
342
634
|
Make a deep copy of current settings.
|
|
635
|
+
创建当前设置的深拷贝。
|
|
343
636
|
|
|
344
|
-
This method returns a new instance of the
|
|
637
|
+
This method returns a new instance of the BaseSettings class,
|
|
345
638
|
populated with the same values and their priorities.
|
|
639
|
+
此方法返回BaseSettings类的新实例,填充相同的值及其优先级。
|
|
346
640
|
|
|
347
641
|
Modifications to the new object won't be reflected on the original
|
|
348
|
-
settings.
|
|
642
|
+
settings, and vice versa.
|
|
643
|
+
对新对象的修改不会反映在原始设置上,反之亦然。
|
|
644
|
+
|
|
645
|
+
Returns:
|
|
646
|
+
BaseSettings: A new instance with the same settings and priorities.
|
|
647
|
+
具有相同设置和优先级的新实例。
|
|
349
648
|
"""
|
|
350
649
|
return copy.deepcopy(self)
|
|
351
650
|
|
|
352
651
|
def freeze(self):
|
|
353
652
|
"""
|
|
354
653
|
Disable further changes to the current settings.
|
|
654
|
+
禁止对当前设置进行进一步更改。
|
|
355
655
|
|
|
356
656
|
After calling this method, the present state of the settings will become
|
|
357
|
-
immutable. Trying to change values through the
|
|
358
|
-
its variants won't be possible and will
|
|
657
|
+
immutable. Trying to change values through the set() method and
|
|
658
|
+
its variants won't be possible and will raise a TypeError.
|
|
659
|
+
调用此方法后,设置的当前状态将变为不可变。
|
|
660
|
+
尝试通过set()方法及其变体更改值将不可能,并将引发TypeError。
|
|
661
|
+
|
|
662
|
+
Returns:
|
|
663
|
+
None
|
|
359
664
|
"""
|
|
360
665
|
self.frozen = True
|
|
361
666
|
|
|
362
667
|
def frozencopy(self):
|
|
363
668
|
"""
|
|
364
669
|
Return an immutable copy of the current settings.
|
|
670
|
+
返回当前设置的不可变副本。
|
|
365
671
|
|
|
366
|
-
|
|
672
|
+
This method creates a copy of the current settings and then freezes it,
|
|
673
|
+
making it immutable.
|
|
674
|
+
此方法创建当前设置的副本,然后冻结它,使其不可变。
|
|
675
|
+
|
|
676
|
+
It's equivalent to calling copy() followed by freeze().
|
|
677
|
+
它相当于调用copy()后跟freeze()。
|
|
678
|
+
|
|
679
|
+
Returns:
|
|
680
|
+
BaseSettings: An immutable copy of the current settings.
|
|
681
|
+
当前设置的不可变副本。
|
|
367
682
|
"""
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
return
|
|
683
|
+
settings_copy = self.copy()
|
|
684
|
+
settings_copy.freeze()
|
|
685
|
+
return settings_copy
|
|
371
686
|
|
|
372
687
|
def __iter__(self):
|
|
373
688
|
return iter(self.attributes)
|
|
@@ -376,90 +691,257 @@ class BaseSettings(MutableMapping):
|
|
|
376
691
|
return len(self.attributes)
|
|
377
692
|
|
|
378
693
|
def _to_dict(self):
|
|
694
|
+
"""
|
|
695
|
+
Internal method to convert settings to a dictionary.
|
|
696
|
+
将设置转换为字典的内部方法。
|
|
697
|
+
|
|
698
|
+
This method recursively converts BaseSettings instances to dictionaries.
|
|
699
|
+
此方法递归地将BaseSettings实例转换为字典。
|
|
700
|
+
|
|
701
|
+
Returns:
|
|
702
|
+
dict: A dictionary representation of the settings.
|
|
703
|
+
设置的字典表示。
|
|
704
|
+
"""
|
|
379
705
|
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
|
|
380
706
|
for k, v in self.items()}
|
|
381
707
|
|
|
382
708
|
def copy_to_dict(self):
|
|
383
709
|
"""
|
|
384
710
|
Make a copy of current settings and convert to a dict.
|
|
711
|
+
创建当前设置的副本并转换为字典。
|
|
385
712
|
|
|
386
713
|
This method returns a new dict populated with the same values
|
|
387
|
-
|
|
714
|
+
as the current settings, with nested BaseSettings also converted to dicts.
|
|
715
|
+
此方法返回一个新字典,填充与当前设置相同的值,嵌套的BaseSettings也转换为字典。
|
|
388
716
|
|
|
389
717
|
Modifications to the returned dict won't be reflected on the original
|
|
390
|
-
settings.
|
|
718
|
+
settings, and vice versa.
|
|
719
|
+
对返回的字典的修改不会反映在原始设置上,反之亦然。
|
|
391
720
|
|
|
392
721
|
This method can be useful for example for printing settings
|
|
393
|
-
|
|
722
|
+
or for serialization.
|
|
723
|
+
此方法例如对于打印设置或序列化很有用。
|
|
724
|
+
|
|
725
|
+
Returns:
|
|
726
|
+
dict: A dictionary representation of the settings.
|
|
727
|
+
设置的字典表示。
|
|
394
728
|
"""
|
|
395
729
|
settings = self.copy()
|
|
396
730
|
return settings._to_dict()
|
|
397
731
|
|
|
398
732
|
def _repr_pretty_(self, p, cycle):
|
|
733
|
+
"""
|
|
734
|
+
Pretty-printing support for IPython/Jupyter.
|
|
735
|
+
IPython/Jupyter的美观打印支持。
|
|
736
|
+
|
|
737
|
+
This method is called by IPython/Jupyter to format the object for display.
|
|
738
|
+
此方法由IPython/Jupyter调用,用于格式化对象以供显示。
|
|
739
|
+
|
|
740
|
+
Args:
|
|
741
|
+
p: The pretty printer instance.
|
|
742
|
+
美观打印器实例。
|
|
743
|
+
cycle: Whether a cycle was detected in the object graph.
|
|
744
|
+
是否在对象图中检测到循环。
|
|
745
|
+
|
|
746
|
+
Returns:
|
|
747
|
+
None
|
|
748
|
+
"""
|
|
399
749
|
if cycle:
|
|
750
|
+
# If a cycle is detected, use the standard repr
|
|
751
|
+
# 如果检测到循环,使用标准的repr
|
|
400
752
|
p.text(repr(self))
|
|
401
753
|
else:
|
|
754
|
+
# Otherwise, format the settings as a pretty-printed dict
|
|
755
|
+
# 否则,将设置格式化为美观打印的字典
|
|
402
756
|
p.text(pformat(self.copy_to_dict()))
|
|
403
757
|
|
|
404
758
|
|
|
405
759
|
class _DictProxy(MutableMapping):
|
|
760
|
+
"""
|
|
761
|
+
Dictionary proxy that updates settings when modified.
|
|
762
|
+
修改时更新设置的字典代理。
|
|
763
|
+
|
|
764
|
+
This class is used internally to provide a dictionary-like interface
|
|
765
|
+
that updates settings with a specific priority when modified.
|
|
766
|
+
此类在内部用于提供类字典接口,在修改时以特定优先级更新设置。
|
|
767
|
+
"""
|
|
406
768
|
|
|
407
769
|
def __init__(self, settings, priority):
|
|
770
|
+
"""
|
|
771
|
+
Initialize a _DictProxy.
|
|
772
|
+
初始化_DictProxy。
|
|
773
|
+
|
|
774
|
+
Args:
|
|
775
|
+
settings: The settings object to update.
|
|
776
|
+
要更新的设置对象。
|
|
777
|
+
priority: The priority to use when updating settings.
|
|
778
|
+
更新设置时使用的优先级。
|
|
779
|
+
"""
|
|
408
780
|
self.o = {}
|
|
409
781
|
self.settings = settings
|
|
410
782
|
self.priority = priority
|
|
411
783
|
|
|
412
784
|
def __len__(self):
|
|
785
|
+
"""
|
|
786
|
+
Return the number of items in the dictionary.
|
|
787
|
+
返回字典中的项目数。
|
|
788
|
+
|
|
789
|
+
Returns:
|
|
790
|
+
int: The number of items in the dictionary.
|
|
791
|
+
字典中的项目数。
|
|
792
|
+
"""
|
|
413
793
|
return len(self.o)
|
|
414
794
|
|
|
415
795
|
def __getitem__(self, k):
|
|
796
|
+
"""
|
|
797
|
+
Get an item from the dictionary.
|
|
798
|
+
从字典中获取项目。
|
|
799
|
+
|
|
800
|
+
Args:
|
|
801
|
+
k: The key to get.
|
|
802
|
+
要获取的键。
|
|
803
|
+
|
|
804
|
+
Returns:
|
|
805
|
+
The value for the key.
|
|
806
|
+
键的值。
|
|
807
|
+
|
|
808
|
+
Raises:
|
|
809
|
+
KeyError: If the key is not found.
|
|
810
|
+
如果未找到键。
|
|
811
|
+
"""
|
|
416
812
|
return self.o[k]
|
|
417
813
|
|
|
418
814
|
def __setitem__(self, k, v):
|
|
815
|
+
"""
|
|
816
|
+
Set an item in the dictionary and update the settings.
|
|
817
|
+
在字典中设置项目并更新设置。
|
|
818
|
+
|
|
819
|
+
Args:
|
|
820
|
+
k: The key to set.
|
|
821
|
+
要设置的键。
|
|
822
|
+
v: The value to set.
|
|
823
|
+
要设置的值。
|
|
824
|
+
"""
|
|
419
825
|
self.settings.set(k, v, priority=self.priority)
|
|
420
826
|
self.o[k] = v
|
|
421
827
|
|
|
422
828
|
def __delitem__(self, k):
|
|
829
|
+
"""
|
|
830
|
+
Delete an item from the dictionary.
|
|
831
|
+
从字典中删除项目。
|
|
832
|
+
|
|
833
|
+
Args:
|
|
834
|
+
k: The key to delete.
|
|
835
|
+
要删除的键。
|
|
836
|
+
|
|
837
|
+
Raises:
|
|
838
|
+
KeyError: If the key is not found.
|
|
839
|
+
如果未找到键。
|
|
840
|
+
"""
|
|
423
841
|
del self.o[k]
|
|
424
842
|
|
|
425
|
-
def __iter__(self
|
|
843
|
+
def __iter__(self):
|
|
844
|
+
"""
|
|
845
|
+
Return an iterator over the dictionary keys.
|
|
846
|
+
返回字典键的迭代器。
|
|
847
|
+
|
|
848
|
+
Returns:
|
|
849
|
+
iterator: An iterator over the dictionary keys.
|
|
850
|
+
字典键的迭代器。
|
|
851
|
+
"""
|
|
426
852
|
return iter(self.o)
|
|
427
853
|
|
|
428
854
|
|
|
429
855
|
class Settings(BaseSettings):
|
|
430
856
|
"""
|
|
431
|
-
|
|
432
|
-
|
|
857
|
+
Settings container for aioscrapy with default settings.
|
|
858
|
+
具有默认设置的aioscrapy设置容器。
|
|
433
859
|
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
860
|
+
This object stores aioscrapy settings for the configuration of internal
|
|
861
|
+
components, and can be used for any further customization.
|
|
862
|
+
此对象存储用于配置内部组件的aioscrapy设置,可用于任何进一步的自定义。
|
|
863
|
+
|
|
864
|
+
It is a direct subclass and supports all methods of BaseSettings.
|
|
865
|
+
Additionally, after instantiation of this class, the new object will
|
|
866
|
+
have the global default settings from default_settings already populated.
|
|
867
|
+
它是BaseSettings的直接子类,支持BaseSettings的所有方法。
|
|
868
|
+
此外,在实例化此类后,新对象将已经填充了来自default_settings的全局默认设置。
|
|
869
|
+
|
|
870
|
+
Attributes:
|
|
871
|
+
frozen (bool): Whether the settings are frozen (immutable).
|
|
872
|
+
设置是否被冻结(不可变)。
|
|
873
|
+
attributes (dict): Dictionary storing the settings as SettingsAttribute objects.
|
|
874
|
+
存储设置为SettingsAttribute对象的字典。
|
|
438
875
|
"""
|
|
439
876
|
|
|
440
877
|
def __init__(self, values=None, priority='project'):
|
|
878
|
+
"""
|
|
879
|
+
Initialize a Settings instance with default settings.
|
|
880
|
+
使用默认设置初始化Settings实例。
|
|
881
|
+
|
|
882
|
+
Args:
|
|
883
|
+
values: Initial settings values as a dict or BaseSettings instance.
|
|
884
|
+
作为字典或BaseSettings实例的初始设置值。
|
|
885
|
+
priority: Priority level for the initial values.
|
|
886
|
+
初始值的优先级。
|
|
887
|
+
"""
|
|
441
888
|
# Do not pass kwarg values here. We don't want to promote user-defined
|
|
442
889
|
# dicts, and we want to update, not replace, default dicts with the
|
|
443
890
|
# values given by the user
|
|
891
|
+
# 不要在这里传递kwarg值。我们不想提升用户定义的字典,
|
|
892
|
+
# 我们想用用户给定的值更新默认字典,而不是替换它们
|
|
444
893
|
super().__init__()
|
|
894
|
+
|
|
895
|
+
# Load default settings with 'default' priority
|
|
896
|
+
# 加载具有'default'优先级的默认设置
|
|
445
897
|
self.setmodule(default_settings, 'default')
|
|
446
|
-
|
|
447
|
-
# priorities
|
|
898
|
+
|
|
899
|
+
# Promote default dictionaries to BaseSettings instances for per-key priorities
|
|
900
|
+
# 将默认字典提升为BaseSettings实例以实现每个键的优先级
|
|
448
901
|
for name, val in self.items():
|
|
449
902
|
if isinstance(val, dict):
|
|
450
903
|
self.set(name, BaseSettings(val, 'default'), 'default')
|
|
904
|
+
|
|
905
|
+
# Update with user-provided values
|
|
906
|
+
# 使用用户提供的值更新
|
|
451
907
|
self.update(values, priority)
|
|
452
908
|
|
|
453
909
|
|
|
454
910
|
def iter_default_settings():
|
|
455
|
-
"""
|
|
911
|
+
"""
|
|
912
|
+
Return the default settings as an iterator.
|
|
913
|
+
将默认设置作为迭代器返回。
|
|
914
|
+
|
|
915
|
+
This function iterates through all uppercase attributes in the default_settings
|
|
916
|
+
module and yields them as (name, value) tuples.
|
|
917
|
+
此函数遍历default_settings模块中的所有大写属性,并将它们作为(名称, 值)元组生成。
|
|
918
|
+
|
|
919
|
+
Returns:
|
|
920
|
+
iterator: An iterator of (name, value) tuples of default settings.
|
|
921
|
+
默认设置的(名称, 值)元组的迭代器。
|
|
922
|
+
"""
|
|
456
923
|
for name in dir(default_settings):
|
|
457
924
|
if name.isupper():
|
|
458
925
|
yield name, getattr(default_settings, name)
|
|
459
926
|
|
|
460
927
|
|
|
461
928
|
def overridden_settings(settings):
|
|
462
|
-
"""
|
|
929
|
+
"""
|
|
930
|
+
Return a dict of the settings that have been overridden.
|
|
931
|
+
返回已被覆盖的设置的字典。
|
|
932
|
+
|
|
933
|
+
This function compares the values in the provided settings object with
|
|
934
|
+
the default values and yields the ones that have been changed.
|
|
935
|
+
此函数将提供的设置对象中的值与默认值进行比较,并生成已更改的值。
|
|
936
|
+
|
|
937
|
+
Args:
|
|
938
|
+
settings: The settings object to check.
|
|
939
|
+
要检查的设置对象。
|
|
940
|
+
|
|
941
|
+
Returns:
|
|
942
|
+
iterator: An iterator of (name, value) tuples of overridden settings.
|
|
943
|
+
已覆盖设置的(名称, 值)元组的迭代器。
|
|
944
|
+
"""
|
|
463
945
|
for name, defvalue in iter_default_settings():
|
|
464
946
|
value = settings[name]
|
|
465
947
|
if not isinstance(defvalue, dict) and value != defvalue:
|