aio-scrapy 2.1.4__py3-none-any.whl → 2.1.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.6.dist-info}/LICENSE +1 -1
  2. {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.6.dist-info}/METADATA +53 -41
  3. aio_scrapy-2.1.6.dist-info/RECORD +134 -0
  4. {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.6.dist-info}/WHEEL +1 -1
  5. aioscrapy/VERSION +1 -1
  6. aioscrapy/cmdline.py +438 -5
  7. aioscrapy/core/downloader/__init__.py +522 -17
  8. aioscrapy/core/downloader/handlers/__init__.py +187 -5
  9. aioscrapy/core/downloader/handlers/aiohttp.py +187 -3
  10. aioscrapy/core/downloader/handlers/curl_cffi.py +124 -3
  11. aioscrapy/core/downloader/handlers/httpx.py +133 -3
  12. aioscrapy/core/downloader/handlers/pyhttpx.py +132 -3
  13. aioscrapy/core/downloader/handlers/requests.py +120 -2
  14. aioscrapy/core/downloader/handlers/webdriver/__init__.py +2 -0
  15. aioscrapy/core/downloader/handlers/webdriver/drissionpage.py +493 -0
  16. aioscrapy/core/downloader/handlers/webdriver/driverpool.py +234 -0
  17. aioscrapy/core/downloader/handlers/webdriver/playwright.py +498 -0
  18. aioscrapy/core/engine.py +381 -20
  19. aioscrapy/core/scheduler.py +350 -36
  20. aioscrapy/core/scraper.py +509 -33
  21. aioscrapy/crawler.py +392 -10
  22. aioscrapy/db/__init__.py +149 -0
  23. aioscrapy/db/absmanager.py +212 -6
  24. aioscrapy/db/aiomongo.py +292 -10
  25. aioscrapy/db/aiomysql.py +363 -10
  26. aioscrapy/db/aiopg.py +299 -2
  27. aioscrapy/db/aiorabbitmq.py +444 -4
  28. aioscrapy/db/aioredis.py +260 -11
  29. aioscrapy/dupefilters/__init__.py +110 -5
  30. aioscrapy/dupefilters/disk.py +124 -2
  31. aioscrapy/dupefilters/redis.py +598 -32
  32. aioscrapy/exceptions.py +151 -13
  33. aioscrapy/http/__init__.py +1 -1
  34. aioscrapy/http/headers.py +237 -3
  35. aioscrapy/http/request/__init__.py +257 -11
  36. aioscrapy/http/request/form.py +83 -3
  37. aioscrapy/http/request/json_request.py +121 -9
  38. aioscrapy/http/response/__init__.py +306 -33
  39. aioscrapy/http/response/html.py +42 -3
  40. aioscrapy/http/response/text.py +496 -49
  41. aioscrapy/http/response/web_driver.py +144 -0
  42. aioscrapy/http/response/xml.py +45 -3
  43. aioscrapy/libs/downloader/defaultheaders.py +66 -2
  44. aioscrapy/libs/downloader/downloadtimeout.py +91 -2
  45. aioscrapy/libs/downloader/ja3fingerprint.py +95 -2
  46. aioscrapy/libs/downloader/retry.py +192 -6
  47. aioscrapy/libs/downloader/stats.py +142 -0
  48. aioscrapy/libs/downloader/useragent.py +93 -2
  49. aioscrapy/libs/extensions/closespider.py +166 -4
  50. aioscrapy/libs/extensions/corestats.py +151 -1
  51. aioscrapy/libs/extensions/logstats.py +145 -1
  52. aioscrapy/libs/extensions/metric.py +370 -1
  53. aioscrapy/libs/extensions/throttle.py +235 -1
  54. aioscrapy/libs/pipelines/__init__.py +345 -2
  55. aioscrapy/libs/pipelines/csv.py +242 -0
  56. aioscrapy/libs/pipelines/excel.py +545 -0
  57. aioscrapy/libs/pipelines/mongo.py +132 -0
  58. aioscrapy/libs/pipelines/mysql.py +67 -0
  59. aioscrapy/libs/pipelines/pg.py +67 -0
  60. aioscrapy/libs/spider/depth.py +141 -3
  61. aioscrapy/libs/spider/httperror.py +144 -4
  62. aioscrapy/libs/spider/offsite.py +202 -2
  63. aioscrapy/libs/spider/referer.py +396 -21
  64. aioscrapy/libs/spider/urllength.py +97 -1
  65. aioscrapy/link.py +115 -8
  66. aioscrapy/logformatter.py +199 -8
  67. aioscrapy/middleware/absmanager.py +328 -2
  68. aioscrapy/middleware/downloader.py +218 -0
  69. aioscrapy/middleware/extension.py +50 -1
  70. aioscrapy/middleware/itempipeline.py +96 -0
  71. aioscrapy/middleware/spider.py +360 -7
  72. aioscrapy/process.py +200 -0
  73. aioscrapy/proxy/__init__.py +142 -3
  74. aioscrapy/proxy/redis.py +136 -2
  75. aioscrapy/queue/__init__.py +168 -16
  76. aioscrapy/scrapyd/runner.py +124 -3
  77. aioscrapy/serializer.py +182 -2
  78. aioscrapy/settings/__init__.py +610 -128
  79. aioscrapy/settings/default_settings.py +313 -13
  80. aioscrapy/signalmanager.py +151 -20
  81. aioscrapy/signals.py +183 -1
  82. aioscrapy/spiderloader.py +165 -12
  83. aioscrapy/spiders/__init__.py +233 -6
  84. aioscrapy/statscollectors.py +312 -1
  85. aioscrapy/utils/conf.py +345 -17
  86. aioscrapy/utils/curl.py +168 -16
  87. aioscrapy/utils/decorators.py +76 -6
  88. aioscrapy/utils/deprecate.py +212 -19
  89. aioscrapy/utils/httpobj.py +55 -3
  90. aioscrapy/utils/log.py +79 -0
  91. aioscrapy/utils/misc.py +189 -21
  92. aioscrapy/utils/ossignal.py +67 -5
  93. aioscrapy/utils/project.py +165 -3
  94. aioscrapy/utils/python.py +254 -44
  95. aioscrapy/utils/reqser.py +75 -1
  96. aioscrapy/utils/request.py +173 -12
  97. aioscrapy/utils/response.py +91 -6
  98. aioscrapy/utils/signal.py +196 -14
  99. aioscrapy/utils/spider.py +51 -4
  100. aioscrapy/utils/template.py +93 -6
  101. aioscrapy/utils/tools.py +191 -17
  102. aioscrapy/utils/trackref.py +198 -12
  103. aioscrapy/utils/url.py +341 -36
  104. aio_scrapy-2.1.4.dist-info/RECORD +0 -133
  105. aioscrapy/core/downloader/handlers/playwright/__init__.py +0 -115
  106. aioscrapy/core/downloader/handlers/playwright/driverpool.py +0 -59
  107. aioscrapy/core/downloader/handlers/playwright/webdriver.py +0 -96
  108. aioscrapy/http/response/playwright.py +0 -36
  109. aioscrapy/libs/pipelines/execl.py +0 -169
  110. {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.6.dist-info}/entry_points.txt +0 -0
  111. {aio_scrapy-2.1.4.dist-info → aio_scrapy-2.1.6.dist-info}/top_level.txt +0 -0
aioscrapy/utils/curl.py CHANGED
@@ -1,3 +1,14 @@
1
+ """
2
+ cURL command parsing utilities for aioscrapy.
3
+ aioscrapy的cURL命令解析实用工具。
4
+
5
+ This module provides utilities for converting cURL commands to aioscrapy Request objects.
6
+ It parses cURL command syntax and extracts relevant parameters like headers, cookies,
7
+ authentication, and request body.
8
+ 此模块提供了将cURL命令转换为aioscrapy Request对象的实用工具。
9
+ 它解析cURL命令语法并提取相关参数,如标头、cookie、身份验证和请求正文。
10
+ """
11
+
1
12
  import argparse
2
13
  import warnings
3
14
  from shlex import split
@@ -8,69 +19,189 @@ from w3lib.http import basic_auth_header
8
19
 
9
20
 
10
21
  class CurlParser(argparse.ArgumentParser):
22
+ """
23
+ Custom ArgumentParser for parsing cURL commands.
24
+ 用于解析cURL命令的自定义ArgumentParser。
25
+
26
+ This class extends the standard ArgumentParser to provide better error handling
27
+ for cURL command parsing. Instead of printing to stderr and exiting, it raises
28
+ a ValueError with a descriptive message when parsing fails.
29
+ 此类扩展了标准ArgumentParser,为cURL命令解析提供更好的错误处理。
30
+ 当解析失败时,它会引发带有描述性消息的ValueError,而不是打印到stderr并退出。
31
+ """
32
+
11
33
  def error(self, message):
34
+ """
35
+ Override the default error method to raise ValueError instead of exiting.
36
+ 覆盖默认的error方法,引发ValueError而不是退出。
37
+
38
+ Args:
39
+ message: The error message from the ArgumentParser.
40
+ 来自ArgumentParser的错误消息。
41
+
42
+ Raises:
43
+ ValueError: Always raised with a descriptive error message.
44
+ 始终引发带有描述性错误消息的ValueError。
45
+ """
12
46
  error_msg = f'There was an error parsing the curl command: {message}'
13
47
  raise ValueError(error_msg)
14
48
 
15
49
 
50
+ # Create a parser instance for cURL commands
51
+ # 创建用于cURL命令的解析器实例
16
52
  curl_parser = CurlParser()
17
- curl_parser.add_argument('url')
18
- curl_parser.add_argument('-H', '--header', dest='headers', action='append')
19
- curl_parser.add_argument('-X', '--request', dest='method')
20
- curl_parser.add_argument('-d', '--data', '--data-raw', dest='data')
21
- curl_parser.add_argument('-u', '--user', dest='auth')
53
+
54
+ # Add arguments for the main cURL options we want to support
55
+ # 添加我们想要支持的主要cURL选项的参数
56
+ curl_parser.add_argument('url') # The target URL (positional argument)
57
+ curl_parser.add_argument('-H', '--header', dest='headers', action='append') # HTTP headers
58
+ curl_parser.add_argument('-X', '--request', dest='method') # HTTP method (GET, POST, etc.)
59
+ curl_parser.add_argument('-d', '--data', '--data-raw', dest='data') # Request body data
60
+ curl_parser.add_argument('-u', '--user', dest='auth') # Basic authentication credentials
22
61
 
23
62
 
63
+ # List of cURL arguments that can be safely ignored
64
+ # These arguments don't affect the Request object we're building
65
+ # cURL参数列表,可以安全地忽略
66
+ # 这些参数不会影响我们正在构建的Request对象
24
67
  safe_to_ignore_arguments = [
25
68
  ['--compressed'],
26
69
  # `--compressed` argument is not safe to ignore, but it's included here
27
70
  # because the `HttpCompressionMiddleware` is enabled by default
28
- ['-s', '--silent'],
29
- ['-v', '--verbose'],
30
- ['-#', '--progress-bar']
71
+ # `--compressed`参数实际上不安全忽略,但它包含在这里
72
+ # 因为`HttpCompressionMiddleware`默认启用
73
+ ['-s', '--silent'], # Don't show progress meter or error messages
74
+ ['-v', '--verbose'], # Make the operation more talkative
75
+ ['-#', '--progress-bar'] # Display transfer progress as a progress bar
31
76
  ]
32
77
 
78
+ # Add all the safe-to-ignore arguments to the parser
79
+ # 将所有可以安全忽略的参数添加到解析器
33
80
  for argument in safe_to_ignore_arguments:
34
81
  curl_parser.add_argument(*argument, action='store_true')
35
82
 
36
83
 
37
84
  def _parse_headers_and_cookies(parsed_args):
85
+ """
86
+ Extract headers and cookies from parsed cURL arguments.
87
+ 从解析的cURL参数中提取标头和cookie。
88
+
89
+ This internal helper function processes the headers from cURL arguments,
90
+ separating regular headers from cookies. It also handles basic authentication
91
+ by converting it to an Authorization header.
92
+ 此内部辅助函数处理来自cURL参数的标头,将常规标头与cookie分开。
93
+ 它还通过将基本身份验证转换为Authorization标头来处理基本身份验证。
94
+
95
+ Args:
96
+ parsed_args: The parsed arguments from the cURL command.
97
+ 来自cURL命令的解析参数。
98
+
99
+ Returns:
100
+ tuple: A tuple containing:
101
+ 包含以下内容的元组:
102
+ - headers (list): List of (name, value) tuples for HTTP headers.
103
+ HTTP标头的(名称, 值)元组列表。
104
+ - cookies (dict): Dictionary of cookie names and values.
105
+ cookie名称和值的字典。
106
+ """
38
107
  headers = []
39
108
  cookies = {}
109
+
110
+ # Process each header from the cURL command
111
+ # 处理来自cURL命令的每个标头
40
112
  for header in parsed_args.headers or ():
113
+ # Split the header into name and value
114
+ # 将标头分成名称和值
41
115
  name, val = header.split(':', 1)
42
116
  name = name.strip()
43
117
  val = val.strip()
118
+
119
+ # Special handling for Cookie headers
120
+ # 对Cookie标头的特殊处理
44
121
  if name.title() == 'Cookie':
122
+ # Parse the cookie string and add each cookie to the cookies dict
123
+ # 解析cookie字符串并将每个cookie添加到cookies字典
45
124
  for name, morsel in SimpleCookie(val).items():
46
125
  cookies[name] = morsel.value
47
126
  else:
127
+ # Add regular headers to the headers list
128
+ # 将常规标头添加到标头列表
48
129
  headers.append((name, val))
49
130
 
131
+ # Handle basic authentication if provided
132
+ # 如果提供了基本身份验证,则处理它
50
133
  if parsed_args.auth:
134
+ # Split the auth string into username and password
135
+ # 将auth字符串分成用户名和密码
51
136
  user, password = parsed_args.auth.split(':', 1)
137
+ # Create and add the Authorization header
138
+ # 创建并添加Authorization标头
52
139
  headers.append(('Authorization', basic_auth_header(user, password)))
53
140
 
54
141
  return headers, cookies
55
142
 
56
143
 
57
144
  def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
58
- """Convert a cURL command syntax to Request kwargs.
59
-
60
- :param str curl_command: string containing the curl command
61
- :param bool ignore_unknown_options: If true, only a warning is emitted when
62
- cURL options are unknown. Otherwise
63
- raises an error. (default: True)
64
- :return: dictionary of Request kwargs
65
145
  """
146
+ Convert a cURL command to Request keyword arguments.
147
+ 将cURL命令转换为Request关键字参数。
148
+
149
+ This function parses a cURL command string and converts it to a dictionary
150
+ of keyword arguments that can be used to create an aioscrapy Request object.
151
+ It handles common cURL options like headers, cookies, method, data, and
152
+ authentication.
153
+ 此函数解析cURL命令字符串,并将其转换为可用于创建aioscrapy Request对象的
154
+ 关键字参数字典。它处理常见的cURL选项,如标头、cookie、方法、数据和身份验证。
155
+
156
+ Args:
157
+ curl_command: String containing the complete curl command.
158
+ 包含完整curl命令的字符串。
159
+ ignore_unknown_options: If True, only a warning is emitted when cURL options
160
+ are unknown. Otherwise raises an error.
161
+ 如果为True,则在cURL选项未知时只发出警告。
162
+ 否则引发错误。
163
+ Defaults to True.
164
+ 默认为True。
66
165
 
166
+ Returns:
167
+ dict: Dictionary of Request keyword arguments, which may include:
168
+ Request关键字参数的字典,可能包括:
169
+ - method: HTTP method (GET, POST, etc.)
170
+ HTTP方法(GET、POST等)
171
+ - url: The target URL
172
+ 目标URL
173
+ - headers: List of (name, value) header tuples
174
+ (名称, 值)标头元组的列表
175
+ - cookies: Dictionary of cookie names and values
176
+ cookie名称和值的字典
177
+ - body: Request body data
178
+ 请求正文数据
179
+
180
+ Raises:
181
+ ValueError: If the command doesn't start with 'curl', or if unknown options
182
+ are encountered and ignore_unknown_options is False.
183
+ 如果命令不以'curl'开头,或者如果遇到未知选项且
184
+ ignore_unknown_options为False。
185
+
186
+ Example:
187
+ >>> kwargs = curl_to_request_kwargs('curl -X POST "http://example.com" -H "Content-Type: application/json" -d \'{"key": "value"}\'')
188
+ >>> request = Request(**kwargs)
189
+ """
190
+ # Split the command string into arguments
191
+ # 将命令字符串分割成参数
67
192
  curl_args = split(curl_command)
68
193
 
194
+ # Verify that the command starts with 'curl'
195
+ # 验证命令以'curl'开头
69
196
  if curl_args[0] != 'curl':
70
197
  raise ValueError('A curl command must start with "curl"')
71
198
 
199
+ # Parse the arguments, separating known from unknown
200
+ # 解析参数,将已知参数与未知参数分开
72
201
  parsed_args, argv = curl_parser.parse_known_args(curl_args[1:])
73
202
 
203
+ # Handle unknown arguments
204
+ # 处理未知参数
74
205
  if argv:
75
206
  msg = f'Unrecognized options: {", ".join(argv)}'
76
207
  if ignore_unknown_options:
@@ -78,26 +209,47 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
78
209
  else:
79
210
  raise ValueError(msg)
80
211
 
212
+ # Get the URL from the parsed arguments
213
+ # 从解析的参数中获取URL
81
214
  url = parsed_args.url
82
215
 
83
216
  # curl automatically prepends 'http' if the scheme is missing, but Request
84
- # needs the scheme to work
217
+ # needs the scheme to work, so we add it if necessary
218
+ # curl在方案缺失时自动添加'http',但Request需要方案才能工作,
219
+ # 因此我们在必要时添加它
85
220
  parsed_url = urlparse(url)
86
221
  if not parsed_url.scheme:
87
222
  url = 'http://' + url
88
223
 
224
+ # Get the HTTP method or default to GET
225
+ # 获取HTTP方法或默认为GET
89
226
  method = parsed_args.method or 'GET'
90
227
 
228
+ # Start building the result dictionary
229
+ # 开始构建结果字典
91
230
  result = {'method': method.upper(), 'url': url}
92
231
 
232
+ # Extract headers and cookies from the parsed arguments
233
+ # 从解析的参数中提取标头和cookie
93
234
  headers, cookies = _parse_headers_and_cookies(parsed_args)
94
235
 
236
+ # Add headers to the result if present
237
+ # 如果存在,将标头添加到结果
95
238
  if headers:
96
239
  result['headers'] = headers
240
+
241
+ # Add cookies to the result if present
242
+ # 如果存在,将cookie添加到结果
97
243
  if cookies:
98
244
  result['cookies'] = cookies
245
+
246
+ # Handle request body data
247
+ # 处理请求正文数据
99
248
  if parsed_args.data:
100
249
  result['body'] = parsed_args.data
250
+
251
+ # If data is provided but method is not specified, default to POST
252
+ # 如果提供了数据但未指定方法,则默认为POST
101
253
  if not parsed_args.method:
102
254
  # if the "data" is specified but the "method" is not specified,
103
255
  # the default method is 'POST'
@@ -1,3 +1,13 @@
1
+ """
2
+ Decorator utilities for aioscrapy.
3
+ aioscrapy的装饰器实用工具。
4
+
5
+ This module provides utility decorators for use in aioscrapy.
6
+ It includes decorators for marking functions and methods as deprecated.
7
+ 此模块提供了在aioscrapy中使用的实用装饰器。
8
+ 它包括用于将函数和方法标记为已弃用的装饰器。
9
+ """
10
+
1
11
  import warnings
2
12
  from functools import wraps
3
13
 
@@ -5,21 +15,81 @@ from aioscrapy.exceptions import AioScrapyDeprecationWarning
5
15
 
6
16
 
7
17
  def deprecated(use_instead=None):
8
- """This is a decorator which can be used to mark functions
9
- as deprecated. It will result in a warning being emitted
10
- when the function is used."""
18
+ """
19
+ Decorator to mark functions or methods as deprecated.
20
+ 用于将函数或方法标记为已弃用的装饰器。
21
+
22
+ This decorator can be used to mark functions or methods as deprecated.
23
+ It will emit a warning when the decorated function is called, informing
24
+ users that the function is deprecated and suggesting an alternative if provided.
25
+ 此装饰器可用于将函数或方法标记为已弃用。
26
+ 当调用被装饰的函数时,它将发出警告,通知用户该函数已弃用,
27
+ 并在提供替代方案时建议使用替代方案。
28
+
29
+ The decorator can be used in two ways:
30
+ 装饰器可以通过两种方式使用:
31
+
32
+ 1. With no arguments: @deprecated
33
+ 2. With an argument specifying the alternative: @deprecated("new_function")
34
+
35
+ Args:
36
+ use_instead: Optional string indicating the alternative function or method to use.
37
+ 可选的字符串,指示要使用的替代函数或方法。
38
+ Defaults to None.
39
+ 默认为None。
11
40
 
41
+ Returns:
42
+ callable: A decorated function that will emit a deprecation warning when called.
43
+ 一个装饰后的函数,在调用时会发出弃用警告。
44
+
45
+ Example:
46
+ >>> @deprecated
47
+ ... def old_function():
48
+ ... return "result"
49
+ ...
50
+ >>> @deprecated("new_improved_function")
51
+ ... def another_old_function():
52
+ ... return "result"
53
+ ...
54
+ >>> # When called, these will emit warnings:
55
+ >>> old_function() # Warning: "Call to deprecated function old_function."
56
+ >>> another_old_function() # Warning: "Call to deprecated function another_old_function. Use new_improved_function instead."
57
+ """
58
+ # Handle case where decorator is used without arguments: @deprecated
59
+ # 处理装饰器不带参数使用的情况:@deprecated
60
+ if callable(use_instead):
61
+ # In this case, use_instead is actually the function being decorated
62
+ # 在这种情况下,use_instead实际上是被装饰的函数
63
+ func = use_instead
64
+ use_instead = None
65
+
66
+ # Apply the decorator directly and return the wrapped function
67
+ # 直接应用装饰器并返回包装后的函数
68
+ @wraps(func)
69
+ def wrapped(*args, **kwargs):
70
+ message = f"Call to deprecated function {func.__name__}."
71
+ warnings.warn(message, category=AioScrapyDeprecationWarning, stacklevel=2)
72
+ return func(*args, **kwargs)
73
+ return wrapped
74
+
75
+ # Handle case where decorator is used with an argument: @deprecated("new_function")
76
+ # 处理装饰器带参数使用的情况:@deprecated("new_function")
12
77
  def deco(func):
13
78
  @wraps(func)
14
79
  def wrapped(*args, **kwargs):
80
+ # Create the warning message
81
+ # 创建警告消息
15
82
  message = f"Call to deprecated function {func.__name__}."
16
83
  if use_instead:
17
84
  message += f" Use {use_instead} instead."
85
+
86
+ # Emit the deprecation warning
87
+ # 发出弃用警告
18
88
  warnings.warn(message, category=AioScrapyDeprecationWarning, stacklevel=2)
89
+
90
+ # Call the original function
91
+ # 调用原始函数
19
92
  return func(*args, **kwargs)
20
93
  return wrapped
21
94
 
22
- if callable(use_instead):
23
- deco = deco(use_instead)
24
- use_instead = None
25
95
  return deco
@@ -1,4 +1,13 @@
1
- """Some helpers for deprecation messages"""
1
+ """
2
+ Deprecation utilities for aioscrapy.
3
+ aioscrapy的弃用实用工具。
4
+
5
+ This module provides utility functions and classes for handling deprecation in aioscrapy.
6
+ It includes tools for creating deprecation warnings, creating deprecated classes,
7
+ and updating class paths to their new locations.
8
+ 此模块提供了用于处理aioscrapy中弃用的实用函数和类。
9
+ 它包括用于创建弃用警告、创建已弃用类以及将类路径更新到其新位置的工具。
10
+ """
2
11
 
3
12
  import inspect
4
13
  import warnings
@@ -7,12 +16,51 @@ from aioscrapy.exceptions import AioScrapyDeprecationWarning
7
16
 
8
17
 
9
18
  def attribute(obj, oldattr, newattr, version='0.12'):
19
+ """
20
+ Issue a deprecation warning for an attribute that has been renamed.
21
+ 为已重命名的属性发出弃用警告。
22
+
23
+ This function issues a deprecation warning when code accesses an old attribute
24
+ that has been renamed. It helps guide users to update their code to use the
25
+ new attribute name.
26
+ 当代码访问已重命名的旧属性时,此函数会发出弃用警告。
27
+ 它有助于指导用户更新其代码以使用新的属性名称。
28
+
29
+ Args:
30
+ obj: The object containing the deprecated attribute.
31
+ 包含已弃用属性的对象。
32
+ oldattr: The name of the deprecated attribute.
33
+ 已弃用属性的名称。
34
+ newattr: The name of the attribute that should be used instead.
35
+ 应该使用的属性的名称。
36
+ version: The version of aioscrapy in which the old attribute will be removed.
37
+ 将删除旧属性的aioscrapy版本。
38
+ Defaults to '0.12'.
39
+ 默认为'0.12'。
40
+
41
+ Example:
42
+ >>> class MyClass:
43
+ ... def __getattr__(self, name):
44
+ ... if name == 'old_name':
45
+ ... attribute(self, 'old_name', 'new_name')
46
+ ... return self.new_name
47
+ ... raise AttributeError(f"{self.__class__.__name__} has no attribute {name}")
48
+ ...
49
+ ... @property
50
+ ... def new_name(self):
51
+ ... return "value"
52
+ """
53
+ # Get the class name of the object
54
+ # 获取对象的类名
10
55
  cname = obj.__class__.__name__
56
+
57
+ # Issue a deprecation warning with information about the old and new attributes
58
+ # 发出包含有关旧属性和新属性信息的弃用警告
11
59
  warnings.warn(
12
60
  f"{cname}.{oldattr} attribute is deprecated and will be no longer supported "
13
61
  f"in Aioscrapy {version}, use {cname}.{newattr} attribute instead",
14
62
  AioScrapyDeprecationWarning,
15
- stacklevel=3)
63
+ stacklevel=3) # Use stacklevel=3 to point to the caller's caller
16
64
 
17
65
 
18
66
  def create_deprecated_class(
@@ -27,28 +75,91 @@ def create_deprecated_class(
27
75
  instance_warn_message="{cls} is deprecated, instantiate {new} instead."
28
76
  ):
29
77
  """
30
- Return a "deprecated" class that causes its subclasses to issue a warning.
31
- Subclasses of ``new_class`` are considered subclasses of this class.
32
- It also warns when the deprecated class is instantiated, but do not when
33
- its subclasses are instantiated.
78
+ Create a deprecated class that redirects to a new class.
79
+ 创建一个重定向到新类的已弃用类。
34
80
 
35
- It can be used to rename a base class in a library. For example, if we
36
- have
81
+ This function creates a "deprecated" class that issues warnings when:
82
+ 1. A user subclasses from the deprecated class
83
+ 2. A user instantiates the deprecated class directly
37
84
 
38
- class OldName(SomeClass):
39
- # ...
85
+ 此函数创建一个"已弃用"类,在以下情况下发出警告:
86
+ 1. 用户从已弃用的类继承
87
+ 2. 用户直接实例化已弃用的类
40
88
 
41
- and we want to rename it to NewName, we can do the following::
89
+ The deprecated class acts as a proxy to the new class, so:
90
+ - Subclasses of the new class are considered subclasses of the deprecated class
91
+ - isinstance() and issubclass() checks work as expected
92
+ - The deprecated class can be instantiated and will create instances of the new class
42
93
 
43
- class NewName(SomeClass):
44
- # ...
94
+ 已弃用的类充当新类的代理,因此:
95
+ - 新类的子类被视为已弃用类的子类
96
+ - isinstance()和issubclass()检查按预期工作
97
+ - 可以实例化已弃用的类,它将创建新类的实例
45
98
 
46
- OldName = create_deprecated_class('OldName', NewName)
99
+ This is particularly useful when renaming or moving classes in a library
100
+ while maintaining backward compatibility.
47
101
 
48
- Then, if user class inherits from OldName, warning is issued. Also, if
49
- some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``
50
- checks they'll still return True if sub is a subclass of NewName instead of
51
- OldName.
102
+ 当在保持向后兼容性的同时重命名或移动库中的类时,这特别有用。
103
+
104
+ Args:
105
+ name: Name of the deprecated class.
106
+ 已弃用类的名称。
107
+ new_class: The class that should be used instead.
108
+ 应该使用的类。
109
+ clsdict: Additional attributes to add to the deprecated class.
110
+ 要添加到已弃用类的其他属性。
111
+ Defaults to None.
112
+ 默认为None。
113
+ warn_category: Warning category to use.
114
+ 要使用的警告类别。
115
+ Defaults to AioScrapyDeprecationWarning.
116
+ 默认为AioScrapyDeprecationWarning。
117
+ warn_once: Whether to warn only once when a subclass inherits from the deprecated class.
118
+ 当子类从已弃用的类继承时是否只警告一次。
119
+ Defaults to True.
120
+ 默认为True。
121
+ old_class_path: Full import path of the old class (for better warning messages).
122
+ 旧类的完整导入路径(用于更好的警告消息)。
123
+ Defaults to None.
124
+ 默认为None。
125
+ new_class_path: Full import path of the new class (for better warning messages).
126
+ 新类的完整导入路径(用于更好的警告消息)。
127
+ Defaults to None.
128
+ 默认为None。
129
+ subclass_warn_message: Warning message template when a class inherits from the deprecated class.
130
+ 当类从已弃用的类继承时的警告消息模板。
131
+ Defaults to "{cls} inherits from deprecated class {old}, please inherit from {new}.".
132
+ 默认为"{cls} inherits from deprecated class {old}, please inherit from {new}."。
133
+ instance_warn_message: Warning message template when the deprecated class is instantiated.
134
+ 当实例化已弃用的类时的警告消息模板。
135
+ Defaults to "{cls} is deprecated, instantiate {new} instead.".
136
+ 默认为"{cls} is deprecated, instantiate {new} instead."。
137
+
138
+ Returns:
139
+ type: A deprecated class that proxies to the new class.
140
+ 代理到新类的已弃用类。
141
+
142
+ Example:
143
+ >>> class OldName(SomeClass):
144
+ ... # ...
145
+ ...
146
+ >>> class NewName(SomeClass):
147
+ ... # ...
148
+ ...
149
+ >>> OldName = create_deprecated_class('OldName', NewName)
150
+ >>>
151
+ >>> # The following will issue a warning but still work:
152
+ >>> class UserClass(OldName):
153
+ ... pass
154
+ ...
155
+ >>> # This will also issue a warning but create a NewName instance:
156
+ >>> instance = OldName()
157
+ >>>
158
+ >>> # These will return True:
159
+ >>> issubclass(UserClass, OldName)
160
+ >>> issubclass(UserClass, NewName)
161
+ >>> isinstance(instance, OldName)
162
+ >>> isinstance(instance, NewName)
52
163
  """
53
164
 
54
165
  class DeprecatedClass(new_class.__class__):
@@ -122,20 +233,102 @@ def create_deprecated_class(
122
233
 
123
234
 
124
235
  def _clspath(cls, forced=None):
236
+ """
237
+ Get the full import path of a class.
238
+ 获取类的完整导入路径。
239
+
240
+ This internal helper function returns the full import path of a class
241
+ (module name + class name). It's used to generate informative deprecation
242
+ warning messages.
243
+ 这个内部辅助函数返回类的完整导入路径(模块名+类名)。
244
+ 它用于生成信息丰富的弃用警告消息。
245
+
246
+ Args:
247
+ cls: The class to get the path for.
248
+ 要获取路径的类。
249
+ forced: An optional string to use instead of the actual class path.
250
+ 可选的字符串,用于替代实际的类路径。
251
+ Defaults to None.
252
+ 默认为None。
253
+
254
+ Returns:
255
+ str: The full import path of the class (e.g., 'package.module.ClassName')
256
+ or the forced path if provided.
257
+ 类的完整导入路径(例如,'package.module.ClassName')
258
+ 或者如果提供了forced参数,则返回forced路径。
259
+ """
260
+ # If a forced path is provided, use it instead of calculating the path
261
+ # 如果提供了强制路径,则使用它而不是计算路径
125
262
  if forced is not None:
126
263
  return forced
264
+
265
+ # Otherwise, construct the path from the module and class name
266
+ # 否则,从模块和类名构造路径
127
267
  return f'{cls.__module__}.{cls.__name__}'
128
268
 
129
269
 
270
+ # List of (prefix, replacement) tuples for updating deprecated class paths
271
+ # Each rule specifies a prefix to match and its replacement for updating import paths
272
+ # 用于更新已弃用类路径的(前缀, 替换)元组列表
273
+ # 每个规则指定要匹配的前缀及其替换,用于更新导入路径
130
274
  DEPRECATION_RULES = []
131
275
 
132
276
 
133
277
  def update_classpath(path):
134
- """Update a deprecated path from an object with its new location"""
278
+ """
279
+ Update a deprecated class path to its new location.
280
+ 将已弃用的类路径更新到其新位置。
281
+
282
+ This function checks if a given import path matches any of the known
283
+ deprecated paths (defined in DEPRECATION_RULES) and returns the updated
284
+ path if a match is found. It also issues a deprecation warning.
285
+ 此函数检查给定的导入路径是否匹配任何已知的已弃用路径
286
+ (在DEPRECATION_RULES中定义),如果找到匹配项,则返回更新的路径。
287
+ 它还会发出弃用警告。
288
+
289
+ This is useful for handling cases where classes have been moved to
290
+ different modules or packages.
291
+ 这对于处理类已移动到不同模块或包的情况很有用。
292
+
293
+ Args:
294
+ path: The import path to check and potentially update.
295
+ 要检查和可能更新的导入路径。
296
+
297
+ Returns:
298
+ str: The updated path if the input path matches a deprecated path,
299
+ otherwise the original path.
300
+ 如果输入路径匹配已弃用路径,则返回更新的路径,
301
+ 否则返回原始路径。
302
+
303
+ Example:
304
+ >>> # Add a deprecation rule
305
+ >>> DEPRECATION_RULES.append(('old.module', 'new.module'))
306
+ >>>
307
+ >>> # This will return 'new.module.MyClass' and issue a warning
308
+ >>> update_classpath('old.module.MyClass')
309
+ 'new.module.MyClass'
310
+ >>>
311
+ >>> # This will return the original path unchanged
312
+ >>> update_classpath('other.module.MyClass')
313
+ 'other.module.MyClass'
314
+ """
315
+ # Check each deprecation rule to see if the path matches
316
+ # 检查每个弃用规则,看路径是否匹配
135
317
  for prefix, replacement in DEPRECATION_RULES:
318
+ # Only process string paths that start with the deprecated prefix
319
+ # 只处理以已弃用前缀开头的字符串路径
136
320
  if isinstance(path, str) and path.startswith(prefix):
321
+ # Replace the deprecated prefix with its replacement (only the first occurrence)
322
+ # 将已弃用的前缀替换为其替换(仅第一次出现)
137
323
  new_path = path.replace(prefix, replacement, 1)
324
+
325
+ # Issue a deprecation warning
326
+ # 发出弃用警告
138
327
  warnings.warn(f"`{path}` class is deprecated, use `{new_path}` instead",
139
328
  AioScrapyDeprecationWarning)
329
+
140
330
  return new_path
331
+
332
+ # If no rules matched, return the original path unchanged
333
+ # 如果没有规则匹配,则原样返回原始路径
141
334
  return path