aio-scrapy 2.1.3__py3-none-any.whl → 2.1.4__py3-none-any.whl

{aio_scrapy-2.1.3.dist-info → aio_scrapy-2.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.1.3
+Version: 2.1.4
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -28,6 +28,7 @@ Requires-Dist: zope.interface >=5.1.0
 Requires-Dist: redis >=4.3.1
 Requires-Dist: aiomultiprocess >=0.9.0
 Requires-Dist: loguru >=0.7.0
+Requires-Dist: anyio >=3.6.2
 Provides-Extra: aio-pika
 Requires-Dist: aio-pika >=8.1.1 ; extra == 'aio-pika'
 Provides-Extra: aiomysql

{aio_scrapy-2.1.3.dist-info → aio_scrapy-2.1.4.dist-info}/RECORD

@@ -1,9 +1,9 @@
-aioscrapy/VERSION,sha256=BuGd6tadzBa8VzzJ8ktotpkH9M_Ur7nXA3LMa0YqmGI,5
+aioscrapy/VERSION,sha256=Z7BD32ByWBAJNuaSjQBe7W_NFoIm-41YzXKXt3z-bUI,5
 aioscrapy/__init__.py,sha256=esJeH66Mz9WV7XbotvZEjNn49jc589YZ_L2DKoD0JvA,858
 aioscrapy/__main__.py,sha256=rvTdJ0cQwbi29aucPj3jJRpccx5SBzvRcV7qvxvX2NQ,80
 aioscrapy/cmdline.py,sha256=1qhNg2Edl-Obmf2re2K4V8pJG7ubGfZZCzcHdKtdE_s,5159
 aioscrapy/crawler.py,sha256=6-ptivIjIGKdojOlZqXV0hV3x1Gont81tOC5u5JqIME,10330
-aioscrapy/exceptions.py,sha256=k1daw1hV_aqsaIKKibdyqcNPyVn5oUb07wmB2DRxfjs,2111
+aioscrapy/exceptions.py,sha256=B1UZUXF_dZNJ5b1wltDemijK8iCNpH-EF2sOooH9AsA,2628
 aioscrapy/link.py,sha256=fXMqsHvYEzsuYi-sNDcElS7jV6Lusq0tjPkPUGOlyZw,1867
 aioscrapy/logformatter.py,sha256=y3etd28ACbpTbcGprJ_cQ086gxQY3k_QX_yxYFoF1AU,3028
 aioscrapy/process.py,sha256=uFkj2wzaBu0Vs3pGFKdJ4R-0Gn7hROX6EU-B5zddnyQ,1603
@@ -24,15 +24,15 @@ aioscrapy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 aioscrapy/core/engine.py,sha256=h02-K2lQqlCxvNIlURgPpnhHCbyiJRIWrFJt5Ys7vZY,9843
 aioscrapy/core/scheduler.py,sha256=czCx5oHknXuHadpISTfoEMSKXXrlwJTmLTUQtHdtaTc,7407
 aioscrapy/core/scraper.py,sha256=eS_qEX_Q9fXZnK8Ou1wDtBJhRKk9JoUSnbn4c04u1cA,10750
-aioscrapy/core/downloader/__init__.py,sha256=2EUQHGS6Q8fy9eoDT_kmA-eYZEUBV9OkmTppfG05AFA,10039
-aioscrapy/core/downloader/handlers/__init__.py,sha256=CriaX2Cp4jUqzDDGZDB7HiIEgUWt2pnYVho6HMV6sJ0,3198
-aioscrapy/core/downloader/handlers/aiohttp.py,sha256=KY04ATlu2cTVF3Uxtvqpx1cGxOn2QVIGWoj7S3_UwF4,4266
-aioscrapy/core/downloader/handlers/curl_cffi.py,sha256=LpwWdHxctDOxVvhzYcgG1qGhPtw9DoqjUKEdrZlerAQ,2579
-aioscrapy/core/downloader/handlers/httpx.py,sha256=4XyLpiaXbO0AtPqctFJgYFX-5rJrKyf469YFpVJdcRY,3387
-aioscrapy/core/downloader/handlers/pyhttpx.py,sha256=phpWXtuvP-9tve3MDnWeto2Dmo25UTxgLYc12QEvMt4,2562
-aioscrapy/core/downloader/handlers/requests.py,sha256=UnV1WDyET8WMwyYYU0DcL_r420uz_0dK1ej5xAl2fwk,2317
-aioscrapy/core/downloader/handlers/playwright/__init__.py,sha256=PXS40Vv3KsV77QoyWCrWcHL6mItXxpTzwaCATB6RXiQ,4504
-aioscrapy/core/downloader/handlers/playwright/driverpool.py,sha256=qfIdGjORdn1MookO-ucIJ8NOeLrIQ0y0UJY_xuMzM_8,1374
+aioscrapy/core/downloader/__init__.py,sha256=OCg21payZbmQPcZ1_Wrhhgos7angRB-w9qya3CxrmSU,10040
+aioscrapy/core/downloader/handlers/__init__.py,sha256=KwID2qt3dhFvvBIF3CJnPR4w4a4_qz4uKaXgQI5b59o,3199
+aioscrapy/core/downloader/handlers/aiohttp.py,sha256=qt8Wys8NrbLatBqEob5lzjKmy_C2Nl9XxLyA2npdv6A,4277
+aioscrapy/core/downloader/handlers/curl_cffi.py,sha256=hYlUf2BzS6GrWaPKLJhuqj8fxOt9AANBoeAp9vx7-KU,2590
+aioscrapy/core/downloader/handlers/httpx.py,sha256=HqidohwQr8G7GNhrS1v23rYmD2dzNW69bObcO0X_6Qs,3398
+aioscrapy/core/downloader/handlers/pyhttpx.py,sha256=djxaNoYVD6TJSN3UruviQBx8_oLVtCn4d__qwsoxRJA,2573
+aioscrapy/core/downloader/handlers/requests.py,sha256=RdRi6Izj-jvWa_8T8axW9EzcUfMqfman7eFKTFjOro4,2328
+aioscrapy/core/downloader/handlers/playwright/__init__.py,sha256=xjPNlvM0zzR8lOIzgJeDnq1p0x1VHGhGiyMQmihdkmM,4676
+aioscrapy/core/downloader/handlers/playwright/driverpool.py,sha256=IlkYB8TlSuDq7-sTLlGvtAsFMalNvzpTJR7wEMYe2jE,1595
 aioscrapy/core/downloader/handlers/playwright/webdriver.py,sha256=QFtAT--2Ea_Gg4x1EhMidyOwQjbqljUl4sKGB_hAA00,3530
 aioscrapy/db/__init__.py,sha256=ISBXM_-cCf5CgTLc3i_emLxV163-ZAbgttkQiRxokD0,2456
 aioscrapy/db/absmanager.py,sha256=6vlPcjDHOtZCHePiUYPe6ezRnM-TB4XLhmuw7APaWDk,1162
@@ -59,7 +59,7 @@ aioscrapy/libs/downloader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 aioscrapy/libs/downloader/defaultheaders.py,sha256=tg_ULA0Y-41bZKG607mowFJQGVfnZ45LdR044DsjA_A,563
 aioscrapy/libs/downloader/downloadtimeout.py,sha256=hNh3OEj7rC0ceQrv_yrhR5lb5AvfxJ6cspj3qsQWj4o,704
 aioscrapy/libs/downloader/ja3fingerprint.py,sha256=DgTw74GXC_Bp94eD_bwoG6A_DphUHTt7bH4glBNXyV8,1058
-aioscrapy/libs/downloader/retry.py,sha256=0670bPz5lc4wUsWmYlhYdGZdeflsQdFhJbnwK1g0c84,4441
+aioscrapy/libs/downloader/retry.py,sha256=uKU8XuPya8Co6vTTTgs1-rFtMsZreSwz0Zo1ErgaA6I,4482
 aioscrapy/libs/downloader/stats.py,sha256=FlkS8Zm4j3SBjHb6caXwq08HvvZ37VKORGCAjlA2U38,1376
 aioscrapy/libs/downloader/useragent.py,sha256=E5x5dk9AxsSCGDDICJlTXwWXRkqAibWgesqG0VhAG8M,743
 aioscrapy/libs/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -125,9 +125,9 @@ aioscrapy/utils/template.py,sha256=HR97X4lpv2WuqhuPfzTgaBN66fYnzHVpP6zQ5IoTwcI,8
 aioscrapy/utils/tools.py,sha256=WJowViZB8XEs2CFqjVvbqXK3H5Uvf4BgWgBD_RcHMaM,2319
 aioscrapy/utils/trackref.py,sha256=0nIpelT1d5WYxALl8SGA8vHNYsh-jS0Z2lwVEAhwx8E,2019
 aioscrapy/utils/url.py,sha256=8W8tAhU7lgfPOfzKp3ejJGEcLj1i_PnA_53Jv5LpxiY,5464
-aio_scrapy-2.1.3.dist-info/LICENSE,sha256=L-UoAEM3fQSjKA7FVWxQM7gwSCbeue6gZRAnpRS_UCo,1088
-aio_scrapy-2.1.3.dist-info/METADATA,sha256=ldFQ8PbukunEAMxkUNbPQsA2Iuo6VfyYwPQbjc7KETk,6506
-aio_scrapy-2.1.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-aio_scrapy-2.1.3.dist-info/entry_points.txt,sha256=WWhoVHZvqhW8a5uFg97K0EP_GjG3uuCIFLkyqDICgaw,56
-aio_scrapy-2.1.3.dist-info/top_level.txt,sha256=8l08KyMt22wfX_5BmhrGH0PgwZdzZIPq-hBUa1GNir4,10
-aio_scrapy-2.1.3.dist-info/RECORD,,
+aio_scrapy-2.1.4.dist-info/LICENSE,sha256=L-UoAEM3fQSjKA7FVWxQM7gwSCbeue6gZRAnpRS_UCo,1088
+aio_scrapy-2.1.4.dist-info/METADATA,sha256=9R1Kw1XYe7yrLJ3h4SeiV69tPphz8sTaIf2Sizfh0GU,6536
+aio_scrapy-2.1.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+aio_scrapy-2.1.4.dist-info/entry_points.txt,sha256=WWhoVHZvqhW8a5uFg97K0EP_GjG3uuCIFLkyqDICgaw,56
+aio_scrapy-2.1.4.dist-info/top_level.txt,sha256=8l08KyMt22wfX_5BmhrGH0PgwZdzZIPq-hBUa1GNir4,10
+aio_scrapy-2.1.4.dist-info/RECORD,,

aioscrapy/VERSION

@@ -1 +1 @@
-2.1.3
+2.1.4

aioscrapy/core/downloader/__init__.py

@@ -143,7 +143,7 @@ class Downloader(BaseDownloader):
         crawler.spider.dupefilter = df  # 将指纹绑定到Spider 在解析成功的时候 调用DUPEFILTER_CLASS的success方法
         return cls(
             crawler,
-            await call_helper(DownloadHandlerManager.for_crawler, crawler),
+            await call_helper(DownloadHandlerManager.from_crawler, crawler),
             await call_helper(DownloaderMiddlewareManager.from_crawler, crawler),
             proxy=crawler.settings.get("PROXY_HANDLER") and await load_instance(crawler.settings["PROXY_HANDLER"],
                                                                                 crawler=crawler),

aioscrapy/core/downloader/handlers/__init__.py

@@ -37,7 +37,7 @@ class DownloadHandlerManager:
         crawler.signals.connect(self._close, signals.engine_stopped)
 
     @classmethod
-    def for_crawler(cls, crawler) -> "DownloadHandlerManager":
+    def from_crawler(cls, crawler) -> "DownloadHandlerManager":
         return cls(crawler)
 
     async def _get_handler(self, scheme: str) -> Optional[BaseDownloadHandler]:

aioscrapy/core/downloader/handlers/aiohttp.py

@@ -37,7 +37,7 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except ClientError as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

aioscrapy/core/downloader/handlers/curl_cffi.py

@@ -24,7 +24,7 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except CurlError as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

aioscrapy/core/downloader/handlers/httpx.py

@@ -32,7 +32,7 @@ class HttpxDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except HttpxError as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

aioscrapy/core/downloader/handlers/playwright/__init__.py

@@ -1,6 +1,10 @@
 from functools import wraps
 
-from playwright._impl._api_types import Error
+try:
+    from playwright._impl._errors import Error
+except ImportError:
+    from playwright._impl._api_types import Error
+
 from playwright.async_api._generated import Response as EventResponse
 
 from aioscrapy import Request, Spider
@@ -17,10 +21,11 @@ class PlaywrightHandler(BaseDownloadHandler):
     def __init__(self, settings: Settings):
         self.settings = settings
         playwright_client_args = settings.getdict('PLAYWRIGHT_CLIENT_ARGS')
+        use_pool = settings.getbool('PLAYWRIGHT_USE_POOL', True)
         self.wait_until = playwright_client_args.get('wait_until', 'domcontentloaded')
         self.url_regexes = playwright_client_args.pop('url_regexes', [])
         pool_size = playwright_client_args.pop('pool_size', settings.getint("CONCURRENT_REQUESTS", 1))
-        self._webdriver_pool = WebDriverPool(pool_size=pool_size, driver_cls=PlaywrightDriver, **playwright_client_args)
+        self._webdriver_pool = WebDriverPool(use_pool=use_pool, pool_size=pool_size, driver_cls=PlaywrightDriver, **playwright_client_args)
 
     @classmethod
     def from_settings(cls, settings: Settings):
@@ -30,7 +35,7 @@ class PlaywrightHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request, spider)
         except Error as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request, spider) -> PlaywrightResponse:
         cookies = dict(request.cookies)

aioscrapy/core/downloader/handlers/playwright/driverpool.py

@@ -9,8 +9,9 @@ from aioscrapy.utils.tools import singleton
 @singleton
 class WebDriverPool:
     def __init__(
-            self, pool_size=5, driver_cls=None, **kwargs
+            self, use_pool=True, pool_size=5, driver_cls=None, **kwargs
     ):
+        self.use_pool = use_pool
         self.pool_size = pool_size
         self.driver_cls = driver_cls
         self.kwargs = kwargs
@@ -32,6 +33,8 @@ class WebDriverPool:
 
     async def get(self, **kwargs):
         async with self.lock:
+            if not self.use_pool:
+                return await self.create_driver(**kwargs)
             if not self.is_full:
                 driver = await self.create_driver(**kwargs)
                 self.driver_count += 1
@@ -40,6 +43,9 @@ class WebDriverPool:
         return driver
 
     async def release(self, driver):
+        if not self.use_pool:
+            await driver.quit()
+            return
         await self.queue.put(driver)
 
     async def remove(self, driver):

aioscrapy/core/downloader/handlers/pyhttpx.py

@@ -27,7 +27,7 @@ class PyhttpxDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except PyHttpxError as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

aioscrapy/core/downloader/handlers/requests.py

@@ -25,7 +25,7 @@ class RequestsDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except RequestsError as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

aioscrapy/exceptions.py

@@ -5,6 +5,7 @@ These exceptions are documented in docs/topics/exceptions.rst. Please don't add
 new exceptions here without documenting them there.
 """
 
+
 # Internal
 
 
@@ -95,4 +96,21 @@ class ProxyException(Exception):
 
 class DownloadError(Exception):
     """下载页面时发生的错误"""
-    pass
+
+    def __init__(self, *args, real_error=None):
+        self.real_error = real_error
+        super().__init__(*args)
+
+    def __str__(self):
+        if not self.real_error:
+            return "DownloadError"
+
+        return f"{self.real_error.__class__.__module__}.{self.real_error.__class__.__name__}: {str(self.real_error)}"
+
+
+if __name__ == '__main__':
+    e = Exception("xxx")
+    reason = DownloadError(real_error=e)
+    print(reason)
+    obj = reason.real_error.__class__
+    print(f"{obj.__module__}.{obj.__name__}")

aioscrapy/libs/downloader/retry.py

@@ -51,7 +51,7 @@ def get_retry_request(
         if callable(reason):
             reason = reason()
         if isinstance(reason, Exception):
-            reason = global_object_name(reason.__class__)
+            reason = global_object_name((getattr(reason, "real_error", None) or reason).__class__)
 
         logger.info(
             "Retrying %(request)s (failed %(retry_times)d times): %(reason)s" % {