aio-scrapy 2.1.3__tar.gz → 2.1.4__tar.gz

{aio-scrapy-2.1.3/aio_scrapy.egg-info → aio-scrapy-2.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.1.3
+Version: 2.1.4
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -28,6 +28,7 @@ Requires-Dist: zope.interface>=5.1.0
 Requires-Dist: redis>=4.3.1
 Requires-Dist: aiomultiprocess>=0.9.0
 Requires-Dist: loguru>=0.7.0
+Requires-Dist: anyio>=3.6.2
 Provides-Extra: all
 Requires-Dist: aiomysql>=0.1.1; extra == "all"
 Requires-Dist: httpx[http2]>=0.23.0; extra == "all"

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4/aio_scrapy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.1.3
+Version: 2.1.4
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -28,6 +28,7 @@ Requires-Dist: zope.interface>=5.1.0
 Requires-Dist: redis>=4.3.1
 Requires-Dist: aiomultiprocess>=0.9.0
 Requires-Dist: loguru>=0.7.0
+Requires-Dist: anyio>=3.6.2
 Provides-Extra: all
 Requires-Dist: aiomysql>=0.1.1; extra == "all"
 Requires-Dist: httpx[http2]>=0.23.0; extra == "all"

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aio_scrapy.egg-info/requires.txt

@@ -7,6 +7,7 @@ zope.interface>=5.1.0
 redis>=4.3.1
 aiomultiprocess>=0.9.0
 loguru>=0.7.0
+anyio>=3.6.2
 
 [aio-pika]
 aio-pika>=8.1.1

aio-scrapy-2.1.4/aioscrapy/VERSION

@@ -0,0 +1 @@
+2.1.4

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aioscrapy/core/downloader/__init__.py

@@ -143,7 +143,7 @@ class Downloader(BaseDownloader):
         crawler.spider.dupefilter = df  # 将指纹绑定到Spider 在解析成功的时候 调用DUPEFILTER_CLASS的success方法
         return cls(
             crawler,
-            await call_helper(DownloadHandlerManager.for_crawler, crawler),
+            await call_helper(DownloadHandlerManager.from_crawler, crawler),
             await call_helper(DownloaderMiddlewareManager.from_crawler, crawler),
             proxy=crawler.settings.get("PROXY_HANDLER") and await load_instance(crawler.settings["PROXY_HANDLER"],
                                                                                 crawler=crawler),

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aioscrapy/core/downloader/handlers/__init__.py

@@ -37,7 +37,7 @@ class DownloadHandlerManager:
         crawler.signals.connect(self._close, signals.engine_stopped)
 
     @classmethod
-    def for_crawler(cls, crawler) -> "DownloadHandlerManager":
+    def from_crawler(cls, crawler) -> "DownloadHandlerManager":
         return cls(crawler)
 
     async def _get_handler(self, scheme: str) -> Optional[BaseDownloadHandler]:

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aioscrapy/core/downloader/handlers/aiohttp.py

@@ -37,7 +37,7 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except ClientError as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aioscrapy/core/downloader/handlers/curl_cffi.py

@@ -24,7 +24,7 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except CurlError as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aioscrapy/core/downloader/handlers/httpx.py

@@ -32,7 +32,7 @@ class HttpxDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except HttpxError as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aioscrapy/core/downloader/handlers/playwright/__init__.py

@@ -1,6 +1,10 @@
 from functools import wraps
 
-from playwright._impl._api_types import Error
+try:
+    from playwright._impl._errors import Error
+except ImportError:
+    from playwright._impl._api_types import Error
+
 from playwright.async_api._generated import Response as EventResponse
 
 from aioscrapy import Request, Spider
@@ -17,10 +21,11 @@ class PlaywrightHandler(BaseDownloadHandler):
     def __init__(self, settings: Settings):
         self.settings = settings
         playwright_client_args = settings.getdict('PLAYWRIGHT_CLIENT_ARGS')
+        use_pool = settings.getbool('PLAYWRIGHT_USE_POOL', True)
         self.wait_until = playwright_client_args.get('wait_until', 'domcontentloaded')
         self.url_regexes = playwright_client_args.pop('url_regexes', [])
         pool_size = playwright_client_args.pop('pool_size', settings.getint("CONCURRENT_REQUESTS", 1))
-        self._webdriver_pool = WebDriverPool(pool_size=pool_size, driver_cls=PlaywrightDriver, **playwright_client_args)
+        self._webdriver_pool = WebDriverPool(use_pool=use_pool, pool_size=pool_size, driver_cls=PlaywrightDriver, **playwright_client_args)
 
     @classmethod
     def from_settings(cls, settings: Settings):
@@ -30,7 +35,7 @@ class PlaywrightHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request, spider)
         except Error as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request, spider) -> PlaywrightResponse:
         cookies = dict(request.cookies)

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aioscrapy/core/downloader/handlers/playwright/driverpool.py

@@ -9,8 +9,9 @@ from aioscrapy.utils.tools import singleton
 @singleton
 class WebDriverPool:
     def __init__(
-            self, pool_size=5, driver_cls=None, **kwargs
+            self, use_pool=True, pool_size=5, driver_cls=None, **kwargs
     ):
+        self.use_pool = use_pool
         self.pool_size = pool_size
         self.driver_cls = driver_cls
         self.kwargs = kwargs
@@ -32,6 +33,8 @@ class WebDriverPool:
 
     async def get(self, **kwargs):
         async with self.lock:
+            if not self.use_pool:
+                return await self.create_driver(**kwargs)
             if not self.is_full:
                 driver = await self.create_driver(**kwargs)
                 self.driver_count += 1
@@ -40,6 +43,9 @@ class WebDriverPool:
         return driver
 
     async def release(self, driver):
+        if not self.use_pool:
+            await driver.quit()
+            return
         await self.queue.put(driver)
 
     async def remove(self, driver):

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aioscrapy/core/downloader/handlers/pyhttpx.py

@@ -27,7 +27,7 @@ class PyhttpxDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except PyHttpxError as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aioscrapy/core/downloader/handlers/requests.py

@@ -25,7 +25,7 @@ class RequestsDownloadHandler(BaseDownloadHandler):
         try:
             return await self._download_request(request)
         except RequestsError as e:
-            raise DownloadError(e) from e
+            raise DownloadError(real_error=e) from e
 
     async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aioscrapy/exceptions.py

@@ -5,6 +5,7 @@ These exceptions are documented in docs/topics/exceptions.rst. Please don't add
 new exceptions here without documenting them there.
 """
 
+
 # Internal
 
 
@@ -95,4 +96,21 @@ class ProxyException(Exception):
 
 class DownloadError(Exception):
     """下载页面时发生的错误"""
-    pass
+
+    def __init__(self, *args, real_error=None):
+        self.real_error = real_error
+        super().__init__(*args)
+
+    def __str__(self):
+        if not self.real_error:
+            return "DownloadError"
+
+        return f"{self.real_error.__class__.__module__}.{self.real_error.__class__.__name__}: {str(self.real_error)}"
+
+
+if __name__ == '__main__':
+    e = Exception("xxx")
+    reason = DownloadError(real_error=e)
+    print(reason)
+    obj = reason.real_error.__class__
+    print(f"{obj.__module__}.{obj.__name__}")

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/aioscrapy/libs/downloader/retry.py

@@ -51,7 +51,7 @@ def get_retry_request(
         if callable(reason):
             reason = reason()
         if isinstance(reason, Exception):
-            reason = global_object_name(reason.__class__)
+            reason = global_object_name((getattr(reason, "real_error", None) or reason).__class__)
 
         logger.info(
             "Retrying %(request)s (failed %(retry_times)d times): %(reason)s" % {

{aio-scrapy-2.1.3 → aio-scrapy-2.1.4}/setup.py

@@ -14,6 +14,7 @@ install_requires = [
     "redis>=4.3.1",
     "aiomultiprocess>=0.9.0",
     "loguru>=0.7.0",
+    "anyio>=3.6.2"
 ]
 extras_require = {
     "all": [

aio-scrapy-2.1.3/aioscrapy/VERSION

@@ -1 +0,0 @@
-2.1.3