aio-scrapy 2.0.10__tar.gz → 2.1.2__tar.gz

{aio-scrapy-2.0.10/aio_scrapy.egg-info → aio-scrapy-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.0.10
+Version: 2.1.2
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -33,12 +33,13 @@ Requires-Dist: aiomysql>=0.1.1; extra == "all"
 Requires-Dist: httpx[http2]>=0.23.0; extra == "all"
 Requires-Dist: aio-pika>=8.1.1; extra == "all"
 Requires-Dist: cryptography; extra == "all"
-Requires-Dist: motor>=3.1.1; extra == "all"
+Requires-Dist: motor>=2.1.0; extra == "all"
 Requires-Dist: pyhttpx>=2.10.1; extra == "all"
 Requires-Dist: asyncpg>=0.27.0; extra == "all"
 Requires-Dist: XlsxWriter>=3.1.2; extra == "all"
 Requires-Dist: pillow>=9.4.0; extra == "all"
 Requires-Dist: requests>=2.28.2; extra == "all"
+Requires-Dist: curl_cffi; extra == "all"
 Provides-Extra: aiomysql
 Requires-Dist: aiomysql>=0.1.1; extra == "aiomysql"
 Requires-Dist: cryptography; extra == "aiomysql"
@@ -47,11 +48,13 @@ Requires-Dist: httpx[http2]>=0.23.0; extra == "httpx"
 Provides-Extra: aio-pika
 Requires-Dist: aio-pika>=8.1.1; extra == "aio-pika"
 Provides-Extra: mongo
-Requires-Dist: motor>=3.1.1; extra == "mongo"
+Requires-Dist: motor>=2.1.0; extra == "mongo"
 Provides-Extra: playwright
 Requires-Dist: playwright>=1.31.1; extra == "playwright"
 Provides-Extra: pyhttpx
 Requires-Dist: pyhttpx>=2.10.4; extra == "pyhttpx"
+Provides-Extra: curl-cffi
+Requires-Dist: curl_cffi>=0.6.1; extra == "curl-cffi"
 Provides-Extra: requests
 Requires-Dist: requests>=2.28.2; extra == "requests"
 Provides-Extra: pg
@@ -86,7 +89,7 @@ The quick way:
 
 ```shell
 # Install the latest aio-scrapy
-pip install git+https://github.com/conlin-huang/aio-scrapy
+pip install git+https://github.com/ConlinH/aio-scrapy
 
 # default
 pip install aio-scrapy

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/README.md

@@ -24,7 +24,7 @@ The quick way:
 
 ```shell
 # Install the latest aio-scrapy
-pip install git+https://github.com/conlin-huang/aio-scrapy
+pip install git+https://github.com/ConlinH/aio-scrapy
 
 # default
 pip install aio-scrapy

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2/aio_scrapy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.0.10
+Version: 2.1.2
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -33,12 +33,13 @@ Requires-Dist: aiomysql>=0.1.1; extra == "all"
 Requires-Dist: httpx[http2]>=0.23.0; extra == "all"
 Requires-Dist: aio-pika>=8.1.1; extra == "all"
 Requires-Dist: cryptography; extra == "all"
-Requires-Dist: motor>=3.1.1; extra == "all"
+Requires-Dist: motor>=2.1.0; extra == "all"
 Requires-Dist: pyhttpx>=2.10.1; extra == "all"
 Requires-Dist: asyncpg>=0.27.0; extra == "all"
 Requires-Dist: XlsxWriter>=3.1.2; extra == "all"
 Requires-Dist: pillow>=9.4.0; extra == "all"
 Requires-Dist: requests>=2.28.2; extra == "all"
+Requires-Dist: curl_cffi; extra == "all"
 Provides-Extra: aiomysql
 Requires-Dist: aiomysql>=0.1.1; extra == "aiomysql"
 Requires-Dist: cryptography; extra == "aiomysql"
@@ -47,11 +48,13 @@ Requires-Dist: httpx[http2]>=0.23.0; extra == "httpx"
 Provides-Extra: aio-pika
 Requires-Dist: aio-pika>=8.1.1; extra == "aio-pika"
 Provides-Extra: mongo
-Requires-Dist: motor>=3.1.1; extra == "mongo"
+Requires-Dist: motor>=2.1.0; extra == "mongo"
 Provides-Extra: playwright
 Requires-Dist: playwright>=1.31.1; extra == "playwright"
 Provides-Extra: pyhttpx
 Requires-Dist: pyhttpx>=2.10.4; extra == "pyhttpx"
+Provides-Extra: curl-cffi
+Requires-Dist: curl_cffi>=0.6.1; extra == "curl-cffi"
 Provides-Extra: requests
 Requires-Dist: requests>=2.28.2; extra == "requests"
 Provides-Extra: pg
@@ -86,7 +89,7 @@ The quick way:
 
 ```shell
 # Install the latest aio-scrapy
-pip install git+https://github.com/conlin-huang/aio-scrapy
+pip install git+https://github.com/ConlinH/aio-scrapy
 
 # default
 pip install aio-scrapy

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aio_scrapy.egg-info/SOURCES.txt

@@ -38,6 +38,7 @@ aioscrapy/core/scraper.py
 aioscrapy/core/downloader/__init__.py
 aioscrapy/core/downloader/handlers/__init__.py
 aioscrapy/core/downloader/handlers/aiohttp.py
+aioscrapy/core/downloader/handlers/curl_cffi.py
 aioscrapy/core/downloader/handlers/httpx.py
 aioscrapy/core/downloader/handlers/pyhttpx.py
 aioscrapy/core/downloader/handlers/requests.py

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aio_scrapy.egg-info/requires.txt

@@ -20,12 +20,16 @@ aiomysql>=0.1.1
 httpx[http2]>=0.23.0
 aio-pika>=8.1.1
 cryptography
-motor>=3.1.1
+motor>=2.1.0
 pyhttpx>=2.10.1
 asyncpg>=0.27.0
 XlsxWriter>=3.1.2
 pillow>=9.4.0
 requests>=2.28.2
+curl_cffi
+
+[curl_cffi]
+curl_cffi>=0.6.1
 
 [execl]
 XlsxWriter>=3.1.2
@@ -35,7 +39,7 @@ pillow>=9.4.0
 httpx[http2]>=0.23.0
 
 [mongo]
-motor>=3.1.1
+motor>=2.1.0
 
 [pg]
 asyncpg>=0.27.0

aio-scrapy-2.1.2/aioscrapy/VERSION

@@ -0,0 +1 @@
+2.1.2

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/core/downloader/__init__.py

@@ -138,14 +138,14 @@ class Downloader(BaseDownloader):
 
     @classmethod
     async def from_crawler(cls, crawler) -> "Downloader":
+        df = crawler.settings.get('DUPEFILTER_CLASS') and await load_instance(crawler.settings['DUPEFILTER_CLASS'], crawler=crawler)
+        crawler.spider.dupefilter = df  # 将指纹绑定到Spider 在解析成功的时候 调用DUPEFILTER_CLASS的success方法
         return cls(
             crawler,
             await call_helper(DownloadHandlerManager.for_crawler, crawler),
             await call_helper(DownloaderMiddlewareManager.from_crawler, crawler),
-            proxy=crawler.settings.get("PROXY_HANDLER") and await load_instance(crawler.settings["PROXY_HANDLER"],
-                                                                                crawler=crawler),
-            dupefilter=crawler.settings.get('DUPEFILTER_CLASS') and await load_instance(
-                crawler.settings['DUPEFILTER_CLASS'], crawler=crawler)
+            proxy=crawler.settings.get("PROXY_HANDLER") and await load_instance(crawler.settings["PROXY_HANDLER"], crawler=crawler),
+            dupefilter=df
         )
 
     async def fetch(self, request: Request) -> None:
@@ -204,6 +204,7 @@ class Downloader(BaseDownloader):
             slot.transferring.remove(request)
             slot.active.remove(request)
             self.active.remove(request)
+            self.dupefilter and not request.dont_filter and await self.dupefilter.done(request, done_type="request_done")
             if isinstance(result, Response):
                 await self.signals.send_catch_log(signal=signals.response_downloaded,
                                                   response=result,

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/core/downloader/handlers/aiohttp.py

@@ -4,9 +4,11 @@ import ssl
 from typing import Optional
 
 import aiohttp
+from aiohttp.client_exceptions import ClientError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -32,6 +34,12 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
         return self.session
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except ClientError as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'verify_ssl': request.meta.get('verify_ssl', self.verify_ssl),
             'timeout': request.meta.get('download_timeout', 180),

aio-scrapy-2.1.2/aioscrapy/core/downloader/handlers/curl_cffi.py

@@ -0,0 +1,67 @@
+from curl_cffi.curl import CurlError
+from curl_cffi.requests import AsyncSession
+
+from aioscrapy import Request
+from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
+from aioscrapy.http import HtmlResponse
+from aioscrapy.settings import Settings
+from aioscrapy.utils.log import logger
+
+
+class CurlCffiDownloadHandler(BaseDownloadHandler):
+
+    def __init__(self, settings):
+        self.settings: Settings = settings
+        self.httpx_client_session_args: dict = self.settings.get('CURL_CFFI_CLIENT_SESSION_ARGS', {})
+        self.verify_ssl: bool = self.settings.get("VERIFY_SSL", True)
+
+    @classmethod
+    def from_settings(cls, settings: Settings):
+        return cls(settings)
+
+    async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except CurlError as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
+        kwargs = {
+            'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
+            'cookies': dict(request.cookies),
+            'verify': request.meta.get('verify_ssl', self.verify_ssl),
+            'allow_redirects': self.settings.getbool('REDIRECT_ENABLED', True) if request.meta.get(
+                'dont_redirect') is None else request.meta.get('dont_redirect'),
+            'impersonate': request.meta.get('impersonate'),
+        }
+        post_data = request.body or None
+        if isinstance(post_data, dict):
+            kwargs['json'] = post_data
+        else:
+            kwargs['data'] = post_data
+
+        headers = request.headers or self.settings.get('DEFAULT_REQUEST_HEADERS')
+        kwargs['headers'] = headers
+
+        proxy = request.meta.get("proxy")
+        if proxy:
+            kwargs["proxies"] = {'http': proxy, 'https': proxy}
+            logger.debug(f"use proxy {proxy}: {request.url}")
+
+        session_args = self.httpx_client_session_args.copy()
+
+        async with AsyncSession(**session_args) as session:
+            response = await session.request(request.method, request.url, **kwargs)
+
+        return HtmlResponse(
+            str(response.url),
+            status=response.status_code,
+            headers=response.headers,
+            body=response.content,
+            cookies={j.name: j.value or '' for j in response.cookies.jar},
+            encoding=response.encoding
+        )
+
+    async def close(self):
+        pass

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/core/downloader/handlers/httpx.py

@@ -1,9 +1,11 @@
 import ssl
 
 import httpx
+from httpx import HTTPError as HttpxError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -27,6 +29,12 @@ class HttpxDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except HttpxError as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
@@ -68,7 +76,7 @@ class HttpxDownloadHandler(BaseDownloadHandler):
             status=response.status_code,
             headers=response.headers,
             body=content,
-            cookies=dict(response.cookies),
+            cookies={j.name: j.value or '' for j in response.cookies.jar},
             encoding=response.encoding
         )
 

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/core/downloader/handlers/playwright/__init__.py

@@ -1,11 +1,13 @@
 from functools import wraps
 
+from playwright._impl._api_types import Error
 from playwright.async_api._generated import Response as EventResponse
 
-from aioscrapy import Request
+from aioscrapy import Request, Spider
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
 from aioscrapy.core.downloader.handlers.playwright.driverpool import WebDriverPool
 from aioscrapy.core.downloader.handlers.playwright.webdriver import PlaywrightDriver
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import PlaywrightResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.tools import call_helper
@@ -24,7 +26,13 @@ class PlaywrightHandler(BaseDownloadHandler):
     def from_settings(cls, settings: Settings):
         return cls(settings)
 
-    async def download_request(self, request: Request, spider) -> PlaywrightResponse:
+    async def download_request(self, request: Request, spider: Spider) -> PlaywrightResponse:
+        try:
+            return await self._download_request(request, spider)
+        except Error as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request, spider) -> PlaywrightResponse:
         cookies = dict(request.cookies)
         timeout = request.meta.get('download_timeout', 30) * 1000
         user_agent = request.headers.get("User-Agent")

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/core/downloader/handlers/pyhttpx.py

@@ -1,9 +1,11 @@
 import asyncio
 
 import pyhttpx
+from pyhttpx.exception import BaseExpetion as PyHttpxError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -22,6 +24,12 @@ class PyhttpxDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except PyHttpxError as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/core/downloader/handlers/requests.py

@@ -1,9 +1,11 @@
 import asyncio
 
 import requests
+from requests.exceptions import RequestException as RequestsError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -21,6 +23,12 @@ class RequestsDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except RequestsError as e:
+            raise DownloadError from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
@@ -48,7 +56,7 @@ class RequestsDownloadHandler(BaseDownloadHandler):
             status=response.status_code,
             headers=response.headers,
             body=response.content,
-            cookies=dict(response.cookies),
+            cookies={k: v or '' for k, v in response.cookies.items()},
             encoding=response.encoding
         )
 

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/core/engine.py

@@ -42,8 +42,6 @@ class ExecutionEngine(object):
         self.signals = crawler.signals
         self.logformatter = crawler.logformatter
 
-        self.enqueue_cache_num = self.settings.getint("ENQUEUE_CACHE_NUM")
-        self.enqueue_cache: Queue = Queue(self.enqueue_cache_num)
         self.slot: Optional[Slot] = None
         self.spider: Optional[Spider] = None
         self.downloader: Optional[DownloaderTV] = None
@@ -53,7 +51,6 @@ class ExecutionEngine(object):
         self.running: bool = False
         self.unlock: bool = True
         self.finish: bool = False
-        self.enqueue_unlock: bool = True
 
     async def start(
             self,
@@ -70,7 +67,6 @@ class ExecutionEngine(object):
         while not self.finish:
             self.running and await self._next_request()
             await asyncio.sleep(1)
-            self.enqueue_cache_num != 1 and create_task(self._crawl())
             self.running and await self._spider_idle(self.spider)
 
     async def stop(self, reason: str = 'shutdown') -> None:
@@ -81,7 +77,6 @@ class ExecutionEngine(object):
 
         while not self.is_idle():
             await asyncio.sleep(0.2)
-            self.enqueue_cache_num != 1 and create_task(self._crawl())
         await self.close_spider(self.spider, reason=reason)
         await self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
         self.finish = True
@@ -212,27 +207,8 @@ class ExecutionEngine(object):
         return True
 
     async def crawl(self, request: Request) -> None:
-        if self.enqueue_cache_num == 1:
-            await self.scheduler.enqueue_request(request)
-            create_task(self._next_request())
-        else:
-            await self.enqueue_cache.put(request)
-
-    async def _crawl(self) -> None:
-        if not self.enqueue_unlock:
-            return
-        self.enqueue_unlock = False
-        requests = []
-        for _ in range(self.enqueue_cache.qsize()):
-            try:
-                request = self.enqueue_cache.get_nowait()
-                requests.append(request)
-            except QueueEmpty:
-                break
-        if requests:
-            await call_helper(self.scheduler.enqueue_request_batch, requests)
-            create_task(self._next_request())
-        self.enqueue_unlock = True
+        await self.scheduler.enqueue_request(request)
+        # create_task(self._next_request())
 
     async def close_spider(self, spider: Spider, reason: str = 'cancelled') -> None:
         """Close (cancel) spider and clear all its outstanding requests"""
@@ -276,7 +252,6 @@ class ExecutionEngine(object):
         # method of 'has_pending_requests' has IO, so method of 'is_idle' execute twice
         if self.is_idle() \
                 and self.slot.start_requests is None \
-                and self.enqueue_unlock and self.enqueue_cache.empty() \
                 and not await self.scheduler.has_pending_requests() \
                 and self.is_idle():
             await self.stop(reason='finished')

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/core/scheduler.py

@@ -31,7 +31,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
     @classmethod
     async def from_crawler(cls, crawler: "aioscrapy.Crawler") -> "BaseScheduler":
         """
-        Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument.
+        Factory method which receives the current :class:`~aioscrapy.crawler.Crawler` object as argument.
         """
         return cls()
 
@@ -103,20 +103,27 @@ class Scheduler(BaseScheduler):
             queue: AbsQueue,
             spider: aioscrapy.Spider,
             stats=Optional[StatsCollector],
-            persist: bool = True
+            persist: bool = True,
+            cache_queue: Optional[AbsQueue] = None
     ):
+
         self.queue = queue
+        self.cache_queue = cache_queue
         self.spider = spider
         self.stats = stats
         self.persist = persist
 
     @classmethod
     async def from_crawler(cls: Type[SchedulerTV], crawler: "aioscrapy.Crawler") -> SchedulerTV:
+        cache_queue = None
+        if crawler.settings.getbool('USE_SCHEDULER_QUEUE_CACHE', False):
+            cache_queue = await load_instance('aioscrapy.queue.memory.SpiderPriorityQueue', spider=crawler.spider)
         instance = cls(
             await load_instance(crawler.settings['SCHEDULER_QUEUE_CLASS'], spider=crawler.spider),
             crawler.spider,
             stats=crawler.stats,
-            persist=crawler.settings.getbool('SCHEDULER_PERSIST', True)
+            persist=crawler.settings.getbool('SCHEDULER_PERSIST', True),
+            cache_queue=cache_queue
         )
 
         if crawler.settings.getbool('SCHEDULER_FLUSH_ON_START', False):
@@ -128,8 +135,20 @@ class Scheduler(BaseScheduler):
         return instance
 
     async def close(self, reason: str) -> None:
+
         if not self.persist:
             await self.flush()
+            return
+
+        # 如果持久化，将缓存中的任务放回到redis等分布式队列中
+        if self.cache_queue is not None:
+            while True:
+                temp = []
+                async for request in self.cache_queue.pop(2000):
+                    temp.append(request)
+                temp and await self.queue.push_batch(temp)
+                if len(temp) < 2000:
+                    break
 
     async def flush(self) -> None:
         await call_helper(self.queue.clear)
@@ -141,16 +160,37 @@ class Scheduler(BaseScheduler):
         return True
 
     async def enqueue_request(self, request: aioscrapy.Request) -> bool:
-        await call_helper(self.queue.push, request)
+        """
+        如果启用了缓存队列(USE_SCHEDULER_QUEUE_CACHE)，则优先将任务放到缓存队列中
+        """
+        if self.cache_queue is not None:
+            await call_helper(self.cache_queue.push, request)
+        else:
+            await call_helper(self.queue.push, request)
         if self.stats:
             self.stats.inc_value(self.queue.inc_key, spider=self.spider)
         return True
 
     async def next_request(self, count: int = 1) -> Optional[aioscrapy.Request]:
+        """
+        如果启用了缓存队列(USE_SCHEDULER_QUEUE_CACHE)，则优先从缓存队列中获取任务，然后从redis等分布式队列中获取任务
+        """
+        flag = False
+        if self.cache_queue is not None:
+            async for request in self.cache_queue.pop(count):
+                if request and self.stats:
+                    self.stats.inc_value(self.queue.inc_key, spider=self.spider)
+                yield request
+                flag = True
+
+        if flag:
+            return
+
         async for request in self.queue.pop(count):
             if request and self.stats:
                 self.stats.inc_value(self.queue.inc_key, spider=self.spider)
             yield request
 
     async def has_pending_requests(self) -> bool:
-        return await call_helper(self.queue.len) > 0
+        return await call_helper(self.queue.len) if self.cache_queue is None \
+            else (await call_helper(self.queue.len) + await call_helper(self.cache_queue.len)) > 0

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/core/scraper.py

@@ -110,8 +110,8 @@ class Scraper:
                     await self.handle_spider_error(e, request, result)
                 else:
                     await self.handle_spider_output(output, request, result)
-            except BaseException:
-                logger.exception('Scraper bug processing %(request)s' % {'request': request})
+            except BaseException as e:
+                await self.handle_spider_error(e, request, result)
             finally:
                 if isinstance(result, PlaywrightResponse):
                     await result.release()
@@ -161,17 +161,23 @@ class Scraper:
         """Iter each Request/Item (given in the output parameter) returned from the given spider"""
         if not result:
             return
-
+        parser_successful = True
         while True:
             try:
                 output = await result.__anext__()
             except StopAsyncIteration:
                 break
             except Exception as e:
+                parser_successful = False
                 await self.handle_spider_error(e, request, response)
             else:
                 await self._process_spidermw_output(output, request, response)
 
+        self.spider.dupefilter and \
+        not request.dont_filter and \
+        parser_successful and \
+        await self.spider.dupefilter.done(request, done_type="parse_done")
+
     async def _process_spidermw_output(self, output: Any, request: Request, response: Response) -> None:
         """Process each Request/Item (given in the output parameter) returned from the given spider"""
 

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/crawler.py

@@ -234,9 +234,12 @@ class CrawlerProcess(CrawlerRunner):
         finally:
             await self.recycle_db_connect()
 
-    def start(self) -> None:
+    def start(self, use_windows_selector_eventLoop: bool = False) -> None:
         if sys.platform.startswith('win'):
-            asyncio.set_event_loop(asyncio.windows_events.ProactorEventLoop())
+            if use_windows_selector_eventLoop:
+                asyncio.set_event_loop_policy(asyncio.windows_events.WindowsSelectorEventLoopPolicy())
+            else:
+                asyncio.set_event_loop(asyncio.windows_events.ProactorEventLoop())
         else:
             try:
                 import uvloop

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/db/aiomongo.py

@@ -1,7 +1,9 @@
 from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo.errors import NetworkTimeout
 
 import aioscrapy
 from aioscrapy.db.absmanager import AbsDBPoolManager
+from loguru import logger
 
 
 class MongoExecutor:
@@ -9,10 +11,16 @@ class MongoExecutor:
         self.alias = alias
         self.pool_manager = pool_manager
 
-    async def insert(self, table_name, values, db_name=None):
+    async def insert(self, table_name, values, db_name=None, ordered=False, retry_times=3):
         client, db_name_default = self.pool_manager.get_pool(self.alias)
         db_name = db_name or db_name_default
-        return await client[f'{db_name}'][f'{table_name}'].insert_many(values)
+        for _ in range(retry_times):
+            try:
+                return await client[f'{db_name}'][f'{table_name}'].insert_many(values, ordered=ordered)
+            except NetworkTimeout:
+                logger.warning("mongo insert error by NetworkTimeout, retrying...")
+
+        raise NetworkTimeout
 
     def __getattr__(self, table_name: str):
         client, db_name_default = self.pool_manager.get_pool(self.alias)

{aio-scrapy-2.0.10 → aio-scrapy-2.1.2}/aioscrapy/dupefilters/__init__.py

@@ -1,3 +1,4 @@
+from typing import Literal
 from abc import ABCMeta, abstractmethod
 
 from aioscrapy import Request, Spider
@@ -37,3 +38,6 @@ class DupeFilterBase(metaclass=ABCMeta):
             self.logdupes = False
 
         spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
+
+    async def done(self, request: Request, done_type: Literal["request_done", "parse_done"]) -> None:
+        """ deal fingerprint on task successful """