aio-scrapy 2.1.0__tar.gz → 2.1.3__tar.gz

{aio-scrapy-2.1.0/aio_scrapy.egg-info → aio-scrapy-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.1.0
+Version: 2.1.3
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -33,7 +33,7 @@ Requires-Dist: aiomysql>=0.1.1; extra == "all"
 Requires-Dist: httpx[http2]>=0.23.0; extra == "all"
 Requires-Dist: aio-pika>=8.1.1; extra == "all"
 Requires-Dist: cryptography; extra == "all"
-Requires-Dist: motor>=3.1.1; extra == "all"
+Requires-Dist: motor>=2.1.0; extra == "all"
 Requires-Dist: pyhttpx>=2.10.1; extra == "all"
 Requires-Dist: asyncpg>=0.27.0; extra == "all"
 Requires-Dist: XlsxWriter>=3.1.2; extra == "all"
@@ -48,7 +48,7 @@ Requires-Dist: httpx[http2]>=0.23.0; extra == "httpx"
 Provides-Extra: aio-pika
 Requires-Dist: aio-pika>=8.1.1; extra == "aio-pika"
 Provides-Extra: mongo
-Requires-Dist: motor>=3.1.1; extra == "mongo"
+Requires-Dist: motor>=2.1.0; extra == "mongo"
 Provides-Extra: playwright
 Requires-Dist: playwright>=1.31.1; extra == "playwright"
 Provides-Extra: pyhttpx
@@ -89,7 +89,7 @@ The quick way:
 
 ```shell
 # Install the latest aio-scrapy
-pip install git+https://github.com/conlin-huang/aio-scrapy
+pip install git+https://github.com/ConlinH/aio-scrapy
 
 # default
 pip install aio-scrapy

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/README.md

@@ -24,7 +24,7 @@ The quick way:
 
 ```shell
 # Install the latest aio-scrapy
-pip install git+https://github.com/conlin-huang/aio-scrapy
+pip install git+https://github.com/ConlinH/aio-scrapy
 
 # default
 pip install aio-scrapy

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3/aio_scrapy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.1.0
+Version: 2.1.3
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -33,7 +33,7 @@ Requires-Dist: aiomysql>=0.1.1; extra == "all"
 Requires-Dist: httpx[http2]>=0.23.0; extra == "all"
 Requires-Dist: aio-pika>=8.1.1; extra == "all"
 Requires-Dist: cryptography; extra == "all"
-Requires-Dist: motor>=3.1.1; extra == "all"
+Requires-Dist: motor>=2.1.0; extra == "all"
 Requires-Dist: pyhttpx>=2.10.1; extra == "all"
 Requires-Dist: asyncpg>=0.27.0; extra == "all"
 Requires-Dist: XlsxWriter>=3.1.2; extra == "all"
@@ -48,7 +48,7 @@ Requires-Dist: httpx[http2]>=0.23.0; extra == "httpx"
 Provides-Extra: aio-pika
 Requires-Dist: aio-pika>=8.1.1; extra == "aio-pika"
 Provides-Extra: mongo
-Requires-Dist: motor>=3.1.1; extra == "mongo"
+Requires-Dist: motor>=2.1.0; extra == "mongo"
 Provides-Extra: playwright
 Requires-Dist: playwright>=1.31.1; extra == "playwright"
 Provides-Extra: pyhttpx
@@ -89,7 +89,7 @@ The quick way:
 
 ```shell
 # Install the latest aio-scrapy
-pip install git+https://github.com/conlin-huang/aio-scrapy
+pip install git+https://github.com/ConlinH/aio-scrapy
 
 # default
 pip install aio-scrapy

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aio_scrapy.egg-info/requires.txt

@@ -20,7 +20,7 @@ aiomysql>=0.1.1
 httpx[http2]>=0.23.0
 aio-pika>=8.1.1
 cryptography
-motor>=3.1.1
+motor>=2.1.0
 pyhttpx>=2.10.1
 asyncpg>=0.27.0
 XlsxWriter>=3.1.2
@@ -39,7 +39,7 @@ pillow>=9.4.0
 httpx[http2]>=0.23.0
 
 [mongo]
-motor>=3.1.1
+motor>=2.1.0
 
 [pg]
 asyncpg>=0.27.0

aio-scrapy-2.1.3/aioscrapy/VERSION

@@ -0,0 +1 @@
+2.1.3

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/__init__.py

@@ -138,14 +138,16 @@ class Downloader(BaseDownloader):
 
     @classmethod
     async def from_crawler(cls, crawler) -> "Downloader":
+        df = crawler.settings.get('DUPEFILTER_CLASS') and await load_instance(crawler.settings['DUPEFILTER_CLASS'],
+                                                                              crawler=crawler)
+        crawler.spider.dupefilter = df  # 将指纹绑定到Spider 在解析成功的时候 调用DUPEFILTER_CLASS的success方法
         return cls(
             crawler,
             await call_helper(DownloadHandlerManager.for_crawler, crawler),
             await call_helper(DownloaderMiddlewareManager.from_crawler, crawler),
             proxy=crawler.settings.get("PROXY_HANDLER") and await load_instance(crawler.settings["PROXY_HANDLER"],
                                                                                 crawler=crawler),
-            dupefilter=crawler.settings.get('DUPEFILTER_CLASS') and await load_instance(
-                crawler.settings['DUPEFILTER_CLASS'], crawler=crawler)
+            dupefilter=df
         )
 
     async def fetch(self, request: Request) -> None:
@@ -204,11 +206,17 @@ class Downloader(BaseDownloader):
             slot.transferring.remove(request)
             slot.active.remove(request)
             self.active.remove(request)
+
             if isinstance(result, Response):
                 await self.signals.send_catch_log(signal=signals.response_downloaded,
                                                   response=result,
                                                   request=request,
                                                   spider=self.spider)
+            #  控制指纹是否移除
+            self.dupefilter and \
+                not request.dont_filter and \
+                await self.dupefilter.done(request, done_type="request_ok" if isinstance(result, Response) else "request_err")
+
             await self._call_engine(result, request)
             await self._process_queue(slot)
 

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/aiohttp.py

@@ -4,9 +4,11 @@ import ssl
 from typing import Optional
 
 import aiohttp
+from aiohttp.client_exceptions import ClientError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -32,6 +34,12 @@ class AioHttpDownloadHandler(BaseDownloadHandler):
         return self.session
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except ClientError as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'verify_ssl': request.meta.get('verify_ssl', self.verify_ssl),
             'timeout': request.meta.get('download_timeout', 180),

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/curl_cffi.py

@@ -1,9 +1,9 @@
-import ssl
-
+from curl_cffi.curl import CurlError
 from curl_cffi.requests import AsyncSession
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -21,6 +21,12 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except CurlError as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
@@ -52,8 +58,8 @@ class CurlCffiDownloadHandler(BaseDownloadHandler):
             str(response.url),
             status=response.status_code,
             headers=response.headers,
-            body=response.text,
-            cookies=dict(response.cookies),
+            body=response.content,
+            cookies={j.name: j.value or '' for j in response.cookies.jar},
             encoding=response.encoding
         )
 

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/httpx.py

@@ -1,9 +1,11 @@
 import ssl
 
 import httpx
+from httpx import HTTPError as HttpxError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -27,6 +29,12 @@ class HttpxDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except HttpxError as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
@@ -68,7 +76,7 @@ class HttpxDownloadHandler(BaseDownloadHandler):
             status=response.status_code,
             headers=response.headers,
             body=content,
-            cookies=dict(response.cookies),
+            cookies={j.name: j.value or '' for j in response.cookies.jar},
             encoding=response.encoding
         )
 

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/playwright/__init__.py

@@ -1,11 +1,13 @@
 from functools import wraps
 
+from playwright._impl._api_types import Error
 from playwright.async_api._generated import Response as EventResponse
 
-from aioscrapy import Request
+from aioscrapy import Request, Spider
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
 from aioscrapy.core.downloader.handlers.playwright.driverpool import WebDriverPool
 from aioscrapy.core.downloader.handlers.playwright.webdriver import PlaywrightDriver
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import PlaywrightResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.tools import call_helper
@@ -24,7 +26,13 @@ class PlaywrightHandler(BaseDownloadHandler):
     def from_settings(cls, settings: Settings):
         return cls(settings)
 
-    async def download_request(self, request: Request, spider) -> PlaywrightResponse:
+    async def download_request(self, request: Request, spider: Spider) -> PlaywrightResponse:
+        try:
+            return await self._download_request(request, spider)
+        except Error as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request, spider) -> PlaywrightResponse:
         cookies = dict(request.cookies)
         timeout = request.meta.get('download_timeout', 30) * 1000
         user_agent = request.headers.get("User-Agent")

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/pyhttpx.py

@@ -1,9 +1,11 @@
 import asyncio
 
 import pyhttpx
+from pyhttpx.exception import BaseExpetion as PyHttpxError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -22,6 +24,12 @@ class PyhttpxDownloadHandler(BaseDownloadHandler):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except PyHttpxError as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/core/downloader/handlers/requests.py

@@ -1,9 +1,11 @@
 import asyncio
 
 import requests
+from requests.exceptions import RequestException as RequestsError
 
 from aioscrapy import Request
 from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.exceptions import DownloadError
 from aioscrapy.http import HtmlResponse
 from aioscrapy.settings import Settings
 from aioscrapy.utils.log import logger
@@ -14,13 +16,18 @@ class RequestsDownloadHandler(BaseDownloadHandler):
     def __init__(self, settings):
         self.settings: Settings = settings
         self.verify_ssl: bool = self.settings.get("VERIFY_SSL", True)
-        self.loop = asyncio.get_running_loop()
 
     @classmethod
     def from_settings(cls, settings: Settings):
         return cls(settings)
 
     async def download_request(self, request: Request, _) -> HtmlResponse:
+        try:
+            return await self._download_request(request)
+        except RequestsError as e:
+            raise DownloadError(e) from e
+
+    async def _download_request(self, request: Request) -> HtmlResponse:
         kwargs = {
             'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
             'cookies': dict(request.cookies),
@@ -48,7 +55,7 @@ class RequestsDownloadHandler(BaseDownloadHandler):
             status=response.status_code,
             headers=response.headers,
             body=response.content,
-            cookies=dict(response.cookies),
+            cookies={k: v or '' for k, v in response.cookies.items()},
             encoding=response.encoding
         )
 

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/core/scheduler.py

@@ -191,7 +191,6 @@ class Scheduler(BaseScheduler):
                 self.stats.inc_value(self.queue.inc_key, spider=self.spider)
             yield request
 
-
     async def has_pending_requests(self) -> bool:
         return await call_helper(self.queue.len) if self.cache_queue is None \
             else (await call_helper(self.queue.len) + await call_helper(self.cache_queue.len)) > 0

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/core/scraper.py

@@ -110,9 +110,14 @@ class Scraper:
                     await self.handle_spider_error(e, request, result)
                 else:
                     await self.handle_spider_output(output, request, result)
-            except BaseException:
-                logger.exception('Scraper bug processing %(request)s' % {'request': request})
+            except BaseException as e:
+                await self.handle_spider_error(e, request, result)
             finally:
+                # 控制指纹是否移除
+                self.spider.dupefilter and \
+                    not request.dont_filter and \
+                    await self.spider.dupefilter.done(request, done_type="parse_ok" if getattr(request, "parse_ok", False) else "parse_err")
+
                 if isinstance(result, PlaywrightResponse):
                     await result.release()
 
@@ -162,16 +167,22 @@ class Scraper:
         if not result:
             return
 
+        parse_ok = True
         while True:
             try:
                 output = await result.__anext__()
             except StopAsyncIteration:
                 break
             except Exception as e:
+                parse_ok = False
                 await self.handle_spider_error(e, request, response)
             else:
                 await self._process_spidermw_output(output, request, response)
 
+        self.spider.dupefilter and \
+            not request.dont_filter and \
+            setattr(request, "parse_ok", parse_ok)
+
     async def _process_spidermw_output(self, output: Any, request: Request, response: Response) -> None:
         """Process each Request/Item (given in the output parameter) returned from the given spider"""
 

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/db/aiomongo.py

@@ -1,7 +1,9 @@
 from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo.errors import NetworkTimeout
 
 import aioscrapy
 from aioscrapy.db.absmanager import AbsDBPoolManager
+from loguru import logger
 
 
 class MongoExecutor:
@@ -9,10 +11,16 @@ class MongoExecutor:
         self.alias = alias
         self.pool_manager = pool_manager
 
-    async def insert(self, table_name, values, db_name=None):
+    async def insert(self, table_name, values, db_name=None, ordered=False, retry_times=3):
         client, db_name_default = self.pool_manager.get_pool(self.alias)
         db_name = db_name or db_name_default
-        return await client[f'{db_name}'][f'{table_name}'].insert_many(values)
+        for _ in range(retry_times):
+            try:
+                return await client[f'{db_name}'][f'{table_name}'].insert_many(values, ordered=ordered)
+            except NetworkTimeout:
+                logger.warning("mongo insert error by NetworkTimeout, retrying...")
+
+        raise NetworkTimeout
 
     def __getattr__(self, table_name: str):
         client, db_name_default = self.pool_manager.get_pool(self.alias)

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/dupefilters/__init__.py

@@ -1,3 +1,4 @@
+from typing import Literal
 from abc import ABCMeta, abstractmethod
 
 from aioscrapy import Request, Spider
@@ -37,3 +38,10 @@ class DupeFilterBase(metaclass=ABCMeta):
             self.logdupes = False
 
         spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
+
+    async def done(
+            self,
+            request: Request,
+            done_type: Literal["request_ok", "request_err", "parse_ok", "parse_err"]
+    ) -> None:
+        """ 根据done_type的状态 控制指纹的移除 """

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/dupefilters/redis.py

@@ -1,3 +1,5 @@
+from typing import Literal
+
 from aioscrapy import Request
 from aioscrapy.db import db_manager
 from aioscrapy.dupefilters import DupeFilterBase
@@ -128,5 +130,68 @@ class RedisBloomDupeFilter(RedisRFPDupeFilter):
         return False
 
 
+class ExRedisBloomDupeFilter(RedisBloomDupeFilter):
+
+    def __init__(self, server, key, key_set, ttl, debug, bit, hash_number, keep_on_close, info):
+        super().__init__(server, key, debug, bit, hash_number, keep_on_close, info)
+        self.key_set = key_set
+        self.ttl = ttl
+
+    @classmethod
+    async def from_crawler(cls, crawler: "aioscrapy.crawler.Crawler"):
+        server = db_manager.redis.queue
+        dupefilter_key = crawler.settings.get("SCHEDULER_DUPEFILTER_KEY", '%(spider)s:bloomfilter')
+        keep_on_close = crawler.settings.getbool("KEEP_DUPEFILTER_DATA_ON_CLOSE", True)
+        key = dupefilter_key % {'spider': crawler.spider.name}
+        debug = crawler.settings.getbool('DUPEFILTER_DEBUG', False)
+        info = crawler.settings.getbool('DUPEFILTER_INFO', False)
+        bit = crawler.settings.getint('BLOOMFILTER_BIT', 30)
+        hash_number = crawler.settings.getint('BLOOMFILTER_HASH_NUMBER', 6)
+        ttl = crawler.settings.getint('DUPEFILTER_SET_KEY_TTL', 180)
+        return cls(server, key=key, key_set=key + "_set", ttl=ttl, debug=debug, bit=bit, hash_number=hash_number,
+                   keep_on_close=keep_on_close, info=info)
+
+    async def request_seen(self, request: Request) -> bool:
+        fp = await self.bf.exists(request.fingerprint)
+        if fp:
+            return True
+        async with self.server.pipeline() as pipe:
+            pipe.sadd(self.key_set, request.fingerprint)
+            pipe.expire(self.key_set, self.ttl)
+            ret, _ = await pipe.execute()
+        return ret == 0
+
+    async def done(
+            self,
+            request: Request,
+            done_type: Literal["request_ok", "request_err", "parse_ok", "parse_err"]
+    ):
+        if done_type == "request_ok" or done_type == "request_err":
+            await self.server.srem(self.key_set, request.fingerprint)
+        elif done_type == "parse_ok":
+            await self.bf.insert(request.fingerprint)
+
+    async def close(self, reason=''):
+        if not self.keep_on_close:
+            await self.clear()
+        await self.server.delete(self.key_set)
+
+
+class ExRedisRFPDupeFilter(RedisRFPDupeFilter):
+
+    async def done(
+            self,
+            request: Request,
+            done_type: Literal["request_ok", "request_err", "parse_ok", "parse_err"]
+    ):
+        # 当请求失败或解析失败的时候 从Redis的Set中移除指纹
+        if done_type == "request_err" or done_type == "parse_err":
+            await self.server.srem(self.key, request.fingerprint)
+
+
 RFPDupeFilter = RedisRFPDupeFilter
+ExRFPDupeFilter = ExRedisRFPDupeFilter
 BloomDupeFilter = RedisBloomDupeFilter
+ExBloomDupeFilter = ExRedisBloomDupeFilter
+BloomSetDupeFilter = ExRedisBloomDupeFilter
+

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/exceptions.py

@@ -91,3 +91,8 @@ class ContractFail(AssertionError):
 
 class ProxyException(Exception):
     pass
+
+
+class DownloadError(Exception):
+    """下载页面时发生的错误"""
+    pass

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/libs/downloader/retry.py

@@ -10,70 +10,21 @@ Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
 from typing import Optional, Union
-from aioscrapy.exceptions import ProxyException
+
+from anyio import EndOfStream
 
 try:
     from asyncio.exceptions import TimeoutError
 except:
     from concurrent.futures._base import TimeoutError
 
-NEED_RETRY_ERROR = (TimeoutError, ConnectionRefusedError, IOError, ProxyException)
-
-try:
-    from aiohttp.client_exceptions import ClientError
-
-    NEED_RETRY_ERROR += (ClientError,)
-except ImportError:
-    pass
-
-try:
-    from anyio import EndOfStream
-
-    NEED_RETRY_ERROR += (EndOfStream,)
-except ImportError:
-    pass
-
-try:
-    from httpx import HTTPError as HttpxError
-
-    NEED_RETRY_ERROR += (HttpxError,)
-except ImportError:
-    pass
-
-try:
-    from pyhttpx.exception import BaseExpetion as PyHttpxError
-
-    NEED_RETRY_ERROR += (PyHttpxError,)
-except ImportError:
-    pass
-
-try:
-    from requests.exceptions import RequestException as RequestsError
-
-    NEED_RETRY_ERROR += (RequestsError,)
-except ImportError:
-    pass
-
-try:
-    from playwright._impl._api_types import Error as PlaywrightError
-
-    NEED_RETRY_ERROR += (PlaywrightError,)
-except ImportError:
-    pass
-
-
-try:
-    from curl_cffi.curl import CurlError
-
-    NEED_RETRY_ERROR += (CurlError,)
-except ImportError:
-    pass
-
-from aioscrapy.exceptions import NotConfigured
+from aioscrapy.exceptions import ProxyException, DownloadError, NotConfigured
 from aioscrapy.http.request import Request
 from aioscrapy.spiders import Spider
-from aioscrapy.utils.python import global_object_name
 from aioscrapy.utils.log import logger as retry_logger
+from aioscrapy.utils.python import global_object_name
+
+NEED_RETRY_ERROR = (TimeoutError, ConnectionRefusedError, IOError, ProxyException, DownloadError, EndOfStream)
 
 
 def get_retry_request(

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/libs/pipelines/mongo.py

@@ -9,6 +9,8 @@ class MongoPipeline(DBPipelineBase):
     def __init__(self, settings, db_type: str):
         super().__init__(settings, db_type)
         self.db_cache = {}
+        self.ordered_cache = {}
+        self.retry_times = settings.getint("MONGO_TIMEOUT_RETRY_TIMES", 3)
 
     @classmethod
     def from_settings(cls, settings):
@@ -17,17 +19,19 @@ class MongoPipeline(DBPipelineBase):
     def parse_item_to_cache(self, item: dict, save_info: dict):
         db_name = save_info.get('db_name')
         table_name = save_info.get('table_name')
+        ordered = save_info.get('ordered', False)
         assert table_name is not None, 'please set table_name'
         db_alias = save_info.get('db_alias', ['default'])
         if isinstance(db_alias, str):
             db_alias = [db_alias]
 
-        cache_key = ''.join(db_alias) + (db_name or '') + table_name
+        cache_key = ''.join(db_alias) + (db_name or '') + table_name + str(ordered)
 
         if self.table_cache.get(cache_key) is None:
             self.db_alias_cache[cache_key] = db_alias
             self.table_cache[cache_key] = table_name
             self.db_cache[cache_key] = db_name
+            self.ordered_cache[cache_key] = ordered
             self.item_cache[cache_key] = []
 
         self.item_cache[cache_key].append(item)
@@ -40,7 +44,8 @@ class MongoPipeline(DBPipelineBase):
                 try:
                     executor = db_manager.mongo.executor(alias)
                     result = await executor.insert(
-                        table_name, self.item_cache[cache_key], db_name=self.db_cache[cache_key]
+                        table_name, self.item_cache[cache_key], db_name=self.db_cache[cache_key],
+                        ordered=self.ordered_cache[cache_key], retry_times=self.retry_times
                     )
                     logger.info(
                         f'table:{alias}->{table_name} sum:{len(self.item_cache[cache_key])} ok:{len(result.inserted_ids)}'

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/spiders/__init__.py

@@ -22,6 +22,7 @@ class Spider(object):
 
     name: Optional[str] = None
     proxy: Optional["aioscrapy.proxy.AbsProxy"] = None
+    dupefilter: Optional["aioscrapy.dupefilters.DupeFilterBase"] = None
     custom_settings: Optional[dict] = None
     stats: Optional[StatsCollector] = None
 
@@ -77,7 +78,7 @@ class Spider(object):
             yield Request(url)
 
     async def request_from_dict(self, d: dict):
-        """集成后重写改方法，将队列中的json根据情况构建成Request对象"""
+        """继承成后重写改方法，将队列中的json根据情况构建成Request对象"""
         pass
 
     async def _parse(self, response: Response, **kwargs):

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/templates/spiders/single.tmpl

@@ -24,11 +24,12 @@ class $classname(Spider):
         pass
 
     async def parse(self, response):
-        item = {
-            'author': quote.xpath('span/small/text()').get(),
-            'text': quote.css('span.text::text').get(),
-        }
-        yield item
+        for quote in response.css('div.quote'):
+            item = {
+                'author': quote.xpath('span/small/text()').get(),
+                'text': quote.css('span.text::text').get(),
+            }
+            yield item
 
     async def process_item(self, item):
         logger.info(item)

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/aioscrapy/utils/python.py

@@ -1,16 +1,12 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
-import errno
 import gc
-import inspect
 import re
 import sys
-import warnings
 import weakref
-from functools import partial, wraps
+from functools import wraps
 
-from aioscrapy.exceptions import AioScrapyDeprecationWarning
 from aioscrapy.utils.decorators import deprecated
 
 
@@ -150,4 +146,3 @@ if hasattr(sys, "pypy_version_info"):
 else:
     def garbage_collect():
         gc.collect()
-

{aio-scrapy-2.1.0 → aio-scrapy-2.1.3}/setup.py

@@ -18,13 +18,13 @@ install_requires = [
 extras_require = {
     "all": [
         "aiomysql>=0.1.1", "httpx[http2]>=0.23.0", "aio-pika>=8.1.1",
-        "cryptography", "motor>=3.1.1", "pyhttpx>=2.10.1", "asyncpg>=0.27.0",
+        "cryptography", "motor>=2.1.0", "pyhttpx>=2.10.1", "asyncpg>=0.27.0",
         "XlsxWriter>=3.1.2", "pillow>=9.4.0", "requests>=2.28.2", "curl_cffi"
     ],
     "aiomysql": ["aiomysql>=0.1.1", "cryptography"],
     "httpx": ["httpx[http2]>=0.23.0"],
     "aio-pika": ["aio-pika>=8.1.1"],
-    "mongo": ["motor>=3.1.1"],
+    "mongo": ["motor>=2.1.0"],
     "playwright": ["playwright>=1.31.1"],
     "pyhttpx": ["pyhttpx>=2.10.4"],
     "curl_cffi": ["curl_cffi>=0.6.1"],

aio-scrapy-2.1.0/aioscrapy/VERSION

@@ -1 +0,0 @@
-2.1.0