aio-scrapy 2.0.10__tar.gz → 2.1.0__tar.gz

{aio-scrapy-2.0.10/aio_scrapy.egg-info → aio-scrapy-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.0.10
+Version: 2.1.0
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -39,6 +39,7 @@ Requires-Dist: asyncpg>=0.27.0; extra == "all"
 Requires-Dist: XlsxWriter>=3.1.2; extra == "all"
 Requires-Dist: pillow>=9.4.0; extra == "all"
 Requires-Dist: requests>=2.28.2; extra == "all"
+Requires-Dist: curl_cffi; extra == "all"
 Provides-Extra: aiomysql
 Requires-Dist: aiomysql>=0.1.1; extra == "aiomysql"
 Requires-Dist: cryptography; extra == "aiomysql"
@@ -52,6 +53,8 @@ Provides-Extra: playwright
 Requires-Dist: playwright>=1.31.1; extra == "playwright"
 Provides-Extra: pyhttpx
 Requires-Dist: pyhttpx>=2.10.4; extra == "pyhttpx"
+Provides-Extra: curl-cffi
+Requires-Dist: curl_cffi>=0.6.1; extra == "curl-cffi"
 Provides-Extra: requests
 Requires-Dist: requests>=2.28.2; extra == "requests"
 Provides-Extra: pg

{aio-scrapy-2.0.10 → aio-scrapy-2.1.0/aio_scrapy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aio-scrapy
-Version: 2.0.10
+Version: 2.1.0
 Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
 Home-page: https://github.com/conlin-huang/aio-scrapy.git
 Author: conlin
@@ -39,6 +39,7 @@ Requires-Dist: asyncpg>=0.27.0; extra == "all"
 Requires-Dist: XlsxWriter>=3.1.2; extra == "all"
 Requires-Dist: pillow>=9.4.0; extra == "all"
 Requires-Dist: requests>=2.28.2; extra == "all"
+Requires-Dist: curl_cffi; extra == "all"
 Provides-Extra: aiomysql
 Requires-Dist: aiomysql>=0.1.1; extra == "aiomysql"
 Requires-Dist: cryptography; extra == "aiomysql"
@@ -52,6 +53,8 @@ Provides-Extra: playwright
 Requires-Dist: playwright>=1.31.1; extra == "playwright"
 Provides-Extra: pyhttpx
 Requires-Dist: pyhttpx>=2.10.4; extra == "pyhttpx"
+Provides-Extra: curl-cffi
+Requires-Dist: curl_cffi>=0.6.1; extra == "curl-cffi"
 Provides-Extra: requests
 Requires-Dist: requests>=2.28.2; extra == "requests"
 Provides-Extra: pg

{aio-scrapy-2.0.10 → aio-scrapy-2.1.0}/aio_scrapy.egg-info/SOURCES.txt

@@ -38,6 +38,7 @@ aioscrapy/core/scraper.py
 aioscrapy/core/downloader/__init__.py
 aioscrapy/core/downloader/handlers/__init__.py
 aioscrapy/core/downloader/handlers/aiohttp.py
+aioscrapy/core/downloader/handlers/curl_cffi.py
 aioscrapy/core/downloader/handlers/httpx.py
 aioscrapy/core/downloader/handlers/pyhttpx.py
 aioscrapy/core/downloader/handlers/requests.py

{aio-scrapy-2.0.10 → aio-scrapy-2.1.0}/aio_scrapy.egg-info/requires.txt

@@ -26,6 +26,10 @@ asyncpg>=0.27.0
 XlsxWriter>=3.1.2
 pillow>=9.4.0
 requests>=2.28.2
+curl_cffi
+
+[curl_cffi]
+curl_cffi>=0.6.1
 
 [execl]
 XlsxWriter>=3.1.2

aio-scrapy-2.1.0/aioscrapy/VERSION

@@ -0,0 +1 @@
+2.1.0

aio-scrapy-2.1.0/aioscrapy/core/downloader/handlers/curl_cffi.py

@@ -0,0 +1,61 @@
+import ssl
+
+from curl_cffi.requests import AsyncSession
+
+from aioscrapy import Request
+from aioscrapy.core.downloader.handlers import BaseDownloadHandler
+from aioscrapy.http import HtmlResponse
+from aioscrapy.settings import Settings
+from aioscrapy.utils.log import logger
+
+
+class CurlCffiDownloadHandler(BaseDownloadHandler):
+
+    def __init__(self, settings):
+        self.settings: Settings = settings
+        self.httpx_client_session_args: dict = self.settings.get('CURL_CFFI_CLIENT_SESSION_ARGS', {})
+        self.verify_ssl: bool = self.settings.get("VERIFY_SSL", True)
+
+    @classmethod
+    def from_settings(cls, settings: Settings):
+        return cls(settings)
+
+    async def download_request(self, request: Request, _) -> HtmlResponse:
+        kwargs = {
+            'timeout': self.settings.get('DOWNLOAD_TIMEOUT'),
+            'cookies': dict(request.cookies),
+            'verify': request.meta.get('verify_ssl', self.verify_ssl),
+            'allow_redirects': self.settings.getbool('REDIRECT_ENABLED', True) if request.meta.get(
+                'dont_redirect') is None else request.meta.get('dont_redirect'),
+            'impersonate': request.meta.get('impersonate'),
+        }
+        post_data = request.body or None
+        if isinstance(post_data, dict):
+            kwargs['json'] = post_data
+        else:
+            kwargs['data'] = post_data
+
+        headers = request.headers or self.settings.get('DEFAULT_REQUEST_HEADERS')
+        kwargs['headers'] = headers
+
+        proxy = request.meta.get("proxy")
+        if proxy:
+            kwargs["proxies"] = {'http': proxy, 'https': proxy}
+            logger.debug(f"use proxy {proxy}: {request.url}")
+
+        session_args = self.httpx_client_session_args.copy()
+
+        async with AsyncSession(**session_args) as session:
+            response = await session.request(request.method, request.url, **kwargs)
+
+        return HtmlResponse(
+            str(response.url),
+            status=response.status_code,
+            headers=response.headers,
+            body=response.text,
+            cookies=dict(response.cookies),
+            encoding=response.encoding
+        )
+
+    async def close(self):
+        pass

{aio-scrapy-2.0.10 → aio-scrapy-2.1.0}/aioscrapy/core/engine.py

@@ -42,8 +42,6 @@ class ExecutionEngine(object):
         self.signals = crawler.signals
         self.logformatter = crawler.logformatter
 
-        self.enqueue_cache_num = self.settings.getint("ENQUEUE_CACHE_NUM")
-        self.enqueue_cache: Queue = Queue(self.enqueue_cache_num)
         self.slot: Optional[Slot] = None
         self.spider: Optional[Spider] = None
         self.downloader: Optional[DownloaderTV] = None
@@ -53,7 +51,6 @@ class ExecutionEngine(object):
         self.running: bool = False
         self.unlock: bool = True
         self.finish: bool = False
-        self.enqueue_unlock: bool = True
 
     async def start(
             self,
@@ -70,7 +67,6 @@ class ExecutionEngine(object):
         while not self.finish:
             self.running and await self._next_request()
             await asyncio.sleep(1)
-            self.enqueue_cache_num != 1 and create_task(self._crawl())
             self.running and await self._spider_idle(self.spider)
 
     async def stop(self, reason: str = 'shutdown') -> None:
@@ -81,7 +77,6 @@ class ExecutionEngine(object):
 
         while not self.is_idle():
             await asyncio.sleep(0.2)
-            self.enqueue_cache_num != 1 and create_task(self._crawl())
         await self.close_spider(self.spider, reason=reason)
         await self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
         self.finish = True
@@ -212,27 +207,8 @@ class ExecutionEngine(object):
         return True
 
     async def crawl(self, request: Request) -> None:
-        if self.enqueue_cache_num == 1:
-            await self.scheduler.enqueue_request(request)
-            create_task(self._next_request())
-        else:
-            await self.enqueue_cache.put(request)
-
-    async def _crawl(self) -> None:
-        if not self.enqueue_unlock:
-            return
-        self.enqueue_unlock = False
-        requests = []
-        for _ in range(self.enqueue_cache.qsize()):
-            try:
-                request = self.enqueue_cache.get_nowait()
-                requests.append(request)
-            except QueueEmpty:
-                break
-        if requests:
-            await call_helper(self.scheduler.enqueue_request_batch, requests)
-            create_task(self._next_request())
-        self.enqueue_unlock = True
+        await self.scheduler.enqueue_request(request)
+        # create_task(self._next_request())
 
     async def close_spider(self, spider: Spider, reason: str = 'cancelled') -> None:
         """Close (cancel) spider and clear all its outstanding requests"""
@@ -276,7 +252,6 @@ class ExecutionEngine(object):
         # method of 'has_pending_requests' has IO, so method of 'is_idle' execute twice
         if self.is_idle() \
                 and self.slot.start_requests is None \
-                and self.enqueue_unlock and self.enqueue_cache.empty() \
                 and not await self.scheduler.has_pending_requests() \
                 and self.is_idle():
             await self.stop(reason='finished')

{aio-scrapy-2.0.10 → aio-scrapy-2.1.0}/aioscrapy/core/scheduler.py

@@ -31,7 +31,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
     @classmethod
     async def from_crawler(cls, crawler: "aioscrapy.Crawler") -> "BaseScheduler":
         """
-        Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument.
+        Factory method which receives the current :class:`~aioscrapy.crawler.Crawler` object as argument.
         """
         return cls()
 
@@ -103,20 +103,27 @@ class Scheduler(BaseScheduler):
             queue: AbsQueue,
             spider: aioscrapy.Spider,
             stats=Optional[StatsCollector],
-            persist: bool = True
+            persist: bool = True,
+            cache_queue: Optional[AbsQueue] = None
     ):
+
         self.queue = queue
+        self.cache_queue = cache_queue
         self.spider = spider
         self.stats = stats
         self.persist = persist
 
     @classmethod
     async def from_crawler(cls: Type[SchedulerTV], crawler: "aioscrapy.Crawler") -> SchedulerTV:
+        cache_queue = None
+        if crawler.settings.getbool('USE_SCHEDULER_QUEUE_CACHE', False):
+            cache_queue = await load_instance('aioscrapy.queue.memory.SpiderPriorityQueue', spider=crawler.spider)
         instance = cls(
             await load_instance(crawler.settings['SCHEDULER_QUEUE_CLASS'], spider=crawler.spider),
             crawler.spider,
             stats=crawler.stats,
-            persist=crawler.settings.getbool('SCHEDULER_PERSIST', True)
+            persist=crawler.settings.getbool('SCHEDULER_PERSIST', True),
+            cache_queue=cache_queue
         )
 
         if crawler.settings.getbool('SCHEDULER_FLUSH_ON_START', False):
@@ -128,8 +135,20 @@ class Scheduler(BaseScheduler):
         return instance
 
     async def close(self, reason: str) -> None:
+
         if not self.persist:
             await self.flush()
+            return
+
+        # 如果持久化，将缓存中的任务放回到redis等分布式队列中
+        if self.cache_queue is not None:
+            while True:
+                temp = []
+                async for request in self.cache_queue.pop(2000):
+                    temp.append(request)
+                temp and await self.queue.push_batch(temp)
+                if len(temp) < 2000:
+                    break
 
     async def flush(self) -> None:
         await call_helper(self.queue.clear)
@@ -141,16 +160,38 @@ class Scheduler(BaseScheduler):
         return True
 
     async def enqueue_request(self, request: aioscrapy.Request) -> bool:
-        await call_helper(self.queue.push, request)
+        """
+        如果启用了缓存队列(USE_SCHEDULER_QUEUE_CACHE)，则优先将任务放到缓存队列中
+        """
+        if self.cache_queue is not None:
+            await call_helper(self.cache_queue.push, request)
+        else:
+            await call_helper(self.queue.push, request)
         if self.stats:
             self.stats.inc_value(self.queue.inc_key, spider=self.spider)
         return True
 
     async def next_request(self, count: int = 1) -> Optional[aioscrapy.Request]:
+        """
+        如果启用了缓存队列(USE_SCHEDULER_QUEUE_CACHE)，则优先从缓存队列中获取任务，然后从redis等分布式队列中获取任务
+        """
+        flag = False
+        if self.cache_queue is not None:
+            async for request in self.cache_queue.pop(count):
+                if request and self.stats:
+                    self.stats.inc_value(self.queue.inc_key, spider=self.spider)
+                yield request
+                flag = True
+
+        if flag:
+            return
+
         async for request in self.queue.pop(count):
             if request and self.stats:
                 self.stats.inc_value(self.queue.inc_key, spider=self.spider)
             yield request
 
+
     async def has_pending_requests(self) -> bool:
-        return await call_helper(self.queue.len) > 0
+        return await call_helper(self.queue.len) if self.cache_queue is None \
+            else (await call_helper(self.queue.len) + await call_helper(self.cache_queue.len)) > 0

{aio-scrapy-2.0.10 → aio-scrapy-2.1.0}/aioscrapy/crawler.py

@@ -234,9 +234,12 @@ class CrawlerProcess(CrawlerRunner):
         finally:
             await self.recycle_db_connect()
 
-    def start(self) -> None:
+    def start(self, use_windows_selector_eventLoop: bool = False) -> None:
         if sys.platform.startswith('win'):
-            asyncio.set_event_loop(asyncio.windows_events.ProactorEventLoop())
+            if use_windows_selector_eventLoop:
+                asyncio.set_event_loop_policy(asyncio.windows_events.WindowsSelectorEventLoopPolicy())
+            else:
+                asyncio.set_event_loop(asyncio.windows_events.ProactorEventLoop())
         else:
             try:
                 import uvloop

{aio-scrapy-2.0.10 → aio-scrapy-2.1.0}/aioscrapy/libs/downloader/retry.py

@@ -61,6 +61,14 @@ try:
 except ImportError:
     pass
 
+
+try:
+    from curl_cffi.curl import CurlError
+
+    NEED_RETRY_ERROR += (CurlError,)
+except ImportError:
+    pass
+
 from aioscrapy.exceptions import NotConfigured
 from aioscrapy.http.request import Request
 from aioscrapy.spiders import Spider

{aio-scrapy-2.0.10 → aio-scrapy-2.1.0}/aioscrapy/libs/pipelines/__init__.py

@@ -8,15 +8,13 @@ class SqlFormat:
 
     @staticmethod
     def pg_insert(table: str, fields: list, *args) -> str:
-        fields = ','.join(fields)
         placeholder = ','.join([f'${i + 1}' for i in range(len(fields))])
-        return f'''INSERT INTO {table} ({fields}) VALUES ({placeholder})'''
+        return f'''INSERT INTO {table} ({",".join(fields)}) VALUES ({placeholder})'''
 
     @staticmethod
     def pg_ignore_insert(table: str, fields: list, *args) -> str:
         placeholder = ','.join([f'${i + 1}' for i in range(len(fields))])
-        fields = ','.join(fields)
-        return f'INSERT INTO {table} ({fields}) VALUES ({placeholder}) ON CONFLICT DO NOTHING'
+        return f'''INSERT INTO {table} ({",".join(fields)}) VALUES ({placeholder}) ON CONFLICT DO NOTHING'''
 
     @staticmethod
     def pg_update_insert(table: str, fields: list, update_fields: list, on_conflict: str, *args) -> str:
@@ -25,8 +23,7 @@ class SqlFormat:
         if not update_fields:
             update_fields = fields
         update_fields = ','.join([f"{key} = excluded.{key}" for key in update_fields])
-        fields = ','.join(fields)
-        return f'INSERT INTO {table} ({fields}) VALUES ({placeholder}) ON CONFLICT({on_conflict}) DO UPDATE SET {update_fields}'
+        return f'''INSERT INTO {table} ({",".join(fields)}) VALUES ({placeholder}) ON CONFLICT({on_conflict}) DO UPDATE SET {update_fields}'''
 
     @staticmethod
     def mysql_insert(table: str, fields: list, *args) -> str:

{aio-scrapy-2.0.10 → aio-scrapy-2.1.0}/aioscrapy/settings/default_settings.py

@@ -72,6 +72,10 @@ DOWNLOAD_HANDLERS_MAP = {
         'http': 'aioscrapy.core.downloader.handlers.playwright.PlaywrightHandler',
         'https': 'aioscrapy.core.downloader.handlers.playwright.PlaywrightHandler',
     },
+    'curl_cffi': {
+        'http': 'aioscrapy.core.downloader.handlers.curl_cffi.CurlCffiDownloadHandler',
+        'https': 'aioscrapy.core.downloader.handlers.curl_cffi.CurlCffiDownloadHandler',
+    },
 }
 
 DOWNLOAD_TIMEOUT = 180  # 3mins

{aio-scrapy-2.0.10 → aio-scrapy-2.1.0}/aioscrapy/spiders/__init__.py

@@ -106,7 +106,7 @@ class Spider(object):
     __repr__ = __str__
 
     @classmethod
-    def start(cls, setting_path=None):
+    def start(cls, setting_path=None, use_windows_selector_eventLoop: bool = False):
         from aioscrapy.crawler import CrawlerProcess
         from aioscrapy.utils.project import get_project_settings
 
@@ -115,7 +115,7 @@ class Spider(object):
             settings.setmodule(setting_path)
         cp = CrawlerProcess(settings)
         cp.crawl(cls)
-        cp.start()
+        cp.start(use_windows_selector_eventLoop)
 
     def spider_idle(self):
         if not self.close_on_idle:

{aio-scrapy-2.0.10 → aio-scrapy-2.1.0}/setup.py

@@ -19,7 +19,7 @@ extras_require = {
     "all": [
         "aiomysql>=0.1.1", "httpx[http2]>=0.23.0", "aio-pika>=8.1.1",
         "cryptography", "motor>=3.1.1", "pyhttpx>=2.10.1", "asyncpg>=0.27.0",
-        "XlsxWriter>=3.1.2", "pillow>=9.4.0", "requests>=2.28.2"
+        "XlsxWriter>=3.1.2", "pillow>=9.4.0", "requests>=2.28.2", "curl_cffi"
     ],
     "aiomysql": ["aiomysql>=0.1.1", "cryptography"],
     "httpx": ["httpx[http2]>=0.23.0"],
@@ -27,6 +27,7 @@ extras_require = {
     "mongo": ["motor>=3.1.1"],
     "playwright": ["playwright>=1.31.1"],
     "pyhttpx": ["pyhttpx>=2.10.4"],
+    "curl_cffi": ["curl_cffi>=0.6.1"],
     "requests": ["requests>=2.28.2"],
     "pg": ["asyncpg>=0.27.0"],
     "execl": ["XlsxWriter>=3.1.2", "pillow>=9.4.0"],

aio-scrapy-2.0.10/aioscrapy/VERSION

@@ -1 +0,0 @@
-2.0.10