aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aio_scrapy-2.1.4.dist-info/METADATA

@@ -1,239 +0,0 @@
-Metadata-Version: 2.1
-Name: aio-scrapy
-Version: 2.1.4
-Summary: A high-level Web Crawling and Web Scraping framework based on Asyncio
-Home-page: https://github.com/conlin-huang/aio-scrapy.git
-Author: conlin
-Author-email: 995018884@qq.com
-License: MIT
-Keywords: aio-scrapy,scrapy,aioscrapy,scrapy redis,asyncio,spider
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Intended Audience :: Developers
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Topic :: Internet :: WWW/HTTP
-Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: aiohttp
-Requires-Dist: ujson
-Requires-Dist: w3lib >=1.17.0
-Requires-Dist: parsel >=1.5.0
-Requires-Dist: PyDispatcher >=2.0.5
-Requires-Dist: zope.interface >=5.1.0
-Requires-Dist: redis >=4.3.1
-Requires-Dist: aiomultiprocess >=0.9.0
-Requires-Dist: loguru >=0.7.0
-Requires-Dist: anyio >=3.6.2
-Provides-Extra: aio-pika
-Requires-Dist: aio-pika >=8.1.1 ; extra == 'aio-pika'
-Provides-Extra: aiomysql
-Requires-Dist: aiomysql >=0.1.1 ; extra == 'aiomysql'
-Requires-Dist: cryptography ; extra == 'aiomysql'
-Provides-Extra: all
-Requires-Dist: aiomysql >=0.1.1 ; extra == 'all'
-Requires-Dist: httpx[http2] >=0.23.0 ; extra == 'all'
-Requires-Dist: aio-pika >=8.1.1 ; extra == 'all'
-Requires-Dist: cryptography ; extra == 'all'
-Requires-Dist: motor >=2.1.0 ; extra == 'all'
-Requires-Dist: pyhttpx >=2.10.1 ; extra == 'all'
-Requires-Dist: asyncpg >=0.27.0 ; extra == 'all'
-Requires-Dist: XlsxWriter >=3.1.2 ; extra == 'all'
-Requires-Dist: pillow >=9.4.0 ; extra == 'all'
-Requires-Dist: requests >=2.28.2 ; extra == 'all'
-Requires-Dist: curl-cffi ; extra == 'all'
-Provides-Extra: curl_cffi
-Requires-Dist: curl-cffi >=0.6.1 ; extra == 'curl_cffi'
-Provides-Extra: execl
-Requires-Dist: XlsxWriter >=3.1.2 ; extra == 'execl'
-Requires-Dist: pillow >=9.4.0 ; extra == 'execl'
-Provides-Extra: httpx
-Requires-Dist: httpx[http2] >=0.23.0 ; extra == 'httpx'
-Provides-Extra: mongo
-Requires-Dist: motor >=2.1.0 ; extra == 'mongo'
-Provides-Extra: pg
-Requires-Dist: asyncpg >=0.27.0 ; extra == 'pg'
-Provides-Extra: playwright
-Requires-Dist: playwright >=1.31.1 ; extra == 'playwright'
-Provides-Extra: pyhttpx
-Requires-Dist: pyhttpx >=2.10.4 ; extra == 'pyhttpx'
-Provides-Extra: requests
-Requires-Dist: requests >=2.28.2 ; extra == 'requests'
-
-<!--
-![aio-scrapy](./doc/images/aio-scrapy.png)
--->
-### aio-scrapy
-
-An asyncio + aiolibs crawler  imitate scrapy framework
-
-English | [中文](./doc/README_ZH.md)
-
-### Overview
-- aio-scrapy framework is base on opensource project Scrapy & scrapy_redis.
-- aio-scrapy implements compatibility with scrapyd.
-- aio-scrapy implements redis queue and rabbitmq queue.
-- aio-scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages.
-- Distributed crawling/scraping.
-### Requirements
-
-- Python 3.9+
-- Works on Linux, Windows, macOS, BSD
-
-### Install
-
-The quick way:
-
-```shell
-# Install the latest aio-scrapy
-pip install git+https://github.com/ConlinH/aio-scrapy
-
-# default
-pip install aio-scrapy
-
-# Install all dependencies 
-pip install aio-scrapy[all]
-
-# When you need to use mysql/httpx/rabbitmq/mongo
-pip install aio-scrapy[aiomysql,httpx,aio-pika,mongo]
-```
-
-### Usage
-
-#### create project spider:
-
-```shell
-aioscrapy startproject project_quotes
-```
-
-```
-cd project_quotes
-aioscrapy genspider quotes 
-```
-
-quotes.py
-
-```python
-from aioscrapy.spiders import Spider
-
-
-class QuotesMemorySpider(Spider):
-    name = 'QuotesMemorySpider'
-
-    start_urls = ['https://quotes.toscrape.com']
-
-    async def parse(self, response):
-        for quote in response.css('div.quote'):
-            yield {
-                'author': quote.xpath('span/small/text()').get(),
-                'text': quote.css('span.text::text').get(),
-            }
-
-        next_page = response.css('li.next a::attr("href")').get()
-        if next_page is not None:
-            yield response.follow(next_page, self.parse)
-
-
-if __name__ == '__main__':
-    QuotesMemorySpider.start()
-
-```
-
-run the spider:
-
-```shell
-aioscrapy crawl quotes
-```
-
-#### create single script spider:
-
-```shell
-aioscrapy genspider single_quotes -t single
-```
-
-single_quotes.py:
-
-```python
-from aioscrapy.spiders import Spider
-
-
-class QuotesMemorySpider(Spider):
-    name = 'QuotesMemorySpider'
-    custom_settings = {
-        "USER_AGENT": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
-        'CLOSE_SPIDER_ON_IDLE': True,
-        # 'DOWNLOAD_DELAY': 3,
-        # 'RANDOMIZE_DOWNLOAD_DELAY': True,
-        # 'CONCURRENT_REQUESTS': 1,
-        # 'LOG_LEVEL': 'INFO'
-    }
-
-    start_urls = ['https://quotes.toscrape.com']
-
-    @staticmethod
-    async def process_request(request, spider):
-        """ request middleware """
-        pass
-
-    @staticmethod
-    async def process_response(request, response, spider):
-        """ response middleware """
-        return response
-
-    @staticmethod
-    async def process_exception(request, exception, spider):
-        """ exception middleware """
-        pass
-
-    async def parse(self, response):
-        for quote in response.css('div.quote'):
-            yield {
-                'author': quote.xpath('span/small/text()').get(),
-                'text': quote.css('span.text::text').get(),
-            }
-
-        next_page = response.css('li.next a::attr("href")').get()
-        if next_page is not None:
-            yield response.follow(next_page, self.parse)
-
-    async def process_item(self, item):
-        print(item)
-
-
-if __name__ == '__main__':
-    QuotesMemorySpider.start()
-
-```
-
-run the spider:
-
-```shell
-aioscrapy runspider quotes.py
-```
-
-
-### more commands:
-
-```shell
-aioscrapy -h
-```
-
-#### [more example](./example)
-
-### Documentation
-[doc](./doc/documentation.md)
-
-### Ready
-
-please submit your sugguestion to owner by issue
-
-## Thanks
-
-[aiohttp](https://github.com/aio-libs/aiohttp/)
-
-[scrapy](https://github.com/scrapy/scrapy)
-

aio_scrapy-2.1.4.dist-info/RECORD

@@ -1,133 +0,0 @@
-aioscrapy/VERSION,sha256=Z7BD32ByWBAJNuaSjQBe7W_NFoIm-41YzXKXt3z-bUI,5
-aioscrapy/__init__.py,sha256=esJeH66Mz9WV7XbotvZEjNn49jc589YZ_L2DKoD0JvA,858
-aioscrapy/__main__.py,sha256=rvTdJ0cQwbi29aucPj3jJRpccx5SBzvRcV7qvxvX2NQ,80
-aioscrapy/cmdline.py,sha256=1qhNg2Edl-Obmf2re2K4V8pJG7ubGfZZCzcHdKtdE_s,5159
-aioscrapy/crawler.py,sha256=6-ptivIjIGKdojOlZqXV0hV3x1Gont81tOC5u5JqIME,10330
-aioscrapy/exceptions.py,sha256=B1UZUXF_dZNJ5b1wltDemijK8iCNpH-EF2sOooH9AsA,2628
-aioscrapy/link.py,sha256=fXMqsHvYEzsuYi-sNDcElS7jV6Lusq0tjPkPUGOlyZw,1867
-aioscrapy/logformatter.py,sha256=y3etd28ACbpTbcGprJ_cQ086gxQY3k_QX_yxYFoF1AU,3028
-aioscrapy/process.py,sha256=uFkj2wzaBu0Vs3pGFKdJ4R-0Gn7hROX6EU-B5zddnyQ,1603
-aioscrapy/serializer.py,sha256=eTMSMHQZJidJ3-LzjmfJd-pkkG29G3NnrtYUx-eKB8w,734
-aioscrapy/signalmanager.py,sha256=S_dTEa8Y75x8SSHEQem014o-OmxwvfwaMH9w34lCkTc,2402
-aioscrapy/signals.py,sha256=rXkPNS6c9rIv_TzKU3lqYGQc8UA3CbqSsHL9O1U6MTE,610
-aioscrapy/spiderloader.py,sha256=XJ-lUUtf9Xy7172VzlvPzLes-2ym3GRw7lAdJ7sankc,3426
-aioscrapy/statscollectors.py,sha256=L1ykz0zCqt7Qw85NyhWH8zfwVOtnQOWMI0065YP03CQ,2036
-aioscrapy/commands/__init__.py,sha256=hNPzpaYpnX3mxLjQgKM211xAuMgjhxMwjpDRCrvFVwI,4769
-aioscrapy/commands/crawl.py,sha256=fKrQEdR-YJuiXpDIX3R0QdhOgjDwDye7C6ilrj77C1A,1020
-aioscrapy/commands/genspider.py,sha256=381W8-pjbASGXvYjrrgYWorJD9BjURa25oolDm0QI1Q,5426
-aioscrapy/commands/list.py,sha256=j62GHgngGRdW-AGKCJBvrc_h26yj0JmEFBkXTvjRoi8,346
-aioscrapy/commands/runspider.py,sha256=albdJM3SUbWzhiOK1zsWklp3HqGSPRISTshg33EnZ_c,2067
-aioscrapy/commands/settings.py,sha256=sc0rwwfBQNySKX8uV3iJqv3i7SelFwNcrlHYxDupKOg,1798
-aioscrapy/commands/startproject.py,sha256=Rcc7JkN75Jp2t2aZIxBzPsWbLXChNAUSByDhcW_6Ig8,4001
-aioscrapy/commands/version.py,sha256=yqqTMlZkkiQhtbU9w_IqUWLMOAjqYlv24friEkPRQYM,485
-aioscrapy/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aioscrapy/core/engine.py,sha256=h02-K2lQqlCxvNIlURgPpnhHCbyiJRIWrFJt5Ys7vZY,9843
-aioscrapy/core/scheduler.py,sha256=czCx5oHknXuHadpISTfoEMSKXXrlwJTmLTUQtHdtaTc,7407
-aioscrapy/core/scraper.py,sha256=eS_qEX_Q9fXZnK8Ou1wDtBJhRKk9JoUSnbn4c04u1cA,10750
-aioscrapy/core/downloader/__init__.py,sha256=OCg21payZbmQPcZ1_Wrhhgos7angRB-w9qya3CxrmSU,10040
-aioscrapy/core/downloader/handlers/__init__.py,sha256=KwID2qt3dhFvvBIF3CJnPR4w4a4_qz4uKaXgQI5b59o,3199
-aioscrapy/core/downloader/handlers/aiohttp.py,sha256=qt8Wys8NrbLatBqEob5lzjKmy_C2Nl9XxLyA2npdv6A,4277
-aioscrapy/core/downloader/handlers/curl_cffi.py,sha256=hYlUf2BzS6GrWaPKLJhuqj8fxOt9AANBoeAp9vx7-KU,2590
-aioscrapy/core/downloader/handlers/httpx.py,sha256=HqidohwQr8G7GNhrS1v23rYmD2dzNW69bObcO0X_6Qs,3398
-aioscrapy/core/downloader/handlers/pyhttpx.py,sha256=djxaNoYVD6TJSN3UruviQBx8_oLVtCn4d__qwsoxRJA,2573
-aioscrapy/core/downloader/handlers/requests.py,sha256=RdRi6Izj-jvWa_8T8axW9EzcUfMqfman7eFKTFjOro4,2328
-aioscrapy/core/downloader/handlers/playwright/__init__.py,sha256=xjPNlvM0zzR8lOIzgJeDnq1p0x1VHGhGiyMQmihdkmM,4676
-aioscrapy/core/downloader/handlers/playwright/driverpool.py,sha256=IlkYB8TlSuDq7-sTLlGvtAsFMalNvzpTJR7wEMYe2jE,1595
-aioscrapy/core/downloader/handlers/playwright/webdriver.py,sha256=QFtAT--2Ea_Gg4x1EhMidyOwQjbqljUl4sKGB_hAA00,3530
-aioscrapy/db/__init__.py,sha256=ISBXM_-cCf5CgTLc3i_emLxV163-ZAbgttkQiRxokD0,2456
-aioscrapy/db/absmanager.py,sha256=6vlPcjDHOtZCHePiUYPe6ezRnM-TB4XLhmuw7APaWDk,1162
-aioscrapy/db/aiomongo.py,sha256=t4JpRPBBisF7_rz02Kp6AejrphLvLWg5rF-yYLIe2MI,3071
-aioscrapy/db/aiomysql.py,sha256=-xCLfeH7RzvghY1jqREAb_Qnz9q_dVjxoHGfz7sCqbU,3799
-aioscrapy/db/aiopg.py,sha256=WG4s_2X0b8LQHbZpoIrwZeuGHNolKj-SvmvAZQlCk00,3213
-aioscrapy/db/aiorabbitmq.py,sha256=tNKl4Kx7KM7H_lOj8xfeA0uD8PuBTVzySApTEn5TyAE,5583
-aioscrapy/db/aioredis.py,sha256=UOoTRTQUvghnq29bVL8v1HvksMXYOzHaS8Btgbpn0bY,2966
-aioscrapy/dupefilters/__init__.py,sha256=Dx_CN-wBYiatLj3cXbK0f5d66CTjzzTeex6565L1EsA,1765
-aioscrapy/dupefilters/disk.py,sha256=EMgxeC2a6aYCGKgp4QOs5xwHp33LUsOZ8pliKBTFx1c,1551
-aioscrapy/dupefilters/redis.py,sha256=YUUsnRQ326PjdM_FUWAWjgOWw92KwswAGnM-FmN8pv0,7559
-aioscrapy/http/__init__.py,sha256=yeQTT5W1iwr6dKznTS5d9vnx2hsB47i9roPM57wQp_0,597
-aioscrapy/http/headers.py,sha256=H-RJ6KqOsFFFAXORfvoyz3V-ud0I8TAj5Jt5fAACcLc,1573
-aioscrapy/http/request/__init__.py,sha256=PFoFU3ncTN-gj6Rx01rjVa_744Qfv3EH29mooW6JX9U,7121
-aioscrapy/http/request/form.py,sha256=pqexRCmGlTiE9FofKa3_OpoK6yoJlf9qk9hxNveDoaw,1382
-aioscrapy/http/request/json_request.py,sha256=qtWdF5UhuGy0QmyLKWm9Y86veISkZS5HyOJiavtEhP4,2051
-aioscrapy/http/response/__init__.py,sha256=6DyNQNVgpJ-Awd8Cu9eCTXF_nmQF87WoAPPj5aBrigA,6731
-aioscrapy/http/response/html.py,sha256=PiBguPg2uE1lY_WNjs419EqbZv38pGsZ5JqTUYQWeMU,303
-aioscrapy/http/response/playwright.py,sha256=0DX0L04hGv1zvtXvdYS7_UQFKbnbx2r7Sj-ZKirxtr0,1051
-aioscrapy/http/response/text.py,sha256=VxpjTIGtnVuUepa3GdsyX5kskqnFVCjmZmfonSynbhM,9844
-aioscrapy/http/response/xml.py,sha256=5iXsTuxFW1eBLrzYggjyf-FTDZwWZwY1ATZceIa5yxM,300
-aioscrapy/libs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aioscrapy/libs/downloader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aioscrapy/libs/downloader/defaultheaders.py,sha256=tg_ULA0Y-41bZKG607mowFJQGVfnZ45LdR044DsjA_A,563
-aioscrapy/libs/downloader/downloadtimeout.py,sha256=hNh3OEj7rC0ceQrv_yrhR5lb5AvfxJ6cspj3qsQWj4o,704
-aioscrapy/libs/downloader/ja3fingerprint.py,sha256=DgTw74GXC_Bp94eD_bwoG6A_DphUHTt7bH4glBNXyV8,1058
-aioscrapy/libs/downloader/retry.py,sha256=uKU8XuPya8Co6vTTTgs1-rFtMsZreSwz0Zo1ErgaA6I,4482
-aioscrapy/libs/downloader/stats.py,sha256=FlkS8Zm4j3SBjHb6caXwq08HvvZ37VKORGCAjlA2U38,1376
-aioscrapy/libs/downloader/useragent.py,sha256=E5x5dk9AxsSCGDDICJlTXwWXRkqAibWgesqG0VhAG8M,743
-aioscrapy/libs/extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aioscrapy/libs/extensions/closespider.py,sha256=LRv4G7RHwFeT9WeH6s0spO909BHvxdtlvHWLN-dlyno,2734
-aioscrapy/libs/extensions/corestats.py,sha256=WCZ4nnk6LUP7AdGx9mnuVm96iWMxHozxdNPr41r8HmQ,1820
-aioscrapy/libs/extensions/logstats.py,sha256=wSLbN9tmsw5I1FBxHjLfIdQo85fxJI7TmOefispaxc4,1844
-aioscrapy/libs/extensions/metric.py,sha256=cx9UnSdj6akzrPe_uwWHh_QKTNzD82VRrEjiiHOoAuc,5479
-aioscrapy/libs/extensions/throttle.py,sha256=yos2D3XZgH40G52kltMKv5_GeAK4MqpRwTu6FCErUh0,3512
-aioscrapy/libs/pipelines/__init__.py,sha256=XW5Ur6bhvGLo-w-tdUeIB4jkFpZxqUU9mbajfAAztb0,5642
-aioscrapy/libs/pipelines/csv.py,sha256=-PEZOt-3ndF0ePO7EnqjEqeCYMJR9wHv3XcpSq6QswI,2454
-aioscrapy/libs/pipelines/execl.py,sha256=a8sfgQCHUc0MIja9cPP4TZ6ghfkxYZuAzLDIK4_nQuo,6284
-aioscrapy/libs/pipelines/mongo.py,sha256=B3dhvspxc4lmPh2noqARYV-rFuHfivdSfZ7ZlPKnk7c,2323
-aioscrapy/libs/pipelines/mysql.py,sha256=gN4DnyuXTQvDvy9Gu-v8F6sT8l7GZEa45AD0d-Ckv8s,1022
-aioscrapy/libs/pipelines/pg.py,sha256=la-SflXtGFw4IQYlOn75Brw2IfmtOUcCh0gUSz_Jg-0,990
-aioscrapy/libs/spider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aioscrapy/libs/spider/depth.py,sha256=IZoLS-JAqLl79JrQQf5p8aubN4c82g44M2iUkhRhXjM,1938
-aioscrapy/libs/spider/httperror.py,sha256=v8_zuTpzyU7nszo-97TEE5iniKPdrvKrPKx6bKDtQyI,1857
-aioscrapy/libs/spider/offsite.py,sha256=QQx15cozRPsjIvYobeiIgTeMwgtgDk5yMgf6Y6qT9s4,2941
-aioscrapy/libs/spider/referer.py,sha256=vdbk_uQI4o0MncCHQauVX7xcxERWSqw5ZkaJVVrYcr0,13768
-aioscrapy/libs/spider/urllength.py,sha256=D2sEt-LeibHYfkO1tUJxENeyI1PrwIb7HrLIBNU3HdA,1151
-aioscrapy/middleware/__init__.py,sha256=Ej7BAmxdHDqaytQT1XA3V73CpBedkZBQsRni0fduuck,396
-aioscrapy/middleware/absmanager.py,sha256=dwm4nEyRIjP_V6BJqI-tl9NZF0HwsafDNeWyqZCwSrA,3410
-aioscrapy/middleware/downloader.py,sha256=SSeHXtECbGE5r6o6EXLo9ZIrPX6wqkFTwEoN4F6qHgM,3199
-aioscrapy/middleware/extension.py,sha256=LS6Q9VFYVa9oemS7DiKsyehhz8244alj4Jtlnl0f4DY,420
-aioscrapy/middleware/itempipeline.py,sha256=_Htfrs3vzIUfajTOzQLdGaX_4xTFtSSoFzyhZJMyZw8,674
-aioscrapy/middleware/spider.py,sha256=QvV5dchOlskBn1sXKd5dj6s9zSZmlT6LibydCjfmYjU,6361
-aioscrapy/proxy/__init__.py,sha256=Cwua97Z-ezxtDSlud7mCOAV-iExY7RX_8O1oP5PS__k,1807
-aioscrapy/proxy/redis.py,sha256=LFfnnkihf6Wq1-HeRzPLVEiy5e5wxJbMY7htU-C_Pd8,2711
-aioscrapy/queue/__init__.py,sha256=MKHNOgcZRAjFHAxoKLujvsBCkB_Ne1-gz5DqNbdTYNA,2037
-aioscrapy/queue/memory.py,sha256=_Dvkd-HXwdR8B-wsEPlHNWbAgaMH9M_UGZhF21LbnHA,3140
-aioscrapy/queue/rabbitmq.py,sha256=rE1GCoIGxoaV4KAi_9Umt623A00FaHHIFV5ZH_nypzY,2516
-aioscrapy/queue/redis.py,sha256=KU31ZNciLI9xxZDxsDhtOPLtmkxZQlRPOx_1z8afdwY,4788
-aioscrapy/scrapyd/__init__.py,sha256=Ey14RVLUP7typ2XqP8RWcUum2fuFyigdhuhBBiEheIo,68
-aioscrapy/scrapyd/runner.py,sha256=tewEkdNTMrBoredCbhmdrswSrF-GWsU3MLgC__ntnzQ,1777
-aioscrapy/settings/__init__.py,sha256=GuiVhezV8U2J1B-WJwSvxxeH_1YWYD_Wighr9owC4HU,15781
-aioscrapy/settings/default_settings.py,sha256=PrUOFYNnPIS8eCdqvRylMLBK-4tT-2MYuU6Nn8dQrx0,5639
-aioscrapy/spiders/__init__.py,sha256=oM_FzqWa46P6cjzarOO1cfDTQD2AuIPgaWZrmdMcuTI,4085
-aioscrapy/templates/project/aioscrapy.cfg,sha256=_nRHP5wtPnZaBi7wCmjWv5BgUu5NYFJZhvCTRVSipyM,112
-aioscrapy/templates/project/module/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aioscrapy/templates/project/module/middlewares.py.tmpl,sha256=0eEf2LC0vYcWPH82HNqieYSORyUuIo3Bgl5t-neRAJ4,3469
-aioscrapy/templates/project/module/pipelines.py.tmpl,sha256=-MYA7MFAffH8FTG1VGAkhIJLQ6MOGMrWVDO7cI9jw9A,164
-aioscrapy/templates/project/module/settings.py.tmpl,sha256=AO2jmyokUhuhFqxMvsMihPgSY4ZrldsMs-BuOEVfvQY,1421
-aioscrapy/templates/project/module/spiders/__init__.py,sha256=Zg1uss1vaNjvld9s9Ccua50SxVZwpFTPwqpBHoCrWdU,164
-aioscrapy/templates/spiders/basic.tmpl,sha256=oO1vh7-TZLjvpwdrYC49TGe-A6Kulc8UIG4Sa0QhDfI,375
-aioscrapy/templates/spiders/single.tmpl,sha256=Ptmo_uFDGEffvpEMyxec7sxIyBbP05x0Grhn5u6lZbQ,1011
-aioscrapy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-aioscrapy/utils/conf.py,sha256=NkSmKjOE7xVvrAWQu4ne3jOzNGucgZdWHPhGbpz8dPU,7208
-aioscrapy/utils/curl.py,sha256=I8eZWFNgvyUiJ2YS9-s3HltGNVG8XMMU0HPhlMxuxdA,3295
-aioscrapy/utils/decorators.py,sha256=gMQArNxF9QQc1bENA0IqDchAjqmfWvHGKOyUdjXdg6A,794
-aioscrapy/utils/deprecate.py,sha256=STy55Q8kZI8q8CQUfxK4QQVu1Rs4En3rlhzWc7p7T00,5467
-aioscrapy/utils/httpobj.py,sha256=ytec7IZzsQY_GwR___051hdbOWs1ZM6S57HwcNiu2es,708
-aioscrapy/utils/log.py,sha256=NRDivw8w21J77qEUeqqLdC4sgdIKaj2UAP6lDvWGotM,1697
-aioscrapy/utils/misc.py,sha256=9NOssEl7CP_c6R9skxyXwmz4bd-nZ_gkw6F0EybeLTQ,3509
-aioscrapy/utils/ossignal.py,sha256=jAsCIKu17KV45-9dZwEkFJHF31Y13KP_zxY0x49j1jo,896
-aioscrapy/utils/project.py,sha256=cT98HaR5JaNmm-Y1UzSuzXj6B5S7GlmMshUfMhjpjJY,2905
-aioscrapy/utils/python.py,sha256=38oD-OSjeGb3XZFJn3bt74PwGbejnBfLWC5-lkUL0g8,4462
-aioscrapy/utils/reqser.py,sha256=qjrYut6KtvGpLLd-HDM0cncNzWCtXgpH6NyERu_5A9g,487
-aioscrapy/utils/request.py,sha256=bkFaLDeebAOp7pF-7vta9LKOB2OR2s7V9jVKfA-XlqA,2418
-aioscrapy/utils/response.py,sha256=UPR1wTTAYZkLGiiIs28kJLhlF7WPrgLuW31l9LZuYKM,1341
-aioscrapy/utils/signal.py,sha256=bkqRgGMqQ82dly_D4tDe_0pHBbc9QUxBJqSsH9RSQf0,2282
-aioscrapy/utils/spider.py,sha256=Usq3UlCaDUvXGp0ojFt39UPKFrR2rbInlJc_q0Xk7Qc,610
-aioscrapy/utils/template.py,sha256=HR97X4lpv2WuqhuPfzTgaBN66fYnzHVpP6zQ5IoTwcI,833
-aioscrapy/utils/tools.py,sha256=WJowViZB8XEs2CFqjVvbqXK3H5Uvf4BgWgBD_RcHMaM,2319
-aioscrapy/utils/trackref.py,sha256=0nIpelT1d5WYxALl8SGA8vHNYsh-jS0Z2lwVEAhwx8E,2019
-aioscrapy/utils/url.py,sha256=8W8tAhU7lgfPOfzKp3ejJGEcLj1i_PnA_53Jv5LpxiY,5464
-aio_scrapy-2.1.4.dist-info/LICENSE,sha256=L-UoAEM3fQSjKA7FVWxQM7gwSCbeue6gZRAnpRS_UCo,1088
-aio_scrapy-2.1.4.dist-info/METADATA,sha256=9R1Kw1XYe7yrLJ3h4SeiV69tPphz8sTaIf2Sizfh0GU,6536
-aio_scrapy-2.1.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-aio_scrapy-2.1.4.dist-info/entry_points.txt,sha256=WWhoVHZvqhW8a5uFg97K0EP_GjG3uuCIFLkyqDICgaw,56
-aio_scrapy-2.1.4.dist-info/top_level.txt,sha256=8l08KyMt22wfX_5BmhrGH0PgwZdzZIPq-hBUa1GNir4,10
-aio_scrapy-2.1.4.dist-info/RECORD,,

aioscrapy/core/downloader/handlers/playwright/__init__.py

@@ -1,115 +0,0 @@
-from functools import wraps
-
-try:
-    from playwright._impl._errors import Error
-except ImportError:
-    from playwright._impl._api_types import Error
-
-from playwright.async_api._generated import Response as EventResponse
-
-from aioscrapy import Request, Spider
-from aioscrapy.core.downloader.handlers import BaseDownloadHandler
-from aioscrapy.core.downloader.handlers.playwright.driverpool import WebDriverPool
-from aioscrapy.core.downloader.handlers.playwright.webdriver import PlaywrightDriver
-from aioscrapy.exceptions import DownloadError
-from aioscrapy.http import PlaywrightResponse
-from aioscrapy.settings import Settings
-from aioscrapy.utils.tools import call_helper
-
-
-class PlaywrightHandler(BaseDownloadHandler):
-    def __init__(self, settings: Settings):
-        self.settings = settings
-        playwright_client_args = settings.getdict('PLAYWRIGHT_CLIENT_ARGS')
-        use_pool = settings.getbool('PLAYWRIGHT_USE_POOL', True)
-        self.wait_until = playwright_client_args.get('wait_until', 'domcontentloaded')
-        self.url_regexes = playwright_client_args.pop('url_regexes', [])
-        pool_size = playwright_client_args.pop('pool_size', settings.getint("CONCURRENT_REQUESTS", 1))
-        self._webdriver_pool = WebDriverPool(use_pool=use_pool, pool_size=pool_size, driver_cls=PlaywrightDriver, **playwright_client_args)
-
-    @classmethod
-    def from_settings(cls, settings: Settings):
-        return cls(settings)
-
-    async def download_request(self, request: Request, spider: Spider) -> PlaywrightResponse:
-        try:
-            return await self._download_request(request, spider)
-        except Error as e:
-            raise DownloadError(real_error=e) from e
-
-    async def _download_request(self, request: Request, spider) -> PlaywrightResponse:
-        cookies = dict(request.cookies)
-        timeout = request.meta.get('download_timeout', 30) * 1000
-        user_agent = request.headers.get("User-Agent")
-        proxy: str = request.meta.get("proxy")
-        url = request.url
-
-        cache_response = {}
-
-        # 为了获取监听事件中的响应结果
-        def on_event_wrap_handler(func):
-            @wraps(func)
-            async def inner(response):
-                ret = await func(response)
-                if ret:
-                    cache_response[ret[0]] = ret[1]
-
-            return inner
-
-        kwargs = dict()
-        if proxy:
-            kwargs['proxy'] = proxy
-        if user_agent:
-            kwargs['user_agent'] = user_agent
-
-        driver: PlaywrightDriver = await self._webdriver_pool.get(**kwargs)
-
-        # 移除所有的事件监听事件后 重新添加
-        driver.page._events = dict()
-        for name in dir(spider):
-            if not name.startswith('on_event_'):
-                continue
-            driver.page.on(name.replace('on_event_', ''), on_event_wrap_handler(getattr(spider, name)))
-
-        try:
-            if cookies:
-                driver.url = url
-                await driver.set_cookies(cookies)
-            await driver.page.goto(url, wait_until=request.meta.get('wait_until', self.wait_until), timeout=timeout)
-
-            if process_action_fn := getattr(spider, 'process_action', None):
-                action_result = await call_helper(process_action_fn, driver)
-                if action_result:
-                    cache_response[action_result[0]] = action_result[1]
-
-            for cache_key in list(cache_response.keys()):
-                if isinstance(cache_response[cache_key], EventResponse):
-                    cache_ret = cache_response[cache_key]
-                    cache_response[cache_key] = PlaywrightResponse(
-                        url=cache_ret.url,
-                        request=request,
-                        intercept_request=dict(
-                            url=cache_ret.request.url,
-                            headers=cache_ret.request.headers,
-                            data=cache_ret.request.post_data,
-                        ),
-                        headers=cache_ret.headers,
-                        body=await cache_ret.body(),
-                        status=cache_ret.status,
-                    )
-
-            return PlaywrightResponse(
-                url=driver.page.url,
-                status=200,
-                text=await driver.page.content(),
-                cookies=await driver.get_cookies(),
-                cache_response=cache_response,
-                driver=driver,
-                driver_pool=self._webdriver_pool
-            )
-        except Exception as e:
-            await self._webdriver_pool.remove(driver)
-            raise e
-
-    async def close(self):
-        await self._webdriver_pool.close()

aioscrapy/core/downloader/handlers/playwright/driverpool.py

@@ -1,59 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from asyncio import Lock
-from asyncio.queues import Queue
-
-from aioscrapy.utils.tools import singleton
-
-
-@singleton
-class WebDriverPool:
-    def __init__(
-            self, use_pool=True, pool_size=5, driver_cls=None, **kwargs
-    ):
-        self.use_pool = use_pool
-        self.pool_size = pool_size
-        self.driver_cls = driver_cls
-        self.kwargs = kwargs
-
-        self.queue = Queue(maxsize=pool_size)
-        self.lock = Lock()
-        self.driver_count = 0
-
-    @property
-    def is_full(self):
-        return self.driver_count >= self.pool_size
-
-    async def create_driver(self, **args):
-        kwargs = self.kwargs.copy()
-        kwargs.update(args)
-        driver = self.driver_cls(**kwargs)
-        await driver.setup()
-        return driver
-
-    async def get(self, **kwargs):
-        async with self.lock:
-            if not self.use_pool:
-                return await self.create_driver(**kwargs)
-            if not self.is_full:
-                driver = await self.create_driver(**kwargs)
-                self.driver_count += 1
-            else:
-                driver = await self.queue.get()
-        return driver
-
-    async def release(self, driver):
-        if not self.use_pool:
-            await driver.quit()
-            return
-        await self.queue.put(driver)
-
-    async def remove(self, driver):
-        await driver.quit()
-        self.driver_count -= 1
-
-    async def close(self):
-        while not self.queue.empty():
-            driver = await self.queue.get()
-            await driver.quit()
-            self.driver_count -= 1

aioscrapy/core/downloader/handlers/playwright/webdriver.py

@@ -1,96 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-from typing import Dict, Optional, Tuple
-
-try:
-    from typing import Literal  # python >= 3.8
-except ImportError:  # python <3.8
-    from typing_extensions import Literal
-
-from urllib.parse import urlparse, urlunparse
-
-from playwright.async_api import Page, BrowserContext, ViewportSize, ProxySettings
-from playwright.async_api import Playwright, Browser
-from playwright.async_api import async_playwright
-
-
-class PlaywrightDriver:
-    def __init__(
-            self,
-            *,
-            driver_type: Literal["chromium", "firefox", "webkit"] = "chromium",
-            proxy: Optional[str] = None,
-            browser_args: Optional[Dict] = None,
-            context_args: Optional[Dict] = None,
-            window_size: Optional[Tuple[int, int]] = None,
-            user_agent: str = None,
-            **kwargs
-    ):
-
-        self.driver_type = driver_type
-        self.proxy = proxy and self.format_context_proxy(proxy)
-        self.viewport = window_size and ViewportSize(width=window_size[0], height=window_size[1])
-        self.browser_args = browser_args or {}
-        self.context_args = context_args or {}
-        self.user_agent = user_agent
-
-        self.driver: Optional[Playwright] = None
-        self.browser: Optional[Browser] = None
-        self.context: Optional[BrowserContext] = None
-        self.page: Optional[Page] = None
-        self.url = None
-
-    async def setup(self):
-        browser_args = self.browser_args.copy()
-        context_args = self.context_args.copy()
-        if browser_args.get('args') is None:
-            browser_args.update({'args': ["--no-sandbox"]})
-
-        if context_args.get("storage_state") is not None:
-            storage_state_path = context_args.get("storage_state")
-            os.makedirs(os.path.dirname(storage_state_path), exist_ok=True)
-
-        if self.proxy:
-            browser_args.update({'proxy': self.proxy})
-            context_args.update({'proxy': self.proxy})
-        if self.viewport:
-            context_args.update({"viewport": self.viewport})
-            context_args.update({"screen": self.viewport})
-        if self.user_agent:
-            context_args.update({'user_agent': self.user_agent})
-
-        self.driver = await async_playwright().start()
-        self.browser: Browser = await getattr(self.driver, self.driver_type).launch(**browser_args)
-        self.context = await self.browser.new_context(**context_args)
-        self.page = await self.context.new_page()
-
-    @staticmethod
-    def format_context_proxy(proxy) -> ProxySettings:
-        parsed_url = urlparse(proxy)
-        return ProxySettings(
-            server=urlunparse(parsed_url._replace(netloc=parsed_url.netloc.split('@')[-1])),
-            username=parsed_url.username,
-            password=parsed_url.password,
-        )
-
-    async def quit(self):
-        await self.page.close()
-        try:
-            await self.context.close()
-        except:
-            pass
-        finally:
-            await self.browser.close()
-            await self.driver.stop()
-
-    async def get_cookies(self):
-        return {
-            cookie["name"]: cookie["value"]
-            for cookie in await self.page.context.cookies()
-        }
-
-    async def set_cookies(self, cookies: dict):
-        await self.page.context.add_cookies([
-            {"name": key, "value": value, "url": self.url or self.page.url} for key, value in cookies.items()
-        ])

aioscrapy/http/response/playwright.py

@@ -1,36 +0,0 @@
-from typing import Optional, Any
-
-from aioscrapy.http.response.text import TextResponse
-
-
-class PlaywrightResponse(TextResponse):
-    def __init__(
-            self,
-            *args,
-            text: str = '',
-            cache_response: Optional[dict] = None,
-            driver: Optional["PlaywrightDriver"] = None,
-            driver_pool: Optional["WebDriverPool"] = None,
-            intercept_request: Optional[dict] = None,
-            **kwargs
-    ):
-        self.driver = driver
-        self.driver_pool = driver_pool
-        self._text = text
-        self.cache_response = cache_response or {}
-        self.intercept_request = intercept_request
-        super().__init__(*args, **kwargs)
-
-    async def release(self):
-        self.driver_pool and self.driver and await self.driver_pool.release(self.driver)
-
-    @property
-    def text(self):
-        return self._text or super().text
-
-    @text.setter
-    def text(self, text):
-        self._text = text
-
-    def get_response(self, key) -> Any:
-        return self.cache_response.get(key)