aio-scrapy 2.1.4__py3-none-any.whl → 2.1.6__py3-none-any.whl

aioscrapy/core/downloader/handlers/playwright/__init__.py

@@ -1,115 +0,0 @@
-from functools import wraps
-
-try:
-    from playwright._impl._errors import Error
-except ImportError:
-    from playwright._impl._api_types import Error
-
-from playwright.async_api._generated import Response as EventResponse
-
-from aioscrapy import Request, Spider
-from aioscrapy.core.downloader.handlers import BaseDownloadHandler
-from aioscrapy.core.downloader.handlers.playwright.driverpool import WebDriverPool
-from aioscrapy.core.downloader.handlers.playwright.webdriver import PlaywrightDriver
-from aioscrapy.exceptions import DownloadError
-from aioscrapy.http import PlaywrightResponse
-from aioscrapy.settings import Settings
-from aioscrapy.utils.tools import call_helper
-
-
-class PlaywrightHandler(BaseDownloadHandler):
-    def __init__(self, settings: Settings):
-        self.settings = settings
-        playwright_client_args = settings.getdict('PLAYWRIGHT_CLIENT_ARGS')
-        use_pool = settings.getbool('PLAYWRIGHT_USE_POOL', True)
-        self.wait_until = playwright_client_args.get('wait_until', 'domcontentloaded')
-        self.url_regexes = playwright_client_args.pop('url_regexes', [])
-        pool_size = playwright_client_args.pop('pool_size', settings.getint("CONCURRENT_REQUESTS", 1))
-        self._webdriver_pool = WebDriverPool(use_pool=use_pool, pool_size=pool_size, driver_cls=PlaywrightDriver, **playwright_client_args)
-
-    @classmethod
-    def from_settings(cls, settings: Settings):
-        return cls(settings)
-
-    async def download_request(self, request: Request, spider: Spider) -> PlaywrightResponse:
-        try:
-            return await self._download_request(request, spider)
-        except Error as e:
-            raise DownloadError(real_error=e) from e
-
-    async def _download_request(self, request: Request, spider) -> PlaywrightResponse:
-        cookies = dict(request.cookies)
-        timeout = request.meta.get('download_timeout', 30) * 1000
-        user_agent = request.headers.get("User-Agent")
-        proxy: str = request.meta.get("proxy")
-        url = request.url
-
-        cache_response = {}
-
-        # 为了获取监听事件中的响应结果
-        def on_event_wrap_handler(func):
-            @wraps(func)
-            async def inner(response):
-                ret = await func(response)
-                if ret:
-                    cache_response[ret[0]] = ret[1]
-
-            return inner
-
-        kwargs = dict()
-        if proxy:
-            kwargs['proxy'] = proxy
-        if user_agent:
-            kwargs['user_agent'] = user_agent
-
-        driver: PlaywrightDriver = await self._webdriver_pool.get(**kwargs)
-
-        # 移除所有的事件监听事件后 重新添加
-        driver.page._events = dict()
-        for name in dir(spider):
-            if not name.startswith('on_event_'):
-                continue
-            driver.page.on(name.replace('on_event_', ''), on_event_wrap_handler(getattr(spider, name)))
-
-        try:
-            if cookies:
-                driver.url = url
-                await driver.set_cookies(cookies)
-            await driver.page.goto(url, wait_until=request.meta.get('wait_until', self.wait_until), timeout=timeout)
-
-            if process_action_fn := getattr(spider, 'process_action', None):
-                action_result = await call_helper(process_action_fn, driver)
-                if action_result:
-                    cache_response[action_result[0]] = action_result[1]
-
-            for cache_key in list(cache_response.keys()):
-                if isinstance(cache_response[cache_key], EventResponse):
-                    cache_ret = cache_response[cache_key]
-                    cache_response[cache_key] = PlaywrightResponse(
-                        url=cache_ret.url,
-                        request=request,
-                        intercept_request=dict(
-                            url=cache_ret.request.url,
-                            headers=cache_ret.request.headers,
-                            data=cache_ret.request.post_data,
-                        ),
-                        headers=cache_ret.headers,
-                        body=await cache_ret.body(),
-                        status=cache_ret.status,
-                    )
-
-            return PlaywrightResponse(
-                url=driver.page.url,
-                status=200,
-                text=await driver.page.content(),
-                cookies=await driver.get_cookies(),
-                cache_response=cache_response,
-                driver=driver,
-                driver_pool=self._webdriver_pool
-            )
-        except Exception as e:
-            await self._webdriver_pool.remove(driver)
-            raise e
-
-    async def close(self):
-        await self._webdriver_pool.close()

aioscrapy/core/downloader/handlers/playwright/driverpool.py

@@ -1,59 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from asyncio import Lock
-from asyncio.queues import Queue
-
-from aioscrapy.utils.tools import singleton
-
-
-@singleton
-class WebDriverPool:
-    def __init__(
-            self, use_pool=True, pool_size=5, driver_cls=None, **kwargs
-    ):
-        self.use_pool = use_pool
-        self.pool_size = pool_size
-        self.driver_cls = driver_cls
-        self.kwargs = kwargs
-
-        self.queue = Queue(maxsize=pool_size)
-        self.lock = Lock()
-        self.driver_count = 0
-
-    @property
-    def is_full(self):
-        return self.driver_count >= self.pool_size
-
-    async def create_driver(self, **args):
-        kwargs = self.kwargs.copy()
-        kwargs.update(args)
-        driver = self.driver_cls(**kwargs)
-        await driver.setup()
-        return driver
-
-    async def get(self, **kwargs):
-        async with self.lock:
-            if not self.use_pool:
-                return await self.create_driver(**kwargs)
-            if not self.is_full:
-                driver = await self.create_driver(**kwargs)
-                self.driver_count += 1
-            else:
-                driver = await self.queue.get()
-        return driver
-
-    async def release(self, driver):
-        if not self.use_pool:
-            await driver.quit()
-            return
-        await self.queue.put(driver)
-
-    async def remove(self, driver):
-        await driver.quit()
-        self.driver_count -= 1
-
-    async def close(self):
-        while not self.queue.empty():
-            driver = await self.queue.get()
-            await driver.quit()
-            self.driver_count -= 1

aioscrapy/core/downloader/handlers/playwright/webdriver.py

@@ -1,96 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import os
-from typing import Dict, Optional, Tuple
-
-try:
-    from typing import Literal  # python >= 3.8
-except ImportError:  # python <3.8
-    from typing_extensions import Literal
-
-from urllib.parse import urlparse, urlunparse
-
-from playwright.async_api import Page, BrowserContext, ViewportSize, ProxySettings
-from playwright.async_api import Playwright, Browser
-from playwright.async_api import async_playwright
-
-
-class PlaywrightDriver:
-    def __init__(
-            self,
-            *,
-            driver_type: Literal["chromium", "firefox", "webkit"] = "chromium",
-            proxy: Optional[str] = None,
-            browser_args: Optional[Dict] = None,
-            context_args: Optional[Dict] = None,
-            window_size: Optional[Tuple[int, int]] = None,
-            user_agent: str = None,
-            **kwargs
-    ):
-
-        self.driver_type = driver_type
-        self.proxy = proxy and self.format_context_proxy(proxy)
-        self.viewport = window_size and ViewportSize(width=window_size[0], height=window_size[1])
-        self.browser_args = browser_args or {}
-        self.context_args = context_args or {}
-        self.user_agent = user_agent
-
-        self.driver: Optional[Playwright] = None
-        self.browser: Optional[Browser] = None
-        self.context: Optional[BrowserContext] = None
-        self.page: Optional[Page] = None
-        self.url = None
-
-    async def setup(self):
-        browser_args = self.browser_args.copy()
-        context_args = self.context_args.copy()
-        if browser_args.get('args') is None:
-            browser_args.update({'args': ["--no-sandbox"]})
-
-        if context_args.get("storage_state") is not None:
-            storage_state_path = context_args.get("storage_state")
-            os.makedirs(os.path.dirname(storage_state_path), exist_ok=True)
-
-        if self.proxy:
-            browser_args.update({'proxy': self.proxy})
-            context_args.update({'proxy': self.proxy})
-        if self.viewport:
-            context_args.update({"viewport": self.viewport})
-            context_args.update({"screen": self.viewport})
-        if self.user_agent:
-            context_args.update({'user_agent': self.user_agent})
-
-        self.driver = await async_playwright().start()
-        self.browser: Browser = await getattr(self.driver, self.driver_type).launch(**browser_args)
-        self.context = await self.browser.new_context(**context_args)
-        self.page = await self.context.new_page()
-
-    @staticmethod
-    def format_context_proxy(proxy) -> ProxySettings:
-        parsed_url = urlparse(proxy)
-        return ProxySettings(
-            server=urlunparse(parsed_url._replace(netloc=parsed_url.netloc.split('@')[-1])),
-            username=parsed_url.username,
-            password=parsed_url.password,
-        )
-
-    async def quit(self):
-        await self.page.close()
-        try:
-            await self.context.close()
-        except:
-            pass
-        finally:
-            await self.browser.close()
-            await self.driver.stop()
-
-    async def get_cookies(self):
-        return {
-            cookie["name"]: cookie["value"]
-            for cookie in await self.page.context.cookies()
-        }
-
-    async def set_cookies(self, cookies: dict):
-        await self.page.context.add_cookies([
-            {"name": key, "value": value, "url": self.url or self.page.url} for key, value in cookies.items()
-        ])

aioscrapy/http/response/playwright.py

@@ -1,36 +0,0 @@
-from typing import Optional, Any
-
-from aioscrapy.http.response.text import TextResponse
-
-
-class PlaywrightResponse(TextResponse):
-    def __init__(
-            self,
-            *args,
-            text: str = '',
-            cache_response: Optional[dict] = None,
-            driver: Optional["PlaywrightDriver"] = None,
-            driver_pool: Optional["WebDriverPool"] = None,
-            intercept_request: Optional[dict] = None,
-            **kwargs
-    ):
-        self.driver = driver
-        self.driver_pool = driver_pool
-        self._text = text
-        self.cache_response = cache_response or {}
-        self.intercept_request = intercept_request
-        super().__init__(*args, **kwargs)
-
-    async def release(self):
-        self.driver_pool and self.driver and await self.driver_pool.release(self.driver)
-
-    @property
-    def text(self):
-        return self._text or super().text
-
-    @text.setter
-    def text(self, text):
-        self._text = text
-
-    def get_response(self, key) -> Any:
-        return self.cache_response.get(key)

aioscrapy/libs/pipelines/execl.py

@@ -1,169 +0,0 @@
-import asyncio
-import math
-from io import BytesIO
-from typing import Tuple, Optional
-
-import requests
-import xlsxwriter
-from PIL import Image, ImageFile
-
-from aioscrapy.utils.log import logger
-
-try:
-    resample = Image.LANCZOS
-except:
-    resample = Image.ANTIALIAS
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-
-
-class ExeclSinkMixin:
-    ws_cache = {}
-    wb_cache = {}
-    fields_cache = {}
-    y_cache = {}
-
-    @staticmethod
-    async def deal_img(url: str, img_size: Optional[Tuple[int, int]]) -> Optional[BytesIO]:
-        if url.startswith('//'):
-            url = 'https:' + url
-        try:
-            img_bytes = requests.get(url).content
-        except Exception as e:
-            logger.error(f"download img error: {e}")
-            return
-        im = Image.open(BytesIO(img_bytes))
-        im_format = im.format
-        if img_size:
-            temp = max(im.size[0] / img_size[0], im.size[1] / img_size[1])
-            img_size = (math.ceil(im.size[0] / temp), math.ceil(im.size[1] / temp))
-            im = im.resize(img_size, resample).convert('P')
-        result = BytesIO()
-        im.save(result, format=im_format)
-        return result
-
-    async def save_item(
-            self,
-            item: dict,
-            *,
-            filename: Optional[str] = None,
-            date_fields: Optional[list] = None,
-            date_format: str = 'yyyy-mm-dd HH:MM:SS',
-            img_fields: Optional[list] = None,
-            img_size: Optional[Tuple[int, int]] = None,
-            **options
-    ):
-        assert filename is not None, "请传入filename参数"
-        if '.xlsx' not in filename:
-            filename = filename + '.xlsx'
-        try:
-            wb, ws, fields, y = self._get_write_class(filename, item, **options)
-            bold_format_1 = wb.add_format({'align': 'left', 'border': 1, 'valign': 'vcenter'})
-            bold_format_2 = wb.add_format({'align': 'left', 'border': 1, 'valign': 'vcenter', 'fg_color': '#D0D3D4'})
-            for x, field in enumerate(fields):
-                if x % 2 == 0:
-                    bold_format = bold_format_1
-                else:
-                    bold_format = bold_format_2
-                if date_fields is not None and field in date_fields:
-                    ws.write_datetime(y, x, item.get(field), wb.add_format({'num_format': date_format}))
-
-                elif img_fields is not None and field in img_fields:
-                    img_size and ws.set_column_pixels(x, x, width=math.ceil(img_size[0]))
-                    url = item.get(field)
-                    img_bytes = await self.deal_img(url, img_size)
-                    if img_bytes is None or ws.insert_image(y, x, '', {'image_data': img_bytes}) == -1:
-                        ws.write(y, x, url, bold_format)
-                else:
-                    ws.write(y, x, item.get(field), bold_format)
-            if img_size is not None:
-                ws.set_column_pixels(0, len(fields), width=math.ceil(img_size[0]))
-                ws.set_row_pixels(y, height=math.ceil(img_size[1]))
-        except Exception as e:
-            logger.exception(f'Save Execl Error, filename:{filename}, item:{item}, errMsg: {e}')
-
-    def _get_write_class(self, filename, item, sheet='sheet1', **options):
-        filename_sheet = filename + sheet
-        if self.ws_cache.get(filename_sheet) is None:
-            if self.wb_cache.get(filename) is None:
-                logger.info(f'Create Execl: {filename}')
-                wb = xlsxwriter.Workbook(filename, options=options)
-                self.wb_cache[filename] = wb
-            else:
-                wb = self.wb_cache[filename]
-            ws = wb.add_worksheet(sheet)
-            bold_format = wb.add_format(
-                {'bold': True, 'font_size': 12, 'border': 1, 'align': 'center', 'valign': 'vcenter'})
-            fields = list(item.keys())
-            ws.write_row('A1', fields, cell_format=bold_format)
-            ws.set_row(0, height=30)
-            self.fields_cache[filename_sheet] = fields
-            self.ws_cache[filename_sheet] = ws
-            self.y_cache[filename_sheet] = 0
-        self.y_cache[filename_sheet] += 1
-        return self.wb_cache[filename], \
-            self.ws_cache[filename_sheet], \
-            self.fields_cache[filename_sheet], \
-            self.y_cache[filename_sheet]
-
-    def close_execl(self, filename=None):
-        if filename not in self.wb_cache:
-            return
-
-        logger.info(f'Closing Execl: {filename}')
-        if wb := self.wb_cache.pop(filename):
-            wb.close()
-        for filename_sheet in list(self.ws_cache.keys()):
-            if not filename_sheet.startswith(filename):
-                continue
-            self.ws_cache.pop(filename_sheet, None)
-            self.y_cache.pop(filename_sheet, None)
-            self.fields_cache.pop(filename_sheet, None)
-
-    def close(self):
-        for filename in list(self.wb_cache.keys()):
-            self.close_execl(filename)
-
-
-class ExeclPipeline(ExeclSinkMixin):
-    def __init__(self, settings):
-        self.lock = asyncio.Lock()
-
-    @classmethod
-    def from_settings(cls, settings):
-        return cls(settings)
-
-    async def process_item(self, item, spider):
-        execl_kw: Optional[dict] = item.pop('__execl__', None)
-        if not execl_kw:
-            logger.warning(f"item Missing key __execl__, not stored")
-            return item
-
-        execl_kw.setdefault('filename', spider.name)
-        async with self.lock:
-            await self.save_item(item, **execl_kw)
-
-    async def close_spider(self, spider):
-        self.close()
-
-
-if __name__ == '__main__':
-    class TestSpider:
-        name = 'TestSpider'
-
-
-    async def test():
-        p = ExeclPipeline({})
-        await p.process_item({
-            'title': 'tttt',
-            'img': '//www.baidu.com/img/flexible/logo/pc/result.png',
-            '__execl__': {
-                'sheet': 'sheet1',
-                # 'filename': 'test',
-                # 'img_fields': ['img'],
-                # 'img_size': (100, 500)
-            }
-        }, TestSpider())
-        await p.close_spider(None)
-
-
-    asyncio.run(test())