crawlo 1.0.0__tar.gz

crawlo-1.0.0/LICENSE

@@ -0,0 +1,23 @@
+MIT License
+
+Modifications:
+
+Copyright (c) 2020 crawl-coder <2251018029@qq.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crawlo-1.0.0/MANIFEST.in

@@ -0,0 +1,11 @@
+include README.md
+include LICENSE
+
+include crawlo/requirements.txt
+include crawlo/VERSION
+
+recursive-include crawlo/utils/js *
+recursive-include crawlo/templates *
+recursive-include tests *
+
+global-exclude __pycache__ *.py[cod]

crawlo-1.0.0/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.4
+Name: crawlo
+Version: 1.0.0
+Summary: feapder是一款支持异步的python爬虫框架
+Home-page: https://github.com/crawl-coder/Crawlo.git
+Author: crawl-coder
+Author-email: crawlo@qq.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiohttp>=3.12.6
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: DBUtils>=2.0
+Requires-Dist: parsel>=1.10.0
+Requires-Dist: pymysql>=1.1.1
+Requires-Dist: ujson>=5.10.0
+Provides-Extra: render
+Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
+Requires-Dist: playwright; extra == "render"
+Requires-Dist: selenium>=3.141.0; extra == "render"
+Provides-Extra: all
+Requires-Dist: bitarray>=1.5.3; extra == "all"
+Requires-Dist: PyExecJS>=1.5.1; extra == "all"
+Requires-Dist: pymongo>=3.10.1; extra == "all"
+Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
+Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: selenium>=3.141.0; extra == "all"
+Dynamic: license-file
+
+# Crawlo
+异步通用爬虫框架

crawlo-1.0.0/README.md

@@ -0,0 +1,2 @@
+# Crawlo
+异步通用爬虫框架

crawlo-1.0.0/crawlo/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.network.request import Request
+from crawlo.network.response import Response
+from crawlo.items.items import Item

crawlo-1.0.0/crawlo/__version__.py

@@ -0,0 +1,2 @@
+
+__version__ = "1.0.0"

crawlo-1.0.0/crawlo/core/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-

crawlo-1.0.0/crawlo/core/engine.py

@@ -0,0 +1,157 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Optional, Generator, Callable
+from inspect import iscoroutine, isgenerator, isasyncgen
+
+from crawlo import Request, Item
+from crawlo.spider import Spider
+from crawlo.core.scheduler import Scheduler
+from crawlo.core.processor import Processor
+from crawlo.utils.log import get_logger
+from crawlo.task_manager import TaskManager
+from crawlo.utils.project import load_class
+from crawlo.downloader import DownloaderBase
+from crawlo.utils.func_tools import transform
+from crawlo.exceptions import OutputError, TransformTypeError
+from crawlo.event import spider_opened, spider_error
+
+
+class Engine(object):
+
+    def __init__(self, crawler):
+        self.running = False
+        self.normal = True
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.spider: Optional[Spider] = None
+        self.downloader: Optional[DownloaderBase] = None
+        self.scheduler: Optional[Scheduler] = None
+        self.processor: Optional[Processor] = None
+        self.start_requests: Optional[Generator] = None
+        self.task_manager: Optional[TaskManager] = TaskManager(self.settings.get_int('CONCURRENCY'))
+
+        self.logger = get_logger(name=self.__class__.__name__)
+
+    def _get_downloader_cls(self):
+        downloader_cls = load_class(self.settings.get('DOWNLOADER'))
+        if not issubclass(downloader_cls, DownloaderBase):
+            raise TypeError(f'Downloader {downloader_cls.__name__} is not subclass of DownloaderBase.')
+        return downloader_cls
+
+    def engine_start(self):
+        self.running = True
+        self.logger.info(
+            f"Crawlo (version {self.settings.get_int('VERSION')}) started. "
+            f"(project name : {self.settings.get('PROJECT_NAME')})"
+        )
+
+    async def start_spider(self, spider):
+        self.spider = spider
+
+        self.scheduler = Scheduler(self.crawler)
+        if hasattr(self.scheduler, 'open'):
+            self.scheduler.open()
+
+        downloader_cls = self._get_downloader_cls()
+        self.downloader = downloader_cls(self.crawler)
+        if hasattr(self.downloader, 'open'):
+            self.downloader.open()
+
+        self.processor = Processor(self.crawler)
+        if hasattr(self.processor, 'open'):
+            self.processor.open()
+
+        self.start_requests = iter(spider.start_requests())
+        await self._open_spider()
+
+    async def crawl(self):
+        """
+        Crawl the spider
+        """
+        while self.running:
+            if request := await self._get_next_request():
+                await self._crawl(request)
+            try:
+                start_request = next(self.start_requests)
+            except StopIteration:
+                self.start_requests = None
+            except Exception as exp:
+                # 1、发去请求的request全部运行完毕
+                # 2、调度器是否空闲
+                # 3、下载器是否空闲
+                if not await self._exit():
+                    continue
+                self.running = False
+                if self.start_requests is not None:
+                    self.logger.error(f"启动请求时发生错误: {str(exp)}")
+            else:
+                # 请求入队
+                await self.enqueue_request(start_request)
+
+        if not self.running:
+            await self.close_spider()
+
+    async def _open_spider(self):
+        asyncio.create_task(self.crawler.subscriber.notify(spider_opened))
+        crawling = asyncio.create_task(self.crawl())
+        await crawling
+
+    async def _crawl(self, request):
+        # TODO 实现并发
+        async def crawl_task():
+            outputs = await self._fetch(request)
+            # TODO 处理output
+            if outputs:
+                await self._handle_spider_output(outputs)
+
+        # asyncio.create_task(crawl_task())
+        self.task_manager.create_task(crawl_task())
+
+    async def _fetch(self, request):
+        async def _successful(_response):
+            callback: Callable = request.callback or self.spider.parse
+            if _outputs := callback(_response):
+                if iscoroutine(_outputs):
+                    await _outputs
+                else:
+                    return transform(_outputs)
+
+        _response = await self.downloader.fetch(request)
+        if _response is None:
+            return None
+        output = await _successful(_response)
+        return output
+
+    async def enqueue_request(self, start_request):
+        await self._schedule_request(start_request)
+
+    async def _schedule_request(self, request):
+        # TODO 去重
+        await self.scheduler.enqueue_request(request)
+
+    async def _get_next_request(self):
+        return await self.scheduler.next_request()
+
+    async def _handle_spider_output(self, outputs):
+        async for spider_output in outputs:
+            if isinstance(spider_output, (Request, Item)):
+                await self.processor.enqueue(spider_output)
+            elif isinstance(spider_output, Exception):
+                asyncio.create_task(
+                    self.crawler.subscriber.notify(spider_error, spider_output, self.spider)
+                )
+                raise spider_output
+            else:
+                raise OutputError(f'{type(self.spider)} must return `Request` or `Item`.')
+
+    async def _exit(self):
+        if self.scheduler.idle() and self.downloader.idle() and self.task_manager.all_done() and self.processor.idle():
+            return True
+        return False
+
+    async def close_spider(self):
+        await asyncio.gather(*self.task_manager.current_task)
+        await self.downloader.close()
+        if self.normal:
+            await self.crawler.close()

crawlo-1.0.0/crawlo/core/processor.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from asyncio import Queue
+from typing import Union, Optional
+
+from crawlo import Request, Item
+from crawlo.pipelines.pipeline_manager import PipelineManager
+
+
+class Processor(object):
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.queue: Queue = Queue()
+        self.pipelines: Optional[PipelineManager] = None
+
+    def open(self):
+        self.pipelines = PipelineManager.create_instance(self.crawler)
+
+    async def process(self):
+        while not self.idle():
+            result = await self.queue.get()
+            if isinstance(result, Request):
+                await self.crawler.engine.enqueue_request(result)
+            else:
+                assert isinstance(result, Item)
+                await self._process_item(result)
+
+    async def _process_item(self, item):
+        await self.pipelines.process_item(item=item)
+
+    async def enqueue(self, output: Union[Request, Item]):
+        await self.queue.put(output)
+        await self.process()
+
+    def idle(self) -> bool:
+        return len(self) == 0
+
+    def __len__(self):
+        return self.queue.qsize()

crawlo-1.0.0/crawlo/core/scheduler.py

@@ -0,0 +1,35 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.event import request_scheduled
+from crawlo.utils.pqueue import SpiderPriorityQueue
+
+
+class Scheduler:
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.request_queue: Optional[SpiderPriorityQueue] = None
+
+        self.item_count = 0
+        self.response_count = 0
+        self.logger = get_logger(name=self.__class__.__name__, level=crawler.settings.get('LOG_LEVEL'))
+
+    def open(self):
+        self.request_queue = SpiderPriorityQueue()
+
+    async def next_request(self):
+        request = await self.request_queue.get()
+        return request
+
+    async def enqueue_request(self, request):
+        await self.request_queue.put(request)
+        asyncio.create_task(self.crawler.subscriber.notify(request_scheduled, request, self.crawler.spider))
+
+    def idle(self) -> bool:
+        return len(self) == 0
+
+    def __len__(self):
+        return self.request_queue.qsize()

crawlo-1.0.0/crawlo/crawler.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*
+import signal
+import asyncio
+from typing import Type, Final, Set, Optional
+
+from crawlo.spider import Spider
+from crawlo.core.engine import Engine
+from crawlo.subscriber import Subscriber
+
+from crawlo.utils.log import get_logger
+from crawlo.extension import ExtensionManager
+from crawlo.exceptions import SpiderTypeError
+from crawlo.utils.project import merge_settings
+from crawlo.stats_collector import StatsCollector
+from crawlo.event import spider_opened, spider_closed
+from crawlo.settings.setting_manager import SettingManager
+
+logger = get_logger(__name__)
+
+
+class Crawler:
+
+    def __init__(self, spider_cls, settings):
+        self.spider_cls = spider_cls
+        self.spider: Optional[Spider] = None
+        self.engine: Optional[Engine] = None
+        self.stats: Optional[StatsCollector] = None
+        self.subscriber: Optional[Subscriber] = None
+        self.extension: Optional[ExtensionManager] = None
+        self.settings: SettingManager = settings.copy()
+
+    async def crawl(self):
+        self.subscriber = self._create_subscriber()
+        self.spider = self._create_spider()
+        self.engine = self._create_engine()
+        self.stats = self._create_stats()
+        self.extension = self._create_extension()
+
+        await self.engine.start_spider(self.spider)
+
+    @staticmethod
+    def _create_subscriber():
+        return Subscriber()
+
+    def _create_spider(self) -> Spider:
+        spider = self.spider_cls.create_instance(self)
+        self._set_spider(spider)
+        return spider
+
+    def _create_engine(self) -> Engine:
+        engine = Engine(self)
+        engine.engine_start()
+        return engine
+
+    def _create_stats(self) -> StatsCollector:
+        stats = StatsCollector(self)
+        return stats
+
+    def _create_extension(self) -> ExtensionManager:
+        extension = ExtensionManager.create_instance(self)
+        return extension
+
+    def _set_spider(self, spider):
+        self.subscriber.subscribe(spider.spider_opened, event=spider_opened)
+        self.subscriber.subscribe(spider.spider_closed, event=spider_closed)
+        merge_settings(spider, self.settings)
+
+    async def close(self, reason='finished') -> None:
+        await asyncio.create_task(self.subscriber.notify(spider_closed))
+        self.stats.close_spider(spider_name=self.spider, reason=reason)
+
+
+class CrawlerProcess:
+
+    def __init__(self, settings=None):
+        self.crawlers: Final[Set] = set()
+        self._active_spiders: Final[Set] = set()
+        self.settings = settings
+
+        signal.signal(signal.SIGINT, self._shutdown)
+
+    async def crawl(self, spider: Type[Spider]):
+        crawler: Crawler = self._create_crawler(spider)
+        self.crawlers.add(crawler)
+        task = await self._crawl(crawler)
+        self._active_spiders.add(task)
+
+    @staticmethod
+    async def _crawl(crawler):
+        return asyncio.create_task(crawler.crawl())
+
+    async def start(self):
+        await asyncio.gather(*self._active_spiders)
+
+    def _create_crawler(self, spider_cls) -> Crawler:
+        if isinstance(spider_cls, str):
+            raise SpiderTypeError(f"{type(self)}.crawl args: String is not supported.")
+        crawler: Crawler = Crawler(spider_cls, self.settings)
+        return crawler
+
+    def _shutdown(self, _signum, _frame):
+        for crawler in self.crawlers:
+            crawler.engine.running = False
+            crawler.engine.normal = False
+            crawler.stats.close_spider(crawler.spider, 'Ctrl C')
+        logger.warning(f'spiders received: `Ctrl C` signal, closed.')

crawlo-1.0.0/crawlo/downloader/__init__.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from abc import abstractmethod, ABCMeta
+from typing_extensions import Self
+from typing import Final, Set, Optional
+from contextlib import asynccontextmanager
+
+from crawlo import Response, Request
+from crawlo.utils.log import get_logger
+from crawlo.middleware.middleware_manager import MiddlewareManager
+
+
+class ActivateRequestManager:
+
+    def __init__(self):
+        self._active: Final[Set] = set()
+
+    def add(self, request):
+        self._active.add(request)
+
+    def remove(self, request):
+        self._active.remove(request)
+
+    @asynccontextmanager
+    async def __call__(self, request):
+        try:
+            yield self.add(request)
+        finally:
+            self.remove(request)
+
+    def __len__(self):
+        return len(self._active)
+
+
+class DownloaderMeta(ABCMeta):
+    def __subclasscheck__(self, subclass):
+        required_methods = ('fetch', 'download', 'create_instance', 'close')
+        is_subclass = all(
+            hasattr(subclass, method) and callable(getattr(subclass, method, None)) for method in required_methods
+        )
+        return is_subclass
+
+
+class DownloaderBase(metaclass=DownloaderMeta):
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self._active = ActivateRequestManager()
+        self.middleware: Optional[MiddlewareManager] = None
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get("LOG_LEVEL"))
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs) -> Self:
+        return cls(*args, **kwargs)
+
+    def open(self) -> None:
+        self.logger.info(
+            f"{self.crawler.spider} <downloader class：{type(self).__name__}>"
+            f"<concurrency：{self.crawler.settings.get_int('CONCURRENCY')}>"
+        )
+        self.middleware = MiddlewareManager.create_instance(self.crawler)
+
+    async def fetch(self, request) -> Optional[Response]:
+        async with self._active(request):
+            response = await self.middleware.download(request)
+            return response
+
+    @abstractmethod
+    async def download(self, request: Request) -> Response:
+        pass
+
+    async def close(self) -> None:
+        pass
+
+    def idle(self) -> bool:
+        return len(self) == 0
+
+    def __len__(self) -> int:
+        return len(self._active)

crawlo-1.0.0/crawlo/downloader/aiohttp_downloader.py

@@ -0,0 +1,96 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Optional
+from aiohttp import ClientSession, TCPConnector, BaseConnector, ClientTimeout, ClientResponse, TraceConfig
+
+from crawlo import Response
+from crawlo.downloader import DownloaderBase
+
+
+class AioHttpDownloader(DownloaderBase):
+    def __init__(self, crawler):
+        super().__init__(crawler)
+        self.session: Optional[ClientSession] = None
+        self.connector: Optional[BaseConnector] = None
+        self._verify_ssl: Optional[bool] = None
+        self._timeout: Optional[ClientTimeout] = None
+        self._use_session: Optional[bool] = None
+        self.trace_config: Optional[TraceConfig] = None
+
+        self.request_method = {
+            "get": self._get,
+            "post": self._post
+        }
+
+    def open(self):
+        super().open()
+        self._timeout = ClientTimeout(total=self.crawler.settings.get_int("DOWNLOAD_TIMEOUT"))
+        self._verify_ssl = self.crawler.settings.get_bool("VERIFY_SSL")
+        self._use_session = self.crawler.settings.get_bool("USE_SESSION")
+        self.trace_config = TraceConfig()
+        self.trace_config.on_request_start.append(self.request_start)
+        if self._use_session:
+            self.connector = TCPConnector(verify_ssl=self._verify_ssl)
+            self.session = ClientSession(
+                connector=self.connector, timeout=self._timeout, trace_configs=[self.trace_config]
+            )
+
+    async def download(self, request) -> Optional[Response]:
+        try:
+            if self._use_session:
+                response = await self.send_request(self.session, request)
+                body = await response.content.read()
+            else:
+                connector = TCPConnector(verify_ssl=self._verify_ssl)
+                async with ClientSession(
+                        connector=connector, timeout=self._timeout, trace_configs=[self.trace_config]
+                ) as session:
+                    response = await self.send_request(session, request)
+                    body = await response.content.read()
+        except Exception as exp:
+            self.logger.error(f"Error downloading {request}: {exp}")
+            raise exp
+
+        return self.structure_response(request=request, response=response, body=body)
+
+    @staticmethod
+    def structure_response(request, response, body):
+        return Response(
+            url=response.url,
+            headers=dict(response.headers),
+            status_code=response.status,
+            body=body,
+            request=request
+        )
+
+    async def send_request(self, session, request) -> ClientResponse:
+        return await self.request_method[request.method.lower()](session, request)
+
+    @staticmethod
+    async def _get(session, request) -> ClientResponse:
+        response = await session.get(
+            request.url,
+            headers=request.headers,
+            cookies=request.cookies
+        )
+        return response
+
+    @staticmethod
+    async def _post(session, request) -> ClientResponse:
+        response = await session.post(
+            request.url,
+            data=request.body,
+            headers=request.headers,
+            cookies=request.cookies,
+            proxy=request.proxy,
+        )
+        return response
+
+    async def request_start(self, _session, _trace_config_ctx, params):
+        self.logger.debug(f"Request start: {params.url}, method：{params.method}")
+
+    async def close(self) -> None:
+        if self.connector:
+            await self.connector.close()
+        if self.session:
+            await self.session.close()

crawlo-1.0.0/crawlo/downloader/httpx_downloader.py

@@ -0,0 +1,48 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Optional
+from httpx import AsyncClient, Timeout
+
+from crawlo import Response
+from crawlo.downloader import DownloaderBase
+
+
+class HttpXDownloader(DownloaderBase):
+    def __init__(self, crawler):
+        super().__init__(crawler)
+        self._client: Optional[AsyncClient] = None
+        self._timeout: Optional[Timeout] = None
+
+    def open(self):
+        super().open()
+        timeout = self.crawler.settings.get_int("DOWNLOAD_TIMEOUT")
+        self._timeout = Timeout(timeout=timeout)
+
+    async def download(self, request) -> Optional[Response]:
+        try:
+            proxies = None
+            async with AsyncClient(timeout=self._timeout, proxy=proxies) as client:
+                self.logger.debug(f"request downloading: {request.url}，method: {request.method}")
+                response = await client.request(
+                    url=request.url,
+                    method=request.method,
+                    headers=request.headers,
+                    cookies=request.cookies,
+                    data=request.body
+                )
+                body = await response.aread()
+        except Exception as exp:
+            self.logger.error(f"Error downloading {request}: {exp}")
+            raise exp
+
+        return self.structure_response(request=request, response=response, body=body)
+
+    @staticmethod
+    def structure_response(request, response, body) -> Response:
+        return Response(
+            url=response.url,
+            headers=dict(response.headers),
+            status_code=response.status_code,
+            body=body,
+            request=request
+        )

crawlo-1.0.0/crawlo/event.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+
+spider_error = "spider_error"
+spider_opened = "spider_open"
+spider_closed = "spider_closed"
+ignore_request = "ignore_request"
+request_scheduled = "request_scheduled"
+response_received = "request_received"
+item_successful = "item_successful"
+item_discard = "item_discard"