crawlo 1.0.0__py3-none-any.whl

crawlo/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.network.request import Request
+from crawlo.network.response import Response
+from crawlo.items.items import Item

crawlo/__version__.py

@@ -0,0 +1,2 @@
+
+__version__ = "1.0.0"

crawlo/core/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-

crawlo/core/engine.py

@@ -0,0 +1,157 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Optional, Generator, Callable
+from inspect import iscoroutine, isgenerator, isasyncgen
+
+from crawlo import Request, Item
+from crawlo.spider import Spider
+from crawlo.core.scheduler import Scheduler
+from crawlo.core.processor import Processor
+from crawlo.utils.log import get_logger
+from crawlo.task_manager import TaskManager
+from crawlo.utils.project import load_class
+from crawlo.downloader import DownloaderBase
+from crawlo.utils.func_tools import transform
+from crawlo.exceptions import OutputError, TransformTypeError
+from crawlo.event import spider_opened, spider_error
+
+
+class Engine(object):
+
+    def __init__(self, crawler):
+        self.running = False
+        self.normal = True
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.spider: Optional[Spider] = None
+        self.downloader: Optional[DownloaderBase] = None
+        self.scheduler: Optional[Scheduler] = None
+        self.processor: Optional[Processor] = None
+        self.start_requests: Optional[Generator] = None
+        self.task_manager: Optional[TaskManager] = TaskManager(self.settings.get_int('CONCURRENCY'))
+
+        self.logger = get_logger(name=self.__class__.__name__)
+
+    def _get_downloader_cls(self):
+        downloader_cls = load_class(self.settings.get('DOWNLOADER'))
+        if not issubclass(downloader_cls, DownloaderBase):
+            raise TypeError(f'Downloader {downloader_cls.__name__} is not subclass of DownloaderBase.')
+        return downloader_cls
+
+    def engine_start(self):
+        self.running = True
+        self.logger.info(
+            f"Crawlo (version {self.settings.get_int('VERSION')}) started. "
+            f"(project name : {self.settings.get('PROJECT_NAME')})"
+        )
+
+    async def start_spider(self, spider):
+        self.spider = spider
+
+        self.scheduler = Scheduler(self.crawler)
+        if hasattr(self.scheduler, 'open'):
+            self.scheduler.open()
+
+        downloader_cls = self._get_downloader_cls()
+        self.downloader = downloader_cls(self.crawler)
+        if hasattr(self.downloader, 'open'):
+            self.downloader.open()
+
+        self.processor = Processor(self.crawler)
+        if hasattr(self.processor, 'open'):
+            self.processor.open()
+
+        self.start_requests = iter(spider.start_requests())
+        await self._open_spider()
+
+    async def crawl(self):
+        """
+        Crawl the spider
+        """
+        while self.running:
+            if request := await self._get_next_request():
+                await self._crawl(request)
+            try:
+                start_request = next(self.start_requests)
+            except StopIteration:
+                self.start_requests = None
+            except Exception as exp:
+                # 1、发去请求的request全部运行完毕
+                # 2、调度器是否空闲
+                # 3、下载器是否空闲
+                if not await self._exit():
+                    continue
+                self.running = False
+                if self.start_requests is not None:
+                    self.logger.error(f"启动请求时发生错误: {str(exp)}")
+            else:
+                # 请求入队
+                await self.enqueue_request(start_request)
+
+        if not self.running:
+            await self.close_spider()
+
+    async def _open_spider(self):
+        asyncio.create_task(self.crawler.subscriber.notify(spider_opened))
+        crawling = asyncio.create_task(self.crawl())
+        await crawling
+
+    async def _crawl(self, request):
+        # TODO 实现并发
+        async def crawl_task():
+            outputs = await self._fetch(request)
+            # TODO 处理output
+            if outputs:
+                await self._handle_spider_output(outputs)
+
+        # asyncio.create_task(crawl_task())
+        self.task_manager.create_task(crawl_task())
+
+    async def _fetch(self, request):
+        async def _successful(_response):
+            callback: Callable = request.callback or self.spider.parse
+            if _outputs := callback(_response):
+                if iscoroutine(_outputs):
+                    await _outputs
+                else:
+                    return transform(_outputs)
+
+        _response = await self.downloader.fetch(request)
+        if _response is None:
+            return None
+        output = await _successful(_response)
+        return output
+
+    async def enqueue_request(self, start_request):
+        await self._schedule_request(start_request)
+
+    async def _schedule_request(self, request):
+        # TODO 去重
+        await self.scheduler.enqueue_request(request)
+
+    async def _get_next_request(self):
+        return await self.scheduler.next_request()
+
+    async def _handle_spider_output(self, outputs):
+        async for spider_output in outputs:
+            if isinstance(spider_output, (Request, Item)):
+                await self.processor.enqueue(spider_output)
+            elif isinstance(spider_output, Exception):
+                asyncio.create_task(
+                    self.crawler.subscriber.notify(spider_error, spider_output, self.spider)
+                )
+                raise spider_output
+            else:
+                raise OutputError(f'{type(self.spider)} must return `Request` or `Item`.')
+
+    async def _exit(self):
+        if self.scheduler.idle() and self.downloader.idle() and self.task_manager.all_done() and self.processor.idle():
+            return True
+        return False
+
+    async def close_spider(self):
+        await asyncio.gather(*self.task_manager.current_task)
+        await self.downloader.close()
+        if self.normal:
+            await self.crawler.close()

crawlo/core/processor.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from asyncio import Queue
+from typing import Union, Optional
+
+from crawlo import Request, Item
+from crawlo.pipelines.pipeline_manager import PipelineManager
+
+
+class Processor(object):
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.queue: Queue = Queue()
+        self.pipelines: Optional[PipelineManager] = None
+
+    def open(self):
+        self.pipelines = PipelineManager.create_instance(self.crawler)
+
+    async def process(self):
+        while not self.idle():
+            result = await self.queue.get()
+            if isinstance(result, Request):
+                await self.crawler.engine.enqueue_request(result)
+            else:
+                assert isinstance(result, Item)
+                await self._process_item(result)
+
+    async def _process_item(self, item):
+        await self.pipelines.process_item(item=item)
+
+    async def enqueue(self, output: Union[Request, Item]):
+        await self.queue.put(output)
+        await self.process()
+
+    def idle(self) -> bool:
+        return len(self) == 0
+
+    def __len__(self):
+        return self.queue.qsize()

crawlo/core/scheduler.py

@@ -0,0 +1,35 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.event import request_scheduled
+from crawlo.utils.pqueue import SpiderPriorityQueue
+
+
+class Scheduler:
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.request_queue: Optional[SpiderPriorityQueue] = None
+
+        self.item_count = 0
+        self.response_count = 0
+        self.logger = get_logger(name=self.__class__.__name__, level=crawler.settings.get('LOG_LEVEL'))
+
+    def open(self):
+        self.request_queue = SpiderPriorityQueue()
+
+    async def next_request(self):
+        request = await self.request_queue.get()
+        return request
+
+    async def enqueue_request(self, request):
+        await self.request_queue.put(request)
+        asyncio.create_task(self.crawler.subscriber.notify(request_scheduled, request, self.crawler.spider))
+
+    def idle(self) -> bool:
+        return len(self) == 0
+
+    def __len__(self):
+        return self.request_queue.qsize()

crawlo/crawler.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*
+import signal
+import asyncio
+from typing import Type, Final, Set, Optional
+
+from crawlo.spider import Spider
+from crawlo.core.engine import Engine
+from crawlo.subscriber import Subscriber
+
+from crawlo.utils.log import get_logger
+from crawlo.extension import ExtensionManager
+from crawlo.exceptions import SpiderTypeError
+from crawlo.utils.project import merge_settings
+from crawlo.stats_collector import StatsCollector
+from crawlo.event import spider_opened, spider_closed
+from crawlo.settings.setting_manager import SettingManager
+
+logger = get_logger(__name__)
+
+
+class Crawler:
+
+    def __init__(self, spider_cls, settings):
+        self.spider_cls = spider_cls
+        self.spider: Optional[Spider] = None
+        self.engine: Optional[Engine] = None
+        self.stats: Optional[StatsCollector] = None
+        self.subscriber: Optional[Subscriber] = None
+        self.extension: Optional[ExtensionManager] = None
+        self.settings: SettingManager = settings.copy()
+
+    async def crawl(self):
+        self.subscriber = self._create_subscriber()
+        self.spider = self._create_spider()
+        self.engine = self._create_engine()
+        self.stats = self._create_stats()
+        self.extension = self._create_extension()
+
+        await self.engine.start_spider(self.spider)
+
+    @staticmethod
+    def _create_subscriber():
+        return Subscriber()
+
+    def _create_spider(self) -> Spider:
+        spider = self.spider_cls.create_instance(self)
+        self._set_spider(spider)
+        return spider
+
+    def _create_engine(self) -> Engine:
+        engine = Engine(self)
+        engine.engine_start()
+        return engine
+
+    def _create_stats(self) -> StatsCollector:
+        stats = StatsCollector(self)
+        return stats
+
+    def _create_extension(self) -> ExtensionManager:
+        extension = ExtensionManager.create_instance(self)
+        return extension
+
+    def _set_spider(self, spider):
+        self.subscriber.subscribe(spider.spider_opened, event=spider_opened)
+        self.subscriber.subscribe(spider.spider_closed, event=spider_closed)
+        merge_settings(spider, self.settings)
+
+    async def close(self, reason='finished') -> None:
+        await asyncio.create_task(self.subscriber.notify(spider_closed))
+        self.stats.close_spider(spider_name=self.spider, reason=reason)
+
+
+class CrawlerProcess:
+
+    def __init__(self, settings=None):
+        self.crawlers: Final[Set] = set()
+        self._active_spiders: Final[Set] = set()
+        self.settings = settings
+
+        signal.signal(signal.SIGINT, self._shutdown)
+
+    async def crawl(self, spider: Type[Spider]):
+        crawler: Crawler = self._create_crawler(spider)
+        self.crawlers.add(crawler)
+        task = await self._crawl(crawler)
+        self._active_spiders.add(task)
+
+    @staticmethod
+    async def _crawl(crawler):
+        return asyncio.create_task(crawler.crawl())
+
+    async def start(self):
+        await asyncio.gather(*self._active_spiders)
+
+    def _create_crawler(self, spider_cls) -> Crawler:
+        if isinstance(spider_cls, str):
+            raise SpiderTypeError(f"{type(self)}.crawl args: String is not supported.")
+        crawler: Crawler = Crawler(spider_cls, self.settings)
+        return crawler
+
+    def _shutdown(self, _signum, _frame):
+        for crawler in self.crawlers:
+            crawler.engine.running = False
+            crawler.engine.normal = False
+            crawler.stats.close_spider(crawler.spider, 'Ctrl C')
+        logger.warning(f'spiders received: `Ctrl C` signal, closed.')

crawlo/downloader/__init__.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from abc import abstractmethod, ABCMeta
+from typing_extensions import Self
+from typing import Final, Set, Optional
+from contextlib import asynccontextmanager
+
+from crawlo import Response, Request
+from crawlo.utils.log import get_logger
+from crawlo.middleware.middleware_manager import MiddlewareManager
+
+
+class ActivateRequestManager:
+
+    def __init__(self):
+        self._active: Final[Set] = set()
+
+    def add(self, request):
+        self._active.add(request)
+
+    def remove(self, request):
+        self._active.remove(request)
+
+    @asynccontextmanager
+    async def __call__(self, request):
+        try:
+            yield self.add(request)
+        finally:
+            self.remove(request)
+
+    def __len__(self):
+        return len(self._active)
+
+
+class DownloaderMeta(ABCMeta):
+    def __subclasscheck__(self, subclass):
+        required_methods = ('fetch', 'download', 'create_instance', 'close')
+        is_subclass = all(
+            hasattr(subclass, method) and callable(getattr(subclass, method, None)) for method in required_methods
+        )
+        return is_subclass
+
+
+class DownloaderBase(metaclass=DownloaderMeta):
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self._active = ActivateRequestManager()
+        self.middleware: Optional[MiddlewareManager] = None
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get("LOG_LEVEL"))
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs) -> Self:
+        return cls(*args, **kwargs)
+
+    def open(self) -> None:
+        self.logger.info(
+            f"{self.crawler.spider} <downloader class：{type(self).__name__}>"
+            f"<concurrency：{self.crawler.settings.get_int('CONCURRENCY')}>"
+        )
+        self.middleware = MiddlewareManager.create_instance(self.crawler)
+
+    async def fetch(self, request) -> Optional[Response]:
+        async with self._active(request):
+            response = await self.middleware.download(request)
+            return response
+
+    @abstractmethod
+    async def download(self, request: Request) -> Response:
+        pass
+
+    async def close(self) -> None:
+        pass
+
+    def idle(self) -> bool:
+        return len(self) == 0
+
+    def __len__(self) -> int:
+        return len(self._active)

crawlo/downloader/aiohttp_downloader.py

@@ -0,0 +1,96 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Optional
+from aiohttp import ClientSession, TCPConnector, BaseConnector, ClientTimeout, ClientResponse, TraceConfig
+
+from crawlo import Response
+from crawlo.downloader import DownloaderBase
+
+
+class AioHttpDownloader(DownloaderBase):
+    def __init__(self, crawler):
+        super().__init__(crawler)
+        self.session: Optional[ClientSession] = None
+        self.connector: Optional[BaseConnector] = None
+        self._verify_ssl: Optional[bool] = None
+        self._timeout: Optional[ClientTimeout] = None
+        self._use_session: Optional[bool] = None
+        self.trace_config: Optional[TraceConfig] = None
+
+        self.request_method = {
+            "get": self._get,
+            "post": self._post
+        }
+
+    def open(self):
+        super().open()
+        self._timeout = ClientTimeout(total=self.crawler.settings.get_int("DOWNLOAD_TIMEOUT"))
+        self._verify_ssl = self.crawler.settings.get_bool("VERIFY_SSL")
+        self._use_session = self.crawler.settings.get_bool("USE_SESSION")
+        self.trace_config = TraceConfig()
+        self.trace_config.on_request_start.append(self.request_start)
+        if self._use_session:
+            self.connector = TCPConnector(verify_ssl=self._verify_ssl)
+            self.session = ClientSession(
+                connector=self.connector, timeout=self._timeout, trace_configs=[self.trace_config]
+            )
+
+    async def download(self, request) -> Optional[Response]:
+        try:
+            if self._use_session:
+                response = await self.send_request(self.session, request)
+                body = await response.content.read()
+            else:
+                connector = TCPConnector(verify_ssl=self._verify_ssl)
+                async with ClientSession(
+                        connector=connector, timeout=self._timeout, trace_configs=[self.trace_config]
+                ) as session:
+                    response = await self.send_request(session, request)
+                    body = await response.content.read()
+        except Exception as exp:
+            self.logger.error(f"Error downloading {request}: {exp}")
+            raise exp
+
+        return self.structure_response(request=request, response=response, body=body)
+
+    @staticmethod
+    def structure_response(request, response, body):
+        return Response(
+            url=response.url,
+            headers=dict(response.headers),
+            status_code=response.status,
+            body=body,
+            request=request
+        )
+
+    async def send_request(self, session, request) -> ClientResponse:
+        return await self.request_method[request.method.lower()](session, request)
+
+    @staticmethod
+    async def _get(session, request) -> ClientResponse:
+        response = await session.get(
+            request.url,
+            headers=request.headers,
+            cookies=request.cookies
+        )
+        return response
+
+    @staticmethod
+    async def _post(session, request) -> ClientResponse:
+        response = await session.post(
+            request.url,
+            data=request.body,
+            headers=request.headers,
+            cookies=request.cookies,
+            proxy=request.proxy,
+        )
+        return response
+
+    async def request_start(self, _session, _trace_config_ctx, params):
+        self.logger.debug(f"Request start: {params.url}, method：{params.method}")
+
+    async def close(self) -> None:
+        if self.connector:
+            await self.connector.close()
+        if self.session:
+            await self.session.close()

crawlo/downloader/httpx_downloader.py

@@ -0,0 +1,48 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Optional
+from httpx import AsyncClient, Timeout
+
+from crawlo import Response
+from crawlo.downloader import DownloaderBase
+
+
+class HttpXDownloader(DownloaderBase):
+    def __init__(self, crawler):
+        super().__init__(crawler)
+        self._client: Optional[AsyncClient] = None
+        self._timeout: Optional[Timeout] = None
+
+    def open(self):
+        super().open()
+        timeout = self.crawler.settings.get_int("DOWNLOAD_TIMEOUT")
+        self._timeout = Timeout(timeout=timeout)
+
+    async def download(self, request) -> Optional[Response]:
+        try:
+            proxies = None
+            async with AsyncClient(timeout=self._timeout, proxy=proxies) as client:
+                self.logger.debug(f"request downloading: {request.url}，method: {request.method}")
+                response = await client.request(
+                    url=request.url,
+                    method=request.method,
+                    headers=request.headers,
+                    cookies=request.cookies,
+                    data=request.body
+                )
+                body = await response.aread()
+        except Exception as exp:
+            self.logger.error(f"Error downloading {request}: {exp}")
+            raise exp
+
+        return self.structure_response(request=request, response=response, body=body)
+
+    @staticmethod
+    def structure_response(request, response, body) -> Response:
+        return Response(
+            url=response.url,
+            headers=dict(response.headers),
+            status_code=response.status_code,
+            body=body,
+            request=request
+        )

crawlo/event.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+
+spider_error = "spider_error"
+spider_opened = "spider_open"
+spider_closed = "spider_closed"
+ignore_request = "ignore_request"
+request_scheduled = "request_scheduled"
+response_received = "request_received"
+item_successful = "item_successful"
+item_discard = "item_discard"

crawlo/exceptions.py

@@ -0,0 +1,64 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+class TransformTypeError(TypeError):
+    pass
+
+
+class OutputError(Exception):
+    pass
+
+
+class SpiderTypeError(TypeError):
+    pass
+
+
+class ItemInitError(Exception):
+    pass
+
+
+class ItemAttributeError(Exception):
+    pass
+
+
+class DecodeError(Exception):
+    pass
+
+
+class MiddlewareInitError(Exception):
+    pass
+
+
+class PipelineInitError(Exception):
+    pass
+
+
+class InvalidOutputError(Exception):
+    pass
+
+
+class RequestMethodError(Exception):
+    pass
+
+
+class IgnoreRequestError(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+        super(IgnoreRequestError, self).__init__(msg)
+
+
+class ItemDiscard(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+        super(ItemDiscard, self).__init__(msg)
+
+
+class NotConfiguredError(Exception):
+    pass
+
+
+class ExtensionInitError(Exception):
+    pass
+
+
+class ReceiverTypeError(Exception):
+    pass

crawlo/extension/__init__.py

@@ -0,0 +1,31 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List
+from pprint import pformat
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.project import load_class
+from crawlo.exceptions import ExtensionInitError
+
+
+class ExtensionManager(object):
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.extensions: List = []
+        extensions = self.crawler.settings.get_list('EXTENSIONS')
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self._add_extensions(extensions)
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs):
+        return cls(*args, **kwargs)
+
+    def _add_extensions(self, extensions):
+        for extension in extensions:
+            extension_cls = load_class(extension)
+            if not hasattr(extension_cls, 'create_instance'):
+                raise ExtensionInitError(f"extension init failed, Must have method 'create_instance()")
+            self.extensions.append(extension_cls.create_instance(self.crawler))
+        if extensions:
+            self.logger.info(f"enabled extensions: \n {pformat(extensions)}")