crawlo 1.0.0__py3-none-any.whl

crawlo/extension/log_interval.py

@@ -0,0 +1,49 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed
+
+
+class LogIntervalExtension(object):
+
+    def __init__(self, crawler):
+        self.task = None
+        self.stats = crawler.stats
+        self.item_count = 0
+        self.response_count = 0
+        self.seconds = crawler.settings.get('INTERVAL')
+        self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
+        self.interval = "" if self.interval == 1 else self.interval
+        self.unit = 'min' if self.seconds % 60 == 0 else 's'
+
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self):
+        self.task = asyncio.create_task(self.interval_log())
+        await self.task
+
+    async def spider_closed(self):
+        if self.task:
+            self.task.cancel()
+
+    async def interval_log(self):
+        while True:
+            last_item_count = self.stats.get_value('item_successful_count', default=0)
+            last_response_count = self.stats.get_value('response_received_count', default=0)
+            item_rate = last_item_count - self.item_count
+            response_rate = last_response_count - self.response_count
+            self.item_count, self.response_count = last_item_count, last_response_count
+            self.logger.info(
+                f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
+                f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
+            )
+            await asyncio.sleep(self.seconds)

crawlo/extension/log_stats.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo import event
+from crawlo.utils.date_tools import now, date_delta
+
+
+class LogStats(object):
+
+    def __init__(self, stats):
+        self._stats = stats
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(crawler.stats)
+        crawler.subscriber.subscribe(o.spider_opened, event=event.spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
+        crawler.subscriber.subscribe(o.item_successful, event=event.item_successful)
+        crawler.subscriber.subscribe(o.item_discard, event=event.item_discard)
+        crawler.subscriber.subscribe(o.response_received, event=event.response_received)
+        crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
+
+        return o
+
+    async def spider_opened(self):
+        self._stats['start_time'] = now()
+
+    async def spider_closed(self):
+        self._stats['end_time'] = now()
+        self._stats['cost_time(s)'] = date_delta(start=self._stats['start_time'], end=self._stats['end_time'])
+
+    async def item_successful(self, _item, _spider):
+        self._stats.inc_value('item_successful_count')
+
+    async def item_discard(self, _item, exc, _spider):
+        self._stats.inc_value('item_discard_count')
+        reason = exc.msg
+        if reason:
+            self._stats.inc_value(f"item_discard/{reason}")
+
+    async def response_received(self, _response, _spider):
+        self._stats.inc_value('response_received_count')
+
+    async def request_scheduled(self, _request, _spider):
+        self._stats.inc_value('request_scheduler_count')

crawlo/items/__init__.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from abc import ABCMeta
+
+
+class Field(dict):
+    pass
+
+
+class ItemMeta(ABCMeta):
+    """
+    元类
+    """
+    def __new__(mcs, name, bases, attrs):
+        field = {}
+        cls_attr = {}
+        for k, v in attrs.items():
+            if isinstance(v, Field):
+                field[k] = v
+            else:
+                cls_attr[k] = v
+        cls_instance = super().__new__(mcs, name, bases, attrs)
+        cls_instance.FIELDS = field
+        return cls_instance

crawlo/items/items.py

@@ -0,0 +1,88 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+from copy import deepcopy
+from pprint import pformat
+from typing import Any, Iterator, Dict
+from collections.abc import MutableMapping
+
+from crawlo.items import ItemMeta, Field
+from crawlo.exceptions import ItemInitError, ItemAttributeError
+
+
+class Item(MutableMapping, metaclass=ItemMeta):
+    FIELDS: Dict[str, Any] = {}
+
+    def __init__(self, *args, **kwargs):
+        if args:
+            raise ItemInitError(f"{self.__class__.__name__} 不支持位置参数：{args}，请使用关键字参数初始化。")
+        if kwargs:
+            for key, value in kwargs.items():
+                self[key] = value
+
+        self._values: Dict[str, Any] = {}
+
+    def __getitem__(self, item: str) -> Any:
+        return self._values[item]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        if key not in self.FIELDS:
+            raise KeyError(f"{self.__class__.__name__} 不包含字段：{key}")
+        self._values[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self._values[key]
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        if not key.startswith("_"):
+            raise AttributeError(
+                f"设置字段值请使用 item[{key!r}] = {value!r}"
+            )
+        super().__setattr__(key, value)
+
+    def __getattr__(self, item: str) -> Any:
+        # 当获取不到属性时触发
+        raise AttributeError(
+            f"{self.__class__.__name__} 不支持字段：{item}。"
+            f"请先在 `{self.__class__.__name__}` 中声明该字段，再通过 item[{item!r}] 获取。"
+        )
+
+    def __getattribute__(self, item: str) -> Any:
+        # 属性拦截器，只要访问属性就会进入该方法
+        try:
+            field = super().__getattribute__("FIELDS")
+            if isinstance(field, dict) and item in field:
+                raise ItemAttributeError(
+                    f"获取字段值请使用 item[{item!r}]"
+                )
+        except AttributeError:
+            pass  # 如果 FIELDS 尚未定义，继续执行后续逻辑
+        return super().__getattribute__(item)
+
+    def __repr__(self) -> str:
+        return pformat(dict(self))
+
+    __str__ = __repr__
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._values)
+
+    def __len__(self) -> int:
+        return len(self._values)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return dict(self)
+
+    def copy(self) -> "Item":
+        return deepcopy(self)
+
+
+if __name__ == '__main__':
+    class TestItem(Item):
+        url = Field()
+        title = Field()
+
+    test_item = TestItem()
+    test_item['title'] = '百度首页'
+    test_item['url'] = 'http://example.com'
+    # test_item.title = 'fffff'
+    print(test_item.title)

crawlo/middleware/__init__.py

@@ -0,0 +1,21 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo import Request, Response
+
+
+class BaseMiddleware(object):
+    def process_request(self, request, spider) -> None | Request | Response:
+        # 请求预处理
+        pass
+
+    def process_response(self, request, response, spider) -> Request | Response:
+        # 响应预处理
+        pass
+
+    def process_exception(self, request, exp, spider) -> None | Request | Response:
+        # 异常预处理
+        pass
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls()

crawlo/middleware/default_header.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.event import spider_opened
+
+
+class DefaultHeaderMiddleware(object):
+
+    def __init__(self, user_agent, headers, spider):
+        self.user_agent = user_agent
+        self.headers = headers
+        self.spider = spider
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(
+            user_agent=crawler.settings.get('USER_AGENT'),
+            headers=crawler.settings.get_dict('DEFAULT_HEADERS'),
+            spider=crawler.spider
+        )
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        return o
+
+    async def spider_opened(self):
+        self.user_agent = getattr(self.spider, 'user_agent', self.user_agent)
+        self.headers = getattr(self.spider, 'headers', self.headers)
+        if self.user_agent:
+            self.headers.setdefault('User-Agent', self.user_agent)
+
+    def process_request(self, request, _spider):
+        if self.headers:
+            for key, value in self.headers.items():
+                request.headers.setdefault(key, value)

crawlo/middleware/download_delay.py

@@ -0,0 +1,28 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from asyncio import sleep
+from random import uniform
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import NotConfiguredError
+
+
+class DownloadDelayMiddleware(object):
+
+    def __init__(self, settings, log_level):
+        self.delay = settings.get_float("DOWNLOAD_DELAY")
+        if not self.delay:
+            raise NotConfiguredError
+        self.randomness = settings.get_bool("RANDOMNESS")
+        self.floor, self.upper = settings.get_list("RANDOM_RANGE")
+        self.logger = get_logger(self.__class__.__name__, log_level)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(settings=crawler.settings, log_level=crawler.settings.get('LOG_LEVEL'))
+        return o
+
+    async def process_request(self, _request, _spider):
+        if self.randomness:
+            await sleep(uniform(self.delay * self.floor, self.delay * self.upper))
+        else:
+            await sleep(self.delay)

crawlo/middleware/middleware_manager.py

@@ -0,0 +1,140 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from pprint import pformat
+from types import MethodType
+from asyncio import create_task
+from collections import defaultdict
+from typing import List, Dict, Callable, Optional
+
+from crawlo import Request, Response
+from crawlo.utils.log import get_logger
+from crawlo.utils.project import load_class
+from crawlo.middleware import BaseMiddleware
+from crawlo.utils.project import common_call
+from crawlo.event import ignore_request, response_received
+from crawlo.exceptions import MiddlewareInitError, InvalidOutputError, RequestMethodError, IgnoreRequestError, \
+    NotConfiguredError
+
+
+class MiddlewareManager:
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.middlewares: List = []
+        self.methods: Dict[str, List[MethodType]] = defaultdict(list)
+        middlewares = self.crawler.settings.get_list('MIDDLEWARES')
+        self._add_middleware(middlewares)
+        self._add_method()
+
+        self.download_method: Callable = crawler.engine.downloader.download
+        self._stats = crawler.stats
+
+    async def _process_request(self, request: Request):
+        for method in self.methods['process_request']:
+            result = await common_call(method, request, self.crawler.spider)
+            if result is None:
+                continue
+            if isinstance(result, (Request, Response)):
+                return result
+            raise InvalidOutputError(
+                f"{method.__self__.__class__.__name__}. must return None or Request or Response, got {type(result).__name__}"
+            )
+        return await self.download_method(request)
+
+    async def _process_response(self, request: Request, response: Response):
+        for method in reversed(self.methods['process_response']):
+            try:
+                response = await common_call(method, request, response, self.crawler.spider)
+            except IgnoreRequestError as exp:
+                create_task(self.crawler.subscriber.notify(ignore_request, exp, request, self.crawler.spider))
+                # self.logger.info(f'{request} ignored.')
+                # self._stats.inc_value('request_ignore_count')
+                # reason = exp.msg
+                # if reason:
+                #     self._stats.inc_value(f'request_ignore_count/{reason}')
+            if isinstance(response, Request):
+                return response
+            if isinstance(response, Response):
+                continue
+            raise InvalidOutputError(
+                f"{method.__self__.__class__.__name__}. must return Request or Response, got {type(response).__name__}"
+            )
+        return response
+
+    async def _process_exception(self, request: Request, exp: Exception):
+        for method in self.methods['process_exception']:
+            response = await common_call(method, request, exp, self.crawler.spider)
+            if response is None:
+                continue
+            if isinstance(response, (Request, Response)):
+                return response
+            if response:
+                break
+            raise InvalidOutputError(
+                f"{method.__self__.__class__.__name__}. must return None or Request or Response, got {type(response).__name__}"
+            )
+        else:
+            raise exp
+
+    async def download(self, request) -> Optional[Response]:
+        """ called in the download method. """
+        try:
+            response = await self._process_request(request)
+        except KeyError:
+            raise RequestMethodError(f"{request.method.lower()} is not supported")
+        except IgnoreRequestError as exp:
+            create_task(self.crawler.subscriber.notify(ignore_request, exp, request, self.crawler.spider))
+            response = await self._process_exception(request, exp)
+        except Exception as exp:
+            self._stats.inc_value(f'download_error/{exp.__class__.__name__}')
+            response = await self._process_exception(request, exp)
+        else:
+            create_task(self.crawler.subscriber.notify(response_received, response, self.crawler.spider))
+            # self.crawler.stats.inc_value('response_received_count')
+        if isinstance(response, Response):
+            response = await self._process_response(request, response)
+        if isinstance(response, Request):
+            await self.crawler.engine.enqueue_request(request)
+            return None
+        return response
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs):
+        return cls(*args, **kwargs)
+
+    def _add_middleware(self, middlewares):
+        enabled_middlewares = [m for m in middlewares if self._validate_middleware(m)]
+        if enabled_middlewares:
+            self.logger.info(f'enabled middleware：\n {pformat(enabled_middlewares)}')
+
+    def _validate_middleware(self, middleware):
+        middleware_cls = load_class(middleware)
+        if not hasattr(middleware_cls, 'create_instance'):
+            raise MiddlewareInitError(
+                f"Middleware init failed, must inherit from `BaseMiddleware` or have a `create_instance` method"
+            )
+        try:
+            instance = middleware_cls.create_instance(self.crawler)
+            self.middlewares.append(instance)
+            return True
+        except NotConfiguredError:
+            return False
+
+    def _add_method(self):
+        for middleware in self.middlewares:
+            if hasattr(middleware, 'process_request'):
+                if self._validate_middleware_method(method_name='process_request', middleware=middleware):
+                    self.methods['process_request'].append(middleware.process_request)
+            if hasattr(middleware, 'process_response'):
+                if self._validate_middleware_method(method_name='process_response', middleware=middleware):
+                    self.methods['process_response'].append(middleware.process_response)
+            if hasattr(middleware, 'process_exception'):
+                if self._validate_middleware_method(method_name='process_exception', middleware=middleware):
+                    self.methods['process_exception'].append(middleware.process_exception)
+
+    @staticmethod
+    def _validate_middleware_method(method_name, middleware) -> bool:
+        method = getattr(type(middleware), method_name)
+        base_method = getattr(BaseMiddleware, method_name)
+        return False if method == base_method else True

crawlo/middleware/request_ignore.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import IgnoreRequestError
+from crawlo.event import ignore_request
+
+
+class RequestIgnoreMiddleware(object):
+
+    def __init__(self, stats, log_level):
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.stats = stats
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(stats=crawler.stats, log_level=crawler.settings.get('LOG_LEVEL'))
+        crawler.subscriber.subscribe(o.request_ignore, event=ignore_request)
+        return o
+
+    async def request_ignore(self, exc, request, _spider):
+        self.logger.info(f'{request} ignored.')
+        self.stats.inc_value('request_ignore_count')
+        reason = exc.msg
+        if reason:
+            self.stats.inc_value(f'request_ignore_count/{reason}')
+
+    @staticmethod
+    def process_exception(_request, exc, _spider):
+        if isinstance(exc, IgnoreRequestError):
+            return True

crawlo/middleware/response_code.py

@@ -0,0 +1,19 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.utils.log import get_logger
+
+
+class ResponseCodeMiddleware(object):
+    def __init__(self, stats, log_level):
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.stats = stats
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(stats=crawler.stats, log_level=crawler.settings.get('LOG_LEVEL'))
+        return o
+
+    def process_response(self, request, response, spider):
+        self.stats.inc_value(f'stats_code/count/{response.status_code}')
+        self.logger.debug(f'Got response from <{response.status_code} {response.url}>')
+        return response

crawlo/middleware/response_filter.py

@@ -0,0 +1,26 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import IgnoreRequestError
+
+
+class ResponseFilterMiddleware:
+
+    def __init__(self, allowed_codes, log_level):
+        self.allowed_codes = allowed_codes
+        self.logger = get_logger(self.__class__.__name__, log_level)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(
+            allowed_codes=crawler.settings.get_list('ALLOWED_CODES'),
+            log_level=crawler.settings.get('LOG_LEVEL')
+        )
+        return o
+
+    def process_response(self, request, response, spider):
+        if 200 <= response.status_code < 300:
+            return response
+        if response.status_code in self.allowed_codes:
+            return response
+        raise IgnoreRequestError(f"response status_code/non-200")

crawlo/middleware/retry.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List
+from anyio import EndOfStream
+from httpcore import ReadError
+from asyncio.exceptions import TimeoutError
+from httpx import RemoteProtocolError, ConnectError, ReadTimeout
+from aiohttp.client_exceptions import ClientConnectionError, ClientPayloadError
+from aiohttp import ClientConnectorError, ClientTimeout, ClientConnectorSSLError, ClientResponseError
+
+from crawlo.utils.log import get_logger
+from crawlo.stats_collector import StatsCollector
+
+_retry_exceptions = [
+    EndOfStream,
+    ReadError,
+    TimeoutError,
+    ConnectError,
+    ReadTimeout,
+    ClientConnectorError,
+    ClientResponseError,
+    RemoteProtocolError,
+    ClientTimeout,
+    ClientConnectorSSLError,
+    ClientPayloadError,
+    ClientConnectionError
+]
+
+
+class RetryMiddleware(object):
+
+    def __init__(
+            self,
+            *,
+            retry_http_codes: List,
+            ignore_http_codes: List,
+            max_retry_times: int,
+            retry_exceptions: List,
+            stats: StatsCollector
+    ):
+        self.retry_http_codes = retry_http_codes
+        self.ignore_http_codes = ignore_http_codes
+        self.max_retry_times = max_retry_times
+        self.retry_exceptions = tuple(retry_exceptions + _retry_exceptions)
+        self.stats = stats
+        self.logger = get_logger(self.__class__.__name__)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(
+            retry_http_codes=crawler.settings.get_list('RETRY_HTTP_CODES'),
+            ignore_http_codes=crawler.settings.get_list('IGNORE_HTTP_CODES'),
+            max_retry_times=crawler.settings.get_int('MAX_RETRY_TIMES'),
+            retry_exceptions=crawler.settings.get_list('RETRY_EXCEPTIONS'),
+            stats=crawler.stats
+        )
+        return o
+
+    def process_response(self, request, response, spider):
+        if request.meta.get('dont_retry', False):
+            return response
+        if response.status_code in self.ignore_http_codes:
+            return response
+        if response.status_code in self.retry_http_codes:
+            # 重试逻辑
+            reason = f"response code {response.status_code}"
+            return self._retry(request, reason, spider) or response
+        return response
+
+    def process_exception(self, request, exc, spider):
+        if isinstance(exc, self.retry_exceptions) and not request.meta.get('dont_retry', False):
+            return self._retry(request=request, reason=type(exc).__name__, spider=spider)
+
+    def _retry(self, request, reason, spider):
+        retry_times = request.meta.get('retry_times', 0)
+        if retry_times < self.max_retry_times:
+            retry_times += 1
+            self.logger.info(f"{spider} {request} {reason} retrying {retry_times} time...")
+            request.meta['retry_times'] = retry_times
+            self.stats.inc_value("retry_count")
+            return request
+        else:
+            self.logger.warning(f"{spider} {request} {reason} retry max {self.max_retry_times} times, give up.")
+            return None

crawlo/network/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 14:07
+# @Author  :   oscar
+# @Desc    :   None
+"""

crawlo/network/request.py

@@ -0,0 +1,52 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import hashlib
+from copy import deepcopy
+from typing import Dict, Optional, Callable
+
+
+class Request(object):
+
+    def __init__(
+            self,
+            url: str,
+            *,
+            callback: Optional[Callable] = None,
+            headers: Optional[Dict[str, str]] = None,
+            body: Optional[bytes] = None,
+            method: Optional[str] = 'GET',
+            cookies: Optional[Dict[str, str]] = None,
+            priority: int = 0,
+            encoding: Optional[str] = 'UTF-8',
+            meta: Optional[Dict[str, str]] = None
+
+    ):
+        self.url = url
+        self.callback = callback
+        self.headers = headers if headers else {}
+        self.body = body
+        self.method = method
+        self.cookies = cookies
+        self.priority = priority
+        self.encoding = encoding
+        self._meta = meta if meta is not None else {}
+
+    def copy(self):
+        return deepcopy(self)
+
+    def fingerprint(self) -> str:
+        data = f"{self.url}{self.method}{self.body or b''}".encode()
+        return hashlib.sha256(data).hexdigest()
+
+    def set_meta(self, key: str, value: str):
+        self._meta[key] = value
+
+    @property
+    def meta(self):
+        return self._meta
+
+    def __str__(self):
+        return f'<Request url={self.url}> method={self.method}>'
+
+    def __lt__(self, other):
+        return self.priority < other.priority