crawlo 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

crawlo/middleware/middleware_manager.py

@@ -1,140 +1,140 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from pprint import pformat
-from types import MethodType
-from asyncio import create_task
-from collections import defaultdict
-from typing import List, Dict, Callable, Optional
-
-from crawlo import Request, Response
-from crawlo.utils.log import get_logger
-from crawlo.utils.project import load_class
-from crawlo.middleware import BaseMiddleware
-from crawlo.utils.project import common_call
-from crawlo.event import ignore_request, response_received
-from crawlo.exceptions import MiddlewareInitError, InvalidOutputError, RequestMethodError, IgnoreRequestError, \
-    NotConfiguredError
-
-
-class MiddlewareManager:
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        self.middlewares: List = []
-        self.methods: Dict[str, List[MethodType]] = defaultdict(list)
-        middlewares = self.crawler.settings.get_list('MIDDLEWARES')
-        self._add_middleware(middlewares)
-        self._add_method()
-
-        self.download_method: Callable = crawler.engine.downloader.download
-        self._stats = crawler.stats
-
-    async def _process_request(self, request: Request):
-        for method in self.methods['process_request']:
-            result = await common_call(method, request, self.crawler.spider)
-            if result is None:
-                continue
-            if isinstance(result, (Request, Response)):
-                return result
-            raise InvalidOutputError(
-                f"{method.__self__.__class__.__name__}. must return None or Request or Response, got {type(result).__name__}"
-            )
-        return await self.download_method(request)
-
-    async def _process_response(self, request: Request, response: Response):
-        for method in reversed(self.methods['process_response']):
-            try:
-                response = await common_call(method, request, response, self.crawler.spider)
-            except IgnoreRequestError as exp:
-                create_task(self.crawler.subscriber.notify(ignore_request, exp, request, self.crawler.spider))
-                # self.logger.info(f'{request} ignored.')
-                # self._stats.inc_value('request_ignore_count')
-                # reason = exp.msg
-                # if reason:
-                #     self._stats.inc_value(f'request_ignore_count/{reason}')
-            if isinstance(response, Request):
-                return response
-            if isinstance(response, Response):
-                continue
-            raise InvalidOutputError(
-                f"{method.__self__.__class__.__name__}. must return Request or Response, got {type(response).__name__}"
-            )
-        return response
-
-    async def _process_exception(self, request: Request, exp: Exception):
-        for method in self.methods['process_exception']:
-            response = await common_call(method, request, exp, self.crawler.spider)
-            if response is None:
-                continue
-            if isinstance(response, (Request, Response)):
-                return response
-            if response:
-                break
-            raise InvalidOutputError(
-                f"{method.__self__.__class__.__name__}. must return None or Request or Response, got {type(response).__name__}"
-            )
-        else:
-            raise exp
-
-    async def download(self, request) -> Optional[Response]:
-        """ called in the download method. """
-        try:
-            response = await self._process_request(request)
-        except KeyError:
-            raise RequestMethodError(f"{request.method.lower()} is not supported")
-        except IgnoreRequestError as exp:
-            create_task(self.crawler.subscriber.notify(ignore_request, exp, request, self.crawler.spider))
-            response = await self._process_exception(request, exp)
-        except Exception as exp:
-            self._stats.inc_value(f'download_error/{exp.__class__.__name__}')
-            response = await self._process_exception(request, exp)
-        else:
-            create_task(self.crawler.subscriber.notify(response_received, response, self.crawler.spider))
-            # self.crawler.stats.inc_value('response_received_count')
-        if isinstance(response, Response):
-            response = await self._process_response(request, response)
-        if isinstance(response, Request):
-            await self.crawler.engine.enqueue_request(request)
-            return None
-        return response
-
-    @classmethod
-    def create_instance(cls, *args, **kwargs):
-        return cls(*args, **kwargs)
-
-    def _add_middleware(self, middlewares):
-        enabled_middlewares = [m for m in middlewares if self._validate_middleware(m)]
-        if enabled_middlewares:
-            self.logger.info(f'enabled middleware：\n {pformat(enabled_middlewares)}')
-
-    def _validate_middleware(self, middleware):
-        middleware_cls = load_class(middleware)
-        if not hasattr(middleware_cls, 'create_instance'):
-            raise MiddlewareInitError(
-                f"Middleware init failed, must inherit from `BaseMiddleware` or have a `create_instance` method"
-            )
-        try:
-            instance = middleware_cls.create_instance(self.crawler)
-            self.middlewares.append(instance)
-            return True
-        except NotConfiguredError:
-            return False
-
-    def _add_method(self):
-        for middleware in self.middlewares:
-            if hasattr(middleware, 'process_request'):
-                if self._validate_middleware_method(method_name='process_request', middleware=middleware):
-                    self.methods['process_request'].append(middleware.process_request)
-            if hasattr(middleware, 'process_response'):
-                if self._validate_middleware_method(method_name='process_response', middleware=middleware):
-                    self.methods['process_response'].append(middleware.process_response)
-            if hasattr(middleware, 'process_exception'):
-                if self._validate_middleware_method(method_name='process_exception', middleware=middleware):
-                    self.methods['process_exception'].append(middleware.process_exception)
-
-    @staticmethod
-    def _validate_middleware_method(method_name, middleware) -> bool:
-        method = getattr(type(middleware), method_name)
-        base_method = getattr(BaseMiddleware, method_name)
-        return False if method == base_method else True
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from pprint import pformat
+from types import MethodType
+from asyncio import create_task
+from collections import defaultdict
+from typing import List, Dict, Callable, Optional
+
+from crawlo import Request, Response
+from crawlo.utils.log import get_logger
+from crawlo.utils.project import load_class
+from crawlo.middleware import BaseMiddleware
+from crawlo.utils.project import common_call
+from crawlo.event import ignore_request, response_received
+from crawlo.exceptions import MiddlewareInitError, InvalidOutputError, RequestMethodError, IgnoreRequestError, \
+    NotConfiguredError
+
+
+class MiddlewareManager:
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.middlewares: List = []
+        self.methods: Dict[str, List[MethodType]] = defaultdict(list)
+        middlewares = self.crawler.settings.get_list('MIDDLEWARES')
+        self._add_middleware(middlewares)
+        self._add_method()
+
+        self.download_method: Callable = crawler.engine.downloader.download
+        self._stats = crawler.stats
+
+    async def _process_request(self, request: Request):
+        for method in self.methods['process_request']:
+            result = await common_call(method, request, self.crawler.spider)
+            if result is None:
+                continue
+            if isinstance(result, (Request, Response)):
+                return result
+            raise InvalidOutputError(
+                f"{method.__self__.__class__.__name__}. must return None or Request or Response, got {type(result).__name__}"
+            )
+        return await self.download_method(request)
+
+    async def _process_response(self, request: Request, response: Response):
+        for method in reversed(self.methods['process_response']):
+            try:
+                response = await common_call(method, request, response, self.crawler.spider)
+            except IgnoreRequestError as exp:
+                create_task(self.crawler.subscriber.notify(ignore_request, exp, request, self.crawler.spider))
+                # self.logger.info(f'{request} ignored.')
+                # self._stats.inc_value('request_ignore_count')
+                # reason = exp.msg
+                # if reason:
+                #     self._stats.inc_value(f'request_ignore_count/{reason}')
+            if isinstance(response, Request):
+                return response
+            if isinstance(response, Response):
+                continue
+            raise InvalidOutputError(
+                f"{method.__self__.__class__.__name__}. must return Request or Response, got {type(response).__name__}"
+            )
+        return response
+
+    async def _process_exception(self, request: Request, exp: Exception):
+        for method in self.methods['process_exception']:
+            response = await common_call(method, request, exp, self.crawler.spider)
+            if response is None:
+                continue
+            if isinstance(response, (Request, Response)):
+                return response
+            if response:
+                break
+            raise InvalidOutputError(
+                f"{method.__self__.__class__.__name__}. must return None or Request or Response, got {type(response).__name__}"
+            )
+        else:
+            raise exp
+
+    async def download(self, request) -> Optional[Response]:
+        """ called in the download method. """
+        try:
+            response = await self._process_request(request)
+        except KeyError:
+            raise RequestMethodError(f"{request.method.lower()} is not supported")
+        except IgnoreRequestError as exp:
+            create_task(self.crawler.subscriber.notify(ignore_request, exp, request, self.crawler.spider))
+            response = await self._process_exception(request, exp)
+        except Exception as exp:
+            self._stats.inc_value(f'download_error/{exp.__class__.__name__}')
+            response = await self._process_exception(request, exp)
+        else:
+            create_task(self.crawler.subscriber.notify(response_received, response, self.crawler.spider))
+            # self.crawler.stats.inc_value('response_received_count')
+        if isinstance(response, Response):
+            response = await self._process_response(request, response)
+        if isinstance(response, Request):
+            await self.crawler.engine.enqueue_request(request)
+            return None
+        return response
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs):
+        return cls(*args, **kwargs)
+
+    def _add_middleware(self, middlewares):
+        enabled_middlewares = [m for m in middlewares if self._validate_middleware(m)]
+        if enabled_middlewares:
+            self.logger.info(f'enabled middleware：\n {pformat(enabled_middlewares)}')
+
+    def _validate_middleware(self, middleware):
+        middleware_cls = load_class(middleware)
+        if not hasattr(middleware_cls, 'create_instance'):
+            raise MiddlewareInitError(
+                f"Middleware init failed, must inherit from `BaseMiddleware` or have a `create_instance` method"
+            )
+        try:
+            instance = middleware_cls.create_instance(self.crawler)
+            self.middlewares.append(instance)
+            return True
+        except NotConfiguredError:
+            return False
+
+    def _add_method(self):
+        for middleware in self.middlewares:
+            if hasattr(middleware, 'process_request'):
+                if self._validate_middleware_method(method_name='process_request', middleware=middleware):
+                    self.methods['process_request'].append(middleware.process_request)
+            if hasattr(middleware, 'process_response'):
+                if self._validate_middleware_method(method_name='process_response', middleware=middleware):
+                    self.methods['process_response'].append(middleware.process_response)
+            if hasattr(middleware, 'process_exception'):
+                if self._validate_middleware_method(method_name='process_exception', middleware=middleware):
+                    self.methods['process_exception'].append(middleware.process_exception)
+
+    @staticmethod
+    def _validate_middleware_method(method_name, middleware) -> bool:
+        method = getattr(type(middleware), method_name)
+        base_method = getattr(BaseMiddleware, method_name)
+        return False if method == base_method else True

crawlo/middleware/request_ignore.py

@@ -1,30 +1,30 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo.utils.log import get_logger
-from crawlo.exceptions import IgnoreRequestError
-from crawlo.event import ignore_request
-
-
-class RequestIgnoreMiddleware(object):
-
-    def __init__(self, stats, log_level):
-        self.logger = get_logger(self.__class__.__name__, log_level)
-        self.stats = stats
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(stats=crawler.stats, log_level=crawler.settings.get('LOG_LEVEL'))
-        crawler.subscriber.subscribe(o.request_ignore, event=ignore_request)
-        return o
-
-    async def request_ignore(self, exc, request, _spider):
-        self.logger.info(f'{request} ignored.')
-        self.stats.inc_value('request_ignore_count')
-        reason = exc.msg
-        if reason:
-            self.stats.inc_value(f'request_ignore_count/{reason}')
-
-    @staticmethod
-    def process_exception(_request, exc, _spider):
-        if isinstance(exc, IgnoreRequestError):
-            return True
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import IgnoreRequestError
+from crawlo.event import ignore_request
+
+
+class RequestIgnoreMiddleware(object):
+
+    def __init__(self, stats, log_level):
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.stats = stats
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(stats=crawler.stats, log_level=crawler.settings.get('LOG_LEVEL'))
+        crawler.subscriber.subscribe(o.request_ignore, event=ignore_request)
+        return o
+
+    async def request_ignore(self, exc, request, _spider):
+        self.logger.info(f'{request} ignored.')
+        self.stats.inc_value('request_ignore_count')
+        reason = exc.msg
+        if reason:
+            self.stats.inc_value(f'request_ignore_count/{reason}')
+
+    @staticmethod
+    def process_exception(_request, exc, _spider):
+        if isinstance(exc, IgnoreRequestError):
+            return True

crawlo/middleware/response_code.py

@@ -1,19 +1,19 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo.utils.log import get_logger
-
-
-class ResponseCodeMiddleware(object):
-    def __init__(self, stats, log_level):
-        self.logger = get_logger(self.__class__.__name__, log_level)
-        self.stats = stats
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(stats=crawler.stats, log_level=crawler.settings.get('LOG_LEVEL'))
-        return o
-
-    def process_response(self, request, response, spider):
-        self.stats.inc_value(f'stats_code/count/{response.status_code}')
-        self.logger.debug(f'Got response from <{response.status_code} {response.url}>')
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.utils.log import get_logger
+
+
+class ResponseCodeMiddleware(object):
+    def __init__(self, stats, log_level):
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.stats = stats
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(stats=crawler.stats, log_level=crawler.settings.get('LOG_LEVEL'))
+        return o
+
+    def process_response(self, request, response, spider):
+        self.stats.inc_value(f'stats_code/count/{response.status_code}')
+        self.logger.debug(f'Got response from <{response.status_code} {response.url}>')
         return response

crawlo/middleware/response_filter.py

@@ -1,26 +1,26 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo.utils.log import get_logger
-from crawlo.exceptions import IgnoreRequestError
-
-
-class ResponseFilterMiddleware:
-
-    def __init__(self, allowed_codes, log_level):
-        self.allowed_codes = allowed_codes
-        self.logger = get_logger(self.__class__.__name__, log_level)
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(
-            allowed_codes=crawler.settings.get_list('ALLOWED_CODES'),
-            log_level=crawler.settings.get('LOG_LEVEL')
-        )
-        return o
-
-    def process_response(self, request, response, spider):
-        if 200 <= response.status_code < 300:
-            return response
-        if response.status_code in self.allowed_codes:
-            return response
-        raise IgnoreRequestError(f"response status_code/non-200")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import IgnoreRequestError
+
+
+class ResponseFilterMiddleware:
+
+    def __init__(self, allowed_codes, log_level):
+        self.allowed_codes = allowed_codes
+        self.logger = get_logger(self.__class__.__name__, log_level)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(
+            allowed_codes=crawler.settings.get_list('ALLOWED_CODES'),
+            log_level=crawler.settings.get('LOG_LEVEL')
+        )
+        return o
+
+    def process_response(self, request, response, spider):
+        if 200 <= response.status_code < 300:
+            return response
+        if response.status_code in self.allowed_codes:
+            return response
+        raise IgnoreRequestError(f"response status_code/non-200")

crawlo/middleware/retry.py

@@ -1,89 +1,90 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import List
-from anyio import EndOfStream
-from httpcore import ReadError
-from asyncio.exceptions import TimeoutError
-from httpx import RemoteProtocolError, ConnectError, ReadTimeout
-from aiohttp.client_exceptions import ClientConnectionError, ClientPayloadError
-from aiohttp import ClientConnectorError, ClientTimeout, ClientConnectorSSLError, ClientResponseError
-
-from crawlo.utils.log import get_logger
-from crawlo.stats_collector import StatsCollector
-
-_retry_exceptions = [
-    EndOfStream,
-    ReadError,
-    TimeoutError,
-    ConnectError,
-    ReadTimeout,
-    ClientConnectorError,
-    ClientResponseError,
-    RemoteProtocolError,
-    ClientTimeout,
-    ClientConnectorSSLError,
-    ClientPayloadError,
-    ClientConnectionError
-]
-
-
-class RetryMiddleware(object):
-
-    def __init__(
-            self,
-            *,
-            retry_http_codes: List,
-            ignore_http_codes: List,
-            max_retry_times: int,
-            retry_exceptions: List,
-            stats: StatsCollector,
-            retry_priority: int
-    ):
-        self.retry_http_codes = retry_http_codes
-        self.ignore_http_codes = ignore_http_codes
-        self.max_retry_times = max_retry_times
-        self.retry_exceptions = tuple(retry_exceptions + _retry_exceptions)
-        self.retry_priority = retry_priority
-        self.stats = stats
-        self.logger = get_logger(self.__class__.__name__)
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(
-            retry_http_codes=crawler.settings.get_list('RETRY_HTTP_CODES'),
-            ignore_http_codes=crawler.settings.get_list('IGNORE_HTTP_CODES'),
-            max_retry_times=crawler.settings.get_int('MAX_RETRY_TIMES'),
-            retry_exceptions=crawler.settings.get_list('RETRY_EXCEPTIONS'),
-            stats=crawler.stats,
-            retry_priority=crawler.settings.get_int('RETRY_PRIORITY')
-        )
-        return o
-
-    def process_response(self, request, response, spider):
-        if request.meta.get('dont_retry', False):
-            return response
-        if response.status_code in self.ignore_http_codes:
-            return response
-        if response.status_code in self.retry_http_codes:
-            # 重试逻辑
-            reason = f"response code {response.status_code}"
-            return self._retry(request, reason, spider) or response
-        return response
-
-    def process_exception(self, request, exc, spider):
-        if isinstance(exc, self.retry_exceptions) and not request.meta.get('dont_retry', False):
-            return self._retry(request=request, reason=type(exc).__name__, spider=spider)
-
-    def _retry(self, request, reason, spider):
-        retry_times = request.meta.get('retry_times', 0)
-        if retry_times < self.max_retry_times:
-            retry_times += 1
-            self.logger.info(f"{spider} {request} {reason} retrying {retry_times} time...")
-            request.meta['retry_times'] = retry_times
-            request.dont_retry = True
-            request.retry_priority = request.priority + self.retry_priority
-            self.stats.inc_value("retry_count")
-            return request
-        else:
-            self.logger.warning(f"{spider} {request} {reason} retry max {self.max_retry_times} times, give up.")
-            return None
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List
+from anyio import EndOfStream
+from httpcore import ReadError
+from asyncio.exceptions import TimeoutError
+from httpx import RemoteProtocolError, ConnectError, ReadTimeout
+from aiohttp.client_exceptions import ClientConnectionError, ClientPayloadError
+from aiohttp import ClientConnectorError, ClientTimeout, ClientConnectorSSLError, ClientResponseError
+
+from crawlo.utils.log import get_logger
+from crawlo.stats_collector import StatsCollector
+
+_retry_exceptions = [
+    EndOfStream,
+    ReadError,
+    TimeoutError,
+    ConnectError,
+    ReadTimeout,
+    ClientConnectorError,
+    ClientResponseError,
+    RemoteProtocolError,
+    ClientTimeout,
+    ClientConnectorSSLError,
+    ClientPayloadError,
+    ClientConnectionError
+]
+
+
+class RetryMiddleware(object):
+
+    def __init__(
+            self,
+            *,
+            retry_http_codes: List,
+            ignore_http_codes: List,
+            max_retry_times: int,
+            retry_exceptions: List,
+            stats: StatsCollector,
+            retry_priority: int
+    ):
+        self.retry_http_codes = retry_http_codes
+        self.ignore_http_codes = ignore_http_codes
+        self.max_retry_times = max_retry_times
+        self.retry_exceptions = tuple(retry_exceptions + _retry_exceptions)
+        self.retry_priority = retry_priority
+        self.stats = stats
+        self.logger = get_logger(self.__class__.__name__)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls(
+            retry_http_codes=crawler.settings.get_list('RETRY_HTTP_CODES'),
+            ignore_http_codes=crawler.settings.get_list('IGNORE_HTTP_CODES'),
+            max_retry_times=crawler.settings.get_int('MAX_RETRY_TIMES'),
+            retry_exceptions=crawler.settings.get_list('RETRY_EXCEPTIONS'),
+            stats=crawler.stats,
+            retry_priority=crawler.settings.get_int('RETRY_PRIORITY')
+        )
+        return o
+
+    def process_response(self, request, response, spider):
+        if request.meta.get('dont_retry', False):
+            return response
+        if response.status_code in self.ignore_http_codes:
+            return response
+        if response.status_code in self.retry_http_codes:
+            # 重试逻辑
+            reason = f"response code {response.status_code}"
+            return self._retry(request, reason, spider) or response
+        return response
+
+    def process_exception(self, request, exc, spider):
+        if isinstance(exc, self.retry_exceptions) and not request.meta.get('dont_retry', False):
+            return self._retry(request=request, reason=type(exc).__name__, spider=spider)
+
+    def _retry(self, request, reason, spider):
+        retry_times = request.meta.get('retry_times', 0)
+        if retry_times < self.max_retry_times:
+            retry_times += 1
+            self.logger.info(f"{spider} {request} {reason} retrying {retry_times} time...")
+            request.meta['retry_times'] = retry_times
+            # request.dont_retry = True
+            request.meta['dont_retry'] = True
+            request.retry_priority = request.priority + self.retry_priority
+            self.stats.inc_value("retry_count")
+            return request
+        else:
+            self.logger.warning(f"{spider} {request} {reason} retry max {self.max_retry_times} times, give up.")
+            return None

crawlo/network/__init__.py

@@ -1,7 +1,7 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 14:07
-# @Author  :   oscar
-# @Desc    :   None
-"""
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 14:07
+# @Author  :   oscar
+# @Desc    :   None
+"""