crawlo 1.0.0__py3-none-any.whl

crawlo/network/response.py

@@ -0,0 +1,93 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import re
+import ujson
+from typing import Dict
+from parsel import Selector
+from http.cookies import SimpleCookie
+from urllib.parse import urljoin as _urljoin
+
+from crawlo import Request
+from crawlo.exceptions import DecodeError
+
+
+class Response(object):
+
+    def __init__(
+            self,
+            url: str,
+            *,
+            headers: Dict,
+            body: bytes = b"",
+            method: str = 'GET',
+            request: Request = None,
+            status_code: int = 200,
+    ):
+        self.url = url
+        self.headers = headers
+        self.body = body
+        self.method = method
+        self.request = request
+        self.status_code = status_code
+        self.encoding = request.encoding
+        self._selector = None
+        self._text_cache = None
+
+    @property
+    def text(self):
+        # 请求缓存
+        if self._text_cache:
+            return self._text_cache
+        try:
+            self._text_cache = self.body.decode(self.encoding)
+        except UnicodeDecodeError:
+            try:
+                _encoding_re = re.compile(r"charset=([\w-]+)", flags=re.I)
+                _encoding_string = self.headers.get('Content-Type', '') or self.headers.get('content-type', '')
+                _encoding = _encoding_re.search(_encoding_string)
+                if _encoding:
+                    _encoding = _encoding.group(1)
+                    self._text_cache = self.body.decode(_encoding)
+                else:
+                    raise DecodeError(f"{self.request} {self.request.encoding} error.")
+            except UnicodeDecodeError as exp:
+                raise UnicodeDecodeError(
+                    exp.encoding, exp.object, exp.start, exp.end, f"{self.request} error."
+                )
+        return self._text_cache
+
+    def json(self):
+        return ujson.loads(self.text)
+
+    def urljoin(self, url):
+        return _urljoin(self.url, url)
+
+    def xpath(self, xpath_str):
+        if self._selector is None:
+            self._selector = Selector(self.text)
+        return self._selector.xpath(xpath_str)
+
+    def css(self, css_str):
+        if self._selector is None:
+            self._selector = Selector(self.text)
+        return self._selector.css(css_str)
+
+    def re_search(self, pattern, flags=re.DOTALL):
+        return re.search(pattern, self.text, flags=flags)
+
+    def re_findall(self, pattern, flags=re.DOTALL):
+        return re.findall(pattern, self.text, flags=flags)
+
+    def get_cookies(self):
+        cookie_headers = self.headers.getlist('Set-Cookie') or []
+        cookies = SimpleCookie()
+        for header in cookie_headers:
+            cookies.load(header)
+        return {k: v.value for k, v in cookies.items()}
+
+    @property
+    def meta(self):
+        return self.request.meta
+
+    def __str__(self):
+        return f"{self.url} {self.status_code} {self.request.encoding} "

crawlo/pipelines/__init__.py

@@ -0,0 +1,13 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo.items.items import Item
+
+
+class BasePipeline:
+
+    def process_item(self, item: Item, spider):
+        raise NotImplementedError
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls()

crawlo/pipelines/console_pipeline.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo import Item
+from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+
+
+class DebugPipeline:
+
+    def __init__(self, logger):
+        self.logger = logger
+
+    @classmethod
+    def create_instance(cls, crawler):
+        logger = get_logger(cls.__name__, crawler.settings.get('LOG_LEVEL'))
+        return cls(logger)
+
+    async def process_item(self, item: Item, spider: Spider) -> Item:
+        self.logger.debug(item.to_dict())
+        return item

crawlo/pipelines/mongo_pipeline.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+
+
+class MongoPipeline(object):
+    pass

crawlo/pipelines/mysql_pipeline.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+
+
+class MySQLPipeline(object):
+    pass

crawlo/pipelines/pipeline_manager.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List
+from pprint import pformat
+from asyncio import create_task
+
+
+from crawlo.utils.log import get_logger
+from crawlo.event import item_successful, item_discard
+from crawlo.utils.project import load_class, common_call
+from crawlo.exceptions import PipelineInitError, ItemDiscard, InvalidOutputError
+
+
+class PipelineManager:
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.pipelines: List = []
+        self.methods: List = []
+
+        self.logger = get_logger(self.__class__.__name__, self.crawler.settings.get('LOG_LEVEL'))
+        pipelines = self.crawler.settings.get_list('PIPELINES')
+        self._add_pipelines(pipelines)
+        self._add_methods()
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs):
+        o = cls(*args, **kwargs)
+        return o
+
+    def _add_pipelines(self, pipelines):
+        for pipeline in pipelines:
+            pipeline_cls = load_class(pipeline)
+            if not hasattr(pipeline_cls, 'create_instance'):
+                raise PipelineInitError(
+                    f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
+                )
+            self.pipelines.append(pipeline_cls.create_instance(self.crawler))
+        if pipelines:
+            self.logger.info(f"enabled pipelines: \n {pformat(pipelines)}")
+
+    def _add_methods(self):
+        for pipeline in self.pipelines:
+            if hasattr(pipeline, 'process_item'):
+                self.methods.append(pipeline.process_item)
+
+    async def process_item(self, item):
+        try:
+            for method in self.methods:
+                item = await common_call(method, item, self.crawler.spider)
+                if item is None:
+                    raise InvalidOutputError(f"{method.__qualname__} return None is not supported.")
+        except ItemDiscard as exc:
+            create_task(self.crawler.subscriber.notify(item_discard, item, exc, self.crawler.spider))
+        else:
+            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))

crawlo/settings/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-05-11 11:08
+# @Author  :   oscar
+# @Desc    :   None
+"""

crawlo/settings/default_settings.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+
+VERSION = 1.0
+
+# 并发数
+CONCURRENCY = 8
+
+# 下载超时时长
+DOWNLOAD_TIMEOUT = 60
+
+INTERVAL = 5
+
+# --------------------------------------------------- delay ------------------------------------------------------------
+# 下载延迟，默认关闭
+DOWNLOAD_DELAY = 0
+# 下载延迟范围
+RANDOM_RANGE = (0.75, 1.25)
+# 是否需要随机
+RANDOMNESS = True
+
+# --------------------------------------------------- retry ------------------------------------------------------------
+MAX_RETRY_TIMES = 2
+IGNORE_HTTP_CODES = [403, 404]
+RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524]
+# 允许通过的状态码
+ALLOWED_CODES = []
+
+STATS_DUMP = True
+# ssl 验证
+VERIFY_SSL = True
+# 是否使用同一个session
+USE_SESSION = True
+# 日志级别
+LOG_LEVEL = 'DEBUG'
+# 选择下载器
+DOWNLOADER = "crawlo.downloader.aiohttp_downloader.AioHttpDownloader"  # HttpXDownloader
+
+EXTENSIONS = []

crawlo/settings/setting_manager.py

@@ -0,0 +1,100 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+import json
+from copy import deepcopy
+from importlib import import_module
+from collections.abc import MutableMapping
+
+from crawlo.settings import default_settings
+
+
+class SettingManager(MutableMapping):
+
+    def __init__(self, values=None):
+        self.attributes = {}
+        self.set_settings(default_settings)
+        self.update_attributes(values)
+
+    def get(self, key, default=None):
+        """安全获取值，不触发递归"""
+        value = self.attributes.get(key, default)
+        return value if value is not None else default
+
+    def get_int(self, key, default=0):
+        return int(self.get(key, default=default))
+
+    def get_float(self, key, default=0.0):
+        return float(self.get(key, default=default))
+
+    def get_bool(self, key, default=False):
+        got = self.get(key, default=default)
+        if isinstance(got, bool):
+            return got
+        if isinstance(got, (int, float)):
+            return bool(got)
+        got_lower = str(got).strip().lower()
+        if got_lower in ('1', 'true'):
+            return True
+        if got_lower in ('0', 'false'):
+            return False
+        raise ValueError(
+            f"Unsupported value for boolean setting: {got}. "
+            "Supported values are: 0/1, True/False, '0'/'1', 'True'/'False' (case-insensitive)."
+        )
+
+    def get_list(self, key, default=None):
+        values = self.get(key, default or [])
+        if isinstance(values, str):
+            return [v.strip() for v in values.split(',') if v.strip()]
+        try:
+            return list(values)
+        except TypeError:
+            return [values]
+
+    def get_dict(self, key, default=None):
+        value = self.get(key, default or {})
+        if isinstance(value, str):
+            value = json.loads(value)
+        try:
+            return dict(value)
+        except TypeError:
+            return value
+
+    def set(self, key, value):
+        self.attributes[key] = value
+
+    def set_settings(self, module):
+        if isinstance(module, str):
+            module = import_module(module)
+        for key in dir(module):
+            if key.isupper():
+                self.set(key, getattr(module, key))
+
+    # 实现 MutableMapping 必须的方法
+    def __getitem__(self, item):
+        return self.attributes[item]
+
+    def __setitem__(self, key, value):
+        self.set(key, value)
+
+    def __delitem__(self, key):
+        del self.attributes[key]
+
+    def __iter__(self):
+        return iter(self.attributes)
+
+    def __len__(self):
+        return len(self.attributes)
+
+    def __str__(self):
+        return f'<Settings: {self.attributes}>'
+
+    __repr__ = __str__
+
+    def update_attributes(self, attributes):
+        if attributes is not None:
+            for key, value in attributes.items():
+                self.set(key, value)
+
+    def copy(self):
+        return deepcopy(self)

crawlo/spider/__init__.py

@@ -0,0 +1,36 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo import Request
+
+
+class Spider(object):
+    def __init__(self):
+        if not hasattr(self, 'start_urls'):
+            self.start_urls = []
+        self.crawler = None
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls()
+        o.crawler = crawler
+        return o
+
+    def start_requests(self):
+        if self.start_urls:
+            for url in self.start_urls:
+                yield Request(url=url)
+        else:
+            if hasattr(self, 'start_url') and isinstance(getattr(self, 'start_url'), str):
+                yield Request(getattr(self, 'start_url'))
+
+    def parse(self, response):
+        raise NotImplementedError
+
+    async def spider_opened(self):
+        pass
+
+    async def spider_closed(self):
+        pass
+
+    def __str__(self):
+        return self.__class__.__name__

crawlo/stats_collector.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-05-17 09:57
+# @Author  :   crawl-coder
+# @Desc    :   统计信息收集器
+"""
+from pprint import pformat
+from crawlo.utils.log import get_logger
+
+
+class StatsCollector(object):
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self._dump = self.crawler.settings.get_bool('STATS_DUMP')
+        self._stats = {}
+        self.logger = get_logger(self.__class__.__name__, "INFO")
+
+    def inc_value(self, key, count=1, start=0):
+        self._stats[key] = self._stats.setdefault(key, start) + count
+
+    def get_value(self, key, default=None):
+        return self._stats.get(key, default)
+
+    def get_stats(self):
+        return self._stats
+
+    def set_stats(self, stats):
+        self._stats = stats
+
+    def clear_stats(self):
+        self._stats.clear()
+
+    def close_spider(self, spider_name, reason):
+        self._stats['reason'] = reason
+        if self._dump:
+            self.logger.info(f'{spider_name} stats: \n{pformat(self._stats)}')
+
+    def __getitem__(self, item):
+        return self._stats[item]
+
+    def __setitem__(self, key, value):
+        self._stats[key] = value
+
+    def __delitem__(self, key):
+        del self._stats[key]

crawlo/subscriber.py

@@ -0,0 +1,27 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from collections import defaultdict
+from inspect import iscoroutinefunction
+from typing import Dict, Set, Callable, Coroutine
+
+from crawlo.exceptions import ReceiverTypeError
+
+
+class Subscriber:
+
+    def __init__(self):
+        self._subscribers: Dict[str, Set[Callable[..., Coroutine]]] = defaultdict(set)
+
+    def subscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+        if not iscoroutinefunction(receiver):
+            raise ReceiverTypeError(f"{receiver.__qualname__} must be a coroutine function")
+        self._subscribers[event].add(receiver)
+
+    def unsubscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+        self._subscribers[event].discard(receiver)
+
+    async def notify(self, event: str, *args, **kwargs) -> None:
+        for receiver in self._subscribers[event]:
+            # 不能 await
+            asyncio.create_task(receiver(*args, **kwargs))

crawlo/task_manager.py

@@ -0,0 +1,27 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from asyncio import Task, Future, Semaphore
+from typing import Set, Final
+
+
+class TaskManager:
+
+    def __init__(self, total_concurrency: int = 8):
+        self.current_task: Final[Set] = set()
+        self.semaphore: Semaphore = Semaphore(total_concurrency)
+
+    def create_task(self, coroutine) -> Task:
+        task = asyncio.create_task(coroutine)
+        self.current_task.add(task)
+
+        def done_callback(_future: Future) -> None:
+            self.current_task.remove(task)
+            self.semaphore.release()
+
+        task.add_done_callback(done_callback)
+
+        return task
+
+    def all_done(self) -> bool:
+        return len(self.current_task) == 0

crawlo/templates/item_template.tmpl

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+"""
+Created on {DATE}
+---------
+@summary:
+---------
+@author: {USER}
+"""
+
+from crawlo import Item
+
+
+class ${item_name}Item(Item):
+    """
+    This class was generated by feapder
+    command: feapder create -i ${command}
+    """
+
+    __table_name__ = "${table_name}"
+
+    def __init__(self, *args, **kwargs):
+        ${propertys}

crawlo/templates/project_template/main.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+Created on {DATE}
+---------
+@summary: 爬虫入口
+---------
+@author: {USER}
+"""
+
+from crawlo import ArgumentParser
+
+from spiders import *
+
+
+
+def crawl_xxx():
+    """
+    Spider爬虫
+    """
+    spider = xxx.XXXSpider(redis_key="xxx:xxx")
+    spider.start()
+
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description="xxx爬虫")
+
+    parser.add_argument(
+        "--crawl_xxx", action="store_true", help="xxx爬虫", function=crawl_xxx
+    )
+    parser.start()
+
+    # main.py作为爬虫启动的统一入口，提供命令行的方式启动多个爬虫