crawlo 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

crawlo/spider/__init__.py

@@ -1,129 +1,129 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from __future__ import annotations
-from typing import Type, Any, Optional, List, Dict
-from ..network.request import Request
-from ..utils.log import get_logger
-
-
-# 全局注册表
-_DEFAULT_SPIDER_REGISTRY: dict[str, Type[Spider]] = {}
-
-
-class SpiderMeta(type):
-    def __new__(mcs, name: str, bases: tuple[type], namespace: dict[str, Any], **kwargs):
-        cls = super().__new__(mcs, name, bases, namespace)
-
-        is_spider_subclass = any(
-            base is Spider or (isinstance(base, type) and issubclass(base, Spider))
-            for base in bases
-        )
-        if not is_spider_subclass:
-            return cls
-
-        spider_name = namespace.get('name')
-        if not isinstance(spider_name, str):
-            raise AttributeError(f"爬虫类 '{cls.__name__}' 必须定义字符串类型的 'name' 属性。")
-
-        if spider_name in _DEFAULT_SPIDER_REGISTRY:
-            raise ValueError(
-                f"爬虫名称 '{spider_name}' 已被 {_DEFAULT_SPIDER_REGISTRY[spider_name].__name__} 占用。"
-                f"请确保每个爬虫的 name 属性全局唯一。"
-            )
-
-        _DEFAULT_SPIDER_REGISTRY[spider_name] = cls
-        get_logger(__name__).debug(f"自动注册爬虫: {spider_name} -> {cls.__name__}")
-
-        return cls
-
-
-class Spider(metaclass=SpiderMeta):
-    name: str = None
-
-    def __init__(self, name=None, **kwargs):
-        if not hasattr(self, 'start_urls'):
-            self.start_urls = []
-        self.crawler = None
-        self.name = name or self.name
-        self.logger = get_logger(self.name or self.__class__.__name__)
-
-    @classmethod
-    def create_instance(cls, crawler) -> Spider:
-        o = cls()
-        o.crawler = crawler
-        return o
-
-    def start_requests(self):
-        if self.start_urls:
-            for url in self.start_urls:
-                yield Request(url=url, dont_filter=True)
-        else:
-            if hasattr(self, 'start_url') and isinstance(getattr(self, 'start_url'), str):
-                yield Request(getattr(self, 'start_url'), dont_filter=True)
-
-    def parse(self, response):
-        raise NotImplementedError
-
-    async def spider_opened(self):
-        pass
-
-    async def spider_closed(self):
-        pass
-
-    def __str__(self):
-        return self.__class__.__name__
-
-
-# === 公共只读接口 ===
-def get_global_spider_registry() -> dict[str, Type[Spider]]:
-    return _DEFAULT_SPIDER_REGISTRY.copy()
-
-
-def get_spider_by_name(name: str) -> Optional[Type[Spider]]:
-    return _DEFAULT_SPIDER_REGISTRY.get(name)
-
-
-def get_all_spider_classes() -> list[Type[Spider]]:
-    return list(set(_DEFAULT_SPIDER_REGISTRY.values()))
-
-# #!/usr/bin/python
-# # -*- coding:UTF-8 -*-
-# from ..network.request import Request
-# from ..utils.log import get_logger
-#
-#
-# class Spider(object):
-#     name = None
-#
-#     def __init__(self, name=None, **kwargs):
-#         if not hasattr(self, 'start_urls'):
-#             self.start_urls = []
-#         self.crawler = None
-#         self.name = name or self.name
-#         self.logger = get_logger(self.name or self.__class__.__name__)
-#
-#     @classmethod
-#     def create_instance(cls, crawler):
-#         o = cls()
-#         o.crawler = crawler
-#         return o
-#
-#     def start_requests(self):
-#         if self.start_urls:
-#             for url in self.start_urls:
-#                 yield Request(url=url, dont_filter=True)
-#         else:
-#             if hasattr(self, 'start_url') and isinstance(getattr(self, 'start_url'), str):
-#                 yield Request(getattr(self, 'start_url'), dont_filter=True)
-#
-#     def parse(self, response):
-#         raise NotImplementedError
-#
-#     async def spider_opened(self):
-#         pass
-#
-#     async def spider_closed(self):
-#         pass
-#
-#     def __str__(self):
-#         return self.__class__.__name__
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from __future__ import annotations
+from typing import Type, Any, Optional, List, Dict
+from ..network.request import Request
+from ..utils.log import get_logger
+
+
+# 全局注册表
+_DEFAULT_SPIDER_REGISTRY: dict[str, Type[Spider]] = {}
+
+
+class SpiderMeta(type):
+    def __new__(mcs, name: str, bases: tuple[type], namespace: dict[str, Any], **kwargs):
+        cls = super().__new__(mcs, name, bases, namespace)
+
+        is_spider_subclass = any(
+            base is Spider or (isinstance(base, type) and issubclass(base, Spider))
+            for base in bases
+        )
+        if not is_spider_subclass:
+            return cls
+
+        spider_name = namespace.get('name')
+        if not isinstance(spider_name, str):
+            raise AttributeError(f"爬虫类 '{cls.__name__}' 必须定义字符串类型的 'name' 属性。")
+
+        if spider_name in _DEFAULT_SPIDER_REGISTRY:
+            raise ValueError(
+                f"爬虫名称 '{spider_name}' 已被 {_DEFAULT_SPIDER_REGISTRY[spider_name].__name__} 占用。"
+                f"请确保每个爬虫的 name 属性全局唯一。"
+            )
+
+        _DEFAULT_SPIDER_REGISTRY[spider_name] = cls
+        get_logger(__name__).debug(f"自动注册爬虫: {spider_name} -> {cls.__name__}")
+
+        return cls
+
+
+class Spider(metaclass=SpiderMeta):
+    name: str = None
+
+    def __init__(self, name=None, **kwargs):
+        if not hasattr(self, 'start_urls'):
+            self.start_urls = []
+        self.crawler = None
+        self.name = name or self.name
+        self.logger = get_logger(self.name or self.__class__.__name__)
+
+    @classmethod
+    def create_instance(cls, crawler) -> Spider:
+        o = cls()
+        o.crawler = crawler
+        return o
+
+    def start_requests(self):
+        if self.start_urls:
+            for url in self.start_urls:
+                yield Request(url=url, dont_filter=True)
+        else:
+            if hasattr(self, 'start_url') and isinstance(getattr(self, 'start_url'), str):
+                yield Request(getattr(self, 'start_url'), dont_filter=True)
+
+    def parse(self, response):
+        raise NotImplementedError
+
+    async def spider_opened(self):
+        pass
+
+    async def spider_closed(self):
+        pass
+
+    def __str__(self):
+        return self.__class__.__name__
+
+
+# === 公共只读接口 ===
+def get_global_spider_registry() -> dict[str, Type[Spider]]:
+    return _DEFAULT_SPIDER_REGISTRY.copy()
+
+
+def get_spider_by_name(name: str) -> Optional[Type[Spider]]:
+    return _DEFAULT_SPIDER_REGISTRY.get(name)
+
+
+def get_all_spider_classes() -> list[Type[Spider]]:
+    return list(set(_DEFAULT_SPIDER_REGISTRY.values()))
+
+# #!/usr/bin/python
+# # -*- coding:UTF-8 -*-
+# from ..network.request import Request
+# from ..utils.log import get_logger
+#
+#
+# class Spider(object):
+#     name = None
+#
+#     def __init__(self, name=None, **kwargs):
+#         if not hasattr(self, 'start_urls'):
+#             self.start_urls = []
+#         self.crawler = None
+#         self.name = name or self.name
+#         self.logger = get_logger(self.name or self.__class__.__name__)
+#
+#     @classmethod
+#     def create_instance(cls, crawler):
+#         o = cls()
+#         o.crawler = crawler
+#         return o
+#
+#     def start_requests(self):
+#         if self.start_urls:
+#             for url in self.start_urls:
+#                 yield Request(url=url, dont_filter=True)
+#         else:
+#             if hasattr(self, 'start_url') and isinstance(getattr(self, 'start_url'), str):
+#                 yield Request(getattr(self, 'start_url'), dont_filter=True)
+#
+#     def parse(self, response):
+#         raise NotImplementedError
+#
+#     async def spider_opened(self):
+#         pass
+#
+#     async def spider_closed(self):
+#         pass
+#
+#     def __str__(self):
+#         return self.__class__.__name__

crawlo/stats_collector.py

@@ -1,59 +1,59 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-05-17 09:57
-# @Author  :   crawl-coder
-# @Desc    :   统计信息收集器
-"""
-from pprint import pformat
-from crawlo.utils.log import get_logger
-
-
-class StatsCollector(object):
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self._dump = self.crawler.settings.get_bool('STATS_DUMP')
-        self._stats = {}
-        self.logger = get_logger(self.__class__.__name__, "INFO")
-
-    def inc_value(self, key, count=1, start=0):
-        self._stats[key] = self._stats.setdefault(key, start) + count
-
-    def get_value(self, key, default=None):
-        return self._stats.get(key, default)
-
-    def get_stats(self):
-        return self._stats
-
-    def set_stats(self, stats):
-        self._stats = stats
-
-    def clear_stats(self):
-        self._stats.clear()
-
-    def close_spider(self, spider, reason):
-        self._stats['reason'] = reason
-
-        # 首选：使用 spider.name
-        # 次选：使用实例的类名
-        # 最后：使用一个完全未知的占位符
-        spider_name = (
-                getattr(spider, 'name', None) or
-                spider.__class__.__name__ or
-                '<Unknown>'
-        )
-
-        self._stats['spider_name'] = spider_name
-
-        if self._dump:
-            self.logger.info(f'{spider_name} stats: \n{pformat(self._stats)}')
-
-    def __getitem__(self, item):
-        return self._stats[item]
-
-    def __setitem__(self, key, value):
-        self._stats[key] = value
-
-    def __delitem__(self, key):
-        del self._stats[key]
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-05-17 09:57
+# @Author  :   crawl-coder
+# @Desc    :   统计信息收集器
+"""
+from pprint import pformat
+from crawlo.utils.log import get_logger
+
+
+class StatsCollector(object):
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self._dump = self.crawler.settings.get_bool('STATS_DUMP')
+        self._stats = {}
+        self.logger = get_logger(self.__class__.__name__, "INFO")
+
+    def inc_value(self, key, count=1, start=0):
+        self._stats[key] = self._stats.setdefault(key, start) + count
+
+    def get_value(self, key, default=None):
+        return self._stats.get(key, default)
+
+    def get_stats(self):
+        return self._stats
+
+    def set_stats(self, stats):
+        self._stats = stats
+
+    def clear_stats(self):
+        self._stats.clear()
+
+    def close_spider(self, spider, reason):
+        self._stats['reason'] = reason
+
+        # 首选：使用 spider.name
+        # 次选：使用实例的类名
+        # 最后：使用一个完全未知的占位符
+        spider_name = (
+                getattr(spider, 'name', None) or
+                spider.__class__.__name__ or
+                '<Unknown>'
+        )
+
+        self._stats['spider_name'] = spider_name
+
+        if self._dump:
+            self.logger.info(f'{spider_name} stats: \n{pformat(self._stats)}')
+
+    def __getitem__(self, item):
+        return self._stats[item]
+
+    def __setitem__(self, key, value):
+        self._stats[key] = value
+
+    def __delitem__(self, key):
+        del self._stats[key]

crawlo/subscriber.py

@@ -1,106 +1,106 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from collections import defaultdict
-from inspect import iscoroutinefunction
-from typing import Dict, Set, Callable, Coroutine, Any, TypeAlias, List
-
-
-class ReceiverTypeError(TypeError):
-    """当订阅的接收者不是一个协程函数时抛出。"""
-    pass
-
-
-ReceiverCoroutine: TypeAlias = Callable[..., Coroutine[Any, Any, Any]]
-
-
-class Subscriber:
-    """
-    一个支持异步协程的发布/订阅（Pub/Sub）模式实现。
-
-    这个类允许你注册（订阅）协程函数来监听特定事件，并在事件发生时
-    以并发的方式异步地通知所有订阅者。
-    """
-
-    def __init__(self):
-        """初始化一个空的订阅者字典。"""
-        self._subscribers: Dict[str, Set[ReceiverCoroutine]] = defaultdict(set)
-
-    def subscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
-        """
-        订阅一个事件。
-
-        Args:
-            receiver: 一个协程函数 (例如 async def my_func(...))。
-            event: 要订阅的事件名称。
-
-        Raises:
-            ReceiverTypeError: 如果提供的 `receiver` 不是一个协程函数。
-        """
-        if not iscoroutinefunction(receiver):
-            raise ReceiverTypeError(f"接收者 '{receiver.__qualname__}' 必须是一个协程函数。")
-        self._subscribers[event].add(receiver)
-
-    def unsubscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
-        """
-        取消订阅一个事件。
-
-        如果事件或接收者不存在，将静默处理。
-
-        Args:
-            receiver: 要取消订阅的协程函数。
-            event: 事件名称。
-        """
-        if event in self._subscribers:
-            self._subscribers[event].discard(receiver)
-
-    async def notify(self, event: str, *args, **kwargs) -> List[Any]:
-        """
-        异步地、并发地通知所有订阅了该事件的接收者。
-
-        此方法会等待所有订阅者任务完成后再返回，并收集所有结果或异常。
-
-        Args:
-            event: 要触发的事件名称。
-            *args: 传递给接收者的位置参数。
-            **kwargs: 传递给接收者的关键字参数。
-
-        Returns:
-            一个列表，包含每个订阅者任务的返回结果或在执行期间捕获的异常。
-        """
-        receivers = self._subscribers.get(event, set())
-        if not receivers:
-            return []
-
-        tasks = [asyncio.create_task(receiver(*args, **kwargs)) for receiver in receivers]
-
-        # 并发执行所有任务并返回结果列表（包括异常）
-        return await asyncio.gather(*tasks, return_exceptions=True)
-
-# #!/usr/bin/python
-# # -*- coding:UTF-8 -*-
-# import asyncio
-# from collections import defaultdict
-# from inspect import iscoroutinefunction
-# from typing import Dict, Set, Callable, Coroutine
-#
-# from crawlo.exceptions import ReceiverTypeError
-#
-#
-# class Subscriber:
-#
-#     def __init__(self):
-#         self._subscribers: Dict[str, Set[Callable[..., Coroutine]]] = defaultdict(set)
-#
-#     def subscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
-#         if not iscoroutinefunction(receiver):
-#             raise ReceiverTypeError(f"{receiver.__qualname__} must be a coroutine function")
-#         self._subscribers[event].add(receiver)
-#
-#     def unsubscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
-#         self._subscribers[event].discard(receiver)
-#
-#     async def notify(self, event: str, *args, **kwargs) -> None:
-#         for receiver in self._subscribers[event]:
-#             # 不能 await
-#             asyncio.create_task(receiver(*args, **kwargs))
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from collections import defaultdict
+from inspect import iscoroutinefunction
+from typing import Dict, Set, Callable, Coroutine, Any, TypeAlias, List
+
+
+class ReceiverTypeError(TypeError):
+    """当订阅的接收者不是一个协程函数时抛出。"""
+    pass
+
+
+ReceiverCoroutine: TypeAlias = Callable[..., Coroutine[Any, Any, Any]]
+
+
+class Subscriber:
+    """
+    一个支持异步协程的发布/订阅（Pub/Sub）模式实现。
+
+    这个类允许你注册（订阅）协程函数来监听特定事件，并在事件发生时
+    以并发的方式异步地通知所有订阅者。
+    """
+
+    def __init__(self):
+        """初始化一个空的订阅者字典。"""
+        self._subscribers: Dict[str, Set[ReceiverCoroutine]] = defaultdict(set)
+
+    def subscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
+        """
+        订阅一个事件。
+
+        Args:
+            receiver: 一个协程函数 (例如 async def my_func(...))。
+            event: 要订阅的事件名称。
+
+        Raises:
+            ReceiverTypeError: 如果提供的 `receiver` 不是一个协程函数。
+        """
+        if not iscoroutinefunction(receiver):
+            raise ReceiverTypeError(f"接收者 '{receiver.__qualname__}' 必须是一个协程函数。")
+        self._subscribers[event].add(receiver)
+
+    def unsubscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
+        """
+        取消订阅一个事件。
+
+        如果事件或接收者不存在，将静默处理。
+
+        Args:
+            receiver: 要取消订阅的协程函数。
+            event: 事件名称。
+        """
+        if event in self._subscribers:
+            self._subscribers[event].discard(receiver)
+
+    async def notify(self, event: str, *args, **kwargs) -> List[Any]:
+        """
+        异步地、并发地通知所有订阅了该事件的接收者。
+
+        此方法会等待所有订阅者任务完成后再返回，并收集所有结果或异常。
+
+        Args:
+            event: 要触发的事件名称。
+            *args: 传递给接收者的位置参数。
+            **kwargs: 传递给接收者的关键字参数。
+
+        Returns:
+            一个列表，包含每个订阅者任务的返回结果或在执行期间捕获的异常。
+        """
+        receivers = self._subscribers.get(event, set())
+        if not receivers:
+            return []
+
+        tasks = [asyncio.create_task(receiver(*args, **kwargs)) for receiver in receivers]
+
+        # 并发执行所有任务并返回结果列表（包括异常）
+        return await asyncio.gather(*tasks, return_exceptions=True)
+
+# #!/usr/bin/python
+# # -*- coding:UTF-8 -*-
+# import asyncio
+# from collections import defaultdict
+# from inspect import iscoroutinefunction
+# from typing import Dict, Set, Callable, Coroutine
+#
+# from crawlo.exceptions import ReceiverTypeError
+#
+#
+# class Subscriber:
+#
+#     def __init__(self):
+#         self._subscribers: Dict[str, Set[Callable[..., Coroutine]]] = defaultdict(set)
+#
+#     def subscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+#         if not iscoroutinefunction(receiver):
+#             raise ReceiverTypeError(f"{receiver.__qualname__} must be a coroutine function")
+#         self._subscribers[event].add(receiver)
+#
+#     def unsubscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+#         self._subscribers[event].discard(receiver)
+#
+#     async def notify(self, event: str, *args, **kwargs) -> None:
+#         for receiver in self._subscribers[event]:
+#             # 不能 await
+#             asyncio.create_task(receiver(*args, **kwargs))

crawlo/task_manager.py

@@ -1,27 +1,27 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from asyncio import Task, Future, Semaphore
-from typing import Set, Final
-
-
-class TaskManager:
-
-    def __init__(self, total_concurrency: int = 8):
-        self.current_task: Final[Set] = set()
-        self.semaphore: Semaphore = Semaphore(total_concurrency)
-
-    def create_task(self, coroutine) -> Task:
-        task = asyncio.create_task(coroutine)
-        self.current_task.add(task)
-
-        def done_callback(_future: Future) -> None:
-            self.current_task.remove(task)
-            self.semaphore.release()
-
-        task.add_done_callback(done_callback)
-
-        return task
-
-    def all_done(self) -> bool:
-        return len(self.current_task) == 0
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from asyncio import Task, Future, Semaphore
+from typing import Set, Final
+
+
+class TaskManager:
+
+    def __init__(self, total_concurrency: int = 8):
+        self.current_task: Final[Set] = set()
+        self.semaphore: Semaphore = Semaphore(total_concurrency)
+
+    def create_task(self, coroutine) -> Task:
+        task = asyncio.create_task(coroutine)
+        self.current_task.add(task)
+
+        def done_callback(_future: Future) -> None:
+            self.current_task.remove(task)
+            self.semaphore.release()
+
+        task.add_done_callback(done_callback)
+
+        return task
+
+    def all_done(self) -> bool:
+        return len(self.current_task) == 0

crawlo/templates/crawlo.cfg.tmpl

@@ -1,11 +1,11 @@
-# crawlo.cfg
-# 项目的配置文件。
-
-[settings]
-# 指定 settings 模块的导入路径
-default = {{project_name}}.settings
-
-[deploy]
-# （可选）用于部署配置
-# url = http://localhost:6800/
+# crawlo.cfg
+# 项目的配置文件。
+
+[settings]
+# 指定 settings 模块的导入路径
+default = {{project_name}}.settings
+
+[deploy]
+# （可选）用于部署配置
+# url = http://localhost:6800/
 # project = {{project_name}}