crawlo 1.0.2__py3-none-any.whl → 1.0.3__py3-none-any.whl

crawlo/subscriber.py

@@ -1,106 +1,106 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from collections import defaultdict
-from inspect import iscoroutinefunction
-from typing import Dict, Set, Callable, Coroutine, Any, TypeAlias, List
-
-
-class ReceiverTypeError(TypeError):
-    """当订阅的接收者不是一个协程函数时抛出。"""
-    pass
-
-
-ReceiverCoroutine: TypeAlias = Callable[..., Coroutine[Any, Any, Any]]
-
-
-class Subscriber:
-    """
-    一个支持异步协程的发布/订阅（Pub/Sub）模式实现。
-
-    这个类允许你注册（订阅）协程函数来监听特定事件，并在事件发生时
-    以并发的方式异步地通知所有订阅者。
-    """
-
-    def __init__(self):
-        """初始化一个空的订阅者字典。"""
-        self._subscribers: Dict[str, Set[ReceiverCoroutine]] = defaultdict(set)
-
-    def subscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
-        """
-        订阅一个事件。
-
-        Args:
-            receiver: 一个协程函数 (例如 async def my_func(...))。
-            event: 要订阅的事件名称。
-
-        Raises:
-            ReceiverTypeError: 如果提供的 `receiver` 不是一个协程函数。
-        """
-        if not iscoroutinefunction(receiver):
-            raise ReceiverTypeError(f"接收者 '{receiver.__qualname__}' 必须是一个协程函数。")
-        self._subscribers[event].add(receiver)
-
-    def unsubscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
-        """
-        取消订阅一个事件。
-
-        如果事件或接收者不存在，将静默处理。
-
-        Args:
-            receiver: 要取消订阅的协程函数。
-            event: 事件名称。
-        """
-        if event in self._subscribers:
-            self._subscribers[event].discard(receiver)
-
-    async def notify(self, event: str, *args, **kwargs) -> List[Any]:
-        """
-        异步地、并发地通知所有订阅了该事件的接收者。
-
-        此方法会等待所有订阅者任务完成后再返回，并收集所有结果或异常。
-
-        Args:
-            event: 要触发的事件名称。
-            *args: 传递给接收者的位置参数。
-            **kwargs: 传递给接收者的关键字参数。
-
-        Returns:
-            一个列表，包含每个订阅者任务的返回结果或在执行期间捕获的异常。
-        """
-        receivers = self._subscribers.get(event, set())
-        if not receivers:
-            return []
-
-        tasks = [asyncio.create_task(receiver(*args, **kwargs)) for receiver in receivers]
-
-        # 并发执行所有任务并返回结果列表（包括异常）
-        return await asyncio.gather(*tasks, return_exceptions=True)
-
-# #!/usr/bin/python
-# # -*- coding:UTF-8 -*-
-# import asyncio
-# from collections import defaultdict
-# from inspect import iscoroutinefunction
-# from typing import Dict, Set, Callable, Coroutine
-#
-# from crawlo.exceptions import ReceiverTypeError
-#
-#
-# class Subscriber:
-#
-#     def __init__(self):
-#         self._subscribers: Dict[str, Set[Callable[..., Coroutine]]] = defaultdict(set)
-#
-#     def subscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
-#         if not iscoroutinefunction(receiver):
-#             raise ReceiverTypeError(f"{receiver.__qualname__} must be a coroutine function")
-#         self._subscribers[event].add(receiver)
-#
-#     def unsubscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
-#         self._subscribers[event].discard(receiver)
-#
-#     async def notify(self, event: str, *args, **kwargs) -> None:
-#         for receiver in self._subscribers[event]:
-#             # 不能 await
-#             asyncio.create_task(receiver(*args, **kwargs))
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from collections import defaultdict
+from inspect import iscoroutinefunction
+from typing import Dict, Set, Callable, Coroutine, Any, TypeAlias, List
+
+
+class ReceiverTypeError(TypeError):
+    """当订阅的接收者不是一个协程函数时抛出。"""
+    pass
+
+
+ReceiverCoroutine: TypeAlias = Callable[..., Coroutine[Any, Any, Any]]
+
+
+class Subscriber:
+    """
+    一个支持异步协程的发布/订阅（Pub/Sub）模式实现。
+
+    这个类允许你注册（订阅）协程函数来监听特定事件，并在事件发生时
+    以并发的方式异步地通知所有订阅者。
+    """
+
+    def __init__(self):
+        """初始化一个空的订阅者字典。"""
+        self._subscribers: Dict[str, Set[ReceiverCoroutine]] = defaultdict(set)
+
+    def subscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
+        """
+        订阅一个事件。
+
+        Args:
+            receiver: 一个协程函数 (例如 async def my_func(...))。
+            event: 要订阅的事件名称。
+
+        Raises:
+            ReceiverTypeError: 如果提供的 `receiver` 不是一个协程函数。
+        """
+        if not iscoroutinefunction(receiver):
+            raise ReceiverTypeError(f"接收者 '{receiver.__qualname__}' 必须是一个协程函数。")
+        self._subscribers[event].add(receiver)
+
+    def unsubscribe(self, receiver: ReceiverCoroutine, *, event: str) -> None:
+        """
+        取消订阅一个事件。
+
+        如果事件或接收者不存在，将静默处理。
+
+        Args:
+            receiver: 要取消订阅的协程函数。
+            event: 事件名称。
+        """
+        if event in self._subscribers:
+            self._subscribers[event].discard(receiver)
+
+    async def notify(self, event: str, *args, **kwargs) -> List[Any]:
+        """
+        异步地、并发地通知所有订阅了该事件的接收者。
+
+        此方法会等待所有订阅者任务完成后再返回，并收集所有结果或异常。
+
+        Args:
+            event: 要触发的事件名称。
+            *args: 传递给接收者的位置参数。
+            **kwargs: 传递给接收者的关键字参数。
+
+        Returns:
+            一个列表，包含每个订阅者任务的返回结果或在执行期间捕获的异常。
+        """
+        receivers = self._subscribers.get(event, set())
+        if not receivers:
+            return []
+
+        tasks = [asyncio.create_task(receiver(*args, **kwargs)) for receiver in receivers]
+
+        # 并发执行所有任务并返回结果列表（包括异常）
+        return await asyncio.gather(*tasks, return_exceptions=True)
+
+# #!/usr/bin/python
+# # -*- coding:UTF-8 -*-
+# import asyncio
+# from collections import defaultdict
+# from inspect import iscoroutinefunction
+# from typing import Dict, Set, Callable, Coroutine
+#
+# from crawlo.exceptions import ReceiverTypeError
+#
+#
+# class Subscriber:
+#
+#     def __init__(self):
+#         self._subscribers: Dict[str, Set[Callable[..., Coroutine]]] = defaultdict(set)
+#
+#     def subscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+#         if not iscoroutinefunction(receiver):
+#             raise ReceiverTypeError(f"{receiver.__qualname__} must be a coroutine function")
+#         self._subscribers[event].add(receiver)
+#
+#     def unsubscribe(self, receiver: Callable[..., Coroutine], *, event: str) -> None:
+#         self._subscribers[event].discard(receiver)
+#
+#     async def notify(self, event: str, *args, **kwargs) -> None:
+#         for receiver in self._subscribers[event]:
+#             # 不能 await
+#             asyncio.create_task(receiver(*args, **kwargs))

crawlo/task_manager.py

@@ -1,27 +1,27 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from asyncio import Task, Future, Semaphore
-from typing import Set, Final
-
-
-class TaskManager:
-
-    def __init__(self, total_concurrency: int = 8):
-        self.current_task: Final[Set] = set()
-        self.semaphore: Semaphore = Semaphore(total_concurrency)
-
-    def create_task(self, coroutine) -> Task:
-        task = asyncio.create_task(coroutine)
-        self.current_task.add(task)
-
-        def done_callback(_future: Future) -> None:
-            self.current_task.remove(task)
-            self.semaphore.release()
-
-        task.add_done_callback(done_callback)
-
-        return task
-
-    def all_done(self) -> bool:
-        return len(self.current_task) == 0
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from asyncio import Task, Future, Semaphore
+from typing import Set, Final
+
+
+class TaskManager:
+
+    def __init__(self, total_concurrency: int = 8):
+        self.current_task: Final[Set] = set()
+        self.semaphore: Semaphore = Semaphore(total_concurrency)
+
+    def create_task(self, coroutine) -> Task:
+        task = asyncio.create_task(coroutine)
+        self.current_task.add(task)
+
+        def done_callback(_future: Future) -> None:
+            self.current_task.remove(task)
+            self.semaphore.release()
+
+        task.add_done_callback(done_callback)
+
+        return task
+
+    def all_done(self) -> bool:
+        return len(self.current_task) == 0

crawlo/templates/item_template.tmpl

@@ -1,22 +1,22 @@
-# -*- coding: utf-8 -*-
-"""
-Created on {DATE}
----------
-@summary:
----------
-@author: {USER}
-"""
-
-from crawlo import Item
-
-
-class ${item_name}Item(Item):
-    """
-    This class was generated by feapder
-    command: feapder create -i ${command}
-    """
-
-    __table_name__ = "${table_name}"
-
-    def __init__(self, *args, **kwargs):
+# -*- coding: utf-8 -*-
+"""
+Created on {DATE}
+---------
+@summary:
+---------
+@author: {USER}
+"""
+
+from crawlo import Item
+
+
+class ${item_name}Item(Item):
+    """
+    This class was generated by feapder
+    command: feapder create -i ${command}
+    """
+
+    __table_name__ = "${table_name}"
+
+    def __init__(self, *args, **kwargs):
         ${propertys}

crawlo/templates/project_template/main.py

@@ -1,33 +1,33 @@
-# -*- coding: utf-8 -*-
-"""
-Created on {DATE}
----------
-@summary: 爬虫入口
----------
-@author: {USER}
-"""
-
-from crawlo import ArgumentParser
-
-from spiders import *
-
-
-
-def crawl_xxx():
-    """
-    Spider爬虫
-    """
-    spider = xxx.XXXSpider(redis_key="xxx:xxx")
-    spider.start()
-
-
-
-if __name__ == "__main__":
-    parser = ArgumentParser(description="xxx爬虫")
-
-    parser.add_argument(
-        "--crawl_xxx", action="store_true", help="xxx爬虫", function=crawl_xxx
-    )
-    parser.start()
-
+# -*- coding: utf-8 -*-
+"""
+Created on {DATE}
+---------
+@summary: 爬虫入口
+---------
+@author: {USER}
+"""
+
+from crawlo import ArgumentParser
+
+from spiders import *
+
+
+
+def crawl_xxx():
+    """
+    Spider爬虫
+    """
+    spider = xxx.XXXSpider(redis_key="xxx:xxx")
+    spider.start()
+
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description="xxx爬虫")
+
+    parser.add_argument(
+        "--crawl_xxx", action="store_true", help="xxx爬虫", function=crawl_xxx
+    )
+    parser.start()
+
     # main.py作为爬虫启动的统一入口，提供命令行的方式启动多个爬虫