coocan 0.5.2.1__tar.gz → 0.5.3__tar.gz

{coocan-0.5.2.1 → coocan-0.5.3}/PKG-INFO

@@ -1,7 +1,8 @@
 Metadata-Version: 2.1
 Name: coocan
-Version: 0.5.2.1
+Version: 0.5.3
 Summary: Air Spider Framework
+Home-page: https://github.com/markadc/coocan
 Author: wauo
 Author-email: markadc@126.com
 Requires-Python: >=3.10

{coocan-0.5.2.1 → coocan-0.5.3}/coocan/cmd/cli.py

@@ -28,7 +28,7 @@ def snake_to_pascal(snake_str: str):
 def main(ctx):
     if ctx.invoked_subcommand is None:
         print(help_info)
-        click.echo("cc new -s <spider_file_name>")
+        click.echo("coocan new -s <spider_file_name>")
 
 
 @main.command()

{coocan-0.5.2.1 → coocan-0.5.3}/coocan/push_project.py

@@ -2,6 +2,7 @@ import os
 import sys
 
 msg = sys.argv[1] if len(sys.argv) == 2 else "Auto Submit"
+msg = sys.argv[1] if len(sys.argv) == 2 else "更新爬虫示例（接收爬虫数据）"
 
 cmd1 = "git add ."
 cmd2 = 'git commit -m "{}"'.format(msg)

{coocan-0.5.2.1 → coocan-0.5.3}/coocan/spider/base.py

@@ -24,6 +24,7 @@ class MiniSpider:
     enable_random_ua = True
     headers_extra_field = {}
     delay = 0
+    item_speed = 100
 
     def start_requests(self):
         """初始请求"""
@@ -55,10 +56,10 @@ class MiniSpider:
     def handle_callback_excetpion(self, e: Exception, request: Request, response: Response):
         logger.error("{} `回调`时出现异常 | {} | {} | {}".format(response.status_code, e, request.callback.__name__, request.url))
 
-    async def worker(self, queue: asyncio.PriorityQueue, semaphore: asyncio.Semaphore):
+    async def request_task(self, q1: asyncio.PriorityQueue, q2: asyncio.Queue, semaphore: asyncio.Semaphore):
         """工作协程，从队列中获取请求并处理"""
         while True:
-            req: Request = await queue.get()
+            req: Request = await q1.get()
 
             # 结束信号
             if req.url == "":
@@ -82,7 +83,7 @@ class MiniSpider:
                         try:
                             result = self.handle_request_excetpion(e, req)
                             if isinstance(result, Request):
-                                await queue.put(result)
+                                await q1.put(result)
                                 break
                         except IgnoreRequest as e:
                             logger.debug("{} 忽略请求 {}".format(e, req.url))
@@ -105,39 +106,72 @@ class MiniSpider:
                         try:
                             cached = req.callback(Response(resp), **req.cb_kwargs)
                             if isinstance(cached, Iterator):
-                                for next_request in cached:
-                                    await queue.put(next_request)  # 把后续请求加入队列
+                                for c in cached:
+                                    if isinstance(c, Request):
+                                        await q1.put(c)  # 把后续请求加入队列
+                                    elif isinstance(c, dict):
+                                        await q2.put(c)
+                                    else:
+                                        logger.warning("Please yield `Request` or `dict` Not {}".format(c))
                         except Exception as e:
                             self.handle_callback_excetpion(e, req, resp)
                         finally:
                             break
 
-            queue.task_done()
+            q1.task_done()
+
+    async def item_task(self, q2: asyncio.Queue):
+        while True:
+            item = await q2.get()
+            if item is None:
+                break
+            self.process_item(item)
+            q2.task_done()
+
+    def process_item(self, item: dict):
+        logger.success(item)
 
     async def run(self):
         """爬取入口"""
-        queue = asyncio.PriorityQueue()
+        request_queue = asyncio.PriorityQueue()
+        item_queue = asyncio.Queue()
         semaphore = asyncio.Semaphore(self.max_requests)
 
-        # 工作协程启动...
-        workers = [
-            asyncio.create_task(self.worker(queue, semaphore))
+        # 处理请求...
+        request_tasks = [
+            asyncio.create_task(self.request_task(request_queue, item_queue, semaphore))
             for _ in range(self.max_requests)
         ]
 
-        # 将初始请求加入队列
+        # 处理数据...
+        item_tasks = [
+            asyncio.create_task(self.item_task(item_queue))
+            for _ in range(self.item_speed)
+        ]
+
+        # 发送最开始的请求
         for req in self.start_requests():
-            await queue.put(req)
+            await request_queue.put(req)
 
-        # 等待队列中的所有任务完成
-        await queue.join()
+        # 等待所有请求处理完成
+        await request_queue.join()
+        logger.debug("处理请求已结束")
 
-        # ...停止工作协程
+        # 等待所有数据处理完成
+        await item_queue.join()
+        logger.debug("处理数据已结束")
+
+        # 退出请求任务
         for _ in range(self.max_requests):
-            await queue.put(Request(url=""))
+            await request_queue.put(Request(url=""))
+
+        # 退出数据任务
+        for _ in range(self.item_speed):
+            await item_queue.put(None)
 
         # 等待所有工作协程完成
-        await asyncio.gather(*workers)
+        await asyncio.gather(*request_tasks)
+        await asyncio.gather(*item_tasks)
 
     def go(self):
         asyncio.run(self.run())

{coocan-0.5.2.1 → coocan-0.5.3}/coocan.egg-info/PKG-INFO

@@ -1,7 +1,8 @@
 Metadata-Version: 2.1
 Name: coocan
-Version: 0.5.2.1
+Version: 0.5.3
 Summary: Air Spider Framework
+Home-page: https://github.com/markadc/coocan
 Author: wauo
 Author-email: markadc@126.com
 Requires-Python: >=3.10

{coocan-0.5.2.1 → coocan-0.5.3}/setup.py

@@ -4,8 +4,9 @@ with open("README.md", "r", encoding="utf-8") as f:
     long_description = f.read()
 
 setup(
+    url="https://github.com/markadc/coocan",
     name="coocan",
-    version="0.5.2.1",
+    version="0.5.3",
     author="wauo",
     author_email="markadc@126.com",
     description="Air Spider Framework",