coocan 0.3.2__tar.gz → 0.3.3__tar.gz

{coocan-0.3.2 → coocan-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: coocan
-Version: 0.3.2
+Version: 0.3.3
 Summary: Air Spider Framework
 Author: wauo
 Author-email: markadc@126.com

coocan-0.3.3/coocan/spider/__init__.py

@@ -0,0 +1 @@
+from coocan.spider.base import MiniSpider, IgnoreRequest

{coocan-0.3.2 → coocan-0.3.3}/coocan/spider/base.py

@@ -3,35 +3,39 @@ from collections.abc import Iterator
 
 from loguru import logger
 
-import coocan
-from coocan.url import Request
+from coocan.url import Request, Response
+
+
+class IgnoreRequest(Exception):
+    pass
 
 
 class MiniSpider:
     start_urls = []
     max_requests = 5
+    max_retry_times = 3
 
     def start_requests(self):
         """初始请求"""
         assert self.start_urls, "没有起始 URL 列表"
         for url in self.start_urls:
-            yield coocan.Request(url, self.parse)
+            yield Request(url, self.parse)
 
     def middleware(self, request: Request):
         request.headers.setdefault("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0")
 
     async def get_response(self, request: Request):
         """发送请求，获取响应"""
-        try:
-            self.middleware(request)
-            response = await request.send()
-            return response
-        except Exception as e:
-            logger.error("{} {}".format(request.url, e))
-
-    def parse(self, response):
+        self.middleware(request)
+        response = await request.send()
+        return response
+
+    def parse(self, response: Response):
         raise NotImplementedError("没有定义回调函数 {}.parse ".format(self.__class__.__name__))
 
+    def when_request_error(self, e: Exception, request: Request):
+        logger.error("{} {}".format(type(e).__name__, request.url))
+
     async def worker(self, queue, semaphore):
         """工作协程，从队列中获取请求并处理"""
         while True:
@@ -43,16 +47,26 @@ class MiniSpider:
 
             # 控制并发
             async with semaphore:
-                response = await self.get_response(request)
-                if response:
+                for i in range(self.max_retry_times + 1):
+                    if i > 0:
+                        logger.debug("正在重试第{}次...{}".format(i, request.url))
                     try:
-                        cached = request.callback(response, **request.cb_kwargs)
+                        response = await self.get_response(request)
+                        cached = request.callback(Response(response), **request.cb_kwargs)
                         if isinstance(cached, Iterator):
                             for next_request in cached:
                                 await queue.put(next_request)  # 将后续请求加入队列
                     except Exception as e:
-                        logger.error(e)
-
+                        try:
+                            result = self.when_request_error(e, request)
+                            if isinstance(result, Request):
+                                await queue.put(result)
+                                logger.debug("新的请求 {}".format(result.url))
+                        except IgnoreRequest as e:
+                            logger.debug("{} 忽略请求 {}".format(e, request.url))
+                            break
+                    else:
+                        break
             queue.task_done()
 
     async def run(self):

coocan-0.3.3/coocan/url/__init__.py

@@ -0,0 +1,2 @@
+from coocan.url.request import Request
+from coocan.url.response import SelectorResponse as Response

{coocan-0.3.2 → coocan-0.3.3}/coocan/url/request.py

@@ -6,7 +6,7 @@ cli = httpx.AsyncClient()
 
 
 class Request:
-    def __init__(self, url, callback: Callable = None, params=None, headers=None, data=None, json=None, timeout=None, cb_kwargs=None):
+    def __init__(self, url: str, callback: Callable = None, params=None, headers=None, data=None, json=None, timeout=None, cb_kwargs=None):
         self.url = url
         self.callback = callback
         self.params = params

{coocan-0.3.2 → coocan-0.3.3}/coocan/url/response.py

@@ -1,15 +1,15 @@
 from typing import Callable
 
 from cocoman.spider.errors import ResponseCodeError, ResponseTextError
+from httpx import Response
 from parsel import Selector
-from requests import Response
 
 
 class SelectorResponse(Response):
     """可以使用Xpath、CSS"""
 
     def __init__(self, response: Response):
-        super().__init__()
+        super().__init__(response.status_code)
         self.__dict__.update(response.__dict__)
         self.selector = Selector(text=response.text)
 

{coocan-0.3.2 → coocan-0.3.3}/coocan.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: coocan
-Version: 0.3.2
+Version: 0.3.3
 Summary: Air Spider Framework
 Author: wauo
 Author-email: markadc@126.com

{coocan-0.3.2 → coocan-0.3.3}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="coocan",
-    version="0.3.2",
+    version="0.3.3",
     author="wauo",
     author_email="markadc@126.com",
     description="Air Spider Framework",

coocan-0.3.2/coocan/spider/__init__.py

@@ -1 +0,0 @@
-from coocan.spider.base import MiniSpider

coocan-0.3.2/coocan/url/__init__.py

@@ -1,2 +0,0 @@
-from coocan.url.request import Request
-from coocan.url.response import SelectorResponse