coocan 0.3.1__tar.gz → 0.3.3__tar.gz

{coocan-0.3.1 → coocan-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: coocan
-Version: 0.3.1
+Version: 0.3.3
 Summary: Air Spider Framework
 Author: wauo
 Author-email: markadc@126.com

coocan-0.3.3/coocan/spider/__init__.py

@@ -0,0 +1 @@
+from coocan.spider.base import MiniSpider, IgnoreRequest

{coocan-0.3.1 → coocan-0.3.3}/coocan/spider/base.py

@@ -3,35 +3,39 @@ from collections.abc import Iterator
 
 from loguru import logger
 
-import coocan
-from coocan.url import Request
+from coocan.url import Request, Response
+
+
+class IgnoreRequest(Exception):
+    pass
 
 
 class MiniSpider:
     start_urls = []
     max_requests = 5
+    max_retry_times = 3
 
     def start_requests(self):
         """初始请求"""
         assert self.start_urls, "没有起始 URL 列表"
         for url in self.start_urls:
-            yield coocan.Request(url, self.parse)
+            yield Request(url, self.parse)
 
     def middleware(self, request: Request):
         request.headers.setdefault("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0")
 
     async def get_response(self, request: Request):
         """发送请求，获取响应"""
-        try:
-            self.middleware(request)
-            response = await request.send()
-            return response
-        except Exception as e:
-            logger.error("{} {}".format(request.url, e))
-
-    def parse(self, response):
+        self.middleware(request)
+        response = await request.send()
+        return response
+
+    def parse(self, response: Response):
         raise NotImplementedError("没有定义回调函数 {}.parse ".format(self.__class__.__name__))
 
+    def when_request_error(self, e: Exception, request: Request):
+        logger.error("{} {}".format(type(e).__name__, request.url))
+
     async def worker(self, queue, semaphore):
         """工作协程，从队列中获取请求并处理"""
         while True:
@@ -43,16 +47,26 @@ class MiniSpider:
 
             # 控制并发
             async with semaphore:
-                response = await self.get_response(request)
-                if response:
+                for i in range(self.max_retry_times + 1):
+                    if i > 0:
+                        logger.debug("正在重试第{}次...{}".format(i, request.url))
                     try:
-                        cached = request.callback(response, **request.cb_kwargs)
+                        response = await self.get_response(request)
+                        cached = request.callback(Response(response), **request.cb_kwargs)
                         if isinstance(cached, Iterator):
                             for next_request in cached:
                                 await queue.put(next_request)  # 将后续请求加入队列
                     except Exception as e:
-                        logger.error(e)
-
+                        try:
+                            result = self.when_request_error(e, request)
+                            if isinstance(result, Request):
+                                await queue.put(result)
+                                logger.debug("新的请求 {}".format(result.url))
+                        except IgnoreRequest as e:
+                            logger.debug("{} 忽略请求 {}".format(e, request.url))
+                            break
+                    else:
+                        break
             queue.task_done()
 
     async def run(self):

coocan-0.3.3/coocan/url/__init__.py

@@ -0,0 +1,2 @@
+from coocan.url.request import Request
+from coocan.url.response import SelectorResponse as Response

coocan-0.3.3/coocan/url/request.py

@@ -0,0 +1,26 @@
+from typing import Callable
+
+import httpx
+
+cli = httpx.AsyncClient()
+
+
+class Request:
+    def __init__(self, url: str, callback: Callable = None, params=None, headers=None, data=None, json=None, timeout=None, cb_kwargs=None):
+        self.url = url
+        self.callback = callback
+        self.params = params
+        self.headers = headers or {}
+        self.data = data
+        self.json = json
+        self.timeout = timeout
+        self.cb_kwargs = cb_kwargs or {}
+
+    async def send(self):
+        if (self.data and self.json) is None:
+            response = await cli.get(self.url, params=self.params, headers=self.headers, timeout=self.timeout)
+        elif self.data or self.json:
+            response = await cli.post(self.url, params=self.params, headers=self.headers, data=self.data, json=self.json, timeout=self.timeout)
+        else:
+            raise Exception("仅支持 GET 和 POST 请求")
+        return response

{coocan-0.3.1 → coocan-0.3.3}/coocan/url/response.py

@@ -1,15 +1,15 @@
 from typing import Callable
 
 from cocoman.spider.errors import ResponseCodeError, ResponseTextError
+from httpx import Response
 from parsel import Selector
-from requests import Response
 
 
 class SelectorResponse(Response):
     """可以使用Xpath、CSS"""
 
     def __init__(self, response: Response):
-        super().__init__()
+        super().__init__(response.status_code)
         self.__dict__.update(response.__dict__)
         self.selector = Selector(text=response.text)
 
@@ -37,7 +37,7 @@ class SelectorResponse(Response):
         if self.status_code not in codes:
             raise ResponseCodeError("{} not in {}".format(self.status_code, codes))
 
-    def raise_for_text(self, validate: Callable[[str], bool] = None):
+    def raise_for_text(self, validate: Callable[[str], bool]):
         if validate(self.text) is False:
             raise ResponseTextError("not ideal text")
 

{coocan-0.3.1 → coocan-0.3.3}/coocan.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: coocan
-Version: 0.3.1
+Version: 0.3.3
 Summary: Air Spider Framework
 Author: wauo
 Author-email: markadc@126.com

{coocan-0.3.1 → coocan-0.3.3}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="coocan",
-    version="0.3.1",
+    version="0.3.3",
     author="wauo",
     author_email="markadc@126.com",
     description="Air Spider Framework",

coocan-0.3.1/coocan/spider/__init__.py

@@ -1 +0,0 @@
-from coocan.spider.base import MiniSpider

coocan-0.3.1/coocan/url/__init__.py

@@ -1,2 +0,0 @@
-from coocan.url.request import Request
-from coocan.url.response import SelectorResponse

coocan-0.3.1/coocan/url/request.py

@@ -1,21 +0,0 @@
-from typing import Callable
-
-import httpx
-
-cli = httpx.AsyncClient()
-
-
-class Request:
-    def __init__(self, url, callback: Callable = None, params=None, headers=None, data=None, json=None, timeout=None, cb_kwargs=None):
-        self.url = url
-        self.callback = callback
-        self.params = params
-        self.headers = headers or {}
-        self.data = data
-        self.json = json
-        self.timeout = timeout
-        self.cb_kwargs = cb_kwargs or {}
-
-    async def send(self):
-        response = await cli.get(self.url, params=self.params, headers=self.headers, timeout=self.timeout)
-        return response