coocan 0.0.1__tar.gz

coocan-0.0.1/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.1
+Name: coocan
+Version: 0.0.1
+Summary: Air Spider Framework
+Author: wauo
+Author-email: markadc@126.com
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# 项目说明
+
+- 一个轻量爬虫框架

coocan-0.0.1/coocan/__init__.py

@@ -0,0 +1,2 @@
+from coocan.spider import *
+from coocan.url import *

coocan-0.0.1/coocan/spider/__init__.py

@@ -0,0 +1 @@
+from coocan.spider.base import Spider

coocan-0.0.1/coocan/spider/base.py

@@ -0,0 +1,84 @@
+import asyncio
+from collections.abc import Iterator
+
+from loguru import logger
+
+import coocan
+from coocan.url import Request
+
+
+class Spider:
+    start_urls = []
+    max_requests = 5
+
+    def start_requests(self):
+        """初始请求"""
+        assert self.start_urls, "没有起始 URL 列表"
+        for url in self.start_urls:
+            yield coocan.Request(url, self.parse)
+
+    def middleware(self, request: Request):
+        request.headers.setdefault("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0")
+
+    async def get_response(self, request: Request):
+        """发送请求，获取响应"""
+        try:
+            self.middleware(request)
+            response = await request.send()
+            return response
+        except Exception as e:
+            logger.error("{} {}".format(request.url, e))
+
+    def parse(self, response):
+        raise NotImplementedError("没有定义回调函数 {}.parse ".format(self.__class__.__name__))
+
+    async def worker(self, queue, semaphore):
+        """工作协程，从队列中获取请求并处理"""
+        while True:
+            request = await queue.get()
+
+            # 结束信号
+            if request is None:
+                break
+
+            # 控制并发
+            async with semaphore:
+                response = await self.get_response(request)
+                if response:
+                    try:
+                        cached = request.callback(response, **request.cb_kwargs)
+                        if isinstance(cached, Iterator):
+                            for next_request in cached:
+                                await queue.put(next_request)  # 将后续请求加入队列
+                    except Exception as e:
+                        logger.error(e)
+
+            queue.task_done()
+
+    async def run(self):
+        """爬取入口"""
+        queue = asyncio.Queue()
+        semaphore = asyncio.Semaphore(self.max_requests)
+
+        # 工作协程启动...
+        workers = [
+            asyncio.create_task(self.worker(queue, semaphore))
+            for _ in range(self.max_requests)
+        ]
+
+        # 将初始请求加入队列
+        for req in self.start_requests():
+            await queue.put(req)
+
+        # 等待队列中的所有任务完成
+        await queue.join()
+
+        # ...停止工作协程
+        for _ in range(self.max_requests):
+            await queue.put(None)
+
+        # 等待所有工作协程完成
+        await asyncio.gather(*workers)
+
+    def go(self):
+        asyncio.run(self.run())

coocan-0.0.1/coocan/url/__init__.py

@@ -0,0 +1,2 @@
+from coocan.url.request import Request
+from coocan.url.response import SelectorResponse

coocan-0.0.1/coocan/url/request.py

@@ -0,0 +1,21 @@
+from typing import Callable
+
+import httpx
+
+cli = httpx.AsyncClient()
+
+
+class Request:
+    def __init__(self, url, callback: Callable = None, params=None, headers=None, data=None, json=None, timeout=None, cb_kwargs=None):
+        self.url = url
+        self.callback = callback
+        self.params = params
+        self.headers = headers or {}
+        self.data = data
+        self.json = json
+        self.timeout = timeout
+        self.cb_kwargs = cb_kwargs or {}
+
+    async def send(self):
+        response = await cli.get(self.url, params=self.params, headers=self.headers, timeout=self.timeout)
+        return response

coocan-0.0.1/coocan/url/response.py

@@ -0,0 +1,51 @@
+from typing import Callable
+
+from parsel import Selector
+from requests import Response
+
+from cocoman.spider.errors import ResponseCodeError, ResponseTextError
+
+
+class SelectorResponse(Response):
+    """可以使用Xpath、CSS"""
+
+    def __init__(self, response: Response):
+        super().__init__()
+        self.__dict__.update(response.__dict__)
+        self.selector = Selector(text=response.text)
+
+    def __str__(self):
+        return "<Response {}>".format(self.status_code)
+
+    def xpath(self, query: str):
+        sel = self.selector.xpath(query)
+        return sel
+
+    def css(self, query: str):
+        sel = self.selector.css(query)
+        return sel
+
+    def get_one(self, query: str, default=None, strip=True):
+        v = self.selector.xpath(query).get(default=default)
+        return v.strip() if strip and isinstance(v, str) else v
+
+    def get_all(self, query: str, strip=True):
+        vs = [v.strip() if strip else v for v in self.selector.xpath(query).getall()]
+        return vs
+
+    def raise_for_status(self, codes: list = None):
+        codes = codes or [200]
+        if self.status_code not in codes:
+            raise ResponseCodeError("{} not in {}".format(self.status_code, codes))
+
+    def raise_for_text(self, validate: Callable[[str], bool] = None):
+        if validate(self.text) is False:
+            raise ResponseTextError("not ideal text")
+
+    def raise_has_text(self, text: str):
+        """有此文本则抛出异常"""
+        assert self.text.find(text) == -1, ResponseTextError("has text: {}".format(text))
+
+    def raise_no_text(self, text: str):
+        """无此文本则抛出异常"""
+        assert self.text.find(text) != -1, ResponseTextError("no text: {}".format(text))

coocan-0.0.1/coocan.egg-info/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.1
+Name: coocan
+Version: 0.0.1
+Summary: Air Spider Framework
+Author: wauo
+Author-email: markadc@126.com
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# 项目说明
+
+- 一个轻量爬虫框架

coocan-0.0.1/coocan.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+setup.py
+coocan/__init__.py
+coocan.egg-info/PKG-INFO
+coocan.egg-info/SOURCES.txt
+coocan.egg-info/dependency_links.txt
+coocan.egg-info/top_level.txt
+coocan/spider/__init__.py
+coocan/spider/base.py
+coocan/url/__init__.py
+coocan/url/request.py
+coocan/url/response.py

coocan-0.0.1/coocan.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

coocan-0.0.1/coocan.egg-info/top_level.txt

@@ -0,0 +1 @@
+coocan

coocan-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

coocan-0.0.1/setup.py

@@ -0,0 +1,16 @@
+from setuptools import setup, find_packages
+
+with open("readme.md", "r", encoding="utf-8") as f:
+    long_description = f.read()
+
+setup(
+    name="coocan",
+    version="0.0.1",
+    author="wauo",
+    author_email="markadc@126.com",
+    description="Air Spider Framework",
+    packages=find_packages(),
+    python_requires=">=3.10",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+)