coocan 0.4.9__py3-none-any.whl

coocan/__init__.py

@@ -0,0 +1,2 @@
+from coocan.spider import *
+from coocan.url import *

coocan/cmd/cli.py

@@ -0,0 +1,72 @@
+import os
+import re
+from pathlib import Path
+
+import click
+
+TEMPLATE_DIR = Path(__file__).parent / "templates"
+
+help_info = """
+ ██████╗ ██████╗  ██████╗  ██████╗ █████╗ ███╗   ██╗
+██╔════╝██╔═══██╗██╔═══██╗██╔════╝██╔══██╗████╗  ██║
+██║     ██║   ██║██║   ██║██║     ███████║██╔██╗ ██║
+██║     ██║   ██║██║   ██║██║     ██╔══██║██║╚██╗██║
+╚██████╗╚██████╔╝╚██████╔╝╚██████╗██║  ██║██║ ╚████║
+ ╚═════╝ ╚═════╝  ╚═════╝  ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝
+"""
+
+
+def show_help_info():
+    print(help_info)
+
+
+def snake_to_pascal(snake_str: str):
+    """小蛇变成大驼峰"""
+    words = snake_str.split('_')
+    pascal_str = ''.join(word.capitalize() for word in words)
+    return pascal_str
+
+
+@click.group(invoke_without_command=True)
+@click.pass_context
+def main(ctx):
+    if ctx.invoked_subcommand is None:
+        show_help_info()
+        click.echo("cc new -s <spider_file_name>")
+
+
+@main.command()
+@click.option('-s', '--spider', required=True, help='爬虫文件名字')
+def new(spider):
+    """新建"""
+    if not re.search("^[a-zA-Z0-9_]*$", spider):
+        click.echo("只支持字母、数字、下划线")
+        return
+
+    pascal = snake_to_pascal(spider)
+    if not pascal.endswith("Spider"):
+        pascal += "Spider"
+
+    try:
+        template_path = TEMPLATE_DIR / "spider.txt"
+        with open(template_path, 'r') as f:
+            text = f.read()
+            spider_py_text = text.replace("{SpiderClassName}", pascal)
+
+        py_file = "{}.py".format(spider)
+        if os.path.exists(py_file):
+            click.echo("File {} already exists".format(py_file))
+            return
+
+        with open(py_file, 'w') as f:
+            f.write(spider_py_text)
+
+        click.echo("Success")
+
+    except Exception as e:
+        click.echo(str(e))
+        raise click.ClickException("Failed")
+
+
+if __name__ == '__main__':
+    main()

coocan/cmd/templates/spider.txt

@@ -0,0 +1,18 @@
+from coocan import Request, Response, MiniSpider
+
+
+class {SpiderClassName}(MiniSpider):
+    start_urls = ['https://github.com/markadc/coocan']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "https://github.com"
+
+    def parse(self, response: Response):
+        print(response.status_code)
+        print(response.get_one("//title/text()"))
+
+
+if __name__ == '__main__':
+    s = {SpiderClassName}()
+    s.go()

coocan/gen.py

@@ -0,0 +1,33 @@
+import random
+
+
+def gen_random_os() -> str:
+    """生成一个随机的操作系统"""
+    os_choices = [
+        "Windows NT 10.0; Win64; x64",
+        "Windows NT 6.1; WOW64",
+        "Macintosh; Intel Mac OS X 10_15_6",
+        "X11; Linux x86_64",
+        "Windows NT 6.3; Trident/7.0",
+    ]
+    return random.choice(os_choices)
+
+
+def gen_random_browser() -> str:
+    """生成一个随机的浏览器类型和版本"""
+    browser_choices = [
+        ("Chrome", random.randint(70, 100)),
+        ("Firefox", random.randint(70, 100)),
+        ("Edge", random.randint(80, 100)),
+        ("Safari", random.randint(10, 14)),
+        ("Opera", random.randint(60, 80)),
+    ]
+    browser, version = random.choice(browser_choices)
+    return f"{browser}/{version}.0"
+
+
+def gen_random_ua() -> str:
+    """生成一个随机的UA"""
+    os, browser = gen_random_os(), gen_random_browser()
+    ua = f"Mozilla/5.0 ({os}) AppleWebKit/537.36 (KHTML, like Gecko) {browser} Safari/537.36"
+    return ua

coocan/spider/__init__.py

@@ -0,0 +1 @@
+from coocan.spider.base import MiniSpider, IgnoreRequest

coocan/spider/base.py

@@ -0,0 +1,143 @@
+import asyncio
+from collections.abc import Iterator
+
+from loguru import logger
+
+from coocan.gen import gen_random_ua
+from coocan.url import Request, Response
+
+
+class IgnoreRequest(Exception):
+    """忽略这个请求，不再重试"""
+    pass
+
+
+class IgnoreResponse(Exception):
+    """忽略这个响应，不进回调"""
+    pass
+
+
+class MiniSpider:
+    start_urls = []
+    max_requests = 5
+    max_retry_times = 3
+    enable_random_ua = True
+    headers_extra_field = {}
+    delay = 0
+
+    def start_requests(self):
+        """初始请求"""
+        assert self.start_urls, "没有起始 URL 列表"
+        for url in self.start_urls:
+            yield Request(url, self.parse)
+
+    def middleware(self, request: Request):
+        # 随机Ua
+        if self.enable_random_ua is True:
+            request.headers.setdefault("User-Agent", gen_random_ua())
+
+        # 为 headers 补充额外字段
+        if self.headers_extra_field:
+            request.headers.update(self.headers_extra_field)
+
+    def validator(self, response: Response):
+        """校验响应"""
+        pass
+
+    def parse(self, response: Response):
+        """默认回调函数"""
+        raise NotImplementedError("没有定义回调函数 {}.parse ".format(self.__class__.__name__))
+
+    def handle_request_excetpion(self, e: Exception, request: Request):
+        """处理请求时的异常"""
+        logger.error("{} {}".format(type(e).__name__, request.url))
+
+    def handle_callback_excetpion(self, e: Exception, request: Request, response: Response):
+        logger.error("{} `回调`时出现异常 | {} | {} | {}".format(response.status_code, e, request.callback.__name__, request.url))
+
+    async def worker(self, queue: asyncio.PriorityQueue, semaphore: asyncio.Semaphore):
+        """工作协程，从队列中获取请求并处理"""
+        while True:
+            req: Request = await queue.get()
+
+            # 结束信号
+            if req.url == "":
+                break
+
+            # 控制并发
+            async with semaphore:
+                for i in range(self.max_retry_times + 1):
+                    # 进入了重试
+                    if i > 0:
+                        logger.debug("正在重试第{}次... {}".format(i, req.url))
+
+                    # 开始请求...
+                    try:
+                        self.middleware(req)
+                        await asyncio.sleep(self.delay)
+                        resp = await req.send()
+
+                    # 请求失败
+                    except Exception as e:
+                        try:
+                            result = self.handle_request_excetpion(e, req)
+                            if isinstance(result, Request):
+                                await queue.put(result)
+                                break
+                        except IgnoreRequest as e:
+                            logger.debug("{} 忽略请求 {}".format(e, req.url))
+                            break
+                        except Exception as e:
+                            logger.error("`处理异常函数`异常了 | {} | {}".format(e, req.url))
+
+                    # 请求成功
+                    else:
+                        # 校验响应
+                        try:
+                            self.validator(resp)
+                        except IgnoreResponse as e:
+                            logger.debug("{} 忽略响应 {}".format(e, req.url))
+                            break
+                        except Exception as e:
+                            logger.error("`校验器`函数异常了 | {} | {}".format(e, req.url))
+
+                        # 进入回调
+                        try:
+                            cached = req.callback(Response(resp), **req.cb_kwargs)
+                            if isinstance(cached, Iterator):
+                                for next_request in cached:
+                                    await queue.put(next_request)  # 把后续请求加入队列
+                        except Exception as e:
+                            self.handle_callback_excetpion(e, req, resp)
+                        finally:
+                            break
+
+            queue.task_done()
+
+    async def run(self):
+        """爬取入口"""
+        queue = asyncio.PriorityQueue()
+        semaphore = asyncio.Semaphore(self.max_requests)
+
+        # 工作协程启动...
+        workers = [
+            asyncio.create_task(self.worker(queue, semaphore))
+            for _ in range(self.max_requests)
+        ]
+
+        # 将初始请求加入队列
+        for req in self.start_requests():
+            await queue.put(req)
+
+        # 等待队列中的所有任务完成
+        await queue.join()
+
+        # ...停止工作协程
+        for _ in range(self.max_requests):
+            await queue.put(Request(url=""))
+
+        # 等待所有工作协程完成
+        await asyncio.gather(*workers)
+
+    def go(self):
+        asyncio.run(self.run())

coocan/url/__init__.py

@@ -0,0 +1,2 @@
+from coocan.url.request import Request
+from coocan.url.response import SelectorResponse as Response

coocan/url/request.py

@@ -0,0 +1,31 @@
+import time
+from typing import Callable
+
+import httpx
+
+cli = httpx.AsyncClient()
+
+
+class Request:
+    def __init__(self, url: str, callback: Callable = None, params=None, headers=None, data=None, json=None, timeout=None, cb_kwargs=None, priority=None):
+        self.url = url
+        self.callback = callback
+        self.params = params
+        self.headers = headers or {}
+        self.data = data
+        self.json = json
+        self.timeout = timeout
+        self.cb_kwargs = cb_kwargs or {}
+        self.priority = priority or time.time()
+
+    async def send(self):
+        if (self.data and self.json) is None:
+            response = await cli.get(self.url, params=self.params, headers=self.headers, timeout=self.timeout)
+        elif self.data or self.json:
+            response = await cli.post(self.url, params=self.params, headers=self.headers, data=self.data, json=self.json, timeout=self.timeout)
+        else:
+            raise Exception("仅支持 GET 和 POST 请求")
+        return response
+
+    def __lt__(self, other):
+        return self.priority < other.priority

coocan/url/response.py

@@ -0,0 +1,50 @@
+from typing import Callable
+
+from cocoman.spider.errors import ResponseCodeError, ResponseTextError
+from httpx import Response
+from parsel import Selector
+
+
+class SelectorResponse(Response):
+    """可以使用Xpath、CSS"""
+
+    def __init__(self, response: Response):
+        super().__init__(response.status_code)
+        self.__dict__.update(response.__dict__)
+        self.selector = Selector(text=response.text)
+
+    def __str__(self):
+        return "<Response {}>".format(self.status_code)
+
+    def xpath(self, query: str):
+        sel = self.selector.xpath(query)
+        return sel
+
+    def css(self, query: str):
+        sel = self.selector.css(query)
+        return sel
+
+    def get_one(self, query: str, default=None, strip=True):
+        v = self.selector.xpath(query).get(default=default)
+        return v.strip() if strip and isinstance(v, str) else v
+
+    def get_all(self, query: str, strip=True):
+        vs = [v.strip() if strip else v for v in self.selector.xpath(query).getall()]
+        return vs
+
+    def raise_for_status(self, codes: list = None):
+        codes = codes or [200]
+        if self.status_code not in codes:
+            raise ResponseCodeError("{} not in {}".format(self.status_code, codes))
+
+    def raise_for_text(self, validate: Callable[[str], bool]):
+        if validate(self.text) is False:
+            raise ResponseTextError("not ideal text")
+
+    def raise_has_text(self, text: str):
+        """有此文本则抛出异常"""
+        assert self.text.find(text) == -1, ResponseTextError("has text: {}".format(text))
+
+    def raise_no_text(self, text: str):
+        """无此文本则抛出异常"""
+        assert self.text.find(text) != -1, ResponseTextError("no text: {}".format(text))

coocan-0.4.9.dist-info/METADATA

@@ -0,0 +1,93 @@
+Metadata-Version: 2.1
+Name: coocan
+Version: 0.4.9
+Summary: Air Spider Framework
+Author: wauo
+Author-email: markadc@126.com
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.0.0
+Requires-Dist: httpx
+Requires-Dist: loguru
+
+# 项目说明
+
+- 一个非常轻量的异步爬虫框架
+
+# 项目地址
+
+- https://github.com/markadc/coocan
+
+## demo
+
+- 效果
+  <br>
+  ![效果](coocan/demo.gif)
+
+
+- 代码
+
+```python
+import json
+
+from loguru import logger
+
+import coocan
+from coocan import Request, MiniSpider
+
+
+class CSDNDetailSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            logger.info(
+                """
+                {} 
+                {} 
+                {}
+                """.format(date, name, detail_url)
+            )
+            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
+
+        logger.info("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_detail(self, response, title):
+        logger.success("{}  已访问 {}".format(response.status_code, title))
+
+
+if __name__ == '__main__':
+    s = CSDNDetailSpider()
+    s.go()
+```

coocan-0.4.9.dist-info/RECORD

@@ -0,0 +1,15 @@
+coocan/__init__.py,sha256=UqFmE7ucuR_xR3OyyBU8pxqLfCJ5AdH_HsDdTsYPf6g,55
+coocan/gen.py,sha256=J6QWXkBVbiCQqey8i0BDqleRNpBswI8AyvrYmkDVQPw,1028
+coocan/cmd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+coocan/cmd/cli.py,sha256=d_sG63wz8RHHa5u7HabOz36yai_D8S3bSr7VprVpTck,2420
+coocan/cmd/templates/spider.txt,sha256=Htd7nOs1EeKbc8LNRUX7xyHkLWz3S0kaTPRW0M3NuUw,480
+coocan/spider/__init__.py,sha256=kMDCGeqtN50raCzwfCn18s_W8xV6KO_Ny9Xol4I48Ag,58
+coocan/spider/base.py,sha256=WMTnMQd7Dnv2aC7rnmkAo_WJu33p9g3GN07A2DnbdLI,5104
+coocan/url/__init__.py,sha256=rEMx66XDy5AIJ9mF_2UVzHW5mRLBAWZEyQ3txrZzuZA,102
+coocan/url/request.py,sha256=seZaQXQRvRMIf9WnCp3mAgNA-kxsj9P2JzAvuIt2Dx8,1116
+coocan/url/response.py,sha256=AnC0xsF34q68r62EVlcHYmDH6skm9RBwRHITTb4iBbU,1785
+coocan-0.4.9.dist-info/METADATA,sha256=WZhIMdcypGrPcE1_bAMxMUa3puvkngstEU4PSTR5UXo,2374
+coocan-0.4.9.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+coocan-0.4.9.dist-info/entry_points.txt,sha256=tOLQN_TVhl_9f2YBASTGBE_ClmG-iQ4rKmyhE2WAOY0,43
+coocan-0.4.9.dist-info/top_level.txt,sha256=VwB-Q4zEljgb9v1Ms1E59B-1pBYORXuhKjgZb-LHOhk,7
+coocan-0.4.9.dist-info/RECORD,,

coocan-0.4.9.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.45.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

coocan-0.4.9.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+cc = coocan.cmd.cli:main

coocan-0.4.9.dist-info/top_level.txt

@@ -0,0 +1 @@
+coocan