coocan 0.3.3__tar.gz → 0.4.4__tar.gz

{coocan-0.3.3 → coocan-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: coocan
-Version: 0.3.3
+Version: 0.4.4
 Summary: Air Spider Framework
 Author: wauo
 Author-email: markadc@126.com

coocan-0.4.4/coocan/cmd/cli.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+
+import click
+
+TEMPLATE_DIR = Path(__file__).parent / "templates"
+
+
+@click.group()
+def main():
+    """爬虫者的贴心助手
+    \b
+    可用命令：
+      new - 创建新的爬虫文件
+    \b
+    示例：
+      cc new -s demo
+    """
+
+
+@main.command()
+@click.option('-s', '--spider', required=True, help='爬虫文')
+def new(spider):
+    """新建"""
+    spider_file_name = "{}.py".format(spider)
+    try:
+        template_path = TEMPLATE_DIR / "spider.py"
+        with open(template_path, 'r') as f:
+            content = f.read()
+
+        with open(spider_file_name, 'w') as f:
+            f.write(content)
+
+        click.echo("Success create spider file {}".format(spider_file_name))
+
+    except Exception as e:
+        click.echo(str(e))
+        raise click.ClickException("Failed")
+
+
+if __name__ == '__main__':
+    main()

coocan-0.4.4/coocan/cmd/templates/spider.py

@@ -0,0 +1,12 @@
+from coocan import Request, Response, MiniSpider
+
+
+class Spider(MiniSpider):
+    start_urls = ['https://github.com/markadc/coocan']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "https://github.com"
+
+    def parse(self, response: Response):
+        pass

coocan-0.4.4/coocan/gen.py

@@ -0,0 +1,33 @@
+import random
+
+
+def gen_random_os() -> str:
+    """生成一个随机的操作系统"""
+    os_choices = [
+        "Windows NT 10.0; Win64; x64",
+        "Windows NT 6.1; WOW64",
+        "Macintosh; Intel Mac OS X 10_15_6",
+        "X11; Linux x86_64",
+        "Windows NT 6.3; Trident/7.0",
+    ]
+    return random.choice(os_choices)
+
+
+def gen_random_browser() -> str:
+    """生成一个随机的浏览器类型和版本"""
+    browser_choices = [
+        ("Chrome", random.randint(70, 100)),
+        ("Firefox", random.randint(70, 100)),
+        ("Edge", random.randint(80, 100)),
+        ("Safari", random.randint(10, 14)),
+        ("Opera", random.randint(60, 80)),
+    ]
+    browser, version = random.choice(browser_choices)
+    return f"{browser}/{version}.0"
+
+
+def gen_random_ua() -> str:
+    """生成一个随机的UA"""
+    os, browser = gen_random_os(), gen_random_browser()
+    ua = f"Mozilla/5.0 ({os}) AppleWebKit/537.36 (KHTML, like Gecko) {browser} Safari/537.36"
+    return ua

coocan-0.4.4/coocan/spider/base.py

@@ -0,0 +1,143 @@
+import asyncio
+from collections.abc import Iterator
+
+from loguru import logger
+
+from coocan.gen import gen_random_ua
+from coocan.url import Request, Response
+
+
+class IgnoreRequest(Exception):
+    """忽略这个请求，不再重试"""
+    pass
+
+
+class IgnoreResponse(Exception):
+    """忽略这个响应，不进回调"""
+    pass
+
+
+class MiniSpider:
+    start_urls = []
+    max_requests = 5
+    max_retry_times = 3
+    enable_random_ua = True
+    headers_extra_field = {}
+    delay = 0
+
+    def start_requests(self):
+        """初始请求"""
+        assert self.start_urls, "没有起始 URL 列表"
+        for url in self.start_urls:
+            yield Request(url, self.parse)
+
+    def middleware(self, request: Request):
+        # 随机Ua
+        if self.enable_random_ua is True:
+            request.headers.setdefault("User-Agent", gen_random_ua())
+
+        # 为 headers 补充额外字段
+        if self.headers_extra_field:
+            request.headers.update(self.headers_extra_field)
+
+    def validator(self, response: Response):
+        """校验响应"""
+        pass
+
+    def parse(self, response: Response):
+        """默认回调函数"""
+        raise NotImplementedError("没有定义回调函数 {}.parse ".format(self.__class__.__name__))
+
+    def handle_request_excetpion(self, e: Exception, request: Request):
+        """处理请求时的异常"""
+        logger.error("{} {}".format(type(e).__name__, request.url))
+
+    def handle_callback_excetpion(self, e: Exception, request: Request, response: Response):
+        logger.error("{} `回调`时出现异常 | {} | {} | {}".format(response.status_code, e, request.callback.__name__, request.url))
+
+    async def worker(self, queue: asyncio.PriorityQueue, semaphore: asyncio.Semaphore):
+        """工作协程，从队列中获取请求并处理"""
+        while True:
+            req: Request = await queue.get()
+
+            # 结束信号
+            if req.url == "":
+                break
+
+            # 控制并发
+            async with semaphore:
+                for i in range(self.max_retry_times + 1):
+                    # 进入了重试
+                    if i > 0:
+                        logger.debug("正在重试第{}次... {}".format(i, req.url))
+
+                    # 开始请求...
+                    try:
+                        self.middleware(req)
+                        await asyncio.sleep(self.delay)
+                        resp = await req.send()
+
+                    # 请求失败
+                    except Exception as e:
+                        try:
+                            result = self.handle_request_excetpion(e, req)
+                            if isinstance(result, Request):
+                                await queue.put(result)
+                                break
+                        except IgnoreRequest as e:
+                            logger.debug("{} 忽略请求 {}".format(e, req.url))
+                            break
+                        except Exception as e:
+                            logger.error("`处理异常函数`异常了 | {} | {}".format(e, req.url))
+
+                    # 请求成功
+                    else:
+                        # 校验响应
+                        try:
+                            self.validator(resp)
+                        except IgnoreResponse as e:
+                            logger.debug("{} 忽略响应 {}".format(e, req.url))
+                            break
+                        except Exception as e:
+                            logger.error("`校验器`函数异常了 | {} | {}".format(e, req.url))
+
+                        # 进入回调
+                        try:
+                            cached = req.callback(Response(resp), **req.cb_kwargs)
+                            if isinstance(cached, Iterator):
+                                for next_request in cached:
+                                    await queue.put(next_request)  # 把后续请求加入队列
+                        except Exception as e:
+                            self.handle_callback_excetpion(e, req, resp)
+                        finally:
+                            break
+
+            queue.task_done()
+
+    async def run(self):
+        """爬取入口"""
+        queue = asyncio.PriorityQueue()
+        semaphore = asyncio.Semaphore(self.max_requests)
+
+        # 工作协程启动...
+        workers = [
+            asyncio.create_task(self.worker(queue, semaphore))
+            for _ in range(self.max_requests)
+        ]
+
+        # 将初始请求加入队列
+        for req in self.start_requests():
+            await queue.put(req)
+
+        # 等待队列中的所有任务完成
+        await queue.join()
+
+        # ...停止工作协程
+        for _ in range(self.max_requests):
+            await queue.put(Request(url=""))
+
+        # 等待所有工作协程完成
+        await asyncio.gather(*workers)
+
+    def go(self):
+        asyncio.run(self.run())

{coocan-0.3.3 → coocan-0.4.4}/coocan/url/request.py

@@ -1,3 +1,4 @@
+import time
 from typing import Callable
 
 import httpx
@@ -6,7 +7,7 @@ cli = httpx.AsyncClient()
 
 
 class Request:
-    def __init__(self, url: str, callback: Callable = None, params=None, headers=None, data=None, json=None, timeout=None, cb_kwargs=None):
+    def __init__(self, url: str, callback: Callable = None, params=None, headers=None, data=None, json=None, timeout=None, cb_kwargs=None, priority=None):
         self.url = url
         self.callback = callback
         self.params = params
@@ -15,6 +16,7 @@ class Request:
         self.json = json
         self.timeout = timeout
         self.cb_kwargs = cb_kwargs or {}
+        self.priority = priority or time.time()
 
     async def send(self):
         if (self.data and self.json) is None:
@@ -24,3 +26,6 @@ class Request:
         else:
             raise Exception("仅支持 GET 和 POST 请求")
         return response
+
+    def __lt__(self, other):
+        return self.priority < other.priority

{coocan-0.3.3 → coocan-0.4.4}/coocan.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: coocan
-Version: 0.3.3
+Version: 0.4.4
 Summary: Air Spider Framework
 Author: wauo
 Author-email: markadc@126.com

{coocan-0.3.3 → coocan-0.4.4}/coocan.egg-info/SOURCES.txt

@@ -1,9 +1,16 @@
 setup.py
 coocan/__init__.py
+coocan/gen.py
 coocan.egg-info/PKG-INFO
 coocan.egg-info/SOURCES.txt
 coocan.egg-info/dependency_links.txt
+coocan.egg-info/entry_points.txt
+coocan.egg-info/requires.txt
 coocan.egg-info/top_level.txt
+coocan/cmd/__init__.py
+coocan/cmd/cli.py
+coocan/cmd/templates/__init__.py
+coocan/cmd/templates/spider.py
 coocan/spider/__init__.py
 coocan/spider/base.py
 coocan/url/__init__.py

coocan-0.4.4/coocan.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+cc = coocan.cmd.cli:main

coocan-0.4.4/coocan.egg-info/requires.txt

@@ -0,0 +1 @@
+click>=8.0.0

{coocan-0.3.3 → coocan-0.4.4}/setup.py

@@ -2,10 +2,18 @@ from setuptools import setup, find_packages
 
 setup(
     name="coocan",
-    version="0.3.3",
+    version="0.4.4",
     author="wauo",
     author_email="markadc@126.com",
     description="Air Spider Framework",
     packages=find_packages(),
     python_requires=">=3.10",
+    install_requires=[
+        'click>=8.0.0',
+    ],
+    entry_points={
+        'console_scripts': [
+            'cc=coocan.cmd.cli:main',
+        ],
+    }
 )

coocan-0.3.3/coocan/spider/base.py

@@ -1,98 +0,0 @@
-import asyncio
-from collections.abc import Iterator
-
-from loguru import logger
-
-from coocan.url import Request, Response
-
-
-class IgnoreRequest(Exception):
-    pass
-
-
-class MiniSpider:
-    start_urls = []
-    max_requests = 5
-    max_retry_times = 3
-
-    def start_requests(self):
-        """初始请求"""
-        assert self.start_urls, "没有起始 URL 列表"
-        for url in self.start_urls:
-            yield Request(url, self.parse)
-
-    def middleware(self, request: Request):
-        request.headers.setdefault("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0")
-
-    async def get_response(self, request: Request):
-        """发送请求，获取响应"""
-        self.middleware(request)
-        response = await request.send()
-        return response
-
-    def parse(self, response: Response):
-        raise NotImplementedError("没有定义回调函数 {}.parse ".format(self.__class__.__name__))
-
-    def when_request_error(self, e: Exception, request: Request):
-        logger.error("{} {}".format(type(e).__name__, request.url))
-
-    async def worker(self, queue, semaphore):
-        """工作协程，从队列中获取请求并处理"""
-        while True:
-            request = await queue.get()
-
-            # 结束信号
-            if request is None:
-                break
-
-            # 控制并发
-            async with semaphore:
-                for i in range(self.max_retry_times + 1):
-                    if i > 0:
-                        logger.debug("正在重试第{}次...{}".format(i, request.url))
-                    try:
-                        response = await self.get_response(request)
-                        cached = request.callback(Response(response), **request.cb_kwargs)
-                        if isinstance(cached, Iterator):
-                            for next_request in cached:
-                                await queue.put(next_request)  # 将后续请求加入队列
-                    except Exception as e:
-                        try:
-                            result = self.when_request_error(e, request)
-                            if isinstance(result, Request):
-                                await queue.put(result)
-                                logger.debug("新的请求 {}".format(result.url))
-                        except IgnoreRequest as e:
-                            logger.debug("{} 忽略请求 {}".format(e, request.url))
-                            break
-                    else:
-                        break
-            queue.task_done()
-
-    async def run(self):
-        """爬取入口"""
-        queue = asyncio.Queue()
-        semaphore = asyncio.Semaphore(self.max_requests)
-
-        # 工作协程启动...
-        workers = [
-            asyncio.create_task(self.worker(queue, semaphore))
-            for _ in range(self.max_requests)
-        ]
-
-        # 将初始请求加入队列
-        for req in self.start_requests():
-            await queue.put(req)
-
-        # 等待队列中的所有任务完成
-        await queue.join()
-
-        # ...停止工作协程
-        for _ in range(self.max_requests):
-            await queue.put(None)
-
-        # 等待所有工作协程完成
-        await asyncio.gather(*workers)
-
-    def go(self):
-        asyncio.run(self.run())