coocan 0.5.5__py3-none-any.whl → 0.5.6__py3-none-any.whl

coocan/__init__.py

@@ -1,2 +1,2 @@
-from coocan.spider import *
-from coocan.url import *
+from coocan.spider import *
+from coocan.url import *

coocan/_examples/crawl_csdn_detail.py

@@ -1,62 +1,62 @@
-import json
-
-from loguru import logger
-
-import coocan
-from coocan import Request, MiniSpider
-
-
-class CSDNDetailSpider(MiniSpider):
-    start_urls = ['http://www.csdn.net']
-    max_requests = 10
-
-    def middleware(self, request: Request):
-        request.headers["Referer"] = "http://www.csdn.net/"
-
-    def parse(self, response):
-        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
-        params = {
-            "page": "1",
-            "size": "20",
-            "businessType": "lately",
-            "noMore": "false",
-            "username": "markadc"
-        }
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-    def parse_page(self, response, api, params):
-        current_page = params["page"]
-        data = json.loads(response.text)
-        some = data["data"]["list"]
-
-        if not some:
-            logger.warning("没有第 {} 页".format(current_page))
-            return
-
-        for one in some:
-            date = one["formatTime"]
-            name = one["title"]
-            detail_url = one["url"]
-            logger.info(
-                """
-                {} 
-                {} 
-                {}
-                """.format(date, name, detail_url)
-            )
-            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
-
-        logger.info("第 {} 页抓取成功".format(params["page"]))
-
-        # 抓取下一页
-        next_page = int(current_page) + 1
-        params["page"] = str(next_page)
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-    def parse_detail(self, response, title):
-        logger.success("{}  已访问 {}".format(response.status_code, title))
-
-
-if __name__ == '__main__':
-    s = CSDNDetailSpider()
-    s.go()
+import json
+
+from loguru import logger
+
+import coocan
+from coocan import Request, MiniSpider
+
+
+class CSDNDetailSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            logger.info(
+                """
+                {} 
+                {} 
+                {}
+                """.format(date, name, detail_url)
+            )
+            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
+
+        logger.info("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_detail(self, response, title):
+        logger.success("{}  已访问 {}".format(response.status_code, title))
+
+
+if __name__ == '__main__':
+    s = CSDNDetailSpider()
+    s.go()

coocan/_examples/crawl_csdn_list.py

@@ -1,50 +1,50 @@
-import json
-
-from loguru import logger
-
-from coocan import Request, MiniSpider
-
-
-class CSDNSpider(MiniSpider):
-    start_urls = ['http://www.csdn.net']
-    max_requests = 10
-
-    def middleware(self, request: Request):
-        request.headers["Referer"] = "http://www.csdn.net/"
-
-    def parse(self, response):
-        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
-        params = {
-            "page": "1",
-            "size": "20",
-            "businessType": "lately",
-            "noMore": "false",
-            "username": "markadc"
-        }
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-    def parse_page(self, response, api, params):
-        current_page = params["page"]
-        data = json.loads(response.text)
-        some = data["data"]["list"]
-
-        if not some:
-            logger.warning("没有第 {} 页".format(current_page))
-            return
-
-        for one in some:
-            date = one["formatTime"]
-            name = one["title"]
-            detail_url = one["url"]
-            print(date, detail_url, name)
-        print("第 {} 页抓取成功".format(params["page"]))
-
-        # 抓取下一页
-        next_page = int(current_page) + 1
-        params["page"] = str(next_page)
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-
-if __name__ == '__main__':
-    s = CSDNSpider()
-    s.go()
+import json
+
+from loguru import logger
+
+from coocan import Request, MiniSpider
+
+
+class CSDNSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            print(date, detail_url, name)
+        print("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+
+if __name__ == '__main__':
+    s = CSDNSpider()
+    s.go()

coocan/_examples/recv_item.py

@@ -1,31 +1,31 @@
-import random
-import time
-
-from loguru import logger
-
-from coocan import MiniSpider, Request, Response
-
-
-class RecvItemSpider(MiniSpider):
-    start_urls = ["https://cn.bing.com/search?q=1"]
-    max_requests = 10
-
-    def parse(self, response: Response):
-        logger.warning("{} {}".format(response.status_code, response.request.url, response.get_one("//title/text()")))
-        for _ in range(10):
-            item = {"timestamp": int(time.time() * 1000), "mark": random.randint(1, 10000)}  # 假设这里是爬虫的数据
-            yield item
-        head, tail = str(response.request.url).split("=")
-        next_url = "{}={}".format(head, int(tail) + 1)
-        if next_url.endswith("11"):
-            yield "coocan"  # 出现警告日志
-            return
-        yield Request(next_url, callback=self.parse)
-
-    def process_item(self, item: dict):
-        logger.success("Get => {}".format(item))
-
-
-if __name__ == '__main__':
-    s = RecvItemSpider()
-    s.go()
+import random
+import time
+
+from loguru import logger
+
+from coocan import MiniSpider, Request, Response
+
+
+class RecvItemSpider(MiniSpider):
+    start_urls = ["https://cn.bing.com/search?q=1"]
+    max_requests = 10
+
+    def parse(self, response: Response):
+        logger.warning("{} {}".format(response.status_code, response.request.url, response.get_one("//title/text()")))
+        for _ in range(10):
+            item = {"timestamp": int(time.time() * 1000), "mark": random.randint(1, 10000)}  # 假设这里是爬虫的数据
+            yield item
+        head, tail = str(response.request.url).split("=")
+        next_url = "{}={}".format(head, int(tail) + 1)
+        if next_url.endswith("11"):
+            yield "coocan"  # 出现警告日志
+            return
+        yield Request(next_url, callback=self.parse)
+
+    def process_item(self, item: dict):
+        logger.success("Get => {}".format(item))
+
+
+if __name__ == '__main__':
+    s = RecvItemSpider()
+    s.go()

coocan/_examples/view_local_ip.py

@@ -1,22 +1,22 @@
-from coocan import Request, Response, MiniSpider
-
-
-class ViewLocalIPSpider(MiniSpider):
-    start_urls = ["https://httpbin.org/ip"]
-    max_requests = 5
-    delay = 5
-
-    def start_requests(self):
-        for _ in range(10):
-            yield Request(self.start_urls[0], callback=self.parse)
-
-    def middleware(self, request: Request):
-        request.headers["Referer"] = "https://httpbin.org"
-
-    def parse(self, response: Response):
-        print(response.status_code, response.json())
-
-
-if __name__ == '__main__':
-    s = ViewLocalIPSpider()
-    s.go()
+from coocan import Request, Response, MiniSpider
+
+
+class ViewLocalIPSpider(MiniSpider):
+    start_urls = ["https://httpbin.org/ip"]
+    max_requests = 5
+    delay = 5
+
+    def start_requests(self):
+        for _ in range(10):
+            yield Request(self.start_urls[0], callback=self.parse)
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "https://httpbin.org"
+
+    def parse(self, response: Response):
+        print(response.status_code, response.json())
+
+
+if __name__ == '__main__':
+    s = ViewLocalIPSpider()
+    s.go()

coocan/cmd/cli.py

@@ -1,68 +1,68 @@
-import os
-import re
-from pathlib import Path
-
-import click
-
-TEMPLATE_DIR = Path(__file__).parent.parent / 'templates'
-
-help_info = """
- ██████╗ ██████╗  ██████╗  ██████╗ █████╗ ███╗   ██╗
-██╔════╝██╔═══██╗██╔═══██╗██╔════╝██╔══██╗████╗  ██║
-██║     ██║   ██║██║   ██║██║     ███████║██╔██╗ ██║
-██║     ██║   ██║██║   ██║██║     ██╔══██║██║╚██╗██║
-╚██████╗╚██████╔╝╚██████╔╝╚██████╗██║  ██║██║ ╚████║
- ╚═════╝ ╚═════╝  ╚═════╝  ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝
-"""
-
-
-def snake_to_pascal(snake_str: str):
-    """小蛇变成大驼峰"""
-    words = snake_str.split('_')
-    pascal_str = ''.join(word.capitalize() for word in words)
-    return pascal_str
-
-
-@click.group(invoke_without_command=True)
-@click.pass_context
-def main(ctx):
-    if ctx.invoked_subcommand is None:
-        print(help_info)
-        click.echo("coocan new -s <spider_file_name>")
-
-
-@main.command()
-@click.option('-s', '--spider', required=True, help='爬虫文件名字')
-def new(spider: str):
-    """新建"""
-    if not re.search("^[a-zA-Z0-9_]*$", spider):
-        click.echo("只支持字母、数字、下划线")
-        return
-
-    spider_class_name = snake_to_pascal(spider)
-    if not spider_class_name.lower().endswith("spider"):
-        spider_class_name += "Spider"
-
-    try:
-        template_path = TEMPLATE_DIR / "spider.txt"
-        with open(template_path, 'r') as f:
-            text = f.read()
-            spider_py_text = text.replace("{SpiderClassName}", spider_class_name)
-
-        py_file = "{}.py".format(spider)
-        if os.path.exists(py_file):
-            click.echo("Failed because file {} already exists".format(py_file))
-            return
-
-        with open(py_file, 'w') as f:
-            f.write(spider_py_text)
-
-        click.echo("Success create {}".format(py_file))
-
-    except Exception as e:
-        click.echo(str(e))
-        raise click.ClickException("Failed")
-
-
-if __name__ == '__main__':
-    main()
+import os
+import re
+from pathlib import Path
+
+import click
+
+TEMPLATE_DIR = Path(__file__).parent.parent / 'templates'
+
+help_info = """
+ ██████╗ ██████╗  ██████╗  ██████╗ █████╗ ███╗   ██╗
+██╔════╝██╔═══██╗██╔═══██╗██╔════╝██╔══██╗████╗  ██║
+██║     ██║   ██║██║   ██║██║     ███████║██╔██╗ ██║
+██║     ██║   ██║██║   ██║██║     ██╔══██║██║╚██╗██║
+╚██████╗╚██████╔╝╚██████╔╝╚██████╗██║  ██║██║ ╚████║
+ ╚═════╝ ╚═════╝  ╚═════╝  ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝
+"""
+
+
+def snake_to_pascal(snake_str: str):
+    """小蛇变成大驼峰"""
+    words = snake_str.split('_')
+    pascal_str = ''.join(word.capitalize() for word in words)
+    return pascal_str
+
+
+@click.group(invoke_without_command=True)
+@click.pass_context
+def main(ctx):
+    if ctx.invoked_subcommand is None:
+        print(help_info)
+        click.echo("coocan new -s <spider_file_name>")
+
+
+@main.command()
+@click.option('-s', '--spider', required=True, help='爬虫文件名字')
+def new(spider: str):
+    """新建"""
+    if not re.search("^[a-zA-Z0-9_]*$", spider):
+        click.echo("只支持字母、数字、下划线")
+        return
+
+    spider_class_name = snake_to_pascal(spider)
+    if not spider_class_name.lower().endswith("spider"):
+        spider_class_name += "Spider"
+
+    try:
+        template_path = TEMPLATE_DIR / "spider.txt"
+        with open(template_path, 'r') as f:
+            text = f.read()
+            spider_py_text = text.replace("{SpiderClassName}", spider_class_name)
+
+        py_file = "{}.py".format(spider)
+        if os.path.exists(py_file):
+            click.echo("Failed because file {} already exists".format(py_file))
+            return
+
+        with open(py_file, 'w') as f:
+            f.write(spider_py_text)
+
+        click.echo("Success create {}".format(py_file))
+
+    except Exception as e:
+        click.echo(str(e))
+        raise click.ClickException("Failed")
+
+
+if __name__ == '__main__':
+    main()

coocan/gen.py

@@ -1,33 +1,33 @@
-import random
-
-
-def gen_random_os() -> str:
-    """生成一个随机的操作系统"""
-    os_choices = [
-        "Windows NT 10.0; Win64; x64",
-        "Windows NT 6.1; WOW64",
-        "Macintosh; Intel Mac OS X 10_15_6",
-        "X11; Linux x86_64",
-        "Windows NT 6.3; Trident/7.0",
-    ]
-    return random.choice(os_choices)
-
-
-def gen_random_browser() -> str:
-    """生成一个随机的浏览器类型和版本"""
-    browser_choices = [
-        ("Chrome", random.randint(70, 100)),
-        ("Firefox", random.randint(70, 100)),
-        ("Edge", random.randint(80, 100)),
-        ("Safari", random.randint(10, 14)),
-        ("Opera", random.randint(60, 80)),
-    ]
-    browser, version = random.choice(browser_choices)
-    return f"{browser}/{version}.0"
-
-
-def gen_random_ua() -> str:
-    """生成一个随机的UA"""
-    os, browser = gen_random_os(), gen_random_browser()
-    ua = f"Mozilla/5.0 ({os}) AppleWebKit/537.36 (KHTML, like Gecko) {browser} Safari/537.36"
-    return ua
+import random
+
+
+def gen_random_os() -> str:
+    """生成一个随机的操作系统"""
+    os_choices = [
+        "Windows NT 10.0; Win64; x64",
+        "Windows NT 6.1; WOW64",
+        "Macintosh; Intel Mac OS X 10_15_6",
+        "X11; Linux x86_64",
+        "Windows NT 6.3; Trident/7.0",
+    ]
+    return random.choice(os_choices)
+
+
+def gen_random_browser() -> str:
+    """生成一个随机的浏览器类型和版本"""
+    browser_choices = [
+        ("Chrome", random.randint(70, 100)),
+        ("Firefox", random.randint(70, 100)),
+        ("Edge", random.randint(80, 100)),
+        ("Safari", random.randint(10, 14)),
+        ("Opera", random.randint(60, 80)),
+    ]
+    browser, version = random.choice(browser_choices)
+    return f"{browser}/{version}.0"
+
+
+def gen_random_ua() -> str:
+    """生成一个随机的UA"""
+    os, browser = gen_random_os(), gen_random_browser()
+    ua = f"Mozilla/5.0 ({os}) AppleWebKit/537.36 (KHTML, like Gecko) {browser} Safari/537.36"
+    return ua

coocan/push_project.py

@@ -1,12 +1,12 @@
-import os
-import sys
-
-msg = sys.argv[1] if len(sys.argv) == 2 else "Auto Submit"
-
-cmd1 = "git add ."
-cmd2 = 'git commit -m "{}"'.format(msg)
-cmd3 = "git push"
-
-os.system(cmd1)
-os.system(cmd2)
-os.system(cmd3)
+import os
+import sys
+
+msg = sys.argv[1] if len(sys.argv) == 2 else "update readme"
+
+cmd1 = "git add ."
+cmd2 = 'git commit -m "{}"'.format(msg)
+cmd3 = "git push"
+
+os.system(cmd1)
+os.system(cmd2)
+os.system(cmd3)

coocan/spider/__init__.py

@@ -1 +1 @@
-from coocan.spider.base import MiniSpider, IgnoreRequest
+from coocan.spider.base import MiniSpider, IgnoreRequest