coocan 0.4.5__tar.gz → 0.4.7__tar.gz

coocan-0.4.7/PKG-INFO

@@ -0,0 +1,90 @@
+Metadata-Version: 2.1
+Name: coocan
+Version: 0.4.7
+Summary: Air Spider Framework
+Author: wauo
+Author-email: markadc@126.com
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# 项目说明
+
+- 一个非常轻量的异步爬虫框架
+
+# 项目地址
+
+- https://github.com/markadc/coocan
+
+## demo
+
+- 效果
+  <br>
+  ![效果](coocan/demo.gif)
+
+
+- 代码
+
+```python
+import json
+
+from loguru import logger
+
+import coocan
+from coocan import Request, MiniSpider
+
+
+class CSDNDetailSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            logger.info(
+                """
+                {} 
+                {} 
+                {}
+                """.format(date, name, detail_url)
+            )
+            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
+
+        logger.info("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_detail(self, response, title):
+        logger.success("{}  已访问 {}".format(response.status_code, title))
+
+
+if __name__ == '__main__':
+    s = CSDNDetailSpider()
+    s.go()
+```

coocan-0.4.7/README.md

@@ -0,0 +1,81 @@
+# 项目说明
+
+- 一个非常轻量的异步爬虫框架
+
+# 项目地址
+
+- https://github.com/markadc/coocan
+
+## demo
+
+- 效果
+  <br>
+  ![效果](coocan/demo.gif)
+
+
+- 代码
+
+```python
+import json
+
+from loguru import logger
+
+import coocan
+from coocan import Request, MiniSpider
+
+
+class CSDNDetailSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            logger.info(
+                """
+                {} 
+                {} 
+                {}
+                """.format(date, name, detail_url)
+            )
+            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
+
+        logger.info("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_detail(self, response, title):
+        logger.success("{}  已访问 {}".format(response.status_code, title))
+
+
+if __name__ == '__main__':
+    s = CSDNDetailSpider()
+    s.go()
+```

coocan-0.4.7/coocan/cmd/cli.py

@@ -0,0 +1,66 @@
+import re
+from pathlib import Path
+
+import click
+
+TEMPLATE_DIR = Path(__file__).parent / "templates"
+
+help_info = """
+ ██████╗ ██████╗  ██████╗  ██████╗ █████╗ ███╗   ██╗
+██╔════╝██╔═══██╗██╔═══██╗██╔════╝██╔══██╗████╗  ██║
+██║     ██║   ██║██║   ██║██║     ███████║██╔██╗ ██║
+██║     ██║   ██║██║   ██║██║     ██╔══██║██║╚██╗██║
+╚██████╗╚██████╔╝╚██████╔╝╚██████╗██║  ██║██║ ╚████║
+ ╚═════╝ ╚═════╝  ╚═════╝  ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝
+"""
+
+
+def show_help_info():
+    print(help_info)
+
+
+def snake_to_pascal(snake_str: str):
+    """小蛇变成大驼峰"""
+    words = snake_str.split('_')
+    pascal_str = ''.join(word.capitalize() for word in words)
+    return pascal_str
+
+
+@click.group(invoke_without_command=True)  # 允许无子命令调用
+@click.pass_context
+def main(ctx):
+    if ctx.invoked_subcommand is None:
+        show_help_info()
+        click.echo("cc new -s <spider_file_name>")
+
+
+@main.command()
+@click.option('-s', '--spider', required=True, help='爬虫文件名字')
+def new(spider):
+    """新建"""
+    if not re.search("^[a-zA-Z0-9_]*$", spider):
+        click.echo("只支持字母、数字、下划线")
+        return
+
+    pascal = snake_to_pascal(spider)
+    if not pascal.endswith("Spider"):
+        pascal += "Spider"
+
+    try:
+        template_path = TEMPLATE_DIR / "spider.txt"
+        with open(template_path, 'r') as f:
+            text = f.read()
+            spider_py_text = text.replace("{SpiderClassName}", pascal)
+
+        with open("{}.py".format(spider), 'w') as f:
+            f.write(spider_py_text)
+
+        click.echo("Success")
+
+    except Exception as e:
+        click.echo(str(e))
+        raise click.ClickException("Failed")
+
+
+if __name__ == '__main__':
+    main()

coocan-0.4.5/coocan/cmd/templates/spider.py → coocan-0.4.7/coocan/cmd/templates/spider.txt

@@ -1,7 +1,7 @@
 from coocan import Request, Response, MiniSpider
 
 
-class Spider(MiniSpider):
+class {SpiderClassName}(MiniSpider):
     start_urls = ['https://github.com/markadc/coocan']
     max_requests = 10
 
@@ -10,3 +10,8 @@ class Spider(MiniSpider):
 
     def parse(self, response: Response):
         pass
+
+
+if __name__ == '__main__':
+    s = {SpiderClassName}()
+    s.go()

coocan-0.4.7/coocan.egg-info/PKG-INFO

@@ -0,0 +1,90 @@
+Metadata-Version: 2.1
+Name: coocan
+Version: 0.4.7
+Summary: Air Spider Framework
+Author: wauo
+Author-email: markadc@126.com
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# 项目说明
+
+- 一个非常轻量的异步爬虫框架
+
+# 项目地址
+
+- https://github.com/markadc/coocan
+
+## demo
+
+- 效果
+  <br>
+  ![效果](coocan/demo.gif)
+
+
+- 代码
+
+```python
+import json
+
+from loguru import logger
+
+import coocan
+from coocan import Request, MiniSpider
+
+
+class CSDNDetailSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            logger.info(
+                """
+                {} 
+                {} 
+                {}
+                """.format(date, name, detail_url)
+            )
+            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
+
+        logger.info("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_detail(self, response, title):
+        logger.success("{}  已访问 {}".format(response.status_code, title))
+
+
+if __name__ == '__main__':
+    s = CSDNDetailSpider()
+    s.go()
+```

{coocan-0.4.5 → coocan-0.4.7}/coocan.egg-info/SOURCES.txt

@@ -1,3 +1,4 @@
+README.md
 setup.py
 coocan/__init__.py
 coocan/gen.py
@@ -9,8 +10,7 @@ coocan.egg-info/requires.txt
 coocan.egg-info/top_level.txt
 coocan/cmd/__init__.py
 coocan/cmd/cli.py
-coocan/cmd/templates/__init__.py
-coocan/cmd/templates/spider.py
+coocan/cmd/templates/spider.txt
 coocan/spider/__init__.py
 coocan/spider/base.py
 coocan/url/__init__.py

{coocan-0.4.5 → coocan-0.4.7}/setup.py

@@ -1,13 +1,20 @@
 from setuptools import setup, find_packages
 
+with open("README.md", "r", encoding="utf-8") as f:
+    long_description = f.read()
+
 setup(
     name="coocan",
-    version="0.4.5",
+    version="0.4.7",
     author="wauo",
     author_email="markadc@126.com",
     description="Air Spider Framework",
     packages=find_packages(),
     python_requires=">=3.10",
+
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+
     install_requires=[
         'click>=8.0.0', 'httpx', 'loguru'
     ],
@@ -15,5 +22,9 @@ setup(
         'console_scripts': [
             'cc=coocan.cmd.cli:main',
         ],
-    }
+    },
+    package_data={
+        'coocan.cmd': ['templates/*'],
+    },
+    include_package_data=True
 )

coocan-0.4.5/PKG-INFO

@@ -1,7 +0,0 @@
-Metadata-Version: 2.1
-Name: coocan
-Version: 0.4.5
-Summary: Air Spider Framework
-Author: wauo
-Author-email: markadc@126.com
-Requires-Python: >=3.10

coocan-0.4.5/coocan/cmd/cli.py

@@ -1,41 +0,0 @@
-from pathlib import Path
-
-import click
-
-TEMPLATE_DIR = Path(__file__).parent / "templates"
-
-
-@click.group()
-def main():
-    """
-    \n
-    可用命令：
-      new - 创建新的爬虫文件
-    \n
-    示例：
-      cc new -s demo
-    """
-
-
-@main.command()
-@click.option('-s', '--spider', required=True, help='爬虫文件')
-def new(spider):
-    """新建"""
-    spider_file_name = "{}.py".format(spider)
-    try:
-        template_path = TEMPLATE_DIR / "spider.py"
-        with open(template_path, 'r') as f:
-            content = f.read()
-
-        with open(spider_file_name, 'w') as f:
-            f.write(content)
-
-        click.echo("Success")
-
-    except Exception as e:
-        click.echo(str(e))
-        raise click.ClickException("Failed")
-
-
-if __name__ == '__main__':
-    main()

coocan-0.4.5/coocan.egg-info/PKG-INFO

@@ -1,7 +0,0 @@
-Metadata-Version: 2.1
-Name: coocan
-Version: 0.4.5
-Summary: Air Spider Framework
-Author: wauo
-Author-email: markadc@126.com
-Requires-Python: >=3.10