coocan 0.5.5__tar.gz → 0.5.6__tar.gz

{coocan-0.5.5 → coocan-0.5.6}/PKG-INFO

@@ -1,99 +1,97 @@
-Metadata-Version: 2.4
-Name: coocan
-Version: 0.5.5
-Summary: Air Async Spider Framework
-Home-page: https://github.com/markadc/coocan
-Author: wauo
-Author-email: wauo <markadc@126.com>
-License-Expression: MIT
-Project-URL: Homepage, https://github.com/markadc/coocan
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-Requires-Dist: click>=8.0.0
-Requires-Dist: httpx
-Requires-Dist: loguru
-Dynamic: author
-Dynamic: home-page
-Dynamic: requires-python
-
-# 项目说明
-
-- 一个非常轻量的异步爬虫框架
-
-# 项目地址
-
-- https://github.com/markadc/coocan
-
-## demo
-
-- 效果
-  <br>
-  ![效果](coocan/demo.gif)
-
-
-- 代码
-
-```python
-import json
-
-from loguru import logger
-
-import coocan
-from coocan import Request, MiniSpider
-
-
-class CSDNDetailSpider(MiniSpider):
-    start_urls = ['http://www.csdn.net']
-    max_requests = 10
-
-    def middleware(self, request: Request):
-        request.headers["Referer"] = "http://www.csdn.net/"
-
-    def parse(self, response):
-        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
-        params = {
-            "page": "1",
-            "size": "20",
-            "businessType": "lately",
-            "noMore": "false",
-            "username": "markadc"
-        }
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-    def parse_page(self, response, api, params):
-        current_page = params["page"]
-        data = json.loads(response.text)
-        some = data["data"]["list"]
-
-        if not some:
-            logger.warning("没有第 {} 页".format(current_page))
-            return
-
-        for one in some:
-            date = one["formatTime"]
-            name = one["title"]
-            detail_url = one["url"]
-            logger.info(
-                """
-                {} 
-                {} 
-                {}
-                """.format(date, name, detail_url)
-            )
-            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
-
-        logger.info("第 {} 页抓取成功".format(params["page"]))
-
-        # 抓取下一页
-        next_page = int(current_page) + 1
-        params["page"] = str(next_page)
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-    def parse_detail(self, response, title):
-        logger.success("{}  已访问 {}".format(response.status_code, title))
-
-
-if __name__ == '__main__':
-    s = CSDNDetailSpider()
-    s.go()
-```
+Metadata-Version: 2.4
+Name: coocan
+Version: 0.5.6
+Summary: Air Async Spider Framework
+Author: wauo
+Author-email: wauo <markadc@126.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/markadc/coocan
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.0.0
+Requires-Dist: httpx
+Requires-Dist: loguru
+Dynamic: author
+Dynamic: requires-python
+
+# 项目说明
+
+- 一个非常轻量的异步爬虫框架
+
+# 项目地址
+
+- https://github.com/markadc/coocan
+
+## demo
+
+- 效果
+  <br>
+  ![效果](coocan/demo.gif)
+
+
+- 代码
+
+```python
+import json
+
+from loguru import logger
+
+import coocan
+from coocan import Request, MiniSpider
+
+
+class CSDNDetailSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            logger.info(
+                """
+                {} 
+                {} 
+                {}
+                """.format(date, name, detail_url)
+            )
+            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
+
+        logger.info("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_detail(self, response, title):
+        logger.success("{}  已访问 {}".format(response.status_code, title))
+
+
+if __name__ == '__main__':
+    s = CSDNDetailSpider()
+    s.go()
+```

{coocan-0.5.5 → coocan-0.5.6}/coocan/__init__.py

@@ -1,2 +1,2 @@
-from coocan.spider import *
-from coocan.url import *
+from coocan.spider import *
+from coocan.url import *

{coocan-0.5.5 → coocan-0.5.6}/coocan/_examples/crawl_csdn_detail.py

@@ -1,62 +1,62 @@
-import json
-
-from loguru import logger
-
-import coocan
-from coocan import Request, MiniSpider
-
-
-class CSDNDetailSpider(MiniSpider):
-    start_urls = ['http://www.csdn.net']
-    max_requests = 10
-
-    def middleware(self, request: Request):
-        request.headers["Referer"] = "http://www.csdn.net/"
-
-    def parse(self, response):
-        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
-        params = {
-            "page": "1",
-            "size": "20",
-            "businessType": "lately",
-            "noMore": "false",
-            "username": "markadc"
-        }
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-    def parse_page(self, response, api, params):
-        current_page = params["page"]
-        data = json.loads(response.text)
-        some = data["data"]["list"]
-
-        if not some:
-            logger.warning("没有第 {} 页".format(current_page))
-            return
-
-        for one in some:
-            date = one["formatTime"]
-            name = one["title"]
-            detail_url = one["url"]
-            logger.info(
-                """
-                {} 
-                {} 
-                {}
-                """.format(date, name, detail_url)
-            )
-            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
-
-        logger.info("第 {} 页抓取成功".format(params["page"]))
-
-        # 抓取下一页
-        next_page = int(current_page) + 1
-        params["page"] = str(next_page)
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-    def parse_detail(self, response, title):
-        logger.success("{}  已访问 {}".format(response.status_code, title))
-
-
-if __name__ == '__main__':
-    s = CSDNDetailSpider()
-    s.go()
+import json
+
+from loguru import logger
+
+import coocan
+from coocan import Request, MiniSpider
+
+
+class CSDNDetailSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            logger.info(
+                """
+                {} 
+                {} 
+                {}
+                """.format(date, name, detail_url)
+            )
+            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
+
+        logger.info("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_detail(self, response, title):
+        logger.success("{}  已访问 {}".format(response.status_code, title))
+
+
+if __name__ == '__main__':
+    s = CSDNDetailSpider()
+    s.go()

{coocan-0.5.5 → coocan-0.5.6}/coocan/_examples/crawl_csdn_list.py

@@ -1,50 +1,50 @@
-import json
-
-from loguru import logger
-
-from coocan import Request, MiniSpider
-
-
-class CSDNSpider(MiniSpider):
-    start_urls = ['http://www.csdn.net']
-    max_requests = 10
-
-    def middleware(self, request: Request):
-        request.headers["Referer"] = "http://www.csdn.net/"
-
-    def parse(self, response):
-        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
-        params = {
-            "page": "1",
-            "size": "20",
-            "businessType": "lately",
-            "noMore": "false",
-            "username": "markadc"
-        }
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-    def parse_page(self, response, api, params):
-        current_page = params["page"]
-        data = json.loads(response.text)
-        some = data["data"]["list"]
-
-        if not some:
-            logger.warning("没有第 {} 页".format(current_page))
-            return
-
-        for one in some:
-            date = one["formatTime"]
-            name = one["title"]
-            detail_url = one["url"]
-            print(date, detail_url, name)
-        print("第 {} 页抓取成功".format(params["page"]))
-
-        # 抓取下一页
-        next_page = int(current_page) + 1
-        params["page"] = str(next_page)
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-
-if __name__ == '__main__':
-    s = CSDNSpider()
-    s.go()
+import json
+
+from loguru import logger
+
+from coocan import Request, MiniSpider
+
+
+class CSDNSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            print(date, detail_url, name)
+        print("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+
+if __name__ == '__main__':
+    s = CSDNSpider()
+    s.go()

{coocan-0.5.5 → coocan-0.5.6}/coocan/_examples/recv_item.py

@@ -1,31 +1,31 @@
-import random
-import time
-
-from loguru import logger
-
-from coocan import MiniSpider, Request, Response
-
-
-class RecvItemSpider(MiniSpider):
-    start_urls = ["https://cn.bing.com/search?q=1"]
-    max_requests = 10
-
-    def parse(self, response: Response):
-        logger.warning("{} {}".format(response.status_code, response.request.url, response.get_one("//title/text()")))
-        for _ in range(10):
-            item = {"timestamp": int(time.time() * 1000), "mark": random.randint(1, 10000)}  # 假设这里是爬虫的数据
-            yield item
-        head, tail = str(response.request.url).split("=")
-        next_url = "{}={}".format(head, int(tail) + 1)
-        if next_url.endswith("11"):
-            yield "coocan"  # 出现警告日志
-            return
-        yield Request(next_url, callback=self.parse)
-
-    def process_item(self, item: dict):
-        logger.success("Get => {}".format(item))
-
-
-if __name__ == '__main__':
-    s = RecvItemSpider()
-    s.go()
+import random
+import time
+
+from loguru import logger
+
+from coocan import MiniSpider, Request, Response
+
+
+class RecvItemSpider(MiniSpider):
+    start_urls = ["https://cn.bing.com/search?q=1"]
+    max_requests = 10
+
+    def parse(self, response: Response):
+        logger.warning("{} {}".format(response.status_code, response.request.url, response.get_one("//title/text()")))
+        for _ in range(10):
+            item = {"timestamp": int(time.time() * 1000), "mark": random.randint(1, 10000)}  # 假设这里是爬虫的数据
+            yield item
+        head, tail = str(response.request.url).split("=")
+        next_url = "{}={}".format(head, int(tail) + 1)
+        if next_url.endswith("11"):
+            yield "coocan"  # 出现警告日志
+            return
+        yield Request(next_url, callback=self.parse)
+
+    def process_item(self, item: dict):
+        logger.success("Get => {}".format(item))
+
+
+if __name__ == '__main__':
+    s = RecvItemSpider()
+    s.go()

{coocan-0.5.5 → coocan-0.5.6}/coocan/_examples/view_local_ip.py

@@ -1,22 +1,22 @@
-from coocan import Request, Response, MiniSpider
-
-
-class ViewLocalIPSpider(MiniSpider):
-    start_urls = ["https://httpbin.org/ip"]
-    max_requests = 5
-    delay = 5
-
-    def start_requests(self):
-        for _ in range(10):
-            yield Request(self.start_urls[0], callback=self.parse)
-
-    def middleware(self, request: Request):
-        request.headers["Referer"] = "https://httpbin.org"
-
-    def parse(self, response: Response):
-        print(response.status_code, response.json())
-
-
-if __name__ == '__main__':
-    s = ViewLocalIPSpider()
-    s.go()
+from coocan import Request, Response, MiniSpider
+
+
+class ViewLocalIPSpider(MiniSpider):
+    start_urls = ["https://httpbin.org/ip"]
+    max_requests = 5
+    delay = 5
+
+    def start_requests(self):
+        for _ in range(10):
+            yield Request(self.start_urls[0], callback=self.parse)
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "https://httpbin.org"
+
+    def parse(self, response: Response):
+        print(response.status_code, response.json())
+
+
+if __name__ == '__main__':
+    s = ViewLocalIPSpider()
+    s.go()