coocan 0.5.5__py3-none-any.whl → 0.5.6__py3-none-any.whl

{coocan-0.5.5.dist-info → coocan-0.5.6.dist-info}/METADATA

@@ -1,99 +1,97 @@
-Metadata-Version: 2.4
-Name: coocan
-Version: 0.5.5
-Summary: Air Async Spider Framework
-Home-page: https://github.com/markadc/coocan
-Author: wauo
-Author-email: wauo <markadc@126.com>
-License-Expression: MIT
-Project-URL: Homepage, https://github.com/markadc/coocan
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-Requires-Dist: click>=8.0.0
-Requires-Dist: httpx
-Requires-Dist: loguru
-Dynamic: author
-Dynamic: home-page
-Dynamic: requires-python
-
-# 项目说明
-
-- 一个非常轻量的异步爬虫框架
-
-# 项目地址
-
-- https://github.com/markadc/coocan
-
-## demo
-
-- 效果
-  <br>
-  ![效果](coocan/demo.gif)
-
-
-- 代码
-
-```python
-import json
-
-from loguru import logger
-
-import coocan
-from coocan import Request, MiniSpider
-
-
-class CSDNDetailSpider(MiniSpider):
-    start_urls = ['http://www.csdn.net']
-    max_requests = 10
-
-    def middleware(self, request: Request):
-        request.headers["Referer"] = "http://www.csdn.net/"
-
-    def parse(self, response):
-        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
-        params = {
-            "page": "1",
-            "size": "20",
-            "businessType": "lately",
-            "noMore": "false",
-            "username": "markadc"
-        }
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-    def parse_page(self, response, api, params):
-        current_page = params["page"]
-        data = json.loads(response.text)
-        some = data["data"]["list"]
-
-        if not some:
-            logger.warning("没有第 {} 页".format(current_page))
-            return
-
-        for one in some:
-            date = one["formatTime"]
-            name = one["title"]
-            detail_url = one["url"]
-            logger.info(
-                """
-                {} 
-                {} 
-                {}
-                """.format(date, name, detail_url)
-            )
-            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
-
-        logger.info("第 {} 页抓取成功".format(params["page"]))
-
-        # 抓取下一页
-        next_page = int(current_page) + 1
-        params["page"] = str(next_page)
-        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
-
-    def parse_detail(self, response, title):
-        logger.success("{}  已访问 {}".format(response.status_code, title))
-
-
-if __name__ == '__main__':
-    s = CSDNDetailSpider()
-    s.go()
-```
+Metadata-Version: 2.4
+Name: coocan
+Version: 0.5.6
+Summary: Air Async Spider Framework
+Author: wauo
+Author-email: wauo <markadc@126.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/markadc/coocan
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.0.0
+Requires-Dist: httpx
+Requires-Dist: loguru
+Dynamic: author
+Dynamic: requires-python
+
+# 项目说明
+
+- 一个非常轻量的异步爬虫框架
+
+# 项目地址
+
+- https://github.com/markadc/coocan
+
+## demo
+
+- 效果
+  <br>
+  ![效果](coocan/demo.gif)
+
+
+- 代码
+
+```python
+import json
+
+from loguru import logger
+
+import coocan
+from coocan import Request, MiniSpider
+
+
+class CSDNDetailSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            logger.info(
+                """
+                {} 
+                {} 
+                {}
+                """.format(date, name, detail_url)
+            )
+            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
+
+        logger.info("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_detail(self, response, title):
+        logger.success("{}  已访问 {}".format(response.status_code, title))
+
+
+if __name__ == '__main__':
+    s = CSDNDetailSpider()
+    s.go()
+```

coocan-0.5.6.dist-info/RECORD

@@ -0,0 +1,20 @@
+coocan/__init__.py,sha256=R1gUEUP9v_2iA1gE5twrxr-XRnPhP9EFftLrDeudAoA,53
+coocan/gen.py,sha256=4MCE3t24m3-rbq2snAzByPe58VAo-ShWn58iXpcEiBE,995
+coocan/push_project.py,sha256=rzxPf4VoddzQnjQvH_gaTSiRGMt1GZxaQve-hiNKqkA,210
+coocan/_examples/crawl_csdn_detail.py,sha256=S3lGihGZF-6KI_Kg5H23BQ9cVzOkZwKrS78n1lYOcAg,1830
+coocan/_examples/crawl_csdn_list.py,sha256=D7j5W0WM_52PoWv-2KLuts2r4rabMXavbMS3wnIg6Gk,1454
+coocan/_examples/recv_item.py,sha256=Iym6RbvL7j87SvK14Hw2Exvxx047jEF4zQV9yo4ZXF4,976
+coocan/_examples/view_local_ip.py,sha256=AcatCwtFF6NeYdzbvD8SXun0wn0IL0pX2D3GWUlQ0Sc,560
+coocan/cmd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+coocan/cmd/cli.py,sha256=T7U0QUtAC3O_ZAsglZoOo3nqUPmfceJWbkdso-SwI18,2412
+coocan/spider/__init__.py,sha256=nqERS5a5eSgJfBiIp5moZvwS6JOToNCM_4kreRrtmaQ,57
+coocan/spider/base.py,sha256=C26B7orcmGczVDq8MmAVH_VCgEQ17gszBZQaXflXkzg,6165
+coocan/templates/spider.txt,sha256=1wcbmnv9mBi-21pdygDMukiMy6lEAbvYVRhVOfNY99k,463
+coocan/url/__init__.py,sha256=KN0lLNVaAISoITrPyjD2HOf2A9UYb-9Bbw4xfs9Zqk4,100
+coocan/url/request.py,sha256=6HJQ3Z_11BYpTqV_u62vks_RS6bQbMbd6B1UytRDq5Y,1085
+coocan/url/response.py,sha256=ruIzOcFcJqszTmbNV9y1BSxdWeyVydsMJd-cDMTJHLo,1735
+coocan-0.5.6.dist-info/METADATA,sha256=G4tEBLo9AN-QJ4q1FVYYU9uStAH2eO6q8t1KPBef4ks,2416
+coocan-0.5.6.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+coocan-0.5.6.dist-info/entry_points.txt,sha256=hNdk42NPboC1o7s7GzMbpII5t2U2jWrtT5bpvliXRcw,47
+coocan-0.5.6.dist-info/top_level.txt,sha256=VwB-Q4zEljgb9v1Ms1E59B-1pBYORXuhKjgZb-LHOhk,7
+coocan-0.5.6.dist-info/RECORD,,

{coocan-0.5.5.dist-info → coocan-0.5.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.0.0)
+Generator: setuptools (80.3.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{coocan-0.5.5.dist-info → coocan-0.5.6.dist-info}/top_level.txt

@@ -1,2 +1 @@
-_test
 coocan

_test/crawl_csdn.py

@@ -1,53 +0,0 @@
-import json
-
-from loguru import logger
-
-import coocan
-from coocan import Request, MiniSpider
-
-api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
-params = {
-    "page": "1",
-    "size": "20",
-    "businessType": "lately",
-    "noMore": "false",
-    "username": "markadc"
-}
-
-
-class CsdnAirAsyncSpider(MiniSpider):
-    start_urls = ['http://www.csdn.net']
-    max_requests = 10
-
-    def parse(self, response):
-        yield coocan.Request(api, self.parse_page, params=params)
-
-    def middleware(self, request: Request):
-        request.headers["Referer"] = "http://www.csdn.net/"
-
-    def parse_page(self, response):
-        current_page = params["page"]
-        data = json.loads(response.text)
-        some = data["data"]["list"]
-        if not some:
-            logger.warning("没有第 {} 页".format(current_page))
-            return
-        for one in some:
-            date = one["formatTime"]
-            name = one["title"]
-            detail_url = one["url"]
-            yield coocan.Request(detail_url, self.parse_detail)
-            print(date, detail_url, name)
-        logger.info("第 {} 页抓取成功".format(params["page"]))
-
-        next_page = int(current_page) + 1
-        params["page"] = str(next_page)
-        yield coocan.Request(api, self.parse_page, params=params)
-
-    def parse_detail(self, response):
-        logger.success("{} {}".format(response.status_code, response.request.url))
-
-
-if __name__ == '__main__':
-    s = CsdnAirAsyncSpider()
-    s.go()

_test/demo.py

@@ -1,33 +0,0 @@
-from loguru import logger
-
-import coocan
-
-
-class DemoSpider(coocan.MiniSpider):
-    start_urls = ["https://cn.bing.com/"]
-    max_requests = 5
-
-    def parse(self, response):
-        print(response.request.headers.get("User-Agent"))
-        logger.debug('{} {}'.format(response.status_code, len(response.text)))
-        for i in range(5):
-            yield coocan.Request('https://cn.bing.com/', self.parse2)
-
-    def parse2(self, response):
-        logger.info('{} {}'.format(response.status_code, len(response.text)))
-        for i in range(3):
-            yield coocan.Request('https://cn.bing.com/', self.parse3)
-
-        for i in range(4):
-            yield coocan.Request('https://cn.bing.com/', self.parse4)
-
-    def parse3(self, response):
-        logger.warning('{} {}'.format(response.status_code, len(response.text)))
-
-    def parse4(self, response):
-        logger.error('{} {}'.format(response.status_code, len(response.text)))
-
-
-if __name__ == '__main__':
-    my_spider = DemoSpider()
-    my_spider.go()

_test/err_demo.py

@@ -1,27 +0,0 @@
-from loguru import logger
-
-import coocan
-from coocan.spider import MiniSpider
-
-
-class ErrDemoSpider(MiniSpider):
-    start_urls = ["https://cn.bing.com/"]
-    max_requests = 5
-
-    def parse(self, response):
-        print(response.request.headers.get("User-Agent"))
-        logger.debug('{} {}'.format(response.status_code, len(response.text)))
-        yield coocan.Request('https://cn.bing.com/', self.parse2, cb_kwargs={"name": "CLOS"})
-
-    def parse2(self, response, name):
-        print(name)
-        logger.debug('{} {}'.format(response.status_code, len(response.text)))
-        yield coocan.Request('https://cn.bing.com/', self.parse3, cb_kwargs={"a1": 1, "a2": 2})
-
-    def parse3(self, response, a1, a22):
-        print(a1, a22)
-
-
-if __name__ == '__main__':
-    my_spider = ErrDemoSpider()
-    my_spider.go()

_test/test_priority.py

@@ -1,21 +0,0 @@
-from coocan import MiniSpider, Request, Response
-
-
-class TestPrioritySpider(MiniSpider):
-    headers_extra_field = {"Name": "Coocan"}
-
-    def start_requests(self):
-        for i in range(100):
-            url = 'https://www.baidu.com/s?w={}'.format(i)
-            yield Request(url, callback=self.parse, priority=100 - i)
-
-    def parse(self, response: Response):
-        print(response.request.url)
-        print(response.request.headers["User-Agent"])
-        print(response.request.headers)
-        print()
-
-
-if __name__ == '__main__':
-    s = TestPrioritySpider()
-    s.go()

_test/test_req_delay.py

@@ -1,19 +0,0 @@
-from coocan import MiniSpider, Request, Response
-
-
-class TestReqDelaySpider(MiniSpider):
-    max_requests = 5
-    delay = 3
-
-    def start_requests(self):
-        for i in range(100):
-            url = 'https://www.baidu.com/s?w={}'.format(i)
-            yield Request(url, callback=self.parse, priority=100 - i)
-
-    def parse(self, response: Response):
-        print(response.request.url)
-
-
-if __name__ == '__main__':
-    s = TestReqDelaySpider()
-    s.go()

_test/test_req_err.py

@@ -1,32 +0,0 @@
-import random
-
-from coocan import MiniSpider, Request, Response, IgnoreRequest
-
-
-class TestReqErrSpider(MiniSpider):
-    def start_requests(self):
-        for i in range(5):
-            url = "https://www.google.com/{}".format(i + 1)
-            yield Request(url, callback=self.parse, timeout=1)
-
-    def handle_request_excetpion(self, e: Exception, request: Request):
-        v = random.randint(1, 3)
-        if v == 1:
-            raise IgnoreRequest("出验证码了")
-        if v == 2:
-            1 / 0
-        if v == 3:
-            new_url = "https://www.baidu.com/s?wd={}".format(random.randint(1, 100))
-            return Request(new_url, callback=self.parse, timeout=1)
-
-    def parse(self, response: Response):
-        v = random.randint(1, 2)
-        if v == 1:
-            print("爬取成功", response.url, len(response.text))
-            print(response.get_one("//title/text()"))
-        aaa
-
-
-if __name__ == '__main__':
-    my_spider = TestReqErrSpider()
-    my_spider.go()

coocan-0.5.5.dist-info/RECORD

@@ -1,26 +0,0 @@
-_test/crawl_csdn.py,sha256=ap2mOq3ps7KEbqqKWH5uJqIK_IQ8YFSRRAMzpreQvww,1555
-_test/demo.py,sha256=ZxfJzWuNVGhDjhUruyVyZ-BoULHMbzgpnxefLSkheZI,1051
-_test/err_demo.py,sha256=EWGqb00KyB192qv3uxMr6YgOr2zKJQb6gkeFtknMLv8,845
-_test/test_priority.py,sha256=K8JLC-PaVM4ztLZdYFCumDQP5m2hB8qWAIWXTOMMUyM,601
-_test/test_req_delay.py,sha256=35afyHcZk3Gmja9xXXjJSHXnU8WVJGph2ZcTQRxRMNk,479
-_test/test_req_err.py,sha256=magK1BUConCBj8TEC29rzmDCbI2u2XXVcPowL6ttP9g,1025
-coocan/__init__.py,sha256=UqFmE7ucuR_xR3OyyBU8pxqLfCJ5AdH_HsDdTsYPf6g,55
-coocan/gen.py,sha256=J6QWXkBVbiCQqey8i0BDqleRNpBswI8AyvrYmkDVQPw,1028
-coocan/push_project.py,sha256=X2fjtYk1oI0ElcibA9wChLx0lCc8hwSelhUNfkJal5o,220
-coocan/_examples/crawl_csdn_detail.py,sha256=J2hiKHCS7RskQ9UmNMjE8i6braFwGchH6BxtdulV9RM,1892
-coocan/_examples/crawl_csdn_list.py,sha256=ZvhFvBbVXQe-qtXf1T_waXuM4tBleBqbpvzP-5z0RCg,1504
-coocan/_examples/recv_item.py,sha256=iJqPuHZ2FykeleFl0Xr0yPwq4UhCnNw84lCPlYyGFzM,1007
-coocan/_examples/view_local_ip.py,sha256=Sl086xNNuZqFoRM31_gMvcISSa2QoL3OGghECkQktxg,582
-coocan/cmd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-coocan/cmd/cli.py,sha256=FRggXqDeLsGs-7u3zhaokfk0GCpItwqudf14W9zUfYE,2480
-coocan/spider/__init__.py,sha256=kMDCGeqtN50raCzwfCn18s_W8xV6KO_Ny9Xol4I48Ag,58
-coocan/spider/base.py,sha256=9Dgn2920Lb9TZGV0cAZSBMvIWuTMqs9M8ZYspx9W0Io,6342
-coocan/templates/spider.txt,sha256=5UEXUzb0ses_4ctn0b3vgbpUJ7tCde91ul6rp-g7Hxw,480
-coocan/url/__init__.py,sha256=rEMx66XDy5AIJ9mF_2UVzHW5mRLBAWZEyQ3txrZzuZA,102
-coocan/url/request.py,sha256=seZaQXQRvRMIf9WnCp3mAgNA-kxsj9P2JzAvuIt2Dx8,1116
-coocan/url/response.py,sha256=AnC0xsF34q68r62EVlcHYmDH6skm9RBwRHITTb4iBbU,1785
-coocan-0.5.5.dist-info/METADATA,sha256=aYnL3IygL_Dw7iwvDtK81oVQRWIygXmiiSO94C01Bv4,2579
-coocan-0.5.5.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
-coocan-0.5.5.dist-info/entry_points.txt,sha256=hNdk42NPboC1o7s7GzMbpII5t2U2jWrtT5bpvliXRcw,47
-coocan-0.5.5.dist-info/top_level.txt,sha256=WiN3Gh529qzUs0jVvEReeZsKxFguIQKrFlMOjtxGblM,13
-coocan-0.5.5.dist-info/RECORD,,