coocan 0.5.3.1__py3-none-any.whl → 0.5.5__py3-none-any.whl

_test/crawl_csdn.py

@@ -0,0 +1,53 @@
+import json
+
+from loguru import logger
+
+import coocan
+from coocan import Request, MiniSpider
+
+api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+params = {
+    "page": "1",
+    "size": "20",
+    "businessType": "lately",
+    "noMore": "false",
+    "username": "markadc"
+}
+
+
+class CsdnAirAsyncSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def parse(self, response):
+        yield coocan.Request(api, self.parse_page, params=params)
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse_page(self, response):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            yield coocan.Request(detail_url, self.parse_detail)
+            print(date, detail_url, name)
+        logger.info("第 {} 页抓取成功".format(params["page"]))
+
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield coocan.Request(api, self.parse_page, params=params)
+
+    def parse_detail(self, response):
+        logger.success("{} {}".format(response.status_code, response.request.url))
+
+
+if __name__ == '__main__':
+    s = CsdnAirAsyncSpider()
+    s.go()

_test/demo.py

@@ -0,0 +1,33 @@
+from loguru import logger
+
+import coocan
+
+
+class DemoSpider(coocan.MiniSpider):
+    start_urls = ["https://cn.bing.com/"]
+    max_requests = 5
+
+    def parse(self, response):
+        print(response.request.headers.get("User-Agent"))
+        logger.debug('{} {}'.format(response.status_code, len(response.text)))
+        for i in range(5):
+            yield coocan.Request('https://cn.bing.com/', self.parse2)
+
+    def parse2(self, response):
+        logger.info('{} {}'.format(response.status_code, len(response.text)))
+        for i in range(3):
+            yield coocan.Request('https://cn.bing.com/', self.parse3)
+
+        for i in range(4):
+            yield coocan.Request('https://cn.bing.com/', self.parse4)
+
+    def parse3(self, response):
+        logger.warning('{} {}'.format(response.status_code, len(response.text)))
+
+    def parse4(self, response):
+        logger.error('{} {}'.format(response.status_code, len(response.text)))
+
+
+if __name__ == '__main__':
+    my_spider = DemoSpider()
+    my_spider.go()

_test/err_demo.py

@@ -0,0 +1,27 @@
+from loguru import logger
+
+import coocan
+from coocan.spider import MiniSpider
+
+
+class ErrDemoSpider(MiniSpider):
+    start_urls = ["https://cn.bing.com/"]
+    max_requests = 5
+
+    def parse(self, response):
+        print(response.request.headers.get("User-Agent"))
+        logger.debug('{} {}'.format(response.status_code, len(response.text)))
+        yield coocan.Request('https://cn.bing.com/', self.parse2, cb_kwargs={"name": "CLOS"})
+
+    def parse2(self, response, name):
+        print(name)
+        logger.debug('{} {}'.format(response.status_code, len(response.text)))
+        yield coocan.Request('https://cn.bing.com/', self.parse3, cb_kwargs={"a1": 1, "a2": 2})
+
+    def parse3(self, response, a1, a22):
+        print(a1, a22)
+
+
+if __name__ == '__main__':
+    my_spider = ErrDemoSpider()
+    my_spider.go()

_test/test_priority.py

@@ -0,0 +1,21 @@
+from coocan import MiniSpider, Request, Response
+
+
+class TestPrioritySpider(MiniSpider):
+    headers_extra_field = {"Name": "Coocan"}
+
+    def start_requests(self):
+        for i in range(100):
+            url = 'https://www.baidu.com/s?w={}'.format(i)
+            yield Request(url, callback=self.parse, priority=100 - i)
+
+    def parse(self, response: Response):
+        print(response.request.url)
+        print(response.request.headers["User-Agent"])
+        print(response.request.headers)
+        print()
+
+
+if __name__ == '__main__':
+    s = TestPrioritySpider()
+    s.go()

_test/test_req_delay.py

@@ -0,0 +1,19 @@
+from coocan import MiniSpider, Request, Response
+
+
+class TestReqDelaySpider(MiniSpider):
+    max_requests = 5
+    delay = 3
+
+    def start_requests(self):
+        for i in range(100):
+            url = 'https://www.baidu.com/s?w={}'.format(i)
+            yield Request(url, callback=self.parse, priority=100 - i)
+
+    def parse(self, response: Response):
+        print(response.request.url)
+
+
+if __name__ == '__main__':
+    s = TestReqDelaySpider()
+    s.go()

_test/test_req_err.py

@@ -0,0 +1,32 @@
+import random
+
+from coocan import MiniSpider, Request, Response, IgnoreRequest
+
+
+class TestReqErrSpider(MiniSpider):
+    def start_requests(self):
+        for i in range(5):
+            url = "https://www.google.com/{}".format(i + 1)
+            yield Request(url, callback=self.parse, timeout=1)
+
+    def handle_request_excetpion(self, e: Exception, request: Request):
+        v = random.randint(1, 3)
+        if v == 1:
+            raise IgnoreRequest("出验证码了")
+        if v == 2:
+            1 / 0
+        if v == 3:
+            new_url = "https://www.baidu.com/s?wd={}".format(random.randint(1, 100))
+            return Request(new_url, callback=self.parse, timeout=1)
+
+    def parse(self, response: Response):
+        v = random.randint(1, 2)
+        if v == 1:
+            print("爬取成功", response.url, len(response.text))
+            print(response.get_one("//title/text()"))
+        aaa
+
+
+if __name__ == '__main__':
+    my_spider = TestReqErrSpider()
+    my_spider.go()

coocan/_examples/crawl_csdn_detail.py

@@ -0,0 +1,62 @@
+import json
+
+from loguru import logger
+
+import coocan
+from coocan import Request, MiniSpider
+
+
+class CSDNDetailSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            logger.info(
+                """
+                {} 
+                {} 
+                {}
+                """.format(date, name, detail_url)
+            )
+            yield coocan.Request(detail_url, self.parse_detail, cb_kwargs={"title": name})
+
+        logger.info("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_detail(self, response, title):
+        logger.success("{}  已访问 {}".format(response.status_code, title))
+
+
+if __name__ == '__main__':
+    s = CSDNDetailSpider()
+    s.go()

coocan/_examples/crawl_csdn_list.py

@@ -0,0 +1,50 @@
+import json
+
+from loguru import logger
+
+from coocan import Request, MiniSpider
+
+
+class CSDNSpider(MiniSpider):
+    start_urls = ['http://www.csdn.net']
+    max_requests = 10
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "http://www.csdn.net/"
+
+    def parse(self, response):
+        api = "https://blog.csdn.net/community/home-api/v1/get-business-list"
+        params = {
+            "page": "1",
+            "size": "20",
+            "businessType": "lately",
+            "noMore": "false",
+            "username": "markadc"
+        }
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+    def parse_page(self, response, api, params):
+        current_page = params["page"]
+        data = json.loads(response.text)
+        some = data["data"]["list"]
+
+        if not some:
+            logger.warning("没有第 {} 页".format(current_page))
+            return
+
+        for one in some:
+            date = one["formatTime"]
+            name = one["title"]
+            detail_url = one["url"]
+            print(date, detail_url, name)
+        print("第 {} 页抓取成功".format(params["page"]))
+
+        # 抓取下一页
+        next_page = int(current_page) + 1
+        params["page"] = str(next_page)
+        yield Request(api, self.parse_page, params=params, cb_kwargs={"api": api, "params": params})
+
+
+if __name__ == '__main__':
+    s = CSDNSpider()
+    s.go()

coocan/_examples/recv_item.py

@@ -0,0 +1,31 @@
+import random
+import time
+
+from loguru import logger
+
+from coocan import MiniSpider, Request, Response
+
+
+class RecvItemSpider(MiniSpider):
+    start_urls = ["https://cn.bing.com/search?q=1"]
+    max_requests = 10
+
+    def parse(self, response: Response):
+        logger.warning("{} {}".format(response.status_code, response.request.url, response.get_one("//title/text()")))
+        for _ in range(10):
+            item = {"timestamp": int(time.time() * 1000), "mark": random.randint(1, 10000)}  # 假设这里是爬虫的数据
+            yield item
+        head, tail = str(response.request.url).split("=")
+        next_url = "{}={}".format(head, int(tail) + 1)
+        if next_url.endswith("11"):
+            yield "coocan"  # 出现警告日志
+            return
+        yield Request(next_url, callback=self.parse)
+
+    def process_item(self, item: dict):
+        logger.success("Get => {}".format(item))
+
+
+if __name__ == '__main__':
+    s = RecvItemSpider()
+    s.go()

coocan/_examples/view_local_ip.py

@@ -0,0 +1,22 @@
+from coocan import Request, Response, MiniSpider
+
+
+class ViewLocalIPSpider(MiniSpider):
+    start_urls = ["https://httpbin.org/ip"]
+    max_requests = 5
+    delay = 5
+
+    def start_requests(self):
+        for _ in range(10):
+            yield Request(self.start_urls[0], callback=self.parse)
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "https://httpbin.org"
+
+    def parse(self, response: Response):
+        print(response.status_code, response.json())
+
+
+if __name__ == '__main__':
+    s = ViewLocalIPSpider()
+    s.go()

{coocan-0.5.3.1.dist-info → coocan-0.5.5.dist-info}/METADATA

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: coocan
-Version: 0.5.3.1
+Version: 0.5.5
 Summary: Air Async Spider Framework
 Home-page: https://github.com/markadc/coocan
 Author: wauo
 Author-email: wauo <markadc@126.com>
+License-Expression: MIT
 Project-URL: Homepage, https://github.com/markadc/coocan
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

coocan-0.5.5.dist-info/RECORD

@@ -0,0 +1,26 @@
+_test/crawl_csdn.py,sha256=ap2mOq3ps7KEbqqKWH5uJqIK_IQ8YFSRRAMzpreQvww,1555
+_test/demo.py,sha256=ZxfJzWuNVGhDjhUruyVyZ-BoULHMbzgpnxefLSkheZI,1051
+_test/err_demo.py,sha256=EWGqb00KyB192qv3uxMr6YgOr2zKJQb6gkeFtknMLv8,845
+_test/test_priority.py,sha256=K8JLC-PaVM4ztLZdYFCumDQP5m2hB8qWAIWXTOMMUyM,601
+_test/test_req_delay.py,sha256=35afyHcZk3Gmja9xXXjJSHXnU8WVJGph2ZcTQRxRMNk,479
+_test/test_req_err.py,sha256=magK1BUConCBj8TEC29rzmDCbI2u2XXVcPowL6ttP9g,1025
+coocan/__init__.py,sha256=UqFmE7ucuR_xR3OyyBU8pxqLfCJ5AdH_HsDdTsYPf6g,55
+coocan/gen.py,sha256=J6QWXkBVbiCQqey8i0BDqleRNpBswI8AyvrYmkDVQPw,1028
+coocan/push_project.py,sha256=X2fjtYk1oI0ElcibA9wChLx0lCc8hwSelhUNfkJal5o,220
+coocan/_examples/crawl_csdn_detail.py,sha256=J2hiKHCS7RskQ9UmNMjE8i6braFwGchH6BxtdulV9RM,1892
+coocan/_examples/crawl_csdn_list.py,sha256=ZvhFvBbVXQe-qtXf1T_waXuM4tBleBqbpvzP-5z0RCg,1504
+coocan/_examples/recv_item.py,sha256=iJqPuHZ2FykeleFl0Xr0yPwq4UhCnNw84lCPlYyGFzM,1007
+coocan/_examples/view_local_ip.py,sha256=Sl086xNNuZqFoRM31_gMvcISSa2QoL3OGghECkQktxg,582
+coocan/cmd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+coocan/cmd/cli.py,sha256=FRggXqDeLsGs-7u3zhaokfk0GCpItwqudf14W9zUfYE,2480
+coocan/spider/__init__.py,sha256=kMDCGeqtN50raCzwfCn18s_W8xV6KO_Ny9Xol4I48Ag,58
+coocan/spider/base.py,sha256=9Dgn2920Lb9TZGV0cAZSBMvIWuTMqs9M8ZYspx9W0Io,6342
+coocan/templates/spider.txt,sha256=5UEXUzb0ses_4ctn0b3vgbpUJ7tCde91ul6rp-g7Hxw,480
+coocan/url/__init__.py,sha256=rEMx66XDy5AIJ9mF_2UVzHW5mRLBAWZEyQ3txrZzuZA,102
+coocan/url/request.py,sha256=seZaQXQRvRMIf9WnCp3mAgNA-kxsj9P2JzAvuIt2Dx8,1116
+coocan/url/response.py,sha256=AnC0xsF34q68r62EVlcHYmDH6skm9RBwRHITTb4iBbU,1785
+coocan-0.5.5.dist-info/METADATA,sha256=aYnL3IygL_Dw7iwvDtK81oVQRWIygXmiiSO94C01Bv4,2579
+coocan-0.5.5.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
+coocan-0.5.5.dist-info/entry_points.txt,sha256=hNdk42NPboC1o7s7GzMbpII5t2U2jWrtT5bpvliXRcw,47
+coocan-0.5.5.dist-info/top_level.txt,sha256=WiN3Gh529qzUs0jVvEReeZsKxFguIQKrFlMOjtxGblM,13
+coocan-0.5.5.dist-info/RECORD,,

{coocan-0.5.3.1.dist-info → coocan-0.5.5.dist-info}/top_level.txt

@@ -1 +1,2 @@
+_test
 coocan

coocan-0.5.3.1.dist-info/RECORD

@@ -1,16 +0,0 @@
-coocan/__init__.py,sha256=UqFmE7ucuR_xR3OyyBU8pxqLfCJ5AdH_HsDdTsYPf6g,55
-coocan/gen.py,sha256=J6QWXkBVbiCQqey8i0BDqleRNpBswI8AyvrYmkDVQPw,1028
-coocan/push_project.py,sha256=X2fjtYk1oI0ElcibA9wChLx0lCc8hwSelhUNfkJal5o,220
-coocan/cmd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-coocan/cmd/cli.py,sha256=FRggXqDeLsGs-7u3zhaokfk0GCpItwqudf14W9zUfYE,2480
-coocan/spider/__init__.py,sha256=kMDCGeqtN50raCzwfCn18s_W8xV6KO_Ny9Xol4I48Ag,58
-coocan/spider/base.py,sha256=9Dgn2920Lb9TZGV0cAZSBMvIWuTMqs9M8ZYspx9W0Io,6342
-coocan/templates/spider.txt,sha256=5UEXUzb0ses_4ctn0b3vgbpUJ7tCde91ul6rp-g7Hxw,480
-coocan/url/__init__.py,sha256=rEMx66XDy5AIJ9mF_2UVzHW5mRLBAWZEyQ3txrZzuZA,102
-coocan/url/request.py,sha256=seZaQXQRvRMIf9WnCp3mAgNA-kxsj9P2JzAvuIt2Dx8,1116
-coocan/url/response.py,sha256=AnC0xsF34q68r62EVlcHYmDH6skm9RBwRHITTb4iBbU,1785
-coocan-0.5.3.1.dist-info/METADATA,sha256=DrjX_0-M_NEvC8AJam9hh1mY1sXKhgkCoglgqjUk8B8,2556
-coocan-0.5.3.1.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
-coocan-0.5.3.1.dist-info/entry_points.txt,sha256=hNdk42NPboC1o7s7GzMbpII5t2U2jWrtT5bpvliXRcw,47
-coocan-0.5.3.1.dist-info/top_level.txt,sha256=VwB-Q4zEljgb9v1Ms1E59B-1pBYORXuhKjgZb-LHOhk,7
-coocan-0.5.3.1.dist-info/RECORD,,