coocan 0.5.6__tar.gz → 0.5.6.1__tar.gz

{coocan-0.5.6 → coocan-0.5.6.1}/PKG-INFO

@@ -1,8 +1,7 @@
 Metadata-Version: 2.4
 Name: coocan
-Version: 0.5.6
+Version: 0.5.6.1
 Summary: Air Async Spider Framework
-Author: wauo
 Author-email: wauo <markadc@126.com>
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/markadc/coocan
@@ -11,8 +10,6 @@ Description-Content-Type: text/markdown
 Requires-Dist: click>=8.0.0
 Requires-Dist: httpx
 Requires-Dist: loguru
-Dynamic: author
-Dynamic: requires-python
 
 # 项目说明
 

coocan-0.5.6.1/coocan/_examples/use_proxy.py

@@ -0,0 +1,22 @@
+from coocan import Request, Response, MiniSpider
+
+
+class UseProxySpider(MiniSpider):
+    start_urls = ["https://httpbin.org/ip"]
+    max_requests = 5
+    delay = 5
+
+    def start_requests(self):
+        proxy = "http://127.0.0.1:1082"
+        yield Request(self.start_urls[0], callback=self.parse, proxy=proxy)
+
+    def middleware(self, request: Request):
+        request.headers["Referer"] = "https://httpbin.org"
+
+    def parse(self, response: Response):
+        print(response.status_code, response.json())
+
+
+if __name__ == '__main__':
+    s = UseProxySpider()
+    s.go()

{coocan-0.5.6 → coocan-0.5.6.1}/coocan/push_project.py

@@ -1,7 +1,7 @@
 import os
 import sys
 
-msg = sys.argv[1] if len(sys.argv) == 2 else "update readme"
+msg = sys.argv[1] if len(sys.argv) == 2 else "支持代理；加入默认超时6秒；优化代码"
 
 cmd1 = "git add ."
 cmd2 = 'git commit -m "{}"'.format(msg)

coocan-0.5.6.1/coocan/url/request.py

@@ -0,0 +1,46 @@
+import time
+from typing import Callable
+
+import httpx
+
+cli = httpx.AsyncClient()
+
+
+class Request:
+    def __init__(
+            self,
+            url: str, callback: Callable = None, cb_kwargs=None,
+            params=None, headers=None,
+            data=None, json=None,
+            proxy=None, timeout=6,
+            priority=None
+    ):
+        self.url = url
+        self.callback = callback
+        self.cb_kwargs = cb_kwargs or {}
+        self.params = params
+        self.headers = headers or {}
+        self.data = data
+        self.json = json
+        self.proxy = proxy
+        self.timeout = timeout
+        self.priority = priority or time.time()
+        self.__client = None
+
+    @property
+    def client(self):
+        if self.proxy is None:
+            return cli
+        return httpx.AsyncClient(proxy=self.proxy)
+
+    async def send(self):
+        if (self.data and self.json) is None:
+            response = await self.client.get(self.url, params=self.params, headers=self.headers, timeout=self.timeout)
+        elif self.data or self.json:
+            response = await self.client.post(self.url, params=self.params, headers=self.headers, data=self.data, json=self.json, timeout=self.timeout)
+        else:
+            raise Exception("仅支持 GET 和 POST 请求")
+        return response
+
+    def __lt__(self, other):
+        return self.priority < other.priority

{coocan-0.5.6 → coocan-0.5.6.1}/coocan.egg-info/PKG-INFO

@@ -1,8 +1,7 @@
 Metadata-Version: 2.4
 Name: coocan
-Version: 0.5.6
+Version: 0.5.6.1
 Summary: Air Async Spider Framework
-Author: wauo
 Author-email: wauo <markadc@126.com>
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/markadc/coocan
@@ -11,8 +10,6 @@ Description-Content-Type: text/markdown
 Requires-Dist: click>=8.0.0
 Requires-Dist: httpx
 Requires-Dist: loguru
-Dynamic: author
-Dynamic: requires-python
 
 # 项目说明
 

{coocan-0.5.6 → coocan-0.5.6.1}/coocan.egg-info/SOURCES.txt

@@ -1,6 +1,5 @@
 README.md
 pyproject.toml
-setup.py
 coocan/__init__.py
 coocan/gen.py
 coocan/push_project.py
@@ -13,6 +12,7 @@ coocan.egg-info/top_level.txt
 coocan/_examples/crawl_csdn_detail.py
 coocan/_examples/crawl_csdn_list.py
 coocan/_examples/recv_item.py
+coocan/_examples/use_proxy.py
 coocan/_examples/view_local_ip.py
 coocan/cmd/__init__.py
 coocan/cmd/cli.py

{coocan-0.5.6 → coocan-0.5.6.1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "coocan"
 description = "Air Async Spider Framework"
-version = "0.5.6"
+version = "0.5.6.1"
 authors = [
     { name = "wauo", email = "markadc@126.com" }
 ]

coocan-0.5.6/coocan/url/request.py

@@ -1,31 +0,0 @@
-import time
-from typing import Callable
-
-import httpx
-
-cli = httpx.AsyncClient()
-
-
-class Request:
-    def __init__(self, url: str, callback: Callable = None, params=None, headers=None, data=None, json=None, timeout=None, cb_kwargs=None, priority=None):
-        self.url = url
-        self.callback = callback
-        self.params = params
-        self.headers = headers or {}
-        self.data = data
-        self.json = json
-        self.timeout = timeout
-        self.cb_kwargs = cb_kwargs or {}
-        self.priority = priority or time.time()
-
-    async def send(self):
-        if (self.data and self.json) is None:
-            response = await cli.get(self.url, params=self.params, headers=self.headers, timeout=self.timeout)
-        elif self.data or self.json:
-            response = await cli.post(self.url, params=self.params, headers=self.headers, data=self.data, json=self.json, timeout=self.timeout)
-        else:
-            raise Exception("仅支持 GET 和 POST 请求")
-        return response
-
-    def __lt__(self, other):
-        return self.priority < other.priority

coocan-0.5.6/setup.py

@@ -1,30 +0,0 @@
-from setuptools import setup, find_packages
-
-with open("README.md", "r", encoding="utf-8") as f:
-    long_description = f.read()
-
-setup(
-    name="coocan",
-    version="0.5.6",
-    author="wauo",
-    author_email="markadc@126.com",
-    description="Air Spider Framework",
-    packages=find_packages(),
-    python_requires=">=3.10",
-
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-
-    install_requires=[
-        'click>=8.0.0', 'httpx', 'loguru'
-    ],
-    entry_points={
-        'console_scripts': [
-            'coocan=coocan.cmd.cli:main',
-        ],
-    },
-    package_data={
-        'coocan': ['templates/*'],
-    },
-    include_package_data=True
-)