cobweb-launcher 1.2.44__tar.gz → 1.2.46__tar.gz

{cobweb-launcher-1.2.44/cobweb_launcher.egg-info → cobweb-launcher-1.2.46}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.44
+Version: 1.2.46
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.44 → cobweb-launcher-1.2.46}/cobweb/crawlers/crawler.py

@@ -1,4 +1,5 @@
 import json
+import os
 import threading
 import time
 import traceback
@@ -6,9 +7,7 @@ from inspect import isgenerator
 from typing import Union, Callable, Mapping
 from urllib.parse import urlparse
 
-import urllib3
-from requests import HTTPError, Response as Res
-from requests.exceptions import ChunkedEncodingError
+from requests import Response as Res
 
 from cobweb.constant import DealModel, LogTemplate
 from cobweb.base import (
@@ -20,6 +19,7 @@ from cobweb.base import (
     logger
 )
 from cobweb.utils import LoghubDot
+proxy_type = os.getenv("PROXY_TYPE", "")
 
 
 class Crawler(threading.Thread):
@@ -149,7 +149,7 @@ class Crawler(threading.Thread):
                                         topic=urlparse(download_item.response.request.url).netloc,
                                         data_size=int(download_item.response.headers.get("content-length", 0)),
                                         cost_time=end_time - start_time, status = 200,
-                                        url=download_item.response.url,
+                                        url=download_item.response.url, proxy_type=proxy_type,
                                     )
                                 parse_iterators = self.parse(download_item)
                                 if not isgenerator(parse_iterators):
@@ -177,7 +177,8 @@ class Crawler(threading.Thread):
                     topic=urlparse(url).netloc,
                     data_size=-1, cost_time=-1,
                     status=status, url=url,
-                    msg=exception_msg
+                    proxy_type=proxy_type,
+                    msg=exception_msg,
                 )
                 logger.info(LogTemplate.download_exception.format(
                     detail=seed_detail_log_info,
@@ -208,6 +209,7 @@ class Crawler(threading.Thread):
         logger.info("spider thread close")
 
     def run(self):
+        threading.Thread(name="loghub_dot", target=self.loghub_dot.build_run).start()
         for index in range(self.thread_num):
             threading.Thread(name=f"spider_{index}", target=self.spider).start()
 

cobweb-launcher-1.2.46/cobweb/utils/dotting.py

@@ -0,0 +1,60 @@
+import json
+import time
+
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+
+from cobweb.base import Queue, logger
+from cobweb import setting
+
+
+class LoghubDot:
+
+    def __init__(self):
+        self.client = LogClient(**setting.LOGHUB_CONFIG)
+        self.queue = Queue()
+
+    def build(self, topic, **kwargs):
+
+        temp = {}
+        log_item = LogItem()
+        for key, value in kwargs.items():
+            if not isinstance(value, str):
+                temp[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                temp[key] = value
+        contents = sorted(temp.items())
+        log_item.set_contents(contents)
+        # log_items.append(log_item)
+        # request = PutLogsRequest(
+        #     project="databee-download-log",
+        #     logstore="log",
+        #     topic=topic,
+        #     logitems=log_items,
+        #     compress=True
+        # )
+        self.queue.push((topic, log_item), direct_insertion=True)
+        # self.client.put_logs(request=request)
+
+    def build_run(self):
+        while True:
+            if self.queue.length < 1000:
+                time.sleep(0.5)
+                continue
+            try:
+                log_item_info = {}
+                for _ in range(1000):
+                    topic, item = self.queue.pop()
+                    if not item:
+                        break
+                    log_item_info.setdefault(topic, []).append(item)
+                for topic, log_items in log_item_info.items():
+                    request = PutLogsRequest(
+                        project="databee-download-log",
+                        logstore="log",
+                        topic=topic,
+                        logitems=log_items,
+                        compress=True
+                    )
+                    self.client.put_logs(request=request)
+            except Exception as e:
+                logger.info(str(e))

{cobweb-launcher-1.2.44 → cobweb-launcher-1.2.46/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.44
+Version: 1.2.46
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.44 → cobweb-launcher-1.2.46}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.44",
+    version="1.2.46",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",

cobweb-launcher-1.2.44/cobweb/utils/dotting.py

@@ -1,32 +0,0 @@
-import json
-
-from aliyun.log import LogClient, LogItem, PutLogsRequest
-from cobweb import setting
-
-
-class LoghubDot:
-
-    def __init__(self):
-        self.client = LogClient(**setting.LOGHUB_CONFIG)
-
-    def build(self, topic, **kwargs):
-
-        temp = {}
-        log_items = []
-        log_item = LogItem()
-        for key, value in kwargs.items():
-            if not isinstance(value, str):
-                temp[key] = json.dumps(value, ensure_ascii=False)
-            else:
-                temp[key] = value
-        contents = sorted(temp.items())
-        log_item.set_contents(contents)
-        log_items.append(log_item)
-        request = PutLogsRequest(
-            project="databee-download-log",
-            logstore="log",
-            topic=topic,
-            logitems=log_items,
-            compress=True
-        )
-        self.client.put_logs(request=request)