cobweb-launcher 1.2.44__py3-none-any.whl → 1.2.46__py3-none-any.whl

cobweb/crawlers/crawler.py

@@ -1,4 +1,5 @@
 import json
+import os
 import threading
 import time
 import traceback
@@ -6,9 +7,7 @@ from inspect import isgenerator
 from typing import Union, Callable, Mapping
 from urllib.parse import urlparse
 
-import urllib3
-from requests import HTTPError, Response as Res
-from requests.exceptions import ChunkedEncodingError
+from requests import Response as Res
 
 from cobweb.constant import DealModel, LogTemplate
 from cobweb.base import (
@@ -20,6 +19,7 @@ from cobweb.base import (
     logger
 )
 from cobweb.utils import LoghubDot
+proxy_type = os.getenv("PROXY_TYPE", "")
 
 
 class Crawler(threading.Thread):
@@ -149,7 +149,7 @@ class Crawler(threading.Thread):
                                         topic=urlparse(download_item.response.request.url).netloc,
                                         data_size=int(download_item.response.headers.get("content-length", 0)),
                                         cost_time=end_time - start_time, status = 200,
-                                        url=download_item.response.url,
+                                        url=download_item.response.url, proxy_type=proxy_type,
                                     )
                                 parse_iterators = self.parse(download_item)
                                 if not isgenerator(parse_iterators):
@@ -177,7 +177,8 @@ class Crawler(threading.Thread):
                     topic=urlparse(url).netloc,
                     data_size=-1, cost_time=-1,
                     status=status, url=url,
-                    msg=exception_msg
+                    proxy_type=proxy_type,
+                    msg=exception_msg,
                 )
                 logger.info(LogTemplate.download_exception.format(
                     detail=seed_detail_log_info,
@@ -208,6 +209,7 @@ class Crawler(threading.Thread):
         logger.info("spider thread close")
 
     def run(self):
+        threading.Thread(name="loghub_dot", target=self.loghub_dot.build_run).start()
         for index in range(self.thread_num):
             threading.Thread(name=f"spider_{index}", target=self.spider).start()
 

cobweb/utils/dotting.py

@@ -1,6 +1,9 @@
 import json
+import time
 
 from aliyun.log import LogClient, LogItem, PutLogsRequest
+
+from cobweb.base import Queue, logger
 from cobweb import setting
 
 
@@ -8,11 +11,11 @@ class LoghubDot:
 
     def __init__(self):
         self.client = LogClient(**setting.LOGHUB_CONFIG)
+        self.queue = Queue()
 
     def build(self, topic, **kwargs):
 
         temp = {}
-        log_items = []
         log_item = LogItem()
         for key, value in kwargs.items():
             if not isinstance(value, str):
@@ -21,12 +24,37 @@ class LoghubDot:
                 temp[key] = value
         contents = sorted(temp.items())
         log_item.set_contents(contents)
-        log_items.append(log_item)
-        request = PutLogsRequest(
-            project="databee-download-log",
-            logstore="log",
-            topic=topic,
-            logitems=log_items,
-            compress=True
-        )
-        self.client.put_logs(request=request)
+        # log_items.append(log_item)
+        # request = PutLogsRequest(
+        #     project="databee-download-log",
+        #     logstore="log",
+        #     topic=topic,
+        #     logitems=log_items,
+        #     compress=True
+        # )
+        self.queue.push((topic, log_item), direct_insertion=True)
+        # self.client.put_logs(request=request)
+
+    def build_run(self):
+        while True:
+            if self.queue.length < 1000:
+                time.sleep(0.5)
+                continue
+            try:
+                log_item_info = {}
+                for _ in range(1000):
+                    topic, item = self.queue.pop()
+                    if not item:
+                        break
+                    log_item_info.setdefault(topic, []).append(item)
+                for topic, log_items in log_item_info.items():
+                    request = PutLogsRequest(
+                        project="databee-download-log",
+                        logstore="log",
+                        topic=topic,
+                        logitems=log_items,
+                        compress=True
+                    )
+                    self.client.put_logs(request=request)
+            except Exception as e:
+                logger.info(str(e))

{cobweb_launcher-1.2.44.dist-info → cobweb_launcher-1.2.46.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.44
+Version: 1.2.46
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.44.dist-info → cobweb_launcher-1.2.46.dist-info}/RECORD

@@ -13,7 +13,7 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=sKu4gVxFTIkppYQ07nr9bEBbznOAEE18zzp6hbP5vJw,8551
+cobweb/crawlers/crawler.py,sha256=s5kImH3lzkyRm2AQoH1fwLaIO_CPZlPhqiBknPNaglM,8676
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
@@ -34,11 +34,11 @@ cobweb/schedulers/scheduler_api.py,sha256=pFEdS1H4zuzxwMhCV-G7CoLz-rEOPv4EVo3xZU
 cobweb/schedulers/scheduler_redis.py,sha256=E5fjc3nNld8GbUhUGT7uY4smRejj2J2ZIzp2g6lhxFM,2205
 cobweb/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
-cobweb/utils/dotting.py,sha256=PgsWdM-724Jy-MZWUsaygNWV-huqLMmdLgop7gaBxlo,872
+cobweb/utils/dotting.py,sha256=cTmhd8e8vSMqn3SGS8HcSa6cSourysaphJZYhWdpSoY,1905
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.44.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.44.dist-info/METADATA,sha256=4poc49J5hteo9dnC2tXYtPWSv_4yQzHM9Yim8S1Wc7U,6510
-cobweb_launcher-1.2.44.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.44.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.44.dist-info/RECORD,,
+cobweb_launcher-1.2.46.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.46.dist-info/METADATA,sha256=wQVxJaBKnlIwb-3Pi0RFjubW-5xLvoRLXdSFYnXQmzM,6510
+cobweb_launcher-1.2.46.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.46.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.46.dist-info/RECORD,,