cobweb-launcher 1.3.7__tar.gz → 1.3.9__tar.gz

{cobweb-launcher-1.3.7/cobweb_launcher.egg-info → cobweb-launcher-1.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.7
+Version: 1.3.9
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.7 → cobweb-launcher-1.3.9}/cobweb/base/__init__.py

@@ -1,3 +1,4 @@
+import os
 import time
 import traceback
 import threading
@@ -12,7 +13,7 @@ from .basic import Seed, Request, Response
 from .item import BaseItem, ConsoleItem
 # from .seed import Seed
 from .log import logger
-# from .dotting import LoghubDot
+from .dotting import LoghubDot
 
 
 class TaskQueue:
@@ -25,8 +26,7 @@ class TaskQueue:
     DONE = Queue()          # 下载完成队列
     UPLOAD = Queue()        # 任务上传队列
     DELETE = Queue()        # 任务删除队列
-
-    # DOT = LoghubDot()
+    DOT = LoghubDot()
 
     @staticmethod
     def is_empty():
@@ -43,6 +43,7 @@ class TaskQueue:
     @staticmethod
     def process_task(it: Union[Seed, Request, Response, BaseItem], crawler_func: Callable):
         try:
+            start_time = time.time()
             iterators = crawler_func(it)
             if not isgenerator(iterators):
                 raise TypeError(f"{crawler_func.__name__} function isn't a generator")
@@ -57,12 +58,11 @@ class TaskQueue:
                     TaskQueue.SEED.push(tk)
                 else:
                     raise TypeError(f"{crawler_func.__name__} function return type isn't supported")
-                # TaskQueue.DOT.build(
-                #     topic=f"{self.project}:{self.task}",
-                #     cost_time=end_time - start_time,
-                #     **download_item.to_dict
-                # )
-                # todo: 数据打点
+                TaskQueue.DOT.build(
+                    topic=f"{os.getenv('PROJECT')}:{os.getenv('TASK')}",
+                    cost_time=round(time.time() - start_time, 2),
+                    **tk.to_dict
+                )
         except Exception as e:
             it.params.retry += 1
             if isinstance(it, Request):

{cobweb-launcher-1.3.7 → cobweb-launcher-1.3.9}/cobweb/base/dotting.py

@@ -27,7 +27,7 @@ class LoghubDot:
             log_items.append(log_item)
             request = PutLogsRequest(
                 project="databee-download-log",
-                logstore="cobweb_log",
+                logstore="download-logging",
                 topic=topic,
                 logitems=log_items,
                 compress=True

{cobweb-launcher-1.3.7 → cobweb-launcher-1.3.9}/cobweb/schedulers/scheduler_api.py

@@ -59,10 +59,12 @@ class ApiScheduler:
     def refresh(self, key, items: dict[str, int]):
         refresh_time = int(time.time())
         its = {k: -refresh_time - v / 1000 for k, v in items}
-        self._db.zadd(key, item=its, xx=True)
+        if its:
+            self._db.zadd(key, item=its, xx=True)
 
     def delete(self, key, values):
-        self._db.zrem(key, *values)
+        if values:
+            self._db.zrem(key, *values)
 
 
 

{cobweb-launcher-1.3.7 → cobweb-launcher-1.3.9}/cobweb/schedulers/scheduler_redis.py

@@ -59,10 +59,12 @@ class RedisScheduler:
     def refresh(self, key, items: dict[str, int]):
         refresh_time = int(time.time())
         its = {k: -refresh_time - v / 1000 for k, v in items}
-        self._db.zadd(key, item=its, xx=True)
+        if its:
+            self._db.zadd(key, item=its, xx=True)
 
     def delete(self, key, values):
-        self._db.zrem(key, *values)
+        if values:
+            self._db.zrem(key, *values)
 
 
 

{cobweb-launcher-1.3.7 → cobweb-launcher-1.3.9}/cobweb/utils/oss.py

@@ -1,9 +1,9 @@
-from typing import List
+# from typing import List
 from cobweb import setting
 from requests import Response
 from oss2 import Auth, Bucket, models, PartIterator
 from cobweb.exceptions import oss_db_exception
-from cobweb.base.decorators import decorator_oss_db
+from cobweb.base import Decorators
 
 
 class OssUtil:
@@ -40,17 +40,17 @@ class OssUtil:
     def head(self, key: str) -> models.HeadObjectResult:
         return self._client.head_object(key)
 
-    @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
+    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
     def init_part(self, key) -> models.InitMultipartUploadResult:
         """初始化分片上传"""
         return self._client.init_multipart_upload(key)
 
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
+    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
     def put(self, key, data) -> models.PutObjectResult:
         """文件上传"""
         return self._client.put_object(key, data)
 
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
+    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
     def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
         """分片上传"""
         return self._client.upload_part(key, upload_id, position, data)
@@ -59,13 +59,13 @@ class OssUtil:
         """获取分片列表"""
         return [part_info for part_info in PartIterator(self._client, key, upload_id)]
 
-    @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
+    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
     def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
         """合并分片"""
         headers = None if parts else {"x-oss-complete-all": "yes"}
         return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
 
-    @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
+    @Decorators.decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
     def append(self, key, position, data) -> models.AppendObjectResult:
         """追加上传"""
         return self._client.append_object(key, position, data)

{cobweb-launcher-1.3.7 → cobweb-launcher-1.3.9/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.7
+Version: 1.3.9
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.7 → cobweb-launcher-1.3.9}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.3.7",
+    version="1.3.9",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",