cobweb-launcher 0.1.13__tar.gz → 0.1.14__tar.gz

{cobweb-launcher-0.1.13 → cobweb-launcher-0.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.13
+Version: 0.1.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.1.13 → cobweb-launcher-0.1.14}/cobweb/__init__.py

@@ -2,6 +2,7 @@ from .bbb import Seed, Queue, DBItem
 from .task import Task
 from .log import log
 from .db.redis_db import RedisDB
+from .db.oss_db import OssDB
 
 from .equip.distributed.launcher import launcher
 from .equip.single.launcher import launcher as single_launcher

{cobweb-launcher-0.1.13 → cobweb-launcher-0.1.14}/cobweb/db/oss_db.py

@@ -13,8 +13,8 @@ class OssDB:
             endpoint,
             access_key,
             secret_key,
-            chunk_size,
-            min_size
+            chunk_size=1024 ** 2,
+            min_size=1024
     ):
         self.endpoint = endpoint
         self.bucket_name = bucket_name
@@ -27,8 +27,8 @@ class OssDB:
             endpoint=endpoint,
             bucket_name=bucket_name
         )
-        self.chunk_size = chunk_size or 1024 ** 2
-        self.min_size = min_size or 1024
+        self.chunk_size = chunk_size
+        self.min_size = min_size
 
     @staticmethod
     def format_upload_len(length):
@@ -104,14 +104,15 @@ class OssDB:
                     f", file size: {format_upload}"
                 )
 
-            else:
-                log.info(
-                    f"file size smaller than min size! "
-                    f"file size: {format_upload}"
-                )
+            # else:
+            #     log.info(
+            #         f"file size smaller than min size! "
+            #         f"file size: {format_upload}"
+            #     )
             status = True
         except ValueError as e:
-            log.exception(str(e))
+            pass
+            # log.exception(str(e))
         except oss2.exceptions.RequestError as e:
             self.bucket = oss2.Bucket(
                 auth=self.auth,

{cobweb-launcher-0.1.13 → cobweb-launcher-0.1.14}/cobweb/equip/__init__.py

@@ -1,4 +1,4 @@
-from .. import Queue, DBItem, RedisDB, Seed, log
+from .. import Queue, DBItem, RedisDB, Seed, log, OssDB
 from ..constant import Setting, DealModel
 from ..utils import (
     struct_queue_name as sqn,

{cobweb-launcher-0.1.13 → cobweb-launcher-0.1.14}/cobweb/equip/single/launcher.py

@@ -2,7 +2,7 @@ import time
 import threading
 
 from .. import log, sqn, rtn, pim
-from .. import Queue, DBItem, RedisDB, Setting
+from .. import Queue, DBItem, RedisDB, Setting, OssDB
 from .models import Scheduler, Spider, Storer
 
 
@@ -81,6 +81,8 @@ def launcher(task):
 
         # new item
         item = type("Item", (object,), {"redis_client": redis_db.client})()
+        if task.oss_config:
+            item.oss = OssDB(**task.oss_config)
 
         log.info("初始化cobweb!")
 

{cobweb-launcher-0.1.13 → cobweb-launcher-0.1.14}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.13
+Version: 0.1.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.1.13 → cobweb-launcher-0.1.14}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="0.1.13",
+    version="0.1.14",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",