cobweb-launcher 0.1.12__tar.gz → 0.1.14__tar.gz

{cobweb-launcher-0.1.12 → cobweb-launcher-0.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.12
+Version: 0.1.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.1.12 → cobweb-launcher-0.1.14}/cobweb/__init__.py

@@ -2,6 +2,7 @@ from .bbb import Seed, Queue, DBItem
 from .task import Task
 from .log import log
 from .db.redis_db import RedisDB
+from .db.oss_db import OssDB
 
 from .equip.distributed.launcher import launcher
 from .equip.single.launcher import launcher as single_launcher

{cobweb-launcher-0.1.12 → cobweb-launcher-0.1.14}/cobweb/constant.py

@@ -22,3 +22,4 @@ class Setting:
     CHECK_LOCK_TIME = None
     DEAL_MODEL = None
     LAUNCHER_MODEL = None
+    SPIDER_RUN_TIME = None

{cobweb-launcher-0.1.12 → cobweb-launcher-0.1.14}/cobweb/db/oss_db.py

@@ -13,8 +13,8 @@ class OssDB:
             endpoint,
             access_key,
             secret_key,
-            chunk_size,
-            min_size
+            chunk_size=1024 ** 2,
+            min_size=1024
     ):
         self.endpoint = endpoint
         self.bucket_name = bucket_name
@@ -27,8 +27,8 @@ class OssDB:
             endpoint=endpoint,
             bucket_name=bucket_name
         )
-        self.chunk_size = chunk_size or 1024 ** 2
-        self.min_size = min_size or 1024
+        self.chunk_size = chunk_size
+        self.min_size = min_size
 
     @staticmethod
     def format_upload_len(length):
@@ -104,14 +104,15 @@ class OssDB:
                     f", file size: {format_upload}"
                 )
 
-            else:
-                log.info(
-                    f"file size smaller than min size! "
-                    f"file size: {format_upload}"
-                )
+            # else:
+            #     log.info(
+            #         f"file size smaller than min size! "
+            #         f"file size: {format_upload}"
+            #     )
             status = True
         except ValueError as e:
-            log.exception(str(e))
+            pass
+            # log.exception(str(e))
         except oss2.exceptions.RequestError as e:
             self.bucket = oss2.Bucket(
                 auth=self.auth,

{cobweb-launcher-0.1.12 → cobweb-launcher-0.1.14}/cobweb/equip/__init__.py

@@ -1,4 +1,4 @@
-from .. import Queue, DBItem, RedisDB, Seed, log
+from .. import Queue, DBItem, RedisDB, Seed, log, OssDB
 from ..constant import Setting, DealModel
 from ..utils import (
     struct_queue_name as sqn,

{cobweb-launcher-0.1.12 → cobweb-launcher-0.1.14}/cobweb/equip/distributed/models.py

@@ -3,7 +3,7 @@ from hashlib import md5
 from inspect import isgenerator
 
 from .. import log, ici
-from .. import DealModel, Queue, Seed
+from .. import DealModel, Queue, Seed, Setting
 # from pympler import asizeof
 
 
@@ -109,6 +109,7 @@ class Spider:
                 log.info(f"{str(seed)} -> {str(e)}")
             finally:
                 self.spider_in_progress.pop()
+                time.sleep(Setting.SPIDER_RUN_TIME)
         log.info(f"close thread: spider")
 
 

{cobweb-launcher-0.1.12 → cobweb-launcher-0.1.14}/cobweb/equip/single/launcher.py

@@ -2,7 +2,7 @@ import time
 import threading
 
 from .. import log, sqn, rtn, pim
-from .. import Queue, DBItem, RedisDB, Setting
+from .. import Queue, DBItem, RedisDB, Setting, OssDB
 from .models import Scheduler, Spider, Storer
 
 
@@ -81,6 +81,8 @@ def launcher(task):
 
         # new item
         item = type("Item", (object,), {"redis_client": redis_db.client})()
+        if task.oss_config:
+            item.oss = OssDB(**task.oss_config)
 
         log.info("初始化cobweb!")
 

{cobweb-launcher-0.1.12 → cobweb-launcher-0.1.14}/cobweb/equip/single/models.py

@@ -2,7 +2,7 @@ import time
 from inspect import isgenerator
 # from pympler import asizeof
 from .. import log, ici
-from .. import DealModel, Queue, Seed
+from .. import DealModel, Queue, Seed, Setting
 
 
 class Scheduler:
@@ -105,6 +105,7 @@ class Spider:
                 log.info(f"{str(seed)} -> {str(e)}")
             finally:
                 self.spider_in_progress.pop()
+                time.sleep(Setting.SPIDER_RUN_TIME)
         log.info(f"close thread: spider")
 
 

{cobweb-launcher-0.1.12 → cobweb-launcher-0.1.14}/cobweb/task.py

@@ -6,6 +6,7 @@ from .utils import parse_info, struct_start_seeds
 def init_task_env():
     Setting.RESET_SCORE = int(os.getenv("RESET_SCORE", 600))
     Setting.CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
+    Setting.SPIDER_RUN_TIME = float(os.getenv("SPIDER_RUN_TIME", 0.2))
     Setting.DEAL_MODEL = os.getenv("DEAL_MODEL", DealModel.failure)
     Setting.LAUNCHER_MODEL = os.getenv("LAUNCHER_MODEL", LauncherModel.task)
 

{cobweb-launcher-0.1.12 → cobweb-launcher-0.1.14}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.12
+Version: 0.1.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.1.12 → cobweb-launcher-0.1.14}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="0.1.12",
+    version="0.1.14",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",