cobweb-launcher 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

cobweb/__init__.py

@@ -2,6 +2,7 @@ from .bbb import Seed, Queue, DBItem
 from .task import Task
 from .log import log
 from .db.redis_db import RedisDB
+from .db.oss_db import OssDB
 
 from .equip.distributed.launcher import launcher
 from .equip.single.launcher import launcher as single_launcher

cobweb/constant.py

@@ -22,3 +22,4 @@ class Setting:
     CHECK_LOCK_TIME = None
     DEAL_MODEL = None
     LAUNCHER_MODEL = None
+    SPIDER_RUN_TIME = None

cobweb/db/oss_db.py

@@ -13,8 +13,8 @@ class OssDB:
             endpoint,
             access_key,
             secret_key,
-            chunk_size,
-            min_size
+            chunk_size=1024 ** 2,
+            min_size=1024
     ):
         self.endpoint = endpoint
         self.bucket_name = bucket_name
@@ -27,8 +27,8 @@ class OssDB:
             endpoint=endpoint,
             bucket_name=bucket_name
         )
-        self.chunk_size = chunk_size or 1024 ** 2
-        self.min_size = min_size or 1024
+        self.chunk_size = chunk_size
+        self.min_size = min_size
 
     @staticmethod
     def format_upload_len(length):
@@ -104,14 +104,15 @@ class OssDB:
                     f", file size: {format_upload}"
                 )
 
-            else:
-                log.info(
-                    f"file size smaller than min size! "
-                    f"file size: {format_upload}"
-                )
+            # else:
+            #     log.info(
+            #         f"file size smaller than min size! "
+            #         f"file size: {format_upload}"
+            #     )
             status = True
         except ValueError as e:
-            log.exception(str(e))
+            pass
+            # log.exception(str(e))
         except oss2.exceptions.RequestError as e:
             self.bucket = oss2.Bucket(
                 auth=self.auth,

cobweb/equip/__init__.py

@@ -1,4 +1,4 @@
-from .. import Queue, DBItem, RedisDB, Seed, log
+from .. import Queue, DBItem, RedisDB, Seed, log, OssDB
 from ..constant import Setting, DealModel
 from ..utils import (
     struct_queue_name as sqn,

cobweb/equip/distributed/models.py

@@ -3,7 +3,7 @@ from hashlib import md5
 from inspect import isgenerator
 
 from .. import log, ici
-from .. import DealModel, Queue, Seed
+from .. import DealModel, Queue, Seed, Setting
 # from pympler import asizeof
 
 
@@ -109,6 +109,7 @@ class Spider:
                 log.info(f"{str(seed)} -> {str(e)}")
             finally:
                 self.spider_in_progress.pop()
+                time.sleep(Setting.SPIDER_RUN_TIME)
         log.info(f"close thread: spider")
 
 

cobweb/equip/single/launcher.py

@@ -2,7 +2,7 @@ import time
 import threading
 
 from .. import log, sqn, rtn, pim
-from .. import Queue, DBItem, RedisDB, Setting
+from .. import Queue, DBItem, RedisDB, Setting, OssDB
 from .models import Scheduler, Spider, Storer
 
 
@@ -81,6 +81,8 @@ def launcher(task):
 
         # new item
         item = type("Item", (object,), {"redis_client": redis_db.client})()
+        if task.oss_config:
+            item.oss = OssDB(**task.oss_config)
 
         log.info("初始化cobweb!")
 

cobweb/equip/single/models.py

@@ -2,7 +2,7 @@ import time
 from inspect import isgenerator
 # from pympler import asizeof
 from .. import log, ici
-from .. import DealModel, Queue, Seed
+from .. import DealModel, Queue, Seed, Setting
 
 
 class Scheduler:
@@ -105,6 +105,7 @@ class Spider:
                 log.info(f"{str(seed)} -> {str(e)}")
             finally:
                 self.spider_in_progress.pop()
+                time.sleep(Setting.SPIDER_RUN_TIME)
         log.info(f"close thread: spider")
 
 

cobweb/task.py

@@ -6,6 +6,7 @@ from .utils import parse_info, struct_start_seeds
 def init_task_env():
     Setting.RESET_SCORE = int(os.getenv("RESET_SCORE", 600))
     Setting.CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
+    Setting.SPIDER_RUN_TIME = float(os.getenv("SPIDER_RUN_TIME", 0.2))
     Setting.DEAL_MODEL = os.getenv("DEAL_MODEL", DealModel.failure)
     Setting.LAUNCHER_MODEL = os.getenv("LAUNCHER_MODEL", LauncherModel.task)
 

{cobweb_launcher-0.1.12.dist-info → cobweb_launcher-0.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.12
+Version: 0.1.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-0.1.12.dist-info → cobweb_launcher-0.1.14.dist-info}/RECORD

@@ -1,14 +1,14 @@
-cobweb/__init__.py,sha256=fIg4v2yl3NHiCSli9EHU1WnMTLqEfDAvOgRtmULwu5A,227
+cobweb/__init__.py,sha256=rd5vkgxe49DzVH4oyvwl-pWQeJrqF1K6XT1NZfvMmTo,256
 cobweb/bbb.py,sha256=zKeCeBVFQfhEv6M8CCktUTM7tXDZmAu6ZN0-ET44pUY,5707
-cobweb/constant.py,sha256=WApaB3mn9cTDzvoCd7UcejO5T5wsCrxzn7D-uBTcIpg,462
+cobweb/constant.py,sha256=t0_2xY_VU6T18DnD8tL9LDQJB6ibE9UWx66H5yz_934,489
 cobweb/decorators.py,sha256=eYQI9rddPVJihAlomLTmbtQhIOzPw8dCrOFpxAq2pLY,318
 cobweb/interface.py,sha256=um_k2AAQl1HTOvfUlq914DjkpfZVwt2m1B65EpPKrmE,802
 cobweb/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
 cobweb/setting.py,sha256=UAu_dLuIFYO98MxtlZ5sZqJcwKAUKq4Bu4KoKlV50Mc,288
-cobweb/task.py,sha256=H9hDK72fSqD7k16exVIXIvCYr1Kvq-pl8PCXtKh8JWA,1704
+cobweb/task.py,sha256=okE6rBFm9VnYOAXShuTXJ0uvc-_ITsuuxhyxNKbymmU,1775
 cobweb/utils.py,sha256=ivmRqJJNtwdOKYT4G7qQCWnL8ar9c-shxeDZzGB2E9c,2651
 cobweb/db/__init__.py,sha256=jC-uOThYLtiDUG6cTJRkDITgOzR4nIOeaZQeZhuk-v0,139
-cobweb/db/oss_db.py,sha256=lFGNuH3tdIMsohVXQ_fTZPyBfS2oxYNmFNuQ-ZBQgm0,4221
+cobweb/db/oss_db.py,sha256=59HwMMfoGUr6l_BI4p84YHYoQwEn22cCM_1GZK34Uo4,4244
 cobweb/db/redis_db.py,sha256=go9IPQQZAl_jXiHDTruWF7N8svs9QEpQPOr7JhuLCQs,8179
 cobweb/db/scheduler/__init__.py,sha256=w5uIGEB1wLJ-H9RqGpzRwOEWW-BBVSk6Cc7FxZIlWCs,51
 cobweb/db/scheduler/default.py,sha256=XDtxNyu5KTpVAbfCOW8mR1zNFNHiMuaQ4sAhZuIYBoM,79
@@ -21,19 +21,19 @@ cobweb/db/storer/textfile.py,sha256=auoXGXLbIbEhMoeYIhy58qw22N2r0fQTtzVjHCjqVGA,
 cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/distributed/launcher.py,sha256=jTtBXBmna_6yFdj6gyGQiiEtg8I0g5uI5h8kbHWt454,7998
 cobweb/distributed/models.py,sha256=PUQokXMGD-H4A99nX7qYA395Ul6IsWGruMTVa05nswY,4568
-cobweb/equip/__init__.py,sha256=UVhm9xl9kj1Ez_9Sf5ElRvkeI5pTpXatWfIHJbXAFx4,240
+cobweb/equip/__init__.py,sha256=LWhbrTnG9kD1et0D40EzLISPuE0PIS-5WD3y3CLDaWk,247
 cobweb/equip/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/equip/distributed/launcher.py,sha256=1LzxibGXWR20XpXawakiRpEMaa9yfaj2rFSKnmEwjFc,7475
-cobweb/equip/distributed/models.py,sha256=5an5JRPvALFB4C-c6gY3KIttgeb4T66uLGwQr7fJJdw,4947
+cobweb/equip/distributed/models.py,sha256=qTGzxLdb2arsZSZK2HE4-MrqhraUhc2Ol5wBvlv_aWA,5008
 cobweb/equip/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/equip/single/launcher.py,sha256=dMd1IJg__yiFnxy3Q6EueebPllN8zyTDwfC0OjEhRf4,6702
-cobweb/equip/single/models.py,sha256=iZhsnf1OAq5L6pIn06iaaWlheS28M5Q2GgM55s58aX0,4579
+cobweb/equip/single/launcher.py,sha256=KRsw7yxklvFM85cel-EyLsNPLyrC9Hd26BMSx6-4Hac,6785
+cobweb/equip/single/models.py,sha256=RFhodfO9UDba2C4a3yHVrbzv-CSqfgbR2zAH8U-O7gk,4640
 cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/single/launcher.py,sha256=IoJbn87j7t7Pib_FxoWZmmX8asXOqNGb-9ospw6EYJI,7302
 cobweb/single/models.py,sha256=UXcxr_Quok91k82plaqbj4deB-UBCWo14WCo6SS5L_o,4247
 cobweb/single/nest.py,sha256=49K6KQ934INfPrWQsrq9rIFpQauLbLGOFbDaHvoQzOk,5015
-cobweb_launcher-0.1.12.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-0.1.12.dist-info/METADATA,sha256=-eWHCdhACz_STCzabluxlwByZ59OlcOKZnmYhjUlvB8,1220
-cobweb_launcher-0.1.12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-cobweb_launcher-0.1.12.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-0.1.12.dist-info/RECORD,,
+cobweb_launcher-0.1.14.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.1.14.dist-info/METADATA,sha256=cHaBgWqZpwy_aGAZF0jBOwYzbDlJUSMmj_mXb0ST7L0,1220
+cobweb_launcher-0.1.14.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+cobweb_launcher-0.1.14.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.1.14.dist-info/RECORD,,