cobweb-launcher 0.0.17__py3-none-any.whl → 0.0.19__py3-none-any.whl

cobweb/db/oss_db.py

@@ -37,8 +37,8 @@ class OssDB:
 
         units = ["KB", "MB", "GB", "TB"]
         for i in range(3):
-            num = length / 1024 ** (i + 1)
-            if num >= 1:
+            num = length / (1024 ** (i + 1))
+            if num <= 1024:
                 return f"{round(num, 2)} {units[i]}"
 
     def assemble(self, ready_data, part_data):
@@ -110,6 +110,8 @@ class OssDB:
                     f"file size: {format_upload}"
                 )
             status = True
+        except ValueError as e:
+            log.exception(str(e))
         except oss2.exceptions.RequestError as e:
             self.bucket = oss2.Bucket(
                 auth=self.auth,

cobweb/db/redis_db.py

@@ -8,6 +8,7 @@ class RedisDB:
 
     def __init__(
             self,
+            model: int,
             project: str,
             task_name: str,
             config: dict
@@ -22,6 +23,7 @@ class RedisDB:
         self.check_lock = f"{project}:{task_name}:check_seed_lock"  # redis type string
         self.scheduler_lock = f"{project}:{task_name}:scheduler_lock"  # redis type string
         self.client = redis.Redis(connection_pool=pool)
+        self.model = model
     
     @check_redis_status
     def _get_lock(self, key, t=15, timeout=3, sleep_time=0.1):
@@ -86,8 +88,13 @@ class RedisDB:
         seeds = [seed if isinstance(seed, Seed) else Seed(seed) for seed in seeds]
 
         if seeds:
-            redis_key = self.succeed_key if spider_status else self.failed_key
-            self.client.sadd(redis_key, *(str(seed) for seed in seeds))
+            # redis_key = self.succeed_key if spider_status else self.failed_key
+            redis_key = None
+            if isinstance(self.model, int):
+                if self.model == 2 and spider_status:
+                    redis_key = self.succeed_key
+            if redis_key:
+                self.client.sadd(redis_key, *(str(seed) for seed in seeds))
             self.client.zrem(self.spider_key, *(seed.format_seed for seed in seeds))
 
     @check_redis_status

cobweb/distributed/launcher.py

@@ -6,29 +6,7 @@ from importlib import import_module
 from cobweb import log, Queue, DBItem, RedisDB, OssDB, StorerInterface
 from cobweb.utils import struct_queue_name, restore_table_name
 from .models import Scheduler, Spider, Storer
-# from collections import namedtuple
-
-# def start_seeds(seeds):
-#     if not seeds:
-#         return None
-#     if any(isinstance(seeds, t) for t in (list, tuple)):
-#         return [Seed(seed) for seed in seeds]
-#     elif any(isinstance(seeds, t) for t in (str, dict)):
-#         return Seed(seeds)
-
-
-# def parse_storer_info(storer_info):
-#     storer_data = {}
-#     storer_info_list = []
-#     if storer_info.__class__.__name__ == 'StorerInfo':
-#         storer_info_list.append(storer_info)
-#     elif any(isinstance(storer_info, t) for t in (list, tuple)):
-#         storer_info_list = storer_info
-#     for info in storer_info_list:
-#         db_name = info.DB.__name__
-#         storer_data.setdefault(db_name, {"StorerDB": info.DB, "db_args_list": []})
-#         storer_data[db_name]["db_args_list"].append(info[1:])
-#     return storer_data
+
 
 def get_scheduler_db(db):
     if isinstance(db, str):
@@ -87,7 +65,10 @@ def check(model, stop, last, spider, scheduler, storer_list, ready_seed_length,
         redis_spider_seed_length = spider_queue_length()
         memory_seed_queue_length = scheduler.queue.length
         storer_upload_queue_list = []
-
+        for storer in storer_list:
+            storer_upload_queue_list.append(
+                f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
+            )
         if (
                 scheduler.stop and
                 not redis_ready_seed_length and
@@ -99,6 +80,7 @@ def check(model, stop, last, spider, scheduler, storer_list, ready_seed_length,
             last.set()
             time.sleep(3)
             storer_queue_empty = True
+            storer_upload_queue_list = []
             for storer in storer_list:
                 if storer.queue.length:
                     storer_queue_empty = False
@@ -116,7 +98,8 @@ def check(model, stop, last, spider, scheduler, storer_list, ready_seed_length,
 
             last.clear()
 
-        storer_upload_queue_length_info = "\n            ".join(storer_upload_queue_list)
+        storer_upload_queue_length_info = "\n            ".join(
+            storer_upload_queue_list) if storer_upload_queue_list else "None"
         log.info(spider_info.format(
             status,
             running_spider_thread_num,

cobweb/distributed/models.py

@@ -60,7 +60,7 @@ class Spider:
                 del_seed(seed, spider_status=False)
                 continue
             try:
-                self.spider_in_progress.push(1)
+                self.spider_in_progress.push(1, direct_insertion=True)
                 # log.info("spider seed: " + str(seed))
                 status = None
                 store_queue = None
@@ -76,6 +76,8 @@ class Spider:
                         self.queue.push([s if isinstance(s, Seed) else Seed(s) for s in it])
                     elif isinstance(it, bool):
                         status = it
+                    elif it is None:
+                        status = False
 
                 if store_queue and store_data:
                     store_data.append(seed)

{cobweb_launcher-0.0.17.dist-info → cobweb_launcher-0.0.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.0.17
+Version: 0.0.19
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-0.0.17.dist-info → cobweb_launcher-0.0.19.dist-info}/RECORD

@@ -6,8 +6,8 @@ cobweb/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
 cobweb/task.py,sha256=0zPomzJOm374cpds0qHMZwX0y2qK_Gi_WnmOVcWPIEM,1401
 cobweb/utils.py,sha256=3iQRn2s4fY-5ClrVo46RO9OhziRAyOn5hGJOQGjMMwQ,2176
 cobweb/db/__init__.py,sha256=4m9lqmxZCRbaih3Z3rl_BT0GugMd0dkOIgu_P9aeC84,63
-cobweb/db/oss_db.py,sha256=pPpWPeOGMVz9X5aqD3c_9zMwTvs6dncE9JgBCmKc3v0,4150
-cobweb/db/redis_db.py,sha256=rpNvhEZJyHgjfmLeXid0sag4PW5LEreuGXolB_oYHX4,7677
+cobweb/db/oss_db.py,sha256=lFGNuH3tdIMsohVXQ_fTZPyBfS2oxYNmFNuQ-ZBQgm0,4221
+cobweb/db/redis_db.py,sha256=OvJMgGKn20h3NKiTzAkYci9yBFIidlyAJP1bme2FDgI,7936
 cobweb/db/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/db/scheduler/default.py,sha256=OxmFX7OvMEhKEq-NF7A8I9cA4V4qWw5vayS-yIbng0A,114
 cobweb/db/scheduler/textfile.py,sha256=atRDeNT-e5toNvyGsCXAxL1FJi77uSYktdCzH_hXGo8,821
@@ -17,13 +17,13 @@ cobweb/db/storer/loghub.py,sha256=4VqZacXWhidzINHXQu2_-E0HOBRCcc86f6LkKfnXD5I,17
 cobweb/db/storer/redis.py,sha256=7Q2XEQwBL6X_M1uvxzzuSBt6iw9piKw-_FWKm2INZDQ,412
 cobweb/db/storer/textfile.py,sha256=3mDHMvF6Sh5fn3IHzWQxyTUd45V-zUoH8vY3EoRlMx0,415
 cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/distributed/launcher.py,sha256=nD4uV8Bi1ymEgxFqRBxNb8Z5DJ_cnpG0riWq2RjzoBE,10688
-cobweb/distributed/models.py,sha256=eYD3yzOLGTI_LrILhzlYBBtwxU2x3KlETFGqVT7_66g,4414
+cobweb/distributed/launcher.py,sha256=CgsrAZX6m0BM91uw085fw5oecGi2gHgXjnNGejar1gM,10116
+cobweb/distributed/models.py,sha256=5rRy16BHBq4EwWZxHcH_hJ6GJ0jEZ9hechSlZxzvdFQ,4513
 cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/single/models.py,sha256=lu8teNWnWcUwZFra8XmqyhzOAf3UyuEztwBr1Ne6pUs,2898
 cobweb/single/nest.py,sha256=mL8q9a5BjtoeUyzXCIVw_vyUsNY8ltbvQpYIIpZEDFU,5012
-cobweb_launcher-0.0.17.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-0.0.17.dist-info/METADATA,sha256=N7hIcJCHqWpxfzlZDBb7CQzDC-PuUA1iaOqtfpme6oY,1226
-cobweb_launcher-0.0.17.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-cobweb_launcher-0.0.17.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-0.0.17.dist-info/RECORD,,
+cobweb_launcher-0.0.19.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.0.19.dist-info/METADATA,sha256=s81yOSvm3whrUbKiii7wwNvbbKY8oYvSNagLrQnU_9o,1226
+cobweb_launcher-0.0.19.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+cobweb_launcher-0.0.19.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.0.19.dist-info/RECORD,,