cobweb-launcher 0.0.18__tar.gz → 0.0.20__tar.gz

{cobweb-launcher-0.0.18 → cobweb-launcher-0.0.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.0.18
+Version: 0.0.20
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.0.18 → cobweb-launcher-0.0.20}/cobweb/db/redis_db.py

@@ -8,6 +8,7 @@ class RedisDB:
 
     def __init__(
             self,
+            model: int,
             project: str,
             task_name: str,
             config: dict
@@ -22,6 +23,7 @@ class RedisDB:
         self.check_lock = f"{project}:{task_name}:check_seed_lock"  # redis type string
         self.scheduler_lock = f"{project}:{task_name}:scheduler_lock"  # redis type string
         self.client = redis.Redis(connection_pool=pool)
+        self.model = model
     
     @check_redis_status
     def _get_lock(self, key, t=15, timeout=3, sleep_time=0.1):
@@ -86,8 +88,13 @@ class RedisDB:
         seeds = [seed if isinstance(seed, Seed) else Seed(seed) for seed in seeds]
 
         if seeds:
-            redis_key = self.succeed_key if spider_status else self.failed_key
-            self.client.sadd(redis_key, *(str(seed) for seed in seeds))
+            # redis_key = self.succeed_key if spider_status else self.failed_key
+            redis_key = None
+            if isinstance(self.model, int):
+                if self.model == 2 and spider_status:
+                    redis_key = self.succeed_key
+            if redis_key:
+                self.client.sadd(redis_key, *(str(seed) for seed in seeds))
             self.client.zrem(self.spider_key, *(seed.format_seed for seed in seeds))
 
     @check_redis_status

{cobweb-launcher-0.0.18 → cobweb-launcher-0.0.20}/cobweb/distributed/launcher.py

@@ -6,29 +6,7 @@ from importlib import import_module
 from cobweb import log, Queue, DBItem, RedisDB, OssDB, StorerInterface
 from cobweb.utils import struct_queue_name, restore_table_name
 from .models import Scheduler, Spider, Storer
-# from collections import namedtuple
-
-# def start_seeds(seeds):
-#     if not seeds:
-#         return None
-#     if any(isinstance(seeds, t) for t in (list, tuple)):
-#         return [Seed(seed) for seed in seeds]
-#     elif any(isinstance(seeds, t) for t in (str, dict)):
-#         return Seed(seeds)
-
-
-# def parse_storer_info(storer_info):
-#     storer_data = {}
-#     storer_info_list = []
-#     if storer_info.__class__.__name__ == 'StorerInfo':
-#         storer_info_list.append(storer_info)
-#     elif any(isinstance(storer_info, t) for t in (list, tuple)):
-#         storer_info_list = storer_info
-#     for info in storer_info_list:
-#         db_name = info.DB.__name__
-#         storer_data.setdefault(db_name, {"StorerDB": info.DB, "db_args_list": []})
-#         storer_data[db_name]["db_args_list"].append(info[1:])
-#     return storer_data
+
 
 def get_scheduler_db(db):
     if isinstance(db, str):
@@ -87,7 +65,10 @@ def check(model, stop, last, spider, scheduler, storer_list, ready_seed_length,
         redis_spider_seed_length = spider_queue_length()
         memory_seed_queue_length = scheduler.queue.length
         storer_upload_queue_list = []
-
+        for storer in storer_list:
+            storer_upload_queue_list.append(
+                f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
+            )
         if (
                 scheduler.stop and
                 not redis_ready_seed_length and
@@ -99,6 +80,7 @@ def check(model, stop, last, spider, scheduler, storer_list, ready_seed_length,
             last.set()
             time.sleep(3)
             storer_queue_empty = True
+            storer_upload_queue_list = []
             for storer in storer_list:
                 if storer.queue.length:
                     storer_queue_empty = False
@@ -116,7 +98,8 @@ def check(model, stop, last, spider, scheduler, storer_list, ready_seed_length,
 
             last.clear()
 
-        storer_upload_queue_length_info = "\n            ".join(storer_upload_queue_list)
+        storer_upload_queue_length_info = "\n            ".join(
+            storer_upload_queue_list) if storer_upload_queue_list else "None"
         log.info(spider_info.format(
             status,
             running_spider_thread_num,
@@ -158,7 +141,7 @@ def launcher(task):
         stop = threading.Event()
 
         # 初始化redis信息
-        redis_db = RedisDB(task.project, task.task_name, task.redis_info)
+        redis_db = RedisDB(task.model, task.project, task.task_name, task.redis_info)
 
         log.info("初始化cobweb!")
 

{cobweb-launcher-0.0.18 → cobweb-launcher-0.0.20}/cobweb/distributed/models.py

@@ -60,7 +60,7 @@ class Spider:
                 del_seed(seed, spider_status=False)
                 continue
             try:
-                self.spider_in_progress.push(1)
+                self.spider_in_progress.push(1, direct_insertion=True)
                 # log.info("spider seed: " + str(seed))
                 status = None
                 store_queue = None

{cobweb-launcher-0.0.18 → cobweb-launcher-0.0.20}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.0.18
+Version: 0.0.20
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.0.18 → cobweb-launcher-0.0.20}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="0.0.18",
+    version="0.0.20",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",