cobweb-launcher 0.0.15__tar.gz → 0.0.16__tar.gz

{cobweb-launcher-0.0.15 → cobweb-launcher-0.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.0.15
+Version: 0.0.16
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.0.15 → cobweb-launcher-0.0.16}/cobweb/distributed/launcher.py

@@ -68,10 +68,11 @@ def get_storer_db(db):
     raise TypeError()
 
 
-def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
+def check(model, stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
+    log.info("run check thread after 30 seconds...")
     time.sleep(30)
     spider_info = """
------------------------ check -----------------------
+------------------- check: %s ------------------
             running_spider_thread_num: {0}
             redis_ready_seed_length:   {1}
             redis_spider_seed_length:  {2}
@@ -80,20 +81,12 @@ def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_
                 {4}
 -----------------------  end  -----------------------"""
     while True:
-
+        status = "running"
         running_spider_thread_num = spider.spider_in_progress.length
         redis_ready_seed_length = ready_seed_length()
         redis_spider_seed_length = spider_queue_length()
         memory_seed_queue_length = scheduler.queue.length
         storer_upload_queue_list = []
-        storer_queue_empty = True
-        for storer in storer_list:
-            if storer.queue.length:
-                storer_queue_empty = False
-            storer_upload_queue_list.append(
-                f"{storer.__class__.__name__}: {storer.queue.length}"
-            )
-        storer_upload_queue_length_info = "\n            ".join(storer_upload_queue_list)
 
         if (
                 scheduler.stop and
@@ -101,14 +94,31 @@ def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_
                 not memory_seed_queue_length and
                 not running_spider_thread_num
         ):
-            log.info("spider is done?")
+            if not model:
+                log.info("spider is done?")
             last.set()
+            time.sleep(3)
+            storer_queue_empty = True
+            for storer in storer_list:
+                if storer.queue.length:
+                    storer_queue_empty = False
+                storer_upload_queue_list.append(
+                    f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
+                )
             if storer_queue_empty and not redis_spider_seed_length:
-                log.info("spider done!")
-                break
-        last.clear()
+                if model:
+                    log.info("waiting for push seeds...")
+                    status = "waiting"
+                    time.sleep(30)
+                else:
+                    log.info("spider done!")
+                    break
+
+            last.clear()
 
+        storer_upload_queue_length_info = "\n            ".join(storer_upload_queue_list)
         log.info(spider_info.format(
+            status,
             running_spider_thread_num,
             redis_ready_seed_length,
             redis_spider_seed_length,
@@ -182,7 +192,7 @@ def launcher(task):
             storer_info_list = [storer_info_list]
 
         # new item
-        item = type("Item", (object,), {"redis_client": redis_db})()
+        item = type("Item", (object,), {"redis_client": redis_db.client})()
 
         if task.oss_config:
             item.oss = OssDB(**task.oss_config)
@@ -289,7 +299,7 @@ def launcher(task):
             # name="check_spider",
             target=check,
             args=(
-                stop, last, spider,
+                task.model, stop, last, spider,
                 scheduler, storer_list,
                 redis_db.ready_seed_length,
                 redis_db.spider_queue_length,

{cobweb-launcher-0.0.15 → cobweb-launcher-0.0.16}/cobweb/task.py

@@ -5,6 +5,7 @@ class Task:
 
     def __init__(
             self,
+            model=None,
             seeds=None,
             project=None,
             task_name=None,
@@ -30,6 +31,8 @@ class Task:
         :param storer_queue_length:
         :param scheduler_queue_length:
         """
+        self.model = model
+
         self.seeds = struct_start_seeds(seeds)
         self.project = project or "test"
         self.task_name = task_name or "spider"

{cobweb-launcher-0.0.15 → cobweb-launcher-0.0.16}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.0.15
+Version: 0.0.16
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.0.15 → cobweb-launcher-0.0.16}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="0.0.15",
+    version="0.0.16",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",