cobweb-launcher 0.0.15__py3-none-any.whl → 0.0.16__py3-none-any.whl

cobweb/distributed/launcher.py

@@ -68,10 +68,11 @@ def get_storer_db(db):
     raise TypeError()
 
 
-def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
+def check(model, stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
+    log.info("run check thread after 30 seconds...")
     time.sleep(30)
     spider_info = """
------------------------ check -----------------------
+------------------- check: %s ------------------
             running_spider_thread_num: {0}
             redis_ready_seed_length:   {1}
             redis_spider_seed_length:  {2}
@@ -80,20 +81,12 @@ def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_
                 {4}
 -----------------------  end  -----------------------"""
     while True:
-
+        status = "running"
         running_spider_thread_num = spider.spider_in_progress.length
         redis_ready_seed_length = ready_seed_length()
         redis_spider_seed_length = spider_queue_length()
         memory_seed_queue_length = scheduler.queue.length
         storer_upload_queue_list = []
-        storer_queue_empty = True
-        for storer in storer_list:
-            if storer.queue.length:
-                storer_queue_empty = False
-            storer_upload_queue_list.append(
-                f"{storer.__class__.__name__}: {storer.queue.length}"
-            )
-        storer_upload_queue_length_info = "\n            ".join(storer_upload_queue_list)
 
         if (
                 scheduler.stop and
@@ -101,14 +94,31 @@ def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_
                 not memory_seed_queue_length and
                 not running_spider_thread_num
         ):
-            log.info("spider is done?")
+            if not model:
+                log.info("spider is done?")
             last.set()
+            time.sleep(3)
+            storer_queue_empty = True
+            for storer in storer_list:
+                if storer.queue.length:
+                    storer_queue_empty = False
+                storer_upload_queue_list.append(
+                    f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
+                )
             if storer_queue_empty and not redis_spider_seed_length:
-                log.info("spider done!")
-                break
-        last.clear()
+                if model:
+                    log.info("waiting for push seeds...")
+                    status = "waiting"
+                    time.sleep(30)
+                else:
+                    log.info("spider done!")
+                    break
+
+            last.clear()
 
+        storer_upload_queue_length_info = "\n            ".join(storer_upload_queue_list)
         log.info(spider_info.format(
+            status,
             running_spider_thread_num,
             redis_ready_seed_length,
             redis_spider_seed_length,
@@ -182,7 +192,7 @@ def launcher(task):
             storer_info_list = [storer_info_list]
 
         # new item
-        item = type("Item", (object,), {"redis_client": redis_db})()
+        item = type("Item", (object,), {"redis_client": redis_db.client})()
 
         if task.oss_config:
             item.oss = OssDB(**task.oss_config)
@@ -289,7 +299,7 @@ def launcher(task):
             # name="check_spider",
             target=check,
             args=(
-                stop, last, spider,
+                task.model, stop, last, spider,
                 scheduler, storer_list,
                 redis_db.ready_seed_length,
                 redis_db.spider_queue_length,

cobweb/task.py

@@ -5,6 +5,7 @@ class Task:
 
     def __init__(
             self,
+            model=None,
             seeds=None,
             project=None,
             task_name=None,
@@ -30,6 +31,8 @@ class Task:
         :param storer_queue_length:
         :param scheduler_queue_length:
         """
+        self.model = model
+
         self.seeds = struct_start_seeds(seeds)
         self.project = project or "test"
         self.task_name = task_name or "spider"

{cobweb_launcher-0.0.15.dist-info → cobweb_launcher-0.0.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.0.15
+Version: 0.0.16
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-0.0.15.dist-info → cobweb_launcher-0.0.16.dist-info}/RECORD

@@ -3,7 +3,7 @@ cobweb/bbb.py,sha256=Sckof5zgzWEz2tIjs6xxoLkrL7wUdogPw3AetlXIDUo,5684
 cobweb/decorators.py,sha256=8KPSKL8xsiXOLv-kckkaDtK8LXM8d5gaRriGpuEgOQk,320
 cobweb/interface.py,sha256=um_k2AAQl1HTOvfUlq914DjkpfZVwt2m1B65EpPKrmE,802
 cobweb/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
-cobweb/task.py,sha256=jfLnhvxIR1RcPVHZk7fMZBOHMRD7HfHPFnkVDpfmlkg,1349
+cobweb/task.py,sha256=0zPomzJOm374cpds0qHMZwX0y2qK_Gi_WnmOVcWPIEM,1401
 cobweb/utils.py,sha256=3iQRn2s4fY-5ClrVo46RO9OhziRAyOn5hGJOQGjMMwQ,2176
 cobweb/db/__init__.py,sha256=4m9lqmxZCRbaih3Z3rl_BT0GugMd0dkOIgu_P9aeC84,63
 cobweb/db/oss_db.py,sha256=pPpWPeOGMVz9X5aqD3c_9zMwTvs6dncE9JgBCmKc3v0,4150
@@ -17,13 +17,13 @@ cobweb/db/storer/loghub.py,sha256=4VqZacXWhidzINHXQu2_-E0HOBRCcc86f6LkKfnXD5I,17
 cobweb/db/storer/redis.py,sha256=7Q2XEQwBL6X_M1uvxzzuSBt6iw9piKw-_FWKm2INZDQ,412
 cobweb/db/storer/textfile.py,sha256=3mDHMvF6Sh5fn3IHzWQxyTUd45V-zUoH8vY3EoRlMx0,415
 cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/distributed/launcher.py,sha256=dVXyhF_QWxGME49CzumJNj7LeQQGaJnp0YkNGiO1ZoY,10270
+cobweb/distributed/launcher.py,sha256=32iFDWo59bzUok2jGuGvCe03J5PdwsF2wGpJLXBYFSw,10687
 cobweb/distributed/models.py,sha256=wnqumVPEX6ENVNcvqo3WvALyb_pXVd1BdT0QS0xpzbQ,4412
 cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/single/models.py,sha256=lu8teNWnWcUwZFra8XmqyhzOAf3UyuEztwBr1Ne6pUs,2898
 cobweb/single/nest.py,sha256=mL8q9a5BjtoeUyzXCIVw_vyUsNY8ltbvQpYIIpZEDFU,5012
-cobweb_launcher-0.0.15.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-0.0.15.dist-info/METADATA,sha256=NSilMj-McfY0n0P0SsHpLF-Dx4tcDZspiLEp_YAR5tY,1226
-cobweb_launcher-0.0.15.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-cobweb_launcher-0.0.15.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-0.0.15.dist-info/RECORD,,
+cobweb_launcher-0.0.16.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.0.16.dist-info/METADATA,sha256=ad7tP97rdRX3pVo6zNm6JXPSrCL1399uLnuZRGeTdvI,1226
+cobweb_launcher-0.0.16.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+cobweb_launcher-0.0.16.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.0.16.dist-info/RECORD,,