cobweb-launcher 1.2.1__py3-none-any.whl → 1.2.3__py3-none-any.whl

cobweb/launchers/launcher.py

@@ -65,6 +65,7 @@ class Launcher(threading.Thread):
         self._Crawler = dynamic_load_class(setting.CRAWLER)
         self._Pipeline = dynamic_load_class(setting.PIPELINE)
 
+        self._before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
         self._scheduler_wait_seconds = setting.SCHEDULER_WAIT_SECONDS
         self._todo_queue_full_wait_seconds = setting.TODO_QUEUE_FULL_WAIT_SECONDS
         self._new_queue_wait_seconds = setting.NEW_QUEUE_WAIT_SECONDS
@@ -83,7 +84,6 @@ class Launcher(threading.Thread):
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL
 
-        # self._upload_queue = Queue()
 
     @property
     def start_seeds(self):
@@ -141,7 +141,7 @@ class Launcher(threading.Thread):
     def _execute(self):
         for func_name in self.__LAUNCHER_FUNC__:
             threading.Thread(name=func_name, target=getattr(self, func_name)).start()
-            time.sleep(2)
+            time.sleep(1)
 
     def run(self):
         threading.Thread(target=self._execute_heartbeat).start()

cobweb/launchers/launcher_pro.py

@@ -16,6 +16,8 @@ class LauncherPro(Launcher):
         self._fail_key = "{%s:%s}:fail" % (project, task)
         self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
         self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+        self._statistics_done_key = "{%s:%s}:statistics:done" % (project, task)
+        self._statistics_fail_key = "{%s:%s}:statistics:fail" % (project, task)
         self._db = RedisDB()
 
         self._heartbeat_start_event = threading.Event()
@@ -25,6 +27,12 @@ class LauncherPro(Launcher):
     def heartbeat(self):
         return self._db.exists(self._heartbeat_key)
 
+    def statistics(self, key, count, expire_time):
+        if self._db.exists(key):
+            self._db._client.incrby(key, count)
+        else:
+            self._db.setex(key, expire_time, int(count))
+
     def _execute_heartbeat(self):
         while not self._stop.is_set():
             if self._heartbeat_start_event.is_set():
@@ -124,19 +132,23 @@ class LauncherPro(Launcher):
                 else:
                     seeds.append(seed.to_string)
             if seeds:
-                self._db.zrem(self._todo_key, *seeds)
+                count = self._db.zrem(self._todo_key, *seeds)
+                self.statistics(self._statistics_done_key, count, 86400 * 30)
                 self._remove_doing_seeds(seeds)
             if s_seeds:
-                self._db.done([self._todo_key, self._done_key], *s_seeds)
+                count = self._db.done([self._todo_key, self._done_key], *s_seeds)
+                self.statistics(self._statistics_done_key, count, 86400 * 30)
                 self._remove_doing_seeds(s_seeds)
             if f_seeds:
-                self._db.done([self._todo_key, self._fail_key], *f_seeds)
+                count = self._db.done([self._todo_key, self._fail_key], *f_seeds)
+                self.statistics(self._statistics_fail_key, count, 86400 * 30)
                 self._remove_doing_seeds(f_seeds)
 
             if status:
                 time.sleep(self._done_queue_wait_seconds)
 
     def _polling(self):
+        wait_scheduler_execute = True
         check_emtpy_times = 0
         while not self._stop.is_set():
             queue_not_empty_count = 0
@@ -145,14 +157,19 @@ class LauncherPro(Launcher):
             for q in self.__LAUNCHER_QUEUE__.values():
                 if q.length != 0:
                     queue_not_empty_count += 1
+                    wait_scheduler_execute = False
 
             if queue_not_empty_count == 0:
                 pooling_wait_seconds = 3
                 if self._pause.is_set():
                     check_emtpy_times = 0
-                    if not self._task_model:
+                    if not self._task_model and (
+                            not wait_scheduler_execute or
+                            int(time.time()) - self._app_time > self._before_scheduler_wait_seconds
+                    ):
                         logger.info("Done! ready to close thread...")
                         self._stop.set()
+
                     elif self._db.zcount(self._todo_key, _min=0, _max="(1000"):
                         logger.info(f"Recovery {self.task} task run！")
                         self._pause.clear()
@@ -161,16 +178,14 @@ class LauncherPro(Launcher):
                         logger.info("pause! waiting for resume...")
                 elif check_emtpy_times > 2:
                     self.__DOING__ = {}
-                    self._pause.set()
+                    if not self._db.zcount(self._todo_key, _min="-inf", _max="(1000"):
+                        self._pause.set()
                 else:
                     logger.info(
                         "check whether the task is complete, "
                         f"reset times {3 - check_emtpy_times}"
                     )
                     check_emtpy_times += 1
-            # elif self._pause.is_set():
-            #     self._pause.clear()
-            #     self._execute()
             else:
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,
@@ -184,36 +199,6 @@ class LauncherPro(Launcher):
                 ))
 
             time.sleep(pooling_wait_seconds)
-            # if self._pause.is_set():
-            #     self._pause.clear()
-            #     self._execute()
-            #
-            # elif queue_not_empty_count == 0:
-            #     pooling_wait_seconds = 5
-            #     check_emtpy_times += 1
-            # else:
-            #     check_emtpy_times = 0
-            #
-            # if not self._db.zcount(self._todo, _min=0, _max="(1000") and check_emtpy_times > 2:
-            #     check_emtpy_times = 0
-            #     self.__DOING__ = {}
-            #     self._pause.set()
-            #
-            # time.sleep(pooling_wait_seconds)
-            #
-            # if not self._pause.is_set():
-            #     logger.info(LogTemplate.launcher_pro_polling.format(
-            #         task=self.task,
-            #         doing_len=len(self.__DOING__.keys()),
-            #         todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-            #         done_len=self.__LAUNCHER_QUEUE__['done'].length,
-            #         redis_seed_count=self._db.zcount(self._todo, "-inf", "+inf"),
-            #         redis_todo_len=self._db.zcount(self._todo, 0, "(1000"),
-            #         redis_doing_len=self._db.zcount(self._todo, "-inf", "(0"),
-            #         upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-            #     ))
-            # elif not self._task_model:
-            #     self._stop.set()
 
         logger.info("Done! Ready to close thread...")
 

cobweb/setting.py

@@ -26,6 +26,9 @@ OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
 OSS_CHUNK_SIZE = 10 * 1024 ** 2
 OSS_MIN_UPLOAD_SIZE = 1024
 
+# message
+MESSAGE = ""
+
 
 # 采集器选择
 CRAWLER = "cobweb.crawlers.Crawler"
@@ -35,6 +38,8 @@ PIPELINE = "cobweb.pipelines.pipeline_console.Console"
 
 
 # Launcher 等待时间
+
+BEFORE_SCHEDULER_WAIT_SECONDS = 60  # 调度前等待时间,只作用于单次任务
 SCHEDULER_WAIT_SECONDS = 15  # 调度等待时间
 TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
 NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间

{cobweb_launcher-1.2.1.dist-info → cobweb_launcher-1.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.1
+Version: 1.2.3
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.1.dist-info → cobweb_launcher-1.2.3.dist-info}/RECORD

@@ -1,6 +1,6 @@
 cobweb/__init__.py,sha256=uMHyf4Fekbyw2xBCbkA8R0LwCpBJf5p_7pWbh60ZWYk,83
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=zOO1cA_zQd4Q0CzY_tdSfdo-10L4QIVpm4382wbP5BQ,1906
+cobweb/setting.py,sha256=_t3LMSpxUNR4dVD9Tox22W9omHPvjeWABFzpgkEOoH0,2016
 cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
 cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
 cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
@@ -18,9 +18,9 @@ cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=af0Y6wrGX8SQZ7w7XL2sOtREjCT3dwad-uCc3nIontY,76
-cobweb/launchers/launcher.py,sha256=_oPoFNXp3Rrf5eK2tcHtwHZi2pKbuNPBf_VRvj9dJUc,5379
+cobweb/launchers/launcher.py,sha256=W2CdS6-tYL0queyT51g__EoQfwsBxE0Z4wEQzuONe6Q,5424
 cobweb/launchers/launcher_air.py,sha256=zHVEJqQCxYU1WDnqQzzEHbEXasR1GmKevujQkCfFt5o,2947
-cobweb/launchers/launcher_pro.py,sha256=046x07CKK26qc69hOAoDPNVkAXAiFwmR0hz2EsDh_tc,8859
+cobweb/launchers/launcher_pro.py,sha256=uu_5S9rzgv0psDr8tYijuWcPQUSiVCcrzWtd6QDPiYY,8349
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
@@ -30,8 +30,8 @@ cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXt
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.1.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.1.dist-info/METADATA,sha256=M-72J_GDC29Vz2KGFB9nZCqHlK8YCYhVioR_qwOnjrs,6489
-cobweb_launcher-1.2.1.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.1.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.1.dist-info/RECORD,,
+cobweb_launcher-1.2.3.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.3.dist-info/METADATA,sha256=YzkV8mWy2OVcj36NSo3N_Im-r8dvSfu5-K2_B43VcuA,6489
+cobweb_launcher-1.2.3.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.3.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.3.dist-info/RECORD,,