cobweb-launcher 1.2.2__py3-none-any.whl → 1.2.3__py3-none-any.whl

cobweb/launchers/launcher.py

@@ -40,7 +40,6 @@ class Launcher(threading.Thread):
         self.task = task
         self.project = project
 
-        self._app_time = int(time.time())
         self._stop = threading.Event()  # 结束事件
         self._pause = threading.Event()  # 暂停事件
 

cobweb/launchers/launcher_pro.py

@@ -16,6 +16,8 @@ class LauncherPro(Launcher):
         self._fail_key = "{%s:%s}:fail" % (project, task)
         self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
         self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+        self._statistics_done_key = "{%s:%s}:statistics:done" % (project, task)
+        self._statistics_fail_key = "{%s:%s}:statistics:fail" % (project, task)
         self._db = RedisDB()
 
         self._heartbeat_start_event = threading.Event()
@@ -25,6 +27,12 @@ class LauncherPro(Launcher):
     def heartbeat(self):
         return self._db.exists(self._heartbeat_key)
 
+    def statistics(self, key, count, expire_time):
+        if self._db.exists(key):
+            self._db._client.incrby(key, count)
+        else:
+            self._db.setex(key, expire_time, int(count))
+
     def _execute_heartbeat(self):
         while not self._stop.is_set():
             if self._heartbeat_start_event.is_set():
@@ -124,13 +132,16 @@ class LauncherPro(Launcher):
                 else:
                     seeds.append(seed.to_string)
             if seeds:
-                self._db.zrem(self._todo_key, *seeds)
+                count = self._db.zrem(self._todo_key, *seeds)
+                self.statistics(self._statistics_done_key, count, 86400 * 30)
                 self._remove_doing_seeds(seeds)
             if s_seeds:
-                self._db.done([self._todo_key, self._done_key], *s_seeds)
+                count = self._db.done([self._todo_key, self._done_key], *s_seeds)
+                self.statistics(self._statistics_done_key, count, 86400 * 30)
                 self._remove_doing_seeds(s_seeds)
             if f_seeds:
-                self._db.done([self._todo_key, self._fail_key], *f_seeds)
+                count = self._db.done([self._todo_key, self._fail_key], *f_seeds)
+                self.statistics(self._statistics_fail_key, count, 86400 * 30)
                 self._remove_doing_seeds(f_seeds)
 
             if status:

{cobweb_launcher-1.2.2.dist-info → cobweb_launcher-1.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.2
+Version: 1.2.3
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.2.dist-info → cobweb_launcher-1.2.3.dist-info}/RECORD

@@ -18,9 +18,9 @@ cobweb/db/redis_db.py,sha256=NNI2QkRV1hEZI-z-COEncXt88z3pZN6wusKlcQzc8V4,4304
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=af0Y6wrGX8SQZ7w7XL2sOtREjCT3dwad-uCc3nIontY,76
-cobweb/launchers/launcher.py,sha256=Mepg-hv9YL5_VnY_Mwqxzd8ZRGyK8gxsl_5B_Ibz0Uc,5466
+cobweb/launchers/launcher.py,sha256=W2CdS6-tYL0queyT51g__EoQfwsBxE0Z4wEQzuONe6Q,5424
 cobweb/launchers/launcher_air.py,sha256=zHVEJqQCxYU1WDnqQzzEHbEXasR1GmKevujQkCfFt5o,2947
-cobweb/launchers/launcher_pro.py,sha256=ihsEcdP8iOrJhHzQTYQow-8pIYK3CCn-iwwI3v9UzMw,7727
+cobweb/launchers/launcher_pro.py,sha256=uu_5S9rzgv0psDr8tYijuWcPQUSiVCcrzWtd6QDPiYY,8349
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
@@ -30,8 +30,8 @@ cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXt
 cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.2.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.2.dist-info/METADATA,sha256=nMfaL4ItGVnr9tHtxNnBEGud1HI_ZRbIGqmfL4TnV7o,6489
-cobweb_launcher-1.2.2.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.2.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.2.dist-info/RECORD,,
+cobweb_launcher-1.2.3.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.3.dist-info/METADATA,sha256=YzkV8mWy2OVcj36NSo3N_Im-r8dvSfu5-K2_B43VcuA,6489
+cobweb_launcher-1.2.3.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.3.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.3.dist-info/RECORD,,