cobweb-launcher 1.2.52__py3-none-any.whl → 1.2.54__py3-none-any.whl

cobweb/db/redis_db.py

@@ -16,6 +16,14 @@ class RedisDB:
         return redis.StrictRedis(connection_pool=self.pool)
         # self._client = redis.Redis(connection_pool=pool)
 
+    def get(self, name):
+        with self.get_connection() as client:
+            return client.get(name)
+
+    def incrby(self, name, value):
+        with self.get_connection() as client:
+            client.incrby(name, value)
+
     def setnx(self, name, value=""):
         with self.get_connection() as client:
             client.setnx(name, value)

cobweb/launchers/launcher.py

@@ -51,6 +51,8 @@ class Launcher(threading.Thread):
         "_delete",
     ]
 
+    __WORKER_THREAD__ = {}
+
     def __init__(self, task, project, custom_setting=None, **kwargs):
         super().__init__()
         self.task = task
@@ -179,9 +181,25 @@ class Launcher(threading.Thread):
 
     def _execute(self):
         for func_name in self.__LAUNCHER_FUNC__:
-            threading.Thread(name=func_name, target=getattr(self, func_name)).start()
+            worker_thread = threading.Thread(name=func_name, target=getattr(self, func_name))
+            self.__WORKER_THREAD__[func_name] = worker_thread
+            worker_thread.start()
             time.sleep(1)
 
+    def _monitor(self):
+        while True:
+            if not self.__WORKER_THREAD__:
+                time.sleep(15)
+                continue
+            for func_name, worker_thread in self.__WORKER_THREAD__.items():
+                if not worker_thread.is_alive():
+                    logger.info(f"{func_name} thread is dead. Restarting...")
+                    target = getattr(self, func_name)
+                    worker_thread = threading.Thread(name=func_name, target=target)
+                    self.__WORKER_THREAD__[func_name] = worker_thread
+                    worker_thread.start()
+                time.sleep(5)
+
     def run(self):
         threading.Thread(target=self._execute_heartbeat).start()
 
@@ -210,6 +228,8 @@ class Launcher(threading.Thread):
             wait_seconds=self._upload_queue_wait_seconds
         ).start()
 
+        threading.Thread(target=self._monitor).start()
+
         self._execute()
         self._polling()
 

cobweb/launchers/launcher_pro.py

@@ -40,11 +40,11 @@ class LauncherPro(Launcher):
         if not self._task_model and not self._db.exists(key):
             self._db.setex(key, 86400 * 30, int(count))
         else:
-            self._db._client.incrby(key, count)
+            self._db.incrby(key, count)
 
     def _get_seed(self) -> Seed:
         if self._speed_control:
-            spider_speed = self._db._client.get(self._speed_control_key)
+            spider_speed = self._db.get(self._speed_control_key)
             if int(spider_speed or 0) > self._spider_max_count:
                 expire_time = self._db.ttl(self._speed_control_key)
                 if expire_time == -1:
@@ -55,7 +55,7 @@ class LauncherPro(Launcher):
                 return None
         seed = self.__LAUNCHER_QUEUE__["todo"].pop()
         if self._speed_control and seed and not self._db.lock(self._speed_control_key, t=self._time_window):
-            self._db._client.incrby(self._speed_control_key, 1)
+            self._db.incrby(self._speed_control_key, 1)
         return seed
 
     @check_pause

{cobweb_launcher-1.2.52.dist-info → cobweb_launcher-1.2.54.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.52
+Version: 1.2.54
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.52.dist-info → cobweb_launcher-1.2.54.dist-info}/RECORD

@@ -17,14 +17,14 @@ cobweb/crawlers/crawler.py,sha256=pePDGroD6JJAht5QTU51L7MkFIY4ob9TKpznZ2wUmsw,89
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
-cobweb/db/redis_db.py,sha256=mEWTBWSBXOhf3qxU9fdXYrhNJ44O8pS9tvqEAvZNhoI,5156
+cobweb/db/redis_db.py,sha256=DG-8l5DKRTs2ln1g9SM6gBMkmmjAcCM7RHL84eIuH60,5385
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
-cobweb/launchers/launcher.py,sha256=_qC3EnD_QxQRP_RBNHHJ8HLfn_0FED1jUF_lyas5Hss,7081
+cobweb/launchers/launcher.py,sha256=y1a72sANV-oJVpy_BI7BwKy7fY4AnpHqJK3t6czoCts,7896
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
 cobweb/launchers/launcher_api.py,sha256=7K7Kl3dk7Ung9iRBwhiMrALEJywcR66ie5RIkLQEM-Y,7909
-cobweb/launchers/launcher_pro.py,sha256=QMumHUTi6IudO5mw9SnwlmP8k_BJ50cpNjYjoQv-_YY,7936
+cobweb/launchers/launcher_pro.py,sha256=QCmOzlIDvdfdfZVK7rQN6PYou9Vedepie1aqqeHlI2w,7912
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -37,8 +37,8 @@ cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.52.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.52.dist-info/METADATA,sha256=lUniniJ1G9qDV9zA4XtI2p5bZPwDkMI6fKPSGnWjb_g,6510
-cobweb_launcher-1.2.52.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.52.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.52.dist-info/RECORD,,
+cobweb_launcher-1.2.54.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.54.dist-info/METADATA,sha256=n9jaFGNujOnCTREmaWA7yM66RKuj5VK1aAvQ_EJRqiU,6510
+cobweb_launcher-1.2.54.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.54.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.54.dist-info/RECORD,,