cobweb-launcher 1.2.51__tar.gz → 1.2.53__tar.gz

{cobweb-launcher-1.2.51/cobweb_launcher.egg-info → cobweb-launcher-1.2.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.51
+Version: 1.2.53
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.51 → cobweb-launcher-1.2.53}/cobweb/db/redis_db.py

@@ -6,60 +6,90 @@ class RedisDB:
 
     def __init__(self, **kwargs):
         redis_config = kwargs or setting.REDIS_CONFIG
-        pool = redis.ConnectionPool(**redis_config)
-        self._client = redis.Redis(connection_pool=pool)
+        # pool = redis.ConnectionPool(**redis_config)
+        self.pool = redis.ConnectionPool(
+            max_connections=25,
+            **redis_config
+        )
+        
+    def get_connection(self):
+        return redis.StrictRedis(connection_pool=self.pool)
+        # self._client = redis.Redis(connection_pool=pool)
+
+    def get(self, name):
+        with self.get_connection() as client:
+            return client.get(name)
+
+    def incrby(self, name, value):
+        with self.get_connection() as client:
+            client.incrby(name, value)
 
     def setnx(self, name, value=""):
-        return self._client.setnx(name, value)
+        with self.get_connection() as client:
+            client.setnx(name, value)
 
     def setex(self, name, t, value=""):
-        return self._client.setex(name, t, value)
+        with self.get_connection() as client:
+            client.setex(name, t, value)
 
     def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
-        return self._client.expire(name, t, nx, xx, gt, lt)
+        with self.get_connection() as client:
+            client.expire(name, t, nx, xx, gt, lt)
 
     def ttl(self, name):
-        return self._client.ttl(name)
+        with self.get_connection() as client:
+            return client.ttl(name)
 
     def delete(self, name):
-        return self._client.delete(name)
+        with self.get_connection() as client:
+            return client.delete(name)
 
     def exists(self, *name) -> bool:
-        return self._client.exists(*name)
+        with self.get_connection() as client:
+            return client.exists(*name)
 
     def sadd(self, name, value):
-        return self._client.sadd(name, value)
+        with self.get_connection() as client:
+            return client.sadd(name, value)
 
     def zcard(self, name) -> bool:
-        return self._client.zcard(name)
+        with self.get_connection() as client:
+            return client.zcard(name)
 
     def zadd(self, name, item: dict, **kwargs):
-        return self._client.zadd(name, item, **kwargs)
+        with self.get_connection() as client:
+            return client.zadd(name, item, **kwargs)
 
     def zrem(self, name, *value):
-        return self._client.zrem(name, *value)
+        with self.get_connection() as client:
+            return client.zrem(name, *value)
 
     def zcount(self, name, _min, _max):
-        return self._client.zcount(name, _min, _max)
+        with self.get_connection() as client:
+            return client.zcount(name, _min, _max)
 
     # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
-    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
+    #     with self.get_connection() as client:
+    #        return client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
 
     def lua(self, script: str, keys: list = None, args: list = None):
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        return self._client.eval(script, keys_count, *keys, *args)
+        with self.get_connection() as client:
+            return client.eval(script, keys_count, *keys, *args)
 
     def lua_sha(self, sha1: str, keys: list = None, args: list = None):
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        return self._client.evalsha(sha1, keys_count, *keys, *args)
+        with self.get_connection() as client:
+            return client.evalsha(sha1, keys_count, *keys, *args)
 
     def execute_lua(self, lua_script: str, keys: list, *args):
-        execute = self._client.register_script(lua_script)
-        return execute(keys=keys, args=args)
+        with self.get_connection() as client:
+            execute = client.register_script(lua_script)
+            return execute(keys=keys, args=args)
 
     def lock(self, key, t=15) -> bool:
         lua_script = """

{cobweb-launcher-1.2.51 → cobweb-launcher-1.2.53}/cobweb/launchers/launcher.py

@@ -51,6 +51,8 @@ class Launcher(threading.Thread):
         "_delete",
     ]
 
+    __WORKER_THREAD__ = {}
+
     def __init__(self, task, project, custom_setting=None, **kwargs):
         super().__init__()
         self.task = task
@@ -179,9 +181,22 @@ class Launcher(threading.Thread):
 
     def _execute(self):
         for func_name in self.__LAUNCHER_FUNC__:
-            threading.Thread(name=func_name, target=getattr(self, func_name)).start()
+            worker_thread = threading.Thread(name=func_name, target=getattr(self, func_name))
+            self.__WORKER_THREAD__[func_name] = worker_thread
+            worker_thread.start()
             time.sleep(1)
 
+    def _monitor(self):
+        while True:
+            for func_name, worker_thread in self.__WORKER_THREAD__.items():
+                if not worker_thread.is_alive():
+                    logger.info(f"{func_name} thread is dead. Restarting...")
+                    target = getattr(self, func_name)
+                    worker_thread = threading.Thread(name=func_name, target=target)
+                    self.__WORKER_THREAD__[func_name] = worker_thread
+                    worker_thread.start()
+                time.sleep(5)
+
     def run(self):
         threading.Thread(target=self._execute_heartbeat).start()
 
@@ -210,6 +225,8 @@ class Launcher(threading.Thread):
             wait_seconds=self._upload_queue_wait_seconds
         ).start()
 
+        threading.Thread(target=self._monitor).start()
+
         self._execute()
         self._polling()
 

{cobweb-launcher-1.2.51 → cobweb-launcher-1.2.53}/cobweb/launchers/launcher_pro.py

@@ -40,11 +40,11 @@ class LauncherPro(Launcher):
         if not self._task_model and not self._db.exists(key):
             self._db.setex(key, 86400 * 30, int(count))
         else:
-            self._db._client.incrby(key, count)
+            self._db.incrby(key, count)
 
     def _get_seed(self) -> Seed:
         if self._speed_control:
-            spider_speed = self._db._client.get(self._speed_control_key)
+            spider_speed = self._db.get(self._speed_control_key)
             if int(spider_speed or 0) > self._spider_max_count:
                 expire_time = self._db.ttl(self._speed_control_key)
                 if expire_time == -1:
@@ -55,7 +55,7 @@ class LauncherPro(Launcher):
                 return None
         seed = self.__LAUNCHER_QUEUE__["todo"].pop()
         if self._speed_control and seed and not self._db.lock(self._speed_control_key, t=self._time_window):
-            self._db._client.incrby(self._speed_control_key, 1)
+            self._db.incrby(self._speed_control_key, 1)
         return seed
 
     @check_pause

{cobweb-launcher-1.2.51 → cobweb-launcher-1.2.53/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.51
+Version: 1.2.53
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.51 → cobweb-launcher-1.2.53}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.51",
+    version="1.2.53",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",