cobweb-launcher 1.2.57__tar.gz → 1.2.59__tar.gz

{cobweb-launcher-1.2.57/cobweb_launcher.egg-info → cobweb-launcher-1.2.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.57
+Version: 1.2.59
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

cobweb-launcher-1.2.59/cobweb/db/redis_db.py

@@ -0,0 +1,215 @@
+import redis
+import time
+from cobweb import setting
+from redis.exceptions import ConnectionError, TimeoutError
+
+
+class RedisDB:
+    def __init__(self, **kwargs):
+        redis_config = kwargs or setting.REDIS_CONFIG
+        self.host = redis_config['host']
+        self.password = redis_config['password']
+        self.port = redis_config['port']
+        self.db = redis_config['db']
+
+        self.max_retries = 5
+        self.retry_delay = 5
+        self.client = None
+        self.connect()
+
+    def connect(self):
+        """尝试连接 Redis"""
+        retries = 0
+        while retries < self.max_retries:
+            try:
+                self.client = redis.Redis(
+                    host=self.host,
+                    port=self.port,
+                    password=self.password,
+                    db=self.db,
+                    socket_timeout=5,  # 设置连接超时时间
+                    socket_connect_timeout=5  # 设置连接超时时间
+                )
+                # 测试连接是否成功
+                self.client.ping()
+                return
+            except (ConnectionError, TimeoutError) as e:
+                retries += 1
+                if retries < self.max_retries:
+                    time.sleep(self.retry_delay)
+                else:
+                    raise Exception("达到最大重试次数，无法连接 Redis")
+
+    def is_connected(self):
+        try:
+            self.client.ping()
+            return True
+        except (ConnectionError, TimeoutError):
+            return False
+
+    def reconnect(self):
+        self.connect()
+
+    def execute_command(self, command, *args, **kwargs):
+        retries = 0
+        while retries < self.max_retries:
+            try:
+                if not self.is_connected():
+                    self.reconnect()
+                return getattr(self.client, command)(*args, **kwargs)
+            except (ConnectionError, TimeoutError) as e:
+                retries += 1
+                if retries < self.max_retries:
+                    time.sleep(self.retry_delay)
+                else:
+                    raise Exception("达到最大重试次数，无法执行命令")
+
+    def get(self, name):
+        # with self.get_connection() as client:
+        #     return client.get(name)
+        return self.execute_command("get", name)
+
+    def incrby(self, name, value):
+        # with self.get_connection() as client:
+        #     client.incrby(name, value)
+        self.execute_command("incrby", name, value)
+
+    def setnx(self, name, value=""):
+        # with self.get_connection() as client:
+        #     client.setnx(name, value)
+        self.execute_command("setnx", name, value)
+
+    def setex(self, name, t, value=""):
+        # with self.get_connection() as client:
+        #     client.setex(name, t, value)
+        self.execute_command("setex", name, t, value)
+
+    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
+        # with self.get_connection() as client:
+        # client.expire(name, t, nx, xx, gt, lt)
+        self.execute_command("expire", name, t, nx, xx, gt, lt)
+
+    def ttl(self, name):
+        # with self.get_connection() as client:
+        #     return client.ttl(name)
+        return self.execute_command("ttl", name)
+
+    def delete(self, name):
+        # with self.get_connection() as client:
+        #     return client.delete(name)
+        return self.execute_command("delete", name)
+
+    def exists(self, *name) -> bool:
+        # with self.get_connection() as client:
+        #     return client.exists(*name)
+        return self.execute_command("exists", *name)
+
+    def sadd(self, name, value):
+        # with self.get_connection() as client:
+        #     return client.sadd(name, value)
+        return self.execute_command("sadd", name, value)
+
+    def zcard(self, name) -> bool:
+        # with self.get_connection() as client:
+        #     return client.zcard(name)
+        return self.execute_command("zcard", name)
+
+    def zadd(self, name, item: dict, **kwargs):
+        # with self.get_connection() as client:
+        #     return client.zadd(name, item, **kwargs)
+        return self.execute_command("zadd", name, item, **kwargs)
+
+    def zrem(self, name, *value):
+        # with self.get_connection() as client:
+        #     return client.zrem(name, *value)
+        return self.execute_command("zrem", name, *value)
+
+    def zcount(self, name, _min, _max):
+        # with self.get_connection() as client:
+        #     return client.zcount(name, _min, _max)
+        return self.execute_command("zcount", name, _min, _max)
+
+    # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
+    #     with self.get_connection() as client:
+    #        return client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
+
+    def lua(self, script: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self.execute_command("eval", script, keys_count, *keys, *args)
+
+    def lua_sha(self, sha1: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self.execute_command("evalsha", sha1, keys_count, *keys, *args)
+
+    def execute_lua(self, lua_script: str, keys: list, *args):
+        execute = self.execute_command("register_script", lua_script)
+        return execute(keys=keys, args=args)
+
+    def lock(self, key, t=15) -> bool:
+        lua_script = """
+        local status = redis.call('setnx', KEYS[1], 1)
+        if ( status == 1 ) then
+            redis.call('expire', KEYS[1], ARGV[1])
+        end 
+        return status 
+        """
+        status = self.execute_lua(lua_script, [key], t)
+        return bool(status)
+
+    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
+        lua_script = """
+        local min = ARGV[1]
+        local max = ARGV[2]
+        local start = ARGV[3]
+        local count = ARGV[4]
+        local score = ARGV[5]
+        local members = nil
+
+        if ( type(count) == string ) then
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
+        else
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
+        end
+
+        local result = {}
+
+        for i = 1, #members, 2 do
+            local priority = nil
+            local member = members[i]
+            local originPriority = nil
+            if ( members[i+1] + 0 < 0 ) then
+                originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
+            else
+                originPriority = math.floor(members[i+1])
+            end
+
+            if ( score + 0 >= 1000 ) then
+                priority = -score - originPriority / 1000
+            elseif ( score + 0 == 0 ) then
+                priority = originPriority
+            else
+                originPriority = score 
+                priority = score
+            end
+            redis.call('zadd', KEYS[1], priority, member)
+            table.insert(result, member)
+            table.insert(result, originPriority)
+        end
+
+        return result
+        """
+        members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
+        return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
+
+    def done(self, keys: list, *args) -> list:
+        lua_script = """
+        for i, member in ipairs(ARGV) do
+            redis.call("zrem", KEYS[1], member)
+            redis.call("sadd", KEYS[2], member)
+        end
+        """
+        self.execute_lua(lua_script, keys, *args)

cobweb-launcher-1.2.57/cobweb/db/redis_db.py → cobweb-launcher-1.2.59/cobweb/db/redis_db_new.py

@@ -1,3 +1,21 @@
+
+
+
+# 示例用法
+# if __name__ == "__main__":
+#     redis_client = RedisClient(
+#         host="r-j6c1t3etiefpmz7cwdpd.redis.rds.aliyuncs.com", port=6379,
+#         password="SpiderLinux666", db=0
+#     )
+#
+#     # 执行 Redis 命令
+#     try:
+#         ss = redis_client.get("host_speed_control:bepls.com")
+#         print(f"获取的值: {ss}")
+#     except Exception as e:
+#         print(f"操作失败: {e}")
+
+
 import redis
 from cobweb import setting
 
@@ -6,86 +24,60 @@ class RedisDB:
 
     def __init__(self, **kwargs):
         redis_config = kwargs or setting.REDIS_CONFIG
-        self.pool = redis.ConnectionPool(**redis_config)
-        
-    def get_connection(self):
-        return redis.StrictRedis(connection_pool=self.pool)
-        # self._client = redis.Redis(connection_pool=pool)
-
-    def get(self, name):
-        with self.get_connection() as client:
-            return client.get(name)
-
-    def incrby(self, name, value):
-        with self.get_connection() as client:
-            client.incrby(name, value)
+        pool = redis.ConnectionPool(**redis_config)
+        self._client = redis.Redis(connection_pool=pool)
 
     def setnx(self, name, value=""):
-        with self.get_connection() as client:
-            client.setnx(name, value)
+        return self._client.setnx(name, value)
 
     def setex(self, name, t, value=""):
-        with self.get_connection() as client:
-            client.setex(name, t, value)
+        return self._client.setex(name, t, value)
 
     def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
-        with self.get_connection() as client:
-            client.expire(name, t, nx, xx, gt, lt)
+        return self._client.expire(name, t, nx, xx, gt, lt)
 
     def ttl(self, name):
-        with self.get_connection() as client:
-            return client.ttl(name)
+        return self._client.ttl(name)
 
     def delete(self, name):
-        with self.get_connection() as client:
-            return client.delete(name)
+        return self._client.delete(name)
 
     def exists(self, *name) -> bool:
-        with self.get_connection() as client:
-            return client.exists(*name)
+        return self._client.exists(*name)
 
     def sadd(self, name, value):
-        with self.get_connection() as client:
-            return client.sadd(name, value)
+        return self._client.sadd(name, value)
 
     def zcard(self, name) -> bool:
-        with self.get_connection() as client:
-            return client.zcard(name)
+        return self._client.zcard(name)
 
     def zadd(self, name, item: dict, **kwargs):
-        with self.get_connection() as client:
-            return client.zadd(name, item, **kwargs)
+        return self._client.zadd(name, item, **kwargs)
 
     def zrem(self, name, *value):
-        with self.get_connection() as client:
-            return client.zrem(name, *value)
+        return self._client.zrem(name, *value)
 
     def zcount(self, name, _min, _max):
-        with self.get_connection() as client:
-            return client.zcount(name, _min, _max)
+        return self._client.zcount(name, _min, _max)
 
     # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
-    #     with self.get_connection() as client:
-    #        return client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
+    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
 
     def lua(self, script: str, keys: list = None, args: list = None):
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        with self.get_connection() as client:
-            return client.eval(script, keys_count, *keys, *args)
+        return self._client.eval(script, keys_count, *keys, *args)
 
     def lua_sha(self, sha1: str, keys: list = None, args: list = None):
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        with self.get_connection() as client:
-            return client.evalsha(sha1, keys_count, *keys, *args)
+        return self._client.evalsha(sha1, keys_count, *keys, *args)
 
     def execute_lua(self, lua_script: str, keys: list, *args):
-        with self.get_connection() as client:
-            execute = client.register_script(lua_script)
-            return execute(keys=keys, args=args)
+        execute = self._client.register_script(lua_script)
+        return execute(keys=keys, args=args)
 
     def lock(self, key, t=15) -> bool:
         lua_script = """
@@ -112,7 +104,7 @@ class RedisDB:
         else
             members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
         end
-        
+
         local result = {}
 
         for i = 1, #members, 2 do
@@ -124,7 +116,7 @@ class RedisDB:
             else
                 originPriority = math.floor(members[i+1])
             end
-            
+
             if ( score + 0 >= 1000 ) then
                 priority = -score - originPriority / 1000
             elseif ( score + 0 == 0 ) then
@@ -143,10 +135,6 @@ class RedisDB:
         members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
         return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
 
-    # def get_member(self):
-    #     with self.get_connection() as client:
-    #         pipeline = client.pipeline()
-
     def done(self, keys: list, *args) -> list:
         lua_script = """
         for i, member in ipairs(ARGV) do

{cobweb-launcher-1.2.57 → cobweb-launcher-1.2.59}/cobweb/launchers/launcher_pro.py

@@ -1,6 +1,7 @@
 import time
 import threading
 
+
 from cobweb.db import RedisDB
 from cobweb.base import Seed, logger
 from cobweb.constant import LogTemplate
@@ -23,9 +24,9 @@ class LauncherPro(Launcher):
         self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
 
         # self._bf_key = "bloom_%s_%s" % (project, task)
-        #
+
         self._db = RedisDB()
-        #
+
         # self._bf = BloomFilter(self._bf_key)
 
         self._heartbeat_start_event = threading.Event()
@@ -46,9 +47,9 @@ class LauncherPro(Launcher):
             spider_speed = self._db.get(self._speed_control_key)
             if int(spider_speed or 0) > self._spider_max_count:
                 expire_time = self._db.ttl(self._speed_control_key)
-                if expire_time == -1:
+                if expire_time <= -1:
                     self._db.delete(self._speed_control_key)
-                else:
+                elif isinstance(expire_time, int):
                     logger.info(f"Too fast! Please wait {expire_time} seconds...")
                     time.sleep(expire_time / 2)
                 return None

{cobweb-launcher-1.2.57 → cobweb-launcher-1.2.59/cobweb_launcher.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.57
+Version: 1.2.59
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.2.57 → cobweb-launcher-1.2.59}/cobweb_launcher.egg-info/SOURCES.txt

@@ -19,6 +19,7 @@ cobweb/crawlers/file_crawler.py
 cobweb/db/__init__.py
 cobweb/db/api_db.py
 cobweb/db/redis_db.py
+cobweb/db/redis_db_new.py
 cobweb/exceptions/__init__.py
 cobweb/exceptions/oss_db_exception.py
 cobweb/launchers/__init__.py

{cobweb-launcher-1.2.57 → cobweb-launcher-1.2.59}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.2.57",
+    version="1.2.59",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",