cobweb-launcher 1.2.57__py3-none-any.whl → 1.2.58__py3-none-any.whl

cobweb/db/redis_db.py

@@ -1,68 +1,133 @@
 import redis
+import time
 from cobweb import setting
+from redis.exceptions import ConnectionError, TimeoutError
 
 
-class RedisDB:
-
+class RedisClient:
     def __init__(self, **kwargs):
         redis_config = kwargs or setting.REDIS_CONFIG
-        self.pool = redis.ConnectionPool(**redis_config)
-        
-    def get_connection(self):
-        return redis.StrictRedis(connection_pool=self.pool)
-        # self._client = redis.Redis(connection_pool=pool)
+        self.host = redis_config['host']
+        self.password = redis_config['password']
+        self.port = redis_config['port']
+        self.db = redis_config['db']
+
+        self.max_retries = 5
+        self.retry_delay = 5
+        self.client = None
+        self.connect()
+
+    def connect(self):
+        """尝试连接 Redis"""
+        retries = 0
+        while retries < self.max_retries:
+            try:
+                self.client = redis.Redis(
+                    host=self.host,
+                    port=self.port,
+                    password=self.password,
+                    db=self.db,
+                    socket_timeout=5,  # 设置连接超时时间
+                    socket_connect_timeout=5  # 设置连接超时时间
+                )
+                # 测试连接是否成功
+                self.client.ping()
+                return
+            except (ConnectionError, TimeoutError) as e:
+                retries += 1
+                if retries < self.max_retries:
+                    time.sleep(self.retry_delay)
+                else:
+                    raise Exception("达到最大重试次数，无法连接 Redis")
+
+    def is_connected(self):
+        try:
+            self.client.ping()
+            return True
+        except (ConnectionError, TimeoutError):
+            return False
+
+    def reconnect(self):
+        self.connect()
+
+    def execute_command(self, command, *args, **kwargs):
+        retries = 0
+        while retries < self.max_retries:
+            try:
+                if not self.is_connected():
+                    self.reconnect()
+                return getattr(self.client, command)(*args, **kwargs)
+            except (ConnectionError, TimeoutError) as e:
+                retries += 1
+                if retries < self.max_retries:
+                    time.sleep(self.retry_delay)
+                else:
+                    raise Exception("达到最大重试次数，无法执行命令")
 
     def get(self, name):
-        with self.get_connection() as client:
-            return client.get(name)
+        # with self.get_connection() as client:
+        #     return client.get(name)
+        return self.execute_command("get", name)
 
     def incrby(self, name, value):
-        with self.get_connection() as client:
-            client.incrby(name, value)
+        # with self.get_connection() as client:
+        #     client.incrby(name, value)
+        self.execute_command("incrby", name, value)
 
     def setnx(self, name, value=""):
-        with self.get_connection() as client:
-            client.setnx(name, value)
+        # with self.get_connection() as client:
+        #     client.setnx(name, value)
+        self.execute_command("setnx", name, value)
 
     def setex(self, name, t, value=""):
-        with self.get_connection() as client:
-            client.setex(name, t, value)
+        # with self.get_connection() as client:
+        #     client.setex(name, t, value)
+        self.execute_command("setex", name, t, value)
 
     def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
-        with self.get_connection() as client:
-            client.expire(name, t, nx, xx, gt, lt)
+        # with self.get_connection() as client:
+        # client.expire(name, t, nx, xx, gt, lt)
+        self.execute_command("expire", name, t, nx, xx, gt, lt)
 
     def ttl(self, name):
-        with self.get_connection() as client:
-            return client.ttl(name)
+        # with self.get_connection() as client:
+        #     return client.ttl(name)
+        return self.execute_command("ttl", name)
 
     def delete(self, name):
-        with self.get_connection() as client:
-            return client.delete(name)
+        # with self.get_connection() as client:
+        #     return client.delete(name)
+        return self.execute_command("delete", name)
 
     def exists(self, *name) -> bool:
-        with self.get_connection() as client:
-            return client.exists(*name)
+        # with self.get_connection() as client:
+        #     return client.exists(*name)
+        return self.execute_command("exists", *name)
 
     def sadd(self, name, value):
-        with self.get_connection() as client:
-            return client.sadd(name, value)
+        # with self.get_connection() as client:
+        #     return client.sadd(name, value)
+        return self.execute_command("sadd", name, value)
 
     def zcard(self, name) -> bool:
-        with self.get_connection() as client:
-            return client.zcard(name)
+        # with self.get_connection() as client:
+        #     return client.zcard(name)
+        return self.execute_command("zcard", name)
 
     def zadd(self, name, item: dict, **kwargs):
-        with self.get_connection() as client:
-            return client.zadd(name, item, **kwargs)
+        # with self.get_connection() as client:
+        #     return client.zadd(name, item, **kwargs)
+        return self.execute_command("zadd", name, item, **kwargs)
 
     def zrem(self, name, *value):
-        with self.get_connection() as client:
-            return client.zrem(name, *value)
+        # with self.get_connection() as client:
+        #     return client.zrem(name, *value)
+        return self.execute_command("zrem", name, *value)
 
     def zcount(self, name, _min, _max):
-        with self.get_connection() as client:
-            return client.zcount(name, _min, _max)
+        # with self.get_connection() as client:
+        #     return client.zcount(name, _min, _max)
+        return self.execute_command("zcount", name, _min, _max)
 
     # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
     #     with self.get_connection() as client:
@@ -72,89 +137,14 @@ class RedisDB:
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        with self.get_connection() as client:
-            return client.eval(script, keys_count, *keys, *args)
+        return self.execute_command("eval", script, keys_count, *keys, *args)
 
     def lua_sha(self, sha1: str, keys: list = None, args: list = None):
         keys = keys or []
         args = args or []
         keys_count = len(keys)
-        with self.get_connection() as client:
-            return client.evalsha(sha1, keys_count, *keys, *args)
+        return self.execute_command("evalsha", sha1, keys_count, *keys, *args)
 
     def execute_lua(self, lua_script: str, keys: list, *args):
-        with self.get_connection() as client:
-            execute = client.register_script(lua_script)
-            return execute(keys=keys, args=args)
-
-    def lock(self, key, t=15) -> bool:
-        lua_script = """
-        local status = redis.call('setnx', KEYS[1], 1)
-        if ( status == 1 ) then
-            redis.call('expire', KEYS[1], ARGV[1])
-        end 
-        return status 
-        """
-        status = self.execute_lua(lua_script, [key], t)
-        return bool(status)
-
-    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
-        lua_script = """
-        local min = ARGV[1]
-        local max = ARGV[2]
-        local start = ARGV[3]
-        local count = ARGV[4]
-        local score = ARGV[5]
-        local members = nil
-
-        if ( type(count) == string ) then
-            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
-        else
-            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
-        end
-        
-        local result = {}
-
-        for i = 1, #members, 2 do
-            local priority = nil
-            local member = members[i]
-            local originPriority = nil
-            if ( members[i+1] + 0 < 0 ) then
-                originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
-            else
-                originPriority = math.floor(members[i+1])
-            end
-            
-            if ( score + 0 >= 1000 ) then
-                priority = -score - originPriority / 1000
-            elseif ( score + 0 == 0 ) then
-                priority = originPriority
-            else
-                originPriority = score 
-                priority = score
-            end
-            redis.call('zadd', KEYS[1], priority, member)
-            table.insert(result, member)
-            table.insert(result, originPriority)
-        end
-
-        return result
-        """
-        members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
-        return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
-
-    # def get_member(self):
-    #     with self.get_connection() as client:
-    #         pipeline = client.pipeline()
-
-    def done(self, keys: list, *args) -> list:
-        lua_script = """
-        for i, member in ipairs(ARGV) do
-            redis.call("zrem", KEYS[1], member)
-            redis.call("sadd", KEYS[2], member)
-        end
-        """
-        self.execute_lua(lua_script, keys, *args)
-
-
-
+        execute = self.execute_command("register_script", lua_script)
+        return execute(keys=keys, args=args)

cobweb/db/redis_db_new.py

@@ -0,0 +1,148 @@
+
+
+
+# 示例用法
+# if __name__ == "__main__":
+#     redis_client = RedisClient(
+#         host="r-j6c1t3etiefpmz7cwdpd.redis.rds.aliyuncs.com", port=6379,
+#         password="SpiderLinux666", db=0
+#     )
+#
+#     # 执行 Redis 命令
+#     try:
+#         ss = redis_client.get("host_speed_control:bepls.com")
+#         print(f"获取的值: {ss}")
+#     except Exception as e:
+#         print(f"操作失败: {e}")
+
+
+import redis
+from cobweb import setting
+
+
+class RedisDB:
+
+    def __init__(self, **kwargs):
+        redis_config = kwargs or setting.REDIS_CONFIG
+        pool = redis.ConnectionPool(**redis_config)
+        self._client = redis.Redis(connection_pool=pool)
+
+    def setnx(self, name, value=""):
+        return self._client.setnx(name, value)
+
+    def setex(self, name, t, value=""):
+        return self._client.setex(name, t, value)
+
+    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
+        return self._client.expire(name, t, nx, xx, gt, lt)
+
+    def ttl(self, name):
+        return self._client.ttl(name)
+
+    def delete(self, name):
+        return self._client.delete(name)
+
+    def exists(self, *name) -> bool:
+        return self._client.exists(*name)
+
+    def sadd(self, name, value):
+        return self._client.sadd(name, value)
+
+    def zcard(self, name) -> bool:
+        return self._client.zcard(name)
+
+    def zadd(self, name, item: dict, **kwargs):
+        return self._client.zadd(name, item, **kwargs)
+
+    def zrem(self, name, *value):
+        return self._client.zrem(name, *value)
+
+    def zcount(self, name, _min, _max):
+        return self._client.zcount(name, _min, _max)
+
+    # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
+    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
+
+    def lua(self, script: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self._client.eval(script, keys_count, *keys, *args)
+
+    def lua_sha(self, sha1: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self._client.evalsha(sha1, keys_count, *keys, *args)
+
+    def execute_lua(self, lua_script: str, keys: list, *args):
+        execute = self._client.register_script(lua_script)
+        return execute(keys=keys, args=args)
+
+    def lock(self, key, t=15) -> bool:
+        lua_script = """
+        local status = redis.call('setnx', KEYS[1], 1)
+        if ( status == 1 ) then
+            redis.call('expire', KEYS[1], ARGV[1])
+        end 
+        return status 
+        """
+        status = self.execute_lua(lua_script, [key], t)
+        return bool(status)
+
+    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
+        lua_script = """
+        local min = ARGV[1]
+        local max = ARGV[2]
+        local start = ARGV[3]
+        local count = ARGV[4]
+        local score = ARGV[5]
+        local members = nil
+
+        if ( type(count) == string ) then
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
+        else
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
+        end
+
+        local result = {}
+
+        for i = 1, #members, 2 do
+            local priority = nil
+            local member = members[i]
+            local originPriority = nil
+            if ( members[i+1] + 0 < 0 ) then
+                originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
+            else
+                originPriority = math.floor(members[i+1])
+            end
+
+            if ( score + 0 >= 1000 ) then
+                priority = -score - originPriority / 1000
+            elseif ( score + 0 == 0 ) then
+                priority = originPriority
+            else
+                originPriority = score 
+                priority = score
+            end
+            redis.call('zadd', KEYS[1], priority, member)
+            table.insert(result, member)
+            table.insert(result, originPriority)
+        end
+
+        return result
+        """
+        members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
+        return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
+
+    def done(self, keys: list, *args) -> list:
+        lua_script = """
+        for i, member in ipairs(ARGV) do
+            redis.call("zrem", KEYS[1], member)
+            redis.call("sadd", KEYS[2], member)
+        end
+        """
+        self.execute_lua(lua_script, keys, *args)
+
+
+

cobweb/launchers/launcher_pro.py

@@ -1,6 +1,8 @@
 import time
 import threading
 
+from h5py.h5pl import insert
+
 from cobweb.db import RedisDB
 from cobweb.base import Seed, logger
 from cobweb.constant import LogTemplate
@@ -23,9 +25,9 @@ class LauncherPro(Launcher):
         self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
 
         # self._bf_key = "bloom_%s_%s" % (project, task)
-        #
+
         self._db = RedisDB()
-        #
+
         # self._bf = BloomFilter(self._bf_key)
 
         self._heartbeat_start_event = threading.Event()
@@ -46,9 +48,9 @@ class LauncherPro(Launcher):
             spider_speed = self._db.get(self._speed_control_key)
             if int(spider_speed or 0) > self._spider_max_count:
                 expire_time = self._db.ttl(self._speed_control_key)
-                if expire_time == -1:
+                if expire_time <= -1:
                     self._db.delete(self._speed_control_key)
-                else:
+                elif isinstance(expire_time, int):
                     logger.info(f"Too fast! Please wait {expire_time} seconds...")
                     time.sleep(expire_time / 2)
                 return None

{cobweb_launcher-1.2.57.dist-info → cobweb_launcher-1.2.58.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.57
+Version: 1.2.58
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.2.57.dist-info → cobweb_launcher-1.2.58.dist-info}/RECORD

@@ -17,14 +17,15 @@ cobweb/crawlers/crawler.py,sha256=pePDGroD6JJAht5QTU51L7MkFIY4ob9TKpznZ2wUmsw,89
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
-cobweb/db/redis_db.py,sha256=ODV_p00TOQb7hvzR1fxFFOFryPL_VxkezdCcFgVKb0s,5397
+cobweb/db/redis_db.py,sha256=7lnEbPjfyvWuuP2f9hZAfKxpK31eJCoDAB5wfNkds34,5495
+cobweb/db/redis_db_new.py,sha256=F09LWVjtC2JFdCaKatZ2bAOLKbsnes85_nZRe2dtSIc,4697
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
 cobweb/launchers/launcher.py,sha256=y1a72sANV-oJVpy_BI7BwKy7fY4AnpHqJK3t6czoCts,7896
 cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
 cobweb/launchers/launcher_api.py,sha256=7K7Kl3dk7Ung9iRBwhiMrALEJywcR66ie5RIkLQEM-Y,7909
-cobweb/launchers/launcher_pro.py,sha256=_KK0KlrzDaQZOdTQJoPZcBsoAiWF13CSbvoepahXIpU,8054
+cobweb/launchers/launcher_pro.py,sha256=EuTsa6ErqpqytdnWhQxHFhO6nXK1V9huvtAAKc_5a6U,8095
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
@@ -37,8 +38,8 @@ cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb/utils/dotting.py,sha256=mVICaa26R-dQ4JGmPK-kkR6QjX38QiRewXZnGb2DCIc,1784
 cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.57.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.57.dist-info/METADATA,sha256=MZ_FzPMBpEfvCRbDFSCbqwHB2VgXXNKB8mrpc9QhHRg,6510
-cobweb_launcher-1.2.57.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.57.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.57.dist-info/RECORD,,
+cobweb_launcher-1.2.58.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.2.58.dist-info/METADATA,sha256=QH36iouuBHu1S7g9-57Zek9aYa0CiEu7aIN7YOoNjzs,6510
+cobweb_launcher-1.2.58.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.2.58.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.2.58.dist-info/RECORD,,