cobweb-launcher 0.1.8__py3-none-any.whl → 1.2.41__py3-none-any.whl

cobweb/db/api_db.py

@@ -0,0 +1,82 @@
+import os
+import json
+import requests
+
+
+class ApiDB:
+
+    def __init__(self, host=None, **kwargs):
+        self.host = host or os.getenv("REDIS_API_HOST", "http://127.0.0.1:4396")
+
+    def _get_response(self, api, params: dict = None):
+        try:
+            url = self.host + api
+            response = requests.get(url, params=params)
+            json_data = response.json()
+            response.close()
+            return json_data["data"]
+        except:
+            return None
+
+    def _post_response(self, api, params: dict = None, data: dict = None):
+        try:
+            url = self.host + api
+            headers = {"Content-Type": "application/json"}
+            response = requests.post(url, headers=headers, params=params, data=json.dumps(data))
+            json_data = response.json()
+            response.close()
+            return json_data["data"]
+        except:
+            return None
+
+    def get(self, name):
+        return self._get_response(api="/get", params=dict(name=name))
+
+    def setnx(self, name, value=""):
+        return self._get_response(api="/setnx", params=dict(name=name, value=value))
+
+    def setex(self, name, t, value=""):
+        return self._get_response(api="/setex", params=dict(name=name, value=value, t=t))
+
+    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
+        return self._get_response(api="/expire", params=dict(name=name, t=t, nx=nx, xx=xx, gt=gt, lt=lt))
+
+    def ttl(self, name):
+        return self._get_response(api="/ttl", params=dict(name=name))
+
+    def delete(self, name):
+        return self._get_response(api="/delete", params=dict(name=name))
+
+    def exists(self, name):
+        return self._get_response(api="/exists", params=dict(name=name))
+
+    def incrby(self, name, value):
+        return self._get_response(api="/incrby", params=dict(name=name, value=value))
+
+    def zcard(self, name) -> bool:
+        return self._get_response(api="/zcard", params=dict(name=name))
+
+    def zadd(self, name, item: dict, **kwargs):
+        return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
+
+    def zrem(self, name, *values):
+        return self._post_response(api="/zrem", data=dict(name=name, values=values))
+
+    def zcount(self, name, _min, _max):
+        return self._get_response(api="/zcount", params=dict(name=name, min=_min, max=_max))
+
+    def lock(self, name, t=15) -> bool:
+        return self._get_response(api="/lock", params=dict(name=name, t=t))
+
+    def auto_incr(self, name, t=15, limit=1000) -> bool:
+        return self._get_response(api="/auto_incr", params=dict(name=name, t=t, limit=limit))
+
+    def members(self, name, score, start=0, count=5000, _min="-inf", _max="+inf"):
+        return self._get_response(api="/members", params=dict(name=name, score=score, start=start, count=count, min=_min, max=_max))
+
+    def done(self, name: list, *values):
+        return self._post_response(api="/done", data=dict(name=name, values=values))
+
+
+
+

cobweb/db/redis_db.py

@@ -1,223 +1,130 @@
-import time
 import redis
-from cobweb import Seed, log
-from cobweb.decorators import check_redis_status
+from cobweb import setting
 
 
 class RedisDB:
 
-    def __init__(
-            self,
-            project: str,
-            task_name: str,
-            config: dict,
-            model: int,
-            cs_lct: int,
-            rs_time: int,
-    ):
-        pool = redis.ConnectionPool(**config)
-        self.heartbeat_key = f"{project}:{task_name}:heartbeat"  # redis type string
-        self.spider_key = f"{project}:{task_name}:seed_info:spider"  # redis type zset, .format(priority)
-        self.storer_key = f"{project}:{task_name}:seed_info:storer:%s"  # redis type set,
-        self.failed_key = f"{project}:{task_name}:seed_info:failed"  # redis type set, .format(priority)
-        self.succeed_key = f"{project}:{task_name}:seed_info:succeed"  # redis type set, .format(priority)
-        self.update_lock = f"{project}:{task_name}:update_seed_lock"  # redis type string
-        self.check_lock = f"{project}:{task_name}:check_seed_lock"  # redis type string
-        self.scheduler_lock = f"{project}:{task_name}:scheduler_lock"  # redis type string
-        self.client = redis.Redis(connection_pool=pool)
-        self.model = model
-        self.cs_lct = cs_lct
-        self.rs_time = rs_time
-
-    @check_redis_status
-    def _get_lock(self, key, t=15, timeout=3, sleep_time=0.1):
-        begin_time = int(time.time())
-        while True:
-            if self.client.setnx(key, ""):
-                self.client.expire(key, t)
-                return True
-            if int(time.time()) - begin_time > timeout:
-                break
-            time.sleep(sleep_time)
-
-        if self.client.ttl(key) == -1:
-            delete_status = True
-            for _ in range(3):
-                if self.client.ttl(key) != -1:
-                    delete_status = False
-                    break
-                time.sleep(0.5)
-            if delete_status:
-                self.client.expire(key, t)
-            return False
-        else:
-            ttl = self.client.ttl(key)
-            log.info("ttl: " + str(ttl))
-            return False
-
-    @check_redis_status
-    def _deal_seed(self, seeds, is_add: bool):
-        if not seeds:
-            return None
-
-        if not isinstance(seeds, list):
-            seeds = [seeds]
-
-        item_info = dict()
-
-        for seed in seeds:
-            if not isinstance(seed, Seed):
-                seed = Seed(seed)
-            item_info[seed.format_seed] = seed._priority
-
-        if item_info:
-            self.client.zadd(self.spider_key, mapping=item_info, nx=is_add, xx=not is_add)
-
-    @check_redis_status
-    def add_seed(self, seeds):
-        self._deal_seed(seeds, is_add=True)
-
-    @check_redis_status
-    def reset_seed(self, seeds):
-        self._deal_seed(seeds, is_add=False)
-
-    @check_redis_status
-    def del_seed(self, seeds, spider_status: bool = True):
-        if not seeds:
-            return None
-
-        if not isinstance(seeds, list):
-            seeds = [seeds]
-
-        seeds = [seed if isinstance(seed, Seed) else Seed(seed) for seed in seeds]
-
-        if seeds:
-            # redis_key = self.succeed_key if spider_status else self.failed_key
-            redis_key = None
-            if spider_status:
-                if isinstance(self.model, int) and self.model == 2:
-                    redis_key = self.succeed_key
-            else:
-                redis_key = self.failed_key
-            if redis_key:
-                self.client.sadd(redis_key, *(str(seed) for seed in seeds))
-            self.client.zrem(self.spider_key, *(seed.format_seed for seed in seeds))
-
-    @check_redis_status
-    def set_storer(self, key, seeds):
-        if not seeds:
-            return None
-
-        if not isinstance(seeds, list):
-            seeds = [seeds]
-
-        item_info = dict()
-        score = -int(time.time())
-        for seed in seeds:
-            if not isinstance(seed, Seed):
-                seed = Seed(seed)
-            item_info[seed.format_seed] = score
-
-        if item_info:
-            self.client.zadd(self.storer_key % key, mapping=item_info)
-            log.info(f"zadd storer key: length {len(item_info.keys())}")
-
-    @check_redis_status
-    def get_seed(self, length: int = 200):
-        cs = time.time()
-
-        if self._get_lock(key=self.update_lock):
-
-            update_item, result = {}, []
-
-            version = int(time.time())
-
-            items = self.client.zrangebyscore(self.spider_key, min=0, max="+inf", start=0, num=length, withscores=True)
-
-            for value, priority in items:
-                score = -(version + int(priority) / 1000)
-                seed = Seed(value, priority=priority, version=version)
-                update_item[value] = score
-                result.append(seed)
-
-            log.info("set seeds into queue time: " + str(time.time() - cs))
-            if result:
-                self.client.zadd(self.spider_key, mapping=update_item, xx=True)
-
-            self.client.delete(self.update_lock)
-            log.info("push seeds into queue time: " + str(time.time() - cs))
-            return result
-
-    @check_redis_status
-    def check_spider_queue(self, stop, storer_num):
-        while not stop.is_set():
-            # 每15s获取check锁,等待600s后仍获取不到锁则重试;获取到锁后，设置锁的存活时间为${cs_lct}s
-            if self._get_lock(key=self.check_lock, t=self.cs_lct, timeout=600, sleep_time=3):
-                heartbeat = True if self.client.exists(self.heartbeat_key) else False
-                # 重启重制score值，否则获取${rs_time}分钟前的分数值
-                score = -int(time.time()) + self.rs_time if heartbeat else "-inf"
-
-                keys = self.client.keys(self.storer_key % "*")
-
-                if keys and len(keys) >= storer_num:
-                    intersection_key = self.storer_key % "intersection"
-                    self.client.delete(intersection_key)
-                    self.client.zinterstore(intersection_key, keys)
-
-                    while True:
-                        members = self.client.zrange(intersection_key, 0, 1999)
-                        if not members:
-                            break
-                        for key in keys:
-                            self.client.zrem(key, *members)
-                        if self.model == 2:
-                            self.client.sadd(self.succeed_key, *members)
-                        self.client.zrem(self.spider_key, *members)
-                        self.client.zrem(intersection_key, *members)
-                        log.info("succeed spider data ...")
-
-                for key in keys:
-                    self.client.zremrangebyscore(key, min=score, max="(0")
-
-                while True:
-                    items = self.client.zrangebyscore(self.spider_key, min=score, max="(0", start=0, num=5000, withscores=True)
-                    if not items:
-                        break
-                    reset_items = {}
-                    for value, priority in items:
-                        reset_score = "{:.3f}".format(priority).split(".")[1]
-                        reset_items[value] = int(reset_score)
-                    if reset_items:
-                        self.client.zadd(self.spider_key, mapping=reset_items, xx=True)
-
-                if not heartbeat:
-                    self.client.setex(self.heartbeat_key, 15, "")
-
-                # self.client.delete(self.check_lock)
-                # time.sleep(3)
-
-    @check_redis_status
-    def set_heartbeat(self, stop):
-        time.sleep(5)
-        while not stop.is_set():
-            self.client.setex(self.heartbeat_key, 5, "")
-            time.sleep(3)
-
-    # @check_redis_status
-    # def heartbeat(self):
-    #     """
-    #     返回心跳key剩余存活时间
-    #     """
-    #     return self.client.ttl(self.heartbeat_key)
-
-    @check_redis_status
-    def spider_queue_length(self):
-        return self.client.zcard(self.spider_key)
-
-    @check_redis_status
-    def ready_seed_length(self):
-        return self.client.zcount(self.spider_key, min=0, max="+inf")
-
-    @check_redis_status
-    def get_scheduler_lock(self):
-        return self._get_lock(self.scheduler_lock)
+    def __init__(self, **kwargs):
+        redis_config = kwargs or setting.REDIS_CONFIG
+        pool = redis.ConnectionPool(**redis_config)
+        self._client = redis.Redis(connection_pool=pool)
+
+    def setnx(self, name, value=""):
+        return self._client.setnx(name, value)
+
+    def setex(self, name, t, value=""):
+        return self._client.setex(name, t, value)
+
+    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
+        return self._client.expire(name, t, nx, xx, gt, lt)
+
+    def ttl(self, name):
+        return self._client.ttl(name)
+
+    def delete(self, name):
+        return self._client.delete(name)
+
+    def exists(self, *name) -> bool:
+        return self._client.exists(*name)
+
+    def sadd(self, name, value):
+        return self._client.sadd(name, value)
+
+    def zcard(self, name) -> bool:
+        return self._client.zcard(name)
+
+    def zadd(self, name, item: dict, **kwargs):
+        return self._client.zadd(name, item, **kwargs)
+
+    def zrem(self, name, *value):
+        return self._client.zrem(name, *value)
+
+    def zcount(self, name, _min, _max):
+        return self._client.zcount(name, _min, _max)
+
+    # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
+    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
+
+    def lua(self, script: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self._client.eval(script, keys_count, *keys, *args)
+
+    def lua_sha(self, sha1: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self._client.evalsha(sha1, keys_count, *keys, *args)
+
+    def execute_lua(self, lua_script: str, keys: list, *args):
+        execute = self._client.register_script(lua_script)
+        return execute(keys=keys, args=args)
+
+    def lock(self, key, t=15) -> bool:
+        lua_script = """
+        local status = redis.call('setnx', KEYS[1], 1)
+        if ( status == 1 ) then
+            redis.call('expire', KEYS[1], ARGV[1])
+        end 
+        return status 
+        """
+        status = self.execute_lua(lua_script, [key], t)
+        return bool(status)
+
+    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
+        lua_script = """
+        local min = ARGV[1]
+        local max = ARGV[2]
+        local start = ARGV[3]
+        local count = ARGV[4]
+        local score = ARGV[5]
+        local members = nil
+
+        if ( type(count) == string ) then
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
+        else
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
+        end
+        
+        local result = {}
+
+        for i = 1, #members, 2 do
+            local priority = nil
+            local member = members[i]
+            local originPriority = nil
+            if ( members[i+1] + 0 < 0 ) then
+                originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
+            else
+                originPriority = math.floor(members[i+1])
+            end
+            
+            if ( score + 0 >= 1000 ) then
+                priority = -score - originPriority / 1000
+            elseif ( score + 0 == 0 ) then
+                priority = originPriority
+            else
+                originPriority = score 
+                priority = score
+            end
+            redis.call('zadd', KEYS[1], priority, member)
+            table.insert(result, member)
+            table.insert(result, originPriority)
+        end
+
+        return result
+        """
+        members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
+        return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
+
+    def done(self, keys: list, *args) -> list:
+        lua_script = """
+        for i, member in ipairs(ARGV) do
+            redis.call("zrem", KEYS[1], member)
+            redis.call("sadd", KEYS[2], member)
+        end
+        """
+        self.execute_lua(lua_script, keys, *args)
+
+
+

cobweb/exceptions/__init__.py

@@ -0,0 +1 @@
+from .oss_db_exception import *

cobweb/exceptions/oss_db_exception.py

@@ -0,0 +1,28 @@
+class OssDBException(Exception):
+    """Base oss client exception that all others inherit."""
+
+
+class OssDBMergeError(OssDBException):
+    """
+    Exception raised when execute merge operation fails.
+    """
+
+
+class OssDBPutPartError(OssDBException):
+    """
+    Exception raised when upload part operation fails.
+    """
+
+
+class OssDBPutObjError(OssDBException):
+    """
+    Exception raised when upload operation fails.
+    """
+
+
+class OssDBAppendObjError(OssDBException):
+    """Exception raised when upload operation fails."""
+
+
+class OssDBInitPartError(OssDBException):
+    """Exception raised when init upload operation fails."""

cobweb/launchers/__init__.py

@@ -0,0 +1,3 @@
+from .launcher_air import LauncherAir
+from .launcher_pro import LauncherPro
+from .launcher_api import LauncherApi