cobweb-launcher 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

cobweb/db/base/redis_dbv3.py

@@ -1,231 +0,0 @@
-import json
-import random
-import time
-import redis
-from datetime import datetime
-from base.bbb import Seed
-
-
-class RedisDB:
-
-    def __init__(
-            self,
-            project: str,
-            task_name: str,
-            # retry_num: int = 3,
-            host=None,
-            port=None,
-            username=None,
-            password=None,
-            db=0
-    ):
-        pool = redis.ConnectionPool(
-            host=host,
-            port=port,
-            username=username,
-            password=password,
-            db=db
-        )
-        self.heartbeat_key = f"{project}:{task_name}:heartbeat"  # redis type string
-        self.ready_key = f"{project}:{task_name}:seed_info:ready"  # redis type zset, .format(priority)
-        self.spider_key = f"{project}:{task_name}:seed_info:spider"  # redis type hash, .format(priority)
-        self.store_key = f"{project}:{task_name}:seed_info:store:%s"  # redis type set,
-        self.failed_key = f"{project}:{task_name}:seed_info:failed"  # redis type set, .format(priority)
-        self.succeed_key = f"{project}:{task_name}:seed_info:succeed"  # redis type set, .format(priority)
-        self.update_lock = f"{project}:{task_name}:update_seed_lock"  # redis type string
-        self.check_lock = f"{project}:{task_name}:check_seed_lock"  # redis type string
-        # self.retry_lock = f"{project}:{task_name}:retry_seed_lock"  # redis type string
-        self.scheduler_lock = f"{project}:{task_name}:scheduler_lock"  # redis type string
-        self.client = redis.Redis(connection_pool=pool)
-        # self.retry_num = retry_num
-
-    def set_heartbeat(self, t=3):
-        self.client.expire(self.heartbeat_key, t)
-
-    # @property
-    def heartbeat(self):
-        return self.client.ttl(self.heartbeat_key)
-
-    def iterate_hash(self, key, count=1000, match=None):
-        cursor = "0"
-        while cursor != 0:
-            # 使用HSCAN命令迭代获取键值对
-            cursor, data = self.client.hscan(key, cursor=cursor, match=match, count=count)
-            if not data:
-                return None
-            for field, value in data.items():
-                yield field.decode(), value.decode()
-
-    def get_lock(self, key, t=15, timeout=3, sleep_time=0.1):
-        begin_time = int(time.time())
-        while True:
-            if self.client.setnx(key, ""):
-                self.client.expire(key, t)
-                return True
-            if int(time.time()) - begin_time > timeout:
-                break
-            time.sleep(sleep_time)
-
-        if self.client.ttl(key) == -1:
-            delete_status = True
-            for _ in range(3):
-                if self.client.ttl(key) != -1:
-                    delete_status = False
-                    break
-                time.sleep(0.5)
-            if delete_status:
-                self.client.expire(key, t)
-            return False
-        else:
-            ttl = self.client.ttl(key)
-            print("ttl: " + str(ttl))
-            return False
-
-    def execute_update(
-            self,
-            set_info,
-            del_info,
-            status: int = 0
-    ):
-        if status not in [0, 1, 2, 3]:
-            return None
-
-        pipe = self.client.pipeline()
-        pipe.multi()
-
-        if status == 0:
-            pipe.hset(self.spider_key, mapping=set_info)
-            pipe.zrem(self.ready_key, *del_info)
-        elif status == 1:
-            pipe.zadd(self.ready_key, mapping=set_info)
-            pipe.hdel(self.spider_key, *del_info)
-        elif status == 2:
-            pipe.sadd(self.failed_key, *set_info)
-            pipe.hdel(self.spider_key, *del_info)
-        else:
-            pipe.sadd(self.succeed_key, *set_info)
-            pipe.hdel(self.spider_key, *del_info)
-        pipe.execute()
-
-    @property
-    def seed_count(self):
-        return self.client.zcard(self.ready_key)
-
-    def deal_seeds(self, sids, status: bool):
-        if isinstance(sids, str):
-            sids = [sids]
-        # if self.get_lock(key=self.retry_lock, t=15):
-        status = 2 if status else 3
-        del_list, fail_set = [], set()
-        for sid in sids:
-            for field, value in self.iterate_hash(self.spider_key, match=f"*{sid}"):
-                _, priority, _sid = field.split("_")
-                if sid != _sid:
-                    continue
-                seed = Seed(value, priority=priority)
-                del_list.append(field)
-                fail_set.add(seed.format_seed)
-            if del_list:
-                self.execute_update(fail_set, del_list, status=status)
-            # self.client.delete(self.retry_lock)
-            print("retry seeds, sids: {}".format(json.dumps(sids)))
-
-    def set_seeds(self, seeds):
-        item_info = {}
-        if any(isinstance(seeds, t) for t in (list, tuple)):
-            for seed in seeds:
-                item_info[seed.format_seed] = seed.priority
-        elif isinstance(seeds, Seed):
-            item_info[seeds.format_seed] = seeds.priority
-        self.client.zadd(self.ready_key, mapping=item_info)
-
-    def get_seeds(self, length: int = 1000):
-        """
-        redis获取种子
-        """
-        cs = time.time()
-
-        if self.get_lock(key=self.update_lock):
-
-            set_dict, del_list, result = {}, [], []
-
-            # version = int(time.time() * 1e3)
-            version = time.time() * 1e6
-
-            items = self.client.zrangebyscore(self.ready_key, min=0, max="+inf", start=0, num=length, withscores=True)
-
-            # for value, priority in items:
-            #     seed = Seed(value, priority=priority, version=version)
-            #     pty = "{:03d}".format(int(priority))
-            #     key = f"{version}_{pty}_{seed.sid}"
-            #     set_dict[key] = value
-            #     del_list.append(value)
-            #     result.append(seed)
-
-            for value, priority in items:
-                v = version + int(priority) / 1000 + random.random() / 1000
-                seed = Seed(value, priority=priority, version=version)
-                pty = "{:03d}".format(int(priority))
-                key = f"{version}_{pty}_{seed.sid}"
-                set_dict[key] = value
-                del_list.append(value)
-                result.append(seed)
-
-            print("\nset seeds into queue time: " + str(time.time() - cs))
-            if result:
-                self.execute_update(set_dict, del_list)
-
-            self.client.delete(self.update_lock)
-            print("push seeds into queue time: " + str(time.time() - cs))
-            return result
-
-    def check_spider_hash(self):
-        cs = time.time()
-        set_dict, del_list, heartbeat = {}, [], False
-        if self.get_lock(key=self.check_lock, t=60, timeout=600, sleep_time=60):
-            count = self.client.hlen(self.spider_key)
-            if self.client.exists(self.heartbeat_key):
-                heartbeat = True
-            now = int(time.time())
-            for field, value in self.iterate_hash(key=self.spider_key, count=count):
-                version, priority, sid = field.split("_")
-                if heartbeat and int(version) + 600 > now:
-                    continue
-                set_dict[value] = priority
-                del_list.append(field)
-
-                if len(del_list) >= 1000:
-                    self.client.expire(self.check_lock, 60)
-                    self.execute_update(set_dict, del_list, status=1)
-                    set_dict, del_list = {}, []
-
-            if set_dict and del_list:
-                self.execute_update(set_dict, del_list, status=1)
-
-            # self.client.delete(self.check_lock)
-            print("init seeds time: " + str(time.time() - cs))
-            if not heartbeat:
-                self.client.setnx(self.heartbeat_key, "")
-                self.set_heartbeat(t=15)
-
-    def add_store_sid(self, key, data):
-        redis_key = self.store_key % key
-        self.client.sadd(redis_key, *data)
-
-
-current_time = datetime.now()
-# 格式化日期时间字符串
-formatted_time = current_time.strftime("%m%d%H%M%S%f")
-c = int(formatted_time)
-print(c)
-d = 200 + 0.9 * random.random()
-print(d)
-print(time.time())
-print(c + d / 1000)
-# for _ in range(100):
-#     redis_db.get_seeds(1000)
-# redis_db.get_seeds(1000)
-# redis_db.check_spider_hash()
-# redis_db.retry_seeds(["dc895aee47f8fc39c479f7cac6025879"])
-# "1705996980_200_dc895aee47f8fc39c479f7cac6025879"
-

cobweb_launcher-0.0.4.dist-info/RECORD

@@ -1,37 +0,0 @@
-cobweb/__init__.py,sha256=cuMlo5UdlfMRnMf6TfMHL8-FIw83zCiCUtfOrrG6gog,356
-cobweb/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/base/bbb.py,sha256=iX2Xn5AukRussL8xHKTGFfgODWTMaFA3JnvQWd02XDQ,5615
-cobweb/base/config.py,sha256=qZY26tziCeENdopGyVSTCyd_8B30S37GbWx0-_g7EiY,5357
-cobweb/base/decorators.py,sha256=gb2puZLRHv_348ZTIAFLhKuJWKZ6a9_tzduCS8p1-UQ,3089
-cobweb/base/hash_table.py,sha256=-EPHRMNOHHbdjxZJUuDxghUfwrbMA05sqrldHAgrIco,1885
-cobweb/base/interface.py,sha256=LM6C0eh-d1b2CxjtiHKfP8I3XhhlYQR5r-3MD6TMIc4,1037
-cobweb/base/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
-cobweb/base/queue_tmp.py,sha256=NS4qBHKq2o-R78Jpv5xp7TtOHIMg8y0livilTVK49M8,1527
-cobweb/base/request.py,sha256=dHTR7qMHbIIW4ggpTAg4io1TBAYH77teYU4bmcWPXH0,2318
-cobweb/base/task.py,sha256=ztgNh4_tgy95pe3REBfMLKkwf7HaShvp-fdRIWJiJXo,1230
-cobweb/base/utils.py,sha256=NSSgCBE4u1yTpXZrjg8RIepYedo4ZdM38rhDObVfRhI,325
-cobweb/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/db/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/db/base/client_db.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-cobweb/db/base/oss_db.py,sha256=LYRsh26-Fttc6sjc9oOVsCd93jZSja6dtRc8G2Em-1E,3812
-cobweb/db/base/redis_db.py,sha256=2-YMrTpiLNGwA7_bt62HYKevVeJr1Y_n88JKjPg1V3s,7636
-cobweb/db/base/redis_dbv3.py,sha256=u-Tmexl0nrYVVRCCbxAjcH6fyRx7CP4J7iW4BdO7q98,8354
-cobweb/db/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/db/scheduler/default.py,sha256=m-zzC2cbHGxplEW5OoB9Vj3nJm30Xl0sAQ94wpPb7Yw,122
-cobweb/db/scheduler/textfile.py,sha256=EiOxV8h99ouIr2HvmpM9B90QY3hqGNPMeQEnps_RG-c,869
-cobweb/db/storer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/db/storer/console.py,sha256=QbHnZ3ou0AR6c35iVy2hYEbKlYYTnFaD8-bUODeNt14,210
-cobweb/db/storer/loghub.py,sha256=Pb0OwVteIllYjL2cIyBlc1WWau1PHn9K1rlAGrr0M3k,1815
-cobweb/db/storer/redis.py,sha256=jK_RirqgSaV4aIWSuySIm5f1ZfZiULqFj2kman2H-Qw,440
-cobweb/db/storer/textfile.py,sha256=yAvtbPkScjZ298H25kWsI0MDg2JuI2Im4m2qmPEUNTM,443
-cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/distributed/launcher.py,sha256=IV2jd1hLyt1YyhyN3leSMrtbN5APsmvqblr2NXy18xg,6163
-cobweb/distributed/models.py,sha256=7ypYQaiHP91LbPE0u5Lb-9LDazg74UW7KacK4-ai1tM,4438
-cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/single/models.py,sha256=lu8teNWnWcUwZFra8XmqyhzOAf3UyuEztwBr1Ne6pUs,2898
-cobweb/single/nest.py,sha256=mL8q9a5BjtoeUyzXCIVw_vyUsNY8ltbvQpYIIpZEDFU,5012
-cobweb_launcher-0.0.4.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-0.0.4.dist-info/METADATA,sha256=4JShM9pxXi1aouOY1OFrEuW7yrTM1K35GUJLKQpr78w,1225
-cobweb_launcher-0.0.4.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-cobweb_launcher-0.0.4.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-0.0.4.dist-info/RECORD,,