cobweb-launcher 0.0.1__py3-none-any.whl

cobweb/db/base/redis_dbv3.py

@@ -0,0 +1,231 @@
+import json
+import random
+import time
+import redis
+from datetime import datetime
+from base.bbb import Seed
+
+
+class RedisDB:
+
+    def __init__(
+            self,
+            project: str,
+            task_name: str,
+            # retry_num: int = 3,
+            host=None,
+            port=None,
+            username=None,
+            password=None,
+            db=0
+    ):
+        pool = redis.ConnectionPool(
+            host=host,
+            port=port,
+            username=username,
+            password=password,
+            db=db
+        )
+        self.heartbeat_key = f"{project}:{task_name}:heartbeat"  # redis type string
+        self.ready_key = f"{project}:{task_name}:seed_info:ready"  # redis type zset, .format(priority)
+        self.spider_key = f"{project}:{task_name}:seed_info:spider"  # redis type hash, .format(priority)
+        self.store_key = f"{project}:{task_name}:seed_info:store:%s"  # redis type set,
+        self.failed_key = f"{project}:{task_name}:seed_info:failed"  # redis type set, .format(priority)
+        self.succeed_key = f"{project}:{task_name}:seed_info:succeed"  # redis type set, .format(priority)
+        self.update_lock = f"{project}:{task_name}:update_seed_lock"  # redis type string
+        self.check_lock = f"{project}:{task_name}:check_seed_lock"  # redis type string
+        # self.retry_lock = f"{project}:{task_name}:retry_seed_lock"  # redis type string
+        self.scheduler_lock = f"{project}:{task_name}:scheduler_lock"  # redis type string
+        self.client = redis.Redis(connection_pool=pool)
+        # self.retry_num = retry_num
+
+    def set_heartbeat(self, t=3):
+        self.client.expire(self.heartbeat_key, t)
+
+    # @property
+    def heartbeat(self):
+        return self.client.ttl(self.heartbeat_key)
+
+    def iterate_hash(self, key, count=1000, match=None):
+        cursor = "0"
+        while cursor != 0:
+            # 使用HSCAN命令迭代获取键值对
+            cursor, data = self.client.hscan(key, cursor=cursor, match=match, count=count)
+            if not data:
+                return None
+            for field, value in data.items():
+                yield field.decode(), value.decode()
+
+    def get_lock(self, key, t=15, timeout=3, sleep_time=0.1):
+        begin_time = int(time.time())
+        while True:
+            if self.client.setnx(key, ""):
+                self.client.expire(key, t)
+                return True
+            if int(time.time()) - begin_time > timeout:
+                break
+            time.sleep(sleep_time)
+
+        if self.client.ttl(key) == -1:
+            delete_status = True
+            for _ in range(3):
+                if self.client.ttl(key) != -1:
+                    delete_status = False
+                    break
+                time.sleep(0.5)
+            if delete_status:
+                self.client.expire(key, t)
+            return False
+        else:
+            ttl = self.client.ttl(key)
+            print("ttl: " + str(ttl))
+            return False
+
+    def execute_update(
+            self,
+            set_info,
+            del_info,
+            status: int = 0
+    ):
+        if status not in [0, 1, 2, 3]:
+            return None
+
+        pipe = self.client.pipeline()
+        pipe.multi()
+
+        if status == 0:
+            pipe.hset(self.spider_key, mapping=set_info)
+            pipe.zrem(self.ready_key, *del_info)
+        elif status == 1:
+            pipe.zadd(self.ready_key, mapping=set_info)
+            pipe.hdel(self.spider_key, *del_info)
+        elif status == 2:
+            pipe.sadd(self.failed_key, *set_info)
+            pipe.hdel(self.spider_key, *del_info)
+        else:
+            pipe.sadd(self.succeed_key, *set_info)
+            pipe.hdel(self.spider_key, *del_info)
+        pipe.execute()
+
+    @property
+    def seed_count(self):
+        return self.client.zcard(self.ready_key)
+
+    def deal_seeds(self, sids, status: bool):
+        if isinstance(sids, str):
+            sids = [sids]
+        # if self.get_lock(key=self.retry_lock, t=15):
+        status = 2 if status else 3
+        del_list, fail_set = [], set()
+        for sid in sids:
+            for field, value in self.iterate_hash(self.spider_key, match=f"*{sid}"):
+                _, priority, _sid = field.split("_")
+                if sid != _sid:
+                    continue
+                seed = Seed(value, priority=priority)
+                del_list.append(field)
+                fail_set.add(seed.format_seed)
+            if del_list:
+                self.execute_update(fail_set, del_list, status=status)
+            # self.client.delete(self.retry_lock)
+            print("retry seeds, sids: {}".format(json.dumps(sids)))
+
+    def set_seeds(self, seeds):
+        item_info = {}
+        if any(isinstance(seeds, t) for t in (list, tuple)):
+            for seed in seeds:
+                item_info[seed.format_seed] = seed.priority
+        elif isinstance(seeds, Seed):
+            item_info[seeds.format_seed] = seeds.priority
+        self.client.zadd(self.ready_key, mapping=item_info)
+
+    def get_seeds(self, length: int = 1000):
+        """
+        redis获取种子
+        """
+        cs = time.time()
+
+        if self.get_lock(key=self.update_lock):
+
+            set_dict, del_list, result = {}, [], []
+
+            # version = int(time.time() * 1e3)
+            version = time.time() * 1e6
+
+            items = self.client.zrangebyscore(self.ready_key, min=0, max="+inf", start=0, num=length, withscores=True)
+
+            # for value, priority in items:
+            #     seed = Seed(value, priority=priority, version=version)
+            #     pty = "{:03d}".format(int(priority))
+            #     key = f"{version}_{pty}_{seed.sid}"
+            #     set_dict[key] = value
+            #     del_list.append(value)
+            #     result.append(seed)
+
+            for value, priority in items:
+                v = version + int(priority) / 1000 + random.random() / 1000
+                seed = Seed(value, priority=priority, version=version)
+                pty = "{:03d}".format(int(priority))
+                key = f"{version}_{pty}_{seed.sid}"
+                set_dict[key] = value
+                del_list.append(value)
+                result.append(seed)
+
+            print("\nset seeds into queue time: " + str(time.time() - cs))
+            if result:
+                self.execute_update(set_dict, del_list)
+
+            self.client.delete(self.update_lock)
+            print("push seeds into queue time: " + str(time.time() - cs))
+            return result
+
+    def check_spider_hash(self):
+        cs = time.time()
+        set_dict, del_list, heartbeat = {}, [], False
+        if self.get_lock(key=self.check_lock, t=60, timeout=600, sleep_time=60):
+            count = self.client.hlen(self.spider_key)
+            if self.client.exists(self.heartbeat_key):
+                heartbeat = True
+            now = int(time.time())
+            for field, value in self.iterate_hash(key=self.spider_key, count=count):
+                version, priority, sid = field.split("_")
+                if heartbeat and int(version) + 600 > now:
+                    continue
+                set_dict[value] = priority
+                del_list.append(field)
+
+                if len(del_list) >= 1000:
+                    self.client.expire(self.check_lock, 60)
+                    self.execute_update(set_dict, del_list, status=1)
+                    set_dict, del_list = {}, []
+
+            if set_dict and del_list:
+                self.execute_update(set_dict, del_list, status=1)
+
+            # self.client.delete(self.check_lock)
+            print("init seeds time: " + str(time.time() - cs))
+            if not heartbeat:
+                self.client.setnx(self.heartbeat_key, "")
+                self.set_heartbeat(t=15)
+
+    def add_store_sid(self, key, data):
+        redis_key = self.store_key % key
+        self.client.sadd(redis_key, *data)
+
+
+current_time = datetime.now()
+# 格式化日期时间字符串
+formatted_time = current_time.strftime("%m%d%H%M%S%f")
+c = int(formatted_time)
+print(c)
+d = 200 + 0.9 * random.random()
+print(d)
+print(time.time())
+print(c + d / 1000)
+# for _ in range(100):
+#     redis_db.get_seeds(1000)
+# redis_db.get_seeds(1000)
+# redis_db.check_spider_hash()
+# redis_db.retry_seeds(["dc895aee47f8fc39c479f7cac6025879"])
+# "1705996980_200_dc895aee47f8fc39c479f7cac6025879"
+

cobweb/db/scheduler/default.py

@@ -0,0 +1,8 @@
+from base.interface import SchedulerInterface
+
+
+class Default(SchedulerInterface):
+
+    def schedule(self):
+        pass
+

cobweb/db/scheduler/textfile.py

@@ -0,0 +1,29 @@
+from base.log import log
+from base.bbb import Seed
+from base.interface import SchedulerInterface
+
+
+class Textfile(SchedulerInterface):
+
+    index = None
+
+    def schedule(self):
+        try:
+            seeds = []
+            with open(self.table, "r") as fp:
+                fp.seek(self.index or 0, 0)
+                for _ in range(self.length):
+                    data = fp.readline().strip()
+                    if not data:
+                        log.info("scheduler end!")
+                        self.stop = True
+                        break
+                    seeds.append(Seed(data))
+                    self.index = fp.tell()
+            return seeds
+        except FileNotFoundError:
+            log.error("task table not found!")
+            return None
+        except TypeError:
+            log.error("task table type error!")
+            return None

cobweb/db/storer/console.py

@@ -0,0 +1,10 @@
+from base.log import log
+from base.interface import StorerInterface
+
+
+class Console(StorerInterface):
+
+    def store(self, data_list):
+        for item in data_list:
+            log.info(f"item info: {item}")
+

cobweb/db/storer/loghub.py

@@ -0,0 +1,55 @@
+import json
+from base.log import log
+from base.interface import StorerInterface
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+
+
+class Loghub(StorerInterface):
+
+    def __init__(self, table, fields, length, queue, config):
+        super().__init__(table, fields, length, queue, config)
+        self.client = None
+
+    def init_loghub_clint(self):
+        try:
+            self.client = LogClient(
+                self.config['endpoint'],
+                self.config['access_key_id'],
+                self.config['access_key']
+            )
+        except Exception as e:
+            self.client = None
+            return False
+
+    def store(self, data_list):
+        try:
+            if not self.client:
+                self.init_loghub_clint()
+
+            log_items = list()
+            for item in data_list:
+                temp = item._asdict()
+                for key, value in temp.items():
+                    if isinstance(value, str):
+                        temp[key] = value
+                    else:
+                        temp[key] = json.dumps(value, ensure_ascii=False)
+                log_item = LogItem()
+                contents = sorted(temp.items())  # dict to tuple
+                log_item.set_contents(contents)
+                log_items.append(log_item)
+            request = PutLogsRequest(
+                project=self.config["project"],
+                logstore=self.table,
+                topic=self.config["topic"],
+                source=self.config.get("source"),
+                logitems=log_items,
+                compress=True
+            )
+            self.client.put_logs(request=request)
+            log.info(f"save data, data length: {len(data_list)}")
+            return True
+        except Exception as e:
+            log.exception(e)
+            return False
+

cobweb/db/storer/redis.py

@@ -0,0 +1,16 @@
+from base.log import log
+from base.interface import StorerInterface
+
+
+class Redis(StorerInterface):
+
+    def store(self, data_list):
+        try:
+            data_str = "\n".join(str(data) for data in data_list)
+            with open(self.table, "a") as fp:
+                fp.write(data_str)
+            log.info(f"save data, data length: {len(data_list)}")
+            return True
+        except Exception as e:
+            return False
+

cobweb/db/storer/textfile.py

@@ -0,0 +1,16 @@
+from base.log import log
+from base.interface import StorerInterface
+
+
+class Textfile(StorerInterface):
+
+    def store(self, data_list):
+        try:
+            data_str = "\n".join(str(data) for data in data_list)
+            with open(self.table, "a") as fp:
+                fp.write(data_str)
+            log.info(f"save data, data length: {len(data_list)}")
+            return True
+        except Exception as e:
+            return False
+

cobweb/distributed/launcher.py

@@ -0,0 +1,194 @@
+import time
+import threading
+from threading import Thread
+from base.log import log
+from db.base.redis_db import RedisDB
+from base.bbb import Queue, Seed, DBItem
+from base.utils import struct_queue_name, restore_table_name
+from models import Scheduler, Spider, Storer
+
+
+def start_seeds(seeds):
+    if not seeds:
+        return None
+    if any(isinstance(seeds, t) for t in (list, tuple)):
+        return [Seed(seed) for seed in seeds]
+    elif any(isinstance(seeds, t) for t in (str, dict)):
+        return Seed(seeds)
+
+
+def parse_storer_info(storer_info):
+    storer_data = {}
+    storer_info_list = []
+    if storer_info.__class__.__name__ == 'StorerInfo':
+        storer_info_list.append(storer_info)
+    elif any(isinstance(storer_info, t) for t in (list, tuple)):
+        storer_info_list = storer_info
+    for info in storer_info_list:
+        db_name = info.DB.__name__
+        storer_data.setdefault(db_name, {"StorerDB": info.DB, "db_args_list": []})
+        storer_data[db_name]["db_args_list"].append(info[1:])
+    return storer_data
+
+
+def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
+    time.sleep(5)
+    while True:
+        if (
+                scheduler.stop and
+                not ready_seed_length() and
+                not scheduler.queue.length and
+                not spider.spider_in_progress.length
+        ):
+            log.info("spider is done?")
+            last.set()
+            time.sleep(5)
+            storer_queue_empty = True
+            for storer in storer_list:
+                if storer.queue.length:
+                    storer_queue_empty = False
+                    break
+            if storer_queue_empty and not spider_queue_length():
+                log.info("spider done!")
+                break
+        last.clear()
+        time.sleep(3)
+    stop.set()
+
+
+def launcher(task):
+    """
+    任务启动装饰器
+    :param task: 任务配置信息
+    """
+    def decorator(func):
+        """
+        Item:
+            Textfile()
+            Loghub()
+            Console()
+        e.g.
+        task.fields = "a,b"
+        func(item, seed)
+            a = "a"
+            b = "b"
+            data = {"a": "a", "b": "b"}
+            yield item.Loghub(**data)
+            yield item.Loghub(a=a, b=b)
+        """
+        storer_list = []
+
+        # 程序结束事件
+        last = threading.Event()
+        # 停止采集事件
+        stop = threading.Event()
+
+        # 初始化redis信息
+        redis_db = RedisDB(task.project, task.task_name, *task.redis_info)
+
+        log.info("初始化cobweb!")
+
+        seed_queue = Queue()
+
+        # 调度器动态继承
+        SchedulerDB, table, sql, length, size, config = task.scheduler_info
+        SchedulerTmp = type(SchedulerDB.__name__, (Scheduler, SchedulerDB), {})
+
+        # 初始化调度器
+        scheduler = SchedulerTmp(table, sql, length, size, seed_queue, config)
+
+        # 初始化采集器
+        spider = Spider(seed_queue, task.max_retries)
+
+        # 解析存储器信息
+        storer_data = parse_storer_info(task.storer_info)
+
+        # new item
+        item = type("Item", (object,), {"redis_client": redis_db})()
+        for db_name in storer_data.keys():
+            # 存储器动态继承
+            StorerDB = storer_data[db_name]["StorerDB"]
+            StorerTmp = type(StorerDB.__name__, (Storer, StorerDB), {})
+            db_args_list = storer_data[db_name]["db_args_list"]
+            for storer_db_args in db_args_list:
+                table, fields, length, config = storer_db_args
+                if not getattr(item, db_name, None):
+                    instance = type(db_name, (DBItem,), {})
+                    setattr(item, db_name, instance)
+                # 创建存储xxx, 创建存储队列
+                storer_item_instance = getattr(item, db_name)
+                storer_item_instance.init_item(table, fields)
+                #
+                storer_queue = struct_queue_name(db_name, table)
+                queue = getattr(storer_item_instance, storer_queue)
+                # 初始话存储器
+                table_name = restore_table_name(table_name=table)
+                storer = StorerTmp(table_name, fields, length, queue, config)
+                storer_list.append(storer)
+
+        Thread(target=redis_db.check_spider_queue, args=(stop, len(storer_list))).start()
+        Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
+
+        # 推送初始种子
+        seeds = start_seeds(task.start_seed)
+        redis_db.add_seed(seeds)
+        # 启动调度器, 调度至redis队列
+        Thread(
+            # name="xxxx_schedule_seeds",
+            target=scheduler.schedule_seed,
+            args=(
+                redis_db.ready_seed_length,
+                redis_db.get_scheduler_lock,
+                redis_db.add_seed
+            )
+        ).start()
+
+        # 启动调度器, 调度任务队列
+        Thread(
+            # name="xxxx_schedule_task",
+            target=scheduler.schedule_task,
+            args=(
+                stop, redis_db.get_seed,
+                redis_db.ready_seed_length
+            )
+        ).start()
+
+        # 启动采集器
+        for index in range(task.spider_num):
+            Thread(
+                # name=f"xxxx_spider_task:{index}",
+                target=spider.spider_task,
+                args=(
+                    stop, func, item,
+                    redis_db.del_seed
+                )
+            ).start()
+
+        # 启动存储器
+        for storer in storer_list:
+            Thread(
+                # name=f"xxxx_store_task:{storer.table}",
+                target=storer.store_task,
+                args=(
+                    stop, last,
+                    redis_db.reset_seed,
+                    redis_db.set_storer
+                )
+            ).start()
+
+        Thread(
+            # name="check_spider",
+            target=check,
+            args=(
+                stop, last, spider,
+                scheduler, storer_list,
+                redis_db.ready_seed_length,
+                redis_db.spider_queue_length,
+            )
+        ).start()
+
+    return decorator
+
+
+
+