cobweb-launcher 1.3.14__py3-none-any.whl → 3.1.0__py3-none-any.whl

cobweb/schedulers/launcher_api.py

@@ -0,0 +1,225 @@
+import time
+import threading
+
+from cobweb.db import ApiDB
+from cobweb.base import Seed, logger
+from cobweb.constant import DealModel, LogTemplate
+from .launcher import Launcher, check_pause
+
+
+class LauncherApi(Launcher):
+
+    def __init__(self, task, project, custom_setting=None, **kwargs):
+        super().__init__(task, project, custom_setting, **kwargs)
+        self._db = ApiDB()
+
+        self._todo_key = "{%s:%s}:todo" % (project, task)
+        self._done_key = "{%s:%s}:done" % (project, task)
+        self._fail_key = "{%s:%s}:fail" % (project, task)
+        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
+
+        self._statistics_done_key = "statistics:%s:%s:done" % (project, task)
+        self._statistics_fail_key = "statistics:%s:%s:fail" % (project, task)
+        self._speed_control_key = "speed_control:%s_%s" % (project, task)
+
+        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+
+        # self._bf_key = "bloom_%s_%s" % (project, task)
+        # self._bf = BloomFilter(self._bf_key)
+
+        self._heartbeat_start_event = threading.Event()
+        self._redis_queue_empty_event = threading.Event()
+
+    @property
+    def heartbeat(self):
+        return self._db.exists(self._heartbeat_key)
+
+    def statistics(self, key, count):
+        if not self._task_model and not self._db.exists(key):
+            self._db.setex(key, 86400 * 30, int(count))
+        else:
+            self._db.incrby(key, count)
+
+    def _get_seed(self) -> Seed:
+        """
+        从队列中获取种子（频控）
+        设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
+        :return: True -> 种子, False -> None
+        """
+        if (self._speed_control and self.__LAUNCHER_QUEUE__["todo"].length and
+                not self._db.auto_incr(self._speed_control_key, t=self._time_window, limit=self._spider_max_count)):
+            expire_time = self._db.ttl(self._speed_control_key)
+            if isinstance(expire_time, int) and expire_time <= -1:
+                self._db.delete(self._speed_control_key)
+            elif isinstance(expire_time, int):
+                logger.info(f"Too fast! Please wait {expire_time} seconds...")
+                time.sleep(expire_time / 2)
+            return None
+        seed = self.__LAUNCHER_QUEUE__["todo"].pop()
+        return seed
+
+    @check_pause
+    def _execute_heartbeat(self):
+        if self._heartbeat_start_event.is_set():
+            self._db.setex(self._heartbeat_key, 5)
+        time.sleep(3)
+
+    @check_pause
+    def _reset(self):
+        """
+        检查过期种子，重新添加到redis缓存中
+        """
+        reset_wait_seconds = 30
+        if self._db.lock(self._reset_lock_key, t=120):
+
+            _min = -int(time.time()) + self._seed_reset_seconds \
+                if self.heartbeat else "-inf"
+
+            self._db.members(self._todo_key, 0, _min=_min, _max="(0")
+
+            if not self.heartbeat:
+                self._heartbeat_start_event.set()
+
+            self._db.delete(self._reset_lock_key)
+
+        time.sleep(reset_wait_seconds)
+
+    @check_pause
+    def _scheduler(self):
+        """
+        调度任务，获取redis队列种子，同时添加到doing字典中
+        """
+        if not self._db.zcount(self._todo_key, 0, "(1000"):
+            time.sleep(self._scheduler_wait_seconds)
+        elif self.__LAUNCHER_QUEUE__['todo'].length >= self._todo_queue_size:
+            time.sleep(self._todo_queue_full_wait_seconds)
+        else:
+            members = self._db.members(
+                self._todo_key, int(time.time()),
+                count=self._todo_queue_size,
+                _min=0, _max="(1000"
+            )
+            for member, priority in members:
+                seed = Seed(member, priority=priority)
+                self.__LAUNCHER_QUEUE__['todo'].push(seed)
+                self.__DOING__[seed.to_string] = seed.params.priority
+
+    @check_pause
+    def _insert(self):
+        """
+        添加新种子到redis队列中
+        """
+        new_seeds = {}
+        del_seeds = set()
+        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
+        for _ in range(self._new_queue_max_size):
+            seed_tuple = self.__LAUNCHER_QUEUE__['new'].pop()
+            if not seed_tuple:
+                break
+            seed, new_seed = seed_tuple
+            new_seeds[new_seed.to_string] = new_seed.params.priority
+            del_seeds.add(seed)
+        if new_seeds:
+            self._db.zadd(self._todo_key, new_seeds, nx=True)
+        if del_seeds:
+            self.__LAUNCHER_QUEUE__['done'].push(list(del_seeds))
+        if status:
+            time.sleep(self._new_queue_wait_seconds)
+
+    @check_pause
+    def _refresh(self):
+        """
+        刷新doing种子过期时间，防止reset重新消费
+        """
+        if self.__DOING__:
+            refresh_time = int(time.time())
+            seeds = {k:-refresh_time - v / 1000 for k, v in self.__DOING__.items()}
+            self._db.zadd(self._todo_key, item=seeds, xx=True)
+        time.sleep(15)
+
+    @check_pause
+    def _delete(self):
+        """
+        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
+        """
+        # seed_info = {"count": 0, "failed": [], "succeed": [], "common": []}
+
+        seed_list = []
+        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
+
+        for _ in range(self._done_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['done'].pop()
+            if not seed:
+                break
+            seed_list.append(seed.to_string)
+
+        if seed_list:
+
+            self._db.zrem(self._todo_key, *seed_list)
+            self._remove_doing_seeds(seed_list)
+
+        if status:
+            time.sleep(self._done_queue_wait_seconds)
+
+    def _polling(self):
+        wait_scheduler_execute = True
+        check_emtpy_times = 0
+        while not self._stop.is_set():
+            queue_not_empty_count = 0
+            pooling_wait_seconds = 30
+
+            for q in self.__LAUNCHER_QUEUE__.values():
+                if q.length != 0:
+                    queue_not_empty_count += 1
+                    wait_scheduler_execute = False
+
+            if queue_not_empty_count == 0:
+                pooling_wait_seconds = 3
+                if self._pause.is_set():
+                    check_emtpy_times = 0
+                    if not self._task_model and (
+                            not wait_scheduler_execute or
+                            int(time.time()) - self._app_time > self._before_scheduler_wait_seconds
+                    ):
+                        logger.info("Done! ready to close thread...")
+                        self._stop.set()
+
+                    elif self._db.zcount(self._todo_key, _min=0, _max="(1000"):
+                        logger.info(f"Recovery {self.task} task run！")
+                        self._pause.clear()
+                        self._execute()
+                    else:
+                        logger.info("pause! waiting for resume...")
+                elif check_emtpy_times > 2:
+                    self.__DOING__ = {}
+                    seed_count = self._db.zcard(self._todo_key)
+                    logger.info(f"队列剩余种子数:{seed_count}")
+                    if not seed_count:
+                        logger.info("Done! pause set...")
+                        self._pause.set()
+                    else:
+                        self._pause.clear()
+                else:
+                    logger.info(
+                        "check whether the task is complete, "
+                        f"reset times {3 - check_emtpy_times}"
+                    )
+                    check_emtpy_times += 1
+            else:
+                if self._pause.is_set():
+                    self._pause.clear()
+                logger.info(LogTemplate.launcher_pro_polling.format(
+                    task=self.task,
+                    doing_len=len(self.__DOING__.keys()),
+                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
+                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
+                    redis_seed_count=self._db.zcount(self._todo_key, "-inf", "+inf"),
+                    redis_todo_len=self._db.zcount(self._todo_key, 0, "(1000"),
+                    redis_doing_len=self._db.zcount(self._todo_key, "-inf", "(0"),
+                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
+                ))
+
+            time.sleep(pooling_wait_seconds)
+
+        logger.info("Done! Ready to close thread...")
+

cobweb/schedulers/scheduler.py

@@ -0,0 +1,85 @@
+import threading
+
+from cobweb import setting
+from typing import Callable
+from cobweb.base import Queue
+from abc import ABC, abstractmethod
+
+
+class Scheduler(ABC, threading.Thread):
+
+    __LAUNCHER_FUNC__ = ["_reset", "_scheduler", "_insert", "_refresh", "_delete"]
+
+    def __init__(
+            self,
+            task,
+            project,
+            stop: threading.Event,
+            pause: threading.Event,
+            new: Queue,
+            todo: Queue,
+            done: Queue,
+            upload: Queue,
+            register: Callable
+    ):
+        super().__init__()
+        self.task = task
+        self.project = project
+
+        self.task_model = setting.TASK_MODEL
+        self.seed_reset_seconds = setting.SEED_RESET_SECONDS
+        self.scheduler_wait_seconds = setting.SCHEDULER_WAIT_SECONDS
+        self.new_queue_wait_seconds = setting.NEW_QUEUE_WAIT_SECONDS
+        self.done_queue_wait_seconds = setting.DONE_QUEUE_WAIT_SECONDS
+        self.todo_queue_full_wait_seconds = setting.TODO_QUEUE_FULL_WAIT_SECONDS
+        self.before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
+
+        self.todo_queue_size = setting.TODO_QUEUE_SIZE
+        self.new_queue_max_size = setting.NEW_QUEUE_MAX_SIZE
+        self.done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
+        self.upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
+
+        self.stop = stop
+        self.pause = pause
+
+        self.new = new
+        self.todo = todo
+        self.done = done
+        self.upload = upload
+
+        self.register = register
+
+        self.working_seeds = dict()
+
+    def is_empty(self):
+        if self.new.length == 0 and self.todo.length == 0 and self.done.length == 0 and self.upload.length == 0:
+            return True
+        else:
+            return False
+
+    def remove_working_seeds(self, seeds: list = None):
+        for seed in seeds:
+            if seed in self.working_seeds:
+                self.working_seeds.pop(seed)
+
+    @abstractmethod
+    def reset(self):
+        ...
+
+    @abstractmethod
+    def schedule(self):
+        ...
+
+    @abstractmethod
+    def insert(self):
+        ...
+
+    @abstractmethod
+    def refresh(self):
+        ...
+
+    @abstractmethod
+    def delete(self):
+        ...
+
+

cobweb/schedulers/scheduler_with_redis.py

@@ -0,0 +1,177 @@
+import os
+import time
+import threading
+from typing import Callable
+from cobweb.db import RedisDB, ApiDB
+from cobweb.utils import check_pause
+from cobweb.base import Queue, Seed, logger
+from cobweb.constant import LogTemplate
+from .scheduler import Scheduler
+use_api = bool(int(os.getenv("REDIS_API")))
+
+
+class RedisScheduler(Scheduler):
+
+    def __init__(
+            self,
+            task,
+            project,
+            stop: threading.Event,
+            pause: threading.Event,
+            new: Queue,
+            todo: Queue,
+            done: Queue,
+            upload: Queue,
+            register: Callable
+    ):
+        super().__init__(task, project, stop, pause, new, todo, done, upload, register)
+        self.todo_key = "{%s:%s}:todo" % (project, task)
+        self.done_key = "{%s:%s}:done" % (project, task)
+        self.fail_key = "{%s:%s}:fail" % (project, task)
+        self.heartbeat_key = "heartbeat:%s_%s" % (project, task)
+        self.speed_control_key = "speed_control:%s_%s" % (project, task)
+        self.reset_lock_key = "lock:reset:%s_%s" % (project, task)
+        self.redis_queue_empty_event = threading.Event()
+        self.db = ApiDB() if use_api else RedisDB()
+
+    @check_pause
+    def reset(self):
+        """
+        检查过期种子，重新添加到redis缓存中
+        """
+        reset_wait_seconds = 30
+        if self.db.lock(self.reset_lock_key, t=60):
+
+            _min = -int(time.time()) + self.seed_reset_seconds
+            self.db.members(self.todo_key, 0, _min=_min, _max="(0")
+            self.db.delete(self.reset_lock_key)
+
+        time.sleep(reset_wait_seconds)
+
+    @check_pause
+    def schedule(self):
+        """
+        调度任务，获取redis队列种子，同时添加到doing字典中
+        """
+        if not self.db.zcount(self.todo_key, 0, "(1000"):
+            time.sleep(self.scheduler_wait_seconds)
+        elif self.todo.length >= self.todo_queue_size:
+            time.sleep(self.todo_queue_full_wait_seconds)
+        else:
+            members = self.db.members(
+                self.todo_key, int(time.time()),
+                count=self.todo_queue_size,
+                _min=0, _max="(1000"
+            )
+            for member, priority in members:
+                seed = Seed(member, priority=priority)
+                self.working_seeds[seed.to_string] = seed.params.priority
+                self.todo.push(seed)
+
+    @check_pause
+    def insert(self):
+        """
+        添加新种子到redis队列中
+        """
+        new_seeds = {}
+        del_seeds = set()
+        status = self.new.length < self.new_queue_max_size
+        for _ in range(self.new_queue_max_size):
+            seed_tuple = self.new.pop()
+            if not seed_tuple:
+                break
+            seed, new_seed = seed_tuple
+            new_seeds[new_seed.to_string] = new_seed.params.priority
+            del_seeds.add(seed)
+        if new_seeds:
+            self.db.zadd(self.todo_key, new_seeds, nx=True)
+        if del_seeds:
+            self.done.push(list(del_seeds))
+        if status:
+            time.sleep(self.new_queue_wait_seconds)
+
+    @check_pause
+    def refresh(self):
+        """
+        刷新doing种子过期时间，防止reset重新消费
+        """
+        if self.working_seeds:
+            refresh_time = int(time.time())
+            seeds = {k:-refresh_time - v / 1000 for k, v in self.working_seeds.items()}
+            self.db.zadd(self.todo_key, item=seeds, xx=True)
+        time.sleep(3)
+
+    @check_pause
+    def delete(self):
+        """
+        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
+        """
+        seed_list = []
+        status = self.done.length < self.done_queue_max_size
+
+        for _ in range(self.done_queue_max_size):
+            seed = self.done.pop()
+            if not seed:
+                break
+            seed_list.append(seed.to_string)
+
+        if seed_list:
+
+            self.db.zrem(self.todo_key, *seed_list)
+            self.remove_working_seeds(seed_list)
+
+        if status:
+            time.sleep(self.done_queue_wait_seconds)
+
+    def run(self):
+        start_time = int(time.time())
+
+        self.register(self.reset, tag="scheduler")
+        self.register(self.insert, tag="scheduler")
+        self.register(self.delete, tag="scheduler")
+        self.register(self.refresh, tag="scheduler")
+        self.register(self.schedule, tag="scheduler")
+
+        while not self.stop.is_set():
+            working_count = len(self.working_seeds.keys())
+            memory_count = self.db.zcount(self.todo_key, "-inf", "(0")
+            todo_count = self.db.zcount(self.todo_key, 0, "(1000")
+            all_count = self.db.zcard(self.todo_key)
+            
+            if self.is_empty():
+                if self.pause.is_set():
+                    execute_time = int(time.time()) - start_time
+                    if not self.task_model and execute_time > self.before_scheduler_wait_seconds:
+                        logger.info("Done! ready to close thread...")
+                        self.stop.set()
+                    elif todo_count:
+                        logger.info(f"Recovery {self.task} task run！todo seeds count: {todo_count}, queue length: {all_count}")
+                        self.pause.clear()
+                        # self.execute()
+                    else:
+                        logger.info("pause! waiting for resume...")
+                else:
+                    if all_count:
+                        logger.info(f"todo seeds count: {todo_count}, queue length: {all_count}")
+                        self.pause.is_set()
+                    else:
+                        logger.info("Done! pause set...")
+                        self.pause.clear()
+            else:
+                if self.pause.is_set():
+                    self.pause.clear()
+                logger.info(LogTemplate.launcher_pro_polling.format(
+                    task=self.task,
+                    doing_len=working_count,
+                    todo_len=self.todo.length,
+                    done_len=self.done.length,
+                    redis_seed_count=all_count,
+                    redis_todo_len=todo_count,
+                    redis_doing_len=memory_count,
+                    upload_len=self.upload.length,
+                ))
+
+            time.sleep(30)
+
+        logger.info("Done! Ready to close thread...")
+

cobweb/setting.py

@@ -30,49 +30,29 @@ OSS_MIN_UPLOAD_SIZE = 1024
 # 采集器选择
 CRAWLER = "cobweb.crawlers.Crawler"
 
-# 数据存储链路
-PIPELINE = "cobweb.pipelines.pipeline_console.Console"
+# 数据管道
+PIPELINE = "cobweb.pipelines.Console"
+
+# 调度器
+SCHEDULER = "cobweb.schedulers.RedisScheduler"
 
 
 # Launcher 等待时间
 
 BEFORE_SCHEDULER_WAIT_SECONDS = 60  # 调度前等待时间,只作用于单次任务
-
-SCHEDULING_WAIT_TIME = 15  # SCHEDULER ITEM 调度等待时间
-INSERTING_WAIT_TIME = 30   # INSERT ITEM 等待时间
-REMOVING_WAIT_TIME = 5     # REMOVE ITEM 等待时间
-RESET_WAIT_TIME = 30       # REST ITEM 等待时间
-UPLOAD_WAIT_TIME = 15      # 上传等待时间
-
-TODO_QUEUE_FULL_WAIT_SECONDS = 5  # 队列已满时等待时间
+SCHEDULER_WAIT_SECONDS = 15  # 调度等待时间
+TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
 NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
 DONE_QUEUE_WAIT_SECONDS = 5   # done队列等待时间
 UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
-SEED_RESET_SECONDS = 30   # 种子重制时间
+SEED_RESET_SECONDS = 60   # 种子重制时间
 
 
 # Launcher 队列长度
-SCHEDULING_SIZE = 100   # 调度队列长度
-INSERTING_SIZE = 100    # INSERT 长度
-REMOVING_SIZE = 100     # REMOVE 长度
-
-# SEED = Queue()          # 添加任务种子队列
-# TODO = Queue()          # 任务种子队列
-# REQUEST = Queue()       # 请求队列
-# DOWNLOAD = Queue()      # 下载任务队列
-# RESPONSE = Queue()      # 响应队列
-# DONE = Queue()          # 下载完成队列
-# UPLOAD = Queue()        # 任务上传队列
-# DELETE = Queue()        # 任务删除队列
-
-SEED_QUEUE_SIZE = 100       # TODO 队列长度
-TODO_QUEUE_SIZE = 100       # TODO 队列长度
-REQUEST_QUEUE_SIZE = 100    # new队列长度
-DOWNLOAD_QUEUE_SIZE = 100   # done队列长度
-RESPONSE_QUEUE_SIZE = 100   # upload队列长度
-DONE_QUEUE_SIZE = 100       # upload队列长度
-UPLOAD_QUEUE_SIZE = 100     # upload队列长度
-DELETE_QUEUE_SIZE = 100     # upload队列长度
+TODO_QUEUE_SIZE = 100  # todo队列长度
+NEW_QUEUE_MAX_SIZE = 100  # new队列长度
+DONE_QUEUE_MAX_SIZE = 100  # done队列长度
+UPLOAD_QUEUE_MAX_SIZE = 100  # upload队列长度
 
 # DONE_MODEL IN (0, 1), 种子完成模式
 DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加至失败队列；1:种子消费成功添加至成功队列，失败添加至失败队列
@@ -81,6 +61,7 @@ DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加
 SPIDER_THREAD_NUM = 10
 SPIDER_MAX_RETRIES = 5
 SPIDER_TIME_SLEEP = 10
+RECORD_FAILED_SPIDER = False
 
 SPIDER_MAX_COUNT = 1000   # 在规定时间窗口内最大采集数
 TIME_WINDOW = 60   # 频控固定时间窗口（秒）
@@ -88,6 +69,8 @@ TIME_WINDOW = 60   # 频控固定时间窗口（秒）
 # 任务模式
 TASK_MODEL = 0  # 0:单次，1:常驻
 
+# 流控措施
+SPEED_CONTROL = 1  # 0:关闭，1:开启
 
 # bloom过滤器
 CAPACITY = 100000000

cobweb/utils/__init__.py

@@ -1,5 +1,6 @@
 from .oss import OssUtil
 from .tools import *
 from .bloom import BloomFilter
-# from .task_queue import TaskQueue
+from .dotting import LoghubDot
+from .decorators import check_pause
 

cobweb/utils/decorators.py

@@ -0,0 +1,43 @@
+import time
+from functools import wraps
+
+from cobweb.base import logger
+
+
+def decorator_oss_db(exception, retries=3):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(callback_func, *args, **kwargs):
+            result = None
+            for i in range(retries):
+                msg = None
+                try:
+                    return func(callback_func, *args, **kwargs)
+                except Exception as e:
+                    result = None
+                    msg = e
+                finally:
+                    if result:
+                        return result
+
+                    if i >= 2 and msg:
+                        raise exception(msg)
+
+        return wrapper
+
+    return decorator
+
+
+def check_pause(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        while not self.pause.is_set():
+            try:
+                func(self, *args, **kwargs)
+            except Exception as e:
+                logger.info(f"{func.__name__}: " + str(e))
+            finally:
+                time.sleep(0.1)
+        logger.info(f"pause: {func.__name__} thread close ...")
+
+    return wrapper

cobweb/utils/dotting.py

@@ -0,0 +1,55 @@
+import json
+import time
+
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+
+from cobweb.base import Queue, logger
+from cobweb import setting
+
+
+class LoghubDot:
+
+    def __init__(self):
+        self.client = LogClient(**setting.LOGHUB_CONFIG)
+        self.queue = Queue()
+
+    def build(self, topic, **kwargs):
+
+        temp = {}
+        log_item = LogItem()
+        for key, value in kwargs.items():
+            if not isinstance(value, str):
+                temp[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                temp[key] = value
+        contents = sorted(temp.items())
+        log_item.set_contents(contents)
+        self.queue.push((topic, log_item), direct_insertion=True)
+
+    def build_run(self):
+        while True:
+            start_time = int(time.time())
+            while True:
+                cost_time = int(time.time()) - start_time
+                if self.queue.length >= 1000 or cost_time > 10:
+                    break
+                time.sleep(0.5)
+            try:
+                log_item_info = {}
+                for _ in range(1000):
+                    its = self.queue.pop()
+                    if not its:
+                        break
+                    topic, item = its
+                    log_item_info.setdefault(topic, []).append(item)
+                for topic, log_items in log_item_info.items():
+                    request = PutLogsRequest(
+                        project="databee-download-log",
+                        logstore="log",
+                        topic=topic,
+                        logitems=log_items,
+                        compress=True
+                    )
+                    self.client.put_logs(request=request)
+            except Exception as e:
+                logger.info(str(e))