cobweb-launcher 1.2.49__py3-none-any.whl → 1.3.2__py3-none-any.whl

cobweb_new/launchers/launcher.py

@@ -0,0 +1,237 @@
+import time
+import inspect
+import threading
+import importlib
+
+from inspect import isgenerator
+from typing import Union, Callable
+
+from constant import DealModel, LogTemplate
+from cobweb.utils import dynamic_load_class
+from cobweb.base import Seed, Queue, logger, TaskQueue
+from cobweb import setting
+
+
+class Launcher(threading.Thread):
+
+    __CUSTOM_FUNC__ = {}
+
+    def __init__(self, task, project, custom_setting=None, **kwargs):
+        super().__init__()
+        self.task = task
+        self.project = project
+        self.custom_func = dict()
+        self.app_time = int(time.time())
+
+        _setting = dict()
+
+        if custom_setting:
+            if isinstance(custom_setting, dict):
+                _setting = custom_setting
+            else:
+                if isinstance(custom_setting, str):
+                    custom_setting = importlib.import_module(custom_setting)
+                if not inspect.ismodule(custom_setting):
+                    raise Exception
+                for k, v in custom_setting.__dict__.items():
+                    if not k.startswith("__") and not inspect.ismodule(v):
+                        _setting[k] = v
+
+        _setting.update(**kwargs)
+
+        for k, v in _setting.items():
+            setattr(setting, k.upper(), v)
+
+        self.scheduling_wait_time = setting.SCHEDULING_WAIT_TIME
+        self.inserting_wait_time = setting.INSERTING_WAIT_TIME
+        self.removing_wait_time = setting.REMOVING_WAIT_TIME
+
+        self.scheduling_size = setting.SCHEDULING_SIZE
+        self.inserting_size = setting.INSERTING_SIZE
+        self.removing_size = setting.REMOVING_SIZE
+
+        self.todo_queue_size = setting.TODO_QUEUE_SIZE
+        self.seed_queue_size = setting.SEED_QUEUE_SIZE
+        self.request_queue_size = setting.REQUEST_QUEUE_SIZE
+        self.download_queue_size = setting.DOWNLOAD_QUEUE_SIZE
+        self.response_queue_size = setting.RESPONSE_QUEUE_SIZE
+        self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
+        self.delete_queue_size = setting.DELETE_QUEUE_SIZE
+        self.done_queue_size = setting.DONE_QUEUE_SIZE
+
+        self.stop = threading.Event()  # 结束事件
+        self.pause = threading.Event()  # 暂停事件
+
+        self.crawler_path = setting.CRAWLER
+        self.pipeline_path = setting.PIPELINE
+
+        # self.crawler = None
+        # self.pipeline = None
+
+        self._threads = []
+
+        self._task_info = dict(todo={}, download={})
+
+        # ------
+
+        self.before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
+
+        self.todo_queue_full_wait_seconds = setting.TODO_QUEUE_FULL_WAIT_SECONDS
+        self.new_queue_wait_seconds = setting.NEW_QUEUE_WAIT_SECONDS
+        self.done_queue_wait_seconds = setting.DONE_QUEUE_WAIT_SECONDS
+        self.upload_queue_wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
+        self.seed_reset_seconds = setting.SEED_RESET_SECONDS
+
+        self.todo_queue_size = setting.TODO_QUEUE_SIZE
+        # self.new_queue_max_size = setting.NEW_QUEUE_MAX_SIZE
+        # self.done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
+        # self.upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
+
+        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
+        self.spider_thread_num = setting.SPIDER_THREAD_NUM
+        self.spider_time_sleep = setting.SPIDER_TIME_SLEEP
+        self.spider_max_count = setting.SPIDER_MAX_COUNT
+        self.time_window = setting.TIME_WINDOW
+
+        self.done_model = setting.DONE_MODEL
+        self.task_model = setting.TASK_MODEL
+
+        self.filter_field = setting.FILTER_FIELD
+
+    @staticmethod
+    def insert_seed(seed: Union[Seed, dict]):
+        if isinstance(seed, dict):
+            seed = Seed(seed)
+        TaskQueue.SEED.push(seed)
+
+    @property
+    def request(self):
+        """
+        自定义request函数
+        use case:
+            from cobweb.base import Request, BaseItem
+            @launcher.request
+            def request(seed: Seed) -> Union[Request, BaseItem]:
+                ...
+                yield Request(seed.url, seed)
+        """
+        def decorator(func):
+            self.custom_func['request'] = func
+        return decorator
+
+    @property
+    def download(self):
+        """
+        自定义download函数
+        use case:
+            from cobweb.base import Request, Response, Seed, BaseItem
+            @launcher.download
+            def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+                ...
+                yield Response(item.seed, response)
+        """
+        def decorator(func):
+            self.custom_func['download'] = func
+        return decorator
+
+    @property
+    def parse(self):
+        """
+        自定义parse函数, xxxItem为自定义的存储数据类型
+        use case:
+            from cobweb.base import Request, Response
+            @launcher.parse
+            def parse(item: Response) -> BaseItem:
+               ...
+               yield xxxItem(seed, **kwargs)
+        """
+        def decorator(func):
+            self.custom_func['parse'] = func
+        return decorator
+
+    def remove_working_items(self, key, items):
+        for item in items:
+            self._task_info[key].pop(item, None)
+
+    def check_alive(self):
+        while not self.stop.is_set():
+            if not self.pause.is_set():
+                for thread in self._threads:
+                    if not thread.is_alive():
+                        thread.start()
+            time.sleep(1)
+
+    def _add_thread(self, func, num=1, obj=None, name=None, args=()):
+        obj = obj or self
+        name = obj.__class__.__name__ + name or func.__name__
+        for i in range(num):
+            func_name = name + "_" + str(i) if num > 1 else name
+            self._threads.append(threading.Thread(name=func_name, target=func, args=(obj,) + args))
+
+    def _init_schedule_thread(self):
+        ...
+
+    def _polling(self):
+        check_emtpy_times = 0
+        while not self.stop.is_set():
+            if TaskQueue.is_empty():
+                if self.pause.is_set():
+                    run_time = int(time.time()) - self.app_time
+                    if not self.task_model and run_time > self.before_scheduler_wait_seconds:
+                        logger.info("Done! ready to close thread...")
+                        self.stop.set()
+                    else:
+                        logger.info("pause! waiting for resume...")
+                elif check_emtpy_times > 2:
+                    logger.info("pause! waiting for resume...")
+                    self.doing_seeds = {}
+                    self.pause.set()
+                else:
+                    logger.info(
+                        "check whether the task is complete, "
+                        f"reset times {3 - check_emtpy_times}"
+                    )
+                    check_emtpy_times += 1
+            elif TaskQueue.TODO.length:
+                logger.info(f"Recovery {self.task} task run！")
+                check_emtpy_times = 0
+                self.pause.clear()
+            else:
+                logger.info(LogTemplate.launcher_polling.format(
+                    task=self.task,
+                    doing_len=len(self.doing_seeds.keys()),
+                    todo_len=TaskQueue.TODO.length,
+                    done_len=TaskQueue.DONE.length,
+                    upload_len=TaskQueue.UPLOAD.length,
+                ))
+
+            time.sleep(10)
+
+        logger.info("Done! Ready to close thread...")
+
+    def run(self):
+        Crawler = dynamic_load_class(self.crawler_path)
+        Pipeline = dynamic_load_class(self.pipeline_path)
+
+        crawler = Crawler(
+            stop=self.stop, pause=self.pause,
+            thread_num=self.spider_thread_num,
+            time_sleep=self.spider_time_sleep,
+            custom_func=self.custom_func
+        )
+
+        pipeline = Pipeline(
+            stop=self.stop, pause=self.pause,
+            upload_size=self.upload_queue_max_size,
+            wait_seconds=self.upload_queue_wait_seconds
+        )
+
+        self._add_thread(obj=crawler, func=crawler.build_request_item)
+        self._add_thread(obj=crawler, func=crawler.build_download_item, num=self.spider_thread_num)
+        self._add_thread(obj=crawler, func=crawler.build_parse_item)
+
+        self._init_schedule_thread()
+        self.check_alive()
+
+
+

cobweb_new/launchers/launcher_air.py

@@ -0,0 +1,88 @@
+import time
+
+from cobweb.base import logger
+from cobweb.constant import LogTemplate
+from .launcher import Launcher, check_pause
+
+
+class LauncherAir(Launcher):
+
+    # def _scheduler(self):
+    #     if self.start_seeds:
+    #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
+
+    @check_pause
+    def _insert(self):
+        seeds = {}
+        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
+        for _ in range(self._new_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['new'].pop()
+            if not seed:
+                break
+            seeds[seed.to_string] = seed.params.priority
+        if seeds:
+            self.__LAUNCHER_QUEUE__['todo'].push(seeds)
+        if status:
+            time.sleep(self._new_queue_wait_seconds)
+
+    @check_pause
+    def _delete(self):
+        seeds = []
+        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
+
+        for _ in range(self._done_queue_max_size):
+            seed = self.__LAUNCHER_QUEUE__['done'].pop()
+            if not seed:
+                break
+            seeds.append(seed.to_string)
+
+        if seeds:
+            self._remove_doing_seeds(seeds)
+
+        if status:
+            time.sleep(self._done_queue_wait_seconds)
+
+    def _polling(self):
+
+        check_emtpy_times = 0
+
+        while not self._stop.is_set():
+
+            queue_not_empty_count = 0
+            pooling_wait_seconds = 30
+
+            for q in self.__LAUNCHER_QUEUE__.values():
+                if q.length != 0:
+                    queue_not_empty_count += 1
+
+            if queue_not_empty_count == 0:
+                pooling_wait_seconds = 3
+                if self._pause.is_set():
+                    check_emtpy_times = 0
+                    if not self._task_model:
+                        logger.info("Done! Ready to close thread...")
+                        self._stop.set()
+                elif check_emtpy_times > 2:
+                    self.__DOING__ = {}
+                    self._pause.set()
+                else:
+                    logger.info(
+                        "check whether the task is complete, "
+                        f"reset times {3 - check_emtpy_times}"
+                    )
+                    check_emtpy_times += 1
+            elif self._pause.is_set():
+                self._pause.clear()
+                self._execute()
+            else:
+                logger.info(LogTemplate.launcher_air_polling.format(
+                    task=self.task,
+                    doing_len=len(self.__DOING__.keys()),
+                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
+                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
+                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
+                ))
+
+            time.sleep(pooling_wait_seconds)
+
+

cobweb_new/launchers/launcher_api.py

@@ -0,0 +1,161 @@
+import time
+import threading
+
+from cobweb.db import ApiDB
+from cobweb.base import Seed, TaskQueue,logger, stop, pause
+from cobweb.constant import DealModel
+from .launcher import Launcher
+
+
+class LauncherApi(Launcher):
+
+    def __init__(self, task, project, custom_setting=None, **kwargs):
+        super().__init__(task, project, custom_setting, **kwargs)
+        self._db = ApiDB()
+
+        self._todo_key = "{%s:%s}:todo" % (project, task)
+        self._done_key = "{%s:%s}:done" % (project, task)
+        self._fail_key = "{%s:%s}:fail" % (project, task)
+        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
+
+        self._statistics_done_key = "statistics:%s:%s:done" % (project, task)
+        self._statistics_fail_key = "statistics:%s:%s:fail" % (project, task)
+        self._speed_control_key = "speed_control:%s_%s" % (project, task)
+
+        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+
+        self._heartbeat_start_event = threading.Event()
+
+    @property
+    def heartbeat(self):
+        return self._db.exists(self._heartbeat_key)
+
+    def statistics(self, key, count):
+        if not self.task_model and not self._db.exists(key):
+            self._db.setex(key, 86400 * 30, int(count))
+        else:
+            self._db.incrby(key, count)
+
+    def _get_seed(self) -> Seed:
+        """
+        从队列中获取种子（频控）
+        设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
+        :return: True -> 种子, False -> None
+        """
+        if TaskQueue.TODO.length and not self._db.auto_incr(
+                self._speed_control_key, 
+                t=self.time_window,
+                limit=self.spider_max_count
+        ):
+            expire_time = self._db.ttl(self._speed_control_key)
+            logger.info(f"Too fast! Please wait {expire_time} seconds...")
+            time.sleep(expire_time / 2)
+            return None
+        return TaskQueue.TODO.pop()
+
+    @stop
+    def _reset(self):
+        """
+        检查过期种子，重新添加到redis缓存中
+        """
+        if self._db.lock(self._reset_lock_key, t=120):
+
+            _min = -int(time.time()) + self.seed_reset_seconds \
+                if self.heartbeat else "-inf"
+
+            self._db.members(self._todo_key, 0, _min=_min, _max="(0")
+
+            if not self.heartbeat:
+                self._heartbeat_start_event.set()
+
+            self._db.delete(self._reset_lock_key)
+
+        time.sleep(30)
+
+    @stop
+    def _refresh(self):
+        """
+        刷新doing种子过期时间，防止reset重新消费
+        """
+        if self.doing_seeds:
+            refresh_time = int(time.time())
+            seeds = {k: -refresh_time - v / 1e3 for k, v in self.doing_seeds.items()}
+            self._db.zadd(self._todo_key, item=seeds, xx=True)
+        time.sleep(3)
+
+    @stop
+    def _scheduler(self):
+        """
+        调度任务，获取redis队列种子，同时添加到doing字典中
+        """
+        if not self._db.zcount(self._todo_key, 0, "(1000"):
+            time.sleep(self.scheduler_wait_seconds)
+        elif TaskQueue.TODO.length >= self.todo_queue_size:
+            time.sleep(self.todo_queue_full_wait_seconds)
+        else:
+            members = self._db.members(
+                self._todo_key, int(time.time()),
+                count=self.todo_queue_size,
+                _min=0, _max="(1000"
+            )
+            for member, priority in members:
+                seed = Seed(member, priority=priority)
+                TaskQueue.TODO.push(seed)
+                self.doing_seeds[seed.to_string] = seed.params.priority
+
+    @pause
+    def _heartbeat(self):
+        if self._heartbeat_start_event.is_set():
+            self._db.setex(self._heartbeat_key, t=5)
+        time.sleep(3)
+
+    @pause
+    def _insert(self):
+        """
+        添加新种子到redis队列中
+        """
+        seeds = {}
+        for _ in range(self.new_queue_max_size):
+            if seed := TaskQueue.SEED.pop():
+                seeds[seed.to_string] = seed.params.priority
+        if seeds:
+            self._db.zadd(self._todo_key, seeds, nx=True)
+        if TaskQueue.SEED.length < self.new_queue_max_size:
+            time.sleep(self.new_queue_wait_seconds)
+
+    @pause
+    def _delete(self):
+        """
+        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
+        """
+        seed_info = {"count": 0, "failed": [], "succeed": [], "common": []}
+        status = TaskQueue.DONE.length < self.done_queue_max_size
+
+        for _ in range(self.done_queue_max_size):
+            seed = TaskQueue.DONE.pop()
+            if not seed:
+                break
+            if seed.params.seed_status == DealModel.fail:
+                seed_info["failed"].append(seed.to_string)
+            elif self.done_model == 1:
+                seed_info["succeed"].append(seed.to_string)
+            else:
+                seed_info["common"].append(seed.to_string)
+            seed_info['count'] += 1
+
+        if seed_info["count"]:
+
+            succeed_count = int(self._db.zrem(self._todo_key, *seed_info["common"]) or 0)
+            succeed_count += int(self._db.done([self._todo_key, self._done_key], *seed_info["succeed"]) or 0)
+            failed_count = int(self._db.done([self._todo_key, self._fail_key], *seed_info["failed"]) or 0)
+
+            if failed_count:
+                self.statistics(self._statistics_fail_key, failed_count)
+            if succeed_count:
+                self.statistics(self._statistics_done_key, succeed_count)
+
+            self._remove_doing_seeds(seed_info["common"] + seed_info["succeed"] + seed_info["failed"])
+
+        if status:
+            time.sleep(self.done_queue_wait_seconds)
+

cobweb_new/launchers/launcher_pro.py

@@ -0,0 +1,96 @@
+import time
+
+from base import TaskQueue
+from cobweb.base import decorators
+from schedulers.scheduler_redis import RedisScheduler
+from .launcher import Launcher
+
+
+class LauncherPro(Launcher):
+
+    def __init__(self, task, project, custom_setting=None, **kwargs):
+        super().__init__(task, project, custom_setting, **kwargs)
+        self._redis_download = "{%s:%s}:download" % (project, task)
+        self._redis_todo = "{%s:%s}:todo" % (project, task)
+        self._scheduler = RedisScheduler(task, project)
+
+    # @decorators.add_thread()
+    @decorators.stop
+    def _schedule(self):
+        thread_sleep = self.scheduling_wait_time
+        for q, key, size in [
+            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size),
+            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size),
+        ]:
+            if q.length < size:
+                for item in self._scheduler.schedule(
+                    key, self.scheduling_size
+                ):
+                    q.push(item)
+                thread_sleep = 0.1
+        time.sleep(thread_sleep)
+
+    # @decorators.add_thread()
+    @decorators.pause
+    def _heartbeat(self):
+        if self._scheduler.working.is_set():
+            self._scheduler.set_heartbeat()
+        time.sleep(3)
+
+    # @decorators.add_thread()
+    @decorators.pause
+    def _reset(self):
+        self._scheduler.reset(
+            keys=[self._redis_todo, self._redis_download],
+            reset_time=self.seed_reset_seconds
+        )
+        time.sleep(15)
+
+    # @decorators.add_thread()
+    @decorators.pause
+    def _insert(self):
+        thread_sleep = 0.1
+        for q, key, size in [
+            (TaskQueue.SEED, self._redis_todo, self.seed_queue_size),
+            (TaskQueue.REQUEST, self._redis_download, self.request_queue_size),
+        ]:
+            items = {}
+            while item := q.pop() and len(items.keys()) < self.inserting_size:
+                items[item.to_string] = item.params.priority
+            if q.length >= size:
+                thread_sleep = self.inserting_wait_time
+            self._scheduler.insert(key, items)
+        time.sleep(thread_sleep)
+
+    # @decorators.add_thread()
+    @decorators.pause
+    def _refresh(self):
+        self._scheduler.refresh(self._redis_todo, self._task_info["todo"])
+        self._scheduler.refresh(self._redis_download, self._task_info["download"])
+        time.sleep(3)
+
+    # @decorators.add_thread()
+    @decorators.pause
+    def _remove(self):
+        thread_sleep = self.removing_wait_time
+        for q, key, size in [
+            (TaskQueue.DELETE, self._redis_todo, self.delete_queue_size),
+            (TaskQueue.DONE, self._redis_download, self.done_queue_size),
+        ]:
+            items = []
+            while item := q.pop() and len(items) < self.removing_size:
+                items.append(item)
+            self._scheduler.delete(key, *items)
+            self.remove_working_items(key.split(":")[-1], items)
+            if q.length >= size:
+                thread_sleep = 0.1
+        time.sleep(thread_sleep)
+
+    def _init_schedule_thread(self):
+        self._add_thread(func=self._heartbeat)
+        self._add_thread(func=self._reset)
+        self._add_thread(func=self._refresh)
+        self._add_thread(func=self._schedule)
+        self._add_thread(func=self._insert)
+        self._add_thread(func=self._remove)
+        self._add_thread(func=self._polling)

cobweb_new/launchers/tesss.py

@@ -0,0 +1,47 @@
+import threading
+import time
+from functools import wraps
+
+
+def add_thread(num=1):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(self, *args):
+            for i in range(num):
+                name = func.__name__ + "_" + str(i) if num > 1 else func.__name__
+                self._threads.append(threading.Thread(name=name, target=func, args=(self,) + args))
+        return wrapper
+
+    return decorator
+
+
+def pause(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        while True:
+            try:
+                func(*args, **kwargs)
+            except Exception as e:
+                print(str(e))
+            finally:
+                time.sleep(0.1)
+
+    return wrapper
+
+
+class TTT:
+    _threads = []
+
+    @add_thread()
+    @pause
+    def tt(self):
+        print("hello")
+        time.sleep(1)
+
+tttt = TTT()
+tttt.tt()
+print(TTT._threads)
+
+
+for _ in TTT._threads:
+    _.start()

cobweb_new/pipelines/__init__.py

@@ -0,0 +1,3 @@
+from .pipeline import Pipeline
+from .pipeline_console import Console
+from .pipeline_loghub import Loghub

cobweb_new/pipelines/pipeline.py

@@ -0,0 +1,68 @@
+import time
+import threading
+
+from abc import ABC, abstractmethod
+
+from cobweb.utils import TaskQueue
+from cobweb.base import BaseItem, logger
+
+
+class Pipeline(threading.Thread, ABC):
+
+    def __init__(
+            self,
+            stop: threading.Event,
+            pause: threading.Event,
+            upload_size: int,
+            wait_seconds: int
+    ):
+        super().__init__()
+        self._stop = stop
+        self._pause = pause
+
+        self.upload_size = upload_size
+        self.wait_seconds = wait_seconds
+
+    @abstractmethod
+    def build(self, item: BaseItem) -> dict:
+        pass
+
+    @abstractmethod
+    def upload(self, table: str, data: list) -> bool:
+        pass
+
+    def run(self):
+        while not self._stop.is_set():
+            if not TaskQueue.UPLOAD.length:
+                time.sleep(self.wait_seconds)
+                continue
+            if TaskQueue.UPLOAD.length < self.upload_size:
+                time.sleep(self.wait_seconds)
+            status = True
+            data_info, seeds = {}, []
+            try:
+                for _ in range(self.upload_size):
+                    item = TaskQueue.UPLOAD.pop()
+                    if not item:
+                        break
+                    seeds.append(item.seed)
+                    data = self.build(item)
+                    data_info.setdefault(item.table, []).append(data)
+                for table, datas in data_info.items():
+                    try:
+                        self.upload(table, datas)
+                    except Exception as e:
+                        logger.info(e)
+                        status = False
+            except Exception as e:
+                logger.info(e)
+                status = False
+            if not status:
+                for seed in seeds:
+                    seed.params.seed_status = "deal model: fail"
+            if seeds:
+                TaskQueue.DONE.push(seeds)
+
+        logger.info("upload pipeline close!")
+
+