cobweb-launcher 1.3.6__py3-none-any.whl → 1.3.7__py3-none-any.whl

cobweb_new/db/redis_db.py

@@ -1,158 +0,0 @@
-import redis
-from cobweb import setting
-
-
-class RedisDB:
-
-    def __init__(self, **kwargs):
-        redis_config = kwargs or setting.REDIS_CONFIG
-        pool = redis.ConnectionPool(**redis_config)
-        self._client = redis.Redis(connection_pool=pool)
-
-    def setnx(self, name, value=""):
-        return self._client.setnx(name, value)
-
-    def setex(self, name, t, value=""):
-        return self._client.setex(name, t, value)
-
-    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
-        return self._client.expire(name, t, nx, xx, gt, lt)
-
-    def ttl(self, name):
-        return self._client.ttl(name)
-
-    def delete(self, name):
-        return self._client.delete(name)
-
-    def exists(self, *name) -> bool:
-        return self._client.exists(*name)
-
-    def incrby(self, name, value):
-        return self._client.incrby(name, value)
-
-    def sadd(self, name, value):
-        return self._client.sadd(name, value)
-
-    def zcard(self, name) -> bool:
-        return self._client.zcard(name)
-
-    def zadd(self, name, item: dict, **kwargs):
-        return self._client.zadd(name, item, **kwargs)
-
-    def zrem(self, name, *value):
-        return self._client.zrem(name, *value)
-
-    def zcount(self, name, _min, _max):
-        return self._client.zcount(name, _min, _max)
-
-    # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
-    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
-
-    def lua(self, script: str, keys: list = None, args: list = None):
-        keys = keys or []
-        args = args or []
-        keys_count = len(keys)
-        return self._client.eval(script, keys_count, *keys, *args)
-
-    def lua_sha(self, sha1: str, keys: list = None, args: list = None):
-        keys = keys or []
-        args = args or []
-        keys_count = len(keys)
-        return self._client.evalsha(sha1, keys_count, *keys, *args)
-
-    def execute_lua(self, lua_script: str, keys: list, *args):
-        execute = self._client.register_script(lua_script)
-        return execute(keys=keys, args=args)
-
-    def lock(self, key, t=15) -> bool:
-        lua_script = """
-        local status = redis.call('setnx', KEYS[1], 1)
-        if ( status == 1 ) then
-            redis.call('expire', KEYS[1], ARGV[1])
-        end 
-        return status 
-        """
-        status = self.execute_lua(lua_script, [key], t)
-        return bool(status)
-
-    def auto_incr(self, name, t=15, limit=1000):
-        lua_script = """
-            local count = 0
-            local status = false
-            local limit = ARGV[2]
-            local expire = redis.call('ttl', KEYS[1])
-
-            if ( expire == -2 ) then
-                redis.call('setnx', KEYS[1], 1)
-            elseif ( expire == -1) then
-                redis.call('expire', KEYS[1], ARGV[1])
-            else
-                count = redis.call('get', KEYS[1])
-            end
-
-            if ( count + 0 < limit + 0 ) then
-                status = true
-                redis.call('incr', KEYS[1])
-            end
-
-            return status
-        """
-        status = self.execute_lua(lua_script, [name], t, limit)
-        return bool(status)
-
-    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
-        lua_script = """
-        local min = ARGV[1]
-        local max = ARGV[2]
-        local start = ARGV[3]
-        local count = ARGV[4]
-        local score = ARGV[5]
-        local members = nil
-
-        if ( type(count) == string ) then
-            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
-        else
-            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
-        end
-        
-        local result = {}
-
-        for i = 1, #members, 2 do
-            local priority = nil
-            local member = members[i]
-            local originPriority = nil
-            if ( members[i+1] + 0 < 0 ) then
-                originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
-            else
-                originPriority = math.floor(members[i+1])
-            end
-            
-            if ( score + 0 >= 1000 ) then
-                priority = -score - originPriority / 1000
-            elseif ( score + 0 == 0 ) then
-                priority = originPriority
-            else
-                originPriority = score 
-                priority = score
-            end
-            redis.call('zadd', KEYS[1], priority, member)
-            table.insert(result, member)
-            table.insert(result, originPriority)
-        end
-
-        return result
-        """
-        members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
-        return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
-
-    def done(self, keys: list, *args) -> list:
-        lua_script = """
-        for i, member in ipairs(ARGV) do
-            redis.call("zrem", KEYS[1], member)
-            redis.call("sadd", KEYS[2], member)
-        end
-        """
-        self.execute_lua(lua_script, keys, *args)
-
-
-

cobweb_new/exceptions/__init__.py

@@ -1 +0,0 @@
-from .oss_db_exception import *

cobweb_new/exceptions/oss_db_exception.py

@@ -1,28 +0,0 @@
-class OssDBException(Exception):
-    """Base oss client exception that all others inherit."""
-
-
-class OssDBMergeError(OssDBException):
-    """
-    Exception raised when execute merge operation fails.
-    """
-
-
-class OssDBPutPartError(OssDBException):
-    """
-    Exception raised when upload part operation fails.
-    """
-
-
-class OssDBPutObjError(OssDBException):
-    """
-    Exception raised when upload operation fails.
-    """
-
-
-class OssDBAppendObjError(OssDBException):
-    """Exception raised when upload operation fails."""
-
-
-class OssDBInitPartError(OssDBException):
-    """Exception raised when init upload operation fails."""

cobweb_new/launchers/__init__.py

@@ -1,3 +0,0 @@
-from .launcher_air import LauncherAir
-from .launcher_pro import LauncherPro
-from .launcher_api import LauncherApi

cobweb_new/launchers/launcher.py

@@ -1,237 +0,0 @@
-import time
-import inspect
-import threading
-import importlib
-
-from inspect import isgenerator
-from typing import Union, Callable
-
-from constant import DealModel, LogTemplate
-from cobweb.utils import dynamic_load_class
-from cobweb.base import Seed, Queue, logger, TaskQueue
-from cobweb import setting
-
-
-class Launcher(threading.Thread):
-
-    __CUSTOM_FUNC__ = {}
-
-    def __init__(self, task, project, custom_setting=None, **kwargs):
-        super().__init__()
-        self.task = task
-        self.project = project
-        self.custom_func = dict()
-        self.app_time = int(time.time())
-
-        _setting = dict()
-
-        if custom_setting:
-            if isinstance(custom_setting, dict):
-                _setting = custom_setting
-            else:
-                if isinstance(custom_setting, str):
-                    custom_setting = importlib.import_module(custom_setting)
-                if not inspect.ismodule(custom_setting):
-                    raise Exception
-                for k, v in custom_setting.__dict__.items():
-                    if not k.startswith("__") and not inspect.ismodule(v):
-                        _setting[k] = v
-
-        _setting.update(**kwargs)
-
-        for k, v in _setting.items():
-            setattr(setting, k.upper(), v)
-
-        self.scheduling_wait_time = setting.SCHEDULING_WAIT_TIME
-        self.inserting_wait_time = setting.INSERTING_WAIT_TIME
-        self.removing_wait_time = setting.REMOVING_WAIT_TIME
-
-        self.scheduling_size = setting.SCHEDULING_SIZE
-        self.inserting_size = setting.INSERTING_SIZE
-        self.removing_size = setting.REMOVING_SIZE
-
-        self.todo_queue_size = setting.TODO_QUEUE_SIZE
-        self.seed_queue_size = setting.SEED_QUEUE_SIZE
-        self.request_queue_size = setting.REQUEST_QUEUE_SIZE
-        self.download_queue_size = setting.DOWNLOAD_QUEUE_SIZE
-        self.response_queue_size = setting.RESPONSE_QUEUE_SIZE
-        self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
-        self.delete_queue_size = setting.DELETE_QUEUE_SIZE
-        self.done_queue_size = setting.DONE_QUEUE_SIZE
-
-        self.stop = threading.Event()  # 结束事件
-        self.pause = threading.Event()  # 暂停事件
-
-        self.crawler_path = setting.CRAWLER
-        self.pipeline_path = setting.PIPELINE
-
-        # self.crawler = None
-        # self.pipeline = None
-
-        self._threads = []
-
-        self._task_info = dict(todo={}, download={})
-
-        # ------
-
-        self.before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
-
-        self.todo_queue_full_wait_seconds = setting.TODO_QUEUE_FULL_WAIT_SECONDS
-        self.new_queue_wait_seconds = setting.NEW_QUEUE_WAIT_SECONDS
-        self.done_queue_wait_seconds = setting.DONE_QUEUE_WAIT_SECONDS
-        self.upload_queue_wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
-        self.seed_reset_seconds = setting.SEED_RESET_SECONDS
-
-        self.todo_queue_size = setting.TODO_QUEUE_SIZE
-        # self.new_queue_max_size = setting.NEW_QUEUE_MAX_SIZE
-        # self.done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
-        # self.upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
-
-        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
-        self.spider_thread_num = setting.SPIDER_THREAD_NUM
-        self.spider_time_sleep = setting.SPIDER_TIME_SLEEP
-        self.spider_max_count = setting.SPIDER_MAX_COUNT
-        self.time_window = setting.TIME_WINDOW
-
-        self.done_model = setting.DONE_MODEL
-        self.task_model = setting.TASK_MODEL
-
-        self.filter_field = setting.FILTER_FIELD
-
-    @staticmethod
-    def insert_seed(seed: Union[Seed, dict]):
-        if isinstance(seed, dict):
-            seed = Seed(seed)
-        TaskQueue.SEED.push(seed)
-
-    @property
-    def request(self):
-        """
-        自定义request函数
-        use case:
-            from cobweb.base import Request, BaseItem
-            @launcher.request
-            def request(seed: Seed) -> Union[Request, BaseItem]:
-                ...
-                yield Request(seed.url, seed)
-        """
-        def decorator(func):
-            self.custom_func['request'] = func
-        return decorator
-
-    @property
-    def download(self):
-        """
-        自定义download函数
-        use case:
-            from cobweb.base import Request, Response, Seed, BaseItem
-            @launcher.download
-            def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-                ...
-                yield Response(item.seed, response)
-        """
-        def decorator(func):
-            self.custom_func['download'] = func
-        return decorator
-
-    @property
-    def parse(self):
-        """
-        自定义parse函数, xxxItem为自定义的存储数据类型
-        use case:
-            from cobweb.base import Request, Response
-            @launcher.parse
-            def parse(item: Response) -> BaseItem:
-               ...
-               yield xxxItem(seed, **kwargs)
-        """
-        def decorator(func):
-            self.custom_func['parse'] = func
-        return decorator
-
-    def remove_working_items(self, key, items):
-        for item in items:
-            self._task_info[key].pop(item, None)
-
-    def check_alive(self):
-        while not self.stop.is_set():
-            if not self.pause.is_set():
-                for thread in self._threads:
-                    if not thread.is_alive():
-                        thread.start()
-            time.sleep(1)
-
-    def _add_thread(self, func, num=1, obj=None, name=None, args=()):
-        obj = obj or self
-        name = obj.__class__.__name__ + name or func.__name__
-        for i in range(num):
-            func_name = name + "_" + str(i) if num > 1 else name
-            self._threads.append(threading.Thread(name=func_name, target=func, args=(obj,) + args))
-
-    def _init_schedule_thread(self):
-        ...
-
-    def _polling(self):
-        check_emtpy_times = 0
-        while not self.stop.is_set():
-            if TaskQueue.is_empty():
-                if self.pause.is_set():
-                    run_time = int(time.time()) - self.app_time
-                    if not self.task_model and run_time > self.before_scheduler_wait_seconds:
-                        logger.info("Done! ready to close thread...")
-                        self.stop.set()
-                    else:
-                        logger.info("pause! waiting for resume...")
-                elif check_emtpy_times > 2:
-                    logger.info("pause! waiting for resume...")
-                    self.doing_seeds = {}
-                    self.pause.set()
-                else:
-                    logger.info(
-                        "check whether the task is complete, "
-                        f"reset times {3 - check_emtpy_times}"
-                    )
-                    check_emtpy_times += 1
-            elif TaskQueue.TODO.length:
-                logger.info(f"Recovery {self.task} task run！")
-                check_emtpy_times = 0
-                self.pause.clear()
-            else:
-                logger.info(LogTemplate.launcher_polling.format(
-                    task=self.task,
-                    doing_len=len(self.doing_seeds.keys()),
-                    todo_len=TaskQueue.TODO.length,
-                    done_len=TaskQueue.DONE.length,
-                    upload_len=TaskQueue.UPLOAD.length,
-                ))
-
-            time.sleep(10)
-
-        logger.info("Done! Ready to close thread...")
-
-    def run(self):
-        Crawler = dynamic_load_class(self.crawler_path)
-        Pipeline = dynamic_load_class(self.pipeline_path)
-
-        crawler = Crawler(
-            stop=self.stop, pause=self.pause,
-            thread_num=self.spider_thread_num,
-            time_sleep=self.spider_time_sleep,
-            custom_func=self.custom_func
-        )
-
-        pipeline = Pipeline(
-            stop=self.stop, pause=self.pause,
-            upload_size=self.upload_queue_max_size,
-            wait_seconds=self.upload_queue_wait_seconds
-        )
-
-        self._add_thread(obj=crawler, func=crawler.build_request_item)
-        self._add_thread(obj=crawler, func=crawler.build_download_item, num=self.spider_thread_num)
-        self._add_thread(obj=crawler, func=crawler.build_parse_item)
-
-        self._init_schedule_thread()
-        self.check_alive()
-
-
-

cobweb_new/launchers/launcher_air.py

@@ -1,88 +0,0 @@
-import time
-
-from cobweb.base import logger
-from cobweb.constant import LogTemplate
-from .launcher import Launcher, check_pause
-
-
-class LauncherAir(Launcher):
-
-    # def _scheduler(self):
-    #     if self.start_seeds:
-    #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
-
-    @check_pause
-    def _insert(self):
-        seeds = {}
-        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-        for _ in range(self._new_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['new'].pop()
-            if not seed:
-                break
-            seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(seeds)
-        if status:
-            time.sleep(self._new_queue_wait_seconds)
-
-    @check_pause
-    def _delete(self):
-        seeds = []
-        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-
-        for _ in range(self._done_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['done'].pop()
-            if not seed:
-                break
-            seeds.append(seed.to_string)
-
-        if seeds:
-            self._remove_doing_seeds(seeds)
-
-        if status:
-            time.sleep(self._done_queue_wait_seconds)
-
-    def _polling(self):
-
-        check_emtpy_times = 0
-
-        while not self._stop.is_set():
-
-            queue_not_empty_count = 0
-            pooling_wait_seconds = 30
-
-            for q in self.__LAUNCHER_QUEUE__.values():
-                if q.length != 0:
-                    queue_not_empty_count += 1
-
-            if queue_not_empty_count == 0:
-                pooling_wait_seconds = 3
-                if self._pause.is_set():
-                    check_emtpy_times = 0
-                    if not self._task_model:
-                        logger.info("Done! Ready to close thread...")
-                        self._stop.set()
-                elif check_emtpy_times > 2:
-                    self.__DOING__ = {}
-                    self._pause.set()
-                else:
-                    logger.info(
-                        "check whether the task is complete, "
-                        f"reset times {3 - check_emtpy_times}"
-                    )
-                    check_emtpy_times += 1
-            elif self._pause.is_set():
-                self._pause.clear()
-                self._execute()
-            else:
-                logger.info(LogTemplate.launcher_air_polling.format(
-                    task=self.task,
-                    doing_len=len(self.__DOING__.keys()),
-                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-                ))
-
-            time.sleep(pooling_wait_seconds)
-
-

cobweb_new/launchers/launcher_api.py

@@ -1,161 +0,0 @@
-import time
-import threading
-
-from cobweb.db import ApiDB
-from cobweb.base import Seed, TaskQueue,logger, stop, pause
-from cobweb.constant import DealModel
-from .launcher import Launcher
-
-
-class LauncherApi(Launcher):
-
-    def __init__(self, task, project, custom_setting=None, **kwargs):
-        super().__init__(task, project, custom_setting, **kwargs)
-        self._db = ApiDB()
-
-        self._todo_key = "{%s:%s}:todo" % (project, task)
-        self._done_key = "{%s:%s}:done" % (project, task)
-        self._fail_key = "{%s:%s}:fail" % (project, task)
-        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
-
-        self._statistics_done_key = "statistics:%s:%s:done" % (project, task)
-        self._statistics_fail_key = "statistics:%s:%s:fail" % (project, task)
-        self._speed_control_key = "speed_control:%s_%s" % (project, task)
-
-        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
-
-        self._heartbeat_start_event = threading.Event()
-
-    @property
-    def heartbeat(self):
-        return self._db.exists(self._heartbeat_key)
-
-    def statistics(self, key, count):
-        if not self.task_model and not self._db.exists(key):
-            self._db.setex(key, 86400 * 30, int(count))
-        else:
-            self._db.incrby(key, count)
-
-    def _get_seed(self) -> Seed:
-        """
-        从队列中获取种子（频控）
-        设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
-        :return: True -> 种子, False -> None
-        """
-        if TaskQueue.TODO.length and not self._db.auto_incr(
-                self._speed_control_key, 
-                t=self.time_window,
-                limit=self.spider_max_count
-        ):
-            expire_time = self._db.ttl(self._speed_control_key)
-            logger.info(f"Too fast! Please wait {expire_time} seconds...")
-            time.sleep(expire_time / 2)
-            return None
-        return TaskQueue.TODO.pop()
-
-    @stop
-    def _reset(self):
-        """
-        检查过期种子，重新添加到redis缓存中
-        """
-        if self._db.lock(self._reset_lock_key, t=120):
-
-            _min = -int(time.time()) + self.seed_reset_seconds \
-                if self.heartbeat else "-inf"
-
-            self._db.members(self._todo_key, 0, _min=_min, _max="(0")
-
-            if not self.heartbeat:
-                self._heartbeat_start_event.set()
-
-            self._db.delete(self._reset_lock_key)
-
-        time.sleep(30)
-
-    @stop
-    def _refresh(self):
-        """
-        刷新doing种子过期时间，防止reset重新消费
-        """
-        if self.doing_seeds:
-            refresh_time = int(time.time())
-            seeds = {k: -refresh_time - v / 1e3 for k, v in self.doing_seeds.items()}
-            self._db.zadd(self._todo_key, item=seeds, xx=True)
-        time.sleep(3)
-
-    @stop
-    def _scheduler(self):
-        """
-        调度任务，获取redis队列种子，同时添加到doing字典中
-        """
-        if not self._db.zcount(self._todo_key, 0, "(1000"):
-            time.sleep(self.scheduler_wait_seconds)
-        elif TaskQueue.TODO.length >= self.todo_queue_size:
-            time.sleep(self.todo_queue_full_wait_seconds)
-        else:
-            members = self._db.members(
-                self._todo_key, int(time.time()),
-                count=self.todo_queue_size,
-                _min=0, _max="(1000"
-            )
-            for member, priority in members:
-                seed = Seed(member, priority=priority)
-                TaskQueue.TODO.push(seed)
-                self.doing_seeds[seed.to_string] = seed.params.priority
-
-    @pause
-    def _heartbeat(self):
-        if self._heartbeat_start_event.is_set():
-            self._db.setex(self._heartbeat_key, t=5)
-        time.sleep(3)
-
-    @pause
-    def _insert(self):
-        """
-        添加新种子到redis队列中
-        """
-        seeds = {}
-        for _ in range(self.new_queue_max_size):
-            if seed := TaskQueue.SEED.pop():
-                seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self._db.zadd(self._todo_key, seeds, nx=True)
-        if TaskQueue.SEED.length < self.new_queue_max_size:
-            time.sleep(self.new_queue_wait_seconds)
-
-    @pause
-    def _delete(self):
-        """
-        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
-        """
-        seed_info = {"count": 0, "failed": [], "succeed": [], "common": []}
-        status = TaskQueue.DONE.length < self.done_queue_max_size
-
-        for _ in range(self.done_queue_max_size):
-            seed = TaskQueue.DONE.pop()
-            if not seed:
-                break
-            if seed.params.seed_status == DealModel.fail:
-                seed_info["failed"].append(seed.to_string)
-            elif self.done_model == 1:
-                seed_info["succeed"].append(seed.to_string)
-            else:
-                seed_info["common"].append(seed.to_string)
-            seed_info['count'] += 1
-
-        if seed_info["count"]:
-
-            succeed_count = int(self._db.zrem(self._todo_key, *seed_info["common"]) or 0)
-            succeed_count += int(self._db.done([self._todo_key, self._done_key], *seed_info["succeed"]) or 0)
-            failed_count = int(self._db.done([self._todo_key, self._fail_key], *seed_info["failed"]) or 0)
-
-            if failed_count:
-                self.statistics(self._statistics_fail_key, failed_count)
-            if succeed_count:
-                self.statistics(self._statistics_done_key, succeed_count)
-
-            self._remove_doing_seeds(seed_info["common"] + seed_info["succeed"] + seed_info["failed"])
-
-        if status:
-            time.sleep(self.done_queue_wait_seconds)
-