cobweb-launcher 1.3.5__py3-none-any.whl → 1.3.7__py3-none-any.whl

cobweb_/db/redis_db.py

@@ -1,130 +0,0 @@
-import redis
-from cobweb import setting
-
-
-class RedisDB:
-
-    def __init__(self, **kwargs):
-        redis_config = kwargs or setting.REDIS_CONFIG
-        pool = redis.ConnectionPool(**redis_config)
-        self._client = redis.Redis(connection_pool=pool)
-
-    def setnx(self, name, value=""):
-        return self._client.setnx(name, value)
-
-    def setex(self, name, t, value=""):
-        return self._client.setex(name, t, value)
-
-    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
-        return self._client.expire(name, t, nx, xx, gt, lt)
-
-    def ttl(self, name):
-        return self._client.ttl(name)
-
-    def delete(self, name):
-        return self._client.delete(name)
-
-    def exists(self, *name) -> bool:
-        return self._client.exists(*name)
-
-    def sadd(self, name, value):
-        return self._client.sadd(name, value)
-
-    def zcard(self, name) -> bool:
-        return self._client.zcard(name)
-
-    def zadd(self, name, item: dict, **kwargs):
-        return self._client.zadd(name, item, **kwargs)
-
-    def zrem(self, name, *value):
-        return self._client.zrem(name, *value)
-
-    def zcount(self, name, _min, _max):
-        return self._client.zcount(name, _min, _max)
-
-    # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
-    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
-
-    def lua(self, script: str, keys: list = None, args: list = None):
-        keys = keys or []
-        args = args or []
-        keys_count = len(keys)
-        return self._client.eval(script, keys_count, *keys, *args)
-
-    def lua_sha(self, sha1: str, keys: list = None, args: list = None):
-        keys = keys or []
-        args = args or []
-        keys_count = len(keys)
-        return self._client.evalsha(sha1, keys_count, *keys, *args)
-
-    def execute_lua(self, lua_script: str, keys: list, *args):
-        execute = self._client.register_script(lua_script)
-        return execute(keys=keys, args=args)
-
-    def lock(self, key, t=15) -> bool:
-        lua_script = """
-        local status = redis.call('setnx', KEYS[1], 1)
-        if ( status == 1 ) then
-            redis.call('expire', KEYS[1], ARGV[1])
-        end 
-        return status 
-        """
-        status = self.execute_lua(lua_script, [key], t)
-        return bool(status)
-
-    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
-        lua_script = """
-        local min = ARGV[1]
-        local max = ARGV[2]
-        local start = ARGV[3]
-        local count = ARGV[4]
-        local score = ARGV[5]
-        local members = nil
-
-        if ( type(count) == string ) then
-            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
-        else
-            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
-        end
-        
-        local result = {}
-
-        for i = 1, #members, 2 do
-            local priority = nil
-            local member = members[i]
-            local originPriority = nil
-            if ( members[i+1] + 0 < 0 ) then
-                originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
-            else
-                originPriority = math.floor(members[i+1])
-            end
-            
-            if ( score + 0 >= 1000 ) then
-                priority = -score - originPriority / 1000
-            elseif ( score + 0 == 0 ) then
-                priority = originPriority
-            else
-                originPriority = score 
-                priority = score
-            end
-            redis.call('zadd', KEYS[1], priority, member)
-            table.insert(result, member)
-            table.insert(result, originPriority)
-        end
-
-        return result
-        """
-        members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
-        return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
-
-    def done(self, keys: list, *args) -> list:
-        lua_script = """
-        for i, member in ipairs(ARGV) do
-            redis.call("zrem", KEYS[1], member)
-            redis.call("sadd", KEYS[2], member)
-        end
-        """
-        self.execute_lua(lua_script, keys, *args)
-
-
-

cobweb_/exceptions/__init__.py

@@ -1 +0,0 @@
-from .oss_db_exception import *

cobweb_/exceptions/oss_db_exception.py

@@ -1,28 +0,0 @@
-class OssDBException(Exception):
-    """Base oss client exception that all others inherit."""
-
-
-class OssDBMergeError(OssDBException):
-    """
-    Exception raised when execute merge operation fails.
-    """
-
-
-class OssDBPutPartError(OssDBException):
-    """
-    Exception raised when upload part operation fails.
-    """
-
-
-class OssDBPutObjError(OssDBException):
-    """
-    Exception raised when upload operation fails.
-    """
-
-
-class OssDBAppendObjError(OssDBException):
-    """Exception raised when upload operation fails."""
-
-
-class OssDBInitPartError(OssDBException):
-    """Exception raised when init upload operation fails."""

cobweb_/launchers/__init__.py

@@ -1,3 +0,0 @@
-from .launcher_air import LauncherAir
-from .launcher_pro import LauncherPro
-from .launcher_api import LauncherApi

cobweb_/launchers/launcher.py

@@ -1,235 +0,0 @@
-import time
-import inspect
-import threading
-import importlib
-from functools import wraps
-
-
-from cobweb import setting
-from cobweb.base import Seed, Queue, logger
-from cobweb.utils.tools import dynamic_load_class
-
-
-def check_pause(func):
-    @wraps(func)
-    def wrapper(self, *args, **kwargs):
-        while not self._pause.is_set():
-            try:
-                func(self, *args, **kwargs)
-            except Exception as e:
-                logger.info(f"{func.__name__}: " + str(e))
-            finally:
-                time.sleep(0.1)
-
-    return wrapper
-
-
-class Launcher(threading.Thread):
-
-    SEEDS = []
-
-    __DOING__ = {}
-
-    __CUSTOM_FUNC__ = {
-        # "download": None,
-        # "request": None,
-        # "parse": None,
-    }
-
-    __LAUNCHER_QUEUE__ = {
-        "new": Queue(),
-        "todo": Queue(),
-        "done": Queue(),
-        "upload": Queue()
-    }
-
-    __LAUNCHER_FUNC__ = [
-        "_reset",
-        "_scheduler",
-        "_insert",
-        "_refresh",
-        "_delete",
-    ]
-
-    def __init__(self, task, project, custom_setting=None, **kwargs):
-        super().__init__()
-        self.task = task
-        self.project = project
-
-        self._app_time = int(time.time())
-        self._stop = threading.Event()  # 结束事件
-        self._pause = threading.Event()  # 暂停事件
-
-        _setting = dict()
-
-        if custom_setting:
-            if isinstance(custom_setting, dict):
-                _setting = custom_setting
-            else:
-                if isinstance(custom_setting, str):
-                    custom_setting = importlib.import_module(custom_setting)
-                if not inspect.ismodule(custom_setting):
-                    raise Exception
-                for k, v in custom_setting.__dict__.items():
-                    if not k.startswith("__") and not inspect.ismodule(v):
-                        _setting[k] = v
-
-        _setting.update(**kwargs)
-
-        for k, v in _setting.items():
-            setattr(setting, k.upper(), v)
-
-        self._Crawler = dynamic_load_class(setting.CRAWLER)
-        self._Pipeline = dynamic_load_class(setting.PIPELINE)
-
-        self._before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
-        self._scheduler_wait_seconds = setting.SCHEDULER_WAIT_SECONDS
-        self._todo_queue_full_wait_seconds = setting.TODO_QUEUE_FULL_WAIT_SECONDS
-        self._new_queue_wait_seconds = setting.NEW_QUEUE_WAIT_SECONDS
-        self._done_queue_wait_seconds = setting.DONE_QUEUE_WAIT_SECONDS
-        self._upload_queue_wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
-        self._seed_reset_seconds = setting.SEED_RESET_SECONDS
-
-        self._todo_queue_size = setting.TODO_QUEUE_SIZE
-        self._new_queue_max_size = setting.NEW_QUEUE_MAX_SIZE
-        self._done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
-        self._upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
-
-        self._spider_max_retries = setting.SPIDER_MAX_RETRIES
-        self._spider_thread_num = setting.SPIDER_THREAD_NUM
-        self._spider_time_sleep = setting.SPIDER_TIME_SLEEP
-        self._spider_max_count = setting.SPIDER_MAX_COUNT
-        self._time_window = setting.TIME_WINDOW
-
-        self._done_model = setting.DONE_MODEL
-        self._task_model = setting.TASK_MODEL
-
-        self._filter_field = setting.FILTER_FIELD
-
-    @property
-    def request(self):
-        """
-        自定义request函数
-        use case:
-            from cobweb.base import Request, BaseItem
-            @launcher.request
-            def request(seed: Seed) -> Union[Request, BaseItem]:
-                ...
-                yield Request(seed.url, seed)
-        """
-        def decorator(func):
-            self.__CUSTOM_FUNC__["request"] = func
-        return decorator
-
-    @property
-    def download(self):
-        """
-        自定义download函数
-        use case:
-            from cobweb.base import Request, Response, Seed, BaseItem
-            @launcher.download
-            def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-                ...
-                yield Response(item.seed, response)
-        """
-        def decorator(func):
-            self.__CUSTOM_FUNC__["download"] = func
-        return decorator
-
-    @property
-    def parse(self):
-        """
-        自定义parse函数, xxxItem为自定义的存储数据类型
-        use case:
-            from cobweb.base import Request, Response
-            @launcher.parse
-            def parse(item: Response) -> BaseItem:
-               ...
-               yield xxxItem(seed, **kwargs)
-        """
-        def decorator(func):
-            self.__CUSTOM_FUNC__["parse"] = func
-        return decorator
-
-    def start_seeds(self):
-        seeds = [Seed(seed) for seed in self.SEEDS]
-        self.__LAUNCHER_QUEUE__['todo'].push(seeds)
-        return seeds
-
-    def _remove_doing_seeds(self, seeds):
-        for seed in seeds:
-            self.__DOING__.pop(seed, None)
-        # logger.info("remove %s seeds from __DOING__" % len(seeds))
-
-    def _get_seed(self) -> Seed:
-        return self.__LAUNCHER_QUEUE__["todo"].pop()
-
-    def _set_seed(self, seed, **kwargs):
-        self.__LAUNCHER_QUEUE__["todo"].push(seed, **kwargs)
-
-    def _upload_data(self, data, **kwargs):
-        self.__LAUNCHER_QUEUE__["upload"].push(data, **kwargs)
-
-    def _add_seed(self, seed, **kwargs):
-        self.__LAUNCHER_QUEUE__["new"].push(seed, **kwargs)
-
-    def _delete_seed(self, seed, **kwargs):
-        self.__LAUNCHER_QUEUE__["done"].push(seed, **kwargs)
-
-    def _execute(self):
-        for func_name in self.__LAUNCHER_FUNC__:
-            threading.Thread(name=func_name, target=getattr(self, func_name)).start()
-            time.sleep(1)
-
-    def run(self):
-        threading.Thread(target=self._execute_heartbeat).start()
-
-        self.start_seeds()
-
-        self._Crawler(
-            task=self.task, project=self.project,
-            stop=self._stop, pause=self._pause,
-            # launcher_queue=self.__LAUNCHER_QUEUE__,
-            get_seed=self._get_seed,
-            set_seed=self._set_seed,
-            add_seed=self._add_seed,
-            delete_seed=self._delete_seed,
-            upload_data=self._upload_data,
-            custom_func=self.__CUSTOM_FUNC__,
-            thread_num = self._spider_thread_num,
-            max_retries = self._spider_max_retries,
-            time_sleep=self._spider_time_sleep
-        ).start()
-
-        self._Pipeline(
-            stop=self._stop, pause=self._pause,
-            upload=self.__LAUNCHER_QUEUE__["upload"],
-            done=self.__LAUNCHER_QUEUE__["done"],
-            upload_size=self._upload_queue_max_size,
-            wait_seconds=self._upload_queue_wait_seconds
-        ).start()
-
-        self._execute()
-        self._polling()
-
-    def _execute_heartbeat(self):
-        pass
-
-    def _reset(self):
-        pass
-
-    def _scheduler(self):
-        pass
-
-    def _insert(self):
-        pass
-
-    def _refresh(self):
-        pass
-
-    def _delete(self):
-        pass
-
-    def _polling(self):
-        pass
-

cobweb_/launchers/launcher_air.py

@@ -1,88 +0,0 @@
-import time
-
-from cobweb.base import logger
-from cobweb.constant import LogTemplate
-from .launcher import Launcher, check_pause
-
-
-class LauncherAir(Launcher):
-
-    # def _scheduler(self):
-    #     if self.start_seeds:
-    #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
-
-    @check_pause
-    def _insert(self):
-        seeds = {}
-        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-        for _ in range(self._new_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['new'].pop()
-            if not seed:
-                break
-            seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(seeds)
-        if status:
-            time.sleep(self._new_queue_wait_seconds)
-
-    @check_pause
-    def _delete(self):
-        seeds = []
-        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-
-        for _ in range(self._done_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['done'].pop()
-            if not seed:
-                break
-            seeds.append(seed.to_string)
-
-        if seeds:
-            self._remove_doing_seeds(seeds)
-
-        if status:
-            time.sleep(self._done_queue_wait_seconds)
-
-    def _polling(self):
-
-        check_emtpy_times = 0
-
-        while not self._stop.is_set():
-
-            queue_not_empty_count = 0
-            pooling_wait_seconds = 30
-
-            for q in self.__LAUNCHER_QUEUE__.values():
-                if q.length != 0:
-                    queue_not_empty_count += 1
-
-            if queue_not_empty_count == 0:
-                pooling_wait_seconds = 3
-                if self._pause.is_set():
-                    check_emtpy_times = 0
-                    if not self._task_model:
-                        logger.info("Done! Ready to close thread...")
-                        self._stop.set()
-                elif check_emtpy_times > 2:
-                    self.__DOING__ = {}
-                    self._pause.set()
-                else:
-                    logger.info(
-                        "check whether the task is complete, "
-                        f"reset times {3 - check_emtpy_times}"
-                    )
-                    check_emtpy_times += 1
-            elif self._pause.is_set():
-                self._pause.clear()
-                self._execute()
-            else:
-                logger.info(LogTemplate.launcher_air_polling.format(
-                    task=self.task,
-                    doing_len=len(self.__DOING__.keys()),
-                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-                ))
-
-            time.sleep(pooling_wait_seconds)
-
-