cobweb-launcher 1.3.5__py3-none-any.whl → 1.3.7__py3-none-any.whl

cobweb_new/db/api_db.py

@@ -1,82 +0,0 @@
-import os
-import json
-import requests
-
-
-class ApiDB:
-
-    def __init__(self, host=None, **kwargs):
-        self.host = host or os.getenv("REDIS_API_HOST", "http://127.0.0.1:4396")
-
-    def _get_response(self, api, params: dict = None):
-        try:
-            url = self.host + api
-            response = requests.get(url, params=params)
-            json_data = response.json()
-            response.close()
-            return json_data["data"]
-        except:
-            return None
-
-    def _post_response(self, api, params: dict = None, data: dict = None):
-        try:
-            url = self.host + api
-            headers = {"Content-Type": "application/json"}
-            response = requests.post(url, headers=headers, params=params, data=json.dumps(data))
-            json_data = response.json()
-            response.close()
-            return json_data["data"]
-        except:
-            return None
-
-    def get(self, name):
-        return self._get_response(api="/get", params=dict(name=name))
-
-    def setnx(self, name, value=""):
-        return self._get_response(api="/setnx", params=dict(name=name, value=value))
-
-    def setex(self, name, t, value=""):
-        return self._get_response(api="/setex", params=dict(name=name, value=value, t=t))
-
-    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
-        return self._get_response(api="/expire", params=dict(name=name, t=t, nx=nx, xx=xx, gt=gt, lt=lt))
-
-    def ttl(self, name):
-        return self._get_response(api="/ttl", params=dict(name=name))
-
-    def delete(self, name):
-        return self._get_response(api="/delete", params=dict(name=name))
-
-    def exists(self, name):
-        return self._get_response(api="/exists", params=dict(name=name))
-
-    def incrby(self, name, value):
-        return self._get_response(api="/incrby", params=dict(name=name, value=value))
-
-    def zcard(self, name) -> bool:
-        return self._get_response(api="/zcard", params=dict(name=name))
-
-    def zadd(self, name, item: dict, **kwargs):
-        return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
-
-    def zrem(self, name, *values):
-        return self._post_response(api="/zrem", data=dict(name=name, values=values))
-
-    def zcount(self, name, _min, _max):
-        return self._get_response(api="/zcount", params=dict(name=name, min=_min, max=_max))
-
-    def lock(self, name, t=15) -> bool:
-        return self._get_response(api="/lock", params=dict(name=name, t=t))
-
-    def auto_incr(self, name, t=15, limit=1000) -> bool:
-        return self._get_response(api="/auto_incr", params=dict(name=name, t=t, limit=limit))
-
-    def members(self, name, score, start=0, count=5000, _min="-inf", _max="+inf"):
-        return self._get_response(api="/members", params=dict(name=name, score=score, start=start, count=count, min=_min, max=_max))
-
-    def done(self, name: list, *values):
-        return self._post_response(api="/done", data=dict(name=name, values=values))
-
-
-
-

cobweb_new/db/redis_db.py

@@ -1,158 +0,0 @@
-import redis
-from cobweb import setting
-
-
-class RedisDB:
-
-    def __init__(self, **kwargs):
-        redis_config = kwargs or setting.REDIS_CONFIG
-        pool = redis.ConnectionPool(**redis_config)
-        self._client = redis.Redis(connection_pool=pool)
-
-    def setnx(self, name, value=""):
-        return self._client.setnx(name, value)
-
-    def setex(self, name, t, value=""):
-        return self._client.setex(name, t, value)
-
-    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
-        return self._client.expire(name, t, nx, xx, gt, lt)
-
-    def ttl(self, name):
-        return self._client.ttl(name)
-
-    def delete(self, name):
-        return self._client.delete(name)
-
-    def exists(self, *name) -> bool:
-        return self._client.exists(*name)
-
-    def incrby(self, name, value):
-        return self._client.incrby(name, value)
-
-    def sadd(self, name, value):
-        return self._client.sadd(name, value)
-
-    def zcard(self, name) -> bool:
-        return self._client.zcard(name)
-
-    def zadd(self, name, item: dict, **kwargs):
-        return self._client.zadd(name, item, **kwargs)
-
-    def zrem(self, name, *value):
-        return self._client.zrem(name, *value)
-
-    def zcount(self, name, _min, _max):
-        return self._client.zcount(name, _min, _max)
-
-    # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
-    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
-
-    def lua(self, script: str, keys: list = None, args: list = None):
-        keys = keys or []
-        args = args or []
-        keys_count = len(keys)
-        return self._client.eval(script, keys_count, *keys, *args)
-
-    def lua_sha(self, sha1: str, keys: list = None, args: list = None):
-        keys = keys or []
-        args = args or []
-        keys_count = len(keys)
-        return self._client.evalsha(sha1, keys_count, *keys, *args)
-
-    def execute_lua(self, lua_script: str, keys: list, *args):
-        execute = self._client.register_script(lua_script)
-        return execute(keys=keys, args=args)
-
-    def lock(self, key, t=15) -> bool:
-        lua_script = """
-        local status = redis.call('setnx', KEYS[1], 1)
-        if ( status == 1 ) then
-            redis.call('expire', KEYS[1], ARGV[1])
-        end 
-        return status 
-        """
-        status = self.execute_lua(lua_script, [key], t)
-        return bool(status)
-
-    def auto_incr(self, name, t=15, limit=1000):
-        lua_script = """
-            local count = 0
-            local status = false
-            local limit = ARGV[2]
-            local expire = redis.call('ttl', KEYS[1])
-
-            if ( expire == -2 ) then
-                redis.call('setnx', KEYS[1], 1)
-            elseif ( expire == -1) then
-                redis.call('expire', KEYS[1], ARGV[1])
-            else
-                count = redis.call('get', KEYS[1])
-            end
-
-            if ( count + 0 < limit + 0 ) then
-                status = true
-                redis.call('incr', KEYS[1])
-            end
-
-            return status
-        """
-        status = self.execute_lua(lua_script, [name], t, limit)
-        return bool(status)
-
-    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
-        lua_script = """
-        local min = ARGV[1]
-        local max = ARGV[2]
-        local start = ARGV[3]
-        local count = ARGV[4]
-        local score = ARGV[5]
-        local members = nil
-
-        if ( type(count) == string ) then
-            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
-        else
-            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
-        end
-        
-        local result = {}
-
-        for i = 1, #members, 2 do
-            local priority = nil
-            local member = members[i]
-            local originPriority = nil
-            if ( members[i+1] + 0 < 0 ) then
-                originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
-            else
-                originPriority = math.floor(members[i+1])
-            end
-            
-            if ( score + 0 >= 1000 ) then
-                priority = -score - originPriority / 1000
-            elseif ( score + 0 == 0 ) then
-                priority = originPriority
-            else
-                originPriority = score 
-                priority = score
-            end
-            redis.call('zadd', KEYS[1], priority, member)
-            table.insert(result, member)
-            table.insert(result, originPriority)
-        end
-
-        return result
-        """
-        members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
-        return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
-
-    def done(self, keys: list, *args) -> list:
-        lua_script = """
-        for i, member in ipairs(ARGV) do
-            redis.call("zrem", KEYS[1], member)
-            redis.call("sadd", KEYS[2], member)
-        end
-        """
-        self.execute_lua(lua_script, keys, *args)
-
-
-

cobweb_new/exceptions/__init__.py

@@ -1 +0,0 @@
-from .oss_db_exception import *

cobweb_new/exceptions/oss_db_exception.py

@@ -1,28 +0,0 @@
-class OssDBException(Exception):
-    """Base oss client exception that all others inherit."""
-
-
-class OssDBMergeError(OssDBException):
-    """
-    Exception raised when execute merge operation fails.
-    """
-
-
-class OssDBPutPartError(OssDBException):
-    """
-    Exception raised when upload part operation fails.
-    """
-
-
-class OssDBPutObjError(OssDBException):
-    """
-    Exception raised when upload operation fails.
-    """
-
-
-class OssDBAppendObjError(OssDBException):
-    """Exception raised when upload operation fails."""
-
-
-class OssDBInitPartError(OssDBException):
-    """Exception raised when init upload operation fails."""

cobweb_new/launchers/__init__.py

@@ -1,3 +0,0 @@
-from .launcher_air import LauncherAir
-from .launcher_pro import LauncherPro
-from .launcher_api import LauncherApi

cobweb_new/launchers/launcher.py

@@ -1,237 +0,0 @@
-import time
-import inspect
-import threading
-import importlib
-
-from inspect import isgenerator
-from typing import Union, Callable
-
-from constant import DealModel, LogTemplate
-from cobweb.utils import dynamic_load_class
-from cobweb.base import Seed, Queue, logger, TaskQueue
-from cobweb import setting
-
-
-class Launcher(threading.Thread):
-
-    __CUSTOM_FUNC__ = {}
-
-    def __init__(self, task, project, custom_setting=None, **kwargs):
-        super().__init__()
-        self.task = task
-        self.project = project
-        self.custom_func = dict()
-        self.app_time = int(time.time())
-
-        _setting = dict()
-
-        if custom_setting:
-            if isinstance(custom_setting, dict):
-                _setting = custom_setting
-            else:
-                if isinstance(custom_setting, str):
-                    custom_setting = importlib.import_module(custom_setting)
-                if not inspect.ismodule(custom_setting):
-                    raise Exception
-                for k, v in custom_setting.__dict__.items():
-                    if not k.startswith("__") and not inspect.ismodule(v):
-                        _setting[k] = v
-
-        _setting.update(**kwargs)
-
-        for k, v in _setting.items():
-            setattr(setting, k.upper(), v)
-
-        self.scheduling_wait_time = setting.SCHEDULING_WAIT_TIME
-        self.inserting_wait_time = setting.INSERTING_WAIT_TIME
-        self.removing_wait_time = setting.REMOVING_WAIT_TIME
-
-        self.scheduling_size = setting.SCHEDULING_SIZE
-        self.inserting_size = setting.INSERTING_SIZE
-        self.removing_size = setting.REMOVING_SIZE
-
-        self.todo_queue_size = setting.TODO_QUEUE_SIZE
-        self.seed_queue_size = setting.SEED_QUEUE_SIZE
-        self.request_queue_size = setting.REQUEST_QUEUE_SIZE
-        self.download_queue_size = setting.DOWNLOAD_QUEUE_SIZE
-        self.response_queue_size = setting.RESPONSE_QUEUE_SIZE
-        self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
-        self.delete_queue_size = setting.DELETE_QUEUE_SIZE
-        self.done_queue_size = setting.DONE_QUEUE_SIZE
-
-        self.stop = threading.Event()  # 结束事件
-        self.pause = threading.Event()  # 暂停事件
-
-        self.crawler_path = setting.CRAWLER
-        self.pipeline_path = setting.PIPELINE
-
-        # self.crawler = None
-        # self.pipeline = None
-
-        self._threads = []
-
-        self._task_info = dict(todo={}, download={})
-
-        # ------
-
-        self.before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
-
-        self.todo_queue_full_wait_seconds = setting.TODO_QUEUE_FULL_WAIT_SECONDS
-        self.new_queue_wait_seconds = setting.NEW_QUEUE_WAIT_SECONDS
-        self.done_queue_wait_seconds = setting.DONE_QUEUE_WAIT_SECONDS
-        self.upload_queue_wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
-        self.seed_reset_seconds = setting.SEED_RESET_SECONDS
-
-        self.todo_queue_size = setting.TODO_QUEUE_SIZE
-        # self.new_queue_max_size = setting.NEW_QUEUE_MAX_SIZE
-        # self.done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
-        # self.upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
-
-        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
-        self.spider_thread_num = setting.SPIDER_THREAD_NUM
-        self.spider_time_sleep = setting.SPIDER_TIME_SLEEP
-        self.spider_max_count = setting.SPIDER_MAX_COUNT
-        self.time_window = setting.TIME_WINDOW
-
-        self.done_model = setting.DONE_MODEL
-        self.task_model = setting.TASK_MODEL
-
-        self.filter_field = setting.FILTER_FIELD
-
-    @staticmethod
-    def insert_seed(seed: Union[Seed, dict]):
-        if isinstance(seed, dict):
-            seed = Seed(seed)
-        TaskQueue.SEED.push(seed)
-
-    @property
-    def request(self):
-        """
-        自定义request函数
-        use case:
-            from cobweb.base import Request, BaseItem
-            @launcher.request
-            def request(seed: Seed) -> Union[Request, BaseItem]:
-                ...
-                yield Request(seed.url, seed)
-        """
-        def decorator(func):
-            self.custom_func['request'] = func
-        return decorator
-
-    @property
-    def download(self):
-        """
-        自定义download函数
-        use case:
-            from cobweb.base import Request, Response, Seed, BaseItem
-            @launcher.download
-            def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
-                ...
-                yield Response(item.seed, response)
-        """
-        def decorator(func):
-            self.custom_func['download'] = func
-        return decorator
-
-    @property
-    def parse(self):
-        """
-        自定义parse函数, xxxItem为自定义的存储数据类型
-        use case:
-            from cobweb.base import Request, Response
-            @launcher.parse
-            def parse(item: Response) -> BaseItem:
-               ...
-               yield xxxItem(seed, **kwargs)
-        """
-        def decorator(func):
-            self.custom_func['parse'] = func
-        return decorator
-
-    def remove_working_items(self, key, items):
-        for item in items:
-            self._task_info[key].pop(item, None)
-
-    def check_alive(self):
-        while not self.stop.is_set():
-            if not self.pause.is_set():
-                for thread in self._threads:
-                    if not thread.is_alive():
-                        thread.start()
-            time.sleep(1)
-
-    def _add_thread(self, func, num=1, obj=None, name=None, args=()):
-        obj = obj or self
-        name = obj.__class__.__name__ + name or func.__name__
-        for i in range(num):
-            func_name = name + "_" + str(i) if num > 1 else name
-            self._threads.append(threading.Thread(name=func_name, target=func, args=(obj,) + args))
-
-    def _init_schedule_thread(self):
-        ...
-
-    def _polling(self):
-        check_emtpy_times = 0
-        while not self.stop.is_set():
-            if TaskQueue.is_empty():
-                if self.pause.is_set():
-                    run_time = int(time.time()) - self.app_time
-                    if not self.task_model and run_time > self.before_scheduler_wait_seconds:
-                        logger.info("Done! ready to close thread...")
-                        self.stop.set()
-                    else:
-                        logger.info("pause! waiting for resume...")
-                elif check_emtpy_times > 2:
-                    logger.info("pause! waiting for resume...")
-                    self.doing_seeds = {}
-                    self.pause.set()
-                else:
-                    logger.info(
-                        "check whether the task is complete, "
-                        f"reset times {3 - check_emtpy_times}"
-                    )
-                    check_emtpy_times += 1
-            elif TaskQueue.TODO.length:
-                logger.info(f"Recovery {self.task} task run！")
-                check_emtpy_times = 0
-                self.pause.clear()
-            else:
-                logger.info(LogTemplate.launcher_polling.format(
-                    task=self.task,
-                    doing_len=len(self.doing_seeds.keys()),
-                    todo_len=TaskQueue.TODO.length,
-                    done_len=TaskQueue.DONE.length,
-                    upload_len=TaskQueue.UPLOAD.length,
-                ))
-
-            time.sleep(10)
-
-        logger.info("Done! Ready to close thread...")
-
-    def run(self):
-        Crawler = dynamic_load_class(self.crawler_path)
-        Pipeline = dynamic_load_class(self.pipeline_path)
-
-        crawler = Crawler(
-            stop=self.stop, pause=self.pause,
-            thread_num=self.spider_thread_num,
-            time_sleep=self.spider_time_sleep,
-            custom_func=self.custom_func
-        )
-
-        pipeline = Pipeline(
-            stop=self.stop, pause=self.pause,
-            upload_size=self.upload_queue_max_size,
-            wait_seconds=self.upload_queue_wait_seconds
-        )
-
-        self._add_thread(obj=crawler, func=crawler.build_request_item)
-        self._add_thread(obj=crawler, func=crawler.build_download_item, num=self.spider_thread_num)
-        self._add_thread(obj=crawler, func=crawler.build_parse_item)
-
-        self._init_schedule_thread()
-        self.check_alive()
-
-
-

cobweb_new/launchers/launcher_air.py

@@ -1,88 +0,0 @@
-import time
-
-from cobweb.base import logger
-from cobweb.constant import LogTemplate
-from .launcher import Launcher, check_pause
-
-
-class LauncherAir(Launcher):
-
-    # def _scheduler(self):
-    #     if self.start_seeds:
-    #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
-
-    @check_pause
-    def _insert(self):
-        seeds = {}
-        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-        for _ in range(self._new_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['new'].pop()
-            if not seed:
-                break
-            seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(seeds)
-        if status:
-            time.sleep(self._new_queue_wait_seconds)
-
-    @check_pause
-    def _delete(self):
-        seeds = []
-        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-
-        for _ in range(self._done_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['done'].pop()
-            if not seed:
-                break
-            seeds.append(seed.to_string)
-
-        if seeds:
-            self._remove_doing_seeds(seeds)
-
-        if status:
-            time.sleep(self._done_queue_wait_seconds)
-
-    def _polling(self):
-
-        check_emtpy_times = 0
-
-        while not self._stop.is_set():
-
-            queue_not_empty_count = 0
-            pooling_wait_seconds = 30
-
-            for q in self.__LAUNCHER_QUEUE__.values():
-                if q.length != 0:
-                    queue_not_empty_count += 1
-
-            if queue_not_empty_count == 0:
-                pooling_wait_seconds = 3
-                if self._pause.is_set():
-                    check_emtpy_times = 0
-                    if not self._task_model:
-                        logger.info("Done! Ready to close thread...")
-                        self._stop.set()
-                elif check_emtpy_times > 2:
-                    self.__DOING__ = {}
-                    self._pause.set()
-                else:
-                    logger.info(
-                        "check whether the task is complete, "
-                        f"reset times {3 - check_emtpy_times}"
-                    )
-                    check_emtpy_times += 1
-            elif self._pause.is_set():
-                self._pause.clear()
-                self._execute()
-            else:
-                logger.info(LogTemplate.launcher_air_polling.format(
-                    task=self.task,
-                    doing_len=len(self.__DOING__.keys()),
-                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-                ))
-
-            time.sleep(pooling_wait_seconds)
-
-