cobweb-launcher 1.2.49__py3-none-any.whl → 1.3.1__py3-none-any.whl

cobweb_new/constant.py

@@ -0,0 +1,105 @@
+
+class CrawlerModel:
+
+    default = "cobweb.crawlers.Crawler"
+    file_air = "cobweb.crawlers.FileCrawlerAir"
+    file_pro = "cobweb.crawlers.FileCrawlerPro"
+
+
+class LauncherModel:
+    task = "launcher model: task"
+    resident = "launcher model: resident"
+
+
+class DownloadModel:
+    common = "download model: common"
+    file = "download model: file"
+
+
+class LogModel:
+    simple = "log model: simple"
+    common = "log model: common"
+    detailed = "log model: detailed"
+
+
+class DealModel:
+    fail = "deal model: fail"
+    done = "deal model: done"
+    poll = "deal model: poll"
+
+
+class LogTemplate:
+
+    console_item = """
+    ----------------------- start - console pipeline -----------------
+            种子详情 \n{seed_detail}
+            解析详情 \n{parse_detail}
+    ----------------------- end  - console pipeline ------------------
+    """
+
+    launcher_polling = """
+    ----------------------- start - 轮训日志: {task} -----------------
+            内存队列 
+                种子数:  {doing_len}
+                待消费:  {todo_len}
+                已消费:  {done_len}
+            存储队列 
+                待上传:  {upload_len}
+    ----------------------- end  - 轮训日志: {task} ------------------
+    """
+
+    launcher_air_polling = """
+    ----------------------- start - 轮训日志: {task} -----------------
+            内存队列 
+                种子数:  {doing_len}
+                待消费:  {todo_len}
+                已消费:  {done_len}
+            存储队列 
+                待上传:  {upload_len}
+    ----------------------- end  - 轮训日志: {task} ------------------
+    """
+
+    launcher_pro_polling = """
+----------------------- start - 轮训日志: {task} -----------------
+        内存队列 
+            种子数:  {doing_len}
+            待消费:  {todo_len}
+            已消费:  {done_len}
+        redis队列 
+            种子数:  {redis_seed_count}
+            待消费:  {redis_todo_len}
+            消费中:  {redis_doing_len}
+        存储队列 
+            待上传:  {upload_len}
+----------------------- end  - 轮训日志: {task} ------------------
+"""
+
+    download_exception = """
+----------------------- download exception -----------------------
+        种子详情 \n{detail}
+        种子参数
+            retry         :    {retry}
+            priority      :    {priority}
+            seed_version  :    {seed_version}
+            identifier    :    {identifier}
+        exception
+            msg           :    {exception}
+------------------------------------------------------------------
+"""
+
+    download_info = """
+------------------------ download info ---------------------------
+        种子详情 \n{detail}
+        种子参数
+            retry         :    {retry}
+            priority      :    {priority}
+            seed_version  :    {seed_version}
+            identifier    :    {identifier}
+        response 
+            status        :    {status} \n{response}
+------------------------------------------------------------------
+"""
+
+    @staticmethod
+    def log_info(item: dict) -> str:
+        return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

cobweb_new/crawlers/__init__.py

@@ -0,0 +1 @@
+from .crawler import Crawler

cobweb_new/crawlers/crawler-new.py

@@ -0,0 +1,85 @@
+import json
+import time
+import threading
+from typing import Union, Callable, Mapping
+
+import setting
+from cobweb.base import (
+    Seed,
+    BaseItem,
+    Request,
+    Response,
+    ConsoleItem,
+    decorators,
+    TaskQueue,
+    logger
+)
+from constant import DealModel
+
+
+class Crawler(threading.Thread):
+
+    def __init__(self, custom_func: Union[Mapping[str, Callable]]):
+        super().__init__()
+
+        for func_name, _callable in custom_func.items():
+            if isinstance(_callable, Callable):
+                self.__setattr__(func_name, _callable)
+
+        self.spider_max_retries = setting.SPIDER_MAX_RETRIES
+        self.request_queue_size = setting.REQUEST_QUEUE_SIZE
+        self.download_queue_size = setting.DOWNLOAD_QUEUE_SIZE
+        self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
+
+    @staticmethod
+    def request(seed: Seed) -> Union[Request, BaseItem]:
+        yield Request(seed.url, seed, timeout=5)
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        response = item.download()
+        yield Response(item.seed, response, **item.to_dict)
+
+    @staticmethod
+    def parse(item: Response) -> BaseItem:
+        upload_item = item.to_dict
+        upload_item["text"] = item.response.text
+        yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
+
+    # @decorators.add_thread()
+    @decorators.pause
+    def build_request_item(self):
+        thread_sleep = 0.1
+        if TaskQueue.REQUEST.length >= self.request_queue_size:
+            thread_sleep = 5
+        elif seed := TaskQueue.TODO.pop():
+            if seed.params.retry > self.spider_max_retries:
+                seed.params.seed_status = DealModel.fail
+            else:
+                TaskQueue.process_task(seed, self.request)
+            TaskQueue.DELETE.push(seed)
+        time.sleep(thread_sleep)
+
+    # @decorators.add_thread(num=setting.SPIDER_THREAD_NUM)
+    @decorators.pause
+    def build_download_item(self):
+        thread_sleep = 0.1
+        if TaskQueue.DOWNLOAD.length >= self.download_queue_size:
+            logger.info(f"download queue is full, sleep {thread_sleep}s")
+            thread_sleep = 5
+        elif request_item := TaskQueue.REQUEST.pop():
+            TaskQueue.process_task(request_item, self.download)
+        time.sleep(thread_sleep)
+
+    # @decorators.add_thread()
+    @decorators.pause
+    def build_parse_item(self):
+        thread_sleep = 0.1
+        if TaskQueue.UPLOAD.length >= self.upload_queue_size:
+            logger.info(f"upload queue is full, sleep {thread_sleep}s")
+            thread_sleep = 5
+        if response_item := TaskQueue.RESPONSE.pop():
+            TaskQueue.process_task(response_item, self.parse)
+        time.sleep(thread_sleep)
+
+

cobweb_new/crawlers/crawler.py

@@ -0,0 +1,170 @@
+import json
+import threading
+import time
+import traceback
+from inspect import isgenerator
+from typing import Union, Callable, Mapping
+
+from cobweb.constant import DealModel, LogTemplate
+from cobweb.base import (
+    Queue, 
+    Seed, 
+    BaseItem, 
+    Request, 
+    Response, 
+    ConsoleItem,
+    logger
+)
+
+
+class Crawler(threading.Thread):
+
+    def __init__(
+            self,
+            stop: threading.Event,
+            pause: threading.Event,
+            # launcher_queue: Union[Mapping[str, Queue]],
+            get_seed: Callable,
+            set_seed: Callable,
+            add_seed: Callable,
+            delete_seed: Callable,
+            upload_data: Callable,
+            custom_func: Union[Mapping[str, Callable]],
+            thread_num: int,
+            max_retries: int,
+            time_sleep: int,
+    ):
+        super().__init__()
+
+        self._stop = stop
+        self._pause = pause
+        self._get_seed = get_seed
+        self._set_seed = set_seed
+        self._add_seed = add_seed
+        self._delete_seed = delete_seed
+        self._upload_data = upload_data
+
+        for func_name, _callable in custom_func.items():
+            if isinstance(_callable, Callable):
+                self.__setattr__(func_name, _callable)
+
+        self.thread_num = thread_num
+        self.time_sleep = time_sleep
+        self.max_retries = max_retries
+
+    @staticmethod
+    def request(seed: Seed) -> Union[Request, BaseItem]:
+        yield Request(seed.url, seed, timeout=5)
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        response = item.download()
+        yield Response(item.seed, response, **item.to_dict)
+
+    @staticmethod
+    def parse(item: Response) -> BaseItem:
+        upload_item = item.to_dict
+        upload_item["text"] = item.response.text
+        yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
+
+    # def get_seed(self) -> Seed:
+    #     return self._todo.pop()
+
+    def distribute(self, item, seed):
+        if isinstance(item, BaseItem):
+            self._upload_data(item)
+        elif isinstance(item, Seed):
+            self._add_seed(item)
+        elif isinstance(item, str) and item == DealModel.poll:
+            self._set_seed(seed)
+        elif isinstance(item, str) and item == DealModel.done:
+            self._delete_seed(seed)
+        elif isinstance(item, str) and item == DealModel.fail:
+            seed.params.seed_status = DealModel.fail
+            self._delete_seed(seed)
+        else:
+            raise TypeError("yield value type error!")
+
+    def spider(self):
+        while not self._stop.is_set():
+
+            seed = self._get_seed()
+
+            if not seed:
+                time.sleep(1)
+                continue
+
+            elif seed.params.retry > self.max_retries:
+                seed.params.seed_status = DealModel.fail
+                self._delete_seed(seed)
+                continue
+
+            seed_detail_log_info = LogTemplate.log_info(seed.to_dict)
+
+            try:
+                request_iterators = self.request(seed)
+
+                if not isgenerator(request_iterators):
+                    raise TypeError("request function isn't a generator!")
+
+                iterator_status = False
+
+                for request_item in request_iterators:
+
+                    iterator_status = True
+
+                    if isinstance(request_item, Request):
+                        iterator_status = False
+                        download_iterators = self.download(request_item)
+                        if not isgenerator(download_iterators):
+                            raise TypeError("download function isn't a generator")
+
+                        for download_item in download_iterators:
+                            iterator_status = True
+                            if isinstance(download_item, Response):
+                                iterator_status = False
+                                logger.info(LogTemplate.download_info.format(
+                                    detail=seed_detail_log_info,
+                                    retry=seed.params.retry,
+                                    priority=seed.params.priority,
+                                    seed_version=seed.params.seed_version,
+                                    identifier=seed.identifier or "",
+                                    status=download_item.response,
+                                    response=LogTemplate.log_info(download_item.to_dict)
+                                ))
+                                parse_iterators = self.parse(download_item)
+                                if not isgenerator(parse_iterators):
+                                    raise TypeError("parse function isn't a generator")
+                                for parse_item in parse_iterators:
+                                    iterator_status = True
+                                    if isinstance(parse_item, Response):
+                                        raise TypeError("upload_item can't be a Response instance")
+                                    self.distribute(parse_item, seed)
+                            else:
+                                self.distribute(download_item, seed)
+                    else:
+                        self.distribute(request_item, seed)
+
+                if not iterator_status:
+                    raise ValueError("request/download/parse function yield value error!")
+            except Exception as e:
+                logger.info(LogTemplate.download_exception.format(
+                    detail=seed_detail_log_info,
+                    retry=seed.params.retry,
+                    priority=seed.params.priority,
+                    seed_version=seed.params.seed_version,
+                    identifier=seed.identifier or "",
+                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                ))
+                seed.params.retry += 1
+                # self._todo.push(seed)
+                self._set_seed(seed)
+                time.sleep(self.time_sleep * seed.params.retry)
+            finally:
+                time.sleep(0.1)
+        logger.info("spider thread close")
+
+    def run(self):
+        for index in range(self.thread_num):
+            threading.Thread(name=f"spider_{index}", target=self.spider).start()
+

cobweb_new/db/__init__.py

@@ -0,0 +1,2 @@
+from .redis_db import RedisDB
+from .api_db import ApiDB

cobweb_new/db/api_db.py

@@ -0,0 +1,82 @@
+import os
+import json
+import requests
+
+
+class ApiDB:
+
+    def __init__(self, host=None, **kwargs):
+        self.host = host or os.getenv("REDIS_API_HOST", "http://127.0.0.1:4396")
+
+    def _get_response(self, api, params: dict = None):
+        try:
+            url = self.host + api
+            response = requests.get(url, params=params)
+            json_data = response.json()
+            response.close()
+            return json_data["data"]
+        except:
+            return None
+
+    def _post_response(self, api, params: dict = None, data: dict = None):
+        try:
+            url = self.host + api
+            headers = {"Content-Type": "application/json"}
+            response = requests.post(url, headers=headers, params=params, data=json.dumps(data))
+            json_data = response.json()
+            response.close()
+            return json_data["data"]
+        except:
+            return None
+
+    def get(self, name):
+        return self._get_response(api="/get", params=dict(name=name))
+
+    def setnx(self, name, value=""):
+        return self._get_response(api="/setnx", params=dict(name=name, value=value))
+
+    def setex(self, name, t, value=""):
+        return self._get_response(api="/setex", params=dict(name=name, value=value, t=t))
+
+    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
+        return self._get_response(api="/expire", params=dict(name=name, t=t, nx=nx, xx=xx, gt=gt, lt=lt))
+
+    def ttl(self, name):
+        return self._get_response(api="/ttl", params=dict(name=name))
+
+    def delete(self, name):
+        return self._get_response(api="/delete", params=dict(name=name))
+
+    def exists(self, name):
+        return self._get_response(api="/exists", params=dict(name=name))
+
+    def incrby(self, name, value):
+        return self._get_response(api="/incrby", params=dict(name=name, value=value))
+
+    def zcard(self, name) -> bool:
+        return self._get_response(api="/zcard", params=dict(name=name))
+
+    def zadd(self, name, item: dict, **kwargs):
+        return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
+
+    def zrem(self, name, *values):
+        return self._post_response(api="/zrem", data=dict(name=name, values=values))
+
+    def zcount(self, name, _min, _max):
+        return self._get_response(api="/zcount", params=dict(name=name, min=_min, max=_max))
+
+    def lock(self, name, t=15) -> bool:
+        return self._get_response(api="/lock", params=dict(name=name, t=t))
+
+    def auto_incr(self, name, t=15, limit=1000) -> bool:
+        return self._get_response(api="/auto_incr", params=dict(name=name, t=t, limit=limit))
+
+    def members(self, name, score, start=0, count=5000, _min="-inf", _max="+inf"):
+        return self._get_response(api="/members", params=dict(name=name, score=score, start=start, count=count, min=_min, max=_max))
+
+    def done(self, name: list, *values):
+        return self._post_response(api="/done", data=dict(name=name, values=values))
+
+
+
+

cobweb_new/db/redis_db.py

@@ -0,0 +1,158 @@
+import redis
+from cobweb import setting
+
+
+class RedisDB:
+
+    def __init__(self, **kwargs):
+        redis_config = kwargs or setting.REDIS_CONFIG
+        pool = redis.ConnectionPool(**redis_config)
+        self._client = redis.Redis(connection_pool=pool)
+
+    def setnx(self, name, value=""):
+        return self._client.setnx(name, value)
+
+    def setex(self, name, t, value=""):
+        return self._client.setex(name, t, value)
+
+    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
+        return self._client.expire(name, t, nx, xx, gt, lt)
+
+    def ttl(self, name):
+        return self._client.ttl(name)
+
+    def delete(self, name):
+        return self._client.delete(name)
+
+    def exists(self, *name) -> bool:
+        return self._client.exists(*name)
+
+    def incrby(self, name, value):
+        return self._client.incrby(name, value)
+
+    def sadd(self, name, value):
+        return self._client.sadd(name, value)
+
+    def zcard(self, name) -> bool:
+        return self._client.zcard(name)
+
+    def zadd(self, name, item: dict, **kwargs):
+        return self._client.zadd(name, item, **kwargs)
+
+    def zrem(self, name, *value):
+        return self._client.zrem(name, *value)
+
+    def zcount(self, name, _min, _max):
+        return self._client.zcount(name, _min, _max)
+
+    # def zrangebyscore(self, name, _min, _max, start, num, withscores: bool = False, *args):
+    #     return self._client.zrangebyscore(name, _min, _max, start, num, withscores, *args)
+
+    def lua(self, script: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self._client.eval(script, keys_count, *keys, *args)
+
+    def lua_sha(self, sha1: str, keys: list = None, args: list = None):
+        keys = keys or []
+        args = args or []
+        keys_count = len(keys)
+        return self._client.evalsha(sha1, keys_count, *keys, *args)
+
+    def execute_lua(self, lua_script: str, keys: list, *args):
+        execute = self._client.register_script(lua_script)
+        return execute(keys=keys, args=args)
+
+    def lock(self, key, t=15) -> bool:
+        lua_script = """
+        local status = redis.call('setnx', KEYS[1], 1)
+        if ( status == 1 ) then
+            redis.call('expire', KEYS[1], ARGV[1])
+        end 
+        return status 
+        """
+        status = self.execute_lua(lua_script, [key], t)
+        return bool(status)
+
+    def auto_incr(self, name, t=15, limit=1000):
+        lua_script = """
+            local count = 0
+            local status = false
+            local limit = ARGV[2]
+            local expire = redis.call('ttl', KEYS[1])
+
+            if ( expire == -2 ) then
+                redis.call('setnx', KEYS[1], 1)
+            elseif ( expire == -1) then
+                redis.call('expire', KEYS[1], ARGV[1])
+            else
+                count = redis.call('get', KEYS[1])
+            end
+
+            if ( count + 0 < limit + 0 ) then
+                status = true
+                redis.call('incr', KEYS[1])
+            end
+
+            return status
+        """
+        status = self.execute_lua(lua_script, [name], t, limit)
+        return bool(status)
+
+    def members(self, key, score, start=0, count=5000, _min="-inf", _max="+inf") -> list:
+        lua_script = """
+        local min = ARGV[1]
+        local max = ARGV[2]
+        local start = ARGV[3]
+        local count = ARGV[4]
+        local score = ARGV[5]
+        local members = nil
+
+        if ( type(count) == string ) then
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES')
+        else
+            members = redis.call('zrangebyscore', KEYS[1], min, max, 'WITHSCORES', 'limit', start, count)
+        end
+        
+        local result = {}
+
+        for i = 1, #members, 2 do
+            local priority = nil
+            local member = members[i]
+            local originPriority = nil
+            if ( members[i+1] + 0 < 0 ) then
+                originPriority = math.ceil(members[i+1]) * 1000 - members[i+1] * 1000
+            else
+                originPriority = math.floor(members[i+1])
+            end
+            
+            if ( score + 0 >= 1000 ) then
+                priority = -score - originPriority / 1000
+            elseif ( score + 0 == 0 ) then
+                priority = originPriority
+            else
+                originPriority = score 
+                priority = score
+            end
+            redis.call('zadd', KEYS[1], priority, member)
+            table.insert(result, member)
+            table.insert(result, originPriority)
+        end
+
+        return result
+        """
+        members = self.execute_lua(lua_script, [key], _min, _max, start, count, score)
+        return [(members[i].decode(), int(members[i + 1])) for i in range(0, len(members), 2)]
+
+    def done(self, keys: list, *args) -> list:
+        lua_script = """
+        for i, member in ipairs(ARGV) do
+            redis.call("zrem", KEYS[1], member)
+            redis.call("sadd", KEYS[2], member)
+        end
+        """
+        self.execute_lua(lua_script, keys, *args)
+
+
+

cobweb_new/exceptions/__init__.py

@@ -0,0 +1 @@
+from .oss_db_exception import *

cobweb_new/exceptions/oss_db_exception.py

@@ -0,0 +1,28 @@
+class OssDBException(Exception):
+    """Base oss client exception that all others inherit."""
+
+
+class OssDBMergeError(OssDBException):
+    """
+    Exception raised when execute merge operation fails.
+    """
+
+
+class OssDBPutPartError(OssDBException):
+    """
+    Exception raised when upload part operation fails.
+    """
+
+
+class OssDBPutObjError(OssDBException):
+    """
+    Exception raised when upload operation fails.
+    """
+
+
+class OssDBAppendObjError(OssDBException):
+    """Exception raised when upload operation fails."""
+
+
+class OssDBInitPartError(OssDBException):
+    """Exception raised when init upload operation fails."""

cobweb_new/launchers/__init__.py

@@ -0,0 +1,3 @@
+from .launcher_air import LauncherAir
+from .launcher_pro import LauncherPro
+from .launcher_api import LauncherApi