cobweb-launcher 0.1.23__py3-none-any.whl → 1.0.0__py3-none-any.whl

cobweb/__init__.py

@@ -1,9 +1 @@
-from .bbb import Seed, Queue, DBItem
-from .task import Task
-from .log import log
-from .db.redis_db import RedisDB
-from .db.oss_db import OssDB
-from .constant import Setting
-
-from .equip.distributed.launcher import launcher
-from .equip.single.launcher import launcher as single_launcher
+from launchers import Launcher, LauncherPro

cobweb/base/__init__.py

@@ -0,0 +1,9 @@
+from .common_queue import Queue
+from .response import Response
+from .request import Request
+from .item import BaseItem
+from .seed import Seed
+
+from .log import logger
+from .decorators import decorator_oss_db
+

cobweb/base/common_queue.py

@@ -0,0 +1,30 @@
+from collections import deque
+
+
+class Queue:
+
+    def __init__(self):
+        self._queue = deque()
+
+    @property
+    def length(self) -> int:
+        return len(self._queue)
+
+    def push(self, data, left: bool = False, direct_insertion: bool = False):
+        try:
+            if not data:
+                return None
+            if not direct_insertion and any(isinstance(data, t) for t in (list, tuple)):
+                self._queue.extendleft(data) if left else self._queue.extend(data)
+            else:
+                self._queue.appendleft(data) if left else self._queue.append(data)
+        except AttributeError:
+            pass
+
+    def pop(self, left: bool = True):
+        try:
+            return self._queue.popleft() if left else self._queue.pop()
+        except IndexError:
+            return None
+        except AttributeError:
+            return None

cobweb/base/decorators.py

@@ -0,0 +1,40 @@
+from functools import wraps
+
+
+# def check_redis_status(func):
+#     @wraps(func)
+#     def wrapper(*args, **kwargs):
+#         try:
+#             result = func(*args, **kwargs)
+#         except Exception:
+#             result = False
+#         return result
+#
+#     return wrapper
+
+
+def decorator_oss_db(exception, retries=3):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(callback_func, *args, **kwargs):
+            result = None
+            for i in range(retries):
+                msg = None
+                try:
+                    return func(callback_func, *args, **kwargs)
+                except Exception as e:
+                    result = None
+                    msg = e
+                finally:
+                    if result:
+                        return result
+
+                    if i >= 2 and msg:
+                        raise exception(msg)
+
+        return wrapper
+
+    return decorator
+
+
+

cobweb/base/item.py

@@ -0,0 +1,39 @@
+from .seed import Seed
+from collections import namedtuple
+
+
+class Item(type):
+
+    def __new__(cls, name, bases, dct):
+        new_class_instance = type.__new__(cls, name, bases, dct)
+        if name != "BaseItem":
+            table = getattr(new_class_instance, "__TABLE__")
+            fields = getattr(new_class_instance, "__FIELDS__")
+            new_class_instance.Data = namedtuple(table, fields)
+        return new_class_instance
+
+
+class BaseItem(metaclass=Item):
+
+    __TABLE__ = ""
+    __FIELDS__ = ""
+
+    def __init__(self, seed: Seed, **kwargs):
+        self.seed = seed
+
+        data = {}
+        for key, value in kwargs.items():
+            if key not in self.__FIELDS__:
+                self.__setattr__(key, value)
+            else:
+                data[key] = value
+
+        self.data = self.Data(**data)
+
+    @property
+    def to_dict(self):
+        return self.data._asdict()
+
+    @property
+    def table(self):
+        return self.Data.__name__

cobweb/base/log.py

@@ -0,0 +1,94 @@
+import logging
+
+
+class ColorCodes:
+    # Text Reset
+    RESET = "\033[0m"
+
+    # Regular Colors
+    RED = "\033[31m"
+    GREEN = "\033[32m"
+    YELLOW = "\033[33m"
+    BLUE = "\033[34m"
+    PURPLE = "\033[35m"
+    CYAN = "\033[36m"
+    WHITE = "\033[37m"
+
+    # Bright Colors
+    BRIGHT_RED = "\033[91m"
+    BRIGHT_GREEN = "\033[92m"
+    BRIGHT_YELLOW = "\033[93m"
+    BRIGHT_BLUE = "\033[94m"
+    BRIGHT_PURPLE = "\033[95m"
+    BRIGHT_CYAN = "\033[96m"
+    BRIGHT_WHITE = "\033[97m"
+
+    # Background Colors
+    BG_RED = "\033[41m"
+    BG_GREEN = "\033[42m"
+    BG_YELLOW = "\033[43m"
+    BG_BLUE = "\033[44m"
+    BG_PURPLE = "\033[45m"
+    BG_CYAN = "\033[46m"
+    BG_WHITE = "\033[47m"
+
+    # Bright Background Colors
+    BG_BRIGHT_RED = "\033[101m"
+    BG_BRIGHT_GREEN = "\033[102m"
+    BG_BRIGHT_YELLOW = "\033[103m"
+    BG_BRIGHT_BLUE = "\033[104m"
+    BG_BRIGHT_PURPLE = "\033[105m"
+    BG_BRIGHT_CYAN = "\033[106m"
+    BG_BRIGHT_WHITE = "\033[107m"
+
+    # Text Styles
+    BOLD = "\033[1m"
+    DIM = "\033[2m"
+    ITALIC = "\033[3m"
+    UNDERLINE = "\033[4m"
+    BLINK = "\033[5m"
+    REVERSE = "\033[7m"
+    HIDDEN = "\033[8m"
+
+
+class Log:
+    logging.getLogger('oss2.api').setLevel(logging.WARNING)
+    logging.basicConfig(
+        level=logging.INFO,
+        format=f'%(asctime)s %(name)s [%(filename)s:%(lineno)d %(funcName)s]'
+               f' %(levelname)s -> %(message)s'
+    )
+    log = logging.getLogger()
+
+    def set_log_name(self, name):
+        self.__class__.log = logging.getLogger(name)
+
+    @property
+    def debug(self):
+        return self.__class__.log.debug
+
+    @property
+    def info(self):
+        return self.__class__.log.info
+
+    @property
+    def warning(self):
+        return self.__class__.log.warning
+
+    @property
+    def exception(self):
+        return self.__class__.log.exception
+
+    @property
+    def error(self):
+        return self.__class__.log.error
+
+    @property
+    def critical(self):
+        return self.__class__.log.critical
+
+
+logger = Log()
+
+
+

cobweb/base/request.py

@@ -0,0 +1,72 @@
+import random
+import requests
+
+
+class Request:
+
+    __REQUEST_ATTRS__ = {
+        "params",
+        "headers",
+        "cookies",
+        "data",
+        "json",
+        "files",
+        "auth",
+        "timeout",
+        "proxies",
+        "hooks",
+        "stream",
+        "verify",
+        "cert",
+        "allow_redirects",
+    }
+
+    def __init__(
+            self,
+            url,
+            seed,
+            random_ua=True,
+            check_status_code=True,
+            **kwargs
+    ):
+        self.url = url
+        self.seed = seed
+        self.check_status_code = check_status_code
+        self.request_setting = {}
+
+        for k, v in kwargs.items():
+            if k in self.__class__.__REQUEST_ATTRS__:
+                self.request_setting[k] = v
+                continue
+            self.__setattr__(k, v)
+
+        if not getattr(self, "method", None):
+            self.method = "POST" if self.request_setting.get("data") or self.request_setting.get("json") else "GET"
+
+        if random_ua:
+            self._build_header()
+
+    @property
+    def _random_ua(self) -> str:
+        v1 = random.randint(4, 15)
+        v2 = random.randint(3, 11)
+        v3 = random.randint(1, 16)
+        v4 = random.randint(533, 605)
+        v5 = random.randint(1000, 6000)
+        v6 = random.randint(10, 80)
+        user_agent = (f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_{v1}_{v2}) AppleWebKit/{v4}.{v3} "
+                      f"(KHTML, like Gecko) Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
+        return user_agent
+
+    def _build_header(self) -> dict:
+        if not self.request_setting.get("headers"):
+            self.request_setting["headers"] = {"accept": "*/*", "user-agent": self._random_ua}
+        elif "user-agent" not in [key.lower() for key in self.request_setting["headers"].keys()]:
+            self.request_setting["headers"]["user-agent"] = self._random_ua
+
+    def download(self) -> requests.Response:
+        response = requests.request(self.method, self.url, **self.request_setting)
+        if self.check_status_code:
+            response.raise_for_status()
+        return response
+

cobweb/base/response.py

@@ -0,0 +1,22 @@
+
+
+class Response:
+
+    def __init__(
+            self,
+            seed,
+            response,
+            **kwargs
+    ):
+        self.seed = seed
+        self.response = response
+
+        for k, v in kwargs.items():
+            self.__setattr__(k, v)
+
+    @property
+    def to_dict(self):
+        _dict = self.__dict__.copy()
+        _dict.pop('seed')
+        _dict.pop('response')
+        return _dict

cobweb/base/seed.py

@@ -0,0 +1,114 @@
+import json
+import time
+import hashlib
+
+
+class SeedParams:
+
+    def __init__(self, retry, priority, seed_version, identifier=None):
+        self.retry = retry or 0
+        self.priority = priority or 300
+        self.seed_version = seed_version or int(time.time())
+        self.identifier = identifier
+
+
+class Seed:
+
+    __SEED_PARAMS__ = [
+        "retry",
+        "priority",
+        "seed_version",
+        "identifier"
+    ]
+
+    def __init__(
+            self,
+            seed,
+            sid=None,
+            retry=None,
+            priority=None,
+            seed_version=None,
+            identifier=None,
+            **kwargs
+    ):
+        if any(isinstance(seed, t) for t in (str, bytes)):
+            try:
+                item = json.loads(seed)
+                self._init_seed(item)
+            except json.JSONDecodeError:
+                self.__setattr__("url", seed)
+        elif isinstance(seed, dict):
+            self._init_seed(seed)
+        else:
+            raise TypeError(Exception(
+                f"seed type error, "
+                f"must be str or dict! "
+                f"seed: {seed}"
+            ))
+
+        seed_params = {
+            "retry": retry,
+            "priority": priority,
+            "seed_version": seed_version,
+            "identifier": identifier
+        }
+
+        if kwargs:
+            self._init_seed(kwargs)
+            seed_params.update({
+                k:v for k, v in kwargs.items()
+                if k in self.__SEED_PARAMS__
+            })
+        if sid or not getattr(self, "sid", None):
+            self._init_id(sid)
+        self.params = SeedParams(**seed_params)
+
+    def __getattr__(self, name):
+        return None
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __str__(self):
+        return json.dumps(self.__dict__, ensure_ascii=False)
+
+    def __repr__(self):
+        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
+        return f'{self.__class__.__name__}({", ".join(chars)})'
+
+    def _init_seed(self, seed_info:dict):
+        for k, v in seed_info.items():
+            if k not in self.__SEED_PARAMS__:
+                self.__setattr__(k, v)
+
+    def _init_id(self, sid):
+        if not sid:
+            sid = hashlib.md5(self.to_string.encode()).hexdigest()
+        self.__setattr__("sid", sid)
+
+    @property
+    def to_dict(self) -> dict:
+        seed = self.__dict__.copy()
+        if seed.get("params"):
+            del seed["params"]
+        return seed
+
+    @property
+    def to_string(self) -> str:
+        return json.dumps(
+            self.to_dict,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
+
+    @property
+    def get_all(self):
+        return json.dumps(
+            self.__dict__,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
+

cobweb/constant.py

@@ -5,6 +5,11 @@ class LauncherModel:
     resident = "launcher model: resident"
 
 
+class DownloadModel:
+    common = "download model: common"
+    file = "download model: file"
+
+
 class LogModel:
     simple = "log model: simple"
     common = "log model: common"
@@ -12,18 +17,50 @@ class LogModel:
 
 
 class DealModel:
-    failure = "deal model: failure"
-    success = "deal model: success"
-    polling = "deal model: polling"
-
-
-class Setting:
-    RESET_SCORE = None
-    CHECK_LOCK_TIME = None
-    SCHEDULER_LOCK_TIME = None
-    DEAL_MODEL = None
-    LAUNCHER_MODEL = None
-    SCHEDULER_WAIT_TIME = None
-    SCHEDULER_BLOCK_TIME = None
-    SPIDER_WAIT_TIME = None
-    SPIDER_SLEEP_TIME = None
+    fail = "deal model: fail"
+    done = "deal model: done"
+    poll = "deal model: poll"
+
+
+class LogTemplate:
+
+    launcher_pro_polling = """
+----------------------- start - 轮训日志: {task} -----------------
+        内存队列 
+            种子数:  {doing_len}
+            待消费:  {todo_len}
+            已消费:  {done_len}
+        redis队列 
+            种子数:  {redis_seed_count}
+            待消费:  {redis_todo_len}
+            消费中:  {redis_doing_len}
+        存储队列 
+            待上传:  {upload_len}
+----------------------- end  - 轮训日志: {task} ------------------
+"""
+
+    download_exception = """
+----------------------- download exception -----------------------
+        种子详情 \n{detail}
+        种子参数
+            retry         :    {retry}
+            priority      :    {priority}
+            seed_version  :    {seed_version}
+            identifier    :    {identifier}
+        exception
+            msg           :    {exception}
+------------------------------------------------------------------
+"""
+
+    download_info = """
+------------------------ download info ---------------------------
+        种子详情 \n{detail}
+        种子参数
+            retry         :    {retry}
+            priority      :    {priority}
+            seed_version  :    {seed_version}
+            identifier    :    {identifier}
+        response 
+            status        :    {status} \n{response}
+------------------------------------------------------------------
+"""

cobweb/crawlers/__init__.py

@@ -0,0 +1,2 @@
+from .base_crawler import Crawler
+from .file_crawler import CrawlerAir

cobweb/crawlers/base_crawler.py

@@ -0,0 +1,121 @@
+import setting
+import threading
+
+from inspect import isgenerator
+from typing import Union, Callable, Mapping
+
+from cobweb.base import Queue, Seed, BaseItem, Request, Response, logger
+from cobweb.constant import DealModel, LogTemplate
+from cobweb.utils import download_log_info
+
+
+class Crawler(threading.Thread):
+
+    def __init__(
+            self,
+            upload_queue: Queue,
+            custom_func: Union[Mapping[str, Callable]],
+            launcher_queue: Union[Mapping[str, Queue]],
+    ):
+        super().__init__()
+
+        self.upload_queue = upload_queue
+        for func_name, _callable in custom_func.items():
+            if isinstance(_callable, Callable):
+                self.__setattr__(func_name, _callable)
+
+        self.launcher_queue = launcher_queue
+
+        self.spider_thread_num = setting.SPIDER_THREAD_NUM
+        self.max_retries = setting.SPIDER_MAX_RETRIES
+
+    @staticmethod
+    def request(seed: Seed) -> Union[Request, BaseItem]:
+        stream = True if setting.DOWNLOAD_MODEL else False
+        return Request(seed.url, seed, stream=stream, timeout=5)
+
+    @staticmethod
+    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+        response = item.download()
+        yield Response(item.seed, response)
+
+    @staticmethod
+    def parse(item: Response) -> BaseItem:
+        pass
+
+    def get(self) -> Seed:
+        return self.launcher_queue['todo'].pop()
+
+    def spider(self):
+        while True:
+            seed = self.get()
+
+            if not seed:
+                continue
+
+            elif seed.params.retry >= self.max_retries:
+                self.launcher_queue['done'].push(seed)
+                continue
+
+            item = self.request(seed)
+
+            if isinstance(item, Request):
+
+                download_iterators = self.download(item)
+
+                if not isgenerator(download_iterators):
+                    raise TypeError("download function isn't a generator")
+
+                seed_detail_log_info = download_log_info(seed.to_dict)
+
+                try:
+                    for it in download_iterators:
+                        if isinstance(it, Response):
+                            response_detail_log_info = download_log_info(it.to_dict())
+                            logger.info(LogTemplate.download_info.format(
+                                detail=seed_detail_log_info, retry=item.seed.params.retry,
+                                priority=item.seed.params.priority,
+                                seed_version=item.seed.params.seed_version,
+                                identifier=item.seed.params.identifier,
+                                status=it.response, response=response_detail_log_info
+                            ))
+                            parse_iterators = self.parse(it)
+                            if not isgenerator(parse_iterators):
+                                raise TypeError("parse function isn't a generator")
+                            for upload_item in parse_iterators:
+                                if not isinstance(upload_item, BaseItem):
+                                    raise TypeError("upload_item isn't BaseItem subclass")
+                                self.upload_queue.push(upload_item)
+                        elif isinstance(it, BaseItem):
+                            self.upload_queue.push(it)
+                        elif isinstance(it, Seed):
+                            self.launcher_queue['new'].push(it)
+                        elif isinstance(it, str) and it == DealModel.poll:
+                            self.launcher_queue['todo'].push(item)
+                            break
+                        elif isinstance(it, str) and it == DealModel.done:
+                            self.launcher_queue['done'].push(seed)
+                            break
+                        elif isinstance(it, str) and it == DealModel.fail:
+                            seed.params.identifier = DealModel.fail
+                            self.launcher_queue['done'].push(seed)
+                            break
+                        else:
+                            raise TypeError("yield value type error!")
+
+                except Exception as e:
+                    logger.info(LogTemplate.download_exception.format(
+                        detail=seed_detail_log_info, retry=seed.params.retry,
+                        priority=seed.params.priority, seed_version=seed.params.seed_version,
+                        identifier=seed.params.identifier, exception=e
+                    ))
+                    seed.params.retry += 1
+                    self.launcher_queue['todo'].push(seed)
+
+            elif isinstance(item, BaseItem):
+                self.upload_queue.push(item)
+
+    def run(self):
+        for index in range(self.spider_thread_num):
+            threading.Thread(name=f"spider_{index}", target=self.spider).start()
+