cobweb-launcher 1.3.4__tar.gz → 1.3.6__tar.gz

{cobweb-launcher-1.3.4 → cobweb-launcher-1.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.4
+Version: 1.3.6
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.4 → cobweb-launcher-1.3.6}/cobweb/launchers/__init__.py

@@ -1,3 +1,3 @@
 # from .launcher_air import LauncherAir
 from .launcher_pro import LauncherPro
-# from .launcher_api import LauncherApi
+from .launcher_api import LauncherApi

{cobweb-launcher-1.3.4 → cobweb-launcher-1.3.6}/cobweb/launchers/launcher_api.py

@@ -5,7 +5,7 @@ from cobweb.schedulers import ApiScheduler
 from .launcher import Launcher
 
 
-class LauncherPro(Launcher):
+class LauncherApi(Launcher):
 
     def __init__(self, task, project, custom_setting=None, **kwargs):
         super().__init__(task, project, custom_setting, **kwargs)

{cobweb-launcher-1.3.4 → cobweb-launcher-1.3.6}/cobweb/utils/__init__.py

@@ -1,5 +1,5 @@
 from .oss import OssUtil
 from .tools import *
 from .bloom import BloomFilter
-from .task_queue import TaskQueue
+# from .task_queue import TaskQueue
 

{cobweb-launcher-1.3.4 → cobweb-launcher-1.3.6}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.4
+Version: 1.3.6
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-1.3.4 → cobweb-launcher-1.3.6}/cobweb_launcher.egg-info/SOURCES.txt

@@ -36,38 +36,6 @@ cobweb/utils/__init__.py
 cobweb/utils/bloom.py
 cobweb/utils/oss.py
 cobweb/utils/tools.py
-cobweb_/__init__.py
-cobweb_/constant.py
-cobweb_/setting.py
-cobweb_/base/__init__.py
-cobweb_/base/common_queue.py
-cobweb_/base/decorators.py
-cobweb_/base/item.py
-cobweb_/base/log.py
-cobweb_/base/request.py
-cobweb_/base/response.py
-cobweb_/base/seed.py
-cobweb_/crawlers/__init__.py
-cobweb_/crawlers/crawler.py
-cobweb_/db/__init__.py
-cobweb_/db/api_db.py
-cobweb_/db/redis_db.py
-cobweb_/exceptions/__init__.py
-cobweb_/exceptions/oss_db_exception.py
-cobweb_/launchers/__init__.py
-cobweb_/launchers/launcher.py
-cobweb_/launchers/launcher_air.py
-cobweb_/launchers/launcher_api.py
-cobweb_/launchers/launcher_pro.py
-cobweb_/pipelines/__init__.py
-cobweb_/pipelines/pipeline.py
-cobweb_/pipelines/pipeline_console.py
-cobweb_/pipelines/pipeline_loghub.py
-cobweb_/utils/__init__.py
-cobweb_/utils/bloom.py
-cobweb_/utils/dotting.py
-cobweb_/utils/oss.py
-cobweb_/utils/tools.py
 cobweb_launcher.egg-info/PKG-INFO
 cobweb_launcher.egg-info/SOURCES.txt
 cobweb_launcher.egg-info/dependency_links.txt

cobweb-launcher-1.3.6/cobweb_launcher.egg-info/top_level.txt

@@ -0,0 +1 @@
+cobweb

{cobweb-launcher-1.3.4 → cobweb-launcher-1.3.6}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="1.3.4",
+    version="1.3.6",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",

cobweb-launcher-1.3.4/cobweb_/__init__.py

@@ -1,2 +0,0 @@
-from .launchers import LauncherAir, LauncherPro, LauncherApi
-from .constant import CrawlerModel

cobweb-launcher-1.3.4/cobweb_/base/__init__.py

@@ -1,9 +0,0 @@
-from .common_queue import Queue
-from .response import Response
-from .request import Request
-from .item import BaseItem, ConsoleItem
-from .seed import Seed
-
-from .log import logger
-from .decorators import decorator_oss_db
-

cobweb-launcher-1.3.4/cobweb_/base/common_queue.py

@@ -1,30 +0,0 @@
-from collections import deque
-
-
-class Queue:
-
-    def __init__(self):
-        self._queue = deque()
-
-    @property
-    def length(self) -> int:
-        return len(self._queue)
-
-    def push(self, data, left: bool = False, direct_insertion: bool = False):
-        try:
-            if not data:
-                return None
-            if not direct_insertion and any(isinstance(data, t) for t in (list, tuple)):
-                self._queue.extendleft(data) if left else self._queue.extend(data)
-            else:
-                self._queue.appendleft(data) if left else self._queue.append(data)
-        except AttributeError:
-            pass
-
-    def pop(self, left: bool = True):
-        try:
-            return self._queue.popleft() if left else self._queue.pop()
-        except IndexError:
-            return None
-        except AttributeError:
-            return None

cobweb-launcher-1.3.4/cobweb_/base/decorators.py

@@ -1,40 +0,0 @@
-from functools import wraps
-
-
-# def check_redis_status(func):
-#     @wraps(func)
-#     def wrapper(*args, **kwargs):
-#         try:
-#             result = func(*args, **kwargs)
-#         except Exception:
-#             result = False
-#         return result
-#
-#     return wrapper
-
-
-def decorator_oss_db(exception, retries=3):
-    def decorator(func):
-        @wraps(func)
-        def wrapper(callback_func, *args, **kwargs):
-            result = None
-            for i in range(retries):
-                msg = None
-                try:
-                    return func(callback_func, *args, **kwargs)
-                except Exception as e:
-                    result = None
-                    msg = e
-                finally:
-                    if result:
-                        return result
-
-                    if i >= 2 and msg:
-                        raise exception(msg)
-
-        return wrapper
-
-    return decorator
-
-
-

cobweb-launcher-1.3.4/cobweb_/base/item.py

@@ -1,46 +0,0 @@
-from .seed import Seed
-from collections import namedtuple
-
-
-class Item(type):
-
-    def __new__(cls, name, bases, dct):
-        new_class_instance = type.__new__(cls, name, bases, dct)
-        if name != "BaseItem":
-            table = getattr(new_class_instance, "__TABLE__")
-            fields = getattr(new_class_instance, "__FIELDS__")
-            new_class_instance.Data = namedtuple(table, fields)
-        return new_class_instance
-
-
-class BaseItem(metaclass=Item):
-
-    __TABLE__ = ""
-    __FIELDS__ = ""
-
-    def __init__(self, seed: Seed, **kwargs):
-        self.seed = seed
-
-        data = {}
-        for key, value in kwargs.items():
-            if key not in self.__FIELDS__:
-                self.__setattr__(key, value)
-            else:
-                data[key] = value
-
-        self.data = self.Data(**data)
-
-    @property
-    def to_dict(self):
-        return self.data._asdict()
-
-    @property
-    def table(self):
-        return self.Data.__name__
-
-
-class ConsoleItem(BaseItem):
-
-    __TABLE__ = "console"
-    __FIELDS__ = "data"
-

cobweb-launcher-1.3.4/cobweb_/base/log.py

@@ -1,94 +0,0 @@
-import logging
-
-
-class ColorCodes:
-    # Text Reset
-    RESET = "\033[0m"
-
-    # Regular Colors
-    RED = "\033[31m"
-    GREEN = "\033[32m"
-    YELLOW = "\033[33m"
-    BLUE = "\033[34m"
-    PURPLE = "\033[35m"
-    CYAN = "\033[36m"
-    WHITE = "\033[37m"
-
-    # Bright Colors
-    BRIGHT_RED = "\033[91m"
-    BRIGHT_GREEN = "\033[92m"
-    BRIGHT_YELLOW = "\033[93m"
-    BRIGHT_BLUE = "\033[94m"
-    BRIGHT_PURPLE = "\033[95m"
-    BRIGHT_CYAN = "\033[96m"
-    BRIGHT_WHITE = "\033[97m"
-
-    # Background Colors
-    BG_RED = "\033[41m"
-    BG_GREEN = "\033[42m"
-    BG_YELLOW = "\033[43m"
-    BG_BLUE = "\033[44m"
-    BG_PURPLE = "\033[45m"
-    BG_CYAN = "\033[46m"
-    BG_WHITE = "\033[47m"
-
-    # Bright Background Colors
-    BG_BRIGHT_RED = "\033[101m"
-    BG_BRIGHT_GREEN = "\033[102m"
-    BG_BRIGHT_YELLOW = "\033[103m"
-    BG_BRIGHT_BLUE = "\033[104m"
-    BG_BRIGHT_PURPLE = "\033[105m"
-    BG_BRIGHT_CYAN = "\033[106m"
-    BG_BRIGHT_WHITE = "\033[107m"
-
-    # Text Styles
-    BOLD = "\033[1m"
-    DIM = "\033[2m"
-    ITALIC = "\033[3m"
-    UNDERLINE = "\033[4m"
-    BLINK = "\033[5m"
-    REVERSE = "\033[7m"
-    HIDDEN = "\033[8m"
-
-
-class Log:
-    logging.getLogger('oss2.api').setLevel(logging.WARNING)
-    logging.basicConfig(
-        level=logging.INFO,
-        format=f'%(asctime)s %(name)s [%(filename)s:%(lineno)d %(funcName)s]'
-               f' %(levelname)s -> %(message)s'
-    )
-    log = logging.getLogger()
-
-    def set_log_name(self, name):
-        self.__class__.log = logging.getLogger(name)
-
-    @property
-    def debug(self):
-        return self.__class__.log.debug
-
-    @property
-    def info(self):
-        return self.__class__.log.info
-
-    @property
-    def warning(self):
-        return self.__class__.log.warning
-
-    @property
-    def exception(self):
-        return self.__class__.log.exception
-
-    @property
-    def error(self):
-        return self.__class__.log.error
-
-    @property
-    def critical(self):
-        return self.__class__.log.critical
-
-
-logger = Log()
-
-
-

cobweb-launcher-1.3.4/cobweb_/base/request.py

@@ -1,82 +0,0 @@
-import random
-import requests
-
-
-class Request:
-
-    __REQUEST_ATTRS__ = {
-        "params",
-        "headers",
-        "cookies",
-        "data",
-        "json",
-        "files",
-        "auth",
-        "timeout",
-        "proxies",
-        "hooks",
-        "stream",
-        "verify",
-        "cert",
-        "allow_redirects",
-    }
-
-    def __init__(
-            self,
-            url,
-            seed,
-            random_ua=True,
-            check_status_code=True,
-            **kwargs
-    ):
-        self.url = url
-        self.seed = seed
-        self.check_status_code = check_status_code
-        self.request_setting = {}
-
-        for k, v in kwargs.items():
-            if k in self.__class__.__REQUEST_ATTRS__:
-                self.request_setting[k] = v
-                continue
-            self.__setattr__(k, v)
-
-        if not getattr(self, "method", None):
-            self.method = "POST" if self.request_setting.get("data") or self.request_setting.get("json") else "GET"
-
-        if random_ua:
-            self._build_header()
-
-    @property
-    def _random_ua(self) -> str:
-        v1 = random.randint(4, 15)
-        v2 = random.randint(3, 11)
-        v3 = random.randint(1, 16)
-        v4 = random.randint(533, 605)
-        v5 = random.randint(1000, 6000)
-        v6 = random.randint(10, 80)
-        user_agent = (f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_{v1}_{v2}) AppleWebKit/{v4}.{v3} "
-                      f"(KHTML, like Gecko) Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
-        return user_agent
-
-    def _build_header(self) -> dict:
-        if not self.request_setting.get("headers"):
-            self.request_setting["headers"] = {"accept": "*/*", "user-agent": self._random_ua}
-        elif "user-agent" not in [key.lower() for key in self.request_setting["headers"].keys()]:
-            self.request_setting["headers"]["user-agent"] = self._random_ua
-
-    def download(self) -> requests.Response:
-        response = requests.request(self.method, self.url, **self.request_setting)
-        if self.check_status_code:
-            response.raise_for_status()
-        return response
-
-    @property
-    def to_dict(self):
-        _dict = self.__dict__.copy()
-        _dict.pop('url')
-        _dict.pop('seed')
-        _dict.pop('check_status_code')
-        _dict.pop('request_setting')
-        return _dict
-
-

cobweb-launcher-1.3.4/cobweb_/base/response.py

@@ -1,23 +0,0 @@
-
-
-class Response:
-
-    def __init__(
-            self,
-            seed,
-            response,
-            **kwargs
-    ):
-        self.seed = seed
-        self.response = response
-
-        for k, v in kwargs.items():
-            self.__setattr__(k, v)
-
-    @property
-    def to_dict(self):
-        _dict = self.__dict__.copy()
-        _dict.pop('seed')
-        _dict.pop('response')
-        return _dict
-

cobweb-launcher-1.3.4/cobweb_/base/seed.py

@@ -1,114 +0,0 @@
-import json
-import time
-import hashlib
-
-
-class SeedParams:
-
-    def __init__(self, retry, priority, seed_version, seed_status=None):
-        self.retry = retry or 0
-        self.priority = priority or 300
-        self.seed_version = seed_version or int(time.time())
-        self.seed_status = seed_status
-
-
-class Seed:
-
-    __SEED_PARAMS__ = [
-        "retry",
-        "priority",
-        "seed_version",
-        "seed_status"
-    ]
-
-    def __init__(
-            self,
-            seed,
-            sid=None,
-            retry=None,
-            priority=None,
-            seed_version=None,
-            seed_status=None,
-            **kwargs
-    ):
-        if any(isinstance(seed, t) for t in (str, bytes)):
-            try:
-                item = json.loads(seed)
-                self._init_seed(item)
-            except json.JSONDecodeError:
-                self.__setattr__("url", seed)
-        elif isinstance(seed, dict):
-            self._init_seed(seed)
-        else:
-            raise TypeError(Exception(
-                f"seed type error, "
-                f"must be str or dict! "
-                f"seed: {seed}"
-            ))
-
-        seed_params = {
-            "retry": retry,
-            "priority": priority,
-            "seed_version": seed_version,
-            "seed_status": seed_status,
-        }
-
-        if kwargs:
-            self._init_seed(kwargs)
-            seed_params.update({
-                k:v for k, v in kwargs.items()
-                if k in self.__SEED_PARAMS__
-            })
-        if sid or not getattr(self, "sid", None):
-            self._init_id(sid)
-        self.params = SeedParams(**seed_params)
-
-    def __getattr__(self, name):
-        return None
-
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-    def __getitem__(self, item):
-        return getattr(self, item)
-
-    def __str__(self):
-        return json.dumps(self.__dict__, ensure_ascii=False)
-
-    def __repr__(self):
-        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
-        return f'{self.__class__.__name__}({", ".join(chars)})'
-
-    def _init_seed(self, seed_info:dict):
-        for k, v in seed_info.items():
-            if k not in self.__SEED_PARAMS__:
-                self.__setattr__(k, v)
-
-    def _init_id(self, sid):
-        if not sid:
-            sid = hashlib.md5(self.to_string.encode()).hexdigest()
-        self.__setattr__("sid", sid)
-
-    @property
-    def to_dict(self) -> dict:
-        seed = self.__dict__.copy()
-        if seed.get("params"):
-            del seed["params"]
-        return seed
-
-    @property
-    def to_string(self) -> str:
-        return json.dumps(
-            self.to_dict,
-            ensure_ascii=False,
-            separators=(",", ":")
-        )
-
-    @property
-    def get_all(self):
-        return json.dumps(
-            self.__dict__,
-            ensure_ascii=False,
-            separators=(",", ":")
-        )
-

cobweb-launcher-1.3.4/cobweb_/constant.py

@@ -1,94 +0,0 @@
-
-class CrawlerModel:
-
-    default = "cobweb.crawlers.Crawler"
-    file_air = "cobweb.crawlers.FileCrawlerAir"
-    file_pro = "cobweb.crawlers.FileCrawlerPro"
-
-
-class LauncherModel:
-    task = "launcher model: task"
-    resident = "launcher model: resident"
-
-
-class DownloadModel:
-    common = "download model: common"
-    file = "download model: file"
-
-
-class LogModel:
-    simple = "log model: simple"
-    common = "log model: common"
-    detailed = "log model: detailed"
-
-
-class DealModel:
-    fail = "deal model: fail"
-    done = "deal model: done"
-    poll = "deal model: poll"
-
-
-class LogTemplate:
-
-    console_item = """
-    ----------------------- start - console pipeline -----------------
-            种子详情 \n{seed_detail}
-            解析详情 \n{parse_detail}
-    ----------------------- end  - console pipeline ------------------
-    """
-
-    launcher_air_polling = """
-    ----------------------- start - 轮训日志: {task} -----------------
-            内存队列 
-                种子数:  {doing_len}
-                待消费:  {todo_len}
-                已消费:  {done_len}
-            存储队列 
-                待上传:  {upload_len}
-    ----------------------- end  - 轮训日志: {task} ------------------
-    """
-
-    launcher_pro_polling = """
------------------------ start - 轮训日志: {task} -----------------
-        内存队列 
-            种子数:  {doing_len}
-            待消费:  {todo_len}
-            已消费:  {done_len}
-        redis队列 
-            种子数:  {redis_seed_count}
-            待消费:  {redis_todo_len}
-            消费中:  {redis_doing_len}
-        存储队列 
-            待上传:  {upload_len}
------------------------ end  - 轮训日志: {task} ------------------
-"""
-
-    download_exception = """
------------------------ download exception -----------------------
-        种子详情 \n{detail}
-        种子参数
-            retry         :    {retry}
-            priority      :    {priority}
-            seed_version  :    {seed_version}
-            identifier    :    {identifier}
-        exception
-            msg           :    {exception}
-------------------------------------------------------------------
-"""
-
-    download_info = """
------------------------- download info ---------------------------
-        种子详情 \n{detail}
-        种子参数
-            retry         :    {retry}
-            priority      :    {priority}
-            seed_version  :    {seed_version}
-            identifier    :    {identifier}
-        response 
-            status        :    {status} \n{response}
-------------------------------------------------------------------
-"""
-
-    @staticmethod
-    def log_info(item: dict) -> str:
-        return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

cobweb-launcher-1.3.4/cobweb_/crawlers/__init__.py

@@ -1 +0,0 @@
-from .crawler import Crawler