cobweb-launcher 1.2.49__py3-none-any.whl → 1.3.2__py3-none-any.whl

cobweb/pipelines/pipeline.py

@@ -2,68 +2,44 @@ import time
 import threading
 
 from abc import ABC, abstractmethod
-from cobweb.base import BaseItem, Queue, logger
+from cobweb.base import BaseItem, TaskQueue, logger, Decorators
+from cobweb import setting
 
 
-class Pipeline(threading.Thread, ABC):
+class Pipeline(ABC):
 
     def __init__(
             self,
-            stop: threading.Event,
             pause: threading.Event,
-            upload: Queue, done: Queue,
-            upload_size: int,
-            wait_seconds: int
     ):
         super().__init__()
-        self._stop = stop
-        self._pause = pause
-        self._upload = upload
-        self._done = done
-
-        self.upload_size = upload_size
-        self.wait_seconds = wait_seconds
+        self.pause = pause
+        self.upload_queue_size = setting.UPLOAD_QUEUE_SIZE
+        self.upload_wait_time = setting.UPLOAD_WAIT_TIME
 
     @abstractmethod
     def build(self, item: BaseItem) -> dict:
-        pass
+        ...
 
     @abstractmethod
     def upload(self, table: str, data: list) -> bool:
-        pass
+        ...
 
+    @Decorators.pause
     def run(self):
-        while not self._stop.is_set():
-            if not self._upload.length:
-                time.sleep(self.wait_seconds)
-                continue
-            if self._upload.length < self.upload_size:
-                time.sleep(self.wait_seconds)
-            status = True
-            data_info, seeds = {}, []
-            try:
-                for _ in range(self.upload_size):
-                    item = self._upload.pop()
-                    if not item:
-                        break
-                    seeds.append(item.seed)
-                    data = self.build(item)
-                    data_info.setdefault(item.table, []).append(data)
-                for table, datas in data_info.items():
-                    try:
-                        self.upload(table, datas)
-                    except Exception as e:
-                        logger.info(e)
-                        status = False
-            except Exception as e:
-                logger.info(e)
-                status = False
-            if not status:
-                for seed in seeds:
-                    seed.params.seed_status = "deal model: fail"
-            if seeds:
-                self._done.push(seeds)
-
-        logger.info("upload pipeline close!")
-
-
+        data_info, seeds = {}, []
+        thread_sleep = self.upload_wait_time if TaskQueue.UPLOAD.length < self.upload_queue_size else 0.1
+        try:
+            while item := TaskQueue.UPLOAD.pop() and len(seeds) <= self.upload_queue_size:
+                data = self.build(item)
+                data_info.setdefault(item.table, []).append(data)
+                seeds.append(item.seed)
+            for table, datas in data_info.items():
+                self.upload(table, datas)
+        except Exception as e:
+            logger.info(e)
+            seeds = None
+        finally:
+            TaskQueue.DONE.push(seeds)
+
+        time.sleep(thread_sleep)

cobweb/schedulers/__init__.py

@@ -1,3 +1 @@
 from .scheduler_redis import RedisScheduler
-from .scheduler_api import ApiScheduler
-

cobweb/schedulers/scheduler_redis.py

@@ -33,7 +33,7 @@ class RedisScheduler:
             members = self._db.members(key, source, count=count, _min=0, _max="(1000")
             for member, priority in members:
                 # seed = Seed(member, priority=priority)
-                yield member, priority
+                yield member.decode(), priority
 
     def insert(self, key, items):
         if items:
@@ -48,8 +48,7 @@ class RedisScheduler:
             _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
 
             for key in keys:
-                if self._db.exists(key):
-                    self._db.members(key, 0, _min=_min, _max="(0")
+                self._db.members(key, 0, _min=_min, _max="(0")
 
             if not self.heartbeat:
                 self.working.set()
@@ -59,13 +58,11 @@ class RedisScheduler:
 
     def refresh(self, key, items: dict[str, int]):
         refresh_time = int(time.time())
-        its = {k: -refresh_time - v / 1000 for k, v in items.items()}
-        if its:
-            self._db.zadd(key, item=its, xx=True)
+        its = {k: -refresh_time - v / 1000 for k, v in items}
+        self._db.zadd(key, item=its, xx=True)
 
     def delete(self, key, values):
-        if values:
-            self._db.zrem(key, *values)
+        self._db.zrem(key, *values)
 
 
 

cobweb/setting.py

@@ -37,8 +37,14 @@ PIPELINE = "cobweb.pipelines.pipeline_console.Console"
 # Launcher 等待时间
 
 BEFORE_SCHEDULER_WAIT_SECONDS = 60  # 调度前等待时间,只作用于单次任务
-SCHEDULER_WAIT_SECONDS = 15  # 调度等待时间
-TODO_QUEUE_FULL_WAIT_SECONDS = 5  # todo队列已满时等待时间
+
+SCHEDULING_WAIT_TIME = 15  # SCHEDULER ITEM 调度等待时间
+INSERTING_WAIT_TIME = 30   # INSERT ITEM 等待时间
+REMOVING_WAIT_TIME = 5     # REMOVE ITEM 等待时间
+RESET_WAIT_TIME = 30       # REST ITEM 等待时间
+UPLOAD_WAIT_TIME = 15      # 上传等待时间
+
+TODO_QUEUE_FULL_WAIT_SECONDS = 5  # 队列已满时等待时间
 NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
 DONE_QUEUE_WAIT_SECONDS = 5   # done队列等待时间
 UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
@@ -46,10 +52,27 @@ SEED_RESET_SECONDS = 30   # 种子重制时间
 
 
 # Launcher 队列长度
-TODO_QUEUE_SIZE = 100  # todo队列长度
-NEW_QUEUE_MAX_SIZE = 100  # new队列长度
-DONE_QUEUE_MAX_SIZE = 100  # done队列长度
-UPLOAD_QUEUE_MAX_SIZE = 100  # upload队列长度
+SCHEDULING_SIZE = 100   # 调度队列长度
+INSERTING_SIZE = 100    # INSERT 长度
+REMOVING_SIZE = 100     # REMOVE 长度
+
+# SEED = Queue()          # 添加任务种子队列
+# TODO = Queue()          # 任务种子队列
+# REQUEST = Queue()       # 请求队列
+# DOWNLOAD = Queue()      # 下载任务队列
+# RESPONSE = Queue()      # 响应队列
+# DONE = Queue()          # 下载完成队列
+# UPLOAD = Queue()        # 任务上传队列
+# DELETE = Queue()        # 任务删除队列
+
+SEED_QUEUE_SIZE = 100       # TODO 队列长度
+TODO_QUEUE_SIZE = 100       # TODO 队列长度
+REQUEST_QUEUE_SIZE = 100    # new队列长度
+DOWNLOAD_QUEUE_SIZE = 100   # done队列长度
+RESPONSE_QUEUE_SIZE = 100   # upload队列长度
+DONE_QUEUE_SIZE = 100       # upload队列长度
+UPLOAD_QUEUE_SIZE = 100     # upload队列长度
+DELETE_QUEUE_SIZE = 100     # upload队列长度
 
 # DONE_MODEL IN (0, 1), 种子完成模式
 DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加至失败队列；1:种子消费成功添加至成功队列，失败添加至失败队列

cobweb/utils/dotting.py

@@ -1,9 +1,6 @@
 import json
-import time
 
 from aliyun.log import LogClient, LogItem, PutLogsRequest
-
-from cobweb.base import Queue, logger
 from cobweb import setting
 
 
@@ -11,11 +8,11 @@ class LoghubDot:
 
     def __init__(self):
         self.client = LogClient(**setting.LOGHUB_CONFIG)
-        self.queue = Queue()
 
     def build(self, topic, **kwargs):
 
         temp = {}
+        log_items = []
         log_item = LogItem()
         for key, value in kwargs.items():
             if not isinstance(value, str):
@@ -24,41 +21,12 @@ class LoghubDot:
                 temp[key] = value
         contents = sorted(temp.items())
         log_item.set_contents(contents)
-        # log_items.append(log_item)
-        # request = PutLogsRequest(
-        #     project="databee-download-log",
-        #     logstore="log",
-        #     topic=topic,
-        #     logitems=log_items,
-        #     compress=True
-        # )
-        self.queue.push((topic, log_item), direct_insertion=True)
-        # self.client.put_logs(request=request)
-
-    def build_run(self):
-        while True:
-            start_time = int(time.time())
-            while True:
-                cost_time = int(time.time()) - start_time
-                if self.queue.length >= 1000 or cost_time > 10:
-                    break
-                time.sleep(0.5)
-            try:
-                log_item_info = {}
-                for _ in range(1000):
-                    its = self.queue.pop()
-                    if not its:
-                        break
-                    topic, item = its
-                    log_item_info.setdefault(topic, []).append(item)
-                for topic, log_items in log_item_info.items():
-                    request = PutLogsRequest(
-                        project="databee-download-log",
-                        logstore="log",
-                        topic=topic,
-                        logitems=log_items,
-                        compress=True
-                    )
-                    self.client.put_logs(request=request)
-            except Exception as e:
-                logger.info(str(e))
+        log_items.append(log_item)
+        request = PutLogsRequest(
+            project="databee-download-log",
+            logstore="cobweb_log",
+            topic=topic,
+            logitems=log_items,
+            compress=True
+        )
+        self.client.put_logs(request=request)

cobweb_/__init__.py

@@ -0,0 +1,2 @@
+from .launchers import LauncherAir, LauncherPro, LauncherApi
+from .constant import CrawlerModel

cobweb_/base/__init__.py

@@ -0,0 +1,9 @@
+from .common_queue import Queue
+from .response import Response
+from .request import Request
+from .item import BaseItem, ConsoleItem
+from .seed import Seed
+
+from .log import logger
+from .decorators import decorator_oss_db
+

cobweb_/base/common_queue.py

@@ -0,0 +1,30 @@
+from collections import deque
+
+
+class Queue:
+
+    def __init__(self):
+        self._queue = deque()
+
+    @property
+    def length(self) -> int:
+        return len(self._queue)
+
+    def push(self, data, left: bool = False, direct_insertion: bool = False):
+        try:
+            if not data:
+                return None
+            if not direct_insertion and any(isinstance(data, t) for t in (list, tuple)):
+                self._queue.extendleft(data) if left else self._queue.extend(data)
+            else:
+                self._queue.appendleft(data) if left else self._queue.append(data)
+        except AttributeError:
+            pass
+
+    def pop(self, left: bool = True):
+        try:
+            return self._queue.popleft() if left else self._queue.pop()
+        except IndexError:
+            return None
+        except AttributeError:
+            return None

cobweb_/base/decorators.py

@@ -0,0 +1,40 @@
+from functools import wraps
+
+
+# def check_redis_status(func):
+#     @wraps(func)
+#     def wrapper(*args, **kwargs):
+#         try:
+#             result = func(*args, **kwargs)
+#         except Exception:
+#             result = False
+#         return result
+#
+#     return wrapper
+
+
+def decorator_oss_db(exception, retries=3):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(callback_func, *args, **kwargs):
+            result = None
+            for i in range(retries):
+                msg = None
+                try:
+                    return func(callback_func, *args, **kwargs)
+                except Exception as e:
+                    result = None
+                    msg = e
+                finally:
+                    if result:
+                        return result
+
+                    if i >= 2 and msg:
+                        raise exception(msg)
+
+        return wrapper
+
+    return decorator
+
+
+

cobweb_/base/item.py

@@ -0,0 +1,46 @@
+from .seed import Seed
+from collections import namedtuple
+
+
+class Item(type):
+
+    def __new__(cls, name, bases, dct):
+        new_class_instance = type.__new__(cls, name, bases, dct)
+        if name != "BaseItem":
+            table = getattr(new_class_instance, "__TABLE__")
+            fields = getattr(new_class_instance, "__FIELDS__")
+            new_class_instance.Data = namedtuple(table, fields)
+        return new_class_instance
+
+
+class BaseItem(metaclass=Item):
+
+    __TABLE__ = ""
+    __FIELDS__ = ""
+
+    def __init__(self, seed: Seed, **kwargs):
+        self.seed = seed
+
+        data = {}
+        for key, value in kwargs.items():
+            if key not in self.__FIELDS__:
+                self.__setattr__(key, value)
+            else:
+                data[key] = value
+
+        self.data = self.Data(**data)
+
+    @property
+    def to_dict(self):
+        return self.data._asdict()
+
+    @property
+    def table(self):
+        return self.Data.__name__
+
+
+class ConsoleItem(BaseItem):
+
+    __TABLE__ = "console"
+    __FIELDS__ = "data"
+

cobweb_/base/log.py

@@ -0,0 +1,94 @@
+import logging
+
+
+class ColorCodes:
+    # Text Reset
+    RESET = "\033[0m"
+
+    # Regular Colors
+    RED = "\033[31m"
+    GREEN = "\033[32m"
+    YELLOW = "\033[33m"
+    BLUE = "\033[34m"
+    PURPLE = "\033[35m"
+    CYAN = "\033[36m"
+    WHITE = "\033[37m"
+
+    # Bright Colors
+    BRIGHT_RED = "\033[91m"
+    BRIGHT_GREEN = "\033[92m"
+    BRIGHT_YELLOW = "\033[93m"
+    BRIGHT_BLUE = "\033[94m"
+    BRIGHT_PURPLE = "\033[95m"
+    BRIGHT_CYAN = "\033[96m"
+    BRIGHT_WHITE = "\033[97m"
+
+    # Background Colors
+    BG_RED = "\033[41m"
+    BG_GREEN = "\033[42m"
+    BG_YELLOW = "\033[43m"
+    BG_BLUE = "\033[44m"
+    BG_PURPLE = "\033[45m"
+    BG_CYAN = "\033[46m"
+    BG_WHITE = "\033[47m"
+
+    # Bright Background Colors
+    BG_BRIGHT_RED = "\033[101m"
+    BG_BRIGHT_GREEN = "\033[102m"
+    BG_BRIGHT_YELLOW = "\033[103m"
+    BG_BRIGHT_BLUE = "\033[104m"
+    BG_BRIGHT_PURPLE = "\033[105m"
+    BG_BRIGHT_CYAN = "\033[106m"
+    BG_BRIGHT_WHITE = "\033[107m"
+
+    # Text Styles
+    BOLD = "\033[1m"
+    DIM = "\033[2m"
+    ITALIC = "\033[3m"
+    UNDERLINE = "\033[4m"
+    BLINK = "\033[5m"
+    REVERSE = "\033[7m"
+    HIDDEN = "\033[8m"
+
+
+class Log:
+    logging.getLogger('oss2.api').setLevel(logging.WARNING)
+    logging.basicConfig(
+        level=logging.INFO,
+        format=f'%(asctime)s %(name)s [%(filename)s:%(lineno)d %(funcName)s]'
+               f' %(levelname)s -> %(message)s'
+    )
+    log = logging.getLogger()
+
+    def set_log_name(self, name):
+        self.__class__.log = logging.getLogger(name)
+
+    @property
+    def debug(self):
+        return self.__class__.log.debug
+
+    @property
+    def info(self):
+        return self.__class__.log.info
+
+    @property
+    def warning(self):
+        return self.__class__.log.warning
+
+    @property
+    def exception(self):
+        return self.__class__.log.exception
+
+    @property
+    def error(self):
+        return self.__class__.log.error
+
+    @property
+    def critical(self):
+        return self.__class__.log.critical
+
+
+logger = Log()
+
+
+

cobweb_/base/request.py

@@ -0,0 +1,82 @@
+import random
+import requests
+
+
+class Request:
+
+    __REQUEST_ATTRS__ = {
+        "params",
+        "headers",
+        "cookies",
+        "data",
+        "json",
+        "files",
+        "auth",
+        "timeout",
+        "proxies",
+        "hooks",
+        "stream",
+        "verify",
+        "cert",
+        "allow_redirects",
+    }
+
+    def __init__(
+            self,
+            url,
+            seed,
+            random_ua=True,
+            check_status_code=True,
+            **kwargs
+    ):
+        self.url = url
+        self.seed = seed
+        self.check_status_code = check_status_code
+        self.request_setting = {}
+
+        for k, v in kwargs.items():
+            if k in self.__class__.__REQUEST_ATTRS__:
+                self.request_setting[k] = v
+                continue
+            self.__setattr__(k, v)
+
+        if not getattr(self, "method", None):
+            self.method = "POST" if self.request_setting.get("data") or self.request_setting.get("json") else "GET"
+
+        if random_ua:
+            self._build_header()
+
+    @property
+    def _random_ua(self) -> str:
+        v1 = random.randint(4, 15)
+        v2 = random.randint(3, 11)
+        v3 = random.randint(1, 16)
+        v4 = random.randint(533, 605)
+        v5 = random.randint(1000, 6000)
+        v6 = random.randint(10, 80)
+        user_agent = (f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_{v1}_{v2}) AppleWebKit/{v4}.{v3} "
+                      f"(KHTML, like Gecko) Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
+        return user_agent
+
+    def _build_header(self) -> dict:
+        if not self.request_setting.get("headers"):
+            self.request_setting["headers"] = {"accept": "*/*", "user-agent": self._random_ua}
+        elif "user-agent" not in [key.lower() for key in self.request_setting["headers"].keys()]:
+            self.request_setting["headers"]["user-agent"] = self._random_ua
+
+    def download(self) -> requests.Response:
+        response = requests.request(self.method, self.url, **self.request_setting)
+        if self.check_status_code:
+            response.raise_for_status()
+        return response
+
+    @property
+    def to_dict(self):
+        _dict = self.__dict__.copy()
+        _dict.pop('url')
+        _dict.pop('seed')
+        _dict.pop('check_status_code')
+        _dict.pop('request_setting')
+        return _dict
+
+

cobweb_/base/response.py

@@ -0,0 +1,23 @@
+
+
+class Response:
+
+    def __init__(
+            self,
+            seed,
+            response,
+            **kwargs
+    ):
+        self.seed = seed
+        self.response = response
+
+        for k, v in kwargs.items():
+            self.__setattr__(k, v)
+
+    @property
+    def to_dict(self):
+        _dict = self.__dict__.copy()
+        _dict.pop('seed')
+        _dict.pop('response')
+        return _dict
+