cobweb-launcher 1.2.49__py3-none-any.whl → 1.3.1__py3-none-any.whl

cobweb/base/__init__.py

@@ -1,9 +1,146 @@
+import time
+import traceback
+import threading
+
+from functools import wraps
+from inspect import isgenerator
+from typing import Callable, Union
+
 from .common_queue import Queue
 from .response import Response
-from .request import Request
+from .basic import Seed, Request, Response
 from .item import BaseItem, ConsoleItem
-from .seed import Seed
-
+# from .seed import Seed
 from .log import logger
-from .decorators import decorator_oss_db
+# from .dotting import LoghubDot
+
+
+class TaskQueue:
+    TODO = Queue()          # 任务种子队列
+    DOWNLOAD = Queue()      # 下载任务队列
+
+    SEED = Queue()          # 添加任务种子队列
+    REQUEST = Queue()       # 请求队列
+    RESPONSE = Queue()      # 响应队列
+    DONE = Queue()          # 下载完成队列
+    UPLOAD = Queue()        # 任务上传队列
+    DELETE = Queue()        # 任务删除队列
+
+    # DOT = LoghubDot()
+
+    @staticmethod
+    def is_empty():
+        total_length = TaskQueue.SEED.length
+        total_length += TaskQueue.TODO.length
+        total_length += TaskQueue.REQUEST.length
+        total_length += TaskQueue.DOWNLOAD.length
+        total_length += TaskQueue.RESPONSE.length
+        total_length += TaskQueue.UPLOAD.length
+        total_length += TaskQueue.DONE.length
+        total_length += TaskQueue.DELETE.length
+        return not bool(total_length)
+
+    @staticmethod
+    def process_task(it: Union[Seed, Request, Response, BaseItem], crawler_func: Callable):
+        try:
+            iterators = crawler_func(it)
+            if not isgenerator(iterators):
+                raise TypeError(f"{crawler_func.__name__} function isn't a generator")
+            for tk in iterators:
+                if isinstance(tk, Request):
+                    TaskQueue.REQUEST.push(tk)
+                elif isinstance(tk, Response):
+                    TaskQueue.RESPONSE.push(tk)
+                elif isinstance(tk, BaseItem):
+                    TaskQueue.UPLOAD.push(tk)
+                elif isinstance(tk, Seed):
+                    TaskQueue.SEED.push(tk)
+                else:
+                    raise TypeError(f"{crawler_func.__name__} function return type isn't supported")
+                # TaskQueue.DOT.build(
+                #     topic=f"{self.project}:{self.task}",
+                #     cost_time=end_time - start_time,
+                #     **download_item.to_dict
+                # )
+                # todo: 数据打点
+        except Exception as e:
+            it.params.retry += 1
+            if isinstance(it, Request):
+                TaskQueue.REQUEST.push(it)
+            elif isinstance(it, Response):
+                TaskQueue.RESPONSE.push(it)
+            elif isinstance(it, Seed):
+                TaskQueue.SEED.push(it)
+            time.sleep(1)
+
+
+class Decorators:
+
+    @staticmethod
+    def add_thread(num=1):
+        def decorator(func):
+            @wraps(func)
+            def wrapper(self, *args):
+                for i in range(num):
+                    name = func.__name__ + "_" + str(i) if num > 1 else func.__name__
+                    self._threads.append(threading.Thread(name=name, target=func, args=(self,) + args))
+
+            return wrapper
+
+        return decorator
+
+    @staticmethod
+    def pause(func):
+        @wraps(func)
+        def wrapper(self, *args, **kwargs):
+            while not self.pause.is_set():
+                try:
+                    func(self, *args, **kwargs)
+                except Exception as e:
+                    logger.info(f"{func.__name__}: " + str(e))
+                finally:
+                    time.sleep(0.1)
+            logger.info(f"{func.__name__}: close!")
+
+        return wrapper
+
+    @staticmethod
+    def stop(func):
+        @wraps(func)
+        def wrapper(self, *args, **kwargs):
+            while not self.stop.is_set():
+                try:
+                    func(self, *args, **kwargs)
+                except Exception as e:
+                    logger.info(
+                        f"{func.__name__} exception: \n" +
+                        ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                    )
+                finally:
+                    time.sleep(0.1)
+
+        return wrapper
+
+    @staticmethod
+    def decorator_oss_db(exception, retries=3):
+        def decorator(func):
+            @wraps(func)
+            def wrapper(callback_func, *args, **kwargs):
+                result = None
+                for i in range(retries):
+                    msg = None
+                    try:
+                        return func(callback_func, *args, **kwargs)
+                    except Exception as e:
+                        result = None
+                        msg = e
+                    finally:
+                        if result:
+                            return result
+
+                        if i >= 2 and msg:
+                            raise exception(msg)
+
+            return wrapper
 
+        return decorator

cobweb/base/basic.py

@@ -15,21 +15,11 @@ class Params:
 
 
 class Seed:
-    __SEED_PARAMS__ = [
-        "retry",
-        "priority",
-        "version",
-        "status"
-    ]
 
     def __init__(
             self,
             seed,
-            sid=None,
-            retry=None,
-            priority=None,
-            version=None,
-            status=None,
+            params = Params(),
             **kwargs
     ):
         if any(isinstance(seed, t) for t in (str, bytes)):
@@ -47,27 +37,11 @@ class Seed:
                 f"seed: {seed}"
             ))
 
-        seed_params = {
-            "retry": retry,
-            "priority": priority,
-            "version": version,
-            "status": status,
-        }
-
         if kwargs:
-            # for k, v in kwargs.items():
-            #     if k in seed_params.keys():
-            #         seed_params[k] = v
-            #     else:
-            #         self.__setattr__(k, v)
             self._init_seed(kwargs)
-            seed_params.update({
-                k: v for k, v in kwargs.items()
-                if k in self.__SEED_PARAMS__
-            })
-        if sid or not getattr(self, "sid", None):
-            self._init_id(sid)
-        self.params = Params(**seed_params)
+        if not getattr(self, "sid", None):
+            self._init_id()
+        self.params = params or Params()
 
     def __getattr__(self, name):
         return None
@@ -85,14 +59,13 @@ class Seed:
         chars = [f"{k}={v}" for k, v in self.__dict__.items()]
         return f'{self.__class__.__name__}({", ".join(chars)})'
 
-    def _init_seed(self, seed_info: dict):
+    def _init_seed(self, seed_info:dict):
         for k, v in seed_info.items():
             if k not in self.__SEED_PARAMS__:
                 self.__setattr__(k, v)
 
-    def _init_id(self, sid):
-        if not sid:
-            sid = hashlib.md5(self.to_string.encode()).hexdigest()
+    def _init_id(self):
+        sid = hashlib.md5(self.to_string.encode()).hexdigest()
         self.__setattr__("sid", sid)
 
     @property
@@ -112,15 +85,16 @@ class Seed:
 
     @property
     def seed(self):
-        return self.to_string
+        return self
 
 
 class Request:
+
     __SEED_PARAMS__ = [
         "retry",
         "priority",
-        "version",
-        "status"
+        "seed_version",
+        "seed_status"
     ]
 
     __REQUEST_ATTRS__ = {
@@ -142,34 +116,27 @@ class Request:
 
     def __init__(
             self,
-            # url,
+            url,
             seed,
             random_ua=True,
             check_status_code=True,
             retry=None,
             priority=None,
-            version=None,
-            status=None,
+            seed_version=None,
+            seed_status=None,
             **kwargs
     ):
-        # self.url = url
+        self.url = url
         self.check_status_code = check_status_code
         self.request_setting = {}
 
         seed_params = {
             "retry": retry,
             "priority": priority,
-            "version": version,
-            "status": status,
+            "seed_version": seed_version,
+            "seed_status": seed_status,
         }
 
-        if isinstance(seed, Seed):
-            kwargs.update(**seed.to_dict)
-        elif isinstance(seed, str):
-            kwargs.update(**json.loads(seed))
-        elif isinstance(seed, dict):
-            kwargs.update(**seed)
-
         for k, v in kwargs.items():
             if k in self.__class__.__REQUEST_ATTRS__:
                 self.request_setting[k] = v
@@ -185,7 +152,12 @@ class Request:
             self._build_header()
 
         self.params = Params(**seed_params)
-        # self.seed = self.to_string
+
+        if isinstance(seed, Seed):
+            kwargs.update(**seed.to_dict)
+        elif isinstance(seed, str):
+            kwargs.update(**json.loads(seed))
+        self.seed = self.to_string
 
     @property
     def _random_ua(self) -> str:
@@ -211,19 +183,10 @@ class Request:
             response.raise_for_status()
         return response
 
-    def __getattr__(self, name):
-        return None
-
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-    def __getitem__(self, item):
-        return getattr(self, item)
-
     @property
     def to_dict(self):
         _dict = self.__dict__.copy()
-        # _dict.pop('seed')
+        _dict.pop('seed')
         _dict.pop('params')
         _dict.pop('check_status_code')
         # _dict.pop('request_setting')
@@ -237,10 +200,6 @@ class Request:
             separators=(",", ":")
         )
 
-    @property
-    def seed(self):
-        return self.to_string
-
 
 class Response:
 
@@ -250,8 +209,8 @@ class Response:
             response,
             retry=None,
             priority=None,
-            version=None,
-            status=None,
+            seed_version=None,
+            seed_status=None,
             **kwargs
     ):
         self.seed = seed
@@ -259,24 +218,20 @@ class Response:
         seed_params = {
             "retry": retry,
             "priority": priority,
-            "version": version,
-            "status": status,
+            "seed_version": seed_version,
+            "seed_status": seed_status,
         }
         for k, v in kwargs.items():
             if k in seed_params.keys():
                 seed_params[k] = v
             else:
                 self.__setattr__(k, v)
-        self.params = Params(**seed_params)
 
     @property
     def to_dict(self):
         _dict = self.__dict__.copy()
         _dict.pop('seed')
         _dict.pop('response')
-        _dict.pop('method')
-        _dict.pop('params')
-        _dict.pop('request_setting')
         return _dict
 
     @property
@@ -286,12 +241,3 @@ class Response:
             ensure_ascii=False,
             separators=(",", ":")
         )
-
-    def __getattr__(self, name):
-        return None
-
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-    def __getitem__(self, item):
-        return getattr(self, item)

cobweb/base/common_queue.py

@@ -1,3 +1,4 @@
+import time
 from collections import deque
 
 
@@ -28,3 +29,15 @@ class Queue:
             return None
         except AttributeError:
             return None
+
+    def clear(self):
+        self._queue.clear()
+
+    def get(self):
+        try:
+            yield self._queue.popleft()
+        except IndexError:
+            time.sleep(1)
+            yield None
+        except AttributeError:
+            yield None

cobweb/base/dotting.py

@@ -27,7 +27,7 @@ class LoghubDot:
             log_items.append(log_item)
             request = PutLogsRequest(
                 project="databee-download-log",
-                logstore="download-logging",
+                logstore="cobweb_log",
                 topic=topic,
                 logitems=log_items,
                 compress=True

cobweb/base/request.py

@@ -1,3 +1,4 @@
+import json
 import random
 import requests
 
@@ -30,7 +31,6 @@ class Request:
             **kwargs
     ):
         self.url = url
-        self.seed = seed
         self.check_status_code = check_status_code
         self.request_setting = {}
 
@@ -46,6 +46,12 @@ class Request:
         if random_ua:
             self._build_header()
 
+        if isinstance(seed, Seed):
+            self.seed = seed.to_string
+        else:
+            kwargs.update(**seed.to_dict)
+            self.seed = self.to_string
+
     @property
     def _random_ua(self) -> str:
         v1 = random.randint(4, 15)
@@ -73,10 +79,16 @@ class Request:
     @property
     def to_dict(self):
         _dict = self.__dict__.copy()
-        _dict.pop('url')
         _dict.pop('seed')
         _dict.pop('check_status_code')
         _dict.pop('request_setting')
         return _dict
 
+    @property
+    def to_string(self) -> str:
+        return json.dumps(
+            self.to_dict,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
 

cobweb/base/seed.py

@@ -104,11 +104,15 @@ class Seed:
             separators=(",", ":")
         )
 
+    # @property
+    # def get_all(self):
+    #     return json.dumps(
+    #         self.__dict__,
+    #         ensure_ascii=False,
+    #         separators=(",", ":")
+    #     )
+
     @property
-    def get_all(self):
-        return json.dumps(
-            self.__dict__,
-            ensure_ascii=False,
-            separators=(",", ":")
-        )
+    def seed(self):
+        return self.to_string
 

cobweb/constant.py

@@ -37,6 +37,22 @@ class LogTemplate:
     ----------------------- end  - console pipeline ------------------
     """
 
+    launcher_polling = """
+    ----------------------- start - 轮训日志: {task} -----------------
+            正在运行任务
+                构造请求任务数: {memory_todo_count}
+                正在下载任务数: {memory_download_count}
+            任务内存队列 
+                待构造请求队列:  {todo_queue_len}
+                待删除请求队列:  {delete_queue_len}
+                待进行下载队列:  {request_queue_len}
+                待解析响应队列:  {response_queue_len}
+                待删除下载队列:  {done_queue_len}
+            存储队列 
+               待上传数据队列:  {upload_queue_len}
+    ----------------------- end  - 轮训日志: {task} ------------------
+    """
+
     launcher_air_polling = """
     ----------------------- start - 轮训日志: {task} -----------------
             内存队列