cobweb-launcher 1.2.49__tar.gz → 1.3.1__tar.gz

{cobweb-launcher-1.2.49/cobweb_launcher.egg-info → cobweb-launcher-1.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.2.49
+Version: 1.3.1
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

cobweb-launcher-1.3.1/cobweb/base/__init__.py

@@ -0,0 +1,146 @@
+import time
+import traceback
+import threading
+
+from functools import wraps
+from inspect import isgenerator
+from typing import Callable, Union
+
+from .common_queue import Queue
+from .response import Response
+from .basic import Seed, Request, Response
+from .item import BaseItem, ConsoleItem
+# from .seed import Seed
+from .log import logger
+# from .dotting import LoghubDot
+
+
+class TaskQueue:
+    TODO = Queue()          # 任务种子队列
+    DOWNLOAD = Queue()      # 下载任务队列
+
+    SEED = Queue()          # 添加任务种子队列
+    REQUEST = Queue()       # 请求队列
+    RESPONSE = Queue()      # 响应队列
+    DONE = Queue()          # 下载完成队列
+    UPLOAD = Queue()        # 任务上传队列
+    DELETE = Queue()        # 任务删除队列
+
+    # DOT = LoghubDot()
+
+    @staticmethod
+    def is_empty():
+        total_length = TaskQueue.SEED.length
+        total_length += TaskQueue.TODO.length
+        total_length += TaskQueue.REQUEST.length
+        total_length += TaskQueue.DOWNLOAD.length
+        total_length += TaskQueue.RESPONSE.length
+        total_length += TaskQueue.UPLOAD.length
+        total_length += TaskQueue.DONE.length
+        total_length += TaskQueue.DELETE.length
+        return not bool(total_length)
+
+    @staticmethod
+    def process_task(it: Union[Seed, Request, Response, BaseItem], crawler_func: Callable):
+        try:
+            iterators = crawler_func(it)
+            if not isgenerator(iterators):
+                raise TypeError(f"{crawler_func.__name__} function isn't a generator")
+            for tk in iterators:
+                if isinstance(tk, Request):
+                    TaskQueue.REQUEST.push(tk)
+                elif isinstance(tk, Response):
+                    TaskQueue.RESPONSE.push(tk)
+                elif isinstance(tk, BaseItem):
+                    TaskQueue.UPLOAD.push(tk)
+                elif isinstance(tk, Seed):
+                    TaskQueue.SEED.push(tk)
+                else:
+                    raise TypeError(f"{crawler_func.__name__} function return type isn't supported")
+                # TaskQueue.DOT.build(
+                #     topic=f"{self.project}:{self.task}",
+                #     cost_time=end_time - start_time,
+                #     **download_item.to_dict
+                # )
+                # todo: 数据打点
+        except Exception as e:
+            it.params.retry += 1
+            if isinstance(it, Request):
+                TaskQueue.REQUEST.push(it)
+            elif isinstance(it, Response):
+                TaskQueue.RESPONSE.push(it)
+            elif isinstance(it, Seed):
+                TaskQueue.SEED.push(it)
+            time.sleep(1)
+
+
+class Decorators:
+
+    @staticmethod
+    def add_thread(num=1):
+        def decorator(func):
+            @wraps(func)
+            def wrapper(self, *args):
+                for i in range(num):
+                    name = func.__name__ + "_" + str(i) if num > 1 else func.__name__
+                    self._threads.append(threading.Thread(name=name, target=func, args=(self,) + args))
+
+            return wrapper
+
+        return decorator
+
+    @staticmethod
+    def pause(func):
+        @wraps(func)
+        def wrapper(self, *args, **kwargs):
+            while not self.pause.is_set():
+                try:
+                    func(self, *args, **kwargs)
+                except Exception as e:
+                    logger.info(f"{func.__name__}: " + str(e))
+                finally:
+                    time.sleep(0.1)
+            logger.info(f"{func.__name__}: close!")
+
+        return wrapper
+
+    @staticmethod
+    def stop(func):
+        @wraps(func)
+        def wrapper(self, *args, **kwargs):
+            while not self.stop.is_set():
+                try:
+                    func(self, *args, **kwargs)
+                except Exception as e:
+                    logger.info(
+                        f"{func.__name__} exception: \n" +
+                        ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                    )
+                finally:
+                    time.sleep(0.1)
+
+        return wrapper
+
+    @staticmethod
+    def decorator_oss_db(exception, retries=3):
+        def decorator(func):
+            @wraps(func)
+            def wrapper(callback_func, *args, **kwargs):
+                result = None
+                for i in range(retries):
+                    msg = None
+                    try:
+                        return func(callback_func, *args, **kwargs)
+                    except Exception as e:
+                        result = None
+                        msg = e
+                    finally:
+                        if result:
+                            return result
+
+                        if i >= 2 and msg:
+                            raise exception(msg)
+
+            return wrapper
+
+        return decorator

cobweb-launcher-1.3.1/cobweb/base/basic.py

@@ -0,0 +1,243 @@
+import json
+import random
+import time
+import hashlib
+import requests
+
+
+class Params:
+
+    def __init__(self, retry=None, priority=None, version=None, status=None):
+        self.retry = retry or 0
+        self.priority = priority or 300
+        self.version = version or int(time.time())
+        self.status = status
+
+
+class Seed:
+
+    def __init__(
+            self,
+            seed,
+            params = Params(),
+            **kwargs
+    ):
+        if any(isinstance(seed, t) for t in (str, bytes)):
+            try:
+                item = json.loads(seed)
+                self._init_seed(item)
+            except json.JSONDecodeError:
+                self.__setattr__("url", seed)
+        elif isinstance(seed, dict):
+            self._init_seed(seed)
+        else:
+            raise TypeError(Exception(
+                f"seed type error, "
+                f"must be str or dict! "
+                f"seed: {seed}"
+            ))
+
+        if kwargs:
+            self._init_seed(kwargs)
+        if not getattr(self, "sid", None):
+            self._init_id()
+        self.params = params or Params()
+
+    def __getattr__(self, name):
+        return None
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __str__(self):
+        return json.dumps(self.__dict__, ensure_ascii=False)
+
+    def __repr__(self):
+        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
+        return f'{self.__class__.__name__}({", ".join(chars)})'
+
+    def _init_seed(self, seed_info:dict):
+        for k, v in seed_info.items():
+            if k not in self.__SEED_PARAMS__:
+                self.__setattr__(k, v)
+
+    def _init_id(self):
+        sid = hashlib.md5(self.to_string.encode()).hexdigest()
+        self.__setattr__("sid", sid)
+
+    @property
+    def to_dict(self) -> dict:
+        seed = self.__dict__.copy()
+        if seed.get("params"):
+            del seed["params"]
+        return seed
+
+    @property
+    def to_string(self) -> str:
+        return json.dumps(
+            self.to_dict,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
+
+    @property
+    def seed(self):
+        return self
+
+
+class Request:
+
+    __SEED_PARAMS__ = [
+        "retry",
+        "priority",
+        "seed_version",
+        "seed_status"
+    ]
+
+    __REQUEST_ATTRS__ = {
+        "params",
+        "headers",
+        "cookies",
+        "data",
+        "json",
+        "files",
+        "auth",
+        "timeout",
+        "proxies",
+        "hooks",
+        "stream",
+        "verify",
+        "cert",
+        "allow_redirects",
+    }
+
+    def __init__(
+            self,
+            url,
+            seed,
+            random_ua=True,
+            check_status_code=True,
+            retry=None,
+            priority=None,
+            seed_version=None,
+            seed_status=None,
+            **kwargs
+    ):
+        self.url = url
+        self.check_status_code = check_status_code
+        self.request_setting = {}
+
+        seed_params = {
+            "retry": retry,
+            "priority": priority,
+            "seed_version": seed_version,
+            "seed_status": seed_status,
+        }
+
+        for k, v in kwargs.items():
+            if k in self.__class__.__REQUEST_ATTRS__:
+                self.request_setting[k] = v
+                continue
+            elif k in self.__SEED_PARAMS__:
+                seed_params[k] = v
+            self.__setattr__(k, v)
+
+        if not getattr(self, "method", None):
+            self.method = "POST" if self.request_setting.get("data") or self.request_setting.get("json") else "GET"
+
+        if random_ua:
+            self._build_header()
+
+        self.params = Params(**seed_params)
+
+        if isinstance(seed, Seed):
+            kwargs.update(**seed.to_dict)
+        elif isinstance(seed, str):
+            kwargs.update(**json.loads(seed))
+        self.seed = self.to_string
+
+    @property
+    def _random_ua(self) -> str:
+        v1 = random.randint(4, 15)
+        v2 = random.randint(3, 11)
+        v3 = random.randint(1, 16)
+        v4 = random.randint(533, 605)
+        v5 = random.randint(1000, 6000)
+        v6 = random.randint(10, 80)
+        user_agent = (f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_{v1}_{v2}) AppleWebKit/{v4}.{v3} "
+                      f"(KHTML, like Gecko) Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
+        return user_agent
+
+    def _build_header(self) -> dict:
+        if not self.request_setting.get("headers"):
+            self.request_setting["headers"] = {"accept": "*/*", "user-agent": self._random_ua}
+        elif "user-agent" not in [key.lower() for key in self.request_setting["headers"].keys()]:
+            self.request_setting["headers"]["user-agent"] = self._random_ua
+
+    def download(self) -> requests.Response:
+        response = requests.request(self.method, self.url, **self.request_setting)
+        if self.check_status_code:
+            response.raise_for_status()
+        return response
+
+    @property
+    def to_dict(self):
+        _dict = self.__dict__.copy()
+        _dict.pop('seed')
+        _dict.pop('params')
+        _dict.pop('check_status_code')
+        # _dict.pop('request_setting')
+        return _dict
+
+    @property
+    def to_string(self) -> str:
+        return json.dumps(
+            self.to_dict,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
+
+
+class Response:
+
+    def __init__(
+            self,
+            seed,
+            response,
+            retry=None,
+            priority=None,
+            seed_version=None,
+            seed_status=None,
+            **kwargs
+    ):
+        self.seed = seed
+        self.response = response
+        seed_params = {
+            "retry": retry,
+            "priority": priority,
+            "seed_version": seed_version,
+            "seed_status": seed_status,
+        }
+        for k, v in kwargs.items():
+            if k in seed_params.keys():
+                seed_params[k] = v
+            else:
+                self.__setattr__(k, v)
+
+    @property
+    def to_dict(self):
+        _dict = self.__dict__.copy()
+        _dict.pop('seed')
+        _dict.pop('response')
+        return _dict
+
+    @property
+    def to_string(self) -> str:
+        return json.dumps(
+            self.to_dict,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )

cobweb-launcher-1.3.1/cobweb/base/common_queue.py

@@ -0,0 +1,43 @@
+import time
+from collections import deque
+
+
+class Queue:
+
+    def __init__(self):
+        self._queue = deque()
+
+    @property
+    def length(self) -> int:
+        return len(self._queue)
+
+    def push(self, data, left: bool = False, direct_insertion: bool = False):
+        try:
+            if not data:
+                return None
+            if not direct_insertion and any(isinstance(data, t) for t in (list, tuple)):
+                self._queue.extendleft(data) if left else self._queue.extend(data)
+            else:
+                self._queue.appendleft(data) if left else self._queue.append(data)
+        except AttributeError:
+            pass
+
+    def pop(self, left: bool = True):
+        try:
+            return self._queue.popleft() if left else self._queue.pop()
+        except IndexError:
+            return None
+        except AttributeError:
+            return None
+
+    def clear(self):
+        self._queue.clear()
+
+    def get(self):
+        try:
+            yield self._queue.popleft()
+        except IndexError:
+            time.sleep(1)
+            yield None
+        except AttributeError:
+            yield None

cobweb-launcher-1.3.1/cobweb/base/dotting.py

@@ -0,0 +1,35 @@
+import os
+import json
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+
+
+class LoghubDot:
+
+    def __init__(self):
+        endpoint = os.getenv("DOTTING_ENDPOINT", "")
+        accessKeyId = os.getenv("DOTTING_ACCESS_KEY", "")
+        accessKey = os.getenv("DOTTING_SECRET_KEY", "")
+        self.client = LogClient(endpoint=endpoint, accessKeyId=accessKeyId, accessKey=accessKey) \
+            if endpoint and accessKeyId and accessKey else None
+
+    def build(self, topic, **kwargs):
+        if self.client:
+            temp = {}
+            log_items = []
+            log_item = LogItem()
+            for key, value in kwargs.items():
+                if not isinstance(value, str):
+                    temp[key] = json.dumps(value, ensure_ascii=False)
+                else:
+                    temp[key] = value
+            contents = sorted(temp.items())
+            log_item.set_contents(contents)
+            log_items.append(log_item)
+            request = PutLogsRequest(
+                project="databee-download-log",
+                logstore="cobweb_log",
+                topic=topic,
+                logitems=log_items,
+                compress=True
+            )
+            self.client.put_logs(request=request)

cobweb-launcher-1.3.1/cobweb/base/request.py

@@ -0,0 +1,94 @@
+import json
+import random
+import requests
+
+
+class Request:
+
+    __REQUEST_ATTRS__ = {
+        "params",
+        "headers",
+        "cookies",
+        "data",
+        "json",
+        "files",
+        "auth",
+        "timeout",
+        "proxies",
+        "hooks",
+        "stream",
+        "verify",
+        "cert",
+        "allow_redirects",
+    }
+
+    def __init__(
+            self,
+            url,
+            seed,
+            random_ua=True,
+            check_status_code=True,
+            **kwargs
+    ):
+        self.url = url
+        self.check_status_code = check_status_code
+        self.request_setting = {}
+
+        for k, v in kwargs.items():
+            if k in self.__class__.__REQUEST_ATTRS__:
+                self.request_setting[k] = v
+                continue
+            self.__setattr__(k, v)
+
+        if not getattr(self, "method", None):
+            self.method = "POST" if self.request_setting.get("data") or self.request_setting.get("json") else "GET"
+
+        if random_ua:
+            self._build_header()
+
+        if isinstance(seed, Seed):
+            self.seed = seed.to_string
+        else:
+            kwargs.update(**seed.to_dict)
+            self.seed = self.to_string
+
+    @property
+    def _random_ua(self) -> str:
+        v1 = random.randint(4, 15)
+        v2 = random.randint(3, 11)
+        v3 = random.randint(1, 16)
+        v4 = random.randint(533, 605)
+        v5 = random.randint(1000, 6000)
+        v6 = random.randint(10, 80)
+        user_agent = (f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_{v1}_{v2}) AppleWebKit/{v4}.{v3} "
+                      f"(KHTML, like Gecko) Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
+        return user_agent
+
+    def _build_header(self) -> dict:
+        if not self.request_setting.get("headers"):
+            self.request_setting["headers"] = {"accept": "*/*", "user-agent": self._random_ua}
+        elif "user-agent" not in [key.lower() for key in self.request_setting["headers"].keys()]:
+            self.request_setting["headers"]["user-agent"] = self._random_ua
+
+    def download(self) -> requests.Response:
+        response = requests.request(self.method, self.url, **self.request_setting)
+        if self.check_status_code:
+            response.raise_for_status()
+        return response
+
+    @property
+    def to_dict(self):
+        _dict = self.__dict__.copy()
+        _dict.pop('seed')
+        _dict.pop('check_status_code')
+        _dict.pop('request_setting')
+        return _dict
+
+    @property
+    def to_string(self) -> str:
+        return json.dumps(
+            self.to_dict,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
+

cobweb-launcher-1.3.1/cobweb/base/seed.py

@@ -0,0 +1,118 @@
+import json
+import time
+import hashlib
+
+
+class SeedParams:
+
+    def __init__(self, retry, priority, seed_version, seed_status=None):
+        self.retry = retry or 0
+        self.priority = priority or 300
+        self.seed_version = seed_version or int(time.time())
+        self.seed_status = seed_status
+
+
+class Seed:
+
+    __SEED_PARAMS__ = [
+        "retry",
+        "priority",
+        "seed_version",
+        "seed_status"
+    ]
+
+    def __init__(
+            self,
+            seed,
+            sid=None,
+            retry=None,
+            priority=None,
+            seed_version=None,
+            seed_status=None,
+            **kwargs
+    ):
+        if any(isinstance(seed, t) for t in (str, bytes)):
+            try:
+                item = json.loads(seed)
+                self._init_seed(item)
+            except json.JSONDecodeError:
+                self.__setattr__("url", seed)
+        elif isinstance(seed, dict):
+            self._init_seed(seed)
+        else:
+            raise TypeError(Exception(
+                f"seed type error, "
+                f"must be str or dict! "
+                f"seed: {seed}"
+            ))
+
+        seed_params = {
+            "retry": retry,
+            "priority": priority,
+            "seed_version": seed_version,
+            "seed_status": seed_status,
+        }
+
+        if kwargs:
+            self._init_seed(kwargs)
+            seed_params.update({
+                k:v for k, v in kwargs.items()
+                if k in self.__SEED_PARAMS__
+            })
+        if sid or not getattr(self, "sid", None):
+            self._init_id(sid)
+        self.params = SeedParams(**seed_params)
+
+    def __getattr__(self, name):
+        return None
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __str__(self):
+        return json.dumps(self.__dict__, ensure_ascii=False)
+
+    def __repr__(self):
+        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
+        return f'{self.__class__.__name__}({", ".join(chars)})'
+
+    def _init_seed(self, seed_info:dict):
+        for k, v in seed_info.items():
+            if k not in self.__SEED_PARAMS__:
+                self.__setattr__(k, v)
+
+    def _init_id(self, sid):
+        if not sid:
+            sid = hashlib.md5(self.to_string.encode()).hexdigest()
+        self.__setattr__("sid", sid)
+
+    @property
+    def to_dict(self) -> dict:
+        seed = self.__dict__.copy()
+        if seed.get("params"):
+            del seed["params"]
+        return seed
+
+    @property
+    def to_string(self) -> str:
+        return json.dumps(
+            self.to_dict,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
+
+    # @property
+    # def get_all(self):
+    #     return json.dumps(
+    #         self.__dict__,
+    #         ensure_ascii=False,
+    #         separators=(",", ":")
+    #     )
+
+    @property
+    def seed(self):
+        return self.to_string
+