cobweb-launcher 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

cobweb/launchers/launcher.py

@@ -1,10 +1,9 @@
 import time
-
-import setting
 import inspect
 import threading
 import importlib
 
+from cobweb import setting
 from cobweb.base import Seed, Queue
 from cobweb.utils.tools import dynamic_load_class
 

cobweb/utils/oss.py

@@ -1,5 +1,4 @@
-import setting
-
+from cobweb import setting
 from requests import Response
 from oss2 import Auth, Bucket, models
 from cobweb.exceptions import oss_db_exception

{cobweb_launcher-1.0.1.dist-info → cobweb_launcher-1.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.0.1
+Version: 1.0.3
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

cobweb_launcher-1.0.3.dist-info/RECORD

@@ -0,0 +1,32 @@
+cobweb/__init__.py,sha256=IkGcdTU6fEBNyzWowcJSSMdErntFM1kmu4WUp1BgImU,45
+cobweb/constant.py,sha256=Aw2ES_nohVRLTWylZp6WMiUAlgyw4kLbae7LpwdZ5y4,1867
+cobweb/setting.py,sha256=T693DAwLFLs9P6ZEvugP99UzXn-8PLeMEgdxRmL6cb4,1955
+cobweb/base/__init__.py,sha256=diiK5MygQaWjlWNLbW6eUIg-93O6glMGC9WLNM5jyOc,209
+cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
+cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
+cobweb/base/item.py,sha256=pMriHStzUXtSvIf5Z3KXsP-bCvjlG1gM3z33wWeuoH8,966
+cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
+cobweb/base/request.py,sha256=b08AtUSZjlFLEFIEw5uGS__FjU6QSldL20-UjZD0LbI,2128
+cobweb/base/response.py,sha256=7h9TwCNqRlwM_fvNmid9zOoRfHbKB8ABSU0eaVUJdVo,405
+cobweb/base/seed.py,sha256=XswH16eEd6iwIBpt71E2S_AsV5UVCcOEOBFoP0r5QRo,2900
+cobweb/crawlers/__init__.py,sha256=1sMhQ0-NJxiff3IqF2aMCXkSXcJFzzoCKIayQ5go4aI,71
+cobweb/crawlers/base_crawler.py,sha256=ZIdmlvL4f41yV7T77F9IhBBxRt1FH-LFm0BmIXAvP8I,4881
+cobweb/crawlers/file_crawler.py,sha256=VVOZ38qNAUPyNDspu3P7-zzDtrUtqefYLjOMnb_-JOw,9685
+cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
+cobweb/db/redis_db.py,sha256=MahFsAyub9OFG-xuZU6Qq0AfWV1lASFAI34g2FRGpq8,4235
+cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
+cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
+cobweb/launchers/__init__.py,sha256=qwlkEJVri7dvCgi45aX3lqAmQS0HrPicAipDvH75kew,69
+cobweb/launchers/launcher.py,sha256=O6Kkvqk-0kOxJom8YO9zW18e_2eMYrA5RTS9Xy4TW5k,5665
+cobweb/launchers/launcher_pro.py,sha256=GlDpyP1XAY2bX5SuSBn3920D5OKNigQgLnJfu6QOmPw,6760
+cobweb/pipelines/__init__.py,sha256=xanY-Z1d7zRR5JhCdW2htzrAywnKBkigiaUlTFa6of0,80
+cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
+cobweb/pipelines/loghub_pipeline.py,sha256=roQ8gLunvuPc4KOMIATN1nKvjOXrc_RSyzXq8YY9ZBU,1015
+cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
+cobweb/utils/oss.py,sha256=8u--dh8rOIznEeSrvGdUglYOg-VBU9yZNAUcScnKayE,3231
+cobweb/utils/tools.py,sha256=8oLxkzwaYcDqKXXuLI3A_lNabyLBr7HSPgTF6x4xbnk,1239
+cobweb_launcher-1.0.3.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.0.3.dist-info/METADATA,sha256=PILf0MMb-RqO3aLG9SvrfDEkk-vmUJ3lDvaWXGmzY1c,1245
+cobweb_launcher-1.0.3.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.0.3.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-1.0.3.dist-info/RECORD,,

cobweb/bbb.py

@@ -1,191 +0,0 @@
-# from typing import Iterable
-import json
-import time
-import hashlib
-from .log import log
-from .utils import struct_queue_name
-from collections import deque, namedtuple
-
-
-class Queue:
-
-    def __init__(self):
-        self._queue = deque()
-
-    @property
-    def length(self) -> int:
-        return len(self._queue)
-    #
-    # @property
-    # def queue_names(self):
-    #     return tuple(self.__dict__.keys())
-    #
-    # @property
-    # def used_memory(self):
-    #     return asizeof.asizeof(self)
-
-    # def create_queue(self, queue_name: str):
-    #     self.__setattr__(queue_name, deque())
-
-    # def push_seed(self, seed):
-    #     self.push("_seed_queue", seed)
-
-    # def pop_seed(self):
-    #     return self.pop("_seed_queue")
-
-    def push(self, data, left: bool = False, direct_insertion: bool = False):
-        try:
-            if not data:
-                return None
-            if direct_insertion or isinstance(data, Seed):
-                self._queue.appendleft(data) if left else self._queue.append(data)
-            elif any(isinstance(data, t) for t in (list, tuple)):
-                self._queue.extendleft(data) if left else self._queue.extend(data)
-        except AttributeError as e:
-            log.exception(e)
-
-    def pop(self, left: bool = True):
-        try:
-            return self._queue.popleft() if left else self._queue.pop()
-        except IndexError:
-            return None
-        except AttributeError as e:
-            log.exception(e)
-            return None
-
-
-class Seed:
-
-    def __init__(
-            self,
-            seed_info=None,
-            priority=300,
-            version=0,
-            retry=0,
-            **kwargs
-    ):
-        if seed_info:
-            if any(isinstance(seed_info, t) for t in (str, bytes)):
-                try:
-                    item = json.loads(seed_info)
-                    for k, v in item.items():
-                        self.__setattr__(k, v)
-                except json.JSONDecodeError:
-                    self.__setattr__("url", seed_info)
-            elif isinstance(seed_info, dict):
-                for k, v in seed_info.items():
-                    self.__setattr__(k, v)
-            else:
-                raise TypeError(Exception(
-                    f"seed type error, "
-                    f"must be str or dict! "
-                    f"seed_info: {seed_info}"
-                ))
-        for k, v in kwargs.items():
-            self.__setattr__(k, v)
-        if not getattr(self, "_priority"):
-            self._priority = min(max(1, int(priority)), 999)
-        if not getattr(self, "_version"):
-            self._version = int(version) or int(time.time())
-        if not getattr(self, "_retry"):
-            self._retry = retry
-        if not getattr(self, "sid"):
-            self.init_id()
-
-    def init_id(self):
-        item_string = self.format_seed
-        seed_id = hashlib.md5(item_string.encode()).hexdigest()
-        self.__setattr__("sid", seed_id)
-
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-    def __getitem__(self, item):
-        return getattr(self, item)
-
-    def __getattr__(self, name):
-        return None
-
-    def __str__(self):
-        return json.dumps(self.__dict__, ensure_ascii=False)
-
-    def __repr__(self):
-        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
-        return f'{self.__class__.__name__}({", ".join(chars)})'
-
-    @property
-    def dict_seed(self):
-        seed = self.__dict__.copy()
-        del seed["_priority"]
-        del seed["_version"]
-        del seed["_retry"]
-        return seed
-
-    @property
-    def format_seed(self):
-        return json.dumps(self.dict_seed, ensure_ascii=False, separators=(",", ":"))
-
-
-class DBItem:
-
-    def __init__(self, **kwargs):
-        self.__setattr__("_index", 0, True)
-        for table in self.__class__.__table__:
-            if set(kwargs.keys()) == set(table._fields):
-                break
-            self._index += 1
-
-        if self._index > len(self.__class__.__table__):
-            raise Exception()
-
-        table = self.__class__.__table__[self._index]
-        self.__setattr__("struct_data", table(**kwargs), True)
-        self.__setattr__("db_name", self.__class__.__name__, True)
-        self.__setattr__("table_name", self.struct_data.__class__.__name__, True)
-
-    @classmethod
-    def init_item(cls, table_name, fields):
-        queue_name = struct_queue_name(cls.__name__, table_name)
-        if getattr(cls, queue_name, None) is None:
-            setattr(cls, queue_name, Queue())
-
-        if getattr(cls, "__table__", None) is None:
-            cls.__table__ = []
-
-        table = namedtuple(table_name, fields)
-
-        if table in getattr(cls, "__table__"):
-            raise Exception()
-        getattr(cls, "__table__").append(table)
-
-    def queue(self):
-        queue_name = struct_queue_name(self.db_name, self.table_name)
-        return getattr(self.__class__, queue_name)
-
-    def __setitem__(self, key, value):
-        self.__setattr__(key, value)
-
-    def __getitem__(self, item):
-        return self.struct_data[item]
-
-    def __getattr__(self, name):
-        return None
-
-    def __setattr__(self, key, value, init=None):
-        if init:
-            super().__setattr__(key, value)
-        elif not getattr(self, "struct_data"):
-            raise Exception(f"no struct_data")
-        else:
-            self.__setattr__(
-                "struct_data",
-                self.struct_data._replace(**{key: value}),
-                init=True
-            )
-
-    def __str__(self):
-        return json.dumps(self.struct_data._asdict(), ensure_ascii=False)
-
-    def __repr__(self):
-        return f'{self.__class__.__name__}:{self.struct_data}'
-

cobweb/config.py

@@ -1,164 +0,0 @@
-import json
-from collections import namedtuple
-from base.utils import struct_table_name
-
-StorerInfo = namedtuple(
-    "StorerInfo",
-    "DB, table, fields, length, config"
-)
-SchedulerInfo = namedtuple(
-    "SchedulerInfo",
-    "DB, table, sql, length, size, config",
-)
-RedisInfo = namedtuple(
-    "RedisInfo",
-    "host, port, username, password, db",
-    defaults=("localhost", 6379, None, None, 0)
-)
-
-# redis_info = dict(
-#     host="localhost",
-#     port=6379,
-#     username=None,
-#     password=None,
-#     db=0
-# )
-
-
-class SchedulerDB:
-
-    @staticmethod
-    def default():
-        from db.scheduler.default import Default
-        return SchedulerInfo(DB=Default, table="", sql="", length=100, size=500000, config=None)
-
-    @staticmethod
-    def textfile(table, sql=None, length=100, size=500000):
-        from db.scheduler.textfile import Textfile
-        return SchedulerInfo(DB=Textfile, table=table, sql=sql, length=length, size=size, config=None)
-
-    @staticmethod
-    def diy(DB, table, sql=None, length=100, size=500000, config=None):
-        from base.interface import SchedulerInterface
-        if not isinstance(DB, SchedulerInterface):
-            raise Exception("DB must be inherit from SchedulerInterface")
-        return SchedulerInfo(DB=DB, table=table, sql=sql, length=length, size=size, config=config)
-
-    # @staticmethod
-    # def info(scheduler_info):
-    #     if not scheduler_info:
-    #         return SchedulerDB.default()
-    #
-    #     if isinstance(scheduler_info, SchedulerInfo):
-    #         return scheduler_info
-    #
-    #     if isinstance(scheduler_info, str):
-    #         scheduler = json.loads(scheduler_info)
-    #         if isinstance(scheduler, dict):
-    #             db_name = scheduler["DB"]
-    #             if db_name in dir(SchedulerDB):
-    #                 del scheduler["DB"]
-    #             else:
-    #                 db_name = "diy"
-    #             func = getattr(SchedulerDB, db_name)
-    #             return func(**scheduler)
-
-
-class StorerDB:
-
-    @staticmethod
-    def console(table, fields, length=200):
-        from db.storer.console import Console
-        table = struct_table_name(table)
-        return StorerInfo(DB=Console, table=table, fields=fields, length=length, config=None)
-
-    @staticmethod
-    def textfile(table, fields, length=200):
-        from db.storer.textfile import Textfile
-        table = struct_table_name(table)
-        return StorerInfo(DB=Textfile, table=table, fields=fields, length=length, config=None)
-
-    @staticmethod
-    def loghub(table, fields, length=200, config=None):
-        from db.storer.loghub import Loghub
-        table = struct_table_name(table)
-        return StorerInfo(DB=Loghub, table=table, fields=fields, length=length, config=config)
-
-    @staticmethod
-    def diy(DB, table, fields, length=200, config=None):
-        from base.interface import StorerInterface
-        if not isinstance(DB, StorerInterface):
-            raise Exception("DB must be inherit from StorerInterface")
-        table = struct_table_name(table)
-        return StorerInfo(DB=DB, table=table, fields=fields, length=length, config=config)
-
-    # @staticmethod
-    # def info(storer_info):
-    #     if not storer_info:
-    #         return None
-    #
-    #     if isinstance(storer_info, str):
-    #         storer_info = json.loads(storer_info)
-    #
-    #     if any(isinstance(storer_info, t) for t in (dict, StorerInfo)):
-    #         storer_info = [storer_info]
-    #
-    #     if not isinstance(storer_info, list):
-    #         raise Exception("StorerDB.info storer_info")
-    #
-    #     storer_info_list = []
-    #     for storer in storer_info:
-    #         if isinstance(storer, StorerInfo):
-    #             storer_info_list.append(storer)
-    #         else:
-    #             db_name = storer["DB"]
-    #             if db_name in dir(StorerDB):
-    #                 del storer["DB"]
-    #             else:
-    #                 db_name = "diy"
-    #             func = getattr(StorerDB, db_name)
-    #             storer_info_list.append(func(**storer))
-    #     return storer_info_list
-
-
-
-def deal(config, tag):
-    if isinstance(config, dict):
-        if tag == 0:
-            return RedisInfo(**config)
-        elif tag == 1:
-            db_name = config["DB"]
-            if db_name in dir(SchedulerDB):
-                del config["DB"]
-            else:
-                db_name = "diy"
-            func = getattr(SchedulerDB, db_name)
-            return func(**config)
-        elif tag == 2:
-            db_name = config["DB"]
-            if db_name in dir(StorerDB):
-                del config["DB"]
-            else:
-                db_name = "diy"
-            func = getattr(StorerDB, db_name)
-            return func(**config)
-        raise ValueError("tag must be in [0, 1, 2]")
-    elif any(isinstance(config, t) for t in (StorerInfo, SchedulerInfo, RedisInfo)):
-        return config
-    raise TypeError("config must be in [StorerInfo, SchedulerInfo, RedisInfo]")
-
-
-def info(configs, tag = 0):
-    if configs is None:
-        return SchedulerDB.default() if tag == 1 else None
-
-    if isinstance(configs, str):
-        configs = json.loads(configs)
-
-    if tag == 0:
-        return deal(configs, tag)
-
-    if not isinstance(configs, list):
-        configs = [configs]
-
-    return [deal(config, tag) for config in configs]

cobweb/db/oss_db.py

@@ -1,128 +0,0 @@
-import oss2
-from typing import Union
-from oss2.models import PartInfo
-from requests import Response
-from cobweb import log
-
-
-class OssDB:
-
-    def __init__(
-            self,
-            bucket_name,
-            endpoint,
-            access_key,
-            secret_key,
-            chunk_size=1024 ** 2,
-            min_size=1024
-    ):
-        self.endpoint = endpoint
-        self.bucket_name = bucket_name
-        self.auth = oss2.Auth(
-            access_key_id=access_key,
-            access_key_secret=secret_key
-        )
-        self.bucket = oss2.Bucket(
-            auth=self.auth,
-            endpoint=endpoint,
-            bucket_name=bucket_name
-        )
-        self.chunk_size = chunk_size
-        self.min_size = min_size
-
-    @staticmethod
-    def format_upload_len(length):
-        if not length:
-            raise ValueError("Length cannot be None or 0")
-
-        units = ["KB", "MB", "GB", "TB"]
-        for i in range(3):
-            num = length / (1024 ** (i + 1))
-            if num <= 1024:
-                return f"{round(num, 2)} {units[i]}"
-
-    def assemble(self, ready_data, part_data):
-        upload_data = None
-        ready_data = ready_data + part_data
-        if len(ready_data) >= self.chunk_size:
-            upload_data = ready_data[:self.chunk_size]
-            ready_data = ready_data[self.chunk_size:]
-
-        return ready_data, upload_data
-
-    def iter_data(self, data):
-        if isinstance(data, Response):
-            for part_data in data.iter_content(self.chunk_size):
-                yield part_data
-        if isinstance(data, bytes):
-            for i in range(0, len(data), self.chunk_size):
-                yield data[i:i + self.chunk_size]
-
-    def upload_split(
-            self, oss_path: str,
-            data: Union[bytes, Response],
-            timeout: int = 300,
-    ):
-        parts = []
-        status = False
-        upload_id = None
-        ready_data = b""
-        upload_data_len = 0
-        headers = {"Expires": str(timeout * 1000)}
-        try:
-            upload_id = self.bucket.init_multipart_upload(oss_path).upload_id
-            for part_data in self.iter_data(data):
-                upload_data_len += len(part_data)
-                ready_data, upload_data = self.assemble(ready_data, part_data)
-                if upload_data:
-                    part_index = len(parts) + 1
-                    upload_info = self.bucket.upload_part(
-                        oss_path, upload_id, part_index, upload_data
-                    )
-                    parts.append(PartInfo(part_index, upload_info.etag))
-
-            format_upload = self.format_upload_len(upload_data_len)
-
-            if parts and ready_data:
-                part_index = len(parts) + 1
-                upload_info = self.bucket.upload_part(
-                    oss_path, upload_id, part_index, ready_data
-                )
-                parts.append(PartInfo(part_index, upload_info.etag))
-                self.bucket.complete_multipart_upload(
-                    oss_path, upload_id, parts
-                )
-                log.info(
-                    f"split upload, file path: {oss_path}"
-                    f", file size: {format_upload}"
-                )
-
-            elif len(ready_data) > self.min_size:
-                self.bucket.put_object(oss_path, ready_data, headers)
-                log.info(
-                    f"upload file, file path: {oss_path}"
-                    f", file size: {format_upload}"
-                )
-
-            # else:
-            #     log.info(
-            #         f"file size smaller than min size! "
-            #         f"file size: {format_upload}"
-            #     )
-            status = True
-        except ValueError as e:
-            pass
-            # log.exception(str(e))
-        except oss2.exceptions.RequestError as e:
-            self.bucket = oss2.Bucket(
-                auth=self.auth,
-                endpoint=self.endpoint,
-                bucket_name=self.bucket_name
-            )
-            log.exception("oss timeout! " + str(e))
-        except Exception as e:
-            self.bucket.abort_multipart_upload(oss_path, upload_id, headers)
-            log.exception("upload file exception: " + str(e))
-
-        return status
-

cobweb/db/scheduler/__init__.py

@@ -1 +0,0 @@
-from .. import log, Seed, SchedulerInterface as Inf

cobweb/db/scheduler/default.py

@@ -1,8 +0,0 @@
-from . import Inf
-
-
-class Default(Inf):
-
-    def schedule(self):
-        pass
-

cobweb/db/scheduler/textfile.py

@@ -1,27 +0,0 @@
-from . import Inf, log, Seed
-
-
-class Textfile(Inf):
-
-    index = None
-
-    def schedule(self):
-        try:
-            seeds = []
-            with open(self.table, "r") as fp:
-                fp.seek(self.index or 0, 0)
-                for _ in range(self.length):
-                    data = fp.readline().strip()
-                    if not data:
-                        log.info("scheduler end!")
-                        self.stop = True
-                        break
-                    seeds.append(Seed(data))
-                    self.index = fp.tell()
-            return seeds
-        except FileNotFoundError:
-            log.error("task table not found!")
-            return None
-        except TypeError:
-            log.error("task table type error!")
-            return None

cobweb/db/storer/__init__.py

@@ -1 +0,0 @@
-from .. import log, Seed, StorerInterface as Inf

cobweb/db/storer/console.py

@@ -1,9 +0,0 @@
-from . import Inf, log
-
-
-class Console(Inf):
-
-    def store(self, data_list):
-        for item in data_list:
-            log.info(f"item info: {item}")
-

cobweb/db/storer/loghub.py

@@ -1,54 +0,0 @@
-import json
-from . import Inf, log
-from aliyun.log import LogClient, LogItem, PutLogsRequest
-
-
-class Loghub(Inf):
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.client = None
-
-    def init_loghub_clint(self):
-        try:
-            self.client = LogClient(
-                self.config['endpoint'],
-                self.config['access_key_id'],
-                self.config['access_key']
-            )
-        except Exception as e:
-            self.client = None
-            return False
-
-    def store(self, data_list):
-        try:
-            if not self.client:
-                self.init_loghub_clint()
-
-            log_items = list()
-            for item in data_list:
-                temp = item._asdict()
-                for key, value in temp.items():
-                    if isinstance(value, str):
-                        temp[key] = value
-                    else:
-                        temp[key] = json.dumps(value, ensure_ascii=False)
-                log_item = LogItem()
-                contents = sorted(temp.items())  # dict to tuple
-                log_item.set_contents(contents)
-                log_items.append(log_item)
-            request = PutLogsRequest(
-                project=self.config["project"],
-                logstore=self.table,
-                topic=self.config["topic"],
-                source=self.config.get("source"),
-                logitems=log_items,
-                compress=True
-            )
-            self.client.put_logs(request=request)
-            log.info(f"save data, data length: {len(data_list)}")
-            return True
-        except Exception as e:
-            log.exception(e)
-            return False
-

cobweb/db/storer/redis.py

@@ -1,15 +0,0 @@
-from cobweb import log, StorerInterface
-
-
-class Redis(StorerInterface):
-
-    def store(self, data_list):
-        try:
-            data_str = "\n".join(str(data) for data in data_list)
-            with open(self.table, "a") as fp:
-                fp.write(data_str)
-            log.info(f"save data, data length: {len(data_list)}")
-            return True
-        except Exception as e:
-            return False
-

cobweb/db/storer/textfile.py

@@ -1,15 +0,0 @@
-from . import Inf, log
-
-
-class Textfile(Inf):
-
-    def store(self, data_list):
-        try:
-            data_str = "\n".join(str(data) for data in data_list)
-            with open(self.table, "a") as fp:
-                fp.write(data_str)
-            log.info(f"save data, data length: {len(data_list)}")
-            return True
-        except Exception as e:
-            return False
-