cobweb-launcher 0.0.1__py3-none-any.whl

cobweb/__init__.py

@@ -0,0 +1,2 @@
+from .distributed import launcher
+from .base import task

cobweb/base/bbb.py

@@ -0,0 +1,187 @@
+# from typing import Iterable
+import json
+import time
+import hashlib
+from log import log
+from utils import struct_queue_name
+from collections import deque, namedtuple
+
+
+class Queue:
+
+    def __init__(self):
+        self._queue = deque()
+
+    @property
+    def length(self) -> int:
+        return len(self._queue)
+    #
+    # @property
+    # def queue_names(self):
+    #     return tuple(self.__dict__.keys())
+    #
+    # @property
+    # def used_memory(self):
+    #     return asizeof.asizeof(self)
+
+    # def create_queue(self, queue_name: str):
+    #     self.__setattr__(queue_name, deque())
+
+    # def push_seed(self, seed):
+    #     self.push("_seed_queue", seed)
+
+    # def pop_seed(self):
+    #     return self.pop("_seed_queue")
+
+    def push(self, data, left: bool = False, direct_insertion: bool = False):
+        try:
+            if not data:
+                return None
+            if direct_insertion or isinstance(data, Seed):
+                self._queue.appendleft(data) if left else self._queue.append(data)
+            elif any(isinstance(data, t) for t in (list, tuple)):
+                self._queue.extendleft(data) if left else self._queue.extend(data)
+        except AttributeError as e:
+            log.exception(e)
+
+    def pop(self, left: bool = True):
+        try:
+            return self._queue.popleft() if left else self._queue.pop()
+        except IndexError:
+            return None
+        except AttributeError as e:
+            log.exception(e)
+            return None
+
+
+class Seed:
+
+    def __init__(
+            self,
+            seed_info=None,
+            priority=300,
+            version=None,
+            retry=0,
+            **kwargs
+    ):
+        if seed_info:
+            if any(isinstance(seed_info, t) for t in (str, bytes)):
+                try:
+                    item = json.loads(seed_info)
+                    for k, v in item.items():
+                        self.__setattr__(k, v)
+                except json.JSONDecodeError:
+                    self.__setattr__("url", seed_info)
+            elif isinstance(seed_info, dict):
+                for k, v in seed_info.items():
+                    self.__setattr__(k, v)
+            else:
+                raise TypeError(Exception(
+                    f"seed type error, "
+                    f"must be str or dict! "
+                    f"seed_info: {seed_info}"
+                ))
+        for k, v in kwargs.items():
+            self.__setattr__(k, v)
+        if not getattr(self, "_priority"):
+            self._priority = min(max(1, int(priority)), 999)
+        if not getattr(self, "_version"):
+            self._version = int(version) or int(time.time())
+        if not getattr(self, "_retry"):
+            self._retry = retry
+        if not getattr(self, "sid"):
+            self.init_id()
+
+    def init_id(self):
+        item_string = self.format_seed
+        seed_id = hashlib.md5(item_string.encode()).hexdigest()
+        self.__setattr__("sid", seed_id)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __getattr__(self, name):
+        return None
+
+    def __str__(self):
+        return json.dumps(self.__dict__, ensure_ascii=False)
+
+    def __repr__(self):
+        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
+        return f'{self.__class__.__name__}({", ".join(chars)})'
+
+    @property
+    def format_seed(self):
+        seed = self.__dict__.copy()
+        del seed["_priority"]
+        del seed["_version"]
+        del seed["_retry"]
+        return json.dumps(seed, ensure_ascii=False)
+
+
+class DBItem:
+
+    def __init__(self, **kwargs):
+        self.__setattr__("_index", 0, True)
+        for table in self.__class__.__table__:
+            if set(kwargs.keys()) == set(table._fields):
+                break
+            self._index += 1
+
+        if self._index > len(self.__class__.__table__):
+            raise Exception()
+
+        table = self.__class__.__table__[self._index]
+        self.__setattr__("struct_data", table(**kwargs), True)
+        self.__setattr__("db_name", self.__class__.__name__, True)
+        self.__setattr__("table_name", self.struct_data.__class__.__name__, True)
+
+    @classmethod
+    def init_item(cls, table_name, fields):
+        queue_name = struct_queue_name(cls.__name__, table_name)
+        if getattr(cls, queue_name, None) is None:
+            setattr(cls, queue_name, Queue())
+
+        if getattr(cls, "__table__", None) is None:
+            cls.__table__ = []
+
+        table = namedtuple(table_name, fields)
+
+        if table in getattr(cls, "__table__"):
+            raise Exception()
+        getattr(cls, "__table__").append(table)
+
+    def queue(self):
+        queue_name = struct_queue_name(self.db_name, self.table_name)
+        return getattr(self.__class__, queue_name)
+
+    def __setitem__(self, key, value):
+        self.__setattr__(key, value)
+
+    def __getitem__(self, item):
+        return self.struct_data[item]
+
+    def __getattr__(self, name):
+        return None
+
+    def __setattr__(self, key, value, init=None):
+        if init:
+            super().__setattr__(key, value)
+        elif not getattr(self, "struct_data"):
+            raise Exception(f"no struct_data")
+        else:
+            self.__setattr__(
+                "struct_data",
+                self.struct_data._replace(**{key: value}),
+                init=True
+            )
+
+    def __str__(self):
+        return json.dumps(self.struct_data._asdict(), ensure_ascii=False)
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}:{self.struct_data}'
+

cobweb/base/config.py

@@ -0,0 +1,164 @@
+import json
+from collections import namedtuple
+from utils import struct_table_name
+
+StorerInfo = namedtuple(
+    "StorerInfo",
+    "DB, table, fields, length, config"
+)
+SchedulerInfo = namedtuple(
+    "SchedulerInfo",
+    "DB, table, sql, length, size, config",
+)
+RedisInfo = namedtuple(
+    "RedisInfo",
+    "host, port, username, password, db",
+    defaults=("localhost", 6379, None, None, 0)
+)
+
+# redis_info = dict(
+#     host="localhost",
+#     port=6379,
+#     username=None,
+#     password=None,
+#     db=0
+# )
+
+
+class SchedulerDB:
+
+    @staticmethod
+    def default():
+        from db.scheduler.default import Default
+        return SchedulerInfo(DB=Default, table="", sql="", length=100, size=500000, config=None)
+
+    @staticmethod
+    def textfile(table, sql=None, length=100, size=500000):
+        from db.scheduler.textfile import Textfile
+        return SchedulerInfo(DB=Textfile, table=table, sql=sql, length=length, size=size, config=None)
+
+    @staticmethod
+    def diy(DB, table, sql=None, length=100, size=500000, config=None):
+        from base.interface import SchedulerInterface
+        if not isinstance(DB, SchedulerInterface):
+            raise Exception("DB must be inherit from SchedulerInterface")
+        return SchedulerInfo(DB=DB, table=table, sql=sql, length=length, size=size, config=config)
+
+    # @staticmethod
+    # def info(scheduler_info):
+    #     if not scheduler_info:
+    #         return SchedulerDB.default()
+    #
+    #     if isinstance(scheduler_info, SchedulerInfo):
+    #         return scheduler_info
+    #
+    #     if isinstance(scheduler_info, str):
+    #         scheduler = json.loads(scheduler_info)
+    #         if isinstance(scheduler, dict):
+    #             db_name = scheduler["DB"]
+    #             if db_name in dir(SchedulerDB):
+    #                 del scheduler["DB"]
+    #             else:
+    #                 db_name = "diy"
+    #             func = getattr(SchedulerDB, db_name)
+    #             return func(**scheduler)
+
+
+class StorerDB:
+
+    @staticmethod
+    def console(table, fields, length=200):
+        from db.storer.console import Console
+        table = struct_table_name(table)
+        return StorerInfo(DB=Console, table=table, fields=fields, length=length, config=None)
+
+    @staticmethod
+    def textfile(table, fields, length=200):
+        from db.storer.textfile import Textfile
+        table = struct_table_name(table)
+        return StorerInfo(DB=Textfile, table=table, fields=fields, length=length, config=None)
+
+    @staticmethod
+    def loghub(table, fields, length=200, config=None):
+        from db.storer.loghub import Loghub
+        table = struct_table_name(table)
+        return StorerInfo(DB=Loghub, table=table, fields=fields, length=length, config=config)
+
+    @staticmethod
+    def diy(DB, table, fields, length=200, config=None):
+        from base.interface import StorerInterface
+        if not isinstance(DB, StorerInterface):
+            raise Exception("DB must be inherit from StorerInterface")
+        table = struct_table_name(table)
+        return StorerInfo(DB=DB, table=table, fields=fields, length=length, config=config)
+
+    # @staticmethod
+    # def info(storer_info):
+    #     if not storer_info:
+    #         return None
+    #
+    #     if isinstance(storer_info, str):
+    #         storer_info = json.loads(storer_info)
+    #
+    #     if any(isinstance(storer_info, t) for t in (dict, StorerInfo)):
+    #         storer_info = [storer_info]
+    #
+    #     if not isinstance(storer_info, list):
+    #         raise Exception("StorerDB.info storer_info")
+    #
+    #     storer_info_list = []
+    #     for storer in storer_info:
+    #         if isinstance(storer, StorerInfo):
+    #             storer_info_list.append(storer)
+    #         else:
+    #             db_name = storer["DB"]
+    #             if db_name in dir(StorerDB):
+    #                 del storer["DB"]
+    #             else:
+    #                 db_name = "diy"
+    #             func = getattr(StorerDB, db_name)
+    #             storer_info_list.append(func(**storer))
+    #     return storer_info_list
+
+
+
+def deal(config, tag):
+    if isinstance(config, dict):
+        if tag == 0:
+            return RedisInfo(**config)
+        elif tag == 1:
+            db_name = config["DB"]
+            if db_name in dir(SchedulerDB):
+                del config["DB"]
+            else:
+                db_name = "diy"
+            func = getattr(SchedulerDB, db_name)
+            return func(**config)
+        elif tag == 2:
+            db_name = config["DB"]
+            if db_name in dir(StorerDB):
+                del config["DB"]
+            else:
+                db_name = "diy"
+            func = getattr(StorerDB, db_name)
+            return func(**config)
+        raise ValueError("tag must be in [0, 1, 2]")
+    elif any(isinstance(config, t) for t in (StorerInfo, SchedulerInfo, RedisInfo)):
+        return config
+    raise TypeError("config must be in [StorerInfo, SchedulerInfo, RedisInfo]")
+
+
+def info(configs, tag = 0):
+    if configs is None:
+        return SchedulerDB.default() if tag == 1 else None
+
+    if isinstance(configs, str):
+        configs = json.loads(configs)
+
+    if tag == 0:
+        return deal(configs, tag)
+
+    if not isinstance(configs, list):
+        configs = [configs]
+
+    return [deal(config, tag) for config in configs]

cobweb/base/decorators.py

@@ -0,0 +1,95 @@
+import time
+from functools import wraps
+
+# from config import DBType
+from log import log
+
+
+# def find_func_name(func_name, name_list):
+#     for name in name_list:
+#         if func_name.find(name) == 0:
+#             return True
+#     return False
+
+
+# def starter_decorator(execute):
+#     @wraps(execute)
+#     def wrapper(starter, *args, **kwargs):
+#         spider_dynamic_funcs = []
+#         scheduler_dynamic_funcs = []
+#         starter_functions = inspect.getmembers(starter, lambda a: inspect.isfunction(a))
+#         for starter_function in starter_functions:
+#             if find_func_name(starter_function[0], starter.scheduler_funcs):
+#                 scheduler_dynamic_funcs.append(starter_function)
+#             elif find_func_name(starter_function[0], starter.spider_funcs):
+#                 spider_dynamic_funcs.append(starter_function)
+#         return execute(starter, scheduler_dynamic_funcs, spider_dynamic_funcs, *args, **kwargs)
+#
+#     return wrapper
+
+
+# def scheduler_decorator(execute):
+#     @wraps(execute)
+#     def wrapper(scheduler, distribute_task):
+#         if not issubclass(scheduler, SchedulerInterface):
+#             scheduler.stop = True
+#         elif getattr(scheduler, "scheduler", None):
+#             execute(scheduler, distribute_task)
+#         else:
+#             log.error(f"scheduler type: {scheduler.db_type} not have add function!")
+#             scheduler.stop = True
+#     return wrapper
+
+
+def storer_decorator(execute):
+    @wraps(execute)
+    def wrapper(storer, stop_event, last_event, table_name, callback):
+        if getattr(storer, "save", None):
+            execute(storer, stop_event, last_event, table_name, callback)
+        else:
+            log.error(f"storer base_type: {storer.data_type} not have add function!")
+            storer.stop = True
+    return wrapper
+
+
+def distribute_scheduler_decorators(func):
+    @wraps(func)
+    def wrapper(distributor, callback):
+        try:
+            func(distributor, callback)
+        except TypeError:
+            pass
+        distributor.event.set()
+    return wrapper
+
+
+def distribute_spider_decorators(func):
+    @wraps(func)
+    def wrapper(distributor, stop_event, db, callback):
+        while not stop_event.is_set():
+            try:
+                seed = distributor.queue_client.pop("_seed_queue")
+                if not seed:
+                    time.sleep(3)
+                    continue
+                distributor.spider_in_progress.append(1)
+                func(distributor, db, seed, callback)
+            except Exception as e:
+                print(e)
+            finally:
+                distributor.spider_in_progress.pop()
+
+    return wrapper
+
+
+def distribute_storer_decorators(func):
+    @wraps(func)
+    def wrapper(distributor, callback, data_type, table_name, last):
+        data_list = []
+        try:
+            func(distributor, callback, data_list, data_type, table_name, last)
+        except Exception as e:
+            log.info("storage exception! " + str(e))
+            # distributor._task_queue.extendleft(data_list)
+
+    return wrapper

cobweb/base/hash_table.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+class DynamicHashTable:
+    def __init__(self):
+        self.capacity = 1000  # 初始容量
+        self.size = 0  # 元素个数
+        self.table = [None] * self.capacity
+
+    def hash_function(self, key):
+        return hash(key) % self.capacity
+
+    def probe(self, index):
+        # 线性探测法
+        return (index + 1) % self.capacity
+
+    def insert(self, key, value):
+        index = self.hash_function(key)
+        while self.table[index] is not None:
+            if self.table[index][0] == key:
+                self.table[index][1] = value
+                return
+            index = self.probe(index)
+        self.table[index] = [key, value]
+        self.size += 1
+
+        # 动态扩容
+        if self.size / self.capacity >= 0.7:
+            self.resize()
+
+    def get(self, key):
+        index = self.hash_function(key)
+        while self.table[index] is not None:
+            if self.table[index][0] == key:
+                return self.table[index][1]
+            index = self.probe(index)
+        raise KeyError("Key not found")
+
+    def remove(self, key):
+        index = self.hash_function(key)
+        while self.table[index] is not None:
+            if self.table[index][0] == key:
+                self.table[index] = None
+                self.size -= 1
+                return
+            index = self.probe(index)
+        raise KeyError("Key not found")
+
+    def resize(self):
+        # 扩容为原容量的两倍
+        self.capacity *= 2
+        new_table = [None] * self.capacity
+        for item in self.table:
+            if item is not None:
+                key, value = item
+                new_index = self.hash_function(key)
+                while new_table[new_index] is not None:
+                    new_index = self.probe(new_index)
+                new_table[new_index] = [key, value]
+        self.table = new_table
+

cobweb/base/interface.py

@@ -0,0 +1,44 @@
+import json
+from abc import ABC, abstractmethod
+
+
+def parse_config(config):
+    if not config:
+        return None
+    if isinstance(config, str):
+        return json.loads(config)
+    if isinstance(config, dict):
+        return config
+    raise TypeError("config type is not in [string, dict]!")
+
+
+class SchedulerInterface(ABC):
+
+    def __init__(self, table, sql, length, size, queue, config=None):
+        self.sql = sql
+        self.table = table
+        self.length = length
+        self.size = size
+        self.queue = queue
+        self.config = parse_config(config)
+        self.stop = False
+
+    @abstractmethod
+    def schedule(self, *args, **kwargs):
+        pass
+
+
+class StorerInterface(ABC):
+
+    def __init__(self, table, fields, length, queue, config=None):
+        self.table = table
+        self.fields = fields
+        self.length = length
+        self.queue = queue
+        self.config = parse_config(config)
+        # self.redis_db = redis_db
+
+    @abstractmethod
+    def store(self, *args, **kwargs):
+        pass
+

cobweb/base/log.py

@@ -0,0 +1,96 @@
+import logging
+
+
+class ColorCodes:
+    # Text Reset
+    RESET = "\033[0m"
+
+    # Regular Colors
+    RED = "\033[31m"
+    GREEN = "\033[32m"
+    YELLOW = "\033[33m"
+    BLUE = "\033[34m"
+    PURPLE = "\033[35m"
+    CYAN = "\033[36m"
+    WHITE = "\033[37m"
+
+    # Bright Colors
+    BRIGHT_RED = "\033[91m"
+    BRIGHT_GREEN = "\033[92m"
+    BRIGHT_YELLOW = "\033[93m"
+    BRIGHT_BLUE = "\033[94m"
+    BRIGHT_PURPLE = "\033[95m"
+    BRIGHT_CYAN = "\033[96m"
+    BRIGHT_WHITE = "\033[97m"
+
+    # Background Colors
+    BG_RED = "\033[41m"
+    BG_GREEN = "\033[42m"
+    BG_YELLOW = "\033[43m"
+    BG_BLUE = "\033[44m"
+    BG_PURPLE = "\033[45m"
+    BG_CYAN = "\033[46m"
+    BG_WHITE = "\033[47m"
+
+    # Bright Background Colors
+    BG_BRIGHT_RED = "\033[101m"
+    BG_BRIGHT_GREEN = "\033[102m"
+    BG_BRIGHT_YELLOW = "\033[103m"
+    BG_BRIGHT_BLUE = "\033[104m"
+    BG_BRIGHT_PURPLE = "\033[105m"
+    BG_BRIGHT_CYAN = "\033[106m"
+    BG_BRIGHT_WHITE = "\033[107m"
+
+    # Text Styles
+    BOLD = "\033[1m"
+    DIM = "\033[2m"
+    ITALIC = "\033[3m"
+    UNDERLINE = "\033[4m"
+    BLINK = "\033[5m"
+    REVERSE = "\033[7m"
+    HIDDEN = "\033[8m"
+
+
+class Log:
+    logging.basicConfig(
+        level=logging.INFO,
+        format=f'%(asctime)s %(name)s [%(filename)s:%(lineno)d %(funcName)s]'
+               f' %(levelname)s -> %(message)s'
+    )
+    log = logging.getLogger()
+
+    def set_log_name(self, name):
+        self.__class__.log = logging.getLogger(name)
+
+    @property
+    def debug(self):
+        return self.__class__.log.debug
+
+    @property
+    def info(self):
+        return self.__class__.log.info
+
+    @property
+    def warning(self):
+        return self.__class__.log.warning
+
+    @property
+    def exception(self):
+        return self.__class__.log.exception
+
+    @property
+    def error(self):
+        return self.__class__.log.error
+
+    @property
+    def critical(self):
+        return self.__class__.log.critical
+
+
+log = Log()
+# log.info("This text will be bold!")
+# print(ColorCodes.BOLD + "This text will be bold!" + ColorCodes.RESET)
+# print(ColorCodes.UNDERLINE + ColorCodes.BLUE + "This text will be underlined and blue!" + ColorCodes.RESET)
+# print(ColorCodes.BG_YELLOW + ColorCodes.RED + "This text will have a yellow background and red text!" + ColorCodes.RESET)
+# print(ColorCodes.BLINK + "This text will blink (if supported by the terminal)!" + ColorCodes.RESET)
+