cobweb-launcher 0.0.1__tar.gz

cobweb-launcher-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Juannie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cobweb-launcher-0.0.1/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.1
+Name: cobweb-launcher
+Version: 0.0.1
+Summary: spider_hole
+Home-page: https://github.com/Juannie-PP/cobweb
+Author: Juannie-PP
+Author-email: 2604868278@qq.com
+License: MIT
+Keywords: cobweb
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# cobweb
+
+> 通用爬虫框架： 1.单机模式采集框架；2.分布式采集框架
+> 
+>  5部分
+>  
+> 1. starter -- 启动器
+> 
+> 2. scheduler -- 调度器
+> 
+> 3. distributor -- 分发器
+> 
+> 4. storer -- 存储器
+> 
+> 5. utils -- 工具函数
+> 
+
+need deal
+- 队列优化完善，使用queue的机制wait()同步各模块执行？
+- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- 去重过滤（布隆过滤器等）
+- 防丢失（单机模式可以通过日志文件进行检查种子）
+- 自定义数据库的功能
+- excel、mysql、redis数据完善
+
+
+![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)

cobweb-launcher-0.0.1/README.md

@@ -0,0 +1,27 @@
+# cobweb
+
+> 通用爬虫框架： 1.单机模式采集框架；2.分布式采集框架
+> 
+>  5部分
+>  
+> 1. starter -- 启动器
+> 
+> 2. scheduler -- 调度器
+> 
+> 3. distributor -- 分发器
+> 
+> 4. storer -- 存储器
+> 
+> 5. utils -- 工具函数
+> 
+
+need deal
+- 队列优化完善，使用queue的机制wait()同步各模块执行？
+- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- 去重过滤（布隆过滤器等）
+- 防丢失（单机模式可以通过日志文件进行检查种子）
+- 自定义数据库的功能
+- excel、mysql、redis数据完善
+
+
+![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)

cobweb-launcher-0.0.1/cobweb/__init__.py

@@ -0,0 +1,2 @@
+from .distributed import launcher
+from .base import task

cobweb-launcher-0.0.1/cobweb/base/bbb.py

@@ -0,0 +1,187 @@
+# from typing import Iterable
+import json
+import time
+import hashlib
+from log import log
+from utils import struct_queue_name
+from collections import deque, namedtuple
+
+
+class Queue:
+
+    def __init__(self):
+        self._queue = deque()
+
+    @property
+    def length(self) -> int:
+        return len(self._queue)
+    #
+    # @property
+    # def queue_names(self):
+    #     return tuple(self.__dict__.keys())
+    #
+    # @property
+    # def used_memory(self):
+    #     return asizeof.asizeof(self)
+
+    # def create_queue(self, queue_name: str):
+    #     self.__setattr__(queue_name, deque())
+
+    # def push_seed(self, seed):
+    #     self.push("_seed_queue", seed)
+
+    # def pop_seed(self):
+    #     return self.pop("_seed_queue")
+
+    def push(self, data, left: bool = False, direct_insertion: bool = False):
+        try:
+            if not data:
+                return None
+            if direct_insertion or isinstance(data, Seed):
+                self._queue.appendleft(data) if left else self._queue.append(data)
+            elif any(isinstance(data, t) for t in (list, tuple)):
+                self._queue.extendleft(data) if left else self._queue.extend(data)
+        except AttributeError as e:
+            log.exception(e)
+
+    def pop(self, left: bool = True):
+        try:
+            return self._queue.popleft() if left else self._queue.pop()
+        except IndexError:
+            return None
+        except AttributeError as e:
+            log.exception(e)
+            return None
+
+
+class Seed:
+
+    def __init__(
+            self,
+            seed_info=None,
+            priority=300,
+            version=None,
+            retry=0,
+            **kwargs
+    ):
+        if seed_info:
+            if any(isinstance(seed_info, t) for t in (str, bytes)):
+                try:
+                    item = json.loads(seed_info)
+                    for k, v in item.items():
+                        self.__setattr__(k, v)
+                except json.JSONDecodeError:
+                    self.__setattr__("url", seed_info)
+            elif isinstance(seed_info, dict):
+                for k, v in seed_info.items():
+                    self.__setattr__(k, v)
+            else:
+                raise TypeError(Exception(
+                    f"seed type error, "
+                    f"must be str or dict! "
+                    f"seed_info: {seed_info}"
+                ))
+        for k, v in kwargs.items():
+            self.__setattr__(k, v)
+        if not getattr(self, "_priority"):
+            self._priority = min(max(1, int(priority)), 999)
+        if not getattr(self, "_version"):
+            self._version = int(version) or int(time.time())
+        if not getattr(self, "_retry"):
+            self._retry = retry
+        if not getattr(self, "sid"):
+            self.init_id()
+
+    def init_id(self):
+        item_string = self.format_seed
+        seed_id = hashlib.md5(item_string.encode()).hexdigest()
+        self.__setattr__("sid", seed_id)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __getattr__(self, name):
+        return None
+
+    def __str__(self):
+        return json.dumps(self.__dict__, ensure_ascii=False)
+
+    def __repr__(self):
+        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
+        return f'{self.__class__.__name__}({", ".join(chars)})'
+
+    @property
+    def format_seed(self):
+        seed = self.__dict__.copy()
+        del seed["_priority"]
+        del seed["_version"]
+        del seed["_retry"]
+        return json.dumps(seed, ensure_ascii=False)
+
+
+class DBItem:
+
+    def __init__(self, **kwargs):
+        self.__setattr__("_index", 0, True)
+        for table in self.__class__.__table__:
+            if set(kwargs.keys()) == set(table._fields):
+                break
+            self._index += 1
+
+        if self._index > len(self.__class__.__table__):
+            raise Exception()
+
+        table = self.__class__.__table__[self._index]
+        self.__setattr__("struct_data", table(**kwargs), True)
+        self.__setattr__("db_name", self.__class__.__name__, True)
+        self.__setattr__("table_name", self.struct_data.__class__.__name__, True)
+
+    @classmethod
+    def init_item(cls, table_name, fields):
+        queue_name = struct_queue_name(cls.__name__, table_name)
+        if getattr(cls, queue_name, None) is None:
+            setattr(cls, queue_name, Queue())
+
+        if getattr(cls, "__table__", None) is None:
+            cls.__table__ = []
+
+        table = namedtuple(table_name, fields)
+
+        if table in getattr(cls, "__table__"):
+            raise Exception()
+        getattr(cls, "__table__").append(table)
+
+    def queue(self):
+        queue_name = struct_queue_name(self.db_name, self.table_name)
+        return getattr(self.__class__, queue_name)
+
+    def __setitem__(self, key, value):
+        self.__setattr__(key, value)
+
+    def __getitem__(self, item):
+        return self.struct_data[item]
+
+    def __getattr__(self, name):
+        return None
+
+    def __setattr__(self, key, value, init=None):
+        if init:
+            super().__setattr__(key, value)
+        elif not getattr(self, "struct_data"):
+            raise Exception(f"no struct_data")
+        else:
+            self.__setattr__(
+                "struct_data",
+                self.struct_data._replace(**{key: value}),
+                init=True
+            )
+
+    def __str__(self):
+        return json.dumps(self.struct_data._asdict(), ensure_ascii=False)
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}:{self.struct_data}'
+

cobweb-launcher-0.0.1/cobweb/base/config.py

@@ -0,0 +1,164 @@
+import json
+from collections import namedtuple
+from utils import struct_table_name
+
+StorerInfo = namedtuple(
+    "StorerInfo",
+    "DB, table, fields, length, config"
+)
+SchedulerInfo = namedtuple(
+    "SchedulerInfo",
+    "DB, table, sql, length, size, config",
+)
+RedisInfo = namedtuple(
+    "RedisInfo",
+    "host, port, username, password, db",
+    defaults=("localhost", 6379, None, None, 0)
+)
+
+# redis_info = dict(
+#     host="localhost",
+#     port=6379,
+#     username=None,
+#     password=None,
+#     db=0
+# )
+
+
+class SchedulerDB:
+
+    @staticmethod
+    def default():
+        from db.scheduler.default import Default
+        return SchedulerInfo(DB=Default, table="", sql="", length=100, size=500000, config=None)
+
+    @staticmethod
+    def textfile(table, sql=None, length=100, size=500000):
+        from db.scheduler.textfile import Textfile
+        return SchedulerInfo(DB=Textfile, table=table, sql=sql, length=length, size=size, config=None)
+
+    @staticmethod
+    def diy(DB, table, sql=None, length=100, size=500000, config=None):
+        from base.interface import SchedulerInterface
+        if not isinstance(DB, SchedulerInterface):
+            raise Exception("DB must be inherit from SchedulerInterface")
+        return SchedulerInfo(DB=DB, table=table, sql=sql, length=length, size=size, config=config)
+
+    # @staticmethod
+    # def info(scheduler_info):
+    #     if not scheduler_info:
+    #         return SchedulerDB.default()
+    #
+    #     if isinstance(scheduler_info, SchedulerInfo):
+    #         return scheduler_info
+    #
+    #     if isinstance(scheduler_info, str):
+    #         scheduler = json.loads(scheduler_info)
+    #         if isinstance(scheduler, dict):
+    #             db_name = scheduler["DB"]
+    #             if db_name in dir(SchedulerDB):
+    #                 del scheduler["DB"]
+    #             else:
+    #                 db_name = "diy"
+    #             func = getattr(SchedulerDB, db_name)
+    #             return func(**scheduler)
+
+
+class StorerDB:
+
+    @staticmethod
+    def console(table, fields, length=200):
+        from db.storer.console import Console
+        table = struct_table_name(table)
+        return StorerInfo(DB=Console, table=table, fields=fields, length=length, config=None)
+
+    @staticmethod
+    def textfile(table, fields, length=200):
+        from db.storer.textfile import Textfile
+        table = struct_table_name(table)
+        return StorerInfo(DB=Textfile, table=table, fields=fields, length=length, config=None)
+
+    @staticmethod
+    def loghub(table, fields, length=200, config=None):
+        from db.storer.loghub import Loghub
+        table = struct_table_name(table)
+        return StorerInfo(DB=Loghub, table=table, fields=fields, length=length, config=config)
+
+    @staticmethod
+    def diy(DB, table, fields, length=200, config=None):
+        from base.interface import StorerInterface
+        if not isinstance(DB, StorerInterface):
+            raise Exception("DB must be inherit from StorerInterface")
+        table = struct_table_name(table)
+        return StorerInfo(DB=DB, table=table, fields=fields, length=length, config=config)
+
+    # @staticmethod
+    # def info(storer_info):
+    #     if not storer_info:
+    #         return None
+    #
+    #     if isinstance(storer_info, str):
+    #         storer_info = json.loads(storer_info)
+    #
+    #     if any(isinstance(storer_info, t) for t in (dict, StorerInfo)):
+    #         storer_info = [storer_info]
+    #
+    #     if not isinstance(storer_info, list):
+    #         raise Exception("StorerDB.info storer_info")
+    #
+    #     storer_info_list = []
+    #     for storer in storer_info:
+    #         if isinstance(storer, StorerInfo):
+    #             storer_info_list.append(storer)
+    #         else:
+    #             db_name = storer["DB"]
+    #             if db_name in dir(StorerDB):
+    #                 del storer["DB"]
+    #             else:
+    #                 db_name = "diy"
+    #             func = getattr(StorerDB, db_name)
+    #             storer_info_list.append(func(**storer))
+    #     return storer_info_list
+
+
+
+def deal(config, tag):
+    if isinstance(config, dict):
+        if tag == 0:
+            return RedisInfo(**config)
+        elif tag == 1:
+            db_name = config["DB"]
+            if db_name in dir(SchedulerDB):
+                del config["DB"]
+            else:
+                db_name = "diy"
+            func = getattr(SchedulerDB, db_name)
+            return func(**config)
+        elif tag == 2:
+            db_name = config["DB"]
+            if db_name in dir(StorerDB):
+                del config["DB"]
+            else:
+                db_name = "diy"
+            func = getattr(StorerDB, db_name)
+            return func(**config)
+        raise ValueError("tag must be in [0, 1, 2]")
+    elif any(isinstance(config, t) for t in (StorerInfo, SchedulerInfo, RedisInfo)):
+        return config
+    raise TypeError("config must be in [StorerInfo, SchedulerInfo, RedisInfo]")
+
+
+def info(configs, tag = 0):
+    if configs is None:
+        return SchedulerDB.default() if tag == 1 else None
+
+    if isinstance(configs, str):
+        configs = json.loads(configs)
+
+    if tag == 0:
+        return deal(configs, tag)
+
+    if not isinstance(configs, list):
+        configs = [configs]
+
+    return [deal(config, tag) for config in configs]

cobweb-launcher-0.0.1/cobweb/base/decorators.py

@@ -0,0 +1,95 @@
+import time
+from functools import wraps
+
+# from config import DBType
+from log import log
+
+
+# def find_func_name(func_name, name_list):
+#     for name in name_list:
+#         if func_name.find(name) == 0:
+#             return True
+#     return False
+
+
+# def starter_decorator(execute):
+#     @wraps(execute)
+#     def wrapper(starter, *args, **kwargs):
+#         spider_dynamic_funcs = []
+#         scheduler_dynamic_funcs = []
+#         starter_functions = inspect.getmembers(starter, lambda a: inspect.isfunction(a))
+#         for starter_function in starter_functions:
+#             if find_func_name(starter_function[0], starter.scheduler_funcs):
+#                 scheduler_dynamic_funcs.append(starter_function)
+#             elif find_func_name(starter_function[0], starter.spider_funcs):
+#                 spider_dynamic_funcs.append(starter_function)
+#         return execute(starter, scheduler_dynamic_funcs, spider_dynamic_funcs, *args, **kwargs)
+#
+#     return wrapper
+
+
+# def scheduler_decorator(execute):
+#     @wraps(execute)
+#     def wrapper(scheduler, distribute_task):
+#         if not issubclass(scheduler, SchedulerInterface):
+#             scheduler.stop = True
+#         elif getattr(scheduler, "scheduler", None):
+#             execute(scheduler, distribute_task)
+#         else:
+#             log.error(f"scheduler type: {scheduler.db_type} not have add function!")
+#             scheduler.stop = True
+#     return wrapper
+
+
+def storer_decorator(execute):
+    @wraps(execute)
+    def wrapper(storer, stop_event, last_event, table_name, callback):
+        if getattr(storer, "save", None):
+            execute(storer, stop_event, last_event, table_name, callback)
+        else:
+            log.error(f"storer base_type: {storer.data_type} not have add function!")
+            storer.stop = True
+    return wrapper
+
+
+def distribute_scheduler_decorators(func):
+    @wraps(func)
+    def wrapper(distributor, callback):
+        try:
+            func(distributor, callback)
+        except TypeError:
+            pass
+        distributor.event.set()
+    return wrapper
+
+
+def distribute_spider_decorators(func):
+    @wraps(func)
+    def wrapper(distributor, stop_event, db, callback):
+        while not stop_event.is_set():
+            try:
+                seed = distributor.queue_client.pop("_seed_queue")
+                if not seed:
+                    time.sleep(3)
+                    continue
+                distributor.spider_in_progress.append(1)
+                func(distributor, db, seed, callback)
+            except Exception as e:
+                print(e)
+            finally:
+                distributor.spider_in_progress.pop()
+
+    return wrapper
+
+
+def distribute_storer_decorators(func):
+    @wraps(func)
+    def wrapper(distributor, callback, data_type, table_name, last):
+        data_list = []
+        try:
+            func(distributor, callback, data_list, data_type, table_name, last)
+        except Exception as e:
+            log.info("storage exception! " + str(e))
+            # distributor._task_queue.extendleft(data_list)
+
+    return wrapper

cobweb-launcher-0.0.1/cobweb/base/hash_table.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+class DynamicHashTable:
+    def __init__(self):
+        self.capacity = 1000  # 初始容量
+        self.size = 0  # 元素个数
+        self.table = [None] * self.capacity
+
+    def hash_function(self, key):
+        return hash(key) % self.capacity
+
+    def probe(self, index):
+        # 线性探测法
+        return (index + 1) % self.capacity
+
+    def insert(self, key, value):
+        index = self.hash_function(key)
+        while self.table[index] is not None:
+            if self.table[index][0] == key:
+                self.table[index][1] = value
+                return
+            index = self.probe(index)
+        self.table[index] = [key, value]
+        self.size += 1
+
+        # 动态扩容
+        if self.size / self.capacity >= 0.7:
+            self.resize()
+
+    def get(self, key):
+        index = self.hash_function(key)
+        while self.table[index] is not None:
+            if self.table[index][0] == key:
+                return self.table[index][1]
+            index = self.probe(index)
+        raise KeyError("Key not found")
+
+    def remove(self, key):
+        index = self.hash_function(key)
+        while self.table[index] is not None:
+            if self.table[index][0] == key:
+                self.table[index] = None
+                self.size -= 1
+                return
+            index = self.probe(index)
+        raise KeyError("Key not found")
+
+    def resize(self):
+        # 扩容为原容量的两倍
+        self.capacity *= 2
+        new_table = [None] * self.capacity
+        for item in self.table:
+            if item is not None:
+                key, value = item
+                new_index = self.hash_function(key)
+                while new_table[new_index] is not None:
+                    new_index = self.probe(new_index)
+                new_table[new_index] = [key, value]
+        self.table = new_table
+

cobweb-launcher-0.0.1/cobweb/base/interface.py

@@ -0,0 +1,44 @@
+import json
+from abc import ABC, abstractmethod
+
+
+def parse_config(config):
+    if not config:
+        return None
+    if isinstance(config, str):
+        return json.loads(config)
+    if isinstance(config, dict):
+        return config
+    raise TypeError("config type is not in [string, dict]!")
+
+
+class SchedulerInterface(ABC):
+
+    def __init__(self, table, sql, length, size, queue, config=None):
+        self.sql = sql
+        self.table = table
+        self.length = length
+        self.size = size
+        self.queue = queue
+        self.config = parse_config(config)
+        self.stop = False
+
+    @abstractmethod
+    def schedule(self, *args, **kwargs):
+        pass
+
+
+class StorerInterface(ABC):
+
+    def __init__(self, table, fields, length, queue, config=None):
+        self.table = table
+        self.fields = fields
+        self.length = length
+        self.queue = queue
+        self.config = parse_config(config)
+        # self.redis_db = redis_db
+
+    @abstractmethod
+    def store(self, *args, **kwargs):
+        pass
+