cobweb-launcher 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

cobweb/base/config.py

@@ -1,164 +0,0 @@
-import json
-from collections import namedtuple
-from utils import struct_table_name
-
-StorerInfo = namedtuple(
-    "StorerInfo",
-    "DB, table, fields, length, config"
-)
-SchedulerInfo = namedtuple(
-    "SchedulerInfo",
-    "DB, table, sql, length, size, config",
-)
-RedisInfo = namedtuple(
-    "RedisInfo",
-    "host, port, username, password, db",
-    defaults=("localhost", 6379, None, None, 0)
-)
-
-# redis_info = dict(
-#     host="localhost",
-#     port=6379,
-#     username=None,
-#     password=None,
-#     db=0
-# )
-
-
-class SchedulerDB:
-
-    @staticmethod
-    def default():
-        from db.scheduler.default import Default
-        return SchedulerInfo(DB=Default, table="", sql="", length=100, size=500000, config=None)
-
-    @staticmethod
-    def textfile(table, sql=None, length=100, size=500000):
-        from db.scheduler.textfile import Textfile
-        return SchedulerInfo(DB=Textfile, table=table, sql=sql, length=length, size=size, config=None)
-
-    @staticmethod
-    def diy(DB, table, sql=None, length=100, size=500000, config=None):
-        from base.interface import SchedulerInterface
-        if not isinstance(DB, SchedulerInterface):
-            raise Exception("DB must be inherit from SchedulerInterface")
-        return SchedulerInfo(DB=DB, table=table, sql=sql, length=length, size=size, config=config)
-
-    # @staticmethod
-    # def info(scheduler_info):
-    #     if not scheduler_info:
-    #         return SchedulerDB.default()
-    #
-    #     if isinstance(scheduler_info, SchedulerInfo):
-    #         return scheduler_info
-    #
-    #     if isinstance(scheduler_info, str):
-    #         scheduler = json.loads(scheduler_info)
-    #         if isinstance(scheduler, dict):
-    #             db_name = scheduler["DB"]
-    #             if db_name in dir(SchedulerDB):
-    #                 del scheduler["DB"]
-    #             else:
-    #                 db_name = "diy"
-    #             func = getattr(SchedulerDB, db_name)
-    #             return func(**scheduler)
-
-
-class StorerDB:
-
-    @staticmethod
-    def console(table, fields, length=200):
-        from db.storer.console import Console
-        table = struct_table_name(table)
-        return StorerInfo(DB=Console, table=table, fields=fields, length=length, config=None)
-
-    @staticmethod
-    def textfile(table, fields, length=200):
-        from db.storer.textfile import Textfile
-        table = struct_table_name(table)
-        return StorerInfo(DB=Textfile, table=table, fields=fields, length=length, config=None)
-
-    @staticmethod
-    def loghub(table, fields, length=200, config=None):
-        from db.storer.loghub import Loghub
-        table = struct_table_name(table)
-        return StorerInfo(DB=Loghub, table=table, fields=fields, length=length, config=config)
-
-    @staticmethod
-    def diy(DB, table, fields, length=200, config=None):
-        from base.interface import StorerInterface
-        if not isinstance(DB, StorerInterface):
-            raise Exception("DB must be inherit from StorerInterface")
-        table = struct_table_name(table)
-        return StorerInfo(DB=DB, table=table, fields=fields, length=length, config=config)
-
-    # @staticmethod
-    # def info(storer_info):
-    #     if not storer_info:
-    #         return None
-    #
-    #     if isinstance(storer_info, str):
-    #         storer_info = json.loads(storer_info)
-    #
-    #     if any(isinstance(storer_info, t) for t in (dict, StorerInfo)):
-    #         storer_info = [storer_info]
-    #
-    #     if not isinstance(storer_info, list):
-    #         raise Exception("StorerDB.info storer_info")
-    #
-    #     storer_info_list = []
-    #     for storer in storer_info:
-    #         if isinstance(storer, StorerInfo):
-    #             storer_info_list.append(storer)
-    #         else:
-    #             db_name = storer["DB"]
-    #             if db_name in dir(StorerDB):
-    #                 del storer["DB"]
-    #             else:
-    #                 db_name = "diy"
-    #             func = getattr(StorerDB, db_name)
-    #             storer_info_list.append(func(**storer))
-    #     return storer_info_list
-
-
-
-def deal(config, tag):
-    if isinstance(config, dict):
-        if tag == 0:
-            return RedisInfo(**config)
-        elif tag == 1:
-            db_name = config["DB"]
-            if db_name in dir(SchedulerDB):
-                del config["DB"]
-            else:
-                db_name = "diy"
-            func = getattr(SchedulerDB, db_name)
-            return func(**config)
-        elif tag == 2:
-            db_name = config["DB"]
-            if db_name in dir(StorerDB):
-                del config["DB"]
-            else:
-                db_name = "diy"
-            func = getattr(StorerDB, db_name)
-            return func(**config)
-        raise ValueError("tag must be in [0, 1, 2]")
-    elif any(isinstance(config, t) for t in (StorerInfo, SchedulerInfo, RedisInfo)):
-        return config
-    raise TypeError("config must be in [StorerInfo, SchedulerInfo, RedisInfo]")
-
-
-def info(configs, tag = 0):
-    if configs is None:
-        return SchedulerDB.default() if tag == 1 else None
-
-    if isinstance(configs, str):
-        configs = json.loads(configs)
-
-    if tag == 0:
-        return deal(configs, tag)
-
-    if not isinstance(configs, list):
-        configs = [configs]
-
-    return [deal(config, tag) for config in configs]

cobweb/base/decorators.py

@@ -1,95 +0,0 @@
-import time
-from functools import wraps
-
-# from config import DBType
-from log import log
-
-
-# def find_func_name(func_name, name_list):
-#     for name in name_list:
-#         if func_name.find(name) == 0:
-#             return True
-#     return False
-
-
-# def starter_decorator(execute):
-#     @wraps(execute)
-#     def wrapper(starter, *args, **kwargs):
-#         spider_dynamic_funcs = []
-#         scheduler_dynamic_funcs = []
-#         starter_functions = inspect.getmembers(starter, lambda a: inspect.isfunction(a))
-#         for starter_function in starter_functions:
-#             if find_func_name(starter_function[0], starter.scheduler_funcs):
-#                 scheduler_dynamic_funcs.append(starter_function)
-#             elif find_func_name(starter_function[0], starter.spider_funcs):
-#                 spider_dynamic_funcs.append(starter_function)
-#         return execute(starter, scheduler_dynamic_funcs, spider_dynamic_funcs, *args, **kwargs)
-#
-#     return wrapper
-
-
-# def scheduler_decorator(execute):
-#     @wraps(execute)
-#     def wrapper(scheduler, distribute_task):
-#         if not issubclass(scheduler, SchedulerInterface):
-#             scheduler.stop = True
-#         elif getattr(scheduler, "scheduler", None):
-#             execute(scheduler, distribute_task)
-#         else:
-#             log.error(f"scheduler type: {scheduler.db_type} not have add function!")
-#             scheduler.stop = True
-#     return wrapper
-
-
-def storer_decorator(execute):
-    @wraps(execute)
-    def wrapper(storer, stop_event, last_event, table_name, callback):
-        if getattr(storer, "save", None):
-            execute(storer, stop_event, last_event, table_name, callback)
-        else:
-            log.error(f"storer base_type: {storer.data_type} not have add function!")
-            storer.stop = True
-    return wrapper
-
-
-def distribute_scheduler_decorators(func):
-    @wraps(func)
-    def wrapper(distributor, callback):
-        try:
-            func(distributor, callback)
-        except TypeError:
-            pass
-        distributor.event.set()
-    return wrapper
-
-
-def distribute_spider_decorators(func):
-    @wraps(func)
-    def wrapper(distributor, stop_event, db, callback):
-        while not stop_event.is_set():
-            try:
-                seed = distributor.queue_client.pop("_seed_queue")
-                if not seed:
-                    time.sleep(3)
-                    continue
-                distributor.spider_in_progress.append(1)
-                func(distributor, db, seed, callback)
-            except Exception as e:
-                print(e)
-            finally:
-                distributor.spider_in_progress.pop()
-
-    return wrapper
-
-
-def distribute_storer_decorators(func):
-    @wraps(func)
-    def wrapper(distributor, callback, data_type, table_name, last):
-        data_list = []
-        try:
-            func(distributor, callback, data_list, data_type, table_name, last)
-        except Exception as e:
-            log.info("storage exception! " + str(e))
-            # distributor._task_queue.extendleft(data_list)
-
-    return wrapper

cobweb/base/hash_table.py

@@ -1,60 +0,0 @@
-# -*- coding: utf-8 -*-
-
-class DynamicHashTable:
-    def __init__(self):
-        self.capacity = 1000  # 初始容量
-        self.size = 0  # 元素个数
-        self.table = [None] * self.capacity
-
-    def hash_function(self, key):
-        return hash(key) % self.capacity
-
-    def probe(self, index):
-        # 线性探测法
-        return (index + 1) % self.capacity
-
-    def insert(self, key, value):
-        index = self.hash_function(key)
-        while self.table[index] is not None:
-            if self.table[index][0] == key:
-                self.table[index][1] = value
-                return
-            index = self.probe(index)
-        self.table[index] = [key, value]
-        self.size += 1
-
-        # 动态扩容
-        if self.size / self.capacity >= 0.7:
-            self.resize()
-
-    def get(self, key):
-        index = self.hash_function(key)
-        while self.table[index] is not None:
-            if self.table[index][0] == key:
-                return self.table[index][1]
-            index = self.probe(index)
-        raise KeyError("Key not found")
-
-    def remove(self, key):
-        index = self.hash_function(key)
-        while self.table[index] is not None:
-            if self.table[index][0] == key:
-                self.table[index] = None
-                self.size -= 1
-                return
-            index = self.probe(index)
-        raise KeyError("Key not found")
-
-    def resize(self):
-        # 扩容为原容量的两倍
-        self.capacity *= 2
-        new_table = [None] * self.capacity
-        for item in self.table:
-            if item is not None:
-                key, value = item
-                new_index = self.hash_function(key)
-                while new_table[new_index] is not None:
-                    new_index = self.probe(new_index)
-                new_table[new_index] = [key, value]
-        self.table = new_table
-

cobweb/base/queue_tmp.py

@@ -1,60 +0,0 @@
-from typing import Iterable
-
-# from pympler import asizeof
-from collections import deque
-
-
-class Queue:
-
-    def __init__(self):
-        self._seed_queue = deque()
-
-    @property
-    def queue_names(self):
-        return tuple(self.__dict__.keys())
-
-    @property
-    def used_memory(self):
-        return asizeof.asizeof(self)
-
-    def create_queue(self, queue_name: str):
-        self.__setattr__(queue_name, deque())
-
-    def push_seed(self, seed):
-        self.push("_seed_queue", seed)
-
-    def pop_seed(self):
-        return self.pop("_seed_queue")
-
-    def push(self, queue_name: str, data, left: bool = False):
-        try:
-            if not data:
-                return None
-            queue = self.__getattribute__(queue_name)
-            if isinstance(data, Iterable):
-                queue.extend(data) if left else queue.extendleft(data)
-            else:
-                queue.appendleft(data) if left else queue.append(data)
-        except AttributeError as e:
-            print(e)
-
-    def pop(self, queue_name: str, left: bool = True):
-        try:
-            queue = self.__getattribute__(queue_name)
-            return queue.pop() if left else queue.popleft()
-        except IndexError as e:
-            print(e)
-            return None
-        except AttributeError as e:
-            print(e)
-            return None
-
-
-# qqueue = Queue()
-# # qqueue.create_queue("test")
-# print(qqueue.queue_names)
-# qqueue.push("task_queue", "key")
-# print(qqueue.used_memory)
-# c = qqueue.pop("task_queue")
-# print(c)
-

cobweb/base/request.py

@@ -1,62 +0,0 @@
-import random
-from typing import Union
-
-import requests
-
-
-class Request:
-
-    def __init__(self):
-        pass
-
-
-def gen_user_agent(platform: str = 'android', redis_client=None):
-    user_agent = ''
-    if platform == 'android':
-        os_version = f'{random.randint(4, 10)}.{random.randint(0, 9)}.{random.randint(0, 9)}'
-        model = (redis_client and redis_client.srandmember('(md)set_android_model').decode()) or ''
-        webkit_version = f'{random.randint(450, 550)}.{random.randint(0, 100)}.{random.randint(0, 100)}'
-        version = f'{random.randint(3, 6)}.{random.randint(0, 9)}.{random.randint(0, 9)}'
-        chrome_version = f'{random.randint(50, 88)}.{random.randint(0, 9)}.{random.randint(1000, 5000)}.{random.randint(0, 1000)}'
-        user_agent = f'Mozilla/5.0 (Linux; U; Android {os_version}; zh-cn; {model} Build/{model}) AppleWebKit/{webkit_version} (KHTML, like Gecko) Version/{version} Chrome/{chrome_version} Mobile Safari/{webkit_version}'
-    elif platform == 'iphone':
-        os_version = f'{random.randint(5, 13)}_{random.randint(0, 9)}_{random.randint(0, 9)}'
-        webkit_version = f'{random.randint(550, 650)}.{random.randint(0, 100)}.{random.randint(0, 100)}'
-        version = f'{random.randint(4, 13)}.{random.randint(0, 9)}.{random.randint(0, 9)}'
-        user_agent = f'Mozilla/5.0 (iPhone; CPU iPhone OS {os_version} like Mac OS X) AppleWebKit/{webkit_version} (KHTML, like Gecko) Version/{version} Mobile Safari/{webkit_version}'
-
-    return user_agent
-
-
-def config(
-        url,
-        method: str = "GET",
-        headers: dict = None,
-        proxies: dict = None,
-        cookies: dict = None,
-        params: dict = None,
-        timeout: int = None,
-        stream: bool = False,
-        data: Union[dict, str, tuple] = None,
-) -> dict:
-    if not headers:
-        headers = {"accept": "*/*", "user-agent": gen_user_agent()}
-
-    elif "user-agent" not in [key.lower() for key in headers.keys()]:
-        headers["user-agent"] = gen_user_agent()
-
-    return {
-        "method": method,
-        "url": url,
-        "data": data,
-        "params": params,
-        "cookies": cookies,
-        "headers": headers,
-        "proxies": proxies,
-        "stream": stream,
-        "timeout": timeout or 3,
-    }
-
-
-def request(**kwargs):
-    return requests.request(**kwargs)

cobweb/base/task.py

@@ -1,38 +0,0 @@
-from config import info
-
-
-class Task:
-
-    def __init__(
-            self,
-            project=None,
-            task_name=None,
-            start_seed=None,
-            spider_num=None,
-            # queue_length=None,
-            max_retries=None,
-            scheduler_info=None,
-            storer_info=None,
-            redis_info=None
-    ):
-        """
-
-        :param project:
-        :param task_name:
-        :param start_seed:
-        :param spider_num:
-        # :param queue_length:
-        :param scheduler_info:
-        :param storer_info: Union(list, DataInfo/namedtuple), 单个元素构成必须有3个值(数据库类型，表名，字段名)
-        """
-        self.project = project or "test"
-        self.task_name = task_name or "spider"
-        self.start_seed = start_seed
-        self.spider_num = spider_num or 1
-        self.max_retries = max_retries or 5
-        # self.redis_info = RedisInfo(**(redis_info or dict()))
-        self.redis_info = info(redis_info, tag=0)
-        # self.scheduler_info = SchedulerDB.info(scheduler_info)
-        self.scheduler_info = info(scheduler_info, tag=1)
-        # self.storer_info = StorerDB.info(storer_info)
-        self.storer_info = info(storer_info, tag=2)

cobweb/base/utils.py

@@ -1,15 +0,0 @@
-import sys
-
-
-def struct_table_name(table_name):
-    return table_name.replace(".", "__p__").replace(":", "__c__")
-
-
-def restore_table_name(table_name):
-    return table_name.replace("__p__", ".").replace("__c__", ":")
-
-
-def struct_queue_name(db_name, table_name):
-    return sys.intern(f"__{db_name}_{table_name}_queue__")
-
-

cobweb/db/base/client_db.py

@@ -1 +0,0 @@
-