cobweb-launcher 0.1.7__py3-none-any.whl → 1.2.41__py3-none-any.whl

cobweb/interface.py

@@ -1,34 +0,0 @@
-from abc import ABC, abstractmethod
-from .utils import parse_info
-
-
-class SchedulerInterface(ABC):
-
-    def __init__(self, table, sql, length, size, queue, config=None):
-        self.sql = sql
-        self.table = table
-        self.length = length
-        self.size = size
-        self.queue = queue
-        self.config = parse_info(config)
-        self.stop = False
-
-    @abstractmethod
-    def schedule(self, *args, **kwargs):
-        pass
-
-
-class StorerInterface(ABC):
-
-    def __init__(self, table, fields, length, queue, config=None):
-        self.table = table
-        self.fields = fields
-        self.length = length
-        self.queue = queue
-        self.config = parse_info(config)
-        # self.redis_db = redis_db
-
-    @abstractmethod
-    def store(self, *args, **kwargs):
-        pass
-

cobweb/single/launcher.py

@@ -1,231 +0,0 @@
-import time
-import threading
-from threading import Thread
-
-from .models import Scheduler, Spider, Storer
-from cobweb import log, Queue, DBItem, RedisDB
-from cobweb.setting import MODEL, RESET_SCORE, CHECK_LOCK_TIME
-from cobweb.utils import (
-    struct_queue_name as sqn,
-    restore_table_name as rtn,
-    parse_import_model as pim,
-)
-
-
-def check(stop, last, spider, scheduler, storer, ready_seed_length, spider_queue_length):
-    log.info("run check thread after 30 seconds...")
-    time.sleep(30)
-    spider_info = """
-------------------- check: {0} ------------------
-            redis_spider_seed_length:  {1}
-            redis_ready_seed_length:   {2}
-            running_spider_thread_num: {3}
-            memory_seed_queue_length:  {4}
-            storer_queue_length_info:  {5}
------------------------  end  -----------------------"""
-    while True:
-        status = "running"
-        running_spider_thread_num = spider.spider_in_progress.length
-        redis_ready_seed_length = ready_seed_length()
-        redis_spider_seed_length = spider_queue_length()
-        memory_seed_queue_length = scheduler.queue.length
-        storer_upload_queue_length = storer.queue.length
-        if (
-                scheduler.stop and
-                # not redis_ready_seed_length and
-                not memory_seed_queue_length and
-                not running_spider_thread_num
-        ):
-            if not MODEL:
-                log.info("spider is done?")
-            last.set()
-            time.sleep(3)
-            storer_queue_empty = True
-            if storer.queue.length:
-                storer_queue_empty = False
-            storer_upload_queue_length = storer.queue.length
-            if (
-                    storer_queue_empty and
-                    not redis_ready_seed_length and
-                    not redis_spider_seed_length
-            ):
-                if MODEL:
-                    log.info("waiting for push seeds...")
-                    status = "waiting"
-                    time.sleep(30)
-                else:
-                    log.info("spider done!")
-                    break
-
-            last.clear()
-
-        log.info(spider_info.format(
-            status,
-            redis_spider_seed_length,
-            redis_ready_seed_length,
-            running_spider_thread_num,
-            memory_seed_queue_length,
-            storer_upload_queue_length
-        ))
-
-        time.sleep(3)
-    stop.set()
-
-
-def launcher(task):
-    """
-    任务启动装饰器
-    :param task: 任务配置信息
-    """
-    def decorator(func):
-        """
-        Item:
-            Textfile()
-            Loghub()
-            Console()
-        e.g.
-        task.fields = "a,b"
-        func(item, seed)
-            a = "a"
-            b = "b"
-            data = {"a": "a", "b": "b"}
-            yield item.Loghub(**data)
-            yield item.Loghub(a=a, b=b)
-        """
-        storer_list = []
-
-        # 程序结束事件
-        last = threading.Event()
-        # 停止采集事件
-        stop = threading.Event()
-
-        # 初始化redis信息
-        redis_db = RedisDB(
-            task.project, task.task_name, task.redis_info,
-            model=MODEL, cs_lct=CHECK_LOCK_TIME, rs_time=RESET_SCORE
-        )
-
-        # new item
-        item = type("Item", (object,), {"redis_client": redis_db.client})()
-
-        log.info("初始化cobweb!")
-
-        seed_queue = Queue()
-
-        scheduler_info = task.scheduler_info or dict()
-
-        # 调度器动态继承
-        sql = scheduler_info.get("sql")
-        table = scheduler_info.get("table")
-        size = scheduler_info.get("size")
-        scheduler_config = scheduler_info.get("config")
-        scheduler_db = scheduler_info.get("db", "default")
-        DB, class_name = pim(scheduler_db, "scheduler")
-        # SchedulerDB, table, sql, length, size, config = task.scheduler_info
-        SchedulerTmp = type(class_name, (Scheduler, DB), {})
-
-        # 初始化调度器
-        scheduler = SchedulerTmp(
-            table=table, sql=sql, size=size, queue=seed_queue,
-            length=task.scheduler_queue_length, config=scheduler_config
-        )
-
-        # 初始化采集器
-        spider = Spider(seed_queue, task.max_retries)
-
-        storer = None
-
-        # 解析存储器信息
-        storer_info = task.storer_info or dict()
-
-        # for storer_info in storer_info_list:
-        if storer_info:
-            storer_db = storer_info["db"]
-            fields = storer_info["fields"]
-            storer_table = storer_info.get("table", "console")
-            storer_config = storer_info.get("config")
-
-            StorerDB, class_name = pim(storer_db, "storer")
-            StorerTmp = type(class_name, (Storer, StorerDB), {})
-
-            db_name = class_name.lower()
-            if not getattr(item, db_name, None):
-                instance = type(db_name, (DBItem,), {})
-                setattr(item, db_name, instance)
-
-            storer_item_instance = getattr(item, db_name)
-            storer_item_instance.init_item(storer_table, fields)
-
-            storer_queue = sqn(db_name, storer_table)
-            queue = getattr(storer_item_instance, storer_queue)
-            # 初始话存储器
-            table_name = rtn(table_name=storer_table)
-            storer = StorerTmp(
-                table=table_name, fields=fields,
-                length=task.storer_queue_length,
-                queue=queue, config=storer_config
-            )
-
-        Thread(target=redis_db.check_spider_queue, args=(stop, len(storer_list))).start()
-        Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
-
-        # 推送初始种子
-        # seeds = start_seeds(task.start_seed)
-        redis_db.add_seed(task.seeds)
-        # 启动调度器, 调度至redis队列
-        Thread(
-            # name="xxxx_schedule_seeds",
-            target=scheduler.schedule_seed,
-            args=(
-                redis_db.ready_seed_length,
-                redis_db.get_scheduler_lock,
-                redis_db.add_seed
-            )
-        ).start()
-
-        # 启动调度器, 调度任务队列
-        Thread(
-            # name="xxxx_schedule_task",
-            target=scheduler.schedule_task,
-            args=(
-                stop, redis_db.get_seed,
-                redis_db.ready_seed_length
-            )
-        ).start()
-
-        # 启动采集器
-        for index in range(task.spider_num):
-            Thread(
-                # name=f"xxxx_spider_task:{index}",
-                target=spider.spider_task,
-                args=(
-                    stop, func, item,
-                    redis_db.del_seed
-                )
-            ).start()
-
-        # 启动存储器
-        if storer:
-            Thread(
-                # name=f"xxxx_store_task:{storer.table}",
-                target=storer.store_task,
-                args=(
-                    stop, last,
-                    redis_db.reset_seed,
-                    redis_db.del_seed
-                )
-            ).start()
-
-        Thread(
-            # name="check_spider",
-            target=check,
-            args=(
-                stop, last, spider,
-                scheduler, storer,
-                redis_db.ready_seed_length,
-                redis_db.spider_queue_length,
-            )
-        ).start()
-
-    return decorator
-

cobweb/single/models.py

@@ -1,134 +0,0 @@
-import time
-from cobweb import log, Queue, Seed
-from cobweb.utils import issubclass_cobweb_inf
-# from pympler import asizeof
-
-
-class Scheduler:
-
-    def schedule_seed(self, ready_seed_length, get_scheduler_lock, add_seed):
-
-        inf_name = "SchedulerInterface"
-        if not issubclass_cobweb_inf(self.__class__, inf_name):
-            raise Exception("not have schedule function!")
-
-        if self.__class__.__name__ == "Default":
-            self.stop = True
-            return None
-
-        while not self.stop:
-            length = ready_seed_length()
-            if length > self.size:
-                time.sleep(15)
-
-            elif get_scheduler_lock():
-                seeds = self.schedule()
-                add_seed(seeds)
-
-        log.info(f"close thread: schedule_seed")
-
-    def schedule_task(self, stop, get_seed, ready_seed_length):
-        time.sleep(3)
-        while not stop.is_set():
-
-            if not ready_seed_length():
-                time.sleep(15)
-                continue
-
-            if self.queue.length >= self.length:
-                time.sleep(3)
-                continue
-
-            seeds = get_seed(self.length)
-            self.queue.push(seeds)
-        log.info(f"close thread: schedule_task")
-
-
-class Spider:
-
-    def __init__(self, queue, max_retries=5):
-        self.spider_in_progress = Queue()
-        self.max_retries = max_retries
-        self.queue = queue
-
-    def spider_task(self, stop, func, item, del_seed):
-        while not stop.is_set():
-            seed = self.queue.pop()
-            if not seed:
-                time.sleep(3)
-                continue
-            elif seed._retry >= self.max_retries:
-                del_seed(seed, spider_status=False)
-                continue
-            try:
-                self.spider_in_progress.push(1, direct_insertion=True)
-                # log.info("spider seed: " + str(seed))
-                ret_count = 0
-                status = None
-                for it in func(item, seed):
-                    ret_count += 1
-                    if getattr(it, "table_name", None):
-                        store_queue = it.queue()
-                        store_queue.push(
-                            [seed, it.struct_data],
-                            direct_insertion=True
-                        )
-                    elif isinstance(it, Seed):
-                        self.queue.push(it)
-                    elif any(isinstance(it, t) for t in (list, tuple)):
-                        self.queue.push([s if isinstance(s, Seed) else Seed(s) for s in it])
-                    elif isinstance(it, bool):
-                        status = it
-
-                if status:
-                    del_seed(seed, spider_status=True)
-                elif not ret_count or status is False:
-                    seed._retry += 1
-                    self.queue.push(seed)
-
-            except Exception as e:
-                seed._retry += 1
-                self.queue.push(seed)
-                log.info(f"{str(seed)} -> {str(e)}")
-            finally:
-                self.spider_in_progress.pop()
-        log.info(f"close thread: spider")
-
-
-class Storer:
-
-    def store_task(self, stop, last, reset_seed, del_seed):
-
-        inf_name = "StorerInterface"
-        if not issubclass_cobweb_inf(self.__class__, inf_name):
-            return None
-
-        if not getattr(self, "store", None):
-            raise Exception("not have store function!")
-
-        storer_name = self.__class__.__name__ + self.table
-
-        while not stop.is_set():
-
-            if last.is_set() or self.queue.length >= self.length:
-                seeds, data_list = [], []
-
-                for _ in range(self.length):
-                    items = self.queue.pop()
-                    if not items:
-                        break
-                    seed, data = items
-                    seeds.append(seed)
-                    data_list.append(data)
-
-                if data_list:
-                    if self.store(data_list):
-                        del_seed(seeds)
-                    else:
-                        reset_seed(seeds)
-                        log.info("reset seeds!")
-                    continue
-
-            time.sleep(3)
-
-        log.info(f"close thread: {storer_name}")

cobweb/single/nest.py

@@ -1,153 +0,0 @@
-import time
-import threading
-
-from single.nest import Seed, DBItem
-from single.nest import struct_queue_name, restore_table_name
-from single.nest import Distributor, Scheduler, Spider, Storer
-
-
-def init_task_seed(seeds):
-    if not seeds:
-        return None
-    if isinstance(seeds, list) or isinstance(seeds, tuple):
-        for seed in seeds:
-            yield Seed(seed)
-    elif isinstance(seeds, str) or isinstance(seeds, dict):
-        yield Seed(seeds)
-
-
-def parse_storer_info(storer_info):
-    storer_data = {}
-    storer_info_list = []
-    if storer_info.__class__.__name__ == 'StorerInfo':
-        storer_info_list.append(storer_info)
-    elif isinstance(storer_info, tuple) or isinstance(storer_info, list):
-        storer_info_list = storer_info
-    for info in storer_info_list:
-        db_name = info.DB.__name__
-        storer_data.setdefault(db_name, {"StorerDB": info.DB, "db_args_list": []})
-        storer_data[db_name]["db_args_list"].append(info[1:])
-    return storer_data
-
-
-def check(stop_event, last_event, distributor, scheduler, spider, storer_list):
-    while True:
-        time.sleep(3)
-        if (
-                scheduler.stop and
-                not distributor.seed_queue.length and
-                not spider.spider_in_progress.length
-        ):
-            last_event.set()
-            time.sleep(10)
-            storer_queue_empty = True
-            for storer in storer_list:
-                if storer.queue.length:
-                    storer_queue_empty = False
-                    break
-            if storer_queue_empty:
-                break
-        last_event.clear()
-    stop_event.set()
-
-
-def cobweb(task):
-    """
-    任务启动装饰器
-    :param task: 任务配置信息
-    """
-    def decorator(func):
-        """
-        func(Item, seed)
-            Item:
-                Item.Textfile()
-                Item.Console()
-        """
-        # project task_name start_seed spider_num queue_length scheduler_info storer_info
-
-        storer_list = []
-
-        # 程序结束事件
-        last_event = threading.Event()
-        # 暂停采集事件
-        stop_event = threading.Event()
-
-        # 创建分发器
-        distributor = Distributor()
-
-        # 调度器动态继承
-        SchedulerDB, table, sql, length, size = task.SchedulerInfo
-        SchedulerTmp = type('Scheduler', (Scheduler, SchedulerDB), {})
-
-        # 初始化调度器
-        scheduler = SchedulerTmp(table=table, sql=sql, length=length, size=size, queue=distributor.seed_queue)
-
-        # 初始化采集器
-        spider = Spider(queue=distributor.seed_queue)
-
-        # 解析存储器信息
-        storer_data = parse_storer_info(task.storer_info)
-
-        # sds
-        item = type("item", (object,), {})
-        for db_name in storer_data.keys():
-            # 存储器动态继承
-            StorerDB = storer_data[db_name]["StorerDB"]
-            StorerTmp = type('Storer', (Storer, StorerDB), {})
-            db_args_list = storer_data[db_name]["db_args_list"]
-            for storer_db_args in db_args_list:
-                table, fields, length = storer_db_args
-                if not getattr(item, db_name, None):
-                    instance = type(db_name, (DBItem,), {})
-                    setattr(item, db_name, instance)
-                # 创建存储xxx
-                getattr(item, db_name).init_item(table, fields)
-                # 创建存储队列
-                storer_queue = struct_queue_name(db_name, table)
-                distributor.create_queue(queue_name=storer_queue)
-                queue = distributor.get_queue(queue_name=storer_queue)
-                # 初始话存储器
-                table_name = restore_table_name(table_name=table)
-                storer = StorerTmp(table=table_name, fields=fields, length=length, queue=queue)
-                storer_list.append(storer)
-
-        # 推送初始种子
-        distributor.distribute(init_task_seed, seeds=task.start_seed)
-
-        # 启动调度器
-        threading.Thread(
-            target=scheduler.schedule_task,
-            args=(distributor.distribute,),
-            name="single_scheduler_task"
-        ).start()
-
-        # 启动采集器
-        for index in range(task.spider_num):
-            threading.Thread(
-                target=spider.spider_task,
-                args=(stop_event, distributor.distribute, func, item),
-                name=f"single_spider_task:{index}"
-            ).start()
-
-        # 启动存储器
-        for storer in storer_list:
-            threading.Thread(
-                target=storer.store_task,
-                args=(stop_event, last_event, distributor.distribute),
-                name=f"single_store_task:{storer.table}",
-            ).start()
-
-        threading.Thread(
-            target=check, name="check",
-            args=(
-                stop_event, last_event, distributor,
-                scheduler, spider, storer_list
-            )
-        ).start()
-
-        # return starter(task, func)
-    return decorator
-
-
-
-

cobweb/task.py

@@ -1,50 +0,0 @@
-from .utils import parse_info, struct_start_seeds
-
-
-class Task:
-
-    def __init__(
-            self,
-            # model=None,
-            seeds=None,
-            project=None,
-            task_name=None,
-            oss_config=None,
-            redis_info=None,
-            storer_info=None,
-            scheduler_info=None,
-            spider_num=None,
-            max_retries=None,
-            storer_queue_length=None,
-            scheduler_queue_length=None,
-    ):
-        """
-
-        :param seeds:
-        :param project:
-        :param task_name:
-        :param redis_info:
-        :param storer_info:
-        :param scheduler_info: dict(DB="", table="", size="", config="")
-        :param spider_num:
-        :param max_retries:
-        :param storer_queue_length:
-        :param scheduler_queue_length:
-        """
-        # self.model = model
-
-        self.seeds = struct_start_seeds(seeds)
-        self.project = project or "test"
-        self.task_name = task_name or "spider"
-
-        self.oss_config = oss_config
-
-        self.redis_info = parse_info(redis_info)
-        self.storer_info = parse_info(storer_info)
-        self.scheduler_info = parse_info(scheduler_info)
-
-        self.spider_num = spider_num or 1
-        self.max_retries = max_retries or 5
-        self.storer_queue_length = storer_queue_length or 100
-        self.scheduler_queue_length = scheduler_queue_length or 100
-

cobweb/utils.py

@@ -1,90 +0,0 @@
-import json
-import re
-import sys
-from abc import ABC
-from typing import Iterable
-from importlib import import_module
-
-
-def struct_table_name(table_name):
-    return table_name.replace(".", "__p__").replace(":", "__c__")
-
-
-def restore_table_name(table_name):
-    return table_name.replace("__p__", ".").replace("__c__", ":")
-
-
-def struct_queue_name(db_name, table_name):
-    return sys.intern(f"__{db_name}_{table_name}_queue__")
-
-
-def parse_info(info):
-    if not info:
-        return info
-
-    if isinstance(info, dict):
-        return info
-
-    if isinstance(info, str):
-        return json.loads(info)
-
-    if isinstance(info, Iterable):
-        result = list()
-        for ii in info:
-            if isinstance(ii, str):
-                result.append(json.loads(ii))
-            elif isinstance(ii, dict):
-                result.append(ii)
-            else:
-                raise TypeError("must be in [str, dict]")
-
-        return result
-
-
-def struct_start_seeds(seeds):
-    from .bbb import Seed
-    if not seeds:
-        return None
-    if any(isinstance(seeds, t) for t in (list, tuple)):
-        return [Seed(seed) for seed in seeds]
-    elif any(isinstance(seeds, t) for t in (str, dict)):
-        return Seed(seeds)
-
-
-def issubclass_cobweb_inf(_class, inf_name):
-    for _c in _class.__mro__[1:]:
-        if _c.__name__ == inf_name:
-            return True
-    return False
-
-
-def parse_import_model(model_info, model_type=None):
-    if model_type not in ["scheduler", "storer"]:
-        raise TypeError("model_type must be in scheduler, storer")
-    if isinstance(model_info, str):
-        if "import" in model_info:
-            model_path, class_name = re.search(
-                r"from (.*?) import (.*?)$", model_info
-            ).groups()
-            model = import_module(model_path)
-            class_object = getattr(model, class_name)
-        elif "." in model_info:
-            info_list = model_info.split(".")
-            class_name = info_list[-1]
-            model_path = ".".join(info_list[:-1])
-            model = import_module(model_path)
-            class_object = getattr(model, class_name)
-        else:
-            model_path = f"cobweb.db.{model_type}.{model_info.lower()}"
-            class_name = model_info.capitalize()
-            model = import_module(model_path)
-            class_object = getattr(model, class_name)
-        return class_object, class_name
-    elif issubclass(model_info, ABC):
-        inf_name = model_type.capitalize() + "Interface"
-        if issubclass_cobweb_inf(model_info, inf_name):
-            return model_info, model_info.__name__
-        raise ImportError()
-    raise TypeError()
-
-