cobweb-launcher 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

cobweb/single/nest.py

@@ -1,153 +0,0 @@
-import time
-import threading
-
-from equip.single import Seed, DBItem
-from equip.single import struct_queue_name, restore_table_name
-from equip.single import Distributor, Scheduler, Spider, Storer
-
-
-def init_task_seed(seeds):
-    if not seeds:
-        return None
-    if isinstance(seeds, list) or isinstance(seeds, tuple):
-        for seed in seeds:
-            yield Seed(seed)
-    elif isinstance(seeds, str) or isinstance(seeds, dict):
-        yield Seed(seeds)
-
-
-def parse_storer_info(storer_info):
-    storer_data = {}
-    storer_info_list = []
-    if storer_info.__class__.__name__ == 'StorerInfo':
-        storer_info_list.append(storer_info)
-    elif isinstance(storer_info, tuple) or isinstance(storer_info, list):
-        storer_info_list = storer_info
-    for info in storer_info_list:
-        db_name = info.DB.__name__
-        storer_data.setdefault(db_name, {"StorerDB": info.DB, "db_args_list": []})
-        storer_data[db_name]["db_args_list"].append(info[1:])
-    return storer_data
-
-
-def check(stop_event, last_event, distributor, scheduler, spider, storer_list):
-    while True:
-        time.sleep(3)
-        if (
-                scheduler.stop and
-                not distributor.seed_queue.length and
-                not spider.spider_in_progress.length
-        ):
-            last_event.set()
-            time.sleep(10)
-            storer_queue_empty = True
-            for storer in storer_list:
-                if storer.queue.length:
-                    storer_queue_empty = False
-                    break
-            if storer_queue_empty:
-                break
-        last_event.clear()
-    stop_event.set()
-
-
-def cobweb(task):
-    """
-    任务启动装饰器
-    :param task: 任务配置信息
-    """
-    def decorator(func):
-        """
-        func(Item, seed)
-            Item:
-                Item.Textfile()
-                Item.Console()
-        """
-        # project task_name start_seed spider_num queue_length scheduler_info storer_info
-
-        storer_list = []
-
-        # 程序结束事件
-        last_event = threading.Event()
-        # 暂停采集事件
-        stop_event = threading.Event()
-
-        # 创建分发器
-        distributor = Distributor()
-
-        # 调度器动态继承
-        SchedulerDB, table, sql, length, size = task.SchedulerInfo
-        SchedulerTmp = type('Scheduler', (Scheduler, SchedulerDB), {})
-
-        # 初始化调度器
-        scheduler = SchedulerTmp(table=table, sql=sql, length=length, size=size, queue=distributor.seed_queue)
-
-        # 初始化采集器
-        spider = Spider(queue=distributor.seed_queue)
-
-        # 解析存储器信息
-        storer_data = parse_storer_info(task.storer_info)
-
-        # sds
-        item = type("item", (object,), {})
-        for db_name in storer_data.keys():
-            # 存储器动态继承
-            StorerDB = storer_data[db_name]["StorerDB"]
-            StorerTmp = type('Storer', (Storer, StorerDB), {})
-            db_args_list = storer_data[db_name]["db_args_list"]
-            for storer_db_args in db_args_list:
-                table, fields, length = storer_db_args
-                if not getattr(item, db_name, None):
-                    instance = type(db_name, (DBItem,), {})
-                    setattr(item, db_name, instance)
-                # 创建存储xxx
-                getattr(item, db_name).init_item(table, fields)
-                # 创建存储队列
-                storer_queue = struct_queue_name(db_name, table)
-                distributor.create_queue(queue_name=storer_queue)
-                queue = distributor.get_queue(queue_name=storer_queue)
-                # 初始话存储器
-                table_name = restore_table_name(table_name=table)
-                storer = StorerTmp(table=table_name, fields=fields, length=length, queue=queue)
-                storer_list.append(storer)
-
-        # 推送初始种子
-        distributor.distribute(init_task_seed, seeds=task.start_seed)
-
-        # 启动调度器
-        threading.Thread(
-            target=scheduler.schedule_task,
-            args=(distributor.distribute,),
-            name="single_scheduler_task"
-        ).start()
-
-        # 启动采集器
-        for index in range(task.spider_num):
-            threading.Thread(
-                target=spider.spider_task,
-                args=(stop_event, distributor.distribute, func, item),
-                name=f"single_spider_task:{index}"
-            ).start()
-
-        # 启动存储器
-        for storer in storer_list:
-            threading.Thread(
-                target=storer.store_task,
-                args=(stop_event, last_event, distributor.distribute),
-                name=f"single_store_task:{storer.table}",
-            ).start()
-
-        threading.Thread(
-            target=check, name="check",
-            args=(
-                stop_event, last_event, distributor,
-                scheduler, spider, storer_list
-            )
-        ).start()
-
-        # return starter(task, func)
-    return decorator
-
-
-
-

cobweb/task.py

@@ -1,61 +0,0 @@
-import os
-from .constant import *
-from .utils import parse_info, struct_start_seeds
-
-
-def init_task_env():
-    Setting.RESET_SCORE = int(os.getenv("RESET_SCORE", 600))
-    Setting.CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
-    Setting.DEAL_MODEL = os.getenv("DEAL_MODEL", DealModel.failure)
-    Setting.LAUNCHER_MODEL = os.getenv("LAUNCHER_MODEL", LauncherModel.task)
-    Setting.SCHEDULER_WAIT_TIME = float(os.getenv("SCHEDULER_WAIT_TIME", 5))
-    Setting.SCHEDULER_BLOCK_TIME = float(os.getenv("SCHEDULER_BLOCK_TIME", 3))
-    Setting.SPIDER_WAIT_TIME = float(os.getenv("SPIDER_WAIT_TIME", 3))
-    Setting.SPIDER_SLEEP_TIME = float(os.getenv("SPIDER_SLEEP_TIME", 0.5))
-
-
-class Task:
-
-    def __init__(
-            self,
-            seeds=None,
-            project=None,
-            task_name=None,
-            oss_config=None,
-            redis_info=None,
-            storer_info=None,
-            scheduler_info=None,
-            spider_num=None,
-            max_retries=None,
-            storer_queue_length=None,
-            scheduler_queue_length=None,
-    ):
-        """
-
-        :param seeds:
-        :param project:
-        :param task_name:
-        :param redis_info:
-        :param storer_info:
-        :param scheduler_info: dict(DB="", table="", size="", config="")
-        :param spider_num:
-        :param max_retries:
-        :param storer_queue_length:
-        :param scheduler_queue_length:
-        """
-        init_task_env()
-        self.seeds = struct_start_seeds(seeds)
-        self.project = project or "test"
-        self.task_name = task_name or "spider"
-
-        self.oss_config = oss_config
-
-        self.redis_info = parse_info(redis_info)
-        self.storer_info = parse_info(storer_info)
-        self.scheduler_info = parse_info(scheduler_info)
-
-        self.spider_num = spider_num or 1
-        self.max_retries = max_retries or 5
-        self.storer_queue_length = storer_queue_length or 100
-        self.scheduler_queue_length = scheduler_queue_length or 100
-

cobweb/utils.py

@@ -1,90 +0,0 @@
-import json
-import re
-import sys
-from abc import ABC
-from typing import Iterable
-from importlib import import_module
-
-
-def struct_table_name(table_name):
-    return table_name.replace(".", "__p__").replace(":", "__c__")
-
-
-def restore_table_name(table_name):
-    return table_name.replace("__p__", ".").replace("__c__", ":")
-
-
-def struct_queue_name(db_name, table_name):
-    return sys.intern(f"__{db_name}_{table_name}_queue__")
-
-
-def parse_info(info):
-    if not info:
-        return info
-
-    if isinstance(info, dict):
-        return info
-
-    if isinstance(info, str):
-        return json.loads(info)
-
-    if isinstance(info, Iterable):
-        result = list()
-        for ii in info:
-            if isinstance(ii, str):
-                result.append(json.loads(ii))
-            elif isinstance(ii, dict):
-                result.append(ii)
-            else:
-                raise TypeError("must be in [str, dict]")
-
-        return result
-
-
-def struct_start_seeds(seeds):
-    from .bbb import Seed
-    if not seeds:
-        return None
-    if any(isinstance(seeds, t) for t in (list, tuple)):
-        return [Seed(seed) for seed in seeds]
-    elif any(isinstance(seeds, t) for t in (str, dict)):
-        return Seed(seeds)
-
-
-def issubclass_cobweb_inf(_class, inf_name):
-    for _c in _class.__mro__[1:]:
-        if _c.__name__ == inf_name:
-            return True
-    return False
-
-
-def parse_import_model(model_info, model_type=None):
-    if model_type not in ["scheduler", "storer"]:
-        raise TypeError("model_type must be in scheduler, storer")
-    if isinstance(model_info, str):
-        if "import" in model_info:
-            model_path, class_name = re.search(
-                r"from (.*?) import (.*?)$", model_info
-            ).groups()
-            model = import_module(model_path)
-            class_object = getattr(model, class_name)
-        elif "." in model_info:
-            info_list = model_info.split(".")
-            class_name = info_list[-1]
-            model_path = ".".join(info_list[:-1])
-            model = import_module(model_path)
-            class_object = getattr(model, class_name)
-        else:
-            model_path = f"cobweb.db.{model_type}.{model_info.lower()}"
-            class_name = model_info.capitalize()
-            model = import_module(model_path)
-            class_object = getattr(model, class_name)
-        return class_object, class_name
-    elif issubclass(model_info, ABC):
-        inf_name = model_type.capitalize() + "Interface"
-        if issubclass_cobweb_inf(model_info, inf_name):
-            return model_info, model_info.__name__
-        raise ImportError()
-    raise TypeError()
-
-

cobweb_launcher-1.0.0.dist-info/RECORD

@@ -1,69 +0,0 @@
-cobweb/__init__.py,sha256=BJu97N8hZsP9aBkMFMhbLFMuFv7JOJvGKnE9NIvcFJo,44
-cobweb/bbb.py,sha256=zKeCeBVFQfhEv6M8CCktUTM7tXDZmAu6ZN0-ET44pUY,5707
-cobweb/config.py,sha256=iWrep4vW9UyUi3hvgUWb4RL2IpEwpo_ttY0EUDKWN4g,5362
-cobweb/constant.py,sha256=Aw2ES_nohVRLTWylZp6WMiUAlgyw4kLbae7LpwdZ5y4,1867
-cobweb/decorators.py,sha256=eYQI9rddPVJihAlomLTmbtQhIOzPw8dCrOFpxAq2pLY,318
-cobweb/interface.py,sha256=um_k2AAQl1HTOvfUlq914DjkpfZVwt2m1B65EpPKrmE,802
-cobweb/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
-cobweb/new.py,sha256=-ZHoLZE4ZGRMBU5YWmWHMfLn2ipuTKEaCTO1pU2fX5o,303
-cobweb/setting.py,sha256=T693DAwLFLs9P6ZEvugP99UzXn-8PLeMEgdxRmL6cb4,1955
-cobweb/task.py,sha256=SyWC43C7hqQAqH-1HECXEEgp_6L6lwDhYe1kZNnHUvA,2006
-cobweb/utils.py,sha256=ivmRqJJNtwdOKYT4G7qQCWnL8ar9c-shxeDZzGB2E9c,2651
-cobweb/base/__init__.py,sha256=diiK5MygQaWjlWNLbW6eUIg-93O6glMGC9WLNM5jyOc,209
-cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
-cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
-cobweb/base/item.py,sha256=pMriHStzUXtSvIf5Z3KXsP-bCvjlG1gM3z33wWeuoH8,966
-cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
-cobweb/base/request.py,sha256=b08AtUSZjlFLEFIEw5uGS__FjU6QSldL20-UjZD0LbI,2128
-cobweb/base/response.py,sha256=7h9TwCNqRlwM_fvNmid9zOoRfHbKB8ABSU0eaVUJdVo,405
-cobweb/base/seed.py,sha256=XswH16eEd6iwIBpt71E2S_AsV5UVCcOEOBFoP0r5QRo,2900
-cobweb/crawlers/__init__.py,sha256=1sMhQ0-NJxiff3IqF2aMCXkSXcJFzzoCKIayQ5go4aI,71
-cobweb/crawlers/base_crawler.py,sha256=ZIdmlvL4f41yV7T77F9IhBBxRt1FH-LFm0BmIXAvP8I,4881
-cobweb/crawlers/file_crawler.py,sha256=VVOZ38qNAUPyNDspu3P7-zzDtrUtqefYLjOMnb_-JOw,9685
-cobweb/db/__init__.py,sha256=ut0iEyBLjcJL06WNG_5_d4hO5PJWvDrKWMkDOdmgh2M,30
-cobweb/db/oss_db.py,sha256=59HwMMfoGUr6l_BI4p84YHYoQwEn22cCM_1GZK34Uo4,4244
-cobweb/db/redis_db.py,sha256=MahFsAyub9OFG-xuZU6Qq0AfWV1lASFAI34g2FRGpq8,4235
-cobweb/db/scheduler/__init__.py,sha256=w5uIGEB1wLJ-H9RqGpzRwOEWW-BBVSk6Cc7FxZIlWCs,51
-cobweb/db/scheduler/default.py,sha256=XDtxNyu5KTpVAbfCOW8mR1zNFNHiMuaQ4sAhZuIYBoM,79
-cobweb/db/scheduler/textfile.py,sha256=P5pk75DUnbXbLNPOaMIbHh2lbwBGBlv0mitX58yK-MU,786
-cobweb/db/storer/__init__.py,sha256=yWUVyq8JLpuUDPnUC0igw3P8Kkw_FqNi0aAoxkMkRmc,49
-cobweb/db/storer/console.py,sha256=096JTALYuB_I3Qy5TjN40yEPeugO_pmqHN9VJu7wD7Y,153
-cobweb/db/storer/loghub.py,sha256=4ImSIpHPNU7Djp72HlUGOd2h5c9gIxGzBKL1jJ3KPkM,1702
-cobweb/db/storer/redis.py,sha256=7Q2XEQwBL6X_M1uvxzzuSBt6iw9piKw-_FWKm2INZDQ,412
-cobweb/db/storer/textfile.py,sha256=auoXGXLbIbEhMoeYIhy58qw22N2r0fQTtzVjHCjqVGA,386
-cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/distributed/launcher.py,sha256=jTtBXBmna_6yFdj6gyGQiiEtg8I0g5uI5h8kbHWt454,7998
-cobweb/distributed/models.py,sha256=PUQokXMGD-H4A99nX7qYA395Ul6IsWGruMTVa05nswY,4568
-cobweb/equip/__init__.py,sha256=LWhbrTnG9kD1et0D40EzLISPuE0PIS-5WD3y3CLDaWk,247
-cobweb/equip/dev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/equip/dev/launcher.py,sha256=KRsw7yxklvFM85cel-EyLsNPLyrC9Hd26BMSx6-4Hac,6785
-cobweb/equip/dev/models.py,sha256=w3LQEhTrgqoYZn5v9TiEK2A68xuC7QH7suRP9OYnoOg,4813
-cobweb/equip/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/equip/distributed/launcher.py,sha256=1LzxibGXWR20XpXawakiRpEMaa9yfaj2rFSKnmEwjFc,7475
-cobweb/equip/distributed/models.py,sha256=qTGzxLdb2arsZSZK2HE4-MrqhraUhc2Ol5wBvlv_aWA,5008
-cobweb/equip/download/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/equip/download/launcher.py,sha256=lZt4WNar0_QQjUSDhaJnxvHDiuQReXeFxquVgJjI5T4,6824
-cobweb/equip/download/models.py,sha256=USwa48PUZ9_J8qwmEoGu4Oq7-oo-xWxL0k5j8q8PoDw,5305
-cobweb/equip/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/equip/single/launcher.py,sha256=lZt4WNar0_QQjUSDhaJnxvHDiuQReXeFxquVgJjI5T4,6824
-cobweb/equip/single/models.py,sha256=mfSmYBUkX-QuuyuajkIw6EI0NYhlbqrCM88YGP7dHUw,5590
-cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
-cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
-cobweb/launchers/__init__.py,sha256=qwlkEJVri7dvCgi45aX3lqAmQS0HrPicAipDvH75kew,69
-cobweb/launchers/launcher.py,sha256=vqRqaBEg04mX6F-EGuoAdeCA163sSZk45mNmmQ2LWUA,5654
-cobweb/launchers/launcher_pro.py,sha256=GlDpyP1XAY2bX5SuSBn3920D5OKNigQgLnJfu6QOmPw,6760
-cobweb/pipelines/__init__.py,sha256=xanY-Z1d7zRR5JhCdW2htzrAywnKBkigiaUlTFa6of0,80
-cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
-cobweb/pipelines/loghub_pipeline.py,sha256=roQ8gLunvuPc4KOMIATN1nKvjOXrc_RSyzXq8YY9ZBU,1015
-cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/single/launcher.py,sha256=IoJbn87j7t7Pib_FxoWZmmX8asXOqNGb-9ospw6EYJI,7302
-cobweb/single/models.py,sha256=wIEV35666lxdzqjDqBHPjjh-r6zD0x24rtQYz7d4Oxw,4332
-cobweb/single/nest.py,sha256=49K6KQ934INfPrWQsrq9rIFpQauLbLGOFbDaHvoQzOk,5015
-cobweb/utils/__init__.py,sha256=JTE4sBfHnKHhD6w9Auk0MIT7O9BMOamCeryhlHNx3Zg,47
-cobweb/utils/oss.py,sha256=8QlVVhXv3GBk53_616MOjbDxgD7c0ooJ0X28BE9pw-M,3220
-cobweb/utils/tools.py,sha256=8oLxkzwaYcDqKXXuLI3A_lNabyLBr7HSPgTF6x4xbnk,1239
-cobweb_launcher-1.0.0.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.0.0.dist-info/METADATA,sha256=aLZS3GZb0OACP69bVQXj1mOkTT3fRYNxiEB2e2G6GrE,1245
-cobweb_launcher-1.0.0.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.0.0.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.0.0.dist-info/RECORD,,