cobweb-launcher 0.1.16__tar.gz → 0.1.18__tar.gz

{cobweb-launcher-0.1.16 → cobweb-launcher-0.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.16
+Version: 0.1.18
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.1.16 → cobweb-launcher-0.1.18}/cobweb/__init__.py

@@ -3,6 +3,7 @@ from .task import Task
 from .log import log
 from .db.redis_db import RedisDB
 from .db.oss_db import OssDB
+from .constant import Setting
 
 from .equip.distributed.launcher import launcher
 from .equip.single.launcher import launcher as single_launcher

{cobweb-launcher-0.1.16 → cobweb-launcher-0.1.18}/cobweb/constant.py

@@ -22,4 +22,6 @@ class Setting:
     CHECK_LOCK_TIME = None
     DEAL_MODEL = None
     LAUNCHER_MODEL = None
-    SPIDER_RUN_TIME = None
+    SCHEDULER_WAIT_TIME = None
+    SCHEDULER_BLOCK_TIME = None
+    SPIDER_WAIT_TIME = None

cobweb-launcher-0.1.18/cobweb/db/storer/redis.py

@@ -0,0 +1,15 @@
+from cobweb import log, StorerInterface
+
+
+class Redis(StorerInterface):
+
+    def store(self, data_list):
+        try:
+            data_str = "\n".join(str(data) for data in data_list)
+            with open(self.table, "a") as fp:
+                fp.write(data_str)
+            log.info(f"save data, data length: {len(data_list)}")
+            return True
+        except Exception as e:
+            return False
+

cobweb-launcher-0.1.18/cobweb/distributed/launcher.py

@@ -0,0 +1,243 @@
+import time
+import threading
+from threading import Thread
+
+from .models import Scheduler, Spider, Storer
+from cobweb import log, Queue, DBItem, RedisDB
+from cobweb.setting import MODEL, RESET_SCORE, CHECK_LOCK_TIME
+from cobweb.utils import (
+    struct_queue_name as sqn,
+    restore_table_name as rtn,
+    parse_import_model as pim,
+)
+
+
+def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
+    log.info("run check thread after 30 seconds...")
+    time.sleep(30)
+    spider_info = """
+------------------- check: {0} ------------------
+            running_spider_thread_num: {1}
+            redis_ready_seed_length:   {2}
+            redis_spider_seed_length:  {3}
+            memory_seed_queue_length:  {4}
+            storer_upload_queue_length_info: 
+                {5}
+-----------------------  end  -----------------------"""
+    while True:
+        status = "running"
+        running_spider_thread_num = spider.spider_in_progress.length
+        redis_ready_seed_length = ready_seed_length()
+        redis_spider_seed_length = spider_queue_length()
+        memory_seed_queue_length = scheduler.queue.length
+        storer_upload_queue_list = []
+        for storer in storer_list:
+            storer_upload_queue_list.append(
+                f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
+            )
+        if (
+                scheduler.stop and
+                # not redis_ready_seed_length and
+                not memory_seed_queue_length and
+                not running_spider_thread_num
+        ):
+            if not MODEL:
+                log.info("spider is done?")
+            last.set()
+            time.sleep(3)
+            storer_queue_empty = True
+            storer_upload_queue_list = []
+            for storer in storer_list:
+                if storer.queue.length:
+                    storer_queue_empty = False
+                storer_upload_queue_list.append(
+                    f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
+                )
+            if (
+                    storer_queue_empty and
+                    not redis_ready_seed_length and
+                    not redis_spider_seed_length
+            ):
+                if MODEL:
+                    log.info("waiting for push seeds...")
+                    status = "waiting"
+                    time.sleep(30)
+                else:
+                    log.info("spider done!")
+                    break
+
+            last.clear()
+
+        storer_upload_queue_length_info = "\n            ".join(
+            storer_upload_queue_list) if storer_upload_queue_list else "None"
+        log.info(spider_info.format(
+            status,
+            running_spider_thread_num,
+            redis_ready_seed_length,
+            redis_spider_seed_length,
+            memory_seed_queue_length,
+            storer_upload_queue_length_info
+        ))
+
+        time.sleep(3)
+    stop.set()
+
+
+def launcher(task):
+    """
+    任务启动装饰器
+    :param task: 任务配置信息
+    """
+    def decorator(func):
+        """
+        Item:
+            Textfile()
+            Loghub()
+            Console()
+        e.g.
+        task.fields = "a,b"
+        func(item, seed)
+            a = "a"
+            b = "b"
+            data = {"a": "a", "b": "b"}
+            yield item.Loghub(**data)
+            yield item.Loghub(a=a, b=b)
+        """
+        storer_list = []
+
+        # 程序结束事件
+        last = threading.Event()
+        # 停止采集事件
+        stop = threading.Event()
+
+        # 初始化redis信息
+        redis_db = RedisDB(
+            task.project, task.task_name, task.redis_info,
+            model=MODEL, cs_lct=CHECK_LOCK_TIME, rs_time=RESET_SCORE
+        )
+
+        log.info("初始化cobweb!")
+
+        seed_queue = Queue()
+
+        if task.scheduler_info is None:
+            task.scheduler_info = dict()
+
+        # 调度器动态继承
+        sql = task.scheduler_info.get("sql")
+        table = task.scheduler_info.get("table")
+        size = task.scheduler_info.get("size")
+        scheduler_config = task.scheduler_info.get("config")
+        scheduler_db = task.scheduler_info.get("db", "default")
+        DB, class_name = pim(scheduler_db, "scheduler")
+        # SchedulerDB, table, sql, length, size, config = task.scheduler_info
+        SchedulerTmp = type(class_name, (Scheduler, DB), {})
+
+        # 初始化调度器
+        scheduler = SchedulerTmp(
+            table=table, sql=sql, size=size, queue=seed_queue,
+            length=task.scheduler_queue_length, config=scheduler_config
+        )
+
+        # 初始化采集器
+        spider = Spider(seed_queue, task.max_retries)
+
+        # 解析存储器信息
+        storer_info_list = task.storer_info or []
+        if not isinstance(storer_info_list, list):
+            storer_info_list = [storer_info_list]
+
+        # new item
+        item = type("Item", (object,), {"redis_client": redis_db.client})()
+
+        for storer_info in storer_info_list:
+            storer_db = storer_info["db"]
+            fields = storer_info["fields"]
+            storer_table = storer_info.get("table", "console")
+            storer_config = storer_info.get("config")
+
+            StorerDB, class_name = pim(storer_db, "storer")
+            StorerTmp = type(class_name, (Storer, StorerDB), {})
+
+            db_name = class_name.lower()
+            if not getattr(item, db_name, None):
+                instance = type(db_name, (DBItem,), {})
+                setattr(item, db_name, instance)
+
+            storer_item_instance = getattr(item, db_name)
+            storer_item_instance.init_item(storer_table, fields)
+
+            storer_queue = sqn(db_name, storer_table)
+            queue = getattr(storer_item_instance, storer_queue)
+            # 初始话存储器
+            table_name = rtn(table_name=storer_table)
+            storer = StorerTmp(
+                table=table_name, fields=fields,
+                length=task.storer_queue_length,
+                queue=queue, config=storer_config
+            )
+            storer_list.append(storer)
+
+        Thread(target=redis_db.check_spider_queue, args=(stop, len(storer_list))).start()
+        Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
+
+        # 推送初始种子
+        # seeds = start_seeds(task.start_seed)
+        redis_db.add_seed(task.seeds)
+        # 启动调度器, 调度至redis队列
+        Thread(
+            # name="xxxx_schedule_seeds",
+            target=scheduler.schedule_seed,
+            args=(
+                redis_db.ready_seed_length,
+                redis_db.get_scheduler_lock,
+                redis_db.add_seed
+            )
+        ).start()
+
+        # 启动调度器, 调度任务队列
+        Thread(
+            # name="xxxx_schedule_task",
+            target=scheduler.schedule_task,
+            args=(
+                stop, redis_db.get_seed,
+                redis_db.ready_seed_length
+            )
+        ).start()
+
+        # 启动采集器
+        for index in range(task.spider_num):
+            Thread(
+                # name=f"xxxx_spider_task:{index}",
+                target=spider.spider_task,
+                args=(
+                    stop, func, item,
+                    redis_db.del_seed
+                )
+            ).start()
+
+        # 启动存储器
+        for storer in storer_list:
+            Thread(
+                # name=f"xxxx_store_task:{storer.table}",
+                target=storer.store_task,
+                args=(
+                    stop, last,
+                    redis_db.reset_seed,
+                    redis_db.set_storer
+                )
+            ).start()
+
+        Thread(
+            # name="check_spider",
+            target=check,
+            args=(
+                stop, last, spider,
+                scheduler, storer_list,
+                redis_db.ready_seed_length,
+                redis_db.spider_queue_length,
+            )
+        ).start()
+
+    return decorator
+

cobweb-launcher-0.1.18/cobweb/distributed/models.py

@@ -0,0 +1,143 @@
+import time
+from hashlib import md5
+from cobweb import log, Queue, Seed
+from cobweb.utils import issubclass_cobweb_inf
+
+# from pympler import asizeof
+
+
+class Scheduler:
+
+    def schedule_seed(self, ready_seed_length, get_scheduler_lock, add_seed):
+
+        inf_name = "SchedulerInterface"
+        if not issubclass_cobweb_inf(self.__class__, inf_name):
+            raise Exception("not have schedule function!")
+
+        if self.__class__.__name__ == "Default":
+            self.stop = True
+            return None
+
+        while not self.stop:
+            length = ready_seed_length()
+            if length > self.size:
+                time.sleep(15)
+
+            elif get_scheduler_lock():
+                seeds = self.schedule()
+                add_seed(seeds)
+
+        log.info(f"close thread: schedule_seed")
+
+    def schedule_task(self, stop, get_seed, ready_seed_length):
+        time.sleep(3)
+        while not stop.is_set():
+
+            if not ready_seed_length():
+                time.sleep(5)
+                continue
+
+            if self.queue.length >= self.length:
+                time.sleep(3)
+                continue
+
+            seeds = get_seed(self.length)
+            self.queue.push(seeds)
+        log.info(f"close thread: schedule_task")
+
+
+class Spider:
+
+    def __init__(self, queue, max_retries=5):
+        self.spider_in_progress = Queue()
+        self.max_retries = max_retries
+        self.queue = queue
+
+    def spider_task(self, stop, func, item, del_seed):
+        while not stop.is_set():
+            seed = self.queue.pop()
+            if not seed:
+                time.sleep(3)
+                continue
+            elif seed._retry >= self.max_retries:
+                del_seed(seed, spider_status=False)
+                continue
+            try:
+                self.spider_in_progress.push(1, direct_insertion=True)
+                # log.info("spider seed: " + str(seed))
+                ret_count = 0
+                status = None
+                store_queue = None
+                store_data = list()
+                for it in func(item, seed):
+                    ret_count += 1
+                    if getattr(it, "table_name", None):
+                        if not store_queue:
+                            store_queue = it.queue()
+                        store_data.append(it.struct_data)
+                    elif isinstance(it, Seed):
+                        self.queue.push(it)
+                    elif any(isinstance(it, t) for t in (list, tuple)):
+                        self.queue.push([s if isinstance(s, Seed) else Seed(s) for s in it])
+                    elif isinstance(it, bool):
+                        status = it
+
+                if store_queue and store_data:
+                    store_data.append(seed)
+                    store_queue.push(store_data)
+
+                if status:
+                    del_seed(seed, spider_status=True)
+                elif not ret_count or status is False:
+                    seed._retry += 1
+                    self.queue.push(seed)
+
+            except Exception as e:
+                seed._retry += 1
+                self.queue.push(seed)
+                log.info(f"{str(seed)} -> {str(e)}")
+            finally:
+                self.spider_in_progress.pop()
+        log.info(f"close thread: spider")
+
+
+class Storer:
+
+    def store_task(self, stop, last, reset_seed, set_storer):
+
+        inf_name = "StorerInterface"
+        if not issubclass_cobweb_inf(self.__class__, inf_name):
+            return None
+
+        if not getattr(self, "store", None):
+            raise Exception("not have store function!")
+
+        storer_name = self.__class__.__name__ + self.table
+        store_key_id = md5(storer_name.encode()).hexdigest()
+
+        while not stop.is_set():
+
+            if last.is_set() or self.queue.length >= self.length:
+                seeds, data_list = [], []
+
+                while True:
+                    data = self.queue.pop()
+                    if not data:
+                        break
+                    if isinstance(data, Seed):
+                        seeds.append(data)
+                        if len(data_list) >= self.length:
+                            break
+                        continue
+                    data_list.append(data)
+
+                if data_list:
+                    if self.store(data_list):
+                        set_storer(store_key_id, seeds)
+                    else:
+                        reset_seed(seeds)
+                    continue
+
+            time.sleep(3)
+
+        log.info(f"close thread: {storer_name}")

cobweb-launcher-0.1.18/cobweb/setting.py

@@ -0,0 +1,13 @@
+import os
+
+
+# model: 0, 1, 2
+MODEL = int(os.getenv("MODEL", "0"))
+
+# 重制score值的等待时间, 默认10分钟
+RESET_SCORE = int(os.getenv("RESET_SCORE", "600"))
+
+# 默认设置检查spider queue队列锁的存活时间为30s
+CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
+
+

cobweb-launcher-0.1.18/cobweb/single/launcher.py

@@ -0,0 +1,231 @@
+import time
+import threading
+from threading import Thread
+
+from .models import Scheduler, Spider, Storer
+from cobweb import log, Queue, DBItem, RedisDB
+from cobweb.setting import MODEL, RESET_SCORE, CHECK_LOCK_TIME
+from cobweb.utils import (
+    struct_queue_name as sqn,
+    restore_table_name as rtn,
+    parse_import_model as pim,
+)
+
+
+def check(stop, last, spider, scheduler, storer, ready_seed_length, spider_queue_length):
+    log.info("run check thread after 30 seconds...")
+    time.sleep(30)
+    spider_info = """
+------------------- check: {0} ------------------
+            redis_spider_seed_length:  {1}
+            redis_ready_seed_length:   {2}
+            running_spider_thread_num: {3}
+            memory_seed_queue_length:  {4}
+            storer_queue_length_info:  {5}
+-----------------------  end  -----------------------"""
+    while True:
+        status = "running"
+        running_spider_thread_num = spider.spider_in_progress.length
+        redis_ready_seed_length = ready_seed_length()
+        redis_spider_seed_length = spider_queue_length()
+        memory_seed_queue_length = scheduler.queue.length
+        storer_upload_queue_length = storer.queue.length
+        if (
+                scheduler.stop and
+                # not redis_ready_seed_length and
+                not memory_seed_queue_length and
+                not running_spider_thread_num
+        ):
+            if not MODEL:
+                log.info("spider is done?")
+            last.set()
+            time.sleep(3)
+            storer_queue_empty = True
+            if storer.queue.length:
+                storer_queue_empty = False
+            storer_upload_queue_length = storer.queue.length
+            if (
+                    storer_queue_empty and
+                    not redis_ready_seed_length and
+                    not redis_spider_seed_length
+            ):
+                if MODEL:
+                    log.info("waiting for push seeds...")
+                    status = "waiting"
+                    time.sleep(30)
+                else:
+                    log.info("spider done!")
+                    break
+
+            last.clear()
+
+        log.info(spider_info.format(
+            status,
+            redis_spider_seed_length,
+            redis_ready_seed_length,
+            running_spider_thread_num,
+            memory_seed_queue_length,
+            storer_upload_queue_length
+        ))
+
+        time.sleep(3)
+    stop.set()
+
+
+def launcher(task):
+    """
+    任务启动装饰器
+    :param task: 任务配置信息
+    """
+    def decorator(func):
+        """
+        Item:
+            Textfile()
+            Loghub()
+            Console()
+        e.g.
+        task.fields = "a,b"
+        func(item, seed)
+            a = "a"
+            b = "b"
+            data = {"a": "a", "b": "b"}
+            yield item.Loghub(**data)
+            yield item.Loghub(a=a, b=b)
+        """
+        storer_list = []
+
+        # 程序结束事件
+        last = threading.Event()
+        # 停止采集事件
+        stop = threading.Event()
+
+        # 初始化redis信息
+        redis_db = RedisDB(
+            task.project, task.task_name, task.redis_info,
+            model=MODEL, cs_lct=CHECK_LOCK_TIME, rs_time=RESET_SCORE
+        )
+
+        # new item
+        item = type("Item", (object,), {"redis_client": redis_db.client})()
+
+        log.info("初始化cobweb!")
+
+        seed_queue = Queue()
+
+        scheduler_info = task.scheduler_info or dict()
+
+        # 调度器动态继承
+        sql = scheduler_info.get("sql")
+        table = scheduler_info.get("table")
+        size = scheduler_info.get("size")
+        scheduler_config = scheduler_info.get("config")
+        scheduler_db = scheduler_info.get("db", "default")
+        DB, class_name = pim(scheduler_db, "scheduler")
+        # SchedulerDB, table, sql, length, size, config = task.scheduler_info
+        SchedulerTmp = type(class_name, (Scheduler, DB), {})
+
+        # 初始化调度器
+        scheduler = SchedulerTmp(
+            table=table, sql=sql, size=size, queue=seed_queue,
+            length=task.scheduler_queue_length, config=scheduler_config
+        )
+
+        # 初始化采集器
+        spider = Spider(seed_queue, task.max_retries)
+
+        storer = None
+
+        # 解析存储器信息
+        storer_info = task.storer_info or dict()
+
+        # for storer_info in storer_info_list:
+        if storer_info:
+            storer_db = storer_info["db"]
+            fields = storer_info["fields"]
+            storer_table = storer_info.get("table", "console")
+            storer_config = storer_info.get("config")
+
+            StorerDB, class_name = pim(storer_db, "storer")
+            StorerTmp = type(class_name, (Storer, StorerDB), {})
+
+            db_name = class_name.lower()
+            if not getattr(item, db_name, None):
+                instance = type(db_name, (DBItem,), {})
+                setattr(item, db_name, instance)
+
+            storer_item_instance = getattr(item, db_name)
+            storer_item_instance.init_item(storer_table, fields)
+
+            storer_queue = sqn(db_name, storer_table)
+            queue = getattr(storer_item_instance, storer_queue)
+            # 初始话存储器
+            table_name = rtn(table_name=storer_table)
+            storer = StorerTmp(
+                table=table_name, fields=fields,
+                length=task.storer_queue_length,
+                queue=queue, config=storer_config
+            )
+
+        Thread(target=redis_db.check_spider_queue, args=(stop, len(storer_list))).start()
+        Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
+
+        # 推送初始种子
+        # seeds = start_seeds(task.start_seed)
+        redis_db.add_seed(task.seeds)
+        # 启动调度器, 调度至redis队列
+        Thread(
+            # name="xxxx_schedule_seeds",
+            target=scheduler.schedule_seed,
+            args=(
+                redis_db.ready_seed_length,
+                redis_db.get_scheduler_lock,
+                redis_db.add_seed
+            )
+        ).start()
+
+        # 启动调度器, 调度任务队列
+        Thread(
+            # name="xxxx_schedule_task",
+            target=scheduler.schedule_task,
+            args=(
+                stop, redis_db.get_seed,
+                redis_db.ready_seed_length
+            )
+        ).start()
+
+        # 启动采集器
+        for index in range(task.spider_num):
+            Thread(
+                # name=f"xxxx_spider_task:{index}",
+                target=spider.spider_task,
+                args=(
+                    stop, func, item,
+                    redis_db.del_seed
+                )
+            ).start()
+
+        # 启动存储器
+        if storer:
+            Thread(
+                # name=f"xxxx_store_task:{storer.table}",
+                target=storer.store_task,
+                args=(
+                    stop, last,
+                    redis_db.reset_seed,
+                    redis_db.del_seed
+                )
+            ).start()
+
+        Thread(
+            # name="check_spider",
+            target=check,
+            args=(
+                stop, last, spider,
+                scheduler, storer,
+                redis_db.ready_seed_length,
+                redis_db.spider_queue_length,
+            )
+        ).start()
+
+    return decorator
+

cobweb-launcher-0.1.18/cobweb/single/models.py

@@ -0,0 +1,134 @@
+import time
+from cobweb import log, Queue, Seed, Setting
+from cobweb.utils import issubclass_cobweb_inf
+# from pympler import asizeof
+
+
+class Scheduler:
+
+    def schedule_seed(self, ready_seed_length, get_scheduler_lock, add_seed):
+
+        inf_name = "SchedulerInterface"
+        if not issubclass_cobweb_inf(self.__class__, inf_name):
+            raise Exception("not have schedule function!")
+
+        if self.__class__.__name__ == "Default":
+            self.stop = True
+            return None
+
+        while not self.stop:
+            length = ready_seed_length()
+            if length > self.size:
+                time.sleep(15)
+
+            elif get_scheduler_lock():
+                seeds = self.schedule()
+                add_seed(seeds)
+
+        log.info(f"close thread: schedule_seed")
+
+    def schedule_task(self, stop, get_seed, ready_seed_length):
+        time.sleep(3)
+        while not stop.is_set():
+
+            if not ready_seed_length():
+                time.sleep(Setting.SCHEDULER_WAIT_TIME)
+                continue
+
+            if self.queue.length >= self.length:
+                time.sleep(Setting.SCHEDULER_BLOCK_TIME)
+                continue
+
+            seeds = get_seed(self.length)
+            self.queue.push(seeds)
+        log.info(f"close thread: schedule_task")
+
+
+class Spider:
+
+    def __init__(self, queue, max_retries=5):
+        self.spider_in_progress = Queue()
+        self.max_retries = max_retries
+        self.queue = queue
+
+    def spider_task(self, stop, func, item, del_seed):
+        while not stop.is_set():
+            seed = self.queue.pop()
+            if not seed:
+                time.sleep(Setting.SPIDER_WAIT_TIME)
+                continue
+            elif seed._retry >= self.max_retries:
+                del_seed(seed, spider_status=False)
+                continue
+            try:
+                self.spider_in_progress.push(1, direct_insertion=True)
+                # log.info("spider seed: " + str(seed))
+                ret_count = 0
+                status = None
+                for it in func(item, seed):
+                    ret_count += 1
+                    if getattr(it, "table_name", None):
+                        store_queue = it.queue()
+                        store_queue.push(
+                            [seed, it.struct_data],
+                            direct_insertion=True
+                        )
+                    elif isinstance(it, Seed):
+                        self.queue.push(it)
+                    elif any(isinstance(it, t) for t in (list, tuple)):
+                        self.queue.push([s if isinstance(s, Seed) else Seed(s) for s in it])
+                    elif isinstance(it, bool):
+                        status = it
+
+                if status:
+                    del_seed(seed, spider_status=True)
+                elif not ret_count or status is False:
+                    seed._retry += 1
+                    self.queue.push(seed)
+
+            except Exception as e:
+                seed._retry += 1
+                self.queue.push(seed)
+                log.info(f"{str(seed)} -> {str(e)}")
+            finally:
+                self.spider_in_progress.pop()
+        log.info(f"close thread: spider")
+
+
+class Storer:
+
+    def store_task(self, stop, last, reset_seed, del_seed):
+
+        inf_name = "StorerInterface"
+        if not issubclass_cobweb_inf(self.__class__, inf_name):
+            return None
+
+        if not getattr(self, "store", None):
+            raise Exception("not have store function!")
+
+        storer_name = self.__class__.__name__ + self.table
+
+        while not stop.is_set():
+
+            if last.is_set() or self.queue.length >= self.length:
+                seeds, data_list = [], []
+
+                for _ in range(self.length):
+                    items = self.queue.pop()
+                    if not items:
+                        break
+                    seed, data = items
+                    seeds.append(seed)
+                    data_list.append(data)
+
+                if data_list:
+                    if self.store(data_list):
+                        del_seed(seeds)
+                    else:
+                        reset_seed(seeds)
+                        log.info("reset seeds!")
+                    continue
+
+            time.sleep(3)
+
+        log.info(f"close thread: {storer_name}")

cobweb-launcher-0.1.18/cobweb/single/nest.py

@@ -0,0 +1,153 @@
+import time
+import threading
+
+from equip.single import Seed, DBItem
+from equip.single import struct_queue_name, restore_table_name
+from equip.single import Distributor, Scheduler, Spider, Storer
+
+
+def init_task_seed(seeds):
+    if not seeds:
+        return None
+    if isinstance(seeds, list) or isinstance(seeds, tuple):
+        for seed in seeds:
+            yield Seed(seed)
+    elif isinstance(seeds, str) or isinstance(seeds, dict):
+        yield Seed(seeds)
+
+
+def parse_storer_info(storer_info):
+    storer_data = {}
+    storer_info_list = []
+    if storer_info.__class__.__name__ == 'StorerInfo':
+        storer_info_list.append(storer_info)
+    elif isinstance(storer_info, tuple) or isinstance(storer_info, list):
+        storer_info_list = storer_info
+    for info in storer_info_list:
+        db_name = info.DB.__name__
+        storer_data.setdefault(db_name, {"StorerDB": info.DB, "db_args_list": []})
+        storer_data[db_name]["db_args_list"].append(info[1:])
+    return storer_data
+
+
+def check(stop_event, last_event, distributor, scheduler, spider, storer_list):
+    while True:
+        time.sleep(3)
+        if (
+                scheduler.stop and
+                not distributor.seed_queue.length and
+                not spider.spider_in_progress.length
+        ):
+            last_event.set()
+            time.sleep(10)
+            storer_queue_empty = True
+            for storer in storer_list:
+                if storer.queue.length:
+                    storer_queue_empty = False
+                    break
+            if storer_queue_empty:
+                break
+        last_event.clear()
+    stop_event.set()
+
+
+def cobweb(task):
+    """
+    任务启动装饰器
+    :param task: 任务配置信息
+    """
+    def decorator(func):
+        """
+        func(Item, seed)
+            Item:
+                Item.Textfile()
+                Item.Console()
+        """
+        # project task_name start_seed spider_num queue_length scheduler_info storer_info
+
+        storer_list = []
+
+        # 程序结束事件
+        last_event = threading.Event()
+        # 暂停采集事件
+        stop_event = threading.Event()
+
+        # 创建分发器
+        distributor = Distributor()
+
+        # 调度器动态继承
+        SchedulerDB, table, sql, length, size = task.SchedulerInfo
+        SchedulerTmp = type('Scheduler', (Scheduler, SchedulerDB), {})
+
+        # 初始化调度器
+        scheduler = SchedulerTmp(table=table, sql=sql, length=length, size=size, queue=distributor.seed_queue)
+
+        # 初始化采集器
+        spider = Spider(queue=distributor.seed_queue)
+
+        # 解析存储器信息
+        storer_data = parse_storer_info(task.storer_info)
+
+        # sds
+        item = type("item", (object,), {})
+        for db_name in storer_data.keys():
+            # 存储器动态继承
+            StorerDB = storer_data[db_name]["StorerDB"]
+            StorerTmp = type('Storer', (Storer, StorerDB), {})
+            db_args_list = storer_data[db_name]["db_args_list"]
+            for storer_db_args in db_args_list:
+                table, fields, length = storer_db_args
+                if not getattr(item, db_name, None):
+                    instance = type(db_name, (DBItem,), {})
+                    setattr(item, db_name, instance)
+                # 创建存储xxx
+                getattr(item, db_name).init_item(table, fields)
+                # 创建存储队列
+                storer_queue = struct_queue_name(db_name, table)
+                distributor.create_queue(queue_name=storer_queue)
+                queue = distributor.get_queue(queue_name=storer_queue)
+                # 初始话存储器
+                table_name = restore_table_name(table_name=table)
+                storer = StorerTmp(table=table_name, fields=fields, length=length, queue=queue)
+                storer_list.append(storer)
+
+        # 推送初始种子
+        distributor.distribute(init_task_seed, seeds=task.start_seed)
+
+        # 启动调度器
+        threading.Thread(
+            target=scheduler.schedule_task,
+            args=(distributor.distribute,),
+            name="single_scheduler_task"
+        ).start()
+
+        # 启动采集器
+        for index in range(task.spider_num):
+            threading.Thread(
+                target=spider.spider_task,
+                args=(stop_event, distributor.distribute, func, item),
+                name=f"single_spider_task:{index}"
+            ).start()
+
+        # 启动存储器
+        for storer in storer_list:
+            threading.Thread(
+                target=storer.store_task,
+                args=(stop_event, last_event, distributor.distribute),
+                name=f"single_store_task:{storer.table}",
+            ).start()
+
+        threading.Thread(
+            target=check, name="check",
+            args=(
+                stop_event, last_event, distributor,
+                scheduler, spider, storer_list
+            )
+        ).start()
+
+        # return starter(task, func)
+    return decorator
+
+
+
+

{cobweb-launcher-0.1.16 → cobweb-launcher-0.1.18}/cobweb/task.py

@@ -6,9 +6,11 @@ from .utils import parse_info, struct_start_seeds
 def init_task_env():
     Setting.RESET_SCORE = int(os.getenv("RESET_SCORE", 600))
     Setting.CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
-    Setting.SPIDER_RUN_TIME = float(os.getenv("SPIDER_RUN_TIME", 0.2))
     Setting.DEAL_MODEL = os.getenv("DEAL_MODEL", DealModel.failure)
     Setting.LAUNCHER_MODEL = os.getenv("LAUNCHER_MODEL", LauncherModel.task)
+    Setting.SCHEDULER_WAIT_TIME = float(os.getenv("SCHEDULER_WAIT_TIME", 5))
+    Setting.SCHEDULER_BLOCK_TIME = float(os.getenv("SCHEDULER_BLOCK_TIME", 3))
+    Setting.SPIDER_WAIT_TIME = float(os.getenv("SPIDER_WAIT_TIME", 3))
 
 
 class Task:

{cobweb-launcher-0.1.16 → cobweb-launcher-0.1.18}/cobweb_launcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.16
+Version: 0.1.18
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb-launcher-0.1.16 → cobweb-launcher-0.1.18}/cobweb_launcher.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ cobweb/constant.py
 cobweb/decorators.py
 cobweb/interface.py
 cobweb/log.py
+cobweb/setting.py
 cobweb/task.py
 cobweb/utils.py
 cobweb/db/__init__.py
@@ -18,7 +19,11 @@ cobweb/db/scheduler/textfile.py
 cobweb/db/storer/__init__.py
 cobweb/db/storer/console.py
 cobweb/db/storer/loghub.py
+cobweb/db/storer/redis.py
 cobweb/db/storer/textfile.py
+cobweb/distributed/__init__.py
+cobweb/distributed/launcher.py
+cobweb/distributed/models.py
 cobweb/equip/__init__.py
 cobweb/equip/dev/__init__.py
 cobweb/equip/dev/launcher.py
@@ -29,6 +34,10 @@ cobweb/equip/distributed/models.py
 cobweb/equip/single/__init__.py
 cobweb/equip/single/launcher.py
 cobweb/equip/single/models.py
+cobweb/single/__init__.py
+cobweb/single/launcher.py
+cobweb/single/models.py
+cobweb/single/nest.py
 cobweb_launcher.egg-info/PKG-INFO
 cobweb_launcher.egg-info/SOURCES.txt
 cobweb_launcher.egg-info/dependency_links.txt

{cobweb-launcher-0.1.16 → cobweb-launcher-0.1.18}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="cobweb-launcher",
-    version="0.1.16",
+    version="0.1.18",
     packages=find_packages(),
     url="https://github.com/Juannie-PP/cobweb",
     license="MIT",