cobweb-launcher 0.1.15__py3-none-any.whl → 0.1.16__py3-none-any.whl

cobweb/equip/dev/launcher.py

@@ -0,0 +1,202 @@
+import time
+import threading
+
+from .. import log, sqn, rtn, pim
+from .. import Queue, DBItem, RedisDB, Setting, OssDB
+from .models import Scheduler, Spider, Storer
+
+
+def check(stop, last, spider, scheduler, storer, ready_seed_length, spider_queue_length):
+    log.info("run check thread after 30 seconds...")
+    time.sleep(30)
+    spider_info = """
+------------------- check: {0} ------------------
+            redis_spider_seed_length:  {1}
+            redis_ready_seed_length:   {2}
+            running_spider_thread_num: {3}
+            memory_seed_queue_length:  {4}
+            storer_queue_length_info:  {5}
+-----------------------  end  -----------------------"""
+    while True:
+        status = "running"
+        running_spider_thread_num = spider.spider_in_progress.length
+        redis_ready_seed_length = ready_seed_length()
+        redis_spider_seed_length = spider_queue_length()
+        memory_seed_queue_length = scheduler.queue.length
+        storer_upload_queue_length = storer.queue.length if storer else None
+        if (
+                scheduler.stop and
+                not memory_seed_queue_length and
+                not running_spider_thread_num
+        ):
+            if not Setting.LAUNCHER_MODEL:
+                log.info("spider is done?")
+            last.set()
+            time.sleep(3)
+            storer_queue_empty = True
+            if storer and storer.queue.length:
+                storer_queue_empty = False
+            storer_upload_queue_length = storer.queue.length if storer else None
+            if (
+                    storer_queue_empty and
+                    not redis_ready_seed_length and
+                    not redis_spider_seed_length
+            ):
+                if Setting.LAUNCHER_MODEL:
+                    log.info("waiting for push seeds...")
+                    status = "waiting"
+                    time.sleep(30)
+                else:
+                    log.info("spider done!")
+                    break
+
+            last.clear()
+
+        log.info(spider_info.format(
+            status,
+            redis_spider_seed_length,
+            redis_ready_seed_length,
+            running_spider_thread_num,
+            memory_seed_queue_length,
+            storer_upload_queue_length
+        ))
+
+        time.sleep(3)
+    stop.set()
+
+
+def launcher(task):
+    """
+    任务启动装饰器
+    :param task: 任务配置信息
+    """
+    def decorator(func):
+        # 程序结束事件
+        last = threading.Event()
+        # 停止采集事件
+        stop = threading.Event()
+
+        # 初始化redis信息
+        redis_db = RedisDB(task.project, task.task_name, task.redis_info)
+
+        # new item
+        item = type("Item", (object,), {"redis_client": redis_db.client})()
+        if task.oss_config:
+            item.oss = OssDB(**task.oss_config)
+
+        log.info("初始化cobweb!")
+
+        seed_queue = Queue()
+
+        scheduler_info = task.scheduler_info or dict()
+        # 调度器动态继承
+        sql = scheduler_info.get("sql")
+        table = scheduler_info.get("table")
+        size = scheduler_info.get("size")
+        scheduler_config = scheduler_info.get("config")
+        scheduler_db = scheduler_info.get("db", "default")
+        DB, class_name = pim(scheduler_db, "scheduler")
+        # SchedulerDB, table, sql, length, size, config = task.scheduler_info
+        SchedulerTmp = type(class_name, (Scheduler, DB), {})
+        # 初始化调度器
+        scheduler = SchedulerTmp(
+            table=table, sql=sql, size=size, queue=seed_queue,
+            length=task.scheduler_queue_length, config=scheduler_config
+        )
+
+        storer = None
+        storer_info = task.storer_info or dict()
+
+        if storer_info:
+            storer_db = storer_info["db"]
+            fields = storer_info["fields"]
+            storer_table = storer_info.get("table", "console")
+            storer_config = storer_info.get("config")
+
+            StorerDB, class_name = pim(storer_db, "storer")
+            StorerTmp = type(class_name, (Storer, StorerDB), {})
+
+            db_name = class_name.lower()
+            if not getattr(item, db_name, None):
+                instance = type(db_name, (DBItem,), {})
+                setattr(item, db_name, instance)
+
+            storer_item_instance = getattr(item, db_name)
+            storer_item_instance.init_item(storer_table, fields)
+
+            storer_queue = sqn(db_name, storer_table)
+            queue = getattr(storer_item_instance, storer_queue)
+            # 初始话存储器
+            table_name = rtn(table_name=storer_table)
+            storer = StorerTmp(
+                table=table_name, fields=fields,
+                length=task.storer_queue_length,
+                queue=queue, config=storer_config
+            )
+
+        # 初始化采集器
+        spider = Spider(seed_queue, storer and True, task.max_retries)
+
+        threading.Thread(target=redis_db.check_spider_queue, args=(stop, 0)).start()
+        threading.Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
+
+        # 推送初始种子
+        # seeds = start_seeds(task.start_seed)
+        redis_db.add_seed(task.seeds)
+        # 启动调度器, 调度至redis队列
+        threading.Thread(
+            # name="xxxx_schedule_seeds",
+            target=scheduler.schedule_seed,
+            args=(
+                redis_db.ready_seed_length,
+                redis_db.get_scheduler_lock,
+                redis_db.add_seed
+            )
+        ).start()
+
+        # 启动调度器, 调度任务队列
+        threading.Thread(
+            # name="xxxx_schedule_task",
+            target=scheduler.schedule_task,
+            args=(
+                stop, redis_db.get_seed,
+                redis_db.ready_seed_length
+            )
+        ).start()
+
+        # 启动采集器
+        for index in range(task.spider_num):
+            threading.Thread(
+                # name=f"xxxx_spider_task:{index}",
+                target=spider.spider_task,
+                args=(
+                    stop, func, item,
+                    redis_db.del_seed
+                )
+            ).start()
+
+        # 启动存储器
+        if storer:
+            threading.Thread(
+                # name=f"xxxx_store_task:{storer.table}",
+                target=storer.store_task,
+                args=(
+                    stop, last,
+                    redis_db.reset_seed,
+                    redis_db.del_seed
+                )
+            ).start()
+
+        threading.Thread(
+            # name="check_spider",
+            target=check,
+            args=(
+                stop, last, spider,
+                scheduler, storer,
+                redis_db.ready_seed_length,
+                redis_db.spider_queue_length,
+            )
+        ).start()
+
+    return decorator
+

cobweb/equip/dev/models.py

@@ -0,0 +1,156 @@
+import time
+from inspect import isgenerator
+# from pympler import asizeof
+from .. import log, ici
+from .. import DealModel, Queue, Seed, Setting
+
+
+class Scheduler:
+
+    def schedule_seed(self, ready_seed_length, get_scheduler_lock, add_seed):
+
+        inf_name = "SchedulerInterface"
+        if not ici(self.__class__, inf_name):
+            raise Exception("not have schedule function!")
+
+        if self.__class__.__name__ == "Default":
+            self.stop = True
+            return None
+
+        while not self.stop:
+            length = ready_seed_length()
+            if length > self.size:
+                time.sleep(15)
+
+            elif get_scheduler_lock():
+                seeds = self.schedule()
+                add_seed(seeds)
+
+        log.info(f"close thread: schedule_seed")
+
+    def schedule_task(self, stop, get_seed, ready_seed_length):
+        time.sleep(3)
+        while not stop.is_set():
+
+            if not ready_seed_length():
+                time.sleep(5)
+                continue
+
+            if self.queue.length >= self.length:
+                time.sleep(3)
+                continue
+
+            seeds = get_seed(self.length)
+            self.queue.push(seeds)
+        log.info(f"close thread: schedule_task")
+
+
+class Spider:
+
+    def __init__(self, queue, storage, max_retries=5):
+        self.spider_in_progress = Queue()
+        self.max_retries = max_retries
+        self.storage = storage
+        self.queue = queue
+
+    def spider_task(self, stop, func, item, del_seed):
+        while not stop.is_set():
+
+            seed = self.queue.pop()
+
+            if not seed:
+                time.sleep(3)
+                continue
+
+            elif seed._retry >= self.max_retries:
+                del_seed(seed, spider_status=False)
+                continue
+
+            try:
+                self.spider_in_progress.push(1, direct_insertion=True)
+                log.info("spider seed: " + str(seed))
+                iterators = func(item, seed)
+
+                if not isgenerator(iterators):
+                    if not self.storage:
+                        del_seed(seed, spider_status=True)
+                        continue
+                    raise TypeError(f"{func.__name__} isn't a generator")
+
+                status = None
+                for it in iterators:
+                    status = True
+                    if getattr(it, "table_name", None):
+                        store_queue = it.queue()
+                        store_queue.push(
+                            [seed, it.struct_data],
+                            direct_insertion=True
+                        )
+                    elif isinstance(it, Seed):
+                        self.queue.push(it)
+
+                    elif isinstance(it, str) and it == DealModel.polling:
+                        self.queue.push(seed)
+                        break
+                    elif isinstance(it, str) and it == DealModel.success:
+                        del_seed(seed, spider_status=True)
+                        break
+                    elif isinstance(it, str) and it == DealModel.failure:
+                        del_seed(seed, spider_status=False)
+                        break
+                    else:
+                        raise TypeError("yield value type error!")
+
+                if not status:
+                    seed._retry += 1
+                    self.queue.push(seed)
+
+            except Exception as e:
+                seed._retry += 1
+                self.queue.push(seed)
+                log.info(f"{str(seed)} -> {str(e)}")
+            finally:
+                self.spider_in_progress.pop()
+                time.sleep(Setting.SPIDER_RUN_TIME)
+        log.info(f"close thread: spider")
+
+
+class Storer:
+
+    def store_task(self, stop, last, reset_seed, del_seed):
+
+        inf_name = "StorerInterface"
+        if not ici(self.__class__, inf_name):
+            return None
+
+        if not getattr(self, "store", None):
+            raise Exception("not have store function!")
+
+        storer_name = self.__class__.__name__ + self.table
+
+        while not stop.is_set():
+
+            storer_length = self.queue.length
+            if not storer_length:
+                time.sleep(5)
+                continue
+
+            if last.is_set() or storer_length >= self.length:
+                seeds, data_list = [], []
+
+                for _ in range(self.length):
+                    items = self.queue.pop()
+                    if not items:
+                        break
+                    seed, data = items
+                    seeds.append(seed)
+                    data_list.append(data)
+
+                if self.store(data_list):
+                    del_seed(seeds)
+                else:
+                    reset_seed(seeds)
+
+            time.sleep(3)
+
+        log.info(f"close thread: {storer_name}")

cobweb/equip/single/models.py

@@ -69,6 +69,9 @@ class Spider:
             try:
                 self.spider_in_progress.push(1, direct_insertion=True)
                 log.info("spider seed: " + str(seed))
+
+                store_queue = None
+                store_data = list()
                 iterators = func(item, seed)
 
                 if not isgenerator(iterators):
@@ -80,12 +83,16 @@ class Spider:
                 status = None
                 for it in iterators:
                     status = True
+                    # if getattr(it, "table_name", None):
+                    #     store_queue = it.queue()
+                    #     store_queue.push(
+                    #         [seed, it.struct_data],
+                    #         direct_insertion=True
+                    #     )
                     if getattr(it, "table_name", None):
-                        store_queue = it.queue()
-                        store_queue.push(
-                            [seed, it.struct_data],
-                            direct_insertion=True
-                        )
+                        if not store_queue:
+                            store_queue = it.queue()
+                        store_data.append(it.struct_data)
                     elif isinstance(it, Seed):
                         self.queue.push(it)
 
@@ -105,6 +112,10 @@ class Spider:
                     seed._retry += 1
                     self.queue.push(seed)
 
+                elif store_queue and store_data:
+                    store_data.append(seed)
+                    store_queue.push(store_data)
+
             except Exception as e:
                 seed._retry += 1
                 self.queue.push(seed)
@@ -134,23 +145,27 @@ class Storer:
             if not storer_length:
                 time.sleep(5)
                 continue
+            elif not last.is_set() and storer_length < self.length:
+                time.sleep(3)
+                continue
 
-            if last.is_set() or storer_length >= self.length:
-                seeds, data_list = [], []
+            seeds, data_list = [], []
 
-                for _ in range(self.length):
-                    items = self.queue.pop()
-                    if not items:
+            while True:
+                data = self.queue.pop()
+                if not data:
+                    break
+                if isinstance(data, Seed):
+                    seeds.append(data)
+                    if len(data_list) >= self.length:
                         break
-                    seed, data = items
-                    seeds.append(seed)
-                    data_list.append(data)
+                    continue
+                data_list.append(data)
 
-                if self.store(data_list):
-                    del_seed(seeds)
-                else:
-                    reset_seed(seeds)
+            if self.store(data_list):
+                del_seed(seeds)
+            else:
+                reset_seed(seeds)
 
-            time.sleep(3)
 
         log.info(f"close thread: {storer_name}")

{cobweb_launcher-0.1.15.dist-info → cobweb_launcher-0.1.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.15
+Version: 0.1.16
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-0.1.15.dist-info → cobweb_launcher-0.1.16.dist-info}/RECORD

@@ -22,18 +22,21 @@ cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 cobweb/distributed/launcher.py,sha256=jTtBXBmna_6yFdj6gyGQiiEtg8I0g5uI5h8kbHWt454,7998
 cobweb/distributed/models.py,sha256=PUQokXMGD-H4A99nX7qYA395Ul6IsWGruMTVa05nswY,4568
 cobweb/equip/__init__.py,sha256=LWhbrTnG9kD1et0D40EzLISPuE0PIS-5WD3y3CLDaWk,247
+cobweb/equip/dev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/equip/dev/launcher.py,sha256=KRsw7yxklvFM85cel-EyLsNPLyrC9Hd26BMSx6-4Hac,6785
+cobweb/equip/dev/models.py,sha256=w3LQEhTrgqoYZn5v9TiEK2A68xuC7QH7suRP9OYnoOg,4813
 cobweb/equip/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/equip/distributed/launcher.py,sha256=1LzxibGXWR20XpXawakiRpEMaa9yfaj2rFSKnmEwjFc,7475
 cobweb/equip/distributed/models.py,sha256=qTGzxLdb2arsZSZK2HE4-MrqhraUhc2Ol5wBvlv_aWA,5008
 cobweb/equip/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/equip/single/launcher.py,sha256=KRsw7yxklvFM85cel-EyLsNPLyrC9Hd26BMSx6-4Hac,6785
-cobweb/equip/single/models.py,sha256=w3LQEhTrgqoYZn5v9TiEK2A68xuC7QH7suRP9OYnoOg,4813
+cobweb/equip/single/models.py,sha256=pWALuEat5v9cNAvJ_PaAlDcIHUWuu38EQl0KFcpSJXA,5344
 cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cobweb/single/launcher.py,sha256=IoJbn87j7t7Pib_FxoWZmmX8asXOqNGb-9ospw6EYJI,7302
 cobweb/single/models.py,sha256=UXcxr_Quok91k82plaqbj4deB-UBCWo14WCo6SS5L_o,4247
 cobweb/single/nest.py,sha256=49K6KQ934INfPrWQsrq9rIFpQauLbLGOFbDaHvoQzOk,5015
-cobweb_launcher-0.1.15.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-0.1.15.dist-info/METADATA,sha256=xkxL2woC_kK4XxUfrCaf80SeA5SE9t_0vwMynXt90lc,1220
-cobweb_launcher-0.1.15.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-cobweb_launcher-0.1.15.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-0.1.15.dist-info/RECORD,,
+cobweb_launcher-0.1.16.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.1.16.dist-info/METADATA,sha256=5LhIX1XNOr4pfnbotXN-lNWdwLUdBeb5Q3O157KORns,1220
+cobweb_launcher-0.1.16.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+cobweb_launcher-0.1.16.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.1.16.dist-info/RECORD,,