cobweb-launcher 0.1.8__py3-none-any.whl → 0.1.9__py3-none-any.whl

cobweb/__init__.py

@@ -1,11 +1,7 @@
 from .bbb import Seed, Queue, DBItem
 from .task import Task
 from .log import log
-from .interface import SchedulerInterface, StorerInterface
 from .db.redis_db import RedisDB
-from .db.oss_db import OssDB
-from .distributed.launcher import launcher
-from .single.launcher import launcher as single_launcher
-from . import setting
-
 
+from .equip.distributed.launcher import launcher
+from .equip.single.launcher import launcher as single_launcher

cobweb/constant.py

@@ -0,0 +1,24 @@
+
+
+class LauncherModel:
+    task = "launcher model: task"
+    resident = "launcher model: resident"
+
+
+class LogModel:
+    simple = "log model: simple"
+    common = "log model: common"
+    detailed = "log model: detailed"
+
+
+class DealModel:
+    failure = "deal model: failure"
+    success = "deal model: success"
+    polling = "deal model: polling"
+
+
+class Setting:
+    RESET_SCORE = None
+    CHECK_LOCK_TIME = None
+    DEAL_MODEL = None
+    LAUNCHER_MODEL = None

cobweb/db/__init__.py

@@ -1,2 +1,3 @@
-from . import oss_db, redis_db
-from . import scheduler, storer
+from .. import log, Seed, decorators
+from ..constant import Setting, DealModel
+from ..interface import SchedulerInterface, StorerInterface

cobweb/db/redis_db.py

@@ -1,7 +1,8 @@
 import time
 import redis
-from cobweb import Seed, log
-from cobweb.decorators import check_redis_status
+from . import log, decorators, Seed, Setting, DealModel
+# from cobweb.decorators import decorators.check_redis_status
+# from cobweb.constant import Setting, DealModel
 
 
 class RedisDB:
@@ -11,9 +12,6 @@ class RedisDB:
             project: str,
             task_name: str,
             config: dict,
-            model: int,
-            cs_lct: int,
-            rs_time: int,
     ):
         pool = redis.ConnectionPool(**config)
         self.heartbeat_key = f"{project}:{task_name}:heartbeat"  # redis type string
@@ -25,11 +23,8 @@ class RedisDB:
         self.check_lock = f"{project}:{task_name}:check_seed_lock"  # redis type string
         self.scheduler_lock = f"{project}:{task_name}:scheduler_lock"  # redis type string
         self.client = redis.Redis(connection_pool=pool)
-        self.model = model
-        self.cs_lct = cs_lct
-        self.rs_time = rs_time
 
-    @check_redis_status
+    @decorators.check_redis_status
     def _get_lock(self, key, t=15, timeout=3, sleep_time=0.1):
         begin_time = int(time.time())
         while True:
@@ -55,7 +50,7 @@ class RedisDB:
             log.info("ttl: " + str(ttl))
             return False
 
-    @check_redis_status
+    @decorators.check_redis_status
     def _deal_seed(self, seeds, is_add: bool):
         if not seeds:
             return None
@@ -73,15 +68,15 @@ class RedisDB:
         if item_info:
             self.client.zadd(self.spider_key, mapping=item_info, nx=is_add, xx=not is_add)
 
-    @check_redis_status
+    @decorators.check_redis_status
     def add_seed(self, seeds):
         self._deal_seed(seeds, is_add=True)
 
-    @check_redis_status
+    @decorators.check_redis_status
     def reset_seed(self, seeds):
         self._deal_seed(seeds, is_add=False)
 
-    @check_redis_status
+    @decorators.check_redis_status
     def del_seed(self, seeds, spider_status: bool = True):
         if not seeds:
             return None
@@ -92,18 +87,16 @@ class RedisDB:
         seeds = [seed if isinstance(seed, Seed) else Seed(seed) for seed in seeds]
 
         if seeds:
-            # redis_key = self.succeed_key if spider_status else self.failed_key
             redis_key = None
-            if spider_status:
-                if isinstance(self.model, int) and self.model == 2:
-                    redis_key = self.succeed_key
-            else:
+            if spider_status and Setting.DEAL_MODEL in [DealModel.success, DealModel.polling]:
+                redis_key = self.succeed_key
+            elif not spider_status:
                 redis_key = self.failed_key
             if redis_key:
                 self.client.sadd(redis_key, *(str(seed) for seed in seeds))
             self.client.zrem(self.spider_key, *(seed.format_seed for seed in seeds))
 
-    @check_redis_status
+    @decorators.check_redis_status
     def set_storer(self, key, seeds):
         if not seeds:
             return None
@@ -122,7 +115,7 @@ class RedisDB:
             self.client.zadd(self.storer_key % key, mapping=item_info)
             log.info(f"zadd storer key: length {len(item_info.keys())}")
 
-    @check_redis_status
+    @decorators.check_redis_status
     def get_seed(self, length: int = 200):
         cs = time.time()
 
@@ -148,14 +141,14 @@ class RedisDB:
             log.info("push seeds into queue time: " + str(time.time() - cs))
             return result
 
-    @check_redis_status
+    @decorators.check_redis_status
     def check_spider_queue(self, stop, storer_num):
         while not stop.is_set():
             # 每15s获取check锁,等待600s后仍获取不到锁则重试;获取到锁后，设置锁的存活时间为${cs_lct}s
-            if self._get_lock(key=self.check_lock, t=self.cs_lct, timeout=600, sleep_time=3):
+            if self._get_lock(key=self.check_lock, t=Setting.CHECK_LOCK_TIME, timeout=600, sleep_time=3):
                 heartbeat = True if self.client.exists(self.heartbeat_key) else False
                 # 重启重制score值，否则获取${rs_time}分钟前的分数值
-                score = -int(time.time()) + self.rs_time if heartbeat else "-inf"
+                score = -int(time.time()) + Setting.RESET_SCORE if heartbeat else "-inf"
 
                 keys = self.client.keys(self.storer_key % "*")
 
@@ -170,7 +163,7 @@ class RedisDB:
                             break
                         for key in keys:
                             self.client.zrem(key, *members)
-                        if self.model == 2:
+                        if Setting.DEAL_MODEL in [DealModel.success, DealModel.polling]:
                             self.client.sadd(self.succeed_key, *members)
                         self.client.zrem(self.spider_key, *members)
                         self.client.zrem(intersection_key, *members)
@@ -193,31 +186,28 @@ class RedisDB:
                 if not heartbeat:
                     self.client.setex(self.heartbeat_key, 15, "")
 
-                # self.client.delete(self.check_lock)
-                # time.sleep(3)
-
-    @check_redis_status
+    @decorators.check_redis_status
     def set_heartbeat(self, stop):
         time.sleep(5)
         while not stop.is_set():
             self.client.setex(self.heartbeat_key, 5, "")
             time.sleep(3)
 
-    # @check_redis_status
+    # @decorators.check_redis_status
     # def heartbeat(self):
     #     """
     #     返回心跳key剩余存活时间
     #     """
     #     return self.client.ttl(self.heartbeat_key)
 
-    @check_redis_status
+    @decorators.check_redis_status
     def spider_queue_length(self):
         return self.client.zcard(self.spider_key)
 
-    @check_redis_status
+    @decorators.check_redis_status
     def ready_seed_length(self):
         return self.client.zcount(self.spider_key, min=0, max="+inf")
 
-    @check_redis_status
+    @decorators.check_redis_status
     def get_scheduler_lock(self):
         return self._get_lock(self.scheduler_lock)

cobweb/db/scheduler/__init__.py

@@ -0,0 +1 @@
+from .. import log, Seed, SchedulerInterface as Inf

cobweb/db/scheduler/default.py

@@ -1,7 +1,7 @@
-from cobweb import SchedulerInterface
+from . import Inf
 
 
-class Default(SchedulerInterface):
+class Default(Inf):
 
     def schedule(self):
         pass

cobweb/db/scheduler/textfile.py

@@ -1,7 +1,7 @@
-from cobweb import log, Seed, SchedulerInterface
+from . import Inf, log, Seed
 
 
-class Textfile(SchedulerInterface):
+class Textfile(Inf):
 
     index = None
 

cobweb/db/storer/__init__.py

@@ -0,0 +1 @@
+from .. import log, Seed, StorerInterface as Inf

cobweb/db/storer/console.py

@@ -1,7 +1,7 @@
-from cobweb import log, StorerInterface
+from . import Inf, log
 
 
-class Console(StorerInterface):
+class Console(Inf):
 
     def store(self, data_list):
         for item in data_list:

cobweb/db/storer/loghub.py

@@ -1,9 +1,9 @@
 import json
+from . import Inf, log
 from aliyun.log import LogClient, LogItem, PutLogsRequest
-from cobweb import log, StorerInterface
 
 
-class Loghub(StorerInterface):
+class Loghub(Inf):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)

cobweb/db/storer/textfile.py

@@ -1,7 +1,7 @@
-from cobweb import log, StorerInterface
+from . import Inf, log
 
 
-class Textfile(StorerInterface):
+class Textfile(Inf):
 
     def store(self, data_list):
         try:

cobweb/decorators.py

@@ -1,5 +1,5 @@
 from functools import wraps
-from cobweb import log
+from .log import log
 
 
 def check_redis_status(func):

cobweb/equip/__init__.py

@@ -0,0 +1,8 @@
+from .. import Queue, DBItem, RedisDB, Seed, log
+from ..constant import Setting, DealModel
+from utils import (
+    struct_queue_name as sqn,
+    restore_table_name as rtn,
+    parse_import_model as pim,
+    issubclass_cobweb_inf as ici
+)

cobweb/equip/distributed/launcher.py

@@ -0,0 +1,219 @@
+import time
+import threading
+
+from .. import log, sqn, rtn, pim
+from .. import Queue, DBItem, RedisDB, Setting
+from .models import Scheduler, Spider, Storer
+
+
+def check(stop, last, spider, scheduler, storer_list, ready_seed_length, spider_queue_length):
+    log.info("run check thread after 30 seconds...")
+    time.sleep(30)
+    spider_info = """
+------------------- check: {0} ------------------
+            running_spider_thread_num: {1}
+            redis_ready_seed_length:   {2}
+            redis_spider_seed_length:  {3}
+            memory_seed_queue_length:  {4}
+            storer_upload_queue_length_info: 
+                {5}
+-----------------------  end  -----------------------"""
+    while True:
+        status = "running"
+        running_spider_thread_num = spider.spider_in_progress.length
+        redis_ready_seed_length = ready_seed_length()
+        redis_spider_seed_length = spider_queue_length()
+        memory_seed_queue_length = scheduler.queue.length
+        storer_upload_queue_list = []
+        for storer in storer_list:
+            storer_upload_queue_list.append(
+                f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
+            )
+        if (
+                scheduler.stop and
+                not memory_seed_queue_length and
+                not running_spider_thread_num
+        ):
+            if not Setting.LAUNCHER_MODEL:
+                log.info("spider is done?")
+            last.set()
+            time.sleep(3)
+            storer_queue_empty = True
+            storer_upload_queue_list = []
+            for storer in storer_list:
+                if storer.queue.length:
+                    storer_queue_empty = False
+                storer_upload_queue_list.append(
+                    f"{storer.__class__.__name__} storer queue length: {storer.queue.length}"
+                )
+            if (
+                    storer_queue_empty and
+                    not redis_ready_seed_length and
+                    not redis_spider_seed_length
+            ):
+                if Setting.LAUNCHER_MODEL:
+                    log.info("waiting for push seeds...")
+                    status = "waiting"
+                    time.sleep(30)
+                else:
+                    log.info("spider done!")
+                    break
+
+            last.clear()
+
+        storer_upload_queue_length_info = "\n            ".join(
+            storer_upload_queue_list) if storer_upload_queue_list else "None"
+        log.info(spider_info.format(
+            status,
+            running_spider_thread_num,
+            redis_ready_seed_length,
+            redis_spider_seed_length,
+            memory_seed_queue_length,
+            storer_upload_queue_length_info
+        ))
+
+        time.sleep(3)
+    stop.set()
+
+
+def launcher(task):
+    """
+    任务启动装饰器
+    :param task: 任务配置信息
+    """
+    def decorator(func):
+        storer_list = []
+
+        # 程序结束事件
+        last = threading.Event()
+        # 停止采集事件
+        stop = threading.Event()
+
+        # 初始化redis信息
+        redis_db = RedisDB(task.project, task.task_name, task.redis_info)
+
+        log.info("初始化cobweb!")
+
+        seed_queue = Queue()
+
+        if task.scheduler_info is None:
+            task.scheduler_info = dict()
+
+        # 调度器动态继承
+        sql = task.scheduler_info.get("sql")
+        table = task.scheduler_info.get("table")
+        size = task.scheduler_info.get("size")
+        scheduler_config = task.scheduler_info.get("config")
+        scheduler_db = task.scheduler_info.get("db", "default")
+        DB, class_name = pim(scheduler_db, "scheduler")
+        # SchedulerDB, table, sql, length, size, config = task.scheduler_info
+        SchedulerTmp = type(class_name, (Scheduler, DB), {})
+
+        # 初始化调度器
+        scheduler = SchedulerTmp(
+            table=table, sql=sql, size=size, queue=seed_queue,
+            length=task.scheduler_queue_length, config=scheduler_config
+        )
+
+        # 解析存储器信息
+        storer_info_list = task.storer_info or []
+        if not isinstance(storer_info_list, list):
+            storer_info_list = [storer_info_list]
+
+        # new item
+        item = type("Item", (object,), {"redis_client": redis_db.client})()
+
+        for storer_info in storer_info_list:
+            storer_db = storer_info["db"]
+            fields = storer_info["fields"]
+            storer_table = storer_info.get("table", "console")
+            storer_config = storer_info.get("config")
+
+            StorerDB, class_name = pim(storer_db, "storer")
+            StorerTmp = type(class_name, (Storer, StorerDB), {})
+
+            db_name = class_name.lower()
+            if not getattr(item, db_name, None):
+                instance = type(db_name, (DBItem,), {})
+                setattr(item, db_name, instance)
+
+            storer_item_instance = getattr(item, db_name)
+            storer_item_instance.init_item(storer_table, fields)
+
+            storer_queue = sqn(db_name, storer_table)
+            queue = getattr(storer_item_instance, storer_queue)
+            # 初始话存储器
+            table_name = rtn(table_name=storer_table)
+            storer = StorerTmp(
+                table=table_name, fields=fields,
+                length=task.storer_queue_length,
+                queue=queue, config=storer_config
+            )
+            storer_list.append(storer)
+
+        # 初始化采集器
+        spider = Spider(seed_queue, storer_list and True, task.max_retries)
+
+        threading.Thread(target=redis_db.check_spider_queue, args=(stop, len(storer_list))).start()
+        threading.Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
+
+        # 推送初始种子
+        # seeds = start_seeds(task.start_seed)
+        redis_db.add_seed(task.seeds)
+        # 启动调度器, 调度至redis队列
+        threading.Thread(
+            # name="xxxx_schedule_seeds",
+            target=scheduler.schedule_seed,
+            args=(
+                redis_db.ready_seed_length,
+                redis_db.get_scheduler_lock,
+                redis_db.add_seed
+            )
+        ).start()
+
+        # 启动调度器, 调度任务队列
+        threading.Thread(
+            # name="xxxx_schedule_task",
+            target=scheduler.schedule_task,
+            args=(
+                stop, redis_db.get_seed,
+                redis_db.ready_seed_length
+            )
+        ).start()
+
+        # 启动采集器
+        for index in range(task.spider_num):
+            threading.Thread(
+                # name=f"xxxx_spider_task:{index}",
+                target=spider.spider_task,
+                args=(
+                    stop, func, item,
+                    redis_db.del_seed
+                )
+            ).start()
+
+        # 启动存储器
+        for storer in storer_list:
+            threading.Thread(
+                # name=f"xxxx_store_task:{storer.table}",
+                target=storer.store_task,
+                args=(
+                    stop, last,
+                    redis_db.reset_seed,
+                    redis_db.set_storer
+                )
+            ).start()
+
+        threading.Thread(
+            # name="check_spider",
+            target=check,
+            args=(
+                stop, last, spider,
+                scheduler, storer_list,
+                redis_db.ready_seed_length,
+                redis_db.spider_queue_length,
+            )
+        ).start()
+
+    return decorator
+

cobweb/equip/distributed/models.py

@@ -0,0 +1,152 @@
+import time
+from hashlib import md5
+from inspect import isgenerator
+
+from .. import log, ici
+from .. import DealModel, Queue, Seed
+# from pympler import asizeof
+
+
+class Scheduler:
+
+    def schedule_seed(self, ready_seed_length, get_scheduler_lock, add_seed):
+
+        inf_name = "SchedulerInterface"
+        if not ici(self.__class__, inf_name):
+            raise Exception("not have schedule function!")
+
+        if self.__class__.__name__ == "Default":
+            self.stop = True
+            return None
+
+        while not self.stop:
+            length = ready_seed_length()
+            if length > self.size:
+                time.sleep(15)
+
+            elif get_scheduler_lock():
+                seeds = self.schedule()
+                add_seed(seeds)
+
+        log.info(f"close thread: schedule_seed")
+
+    def schedule_task(self, stop, get_seed, ready_seed_length):
+        time.sleep(3)
+        while not stop.is_set():
+
+            if not ready_seed_length():
+                time.sleep(5)
+                continue
+
+            if self.queue.length >= self.length:
+                time.sleep(3)
+                continue
+
+            seeds = get_seed(self.length)
+            self.queue.push(seeds)
+        log.info(f"close thread: schedule_task")
+
+
+class Spider:
+
+    def __init__(self, queue, storage, max_retries=5):
+        self.spider_in_progress = Queue()
+        self.max_retries = max_retries
+        self.storage = storage
+        self.queue = queue
+
+    def spider_task(self, stop, func, item, del_seed):
+        while not stop.is_set():
+            seed = self.queue.pop()
+            if not seed:
+                time.sleep(3)
+                continue
+            elif seed._retry >= self.max_retries:
+                del_seed(seed, spider_status=False)
+                continue
+            try:
+                self.spider_in_progress.push(1, direct_insertion=True)
+                # log.info("spider seed: " + str(seed))
+
+                store_queue = None
+                store_data = list()
+
+                iterators = func(item, seed)
+
+                if not isgenerator(iterators):
+                    if not self.storage:
+                        del_seed(seed, spider_status=True)
+                        continue
+                    raise TypeError(f"{func.__name__} isn't a generator")
+
+                for it in iterators:
+                    if getattr(it, "table_name", None):
+                        if not store_queue:
+                            store_queue = it.queue()
+                        store_data.append(it.struct_data)
+                    elif isinstance(it, Seed):
+                        self.queue.push(it)
+
+                    elif isinstance(it, str) and it == DealModel.polling:
+                        self.queue.push(seed)
+                        break
+                    elif isinstance(it, str) and it == DealModel.success:
+                        del_seed(seed, spider_status=True)
+                        break
+                    elif isinstance(it, str) and it == DealModel.failure:
+                        del_seed(seed, spider_status=False)
+                        break
+                    else:
+                        raise TypeError("yield value type error!")
+
+                if store_queue and store_data:
+                    store_data.append(seed)
+                    store_queue.push(store_data)
+
+            except Exception as e:
+                seed._retry += 1
+                self.queue.push(seed)
+                log.info(f"{str(seed)} -> {str(e)}")
+            finally:
+                self.spider_in_progress.pop()
+        log.info(f"close thread: spider")
+
+
+class Storer:
+
+    def store_task(self, stop, last, reset_seed, set_storer):
+
+        inf_name = "StorerInterface"
+        if not ici(self.__class__, inf_name):
+            return None
+
+        if not getattr(self, "store", None):
+            raise Exception("not have store function!")
+
+        storer_name = self.__class__.__name__ + self.table
+        store_key_id = md5(storer_name.encode()).hexdigest()
+
+        while not stop.is_set():
+
+            if last.is_set() or self.queue.length >= self.length:
+                seeds, data_list = [], []
+
+                while True:
+                    data = self.queue.pop()
+                    if not data:
+                        break
+                    if isinstance(data, Seed):
+                        seeds.append(data)
+                        if len(data_list) >= self.length:
+                            break
+                        continue
+                    data_list.append(data)
+
+                if self.store(data_list):
+                    set_storer(store_key_id, seeds)
+                else:
+                    reset_seed(seeds)
+
+            time.sleep(3)
+
+        log.info(f"close thread: {storer_name}")

cobweb/equip/single/launcher.py

@@ -0,0 +1,200 @@
+import time
+import threading
+
+from .. import log, sqn, rtn, pim
+from .. import Queue, DBItem, RedisDB, Setting
+from .models import Scheduler, Spider, Storer
+
+
+def check(stop, last, spider, scheduler, storer, ready_seed_length, spider_queue_length):
+    log.info("run check thread after 30 seconds...")
+    time.sleep(30)
+    spider_info = """
+------------------- check: {0} ------------------
+            redis_spider_seed_length:  {1}
+            redis_ready_seed_length:   {2}
+            running_spider_thread_num: {3}
+            memory_seed_queue_length:  {4}
+            storer_queue_length_info:  {5}
+-----------------------  end  -----------------------"""
+    while True:
+        status = "running"
+        running_spider_thread_num = spider.spider_in_progress.length
+        redis_ready_seed_length = ready_seed_length()
+        redis_spider_seed_length = spider_queue_length()
+        memory_seed_queue_length = scheduler.queue.length
+        storer_upload_queue_length = storer.queue.length if storer else None
+        if (
+                scheduler.stop and
+                not memory_seed_queue_length and
+                not running_spider_thread_num
+        ):
+            if not Setting.LAUNCHER_MODEL:
+                log.info("spider is done?")
+            last.set()
+            time.sleep(3)
+            storer_queue_empty = True
+            if storer and storer.queue.length:
+                storer_queue_empty = False
+            storer_upload_queue_length = storer.queue.length if storer else None
+            if (
+                    storer_queue_empty and
+                    not redis_ready_seed_length and
+                    not redis_spider_seed_length
+            ):
+                if Setting.LAUNCHER_MODEL:
+                    log.info("waiting for push seeds...")
+                    status = "waiting"
+                    time.sleep(30)
+                else:
+                    log.info("spider done!")
+                    break
+
+            last.clear()
+
+        log.info(spider_info.format(
+            status,
+            redis_spider_seed_length,
+            redis_ready_seed_length,
+            running_spider_thread_num,
+            memory_seed_queue_length,
+            storer_upload_queue_length
+        ))
+
+        time.sleep(3)
+    stop.set()
+
+
+def launcher(task):
+    """
+    任务启动装饰器
+    :param task: 任务配置信息
+    """
+    def decorator(func):
+        # 程序结束事件
+        last = threading.Event()
+        # 停止采集事件
+        stop = threading.Event()
+
+        # 初始化redis信息
+        redis_db = RedisDB(task.project, task.task_name, task.redis_info)
+
+        # new item
+        item = type("Item", (object,), {"redis_client": redis_db.client})()
+
+        log.info("初始化cobweb!")
+
+        seed_queue = Queue()
+
+        scheduler_info = task.scheduler_info or dict()
+        # 调度器动态继承
+        sql = scheduler_info.get("sql")
+        table = scheduler_info.get("table")
+        size = scheduler_info.get("size")
+        scheduler_config = scheduler_info.get("config")
+        scheduler_db = scheduler_info.get("db", "default")
+        DB, class_name = pim(scheduler_db, "scheduler")
+        # SchedulerDB, table, sql, length, size, config = task.scheduler_info
+        SchedulerTmp = type(class_name, (Scheduler, DB), {})
+        # 初始化调度器
+        scheduler = SchedulerTmp(
+            table=table, sql=sql, size=size, queue=seed_queue,
+            length=task.scheduler_queue_length, config=scheduler_config
+        )
+
+        storer = None
+        storer_info = task.storer_info or dict()
+
+        if storer_info:
+            storer_db = storer_info["db"]
+            fields = storer_info["fields"]
+            storer_table = storer_info.get("table", "console")
+            storer_config = storer_info.get("config")
+
+            StorerDB, class_name = pim(storer_db, "storer")
+            StorerTmp = type(class_name, (Storer, StorerDB), {})
+
+            db_name = class_name.lower()
+            if not getattr(item, db_name, None):
+                instance = type(db_name, (DBItem,), {})
+                setattr(item, db_name, instance)
+
+            storer_item_instance = getattr(item, db_name)
+            storer_item_instance.init_item(storer_table, fields)
+
+            storer_queue = sqn(db_name, storer_table)
+            queue = getattr(storer_item_instance, storer_queue)
+            # 初始话存储器
+            table_name = rtn(table_name=storer_table)
+            storer = StorerTmp(
+                table=table_name, fields=fields,
+                length=task.storer_queue_length,
+                queue=queue, config=storer_config
+            )
+
+        # 初始化采集器
+        spider = Spider(seed_queue, storer and True, task.max_retries)
+
+        threading.Thread(target=redis_db.check_spider_queue, args=(stop, 0)).start()
+        threading.Thread(target=redis_db.set_heartbeat, args=(stop,)).start()
+
+        # 推送初始种子
+        # seeds = start_seeds(task.start_seed)
+        redis_db.add_seed(task.seeds)
+        # 启动调度器, 调度至redis队列
+        threading.Thread(
+            # name="xxxx_schedule_seeds",
+            target=scheduler.schedule_seed,
+            args=(
+                redis_db.ready_seed_length,
+                redis_db.get_scheduler_lock,
+                redis_db.add_seed
+            )
+        ).start()
+
+        # 启动调度器, 调度任务队列
+        threading.Thread(
+            # name="xxxx_schedule_task",
+            target=scheduler.schedule_task,
+            args=(
+                stop, redis_db.get_seed,
+                redis_db.ready_seed_length
+            )
+        ).start()
+
+        # 启动采集器
+        for index in range(task.spider_num):
+            threading.Thread(
+                # name=f"xxxx_spider_task:{index}",
+                target=spider.spider_task,
+                args=(
+                    stop, func, item,
+                    redis_db.del_seed
+                )
+            ).start()
+
+        # 启动存储器
+        if storer:
+            threading.Thread(
+                # name=f"xxxx_store_task:{storer.table}",
+                target=storer.store_task,
+                args=(
+                    stop, last,
+                    redis_db.reset_seed,
+                    redis_db.del_seed
+                )
+            ).start()
+
+        threading.Thread(
+            # name="check_spider",
+            target=check,
+            args=(
+                stop, last, spider,
+                scheduler, storer,
+                redis_db.ready_seed_length,
+                redis_db.spider_queue_length,
+            )
+        ).start()
+
+    return decorator
+

cobweb/equip/single/models.py

@@ -0,0 +1,144 @@
+import time
+from inspect import isgenerator
+# from pympler import asizeof
+from .. import log, ici
+from .. import DealModel, Queue, Seed
+
+
+class Scheduler:
+
+    def schedule_seed(self, ready_seed_length, get_scheduler_lock, add_seed):
+
+        inf_name = "SchedulerInterface"
+        if not ici(self.__class__, inf_name):
+            raise Exception("not have schedule function!")
+
+        if self.__class__.__name__ == "Default":
+            self.stop = True
+            return None
+
+        while not self.stop:
+            length = ready_seed_length()
+            if length > self.size:
+                time.sleep(15)
+
+            elif get_scheduler_lock():
+                seeds = self.schedule()
+                add_seed(seeds)
+
+        log.info(f"close thread: schedule_seed")
+
+    def schedule_task(self, stop, get_seed, ready_seed_length):
+        time.sleep(3)
+        while not stop.is_set():
+
+            if not ready_seed_length():
+                time.sleep(5)
+                continue
+
+            if self.queue.length >= self.length:
+                time.sleep(3)
+                continue
+
+            seeds = get_seed(self.length)
+            self.queue.push(seeds)
+        log.info(f"close thread: schedule_task")
+
+
+class Spider:
+
+    def __init__(self, queue, storage, max_retries=5):
+        self.spider_in_progress = Queue()
+        self.max_retries = max_retries
+        self.storage = storage
+        self.queue = queue
+
+    def spider_task(self, stop, func, item, del_seed):
+        while not stop.is_set():
+
+            seed = self.queue.pop()
+
+            if not seed:
+                time.sleep(3)
+                continue
+
+            elif seed._retry >= self.max_retries:
+                del_seed(seed, spider_status=False)
+                continue
+
+            try:
+                self.spider_in_progress.push(1, direct_insertion=True)
+                # log.info("spider seed: " + str(seed))
+                iterators = func(item, seed)
+
+                if not isgenerator(iterators):
+                    if not self.storage:
+                        del_seed(seed, spider_status=True)
+                        continue
+                    raise TypeError(f"{func.__name__} isn't a generator")
+
+                for it in iterators:
+                    if getattr(it, "table_name", None):
+                        store_queue = it.queue()
+                        store_queue.push(
+                            [seed, it.struct_data],
+                            direct_insertion=True
+                        )
+                    elif isinstance(it, Seed):
+                        self.queue.push(it)
+
+                    elif isinstance(it, str) and it == DealModel.polling:
+                        self.queue.push(seed)
+                        break
+                    elif isinstance(it, str) and it == DealModel.success:
+                        del_seed(seed, spider_status=True)
+                        break
+                    elif isinstance(it, str) and it == DealModel.failure:
+                        del_seed(seed, spider_status=False)
+                        break
+                    else:
+                        raise TypeError("yield value type error!")
+
+            except Exception as e:
+                seed._retry += 1
+                self.queue.push(seed)
+                log.info(f"{str(seed)} -> {str(e)}")
+            finally:
+                self.spider_in_progress.pop()
+        log.info(f"close thread: spider")
+
+
+class Storer:
+
+    def store_task(self, stop, last, reset_seed, del_seed):
+
+        inf_name = "StorerInterface"
+        if not ici(self.__class__, inf_name):
+            return None
+
+        if not getattr(self, "store", None):
+            raise Exception("not have store function!")
+
+        storer_name = self.__class__.__name__ + self.table
+
+        while not stop.is_set():
+
+            if last.is_set() or self.queue.length >= self.length:
+                seeds, data_list = [], []
+
+                for _ in range(self.length):
+                    items = self.queue.pop()
+                    if not items:
+                        break
+                    seed, data = items
+                    seeds.append(seed)
+                    data_list.append(data)
+
+                if self.store(data_list):
+                    del_seed(seeds)
+                else:
+                    reset_seed(seeds)
+
+            time.sleep(3)
+
+        log.info(f"close thread: {storer_name}")

cobweb/single/nest.py

@@ -1,9 +1,9 @@
 import time
 import threading
 
-from single.nest import Seed, DBItem
-from single.nest import struct_queue_name, restore_table_name
-from single.nest import Distributor, Scheduler, Spider, Storer
+from equip.single import Seed, DBItem
+from equip.single import struct_queue_name, restore_table_name
+from equip.single import Distributor, Scheduler, Spider, Storer
 
 
 def init_task_seed(seeds):

cobweb/task.py

@@ -1,11 +1,19 @@
+import os
+from .constant import *
 from .utils import parse_info, struct_start_seeds
 
 
+def init_task_env():
+    Setting.RESET_SCORE = int(os.getenv("RESET_SCORE", 600))
+    Setting.CHECK_LOCK_TIME = int(os.getenv("CHECK_LOCK_TIME", 30))
+    Setting.DEAL_MODEL = os.getenv("DEAL_MODEL", DealModel.failure)
+    Setting.LAUNCHER_MODEL = os.getenv("LAUNCHER_MODEL", LauncherModel.task)
+
+
 class Task:
 
     def __init__(
             self,
-            # model=None,
             seeds=None,
             project=None,
             task_name=None,
@@ -31,8 +39,7 @@ class Task:
         :param storer_queue_length:
         :param scheduler_queue_length:
         """
-        # self.model = model
-
+        init_task_env()
         self.seeds = struct_start_seeds(seeds)
         self.project = project or "test"
         self.task_name = task_name or "spider"

{cobweb_launcher-0.1.8.dist-info → cobweb_launcher-0.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 0.1.8
+Version: 0.1.9
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
@@ -11,9 +11,9 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: requests (>=2.19.1)
-Requires-Dist: oss2 (>=2.18.1)
-Requires-Dist: redis (>=4.4.4)
+Requires-Dist: requests >=2.19.1
+Requires-Dist: oss2 >=2.18.1
+Requires-Dist: redis >=4.4.4
 Requires-Dist: aliyun-log-python-sdk
 
 # cobweb

cobweb_launcher-0.1.9.dist-info/RECORD

@@ -0,0 +1,39 @@
+cobweb/__init__.py,sha256=fIg4v2yl3NHiCSli9EHU1WnMTLqEfDAvOgRtmULwu5A,227
+cobweb/bbb.py,sha256=Sckof5zgzWEz2tIjs6xxoLkrL7wUdogPw3AetlXIDUo,5684
+cobweb/constant.py,sha256=WApaB3mn9cTDzvoCd7UcejO5T5wsCrxzn7D-uBTcIpg,462
+cobweb/decorators.py,sha256=eYQI9rddPVJihAlomLTmbtQhIOzPw8dCrOFpxAq2pLY,318
+cobweb/interface.py,sha256=um_k2AAQl1HTOvfUlq914DjkpfZVwt2m1B65EpPKrmE,802
+cobweb/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
+cobweb/setting.py,sha256=UAu_dLuIFYO98MxtlZ5sZqJcwKAUKq4Bu4KoKlV50Mc,288
+cobweb/task.py,sha256=H9hDK72fSqD7k16exVIXIvCYr1Kvq-pl8PCXtKh8JWA,1704
+cobweb/utils.py,sha256=ivmRqJJNtwdOKYT4G7qQCWnL8ar9c-shxeDZzGB2E9c,2651
+cobweb/db/__init__.py,sha256=jC-uOThYLtiDUG6cTJRkDITgOzR4nIOeaZQeZhuk-v0,139
+cobweb/db/oss_db.py,sha256=lFGNuH3tdIMsohVXQ_fTZPyBfS2oxYNmFNuQ-ZBQgm0,4221
+cobweb/db/redis_db.py,sha256=go9IPQQZAl_jXiHDTruWF7N8svs9QEpQPOr7JhuLCQs,8179
+cobweb/db/scheduler/__init__.py,sha256=w5uIGEB1wLJ-H9RqGpzRwOEWW-BBVSk6Cc7FxZIlWCs,51
+cobweb/db/scheduler/default.py,sha256=XDtxNyu5KTpVAbfCOW8mR1zNFNHiMuaQ4sAhZuIYBoM,79
+cobweb/db/scheduler/textfile.py,sha256=P5pk75DUnbXbLNPOaMIbHh2lbwBGBlv0mitX58yK-MU,786
+cobweb/db/storer/__init__.py,sha256=yWUVyq8JLpuUDPnUC0igw3P8Kkw_FqNi0aAoxkMkRmc,49
+cobweb/db/storer/console.py,sha256=096JTALYuB_I3Qy5TjN40yEPeugO_pmqHN9VJu7wD7Y,153
+cobweb/db/storer/loghub.py,sha256=4ImSIpHPNU7Djp72HlUGOd2h5c9gIxGzBKL1jJ3KPkM,1702
+cobweb/db/storer/redis.py,sha256=7Q2XEQwBL6X_M1uvxzzuSBt6iw9piKw-_FWKm2INZDQ,412
+cobweb/db/storer/textfile.py,sha256=auoXGXLbIbEhMoeYIhy58qw22N2r0fQTtzVjHCjqVGA,386
+cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/distributed/launcher.py,sha256=jTtBXBmna_6yFdj6gyGQiiEtg8I0g5uI5h8kbHWt454,7998
+cobweb/distributed/models.py,sha256=PUQokXMGD-H4A99nX7qYA395Ul6IsWGruMTVa05nswY,4568
+cobweb/equip/__init__.py,sha256=Cjx35k6fGGMeZcxrvtBiBwl8ajOjsJmSR1yvLAZdDys,238
+cobweb/equip/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/equip/distributed/launcher.py,sha256=1LzxibGXWR20XpXawakiRpEMaa9yfaj2rFSKnmEwjFc,7475
+cobweb/equip/distributed/models.py,sha256=pIQSac-WuaKsFGrT1ImOMqYoD2zySTkOK1ZLtHw33d8,4815
+cobweb/equip/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/equip/single/launcher.py,sha256=dMd1IJg__yiFnxy3Q6EueebPllN8zyTDwfC0OjEhRf4,6702
+cobweb/equip/single/models.py,sha256=kf6ZSlcY9XVdDx6WqGahO4uuGCAdHK9ayeJ6Kuu3JiE,4447
+cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/single/launcher.py,sha256=IoJbn87j7t7Pib_FxoWZmmX8asXOqNGb-9ospw6EYJI,7302
+cobweb/single/models.py,sha256=UXcxr_Quok91k82plaqbj4deB-UBCWo14WCo6SS5L_o,4247
+cobweb/single/nest.py,sha256=49K6KQ934INfPrWQsrq9rIFpQauLbLGOFbDaHvoQzOk,5015
+cobweb_launcher-0.1.9.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.1.9.dist-info/METADATA,sha256=wJXLwwIpVNvi5DS_PUgX5wcQyx2pzfq0r0TPQJgN-ak,1219
+cobweb_launcher-0.1.9.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+cobweb_launcher-0.1.9.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.1.9.dist-info/RECORD,,

{cobweb_launcher-0.1.8.dist-info → cobweb_launcher-0.1.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

cobweb_launcher-0.1.8.dist-info/RECORD

@@ -1,31 +0,0 @@
-cobweb/__init__.py,sha256=hEucL3DxmJNXKLwwcSlJrMlwMarlDV4VbILjJVk1s64,326
-cobweb/bbb.py,sha256=Sckof5zgzWEz2tIjs6xxoLkrL7wUdogPw3AetlXIDUo,5684
-cobweb/decorators.py,sha256=8KPSKL8xsiXOLv-kckkaDtK8LXM8d5gaRriGpuEgOQk,320
-cobweb/interface.py,sha256=um_k2AAQl1HTOvfUlq914DjkpfZVwt2m1B65EpPKrmE,802
-cobweb/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
-cobweb/setting.py,sha256=UAu_dLuIFYO98MxtlZ5sZqJcwKAUKq4Bu4KoKlV50Mc,288
-cobweb/task.py,sha256=77F5EaopSVlSX2TANv1lhuPHFI8ER8Jh4tSGrwDWAc0,1405
-cobweb/utils.py,sha256=ivmRqJJNtwdOKYT4G7qQCWnL8ar9c-shxeDZzGB2E9c,2651
-cobweb/db/__init__.py,sha256=4m9lqmxZCRbaih3Z3rl_BT0GugMd0dkOIgu_P9aeC84,63
-cobweb/db/oss_db.py,sha256=lFGNuH3tdIMsohVXQ_fTZPyBfS2oxYNmFNuQ-ZBQgm0,4221
-cobweb/db/redis_db.py,sha256=yoWy-GI0rjVmT-68Che-pypfqNwNti5JGkc9bYvJH2o,8202
-cobweb/db/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/db/scheduler/default.py,sha256=OxmFX7OvMEhKEq-NF7A8I9cA4V4qWw5vayS-yIbng0A,114
-cobweb/db/scheduler/textfile.py,sha256=atRDeNT-e5toNvyGsCXAxL1FJi77uSYktdCzH_hXGo8,821
-cobweb/db/storer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/db/storer/console.py,sha256=f7yZFo4qTieaB9JxbGfrVAclAb2H_wji82dWoZp7HUw,182
-cobweb/db/storer/loghub.py,sha256=4VqZacXWhidzINHXQu2_-E0HOBRCcc86f6LkKfnXD5I,1731
-cobweb/db/storer/redis.py,sha256=7Q2XEQwBL6X_M1uvxzzuSBt6iw9piKw-_FWKm2INZDQ,412
-cobweb/db/storer/textfile.py,sha256=3mDHMvF6Sh5fn3IHzWQxyTUd45V-zUoH8vY3EoRlMx0,415
-cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/distributed/launcher.py,sha256=jTtBXBmna_6yFdj6gyGQiiEtg8I0g5uI5h8kbHWt454,7998
-cobweb/distributed/models.py,sha256=PUQokXMGD-H4A99nX7qYA395Ul6IsWGruMTVa05nswY,4568
-cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cobweb/single/launcher.py,sha256=IoJbn87j7t7Pib_FxoWZmmX8asXOqNGb-9ospw6EYJI,7302
-cobweb/single/models.py,sha256=UXcxr_Quok91k82plaqbj4deB-UBCWo14WCo6SS5L_o,4247
-cobweb/single/nest.py,sha256=mL8q9a5BjtoeUyzXCIVw_vyUsNY8ltbvQpYIIpZEDFU,5012
-cobweb_launcher-0.1.8.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-0.1.8.dist-info/METADATA,sha256=9NTzvHe-pPNFJdbyGuE9SRqvtKQd6Kar1WG_GwUS4Ss,1225
-cobweb_launcher-0.1.8.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-cobweb_launcher-0.1.8.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-0.1.8.dist-info/RECORD,,