cobweb-launcher 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

cobweb/__init__.py

@@ -1,2 +1,2 @@
-from .launchers import LauncherAir, LauncherPro, LauncherApi
+from .launchers import LauncherPro, LauncherApi
 from .constant import CrawlerModel

cobweb/base/__init__.py

@@ -66,11 +66,17 @@ class TaskQueue:
         except Exception as e:
             it.params.retry += 1
             if isinstance(it, Request):
-                TaskQueue.REQUEST.push(it)
+                TaskQueue.DOWNLOAD.push(it)
             elif isinstance(it, Response):
                 TaskQueue.RESPONSE.push(it)
             elif isinstance(it, Seed):
-                TaskQueue.SEED.push(it)
+                TaskQueue.TODO.push(it)
+            elif isinstance(it, BaseItem):
+                TaskQueue.UPLOAD.push(it)
+            logger.info(
+                f"{crawler_func.__name__} failed: "
+                f"{''.join(traceback.format_exception(type(e), e, e.__traceback__))}"
+            )
             time.sleep(1)
 
 
@@ -95,7 +101,7 @@ class Decorators:
         def wrapper(self, *args, **kwargs):
             while not self.pause.is_set():
                 try:
-                    func(self, *args, **kwargs)
+                    func(self)
                 except Exception as e:
                     logger.info(f"{func.__name__}: " + str(e))
                 finally:

cobweb/base/basic.py

@@ -15,11 +15,21 @@ class Params:
 
 
 class Seed:
+    __SEED_PARAMS__ = [
+        "retry",
+        "priority",
+        "version",
+        "status"
+    ]
 
     def __init__(
             self,
             seed,
-            params = Params(),
+            sid=None,
+            retry=None,
+            priority=None,
+            version=None,
+            status=None,
             **kwargs
     ):
         if any(isinstance(seed, t) for t in (str, bytes)):
@@ -37,11 +47,27 @@ class Seed:
                 f"seed: {seed}"
             ))
 
+        seed_params = {
+            "retry": retry,
+            "priority": priority,
+            "version": version,
+            "status": status,
+        }
+
         if kwargs:
+            # for k, v in kwargs.items():
+            #     if k in seed_params.keys():
+            #         seed_params[k] = v
+            #     else:
+            #         self.__setattr__(k, v)
             self._init_seed(kwargs)
-        if not getattr(self, "sid", None):
-            self._init_id()
-        self.params = params or Params()
+            seed_params.update({
+                k: v for k, v in kwargs.items()
+                if k in self.__SEED_PARAMS__
+            })
+        if sid or not getattr(self, "sid", None):
+            self._init_id(sid)
+        self.params = Params(**seed_params)
 
     def __getattr__(self, name):
         return None
@@ -59,13 +85,14 @@ class Seed:
         chars = [f"{k}={v}" for k, v in self.__dict__.items()]
         return f'{self.__class__.__name__}({", ".join(chars)})'
 
-    def _init_seed(self, seed_info:dict):
+    def _init_seed(self, seed_info: dict):
         for k, v in seed_info.items():
             if k not in self.__SEED_PARAMS__:
                 self.__setattr__(k, v)
 
-    def _init_id(self):
-        sid = hashlib.md5(self.to_string.encode()).hexdigest()
+    def _init_id(self, sid):
+        if not sid:
+            sid = hashlib.md5(self.to_string.encode()).hexdigest()
         self.__setattr__("sid", sid)
 
     @property
@@ -85,16 +112,15 @@ class Seed:
 
     @property
     def seed(self):
-        return self
+        return self.to_string
 
 
 class Request:
-
     __SEED_PARAMS__ = [
         "retry",
         "priority",
-        "seed_version",
-        "seed_status"
+        "version",
+        "status"
     ]
 
     __REQUEST_ATTRS__ = {
@@ -122,8 +148,8 @@ class Request:
             check_status_code=True,
             retry=None,
             priority=None,
-            seed_version=None,
-            seed_status=None,
+            version=None,
+            status=None,
             **kwargs
     ):
         self.url = url
@@ -133,10 +159,15 @@ class Request:
         seed_params = {
             "retry": retry,
             "priority": priority,
-            "seed_version": seed_version,
-            "seed_status": seed_status,
+            "version": version,
+            "status": status,
         }
 
+        if isinstance(seed, Seed):
+            kwargs.update(**seed.to_dict)
+        elif isinstance(seed, str):
+            kwargs.update(**json.loads(seed))
+
         for k, v in kwargs.items():
             if k in self.__class__.__REQUEST_ATTRS__:
                 self.request_setting[k] = v
@@ -152,12 +183,7 @@ class Request:
             self._build_header()
 
         self.params = Params(**seed_params)
-
-        if isinstance(seed, Seed):
-            kwargs.update(**seed.to_dict)
-        elif isinstance(seed, str):
-            kwargs.update(**json.loads(seed))
-        self.seed = self.to_string
+        # self.seed = self.to_string
 
     @property
     def _random_ua(self) -> str:
@@ -183,10 +209,19 @@ class Request:
             response.raise_for_status()
         return response
 
+    def __getattr__(self, name):
+        return None
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
     @property
     def to_dict(self):
         _dict = self.__dict__.copy()
-        _dict.pop('seed')
+        # _dict.pop('seed')
         _dict.pop('params')
         _dict.pop('check_status_code')
         # _dict.pop('request_setting')
@@ -200,6 +235,10 @@ class Request:
             separators=(",", ":")
         )
 
+    @property
+    def seed(self):
+        return self.to_string
+
 
 class Response:
 
@@ -209,8 +248,8 @@ class Response:
             response,
             retry=None,
             priority=None,
-            seed_version=None,
-            seed_status=None,
+            version=None,
+            status=None,
             **kwargs
     ):
         self.seed = seed
@@ -218,20 +257,24 @@ class Response:
         seed_params = {
             "retry": retry,
             "priority": priority,
-            "seed_version": seed_version,
-            "seed_status": seed_status,
+            "version": version,
+            "status": status,
         }
         for k, v in kwargs.items():
             if k in seed_params.keys():
                 seed_params[k] = v
             else:
                 self.__setattr__(k, v)
+        self.params = Params(**seed_params)
 
     @property
     def to_dict(self):
         _dict = self.__dict__.copy()
         _dict.pop('seed')
         _dict.pop('response')
+        _dict.pop('method')
+        _dict.pop('params')
+        _dict.pop('request_setting')
         return _dict
 
     @property
@@ -241,3 +284,12 @@ class Response:
             ensure_ascii=False,
             separators=(",", ":")
         )
+
+    def __getattr__(self, name):
+        return None
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+    def __getitem__(self, item):
+        return getattr(self, item)

cobweb/crawlers/crawler.py

@@ -3,7 +3,7 @@ import time
 import threading
 from typing import Union, Callable, Mapping
 
-import setting
+from cobweb import setting
 from cobweb.base import (
     Seed,
     BaseItem,
@@ -14,7 +14,7 @@ from cobweb.base import (
     TaskQueue,
     logger
 )
-from constant import DealModel
+from cobweb.constant import DealModel
 
 
 class Crawler(threading.Thread):
@@ -66,8 +66,8 @@ class Crawler(threading.Thread):
     def build_download_item(self):
         thread_sleep = 0.1
         if TaskQueue.RESPONSE.length >= self.download_queue_size:
-            logger.info(f"download queue is full, sleep {thread_sleep}s")
             thread_sleep = 5
+            # logger.info(f"download queue is full, sleep {thread_sleep}s")
         elif request_info := TaskQueue.DOWNLOAD.pop():
             member, priority = request_info
             request_setting = json.loads(member)
@@ -79,7 +79,7 @@ class Crawler(threading.Thread):
     def build_parse_item(self):
         thread_sleep = 0.1
         if TaskQueue.UPLOAD.length >= self.upload_queue_size:
-            logger.info(f"upload queue is full, sleep {thread_sleep}s")
+            # logger.info(f"upload queue is full, sleep {thread_sleep}s")
             thread_sleep = 5
         if response_item := TaskQueue.RESPONSE.pop():
             TaskQueue.process_task(response_item, self.parse)

cobweb/launchers/launcher.py

@@ -137,10 +137,10 @@ class Launcher(threading.Thread):
 
     def _add_thread(self, func, num=1, obj=None, name=None, args=()):
         obj = obj or self
-        name = obj.__class__.__name__ + name or func.__name__
+        name = obj.__class__.__name__ + ":" + (name or func.__name__)
         for i in range(num):
             func_name = name + "_" + str(i) if num > 1 else name
-            self._threads.append(threading.Thread(name=func_name, target=func, args=(obj,) + args))
+            self._threads.append(threading.Thread(name=func_name, target=func, args=()))
 
     @Decorators.stop
     def _polling(self):
@@ -150,6 +150,10 @@ class Launcher(threading.Thread):
                 if not self.task_model and run_time > self.before_scheduler_wait_seconds:
                     logger.info("Done! ready to close thread...")
                     self.stop.set()
+                elif TaskQueue.TODO.length or TaskQueue.DOWNLOAD.length:
+                    logger.info(f"Recovery {self.task} task run！")
+                    self.check_emtpy_times = 0
+                    self.pause.clear()
                 else:
                     logger.info("pause! waiting for resume...")
             elif self.check_emtpy_times > 2:
@@ -164,10 +168,6 @@ class Launcher(threading.Thread):
                     f"reset times {3 - self.check_emtpy_times}"
                 )
                 self.check_emtpy_times += 1
-        elif TaskQueue.TODO.length:
-            logger.info(f"Recovery {self.task} task run！")
-            self.check_emtpy_times = 0
-            self.pause.clear()
         else:
             logger.info(LogTemplate.launcher_polling.format(
                 task=self.task,
@@ -179,6 +179,8 @@ class Launcher(threading.Thread):
                 response_queue_len=TaskQueue.RESPONSE.length,
                 done_queue_len=TaskQueue.DONE.length,
                 upload_queue_len=TaskQueue.UPLOAD.length,
+                seed_queue_len=TaskQueue.SEED.length,
+                download_queue_len=TaskQueue.DOWNLOAD.length
             ))
         time.sleep(10)
 

cobweb/launchers/launcher_air.py

@@ -1,88 +1,88 @@
-import time
-
-from cobweb.base import logger
-from cobweb.constant import LogTemplate
-from .launcher import Launcher, check_pause
-
-
-class LauncherAir(Launcher):
-
-    # def _scheduler(self):
-    #     if self.start_seeds:
-    #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
-
-    @check_pause
-    def _insert(self):
-        seeds = {}
-        status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
-        for _ in range(self._new_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['new'].pop()
-            if not seed:
-                break
-            seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self.__LAUNCHER_QUEUE__['todo'].push(seeds)
-        if status:
-            time.sleep(self._new_queue_wait_seconds)
-
-    @check_pause
-    def _delete(self):
-        seeds = []
-        status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
-
-        for _ in range(self._done_queue_max_size):
-            seed = self.__LAUNCHER_QUEUE__['done'].pop()
-            if not seed:
-                break
-            seeds.append(seed.to_string)
-
-        if seeds:
-            self._remove_doing_seeds(seeds)
-
-        if status:
-            time.sleep(self._done_queue_wait_seconds)
-
-    def _polling(self):
-
-        check_emtpy_times = 0
-
-        while not self._stop.is_set():
-
-            queue_not_empty_count = 0
-            pooling_wait_seconds = 30
-
-            for q in self.__LAUNCHER_QUEUE__.values():
-                if q.length != 0:
-                    queue_not_empty_count += 1
-
-            if queue_not_empty_count == 0:
-                pooling_wait_seconds = 3
-                if self._pause.is_set():
-                    check_emtpy_times = 0
-                    if not self._task_model:
-                        logger.info("Done! Ready to close thread...")
-                        self._stop.set()
-                elif check_emtpy_times > 2:
-                    self.__DOING__ = {}
-                    self._pause.set()
-                else:
-                    logger.info(
-                        "check whether the task is complete, "
-                        f"reset times {3 - check_emtpy_times}"
-                    )
-                    check_emtpy_times += 1
-            elif self._pause.is_set():
-                self._pause.clear()
-                self._execute()
-            else:
-                logger.info(LogTemplate.launcher_air_polling.format(
-                    task=self.task,
-                    doing_len=len(self.__DOING__.keys()),
-                    todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
-                    done_len=self.__LAUNCHER_QUEUE__['done'].length,
-                    upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
-                ))
-
-            time.sleep(pooling_wait_seconds)
-
-
+# import time
+#
+# from cobweb.base import logger
+# from cobweb.constant import LogTemplate
+# from .launcher import Launcher, check_pause
+#
+#
+# class LauncherAir(Launcher):
+#
+#     # def _scheduler(self):
+#     #     if self.start_seeds:
+#     #         self.__LAUNCHER_QUEUE__['todo'].push(self.start_seeds)
+#
+#     @check_pause
+#     def _insert(self):
+#         seeds = {}
+#         status = self.__LAUNCHER_QUEUE__['new'].length < self._new_queue_max_size
+#         for _ in range(self._new_queue_max_size):
+#             seed = self.__LAUNCHER_QUEUE__['new'].pop()
+#             if not seed:
+#                 break
+#             seeds[seed.to_string] = seed.params.priority
+#         if seeds:
+#             self.__LAUNCHER_QUEUE__['todo'].push(seeds)
+#         if status:
+#             time.sleep(self._new_queue_wait_seconds)
+#
+#     @check_pause
+#     def _delete(self):
+#         seeds = []
+#         status = self.__LAUNCHER_QUEUE__['done'].length < self._done_queue_max_size
+#
+#         for _ in range(self._done_queue_max_size):
+#             seed = self.__LAUNCHER_QUEUE__['done'].pop()
+#             if not seed:
+#                 break
+#             seeds.append(seed.to_string)
+#
+#         if seeds:
+#             self._remove_doing_seeds(seeds)
+#
+#         if status:
+#             time.sleep(self._done_queue_wait_seconds)
+#
+#     def _polling(self):
+#
+#         check_emtpy_times = 0
+#
+#         while not self._stop.is_set():
+#
+#             queue_not_empty_count = 0
+#             pooling_wait_seconds = 30
+#
+#             for q in self.__LAUNCHER_QUEUE__.values():
+#                 if q.length != 0:
+#                     queue_not_empty_count += 1
+#
+#             if queue_not_empty_count == 0:
+#                 pooling_wait_seconds = 3
+#                 if self._pause.is_set():
+#                     check_emtpy_times = 0
+#                     if not self._task_model:
+#                         logger.info("Done! Ready to close thread...")
+#                         self._stop.set()
+#                 elif check_emtpy_times > 2:
+#                     self.__DOING__ = {}
+#                     self._pause.set()
+#                 else:
+#                     logger.info(
+#                         "check whether the task is complete, "
+#                         f"reset times {3 - check_emtpy_times}"
+#                     )
+#                     check_emtpy_times += 1
+#             elif self._pause.is_set():
+#                 self._pause.clear()
+#                 self._execute()
+#             else:
+#                 logger.info(LogTemplate.launcher_air_polling.format(
+#                     task=self.task,
+#                     doing_len=len(self.__DOING__.keys()),
+#                     todo_len=self.__LAUNCHER_QUEUE__['todo'].length,
+#                     done_len=self.__LAUNCHER_QUEUE__['done'].length,
+#                     upload_len=self.__LAUNCHER_QUEUE__['upload'].length,
+#                 ))
+#
+#             time.sleep(pooling_wait_seconds)
+#
+#

cobweb/launchers/launcher_api.py

@@ -1,161 +1,88 @@
 import time
-import threading
 
-from cobweb.db import ApiDB
-from cobweb.base import Seed, TaskQueue,logger, stop, pause
-from cobweb.constant import DealModel
+from cobweb.base import TaskQueue, Decorators
+from cobweb.schedulers import ApiScheduler
 from .launcher import Launcher
 
 
-class LauncherApi(Launcher):
+class LauncherPro(Launcher):
 
     def __init__(self, task, project, custom_setting=None, **kwargs):
         super().__init__(task, project, custom_setting, **kwargs)
-        self._db = ApiDB()
-
-        self._todo_key = "{%s:%s}:todo" % (project, task)
-        self._done_key = "{%s:%s}:done" % (project, task)
-        self._fail_key = "{%s:%s}:fail" % (project, task)
-        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
-
-        self._statistics_done_key = "statistics:%s:%s:done" % (project, task)
-        self._statistics_fail_key = "statistics:%s:%s:fail" % (project, task)
-        self._speed_control_key = "speed_control:%s_%s" % (project, task)
-
-        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
-
-        self._heartbeat_start_event = threading.Event()
-
-    @property
-    def heartbeat(self):
-        return self._db.exists(self._heartbeat_key)
-
-    def statistics(self, key, count):
-        if not self.task_model and not self._db.exists(key):
-            self._db.setex(key, 86400 * 30, int(count))
-        else:
-            self._db.incrby(key, count)
-
-    def _get_seed(self) -> Seed:
-        """
-        从队列中获取种子（频控）
-        设置时间窗口为self._time_window（秒），判断在该窗口内的采集量是否满足阈值（self._spider_max_speed）
-        :return: True -> 种子, False -> None
-        """
-        if TaskQueue.TODO.length and not self._db.auto_incr(
-                self._speed_control_key, 
-                t=self.time_window,
-                limit=self.spider_max_count
-        ):
-            expire_time = self._db.ttl(self._speed_control_key)
-            logger.info(f"Too fast! Please wait {expire_time} seconds...")
-            time.sleep(expire_time / 2)
-            return None
-        return TaskQueue.TODO.pop()
+        self._redis_download = "{%s:%s}:download" % (project, task)
+        self._redis_todo = "{%s:%s}:todo" % (project, task)
+        self._scheduler = ApiScheduler(task, project)
+
+    @Decorators.stop
+    def _schedule(self):
+        thread_sleep = self.scheduling_wait_time
+        for q, key, size, item_info in [
+            (TaskQueue.TODO, self._redis_todo, self.todo_queue_size, self._task_info["todo"]),
+            (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"]),
+        ]:
+            if q.length < size:
+                for member, priority in self._scheduler.schedule(key, self.scheduling_size):
+                    q.push((member, priority), direct_insertion=True)
+                    self.add_working_item(key.split(":")[-1], member, priority)
+                thread_sleep = 0.1
+        time.sleep(thread_sleep)
+
+    @Decorators.stop
+    def _heartbeat(self):
+        if self._scheduler.working.is_set():
+            self._scheduler.set_heartbeat()
+        time.sleep(3)
 
-    @stop
+    @Decorators.stop
     def _reset(self):
-        """
-        检查过期种子，重新添加到redis缓存中
-        """
-        if self._db.lock(self._reset_lock_key, t=120):
-
-            _min = -int(time.time()) + self.seed_reset_seconds \
-                if self.heartbeat else "-inf"
-
-            self._db.members(self._todo_key, 0, _min=_min, _max="(0")
-
-            if not self.heartbeat:
-                self._heartbeat_start_event.set()
-
-            self._db.delete(self._reset_lock_key)
-
+        self._scheduler.reset(
+            keys=[self._redis_todo, self._redis_download],
+            reset_time=self.seed_reset_seconds
+        )
         time.sleep(30)
 
-    @stop
-    def _refresh(self):
-        """
-        刷新doing种子过期时间，防止reset重新消费
-        """
-        if self.doing_seeds:
-            refresh_time = int(time.time())
-            seeds = {k: -refresh_time - v / 1e3 for k, v in self.doing_seeds.items()}
-            self._db.zadd(self._todo_key, item=seeds, xx=True)
-        time.sleep(3)
-
-    @stop
-    def _scheduler(self):
-        """
-        调度任务，获取redis队列种子，同时添加到doing字典中
-        """
-        if not self._db.zcount(self._todo_key, 0, "(1000"):
-            time.sleep(self.scheduler_wait_seconds)
-        elif TaskQueue.TODO.length >= self.todo_queue_size:
-            time.sleep(self.todo_queue_full_wait_seconds)
-        else:
-            members = self._db.members(
-                self._todo_key, int(time.time()),
-                count=self.todo_queue_size,
-                _min=0, _max="(1000"
-            )
-            for member, priority in members:
-                seed = Seed(member, priority=priority)
-                TaskQueue.TODO.push(seed)
-                self.doing_seeds[seed.to_string] = seed.params.priority
-
-    @pause
-    def _heartbeat(self):
-        if self._heartbeat_start_event.is_set():
-            self._db.setex(self._heartbeat_key, t=5)
-        time.sleep(3)
-
-    @pause
+    @Decorators.pause
     def _insert(self):
-        """
-        添加新种子到redis队列中
-        """
-        seeds = {}
-        for _ in range(self.new_queue_max_size):
-            if seed := TaskQueue.SEED.pop():
-                seeds[seed.to_string] = seed.params.priority
-        if seeds:
-            self._db.zadd(self._todo_key, seeds, nx=True)
-        if TaskQueue.SEED.length < self.new_queue_max_size:
-            time.sleep(self.new_queue_wait_seconds)
-
-    @pause
-    def _delete(self):
-        """
-        删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
-        """
-        seed_info = {"count": 0, "failed": [], "succeed": [], "common": []}
-        status = TaskQueue.DONE.length < self.done_queue_max_size
-
-        for _ in range(self.done_queue_max_size):
-            seed = TaskQueue.DONE.pop()
-            if not seed:
-                break
-            if seed.params.seed_status == DealModel.fail:
-                seed_info["failed"].append(seed.to_string)
-            elif self.done_model == 1:
-                seed_info["succeed"].append(seed.to_string)
-            else:
-                seed_info["common"].append(seed.to_string)
-            seed_info['count'] += 1
-
-        if seed_info["count"]:
-
-            succeed_count = int(self._db.zrem(self._todo_key, *seed_info["common"]) or 0)
-            succeed_count += int(self._db.done([self._todo_key, self._done_key], *seed_info["succeed"]) or 0)
-            failed_count = int(self._db.done([self._todo_key, self._fail_key], *seed_info["failed"]) or 0)
-
-            if failed_count:
-                self.statistics(self._statistics_fail_key, failed_count)
-            if succeed_count:
-                self.statistics(self._statistics_done_key, succeed_count)
-
-            self._remove_doing_seeds(seed_info["common"] + seed_info["succeed"] + seed_info["failed"])
-
-        if status:
-            time.sleep(self.done_queue_wait_seconds)
-
+        thread_sleep = 0.1
+        for q, key, size in [
+            (TaskQueue.SEED, self._redis_todo, self.seed_queue_size),
+            (TaskQueue.REQUEST, self._redis_download, self.request_queue_size),
+        ]:
+            item_info = {}
+            while (item := q.pop()) and len(item_info.keys()) < self.inserting_size:
+                item_info[item.seed] = item.params.priority
+            if q.length >= size:
+                thread_sleep = self.inserting_wait_time
+            self._scheduler.insert(key, item_info)
+        time.sleep(thread_sleep)
+
+    @Decorators.pause
+    def _refresh(self):
+        self._scheduler.refresh(self._redis_todo, self._task_info["todo"])
+        self._scheduler.refresh(self._redis_download, self._task_info["download"])
+        time.sleep(10)
+
+    @Decorators.pause
+    def _remove(self):
+        thread_sleep = self.removing_wait_time
+        for q, key, size in [
+            (TaskQueue.DELETE, self._redis_todo, self.delete_queue_size),
+            (TaskQueue.DONE, self._redis_download, self.done_queue_size),
+        ]:
+            items = []
+            while (item := q.pop()) and len(items) < self.removing_size:
+                items.append(item)
+            self._scheduler.delete(key, items)
+            self.remove_working_items(key.split(":")[-1], items)
+            if q.length >= size:
+                thread_sleep = 0.1
+        time.sleep(thread_sleep)
+
+    def _init_schedule_thread(self):
+        self._add_thread(func=self._heartbeat)
+        self._add_thread(func=self._reset)
+        self._add_thread(func=self._refresh)
+        self._add_thread(func=self._schedule)
+        self._add_thread(func=self._insert)
+        self._add_thread(func=self._remove)
+        # self._add_thread(func=self._polling)

cobweb/launchers/launcher_pro.py

@@ -21,21 +21,19 @@ class LauncherPro(Launcher):
             (TaskQueue.DOWNLOAD, self._redis_download, self.download_queue_size, self._task_info["download"]),
         ]:
             if q.length < size:
-                for member, priority in self._scheduler.schedule(
-                    key, self.scheduling_size
-                ):
+                for member, priority in self._scheduler.schedule(key, self.scheduling_size):
                     q.push((member, priority), direct_insertion=True)
                     self.add_working_item(key.split(":")[-1], member, priority)
                 thread_sleep = 0.1
         time.sleep(thread_sleep)
 
-    @Decorators.pause
+    @Decorators.stop
     def _heartbeat(self):
         if self._scheduler.working.is_set():
             self._scheduler.set_heartbeat()
         time.sleep(3)
 
-    @Decorators.pause
+    @Decorators.stop
     def _reset(self):
         self._scheduler.reset(
             keys=[self._redis_todo, self._redis_download],
@@ -51,7 +49,7 @@ class LauncherPro(Launcher):
             (TaskQueue.REQUEST, self._redis_download, self.request_queue_size),
         ]:
             item_info = {}
-            while item := q.pop() and len(item_info.keys()) < self.inserting_size:
+            while (item := q.pop()) and len(item_info.keys()) < self.inserting_size:
                 item_info[item.seed] = item.params.priority
             if q.length >= size:
                 thread_sleep = self.inserting_wait_time
@@ -72,9 +70,9 @@ class LauncherPro(Launcher):
             (TaskQueue.DONE, self._redis_download, self.done_queue_size),
         ]:
             items = []
-            while item := q.pop() and len(items) < self.removing_size:
+            while (item := q.pop()) and len(items) < self.removing_size:
                 items.append(item)
-            self._scheduler.delete(key, *items)
+            self._scheduler.delete(key, items)
             self.remove_working_items(key.split(":")[-1], items)
             if q.length >= size:
                 thread_sleep = 0.1
@@ -87,4 +85,4 @@ class LauncherPro(Launcher):
         self._add_thread(func=self._schedule)
         self._add_thread(func=self._insert)
         self._add_thread(func=self._remove)
-        self._add_thread(func=self._polling)
+        # self._add_thread(func=self._polling)

cobweb/pipelines/pipeline.py

@@ -30,7 +30,7 @@ class Pipeline(ABC):
         data_info, seeds = {}, []
         thread_sleep = self.upload_wait_time if TaskQueue.UPLOAD.length < self.upload_queue_size else 0.1
         try:
-            while item := TaskQueue.UPLOAD.pop() and len(seeds) <= self.upload_queue_size:
+            while (item := TaskQueue.UPLOAD.pop()) and len(seeds) <= self.upload_queue_size:
                 data = self.build(item)
                 data_info.setdefault(item.table, []).append(data)
                 seeds.append(item.seed)
@@ -39,6 +39,7 @@ class Pipeline(ABC):
         except Exception as e:
             logger.info(e)
             seeds = None
+            # todo: retry
         finally:
             TaskQueue.DONE.push(seeds)
 

cobweb/schedulers/__init__.py

@@ -1 +1,3 @@
 from .scheduler_redis import RedisScheduler
+from .scheduler_api import ApiScheduler
+

cobweb/schedulers/scheduler_api.py

@@ -0,0 +1,69 @@
+import threading
+import time
+
+# from cobweb.base import Seed
+from cobweb.db import ApiDB
+
+
+class ApiScheduler:
+
+    def __init__(self, task, project, scheduler_wait_seconds=30):
+        self._todo_key = "{%s:%s}:todo" % (project, task)
+        self._download_key = "{%s:%s}:download" % (project, task)
+        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
+        self._speed_control_key = "speed_control:%s_%s" % (project, task)
+        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
+        self._db = ApiDB()
+
+        self.scheduler_wait_seconds = scheduler_wait_seconds
+        self.working = threading.Event()
+
+    @property
+    def heartbeat(self):
+        return self._db.exists(self._heartbeat_key)
+
+    def set_heartbeat(self):
+        return self._db.setex(self._heartbeat_key, 5)
+
+    def schedule(self, key, count):
+        if not self._db.zcount(key, 0, "(1000"):
+            time.sleep(self.scheduler_wait_seconds)
+        else:
+            source = int(time.time())
+            members = self._db.members(key, source, count=count, _min=0, _max="(1000")
+            for member, priority in members:
+                # seed = Seed(member, priority=priority)
+                yield member.decode(), priority
+
+    def insert(self, key, items):
+        if items:
+            self._db.zadd(key, items, nx=True)
+
+    def reset(self, keys, reset_time=30):
+        if self._db.lock(self._reset_lock_key, t=120):
+
+            if isinstance(keys, str):
+                keys = [keys]
+
+            _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
+
+            for key in keys:
+                self._db.members(key, 0, _min=_min, _max="(0")
+
+            if not self.heartbeat:
+                self.working.set()
+                time.sleep(10)
+
+            self._db.delete(self._reset_lock_key)
+
+    def refresh(self, key, items: dict[str, int]):
+        refresh_time = int(time.time())
+        its = {k: -refresh_time - v / 1000 for k, v in items}
+        self._db.zadd(key, item=its, xx=True)
+
+    def delete(self, key, values):
+        self._db.zrem(key, *values)
+
+
+
+

{cobweb_launcher-1.3.2.dist-info → cobweb_launcher-1.3.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 1.3.2
+Version: 1.3.4
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-1.3.2.dist-info → cobweb_launcher-1.3.4.dist-info}/RECORD

@@ -1,8 +1,8 @@
-cobweb/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
+cobweb/__init__.py,sha256=oaEfsGUuGP0s39UbFRwrnsjMUeuB6QvQIAwStKFyUTk,83
 cobweb/constant.py,sha256=eofONAntk9O6S-cb4KbYGYHL_u7nBlOqqFOw_HzJHAU,3588
 cobweb/setting.py,sha256=pY6LKsgWI3164GiGA1z_y26LVf5-3mpiEgmm86mKRdY,3135
-cobweb/base/__init__.py,sha256=yrf__ULml-wN2CuLDgu7o1slbFhiSJwvOu_dsYstPLU,4849
-cobweb/base/basic.py,sha256=ukWBLDdBgJhV15pO13ZxcYYXVLmnWFctOyHuDvZQh4M,6390
+cobweb/base/__init__.py,sha256=Na385Hhl9l2S8aPhcdJVPjmb02wkVM969bWQ84bCSQs,5095
+cobweb/base/basic.py,sha256=s5G4LBZiLUfoymV-gLSIqeH-OJ7q7-L35sBa6xEH3EI,7666
 cobweb/base/common_queue.py,sha256=Gor7sR3h1hlZWaI0XcNAbf0S15Ftjr3DFRWNTGL13uU,1137
 cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
 cobweb/base/dotting.py,sha256=0SH8F2uAGWZjfODpTAXngYHz8JgfCm-RqpmQbfQ3NCY,1233
@@ -13,7 +13,7 @@ cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
 cobweb/base/seed.py,sha256=PN5J4gKPEXylwyQeSGOBfauxHktxFr7RJe8nVX1hBw4,2987
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=rdXLgY91kKDSjOvDml3CUqJTR9RFifcLEYNiF6-gXEs,2964
+cobweb/crawlers/crawler.py,sha256=ZQ6yVA1EaQRdKJEY3DNqShzp9HPMwlSXapnsRW9E5Wc,2987
 cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
 cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
@@ -21,17 +21,18 @@ cobweb/db/redis_db.py,sha256=FvMzckJtmhwKhZqKoS23iXmJti5P2dnMVD5rJ__5LUw,5139
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=uzfPkLbY2m0wsIR_s93VFxmO0U49GgUG7hXPzMYdye0,118
-cobweb/launchers/launcher.py,sha256=0BCXzZ05PFB6aQejwVBuMPwoxhlc2BLYL541HO5yiYM,7238
-cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_api.py,sha256=qPazoC7U-UmgebbiTkhl6f4yQmN34XMl6HawekhAhEo,5789
-cobweb/launchers/launcher_pro.py,sha256=np6cY7UmW7-649mORdsCvbLy2_mkoXhNZ3OLtev555I,3409
+cobweb/launchers/launcher.py,sha256=NFwpc_0Um0hbDm1A8glWA4fcW6mNYL1eon4t3JAQUlw,7411
+cobweb/launchers/launcher_air.py,sha256=yPr395HVIIHAq6lqRcYJu7c0KkfO9V8O-2sn0hC96p0,2990
+cobweb/launchers/launcher_api.py,sha256=c0bnnZQCqkk_cX-WyFsjc6jpliCwZCuAJeGAvUATODk,3370
+cobweb/launchers/launcher_pro.py,sha256=2H-TcvQx-ga78GLNTa-GXMLYAj9nEeCJSWf8xl-1ISQ,3374
 cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
 cobweb/pipelines/base_pipeline.py,sha256=fYnWf79GmhufXpcnMa3te18SbmnVeYLwxfyo-zLd9CY,1577
 cobweb/pipelines/loghub_pipeline.py,sha256=cjPO6w6UJ0jNw2fVvdX0BCdlm58T7dmYXlxzXOBpvfY,1027
-cobweb/pipelines/pipeline.py,sha256=FTl2JOoF3ysWa-mxb1pbmayEM4IXQH5a4L5CymlBQNU,1293
+cobweb/pipelines/pipeline.py,sha256=Pycm22bHId9a3gdP81D5y7SsuMndYooTb5n4zQxP7dM,1321
 cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
 cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
-cobweb/schedulers/__init__.py,sha256=WJgBXsPE8zpJQ9L_-bHIUpBcaB2G4HmutDdWF3ud1Bs,44
+cobweb/schedulers/__init__.py,sha256=y7Lv_7b0zfTl0OhIONb_8u1K1C9gVlBA-xz_XG_kI9g,85
+cobweb/schedulers/scheduler_api.py,sha256=mC54QOS0PEu4SFvxfD5Qr9239hAxwMrKTg-33rirANE,2112
 cobweb/schedulers/scheduler_redis.py,sha256=Aw7de0sXigRAxJgqUhHWu30hMBzgEWjkj-3OXXqmldg,2118
 cobweb/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
 cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
@@ -103,8 +104,8 @@ cobweb_new/utils/__init__.py,sha256=c9macpjc15hrCUCdzO5RR_sgK_B9kvJKreSGprZ1ld4,
 cobweb_new/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
 cobweb_new/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
 cobweb_new/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.3.2.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.3.2.dist-info/METADATA,sha256=bnweVxLU9zIGyMqQ7mvKdMvzTYjVTIhJ1ei0tm35SOg,6509
-cobweb_launcher-1.3.2.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.3.2.dist-info/top_level.txt,sha256=A0GPGeX6QtxXg7AJno3SVRTHtVCCqeRIOrpwDoXg9qs,15
-cobweb_launcher-1.3.2.dist-info/RECORD,,
+cobweb_launcher-1.3.4.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-1.3.4.dist-info/METADATA,sha256=AMoOPhP8ILf0uUUBrQUpn3_-S3qF-_-ef-_DSPQgJZA,6509
+cobweb_launcher-1.3.4.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-1.3.4.dist-info/top_level.txt,sha256=A0GPGeX6QtxXg7AJno3SVRTHtVCCqeRIOrpwDoXg9qs,15
+cobweb_launcher-1.3.4.dist-info/RECORD,,