cobweb-launcher 3.1.22__py3-none-any.whl → 3.1.23__py3-none-any.whl

cobweb/base/__init__.py

@@ -4,4 +4,6 @@ from .response import Response
 from .request import Request
 from .logger import logger
 from .seed import Seed
+from .task_queue import TaskQueue, Status
+
 

cobweb/base/task_queue.py

@@ -0,0 +1,179 @@
+import time
+import threading
+from enum import Enum
+from hashlib import md5
+from dataclasses import dataclass
+from typing import Dict, Any, Optional, List
+
+
+class Status(Enum):
+    PENDING = 0     # 待处理
+    PROCESSING = 1  # 处理中
+    FINISHED = 2    # 已完成
+    INSERT = 3      # 失败
+    UPLOAD = 4     # 过期
+
+
+@dataclass
+class Task:
+    task_id: str         # 种子唯一ID
+    data: Any            # 种子内容
+    status: Status       # 当前状态
+    priority: int            # 优先级（数值越小越优先）
+    created_at: float        # 创建时间戳
+    parent_id: Optional[str] = None   # 父种子 ID
+    children_ids: List[str] = None    # 子种子 ID 列表
+    ttl_seconds: Optional[int] = None  # 可选 TTL 时间（秒）
+
+    def __post_init__(self):
+        if self.children_ids is None:
+            self.children_ids = []
+
+
+class TaskQueue:
+
+    def __init__(self, cleanup_interval=60):
+        self._tasks: Dict[str, Task] = {}
+        self._lock = threading.Lock()
+        # self.cleanup_interval = cleanup_interval
+        # self._start_cleanup_task()
+
+    # def _start_cleanup_task(self):
+    #     """启动后台线程清理过期种子"""
+    #     def run():
+    #         while True:
+    #             time.sleep(self.cleanup_interval)
+    #             self._cleanup_expired_seeds()
+    #     threading.Thread(target=run, daemon=True).start()
+
+    def length(self) -> int:
+        with self._lock:
+            return len(self._tasks)
+
+    def status_length(self, status) -> int:
+        with self._lock:
+            return len([it for it in self._tasks.values() if it.status == status])
+
+    def get_task(self, task_id) -> Task:
+        with self._lock:
+            if task_id in self._tasks:
+                return self._tasks[task_id]
+
+    def get_task_by_status(self, status: list, limit: int = None) -> List[Task]:
+        with self._lock:
+            if not isinstance(status, list):
+                status = [status]
+            task_list = [it for it in self._tasks.values() if it.status in status]
+            task_list.sort(key=lambda x: (x.priority, x.created_at))
+            return task_list[:limit] if limit else task_list
+
+    def get_pending_task(self) -> Task:
+        with self._lock:
+            if items := [it for it in self._tasks.values() if it.status == Status.PENDING]:
+                items.sort(key=lambda x: (x.priority, x.created_at))
+                task_item = items[0]
+                task_item.status = Status.PROCESSING
+                self._tasks[task_item.task_id] = task_item
+                return task_item
+
+    def pop_task(self, status) -> Task:
+        with self._lock:
+            if items := [it for it in self._tasks.values() if it.status == status]:
+                items.sort(key=lambda x: (x.priority, x.created_at))
+                task_item = items[0]
+
+                to_remove = set()
+                queue = [task_item.task_id]
+
+                while queue:
+                    current = queue.pop(0)
+                    if current in self._tasks:
+                        to_remove.add(current)
+                        queue.extend(self._tasks[current].children_ids)
+                        del self._tasks[current]
+
+                for tid in to_remove:
+                    if task_item := self._tasks.get(tid):
+                        if task_item.parent_id in self._tasks:
+                            if tid in self._tasks[task_item.parent_id].children_ids:
+                                self._tasks[task_item.parent_id].children_ids.remove(tid)
+
+    def add_task(
+            self,
+            task_id: str = None,
+            data: Any = None,
+            status=Status.PENDING,
+            priority: int = 500,
+            parent_id: Optional[str] = None,
+            ttl_seconds: Optional[int] = None
+    ) -> bool:
+        """添加新种子，可指定父种子"""
+        with self._lock:
+            if not task_id:
+                task_id = md5(str(time.time()).encode()).hexdigest()
+
+            if task_id in self._tasks:
+                return False  # 防止重复添加
+
+            task_item = Task(
+                task_id=task_id,
+                data=data,
+                status=status,
+                priority=priority,
+                created_at=int(time.time()),
+                parent_id=parent_id,
+                ttl_seconds=ttl_seconds
+            )
+            self._tasks[task_id] = task_item
+
+            if parent_id and parent_id in self._tasks:
+                self._tasks[parent_id].children_ids.append(task_id)
+
+            return True
+
+    def update_task(self, task_id, status, data=None) -> Task:
+        with self._lock:
+            task_item = self._tasks[task_id]
+            task_item.status = status
+            if data:
+                task_item.data = data
+
+            for tid in task_item.children_ids:
+                if self._tasks[tid].status == Status.INSERT:
+                    del self._tasks[tid]
+
+            task_item.children_ids = []
+            self._tasks[task_id] = task_item
+
+            return task_item
+
+    def remove(self, task_ids: list) -> bool:
+        with self._lock:
+            for task_id in task_ids:
+                if task_item := self._tasks.get(task_id):
+
+                    if task_item.children_ids:
+                        continue
+
+                    if task_item.parent_id in self._tasks:
+                        if task_id in self._tasks[task_item.parent_id].children_ids:
+                            self._tasks[task_item.parent_id].children_ids.remove(task_id)
+
+                    del self._tasks[task_id]
+
+    def count_children(self, task_id: str) -> int:
+        with self._lock:
+            if task_id in self._tasks:
+                return len(self._tasks[task_id].children_ids)
+            return 0
+
+    # def _cleanup_expired_seeds(self):
+    #     now = time.time()
+    #     expired_ids = []
+    #     with self._lock:
+    #         for seed_id, seed in self._seeds.items():
+    #             if seed.ttl_seconds and now - seed.created_at > seed.ttl_seconds:
+    #                 expired_ids.append(seed_id)
+    #         for seed_id in expired_ids:
+    #             self._seeds[seed_id] = self._seeds[seed_id]._replace(status=SeedStatus.EXPIRED)
+    #         print(f"清理了 {len(expired_ids)} 个过期种子")

cobweb/launchers/distributor.py

@@ -2,15 +2,15 @@ import time
 import threading
 import traceback
 
-from typing import Callable, Type
 from inspect import isgenerator
+from typing import Callable, Type
 from urllib.parse import urlparse
 from requests import Response as Res
 
 from cobweb.crawlers import Crawler
-from cobweb.constant import DealModel, LogTemplate
 from cobweb.utils import LoghubDot, check_pause
-from cobweb.base import Seed, Queue, BaseItem, Request, Response, logger
+from cobweb.constant import DealModel, LogTemplate
+from cobweb.base import Seed, Status, TaskQueue, BaseItem, Request, Response, logger
 
 
 class Distributor(threading.Thread):
@@ -19,10 +19,7 @@ class Distributor(threading.Thread):
             self,
             task: str,
             project: str,
-            new: Queue,
-            todo: Queue,
-            done: Queue,
-            upload: Queue,
+            task_queue: TaskQueue,
             stop: threading.Event,
             pause: threading.Event,
             callback_register: Callable,
@@ -34,10 +31,8 @@ class Distributor(threading.Thread):
         self.stop = stop
         self.pause = pause
 
-        self.new = new
-        self.todo = todo
-        self.done = done
-        self.upload = upload
+        self.task_queue = task_queue
+
         self.callback_register = callback_register
         self.Crawler = SpiderCrawler
 
@@ -46,69 +41,83 @@ class Distributor(threading.Thread):
         self.thread_num = setting.SPIDER_THREAD_NUM
         self.max_retries = setting.SPIDER_MAX_RETRIES
         self.record_failed = setting.RECORD_FAILED_SPIDER
-        self.loghub_dot = LoghubDot(stop=stop)  # todo: 解偶
+        self.loghub_dot = LoghubDot(stop=stop)  # todo
 
         logger.debug(f"Distribute instance attrs: {self.__dict__}")
 
-    def distribute(self, item, seed, _id: int):
+    def distribute(self, task_id, item, _id: int):
         if isinstance(item, Request):
-            seed.params.start_time = time.time()
-            self.process(item=item, seed=seed, callback=self.Crawler.download, _id=1)
+            item.seed.params.start_time = time.time()
+            self.process(task_id=task_id, item=item, callback=self.Crawler.download, _id=1)
+
         elif isinstance(item, Response):
             if _id == 2:
                 raise TypeError("parse function can't yield a Response instance")
             dot = isinstance(item.response, Res)
-            # TODO: 请求成功打点
-            self.spider_logging(seed, item, dot=dot)
-            self.process(item=item, seed=seed, callback=self.Crawler.parse, _id=2)
+            self.spider_logging(item.seed, item, dot=dot)  # todo: update
+            self.process(task_id=task_id, item=item, callback=self.Crawler.parse, _id=2)
+
         elif isinstance(item, BaseItem):
-            self.upload.push(item)
+            self.task_queue.add_task(
+                data=item,
+                status=Status.UPLOAD,
+                parent_id=task_id
+            )
+
         elif isinstance(item, Seed):
-            self.new.push((seed, item), direct_insertion=True)
-        elif isinstance(item, str) and item == DealModel.poll:
-            self.todo.push(seed)
-        elif isinstance(item, str) and item == DealModel.done:
-            self.done.push(seed)
-        elif isinstance(item, str) and item == DealModel.fail:
-            seed.params.retry += 1
-            if seed.params.retry < self.max_retries:
-                self.todo.push(seed)
-            else:
-                if record_failed := self.record_failed:
-                    try:
-                        response = Response(seed, "failed", max_retries=True)
-                        self.process(response, seed, self.Crawler.parse, _id=2)
-                    except Exception as e:
-                        msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                        logger.error(msg = msg)
-                        record_failed = False
-                if not record_failed:
-                    self.done.push(seed)
-        else:
+            self.task_queue.add_task(
+                task_id=item.sid,
+                data=item,
+                status=Status.INSERT,
+                priority=item.params.priority,
+                parent_id=task_id
+            )
+
+        elif isinstance(item, str) and item != DealModel.done:
             raise TypeError("yield value type error!")
 
-    def process(self, item, seed, callback, _id: int):
-        result_iterators = callback(item)
-        if not isgenerator(result_iterators):
+    def process(self, task_id, item, callback, _id: int):
+        iterators = callback(item)
+        if not isgenerator(iterators):
             raise TypeError(f"{callback.__name__} function isn't a generator!")
-        for result_item in result_iterators:
-            self.distribute(result_item, seed, _id)
+        for it in iterators:
+            self.distribute(task_id=task_id, item=it, _id=_id)
 
     @check_pause
     def spider(self):
-        # TODO: 限流措施
-        if seed := self.todo.pop():
+        if task_item := self.task_queue.get_pending_task():
+            seed = task_item.data
+            status = Status.FINISHED
+            task_id = task_item.task_id
+
             try:
-                self.process(item=seed, seed=seed, callback=self.Crawler.request, _id=0)
+                self.process(task_id=task_id, item=seed, callback=self.Crawler.request, _id=0)
+
             except Exception as e:
-                url, status = seed.url, e.__class__.__name__
+
+                seed.params.retry += 1
+                url, _status = seed.url, e.__class__.__name__
+
                 msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
                 if getattr(e, "response", None) and isinstance(e.response, Res):
                     url = e.response.request.url
-                    status = e.response.status_code
-                    # TODO:失败请求打点
-                self.spider_logging(seed, None, error=True, url=url, status=status, msg=msg)
-                self.distribute(DealModel.fail, seed, _id=-1)
+                    _status = e.response.status_code
+
+                self.spider_logging(seed, None, error=True, url=url, status=_status, msg=msg)
+
+                if seed.params.retry < self.max_retries:
+                    status = Status.PENDING
+
+                elif self.record_failed:
+                    try:
+                        response = Response(seed, "failed", max_retries=True)
+                        self.process(task_id=task_id, item=response, callback=self.Crawler.parse, _id=2)
+                    except Exception as e:
+                        msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+                        logger.error(msg=msg)
+
+            finally:
+                self.task_queue.update_task(task_id, status=status, data=seed)
 
     def spider_logging(
             self, seed,

cobweb/launchers/launcher.py

@@ -5,20 +5,15 @@ import threading
 import importlib
 
 from cobweb import setting
-from cobweb.base import Seed, Queue, logger
+from cobweb.launchers.uploader import Uploader
 from cobweb.utils.tools import dynamic_load_class
 from cobweb.launchers.distributor import Distributor
-from cobweb.launchers.uploader import Uploader
+from cobweb.base import Seed, logger, TaskQueue, Status
 from typing import Optional, Union, Dict, Any, Callable
 
 
 class Launcher:
 
-    _NEW_QUEUE_ = Queue()
-    _TODO_QUEUE_ = Queue()
-    _DONE_QUEUE_ = Queue()
-    _UPLOAD_QUEUE_ = Queue()
-
     __REGISTER_FUNC__: Dict[str, Callable] = {}
     __WORKER_THREAD__: Dict[str, threading.Thread] = {}
 
@@ -33,6 +28,7 @@ class Launcher:
         self._pause = threading.Event()  # 暂停事件
 
         _setting = self._load_custom_settings(custom_setting)
+
         _setting.update(kwargs)
         for key, value in _setting.items():
             setattr(setting, key.upper(), value)
@@ -40,6 +36,8 @@ class Launcher:
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL
 
+        self._task_queue = TaskQueue()
+
         self.Scheduler = dynamic_load_class(setting.SCHEDULER)
         self.SpiderCrawler = dynamic_load_class(setting.CRAWLER)
         self.SpiderPipeline = dynamic_load_class(setting.PIPELINE)
@@ -108,7 +106,15 @@ class Launcher:
 
     def start_seeds(self, seeds: list[Union[str, Dict]]) -> list[Seed]:
         seed_list = [Seed(seed) for seed in seeds]
-        self._TODO_QUEUE_.push(seed_list)
+        for seed in seed_list:
+            self._task_queue.add_task(
+                task_id=seed.sid,
+                data=seed,
+                status=Status.PENDING,
+                priority=seed.params.priority,
+                parent_id=None,
+                ttl_seconds=None
+            )
         return seed_list
 
     def _register(self, func: Callable, tag: str = "launcher"):
@@ -139,20 +145,14 @@ class Launcher:
             project=self.project,
             stop=self._stop,
             pause=self._pause,
-            new=self._NEW_QUEUE_,
-            todo=self._TODO_QUEUE_,
-            done=self._DONE_QUEUE_,
-            upload=self._UPLOAD_QUEUE_,
+            task_queue=self._task_queue,
             callback_register=self._register
         ).start()
 
         Distributor(
             task=self.task,
             project=self.project,
-            new=self._NEW_QUEUE_,
-            todo=self._TODO_QUEUE_,
-            done=self._DONE_QUEUE_,
-            upload=self._UPLOAD_QUEUE_,
+            task_queue=self._task_queue,
             callback_register=self._register,
             stop=self._stop, pause=self._pause,
             SpiderCrawler=self.SpiderCrawler
@@ -161,8 +161,7 @@ class Launcher:
         Uploader(
             task=self.task, project=self.project,
             stop=self._stop, pause=self._pause,
-            done=self._DONE_QUEUE_,
-            upload=self._UPLOAD_QUEUE_,
+            task_queue=self._task_queue,
             callback_register=self._register,
             SpiderPipeline=self.SpiderPipeline
         ).start()

cobweb/launchers/uploader.py

@@ -2,7 +2,7 @@ import time
 import threading
 from typing import Callable, Type
 from cobweb.pipelines import Pipeline
-from cobweb.base import Queue, logger
+from cobweb.base import TaskQueue, logger, Status
 from cobweb.utils import check_pause
 
 
@@ -14,7 +14,7 @@ class Uploader(threading.Thread):
             project: str,
             stop: threading.Event,
             pause: threading.Event,
-            upload: Queue, done: Queue,
+            task_queue: TaskQueue,
             callback_register: Callable,
             SpiderPipeline: Type[Pipeline]
     ):
@@ -25,8 +25,7 @@ class Uploader(threading.Thread):
         self.stop = stop
         self.pause = pause
 
-        self.done = done
-        self.upload = upload
+        self.task_queue = task_queue
         self.callback_register = callback_register
 
         from cobweb import setting
@@ -40,30 +39,45 @@ class Uploader(threading.Thread):
 
     @check_pause
     def upload_data(self):
-        if not self.upload.length:
-            time.sleep(self.wait_seconds)
-            return
-        if self.upload.length < self.upload_size:
-            time.sleep(self.wait_seconds)
-        data_info, seeds = {}, []
-        try:
-            for _ in range(self.upload_size):
-                item = self.upload.pop()
-                if not item:
-                    break
-                seeds.append(item.seed)
-                data = self.pipeline.build(item)
-                data_info.setdefault(item.table, []).append(data)
-            for table, datas in data_info.items():
-                try:
-                    self.pipeline.upload(table, datas)
-                    # TODO: 上传打点
-                except Exception as e:
-                    logger.info(e)
-        except Exception as e:
-            logger.info(e)
-        if seeds:
-            self.done.push(seeds)
+        if task_list := self.task_queue.get_task_by_status(
+                status=Status.UPLOAD, limit=self.upload_size
+        ):
+            try:
+                data_info, task_ids = dict(), set()
+                for task_item in task_list:
+                    upload_data = self.pipeline.build(task_item.data)
+                    data_info.setdefault(task_item.data.table, []).append(upload_data)
+
+                for table, datas in data_info.items():
+                    try:
+                        self.pipeline.upload(table, datas)
+                    except Exception as e:
+                        logger.info(e)
+            except Exception as e:
+                logger.info(e)
+
+            if self.task_queue.status_length(status=Status.UPLOAD) < self.upload_size:
+                time.sleep(self.wait_seconds)
+
+        # data_info, seeds = {}, []
+        # try:
+        #     for _ in range(self.upload_size):
+        #         item = self.upload.pop()
+        #         if not item:
+        #             break
+        #         # seeds.append(item.seed)
+        #         data = self.pipeline.build(item)
+        #         data_info.setdefault(item.table, []).append(data)
+        #     for table, datas in data_info.items():
+        #         try:
+        #             self.pipeline.upload(table, datas)
+        #             # TODO: 上传打点
+        #         except Exception as e:
+        #             logger.info(e)
+        # except Exception as e:
+        #     logger.info(e)
+        # if self.upload.length < self.upload_size:
+        #     time.sleep(self.wait_seconds)
 
     def run(self):
         self.callback_register(self.upload_data, tag="Uploader")

cobweb/schedulers/scheduler.py

@@ -1,15 +1,13 @@
 import threading
 
-
 from typing import Callable
-from cobweb.base import Queue
+from cobweb.base import TaskQueue
 from abc import ABC, abstractmethod
 
 
 class Scheduler(ABC, threading.Thread):
 
-    __WORKING_ITEMS__ = {}
-    __LAUNCHER_FUNC__ = ["_reset", "_scheduler", "_insert", "_refresh", "_delete"]
+    # __LAUNCHER_FUNC__ = ["_reset", "_scheduler", "_insert", "_refresh", "_delete"]
 
     def __init__(
             self,
@@ -17,10 +15,7 @@ class Scheduler(ABC, threading.Thread):
             project,
             stop: threading.Event,
             pause: threading.Event,
-            new: Queue,
-            todo: Queue,
-            done: Queue,
-            upload: Queue,
+            task_queue: TaskQueue,
             callback_register: Callable
     ):
         super().__init__()
@@ -44,45 +39,9 @@ class Scheduler(ABC, threading.Thread):
         self.stop = stop
         self.pause = pause
 
-        self.new = new
-        self.todo = todo
-        self.done = done
-        self.upload = upload
+        self.task_queue = task_queue
 
         self.callback_register = callback_register
-        self.lock = threading.Lock()
-
-    def is_empty(self):
-        return all(queue.empty() for queue in (self.new, self.todo, self.done, self.upload))
-
-    def set_working_items(self, item_info: dict = None):
-        if not item_info:
-            return False
-        with self.lock:
-            self.__WORKING_ITEMS__.update(item_info)
-
-    def get_working_items(self) -> dict:
-        with self.lock:
-            return self.__WORKING_ITEMS__.copy()
-
-    def remove_working_items(self, items: list[str] = None) -> int:
-        if not items:
-            return 0
-        with self.lock:
-            deleted_count = 0
-            for item in items:
-                if item in self.__WORKING_ITEMS__:
-                    del self.__WORKING_ITEMS__[item]
-                    deleted_count += 1
-            return deleted_count
-
-    def get_working_items_count(self) -> int:
-        with self.lock:
-            return len(self.__WORKING_ITEMS__)
-
-    def clear_working_items(self):
-        with self.lock:
-            self.__WORKING_ITEMS__.clear()
 
     @abstractmethod
     def reset(self):

cobweb/schedulers/scheduler_with_redis.py

@@ -4,7 +4,7 @@ import threading
 from typing import Callable
 from cobweb.db import RedisDB, ApiDB
 from cobweb.utils import check_pause
-from cobweb.base import Queue, Seed, logger
+from cobweb.base import Seed, logger, TaskQueue, Status
 from cobweb.constant import LogTemplate
 from .scheduler import Scheduler
 use_api = bool(os.getenv("REDIS_API_HOST", 0))
@@ -18,13 +18,10 @@ class RedisScheduler(Scheduler):
             project,
             stop: threading.Event,
             pause: threading.Event,
-            new: Queue,
-            todo: Queue,
-            done: Queue,
-            upload: Queue,
+            task_queue: TaskQueue,
             callback_register: Callable
     ):
-        super().__init__(task, project, stop, pause, new, todo, done, upload, callback_register)
+        super().__init__(task, project, stop, pause, task_queue, callback_register)
         self.todo_key = f"{{{project}:{task}}}:todo"
         self.done_key = f"{{{project}:{task}}}:done"
         self.fail_key = f"{{{project}:{task}}}:fail"
@@ -38,14 +35,13 @@ class RedisScheduler(Scheduler):
         检查过期种子，重新添加到redis缓存中
         """
         while not self.stop.is_set():
-            if self.db.lock(self.reset_lock_key, t=60):
+            if self.db.lock(self.reset_lock_key, t=360):
 
-                # _min = -int(time.time()) + self.seed_reset_seconds
-                _min = -int(time.time()) + 3600
+                _min = -int(time.time()) + self.seed_reset_seconds
                 self.db.members(self.todo_key, 0, _min=_min, _max="(0")
                 self.db.delete(self.reset_lock_key)
 
-            time.sleep(60)
+            time.sleep(self.seed_reset_seconds)
 
     @check_pause
     def schedule(self):
@@ -56,42 +52,43 @@ class RedisScheduler(Scheduler):
             time.sleep(self.scheduler_wait_seconds)
             return
 
-        if self.todo.length >= self.todo_queue_size:
+        if self.task_queue.status_length(Status.PENDING) >= self.todo_queue_size:
             time.sleep(self.todo_queue_full_wait_seconds)
             return
 
-        members = self.db.members(
+        if members := self.db.members(
             self.todo_key, int(time.time()),
             count=self.todo_queue_size,
             _min=0, _max="(1000"
-        )
-
-        logger.debug(f"Retrieved {len(members)} seeds from Redis.")
-
-        seeds, item_info = list(), dict()
-        for member, priority in members:
-            seed = Seed(member, priority=int(priority % 1000))
-            item_info[seed.to_string] = seed.params.priority
-            seeds.append(seed)
-
-        # self.set_working_items(item_info)
-        self.todo.push(seeds)
-        # time.sleep(1)
+        ):
+            for member, priority in members:
+                seed = Seed(member, priority=int(priority % 1000))
+                self.task_queue.add_task(
+                    task_id=seed.sid,
+                    data=seed,
+                    status=Status.PENDING,
+                    priority=seed.params.priority
+                )
 
     @check_pause
     def insert(self):
         """
         添加新种子到redis队列中
         """
-        seeds, delete_seeds = dict(), set()
-        for seed, new_seed in self.new.iter_items(limit=self.new_queue_max_size):
-            seeds[new_seed.to_string] = new_seed.params.priority
-            delete_seeds.add(seed)
+        if task_list := self.task_queue.get_task_by_status(
+            status=Status.INSERT, limit=self.new_queue_max_size
+        ):
+            seed_info, task_ids = dict(), set()
 
-        self.db.zadd(self.todo_key, seeds, nx=True)
-        self.done.push(delete_seeds)
+            for task_item in task_list:
+                seed = task_item.data
+                task_ids.add(task_item.task_id)
+                seed_info[seed.to_string] = seed.params.priority
 
-        if self.new.length < self.new_queue_max_size:
+            self.db.zadd(self.todo_key, seed_info, nx=True)
+            self.task_queue.remove(task_ids)
+
+        if self.task_queue.status_length(status=Status.INSERT) < self.new_queue_max_size:
             time.sleep(self.scheduler_wait_seconds)
 
     @check_pause
@@ -99,27 +96,28 @@ class RedisScheduler(Scheduler):
         """
         刷新doing种子过期时间，防止reset重新消费
         """
-        if item_info := self.get_working_items():
+        if task_list := self.task_queue.get_task_by_status(
+            status=[Status.PENDING, Status.PROCESSING, Status.FINISHED],
+        ):
             refresh_time = int(time.time())
-            seed_info = {k: -refresh_time - v / 1000 for k, v in item_info.items()}
+            seed_info = {it.data.to_string: -refresh_time - it.data.params.priority / 1000 for it in task_list}
             self.db.zadd(self.todo_key, seed_info, xx=True)
-            # self.set_working_items(seed_info)
-        time.sleep(20)
+        time.sleep(self.seed_reset_seconds // 3)
 
     @check_pause
     def delete(self):
         """
         删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
         """
-        seeds = [seed for seed in self.done.iter_items(limit=self.done_queue_max_size)]
-        items = [seed.to_string for seed in seeds]
-
-        if self.remove_working_items(items):
-            self.db.zrem(self.todo_key, *items)
-        elif seeds:
-            self.done.push(seeds)
-
-        if self.done.length < self.done_queue_max_size:
+        if task_list := self.task_queue.get_task_by_status(
+                status=Status.FINISHED, limit=self.done_queue_max_size
+        ):
+            zrem_items = [it.data.to_string for it in task_list]
+            remove_task_ids = [it.task_id for it in task_list]
+            self.db.zrem(self.todo_key, *zrem_items)
+            self.task_queue.remove(remove_task_ids)
+
+        if self.task_queue.status_length(status=Status.FINISHED) < self.done_queue_max_size:
             time.sleep(self.done_queue_wait_seconds)
 
     def run(self):
@@ -129,32 +127,41 @@ class RedisScheduler(Scheduler):
             self.callback_register(func, tag="scheduler")
 
         while not self.stop.is_set():
-            working_count = self.get_working_items_count()
-            memory_count = self.db.zcount(self.todo_key, "-inf", "(0")
-            todo_count = self.db.zcount(self.todo_key, 0, "(1000")
-            all_count = self.db.zcard(self.todo_key)
+            todo_len = self.task_queue.status_length(status=Status.PENDING)
+            doing_len = self.task_queue.status_length(status=Status.PROCESSING)
+            done_len = self.task_queue.status_length(status=Status.FINISHED)
+            upload_len = self.task_queue.status_length(status=Status.UPLOAD)
+
+            redis_doing_count = self.db.zcount(self.todo_key, "-inf", "(0")
+            redis_todo_len = self.db.zcount(self.todo_key, 0, "(1000")
+            redis_seed_count = self.db.zcard(self.todo_key)
 
             if self.pause.is_set():
                 execute_time = int(time.time()) - start_time
                 if not self.task_model and execute_time > self.before_scheduler_wait_seconds:
                     logger.info("Done! ready to close thread...")
                     self.stop.set()
-                elif todo_count:
+                elif redis_todo_len:
                     logger.info(
-                        f"Recovery {self.task} task run！Todo seeds count: {todo_count}, queue length: {all_count}")
+                        f"Recovery {self.task} task run！"
+                        f"Todo seeds count: {redis_todo_len}"
+                        f", queue length: {redis_seed_count}"
+                    )
                     self.pause.clear()
                 else:
                     logger.info("Pause! waiting for resume...")
 
-            elif self.is_empty():
-
-                if all_count:
-                    logger.info(f"Todo seeds count: {todo_count}, queue length: {all_count}")
+            elif self.task_queue.length() == 0:
+                if redis_seed_count:
+                    logger.info(
+                        f"Todo seeds count: {redis_todo_len}"
+                        f", queue length: {redis_seed_count}"
+                    )
                     self.pause.clear()
                 else:
                     count = 0
                     for _ in range(3):
-                        if not all_count:
+                        if not redis_seed_count:
                             count += 1
                             time.sleep(5)
                             logger.info("Checking count...")
@@ -162,19 +169,18 @@ class RedisScheduler(Scheduler):
                             break
                     if count >= 3:
                         logger.info("Todo queue is empty! Pause set...")
-                        self.clear_working_items()
                         self.pause.set()
 
             else:
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,
-                    doing_len=working_count,
-                    todo_len=self.todo.length,
-                    done_len=self.done.length,
-                    redis_seed_count=all_count,
-                    redis_todo_len=todo_count,
-                    redis_doing_len=memory_count,
-                    upload_len=self.upload.length,
+                    doing_len=doing_len,
+                    todo_len=todo_len,
+                    done_len=done_len,
+                    redis_seed_count=redis_seed_count,
+                    redis_todo_len=redis_todo_len,
+                    redis_doing_len=redis_doing_count,
+                    upload_len=upload_len,
                 ))
 
             time.sleep(30)

{cobweb_launcher-3.1.22.dist-info → cobweb_launcher-3.1.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.22
+Version: 3.1.23
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP

{cobweb_launcher-3.1.22.dist-info → cobweb_launcher-3.1.23.dist-info}/RECORD

@@ -1,13 +1,14 @@
 cobweb/__init__.py,sha256=YdBi3uytEFRXan155xU1kKMpiUKUupO2RGeJyXmH0zk,129
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
 cobweb/setting.py,sha256=rHPQfc4a1xMTbkt3_KXBfUomhYcbTXogsz7ew-QsqHw,1670
-cobweb/base/__init__.py,sha256=c1qwQrpe5atW_OK_Qw9kaiZpVDey3t_nvRwlH7QRIqY,179
+cobweb/base/__init__.py,sha256=NanSxJr0WsqjqCNOQAlxlkt-vQEsERHYBzacFC057oI,222
 cobweb/base/common_queue.py,sha256=hYdaM70KrWjvACuLKaGhkI2VqFCnd87NVvWzmnfIg8Q,1423
 cobweb/base/item.py,sha256=1bS4U_3vzI2jzSSeoEbLoLT_5CfgLPopWiEYtaahbvw,1674
 cobweb/base/logger.py,sha256=Vsg1bD4LXW91VgY-ANsmaUu-mD88hU_WS83f7jX3qF8,2011
 cobweb/base/request.py,sha256=MBYYjWpbRQRulPG0zPbK0DO3LKmScqQ4tBzFXekYkao,2652
 cobweb/base/response.py,sha256=g8e5H0hEiRfqseh3nD7t6a1rhIJYRMV7nI47kqNOd-U,446
 cobweb/base/seed.py,sha256=ddaWCq_KaWwpmPl1CToJlfCxEEnoJ16kjo6azJs9uls,5000
+cobweb/base/task_queue.py,sha256=3ScPKnjlPEuuCzWyG9D2iHiND3L9lLM7fo1LNOkw8CY,6337
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/crawler.py,sha256=ZZVZJ17RWuvzUFGLjqdvyVZpmuq-ynslJwXQzdm_UdQ,709
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
@@ -16,24 +17,24 @@ cobweb/db/redis_db.py,sha256=X7dUpW50QcmRPjYlYg7b-fXF_fcjuRRk3DBx2ggetXk,7687
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=6_v2jd2sgj6YnOB1nPKiYBskuXVb5xpQnq2YaDGJgQ8,100
-cobweb/launchers/distributor.py,sha256=ALvu7MVZLSQPmWJc_FR-UUIlTMv4PAu8q7tt-KzK1v8,6810
-cobweb/launchers/launcher.py,sha256=L75eYKemPVqT0cuwfBy_Vh0CObWilDpJ9ibD29g5L38,5742
-cobweb/launchers/uploader.py,sha256=5Hm1pmco8PsFrtBDRN9aw6IjAElfX2wdN1yaILtp03w,2059
+cobweb/launchers/distributor.py,sha256=I5QBs2hFiyGGkqLLkMw9uzf4_oRW2JvahNW9yc866cc,6748
+cobweb/launchers/launcher.py,sha256=Shb6o6MAM38d32ybW2gY6qpGmhuiV7jo9TDh0f7rud8,5694
+cobweb/launchers/uploader.py,sha256=dDBv6Vfy1ciaTAJA3TebJV-2oM3OMrqTfzpNX8VGv-0,2766
 cobweb/pipelines/__init__.py,sha256=rtkaaCZ4u1XcxpkDLHztETQjEcLZ_6DXTHjdfcJlyxQ,97
 cobweb/pipelines/pipeline.py,sha256=OgSEZ2DdqofpZcer1Wj1tuBqn8OHVjrYQ5poqt75czQ,357
 cobweb/pipelines/pipeline_csv.py,sha256=TFqxqgVUqkBF6Jott4zd6fvCSxzG67lpafRQtXPw1eg,807
 cobweb/pipelines/pipeline_loghub.py,sha256=zwIa_pcWBB2UNGd32Cu-i1jKGNruTbo2STdxl1WGwZ0,1829
 cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0,49
-cobweb/schedulers/scheduler.py,sha256=qb_u3tJELp4zRGOyT8OsNWndKDknhQslnKnjUlwEpiE,2943
-cobweb/schedulers/scheduler_with_redis.py,sha256=i7EfjXY4p5BJrZfCNxyUqBqIcyg1ZICjm7gbxrhHbN0,6448
+cobweb/schedulers/scheduler.py,sha256=Of-BjbBh679R6glc12Kc8iugeERCSusP7jolpCc1UMI,1740
+cobweb/schedulers/scheduler_with_redis.py,sha256=SUiEjYhzbbzc5kt_zpK8bXaEjIpwqC-JBk8ApHcVa18,7149
 cobweb/utils/__init__.py,sha256=TRFJyyBjaQH_sejU6G_msOeHpjc3ZXU0dUOO5GQfknM,171
 cobweb/utils/bloom.py,sha256=A8xqtHXp7jgRoBuUlpovmq8lhU5y7IEF0FOCjfQDb6s,1855
 cobweb/utils/decorators.py,sha256=ZwVQlz-lYHgXgKf9KRCp15EWPzTDdhoikYUNUCIqNeM,1140
 cobweb/utils/dotting.py,sha256=L-jGSApdnFIP4jUWH6p5qIme0aJ1vyDrxAx8wOJWvcs,1960
 cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
 cobweb/utils/tools.py,sha256=14TCedqt07m4z6bCnFAsITOFixeGr8V3aOKk--L7Cr0,879
-cobweb_launcher-3.1.22.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-3.1.22.dist-info/METADATA,sha256=L5ozmjklF83tH1LhHCgFp2U-rRBzEP_sj6x8WSZda5s,5998
-cobweb_launcher-3.1.22.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-3.1.22.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-3.1.22.dist-info/RECORD,,
+cobweb_launcher-3.1.23.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.1.23.dist-info/METADATA,sha256=QwwqDS7cSVmiivRXj_Kgu2BZW527APBQ-Qe6frnjIls,5998
+cobweb_launcher-3.1.23.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-3.1.23.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.1.23.dist-info/RECORD,,