cobweb-launcher 3.1.13__py3-none-any.whl → 3.1.14__py3-none-any.whl

cobweb/base/common_queue.py

@@ -1,30 +1,51 @@
 from collections import deque
+from typing import Any, Iterable, Union
 
 
 class Queue:
-
     def __init__(self):
+        """初始化队列"""
         self._queue = deque()
 
     @property
     def length(self) -> int:
+        """返回队列长度"""
         return len(self._queue)
 
-    def push(self, data, left: bool = False, direct_insertion: bool = False):
-        try:
-            if not data:
-                return None
-            if not direct_insertion and any(isinstance(data, t) for t in (list, tuple)):
-                self._queue.extendleft(data) if left else self._queue.extend(data)
-            else:
-                self._queue.appendleft(data) if left else self._queue.append(data)
-        except AttributeError:
-            pass
-
-    def pop(self, left: bool = True):
+    def empty(self) -> bool:
+        """检查队列是否为空"""
+        return not self._queue
+
+    def push(self, data: Union[Any, Iterable], direct_insertion: bool = False):
+        """
+        向队列中添加数据。
+        如果数据是可迭代对象（如列表、元组或集合），可以选择直接扩展队列。
+        """
+        if not data:
+            return
+
+        if not direct_insertion and isinstance(data, (list, tuple, set)):
+            self._queue.extend(data)
+        else:
+            self._queue.append(data)
+
+    def pop(self) -> Any:
+        """
+        从队列左侧弹出一个元素。
+        如果队列为空，返回 None。
+        """
         try:
-            return self._queue.popleft() if left else self._queue.pop()
+            return self._queue.popleft()
         except IndexError:
             return None
-        except AttributeError:
-            return None
+
+    def iter_items(self, limit: int = 1) -> Iterable:
+        """
+        按指定数量从队列中弹出多个元素并生成它们。
+        如果队列为空或达到限制，则停止生成。
+        """
+        for _ in range(limit):
+            item = self.pop()
+            if item is None:
+                break
+            yield item

cobweb/base/item.py

@@ -1,19 +1,22 @@
 from .seed import Seed
+from typing import Dict, Any
 from collections import namedtuple
 
 
-class Item(type):
+class ItemMeta(type):
 
-    def __new__(cls, name, bases, dct):
-        new_class_instance = type.__new__(cls, name, bases, dct)
+    def __new__(cls, name: str, bases: tuple, dct: dict) -> type:
+        new_class = super().__new__(cls, name, bases, dct)
         if name != "BaseItem":
-            table = getattr(new_class_instance, "__TABLE__")
-            fields = getattr(new_class_instance, "__FIELDS__")
-            new_class_instance.Data = namedtuple(table, fields)
-        return new_class_instance
+            table = getattr(new_class, "__TABLE__")
+            fields = getattr(new_class, "__FIELDS__")
+            if not table or not fields:
+                raise ValueError(f"Missing required attributes '__TABLE__' or '__FIELDS__' in class {name}")
+            new_class.Data = namedtuple(table, fields)
+        return new_class
 
 
-class BaseItem(metaclass=Item):
+class BaseItem(metaclass=ItemMeta):
 
     __TABLE__ = ""
     __FIELDS__ = ""
@@ -23,34 +26,36 @@ class BaseItem(metaclass=Item):
 
         data = {}
         for key, value in kwargs.items():
-            if key not in self.__FIELDS__:
-                self.__setattr__(key, value)
-            else:
+            if key in self.__FIELDS__:
                 data[key] = value
+            else:
+                setattr(self, key, value)
 
-        self.data = self.Data(**data)
+        try:
+            self.data = self.Data(**data)
+        except TypeError as e:
+            raise ValueError(f"Invalid field values for Data: {e}") from e
 
     @property
-    def to_dict(self):
+    def to_dict(self) -> Dict[str, Any]:
         return self.data._asdict()
 
     @property
-    def table(self):
-        return self.Data.__name__
+    def fields(self) -> tuple[str]:
+        return self.data._fields
 
     @property
-    def fields(self):
-        return self.__FIELDS__
-
-    def __getattr__(self, name):
-        return None
+    def table(self) -> str:
+        return self.Data.__name__
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: str, value: Any):
         setattr(self, key, value)
 
-    def __getitem__(self, item):
-        return getattr(self, item)
+    def __getitem__(self, key: str) -> Any:
+        return getattr(self, key, None)
 
+    def __getattr__(self, name: str) -> Any:
+        return None
 
 
 class CSVItem(BaseItem):

cobweb/base/request.py

@@ -1,8 +1,12 @@
 import random
 import requests
+from typing import Any, Dict
 
 
 class Request:
+    """
+    请求类，用于封装 HTTP 请求并提供相关功能。
+    """
 
     __REQUEST_ATTRS__ = {
         "params",
@@ -22,26 +26,35 @@ class Request:
     }
 
     def __init__(
-            self,
-            url,
-            seed,
-            random_ua=True,
-            check_status_code=True,
-            **kwargs
+        self,
+        url: str,
+        seed: Any,
+        random_ua: bool = True,
+        check_status_code: bool = True,
+        **kwargs,
     ):
+        """
+        初始化请求对象。
+        :param url: 请求的 URL。
+        :param seed: 种子对象或标识符。
+        :param random_ua: 是否使用随机 User-Agent，默认为 True。
+        :param check_status_code: 是否检查响应状态码，默认为 True。
+        :param kwargs: 其他扩展参数。
+        """
         self.url = url
         self.seed = seed
         self.check_status_code = check_status_code
-        self.request_setting = {}
+        self.request_setting: Dict[str, Any] = {}
 
-        for k, v in kwargs.items():
-            if k in self.__class__.__REQUEST_ATTRS__:
-                self.request_setting[k] = v
-                continue
-            self.__setattr__(k, v)
+        for key, value in kwargs.items():
+            if key in self.__class__.__REQUEST_ATTRS__:
+                self.request_setting[key] = value
+            else:
+                setattr(self, key, value)
 
-        if not getattr(self, "method", None):
-            self.method = "POST" if self.request_setting.get("data") or self.request_setting.get("json") else "GET"
+        self.method = getattr(self, "method", None) or (
+            "POST" if self.request_setting.get("data") or self.request_setting.get("json") else "GET"
+        )
 
         if random_ua:
             self._build_header()
@@ -54,15 +67,14 @@ class Request:
         v4 = random.randint(533, 605)
         v5 = random.randint(1000, 6000)
         v6 = random.randint(10, 80)
-        user_agent = (f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_{v1}_{v2}) AppleWebKit/{v4}.{v3} "
-                      f"(KHTML, like Gecko) Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
-        return user_agent
+        return (f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_{v1}_{v2}) "
+                f"AppleWebKit/{v4}.{v3} (KHTML, like Gecko) "
+                f"Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
 
     def _build_header(self):
-        if not self.request_setting.get("headers"):
-            self.request_setting["headers"] = {"accept": "*/*", "user-agent": self._random_ua}
-        elif "user-agent" not in [key.lower() for key in self.request_setting["headers"].keys()]:
-            self.request_setting["headers"]["user-agent"] = self._random_ua
+        headers = self.request_setting.setdefault("headers", {})
+        if not headers.get("user-agent"):
+            headers["user-agent"] = self._random_ua
 
     def download(self) -> requests.Response:
         response = requests.request(self.method, self.url, **self.request_setting)
@@ -71,12 +83,6 @@ class Request:
         return response
 
     @property
-    def to_dict(self):
-        _dict = self.__dict__.copy()
-        _dict.pop('url')
-        _dict.pop('seed')
-        _dict.pop('check_status_code')
-        _dict.pop('request_setting')
-        return _dict
-
-
+    def to_dict(self) -> Dict[str, Any]:
+        excluded_keys = {"url", "seed", "check_status_code", "request_setting"}
+        return {k: v for k, v in self.__dict__.items() if k not in excluded_keys}

cobweb/base/seed.py

@@ -1,11 +1,23 @@
 import json
 import time
 import hashlib
+from typing import Any, Dict, Optional, Union
 
 
 class SeedParams:
+    """
+    定义种子参数类，用于存储种子的元信息。
+    """
 
-    def __init__(self, retry, priority, seed_version, seed_status=None, proxy_type=None, proxy=None):
+    def __init__(
+        self,
+        retry: Optional[int] = None,
+        priority: Optional[int] = None,
+        seed_version: Optional[int] = None,
+        seed_status: Optional[str] = None,
+        proxy_type: Optional[str] = None,
+        proxy: Optional[str] = None,
+    ):
         self.retry = retry or 0
         self.priority = priority or 300
         self.seed_version = seed_version or int(time.time())
@@ -13,8 +25,15 @@ class SeedParams:
         self.proxy_type = proxy_type
         self.proxy = proxy
 
+    def __getattr__(self, name: str) -> Any:
+        """动态获取未定义的属性，返回 None"""
+        return None
+
 
 class Seed:
+    """
+    种子类，用于表示一个种子对象，包含种子的基本属性和方法。
+    """
 
     __SEED_PARAMS__ = [
         "retry",
@@ -22,90 +41,111 @@ class Seed:
         "seed_version",
         "seed_status",
         "proxy_type",
-        "proxy"
+        "proxy",
     ]
 
     def __init__(
-            self,
-            seed,
-            sid=None,
-            retry=None,
-            priority=None,
-            seed_version=None,
-            seed_status=None,
-            proxy_type=None,
-            proxy=None,
-            **kwargs
+        self,
+        seed: Union[str, bytes, Dict[str, Any]] = None,
+        sid: Optional[str] = None,
+        retry: Optional[int] = None,
+        priority: Optional[int] = None,
+        seed_version: Optional[int] = None,
+        seed_status: Optional[str] = None,
+        proxy_type: Optional[str] = None,
+        proxy: Optional[str] = None,
+        **kwargs,
     ):
-        if any(isinstance(seed, t) for t in (str, bytes)):
-            try:
-                item = json.loads(seed)
-                self._init_seed(item)
-            except json.JSONDecodeError:
-                self.__setattr__("url", seed)
-        elif isinstance(seed, dict):
-            self._init_seed(seed)
-        else:
-            raise TypeError(Exception(
-                f"seed type error, "
-                f"must be str or dict! "
-                f"seed: {seed}"
-            ))
-
+        """
+        初始化种子对象。
+        :param seed: 种子数据，可以是字符串、字节或字典。
+        :param sid: 种子的唯一标识符。
+        :param retry: 重试次数。
+        :param priority: 优先级。
+        :param seed_version: 种子版本。
+        :param seed_status: 种子状态。
+        :param proxy_type: 代理类型。
+        :param proxy: 代理地址。
+        :param kwargs: 其他扩展参数。
+        """
+        # 初始化种子数据
+        if seed:
+            if isinstance(seed, (str, bytes)):
+                try:
+                    item = json.loads(seed)
+                    self._init_seed(item)
+                except json.JSONDecodeError as e:
+                    raise ValueError(f"Invalid JSON format for seed: {seed}") from e
+            elif isinstance(seed, dict):
+                self._init_seed(seed)
+            else:
+                raise TypeError(f"Seed type error, must be str, bytes, or dict! Seed: {seed}")
+
+        # 初始化种子参数
         seed_params = {
             "retry": retry,
             "priority": priority,
             "seed_version": seed_version,
             "seed_status": seed_status,
             "proxy_type": proxy_type,
-            "proxy": proxy
+            "proxy": proxy,
         }
 
+        # 合并扩展参数
         if kwargs:
             self._init_seed(kwargs)
-            seed_params.update({
-                k:v for k, v in kwargs.items()
-                if k in self.__SEED_PARAMS__
-            })
+            seed_params.update({k: v for k, v in kwargs.items() if k in self.__SEED_PARAMS__})
+
+        # 初始化唯一标识符
         if sid or not getattr(self, "sid", None):
             self._init_id(sid)
+
+        # 设置参数对象
         self.params = SeedParams(**seed_params)
 
-    def __getattr__(self, name):
+    def __getattr__(self, name: str) -> Any:
+        """动态获取未定义的属性，返回 None"""
         return None
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: str, value: Any):
+        """支持字典式设置属性"""
         setattr(self, key, value)
 
-    def __getitem__(self, item):
-        return getattr(self, item)
+    def __getitem__(self, key: str) -> Any:
+        """支持字典式获取属性"""
+        return getattr(self, key, None)
 
-    def __str__(self):
-        return json.dumps(self.__dict__, ensure_ascii=False)
+    def __str__(self) -> str:
+        """返回种子的 JSON 字符串表示"""
+        return self.to_string
 
-    def __repr__(self):
-        chars = [f"{k}={v}" for k, v in self.__dict__.items()]
-        return f'{self.__class__.__name__}({", ".join(chars)})'
+    def __repr__(self) -> str:
+        """返回种子的调试字符串表示"""
+        attrs = [f"{k}={v}" for k, v in self.__dict__.items()]
+        return f"{self.__class__.__name__}({', '.join(attrs)})"
 
-    def _init_seed(self, seed_info:dict):
-        for k, v in seed_info.items():
-            if k not in self.__SEED_PARAMS__:
-                self.__setattr__(k, v)
+    def _init_seed(self, seed_info: Dict[str, Any]):
+        """初始化种子数据"""
+        for key, value in seed_info.items():
+            if key not in self.__SEED_PARAMS__:
+                self.__setattr__(key, value)
 
-    def _init_id(self, sid):
+    def _init_id(self, sid: Optional[str]):
+        """初始化种子的唯一标识符"""
         if not sid:
             sid = hashlib.md5(self.to_string.encode()).hexdigest()
         self.__setattr__("sid", sid)
 
     @property
-    def to_dict(self) -> dict:
+    def to_dict(self) -> Dict[str, Any]:
+        """返回种子的字典表示（不包含 params 属性）"""
         seed = self.__dict__.copy()
-        if seed.get("params"):
-            del seed["params"]
+        seed.pop("params", None)
         return seed
 
     @property
     def to_string(self) -> str:
+        """返回种子的紧凑 JSON 字符串表示"""
         return json.dumps(
             self.to_dict,
             ensure_ascii=False,
@@ -113,10 +153,10 @@ class Seed:
         )
 
     @property
-    def get_all(self):
+    def get_all(self) -> str:
+        """返回种子的所有属性（包括 params）的 JSON 字符串表示"""
         return json.dumps(
             self.__dict__,
             ensure_ascii=False,
             separators=(",", ":")
         )
-

cobweb/db/api_db.py

@@ -57,7 +57,8 @@ class ApiDB:
         return self._get_response(api="/zcard", params=dict(name=name))
 
     def zadd(self, name, item: dict, **kwargs):
-        return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
+        if item:
+            return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
 
     def zrem(self, name, *values):
         return self._post_response(api="/zrem", data=dict(name=name, values=values))

cobweb/db/redis_db.py

@@ -120,7 +120,8 @@ class RedisDB:
     def zadd(self, name, item: dict, **kwargs):
         # with self.get_connection() as client:
         #     return client.zadd(name, item, **kwargs)
-        return self.execute_command("zadd", name, item, **kwargs)
+        if item:
+            return self.execute_command("zadd", name, item, **kwargs)
 
     def zrem(self, name, *value):
         # with self.get_connection() as client:

cobweb/launchers/launcher.py

@@ -9,6 +9,7 @@ from cobweb.base import Seed, Queue, logger
 from cobweb.utils.tools import dynamic_load_class
 from cobweb.launchers.distributor import Distributor
 from cobweb.launchers.uploader import Uploader
+from typing import Optional, Union, Dict, Any, Callable
 
 
 class Launcher:
@@ -18,8 +19,8 @@ class Launcher:
     _DONE_QUEUE_ = Queue()
     _UPLOAD_QUEUE_ = Queue()
 
-    __WORKER_THREAD__ = dict()
-    __REGISTER_FUNC__ = dict()
+    __REGISTER_FUNC__: Dict[str, Callable] = {}
+    __WORKER_THREAD__: Dict[str, threading.Thread] = {}
 
     def __init__(self, task, project, custom_setting=None, **kwargs):
         super().__init__()
@@ -31,24 +32,10 @@ class Launcher:
         self._stop = threading.Event()  # 结束事件
         self._pause = threading.Event()  # 暂停事件
 
-        _setting = dict()
-
-        if custom_setting:
-            if isinstance(custom_setting, dict):
-                _setting = custom_setting
-            else:
-                if isinstance(custom_setting, str):
-                    custom_setting = importlib.import_module(custom_setting)
-                if not inspect.ismodule(custom_setting):
-                    raise Exception
-                for k, v in custom_setting.__dict__.items():
-                    if not k.startswith("__") and not inspect.ismodule(v):
-                        _setting[k] = v
-
-        _setting.update(**kwargs)
-
-        for k, v in _setting.items():
-            setattr(setting, k.upper(), v)
+        _setting = self._load_custom_settings(custom_setting)
+        _setting.update(kwargs)
+        for key, value in _setting.items():
+            setattr(setting, key.upper(), value)
 
         self._done_model = setting.DONE_MODEL
         self._task_model = setting.TASK_MODEL
@@ -57,6 +44,23 @@ class Launcher:
         self.SpiderCrawler = dynamic_load_class(setting.CRAWLER)
         self.SpiderPipeline = dynamic_load_class(setting.PIPELINE)
 
+    @staticmethod
+    def _load_custom_settings(custom_setting: Optional[Union[str, Dict]]) -> Dict[str, Any]:
+        _setting = {}
+        if custom_setting:
+            if isinstance(custom_setting, dict):
+                _setting = custom_setting
+            elif isinstance(custom_setting, str):
+                module = importlib.import_module(custom_setting)
+                _setting = {
+                    k: v
+                    for k, v in module.__dict__.items()
+                    if not k.startswith("__") and not inspect.ismodule(v)
+                }
+            else:
+                raise ValueError("custom_setting must be a dictionary or a module path.")
+        return _setting
+
     @property
     def request(self):
         """
@@ -102,18 +106,17 @@ class Launcher:
             self.SpiderCrawler.parse = func
         return decorator
 
-    def start_seeds(self, seeds: list):
+    def start_seeds(self, seeds: list[Union[str, Dict]]) -> list[Seed]:
         seed_list = [Seed(seed) for seed in seeds]
         self._TODO_QUEUE_.push(seed_list)
         return seed_list
 
-    def _register(self, func, tag: str = "launcher"):
-        name = tag + ":" + func.__name__ + "_" + str(uuid.uuid4())
+    def _register(self, func: Callable, tag: str = "launcher"):
+        name = f"{tag}:{func.__name__}_{uuid.uuid4()}"
         self.__REGISTER_FUNC__[name] = func
         if not self.__WORKER_THREAD__.get(name):
             worker_thread = threading.Thread(name=name, target=func)
             self.__WORKER_THREAD__[name] = worker_thread
-            # worker_thread.start()
 
     def _monitor(self):
         while not self._stop.is_set():

cobweb/schedulers/scheduler.py

@@ -8,6 +8,7 @@ from abc import ABC, abstractmethod
 
 class Scheduler(ABC, threading.Thread):
 
+    __WORKING_ITEMS__ = {}
     __LAUNCHER_FUNC__ = ["_reset", "_scheduler", "_insert", "_refresh", "_delete"]
 
     def __init__(
@@ -49,19 +50,35 @@ class Scheduler(ABC, threading.Thread):
         self.upload = upload
 
         self.callback_register = callback_register
-
-        self.working_seeds = dict()
+        self.lock = threading.Lock()
 
     def is_empty(self):
-        if self.new.length == 0 and self.todo.length == 0 and self.done.length == 0 and self.upload.length == 0:
-            return True
-        else:
-            return False
-
-    def remove_working_seeds(self, seeds: list = None):
-        for seed in seeds:
-            if seed in self.working_seeds:
-                self.working_seeds.pop(seed)
+        return all(queue.empty() for queue in (self.new, self.todo, self.done, self.upload))
+
+    def set_working_items(self, item_info: dict = None):
+        if not item_info:
+            return
+        with self.lock:
+            self.__WORKING_ITEMS__.update(item_info)
+
+    def get_working_items(self) -> dict:
+        with self.lock:
+            return self.__WORKING_ITEMS__.copy()
+
+    def remove_working_items(self, items: list[str] = None):
+        if not items:
+            return
+        with self.lock:
+            for item in items:
+                self.__WORKING_ITEMS__.pop(item, None)
+
+    def get_working_items_count(self) -> int:
+        with self.lock:
+            return len(self.__WORKING_ITEMS__)
+
+    def clear_working_items(self):
+        with self.lock:
+            self.__WORKING_ITEMS__.clear()
 
     @abstractmethod
     def reset(self):

cobweb/schedulers/scheduler_with_redis.py

@@ -25,28 +25,26 @@ class RedisScheduler(Scheduler):
             callback_register: Callable
     ):
         super().__init__(task, project, stop, pause, new, todo, done, upload, callback_register)
-        self.todo_key = "{%s:%s}:todo" % (project, task)
-        self.done_key = "{%s:%s}:done" % (project, task)
-        self.fail_key = "{%s:%s}:fail" % (project, task)
-        self.heartbeat_key = "heartbeat:%s_%s" % (project, task)
-        self.speed_control_key = "speed_control:%s_%s" % (project, task)
-        self.reset_lock_key = "lock:reset:%s_%s" % (project, task)
-        # self.redis_queue_empty_event = threading.Event()
+        self.todo_key = f"{{{project}:{task}}}:todo"
+        self.done_key = f"{{{project}:{task}}}:done"
+        self.fail_key = f"{{{project}:{task}}}:fail"
+        self.heartbeat_key = f"heartbeat:{project}_{task}"
+        self.speed_control_key = f"speed_control:{project}_{task}"
+        self.reset_lock_key = f"lock:reset:{project}_{task}"
         self.db = ApiDB() if use_api else RedisDB()
 
-    @check_pause
     def reset(self):
         """
         检查过期种子，重新添加到redis缓存中
         """
-        reset_wait_seconds = 30
-        if self.db.lock(self.reset_lock_key, t=60):
+        while not self.stop.is_set():
+            if self.db.lock(self.reset_lock_key, t=60):
 
-            _min = -int(time.time()) + self.seed_reset_seconds
-            self.db.members(self.todo_key, 0, _min=_min, _max="(0")
-            self.db.delete(self.reset_lock_key)
+                _min = -int(time.time()) + self.seed_reset_seconds
+                self.db.members(self.todo_key, 0, _min=_min, _max="(0")
+                self.db.delete(self.reset_lock_key)
 
-        time.sleep(reset_wait_seconds)
+            time.sleep(60)
 
     @check_pause
     def schedule(self):
@@ -55,109 +53,115 @@ class RedisScheduler(Scheduler):
         """
         if not self.db.zcount(self.todo_key, 0, "(1000"):
             time.sleep(self.scheduler_wait_seconds)
-        elif self.todo.length >= self.todo_queue_size:
+            return
+
+        if self.todo.length >= self.todo_queue_size:
             time.sleep(self.todo_queue_full_wait_seconds)
-        else:
-            members = self.db.members(
-                self.todo_key, int(time.time()),
-                count=self.todo_queue_size,
-                _min=0, _max="(1000"
-            )
-            for member, priority in members:
-                seed = Seed(member, priority=priority)
-                self.working_seeds[seed.to_string] = seed.params.priority
-                self.todo.push(seed)
+            return
+
+        members = self.db.members(
+            self.todo_key, int(time.time()),
+            count=self.todo_queue_size,
+            _min=0, _max="(1000"
+        )
+
+        logger.debug(f"Retrieved {len(members)} seeds from Redis.")
+
+        seeds, item_info = list(), dict()
+        for member, priority in members:
+            seed = Seed(member, priority=priority)
+            item_info[seed.to_string] = seed.params.priority
+            seeds.append(seed)
+
+        self.set_working_items(item_info)
+        self.todo.push(seeds)
 
     @check_pause
     def insert(self):
         """
         添加新种子到redis队列中
         """
-        new_seeds = {}
-        del_seeds = set()
-        status = self.new.length < self.new_queue_max_size
-        for _ in range(self.new_queue_max_size):
-            seed_tuple = self.new.pop()
-            if not seed_tuple:
-                break
-            seed, new_seed = seed_tuple
-            new_seeds[new_seed.to_string] = new_seed.params.priority
-            del_seeds.add(seed)
-        if new_seeds:
-            self.db.zadd(self.todo_key, new_seeds, nx=True)
-        if del_seeds:
-            self.done.push(list(del_seeds))
-        if status:
-            time.sleep(self.new_queue_wait_seconds)
+        seeds, delete_seeds = dict(), set()
+        for seed, new_seed in self.new.iter_items(limit=self.new_queue_max_size):
+            seeds[new_seed.to_string] = new_seed.params.priority
+            delete_seeds.add(seed)
+
+        self.db.zadd(self.todo_key, seeds, nx=True)
+        self.done.push(delete_seeds)
+
+        if self.new.length < self.new_queue_max_size:
+            time.sleep(self.scheduler_wait_seconds)
 
     @check_pause
     def refresh(self):
         """
         刷新doing种子过期时间，防止reset重新消费
         """
-        if self.working_seeds:
+        if item_info := self.get_working_items():
             refresh_time = int(time.time())
-            seeds = {k:-refresh_time - v / 1000 for k, v in self.working_seeds.items()}
-            self.db.zadd(self.todo_key, item=seeds, xx=True)
-        time.sleep(3)
+            seed_info = {k: -refresh_time - v / 1000 for k, v in item_info.items()}
+            self.db.zadd(self.todo_key, seed_info, xx=True)
+            self.set_working_items(seed_info)
+        time.sleep(20)
 
     @check_pause
     def delete(self):
         """
         删除队列种子，根据状态添加至成功或失败队列，移除doing字典种子索引
         """
-        seed_list = []
-        status = self.done.length < self.done_queue_max_size
+        seeds = [seed.to_string for seed in self.done.iter_items(limit=self.done_queue_max_size)]
 
-        for _ in range(self.done_queue_max_size):
-            seed = self.done.pop()
-            if not seed:
-                break
-            seed_list.append(seed.to_string)
+        if seeds:
+            self.db.zrem(name=self.todo_key, *seeds)
+            self.remove_working_items(seeds)
 
-        if seed_list:
-
-            self.db.zrem(self.todo_key, *seed_list)
-            self.remove_working_seeds(seed_list)
-
-        if status:
+        if self.done.length < self.done_queue_max_size:
             time.sleep(self.done_queue_wait_seconds)
 
     def run(self):
         start_time = int(time.time())
 
-        self.callback_register(self.reset, tag="scheduler")
-        self.callback_register(self.insert, tag="scheduler")
-        self.callback_register(self.delete, tag="scheduler")
-        self.callback_register(self.refresh, tag="scheduler")
-        self.callback_register(self.schedule, tag="scheduler")
+        for func in [self.reset, self.insert, self.delete, self.refresh, self.schedule]:
+            self.callback_register(func, tag="scheduler")
 
         while not self.stop.is_set():
-            working_count = len(self.working_seeds.keys())
+            working_count = self.get_working_items_count()
             memory_count = self.db.zcount(self.todo_key, "-inf", "(0")
             todo_count = self.db.zcount(self.todo_key, 0, "(1000")
             all_count = self.db.zcard(self.todo_key)
-            
-            if self.is_empty():
-                if self.pause.is_set():
-                    execute_time = int(time.time()) - start_time
-                    if not self.task_model and execute_time > self.before_scheduler_wait_seconds:
-                        logger.info("Done! ready to close thread...")
-                        self.stop.set()
-                    elif todo_count:
-                        logger.info(f"Recovery {self.task} task run！todo seeds count: {todo_count}, queue length: {all_count}")
-                        self.pause.clear()
-                    else:
-                        logger.info("pause! waiting for resume...")
-                elif all_count:
-                    logger.info(f"todo seeds count: {todo_count}, queue length: {all_count}")
+
+            if self.pause.is_set():
+                execute_time = int(time.time()) - start_time
+                if not self.task_model and execute_time > self.before_scheduler_wait_seconds:
+                    logger.info("Done! ready to close thread...")
+                    self.stop.set()
+                elif todo_count:
+                    logger.info(
+                        f"Recovery {self.task} task run！Todo seeds count: {todo_count}, queue length: {all_count}")
                     self.pause.clear()
                 else:
-                    logger.info("TODO queue is empty! pause set...")
-                    self.pause.set()
-            else:
-                if self.pause.is_set():
+                    logger.info("Pause! waiting for resume...")
+
+            elif self.is_empty():
+
+                if all_count:
+                    logger.info(f"Todo seeds count: {todo_count}, queue length: {all_count}")
                     self.pause.clear()
+                else:
+                    count = 0
+                    for _ in range(3):
+                        if not all_count:
+                            count += 1
+                            time.sleep(5)
+                            logger.info("Checking count...")
+                        else:
+                            break
+                    if count >= 3:
+                        logger.info("Todo queue is empty! Pause set...")
+                        self.clear_working_items()
+                        self.pause.set()
+
+            else:
                 logger.info(LogTemplate.launcher_pro_polling.format(
                     task=self.task,
                     doing_len=working_count,

cobweb/setting.py

@@ -43,12 +43,12 @@ TASK_MODEL = 0
 # 流控措施, 0:关闭，1:开启
 SPEED_CONTROL = 1
 
-
-# scheduler redis config
-# os.getenv("REDIS_HOST", "127.0.0.1")
-# os.getenv("REDIS_PASSWORD")
-# os.getenv("REDIS_PORT", 6379)
-# os.getenv("REDIS_DB", 0)
+# redis config
+REDIS_CONFIG = {
+    "host": "127.0.0.1",
+    "port": 6379,
+    "db": 0
+}
 
 # loghub pipeline config
 # os.getenv("LOGHUB_ENDPOINT"),

cobweb/utils/decorators.py

@@ -1,6 +1,5 @@
 import time
 from functools import wraps
-
 from cobweb.base import logger
 
 
@@ -38,6 +37,6 @@ def check_pause(func):
                 logger.info(f"{func.__name__}: " + str(e))
             finally:
                 time.sleep(0.1)
-        logger.info(f"pause: {func.__name__} thread close ...")
+        logger.info(f"Pause detected: {func.__name__} thread closing...")
 
     return wrapper

{cobweb_launcher-3.1.13.dist-info → cobweb_launcher-3.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.13
+Version: 3.1.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
@@ -32,12 +32,12 @@ pip3 install --upgrade cobweb-launcher
 ```
 ## 使用方法介绍
 ### 1. 任务创建
-- LauncherAir任务创建
+- Launcher任务创建
 ```python
-from cobweb import LauncherAir
+from cobweb import Launcher
 
 # 创建启动器
-app = LauncherAir(task="test", project="test")
+app = Launcher(task="test", project="test")
 
 # 设置采集种子
 app.SEEDS = [{
@@ -47,29 +47,15 @@ app.SEEDS = [{
 # 启动任务
 app.start()
 ```
-- LauncherPro任务创建  
-LauncherPro依赖redis实现分布式调度，使用LauncherPro启动器需要完成环境变量的配置或自定义setting文件中的redis配置，如何配置查看`2. 自定义配置文件参数`
-```python
-from cobweb import LauncherPro
-
-# 创建启动器
-app = LauncherPro(
-    task="test",
-    project="test"
-)
-...
-# 启动任务
-app.start()
-```
 ### 2. 自定义配置文件参数
 - 通过自定义setting文件，配置文件导入字符串方式  
 > 默认配置文件：import cobweb.setting  
 > 不推荐！！！目前有bug，随缘使用...
 例如：同级目录下自定义创建了setting.py文件。
 ```python
-from cobweb import LauncherAir
+from cobweb import Launcher
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test",
     setting="import setting"
@@ -81,10 +67,10 @@ app.start()
 ```
 - 自定义修改setting中对象值
 ```python
-from cobweb import LauncherPro
+from cobweb import Launcher
 
 # 创建启动器
-app = LauncherPro(
+app = Launcher(
     task="test",
     project="test",
     REDIS_CONFIG = {
@@ -102,10 +88,10 @@ app.start()
 `@app.request`使用装饰器封装自定义请求方法，作用于发生请求前的操作，返回Request对象或继承于BaseItem对象，用于控制请求参数。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Seed, Request, BaseItem
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -130,10 +116,10 @@ app.start()
 `@app.download`使用装饰器封装自定义下载方法，作用于发生请求时的操作，返回Response对象或继承于BaseItem对象，用于控制请求参数。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Request, Response, BaseItem
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -161,14 +147,14 @@ app.start()
 解析方法返回继承于BaseItem的对象，yield返回进行控制数据存储流程。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Seed, Response, BaseItem
 
 class TestItem(BaseItem):
     __TABLE__ = "test_data" # 表名
     __FIELDS__ = "field1, field2, field3" # 字段名
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -190,12 +176,11 @@ app.start()
 >     upload_item = item.to_dict  
 >     upload_item["text"] = item.response.text  
 >     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
-## need deal
+## todo
 - 队列优化完善，使用queue的机制wait()同步各模块执行？
 - 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
 - 去重过滤（布隆过滤器等）
 - 单机防丢失
-- excel、mysql、redis数据完善
 
 > 未更新流程图！！！
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)

{cobweb_launcher-3.1.13.dist-info → cobweb_launcher-3.1.14.dist-info}/RECORD

@@ -1,39 +1,39 @@
 cobweb/__init__.py,sha256=YdBi3uytEFRXan155xU1kKMpiUKUupO2RGeJyXmH0zk,129
 cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=TjL5IyAYZkhj1941RQ4n45zOdWzO8X53WZvVWB0MnPY,1735
+cobweb/setting.py,sha256=rHPQfc4a1xMTbkt3_KXBfUomhYcbTXogsz7ew-QsqHw,1670
 cobweb/base/__init__.py,sha256=c1qwQrpe5atW_OK_Qw9kaiZpVDey3t_nvRwlH7QRIqY,179
-cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
-cobweb/base/item.py,sha256=uEckxV1SXgrFk194oNSHA-NaWiR1f-AGDCZfYGZM7Ys,1310
+cobweb/base/common_queue.py,sha256=hYdaM70KrWjvACuLKaGhkI2VqFCnd87NVvWzmnfIg8Q,1423
+cobweb/base/item.py,sha256=1bS4U_3vzI2jzSSeoEbLoLT_5CfgLPopWiEYtaahbvw,1674
 cobweb/base/logger.py,sha256=Vsg1bD4LXW91VgY-ANsmaUu-mD88hU_WS83f7jX3qF8,2011
-cobweb/base/request.py,sha256=UduMEPqmvYknrUl2M8pHffD5ESfezQIk6wYtX1i4w4Y,2344
+cobweb/base/request.py,sha256=MBYYjWpbRQRulPG0zPbK0DO3LKmScqQ4tBzFXekYkao,2652
 cobweb/base/response.py,sha256=g8e5H0hEiRfqseh3nD7t6a1rhIJYRMV7nI47kqNOd-U,446
-cobweb/base/seed.py,sha256=A-F1urjbE5hYNWTCwq3sUV4nrxlK_RGMoCmjBmIwYsI,3158
+cobweb/base/seed.py,sha256=ddaWCq_KaWwpmPl1CToJlfCxEEnoJ16kjo6azJs9uls,5000
 cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
 cobweb/crawlers/crawler.py,sha256=ZZVZJ17RWuvzUFGLjqdvyVZpmuq-ynslJwXQzdm_UdQ,709
 cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
-cobweb/db/api_db.py,sha256=kMJhQnhdmCewldG5It0Go8Ge9LBqw0eGwyoynVGeCrQ,3015
-cobweb/db/redis_db.py,sha256=t3fvAzOfnfvoVcNUA_EPbiVOHGFWFaleX-nNo3N35hk,7666
+cobweb/db/api_db.py,sha256=qIhEGB-reKPVFtWPIJYFVK16Us32GBgYjgFjcF-V0GM,3036
+cobweb/db/redis_db.py,sha256=X7dUpW50QcmRPjYlYg7b-fXF_fcjuRRk3DBx2ggetXk,7687
 cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
 cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
 cobweb/launchers/__init__.py,sha256=6_v2jd2sgj6YnOB1nPKiYBskuXVb5xpQnq2YaDGJgQ8,100
 cobweb/launchers/distributor.py,sha256=ALvu7MVZLSQPmWJc_FR-UUIlTMv4PAu8q7tt-KzK1v8,6810
-cobweb/launchers/launcher.py,sha256=cV8cG6oyUdqj95mX_02CXEEx_Whx61mVfe6cRSqLtOo,5440
+cobweb/launchers/launcher.py,sha256=L75eYKemPVqT0cuwfBy_Vh0CObWilDpJ9ibD29g5L38,5742
 cobweb/launchers/uploader.py,sha256=5Hm1pmco8PsFrtBDRN9aw6IjAElfX2wdN1yaILtp03w,2059
 cobweb/pipelines/__init__.py,sha256=rtkaaCZ4u1XcxpkDLHztETQjEcLZ_6DXTHjdfcJlyxQ,97
 cobweb/pipelines/pipeline.py,sha256=OgSEZ2DdqofpZcer1Wj1tuBqn8OHVjrYQ5poqt75czQ,357
 cobweb/pipelines/pipeline_csv.py,sha256=TFqxqgVUqkBF6Jott4zd6fvCSxzG67lpafRQtXPw1eg,807
 cobweb/pipelines/pipeline_loghub.py,sha256=zwIa_pcWBB2UNGd32Cu-i1jKGNruTbo2STdxl1WGwZ0,1829
 cobweb/schedulers/__init__.py,sha256=LEya11fdAv0X28YzbQTeC1LQZ156Fj4cyEMGqQHUWW0,49
-cobweb/schedulers/scheduler.py,sha256=Uo_BCJBvBJN2rzFQyCJXZca3_NHFAegGV8YRyTKtBgE,2276
-cobweb/schedulers/scheduler_with_redis.py,sha256=jPP4FkpjCYlTdMrkl3VE4C_7x1_V8q30UWjSfamuNAQ,6333
+cobweb/schedulers/scheduler.py,sha256=sFlxQY-ehd-Lbskvr21v0wjkb88bs551djAY96Bz1Qs,2777
+cobweb/schedulers/scheduler_with_redis.py,sha256=x68CVJqcMkoUD5_AraG7lcJ6M1UdMCjt1e3i1K8Ea40,6286
 cobweb/utils/__init__.py,sha256=TRFJyyBjaQH_sejU6G_msOeHpjc3ZXU0dUOO5GQfknM,171
 cobweb/utils/bloom.py,sha256=A8xqtHXp7jgRoBuUlpovmq8lhU5y7IEF0FOCjfQDb6s,1855
-cobweb/utils/decorators.py,sha256=066JCY_RNMr2mXkhEv8XTtOOKkv9CFiBm0ZNCcC-2ag,1131
+cobweb/utils/decorators.py,sha256=ZwVQlz-lYHgXgKf9KRCp15EWPzTDdhoikYUNUCIqNeM,1140
 cobweb/utils/dotting.py,sha256=L-jGSApdnFIP4jUWH6p5qIme0aJ1vyDrxAx8wOJWvcs,1960
 cobweb/utils/oss.py,sha256=wmToIIVNO8nCQVRmreVaZejk01aCWS35e1NV6cr0yGI,4192
 cobweb/utils/tools.py,sha256=14TCedqt07m4z6bCnFAsITOFixeGr8V3aOKk--L7Cr0,879
-cobweb_launcher-3.1.13.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-3.1.13.dist-info/METADATA,sha256=1KaV4nUFU8lX-zYaT-DURpNvbjPZySGOsGOVwcwnUsA,6459
-cobweb_launcher-3.1.13.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-3.1.13.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-3.1.13.dist-info/RECORD,,
+cobweb_launcher-3.1.14.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-3.1.14.dist-info/METADATA,sha256=OX4pI6FO7KD_XCkpBj2efE674q2E2PVpem_Xnaq2Z_0,5998
+cobweb_launcher-3.1.14.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+cobweb_launcher-3.1.14.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-3.1.14.dist-info/RECORD,,