cobweb-launcher 3.1.13__tar.gz → 3.1.14__tar.gz

{cobweb-launcher-3.1.13 → cobweb-launcher-3.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cobweb-launcher
-Version: 3.1.13
+Version: 3.1.14
 Summary: spider_hole
 Home-page: https://github.com/Juannie-PP/cobweb
 Author: Juannie-PP
@@ -29,12 +29,12 @@ pip3 install --upgrade cobweb-launcher
 ```
 ## 使用方法介绍
 ### 1. 任务创建
-- LauncherAir任务创建
+- Launcher任务创建
 ```python
-from cobweb import LauncherAir
+from cobweb import Launcher
 
 # 创建启动器
-app = LauncherAir(task="test", project="test")
+app = Launcher(task="test", project="test")
 
 # 设置采集种子
 app.SEEDS = [{
@@ -44,29 +44,15 @@ app.SEEDS = [{
 # 启动任务
 app.start()
 ```
-- LauncherPro任务创建  
-LauncherPro依赖redis实现分布式调度，使用LauncherPro启动器需要完成环境变量的配置或自定义setting文件中的redis配置，如何配置查看`2. 自定义配置文件参数`
-```python
-from cobweb import LauncherPro
-
-# 创建启动器
-app = LauncherPro(
-    task="test",
-    project="test"
-)
-...
-# 启动任务
-app.start()
-```
 ### 2. 自定义配置文件参数
 - 通过自定义setting文件，配置文件导入字符串方式  
 > 默认配置文件：import cobweb.setting  
 > 不推荐！！！目前有bug，随缘使用...
 例如：同级目录下自定义创建了setting.py文件。
 ```python
-from cobweb import LauncherAir
+from cobweb import Launcher
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test",
     setting="import setting"
@@ -78,10 +64,10 @@ app.start()
 ```
 - 自定义修改setting中对象值
 ```python
-from cobweb import LauncherPro
+from cobweb import Launcher
 
 # 创建启动器
-app = LauncherPro(
+app = Launcher(
     task="test",
     project="test",
     REDIS_CONFIG = {
@@ -99,10 +85,10 @@ app.start()
 `@app.request`使用装饰器封装自定义请求方法，作用于发生请求前的操作，返回Request对象或继承于BaseItem对象，用于控制请求参数。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Seed, Request, BaseItem
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -127,10 +113,10 @@ app.start()
 `@app.download`使用装饰器封装自定义下载方法，作用于发生请求时的操作，返回Response对象或继承于BaseItem对象，用于控制请求参数。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Request, Response, BaseItem
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -158,14 +144,14 @@ app.start()
 解析方法返回继承于BaseItem的对象，yield返回进行控制数据存储流程。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Seed, Response, BaseItem
 
 class TestItem(BaseItem):
     __TABLE__ = "test_data" # 表名
     __FIELDS__ = "field1, field2, field3" # 字段名
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -187,12 +173,11 @@ app.start()
 >     upload_item = item.to_dict  
 >     upload_item["text"] = item.response.text  
 >     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
-## need deal
+## todo
 - 队列优化完善，使用queue的机制wait()同步各模块执行？
 - 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
 - 去重过滤（布隆过滤器等）
 - 单机防丢失
-- excel、mysql、redis数据完善
 
 > 未更新流程图！！！
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)

{cobweb-launcher-3.1.13 → cobweb-launcher-3.1.14}/README.md

@@ -14,12 +14,12 @@ pip3 install --upgrade cobweb-launcher
 ```
 ## 使用方法介绍
 ### 1. 任务创建
-- LauncherAir任务创建
+- Launcher任务创建
 ```python
-from cobweb import LauncherAir
+from cobweb import Launcher
 
 # 创建启动器
-app = LauncherAir(task="test", project="test")
+app = Launcher(task="test", project="test")
 
 # 设置采集种子
 app.SEEDS = [{
@@ -29,29 +29,15 @@ app.SEEDS = [{
 # 启动任务
 app.start()
 ```
-- LauncherPro任务创建  
-LauncherPro依赖redis实现分布式调度，使用LauncherPro启动器需要完成环境变量的配置或自定义setting文件中的redis配置，如何配置查看`2. 自定义配置文件参数`
-```python
-from cobweb import LauncherPro
-
-# 创建启动器
-app = LauncherPro(
-    task="test",
-    project="test"
-)
-...
-# 启动任务
-app.start()
-```
 ### 2. 自定义配置文件参数
 - 通过自定义setting文件，配置文件导入字符串方式  
 > 默认配置文件：import cobweb.setting  
 > 不推荐！！！目前有bug，随缘使用...
 例如：同级目录下自定义创建了setting.py文件。
 ```python
-from cobweb import LauncherAir
+from cobweb import Launcher
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test",
     setting="import setting"
@@ -63,10 +49,10 @@ app.start()
 ```
 - 自定义修改setting中对象值
 ```python
-from cobweb import LauncherPro
+from cobweb import Launcher
 
 # 创建启动器
-app = LauncherPro(
+app = Launcher(
     task="test",
     project="test",
     REDIS_CONFIG = {
@@ -84,10 +70,10 @@ app.start()
 `@app.request`使用装饰器封装自定义请求方法，作用于发生请求前的操作，返回Request对象或继承于BaseItem对象，用于控制请求参数。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Seed, Request, BaseItem
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -112,10 +98,10 @@ app.start()
 `@app.download`使用装饰器封装自定义下载方法，作用于发生请求时的操作，返回Response对象或继承于BaseItem对象，用于控制请求参数。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Request, Response, BaseItem
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -143,14 +129,14 @@ app.start()
 解析方法返回继承于BaseItem的对象，yield返回进行控制数据存储流程。
 ```python
 from typing import Union
-from cobweb import LauncherAir
+from cobweb import Launcher
 from cobweb.base import Seed, Response, BaseItem
 
 class TestItem(BaseItem):
     __TABLE__ = "test_data" # 表名
     __FIELDS__ = "field1, field2, field3" # 字段名
 
-app = LauncherAir(
+app = Launcher(
     task="test", 
     project="test"
 )
@@ -172,12 +158,11 @@ app.start()
 >     upload_item = item.to_dict  
 >     upload_item["text"] = item.response.text  
 >     yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
-## need deal
+## todo
 - 队列优化完善，使用queue的机制wait()同步各模块执行？
 - 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
 - 去重过滤（布隆过滤器等）
 - 单机防丢失
-- excel、mysql、redis数据完善
 
 > 未更新流程图！！！
 ![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)

cobweb-launcher-3.1.14/cobweb/base/common_queue.py

@@ -0,0 +1,51 @@
+from collections import deque
+from typing import Any, Iterable, Union
+
+
+class Queue:
+    def __init__(self):
+        """初始化队列"""
+        self._queue = deque()
+
+    @property
+    def length(self) -> int:
+        """返回队列长度"""
+        return len(self._queue)
+
+    def empty(self) -> bool:
+        """检查队列是否为空"""
+        return not self._queue
+
+    def push(self, data: Union[Any, Iterable], direct_insertion: bool = False):
+        """
+        向队列中添加数据。
+        如果数据是可迭代对象（如列表、元组或集合），可以选择直接扩展队列。
+        """
+        if not data:
+            return
+
+        if not direct_insertion and isinstance(data, (list, tuple, set)):
+            self._queue.extend(data)
+        else:
+            self._queue.append(data)
+
+    def pop(self) -> Any:
+        """
+        从队列左侧弹出一个元素。
+        如果队列为空，返回 None。
+        """
+        try:
+            return self._queue.popleft()
+        except IndexError:
+            return None
+
+    def iter_items(self, limit: int = 1) -> Iterable:
+        """
+        按指定数量从队列中弹出多个元素并生成它们。
+        如果队列为空或达到限制，则停止生成。
+        """
+        for _ in range(limit):
+            item = self.pop()
+            if item is None:
+                break
+            yield item

cobweb-launcher-3.1.14/cobweb/base/item.py

@@ -0,0 +1,65 @@
+from .seed import Seed
+from typing import Dict, Any
+from collections import namedtuple
+
+
+class ItemMeta(type):
+
+    def __new__(cls, name: str, bases: tuple, dct: dict) -> type:
+        new_class = super().__new__(cls, name, bases, dct)
+        if name != "BaseItem":
+            table = getattr(new_class, "__TABLE__")
+            fields = getattr(new_class, "__FIELDS__")
+            if not table or not fields:
+                raise ValueError(f"Missing required attributes '__TABLE__' or '__FIELDS__' in class {name}")
+            new_class.Data = namedtuple(table, fields)
+        return new_class
+
+
+class BaseItem(metaclass=ItemMeta):
+
+    __TABLE__ = ""
+    __FIELDS__ = ""
+
+    def __init__(self, seed: Seed, **kwargs):
+        self.seed = seed
+
+        data = {}
+        for key, value in kwargs.items():
+            if key in self.__FIELDS__:
+                data[key] = value
+            else:
+                setattr(self, key, value)
+
+        try:
+            self.data = self.Data(**data)
+        except TypeError as e:
+            raise ValueError(f"Invalid field values for Data: {e}") from e
+
+    @property
+    def to_dict(self) -> Dict[str, Any]:
+        return self.data._asdict()
+
+    @property
+    def fields(self) -> tuple[str]:
+        return self.data._fields
+
+    @property
+    def table(self) -> str:
+        return self.Data.__name__
+
+    def __setitem__(self, key: str, value: Any):
+        setattr(self, key, value)
+
+    def __getitem__(self, key: str) -> Any:
+        return getattr(self, key, None)
+
+    def __getattr__(self, name: str) -> Any:
+        return None
+
+
+class CSVItem(BaseItem):
+
+    __TABLE__ = "cobweb"
+    __FIELDS__ = "data"
+

cobweb-launcher-3.1.14/cobweb/base/request.py

@@ -0,0 +1,88 @@
+import random
+import requests
+from typing import Any, Dict
+
+
+class Request:
+    """
+    请求类，用于封装 HTTP 请求并提供相关功能。
+    """
+
+    __REQUEST_ATTRS__ = {
+        "params",
+        "headers",
+        "cookies",
+        "data",
+        "json",
+        "files",
+        "auth",
+        "timeout",
+        "proxies",
+        "hooks",
+        "stream",
+        "verify",
+        "cert",
+        "allow_redirects",
+    }
+
+    def __init__(
+        self,
+        url: str,
+        seed: Any,
+        random_ua: bool = True,
+        check_status_code: bool = True,
+        **kwargs,
+    ):
+        """
+        初始化请求对象。
+        :param url: 请求的 URL。
+        :param seed: 种子对象或标识符。
+        :param random_ua: 是否使用随机 User-Agent，默认为 True。
+        :param check_status_code: 是否检查响应状态码，默认为 True。
+        :param kwargs: 其他扩展参数。
+        """
+        self.url = url
+        self.seed = seed
+        self.check_status_code = check_status_code
+        self.request_setting: Dict[str, Any] = {}
+
+        for key, value in kwargs.items():
+            if key in self.__class__.__REQUEST_ATTRS__:
+                self.request_setting[key] = value
+            else:
+                setattr(self, key, value)
+
+        self.method = getattr(self, "method", None) or (
+            "POST" if self.request_setting.get("data") or self.request_setting.get("json") else "GET"
+        )
+
+        if random_ua:
+            self._build_header()
+
+    @property
+    def _random_ua(self) -> str:
+        v1 = random.randint(4, 15)
+        v2 = random.randint(3, 11)
+        v3 = random.randint(1, 16)
+        v4 = random.randint(533, 605)
+        v5 = random.randint(1000, 6000)
+        v6 = random.randint(10, 80)
+        return (f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_{v1}_{v2}) "
+                f"AppleWebKit/{v4}.{v3} (KHTML, like Gecko) "
+                f"Chrome/105.0.0.0 Safari/{v4}.{v3} Edg/105.0.{v5}.{v6}")
+
+    def _build_header(self):
+        headers = self.request_setting.setdefault("headers", {})
+        if not headers.get("user-agent"):
+            headers["user-agent"] = self._random_ua
+
+    def download(self) -> requests.Response:
+        response = requests.request(self.method, self.url, **self.request_setting)
+        if self.check_status_code:
+            response.raise_for_status()
+        return response
+
+    @property
+    def to_dict(self) -> Dict[str, Any]:
+        excluded_keys = {"url", "seed", "check_status_code", "request_setting"}
+        return {k: v for k, v in self.__dict__.items() if k not in excluded_keys}

cobweb-launcher-3.1.14/cobweb/base/seed.py

@@ -0,0 +1,162 @@
+import json
+import time
+import hashlib
+from typing import Any, Dict, Optional, Union
+
+
+class SeedParams:
+    """
+    定义种子参数类，用于存储种子的元信息。
+    """
+
+    def __init__(
+        self,
+        retry: Optional[int] = None,
+        priority: Optional[int] = None,
+        seed_version: Optional[int] = None,
+        seed_status: Optional[str] = None,
+        proxy_type: Optional[str] = None,
+        proxy: Optional[str] = None,
+    ):
+        self.retry = retry or 0
+        self.priority = priority or 300
+        self.seed_version = seed_version or int(time.time())
+        self.seed_status = seed_status
+        self.proxy_type = proxy_type
+        self.proxy = proxy
+
+    def __getattr__(self, name: str) -> Any:
+        """动态获取未定义的属性，返回 None"""
+        return None
+
+
+class Seed:
+    """
+    种子类，用于表示一个种子对象，包含种子的基本属性和方法。
+    """
+
+    __SEED_PARAMS__ = [
+        "retry",
+        "priority",
+        "seed_version",
+        "seed_status",
+        "proxy_type",
+        "proxy",
+    ]
+
+    def __init__(
+        self,
+        seed: Union[str, bytes, Dict[str, Any]] = None,
+        sid: Optional[str] = None,
+        retry: Optional[int] = None,
+        priority: Optional[int] = None,
+        seed_version: Optional[int] = None,
+        seed_status: Optional[str] = None,
+        proxy_type: Optional[str] = None,
+        proxy: Optional[str] = None,
+        **kwargs,
+    ):
+        """
+        初始化种子对象。
+        :param seed: 种子数据，可以是字符串、字节或字典。
+        :param sid: 种子的唯一标识符。
+        :param retry: 重试次数。
+        :param priority: 优先级。
+        :param seed_version: 种子版本。
+        :param seed_status: 种子状态。
+        :param proxy_type: 代理类型。
+        :param proxy: 代理地址。
+        :param kwargs: 其他扩展参数。
+        """
+        # 初始化种子数据
+        if seed:
+            if isinstance(seed, (str, bytes)):
+                try:
+                    item = json.loads(seed)
+                    self._init_seed(item)
+                except json.JSONDecodeError as e:
+                    raise ValueError(f"Invalid JSON format for seed: {seed}") from e
+            elif isinstance(seed, dict):
+                self._init_seed(seed)
+            else:
+                raise TypeError(f"Seed type error, must be str, bytes, or dict! Seed: {seed}")
+
+        # 初始化种子参数
+        seed_params = {
+            "retry": retry,
+            "priority": priority,
+            "seed_version": seed_version,
+            "seed_status": seed_status,
+            "proxy_type": proxy_type,
+            "proxy": proxy,
+        }
+
+        # 合并扩展参数
+        if kwargs:
+            self._init_seed(kwargs)
+            seed_params.update({k: v for k, v in kwargs.items() if k in self.__SEED_PARAMS__})
+
+        # 初始化唯一标识符
+        if sid or not getattr(self, "sid", None):
+            self._init_id(sid)
+
+        # 设置参数对象
+        self.params = SeedParams(**seed_params)
+
+    def __getattr__(self, name: str) -> Any:
+        """动态获取未定义的属性，返回 None"""
+        return None
+
+    def __setitem__(self, key: str, value: Any):
+        """支持字典式设置属性"""
+        setattr(self, key, value)
+
+    def __getitem__(self, key: str) -> Any:
+        """支持字典式获取属性"""
+        return getattr(self, key, None)
+
+    def __str__(self) -> str:
+        """返回种子的 JSON 字符串表示"""
+        return self.to_string
+
+    def __repr__(self) -> str:
+        """返回种子的调试字符串表示"""
+        attrs = [f"{k}={v}" for k, v in self.__dict__.items()]
+        return f"{self.__class__.__name__}({', '.join(attrs)})"
+
+    def _init_seed(self, seed_info: Dict[str, Any]):
+        """初始化种子数据"""
+        for key, value in seed_info.items():
+            if key not in self.__SEED_PARAMS__:
+                self.__setattr__(key, value)
+
+    def _init_id(self, sid: Optional[str]):
+        """初始化种子的唯一标识符"""
+        if not sid:
+            sid = hashlib.md5(self.to_string.encode()).hexdigest()
+        self.__setattr__("sid", sid)
+
+    @property
+    def to_dict(self) -> Dict[str, Any]:
+        """返回种子的字典表示（不包含 params 属性）"""
+        seed = self.__dict__.copy()
+        seed.pop("params", None)
+        return seed
+
+    @property
+    def to_string(self) -> str:
+        """返回种子的紧凑 JSON 字符串表示"""
+        return json.dumps(
+            self.to_dict,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )
+
+    @property
+    def get_all(self) -> str:
+        """返回种子的所有属性（包括 params）的 JSON 字符串表示"""
+        return json.dumps(
+            self.__dict__,
+            ensure_ascii=False,
+            separators=(",", ":")
+        )

{cobweb-launcher-3.1.13 → cobweb-launcher-3.1.14}/cobweb/db/api_db.py

@@ -57,7 +57,8 @@ class ApiDB:
         return self._get_response(api="/zcard", params=dict(name=name))
 
     def zadd(self, name, item: dict, **kwargs):
-        return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
+        if item:
+            return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
 
     def zrem(self, name, *values):
         return self._post_response(api="/zrem", data=dict(name=name, values=values))

{cobweb-launcher-3.1.13 → cobweb-launcher-3.1.14}/cobweb/db/redis_db.py

@@ -120,7 +120,8 @@ class RedisDB:
     def zadd(self, name, item: dict, **kwargs):
         # with self.get_connection() as client:
         #     return client.zadd(name, item, **kwargs)
-        return self.execute_command("zadd", name, item, **kwargs)
+        if item:
+            return self.execute_command("zadd", name, item, **kwargs)
 
     def zrem(self, name, *value):
         # with self.get_connection() as client: