cobweb-launcher 1.0.5__py3-none-any.whl → 3.2.18__py3-none-any.whl

cobweb/base/task_queue.py

@@ -0,0 +1,180 @@
+import time
+import threading
+from enum import Enum
+from hashlib import md5
+from dataclasses import dataclass
+from typing import Dict, Any, Optional, List
+
+
+class Status(Enum):
+    PENDING = 0     # 待处理
+    PROCESSING = 1  # 处理中
+    FINISHED = 2    # 已完成
+    INSERT = 3      # 新增
+    UPLOAD = 4     # 上传
+
+
+@dataclass
+class Task:
+    task_id: str         # 种子唯一ID
+    data: Any            # 种子内容
+    status: Status       # 当前状态
+    priority: int            # 优先级（数值越小越优先）
+    created_at: float        # 创建时间戳
+    parent_id: Optional[str] = None   # 父种子 ID
+    children_ids: List[str] = None    # 子种子 ID 列表
+    ttl_seconds: Optional[int] = None  # 可选 TTL 时间（秒）
+
+    def __post_init__(self):
+        if self.children_ids is None:
+            self.children_ids = []
+
+
+class TaskQueue:
+
+    def __init__(self, cleanup_interval=60):
+        self._tasks: Dict[str, Task] = {}
+        self._lock = threading.Lock()
+        # self.cleanup_interval = cleanup_interval
+        # self._start_cleanup_task()
+
+    # def _start_cleanup_task(self):
+    #     """启动后台线程清理过期种子"""
+    #     def run():
+    #         while True:
+    #             time.sleep(self.cleanup_interval)
+    #             self._cleanup_expired_seeds()
+    #     threading.Thread(target=run, daemon=True).start()
+
+    def length(self) -> int:
+        with self._lock:
+            return len(self._tasks)
+
+    def status_length(self, status) -> int:
+        with self._lock:
+            return len([it for it in self._tasks.values() if it.status == status])
+
+    def get_task(self, task_id) -> Task:
+        with self._lock:
+            if task_id in self._tasks:
+                return self._tasks[task_id]
+
+    def get_task_by_status(self, status: list, limit: int = None) -> List[Task]:
+        with self._lock:
+            if not isinstance(status, list):
+                status = [status]
+            task_list = [it for it in self._tasks.values() if it.status in status]
+            task_list.sort(key=lambda x: (x.priority, x.created_at))
+            return task_list[:limit] if limit else task_list
+
+    def get_pending_task(self) -> Task:
+        with self._lock:
+            if items := [it for it in self._tasks.values() if it.status == Status.PENDING]:
+                items.sort(key=lambda x: (x.priority, x.created_at))
+                task_item = items[0]
+                task_item.status = Status.PROCESSING
+                self._tasks[task_item.task_id] = task_item
+                return task_item
+
+    def pop_task(self, status) -> Task:
+        with self._lock:
+            if items := [it for it in self._tasks.values() if it.status == status]:
+                items.sort(key=lambda x: (x.priority, x.created_at))
+                task_item = items[0]
+
+                to_remove = set()
+                queue = [task_item.task_id]
+
+                while queue:
+                    current = queue.pop(0)
+                    if current in self._tasks:
+                        to_remove.add(current)
+                        queue.extend(self._tasks[current].children_ids)
+                        del self._tasks[current]
+
+                for tid in to_remove:
+                    if task_item := self._tasks.get(tid):
+                        if task_item.parent_id in self._tasks:
+                            if tid in self._tasks[task_item.parent_id].children_ids:
+                                self._tasks[task_item.parent_id].children_ids.remove(tid)
+
+    def add_task(
+            self,
+            task_id: str = None,
+            data: Any = None,
+            status=Status.PENDING,
+            priority: int = 500,
+            parent_id: Optional[str] = None,
+            ttl_seconds: Optional[int] = None
+    ) -> bool:
+        """添加新种子，可指定父种子"""
+        with self._lock:
+            if not task_id:
+                task_id = md5(str(time.time()).encode()).hexdigest()
+
+            if task_id in self._tasks:
+                return False  # 防止重复添加
+
+            task_item = Task(
+                task_id=task_id,
+                data=data,
+                status=status,
+                priority=priority,
+                created_at=int(time.time()),
+                parent_id=parent_id,
+                ttl_seconds=ttl_seconds
+            )
+            self._tasks[task_id] = task_item
+
+            if parent_id and parent_id in self._tasks:
+                self._tasks[parent_id].children_ids.append(task_id)
+
+            return True
+
+    def update_task(self, task_id, status, data=None) -> Task:
+        with self._lock:
+            task_item = self._tasks[task_id]
+            task_item.status = status
+            if data:
+                task_item.data = data
+
+            if task_item.status != Status.FINISHED:
+                for tid in task_item.children_ids:
+                    if self._tasks[tid].status == Status.INSERT:
+                        del self._tasks[tid]
+
+            task_item.children_ids = []
+            self._tasks[task_id] = task_item
+
+            return task_item
+
+    def remove(self, task_ids: list) -> bool:
+        with self._lock:
+            for task_id in task_ids:
+                if task_item := self._tasks.get(task_id):
+
+                    if task_item.children_ids:
+                        continue
+
+                    if task_item.parent_id in self._tasks:
+                        if task_id in self._tasks[task_item.parent_id].children_ids:
+                            self._tasks[task_item.parent_id].children_ids.remove(task_id)
+
+                    del self._tasks[task_id]
+
+    def count_children(self, task_id: str) -> int:
+        with self._lock:
+            if task_id in self._tasks:
+                return len(self._tasks[task_id].children_ids)
+            return 0
+
+    # def _cleanup_expired_seeds(self):
+    #     now = time.time()
+    #     expired_ids = []
+    #     with self._lock:
+    #         for seed_id, seed in self._seeds.items():
+    #             if seed.ttl_seconds and now - seed.created_at > seed.ttl_seconds:
+    #                 expired_ids.append(seed_id)
+    #         for seed_id in expired_ids:
+    #             self._seeds[seed_id] = self._seeds[seed_id]._replace(status=SeedStatus.EXPIRED)
+    #         print(f"清理了 {len(expired_ids)} 个过期种子")

cobweb/base/test.py

@@ -0,0 +1,257 @@
+import requests
+from urllib.parse import urlparse
+from typing import Dict, Optional
+
+
+class FileTypeDetector:
+    def __init__(self):
+        self.file_signatures = {
+            # 图片格式
+            b'\x89PNG\r\n\x1a\n': 'PNG',
+            b'\xff\xd8\xff': 'JPEG',
+            b'GIF87a': 'GIF',
+            b'GIF89a': 'GIF',
+            b'RIFF': 'WEBP',  # 需要进一步检查
+            b'BM': 'BMP',
+            b'II*\x00': 'TIFF',
+            b'MM\x00*': 'TIFF',
+            b'\x00\x00\x01\x00': 'ICO',
+            b'\x00\x00\x02\x00': 'CUR',
+
+            # 视频格式
+            b'\x00\x00\x00\x18ftypmp4': 'MP4',
+            b'\x00\x00\x00\x20ftypM4V': 'M4V',
+            b'FLV\x01': 'FLV',
+            b'\x1aE\xdf\xa3': 'WEBM',
+            b'RIFF': 'AVI',  # 需要进一步检查
+            b'\x00\x00\x01\xba': 'MPEG',
+            b'\x00\x00\x01\xb3': 'MPEG',
+            b'OggS': 'OGV',
+
+            # 音频格式
+            b'ID3': 'MP3',
+            b'\xff\xfb': 'MP3',
+            b'\xff\xf3': 'MP3',
+            b'\xff\xf2': 'MP3',
+            b'fLaC': 'FLAC',
+            b'RIFF': 'WAV',  # 需要进一步检查
+            b'OggS': 'OGG',  # 需要进一步检查
+            b'ftypM4A': 'M4A',
+            b'MAC ': 'APE',
+
+            # 其他格式
+            b'%PDF': 'PDF',
+            b'PK\x03\x04': 'ZIP',
+            b'Rar!\x1a\x07\x00': 'RAR',
+            b'\x37\x7a\xbc\xaf\x27\x1c': '7Z',
+        }
+
+        # 扩展名映射
+        self.extension_map = {
+            # 图片
+            '.jpg': 'JPEG', '.jpeg': 'JPEG', '.png': 'PNG', '.gif': 'GIF',
+            '.webp': 'WEBP', '.bmp': 'BMP', '.tiff': 'TIFF', '.tif': 'TIFF',
+            '.ico': 'ICO', '.svg': 'SVG', '.heic': 'HEIC', '.avif': 'AVIF',
+
+            # 视频
+            '.mp4': 'MP4', '.avi': 'AVI', '.mov': 'MOV', '.wmv': 'WMV',
+            '.flv': 'FLV', '.webm': 'WEBM', '.mkv': 'MKV', '.m4v': 'M4V',
+            '.mpg': 'MPEG', '.mpeg': 'MPEG', '.3gp': '3GP', '.ogv': 'OGV',
+            '.ts': 'TS', '.mts': 'MTS', '.vob': 'VOB',
+
+            # 音频
+            '.mp3': 'MP3', '.wav': 'WAV', '.flac': 'FLAC', '.aac': 'AAC',
+            '.ogg': 'OGG', '.wma': 'WMA', '.m4a': 'M4A', '.ape': 'APE',
+            '.opus': 'OPUS', '.aiff': 'AIFF', '.au': 'AU',
+        }
+
+        # MIME类型映射
+        self.mime_type_map = {
+            # 图片
+            'image/jpeg': 'JPEG', 'image/png': 'PNG', 'image/gif': 'GIF',
+            'image/webp': 'WEBP', 'image/bmp': 'BMP', 'image/tiff': 'TIFF',
+            'image/svg+xml': 'SVG', 'image/x-icon': 'ICO',
+
+            # 视频
+            'video/mp4': 'MP4', 'video/avi': 'AVI', 'video/quicktime': 'MOV',
+            'video/x-msvideo': 'AVI', 'video/webm': 'WEBM', 'video/x-flv': 'FLV',
+            'video/3gpp': '3GP', 'video/ogg': 'OGV',
+
+            # 音频
+            'audio/mpeg': 'MP3', 'audio/wav': 'WAV', 'audio/flac': 'FLAC',
+            'audio/aac': 'AAC', 'audio/ogg': 'OGG', 'audio/x-ms-wma': 'WMA',
+            'audio/mp4': 'M4A', 'audio/opus': 'OPUS',
+        }
+
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+        })
+
+    def get_file_extension(self, url: str) -> str:
+        """从URL获取文件扩展名"""
+        parsed = urlparse(url)
+        path = parsed.path.lower()
+        site = parsed.netloc
+
+        # 移除查询参数
+        if '?' in path:
+            path = path.split('?')[0]
+
+        # 获取扩展名
+        if '.' in path:
+            return '.' + path.split('.')[-1], site
+        return '', site
+
+    def detect_by_extension(self, url: str) -> Optional[str]:
+        """通过文件扩展名检测类型"""
+        ext, site = self.get_file_extension(url)
+        return self.extension_map.get(ext)
+
+    def detect_by_mime_type(self, content_type: str) -> Optional[str]:
+        """通过MIME类型检测"""
+        if not content_type:
+            return None
+
+        # 清理content-type，移除参数
+        mime_type = content_type.split(';')[0].strip().lower()
+        return self.mime_type_map.get(mime_type)
+
+    def get_partial_content(self, url: str, max_bytes: int = 64) -> Optional[bytes]:
+        """获取文件的前几个字节"""
+        try:
+            headers = {'Range': f'bytes=0-{max_bytes - 1}'}
+            response = self.session.get(url, headers=headers, timeout=10)
+
+            if response.status_code in [200, 206]:
+                return response.content
+        except Exception as e:
+            print(f"获取内容失败: {e}")
+        return None
+
+    def detect_by_signature(self, data: bytes) -> Optional[str]:
+        """通过文件签名检测类型"""
+        if not data:
+            return None
+
+        # 检查各种文件签名
+        for signature, file_type in self.file_signatures.items():
+            if data.startswith(signature):
+                # 特殊处理需要进一步检查的格式
+                if signature == b'RIFF' and len(data) >= 12:
+                    # 检查是WEBP、AVI还是WAV
+                    if data[8:12] == b'WEBP':
+                        return 'WEBP'
+                    elif data[8:12] == b'AVI ':
+                        return 'AVI'
+                    elif data[8:12] == b'WAVE':
+                        return 'WAV'
+                elif signature == b'OggS' and len(data) >= 32:
+                    # 检查是OGG音频还是OGV视频
+                    if b'vorbis' in data[:64].lower():
+                        return 'OGG'
+                    elif b'theora' in data[:64].lower():
+                        return 'OGV'
+                    else:
+                        return 'OGG'
+                else:
+                    return file_type
+
+        # 检查MP4相关格式
+        if len(data) >= 12 and data[4:8] == b'ftyp':
+            brand = data[8:12]
+            if brand in [b'mp41', b'mp42', b'isom', b'avc1']:
+                return 'MP4'
+            elif brand == b'M4A ':
+                return 'M4A'
+            elif brand == b'M4V ':
+                return 'M4V'
+            elif brand == b'qt  ':
+                return 'MOV'
+
+        return None
+
+    def get_detailed_info(self, url, content_type, data) -> Dict:
+        """获取详细的文件信息"""
+        result = {
+            'url': url,
+            'site': None,
+            'detected_type': None,
+            'confidence': 'unknown',
+            'methods_used': [],
+            'content_type': content_type,
+            'extension': None
+        }
+
+        # 1. 先尝试HEAD请求获取HTTP头信息
+        try:
+            result['content_type'] = content_type
+            # result['file_size'] = content_length
+
+            # 通过MIME类型检测
+            mime_detected = self.detect_by_mime_type(content_type)
+            if mime_detected:
+                result['detected_type'] = mime_detected
+                result['confidence'] = 'high'
+                result['methods_used'].append('mime_type')
+        except Exception as e:
+            print(f"HEAD请求失败: {e}")
+
+        # 2. 通过扩展名检测
+        ext_detected = self.detect_by_extension(url)
+        result['extension'], result['site'] = self.get_file_extension(url)
+
+        if ext_detected:
+            if not result['detected_type']:
+                result['detected_type'] = ext_detected
+                result['confidence'] = 'medium'
+            elif result['detected_type'] == ext_detected:
+                result['confidence'] = 'very_high'  # MIME和扩展名一致
+            result['methods_used'].append('extension')
+
+        # 3. 如果前两种方法不确定，使用文件签名检测
+        if result['confidence'] in ['unknown', 'medium']:
+            signature_detected = self.detect_by_signature(data)
+            if signature_detected:
+                if not result['detected_type']:
+                    result['detected_type'] = signature_detected
+                    result['confidence'] = 'high'
+                elif result['detected_type'] == signature_detected:
+                    result['confidence'] = 'very_high'
+                else:
+                    # 冲突时，优先相信文件签名
+                    result['detected_type'] = signature_detected
+                    result['confidence'] = 'high'
+                result['methods_used'].append('file_signature')
+
+        return result
+
+    def detect_file_type(self, url: str) -> str:
+        """简单的文件类型检测，返回类型字符串"""
+        info = self.get_detailed_info(url)
+        return info.get('detected_type', 'Unknown')
+
+    def get_file_category(self, file_type: str) -> str:
+        """获取文件类别"""
+        if not file_type or file_type == 'Unknown':
+            return 'Unknown'
+
+        image_types = {'PNG', 'JPEG', 'GIF', 'WEBP', 'BMP', 'TIFF', 'ICO', 'SVG', 'HEIC', 'AVIF'}
+        video_types = {'MP4', 'AVI', 'MOV', 'WMV', 'FLV', 'WEBM', 'MKV', 'M4V', 'MPEG', '3GP', 'OGV', 'TS', 'MTS',
+                       'VOB'}
+        audio_types = {'MP3', 'WAV', 'FLAC', 'AAC', 'OGG', 'WMA', 'M4A', 'APE', 'OPUS', 'AIFF', 'AU'}
+
+        if file_type in image_types:
+            return 'Image'
+        elif file_type in video_types:
+            return 'Video'
+        elif file_type in audio_types:
+            return 'Audio'
+        else:
+            return 'Other'
+
+
+# if __name__ == "__main__":
+#     detector = FileTypeDetector()
+#     result = detector.get_detailed_info("https://cdn.pixabay.com/user/2024/12/10/12-18-33-812_96x96.jpeg")
+#     print(result)

cobweb/constant.py

@@ -1,3 +1,11 @@
+from enum import Enum
+
+
+class CrawlerModel:
+
+    default = "cobweb.crawlers.Crawler"
+    file_air = "cobweb.crawlers.FileCrawlerAir"
+    file_pro = "cobweb.crawlers.FileCrawlerPro"
 
 
 class LauncherModel:
@@ -22,12 +30,37 @@ class DealModel:
     poll = "deal model: poll"
 
 
+class ResponseStatus(Enum):
+    failed = "failed"
+    succeed = "succeed"
+    filter = "filter"
+    max_retry = "max retry"
+
+
 class LogTemplate:
 
+    console_item = """
+    ----------------------- start - console pipeline -----------------
+            种子详情 \n{seed_detail}
+            解析详情 \n{parse_detail}
+    ----------------------- end  - console pipeline ------------------
+    """
+
+    launcher_air_polling = """
+    ----------------------- start - 轮训日志: {task} -----------------
+            内存队列 
+                种子数:  {doing_len}
+                待消费:  {todo_len}
+                已消费:  {done_len}
+            存储队列 
+                待上传:  {upload_len}
+    ----------------------- end  - 轮训日志: {task} ------------------
+    """
+
     launcher_pro_polling = """
 ----------------------- start - 轮训日志: {task} -----------------
         内存队列 
-            种子数:  {doing_len}
+            消费中:  {doing_len}
             待消费:  {todo_len}
             已消费:  {done_len}
         redis队列 
@@ -63,4 +96,8 @@ class LogTemplate:
         response 
             status        :    {status} \n{response}
 ------------------------------------------------------------------
-"""
+"""
+
+    @staticmethod
+    def log_info(item: dict) -> str:
+        return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

cobweb/crawlers/__init__.py

@@ -1,2 +1 @@
-from .base_crawler import Crawler
-from .file_crawler import CrawlerAir
+from .crawler import Crawler

cobweb/crawlers/crawler.py

@@ -0,0 +1,27 @@
+from typing import Any, Generator
+from cobweb.base import (
+    Seed,
+    BaseItem,
+    Request,
+    Response,
+    CSVItem,
+)
+
+
+class Crawler:
+
+    @staticmethod
+    def request(seed: Seed) -> Generator[Request, Response, None]:
+        yield Request(seed.url, seed, timeout=5)
+
+    @staticmethod
+    def download(item: Request) -> Generator[Response, Any, None]:
+        response = item.download()
+        yield Response(item.seed, response, **item.to_dict)
+
+    @staticmethod
+    def parse(item: Response) -> Generator[BaseItem, Any, None]:
+        upload_item = item.to_dict
+        upload_item["content"] = getattr(item.response, "text", item.response)
+        yield CSVItem(item.seed, data=upload_item)
+

cobweb/db/__init__.py

@@ -1 +1,2 @@
 from .redis_db import RedisDB
+from .api_db import ApiDB

cobweb/db/api_db.py

@@ -0,0 +1,83 @@
+import os
+import json
+import requests
+
+
+class ApiDB:
+
+    def __init__(self, host=None, **kwargs):
+        self.host = host or os.getenv("REDIS_API_HOST", "http://127.0.0.1:4396")
+
+    def _get_response(self, api, params: dict = None):
+        try:
+            url = self.host + api
+            response = requests.get(url, params=params)
+            json_data = response.json()
+            response.close()
+            return json_data["data"]
+        except:
+            return None
+
+    def _post_response(self, api, params: dict = None, data: dict = None):
+        try:
+            url = self.host + api
+            headers = {"Content-Type": "application/json"}
+            response = requests.post(url, headers=headers, params=params, data=json.dumps(data))
+            json_data = response.json()
+            response.close()
+            return json_data["data"]
+        except:
+            return None
+
+    def get(self, name):
+        return self._get_response(api="/get", params=dict(name=name))
+
+    def setnx(self, name, value=""):
+        return self._get_response(api="/setnx", params=dict(name=name, value=value))
+
+    def setex(self, name, t, value=""):
+        return self._get_response(api="/setex", params=dict(name=name, value=value, t=t))
+
+    def expire(self, name, t, nx: bool = False, xx: bool = False, gt: bool = False, lt: bool = False):
+        return self._get_response(api="/expire", params=dict(name=name, t=t, nx=nx, xx=xx, gt=gt, lt=lt))
+
+    def ttl(self, name):
+        return self._get_response(api="/ttl", params=dict(name=name))
+
+    def delete(self, name):
+        return self._get_response(api="/delete", params=dict(name=name))
+
+    def exists(self, name):
+        return self._get_response(api="/exists", params=dict(name=name))
+
+    def incrby(self, name, value):
+        return self._get_response(api="/incrby", params=dict(name=name, value=value))
+
+    def zcard(self, name) -> bool:
+        return self._get_response(api="/zcard", params=dict(name=name))
+
+    def zadd(self, name, item: dict, **kwargs):
+        if item:
+            return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
+
+    def zrem(self, name, *values):
+        return self._post_response(api="/zrem", data=dict(name=name, values=values))
+
+    def zcount(self, name, _min, _max):
+        return self._get_response(api="/zcount", params=dict(name=name, min=_min, max=_max))
+
+    def lock(self, name, t=15) -> bool:
+        return self._get_response(api="/lock", params=dict(name=name, t=t))
+
+    def auto_incr(self, name, t=15, limit=1000) -> bool:
+        return self._get_response(api="/auto_incr", params=dict(name=name, t=t, limit=limit))
+
+    def members(self, name, score, start=0, count=1000, _min="-inf", _max="+inf"):
+        return self._get_response(api="/members", params=dict(name=name, score=score, start=start, count=count, min=_min, max=_max))
+
+    def done(self, name: list, *values):
+        return self._post_response(api="/done", data=dict(name=name, values=values))
+
+
+
+