cobweb-launcher 1.2.25__py3-none-any.whl → 3.2.20__py3-none-any.whl

cobweb/base/task_queue.py

@@ -0,0 +1,180 @@
+import time
+import threading
+from enum import Enum
+from hashlib import md5
+from dataclasses import dataclass
+from typing import Dict, Any, Optional, List
+
+
+class Status(Enum):
+    PENDING = 0     # 待处理
+    PROCESSING = 1  # 处理中
+    FINISHED = 2    # 已完成
+    INSERT = 3      # 新增
+    UPLOAD = 4     # 上传
+
+
+@dataclass
+class Task:
+    task_id: str         # 种子唯一ID
+    data: Any            # 种子内容
+    status: Status       # 当前状态
+    priority: int            # 优先级（数值越小越优先）
+    created_at: float        # 创建时间戳
+    parent_id: Optional[str] = None   # 父种子 ID
+    children_ids: List[str] = None    # 子种子 ID 列表
+    ttl_seconds: Optional[int] = None  # 可选 TTL 时间（秒）
+
+    def __post_init__(self):
+        if self.children_ids is None:
+            self.children_ids = []
+
+
+class TaskQueue:
+
+    def __init__(self, cleanup_interval=60):
+        self._tasks: Dict[str, Task] = {}
+        self._lock = threading.Lock()
+        # self.cleanup_interval = cleanup_interval
+        # self._start_cleanup_task()
+
+    # def _start_cleanup_task(self):
+    #     """启动后台线程清理过期种子"""
+    #     def run():
+    #         while True:
+    #             time.sleep(self.cleanup_interval)
+    #             self._cleanup_expired_seeds()
+    #     threading.Thread(target=run, daemon=True).start()
+
+    def length(self) -> int:
+        with self._lock:
+            return len(self._tasks)
+
+    def status_length(self, status) -> int:
+        with self._lock:
+            return len([it for it in self._tasks.values() if it.status == status])
+
+    def get_task(self, task_id) -> Task:
+        with self._lock:
+            if task_id in self._tasks:
+                return self._tasks[task_id]
+
+    def get_task_by_status(self, status: list, limit: int = None) -> List[Task]:
+        with self._lock:
+            if not isinstance(status, list):
+                status = [status]
+            task_list = [it for it in self._tasks.values() if it.status in status]
+            task_list.sort(key=lambda x: (x.priority, x.created_at))
+            return task_list[:limit] if limit else task_list
+
+    def get_pending_task(self) -> Task:
+        with self._lock:
+            if items := [it for it in self._tasks.values() if it.status == Status.PENDING]:
+                items.sort(key=lambda x: (x.priority, x.created_at))
+                task_item = items[0]
+                task_item.status = Status.PROCESSING
+                self._tasks[task_item.task_id] = task_item
+                return task_item
+
+    def pop_task(self, status) -> Task:
+        with self._lock:
+            if items := [it for it in self._tasks.values() if it.status == status]:
+                items.sort(key=lambda x: (x.priority, x.created_at))
+                task_item = items[0]
+
+                to_remove = set()
+                queue = [task_item.task_id]
+
+                while queue:
+                    current = queue.pop(0)
+                    if current in self._tasks:
+                        to_remove.add(current)
+                        queue.extend(self._tasks[current].children_ids)
+                        del self._tasks[current]
+
+                for tid in to_remove:
+                    if task_item := self._tasks.get(tid):
+                        if task_item.parent_id in self._tasks:
+                            if tid in self._tasks[task_item.parent_id].children_ids:
+                                self._tasks[task_item.parent_id].children_ids.remove(tid)
+
+    def add_task(
+            self,
+            task_id: str = None,
+            data: Any = None,
+            status=Status.PENDING,
+            priority: int = 500,
+            parent_id: Optional[str] = None,
+            ttl_seconds: Optional[int] = None
+    ) -> bool:
+        """添加新种子，可指定父种子"""
+        with self._lock:
+            if not task_id:
+                task_id = md5(str(time.time()).encode()).hexdigest()
+
+            if task_id in self._tasks:
+                return False  # 防止重复添加
+
+            task_item = Task(
+                task_id=task_id,
+                data=data,
+                status=status,
+                priority=priority,
+                created_at=int(time.time()),
+                parent_id=parent_id,
+                ttl_seconds=ttl_seconds
+            )
+            self._tasks[task_id] = task_item
+
+            if parent_id and parent_id in self._tasks:
+                self._tasks[parent_id].children_ids.append(task_id)
+
+            return True
+
+    def update_task(self, task_id, status, data=None) -> Task:
+        with self._lock:
+            task_item = self._tasks[task_id]
+            task_item.status = status
+            if data:
+                task_item.data = data
+
+            if task_item.status != Status.FINISHED:
+                for tid in task_item.children_ids:
+                    if self._tasks[tid].status == Status.INSERT:
+                        del self._tasks[tid]
+
+            task_item.children_ids = []
+            self._tasks[task_id] = task_item
+
+            return task_item
+
+    def remove(self, task_ids: list) -> bool:
+        with self._lock:
+            for task_id in task_ids:
+                if task_item := self._tasks.get(task_id):
+
+                    if task_item.children_ids:
+                        continue
+
+                    if task_item.parent_id in self._tasks:
+                        if task_id in self._tasks[task_item.parent_id].children_ids:
+                            self._tasks[task_item.parent_id].children_ids.remove(task_id)
+
+                    del self._tasks[task_id]
+
+    def count_children(self, task_id: str) -> int:
+        with self._lock:
+            if task_id in self._tasks:
+                return len(self._tasks[task_id].children_ids)
+            return 0
+
+    # def _cleanup_expired_seeds(self):
+    #     now = time.time()
+    #     expired_ids = []
+    #     with self._lock:
+    #         for seed_id, seed in self._seeds.items():
+    #             if seed.ttl_seconds and now - seed.created_at > seed.ttl_seconds:
+    #                 expired_ids.append(seed_id)
+    #         for seed_id in expired_ids:
+    #             self._seeds[seed_id] = self._seeds[seed_id]._replace(status=SeedStatus.EXPIRED)
+    #         print(f"清理了 {len(expired_ids)} 个过期种子")

cobweb/base/test.py

@@ -0,0 +1,257 @@
+import requests
+from urllib.parse import urlparse
+from typing import Dict, Optional
+
+
+class FileTypeDetector:
+    def __init__(self):
+        self.file_signatures = {
+            # 图片格式
+            b'\x89PNG\r\n\x1a\n': 'PNG',
+            b'\xff\xd8\xff': 'JPEG',
+            b'GIF87a': 'GIF',
+            b'GIF89a': 'GIF',
+            b'RIFF': 'WEBP',  # 需要进一步检查
+            b'BM': 'BMP',
+            b'II*\x00': 'TIFF',
+            b'MM\x00*': 'TIFF',
+            b'\x00\x00\x01\x00': 'ICO',
+            b'\x00\x00\x02\x00': 'CUR',
+
+            # 视频格式
+            b'\x00\x00\x00\x18ftypmp4': 'MP4',
+            b'\x00\x00\x00\x20ftypM4V': 'M4V',
+            b'FLV\x01': 'FLV',
+            b'\x1aE\xdf\xa3': 'WEBM',
+            b'RIFF': 'AVI',  # 需要进一步检查
+            b'\x00\x00\x01\xba': 'MPEG',
+            b'\x00\x00\x01\xb3': 'MPEG',
+            b'OggS': 'OGV',
+
+            # 音频格式
+            b'ID3': 'MP3',
+            b'\xff\xfb': 'MP3',
+            b'\xff\xf3': 'MP3',
+            b'\xff\xf2': 'MP3',
+            b'fLaC': 'FLAC',
+            b'RIFF': 'WAV',  # 需要进一步检查
+            b'OggS': 'OGG',  # 需要进一步检查
+            b'ftypM4A': 'M4A',
+            b'MAC ': 'APE',
+
+            # 其他格式
+            b'%PDF': 'PDF',
+            b'PK\x03\x04': 'ZIP',
+            b'Rar!\x1a\x07\x00': 'RAR',
+            b'\x37\x7a\xbc\xaf\x27\x1c': '7Z',
+        }
+
+        # 扩展名映射
+        self.extension_map = {
+            # 图片
+            '.jpg': 'JPEG', '.jpeg': 'JPEG', '.png': 'PNG', '.gif': 'GIF',
+            '.webp': 'WEBP', '.bmp': 'BMP', '.tiff': 'TIFF', '.tif': 'TIFF',
+            '.ico': 'ICO', '.svg': 'SVG', '.heic': 'HEIC', '.avif': 'AVIF',
+
+            # 视频
+            '.mp4': 'MP4', '.avi': 'AVI', '.mov': 'MOV', '.wmv': 'WMV',
+            '.flv': 'FLV', '.webm': 'WEBM', '.mkv': 'MKV', '.m4v': 'M4V',
+            '.mpg': 'MPEG', '.mpeg': 'MPEG', '.3gp': '3GP', '.ogv': 'OGV',
+            '.ts': 'TS', '.mts': 'MTS', '.vob': 'VOB',
+
+            # 音频
+            '.mp3': 'MP3', '.wav': 'WAV', '.flac': 'FLAC', '.aac': 'AAC',
+            '.ogg': 'OGG', '.wma': 'WMA', '.m4a': 'M4A', '.ape': 'APE',
+            '.opus': 'OPUS', '.aiff': 'AIFF', '.au': 'AU',
+        }
+
+        # MIME类型映射
+        self.mime_type_map = {
+            # 图片
+            'image/jpeg': 'JPEG', 'image/png': 'PNG', 'image/gif': 'GIF',
+            'image/webp': 'WEBP', 'image/bmp': 'BMP', 'image/tiff': 'TIFF',
+            'image/svg+xml': 'SVG', 'image/x-icon': 'ICO',
+
+            # 视频
+            'video/mp4': 'MP4', 'video/avi': 'AVI', 'video/quicktime': 'MOV',
+            'video/x-msvideo': 'AVI', 'video/webm': 'WEBM', 'video/x-flv': 'FLV',
+            'video/3gpp': '3GP', 'video/ogg': 'OGV',
+
+            # 音频
+            'audio/mpeg': 'MP3', 'audio/wav': 'WAV', 'audio/flac': 'FLAC',
+            'audio/aac': 'AAC', 'audio/ogg': 'OGG', 'audio/x-ms-wma': 'WMA',
+            'audio/mp4': 'M4A', 'audio/opus': 'OPUS',
+        }
+
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+        })
+
+    def get_file_extension(self, url: str) -> str:
+        """从URL获取文件扩展名"""
+        parsed = urlparse(url)
+        path = parsed.path.lower()
+        site = parsed.netloc
+
+        # 移除查询参数
+        if '?' in path:
+            path = path.split('?')[0]
+
+        # 获取扩展名
+        if '.' in path:
+            return '.' + path.split('.')[-1], site
+        return '', site
+
+    def detect_by_extension(self, url: str) -> Optional[str]:
+        """通过文件扩展名检测类型"""
+        ext, site = self.get_file_extension(url)
+        return self.extension_map.get(ext)
+
+    def detect_by_mime_type(self, content_type: str) -> Optional[str]:
+        """通过MIME类型检测"""
+        if not content_type:
+            return None
+
+        # 清理content-type，移除参数
+        mime_type = content_type.split(';')[0].strip().lower()
+        return self.mime_type_map.get(mime_type)
+
+    def get_partial_content(self, url: str, max_bytes: int = 64) -> Optional[bytes]:
+        """获取文件的前几个字节"""
+        try:
+            headers = {'Range': f'bytes=0-{max_bytes - 1}'}
+            response = self.session.get(url, headers=headers, timeout=10)
+
+            if response.status_code in [200, 206]:
+                return response.content
+        except Exception as e:
+            print(f"获取内容失败: {e}")
+        return None
+
+    def detect_by_signature(self, data: bytes) -> Optional[str]:
+        """通过文件签名检测类型"""
+        if not data:
+            return None
+
+        # 检查各种文件签名
+        for signature, file_type in self.file_signatures.items():
+            if data.startswith(signature):
+                # 特殊处理需要进一步检查的格式
+                if signature == b'RIFF' and len(data) >= 12:
+                    # 检查是WEBP、AVI还是WAV
+                    if data[8:12] == b'WEBP':
+                        return 'WEBP'
+                    elif data[8:12] == b'AVI ':
+                        return 'AVI'
+                    elif data[8:12] == b'WAVE':
+                        return 'WAV'
+                elif signature == b'OggS' and len(data) >= 32:
+                    # 检查是OGG音频还是OGV视频
+                    if b'vorbis' in data[:64].lower():
+                        return 'OGG'
+                    elif b'theora' in data[:64].lower():
+                        return 'OGV'
+                    else:
+                        return 'OGG'
+                else:
+                    return file_type
+
+        # 检查MP4相关格式
+        if len(data) >= 12 and data[4:8] == b'ftyp':
+            brand = data[8:12]
+            if brand in [b'mp41', b'mp42', b'isom', b'avc1']:
+                return 'MP4'
+            elif brand == b'M4A ':
+                return 'M4A'
+            elif brand == b'M4V ':
+                return 'M4V'
+            elif brand == b'qt  ':
+                return 'MOV'
+
+        return None
+
+    def get_detailed_info(self, url, content_type, data) -> Dict:
+        """获取详细的文件信息"""
+        result = {
+            'url': url,
+            'site': None,
+            'detected_type': None,
+            'confidence': 'unknown',
+            'methods_used': [],
+            'content_type': content_type,
+            'extension': None
+        }
+
+        # 1. 先尝试HEAD请求获取HTTP头信息
+        try:
+            result['content_type'] = content_type
+            # result['file_size'] = content_length
+
+            # 通过MIME类型检测
+            mime_detected = self.detect_by_mime_type(content_type)
+            if mime_detected:
+                result['detected_type'] = mime_detected
+                result['confidence'] = 'high'
+                result['methods_used'].append('mime_type')
+        except Exception as e:
+            print(f"HEAD请求失败: {e}")
+
+        # 2. 通过扩展名检测
+        ext_detected = self.detect_by_extension(url)
+        result['extension'], result['site'] = self.get_file_extension(url)
+
+        if ext_detected:
+            if not result['detected_type']:
+                result['detected_type'] = ext_detected
+                result['confidence'] = 'medium'
+            elif result['detected_type'] == ext_detected:
+                result['confidence'] = 'very_high'  # MIME和扩展名一致
+            result['methods_used'].append('extension')
+
+        # 3. 如果前两种方法不确定，使用文件签名检测
+        if result['confidence'] in ['unknown', 'medium']:
+            signature_detected = self.detect_by_signature(data)
+            if signature_detected:
+                if not result['detected_type']:
+                    result['detected_type'] = signature_detected
+                    result['confidence'] = 'high'
+                elif result['detected_type'] == signature_detected:
+                    result['confidence'] = 'very_high'
+                else:
+                    # 冲突时，优先相信文件签名
+                    result['detected_type'] = signature_detected
+                    result['confidence'] = 'high'
+                result['methods_used'].append('file_signature')
+
+        return result
+
+    def detect_file_type(self, url: str) -> str:
+        """简单的文件类型检测，返回类型字符串"""
+        info = self.get_detailed_info(url)
+        return info.get('detected_type', 'Unknown')
+
+    def get_file_category(self, file_type: str) -> str:
+        """获取文件类别"""
+        if not file_type or file_type == 'Unknown':
+            return 'Unknown'
+
+        image_types = {'PNG', 'JPEG', 'GIF', 'WEBP', 'BMP', 'TIFF', 'ICO', 'SVG', 'HEIC', 'AVIF'}
+        video_types = {'MP4', 'AVI', 'MOV', 'WMV', 'FLV', 'WEBM', 'MKV', 'M4V', 'MPEG', '3GP', 'OGV', 'TS', 'MTS',
+                       'VOB'}
+        audio_types = {'MP3', 'WAV', 'FLAC', 'AAC', 'OGG', 'WMA', 'M4A', 'APE', 'OPUS', 'AIFF', 'AU'}
+
+        if file_type in image_types:
+            return 'Image'
+        elif file_type in video_types:
+            return 'Video'
+        elif file_type in audio_types:
+            return 'Audio'
+        else:
+            return 'Other'
+
+
+# if __name__ == "__main__":
+#     detector = FileTypeDetector()
+#     result = detector.get_detailed_info("https://cdn.pixabay.com/user/2024/12/10/12-18-33-812_96x96.jpeg")
+#     print(result)

cobweb/constant.py

@@ -1,3 +1,5 @@
+from enum import Enum
+
 
 class CrawlerModel:
 
@@ -28,6 +30,13 @@ class DealModel:
     poll = "deal model: poll"
 
 
+class ResponseStatus(Enum):
+    failed = "failed"
+    succeed = "succeed"
+    filter = "filter"
+    max_retry = "max retry"
+
+
 class LogTemplate:
 
     console_item = """
@@ -51,7 +60,7 @@ class LogTemplate:
     launcher_pro_polling = """
 ----------------------- start - 轮训日志: {task} -----------------
         内存队列 
-            种子数:  {doing_len}
+            消费中:  {doing_len}
             待消费:  {todo_len}
             已消费:  {done_len}
         redis队列 

cobweb/crawlers/crawler.py

@@ -1,170 +1,27 @@
-import json
-import threading
-import time
-import traceback
-from inspect import isgenerator
-from typing import Union, Callable, Mapping
-
-from cobweb.constant import DealModel, LogTemplate
+from typing import Any, Generator
 from cobweb.base import (
-    Queue, 
-    Seed, 
-    BaseItem, 
-    Request, 
-    Response, 
-    ConsoleItem,
-    logger
+    Seed,
+    BaseItem,
+    Request,
+    Response,
+    CSVItem,
 )
 
 
-class Crawler(threading.Thread):
-
-    def __init__(
-            self,
-            stop: threading.Event,
-            pause: threading.Event,
-            # launcher_queue: Union[Mapping[str, Queue]],
-            get_seed: Callable,
-            set_seed: Callable,
-            add_seed: Callable,
-            delete_seed: Callable,
-            upload_data: Callable,
-            custom_func: Union[Mapping[str, Callable]],
-            thread_num: int,
-            max_retries: int,
-            time_sleep: int,
-    ):
-        super().__init__()
-
-        self._stop = stop
-        self._pause = pause
-        self._get_seed = get_seed
-        self._set_seed = set_seed
-        self._add_seed = add_seed
-        self._delete_seed = delete_seed
-        self._upload_data = upload_data
-
-        for func_name, _callable in custom_func.items():
-            if isinstance(_callable, Callable):
-                self.__setattr__(func_name, _callable)
-
-        self.thread_num = thread_num
-        self.time_sleep = time_sleep
-        self.max_retries = max_retries
+class Crawler:
 
     @staticmethod
-    def request(seed: Seed) -> Union[Request, BaseItem]:
+    def request(seed: Seed) -> Generator[Request, Response, None]:
         yield Request(seed.url, seed, timeout=5)
 
     @staticmethod
-    def download(item: Request) -> Union[Seed, BaseItem, Response, str]:
+    def download(item: Request) -> Generator[Response, Any, None]:
         response = item.download()
         yield Response(item.seed, response, **item.to_dict)
 
     @staticmethod
-    def parse(item: Response) -> BaseItem:
+    def parse(item: Response) -> Generator[BaseItem, Any, None]:
         upload_item = item.to_dict
-        upload_item["text"] = item.response.text
-        yield ConsoleItem(item.seed, data=json.dumps(upload_item, ensure_ascii=False))
-
-    # def get_seed(self) -> Seed:
-    #     return self._todo.pop()
-
-    def distribute(self, item, seed):
-        if isinstance(item, BaseItem):
-            self._upload_data(item)
-        elif isinstance(item, Seed):
-            self._add_seed(item)
-        elif isinstance(item, str) and item == DealModel.poll:
-            self._set_seed(seed)
-        elif isinstance(item, str) and item == DealModel.done:
-            self._delete_seed(seed)
-        elif isinstance(item, str) and item == DealModel.fail:
-            seed.params.seed_status = DealModel.fail
-            self._delete_seed(seed)
-        else:
-            raise TypeError("yield value type error!")
-
-    def spider(self):
-        while not self._stop.is_set():
-
-            seed = self._get_seed()
-
-            if not seed:
-                time.sleep(1)
-                continue
-
-            elif seed.params.retry > self.max_retries:
-                seed.params.seed_status = DealModel.fail
-                self._delete_seed(seed)
-                continue
-
-            seed_detail_log_info = LogTemplate.log_info(seed.to_dict)
-
-            try:
-                request_iterators = self.request(seed)
-
-                if not isgenerator(request_iterators):
-                    raise TypeError("request function isn't a generator!")
-
-                iterator_status = False
-
-                for request_item in request_iterators:
-
-                    iterator_status = True
-
-                    if isinstance(request_item, Request):
-                        iterator_status = False
-                        download_iterators = self.download(request_item)
-                        if not isgenerator(download_iterators):
-                            raise TypeError("download function isn't a generator")
-
-                        for download_item in download_iterators:
-                            iterator_status = True
-                            if isinstance(download_item, Response):
-                                iterator_status = False
-                                logger.info(LogTemplate.download_info.format(
-                                    detail=seed_detail_log_info,
-                                    retry=seed.params.retry,
-                                    priority=seed.params.priority,
-                                    seed_version=seed.params.seed_version,
-                                    identifier=seed.identifier or "",
-                                    status=download_item.response,
-                                    response=LogTemplate.log_info(download_item.to_dict)
-                                ))
-                                parse_iterators = self.parse(download_item)
-                                if not isgenerator(parse_iterators):
-                                    raise TypeError("parse function isn't a generator")
-                                for parse_item in parse_iterators:
-                                    iterator_status = True
-                                    if isinstance(parse_item, Response):
-                                        raise TypeError("upload_item can't be a Response instance")
-                                    self.distribute(parse_item, seed)
-                            else:
-                                self.distribute(download_item, seed)
-                    else:
-                        self.distribute(request_item, seed)
-
-                if not iterator_status:
-                    raise ValueError("request/download/parse function yield value error!")
-            except Exception as e:
-                logger.info(LogTemplate.download_exception.format(
-                    detail=seed_detail_log_info,
-                    retry=seed.params.retry,
-                    priority=seed.params.priority,
-                    seed_version=seed.params.seed_version,
-                    identifier=seed.identifier or "",
-                    exception=''.join(traceback.format_exception(type(e), e, e.__traceback__))
-                ))
-                seed.params.retry += 1
-                # self._todo.push(seed)
-                self._set_seed(seed)
-                time.sleep(self.time_sleep * seed.params.retry)
-            finally:
-                time.sleep(0.1)
-        logger.info("spider thread close")
-
-    def run(self):
-        for index in range(self.thread_num):
-            threading.Thread(name=f"spider_{index}", target=self.spider).start()
+        upload_item["content"] = getattr(item.response, "text", item.response)
+        yield CSVItem(item.seed, data=upload_item)
 

cobweb/db/api_db.py

@@ -57,7 +57,8 @@ class ApiDB:
         return self._get_response(api="/zcard", params=dict(name=name))
 
     def zadd(self, name, item: dict, **kwargs):
-        return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
+        if item:
+            return self._post_response(api="/zadd", data=dict(name=name, mapping=item, **kwargs))
 
     def zrem(self, name, *values):
         return self._post_response(api="/zrem", data=dict(name=name, values=values))
@@ -71,7 +72,7 @@ class ApiDB:
     def auto_incr(self, name, t=15, limit=1000) -> bool:
         return self._get_response(api="/auto_incr", params=dict(name=name, t=t, limit=limit))
 
-    def members(self, name, score, start=0, count=5000, _min="-inf", _max="+inf"):
+    def members(self, name, score, start=0, count=1000, _min="-inf", _max="+inf"):
         return self._get_response(api="/members", params=dict(name=name, score=score, start=start, count=count, min=_min, max=_max))
 
     def done(self, name: list, *values):