cobweb-launcher 1.0.5__py3-none-any.whl → 3.2.18__py3-none-any.whl

cobweb/__init__.py

@@ -1 +1,5 @@
-from .launchers import Launcher, LauncherPro
+from .launchers import Launcher
+from .constant import CrawlerModel
+from .pipelines import Pipeline
+from .crawlers import Crawler
+from .log_dots import Dot

cobweb/base/__init__.py

@@ -1,9 +1,9 @@
+from .item import BaseItem, CSVItem
 from .common_queue import Queue
 from .response import Response
 from .request import Request
-from .item import BaseItem
+from .logger import logger
 from .seed import Seed
+from .task_queue import TaskQueue, Status
 
-from .log import logger
-from .decorators import decorator_oss_db
 

cobweb/base/common_queue.py

@@ -1,30 +1,51 @@
 from collections import deque
+from typing import Any, Iterable, Union
 
 
 class Queue:
-
     def __init__(self):
+        """初始化队列"""
         self._queue = deque()
 
     @property
     def length(self) -> int:
+        """返回队列长度"""
         return len(self._queue)
 
-    def push(self, data, left: bool = False, direct_insertion: bool = False):
-        try:
-            if not data:
-                return None
-            if not direct_insertion and any(isinstance(data, t) for t in (list, tuple)):
-                self._queue.extendleft(data) if left else self._queue.extend(data)
-            else:
-                self._queue.appendleft(data) if left else self._queue.append(data)
-        except AttributeError:
-            pass
-
-    def pop(self, left: bool = True):
+    def empty(self) -> bool:
+        """检查队列是否为空"""
+        return not self._queue
+
+    def push(self, data: Union[Any, Iterable], direct_insertion: bool = False):
+        """
+        向队列中添加数据。
+        如果数据是可迭代对象（如列表、元组或集合），可以选择直接扩展队列。
+        """
+        if not data:
+            return
+
+        if not direct_insertion and isinstance(data, (list, tuple, set)):
+            self._queue.extend(data)
+        else:
+            self._queue.append(data)
+
+    def pop(self) -> Any:
+        """
+        从队列左侧弹出一个元素。
+        如果队列为空，返回 None。
+        """
         try:
-            return self._queue.popleft() if left else self._queue.pop()
+            return self._queue.popleft()
         except IndexError:
             return None
-        except AttributeError:
-            return None
+
+    def iter_items(self, limit: int = 1) -> Iterable:
+        """
+        按指定数量从队列中弹出多个元素并生成它们。
+        如果队列为空或达到限制，则停止生成。
+        """
+        for _ in range(limit):
+            item = self.pop()
+            if item is None:
+                break
+            yield item

cobweb/base/item.py

@@ -1,19 +1,22 @@
 from .seed import Seed
+from typing import Dict, Any
 from collections import namedtuple
 
 
-class Item(type):
+class ItemMeta(type):
 
-    def __new__(cls, name, bases, dct):
-        new_class_instance = type.__new__(cls, name, bases, dct)
+    def __new__(cls, name: str, bases: tuple, dct: dict) -> type:
+        new_class = super().__new__(cls, name, bases, dct)
         if name != "BaseItem":
-            table = getattr(new_class_instance, "__TABLE__")
-            fields = getattr(new_class_instance, "__FIELDS__")
-            new_class_instance.Data = namedtuple(table, fields)
-        return new_class_instance
+            table = getattr(new_class, "__TABLE__")
+            fields = getattr(new_class, "__FIELDS__")
+            if not table or not fields:
+                raise ValueError(f"Missing required attributes '__TABLE__' or '__FIELDS__' in class {name}")
+            new_class.Data = namedtuple(table, fields)
+        return new_class
 
 
-class BaseItem(metaclass=Item):
+class BaseItem(metaclass=ItemMeta):
 
     __TABLE__ = ""
     __FIELDS__ = ""
@@ -23,17 +26,40 @@ class BaseItem(metaclass=Item):
 
         data = {}
         for key, value in kwargs.items():
-            if key not in self.__FIELDS__:
-                self.__setattr__(key, value)
-            else:
+            if key in self.__FIELDS__:
                 data[key] = value
+            else:
+                setattr(self, key, value)
 
-        self.data = self.Data(**data)
+        try:
+            self.data = self.Data(**data)
+        except TypeError as e:
+            raise ValueError(f"Invalid field values for Data: {e}") from e
 
     @property
-    def to_dict(self):
+    def to_dict(self) -> Dict[str, Any]:
         return self.data._asdict()
 
     @property
-    def table(self):
+    def fields(self) -> tuple[str]:
+        return self.data._fields
+
+    @property
+    def table(self) -> str:
         return self.Data.__name__
+
+    def __setitem__(self, key: str, value: Any):
+        setattr(self, key, value)
+
+    def __getitem__(self, key: str) -> Any:
+        return getattr(self, key, None)
+
+    def __getattr__(self, name: str) -> Any:
+        return None
+
+
+class CSVItem(BaseItem):
+
+    __TABLE__ = "cobweb"
+    __FIELDS__ = "data"
+

cobweb/base/{log.py → logger.py}

@@ -51,8 +51,8 @@ class ColorCodes:
     HIDDEN = "\033[8m"
 
 
-class Log:
-    logging.getLogger('oss2.api').setLevel(logging.WARNING)
+class Logger:
+    # logging.getLogger('oss2.api').setLevel(logging.WARNING)
     logging.basicConfig(
         level=logging.INFO,
         format=f'%(asctime)s %(name)s [%(filename)s:%(lineno)d %(funcName)s]'
@@ -88,7 +88,7 @@ class Log:
         return self.__class__.log.critical
 
 
-logger = Log()
+logger = Logger()