crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

crawlo/items/base.py

@@ -1,23 +1,23 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-基础元类定义
-"""
-from abc import ABCMeta
-from .fields import Field
-
-
-class ItemMeta(ABCMeta):
-    def __new__(mcs, name, bases, attrs):
-        fields = {}
-        cls_attrs = {}
-
-        for attr_name, attr_value in attrs.items():
-            if isinstance(attr_value, Field):
-                fields[attr_name] = attr_value
-            else:
-                cls_attrs[attr_name] = attr_value
-
-        cls_instance = super().__new__(mcs, name, bases, cls_attrs)
-        cls_instance.FIELDS = fields
-        return cls_instance
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+基础元类定义
+"""
+from abc import ABCMeta
+from .fields import Field
+
+
+class ItemMeta(ABCMeta):
+    def __new__(mcs, name, bases, attrs):
+        fields = {}
+        cls_attrs = {}
+
+        for attr_name, attr_value in attrs.items():
+            if isinstance(attr_value, Field):
+                fields[attr_name] = attr_value
+            else:
+                cls_attrs[attr_name] = attr_value
+
+        cls_instance = super().__new__(mcs, name, bases, cls_attrs)
+        cls_instance.FIELDS = fields
+        return cls_instance

crawlo/items/fields.py

@@ -1,53 +1,53 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Field 类定义
-"""
-from typing import Any, Optional, Type
-
-
-class Field:
-    """
-    字段定义类，用于定义 Item 的字段属性和验证规则
-    """
-    def __init__(
-        self,
-        nullable: bool = True,
-        *,
-        default: Any = None,
-        field_type: Optional[Type] = None,
-        max_length: Optional[int] = None,
-        description: str = ""
-    ):
-        self.nullable = nullable
-        self.default = default
-        self.field_type = field_type
-        self.max_length = max_length
-        self.description = description
-
-    def validate(self, value: Any, field_name: str = "") -> Any:
-        """
-        验证字段值是否符合规则
-        """
-        if value is None or (isinstance(value, str) and value.strip() == ""):
-            if self.default is not None:
-                return self.default
-            elif not self.nullable:
-                raise ValueError(
-                    f"字段 '{field_name}' 不允许为空。"
-                )
-
-        if value is not None and not (isinstance(value, str) and value.strip() == ""):
-            if self.field_type and not isinstance(value, self.field_type):
-                raise TypeError(
-                    f"字段 '{field_name}' 类型错误：期望类型 {self.field_type}, 得到 {type(value)}，值：{value!r}"
-                )
-            if self.max_length and len(str(value)) > self.max_length:
-                raise ValueError(
-                    f"字段 '{field_name}' 长度超限：最大长度 {self.max_length}，当前长度 {len(str(value))}，值：{value!r}"
-                )
-
-        return value
-
-    def __repr__(self):
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Field 类定义
+"""
+from typing import Any, Optional, Type
+
+
+class Field:
+    """
+    字段定义类，用于定义 Item 的字段属性和验证规则
+    """
+    def __init__(
+        self,
+        nullable: bool = True,
+        *,
+        default: Any = None,
+        field_type: Optional[Type] = None,
+        max_length: Optional[int] = None,
+        description: str = ""
+    ):
+        self.nullable = nullable
+        self.default = default
+        self.field_type = field_type
+        self.max_length = max_length
+        self.description = description
+
+    def validate(self, value: Any, field_name: str = "") -> Any:
+        """
+        验证字段值是否符合规则
+        """
+        if value is None or (isinstance(value, str) and value.strip() == ""):
+            if self.default is not None:
+                return self.default
+            elif not self.nullable:
+                raise ValueError(
+                    f"字段 '{field_name}' 不允许为空。"
+                )
+
+        if value is not None and not (isinstance(value, str) and value.strip() == ""):
+            if self.field_type and not isinstance(value, self.field_type):
+                raise TypeError(
+                    f"字段 '{field_name}' 类型错误：期望类型 {self.field_type}, 得到 {type(value)}，值：{value!r}"
+                )
+            if self.max_length and len(str(value)) > self.max_length:
+                raise ValueError(
+                    f"字段 '{field_name}' 长度超限：最大长度 {self.max_length}，当前长度 {len(str(value))}，值：{value!r}"
+                )
+
+        return value
+
+    def __repr__(self):
         return f"<Field nullable={self.nullable} type={self.field_type} default={self.default}>"

crawlo/items/items.py

@@ -1,105 +1,105 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Item 类定义
-"""
-from copy import deepcopy
-from pprint import pformat
-from typing import Any, Iterator, Dict
-from collections.abc import MutableMapping
-
-from .base import ItemMeta
-from crawlo.exceptions import ItemInitError, ItemAttributeError
-
-
-class Item(MutableMapping, metaclass=ItemMeta):
-    """
-    数据项基类，用于定义结构化数据
-    """
-    FIELDS: Dict[str, Any] = {}
-
-    def __init__(self, *args, **kwargs):
-        if args:
-            raise ItemInitError(f"{self.__class__.__name__} 不支持位置参数：{args}，请使用关键字参数初始化。")
-
-        self._values: Dict[str, Any] = {}
-
-        # 初始化字段，默认值填充
-        for field_name, field_obj in self.FIELDS.items():
-            if field_obj.default is not None:
-                self._values[field_name] = field_obj.default
-
-        # 覆盖默认值或设置新值
-        for key, value in kwargs.items():
-            self[key] = value
-
-    def __getitem__(self, item: str) -> Any:
-        return self._values[item]
-
-    def __setitem__(self, key: str, value: Any) -> None:
-        if key not in self.FIELDS:
-            raise KeyError(f"{self.__class__.__name__} 不包含字段：{key}")
-
-        field = self.FIELDS[key]
-        try:
-            validated_value = field.validate(value, field_name=key)
-            self._values[key] = validated_value
-        except Exception as e:
-            error_lines = [
-                "",
-                "【字段校验失败】",
-                f"字段名称: {key}",
-                f"数据类型: {type(value)}",
-                f"原始值:   {repr(value)}",
-                f"是否允许空值: {field.nullable}",
-                f"错误原因: {str(e)}",
-                ""
-            ]
-            detailed_error = "\n".join(error_lines)
-            raise type(e)(detailed_error) from e
-
-    def __delitem__(self, key: str) -> None:
-        del self._values[key]
-
-    def __setattr__(self, key: str, value: Any) -> None:
-        if not key.startswith("_"):
-            raise AttributeError(
-                f"设置字段值请使用 item[{key!r}] = {value!r}"
-            )
-        super().__setattr__(key, value)
-
-    def __getattr__(self, item: str) -> Any:
-        raise AttributeError(
-            f"{self.__class__.__name__} 不支持字段：{item}。"
-            f"请先在 `{self.__class__.__name__}` 中声明该字段，再通过 item[{item!r}] 获取。"
-        )
-
-    def __getattribute__(self, item: str) -> Any:
-        try:
-            field = super().__getattribute__("FIELDS")
-            if isinstance(field, dict) and item in field:
-                raise ItemAttributeError(
-                    f"获取字段值请使用 item[{item!r}]"
-                )
-        except AttributeError:
-            pass  # 如果 FIELDS 尚未定义，继续执行后续逻辑
-        return super().__getattribute__(item)
-
-    def __repr__(self) -> str:
-        return pformat(dict(self))
-
-    __str__ = __repr__
-
-    def __iter__(self) -> Iterator[str]:
-        return iter(self._values)
-
-    def __len__(self) -> int:
-        return len(self._values)
-
-    def to_dict(self) -> Dict[str, Any]:
-        """转换为字典"""
-        return dict(self)
-
-    def copy(self) -> "Item":
-        """深拷贝当前 Item"""
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Item 类定义
+"""
+from copy import deepcopy
+from pprint import pformat
+from typing import Any, Iterator, Dict
+from collections.abc import MutableMapping
+
+from .base import ItemMeta
+from crawlo.exceptions import ItemInitError, ItemAttributeError
+
+
+class Item(MutableMapping, metaclass=ItemMeta):
+    """
+    数据项基类，用于定义结构化数据
+    """
+    FIELDS: Dict[str, Any] = {}
+
+    def __init__(self, *args, **kwargs):
+        if args:
+            raise ItemInitError(f"{self.__class__.__name__} 不支持位置参数：{args}，请使用关键字参数初始化。")
+
+        self._values: Dict[str, Any] = {}
+
+        # 初始化字段，默认值填充
+        for field_name, field_obj in self.FIELDS.items():
+            if field_obj.default is not None:
+                self._values[field_name] = field_obj.default
+
+        # 覆盖默认值或设置新值
+        for key, value in kwargs.items():
+            self[key] = value
+
+    def __getitem__(self, item: str) -> Any:
+        return self._values[item]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        if key not in self.FIELDS:
+            raise KeyError(f"{self.__class__.__name__} 不包含字段：{key}")
+
+        field = self.FIELDS[key]
+        try:
+            validated_value = field.validate(value, field_name=key)
+            self._values[key] = validated_value
+        except Exception as e:
+            error_lines = [
+                "",
+                "【字段校验失败】",
+                f"字段名称: {key}",
+                f"数据类型: {type(value)}",
+                f"原始值:   {repr(value)}",
+                f"是否允许空值: {field.nullable}",
+                f"错误原因: {str(e)}",
+                ""
+            ]
+            detailed_error = "\n".join(error_lines)
+            raise type(e)(detailed_error) from e
+
+    def __delitem__(self, key: str) -> None:
+        del self._values[key]
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        if not key.startswith("_"):
+            raise AttributeError(
+                f"设置字段值请使用 item[{key!r}] = {value!r}"
+            )
+        super().__setattr__(key, value)
+
+    def __getattr__(self, item: str) -> Any:
+        raise AttributeError(
+            f"{self.__class__.__name__} 不支持字段：{item}。"
+            f"请先在 `{self.__class__.__name__}` 中声明该字段，再通过 item[{item!r}] 获取。"
+        )
+
+    def __getattribute__(self, item: str) -> Any:
+        try:
+            field = super().__getattribute__("FIELDS")
+            if isinstance(field, dict) and item in field:
+                raise ItemAttributeError(
+                    f"获取字段值请使用 item[{item!r}]"
+                )
+        except AttributeError:
+            pass  # 如果 FIELDS 尚未定义，继续执行后续逻辑
+        return super().__getattribute__(item)
+
+    def __repr__(self) -> str:
+        return pformat(dict(self))
+
+    __str__ = __repr__
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._values)
+
+    def __len__(self) -> int:
+        return len(self._values)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """转换为字典"""
+        return dict(self)
+
+    def copy(self) -> "Item":
+        """深拷贝当前 Item"""
         return deepcopy(self)

crawlo/logging/__init__.py

@@ -1,38 +1,46 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Crawlo统一日志系统
-=================
-
-设计原则：
-1. 简单优先 - 避免过度设计
-2. 性能优先 - 减少锁竞争和复杂逻辑  
-3. 一致性 - 统一的日志接口
-4. 可靠性 - 确保日志始终可用
-"""
-
-from .manager import LogManager
-from .factory import LoggerFactory
-from .config import LogConfig
-
-# 统一的公共接口
-def get_logger(name: str = 'default'):
-    """获取logger实例"""
-    return LoggerFactory.get_logger(name)
-
-def configure_logging(settings=None, **kwargs):
-    """配置日志系统"""
-    return LogManager().configure(settings, **kwargs)
-
-def is_configured() -> bool:
-    """检查日志系统是否已配置"""
-    return LogManager().is_configured
-
-__all__ = [
-    'LogManager',
-    'LoggerFactory', 
-    'LogConfig',
-    'get_logger',
-    'configure_logging',
-    'is_configured'
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo统一日志系统
+=================
+
+设计原则：
+1. 简单优先 - 避免过度设计
+2. 性能优先 - 减少锁竞争和复杂逻辑  
+3. 一致性 - 统一的日志接口
+4. 可靠性 - 确保日志始终可用
+"""
+
+from .manager import LogManager
+from .factory import LoggerFactory
+from .config import LogConfig
+from .monitor import LogPerformanceMonitor
+
+# 统一的公共接口
+def get_logger(name: str = 'default'):
+    """获取logger实例"""
+    return LoggerFactory.get_logger(name)
+
+def configure_logging(settings=None, **kwargs):
+    """配置日志系统"""
+    return LogManager().configure(settings, **kwargs)
+
+def is_configured() -> bool:
+    """检查日志系统是否已配置"""
+    return LogManager().is_configured
+
+def get_monitor() -> LogPerformanceMonitor:
+    """获取日志性能监控器"""
+    from .monitor import get_monitor as _get_monitor
+    return _get_monitor()
+
+__all__ = [
+    'LogManager',
+    'LoggerFactory', 
+    'LogConfig',
+    'LogPerformanceMonitor',
+    'get_logger',
+    'configure_logging',
+    'is_configured',
+    'get_monitor'
 ]

crawlo/logging/async_handler.py

@@ -0,0 +1,181 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+异步日志处理器
+用于提高日志写入性能
+"""
+
+import asyncio
+import logging
+import threading
+import queue
+from typing import Optional
+from concurrent_log_handler import ConcurrentRotatingFileHandler
+
+
+class AsyncLogHandler(logging.Handler):
+    """
+    异步日志处理器
+    将日志记录放入队列中，由后台线程异步处理
+    """
+    
+    def __init__(self, handler: logging.Handler, queue_size: int = 10000):
+        """
+        初始化异步日志处理器
+        
+        Args:
+            handler: 实际的日志处理器
+            queue_size: 队列大小
+        """
+        super().__init__()
+        self._handler = handler
+        self._queue = queue.Queue(maxsize=queue_size)
+        self._thread: Optional[threading.Thread] = None
+        self._stop_event = threading.Event()
+        self._started = False
+        
+    def start(self):
+        """启动异步处理线程"""
+        if self._started:
+            return
+            
+        self._started = True
+        self._stop_event.clear()
+        self._thread = threading.Thread(target=self._worker, daemon=True)
+        self._thread.start()
+        
+    def stop(self):
+        """停止异步处理线程"""
+        if not self._started:
+            return
+            
+        self._started = False
+        self._stop_event.set()
+        
+        # 发送一个哨兵消息来唤醒工作线程
+        try:
+            self._queue.put_nowait(None)
+        except queue.Full:
+            pass
+            
+        # 等待线程结束
+        if self._thread and self._thread.is_alive():
+            self._thread.join(timeout=5.0)
+            
+        # 关闭底层处理器
+        if self._handler:
+            self._handler.close()
+            
+    def _worker(self):
+        """工作线程函数"""
+        while not self._stop_event.is_set():
+            try:
+                # 从队列中获取日志记录
+                record = self._queue.get(timeout=1.0)
+                
+                # 哨兵消息，表示停止
+                if record is None:
+                    break
+                    
+                # 处理日志记录
+                try:
+                    self._handler.emit(record)
+                except Exception:
+                    pass  # 忽略处理错误
+                    
+                self._queue.task_done()
+                
+            except queue.Empty:
+                continue
+            except Exception:
+                if not self._stop_event.is_set():
+                    continue
+                else:
+                    break
+                    
+    def emit(self, record):
+        """
+        发出日志记录
+        
+        Args:
+            record: 日志记录
+        """
+        if not self._started:
+            self.start()
+            
+        # 将日志记录放入队列
+        try:
+            self._queue.put_nowait(record)
+        except queue.Full:
+            # 队列满时丢弃日志记录
+            pass
+            
+    def flush(self):
+        """刷新日志处理器"""
+        if self._handler:
+            self._handler.flush()
+            
+    def close(self):
+        """关闭日志处理器"""
+        self.stop()
+        super().close()
+
+
+class AsyncConcurrentRotatingFileHandler(AsyncLogHandler):
+    """
+    异步并发轮转文件处理器
+    结合了异步处理和并发轮转文件的功能
+    """
+    
+    def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, 
+                 encoding=None, delay=False, queue_size: int = 10000):
+        """
+        初始化异步并发轮转文件处理器
+        
+        Args:
+            filename: 日志文件名
+            mode: 文件打开模式
+            maxBytes: 最大文件大小
+            backupCount: 备份文件数量
+            encoding: 文件编码
+            delay: 是否延迟打开文件
+            queue_size: 队列大小
+        """
+        handler = ConcurrentRotatingFileHandler(
+            filename=filename,
+            mode=mode,
+            maxBytes=maxBytes,
+            backupCount=backupCount,
+            encoding=encoding,
+            delay=delay
+        )
+        super().__init__(handler, queue_size)
+        
+    @property
+    def baseFilename(self):
+        """获取基础文件名"""
+        return self._handler.baseFilename if self._handler else None
+        
+    @property
+    def maxBytes(self):
+        """获取最大字节数"""
+        return self._handler.maxBytes if self._handler else 0
+        
+    @property
+    def backupCount(self):
+        """获取备份计数"""
+        return self._handler.backupCount if self._handler else 0
+
+
+def wrap_handler_async(handler: logging.Handler, queue_size: int = 10000) -> AsyncLogHandler:
+    """
+    将现有的日志处理器包装为异步处理器
+    
+    Args:
+        handler: 要包装的日志处理器
+        queue_size: 队列大小
+        
+    Returns:
+        异步日志处理器
+    """
+    return AsyncLogHandler(handler, queue_size)