tamar-file-hub-client 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

file_hub_client/utils/__init__.py

@@ -11,7 +11,18 @@ from .file_utils import (
     calculate_file_hash,
     split_file_chunks,
 )
-from .retry import retry_with_backoff
+from .retry import (
+    retry_with_backoff,
+    retry_on_lock_conflict
+)
+from .smart_retry import (
+    smart_retry,
+    retry_on_network_errors,
+    retry_on_conflict,
+    no_retry,
+    ErrorClassifier,
+    RetryStrategy
+)
 from .upload_helper import (
     HttpUploader,
     AsyncHttpUploader,
@@ -23,6 +34,18 @@ from .download_helper import (
     AsyncHttpDownloader,
     DownloadProgress,
 )
+from .idempotency import (
+    IdempotencyKeyGenerator,
+    IdempotencyManager,
+    generate_idempotency_key
+)
+from .logging import (
+    setup_logging,
+    get_logger,
+    GrpcRequestLogger,
+    grpc_request_context,
+    log_grpc_call,
+)
 
 __all__ = [
     # 文件工具
@@ -34,6 +57,13 @@ __all__ = [
 
     # 重试工具
     "retry_with_backoff",
+    "retry_on_lock_conflict",
+    "smart_retry",
+    "retry_on_network_errors",
+    "retry_on_conflict",
+    "no_retry",
+    "ErrorClassifier",
+    "RetryStrategy",
 
     # 上传助手
     "HttpUploader",
@@ -45,4 +75,16 @@ __all__ = [
     "HttpDownloader",
     "AsyncHttpDownloader",
     "DownloadProgress",
+    
+    # 幂等性工具
+    "IdempotencyKeyGenerator",
+    "IdempotencyManager",
+    "generate_idempotency_key",
+    
+    # 日志工具
+    "setup_logging",
+    "get_logger",
+    "GrpcRequestLogger",
+    "grpc_request_context",
+    "log_grpc_call",
 ]

file_hub_client/utils/file_utils.py

@@ -18,8 +18,56 @@ def get_file_mime_type(file_path: Union[str, Path]) -> str:
         MIME类型
     """
     file_path = Path(file_path)
-    mime_type, _ = mimetypes.guess_type(str(file_path))
-    return mime_type or "application/octet-stream"
+    
+    # 定义常见文件扩展名到MIME类型的映射，确保跨平台一致性
+    extension_mime_map = {
+        '.csv': 'text/csv',
+        '.txt': 'text/plain',
+        '.json': 'application/json',
+        '.xml': 'application/xml',
+        '.html': 'text/html',
+        '.htm': 'text/html',
+        '.pdf': 'application/pdf',
+        '.doc': 'application/msword',
+        '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        '.xls': 'application/vnd.ms-excel',
+        '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+        '.ppt': 'application/vnd.ms-powerpoint',
+        '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+        '.jpg': 'image/jpeg',
+        '.jpeg': 'image/jpeg',
+        '.png': 'image/png',
+        '.gif': 'image/gif',
+        '.bmp': 'image/bmp',
+        '.webp': 'image/webp',
+        '.mp3': 'audio/mpeg',
+        '.wav': 'audio/wav',
+        '.mp4': 'video/mp4',
+        '.avi': 'video/x-msvideo',
+        '.mov': 'video/quicktime',
+        '.zip': 'application/zip',
+        '.rar': 'application/vnd.rar',
+        '.7z': 'application/x-7z-compressed',
+        '.tar': 'application/x-tar',
+        '.gz': 'application/gzip',
+    }
+    
+    # 获取文件扩展名（转为小写）
+    extension = file_path.suffix.lower()
+    
+    # 优先使用自定义映射，确保常见文件类型的一致性
+    if extension in extension_mime_map:
+        return extension_mime_map[extension]
+    
+    # 如果自定义映射中没有，尝试使用magic进行内容检测
+    try:
+        import magic
+        mime = magic.Magic(mime=True)
+        return mime.from_file(str(file_path))
+    except ImportError:
+        # 如果magic不可用，使用mimetypes作为fallback
+        mime_type, _ = mimetypes.guess_type(str(file_path))
+        return mime_type or "application/octet-stream"
 
 
 def get_file_extension(file_name: str) -> str:
@@ -65,18 +113,18 @@ def calculate_file_hash(file_path: Union[str, Path], algorithm: str = "sha256")
     """
     file_path = Path(file_path)
     hash_obj = hashlib.new(algorithm)
-    
+
     with open(file_path, "rb") as f:
         while chunk := f.read(8192):
             hash_obj.update(chunk)
-    
+
     return hash_obj.hexdigest()
 
 
 def split_file_chunks(
-    file_obj: BinaryIO,
-    chunk_size: int = 1024 * 1024,  # 默认1MB
-    start_offset: int = 0
+        file_obj: BinaryIO,
+        chunk_size: int = 1024 * 1024,  # 默认1MB
+        start_offset: int = 0
 ) -> Generator[tuple[bytes, int, bool], None, None]:
     """
     将文件分割成块
@@ -91,15 +139,15 @@ def split_file_chunks(
     """
     file_obj.seek(start_offset)
     offset = start_offset
-    
+
     while True:
         chunk = file_obj.read(chunk_size)
         if not chunk:
             break
-            
+
         is_last = len(chunk) < chunk_size
         yield chunk, offset, is_last
-        
+
         offset += len(chunk)
         if is_last:
-            break 
+            break

file_hub_client/utils/idempotency.py

@@ -0,0 +1,196 @@
+"""
+幂等性支持工具
+"""
+import uuid
+import hashlib
+from typing import Optional, Dict, Any, Union
+from datetime import datetime, timedelta
+
+
+class IdempotencyKeyGenerator:
+    """幂等性键生成器"""
+    
+    @staticmethod
+    def generate_uuid_key() -> str:
+        """生成基于UUID的幂等性键"""
+        return str(uuid.uuid4())
+    
+    @staticmethod
+    def generate_content_based_key(content: Dict[str, Any]) -> str:
+        """
+        基于内容生成幂等性键
+        
+        Args:
+            content: 要处理的内容字典
+            
+        Returns:
+            基于内容生成的幂等性键
+        """
+        # 排序内容以确保一致性
+        sorted_content = _sort_dict_recursively(content)
+        content_str = str(sorted_content)
+        
+        # 生成SHA256哈希
+        hash_obj = hashlib.sha256(content_str.encode('utf-8'))
+        return hash_obj.hexdigest()
+    
+    @staticmethod
+    def generate_timestamp_key(prefix: str = "taple") -> str:
+        """
+        基于时间戳生成幂等性键
+        
+        Args:
+            prefix: 键前缀
+            
+        Returns:
+            基于时间戳的幂等性键
+        """
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+        return f"{prefix}_{timestamp}_{uuid.uuid4().hex[:8]}"
+
+
+class IdempotencyManager:
+    """幂等性管理器"""
+    
+    def __init__(self, default_ttl_minutes: int = 60):
+        """
+        初始化幂等性管理器
+        
+        Args:
+            default_ttl_minutes: 默认的幂等性键TTL（分钟）
+        """
+        self._cache: Dict[str, Dict[str, Any]] = {}
+        self._default_ttl = timedelta(minutes=default_ttl_minutes)
+    
+    def register_operation(
+        self,
+        idempotency_key: str,
+        operation_type: str,
+        params: Dict[str, Any],
+        ttl_minutes: Optional[int] = None
+    ) -> bool:
+        """
+        注册幂等性操作
+        
+        Args:
+            idempotency_key: 幂等性键
+            operation_type: 操作类型
+            params: 操作参数
+            ttl_minutes: TTL分钟数（可选）
+            
+        Returns:
+            True if 新注册，False if 已存在
+        """
+        if idempotency_key in self._cache:
+            # 检查是否过期
+            cache_entry = self._cache[idempotency_key]
+            if datetime.now() > cache_entry['expires_at']:
+                # 过期，删除并重新注册
+                del self._cache[idempotency_key]
+            else:
+                # 未过期，检查操作是否一致
+                if (cache_entry['operation_type'] == operation_type and 
+                    cache_entry['params'] == params):
+                    return False  # 相同操作，幂等
+                else:
+                    raise ValueError(f"幂等性键冲突: {idempotency_key}")
+        
+        # 注册新操作
+        ttl = timedelta(minutes=ttl_minutes) if ttl_minutes else self._default_ttl
+        self._cache[idempotency_key] = {
+            'operation_type': operation_type,
+            'params': params,
+            'created_at': datetime.now(),
+            'expires_at': datetime.now() + ttl,
+            'result': None
+        }
+        return True
+    
+    def get_cached_result(self, idempotency_key: str) -> Optional[Any]:
+        """
+        获取缓存的结果
+        
+        Args:
+            idempotency_key: 幂等性键
+            
+        Returns:
+            缓存的结果，如果不存在返回None
+        """
+        if idempotency_key not in self._cache:
+            return None
+            
+        cache_entry = self._cache[idempotency_key]
+        if datetime.now() > cache_entry['expires_at']:
+            del self._cache[idempotency_key]
+            return None
+            
+        return cache_entry.get('result')
+    
+    def cache_result(self, idempotency_key: str, result: Any) -> None:
+        """
+        缓存操作结果
+        
+        Args:
+            idempotency_key: 幂等性键
+            result: 操作结果
+        """
+        if idempotency_key in self._cache:
+            self._cache[idempotency_key]['result'] = result
+    
+    def cleanup_expired(self) -> int:
+        """
+        清理过期的缓存条目
+        
+        Returns:
+            清理的条目数量
+        """
+        now = datetime.now()
+        expired_keys = [
+            key for key, entry in self._cache.items()
+            if now > entry['expires_at']
+        ]
+        
+        for key in expired_keys:
+            del self._cache[key]
+            
+        return len(expired_keys)
+
+
+def _sort_dict_recursively(obj: Any) -> Any:
+    """递归排序字典，确保生成一致的哈希"""
+    if isinstance(obj, dict):
+        return {k: _sort_dict_recursively(v) for k, v in sorted(obj.items())}
+    elif isinstance(obj, list):
+        return [_sort_dict_recursively(item) for item in obj]
+    else:
+        return obj
+
+
+def generate_idempotency_key(
+    operation_type: str,
+    params: Optional[Dict[str, Any]] = None,
+    method: str = "uuid"
+) -> str:
+    """
+    生成幂等性键的便捷函数
+    
+    Args:
+        operation_type: 操作类型
+        params: 操作参数（可选）
+        method: 生成方法（"uuid", "content", "timestamp"）
+        
+    Returns:
+        生成的幂等性键
+    """
+    generator = IdempotencyKeyGenerator()
+    
+    if method == "uuid":
+        return generator.generate_uuid_key()
+    elif method == "content" and params:
+        content = {"operation_type": operation_type, "params": params}
+        return generator.generate_content_based_key(content)
+    elif method == "timestamp":
+        return generator.generate_timestamp_key(operation_type)
+    else:
+        # 默认使用UUID
+        return generator.generate_uuid_key()

file_hub_client/utils/logging.py

@@ -0,0 +1,315 @@
+"""
+日志配置和工具
+"""
+import logging
+import sys
+import time
+import json
+import traceback
+from typing import Optional, Any, Dict
+from functools import wraps
+from contextlib import contextmanager
+from datetime import datetime
+
+# 创建SDK专用的日志记录器 - 使用独立的命名空间避免冲突
+SDK_LOGGER_NAME = "file_hub_client.grpc"
+logger = logging.getLogger(SDK_LOGGER_NAME)
+
+
+class GrpcJSONFormatter(logging.Formatter):
+    """gRPC请求的JSON格式化器"""
+    
+    def format(self, record):
+        log_type = getattr(record, "log_type", "info")
+        log_data = {
+            "timestamp": datetime.fromtimestamp(record.created).isoformat(),
+            "level": record.levelname,
+            "type": log_type,
+            "uri": getattr(record, "uri", None),
+            "request_id": getattr(record, "request_id", None),
+            "data": getattr(record, "data", None),
+            "message": record.getMessage(),
+            "duration": getattr(record, "duration", None),
+            "logger": record.name,  # 添加logger名称以区分SDK日志
+        }
+        
+        # 增加 trace 支持
+        if hasattr(record, "trace"):
+            log_data["trace"] = getattr(record, "trace")
+            
+        # 添加异常信息（如果有的话）
+        if hasattr(record, "exc_info") and record.exc_info:
+            log_data["exception"] = {
+                "type": record.exc_info[0].__name__ if record.exc_info[0] else None,
+                "message": str(record.exc_info[1]) if record.exc_info[1] else None,
+                "traceback": traceback.format_exception(*record.exc_info)
+            }
+            
+        # 过滤掉None值
+        log_data = {k: v for k, v in log_data.items() if v is not None}
+        
+        return json.dumps(log_data, ensure_ascii=False)
+
+
+def get_default_formatter() -> logging.Formatter:
+    """获取默认的JSON格式化器"""
+    return GrpcJSONFormatter()
+
+
+def setup_logging(
+    level: str = "INFO",
+    format_string: Optional[str] = None,
+    enable_grpc_logging: bool = True,
+    log_request_payload: bool = False,
+    log_response_payload: bool = False,
+    handler: Optional[logging.Handler] = None,
+    use_json_format: bool = True
+):
+    """
+    设置SDK日志记录配置
+    
+    Args:
+        level: 日志级别 (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+        format_string: 自定义日志格式（当use_json_format=False时使用）
+        enable_grpc_logging: 是否启用gRPC请求日志
+        log_request_payload: 是否记录请求载荷
+        log_response_payload: 是否记录响应载荷
+        handler: 自定义日志处理器
+        use_json_format: 是否使用JSON格式（默认True）
+    """
+    # 设置日志级别
+    log_level = getattr(logging, level.upper(), logging.INFO)
+    logger.setLevel(log_level)
+    
+    # 清除现有的处理器（只清除SDK的logger）
+    logger.handlers.clear()
+    
+    # 创建处理器
+    if handler is None:
+        handler = logging.StreamHandler(sys.stdout)
+    
+    # 设置日志格式
+    if use_json_format:
+        formatter = get_default_formatter()
+    else:
+        if format_string is None:
+            format_string = "[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s"
+        formatter = logging.Formatter(format_string, datefmt="%Y-%m-%d %H:%M:%S")
+    
+    handler.setFormatter(formatter)
+    
+    # 添加处理器
+    logger.addHandler(handler)
+    
+    # 设置gRPC日志配置
+    logger.grpc_logging_enabled = enable_grpc_logging
+    logger.log_request_payload = log_request_payload
+    logger.log_response_payload = log_response_payload
+    
+    # 防止日志传播到根日志记录器 - 保持SDK日志独立
+    logger.propagate = False
+    
+    # 初始化日志（使用JSON格式）
+    if enable_grpc_logging:
+        log_record = logging.LogRecord(
+            name=logger.name,
+            level=logging.INFO,
+            pathname="",
+            lineno=0,
+            msg="📡 文件中心客户端 gRPC 日志已初始化",
+            args=(),
+            exc_info=None
+        )
+        log_record.log_type = "info"
+        log_record.data = {
+            "level": level,
+            "grpc_logging": enable_grpc_logging,
+            "json_format": use_json_format
+        }
+        logger.handle(log_record)
+
+
+def get_logger() -> logging.Logger:
+    """获取SDK日志记录器"""
+    return logger
+
+
+class GrpcRequestLogger:
+    """gRPC请求日志记录器"""
+    
+    def __init__(self, logger: logging.Logger):
+        self.logger = logger
+        self.enable_grpc_logging = getattr(logger, 'grpc_logging_enabled', True)
+        self.log_request_payload = getattr(logger, 'log_request_payload', False)
+        self.log_response_payload = getattr(logger, 'log_response_payload', False)
+    
+    def log_request_start(self, method_name: str, request_id: str, metadata: Dict[str, Any], 
+                         request_payload: Any = None):
+        """记录请求开始"""
+        if not self.enable_grpc_logging:
+            return
+            
+        # 提取关键元数据
+        user_info = {}
+        if metadata:
+            metadata_dict = dict(metadata) if isinstance(metadata, list) else metadata
+            user_info = {
+                'org_id': metadata_dict.get('x-org-id'),
+                'user_id': metadata_dict.get('x-user-id'),
+                'client_ip': metadata_dict.get('x-client-ip'),
+                'client_version': metadata_dict.get('x-client-version')
+            }
+            user_info = {k: v for k, v in user_info.items() if v is not None}
+        
+        # 创建日志记录
+        log_record = logging.LogRecord(
+            name=self.logger.name,
+            level=logging.INFO,
+            pathname="",
+            lineno=0,
+            msg=f"📤 gRPC 请求: {method_name}",
+            args=(),
+            exc_info=None
+        )
+        
+        # 添加自定义字段
+        log_record.log_type = "request"
+        log_record.uri = method_name
+        log_record.request_id = request_id
+        log_record.data = user_info
+        
+        # 记录请求载荷
+        if request_payload is not None:
+            if isinstance(request_payload, dict):
+                # 已经是字典格式，直接合并
+                log_record.data.update(request_payload)
+            else:
+                # 其他格式，添加到payload字段
+                log_record.data["payload"] = request_payload
+        
+        self.logger.handle(log_record)
+    
+    def log_request_end(self, method_name: str, request_id: str, duration_ms: float, 
+                       response_payload: Any = None, error: Exception = None):
+        """记录请求结束"""
+        if not self.enable_grpc_logging:
+            return
+            
+        if error:
+            # 错误日志
+            log_record = logging.LogRecord(
+                name=self.logger.name,
+                level=logging.ERROR,
+                pathname="",
+                lineno=0,
+                msg=f"❌ gRPC 错误: {method_name} - {str(error)}",
+                args=(),
+                exc_info=(type(error), error, error.__traceback__) if error else None
+            )
+            log_record.log_type = "error"
+            log_record.uri = method_name
+            log_record.request_id = request_id
+            log_record.duration = duration_ms
+            log_record.data = {"error": str(error)}
+            
+            self.logger.handle(log_record)
+        else:
+            # 响应日志
+            log_record = logging.LogRecord(
+                name=self.logger.name,
+                level=logging.INFO,
+                pathname="",
+                lineno=0,
+                msg=f"✅ gRPC 响应: {method_name}",
+                args=(),
+                exc_info=None
+            )
+            log_record.log_type = "response"
+            log_record.uri = method_name
+            log_record.request_id = request_id
+            log_record.duration = duration_ms
+            
+            # 记录响应载荷（如果启用）
+            if self.log_response_payload and response_payload is not None:
+                log_record.data = {"response_payload": self._safe_serialize(response_payload)}
+            
+            self.logger.handle(log_record)
+    
+    def _safe_serialize(self, obj: Any) -> str:
+        """安全地序列化对象，避免敏感信息泄露"""
+        try:
+            if hasattr(obj, 'SerializeToString'):
+                # protobuf 对象
+                return f"<Proto object: {type(obj).__name__}>"
+            elif hasattr(obj, '__dict__'):
+                # 普通对象
+                return f"<Object: {type(obj).__name__}>"
+            else:
+                # 基本类型
+                return str(obj)[:200]  # 限制长度
+        except Exception:
+            return f"<Unserializable: {type(obj).__name__}>"
+
+
+@contextmanager
+def grpc_request_context(method_name: str, request_id: str, metadata: Dict[str, Any], 
+                        request_payload: Any = None):
+    """gRPC请求上下文管理器"""
+    request_logger = GrpcRequestLogger(get_logger())
+    start_time = time.time()
+    
+    try:
+        # 记录请求开始
+        request_logger.log_request_start(method_name, request_id, metadata, request_payload)
+        yield request_logger
+        
+    except Exception as e:
+        # 记录请求错误
+        duration_ms = (time.time() - start_time) * 1000
+        request_logger.log_request_end(method_name, request_id, duration_ms, error=e)
+        raise
+        
+    else:
+        # 记录请求成功结束
+        duration_ms = (time.time() - start_time) * 1000
+        request_logger.log_request_end(method_name, request_id, duration_ms)
+
+
+def log_grpc_call(method_name: str):
+    """gRPC调用日志装饰器"""
+    def decorator(func):
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            # 提取request_id和metadata
+            request_id = kwargs.get('request_id', 'unknown')
+            metadata = kwargs.get('metadata', {})
+            
+            with grpc_request_context(method_name, request_id, metadata) as request_logger:
+                result = func(*args, **kwargs)
+                request_logger.log_request_end(method_name, request_id, 0, response_payload=result)
+                return result
+                
+        @wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            # 提取request_id和metadata
+            request_id = kwargs.get('request_id', 'unknown')
+            metadata = kwargs.get('metadata', {})
+            
+            with grpc_request_context(method_name, request_id, metadata) as request_logger:
+                result = await func(*args, **kwargs)
+                request_logger.log_request_end(method_name, request_id, 0, response_payload=result)
+                return result
+        
+        # 根据函数类型返回对应的包装器
+        import asyncio
+        if asyncio.iscoroutinefunction(func):
+            return async_wrapper
+        else:
+            return sync_wrapper
+    
+    return decorator
+
+
+# 默认初始化（可以被用户重新配置）
+if not logger.handlers:
+    setup_logging()