tamar-model-client 0.1.20__tar.gz → 0.1.22__tar.gz

{tamar_model_client-0.1.20 → tamar_model_client-0.1.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tamar-model-client
-Version: 0.1.20
+Version: 0.1.22
 Summary: A Python SDK for interacting with the Model Manager gRPC service
 Home-page: http://gitlab.tamaredge.top/project-tap/AgentOS/model-manager-client
 Author: Oscar Ou
@@ -532,6 +532,61 @@ response = client.invoke(
    - 启用流式响应减少首字延迟
    - 合理设置 max_tokens 避免浪费
 
+### 🛡️ 熔断降级功能（高可用保障）
+
+SDK 内置了熔断降级机制，当 gRPC 服务不可用时自动切换到 HTTP 服务，确保业务连续性。
+
+#### 工作原理
+1. **正常状态**：所有请求通过高性能的 gRPC 协议
+2. **熔断触发**：当连续失败达到阈值时，熔断器打开
+3. **自动降级**：切换到 HTTP 协议继续提供服务
+4. **定期恢复**：熔断器会定期尝试恢复到 gRPC
+
+#### 启用方式
+```bash
+# 设置环境变量
+export MODEL_CLIENT_RESILIENT_ENABLED=true
+export MODEL_CLIENT_HTTP_FALLBACK_URL=http://localhost:8080
+export MODEL_CLIENT_CIRCUIT_BREAKER_THRESHOLD=5
+export MODEL_CLIENT_CIRCUIT_BREAKER_TIMEOUT=60
+```
+
+#### 使用示例
+```python
+from tamar_model_client import TamarModelClient
+
+# 客户端会自动处理熔断降级，对使用者透明
+client = TamarModelClient()
+
+# 正常使用，无需关心底层协议
+response = client.invoke(request)
+
+# 获取熔断器状态（可选）
+metrics = client.get_resilient_metrics()
+if metrics:
+    print(f"熔断器状态: {metrics['circuit_state']}")
+    print(f"失败次数: {metrics['failure_count']}")
+```
+
+#### 熔断器状态
+- **CLOSED**（关闭）：正常工作状态，请求正常通过
+- **OPEN**（打开）：熔断状态，所有请求直接降级到 HTTP
+- **HALF_OPEN**（半开）：恢复测试状态，允许少量请求测试 gRPC 是否恢复
+
+#### 监控指标
+```python
+# 获取熔断降级指标
+metrics = client.get_resilient_metrics()
+# 返回示例：
+# {
+#     "enabled": true,
+#     "circuit_state": "closed",
+#     "failure_count": 0,
+#     "last_failure_time": null,
+#     "http_fallback_url": "http://localhost:8080"
+# }
+```
+
 ### ⚠️ 注意事项
 
 1. **参数说明**
@@ -595,6 +650,23 @@ MODEL_MANAGER_SERVER_GRPC_MAX_RETRIES=3
 
 # 初始重试延迟（秒，默认 1.0），指数退避
 MODEL_MANAGER_SERVER_GRPC_RETRY_DELAY=1.0
+
+
+# ========================
+# 🛡️ 熔断降级配置（可选）
+# ========================
+
+# 是否启用熔断降级功能（默认 false）
+MODEL_CLIENT_RESILIENT_ENABLED=false
+
+# HTTP 降级服务地址（当 gRPC 不可用时的备用地址）
+MODEL_CLIENT_HTTP_FALLBACK_URL=http://localhost:8080
+
+# 熔断器触发阈值（连续失败多少次后熔断，默认 5）
+MODEL_CLIENT_CIRCUIT_BREAKER_THRESHOLD=5
+
+# 熔断器恢复超时（秒，熔断后多久尝试恢复，默认 60）
+MODEL_CLIENT_CIRCUIT_BREAKER_TIMEOUT=60
 ```
 
 加载后，初始化时无需传参：

{tamar_model_client-0.1.20 → tamar_model_client-0.1.22}/README.md

@@ -502,6 +502,61 @@ response = client.invoke(
    - 启用流式响应减少首字延迟
    - 合理设置 max_tokens 避免浪费
 
+### 🛡️ 熔断降级功能（高可用保障）
+
+SDK 内置了熔断降级机制，当 gRPC 服务不可用时自动切换到 HTTP 服务，确保业务连续性。
+
+#### 工作原理
+1. **正常状态**：所有请求通过高性能的 gRPC 协议
+2. **熔断触发**：当连续失败达到阈值时，熔断器打开
+3. **自动降级**：切换到 HTTP 协议继续提供服务
+4. **定期恢复**：熔断器会定期尝试恢复到 gRPC
+
+#### 启用方式
+```bash
+# 设置环境变量
+export MODEL_CLIENT_RESILIENT_ENABLED=true
+export MODEL_CLIENT_HTTP_FALLBACK_URL=http://localhost:8080
+export MODEL_CLIENT_CIRCUIT_BREAKER_THRESHOLD=5
+export MODEL_CLIENT_CIRCUIT_BREAKER_TIMEOUT=60
+```
+
+#### 使用示例
+```python
+from tamar_model_client import TamarModelClient
+
+# 客户端会自动处理熔断降级，对使用者透明
+client = TamarModelClient()
+
+# 正常使用，无需关心底层协议
+response = client.invoke(request)
+
+# 获取熔断器状态（可选）
+metrics = client.get_resilient_metrics()
+if metrics:
+    print(f"熔断器状态: {metrics['circuit_state']}")
+    print(f"失败次数: {metrics['failure_count']}")
+```
+
+#### 熔断器状态
+- **CLOSED**（关闭）：正常工作状态，请求正常通过
+- **OPEN**（打开）：熔断状态，所有请求直接降级到 HTTP
+- **HALF_OPEN**（半开）：恢复测试状态，允许少量请求测试 gRPC 是否恢复
+
+#### 监控指标
+```python
+# 获取熔断降级指标
+metrics = client.get_resilient_metrics()
+# 返回示例：
+# {
+#     "enabled": true,
+#     "circuit_state": "closed",
+#     "failure_count": 0,
+#     "last_failure_time": null,
+#     "http_fallback_url": "http://localhost:8080"
+# }
+```
+
 ### ⚠️ 注意事项
 
 1. **参数说明**
@@ -565,6 +620,23 @@ MODEL_MANAGER_SERVER_GRPC_MAX_RETRIES=3
 
 # 初始重试延迟（秒，默认 1.0），指数退避
 MODEL_MANAGER_SERVER_GRPC_RETRY_DELAY=1.0
+
+
+# ========================
+# 🛡️ 熔断降级配置（可选）
+# ========================
+
+# 是否启用熔断降级功能（默认 false）
+MODEL_CLIENT_RESILIENT_ENABLED=false
+
+# HTTP 降级服务地址（当 gRPC 不可用时的备用地址）
+MODEL_CLIENT_HTTP_FALLBACK_URL=http://localhost:8080
+
+# 熔断器触发阈值（连续失败多少次后熔断，默认 5）
+MODEL_CLIENT_CIRCUIT_BREAKER_THRESHOLD=5
+
+# 熔断器恢复超时（秒，熔断后多久尝试恢复，默认 60）
+MODEL_CLIENT_CIRCUIT_BREAKER_TIMEOUT=60
 ```
 
 加载后，初始化时无需传参：

{tamar_model_client-0.1.20 → tamar_model_client-0.1.22}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="tamar-model-client",
-    version="0.1.20",
+    version="0.1.22",
     description="A Python SDK for interacting with the Model Manager gRPC service",
     author="Oscar Ou",
     author_email="oscar.ou@tamaredge.ai",

{tamar_model_client-0.1.20 → tamar_model_client-0.1.22}/tamar_model_client/async_client.py

@@ -22,6 +22,7 @@ import asyncio
 import atexit
 import json
 import logging
+import random
 import time
 from typing import Optional, AsyncIterator, Union
 
@@ -31,7 +32,7 @@ from grpc import RpcError
 from .core import (
     generate_request_id,
     set_request_id,
-    setup_logger,
+    get_protected_logger,
     MAX_MESSAGE_LENGTH
 )
 from .core.base_client import BaseClient
@@ -42,12 +43,13 @@ from .exceptions import ConnectionError, TamarModelException
 from .error_handler import EnhancedRetryHandler
 from .schemas import ModelRequest, ModelResponse, BatchModelRequest, BatchModelResponse
 from .generated import model_service_pb2, model_service_pb2_grpc
+from .core.http_fallback import AsyncHttpFallbackMixin
 
-# 配置日志记录器
-logger = setup_logger(__name__)
+# 配置日志记录器（使用受保护的logger）
+logger = get_protected_logger(__name__)
 
 
-class AsyncTamarModelClient(BaseClient):
+class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
     """
     Tamar Model Client 异步客户端
     
@@ -127,6 +129,10 @@ class AsyncTamarModelClient(BaseClient):
             self._closed = True
             logger.info("🔒 gRPC channel closed",
                         extra={"log_type": "info", "data": {"status": "closed"}})
+        
+        # 清理 HTTP session（如果有）
+        if self.resilient_enabled:
+            await self._cleanup_http_session()
 
     async def __aenter__(self):
         """异步上下文管理器入口"""
@@ -238,7 +244,16 @@ class AsyncTamarModelClient(BaseClient):
         Raises:
             TamarModelException: 当所有重试都失败时
         """
-        return await self.retry_handler.execute_with_retry(func, *args, **kwargs)
+        # 从kwargs中提取request_id（如果有的话），然后移除它
+        request_id = kwargs.pop('request_id', None) or get_request_id()
+        
+        # 构建包含request_id的上下文
+        context = {
+            'method': func.__name__ if hasattr(func, '__name__') else 'unknown',
+            'client_version': 'async',
+            'request_id': request_id,
+        }
+        return await self.retry_handler.execute_with_retry(func, *args, context=context, **kwargs)
 
     async def _retry_request_stream(self, func, *args, **kwargs):
         """
@@ -254,33 +269,174 @@ class AsyncTamarModelClient(BaseClient):
         Returns:
             AsyncIterator: 流式响应迭代器
         """
+        # 记录方法开始时间
+        import time
+        method_start_time = time.time()
+        
+        # 从kwargs中提取request_id（如果有的话），然后移除它
+        request_id = kwargs.pop('request_id', None) or get_request_id()
+        
         last_exception = None
+        context = {
+            'method': 'stream',
+            'client_version': 'async',
+            'request_id': request_id,
+        }
         
         for attempt in range(self.max_retries + 1):
             try:
+                context['retry_count'] = attempt
                 # 尝试创建流
                 async for item in func(*args, **kwargs):
                     yield item
                 return
                 
             except RpcError as e:
-                last_exception = e
-                if attempt < self.max_retries:
+                # 使用智能重试判断
+                context['retry_count'] = attempt
+                
+                # 创建错误上下文并判断是否应该重试
+                from .exceptions import ErrorContext, get_retry_policy
+                error_context = ErrorContext(e, context)
+                error_code = e.code()
+                policy = get_retry_policy(error_code)
+                
+                # 先检查错误级别的 max_attempts 配置
+                # max_attempts 表示最大重试次数（不包括初始请求）
+                error_max_attempts = policy.get('max_attempts', self.max_retries)
+                if attempt >= error_max_attempts:
+                    should_retry = False
+                elif attempt >= self.max_retries:
+                    should_retry = False
+                else:
+                    retryable = policy.get('retryable', False)
+                    if retryable == True:
+                        should_retry = True
+                    elif retryable == 'conditional':
+                        # 条件重试，特殊处理 CANCELLED
+                        if error_code == grpc.StatusCode.CANCELLED:
+                            should_retry = error_context.is_network_cancelled()
+                        else:
+                            should_retry = self._check_error_details_for_retry(e)
+                    else:
+                        should_retry = False
+                
+                if should_retry:
+                    current_duration = time.time() - method_start_time
+                    log_data = {
+                        "log_type": "info",
+                        "request_id": context.get('request_id'),
+                        "data": {
+                            "error_code": e.code().name if e.code() else 'UNKNOWN',
+                            "retry_count": attempt,
+                            "max_retries": self.max_retries,
+                            "method": "stream"
+                        },
+                        "duration": current_duration
+                    }
                     logger.warning(
-                        f"Stream attempt {attempt + 1}/{self.max_retries + 1} failed: {e.code()}",
-                        extra={"retry_count": attempt, "error_code": str(e.code())}
+                        f"Stream attempt {attempt + 1}/{self.max_retries + 1} failed: {e.code()} (will retry)",
+                        extra=log_data
                     )
-                    await asyncio.sleep(self.retry_delay * (attempt + 1))
+                    
+                    # 计算退避时间
+                    delay = self._calculate_backoff(attempt, error_code)
+                    await asyncio.sleep(delay)
                 else:
+                    # 不重试或已达到最大重试次数
+                    current_duration = time.time() - method_start_time
+                    log_data = {
+                        "log_type": "info",
+                        "request_id": context.get('request_id'),
+                        "data": {
+                            "error_code": e.code().name if e.code() else 'UNKNOWN',
+                            "retry_count": attempt,
+                            "max_retries": self.max_retries,
+                            "method": "stream",
+                            "will_retry": False
+                        },
+                        "duration": current_duration
+                    }
+                    logger.error(
+                        f"Stream failed: {e.code()} (no retry)",
+                        extra=log_data
+                    )
+                    context['duration'] = current_duration
+                    last_exception = self.error_handler.handle_error(e, context)
                     break
+                    
+                last_exception = e
+                
             except Exception as e:
+                context['retry_count'] = attempt
                 raise TamarModelException(str(e)) from e
         
         if last_exception:
-            raise self.error_handler.handle_error(last_exception, {"retry_count": self.max_retries})
+            if isinstance(last_exception, TamarModelException):
+                raise last_exception
+            else:
+                raise self.error_handler.handle_error(last_exception, context)
         else:
             raise TamarModelException("Unknown streaming error occurred")
 
+    def _check_error_details_for_retry(self, error: RpcError) -> bool:
+        """检查错误详情决定是否重试"""
+        error_message = error.details().lower() if error.details() else ""
+        
+        # 可重试的错误模式
+        retryable_patterns = [
+            'temporary', 'timeout', 'unavailable', 
+            'connection', 'network', 'try again'
+        ]
+        
+        for pattern in retryable_patterns:
+            if pattern in error_message:
+                return True
+                
+        return False
+    
+    def _calculate_backoff(self, attempt: int, error_code = None) -> float:
+        """
+        计算退避时间，支持不同的退避策略
+        
+        Args:
+            attempt: 当前重试次数
+            error_code: gRPC错误码，用于确定退避策略
+        """
+        max_delay = 60.0
+        base_delay = self.retry_delay
+        
+        # 获取错误的重试策略
+        if error_code:
+            from .exceptions import get_retry_policy
+            policy = get_retry_policy(error_code)
+            backoff_type = policy.get('backoff', 'exponential')
+            use_jitter = policy.get('jitter', False)
+        else:
+            backoff_type = 'exponential'
+            use_jitter = False
+        
+        # 根据退避类型计算延迟
+        if backoff_type == 'linear':
+            # 线性退避：delay * (attempt + 1)
+            delay = min(base_delay * (attempt + 1), max_delay)
+        else:
+            # 指数退避：delay * 2^attempt
+            delay = min(base_delay * (2 ** attempt), max_delay)
+        
+        # 添加抖动
+        if use_jitter:
+            jitter_factor = 0.2  # 增加抖动范围，减少竞争
+            jitter = random.uniform(0, delay * jitter_factor)
+            delay += jitter
+        else:
+            # 默认的小量抖动，避免完全同步
+            jitter_factor = 0.05
+            jitter = random.uniform(0, delay * jitter_factor)
+            delay += jitter
+            
+        return delay
+
     async def _stream(self, request, metadata, invoke_timeout) -> AsyncIterator[ModelResponse]:
         """
         处理流式响应
@@ -328,7 +484,7 @@ class AsyncTamarModelClient(BaseClient):
         chunk_count = 0
 
         # 使用重试逻辑获取流生成器
-        stream_generator = self._retry_request_stream(self._stream, request, metadata, invoke_timeout)
+        stream_generator = self._retry_request_stream(self._stream, request, metadata, invoke_timeout, request_id=get_request_id())
 
         try:
             async for response in stream_generator:
@@ -424,6 +580,12 @@ class AsyncTamarModelClient(BaseClient):
             ValidationError: 输入验证失败。
             ConnectionError: 连接服务端失败。
         """
+        # 如果启用了熔断且熔断器打开，直接走 HTTP
+        if self.resilient_enabled and self.circuit_breaker and self.circuit_breaker.is_open:
+            if self.http_fallback_url:
+                logger.warning("🔻 Circuit breaker is OPEN, using HTTP fallback")
+                return await self._invoke_http_fallback(model_request, timeout, request_id)
+                
         await self._ensure_initialized()
 
         if not self.default_payload:
@@ -477,7 +639,7 @@ class AsyncTamarModelClient(BaseClient):
                 # 对于流式响应，直接返回带日志记录的包装器
                 return self._stream_with_logging(request, metadata, invoke_timeout, start_time, model_request)
             else:
-                result = await self._retry_request(self._invoke_request, request, metadata, invoke_timeout)
+                result = await self._retry_request(self._invoke_request, request, metadata, invoke_timeout, request_id=request_id)
 
                 # 记录非流式响应的成功日志
                 duration = time.time() - start_time
@@ -491,9 +653,14 @@ class AsyncTamarModelClient(BaseClient):
                         "data": ResponseHandler.build_log_data(model_request, result)
                     }
                 )
+                
+                # 记录成功（如果启用了熔断）
+                if self.resilient_enabled and self.circuit_breaker:
+                    self.circuit_breaker.record_success()
+                    
                 return result
                 
-        except grpc.RpcError as e:
+        except (ConnectionError, grpc.RpcError) as e:
             duration = time.time() - start_time
             error_message = f"❌ Invoke gRPC failed: {str(e)}"
             logger.error(error_message, exc_info=True,
@@ -506,6 +673,18 @@ class AsyncTamarModelClient(BaseClient):
                                  error=e
                              )
                          })
+            
+            # 记录失败并尝试降级（如果启用了熔断）
+            if self.resilient_enabled and self.circuit_breaker:
+                # 将错误码传递给熔断器，用于智能失败统计
+                error_code = e.code() if hasattr(e, 'code') else None
+                self.circuit_breaker.record_failure(error_code)
+                
+                # 如果可以降级，则降级
+                if self.http_fallback_url and self.circuit_breaker.should_fallback():
+                    logger.warning(f"🔻 gRPC failed, falling back to HTTP: {str(e)}")
+                    return await self._invoke_http_fallback(model_request, timeout, request_id)
+            
             raise e
         except Exception as e:
             duration = time.time() - start_time
@@ -590,7 +769,8 @@ class AsyncTamarModelClient(BaseClient):
                 self.stub.BatchInvoke,
                 batch_request,
                 metadata=metadata,
-                timeout=invoke_timeout
+                timeout=invoke_timeout,
+                request_id=request_id
             )
 
             # 构建响应对象

tamar_model_client-0.1.22/tamar_model_client/circuit_breaker.py

@@ -0,0 +1,140 @@
+"""
+Circuit Breaker implementation for resilient client
+
+This module provides a thread-safe circuit breaker pattern implementation
+to handle failures gracefully and prevent cascading failures.
+"""
+
+import time
+import logging
+from enum import Enum
+from threading import Lock
+from typing import Optional
+
+from .core.logging_setup import get_protected_logger
+
+logger = get_protected_logger(__name__)
+
+
+class CircuitState(Enum):
+    """Circuit breaker states"""
+    CLOSED = "closed"      # Normal operation
+    OPEN = "open"          # Circuit is broken, requests fail fast
+    HALF_OPEN = "half_open"  # Testing if service has recovered
+
+
+class CircuitBreaker:
+    """
+    Thread-safe circuit breaker implementation
+    
+    The circuit breaker prevents cascading failures by failing fast when
+    a service is unavailable, and automatically recovers when the service
+    becomes available again.
+    """
+    
+    def __init__(self, failure_threshold: int = 5, recovery_timeout: int = 60):
+        """
+        Initialize the circuit breaker
+        
+        Args:
+            failure_threshold: Number of consecutive failures before opening circuit
+            recovery_timeout: Seconds to wait before attempting recovery
+        """
+        self.failure_threshold = failure_threshold
+        self.recovery_timeout = recovery_timeout
+        self.failure_count = 0
+        self.last_failure_time: Optional[float] = None
+        self.state = CircuitState.CLOSED
+        self._lock = Lock()
+        
+    @property
+    def is_open(self) -> bool:
+        """Check if circuit breaker is open"""
+        with self._lock:
+            if self.state == CircuitState.OPEN:
+                # Check if we should attempt recovery
+                if (self.last_failure_time and 
+                    time.time() - self.last_failure_time > self.recovery_timeout):
+                    self.state = CircuitState.HALF_OPEN
+                    logger.info("🔄 Circuit breaker entering HALF_OPEN state")
+                    return False
+                return True
+            return False
+    
+    def record_success(self) -> None:
+        """Record a successful request"""
+        with self._lock:
+            if self.state == CircuitState.HALF_OPEN:
+                # Success in half-open state means service has recovered
+                self.state = CircuitState.CLOSED
+                self.failure_count = 0
+                logger.info("🔺 Circuit breaker recovered to CLOSED state")
+            elif self.state == CircuitState.CLOSED and self.failure_count > 0:
+                # Reset failure count on success
+                self.failure_count = 0
+    
+    def record_failure(self, error_code=None) -> None:
+        """
+        Record a failed request
+        
+        Args:
+            error_code: gRPC error code for failure classification
+        """
+        with self._lock:
+            # 对于某些错误类型，不计入熔断统计或权重较低
+            if error_code and self._should_ignore_for_circuit_breaker(error_code):
+                return
+                
+            # ABORTED 错误权重较低，因为通常是瞬时的并发问题
+            import grpc
+            if error_code == grpc.StatusCode.ABORTED:
+                # ABORTED 错误只计算半个失败
+                self.failure_count += 0.5
+            else:
+                self.failure_count += 1
+                
+            self.last_failure_time = time.time()
+            
+            if self.failure_count >= self.failure_threshold:
+                if self.state != CircuitState.OPEN:
+                    self.state = CircuitState.OPEN
+                    logger.warning(
+                        f"🔻 Circuit breaker OPENED after {self.failure_count} failures",
+                        extra={
+                            "failure_count": self.failure_count,
+                            "threshold": self.failure_threshold,
+                            "trigger_error": error_code.name if error_code else "unknown"
+                        }
+                    )
+    
+    def _should_ignore_for_circuit_breaker(self, error_code) -> bool:
+        """
+        判断错误是否应该被熔断器忽略
+        
+        某些错误不应该触发熔断：
+        - 客户端主动取消的请求
+        - 认证相关错误（不代表服务不可用）
+        """
+        import grpc
+        ignored_codes = {
+            grpc.StatusCode.UNAUTHENTICATED,    # 认证问题，不是服务问题
+            grpc.StatusCode.PERMISSION_DENIED,  # 权限问题，不是服务问题
+            grpc.StatusCode.INVALID_ARGUMENT,   # 参数错误，不是服务问题
+        }
+        return error_code in ignored_codes
+    
+    def should_fallback(self) -> bool:
+        """Check if fallback should be used"""
+        return self.is_open and self.state != CircuitState.HALF_OPEN
+    
+    def get_state(self) -> str:
+        """Get current circuit state"""
+        return self.state.value
+    
+    def reset(self) -> None:
+        """Reset circuit breaker to initial state"""
+        with self._lock:
+            self.state = CircuitState.CLOSED
+            self.failure_count = 0
+            self.last_failure_time = None
+            logger.info("🔄 Circuit breaker reset to CLOSED state")

{tamar_model_client-0.1.20 → tamar_model_client-0.1.22}/tamar_model_client/core/__init__.py

@@ -16,6 +16,9 @@ from .utils import (
 from .logging_setup import (
     setup_logger,
     RequestIdFilter,
+    TamarLoggerAdapter,
+    get_protected_logger,
+    reset_logger_config,
     MAX_MESSAGE_LENGTH
 )
 
@@ -30,5 +33,8 @@ __all__ = [
     # Logging
     'setup_logger',
     'RequestIdFilter',
+    'TamarLoggerAdapter',
+    'get_protected_logger',
+    'reset_logger_config',
     'MAX_MESSAGE_LENGTH',
 ]