tamar-model-client 0.1.20__py3-none-any.whl → 0.1.22__py3-none-any.whl

tamar_model_client/async_client.py

@@ -22,6 +22,7 @@ import asyncio
 import atexit
 import json
 import logging
+import random
 import time
 from typing import Optional, AsyncIterator, Union
 
@@ -31,7 +32,7 @@ from grpc import RpcError
 from .core import (
     generate_request_id,
     set_request_id,
-    setup_logger,
+    get_protected_logger,
     MAX_MESSAGE_LENGTH
 )
 from .core.base_client import BaseClient
@@ -42,12 +43,13 @@ from .exceptions import ConnectionError, TamarModelException
 from .error_handler import EnhancedRetryHandler
 from .schemas import ModelRequest, ModelResponse, BatchModelRequest, BatchModelResponse
 from .generated import model_service_pb2, model_service_pb2_grpc
+from .core.http_fallback import AsyncHttpFallbackMixin
 
-# 配置日志记录器
-logger = setup_logger(__name__)
+# 配置日志记录器（使用受保护的logger）
+logger = get_protected_logger(__name__)
 
 
-class AsyncTamarModelClient(BaseClient):
+class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
     """
     Tamar Model Client 异步客户端
     
@@ -127,6 +129,10 @@ class AsyncTamarModelClient(BaseClient):
             self._closed = True
             logger.info("🔒 gRPC channel closed",
                         extra={"log_type": "info", "data": {"status": "closed"}})
+        
+        # 清理 HTTP session（如果有）
+        if self.resilient_enabled:
+            await self._cleanup_http_session()
 
     async def __aenter__(self):
         """异步上下文管理器入口"""
@@ -238,7 +244,16 @@ class AsyncTamarModelClient(BaseClient):
         Raises:
             TamarModelException: 当所有重试都失败时
         """
-        return await self.retry_handler.execute_with_retry(func, *args, **kwargs)
+        # 从kwargs中提取request_id（如果有的话），然后移除它
+        request_id = kwargs.pop('request_id', None) or get_request_id()
+        
+        # 构建包含request_id的上下文
+        context = {
+            'method': func.__name__ if hasattr(func, '__name__') else 'unknown',
+            'client_version': 'async',
+            'request_id': request_id,
+        }
+        return await self.retry_handler.execute_with_retry(func, *args, context=context, **kwargs)
 
     async def _retry_request_stream(self, func, *args, **kwargs):
         """
@@ -254,33 +269,174 @@ class AsyncTamarModelClient(BaseClient):
         Returns:
             AsyncIterator: 流式响应迭代器
         """
+        # 记录方法开始时间
+        import time
+        method_start_time = time.time()
+        
+        # 从kwargs中提取request_id（如果有的话），然后移除它
+        request_id = kwargs.pop('request_id', None) or get_request_id()
+        
         last_exception = None
+        context = {
+            'method': 'stream',
+            'client_version': 'async',
+            'request_id': request_id,
+        }
         
         for attempt in range(self.max_retries + 1):
             try:
+                context['retry_count'] = attempt
                 # 尝试创建流
                 async for item in func(*args, **kwargs):
                     yield item
                 return
                 
             except RpcError as e:
-                last_exception = e
-                if attempt < self.max_retries:
+                # 使用智能重试判断
+                context['retry_count'] = attempt
+                
+                # 创建错误上下文并判断是否应该重试
+                from .exceptions import ErrorContext, get_retry_policy
+                error_context = ErrorContext(e, context)
+                error_code = e.code()
+                policy = get_retry_policy(error_code)
+                
+                # 先检查错误级别的 max_attempts 配置
+                # max_attempts 表示最大重试次数（不包括初始请求）
+                error_max_attempts = policy.get('max_attempts', self.max_retries)
+                if attempt >= error_max_attempts:
+                    should_retry = False
+                elif attempt >= self.max_retries:
+                    should_retry = False
+                else:
+                    retryable = policy.get('retryable', False)
+                    if retryable == True:
+                        should_retry = True
+                    elif retryable == 'conditional':
+                        # 条件重试，特殊处理 CANCELLED
+                        if error_code == grpc.StatusCode.CANCELLED:
+                            should_retry = error_context.is_network_cancelled()
+                        else:
+                            should_retry = self._check_error_details_for_retry(e)
+                    else:
+                        should_retry = False
+                
+                if should_retry:
+                    current_duration = time.time() - method_start_time
+                    log_data = {
+                        "log_type": "info",
+                        "request_id": context.get('request_id'),
+                        "data": {
+                            "error_code": e.code().name if e.code() else 'UNKNOWN',
+                            "retry_count": attempt,
+                            "max_retries": self.max_retries,
+                            "method": "stream"
+                        },
+                        "duration": current_duration
+                    }
                     logger.warning(
-                        f"Stream attempt {attempt + 1}/{self.max_retries + 1} failed: {e.code()}",
-                        extra={"retry_count": attempt, "error_code": str(e.code())}
+                        f"Stream attempt {attempt + 1}/{self.max_retries + 1} failed: {e.code()} (will retry)",
+                        extra=log_data
                     )
-                    await asyncio.sleep(self.retry_delay * (attempt + 1))
+                    
+                    # 计算退避时间
+                    delay = self._calculate_backoff(attempt, error_code)
+                    await asyncio.sleep(delay)
                 else:
+                    # 不重试或已达到最大重试次数
+                    current_duration = time.time() - method_start_time
+                    log_data = {
+                        "log_type": "info",
+                        "request_id": context.get('request_id'),
+                        "data": {
+                            "error_code": e.code().name if e.code() else 'UNKNOWN',
+                            "retry_count": attempt,
+                            "max_retries": self.max_retries,
+                            "method": "stream",
+                            "will_retry": False
+                        },
+                        "duration": current_duration
+                    }
+                    logger.error(
+                        f"Stream failed: {e.code()} (no retry)",
+                        extra=log_data
+                    )
+                    context['duration'] = current_duration
+                    last_exception = self.error_handler.handle_error(e, context)
                     break
+                    
+                last_exception = e
+                
             except Exception as e:
+                context['retry_count'] = attempt
                 raise TamarModelException(str(e)) from e
         
         if last_exception:
-            raise self.error_handler.handle_error(last_exception, {"retry_count": self.max_retries})
+            if isinstance(last_exception, TamarModelException):
+                raise last_exception
+            else:
+                raise self.error_handler.handle_error(last_exception, context)
         else:
             raise TamarModelException("Unknown streaming error occurred")
 
+    def _check_error_details_for_retry(self, error: RpcError) -> bool:
+        """检查错误详情决定是否重试"""
+        error_message = error.details().lower() if error.details() else ""
+        
+        # 可重试的错误模式
+        retryable_patterns = [
+            'temporary', 'timeout', 'unavailable', 
+            'connection', 'network', 'try again'
+        ]
+        
+        for pattern in retryable_patterns:
+            if pattern in error_message:
+                return True
+                
+        return False
+    
+    def _calculate_backoff(self, attempt: int, error_code = None) -> float:
+        """
+        计算退避时间，支持不同的退避策略
+        
+        Args:
+            attempt: 当前重试次数
+            error_code: gRPC错误码，用于确定退避策略
+        """
+        max_delay = 60.0
+        base_delay = self.retry_delay
+        
+        # 获取错误的重试策略
+        if error_code:
+            from .exceptions import get_retry_policy
+            policy = get_retry_policy(error_code)
+            backoff_type = policy.get('backoff', 'exponential')
+            use_jitter = policy.get('jitter', False)
+        else:
+            backoff_type = 'exponential'
+            use_jitter = False
+        
+        # 根据退避类型计算延迟
+        if backoff_type == 'linear':
+            # 线性退避：delay * (attempt + 1)
+            delay = min(base_delay * (attempt + 1), max_delay)
+        else:
+            # 指数退避：delay * 2^attempt
+            delay = min(base_delay * (2 ** attempt), max_delay)
+        
+        # 添加抖动
+        if use_jitter:
+            jitter_factor = 0.2  # 增加抖动范围，减少竞争
+            jitter = random.uniform(0, delay * jitter_factor)
+            delay += jitter
+        else:
+            # 默认的小量抖动，避免完全同步
+            jitter_factor = 0.05
+            jitter = random.uniform(0, delay * jitter_factor)
+            delay += jitter
+            
+        return delay
+
     async def _stream(self, request, metadata, invoke_timeout) -> AsyncIterator[ModelResponse]:
         """
         处理流式响应
@@ -328,7 +484,7 @@ class AsyncTamarModelClient(BaseClient):
         chunk_count = 0
 
         # 使用重试逻辑获取流生成器
-        stream_generator = self._retry_request_stream(self._stream, request, metadata, invoke_timeout)
+        stream_generator = self._retry_request_stream(self._stream, request, metadata, invoke_timeout, request_id=get_request_id())
 
         try:
             async for response in stream_generator:
@@ -424,6 +580,12 @@ class AsyncTamarModelClient(BaseClient):
             ValidationError: 输入验证失败。
             ConnectionError: 连接服务端失败。
         """
+        # 如果启用了熔断且熔断器打开，直接走 HTTP
+        if self.resilient_enabled and self.circuit_breaker and self.circuit_breaker.is_open:
+            if self.http_fallback_url:
+                logger.warning("🔻 Circuit breaker is OPEN, using HTTP fallback")
+                return await self._invoke_http_fallback(model_request, timeout, request_id)
+                
         await self._ensure_initialized()
 
         if not self.default_payload:
@@ -477,7 +639,7 @@ class AsyncTamarModelClient(BaseClient):
                 # 对于流式响应，直接返回带日志记录的包装器
                 return self._stream_with_logging(request, metadata, invoke_timeout, start_time, model_request)
             else:
-                result = await self._retry_request(self._invoke_request, request, metadata, invoke_timeout)
+                result = await self._retry_request(self._invoke_request, request, metadata, invoke_timeout, request_id=request_id)
 
                 # 记录非流式响应的成功日志
                 duration = time.time() - start_time
@@ -491,9 +653,14 @@ class AsyncTamarModelClient(BaseClient):
                         "data": ResponseHandler.build_log_data(model_request, result)
                     }
                 )
+                
+                # 记录成功（如果启用了熔断）
+                if self.resilient_enabled and self.circuit_breaker:
+                    self.circuit_breaker.record_success()
+                    
                 return result
                 
-        except grpc.RpcError as e:
+        except (ConnectionError, grpc.RpcError) as e:
             duration = time.time() - start_time
             error_message = f"❌ Invoke gRPC failed: {str(e)}"
             logger.error(error_message, exc_info=True,
@@ -506,6 +673,18 @@ class AsyncTamarModelClient(BaseClient):
                                  error=e
                              )
                          })
+            
+            # 记录失败并尝试降级（如果启用了熔断）
+            if self.resilient_enabled and self.circuit_breaker:
+                # 将错误码传递给熔断器，用于智能失败统计
+                error_code = e.code() if hasattr(e, 'code') else None
+                self.circuit_breaker.record_failure(error_code)
+                
+                # 如果可以降级，则降级
+                if self.http_fallback_url and self.circuit_breaker.should_fallback():
+                    logger.warning(f"🔻 gRPC failed, falling back to HTTP: {str(e)}")
+                    return await self._invoke_http_fallback(model_request, timeout, request_id)
+            
             raise e
         except Exception as e:
             duration = time.time() - start_time
@@ -590,7 +769,8 @@ class AsyncTamarModelClient(BaseClient):
                 self.stub.BatchInvoke,
                 batch_request,
                 metadata=metadata,
-                timeout=invoke_timeout
+                timeout=invoke_timeout,
+                request_id=request_id
             )
 
             # 构建响应对象

tamar_model_client/circuit_breaker.py

@@ -0,0 +1,140 @@
+"""
+Circuit Breaker implementation for resilient client
+
+This module provides a thread-safe circuit breaker pattern implementation
+to handle failures gracefully and prevent cascading failures.
+"""
+
+import time
+import logging
+from enum import Enum
+from threading import Lock
+from typing import Optional
+
+from .core.logging_setup import get_protected_logger
+
+logger = get_protected_logger(__name__)
+
+
+class CircuitState(Enum):
+    """Circuit breaker states"""
+    CLOSED = "closed"      # Normal operation
+    OPEN = "open"          # Circuit is broken, requests fail fast
+    HALF_OPEN = "half_open"  # Testing if service has recovered
+
+
+class CircuitBreaker:
+    """
+    Thread-safe circuit breaker implementation
+    
+    The circuit breaker prevents cascading failures by failing fast when
+    a service is unavailable, and automatically recovers when the service
+    becomes available again.
+    """
+    
+    def __init__(self, failure_threshold: int = 5, recovery_timeout: int = 60):
+        """
+        Initialize the circuit breaker
+        
+        Args:
+            failure_threshold: Number of consecutive failures before opening circuit
+            recovery_timeout: Seconds to wait before attempting recovery
+        """
+        self.failure_threshold = failure_threshold
+        self.recovery_timeout = recovery_timeout
+        self.failure_count = 0
+        self.last_failure_time: Optional[float] = None
+        self.state = CircuitState.CLOSED
+        self._lock = Lock()
+        
+    @property
+    def is_open(self) -> bool:
+        """Check if circuit breaker is open"""
+        with self._lock:
+            if self.state == CircuitState.OPEN:
+                # Check if we should attempt recovery
+                if (self.last_failure_time and 
+                    time.time() - self.last_failure_time > self.recovery_timeout):
+                    self.state = CircuitState.HALF_OPEN
+                    logger.info("🔄 Circuit breaker entering HALF_OPEN state")
+                    return False
+                return True
+            return False
+    
+    def record_success(self) -> None:
+        """Record a successful request"""
+        with self._lock:
+            if self.state == CircuitState.HALF_OPEN:
+                # Success in half-open state means service has recovered
+                self.state = CircuitState.CLOSED
+                self.failure_count = 0
+                logger.info("🔺 Circuit breaker recovered to CLOSED state")
+            elif self.state == CircuitState.CLOSED and self.failure_count > 0:
+                # Reset failure count on success
+                self.failure_count = 0
+    
+    def record_failure(self, error_code=None) -> None:
+        """
+        Record a failed request
+        
+        Args:
+            error_code: gRPC error code for failure classification
+        """
+        with self._lock:
+            # 对于某些错误类型，不计入熔断统计或权重较低
+            if error_code and self._should_ignore_for_circuit_breaker(error_code):
+                return
+                
+            # ABORTED 错误权重较低，因为通常是瞬时的并发问题
+            import grpc
+            if error_code == grpc.StatusCode.ABORTED:
+                # ABORTED 错误只计算半个失败
+                self.failure_count += 0.5
+            else:
+                self.failure_count += 1
+                
+            self.last_failure_time = time.time()
+            
+            if self.failure_count >= self.failure_threshold:
+                if self.state != CircuitState.OPEN:
+                    self.state = CircuitState.OPEN
+                    logger.warning(
+                        f"🔻 Circuit breaker OPENED after {self.failure_count} failures",
+                        extra={
+                            "failure_count": self.failure_count,
+                            "threshold": self.failure_threshold,
+                            "trigger_error": error_code.name if error_code else "unknown"
+                        }
+                    )
+    
+    def _should_ignore_for_circuit_breaker(self, error_code) -> bool:
+        """
+        判断错误是否应该被熔断器忽略
+        
+        某些错误不应该触发熔断：
+        - 客户端主动取消的请求
+        - 认证相关错误（不代表服务不可用）
+        """
+        import grpc
+        ignored_codes = {
+            grpc.StatusCode.UNAUTHENTICATED,    # 认证问题，不是服务问题
+            grpc.StatusCode.PERMISSION_DENIED,  # 权限问题，不是服务问题
+            grpc.StatusCode.INVALID_ARGUMENT,   # 参数错误，不是服务问题
+        }
+        return error_code in ignored_codes
+    
+    def should_fallback(self) -> bool:
+        """Check if fallback should be used"""
+        return self.is_open and self.state != CircuitState.HALF_OPEN
+    
+    def get_state(self) -> str:
+        """Get current circuit state"""
+        return self.state.value
+    
+    def reset(self) -> None:
+        """Reset circuit breaker to initial state"""
+        with self._lock:
+            self.state = CircuitState.CLOSED
+            self.failure_count = 0
+            self.last_failure_time = None
+            logger.info("🔄 Circuit breaker reset to CLOSED state")

tamar_model_client/core/__init__.py

@@ -16,6 +16,9 @@ from .utils import (
 from .logging_setup import (
     setup_logger,
     RequestIdFilter,
+    TamarLoggerAdapter,
+    get_protected_logger,
+    reset_logger_config,
     MAX_MESSAGE_LENGTH
 )
 
@@ -30,5 +33,8 @@ __all__ = [
     # Logging
     'setup_logger',
     'RequestIdFilter',
+    'TamarLoggerAdapter',
+    'get_protected_logger',
+    'reset_logger_config',
     'MAX_MESSAGE_LENGTH',
 ]

tamar_model_client/core/base_client.py

@@ -12,7 +12,7 @@ from abc import ABC, abstractmethod
 
 from ..auth import JWTAuthHandler
 from ..error_handler import GrpcErrorHandler, ErrorRecoveryStrategy
-from .logging_setup import MAX_MESSAGE_LENGTH, setup_logger
+from .logging_setup import MAX_MESSAGE_LENGTH, get_protected_logger
 
 
 class BaseClient(ABC):
@@ -79,7 +79,7 @@ class BaseClient(ABC):
             os.getenv("MODEL_MANAGER_SERVER_GRPC_RETRY_DELAY", 1.0))
         
         # === 日志配置 ===
-        self.logger = setup_logger(logger_name or __name__)
+        self.logger = get_protected_logger(logger_name or __name__)
         
         # === 错误处理器 ===
         self.error_handler = GrpcErrorHandler(self.logger)
@@ -87,6 +87,9 @@ class BaseClient(ABC):
         
         # === 连接状态 ===
         self._closed = False
+        
+        # === 熔断降级配置 ===
+        self._init_resilient_features()
     
     def build_channel_options(self) -> list:
         """
@@ -165,4 +168,54 @@ class BaseClient(ABC):
     @abstractmethod
     def __exit__(self, exc_type, exc_val, exc_tb):
         """退出上下文管理器（由子类实现）"""
-        pass
+        pass
+    
+    def _init_resilient_features(self):
+        """初始化熔断降级特性"""
+        # 是否启用熔断降级
+        self.resilient_enabled = os.getenv('MODEL_CLIENT_RESILIENT_ENABLED', 'false').lower() == 'true'
+        
+        if self.resilient_enabled:
+            # HTTP 降级地址
+            self.http_fallback_url = os.getenv('MODEL_CLIENT_HTTP_FALLBACK_URL')
+            
+            if not self.http_fallback_url:
+                self.logger.warning("🔶 Resilient mode enabled but MODEL_CLIENT_HTTP_FALLBACK_URL not set")
+                self.resilient_enabled = False
+                return
+            
+            # 初始化熔断器
+            from ..circuit_breaker import CircuitBreaker
+            self.circuit_breaker = CircuitBreaker(
+                failure_threshold=int(os.getenv('MODEL_CLIENT_CIRCUIT_BREAKER_THRESHOLD', '5')),
+                recovery_timeout=int(os.getenv('MODEL_CLIENT_CIRCUIT_BREAKER_TIMEOUT', '60'))
+            )
+            
+            # HTTP 客户端（延迟初始化）
+            self._http_client = None
+            self._http_session = None  # 异步客户端使用
+            
+            self.logger.info(
+                "🛡️ Resilient mode enabled",
+                extra={
+                    "http_fallback_url": self.http_fallback_url,
+                    "circuit_breaker_threshold": self.circuit_breaker.failure_threshold,
+                    "circuit_breaker_timeout": self.circuit_breaker.recovery_timeout
+                }
+            )
+        else:
+            self.circuit_breaker = None
+            self.http_fallback_url = None
+    
+    def get_resilient_metrics(self):
+        """获取熔断降级指标"""
+        if not self.resilient_enabled or not self.circuit_breaker:
+            return None
+            
+        return {
+            "enabled": self.resilient_enabled,
+            "circuit_state": self.circuit_breaker.get_state(),
+            "failure_count": self.circuit_breaker.failure_count,
+            "last_failure_time": self.circuit_breaker.last_failure_time,
+            "http_fallback_url": self.http_fallback_url
+        }