tamar-model-client 0.1.27__py3-none-any.whl → 0.1.28__py3-none-any.whl

tamar_model_client/async_client.py

@@ -32,6 +32,7 @@ from grpc import RpcError
 from .core import (
     generate_request_id,
     set_request_id,
+    set_origin_request_id,
     get_protected_logger,
     MAX_MESSAGE_LENGTH, 
     get_request_id,
@@ -113,6 +114,9 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
             base_delay=self.retry_delay
         )
         
+        # 设置client引用，用于快速降级
+        self.retry_handler.error_handler.client = self
+        
         # 注册退出时的清理函数
         atexit.register(self._cleanup_atexit)
 
@@ -739,7 +743,12 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
         if self.resilient_enabled and self.circuit_breaker and self.circuit_breaker.is_open:
             if self.http_fallback_url:
                 logger.warning("🔻 Circuit breaker is OPEN, using HTTP fallback")
-                return await self._invoke_http_fallback(model_request, timeout, request_id)
+                # 在这里还没有计算origin_request_id，所以先计算
+                temp_origin_request_id = None
+                temp_request_id = request_id
+                if request_id:
+                    temp_request_id, temp_origin_request_id = self._request_id_manager.get_composite_id(request_id)
+                return await self._invoke_http_fallback(model_request, timeout, temp_request_id, temp_origin_request_id)
                 
         await self._ensure_initialized()
 
@@ -759,6 +768,8 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
             request_id = generate_request_id()
             
         set_request_id(request_id)
+        if origin_request_id:
+            set_origin_request_id(origin_request_id)
         metadata = self._build_auth_metadata(request_id, origin_request_id)
 
         # 构建日志数据
@@ -806,7 +817,17 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
                 # 对于流式响应，直接返回带日志记录的包装器
                 return self._stream_with_logging(request, metadata, invoke_timeout, start_time, model_request)
             else:
-                result = await self._retry_request(self._invoke_request, request, metadata, invoke_timeout, request_id=request_id)
+                # 存储model_request和origin_request_id供重试方法使用
+                self._current_model_request = model_request
+                self._current_origin_request_id = origin_request_id
+                try:
+                    result = await self._retry_request(self._invoke_request, request, metadata, invoke_timeout, request_id=request_id)
+                finally:
+                    # 清理临时存储
+                    if hasattr(self, '_current_model_request'):
+                        delattr(self, '_current_model_request')
+                    if hasattr(self, '_current_origin_request_id'):
+                        delattr(self, '_current_origin_request_id')
 
                 # 记录非流式响应的成功日志
                 duration = time.time() - start_time
@@ -854,16 +875,11 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
             if isinstance(e, grpc.RpcError):
                 self._record_channel_error(e)
             
-            # 记录失败并尝试降级（如果启用了熔断）
+            # 记录失败（如果启用了熔断）
             if self.resilient_enabled and self.circuit_breaker:
                 # 将错误码传递给熔断器，用于智能失败统计
                 error_code = e.code() if hasattr(e, 'code') else None
                 self.circuit_breaker.record_failure(error_code)
-                
-                # 如果可以降级，则降级
-                if self.http_fallback_url and self.circuit_breaker.should_fallback():
-                    logger.warning(f"🔻 gRPC failed, falling back to HTTP: {str(e)}")
-                    return await self._invoke_http_fallback(model_request, timeout, request_id)
             
             raise e
         except Exception as e:
@@ -893,6 +909,17 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
         Returns:
             BatchModelResponse: 批量请求的结果
         """
+        # 如果启用了熔断且熔断器打开，直接走 HTTP
+        if self.resilient_enabled and self.circuit_breaker and self.circuit_breaker.is_open:
+            if self.http_fallback_url:
+                logger.warning("🔻 Circuit breaker is OPEN, using HTTP fallback for batch request")
+                # 在这里还没有计算origin_request_id，所以先计算
+                temp_origin_request_id = None
+                temp_request_id = request_id
+                if request_id:
+                    temp_request_id, temp_origin_request_id = self._request_id_manager.get_composite_id(request_id)
+                return await self._invoke_batch_http_fallback(batch_request_model, timeout, temp_request_id, temp_origin_request_id)
+                
         await self._ensure_initialized()
 
         if not self.default_payload:
@@ -911,6 +938,8 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
             request_id = generate_request_id()
             
         set_request_id(request_id)
+        if origin_request_id:
+            set_origin_request_id(origin_request_id)
         metadata = self._build_auth_metadata(request_id, origin_request_id)
 
         # 构建日志数据
@@ -957,6 +986,11 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
 
         try:
             invoke_timeout = timeout or self.default_invoke_timeout
+            
+            # 保存批量请求信息用于降级
+            self._current_batch_request = batch_request_model
+            self._current_origin_request_id = origin_request_id
+            
             batch_response = await self._retry_request(
                 self.stub.BatchInvoke,
                 batch_request,

tamar_model_client/circuit_breaker.py

@@ -101,9 +101,12 @@ class CircuitBreaker:
                     logger.warning(
                         f"🔻 Circuit breaker OPENED after {self.failure_count} failures",
                         extra={
-                            "failure_count": self.failure_count,
-                            "threshold": self.failure_threshold,
-                            "trigger_error": error_code.name if error_code else "unknown"
+                            "log_type": "info",
+                            "data": {
+                                "failure_count": self.failure_count,
+                                "threshold": self.failure_threshold,
+                                "trigger_error": error_code.name if error_code else "unknown"
+                            }
                         }
                     )
     

tamar_model_client/core/__init__.py

@@ -10,7 +10,9 @@ from .utils import (
     remove_none_from_dict,
     generate_request_id,
     set_request_id,
-    get_request_id
+    get_request_id,
+    set_origin_request_id,
+    get_origin_request_id
 )
 
 from .logging_setup import (
@@ -32,6 +34,8 @@ __all__ = [
     'generate_request_id',
     'set_request_id',
     'get_request_id',
+    'set_origin_request_id',
+    'get_origin_request_id',
     # Logging
     'setup_logger',
     'RequestIdFilter',

tamar_model_client/core/base_client.py

@@ -6,8 +6,7 @@ and configuration management for both sync and async clients.
 """
 
 import os
-import logging
-from typing import Optional, Dict, Any
+from typing import Optional
 from abc import ABC, abstractmethod
 
 from ..auth import JWTAuthHandler
@@ -25,7 +24,7 @@ class BaseClient(ABC):
     - 连接选项构建
     - 错误处理器初始化
     """
-    
+
     def __init__(
             self,
             server_address: Optional[str] = None,
@@ -57,40 +56,43 @@ class BaseClient(ABC):
         self.server_address = server_address or os.getenv("MODEL_MANAGER_SERVER_ADDRESS")
         if not self.server_address:
             raise ValueError("Server address must be provided via argument or environment variable.")
-        
+
         # 默认调用超时时间
         self.default_invoke_timeout = float(os.getenv("MODEL_MANAGER_SERVER_INVOKE_TIMEOUT", 30.0))
-        
+
         # === JWT 认证配置 ===
         self.jwt_secret_key = jwt_secret_key or os.getenv("MODEL_MANAGER_SERVER_JWT_SECRET_KEY")
         self.jwt_handler = JWTAuthHandler(self.jwt_secret_key) if self.jwt_secret_key else None
         self.jwt_token = jwt_token  # 用户传入的预生成 Token（可选）
         self.default_payload = default_payload
         self.token_expires_in = token_expires_in
-        
+
         # === TLS/Authority 配置 ===
         self.use_tls = os.getenv("MODEL_MANAGER_SERVER_GRPC_USE_TLS", "true").lower() == "true"
         self.default_authority = os.getenv("MODEL_MANAGER_SERVER_GRPC_DEFAULT_AUTHORITY")
-        
+
         # === 重试配置 ===
         self.max_retries = max_retries if max_retries is not None else int(
             os.getenv("MODEL_MANAGER_SERVER_GRPC_MAX_RETRIES", 6))
         self.retry_delay = retry_delay if retry_delay is not None else float(
             os.getenv("MODEL_MANAGER_SERVER_GRPC_RETRY_DELAY", 1.0))
-        
+
         # === 日志配置 ===
         self.logger = get_protected_logger(logger_name or __name__)
-        
+
         # === 错误处理器 ===
         self.error_handler = GrpcErrorHandler(self.logger)
         self.recovery_strategy = ErrorRecoveryStrategy(self)
-        
+
         # === 连接状态 ===
         self._closed = False
-        
+
         # === 熔断降级配置 ===
         self._init_resilient_features()
-    
+
+        # === 快速降级配置 ===
+        self._init_fast_fallback_config()
+
     def build_channel_options(self) -> list:
         """
         构建 gRPC 通道选项
@@ -108,29 +110,43 @@ class BaseClient(ABC):
             # 消息大小限制
             ('grpc.max_send_message_length', MAX_MESSAGE_LENGTH),
             ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH),
-            
+
             # Keepalive 核心配置
             ('grpc.keepalive_time_ms', 30000),  # 30秒发送一次 keepalive ping
             ('grpc.keepalive_timeout_ms', 10000),  # ping 响应超时时间 10秒
             ('grpc.keepalive_permit_without_calls', True),  # 空闲时也发送 keepalive
             ('grpc.http2.max_pings_without_data', 2),  # 无数据时最大 ping 次数
-            
+
             # 连接管理增强配置
             ('grpc.http2.min_time_between_pings_ms', 10000),  # ping 最小间隔 10秒
             ('grpc.http2.max_connection_idle_ms', 300000),  # 最大空闲时间 5分钟
             ('grpc.http2.max_connection_age_ms', 3600000),  # 连接最大生存时间 1小时
             ('grpc.http2.max_connection_age_grace_ms', 5000),  # 优雅关闭时间 5秒
-            
+
             # 性能相关配置
             ('grpc.http2.bdp_probe', 1),  # 启用带宽延迟探测
             ('grpc.enable_retries', 1),  # 启用内置重试
+
+            # 启用连接池配置（如果 gRPC 客户端支持）
+            ('grpc.keepalive_time_ms', 30000),  # 保持活跃的连接时间（30秒）
+            ('grpc.keepalive_timeout_ms', 10000),  # ping 响应超时时间（10秒）
+            ('grpc.max_connection_idle_ms', 300000),  # 连接最大空闲时间（5分钟）
+
+            # 设置资源配额
+            ('grpc.resource_quota_size', 1048576000),  # 设置资源配额为1GB
+
+            # 启用负载均衡配置
+            ('grpc.lb_policy', 'round_robin'),  # 设置负载均衡策略为 round_robin（轮询）
+
+            # 启用详细的日志记录
+            ('grpc.debug', 1),  # 启用 gRPC 的调试日志，记录更多的连接和请求信息
         ]
-        
+
         if self.default_authority:
             options.append(("grpc.default_authority", self.default_authority))
-            
+
         return options
-    
+
     def _build_auth_metadata(self, request_id: str, origin_request_id: Optional[str] = None) -> list:
         """
         构建认证元数据
@@ -146,81 +162,160 @@ class BaseClient(ABC):
             list: gRPC元数据列表，包含请求ID和认证令牌
         """
         metadata = [("x-request-id", request_id)]  # 将 request_id 添加到 headers
-        
+
         # 如果有原始请求ID，也添加到 headers
         if origin_request_id:
             metadata.append(("x-origin-request-id", origin_request_id))
-        
+
         if self.jwt_handler:
             self.jwt_token = self.jwt_handler.encode_token(
-                self.default_payload, 
+                self.default_payload,
                 expires_in=self.token_expires_in
             )
             metadata.append(("authorization", f"Bearer {self.jwt_token}"))
-            
+
         return metadata
-    
+
     @abstractmethod
     def close(self):
         """关闭客户端连接（由子类实现）"""
         pass
-    
+
     @abstractmethod
     def __enter__(self):
         """进入上下文管理器（由子类实现）"""
         pass
-    
+
     @abstractmethod
     def __exit__(self, exc_type, exc_val, exc_tb):
         """退出上下文管理器（由子类实现）"""
         pass
-    
+
     def _init_resilient_features(self):
         """初始化熔断降级特性"""
         # 是否启用熔断降级
         self.resilient_enabled = os.getenv('MODEL_CLIENT_RESILIENT_ENABLED', 'false').lower() == 'true'
-        
+
         if self.resilient_enabled:
             # HTTP 降级地址
             self.http_fallback_url = os.getenv('MODEL_CLIENT_HTTP_FALLBACK_URL')
-            
+
             if not self.http_fallback_url:
                 self.logger.warning("🔶 Resilient mode enabled but MODEL_CLIENT_HTTP_FALLBACK_URL not set")
                 self.resilient_enabled = False
                 return
-            
+
             # 初始化熔断器
             from ..circuit_breaker import CircuitBreaker
             self.circuit_breaker = CircuitBreaker(
                 failure_threshold=int(os.getenv('MODEL_CLIENT_CIRCUIT_BREAKER_THRESHOLD', '5')),
                 recovery_timeout=int(os.getenv('MODEL_CLIENT_CIRCUIT_BREAKER_TIMEOUT', '60'))
             )
-            
+
             # HTTP 客户端（延迟初始化）
             self._http_client = None
             self._http_session = None  # 异步客户端使用
-            
+
             self.logger.info(
                 "🛡️ Resilient mode enabled",
                 extra={
-                    "http_fallback_url": self.http_fallback_url,
-                    "circuit_breaker_threshold": self.circuit_breaker.failure_threshold,
-                    "circuit_breaker_timeout": self.circuit_breaker.recovery_timeout
+                    "log_type": "info",
+                    "data": {
+                        "http_fallback_url": self.http_fallback_url,
+                        "circuit_breaker_threshold": self.circuit_breaker.failure_threshold,
+                        "circuit_breaker_timeout": self.circuit_breaker.recovery_timeout
+                    }
                 }
             )
         else:
             self.circuit_breaker = None
             self.http_fallback_url = None
-    
+            self._http_client = None
+            self._http_session = None
+
     def get_resilient_metrics(self):
         """获取熔断降级指标"""
         if not self.resilient_enabled or not self.circuit_breaker:
             return None
-            
+
         return {
             "enabled": self.resilient_enabled,
             "circuit_state": self.circuit_breaker.get_state(),
             "failure_count": self.circuit_breaker.failure_count,
             "last_failure_time": self.circuit_breaker.last_failure_time,
             "http_fallback_url": self.http_fallback_url
-        }
+        }
+
+    def _init_fast_fallback_config(self):
+        """初始化快速降级配置"""
+        import grpc
+
+        # 是否启用快速降级
+        self.fast_fallback_enabled = os.getenv('MODEL_CLIENT_FAST_FALLBACK_ENABLED', 'false').lower() == 'true'
+
+        # 降级前的最大gRPC重试次数
+        self.fallback_after_retries = int(os.getenv('MODEL_CLIENT_FALLBACK_AFTER_RETRIES', '1'))
+
+        # 立即降级的错误码配置
+        immediate_fallback_errors = os.getenv('MODEL_CLIENT_IMMEDIATE_FALLBACK_ERRORS',
+                                              'UNAVAILABLE,DEADLINE_EXCEEDED,CANCELLED')
+        self.immediate_fallback_errors = set()
+
+        if immediate_fallback_errors:
+            for error_name in immediate_fallback_errors.split(','):
+                error_name = error_name.strip()
+                if hasattr(grpc.StatusCode, error_name):
+                    self.immediate_fallback_errors.add(getattr(grpc.StatusCode, error_name))
+
+        # 永不降级的错误码
+        never_fallback_errors = os.getenv('MODEL_CLIENT_NEVER_FALLBACK_ERRORS',
+                                          'UNAUTHENTICATED,PERMISSION_DENIED,INVALID_ARGUMENT')
+        self.never_fallback_errors = set()
+
+        if never_fallback_errors:
+            for error_name in never_fallback_errors.split(','):
+                error_name = error_name.strip()
+                if hasattr(grpc.StatusCode, error_name):
+                    self.never_fallback_errors.add(getattr(grpc.StatusCode, error_name))
+
+        if self.fast_fallback_enabled:
+            self.logger.info(
+                "🚀 Fast fallback enabled",
+                extra={
+                    "data": {
+                        "fallback_after_retries": self.fallback_after_retries,
+                        "immediate_fallback_errors": [e.name for e in self.immediate_fallback_errors],
+                        "never_fallback_errors": [e.name for e in self.never_fallback_errors]
+                    }
+                }
+            )
+
+    def _should_try_fallback(self, error_code, attempt: int) -> bool:
+        """
+        判断是否应该尝试降级
+        
+        Args:
+            error_code: gRPC错误码
+            attempt: 当前重试次数
+            
+        Returns:
+            bool: 是否应该尝试降级
+        """
+        # 未启用快速降级
+        if not self.fast_fallback_enabled:
+            return False
+
+        # 未启用熔断降级功能
+        if not self.resilient_enabled or not self.http_fallback_url:
+            return False
+
+        # 永不降级的错误类型
+        if error_code in self.never_fallback_errors:
+            return False
+
+        # 立即降级的错误类型
+        if error_code in self.immediate_fallback_errors:
+            return True
+
+        # 其他错误在达到重试次数后降级
+        return attempt >= self.fallback_after_retries