tamar-model-client 0.1.26__py3-none-any.whl → 0.1.28__py3-none-any.whl

tamar_model_client/async_client.py

@@ -32,8 +32,11 @@ from grpc import RpcError
 from .core import (
     generate_request_id,
     set_request_id,
+    set_origin_request_id,
     get_protected_logger,
-    MAX_MESSAGE_LENGTH, get_request_id
+    MAX_MESSAGE_LENGTH, 
+    get_request_id,
+    RequestIdManager
 )
 from .core.base_client import BaseClient
 from .core.request_builder import RequestBuilder
@@ -102,12 +105,18 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
         self._last_channel_error_time = None
         self._channel_lock = asyncio.Lock()  # 异步锁
         
+        # === Request ID 管理 ===
+        self._request_id_manager = RequestIdManager()
+        
         # === 增强的重试处理器 ===
         self.retry_handler = EnhancedRetryHandler(
             max_retries=self.max_retries,
             base_delay=self.retry_delay
         )
         
+        # 设置client引用，用于快速降级
+        self.retry_handler.error_handler.client = self
+        
         # 注册退出时的清理函数
         atexit.register(self._cleanup_atexit)
 
@@ -734,7 +743,12 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
         if self.resilient_enabled and self.circuit_breaker and self.circuit_breaker.is_open:
             if self.http_fallback_url:
                 logger.warning("🔻 Circuit breaker is OPEN, using HTTP fallback")
-                return await self._invoke_http_fallback(model_request, timeout, request_id)
+                # 在这里还没有计算origin_request_id，所以先计算
+                temp_origin_request_id = None
+                temp_request_id = request_id
+                if request_id:
+                    temp_request_id, temp_origin_request_id = self._request_id_manager.get_composite_id(request_id)
+                return await self._invoke_http_fallback(model_request, timeout, temp_request_id, temp_origin_request_id)
                 
         await self._ensure_initialized()
 
@@ -744,10 +758,24 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
                 "user_id": model_request.user_context.user_id or ""
             }
 
-        if not request_id:
+        # 处理 request_id
+        origin_request_id = None
+        if request_id:
+            # 用户提供了 request_id，生成组合 ID
+            request_id, origin_request_id = self._request_id_manager.get_composite_id(request_id)
+        else:
+            # 没有提供，生成新的
             request_id = generate_request_id()
+            
         set_request_id(request_id)
-        metadata = self._build_auth_metadata(request_id)
+        if origin_request_id:
+            set_origin_request_id(origin_request_id)
+        metadata = self._build_auth_metadata(request_id, origin_request_id)
+
+        # 构建日志数据
+        log_data = ResponseHandler.build_log_data(model_request)
+        if origin_request_id:
+            log_data['origin_request_id'] = origin_request_id
 
         # 记录开始日志
         start_time = time.time()
@@ -756,7 +784,7 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
             extra={
                 "log_type": "request",
                 "uri": f"/invoke/{model_request.provider.value}/{model_request.invoke_type.value}",
-                "data": ResponseHandler.build_log_data(model_request)
+                "data": log_data
             })
 
         try:
@@ -789,18 +817,34 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
                 # 对于流式响应，直接返回带日志记录的包装器
                 return self._stream_with_logging(request, metadata, invoke_timeout, start_time, model_request)
             else:
-                result = await self._retry_request(self._invoke_request, request, metadata, invoke_timeout, request_id=request_id)
+                # 存储model_request和origin_request_id供重试方法使用
+                self._current_model_request = model_request
+                self._current_origin_request_id = origin_request_id
+                try:
+                    result = await self._retry_request(self._invoke_request, request, metadata, invoke_timeout, request_id=request_id)
+                finally:
+                    # 清理临时存储
+                    if hasattr(self, '_current_model_request'):
+                        delattr(self, '_current_model_request')
+                    if hasattr(self, '_current_origin_request_id'):
+                        delattr(self, '_current_origin_request_id')
 
                 # 记录非流式响应的成功日志
                 duration = time.time() - start_time
                 content_length = len(result.content) if result.content else 0
+                
+                # 构建响应日志数据
+                response_log_data = ResponseHandler.build_log_data(model_request, result)
+                if origin_request_id:
+                    response_log_data['origin_request_id'] = origin_request_id
+                    
                 logger.info(
                     f"✅ Request completed | content_length: {content_length}",
                     extra={
                         "log_type": "response",
                         "uri": f"/invoke/{model_request.provider.value}/{model_request.invoke_type.value}",
                         "duration": duration,
-                        "data": ResponseHandler.build_log_data(model_request, result)
+                        "data": response_log_data
                     }
                 )
                 
@@ -813,31 +857,29 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
         except (ConnectionError, grpc.RpcError) as e:
             duration = time.time() - start_time
             error_message = f"❌ Invoke gRPC failed: {str(e)}"
+            
+            # 构建错误日志数据
+            error_log_data = ResponseHandler.build_log_data(model_request, error=e)
+            if origin_request_id:
+                error_log_data['origin_request_id'] = origin_request_id
+                
             logger.error(error_message, exc_info=True,
                          extra={
                              "log_type": "response",
                              "uri": f"/invoke/{model_request.provider.value}/{model_request.invoke_type.value}",
                              "duration": duration,
-                             "data": ResponseHandler.build_log_data(
-                                 model_request,
-                                 error=e
-                             )
+                             "data": error_log_data
                          })
             
             # 记录 channel 错误
             if isinstance(e, grpc.RpcError):
                 self._record_channel_error(e)
             
-            # 记录失败并尝试降级（如果启用了熔断）
+            # 记录失败（如果启用了熔断）
             if self.resilient_enabled and self.circuit_breaker:
                 # 将错误码传递给熔断器，用于智能失败统计
                 error_code = e.code() if hasattr(e, 'code') else None
                 self.circuit_breaker.record_failure(error_code)
-                
-                # 如果可以降级，则降级
-                if self.http_fallback_url and self.circuit_breaker.should_fallback():
-                    logger.warning(f"🔻 gRPC failed, falling back to HTTP: {str(e)}")
-                    return await self._invoke_http_fallback(model_request, timeout, request_id)
             
             raise e
         except Exception as e:
@@ -867,6 +909,17 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
         Returns:
             BatchModelResponse: 批量请求的结果
         """
+        # 如果启用了熔断且熔断器打开，直接走 HTTP
+        if self.resilient_enabled and self.circuit_breaker and self.circuit_breaker.is_open:
+            if self.http_fallback_url:
+                logger.warning("🔻 Circuit breaker is OPEN, using HTTP fallback for batch request")
+                # 在这里还没有计算origin_request_id，所以先计算
+                temp_origin_request_id = None
+                temp_request_id = request_id
+                if request_id:
+                    temp_request_id, temp_origin_request_id = self._request_id_manager.get_composite_id(request_id)
+                return await self._invoke_batch_http_fallback(batch_request_model, timeout, temp_request_id, temp_origin_request_id)
+                
         await self._ensure_initialized()
 
         if not self.default_payload:
@@ -875,10 +928,29 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
                 "user_id": batch_request_model.user_context.user_id or ""
             }
 
-        if not request_id:
+        # 处理 request_id
+        origin_request_id = None
+        if request_id:
+            # 用户提供了 request_id，生成组合 ID
+            request_id, origin_request_id = self._request_id_manager.get_composite_id(request_id)
+        else:
+            # 没有提供，生成新的
             request_id = generate_request_id()
+            
         set_request_id(request_id)
-        metadata = self._build_auth_metadata(request_id)
+        if origin_request_id:
+            set_origin_request_id(origin_request_id)
+        metadata = self._build_auth_metadata(request_id, origin_request_id)
+
+        # 构建日志数据
+        batch_log_data = {
+            "batch_size": len(batch_request_model.items),
+            "org_id": batch_request_model.user_context.org_id,
+            "user_id": batch_request_model.user_context.user_id,
+            "client_type": batch_request_model.user_context.client_type
+        }
+        if origin_request_id:
+            batch_log_data['origin_request_id'] = origin_request_id
 
         # 记录开始日志
         start_time = time.time()
@@ -887,12 +959,7 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
             extra={
                 "log_type": "request",
                 "uri": "/batch_invoke",
-                "data": {
-                    "batch_size": len(batch_request_model.items),
-                    "org_id": batch_request_model.user_context.org_id,
-                    "user_id": batch_request_model.user_context.user_id,
-                    "client_type": batch_request_model.user_context.client_type
-                }
+                "data": batch_log_data
             })
 
         try:
@@ -919,6 +986,11 @@ class AsyncTamarModelClient(BaseClient, AsyncHttpFallbackMixin):
 
         try:
             invoke_timeout = timeout or self.default_invoke_timeout
+            
+            # 保存批量请求信息用于降级
+            self._current_batch_request = batch_request_model
+            self._current_origin_request_id = origin_request_id
+            
             batch_response = await self._retry_request(
                 self.stub.BatchInvoke,
                 batch_request,

tamar_model_client/circuit_breaker.py

@@ -101,9 +101,12 @@ class CircuitBreaker:
                     logger.warning(
                         f"🔻 Circuit breaker OPENED after {self.failure_count} failures",
                         extra={
-                            "failure_count": self.failure_count,
-                            "threshold": self.failure_threshold,
-                            "trigger_error": error_code.name if error_code else "unknown"
+                            "log_type": "info",
+                            "data": {
+                                "failure_count": self.failure_count,
+                                "threshold": self.failure_threshold,
+                                "trigger_error": error_code.name if error_code else "unknown"
+                            }
                         }
                     )
     

tamar_model_client/core/__init__.py

@@ -10,7 +10,9 @@ from .utils import (
     remove_none_from_dict,
     generate_request_id,
     set_request_id,
-    get_request_id
+    get_request_id,
+    set_origin_request_id,
+    get_origin_request_id
 )
 
 from .logging_setup import (
@@ -22,6 +24,8 @@ from .logging_setup import (
     MAX_MESSAGE_LENGTH
 )
 
+from .request_id_manager import RequestIdManager
+
 __all__ = [
     # Utils
     'is_effective_value',
@@ -30,6 +34,8 @@ __all__ = [
     'generate_request_id',
     'set_request_id',
     'get_request_id',
+    'set_origin_request_id',
+    'get_origin_request_id',
     # Logging
     'setup_logger',
     'RequestIdFilter',
@@ -37,4 +43,6 @@ __all__ = [
     'get_protected_logger',
     'reset_logger_config',
     'MAX_MESSAGE_LENGTH',
+    # Request ID Management
+    'RequestIdManager',
 ]

tamar_model_client/core/base_client.py

@@ -6,8 +6,7 @@ and configuration management for both sync and async clients.
 """
 
 import os
-import logging
-from typing import Optional, Dict, Any
+from typing import Optional
 from abc import ABC, abstractmethod
 
 from ..auth import JWTAuthHandler
@@ -25,7 +24,7 @@ class BaseClient(ABC):
     - 连接选项构建
     - 错误处理器初始化
     """
-    
+
     def __init__(
             self,
             server_address: Optional[str] = None,
@@ -57,40 +56,43 @@ class BaseClient(ABC):
         self.server_address = server_address or os.getenv("MODEL_MANAGER_SERVER_ADDRESS")
         if not self.server_address:
             raise ValueError("Server address must be provided via argument or environment variable.")
-        
+
         # 默认调用超时时间
         self.default_invoke_timeout = float(os.getenv("MODEL_MANAGER_SERVER_INVOKE_TIMEOUT", 30.0))
-        
+
         # === JWT 认证配置 ===
         self.jwt_secret_key = jwt_secret_key or os.getenv("MODEL_MANAGER_SERVER_JWT_SECRET_KEY")
         self.jwt_handler = JWTAuthHandler(self.jwt_secret_key) if self.jwt_secret_key else None
         self.jwt_token = jwt_token  # 用户传入的预生成 Token（可选）
         self.default_payload = default_payload
         self.token_expires_in = token_expires_in
-        
+
         # === TLS/Authority 配置 ===
         self.use_tls = os.getenv("MODEL_MANAGER_SERVER_GRPC_USE_TLS", "true").lower() == "true"
         self.default_authority = os.getenv("MODEL_MANAGER_SERVER_GRPC_DEFAULT_AUTHORITY")
-        
+
         # === 重试配置 ===
         self.max_retries = max_retries if max_retries is not None else int(
             os.getenv("MODEL_MANAGER_SERVER_GRPC_MAX_RETRIES", 6))
         self.retry_delay = retry_delay if retry_delay is not None else float(
             os.getenv("MODEL_MANAGER_SERVER_GRPC_RETRY_DELAY", 1.0))
-        
+
         # === 日志配置 ===
         self.logger = get_protected_logger(logger_name or __name__)
-        
+
         # === 错误处理器 ===
         self.error_handler = GrpcErrorHandler(self.logger)
         self.recovery_strategy = ErrorRecoveryStrategy(self)
-        
+
         # === 连接状态 ===
         self._closed = False
-        
+
         # === 熔断降级配置 ===
         self._init_resilient_features()
-    
+
+        # === 快速降级配置 ===
+        self._init_fast_fallback_config()
+
     def build_channel_options(self) -> list:
         """
         构建 gRPC 通道选项
@@ -108,30 +110,44 @@ class BaseClient(ABC):
             # 消息大小限制
             ('grpc.max_send_message_length', MAX_MESSAGE_LENGTH),
             ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH),
-            
+
             # Keepalive 核心配置
             ('grpc.keepalive_time_ms', 30000),  # 30秒发送一次 keepalive ping
             ('grpc.keepalive_timeout_ms', 10000),  # ping 响应超时时间 10秒
             ('grpc.keepalive_permit_without_calls', True),  # 空闲时也发送 keepalive
             ('grpc.http2.max_pings_without_data', 2),  # 无数据时最大 ping 次数
-            
+
             # 连接管理增强配置
             ('grpc.http2.min_time_between_pings_ms', 10000),  # ping 最小间隔 10秒
             ('grpc.http2.max_connection_idle_ms', 300000),  # 最大空闲时间 5分钟
             ('grpc.http2.max_connection_age_ms', 3600000),  # 连接最大生存时间 1小时
             ('grpc.http2.max_connection_age_grace_ms', 5000),  # 优雅关闭时间 5秒
-            
+
             # 性能相关配置
             ('grpc.http2.bdp_probe', 1),  # 启用带宽延迟探测
             ('grpc.enable_retries', 1),  # 启用内置重试
+
+            # 启用连接池配置（如果 gRPC 客户端支持）
+            ('grpc.keepalive_time_ms', 30000),  # 保持活跃的连接时间（30秒）
+            ('grpc.keepalive_timeout_ms', 10000),  # ping 响应超时时间（10秒）
+            ('grpc.max_connection_idle_ms', 300000),  # 连接最大空闲时间（5分钟）
+
+            # 设置资源配额
+            ('grpc.resource_quota_size', 1048576000),  # 设置资源配额为1GB
+
+            # 启用负载均衡配置
+            ('grpc.lb_policy', 'round_robin'),  # 设置负载均衡策略为 round_robin（轮询）
+
+            # 启用详细的日志记录
+            ('grpc.debug', 1),  # 启用 gRPC 的调试日志，记录更多的连接和请求信息
         ]
-        
+
         if self.default_authority:
             options.append(("grpc.default_authority", self.default_authority))
-            
+
         return options
-    
-    def _build_auth_metadata(self, request_id: str) -> list:
+
+    def _build_auth_metadata(self, request_id: str, origin_request_id: Optional[str] = None) -> list:
         """
         构建认证元数据
         
@@ -140,82 +156,166 @@ class BaseClient(ABC):
         
         Args:
             request_id: 当前请求的唯一标识符
+            origin_request_id: 原始请求ID（可选）
             
         Returns:
             list: gRPC元数据列表，包含请求ID和认证令牌
         """
         metadata = [("x-request-id", request_id)]  # 将 request_id 添加到 headers
-        
+
+        # 如果有原始请求ID，也添加到 headers
+        if origin_request_id:
+            metadata.append(("x-origin-request-id", origin_request_id))
+
         if self.jwt_handler:
             self.jwt_token = self.jwt_handler.encode_token(
-                self.default_payload, 
+                self.default_payload,
                 expires_in=self.token_expires_in
             )
             metadata.append(("authorization", f"Bearer {self.jwt_token}"))
-            
+
         return metadata
-    
+
     @abstractmethod
     def close(self):
         """关闭客户端连接（由子类实现）"""
         pass
-    
+
     @abstractmethod
     def __enter__(self):
         """进入上下文管理器（由子类实现）"""
         pass
-    
+
     @abstractmethod
     def __exit__(self, exc_type, exc_val, exc_tb):
         """退出上下文管理器（由子类实现）"""
         pass
-    
+
     def _init_resilient_features(self):
         """初始化熔断降级特性"""
         # 是否启用熔断降级
         self.resilient_enabled = os.getenv('MODEL_CLIENT_RESILIENT_ENABLED', 'false').lower() == 'true'
-        
+
         if self.resilient_enabled:
             # HTTP 降级地址
             self.http_fallback_url = os.getenv('MODEL_CLIENT_HTTP_FALLBACK_URL')
-            
+
             if not self.http_fallback_url:
                 self.logger.warning("🔶 Resilient mode enabled but MODEL_CLIENT_HTTP_FALLBACK_URL not set")
                 self.resilient_enabled = False
                 return
-            
+
             # 初始化熔断器
             from ..circuit_breaker import CircuitBreaker
             self.circuit_breaker = CircuitBreaker(
                 failure_threshold=int(os.getenv('MODEL_CLIENT_CIRCUIT_BREAKER_THRESHOLD', '5')),
                 recovery_timeout=int(os.getenv('MODEL_CLIENT_CIRCUIT_BREAKER_TIMEOUT', '60'))
             )
-            
+
             # HTTP 客户端（延迟初始化）
             self._http_client = None
             self._http_session = None  # 异步客户端使用
-            
+
             self.logger.info(
                 "🛡️ Resilient mode enabled",
                 extra={
-                    "http_fallback_url": self.http_fallback_url,
-                    "circuit_breaker_threshold": self.circuit_breaker.failure_threshold,
-                    "circuit_breaker_timeout": self.circuit_breaker.recovery_timeout
+                    "log_type": "info",
+                    "data": {
+                        "http_fallback_url": self.http_fallback_url,
+                        "circuit_breaker_threshold": self.circuit_breaker.failure_threshold,
+                        "circuit_breaker_timeout": self.circuit_breaker.recovery_timeout
+                    }
                 }
             )
         else:
             self.circuit_breaker = None
             self.http_fallback_url = None
-    
+            self._http_client = None
+            self._http_session = None
+
     def get_resilient_metrics(self):
         """获取熔断降级指标"""
         if not self.resilient_enabled or not self.circuit_breaker:
             return None
-            
+
         return {
             "enabled": self.resilient_enabled,
             "circuit_state": self.circuit_breaker.get_state(),
             "failure_count": self.circuit_breaker.failure_count,
             "last_failure_time": self.circuit_breaker.last_failure_time,
             "http_fallback_url": self.http_fallback_url
-        }
+        }
+
+    def _init_fast_fallback_config(self):
+        """初始化快速降级配置"""
+        import grpc
+
+        # 是否启用快速降级
+        self.fast_fallback_enabled = os.getenv('MODEL_CLIENT_FAST_FALLBACK_ENABLED', 'false').lower() == 'true'
+
+        # 降级前的最大gRPC重试次数
+        self.fallback_after_retries = int(os.getenv('MODEL_CLIENT_FALLBACK_AFTER_RETRIES', '1'))
+
+        # 立即降级的错误码配置
+        immediate_fallback_errors = os.getenv('MODEL_CLIENT_IMMEDIATE_FALLBACK_ERRORS',
+                                              'UNAVAILABLE,DEADLINE_EXCEEDED,CANCELLED')
+        self.immediate_fallback_errors = set()
+
+        if immediate_fallback_errors:
+            for error_name in immediate_fallback_errors.split(','):
+                error_name = error_name.strip()
+                if hasattr(grpc.StatusCode, error_name):
+                    self.immediate_fallback_errors.add(getattr(grpc.StatusCode, error_name))
+
+        # 永不降级的错误码
+        never_fallback_errors = os.getenv('MODEL_CLIENT_NEVER_FALLBACK_ERRORS',
+                                          'UNAUTHENTICATED,PERMISSION_DENIED,INVALID_ARGUMENT')
+        self.never_fallback_errors = set()
+
+        if never_fallback_errors:
+            for error_name in never_fallback_errors.split(','):
+                error_name = error_name.strip()
+                if hasattr(grpc.StatusCode, error_name):
+                    self.never_fallback_errors.add(getattr(grpc.StatusCode, error_name))
+
+        if self.fast_fallback_enabled:
+            self.logger.info(
+                "🚀 Fast fallback enabled",
+                extra={
+                    "data": {
+                        "fallback_after_retries": self.fallback_after_retries,
+                        "immediate_fallback_errors": [e.name for e in self.immediate_fallback_errors],
+                        "never_fallback_errors": [e.name for e in self.never_fallback_errors]
+                    }
+                }
+            )
+
+    def _should_try_fallback(self, error_code, attempt: int) -> bool:
+        """
+        判断是否应该尝试降级
+        
+        Args:
+            error_code: gRPC错误码
+            attempt: 当前重试次数
+            
+        Returns:
+            bool: 是否应该尝试降级
+        """
+        # 未启用快速降级
+        if not self.fast_fallback_enabled:
+            return False
+
+        # 未启用熔断降级功能
+        if not self.resilient_enabled or not self.http_fallback_url:
+            return False
+
+        # 永不降级的错误类型
+        if error_code in self.never_fallback_errors:
+            return False
+
+        # 立即降级的错误类型
+        if error_code in self.immediate_fallback_errors:
+            return True
+
+        # 其他错误在达到重试次数后降级
+        return attempt >= self.fallback_after_retries