tamar-model-client 0.1.20__py3-none-any.whl → 0.1.22__py3-none-any.whl

tamar_model_client/sync_client.py

@@ -30,22 +30,23 @@ import grpc
 from .core import (
     generate_request_id,
     set_request_id,
-    setup_logger,
+    get_protected_logger,
     MAX_MESSAGE_LENGTH
 )
 from .core.base_client import BaseClient
 from .core.request_builder import RequestBuilder
 from .core.response_handler import ResponseHandler
-from .exceptions import ConnectionError, TamarModelException, is_retryable_error
+from .exceptions import ConnectionError, TamarModelException
 from .generated import model_service_pb2, model_service_pb2_grpc
 from .schemas import BatchModelResponse, ModelResponse
 from .schemas.inputs import BatchModelRequest, ModelRequest
+from .core.http_fallback import HttpFallbackMixin
 
-# 配置日志记录器
-logger = setup_logger(__name__)
+# 配置日志记录器（使用受保护的logger）
+logger = get_protected_logger(__name__)
 
 
-class TamarModelClient(BaseClient):
+class TamarModelClient(BaseClient, HttpFallbackMixin):
     """
     Tamar Model Client 同步客户端
     
@@ -200,10 +201,17 @@ class TamarModelClient(BaseClient):
         """
         使用增强的错误处理器进行重试（同步版本）
         """
+        # 记录方法开始时间
+        method_start_time = time.time()
+        
+        # 从kwargs中提取request_id（如果有的话），然后移除它
+        request_id = kwargs.pop('request_id', None) or get_request_id()
+        
         # 构建请求上下文
         context = {
             'method': func.__name__ if hasattr(func, '__name__') else 'unknown',
             'client_version': 'sync',
+            'request_id': request_id,
         }
 
         last_exception = None
@@ -218,22 +226,40 @@ class TamarModelClient(BaseClient):
                 context['retry_count'] = attempt
 
                 # 判断是否可以重试
-                if not is_retryable_error(e.code()) or attempt >= self.max_retries:
+                should_retry = self._should_retry(e, attempt)
+                if not should_retry or attempt >= self.max_retries:
                     # 不可重试或已达到最大重试次数
+                    current_duration = time.time() - method_start_time
+                    context['duration'] = current_duration
                     last_exception = self.error_handler.handle_error(e, context)
                     break
 
+                # 计算当前的耗时
+                current_duration = time.time() - method_start_time
+                
                 # 记录重试日志
+                log_data = {
+                    "log_type": "info",
+                    "request_id": context.get('request_id'),
+                    "data": {
+                        "error_code": e.code().name if e.code() else 'UNKNOWN',
+                        "retry_count": attempt,
+                        "max_retries": self.max_retries,
+                        "method": context.get('method', 'unknown')
+                    },
+                    "duration": current_duration
+                }
                 logger.warning(
                     f"Attempt {attempt + 1}/{self.max_retries + 1} failed: {e.code()}",
-                    extra=context
+                    extra=log_data
                 )
 
                 # 执行退避等待
                 if attempt < self.max_retries:
-                    delay = self._calculate_backoff(attempt)
+                    delay = self._calculate_backoff(attempt, e.code())
                     time.sleep(delay)
 
+                context['duration'] = current_duration
                 last_exception = self.error_handler.handle_error(e, context)
 
             except Exception as e:
@@ -248,14 +274,114 @@ class TamarModelClient(BaseClient):
         else:
             raise TamarModelException("Unknown error occurred")
 
-    def _calculate_backoff(self, attempt: int) -> float:
-        """计算退避时间"""
-        max_delay = 60.0
-        jitter_factor = 0.1
+    def _should_retry(self, error: grpc.RpcError, attempt: int) -> bool:
+        """
+        判断是否应该重试
+        
+        Args:
+            error: gRPC错误
+            attempt: 当前重试次数
+            
+        Returns:
+            bool: 是否应该重试
+        """
+        error_code = error.code()
+        from .exceptions import get_retry_policy, ErrorContext
+        policy = get_retry_policy(error_code)
+        
+        # 先检查错误级别的 max_attempts 配置
+        # max_attempts 表示最大重试次数（不包括初始请求）
+        error_max_attempts = policy.get('max_attempts', self.max_retries)
+        if attempt >= error_max_attempts:
+            return False
+            
+        # 再检查全局的 max_retries
+        if attempt >= self.max_retries:
+            return False
+        
+        retryable = policy.get('retryable', False)
+        
+        if retryable == False:
+            return False
+        elif retryable == True:
+            return True
+        elif retryable == 'conditional':
+            # 条件重试，特殊处理
+            if error_code == grpc.StatusCode.CANCELLED:
+                # 检查是否是网络中断导致的取消
+                context = {'method': 'unknown', 'client_version': 'sync'}
+                error_context = ErrorContext(error, context)
+                return error_context.is_network_cancelled()
+            else:
+                return self._check_error_details_for_retry(error)
+        
+        return False
+    
+    def _check_error_details_for_retry(self, error: grpc.RpcError) -> bool:
+        """
+        检查错误详情决定是否重试
+        
+        Args:
+            error: gRPC错误
+            
+        Returns:
+            bool: 是否应该重试
+        """
+        error_message = error.details().lower() if error.details() else ""
+        
+        # 可重试的错误模式
+        retryable_patterns = [
+            'temporary', 'timeout', 'unavailable', 
+            'connection', 'network', 'try again'
+        ]
+        
+        for pattern in retryable_patterns:
+            if pattern in error_message:
+                return True
+                
+        return False
 
-        delay = min(self.retry_delay * (2 ** attempt), max_delay)
-        jitter = random.uniform(0, delay * jitter_factor)
-        return delay + jitter
+    def _calculate_backoff(self, attempt: int, error_code: grpc.StatusCode = None) -> float:
+        """
+        计算退避时间，支持不同的退避策略
+        
+        Args:
+            attempt: 当前重试次数
+            error_code: gRPC错误码，用于确定退避策略
+        """
+        max_delay = 60.0
+        base_delay = self.retry_delay
+        
+        # 获取错误的重试策略
+        if error_code:
+            from .exceptions import get_retry_policy
+            policy = get_retry_policy(error_code)
+            backoff_type = policy.get('backoff', 'exponential')
+            use_jitter = policy.get('jitter', False)
+        else:
+            backoff_type = 'exponential'
+            use_jitter = False
+        
+        # 根据退避类型计算延迟
+        if backoff_type == 'linear':
+            # 线性退避：delay * (attempt + 1)
+            delay = min(base_delay * (attempt + 1), max_delay)
+        else:
+            # 指数退避：delay * 2^attempt
+            delay = min(base_delay * (2 ** attempt), max_delay)
+        
+        # 添加抖动
+        if use_jitter:
+            jitter_factor = 0.2  # 增加抖动范围，减少竞争
+            jitter = random.uniform(0, delay * jitter_factor)
+            delay += jitter
+        else:
+            # 默认的小量抖动，避免完全同步
+            jitter_factor = 0.05
+            jitter = random.uniform(0, delay * jitter_factor)
+            delay += jitter
+            
+        return delay
 
     def _retry_request_stream(self, func, *args, **kwargs):
         """
@@ -271,30 +397,91 @@ class TamarModelClient(BaseClient):
         Yields:
             流式响应的每个元素
         """
+        # 记录方法开始时间
+        method_start_time = time.time()
+        
+        # 从kwargs中提取request_id（如果有的话），然后移除它
+        request_id = kwargs.pop('request_id', None) or get_request_id()
+        
         last_exception = None
+        context = {
+            'method': 'stream',
+            'client_version': 'sync',
+            'request_id': request_id,
+        }
         
         for attempt in range(self.max_retries + 1):
             try:
+                context['retry_count'] = attempt
                 # 尝试创建流
                 for item in func(*args, **kwargs):
                     yield item
                 return
                 
             except grpc.RpcError as e:
-                last_exception = e
-                if attempt < self.max_retries:
-                    logger.warning(
-                        f"Stream attempt {attempt + 1}/{self.max_retries + 1} failed: {e.code()}",
-                        extra={"retry_count": attempt, "error_code": str(e.code())}
+                # 使用智能重试判断
+                context['retry_count'] = attempt
+                
+                # 计算当前的耗时
+                current_duration = time.time() - method_start_time
+                
+                # 判断是否应该重试
+                should_retry = self._should_retry(e, attempt)
+                if not should_retry or attempt >= self.max_retries:
+                    # 不重试或已达到最大重试次数
+                    log_data = {
+                        "log_type": "info",
+                        "request_id": context.get('request_id'),
+                        "data": {
+                            "error_code": e.code().name if e.code() else 'UNKNOWN',
+                            "retry_count": attempt,
+                            "max_retries": self.max_retries,
+                            "method": "stream",
+                            "will_retry": False
+                        },
+                        "duration": current_duration
+                    }
+                    logger.error(
+                        f"Stream failed: {e.code()} (no retry)",
+                        extra=log_data
                     )
-                    time.sleep(self.retry_delay * (attempt + 1))
-                else:
+                    context['duration'] = current_duration
+                    last_exception = self.error_handler.handle_error(e, context)
                     break
+                
+                # 记录重试日志
+                log_data = {
+                    "log_type": "info",
+                    "request_id": context.get('request_id'),
+                    "data": {
+                        "error_code": e.code().name if e.code() else 'UNKNOWN',
+                        "retry_count": attempt,
+                        "max_retries": self.max_retries,
+                        "method": "stream"
+                    },
+                    "duration": current_duration
+                }
+                logger.warning(
+                    f"Stream attempt {attempt + 1}/{self.max_retries + 1} failed: {e.code()} (will retry)",
+                    extra=log_data
+                )
+                
+                # 执行退避等待
+                if attempt < self.max_retries:
+                    delay = self._calculate_backoff(attempt, e.code())
+                    time.sleep(delay)
+                    
+                last_exception = e
+                
             except Exception as e:
+                context['retry_count'] = attempt
                 raise TamarModelException(str(e)) from e
         
         if last_exception:
-            raise self.error_handler.handle_error(last_exception, {"retry_count": self.max_retries})
+            if isinstance(last_exception, TamarModelException):
+                raise last_exception
+            else:
+                raise self.error_handler.handle_error(last_exception, context)
         else:
             raise TamarModelException("Unknown streaming error occurred")
 
@@ -457,6 +644,12 @@ class TamarModelClient(BaseClient):
             ValidationError: 输入验证失败。
             ConnectionError: 连接服务端失败。
         """
+        # 如果启用了熔断且熔断器打开，直接走 HTTP
+        if self.resilient_enabled and self.circuit_breaker and self.circuit_breaker.is_open:
+            if self.http_fallback_url:
+                logger.warning("🔻 Circuit breaker is OPEN, using HTTP fallback")
+                return self._invoke_http_fallback(model_request, timeout, request_id)
+                
         self._ensure_initialized()
 
         if not self.default_payload:
@@ -510,10 +703,11 @@ class TamarModelClient(BaseClient):
                 # 对于流式响应，使用重试包装器
                 return self._retry_request_stream(
                     self._stream_with_logging,
-                    request, metadata, invoke_timeout, start_time, model_request
+                    request, metadata, invoke_timeout, start_time, model_request,
+                    request_id=request_id
                 )
             else:
-                result = self._retry_request(self._invoke_request, request, metadata, invoke_timeout)
+                result = self._retry_request(self._invoke_request, request, metadata, invoke_timeout, request_id=request_id)
 
                 # 记录非流式响应的成功日志
                 duration = time.time() - start_time
@@ -527,9 +721,14 @@ class TamarModelClient(BaseClient):
                         "data": ResponseHandler.build_log_data(model_request, result)
                     }
                 )
+                
+                # 记录成功（如果启用了熔断）
+                if self.resilient_enabled and self.circuit_breaker:
+                    self.circuit_breaker.record_success()
+                    
                 return result
                 
-        except grpc.RpcError as e:
+        except (ConnectionError, grpc.RpcError) as e:
             duration = time.time() - start_time
             error_message = f"❌ Invoke gRPC failed: {str(e)}"
             logger.error(error_message, exc_info=True,
@@ -542,6 +741,18 @@ class TamarModelClient(BaseClient):
                                  error=e
                              )
                          })
+            
+            # 记录失败并尝试降级（如果启用了熔断）
+            if self.resilient_enabled and self.circuit_breaker:
+                # 将错误码传递给熔断器，用于智能失败统计
+                error_code = e.code() if hasattr(e, 'code') else None
+                self.circuit_breaker.record_failure(error_code)
+                
+                # 如果可以降级，则降级
+                if self.http_fallback_url and self.circuit_breaker.should_fallback():
+                    logger.warning(f"🔻 gRPC failed, falling back to HTTP: {str(e)}")
+                    return self._invoke_http_fallback(model_request, timeout, request_id)
+            
             raise e
         except Exception as e:
             duration = time.time() - start_time
@@ -626,7 +837,8 @@ class TamarModelClient(BaseClient):
                 self.stub.BatchInvoke,
                 batch_request,
                 metadata=metadata,
-                timeout=invoke_timeout
+                timeout=invoke_timeout,
+                request_id=request_id
             )
 
             # 构建响应对象

{tamar_model_client-0.1.20.dist-info → tamar_model_client-0.1.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tamar-model-client
-Version: 0.1.20
+Version: 0.1.22
 Summary: A Python SDK for interacting with the Model Manager gRPC service
 Home-page: http://gitlab.tamaredge.top/project-tap/AgentOS/model-manager-client
 Author: Oscar Ou
@@ -532,6 +532,61 @@ response = client.invoke(
    - 启用流式响应减少首字延迟
    - 合理设置 max_tokens 避免浪费
 
+### 🛡️ 熔断降级功能（高可用保障）
+
+SDK 内置了熔断降级机制，当 gRPC 服务不可用时自动切换到 HTTP 服务，确保业务连续性。
+
+#### 工作原理
+1. **正常状态**：所有请求通过高性能的 gRPC 协议
+2. **熔断触发**：当连续失败达到阈值时，熔断器打开
+3. **自动降级**：切换到 HTTP 协议继续提供服务
+4. **定期恢复**：熔断器会定期尝试恢复到 gRPC
+
+#### 启用方式
+```bash
+# 设置环境变量
+export MODEL_CLIENT_RESILIENT_ENABLED=true
+export MODEL_CLIENT_HTTP_FALLBACK_URL=http://localhost:8080
+export MODEL_CLIENT_CIRCUIT_BREAKER_THRESHOLD=5
+export MODEL_CLIENT_CIRCUIT_BREAKER_TIMEOUT=60
+```
+
+#### 使用示例
+```python
+from tamar_model_client import TamarModelClient
+
+# 客户端会自动处理熔断降级，对使用者透明
+client = TamarModelClient()
+
+# 正常使用，无需关心底层协议
+response = client.invoke(request)
+
+# 获取熔断器状态（可选）
+metrics = client.get_resilient_metrics()
+if metrics:
+    print(f"熔断器状态: {metrics['circuit_state']}")
+    print(f"失败次数: {metrics['failure_count']}")
+```
+
+#### 熔断器状态
+- **CLOSED**（关闭）：正常工作状态，请求正常通过
+- **OPEN**（打开）：熔断状态，所有请求直接降级到 HTTP
+- **HALF_OPEN**（半开）：恢复测试状态，允许少量请求测试 gRPC 是否恢复
+
+#### 监控指标
+```python
+# 获取熔断降级指标
+metrics = client.get_resilient_metrics()
+# 返回示例：
+# {
+#     "enabled": true,
+#     "circuit_state": "closed",
+#     "failure_count": 0,
+#     "last_failure_time": null,
+#     "http_fallback_url": "http://localhost:8080"
+# }
+```
+
 ### ⚠️ 注意事项
 
 1. **参数说明**
@@ -595,6 +650,23 @@ MODEL_MANAGER_SERVER_GRPC_MAX_RETRIES=3
 
 # 初始重试延迟（秒，默认 1.0），指数退避
 MODEL_MANAGER_SERVER_GRPC_RETRY_DELAY=1.0
+
+
+# ========================
+# 🛡️ 熔断降级配置（可选）
+# ========================
+
+# 是否启用熔断降级功能（默认 false）
+MODEL_CLIENT_RESILIENT_ENABLED=false
+
+# HTTP 降级服务地址（当 gRPC 不可用时的备用地址）
+MODEL_CLIENT_HTTP_FALLBACK_URL=http://localhost:8080
+
+# 熔断器触发阈值（连续失败多少次后熔断，默认 5）
+MODEL_CLIENT_CIRCUIT_BREAKER_THRESHOLD=5
+
+# 熔断器恢复超时（秒，熔断后多久尝试恢复，默认 60）
+MODEL_CLIENT_CIRCUIT_BREAKER_TIMEOUT=60
 ```
 
 加载后，初始化时无需传参：

{tamar_model_client-0.1.20.dist-info → tamar_model_client-0.1.22.dist-info}/RECORD

@@ -1,15 +1,17 @@
 tamar_model_client/__init__.py,sha256=4DEIUGlLTeiaECjJQbGYik7C0JO6hHwwfbLYpYpMdzg,444
-tamar_model_client/async_client.py,sha256=t945Qyw2s2MQpj9ArxCSGrMH625Asmt52QXD8S5CEPM,26314
+tamar_model_client/async_client.py,sha256=8BbzeYX735xIaPJPF0oI3g-8oDKyl6GugxxCdBEGi5s,34400
 tamar_model_client/auth.py,sha256=gbwW5Aakeb49PMbmYvrYlVx1mfyn1LEDJ4qQVs-9DA4,438
-tamar_model_client/error_handler.py,sha256=_KUCTCpXhQm7_7a0-luClOW3e7FbaNC8gDHvGp0Wtxg,10019
-tamar_model_client/exceptions.py,sha256=vATn4LUNiD-0sL2Cn4E-HqQEKmIBd96xcoUlkArus7w,11188
+tamar_model_client/circuit_breaker.py,sha256=0XHJXBYA4O8vwsDGwqNrae9zxNJphY5Rfucc9ytVFGA,5419
+tamar_model_client/error_handler.py,sha256=eEuwMcecJvQbFkIieri60uLoQX5sBMawnxxedxzxj5I,12745
+tamar_model_client/exceptions.py,sha256=D6G8igA-YO4AroeCa-9CDDPt4hSqBFX5C_4w-NCIL1w,13063
 tamar_model_client/json_formatter.py,sha256=IyBv_pEEzjF-KaMF-7rxRpNc_fxRYK2A-pu_2n4Liow,1990
 tamar_model_client/logging_icons.py,sha256=MRTZ1Xvkep9ce_jdltj54_XZUXvIpQ95soRNmLdJ4qw,1837
-tamar_model_client/sync_client.py,sha256=2LM3ZQ0M0MNE732sfwDeDruOAAiEEFB7BcjEeNmQIb8,27401
+tamar_model_client/sync_client.py,sha256=mpWc6T9RjKnNDiETE8UYtKu2Zu0U6AME1kQMnDwnfMI,36225
 tamar_model_client/utils.py,sha256=Kn6pFz9GEC96H4eejEax66AkzvsrXI3WCSDtgDjnVTI,5238
-tamar_model_client/core/__init__.py,sha256=PDQ2emPz3eHg_dhmvsd3pXlJf93A-Pl5Qh6E6mHO4XQ,670
-tamar_model_client/core/base_client.py,sha256=YogODGjDnQE2b2P_fcpgB1uYt0A88h_oUfFN63qZLf8,6699
-tamar_model_client/core/logging_setup.py,sha256=YXX0MEe83mpXwGH8A-D3bYOTNwX14NQbgfA7b3T_vbs,2626
+tamar_model_client/core/__init__.py,sha256=bJRJllrp4Xc0g_qu1pW9G-lsXNB7c1r0NBIfb2Ypxe0,832
+tamar_model_client/core/base_client.py,sha256=sYvJZsDu_66akddAMowSnihFtgOoVKaQJxxnVruF9Ms,8995
+tamar_model_client/core/http_fallback.py,sha256=1OuSMxzhDyxy07JZa5artMTNdPNMyAhI7By3RUCSPDw,9872
+tamar_model_client/core/logging_setup.py,sha256=h1aky1uslIQnx4NxMqjoDMxwlc4Vg46KYTjW9yPu2xQ,6032
 tamar_model_client/core/request_builder.py,sha256=yi8iy2Ps2m4d1YwIFiQLRxTvxQxgEGV576aXnNYRl7E,8507
 tamar_model_client/core/response_handler.py,sha256=_q5galAT0_RaUT5C_yZsjg-9VnT9CBjmIASOt28BUmQ,4616
 tamar_model_client/core/utils.py,sha256=8jSx8UOE6ukbiIgruCX7SXN8J5FyuGbqENOmJDsxaSM,5084
@@ -25,9 +27,10 @@ tamar_model_client/schemas/inputs.py,sha256=dz1m8NbUIxA99JXZc8WlyzbKpDuz1lEzx3Vg
 tamar_model_client/schemas/outputs.py,sha256=M_fcqUtXPJnfiLabHlyA8BorlC5pYkf5KLjXO1ysKIQ,1031
 tests/__init__.py,sha256=kbmImddLDwdqlkkmkyKtl4bQy_ipe-R8eskpaBylU9w,38
 tests/stream_hanging_analysis.py,sha256=W3W48IhQbNAR6-xvMpoWZvnWOnr56CTaH4-aORNBuD4,14807
-tests/test_google_azure_final.py,sha256=wAnfodYCs8VIqYlgT6nm1YnLnufqSuYfXBaVqCXkmfU,17019
+tests/test_google_azure_final.py,sha256=YFhjx2mQlFijcuHqOVnnS7ZD8mQWCf2Uv1oiqOFxASs,26393
+tests/test_logging_issue.py,sha256=JTMbotfHpAEPMBj73pOwxPn-Zn4QVQJX6scMz48FRDQ,2427
 tests/test_simple.py,sha256=Xf0U-J9_xn_LzUsmYu06suK0_7DrPeko8OHoHldsNxE,7169
-tamar_model_client-0.1.20.dist-info/METADATA,sha256=FDp1ZkTf3FJ08gVvIVAx2_CmGRz1aNqa7LJFH0WPqnY,21147
-tamar_model_client-0.1.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-tamar_model_client-0.1.20.dist-info/top_level.txt,sha256=f1I-S8iWN-cgv4gB8gxRg9jJOTJMumvm4oGKVPfGg6A,25
-tamar_model_client-0.1.20.dist-info/RECORD,,
+tamar_model_client-0.1.22.dist-info/METADATA,sha256=jEEq8UTzqVcutF26FefqaxZ08WhNW58xiyEWZgDO7WA,23453
+tamar_model_client-0.1.22.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tamar_model_client-0.1.22.dist-info/top_level.txt,sha256=f1I-S8iWN-cgv4gB8gxRg9jJOTJMumvm4oGKVPfGg6A,25
+tamar_model_client-0.1.22.dist-info/RECORD,,