tamar-model-client 0.1.20__py3-none-any.whl → 0.1.21__py3-none-any.whl

tamar_model_client/sync_client.py

@@ -30,22 +30,23 @@ import grpc
 from .core import (
     generate_request_id,
     set_request_id,
-    setup_logger,
+    get_protected_logger,
     MAX_MESSAGE_LENGTH
 )
 from .core.base_client import BaseClient
 from .core.request_builder import RequestBuilder
 from .core.response_handler import ResponseHandler
-from .exceptions import ConnectionError, TamarModelException, is_retryable_error
+from .exceptions import ConnectionError, TamarModelException
 from .generated import model_service_pb2, model_service_pb2_grpc
 from .schemas import BatchModelResponse, ModelResponse
 from .schemas.inputs import BatchModelRequest, ModelRequest
+from .core.http_fallback import HttpFallbackMixin
 
-# 配置日志记录器
-logger = setup_logger(__name__)
+# 配置日志记录器（使用受保护的logger）
+logger = get_protected_logger(__name__)
 
 
-class TamarModelClient(BaseClient):
+class TamarModelClient(BaseClient, HttpFallbackMixin):
     """
     Tamar Model Client 同步客户端
     
@@ -218,20 +219,31 @@ class TamarModelClient(BaseClient):
                 context['retry_count'] = attempt
 
                 # 判断是否可以重试
-                if not is_retryable_error(e.code()) or attempt >= self.max_retries:
+                should_retry = self._should_retry(e, attempt)
+                if not should_retry or attempt >= self.max_retries:
                     # 不可重试或已达到最大重试次数
                     last_exception = self.error_handler.handle_error(e, context)
                     break
 
                 # 记录重试日志
+                log_data = {
+                    "log_type": "info",
+                    "request_id": context.get('request_id'),
+                    "data": {
+                        "error_code": e.code().name if e.code() else 'UNKNOWN',
+                        "retry_count": attempt,
+                        "max_retries": self.max_retries,
+                        "method": context.get('method', 'unknown')
+                    }
+                }
                 logger.warning(
                     f"Attempt {attempt + 1}/{self.max_retries + 1} failed: {e.code()}",
-                    extra=context
+                    extra=log_data
                 )
 
                 # 执行退避等待
                 if attempt < self.max_retries:
-                    delay = self._calculate_backoff(attempt)
+                    delay = self._calculate_backoff(attempt, e.code())
                     time.sleep(delay)
 
                 last_exception = self.error_handler.handle_error(e, context)
@@ -248,14 +260,47 @@ class TamarModelClient(BaseClient):
         else:
             raise TamarModelException("Unknown error occurred")
 
-    def _calculate_backoff(self, attempt: int) -> float:
-        """计算退避时间"""
+    def _calculate_backoff(self, attempt: int, error_code: grpc.StatusCode = None) -> float:
+        """
+        计算退避时间，支持不同的退避策略
+        
+        Args:
+            attempt: 当前重试次数
+            error_code: gRPC错误码，用于确定退避策略
+        """
         max_delay = 60.0
-        jitter_factor = 0.1
-
-        delay = min(self.retry_delay * (2 ** attempt), max_delay)
-        jitter = random.uniform(0, delay * jitter_factor)
-        return delay + jitter
+        base_delay = self.retry_delay
+        
+        # 获取错误的重试策略
+        if error_code:
+            from .exceptions import get_retry_policy
+            policy = get_retry_policy(error_code)
+            backoff_type = policy.get('backoff', 'exponential')
+            use_jitter = policy.get('jitter', False)
+        else:
+            backoff_type = 'exponential'
+            use_jitter = False
+        
+        # 根据退避类型计算延迟
+        if backoff_type == 'linear':
+            # 线性退避：delay * (attempt + 1)
+            delay = min(base_delay * (attempt + 1), max_delay)
+        else:
+            # 指数退避：delay * 2^attempt
+            delay = min(base_delay * (2 ** attempt), max_delay)
+        
+        # 添加抖动
+        if use_jitter:
+            jitter_factor = 0.2  # 增加抖动范围，减少竞争
+            jitter = random.uniform(0, delay * jitter_factor)
+            delay += jitter
+        else:
+            # 默认的小量抖动，避免完全同步
+            jitter_factor = 0.05
+            jitter = random.uniform(0, delay * jitter_factor)
+            delay += jitter
+            
+        return delay
 
     def _retry_request_stream(self, func, *args, **kwargs):
         """
@@ -272,29 +317,77 @@ class TamarModelClient(BaseClient):
             流式响应的每个元素
         """
         last_exception = None
+        context = {
+            'method': 'stream',
+            'client_version': 'sync',
+        }
         
         for attempt in range(self.max_retries + 1):
             try:
+                context['retry_count'] = attempt
                 # 尝试创建流
                 for item in func(*args, **kwargs):
                     yield item
                 return
                 
             except grpc.RpcError as e:
-                last_exception = e
-                if attempt < self.max_retries:
-                    logger.warning(
-                        f"Stream attempt {attempt + 1}/{self.max_retries + 1} failed: {e.code()}",
-                        extra={"retry_count": attempt, "error_code": str(e.code())}
+                # 使用智能重试判断
+                context['retry_count'] = attempt
+                
+                # 判断是否应该重试
+                should_retry = self._should_retry(e, attempt)
+                if not should_retry or attempt >= self.max_retries:
+                    # 不重试或已达到最大重试次数
+                    log_data = {
+                        "log_type": "info",
+                        "request_id": context.get('request_id'),
+                        "data": {
+                            "error_code": e.code().name if e.code() else 'UNKNOWN',
+                            "retry_count": attempt,
+                            "max_retries": self.max_retries,
+                            "method": "stream",
+                            "will_retry": False
+                        }
+                    }
+                    logger.error(
+                        f"Stream failed: {e.code()} (no retry)",
+                        extra=log_data
                     )
-                    time.sleep(self.retry_delay * (attempt + 1))
-                else:
+                    last_exception = self.error_handler.handle_error(e, context)
                     break
+                
+                # 记录重试日志
+                log_data = {
+                    "log_type": "info",
+                    "request_id": context.get('request_id'),
+                    "data": {
+                        "error_code": e.code().name if e.code() else 'UNKNOWN',
+                        "retry_count": attempt,
+                        "max_retries": self.max_retries,
+                        "method": "stream"
+                    }
+                }
+                logger.warning(
+                    f"Stream attempt {attempt + 1}/{self.max_retries + 1} failed: {e.code()} (will retry)",
+                    extra=log_data
+                )
+                
+                # 执行退避等待
+                if attempt < self.max_retries:
+                    delay = self._calculate_backoff(attempt, e.code())
+                    time.sleep(delay)
+                    
+                last_exception = e
+                
             except Exception as e:
+                context['retry_count'] = attempt
                 raise TamarModelException(str(e)) from e
         
         if last_exception:
-            raise self.error_handler.handle_error(last_exception, {"retry_count": self.max_retries})
+            if isinstance(last_exception, TamarModelException):
+                raise last_exception
+            else:
+                raise self.error_handler.handle_error(last_exception, context)
         else:
             raise TamarModelException("Unknown streaming error occurred")
 
@@ -457,6 +550,12 @@ class TamarModelClient(BaseClient):
             ValidationError: 输入验证失败。
             ConnectionError: 连接服务端失败。
         """
+        # 如果启用了熔断且熔断器打开，直接走 HTTP
+        if self.resilient_enabled and self.circuit_breaker and self.circuit_breaker.is_open:
+            if self.http_fallback_url:
+                logger.warning("🔻 Circuit breaker is OPEN, using HTTP fallback")
+                return self._invoke_http_fallback(model_request, timeout, request_id)
+                
         self._ensure_initialized()
 
         if not self.default_payload:
@@ -527,9 +626,14 @@ class TamarModelClient(BaseClient):
                         "data": ResponseHandler.build_log_data(model_request, result)
                     }
                 )
+                
+                # 记录成功（如果启用了熔断）
+                if self.resilient_enabled and self.circuit_breaker:
+                    self.circuit_breaker.record_success()
+                    
                 return result
                 
-        except grpc.RpcError as e:
+        except (ConnectionError, grpc.RpcError) as e:
             duration = time.time() - start_time
             error_message = f"❌ Invoke gRPC failed: {str(e)}"
             logger.error(error_message, exc_info=True,
@@ -542,6 +646,18 @@ class TamarModelClient(BaseClient):
                                  error=e
                              )
                          })
+            
+            # 记录失败并尝试降级（如果启用了熔断）
+            if self.resilient_enabled and self.circuit_breaker:
+                # 将错误码传递给熔断器，用于智能失败统计
+                error_code = e.code() if hasattr(e, 'code') else None
+                self.circuit_breaker.record_failure(error_code)
+                
+                # 如果可以降级，则降级
+                if self.http_fallback_url and self.circuit_breaker.should_fallback():
+                    logger.warning(f"🔻 gRPC failed, falling back to HTTP: {str(e)}")
+                    return self._invoke_http_fallback(model_request, timeout, request_id)
+            
             raise e
         except Exception as e:
             duration = time.time() - start_time

{tamar_model_client-0.1.20.dist-info → tamar_model_client-0.1.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tamar-model-client
-Version: 0.1.20
+Version: 0.1.21
 Summary: A Python SDK for interacting with the Model Manager gRPC service
 Home-page: http://gitlab.tamaredge.top/project-tap/AgentOS/model-manager-client
 Author: Oscar Ou
@@ -532,6 +532,61 @@ response = client.invoke(
    - 启用流式响应减少首字延迟
    - 合理设置 max_tokens 避免浪费
 
+### 🛡️ 熔断降级功能（高可用保障）
+
+SDK 内置了熔断降级机制，当 gRPC 服务不可用时自动切换到 HTTP 服务，确保业务连续性。
+
+#### 工作原理
+1. **正常状态**：所有请求通过高性能的 gRPC 协议
+2. **熔断触发**：当连续失败达到阈值时，熔断器打开
+3. **自动降级**：切换到 HTTP 协议继续提供服务
+4. **定期恢复**：熔断器会定期尝试恢复到 gRPC
+
+#### 启用方式
+```bash
+# 设置环境变量
+export MODEL_CLIENT_RESILIENT_ENABLED=true
+export MODEL_CLIENT_HTTP_FALLBACK_URL=http://localhost:8080
+export MODEL_CLIENT_CIRCUIT_BREAKER_THRESHOLD=5
+export MODEL_CLIENT_CIRCUIT_BREAKER_TIMEOUT=60
+```
+
+#### 使用示例
+```python
+from tamar_model_client import TamarModelClient
+
+# 客户端会自动处理熔断降级，对使用者透明
+client = TamarModelClient()
+
+# 正常使用，无需关心底层协议
+response = client.invoke(request)
+
+# 获取熔断器状态（可选）
+metrics = client.get_resilient_metrics()
+if metrics:
+    print(f"熔断器状态: {metrics['circuit_state']}")
+    print(f"失败次数: {metrics['failure_count']}")
+```
+
+#### 熔断器状态
+- **CLOSED**（关闭）：正常工作状态，请求正常通过
+- **OPEN**（打开）：熔断状态，所有请求直接降级到 HTTP
+- **HALF_OPEN**（半开）：恢复测试状态，允许少量请求测试 gRPC 是否恢复
+
+#### 监控指标
+```python
+# 获取熔断降级指标
+metrics = client.get_resilient_metrics()
+# 返回示例：
+# {
+#     "enabled": true,
+#     "circuit_state": "closed",
+#     "failure_count": 0,
+#     "last_failure_time": null,
+#     "http_fallback_url": "http://localhost:8080"
+# }
+```
+
 ### ⚠️ 注意事项
 
 1. **参数说明**
@@ -595,6 +650,23 @@ MODEL_MANAGER_SERVER_GRPC_MAX_RETRIES=3
 
 # 初始重试延迟（秒，默认 1.0），指数退避
 MODEL_MANAGER_SERVER_GRPC_RETRY_DELAY=1.0
+
+
+# ========================
+# 🛡️ 熔断降级配置（可选）
+# ========================
+
+# 是否启用熔断降级功能（默认 false）
+MODEL_CLIENT_RESILIENT_ENABLED=false
+
+# HTTP 降级服务地址（当 gRPC 不可用时的备用地址）
+MODEL_CLIENT_HTTP_FALLBACK_URL=http://localhost:8080
+
+# 熔断器触发阈值（连续失败多少次后熔断，默认 5）
+MODEL_CLIENT_CIRCUIT_BREAKER_THRESHOLD=5
+
+# 熔断器恢复超时（秒，熔断后多久尝试恢复，默认 60）
+MODEL_CLIENT_CIRCUIT_BREAKER_TIMEOUT=60
 ```
 
 加载后，初始化时无需传参：

{tamar_model_client-0.1.20.dist-info → tamar_model_client-0.1.21.dist-info}/RECORD

@@ -1,15 +1,17 @@
 tamar_model_client/__init__.py,sha256=4DEIUGlLTeiaECjJQbGYik7C0JO6hHwwfbLYpYpMdzg,444
-tamar_model_client/async_client.py,sha256=t945Qyw2s2MQpj9ArxCSGrMH625Asmt52QXD8S5CEPM,26314
+tamar_model_client/async_client.py,sha256=cU3cUrwP75zacyFa3KibcYfacJBsxRNLIu1vtQZrrcU,32836
 tamar_model_client/auth.py,sha256=gbwW5Aakeb49PMbmYvrYlVx1mfyn1LEDJ4qQVs-9DA4,438
-tamar_model_client/error_handler.py,sha256=_KUCTCpXhQm7_7a0-luClOW3e7FbaNC8gDHvGp0Wtxg,10019
-tamar_model_client/exceptions.py,sha256=vATn4LUNiD-0sL2Cn4E-HqQEKmIBd96xcoUlkArus7w,11188
+tamar_model_client/circuit_breaker.py,sha256=0XHJXBYA4O8vwsDGwqNrae9zxNJphY5Rfucc9ytVFGA,5419
+tamar_model_client/error_handler.py,sha256=kVfHL7DWvO3sIobjVuJbqjV4mtI4oqbS4Beax7Dmm9w,11788
+tamar_model_client/exceptions.py,sha256=FImLCBpYQ8DpsNbH-ZttxyClEZCL6ICmQGESIlbI--s,12038
 tamar_model_client/json_formatter.py,sha256=IyBv_pEEzjF-KaMF-7rxRpNc_fxRYK2A-pu_2n4Liow,1990
 tamar_model_client/logging_icons.py,sha256=MRTZ1Xvkep9ce_jdltj54_XZUXvIpQ95soRNmLdJ4qw,1837
-tamar_model_client/sync_client.py,sha256=2LM3ZQ0M0MNE732sfwDeDruOAAiEEFB7BcjEeNmQIb8,27401
+tamar_model_client/sync_client.py,sha256=AhNFlhk9aC7JhNrI2BEZJDLjXZwVT9pMy3u9jgjO1QU,32603
 tamar_model_client/utils.py,sha256=Kn6pFz9GEC96H4eejEax66AkzvsrXI3WCSDtgDjnVTI,5238
-tamar_model_client/core/__init__.py,sha256=PDQ2emPz3eHg_dhmvsd3pXlJf93A-Pl5Qh6E6mHO4XQ,670
-tamar_model_client/core/base_client.py,sha256=YogODGjDnQE2b2P_fcpgB1uYt0A88h_oUfFN63qZLf8,6699
-tamar_model_client/core/logging_setup.py,sha256=YXX0MEe83mpXwGH8A-D3bYOTNwX14NQbgfA7b3T_vbs,2626
+tamar_model_client/core/__init__.py,sha256=bJRJllrp4Xc0g_qu1pW9G-lsXNB7c1r0NBIfb2Ypxe0,832
+tamar_model_client/core/base_client.py,sha256=sYvJZsDu_66akddAMowSnihFtgOoVKaQJxxnVruF9Ms,8995
+tamar_model_client/core/http_fallback.py,sha256=1OuSMxzhDyxy07JZa5artMTNdPNMyAhI7By3RUCSPDw,9872
+tamar_model_client/core/logging_setup.py,sha256=h1aky1uslIQnx4NxMqjoDMxwlc4Vg46KYTjW9yPu2xQ,6032
 tamar_model_client/core/request_builder.py,sha256=yi8iy2Ps2m4d1YwIFiQLRxTvxQxgEGV576aXnNYRl7E,8507
 tamar_model_client/core/response_handler.py,sha256=_q5galAT0_RaUT5C_yZsjg-9VnT9CBjmIASOt28BUmQ,4616
 tamar_model_client/core/utils.py,sha256=8jSx8UOE6ukbiIgruCX7SXN8J5FyuGbqENOmJDsxaSM,5084
@@ -27,7 +29,7 @@ tests/__init__.py,sha256=kbmImddLDwdqlkkmkyKtl4bQy_ipe-R8eskpaBylU9w,38
 tests/stream_hanging_analysis.py,sha256=W3W48IhQbNAR6-xvMpoWZvnWOnr56CTaH4-aORNBuD4,14807
 tests/test_google_azure_final.py,sha256=wAnfodYCs8VIqYlgT6nm1YnLnufqSuYfXBaVqCXkmfU,17019
 tests/test_simple.py,sha256=Xf0U-J9_xn_LzUsmYu06suK0_7DrPeko8OHoHldsNxE,7169
-tamar_model_client-0.1.20.dist-info/METADATA,sha256=FDp1ZkTf3FJ08gVvIVAx2_CmGRz1aNqa7LJFH0WPqnY,21147
-tamar_model_client-0.1.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-tamar_model_client-0.1.20.dist-info/top_level.txt,sha256=f1I-S8iWN-cgv4gB8gxRg9jJOTJMumvm4oGKVPfGg6A,25
-tamar_model_client-0.1.20.dist-info/RECORD,,
+tamar_model_client-0.1.21.dist-info/METADATA,sha256=gj8tUbP3goUZKi3pVVWMxEpmmK6W72IV23Ym2ohlcBs,23453
+tamar_model_client-0.1.21.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tamar_model_client-0.1.21.dist-info/top_level.txt,sha256=f1I-S8iWN-cgv4gB8gxRg9jJOTJMumvm4oGKVPfGg6A,25
+tamar_model_client-0.1.21.dist-info/RECORD,,