sycommon-python-lib 0.1.57b4__py3-none-any.whl → 0.1.57b6__py3-none-any.whl

sycommon/llm/embedding.py

@@ -30,18 +30,12 @@ class Embedding(metaclass=SingletonMeta):
             self.default_reranker_model: self.reranker_base_url
         }
 
-        # [修复] 缓存模型的向量维度，用于生成兜底零向量
-        self._model_dim_cache: Dict[str, int] = {}
-
         # 并发信号量
         self.semaphore = asyncio.Semaphore(self.max_concurrency)
         self.default_timeout = aiohttp.ClientTimeout(total=None)
 
         # 核心优化：创建全局可复用的ClientSession（连接池复用）
         self.session = None
-        # 重试配置（可根据需要调整）
-        self.max_retry_attempts = 3  # 最大重试次数
-        self.retry_wait_base = 0.5   # 基础等待时间（秒）
 
         # [修复] 注册退出钩子，确保程序结束时关闭连接池
         atexit.register(self._sync_close_session)
@@ -51,7 +45,7 @@ class Embedding(metaclass=SingletonMeta):
         if self.session is None or self.session.closed:
             # 配置连接池参数，适配高并发
             connector = aiohttp.TCPConnector(
-                limit=self.max_concurrency * 2,  # 连接池最大连接数（建议是并发数的2倍）
+                limit=self.max_concurrency,  # 连接池最大连接数
                 limit_per_host=self.max_concurrency,  # 每个域名的最大连接数
                 ttl_dns_cache=300,  # DNS缓存时间
                 enable_cleanup_closed=True  # 自动清理关闭的连接
@@ -68,52 +62,23 @@ class Embedding(metaclass=SingletonMeta):
 
     def _sync_close_session(self):
         """同步关闭Session的封装，供atexit调用"""
-        # 注意：atexit在主线程运行，如果当前没有事件循环，这个操作可能会受限
-        # 但它能捕获大多数正常退出的场景。对于asyncio程序，建议显式调用cleanup
         try:
             loop = asyncio.get_event_loop()
             if loop.is_running():
-                # 如果loop还在跑，创建一个任务去关闭
-                loop.create_task(self.close_session())
+                # [修复] 修正缩进，确保 create_task 的异常能被捕获
+                try:
+                    loop.create_task(self.close_session())
+                except Exception:
+                    pass
             else:
-                # 如果loop已经停止，尝试运行一次
-                loop.run_until_complete(self.close_session())
+                try:
+                    loop.run_until_complete(self.close_session())
+                except Exception:
+                    pass
         except Exception:
-            # 静默处理清理失败，避免退出报错
+            # 捕获获取 loop 时的异常
             pass
 
-    async def _retry_request(self, func, *args, **kwargs):
-        """
-        原生异步重试封装函数
-        Args:
-            func: 待重试的异步函数
-            *args: 函数参数
-            **kwargs: 函数关键字参数
-        Returns:
-            函数执行结果，重试失败返回None
-        """
-        attempt = 0
-        while attempt < self.max_retry_attempts:
-            try:
-                return await func(*args, **kwargs)
-            except (aiohttp.ClientConnectionResetError, asyncio.TimeoutError, aiohttp.ClientError) as e:
-                attempt += 1
-                if attempt >= self.max_retry_attempts:
-                    SYLogger.error(
-                        f"Request failed after {attempt} retries: {str(e)}")
-                    return None
-                # 指数退避等待：0.5s → 1s → 2s（最大不超过5s）
-                wait_time = min(self.retry_wait_base * (2 ** (attempt - 1)), 5)
-                SYLogger.warning(
-                    f"Retry {func.__name__} (attempt {attempt}/{self.max_retry_attempts}): {str(e)}, wait {wait_time}s")
-                await asyncio.sleep(wait_time)
-            except Exception as e:
-                # 非重试类异常直接返回None
-                SYLogger.error(
-                    f"Non-retryable error in {func.__name__}: {str(e)}")
-                return None
-        return None
-
     def _get_embedding_url(self, model: str) -> str:
         """获取Embedding URL（带缓存）"""
         if model not in self._embedding_url_cache:
@@ -136,7 +101,7 @@ class Embedding(metaclass=SingletonMeta):
         timeout: aiohttp.ClientTimeout = None,
         **kwargs
     ):
-        """embedding请求核心逻辑（剥离重试，供重试封装调用）"""
+        """embedding请求核心逻辑"""
         await self.init_session()  # 确保Session已初始化
         async with self.semaphore:
             request_timeout = timeout or self.default_timeout
@@ -154,17 +119,33 @@ class Embedding(metaclass=SingletonMeta):
             request_body.update(kwargs)
 
             # 复用全局Session
-            async with self.session.post(
-                url,
-                json=request_body,
-                timeout=request_timeout
-            ) as response:
-                if response.status != 200:
-                    error_detail = await response.text()
-                    SYLogger.error(
-                        f"Embedding request failed (model: {target_model}): {error_detail}")
-                    return None
-                return await response.json()
+            try:
+                async with self.session.post(
+                    url,
+                    json=request_body,
+                    timeout=request_timeout
+                ) as response:
+                    if response.status != 200:
+                        error_detail = await response.text()
+                        # [日志] 记录详细的HTTP错误响应
+                        SYLogger.error(
+                            f"Embedding request HTTP Error. Status: {response.status}, "
+                            f"Model: {target_model}, URL: {url}. Detail: {error_detail}"
+                        )
+                        return None
+                    return await response.json()
+            except (aiohttp.ClientConnectionResetError, asyncio.TimeoutError, aiohttp.ClientError) as e:
+                # [日志] 记录网络错误
+                SYLogger.error(
+                    f"Embedding request Network Error. Model: {target_model}, URL: {url}. "
+                    f"Error: {e.__class__.__name__} - {str(e)}"
+                )
+                return None
+            except Exception as e:
+                # 记录其他未预期的异常
+                SYLogger.error(
+                    f"Unexpected error in _get_embeddings_http_core: {str(e)}", exc_info=True)
+                return None
 
     async def _get_embeddings_http_async(
         self,
@@ -173,9 +154,8 @@ class Embedding(metaclass=SingletonMeta):
         model: str = None,
         timeout: aiohttp.ClientTimeout = None, ** kwargs
     ):
-        """对外暴露的embedding请求方法（包含重试）"""
-        return await self._retry_request(
-            self._get_embeddings_http_core,
+        """对外暴露的embedding请求方法"""
+        return await self._get_embeddings_http_core(
             input, encoding_format, model, timeout, ** kwargs
         )
 
@@ -190,7 +170,7 @@ class Embedding(metaclass=SingletonMeta):
         return_len: Optional[bool] = True,
         timeout: aiohttp.ClientTimeout = None, ** kwargs
     ):
-        """reranker请求核心逻辑（剥离重试，供重试封装调用）"""
+        """reranker请求核心逻辑"""
         await self.init_session()  # 确保Session已初始化
         async with self.semaphore:
             request_timeout = timeout or self.default_timeout
@@ -212,17 +192,33 @@ class Embedding(metaclass=SingletonMeta):
             request_body.update(kwargs)
 
             # 复用全局Session
-            async with self.session.post(
-                url,
-                json=request_body,
-                timeout=request_timeout
-            ) as response:
-                if response.status != 200:
-                    error_detail = await response.text()
-                    SYLogger.error(
-                        f"Rerank request failed (model: {target_model}): {error_detail}")
-                    return None
-                return await response.json()
+            try:
+                async with self.session.post(
+                    url,
+                    json=request_body,
+                    timeout=request_timeout
+                ) as response:
+                    if response.status != 200:
+                        error_detail = await response.text()
+                        # [日志] 记录详细的HTTP错误响应
+                        SYLogger.error(
+                            f"Reranker request HTTP Error. Status: {response.status}, "
+                            f"Model: {target_model}, URL: {url}. Detail: {error_detail}"
+                        )
+                        return None
+                    return await response.json()
+            except (aiohttp.ClientConnectionResetError, asyncio.TimeoutError, aiohttp.ClientError) as e:
+                # [日志] 记录网络错误
+                SYLogger.error(
+                    f"Reranker request Network Error. Model: {target_model}, URL: {url}. "
+                    f"Error: {e.__class__.__name__} - {str(e)}"
+                )
+                return None
+            except Exception as e:
+                # 记录其他未预期的异常
+                SYLogger.error(
+                    f"Unexpected error in _get_reranker_http_core: {str(e)}", exc_info=True)
+                return None
 
     async def _get_reranker_http_async(
         self,
@@ -235,13 +231,23 @@ class Embedding(metaclass=SingletonMeta):
         return_len: Optional[bool] = True,
         timeout: aiohttp.ClientTimeout = None, ** kwargs
     ):
-        """对外暴露的reranker请求方法（包含重试）"""
-        return await self._retry_request(
-            self._get_reranker_http_core,
+        """对外暴露的reranker请求方法"""
+        return await self._get_reranker_http_core(
             documents, query, top_n, model, max_chunks_per_doc,
             return_documents, return_len, timeout, **kwargs
         )
 
+    def _get_dimension(self, model: str) -> int:
+        """获取模型维度，用于生成兜底零向量"""
+        try:
+            config = EmbeddingConfig.from_config(model)
+            if hasattr(config, 'dimension'):
+                return int(config.dimension)
+        except Exception:
+            pass
+        # 默认兜底 1024
+        return 1024
+
     async def get_embeddings(
         self,
         corpus: List[str],
@@ -287,44 +293,23 @@ class Embedding(metaclass=SingletonMeta):
 
             for result in results:
                 if result is None:
-                    # [修复] 尝试获取真实维度或使用配置兜底，不再硬编码 1024
-                    dim = self._model_dim_cache.get(actual_model)
-
-                    # 如果缓存中没有维度，尝试从配置对象获取（假设Config类有dimension属性）
-                    if dim is None:
-                        try:
-                            config = EmbeddingConfig.from_config(actual_model)
-                            if hasattr(config, 'dimension'):
-                                dim = config.dimension
-                            else:
-                                # 最后的兜底：如果配置也没有，必须有一个默认值防止崩溃
-                                # bge-large 通常是 1024
-                                dim = 1024
-                                SYLogger.warning(
-                                    f"Cannot get dimension from config for {actual_model}, use default 1024")
-                        except Exception:
-                            dim = 1024
+                    dim = self._get_dimension(actual_model)
 
                     zero_vector = [0.0] * dim
                     all_vectors.append(zero_vector)
+                    # [日志] 补充日志，明确是补零操作
                     SYLogger.warning(
-                        f"Embedding request failed, append zero vector ({dim}D) for model {actual_model}")
+                        f"Embedding request failed (returned None), appending zero vector ({dim}D) for model {actual_model}")
                     continue
 
-                # 从返回结果中提取向量并更新维度缓存
-                # 正常情况下 result["data"] 是一个列表
+                # 从返回结果中提取向量
                 try:
                     for item in result["data"]:
                         embedding = item["embedding"]
-                        # [修复] 动态学习并缓存维度
-                        if actual_model not in self._model_dim_cache:
-                            self._model_dim_cache[actual_model] = len(
-                                embedding)
                         all_vectors.append(embedding)
                 except (KeyError, TypeError) as e:
                     SYLogger.error(f"Failed to parse embedding result: {e}")
-                    # 解析失败也补零
-                    dim = self._model_dim_cache.get(actual_model, 1024)
+                    dim = self._get_dimension(actual_model)
                     all_vectors.append([0.0] * dim)
 
         SYLogger.info(

sycommon/rabbitmq/rabbitmq_client.py

@@ -256,75 +256,68 @@ class RabbitMQClient:
             self._message_handler = handler
 
     async def _process_message_callback(self, message: AbstractIncomingMessage):
-        # 定义标志位，记录我们是否需要重试（即业务是否失败）
-        should_retry = False
+        # 记录消息的原始追踪ID
+        # original_trace_id = message.headers.get(
+        #     "trace-id") if message.headers else None
+        # current_retry = 0
 
         try:
             msg_obj: MQMsgModel
+
+            # 1. 解析消息
             if self.auto_parse_json:
                 try:
                     body_dict = json.loads(message.body.decode("utf-8"))
                     msg_obj = MQMsgModel(**body_dict)
                 except json.JSONDecodeError as e:
                     logger.error(f"JSON解析失败: {e}")
-                    # 格式错误，无法处理，直接拒绝不重试
                     await message.reject(requeue=False)
-                    return  # 这里 return 了，不会走下面的 finally
+                    return
             else:
                 msg_obj = MQMsgModel(
                     body=message.body.decode("utf-8"),
                     routing_key=message.routing_key,
                     delivery_tag=message.delivery_tag,
-                    traceId=message.headers.get("trace-id"),
+                    traceId=message.headers.get(
+                        "trace-id") if message.headers else SYLogger.get_trace_id(),
                 )
 
+            # 2. 设置日志上下文
+            # 注意：如果 header 中有 x-last-retry-ts，说明之前重试过
+            # current_retry = int(message.headers.get("x-retry-count", 0))
             SYLogger.set_trace_id(msg_obj.traceId)
 
+            # 3. 执行业务逻辑
             if self._message_handler:
                 await self._message_handler(msg_obj, message)
 
-            # 如果正常执行到这里，说明业务成功
-            # await message.ack()
-            # 我们移除这里的 ack，统一交给 finally 处理
+            # 4. 业务成功，Ack (移除 finally 中的 ack，成功即确认)
+            await message.ack()
 
         except Exception as e:
-            logger.error(f"消息处理异常: {e}", exc_info=True)
-            # 业务异常，标记需要重试
-            should_retry = True
-
-        finally:
-            # 【核心修复】无论发生什么，最后都要给 MQ 一个交待
-            if should_retry:
-                headers = dict(message.headers) if message.headers else {}
-                current_retry = int(headers.get("x-retry-count", 0))
-
-                if current_retry >= 3:
-                    logger.warning(f"重试次数超限，丢弃消息: {message.delivery_tag}")
-                    await message.reject(requeue=False)
-                else:
-                    headers["x-retry-count"] = current_retry + 1
-                    try:
-                        new_msg = Message(
-                            body=message.body,
-                            headers=headers,
-                            content_type=message.content_type,
-                            delivery_mode=message.delivery_mode
-                        )
-                        # 发送新消息用于重试
-                        await self._exchange.publish(new_msg, routing_key=message.routing_key)
-
-                        # 【关键】新消息发成功了，现在可以安全地 Ack 掉旧消息了
-                        # 这样旧消息才会从队列中移除，避免死循环
-                        await message.ack()
-
-                    except Exception as pub_err:
-                        logger.error(f"重试发布失败，消息将丢失: {pub_err}")
-                        # 发布失败，无法重试，只能丢弃旧消息（或者 Nack requeue=True）
-                        # 为了防止死循环，这里通常建议 Reject (False) 并配置死信队列
-                        await message.reject(requeue=False)
-            else:
-                # 业务正常执行，直接 Ack
-                await message.ack()
+            # logger.error(f"消息处理异常 (第 {current_retry} 次尝试): {e}", exc_info=True)
+
+            # # 【核心修复】使用原生 Nack + Requeue
+            # if current_retry >= 3:
+            #     # 超过重试次数，丢弃消息（或进入死信队列）
+            #     logger.warning(f"重试次数超限 (3次)，丢弃消息: {message.delivery_tag}")
+            #     await message.reject(requeue=False)
+            # else:
+            #     # 还没到重试上限，重新入队
+            #     # 为了防止立即重试导致的死循环，我们需要人为增加一点延迟
+            #     # 但 Nack 本身不支持延迟，所以这里只能快速 Nack 让它尽快回来，
+            #     # 并在业务层（或外层）做好限流保护。
+
+            #     # 如果你有延迟队列插件，可以 publish 到延迟交换机。
+            #     # 如果没有，直接 requeue 是最安全的不丢包方案。
+            #     logger.info(f"消息处理失败，重新入队等待重试... (当前重试: {current_retry})")
+
+            #     # 技巧：如果你不想立即重试，可以 Nack(False) 然后手动 Publish 延迟消息
+            #     # 但为了解决你当前的“死循环”问题，直接 Nack(True) 是最有效的
+            #     # 延迟5秒
+            #     await asyncio.sleep(5)
+            #     await message.nack(requeue=True)
+            await message.ack()
 
     async def start_consuming(self) -> Optional[ConsumerTag]:
         if self._closed:

{sycommon_python_lib-0.1.57b4.dist-info → sycommon_python_lib-0.1.57b6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.1.57b4
+Version: 0.1.57b6
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.1.57b4.dist-info → sycommon_python_lib-0.1.57b6.dist-info}/RECORD

@@ -19,7 +19,7 @@ sycommon/health/health_check.py,sha256=EhfbhspRpQiKJaxdtE-PzpKQO_ucaFKtQxIm16F5M
 sycommon/health/metrics.py,sha256=fHqO73JuhoZkNPR-xIlxieXiTCvttq-kG-tvxag1s1s,268
 sycommon/health/ping.py,sha256=FTlnIKk5y1mPfS1ZGOeT5IM_2udF5aqVLubEtuBp18M,250
 sycommon/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sycommon/llm/embedding.py,sha256=rasx8xBOq-mQdWZ5RSzpwjbkAKi7Da-FDWiTm-Ga2Bs,15606
+sycommon/llm/embedding.py,sha256=qf7snCJQNemiB6Yz3nHEVB70sMjGXQx59Fz5RpT82sE,14372
 sycommon/llm/get_llm.py,sha256=C48gt9GCwEpR26M-cUjM74_t-el18ZvlwpGhcQfR3gs,1054
 sycommon/llm/llm_logger.py,sha256=n4UeNy_-g4oHQOsw-VUzF4uo3JVRLtxaMp1FcI8FiEo,5437
 sycommon/llm/llm_tokens.py,sha256=-udDyFcmyzx6UAwIi6_d_wwI5kMd5w0-WcS2soVPQxg,4309
@@ -51,7 +51,7 @@ sycommon/models/mqsend_config.py,sha256=NQX9dc8PpuquMG36GCVhJe8omAW1KVXXqr6lSRU6
 sycommon/models/sso_user.py,sha256=i1WAN6k5sPcPApQEdtjpWDy7VrzWLpOrOQewGLGoGIw,2702
 sycommon/notice/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sycommon/notice/uvicorn_monitor.py,sha256=VryQYcAtjijJuGDBimbVurgwxlsLaLtkNnABPDY5Tao,7332
-sycommon/rabbitmq/rabbitmq_client.py,sha256=PaO9shCi665MOuDXhSmRJtOBC-ayo9sD9U3rQq90Rqc,19071
+sycommon/rabbitmq/rabbitmq_client.py,sha256=XN_I-W2PWqjpxk_DrvsgVEHQaWQ1S7Y-N2_psNAFbg8,18833
 sycommon/rabbitmq/rabbitmq_pool.py,sha256=BiFQgZPzSAFR-n5XhyIafoeWQXETF_31nFRDhMbe6aU,15577
 sycommon/rabbitmq/rabbitmq_service.py,sha256=XSHo9HuIJ_lq-vizRh4xJVdZr_2zLqeLhot09qb0euA,2025
 sycommon/rabbitmq/rabbitmq_service_client_manager.py,sha256=IP9TMFeG5LSrwFPEmOy1ce4baPxBUZnWJZR3nN_-XR4,8009
@@ -82,8 +82,8 @@ sycommon/tools/env.py,sha256=Ah-tBwG2C0_hwLGFebVQgKdWWXCjTzBuF23gCkLHYy4,2437
 sycommon/tools/merge_headers.py,sha256=u9u8_1ZIuGIminWsw45YJ5qnsx9MB-Fot0VPge7itPw,4941
 sycommon/tools/snowflake.py,sha256=xQlYXwYnI85kSJ1rZ89gMVBhzemP03xrMPVX9vVa3MY,9228
 sycommon/tools/timing.py,sha256=OiiE7P07lRoMzX9kzb8sZU9cDb0zNnqIlY5pWqHcnkY,2064
-sycommon_python_lib-0.1.57b4.dist-info/METADATA,sha256=DzUZnbSOLPma462MbLY8-WQAaj9wh9BjkB1eSGxzq1A,7301
-sycommon_python_lib-0.1.57b4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sycommon_python_lib-0.1.57b4.dist-info/entry_points.txt,sha256=q_h2nbvhhmdnsOUZEIwpuoDjaNfBF9XqppDEmQn9d_A,46
-sycommon_python_lib-0.1.57b4.dist-info/top_level.txt,sha256=98CJ-cyM2WIKxLz-Pf0AitWLhJyrfXvyY8slwjTXNuc,17
-sycommon_python_lib-0.1.57b4.dist-info/RECORD,,
+sycommon_python_lib-0.1.57b6.dist-info/METADATA,sha256=7jdIE6yCJt89yjcZoBiDUujDJijuyL3x1U8NRH-Tir4,7301
+sycommon_python_lib-0.1.57b6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sycommon_python_lib-0.1.57b6.dist-info/entry_points.txt,sha256=q_h2nbvhhmdnsOUZEIwpuoDjaNfBF9XqppDEmQn9d_A,46
+sycommon_python_lib-0.1.57b6.dist-info/top_level.txt,sha256=98CJ-cyM2WIKxLz-Pf0AitWLhJyrfXvyY8slwjTXNuc,17
+sycommon_python_lib-0.1.57b6.dist-info/RECORD,,