hello-datap-component-base 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

hello_datap_component_base/__init__.py

@@ -13,7 +13,7 @@ from .discover import find_service_classes, get_single_service_class
 # 导入 logger 实例
 from .logger import logger
 
-__version__ = "0.1.9"
+__version__ = "0.2.3"
 __author__ = "zhaohaidong"
 __email__ = "zhaohaidong389@hellobike.com"
 

hello_datap_component_base/cli.py

@@ -11,7 +11,7 @@ from .config import ServerConfig
 
 
 @click.group()
-@click.version_option(version="0.1.9")
+@click.version_option(version="0.2.3")
 def cli():
     """数据处理平台组件基类 - 统一的服务管理框架"""
     pass

hello_datap_component_base/mns_client.py

@@ -9,6 +9,11 @@ import logging
 
 logger = logging.getLogger(__name__)
 
+# MNS 消息体最大长度（64KB），预留一些空间给 JSON 结构
+MNS_MAX_MESSAGE_SIZE = 64 * 1024
+# 错误消息最大长度（预留空间给其他字段）
+MAX_ERROR_MESSAGE_LENGTH = 8 * 1024  # 8KB
+
 
 class MNSClient:
     """阿里云 MNS 消息队列客户端"""
@@ -145,6 +150,49 @@ class MNSClient:
         self._queue.send_message(msg)
         return True
     
+    def _truncate_message(self, message: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        截断消息中过长的字段，确保消息体不超过 MNS 限制
+        
+        Args:
+            message: 原始消息字典
+            
+        Returns:
+            截断后的消息字典
+        """
+        # 深拷贝，避免修改原始数据
+        import copy
+        truncated = copy.deepcopy(message)
+        
+        # 截断 message 字段（错误信息）
+        if 'message' in truncated and isinstance(truncated['message'], str):
+            original_len = len(truncated['message'])
+            if original_len > MAX_ERROR_MESSAGE_LENGTH:
+                truncated['message'] = (
+                    truncated['message'][:MAX_ERROR_MESSAGE_LENGTH] + 
+                    f"\n... [truncated, original length: {original_len}]"
+                )
+                logger.warning(
+                    f"消息的 message 字段过长（{original_len} 字符），已截断至 {MAX_ERROR_MESSAGE_LENGTH} 字符"
+                )
+        
+        # 检查整体消息大小，如果仍然过大，进一步截断 out_put
+        message_body = json.dumps(truncated, ensure_ascii=False)
+        if len(message_body.encode('utf-8')) > MNS_MAX_MESSAGE_SIZE:
+            if 'data' in truncated and isinstance(truncated['data'], dict):
+                if 'out_put' in truncated['data'] and truncated['data']['out_put'] is not None:
+                    out_put_str = json.dumps(truncated['data']['out_put'], ensure_ascii=False)
+                    if len(out_put_str) > MAX_ERROR_MESSAGE_LENGTH:
+                        truncated['data']['out_put'] = {
+                            '_truncated': True,
+                            '_message': f'Output too large ({len(out_put_str)} chars), truncated'
+                        }
+                        logger.warning(
+                            f"消息的 out_put 字段过大（{len(out_put_str)} 字符），已截断"
+                        )
+        
+        return truncated
+
     def send_message(self, message: Dict[str, Any]) -> bool:
         """
         发送消息到队列（带重试逻辑）
@@ -161,8 +209,11 @@ class MNSClient:
             logger.warning("MNS 队列未初始化，跳过消息发送")
             return False
         
+        # 截断过长的消息字段
+        truncated_message = self._truncate_message(message)
+        
         # 将消息转换为 JSON 字符串
-        message_body = json.dumps(message, ensure_ascii=False)
+        message_body = json.dumps(truncated_message, ensure_ascii=False)
         
         # 重试逻辑
         last_exception = None

hello_datap_component_base/oss_client.py

@@ -29,10 +29,10 @@ class OSSClient:
             import oss2
             
             # 从环境变量获取 OSS 配置
-            endpoint = os.environ.get('OSS_ENDPOINT')
-            access_key_id = os.environ.get('OSS_ACCESS_KEY_ID')
-            access_key_secret = os.environ.get('OSS_ACCESS_KEY_SECRET')
-            bucket_name = os.environ.get('OSS_BUCKET_NAME')
+            endpoint = os.environ.get('OSS_ENDPOINT_FOR_LOG')
+            access_key_id = os.environ.get('OSS_ACCESS_KEY_ID_FOR_LOG')
+            access_key_secret = os.environ.get('OSS_ACCESS_KEY_SECRET_FOR_LOG')
+            bucket_name = os.environ.get('OSS_BUCKET_NAME_FOR_LOG')
             
             # 如果 OSS_ENDPOINT 不存在，静默跳过（向下兼容）
             if not endpoint:
@@ -43,7 +43,7 @@ class OSSClient:
             if not all([access_key_id, access_key_secret, bucket_name]):
                 logger.warning(
                     "OSS 配置不完整，无法上传日志文件。"
-                    "需要设置环境变量: OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET, OSS_BUCKET_NAME"
+                    "需要设置环境变量: OSS_ACCESS_KEY_ID_FOR_LOG, OSS_ACCESS_KEY_SECRET_FOR_LOG, OSS_BUCKET_NAME_FOR_LOG"
                 )
                 self._initialized = True
                 return
@@ -139,7 +139,7 @@ def get_oss_client() -> Optional[OSSClient]:
         OSSClient 实例，如果配置不完整则返回 None
     """
     # 如果环境变量中没有 OSS_ENDPOINT 配置，则不使用 OSS，直接返回 None（向下兼容）
-    if not os.environ.get('OSS_ENDPOINT'):
+    if not os.environ.get('OSS_ENDPOINT_FOR_LOG'):
         return None
     
     client = OSSClient()

hello_datap_component_base/runner.py

@@ -124,7 +124,9 @@ class ServiceRunner:
         print(f"Starting service: {self.config.name}")
         if self.config.version:
             print(f"Version: {self.config.version}")
-        print(f"Config: {self.config.model_dump_json(indent=2)}")
+        import json
+        params_to_print = self.config.params if self.config.params is not None else {}
+        print(f"Param: {json.dumps(params_to_print, indent=2, ensure_ascii=False)}")
         print("\n" + "=" * 60)
         print("Processing request...")
         print("=" * 60 + "\n")

{hello_datap_component_base-0.2.1.dist-info → hello_datap_component_base-0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hello-datap-component-base
-Version: 0.2.1
+Version: 0.2.3
 Summary: A unified server management framework for data processing component
 Author-email: zhaohaidong <zhaohaidong389@hellobike.com>
 License: MIT

hello_datap_component_base-0.2.3.dist-info/RECORD

@@ -0,0 +1,14 @@
+hello_datap_component_base/__init__.py,sha256=5bGlAUivYRN9_H91I9tYoUrokl6YJJgJGtlM4PkLU_A,811
+hello_datap_component_base/base.py,sha256=jkjoUx9QQ4IqiwR0WZrb2-ZX9KEKF1_fOF_415qwsxc,6436
+hello_datap_component_base/cli.py,sha256=chRlcOTaqHJFdKNPe0OTFYvw_M4c_1_kHBlQ82BwB4g,8161
+hello_datap_component_base/config.py,sha256=XV4OY0iCEjVf0PNxLRdWLgOB5pPB0OvASdkysZXukms,6992
+hello_datap_component_base/discover.py,sha256=70sFO9iVnpsjo_eTViQsStI-n0N_eJNvDRvLvm_dqZQ,7459
+hello_datap_component_base/logger.py,sha256=JIvy_gctDf0Vpe_itSQCwf-ZhVigMdDFwpwbmMlcNNE,10606
+hello_datap_component_base/mns_client.py,sha256=uaWb4P99iRk8vpREqEDqjD9HqaKwlhOYEtHfaW_Y2Pg,12521
+hello_datap_component_base/oss_client.py,sha256=C4Dajeey3JBKh_EgGJzzNMMdpzkm85Z6tpO1kC1wU_s,5057
+hello_datap_component_base/runner.py,sha256=korZY_Qoa-ZzwIFsWK4zaetLF17BE9oT-IRDueoaSbs,11576
+hello_datap_component_base-0.2.3.dist-info/METADATA,sha256=fHLMMFXgwraItePBCFDAHvCUf4bnW8JHE85yAyeiobQ,19865
+hello_datap_component_base-0.2.3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+hello_datap_component_base-0.2.3.dist-info/entry_points.txt,sha256=Q2YteaAVN0UW9MEBfPZ3EY6FI6dRaoCmQZpcvAzmSVQ,74
+hello_datap_component_base-0.2.3.dist-info/top_level.txt,sha256=V4aKCkt0lrJbO4RBPymJ6dz5mXo6vjYy02kusdWKGqM,27
+hello_datap_component_base-0.2.3.dist-info/RECORD,,

{hello_datap_component_base-0.2.1.dist-info → hello_datap_component_base-0.2.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.10.1)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

hello_datap_component_base-0.2.1.dist-info/RECORD

@@ -1,14 +0,0 @@
-hello_datap_component_base/__init__.py,sha256=uEkEyRNAWoC3Lh6TkCvA1hCaQjMghjuJ_nOsGnNVlqU,811
-hello_datap_component_base/base.py,sha256=jkjoUx9QQ4IqiwR0WZrb2-ZX9KEKF1_fOF_415qwsxc,6436
-hello_datap_component_base/cli.py,sha256=FVHMk599_L8u_SMPTxGAiyHUkimD-Wbt6upeMA9229s,8161
-hello_datap_component_base/config.py,sha256=XV4OY0iCEjVf0PNxLRdWLgOB5pPB0OvASdkysZXukms,6992
-hello_datap_component_base/discover.py,sha256=70sFO9iVnpsjo_eTViQsStI-n0N_eJNvDRvLvm_dqZQ,7459
-hello_datap_component_base/logger.py,sha256=JIvy_gctDf0Vpe_itSQCwf-ZhVigMdDFwpwbmMlcNNE,10606
-hello_datap_component_base/mns_client.py,sha256=wEPlU23_O8x1lUkz51knEUHjPzzMnLsmQEVVO_Un9gI,10152
-hello_datap_component_base/oss_client.py,sha256=9DD26Mu9TxX9z0mhLgDKQxqx8VI1yuxUNlYAcLDgKnE,4993
-hello_datap_component_base/runner.py,sha256=eccCY78v69hEWu0fJ4Z9mTWKauG8t5o_gWagafn0qMo,11450
-hello_datap_component_base-0.2.1.dist-info/METADATA,sha256=ruYGekCeTnMcVbpQz8k4mcRtaNeiQnBoJACQsn4r_QY,19865
-hello_datap_component_base-0.2.1.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
-hello_datap_component_base-0.2.1.dist-info/entry_points.txt,sha256=Q2YteaAVN0UW9MEBfPZ3EY6FI6dRaoCmQZpcvAzmSVQ,74
-hello_datap_component_base-0.2.1.dist-info/top_level.txt,sha256=V4aKCkt0lrJbO4RBPymJ6dz5mXo6vjYy02kusdWKGqM,27
-hello_datap_component_base-0.2.1.dist-info/RECORD,,