sycommon-python-lib 0.2.5a5__tar.gz → 0.2.5a6__tar.gz

{sycommon_python_lib-0.2.5a5 → sycommon_python_lib-0.2.5a6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.5a5
+Version: 0.2.5a6
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.5a5 → sycommon_python_lib-0.2.5a6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sycommon-python-lib"
-version = "0.2.5a5"
+version = "0.2.5a6"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.11"

{sycommon_python_lib-0.2.5a5 → sycommon_python_lib-0.2.5a6}/src/sycommon/llm/struct_token.py

@@ -10,6 +10,7 @@ from sycommon.llm.llm_logger import LLMLogger
 from langfuse import Langfuse, LangfuseSpan, propagate_attributes
 from typing import Dict, List, Optional, Any
 import tiktoken
+import threading
 import warnings
 warnings.filterwarnings("ignore", message="Pydantic serializer warnings")
 
@@ -48,18 +49,29 @@ class StructuredRunnableWithToken(Runnable):
         self.max_compression_attempts = max_compression_attempts  # 最大压缩尝试次数
         self.is_native_mode = is_native_mode  # 是否为原生模式
 
-        # 初始化 Tokenizer
-        try:
-            self.encoding = tiktoken.encoding_for_model(model_name)
-        except KeyError:
+        # 初始化 Tokenizer（5秒超时保护）
+        self.encoding = None
+
+        def _init_encoding():
+            nonlocal encoding_result
             try:
-                self.encoding = tiktoken.get_encoding("cl100k_base")
+                encoding_result = tiktoken.encoding_for_model(model_name)
+            except KeyError:
+                try:
+                    encoding_result = tiktoken.get_encoding("cl100k_base")
+                except Exception as e:
+                    SYLogger.error(f"tiktoken get_encoding 失败: {e}")
             except Exception as e:
-                SYLogger.error(f"tiktoken 初始化失败，将使用字符数近似估算: {e}")
-                self.encoding = None
-        except Exception as e:
-            SYLogger.error(f"tiktoken 初始化失败，将使用字符数近似估算: {e}")
-            self.encoding = None
+                SYLogger.error(f"tiktoken encoding_for_model 失败: {e}")
+
+        encoding_result = None
+        t = threading.Thread(target=_init_encoding, daemon=True)
+        t.start()
+        t.join(timeout=5)
+        if t.is_alive():
+            SYLogger.error("tiktoken 初始化超时(5s)，将使用字符数近似估算")
+        elif encoding_result is not None:
+            self.encoding = encoding_result
 
     def _encode_text(self, text: str) -> int:
         """计算文本的 token 数量，encoding 不可用时按字符数近似估算"""

{sycommon_python_lib-0.2.5a5 → sycommon_python_lib-0.2.5a6}/src/sycommon_python_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.5a5
+Version: 0.2.5a6
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown