sycommon-python-lib 0.1.56b14__tar.gz → 0.1.56b16__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/PKG-INFO +1 -1
  2. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/pyproject.toml +1 -1
  3. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/llm/get_llm.py +87 -43
  4. sycommon_python_lib-0.1.56b16/src/sycommon/tools/snowflake.py +248 -0
  5. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon_python_lib.egg-info/PKG-INFO +1 -1
  6. sycommon_python_lib-0.1.56b14/src/sycommon/tools/snowflake.py +0 -300
  7. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/README.md +0 -0
  8. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/setup.cfg +0 -0
  9. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/command/cli.py +0 -0
  10. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/__init__.py +0 -0
  11. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/config/Config.py +0 -0
  12. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/config/DatabaseConfig.py +0 -0
  13. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/config/EmbeddingConfig.py +0 -0
  14. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/config/LLMConfig.py +0 -0
  15. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/config/LangfuseConfig.py +0 -0
  16. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/config/MQConfig.py +0 -0
  17. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/config/RerankerConfig.py +0 -0
  18. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/config/SentryConfig.py +0 -0
  19. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/config/__init__.py +0 -0
  20. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/database/async_base_db_service.py +0 -0
  21. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/database/async_database_service.py +0 -0
  22. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/database/base_db_service.py +0 -0
  23. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/database/database_service.py +0 -0
  24. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/health/__init__.py +0 -0
  25. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/health/health_check.py +0 -0
  26. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/health/metrics.py +0 -0
  27. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/health/ping.py +0 -0
  28. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/llm/__init__.py +0 -0
  29. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/llm/embedding.py +0 -0
  30. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/llm/llm_logger.py +0 -0
  31. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/llm/llm_tokens.py +0 -0
  32. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/logging/__init__.py +0 -0
  33. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/logging/async_sql_logger.py +0 -0
  34. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/logging/kafka_log.py +0 -0
  35. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/logging/logger_levels.py +0 -0
  36. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/logging/logger_wrapper.py +0 -0
  37. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/logging/sql_logger.py +0 -0
  38. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/middleware/__init__.py +0 -0
  39. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/middleware/context.py +0 -0
  40. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/middleware/cors.py +0 -0
  41. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/middleware/docs.py +0 -0
  42. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/middleware/exception.py +0 -0
  43. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/middleware/middleware.py +0 -0
  44. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/middleware/monitor_memory.py +0 -0
  45. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/middleware/mq.py +0 -0
  46. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/middleware/timeout.py +0 -0
  47. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/middleware/traceid.py +0 -0
  48. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/models/__init__.py +0 -0
  49. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/models/base_http.py +0 -0
  50. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/models/log.py +0 -0
  51. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/models/mqlistener_config.py +0 -0
  52. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/models/mqmsg_model.py +0 -0
  53. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/models/mqsend_config.py +0 -0
  54. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/models/sso_user.py +0 -0
  55. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/notice/__init__.py +0 -0
  56. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/notice/uvicorn_monitor.py +0 -0
  57. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/rabbitmq/rabbitmq_client.py +0 -0
  58. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/rabbitmq/rabbitmq_pool.py +0 -0
  59. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/rabbitmq/rabbitmq_service.py +0 -0
  60. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/rabbitmq/rabbitmq_service_client_manager.py +0 -0
  61. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/rabbitmq/rabbitmq_service_connection_monitor.py +0 -0
  62. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/rabbitmq/rabbitmq_service_consumer_manager.py +0 -0
  63. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/rabbitmq/rabbitmq_service_core.py +0 -0
  64. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/rabbitmq/rabbitmq_service_producer_manager.py +0 -0
  65. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/sentry/__init__.py +0 -0
  66. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/sentry/sy_sentry.py +0 -0
  67. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/services.py +0 -0
  68. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/sse/__init__.py +0 -0
  69. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/sse/event.py +0 -0
  70. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/sse/sse.py +0 -0
  71. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/__init__.py +0 -0
  72. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/example.py +0 -0
  73. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/example2.py +0 -0
  74. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/feign.py +0 -0
  75. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/feign_client.py +0 -0
  76. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/nacos_client_base.py +0 -0
  77. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/nacos_config_manager.py +0 -0
  78. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/nacos_heartbeat_manager.py +0 -0
  79. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/nacos_service.py +0 -0
  80. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/nacos_service_discovery.py +0 -0
  81. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/nacos_service_registration.py +0 -0
  82. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/synacos/param.py +0 -0
  83. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/tools/__init__.py +0 -0
  84. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/tools/docs.py +0 -0
  85. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/tools/env.py +0 -0
  86. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/tools/merge_headers.py +0 -0
  87. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon/tools/timing.py +0 -0
  88. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon_python_lib.egg-info/SOURCES.txt +0 -0
  89. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon_python_lib.egg-info/dependency_links.txt +0 -0
  90. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon_python_lib.egg-info/entry_points.txt +0 -0
  91. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon_python_lib.egg-info/requires.txt +0 -0
  92. {sycommon_python_lib-0.1.56b14 → sycommon_python_lib-0.1.56b16}/src/sycommon_python_lib.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sycommon-python-lib
3
- Version: 0.1.56b14
3
+ Version: 0.1.56b16
4
4
  Summary: Add your description here
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sycommon-python-lib"
3
- version = "0.1.56-beta14"
3
+ version = "0.1.56-beta16"
4
4
  description = "Add your description here"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.11"
@@ -1,5 +1,6 @@
1
1
  import os
2
- from typing import Dict, Type, List, Optional, Callable, Any
2
+ from typing import Dict, Tuple, Type, List, Optional, Callable, Any
3
+ from langfuse import Langfuse, get_client, propagate_attributes
3
4
  from sycommon.config.Config import Config
4
5
  from sycommon.llm.llm_logger import LLMLogger
5
6
  from langchain_core.language_models import BaseChatModel
@@ -12,15 +13,18 @@ from pydantic import BaseModel, ValidationError, Field
12
13
  from sycommon.config.LLMConfig import LLMConfig
13
14
  from sycommon.llm.llm_tokens import TokensCallbackHandler
14
15
  from sycommon.logging.kafka_log import SYLogger
16
+ from sycommon.tools.env import get_env_var
15
17
  from langfuse.langchain import CallbackHandler
16
18
 
17
19
 
18
20
  class StructuredRunnableWithToken(Runnable):
19
21
  """带Token统计的Runnable类"""
20
22
 
21
- def __init__(self, retry_chain: Runnable):
23
+ def __init__(self, retry_chain: Runnable, langfuse: Langfuse):
22
24
  super().__init__()
23
25
  self.retry_chain = retry_chain
26
+ self.langfuse = langfuse
27
+ self.metadata = {"langfuse_session_id": SYLogger.get_trace_id()}
24
28
 
25
29
  def _adapt_input(self, input: Any) -> List[BaseMessage]:
26
30
  """适配输入格式"""
@@ -72,21 +76,28 @@ class StructuredRunnableWithToken(Runnable):
72
76
  # 同步调用
73
77
  def invoke(self, input: Any, config: Optional[RunnableConfig] = None, ** kwargs) -> Dict[str, Any]:
74
78
  try:
75
- processed_config, token_handler = self._get_callback_config(
76
- config)
77
- adapted_input = self._adapt_input(input)
79
+ with self.langfuse.start_as_current_observation(as_type="span", name="invoke") as span:
80
+ with propagate_attributes(session_id=SYLogger.get_trace_id(), user_id=get_env_var('VERSION')):
81
+ processed_config, token_handler = self._get_callback_config(
82
+ config)
83
+ adapted_input = self._adapt_input(input)
78
84
 
79
- structured_result = self.retry_chain.invoke(
80
- {"messages": adapted_input},
81
- config=processed_config,
82
- **kwargs
83
- )
85
+ span.update_trace(
86
+ input={"messages": adapted_input}
87
+ )
84
88
 
85
- # 获取Token统计结果
86
- token_usage = token_handler.usage_metadata
87
- structured_result._token_usage_ = token_usage
89
+ structured_result = self.retry_chain.invoke(
90
+ {"messages": adapted_input},
91
+ config={**processed_config, **self.metadata},
92
+ **kwargs
93
+ )
88
94
 
89
- return structured_result
95
+ span.update_trace(output=structured_result)
96
+
97
+ token_usage = token_handler.usage_metadata
98
+ structured_result._token_usage_ = token_usage
99
+
100
+ return structured_result
90
101
 
91
102
  except Exception as e:
92
103
  SYLogger.error(f"同步LLM调用失败: {str(e)}", exc_info=True)
@@ -95,20 +106,28 @@ class StructuredRunnableWithToken(Runnable):
95
106
  # 异步调用
96
107
  async def ainvoke(self, input: Any, config: Optional[RunnableConfig] = None, ** kwargs) -> Dict[str, Any]:
97
108
  try:
98
- processed_config, token_handler = self._get_callback_config(
99
- config)
100
- adapted_input = self._adapt_input(input)
109
+ with self.langfuse.start_as_current_observation(as_type="span", name="ainvoke") as span:
110
+ with propagate_attributes(session_id=SYLogger.get_trace_id(), user_id=get_env_var('VERSION')):
111
+ processed_config, token_handler = self._get_callback_config(
112
+ config)
113
+ adapted_input = self._adapt_input(input)
114
+
115
+ span.update_trace(
116
+ input={"messages": adapted_input}
117
+ )
118
+
119
+ structured_result = await self.retry_chain.ainvoke(
120
+ {"messages": adapted_input},
121
+ config={**processed_config, **self.metadata},
122
+ **kwargs
123
+ )
101
124
 
102
- structured_result = await self.retry_chain.ainvoke(
103
- {"messages": adapted_input},
104
- config=processed_config,
105
- **kwargs
106
- )
125
+ span.update_trace(output=structured_result)
107
126
 
108
- token_usage = token_handler.usage_metadata
109
- structured_result._token_usage_ = token_usage
127
+ token_usage = token_handler.usage_metadata
128
+ structured_result._token_usage_ = token_usage
110
129
 
111
- return structured_result
130
+ return structured_result
112
131
 
113
132
  except Exception as e:
114
133
  SYLogger.error(f"异步LLM调用失败: {str(e)}", exc_info=True)
@@ -118,9 +137,10 @@ class StructuredRunnableWithToken(Runnable):
118
137
  class LLMWithAutoTokenUsage(BaseChatModel):
119
138
  """自动为结构化调用返回token_usage的LLM包装类"""
120
139
  llm: BaseChatModel = Field(default=None)
140
+ langfuse: Optional[Langfuse] = Field(default=None, exclude=True)
121
141
 
122
- def __init__(self, llm: BaseChatModel, **kwargs):
123
- super().__init__(llm=llm, ** kwargs)
142
+ def __init__(self, llm: BaseChatModel, langfuse: Langfuse, **kwargs):
143
+ super().__init__(llm=llm, langfuse=langfuse, **kwargs)
124
144
 
125
145
  def with_structured_output(
126
146
  self,
@@ -211,7 +231,7 @@ class LLMWithAutoTokenUsage(BaseChatModel):
211
231
  "initial": 0.1, "max": 3.0, "exp_base": 2.0, "jitter": 1.0}
212
232
  )
213
233
 
214
- return StructuredRunnableWithToken(retry_chain)
234
+ return StructuredRunnableWithToken(retry_chain, self.langfuse)
215
235
 
216
236
  # ========== 实现BaseChatModel抽象方法 ==========
217
237
  def _generate(self, messages, stop=None, run_manager=None, ** kwargs):
@@ -222,6 +242,43 @@ class LLMWithAutoTokenUsage(BaseChatModel):
222
242
  return self.llm._llm_type
223
243
 
224
244
 
245
+ def _init_langfuse() -> Tuple[List[CallbackHandler], Optional[Langfuse]]:
246
+ """
247
+ 初始化 Langfuse 组件的辅助函数
248
+ """
249
+ callbacks = []
250
+ langfuse = None
251
+
252
+ # 基础日志回调
253
+ callbacks.append(LLMLogger())
254
+ config_dict = Config().config
255
+
256
+ server_name = config_dict.get('Name', '')
257
+ langfuse_configs = config_dict.get('LangfuseConfig', [])
258
+ environment = config_dict.get('Nacos', {}).get('namespaceId', '')
259
+
260
+ # 查找当前服务对应的 Langfuse 配置
261
+ target_config = next(
262
+ (item for item in langfuse_configs if item.get('name') == server_name), None
263
+ )
264
+
265
+ if target_config and target_config.get('enable', False):
266
+ # 设置环境变量
267
+ os.environ["LANGFUSE_SECRET_KEY"] = target_config.get('secretKey', '')
268
+ os.environ["LANGFUSE_PUBLIC_KEY"] = target_config.get('publicKey', '')
269
+ os.environ["LANGFUSE_BASE_URL"] = target_config.get('baseUrl', '')
270
+ os.environ["LANGFUSE_TRACING_ENVIRONMENT"] = environment
271
+
272
+ # 创建 Langfuse Handler 和 Client
273
+ langfuse_handler = CallbackHandler(
274
+ trace_context={"trace_id": SYLogger.get_trace_id()}
275
+ )
276
+ callbacks.append(langfuse_handler)
277
+ langfuse = get_client()
278
+
279
+ return callbacks, langfuse
280
+
281
+
225
282
  def get_llm(
226
283
  model: str = None,
227
284
  streaming: bool = False
@@ -233,20 +290,7 @@ def get_llm(
233
290
  if not llmConfig:
234
291
  raise Exception(f"无效的模型配置:{model}")
235
292
 
236
- callbacks = [LLMLogger()]
237
-
238
- config = Config().config
239
- server_name = config.get('Name', '')
240
- langfuse_configs = config.get('LangfuseConfig', [])
241
- target_config = next(
242
- (item for item in langfuse_configs if item.get('name') == server_name), None)
243
- if target_config and target_config.get('enable', False):
244
- os.environ["LANGFUSE_SECRET_KEY"] = target_config.get('secretKey', '')
245
- os.environ["LANGFUSE_PUBLIC_KEY"] = target_config.get('publicKey', '')
246
- os.environ["LANGFUSE_BASE_URL"] = target_config.get('baseUrl', '')
247
-
248
- langfuse_handler = CallbackHandler()
249
- callbacks += [langfuse_handler]
293
+ callbacks, langfuse = _init_langfuse()
250
294
 
251
295
  llm = init_chat_model(
252
296
  model_provider=llmConfig.provider,
@@ -261,4 +305,4 @@ def get_llm(
261
305
  if llm is None:
262
306
  raise Exception(f"初始化原始LLM实例失败:{model}")
263
307
 
264
- return LLMWithAutoTokenUsage(llm)
308
+ return LLMWithAutoTokenUsage(llm, langfuse)
@@ -0,0 +1,248 @@
1
+ import time
2
+ import threading
3
+ import socket
4
+ import hashlib
5
+ import random
6
+ import os
7
+ from typing import Optional, Type, Any
8
+ from os import environ
9
+ import psutil
10
+
11
+
12
+ class ClassProperty:
13
+ """支持通过 类.属性 的方式访问,无需实例化"""
14
+
15
+ def __init__(self, func):
16
+ self.func = func
17
+
18
+ def __get__(self, instance: Any, cls: Type) -> str:
19
+ return self.func(cls)
20
+
21
+
22
+ class Snowflake:
23
+ """
24
+ 雪花算法生成器(高并发终极优化版 - 修复语法错误)
25
+
26
+ 终极优化点:
27
+ 1. 解决溢出死锁:当序列号溢出时,释放锁自旋等待,避免阻塞其他线程。
28
+ 2. 锁内原子更新:确保时钟回拨判断与状态更新的原子性。
29
+ 3. 移除 sleep:全流程无休眠,纯自旋保证极致吞吐。
30
+ """
31
+ START_TIMESTAMP = 1388534400000 # 2014-01-01 00:00:00
32
+ SEQUENCE_BITS = 12
33
+ MACHINE_ID_BITS = 10
34
+ MAX_MACHINE_ID = (1 << MACHINE_ID_BITS) - 1 # 0~1023
35
+ MAX_SEQUENCE = (1 << SEQUENCE_BITS) - 1 # 4095
36
+ MACHINE_ID_SHIFT = SEQUENCE_BITS
37
+ TIMESTAMP_SHIFT = SEQUENCE_BITS + MACHINE_ID_BITS
38
+ CLOCK_BACKWARD_THRESHOLD = 5
39
+ _MAX_JAVA_LONG = 9223372036854775807
40
+
41
+ _instance = None
42
+ _instance_lock = threading.Lock()
43
+
44
+ def __init__(self, machine_id: Optional[int] = None):
45
+ self._validate_timestamp_range()
46
+ if machine_id is None:
47
+ machine_id = self._get_k8s_machine_id()
48
+ if not (0 <= machine_id <= self.MAX_MACHINE_ID):
49
+ raise ValueError(f"机器ID必须在0~{self.MAX_MACHINE_ID}之间")
50
+
51
+ self.machine_id = machine_id
52
+ self.last_timestamp = -1
53
+ self.sequence = 0
54
+ self.lock = threading.Lock()
55
+
56
+ def _validate_timestamp_range(self):
57
+ max_support_timestamp = self.START_TIMESTAMP + \
58
+ (1 << (64 - self.TIMESTAMP_SHIFT)) - 1
59
+ current_timestamp = self._get_current_timestamp()
60
+ if current_timestamp > max_support_timestamp:
61
+ raise RuntimeError(f"当前时间戳({current_timestamp})超过支持范围")
62
+
63
+ def _get_k8s_machine_id(self) -> int:
64
+ pod_name = environ.get("POD_NAME")
65
+ if pod_name:
66
+ return self._hash_to_machine_id(pod_name)
67
+ pod_ip = environ.get("POD_IP")
68
+ if pod_ip:
69
+ return self._hash_to_machine_id(pod_ip)
70
+ try:
71
+ local_ip = self._get_local_internal_ip()
72
+ if local_ip:
73
+ return self._hash_to_machine_id(local_ip)
74
+ except Exception:
75
+ pass
76
+ hostname = socket.gethostname()
77
+ if hostname:
78
+ return self._hash_to_machine_id(hostname)
79
+ fallback_text = f"{os.getpid()}_{int(time.time()*1000)}_{random.randint(0, 100000)}"
80
+ return self._hash_to_machine_id(fallback_text)
81
+
82
+ def _get_local_internal_ip(self) -> Optional[str]:
83
+ try:
84
+ net_if_addrs = psutil.net_if_addrs()
85
+ for interface_name, addrs in net_if_addrs.items():
86
+ if (interface_name.lower().startswith("lo")
87
+ or interface_name.lower() in ["loopback", "virtual", "docker", "veth"]):
88
+ continue
89
+ for addr in addrs:
90
+ if addr.family == psutil.AF_INET:
91
+ ip = addr.address
92
+ if ip and not ip.startswith('127.') and not ip.startswith('0.'):
93
+ return ip
94
+ return None
95
+ except Exception:
96
+ return self._get_local_ip_fallback()
97
+
98
+ def _get_local_ip_fallback(self) -> Optional[str]:
99
+ try:
100
+ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
101
+ s.connect(("10.0.0.1", 80))
102
+ local_ip = s.getsockname()[0]
103
+ s.close()
104
+ if not local_ip.startswith('127.') and not local_ip.startswith('0.'):
105
+ return local_ip
106
+ except Exception:
107
+ pass
108
+ try:
109
+ hostname = socket.gethostname()
110
+ ip_list = socket.gethostbyname_ex(hostname)[2]
111
+ for ip in ip_list:
112
+ if not ip.startswith('127.') and not ip.startswith('0.'):
113
+ return ip
114
+ except Exception:
115
+ pass
116
+ return None
117
+
118
+ def _hash_to_machine_id(self, text: str) -> int:
119
+ hash_bytes = hashlib.md5(text.encode("utf-8")).digest()
120
+ return int.from_bytes(hash_bytes[:4], byteorder="big") % (self.MAX_MACHINE_ID + 1)
121
+
122
+ def _get_current_timestamp(self) -> int:
123
+ return int(time.time() * 1000)
124
+
125
+ def generate_id(self) -> int:
126
+ """
127
+ 生成雪花ID(高并发优化版)
128
+ 使用 while True 循环来处理序列号溢出时的重试逻辑
129
+ """
130
+ while True:
131
+ # 1. 快速获取当前时间
132
+ current_timestamp = self._get_current_timestamp()
133
+
134
+ # 2. 加锁,保证状态更新的原子性
135
+ with self.lock:
136
+ # 读取当前状态(快照)
137
+ last_timestamp = self.last_timestamp
138
+ sequence = self.sequence
139
+
140
+ # 2.1 处理时钟回拨
141
+ time_diff = last_timestamp - current_timestamp
142
+ if time_diff > 0:
143
+ if time_diff > self.CLOCK_BACKWARD_THRESHOLD:
144
+ # 大幅回拨:直接“借用”未来1ms
145
+ current_timestamp = last_timestamp + 1
146
+ else:
147
+ # 微小回拨:锁内自旋等待追上(通常很快)
148
+ while current_timestamp <= last_timestamp:
149
+ current_timestamp = self._get_current_timestamp()
150
+
151
+ # 2.2 计算序列号
152
+ if current_timestamp == last_timestamp:
153
+ # 同一毫秒内
154
+ sequence = (sequence + 1) & self.MAX_SEQUENCE
155
+ if sequence == 0:
156
+ self.lock.release() # 手动释放锁
157
+
158
+ # 锁外自旋等待下一毫秒
159
+ while current_timestamp <= last_timestamp:
160
+ current_timestamp = self._get_current_timestamp()
161
+
162
+ # 等待到了新毫秒,进入下一轮循环(continue 默认行为),重新抢锁竞争
163
+ continue
164
+
165
+ elif current_timestamp > last_timestamp:
166
+ # 时间推进,重置序列号
167
+ sequence = 0
168
+ self.last_timestamp = current_timestamp
169
+ self.sequence = sequence
170
+ else:
171
+ # current < last 的情况通常已被回拨逻辑处理
172
+ pass
173
+
174
+ # 2.4 生成 ID
175
+ snowflake_id = (
176
+ ((current_timestamp - self.START_TIMESTAMP)
177
+ << self.TIMESTAMP_SHIFT)
178
+ | (self.machine_id << self.MACHINE_ID_SHIFT)
179
+ | sequence
180
+ )
181
+
182
+ # 更新全局状态(如果是同一毫秒)
183
+ if current_timestamp == self.last_timestamp:
184
+ self.sequence = sequence
185
+
186
+ # 成功生成 ID,退出循环
187
+ return snowflake_id
188
+
189
+ @staticmethod
190
+ def parse_id(snowflake_id: int) -> dict:
191
+ from datetime import datetime
192
+ sequence = snowflake_id & Snowflake.MAX_SEQUENCE
193
+ machine_id = (snowflake_id >>
194
+ Snowflake.MACHINE_ID_SHIFT) & Snowflake.MAX_MACHINE_ID
195
+ timestamp = (snowflake_id >> Snowflake.TIMESTAMP_SHIFT) + \
196
+ Snowflake.START_TIMESTAMP
197
+ generate_time = datetime.fromtimestamp(
198
+ timestamp / 1000).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
199
+ return {
200
+ "snowflake_id": snowflake_id,
201
+ "generate_time": generate_time,
202
+ "machine_id": machine_id,
203
+ "sequence": sequence,
204
+ "is_java_long_safe": snowflake_id <= Snowflake._MAX_JAVA_LONG
205
+ }
206
+
207
+ @classmethod
208
+ def next_id(cls) -> str:
209
+ if cls._instance is None:
210
+ with cls._instance_lock:
211
+ if cls._instance is None:
212
+ cls._instance = cls()
213
+ return str(cls._instance.generate_id())
214
+
215
+ @ClassProperty
216
+ def id(cls) -> str:
217
+ return cls.next_id()
218
+
219
+
220
+ if __name__ == "__main__":
221
+ import concurrent.futures
222
+
223
+ print("=== 高并发终极版 Snowflake 性能测试 ===")
224
+ count = 100000
225
+ workers = 100
226
+
227
+ ids = []
228
+ start_time = time.perf_counter()
229
+
230
+ def task():
231
+ return Snowflake.id
232
+
233
+ with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
234
+ futures = [executor.submit(task) for _ in range(count)]
235
+ for future in concurrent.futures.as_completed(futures):
236
+ ids.append(future.result())
237
+
238
+ end_time = time.perf_counter()
239
+ duration = end_time - start_time
240
+
241
+ unique_count = len(set(ids))
242
+ print(f"生成数量: {len(ids)}")
243
+ print(f"唯一ID数量: {unique_count}")
244
+ print(f"是否有重复: {'是 ❌' if unique_count != len(ids) else '否 ✅'}")
245
+ print(f"总耗时: {duration:.4f} 秒")
246
+ print(f"吞吐量 (QPS): {len(ids) / duration:,.2f}")
247
+ print("\n最后一个ID解析:")
248
+ print(Snowflake.parse_id(int(ids[-1])))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sycommon-python-lib
3
- Version: 0.1.56b14
3
+ Version: 0.1.56b16
4
4
  Summary: Add your description here
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -1,300 +0,0 @@
1
- import time
2
- import threading
3
- import socket
4
- import hashlib
5
- import random
6
- import os
7
- from typing import Optional, Type, Any
8
- from os import environ
9
- import psutil
10
-
11
-
12
- class ClassProperty:
13
- """
14
- 自定义类属性描述符,替代 @classmethod + @property 的废弃写法
15
- 支持通过 类.属性 的方式访问,无需实例化
16
- """
17
-
18
- def __init__(self, func):
19
- self.func = func
20
-
21
- def __get__(self, instance: Any, cls: Type) -> str:
22
- # 调用传入的函数,并传入类本身作为第一个参数
23
- return self.func(cls)
24
-
25
-
26
- class Snowflake:
27
- """雪花算法生成器(生产级优化版,无公网依赖,适配内网/K8s环境)"""
28
- # 基础配置(可根据业务调整)
29
- START_TIMESTAMP = 1388534400000 # 2014-01-01 00:00:00
30
- SEQUENCE_BITS = 12
31
- MACHINE_ID_BITS = 10
32
- MAX_MACHINE_ID = (1 << MACHINE_ID_BITS) - 1 # 0~1023
33
- MAX_SEQUENCE = (1 << SEQUENCE_BITS) - 1
34
- MACHINE_ID_SHIFT = SEQUENCE_BITS
35
- TIMESTAMP_SHIFT = SEQUENCE_BITS + MACHINE_ID_BITS
36
- CLOCK_BACKWARD_THRESHOLD = 5 # 容忍的时钟回拨阈值(毫秒)
37
- _MAX_JAVA_LONG = 9223372036854775807 # Java Long最大值
38
-
39
- # 类级别的单例实例(线程安全)
40
- _instance = None
41
- _instance_lock = threading.Lock()
42
-
43
- def __init__(self, machine_id: Optional[int] = None):
44
- """
45
- 初始化:优先使用传入的machine_id,否则自动从K8s环境获取
46
- :param machine_id: 手动指定机器ID(None则自动计算)
47
- """
48
- # 前置校验:确保雪花ID不会超过Java Long最大值
49
- self._validate_timestamp_range()
50
-
51
- # 自动计算K8s环境下的machine_id
52
- if machine_id is None:
53
- machine_id = self._get_k8s_machine_id()
54
-
55
- # 校验machine_id合法性
56
- if not (0 <= machine_id <= self.MAX_MACHINE_ID):
57
- raise ValueError(f"机器ID必须在0~{self.MAX_MACHINE_ID}之间")
58
-
59
- # 初始化核心参数
60
- self.machine_id = machine_id
61
- self.last_timestamp = -1
62
- self.sequence = 0
63
- self.lock = threading.Lock()
64
-
65
- def _validate_timestamp_range(self):
66
- """校验当前时间戳是否在雪花ID支持的范围内,避免超过Java Long最大值"""
67
- max_support_timestamp = self.START_TIMESTAMP + \
68
- (1 << (64 - self.TIMESTAMP_SHIFT)) - 1
69
- current_timestamp = self._get_current_timestamp()
70
- if current_timestamp > max_support_timestamp:
71
- raise RuntimeError(
72
- f"当前时间戳({current_timestamp})超过雪花ID支持的最大时间戳({max_support_timestamp}),"
73
- f"请调整START_TIMESTAMP或减少TIMESTAMP_SHIFT位数"
74
- )
75
-
76
- def _get_k8s_machine_id(self) -> int:
77
- """
78
- 从K8s环境自动计算唯一machine_id(无公网依赖,多层兜底,降低重复风险):
79
- 优先级:POD_NAME > POD_IP > 容器内网IP(psutil读取) > 容器主机名 > 进程+时间+随机数(最终兜底)
80
- """
81
- # 1. 优先读取K8s内置的POD_NAME(默认注入,优先级最高)
82
- pod_name = environ.get("POD_NAME")
83
- if pod_name:
84
- return self._hash_to_machine_id(pod_name)
85
-
86
- # 2. 读取POD_IP(手动配置downwardAPI后必存在)
87
- pod_ip = environ.get("POD_IP")
88
- if pod_ip:
89
- return self._hash_to_machine_id(pod_ip)
90
-
91
- # 3. 兜底1:读取本机网卡获取内网IP(替换netifaces,使用psutil)
92
- try:
93
- local_ip = self._get_local_internal_ip()
94
- if local_ip:
95
- return self._hash_to_machine_id(local_ip)
96
- except Exception:
97
- pass
98
-
99
- # 4. 兜底2:获取容器主机名(K8s中默认等于Pod名称,保证唯一)
100
- hostname = socket.gethostname()
101
- if hostname:
102
- return self._hash_to_machine_id(hostname)
103
-
104
- # 5. 最终兜底:增加熵值(进程ID+毫秒时间戳+随机数),大幅降低重复概率
105
- fallback_text = f"{os.getpid()}_{int(time.time()*1000)}_{random.randint(0, 100000)}"
106
- return self._hash_to_machine_id(fallback_text)
107
-
108
- def _get_local_internal_ip(self) -> Optional[str]:
109
- """
110
- 使用psutil读取本机网卡信息,获取非回环的内网IP(跨平台兼容,过滤lo/lo0等回环网卡)
111
- :return: 内网IP字符串,失败返回None
112
- """
113
- try:
114
- # 遍历所有网卡接口
115
- net_if_addrs = psutil.net_if_addrs()
116
- for interface_name, addrs in net_if_addrs.items():
117
- # 过滤回环/虚拟网卡(兼容lo、lo0、lo1、Loopback、virtual等)
118
- if (interface_name.lower().startswith("lo")
119
- or interface_name.lower() in ["loopback", "virtual"]):
120
- continue
121
- # 遍历该网卡的所有地址,优先返回第一个非回环IPv4
122
- for addr in addrs:
123
- if addr.family == psutil.AF_INET:
124
- ip = addr.address
125
- if ip and not ip.startswith('127.'):
126
- return ip
127
- return None
128
- except Exception:
129
- # psutil调用失败,降级到纯内置方法
130
- return self._get_local_ip_fallback()
131
-
132
- def _get_local_ip_fallback(self) -> Optional[str]:
133
- """
134
- 增强版降级方案:纯Python内置方法,多维度获取内网IP(无第三方依赖)
135
- """
136
- # 方案1:socket绑定内网地址(避免访问公网)
137
- try:
138
- s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
139
- s.connect(("192.168.0.1", 80))
140
- local_ip = s.getsockname()[0]
141
- s.close()
142
- if not local_ip.startswith('127.'):
143
- return local_ip
144
- except Exception:
145
- pass
146
-
147
- # 方案2:遍历所有本地IP(通过hostname解析)
148
- try:
149
- hostname = socket.gethostname()
150
- ip_list = socket.gethostbyname_ex(hostname)[2]
151
- for ip in ip_list:
152
- if not ip.startswith('127.'):
153
- return ip
154
- except Exception:
155
- pass
156
-
157
- return None
158
-
159
- def _hash_to_machine_id(self, text: str) -> int:
160
- """将字符串哈希后取模,得到0~1023的machine_id(保证分布均匀)"""
161
- hash_bytes = hashlib.md5(text.encode("utf-8")).digest()
162
- hash_int = int.from_bytes(hash_bytes[:4], byteorder="big")
163
- return hash_int % self.MAX_MACHINE_ID
164
-
165
- def _get_current_timestamp(self) -> int:
166
- """获取当前毫秒级时间戳"""
167
- return int(time.time() * 1000)
168
-
169
- def _wait_next_millisecond(self, current_timestamp: int) -> int:
170
- """等待直到下一个毫秒,避免序列耗尽"""
171
- while current_timestamp <= self.last_timestamp:
172
- current_timestamp = self._get_current_timestamp()
173
- return current_timestamp
174
-
175
- def generate_id(self) -> int:
176
- """生成雪花ID(生产级优化:优化锁粒度,容忍轻微时钟回拨)"""
177
- current_timestamp = self._get_current_timestamp()
178
-
179
- # 1. 处理时钟回拨:容忍CLOCK_BACKWARD_THRESHOLD内的微调,超过则抛异常
180
- time_diff = self.last_timestamp - current_timestamp
181
- if time_diff > 0:
182
- if time_diff > self.CLOCK_BACKWARD_THRESHOLD:
183
- raise RuntimeError(
184
- f"时钟回拨检测:当前时间戳({current_timestamp}) < 上一次时间戳({self.last_timestamp}),"
185
- f"差值{time_diff}ms(阈值{self.CLOCK_BACKWARD_THRESHOLD}ms)"
186
- )
187
- # 轻微回拨:等待时钟追上
188
- current_timestamp = self._wait_next_millisecond(current_timestamp)
189
-
190
- # 2. 优化锁粒度:仅在同一毫秒内递增序列时加锁
191
- if current_timestamp != self.last_timestamp:
192
- with self.lock:
193
- self.last_timestamp = current_timestamp
194
- self.sequence = 0
195
- else:
196
- with self.lock:
197
- self.sequence = (self.sequence + 1) & self.MAX_SEQUENCE
198
- if self.sequence == 0:
199
- current_timestamp = self._wait_next_millisecond(
200
- current_timestamp)
201
- self.last_timestamp = current_timestamp
202
-
203
- # 3. 计算最终雪花ID
204
- snowflake_id = (
205
- ((current_timestamp - self.START_TIMESTAMP) << self.TIMESTAMP_SHIFT)
206
- | (self.machine_id << self.MACHINE_ID_SHIFT)
207
- | self.sequence
208
- )
209
-
210
- # 最终校验:确保不超过Java Long最大值
211
- if snowflake_id > self._MAX_JAVA_LONG:
212
- raise RuntimeError(
213
- f"生成的雪花ID({snowflake_id})超过Java Long最大值({self._MAX_JAVA_LONG})")
214
-
215
- return snowflake_id
216
-
217
- @staticmethod
218
- def parse_id(snowflake_id: int) -> dict:
219
- """解析雪花ID,返回生成时间、机器ID、序列等信息"""
220
- from datetime import datetime
221
- sequence = snowflake_id & Snowflake.MAX_SEQUENCE
222
- machine_id = (snowflake_id >>
223
- Snowflake.MACHINE_ID_SHIFT) & Snowflake.MAX_MACHINE_ID
224
- timestamp = (snowflake_id >> Snowflake.TIMESTAMP_SHIFT) + \
225
- Snowflake.START_TIMESTAMP
226
- generate_time = datetime.fromtimestamp(
227
- timestamp / 1000).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
228
-
229
- return {
230
- "snowflake_id": snowflake_id,
231
- "generate_time": generate_time,
232
- "machine_id": machine_id,
233
- "sequence": sequence,
234
- "is_java_long_safe": snowflake_id <= Snowflake._MAX_JAVA_LONG
235
- }
236
-
237
- @classmethod
238
- def next_id(cls) -> str:
239
- """
240
- 生成雪花ID(线程安全单例模式,避免重复创建实例,锁内完成所有初始化)
241
- :return: 雪花ID字符串
242
- """
243
- if cls._instance is None:
244
- with cls._instance_lock:
245
- if cls._instance is None:
246
- # 锁内初始化,避免多线程重复计算machine_id
247
- cls._instance = cls()
248
- return str(cls._instance.generate_id())
249
-
250
- @ClassProperty
251
- def id(cls) -> str:
252
- """
253
- 直接通过 `Snowflake.id` 属性生成雪花ID(兼容Python 3.11+)
254
- :return: 雪花ID字符串
255
- """
256
- return cls.next_id()
257
-
258
-
259
- if __name__ == "__main__":
260
- print("=== 生产级雪花算法ID生成测试 ===")
261
- # 1. 基础生成测试
262
- id1 = Snowflake.id
263
- id2 = Snowflake.id
264
- id3 = Snowflake.id
265
- print(f"生成ID1: {id1}")
266
- print(f"生成ID2: {id2}")
267
- print(f"生成ID3: {id3}")
268
- print(f"ID是否唯一: {len({id1, id2, id3}) == 3}")
269
-
270
- # 2. 解析ID信息
271
- print("\n=== 雪花ID解析 ===")
272
- parse_info = Snowflake.parse_id(int(id3))
273
- for key, value in parse_info.items():
274
- print(f"{key}: {value}")
275
-
276
- # 3. 批量唯一性验证(10000个ID)
277
- print("\n=== 批量唯一性验证(10000个)===")
278
- id_set = set()
279
- duplicate_count = 0
280
- for i in range(10000):
281
- snow_id = Snowflake.id
282
- if snow_id in id_set:
283
- duplicate_count += 1
284
- id_set.add(snow_id)
285
- print(f"总生成数量: 10000")
286
- print(f"唯一ID数量: {len(id_set)}")
287
- print(f"重复ID数量: {duplicate_count}")
288
- print(f"机器ID: {Snowflake._instance.machine_id}")
289
-
290
- # 4. 高并发测试
291
- import concurrent.futures
292
- print("\n=== 高并发测试(100线程)===")
293
- id_set_concurrent = set()
294
- with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
295
- futures = [executor.submit(lambda: Snowflake.id) for _ in range(10000)]
296
- for future in concurrent.futures.as_completed(futures):
297
- id_set_concurrent.add(future.result())
298
- print(f"高并发生成唯一ID数量: {len(id_set_concurrent)}")
299
-
300
- print("\n=== 生产级雪花算法验证通过 ===")