sycommon-python-lib 0.1.56b13__tar.gz → 0.1.56b15__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/PKG-INFO +2 -1
  2. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/pyproject.toml +2 -1
  3. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/config/Config.py +7 -0
  4. sycommon_python_lib-0.1.56b15/src/sycommon/config/LangfuseConfig.py +15 -0
  5. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/llm/get_llm.py +19 -1
  6. sycommon_python_lib-0.1.56b15/src/sycommon/tools/snowflake.py +253 -0
  7. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon_python_lib.egg-info/PKG-INFO +2 -1
  8. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon_python_lib.egg-info/SOURCES.txt +1 -0
  9. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon_python_lib.egg-info/requires.txt +1 -0
  10. sycommon_python_lib-0.1.56b13/src/sycommon/tools/snowflake.py +0 -300
  11. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/README.md +0 -0
  12. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/setup.cfg +0 -0
  13. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/command/cli.py +0 -0
  14. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/__init__.py +0 -0
  15. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/config/DatabaseConfig.py +0 -0
  16. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/config/EmbeddingConfig.py +0 -0
  17. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/config/LLMConfig.py +0 -0
  18. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/config/MQConfig.py +0 -0
  19. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/config/RerankerConfig.py +0 -0
  20. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/config/SentryConfig.py +0 -0
  21. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/config/__init__.py +0 -0
  22. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/database/async_base_db_service.py +0 -0
  23. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/database/async_database_service.py +0 -0
  24. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/database/base_db_service.py +0 -0
  25. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/database/database_service.py +0 -0
  26. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/health/__init__.py +0 -0
  27. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/health/health_check.py +0 -0
  28. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/health/metrics.py +0 -0
  29. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/health/ping.py +0 -0
  30. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/llm/__init__.py +0 -0
  31. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/llm/embedding.py +0 -0
  32. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/llm/llm_logger.py +0 -0
  33. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/llm/llm_tokens.py +0 -0
  34. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/logging/__init__.py +0 -0
  35. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/logging/async_sql_logger.py +0 -0
  36. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/logging/kafka_log.py +0 -0
  37. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/logging/logger_levels.py +0 -0
  38. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/logging/logger_wrapper.py +0 -0
  39. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/logging/sql_logger.py +0 -0
  40. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/middleware/__init__.py +0 -0
  41. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/middleware/context.py +0 -0
  42. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/middleware/cors.py +0 -0
  43. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/middleware/docs.py +0 -0
  44. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/middleware/exception.py +0 -0
  45. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/middleware/middleware.py +0 -0
  46. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/middleware/monitor_memory.py +0 -0
  47. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/middleware/mq.py +0 -0
  48. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/middleware/timeout.py +0 -0
  49. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/middleware/traceid.py +0 -0
  50. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/models/__init__.py +0 -0
  51. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/models/base_http.py +0 -0
  52. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/models/log.py +0 -0
  53. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/models/mqlistener_config.py +0 -0
  54. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/models/mqmsg_model.py +0 -0
  55. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/models/mqsend_config.py +0 -0
  56. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/models/sso_user.py +0 -0
  57. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/notice/__init__.py +0 -0
  58. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/notice/uvicorn_monitor.py +0 -0
  59. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/rabbitmq/rabbitmq_client.py +0 -0
  60. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/rabbitmq/rabbitmq_pool.py +0 -0
  61. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/rabbitmq/rabbitmq_service.py +0 -0
  62. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/rabbitmq/rabbitmq_service_client_manager.py +0 -0
  63. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/rabbitmq/rabbitmq_service_connection_monitor.py +0 -0
  64. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/rabbitmq/rabbitmq_service_consumer_manager.py +0 -0
  65. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/rabbitmq/rabbitmq_service_core.py +0 -0
  66. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/rabbitmq/rabbitmq_service_producer_manager.py +0 -0
  67. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/sentry/__init__.py +0 -0
  68. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/sentry/sy_sentry.py +0 -0
  69. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/services.py +0 -0
  70. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/sse/__init__.py +0 -0
  71. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/sse/event.py +0 -0
  72. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/sse/sse.py +0 -0
  73. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/__init__.py +0 -0
  74. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/example.py +0 -0
  75. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/example2.py +0 -0
  76. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/feign.py +0 -0
  77. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/feign_client.py +0 -0
  78. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/nacos_client_base.py +0 -0
  79. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/nacos_config_manager.py +0 -0
  80. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/nacos_heartbeat_manager.py +0 -0
  81. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/nacos_service.py +0 -0
  82. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/nacos_service_discovery.py +0 -0
  83. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/nacos_service_registration.py +0 -0
  84. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/synacos/param.py +0 -0
  85. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/tools/__init__.py +0 -0
  86. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/tools/docs.py +0 -0
  87. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/tools/env.py +0 -0
  88. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/tools/merge_headers.py +0 -0
  89. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon/tools/timing.py +0 -0
  90. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon_python_lib.egg-info/dependency_links.txt +0 -0
  91. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon_python_lib.egg-info/entry_points.txt +0 -0
  92. {sycommon_python_lib-0.1.56b13 → sycommon_python_lib-0.1.56b15}/src/sycommon_python_lib.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sycommon-python-lib
3
- Version: 0.1.56b13
3
+ Version: 0.1.56b15
4
4
  Summary: Add your description here
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -13,6 +13,7 @@ Requires-Dist: kafka-python>=2.3.0
13
13
  Requires-Dist: langchain>=1.2.0
14
14
  Requires-Dist: langchain-core>=1.2.6
15
15
  Requires-Dist: langchain-openai>=1.1.6
16
+ Requires-Dist: langfuse>=3.11.2
16
17
  Requires-Dist: langgraph>=1.0.5
17
18
  Requires-Dist: loguru>=0.7.3
18
19
  Requires-Dist: mysql-connector-python>=9.5.0
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sycommon-python-lib"
3
- version = "0.1.56-beta13"
3
+ version = "0.1.56-beta15"
4
4
  description = "Add your description here"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.11"
@@ -14,6 +14,7 @@ dependencies = [
14
14
  "langchain>=1.2.0",
15
15
  "langchain-core>=1.2.6",
16
16
  "langchain-openai>=1.1.6",
17
+ "langfuse>=3.11.2",
17
18
  "langgraph>=1.0.5",
18
19
  "loguru>=0.7.3",
19
20
  "mysql-connector-python>=9.5.0",
@@ -21,6 +21,7 @@ class Config(metaclass=SingletonMeta):
21
21
  self.embedding_configs = []
22
22
  self.reranker_configs = []
23
23
  self.sentry_configs = []
24
+ self.langfuse_configs = []
24
25
  self._process_config()
25
26
 
26
27
  def get_llm_config(self, model_name):
@@ -47,6 +48,12 @@ class Config(metaclass=SingletonMeta):
47
48
  return sentry
48
49
  raise ValueError(f"No configuration found for server: {name}")
49
50
 
51
+ def get_langfuse_config(self, name):
52
+ for langfuse in self.langfuse_configs:
53
+ if langfuse.get('name') == name:
54
+ return langfuse
55
+ raise ValueError(f"No configuration found for server: {name}")
56
+
50
57
  def _process_config(self):
51
58
  llm_config_list = self.config.get('LLMConfig', [])
52
59
  for llm_config in llm_config_list:
@@ -0,0 +1,15 @@
1
+ from pydantic import BaseModel
2
+
3
+
4
+ class LangfuseConfig(BaseModel):
5
+ name: str
6
+ secretKey: str
7
+ publicKey: str
8
+ baseUrl: str
9
+ enable: bool
10
+
11
+ @classmethod
12
+ def from_config(cls, server_name: str):
13
+ from sycommon.config.Config import Config
14
+ langfuse_config = Config().get_langfuse_config(server_name)
15
+ return cls(**langfuse_config)
@@ -1,4 +1,6 @@
1
+ import os
1
2
  from typing import Dict, Type, List, Optional, Callable, Any
3
+ from sycommon.config.Config import Config
2
4
  from sycommon.llm.llm_logger import LLMLogger
3
5
  from langchain_core.language_models import BaseChatModel
4
6
  from langchain_core.runnables import Runnable, RunnableLambda, RunnableConfig
@@ -10,6 +12,7 @@ from pydantic import BaseModel, ValidationError, Field
10
12
  from sycommon.config.LLMConfig import LLMConfig
11
13
  from sycommon.llm.llm_tokens import TokensCallbackHandler
12
14
  from sycommon.logging.kafka_log import SYLogger
15
+ from langfuse.langchain import CallbackHandler
13
16
 
14
17
 
15
18
  class StructuredRunnableWithToken(Runnable):
@@ -230,6 +233,21 @@ def get_llm(
230
233
  if not llmConfig:
231
234
  raise Exception(f"无效的模型配置:{model}")
232
235
 
236
+ callbacks = [LLMLogger()]
237
+
238
+ config = Config().config
239
+ server_name = config.get('Name', '')
240
+ langfuse_configs = config.get('LangfuseConfig', [])
241
+ target_config = next(
242
+ (item for item in langfuse_configs if item.get('name') == server_name), None)
243
+ if target_config and target_config.get('enable', False):
244
+ os.environ["LANGFUSE_SECRET_KEY"] = target_config.get('secretKey', '')
245
+ os.environ["LANGFUSE_PUBLIC_KEY"] = target_config.get('publicKey', '')
246
+ os.environ["LANGFUSE_BASE_URL"] = target_config.get('baseUrl', '')
247
+
248
+ langfuse_handler = CallbackHandler()
249
+ callbacks += [langfuse_handler]
250
+
233
251
  llm = init_chat_model(
234
252
  model_provider=llmConfig.provider,
235
253
  model=llmConfig.model,
@@ -237,7 +255,7 @@ def get_llm(
237
255
  api_key="-",
238
256
  temperature=0.1,
239
257
  streaming=streaming,
240
- callbacks=[LLMLogger()]
258
+ callbacks=callbacks
241
259
  )
242
260
 
243
261
  if llm is None:
@@ -0,0 +1,253 @@
1
+ import time
2
+ import threading
3
+ import socket
4
+ import hashlib
5
+ import random
6
+ import os
7
+ from typing import Optional, Type, Any
8
+ from os import environ
9
+ import psutil
10
+
11
+
12
+ class ClassProperty:
13
+ """支持通过 类.属性 的方式访问,无需实例化"""
14
+
15
+ def __init__(self, func):
16
+ self.func = func
17
+
18
+ def __get__(self, instance: Any, cls: Type) -> str:
19
+ return self.func(cls)
20
+
21
+
22
+ class Snowflake:
23
+ """
24
+ 雪花算法生成器(高并发终极优化版 - 修复语法错误)
25
+
26
+ 终极优化点:
27
+ 1. 解决溢出死锁:当序列号溢出时,释放锁自旋等待,避免阻塞其他线程。
28
+ 2. 锁内原子更新:确保时钟回拨判断与状态更新的原子性。
29
+ 3. 移除 sleep:全流程无休眠,纯自旋保证极致吞吐。
30
+ """
31
+ START_TIMESTAMP = 1388534400000 # 2014-01-01 00:00:00
32
+ SEQUENCE_BITS = 12
33
+ MACHINE_ID_BITS = 10
34
+ MAX_MACHINE_ID = (1 << MACHINE_ID_BITS) - 1 # 0~1023
35
+ MAX_SEQUENCE = (1 << SEQUENCE_BITS) - 1 # 4095
36
+ MACHINE_ID_SHIFT = SEQUENCE_BITS
37
+ TIMESTAMP_SHIFT = SEQUENCE_BITS + MACHINE_ID_BITS
38
+ CLOCK_BACKWARD_THRESHOLD = 5
39
+ _MAX_JAVA_LONG = 9223372036854775807
40
+
41
+ _instance = None
42
+ _instance_lock = threading.Lock()
43
+
44
+ def __init__(self, machine_id: Optional[int] = None):
45
+ self._validate_timestamp_range()
46
+ if machine_id is None:
47
+ machine_id = self._get_k8s_machine_id()
48
+ if not (0 <= machine_id <= self.MAX_MACHINE_ID):
49
+ raise ValueError(f"机器ID必须在0~{self.MAX_MACHINE_ID}之间")
50
+
51
+ self.machine_id = machine_id
52
+ self.last_timestamp = -1
53
+ self.sequence = 0
54
+ self.lock = threading.Lock()
55
+
56
+ def _validate_timestamp_range(self):
57
+ max_support_timestamp = self.START_TIMESTAMP + \
58
+ (1 << (64 - self.TIMESTAMP_SHIFT)) - 1
59
+ current_timestamp = self._get_current_timestamp()
60
+ if current_timestamp > max_support_timestamp:
61
+ raise RuntimeError(f"当前时间戳({current_timestamp})超过支持范围")
62
+
63
+ def _get_k8s_machine_id(self) -> int:
64
+ pod_name = environ.get("POD_NAME")
65
+ if pod_name:
66
+ return self._hash_to_machine_id(pod_name)
67
+ pod_ip = environ.get("POD_IP")
68
+ if pod_ip:
69
+ return self._hash_to_machine_id(pod_ip)
70
+ try:
71
+ local_ip = self._get_local_internal_ip()
72
+ if local_ip:
73
+ return self._hash_to_machine_id(local_ip)
74
+ except Exception:
75
+ pass
76
+ hostname = socket.gethostname()
77
+ if hostname:
78
+ return self._hash_to_machine_id(hostname)
79
+ fallback_text = f"{os.getpid()}_{int(time.time()*1000)}_{random.randint(0, 100000)}"
80
+ return self._hash_to_machine_id(fallback_text)
81
+
82
+ def _get_local_internal_ip(self) -> Optional[str]:
83
+ try:
84
+ net_if_addrs = psutil.net_if_addrs()
85
+ for interface_name, addrs in net_if_addrs.items():
86
+ if (interface_name.lower().startswith("lo")
87
+ or interface_name.lower() in ["loopback", "virtual", "docker", "veth"]):
88
+ continue
89
+ for addr in addrs:
90
+ if addr.family == psutil.AF_INET:
91
+ ip = addr.address
92
+ if ip and not ip.startswith('127.') and not ip.startswith('0.'):
93
+ return ip
94
+ return None
95
+ except Exception:
96
+ return self._get_local_ip_fallback()
97
+
98
+ def _get_local_ip_fallback(self) -> Optional[str]:
99
+ try:
100
+ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
101
+ s.connect(("10.0.0.1", 80))
102
+ local_ip = s.getsockname()[0]
103
+ s.close()
104
+ if not local_ip.startswith('127.') and not local_ip.startswith('0.'):
105
+ return local_ip
106
+ except Exception:
107
+ pass
108
+ try:
109
+ hostname = socket.gethostname()
110
+ ip_list = socket.gethostbyname_ex(hostname)[2]
111
+ for ip in ip_list:
112
+ if not ip.startswith('127.') and not ip.startswith('0.'):
113
+ return ip
114
+ except Exception:
115
+ pass
116
+ return None
117
+
118
+ def _hash_to_machine_id(self, text: str) -> int:
119
+ hash_bytes = hashlib.md5(text.encode("utf-8")).digest()
120
+ return int.from_bytes(hash_bytes[:4], byteorder="big") % (self.MAX_MACHINE_ID + 1)
121
+
122
+ def _get_current_timestamp(self) -> int:
123
+ return int(time.time() * 1000)
124
+
125
+ def generate_id(self) -> int:
126
+ """
127
+ 生成雪花ID(高并发优化版)
128
+ 使用 while True 循环来处理序列号溢出时的重试逻辑
129
+ """
130
+ while True:
131
+ # 1. 快速获取当前时间
132
+ current_timestamp = self._get_current_timestamp()
133
+
134
+ # 2. 加锁,保证状态更新的原子性
135
+ with self.lock:
136
+ # 读取当前状态(快照)
137
+ last_timestamp = self.last_timestamp
138
+ sequence = self.sequence
139
+
140
+ # 2.1 处理时钟回拨
141
+ time_diff = last_timestamp - current_timestamp
142
+ if time_diff > 0:
143
+ if time_diff > self.CLOCK_BACKWARD_THRESHOLD:
144
+ # 大幅回拨:直接“借用”未来1ms
145
+ current_timestamp = last_timestamp + 1
146
+ else:
147
+ # 微小回拨:锁内自旋等待追上(通常很快)
148
+ while current_timestamp <= last_timestamp:
149
+ current_timestamp = self._get_current_timestamp()
150
+
151
+ # 2.2 计算序列号
152
+ if current_timestamp == last_timestamp:
153
+ # 同一毫秒内
154
+ sequence = (sequence + 1) & self.MAX_SEQUENCE
155
+ if sequence == 0:
156
+ # 【关键优化】:序列号溢出!
157
+ # 此时必须等待下一毫秒。为了不阻塞其他线程,我们:
158
+ # 1. 更新 last_timestamp 标记进入新毫秒 (可选,但这里直接 release 更安全)
159
+ # 2. 释放锁
160
+ # 3. 锁外自旋
161
+ # 4. 下一轮循环重新抢锁
162
+
163
+ self.lock.release() # 手动释放锁
164
+
165
+ # 锁外自旋等待下一毫秒
166
+ while current_timestamp <= last_timestamp:
167
+ current_timestamp = self._get_current_timestamp()
168
+
169
+ # 等待到了新毫秒,进入下一轮循环(continue 默认行为),重新抢锁竞争
170
+ continue
171
+
172
+ elif current_timestamp > last_timestamp:
173
+ # 时间推进,重置序列号
174
+ sequence = 0
175
+ self.last_timestamp = current_timestamp
176
+ self.sequence = sequence
177
+ else:
178
+ # current < last 的情况通常已被回拨逻辑处理
179
+ pass
180
+
181
+ # 2.4 生成 ID
182
+ snowflake_id = (
183
+ ((current_timestamp - self.START_TIMESTAMP)
184
+ << self.TIMESTAMP_SHIFT)
185
+ | (self.machine_id << self.MACHINE_ID_SHIFT)
186
+ | sequence
187
+ )
188
+
189
+ # 更新全局状态(如果是同一毫秒)
190
+ if current_timestamp == self.last_timestamp:
191
+ self.sequence = sequence
192
+
193
+ # 成功生成 ID,退出循环
194
+ return snowflake_id
195
+
196
+ @staticmethod
197
+ def parse_id(snowflake_id: int) -> dict:
198
+ from datetime import datetime
199
+ sequence = snowflake_id & Snowflake.MAX_SEQUENCE
200
+ machine_id = (snowflake_id >>
201
+ Snowflake.MACHINE_ID_SHIFT) & Snowflake.MAX_MACHINE_ID
202
+ timestamp = (snowflake_id >> Snowflake.TIMESTAMP_SHIFT) + \
203
+ Snowflake.START_TIMESTAMP
204
+ generate_time = datetime.fromtimestamp(
205
+ timestamp / 1000).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
206
+ return {
207
+ "snowflake_id": snowflake_id,
208
+ "generate_time": generate_time,
209
+ "machine_id": machine_id,
210
+ "sequence": sequence,
211
+ "is_java_long_safe": snowflake_id <= Snowflake._MAX_JAVA_LONG
212
+ }
213
+
214
+ @classmethod
215
+ def next_id(cls) -> str:
216
+ if cls._instance is None:
217
+ with cls._instance_lock:
218
+ if cls._instance is None:
219
+ cls._instance = cls()
220
+ return str(cls._instance.generate_id())
221
+
222
+ @ClassProperty
223
+ def id(cls) -> str:
224
+ return cls.next_id()
225
+
226
+
227
+ if __name__ == "__main__":
228
+ import concurrent.futures
229
+
230
+ print("=== 高并发终极版 Snowflake 性能测试 ===")
231
+ count = 100000
232
+ workers = 100
233
+
234
+ ids = []
235
+ start_time = time.perf_counter()
236
+
237
+ def task():
238
+ return Snowflake.id
239
+
240
+ with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
241
+ futures = [executor.submit(task) for _ in range(count)]
242
+ for future in concurrent.futures.as_completed(futures):
243
+ ids.append(future.result())
244
+
245
+ end_time = time.perf_counter()
246
+ duration = end_time - start_time
247
+
248
+ unique_count = len(set(ids))
249
+ print(f"生成数量: {len(ids)}")
250
+ print(f"唯一ID数量: {unique_count}")
251
+ print(f"是否有重复: {'是 ❌' if unique_count != len(ids) else '否 ✅'}")
252
+ print(f"总耗时: {duration:.4f} 秒")
253
+ print(f"吞吐量 (QPS): {len(ids) / duration:,.2f}")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sycommon-python-lib
3
- Version: 0.1.56b13
3
+ Version: 0.1.56b15
4
4
  Summary: Add your description here
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -13,6 +13,7 @@ Requires-Dist: kafka-python>=2.3.0
13
13
  Requires-Dist: langchain>=1.2.0
14
14
  Requires-Dist: langchain-core>=1.2.6
15
15
  Requires-Dist: langchain-openai>=1.1.6
16
+ Requires-Dist: langfuse>=3.11.2
16
17
  Requires-Dist: langgraph>=1.0.5
17
18
  Requires-Dist: loguru>=0.7.3
18
19
  Requires-Dist: mysql-connector-python>=9.5.0
@@ -7,6 +7,7 @@ src/sycommon/config/Config.py
7
7
  src/sycommon/config/DatabaseConfig.py
8
8
  src/sycommon/config/EmbeddingConfig.py
9
9
  src/sycommon/config/LLMConfig.py
10
+ src/sycommon/config/LangfuseConfig.py
10
11
  src/sycommon/config/MQConfig.py
11
12
  src/sycommon/config/RerankerConfig.py
12
13
  src/sycommon/config/SentryConfig.py
@@ -7,6 +7,7 @@ kafka-python>=2.3.0
7
7
  langchain>=1.2.0
8
8
  langchain-core>=1.2.6
9
9
  langchain-openai>=1.1.6
10
+ langfuse>=3.11.2
10
11
  langgraph>=1.0.5
11
12
  loguru>=0.7.3
12
13
  mysql-connector-python>=9.5.0
@@ -1,300 +0,0 @@
1
- import time
2
- import threading
3
- import socket
4
- import hashlib
5
- import random
6
- import os
7
- from typing import Optional, Type, Any
8
- from os import environ
9
- import psutil
10
-
11
-
12
- class ClassProperty:
13
- """
14
- 自定义类属性描述符,替代 @classmethod + @property 的废弃写法
15
- 支持通过 类.属性 的方式访问,无需实例化
16
- """
17
-
18
- def __init__(self, func):
19
- self.func = func
20
-
21
- def __get__(self, instance: Any, cls: Type) -> str:
22
- # 调用传入的函数,并传入类本身作为第一个参数
23
- return self.func(cls)
24
-
25
-
26
- class Snowflake:
27
- """雪花算法生成器(生产级优化版,无公网依赖,适配内网/K8s环境)"""
28
- # 基础配置(可根据业务调整)
29
- START_TIMESTAMP = 1388534400000 # 2014-01-01 00:00:00
30
- SEQUENCE_BITS = 12
31
- MACHINE_ID_BITS = 10
32
- MAX_MACHINE_ID = (1 << MACHINE_ID_BITS) - 1 # 0~1023
33
- MAX_SEQUENCE = (1 << SEQUENCE_BITS) - 1
34
- MACHINE_ID_SHIFT = SEQUENCE_BITS
35
- TIMESTAMP_SHIFT = SEQUENCE_BITS + MACHINE_ID_BITS
36
- CLOCK_BACKWARD_THRESHOLD = 5 # 容忍的时钟回拨阈值(毫秒)
37
- _MAX_JAVA_LONG = 9223372036854775807 # Java Long最大值
38
-
39
- # 类级别的单例实例(线程安全)
40
- _instance = None
41
- _instance_lock = threading.Lock()
42
-
43
- def __init__(self, machine_id: Optional[int] = None):
44
- """
45
- 初始化:优先使用传入的machine_id,否则自动从K8s环境获取
46
- :param machine_id: 手动指定机器ID(None则自动计算)
47
- """
48
- # 前置校验:确保雪花ID不会超过Java Long最大值
49
- self._validate_timestamp_range()
50
-
51
- # 自动计算K8s环境下的machine_id
52
- if machine_id is None:
53
- machine_id = self._get_k8s_machine_id()
54
-
55
- # 校验machine_id合法性
56
- if not (0 <= machine_id <= self.MAX_MACHINE_ID):
57
- raise ValueError(f"机器ID必须在0~{self.MAX_MACHINE_ID}之间")
58
-
59
- # 初始化核心参数
60
- self.machine_id = machine_id
61
- self.last_timestamp = -1
62
- self.sequence = 0
63
- self.lock = threading.Lock()
64
-
65
- def _validate_timestamp_range(self):
66
- """校验当前时间戳是否在雪花ID支持的范围内,避免超过Java Long最大值"""
67
- max_support_timestamp = self.START_TIMESTAMP + \
68
- (1 << (64 - self.TIMESTAMP_SHIFT)) - 1
69
- current_timestamp = self._get_current_timestamp()
70
- if current_timestamp > max_support_timestamp:
71
- raise RuntimeError(
72
- f"当前时间戳({current_timestamp})超过雪花ID支持的最大时间戳({max_support_timestamp}),"
73
- f"请调整START_TIMESTAMP或减少TIMESTAMP_SHIFT位数"
74
- )
75
-
76
- def _get_k8s_machine_id(self) -> int:
77
- """
78
- 从K8s环境自动计算唯一machine_id(无公网依赖,多层兜底,降低重复风险):
79
- 优先级:POD_NAME > POD_IP > 容器内网IP(psutil读取) > 容器主机名 > 进程+时间+随机数(最终兜底)
80
- """
81
- # 1. 优先读取K8s内置的POD_NAME(默认注入,优先级最高)
82
- pod_name = environ.get("POD_NAME")
83
- if pod_name:
84
- return self._hash_to_machine_id(pod_name)
85
-
86
- # 2. 读取POD_IP(手动配置downwardAPI后必存在)
87
- pod_ip = environ.get("POD_IP")
88
- if pod_ip:
89
- return self._hash_to_machine_id(pod_ip)
90
-
91
- # 3. 兜底1:读取本机网卡获取内网IP(替换netifaces,使用psutil)
92
- try:
93
- local_ip = self._get_local_internal_ip()
94
- if local_ip:
95
- return self._hash_to_machine_id(local_ip)
96
- except Exception:
97
- pass
98
-
99
- # 4. 兜底2:获取容器主机名(K8s中默认等于Pod名称,保证唯一)
100
- hostname = socket.gethostname()
101
- if hostname:
102
- return self._hash_to_machine_id(hostname)
103
-
104
- # 5. 最终兜底:增加熵值(进程ID+毫秒时间戳+随机数),大幅降低重复概率
105
- fallback_text = f"{os.getpid()}_{int(time.time()*1000)}_{random.randint(0, 100000)}"
106
- return self._hash_to_machine_id(fallback_text)
107
-
108
- def _get_local_internal_ip(self) -> Optional[str]:
109
- """
110
- 使用psutil读取本机网卡信息,获取非回环的内网IP(跨平台兼容,过滤lo/lo0等回环网卡)
111
- :return: 内网IP字符串,失败返回None
112
- """
113
- try:
114
- # 遍历所有网卡接口
115
- net_if_addrs = psutil.net_if_addrs()
116
- for interface_name, addrs in net_if_addrs.items():
117
- # 过滤回环/虚拟网卡(兼容lo、lo0、lo1、Loopback、virtual等)
118
- if (interface_name.lower().startswith("lo")
119
- or interface_name.lower() in ["loopback", "virtual"]):
120
- continue
121
- # 遍历该网卡的所有地址,优先返回第一个非回环IPv4
122
- for addr in addrs:
123
- if addr.family == psutil.AF_INET:
124
- ip = addr.address
125
- if ip and not ip.startswith('127.'):
126
- return ip
127
- return None
128
- except Exception:
129
- # psutil调用失败,降级到纯内置方法
130
- return self._get_local_ip_fallback()
131
-
132
- def _get_local_ip_fallback(self) -> Optional[str]:
133
- """
134
- 增强版降级方案:纯Python内置方法,多维度获取内网IP(无第三方依赖)
135
- """
136
- # 方案1:socket绑定内网地址(避免访问公网)
137
- try:
138
- s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
139
- s.connect(("192.168.0.1", 80))
140
- local_ip = s.getsockname()[0]
141
- s.close()
142
- if not local_ip.startswith('127.'):
143
- return local_ip
144
- except Exception:
145
- pass
146
-
147
- # 方案2:遍历所有本地IP(通过hostname解析)
148
- try:
149
- hostname = socket.gethostname()
150
- ip_list = socket.gethostbyname_ex(hostname)[2]
151
- for ip in ip_list:
152
- if not ip.startswith('127.'):
153
- return ip
154
- except Exception:
155
- pass
156
-
157
- return None
158
-
159
- def _hash_to_machine_id(self, text: str) -> int:
160
- """将字符串哈希后取模,得到0~1023的machine_id(保证分布均匀)"""
161
- hash_bytes = hashlib.md5(text.encode("utf-8")).digest()
162
- hash_int = int.from_bytes(hash_bytes[:4], byteorder="big")
163
- return hash_int % self.MAX_MACHINE_ID
164
-
165
- def _get_current_timestamp(self) -> int:
166
- """获取当前毫秒级时间戳"""
167
- return int(time.time() * 1000)
168
-
169
- def _wait_next_millisecond(self, current_timestamp: int) -> int:
170
- """等待直到下一个毫秒,避免序列耗尽"""
171
- while current_timestamp <= self.last_timestamp:
172
- current_timestamp = self._get_current_timestamp()
173
- return current_timestamp
174
-
175
- def generate_id(self) -> int:
176
- """生成雪花ID(生产级优化:优化锁粒度,容忍轻微时钟回拨)"""
177
- current_timestamp = self._get_current_timestamp()
178
-
179
- # 1. 处理时钟回拨:容忍CLOCK_BACKWARD_THRESHOLD内的微调,超过则抛异常
180
- time_diff = self.last_timestamp - current_timestamp
181
- if time_diff > 0:
182
- if time_diff > self.CLOCK_BACKWARD_THRESHOLD:
183
- raise RuntimeError(
184
- f"时钟回拨检测:当前时间戳({current_timestamp}) < 上一次时间戳({self.last_timestamp}),"
185
- f"差值{time_diff}ms(阈值{self.CLOCK_BACKWARD_THRESHOLD}ms)"
186
- )
187
- # 轻微回拨:等待时钟追上
188
- current_timestamp = self._wait_next_millisecond(current_timestamp)
189
-
190
- # 2. 优化锁粒度:仅在同一毫秒内递增序列时加锁
191
- if current_timestamp != self.last_timestamp:
192
- with self.lock:
193
- self.last_timestamp = current_timestamp
194
- self.sequence = 0
195
- else:
196
- with self.lock:
197
- self.sequence = (self.sequence + 1) & self.MAX_SEQUENCE
198
- if self.sequence == 0:
199
- current_timestamp = self._wait_next_millisecond(
200
- current_timestamp)
201
- self.last_timestamp = current_timestamp
202
-
203
- # 3. 计算最终雪花ID
204
- snowflake_id = (
205
- ((current_timestamp - self.START_TIMESTAMP) << self.TIMESTAMP_SHIFT)
206
- | (self.machine_id << self.MACHINE_ID_SHIFT)
207
- | self.sequence
208
- )
209
-
210
- # 最终校验:确保不超过Java Long最大值
211
- if snowflake_id > self._MAX_JAVA_LONG:
212
- raise RuntimeError(
213
- f"生成的雪花ID({snowflake_id})超过Java Long最大值({self._MAX_JAVA_LONG})")
214
-
215
- return snowflake_id
216
-
217
- @staticmethod
218
- def parse_id(snowflake_id: int) -> dict:
219
- """解析雪花ID,返回生成时间、机器ID、序列等信息"""
220
- from datetime import datetime
221
- sequence = snowflake_id & Snowflake.MAX_SEQUENCE
222
- machine_id = (snowflake_id >>
223
- Snowflake.MACHINE_ID_SHIFT) & Snowflake.MAX_MACHINE_ID
224
- timestamp = (snowflake_id >> Snowflake.TIMESTAMP_SHIFT) + \
225
- Snowflake.START_TIMESTAMP
226
- generate_time = datetime.fromtimestamp(
227
- timestamp / 1000).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
228
-
229
- return {
230
- "snowflake_id": snowflake_id,
231
- "generate_time": generate_time,
232
- "machine_id": machine_id,
233
- "sequence": sequence,
234
- "is_java_long_safe": snowflake_id <= Snowflake._MAX_JAVA_LONG
235
- }
236
-
237
- @classmethod
238
- def next_id(cls) -> str:
239
- """
240
- 生成雪花ID(线程安全单例模式,避免重复创建实例,锁内完成所有初始化)
241
- :return: 雪花ID字符串
242
- """
243
- if cls._instance is None:
244
- with cls._instance_lock:
245
- if cls._instance is None:
246
- # 锁内初始化,避免多线程重复计算machine_id
247
- cls._instance = cls()
248
- return str(cls._instance.generate_id())
249
-
250
- @ClassProperty
251
- def id(cls) -> str:
252
- """
253
- 直接通过 `Snowflake.id` 属性生成雪花ID(兼容Python 3.11+)
254
- :return: 雪花ID字符串
255
- """
256
- return cls.next_id()
257
-
258
-
259
- if __name__ == "__main__":
260
- print("=== 生产级雪花算法ID生成测试 ===")
261
- # 1. 基础生成测试
262
- id1 = Snowflake.id
263
- id2 = Snowflake.id
264
- id3 = Snowflake.id
265
- print(f"生成ID1: {id1}")
266
- print(f"生成ID2: {id2}")
267
- print(f"生成ID3: {id3}")
268
- print(f"ID是否唯一: {len({id1, id2, id3}) == 3}")
269
-
270
- # 2. 解析ID信息
271
- print("\n=== 雪花ID解析 ===")
272
- parse_info = Snowflake.parse_id(int(id3))
273
- for key, value in parse_info.items():
274
- print(f"{key}: {value}")
275
-
276
- # 3. 批量唯一性验证(10000个ID)
277
- print("\n=== 批量唯一性验证(10000个)===")
278
- id_set = set()
279
- duplicate_count = 0
280
- for i in range(10000):
281
- snow_id = Snowflake.id
282
- if snow_id in id_set:
283
- duplicate_count += 1
284
- id_set.add(snow_id)
285
- print(f"总生成数量: 10000")
286
- print(f"唯一ID数量: {len(id_set)}")
287
- print(f"重复ID数量: {duplicate_count}")
288
- print(f"机器ID: {Snowflake._instance.machine_id}")
289
-
290
- # 4. 高并发测试
291
- import concurrent.futures
292
- print("\n=== 高并发测试(100线程)===")
293
- id_set_concurrent = set()
294
- with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
295
- futures = [executor.submit(lambda: Snowflake.id) for _ in range(10000)]
296
- for future in concurrent.futures.as_completed(futures):
297
- id_set_concurrent.add(future.result())
298
- print(f"高并发生成唯一ID数量: {len(id_set_concurrent)}")
299
-
300
- print("\n=== 生产级雪花算法验证通过 ===")