sycommon-python-lib 0.1.56__py3-none-any.whl → 0.1.56b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sycommon/config/Config.py +3 -24
- sycommon/llm/embedding.py +23 -78
- sycommon/llm/get_llm.py +164 -24
- sycommon/logging/kafka_log.py +433 -187
- sycommon/middleware/exception.py +16 -10
- sycommon/middleware/timeout.py +1 -2
- sycommon/middleware/traceid.py +76 -81
- sycommon/rabbitmq/rabbitmq_client.py +242 -232
- sycommon/rabbitmq/rabbitmq_pool.py +218 -278
- sycommon/rabbitmq/rabbitmq_service.py +843 -25
- sycommon/services.py +96 -122
- sycommon/synacos/nacos_service.py +779 -63
- sycommon/tools/merge_headers.py +0 -20
- sycommon/tools/snowflake.py +153 -101
- {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/METADATA +8 -10
- {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/RECORD +19 -40
- sycommon/config/LangfuseConfig.py +0 -15
- sycommon/config/SentryConfig.py +0 -13
- sycommon/llm/llm_tokens.py +0 -119
- sycommon/llm/struct_token.py +0 -192
- sycommon/llm/sy_langfuse.py +0 -103
- sycommon/llm/usage_token.py +0 -117
- sycommon/notice/__init__.py +0 -0
- sycommon/notice/uvicorn_monitor.py +0 -200
- sycommon/rabbitmq/rabbitmq_service_client_manager.py +0 -206
- sycommon/rabbitmq/rabbitmq_service_connection_monitor.py +0 -73
- sycommon/rabbitmq/rabbitmq_service_consumer_manager.py +0 -285
- sycommon/rabbitmq/rabbitmq_service_core.py +0 -117
- sycommon/rabbitmq/rabbitmq_service_producer_manager.py +0 -238
- sycommon/sentry/__init__.py +0 -0
- sycommon/sentry/sy_sentry.py +0 -35
- sycommon/synacos/nacos_client_base.py +0 -119
- sycommon/synacos/nacos_config_manager.py +0 -107
- sycommon/synacos/nacos_heartbeat_manager.py +0 -144
- sycommon/synacos/nacos_service_discovery.py +0 -157
- sycommon/synacos/nacos_service_registration.py +0 -270
- sycommon/tools/env.py +0 -62
- {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/WHEEL +0 -0
- {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/entry_points.txt +0 -0
- {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/top_level.txt +0 -0
sycommon/tools/merge_headers.py
CHANGED
|
@@ -95,23 +95,3 @@ def merge_headers(
|
|
|
95
95
|
processed_headers[key_lower] = value
|
|
96
96
|
|
|
97
97
|
return processed_headers
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
def get_header_value(headers: list, target_key: str, default=None):
|
|
101
|
-
"""
|
|
102
|
-
从列表中查找指定 header 的值
|
|
103
|
-
|
|
104
|
-
Args:
|
|
105
|
-
headers: header 列表,例如 [('Content-Type', 'application/json'), ...]
|
|
106
|
-
target_key: 要查找的 key
|
|
107
|
-
default: 如果没找到返回的默认值
|
|
108
|
-
"""
|
|
109
|
-
if not headers:
|
|
110
|
-
return default
|
|
111
|
-
|
|
112
|
-
for item in headers:
|
|
113
|
-
# 兼容 list 和 tuple,确保长度为2
|
|
114
|
-
if isinstance(item, (list, tuple)) and len(item) == 2 and item[0] == target_key:
|
|
115
|
-
return item[1]
|
|
116
|
-
|
|
117
|
-
return default
|
sycommon/tools/snowflake.py
CHANGED
|
@@ -10,184 +10,213 @@ import psutil
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class ClassProperty:
|
|
13
|
-
"""
|
|
13
|
+
"""
|
|
14
|
+
自定义类属性描述符,替代 @classmethod + @property 的废弃写法
|
|
15
|
+
支持通过 类.属性 的方式访问,无需实例化
|
|
16
|
+
"""
|
|
14
17
|
|
|
15
18
|
def __init__(self, func):
|
|
16
19
|
self.func = func
|
|
17
20
|
|
|
18
21
|
def __get__(self, instance: Any, cls: Type) -> str:
|
|
22
|
+
# 调用传入的函数,并传入类本身作为第一个参数
|
|
19
23
|
return self.func(cls)
|
|
20
24
|
|
|
21
25
|
|
|
22
26
|
class Snowflake:
|
|
23
|
-
"""
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
终极优化点:
|
|
27
|
-
1. 解决溢出死锁:当序列号溢出时,释放锁自旋等待,避免阻塞其他线程。
|
|
28
|
-
2. 锁内原子更新:确保时钟回拨判断与状态更新的原子性。
|
|
29
|
-
3. 移除 sleep:全流程无休眠,纯自旋保证极致吞吐。
|
|
30
|
-
"""
|
|
27
|
+
"""雪花算法生成器(生产级优化版,无公网依赖,适配内网/K8s环境)"""
|
|
28
|
+
# 基础配置(可根据业务调整)
|
|
31
29
|
START_TIMESTAMP = 1388534400000 # 2014-01-01 00:00:00
|
|
32
30
|
SEQUENCE_BITS = 12
|
|
33
31
|
MACHINE_ID_BITS = 10
|
|
34
32
|
MAX_MACHINE_ID = (1 << MACHINE_ID_BITS) - 1 # 0~1023
|
|
35
|
-
MAX_SEQUENCE = (1 << SEQUENCE_BITS) - 1
|
|
33
|
+
MAX_SEQUENCE = (1 << SEQUENCE_BITS) - 1
|
|
36
34
|
MACHINE_ID_SHIFT = SEQUENCE_BITS
|
|
37
35
|
TIMESTAMP_SHIFT = SEQUENCE_BITS + MACHINE_ID_BITS
|
|
38
|
-
CLOCK_BACKWARD_THRESHOLD = 5
|
|
39
|
-
_MAX_JAVA_LONG = 9223372036854775807
|
|
36
|
+
CLOCK_BACKWARD_THRESHOLD = 5 # 容忍的时钟回拨阈值(毫秒)
|
|
37
|
+
_MAX_JAVA_LONG = 9223372036854775807 # Java Long最大值
|
|
40
38
|
|
|
39
|
+
# 类级别的单例实例(线程安全)
|
|
41
40
|
_instance = None
|
|
42
41
|
_instance_lock = threading.Lock()
|
|
43
42
|
|
|
44
43
|
def __init__(self, machine_id: Optional[int] = None):
|
|
44
|
+
"""
|
|
45
|
+
初始化:优先使用传入的machine_id,否则自动从K8s环境获取
|
|
46
|
+
:param machine_id: 手动指定机器ID(None则自动计算)
|
|
47
|
+
"""
|
|
48
|
+
# 前置校验:确保雪花ID不会超过Java Long最大值
|
|
45
49
|
self._validate_timestamp_range()
|
|
50
|
+
|
|
51
|
+
# 自动计算K8s环境下的machine_id
|
|
46
52
|
if machine_id is None:
|
|
47
53
|
machine_id = self._get_k8s_machine_id()
|
|
54
|
+
|
|
55
|
+
# 校验machine_id合法性
|
|
48
56
|
if not (0 <= machine_id <= self.MAX_MACHINE_ID):
|
|
49
57
|
raise ValueError(f"机器ID必须在0~{self.MAX_MACHINE_ID}之间")
|
|
50
58
|
|
|
59
|
+
# 初始化核心参数
|
|
51
60
|
self.machine_id = machine_id
|
|
52
61
|
self.last_timestamp = -1
|
|
53
62
|
self.sequence = 0
|
|
54
63
|
self.lock = threading.Lock()
|
|
55
64
|
|
|
56
65
|
def _validate_timestamp_range(self):
|
|
66
|
+
"""校验当前时间戳是否在雪花ID支持的范围内,避免超过Java Long最大值"""
|
|
57
67
|
max_support_timestamp = self.START_TIMESTAMP + \
|
|
58
68
|
(1 << (64 - self.TIMESTAMP_SHIFT)) - 1
|
|
59
69
|
current_timestamp = self._get_current_timestamp()
|
|
60
70
|
if current_timestamp > max_support_timestamp:
|
|
61
|
-
raise RuntimeError(
|
|
71
|
+
raise RuntimeError(
|
|
72
|
+
f"当前时间戳({current_timestamp})超过雪花ID支持的最大时间戳({max_support_timestamp}),"
|
|
73
|
+
f"请调整START_TIMESTAMP或减少TIMESTAMP_SHIFT位数"
|
|
74
|
+
)
|
|
62
75
|
|
|
63
76
|
def _get_k8s_machine_id(self) -> int:
|
|
77
|
+
"""
|
|
78
|
+
从K8s环境自动计算唯一machine_id(无公网依赖,多层兜底,降低重复风险):
|
|
79
|
+
优先级:POD_NAME > POD_IP > 容器内网IP(psutil读取) > 容器主机名 > 进程+时间+随机数(最终兜底)
|
|
80
|
+
"""
|
|
81
|
+
# 1. 优先读取K8s内置的POD_NAME(默认注入,优先级最高)
|
|
64
82
|
pod_name = environ.get("POD_NAME")
|
|
65
83
|
if pod_name:
|
|
66
84
|
return self._hash_to_machine_id(pod_name)
|
|
85
|
+
|
|
86
|
+
# 2. 读取POD_IP(手动配置downwardAPI后必存在)
|
|
67
87
|
pod_ip = environ.get("POD_IP")
|
|
68
88
|
if pod_ip:
|
|
69
89
|
return self._hash_to_machine_id(pod_ip)
|
|
90
|
+
|
|
91
|
+
# 3. 兜底1:读取本机网卡获取内网IP(替换netifaces,使用psutil)
|
|
70
92
|
try:
|
|
71
93
|
local_ip = self._get_local_internal_ip()
|
|
72
94
|
if local_ip:
|
|
73
95
|
return self._hash_to_machine_id(local_ip)
|
|
74
96
|
except Exception:
|
|
75
97
|
pass
|
|
98
|
+
|
|
99
|
+
# 4. 兜底2:获取容器主机名(K8s中默认等于Pod名称,保证唯一)
|
|
76
100
|
hostname = socket.gethostname()
|
|
77
101
|
if hostname:
|
|
78
102
|
return self._hash_to_machine_id(hostname)
|
|
103
|
+
|
|
104
|
+
# 5. 最终兜底:增加熵值(进程ID+毫秒时间戳+随机数),大幅降低重复概率
|
|
79
105
|
fallback_text = f"{os.getpid()}_{int(time.time()*1000)}_{random.randint(0, 100000)}"
|
|
80
106
|
return self._hash_to_machine_id(fallback_text)
|
|
81
107
|
|
|
82
108
|
def _get_local_internal_ip(self) -> Optional[str]:
|
|
109
|
+
"""
|
|
110
|
+
使用psutil读取本机网卡信息,获取非回环的内网IP(跨平台兼容,过滤lo/lo0等回环网卡)
|
|
111
|
+
:return: 内网IP字符串,失败返回None
|
|
112
|
+
"""
|
|
83
113
|
try:
|
|
114
|
+
# 遍历所有网卡接口
|
|
84
115
|
net_if_addrs = psutil.net_if_addrs()
|
|
85
116
|
for interface_name, addrs in net_if_addrs.items():
|
|
117
|
+
# 过滤回环/虚拟网卡(兼容lo、lo0、lo1、Loopback、virtual等)
|
|
86
118
|
if (interface_name.lower().startswith("lo")
|
|
87
|
-
or interface_name.lower() in ["loopback", "virtual"
|
|
119
|
+
or interface_name.lower() in ["loopback", "virtual"]):
|
|
88
120
|
continue
|
|
121
|
+
# 遍历该网卡的所有地址,优先返回第一个非回环IPv4
|
|
89
122
|
for addr in addrs:
|
|
90
123
|
if addr.family == psutil.AF_INET:
|
|
91
124
|
ip = addr.address
|
|
92
|
-
if ip and not ip.startswith('127.')
|
|
125
|
+
if ip and not ip.startswith('127.'):
|
|
93
126
|
return ip
|
|
94
127
|
return None
|
|
95
128
|
except Exception:
|
|
129
|
+
# psutil调用失败,降级到纯内置方法
|
|
96
130
|
return self._get_local_ip_fallback()
|
|
97
131
|
|
|
98
132
|
def _get_local_ip_fallback(self) -> Optional[str]:
|
|
133
|
+
"""
|
|
134
|
+
增强版降级方案:纯Python内置方法,多维度获取内网IP(无第三方依赖)
|
|
135
|
+
"""
|
|
136
|
+
# 方案1:socket绑定内网地址(避免访问公网)
|
|
99
137
|
try:
|
|
100
138
|
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
101
|
-
s.connect(("
|
|
139
|
+
s.connect(("192.168.0.1", 80))
|
|
102
140
|
local_ip = s.getsockname()[0]
|
|
103
141
|
s.close()
|
|
104
|
-
if not local_ip.startswith('127.')
|
|
142
|
+
if not local_ip.startswith('127.'):
|
|
105
143
|
return local_ip
|
|
106
144
|
except Exception:
|
|
107
145
|
pass
|
|
146
|
+
|
|
147
|
+
# 方案2:遍历所有本地IP(通过hostname解析)
|
|
108
148
|
try:
|
|
109
149
|
hostname = socket.gethostname()
|
|
110
150
|
ip_list = socket.gethostbyname_ex(hostname)[2]
|
|
111
151
|
for ip in ip_list:
|
|
112
|
-
if not ip.startswith('127.')
|
|
152
|
+
if not ip.startswith('127.'):
|
|
113
153
|
return ip
|
|
114
154
|
except Exception:
|
|
115
155
|
pass
|
|
156
|
+
|
|
116
157
|
return None
|
|
117
158
|
|
|
118
159
|
def _hash_to_machine_id(self, text: str) -> int:
|
|
160
|
+
"""将字符串哈希后取模,得到0~1023的machine_id(保证分布均匀)"""
|
|
119
161
|
hash_bytes = hashlib.md5(text.encode("utf-8")).digest()
|
|
120
|
-
|
|
162
|
+
hash_int = int.from_bytes(hash_bytes[:4], byteorder="big")
|
|
163
|
+
return hash_int % self.MAX_MACHINE_ID
|
|
121
164
|
|
|
122
165
|
def _get_current_timestamp(self) -> int:
|
|
166
|
+
"""获取当前毫秒级时间戳"""
|
|
123
167
|
return int(time.time() * 1000)
|
|
124
168
|
|
|
125
|
-
def
|
|
126
|
-
"""
|
|
127
|
-
|
|
128
|
-
使用 while True 循环来处理序列号溢出时的重试逻辑
|
|
129
|
-
"""
|
|
130
|
-
while True:
|
|
131
|
-
# 1. 快速获取当前时间
|
|
169
|
+
def _wait_next_millisecond(self, current_timestamp: int) -> int:
|
|
170
|
+
"""等待直到下一个毫秒,避免序列耗尽"""
|
|
171
|
+
while current_timestamp <= self.last_timestamp:
|
|
132
172
|
current_timestamp = self._get_current_timestamp()
|
|
173
|
+
return current_timestamp
|
|
174
|
+
|
|
175
|
+
def generate_id(self) -> int:
|
|
176
|
+
"""生成雪花ID(生产级优化:优化锁粒度,容忍轻微时钟回拨)"""
|
|
177
|
+
current_timestamp = self._get_current_timestamp()
|
|
133
178
|
|
|
134
|
-
|
|
179
|
+
# 1. 处理时钟回拨:容忍CLOCK_BACKWARD_THRESHOLD内的微调,超过则抛异常
|
|
180
|
+
time_diff = self.last_timestamp - current_timestamp
|
|
181
|
+
if time_diff > 0:
|
|
182
|
+
if time_diff > self.CLOCK_BACKWARD_THRESHOLD:
|
|
183
|
+
raise RuntimeError(
|
|
184
|
+
f"时钟回拨检测:当前时间戳({current_timestamp}) < 上一次时间戳({self.last_timestamp}),"
|
|
185
|
+
f"差值{time_diff}ms(阈值{self.CLOCK_BACKWARD_THRESHOLD}ms)"
|
|
186
|
+
)
|
|
187
|
+
# 轻微回拨:等待时钟追上
|
|
188
|
+
current_timestamp = self._wait_next_millisecond(current_timestamp)
|
|
189
|
+
|
|
190
|
+
# 2. 优化锁粒度:仅在同一毫秒内递增序列时加锁
|
|
191
|
+
if current_timestamp != self.last_timestamp:
|
|
192
|
+
with self.lock:
|
|
193
|
+
self.last_timestamp = current_timestamp
|
|
194
|
+
self.sequence = 0
|
|
195
|
+
else:
|
|
135
196
|
with self.lock:
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
# 2.1 处理时钟回拨
|
|
141
|
-
time_diff = last_timestamp - current_timestamp
|
|
142
|
-
if time_diff > 0:
|
|
143
|
-
if time_diff > self.CLOCK_BACKWARD_THRESHOLD:
|
|
144
|
-
# 大幅回拨:直接“借用”未来1ms
|
|
145
|
-
current_timestamp = last_timestamp + 1
|
|
146
|
-
else:
|
|
147
|
-
# 微小回拨:锁内自旋等待追上(通常很快)
|
|
148
|
-
while current_timestamp <= last_timestamp:
|
|
149
|
-
current_timestamp = self._get_current_timestamp()
|
|
150
|
-
|
|
151
|
-
# 2.2 计算序列号
|
|
152
|
-
if current_timestamp == last_timestamp:
|
|
153
|
-
# 同一毫秒内
|
|
154
|
-
sequence = (sequence + 1) & self.MAX_SEQUENCE
|
|
155
|
-
if sequence == 0:
|
|
156
|
-
self.lock.release() # 手动释放锁
|
|
157
|
-
|
|
158
|
-
# 锁外自旋等待下一毫秒
|
|
159
|
-
while current_timestamp <= last_timestamp:
|
|
160
|
-
current_timestamp = self._get_current_timestamp()
|
|
161
|
-
|
|
162
|
-
# 等待到了新毫秒,进入下一轮循环(continue 默认行为),重新抢锁竞争
|
|
163
|
-
continue
|
|
164
|
-
|
|
165
|
-
elif current_timestamp > last_timestamp:
|
|
166
|
-
# 时间推进,重置序列号
|
|
167
|
-
sequence = 0
|
|
197
|
+
self.sequence = (self.sequence + 1) & self.MAX_SEQUENCE
|
|
198
|
+
if self.sequence == 0:
|
|
199
|
+
current_timestamp = self._wait_next_millisecond(
|
|
200
|
+
current_timestamp)
|
|
168
201
|
self.last_timestamp = current_timestamp
|
|
169
|
-
self.sequence = sequence
|
|
170
|
-
else:
|
|
171
|
-
# current < last 的情况通常已被回拨逻辑处理
|
|
172
|
-
pass
|
|
173
|
-
|
|
174
|
-
# 2.4 生成 ID
|
|
175
|
-
snowflake_id = (
|
|
176
|
-
((current_timestamp - self.START_TIMESTAMP)
|
|
177
|
-
<< self.TIMESTAMP_SHIFT)
|
|
178
|
-
| (self.machine_id << self.MACHINE_ID_SHIFT)
|
|
179
|
-
| sequence
|
|
180
|
-
)
|
|
181
202
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
203
|
+
# 3. 计算最终雪花ID
|
|
204
|
+
snowflake_id = (
|
|
205
|
+
((current_timestamp - self.START_TIMESTAMP) << self.TIMESTAMP_SHIFT)
|
|
206
|
+
| (self.machine_id << self.MACHINE_ID_SHIFT)
|
|
207
|
+
| self.sequence
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
# 最终校验:确保不超过Java Long最大值
|
|
211
|
+
if snowflake_id > self._MAX_JAVA_LONG:
|
|
212
|
+
raise RuntimeError(
|
|
213
|
+
f"生成的雪花ID({snowflake_id})超过Java Long最大值({self._MAX_JAVA_LONG})")
|
|
185
214
|
|
|
186
|
-
|
|
187
|
-
return snowflake_id
|
|
215
|
+
return snowflake_id
|
|
188
216
|
|
|
189
217
|
@staticmethod
|
|
190
218
|
def parse_id(snowflake_id: int) -> dict:
|
|
219
|
+
"""解析雪花ID,返回生成时间、机器ID、序列等信息"""
|
|
191
220
|
from datetime import datetime
|
|
192
221
|
sequence = snowflake_id & Snowflake.MAX_SEQUENCE
|
|
193
222
|
machine_id = (snowflake_id >>
|
|
@@ -196,6 +225,7 @@ class Snowflake:
|
|
|
196
225
|
Snowflake.START_TIMESTAMP
|
|
197
226
|
generate_time = datetime.fromtimestamp(
|
|
198
227
|
timestamp / 1000).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
|
|
228
|
+
|
|
199
229
|
return {
|
|
200
230
|
"snowflake_id": snowflake_id,
|
|
201
231
|
"generate_time": generate_time,
|
|
@@ -206,43 +236,65 @@ class Snowflake:
|
|
|
206
236
|
|
|
207
237
|
@classmethod
|
|
208
238
|
def next_id(cls) -> str:
|
|
239
|
+
"""
|
|
240
|
+
生成雪花ID(线程安全单例模式,避免重复创建实例,锁内完成所有初始化)
|
|
241
|
+
:return: 雪花ID字符串
|
|
242
|
+
"""
|
|
209
243
|
if cls._instance is None:
|
|
210
244
|
with cls._instance_lock:
|
|
211
245
|
if cls._instance is None:
|
|
246
|
+
# 锁内初始化,避免多线程重复计算machine_id
|
|
212
247
|
cls._instance = cls()
|
|
213
248
|
return str(cls._instance.generate_id())
|
|
214
249
|
|
|
215
250
|
@ClassProperty
|
|
216
251
|
def id(cls) -> str:
|
|
252
|
+
"""
|
|
253
|
+
直接通过 `Snowflake.id` 属性生成雪花ID(兼容Python 3.11+)
|
|
254
|
+
:return: 雪花ID字符串
|
|
255
|
+
"""
|
|
217
256
|
return cls.next_id()
|
|
218
257
|
|
|
219
258
|
|
|
220
259
|
if __name__ == "__main__":
|
|
260
|
+
print("=== 生产级雪花算法ID生成测试 ===")
|
|
261
|
+
# 1. 基础生成测试
|
|
262
|
+
id1 = Snowflake.id
|
|
263
|
+
id2 = Snowflake.id
|
|
264
|
+
id3 = Snowflake.id
|
|
265
|
+
print(f"生成ID1: {id1}")
|
|
266
|
+
print(f"生成ID2: {id2}")
|
|
267
|
+
print(f"生成ID3: {id3}")
|
|
268
|
+
print(f"ID是否唯一: {len({id1, id2, id3}) == 3}")
|
|
269
|
+
|
|
270
|
+
# 2. 解析ID信息
|
|
271
|
+
print("\n=== 雪花ID解析 ===")
|
|
272
|
+
parse_info = Snowflake.parse_id(int(id3))
|
|
273
|
+
for key, value in parse_info.items():
|
|
274
|
+
print(f"{key}: {value}")
|
|
275
|
+
|
|
276
|
+
# 3. 批量唯一性验证(10000个ID)
|
|
277
|
+
print("\n=== 批量唯一性验证(10000个)===")
|
|
278
|
+
id_set = set()
|
|
279
|
+
duplicate_count = 0
|
|
280
|
+
for i in range(10000):
|
|
281
|
+
snow_id = Snowflake.id
|
|
282
|
+
if snow_id in id_set:
|
|
283
|
+
duplicate_count += 1
|
|
284
|
+
id_set.add(snow_id)
|
|
285
|
+
print(f"总生成数量: 10000")
|
|
286
|
+
print(f"唯一ID数量: {len(id_set)}")
|
|
287
|
+
print(f"重复ID数量: {duplicate_count}")
|
|
288
|
+
print(f"机器ID: {Snowflake._instance.machine_id}")
|
|
289
|
+
|
|
290
|
+
# 4. 高并发测试
|
|
221
291
|
import concurrent.futures
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
count = 100000
|
|
225
|
-
workers = 100
|
|
226
|
-
|
|
227
|
-
ids = []
|
|
228
|
-
start_time = time.perf_counter()
|
|
229
|
-
|
|
230
|
-
def task():
|
|
231
|
-
return Snowflake.id
|
|
232
|
-
|
|
292
|
+
print("\n=== 高并发测试(100线程)===")
|
|
293
|
+
id_set_concurrent = set()
|
|
233
294
|
with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
|
|
234
|
-
futures = [executor.submit(
|
|
295
|
+
futures = [executor.submit(lambda: Snowflake.id) for _ in range(10000)]
|
|
235
296
|
for future in concurrent.futures.as_completed(futures):
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
unique_count = len(set(ids))
|
|
242
|
-
print(f"生成数量: {len(ids)}")
|
|
243
|
-
print(f"唯一ID数量: {unique_count}")
|
|
244
|
-
print(f"是否有重复: {'是 ❌' if unique_count != len(ids) else '否 ✅'}")
|
|
245
|
-
print(f"总耗时: {duration:.4f} 秒")
|
|
246
|
-
print(f"吞吐量 (QPS): {len(ids) / duration:,.2f}")
|
|
247
|
-
print("\n最后一个ID解析:")
|
|
248
|
-
print(Snowflake.parse_id(int(ids[-1])))
|
|
297
|
+
id_set_concurrent.add(future.result())
|
|
298
|
+
print(f"高并发生成唯一ID数量: {len(id_set_concurrent)}")
|
|
299
|
+
|
|
300
|
+
print("\n=== 生产级雪花算法验证通过 ===")
|
|
@@ -1,28 +1,26 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sycommon-python-lib
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.56b2
|
|
4
4
|
Summary: Add your description here
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
7
|
Requires-Dist: aio-pika>=9.5.8
|
|
8
|
-
Requires-Dist: aiohttp>=3.13.
|
|
8
|
+
Requires-Dist: aiohttp>=3.13.2
|
|
9
9
|
Requires-Dist: aiomysql>=0.3.2
|
|
10
10
|
Requires-Dist: decorator>=5.2.1
|
|
11
|
-
Requires-Dist: fastapi>=0.
|
|
11
|
+
Requires-Dist: fastapi>=0.127.0
|
|
12
12
|
Requires-Dist: kafka-python>=2.3.0
|
|
13
|
-
Requires-Dist: langchain>=1.2.
|
|
14
|
-
Requires-Dist: langchain-core>=1.2.
|
|
15
|
-
Requires-Dist: langchain-openai>=1.1.
|
|
16
|
-
Requires-Dist:
|
|
17
|
-
Requires-Dist: langgraph>=1.0.6
|
|
13
|
+
Requires-Dist: langchain>=1.2.0
|
|
14
|
+
Requires-Dist: langchain-core>=1.2.6
|
|
15
|
+
Requires-Dist: langchain-openai>=1.1.6
|
|
16
|
+
Requires-Dist: langgraph>=1.0.5
|
|
18
17
|
Requires-Dist: loguru>=0.7.3
|
|
19
18
|
Requires-Dist: mysql-connector-python>=9.5.0
|
|
20
19
|
Requires-Dist: nacos-sdk-python<3.0,>=2.0.9
|
|
21
|
-
Requires-Dist: psutil>=7.
|
|
20
|
+
Requires-Dist: psutil>=7.1.3
|
|
22
21
|
Requires-Dist: pydantic>=2.12.5
|
|
23
22
|
Requires-Dist: python-dotenv>=1.2.1
|
|
24
23
|
Requires-Dist: pyyaml>=6.0.3
|
|
25
|
-
Requires-Dist: sentry-sdk[fastapi]>=2.49.0
|
|
26
24
|
Requires-Dist: sqlalchemy[asyncio]>=2.0.45
|
|
27
25
|
Requires-Dist: starlette>=0.50.0
|
|
28
26
|
Requires-Dist: uvicorn>=0.40.0
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
command/cli.py,sha256=bP2LCLkRvfETIwWkVD70q5xFxMI4D3BpH09Ws1f-ENc,5849
|
|
2
2
|
sycommon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
sycommon/services.py,sha256=
|
|
4
|
-
sycommon/config/Config.py,sha256=
|
|
3
|
+
sycommon/services.py,sha256=pioA4E2RrV6nyAY8EuETUt4mKzCYoxD1rTMYCcppqjE,11581
|
|
4
|
+
sycommon/config/Config.py,sha256=XWXcE_6RaoB9og0Q6ylQs4oW-lX9mhblr-wWq7S2dVY,3157
|
|
5
5
|
sycommon/config/DatabaseConfig.py,sha256=ILiUuYT9_xJZE2W-RYuC3JCt_YLKc1sbH13-MHIOPhg,804
|
|
6
6
|
sycommon/config/EmbeddingConfig.py,sha256=gPKwiDYbeu1GpdIZXMmgqM7JqBIzCXi0yYuGRLZooMI,362
|
|
7
7
|
sycommon/config/LLMConfig.py,sha256=yU-aIqePIeF6msfRVEtGq7SXZVDfHyTi6JduKjhMO_4,371
|
|
8
|
-
sycommon/config/LangfuseConfig.py,sha256=t2LulAtnMUvIINOKHXNWlT5PtgNb7IuaHURjWlbma38,370
|
|
9
8
|
sycommon/config/MQConfig.py,sha256=_RDcmIdyWKjmgM5ZnriOoI-DpaxgXs7CD0awdAD6z88,252
|
|
10
9
|
sycommon/config/RerankerConfig.py,sha256=35sVwzus2IscvTHnCG63Orl2pC-pMsrVi6wAGDmOH3U,341
|
|
11
|
-
sycommon/config/SentryConfig.py,sha256=OsLb3G9lTsCSZ7tWkcXWJHmvfILQopBxje5pjnkFJfo,320
|
|
12
10
|
sycommon/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
11
|
sycommon/database/async_base_db_service.py,sha256=w6ONUiTtF4-bXRnkBt9QpL9BAy0XUDbQG7F9Hf2rfjw,1337
|
|
14
12
|
sycommon/database/async_database_service.py,sha256=4Ag5PH6DFEcJOXR8MRF9V_Jho5uCoU9Ibo3PqulDsXw,3916
|
|
@@ -19,16 +17,12 @@ sycommon/health/health_check.py,sha256=EhfbhspRpQiKJaxdtE-PzpKQO_ucaFKtQxIm16F5M
|
|
|
19
17
|
sycommon/health/metrics.py,sha256=fHqO73JuhoZkNPR-xIlxieXiTCvttq-kG-tvxag1s1s,268
|
|
20
18
|
sycommon/health/ping.py,sha256=FTlnIKk5y1mPfS1ZGOeT5IM_2udF5aqVLubEtuBp18M,250
|
|
21
19
|
sycommon/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
|
-
sycommon/llm/embedding.py,sha256=
|
|
23
|
-
sycommon/llm/get_llm.py,sha256=
|
|
20
|
+
sycommon/llm/embedding.py,sha256=Wcm2W7JU3FyZXvOhMSdyhiZJhJS1MwW8bMqdrOzD2TY,5768
|
|
21
|
+
sycommon/llm/get_llm.py,sha256=QnmNwah2wzO2P1o37lNTrpzg7ZYvaDHjIzgOD9n3Ais,8325
|
|
24
22
|
sycommon/llm/llm_logger.py,sha256=n4UeNy_-g4oHQOsw-VUzF4uo3JVRLtxaMp1FcI8FiEo,5437
|
|
25
|
-
sycommon/llm/llm_tokens.py,sha256=-udDyFcmyzx6UAwIi6_d_wwI5kMd5w0-WcS2soVPQxg,4309
|
|
26
|
-
sycommon/llm/struct_token.py,sha256=jlpZnTOLDmRDdrCuxZe-1pQopd6OmCM9B_gWZ48CnEQ,7655
|
|
27
|
-
sycommon/llm/sy_langfuse.py,sha256=NZv6ydfn3-cxqQvuB5WdnM9GYliO9qB_RWh_XqIS3VU,3692
|
|
28
|
-
sycommon/llm/usage_token.py,sha256=n0hytuaHI4tJi6wuOS3bd-yWzQjZ-lx5w9egHs8uYgg,5140
|
|
29
23
|
sycommon/logging/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
30
24
|
sycommon/logging/async_sql_logger.py,sha256=_OY36XkUm__U3NhMgiecy-qd-nptZ_0gpE3J8lGAr58,2619
|
|
31
|
-
sycommon/logging/kafka_log.py,sha256=
|
|
25
|
+
sycommon/logging/kafka_log.py,sha256=sVw-dFZKEgCosjSUqgTj7YrpK-ggXhleZFwMUVhl-K0,21416
|
|
32
26
|
sycommon/logging/logger_levels.py,sha256=_-uQ_T1N8NkNgcAmLrMmJ83nHTDw5ZNvXFPvdk89XGY,1144
|
|
33
27
|
sycommon/logging/logger_wrapper.py,sha256=TiHsrIIHiQMzXgXK12-0KIpU9GhwQJOoHslakzmq2zc,357
|
|
34
28
|
sycommon/logging/sql_logger.py,sha256=aEU3OGnI_51Tjyuuf4FpUi9KPTceFRuKAOyQbPzGhzM,2021
|
|
@@ -36,12 +30,12 @@ sycommon/middleware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
|
|
|
36
30
|
sycommon/middleware/context.py,sha256=sc1UjN55dYww2tT9KdFMJV8mgpGro67SnETzbVzty-s,155
|
|
37
31
|
sycommon/middleware/cors.py,sha256=0B5d_ovD56wcH9TfktRs88Q09R9f8Xx5h5ALWYvE8Iw,600
|
|
38
32
|
sycommon/middleware/docs.py,sha256=bVdDBIHXGVBv562MQLSroa1DgHoObxR9gFzv71PIejg,1187
|
|
39
|
-
sycommon/middleware/exception.py,sha256=
|
|
33
|
+
sycommon/middleware/exception.py,sha256=Bk8IchNND1wg6tUX9hf4xWeEJhvA-E_zE9ysBpRZrqE,3010
|
|
40
34
|
sycommon/middleware/middleware.py,sha256=SzZ4wufSNdwC4Ppw99TE7a6AVGkrZRc55NHSrA3PiC8,1447
|
|
41
35
|
sycommon/middleware/monitor_memory.py,sha256=pYRK-wRuDd6enSg9Pf8tQxPdYQS6S0AyjyXeKFRLKEs,628
|
|
42
36
|
sycommon/middleware/mq.py,sha256=4wBqiT5wJGcrfjk2GSr0_U3TStBxoNpHTzcRxVlMEHE,183
|
|
43
|
-
sycommon/middleware/timeout.py,sha256=
|
|
44
|
-
sycommon/middleware/traceid.py,sha256=
|
|
37
|
+
sycommon/middleware/timeout.py,sha256=fImlAPLm4Oa8N9goXtT_0os1GZPCi9F92OgXU81DgDU,656
|
|
38
|
+
sycommon/middleware/traceid.py,sha256=0WYLV5PR6sGeqhLdviHYW94UhUC7NJua2g5HWf2Zhbc,13447
|
|
45
39
|
sycommon/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
46
40
|
sycommon/models/base_http.py,sha256=EICAAibx3xhjBsLqm35Mi3DCqxp0FME4rD_3iQVjT_E,3051
|
|
47
41
|
sycommon/models/log.py,sha256=rZpj6VkDRxK3B6H7XSeWdYZshU8F0Sks8bq1p6pPlDw,500
|
|
@@ -49,18 +43,9 @@ sycommon/models/mqlistener_config.py,sha256=vXp2uMmd0XQ5B9noSRXWHewTy-juQ2y7IsWt
|
|
|
49
43
|
sycommon/models/mqmsg_model.py,sha256=cxn0M5b0utQK6crMYmL-1waeGYHvK3AlGaRy23clqTE,277
|
|
50
44
|
sycommon/models/mqsend_config.py,sha256=NQX9dc8PpuquMG36GCVhJe8omAW1KVXXqr6lSRU6D7I,268
|
|
51
45
|
sycommon/models/sso_user.py,sha256=i1WAN6k5sPcPApQEdtjpWDy7VrzWLpOrOQewGLGoGIw,2702
|
|
52
|
-
sycommon/
|
|
53
|
-
sycommon/
|
|
54
|
-
sycommon/rabbitmq/
|
|
55
|
-
sycommon/rabbitmq/rabbitmq_pool.py,sha256=BiFQgZPzSAFR-n5XhyIafoeWQXETF_31nFRDhMbe6aU,15577
|
|
56
|
-
sycommon/rabbitmq/rabbitmq_service.py,sha256=XSHo9HuIJ_lq-vizRh4xJVdZr_2zLqeLhot09qb0euA,2025
|
|
57
|
-
sycommon/rabbitmq/rabbitmq_service_client_manager.py,sha256=MM4r8Pa2rjAmzy_NpHFb4thGznr6AYk6m__IC8IIxEM,7852
|
|
58
|
-
sycommon/rabbitmq/rabbitmq_service_connection_monitor.py,sha256=uvoMuJDzJ9i63uVRq1NKFV10CvkbGnTMyEoq2rgjQx8,3013
|
|
59
|
-
sycommon/rabbitmq/rabbitmq_service_consumer_manager.py,sha256=489r1RKd5WrTNMAcWCxUZpt9yWGrNunZlLCCp-M_rzM,11497
|
|
60
|
-
sycommon/rabbitmq/rabbitmq_service_core.py,sha256=6RMvIf78DmEOZmN8dA0duA9oy4ieNswdGrOeyJdD6tU,4753
|
|
61
|
-
sycommon/rabbitmq/rabbitmq_service_producer_manager.py,sha256=TJrLbvsjF55P9lwr7aCm9uRIRuC3z4EZNx74KEVKBtU,10190
|
|
62
|
-
sycommon/sentry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
63
|
-
sycommon/sentry/sy_sentry.py,sha256=e5Fbt9Gi2gIb048z0nuKbuhp3uCAEqYH2xXbF6qrZq4,1471
|
|
46
|
+
sycommon/rabbitmq/rabbitmq_client.py,sha256=GkuYILMZJnvgZs4ID46I-w_UGzzI28YUydKgkTIDhIs,20226
|
|
47
|
+
sycommon/rabbitmq/rabbitmq_pool.py,sha256=QtUcK4HuepRqRmy5XkUQo8gDgj74fr77CX7T5rN0y4I,15640
|
|
48
|
+
sycommon/rabbitmq/rabbitmq_service.py,sha256=wpEJynP0gzbnmMyB02sfR9nTWB4hfTTP00xQxDJu644,38119
|
|
64
49
|
sycommon/sse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
65
50
|
sycommon/sse/event.py,sha256=k_rBJy23R7crtzQeetT0Q73D8o5-5p-eESGSs_BPOj0,2797
|
|
66
51
|
sycommon/sse/sse.py,sha256=__CfWEcYxOxQ-HpLor4LTZ5hLWqw9-2X7CngqbVHsfw,10128
|
|
@@ -69,21 +54,15 @@ sycommon/synacos/example.py,sha256=61XL03tU8WTNOo3FUduf93F2fAwah1S0lbH1ufhRhRk,5
|
|
|
69
54
|
sycommon/synacos/example2.py,sha256=adUaru3Hy482KrOA17DfaC4nwvLj8etIDS_KrWLWmCU,4811
|
|
70
55
|
sycommon/synacos/feign.py,sha256=frB3D5LeFDtT3pJLFOwFzEOrNAJKeQNGk-BzUg9T3WM,8295
|
|
71
56
|
sycommon/synacos/feign_client.py,sha256=ExO7Pd5B3eFKDjXqBRc260K1jkI49IYguLwJJaD2R-o,16166
|
|
72
|
-
sycommon/synacos/
|
|
73
|
-
sycommon/synacos/nacos_config_manager.py,sha256=Cff-4gpp0aD7sQVi-nEvDO4BWqK9abEDDDJ9qXKFQgs,4399
|
|
74
|
-
sycommon/synacos/nacos_heartbeat_manager.py,sha256=G80_pOn37WdO_HpYUiAfpwMqAxW0ff0Bnw0NEuge9v0,5568
|
|
75
|
-
sycommon/synacos/nacos_service.py,sha256=BezQ1eDIYwBPE567Po_Qh1Ki_z9WmhZy1J1NiTPbdHY,6118
|
|
76
|
-
sycommon/synacos/nacos_service_discovery.py,sha256=O0M4facMa3I9YDFaovrfiTnCMtJWTuewbt9ce4a2-CA,6828
|
|
77
|
-
sycommon/synacos/nacos_service_registration.py,sha256=plg2PmH8CWgmVnPtiIXBxtj-3BpyMdSzKr1wyWRdzh4,10968
|
|
57
|
+
sycommon/synacos/nacos_service.py,sha256=cvmjd9dJIzaOFm6IXGjy2BCq1gHTHRxLYdgt5kBguUw,35976
|
|
78
58
|
sycommon/synacos/param.py,sha256=KcfSkxnXOa0TGmCjY8hdzU9pzUsA8-4PeyBKWI2-568,1765
|
|
79
59
|
sycommon/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
80
60
|
sycommon/tools/docs.py,sha256=OPj2ETheuWjXLyaXtaZPbwmJKfJaYXV5s4XMVAUNrms,1607
|
|
81
|
-
sycommon/tools/
|
|
82
|
-
sycommon/tools/
|
|
83
|
-
sycommon/tools/snowflake.py,sha256=xQlYXwYnI85kSJ1rZ89gMVBhzemP03xrMPVX9vVa3MY,9228
|
|
61
|
+
sycommon/tools/merge_headers.py,sha256=HV_i52Q-9se3SP8qh7ZGYl8bP7Fxtal4CGVkyMwEdM8,4373
|
|
62
|
+
sycommon/tools/snowflake.py,sha256=lVEe5mNCOgz5OqGQpf5_nXaGnRJlI2STX2s-ppTtanA,11947
|
|
84
63
|
sycommon/tools/timing.py,sha256=OiiE7P07lRoMzX9kzb8sZU9cDb0zNnqIlY5pWqHcnkY,2064
|
|
85
|
-
sycommon_python_lib-0.1.
|
|
86
|
-
sycommon_python_lib-0.1.
|
|
87
|
-
sycommon_python_lib-0.1.
|
|
88
|
-
sycommon_python_lib-0.1.
|
|
89
|
-
sycommon_python_lib-0.1.
|
|
64
|
+
sycommon_python_lib-0.1.56b2.dist-info/METADATA,sha256=HzNJOWrTsJrz8BkzZwJRiq6r9ugYormfdmCEFPM_e_g,7226
|
|
65
|
+
sycommon_python_lib-0.1.56b2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
66
|
+
sycommon_python_lib-0.1.56b2.dist-info/entry_points.txt,sha256=q_h2nbvhhmdnsOUZEIwpuoDjaNfBF9XqppDEmQn9d_A,46
|
|
67
|
+
sycommon_python_lib-0.1.56b2.dist-info/top_level.txt,sha256=98CJ-cyM2WIKxLz-Pf0AitWLhJyrfXvyY8slwjTXNuc,17
|
|
68
|
+
sycommon_python_lib-0.1.56b2.dist-info/RECORD,,
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
from pydantic import BaseModel
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
class LangfuseConfig(BaseModel):
|
|
5
|
-
name: str
|
|
6
|
-
secretKey: str
|
|
7
|
-
publicKey: str
|
|
8
|
-
baseUrl: str
|
|
9
|
-
enable: bool
|
|
10
|
-
|
|
11
|
-
@classmethod
|
|
12
|
-
def from_config(cls, server_name: str):
|
|
13
|
-
from sycommon.config.Config import Config
|
|
14
|
-
langfuse_config = Config().get_langfuse_config(server_name)
|
|
15
|
-
return cls(**langfuse_config)
|
sycommon/config/SentryConfig.py
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
from pydantic import BaseModel
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
class SentryConfig(BaseModel):
|
|
5
|
-
name: str
|
|
6
|
-
dsn: str
|
|
7
|
-
enable: bool
|
|
8
|
-
|
|
9
|
-
@classmethod
|
|
10
|
-
def from_config(cls, server_name: str):
|
|
11
|
-
from sycommon.config.Config import Config
|
|
12
|
-
sentry_config = Config().get_sentry_config(server_name)
|
|
13
|
-
return cls(**sentry_config)
|