sycommon-python-lib 0.1.56b5__py3-none-any.whl → 0.1.57b4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sycommon/config/Config.py +24 -3
- sycommon/config/LangfuseConfig.py +15 -0
- sycommon/config/SentryConfig.py +13 -0
- sycommon/llm/embedding.py +269 -50
- sycommon/llm/get_llm.py +9 -218
- sycommon/llm/struct_token.py +192 -0
- sycommon/llm/sy_langfuse.py +103 -0
- sycommon/llm/usage_token.py +117 -0
- sycommon/logging/kafka_log.py +187 -433
- sycommon/middleware/exception.py +10 -16
- sycommon/middleware/timeout.py +2 -1
- sycommon/middleware/traceid.py +81 -76
- sycommon/notice/uvicorn_monitor.py +32 -27
- sycommon/rabbitmq/rabbitmq_client.py +247 -242
- sycommon/rabbitmq/rabbitmq_pool.py +201 -123
- sycommon/rabbitmq/rabbitmq_service.py +25 -843
- sycommon/rabbitmq/rabbitmq_service_client_manager.py +211 -0
- sycommon/rabbitmq/rabbitmq_service_connection_monitor.py +73 -0
- sycommon/rabbitmq/rabbitmq_service_consumer_manager.py +285 -0
- sycommon/rabbitmq/rabbitmq_service_core.py +117 -0
- sycommon/rabbitmq/rabbitmq_service_producer_manager.py +238 -0
- sycommon/sentry/__init__.py +0 -0
- sycommon/sentry/sy_sentry.py +35 -0
- sycommon/services.py +122 -96
- sycommon/synacos/nacos_client_base.py +121 -0
- sycommon/synacos/nacos_config_manager.py +107 -0
- sycommon/synacos/nacos_heartbeat_manager.py +144 -0
- sycommon/synacos/nacos_service.py +63 -783
- sycommon/synacos/nacos_service_discovery.py +157 -0
- sycommon/synacos/nacos_service_registration.py +270 -0
- sycommon/tools/env.py +62 -0
- sycommon/tools/merge_headers.py +20 -0
- sycommon/tools/snowflake.py +101 -153
- {sycommon_python_lib-0.1.56b5.dist-info → sycommon_python_lib-0.1.57b4.dist-info}/METADATA +10 -8
- {sycommon_python_lib-0.1.56b5.dist-info → sycommon_python_lib-0.1.57b4.dist-info}/RECORD +38 -20
- {sycommon_python_lib-0.1.56b5.dist-info → sycommon_python_lib-0.1.57b4.dist-info}/WHEEL +0 -0
- {sycommon_python_lib-0.1.56b5.dist-info → sycommon_python_lib-0.1.57b4.dist-info}/entry_points.txt +0 -0
- {sycommon_python_lib-0.1.56b5.dist-info → sycommon_python_lib-0.1.57b4.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import threading
|
|
2
|
+
import time
|
|
3
|
+
from typing import Optional
|
|
4
|
+
import nacos
|
|
5
|
+
from sycommon.config.Config import Config
|
|
6
|
+
from sycommon.logging.kafka_log import SYLogger
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class NacosClientBase:
|
|
10
|
+
"""Nacos客户端基础类 - 负责客户端初始化和连接管理"""
|
|
11
|
+
|
|
12
|
+
def __init__(self, nacos_config: dict, enable_register_nacos: bool):
|
|
13
|
+
self.nacos_config = nacos_config
|
|
14
|
+
self.enable_register_nacos = enable_register_nacos
|
|
15
|
+
|
|
16
|
+
# 客户端配置
|
|
17
|
+
self.max_retries = self.nacos_config.get('maxRetries', 5)
|
|
18
|
+
self.retry_delay = self.nacos_config.get('retryDelay', 5)
|
|
19
|
+
self.max_retry_delay = self.nacos_config.get('maxRetryDelay', 30)
|
|
20
|
+
|
|
21
|
+
# 状态管理
|
|
22
|
+
self._client_initialized = False
|
|
23
|
+
self._state_lock = threading.RLock()
|
|
24
|
+
self._shutdown_event = threading.Event()
|
|
25
|
+
self.nacos_client: Optional[nacos.NacosClient] = None
|
|
26
|
+
|
|
27
|
+
def _initialize_client(self) -> bool:
|
|
28
|
+
"""初始化Nacos客户端(仅首次调用时执行)"""
|
|
29
|
+
if self._client_initialized:
|
|
30
|
+
return True
|
|
31
|
+
|
|
32
|
+
for attempt in range(self.max_retries):
|
|
33
|
+
try:
|
|
34
|
+
register_ip = self.nacos_config['registerIp']
|
|
35
|
+
namespace_id = self.nacos_config['namespaceId']
|
|
36
|
+
self.nacos_client = nacos.NacosClient(
|
|
37
|
+
server_addresses=register_ip,
|
|
38
|
+
namespace=namespace_id
|
|
39
|
+
)
|
|
40
|
+
SYLogger.info("nacos:客户端初始化成功")
|
|
41
|
+
self._client_initialized = True
|
|
42
|
+
return True
|
|
43
|
+
except Exception as e:
|
|
44
|
+
delay = min(self.retry_delay, self.max_retry_delay)
|
|
45
|
+
SYLogger.error(
|
|
46
|
+
f"nacos:客户端初始化失败 (尝试 {attempt+1}/{self.max_retries}): {e}")
|
|
47
|
+
time.sleep(delay)
|
|
48
|
+
|
|
49
|
+
SYLogger.warning("nacos:无法连接到 Nacos 服务器,已达到最大重试次数")
|
|
50
|
+
return False
|
|
51
|
+
|
|
52
|
+
def ensure_client_connected(self, retry_once: bool = False) -> bool:
|
|
53
|
+
"""确保Nacos客户端已连接,返回连接状态"""
|
|
54
|
+
with self._state_lock:
|
|
55
|
+
if self._client_initialized:
|
|
56
|
+
return True
|
|
57
|
+
|
|
58
|
+
SYLogger.warning("nacos:客户端未初始化,尝试连接...")
|
|
59
|
+
|
|
60
|
+
max_attempts = 2 if retry_once else self.max_retries
|
|
61
|
+
attempt = 0
|
|
62
|
+
|
|
63
|
+
while attempt < max_attempts:
|
|
64
|
+
try:
|
|
65
|
+
register_ip = self.nacos_config['registerIp']
|
|
66
|
+
namespace_id = self.nacos_config['namespaceId']
|
|
67
|
+
|
|
68
|
+
self.nacos_client = nacos.NacosClient(
|
|
69
|
+
server_addresses=register_ip,
|
|
70
|
+
namespace=namespace_id
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
if self._verify_client_connection():
|
|
74
|
+
with self._state_lock:
|
|
75
|
+
self._client_initialized = True
|
|
76
|
+
SYLogger.info("nacos:客户端初始化成功")
|
|
77
|
+
return True
|
|
78
|
+
else:
|
|
79
|
+
raise ConnectionError("nacos:客户端初始化后无法验证连接")
|
|
80
|
+
|
|
81
|
+
except Exception as e:
|
|
82
|
+
attempt += 1
|
|
83
|
+
delay = min(self.retry_delay, self.max_retry_delay)
|
|
84
|
+
SYLogger.error(
|
|
85
|
+
f"nacos:客户端初始化失败 (尝试 {attempt}/{max_attempts}): {e}")
|
|
86
|
+
time.sleep(delay)
|
|
87
|
+
|
|
88
|
+
SYLogger.error("nacos:无法连接到 Nacos 服务器,已达到最大重试次数")
|
|
89
|
+
return False
|
|
90
|
+
|
|
91
|
+
def _verify_client_connection(self) -> bool:
|
|
92
|
+
"""验证客户端是否真正连接成功"""
|
|
93
|
+
if not self.enable_register_nacos:
|
|
94
|
+
return True
|
|
95
|
+
|
|
96
|
+
try:
|
|
97
|
+
namespace_id = self.nacos_config['namespaceId']
|
|
98
|
+
service_name = Config().config.get('Name', '')
|
|
99
|
+
self.nacos_client.list_naming_instance(
|
|
100
|
+
service_name=service_name,
|
|
101
|
+
namespace_id=namespace_id,
|
|
102
|
+
group_name="DEFAULT_GROUP",
|
|
103
|
+
healthy_only=True
|
|
104
|
+
)
|
|
105
|
+
return True
|
|
106
|
+
except Exception as e:
|
|
107
|
+
SYLogger.warning(f"nacos:客户端连接验证失败: {e}")
|
|
108
|
+
return False
|
|
109
|
+
|
|
110
|
+
def reconnect_nacos_client(self) -> bool:
|
|
111
|
+
"""重新连接Nacos客户端"""
|
|
112
|
+
SYLogger.warning("nacos:尝试重新连接Nacos客户端")
|
|
113
|
+
with self._state_lock:
|
|
114
|
+
self._client_initialized = False
|
|
115
|
+
return self.ensure_client_connected()
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def is_connected(self) -> bool:
|
|
119
|
+
"""检查客户端是否已连接"""
|
|
120
|
+
with self._state_lock:
|
|
121
|
+
return self._client_initialized
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import threading
|
|
3
|
+
import time
|
|
4
|
+
from typing import Callable, Optional, Dict, List
|
|
5
|
+
from sycommon.synacos.nacos_client_base import NacosClientBase
|
|
6
|
+
import yaml
|
|
7
|
+
from sycommon.logging.kafka_log import SYLogger
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class NacosConfigManager:
|
|
11
|
+
"""Nacos配置管理类 - 负责配置读取、监听和更新"""
|
|
12
|
+
|
|
13
|
+
def __init__(self, client_base: NacosClientBase):
|
|
14
|
+
self.client_base = client_base
|
|
15
|
+
|
|
16
|
+
# 配置
|
|
17
|
+
self.config_watch_interval = self.client_base.nacos_config.get(
|
|
18
|
+
'configWatchInterval', 30)
|
|
19
|
+
|
|
20
|
+
# 状态
|
|
21
|
+
self.share_configs: Dict = {}
|
|
22
|
+
self._config_listeners: Dict[str, Callable[[str], None]] = {}
|
|
23
|
+
self._config_cache: Dict[str, str] = {}
|
|
24
|
+
self._watch_thread: Optional[threading.Thread] = None
|
|
25
|
+
|
|
26
|
+
def read_configs(self, shared_configs: List[Dict]) -> dict:
|
|
27
|
+
"""读取共享配置"""
|
|
28
|
+
configs = {}
|
|
29
|
+
|
|
30
|
+
for config in shared_configs:
|
|
31
|
+
data_id = config['dataId']
|
|
32
|
+
group = config['group']
|
|
33
|
+
|
|
34
|
+
for attempt in range(self.client_base.max_retries):
|
|
35
|
+
try:
|
|
36
|
+
if not self.client_base.ensure_client_connected():
|
|
37
|
+
self.client_base.reconnect_nacos_client()
|
|
38
|
+
|
|
39
|
+
content = self.client_base.nacos_client.get_config(
|
|
40
|
+
data_id, group)
|
|
41
|
+
|
|
42
|
+
try:
|
|
43
|
+
configs[data_id] = json.loads(content)
|
|
44
|
+
except json.JSONDecodeError:
|
|
45
|
+
try:
|
|
46
|
+
configs[data_id] = yaml.safe_load(content)
|
|
47
|
+
except yaml.YAMLError:
|
|
48
|
+
SYLogger.error(f"nacos:无法解析 {data_id} 的内容")
|
|
49
|
+
break
|
|
50
|
+
except Exception as e:
|
|
51
|
+
if attempt < self.client_base.max_retries - 1:
|
|
52
|
+
SYLogger.warning(
|
|
53
|
+
f"nacos:读取配置 {data_id} 失败 (尝试 {attempt+1}/{self.client_base.max_retries}): {e}")
|
|
54
|
+
time.sleep(self.client_base.retry_delay)
|
|
55
|
+
else:
|
|
56
|
+
SYLogger.error(
|
|
57
|
+
f"nacos:读取配置 {data_id} 失败,已达到最大重试次数: {e}")
|
|
58
|
+
|
|
59
|
+
self.share_configs = configs
|
|
60
|
+
return configs
|
|
61
|
+
|
|
62
|
+
def add_config_listener(self, data_id: str, callback: Callable[[str], None]):
|
|
63
|
+
"""添加配置变更监听器"""
|
|
64
|
+
self._config_listeners[data_id] = callback
|
|
65
|
+
if config := self.get_config(data_id):
|
|
66
|
+
callback(config)
|
|
67
|
+
|
|
68
|
+
def get_config(self, data_id: str, group: str = "DEFAULT_GROUP") -> Optional[str]:
|
|
69
|
+
"""获取配置内容"""
|
|
70
|
+
if not self.client_base.ensure_client_connected():
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
try:
|
|
74
|
+
return self.client_base.nacos_client.get_config(data_id, group=group)
|
|
75
|
+
except Exception as e:
|
|
76
|
+
SYLogger.error(f"nacos:获取配置 {data_id} 失败: {str(e)}")
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
def start_watch_configs(self):
|
|
80
|
+
"""启动配置监视线程"""
|
|
81
|
+
self._watch_thread = threading.Thread(
|
|
82
|
+
target=self._watch_configs, daemon=True)
|
|
83
|
+
self._watch_thread.start()
|
|
84
|
+
|
|
85
|
+
def _watch_configs(self):
|
|
86
|
+
"""配置监听线程"""
|
|
87
|
+
check_interval = self.config_watch_interval
|
|
88
|
+
|
|
89
|
+
while not self.client_base._shutdown_event.is_set():
|
|
90
|
+
try:
|
|
91
|
+
for data_id, callback in list(self._config_listeners.items()):
|
|
92
|
+
new_config = self.get_config(data_id)
|
|
93
|
+
if new_config and new_config != self._config_cache.get(data_id):
|
|
94
|
+
callback(new_config)
|
|
95
|
+
self._config_cache[data_id] = new_config
|
|
96
|
+
try:
|
|
97
|
+
self.share_configs[data_id] = json.loads(
|
|
98
|
+
new_config)
|
|
99
|
+
except json.JSONDecodeError:
|
|
100
|
+
try:
|
|
101
|
+
self.share_configs[data_id] = yaml.safe_load(
|
|
102
|
+
new_config)
|
|
103
|
+
except yaml.YAMLError:
|
|
104
|
+
SYLogger.error(f"nacos:无法解析 {data_id} 的内容")
|
|
105
|
+
except Exception as e:
|
|
106
|
+
SYLogger.error(f"nacos:配置监视线程异常: {str(e)}")
|
|
107
|
+
self.client_base._shutdown_event.wait(check_interval)
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import threading
|
|
2
|
+
import time
|
|
3
|
+
from sycommon.logging.kafka_log import SYLogger
|
|
4
|
+
from sycommon.synacos.nacos_client_base import NacosClientBase
|
|
5
|
+
from sycommon.synacos.nacos_service_registration import NacosServiceRegistration
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class NacosHeartbeatManager:
|
|
9
|
+
"""Nacos心跳管理类 - 负责心跳发送和监控"""
|
|
10
|
+
|
|
11
|
+
def __init__(self, client_base: NacosClientBase, registration: NacosServiceRegistration, heartbeat_interval: int = 15):
|
|
12
|
+
self.client_base = client_base
|
|
13
|
+
self.registration = registration
|
|
14
|
+
|
|
15
|
+
# 心跳配置
|
|
16
|
+
self.heartbeat_interval = heartbeat_interval
|
|
17
|
+
self.heartbeat_timeout = 15
|
|
18
|
+
self.max_heartbeat_timeout = self.client_base.nacos_config.get(
|
|
19
|
+
'maxHeartbeatTimeout', 30)
|
|
20
|
+
|
|
21
|
+
# 状态管理
|
|
22
|
+
self._heartbeat_lock = threading.Lock()
|
|
23
|
+
self._heartbeat_thread = None
|
|
24
|
+
self._last_heartbeat_time = 0
|
|
25
|
+
self._heartbeat_fail_count = 0
|
|
26
|
+
|
|
27
|
+
def start_heartbeat(self):
|
|
28
|
+
"""启动心跳线程(确保单例)"""
|
|
29
|
+
with self._heartbeat_lock:
|
|
30
|
+
if self._heartbeat_thread is not None and self._heartbeat_thread.is_alive():
|
|
31
|
+
return
|
|
32
|
+
|
|
33
|
+
self._heartbeat_thread = None
|
|
34
|
+
|
|
35
|
+
self._heartbeat_thread = threading.Thread(
|
|
36
|
+
target=self._send_heartbeat_loop,
|
|
37
|
+
name="NacosHeartbeatThread",
|
|
38
|
+
daemon=True
|
|
39
|
+
)
|
|
40
|
+
self._heartbeat_thread.start()
|
|
41
|
+
SYLogger.info(
|
|
42
|
+
f"nacos:心跳线程启动,线程ID: {self._heartbeat_thread.ident},"
|
|
43
|
+
f"心跳间隔: {self.heartbeat_interval}秒,"
|
|
44
|
+
f"心跳超时: {self.heartbeat_timeout}秒"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
def _send_heartbeat_loop(self):
|
|
48
|
+
"""心跳发送循环"""
|
|
49
|
+
current_thread = threading.current_thread()
|
|
50
|
+
thread_ident = current_thread.ident
|
|
51
|
+
SYLogger.info(
|
|
52
|
+
f"nacos:心跳循环启动 - 线程ID: {thread_ident}, "
|
|
53
|
+
f"配置间隔: {self.heartbeat_interval}秒, "
|
|
54
|
+
f"超时时间: {self.heartbeat_timeout}秒"
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
consecutive_fail = 0
|
|
58
|
+
|
|
59
|
+
while not self.client_base._shutdown_event.is_set():
|
|
60
|
+
current_time = time.time()
|
|
61
|
+
|
|
62
|
+
try:
|
|
63
|
+
registered_status = self.registration.registered
|
|
64
|
+
|
|
65
|
+
if not registered_status:
|
|
66
|
+
SYLogger.warning(
|
|
67
|
+
f"nacos:服务未注册,跳过心跳 - 线程ID: {thread_ident}")
|
|
68
|
+
consecutive_fail = 0
|
|
69
|
+
else:
|
|
70
|
+
success = self.send_heartbeat()
|
|
71
|
+
if success:
|
|
72
|
+
consecutive_fail = 0
|
|
73
|
+
SYLogger.info(
|
|
74
|
+
f"nacos:心跳发送成功 - 时间: {current_time:.3f}, "
|
|
75
|
+
f"间隔: {self.heartbeat_interval}秒"
|
|
76
|
+
)
|
|
77
|
+
else:
|
|
78
|
+
consecutive_fail += 1
|
|
79
|
+
SYLogger.warning(
|
|
80
|
+
f"nacos:心跳发送失败 - 连续失败: {consecutive_fail}次"
|
|
81
|
+
)
|
|
82
|
+
if consecutive_fail >= 5:
|
|
83
|
+
SYLogger.error("nacos:心跳连续失败5次,尝试重连")
|
|
84
|
+
self.client_base.reconnect_nacos_client()
|
|
85
|
+
consecutive_fail = 0
|
|
86
|
+
|
|
87
|
+
except Exception as e:
|
|
88
|
+
consecutive_fail += 1
|
|
89
|
+
SYLogger.error(
|
|
90
|
+
f"nacos:心跳异常: {str(e)}, 连续失败: {consecutive_fail}次")
|
|
91
|
+
|
|
92
|
+
self.client_base._shutdown_event.wait(self.heartbeat_interval)
|
|
93
|
+
|
|
94
|
+
SYLogger.info(f"nacos:心跳循环已停止 - 线程ID: {thread_ident}")
|
|
95
|
+
|
|
96
|
+
def send_heartbeat(self) -> bool:
|
|
97
|
+
"""发送心跳并添加超时控制"""
|
|
98
|
+
if not self.client_base.ensure_client_connected():
|
|
99
|
+
SYLogger.warning("nacos:客户端未连接,心跳发送失败")
|
|
100
|
+
return False
|
|
101
|
+
|
|
102
|
+
result_list = []
|
|
103
|
+
|
|
104
|
+
def heartbeat_task():
|
|
105
|
+
try:
|
|
106
|
+
result = self._send_heartbeat_internal()
|
|
107
|
+
result_list.append(result)
|
|
108
|
+
except Exception as e:
|
|
109
|
+
SYLogger.error(f"nacos:心跳任务执行异常: {e}")
|
|
110
|
+
result_list.append(False)
|
|
111
|
+
|
|
112
|
+
task_thread = threading.Thread(
|
|
113
|
+
target=heartbeat_task,
|
|
114
|
+
daemon=True,
|
|
115
|
+
name="NacosHeartbeatTaskThread"
|
|
116
|
+
)
|
|
117
|
+
task_thread.start()
|
|
118
|
+
task_thread.join(timeout=self.heartbeat_timeout)
|
|
119
|
+
|
|
120
|
+
if not result_list:
|
|
121
|
+
SYLogger.error(f"nacos:心跳发送超时({self.heartbeat_timeout}秒)")
|
|
122
|
+
self.client_base._client_initialized = False
|
|
123
|
+
return False
|
|
124
|
+
|
|
125
|
+
return result_list[0]
|
|
126
|
+
|
|
127
|
+
def _send_heartbeat_internal(self) -> bool:
|
|
128
|
+
"""实际的心跳发送逻辑"""
|
|
129
|
+
result = self.client_base.nacos_client.send_heartbeat(
|
|
130
|
+
service_name=self.registration.service_name,
|
|
131
|
+
ip=self.registration.real_ip,
|
|
132
|
+
port=int(self.registration.port),
|
|
133
|
+
cluster_name="DEFAULT",
|
|
134
|
+
weight=1.0,
|
|
135
|
+
metadata={
|
|
136
|
+
"version": self.registration.version} if self.registration.version else None
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
if result and isinstance(result, dict) and result.get('lightBeatEnabled', False):
|
|
140
|
+
SYLogger.info(f"nacos:心跳发送成功,Nacos返回: {result}")
|
|
141
|
+
return True
|
|
142
|
+
else:
|
|
143
|
+
SYLogger.warning(f"nacos:心跳发送失败,Nacos返回: {result}")
|
|
144
|
+
return False
|