sycommon-python-lib 0.2.7a28__py3-none-any.whl → 0.2.7a30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sycommon/agent/deep_agent.py +114 -30
- sycommon/agent/mcp/tool_loader.py +242 -68
- sycommon/agent/sandbox/minio_sync.py +35 -17
- {sycommon_python_lib-0.2.7a28.dist-info → sycommon_python_lib-0.2.7a30.dist-info}/METADATA +1 -1
- {sycommon_python_lib-0.2.7a28.dist-info → sycommon_python_lib-0.2.7a30.dist-info}/RECORD +8 -8
- {sycommon_python_lib-0.2.7a28.dist-info → sycommon_python_lib-0.2.7a30.dist-info}/WHEEL +0 -0
- {sycommon_python_lib-0.2.7a28.dist-info → sycommon_python_lib-0.2.7a30.dist-info}/entry_points.txt +0 -0
- {sycommon_python_lib-0.2.7a28.dist-info → sycommon_python_lib-0.2.7a30.dist-info}/top_level.txt +0 -0
sycommon/agent/deep_agent.py
CHANGED
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
"""
|
|
36
36
|
|
|
37
37
|
import asyncio
|
|
38
|
+
import hashlib
|
|
38
39
|
import json
|
|
39
40
|
import os
|
|
40
41
|
import re
|
|
@@ -1366,6 +1367,79 @@ def _parse_skill_version(content: str) -> str:
|
|
|
1366
1367
|
return ""
|
|
1367
1368
|
|
|
1368
1369
|
|
|
1370
|
+
def _decode_remote_skillmd(content: str) -> str:
|
|
1371
|
+
"""把沙箱 /read 返回的 SKILL.md 内容统一解成明文。
|
|
1372
|
+
|
|
1373
|
+
沙箱 /read 对大文件(如 skill-creator 的 34KB SKILL.md)返回 base64,
|
|
1374
|
+
且 encoding 标志位可能漏标/误标。判断策略(不依赖 encoding 字段):
|
|
1375
|
+
- 明文 SKILL.md 一定含 frontmatter 标志 `---`;含则当明文直接返回。
|
|
1376
|
+
- 不含 `---` 则当作 base64,解码后返回;解码失败或解码后仍无 `---` 返回原值。
|
|
1377
|
+
"""
|
|
1378
|
+
if not content:
|
|
1379
|
+
return content
|
|
1380
|
+
if "---" in content:
|
|
1381
|
+
return content # 明文
|
|
1382
|
+
try:
|
|
1383
|
+
import base64 as _b64
|
|
1384
|
+
decoded = _b64.b64decode(content).decode("utf-8", errors="replace")
|
|
1385
|
+
return decoded
|
|
1386
|
+
except Exception:
|
|
1387
|
+
return content
|
|
1388
|
+
|
|
1389
|
+
|
|
1390
|
+
def _skill_signature(content: str) -> str:
|
|
1391
|
+
"""计算 SKILL.md 正文内容的指纹,用于检测「版本号一致但正文未同步」的脏数据。
|
|
1392
|
+
|
|
1393
|
+
取正文(去掉 frontmatter)的前 N 行规范化后的拼接 + md5,作为轻量指纹。
|
|
1394
|
+
版本号相同时若指纹不同,说明沙箱里的正文是旧的,需强制重传。
|
|
1395
|
+
frontmatter 跳过逻辑与 _parse_skill_version 一致(逐行数 ---)。
|
|
1396
|
+
"""
|
|
1397
|
+
lines = content.split("\n")
|
|
1398
|
+
in_frontmatter = False
|
|
1399
|
+
body_start = 0
|
|
1400
|
+
for i, line in enumerate(lines):
|
|
1401
|
+
if line.strip() == "---":
|
|
1402
|
+
if not in_frontmatter:
|
|
1403
|
+
in_frontmatter = True # 进入 frontmatter
|
|
1404
|
+
else:
|
|
1405
|
+
body_start = i + 1 # 离开 frontmatter,正文从这里开始
|
|
1406
|
+
break
|
|
1407
|
+
body = "\n".join(lines[body_start:])
|
|
1408
|
+
# 规范化:去掉空白行和首尾空格,只取前 40 行做指纹(足够区分新旧大改)
|
|
1409
|
+
norm_lines = [l.rstrip() for l in body.split("\n") if l.strip()]
|
|
1410
|
+
head = "\n".join(norm_lines[:40])
|
|
1411
|
+
return f"{len(norm_lines)}:{hashlib.md5(head.encode('utf-8')).hexdigest()[:12]}"
|
|
1412
|
+
|
|
1413
|
+
|
|
1414
|
+
async def _verify_skill_content_fresh(
|
|
1415
|
+
backend, sandbox_dest: str, skill_name: str, local_src: str
|
|
1416
|
+
) -> bool:
|
|
1417
|
+
"""版本一致时,再校验沙箱 SKILL.md 正文指纹是否与本地一致。
|
|
1418
|
+
一致返回 True(可跳过同步);不一致返回 False(需强制重传)。
|
|
1419
|
+
任何读取异常返回 False(保守起见触发重传,宁可多传不可漏传)。"""
|
|
1420
|
+
local_md = os.path.join(local_src, "SKILL.md")
|
|
1421
|
+
if not os.path.isfile(local_md):
|
|
1422
|
+
return True # 本地没 SKILL.md,无法校验,交给版本逻辑
|
|
1423
|
+
try:
|
|
1424
|
+
with open(local_md, "r", encoding="utf-8") as f:
|
|
1425
|
+
local_sig = _skill_signature(f.read())
|
|
1426
|
+
except Exception:
|
|
1427
|
+
return True
|
|
1428
|
+
|
|
1429
|
+
try:
|
|
1430
|
+
remote = await backend.aread(
|
|
1431
|
+
f"{sandbox_dest}/{skill_name}/SKILL.md", _skip_lazy_sync=True)
|
|
1432
|
+
if not remote or not remote.file_data:
|
|
1433
|
+
return False # 读不到 → 当作不一致,触发重传
|
|
1434
|
+
raw = remote.file_data.get("content", "") or ""
|
|
1435
|
+
# 统一用 _decode_remote_skillmd 处理明文/base64 两种返回
|
|
1436
|
+
content = _decode_remote_skillmd(raw)
|
|
1437
|
+
remote_sig = _skill_signature(content)
|
|
1438
|
+
return bool(remote_sig) and local_sig == remote_sig
|
|
1439
|
+
except Exception:
|
|
1440
|
+
return False
|
|
1441
|
+
|
|
1442
|
+
|
|
1369
1443
|
async def _sync_skills_to_sandbox(
|
|
1370
1444
|
skills_dir: str,
|
|
1371
1445
|
backend: HTTPSandboxBackend,
|
|
@@ -1424,21 +1498,11 @@ async def _sync_skills_to_sandbox(
|
|
|
1424
1498
|
if remote_content and remote_content.file_data:
|
|
1425
1499
|
# FileData 是 TypedDict(dict), 用键访问而非属性
|
|
1426
1500
|
content = remote_content.file_data.get("content", "") or ""
|
|
1427
|
-
#
|
|
1428
|
-
#
|
|
1429
|
-
#
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
# 不依赖 encoding 字段, 对明文/base64 两种返回都正确。
|
|
1433
|
-
version = _parse_skill_version(content)
|
|
1434
|
-
if version:
|
|
1435
|
-
return version
|
|
1436
|
-
try:
|
|
1437
|
-
import base64 as _b64
|
|
1438
|
-
decoded = _b64.b64decode(content).decode("utf-8", errors="replace")
|
|
1439
|
-
return _parse_skill_version(decoded)
|
|
1440
|
-
except Exception:
|
|
1441
|
-
return ""
|
|
1501
|
+
# 沙箱 /read 对大文件(如 skill-creator 的 34KB SKILL.md)返回 base64,
|
|
1502
|
+
# 且 encoding 标志位可能漏标/误标。统一用 _decode_remote_skillmd
|
|
1503
|
+
# 处理明文/base64 两种返回(不依赖 encoding 字段),再解 version。
|
|
1504
|
+
plain = _decode_remote_skillmd(content)
|
|
1505
|
+
return _parse_skill_version(plain)
|
|
1442
1506
|
except Exception:
|
|
1443
1507
|
pass
|
|
1444
1508
|
return ""
|
|
@@ -1456,25 +1520,45 @@ async def _sync_skills_to_sandbox(
|
|
|
1456
1520
|
remote_version = remote_version_map.get(skill_name, "")
|
|
1457
1521
|
|
|
1458
1522
|
if remote_version and remote_version == local_version:
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1523
|
+
# 版本一致仍校验正文关键内容,防止"version 字段已更新但正文未同步"
|
|
1524
|
+
# (历史上出现过沙箱 SKILL.md 版本号新、正文旧的脏数据,导致 agent
|
|
1525
|
+
# 永远读不到新版指令)。校验失败则强制走删除+重传。
|
|
1526
|
+
if not await _verify_skill_content_fresh(
|
|
1527
|
+
backend, sandbox_dest, skill_name, src
|
|
1528
|
+
):
|
|
1529
|
+
SYLogger.warning(
|
|
1530
|
+
f"[DeepAgent] 技能版本一致但正文不一致,强制重传: {skill_name} "
|
|
1531
|
+
f"v{local_version}")
|
|
1532
|
+
# 落到下方的删除+重传分支
|
|
1533
|
+
else:
|
|
1534
|
+
SYLogger.debug(
|
|
1535
|
+
f"[DeepAgent] 技能版本一致,跳过: {skill_name} v{local_version}")
|
|
1536
|
+
backend._synced_skills.add(skill_name)
|
|
1537
|
+
skip_count += 1
|
|
1538
|
+
continue
|
|
1464
1539
|
|
|
1465
|
-
#
|
|
1540
|
+
# 版本不一致或无版本(或正文校验失败),删除旧目录(走 /delete,跳过懒同步,毫秒级)
|
|
1466
1541
|
if skill_name in existing_dirs:
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1542
|
+
deleted_ok = False
|
|
1543
|
+
for _attempt in range(2): # 最多重试一次,确保旧内容真删掉
|
|
1544
|
+
try:
|
|
1545
|
+
await backend.adelete(
|
|
1546
|
+
f"{sandbox_dest}/{skill_name}",
|
|
1547
|
+
recursive=True,
|
|
1548
|
+
_skip_lazy_sync=True)
|
|
1549
|
+
deleted_ok = True
|
|
1550
|
+
break
|
|
1551
|
+
except Exception as e:
|
|
1552
|
+
SYLogger.warning(
|
|
1553
|
+
f"[DeepAgent] 删除旧版技能失败(尝试{_attempt+1}): "
|
|
1554
|
+
f"{skill_name}, {e}")
|
|
1555
|
+
if deleted_ok:
|
|
1472
1556
|
SYLogger.info(
|
|
1473
|
-
f"[DeepAgent]
|
|
1557
|
+
f"[DeepAgent] 技能版本不一致,已删除旧目录: {skill_name} "
|
|
1474
1558
|
f"({remote_version or '无版本'} → {local_version})")
|
|
1475
|
-
|
|
1476
|
-
SYLogger.
|
|
1477
|
-
f"[DeepAgent]
|
|
1559
|
+
else:
|
|
1560
|
+
SYLogger.error(
|
|
1561
|
+
f"[DeepAgent] 删除旧版技能彻底失败,本次跳过: {skill_name}")
|
|
1478
1562
|
|
|
1479
1563
|
# 全量上传整个技能目录到沙箱(batch_upload 服务端无条件覆盖)
|
|
1480
1564
|
await backend.async_sync_dirs([(src, f"{sandbox_dest}/{skill_name}")])
|
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
|
|
7
7
|
import asyncio
|
|
8
8
|
import re
|
|
9
|
+
import time
|
|
9
10
|
import hashlib
|
|
10
|
-
from typing import List, Optional, Callable, Awaitable, TYPE_CHECKING
|
|
11
|
+
from typing import List, Optional, Callable, Awaitable, Tuple, TYPE_CHECKING
|
|
11
12
|
|
|
12
13
|
from langchain_core.tools import BaseTool
|
|
13
14
|
|
|
@@ -18,6 +19,122 @@ if TYPE_CHECKING:
|
|
|
18
19
|
from langchain_mcp_adapters.interceptors import MCPToolCallRequest
|
|
19
20
|
|
|
20
21
|
|
|
22
|
+
# ============== MCP 工具发现缓存 ==============
|
|
23
|
+
# 缓存的是「发现的 raw MCP tools」(mcp.types.Tool 列表),用户无关、可安全共享。
|
|
24
|
+
# 命中后跳过 TCP/TLS 握手 + MCP initialize + list_tools(这才是慢的根因——
|
|
25
|
+
# 每次 create_session 都重建连接)。LangChain BaseTool 转换(per-user interceptor
|
|
26
|
+
# + display_name 注入)仍在每次调用做,但那是纯内存操作,毫秒级。
|
|
27
|
+
# 失效:TTL 过期 / 配置变更(create/update/delete) 主动 invalidate。
|
|
28
|
+
_MCP_DISCOVERY_TTL = 300 # 5 分钟
|
|
29
|
+
# key = (server_url, headers_repr);value = (raw_tools, expire_ts)
|
|
30
|
+
_mcp_discovery_cache: dict[Tuple[str, str], Tuple[list, float]] = {}
|
|
31
|
+
# 单飞锁:同一 (url, headers) 并发发现只跑一次,其余等结果
|
|
32
|
+
_mcp_inflight: dict[Tuple[str, str], asyncio.Future] = {}
|
|
33
|
+
_mcp_cache_lock = asyncio.Lock()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _discovery_key(config: "MCPServerConfig") -> Tuple[str, str]:
|
|
37
|
+
"""计算发现缓存 key。headers 影响鉴权 → 影响可见工具,必须纳入 key。"""
|
|
38
|
+
headers_repr = ""
|
|
39
|
+
if config.headers:
|
|
40
|
+
# 排序保证 dict 顺序不影响 key
|
|
41
|
+
try:
|
|
42
|
+
headers_repr = repr(sorted(config.headers.items()))
|
|
43
|
+
except Exception:
|
|
44
|
+
headers_repr = str(config.headers)
|
|
45
|
+
return (config.server_url, headers_repr)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
async def invalidate_mcp_cache(server_url: str = None) -> None:
|
|
49
|
+
"""使发现缓存失效。配置变更(create/update/delete)时调用。
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
server_url: 指定则只失效该 url 的条目;None 清空全部。
|
|
53
|
+
"""
|
|
54
|
+
async with _mcp_cache_lock:
|
|
55
|
+
if server_url is None:
|
|
56
|
+
n = len(_mcp_discovery_cache)
|
|
57
|
+
_mcp_discovery_cache.clear()
|
|
58
|
+
if n:
|
|
59
|
+
SYLogger.info(f"[MCP] 发现缓存已全部清空({n} 条)")
|
|
60
|
+
else:
|
|
61
|
+
# 按第一维(url)匹配,清除该 url 所有 headers 变体
|
|
62
|
+
keys_to_drop = [k for k in _mcp_discovery_cache if k[0] == server_url]
|
|
63
|
+
for k in keys_to_drop:
|
|
64
|
+
_mcp_discovery_cache.pop(k, None)
|
|
65
|
+
if keys_to_drop:
|
|
66
|
+
SYLogger.info(f"[MCP] 发现缓存已失效: {server_url}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
async def _discover_with_cache(config: "MCPServerConfig") -> list:
|
|
70
|
+
"""带缓存 + 单飞地发现单个 server 的 raw MCP tools。
|
|
71
|
+
|
|
72
|
+
命中缓存(未过期) → 直接返回。未命中 → 并发请求中已有 in-flight future → await 它;
|
|
73
|
+
否则自己建立连接发现,写回缓存。
|
|
74
|
+
"""
|
|
75
|
+
from langchain_mcp_adapters.client import MultiServerMCPClient
|
|
76
|
+
from langchain_mcp_adapters.tools import _list_all_tools
|
|
77
|
+
|
|
78
|
+
key = sanitize_name(config.name)
|
|
79
|
+
cache_key = _discovery_key(config)
|
|
80
|
+
|
|
81
|
+
# 1. 查缓存(无锁快路径,dict 读本身线程安全足够)
|
|
82
|
+
cached = _mcp_discovery_cache.get(cache_key)
|
|
83
|
+
if cached is not None:
|
|
84
|
+
raw_tools, expire_ts = cached
|
|
85
|
+
if time.monotonic() < expire_ts:
|
|
86
|
+
SYLogger.debug(f"[MCP] 发现缓存命中: {config.name} ({len(raw_tools)} tools)")
|
|
87
|
+
return raw_tools
|
|
88
|
+
# 过期,清掉
|
|
89
|
+
_mcp_discovery_cache.pop(cache_key, None)
|
|
90
|
+
|
|
91
|
+
# 2. 单飞:同一 key 并发只发一个请求
|
|
92
|
+
is_leader = False
|
|
93
|
+
async with _mcp_cache_lock:
|
|
94
|
+
inflight = _mcp_inflight.get(cache_key)
|
|
95
|
+
if inflight is not None:
|
|
96
|
+
waiter = inflight
|
|
97
|
+
else:
|
|
98
|
+
waiter = asyncio.get_running_loop().create_future()
|
|
99
|
+
_mcp_inflight[cache_key] = waiter
|
|
100
|
+
is_leader = True
|
|
101
|
+
if not is_leader:
|
|
102
|
+
# 有并发请求在跑,等它的结果(不重复建连)
|
|
103
|
+
return await asyncio.shield(waiter)
|
|
104
|
+
|
|
105
|
+
# 3. 自己建连发现(leader)
|
|
106
|
+
try:
|
|
107
|
+
client_config = {
|
|
108
|
+
key: {
|
|
109
|
+
"url": config.server_url,
|
|
110
|
+
"transport": "streamable_http",
|
|
111
|
+
**({"headers": config.headers} if config.headers else {}),
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
client = MultiServerMCPClient(client_config)
|
|
115
|
+
|
|
116
|
+
async def _connect_and_list():
|
|
117
|
+
async with client.session(key) as session:
|
|
118
|
+
return await _list_all_tools(session)
|
|
119
|
+
|
|
120
|
+
raw_tools = await asyncio.wait_for(
|
|
121
|
+
_connect_and_list(), timeout=_MCP_SERVER_TIMEOUT)
|
|
122
|
+
|
|
123
|
+
# 写回缓存
|
|
124
|
+
_mcp_discovery_cache[cache_key] = (raw_tools, time.monotonic() + _MCP_DISCOVERY_TTL)
|
|
125
|
+
# 唤醒所有等待者
|
|
126
|
+
if not waiter.done():
|
|
127
|
+
waiter.set_result(raw_tools)
|
|
128
|
+
return raw_tools
|
|
129
|
+
except BaseException as e:
|
|
130
|
+
# 失败不写缓存,把真实异常透传给所有等待者(各自在 _load_one_server 里兜底返回 [])
|
|
131
|
+
if not waiter.done():
|
|
132
|
+
waiter.set_exception(RuntimeError(f"MCP 发现失败: {e}"))
|
|
133
|
+
raise
|
|
134
|
+
finally:
|
|
135
|
+
_mcp_inflight.pop(cache_key, None)
|
|
136
|
+
|
|
137
|
+
|
|
21
138
|
def sanitize_name(name: str) -> str:
|
|
22
139
|
"""将名称转为合法的标识符,中文等非ASCII字符做 transliterate"""
|
|
23
140
|
sanitized = re.sub(r'[^a-zA-Z0-9_]', '_', name).strip('_')
|
|
@@ -144,6 +261,115 @@ def _make_user_id_interceptor(user_id: str):
|
|
|
144
261
|
return inject_user_id
|
|
145
262
|
|
|
146
263
|
|
|
264
|
+
# 单个 MCP 服务器连接超时(秒)。慢/不健康的服务器不应拖垮整批并发加载。
|
|
265
|
+
_MCP_SERVER_TIMEOUT = 10
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
async def _load_one_server(
|
|
269
|
+
config: "MCPServerConfig",
|
|
270
|
+
interceptors: list,
|
|
271
|
+
tool_display_names: Optional[dict],
|
|
272
|
+
on_tool_success=None,
|
|
273
|
+
on_tool_error=None,
|
|
274
|
+
on_batch_available=None,
|
|
275
|
+
on_server_failure=None,
|
|
276
|
+
) -> List[BaseTool]:
|
|
277
|
+
"""连接单个 MCP 服务器并加载其工具。
|
|
278
|
+
|
|
279
|
+
独立的异常隔离单元:单个服务器失败/超时返回空列表,不拖累其他服务器。
|
|
280
|
+
被并发调度(load_mcp_tools 里 asyncio.gather)。
|
|
281
|
+
|
|
282
|
+
慢的根因是「建连 + MCP initialize + list_tools」(每次 create_session 都重建)。
|
|
283
|
+
这一步用 _discover_with_cache 命中缓存(进程级 + 单飞),命中即跳过建连。
|
|
284
|
+
后续 LangChain BaseTool 转换(per-user interceptor + display_name 注入)是纯内存操作,
|
|
285
|
+
不缓存,每次都做——保持 per-user 隔离正确。
|
|
286
|
+
"""
|
|
287
|
+
try:
|
|
288
|
+
from langchain_mcp_adapters.tools import convert_mcp_tool_to_langchain_tool
|
|
289
|
+
|
|
290
|
+
key = sanitize_name(config.name)
|
|
291
|
+
client_config = {
|
|
292
|
+
key: {
|
|
293
|
+
"url": config.server_url,
|
|
294
|
+
"transport": "streamable_http",
|
|
295
|
+
**({"headers": config.headers} if config.headers else {}),
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
# 🔑 命中缓存则跳过建连+握手+list_tools(毫秒级),未命中才真正连一次
|
|
300
|
+
raw_tools = await _discover_with_cache(config)
|
|
301
|
+
|
|
302
|
+
original_names = []
|
|
303
|
+
server_tools: List[BaseTool] = []
|
|
304
|
+
for tool in raw_tools:
|
|
305
|
+
original_name = tool.name
|
|
306
|
+
original_names.append(original_name)
|
|
307
|
+
|
|
308
|
+
lc_tool = convert_mcp_tool_to_langchain_tool(
|
|
309
|
+
None,
|
|
310
|
+
tool,
|
|
311
|
+
connection=client_config[key],
|
|
312
|
+
tool_interceptors=interceptors if interceptors else None,
|
|
313
|
+
server_name=config.name,
|
|
314
|
+
tool_name_prefix=False,
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
lc_tool.name = f"mcp__{key}__{original_name}"
|
|
318
|
+
if lc_tool.description and not lc_tool.description.startswith("[MCP"):
|
|
319
|
+
lc_tool.description = f"[MCP:{config.name}] {lc_tool.description}"
|
|
320
|
+
# 注入中文别名到 description,让 LLM 能识别用户说的中文名
|
|
321
|
+
if tool_display_names:
|
|
322
|
+
dn = tool_display_names.get(lc_tool.name)
|
|
323
|
+
if dn:
|
|
324
|
+
lc_tool.description = f"[别名: {dn}] {lc_tool.description}"
|
|
325
|
+
lc_tool = wrap_tool_with_error_handler(
|
|
326
|
+
lc_tool, config.id, config.name, original_name,
|
|
327
|
+
on_tool_success=on_tool_success,
|
|
328
|
+
on_tool_error=on_tool_error,
|
|
329
|
+
)
|
|
330
|
+
server_tools.append(lc_tool)
|
|
331
|
+
|
|
332
|
+
if on_batch_available:
|
|
333
|
+
try:
|
|
334
|
+
await on_batch_available(config.id, original_names)
|
|
335
|
+
except Exception as e:
|
|
336
|
+
SYLogger.warning(f"[MCP] 写入工具状态失败: {e}")
|
|
337
|
+
|
|
338
|
+
SYLogger.info(
|
|
339
|
+
f"[MCP] 服务器 '{config.name}' 加载了 {len(raw_tools)} 个工具")
|
|
340
|
+
return server_tools
|
|
341
|
+
|
|
342
|
+
except asyncio.TimeoutError:
|
|
343
|
+
SYLogger.warning(
|
|
344
|
+
f"[MCP] 服务器 '{config.name}' ({config.server_url}) "
|
|
345
|
+
f"连接超时(>{_MCP_SERVER_TIMEOUT}s),跳过")
|
|
346
|
+
if on_server_failure:
|
|
347
|
+
try:
|
|
348
|
+
await on_server_failure(config.id, f"连接超时(>{_MCP_SERVER_TIMEOUT}s)")
|
|
349
|
+
except Exception:
|
|
350
|
+
pass
|
|
351
|
+
return []
|
|
352
|
+
except asyncio.CancelledError:
|
|
353
|
+
# 单飞失败透传的哨兵(_discover_with_cache 失败时唤醒等待者用),按失败处理
|
|
354
|
+
SYLogger.warning(
|
|
355
|
+
f"[MCP] 服务器 '{config.name}' ({config.server_url}) 发现失败,跳过")
|
|
356
|
+
if on_server_failure:
|
|
357
|
+
try:
|
|
358
|
+
await on_server_failure(config.id, "发现失败")
|
|
359
|
+
except Exception:
|
|
360
|
+
pass
|
|
361
|
+
return []
|
|
362
|
+
except Exception as e:
|
|
363
|
+
SYLogger.warning(
|
|
364
|
+
f"[MCP] 服务器 '{config.name}' ({config.server_url}) 连接失败,跳过: {e}")
|
|
365
|
+
if on_server_failure:
|
|
366
|
+
try:
|
|
367
|
+
await on_server_failure(config.id, str(e)[:300])
|
|
368
|
+
except Exception:
|
|
369
|
+
pass
|
|
370
|
+
return []
|
|
371
|
+
|
|
372
|
+
|
|
147
373
|
async def load_mcp_tools(
|
|
148
374
|
configs: List[MCPServerConfig],
|
|
149
375
|
user_id: str = None,
|
|
@@ -156,7 +382,7 @@ async def load_mcp_tools(
|
|
|
156
382
|
) -> List[BaseTool]:
|
|
157
383
|
"""加载 MCP 工具列表
|
|
158
384
|
|
|
159
|
-
接受 MCPServerConfig
|
|
385
|
+
接受 MCPServerConfig 列表,并发连接所有服务器并加载工具。
|
|
160
386
|
|
|
161
387
|
Args:
|
|
162
388
|
configs: MCP 服务器配置列表(仅 enabled 的会被处理)
|
|
@@ -176,72 +402,20 @@ async def load_mcp_tools(
|
|
|
176
402
|
if user_id:
|
|
177
403
|
interceptors.append(_make_user_id_interceptor(user_id))
|
|
178
404
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
client = MultiServerMCPClient(client_config)
|
|
194
|
-
|
|
195
|
-
async with client.session(key) as session:
|
|
196
|
-
from langchain_mcp_adapters.tools import _list_all_tools, convert_mcp_tool_to_langchain_tool
|
|
197
|
-
raw_tools = await _list_all_tools(session)
|
|
198
|
-
|
|
199
|
-
original_names = []
|
|
200
|
-
for tool in raw_tools:
|
|
201
|
-
original_name = tool.name
|
|
202
|
-
original_names.append(original_name)
|
|
203
|
-
|
|
204
|
-
lc_tool = convert_mcp_tool_to_langchain_tool(
|
|
205
|
-
None,
|
|
206
|
-
tool,
|
|
207
|
-
connection=client_config[key],
|
|
208
|
-
tool_interceptors=interceptors if interceptors else None,
|
|
209
|
-
server_name=config.name,
|
|
210
|
-
tool_name_prefix=False,
|
|
211
|
-
)
|
|
212
|
-
|
|
213
|
-
lc_tool.name = f"mcp__{key}__{original_name}"
|
|
214
|
-
if lc_tool.description and not lc_tool.description.startswith("[MCP"):
|
|
215
|
-
lc_tool.description = f"[MCP:{config.name}] {lc_tool.description}"
|
|
216
|
-
# 注入中文别名到 description,让 LLM 能识别用户说的中文名
|
|
217
|
-
if tool_display_names:
|
|
218
|
-
dn = tool_display_names.get(lc_tool.name)
|
|
219
|
-
if dn:
|
|
220
|
-
lc_tool.description = f"[别名: {dn}] {lc_tool.description}"
|
|
221
|
-
lc_tool = wrap_tool_with_error_handler(
|
|
222
|
-
lc_tool, config.id, config.name, original_name,
|
|
223
|
-
on_tool_success=on_tool_success,
|
|
224
|
-
on_tool_error=on_tool_error,
|
|
225
|
-
)
|
|
226
|
-
all_tools.append(lc_tool)
|
|
227
|
-
|
|
228
|
-
if on_batch_available:
|
|
229
|
-
try:
|
|
230
|
-
await on_batch_available(config.id, original_names)
|
|
231
|
-
except Exception as e:
|
|
232
|
-
SYLogger.warning(f"[MCP] 写入工具状态失败: {e}")
|
|
233
|
-
|
|
234
|
-
SYLogger.info(
|
|
235
|
-
f"[MCP] 服务器 '{config.name}' 加载了 {len(raw_tools)} 个工具")
|
|
236
|
-
|
|
237
|
-
except Exception as e:
|
|
238
|
-
SYLogger.warning(
|
|
239
|
-
f"[MCP] 服务器 '{config.name}' ({config.server_url}) 连接失败,跳过: {e}")
|
|
240
|
-
if on_server_failure:
|
|
241
|
-
try:
|
|
242
|
-
await on_server_failure(config.id, str(e)[:300])
|
|
243
|
-
except Exception:
|
|
244
|
-
pass
|
|
405
|
+
# 并发连接所有服务器:总耗时 ≈ max(单服务器),而非 Σ(单服务器)。
|
|
406
|
+
# 每个服务器在 _load_one_server 内独立异常隔离 + 超时兜底。
|
|
407
|
+
per_server_results = await asyncio.gather(*[
|
|
408
|
+
_load_one_server(
|
|
409
|
+
config, interceptors, tool_display_names,
|
|
410
|
+
on_tool_success, on_tool_error, on_batch_available, on_server_failure,
|
|
411
|
+
)
|
|
412
|
+
for config in enabled_configs
|
|
413
|
+
], return_exceptions=False)
|
|
414
|
+
|
|
415
|
+
all_tools: List[BaseTool] = []
|
|
416
|
+
for tools in per_server_results:
|
|
417
|
+
if tools:
|
|
418
|
+
all_tools.extend(tools)
|
|
245
419
|
|
|
246
420
|
if all_tools:
|
|
247
421
|
SYLogger.info(f"[MCP] 共加载 {len(all_tools)} 个 MCP 工具")
|
|
@@ -24,6 +24,10 @@ from sycommon.logging.kafka_log import SYLogger
|
|
|
24
24
|
# 日期快照功能预留,当前暂不启用
|
|
25
25
|
_SNAPSHOT_ENABLED = False
|
|
26
26
|
|
|
27
|
+
# restore_from_minio 的 MinIO 下载并发度(同时下载的文件数上限)。
|
|
28
|
+
# 与 _sync_batch(sync→minio 方向)的 batch_size 对齐为 5,避免压垮 MinIO/网络。
|
|
29
|
+
_RESTORE_CONCURRENCY = 5
|
|
30
|
+
|
|
27
31
|
|
|
28
32
|
class MinioSyncService(metaclass=SingletonMeta):
|
|
29
33
|
"""MinIO 文件同步服务(单例)"""
|
|
@@ -759,12 +763,15 @@ class MinioSyncService(metaclass=SingletonMeta):
|
|
|
759
763
|
|
|
760
764
|
SYLogger.info(f"[MinIO] 需恢复 {len(to_restore)} 个文件到沙箱: user={user_id}")
|
|
761
765
|
|
|
762
|
-
# 4.
|
|
766
|
+
# 4. 并发恢复:MinIO 下载用全局 Semaphore(5) 限流并发,沙箱上传仍按批 batch_upload。
|
|
767
|
+
# 旧实现是「批内串行下载 + 批间串行」,N 个文件 ≈ N 次串行 MinIO GET,27 个文件要数十秒。
|
|
768
|
+
# 改为 Semaphore(5) 后下载并发,总耗时 ≈ ceil(N/5) 次串行往返,对齐 _sync_batch 的并发度。
|
|
763
769
|
restored = 0
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
770
|
+
download_sem = asyncio.Semaphore(_RESTORE_CONCURRENCY)
|
|
771
|
+
|
|
772
|
+
async def _download_one(rel_path: str):
|
|
773
|
+
"""下载单个 MinIO 对象到内存,返回 (沙箱绝对路径, 内容) 或 None。"""
|
|
774
|
+
async with download_sem:
|
|
768
775
|
try:
|
|
769
776
|
def _download_object(bucket, key):
|
|
770
777
|
resp = self._client.get_object(bucket, key)
|
|
@@ -777,21 +784,32 @@ class MinioSyncService(metaclass=SingletonMeta):
|
|
|
777
784
|
content = await asyncio.to_thread(
|
|
778
785
|
_download_object, self._bucket, f"{prefix}{rel_path}"
|
|
779
786
|
)
|
|
780
|
-
|
|
787
|
+
return (f"/{rel_path}", content)
|
|
781
788
|
except Exception as e:
|
|
782
789
|
SYLogger.warning(f"[MinIO] 下载文件失败: {rel_path}, error={e}")
|
|
790
|
+
return None
|
|
783
791
|
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
792
|
+
# 全量并发下载(受 Semaphore 限流),比按批串行快得多
|
|
793
|
+
download_results = await asyncio.gather(
|
|
794
|
+
*[_download_one(rp) for rp in to_restore]
|
|
795
|
+
)
|
|
796
|
+
downloaded = [r for r in download_results if r is not None]
|
|
797
|
+
|
|
798
|
+
# 上传仍走 batch_upload(服务端并发),按 5 个一批发(沙箱端点对超大批量可能限流)
|
|
799
|
+
for i in range(0, len(downloaded), _RESTORE_CONCURRENCY):
|
|
800
|
+
batch = downloaded[i:i + _RESTORE_CONCURRENCY]
|
|
801
|
+
if not batch:
|
|
802
|
+
continue
|
|
803
|
+
try:
|
|
804
|
+
results = await sandbox_backend.aupload_files(batch)
|
|
805
|
+
for r in results:
|
|
806
|
+
if not r.error:
|
|
807
|
+
restored += 1
|
|
808
|
+
else:
|
|
809
|
+
SYLogger.warning(
|
|
810
|
+
f"[MinIO] 上传到沙箱失败: {r.path}, error={r.error}")
|
|
811
|
+
except Exception as e:
|
|
812
|
+
SYLogger.warning(f"[MinIO] 批量上传到沙箱失败: {e}")
|
|
795
813
|
|
|
796
814
|
SYLogger.info(
|
|
797
815
|
f"[MinIO] 恢复完成: user={user_id}, restored={restored}/{len(to_restore)}")
|
|
@@ -136,7 +136,7 @@ sycommon/services.py,sha256=iC73h5d5bpNqW2qjRUGODZFhUhcwvKVnKUFLKJlBSq0,25322
|
|
|
136
136
|
sycommon/agent/__init__.py,sha256=mxceAeUifQ-DKvWp7ZEJIFlmOCb5wpYHPGQw3rwEN8I,4378
|
|
137
137
|
sycommon/agent/agent_manager.py,sha256=UhhaekEumT7g4v_Z1UB4jTp13X0n8M8erYaQdkGGWkA,13620
|
|
138
138
|
sycommon/agent/chat_events.py,sha256=t7qWa6OrIWLfqtd1AnqaVP67QWkn1JxpCBTZYQpoLuM,14226
|
|
139
|
-
sycommon/agent/deep_agent.py,sha256=
|
|
139
|
+
sycommon/agent/deep_agent.py,sha256=uLqhiBLcNQlmaYcg0L6dreJGKHDJ4XFU7GWjuVioIkY,78708
|
|
140
140
|
sycommon/agent/multi_agent_team.py,sha256=RFXHB9q5j1_2B-MDZVRTcbXHdEpRIXpAhrm1yroUDYI,30460
|
|
141
141
|
sycommon/agent/summarization_utils.py,sha256=bqZRFqSekNGlv2mYQPaT_s2QNA5VDTKddi2Va_SkHWc,18376
|
|
142
142
|
sycommon/agent/acp/__init__.py,sha256=vQGMQYAA5-ZaejYmDZ4r4Fklqs17qH3cDXQ5pZlwj-0,1844
|
|
@@ -146,7 +146,7 @@ sycommon/agent/acp/ssh_init.py,sha256=5y1kIyNHpsh9ayx9RWO5gS0GOOn2GfZpU4fNrGuhOY
|
|
|
146
146
|
sycommon/agent/acp/tool.py,sha256=yt6ohKL2y7wIZMNiLv-m5oCg5u1p5GAbqgjzQVt285Y,10975
|
|
147
147
|
sycommon/agent/mcp/__init__.py,sha256=iKrdDhIrFsNIkqG_kgcwNe-nOiM6uVfolKv44LfQ-FQ,636
|
|
148
148
|
sycommon/agent/mcp/models.py,sha256=zda4ho-UTYOJ5oPAZPZ-vxhfDRLghUCsMroHnA2A2QQ,1490
|
|
149
|
-
sycommon/agent/mcp/tool_loader.py,sha256=
|
|
149
|
+
sycommon/agent/mcp/tool_loader.py,sha256=B0dCaRKm2WGGsK6d9TIp4zmuun6QxPwXCVFY7drKEkA,17824
|
|
150
150
|
sycommon/agent/middleware/sandbox_path_guard.py,sha256=K1SnxA8pDxkCSucOkranwUvC9_8rd3l_-TIR8gMMTK8,8102
|
|
151
151
|
sycommon/agent/middleware/sensitive_guard.py,sha256=8e7qSAoak0MNB5fRLscQF0PVGZ07SKdwmGk590GubOI,22870
|
|
152
152
|
sycommon/agent/middleware/sitecustomize.py,sha256=Bt7XFnWV_LJo89l858AGp5VUkXwefpLGmjkCtLe-CMw,4800
|
|
@@ -157,7 +157,7 @@ sycommon/agent/sandbox/__init__.py,sha256=jR7LlkD4J4Y6QYyRXQClkwmqDBCCPmycV_hQV9
|
|
|
157
157
|
sycommon/agent/sandbox/consistent_hash.py,sha256=8Jgk-W4NAD2-u5_vKRVlPmql_e0Vy4aNORiXOuzvIrs,7277
|
|
158
158
|
sycommon/agent/sandbox/file_ops.py,sha256=VTK7xduMMWLVTr9JViMnmPeh7s8Ojr8YwYA4ovMZZg0,37138
|
|
159
159
|
sycommon/agent/sandbox/http_sandbox_backend.py,sha256=1Cj_JYdvwP3PFPa_4xFPnrGdt67Wp5iQdoziSzjeqoE,70481
|
|
160
|
-
sycommon/agent/sandbox/minio_sync.py,sha256=
|
|
160
|
+
sycommon/agent/sandbox/minio_sync.py,sha256=c2TQmnawlpfTyEalSmcIRSK4DNkrZluq-9SjoySrf-M,34515
|
|
161
161
|
sycommon/agent/sandbox/sandbox_pool.py,sha256=eMn8sLakCWf90l6ni2-333QM8oBdX1CflV-WzneFp_k,9133
|
|
162
162
|
sycommon/agent/sandbox/sandbox_recovery.py,sha256=X-eDODx1tmGMh_iTngV6e1ppfDBHpTdkPreJusN5MHY,7358
|
|
163
163
|
sycommon/agent/sandbox/session.py,sha256=TjzC3yFC-VaJ75UwCyL26QX4PRTGNNfQae1FKFuOsYI,2365
|
|
@@ -308,8 +308,8 @@ sycommon/tools/syemail.py,sha256=BDFhgf7WDOQeTcjxJEQdu0dQhnHFPO_p3eI0-Ni3LhQ,561
|
|
|
308
308
|
sycommon/tools/timing.py,sha256=OiiE7P07lRoMzX9kzb8sZU9cDb0zNnqIlY5pWqHcnkY,2064
|
|
309
309
|
sycommon/xxljob/__init__.py,sha256=7eoBlQxv-B39IfRSCY2bkqdGYs1QRe1umAWd88VMEEM,86
|
|
310
310
|
sycommon/xxljob/xxljob_service.py,sha256=1yifwIBNGsCIxLnQjHKiBlbsigc_zvPH-dMTZcNxe-Q,7649
|
|
311
|
-
sycommon_python_lib-0.2.
|
|
312
|
-
sycommon_python_lib-0.2.
|
|
313
|
-
sycommon_python_lib-0.2.
|
|
314
|
-
sycommon_python_lib-0.2.
|
|
315
|
-
sycommon_python_lib-0.2.
|
|
311
|
+
sycommon_python_lib-0.2.7a30.dist-info/METADATA,sha256=A_VXT1ygWDo4Hkvwonq65az8FeY0k_8ik6hkReVPKWY,7962
|
|
312
|
+
sycommon_python_lib-0.2.7a30.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
313
|
+
sycommon_python_lib-0.2.7a30.dist-info/entry_points.txt,sha256=gsR4SssKxDWjRU8ggidzNcdMXDPRSKRS7UaGyNP84Qg,92
|
|
314
|
+
sycommon_python_lib-0.2.7a30.dist-info/top_level.txt,sha256=RgphKrg7nJyZ7irJqbxFr-5H2LUYTvI7ivoWZH2hcD0,29
|
|
315
|
+
sycommon_python_lib-0.2.7a30.dist-info/RECORD,,
|
|
File without changes
|
{sycommon_python_lib-0.2.7a28.dist-info → sycommon_python_lib-0.2.7a30.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.7a28.dist-info → sycommon_python_lib-0.2.7a30.dist-info}/top_level.txt
RENAMED
|
File without changes
|