sycommon-python-lib 0.2.7a28__py3-none-any.whl → 0.2.7a29__py3-none-any.whl

sycommon/agent/mcp/tool_loader.py

@@ -6,8 +6,9 @@
 
 import asyncio
 import re
+import time
 import hashlib
-from typing import List, Optional, Callable, Awaitable, TYPE_CHECKING
+from typing import List, Optional, Callable, Awaitable, Tuple, TYPE_CHECKING
 
 from langchain_core.tools import BaseTool
 
@@ -18,6 +19,122 @@ if TYPE_CHECKING:
     from langchain_mcp_adapters.interceptors import MCPToolCallRequest
 
 
+# ============== MCP 工具发现缓存 ==============
+# 缓存的是「发现的 raw MCP tools」(mcp.types.Tool 列表)，用户无关、可安全共享。
+# 命中后跳过 TCP/TLS 握手 + MCP initialize + list_tools（这才是慢的根因——
+# 每次 create_session 都重建连接）。LangChain BaseTool 转换(per-user interceptor
+# + display_name 注入)仍在每次调用做，但那是纯内存操作，毫秒级。
+# 失效：TTL 过期 / 配置变更(create/update/delete) 主动 invalidate。
+_MCP_DISCOVERY_TTL = 300  # 5 分钟
+# key = (server_url, headers_repr)；value = (raw_tools, expire_ts)
+_mcp_discovery_cache: dict[Tuple[str, str], Tuple[list, float]] = {}
+# 单飞锁：同一 (url, headers) 并发发现只跑一次，其余等结果
+_mcp_inflight: dict[Tuple[str, str], asyncio.Future] = {}
+_mcp_cache_lock = asyncio.Lock()
+
+
+def _discovery_key(config: "MCPServerConfig") -> Tuple[str, str]:
+    """计算发现缓存 key。headers 影响鉴权 → 影响可见工具，必须纳入 key。"""
+    headers_repr = ""
+    if config.headers:
+        # 排序保证 dict 顺序不影响 key
+        try:
+            headers_repr = repr(sorted(config.headers.items()))
+        except Exception:
+            headers_repr = str(config.headers)
+    return (config.server_url, headers_repr)
+
+
+async def invalidate_mcp_cache(server_url: str = None) -> None:
+    """使发现缓存失效。配置变更(create/update/delete)时调用。
+
+    Args:
+        server_url: 指定则只失效该 url 的条目；None 清空全部。
+    """
+    async with _mcp_cache_lock:
+        if server_url is None:
+            n = len(_mcp_discovery_cache)
+            _mcp_discovery_cache.clear()
+            if n:
+                SYLogger.info(f"[MCP] 发现缓存已全部清空({n} 条)")
+        else:
+            # 按第一维(url)匹配，清除该 url 所有 headers 变体
+            keys_to_drop = [k for k in _mcp_discovery_cache if k[0] == server_url]
+            for k in keys_to_drop:
+                _mcp_discovery_cache.pop(k, None)
+            if keys_to_drop:
+                SYLogger.info(f"[MCP] 发现缓存已失效: {server_url}")
+
+
+async def _discover_with_cache(config: "MCPServerConfig") -> list:
+    """带缓存 + 单飞地发现单个 server 的 raw MCP tools。
+
+    命中缓存(未过期) → 直接返回。未命中 → 并发请求中已有 in-flight future → await 它；
+    否则自己建立连接发现，写回缓存。
+    """
+    from langchain_mcp_adapters.client import MultiServerMCPClient
+    from langchain_mcp_adapters.tools import _list_all_tools
+
+    key = sanitize_name(config.name)
+    cache_key = _discovery_key(config)
+
+    # 1. 查缓存（无锁快路径，dict 读本身线程安全足够）
+    cached = _mcp_discovery_cache.get(cache_key)
+    if cached is not None:
+        raw_tools, expire_ts = cached
+        if time.monotonic() < expire_ts:
+            SYLogger.debug(f"[MCP] 发现缓存命中: {config.name} ({len(raw_tools)} tools)")
+            return raw_tools
+        # 过期，清掉
+        _mcp_discovery_cache.pop(cache_key, None)
+
+    # 2. 单飞：同一 key 并发只发一个请求
+    is_leader = False
+    async with _mcp_cache_lock:
+        inflight = _mcp_inflight.get(cache_key)
+        if inflight is not None:
+            waiter = inflight
+        else:
+            waiter = asyncio.get_running_loop().create_future()
+            _mcp_inflight[cache_key] = waiter
+            is_leader = True
+    if not is_leader:
+        # 有并发请求在跑，等它的结果（不重复建连）
+        return await asyncio.shield(waiter)
+
+    # 3. 自己建连发现（leader）
+    try:
+        client_config = {
+            key: {
+                "url": config.server_url,
+                "transport": "streamable_http",
+                **({"headers": config.headers} if config.headers else {}),
+            }
+        }
+        client = MultiServerMCPClient(client_config)
+
+        async def _connect_and_list():
+            async with client.session(key) as session:
+                return await _list_all_tools(session)
+
+        raw_tools = await asyncio.wait_for(
+            _connect_and_list(), timeout=_MCP_SERVER_TIMEOUT)
+
+        # 写回缓存
+        _mcp_discovery_cache[cache_key] = (raw_tools, time.monotonic() + _MCP_DISCOVERY_TTL)
+        # 唤醒所有等待者
+        if not waiter.done():
+            waiter.set_result(raw_tools)
+        return raw_tools
+    except BaseException as e:
+        # 失败不写缓存，把真实异常透传给所有等待者（各自在 _load_one_server 里兜底返回 []）
+        if not waiter.done():
+            waiter.set_exception(RuntimeError(f"MCP 发现失败: {e}"))
+        raise
+    finally:
+        _mcp_inflight.pop(cache_key, None)
+
+
 def sanitize_name(name: str) -> str:
     """将名称转为合法的标识符，中文等非ASCII字符做 transliterate"""
     sanitized = re.sub(r'[^a-zA-Z0-9_]', '_', name).strip('_')
@@ -144,6 +261,115 @@ def _make_user_id_interceptor(user_id: str):
     return inject_user_id
 
 
+# 单个 MCP 服务器连接超时(秒)。慢/不健康的服务器不应拖垮整批并发加载。
+_MCP_SERVER_TIMEOUT = 10
+
+
+async def _load_one_server(
+    config: "MCPServerConfig",
+    interceptors: list,
+    tool_display_names: Optional[dict],
+    on_tool_success=None,
+    on_tool_error=None,
+    on_batch_available=None,
+    on_server_failure=None,
+) -> List[BaseTool]:
+    """连接单个 MCP 服务器并加载其工具。
+
+    独立的异常隔离单元：单个服务器失败/超时返回空列表，不拖累其他服务器。
+    被并发调度（load_mcp_tools 里 asyncio.gather）。
+
+    慢的根因是「建连 + MCP initialize + list_tools」（每次 create_session 都重建）。
+    这一步用 _discover_with_cache 命中缓存(进程级 + 单飞)，命中即跳过建连。
+    后续 LangChain BaseTool 转换(per-user interceptor + display_name 注入)是纯内存操作，
+    不缓存，每次都做——保持 per-user 隔离正确。
+    """
+    try:
+        from langchain_mcp_adapters.tools import convert_mcp_tool_to_langchain_tool
+
+        key = sanitize_name(config.name)
+        client_config = {
+            key: {
+                "url": config.server_url,
+                "transport": "streamable_http",
+                **({"headers": config.headers} if config.headers else {}),
+            }
+        }
+
+        # 🔑 命中缓存则跳过建连+握手+list_tools（毫秒级），未命中才真正连一次
+        raw_tools = await _discover_with_cache(config)
+
+        original_names = []
+        server_tools: List[BaseTool] = []
+        for tool in raw_tools:
+            original_name = tool.name
+            original_names.append(original_name)
+
+            lc_tool = convert_mcp_tool_to_langchain_tool(
+                None,
+                tool,
+                connection=client_config[key],
+                tool_interceptors=interceptors if interceptors else None,
+                server_name=config.name,
+                tool_name_prefix=False,
+            )
+
+            lc_tool.name = f"mcp__{key}__{original_name}"
+            if lc_tool.description and not lc_tool.description.startswith("[MCP"):
+                lc_tool.description = f"[MCP:{config.name}] {lc_tool.description}"
+            # 注入中文别名到 description，让 LLM 能识别用户说的中文名
+            if tool_display_names:
+                dn = tool_display_names.get(lc_tool.name)
+                if dn:
+                    lc_tool.description = f"[别名: {dn}] {lc_tool.description}"
+            lc_tool = wrap_tool_with_error_handler(
+                lc_tool, config.id, config.name, original_name,
+                on_tool_success=on_tool_success,
+                on_tool_error=on_tool_error,
+            )
+            server_tools.append(lc_tool)
+
+        if on_batch_available:
+            try:
+                await on_batch_available(config.id, original_names)
+            except Exception as e:
+                SYLogger.warning(f"[MCP] 写入工具状态失败: {e}")
+
+        SYLogger.info(
+            f"[MCP] 服务器 '{config.name}' 加载了 {len(raw_tools)} 个工具")
+        return server_tools
+
+    except asyncio.TimeoutError:
+        SYLogger.warning(
+            f"[MCP] 服务器 '{config.name}' ({config.server_url}) "
+            f"连接超时(>{_MCP_SERVER_TIMEOUT}s)，跳过")
+        if on_server_failure:
+            try:
+                await on_server_failure(config.id, f"连接超时(>{_MCP_SERVER_TIMEOUT}s)")
+            except Exception:
+                pass
+        return []
+    except asyncio.CancelledError:
+        # 单飞失败透传的哨兵（_discover_with_cache 失败时唤醒等待者用），按失败处理
+        SYLogger.warning(
+            f"[MCP] 服务器 '{config.name}' ({config.server_url}) 发现失败，跳过")
+        if on_server_failure:
+            try:
+                await on_server_failure(config.id, "发现失败")
+            except Exception:
+                pass
+        return []
+    except Exception as e:
+        SYLogger.warning(
+            f"[MCP] 服务器 '{config.name}' ({config.server_url}) 连接失败，跳过: {e}")
+        if on_server_failure:
+            try:
+                await on_server_failure(config.id, str(e)[:300])
+            except Exception:
+                pass
+        return []
+
+
 async def load_mcp_tools(
     configs: List[MCPServerConfig],
     user_id: str = None,
@@ -156,7 +382,7 @@ async def load_mcp_tools(
 ) -> List[BaseTool]:
     """加载 MCP 工具列表
 
-    接受 MCPServerConfig 列表，逐个服务器连接并加载工具。
+    接受 MCPServerConfig 列表，并发连接所有服务器并加载工具。
 
     Args:
         configs: MCP 服务器配置列表（仅 enabled 的会被处理）
@@ -176,72 +402,20 @@ async def load_mcp_tools(
     if user_id:
         interceptors.append(_make_user_id_interceptor(user_id))
 
-    all_tools = []
-    for config in enabled_configs:
-        try:
-            from langchain_mcp_adapters.client import MultiServerMCPClient
-
-            key = sanitize_name(config.name)
-            client_config = {
-                key: {
-                    "url": config.server_url,
-                    "transport": "streamable_http",
-                    **({"headers": config.headers} if config.headers else {}),
-                }
-            }
-
-            client = MultiServerMCPClient(client_config)
-
-            async with client.session(key) as session:
-                from langchain_mcp_adapters.tools import _list_all_tools, convert_mcp_tool_to_langchain_tool
-                raw_tools = await _list_all_tools(session)
-
-                original_names = []
-                for tool in raw_tools:
-                    original_name = tool.name
-                    original_names.append(original_name)
-
-                    lc_tool = convert_mcp_tool_to_langchain_tool(
-                        None,
-                        tool,
-                        connection=client_config[key],
-                        tool_interceptors=interceptors if interceptors else None,
-                        server_name=config.name,
-                        tool_name_prefix=False,
-                    )
-
-                    lc_tool.name = f"mcp__{key}__{original_name}"
-                    if lc_tool.description and not lc_tool.description.startswith("[MCP"):
-                        lc_tool.description = f"[MCP:{config.name}] {lc_tool.description}"
-                    # 注入中文别名到 description，让 LLM 能识别用户说的中文名
-                    if tool_display_names:
-                        dn = tool_display_names.get(lc_tool.name)
-                        if dn:
-                            lc_tool.description = f"[别名: {dn}] {lc_tool.description}"
-                    lc_tool = wrap_tool_with_error_handler(
-                        lc_tool, config.id, config.name, original_name,
-                        on_tool_success=on_tool_success,
-                        on_tool_error=on_tool_error,
-                    )
-                    all_tools.append(lc_tool)
-
-            if on_batch_available:
-                try:
-                    await on_batch_available(config.id, original_names)
-                except Exception as e:
-                    SYLogger.warning(f"[MCP] 写入工具状态失败: {e}")
-
-            SYLogger.info(
-                f"[MCP] 服务器 '{config.name}' 加载了 {len(raw_tools)} 个工具")
-
-        except Exception as e:
-            SYLogger.warning(
-                f"[MCP] 服务器 '{config.name}' ({config.server_url}) 连接失败，跳过: {e}")
-            if on_server_failure:
-                try:
-                    await on_server_failure(config.id, str(e)[:300])
-                except Exception:
-                    pass
+    # 并发连接所有服务器：总耗时 ≈ max(单服务器)，而非 Σ(单服务器)。
+    # 每个服务器在 _load_one_server 内独立异常隔离 + 超时兜底。
+    per_server_results = await asyncio.gather(*[
+        _load_one_server(
+            config, interceptors, tool_display_names,
+            on_tool_success, on_tool_error, on_batch_available, on_server_failure,
+        )
+        for config in enabled_configs
+    ], return_exceptions=False)
+
+    all_tools: List[BaseTool] = []
+    for tools in per_server_results:
+        if tools:
+            all_tools.extend(tools)
 
     if all_tools:
         SYLogger.info(f"[MCP] 共加载 {len(all_tools)} 个 MCP 工具")

sycommon/agent/sandbox/minio_sync.py

@@ -24,6 +24,10 @@ from sycommon.logging.kafka_log import SYLogger
 # 日期快照功能预留，当前暂不启用
 _SNAPSHOT_ENABLED = False
 
+# restore_from_minio 的 MinIO 下载并发度（同时下载的文件数上限）。
+# 与 _sync_batch(sync→minio 方向)的 batch_size 对齐为 5，避免压垮 MinIO/网络。
+_RESTORE_CONCURRENCY = 5
+
 
 class MinioSyncService(metaclass=SingletonMeta):
     """MinIO 文件同步服务（单例）"""
@@ -759,12 +763,15 @@ class MinioSyncService(metaclass=SingletonMeta):
 
         SYLogger.info(f"[MinIO] 需恢复 {len(to_restore)} 个文件到沙箱: user={user_id}")
 
-        # 4. 批量恢复
+        # 4. 并发恢复：MinIO 下载用全局 Semaphore(5) 限流并发，沙箱上传仍按批 batch_upload。
+        # 旧实现是「批内串行下载 + 批间串行」，N 个文件 ≈ N 次串行 MinIO GET，27 个文件要数十秒。
+        # 改为 Semaphore(5) 后下载并发，总耗时 ≈ ceil(N/5) 次串行往返，对齐 _sync_batch 的并发度。
         restored = 0
-        for i in range(0, len(to_restore), 5):
-            batch = to_restore[i:i + 5]
-            files_to_upload = []
-            for rel_path in batch:
+        download_sem = asyncio.Semaphore(_RESTORE_CONCURRENCY)
+
+        async def _download_one(rel_path: str):
+            """下载单个 MinIO 对象到内存，返回 (沙箱绝对路径, 内容) 或 None。"""
+            async with download_sem:
                 try:
                     def _download_object(bucket, key):
                         resp = self._client.get_object(bucket, key)
@@ -777,21 +784,32 @@ class MinioSyncService(metaclass=SingletonMeta):
                     content = await asyncio.to_thread(
                         _download_object, self._bucket, f"{prefix}{rel_path}"
                     )
-                    files_to_upload.append((f"/{rel_path}", content))
+                    return (f"/{rel_path}", content)
                 except Exception as e:
                     SYLogger.warning(f"[MinIO] 下载文件失败: {rel_path}, error={e}")
+                    return None
 
-            if files_to_upload:
-                try:
-                    results = await sandbox_backend.aupload_files(files_to_upload)
-                    for r in results:
-                        if not r.error:
-                            restored += 1
-                        else:
-                            SYLogger.warning(
-                                f"[MinIO] 上传到沙箱失败: {r.path}, error={r.error}")
-                except Exception as e:
-                    SYLogger.warning(f"[MinIO] 批量上传到沙箱失败: {e}")
+        # 全量并发下载(受 Semaphore 限流)，比按批串行快得多
+        download_results = await asyncio.gather(
+            *[_download_one(rp) for rp in to_restore]
+        )
+        downloaded = [r for r in download_results if r is not None]
+
+        # 上传仍走 batch_upload(服务端并发)，按 5 个一批发(沙箱端点对超大批量可能限流)
+        for i in range(0, len(downloaded), _RESTORE_CONCURRENCY):
+            batch = downloaded[i:i + _RESTORE_CONCURRENCY]
+            if not batch:
+                continue
+            try:
+                results = await sandbox_backend.aupload_files(batch)
+                for r in results:
+                    if not r.error:
+                        restored += 1
+                    else:
+                        SYLogger.warning(
+                            f"[MinIO] 上传到沙箱失败: {r.path}, error={r.error}")
+            except Exception as e:
+                SYLogger.warning(f"[MinIO] 批量上传到沙箱失败: {e}")
 
         SYLogger.info(
             f"[MinIO] 恢复完成: user={user_id}, restored={restored}/{len(to_restore)}")

{sycommon_python_lib-0.2.7a28.dist-info → sycommon_python_lib-0.2.7a29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.7a28
+Version: 0.2.7a29
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.7a28.dist-info → sycommon_python_lib-0.2.7a29.dist-info}/RECORD

@@ -146,7 +146,7 @@ sycommon/agent/acp/ssh_init.py,sha256=5y1kIyNHpsh9ayx9RWO5gS0GOOn2GfZpU4fNrGuhOY
 sycommon/agent/acp/tool.py,sha256=yt6ohKL2y7wIZMNiLv-m5oCg5u1p5GAbqgjzQVt285Y,10975
 sycommon/agent/mcp/__init__.py,sha256=iKrdDhIrFsNIkqG_kgcwNe-nOiM6uVfolKv44LfQ-FQ,636
 sycommon/agent/mcp/models.py,sha256=zda4ho-UTYOJ5oPAZPZ-vxhfDRLghUCsMroHnA2A2QQ,1490
-sycommon/agent/mcp/tool_loader.py,sha256=OQa-lE7RL2s3Lqh-kNOKaUrb130hvorB_HjyFBqZBZs,10960
+sycommon/agent/mcp/tool_loader.py,sha256=B0dCaRKm2WGGsK6d9TIp4zmuun6QxPwXCVFY7drKEkA,17824
 sycommon/agent/middleware/sandbox_path_guard.py,sha256=K1SnxA8pDxkCSucOkranwUvC9_8rd3l_-TIR8gMMTK8,8102
 sycommon/agent/middleware/sensitive_guard.py,sha256=8e7qSAoak0MNB5fRLscQF0PVGZ07SKdwmGk590GubOI,22870
 sycommon/agent/middleware/sitecustomize.py,sha256=Bt7XFnWV_LJo89l858AGp5VUkXwefpLGmjkCtLe-CMw,4800
@@ -157,7 +157,7 @@ sycommon/agent/sandbox/__init__.py,sha256=jR7LlkD4J4Y6QYyRXQClkwmqDBCCPmycV_hQV9
 sycommon/agent/sandbox/consistent_hash.py,sha256=8Jgk-W4NAD2-u5_vKRVlPmql_e0Vy4aNORiXOuzvIrs,7277
 sycommon/agent/sandbox/file_ops.py,sha256=VTK7xduMMWLVTr9JViMnmPeh7s8Ojr8YwYA4ovMZZg0,37138
 sycommon/agent/sandbox/http_sandbox_backend.py,sha256=1Cj_JYdvwP3PFPa_4xFPnrGdt67Wp5iQdoziSzjeqoE,70481
-sycommon/agent/sandbox/minio_sync.py,sha256=KUVuHt43wNJxPewoTacFZii8WZNpQULEZjidQKEKKFA,33379
+sycommon/agent/sandbox/minio_sync.py,sha256=c2TQmnawlpfTyEalSmcIRSK4DNkrZluq-9SjoySrf-M,34515
 sycommon/agent/sandbox/sandbox_pool.py,sha256=eMn8sLakCWf90l6ni2-333QM8oBdX1CflV-WzneFp_k,9133
 sycommon/agent/sandbox/sandbox_recovery.py,sha256=X-eDODx1tmGMh_iTngV6e1ppfDBHpTdkPreJusN5MHY,7358
 sycommon/agent/sandbox/session.py,sha256=TjzC3yFC-VaJ75UwCyL26QX4PRTGNNfQae1FKFuOsYI,2365
@@ -308,8 +308,8 @@ sycommon/tools/syemail.py,sha256=BDFhgf7WDOQeTcjxJEQdu0dQhnHFPO_p3eI0-Ni3LhQ,561
 sycommon/tools/timing.py,sha256=OiiE7P07lRoMzX9kzb8sZU9cDb0zNnqIlY5pWqHcnkY,2064
 sycommon/xxljob/__init__.py,sha256=7eoBlQxv-B39IfRSCY2bkqdGYs1QRe1umAWd88VMEEM,86
 sycommon/xxljob/xxljob_service.py,sha256=1yifwIBNGsCIxLnQjHKiBlbsigc_zvPH-dMTZcNxe-Q,7649
-sycommon_python_lib-0.2.7a28.dist-info/METADATA,sha256=ZwQSR_5KtZWzfjtCNP0r4LmgCQ4rt0_MpssHyLnqMPM,7962
-sycommon_python_lib-0.2.7a28.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
-sycommon_python_lib-0.2.7a28.dist-info/entry_points.txt,sha256=gsR4SssKxDWjRU8ggidzNcdMXDPRSKRS7UaGyNP84Qg,92
-sycommon_python_lib-0.2.7a28.dist-info/top_level.txt,sha256=RgphKrg7nJyZ7irJqbxFr-5H2LUYTvI7ivoWZH2hcD0,29
-sycommon_python_lib-0.2.7a28.dist-info/RECORD,,
+sycommon_python_lib-0.2.7a29.dist-info/METADATA,sha256=gq42NKV6-MSlGgQftI3YmU0DiUDErLKMZZOEbMtrOGQ,7962
+sycommon_python_lib-0.2.7a29.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+sycommon_python_lib-0.2.7a29.dist-info/entry_points.txt,sha256=gsR4SssKxDWjRU8ggidzNcdMXDPRSKRS7UaGyNP84Qg,92
+sycommon_python_lib-0.2.7a29.dist-info/top_level.txt,sha256=RgphKrg7nJyZ7irJqbxFr-5H2LUYTvI7ivoWZH2hcD0,29
+sycommon_python_lib-0.2.7a29.dist-info/RECORD,,