sycommon-python-lib 0.2.7a28__tar.gz → 0.2.7a29__tar.gz

{sycommon_python_lib-0.2.7a28 → sycommon_python_lib-0.2.7a29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.7a28
+Version: 0.2.7a29
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.7a28 → sycommon_python_lib-0.2.7a29}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sycommon-python-lib"
-version = "0.2.7a28"
+version = "0.2.7a29"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.11"

sycommon_python_lib-0.2.7a29/src/sycommon/agent/mcp/tool_loader.py

@@ -0,0 +1,449 @@
+"""MCP 工具加载器
+
+使用 langchain-mcp-adapters 官方库连接远程 MCP 服务器，
+发现工具并作为 LangChain BaseTool 注入 Agent。
+"""
+
+import asyncio
+import re
+import time
+import hashlib
+from typing import List, Optional, Callable, Awaitable, Tuple, TYPE_CHECKING
+
+from langchain_core.tools import BaseTool
+
+from sycommon.logging.kafka_log import SYLogger
+from sycommon.agent.mcp.models import MCPServerConfig
+
+if TYPE_CHECKING:
+    from langchain_mcp_adapters.interceptors import MCPToolCallRequest
+
+
+# ============== MCP 工具发现缓存 ==============
+# 缓存的是「发现的 raw MCP tools」(mcp.types.Tool 列表)，用户无关、可安全共享。
+# 命中后跳过 TCP/TLS 握手 + MCP initialize + list_tools（这才是慢的根因——
+# 每次 create_session 都重建连接）。LangChain BaseTool 转换(per-user interceptor
+# + display_name 注入)仍在每次调用做，但那是纯内存操作，毫秒级。
+# 失效：TTL 过期 / 配置变更(create/update/delete) 主动 invalidate。
+_MCP_DISCOVERY_TTL = 300  # 5 分钟
+# key = (server_url, headers_repr)；value = (raw_tools, expire_ts)
+_mcp_discovery_cache: dict[Tuple[str, str], Tuple[list, float]] = {}
+# 单飞锁：同一 (url, headers) 并发发现只跑一次，其余等结果
+_mcp_inflight: dict[Tuple[str, str], asyncio.Future] = {}
+_mcp_cache_lock = asyncio.Lock()
+
+
+def _discovery_key(config: "MCPServerConfig") -> Tuple[str, str]:
+    """计算发现缓存 key。headers 影响鉴权 → 影响可见工具，必须纳入 key。"""
+    headers_repr = ""
+    if config.headers:
+        # 排序保证 dict 顺序不影响 key
+        try:
+            headers_repr = repr(sorted(config.headers.items()))
+        except Exception:
+            headers_repr = str(config.headers)
+    return (config.server_url, headers_repr)
+
+
+async def invalidate_mcp_cache(server_url: str = None) -> None:
+    """使发现缓存失效。配置变更(create/update/delete)时调用。
+
+    Args:
+        server_url: 指定则只失效该 url 的条目；None 清空全部。
+    """
+    async with _mcp_cache_lock:
+        if server_url is None:
+            n = len(_mcp_discovery_cache)
+            _mcp_discovery_cache.clear()
+            if n:
+                SYLogger.info(f"[MCP] 发现缓存已全部清空({n} 条)")
+        else:
+            # 按第一维(url)匹配，清除该 url 所有 headers 变体
+            keys_to_drop = [k for k in _mcp_discovery_cache if k[0] == server_url]
+            for k in keys_to_drop:
+                _mcp_discovery_cache.pop(k, None)
+            if keys_to_drop:
+                SYLogger.info(f"[MCP] 发现缓存已失效: {server_url}")
+
+
+async def _discover_with_cache(config: "MCPServerConfig") -> list:
+    """带缓存 + 单飞地发现单个 server 的 raw MCP tools。
+
+    命中缓存(未过期) → 直接返回。未命中 → 并发请求中已有 in-flight future → await 它；
+    否则自己建立连接发现，写回缓存。
+    """
+    from langchain_mcp_adapters.client import MultiServerMCPClient
+    from langchain_mcp_adapters.tools import _list_all_tools
+
+    key = sanitize_name(config.name)
+    cache_key = _discovery_key(config)
+
+    # 1. 查缓存（无锁快路径，dict 读本身线程安全足够）
+    cached = _mcp_discovery_cache.get(cache_key)
+    if cached is not None:
+        raw_tools, expire_ts = cached
+        if time.monotonic() < expire_ts:
+            SYLogger.debug(f"[MCP] 发现缓存命中: {config.name} ({len(raw_tools)} tools)")
+            return raw_tools
+        # 过期，清掉
+        _mcp_discovery_cache.pop(cache_key, None)
+
+    # 2. 单飞：同一 key 并发只发一个请求
+    is_leader = False
+    async with _mcp_cache_lock:
+        inflight = _mcp_inflight.get(cache_key)
+        if inflight is not None:
+            waiter = inflight
+        else:
+            waiter = asyncio.get_running_loop().create_future()
+            _mcp_inflight[cache_key] = waiter
+            is_leader = True
+    if not is_leader:
+        # 有并发请求在跑，等它的结果（不重复建连）
+        return await asyncio.shield(waiter)
+
+    # 3. 自己建连发现（leader）
+    try:
+        client_config = {
+            key: {
+                "url": config.server_url,
+                "transport": "streamable_http",
+                **({"headers": config.headers} if config.headers else {}),
+            }
+        }
+        client = MultiServerMCPClient(client_config)
+
+        async def _connect_and_list():
+            async with client.session(key) as session:
+                return await _list_all_tools(session)
+
+        raw_tools = await asyncio.wait_for(
+            _connect_and_list(), timeout=_MCP_SERVER_TIMEOUT)
+
+        # 写回缓存
+        _mcp_discovery_cache[cache_key] = (raw_tools, time.monotonic() + _MCP_DISCOVERY_TTL)
+        # 唤醒所有等待者
+        if not waiter.done():
+            waiter.set_result(raw_tools)
+        return raw_tools
+    except BaseException as e:
+        # 失败不写缓存，把真实异常透传给所有等待者（各自在 _load_one_server 里兜底返回 []）
+        if not waiter.done():
+            waiter.set_exception(RuntimeError(f"MCP 发现失败: {e}"))
+        raise
+    finally:
+        _mcp_inflight.pop(cache_key, None)
+
+
+def sanitize_name(name: str) -> str:
+    """将名称转为合法的标识符，中文等非ASCII字符做 transliterate"""
+    sanitized = re.sub(r'[^a-zA-Z0-9_]', '_', name).strip('_')
+    if not sanitized:
+        h = hashlib.md5(name.encode()).hexdigest()[:8]
+        sanitized = f"mcp_{h}"
+    return sanitized
+
+
+def _contains_auth_error(err_msg: str) -> bool:
+    """检查错误信息中是否包含认证相关关键词"""
+    auth_keywords = [
+        "401", "unauthorized", "token expired", "authentication failed",
+        "440", "login timeout", "403", "forbidden",
+        "认证失败", "登录过期", "token无效", "token过期",
+        "syt_token", "SYT_TOKEN",
+    ]
+    msg_lower = err_msg.lower()
+    return any(kw.lower() in msg_lower for kw in auth_keywords)
+
+
+def _auth_error_hint() -> str:
+    """返回认证错误时的 authorize 工具调用提示"""
+    return (
+        "\n\n⚠️ **检测到认证错误**：这可能是 SYT_TOKEN 过期或未设置导致的。"
+        "请立即调用 `authorize` 工具（传入失败原因和待重试的命令），"
+        "让用户重新授权后再自动重试。不要只是告知用户有问题。"
+    )
+
+
+def wrap_tool_with_error_handler(
+    tool: BaseTool,
+    server_config_id: str,
+    server_name: str,
+    original_tool_name: str,
+    on_tool_success: Optional[Callable[[str, str], Awaitable[None]]] = None,
+    on_tool_error: Optional[Callable[[str, str, str], Awaitable[None]]] = None,
+) -> BaseTool:
+    """包装 MCP 工具的 coroutine，捕获连接/超时等异常，返回友好的错误信息而非抛出异常。
+
+    Args:
+        on_tool_success: 异步回调 (server_config_id, tool_name) -> None
+        on_tool_error: 异步回调 (server_config_id, tool_name, error_msg) -> None
+    """
+    is_content_and_artifact = getattr(
+        tool, 'response_format', '') == 'content_and_artifact'
+    original_coroutine = tool.coroutine
+    tool_name = tool.name
+
+    MAX_RETRIES = 2
+    RETRY_DELAY_BASE = 1.0  # 秒
+
+    async def _safe_coroutine(*args, **kwargs):
+        last_exception = None
+        for attempt in range(MAX_RETRIES + 1):
+            try:
+                result = await original_coroutine(*args, **kwargs)
+                if on_tool_success:
+                    try:
+                        await on_tool_success(server_config_id, original_tool_name)
+                    except Exception:
+                        pass
+                return result
+            except Exception as e:
+                last_exception = e
+                if attempt < MAX_RETRIES:
+                    delay = RETRY_DELAY_BASE * (attempt + 1)
+                    SYLogger.info(
+                        f"[MCP] 工具 '{tool_name}' 第 {attempt + 1}/{MAX_RETRIES} 次重试 "
+                        f"({type(e).__name__}: {str(e)[:200]}), {delay}s 后重试")
+                    await asyncio.sleep(delay)
+                    continue
+
+                # 重试耗尽，返回错误
+                from langchain_core.tools import ToolException
+                err_type = type(e).__name__
+                err_msg = str(e)[:500]
+                SYLogger.warning(
+                    f"[MCP] 工具 '{tool_name}' 调用失败，已重试 {MAX_RETRIES} 次 "
+                    f"({err_type}): {err_msg}")
+
+                if on_tool_error:
+                    try:
+                        await on_tool_error(server_config_id, original_tool_name, f"{err_type}: {err_msg}")
+                    except Exception:
+                        pass
+
+                if isinstance(e, ToolException):
+                    friendly_msg = (
+                        f"MCP 工具 '{tool_name}' 调用失败（已重试 {MAX_RETRIES} 次）。\n"
+                        f"错误详情: {err_msg}\n"
+                        f"请根据错误信息判断是否可以不使用该工具继续完成任务。"
+                    )
+                else:
+                    friendly_msg = (
+                        f"MCP 工具调用失败：工具 '{tool_name}' (服务器: {server_name}) 当前不可用（已重试 {MAX_RETRIES} 次）。\n"
+                        f"错误类型: {err_type}\n"
+                        f"可能原因: MCP 服务器未启动、网络不可达或连接超时。\n"
+                        f"请尝试不使用该工具继续完成任务，或联系管理员检查 MCP 服务 '{server_name}'。"
+                    )
+
+                # 检测认证错误，追加 authorize 工具提示
+                if _contains_auth_error(err_msg):
+                    friendly_msg += _auth_error_hint()
+                if is_content_and_artifact:
+                    return friendly_msg, None
+                return [friendly_msg]
+
+    tool.coroutine = _safe_coroutine
+    return tool
+
+
+def _make_user_id_interceptor(user_id: str):
+    """创建 MCP 工具调用拦截器，注入 X-User-Id header"""
+    async def inject_user_id(
+        request: "MCPToolCallRequest",
+        handler: Callable[["MCPToolCallRequest"], Awaitable],
+    ):
+        headers = dict(request.headers or {})
+        headers["X-User-Id"] = user_id
+        request = request.override(headers=headers)
+        return await handler(request)
+
+    return inject_user_id
+
+
+# 单个 MCP 服务器连接超时(秒)。慢/不健康的服务器不应拖垮整批并发加载。
+_MCP_SERVER_TIMEOUT = 10
+
+
+async def _load_one_server(
+    config: "MCPServerConfig",
+    interceptors: list,
+    tool_display_names: Optional[dict],
+    on_tool_success=None,
+    on_tool_error=None,
+    on_batch_available=None,
+    on_server_failure=None,
+) -> List[BaseTool]:
+    """连接单个 MCP 服务器并加载其工具。
+
+    独立的异常隔离单元：单个服务器失败/超时返回空列表，不拖累其他服务器。
+    被并发调度（load_mcp_tools 里 asyncio.gather）。
+
+    慢的根因是「建连 + MCP initialize + list_tools」（每次 create_session 都重建）。
+    这一步用 _discover_with_cache 命中缓存(进程级 + 单飞)，命中即跳过建连。
+    后续 LangChain BaseTool 转换(per-user interceptor + display_name 注入)是纯内存操作，
+    不缓存，每次都做——保持 per-user 隔离正确。
+    """
+    try:
+        from langchain_mcp_adapters.tools import convert_mcp_tool_to_langchain_tool
+
+        key = sanitize_name(config.name)
+        client_config = {
+            key: {
+                "url": config.server_url,
+                "transport": "streamable_http",
+                **({"headers": config.headers} if config.headers else {}),
+            }
+        }
+
+        # 🔑 命中缓存则跳过建连+握手+list_tools（毫秒级），未命中才真正连一次
+        raw_tools = await _discover_with_cache(config)
+
+        original_names = []
+        server_tools: List[BaseTool] = []
+        for tool in raw_tools:
+            original_name = tool.name
+            original_names.append(original_name)
+
+            lc_tool = convert_mcp_tool_to_langchain_tool(
+                None,
+                tool,
+                connection=client_config[key],
+                tool_interceptors=interceptors if interceptors else None,
+                server_name=config.name,
+                tool_name_prefix=False,
+            )
+
+            lc_tool.name = f"mcp__{key}__{original_name}"
+            if lc_tool.description and not lc_tool.description.startswith("[MCP"):
+                lc_tool.description = f"[MCP:{config.name}] {lc_tool.description}"
+            # 注入中文别名到 description，让 LLM 能识别用户说的中文名
+            if tool_display_names:
+                dn = tool_display_names.get(lc_tool.name)
+                if dn:
+                    lc_tool.description = f"[别名: {dn}] {lc_tool.description}"
+            lc_tool = wrap_tool_with_error_handler(
+                lc_tool, config.id, config.name, original_name,
+                on_tool_success=on_tool_success,
+                on_tool_error=on_tool_error,
+            )
+            server_tools.append(lc_tool)
+
+        if on_batch_available:
+            try:
+                await on_batch_available(config.id, original_names)
+            except Exception as e:
+                SYLogger.warning(f"[MCP] 写入工具状态失败: {e}")
+
+        SYLogger.info(
+            f"[MCP] 服务器 '{config.name}' 加载了 {len(raw_tools)} 个工具")
+        return server_tools
+
+    except asyncio.TimeoutError:
+        SYLogger.warning(
+            f"[MCP] 服务器 '{config.name}' ({config.server_url}) "
+            f"连接超时(>{_MCP_SERVER_TIMEOUT}s)，跳过")
+        if on_server_failure:
+            try:
+                await on_server_failure(config.id, f"连接超时(>{_MCP_SERVER_TIMEOUT}s)")
+            except Exception:
+                pass
+        return []
+    except asyncio.CancelledError:
+        # 单飞失败透传的哨兵（_discover_with_cache 失败时唤醒等待者用），按失败处理
+        SYLogger.warning(
+            f"[MCP] 服务器 '{config.name}' ({config.server_url}) 发现失败，跳过")
+        if on_server_failure:
+            try:
+                await on_server_failure(config.id, "发现失败")
+            except Exception:
+                pass
+        return []
+    except Exception as e:
+        SYLogger.warning(
+            f"[MCP] 服务器 '{config.name}' ({config.server_url}) 连接失败，跳过: {e}")
+        if on_server_failure:
+            try:
+                await on_server_failure(config.id, str(e)[:300])
+            except Exception:
+                pass
+        return []
+
+
+async def load_mcp_tools(
+    configs: List[MCPServerConfig],
+    user_id: str = None,
+    tool_display_names: Optional[dict] = None,
+    on_tool_success: Optional[Callable[[str, str], Awaitable[None]]] = None,
+    on_tool_error: Optional[Callable[[str, str, str], Awaitable[None]]] = None,
+    on_batch_available: Optional[Callable[[
+        str, list], Awaitable[None]]] = None,
+    on_server_failure: Optional[Callable[[str, str], Awaitable[None]]] = None,
+) -> List[BaseTool]:
+    """加载 MCP 工具列表
+
+    接受 MCPServerConfig 列表，并发连接所有服务器并加载工具。
+
+    Args:
+        configs: MCP 服务器配置列表（仅 enabled 的会被处理）
+        user_id: 当前用户 ID，通过 interceptor 注入到每次 MCP 工具调用的 header 中
+        tool_display_names: 可选的工具中文显示名映射 {tool_full_name: display_name}
+        on_tool_success: 单个工具调用成功回调
+        on_tool_error: 单个工具调用失败回调
+        on_batch_available: 服务器连接成功后批量标记工具可用回调 (config_id, tool_names)
+        on_server_failure: 服务器连接失败回调 (config_id, error_msg)
+    """
+    enabled_configs = [c for c in configs if c.enabled]
+    if not enabled_configs:
+        return []
+
+    # 构建 interceptor 链
+    interceptors = []
+    if user_id:
+        interceptors.append(_make_user_id_interceptor(user_id))
+
+    # 并发连接所有服务器：总耗时 ≈ max(单服务器)，而非 Σ(单服务器)。
+    # 每个服务器在 _load_one_server 内独立异常隔离 + 超时兜底。
+    per_server_results = await asyncio.gather(*[
+        _load_one_server(
+            config, interceptors, tool_display_names,
+            on_tool_success, on_tool_error, on_batch_available, on_server_failure,
+        )
+        for config in enabled_configs
+    ], return_exceptions=False)
+
+    all_tools: List[BaseTool] = []
+    for tools in per_server_results:
+        if tools:
+            all_tools.extend(tools)
+
+    if all_tools:
+        SYLogger.info(f"[MCP] 共加载 {len(all_tools)} 个 MCP 工具")
+    return all_tools
+
+
+async def test_mcp_connection(server_url: str, headers: dict = None) -> dict:
+    """测试 MCP 服务器连接，返回发现的工具列表"""
+    try:
+        from langchain_mcp_adapters.client import MultiServerMCPClient
+
+        client_config = {
+            "test": {
+                "url": server_url,
+                "transport": "streamable_http",
+                **({"headers": headers} if headers else {}),
+            }
+        }
+
+        client = MultiServerMCPClient(client_config)
+        tools = await client.get_tools()
+
+        return {
+            "success": True,
+            "tools": [{"name": t.name, "description": t.description or ""} for t in tools],
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "error": str(e),
+        }

{sycommon_python_lib-0.2.7a28 → sycommon_python_lib-0.2.7a29}/src/sycommon/agent/sandbox/minio_sync.py

@@ -24,6 +24,10 @@ from sycommon.logging.kafka_log import SYLogger
 # 日期快照功能预留，当前暂不启用
 _SNAPSHOT_ENABLED = False
 
+# restore_from_minio 的 MinIO 下载并发度（同时下载的文件数上限）。
+# 与 _sync_batch(sync→minio 方向)的 batch_size 对齐为 5，避免压垮 MinIO/网络。
+_RESTORE_CONCURRENCY = 5
+
 
 class MinioSyncService(metaclass=SingletonMeta):
     """MinIO 文件同步服务（单例）"""
@@ -759,12 +763,15 @@ class MinioSyncService(metaclass=SingletonMeta):
 
         SYLogger.info(f"[MinIO] 需恢复 {len(to_restore)} 个文件到沙箱: user={user_id}")
 
-        # 4. 批量恢复
+        # 4. 并发恢复：MinIO 下载用全局 Semaphore(5) 限流并发，沙箱上传仍按批 batch_upload。
+        # 旧实现是「批内串行下载 + 批间串行」，N 个文件 ≈ N 次串行 MinIO GET，27 个文件要数十秒。
+        # 改为 Semaphore(5) 后下载并发，总耗时 ≈ ceil(N/5) 次串行往返，对齐 _sync_batch 的并发度。
         restored = 0
-        for i in range(0, len(to_restore), 5):
-            batch = to_restore[i:i + 5]
-            files_to_upload = []
-            for rel_path in batch:
+        download_sem = asyncio.Semaphore(_RESTORE_CONCURRENCY)
+
+        async def _download_one(rel_path: str):
+            """下载单个 MinIO 对象到内存，返回 (沙箱绝对路径, 内容) 或 None。"""
+            async with download_sem:
                 try:
                     def _download_object(bucket, key):
                         resp = self._client.get_object(bucket, key)
@@ -777,21 +784,32 @@ class MinioSyncService(metaclass=SingletonMeta):
                     content = await asyncio.to_thread(
                         _download_object, self._bucket, f"{prefix}{rel_path}"
                     )
-                    files_to_upload.append((f"/{rel_path}", content))
+                    return (f"/{rel_path}", content)
                 except Exception as e:
                     SYLogger.warning(f"[MinIO] 下载文件失败: {rel_path}, error={e}")
+                    return None
 
-            if files_to_upload:
-                try:
-                    results = await sandbox_backend.aupload_files(files_to_upload)
-                    for r in results:
-                        if not r.error:
-                            restored += 1
-                        else:
-                            SYLogger.warning(
-                                f"[MinIO] 上传到沙箱失败: {r.path}, error={r.error}")
-                except Exception as e:
-                    SYLogger.warning(f"[MinIO] 批量上传到沙箱失败: {e}")
+        # 全量并发下载(受 Semaphore 限流)，比按批串行快得多
+        download_results = await asyncio.gather(
+            *[_download_one(rp) for rp in to_restore]
+        )
+        downloaded = [r for r in download_results if r is not None]
+
+        # 上传仍走 batch_upload(服务端并发)，按 5 个一批发(沙箱端点对超大批量可能限流)
+        for i in range(0, len(downloaded), _RESTORE_CONCURRENCY):
+            batch = downloaded[i:i + _RESTORE_CONCURRENCY]
+            if not batch:
+                continue
+            try:
+                results = await sandbox_backend.aupload_files(batch)
+                for r in results:
+                    if not r.error:
+                        restored += 1
+                    else:
+                        SYLogger.warning(
+                            f"[MinIO] 上传到沙箱失败: {r.path}, error={r.error}")
+            except Exception as e:
+                SYLogger.warning(f"[MinIO] 批量上传到沙箱失败: {e}")
 
         SYLogger.info(
             f"[MinIO] 恢复完成: user={user_id}, restored={restored}/{len(to_restore)}")

{sycommon_python_lib-0.2.7a28 → sycommon_python_lib-0.2.7a29}/src/sycommon_python_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.7a28
+Version: 0.2.7a29
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown