rexp-web 0.0.0-stage → 0.1.1

package/server/bridge.mjs

@@ -0,0 +1,891 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+import { npuQuery, parseNpuSnapshot } from "./accelerators.mjs";
+import { RunMonitor } from "./run-monitor.mjs";
+import { errorInfo } from "./errors.mjs";
+import { createServerDirectory, serverState } from "./server-directory.mjs";
+import {autostartStatus,setAutostart} from './autostart.mjs';
+import {webPaths} from './web-paths.mjs';
+
+const settingsFile = webPaths().settingsFile;
+
+export function mcpSettings() {
+  let settings = {};
+  try { settings = JSON.parse(fs.readFileSync(settingsFile, "utf8")); }
+  catch (error) {
+    if (error.code !== "ENOENT") throw new Error(`无法读取 Web 连接配置：${settingsFile}`);
+  }
+  if (!settings || typeof settings !== "object" || Array.isArray(settings)) {
+    throw new Error(`Web 连接配置应为 JSON 对象：${settingsFile}`);
+  }
+  for (const key of ["mcpDir", "mcpConfig", "mcpPlugins"]) {
+    if (settings[key] !== undefined && (typeof settings[key] !== "string" || !path.isAbsolute(settings[key]))) {
+      throw new Error(`Web 连接配置的 ${key} 必须是绝对路径：${settingsFile}`);
+    }
+  }
+  const configuredDir = process.env.HANDFREE_MCP_DIR || settings.mcpDir;
+  if (!configuredDir) throw new Error('尚未接入 SSH-MCP；请通过 rexp-web 插件完成 configure，或在 Web 个人配置中指定 mcpDir。');
+  const mcpDir = path.resolve(configuredDir);
+  const configHome = process.env.XDG_CONFIG_HOME || path.join(os.homedir(), ".config");
+  const mcpConfig = path.resolve(process.env.HANDFREE_MCP_CONFIG || settings.mcpConfig || path.join(configHome, "handfree-ssh-mcp", "servers.yaml"));
+  const mcpPlugins = path.resolve(process.env.HANDFREE_PLUGINS_CONFIG || settings.mcpPlugins || path.join(configHome, "handfree-ssh-mcp", "plugins.yaml"));
+  return { mcpDir, mcpConfig, mcpPlugins };
+}
+
+const runtimeDir = path.join(os.homedir(), ".cache", "rexp-hub", "ssh-mcp");
+const outputLogDir = path.join(runtimeDir, ".handfree-output");
+let clientPromise;
+const serverDirectory = createServerDirectory({ invoke: (...args) => callTool(...args) });
+const gpuStreams = new Map();
+const acceleratorFacts = new Map();
+const npuSamples = new Map();
+const activeStreamLogs = new Set();
+const runMonitor = new RunMonitor({
+  listServers: () => listServers(),
+  readRuns: (alias) => runsData(alias),
+  stateFile: path.join(os.homedir(), ".cache", "rexp-hub", "web", "run-monitor.json"),
+});
+
+function pruneOutputLogs(maxAgeMs = 2000) {
+  if (!fs.existsSync(outputLogDir)) return;
+  const visit = (dir) => {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      const file = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        visit(file);
+        if (!fs.readdirSync(file).length) fs.rmdirSync(file);
+      } else if (entry.isFile() && entry.name.endsWith(".log") && !activeStreamLogs.has(file)) {
+        if (Date.now() - fs.statSync(file).mtimeMs >= maxAgeMs) fs.unlinkSync(file);
+      }
+    }
+  };
+  try { visit(outputLogDir); } catch { /* Cache cleanup must not interrupt data reading. */ }
+}
+
+export async function closeBridge() {
+  runMonitor.stop();
+  for (const stream of gpuStreams.values()) {
+    stream.closed = true;
+    for (const subscriber of stream.subscribers) subscriber.end();
+    stream.subscribers.clear();
+  }
+  gpuStreams.clear();
+  const pending = clientPromise;
+  clientPromise = undefined;
+  serverDirectory.clear();
+  acceleratorFacts.clear();
+  npuSamples.clear();
+  if (pending) {
+    try { await (await pending).close(); }
+    catch { /* The child may already have exited. */ }
+  }
+  try { fs.rmSync(outputLogDir, { recursive: true, force: true }); } catch { /* Cache can be cleared later. */ }
+}
+
+const bridgeKey = Symbol.for("rexp-hub.ssh-mcp-bridge");
+if (globalThis[bridgeKey]) void globalThis[bridgeKey]();
+globalThis[bridgeKey] = closeBridge;
+
+async function mcpClient() {
+  if (!clientPromise) {
+    clientPromise = (async () => {
+      const { mcpDir, mcpConfig, mcpPlugins } = mcpSettings();
+      const entry = path.join(mcpDir, "build/index.js");
+      if (!fs.existsSync(entry)) throw new Error(`找不到 SSH-MCP 程序：${entry}`);
+      fs.mkdirSync(runtimeDir, { recursive: true, mode: 0o700 });
+      const client = new Client({ name: "rexp-hub", version: "0.1.0" });
+      const transport = new StdioClientTransport({
+        command: process.execPath,
+        args: [entry, ...(fs.existsSync(mcpConfig) ? ["--config", mcpConfig] : []), "--plugins", mcpPlugins],
+        cwd: runtimeDir,
+        stderr: "pipe",
+      });
+      let startupError = "";
+      transport.stderr?.on("data", (chunk) => {
+        const errorLine = chunk.toString().split("\n").find((line) => line.includes("[ERROR]"));
+        if (errorLine) startupError = errorLine.slice(0, 1000);
+      });
+      try { await client.connect(transport); }
+      catch (error) {
+        await client.close().catch(() => {});
+        throw new Error(`SSH-MCP 启动失败：${startupError || error.message}`);
+      }
+      client.onclose = () => { clientPromise = undefined; serverDirectory.clear(); };
+      return client;
+    })().catch((error) => { clientPromise = undefined; throw error; });
+  }
+  return clientPromise;
+}
+
+async function callTool(name, args, timeout = 30_000) {
+  const client = await mcpClient();
+  const result = await client.callTool({ name, arguments: args }, undefined, { timeout });
+  const body = result.content?.filter((part) => part.type === "text").map((part) => part.text).join("\n") ?? "";
+  if (result.isError) throw new Error(body || `${name} 调用失败`);
+  if (name !== "command-status") pruneOutputLogs();
+  return body;
+}
+
+function parseJsonTool(text) {
+  try { return JSON.parse(text); }
+  catch { throw new Error("SSH-MCP 返回的数据格式无法识别"); }
+}
+
+function readCommandOutput(text) {
+  if (/\[EXIT CODE: [1-9]\d*\]\s*$/u.test(text)) throw new Error(text.trim());
+  if (text.startsWith("[OUTPUT TRUNCATED]")) throw new Error("远端输出被截断");
+  return text.trim();
+}
+
+function numberOrUndefined(value) {
+  const text = String(value ?? "").trim();
+  if (!text || text === "—" || /^N\/A$/iu.test(text)) return undefined;
+  const n = Number(text);
+  return Number.isFinite(n) ? n : undefined;
+}
+
+function parseProgress(text) {
+  const value = String(text ?? "").trim();
+  const fraction = /^(\d+(?:\.\d+)?|—)\s*\/\s*(\d+)$/u.exec(value);
+  if (fraction) return { progress: numberOrUndefined(fraction[1]), total: Number(fraction[2]) };
+  return { progress: numberOrUndefined(value) };
+}
+
+export function parseGpuCsv(text) {
+  return text.split(/\r?\n/u).filter(Boolean).map((line) => {
+    const cells = line.split(",").map((part) => part.trim());
+    if (cells.length < 14) return null;
+    const [index, uuid, name, utilization, usedMiB, totalMiB, temperature, fanPercent, powerWatts, powerLimitWatts, gpuClockMHz, memoryClockMHz, pcieGeneration, pcieWidth] = cells;
+    const device = numberOrUndefined(index);
+    if (device === undefined) return null;
+    return {
+      index: device, uuid, name,
+      utilization: numberOrUndefined(utilization),
+      usedMemory: numberOrUndefined(usedMiB) === undefined ? undefined : numberOrUndefined(usedMiB) / 1024,
+      totalMemory: numberOrUndefined(totalMiB) === undefined ? undefined : numberOrUndefined(totalMiB) / 1024,
+      temperature: numberOrUndefined(temperature), fanPercent: numberOrUndefined(fanPercent),
+      powerWatts: numberOrUndefined(powerWatts), powerLimitWatts: numberOrUndefined(powerLimitWatts),
+      gpuClockMHz: numberOrUndefined(gpuClockMHz), memoryClockMHz: numberOrUndefined(memoryClockMHz),
+      pcieGeneration: numberOrUndefined(pcieGeneration), pcieWidth: numberOrUndefined(pcieWidth),
+    };
+  }).filter(Boolean);
+}
+
+export function parseRunsTable(text) {
+  const rows = [];
+  let group = "";
+  for (const line of text.split(/\r?\n/u)) {
+    if (!line.startsWith("│")) continue;
+    const cells = line.slice(1, line.endsWith("│") ? -1 : undefined).split("│").map((value) => value.trim());
+    if (cells.length !== 5 && cells.length !== 7) continue;
+    if (cells[0] === "run_id") continue;
+    const [id, a, b, c, d, e, f] = cells;
+    if (!id && a.includes("/") && cells.slice(2).every((value) => !value)) {
+      group = a;
+      continue;
+    }
+    if (/^run_\d+$/u.test(id)) {
+      const wide = cells.length === 7;
+      const projectVersion = wide ? a : group;
+      const separator = projectVersion.lastIndexOf("/");
+      rows.push({
+        id,
+        project: separator >= 0 ? projectVersion.slice(0, separator) : projectVersion,
+        version: separator >= 0 ? projectVersion.slice(separator + 1) : "",
+        experiment: wide ? b : a,
+        status: wide ? c : b,
+        ...parseProgress(wide ? d : c),
+        environment: wide ? e : undefined,
+        submitted: wide ? f : d,
+      });
+      continue;
+    }
+    const last = rows.at(-1);
+    if (last && !id) {
+      last.experiment += cells.length === 7 ? b : a;
+      last.status += cells.length === 7 ? c : b;
+    }
+  }
+  return rows.sort((left, right) => Number(right.id.slice(4)) - Number(left.id.slice(4)));
+}
+
+export function parseRunsJson(text) {
+  const payload = JSON.parse(text);
+  if (payload?.schema_version !== 1 || !Array.isArray(payload.runs)) {
+    throw new Error("不支持的平台实验列表格式");
+  }
+  return payload.runs.map((record) => {
+    if (!/^run_\d+$/u.test(record.run_id) || typeof record.status !== "string") {
+      throw new Error("平台实验列表包含无效记录");
+    }
+    return {
+      id: record.run_id,
+      project: record.project ?? "",
+      version: record.version ?? "",
+      experiment: record.experiment_name ?? "",
+      description: record.description ?? undefined,
+      ...(record.research_topic ? { researchTopic: record.research_topic } : {}),
+      ...(record.study ? { study: record.study } : {}),
+      ...(record.stage ? { stage: record.stage } : {}),
+      gpuIds: Array.isArray(record.allocated_gpu_ids) ? record.allocated_gpu_ids : undefined,
+      status: record.status,
+      progress: numberOrUndefined(record.progress),
+      total: numberOrUndefined(record.total_samples),
+      ...(record.run_kind ? { kind: record.run_kind, progressUnit: record.progress_unit } : {}),
+      environment: record.environment ?? undefined,
+      submitted: record.submitted_display || record.submitted_at || "",
+    };
+  }).sort((left, right) => Number(right.id.slice(4)) - Number(left.id.slice(4)));
+}
+
+export function parseRunDetails(text) {
+  const fields = {};
+  let key;
+  for (const rawLine of text.replace(/\x1b\[[0-?]*[ -/]*[@-~]/gu, "").split(/\r?\n/u)) {
+    const line = rawLine.trim();
+    if (!line) continue;
+    const match = /^([a-z][a-z_ ]*):\s*(.*)$/iu.exec(line);
+    if (match) {
+      key = match[1].trim().toLowerCase().replaceAll(" ", "_");
+      fields[key] = match[2];
+    } else if (key) {
+      fields[key] = `${fields[key]} ${line}`.trim();
+    }
+  }
+  return fields;
+}
+
+function parseProcesses(text, gpus) {
+  const byUuid = new Map(gpus.map((gpu) => [gpu.uuid, gpu.index]));
+  return text.split(/\r?\n/u).filter(Boolean).map((line) => {
+    const [uuid, pid, command, memory] = line.split(",").map((part) => part.trim());
+    const device = byUuid.get(uuid);
+    if (device === undefined || !/^\d+$/u.test(pid)) return null;
+    return { device, pid: Number(pid), command, type: "Compute", gpuMemoryMiB: numberOrUndefined(memory) };
+  }).filter(Boolean);
+}
+
+function enrichProcesses(text, processes) {
+  const byPid = new Map(processes.map((item) => [item.pid, item]));
+  for (const line of text.split(/\r?\n/u)) {
+    const match = /^\s*(\d+)\s+(\S+)\s+(\S+)\s+(\d+)\s+(.+)$/u.exec(line);
+    if (!match) continue;
+    const process = byPid.get(Number(match[1]));
+    if (!process) continue;
+    process.user = match[2];
+    process.cpuPercent = numberOrUndefined(match[3]);
+    process.hostMemoryMiB = Math.round(Number(match[4]) / 1024);
+    process.command = match[5];
+  }
+  return processes;
+}
+
+export function isWindows(osName) { return /windows/iu.test(osName || ""); }
+export function gpuSamplingMode(osName) { return /^linux$/iu.test(osName || "") ? "stream" : "poll"; }
+
+export function processDetailsCommand(osName, pids) {
+  if (!/^\d+(?:,\d+)*$/u.test(pids)) throw new Error("无效的进程编号");
+  if (!isWindows(osName)) return `ps -p ${pids} -o pid=,user:32=,pcpu=,rss=,args=`;
+  const filter = pids.split(",").map((pid) => `ProcessId=${pid}`).join(" OR ");
+  const script = `[Console]::OutputEncoding = [System.Text.Encoding]::UTF8; $ErrorActionPreference = 'Stop'; try { $items = @(Get-CimInstance Win32_Process -Filter '${filter}' | Select-Object ProcessId,WorkingSetSize,CommandLine); ConvertTo-Json -InputObject $items -Compress; exit 0 } catch { [Console]::Error.WriteLine($_.ToString()); exit 1 }`;
+  return `powershell.exe -NoProfile -NonInteractive -EncodedCommand ${Buffer.from(script, "utf16le").toString("base64")}`;
+}
+
+export function enrichWindowsProcesses(text, processes) {
+  const payload = JSON.parse(text);
+  const rows = new Map((Array.isArray(payload) ? payload : [payload]).filter(Boolean).map((row) => [Number(row.ProcessId), row]));
+  return processes.map((process) => {
+    const row = rows.get(process.pid);
+    if (!row) return process;
+    return { ...process, command: row.CommandLine || process.command,
+      hostMemoryMiB: row.WorkingSetSize == null ? undefined : Math.round(Number(row.WorkingSetSize) / 1024 / 1024) };
+  });
+}
+
+async function listServers(force = false) {
+  const raw = await serverDirectory.list(force);
+  const items = raw.filter((item) => item.enabled).map((item) => {
+    const gpus = item.status?.gpus ?? [];
+    const busy = gpus.some((gpu) => (numberOrUndefined(String(gpu.usage).replace("%", "")) ?? 0) >= 20);
+    return {
+      alias: item.name,
+      label: item.status?.hostname || item.name,
+      state: serverState(item, busy),
+      gpuCount: Array.isArray(item.status?.gpus) ? gpus.length : undefined,
+      osName: item.status?.osName,
+      lastUpdated: item.status?.lastUpdated,
+      ...acceleratorFacts.get(item.name),
+    };
+  });
+  return items;
+}
+
+async function assertServer(alias) {
+  const servers = await listServers();
+  const server = servers.find((server) => server.alias === alias)
+    ?? (await listServers(true)).find((server) => server.alias === alias);
+  if (!server) throw new Error("未知的 SSH 服务器别名");
+  return server;
+}
+
+const gpuQuery = "nvidia-smi --query-gpu=index,uuid,name,utilization.gpu,memory.used,memory.total,temperature.gpu,fan.speed,power.draw,power.limit,clocks.current.graphics,clocks.current.memory,pcie.link.gen.current,pcie.link.width.current --format=csv,noheader,nounits";
+const processQuery = "nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_gpu_memory --format=csv,noheader,nounits";
+
+function sendEvent(res, name, data) {
+  if (!res.writableEnded) res.write(`event: ${name}\ndata: ${JSON.stringify(data)}\n\n`);
+}
+
+function monitorEvent(res, kind, value) {
+  if (res.writableEnded) return;
+  if (kind === "transition") res.write(`id: ${value.event.id}\nevent: transition\ndata: ${JSON.stringify(value.event)}\n\n`);
+  else if (kind === "runs") res.write(`event: runs\ndata: ${JSON.stringify({ alias: value.alias, ...value.snapshot })}\n\n`);
+  else if (kind === "monitor-status") res.write(`event: monitor-status\ndata: ${JSON.stringify(value.status)}\n\n`);
+}
+
+function subscribeRunEvents(req, res, url) {
+  const requested = Number(url.searchParams.get("after") || runMonitor.cursor());
+  const lastReceived = Number(req.headers["last-event-id"] || 0);
+  const cursor = Math.max(requested, lastReceived);
+  res.writeHead(200, {
+    "Content-Type": "text/event-stream; charset=utf-8", "Cache-Control": "no-cache, no-transform",
+    "Connection": "keep-alive", "X-Accel-Buffering": "no",
+  });
+  res.write("retry: 5000\n\n");
+  monitorEvent(res, "monitor-status", { status: runMonitor.status() });
+  for (const event of runMonitor.eventsAfter(Number.isSafeInteger(cursor) ? cursor : runMonitor.cursor())) monitorEvent(res, "transition", { event });
+  const unsubscribe = runMonitor.subscribe((value) => monitorEvent(res, value.type, value));
+  runMonitor.start();
+  const heartbeat = setInterval(() => { if (!res.writableEnded) res.write(": keep-alive\n\n"); }, 25_000);
+  res.on("close", () => {
+    clearInterval(heartbeat);
+    unsubscribe();
+    if (!runMonitor.listeners.size) runMonitor.stop();
+  });
+}
+
+function publish(stream, name, data) {
+  for (const subscriber of stream.subscribers) sendEvent(subscriber, name, data);
+}
+
+function acceptStreamChunk(stream, chunk) {
+  stream.carry += chunk;
+  const lines = stream.carry.split(/\r?\n/u);
+  stream.carry = lines.pop() ?? "";
+  for (const line of lines) {
+    const [gpu] = parseGpuCsv(line);
+    if (!gpu) continue;
+    if (stream.pending.size && stream.pending.has(gpu.index)) flushStreamSample(stream);
+    stream.pending.set(gpu.index, gpu);
+    if (stream.expectedCount > 0 && stream.pending.size >= stream.expectedCount) flushStreamSample(stream);
+  }
+}
+
+function flushStreamSample(stream) {
+  if (!stream.pending.size) return;
+  const gpus = [...stream.pending.values()].sort((a, b) => a.index - b.index);
+  stream.pending.clear();
+  publish(stream, "sample", { acceleratorBackend: "cuda", gpus, sampledAt: new Date().toISOString() });
+}
+
+async function runGpuStream(stream) {
+  try {
+    const initial = await gpuData(stream.alias, false);
+    publish(stream, "sample", initial);
+    if (initial.acceleratorBackend === "npu" || gpuSamplingMode(stream.osName) === "poll") {
+      while (!stream.closed && stream.subscribers.size) {
+        const sample = await gpuData(stream.alias, false);
+        if (!stream.closed && stream.subscribers.size) publish(stream, "sample", sample);
+        await new Promise((resolve) => setTimeout(resolve, 2000));
+      }
+      return;
+    }
+    stream.expectedCount = initial.gpus.length;
+    while (!stream.closed && stream.subscribers.size) {
+      stream.carry = "";
+      stream.pending.clear();
+      let logPath;
+      let runId;
+      try {
+        const command = `timeout 12s ${gpuQuery} --loop=2`;
+        const started = parseJsonTool(await callTool("execute-command", {
+          connectionName: stream.alias, cmdString: command, stream: true, timeout: 18_000, maxOutputBytes: 8192,
+        }, 22_000));
+        if (!started.runId) throw new Error("SSH-MCP 未启动 GPU 采样");
+        runId = started.runId;
+        logPath = started.logPath;
+        if (logPath) activeStreamLogs.add(logPath);
+        let status = "running";
+        while (status === "running" && !stream.closed) {
+          await new Promise((resolve) => setTimeout(resolve, stream.subscribers.size ? 900 : 1700));
+          const update = parseJsonTool(await callTool("command-status", { runId: started.runId, maxOutputBytes: 8192 }, 10_000));
+          if (update.outputChunk) acceptStreamChunk(stream, update.outputChunk);
+          status = update.status;
+          if (status === "failed" && stream.subscribers.size) throw new Error("远端 GPU 采样中断");
+        }
+        if (stream.carry) acceptStreamChunk(stream, "\n");
+        flushStreamSample(stream);
+      } finally {
+        if (logPath) activeStreamLogs.delete(logPath);
+        if (logPath && runId && path.resolve(logPath).startsWith(`${outputLogDir}${path.sep}`) && path.basename(logPath).includes(runId) && logPath.endsWith(".log")) {
+          try { await fs.promises.unlink(logPath); } catch { /* The log may have been removed already. */ }
+        }
+        pruneOutputLogs();
+      }
+    }
+  } catch (error) {
+    if (!stream.closed && stream.subscribers.size) {
+      publish(stream, "stream-error", { error: (error instanceof Error ? error.message : "流式采样失败").slice(0, 300) });
+    }
+  } finally {
+    for (const subscriber of stream.subscribers) subscriber.end();
+    stream.subscribers.clear();
+    if (gpuStreams.get(stream.alias) === stream) gpuStreams.delete(stream.alias);
+  }
+}
+
+async function subscribeGpuStream(alias, req, res) {
+  const server = await assertServer(alias);
+  res.writeHead(200, {
+    "Content-Type": "text/event-stream; charset=utf-8",
+    "Cache-Control": "no-cache, no-transform",
+    "Connection": "keep-alive",
+    "X-Accel-Buffering": "no",
+  });
+  res.flushHeaders?.();
+  let stream = gpuStreams.get(alias);
+  if (!stream) {
+    stream = { alias, osName: server.osName, expectedCount: server.gpuCount, pending: new Map(), carry: "", subscribers: new Set(), closed: false };
+    gpuStreams.set(alias, stream);
+  }
+  stream.subscribers.add(res);
+  sendEvent(res, "connected", { intervalSeconds: 2 });
+  res.on("close", () => stream.subscribers.delete(res));
+  if (!stream.running) {
+    stream.running = true;
+    void runGpuStream(stream);
+  }
+}
+
+async function npuData(alias) {
+  const cached = npuSamples.get(alias);
+  if (cached && Date.now() - cached.at < 2000) return cached.pending;
+  const pending = (async () => {
+    const output = readCommandOutput(await callTool("execute-command", {
+      connectionName: alias, cmdString: npuQuery, stream: false, timeout: 15_000,
+    }, 25_000));
+    const data = parseNpuSnapshot(output);
+    acceleratorFacts.set(alias, { acceleratorBackend: "npu", gpuCount: data.gpus.length });
+    return data;
+  })();
+  const sample = { at: Infinity, pending };
+  npuSamples.set(alias, sample);
+  try { const data = await pending; sample.at = Date.now(); return data; }
+  catch (error) { npuSamples.delete(alias); throw error; }
+}
+
+async function gpuData(alias, detail) {
+  const server = await assertServer(alias);
+  if (acceleratorFacts.get(alias)?.acceleratorBackend === "npu") return npuData(alias);
+  let raw;
+  try {
+    raw = readCommandOutput(await callTool("execute-command", { connectionName: alias, cmdString: gpuQuery, stream: false, timeout: 15_000 }, 25_000));
+  } catch (error) {
+    if (/nvidia-smi.*(?:not found|command not found|not recognized)|\[EXIT CODE: 127\]/iu.test(String(error))) {
+      if (!isWindows(server.osName)) {
+        try { return await npuData(alias); }
+        catch (npuError) {
+          if (!/npu-smi.*(?:not found|command not found)|\[EXIT CODE: 127\]/iu.test(String(npuError))) throw npuError;
+        }
+      }
+      return { gpus: [], processes: [], sampledAt: new Date().toISOString() };
+    }
+    throw error;
+  }
+  const gpus = parseGpuCsv(raw);
+  if (!gpus.length && raw) throw new Error(raw.slice(0, 300));
+  let processes = [];
+  let processError;
+  if (detail && gpus.length) {
+    try {
+      const output = readCommandOutput(await callTool("execute-command", { connectionName: alias, cmdString: processQuery, stream: false, timeout: 15_000 }, 25_000));
+      processes = parseProcesses(output, gpus);
+    } catch (error) { processError = error instanceof Error ? error.message : "进程信息不可用"; }
+    if (processes.length) {
+      const pids = [...new Set(processes.map((process) => process.pid))].join(",");
+      try {
+        const output = readCommandOutput(await callTool("execute-command", { connectionName: alias, cmdString: processDetailsCommand(server.osName, pids), stream: false, timeout: 10_000 }, 15_000));
+        processes = isWindows(server.osName) ? enrichWindowsProcesses(output, processes) : enrichProcesses(output, processes);
+      } catch { /* GPU process records remain valid without host metrics. */ }
+    }
+  }
+  acceleratorFacts.set(alias, { acceleratorBackend: "cuda", gpuCount: gpus.length });
+  return { acceleratorBackend: "cuda", gpus, processes, processError, sampledAt: new Date().toISOString() };
+}
+
+export async function readPluginRuns(alias, invoke = callTool) {
+  try {
+    const envelope = parseJsonTool(await invoke("plugin-run", { plugin: "rexp", connectionName: alias, args: ["runs", "--json"] }, 45_000));
+    const rows = parseRunsJson(envelope.output || "");
+    return { available: true, runs: rows, sampledAt: new Date().toISOString() };
+  } catch (error) {
+    const info = errorInfo(error);
+    if (info.state !== "error") {
+      return { available: false, platformState: info.state, message: info.message, diagnostic: info.details, runs: [], sampledAt: new Date().toISOString() };
+    }
+    throw error;
+  }
+}
+
+export function parseSearchJson(text, runId) {
+  const payload = JSON.parse(text);
+  if (payload?.schema_version !== 1) throw new Error("不支持的参数搜索数据格式");
+  const rows = runId ? [payload] : payload.searches;
+  if (!Array.isArray(rows) || rows.some((row) => !/^run_\d+$/u.test(row.run_id) || typeof row.status !== "string")) {
+    throw new Error("参数搜索数据包含无效记录");
+  }
+  if (runId && (payload.run_id !== runId || payload.query_version !== 1 || !payload.search?.metrics)) {
+    throw new Error("平台返回的搜索详情与请求不一致");
+  }
+  return payload;
+}
+
+export async function readPluginSearches(alias, invoke = callTool, runId) {
+  try {
+    const args = runId ? ["tune", "show", runId, "--summary", "--json"] : ["tune", "list", "--json"];
+    const envelope = parseJsonTool(await invoke("plugin-run", { plugin: "rexp", connectionName: alias, args }, 30_000));
+    const data = parseSearchJson(envelope.output || "", runId);
+    return { available: true, supported: true, ...data, sampledAt: new Date().toISOString() };
+  } catch (error) {
+    if(runId && /No such option.*--summary|No such command.*(?:trials|trial|charts)/iu.test(String(error)))
+      throw new Error('此节点的平台需要更新以支持按需读取；不会回退到全量搜索详情。');
+    if (/No such command ['"‘]?tune|invalid choice: ['"]tune/iu.test(String(error))) {
+      return { available: true, supported: false, searches: [], message: "此节点的平台版本暂不支持参数搜索。", sampledAt: new Date().toISOString() };
+    }
+    const info = errorInfo(error);
+    if (info.state !== "error") return { available: false, supported: false, platformState: info.state, searches: [], message: info.message, diagnostic: info.details, sampledAt: new Date().toISOString() };
+    throw error;
+  }
+}
+
+export function tuneQueryArgs(runId, kind, params, trialId) {
+  if (!/^run_\d+$/u.test(runId)) throw new Error('无效的搜索编号');
+  if (kind === 'trial') {
+    if (!/^[A-Za-z0-9_-]{1,128}$/u.test(trialId || '')) throw new Error('无效的尝试编号');
+    return ['tune', 'trial', runId, trialId, '--json'];
+  }
+  if (kind === 'charts') {
+    const metric=params.get('metric');
+    if (!metric || metric.length>200) throw new Error('请选择有效的目标指标');
+    return ['tune','charts',runId,'--metric',metric,'--json'];
+  }
+  const page=Number(params.get('page') || 1), size=Number(params.get('pageSize') || 50);
+  if (!Number.isInteger(page)||page<1||!Number.isInteger(size)||size<1||size>100) throw new Error('无效的分页参数');
+  const sort=params.get('sort') || 'default', status=params.get('status') || 'all', query=params.get('query') || '';
+  if (!(sort==='default'||/^time:(newest|oldest)$/.test(sort)||/^metric:.{1,200}$/.test(sort)) ||
+      !['all','pareto','PENDING','RUNNING','PAUSING','PAUSED','TERMINATED','ERROR','CANCELLED'].includes(status)||query.length>500) throw new Error('无效的排序或筛选条件');
+  return ['tune','trials',runId,'--page',String(page),'--page-size',String(size),'--sort',sort,'--status',status,'--query',query,
+    ...(params.get('active')==='1'?['--active']:[]),'--json'];
+}
+
+export async function readPluginTuneView(alias, runId, kind, params, trialId, invoke=callTool) {
+  const args=tuneQueryArgs(runId,kind,params,trialId);
+  const envelope=parseJsonTool(await invoke('plugin-run',{plugin:'rexp',connectionName:alias,args},30_000));
+  const data=JSON.parse(envelope.output || '');
+  if(data.schema_version!==1||data.run_id!==runId) throw new Error('平台返回的搜索视图与请求不一致');
+  if(kind==='trials' && (!Array.isArray(data.trials)||data.trials.length>Number(params.get('pageSize')||50))) throw new Error('平台未返回请求的分页数据');
+  if(kind==='trial' && data.trial?.trial_id!==trialId) throw new Error('平台返回的尝试与请求不一致');
+  if(kind==='charts' && data.analytics?.metric!==params.get('metric')) throw new Error('平台返回的图表与请求不一致');
+  return data;
+}
+
+export function sameOrigin(req) {
+  try { const origin = new URL(req.headers.origin); return origin.host === req.headers.host && ['http:', 'https:'].includes(origin.protocol); }
+  catch { return false; }
+}
+
+export async function setPluginSearchPolicy(alias, runId, enabled, invoke = callTool) {
+  if (typeof enabled !== 'boolean') throw new Error('让位开关必须是布尔值');
+  const envelope = parseJsonTool(await invoke('plugin-run', {
+    plugin: 'rexp', connectionName: alias,
+    args: ['tune', 'policy', runId, enabled ? '--yield-to-normal' : '--no-yield-to-normal', '--json'],
+  }, 30_000));
+  const result = JSON.parse(envelope.output || '');
+  if (result.run_id !== runId || result.yield_to_normal !== enabled || result.trial_priority !== Number(enabled)) {
+    throw new Error('平台未确认调度策略，请刷新后检查');
+  }
+  return result;
+}
+
+export async function readPolicyBody(req) {
+  let body = '';
+  for await (const chunk of req) {
+    body += chunk;
+    if (Buffer.byteLength(body) > 1024) throw new Error('请求过大');
+  }
+  const value = JSON.parse(body);
+  if (typeof value?.yield_to_normal !== 'boolean') throw new Error('让位开关必须是布尔值');
+  return value.yield_to_normal;
+}
+
+async function runsData(alias) {
+  await assertServer(alias);
+  return readPluginRuns(alias);
+}
+
+export function parseRunDetailsJson(text) {
+  const record = JSON.parse(text);
+  if (record?.schema_version !== 1 || !/^run_\d+$/u.test(record.run_id)) throw new Error("不支持的平台实验详情格式");
+  const fields = Object.fromEntries(Object.entries(record).filter(([key]) => !["schema_version", "config_snapshot", "management", "scheduler"].includes(key))
+    .map(([key, value]) => [key, value == null ? "—" : typeof value === "object" ? JSON.stringify(value, null, 2) : String(value)]));
+  if (record.total_samples != null && record.progress != null) fields[record.progress_unit === "trials" ? "remaining_trials" : "remaining_samples"] = String(Math.max(0, record.total_samples - record.progress));
+  return fields;
+}
+
+export async function readPluginRunDetails(alias, runId, invoke = callTool) {
+  if (!/^run_\d+$/u.test(runId)) throw new Error("无效的实验编号");
+  const envelope = parseJsonTool(await invoke("plugin-run", { plugin: "rexp", connectionName: alias, args: ["run", "show", runId, "--json"] }, 30_000));
+  const record = JSON.parse(envelope.output || "");
+  const fields = parseRunDetailsJson(envelope.output || "");
+  if (fields.run_id !== runId) throw new Error("平台返回的实验详情与请求不一致");
+  const terminal = ['COMPLETED', 'FAILED', 'CANCELLED'].includes(record.status);
+  const management = record.management || { schema_version: 0, actions: {
+    pause: { enabled: false, reason: '需更新节点平台后使用暂停' },
+    resume: { enabled: record.checkpoint?.can_resume === true, reason: record.checkpoint?.resume_blocked_reason },
+    cancel: { enabled: !terminal, reason: '实验已结束' },
+    clean: { enabled: terminal, reason: '只允许清理已结束的实验' },
+  } };
+  return { fields, management, scheduler: record.scheduler, sampledAt: new Date().toISOString() };
+}
+
+export async function readPluginCleanup(alias, runId, invoke = callTool, confirm = false) {
+  if (!/^run_\d+$/u.test(runId)) throw new Error('无效的实验编号');
+  const envelope = parseJsonTool(await invoke('plugin-run', { plugin: 'rexp', connectionName: alias,
+    args: ['run', 'clean', runId, ...(confirm ? ['--yes'] : []), '--json'],
+  }, 90_000));
+  const data = JSON.parse(envelope.output || '');
+  if (data.run_id !== runId || data.dry_run !== !confirm || !Array.isArray(data.entries)) throw new Error('平台未确认清理结果');
+  return data;
+}
+
+export async function managePluginRun(alias, runId, action, body, invoke = callTool) {
+  if (!['pause', 'resume', 'cancel', 'clean'].includes(action)) throw new Error('不支持此实验操作');
+  if (['cancel', 'clean'].includes(action) && body?.confirmRunId !== runId) throw new Error('请确认要操作的实验编号');
+  const before = await readPluginRunDetails(alias, runId, invoke);
+  if (!before.management.actions[action]?.enabled) throw new Error(before.management.actions[action]?.reason || '当前状态无法执行此操作');
+  let cleanup;
+  if (action === 'clean') cleanup = await readPluginCleanup(alias, runId, invoke, true);
+  else {
+    // Legacy resume has no --json option. Verify state with run show after every operation.
+    const args = ['run', action, runId, ...(action === 'resume' ? [] : ['--json'])];
+    await invoke('plugin-run', { plugin: 'rexp', connectionName: alias, args }, 30_000);
+  }
+  let details;
+  try { details = await readPluginRunDetails(alias, runId, invoke); }
+  catch { return { cleanup, message: '请求已发送，但状态暂时无法读取。请刷新核实后再操作。' }; }
+  return { details, cleanup, message: action === 'pause' ? '已请求暂停；等待实验在安全边界保存检查点。'
+    : action === 'resume' ? (details.scheduler && !details.scheduler.accepting_work ? '已加入续跑队列；调度器未启动，请先启动节点调度器。' : '已加入续跑队列，等待调度和可用资源。')
+    : action === 'cancel' ? '已取消后续调度；运行中的进程需要时间响应退出请求。' : '清理完成，实验记录与分析结果已保留。' };
+}
+
+export async function readRunActionBody(req) {
+  let body = '';
+  for await (const chunk of req) {
+    body += chunk;
+    if (Buffer.byteLength(body) > 1024) throw new Error('请求过大');
+  }
+  const value = JSON.parse(body);
+  if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).some(k => k !== 'confirmRunId') ||
+      (value.confirmRunId !== undefined && !/^run_\d+$/u.test(value.confirmRunId))) throw new Error('实验操作参数不正确');
+  return value;
+}
+
+function send(res, status, value) {
+  res.writeHead(status, { "Content-Type": "application/json; charset=utf-8", "Cache-Control": "no-store" });
+  res.end(JSON.stringify(value));
+}
+
+export async function pluginDevicePolicy(alias, policy, invoke = callTool) {
+  const args = ['node', 'devices', '--json'];
+  if (policy) {
+    if (!['all','selected'].includes(policy.mode) || !Array.isArray(policy.allowed_device_uuids) ||
+        policy.allowed_device_uuids.length > 128 || policy.allowed_device_uuids.some(id => typeof id !== 'string' || !/^(?:GPU|MIG)-[A-Za-z0-9/._-]{1,160}$/u.test(id)))
+      throw new Error('算力卡范围设置不正确');
+    if (policy.mode === 'all') args.push('--all');
+    else if (!policy.allowed_device_uuids.length) args.push('--none');
+    else for (const uuid of policy.allowed_device_uuids) args.push('--allow', uuid);
+  }
+  try {
+    const envelope = parseJsonTool(await invoke('plugin-run', {plugin:'rexp',connectionName:alias,args}, 30_000));
+    const data = JSON.parse(envelope.output);
+    if(data.schema_version !== 1 || !Array.isArray(data.devices)) throw new Error('算力卡范围数据格式不正确');
+    return {available:true,...data};
+  } catch(error) {
+    if (/No such command ['"‘]?devices/iu.test(String(error))) {
+      if(policy) throw new Error('请先更新此节点的 rexprun，以支持算力卡范围设置');
+      return {available:false,message:'此节点的平台需要更新后，才能设置允许使用的算力卡。'};
+    }
+    const info=errorInfo(error);
+    if(!policy && info.state !== 'error') return {available:false,message:info.message,platformState:info.state};
+    throw error;
+  }
+}
+
+export async function pluginSchedulerAutostart(alias, enabled, invoke = callTool) {
+  if (enabled !== undefined && typeof enabled !== 'boolean') throw new Error('自启动设置参数不正确');
+  const args = ['autostart', '--json'];
+  if (enabled !== undefined) args.push(enabled ? '--enable' : '--disable');
+  try {
+    const envelope = parseJsonTool(await invoke('plugin-run', {plugin:'rexp',connectionName:alias,args}, 60_000));
+    const data = JSON.parse(envelope.output);
+    if (typeof data.supported !== 'boolean' || typeof data.enabled !== 'boolean' || typeof data.boot_ready !== 'boolean') throw new Error('自启动数据格式不正确');
+    return {available:true,...data};
+  } catch(error) {
+    if (/No such command ['"‘]?autostart/iu.test(String(error))) {
+      if(enabled !== undefined) throw new Error('请先更新此节点的 rexprun，以支持调度器开机自启动');
+      return {available:false,message:'此节点的平台需要更新后，才能设置调度器开机自启动。'};
+    }
+    const info=errorInfo(error);
+    if(enabled === undefined && info.state !== 'error') return {available:false,message:info.message};
+    throw error;
+  }
+}
+
+export async function handleApi(req, res, next) {
+  const url = new URL(req.url || "/", "http://127.0.0.1");
+  if (!url.pathname.startsWith("/api/")) return next();
+  if (url.pathname === '/api/web/autostart') {
+    if(req.method === 'GET') {
+      try {return send(res,200,autostartStatus());}
+      catch(error){return send(res,500,{error:'自启动设置读取失败',details:error.message});}
+    }
+    if(req.method !== 'POST') return send(res,405,{error:'Method not allowed'});
+    if(!sameOrigin(req)) return send(res,403,{error:'只接受来自当前网页的请求'});
+    let enabled;
+    try {
+      let body=''; for await(const chunk of req) {body+=chunk; if(Buffer.byteLength(body)>1024) throw new Error('Too large');}
+      const data=JSON.parse(body);
+      if(!data || Object.keys(data).length!==1 || typeof data.enabled!=='boolean') throw new Error('Invalid settings');
+      enabled=data.enabled;
+    } catch{return send(res,400,{error:'自启动设置参数不正确'});}
+    try{return send(res,200,setAutostart(enabled));}
+    catch(error){return send(res,500,{error:'自启动设置未能保存',details:error.message});}
+  }
+  const schedulerAutostartMatch = /^\/api\/servers\/([^/]+)\/scheduler-autostart$/u.exec(url.pathname);
+  if (schedulerAutostartMatch) {
+    if(!['GET','POST'].includes(req.method)) return send(res,405,{error:'Method not allowed'});
+    if(req.method === 'POST' && !sameOrigin(req)) return send(res,403,{error:'只接受来自当前网页的请求'});
+    let enabled;
+    if(req.method === 'POST') {
+      try {
+        let raw=''; for await(const chunk of req) {raw+=chunk;if(Buffer.byteLength(raw)>1024)throw new Error('Too large');}
+        const data=JSON.parse(raw);
+        if(!data || Object.keys(data).length!==1 || typeof data.enabled!=='boolean') throw new Error('Invalid settings');
+        enabled=data.enabled;
+      } catch {return send(res,400,{error:'自启动设置参数不正确'});}
+    }
+    try {
+      const alias=decodeURIComponent(schedulerAutostartMatch[1]);await assertServer(alias);
+      return send(res,200,await pluginSchedulerAutostart(alias,enabled));
+    } catch(error) {return send(res,502,{error:req.method === 'POST'?'调度器自启动未能保存':'调度器自启动暂时无法读取',details:String(error.message || error)});}
+  }
+  const devicesMatch = /^\/api\/servers\/([^/]+)\/device-policy$/u.exec(url.pathname);
+  if (devicesMatch && ['GET','POST'].includes(req.method)) {
+    if(req.method === 'POST' && !sameOrigin(req)) return send(res,403,{error:'只接受来自当前网页的请求'});
+    let policy;
+    if(req.method === 'POST') {
+      try {
+        let raw=''; for await(const chunk of req) {raw+=chunk;if(Buffer.byteLength(raw)>24_000)throw new Error('Too large');}
+        policy=JSON.parse(raw);
+        if(!policy || Object.keys(policy).some(k=>!['mode','allowed_device_uuids'].includes(k))) throw new Error('Invalid policy');
+      } catch {return send(res,400,{error:'算力卡范围设置不正确'});}
+    }
+    try {
+      const alias=decodeURIComponent(devicesMatch[1]);await assertServer(alias);
+      return send(res,200,await pluginDevicePolicy(alias,policy));
+    } catch(error) {return send(res,502,{error:req.method === 'POST'?'算力卡范围未能保存':'算力卡范围暂时无法读取',details:String(error.message || error)});}
+  }
+  const actionMatch = /^\/api\/servers\/([^/]+)\/runs\/(run_\d+)\/actions\/(pause|resume|cancel|clean)$/u.exec(url.pathname);
+  if (actionMatch && req.method === 'POST') {
+    if (!sameOrigin(req)) return send(res, 403, { error: '只接受来自当前网页的请求' });
+    let body;
+    try {
+      body = await readRunActionBody(req);
+      if (['cancel', 'clean'].includes(actionMatch[3]) && body.confirmRunId !== actionMatch[2]) throw new Error('confirmation');
+    } catch { return send(res, 400, { error: '实验操作参数不正确或尚未确认' }); }
+    try {
+      const alias = decodeURIComponent(actionMatch[1]);
+      await assertServer(alias);
+      const result = await managePluginRun(alias, actionMatch[2], actionMatch[3], body);
+      runMonitor.snapshots.delete(alias);
+      return send(res, 200, result);
+    } catch (error) {
+      return send(res, 502, { error: '操作未完成，请刷新状态后检查', details: String(error.message || error) });
+    }
+  }
+  const claimMatch = /^\/api\/notifications\/(\d+)\/claim$/u.exec(url.pathname);
+  if (claimMatch && req.method === "POST") {
+    if (!sameOrigin(req)) return send(res, 403, { error: "只接受来自当前网页的请求" });
+    return send(res, 200, { claimed: runMonitor.claim(Number(claimMatch[1])) });
+  }
+  const policyMatch = /^\/api\/servers\/([^/]+)\/tune\/(run_\d+)\/policy$/u.exec(url.pathname);
+  if (policyMatch && req.method === 'POST') {
+    if (!sameOrigin(req)) return send(res, 403, { error: '只接受来自当前网页的请求' });
+    let enabled;
+    try { enabled = await readPolicyBody(req); }
+    catch { return send(res, 400, { error: '调度策略格式不正确' }); }
+    try {
+      const alias = decodeURIComponent(policyMatch[1]);
+      await assertServer(alias);
+      return send(res, 200, await setPluginSearchPolicy(alias, policyMatch[2], enabled));
+    } catch (error) {
+      const info = errorInfo(error);
+      return send(res, 502, { error: '调度策略更新失败，请刷新后检查', details: info.details });
+    }
+  }
+  if (req.method !== "GET") return send(res, 405, { error: "不支持此操作" });
+  try {
+    if (url.pathname === "/api/notifications/state") return send(res, 200, runMonitor.status());
+    if (url.pathname === "/api/notifications/events") return subscribeRunEvents(req, res, url);
+    if (url.pathname === "/api/servers") return send(res, 200, { servers: await listServers() });
+    const cleanupMatch = /^\/api\/servers\/([^/]+)\/runs\/(run_\d+)\/storage$/u.exec(url.pathname);
+    if (cleanupMatch) {
+      const alias = decodeURIComponent(cleanupMatch[1]);
+      await assertServer(alias);
+      return send(res, 200, await readPluginCleanup(alias, cleanupMatch[2]));
+    }
+    const tuneViewMatch=/^\/api\/servers\/([^/]+)\/tune\/(run_\d+)\/(trials|charts|trial)(?:\/([A-Za-z0-9_-]+))?$/u.exec(url.pathname);
+    if(tuneViewMatch) {
+      try {tuneQueryArgs(tuneViewMatch[2],tuneViewMatch[3],url.searchParams,tuneViewMatch[4]);}
+      catch(error){return send(res,400,{error:error.message});}
+      const alias=decodeURIComponent(tuneViewMatch[1]);
+      await assertServer(alias);
+      return send(res,200,await readPluginTuneView(alias,tuneViewMatch[2],tuneViewMatch[3],url.searchParams,tuneViewMatch[4]));
+    }
+    const tuneMatch = /^\/api\/servers\/([^/]+)\/tune(?:\/(run_\d+))?$/u.exec(url.pathname);
+    if (tuneMatch) {
+      const alias = decodeURIComponent(tuneMatch[1]);
+      await assertServer(alias);
+      return send(res, 200, await readPluginSearches(alias, callTool, tuneMatch[2]));
+    }
+    const runDetailMatch = /^\/api\/servers\/([^/]+)\/runs\/(run_\d+)$/u.exec(url.pathname);
+    if (runDetailMatch) {
+      const alias = decodeURIComponent(runDetailMatch[1]);
+      await assertServer(alias);
+      return send(res, 200, await readPluginRunDetails(alias, runDetailMatch[2]));
+    }
+    const streamMatch = /^\/api\/servers\/([^/]+)\/gpu\/stream$/u.exec(url.pathname);
+    if (streamMatch) return await subscribeGpuStream(decodeURIComponent(streamMatch[1]), req, res);
+    const match = /^\/api\/servers\/([^/]+)\/(gpu|runs)$/u.exec(url.pathname);
+    if (!match) return send(res, 404, { error: "接口不存在" });
+    const alias = decodeURIComponent(match[1]);
+    const body = match[2] === "gpu" ? await gpuData(alias, url.searchParams.get("detail") === "1") : (!runMonitor.stopped && runMonitor.snapshot(alias)) || await runsData(alias);
+    return send(res, 200, body);
+  } catch (error) {
+    const info = errorInfo(error);
+    return send(res, 502, { error: info.message, details: info.details });
+  }
+}