superlab 0.1.12 → 0.1.13

package/lib/auto_state.cjs

@@ -0,0 +1,227 @@
+const fs = require("node:fs");
+const path = require("node:path");
+const {
+  contextFile,
+  extractValue,
+  isMeaningful,
+  normalizeAllowedStages,
+  normalizeScalar,
+  readFileIfExists,
+  readWorkflowLanguage,
+} = require("./auto_common.cjs");
+
+function parseAutoMode(targetDir) {
+  const text = readFileIfExists(contextFile(targetDir, "auto-mode.md"));
+  const allowedStages = normalizeAllowedStages(extractValue(text, ["Allowed stages", "允许阶段"]));
+  return {
+    path: contextFile(targetDir, "auto-mode.md"),
+    text,
+    objective: extractValue(text, ["Objective", "目标"]),
+    autonomyLevel: normalizeScalar(extractValue(text, ["Autonomy level", "自治级别"])),
+    approvalStatus: normalizeScalar(extractValue(text, ["Approval status", "批准状态"])),
+    allowedStages,
+    successCriteria: extractValue(text, ["Success criteria", "成功标准"]),
+    terminalGoalType: normalizeScalar(extractValue(text, ["Terminal goal type", "终止目标类型"])),
+    terminalGoalTarget: extractValue(text, ["Terminal goal target", "终止目标目标值"]),
+    requiredTerminalArtifact: extractValue(text, ["Required terminal artifact", "终止目标工件"]),
+    maxIterations: extractValue(text, ["Max iterations", "最大迭代轮次"]),
+    maxWallClockTime: extractValue(text, ["Max wall-clock time", "最大运行时长"]),
+    maxFailures: extractValue(text, ["Max failures", "最大失败次数"]),
+    pollInterval: extractValue(text, ["Poll interval", "轮询间隔"]),
+    stageCommands: {
+      run: extractValue(text, ["Run command", "运行命令"]),
+      iterate: extractValue(text, ["Iterate command", "迭代命令"]),
+      review: extractValue(text, ["Review command", "审查命令"]),
+      report: extractValue(text, ["Report command", "报告命令"]),
+      write: extractValue(text, ["Write command", "写作命令"]),
+    },
+    successCheckCommand: extractValue(text, ["Success check command", "成功检查命令"]),
+    stopCheckCommand: extractValue(text, ["Stop check command", "停止检查命令"]),
+    promotionCheckCommand: extractValue(text, ["Promotion check command", "升格检查命令"]),
+    promotionCommand: extractValue(text, ["Promotion command", "升格命令"]),
+    promotionPolicy: extractValue(text, ["Promotion policy", "升格策略"]),
+    frozenCore: extractValue(text, ["Frozen core", "冻结核心"]),
+    explorationEnvelope: extractValue(text, ["Exploration envelope", "探索边界"]),
+    stopConditions: extractValue(text, ["Stop conditions", "停止条件"]),
+    escalationConditions: extractValue(text, ["Escalation conditions", "升级条件"]),
+  };
+}
+
+function parseAutoStatus(targetDir) {
+  const text = readFileIfExists(contextFile(targetDir, "auto-status.md"));
+  return {
+    path: contextFile(targetDir, "auto-status.md"),
+    text,
+    status: extractValue(text, ["Status", "状态"]),
+    currentStage: extractValue(text, ["Current stage", "当前阶段"]),
+    currentCommand: extractValue(text, ["Current command", "当前命令"]),
+    activeRunId: extractValue(text, ["Active run id", "当前活跃 run id"]),
+    iterationCount: extractValue(text, ["Iteration count", "迭代计数"]),
+    currentRung: extractValue(text, ["Current rung", "当前 rung"]),
+    watchTarget: extractValue(text, ["Watch target", "监视目标"]),
+    nextRung: extractValue(text, ["Next rung", "下一 rung"]),
+    startedAt: extractValue(text, ["Started at", "开始时间"]),
+    lastHeartbeat: extractValue(text, ["Last heartbeat", "最近心跳"]),
+    lastCheckpoint: extractValue(text, ["Last checkpoint", "最近 checkpoint"]),
+    lastSummary: extractValue(text, ["Last summary", "最近 summary"]),
+    decision: extractValue(text, ["Current decision", "当前决策"]),
+  };
+}
+
+function renderAutoStatus(status, { lang = "en" } = {}) {
+  if (lang === "zh") {
+    return `# 自动模式状态
+
+## 运行状态
+
+- 状态: ${status.status || "idle"}
+- 当前阶段: ${status.currentStage || ""}
+- 当前命令: ${status.currentCommand || ""}
+- 当前活跃 run id: ${status.activeRunId || ""}
+- 迭代计数: ${status.iterationCount || "0"}
+- 当前 rung: ${status.currentRung || ""}
+- 监视目标: ${status.watchTarget || ""}
+- 下一 rung: ${status.nextRung || ""}
+
+## 时间
+
+- 开始时间: ${status.startedAt || ""}
+- 最近心跳: ${status.lastHeartbeat || ""}
+
+## 工件
+
+- 最近 checkpoint: ${status.lastCheckpoint || ""}
+- 最近 summary: ${status.lastSummary || ""}
+
+## 决策
+
+- 当前决策: ${status.decision || ""}
+`;
+  }
+
+  return `# Auto Mode Status
+
+## Runtime State
+
+- Status: ${status.status || "idle"}
+- Current stage: ${status.currentStage || ""}
+- Current command: ${status.currentCommand || ""}
+- Active run id: ${status.activeRunId || ""}
+- Iteration count: ${status.iterationCount || "0"}
+- Current rung: ${status.currentRung || ""}
+- Watch target: ${status.watchTarget || ""}
+- Next rung: ${status.nextRung || ""}
+
+## Timing
+
+- Started at: ${status.startedAt || ""}
+- Last heartbeat: ${status.lastHeartbeat || ""}
+
+## Artifacts
+
+- Last checkpoint: ${status.lastCheckpoint || ""}
+- Last summary: ${status.lastSummary || ""}
+
+## Decision
+
+- Current decision: ${status.decision || ""}
+`;
+}
+
+function writeAutoStatus(targetDir, status, { lang = "en" } = {}) {
+  const filePath = contextFile(targetDir, "auto-status.md");
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, renderAutoStatus(status, { lang }).trimEnd() + "\n");
+}
+
+function renderAutoOutcome(outcome, { lang = "en" } = {}) {
+  if (lang === "zh") {
+    return `# 自动结果
+
+## 目标
+
+- Objective: ${outcome.objective || ""}
+- Experiment ladder: ${outcome.experimentLadder || ""}
+- Metric glossary: ${outcome.metricGlossary || ""}
+- Metric source papers: ${outcome.metricSourcePapers || ""}
+- Metric implementation source: ${outcome.metricImplementationSource || ""}
+- Comparison source papers: ${outcome.comparisonSourcePapers || ""}
+- Comparison implementation source: ${outcome.comparisonImplementationSource || ""}
+- Deviation from original implementation: ${outcome.deviationFromOriginalImplementation || ""}
+- Terminal goal type: ${outcome.terminalGoalType || ""}
+- Terminal goal target: ${outcome.terminalGoalTarget || ""}
+- Required terminal artifact: ${outcome.requiredTerminalArtifact || ""}
+
+## 结果
+
+- Status: ${outcome.status || ""}
+- Goal reached: ${outcome.goalReached ? "yes" : "no"}
+- Stop reason: ${outcome.stopReason || ""}
+- Promotion applied: ${outcome.promotionApplied ? "yes" : "no"}
+- Final artifact: ${outcome.finalArtifact || ""}
+- Final rung: ${outcome.finalRung || ""}
+- Executed stages: ${outcome.executedStages || ""}
+- Iterations completed: ${outcome.iterationsCompleted || "0"}
+- Started at: ${outcome.startedAt || ""}
+- Finished at: ${outcome.finishedAt || ""}
+`;
+  }
+
+  return `# Auto Outcome
+
+## Goal
+
+- Objective: ${outcome.objective || ""}
+- Experiment ladder: ${outcome.experimentLadder || ""}
+- Metric glossary: ${outcome.metricGlossary || ""}
+- Metric source papers: ${outcome.metricSourcePapers || ""}
+- Metric implementation source: ${outcome.metricImplementationSource || ""}
+- Comparison source papers: ${outcome.comparisonSourcePapers || ""}
+- Comparison implementation source: ${outcome.comparisonImplementationSource || ""}
+- Deviation from original implementation: ${outcome.deviationFromOriginalImplementation || ""}
+- Terminal goal type: ${outcome.terminalGoalType || ""}
+- Terminal goal target: ${outcome.terminalGoalTarget || ""}
+- Required terminal artifact: ${outcome.requiredTerminalArtifact || ""}
+
+## Outcome
+
+- Status: ${outcome.status || ""}
+- Goal reached: ${outcome.goalReached ? "yes" : "no"}
+- Stop reason: ${outcome.stopReason || ""}
+- Promotion applied: ${outcome.promotionApplied ? "yes" : "no"}
+- Final artifact: ${outcome.finalArtifact || ""}
+- Final rung: ${outcome.finalRung || ""}
+- Executed stages: ${outcome.executedStages || ""}
+- Iterations completed: ${outcome.iterationsCompleted || "0"}
+- Started at: ${outcome.startedAt || ""}
+- Finished at: ${outcome.finishedAt || ""}
+`;
+}
+
+function writeAutoOutcome(targetDir, outcome, { lang = "en" } = {}) {
+  const filePath = contextFile(targetDir, "auto-outcome.md");
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, renderAutoOutcome(outcome, { lang }).trimEnd() + "\n");
+}
+
+function resolveRequiredArtifact(targetDir, configuredPath) {
+  if (!isMeaningful(configuredPath)) {
+    return { relativePath: "", absolutePath: "" };
+  }
+  const trimmed = configuredPath.trim();
+  return {
+    relativePath: trimmed,
+    absolutePath: path.isAbsolute(trimmed) ? trimmed : path.resolve(targetDir, trimmed),
+  };
+}
+
+module.exports = {
+  parseAutoMode,
+  parseAutoStatus,
+  readWorkflowLanguage,
+  renderAutoOutcome,
+  renderAutoStatus,
+  resolveRequiredArtifact,
+  writeAutoOutcome,
+  writeAutoStatus,
+};

package/lib/context.cjs

@@ -1,5 +1,6 @@
 const fs = require("node:fs");
 const path = require("node:path");
+const { parseEvalProtocol } = require("./eval_protocol.cjs");
 
 function contextFile(targetDir, name) {
   return path.join(targetDir, ".lab", "context", name);
@@ -86,7 +87,25 @@ function renderSummary(lang, data) {
 - Active stage: ${data.stage || "待补充"}
 - Auto mode: ${data.autoStatus || "未启用"}
 - Auto objective: ${data.autoObjective || "待补充"}
+- Auto rung: ${data.autoCurrentRung || "待补充"}
+- Auto watch target: ${data.autoWatchTarget || "待补充"}
 - Auto decision: ${data.autoDecision || "待补充"}
+- Auto terminal goal: ${joinNonEmpty([data.autoGoalType, data.autoGoalTarget], " | ") || "待补充"}
+- Auto goal reached: ${data.autoGoalReached || "待补充"}
+- Auto stop reason: ${data.autoStopReason || "待补充"}
+- Auto final artifact: ${data.autoFinalArtifact || "待补充"}
+- Auto final rung: ${data.autoFinalRung || "待补充"}
+- Eval objective: ${data.evalObjective || "待补充"}
+- Table plan: ${data.evalTablePlan || "待补充"}
+- Metric glossary: ${data.evalMetricGlossary || "待补充"}
+- Metric source papers: ${data.evalMetricSourcePapers || "待补充"}
+- Metric implementation source: ${data.evalMetricImplementationSource || "待补充"}
+- Comparison source papers: ${data.evalComparisonSourcePapers || "待补充"}
+- Comparison implementation source: ${data.evalComparisonImplementationSource || "待补充"}
+- Deviation from original implementation: ${data.evalDeviationFromOriginalImplementation || "待补充"}
+- Experiment ladder: ${data.evalExperimentLadder || "待补充"}
+- Benchmark ladder: ${data.evalBenchmarkLadder || "待补充"}
+- Promotion gate: ${data.evalPromotionGate || "待补充"}
 - Dataset package: ${data.datasetPackage || "待补充"}
 - Dataset years: ${data.datasetYears || "待补充"}
 - Benchmark role: ${data.benchmarkRole || "待补充"}
@@ -120,7 +139,25 @@ function renderSummary(lang, data) {
 - Active stage: ${data.stage || "TBD"}
 - Auto mode: ${data.autoStatus || "inactive"}
 - Auto objective: ${data.autoObjective || "TBD"}
+- Auto rung: ${data.autoCurrentRung || "TBD"}
+- Auto watch target: ${data.autoWatchTarget || "TBD"}
 - Auto decision: ${data.autoDecision || "TBD"}
+- Auto terminal goal: ${joinNonEmpty([data.autoGoalType, data.autoGoalTarget], " | ") || "TBD"}
+- Auto goal reached: ${data.autoGoalReached || "TBD"}
+- Auto stop reason: ${data.autoStopReason || "TBD"}
+- Auto final artifact: ${data.autoFinalArtifact || "TBD"}
+- Auto final rung: ${data.autoFinalRung || "TBD"}
+- Eval objective: ${data.evalObjective || "TBD"}
+- Table plan: ${data.evalTablePlan || "TBD"}
+- Metric glossary: ${data.evalMetricGlossary || "TBD"}
+- Metric source papers: ${data.evalMetricSourcePapers || "TBD"}
+- Metric implementation source: ${data.evalMetricImplementationSource || "TBD"}
+- Comparison source papers: ${data.evalComparisonSourcePapers || "TBD"}
+- Comparison implementation source: ${data.evalComparisonImplementationSource || "TBD"}
+- Deviation from original implementation: ${data.evalDeviationFromOriginalImplementation || "TBD"}
+- Experiment ladder: ${data.evalExperimentLadder || "TBD"}
+- Benchmark ladder: ${data.evalBenchmarkLadder || "TBD"}
+- Promotion gate: ${data.evalPromotionGate || "TBD"}
 - Dataset package: ${data.datasetPackage || "TBD"}
 - Dataset years: ${data.datasetYears || "TBD"}
 - Benchmark role: ${data.benchmarkRole || "TBD"}
@@ -209,7 +246,25 @@ ${data.problem || "待补充"}
 - Why this is the active path: ${data.why || "当前已批准方向"}
 - Auto mode: ${data.autoStatus || "未启用"}
 - Auto objective: ${data.autoObjective || "待补充"}
+- Auto rung: ${data.autoCurrentRung || "待补充"}
+- Auto watch target: ${data.autoWatchTarget || "待补充"}
 - Auto decision: ${data.autoDecision || "待补充"}
+- Auto terminal goal: ${joinNonEmpty([data.autoGoalType, data.autoGoalTarget], " | ") || "待补充"}
+- Auto goal reached: ${data.autoGoalReached || "待补充"}
+- Auto stop reason: ${data.autoStopReason || "待补充"}
+- Auto final artifact: ${data.autoFinalArtifact || "待补充"}
+- Auto final rung: ${data.autoFinalRung || "待补充"}
+- Eval objective: ${data.evalObjective || "待补充"}
+- Table plan: ${data.evalTablePlan || "待补充"}
+- Metric glossary: ${data.evalMetricGlossary || "待补充"}
+- Metric source papers: ${data.evalMetricSourcePapers || "待补充"}
+- Metric implementation source: ${data.evalMetricImplementationSource || "待补充"}
+- Comparison source papers: ${data.evalComparisonSourcePapers || "待补充"}
+- Comparison implementation source: ${data.evalComparisonImplementationSource || "待补充"}
+- Deviation from original implementation: ${data.evalDeviationFromOriginalImplementation || "待补充"}
+- Experiment ladder: ${data.evalExperimentLadder || "待补充"}
+- Benchmark ladder: ${data.evalBenchmarkLadder || "待补充"}
+- Promotion gate: ${data.evalPromotionGate || "待补充"}
 - Dataset package: ${data.datasetPackage || "待补充"}
 - Dataset years: ${data.datasetYears || "待补充"}
 - Benchmark role: ${data.benchmarkRole || "待补充"}
@@ -254,7 +309,25 @@ ${data.problem || "TBD"}
 - Why this is the active path: ${data.why || "This is the approved direction"}
 - Auto mode: ${data.autoStatus || "inactive"}
 - Auto objective: ${data.autoObjective || "TBD"}
+- Auto rung: ${data.autoCurrentRung || "TBD"}
+- Auto watch target: ${data.autoWatchTarget || "TBD"}
 - Auto decision: ${data.autoDecision || "TBD"}
+- Auto terminal goal: ${joinNonEmpty([data.autoGoalType, data.autoGoalTarget], " | ") || "TBD"}
+- Auto goal reached: ${data.autoGoalReached || "TBD"}
+- Auto stop reason: ${data.autoStopReason || "TBD"}
+- Auto final artifact: ${data.autoFinalArtifact || "TBD"}
+- Auto final rung: ${data.autoFinalRung || "TBD"}
+- Eval objective: ${data.evalObjective || "TBD"}
+- Table plan: ${data.evalTablePlan || "TBD"}
+- Metric glossary: ${data.evalMetricGlossary || "TBD"}
+- Metric source papers: ${data.evalMetricSourcePapers || "TBD"}
+- Metric implementation source: ${data.evalMetricImplementationSource || "TBD"}
+- Comparison source papers: ${data.evalComparisonSourcePapers || "TBD"}
+- Comparison implementation source: ${data.evalComparisonImplementationSource || "TBD"}
+- Deviation from original implementation: ${data.evalDeviationFromOriginalImplementation || "TBD"}
+- Experiment ladder: ${data.evalExperimentLadder || "TBD"}
+- Benchmark ladder: ${data.evalBenchmarkLadder || "TBD"}
+- Promotion gate: ${data.evalPromotionGate || "TBD"}
 - Dataset package: ${data.datasetPackage || "TBD"}
 - Dataset years: ${data.datasetYears || "TBD"}
 - Benchmark role: ${data.benchmarkRole || "TBD"}
@@ -291,6 +364,8 @@ function buildContextSnapshot(targetDir) {
   const dataDecisions = readFileIfExists(contextFile(targetDir, "data-decisions.md"));
   const autoMode = readFileIfExists(contextFile(targetDir, "auto-mode.md"));
   const autoStatus = readFileIfExists(contextFile(targetDir, "auto-status.md"));
+  const autoOutcome = readFileIfExists(contextFile(targetDir, "auto-outcome.md"));
+  const evalProtocol = parseEvalProtocol(targetDir);
   const classicBenchmarks = labelValue(
     dataDecisions,
     ["Classic public benchmarks", "Classic benchmarks"],
@@ -484,7 +559,26 @@ function buildContextSnapshot(targetDir) {
     ),
     autoObjective: extractValue(autoMode, ["Objective", "目标"]),
     autoStatus: extractValue(autoStatus, ["Status", "状态"]),
+    autoCurrentRung: extractValue(autoStatus, ["Current rung", "当前 rung"]),
+    autoWatchTarget: extractValue(autoStatus, ["Watch target", "监视目标"]),
     autoDecision: extractValue(autoStatus, ["Current decision", "当前决策"]),
+    autoGoalType: extractValue(autoOutcome, ["Terminal goal type", "终止目标类型"]),
+    autoGoalTarget: extractValue(autoOutcome, ["Terminal goal target", "终止目标目标值"]),
+    autoGoalReached: extractValue(autoOutcome, ["Goal reached", "目标是否达成"]),
+    autoStopReason: extractValue(autoOutcome, ["Stop reason", "停止原因"]),
+    autoFinalArtifact: extractValue(autoOutcome, ["Final artifact", "最终工件"]),
+    autoFinalRung: extractValue(autoOutcome, ["Final rung", "最终 rung"]),
+    evalObjective: evalProtocol.primaryEvaluationObjective,
+    evalTablePlan: evalProtocol.tablePlan,
+    evalMetricGlossary: evalProtocol.metricGlossary,
+    evalMetricSourcePapers: evalProtocol.metricSourcePapers,
+    evalMetricImplementationSource: evalProtocol.metricImplementationSource,
+    evalComparisonSourcePapers: evalProtocol.comparisonSourcePapers,
+    evalComparisonImplementationSource: evalProtocol.comparisonImplementationSource,
+    evalDeviationFromOriginalImplementation: evalProtocol.deviationFromOriginalImplementation,
+    evalExperimentLadder: evalProtocol.experimentLadder,
+    evalBenchmarkLadder: evalProtocol.benchmarkLadder,
+    evalPromotionGate: evalProtocol.promotionGate,
   };
 }
 

package/lib/eval_protocol.cjs

@@ -0,0 +1,236 @@
+const fs = require("node:fs");
+const path = require("node:path");
+
+const PLACEHOLDER_VALUES = new Set(["", "tbd", "none", "待补充", "无"]);
+const VALID_LADDER_STAGES = new Set(["run", "iterate", "review", "report", "write"]);
+const EVAL_PROTOCOL_FIELDS = [
+  {
+    name: "Primary evaluation objective",
+    key: "primaryEvaluationObjective",
+    labels: ["Primary evaluation objective", "主评估目标"],
+  },
+  {
+    name: "Primary metrics",
+    key: "primaryMetrics",
+    labels: ["Primary metrics", "主指标"],
+  },
+  {
+    name: "Secondary metrics",
+    key: "secondaryMetrics",
+    labels: ["Secondary metrics", "次级指标"],
+  },
+  {
+    name: "Required terminal evidence",
+    key: "requiredTerminalEvidence",
+    labels: ["Required terminal evidence", "必要终局证据"],
+  },
+  {
+    name: "Table plan",
+    key: "tablePlan",
+    labels: ["Table plan", "主表计划"],
+  },
+  {
+    name: "Required claims per table",
+    key: "requiredClaimsPerTable",
+    labels: ["Required claims per table", "每张表必须支撑的 claims"],
+  },
+  {
+    name: "Metric glossary",
+    key: "metricGlossary",
+    labels: ["Metric glossary", "指标释义"],
+  },
+  {
+    name: "Metric source papers",
+    key: "metricSourcePapers",
+    labels: ["Metric source papers", "指标来源论文"],
+  },
+  {
+    name: "Metric implementation source",
+    key: "metricImplementationSource",
+    labels: ["Metric implementation source", "指标实现来源"],
+  },
+  {
+    name: "Comparison source papers",
+    key: "comparisonSourcePapers",
+    labels: ["Comparison source papers", "对比方法来源论文"],
+  },
+  {
+    name: "Comparison implementation source",
+    key: "comparisonImplementationSource",
+    labels: ["Comparison implementation source", "对比方法实现来源"],
+  },
+  {
+    name: "Deviation from original implementation",
+    key: "deviationFromOriginalImplementation",
+    labels: ["Deviation from original implementation", "与原始实现的偏差"],
+  },
+  {
+    name: "Benchmark ladder",
+    key: "benchmarkLadder",
+    labels: ["Benchmark ladder", "benchmark 阶梯"],
+  },
+  {
+    name: "Experiment ladder",
+    key: "experimentLadder",
+    labels: ["Experiment ladder", "实验阶梯"],
+  },
+  {
+    name: "Comparison gate",
+    key: "comparisonGate",
+    labels: ["Comparison gate", "对比方法 gate"],
+  },
+  {
+    name: "Promotion gate",
+    key: "promotionGate",
+    labels: ["Promotion gate", "升格 gate"],
+  },
+  {
+    name: "Minimum sample sizes",
+    key: "minimumSampleSizes",
+    labels: ["Minimum sample sizes", "最小样本量"],
+  },
+  {
+    name: "Required output artifacts",
+    key: "requiredOutputArtifacts",
+    labels: ["Required output artifacts", "必要输出工件"],
+  },
+];
+const EXPERIMENT_RUNG_FIELDS = [
+  { name: "Stage", key: "stage", labels: ["Stage", "阶段"] },
+  { name: "Goal", key: "goal", labels: ["Goal", "目标"] },
+  { name: "Command", key: "command", labels: ["Command", "命令"] },
+  { name: "Watch", key: "watch", labels: ["Watch", "监视目标"] },
+  { name: "Gate", key: "gate", labels: ["Gate", "gate 命令", "检查命令"] },
+  { name: "On pass", key: "onPass", labels: ["On pass", "通过后"] },
+  { name: "On fail", key: "onFail", labels: ["On fail", "失败后"] },
+  { name: "On stop", key: "onStop", labels: ["On stop", "停止后"] },
+];
+
+function contextFile(targetDir, name) {
+  return path.join(targetDir, ".lab", "context", name);
+}
+
+function readFileIfExists(filePath) {
+  if (!fs.existsSync(filePath)) {
+    return "";
+  }
+  return fs.readFileSync(filePath, "utf8");
+}
+
+function extractValue(text, labels) {
+  for (const label of labels) {
+    const escaped = label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const regex = new RegExp(`^\\s*-\\s*${escaped}:[ \\t]*([^\\n\\r]+?)[ \\t]*$`, "im");
+    const match = text.match(regex);
+    if (match && match[1]) {
+      return match[1].trim();
+    }
+  }
+  return "";
+}
+
+function isMeaningful(value) {
+  return !PLACEHOLDER_VALUES.has((value || "").trim().toLowerCase());
+}
+
+function parseEvalProtocol(targetDir) {
+  const filePath = contextFile(targetDir, "eval-protocol.md");
+  const text = readFileIfExists(filePath);
+  const fields = {};
+
+  for (const field of EVAL_PROTOCOL_FIELDS) {
+    fields[field.key] = extractValue(text, field.labels);
+  }
+
+  const experimentRungs = [];
+  const rungHeadingPattern = /^###\s*Rung:\s*(.+?)\s*$/gm;
+  const rungHeadings = Array.from(text.matchAll(rungHeadingPattern));
+  for (let index = 0; index < rungHeadings.length; index += 1) {
+    const match = rungHeadings[index];
+    const rungId = match[1];
+    const blockStart = match.index || 0;
+    const bodyStart = blockStart + match[0].length;
+    const nextBlockStart = index + 1 < rungHeadings.length ? rungHeadings[index + 1].index : text.length;
+    const rungBody = text.slice(bodyStart, nextBlockStart).trim();
+    const rung = {
+      id: rungId.trim(),
+      block: text.slice(blockStart, nextBlockStart).trim(),
+    };
+    for (const field of EXPERIMENT_RUNG_FIELDS) {
+      rung[field.key] = extractValue(rungBody, field.labels);
+    }
+    rung.stage = (rung.stage || "").trim().toLowerCase();
+    experimentRungs.push(rung);
+  }
+
+  return {
+    path: filePath,
+    text,
+    experimentRungs,
+    ...fields,
+  };
+}
+
+function validateEvalProtocol(targetDir) {
+  const protocol = parseEvalProtocol(targetDir);
+  if (!fs.existsSync(protocol.path)) {
+    return [];
+  }
+
+  const stateText = readFileIfExists(contextFile(targetDir, "state.md"));
+  const autoModeText = readFileIfExists(contextFile(targetDir, "auto-mode.md"));
+  const activeStage = extractValue(stateText, ["Active stage", "当前阶段", "Stage"]).trim().toLowerCase();
+  const autoObjective = extractValue(autoModeText, ["Objective", "目标"]);
+  const activated =
+    EVAL_PROTOCOL_FIELDS.some((field) => isMeaningful(protocol[field.key])) ||
+    ["run", "iterate", "review", "report", "write", "auto"].includes(activeStage) ||
+    isMeaningful(autoObjective);
+
+  if (!activated) {
+    return [];
+  }
+
+  const missing = EVAL_PROTOCOL_FIELDS.filter((field) => !isMeaningful(protocol[field.key])).map(
+    (field) => field.name
+  );
+  if (missing.length === 0) {
+    const rungIssues = [];
+    const rungIds = new Set();
+    for (const rung of protocol.experimentRungs) {
+      if (!isMeaningful(rung.id)) {
+        rungIssues.push("experiment ladder has a rung with an empty id");
+        continue;
+      }
+      if (rungIds.has(rung.id)) {
+        rungIssues.push(`experiment ladder has duplicate rung id: ${rung.id}`);
+      }
+      rungIds.add(rung.id);
+
+      const missingRungFields = EXPERIMENT_RUNG_FIELDS.filter((field) => {
+        if (field.key === "command" && !isMeaningful(rung[field.key])) {
+          return false;
+        }
+        return !isMeaningful(rung[field.key]);
+      }).map((field) => field.name);
+
+      if (missingRungFields.length > 0) {
+        rungIssues.push(`experiment ladder rung ${rung.id} is missing: ${missingRungFields.join(", ")}`);
+      }
+
+      if (isMeaningful(rung.stage) && !VALID_LADDER_STAGES.has(rung.stage)) {
+        rungIssues.push(`experiment ladder rung ${rung.id} has invalid stage: ${rung.stage}`);
+      }
+    }
+
+    return rungIssues;
+  }
+
+  return [`missing evaluation protocol fields: ${missing.join(", ")}`];
+}
+
+module.exports = {
+  EVAL_PROTOCOL_FIELDS,
+  EXPERIMENT_RUNG_FIELDS,
+  parseEvalProtocol,
+  validateEvalProtocol,
+};