soturail 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -16
- package/dist/commands/bench.d.ts +1 -0
- package/dist/commands/bench.js +18 -6
- package/dist/commands/bench.js.map +1 -1
- package/dist/commands/brain.js +14 -7
- package/dist/commands/brain.js.map +1 -1
- package/dist/commands/native.js +12 -0
- package/dist/commands/native.js.map +1 -1
- package/dist/commands/rules.js +20 -1
- package/dist/commands/rules.js.map +1 -1
- package/dist/commands/self.js +19 -0
- package/dist/commands/self.js.map +1 -1
- package/dist/core/baseline-snapshot.d.ts +27 -0
- package/dist/core/baseline-snapshot.js +225 -0
- package/dist/core/baseline-snapshot.js.map +1 -0
- package/dist/core/benchmark-rail.d.ts +44 -0
- package/dist/core/benchmark-rail.js +322 -0
- package/dist/core/benchmark-rail.js.map +1 -0
- package/dist/core/config.d.ts +5 -0
- package/dist/core/config.js +6 -1
- package/dist/core/config.js.map +1 -1
- package/dist/core/evaluation-suite.js +117 -2
- package/dist/core/evaluation-suite.js.map +1 -1
- package/dist/core/evidence-pack.js +17 -0
- package/dist/core/evidence-pack.js.map +1 -1
- package/dist/core/harness-rail.js +2 -0
- package/dist/core/harness-rail.js.map +1 -1
- package/dist/core/native-candidates.d.ts +44 -0
- package/dist/core/native-candidates.js +163 -0
- package/dist/core/native-candidates.js.map +1 -0
- package/dist/core/project-brain.d.ts +70 -4
- package/dist/core/project-brain.js +475 -30
- package/dist/core/project-brain.js.map +1 -1
- package/dist/core/release-preflight.js +8 -0
- package/dist/core/release-preflight.js.map +1 -1
- package/dist/core/version.d.ts +1 -1
- package/dist/core/version.js +1 -1
- package/docs/agents.md +9 -1
- package/docs/baseline-snapshots.md +54 -0
- package/docs/benchmarking.md +69 -0
- package/docs/code-graph.md +33 -0
- package/docs/diagram-rail.md +2 -0
- package/docs/evaluation-suite.md +23 -0
- package/docs/harness-rail.md +10 -0
- package/docs/knowledge-to-rules.md +23 -0
- package/docs/memory-rail.md +2 -0
- package/docs/native-performance-policy.md +62 -0
- package/docs/native-runner.md +24 -0
- package/docs/project-brain.md +101 -4
- package/docs/release-workflow.md +13 -0
- package/docs/releases/README.md +2 -0
- package/docs/releases/RELEASE_NOTES_v0.8.1.md +42 -0
- package/docs/releases/RELEASE_NOTES_v0.9.0.md +35 -0
- package/docs/reverse-specification-rail.md +19 -0
- package/docs/rules.md +3 -0
- package/docs/workflow-rail.md +23 -0
- package/package.json +1 -1
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { promises as fs } from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
import { getWorkspacePaths, relativeToRoot, writeJson } from "./config.js";
|
|
7
|
+
import { SOTURAIL_VERSION } from "./version.js";
|
|
8
|
+
const execFileAsync = promisify(execFile);
|
|
9
|
+
export async function baselineSnapshot(root = process.cwd(), mode) {
|
|
10
|
+
const resolvedRoot = path.resolve(root);
|
|
11
|
+
const packageJson = await readJson(path.join(resolvedRoot, "package.json"));
|
|
12
|
+
const version = typeof packageJson.version === "string" ? packageJson.version : SOTURAIL_VERSION;
|
|
13
|
+
const dir = path.join(getWorkspacePaths(resolvedRoot).workspace, "baselines");
|
|
14
|
+
await fs.mkdir(dir, { recursive: true });
|
|
15
|
+
const status = await run("git", ["status", "--short"], resolvedRoot, true);
|
|
16
|
+
const commit = await run("git", ["rev-parse", "--short", "HEAD"], resolvedRoot, true);
|
|
17
|
+
const releaseNotes = path.join("docs", "releases", `RELEASE_NOTES_v${version}.md`);
|
|
18
|
+
const signals = await baselineSignals(resolvedRoot, version);
|
|
19
|
+
const commands = [];
|
|
20
|
+
const artifacts = [];
|
|
21
|
+
const warnings = [
|
|
22
|
+
"Do not manually zip the whole working directory.",
|
|
23
|
+
"Use git archive for source zip.",
|
|
24
|
+
"Use git bundle for history backup.",
|
|
25
|
+
"Use npm pack for npm package snapshot."
|
|
26
|
+
];
|
|
27
|
+
if (mode === "zip") {
|
|
28
|
+
const output = path.join(dir, `soturail-v${version}-source.zip`);
|
|
29
|
+
const command = `git archive --format=zip --output ${relativeToRoot(resolvedRoot, output)} HEAD`;
|
|
30
|
+
commands.push(command);
|
|
31
|
+
const result = await run("git", ["archive", "--format=zip", "--output", output, "HEAD"], resolvedRoot, true);
|
|
32
|
+
if (result.code === 0)
|
|
33
|
+
artifacts.push(output);
|
|
34
|
+
else
|
|
35
|
+
warnings.push(`git archive was not executed successfully; recommended command: ${command}`);
|
|
36
|
+
}
|
|
37
|
+
if (mode === "bundle") {
|
|
38
|
+
const output = path.join(dir, `soturail-v${version}.bundle`);
|
|
39
|
+
const command = `git bundle create ${relativeToRoot(resolvedRoot, output)} --all`;
|
|
40
|
+
commands.push(command);
|
|
41
|
+
const result = await run("git", ["bundle", "create", output, "--all"], resolvedRoot, true);
|
|
42
|
+
if (result.code === 0)
|
|
43
|
+
artifacts.push(output);
|
|
44
|
+
else
|
|
45
|
+
warnings.push(`git bundle was not executed successfully; recommended command: ${command}`);
|
|
46
|
+
}
|
|
47
|
+
if (mode === "pack") {
|
|
48
|
+
const command = `npm pack --pack-destination ${relativeToRoot(resolvedRoot, dir)}`;
|
|
49
|
+
commands.push(command);
|
|
50
|
+
const result = await run("npm", ["pack", "--pack-destination", dir], resolvedRoot, true);
|
|
51
|
+
const tarball = extractTarballName(`${result.stdout}\n${result.stderr}`);
|
|
52
|
+
if (result.code === 0 && tarball)
|
|
53
|
+
artifacts.push(path.join(dir, path.basename(tarball)));
|
|
54
|
+
else
|
|
55
|
+
warnings.push(`npm pack was not executed successfully; recommended command: ${command}`);
|
|
56
|
+
}
|
|
57
|
+
if (mode === "check") {
|
|
58
|
+
commands.push(`git archive --format=zip --output .soturail/baselines/soturail-v${version}-source.zip HEAD`, `git bundle create .soturail/baselines/soturail-v${version}.bundle --all`, `npm pack --pack-destination .soturail/baselines`);
|
|
59
|
+
}
|
|
60
|
+
const report = {
|
|
61
|
+
schemaVersion: "soturail.baseline.v1",
|
|
62
|
+
createdAt: new Date().toISOString(),
|
|
63
|
+
version,
|
|
64
|
+
gitCommit: commit.stdout.trim() || "unknown",
|
|
65
|
+
dirtyStatus: status.stdout.trim() ? "dirty" : "clean",
|
|
66
|
+
githubFolderStatus: existsSync(path.join(resolvedRoot, ".github")) ? "present" : "missing",
|
|
67
|
+
nodeModulesStatus: existsSync(path.join(resolvedRoot, "node_modules")) ? "present-not-for-source-snapshot" : "not present",
|
|
68
|
+
releaseNotesPath: releaseNotes,
|
|
69
|
+
mode,
|
|
70
|
+
signals,
|
|
71
|
+
recommendedArtifacts: [
|
|
72
|
+
`.soturail/baselines/soturail-v${version}-source.zip`,
|
|
73
|
+
`.soturail/baselines/soturail-v${version}.bundle`,
|
|
74
|
+
`soturail-${version}.tgz`
|
|
75
|
+
],
|
|
76
|
+
commandsUsedOrRecommended: commands,
|
|
77
|
+
artifacts: artifacts.map((artifact) => relativeToRoot(resolvedRoot, artifact)),
|
|
78
|
+
warnings
|
|
79
|
+
};
|
|
80
|
+
await writeBaselineReports(resolvedRoot, report);
|
|
81
|
+
return { report, output: renderBaselineOutput(resolvedRoot, report) };
|
|
82
|
+
}
|
|
83
|
+
async function baselineSignals(root, version) {
|
|
84
|
+
const changelog = await readText(path.join(root, "CHANGELOG.md"));
|
|
85
|
+
const readme = await readText(path.join(root, "README.md"));
|
|
86
|
+
const versionTs = await readText(path.join(root, "src", "core", "version.ts"));
|
|
87
|
+
const releaseNotes = path.join(root, "docs", "releases", `RELEASE_NOTES_v${version}.md`);
|
|
88
|
+
const pack = await run("npm", ["pack", "--dry-run", "--json"], root, true);
|
|
89
|
+
const packText = `${pack.stdout}\n${pack.stderr}`;
|
|
90
|
+
return [
|
|
91
|
+
signal("git_dir", existsSync(path.join(root, ".git")), ".git exists"),
|
|
92
|
+
signal("github_dir", existsSync(path.join(root, ".github")), ".github exists"),
|
|
93
|
+
signal("github_ci", existsSync(path.join(root, ".github", "workflows", "ci.yml")), ".github/workflows/ci.yml exists"),
|
|
94
|
+
signal("node_modules_source_snapshot", true, existsSync(path.join(root, "node_modules")) ? "node_modules present; exclude from source snapshots" : "node_modules not present"),
|
|
95
|
+
signal("docs_releases", existsSync(path.join(root, "docs", "releases")), "docs/releases exists"),
|
|
96
|
+
signal("readme", readme.length > 0, "README.md exists"),
|
|
97
|
+
signal("version_sync", versionTs.includes(`"${version}"`) || SOTURAIL_VERSION === version, `package=${version}, cli=${SOTURAIL_VERSION}`),
|
|
98
|
+
signal("changelog_entry", changelog.includes(`## [${version}]`), `expected CHANGELOG entry ## [${version}]`),
|
|
99
|
+
signal("release_notes", existsSync(releaseNotes), path.join("docs", "releases", `RELEASE_NOTES_v${version}.md`)),
|
|
100
|
+
signal("npm_pack_metadata", pack.code === 0 && packText.includes("README.md") && packText.includes("package.json"), "npm pack dry-run includes package metadata")
|
|
101
|
+
];
|
|
102
|
+
}
|
|
103
|
+
async function writeBaselineReports(root, report) {
|
|
104
|
+
const dir = path.join(getWorkspacePaths(root).workspace, "baselines");
|
|
105
|
+
await fs.mkdir(dir, { recursive: true });
|
|
106
|
+
await writeJson(path.join(dir, "latest.json"), report);
|
|
107
|
+
await fs.writeFile(path.join(dir, "latest.md"), renderBaselineMarkdown(report), "utf8");
|
|
108
|
+
}
|
|
109
|
+
function renderBaselineOutput(root, report) {
|
|
110
|
+
return [
|
|
111
|
+
"SotuRail self baseline",
|
|
112
|
+
`mode: ${report.mode}`,
|
|
113
|
+
`version: ${report.version}`,
|
|
114
|
+
`git_commit: ${report.gitCommit}`,
|
|
115
|
+
`dirty_status: ${report.dirtyStatus}`,
|
|
116
|
+
`release_notes_path: ${report.releaseNotesPath}`,
|
|
117
|
+
`signals_passed: ${report.signals.filter((item) => item.ok).length}`,
|
|
118
|
+
`signals_failed: ${report.signals.filter((item) => !item.ok && item.required).length}`,
|
|
119
|
+
`artifacts: ${report.artifacts.length === 0 ? "none" : report.artifacts.join(", ")}`,
|
|
120
|
+
`json: ${relativeToRoot(root, path.join(getWorkspacePaths(root).workspace, "baselines", "latest.json"))}`,
|
|
121
|
+
`markdown: ${relativeToRoot(root, path.join(getWorkspacePaths(root).workspace, "baselines", "latest.md"))}`,
|
|
122
|
+
"",
|
|
123
|
+
"Warnings:",
|
|
124
|
+
...report.warnings.map((warning) => `- ${warning}`),
|
|
125
|
+
"",
|
|
126
|
+
"Commands:",
|
|
127
|
+
...report.commandsUsedOrRecommended.map((command) => `- ${command}`)
|
|
128
|
+
].join("\n") + "\n";
|
|
129
|
+
}
|
|
130
|
+
function renderBaselineMarkdown(report) {
|
|
131
|
+
return [
|
|
132
|
+
"# SotuRail Baseline Snapshot Report",
|
|
133
|
+
"",
|
|
134
|
+
`schemaVersion: ${report.schemaVersion}`,
|
|
135
|
+
`createdAt: ${report.createdAt}`,
|
|
136
|
+
`mode: ${report.mode}`,
|
|
137
|
+
`version: ${report.version}`,
|
|
138
|
+
`gitCommit: ${report.gitCommit}`,
|
|
139
|
+
`dirtyStatus: ${report.dirtyStatus}`,
|
|
140
|
+
`githubFolderStatus: ${report.githubFolderStatus}`,
|
|
141
|
+
`nodeModulesStatus: ${report.nodeModulesStatus}`,
|
|
142
|
+
`releaseNotesPath: ${report.releaseNotesPath}`,
|
|
143
|
+
"",
|
|
144
|
+
"## Signals",
|
|
145
|
+
"",
|
|
146
|
+
...report.signals.map((item) => `- ${item.ok ? "PASS" : item.required ? "FAIL" : "WARN"} ${item.id}: ${item.details}`),
|
|
147
|
+
"",
|
|
148
|
+
"## Recommended Artifacts",
|
|
149
|
+
"",
|
|
150
|
+
...report.recommendedArtifacts.map((artifact) => `- ${artifact}`),
|
|
151
|
+
"",
|
|
152
|
+
"## Commands",
|
|
153
|
+
"",
|
|
154
|
+
...report.commandsUsedOrRecommended.map((command) => `- ${command}`),
|
|
155
|
+
"",
|
|
156
|
+
"## Artifacts",
|
|
157
|
+
"",
|
|
158
|
+
...(report.artifacts.length > 0 ? report.artifacts.map((artifact) => `- ${artifact}`) : ["- none"]),
|
|
159
|
+
"",
|
|
160
|
+
"## Warnings",
|
|
161
|
+
"",
|
|
162
|
+
...report.warnings.map((warning) => `- ${warning}`),
|
|
163
|
+
""
|
|
164
|
+
].join("\n");
|
|
165
|
+
}
|
|
166
|
+
function signal(id, ok, details, required = true) {
|
|
167
|
+
return { id, ok, details, required };
|
|
168
|
+
}
|
|
169
|
+
async function readText(filePath) {
|
|
170
|
+
return fs.readFile(filePath, "utf8").catch(() => "");
|
|
171
|
+
}
|
|
172
|
+
async function readJson(filePath) {
|
|
173
|
+
const raw = await readText(filePath);
|
|
174
|
+
if (!raw)
|
|
175
|
+
return {};
|
|
176
|
+
try {
|
|
177
|
+
return JSON.parse(raw);
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
return {};
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
async function run(command, args, cwd, allowFailure = false) {
|
|
184
|
+
try {
|
|
185
|
+
const executable = resolveWindowsCommand(command);
|
|
186
|
+
const runner = windowsScriptRunner(executable, args);
|
|
187
|
+
const result = await execFileAsync(runner.command, runner.args, { cwd, windowsHide: true, timeout: 30000, maxBuffer: 16 * 1024 * 1024 });
|
|
188
|
+
return { code: 0, stdout: result.stdout, stderr: result.stderr };
|
|
189
|
+
}
|
|
190
|
+
catch (error) {
|
|
191
|
+
const maybe = error;
|
|
192
|
+
if (allowFailure) {
|
|
193
|
+
return { code: typeof maybe.code === "number" ? maybe.code : 1, stdout: maybe.stdout ?? "", stderr: maybe.stderr ?? maybe.message ?? "" };
|
|
194
|
+
}
|
|
195
|
+
throw error;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
function extractTarballName(text) {
|
|
199
|
+
const filename = /"filename"\s*:\s*"([^"]+\.tgz)"/.exec(text)?.[1];
|
|
200
|
+
if (filename)
|
|
201
|
+
return filename;
|
|
202
|
+
return text.split(/\r?\n/).map((line) => line.trim()).find((line) => line.endsWith(".tgz"));
|
|
203
|
+
}
|
|
204
|
+
function windowsScriptRunner(command, args) {
|
|
205
|
+
if (process.platform !== "win32" || !/\.(cmd|bat)$/i.test(command))
|
|
206
|
+
return { command, args };
|
|
207
|
+
const comspec = process.env.ComSpec ?? "cmd.exe";
|
|
208
|
+
return { command: comspec, args: ["/d", "/c", "call", command, ...args] };
|
|
209
|
+
}
|
|
210
|
+
function resolveWindowsCommand(command) {
|
|
211
|
+
if (process.platform !== "win32")
|
|
212
|
+
return command;
|
|
213
|
+
if (command === "node")
|
|
214
|
+
return process.execPath;
|
|
215
|
+
const extensions = path.extname(command) ? [""] : [".cmd", ".exe", ".bat", ""];
|
|
216
|
+
for (const entry of (process.env.PATH ?? "").split(path.delimiter).filter(Boolean)) {
|
|
217
|
+
for (const extension of extensions) {
|
|
218
|
+
const candidate = path.resolve(entry, `${command}${extension}`);
|
|
219
|
+
if (existsSync(candidate))
|
|
220
|
+
return candidate;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return command;
|
|
224
|
+
}
|
|
225
|
+
//# sourceMappingURL=baseline-snapshot.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"baseline-snapshot.js","sourceRoot":"","sources":["../../src/core/baseline-snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AA4B1C,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,EAAE,IAAkB;IAC7E,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAG,OAAO,WAAW,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACjG,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAC9E,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IACtF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,kBAAkB,OAAO,KAAK,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAG;QACf,kDAAkD;QAClD,iCAAiC;QACjC,oCAAoC;QACpC,wCAAwC;KACzC,CAAC;IAEF,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,OAAO,aAAa,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,qCAAqC,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC;QACjG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;QAC7G,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;YAAE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;YACzC,QAAQ,CAAC,IAAI,CAAC,mEAAmE,OAAO,EAAE,CAAC,CAAC;IACnG,CAAC;IAED,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,OAAO,SAAS,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,qBAAqB,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC;QAClF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;QAC3F,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;YAAE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;;YACzC,QAAQ,CAAC,IAAI,CAAC,kEAAkE,OAAO,EAAE,CAAC,CAAC;IAClG,CAAC;IAED,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,+BAA+B,cAAc,CAAC,YAAY,EAAE,GAAG,CAAC,EAAE,CAAC;QACnF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,oBAAoB,EAAE,GAAG,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;QACzF,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACzE,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,OAAO;YAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;;YACpF,QAAQ,CAAC,IAAI,CAAC,gEAAgE,OAAO,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACrB,QAAQ,CAAC,IAAI,CACX,mEAAmE,OAAO,kBAAkB,EAC5F,mDAAmD,OAAO,eAAe,EACzE,iDAAiD,CAClD,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAmB;QAC7B,aAAa,EAAE,sBAAsB;QACrC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO;QACP,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,SAAS;QAC5C,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO;QACrD,kBAAkB,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;QAC1F,iBAAiB,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,iCAAiC,CAAC,CAAC,CAAC,aAAa;QAC1H,gBAAgB,EAAE,YAAY;QAC9B,IAAI;QACJ,OAAO;QACP,oBAAoB,EAAE;YACpB,iCAAiC,OAAO,aAAa;YACrD,iCAAiC,OAAO,SAAS;YACjD,YAAY,OAAO,MAAM;SAC1B;QACD,yBAAyB,EAAE,QAAQ;QACnC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,cAAc,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QAC9E,QAAQ;KACT,CAAC;IACF,MAAM,oBAAoB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACjD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,oBAAoB,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,OAAe;IAC1D,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;IAC/E,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,kBAAkB,OAAO,KAAK,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;IAClD,OAAO;QACL,MAAM,CAAC,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,aAAa,CAAC;QACrE,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,EAAE,gBAAgB,CAAC;QAC9E,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,EAAE,iCAAiC,CAAC;QACrH,MAAM,CAAC,8BAA8B,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,qDAAqD,CAAC,CAAC,CAAC,0BAA0B,CAAC;QAC9K,MAAM,CAAC,eAAe,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,EAAE,sBAAsB,CAAC;QAChG,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,kBAAkB,CAAC;QACvD,MAAM,CAAC,cAAc,EAAE,SAAS,CAAC,QAAQ,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,gBAAgB,KAAK,OAAO,EAAE,WAAW,OAAO,SAAS,gBAAgB,EAAE,CAAC;QACzI,MAAM,CAAC,iBAAiB,EAAE,SAAS,CAAC,QAAQ,CAAC,OAAO,OAAO,GAAG,CAAC,EAAE,gCAAgC,OAAO,GAAG,CAAC;QAC5G,MAAM,CAAC,eAAe,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,kBAAkB,OAAO,KAAK,CAAC,CAAC;QAChH,MAAM,CAAC,mBAAmB,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,4CAA4C,CAAC;KAClK,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,IAAY,EAAE,MAAsB;IACtE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACtE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;IACvD,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,sBAAsB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;AAC1F,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY,EAAE,MAAsB;IAChE,OAAO;QACL,wBAAwB;QACxB,SAAS,MAAM,CAAC,IAAI,EAAE;QACtB,YAAY,MAAM,CAAC,OAAO,EAAE;QAC5B,eAAe,MAAM,CAAC,SAAS,EAAE;QACjC,iBAAiB,MAAM,CAAC,WAAW,EAAE;QACrC,uBAAuB,MAAM,CAAC,gBAAgB,EAAE;QAChD,mBAAmB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;QACpE,mBAAmB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE;QACtF,cAAc,MAAM,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpF,SAAS,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC,EAAE;QACzG,aAAa,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,EAAE;QAC3G,EAAE;QACF,WAAW;QACX,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,OAAO,EAAE,CAAC;QACnD,EAAE;QACF,WAAW;QACX,GAAG,MAAM,CAAC,yBAAyB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,OAAO,EAAE,CAAC;KACrE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACtB,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAsB;IACpD,OAAO;QACL,qCAAqC;QACrC,EAAE;QACF,kBAAkB,MAAM,CAAC,aAAa,EAAE;QACxC,cAAc,MAAM,CAAC,SAAS,EAAE;QAChC,SAAS,MAAM,CAAC,IAAI,EAAE;QACtB,YAAY,MAAM,CAAC,OAAO,EAAE;QAC5B,cAAc,MAAM,CAAC,SAAS,EAAE;QAChC,gBAAgB,MAAM,CAAC,WAAW,EAAE;QACpC,uBAAuB,MAAM,CAAC,kBAAkB,EAAE;QAClD,sBAAsB,MAAM,CAAC,iBAAiB,EAAE;QAChD,qBAAqB,MAAM,CAAC,gBAAgB,EAAE;QAC9C,EAAE;QACF,YAAY;QACZ,EAAE;QACF,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;QACtH,EAAE;QACF,0BAA0B;QAC1B,EAAE;QACF,GAAG,MAAM,CAAC,oBAAoB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,QAAQ,EAAE,CAAC;QACjE,EAAE;QACF,aAAa;QACb,EAAE;QACF,GAAG,MAAM,CAAC,yBAAyB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,OAAO,EAAE,CAAC;QACpE,EAAE;QACF,cAAc;QACd,EAAE;QACF,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACnG,EAAE;QACF,aAAa;QACb,EAAE;QACF,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,OAAO,EAAE,CAAC;QACnD,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,MAAM,CAAC,EAAU,EAAE,EAAW,EAAE,OAAe,EAAE,QAAQ,GAAG,IAAI;IACvE,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,QAAgB;IACtC,OAAO,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,QAAgB;IACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACrC,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,GAAG,CAAC,OAAe,EAAE,IAAc,EAAE,GAAW,EAAE,YAAY,GAAG,KAAK;IACnF,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,mBAAmB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC;QACzI,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IACnE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,KAA8E,CAAC;QAC7F,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,EAAE,IAAI,EAAE,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QAC5I,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,QAAQ,GAAG,iCAAiC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACnE,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AAC9F,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAe,EAAE,IAAc;IAC1D,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC7F,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,SAAS,CAAC;IACjD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAe;IAC5C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,OAAO,CAAC;IACjD,IAAI,OAAO,KAAK,MAAM;QAAE,OAAO,OAAO,CAAC,QAAQ,CAAC;IAChD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;IAC/E,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACnF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,OAAO,GAAG,SAAS,EAAE,CAAC,CAAC;YAChE,IAAI,UAAU,CAAC,SAAS,CAAC;gBAAE,OAAO,SAAS,CAAC;QAC9C,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
export type BenchmarkRailSuite = "all" | "brain" | "reducers" | "filesystem" | "release";
|
|
2
|
+
export type BenchmarkRailCategory = "brain-scan" | "brain-stale" | "brain-consolidate" | "reverse-claims" | "reducer-large-log" | "jsonl-read-write" | "range-hash" | "file-scan" | "workflow-evidence" | "format-compare" | "json-validate" | "release-preflight";
|
|
3
|
+
export interface BenchmarkRailCase {
|
|
4
|
+
id: string;
|
|
5
|
+
category: BenchmarkRailCategory;
|
|
6
|
+
operation: string;
|
|
7
|
+
durationMs: number;
|
|
8
|
+
recordsRead: number;
|
|
9
|
+
recordsWritten: number;
|
|
10
|
+
filesScanned: number;
|
|
11
|
+
engine: "typescript" | "native";
|
|
12
|
+
status: "passed" | "failed" | "warning";
|
|
13
|
+
warnings: string[];
|
|
14
|
+
}
|
|
15
|
+
export interface BenchmarkRailReport {
|
|
16
|
+
schemaVersion: "soturail.bench.v1";
|
|
17
|
+
id: string;
|
|
18
|
+
createdAt: string;
|
|
19
|
+
suite: BenchmarkRailSuite;
|
|
20
|
+
version: string;
|
|
21
|
+
environment: {
|
|
22
|
+
platform: string;
|
|
23
|
+
node: string;
|
|
24
|
+
nativeAvailable: boolean;
|
|
25
|
+
engine: "typescript" | "native";
|
|
26
|
+
};
|
|
27
|
+
cases: BenchmarkRailCase[];
|
|
28
|
+
summary: {
|
|
29
|
+
passed: number;
|
|
30
|
+
failed: number;
|
|
31
|
+
warnings: number;
|
|
32
|
+
totalDurationMs: number;
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
export declare function benchmarkSuites(): Array<{
|
|
36
|
+
suite: BenchmarkRailSuite;
|
|
37
|
+
categories: BenchmarkRailCategory[];
|
|
38
|
+
}>;
|
|
39
|
+
export declare function renderBenchmarkList(): string;
|
|
40
|
+
export declare function runBenchmarkRail(root?: string, suite?: BenchmarkRailSuite): Promise<BenchmarkRailReport>;
|
|
41
|
+
export declare function readBenchmarkRailReport(root?: string): Promise<string>;
|
|
42
|
+
export declare function compareBenchmarkRail(root?: string): Promise<string>;
|
|
43
|
+
export declare function renderBenchmarkRun(report: BenchmarkRailReport, root?: string): string;
|
|
44
|
+
export declare function parseBenchmarkSuite(value: string | undefined): BenchmarkRailSuite;
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
import { promises as fs } from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { performance } from "node:perf_hooks";
|
|
5
|
+
import { compressOutputWithEngine } from "../compressors/index.js";
|
|
6
|
+
import { compareFormats } from "./format-compare.js";
|
|
7
|
+
import { appendJsonl, defaultConfig, ensureWorkspace, getWorkspacePaths, readJsonl, relativeToRoot, writeJson } from "./config.js";
|
|
8
|
+
import { scanRepository } from "./file-scanner.js";
|
|
9
|
+
import { validateJsonFile } from "./json-validator.js";
|
|
10
|
+
import { detectNativeEngine } from "./native-engine.js";
|
|
11
|
+
import { consolidateBrain, scanBrain, staleBrain } from "./project-brain.js";
|
|
12
|
+
import { reverseClaims } from "./reverse-specification.js";
|
|
13
|
+
import { sha256Text } from "./rail-utils.js";
|
|
14
|
+
import { buildWorkflowEvidence } from "./evidence-pack.js";
|
|
15
|
+
import { createWorkflowPlan } from "./workflow-store.js";
|
|
16
|
+
import { runReleasePreflight } from "./release-preflight.js";
|
|
17
|
+
import { SOTURAIL_VERSION } from "./version.js";
|
|
18
|
+
const categorySuites = {
|
|
19
|
+
brain: ["brain-scan", "brain-stale", "brain-consolidate", "reverse-claims", "jsonl-read-write", "range-hash"],
|
|
20
|
+
reducers: ["reducer-large-log", "format-compare", "json-validate"],
|
|
21
|
+
filesystem: ["file-scan", "workflow-evidence"],
|
|
22
|
+
release: ["release-preflight"]
|
|
23
|
+
};
|
|
24
|
+
export function benchmarkSuites() {
|
|
25
|
+
return [
|
|
26
|
+
{ suite: "all", categories: allCategories() },
|
|
27
|
+
...Object.entries(categorySuites).map(([suite, categories]) => ({
|
|
28
|
+
suite: suite,
|
|
29
|
+
categories
|
|
30
|
+
}))
|
|
31
|
+
];
|
|
32
|
+
}
|
|
33
|
+
export function renderBenchmarkList() {
|
|
34
|
+
return [
|
|
35
|
+
"SotuRail bench list",
|
|
36
|
+
"schemaVersion: soturail.bench.v1",
|
|
37
|
+
"",
|
|
38
|
+
...benchmarkSuites().flatMap((suite) => [
|
|
39
|
+
`## ${suite.suite}`,
|
|
40
|
+
...suite.categories.map((category) => `- ${category}`),
|
|
41
|
+
""
|
|
42
|
+
]),
|
|
43
|
+
"Rules:",
|
|
44
|
+
"- Benchmarks are local and deterministic smoke fixtures by default.",
|
|
45
|
+
"- Timing values are evidence, not release claims by themselves.",
|
|
46
|
+
"- Native acceleration remains optional and benchmark-gated."
|
|
47
|
+
].join("\n") + "\n";
|
|
48
|
+
}
|
|
49
|
+
export async function runBenchmarkRail(root = process.cwd(), suite = "all") {
|
|
50
|
+
await ensureWorkspace(root);
|
|
51
|
+
const native = await detectNativeEngine(root);
|
|
52
|
+
const fixtureRoot = await createBenchmarkProject();
|
|
53
|
+
const categories = suite === "all" ? allCategories() : categorySuites[suite];
|
|
54
|
+
const cases = [];
|
|
55
|
+
for (const category of categories) {
|
|
56
|
+
cases.push(await runCase(category, fixtureRoot));
|
|
57
|
+
}
|
|
58
|
+
const report = {
|
|
59
|
+
schemaVersion: "soturail.bench.v1",
|
|
60
|
+
id: `bench_${Date.now().toString(36)}`,
|
|
61
|
+
createdAt: new Date().toISOString(),
|
|
62
|
+
suite,
|
|
63
|
+
version: SOTURAIL_VERSION,
|
|
64
|
+
environment: {
|
|
65
|
+
platform: `${process.platform}/${process.arch}`,
|
|
66
|
+
node: process.version,
|
|
67
|
+
nativeAvailable: native.available,
|
|
68
|
+
engine: "typescript"
|
|
69
|
+
},
|
|
70
|
+
cases,
|
|
71
|
+
summary: {
|
|
72
|
+
passed: cases.filter((item) => item.status === "passed").length,
|
|
73
|
+
failed: cases.filter((item) => item.status === "failed").length,
|
|
74
|
+
warnings: cases.filter((item) => item.status === "warning").length + cases.reduce((sum, item) => sum + item.warnings.length, 0),
|
|
75
|
+
totalDurationMs: Number(cases.reduce((sum, item) => sum + item.durationMs, 0).toFixed(3))
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
await writeBenchmarkRailReports(root, report);
|
|
79
|
+
return report;
|
|
80
|
+
}
|
|
81
|
+
export async function readBenchmarkRailReport(root = process.cwd()) {
|
|
82
|
+
const latest = path.join(getWorkspacePaths(root).workspace, "bench", "latest.md");
|
|
83
|
+
return fs.readFile(latest, "utf8").catch(() => fs.readFile(path.join(root, "benchmarks", "reports", "latest.md"), "utf8")
|
|
84
|
+
.catch(() => "No benchmark report found. Run: soturail bench run\n"));
|
|
85
|
+
}
|
|
86
|
+
export async function compareBenchmarkRail(root = process.cwd()) {
|
|
87
|
+
const latestPath = path.join(getWorkspacePaths(root).workspace, "bench", "latest.json");
|
|
88
|
+
const latest = await fs.readFile(latestPath, "utf8").catch(() => "");
|
|
89
|
+
if (!latest) {
|
|
90
|
+
return "SotuRail bench compare\nresult: no latest benchmark report\nnext: soturail bench run --suite brain\n";
|
|
91
|
+
}
|
|
92
|
+
const report = JSON.parse(latest);
|
|
93
|
+
const candidateRows = report.cases.map((item) => {
|
|
94
|
+
const classification = item.durationMs >= 25 || item.recordsRead >= 50 || item.filesScanned >= 30 ? "candidate" : "baseline";
|
|
95
|
+
return `- ${item.category}: ${classification} (${item.durationMs}ms, records_read=${item.recordsRead}, files_scanned=${item.filesScanned})`;
|
|
96
|
+
});
|
|
97
|
+
return [
|
|
98
|
+
"SotuRail bench compare",
|
|
99
|
+
`latest: ${relativeToRoot(root, latestPath)}`,
|
|
100
|
+
`suite: ${report.suite}`,
|
|
101
|
+
`cases: ${report.cases.length}`,
|
|
102
|
+
"comparison: latest local TypeScript baseline only",
|
|
103
|
+
"native_claim: no native speedup claimed without a native report",
|
|
104
|
+
"",
|
|
105
|
+
...candidateRows,
|
|
106
|
+
""
|
|
107
|
+
].join("\n");
|
|
108
|
+
}
|
|
109
|
+
export function renderBenchmarkRun(report, root = process.cwd()) {
|
|
110
|
+
const paths = benchmarkReportPaths(root);
|
|
111
|
+
return [
|
|
112
|
+
"SotuRail bench run",
|
|
113
|
+
`schemaVersion: ${report.schemaVersion}`,
|
|
114
|
+
`suite: ${report.suite}`,
|
|
115
|
+
`version: ${report.version}`,
|
|
116
|
+
`cases_count: ${report.cases.length}`,
|
|
117
|
+
`passed: ${report.summary.passed}`,
|
|
118
|
+
`failed: ${report.summary.failed}`,
|
|
119
|
+
`warnings: ${report.summary.warnings}`,
|
|
120
|
+
`totalDurationMs: ${report.summary.totalDurationMs}`,
|
|
121
|
+
`json: ${relativeToRoot(root, paths.latestJson)}`,
|
|
122
|
+
`markdown: ${relativeToRoot(root, paths.latestMd)}`,
|
|
123
|
+
`versioned_json: ${relativeToRoot(root, paths.versionedJson)}`,
|
|
124
|
+
`versioned_markdown: ${relativeToRoot(root, paths.versionedMd)}`
|
|
125
|
+
].join("\n") + "\n";
|
|
126
|
+
}
|
|
127
|
+
export function parseBenchmarkSuite(value) {
|
|
128
|
+
if (!value || value === "all")
|
|
129
|
+
return "all";
|
|
130
|
+
if (value === "brain" || value === "reducers" || value === "filesystem" || value === "release")
|
|
131
|
+
return value;
|
|
132
|
+
throw new Error(`Unknown benchmark suite "${value}". Supported: all, brain, reducers, filesystem, release.`);
|
|
133
|
+
}
|
|
134
|
+
async function runCase(category, root) {
|
|
135
|
+
const start = performance.now();
|
|
136
|
+
try {
|
|
137
|
+
const details = await caseOperation(category, root);
|
|
138
|
+
return {
|
|
139
|
+
id: `${category}-small`,
|
|
140
|
+
category,
|
|
141
|
+
operation: operationName(category),
|
|
142
|
+
durationMs: elapsed(start),
|
|
143
|
+
recordsRead: details.recordsRead,
|
|
144
|
+
recordsWritten: details.recordsWritten,
|
|
145
|
+
filesScanned: details.filesScanned,
|
|
146
|
+
engine: "typescript",
|
|
147
|
+
status: details.warnings.length > 0 ? "warning" : "passed",
|
|
148
|
+
warnings: details.warnings
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
return {
|
|
153
|
+
id: `${category}-small`,
|
|
154
|
+
category,
|
|
155
|
+
operation: operationName(category),
|
|
156
|
+
durationMs: elapsed(start),
|
|
157
|
+
recordsRead: 0,
|
|
158
|
+
recordsWritten: 0,
|
|
159
|
+
filesScanned: 0,
|
|
160
|
+
engine: "typescript",
|
|
161
|
+
status: "failed",
|
|
162
|
+
warnings: [error instanceof Error ? error.message : String(error)]
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
async function caseOperation(category, root) {
|
|
167
|
+
switch (category) {
|
|
168
|
+
case "brain-scan": {
|
|
169
|
+
const result = await scanBrain(root);
|
|
170
|
+
return { recordsRead: 0, recordsWritten: result.claimsAdded, filesScanned: 8, warnings: [] };
|
|
171
|
+
}
|
|
172
|
+
case "brain-stale": {
|
|
173
|
+
await scanBrain(root);
|
|
174
|
+
await fs.appendFile(path.join(root, "src", "core", "release-preflight.ts"), "\n// benchmark drift\n", "utf8");
|
|
175
|
+
const result = await staleBrain(root);
|
|
176
|
+
return { recordsRead: result.freshness.checkedRecords, recordsWritten: result.freshness.events.length, filesScanned: 1, warnings: result.freshness.warnings };
|
|
177
|
+
}
|
|
178
|
+
case "brain-consolidate": {
|
|
179
|
+
await scanBrain(root);
|
|
180
|
+
const paths = getWorkspacePaths(root);
|
|
181
|
+
const claims = await readJsonl(paths.brainClaimsFile);
|
|
182
|
+
const first = claims[0];
|
|
183
|
+
if (first)
|
|
184
|
+
await appendJsonl(paths.brainClaimsFile, { ...first, id: "claim_bench_duplicate" });
|
|
185
|
+
const result = await consolidateBrain(root, { dryRun: true });
|
|
186
|
+
return { recordsRead: result.report.claimsRead, recordsWritten: result.report.canonicalClaims, filesScanned: 0, warnings: [] };
|
|
187
|
+
}
|
|
188
|
+
case "reverse-claims": {
|
|
189
|
+
const result = await reverseClaims("./src", root);
|
|
190
|
+
return { recordsRead: result.claims.length, recordsWritten: result.claims.length, filesScanned: 6, warnings: [] };
|
|
191
|
+
}
|
|
192
|
+
case "reducer-large-log": {
|
|
193
|
+
const raw = Array.from({ length: 220 }, (_, index) => `FAIL tests/unit-${index}.test.ts AssertionError at src/app.ts:${index + 1}`).join("\n");
|
|
194
|
+
const reduced = await compressOutputWithEngine("vitest run", raw, "bench-v090-reducer", "ts", root);
|
|
195
|
+
return { recordsRead: 1, recordsWritten: 1, filesScanned: 0, warnings: reduced.summary.includes("AssertionError") ? [] : ["reducer summary did not preserve AssertionError"] };
|
|
196
|
+
}
|
|
197
|
+
case "jsonl-read-write": {
|
|
198
|
+
const file = path.join(getWorkspacePaths(root).workspace, "bench", "jsonl-fixture.jsonl");
|
|
199
|
+
for (let index = 0; index < 12; index += 1)
|
|
200
|
+
await appendJsonl(file, { index, value: `record-${index}` });
|
|
201
|
+
const records = await readJsonl(file);
|
|
202
|
+
return { recordsRead: records.length, recordsWritten: 12, filesScanned: 1, warnings: records.length === 12 ? [] : ["jsonl record count mismatch"] };
|
|
203
|
+
}
|
|
204
|
+
case "range-hash": {
|
|
205
|
+
const text = await fs.readFile(path.join(root, "src", "core", "release-preflight.ts"), "utf8");
|
|
206
|
+
const hashes = Array.from({ length: 20 }, (_, index) => sha256Text(text.slice(index, index + 120)));
|
|
207
|
+
return { recordsRead: hashes.length, recordsWritten: hashes.length, filesScanned: 1, warnings: hashes.every(Boolean) ? [] : ["empty hash"] };
|
|
208
|
+
}
|
|
209
|
+
case "file-scan": {
|
|
210
|
+
const repo = await scanRepository(root, defaultConfig);
|
|
211
|
+
return { recordsRead: repo.files.length, recordsWritten: repo.files.length, filesScanned: repo.total_files, warnings: [] };
|
|
212
|
+
}
|
|
213
|
+
case "workflow-evidence": {
|
|
214
|
+
const plan = await createWorkflowPlan("Benchmark workflow evidence", root);
|
|
215
|
+
const evidence = await buildWorkflowEvidence(plan.record.id, root);
|
|
216
|
+
return { recordsRead: 1, recordsWritten: 1, filesScanned: evidence.content.includes("Workflow") ? 1 : 0, warnings: [] };
|
|
217
|
+
}
|
|
218
|
+
case "format-compare": {
|
|
219
|
+
const file = path.join(root, "payload.json");
|
|
220
|
+
await fs.writeFile(file, JSON.stringify({ evidence: "ERR_BENCH", command: "npm test", items: [1, 2, 3] }, null, 2), "utf8");
|
|
221
|
+
const report = await compareFormats("payload.json", root);
|
|
222
|
+
return { recordsRead: 1, recordsWritten: 1, filesScanned: 1, warnings: report.warnings };
|
|
223
|
+
}
|
|
224
|
+
case "json-validate": {
|
|
225
|
+
await fs.writeFile(path.join(root, "payload.json"), "{\"ok\":true,\"items\":[1,2,3]}\n", "utf8");
|
|
226
|
+
const result = await validateJsonFile("payload.json", { strict: true }, root);
|
|
227
|
+
return { recordsRead: 1, recordsWritten: 1, filesScanned: 1, warnings: result.result === "pass" ? [] : result.probableSecrets };
|
|
228
|
+
}
|
|
229
|
+
case "release-preflight": {
|
|
230
|
+
const result = await runReleasePreflight(root, { runAudit: false, runPack: false, cliCommand: [process.execPath, path.join(root, "dist", "cli.js"), "--version"] });
|
|
231
|
+
return { recordsRead: result.gates.length, recordsWritten: result.gates.length, filesScanned: 6, warnings: result.ok ? [] : result.gates.filter((gate) => !gate.ok).map((gate) => gate.id) };
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
async function writeBenchmarkRailReports(root, report) {
|
|
236
|
+
const paths = benchmarkReportPaths(root);
|
|
237
|
+
await fs.mkdir(path.dirname(paths.latestJson), { recursive: true });
|
|
238
|
+
await fs.mkdir(path.dirname(paths.versionedJson), { recursive: true });
|
|
239
|
+
await writeJson(paths.latestJson, report);
|
|
240
|
+
await fs.writeFile(paths.latestMd, renderBenchmarkMarkdown(report, root), "utf8");
|
|
241
|
+
await writeJson(paths.versionedJson, report);
|
|
242
|
+
await fs.writeFile(paths.versionedMd, renderBenchmarkMarkdown(report, root), "utf8");
|
|
243
|
+
}
|
|
244
|
+
function benchmarkReportPaths(root) {
|
|
245
|
+
return {
|
|
246
|
+
latestJson: path.join(getWorkspacePaths(root).workspace, "bench", "latest.json"),
|
|
247
|
+
latestMd: path.join(getWorkspacePaths(root).workspace, "bench", "latest.md"),
|
|
248
|
+
versionedJson: path.join(root, "benchmarks", "reports", "bench-v0.9.0.json"),
|
|
249
|
+
versionedMd: path.join(root, "benchmarks", "reports", "bench-v0.9.0.md")
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
function renderBenchmarkMarkdown(report, root) {
|
|
253
|
+
return [
|
|
254
|
+
"# SotuRail Benchmark Rail 2 Report",
|
|
255
|
+
"",
|
|
256
|
+
"SotuRail does not claim native speedups unless a local benchmark report proves them.",
|
|
257
|
+
"Native acceleration is optional. TypeScript remains the portable baseline.",
|
|
258
|
+
"",
|
|
259
|
+
`schemaVersion: ${report.schemaVersion}`,
|
|
260
|
+
`id: ${report.id}`,
|
|
261
|
+
`suite: ${report.suite}`,
|
|
262
|
+
`version: ${report.version}`,
|
|
263
|
+
`createdAt: ${report.createdAt}`,
|
|
264
|
+
`platform: ${report.environment.platform}`,
|
|
265
|
+
`node: ${report.environment.node}`,
|
|
266
|
+
`nativeAvailable: ${report.environment.nativeAvailable}`,
|
|
267
|
+
`engine: ${report.environment.engine}`,
|
|
268
|
+
`passed: ${report.summary.passed}`,
|
|
269
|
+
`failed: ${report.summary.failed}`,
|
|
270
|
+
`warnings: ${report.summary.warnings}`,
|
|
271
|
+
`totalDurationMs: ${report.summary.totalDurationMs}`,
|
|
272
|
+
"",
|
|
273
|
+
"| case | category | operation | durationMs | recordsRead | recordsWritten | filesScanned | engine | status | warnings |",
|
|
274
|
+
"|---|---|---|---:|---:|---:|---:|---|---|---|",
|
|
275
|
+
...report.cases.map((item) => `| ${item.id} | ${item.category} | ${item.operation} | ${item.durationMs} | ${item.recordsRead} | ${item.recordsWritten} | ${item.filesScanned} | ${item.engine} | ${item.status} | ${item.warnings.join("; ") || "none"} |`),
|
|
276
|
+
"",
|
|
277
|
+
"## Report Paths",
|
|
278
|
+
"",
|
|
279
|
+
`- latest_json: ${relativeToRoot(root, benchmarkReportPaths(root).latestJson)}`,
|
|
280
|
+
`- latest_markdown: ${relativeToRoot(root, benchmarkReportPaths(root).latestMd)}`,
|
|
281
|
+
`- versioned_json: ${relativeToRoot(root, benchmarkReportPaths(root).versionedJson)}`,
|
|
282
|
+
`- versioned_markdown: ${relativeToRoot(root, benchmarkReportPaths(root).versionedMd)}`,
|
|
283
|
+
""
|
|
284
|
+
].join("\n");
|
|
285
|
+
}
|
|
286
|
+
async function createBenchmarkProject() {
|
|
287
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "soturail-bench-v090-"));
|
|
288
|
+
await fs.mkdir(path.join(root, "src", "core"), { recursive: true });
|
|
289
|
+
await fs.mkdir(path.join(root, "src", "commands"), { recursive: true });
|
|
290
|
+
await fs.mkdir(path.join(root, "docs", "releases"), { recursive: true });
|
|
291
|
+
await fs.mkdir(path.join(root, ".github", "workflows"), { recursive: true });
|
|
292
|
+
await fs.mkdir(path.join(root, "dist"), { recursive: true });
|
|
293
|
+
await fs.writeFile(path.join(root, "README.md"), "npx soturail --help\nnpm install -g soturail\nsoturail --version\n", "utf8");
|
|
294
|
+
await fs.writeFile(path.join(root, "CHANGELOG.md"), "## [0.9.0]\n\n- Benchmark fixture.\n", "utf8");
|
|
295
|
+
await fs.writeFile(path.join(root, "LICENSE"), "MIT\n", "utf8");
|
|
296
|
+
await fs.writeFile(path.join(root, "package.json"), JSON.stringify({ name: "soturail", version: "0.9.0", engines: { node: ">=20" } }, null, 2), "utf8");
|
|
297
|
+
await fs.writeFile(path.join(root, "package-lock.json"), JSON.stringify({ name: "soturail", version: "0.9.0", lockfileVersion: 3, packages: { "": { name: "soturail", version: "0.9.0" } } }, null, 2), "utf8");
|
|
298
|
+
await fs.writeFile(path.join(root, "dist", "cli.js"), "if (process.argv.includes('--version')) console.log('0.9.0');\n", "utf8");
|
|
299
|
+
await fs.writeFile(path.join(root, ".github", "workflows", "ci.yml"), "name: ci\n", "utf8");
|
|
300
|
+
await fs.writeFile(path.join(root, "src", "cli.ts"), "registerBrainCommand(program);\nregisterReverseCommand(program);\nregisterWorkflowCommand(program);\n", "utf8");
|
|
301
|
+
await fs.writeFile(path.join(root, "src", "core", "version.ts"), "export const SOTURAIL_VERSION = \"0.9.0\";\n", "utf8");
|
|
302
|
+
await fs.writeFile(path.join(root, "src", "core", "release-preflight.ts"), "export const releaseNotesPath = \"docs/releases/RELEASE_NOTES_v0.9.0.md\";\n", "utf8");
|
|
303
|
+
await fs.writeFile(path.join(root, "src", "core", "agent-runtime.ts"), "export const policy = \"No arbitrary shell execution through MCP by default.\";\n", "utf8");
|
|
304
|
+
await fs.writeFile(path.join(root, "src", "core", "workflow-store.ts"), "export const workflowStorage = \".soturail/workflows/\";\n", "utf8");
|
|
305
|
+
await fs.writeFile(path.join(root, "src", "core", "diagram-validator.ts"), "export const diagramNote = \"lightweight Mermaid validation, not a full parser\";\n", "utf8");
|
|
306
|
+
await fs.writeFile(path.join(root, "src", "commands", "brain.ts"), "program.command(\"brain\");\n", "utf8");
|
|
307
|
+
await fs.writeFile(path.join(root, "src", "commands", "reverse.ts"), "program.command(\"reverse\");\n", "utf8");
|
|
308
|
+
await fs.writeFile(path.join(root, "docs", "releases", "README.md"), "# Release Notes\n\nRelease notes live under docs/releases/.\n", "utf8");
|
|
309
|
+
await fs.writeFile(path.join(root, "docs", "releases", "RELEASE_NOTES_v0.9.0.md"), "# Notes\n", "utf8");
|
|
310
|
+
await ensureWorkspace(root);
|
|
311
|
+
return root;
|
|
312
|
+
}
|
|
313
|
+
function allCategories() {
|
|
314
|
+
return [...new Set(Object.values(categorySuites).flat())];
|
|
315
|
+
}
|
|
316
|
+
function operationName(category) {
|
|
317
|
+
return category.replace(/-/g, " ");
|
|
318
|
+
}
|
|
319
|
+
function elapsed(start) {
|
|
320
|
+
return Number(Math.max(0, performance.now() - start).toFixed(3));
|
|
321
|
+
}
|
|
322
|
+
//# sourceMappingURL=benchmark-rail.js.map
|