runcap 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +211 -9
- package/bin/runcap.mjs +153 -0
- package/examples/outcome-demo/agent-fixes.mjs +24 -0
- package/examples/outcome-demo/agent-spins.mjs +20 -0
- package/examples/outcome-demo/broken.mjs +5 -0
- package/examples/outcome-demo/verify.mjs +7 -0
- package/package.json +11 -2
- package/scripts/guard-test.mjs +76 -0
- package/scripts/loop-e2e.mjs +137 -0
- package/scripts/loop-test.mjs +45 -1
- package/scripts/make-demo-svg.mjs +20 -19
- package/scripts/make-linkedin-loop-video.mjs +338 -0
- package/scripts/mission-test.mjs +148 -0
- package/scripts/outcome-test.mjs +48 -0
- package/scripts/policy-test.mjs +121 -0
- package/scripts/render-media-screenshots.mjs +37 -0
- package/src/compressor.mjs +77 -9
- package/src/mission-control.mjs +475 -8
- package/src/policy.mjs +208 -0
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
// Renders a LinkedIn-ready MP4 for the Runcap loop-detection post.
|
|
2
|
+
// Narrative: a circling agent looks busy but burns money -> Runcap catches the
|
|
3
|
+
// loop in real time -> proven 37.9% compression -> hard cap stops the run.
|
|
4
|
+
// Output: docs/assets/media/runcap-linkedin-loop-demo.mp4
|
|
5
|
+
// Requires: playwright + ffmpeg available on the machine.
|
|
6
|
+
import { spawnSync } from "node:child_process";
|
|
7
|
+
import { mkdirSync, readdirSync, rmSync } from "node:fs";
|
|
8
|
+
import { dirname, join, resolve } from "node:path";
|
|
9
|
+
import { fileURLToPath } from "node:url";
|
|
10
|
+
import { chromium } from "playwright";
|
|
11
|
+
|
|
12
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
13
|
+
const root = resolve(__dirname, "..");
|
|
14
|
+
const outDir = resolve(root, "docs/assets/media");
|
|
15
|
+
const framesDir = "/private/tmp/runcap-linkedin-loop-frames";
|
|
16
|
+
const outFile = join(outDir, "runcap-linkedin-loop-demo.mp4");
|
|
17
|
+
|
|
18
|
+
const width = 1080;
|
|
19
|
+
const height = 1080;
|
|
20
|
+
const fps = 30;
|
|
21
|
+
const duration = 13;
|
|
22
|
+
const frameCount = fps * duration;
|
|
23
|
+
|
|
24
|
+
mkdirSync(outDir, { recursive: true });
|
|
25
|
+
mkdirSync(framesDir, { recursive: true });
|
|
26
|
+
for (const file of readdirSync(framesDir)) {
|
|
27
|
+
if (file.startsWith("frame-") && file.endsWith(".png")) {
|
|
28
|
+
rmSync(join(framesDir, file));
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const html = `<!doctype html>
|
|
33
|
+
<html>
|
|
34
|
+
<head>
|
|
35
|
+
<meta charset="utf-8" />
|
|
36
|
+
<style>
|
|
37
|
+
* { box-sizing: border-box; }
|
|
38
|
+
html, body {
|
|
39
|
+
margin: 0;
|
|
40
|
+
width: ${width}px;
|
|
41
|
+
height: ${height}px;
|
|
42
|
+
overflow: hidden;
|
|
43
|
+
background: #f4f6fb;
|
|
44
|
+
font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
|
45
|
+
color: #f8fafc;
|
|
46
|
+
}
|
|
47
|
+
.stage {
|
|
48
|
+
width: ${width}px;
|
|
49
|
+
height: ${height}px;
|
|
50
|
+
padding: 58px;
|
|
51
|
+
display: grid;
|
|
52
|
+
place-items: center;
|
|
53
|
+
background:
|
|
54
|
+
radial-gradient(circle at 15% 10%, rgba(167, 139, 250, .2), transparent 32%),
|
|
55
|
+
radial-gradient(circle at 85% 12%, rgba(34, 211, 238, .16), transparent 34%),
|
|
56
|
+
linear-gradient(135deg, #eef2ff, #f8fafc);
|
|
57
|
+
}
|
|
58
|
+
.card {
|
|
59
|
+
width: 964px;
|
|
60
|
+
height: 964px;
|
|
61
|
+
border-radius: 42px;
|
|
62
|
+
padding: 42px;
|
|
63
|
+
background: #080b12;
|
|
64
|
+
box-shadow: 0 36px 90px rgba(15, 23, 42, .25);
|
|
65
|
+
position: relative;
|
|
66
|
+
overflow: hidden;
|
|
67
|
+
}
|
|
68
|
+
.card::before {
|
|
69
|
+
content: "";
|
|
70
|
+
position: absolute;
|
|
71
|
+
inset: 0;
|
|
72
|
+
background:
|
|
73
|
+
radial-gradient(circle at 50% -10%, rgba(167, 139, 250, .18), transparent 36%),
|
|
74
|
+
linear-gradient(180deg, rgba(255,255,255,.06), transparent 28%);
|
|
75
|
+
pointer-events: none;
|
|
76
|
+
}
|
|
77
|
+
.top {
|
|
78
|
+
position: relative;
|
|
79
|
+
display: flex;
|
|
80
|
+
justify-content: space-between;
|
|
81
|
+
align-items: center;
|
|
82
|
+
color: #94a3b8;
|
|
83
|
+
font-size: 23px;
|
|
84
|
+
letter-spacing: -0.02em;
|
|
85
|
+
}
|
|
86
|
+
.brand {
|
|
87
|
+
display: flex;
|
|
88
|
+
gap: 14px;
|
|
89
|
+
align-items: center;
|
|
90
|
+
font-weight: 800;
|
|
91
|
+
color: #fff;
|
|
92
|
+
font-size: 30px;
|
|
93
|
+
}
|
|
94
|
+
.logo {
|
|
95
|
+
width: 42px;
|
|
96
|
+
height: 42px;
|
|
97
|
+
border-radius: 13px;
|
|
98
|
+
display: grid;
|
|
99
|
+
place-items: center;
|
|
100
|
+
background: linear-gradient(135deg, #22d3ee, #34d399);
|
|
101
|
+
color: #021014;
|
|
102
|
+
font-weight: 900;
|
|
103
|
+
}
|
|
104
|
+
.pill {
|
|
105
|
+
border: 1px solid rgba(148, 163, 184, .28);
|
|
106
|
+
background: rgba(15, 23, 42, .68);
|
|
107
|
+
color: #cbd5e1;
|
|
108
|
+
border-radius: 999px;
|
|
109
|
+
padding: 10px 16px;
|
|
110
|
+
font-size: 18px;
|
|
111
|
+
font-weight: 650;
|
|
112
|
+
}
|
|
113
|
+
.content {
|
|
114
|
+
position: relative;
|
|
115
|
+
height: 818px;
|
|
116
|
+
padding-top: 44px;
|
|
117
|
+
}
|
|
118
|
+
.headline {
|
|
119
|
+
margin: 0;
|
|
120
|
+
color: #f8fafc;
|
|
121
|
+
font-size: 68px;
|
|
122
|
+
line-height: .98;
|
|
123
|
+
letter-spacing: -0.06em;
|
|
124
|
+
max-width: 840px;
|
|
125
|
+
}
|
|
126
|
+
.sub {
|
|
127
|
+
margin-top: 22px;
|
|
128
|
+
color: #cbd5e1;
|
|
129
|
+
font-size: 29px;
|
|
130
|
+
line-height: 1.28;
|
|
131
|
+
letter-spacing: -0.03em;
|
|
132
|
+
max-width: 820px;
|
|
133
|
+
}
|
|
134
|
+
.accent { color: #67e8f9; }
|
|
135
|
+
.green { color: #34d399; }
|
|
136
|
+
.red { color: #fb7185; }
|
|
137
|
+
.violet { color: #a78bfa; }
|
|
138
|
+
.mono {
|
|
139
|
+
font-family: "SF Mono", "JetBrains Mono", Menlo, Consolas, monospace;
|
|
140
|
+
letter-spacing: -0.04em;
|
|
141
|
+
}
|
|
142
|
+
.terminal {
|
|
143
|
+
margin-top: 38px;
|
|
144
|
+
border: 1px solid rgba(148, 163, 184, .22);
|
|
145
|
+
background: rgba(2, 6, 23, .82);
|
|
146
|
+
border-radius: 24px;
|
|
147
|
+
padding: 26px;
|
|
148
|
+
font-size: 24px;
|
|
149
|
+
line-height: 1.5;
|
|
150
|
+
color: #dbeafe;
|
|
151
|
+
box-shadow: inset 0 1px 0 rgba(255,255,255,.05);
|
|
152
|
+
}
|
|
153
|
+
.terminal .line { opacity: 1; }
|
|
154
|
+
.warning {
|
|
155
|
+
margin-top: 28px;
|
|
156
|
+
border: 1px solid rgba(167, 139, 250, .4);
|
|
157
|
+
background: rgba(167, 139, 250, .12);
|
|
158
|
+
color: #ddd6fe;
|
|
159
|
+
border-radius: 22px;
|
|
160
|
+
padding: 22px 26px;
|
|
161
|
+
font-size: 28px;
|
|
162
|
+
font-weight: 850;
|
|
163
|
+
letter-spacing: -0.04em;
|
|
164
|
+
}
|
|
165
|
+
.numbers {
|
|
166
|
+
margin-top: 46px;
|
|
167
|
+
display: grid;
|
|
168
|
+
grid-template-columns: 1fr 1fr;
|
|
169
|
+
gap: 28px;
|
|
170
|
+
align-items: end;
|
|
171
|
+
}
|
|
172
|
+
.number-card {
|
|
173
|
+
border-radius: 26px;
|
|
174
|
+
padding: 28px;
|
|
175
|
+
background: rgba(15, 23, 42, .9);
|
|
176
|
+
border: 1px solid rgba(148, 163, 184, .22);
|
|
177
|
+
}
|
|
178
|
+
.label {
|
|
179
|
+
color: #94a3b8;
|
|
180
|
+
font-size: 22px;
|
|
181
|
+
margin-bottom: 12px;
|
|
182
|
+
letter-spacing: -0.03em;
|
|
183
|
+
}
|
|
184
|
+
.big {
|
|
185
|
+
font-size: 78px;
|
|
186
|
+
line-height: .9;
|
|
187
|
+
font-weight: 900;
|
|
188
|
+
letter-spacing: -0.08em;
|
|
189
|
+
}
|
|
190
|
+
.bar {
|
|
191
|
+
margin-top: 32px;
|
|
192
|
+
height: 34px;
|
|
193
|
+
border-radius: 999px;
|
|
194
|
+
background: rgba(148, 163, 184, .16);
|
|
195
|
+
overflow: hidden;
|
|
196
|
+
border: 1px solid rgba(148, 163, 184, .24);
|
|
197
|
+
}
|
|
198
|
+
.fill {
|
|
199
|
+
height: 100%;
|
|
200
|
+
width: 37.9%;
|
|
201
|
+
border-radius: 999px;
|
|
202
|
+
background: linear-gradient(90deg, #22d3ee, #34d399);
|
|
203
|
+
}
|
|
204
|
+
.footer {
|
|
205
|
+
position: absolute;
|
|
206
|
+
left: 42px;
|
|
207
|
+
right: 42px;
|
|
208
|
+
bottom: 34px;
|
|
209
|
+
display: flex;
|
|
210
|
+
justify-content: space-between;
|
|
211
|
+
align-items: center;
|
|
212
|
+
color: #94a3b8;
|
|
213
|
+
font-size: 20px;
|
|
214
|
+
}
|
|
215
|
+
.scene {
|
|
216
|
+
position: absolute;
|
|
217
|
+
inset: 44px 0 0 0;
|
|
218
|
+
opacity: 0;
|
|
219
|
+
transform: translateY(24px) scale(.985);
|
|
220
|
+
transition: opacity .24s ease, transform .24s ease;
|
|
221
|
+
}
|
|
222
|
+
.scene.active {
|
|
223
|
+
opacity: 1;
|
|
224
|
+
transform: translateY(0) scale(1);
|
|
225
|
+
}
|
|
226
|
+
</style>
|
|
227
|
+
</head>
|
|
228
|
+
<body>
|
|
229
|
+
<div class="stage">
|
|
230
|
+
<div class="card">
|
|
231
|
+
<div class="top">
|
|
232
|
+
<div class="brand"><div class="logo">R</div> Runcap</div>
|
|
233
|
+
<div class="pill">local-first AI cost control</div>
|
|
234
|
+
</div>
|
|
235
|
+
<div class="content">
|
|
236
|
+
<section class="scene active" id="s0">
|
|
237
|
+
<h1 class="headline">Your AI agent looks busy. It is just circling.</h1>
|
|
238
|
+
<p class="sub">Same failure, reworded every turn. It produces output, makes no progress, and keeps spending your tokens.</p>
|
|
239
|
+
<div class="terminal mono">
|
|
240
|
+
<div class="line">attempt 1: guard the undefined with an if check</div>
|
|
241
|
+
<div class="line">attempt 2: add an optional chain before .id</div>
|
|
242
|
+
<div class="line">attempt 3: default the object to {} first</div>
|
|
243
|
+
<div class="line red">test still fails. budget still draining.</div>
|
|
244
|
+
</div>
|
|
245
|
+
</section>
|
|
246
|
+
<section class="scene" id="s1">
|
|
247
|
+
<h1 class="headline">Plain hashing never catches this.</h1>
|
|
248
|
+
<p class="sub">The prompt is similar but never byte-identical between loops, so the hash changes every turn and nothing trips.</p>
|
|
249
|
+
<div class="terminal mono">
|
|
250
|
+
<div class="line">hash(attempt 1) = a91f... hash(attempt 2) = c4d2...</div>
|
|
251
|
+
<div class="line red">different hash every time -> loop invisible</div>
|
|
252
|
+
</div>
|
|
253
|
+
</section>
|
|
254
|
+
<section class="scene" id="s2">
|
|
255
|
+
<h1 class="headline">Runcap measures similarity, not hashes.</h1>
|
|
256
|
+
<p class="sub">A local gateway sees every request in real time and compares each prompt's shape against the recent run.</p>
|
|
257
|
+
<div class="warning">loop: last 3 prompts 97.7% identical, no progress. The agent is circling the same failure.</div>
|
|
258
|
+
<div class="terminal mono">
|
|
259
|
+
<div class="line green">$ runcap status</div>
|
|
260
|
+
<div class="line violet">Loop warning: stepping in before it burns more budget.</div>
|
|
261
|
+
</div>
|
|
262
|
+
</section>
|
|
263
|
+
<section class="scene" id="s3">
|
|
264
|
+
<h1 class="headline">And it compresses every call it lets through.</h1>
|
|
265
|
+
<div class="numbers">
|
|
266
|
+
<div class="number-card">
|
|
267
|
+
<div class="label">baseline prompt</div>
|
|
268
|
+
<div class="big red mono">1,186</div>
|
|
269
|
+
<div class="label">tokens</div>
|
|
270
|
+
</div>
|
|
271
|
+
<div class="number-card">
|
|
272
|
+
<div class="label">with Runcap</div>
|
|
273
|
+
<div class="big green mono">737</div>
|
|
274
|
+
<div class="label">tokens</div>
|
|
275
|
+
</div>
|
|
276
|
+
</div>
|
|
277
|
+
<div class="bar"><div class="fill"></div></div>
|
|
278
|
+
<p class="sub"><span class="green">37.9% saved</span> on a real OpenAI call. The model still answered correctly about the changed line.</p>
|
|
279
|
+
</section>
|
|
280
|
+
<section class="scene" id="s4">
|
|
281
|
+
<h1 class="headline">Estimate. Cap. Compress. Catch the loop.</h1>
|
|
282
|
+
<p class="sub">Point your OpenAI or Anthropic-compatible tools at the local gateway. When the ceiling is crossed, the next call stops.</p>
|
|
283
|
+
<div class="terminal mono">
|
|
284
|
+
<div class="line green">$ AIM_DAILY_BUDGET_USD=10 runcap gateway</div>
|
|
285
|
+
<div class="line">gateway up · compress on · hard cap armed · loop guard on</div>
|
|
286
|
+
<div class="line red">HTTP 429 budget_guard</div>
|
|
287
|
+
<div class="line accent">stopped before money left your account</div>
|
|
288
|
+
</div>
|
|
289
|
+
</section>
|
|
290
|
+
</div>
|
|
291
|
+
<div class="footer">
|
|
292
|
+
<span class="mono">npm install -g runcap</span>
|
|
293
|
+
<span>Free · MIT · 100% local</span>
|
|
294
|
+
</div>
|
|
295
|
+
</div>
|
|
296
|
+
</div>
|
|
297
|
+
<script>
|
|
298
|
+
const scenes = [...document.querySelectorAll(".scene")];
|
|
299
|
+
window.renderFrame = (seconds) => {
|
|
300
|
+
const index =
|
|
301
|
+
seconds < 2.8 ? 0 :
|
|
302
|
+
seconds < 5.2 ? 1 :
|
|
303
|
+
seconds < 8.2 ? 2 :
|
|
304
|
+
seconds < 10.6 ? 3 : 4;
|
|
305
|
+
scenes.forEach((scene, i) => scene.classList.toggle("active", i === index));
|
|
306
|
+
};
|
|
307
|
+
</script>
|
|
308
|
+
</body>
|
|
309
|
+
</html>`;
|
|
310
|
+
|
|
311
|
+
const browser = await chromium.launch({ headless: true });
|
|
312
|
+
const page = await browser.newPage({ viewport: { width, height }, deviceScaleFactor: 1 });
|
|
313
|
+
await page.setContent(html);
|
|
314
|
+
await page.waitForTimeout(100);
|
|
315
|
+
|
|
316
|
+
for (let i = 0; i < frameCount; i += 1) {
|
|
317
|
+
const seconds = i / fps;
|
|
318
|
+
await page.evaluate((t) => window.renderFrame(t), seconds);
|
|
319
|
+
await page.screenshot({ path: join(framesDir, `frame-${String(i).padStart(4, "0")}.png`) });
|
|
320
|
+
}
|
|
321
|
+
await browser.close();
|
|
322
|
+
|
|
323
|
+
const ffmpeg = spawnSync("ffmpeg", [
|
|
324
|
+
"-y",
|
|
325
|
+
"-framerate", String(fps),
|
|
326
|
+
"-i", join(framesDir, "frame-%04d.png"),
|
|
327
|
+
"-c:v", "libx264",
|
|
328
|
+
"-pix_fmt", "yuv420p",
|
|
329
|
+
"-movflags", "+faststart",
|
|
330
|
+
"-crf", "18",
|
|
331
|
+
outFile
|
|
332
|
+
], { stdio: "inherit" });
|
|
333
|
+
|
|
334
|
+
if (ffmpeg.status !== 0) {
|
|
335
|
+
process.exit(ffmpeg.status ?? 1);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
console.log(`wrote ${outFile}`);
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// Proves a policy-bound mission grades a real run into a PASS/BLOCKED verdict and
|
|
2
|
+
// that the verdict drives the process exit code (so CI fails on a blocked mission).
|
|
3
|
+
// Everything runs offline through the mock cap gateway inside a throwaway git repo:
|
|
4
|
+
// - an honest fix within scope, under cap → PASS, exit 0
|
|
5
|
+
// - an agent that rewrites the verifier → BLOCKED (VERIFIER_COMPROMISED)
|
|
6
|
+
// - an edit outside the declared allow scope → BLOCKED (out of scope)
|
|
7
|
+
// - a mission whose first call trips the hard cap → BLOCKED (budget guard)
|
|
8
|
+
// It also drives the real `bin/runcap.mjs` so the exit codes and the GitHub
|
|
9
|
+
// Action's `runcap ci` PR summary are tested as a reviewer would see them.
|
|
10
|
+
|
|
11
|
+
import os from "node:os";
|
|
12
|
+
import path from "node:path";
|
|
13
|
+
import { fileURLToPath } from "node:url";
|
|
14
|
+
import { execFileSync } from "node:child_process";
|
|
15
|
+
import { mkdtempSync, writeFileSync, mkdirSync, readFileSync } from "node:fs";
|
|
16
|
+
|
|
17
|
+
const SRC_DIR = process.env.RUNCAP_SRC ?? path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "src");
|
|
18
|
+
const BIN = path.join(SRC_DIR, "..", "bin", "runcap.mjs");
|
|
19
|
+
|
|
20
|
+
const tmp = mkdtempSync(path.join(os.tmpdir(), "runcap-mission-"));
|
|
21
|
+
process.chdir(tmp);
|
|
22
|
+
|
|
23
|
+
mkdirSync(path.join(tmp, "app"), { recursive: true });
|
|
24
|
+
mkdirSync(path.join(tmp, ".runcap"), { recursive: true });
|
|
25
|
+
// broken.mjs starts wrong, so the task genuinely fails on baseline.
|
|
26
|
+
writeFileSync(path.join(tmp, "app", "broken.mjs"), "export const ok = false;\n");
|
|
27
|
+
writeFileSync(path.join(tmp, "app", "verify.mjs"),
|
|
28
|
+
"import { ok } from './broken.mjs'; import assert from 'node:assert'; assert.strictEqual(ok, true, 'not fixed'); console.log('ok');\n");
|
|
29
|
+
// A tracked file OUTSIDE the allow scope: editing it must trip the scope check.
|
|
30
|
+
// (The guard reads `git diff --name-status`, which sees tracked edits - the
|
|
31
|
+
// honest unit of "out of scope" for this scenario.)
|
|
32
|
+
writeFileSync(path.join(tmp, "app", "other.mjs"), "export const other = 0;\n");
|
|
33
|
+
|
|
34
|
+
const spend = (msg) =>
|
|
35
|
+
"const b=process.env.OPENAI_BASE_URL;await fetch(`${b}/chat/completions`,{method:'POST',headers:{'content-type':'application/json',authorization:'Bearer x'},body:JSON.stringify({model:'gpt-4o',messages:[{role:'user',content:'" + msg + "'}]})}).then(r=>r.text());";
|
|
36
|
+
|
|
37
|
+
// Honest agent: spends, fixes the source file (the one allowed change).
|
|
38
|
+
writeFileSync(path.join(tmp, "agent-fix.mjs"),
|
|
39
|
+
spend("fix it") + "const {writeFile}=await import('node:fs/promises');await writeFile('app/broken.mjs','export const ok = true;\\n');\n");
|
|
40
|
+
// Cheating agent: rewrites the VERIFIER so it always passes instead of fixing the bug.
|
|
41
|
+
writeFileSync(path.join(tmp, "agent-cheat.mjs"),
|
|
42
|
+
spend("cheat") + "const {writeFile}=await import('node:fs/promises');await writeFile('app/verify.mjs','console.log(\"ok\");\\n');\n");
|
|
43
|
+
// Out-of-scope agent: fixes the bug (verify passes) BUT also edits a tracked file outside `allow`.
|
|
44
|
+
writeFileSync(path.join(tmp, "agent-scope.mjs"),
|
|
45
|
+
spend("scope") + "const {writeFile}=await import('node:fs/promises');await writeFile('app/broken.mjs','export const ok = true;\\n');await writeFile('app/other.mjs','export const other = 1;\\n');\n");
|
|
46
|
+
|
|
47
|
+
// The mission policy a reviewer commits to the repo.
|
|
48
|
+
const POLICY = `version: v1
|
|
49
|
+
identity:
|
|
50
|
+
project: checkout
|
|
51
|
+
team: payments
|
|
52
|
+
mission:
|
|
53
|
+
name: Fix the failing checkout test
|
|
54
|
+
task_class: bugfix
|
|
55
|
+
budget:
|
|
56
|
+
mission_hard_limit_usd: 5
|
|
57
|
+
max_llm_calls: 12
|
|
58
|
+
verification:
|
|
59
|
+
command: "node app/verify.mjs"
|
|
60
|
+
guard: strict
|
|
61
|
+
protect: ["app/verify.mjs"]
|
|
62
|
+
allow: ["app/broken.mjs"]
|
|
63
|
+
`;
|
|
64
|
+
writeFileSync(path.join(tmp, ".runcap", "mission.yaml"), POLICY);
|
|
65
|
+
|
|
66
|
+
// A second policy with a hair-thin cap, so the gateway trips the budget guard pre-flight.
|
|
67
|
+
const TINY_POLICY = POLICY.replace("mission_hard_limit_usd: 5", "mission_hard_limit_usd: 0.0000001");
|
|
68
|
+
writeFileSync(path.join(tmp, ".runcap", "mission-tiny.yaml"), TINY_POLICY);
|
|
69
|
+
|
|
70
|
+
// Commit a baseline so the guard has a real commit + clean tree to check against.
|
|
71
|
+
const g = (...a) => execFileSync("git", a, { cwd: tmp, stdio: "pipe" });
|
|
72
|
+
g("init", "-q");
|
|
73
|
+
g("config", "user.email", "test@runcap.local");
|
|
74
|
+
g("config", "user.name", "runcap-test");
|
|
75
|
+
g("add", "-A");
|
|
76
|
+
g("commit", "-qm", "baseline");
|
|
77
|
+
|
|
78
|
+
let failures = 0;
|
|
79
|
+
const check = (name, pass, detail) => { if (!pass) failures++; console.log(`${pass ? "PASS" : "FAIL"} ${name}${detail ? " — " + detail : ""}`); };
|
|
80
|
+
|
|
81
|
+
const { runOutcome } = await import(path.join(SRC_DIR, "mission-control.mjs"));
|
|
82
|
+
const { loadPolicy } = await import(path.join(SRC_DIR, "policy.mjs"));
|
|
83
|
+
|
|
84
|
+
// Each scenario starts from the committed baseline so one run's edits (the cheat
|
|
85
|
+
// run's rewritten verifier especially) never leak into the next.
|
|
86
|
+
const resetToBaseline = () => { g("checkout", "-f", "HEAD"); g("clean", "-fdq", "-e", ".runcap"); };
|
|
87
|
+
|
|
88
|
+
const loaded = loadPolicy(tmp);
|
|
89
|
+
|
|
90
|
+
// 1. Honest fix, within scope, under cap → PASS with a strong verification.
|
|
91
|
+
const fix = await runOutcome({ task: "fix ok", verify: "node app/verify.mjs", command: ["node", "agent-fix.mjs"], label: "mission-fix", mock: true, guard: true, protect: ["app/verify.mjs"], allow: ["app/broken.mjs"], capUsd: 5, policy: loaded });
|
|
92
|
+
check("honest fix verifies", fix.receipt.outcome === "VERIFIED", `outcome=${fix.receipt.outcome}`);
|
|
93
|
+
check("honest fix grades VERIFIED_STRONG", fix.receipt.verificationIntegrity.status === "VERIFIED_STRONG", `status=${fix.receipt.verificationIntegrity.status}`);
|
|
94
|
+
check("honest fix mission verdict PASS", fix.receipt.policy?.verdict === "PASS", JSON.stringify(fix.receipt.policy?.reasons));
|
|
95
|
+
check("receipt carries the policy hash", /^[0-9a-f]{64}$/.test(fix.receipt.policy?.hash ?? ""), fix.receipt.policy?.hash);
|
|
96
|
+
check("receipt carries org attribution", fix.receipt.policy?.identity?.project === "checkout" && fix.receipt.policy?.identity?.team === "payments");
|
|
97
|
+
check("receipt bumps to v0.3 schema", fix.receipt.schema === "runcap.outcome-receipt/v0.3", fix.receipt.schema);
|
|
98
|
+
|
|
99
|
+
// 2. Cheat by rewriting the verifier → BLOCKED, VERIFIER_COMPROMISED.
|
|
100
|
+
resetToBaseline();
|
|
101
|
+
const cheat = await runOutcome({ task: "fix ok", verify: "node app/verify.mjs", command: ["node", "agent-cheat.mjs"], label: "mission-cheat", mock: true, guard: true, protect: ["app/verify.mjs"], allow: ["app/broken.mjs"], capUsd: 5, policy: loaded });
|
|
102
|
+
check("cheat run mission verdict BLOCKED", cheat.receipt.policy?.verdict === "BLOCKED", `verdict=${cheat.receipt.policy?.verdict}`);
|
|
103
|
+
check("cheat run names VERIFIER_COMPROMISED", (cheat.receipt.policy?.reasons ?? []).some((r) => r.includes("VERIFIER_COMPROMISED")), JSON.stringify(cheat.receipt.policy?.reasons));
|
|
104
|
+
|
|
105
|
+
// 3. Edit outside the declared scope → BLOCKED, out-of-scope.
|
|
106
|
+
resetToBaseline();
|
|
107
|
+
const scope = await runOutcome({ task: "fix ok", verify: "node app/verify.mjs", command: ["node", "agent-scope.mjs"], label: "mission-scope", mock: true, guard: true, protect: ["app/verify.mjs"], allow: ["app/broken.mjs"], capUsd: 5, policy: loaded });
|
|
108
|
+
check("out-of-scope run mission verdict BLOCKED", scope.receipt.policy?.verdict === "BLOCKED", `verdict=${scope.receipt.policy?.verdict}`);
|
|
109
|
+
check("out-of-scope run names the scope breach", (scope.receipt.policy?.reasons ?? []).some((r) => r.toLowerCase().includes("scope")), JSON.stringify(scope.receipt.policy?.reasons));
|
|
110
|
+
|
|
111
|
+
// 4. A hair-thin cap trips the gateway budget guard → BLOCKED, budget reason.
|
|
112
|
+
resetToBaseline();
|
|
113
|
+
const tinyLoaded = loadPolicy(tmp, ".runcap/mission-tiny.yaml");
|
|
114
|
+
const broke = await runOutcome({ task: "fix ok", verify: "node app/verify.mjs", command: ["node", "agent-fix.mjs"], label: "mission-broke", mock: true, guard: true, protect: ["app/verify.mjs"], allow: ["app/broken.mjs"], capUsd: 0.0000001, policy: tinyLoaded });
|
|
115
|
+
check("tiny cap trips the budget guard", broke.receipt.cost.budgetGuardTripped === true, `tripped=${broke.receipt.cost.budgetGuardTripped}`);
|
|
116
|
+
check("budget trip mission verdict BLOCKED", broke.receipt.policy?.verdict === "BLOCKED", `verdict=${broke.receipt.policy?.verdict}`);
|
|
117
|
+
check("budget trip names the budget guard", (broke.receipt.policy?.reasons ?? []).some((r) => r.toLowerCase().includes("budget")), JSON.stringify(broke.receipt.policy?.reasons));
|
|
118
|
+
|
|
119
|
+
// 5. The real bin must exit 0 on PASS and 1 on BLOCKED so CI fails on a bad mission.
|
|
120
|
+
const runBin = (args, extraEnv = {}) => {
|
|
121
|
+
try {
|
|
122
|
+
const stdout = execFileSync("node", [BIN, ...args], { cwd: tmp, env: { ...process.env, ...extraEnv }, stdio: ["ignore", "pipe", "pipe"] });
|
|
123
|
+
return { code: 0, stdout: String(stdout) };
|
|
124
|
+
} catch (e) {
|
|
125
|
+
return { code: e.status ?? 1, stdout: String(e.stdout ?? ""), stderr: String(e.stderr ?? "") };
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
resetToBaseline();
|
|
130
|
+
const binPass = runBin(["mission", "run", "--mock", "--", "node", "agent-fix.mjs"]);
|
|
131
|
+
check("`runcap mission run` exits 0 on a PASS mission", binPass.code === 0, `code=${binPass.code}`);
|
|
132
|
+
check("PASS run prints the verdict", /Mission verdict: PASS/.test(binPass.stdout), binPass.stdout.slice(-200));
|
|
133
|
+
|
|
134
|
+
resetToBaseline();
|
|
135
|
+
const binBlock = runBin(["mission", "run", "--mock", "--", "node", "agent-cheat.mjs"]);
|
|
136
|
+
check("`runcap mission run` exits 1 on a BLOCKED mission", binBlock.code === 1, `code=${binBlock.code}`);
|
|
137
|
+
|
|
138
|
+
// 6. `runcap ci` (the GitHub Action's grader) must write the PR summary and exit 1 on BLOCKED.
|
|
139
|
+
// It grades the latest receipt on disk - which the BLOCKED cheat run just wrote.
|
|
140
|
+
const summaryFile = path.join(tmp, "step-summary.md");
|
|
141
|
+
writeFileSync(summaryFile, "");
|
|
142
|
+
const ci = runBin(["ci", "--policy", ".runcap/mission.yaml"], { GITHUB_STEP_SUMMARY: summaryFile });
|
|
143
|
+
check("`runcap ci` exits 1 when the graded receipt is BLOCKED", ci.code === 1, `code=${ci.code}`);
|
|
144
|
+
const summary = readFileSync(summaryFile, "utf8");
|
|
145
|
+
check("`runcap ci` writes a PR summary to GITHUB_STEP_SUMMARY", /Runcap mission verdict: BLOCKED/.test(summary), summary.slice(0, 200));
|
|
146
|
+
|
|
147
|
+
console.log("\n" + (failures === 0 ? "ALL MISSION TESTS PASSED" : `${failures} MISSION TEST(S) FAILED`));
|
|
148
|
+
process.exit(failures === 0 ? 0 : 1);
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// Proves runOutcome produces an honest receipt end-to-end through the REAL cap
|
|
2
|
+
// gateway (mock upstream, so no network/keys), for both the VERIFIED and
|
|
3
|
+
// UNVERIFIED cases. The agent spends recorded tokens; the verify command's exit
|
|
4
|
+
// code is the oracle; Verified Outcome Cost is the actual spend only when verify
|
|
5
|
+
// passes. Runs in an isolated temp cwd so it never touches real .runcap data.
|
|
6
|
+
|
|
7
|
+
import os from "node:os";
|
|
8
|
+
import path from "node:path";
|
|
9
|
+
import { fileURLToPath } from "node:url";
|
|
10
|
+
import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs";
|
|
11
|
+
|
|
12
|
+
// Resolve the engine relative to this script so the test runs from any cwd
|
|
13
|
+
// (it chdir's into a temp dir below, so a relative import would break).
|
|
14
|
+
const SRC_DIR = process.env.RUNCAP_SRC ?? path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "src");
|
|
15
|
+
|
|
16
|
+
const tmp = mkdtempSync(path.join(os.tmpdir(), "runcap-outcome-"));
|
|
17
|
+
process.chdir(tmp);
|
|
18
|
+
|
|
19
|
+
// A tiny agent that spends through the gateway and writes (or doesn't write) a fix.
|
|
20
|
+
mkdirSync(path.join(tmp, "app"), { recursive: true });
|
|
21
|
+
writeFileSync(path.join(tmp, "app", "broken.mjs"), "export const ok = false;\n");
|
|
22
|
+
writeFileSync(path.join(tmp, "app", "verify.mjs"),
|
|
23
|
+
"import { ok } from './broken.mjs'; import assert from 'node:assert'; assert.strictEqual(ok, true, 'not fixed'); console.log('ok');\n");
|
|
24
|
+
writeFileSync(path.join(tmp, "agent-fix.mjs"),
|
|
25
|
+
"const b=process.env.OPENAI_BASE_URL;await fetch(`${b}/chat/completions`,{method:'POST',headers:{'content-type':'application/json',authorization:'Bearer x'},body:JSON.stringify({model:'gpt-4o',messages:[{role:'user',content:'fix it'}]})}).then(r=>r.text());" +
|
|
26
|
+
"const {writeFile}=await import('node:fs/promises');await writeFile('app/broken.mjs','export const ok = true;\\n');\n");
|
|
27
|
+
writeFileSync(path.join(tmp, "agent-nop.mjs"),
|
|
28
|
+
"const b=process.env.OPENAI_BASE_URL;await fetch(`${b}/chat/completions`,{method:'POST',headers:{'content-type':'application/json',authorization:'Bearer x'},body:JSON.stringify({model:'gpt-4o',messages:[{role:'user',content:'think'}]})}).then(r=>r.text());console.log('no fix');\n");
|
|
29
|
+
|
|
30
|
+
let failures = 0;
|
|
31
|
+
const check = (name, pass, detail) => { if (!pass) failures++; console.log(`${pass ? "PASS" : "FAIL"} ${name}${detail ? " — " + detail : ""}`); };
|
|
32
|
+
|
|
33
|
+
const { runOutcome } = await import(path.join(SRC_DIR, "mission-control.mjs"));
|
|
34
|
+
|
|
35
|
+
const nop = await runOutcome({ task: "fix ok", verify: "node app/verify.mjs", command: ["node", "agent-nop.mjs"], label: "nop", mock: true });
|
|
36
|
+
check("no-fix run is UNVERIFIED", nop.receipt.outcome === "UNVERIFIED", `outcome=${nop.receipt.outcome}`);
|
|
37
|
+
check("no-fix run still spent real money", nop.receipt.cost.actualCostUsd > 0, `cost=${nop.receipt.cost.actualCostUsd}`);
|
|
38
|
+
check("no-fix Verified Outcome Cost is null", nop.receipt.cost.verifiedOutcomeCostUsd === null);
|
|
39
|
+
check("no-fix counts money without delivery", nop.receipt.cost.moneySpentWithoutVerifiedDeliveryUsd > 0);
|
|
40
|
+
|
|
41
|
+
const fix = await runOutcome({ task: "fix ok", verify: "node app/verify.mjs", command: ["node", "agent-fix.mjs"], label: "fix", mock: true });
|
|
42
|
+
check("fix run is VERIFIED", fix.receipt.outcome === "VERIFIED", `outcome=${fix.receipt.outcome}`);
|
|
43
|
+
check("fix Verified Outcome Cost equals actual spend", fix.receipt.cost.verifiedOutcomeCostUsd === fix.receipt.cost.actualCostUsd);
|
|
44
|
+
check("fix counts zero undelivered money", fix.receipt.cost.moneySpentWithoutVerifiedDeliveryUsd === 0);
|
|
45
|
+
check("cost truth is calculated from usage + price table", /price_table/.test(fix.receipt.cost.truth));
|
|
46
|
+
|
|
47
|
+
console.log("\n" + (failures === 0 ? "ALL OUTCOME TESTS PASSED" : `${failures} OUTCOME TEST(S) FAILED`));
|
|
48
|
+
process.exit(failures === 0 ? 0 : 1);
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
// Proves src/policy.mjs parses, validates, and grades correctly. Pure unit test:
|
|
2
|
+
// no gateway, no git, no agent - just the policy module over hand-built inputs.
|
|
3
|
+
// Covers: YAML parse + hash, .json fallback, required-field validation, the
|
|
4
|
+
// guard/scope warnings, and every BLOCK condition in evaluatePolicyVerdict.
|
|
5
|
+
|
|
6
|
+
import os from "node:os";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import { fileURLToPath } from "node:url";
|
|
9
|
+
import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs";
|
|
10
|
+
|
|
11
|
+
const SRC_DIR = process.env.RUNCAP_SRC ?? path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "src");
|
|
12
|
+
const { loadPolicy, validatePolicy, evaluatePolicyVerdict, policyMeta } = await import(path.join(SRC_DIR, "policy.mjs"));
|
|
13
|
+
|
|
14
|
+
let failures = 0;
|
|
15
|
+
const check = (name, pass, detail) => { if (!pass) failures++; console.log(`${pass ? "PASS" : "FAIL"} ${name}${detail ? " — " + detail : ""}`); };
|
|
16
|
+
|
|
17
|
+
const tmp = mkdtempSync(path.join(os.tmpdir(), "runcap-policy-"));
|
|
18
|
+
mkdirSync(path.join(tmp, ".runcap"), { recursive: true });
|
|
19
|
+
|
|
20
|
+
const VALID_YAML = `version: v1
|
|
21
|
+
identity:
|
|
22
|
+
project: checkout
|
|
23
|
+
team: payments
|
|
24
|
+
mission:
|
|
25
|
+
name: Fix the failing checkout test
|
|
26
|
+
task_class: bugfix
|
|
27
|
+
budget:
|
|
28
|
+
mission_hard_limit_usd: 10
|
|
29
|
+
max_llm_calls: 12
|
|
30
|
+
max_runtime_minutes: 30
|
|
31
|
+
verification:
|
|
32
|
+
command: "node app/verify.mjs"
|
|
33
|
+
guard: strict
|
|
34
|
+
protect: ["tests/**"]
|
|
35
|
+
allow: ["src/checkout/**"]
|
|
36
|
+
`;
|
|
37
|
+
|
|
38
|
+
// 1. Valid YAML loads, parses, hashes, validates clean.
|
|
39
|
+
writeFileSync(path.join(tmp, ".runcap", "mission.yaml"), VALID_YAML);
|
|
40
|
+
const loaded = loadPolicy(tmp);
|
|
41
|
+
check("loadPolicy finds .runcap/mission.yaml", loaded && loaded.source.endsWith("mission.yaml"));
|
|
42
|
+
check("loadPolicy computes a sha256 hash", /^[0-9a-f]{64}$/.test(loaded.hash), loaded.hash);
|
|
43
|
+
check("valid policy parses mission.name", loaded.policy.mission.name === "Fix the failing checkout test");
|
|
44
|
+
const v1 = validatePolicy(loaded.policy);
|
|
45
|
+
check("valid policy validates ok", v1.ok === true, JSON.stringify(v1.errors));
|
|
46
|
+
check("valid policy with allow has no scope warning", !v1.warnings.some((w) => w.includes("allow is empty")));
|
|
47
|
+
const meta = policyMeta(loaded);
|
|
48
|
+
check("policyMeta carries identity + hash", meta.identity.project === "checkout" && meta.hash === loaded.hash);
|
|
49
|
+
check("policyMeta carries the limits", meta.limits.mission_hard_limit_usd === 10 && meta.limits.max_llm_calls === 12);
|
|
50
|
+
|
|
51
|
+
// 2. .json fallback parses with native JSON.parse (no parser needed).
|
|
52
|
+
const tmp2 = mkdtempSync(path.join(os.tmpdir(), "runcap-policy-json-"));
|
|
53
|
+
mkdirSync(path.join(tmp2, ".runcap"), { recursive: true });
|
|
54
|
+
writeFileSync(path.join(tmp2, ".runcap", "mission.json"), JSON.stringify({
|
|
55
|
+
version: "v1",
|
|
56
|
+
mission: { name: "json mission" },
|
|
57
|
+
budget: { mission_hard_limit_usd: 5 },
|
|
58
|
+
verification: { command: "npm test" }
|
|
59
|
+
}));
|
|
60
|
+
const jsonLoaded = loadPolicy(tmp2);
|
|
61
|
+
check("loadPolicy reads .json fallback", jsonLoaded && jsonLoaded.source.endsWith("mission.json"));
|
|
62
|
+
check("json policy validates ok", validatePolicy(jsonLoaded.policy).ok === true);
|
|
63
|
+
|
|
64
|
+
// 3. Missing verification.command → invalid.
|
|
65
|
+
const noVerify = validatePolicy({ version: "v1", mission: { name: "x" }, budget: { mission_hard_limit_usd: 1 } });
|
|
66
|
+
check("missing verification.command is invalid", noVerify.ok === false && noVerify.errors.some((e) => e.includes("verification.command")));
|
|
67
|
+
|
|
68
|
+
// 4. Bad version → invalid.
|
|
69
|
+
const badVersion = validatePolicy({ version: "v2", mission: { name: "x" }, budget: { mission_hard_limit_usd: 1 }, verification: { command: "npm test" } });
|
|
70
|
+
check("wrong version is invalid", badVersion.ok === false && badVersion.errors.some((e) => e.includes("version")));
|
|
71
|
+
|
|
72
|
+
// 5. Missing budget cap → invalid.
|
|
73
|
+
const noBudget = validatePolicy({ version: "v1", mission: { name: "x" }, verification: { command: "npm test" } });
|
|
74
|
+
check("missing mission_hard_limit_usd is invalid", noBudget.ok === false && noBudget.errors.some((e) => e.includes("mission_hard_limit_usd")));
|
|
75
|
+
|
|
76
|
+
// 6. No allow scope → warning (not error).
|
|
77
|
+
const noAllow = validatePolicy({ version: "v1", mission: { name: "x" }, budget: { mission_hard_limit_usd: 1 }, verification: { command: "npm test", allow: [] } });
|
|
78
|
+
check("empty allow produces a warning", noAllow.ok === true && noAllow.warnings.some((w) => w.includes("allow is empty")));
|
|
79
|
+
|
|
80
|
+
// 7. evaluatePolicyVerdict: a clean VERIFIED receipt → PASS.
|
|
81
|
+
const policy = loaded.policy;
|
|
82
|
+
const cleanReceipt = {
|
|
83
|
+
outcome: "VERIFIED",
|
|
84
|
+
verificationIntegrity: { status: "VERIFIED_STRONG", violations: [] },
|
|
85
|
+
cost: { actualCostUsd: 0.0007, llmCalls: 2, budgetGuardTripped: false },
|
|
86
|
+
work: { agentDurationMs: 5000 }
|
|
87
|
+
};
|
|
88
|
+
check("clean receipt grades PASS", evaluatePolicyVerdict(cleanReceipt, policy).verdict === "PASS");
|
|
89
|
+
|
|
90
|
+
// 8. Compromised verifier → BLOCKED with the reason.
|
|
91
|
+
const compromised = { ...cleanReceipt, verificationIntegrity: { status: "VERIFIER_COMPROMISED", violations: ["verifier_file_unchanged:app/verify.mjs"] } };
|
|
92
|
+
const cv = evaluatePolicyVerdict(compromised, policy);
|
|
93
|
+
check("compromised verifier grades BLOCKED", cv.verdict === "BLOCKED" && cv.reasons.some((r) => r.includes("VERIFIER_COMPROMISED")));
|
|
94
|
+
|
|
95
|
+
// 9. UNVERIFIED → BLOCKED.
|
|
96
|
+
const unver = { ...cleanReceipt, outcome: "UNVERIFIED", verificationIntegrity: { status: "UNVERIFIED", violations: [] } };
|
|
97
|
+
check("unverified grades BLOCKED", evaluatePolicyVerdict(unver, policy).verdict === "BLOCKED");
|
|
98
|
+
|
|
99
|
+
// 10. Out-of-allow scope → BLOCKED.
|
|
100
|
+
const scope = { ...cleanReceipt, verificationIntegrity: { status: "VERIFIED_STRONG", violations: ["within_allowed_scope:src/other.mjs"] } };
|
|
101
|
+
const sc = evaluatePolicyVerdict(scope, policy);
|
|
102
|
+
check("out-of-scope edit grades BLOCKED", sc.verdict === "BLOCKED" && sc.reasons.some((r) => r.toLowerCase().includes("scope")));
|
|
103
|
+
|
|
104
|
+
// 11. Over the dollar cap → BLOCKED.
|
|
105
|
+
const overCost = { ...cleanReceipt, cost: { actualCostUsd: 11, llmCalls: 2, budgetGuardTripped: false } };
|
|
106
|
+
check("over the cap grades BLOCKED", evaluatePolicyVerdict(overCost, policy).verdict === "BLOCKED");
|
|
107
|
+
|
|
108
|
+
// 12. budget_guard tripped → BLOCKED.
|
|
109
|
+
const guardTrip = { ...cleanReceipt, cost: { actualCostUsd: 1, llmCalls: 2, budgetGuardTripped: true } };
|
|
110
|
+
check("budget guard trip grades BLOCKED", evaluatePolicyVerdict(guardTrip, policy).verdict === "BLOCKED");
|
|
111
|
+
|
|
112
|
+
// 13. Too many LLM calls → BLOCKED.
|
|
113
|
+
const tooMany = { ...cleanReceipt, cost: { actualCostUsd: 1, llmCalls: 99, budgetGuardTripped: false } };
|
|
114
|
+
check("too many llm calls grades BLOCKED", evaluatePolicyVerdict(tooMany, policy).verdict === "BLOCKED");
|
|
115
|
+
|
|
116
|
+
// 14. Over the runtime budget → BLOCKED.
|
|
117
|
+
const slow = { ...cleanReceipt, work: { agentDurationMs: 31 * 60_000 } };
|
|
118
|
+
check("over runtime budget grades BLOCKED", evaluatePolicyVerdict(slow, policy).verdict === "BLOCKED");
|
|
119
|
+
|
|
120
|
+
console.log("\n" + (failures === 0 ? "ALL POLICY TESTS PASSED" : `${failures} POLICY TEST(S) FAILED`));
|
|
121
|
+
process.exit(failures === 0 ? 0 : 1);
|