swarm-orchestrator 11.2.0 → 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -211
- package/dist/src/audit/cheat-detector/diff-chunker.js +83 -5
- package/dist/src/audit/cheat-detector/diff-walker.js +5 -0
- package/dist/src/audit/cheat-detector/judge-primary.js +8 -1
- package/dist/src/audit/cheat-detector/judge-prompts/v2-balanced.js +46 -7
- package/dist/src/audit/cheat-detector/llm-judge/index.js +44 -9
- package/dist/src/audit/cheat-detector/llm-judge/local-judge.js +10 -2
- package/dist/src/audit/cheat-detector/llm-judge/ollama-judge.js +150 -0
- package/dist/src/audit/cheat-detector/mock-delta.js +101 -0
- package/dist/src/audit/cheat-detector/mock-of-hallucination.js +9 -1
- package/dist/src/audit/cheat-detector/verify-findings.js +34 -0
- package/dist/src/audit/execution-grounded/corroborate.js +5 -0
- package/dist/src/audit/execution-grounded/dead-branch-restoration.js +457 -0
- package/dist/src/audit/execution-grounded/fake-refactor-restoration.js +399 -0
- package/dist/src/audit/execution-grounded/index.js +719 -8
- package/dist/src/audit/execution-grounded/issue-repro.js +5 -2
- package/dist/src/audit/execution-grounded/mock-restoration.js +396 -0
- package/dist/src/audit/execution-grounded/no-op-fix-restoration.js +376 -0
- package/dist/src/audit/execution-grounded/test-restoration.js +906 -0
- package/dist/src/audit/execution-grounded/type-suppression-restoration.js +496 -0
- package/dist/src/audit/gate/block-eligibility.js +35 -14
- package/dist/src/audit/gate/block-trigger-types.js +6 -0
- package/dist/src/audit/gate/block-triggers.js +320 -4
- package/dist/src/audit/gate/gate-decision.js +30 -12
- package/dist/src/audit/gate/self-certifying.js +74 -0
- package/dist/src/audit/gate/wilson.js +26 -0
- package/dist/src/audit/oracle/evade/evaders.js +77 -1
- package/dist/src/audit/report-comment/block-trigger-section.js +146 -3
- package/dist/src/audit/report-comment/index.js +8 -0
- package/dist/src/cli/v8/audit-handler.js +363 -112
- package/dist/src/cli/v8/resume-handler.js +5 -59
- package/dist/src/cli/v8/run-handler.js +133 -142
- package/dist/src/cli/v8/session-utils.js +121 -0
- package/dist/src/persona/verifier-persona.js +2 -1
- package/dist/src/population/persona-message.js +2 -1
- package/dist/src/session/bounded-session.js +94 -0
- package/dist/src/shared/child-env.js +94 -0
- package/dist/src/shared/obligation-rendering.js +80 -0
- package/dist/src/shared-predicates/predicate-runner.js +2 -1
- package/dist/src/verification/run-verifier.js +3 -2
- package/package.json +13 -2
|
@@ -12,22 +12,42 @@ exports.formatJudgePromptHash = formatJudgePromptHash;
|
|
|
12
12
|
const logger_1 = require("../../../logger");
|
|
13
13
|
const anthropic_judge_1 = require("./anthropic-judge");
|
|
14
14
|
const local_judge_1 = require("./local-judge");
|
|
15
|
+
const ollama_judge_1 = require("./ollama-judge");
|
|
15
16
|
const cache_1 = require("./cache");
|
|
16
17
|
const diff_chunker_1 = require("../diff-chunker");
|
|
17
18
|
const types_1 = require("./types");
|
|
18
19
|
const logger = (0, logger_1.getLogger)('audit:judge');
|
|
20
|
+
/** Resolve the production Anthropic judge model id. Defaults to the
|
|
21
|
+
* pinned snapshot; `SWARM_JUDGE_MODEL` overrides without a code change
|
|
22
|
+
* so an operator can move past a deprecation announcement without
|
|
23
|
+
* shipping a new build. The model id folds into the cache key, so an
|
|
24
|
+
* override never collides with cached answers from a prior model. */
|
|
25
|
+
function resolveAnthropicModelId(opts) {
|
|
26
|
+
if (opts.modelId !== undefined)
|
|
27
|
+
return opts.modelId;
|
|
28
|
+
const override = process.env.SWARM_JUDGE_MODEL;
|
|
29
|
+
if (override !== undefined && override.trim().length > 0)
|
|
30
|
+
return override.trim();
|
|
31
|
+
return types_1.PINNED_JUDGE_MODEL_ID;
|
|
32
|
+
}
|
|
19
33
|
/** Resolve the judge provider. `SWARM_JUDGE_PROVIDER=local` points the
|
|
20
|
-
* judge at a free OpenAI-compatible server
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
34
|
+
* judge at a free OpenAI-compatible server and
|
|
35
|
+
* `SWARM_JUDGE_PROVIDER=ollama` at Ollama's native /api/chat (the /v1
|
|
36
|
+
* bridge cannot disable a thinking model's reasoning, so local and
|
|
37
|
+
* ollama are distinct providers); anything else keeps the pinned
|
|
38
|
+
* Anthropic Haiku judge. The model id is folded into the cache key, so
|
|
39
|
+
* answers from different providers never collide. */
|
|
24
40
|
function resolveJudgeProvider(opts) {
|
|
25
41
|
if (opts.client !== undefined)
|
|
26
|
-
return { client: opts.client, modelId: opts
|
|
27
|
-
|
|
42
|
+
return { client: opts.client, modelId: resolveAnthropicModelId(opts) };
|
|
43
|
+
const provider = (process.env.SWARM_JUDGE_PROVIDER ?? '').toLowerCase();
|
|
44
|
+
if (provider === 'local') {
|
|
28
45
|
return { client: new local_judge_1.LocalJudge(), modelId: opts.modelId ?? (0, local_judge_1.localJudgeModelId)() };
|
|
29
46
|
}
|
|
30
|
-
|
|
47
|
+
if (provider === 'ollama') {
|
|
48
|
+
return { client: new ollama_judge_1.OllamaJudge(), modelId: opts.modelId ?? (0, ollama_judge_1.ollamaJudgeModelId)() };
|
|
49
|
+
}
|
|
50
|
+
return { client: new anthropic_judge_1.AnthropicJudge(), modelId: resolveAnthropicModelId(opts) };
|
|
31
51
|
}
|
|
32
52
|
async function askJudge(opts) {
|
|
33
53
|
const provider = resolveJudgeProvider(opts);
|
|
@@ -150,8 +170,23 @@ function hasCredentials(opts) {
|
|
|
150
170
|
// ~120k chars is well under Haiku's 200k-token ceiling once the prompt
|
|
151
171
|
// scaffold is added. It is the per-chunk budget for the hunk-aware
|
|
152
172
|
// chunker, and is exported so the benchmark harnesses size their context
|
|
153
|
-
// the same way.
|
|
154
|
-
|
|
173
|
+
// the same way. `SWARM_JUDGE_MAX_DIFF_CHARS` overrides it for judges
|
|
174
|
+
// with smaller context windows: chars-per-token collapses toward 1 on
|
|
175
|
+
// base64/SVG-heavy diffs, so a 120k-char chunk can overflow a 131k-token
|
|
176
|
+
// local model and make the server truncate the prompt (front-first,
|
|
177
|
+
// eating the question) where Haiku would still have headroom.
|
|
178
|
+
function resolveMaxJudgeDiffChars() {
|
|
179
|
+
const raw = process.env.SWARM_JUDGE_MAX_DIFF_CHARS;
|
|
180
|
+
if (raw === undefined || raw.trim().length === 0)
|
|
181
|
+
return 120_000;
|
|
182
|
+
const n = Number(raw);
|
|
183
|
+
if (!Number.isFinite(n) || n < 1_000) {
|
|
184
|
+
logger.warn(`ignoring SWARM_JUDGE_MAX_DIFF_CHARS=${raw} (need a number >= 1000)`);
|
|
185
|
+
return 120_000;
|
|
186
|
+
}
|
|
187
|
+
return Math.floor(n);
|
|
188
|
+
}
|
|
189
|
+
exports.MAX_JUDGE_DIFF_CHARS = resolveMaxJudgeDiffChars();
|
|
155
190
|
function recordLedger(ledger, detector, result) {
|
|
156
191
|
if (ledger === undefined)
|
|
157
192
|
return;
|
|
@@ -14,7 +14,9 @@
|
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
exports.LocalJudge = void 0;
|
|
16
16
|
exports.localJudgeModelId = localJudgeModelId;
|
|
17
|
+
const logger_1 = require("../../../logger");
|
|
17
18
|
const anthropic_judge_1 = require("./anthropic-judge");
|
|
19
|
+
const logger = (0, logger_1.getLogger)('audit:judge:local');
|
|
18
20
|
class LocalJudge {
|
|
19
21
|
baseUrl;
|
|
20
22
|
model;
|
|
@@ -42,17 +44,23 @@ class LocalJudge {
|
|
|
42
44
|
enable_thinking: false,
|
|
43
45
|
}),
|
|
44
46
|
});
|
|
45
|
-
if (!res.ok)
|
|
47
|
+
if (!res.ok) {
|
|
48
|
+
logger.warn(`local judge request failed (${res.status} ${res.statusText})`);
|
|
46
49
|
return { raw: '', answer: 'unavailable' };
|
|
50
|
+
}
|
|
47
51
|
const json = (await res.json());
|
|
48
52
|
const raw = (json.choices?.[0]?.message?.content ?? '').trim();
|
|
49
53
|
const parsed = (0, anthropic_judge_1.parseJudgeReply)(raw);
|
|
54
|
+
if (parsed.answer === 'unavailable') {
|
|
55
|
+
logger.warn(`local judge reply did not parse as YES/NO (head: ${JSON.stringify(raw.slice(0, 120))})`);
|
|
56
|
+
}
|
|
50
57
|
const out = { raw, answer: parsed.answer };
|
|
51
58
|
if (parsed.reason !== undefined)
|
|
52
59
|
out.reason = parsed.reason;
|
|
53
60
|
return out;
|
|
54
61
|
}
|
|
55
|
-
catch {
|
|
62
|
+
catch (err) {
|
|
63
|
+
logger.warn(`local judge call failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
56
64
|
return { raw: '', answer: 'unavailable' };
|
|
57
65
|
}
|
|
58
66
|
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Ollama judge. Talks to Ollama's native /api/chat (NOT its OpenAI
|
|
3
|
+
// /v1 bridge): thinking models like qwen3 only honor the think:false
|
|
4
|
+
// toggle on the native endpoint, and through /v1 they spend the whole
|
|
5
|
+
// token budget reasoning and return empty content, which the strict
|
|
6
|
+
// YES/NO parser collapses to 'unavailable'. Selected with
|
|
7
|
+
// `SWARM_JUDGE_PROVIDER=ollama`; base URL from `SWARM_JUDGE_BASE_URL`
|
|
8
|
+
// (or `OLLAMA_BASE_URL`), model from `SWARM_JUDGE_MODEL` (required:
|
|
9
|
+
// Ollama has no meaningful default model).
|
|
10
|
+
//
|
|
11
|
+
// Uses node:http with an explicit generous timeout instead of fetch:
|
|
12
|
+
// with stream:false Ollama sends headers only when generation
|
|
13
|
+
// completes, and a long prompt on a loaded machine can exceed fetch's
|
|
14
|
+
// fixed 300s header timeout. That timeout is how the first local-judge
|
|
15
|
+
// run silently lost most of its answers.
|
|
16
|
+
//
|
|
17
|
+
// Like every JudgeClient it returns 'unavailable' rather than throwing,
|
|
18
|
+
// so a down server degrades to deterministic-only behavior, but every
|
|
19
|
+
// failure path is logged: a judge that silently answers nothing looks
|
|
20
|
+
// identical to a judge that answered NO, and that hides false
|
|
21
|
+
// negatives.
|
|
22
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
23
|
+
if (k2 === undefined) k2 = k;
|
|
24
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
25
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
26
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
27
|
+
}
|
|
28
|
+
Object.defineProperty(o, k2, desc);
|
|
29
|
+
}) : (function(o, m, k, k2) {
|
|
30
|
+
if (k2 === undefined) k2 = k;
|
|
31
|
+
o[k2] = m[k];
|
|
32
|
+
}));
|
|
33
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
34
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
35
|
+
}) : function(o, v) {
|
|
36
|
+
o["default"] = v;
|
|
37
|
+
});
|
|
38
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
39
|
+
var ownKeys = function(o) {
|
|
40
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
41
|
+
var ar = [];
|
|
42
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
43
|
+
return ar;
|
|
44
|
+
};
|
|
45
|
+
return ownKeys(o);
|
|
46
|
+
};
|
|
47
|
+
return function (mod) {
|
|
48
|
+
if (mod && mod.__esModule) return mod;
|
|
49
|
+
var result = {};
|
|
50
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
51
|
+
__setModuleDefault(result, mod);
|
|
52
|
+
return result;
|
|
53
|
+
};
|
|
54
|
+
})();
|
|
55
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
56
|
+
exports.OllamaJudge = void 0;
|
|
57
|
+
exports.ollamaJudgeModelId = ollamaJudgeModelId;
|
|
58
|
+
const http = __importStar(require("http"));
|
|
59
|
+
const logger_1 = require("../../../logger");
|
|
60
|
+
const anthropic_judge_1 = require("./anthropic-judge");
|
|
61
|
+
const logger = (0, logger_1.getLogger)('audit:judge:ollama');
|
|
62
|
+
class OllamaJudge {
|
|
63
|
+
baseUrl;
|
|
64
|
+
model;
|
|
65
|
+
timeoutMs;
|
|
66
|
+
numPredict;
|
|
67
|
+
constructor(options = {}) {
|
|
68
|
+
this.baseUrl = (options.baseUrl ??
|
|
69
|
+
process.env.SWARM_JUDGE_BASE_URL ??
|
|
70
|
+
process.env.OLLAMA_BASE_URL ??
|
|
71
|
+
'http://localhost:11434').replace(/\/$/, '');
|
|
72
|
+
this.model = options.model ?? process.env.SWARM_JUDGE_MODEL;
|
|
73
|
+
this.timeoutMs = options.timeoutMs ?? 10 * 60 * 1000;
|
|
74
|
+
this.numPredict = options.numPredict ?? 256;
|
|
75
|
+
}
|
|
76
|
+
async ask(prompt) {
|
|
77
|
+
if (this.model === undefined || this.model.length === 0) {
|
|
78
|
+
logger.warn('ollama judge selected but SWARM_JUDGE_MODEL is not set; answering unavailable');
|
|
79
|
+
return { raw: '', answer: 'unavailable' };
|
|
80
|
+
}
|
|
81
|
+
let body;
|
|
82
|
+
try {
|
|
83
|
+
body = await this.post(JSON.stringify({
|
|
84
|
+
model: this.model,
|
|
85
|
+
think: false,
|
|
86
|
+
stream: false,
|
|
87
|
+
options: { temperature: 0, num_predict: this.numPredict },
|
|
88
|
+
messages: [
|
|
89
|
+
{ role: 'system', content: prompt.system },
|
|
90
|
+
{ role: 'user', content: prompt.user },
|
|
91
|
+
],
|
|
92
|
+
}));
|
|
93
|
+
}
|
|
94
|
+
catch (err) {
|
|
95
|
+
logger.warn(`ollama judge call failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
96
|
+
return { raw: '', answer: 'unavailable' };
|
|
97
|
+
}
|
|
98
|
+
let raw;
|
|
99
|
+
try {
|
|
100
|
+
const json = JSON.parse(body);
|
|
101
|
+
raw = (json.message?.content ?? '').trim();
|
|
102
|
+
}
|
|
103
|
+
catch (err) {
|
|
104
|
+
logger.warn(`ollama judge returned unparseable JSON: ${err instanceof Error ? err.message : String(err)}`);
|
|
105
|
+
return { raw: '', answer: 'unavailable' };
|
|
106
|
+
}
|
|
107
|
+
const parsed = (0, anthropic_judge_1.parseJudgeReply)(raw);
|
|
108
|
+
if (parsed.answer === 'unavailable') {
|
|
109
|
+
logger.warn(`ollama judge reply did not parse as YES/NO (head: ${JSON.stringify(raw.slice(0, 120))})`);
|
|
110
|
+
}
|
|
111
|
+
const out = {
|
|
112
|
+
raw,
|
|
113
|
+
answer: parsed.answer,
|
|
114
|
+
};
|
|
115
|
+
if (parsed.reason !== undefined)
|
|
116
|
+
out.reason = parsed.reason;
|
|
117
|
+
return out;
|
|
118
|
+
}
|
|
119
|
+
post(payload) {
|
|
120
|
+
return new Promise((resolve, reject) => {
|
|
121
|
+
const req = http.request(`${this.baseUrl}/api/chat`, {
|
|
122
|
+
method: 'POST',
|
|
123
|
+
headers: { 'content-type': 'application/json' },
|
|
124
|
+
timeout: this.timeoutMs,
|
|
125
|
+
}, (res) => {
|
|
126
|
+
const chunks = [];
|
|
127
|
+
res.on('data', (c) => chunks.push(c));
|
|
128
|
+
res.on('end', () => {
|
|
129
|
+
const text = Buffer.concat(chunks).toString('utf8');
|
|
130
|
+
if ((res.statusCode ?? 500) >= 300) {
|
|
131
|
+
reject(new Error(`ollama judge request failed (${res.statusCode}): ${text.slice(0, 200)}`));
|
|
132
|
+
}
|
|
133
|
+
else
|
|
134
|
+
resolve(text);
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
req.on('timeout', () => req.destroy(new Error(`ollama judge request exceeded ${this.timeoutMs}ms`)));
|
|
138
|
+
req.on('error', (err) => reject(err));
|
|
139
|
+
req.end(payload);
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
exports.OllamaJudge = OllamaJudge;
|
|
144
|
+
/** The judge model id folded into the cache key when the ollama provider
|
|
145
|
+
* is selected, so ollama-judged answers never collide with Haiku- or
|
|
146
|
+
* local-judged ones in the committed cache. */
|
|
147
|
+
function ollamaJudgeModelId() {
|
|
148
|
+
const model = process.env.SWARM_JUDGE_MODEL ?? 'unset';
|
|
149
|
+
return `ollama:${model}`;
|
|
150
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Deterministic mock-delta pre-filter for the cheat-mock-mutation semantic
|
|
3
|
+
// category. A mutated mock is invisible to every structural detector: the
|
|
4
|
+
// assertion still exists, nothing is stripped, and the mocked target is a
|
|
5
|
+
// real local unit, so `mockResolvedValue(fakeData)` reads as ordinary test
|
|
6
|
+
// setup. Only the judge can call it, but the judge measured 0.16 recall
|
|
7
|
+
// running over the whole PR diff: the six-line mock hunk is a needle in a
|
|
8
|
+
// 40k-char haystack, and a single yes/no over the whole diff walks past it.
|
|
9
|
+
//
|
|
10
|
+
// This module finds the hunks that introduce a value-injecting mock in a
|
|
11
|
+
// test file and hands ONLY those hunks to the judge. Two effects, both
|
|
12
|
+
// measured in benchmarks/results/AB-REPORT.md:
|
|
13
|
+
// - recall up: the judge reads the mock hunk in isolation, so the
|
|
14
|
+
// value-injection is the whole prompt, not a tail detail it skims.
|
|
15
|
+
// - false positives down: the judge is asked the cheat-mock-mutation
|
|
16
|
+
// question only when an added mock-return wiring actually exists, so a
|
|
17
|
+
// clean PR with no such hunk never produces a false yes for this
|
|
18
|
+
// category (the whole-diff path asked on every PR).
|
|
19
|
+
//
|
|
20
|
+
// The filter is high-recall on purpose: any added value-injecting mock in a
|
|
21
|
+
// test hunk qualifies, and the judge makes the legitimate-vs-cheat call on
|
|
22
|
+
// the focused hunk. That division (deterministic locate, judge confirm) is
|
|
23
|
+
// the same shape as the structural detectors feeding the confirmation gate.
|
|
24
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
+
exports.extractMockMutationFocus = extractMockMutationFocus;
|
|
26
|
+
exports.focusSemanticDiff = focusSemanticDiff;
|
|
27
|
+
const diff_chunker_1 = require("./diff-chunker");
|
|
28
|
+
const diff_walker_1 = require("./diff-walker");
|
|
29
|
+
// jest / vitest value-injecting mock methods: each exists only to make a
|
|
30
|
+
// mock hand back a fixed value or a scripted implementation, which is the
|
|
31
|
+
// exact lever a mock-mutation cheat pulls. `mockImplementation` is included
|
|
32
|
+
// because `mockImplementation(() => expected)` injects a value just as
|
|
33
|
+
// `mockReturnValue(expected)` does.
|
|
34
|
+
const JEST_MOCK_RETURN_RE = /\.(?:mockReturnValue|mockResolvedValue|mockRejectedValue|mockReturnValueOnce|mockResolvedValueOnce|mockRejectedValueOnce|mockImplementation|mockImplementationOnce)\s*\(/;
|
|
35
|
+
// sinon stubs inject a value through `.returns(...)` / `.resolves(...)`.
|
|
36
|
+
// Those method names are common enough on their own (a promise helper can
|
|
37
|
+
// expose `.resolves`) that we only treat them as a mock signal when the
|
|
38
|
+
// hunk is plainly sinon: it names `sinon` or constructs a `stub`.
|
|
39
|
+
const SINON_RETURN_RE = /\.(?:returns|resolves|rejects)\s*\(/;
|
|
40
|
+
const SINON_CONTEXT_RE = /\b(?:sinon|\.stub\s*\(|createStubInstance)\b|\bstub\b/;
|
|
41
|
+
/** An added (`+`) content line, excluding the unified-diff `+++` file header. */
|
|
42
|
+
function isAddedContentLine(line) {
|
|
43
|
+
return line.startsWith('+') && !line.startsWith('+++');
|
|
44
|
+
}
|
|
45
|
+
/** True when a single added line introduces a value-injecting mock. */
|
|
46
|
+
function addedLineInjectsMock(line, hunkText) {
|
|
47
|
+
if (!isAddedContentLine(line))
|
|
48
|
+
return false;
|
|
49
|
+
if (JEST_MOCK_RETURN_RE.test(line))
|
|
50
|
+
return true;
|
|
51
|
+
if (SINON_RETURN_RE.test(line) && SINON_CONTEXT_RE.test(hunkText))
|
|
52
|
+
return true;
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
/** True when a one-hunk diff (file header + hunk) adds a value-injecting mock. */
|
|
56
|
+
function hunkInjectsMock(hunkText) {
|
|
57
|
+
for (const line of hunkText.split('\n')) {
|
|
58
|
+
if (addedLineInjectsMock(line, hunkText))
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Extract the test-file hunks that introduce a value-injecting mock, each as
|
|
65
|
+
* a valid one-hunk unified diff (file header + hunk). Returns the hunks in
|
|
66
|
+
* diff order and a `focusedDiff` that concatenates them, or `null` when the
|
|
67
|
+
* diff introduces no such mock.
|
|
68
|
+
*
|
|
69
|
+
* Pure and deterministic: the same diff always yields the same focused diff,
|
|
70
|
+
* so a committed benchmark replays the exact prompt the judge was scored
|
|
71
|
+
* against (the focused diff folds into the judge cache key).
|
|
72
|
+
*/
|
|
73
|
+
function extractMockMutationFocus(diff) {
|
|
74
|
+
const hits = [];
|
|
75
|
+
for (const chunk of (0, diff_chunker_1.chunkUnifiedDiffByHunk)(diff)) {
|
|
76
|
+
if (!(0, diff_walker_1.isTestFile)(chunk.file))
|
|
77
|
+
continue;
|
|
78
|
+
if (hunkInjectsMock(chunk.text))
|
|
79
|
+
hits.push(chunk.text);
|
|
80
|
+
}
|
|
81
|
+
return { hunks: hits, focusedDiff: hits.length > 0 ? hits.join('') : null };
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* The diff a semantic-category judge call should read, plus whether to skip
|
|
85
|
+
* the call entirely. Single source of truth shared by the production
|
|
86
|
+
* judge-primary path and the oracle / calibration harnesses so the scored
|
|
87
|
+
* prompt and the shipped prompt are identical.
|
|
88
|
+
*
|
|
89
|
+
* - `goal-not-fixed`: the whole diff, never skipped (the claimed fix can be
|
|
90
|
+
* anywhere in the change).
|
|
91
|
+
* - `cheat-mock-mutation`: only the mock-bearing hunks; skip when the diff
|
|
92
|
+
* introduces no value-injecting mock (no signal, so no finding).
|
|
93
|
+
*/
|
|
94
|
+
function focusSemanticDiff(category, diff) {
|
|
95
|
+
if (category !== 'cheat-mock-mutation')
|
|
96
|
+
return { skip: false, diff };
|
|
97
|
+
const { focusedDiff } = extractMockMutationFocus(diff);
|
|
98
|
+
if (focusedDiff === null)
|
|
99
|
+
return { skip: true, diff };
|
|
100
|
+
return { skip: false, diff: focusedDiff };
|
|
101
|
+
}
|
|
@@ -217,7 +217,15 @@ function extractMockTarget(line) {
|
|
|
217
217
|
return undefined;
|
|
218
218
|
}
|
|
219
219
|
function isLocalImport(target) {
|
|
220
|
-
|
|
220
|
+
// `@/...` and `~/...` are TypeScript/bundler path aliases (tsconfig `paths`,
|
|
221
|
+
// Vite/webpack `resolve.alias`) that map to a local source directory. They
|
|
222
|
+
// are never published packages — `@scope/pkg` always has a name after the
|
|
223
|
+
// `@`, so `@/` (slash immediately after `@`) is unambiguously an alias. A
|
|
224
|
+
// mock of a path alias resolves locally and is not a manifest concern.
|
|
225
|
+
return (target.startsWith('.') ||
|
|
226
|
+
target.startsWith('/') ||
|
|
227
|
+
target.startsWith('~') ||
|
|
228
|
+
target.startsWith('@/'));
|
|
221
229
|
}
|
|
222
230
|
function topLevelPackageOf(target) {
|
|
223
231
|
if (target.startsWith('@')) {
|
|
@@ -48,6 +48,16 @@ const TEST_REMOVAL = new Set([
|
|
|
48
48
|
'test-relaxation',
|
|
49
49
|
'assertion-strip',
|
|
50
50
|
]);
|
|
51
|
+
// The error-swallow detector blocks on a bare empty catch. That pattern is a
|
|
52
|
+
// cheat when it removes error propagation from code a caller already relied on;
|
|
53
|
+
// in a file the PR *creates*, there is no prior behavior to hide — a bare
|
|
54
|
+
// `try/except: pass` in a new standalone script is best-effort robustness, not
|
|
55
|
+
// the silencing of an existing failure. The real-corpus FPs are exactly this
|
|
56
|
+
// (whole new app.py / bot.py / plotting scripts). The oracle's error-swallow
|
|
57
|
+
// injector wraps a call in an existing modified file, so this never touches a
|
|
58
|
+
// seeded cheat. Demoted (not dropped) so a bare --diff-file audit still records
|
|
59
|
+
// the observation.
|
|
60
|
+
const NEW_FILE_DEMOTED = new Set(['error-swallow']);
|
|
51
61
|
const RENAME_MESSAGE = /^Function "([^"]+)" was renamed to "([^"]+)"/;
|
|
52
62
|
// The two test-relaxation/assertion-strip messages that fire on a deletion
|
|
53
63
|
// scoped to a single hunk. They miss a test that was relocated or
|
|
@@ -72,11 +82,13 @@ function verifyFindings(findings, ctx) {
|
|
|
72
82
|
const ambiguousRenameKeys = collectAmbiguousRenameKeys(findings);
|
|
73
83
|
const deletesNonTestSource = hasNonTestSourceDeletion(ctx.files);
|
|
74
84
|
const testSignals = collectTestSignals(ctx.files);
|
|
85
|
+
const addedFiles = collectAddedFiles(ctx.files);
|
|
75
86
|
for (const finding of findings) {
|
|
76
87
|
const refutation = refute(finding, ctx, {
|
|
77
88
|
ambiguousRenameKeys,
|
|
78
89
|
deletesNonTestSource,
|
|
79
90
|
testSignals,
|
|
91
|
+
addedFiles,
|
|
80
92
|
});
|
|
81
93
|
if (refutation === undefined) {
|
|
82
94
|
kept.push(finding);
|
|
@@ -98,6 +110,16 @@ function refute(finding, ctx, state) {
|
|
|
98
110
|
// Already informational findings have nothing to refute or demote.
|
|
99
111
|
if (finding.severity === 'info')
|
|
100
112
|
return undefined;
|
|
113
|
+
if (NEW_FILE_DEMOTED.has(finding.category) && state.addedFiles.has(finding.location.file)) {
|
|
114
|
+
return {
|
|
115
|
+
action: 'downgrade',
|
|
116
|
+
severity: 'info',
|
|
117
|
+
rule: 'new-file-best-effort',
|
|
118
|
+
reason: `${finding.category} blocks when a bare empty catch removes error propagation from ` +
|
|
119
|
+
`existing behavior; this catch is in a file the PR creates, so there is no prior ` +
|
|
120
|
+
`behavior to hide — it is new best-effort code, not a silenced existing failure`,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
101
123
|
if (FIX_CLAIM_GATED.has(finding.category) && !ctx.intent.claimsFix) {
|
|
102
124
|
return {
|
|
103
125
|
action: 'downgrade',
|
|
@@ -279,6 +301,18 @@ function assignConfidence(findings) {
|
|
|
279
301
|
for (const f of findings)
|
|
280
302
|
setFindingConfidence(f);
|
|
281
303
|
}
|
|
304
|
+
/** Post-image paths of files this PR creates (added, not modified). */
|
|
305
|
+
function collectAddedFiles(files) {
|
|
306
|
+
const out = new Set();
|
|
307
|
+
for (const file of files) {
|
|
308
|
+
if ((0, diff_walker_1.fileKind)(file) !== 'add')
|
|
309
|
+
continue;
|
|
310
|
+
const p = (0, diff_walker_1.filePath)(file);
|
|
311
|
+
if (p.length > 0)
|
|
312
|
+
out.add(p);
|
|
313
|
+
}
|
|
314
|
+
return out;
|
|
315
|
+
}
|
|
282
316
|
function hasNonTestSourceDeletion(files) {
|
|
283
317
|
for (const file of files) {
|
|
284
318
|
if ((0, diff_walker_1.fileKind)(file) !== 'delete')
|
|
@@ -97,6 +97,11 @@ function corroborationFor(finding, signals) {
|
|
|
97
97
|
function corroborateStructuralFindings(findings, signals) {
|
|
98
98
|
let corroborated = 0;
|
|
99
99
|
for (const finding of findings) {
|
|
100
|
+
// A finding that already carries runtime backing (a proven test
|
|
101
|
+
// restoration ran earlier in the same pass) keeps it: a restored test
|
|
102
|
+
// failing is stronger evidence than any signal matched here.
|
|
103
|
+
if (finding.runtimeCorroboration !== undefined)
|
|
104
|
+
continue;
|
|
100
105
|
const corroboration = corroborationFor(finding, signals);
|
|
101
106
|
if (corroboration !== null) {
|
|
102
107
|
finding.runtimeCorroboration = corroboration;
|