pi-jev-wiki 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/LICENSE +21 -0
- package/README.md +152 -0
- package/package.json +63 -0
- package/skills/llm-wiki/SKILL.md +143 -0
- package/skills/llm-wiki/references/decision.md +32 -0
- package/skills/llm-wiki/references/flow.md +32 -0
- package/skills/llm-wiki/references/gotcha.md +16 -0
- package/skills/llm-wiki/references/invariant.md +20 -0
- package/skills/llm-wiki/references/module.md +30 -0
- package/src/config.ts +168 -0
- package/src/doctor.ts +171 -0
- package/src/extension.ts +1674 -0
- package/src/git.ts +69 -0
- package/src/grounding.ts +46 -0
- package/src/jev.ts +207 -0
- package/src/ledger.ts +85 -0
- package/src/lint.ts +407 -0
- package/src/metrics.ts +61 -0
- package/src/pipeline/adjudicate.ts +416 -0
- package/src/pipeline/capture.ts +109 -0
- package/src/pipeline/extract.ts +150 -0
- package/src/pipeline/write.ts +263 -0
- package/src/provenance.ts +137 -0
- package/src/redact.ts +46 -0
- package/src/review.ts +146 -0
- package/src/sessionlog.ts +107 -0
- package/src/structure.ts +215 -0
- package/src/sync.ts +295 -0
- package/src/wiki/frontmatter.ts +164 -0
- package/src/wiki/layout.ts +126 -0
- package/src/wiki/links.ts +17 -0
- package/src/wiki/lock.ts +86 -0
- package/src/wiki/search.ts +263 -0
- package/src/wiki/toc.ts +198 -0
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Writer pipeline: turn accepted claims into wiki pages or drafts.
|
|
3
|
+
*
|
|
4
|
+
* The model writes prose only; code assembles frontmatter from the adjudication
|
|
5
|
+
* results, so status/support/evidence cannot be hallucinated. Mode is adaptive:
|
|
6
|
+
* high-criticality claims downgrade auto -> draft -> guided.
|
|
7
|
+
*/
|
|
8
|
+
import { existsSync } from "node:fs";
|
|
9
|
+
import { readFile } from "node:fs/promises";
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
12
|
+
import type { ResolvedConfig, WriterMode } from "../config.ts";
|
|
13
|
+
import { checkLiterals } from "../grounding.ts";
|
|
14
|
+
import { redact } from "../redact.ts";
|
|
15
|
+
import { appendLedger } from "../ledger.ts";
|
|
16
|
+
import { readPage, todayISO, writePage, writeTextAtomic, type WikiLayout } from "../wiki/layout.ts";
|
|
17
|
+
import { appendLog, entryFromPage, updateIndex, upsertEntries, type TocEntry } from "../wiki/toc.ts";
|
|
18
|
+
|
|
19
|
+
export interface WriterClaim {
|
|
20
|
+
text: string;
|
|
21
|
+
kind?: string;
|
|
22
|
+
pageType?: string;
|
|
23
|
+
topic?: string;
|
|
24
|
+
target?: string;
|
|
25
|
+
trustTier?: string;
|
|
26
|
+
files: string[];
|
|
27
|
+
evidence: string[];
|
|
28
|
+
grounded: number;
|
|
29
|
+
criticality: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface WriterPlan {
|
|
33
|
+
title: string;
|
|
34
|
+
topic: string;
|
|
35
|
+
sourcePath?: string;
|
|
36
|
+
claims: WriterClaim[];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface WriterResult {
|
|
40
|
+
mode: WriterMode;
|
|
41
|
+
written: string[];
|
|
42
|
+
drafted: string[];
|
|
43
|
+
groups: number;
|
|
44
|
+
flagged: Array<{ page: string; missing: string[] }>;
|
|
45
|
+
usage: { input_tokens: number; output_tokens: number };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function resolveWriterMode(requested: WriterMode, claims: WriterClaim[], config: ResolvedConfig): WriterMode {
|
|
49
|
+
const worst = claims.reduce((max, claim) => Math.max(max, claim.criticality), 0);
|
|
50
|
+
let mode = requested;
|
|
51
|
+
if (worst >= config.review.escalateCriticality) {
|
|
52
|
+
if (mode === "auto") mode = "draft";
|
|
53
|
+
else if (mode === "draft") mode = "guided";
|
|
54
|
+
}
|
|
55
|
+
if (worst >= 0.95) mode = "guided";
|
|
56
|
+
return mode;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
interface WriterGroup {
|
|
60
|
+
target?: string;
|
|
61
|
+
pageType: string;
|
|
62
|
+
topic: string;
|
|
63
|
+
claims: WriterClaim[];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function groupClaims(plan: WriterPlan): WriterGroup[] {
|
|
67
|
+
const groups: WriterGroup[] = [];
|
|
68
|
+
for (const claim of plan.claims) {
|
|
69
|
+
const key = claim.target ?? `${claim.pageType ?? "concept"}:${claim.topic ?? plan.topic}`;
|
|
70
|
+
let group = groups.find((candidate) => (candidate.target ?? `${candidate.pageType}:${candidate.topic}`) === key);
|
|
71
|
+
if (!group) {
|
|
72
|
+
group = {
|
|
73
|
+
target: claim.target,
|
|
74
|
+
pageType: claim.pageType ?? "concept",
|
|
75
|
+
topic: claim.topic ?? plan.topic,
|
|
76
|
+
claims: [],
|
|
77
|
+
};
|
|
78
|
+
groups.push(group);
|
|
79
|
+
}
|
|
80
|
+
group.claims.push(claim);
|
|
81
|
+
}
|
|
82
|
+
return groups;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function stripFences(text: string): string {
|
|
86
|
+
const trimmed = text.trim();
|
|
87
|
+
return trimmed.startsWith("```") ? trimmed.replace(/^```[a-z]*\s*/i, "").replace(/```\s*$/, "") : trimmed;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function parseWriterJson(text: string): { title?: string; summary?: string; tags?: string[]; body?: string } | undefined {
|
|
91
|
+
const cleaned = stripFences(text);
|
|
92
|
+
const start = cleaned.indexOf("{");
|
|
93
|
+
const end = cleaned.lastIndexOf("}");
|
|
94
|
+
if (start === -1 || end <= start) return undefined;
|
|
95
|
+
try {
|
|
96
|
+
return JSON.parse(cleaned.slice(start, end + 1)) as { title?: string; summary?: string; tags?: string[]; body?: string };
|
|
97
|
+
} catch {
|
|
98
|
+
return undefined;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const WRITER_RULES = `You maintain a project wiki that gives coding agents the mental model of a system.
|
|
103
|
+
Write the page body in markdown: concrete, decision-oriented, no code blocks unless essential, no frontmatter.
|
|
104
|
+
Use short sections (Responsibility, Invariants, Change impact, See also) as appropriate.
|
|
105
|
+
Every factual statement must come from the supplied claims and evidence. Do not invent file paths, numbers, or behavior.
|
|
106
|
+
Prefer updating an existing page over restating it: merge the new claims into the existing text, keeping it coherent.
|
|
107
|
+
Return ONLY JSON: { "title": "...", "summary": "one line", "tags": ["..."], "body": "markdown body" }`;
|
|
108
|
+
|
|
109
|
+
function buildUserPrompt(group: WriterGroup, plan: WriterPlan, existing: { body: string; data: Record<string, unknown> } | undefined, sourceExcerpt: string): string {
|
|
110
|
+
const claims = group.claims.map((claim) => `- (${claim.trustTier ?? "source_document"}) ${claim.text}`).join("\n");
|
|
111
|
+
const files = [...new Set(group.claims.flatMap((claim) => claim.files))];
|
|
112
|
+
const evidence = [...new Set(group.claims.flatMap((claim) => claim.evidence))];
|
|
113
|
+
return [
|
|
114
|
+
`Source title: ${plan.title}`,
|
|
115
|
+
`Page type: ${group.pageType}`,
|
|
116
|
+
`Topic: ${group.topic}`,
|
|
117
|
+
files.length ? `Related code files: ${files.join(", ")}` : "",
|
|
118
|
+
evidence.length ? `Evidence: ${evidence.join(", ")}` : "",
|
|
119
|
+
existing ? `\nExisting page to update (keep what is still true, integrate the claims):\n---\n${existing.body.slice(0, 6000)}\n---` : "",
|
|
120
|
+
`\nClaims to file:\n${claims}`,
|
|
121
|
+
sourceExcerpt ? `\nSource material (may be truncated):\n---\n${sourceExcerpt}\n---` : "",
|
|
122
|
+
].filter(Boolean).join("\n");
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async function callWriter(
|
|
126
|
+
ctx: ExtensionContext,
|
|
127
|
+
group: WriterGroup,
|
|
128
|
+
plan: WriterPlan,
|
|
129
|
+
existing: { body: string; data: Record<string, unknown> } | undefined,
|
|
130
|
+
sourceExcerpt: string,
|
|
131
|
+
): Promise<{ result?: { title?: string; summary?: string; tags?: string[]; body?: string }; usage: { input_tokens: number; output_tokens: number } }> {
|
|
132
|
+
const model = ctx.model;
|
|
133
|
+
if (!model) return { usage: { input_tokens: 0, output_tokens: 0 } };
|
|
134
|
+
const response = await ctx.modelRegistry.complete(
|
|
135
|
+
model,
|
|
136
|
+
{
|
|
137
|
+
systemPrompt: WRITER_RULES,
|
|
138
|
+
messages: [{ role: "user", content: [{ type: "text", text: buildUserPrompt(group, plan, existing, sourceExcerpt) }], timestamp: Date.now() }],
|
|
139
|
+
},
|
|
140
|
+
{ maxTokens: 6000, signal: ctx.signal, cacheRetention: "none" },
|
|
141
|
+
);
|
|
142
|
+
const text = response.content
|
|
143
|
+
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
|
144
|
+
.map((part) => part.text)
|
|
145
|
+
.join("\n");
|
|
146
|
+
return {
|
|
147
|
+
result: parseWriterJson(text),
|
|
148
|
+
usage: response.usage
|
|
149
|
+
? { input_tokens: response.usage.input ?? 0, output_tokens: response.usage.output ?? 0 }
|
|
150
|
+
: { input_tokens: 0, output_tokens: 0 },
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export async function writeAcceptedPages(
|
|
155
|
+
ctx: ExtensionContext,
|
|
156
|
+
layout: WikiLayout,
|
|
157
|
+
config: ResolvedConfig,
|
|
158
|
+
plan: WriterPlan,
|
|
159
|
+
requestedMode: WriterMode,
|
|
160
|
+
): Promise<WriterResult> {
|
|
161
|
+
const mode = resolveWriterMode(requestedMode, plan.claims, config);
|
|
162
|
+
const usage = { input_tokens: 0, output_tokens: 0 };
|
|
163
|
+
const flagged: Array<{ page: string; missing: string[] }> = [];
|
|
164
|
+
if (mode === "guided" || plan.claims.length === 0) {
|
|
165
|
+
return { mode, written: [], drafted: [], groups: 0, flagged, usage };
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const groups = groupClaims(plan);
|
|
169
|
+
const sourceExcerpt =
|
|
170
|
+
plan.sourcePath && existsSync(join(layout.root, plan.sourcePath))
|
|
171
|
+
? redact((await readFile(join(layout.root, plan.sourcePath), "utf8")).slice(0, 12_000)).text
|
|
172
|
+
: "";
|
|
173
|
+
|
|
174
|
+
const written: string[] = [];
|
|
175
|
+
const drafted: string[] = [];
|
|
176
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
177
|
+
const draftDir = join(layout.stateDir, "drafts", stamp);
|
|
178
|
+
|
|
179
|
+
for (const group of groups) {
|
|
180
|
+
const existing = group.target && existsSync(join(layout.wikiDir, group.target)) ? await readPage(join(layout.wikiDir, group.target)) : undefined;
|
|
181
|
+
|
|
182
|
+
const { result, usage: callUsage } = await callWriter(ctx, group, plan, existing, sourceExcerpt);
|
|
183
|
+
usage.input_tokens += callUsage.input_tokens;
|
|
184
|
+
usage.output_tokens += callUsage.output_tokens;
|
|
185
|
+
if (!result?.body) continue;
|
|
186
|
+
|
|
187
|
+
const typePrefix = group.pageType.includes("/") ? group.pageType.split("/")[1] : group.pageType;
|
|
188
|
+
const finalRel = group.target ?? `${group.topic}/${typePrefix}-${slugPart(result.title ?? group.claims[0].text)}.md`;
|
|
189
|
+
const absolute = join(layout.wikiDir, finalRel);
|
|
190
|
+
|
|
191
|
+
const claimRecords = group.claims.map((claim, index) => ({
|
|
192
|
+
id: `c${index + 1}`,
|
|
193
|
+
text: claim.text,
|
|
194
|
+
status: claim.trustTier === "user_stated" ? "user-stated" : "verified",
|
|
195
|
+
support: Number(claim.grounded.toFixed(2)),
|
|
196
|
+
evidence: claim.evidence.length > 0 ? claim.evidence : plan.sourcePath ? [plan.sourcePath] : [],
|
|
197
|
+
}));
|
|
198
|
+
const evidenceForCheck = [
|
|
199
|
+
group.claims.map((claim) => claim.text).join("\n"),
|
|
200
|
+
group.claims.flatMap((claim) => claim.evidence).join("\n"),
|
|
201
|
+
sourceExcerpt,
|
|
202
|
+
].join("\n");
|
|
203
|
+
const grounding = checkLiterals(result.body, evidenceForCheck);
|
|
204
|
+
if (grounding.missing.length > 0) {
|
|
205
|
+
flagged.push({ page: finalRel, missing: grounding.missing.slice(0, 5) });
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const data: Record<string, unknown> = {
|
|
209
|
+
title: result.title ?? existing?.data.title ?? plan.title,
|
|
210
|
+
type: group.pageType,
|
|
211
|
+
topic: group.topic,
|
|
212
|
+
summary: result.summary ?? existing?.data.summary ?? "",
|
|
213
|
+
tags: Array.isArray(result.tags) ? result.tags.map(String).slice(0, 8) : [],
|
|
214
|
+
updated: todayISO(),
|
|
215
|
+
sources: [...new Set([...(Array.isArray(existing?.data.sources) ? existing!.data.sources.map(String) : []), ...(plan.sourcePath ? [plan.sourcePath] : [])])],
|
|
216
|
+
files: [...new Set([...(Array.isArray(existing?.data.files) ? existing!.data.files.map(String) : []), ...group.claims.flatMap((claim) => claim.files)])],
|
|
217
|
+
claims: [...(Array.isArray(existing?.data.claims) ? (existing!.data.claims as unknown[]) : []), ...claimRecords],
|
|
218
|
+
...(grounding.missing.length > 0 ? { needs_review: true } : {}),
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
if (mode === "draft") {
|
|
222
|
+
const draftPath = join(draftDir, finalRel);
|
|
223
|
+
await writePage(draftPath, data, result.body);
|
|
224
|
+
drafted.push(finalRel);
|
|
225
|
+
} else {
|
|
226
|
+
await writePage(absolute, data, result.body);
|
|
227
|
+
written.push(finalRel);
|
|
228
|
+
}
|
|
229
|
+
await appendLedger(layout, {
|
|
230
|
+
actor: "agent",
|
|
231
|
+
op: "wiki.write",
|
|
232
|
+
subject: finalRel,
|
|
233
|
+
action: mode,
|
|
234
|
+
reason: `${group.claims.length} accepted claim(s)`,
|
|
235
|
+
verdict: { mode, claims: group.claims.length, grounded: group.claims.map((claim) => claim.grounded), ungroundedLiterals: grounding.missing },
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
if (written.length > 0) {
|
|
240
|
+
const updates: TocEntry[] = [];
|
|
241
|
+
for (const rel of written) {
|
|
242
|
+
const page = await readPage(join(layout.wikiDir, rel));
|
|
243
|
+
updates.push(entryFromPage(rel, page.data));
|
|
244
|
+
}
|
|
245
|
+
await updateIndex(layout, (entries) => upsertEntries(entries, updates));
|
|
246
|
+
await appendLog(layout, "write", `${written.length} page(s) (${mode})`, written.map((rel) => `Written: ${rel}`));
|
|
247
|
+
}
|
|
248
|
+
if (drafted.length > 0) {
|
|
249
|
+
await writeTextAtomic(join(draftDir, "manifest.json"), `${JSON.stringify({ created: stamp, plan: plan.title, pages: drafted }, null, "\t")}\n`);
|
|
250
|
+
await appendLog(layout, "draft", `${drafted.length} page(s)`, drafted.map((rel) => `Draft: ${rel}`));
|
|
251
|
+
}
|
|
252
|
+
return { mode, written, drafted, groups: groups.length, flagged, usage };
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function slugPart(text: string): string {
|
|
256
|
+
return text
|
|
257
|
+
.toLowerCase()
|
|
258
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
259
|
+
.replace(/^-+|-+$/g, "")
|
|
260
|
+
.split("-")
|
|
261
|
+
.slice(0, 5)
|
|
262
|
+
.join("-");
|
|
263
|
+
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claim provenance: corroboration counting and supersession.
|
|
3
|
+
* These are the compounding mechanics — repeated insight strengthens a claim,
|
|
4
|
+
* newer knowledge retires it with a link instead of deleting it.
|
|
5
|
+
*/
|
|
6
|
+
import { existsSync } from "node:fs";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { readPage, todayISO, writePage, type WikiLayout } from "./wiki/layout.ts";
|
|
9
|
+
|
|
10
|
+
export interface ReinforcementResult {
|
|
11
|
+
page: string;
|
|
12
|
+
claimId?: string;
|
|
13
|
+
corroborations: number;
|
|
14
|
+
support: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface SupersessionResult {
|
|
18
|
+
page: string;
|
|
19
|
+
claimId?: string;
|
|
20
|
+
supersededText: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function tokens(text: string): Set<string> {
|
|
24
|
+
return new Set(
|
|
25
|
+
text
|
|
26
|
+
.toLowerCase()
|
|
27
|
+
.split(/[^a-z0-9_]+/)
|
|
28
|
+
.filter((token) => token.length > 3),
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function similarity(a: Set<string>, b: Set<string>): number {
|
|
33
|
+
if (a.size === 0 || b.size === 0) return 0;
|
|
34
|
+
let shared = 0;
|
|
35
|
+
for (const token of a) if (b.has(token)) shared++;
|
|
36
|
+
return shared / Math.min(a.size, b.size);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function findClaim(claims: Record<string, unknown>[], text: string, minimum = 0.5): number {
|
|
40
|
+
const needle = tokens(text);
|
|
41
|
+
let bestIndex = -1;
|
|
42
|
+
let bestScore = 0;
|
|
43
|
+
claims.forEach((claim, index) => {
|
|
44
|
+
if (typeof claim.text !== "string") return;
|
|
45
|
+
const status = String(claim.status ?? "verified");
|
|
46
|
+
if (status === "rejected" || status === "superseded") return;
|
|
47
|
+
const score = similarity(tokens(claim.text), needle);
|
|
48
|
+
if (score > bestScore) {
|
|
49
|
+
bestScore = score;
|
|
50
|
+
bestIndex = index;
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
return bestScore >= minimum ? bestIndex : -1;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Increment corroboration for the best-matching claim on a page. */
|
|
57
|
+
export async function applyReinforcement(
|
|
58
|
+
layout: WikiLayout,
|
|
59
|
+
pageRel: string,
|
|
60
|
+
text: string,
|
|
61
|
+
options?: { evidence?: string[] },
|
|
62
|
+
): Promise<ReinforcementResult | undefined> {
|
|
63
|
+
const pagePath = join(layout.wikiDir, pageRel);
|
|
64
|
+
if (!existsSync(pagePath)) return undefined;
|
|
65
|
+
const page = await readPage(pagePath);
|
|
66
|
+
const claims = Array.isArray(page.data.claims) ? (page.data.claims as Record<string, unknown>[]) : [];
|
|
67
|
+
const index = findClaim(claims, text);
|
|
68
|
+
if (index === -1) return undefined;
|
|
69
|
+
const claim = claims[index];
|
|
70
|
+
const corroborations = Number(claim.corroborations ?? 1) + 1;
|
|
71
|
+
claim.corroborations = corroborations;
|
|
72
|
+
claim.last_confirmed = todayISO();
|
|
73
|
+
claim.support = Math.min(0.99, Math.max(Number(claim.support ?? 0.8), 0.85 + 0.02 * corroborations));
|
|
74
|
+
if (options?.evidence?.length) {
|
|
75
|
+
const evidence = Array.isArray(claim.evidence) ? claim.evidence.map(String) : [];
|
|
76
|
+
for (const item of options.evidence) if (!evidence.includes(item)) evidence.push(item);
|
|
77
|
+
claim.evidence = evidence;
|
|
78
|
+
}
|
|
79
|
+
claims[index] = claim;
|
|
80
|
+
page.data.claims = claims;
|
|
81
|
+
page.data.updated = todayISO();
|
|
82
|
+
await writePage(pagePath, page.data, page.body);
|
|
83
|
+
return {
|
|
84
|
+
page: pageRel,
|
|
85
|
+
claimId: typeof claim.id === "string" ? claim.id : undefined,
|
|
86
|
+
corroborations,
|
|
87
|
+
support: Number(claim.support),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Mark the best-matching claim on a page as superseded by newer knowledge. */
|
|
92
|
+
export async function applySupersession(
|
|
93
|
+
layout: WikiLayout,
|
|
94
|
+
pageRel: string,
|
|
95
|
+
byText: string,
|
|
96
|
+
): Promise<SupersessionResult | undefined> {
|
|
97
|
+
const pagePath = join(layout.wikiDir, pageRel);
|
|
98
|
+
if (!existsSync(pagePath)) return undefined;
|
|
99
|
+
const page = await readPage(pagePath);
|
|
100
|
+
const claims = Array.isArray(page.data.claims) ? (page.data.claims as Record<string, unknown>[]) : [];
|
|
101
|
+
// find the claim that the new text supersedes: highest overlap with the new text
|
|
102
|
+
const index = findClaim(claims, byText, 0.35);
|
|
103
|
+
if (index === -1) return undefined;
|
|
104
|
+
const claim = claims[index];
|
|
105
|
+
const supersededText = String(claim.text ?? "");
|
|
106
|
+
claim.status = "superseded";
|
|
107
|
+
claim.superseded_by = byText.slice(0, 100);
|
|
108
|
+
claim.superseded_at = todayISO();
|
|
109
|
+
claims[index] = claim;
|
|
110
|
+
page.data.claims = claims;
|
|
111
|
+
page.data.updated = todayISO();
|
|
112
|
+
await writePage(pagePath, page.data, page.body);
|
|
113
|
+
return {
|
|
114
|
+
page: pageRel,
|
|
115
|
+
claimId: typeof claim.id === "string" ? claim.id : undefined,
|
|
116
|
+
supersededText,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Best-matching candidate claim page for a relation verdict. */
|
|
121
|
+
export function bestCandidatePage(
|
|
122
|
+
candidates: Array<{ text: string; page?: string }>,
|
|
123
|
+
text: string,
|
|
124
|
+
minimum = 0.4,
|
|
125
|
+
): string | undefined {
|
|
126
|
+
let best: string | undefined;
|
|
127
|
+
let bestScore = 0;
|
|
128
|
+
for (const candidate of candidates) {
|
|
129
|
+
if (!candidate.page) continue;
|
|
130
|
+
const score = similarity(tokens(candidate.text), tokens(text));
|
|
131
|
+
if (score > bestScore) {
|
|
132
|
+
bestScore = score;
|
|
133
|
+
best = candidate.page;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return bestScore >= minimum ? best : undefined;
|
|
137
|
+
}
|
package/src/redact.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Redaction: best-effort secret/PII stripping before content is written to raw/
|
|
3
|
+
* or sent to a model. Redaction is defensive, not a guarantee — regex cannot
|
|
4
|
+
* catch everything, and it is logged so the ledger shows what was removed.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export interface RedactionFinding {
|
|
8
|
+
type: string;
|
|
9
|
+
count: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface RedactionResult {
|
|
13
|
+
text: string;
|
|
14
|
+
findings: RedactionFinding[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const PATTERNS: Array<{ type: string; regex: RegExp; replacement: string }> = [
|
|
18
|
+
{ type: "private_key", regex: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, replacement: "[REDACTED_PRIVATE_KEY]" },
|
|
19
|
+
{ type: "openai_key", regex: /\bsk-(?!or-v1-|ant-)[A-Za-z0-9_-]{20,}\b/g, replacement: "[REDACTED_OPENAI_KEY]" },
|
|
20
|
+
{ type: "openrouter_key", regex: /\bsk-or-v1-[A-Za-z0-9]{20,}\b/g, replacement: "[REDACTED_OPENROUTER_KEY]" },
|
|
21
|
+
{ type: "anthropic_key", regex: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g, replacement: "[REDACTED_ANTHROPIC_KEY]" },
|
|
22
|
+
{ type: "github_token", regex: /\bgh[pousr]_[A-Za-z0-9]{20,}\b/g, replacement: "[REDACTED_GITHUB_TOKEN]" },
|
|
23
|
+
{ type: "aws_key", regex: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g, replacement: "[REDACTED_AWS_KEY]" },
|
|
24
|
+
{ type: "bearer_token", regex: /\bBearer\s+[A-Za-z0-9._~+/-]{24,}=*/g, replacement: "Bearer [REDACTED_TOKEN]" },
|
|
25
|
+
{ type: "jwt", regex: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, replacement: "[REDACTED_JWT]" },
|
|
26
|
+
{ type: "slack_token", regex: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g, replacement: "[REDACTED_SLACK_TOKEN]" },
|
|
27
|
+
{
|
|
28
|
+
type: "generic_secret_assignment",
|
|
29
|
+
regex: /\b(?:api[_-]?key|secret|password|passwd|token)\s*[:=]\s*["']?[A-Za-z0-9_./+-]{16,}["']?/gi,
|
|
30
|
+
replacement: "[REDACTED_SECRET]",
|
|
31
|
+
},
|
|
32
|
+
{ type: "email", regex: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g, replacement: "[REDACTED_EMAIL]" },
|
|
33
|
+
];
|
|
34
|
+
|
|
35
|
+
/** Redact secrets and PII, returning what was found. */
|
|
36
|
+
export function redact(text: string): RedactionResult {
|
|
37
|
+
let output = text;
|
|
38
|
+
const findings: RedactionFinding[] = [];
|
|
39
|
+
for (const pattern of PATTERNS) {
|
|
40
|
+
const matches = output.match(pattern.regex);
|
|
41
|
+
if (!matches || matches.length === 0) continue;
|
|
42
|
+
findings.push({ type: pattern.type, count: matches.length });
|
|
43
|
+
output = output.replace(pattern.regex, pattern.replacement);
|
|
44
|
+
}
|
|
45
|
+
return { text: output, findings };
|
|
46
|
+
}
|
package/src/review.ts
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Review queue: items that need a decision (low confidence, needs recheck, disputes).
|
|
3
|
+
* The agent works the queue via the wiki_review tool; the user is escalated only for
|
|
4
|
+
* critical items. Code applies resolutions to page frontmatter.
|
|
5
|
+
*/
|
|
6
|
+
import { randomBytes } from "node:crypto";
|
|
7
|
+
import { existsSync } from "node:fs";
|
|
8
|
+
import { readFile } from "node:fs/promises";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
import { readPage, todayISO, writePage, writeTextAtomic, type WikiLayout } from "./wiki/layout.ts";
|
|
11
|
+
import { withWikiLock } from "./wiki/lock.ts";
|
|
12
|
+
|
|
13
|
+
export type ReviewKind = "claim_review" | "needs_recheck" | "dispute";
|
|
14
|
+
export type ReviewStatus = "open" | "resolved" | "deferred";
|
|
15
|
+
export type ReviewResolution = "accept" | "reject" | "supersede" | "defer";
|
|
16
|
+
|
|
17
|
+
export interface ReviewItem {
|
|
18
|
+
id: string;
|
|
19
|
+
ts: string;
|
|
20
|
+
kind: ReviewKind;
|
|
21
|
+
status: ReviewStatus;
|
|
22
|
+
claimText: string;
|
|
23
|
+
page?: string;
|
|
24
|
+
claimId?: string;
|
|
25
|
+
criticality: number;
|
|
26
|
+
reason?: string;
|
|
27
|
+
verdicts?: unknown;
|
|
28
|
+
resolution?: ReviewResolution;
|
|
29
|
+
note?: string;
|
|
30
|
+
resolvedAt?: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function newId(): string {
|
|
34
|
+
return `${Date.now().toString(36)}-${randomBytes(3).toString("hex")}`;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export async function readReviews(layout: WikiLayout): Promise<ReviewItem[]> {
|
|
38
|
+
if (!existsSync(layout.reviewQueuePath)) return [];
|
|
39
|
+
const text = await readFile(layout.reviewQueuePath, "utf8");
|
|
40
|
+
return text
|
|
41
|
+
.split(/\r?\n/)
|
|
42
|
+
.filter(Boolean)
|
|
43
|
+
.map((line) => {
|
|
44
|
+
try {
|
|
45
|
+
return JSON.parse(line) as ReviewItem;
|
|
46
|
+
} catch {
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
})
|
|
50
|
+
.filter((item): item is ReviewItem => Boolean(item));
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export async function writeReviews(layout: WikiLayout, items: ReviewItem[]): Promise<void> {
|
|
54
|
+
const body = items.map((item) => JSON.stringify(item)).join("\n");
|
|
55
|
+
await writeTextAtomic(layout.reviewQueuePath, body ? `${body}\n` : "");
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export async function openReviewCount(layout: WikiLayout): Promise<number> {
|
|
59
|
+
return (await readReviews(layout)).filter((item) => item.status === "open").length;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export async function enqueueReview(
|
|
63
|
+
layout: WikiLayout,
|
|
64
|
+
item: Omit<ReviewItem, "id" | "ts" | "status">,
|
|
65
|
+
): Promise<ReviewItem | undefined> {
|
|
66
|
+
return withWikiLock(layout, async () => {
|
|
67
|
+
const items = await readReviews(layout);
|
|
68
|
+
const duplicate = items.find(
|
|
69
|
+
(existing) =>
|
|
70
|
+
existing.status === "open" &&
|
|
71
|
+
existing.kind === item.kind &&
|
|
72
|
+
existing.page === item.page &&
|
|
73
|
+
existing.claimText === item.claimText,
|
|
74
|
+
);
|
|
75
|
+
if (duplicate) return undefined;
|
|
76
|
+
const record: ReviewItem = { id: newId(), ts: new Date().toISOString(), status: "open", ...item };
|
|
77
|
+
items.push(record);
|
|
78
|
+
await writeReviews(layout, items);
|
|
79
|
+
return record;
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export async function listOpenReviews(layout: WikiLayout, limit = 20): Promise<ReviewItem[]> {
|
|
84
|
+
const items = await readReviews(layout);
|
|
85
|
+
const order: Record<ReviewKind, number> = { dispute: 0, needs_recheck: 1, claim_review: 2 };
|
|
86
|
+
return items
|
|
87
|
+
.filter((item) => item.status === "open")
|
|
88
|
+
.sort((a, b) => order[a.kind] - order[b.kind] || b.criticality - a.criticality)
|
|
89
|
+
.slice(0, limit);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export async function resolveReview(
|
|
93
|
+
layout: WikiLayout,
|
|
94
|
+
id: string,
|
|
95
|
+
resolution: ReviewResolution,
|
|
96
|
+
note?: string,
|
|
97
|
+
): Promise<ReviewItem | undefined> {
|
|
98
|
+
return withWikiLock(layout, async () => {
|
|
99
|
+
const items = await readReviews(layout);
|
|
100
|
+
const item = items.find((candidate) => candidate.id === id);
|
|
101
|
+
if (!item) return undefined;
|
|
102
|
+
item.status = resolution === "defer" ? "deferred" : "resolved";
|
|
103
|
+
item.resolution = resolution;
|
|
104
|
+
item.note = note;
|
|
105
|
+
item.resolvedAt = new Date().toISOString();
|
|
106
|
+
await writeReviews(layout, items);
|
|
107
|
+
return item;
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Apply a resolution to the claim in its page frontmatter.
|
|
113
|
+
* Returns a status message; a missing page/claim is not fatal (the agent may handle it).
|
|
114
|
+
*/
|
|
115
|
+
export async function applyReviewResolution(
|
|
116
|
+
layout: WikiLayout,
|
|
117
|
+
item: ReviewItem,
|
|
118
|
+
resolution: ReviewResolution,
|
|
119
|
+
): Promise<string> {
|
|
120
|
+
if (!item.page) return "no page attached; resolution recorded only";
|
|
121
|
+
const pagePath = join(layout.wikiDir, item.page);
|
|
122
|
+
if (!existsSync(pagePath)) return `page not found: ${item.page}`;
|
|
123
|
+
const page = await readPage(pagePath);
|
|
124
|
+
const claims = Array.isArray(page.data.claims) ? (page.data.claims as Record<string, unknown>[]) : [];
|
|
125
|
+
const index = claims.findIndex(
|
|
126
|
+
(claim) => (item.claimId && claim.id === item.claimId) || claim.text === item.claimText,
|
|
127
|
+
);
|
|
128
|
+
if (index === -1) return `claim not found on ${item.page}`;
|
|
129
|
+
const claim = claims[index];
|
|
130
|
+
if (resolution === "accept") {
|
|
131
|
+
if (claim.status !== "user-stated") {
|
|
132
|
+
claim.status = "verified";
|
|
133
|
+
claim.support = Math.max(Number(claim.support ?? 0), 0.8);
|
|
134
|
+
}
|
|
135
|
+
} else if (resolution === "reject") {
|
|
136
|
+
claim.status = "rejected";
|
|
137
|
+
} else if (resolution === "supersede") {
|
|
138
|
+
claim.status = "superseded";
|
|
139
|
+
}
|
|
140
|
+
claim.reviewed = todayISO();
|
|
141
|
+
claims[index] = claim;
|
|
142
|
+
page.data.claims = claims;
|
|
143
|
+
page.data.updated = todayISO();
|
|
144
|
+
await writePage(pagePath, page.data, page.body);
|
|
145
|
+
return `${resolution} applied to ${item.page}${item.claimId ? `#${item.claimId}` : ""}`;
|
|
146
|
+
}
|