pi-jev-wiki 0.2.0

package/src/sessionlog.ts

@@ -0,0 +1,107 @@
+/**
+ * Session log: every captured candidate with its verdict, so recurrence can be
+ * counted in code. Recurring rejected candidates are promoted to the review queue.
+ */
+import { existsSync } from "node:fs";
+import { appendFile, readFile } from "node:fs/promises";
+import type { WikiLayout } from "./wiki/layout.ts";
+import { enqueueReview } from "./review.ts";
+
+export interface SessionLogEntry {
+	ts: string;
+	text: string;
+	kind?: string;
+	source: "tool" | "compact" | "settled";
+	action: string;
+	reason?: string;
+	grounded?: number;
+	derivable?: number;
+	importance?: number;
+}
+
+export async function appendSessionLog(layout: WikiLayout, entry: Omit<SessionLogEntry, "ts">): Promise<void> {
+	try {
+		await appendFile(layout.sessionLogPath, `${JSON.stringify({ ts: new Date().toISOString(), ...entry })}\n`, "utf8");
+	} catch {
+		/* session log is best-effort */
+	}
+}
+
+export async function readSessionLog(layout: WikiLayout): Promise<SessionLogEntry[]> {
+	if (!existsSync(layout.sessionLogPath)) return [];
+	const text = await readFile(layout.sessionLogPath, "utf8");
+	return text
+		.split(/\r?\n/)
+		.filter(Boolean)
+		.map((line) => {
+			try {
+				return JSON.parse(line) as SessionLogEntry;
+			} catch {
+				return undefined;
+			}
+		})
+		.filter((entry): entry is SessionLogEntry => Boolean(entry));
+}
+
+function tokens(text: string): Set<string> {
+	return new Set(
+		text
+			.toLowerCase()
+			.split(/[^a-z0-9_]+/)
+			.filter((token) => token.length > 3),
+	);
+}
+
+function similarity(a: Set<string>, b: Set<string>): number {
+	if (a.size === 0 || b.size === 0) return 0;
+	let shared = 0;
+	for (const token of a) if (b.has(token)) shared++;
+	return shared / Math.min(a.size, b.size);
+}
+
+/** How many prior log entries describe substantially the same candidate. */
+export function countRecurrence(entries: SessionLogEntry[], text: string, threshold = 0.7): number {
+	const needle = tokens(text);
+	return entries.filter((entry) => similarity(tokens(entry.text), needle) >= threshold).length - 1;
+}
+
+/**
+ * Promote candidates that keep recurring after rejection. A fact that appears in
+ * three sessions but never gets filed is usually evidence of a documentation gap
+ * or a claim that needs a stronger artifact.
+ */
+export async function promoteRecurring(layout: WikiLayout, minimum = 3): Promise<number> {
+	const entries = await readSessionLog(layout);
+	const groups: Array<{ text: string; entries: SessionLogEntry[] }> = [];
+	for (const entry of entries) {
+		const group = groups.find((candidate) => similarity(tokens(candidate.text), tokens(entry.text)) >= 0.7);
+		if (group) group.entries.push(entry);
+		else groups.push({ text: entry.text, entries: [entry] });
+	}
+
+	let promoted = 0;
+	for (const group of groups) {
+		if (group.entries.length < minimum) continue;
+		const accepted = group.entries.some((entry) => ["file", "reinforce", "file_user_stated"].includes(entry.action));
+		if (accepted) continue;
+		const latest = group.entries[group.entries.length - 1];
+		const alreadyQueued = group.entries.some((entry) => entry.action === "promoted");
+		if (alreadyQueued) continue;
+		await enqueueReview(layout, {
+			kind: "claim_review",
+			claimText: latest.text,
+			criticality: 0.5,
+			reason: `recurred ${group.entries.length} times without filing; needs a stronger artifact or a decision`,
+			verdicts: { recurrences: group.entries.length, lastAction: latest.action },
+		});
+		await appendSessionLog(layout, {
+			text: latest.text,
+			kind: latest.kind,
+			source: latest.source,
+			action: "promoted",
+			reason: `recurred ${group.entries.length} times`,
+		});
+		promoted++;
+	}
+	return promoted;
+}

package/src/structure.ts

@@ -0,0 +1,215 @@
+/**
+ * Structure scan: a deterministic map of modules, dependency edges, entry points,
+ * and test surface — plus coverage checks against the wiki's architecture pages.
+ * No model calls: this is the reproducible substrate an agent can reason over.
+ */
+import { existsSync } from "node:fs";
+import { readdir, readFile } from "node:fs/promises";
+import { dirname, extname, join, relative, resolve } from "node:path";
+import { listMarkdownFiles, readPage, type WikiLayout } from "./wiki/layout.ts";
+import { readIndex } from "./wiki/toc.ts";
+
+export interface ModuleInfo {
+	dir: string;
+	files: number;
+	imports: string[];
+	importedBy: string[];
+	entry: boolean;
+}
+
+export interface StructureReport {
+	manifests: string[];
+	entryPoints: string[];
+	totalFiles: number;
+	testFiles: number;
+	modules: ModuleInfo[];
+	wiki: {
+		architecturePages: number;
+		undocumentedModules: string[];
+		staleFileReferences: string[];
+	};
+}
+
+const CODE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rs", ".java", ".rb"]);
+const SKIP_DIRS = new Set([
+	"node_modules",
+	".git",
+	"dist",
+	"build",
+	"out",
+	".next",
+	"coverage",
+	"vendor",
+	"target",
+	".venv",
+	"venv",
+	"__pycache__",
+	".jev-wiki",
+	".pi",
+]);
+const ENTRY_CANDIDATES = ["src/index.ts", "src/main.ts", "src/extension.ts", "src/index.js", "index.js", "main.py", "src/main.rs", "src/main.go"];
+const MAX_FILES = 6000;
+
+async function walk(root: string, dir: string, files: string[], depth = 0): Promise<void> {
+	if (files.length >= MAX_FILES || depth > 7) return;
+	const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
+	for (const entry of entries) {
+		const full = join(dir, entry.name);
+		if (entry.isDirectory()) {
+			if (SKIP_DIRS.has(entry.name) || entry.name.startsWith(".")) continue;
+			await walk(root, full, files, depth + 1);
+		} else if (entry.isFile() && CODE_EXTENSIONS.has(extname(entry.name))) {
+			files.push(full);
+		}
+	}
+}
+
+function moduleOf(root: string, file: string): string {
+	const rel = relative(root, file).split("\\").join("/");
+	const parts = rel.split("/");
+	if (parts.length === 1) return "(root)";
+	if (["src", "packages", "apps", "lib", "modules"].includes(parts[0]) && parts.length > 2) return `${parts[0]}/${parts[1]}`;
+	return parts[0];
+}
+
+function importSpecifiers(content: string, extension: string): string[] {
+	const specs: string[] = [];
+	if (extension === ".py") {
+		for (const match of content.matchAll(/^\s*(?:from|import)\s+([.\w/]+)/gm)) specs.push(match[1]);
+		return specs;
+	}
+	for (const match of content.matchAll(/from\s+["']([^"']+)["']|import\s+["']([^"']+)["']|require\(\s*["']([^"']+)["']\s*\)/g)) {
+		const spec = match[1] ?? match[2] ?? match[3];
+		if (spec) specs.push(spec);
+	}
+	return specs;
+}
+
+async function findManifests(root: string): Promise<string[]> {
+	const candidates = ["package.json", "pyproject.toml", "go.mod", "Cargo.toml", "requirements.txt", "pom.xml", "build.gradle"];
+	const found: string[] = [];
+	for (const candidate of candidates) if (existsSync(join(root, candidate))) found.push(candidate);
+	if (existsSync(join(root, "packages"))) {
+		const entries = await readdir(join(root, "packages"), { withFileTypes: true }).catch(() => []);
+		for (const entry of entries) {
+			if (entry.isDirectory() && existsSync(join(root, "packages", entry.name, "package.json"))) {
+				found.push(`packages/${entry.name}/package.json`);
+			}
+		}
+	}
+	return found;
+}
+
+export async function scanStructure(root: string, layout?: WikiLayout): Promise<StructureReport> {
+	const files: string[] = [];
+	await walk(root, root, files);
+
+	const moduleMap = new Map<string, { files: number; imports: Set<string>; importedBy: Set<string>; entry: boolean }>();
+	const entryPoints: string[] = [];
+	let testFiles = 0;
+
+	for (const file of files) {
+		const rel = relative(root, file).split("\\").join("/");
+		const moduleName = moduleOf(root, file);
+		const record = moduleMap.get(moduleName) ?? { files: 0, imports: new Set<string>(), importedBy: new Set<string>(), entry: false };
+		record.files++;
+		if (ENTRY_CANDIDATES.includes(rel) || /^src\/(cli|server|index)\.(ts|js|py)$/.test(rel)) {
+			record.entry = true;
+			entryPoints.push(rel);
+		}
+		if (/(^|[._-])(test|spec)\./i.test(rel) || rel.includes("/tests/") || rel.startsWith("tests/")) testFiles++;
+		moduleMap.set(moduleName, record);
+	}
+
+	// import edges (second pass, so all modules exist)
+	for (const file of files) {
+		const content = await readFile(file, "utf8").catch(() => "");
+		if (!content) continue;
+		if (content.length > 200_000) continue;
+		const from = moduleOf(root, file);
+		for (const spec of importSpecifiers(content, extname(file))) {
+			if (!spec.startsWith(".")) continue;
+			const target = resolve(dirname(file), spec);
+			const targetModule = moduleOf(root, target);
+			if (targetModule === from) continue;
+			const fromRecord = moduleMap.get(from);
+			const toRecord = moduleMap.get(targetModule);
+			if (fromRecord && toRecord) {
+				fromRecord.imports.add(targetModule);
+				toRecord.importedBy.add(from);
+			}
+		}
+	}
+
+	const manifests = await findManifests(root);
+	const modules: ModuleInfo[] = [...moduleMap.entries()]
+		.map(([dir, record]) => ({
+			dir,
+			files: record.files,
+			imports: [...record.imports].sort(),
+			importedBy: [...record.importedBy].sort(),
+			entry: record.entry,
+		}))
+		.sort((a, b) => b.files - a.files);
+
+	const wiki = { architecturePages: 0, undocumentedModules: [] as string[], staleFileReferences: [] as string[] };
+	if (layout && existsSync(layout.wikiDir)) {
+		const entries = await readIndex(layout);
+		const architecture = entries.filter((entry) => entry.type.startsWith("architecture/"));
+		wiki.architecturePages = architecture.length;
+		const documentedText = architecture.map((entry) => `${entry.title} ${entry.summary} ${entry.path}`.toLowerCase());
+		const documentedFiles: string[] = [];
+		for (const file of await listMarkdownFiles(layout.wikiDir)) {
+			if (file.endsWith("index.md") || file.endsWith("log.md")) continue;
+			const page = await readPage(file).catch(() => undefined);
+			if (!page) continue;
+			const referenced = Array.isArray(page.data.files) ? page.data.files.map(String) : [];
+			const pageType = String(page.data.type ?? "");
+			if (pageType.startsWith("architecture/")) documentedFiles.push(...referenced);
+			for (const ref of referenced) {
+				if (!existsSync(resolve(root, ref))) {
+					wiki.staleFileReferences.push(`${relative(layout.wikiDir, file).split("\\").join("/")} → ${ref}`);
+				}
+			}
+		}
+		for (const module of modules) {
+			if (module.dir === "(root)") continue;
+			const name = module.dir.split("/").pop()!.toLowerCase();
+			const prefix = `${module.dir}/`;
+			const documented =
+				documentedText.some((text) => text.includes(name)) || documentedFiles.some((ref) => ref === module.dir || ref.startsWith(prefix));
+			if (!documented) wiki.undocumentedModules.push(module.dir);
+		}
+	}
+
+	return { manifests, entryPoints, totalFiles: files.length, testFiles, modules, wiki };
+}
+
+/** Compact markdown rendering for the wiki_structure tool. */
+export function renderStructure(report: StructureReport): string {
+	const lines = [
+		"# Repository structure",
+		"",
+		`Files scanned: ${report.totalFiles} · test files: ${report.testFiles}`,
+		`Manifests: ${report.manifests.join(", ") || "(none)"}`,
+		`Entry points: ${report.entryPoints.join(", ") || "(none detected)"}`,
+		"",
+		"| Module | Files | Imports | Imported by | Entry |",
+		"|--------|-------|---------|-------------|-------|",
+	];
+	for (const module of report.modules.slice(0, 40)) {
+		lines.push(
+			`| \`${module.dir}\` | ${module.files} | ${module.imports.slice(0, 5).join(", ") || "—"} | ${module.importedBy.slice(0, 5).join(", ") || "—"} | ${module.entry ? "yes" : ""} |`,
+		);
+	}
+	if (report.wiki.architecturePages > 0 || report.wiki.undocumentedModules.length > 0) {
+		lines.push(
+			"",
+			`Architecture pages: ${report.wiki.architecturePages}`,
+			`Modules without an architecture page: ${report.wiki.undocumentedModules.join(", ") || "none"}`,
+			`Stale file references in wiki pages: ${report.wiki.staleFileReferences.length}`,
+			...report.wiki.staleFileReferences.slice(0, 10).map((ref) => `- ${ref}`),
+		);
+	}
+	return lines.join("\n");
+}

package/src/sync.ts

@@ -0,0 +1,295 @@
+/**
+ * Change-driven invalidation: diff since the last synced commit, ask Jev which
+ * file-linked claims are affected, and update their status (recheck/supersede/dispute).
+ */
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { ResolvedConfig } from "./config.ts";
+import { changedFiles, diffForFiles, fileMatches, headCommit, isGitRepo } from "./git.ts";
+import { choice, isChoice, isNoul, noul, type JevClient } from "./jev.ts";
+import { appendLedger } from "./ledger.ts";
+import { enqueueReview } from "./review.ts";
+import { listMarkdownFiles, readPage, todayISO, writePage, writeTextAtomic, type WikiLayout } from "./wiki/layout.ts";
+import { appendLog } from "./wiki/toc.ts";
+
+export interface SyncState {
+	lastSyncCommit?: string;
+	lastSyncAt?: string;
+}
+
+export interface MappedClaim {
+	page: string;
+	pagePath: string;
+	claimId?: string;
+	text: string;
+	status: string;
+	files: string[];
+	claim: Record<string, unknown>;
+	pageData: Record<string, unknown>;
+}
+
+export interface SyncImpact {
+	page: string;
+	claimId?: string;
+	text: string;
+	impact: string;
+	confidence: number;
+	stillTrue: number;
+	changedFiles: string[];
+}
+
+export interface SyncReport {
+	repo: boolean;
+	baseline?: string;
+	head?: string;
+	baselineInitialized: boolean;
+	changedFiles: string[];
+	matchedClaims: number;
+	impacts: SyncImpact[];
+	applied: string[];
+	dryRun: boolean;
+	usage: { input_tokens: number; output_tokens: number };
+}
+
+export interface SyncOptions {
+	baseline?: string;
+	dryRun?: boolean;
+	maxClaims?: number;
+	maxDiffChars?: number;
+	signal?: AbortSignal;
+}
+
+function statePath(layout: WikiLayout): string {
+	return join(layout.stateDir, "state.json");
+}
+
+export async function readSyncState(layout: WikiLayout): Promise<SyncState> {
+	if (!existsSync(statePath(layout))) return {};
+	try {
+		return JSON.parse(await readFile(statePath(layout), "utf8")) as SyncState;
+	} catch {
+		return {};
+	}
+}
+
+export async function writeSyncState(layout: WikiLayout, state: SyncState): Promise<void> {
+	await writeTextAtomic(statePath(layout), `${JSON.stringify(state, null, "\t")}\n`);
+}
+
+export async function collectMappedClaims(layout: WikiLayout): Promise<MappedClaim[]> {
+	const files = (await listMarkdownFiles(layout.wikiDir)).filter(
+		(file) => !file.endsWith("index.md") && !file.endsWith("log.md"),
+	);
+	const mapped: MappedClaim[] = [];
+	for (const file of files) {
+		try {
+			const page = await readPage(file);
+			const pageFiles = normalizeFiles(page.data.files);
+			const claims = Array.isArray(page.data.claims) ? (page.data.claims as Record<string, unknown>[]) : [];
+			const pagePath = file; // listMarkdownFiles returns absolute paths
+			const rel = file.split("\\").join("/").replace(`${layout.wikiDir.split("\\").join("/")}/`, "");
+			for (const claim of claims) {
+				if (typeof claim.text !== "string") continue;
+				const claimFiles = normalizeFiles(claim.files);
+				const relevant = claimFiles.length > 0 ? claimFiles : pageFiles;
+				if (relevant.length === 0) continue;
+				mapped.push({
+					page: rel,
+					pagePath,
+					claimId: typeof claim.id === "string" ? claim.id : undefined,
+					text: claim.text,
+					status: typeof claim.status === "string" ? claim.status : "unknown",
+					files: relevant,
+					claim,
+					pageData: page.data,
+				});
+			}
+		} catch {
+			/* skip unreadable */
+		}
+	}
+	return mapped;
+}
+
+function normalizeFiles(value: unknown): string[] {
+	if (Array.isArray(value)) return value.map(String).filter(Boolean);
+	if (typeof value === "string" && value) return [value];
+	return [];
+}
+
+export async function syncWiki(
+	layout: WikiLayout,
+	client: JevClient,
+	config: ResolvedConfig,
+	cwd: string,
+	options?: SyncOptions,
+): Promise<SyncReport> {
+	const usage = { input_tokens: 0, output_tokens: 0 };
+	const empty: SyncReport = {
+		repo: false,
+		baselineInitialized: false,
+		changedFiles: [],
+		matchedClaims: 0,
+		impacts: [],
+		applied: [],
+		dryRun: Boolean(options?.dryRun),
+		usage,
+	};
+
+	if (!(await isGitRepo(cwd))) return empty;
+	const head = await headCommit(cwd);
+	if (!head) return { ...empty, repo: true };
+
+	const state = await readSyncState(layout);
+	const baseline = options?.baseline ?? state.lastSyncCommit;
+	if (!baseline) {
+		if (!options?.dryRun) await writeSyncState(layout, { lastSyncCommit: head, lastSyncAt: new Date().toISOString() });
+		return { ...empty, repo: true, head, baselineInitialized: true };
+	}
+
+	const changed = await changedFiles(cwd, baseline, "HEAD");
+	if (changed.length === 0) {
+		if (!options?.dryRun) await writeSyncState(layout, { lastSyncCommit: head, lastSyncAt: new Date().toISOString() });
+		return { ...empty, repo: true, baseline, head, baselineInitialized: false };
+	}
+
+	const mapped = await collectMappedClaims(layout);
+	const affected = mapped
+		.filter((claim) => claim.files.some((pattern) => changed.some((path) => fileMatches(path, pattern))))
+		.slice(0, options?.maxClaims ?? 40);
+
+	const impacts: SyncImpact[] = [];
+	const applied: string[] = [];
+	const touchedPages = new Map<string, Record<string, unknown>>();
+
+	for (const claim of affected) {
+		const relevantChanges = changed.filter((path) => claim.files.some((pattern) => fileMatches(path, pattern)));
+		const diff = await diffForFiles(cwd, baseline, "HEAD", relevantChanges, options?.maxDiffChars ?? 6000);
+		const response = await client.systemOne(
+			{
+				page: { title: claim.pageData.title ?? claim.page, summary: claim.pageData.summary ?? null },
+				claim: { id: claim.claimId ?? null, text: claim.text, status: claim.status, files: claim.files },
+				code_change: { files: relevantChanges, diff },
+			},
+			{
+				impact: choice("Given this code change, how should the wiki claim be updated?", {
+					no_impact: "The change does not affect what the claim says",
+					needs_recheck: "The change may invalidate the claim; it should be re-verified against current code",
+					supersede: "The claim is now outdated and should be marked superseded",
+					contradict: "The change directly contradicts the claim",
+				}),
+				still_true: noul("The claim is still true of the current code after this change."),
+			},
+			{ signal: options?.signal },
+		);
+		usage.input_tokens += response.usage.input_tokens;
+		usage.output_tokens += response.usage.output_tokens;
+
+		const impactAnswer = response.answers.impact;
+		const impact = isChoice(impactAnswer) ? impactAnswer.choice : "needs_recheck";
+		const confidence = isChoice(impactAnswer) ? impactAnswer.confidence : 0;
+		const stillTrue = isNoul(response.answers.still_true) ? response.answers.still_true.noul : 0.5;
+		const record: SyncImpact = {
+			page: claim.page,
+			claimId: claim.claimId,
+			text: claim.text,
+			impact,
+			confidence,
+			stillTrue,
+			changedFiles: relevantChanges,
+		};
+		impacts.push(record);
+
+		await appendLedger(layout, {
+			actor: "jev",
+			op: "sync.impact",
+			subject: `${claim.page}#${claim.claimId ?? claim.text.slice(0, 40)}`,
+			verdict: { impact, confidence, stillTrue },
+			action: impact,
+			usage: { input_tokens: response.usage.input_tokens, output_tokens: response.usage.output_tokens },
+		});
+
+		if (options?.dryRun || impact === "no_impact") {
+			if (!options?.dryRun) {
+				claim.claim.last_checked = todayISO();
+				touchedPages.set(claim.pagePath, claim.pageData);
+			}
+			continue;
+		}
+
+		const criticality = impact === "contradict" ? Math.max(0.85, 1 - stillTrue) : Math.max(0.4, 1 - stillTrue);
+		if (impact === "needs_recheck") {
+			claim.claim.status = "needs_recheck";
+			await enqueueReview(layout, {
+				kind: "needs_recheck",
+				claimText: claim.text,
+				page: claim.page,
+				claimId: claim.claimId,
+				criticality,
+				reason: `code changed: ${relevantChanges.slice(0, 4).join(", ")}`,
+				verdicts: { impact, confidence, stillTrue },
+			});
+		} else if (impact === "supersede") {
+			claim.claim.status = "superseded";
+			claim.claim.superseded_by = `commit ${head.slice(0, 7)}`;
+			await enqueueReview(layout, {
+				kind: "needs_recheck",
+				claimText: claim.text,
+				page: claim.page,
+				claimId: claim.claimId,
+				criticality,
+				reason: `diff supersedes claim: ${relevantChanges.slice(0, 4).join(", ")}`,
+				verdicts: { impact, confidence, stillTrue },
+			});
+		} else if (impact === "contradict") {
+			claim.claim.status = "disputed";
+			await enqueueReview(layout, {
+				kind: "dispute",
+				claimText: claim.text,
+				page: claim.page,
+				claimId: claim.claimId,
+				criticality,
+				reason: `diff contradicts claim: ${relevantChanges.slice(0, 4).join(", ")}`,
+				verdicts: { impact, confidence, stillTrue },
+			});
+		}
+		claim.claim.last_checked = todayISO();
+		claim.pageData.updated = todayISO();
+		touchedPages.set(claim.pagePath, claim.pageData);
+		applied.push(`${claim.page}#${claim.claimId ?? "?"} → ${impact}`);
+		await appendLedger(layout, {
+			actor: "code",
+			op: "sync.apply",
+			subject: `${claim.page}#${claim.claimId ?? claim.text.slice(0, 40)}`,
+			action: impact,
+			verdict: { status: claim.claim.status ?? null, changedFiles: relevantChanges },
+		});
+	}
+
+	if (!options?.dryRun) {
+		for (const [pagePath, data] of touchedPages) {
+			const page = await readPage(pagePath);
+			await writePage(pagePath, data, page.body);
+		}
+		await writeSyncState(layout, { lastSyncCommit: head, lastSyncAt: new Date().toISOString() });
+		await appendLog(layout, "sync", `${impacts.length} claim(s) checked`, [
+			`Baseline: ${baseline.slice(0, 7)} → ${head.slice(0, 7)}`,
+			`Changed files: ${changed.length}`,
+			...applied.map((line) => `Applied: ${line}`),
+		]);
+	}
+
+	return {
+		repo: true,
+		baseline,
+		head,
+		baselineInitialized: false,
+		changedFiles: changed,
+		matchedClaims: affected.length,
+		impacts,
+		applied,
+		dryRun: Boolean(options?.dryRun),
+		usage,
+	};
+}