pi-jev-wiki 0.2.0

package/src/wiki/frontmatter.ts

@@ -0,0 +1,164 @@
+/**
+ * Minimal YAML frontmatter support for the subset jev-wiki writes:
+ * flat scalars, inline arrays of scalars, and lists of flat maps (with inline arrays).
+ */
+
+export interface Frontmatter {
+	data: Record<string, unknown>;
+	body: string;
+}
+
+const KEY_RE = /^([A-Za-z0-9_-]+):(.*)$/;
+
+function parseScalar(raw: string): unknown {
+	const value = raw.trim();
+	if (value === "") return "";
+	if (value === "null" || value === "~") return null;
+	if (value === "true") return true;
+	if (value === "false") return false;
+	if (/^-?\d+(\.\d+)?$/.test(value)) return Number(value);
+	if (value.startsWith('"') && value.endsWith('"') && value.length >= 2) {
+		try {
+			return JSON.parse(value);
+		} catch {
+			return value.slice(1, -1);
+		}
+	}
+	if (value.startsWith("'") && value.endsWith("'") && value.length >= 2) {
+		return value.slice(1, -1).replace(/''/g, "'");
+	}
+	if (value.startsWith("[") && value.endsWith("]")) {
+		const inner = value.slice(1, -1).trim();
+		if (!inner) return [];
+		return splitInline(inner).map(parseScalar);
+	}
+	return value;
+}
+
+function splitInline(input: string): string[] {
+	const parts: string[] = [];
+	let current = "";
+	let quote: string | null = null;
+	for (const char of input) {
+		if (quote) {
+			current += char;
+			if (char === quote) quote = null;
+			continue;
+		}
+		if (char === '"' || char === "'") {
+			quote = char;
+			current += char;
+			continue;
+		}
+		if (char === ",") {
+			parts.push(current);
+			current = "";
+			continue;
+		}
+		current += char;
+	}
+	if (current.trim()) parts.push(current);
+	return parts.map((part) => part.trim());
+}
+
+export function parseFrontmatter(text: string): Frontmatter {
+	const normalized = text.replace(/^\uFEFF/, "");
+	if (!normalized.startsWith("---")) return { data: {}, body: normalized };
+	const end = normalized.indexOf("\n---", 3);
+	if (end === -1) return { data: {}, body: normalized };
+	const block = normalized.slice(4, end).replace(/\r/g, "");
+	const body = normalized.slice(end + 4).replace(/^\r?\n/, "");
+	return { data: parseYamlBlock(block), body };
+}
+
+export function parseYamlBlock(block: string): Record<string, unknown> {
+	const root: Record<string, unknown> = {};
+	let currentListKey: string | null = null;
+	let currentItem: Record<string, unknown> | null = null;
+
+	for (const rawLine of block.split("\n")) {
+		if (!rawLine.trim() || rawLine.trim().startsWith("#")) continue;
+		const indent = rawLine.match(/^ */)?.[0].length ?? 0;
+		const line = rawLine.trim();
+
+		if (indent === 0) {
+			currentListKey = null;
+			currentItem = null;
+			const match = line.match(KEY_RE);
+			if (!match) continue;
+			const [, key, rest] = match;
+			if (rest.trim() === "") {
+				root[key] = [];
+				currentListKey = key;
+			} else {
+				root[key] = parseScalar(rest);
+			}
+			continue;
+		}
+
+		if (line.startsWith("- ")) {
+			if (!currentListKey) continue;
+			const rest = line.slice(2).trim();
+			const list = root[currentListKey] as unknown[];
+			const match = rest.match(KEY_RE);
+			if (match) {
+				currentItem = { [match[1]]: parseScalar(match[2]) };
+				list.push(currentItem);
+			} else {
+				currentItem = null;
+				list.push(parseScalar(rest));
+			}
+			continue;
+		}
+
+		const match = line.match(KEY_RE);
+		if (match && currentItem) currentItem[match[1]] = parseScalar(match[2]);
+	}
+	return root;
+}
+
+function formatScalar(value: unknown): string {
+	if (value === null) return "null";
+	if (typeof value === "number" || typeof value === "boolean") return String(value);
+	const text = String(value);
+	if (text === "") return '""';
+	if (/^[\s]|[\s]$|[:#\[\]{},"'|>&*!%@`]|^-|^[?]/.test(text) || text.includes("\n")) {
+		return JSON.stringify(text);
+	}
+	if (/^(true|false|null|~)$/i.test(text) || /^-?\d+(\.\d+)?$/.test(text)) return JSON.stringify(text);
+	return text;
+}
+
+function formatInlineArray(values: unknown[]): string {
+	return `[${values.map((value) => (Array.isArray(value) ? formatInlineArray(value) : formatScalar(value))).join(", ")}]`;
+}
+
+export function serializeFrontmatter(data: Record<string, unknown>, body: string): string {
+	const lines = ["---"];
+	for (const [key, value] of Object.entries(data)) {
+		if (value === undefined) continue;
+		if (Array.isArray(value)) {
+			if (value.length === 0) {
+				lines.push(`${key}: []`);
+				continue;
+			}
+			if (value.every((item) => item === null || typeof item !== "object" || Array.isArray(item))) {
+				lines.push(`${key}: ${formatInlineArray(value)}`);
+				continue;
+			}
+			lines.push(`${key}:`);
+			for (const item of value) {
+				const entries = Object.entries(item as Record<string, unknown>).filter(([, v]) => v !== undefined);
+				entries.forEach(([childKey, childValue], index) => {
+					const prefix = index === 0 ? "  - " : "    ";
+					const rendered = Array.isArray(childValue) ? formatInlineArray(childValue) : formatScalar(childValue);
+					lines.push(`${prefix}${childKey}: ${rendered}`);
+				});
+			}
+			continue;
+		}
+		lines.push(`${key}: ${formatScalar(value)}`);
+	}
+	lines.push("---", "", body.replace(/\s+$/, ""), "");
+	return lines.join("\n");
+}

package/src/wiki/layout.ts

@@ -0,0 +1,126 @@
+/**
+ * Wiki filesystem layout: paths, atomic writes, hashing, and page I/O.
+ */
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
+import { dirname, isAbsolute, join, relative, resolve } from "node:path";
+import { withFileMutationQueue } from "@earendil-works/pi-coding-agent";
+import { parseFrontmatter, serializeFrontmatter, type Frontmatter } from "./frontmatter.ts";
+
+export interface WikiLayout {
+	root: string;
+	rawDir: string;
+	wikiDir: string;
+	stateDir: string;
+	ledgerPath: string;
+	reviewQueuePath: string;
+	sessionLogPath: string;
+}
+
+export function resolveLayout(cwd: string, wikiRoot: string, stateRoot = ".jev-wiki"): WikiLayout {
+	const root = resolve(cwd, wikiRoot);
+	const stateDir = isAbsolute(stateRoot) ? stateRoot : join(root, stateRoot);
+	return {
+		root,
+		rawDir: join(root, "raw"),
+		wikiDir: join(root, "wiki"),
+		stateDir,
+		ledgerPath: join(stateDir, "decisions.jsonl"),
+		reviewQueuePath: join(stateDir, "review-queue.jsonl"),
+		sessionLogPath: join(stateDir, "session-log.jsonl"),
+	};
+}
+
+export async function ensureLayout(layout: WikiLayout): Promise<void> {
+	for (const dir of [layout.rawDir, layout.wikiDir, layout.stateDir]) {
+		await mkdir(dir, { recursive: true });
+	}
+}
+
+export function slugify(input: string, maxLength = 60): string {
+	const slug = input
+		.toLowerCase()
+		.replace(/[^a-z0-9]+/g, "-")
+		.replace(/^-+|-+$/g, "")
+		.slice(0, maxLength)
+		.replace(/-+$/g, "");
+	return slug || "untitled";
+}
+
+export function todayISO(date = new Date()): string {
+	return date.toISOString().slice(0, 10);
+}
+
+export function timeStamp(date = new Date()): string {
+	return date.toISOString().slice(11, 16).replace(":", "");
+}
+
+export async function sha256Hex(text: string): Promise<string> {
+	return createHash("sha256").update(text, "utf8").digest("hex");
+}
+
+export async function writeTextAtomic(absPath: string, text: string): Promise<void> {
+	await withFileMutationQueue(absPath, async () => {
+		await mkdir(dirname(absPath), { recursive: true });
+		const temp = `${absPath}.tmp-${process.pid}-${Date.now()}`;
+		await writeFile(temp, text, "utf8");
+		await rename(temp, absPath);
+	});
+}
+
+export async function readPage(absPath: string): Promise<Frontmatter & { path: string }> {
+	const text = await readFile(absPath, "utf8");
+	const { data, body } = parseFrontmatter(text);
+	return { data, body, path: absPath };
+}
+
+export async function writePage(absPath: string, data: Record<string, unknown>, body: string): Promise<void> {
+	await writeTextAtomic(absPath, serializeFrontmatter(data, body));
+}
+
+export async function writeRawSource(
+	layout: WikiLayout,
+	topic: string,
+	slug: string,
+	data: Record<string, unknown>,
+	body: string,
+): Promise<string> {
+	const dir = join(layout.rawDir, slugify(topic, 40));
+	const filename = `${todayISO()}-${slugify(slug)}.md`;
+	const path = join(dir, filename);
+	await writePage(path, data, body);
+	return path;
+}
+
+export function relativeTo(fromDir: string, to: string): string {
+	const rel = relative(fromDir, to).split("\\").join("/");
+	return rel.startsWith(".") ? rel : `./${rel}`;
+}
+
+export function toPosix(path: string): string {
+	return path.split("\\").join("/");
+}
+
+export async function listMarkdownFiles(dir: string): Promise<string[]> {
+	if (!existsSync(dir)) return [];
+	const out: string[] = [];
+	async function walk(current: string): Promise<void> {
+		const entries = await readdir(current, { withFileTypes: true });
+		for (const entry of entries) {
+			const full = join(current, entry.name);
+			if (entry.isDirectory()) {
+				if (entry.name === ".jev-wiki" || entry.name === "raw" || entry.name === "toc") continue;
+				await walk(full);
+			} else if (entry.isFile() && entry.name.endsWith(".md")) {
+				out.push(full);
+			}
+		}
+	}
+	await walk(dir);
+	return out.sort();
+}
+
+export async function removeFileIfExists(path: string): Promise<void> {
+	if (existsSync(path)) await rm(path, { force: true });
+}

package/src/wiki/links.ts

@@ -0,0 +1,17 @@
+/** Markdown link helpers shared by the extension and lint. */
+
+export function extractMarkdownLinks(markdown: string): string[] {
+	const links: string[] = [];
+	for (const match of markdown.matchAll(/\[[^\]]*\]\(([^)]+)\)/g)) {
+		links.push(match[1].trim());
+	}
+	return links;
+}
+
+export function isExternalLink(link: string): boolean {
+	return /^[a-z][a-z0-9+.-]*:/i.test(link) || link.startsWith("#");
+}
+
+export function linkTarget(link: string): string {
+	return link.split("#")[0].trim();
+}

package/src/wiki/lock.ts

@@ -0,0 +1,86 @@
+/**
+ * Cross-process wiki lock.
+ *
+ * `withFileMutationQueue` only serializes within one pi process; two sessions in
+ * the same project can otherwise interleave read-modify-write cycles on the TOC,
+ * log, review queue, and raw index. This is a simple exclusive-create lock file
+ * with stale takeover. Locks are held only around short mutations — never across
+ * model or Jev calls — and are not reentrant.
+ */
+import { mkdir, open, readFile, rm } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import type { WikiLayout } from "./layout.ts";
+
+export interface LockOptions {
+	timeoutMs?: number;
+	staleMs?: number;
+}
+
+export class WikiLockError extends Error {
+	constructor(message: string) {
+		super(message);
+		this.name = "WikiLockError";
+	}
+}
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+export function lockPath(layout: WikiLayout): string {
+	return join(layout.stateDir, "lock");
+}
+
+export async function isLocked(layout: WikiLayout, staleMs = 30_000): Promise<{ locked: boolean; stale: boolean; ageMs?: number }> {
+	try {
+		const raw = await readFile(lockPath(layout), "utf8");
+		const info = JSON.parse(raw) as { ts?: number };
+		const ageMs = info.ts ? Date.now() - info.ts : undefined;
+		return { locked: true, stale: ageMs === undefined || ageMs > staleMs, ageMs };
+	} catch {
+		return { locked: false, stale: false };
+	}
+}
+
+/** Run `fn` while holding the wiki lock. Not reentrant. */
+export async function withWikiLock<T>(layout: WikiLayout, fn: () => Promise<T>, options?: LockOptions): Promise<T> {
+	const path = lockPath(layout);
+	const timeoutMs = options?.timeoutMs ?? 15_000;
+	const staleMs = options?.staleMs ?? 30_000;
+	const started = Date.now();
+	await mkdir(dirname(path), { recursive: true });
+
+	let handle: Awaited<ReturnType<typeof open>>;
+	for (;;) {
+		try {
+			handle = await open(path, "wx");
+			await handle.writeFile(JSON.stringify({ pid: process.pid, ts: Date.now(), host: process.env.COMPUTERNAME ?? process.env.HOSTNAME ?? "" }));
+			break;
+		} catch (error) {
+			if ((error as { code?: string }).code !== "EEXIST") throw error;
+			// Existing lock: take it over if stale, otherwise wait.
+			try {
+				const raw = await readFile(path, "utf8");
+				const info = JSON.parse(raw) as { ts?: number };
+				if (!info.ts || Date.now() - info.ts > staleMs) {
+					await rm(path, { force: true });
+					continue;
+				}
+			} catch {
+				await rm(path, { force: true }).catch(() => undefined);
+				continue;
+			}
+			if (Date.now() - started > timeoutMs) {
+				throw new WikiLockError(
+					`Wiki is locked by another pi session (${path}). Retry in a moment, or delete the lock file if it is stale.`,
+				);
+			}
+			await sleep(100 + Math.floor(Math.random() * 150));
+		}
+	}
+
+	try {
+		return await fn();
+	} finally {
+		await handle.close().catch(() => undefined);
+		await rm(path, { force: true }).catch(() => undefined);
+	}
+}

package/src/wiki/search.ts

@@ -0,0 +1,263 @@
+/**
+ * Wiki search: pluggable engines behind one interface.
+ *   index  — table-of-contents matching plus naive token scan (small wikis)
+ *   bm25   — in-process BM25 over page content (up to a few thousand pages)
+ *   qmd    — optional adapter to the qmd CLI (hybrid BM25 + vectors + rerank)
+ */
+import { existsSync } from "node:fs";
+import { readFile, stat } from "node:fs/promises";
+import { relative } from "node:path";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import type { ResolvedConfig } from "../config.ts";
+import { listMarkdownFiles, type WikiLayout } from "./layout.ts";
+import { readIndex, isWikiMetaFile, type TocEntry } from "./toc.ts";
+
+const run = promisify(execFile);
+
+export interface SearchResult {
+	path: string;
+	title: string;
+	score: number;
+	excerpt: string;
+	source?: "project" | "global";
+}
+
+export interface SearchOptions {
+	limit?: number;
+	query: string;
+}
+
+export interface SearchEngine {
+	name: string;
+	search(options: SearchOptions): Promise<SearchResult[]>;
+}
+
+const STOPWORDS = new Set([
+	"the", "and", "for", "with", "that", "this", "from", "into", "when", "what", "where", "which", "does", "how", "why",
+	"are", "was", "were", "has", "have", "had", "not", "but", "can", "could", "should", "would", "about", "over", "under",
+]);
+
+export function tokenize(text: string): string[] {
+	return text
+		.toLowerCase()
+		.split(/[^a-z0-9_]+/)
+		.filter((token) => token.length > 2 && !STOPWORDS.has(token));
+}
+
+interface Doc {
+	path: string;
+	title: string;
+	tags: string[];
+	summary: string;
+	text: string;
+	length: number;
+}
+
+async function loadDocs(layout: WikiLayout): Promise<Doc[]> {
+	const files = await listMarkdownFiles(layout.wikiDir);
+	const entries = await readIndex(layout);
+	const byPath = new Map<string, TocEntry>(entries.map((entry) => [entry.path, entry]));
+	const docs: Doc[] = [];
+	for (const file of files) {
+		const rel = relative(layout.wikiDir, file).split("\\").join("/");
+		if (isWikiMetaFile(rel)) continue;
+		try {
+			const raw = await readFile(file, "utf8");
+			const entry = byPath.get(rel);
+			docs.push({
+				path: rel,
+				title: entry?.title ?? rel,
+				tags: entry?.tags ?? [],
+				summary: entry?.summary ?? "",
+				text: raw,
+				length: tokenize(raw).length || 1,
+			});
+		} catch {
+			/* skip unreadable */
+		}
+	}
+	return docs;
+}
+
+function excerptFor(doc: Doc, queryTokens: Set<string>): string {
+	const lines = doc.text.split(/\r?\n/);
+	const hits = lines
+		.filter((line) => {
+			const lower = line.toLowerCase();
+			return [...queryTokens].some((token) => lower.includes(token));
+		})
+		.slice(0, 3);
+	return (hits.length > 0 ? hits : lines.filter((line) => line.trim()).slice(0, 2))
+		.map((line) => line.trim().slice(0, 240))
+		.join("\n");
+}
+
+async function directoryFingerprint(layout: WikiLayout): Promise<string> {
+	const files = await listMarkdownFiles(layout.wikiDir);
+	let newest = 0;
+	for (const file of files) {
+		const info = await stat(file).catch(() => undefined);
+		if (info && info.mtimeMs > newest) newest = info.mtimeMs;
+	}
+	return `${files.length}:${Math.round(newest)}`;
+}
+
+/** BM25 with field boosts: title x3, summary/tags x2, body x1. */
+class Bm25Index {
+	private docs: Doc[] = [];
+	private df = new Map<string, number>();
+	private avgLength = 1;
+	private built = 0;
+	private fingerprint = "";
+	private checkedAt = 0;
+
+	async ensure(layout: WikiLayout): Promise<void> {
+		const now = Date.now();
+		if (this.built > 0 && now - this.checkedAt < 2000) return;
+		const fingerprint = await directoryFingerprint(layout);
+		this.checkedAt = now;
+		if (this.built > 0 && fingerprint === this.fingerprint) return;
+		this.docs = await loadDocs(layout);
+		this.df = new Map();
+		let total = 0;
+		for (const doc of this.docs) {
+			const unique = new Set(tokenize(doc.text));
+			for (const token of unique) this.df.set(token, (this.df.get(token) ?? 0) + 1);
+			total += doc.length;
+		}
+		this.avgLength = this.docs.length > 0 ? total / this.docs.length : 1;
+		this.fingerprint = fingerprint;
+		this.built = Date.now();
+	}
+
+	search(query: string, limit: number): SearchResult[] {
+		const tokens = tokenize(query);
+		if (tokens.length === 0) return [];
+		const querySet = new Set(tokens);
+		const k1 = 1.2;
+		const b = 0.75;
+		const scored = this.docs.map((doc) => {
+			const bodyTokens = tokenize(doc.text);
+			const counts = new Map<string, number>();
+			for (const token of bodyTokens) counts.set(token, (counts.get(token) ?? 0) + 1);
+			const titleTokens = new Set(tokenize(doc.title));
+			const metaTokens = new Set([...tokenize(doc.summary), ...tokenize(doc.tags.join(" "))]);
+			let score = 0;
+			for (const token of querySet) {
+				const df = this.df.get(token) ?? 0;
+				if (df === 0) continue;
+				const idf = Math.log(1 + (this.docs.length - df + 0.5) / (df + 0.5));
+				const tf = counts.get(token) ?? 0;
+				const bodyScore = (tf * (k1 + 1)) / (tf + k1 * (1 - b + (b * doc.length) / this.avgLength));
+				const boost = titleTokens.has(token) ? 3 : metaTokens.has(token) ? 2 : 1;
+				score += idf * bodyScore * boost;
+			}
+			return { doc, score };
+		});
+		return scored
+			.filter((entry) => entry.score > 0)
+			.sort((a, b2) => b2.score - a.score)
+			.slice(0, limit)
+			.map((entry) => ({
+				path: entry.doc.path,
+				title: entry.doc.title,
+				score: Number(entry.score.toFixed(3)),
+				excerpt: excerptFor(entry.doc, querySet),
+			}));
+	}
+}
+
+const bm25Cache = new Map<string, Bm25Index>();
+
+export class IndexSearchEngine implements SearchEngine {
+	readonly name = "index";
+	constructor(private readonly layout: WikiLayout) {}
+	async search(options: SearchOptions): Promise<SearchResult[]> {
+		return this.indexFallback(options);
+	}
+	private async indexFallback(options: SearchOptions): Promise<SearchResult[]> {
+		const entries = await readIndex(this.layout);
+		const tokens = tokenize(options.query);
+		const scored = entries
+			.map((entry) => {
+				const haystack = `${entry.title} ${entry.summary} ${entry.tags.join(" ")}`.toLowerCase();
+				const score = tokens.reduce((sum, token) => sum + (haystack.includes(token) ? 1 : 0), 0);
+				return { entry, score };
+			})
+			.filter((item) => item.score > 0)
+			.sort((a, b) => b.score - a.score)
+			.slice(0, options.limit ?? 5);
+		return scored.map(({ entry, score }) => ({
+			path: entry.path,
+			title: entry.title,
+			score,
+			excerpt: entry.summary,
+		}));
+	}
+}
+
+export class Bm25SearchEngine implements SearchEngine {
+	readonly name = "bm25";
+	constructor(private readonly layout: WikiLayout) {}
+	async search(options: SearchOptions): Promise<SearchResult[]> {
+		const key = this.layout.wikiDir;
+		let index = bm25Cache.get(key);
+		if (!index) {
+			index = new Bm25Index();
+			bm25Cache.set(key, index);
+		}
+		await index.ensure(this.layout);
+		return index.search(options.query, options.limit ?? 5);
+	}
+}
+
+export class QmdSearchEngine implements SearchEngine {
+	readonly name = "qmd";
+	private readonly fallback: SearchEngine;
+	constructor(
+		private readonly layout: WikiLayout,
+		collection?: string,
+	) {
+		this.collection = collection;
+		this.fallback = new Bm25SearchEngine(layout);
+	}
+	private collection?: string;
+	async search(options: SearchOptions): Promise<SearchResult[]> {
+		try {
+			const args = ["search", options.query, "--json", "-n", String(options.limit ?? 5)];
+			if (this.collection) args.push("-c", this.collection);
+			const { stdout } = await run("qmd", args, { maxBuffer: 4 * 1024 * 1024 });
+			const parsed = JSON.parse(stdout) as Array<{ path?: string; docid?: string; score?: number; snippet?: string; title?: string }>;
+			return parsed.slice(0, options.limit ?? 5).map((item) => ({
+				path: item.path ?? item.docid ?? "(unknown)",
+				title: item.title ?? item.path ?? "(unknown)",
+				score: typeof item.score === "number" ? item.score : 0,
+				excerpt: item.snippet ?? "",
+			}));
+		} catch {
+			return this.fallback.search(options);
+		}
+	}
+}
+
+export function createSearchEngine(config: ResolvedConfig, layout: WikiLayout, globalLayout?: WikiLayout): SearchEngine {
+	const engines: SearchEngine[] = [];
+	if (config.search.engine === "qmd") engines.push(new QmdSearchEngine(layout, config.search.qmdCollection));
+	if (config.search.engine === "bm25") engines.push(new Bm25SearchEngine(layout));
+	if (engines.length === 0) engines.push(new IndexSearchEngine(layout));
+	if (globalLayout && existsSync(globalLayout.wikiDir)) engines.push(new Bm25SearchEngine(globalLayout));
+	const primary = engines[0];
+	if (engines.length <= 1) return primary;
+	return {
+		name: `${primary.name}+global`,
+		async search(options) {
+			const results: SearchResult[] = [];
+			for (const [index, engine] of engines.entries()) {
+				const hits = await engine.search({ ...options, limit: Math.max(3, Math.round((options.limit ?? 5) / engines.length)) });
+				for (const hit of hits) results.push({ ...hit, source: index === 0 ? "project" : "global" });
+			}
+			return results.sort((a, b) => b.score - a.score).slice(0, options.limit ?? 5);
+		},
+	};
+}