pi-memoria 0.2.0 → 0.3.0

package/package.json

@@ -1,71 +1,30 @@
 {
-	"name": "pi-memoria",
-	"version": "0.2.0",
-	"description": "Agent-first long-term memory for pi: a categorized markdown library with automatic, sub-millisecond recall.",
-	"type": "module",
-	"license": "MIT",
-	"author": "rcsaquino",
-	"homepage": "https://github.com/rcsaquino/pi-memoria#readme",
-	"repository": {
-		"type": "git",
-		"url": "git+https://github.com/rcsaquino/pi-memoria.git"
-	},
-	"bugs": {
-		"url": "https://github.com/rcsaquino/pi-memoria/issues"
-	},
-	"keywords": [
-		"pi-package",
-		"pi-extension",
-		"memory",
-		"recall",
-		"markdown"
-	],
-	"engines": {
-		"node": ">=22.19.0"
-	},
-	"pi": {
-		"extensions": [
-			"./index.ts"
-		],
-		"skills": [
-			"./skills"
-		]
-	},
-	"files": [
-		"index.ts",
-		"src",
-		"skills",
-		"README.md",
-		"LICENSE"
-	],
-	"scripts": {
-		"test": "node --test tests/*.test.ts",
-		"eval": "node tests/eval.ts",
-		"bench": "node tests/bench.ts",
-		"bench:real": "node tests/bench.ts 1000 5000 --real",
-		"typecheck": "tsc --noEmit"
-	},
-	"peerDependencies": {
-		"@earendil-works/pi-coding-agent": "*",
-		"@earendil-works/pi-tui": "*",
-		"typebox": "*"
-	},
-	"devDependencies": {
-		"@earendil-works/pi-coding-agent": "*",
-		"@earendil-works/pi-tui": "*",
-		"@types/node": "^24.0.0",
-		"typebox": "*",
-		"typescript": "^5.6.0"
-	},
-	"peerDependenciesMeta": {
-		"@earendil-works/pi-coding-agent": {
-			"optional": true
-		},
-		"@earendil-works/pi-tui": {
-			"optional": true
-		},
-		"typebox": {
-			"optional": true
-		}
-	}
+  "name": "pi-memoria",
+  "version": "0.3.0",
+  "description": "Global working memory, SQLite FTS5 recall, and complete Pi session search",
+  "type": "module",
+  "license": "MIT",
+  "keywords": ["pi-package", "pi-extension", "memory", "sqlite", "fts5"],
+  "files": ["index.ts", "src", "README.md", "AGENTS.md", "LICENSE"],
+  "pi": { "extensions": ["./index.ts"] },
+  "engines": { "node": ">=22.19.0" },
+  "scripts": {
+    "check": "tsc --noEmit",
+    "test": "tsx --test test/*.test.ts",
+    "bench": "tsx scripts/benchmark.ts",
+    "verify": "npm run check && npm test"
+  },
+  "dependencies": { "proper-lockfile": "^4.1.2" },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "typebox": "*"
+  },
+  "devDependencies": {
+    "@earendil-works/pi-coding-agent": "0.87.1",
+    "@types/node": "^22.19.0",
+    "@types/proper-lockfile": "^4.1.4",
+    "tsx": "^4.20.0",
+    "typebox": "1.3.27",
+    "typescript": "^5.9.3"
+  }
 }

package/src/aliases.ts

@@ -0,0 +1,104 @@
+import { closeSync, openSync, readSync } from "node:fs";
+import { isMissing } from "./files.ts";
+
+export const MAX_ALIAS_FILE_BYTES = 64 * 1024;
+export const MAX_ALIAS_KEYS = 256;
+export const MAX_ALIAS_TARGETS = 8;
+export const MAX_ALIAS_TOKEN_UNITS = 80;
+export const MAX_ADDED_ALTERNATIVES = 128;
+
+const TOKEN = /^[\p{L}\p{N}\p{M}_]+$/u;
+
+/** Directional, one-hop terminology alternatives. User keys replace these entries wholesale. */
+const BUILT_IN: ReadonlyArray<readonly [string, readonly string[]]> = [
+  ["k8s", ["kubernetes"]],
+  ["db", ["database"]],
+  ["repo", ["repository"]],
+  ["prefs", ["preferences"]],
+];
+
+export interface AliasConfiguration { aliases: Map<string, string[]>; warning?: string }
+export interface QueryExpansion { expansions: Array<{ term: string; aliases: string[] }>; warning?: string }
+
+function boundedRead(path: string): { text: string; bytes: number } {
+  const descriptor = openSync(path, "r");
+  try {
+    const buffer = Buffer.alloc(MAX_ALIAS_FILE_BYTES + 1);
+    const bytes = readSync(descriptor, buffer, 0, buffer.length, 0);
+    return { text: buffer.subarray(0, bytes).toString("utf8"), bytes };
+  } finally {
+    closeSync(descriptor);
+  }
+}
+
+function invalid(aliases: Map<string, string[]>, reason: string): AliasConfiguration {
+  return { aliases, warning: `${reason} Built-in aliases remain active; fix or remove the file and restart Pi or /reload.` };
+}
+
+/**
+ * Load optional `synonyms.json` from the global memory directory. A missing
+ * file is normal. Any malformed, unreadable, or oversized file is ignored
+ * whole so a partial configuration is never merged.
+ */
+export function loadAliases(path?: string): AliasConfiguration {
+  const aliases = new Map<string, string[]>(BUILT_IN.map(([key, targets]) => [key, [...targets]]));
+  if (!path) return { aliases };
+  let text: string;
+  try {
+    const read = boundedRead(path);
+    if (read.bytes > MAX_ALIAS_FILE_BYTES) return invalid(aliases, `synonyms.json is larger than ${MAX_ALIAS_FILE_BYTES} bytes and was ignored.`);
+    text = read.text;
+  } catch (error) {
+    if (isMissing(error)) return { aliases };
+    return invalid(aliases, `synonyms.json could not be read (${(error as Error).message}) and was ignored.`);
+  }
+  let value: unknown;
+  try { value = JSON.parse(text); }
+  catch { return invalid(aliases, "synonyms.json is not valid JSON and was ignored."); }
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    return invalid(aliases, "synonyms.json must be a JSON object mapping single tokens to arrays of single tokens.");
+  }
+  const entries = Object.entries(value as Record<string, unknown>);
+  if (entries.length > MAX_ALIAS_KEYS) return invalid(aliases, `synonyms.json has more than ${MAX_ALIAS_KEYS} keys and was ignored.`);
+  const user = new Map<string, string[]>();
+  for (const [key, targets] of entries) {
+    const normalized = key.trim().toLowerCase();
+    if (!TOKEN.test(normalized) || normalized.length > MAX_ALIAS_TOKEN_UNITS) {
+      return invalid(aliases, `synonyms.json key ${JSON.stringify(key)} is invalid: keys must be single tokens of at most ${MAX_ALIAS_TOKEN_UNITS} characters.`);
+    }
+    if (user.has(normalized)) return invalid(aliases, `synonyms.json contains duplicate key ${JSON.stringify(normalized)} after lowercasing and was ignored.`);
+    if (!Array.isArray(targets) || targets.length > MAX_ALIAS_TARGETS) {
+      return invalid(aliases, `synonyms.json key ${JSON.stringify(key)} must map to an array of at most ${MAX_ALIAS_TARGETS} tokens.`);
+    }
+    const list: string[] = [];
+    for (const target of targets) {
+      if (typeof target !== "string") {
+        return invalid(aliases, `synonyms.json key ${JSON.stringify(key)} contains a non-string alternative.`);
+      }
+      const normalizedTarget = target.trim().toLowerCase();
+      if (!TOKEN.test(normalizedTarget) || normalizedTarget.length > MAX_ALIAS_TOKEN_UNITS) {
+        return invalid(aliases, `synonyms.json alternative ${JSON.stringify(target)} is invalid: alternatives must be single tokens of at most ${MAX_ALIAS_TOKEN_UNITS} characters.`);
+      }
+      if (normalizedTarget !== normalized && !list.includes(normalizedTarget)) list.push(normalizedTarget);
+    }
+    user.set(normalized, list);
+  }
+  for (const [key, targets] of user) aliases.set(key, targets);
+  return { aliases };
+}
+
+/** One-hop expansion for the original query tokens, bounded in total added alternatives. */
+export function expandQuery(terms: string[], aliases: Map<string, string[]>): QueryExpansion {
+  const expansions: Array<{ term: string; aliases: string[] }> = [];
+  let added = 0;
+  for (const term of terms) {
+    const alternatives = aliases.get(term.toLowerCase()) ?? [];
+    if (!alternatives.length) continue;
+    added += alternatives.length;
+    expansions.push({ term, aliases: alternatives });
+  }
+  if (added > MAX_ADDED_ALTERNATIVES) {
+    return { expansions: [], warning: `Alias expansion adds ${added} alternatives, over the ${MAX_ADDED_ALTERNATIVES}-term limit; the original terms were searched unexpanded.` };
+  }
+  return { expansions };
+}

package/src/config.ts

@@ -1,190 +1,28 @@
-/**
- * Configuration and path resolution for memoria.
- *
- * A store root has this shape:
- *
- *   memoria/
- *   ├── MEMORY.md        curated memory injected into every session (<= hotLimit chars)
- *   ├── config.json      optional config overrides
- *   ├── .index/          derived index data (never indexed)
- *   ├── .trash/          soft-deleted memories
- *   └── library/         categorized markdown memories
- */
-
 import { homedir } from "node:os";
 import { isAbsolute, join, resolve } from "node:path";
-import { readFileOrUndefined } from "./util.ts";
-import type { MemoriaConfig } from "./types.ts";
-
-export const INDEX_DIR = ".index";
-export const TRASH_DIR = ".trash";
-export const LIBRARY_DIR = "library";
-export const HOT_FILE = "MEMORY.md";
-export const CONFIG_FILE = "config.json";
-export const INDEX_FILE = "index.json";
-export const INDEX_BIN_FILE = "index.bin";
-export const USAGE_FILE = "usage.json";
-export const SYNONYMS_FILE = "synonyms.json";
-
-export const DEFAULT_CONFIG: MemoriaConfig = {
-	version: 1,
-	rootDir: "$AGENT_DIR/memoria",
-	projectRoot: "",
-	extraRoots: [],
-	hotLimit: 5000,
-	snippetChars: 280,
-	autoRecall: true,
-	autoRecallLimit: 3,
-	autoRecallMinRatio: 0.3,
-	autoRecallMinScore: 1.4,
-	autoRecallMaxChars: 2400,
-	autoRecallLastTurns: 1,
-	autoRecallPriorWeight: 0.4,
-	bodyCacheBytes: 8 * 1024 * 1024,
-	scanIntervalMs: 2000,
-	exclude: [],
-	defaultCategory: "inbox",
-	topicMaxWords: 6,
-	topicMaxChars: 8000,
-	usageTracking: true,
-	lastUsedWriteIntervalMs: 21_600_000,
-	staleAfterDays: 365,
-	promoteAfterWrites: 3,
-	dedupeThreshold: 0.62,
-	relatedHits: 3,
-	relatedBoost: 0.35,
-	synonyms: {},
-	synonymWeight: 0.6,
-	timeHints: true,
-	searchCacheSize: 32,
-	rerank: false,
-	rerankModel: "",
-	rerankTopK: 12,
-	rerankTimeoutMs: 2500,
-	watcherSettleMs: 300,
-	indexFormat: "auto",
-	sessionSearch: true,
-	sessionRoots: [],
-	sessionFallback: true,
-	sessionScanMs: 1500,
-	sessionExcerptChars: 400,
-	sessionCacheBytes: 32 * 1024 * 1024,
-	sessionIncludeTools: false,
-	sessionRipgrep: true,
-};
-
-/** Filesystem context used to resolve store paths. */
-export interface PathContext {
-	home: string;
-	/** Pi's user config directory (`~/.pi/agent` by default, or `PI_CODING_AGENT_DIR`). */
-	agentDir: string;
-}
-
-export interface ResolvedRoots {
-	/** The main user-level store: `<agent dir>/memoria` by default. */
-	primary: string;
-	/** Optional per-project store (writable). Empty string when disabled. */
-	project: string;
-	/** Additional read-only store roots declared in config. */
-	extras: string[];
-}
-
-/**
- * Expand `$AGENT_DIR`, a leading `~`, absolute paths and cwd-relative paths.
- * Returns undefined for an empty value.
- */
-export function resolveStorePath(input: string, cwd: string, context: PathContext): string | undefined {
-	const value = (input ?? "").trim();
-	if (!value) return undefined;
-	let expanded = value.replace(/\$AGENT_DIR/g, context.agentDir);
-	if (expanded === "~") return context.home;
-	if (expanded.startsWith("~/") || expanded.startsWith("~\\")) expanded = join(context.home, expanded.slice(2));
-	return isAbsolute(expanded) ? expanded : resolve(cwd, expanded);
-}
-
-/** Resolve the store roots for a working directory. */
-export function resolveRoots(cwd: string, config: MemoriaConfig = DEFAULT_CONFIG, context?: Partial<PathContext>): ResolvedRoots {
-	const home = context?.home ?? homedir();
-	const pathContext: PathContext = { home, agentDir: context?.agentDir ?? join(home, ".pi", "agent") };
-	const primary = resolveStorePath(config.rootDir, cwd, pathContext) ?? join(pathContext.agentDir, "memoria");
-	const project = resolveStorePath(config.projectRoot ?? "", cwd, pathContext) ?? "";
-	const extras = config.extraRoots
-		.map((entry) => resolveStorePath(entry, cwd, pathContext))
-		.filter((entry): entry is string => Boolean(entry) && entry !== primary && entry !== project);
-	return { primary, project, extras: [...new Set(extras)] };
-}
 
-function coerceConfig(raw: unknown): Partial<MemoriaConfig> {
-	if (!raw || typeof raw !== "object") return {};
-	const input = raw as Record<string, unknown>;
-	const out: Partial<MemoriaConfig> = {};
-	const number = (value: unknown): number | undefined => (typeof value === "number" && Number.isFinite(value) ? value : undefined);
-	const bool = (value: unknown): boolean | undefined => (typeof value === "boolean" ? value : undefined);
-	const str = (value: unknown): string | undefined => (typeof value === "string" && value.length > 0 ? value : undefined);
-	if (str(input.rootDir)) out.rootDir = input.rootDir as string;
-	if (Array.isArray(input.extraRoots)) out.extraRoots = input.extraRoots.filter((item): item is string => typeof item === "string");
-	if (Array.isArray(input.exclude)) out.exclude = input.exclude.filter((item): item is string => typeof item === "string");
-	if (number(input.hotLimit) !== undefined) out.hotLimit = Math.max(500, number(input.hotLimit)!);
-	if (number(input.snippetChars) !== undefined) out.snippetChars = Math.max(80, number(input.snippetChars)!);
-	if (bool(input.autoRecall) !== undefined) out.autoRecall = bool(input.autoRecall);
-	if (number(input.autoRecallLimit) !== undefined) out.autoRecallLimit = Math.max(0, Math.min(20, number(input.autoRecallLimit)!));
-	if (number(input.autoRecallMinRatio) !== undefined) out.autoRecallMinRatio = Math.max(0, Math.min(1, number(input.autoRecallMinRatio)!));
-	if (number(input.autoRecallMinScore) !== undefined) out.autoRecallMinScore = Math.max(0, number(input.autoRecallMinScore)!);
-	if (number(input.autoRecallMaxChars) !== undefined) out.autoRecallMaxChars = Math.max(200, number(input.autoRecallMaxChars)!);
-	if (number(input.autoRecallLastTurns) !== undefined) out.autoRecallLastTurns = Math.max(0, Math.min(5, number(input.autoRecallLastTurns)!));
-	if (number(input.bodyCacheBytes) !== undefined) out.bodyCacheBytes = Math.max(0, number(input.bodyCacheBytes)!);
-	if (number(input.scanIntervalMs) !== undefined) out.scanIntervalMs = Math.max(250, number(input.scanIntervalMs)!);
-	if (str(input.defaultCategory)) out.defaultCategory = input.defaultCategory as string;
-	if (number(input.topicMaxWords) !== undefined) out.topicMaxWords = Math.max(1, Math.min(12, number(input.topicMaxWords)!));
-	if (number(input.topicMaxChars) !== undefined) out.topicMaxChars = Math.max(500, number(input.topicMaxChars)!);
-	if (str(input.projectRoot)) out.projectRoot = input.projectRoot as string;
-	if (number(input.autoRecallPriorWeight) !== undefined) out.autoRecallPriorWeight = Math.max(0, Math.min(1, number(input.autoRecallPriorWeight)!));
-	if (bool(input.usageTracking) !== undefined) out.usageTracking = bool(input.usageTracking);
-	if (number(input.lastUsedWriteIntervalMs) !== undefined) out.lastUsedWriteIntervalMs = Math.max(60_000, number(input.lastUsedWriteIntervalMs)!);
-	if (number(input.staleAfterDays) !== undefined) out.staleAfterDays = Math.max(1, number(input.staleAfterDays)!);
-	if (number(input.promoteAfterWrites) !== undefined) out.promoteAfterWrites = Math.max(1, number(input.promoteAfterWrites)!);
-	if (number(input.dedupeThreshold) !== undefined) out.dedupeThreshold = Math.max(0.2, Math.min(0.98, number(input.dedupeThreshold)!));
-	if (number(input.relatedHits) !== undefined) out.relatedHits = Math.max(0, Math.min(10, number(input.relatedHits)!));
-	if (number(input.relatedBoost) !== undefined) out.relatedBoost = Math.max(0, Math.min(2, number(input.relatedBoost)!));
-	if (bool(input.timeHints) !== undefined) out.timeHints = bool(input.timeHints);
-	if (number(input.searchCacheSize) !== undefined) out.searchCacheSize = Math.max(0, Math.min(512, number(input.searchCacheSize)!));
-	if (bool(input.rerank) !== undefined) out.rerank = bool(input.rerank);
-	if (str(input.rerankModel)) out.rerankModel = input.rerankModel as string;
-	if (number(input.rerankTopK) !== undefined) out.rerankTopK = Math.max(1, Math.min(50, number(input.rerankTopK)!));
-	if (number(input.rerankTimeoutMs) !== undefined) out.rerankTimeoutMs = Math.max(250, Math.min(30_000, number(input.rerankTimeoutMs)!));
-	if (number(input.synonymWeight) !== undefined) out.synonymWeight = Math.max(0.1, Math.min(1, number(input.synonymWeight)!));
-	if (number(input.watcherSettleMs) !== undefined) out.watcherSettleMs = Math.max(0, Math.min(5000, number(input.watcherSettleMs)!));
-	if (str(input.indexFormat) && ["auto", "json", "binary"].includes(input.indexFormat as string)) out.indexFormat = input.indexFormat as string;
-	if (bool(input.sessionSearch) !== undefined) out.sessionSearch = bool(input.sessionSearch);
-	if (bool(input.sessionFallback) !== undefined) out.sessionFallback = bool(input.sessionFallback);
-	if (bool(input.sessionIncludeTools) !== undefined) out.sessionIncludeTools = bool(input.sessionIncludeTools);
-	if (number(input.sessionScanMs) !== undefined) out.sessionScanMs = Math.max(50, Math.min(60_000, number(input.sessionScanMs)!));
-	if (number(input.sessionExcerptChars) !== undefined) out.sessionExcerptChars = Math.max(80, Math.min(4000, number(input.sessionExcerptChars)!));
-	if (number(input.sessionCacheBytes) !== undefined) out.sessionCacheBytes = Math.max(0, number(input.sessionCacheBytes)!);
-	if (bool(input.sessionRipgrep) !== undefined) out.sessionRipgrep = bool(input.sessionRipgrep);
-	if (Array.isArray(input.sessionRoots)) out.sessionRoots = input.sessionRoots.filter((item): item is string => typeof item === "string");
-	if (input.synonyms && typeof input.synonyms === "object" && !Array.isArray(input.synonyms)) {
-		const table: Record<string, string[]> = {};
-		for (const [key, value] of Object.entries(input.synonyms as Record<string, unknown>)) {
-			const term = key.trim().toLowerCase();
-			if (!term) continue;
-			const list = Array.isArray(value) ? value : typeof value === "string" ? value.split(/[,\s]+/) : [];
-			const cleaned = list.map((entry) => String(entry).trim().toLowerCase()).filter((entry) => entry && entry !== term);
-			if (cleaned.length > 0) table[term] = [...new Set(cleaned)].slice(0, 12);
-		}
-		out.synonyms = table;
-	}
-	return out;
+export function absolutePath(value: string, name: string): string {
+  const expanded = value === "~" ? homedir() : value.startsWith("~/") ? join(homedir(), value.slice(2)) : value;
+  if (!expanded || !isAbsolute(expanded)) throw new Error(`${name} must be an absolute path (or start with ~/).`);
+  return resolve(expanded);
 }
 
-/** Load `<root>/config.json`, merged over defaults. Malformed files fall back to defaults. */
-export async function loadConfig(root: string): Promise<MemoriaConfig> {
-	const raw = await readFileOrUndefined(join(root, CONFIG_FILE));
-	if (!raw) return { ...DEFAULT_CONFIG };
-	try {
-		const parsed = JSON.parse(raw) as unknown;
-		return { ...DEFAULT_CONFIG, ...coerceConfig(parsed) };
-	} catch {
-		return { ...DEFAULT_CONFIG };
-	}
+export function configuration(agentDir: string, env: NodeJS.ProcessEnv = process.env) {
+  agentDir = absolutePath(agentDir, "Pi agent directory");
+  const directory = env.PI_MEMORIA_DIR ? absolutePath(env.PI_MEMORIA_DIR, "PI_MEMORIA_DIR") : join(agentDir, "memoria");
+  let additionalRoots: string[] = [];
+  if (env.PI_MEMORIA_SESSION_DIRS) {
+    const value: unknown = JSON.parse(env.PI_MEMORIA_SESSION_DIRS);
+    if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) {
+      throw new Error("PI_MEMORIA_SESSION_DIRS must be a JSON array of absolute directory paths.");
+    }
+    additionalRoots = value.map((root: string) => absolutePath(root, "PI_MEMORIA_SESSION_DIRS entry"));
+  }
+  return {
+    directory,
+    hotPath: join(directory, "MEMORY.md"),
+    databasePath: join(directory, "memoria.sqlite"),
+    aliasesPath: join(directory, "synonyms.json"),
+    sessionRoots: [join(agentDir, "sessions"), ...additionalRoots],
+  };
 }