pi-memoria 0.1.1

package/src/synonyms.ts

@@ -0,0 +1,105 @@
+/**
+ * Query-side synonym expansion.
+ *
+ * `k8s` should find notes that say `kubernetes` without re-indexing them, so
+ * synonyms are applied when the query is analyzed. The table comes from
+ * `config.synonyms` (inline) merged with `<root>/synonyms.json`, which makes it
+ * per-store: one person's shorthand never leaks into another store.
+ *
+ * File format (all keys and values lowercase; expansion is bidirectional):
+ *
+ *   {
+ *     "k8s": ["kubernetes", "kube"],
+ *     "postgres": ["postgresql", "pg"]
+ *   }
+ */
+
+import { join } from "node:path";
+import { SYNONYMS_FILE } from "./config.ts";
+import { readFileOrUndefined } from "./util.ts";
+
+/** Upper bound on the size of a store's synonym table. */
+export const MAX_SYNONYM_TERMS = 2000;
+
+/** Normalize an arbitrary JSON value into a clean `term -> alternatives` table. */
+export function normalizeSynonymTable(raw: unknown): Record<string, string[]> {
+	if (!raw || typeof raw !== "object" || Array.isArray(raw)) return {};
+	const out: Record<string, string[]> = {};
+	for (const [key, value] of Object.entries(raw as Record<string, unknown>)) {
+		const term = key.trim().toLowerCase();
+		if (!term || term.length > 60) continue;
+		const list = Array.isArray(value) ? value : typeof value === "string" ? value.split(/[,\s]+/) : [];
+		const cleaned = list
+			.map((entry) => String(entry ?? "").trim().toLowerCase())
+			.filter((entry) => entry && entry.length <= 60 && entry !== term);
+		if (cleaned.length > 0) out[term] = [...new Set(cleaned)].slice(0, 12);
+		if (Object.keys(out).length >= MAX_SYNONYM_TERMS) break;
+	}
+	return out;
+}
+
+/** Merge tables left to right; later entries append alternatives. */
+export function mergeSynonymTables(...tables: Array<Record<string, string[]> | undefined>): Record<string, string[]> {
+	const out = new Map<string, string[]>();
+	for (const table of tables) {
+		if (!table) continue;
+		for (const [term, values] of Object.entries(table)) {
+			const existing = out.get(term) ?? [];
+			out.set(term, [...new Set([...existing, ...values])].slice(0, 12));
+		}
+	}
+	return Object.fromEntries(out);
+}
+
+/**
+ * Make expansion symmetric: if `k8s -> kubernetes` then `kubernetes -> k8s`
+ * too, so the feature works from whichever word the query happens to use.
+ */
+export function expandSynonymTable(table: Record<string, string[]>): Record<string, string[]> {
+	const out = new Map<string, Set<string>>();
+	const add = (from: string, to: string): void => {
+		if (from === to) return;
+		const set = out.get(from) ?? new Set<string>();
+		set.add(to);
+		out.set(from, set);
+	};
+	for (const [term, values] of Object.entries(table)) {
+		for (const value of values) {
+			add(term, value);
+			add(value, term);
+		}
+	}
+	return Object.fromEntries([...out.entries()].map(([term, set]) => [term, [...set].slice(0, 12)]));
+}
+
+/** Load `<root>/synonyms.json`; a missing or malformed file yields an empty table. */
+export async function loadSynonymsFile(root: string): Promise<Record<string, string[]>> {
+	const raw = await readFileOrUndefined(join(root, SYNONYMS_FILE));
+	if (!raw) return {};
+	try {
+		return normalizeSynonymTable(JSON.parse(raw));
+	} catch {
+		return {};
+	}
+}
+
+/** Alternatives for a term, already lowercased and deduplicated. */
+export function synonymsFor(table: Record<string, string[]>, term: string): string[] {
+	return table[term] ?? [];
+}
+
+/** Example file content (used by `/memoria doctor` and the README). */
+export function synonymsFileTemplate(): string {
+	return `${JSON.stringify(
+		{
+			k8s: ["kubernetes"],
+			postgres: ["postgresql", "pg"],
+			auth: ["authentication", "login"],
+			deps: ["dependencies"],
+			repo: ["repository"],
+			perf: ["performance"],
+		},
+		null,
+		2,
+	)}\n`;
+}

package/src/timeexpr.ts

@@ -0,0 +1,145 @@
+/**
+ * Natural-language time expressions in queries.
+ *
+ * "what did we decide last week?" should not depend on the word "week" being in
+ * the stored note, so the window is parsed out of the query and applied as a
+ * recency filter instead. Pure and locale-independent in its assertions: all
+ * windows are derived from the supplied `now`.
+ */
+
+import type { TimeWindow } from "./types.ts";
+
+const DAY = 86_400_000;
+
+/** Start of the local day containing `epochMs`. */
+export function startOfDay(epochMs: number): number {
+	const date = new Date(epochMs);
+	date.setHours(0, 0, 0, 0);
+	return date.getTime();
+}
+
+/** Start of the ISO week (Monday) containing `epochMs`. */
+function startOfWeek(epochMs: number): number {
+	const date = new Date(startOfDay(epochMs));
+	const day = (date.getDay() + 6) % 7; // Monday = 0
+	date.setDate(date.getDate() - day);
+	return date.getTime();
+}
+
+/** Start of the month containing `epochMs`. */
+function startOfMonth(epochMs: number): number {
+	const date = new Date(epochMs);
+	return new Date(date.getFullYear(), date.getMonth(), 1).getTime();
+}
+
+/** Start of the year containing `epochMs`. */
+function startOfYear(epochMs: number): number {
+	return new Date(new Date(epochMs).getFullYear(), 0, 1).getTime();
+}
+
+const MONTHS = [
+	"january",
+	"february",
+	"march",
+	"april",
+	"may",
+	"june",
+	"july",
+	"august",
+	"september",
+	"october",
+	"november",
+	"december",
+];
+
+const WEEKDAYS = ["sunday", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday"];
+
+/**
+ * Parse a time expression from a query.
+ *
+ * Recognizes: today, yesterday, this week, last week, this month, last month,
+ * this year, last year, last N days/weeks/months/years, N days ago, recently /
+ * lately, this/last quarter, weekday names, and month names ("in March").
+ * Returns undefined when the query has no usable time hint.
+ */
+export function parseTimeExpression(text: string, now = Date.now()): TimeWindow | undefined {
+	const lower = ` ${text.toLowerCase()} `;
+	const relative = (since: number, label: string, until?: number): TimeWindow => ({ label, since, until });
+
+	if (/\b(today|this morning|this afternoon|tonight)\b/.test(lower)) {
+		return relative(startOfDay(now), "today", now);
+	}
+	if (/\byesterday\b/.test(lower)) {
+		const start = startOfDay(now) - DAY;
+		return relative(start, "yesterday", start + DAY);
+	}
+	if (/\b(just now|moments ago|a moment ago)\b/.test(lower)) {
+		return relative(now - 3_600_000, "the last hour", now);
+	}
+
+	const countMatch = lower.match(/\b(?:last|past|previous)\s+(\d{1,3})\s*(day|days|week|weeks|month|months|year|years)\b/);
+	const agoMatch = lower.match(/\b(\d{1,3})\s*(day|days|week|weeks|month|months|year|years)\s+ago\b/);
+	const span = countMatch ?? agoMatch;
+	if (span) {
+		const count = Number(span[1]);
+		const unit = span[2].replace(/s$/, "");
+		const ms = unit === "day" ? DAY : unit === "week" ? 7 * DAY : unit === "month" ? 30 * DAY : 365 * DAY;
+		if (Number.isFinite(count) && count > 0) return relative(now - count * ms, `the last ${count} ${span[2]}`, now);
+	}
+
+	if (/\b(this|last|past|previous)\s+week\b/.test(lower)) {
+		const thisWeek = startOfWeek(now);
+		return /\blast|previous|past\b/.test(lower)
+			? relative(thisWeek - 7 * DAY, "last week", thisWeek)
+			: relative(thisWeek, "this week", now);
+	}
+	if (/\b(this|last|past|previous)\s+month\b/.test(lower)) {
+		const thisMonth = startOfMonth(now);
+		return /\blast|previous|past\b/.test(lower)
+			? relative(startOfMonth(thisMonth - 1), "last month", thisMonth)
+			: relative(thisMonth, "this month", now);
+	}
+	if (/\b(this|last|past|previous)\s+year\b/.test(lower)) {
+		const thisYear = startOfYear(now);
+		return /\blast|previous|past\b/.test(lower)
+			? relative(startOfYear(thisYear - 1), "last year", thisYear)
+			: relative(thisYear, "this year", now);
+	}
+	const quarter = lower.match(/\b(this|last|previous)\s+quarter\b/);
+	if (quarter) {
+		const month = new Date(now).getMonth();
+		const quarterStartMonth = month - (month % 3);
+		const start = new Date(new Date(now).getFullYear(), quarterStartMonth, 1).getTime();
+		if (/last|previous/.test(quarter[1])) {
+			const previousStart = new Date(new Date(now).getFullYear(), quarterStartMonth - 3, 1).getTime();
+			return relative(previousStart, "last quarter", start);
+		}
+		return relative(start, "this quarter", now);
+	}
+	if (/\b(recently|lately|these days|the other day)\b/.test(lower)) {
+		return relative(now - 30 * DAY, "the last 30 days", now);
+	}
+
+	// Weekday names: "on Monday" means the most recent Monday.
+	for (let index = 0; index < WEEKDAYS.length; index += 1) {
+		if (!new RegExp(`\\b(on|last|this|since)?\\s*${WEEKDAYS[index]}\\b`).test(lower)) continue;
+		const start = startOfDay(now);
+		const current = (new Date(start).getDay() + 6) % 7; // Monday = 0
+		const target = (index + 6) % 7;
+		let delta = current - target;
+		if (delta < 0) delta += 7;
+		return relative(start - delta * DAY, `the last ${WEEKDAYS[index]}`, start - delta * DAY + DAY);
+	}
+
+	// Month names, with an optional 4-digit year: "in March", "since March 2024".
+	for (let index = 0; index < MONTHS.length; index += 1) {
+		const pattern = new RegExp(`\\b(?:in|during|since|from)\\s+${MONTHS[index]}(?:\\s+(\\d{4}))?\\b`);
+		const match = lower.match(pattern);
+		if (!match) continue;
+		const year = match[1] ? Number(match[1]) : new Date(now).getFullYear();
+		const start = new Date(year, index, 1).getTime();
+		const end = new Date(year, index + 1, 1).getTime();
+		return { label: `${MONTHS[index][0].toUpperCase()}${MONTHS[index].slice(1)}${match[1] ? ` ${year}` : ""}`, since: start, until: end };
+	}
+	return undefined;
+}

package/src/tokenize.ts

@@ -0,0 +1,295 @@
+/**
+ * Tokenizer for memoria.
+ *
+ * Design goals:
+ * - Exact recall for identifiers, paths, versions and error codes.
+ * - Tolerance for plural/verb/derivational variants via a conservative stemmer.
+ * - CJK support via unigrams + bigrams.
+ * - Zero dependencies and fast enough to re-index thousands of notes per second.
+ *
+ * Both the raw token and its stem are emitted, so exact matches always outrank
+ * stemmed coincidences.
+ */
+
+export interface TokenizeOptions {
+	/** Drop common English stopwords. Default true. */
+	stopwords?: boolean;
+	/** Emit stemmed variants. Default true. */
+	stem?: boolean;
+	/** Minimum length for latin tokens. Default 2. */
+	minLength?: number;
+	/** Hard cap on emitted unique tokens. Default 6000. */
+	maxTokens?: number;
+}
+
+const STOPWORDS = new Set(
+	`a about above after again against all also am an and any are aren't as at be because been before being below between both but by can cannot can't could couldn't did didn't do does doesn't doing don't down during each few for from further had hadn't has hasn't have haven't having he her here hers herself him himself his how i if in into is isn't it its itself just let's me more most mustn't my myself no nor not of off on once only or other ought our ours ourselves out over own same shan't she should shouldn't so some such than that the their theirs them themselves then there these they this those through to too under until up very was wasn't we were weren't what when where which while who whom why with won't would wouldn't you your yours yourself yourselves`.split(
+		/\s+/,
+	),
+);
+
+const CJK_CHAR_RE = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]/;
+const IDENT_CHAR_RE = /[\p{L}\p{N}_]/u;
+const IDENT_SEPARATOR_RE = /[._/\\-]/;
+const HAN_RE = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/;
+const HANGUL_RE = /[\uac00-\ud7af]/;
+
+type CjkClass = "han" | "kana" | "hangul";
+
+function cjkClass(ch: string): CjkClass | undefined {
+	if (HAN_RE.test(ch)) return "han";
+	if (HANGUL_RE.test(ch)) return "hangul";
+	if (CJK_CHAR_RE.test(ch)) return "kana";
+	return undefined;
+}
+
+export function isCjkChar(ch: string): boolean {
+	return CJK_CHAR_RE.test(ch);
+}
+
+/** Light, targeted stemmer. Conservative on purpose: false merges hurt precision. */
+export function stem(token: string): string {
+	let t = token;
+	if (t.length < 4) return t;
+	const rules: Array<[RegExp, string]> = [
+		[/sses$/, "ss"],
+		[/ies$/, "y"],
+		[/izations?$/, "iz"],
+		[/isations?$/, "is"],
+		[/izing$/, "iz"],
+		[/ized$/, "iz"],
+		[/izes$/, "iz"],
+		[/ize$/, "iz"],
+		[/ising$/, "is"],
+		[/ised$/, "is"],
+		[/ises$/, "is"],
+		[/ise$/, "is"],
+		[/ments?$/, "ment"],
+		[/nesses$/, "ness"],
+		[/ness$/, "ness"],
+		[/ingly$/, ""],
+		[/edly$/, ""],
+		[/ing$/, ""],
+		[/ed$/, ""],
+		[/ly$/, ""],
+		[/s$/, ""],
+	];
+	for (const [pattern, replacement] of rules) {
+		if (!pattern.test(t)) continue;
+		if (pattern.source === "s$" && /(ss|us|is|as|os)$/.test(t)) continue;
+		const candidate = t.replace(pattern, replacement);
+		if (candidate.length < 3) continue;
+		// Undo a doubled final consonant produced by dropping a suffix: runn -> run.
+		t = /([bdfgmnprt])\1$/.test(candidate) ? candidate.slice(0, -1) : candidate;
+		break;
+	}
+	return t;
+}
+
+function splitCamelAndCase(word: string): string[] {
+	if (!/[A-Z]/.test(word)) return [word];
+	const parts = word
+		.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+		.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
+		.split(/\s+/)
+		.filter(Boolean);
+	return parts.length > 1 ? parts : [word];
+}
+
+/** Strip diacritics: `café` -> `cafe`. */
+function stripDiacritics(input: string): string {
+	return input.normalize("NFKD").replace(/[\u0300-\u036f]/g, "");
+}
+
+/**
+ * Split text into raw surface tokens before stopword removal and stemming.
+ *
+ * Identifier runs (paths, dotted names, snake_case, kebab-case, camelCase) are
+ * emitted whole and also decomposed, so both `src/core/index.ts` and `index.ts`
+ * are searchable. CJK runs are split by script and emitted as unigrams, bigrams
+ * and the full run.
+ */
+export function splitSurfaceTokens(text: string): string[] {
+	const out: string[] = [];
+	const source = stripDiacritics(text);
+
+	const emitRun = (run: string) => {
+		// Deduplicate only within one run so repeated words keep their frequency.
+		const local = new Set<string>();
+		const pushLocal = (value: string) => {
+			const lowered = value.toLowerCase();
+			if (!lowered || local.has(lowered)) return;
+			local.add(lowered);
+			out.push(lowered);
+		};
+		pushLocal(run);
+		// Split on path separators first so `index.ts` survives as a segment,
+		// then decompose each segment further.
+		for (const segment of run.split(/[/\\]+/)) {
+			if (!segment) continue;
+			if (segment !== run) pushLocal(segment);
+			for (const camelPart of splitCamelAndCase(segment)) {
+				if (camelPart !== segment) pushLocal(camelPart);
+				const subs = camelPart.split(/[._-]+/).filter(Boolean);
+				if (subs.length > 1) for (const sub of subs) if (sub !== camelPart) pushLocal(sub);
+			}
+		}
+	};
+
+	let latin = "";
+	const flushLatin = () => {
+		if (latin) emitRun(latin);
+		latin = "";
+	};
+
+	let i = 0;
+	while (i < source.length) {
+		const ch = source[i];
+		const script = cjkClass(ch);
+		if (script) {
+			flushLatin();
+			let j = i;
+			while (j < source.length && cjkClass(source[j]) === script) j += 1;
+			const run = source.slice(i, j);
+			if (run.length <= 12) out.push(run.toLowerCase());
+			const local = new Set<string>();
+			for (let k = 0; k < run.length; k += 1) {
+				const unigram = run[k].toLowerCase();
+				if (!local.has(unigram)) {
+					local.add(unigram);
+					out.push(unigram);
+				}
+				if (k + 1 < run.length) {
+					const bigram = run.slice(k, k + 2).toLowerCase();
+					if (!local.has(bigram)) {
+						local.add(bigram);
+						out.push(bigram);
+					}
+				}
+			}
+			i = j;
+			continue;
+		}
+		if (IDENT_CHAR_RE.test(ch)) {
+			latin += ch;
+			i += 1;
+			continue;
+		}
+		if (IDENT_SEPARATOR_RE.test(ch) && latin && i + 1 < source.length && IDENT_CHAR_RE.test(source[i + 1])) {
+			latin += ch;
+			i += 1;
+			continue;
+		}
+		flushLatin();
+		i += 1;
+	}
+	flushLatin();
+	return out;
+}
+
+function acceptToken(token: string, stopwords: boolean, minLength: number): boolean {
+	if (!token) return false;
+	const isCjk = CJK_CHAR_RE.test(token);
+	if (token.length < (isCjk ? 1 : minLength)) return false;
+	if (stopwords && !isCjk && token.length <= 12 && STOPWORDS.has(token)) return false;
+	return true;
+}
+
+/** Tokenize into unique surface terms (no stemming). Used for query analysis. */
+export function tokenizeRaw(text: string, options: TokenizeOptions = {}): string[] {
+	const { stopwords = true, minLength = 2, maxTokens = 6000 } = options;
+	const out: string[] = [];
+	const seen = new Set<string>();
+	for (const token of splitSurfaceTokens(text)) {
+		if (!acceptToken(token, stopwords, minLength)) continue;
+		if (seen.has(token)) continue;
+		seen.add(token);
+		out.push(token);
+		if (out.length >= maxTokens) break;
+	}
+	return out;
+}
+
+/** Tokenize into unique surface + stemmed terms. Used for query analysis. */
+export function tokenize(text: string, options: TokenizeOptions = {}): string[] {
+	const { stem: doStem = true } = options;
+	const base = tokenizeRaw(text, options);
+	if (!doStem) return base;
+	const out: string[] = [];
+	const seen = new Set<string>();
+	for (const token of base) {
+		if (!seen.has(token)) {
+			seen.add(token);
+			out.push(token);
+		}
+		const reduced = stem(token);
+		if (reduced !== token && !seen.has(reduced)) {
+			seen.add(reduced);
+			out.push(reduced);
+		}
+	}
+	return out;
+}
+
+/** Raw term frequencies (surface tokens only) with stopword/min-length filtering. */
+export function termCounts(text: string, options: TokenizeOptions = {}): Map<string, number> {
+	const { stopwords = true, minLength = 2, maxTokens = 6000 } = options;
+	const counts = new Map<string, number>();
+	for (const token of splitSurfaceTokens(text)) {
+		if (!acceptToken(token, stopwords, minLength)) continue;
+		counts.set(token, (counts.get(token) ?? 0) + 1);
+		if (counts.size >= maxTokens) break;
+	}
+	return counts;
+}
+
+/**
+ * Tokenize `text` and accumulate weighted frequencies into `target`.
+ * Field boosts are applied by calling this once per field with a multiplier.
+ */
+export function accumulateTokens(
+	text: string,
+	weight: number,
+	target: Record<string, number>,
+	options: TokenizeOptions = {},
+): void {
+	if (!text) return;
+	const { stem: doStem = true } = options;
+	for (const [token, count] of termCounts(text, options)) {
+		const weighted = count * weight;
+		target[token] = (target[token] ?? 0) + weighted;
+		if (doStem) {
+			const reduced = stem(token);
+			if (reduced !== token) target[reduced] = (target[reduced] ?? 0) + weighted * 0.9;
+		}
+	}
+}
+
+/** Character trigrams of a term, used for fuzzy term lookup. */
+export function termTrigrams(term: string): string[] {
+	const padded = `^${term}$`;
+	const out: string[] = [];
+	for (let i = 0; i + 3 <= padded.length; i += 1) out.push(padded.slice(i, i + 3));
+	return out;
+}
+
+/** Levenshtein distance with an early-out band. */
+export function editDistance(a: string, b: string, max = 2): number {
+	if (a === b) return 0;
+	if (Math.abs(a.length - b.length) > max) return max + 1;
+	const prev = new Array<number>(b.length + 1);
+	const curr = new Array<number>(b.length + 1);
+	for (let j = 0; j <= b.length; j += 1) prev[j] = j;
+	for (let i = 1; i <= a.length; i += 1) {
+		curr[0] = i;
+		let rowMin = curr[0];
+		for (let j = 1; j <= b.length; j += 1) {
+			const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+			curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
+			if (curr[j] < rowMin) rowMin = curr[j];
+		}
+		if (rowMin > max) return max + 1;
+		for (let j = 0; j <= b.length; j += 1) prev[j] = curr[j];
+	}
+	return prev[b.length];
+}