pi-mem 0.1.0

package/observer-agent.ts

@@ -0,0 +1,266 @@
+/**
+ * Observer agent for pi-mem.
+ * Spawns a headless pi sub-agent per tool execution to extract
+ * structured observations from full tool output.
+ */
+
+import { spawn, type ChildProcess } from "node:child_process";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+import { CODE_MODE } from "./mode-config.js";
+import { parseObservation, type ParsedObservation } from "./xml-parser.js";
+import type { PiMemConfig } from "./config.js";
+
+const DEBUG_LOG_PATH = path.join(os.homedir(), ".pi-mem", "debug-observer.log");
+
+function debugLog(msg: string) {
+	try {
+		fs.appendFileSync(DEBUG_LOG_PATH, `[${new Date().toISOString()}] ${msg}\n`);
+	} catch {}
+}
+
+// Tools that are pi-mem's own — skip to avoid meta-observations
+const SKIP_TOOLS = new Set([
+	"search",
+	"timeline",
+	"get_observations",
+	"save_memory",
+]);
+
+const MIN_OUTPUT_LENGTH = 50;
+
+// ─── Skip Logic ───────────────────────────────────────────────
+
+/**
+ * Check if a tool execution should be skipped (no observer call).
+ */
+export function shouldSkipObservation(
+	toolName: string,
+	output: string,
+): boolean {
+	if (SKIP_TOOLS.has(toolName)) return true;
+	if (output.length < MIN_OUTPUT_LENGTH) return true;
+	return false;
+}
+
+// ─── Prompt Building ──────────────────────────────────────────
+
+/**
+ * Build the observer system prompt from CODE_MODE config.
+ */
+function buildSystemPrompt(): string {
+	const p = CODE_MODE.prompts;
+	return [
+		p.system_identity,
+		"",
+		p.observer_role,
+		"",
+		p.recording_focus,
+		"",
+		p.skip_guidance,
+		"",
+		p.type_guidance,
+		"",
+		p.concept_guidance,
+		"",
+		p.field_guidance,
+		"",
+		p.xml_format,
+		"",
+		p.footer,
+	].join("\n");
+}
+
+/**
+ * Build the user prompt for a single tool execution.
+ */
+export function buildObserverPrompt(
+	toolName: string,
+	input: Record<string, unknown>,
+	output: string,
+): string {
+	return `<observed_from_primary_session>
+  <what_happened>${toolName}</what_happened>
+  <occurred_at>${new Date().toISOString()}</occurred_at>
+  <parameters>${JSON.stringify(input, null, 2)}</parameters>
+  <outcome>${output}</outcome>
+</observed_from_primary_session>`;
+}
+
+// ─── Sub-Agent Spawn ──────────────────────────────────────────
+
+function killProcess(proc: ChildProcess): void {
+	try {
+		proc.kill("SIGTERM");
+	} catch {}
+	setTimeout(() => {
+		try {
+			proc.kill("SIGKILL");
+		} catch {}
+	}, 2000);
+}
+
+/**
+ * Run a pi sub-agent and return the response text.
+ */
+function runSubAgent(
+	prompt: string,
+	systemPrompt: string,
+	model: string,
+	thinkingLevel: string,
+): Promise<{ ok: true; response: string } | { ok: false; error: string }> {
+	return new Promise((resolve) => {
+		const proc = spawn(
+			"pi",
+			[
+				"--mode", "json",
+				"-p",
+				"--no-session",
+				"--no-tools",
+				"--system-prompt", systemPrompt,
+				"--model", model,
+				"--thinking", thinkingLevel,
+				prompt,
+			],
+			{
+				stdio: ["ignore", "pipe", "pipe"],
+				env: { ...process.env, PI_MEM_SUB_AGENT: "1" },
+			},
+		);
+
+		let buffer = "";
+		let lastAssistantText = "";
+		let stderr = "";
+
+		const timeout = setTimeout(() => {
+			killProcess(proc);
+			resolve({ ok: false, error: "Observer timeout (30s)" });
+		}, 30_000);
+
+		const processLine = (line: string) => {
+			if (!line.trim()) return;
+			try {
+				const event = JSON.parse(line);
+				if (
+					event.type === "message_end" &&
+					event.message?.role === "assistant"
+				) {
+					for (const part of event.message.content) {
+						if (part.type === "text") {
+							lastAssistantText = part.text;
+						}
+					}
+				}
+			} catch {
+				// ignore non-JSON lines
+			}
+		};
+
+		proc.stdout!.on("data", (data: Buffer) => {
+			buffer += data.toString();
+			const lines = buffer.split("\n");
+			buffer = lines.pop() || "";
+			for (const line of lines) processLine(line);
+		});
+
+		proc.stderr!.on("data", (data: Buffer) => {
+			stderr += data.toString();
+		});
+
+		proc.on("close", (code) => {
+			clearTimeout(timeout);
+			if (buffer.trim()) processLine(buffer);
+
+			if (lastAssistantText) {
+				resolve({ ok: true, response: lastAssistantText });
+			} else if (code !== 0) {
+				resolve({
+					ok: false,
+					error: `Sub-agent failed (exit ${code}): ${stderr.trim().slice(0, 500) || "(no output)"}`,
+				});
+			} else {
+				resolve({ ok: false, error: "Sub-agent returned no response" });
+			}
+		});
+
+		proc.on("error", (err) => {
+			clearTimeout(timeout);
+			resolve({ ok: false, error: `Failed to spawn pi: ${err.message}` });
+		});
+	});
+}
+
+// ─── Main Entry Point ─────────────────────────────────────────
+
+export interface ObserverContext {
+	/** Current session model */
+	model: any;
+	/** Current session thinking level */
+	thinkingLevel: string;
+}
+
+/**
+ * Extract a structured observation from a tool execution.
+ * Spawns observer LLM, parses XML output.
+ * Returns null if extraction fails or is skipped.
+ */
+export async function extractObservation(
+	toolName: string,
+	input: Record<string, unknown>,
+	output: string,
+	config: PiMemConfig,
+	context: ObserverContext,
+): Promise<ParsedObservation | null> {
+	// Skip logic
+	if (shouldSkipObservation(toolName, output)) {
+		return null;
+	}
+
+	// Resolve model: observerModel → summaryModel → session model
+	const model =
+		config.observerModel ||
+		config.summaryModel ||
+		(context.model
+			? `${context.model.provider}/${context.model.id}`
+			: undefined);
+
+	if (!model) {
+		debugLog("No model available for observer extraction");
+		return null;
+	}
+
+	const thinkingLevel = config.thinkingLevel || context.thinkingLevel || "none";
+
+	const systemPrompt = buildSystemPrompt();
+	const userPrompt = buildObserverPrompt(toolName, input, output);
+
+	debugLog(
+		`Extracting observation: ${toolName} (output: ${output.length} chars, model: ${model})`,
+	);
+
+	const result = await runSubAgent(
+		userPrompt,
+		systemPrompt,
+		model,
+		thinkingLevel,
+	);
+
+	if (!result.ok) {
+		debugLog(`Observer failed: ${result.error}`);
+		return null;
+	}
+
+	debugLog(
+		`Observer response: ${result.response.length} chars`,
+	);
+
+	const parsed = parseObservation(result.response);
+	if (!parsed) {
+		debugLog("Observer returned no parseable observation XML");
+		return null;
+	}
+
+	debugLog(`Extracted: [${parsed.type}] ${parsed.title}`);
+	return parsed;
+}

package/observer.ts

@@ -0,0 +1,11 @@
+/**
+ * Observation utilities for pi-mem.
+ * Privacy filtering helper.
+ */
+
+/**
+ * Strip <private>...</private> tags from text, replacing with [REDACTED].
+ */
+export function stripPrivateTags(text: string): string {
+	return text.replace(/<private>[\s\S]*?<\/private>/g, "[REDACTED]");
+}

package/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "pi-mem",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Persistent memory extension for pi — captures observations, compresses them into searchable memories, and injects context into future sessions",
+  "keywords": ["pi-package"],
+  "author": "George Bashi",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/georgebashi/pi-mem.git"
+  },
+  "homepage": "https://github.com/georgebashi/pi-mem",
+  "bugs": {
+    "url": "https://github.com/georgebashi/pi-mem/issues"
+  },
+  "pi": {
+    "extensions": [
+      "./index.ts"
+    ]
+  },
+  "files": [
+    "*.ts",
+    "README.md",
+    "LICENSE",
+    "!test-*.ts",
+    "!project.test.ts",
+    "!*.mjs"
+  ],
+  "peerDependencies": {
+    "@mariozechner/pi-coding-agent": "*",
+    "@sinclair/typebox": "*"
+  },
+  "dependencies": {
+    "@lancedb/lancedb": "^0.26.2"
+  }
+}

package/privacy.ts

@@ -0,0 +1,69 @@
+/**
+ * Privacy filtering for pi-mem.
+ * Loads .pi-mem-ignore patterns and checks file paths.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { PI_MEM_DIR } from "./config.js";
+
+/**
+ * Load ignore patterns from .pi-mem-ignore files.
+ * Checks project root and ~/.pi-mem/ for patterns.
+ */
+export function loadIgnorePatterns(cwd: string): string[] {
+	const patterns: string[] = [];
+
+	const locations = [
+		path.join(cwd, ".pi-mem-ignore"),
+		path.join(PI_MEM_DIR, ".pi-mem-ignore"),
+	];
+
+	for (const loc of locations) {
+		try {
+			if (fs.existsSync(loc)) {
+				const content = fs.readFileSync(loc, "utf-8");
+				const lines = content.split("\n")
+					.map((l) => l.trim())
+					.filter((l) => l && !l.startsWith("#"));
+				patterns.push(...lines);
+			}
+		} catch {
+			// Ignore read errors
+		}
+	}
+
+	return patterns;
+}
+
+/**
+ * Check if a file path matches any ignore patterns.
+ * Uses simple glob-style matching (*.ext, exact names).
+ */
+export function shouldIgnorePath(filePath: string, patterns: string[]): boolean {
+	if (patterns.length === 0) return false;
+
+	const basename = path.basename(filePath);
+
+	for (const pattern of patterns) {
+		// Exact match
+		if (basename === pattern || filePath === pattern) return true;
+
+		// Glob match: *.ext
+		if (pattern.startsWith("*.")) {
+			const ext = pattern.slice(1); // ".ext"
+			if (basename.endsWith(ext)) return true;
+		}
+
+		// Directory match: dir/
+		if (pattern.endsWith("/")) {
+			const dir = pattern.slice(0, -1);
+			if (filePath.includes(`/${dir}/`) || filePath.startsWith(`${dir}/`)) return true;
+		}
+
+		// Contains match
+		if (filePath.includes(pattern)) return true;
+	}
+
+	return false;
+}

package/project.ts

@@ -0,0 +1,64 @@
+/**
+ * Project identity detection.
+ * Derives a project slug from git remote or cwd basename.
+ */
+
+import { execSync } from "node:child_process";
+import * as path from "node:path";
+
+/**
+ * Normalize a git remote URL to a filesystem-safe slug.
+ *
+ * Examples:
+ *   git@github.com:user/repo.git   → github.com-user-repo
+ *   https://github.com/user/repo   → github.com-user-repo
+ *   ssh://git@github.com/user/repo → github.com-user-repo
+ */
+export function normalizeRemoteUrl(url: string): string {
+	let normalized = url.trim();
+
+	// Remove .git suffix
+	normalized = normalized.replace(/\.git$/, "");
+
+	// Handle SSH format: git@host:user/repo
+	const sshMatch = normalized.match(/^[\w.-]+@([\w.-]+):(.+)$/);
+	if (sshMatch) {
+		normalized = `${sshMatch[1]}/${sshMatch[2]}`;
+	} else {
+		// Handle protocol-based URLs: https://host/path, ssh://git@host/path
+		normalized = normalized.replace(/^[a-zA-Z+]+:\/\//, "");
+		// Remove user@ prefix
+		normalized = normalized.replace(/^[^@]+@/, "");
+	}
+
+	// Replace / and : with -
+	normalized = normalized.replace(/[/:]/g, "-");
+
+	// Remove leading/trailing dashes
+	normalized = normalized.replace(/^-+|-+$/g, "");
+
+	return normalized;
+}
+
+/**
+ * Get the project slug for the given working directory.
+ * Tries git remote first, falls back to directory basename.
+ */
+export async function getProjectSlug(cwd: string): Promise<string> {
+	try {
+		const remote = execSync("git remote get-url origin", {
+			cwd,
+			encoding: "utf-8",
+			stdio: ["ignore", "pipe", "ignore"],
+			timeout: 5000,
+		}).trim();
+
+		if (remote) {
+			return normalizeRemoteUrl(remote);
+		}
+	} catch {
+		// Not a git repo or no remote
+	}
+
+	return path.basename(cwd);
+}

package/tools.ts

@@ -0,0 +1,258 @@
+/**
+ * Tool registration for pi-mem.
+ * Registers search, timeline, get_observations, and save_memory tools.
+ * Implements a 3-layer progressive disclosure pattern for memory search.
+ */
+
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { Type } from "@sinclair/typebox";
+import { StringEnum } from "@mariozechner/pi-ai";
+import type { IndexResult, TimelineResult, FullResult } from "./observation-store.js";
+
+// ─── Result Formatters ────────────────────────────────────────
+
+function formatIndexResults(results: IndexResult[]): string {
+	if (results.length === 0) {
+		return "No results found. Try broader search terms or different filters.";
+	}
+
+	const header = "| id | timestamp | type | tool | title |";
+	const sep = "|---|---|---|---|---|";
+	const rows = results.map(
+		(r) =>
+			`| ${r.id} | ${r.timestamp.slice(0, 16)} | ${r.obs_type || r.type} | ${r.tool_name || "-"} | ${r.title.slice(0, 60)} |`,
+	);
+
+	return [
+		`Found ${results.length} result(s):`,
+		"",
+		header,
+		sep,
+		...rows,
+		"",
+		"Use `timeline(anchor=ID)` for context or `get_observations(ids=[...])` for full details.",
+	].join("\n");
+}
+
+function formatTimelineResults(results: TimelineResult[]): string {
+	if (results.length === 0) {
+		return "No timeline results found.";
+	}
+
+	const lines = results.map((r) => {
+		const tool = r.tool_name ? ` [${r.tool_name}]` : "";
+		const preview = r.narrative_preview ? `\n  ${r.narrative_preview}` : "";
+		const sub = r.subtitle ? ` — ${r.subtitle}` : "";
+		return `**#${r.id}** ${r.timestamp.slice(0, 16)} ${r.obs_type || r.type}${tool} — ${r.title}${sub}${preview}`;
+	});
+
+	return [
+		`Timeline (${results.length} entries):`,
+		"",
+		...lines,
+		"",
+		"Use `get_observations(ids=[...])` for full details on specific entries.",
+	].join("\n");
+}
+
+function formatFullResults(results: FullResult[]): string {
+	if (results.length === 0) {
+		return "No observations found for the given IDs.";
+	}
+
+	const entries = results.map((r) => {
+		const parts = [
+			`## #${r.id} — ${r.title}`,
+			`**Type:** ${r.obs_type || r.type} | **Session:** ${r.session_id} | **Project:** ${r.project}`,
+			`**Timestamp:** ${r.timestamp}`,
+		];
+		if (r.tool_name) parts.push(`**Tool:** ${r.tool_name}`);
+		if (r.subtitle) parts.push(`**Subtitle:** ${r.subtitle}`);
+		if (r.concepts.length > 0) parts.push(`**Concepts:** ${r.concepts.join(", ")}`);
+		if (r.files_read.length > 0) parts.push(`**Files Read:** ${r.files_read.join(", ")}`);
+		if (r.files_modified.length > 0) parts.push(`**Files Modified:** ${r.files_modified.join(", ")}`);
+		if (r.facts.length > 0) {
+			parts.push("", "**Facts:**");
+			for (const fact of r.facts) {
+				parts.push(`- ${fact}`);
+			}
+		}
+		if (r.narrative) parts.push("", r.narrative);
+		return parts.join("\n");
+	});
+
+	return entries.join("\n\n---\n\n");
+}
+
+// ─── Tool Registration ───────────────────────────────────────
+
+export interface ToolCallbacks {
+	onSearch: (params: any) => Promise<IndexResult[]>;
+	onTimeline: (params: any) => Promise<TimelineResult[]>;
+	onGetObservations: (params: any) => Promise<FullResult[]>;
+	onSaveMemory: (params: any) => Promise<string>;
+}
+
+export function registerTools(pi: ExtensionAPI, callbacks: ToolCallbacks): void {
+	// ─── search ───────────────────────────────────────────────
+	pi.registerTool({
+		name: "search",
+		label: "Memory Search",
+		description:
+			"Step 1: Search memory. Returns compact index with IDs. " +
+			"Params: query, limit, project, type, obs_type, dateStart, dateEnd, offset, orderBy",
+		parameters: Type.Object({
+			query: Type.String({ description: "Full-text search query (supports AND, OR, NOT)" }),
+			limit: Type.Optional(Type.Number({ description: "Max results, default 20", default: 20 })),
+			offset: Type.Optional(Type.Number({ description: "Skip first N results for pagination", default: 0 })),
+			project: Type.Optional(Type.String({ description: "Filter by project slug" })),
+			obs_type: Type.Optional(
+				StringEnum(["observation", "summary", "prompt", "manual"] as const, {
+					description: "Filter by record type",
+				}),
+			),
+			dateStart: Type.Optional(Type.String({ description: "Filter by start date (YYYY-MM-DD)" })),
+			dateEnd: Type.Optional(Type.String({ description: "Filter by end date (YYYY-MM-DD)" })),
+			orderBy: Type.Optional(
+				StringEnum(["date_desc", "date_asc", "relevance"] as const, {
+					description: "Sort order (default: relevance for FTS)",
+				}),
+			),
+		}),
+
+		async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
+			try {
+				const results = await callbacks.onSearch(params);
+				return {
+					content: [{ type: "text", text: formatIndexResults(results) }],
+					details: {},
+				};
+			} catch (e: any) {
+				return {
+					content: [{ type: "text", text: `Error searching: ${e.message}` }],
+					isError: true,
+					details: {},
+				};
+			}
+		},
+	});
+
+	// ─── timeline ─────────────────────────────────────────────
+	pi.registerTool({
+		name: "timeline",
+		label: "Memory Timeline",
+		description:
+			"Step 2: Get chronological context around a result. " +
+			"Provide anchor (observation ID) OR query to find the anchor automatically.",
+		parameters: Type.Object({
+			anchor: Type.Optional(Type.String({ description: "Observation ID to center timeline around" })),
+			query: Type.Optional(Type.String({ description: "Search query to find anchor automatically" })),
+			depth_before: Type.Optional(Type.Number({ description: "Observations before anchor, default 3", default: 3 })),
+			depth_after: Type.Optional(Type.Number({ description: "Observations after anchor, default 3", default: 3 })),
+			project: Type.Optional(Type.String({ description: "Filter by project slug" })),
+		}),
+
+		async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
+			if (!params.anchor && !params.query) {
+				return {
+					content: [{ type: "text", text: "Provide either 'anchor' (observation ID) or 'query' to find the anchor." }],
+					isError: true,
+					details: {},
+				};
+			}
+
+			try {
+				const results = await callbacks.onTimeline(params);
+				return {
+					content: [{ type: "text", text: formatTimelineResults(results) }],
+					details: {},
+				};
+			} catch (e: any) {
+				return {
+					content: [{ type: "text", text: `Error building timeline: ${e.message}` }],
+					isError: true,
+					details: {},
+				};
+			}
+		},
+	});
+
+	// ─── get_observations ─────────────────────────────────────
+	pi.registerTool({
+		name: "get_observations",
+		label: "Get Observations",
+		description:
+			"Step 3: Fetch full details for specific IDs. Always batch multiple IDs in a single call.",
+		parameters: Type.Object({
+			ids: Type.Array(Type.String(), {
+				description: "Array of observation IDs to fetch (required)",
+			}),
+			orderBy: Type.Optional(
+				StringEnum(["date_desc", "date_asc"] as const, {
+					description: "Sort order",
+				}),
+			),
+			limit: Type.Optional(Type.Number({ description: "Maximum observations to return" })),
+			project: Type.Optional(Type.String({ description: "Filter by project slug" })),
+		}),
+
+		async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
+			if (!params.ids || params.ids.length === 0) {
+				return {
+					content: [{ type: "text", text: "Provide at least one observation ID." }],
+					isError: true,
+					details: {},
+				};
+			}
+
+			try {
+				const results = await callbacks.onGetObservations(params);
+				return {
+					content: [{ type: "text", text: formatFullResults(results) }],
+					details: {},
+				};
+			} catch (e: any) {
+				return {
+					content: [{ type: "text", text: `Error fetching observations: ${e.message}` }],
+					isError: true,
+					details: {},
+				};
+			}
+		},
+	});
+
+	// ─── save_memory ──────────────────────────────────────────
+	pi.registerTool({
+		name: "save_memory",
+		label: "Save Memory",
+		description:
+			"Save important information to memory for future sessions. " +
+			"Use for decisions, discoveries, or context that should be remembered.",
+		parameters: Type.Object({
+			text: Type.String({ description: "Content to remember (required)" }),
+			title: Type.Optional(Type.String({ description: "Short title (auto-generated if omitted)" })),
+			project: Type.Optional(Type.String({ description: "Project slug (defaults to current)" })),
+			concepts: Type.Optional(
+				Type.Array(Type.String(), {
+					description: 'Concept tags, e.g. ["decision", "architecture"]',
+				}),
+			),
+		}),
+
+		async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
+			try {
+				const result = await callbacks.onSaveMemory(params);
+				return {
+					content: [{ type: "text", text: result }],
+					details: {},
+				};
+			} catch (e: any) {
+				return {
+					content: [{ type: "text", text: `Error saving memory: ${e.message}` }],
+					isError: true,
+					details: {},
+				};
+			}
+		},
+	});
+}