pi-hermes-memory 0.1.0 → 0.2.1

package/src/index.ts

@@ -7,48 +7,101 @@
  * 1. Persistent Memory — MEMORY.md + USER.md that survive across sessions
  * 2. Background Learning Loop — auto-saves notable facts every N turns
  * 3. Session-End Flush — saves memories before compaction/shutdown
- * 4. /memory-insights — shows what's stored
+ * 4. Auto-Consolidation — merges memory when full instead of erroring
+ * 5. Correction Detection — immediate save on user corrections
+ * 6. Procedural Skills — SKILL.md files for reusable procedures
+ * 7. Tool-Call-Aware Nudge — review triggers on tool call count too
+ * 8. /memory-insights — shows what's stored
+ * 9. /memory-skills — lists procedural skills
+ * 10. /memory-consolidate — manual consolidation trigger
  *
- * See PLAN.md for full architecture and Hermes source references.
+ * See docs/ROADMAP.md for full roadmap and Hermes competitive analysis.
  */
 
+import * as path from "node:path";
+import * as os from "node:os";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { MemoryStore } from "./store/memory-store.js";
+import { SkillStore } from "./store/skill-store.js";
 import { registerMemoryTool } from "./tools/memory-tool.js";
+import { registerSkillTool } from "./tools/skill-tool.js";
 import { setupBackgroundReview } from "./handlers/background-review.js";
 import { setupSessionFlush } from "./handlers/session-flush.js";
 import { registerInsightsCommand } from "./handlers/insights.js";
+import { triggerConsolidation, registerConsolidateCommand } from "./handlers/auto-consolidate.js";
+import { setupCorrectionDetector } from "./handlers/correction-detector.js";
+import { setupSkillAutoTrigger } from "./handlers/skill-auto-trigger.js";
+import { registerSkillsCommand } from "./handlers/skills-command.js";
 import { loadConfig } from "./config.js";
 
 export default function (pi: ExtensionAPI) {
   const config = loadConfig();
 
+  const globalDir = config.memoryDir ?? path.join(os.homedir(), ".pi", "agent", "memory");
   const store = new MemoryStore(config);
+  const skillStore = new SkillStore(path.join(globalDir, "skills"));
+
+  // Detect project name from cwd — skip if running from home directory
+  const cwd = process.cwd();
+  const homeDir = os.homedir();
+  const projectName = path.basename(cwd);
+  const hasProject = cwd !== homeDir;
+
+  // Project-scoped store: ~/.pi/agent/<project_name>/
+  // Uses memoryCharLimit overridden to projectCharLimit for the "memory" target
+  const projectDir = hasProject ? path.join(homeDir, ".pi", "agent", projectName) : null;
+  const projectConfig = { ...config, memoryCharLimit: config.projectCharLimit, memoryDir: projectDir ?? undefined };
+  const projectStore = hasProject ? new MemoryStore(projectConfig) : null;
 
   // ── 1. Load memory from disk on session start ──
   pi.on("session_start", async (_event, _ctx) => {
     await store.loadFromDisk();
+    if (projectStore) await projectStore.loadFromDisk();
   });
 
-  // ── 2. Inject frozen snapshot into system prompt ──
+  // ── 2. Inject frozen snapshot + skill index + project memory into system prompt ──
   pi.on("before_agent_start", async (event, _ctx) => {
     const memoryBlock = store.formatForSystemPrompt();
-    if (memoryBlock) {
+    const skillIndex = await skillStore.formatIndexForSystemPrompt();
+    const projectBlock = projectStore ? projectStore.formatProjectBlock(projectName) : "";
+
+    const parts: string[] = [];
+    if (memoryBlock) parts.push(memoryBlock);
+    if (projectBlock) parts.push(projectBlock);
+    if (skillIndex) parts.push(skillIndex);
+
+    if (parts.length > 0) {
       return {
-        systemPrompt: event.systemPrompt + "\n\n" + memoryBlock,
+        systemPrompt: event.systemPrompt + "\n\n" + parts.join("\n\n"),
       };
     }
   });
 
-  // ── 3. Register the memory tool ──
-  registerMemoryTool(pi, store);
+  // ── 3. Register the memory tool (with project store) ──
+  registerMemoryTool(pi, store, projectStore);
+
+  // ── 4. Register the skill tool ──
+  registerSkillTool(pi, skillStore);
+
+  // ── 5. Setup background learning loop (with tool-call-aware nudge) ──
+  setupBackgroundReview(pi, store, projectStore, config);
+
+  // ── 6. Setup session-end flush ──
+  setupSessionFlush(pi, store, projectStore, config);
+
+  // ── 7. Setup auto-consolidation (inject consolidator into store) ──
+  store.setConsolidator(async (target, signal) => {
+    return triggerConsolidation(pi, store, target, signal);
+  });
+  registerConsolidateCommand(pi, store);
 
-  // ── 4. Setup background learning loop ──
-  setupBackgroundReview(pi, store, config);
+  // ── 8. Setup correction detection ──
+  setupCorrectionDetector(pi, store, projectStore, config);
 
-  // ── 5. Setup session-end flush ──
-  setupSessionFlush(pi, store, config);
+  // ── 9. Setup skill auto-trigger ──
+  setupSkillAutoTrigger(pi, store, skillStore, config);
 
-  // ── 6. Register insights command ──
-  registerInsightsCommand(pi, store);
+  // ── 10. Register commands ──
+  registerInsightsCommand(pi, store, projectStore, projectName);
+  registerSkillsCommand(pi, skillStore);
 }

package/src/store/memory-store.ts

@@ -22,15 +22,24 @@ import {
   MEMORY_FILE,
   USER_FILE,
 } from "../constants.js";
-import type { MemoryConfig, MemoryResult, MemorySnapshot } from "../types.js";
+import type { MemoryConfig, MemoryResult, MemorySnapshot, ConsolidationResult } from "../types.js";
 
 export class MemoryStore {
   private memoryEntries: string[] = [];
   private userEntries: string[] = [];
   private snapshot: MemorySnapshot = { memory: "", user: "" };
+  private consolidator: ((target: "memory" | "user", signal?: AbortSignal) => Promise<ConsolidationResult>) | null = null;
 
   constructor(private config: MemoryConfig) {}
 
+  /**
+   * Inject a consolidation function (avoids circular imports).
+   * Called from index.ts after both store and pi are available.
+   */
+  setConsolidator(fn: (target: "memory" | "user", signal?: AbortSignal) => Promise<ConsolidationResult>): void {
+    this.consolidator = fn;
+  }
+
   // ─── Path helpers ───
 
   private get memoryDir(): string {
@@ -79,7 +88,7 @@ export class MemoryStore {
 
   // ─── CRUD ───
 
-  add(target: "memory" | "user", content: string): MemoryResult {
+  async add(target: "memory" | "user", content: string, signal?: AbortSignal): Promise<MemoryResult> {
     content = content.trim();
     if (!content) return { success: false, error: "Content cannot be empty." };
 
@@ -95,6 +104,32 @@ export class MemoryStore {
 
     const newTotal = [...entries, content].join(ENTRY_DELIMITER).length;
     if (newTotal > limit) {
+      // Auto-consolidate if configured and consolidator available
+      if (this.config.autoConsolidate && this.consolidator) {
+        // Track consolidation attempts to prevent infinite recursion
+        // when the consolidator fails to free enough space
+        const beforeCount = entries.length;
+        try {
+          const result = await this.consolidator(target, signal);
+          if (result.consolidated) {
+            // CRITICAL: reload from disk — child process modified files, our arrays are stale
+            await this.loadFromDisk();
+            // Guard: if consolidation didn't reduce entries, stop recursing
+            const afterEntries = this.entriesFor(target);
+            const afterCount = afterEntries.length;
+            if (afterCount >= beforeCount && afterCount > 0) {
+              return {
+                success: false,
+                error: `Memory at capacity and consolidation did not free enough space. Entry count unchanged at ${afterCount}.`,
+              };
+            }
+            // Retry the add with fresh data
+            return this.add(target, content, signal);
+          }
+        } catch {
+          // Consolidation failed — fall through to error
+        }
+      }
       const current = this.charCount(target);
       return {
         success: false,
@@ -104,12 +139,12 @@ export class MemoryStore {
 
     entries.push(content);
     this.setEntries(target, entries);
-    this.saveToDisk(target);
+    await this.saveToDisk(target);
 
     return this.successResponse(target, "Entry added.");
   }
 
-  replace(target: "memory" | "user", oldText: string, newContent: string): MemoryResult {
+  async replace(target: "memory" | "user", oldText: string, newContent: string): Promise<MemoryResult> {
     oldText = oldText.trim();
     newContent = newContent.trim();
     if (!oldText) return { success: false, error: "old_text cannot be empty." };
@@ -144,12 +179,12 @@ export class MemoryStore {
 
     entries[idx] = newContent;
     this.setEntries(target, entries);
-    this.saveToDisk(target);
+    await this.saveToDisk(target);
 
     return this.successResponse(target, "Entry replaced.");
   }
 
-  remove(target: "memory" | "user", oldText: string): MemoryResult {
+  async remove(target: "memory" | "user", oldText: string): Promise<MemoryResult> {
     oldText = oldText.trim();
     if (!oldText) return { success: false, error: "old_text cannot be empty." };
 
@@ -168,7 +203,7 @@ export class MemoryStore {
     const idx = entries.indexOf(matches[0]);
     entries.splice(idx, 1);
     this.setEntries(target, entries);
-    this.saveToDisk(target);
+    await this.saveToDisk(target);
 
     return this.successResponse(target, "Entry removed.");
   }
@@ -182,6 +217,14 @@ export class MemoryStore {
     return parts.join("\n\n");
   }
 
+  /**
+   * Render a project-specific memory block for system prompt injection.
+   * Uses only the memory entries (no user split) with a project-labelled header.
+   */
+  formatProjectBlock(projectName: string): string {
+    return this.renderProjectBlock(projectName, this.memoryEntries);
+  }
+
   getMemoryEntries(): string[] {
     return [...this.memoryEntries];
   }
@@ -224,6 +267,18 @@ export class MemoryStore {
     return `${separator}\n${header}\n${separator}\n${content}`;
   }
 
+  private renderProjectBlock(projectName: string, entries: string[]): string {
+    if (!entries.length) return "";
+    const limit = this.config.memoryCharLimit;
+    const content = entries.join(ENTRY_DELIMITER);
+    const current = content.length;
+    const pct = limit > 0 ? Math.min(100, Math.floor((current / limit) * 100)) : 0;
+
+    const header = `PROJECT MEMORY: ${projectName} [${pct}% — ${current}/${limit} chars]`;
+    const separator = "═".repeat(46);
+    return `${separator}\n${header}\n${separator}\n${content}`;
+  }
+
   private async readFile(filePath: string): Promise<string[]> {
     try {
       const raw = await fs.readFile(filePath, "utf-8");
@@ -235,23 +290,22 @@ export class MemoryStore {
   }
 
   /** Atomic write: temp file + fs.rename() — same crash-safety as Hermes. */
-  private saveToDisk(target: "memory" | "user"): void {
+  private async saveToDisk(target: "memory" | "user"): Promise<void> {
     const filePath = this.pathFor(target);
     const entries = this.entriesFor(target);
     const content = entries.length ? entries.join(ENTRY_DELIMITER) : "";
 
-    // Fire-and-forget atomic write
-    fs.mkdtemp(path.join(os.tmpdir(), "pi-memory-")).then((tmpDir) => {
-      const tmpPath = path.join(tmpDir, "write.tmp");
-      return fs
-        .writeFile(tmpPath, content, "utf-8")
-        .then(() => fs.rename(tmpPath, filePath))
-        .catch(async () => {
-          try { await fs.unlink(tmpPath); } catch { /* ignore */ }
-        })
-        .finally(async () => {
-          try { await fs.rmdir(tmpDir); } catch { /* ignore */ }
-        });
-    });
+    const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "pi-memory-"));
+    const tmpPath = path.join(tmpDir, "write.tmp");
+
+    try {
+      await fs.writeFile(tmpPath, content, "utf-8");
+      await fs.rename(tmpPath, filePath);
+    } catch (err) {
+      try { await fs.unlink(tmpPath); } catch { /* ignore */ }
+      throw err;
+    } finally {
+      try { await fs.rmdir(tmpDir); } catch { /* ignore */ }
+    }
   }
 }

package/src/store/skill-store.ts

@@ -0,0 +1,292 @@
+/**
+ * SkillStore — procedural memory stored as SKILL.md files.
+ *
+ * Skills capture HOW to do something (procedural knowledge), as opposed
+ * to MemoryStore which captures WHAT (declarative knowledge).
+ *
+ * Storage: ~/.pi/agent/memory/skills/<slug>.md
+ * Format: YAML-like frontmatter + markdown body (no yaml dependency)
+ * Progressive disclosure: index (name+description) in system prompt,
+ *   full content loaded on demand via skill tool.
+ */
+
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import * as os from "node:os";
+import { scanContent } from "./content-scanner.js";
+import type { SkillIndex, SkillDocument, SkillResult } from "../types.js";
+
+// ─── Frontmatter parsing ───
+
+function parseFrontmatter(raw: string): { meta: Record<string, string>; body: string } {
+  const match = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+  if (!match) return { meta: {}, body: raw };
+
+  const meta: Record<string, string> = {};
+  for (const line of match[1].split("\n")) {
+    const idx = line.indexOf(":");
+    if (idx > 0) {
+      const key = line.slice(0, idx).trim();
+      const value = line.slice(idx + 1).trim();
+      meta[key] = value;
+    }
+  }
+  return { meta, body: match[2].trim() };
+}
+
+function formatFrontmatter(doc: Omit<SkillDocument, "fileName">): string {
+  return [
+    "---",
+    `name: ${doc.name}`,
+    `description: ${doc.description}`,
+    `version: ${doc.version}`,
+    `created: ${doc.created}`,
+    `updated: ${doc.updated}`,
+    "---",
+    doc.body,
+  ].join("\n");
+}
+
+// ─── Slugify ───
+
+function slugify(name: string): string {
+  return name
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, 64);
+}
+
+// ─── SkillStore ───
+
+export class SkillStore {
+  private skillsDir: string;
+
+  constructor(skillsDir?: string) {
+    this.skillsDir = skillsDir ?? path.join(os.homedir(), ".pi", "agent", "memory", "skills");
+  }
+
+  // ─── Read ───
+
+  async loadIndex(): Promise<SkillIndex[]> {
+    await fs.mkdir(this.skillsDir, { recursive: true });
+    const files = await fs.readdir(this.skillsDir);
+    const skills: SkillIndex[] = [];
+
+    for (const file of files) {
+      if (!file.endsWith(".md")) continue;
+      const doc = await this.loadSkill(file);
+      if (doc) {
+        skills.push({ fileName: doc.fileName, name: doc.name, description: doc.description });
+      }
+    }
+
+    return skills;
+  }
+
+  async loadSkill(fileName: string): Promise<SkillDocument | null> {
+    try {
+      const raw = await fs.readFile(path.join(this.skillsDir, fileName), "utf-8");
+      const { meta, body } = parseFrontmatter(raw);
+      if (!meta.name) return null;
+      return {
+        fileName,
+        name: meta.name,
+        description: meta.description || "",
+        version: parseInt(meta.version || "1", 10),
+        created: meta.created || new Date().toISOString().split("T")[0],
+        updated: meta.updated || new Date().toISOString().split("T")[0],
+        body,
+      };
+    } catch {
+      return null;
+    }
+  }
+
+  // ─── Write ───
+
+  async create(name: string, description: string, body: string): Promise<SkillResult> {
+    name = name.trim();
+    description = description.trim();
+    body = body.trim();
+
+    if (!name) return { success: false, error: "Skill name is required." };
+    if (!description) return { success: false, error: "Skill description is required." };
+    if (!body) return { success: false, error: "Skill body is required." };
+
+    // Scan content for security
+    const scanError = scanContent(name + " " + description + " " + body);
+    if (scanError) return { success: false, error: scanError };
+
+    const slug = slugify(name);
+    if (!slug) return { success: false, error: "Skill name produces empty slug." };
+
+    const fileName = `${slug}.md`;
+    const filePath = path.join(this.skillsDir, fileName);
+
+    // Check if file already exists
+    try {
+      await fs.access(filePath);
+      return {
+        success: false,
+        error: `Skill '${name}' already exists (file: ${fileName}). Use 'patch' or 'edit' to update it.`,
+      };
+    } catch {
+      // File doesn't exist — good
+    }
+
+    await fs.mkdir(this.skillsDir, { recursive: true });
+
+    const today = new Date().toISOString().split("T")[0];
+    const doc: Omit<SkillDocument, "fileName"> = {
+      name,
+      description,
+      version: 1,
+      created: today,
+      updated: today,
+      body,
+    };
+
+    await this.atomicWrite(fileName, formatFrontmatter(doc));
+
+    return { success: true, message: `Skill '${name}' created.`, fileName };
+  }
+
+  async patch(fileName: string, section: string, newContent: string): Promise<SkillResult> {
+    newContent = newContent.trim();
+    if (!newContent) return { success: false, error: "New content is required for patch." };
+
+    const scanError = scanContent(newContent);
+    if (scanError) return { success: false, error: scanError };
+
+    const doc = await this.loadSkill(fileName);
+    if (!doc) return { success: false, error: `Skill file '${fileName}' not found.` };
+
+    // Replace or append the section in the body
+    const sectionHeader = `## ${section}`;
+    const lines = doc.body.split("\n");
+    let found = false;
+    const result: string[] = [];
+
+    for (let i = 0; i < lines.length; i++) {
+      if (lines[i].startsWith(sectionHeader)) {
+        // Replace this section — skip old content until next section or end
+        result.push(sectionHeader);
+        result.push(newContent);
+        found = true;
+        // Skip lines until next ## header or end
+        i++;
+        while (i < lines.length && !lines[i].startsWith("## ")) {
+          i++;
+        }
+        // Don't skip the next ## header
+        if (i < lines.length) {
+          result.push(lines[i]);
+        }
+      } else {
+        result.push(lines[i]);
+      }
+    }
+
+    if (!found) {
+      // Append the section
+      result.push("", sectionHeader, newContent);
+    }
+
+    const today = new Date().toISOString().split("T")[0];
+    const updated: Omit<SkillDocument, "fileName"> = {
+      name: doc.name,
+      description: doc.description,
+      version: doc.version + 1,
+      created: doc.created,
+      updated: today,
+      body: result.join("\n").trim(),
+    };
+
+    await this.atomicWrite(fileName, formatFrontmatter(updated));
+
+    return { success: true, message: `Skill '${doc.name}' section '${section}' updated.`, fileName };
+  }
+
+  async edit(fileName: string, description: string, body: string): Promise<SkillResult> {
+    description = description.trim();
+    body = body.trim();
+
+    if (!description && !body) {
+      return { success: false, error: "At least one of description or body is required." };
+    }
+
+    const doc = await this.loadSkill(fileName);
+    if (!doc) return { success: false, error: `Skill file '${fileName}' not found.` };
+
+    const newDesc = description || doc.description;
+    const newBody = body || doc.body;
+
+    // Scan combined content
+    const scanError = scanContent(newDesc + " " + newBody);
+    if (scanError) return { success: false, error: scanError };
+
+    const today = new Date().toISOString().split("T")[0];
+    const updated: Omit<SkillDocument, "fileName"> = {
+      name: doc.name,
+      description: newDesc,
+      version: doc.version + 1,
+      created: doc.created,
+      updated: today,
+      body: newBody,
+    };
+
+    await this.atomicWrite(fileName, formatFrontmatter(updated));
+
+    return { success: true, message: `Skill '${doc.name}' updated.`, fileName };
+  }
+
+  async delete(fileName: string): Promise<SkillResult> {
+    const doc = await this.loadSkill(fileName);
+    if (!doc) return { success: false, error: `Skill file '${fileName}' not found.` };
+
+    await fs.unlink(path.join(this.skillsDir, fileName));
+
+    return { success: true, message: `Skill '${doc.name}' deleted.`, fileName };
+  }
+
+  // ─── System prompt injection (progressive disclosure) ───
+
+  async formatIndexForSystemPrompt(): Promise<string> {
+    const skills = await this.loadIndex();
+    if (skills.length === 0) return "";
+
+    const lines: string[] = [
+      "═".repeat(46),
+      `SKILLS (procedural memory) [${skills.length} skills]`,
+      "═".repeat(46),
+      "Use the 'skill' tool with action 'view' to load full content on demand.",
+      "",
+    ];
+
+    for (const skill of skills) {
+      lines.push(`• ${skill.name}: ${skill.description}`);
+    }
+
+    return lines.join("\n");
+  }
+
+  // ─── Internal helpers ───
+
+  /** Atomic write: temp file + rename (same crash-safety as MemoryStore) */
+  private async atomicWrite(fileName: string, content: string): Promise<void> {
+    const filePath = path.join(this.skillsDir, fileName);
+    const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "pi-skill-"));
+    const tmpPath = path.join(tmpDir, "write.tmp");
+
+    try {
+      await fs.writeFile(tmpPath, content, "utf-8");
+      await fs.rename(tmpPath, filePath);
+    } catch (err) {
+      try { await fs.unlink(tmpPath); } catch { /* ignore */ }
+      throw err;
+    } finally {
+      try { await fs.rmdir(tmpDir); } catch { /* ignore */ }
+    }
+  }
+}

package/src/tools/memory-tool.ts

@@ -10,7 +10,7 @@ import { StringEnum } from "@mariozechner/pi-ai";
 import { MemoryStore } from "../store/memory-store.js";
 import { MEMORY_TOOL_DESCRIPTION } from "../constants.js";
 
-export function registerMemoryTool(pi: ExtensionAPI, store: MemoryStore): void {
+export function registerMemoryTool(pi: ExtensionAPI, store: MemoryStore, projectStore: MemoryStore | null): void {
   pi.registerTool({
     name: "memory",
     label: "Memory",
@@ -24,7 +24,7 @@ export function registerMemoryTool(pi: ExtensionAPI, store: MemoryStore): void {
     ],
     parameters: Type.Object({
       action: StringEnum(["add", "replace", "remove"] as const),
-      target: StringEnum(["memory", "user"] as const),
+      target: StringEnum(["memory", "user", "project"] as const),
       content: Type.Optional(
         Type.String({ description: "Entry content for add/replace" })
       ),
@@ -36,7 +36,21 @@ export function registerMemoryTool(pi: ExtensionAPI, store: MemoryStore): void {
       ),
     }),
     async execute(toolCallId, params, signal, onUpdate, ctx) {
-      const { action, target, content, old_text } = params;
+      const { action, target: rawTarget, content, old_text } = params;
+
+      // Route 'project' to projectStore (internal target 'memory')
+      const target = rawTarget as "memory" | "user";
+      const activeStore = rawTarget === "project" ? projectStore : store;
+
+      if (rawTarget === "project" && !projectStore) {
+        return {
+          content: [{ type: "text", text: JSON.stringify({ success: false, error: "Project memory is not available (no project detected)." }) }],
+          details: {},
+        };
+      }
+
+      // After the guard above, activeStore is guaranteed non-null when rawTarget === 'project'
+      const store_ = activeStore!;
 
       let result;
       switch (action) {
@@ -55,7 +69,7 @@ export function registerMemoryTool(pi: ExtensionAPI, store: MemoryStore): void {
               details: {},
             };
           }
-          result = store.add(target, content);
+          result = await store_.add(target, content);
           break;
 
         case "replace":
@@ -87,7 +101,7 @@ export function registerMemoryTool(pi: ExtensionAPI, store: MemoryStore): void {
               details: {},
             };
           }
-          result = store.replace(target, old_text, content);
+          result = await store_.replace(target, old_text, content);
           break;
 
         case "remove":
@@ -105,7 +119,7 @@ export function registerMemoryTool(pi: ExtensionAPI, store: MemoryStore): void {
               details: {},
             };
           }
-          result = store.remove(target, old_text);
+          result = await store_.remove(target, old_text);
           break;
 
         default:
@@ -115,6 +129,11 @@ export function registerMemoryTool(pi: ExtensionAPI, store: MemoryStore): void {
           };
       }
 
+      // Tag project results so the caller knows the scope
+      if (rawTarget === "project" && result.success) {
+        (result as any).target = "project";
+      }
+
       return {
         content: [{ type: "text", text: JSON.stringify(result) }],
         details: result,