pi-orchestrator-config 4.1.5

package/extensions/orchestrator/memory-promotion.ts

@@ -0,0 +1,449 @@
+/**
+ * Memory promotion — graduate high-evidence memories into structure.
+ *
+ * Safe auto-apply: enforcement metadata on existing memories.
+ * Propose-only: project_rule (never writes rules files).
+ * Skill: queue + dream may auto-create under .pi/skills/.
+ */
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  type EnforcementAction,
+  type EnforcementTrigger,
+  type MemoryCategory,
+  type ScoredEntry,
+  entryHash,
+  loadScores,
+  saveScores,
+} from "./memory-scoring.js";
+import { CATEGORY_TO_TOPIC, readAllTopicEntries } from "./memory-tree.js";
+import {
+  type PromotionCandidate,
+  type PromotionCandidatePatch,
+  appendPromotions,
+  formatPromotionsForReport,
+  loadPromotions,
+  promotionId,
+  updatePromotionCandidates,
+  updatePromotionStatus,
+} from "./promotion-queue.js";
+
+export const EVIDENCE_ENFORCEMENT = 3;
+export const EVIDENCE_SKILL = 3;
+export const EVIDENCE_PROJECT_RULE = 5;
+
+export interface InferredEnforcement {
+  trigger: EnforcementTrigger;
+  action: EnforcementAction;
+  actionCommand?: string;
+  verifier?: string;
+  confidence: "high" | "low";
+}
+
+/** Detect unambiguous mechanical enforcement from memory text. */
+export function inferMechanicalEnforcement(text: string): InferredEnforcement | null {
+  const t = text.trim();
+
+  // never/don't use|run `cmd` or "cmd"
+  const neverCmd = t.match(
+    /\b(?:never|don'?t|do not)\s+(?:use|run|execute)\s+[`"']([^`"']+)[`"']/i,
+  );
+  if (neverCmd?.[1]) {
+    return {
+      trigger: `bash_contains ${neverCmd[1].trim()}` as EnforcementTrigger,
+      action: "block",
+      confidence: "high",
+    };
+  }
+
+  // never/don't git add .
+  const neverGitAdd = t.match(/\b(?:never|don'?t|do not)\s+.*\bgit\s+add\s+\./i);
+  if (neverGitAdd) {
+    return {
+      trigger: "bash_contains git add ." as EnforcementTrigger,
+      action: "block",
+      confidence: "high",
+    };
+  }
+
+  // never use tool X / never call X
+  const neverTool = t.match(
+    /\b(?:never|don'?t|do not)\s+(?:use|call|invoke)\s+(?:the\s+)?([a-z_][a-z0-9_-]{1,40})\b/i,
+  );
+  if (neverTool?.[1]) {
+    const tool = neverTool[1].toLowerCase();
+    // Avoid matching generic words
+    if (!["this", "that", "the", "a", "an", "any", "our"].includes(tool)) {
+      return {
+        trigger: `tool_name ${tool}` as EnforcementTrigger,
+        action: "block",
+        confidence: "high",
+      };
+    }
+  }
+
+  // always ask_user before <command>
+  const askBefore = t.match(
+    /\balways\s+(?:call\s+)?ask_user\s+before\s+(.+)$/i,
+  );
+  if (askBefore?.[1]) {
+    const cmd = askBefore[1].replace(/[.!]+$/, "").trim();
+    if (cmd.length > 2 && cmd.length < 80) {
+      return {
+        trigger: "tool_name bash" as EnforcementTrigger,
+        action: "warn",
+        verifier: `tool_called ask_user before ${cmd}`,
+        confidence: "high",
+      };
+    }
+  }
+
+  // warn when modifying <pattern>
+  // Only high-confidence when the pattern is matchable by enforcement runtime:
+  // `*.ext` suffix or a plain path fragment (no glob metacharacters).
+  const warnFile = t.match(
+    /\bwarn\s+when\s+modifying\s+([^\s,]+)/i,
+  );
+  if (warnFile?.[1]) {
+    const pattern = warnFile[1].trim();
+    const isExtGlob = /^\*\.[A-Za-z0-9]+$/.test(pattern);
+    const hasGlobMeta = /[*?[\]{}]|\*\*/.test(pattern);
+    if (isExtGlob || !hasGlobMeta) {
+      return {
+        trigger: `file_modified ${pattern}` as EnforcementTrigger,
+        action: "warn",
+        confidence: "high",
+      };
+    }
+    // Glob-like patterns (e.g. src/**/*.py) are not matchable — leave for propose-only
+    return null;
+  }
+
+  // Soft mechanical cue without extractable trigger
+  if (
+    /\b(?:never|always|don'?t|do not)\b/i.test(t) &&
+    /\b(?:git|bash|npm|uv|docker|kubectl|gh|curl)\b/i.test(t)
+  ) {
+    return null; // mechanical-looking but ambiguous — caller may queue low-confidence
+  }
+
+  return null;
+}
+
+function looksProcedural(text: string): boolean {
+  return (
+    /\b(step\s*\d|then\s+|after that|workflow|pipeline)\b/i.test(text) ||
+    (text.includes(" → ") || text.includes("->"))
+  );
+}
+
+function looksProjectConvention(text: string): boolean {
+  return /\b(always|never|must|convention|in this (repo|project)|project-wide)\b/i.test(
+    text,
+  );
+}
+
+function hasEnforcement(entry: ScoredEntry): boolean {
+  return !!((entry.trigger && entry.action) || entry.verifier);
+}
+
+/** Mark *(enforced)* on the topic line for an entry. */
+export function markTopicEntryEnforced(
+  cwd: string,
+  category: MemoryCategory,
+  text: string,
+): boolean {
+  const topicName = CATEGORY_TO_TOPIC[category];
+  if (!topicName) return false;
+  const topicPath = join(cwd, ".pi", "memory", "topics", `${topicName}.md`);
+  if (!existsSync(topicPath)) return false;
+
+  let content = readFileSync(topicPath, "utf-8");
+  const escaped = text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const lineRe = new RegExp(
+    `^(- \\[${category}\\] ${escaped})((?: \\*\\(pinned\\)\\*)?)((?: \\*\\(enforced\\)\\*)?)(\\s*)$`,
+    "m",
+  );
+  const m = content.match(lineRe);
+  if (!m) return false;
+  if (m[3]) return true; // already enforced
+
+  content = content.replace(lineRe, `$1$2 *(enforced)*$4`);
+  writeFileSync(topicPath, content, "utf-8");
+  return true;
+}
+
+export interface TopicScored {
+  text: string;
+  category: MemoryCategory;
+  pinned: boolean;
+  hash: string;
+  scored: ScoredEntry;
+}
+
+function loadTopicScored(cwd: string): TopicScored[] {
+  const scores = loadScores(cwd);
+  const out: TopicScored[] = [];
+  for (const te of readAllTopicEntries(cwd)) {
+    const hash = entryHash(`- [${te.category}] ${te.text}`);
+    const scored = scores.entries[hash];
+    if (!scored) continue;
+    out.push({
+      text: te.text,
+      category: te.category,
+      pinned: te.pinned,
+      hash,
+      scored,
+    });
+  }
+  return out;
+}
+
+/**
+ * Scan memories for promotion candidates (does not write).
+ */
+export function scanPromotionCandidates(cwd: string): PromotionCandidate[] {
+  const now = new Date().toISOString();
+  const candidates: PromotionCandidate[] = [];
+  const entries = loadTopicScored(cwd);
+
+  for (const e of entries) {
+    const evidence = e.scored.evidenceCount;
+    const inferred = inferMechanicalEnforcement(e.text);
+
+    // Enforcement path
+    if (
+      evidence >= EVIDENCE_ENFORCEMENT &&
+      !hasEnforcement(e.scored) &&
+      (e.category === "lesson" ||
+        e.category === "mistake" ||
+        e.category === "preference")
+    ) {
+      if (inferred && inferred.confidence === "high") {
+        candidates.push({
+          id: promotionId("enforcement", e.category, e.text),
+          destination: "enforcement",
+          text: e.text,
+          category: e.category,
+          reason: `evidenceCount=${evidence} with extractable mechanical trigger`,
+          evidenceCount: evidence,
+          status: "proposed",
+          trigger: inferred.trigger,
+          action: inferred.actionCommand
+            ? `run_after ${inferred.actionCommand}`
+            : inferred.action,
+          verifier: inferred.verifier,
+          createdAt: now,
+        });
+      } else if (
+        /\b(?:never|always|don'?t|do not)\b/i.test(e.text) &&
+        !inferred
+      ) {
+        candidates.push({
+          id: promotionId("enforcement", e.category, e.text),
+          destination: "enforcement",
+          text: e.text,
+          category: e.category,
+          reason: `evidenceCount=${evidence}; mechanical language but trigger needs human/LLM fill-in`,
+          evidenceCount: evidence,
+          status: "proposed",
+          createdAt: now,
+        });
+      }
+    }
+
+    // Skill path
+    if (
+      evidence >= EVIDENCE_SKILL &&
+      looksProcedural(e.text) &&
+      (e.category === "pattern" || e.category === "lesson" || e.category === "done")
+    ) {
+      const skillName = e.text
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-|-$/g, "")
+        .slice(0, 40) || "workflow";
+      candidates.push({
+        id: promotionId("skill", e.category, e.text),
+        destination: "skill",
+        text: e.text,
+        category: e.category,
+        reason: `evidenceCount=${evidence}; looks like a multi-step workflow`,
+        evidenceCount: evidence,
+        status: "proposed",
+        skillName,
+        skillCreated: false,
+        createdAt: now,
+      });
+    }
+
+    // Project rule path (propose only)
+    if (
+      evidence >= EVIDENCE_PROJECT_RULE &&
+      looksProjectConvention(e.text) &&
+      !hasEnforcement(e.scored) &&
+      !inferred
+    ) {
+      candidates.push({
+        id: promotionId("project_rule", e.category, e.text),
+        destination: "project_rule",
+        text: e.text,
+        category: e.category,
+        reason: `evidenceCount=${evidence}; project-wide convention — propose only, never auto-write rules/`,
+        evidenceCount: evidence,
+        status: "proposed",
+        createdAt: now,
+      });
+    }
+  }
+
+  return candidates;
+}
+
+export interface ApplySafeResult {
+  applied: number;
+  queued: number;
+  details: string[];
+}
+
+/**
+ * Apply high-confidence enforcement promotions; queue the rest.
+ * Never writes rules files.
+ */
+export function applySafePromotions(cwd: string): ApplySafeResult {
+  const details: string[] = [];
+  let applied = 0;
+
+  const scanned = scanPromotionCandidates(cwd);
+  const toQueue: PromotionCandidate[] = [];
+  const scores = loadScores(cwd);
+
+  for (const c of scanned) {
+    if (c.destination !== "enforcement" || !c.trigger || !c.action) {
+      toQueue.push(c);
+      continue;
+    }
+
+    // Only auto-apply when action is block/warn (not run_after — safer)
+    if (c.action !== "block" && c.action !== "warn") {
+      toQueue.push(c);
+      continue;
+    }
+
+    const hash = entryHash(`- [${c.category}] ${c.text}`);
+    const entry = scores.entries[hash];
+    if (!entry || hasEnforcement(entry)) {
+      toQueue.push({ ...c, status: "proposed" });
+      continue;
+    }
+
+    entry.trigger = c.trigger as EnforcementTrigger;
+    entry.action = c.action as EnforcementAction;
+    if (c.verifier) entry.verifier = c.verifier;
+    entry.lifecycle = "active";
+    markTopicEntryEnforced(cwd, c.category as MemoryCategory, c.text);
+
+    const appliedCandidate: PromotionCandidate = { ...c, status: "applied" };
+    toQueue.push(appliedCandidate);
+    applied += 1;
+    details.push(`enforced: [${c.category}] ${c.text}`);
+  }
+
+  saveScores(cwd, scores);
+
+  // Append-only / batched status updates — avoid lossy full rewrite and per-item I/O.
+  const existing = loadPromotions(cwd);
+  const byId = new Map(existing.map((x) => [x.id, x]));
+  const toAppend: PromotionCandidate[] = [];
+  const patches: { id: string; patch: PromotionCandidatePatch }[] = [];
+
+  for (const c of toQueue) {
+    const prev = byId.get(c.id);
+    if (!prev) {
+      toAppend.push(c);
+      byId.set(c.id, c);
+      continue;
+    }
+    // Don't reopen applied/rejected as proposed
+    if (prev.status !== "proposed" && c.status === "proposed") continue;
+    if (c.status === "applied" && prev.status === "proposed") {
+      // Merge enforcement metadata for auditability; keep human reason + createdAt
+      const merged: PromotionCandidate = {
+        ...prev,
+        ...c,
+        status: "applied",
+        reason: prev.reason || c.reason,
+        createdAt: prev.createdAt,
+        // Prefer new scan values when present; otherwise keep queued metadata
+        evidenceCount: c.evidenceCount ?? prev.evidenceCount,
+        trigger: c.trigger ?? prev.trigger,
+        action: c.action ?? prev.action,
+        verifier: c.verifier ?? prev.verifier,
+        skillName: c.skillName ?? prev.skillName,
+        skillCreated: c.skillCreated ?? prev.skillCreated,
+      };
+      patches.push({
+        id: c.id,
+        patch: {
+          status: "applied",
+          reason: merged.reason,
+          ...(c.evidenceCount !== undefined
+            ? { evidenceCount: c.evidenceCount }
+            : {}),
+          ...(c.trigger !== undefined ? { trigger: c.trigger } : {}),
+          ...(c.action !== undefined ? { action: c.action } : {}),
+          ...(c.verifier !== undefined ? { verifier: c.verifier } : {}),
+          ...(c.skillName !== undefined ? { skillName: c.skillName } : {}),
+          ...(c.skillCreated !== undefined
+            ? { skillCreated: c.skillCreated }
+            : {}),
+        },
+      });
+      byId.set(c.id, merged);
+    }
+  }
+  if (patches.length > 0) updatePromotionCandidates(cwd, patches);
+  if (toAppend.length > 0) appendPromotions(cwd, toAppend);
+
+  const queued = [...byId.values()].filter((c) => c.status === "proposed").length;
+  return { applied, queued, details };
+}
+
+/**
+ * Full promotion pass: scan, apply safe enforcement, refresh queue.
+ * Call after reinforce threshold crossings and after dream completes.
+ */
+export function runPromotionPass(cwd: string): ApplySafeResult {
+  try {
+    return applySafePromotions(cwd);
+  } catch (e: any) {
+    console.debug("[memory-promotion] pass failed:", e?.message || e);
+    return { applied: 0, queued: 0, details: [] };
+  }
+}
+
+/**
+ * After reinforce: if evidence crossed a threshold, run promotion pass.
+ */
+export function maybePromoteAfterReinforce(
+  cwd: string,
+  entryLine: string,
+): ApplySafeResult | null {
+  const scores = loadScores(cwd);
+  const hash = entryHash(entryLine);
+  const entry = scores.entries[hash];
+  if (!entry) return null;
+
+  const crossed =
+    entry.evidenceCount === EVIDENCE_ENFORCEMENT ||
+    entry.evidenceCount === EVIDENCE_SKILL ||
+    entry.evidenceCount === EVIDENCE_PROJECT_RULE ||
+    (entry.evidenceCount > EVIDENCE_ENFORCEMENT &&
+      entry.evidenceCount % 3 === 0);
+
+  if (!crossed) return null;
+  return runPromotionPass(cwd);
+}
+
+export { formatPromotionsForReport, updatePromotionStatus, appendPromotions };

package/extensions/orchestrator/memory-provenance.ts

@@ -0,0 +1,80 @@
+/**
+ * Provenance pending merge — dream writes a sidecar JSON; onComplete merges
+ * into memory-scores.json without asking the LLM to edit scores JSON.
+ *
+ * Sidecar: `.pi/memory/provenance-pending.json`
+ * Shape: { entries: [{ category, text, sourceSession?, derivedFrom?, informs? }] }
+ */
+
+import { existsSync, readFileSync, unlinkSync, writeFileSync, mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { entryHash, loadScores, saveScores, type MemoryCategory } from "./memory-scoring.js";
+
+export interface ProvenancePendingItem {
+  category: MemoryCategory | string;
+  text: string;
+  sourceSession?: string;
+  derivedFrom?: string;
+  informs?: string[];
+}
+
+export interface ProvenancePendingFile {
+  entries: ProvenancePendingItem[];
+}
+
+export function getProvenancePendingPath(cwd: string): string {
+  return join(cwd, ".pi", "memory", "provenance-pending.json");
+}
+
+export function writeProvenancePending(
+  cwd: string,
+  entries: ProvenancePendingItem[],
+): void {
+  const path = getProvenancePendingPath(cwd);
+  const dir = dirname(path);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  const payload: ProvenancePendingFile = { entries };
+  writeFileSync(path, JSON.stringify(payload, null, 2) + "\n", "utf-8");
+}
+
+/**
+ * Merge provenance-pending.json into scores, then delete the sidecar.
+ * Returns number of entries updated.
+ */
+export function mergeProvenancePending(cwd: string): number {
+  const path = getProvenancePendingPath(cwd);
+  if (!existsSync(path)) return 0;
+
+  let pending: ProvenancePendingFile;
+  try {
+    pending = JSON.parse(readFileSync(path, "utf-8")) as ProvenancePendingFile;
+  } catch (e: any) {
+    console.debug("[memory-provenance] parse failed:", e?.message || e);
+    return 0;
+  }
+
+  if (!pending?.entries?.length) {
+    try { unlinkSync(path); } catch { /* ignore */ }
+    return 0;
+  }
+
+  const scores = loadScores(cwd);
+  let updated = 0;
+
+  for (const item of pending.entries) {
+    if (!item?.category || !item?.text) continue;
+    const hash = entryHash(`- [${item.category}] ${item.text}`);
+    const entry = scores.entries[hash];
+    if (!entry) continue;
+
+    if (item.sourceSession) entry.sourceSession = item.sourceSession;
+    if (item.derivedFrom) entry.derivedFrom = item.derivedFrom;
+    if (item.informs?.length) entry.informs = item.informs;
+    updated += 1;
+  }
+
+  if (updated > 0) saveScores(cwd, scores);
+
+  try { unlinkSync(path); } catch { /* ignore */ }
+  return updated;
+}

package/extensions/orchestrator/memory-query-class.ts

@@ -0,0 +1,90 @@
+/**
+ * Query-class classifier for memory injection routing.
+ * Cheap heuristics only — no LLM call.
+ */
+
+export type QueryClass = "pr_review" | "git_release" | "debug" | "general";
+
+export interface QueryClassBias {
+  /** Categories to boost (higher section budget / earlier priority) */
+  boostCategories: string[];
+  /** Vector topK override */
+  vectorTopK: number;
+  /** Extra note injected into situation report when relevant */
+  hint?: string;
+}
+
+const CLASS_BIAS: Record<QueryClass, QueryClassBias> = {
+  pr_review: {
+    boostCategories: ["mistake", "pattern", "lesson"],
+    vectorTopK: 7,
+    hint: "PR review context — prefer mistakes/patterns; check `.pi/data/review-guidelines.md` when reviewing.",
+  },
+  git_release: {
+    boostCategories: ["lesson", "decision", "preference"],
+    vectorTopK: 6,
+    hint: "Git/release context — prefer enforced lessons and decisions.",
+  },
+  debug: {
+    boostCategories: ["mistake", "lesson"],
+    vectorTopK: 7,
+    hint: "Debug context — prefer mistakes and lessons.",
+  },
+  general: {
+    boostCategories: [],
+    vectorTopK: 5,
+  },
+};
+
+const PR_REVIEW_RE =
+  /\b(pr review|code review|review (this |the )?(pr|pull request)|gh pr\b|pull request)\b/i;
+const GIT_RELEASE_RE =
+  /\b(git (commit|push|tag|rebase|merge)|release|changelog|semver|deploy(ment)?)\b/i;
+const DEBUG_RE =
+  /\b(error|stack( ?trace)?|traceback|fail(ed|ing|ure)?|exception|bug|debug|segfault|panic)\b/i;
+
+/**
+ * Classify the user prompt into a query class for injection bias.
+ */
+export function classifyQueryClass(prompt: string): QueryClass {
+  const text = (prompt || "").trim();
+  if (!text) return "general";
+
+  if (PR_REVIEW_RE.test(text)) return "pr_review";
+  if (GIT_RELEASE_RE.test(text)) return "git_release";
+  if (DEBUG_RE.test(text)) return "debug";
+  return "general";
+}
+
+export function getQueryClassBias(queryClass: QueryClass): QueryClassBias {
+  return CLASS_BIAS[queryClass];
+}
+
+/**
+ * Reorder section priority: boosted categories get a modest priority lift
+ * (lower number = earlier). Non-boosted keep base order.
+ */
+export function sectionPriorityBoost(
+  sectionName: string,
+  basePriority: number,
+  queryClass: QueryClass,
+): number {
+  const bias = getQueryClassBias(queryClass);
+  if (bias.boostCategories.length === 0) return basePriority;
+
+  const topicMap: Record<string, string> = {
+    "Active Preferences": "preference",
+    "Active Lessons": "lesson",
+    "Vetoes & Mistakes": "mistake",
+    Patterns: "pattern",
+    "Recent Decisions": "decision",
+    "Recent Completions": "done",
+  };
+  const cat = topicMap[sectionName];
+  if (cat && bias.boostCategories.includes(cat)) {
+    // Lower number sorts earlier; allow 0/negative so boosted sections
+    // beat unboosted priority-1 sections (e.g. preferences).
+    return basePriority - 3;
+  }
+  return basePriority;
+}