ultimate-pi 0.1.7 → 0.2.2

package/node_modules/@sting8k/pi-vcc/src/core/report.ts

@@ -0,0 +1,237 @@
+import type { Message } from "@mariozechner/pi-ai";
+import { buildSections } from "./build-sections";
+import { clip } from "./content";
+import { normalize } from "./normalize";
+import { renderMessage } from "./render-entries";
+import { searchEntries } from "./search-entries";
+import { type CompileInput, compile } from "./summarize";
+
+const SECTION_HEADERS = ["Session Goal", "Files And Changes", "Commits", "Outstanding Context"];
+
+interface RoleCounts {
+  user: number;
+  assistant: number;
+  toolResult: number;
+}
+
+interface BlockCounts {
+  user: number;
+  assistant: number;
+  toolCalls: number;
+  toolResults: number;
+  thinking: number;
+}
+
+export interface RecallProbe {
+  label: string;
+  sourceText: string;
+  query: string;
+  summaryMentioned: boolean;
+  recallHits: number;
+}
+
+export interface CompactReport {
+  summary: string;
+  before: {
+    messageCount: number;
+    roleCounts: RoleCounts;
+    blockCounts: BlockCounts;
+    inputChars: number;
+    estimatedTokens: number;
+    topFiles: string[];
+    preview: string;
+  };
+  after: {
+    summaryLength: number;
+    estimatedTokens: number;
+    sectionCount: number;
+    summaryPreview: string;
+    goalsCount: number;
+    blockersCount: number;
+    briefTranscriptLines: number;
+  };
+  compression: {
+    charsBefore: number;
+    charsAfter: number;
+    ratio: number;
+    messagesBefore: number;
+  };
+  recall: {
+    probes: RecallProbe[];
+  };
+}
+
+const estimateTokensFromChars = (chars: number): number =>
+  Math.ceil(chars / 4);
+
+const countRoles = (messages: Message[]): RoleCounts => {
+  const counts: RoleCounts = { user: 0, assistant: 0, toolResult: 0 };
+  for (const msg of messages) {
+    if (msg.role === "user") counts.user += 1;
+    else if (msg.role === "assistant") counts.assistant += 1;
+    else if (msg.role === "toolResult") counts.toolResult += 1;
+  }
+  return counts;
+};
+
+const countBlocks = (messages: Message[]): BlockCounts => {
+  const counts: BlockCounts = {
+    user: 0,
+    assistant: 0,
+    toolCalls: 0,
+    toolResults: 0,
+    thinking: 0,
+  };
+
+  for (const block of normalize(messages)) {
+    if (block.kind === "user") counts.user += 1;
+    else if (block.kind === "assistant") counts.assistant += 1;
+    else if (block.kind === "tool_call") counts.toolCalls += 1;
+    else if (block.kind === "tool_result") counts.toolResults += 1;
+    else if (block.kind === "thinking") counts.thinking += 1;
+  }
+
+  return counts;
+};
+
+const inputCharsOf = (messages: Message[]): number =>
+  messages
+    .map((msg, index) => renderMessage(msg, index, true).summary.length)
+    .reduce((sum, len) => sum + len, 0);
+
+const topFilesOf = (messages: Message[]): string[] => {
+  const files = new Set<string>();
+  for (const block of normalize(messages)) {
+    if (block.kind === "tool_call") {
+      for (const key of ["path", "file_path", "filePath", "file"]) {
+        const val = block.args[key];
+        if (typeof val === "string") { files.add(val); break; }
+      }
+    }
+  }
+  return [...files].slice(0, 10);
+};
+
+const previewOf = (messages: Message[], edgeCount = 3): string => {
+  const rendered = messages.map((msg, index) => renderMessage(msg, index));
+  if (rendered.length === 0) return "(empty)";
+  if (rendered.length <= edgeCount * 2) {
+    return rendered
+      .map((entry) => `#${entry.index} [${entry.role}] ${clip(entry.summary, 220)}`)
+      .join("\n");
+  }
+
+  const first = rendered.slice(0, edgeCount);
+  const last = rendered.slice(-edgeCount);
+  return [
+    ...first.map((entry) => `#${entry.index} [${entry.role}] ${clip(entry.summary, 220)}`),
+    "...",
+    ...last.map((entry) => `#${entry.index} [${entry.role}] ${clip(entry.summary, 220)}`),
+  ].join("\n");
+};
+
+const sectionCountOf = (summary: string): number =>
+  SECTION_HEADERS.filter((header) => summary.includes(`[${header}]`)).length;
+
+const briefLineCountOf = (summary: string): number => {
+  const sep = "\n\n---\n\n";
+  const idx = summary.indexOf(sep);
+  if (idx < 0) return 0;
+  return summary.slice(idx + sep.length).split("\n").length;
+};
+
+const queryTermsOf = (text: string): string[] =>
+  (text.match(/[\p{L}\p{N}_./-]{3,}/gu) ?? [])
+    .map((part) => part.trim())
+    .filter(Boolean);
+
+const queryOf = (text: string): string => {
+  const terms = queryTermsOf(text);
+  return terms.slice(0, 6).join(" ");
+};
+
+const matchesQuery = (text: string, query: string): boolean => {
+  const hay = text.toLowerCase();
+  return query
+    .toLowerCase()
+    .split(/\s+/)
+    .filter(Boolean)
+    .every((term) => hay.includes(term));
+};
+
+const probesOf = (messages: Message[], summary: string): RecallProbe[] => {
+  const blocks = normalize(messages);
+  const data = buildSections({ blocks });
+
+  // Find first file from tool calls
+  let firstFile = "";
+  for (const b of blocks) {
+    if (b.kind === "tool_call") {
+      for (const key of ["path", "file_path", "filePath", "file"]) {
+        if (typeof b.args[key] === "string") { firstFile = b.args[key] as string; break; }
+      }
+      if (firstFile) break;
+    }
+  }
+
+  const rawProbes = [
+    { label: "goal", text: data.sessionGoal[0] ?? "" },
+    { label: "file", text: firstFile },
+    { label: "problem", text: data.outstandingContext[0] ?? "" },
+  ];
+
+  const rendered = messages.map((msg, index) => renderMessage(msg, index));
+
+  return rawProbes
+    .map(({ label, text }) => {
+      const sourceText = text.trim();
+      const query = queryOf(sourceText);
+      if (!query) return null;
+      return {
+        label,
+        sourceText,
+        query,
+        summaryMentioned: matchesQuery(summary, query),
+        recallHits: searchEntries(rendered, query).length,
+      };
+    })
+    .filter((probe): probe is RecallProbe => probe !== null);
+};
+
+export const buildCompactReport = (input: CompileInput): CompactReport => {
+  const summary = compile(input);
+  const data = buildSections({ blocks: normalize(input.messages) });
+  const inputChars = inputCharsOf(input.messages);
+  const topFiles = topFilesOf(input.messages);
+
+  return {
+    summary,
+    before: {
+      messageCount: input.messages.length,
+      roleCounts: countRoles(input.messages),
+      blockCounts: countBlocks(input.messages),
+      inputChars,
+      estimatedTokens: estimateTokensFromChars(inputChars),
+      topFiles,
+      preview: previewOf(input.messages),
+    },
+    after: {
+      summaryLength: summary.length,
+      estimatedTokens: estimateTokensFromChars(summary.length),
+      sectionCount: sectionCountOf(summary),
+      summaryPreview: summary,
+      goalsCount: data.sessionGoal.length,
+      blockersCount: data.outstandingContext.length,
+      briefTranscriptLines: briefLineCountOf(summary),
+    },
+    compression: {
+      charsBefore: inputChars,
+      charsAfter: summary.length,
+      ratio: summary.length === 0 ? 0 : Number((inputChars / summary.length).toFixed(2)),
+      messagesBefore: input.messages.length,
+    },
+    recall: {
+      probes: probesOf(input.messages, summary),
+    },
+  };
+};

package/node_modules/@sting8k/pi-vcc/src/core/sanitize.ts

@@ -0,0 +1,5 @@
+const ANSI_RE = /\x1b\[[0-9;]*[A-Za-z]/g;
+const CTRL_RE = /[\x00-\x08\x0b\x0c\x0e-\x1f]/g;
+
+export const sanitize = (text: string): string =>
+  text.replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(ANSI_RE, "").replace(CTRL_RE, "");

package/node_modules/@sting8k/pi-vcc/src/core/search-entries.ts

@@ -0,0 +1,221 @@
+import type { Message } from "@mariozechner/pi-ai";
+import type { RenderedEntry } from "./render-entries";
+import { textOf } from "./content";
+
+export interface SearchHit extends RenderedEntry {
+  /** Context snippet around the first matched term (only when query provided) */
+  snippet?: string;
+  /** Number of query terms matched (for ranking) */
+  matchCount?: number;
+}
+
+const escapeRegex = (s: string): string =>
+  s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/** Try to compile as regex; fall back to escaped literal. */
+const safeRegex = (pattern: string): RegExp => {
+  try {
+    return new RegExp(pattern, "i");
+  } catch {
+    return new RegExp(escapeRegex(pattern), "i");
+  }
+};
+
+/** Detect if the query looks like a single regex pattern (contains regex metacharacters). */
+const looksLikeRegex = (query: string): boolean =>
+  /[|*+?{}()[\]\\^$.]/.test(query);
+
+/** Build a regex for snippet highlighting — matches first available term. */
+const snippetRegex = (terms: string[]): RegExp => {
+  const alts = terms.map((t) => {
+    try {
+      // Validate that it's a valid regex
+      new RegExp(t, "i");
+      return t;
+    } catch {
+      return escapeRegex(t);
+    }
+  });
+  return new RegExp(alts.join("|"), "i");
+};
+
+// ── Stopwords for natural language queries ──
+const STOPWORDS = new Set([
+  // English
+  "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+  "have", "has", "had", "do", "does", "did", "will", "would", "could",
+  "should", "may", "might", "can", "shall", "of", "in", "to", "for",
+  "with", "on", "at", "from", "by", "as", "into", "through", "during",
+  "before", "after", "above", "below", "between", "out", "off", "over",
+  "under", "again", "further", "then", "once", "here", "there", "when",
+  "where", "why", "how", "all", "both", "each", "few", "more", "most",
+  "other", "some", "such", "no", "nor", "not", "only", "own", "same",
+  "so", "than", "too", "very", "just", "about", "it", "its", "that",
+  "this", "what", "which", "who", "whom", "these", "those",
+]);
+
+/** Remove stopwords, keep meaningful terms. */
+const filterStopwords = (terms: string[]): string[] => {
+  const meaningful = terms.filter((t) => !STOPWORDS.has(t.toLowerCase()) && t.length > 1);
+  // If all terms were stopwords, return original (don't lose everything)
+  return meaningful.length > 0 ? meaningful : terms;
+};
+
+/** Count how many distinct terms match the haystack. */
+const countMatches = (hay: string, terms: string[]): number => {
+  let count = 0;
+  for (const t of terms) {
+    if (safeRegex(t).test(hay)) count++;
+  }
+  return count;
+};
+
+// ── BM25-lite scoring ──
+const BM25_K = 1.2;
+const BM25_B = 0.75;
+
+/** Count occurrences of a regex pattern in text. */
+const termFreq = (text: string, pattern: RegExp): number => {
+  const matches = text.match(new RegExp(pattern.source, "gi"));
+  return matches ? matches.length : 0;
+};
+
+interface BM25Context {
+  n: number;         // total docs
+  avgDl: number;     // average doc length (words)
+  df: Map<string, number>; // term -> number of docs containing it
+}
+
+/** Precompute IDF and avgDl across all docs. */
+const buildBM25Context = (docs: string[], terms: string[]): BM25Context => {
+  const n = docs.length;
+  const df = new Map<string, number>();
+  let totalLen = 0;
+
+  for (const doc of docs) {
+    totalLen += doc.split(/\s+/).length;
+    for (const t of terms) {
+      if (safeRegex(t).test(doc)) {
+        df.set(t, (df.get(t) ?? 0) + 1);
+      }
+    }
+  }
+
+  return { n, avgDl: totalLen / Math.max(n, 1), df };
+};
+
+/** BM25 score for a single doc against query terms. */
+const bm25Score = (doc: string, terms: string[], ctx: BM25Context): number => {
+  const dl = doc.split(/\s+/).length;
+  let score = 0;
+
+  for (const t of terms) {
+    const tf = termFreq(doc, safeRegex(t));
+    if (tf === 0) continue;
+
+    const docFreq = ctx.df.get(t) ?? 0;
+    // IDF: log((N - df + 0.5) / (df + 0.5) + 1)
+    const idf = Math.log((ctx.n - docFreq + 0.5) / (docFreq + 0.5) + 1);
+    // TF saturation with length normalization
+    const tfNorm = (tf * (BM25_K + 1)) / (tf + BM25_K * (1 - BM25_B + BM25_B * dl / ctx.avgDl));
+    score += idf * tfNorm;
+  }
+
+  return score;
+};
+
+/** Line-based snippet: ±contextLines around first regex match. */
+const lineSnippet = (text: string, regex: RegExp, contextLines = 2): string | undefined => {
+  const lines = text.split("\n");
+  let matchIdx = -1;
+  for (let i = 0; i < lines.length; i++) {
+    if (regex.test(lines[i])) {
+      matchIdx = i;
+      break;
+    }
+  }
+  if (matchIdx === -1) return undefined;
+
+  const start = Math.max(0, matchIdx - contextLines);
+  const end = Math.min(lines.length, matchIdx + contextLines + 1);
+  const slice = lines.slice(start, end);
+
+  const parts: string[] = [];
+  if (start > 0) parts.push(`...(${start} lines above)`);
+  parts.push(...slice);
+  if (end < lines.length) parts.push(`...(${lines.length - end} lines below)`);
+  return parts.join("\n");
+};
+
+/** Build full searchable text for a message. */
+const fullText = (msg: Message): string => {
+  if ((msg as any).role === "bashExecution") {
+    return `${(msg as any).command ?? ""} ${(msg as any).output ?? ""}`;
+  }
+  return textOf(msg.content);
+};
+
+export const searchEntries = (
+  entries: RenderedEntry[],
+  messages: Message[],
+  query?: string,
+): SearchHit[] => {
+  if (!query?.trim()) return entries;
+
+  const rawQuery = query.trim();
+
+  // If query looks like a single regex pattern (contains metacharacters),
+  // treat the whole thing as one pattern — don't split into terms
+  if (looksLikeRegex(rawQuery)) {
+    const regex = safeRegex(rawQuery);
+    const hits: SearchHit[] = [];
+    for (let i = 0; i < entries.length; i++) {
+      const e = entries[i];
+      const msg = messages[i];
+      const text = msg ? fullText(msg) : e.summary;
+      const filePart = e.files?.join(" ") ?? "";
+      const hay = `${e.role} ${text} ${filePart}`;
+      if (regex.test(hay)) {
+        const snip = lineSnippet(text, regex);
+        hits.push({ ...e, snippet: snip, matchCount: 1 });
+      }
+    }
+    return hits;
+  }
+
+  // Natural language / multi-word query: BM25 scoring
+  const rawTerms = rawQuery.split(/\s+/);
+  const terms = filterStopwords(rawTerms);
+  const snipRe = snippetRegex(terms);
+
+  // Build all docs for BM25 context
+  const docs: string[] = [];
+  for (let i = 0; i < entries.length; i++) {
+    const e = entries[i];
+    const msg = messages[i];
+    const text = msg ? fullText(msg) : e.summary;
+    const filePart = e.files?.join(" ") ?? "";
+    docs.push(`${e.role} ${text} ${filePart}`);
+  }
+
+  const ctx = buildBM25Context(docs, terms);
+
+  const scored: Array<{ hit: SearchHit; score: number }> = [];
+  for (let i = 0; i < entries.length; i++) {
+    const e = entries[i];
+    const hay = docs[i];
+    const mc = countMatches(hay, terms);
+    if (mc === 0) continue;
+    const score = bm25Score(hay, terms, ctx);
+    const text = messages[i] ? fullText(messages[i]) : e.summary;
+    const snip = lineSnippet(text, snipRe);
+    scored.push({
+      hit: { ...e, snippet: snip, matchCount: mc },
+      score,
+    });
+  }
+
+  // Sort by BM25 score desc
+  scored.sort((a, b) => b.score - a.score);
+  return scored.map((s) => s.hit);
+};

package/node_modules/@sting8k/pi-vcc/src/core/settings.ts

@@ -0,0 +1,77 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { homedir } from "os";
+import { dirname, join } from "path";
+
+export const SETTINGS_PATH_DEFAULT = join(homedir(), ".pi", "agent", "pi-vcc-config.json");
+const settingsPath = (): string => process.env.PI_VCC_CONFIG_PATH ?? SETTINGS_PATH_DEFAULT;
+/** Backwards-compat export. Resolves at access time, not import time. */
+export const SETTINGS_PATH = settingsPath();
+
+export interface PiVccSettings {
+  /**
+   * When true, pi-vcc handles ALL compactions:
+   *   - /compact (no args)
+   *   - /compact <text>
+   *   - auto threshold / overflow
+   *   - /pi-vcc (always handled regardless)
+   *
+   * When false (default), pi-vcc only handles /pi-vcc; everything else
+   * falls back to pi core's default LLM-based compaction.
+   */
+  overrideDefaultCompaction: boolean;
+  /** Write debug snapshot to /tmp/pi-vcc-debug.json on each compaction. */
+  debug: boolean;
+}
+
+export const DEFAULT_SETTINGS: PiVccSettings = {
+  overrideDefaultCompaction: false,
+  debug: false,
+};
+
+const readJson = (path: string): Record<string, unknown> | null => {
+  try {
+    return JSON.parse(readFileSync(path, "utf-8"));
+  } catch {
+    return null;
+  }
+};
+
+export function loadSettings(): PiVccSettings {
+  const parsed = readJson(settingsPath());
+  if (!parsed || typeof parsed !== "object") return { ...DEFAULT_SETTINGS };
+  return { ...DEFAULT_SETTINGS, ...(parsed as Partial<PiVccSettings>) };
+}
+
+/**
+ * Ensure ~/.pi/agent/pi-vcc-config.json exists with default keys.
+ * - File missing → create with full default block.
+ * - File exists but invalid JSON → no-op (don't clobber user file).
+ * - File exists and valid → fill in missing default keys, preserve existing values.
+ */
+export function scaffoldSettings(): void {
+  try {
+    const path = settingsPath();
+    const dir = dirname(path);
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+
+    if (!existsSync(path)) {
+      writeFileSync(path, `${JSON.stringify(DEFAULT_SETTINGS, null, 2)}\n`);
+      return;
+    }
+
+    const parsed = readJson(path);
+    if (!parsed || typeof parsed !== "object") return; // don't clobber
+
+    let changed = false;
+    const next: Record<string, unknown> = { ...parsed };
+    for (const [key, value] of Object.entries(DEFAULT_SETTINGS)) {
+      if (!(key in next)) {
+        next[key] = value;
+        changed = true;
+      }
+    }
+    if (changed) writeFileSync(path, `${JSON.stringify(next, null, 2)}\n`);
+  } catch {
+    // best-effort; never crash extension load
+  }
+}

package/node_modules/@sting8k/pi-vcc/src/core/skill-collapse.ts

@@ -0,0 +1,35 @@
+/** Shared skill-tag collapse utilities */
+
+const SKILL_TAG_RE = /^-?\s*<skill\s+name="([^"]+)"/;
+const SKILL_CLOSE_RE = /^-?\s*<\/skill>/;
+
+/** Collapse skill tags in an array of lines — dedup by name, drop all content inside block */
+export const collapseSkillLines = (lines: string[]): string[] => {
+  const result: string[] = [];
+  const seenSkills = new Set<string>();
+  let insideSkill = false;
+
+  for (const line of lines) {
+    const skillMatch = line.match(SKILL_TAG_RE);
+    if (skillMatch) {
+      insideSkill = true;
+      const name = skillMatch[1];
+      if (!seenSkills.has(name)) {
+        seenSkills.add(name);
+        result.push(`[skill: ${name}]`);
+      }
+      continue;
+    }
+    if (insideSkill) {
+      if (SKILL_CLOSE_RE.test(line)) insideSkill = false;
+      continue;
+    }
+    result.push(line);
+  }
+  return result;
+};
+
+/** Collapse <skill name="X" ...>...</skill> blocks in raw text */
+const SKILL_BLOCK_RE = /<skill\s+name="([^"]+)"[^>]*>[\s\S]*?(?:<\/skill>|$)/g;
+export const collapseSkillText = (text: string): string =>
+  text.replace(SKILL_BLOCK_RE, (_, name) => `[skill: ${name}]`);