pi-continuous-learning 0.5.1 → 0.6.0

package/src/observation-preprocessor.ts

@@ -0,0 +1,48 @@
+/**
+ * Observation preprocessing for cost reduction.
+ *
+ * Strips high-volume, low-signal data from raw observation events before
+ * sending to the LLM analyzer. Reduces context size by ~80% on typical sessions.
+ *
+ * Rules:
+ * - turn_start  → DROP (no information not already in turn_end)
+ * - tool_start  → DROP (tool name + sequence captured by tool_complete)
+ * - tool_complete, is_error: false → KEEP, strip output field
+ * - tool_complete, is_error: true  → KEEP as-is (error message needed)
+ * - all others  → KEEP as-is
+ */
+import type { Observation } from "./types.js";
+
+/**
+ * Preprocess a single observation.
+ * Returns null if the observation should be dropped entirely.
+ * Returns a new (immutable) observation with large fields stripped if applicable.
+ */
+export function preprocessObservation(obs: Observation): Observation | null {
+  if (obs.event === "turn_start" || obs.event === "tool_start") {
+    return null;
+  }
+
+  if (obs.event === "tool_complete" && !obs.is_error) {
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    const { output: _, ...stripped } = obs;
+    return stripped as Observation;
+  }
+
+  return obs;
+}
+
+/**
+ * Preprocess an array of raw observations.
+ * Drops nulls and returns only the meaningful events.
+ */
+export function preprocessObservations(observations: Observation[]): Observation[] {
+  const result: Observation[] = [];
+  for (const obs of observations) {
+    const processed = preprocessObservation(obs);
+    if (processed !== null) {
+      result.push(processed);
+    }
+  }
+  return result;
+}

package/src/prompts/analyzer-system-single-shot.ts

@@ -0,0 +1,123 @@
+/**
+ * System prompt for the single-shot (non-agentic) background analyzer.
+ * Instructs the model to return a JSON change-set instead of using tool calls.
+ */
+export function buildSingleShotSystemPrompt(): string {
+  return `You are a coding behavior analyst. Your job is to read session observations
+and produce a JSON change-set to create or update instinct files that capture reusable coding patterns.
+
+## Output Format
+
+Return ONLY a valid JSON object (no prose, no markdown fences) with this structure:
+
+{
+  "changes": [
+    {
+      "action": "create",
+      "instinct": {
+        "id": "kebab-case-id",
+        "title": "Short title",
+        "trigger": "When this should activate",
+        "action": "What the agent should do (verb phrase)",
+        "confidence": 0.5,
+        "domain": "typescript",
+        "scope": "project",
+        "observation_count": 3,
+        "confirmed_count": 0,
+        "contradicted_count": 0,
+        "inactive_count": 0,
+        "evidence": ["brief note 1", "brief note 2"]
+      }
+    },
+    {
+      "action": "update",
+      "instinct": { "...same fields as create..." }
+    },
+    {
+      "action": "delete",
+      "id": "instinct-id-to-delete",
+      "scope": "project"
+    }
+  ]
+}
+
+Return { "changes": [] } if no changes are needed.
+
+## Pattern Detection Heuristics
+
+Analyze observations for these categories:
+
+### User Corrections
+- User rephrases a request after an agent response
+- User explicitly rejects an approach
+- Trigger: the corrected behavior; Action: the preferred approach
+
+### Error Resolutions
+- Tool call returns is_error: true followed by a successful retry
+- Trigger: the error condition; Action: the proven resolution
+
+### Repeated Workflows
+- Same sequence of tool calls appears 3+ times
+- Trigger: the workflow start condition; Action: the efficient path
+
+### Tool Preferences
+- Agent consistently uses one tool over alternatives
+- Trigger: the task type; Action: the preferred tool and parameters
+
+### Anti-Patterns
+- Actions that consistently lead to errors or user corrections
+- Trigger: the bad pattern situation; Action: what to do instead
+
+### Turn Structure
+- turn_end events summarize turns: tool_count and error_count
+- High error_count turns suggest inefficient approaches
+
+### Context Pressure
+- session_compact events signal context window pressure
+
+### User Shell Commands
+- user_bash events capture manual shell commands the user runs
+- Repeated commands after agent actions reveal verification patterns
+
+### Model Preferences
+- model_select events track when users switch models
+
+## Feedback Analysis
+
+Each observation may include an active_instincts field listing instinct IDs
+that were injected into the agent's system prompt before that turn.
+
+Use this to update existing instinct confidence scores:
+- Confirmed (+0.05): instinct was active and agent followed guidance without correction
+- Contradicted (-0.15): instinct was active but user corrected the agent
+- Inactive (no change): instinct was injected but trigger never arose
+
+When updating, increment the corresponding count field and recalculate confidence.
+
+## Confidence Scoring Rules
+
+### Initial Confidence (new instincts)
+- 1-2 observations  -> 0.3
+- 3-5 observations  -> 0.5
+- 6-10 observations -> 0.7
+- 11+ observations  -> 0.85
+
+### Clamping
+- Always clamp to [0.1, 0.9]
+
+## Scope Decision Guide
+
+Use project scope when the pattern is specific to this project's tech stack or conventions.
+Use global scope when the pattern applies universally to any coding session.
+When in doubt, prefer project scope.
+
+## Conservativeness Rules
+
+1. Only create a new instinct with 3+ clear independent observations supporting the pattern.
+2. No code snippets in the action field - plain language only.
+3. Each instinct must have one well-defined trigger.
+4. New instincts from observation data alone are capped at 0.85 confidence.
+5. Check existing instincts (provided in the user message) for duplicates before creating. Update instead.
+6. Write actions as clear instructions starting with a verb.
+7. Be skeptical of outliers - patterns seen only in unusual circumstances should not become instincts.`;
+}

package/src/prompts/analyzer-user-single-shot.ts

@@ -0,0 +1,88 @@
+/**
+ * User prompt builder for the single-shot background analyzer.
+ * Includes current instincts inline (no tool calls needed) and filtered observations.
+ */
+import type { InstalledSkill, Instinct, ProjectEntry } from "../types.js";
+import { formatInstinctsForPrompt } from "../cli/analyze-single-shot.js";
+
+export interface SingleShotPromptOptions {
+  agentsMdProject?: string | null;
+  agentsMdGlobal?: string | null;
+  installedSkills?: InstalledSkill[];
+}
+
+/**
+ * Builds the user prompt for the single-shot analyzer.
+ * Embeds all current instincts inline so the model has full context
+ * without making any tool calls.
+ *
+ * @param project          - Project metadata
+ * @param existingInstincts - All current instincts (project + global)
+ * @param observationLines  - Preprocessed observation lines (JSONL strings)
+ * @param options           - Optional AGENTS.md content and installed skills
+ */
+export function buildSingleShotUserPrompt(
+  project: ProjectEntry,
+  existingInstincts: Instinct[],
+  observationLines: string[],
+  options: SingleShotPromptOptions = {}
+): string {
+  const { agentsMdProject = null, agentsMdGlobal = null, installedSkills = [] } = options;
+
+  const observationBlock =
+    observationLines.length > 0
+      ? observationLines.join("\n")
+      : "(no observations recorded yet)";
+
+  const instinctBlock = formatInstinctsForPrompt(existingInstincts);
+
+  const parts: string[] = [
+    "## Project Context",
+    "",
+    `project_id: ${project.id}`,
+    `project_name: ${project.name}`,
+    "",
+    "## Existing Instincts",
+    "",
+    instinctBlock,
+    "",
+    "## New Observations (preprocessed)",
+    "",
+    "```",
+    observationBlock,
+    "```",
+  ];
+
+  if (agentsMdProject != null || agentsMdGlobal != null) {
+    parts.push("", "## Existing Guidelines", "");
+    if (agentsMdProject != null) {
+      parts.push("### Project AGENTS.md", "", agentsMdProject, "");
+    }
+    if (agentsMdGlobal != null) {
+      parts.push("### Global AGENTS.md", "", agentsMdGlobal, "");
+    }
+  }
+
+  if (installedSkills.length > 0) {
+    parts.push("", "## Installed Skills", "");
+    for (const skill of installedSkills) {
+      parts.push(`- **${skill.name}**: ${skill.description}`);
+    }
+    parts.push("");
+  }
+
+  parts.push(
+    "",
+    "## Instructions",
+    "",
+    "1. Review the existing instincts above.",
+    "2. Analyze the new observations for patterns per the system prompt rules.",
+    "3. Return a JSON change-set: create new instincts, update existing ones, or delete obsolete ones.",
+    "4. Apply feedback analysis using the active_instincts field in each observation.",
+    "5. Passive confidence decay has already been applied before this analysis.",
+    "",
+    "Return ONLY the JSON object. No prose, no markdown fences."
+  );
+
+  return parts.join("\n");
+}

package/src/prompts/analyzer-user.ts

@@ -5,7 +5,8 @@
  */
 
 import { existsSync, readFileSync } from "node:fs";
-import type { InstalledSkill, ProjectEntry } from "../types.js";
+import type { InstalledSkill, Observation, ProjectEntry } from "../types.js";
+import { preprocessObservations } from "../observation-preprocessor.js";
 
 /** Maximum number of observation lines to include in analysis. */
 const MAX_TAIL_ENTRIES = 500;
@@ -35,15 +36,18 @@ export function tailObservations(
 export interface TailSinceResult {
   lines: string[];
   totalLineCount: number;
+  /** Number of raw new lines before preprocessing. */
+  rawLineCount: number;
 }
 
 export function tailObservationsSince(
   observationsPath: string,
   sinceLineCount: number,
-  maxEntries = MAX_TAIL_ENTRIES
+  maxEntries = MAX_TAIL_ENTRIES,
+  preprocess = true
 ): TailSinceResult {
   if (!existsSync(observationsPath)) {
-    return { lines: [], totalLineCount: 0 };
+    return { lines: [], totalLineCount: 0, rawLineCount: 0 };
   }
   const content = readFileSync(observationsPath, "utf-8");
   const allLines = content
@@ -55,12 +59,26 @@ export function tailObservationsSince(
 
   // If file was archived/reset (fewer lines than cursor), treat as fresh
   const effectiveSince = totalLineCount < sinceLineCount ? 0 : sinceLineCount;
-  const newLines = allLines.slice(effectiveSince);
+  const newLines = allLines.slice(effectiveSince).slice(-maxEntries);
+  const rawLineCount = newLines.length;
+
+  if (!preprocess) {
+    return { lines: newLines, totalLineCount, rawLineCount };
+  }
+
+  const parsed: Observation[] = [];
+  for (const line of newLines) {
+    try {
+      parsed.push(JSON.parse(line) as Observation);
+    } catch {
+      // skip malformed lines
+    }
+  }
+
+  const filtered = preprocessObservations(parsed);
+  const lines = filtered.map((obs) => JSON.stringify(obs));
 
-  return {
-    lines: newLines.slice(-maxEntries),
-    totalLineCount,
-  };
+  return { lines, totalLineCount, rawLineCount };
 }
 
 export interface AnalyzerUserPromptOptions {