pi-continuous-learning 0.3.0

package/src/prompt-observer.ts

@@ -0,0 +1,92 @@
+/**
+ * User prompt and agent end observation handlers for pi-continuous-learning.
+ * Captures before_agent_start (user_prompt) and agent_end events as JSONL observations.
+ */
+
+import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
+
+// Local event type definitions (not all pi-coding-agent versions re-export these at top level)
+export interface BeforeAgentStartEvent {
+  type: "before_agent_start";
+  prompt: string;
+  systemPrompt: string;
+}
+
+export interface AgentEndEvent {
+  type: "agent_end";
+}
+
+import { getCurrentActiveInstincts } from "./active-instincts.js";
+import { appendObservation } from "./observations.js";
+import { shouldSkipObservation } from "./observer-guard.js";
+import { scrubSecrets } from "./scrubber.js";
+import { logError } from "./error-logger.js";
+import type { Observation, ProjectEntry } from "./types.js";
+
+function getSessionId(ctx: ExtensionContext): string {
+  return ctx.sessionManager.getSessionId();
+}
+
+function buildActiveInstincts(): Pick<Observation, "active_instincts"> {
+  const ids = getCurrentActiveInstincts();
+  return ids.length > 0 ? { active_instincts: ids } : {};
+}
+
+/**
+ * Handles before_agent_start events.
+ * Records an observation with event: user_prompt and scrubbed prompt text.
+ */
+export function handleBeforeAgentStart(
+  event: BeforeAgentStartEvent,
+  ctx: ExtensionContext,
+  project: ProjectEntry,
+  baseDir?: string
+): void {
+  try {
+    if (shouldSkipObservation()) return;
+
+    const input = scrubSecrets(event.prompt);
+
+    const observation: Observation = {
+      timestamp: new Date().toISOString(),
+      event: "user_prompt",
+      session: getSessionId(ctx),
+      project_id: project.id,
+      project_name: project.name,
+      input,
+      ...buildActiveInstincts(),
+    };
+
+    appendObservation(observation, project.id, baseDir);
+  } catch (err) {
+    logError(project.id, "prompt-observer:handleBeforeAgentStart", err, baseDir);
+  }
+}
+
+/**
+ * Handles agent_end events.
+ * Records an observation with event: agent_end.
+ */
+export function handleAgentEnd(
+  _event: AgentEndEvent,
+  ctx: ExtensionContext,
+  project: ProjectEntry,
+  baseDir?: string
+): void {
+  try {
+    if (shouldSkipObservation()) return;
+
+    const observation: Observation = {
+      timestamp: new Date().toISOString(),
+      event: "agent_end",
+      session: getSessionId(ctx),
+      project_id: project.id,
+      project_name: project.name,
+      ...buildActiveInstincts(),
+    };
+
+    appendObservation(observation, project.id, baseDir);
+  } catch (err) {
+    logError(project.id, "prompt-observer:handleAgentEnd", err, baseDir);
+  }
+}

package/src/prompts/analyzer-user.ts

@@ -0,0 +1,156 @@
+/**
+ * Analyzer user prompt construction.
+ * Returns the user prompt string used by the Haiku background analyzer
+ * to locate observations and instinct files for the current project.
+ */
+
+import { existsSync, readFileSync } from "node:fs";
+import type { InstalledSkill, ProjectEntry } from "../types.js";
+
+/** Maximum number of observation lines to include in analysis. */
+const MAX_TAIL_ENTRIES = 500;
+
+/**
+ * Reads the last `maxEntries` lines from a JSONL observations file.
+ * Returns an empty array if the file does not exist.
+ *
+ * @param observationsPath - Absolute path to observations.jsonl
+ * @param maxEntries - Maximum number of lines to return (default 500)
+ */
+export function tailObservations(
+  observationsPath: string,
+  maxEntries = MAX_TAIL_ENTRIES
+): string[] {
+  if (!existsSync(observationsPath)) {
+    return [];
+  }
+  const content = readFileSync(observationsPath, "utf-8");
+  const lines = content
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l.length > 0);
+  return lines.slice(-maxEntries);
+}
+
+export interface TailSinceResult {
+  lines: string[];
+  totalLineCount: number;
+}
+
+export function tailObservationsSince(
+  observationsPath: string,
+  sinceLineCount: number,
+  maxEntries = MAX_TAIL_ENTRIES
+): TailSinceResult {
+  if (!existsSync(observationsPath)) {
+    return { lines: [], totalLineCount: 0 };
+  }
+  const content = readFileSync(observationsPath, "utf-8");
+  const allLines = content
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l.length > 0);
+
+  const totalLineCount = allLines.length;
+
+  // If file was archived/reset (fewer lines than cursor), treat as fresh
+  const effectiveSince = totalLineCount < sinceLineCount ? 0 : sinceLineCount;
+  const newLines = allLines.slice(effectiveSince);
+
+  return {
+    lines: newLines.slice(-maxEntries),
+    totalLineCount,
+  };
+}
+
+export interface AnalyzerUserPromptOptions {
+  agentsMdProject?: string | null;
+  agentsMdGlobal?: string | null;
+  installedSkills?: InstalledSkill[];
+  observationLines?: string[];
+}
+
+/**
+ * Builds the user prompt for the background Haiku analyzer.
+ * Includes observation and instinct file paths plus project context.
+ * Optionally includes AGENTS.md content and installed skills for deduplication.
+ * Template construction only - no subprocess I/O.
+ *
+ * @param observationsPath - Absolute path to the project's observations.jsonl
+ * @param instinctsDir     - Absolute path to the project's instincts directory
+ * @param project          - ProjectEntry with id and name
+ * @param options          - Optional AGENTS.md content and installed skills
+ */
+export function buildAnalyzerUserPrompt(
+  observationsPath: string,
+  instinctsDir: string,
+  project: ProjectEntry,
+  options: AnalyzerUserPromptOptions = {}
+): string {
+  const { agentsMdProject = null, agentsMdGlobal = null, installedSkills = [], observationLines } = options;
+
+  const tailedLines = observationLines ?? tailObservations(observationsPath);
+  const observationBlock =
+    tailedLines.length > 0
+      ? tailedLines.join("\n")
+      : "(no observations recorded yet)";
+
+  const entriesLabel = observationLines
+    ? `new observations since last analysis (up to ${MAX_TAIL_ENTRIES})`
+    : `most recent entries (up to ${MAX_TAIL_ENTRIES})`;
+
+  const parts: string[] = [
+    "## Analysis Task",
+    "",
+    "Analyze the following session observations and update the instinct files accordingly.",
+    "",
+    "## Project Context",
+    "",
+    `project_id: ${project.id}`,
+    `project_name: ${project.name}`,
+    "",
+    "## File Paths",
+    "",
+    `Observations file: ${observationsPath}`,
+    `Instincts directory: ${instinctsDir}`,
+    "",
+    `The following observations are ${entriesLabel}:`,
+    "",
+    "```",
+    observationBlock,
+    "```",
+  ];
+
+  if (agentsMdProject != null || agentsMdGlobal != null) {
+    parts.push("", "## Existing Guidelines", "");
+    if (agentsMdProject != null) {
+      parts.push("### Project AGENTS.md", "", agentsMdProject, "");
+    }
+    if (agentsMdGlobal != null) {
+      parts.push("### Global AGENTS.md", "", agentsMdGlobal, "");
+    }
+  }
+
+  if (installedSkills.length > 0) {
+    parts.push("", "## Installed Skills", "");
+    for (const skill of installedSkills) {
+      parts.push(`- **${skill.name}**: ${skill.description}`);
+    }
+    parts.push("");
+  }
+
+  parts.push(
+    "",
+    "## Instructions",
+    "",
+    "1. Read existing instinct files from the instincts directory.",
+    "2. Analyze the observations above for patterns following the system prompt rules.",
+    "3. Create new instinct files or update existing ones in the instincts directory.",
+    "4. Apply feedback analysis using the active_instincts field in each observation.",
+    "5. Do not delete any instinct files - only create or update.",
+    "",
+    "Note: Passive confidence decay has already been applied to existing instincts before this analysis."
+  );
+
+  return parts.join("\n");
+}

package/src/prompts/evolve-prompt.ts

@@ -0,0 +1,71 @@
+import type { Instinct, InstalledSkill } from "../types.js";
+
+interface InstinctSummary {
+  id: string;
+  title: string;
+  trigger: string;
+  action: string;
+  confidence: number;
+  domain: string;
+  scope: string;
+}
+
+function summarizeInstinct(i: Instinct): InstinctSummary {
+  return {
+    id: i.id,
+    title: i.title,
+    trigger: i.trigger,
+    action: i.action,
+    confidence: i.confidence,
+    domain: i.domain,
+    scope: i.scope,
+  };
+}
+
+export function buildEvolvePrompt(
+  instincts: Instinct[],
+  agentsMdProject?: string | null,
+  agentsMdGlobal?: string | null,
+  installedSkills?: InstalledSkill[]
+): string {
+  const parts: string[] = [
+    "Analyze the following learned instincts and identify opportunities for improvement.",
+    "You have access to instinct tools (instinct_merge, instinct_delete, instinct_write) to act on your findings.",
+    "",
+    "## Instincts",
+    "",
+    "```json",
+    JSON.stringify(instincts.map(summarizeInstinct), null, 2),
+    "```",
+    "",
+    "## Analysis Tasks",
+    "",
+    "1. **Merge candidates**: Find instincts with semantically similar triggers or actions (even if worded differently). Offer to merge them using the instinct_merge tool.",
+    "2. **Duplicates of AGENTS.md**: Flag instincts already covered by the guidelines below. Offer to delete them.",
+    "3. **Promotion candidates**: Project-scoped instincts with confidence >= 0.7 that could become global.",
+    "4. **Skill shadows**: Instincts whose purpose is already served by an installed skill (listed below). Offer to delete them.",
+    "5. **Low-confidence cleanup**: Instincts with confidence < 0.3 or flagged_for_removal that should be deleted.",
+    "",
+    "Present your findings conversationally. For each suggestion, explain your reasoning and ask if I'd like you to take action using the instinct tools.",
+    "If there are no issues, say so briefly.",
+  ];
+
+  if (agentsMdProject || agentsMdGlobal) {
+    parts.push("", "## Current Guidelines (AGENTS.md)", "");
+    if (agentsMdProject) {
+      parts.push("### Project AGENTS.md", "", agentsMdProject, "");
+    }
+    if (agentsMdGlobal) {
+      parts.push("### Global AGENTS.md", "", agentsMdGlobal, "");
+    }
+  }
+
+  if (installedSkills && installedSkills.length > 0) {
+    parts.push("", "## Installed Skills", "");
+    for (const skill of installedSkills) {
+      parts.push(`- **${skill.name}**: ${skill.description}`);
+    }
+  }
+
+  return parts.join("\n");
+}

package/src/scrubber.ts

@@ -0,0 +1,42 @@
+/** Secret scrubbing - replaces sensitive values with [REDACTED] before writing to disk */
+
+export const REDACTED = "[REDACTED]";
+
+/**
+ * Ordered list of regex patterns that match common secret formats.
+ * Full matches are replaced with REDACTED.
+ */
+const SECRET_PATTERNS: readonly RegExp[] = [
+  // Authorization header value (bearer, basic, token schemes)
+  /authorization\s*:\s*(?:bearer|basic|token)\s+\S+/gi,
+  // Standalone bearer token (match full token value, any non-whitespace chars)
+  /bearer\s+\S+/gi,
+  // HTTP header shorthand: x-api-key: <value>, x-auth-token: <value>
+  /x-(?:api-key|auth-token)\s*:\s*\S+/gi,
+  // API key assignments: api_key=, apiKey:, api-key =, access_key=, secret_key=
+  /(?:api[_-]?key|apikey|access[_-]?key|secret[_-]?key)\s*[:=]\s*\S+/gi,
+  // Token assignments: token=, auth_token:, access_token=, refresh_token=
+  /(?:auth[_-]?token|access[_-]?token|refresh[_-]?token)\s*[:=]\s*\S+/gi,
+  // Password assignments: password=, passwd:, pwd =
+  /(?:password|passwd|pwd)\s*[:=]\s*\S+/gi,
+  // Secret / credential / private_key assignments
+  /(?:secret|credential|private[_-]?key)\s*[:=]\s*\S+/gi,
+  // AWS Access Key IDs (AKIA...)
+  /AKIA[0-9A-Z]{16}/g,
+  // OpenAI / Anthropic SDK keys: sk-..., sk-ant-...
+  /sk-(?:ant-api03-)?[a-zA-Z0-9]{32,}/g,
+];
+
+/**
+ * Scrub secrets from arbitrary text.
+ * Replaces all matched secret patterns with [REDACTED].
+ * Non-secret text is returned unchanged.
+ */
+export function scrubSecrets(text: string): string {
+  let result = text;
+  for (const pattern of SECRET_PATTERNS) {
+    pattern.lastIndex = 0;
+    result = result.replace(pattern, REDACTED);
+  }
+  return result;
+}

package/src/storage.ts

@@ -0,0 +1,91 @@
+/**
+ * Storage layout creation for pi-continuous-learning.
+ * Creates the directory structure under ~/.pi/continuous-learning/
+ * on first use, and maintains the projects.json registry.
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import type { ProjectEntry } from "./types.js";
+
+export function getBaseDir(): string {
+  return join(homedir(), ".pi", "continuous-learning");
+}
+
+export function getProjectDir(projectId: string, baseDir = getBaseDir()): string {
+  return join(baseDir, "projects", projectId);
+}
+
+export function getObservationsPath(projectId: string, baseDir = getBaseDir()): string {
+  return join(getProjectDir(projectId, baseDir), "observations.jsonl");
+}
+
+export function getArchiveDir(projectId: string, baseDir = getBaseDir()): string {
+  return join(getProjectDir(projectId, baseDir), "observations.archive");
+}
+
+export function getProjectInstinctsDir(
+  projectId: string,
+  source: "personal" | "inherited",
+  baseDir = getBaseDir()
+): string {
+  return join(getProjectDir(projectId, baseDir), "instincts", source);
+}
+
+export function getGlobalInstinctsDir(
+  source: "personal" | "inherited",
+  baseDir = getBaseDir()
+): string {
+  return join(baseDir, "instincts", source);
+}
+
+export function getProjectsRegistryPath(baseDir = getBaseDir()): string {
+  return join(baseDir, "projects.json");
+}
+
+function ensureDir(dir: string): void {
+  mkdirSync(dir, { recursive: true });
+}
+
+function readProjectsRegistry(registryPath: string): Record<string, ProjectEntry> {
+  if (!existsSync(registryPath)) {
+    return {};
+  }
+  try {
+    return JSON.parse(readFileSync(registryPath, "utf-8")) as Record<string, ProjectEntry>;
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Ensures the full storage directory layout exists.
+ * Idempotent - safe to call multiple times.
+ *
+ * @param project - Project entry with metadata
+ * @param baseDir - Base directory (defaults to ~/.pi/continuous-learning/)
+ */
+export function ensureStorageLayout(project: ProjectEntry, baseDir = getBaseDir()): void {
+  // Global instinct directories
+  ensureDir(join(baseDir, "instincts", "personal"));
+  ensureDir(join(baseDir, "instincts", "inherited"));
+
+  // Project-scoped directories
+  const projectDir = getProjectDir(project.id, baseDir);
+  ensureDir(join(projectDir, "instincts", "personal"));
+  ensureDir(join(projectDir, "instincts", "inherited"));
+  ensureDir(join(projectDir, "observations.archive"));
+
+  // Write project.json only if it does not yet exist
+  const projectJsonPath = join(projectDir, "project.json");
+  if (!existsSync(projectJsonPath)) {
+    writeFileSync(projectJsonPath, JSON.stringify(project, null, 2), "utf-8");
+  }
+
+  // Update projects.json registry (always updates the entry)
+  const registryPath = getProjectsRegistryPath(baseDir);
+  const registry = readProjectsRegistry(registryPath);
+  const updated = { ...registry, [project.id]: project };
+  writeFileSync(registryPath, JSON.stringify(updated, null, 2), "utf-8");
+}

package/src/tool-observer.ts

@@ -0,0 +1,114 @@
+/**
+ * Tool call observation handlers for pi-continuous-learning.
+ * Captures tool_execution_start and tool_execution_end events as JSONL observations.
+ */
+
+import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
+
+// Local event type definitions (not all pi-coding-agent versions re-export these at top level)
+export interface ToolExecutionStartEvent {
+  type: "tool_execution_start";
+  toolCallId: string;
+  toolName: string;
+  args: unknown;
+}
+
+export interface ToolExecutionEndEvent {
+  type: "tool_execution_end";
+  toolCallId: string;
+  toolName: string;
+  result: unknown;
+  isError: boolean;
+}
+
+import { getCurrentActiveInstincts } from "./active-instincts.js";
+import { appendObservation } from "./observations.js";
+import { shouldSkipObservation } from "./observer-guard.js";
+import { scrubSecrets } from "./scrubber.js";
+import { logError } from "./error-logger.js";
+import type { Observation, ProjectEntry } from "./types.js";
+
+export const MAX_TOOL_INPUT_LENGTH = 5000;
+export const MAX_TOOL_OUTPUT_LENGTH = 5000;
+
+function truncate(text: string, maxLen: number): string {
+  if (text.length <= maxLen) return text;
+  return text.slice(0, maxLen);
+}
+
+function getSessionId(ctx: ExtensionContext): string {
+  return ctx.sessionManager.getSessionId();
+}
+
+function buildActiveInstincts(): Pick<Observation, "active_instincts"> {
+  const ids = getCurrentActiveInstincts();
+  return ids.length > 0 ? { active_instincts: ids } : {};
+}
+
+/**
+ * Handles tool_execution_start events.
+ * Records an observation with event: tool_start, tool name, and scrubbed/truncated input.
+ */
+export function handleToolStart(
+  event: ToolExecutionStartEvent,
+  ctx: ExtensionContext,
+  project: ProjectEntry,
+  baseDir?: string
+): void {
+  try {
+    if (shouldSkipObservation()) return;
+
+    const raw = typeof event.args === "string" ? event.args : JSON.stringify(event.args);
+    const input = truncate(scrubSecrets(raw), MAX_TOOL_INPUT_LENGTH);
+
+    const observation: Observation = {
+      timestamp: new Date().toISOString(),
+      event: "tool_start",
+      session: getSessionId(ctx),
+      project_id: project.id,
+      project_name: project.name,
+      tool: event.toolName,
+      input,
+      ...buildActiveInstincts(),
+    };
+
+    appendObservation(observation, project.id, baseDir);
+  } catch (err) {
+    logError(project.id, "tool-observer:handleToolStart", err, baseDir);
+  }
+}
+
+/**
+ * Handles tool_execution_end events.
+ * Records an observation with event: tool_complete, tool name, scrubbed/truncated output, and is_error.
+ */
+export function handleToolEnd(
+  event: ToolExecutionEndEvent,
+  ctx: ExtensionContext,
+  project: ProjectEntry,
+  baseDir?: string
+): void {
+  try {
+    if (shouldSkipObservation()) return;
+
+    const raw =
+      typeof event.result === "string" ? event.result : JSON.stringify(event.result);
+    const output = truncate(scrubSecrets(raw), MAX_TOOL_OUTPUT_LENGTH);
+
+    const observation: Observation = {
+      timestamp: new Date().toISOString(),
+      event: "tool_complete",
+      session: getSessionId(ctx),
+      project_id: project.id,
+      project_name: project.name,
+      tool: event.toolName,
+      output,
+      is_error: event.isError,
+      ...buildActiveInstincts(),
+    };
+
+    appendObservation(observation, project.id, baseDir);
+  } catch (err) {
+    logError(project.id, "tool-observer:handleToolEnd", err, baseDir);
+  }
+}

package/src/types.ts

@@ -0,0 +1,90 @@
+/**
+ * Shared TypeScript interfaces for pi-continuous-learning.
+ * All modules import from this file for consistent data contracts.
+ */
+
+// ---------------------------------------------------------------------------
+// Observation
+// ---------------------------------------------------------------------------
+
+export type ObservationEvent =
+  | "tool_start"
+  | "tool_complete"
+  | "user_prompt"
+  | "agent_end";
+
+export interface Observation {
+  timestamp: string; // ISO 8601 UTC
+  event: ObservationEvent;
+  session: string;
+  project_id: string;
+  project_name: string;
+  tool?: string;
+  input?: string;
+  output?: string;
+  is_error?: boolean;
+  active_instincts?: string[];
+}
+
+// ---------------------------------------------------------------------------
+// Instinct
+// ---------------------------------------------------------------------------
+
+export type InstinctScope = "project" | "global";
+export type InstinctSource = "personal" | "inherited";
+
+export interface Instinct {
+  id: string; // kebab-case
+  title: string;
+  trigger: string;
+  action: string;
+  confidence: number; // 0.1 - 0.9
+  domain: string;
+  source: InstinctSource;
+  scope: InstinctScope;
+  project_id?: string;
+  project_name?: string;
+  created_at: string; // ISO 8601
+  updated_at: string; // ISO 8601
+  observation_count: number;
+  confirmed_count: number;
+  contradicted_count: number;
+  inactive_count: number;
+  evidence?: string[];
+  flagged_for_removal?: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// ProjectEntry
+// ---------------------------------------------------------------------------
+
+export interface ProjectEntry {
+  id: string;
+  name: string;
+  root: string;
+  remote: string;
+  created_at: string; // ISO 8601
+  last_seen: string; // ISO 8601
+}
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+export interface InstalledSkill {
+  name: string;
+  description: string;
+}
+
+export interface Config {
+  run_interval_minutes: number;
+  min_observations_to_analyze: number;
+  min_confidence: number;
+  max_instincts: number;
+  max_injection_chars: number;
+  model: string;
+  timeout_seconds: number;
+  active_hours_start: number; // 0-23
+  active_hours_end: number; // 0-23
+  max_idle_seconds: number;
+}