prjct-cli 0.11.4 → 0.12.0

package/core/outcomes/analyzer.ts

@@ -0,0 +1,333 @@
+/**
+ * Outcome Analyzer
+ *
+ * Analyzes outcomes to extract patterns and insights.
+ * Powers the learning loop for better estimates and agent selection.
+ */
+
+import outcomeRecorder from './recorder'
+import type { Outcome, OutcomeSummary, QualityScore } from './types'
+
+/**
+ * Pattern detected from outcomes.
+ */
+export interface DetectedPattern {
+  /** Pattern description */
+  description: string
+
+  /** Confidence level (0-1) */
+  confidence: number
+
+  /** Number of occurrences supporting this pattern */
+  occurrences: number
+
+  /** Suggested action based on pattern */
+  suggestedAction?: string
+}
+
+/**
+ * Agent performance metrics.
+ */
+export interface AgentMetrics {
+  /** Agent name */
+  agent: string
+
+  /** Number of tasks completed */
+  tasksCompleted: number
+
+  /** Success rate (0-100) */
+  successRate: number
+
+  /** Average quality score */
+  avgQualityScore: number
+
+  /** Estimate accuracy */
+  estimateAccuracy: number
+
+  /** Best task types for this agent */
+  bestFor: string[]
+}
+
+/**
+ * OutcomeAnalyzer - Extracts insights from outcomes.
+ */
+export class OutcomeAnalyzer {
+  /**
+   * Generate summary of all outcomes.
+   */
+  async summarize(projectId: string): Promise<OutcomeSummary> {
+    const outcomes = await outcomeRecorder.getAll(projectId)
+
+    if (outcomes.length === 0) {
+      return {
+        totalOutcomes: 0,
+        avgQualityScore: 0,
+        estimateAccuracy: 0,
+        topBlockers: [],
+        topAgents: [],
+        patternsDetected: [],
+      }
+    }
+
+    // Calculate average quality
+    const avgQuality =
+      outcomes.reduce((sum, o) => sum + o.qualityScore, 0) / outcomes.length
+
+    // Calculate estimate accuracy
+    const estimateAccuracy = await outcomeRecorder.getEstimateAccuracy(projectId)
+
+    // Find top blockers
+    const blockerCounts = new Map<string, number>()
+    for (const outcome of outcomes) {
+      for (const blocker of outcome.blockers || []) {
+        blockerCounts.set(blocker, (blockerCounts.get(blocker) || 0) + 1)
+      }
+    }
+    const topBlockers = [...blockerCounts.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 5)
+      .map(([blocker]) => blocker)
+
+    // Find top agents
+    const agentMetrics = await this.getAgentMetrics(projectId)
+    const topAgents = agentMetrics
+      .sort((a, b) => b.successRate - a.successRate)
+      .slice(0, 3)
+      .map((m) => m.agent)
+
+    // Detect patterns
+    const patterns = await this.detectPatterns(projectId)
+    const patternsDetected = patterns.map((p) => p.description)
+
+    return {
+      totalOutcomes: outcomes.length,
+      avgQualityScore: Math.round(avgQuality * 10) / 10,
+      estimateAccuracy,
+      topBlockers,
+      topAgents,
+      patternsDetected,
+    }
+  }
+
+  /**
+   * Get metrics for each agent.
+   */
+  async getAgentMetrics(projectId: string): Promise<AgentMetrics[]> {
+    const outcomes = await outcomeRecorder.getAll(projectId)
+
+    // Group by agent
+    const byAgent = new Map<string, Outcome[]>()
+    for (const outcome of outcomes) {
+      const agent = outcome.agentUsed || 'unknown'
+      if (!byAgent.has(agent)) {
+        byAgent.set(agent, [])
+      }
+      byAgent.get(agent)!.push(outcome)
+    }
+
+    const metrics: AgentMetrics[] = []
+
+    for (const [agent, agentOutcomes] of byAgent) {
+      const tasksCompleted = agentOutcomes.length
+      const successful = agentOutcomes.filter((o) => o.completedAsPlanned)
+      const successRate = Math.round((successful.length / tasksCompleted) * 100)
+
+      const avgQuality =
+        agentOutcomes.reduce((sum, o) => sum + o.qualityScore, 0) / tasksCompleted
+
+      // Calculate estimate accuracy for this agent
+      const accurateEstimates = agentOutcomes.filter((o) => {
+        if (!o.variance) return false
+        const variance = this.parseVariance(o.variance)
+        const estimated = this.parseDuration(o.estimatedDuration)
+        if (estimated === 0) return false
+        return Math.abs(variance) / estimated <= 0.2
+      })
+      const estimateAccuracy = Math.round(
+        (accurateEstimates.length / tasksCompleted) * 100
+      )
+
+      // Find best task types
+      const taskTypes = new Map<string, number>()
+      for (const o of agentOutcomes.filter((o) => o.completedAsPlanned)) {
+        for (const tag of o.tags || []) {
+          taskTypes.set(tag, (taskTypes.get(tag) || 0) + 1)
+        }
+      }
+      const bestFor = [...taskTypes.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, 3)
+        .map(([type]) => type)
+
+      metrics.push({
+        agent,
+        tasksCompleted,
+        successRate,
+        avgQualityScore: Math.round(avgQuality * 10) / 10,
+        estimateAccuracy,
+        bestFor,
+      })
+    }
+
+    return metrics
+  }
+
+  /**
+   * Detect patterns from outcomes.
+   */
+  async detectPatterns(projectId: string): Promise<DetectedPattern[]> {
+    const outcomes = await outcomeRecorder.getAll(projectId)
+    const patterns: DetectedPattern[] = []
+
+    if (outcomes.length < 3) {
+      return patterns // Need at least 3 outcomes to detect patterns
+    }
+
+    // Pattern: Consistent underestimation
+    const underestimated = outcomes.filter((o) => {
+      const variance = this.parseVariance(o.variance)
+      return variance > 0
+    })
+    if (underestimated.length / outcomes.length > 0.6) {
+      patterns.push({
+        description: 'Tasks consistently take longer than estimated',
+        confidence: underestimated.length / outcomes.length,
+        occurrences: underestimated.length,
+        suggestedAction: 'Add 30% buffer to estimates',
+      })
+    }
+
+    // Pattern: Consistent overestimation
+    const overestimated = outcomes.filter((o) => {
+      const variance = this.parseVariance(o.variance)
+      return variance < 0
+    })
+    if (overestimated.length / outcomes.length > 0.6) {
+      patterns.push({
+        description: 'Tasks consistently finish faster than estimated',
+        confidence: overestimated.length / outcomes.length,
+        occurrences: overestimated.length,
+        suggestedAction: 'Reduce estimates by 20%',
+      })
+    }
+
+    // Pattern: Common blockers
+    const blockerCounts = new Map<string, number>()
+    for (const outcome of outcomes) {
+      for (const blocker of outcome.blockers || []) {
+        blockerCounts.set(blocker, (blockerCounts.get(blocker) || 0) + 1)
+      }
+    }
+    for (const [blocker, count] of blockerCounts) {
+      if (count >= 3) {
+        patterns.push({
+          description: `Recurring blocker: ${blocker}`,
+          confidence: count / outcomes.length,
+          occurrences: count,
+          suggestedAction: `Address root cause of "${blocker}"`,
+        })
+      }
+    }
+
+    // Pattern: Agent performance
+    const agentMetrics = await this.getAgentMetrics(projectId)
+    for (const metrics of agentMetrics) {
+      if (metrics.tasksCompleted >= 5 && metrics.successRate > 90) {
+        patterns.push({
+          description: `${metrics.agent} has high success rate (${metrics.successRate}%)`,
+          confidence: 0.9,
+          occurrences: metrics.tasksCompleted,
+          suggestedAction: `Prefer ${metrics.agent} for similar tasks`,
+        })
+      }
+    }
+
+    return patterns.sort((a, b) => b.confidence - a.confidence)
+  }
+
+  /**
+   * Suggest estimate for a task type based on history.
+   */
+  async suggestEstimate(
+    projectId: string,
+    taskType: string
+  ): Promise<string | null> {
+    const outcomes = await outcomeRecorder.getAll(projectId)
+
+    // Filter by task type (using tags)
+    const relevant = outcomes.filter((o) => o.tags?.includes(taskType))
+
+    if (relevant.length < 2) {
+      return null // Not enough data
+    }
+
+    // Calculate average actual duration
+    const totalMinutes = relevant.reduce((sum, o) => {
+      return sum + this.parseDuration(o.actualDuration)
+    }, 0)
+
+    const avgMinutes = Math.round(totalMinutes / relevant.length)
+
+    // Format as duration string
+    if (avgMinutes >= 60) {
+      const hours = Math.floor(avgMinutes / 60)
+      const mins = avgMinutes % 60
+      return mins > 0 ? `${hours}h ${mins}m` : `${hours}h`
+    }
+    return `${avgMinutes}m`
+  }
+
+  /**
+   * Suggest best agent for a task type.
+   */
+  async suggestAgent(projectId: string, taskType: string): Promise<string | null> {
+    const agentMetrics = await this.getAgentMetrics(projectId)
+
+    // Find agents good at this task type
+    const suitable = agentMetrics.filter((m) => m.bestFor.includes(taskType))
+
+    if (suitable.length === 0) {
+      return null
+    }
+
+    // Return the one with highest success rate
+    return suitable.sort((a, b) => b.successRate - a.successRate)[0].agent
+  }
+
+  /**
+   * Parse variance string to minutes.
+   */
+  private parseVariance(variance: string): number {
+    const match = variance.match(/^([+-])(\d+)([mh])$/)
+    if (!match) return 0
+
+    const sign = match[1] === '-' ? -1 : 1
+    const value = parseInt(match[2], 10)
+    const unit = match[3]
+
+    return sign * (unit === 'h' ? value * 60 : value)
+  }
+
+  /**
+   * Parse duration string to minutes.
+   */
+  private parseDuration(duration: string): number {
+    let minutes = 0
+
+    const hourMatch = duration.match(/(\d+)h/)
+    if (hourMatch) {
+      minutes += parseInt(hourMatch[1], 10) * 60
+    }
+
+    const minMatch = duration.match(/(\d+)m/)
+    if (minMatch) {
+      minutes += parseInt(minMatch[1], 10)
+    }
+
+    return minutes
+  }
+}
+
+// Singleton instance
+const outcomeAnalyzer = new OutcomeAnalyzer()
+export default outcomeAnalyzer

package/core/outcomes/index.ts

@@ -0,0 +1,34 @@
+/**
+ * Outcomes Module
+ *
+ * Records and analyzes execution outcomes for learning.
+ *
+ * @example
+ * ```typescript
+ * import { outcomeRecorder, outcomeAnalyzer } from './outcomes'
+ *
+ * // Record an outcome
+ * await outcomeRecorder.record(projectId, {
+ *   sessionId: 'session_123',
+ *   command: '/p:now',
+ *   task: 'implement auth',
+ *   startedAt: '2025-12-09T14:00:00Z',
+ *   completedAt: '2025-12-09T16:30:00Z',
+ *   estimatedDuration: '2h',
+ *   actualDuration: '2h 30m',
+ *   variance: '+30m',
+ *   completedAsPlanned: true,
+ *   qualityScore: 4,
+ *   agentUsed: 'backend-specialist',
+ *   tags: ['auth', 'backend']
+ * })
+ *
+ * // Analyze outcomes
+ * const summary = await outcomeAnalyzer.summarize(projectId)
+ * const patterns = await outcomeAnalyzer.detectPatterns(projectId)
+ * ```
+ */
+
+export { OutcomeRecorder, default as outcomeRecorder } from './recorder'
+export { OutcomeAnalyzer, default as outcomeAnalyzer } from './analyzer'
+export * from './types'

package/core/outcomes/recorder.ts

@@ -0,0 +1,194 @@
+/**
+ * Outcome Recorder
+ *
+ * Records execution outcomes for learning and analysis.
+ * Appends to JSONL files for efficient streaming.
+ */
+
+import path from 'path'
+import * as fileHelper from '../utils/file-helper'
+import pathManager from '../infrastructure/path-manager'
+import type { Outcome, OutcomeInput, OutcomeFilter } from './types'
+
+const OUTCOMES_DIR = 'outcomes'
+const OUTCOMES_FILE = 'outcomes.jsonl'
+
+/**
+ * OutcomeRecorder - Records and retrieves execution outcomes.
+ */
+export class OutcomeRecorder {
+  /**
+   * Get outcomes directory path for a project.
+   */
+  private getOutcomesDir(projectId: string): string {
+    const globalPath = pathManager.getGlobalProjectPath(projectId)
+    return path.join(globalPath, OUTCOMES_DIR)
+  }
+
+  /**
+   * Get outcomes file path for a project.
+   */
+  private getOutcomesPath(projectId: string): string {
+    return path.join(this.getOutcomesDir(projectId), OUTCOMES_FILE)
+  }
+
+  /**
+   * Record an outcome.
+   */
+  async record(projectId: string, input: OutcomeInput): Promise<Outcome> {
+    const outcome: Outcome = {
+      ...input,
+      id: `outcome_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+    }
+
+    const outcomesPath = this.getOutcomesPath(projectId)
+
+    // Ensure directory exists
+    await fileHelper.ensureDir(path.dirname(outcomesPath))
+
+    // Append to JSONL
+    await fileHelper.appendLine(outcomesPath, JSON.stringify(outcome))
+
+    return outcome
+  }
+
+  /**
+   * Get all outcomes for a project.
+   */
+  async getAll(projectId: string): Promise<Outcome[]> {
+    const outcomesPath = this.getOutcomesPath(projectId)
+
+    if (!(await fileHelper.fileExists(outcomesPath))) {
+      return []
+    }
+
+    const content = await fileHelper.readFile(outcomesPath)
+    if (!content.trim()) {
+      return []
+    }
+
+    return content
+      .trim()
+      .split('\n')
+      .filter((line) => line.trim())
+      .map((line) => JSON.parse(line) as Outcome)
+  }
+
+  /**
+   * Get outcomes matching a filter.
+   */
+  async filter(projectId: string, filter: OutcomeFilter): Promise<Outcome[]> {
+    const all = await this.getAll(projectId)
+
+    return all.filter((outcome) => {
+      if (filter.sessionId && outcome.sessionId !== filter.sessionId) {
+        return false
+      }
+      if (filter.command && outcome.command !== filter.command) {
+        return false
+      }
+      if (filter.agent && outcome.agentUsed !== filter.agent) {
+        return false
+      }
+      if (filter.fromDate && outcome.startedAt < filter.fromDate) {
+        return false
+      }
+      if (filter.toDate && outcome.completedAt > filter.toDate) {
+        return false
+      }
+      if (filter.minQuality && outcome.qualityScore < filter.minQuality) {
+        return false
+      }
+      if (filter.tags && filter.tags.length > 0) {
+        const outcomeTags = outcome.tags || []
+        if (!filter.tags.some((tag) => outcomeTags.includes(tag))) {
+          return false
+        }
+      }
+      return true
+    })
+  }
+
+  /**
+   * Get recent outcomes (last N).
+   */
+  async getRecent(projectId: string, count: number = 10): Promise<Outcome[]> {
+    const all = await this.getAll(projectId)
+    return all.slice(-count)
+  }
+
+  /**
+   * Get outcomes for a specific command.
+   */
+  async getByCommand(projectId: string, command: string): Promise<Outcome[]> {
+    return this.filter(projectId, { command })
+  }
+
+  /**
+   * Get outcomes for a specific agent.
+   */
+  async getByAgent(projectId: string, agent: string): Promise<Outcome[]> {
+    return this.filter(projectId, { agent })
+  }
+
+  /**
+   * Calculate estimate accuracy.
+   */
+  async getEstimateAccuracy(projectId: string): Promise<number> {
+    const outcomes = await this.getAll(projectId)
+
+    if (outcomes.length === 0) {
+      return 0
+    }
+
+    // Count outcomes where variance was within 20%
+    const accurate = outcomes.filter((o) => {
+      if (!o.variance) return false
+      const variance = this.parseVariance(o.variance)
+      const estimated = this.parseDuration(o.estimatedDuration)
+      if (estimated === 0) return false
+      return Math.abs(variance) / estimated <= 0.2
+    })
+
+    return Math.round((accurate.length / outcomes.length) * 100)
+  }
+
+  /**
+   * Parse variance string to minutes.
+   * "+30m" → 30, "-15m" → -15
+   */
+  private parseVariance(variance: string): number {
+    const match = variance.match(/^([+-])(\d+)([mh])$/)
+    if (!match) return 0
+
+    const sign = match[1] === '-' ? -1 : 1
+    const value = parseInt(match[2], 10)
+    const unit = match[3]
+
+    return sign * (unit === 'h' ? value * 60 : value)
+  }
+
+  /**
+   * Parse duration string to minutes.
+   * "2h" → 120, "30m" → 30, "1h 30m" → 90
+   */
+  private parseDuration(duration: string): number {
+    let minutes = 0
+
+    const hourMatch = duration.match(/(\d+)h/)
+    if (hourMatch) {
+      minutes += parseInt(hourMatch[1], 10) * 60
+    }
+
+    const minMatch = duration.match(/(\d+)m/)
+    if (minMatch) {
+      minutes += parseInt(minMatch[1], 10)
+    }
+
+    return minutes
+  }
+}
+
+// Singleton instance
+const outcomeRecorder = new OutcomeRecorder()
+export default outcomeRecorder

package/core/outcomes/types.ts

@@ -0,0 +1,145 @@
+/**
+ * Outcome Types
+ *
+ * Types for recording execution results.
+ * Enables learning from past executions.
+ */
+
+/**
+ * Quality score for task completion.
+ * 1 = Poor, 5 = Excellent
+ */
+export type QualityScore = 1 | 2 | 3 | 4 | 5
+
+/**
+ * Outcome of a command or task execution.
+ *
+ * @property id - Unique outcome ID
+ * @property sessionId - Session this outcome belongs to
+ * @property command - Command that was executed
+ * @property task - Task description
+ */
+export interface Outcome {
+  /** Unique outcome ID */
+  id: string
+
+  /** Session this outcome belongs to */
+  sessionId: string
+
+  /** Command that was executed */
+  command: string
+
+  /** Task description */
+  task: string
+
+  // =========== Timing ===========
+
+  /** When execution started */
+  startedAt: string
+
+  /** When execution completed */
+  completedAt: string
+
+  /** Estimated duration (before execution) */
+  estimatedDuration: string
+
+  /** Actual duration */
+  actualDuration: string
+
+  /**
+   * Variance from estimate.
+   * Positive = took longer ("+30m")
+   * Negative = faster ("-15m")
+   */
+  variance: string
+
+  // =========== Quality ===========
+
+  /** Whether task was completed as planned */
+  completedAsPlanned: boolean
+
+  /** Quality score (1-5) */
+  qualityScore: QualityScore
+
+  /** Blockers encountered during execution */
+  blockers?: string[]
+
+  /** Error messages if any */
+  errors?: string[]
+
+  // =========== Agent ===========
+
+  /** Agent used for this task */
+  agentUsed?: string
+
+  /** Agent confidence level (0-1) */
+  agentConfidence?: number
+
+  /** Whether agent performed well */
+  agentPerformedWell?: boolean
+
+  // =========== Learning ===========
+
+  /** Pattern detected from this execution */
+  patternDetected?: string
+
+  /** Suggested estimate for similar tasks */
+  nextTimeEstimate?: string
+
+  /** Tags for categorization */
+  tags?: string[]
+}
+
+/**
+ * Summary of outcomes for analysis.
+ */
+export interface OutcomeSummary {
+  /** Total outcomes analyzed */
+  totalOutcomes: number
+
+  /** Average quality score */
+  avgQualityScore: number
+
+  /** Estimate accuracy percentage (0-100) */
+  estimateAccuracy: number
+
+  /** Most common blockers */
+  topBlockers: string[]
+
+  /** Best performing agents */
+  topAgents: string[]
+
+  /** Patterns detected */
+  patternsDetected: string[]
+}
+
+/**
+ * Outcome filter options.
+ */
+export interface OutcomeFilter {
+  /** Filter by session ID */
+  sessionId?: string
+
+  /** Filter by command */
+  command?: string
+
+  /** Filter by agent */
+  agent?: string
+
+  /** Filter by date range start */
+  fromDate?: string
+
+  /** Filter by date range end */
+  toDate?: string
+
+  /** Filter by minimum quality */
+  minQuality?: QualityScore
+
+  /** Filter by tags */
+  tags?: string[]
+}
+
+/**
+ * Outcome creation input.
+ */
+export type OutcomeInput = Omit<Outcome, 'id'>