smart-context-mcp 1.1.0 → 1.3.0

package/scripts/report-workflow-metrics.js

@@ -0,0 +1,255 @@
+#!/usr/bin/env node
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+// Dynamic import to handle SQLite availability
+let workflowTracker;
+try {
+  workflowTracker = await import('../src/workflow-tracker.js');
+} catch {
+  workflowTracker = await import('../src/workflow-tracker-stub.js');
+}
+
+const { getWorkflowMetrics, getWorkflowSummaryByType, WORKFLOW_DEFINITIONS } = workflowTracker;
+
+const parseArgs = (argv) => {
+  const options = {
+    type: null,
+    sessionId: null,
+    limit: 10,
+    json: false,
+    summary: false,
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const token = argv[index];
+
+    if (token === '--type') {
+      options.type = argv[index + 1];
+      index += 1;
+      continue;
+    }
+
+    if (token === '--session') {
+      options.sessionId = argv[index + 1];
+      index += 1;
+      continue;
+    }
+
+    if (token === '--limit') {
+      options.limit = parseInt(argv[index + 1], 10);
+      index += 1;
+      continue;
+    }
+
+    if (token === '--json') {
+      options.json = true;
+      continue;
+    }
+
+    if (token === '--summary') {
+      options.summary = true;
+      continue;
+    }
+
+    if (token === '--help' || token === '-h') {
+      console.log(`
+Usage: report-workflow-metrics [options]
+
+Options:
+  --type <type>       Filter by workflow type (debugging, code-review, refactoring, testing, architecture)
+  --session <id>      Filter by session ID
+  --limit <n>         Limit number of workflows (default: 10)
+  --summary           Show summary by workflow type
+  --json              Output as JSON
+  --help, -h          Show this help
+
+Examples:
+  # Show summary by workflow type
+  npm run report:workflows -- --summary
+
+  # Show last 10 debugging workflows
+  npm run report:workflows -- --type debugging
+
+  # Show workflows for specific session
+  npm run report:workflows -- --session abc123
+
+  # Output as JSON
+  npm run report:workflows -- --json
+      `);
+      process.exit(0);
+    }
+
+    throw new Error(`Unknown argument: ${token}`);
+  }
+
+  return options;
+};
+
+const formatNumber = (value) => new Intl.NumberFormat('en-US').format(value);
+
+const formatDuration = (ms) => {
+  if (ms < 1000) return `${ms}ms`;
+  if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
+  if (ms < 3600000) return `${(ms / 60000).toFixed(1)}m`;
+  return `${(ms / 3600000).toFixed(1)}h`;
+};
+
+const printSummary = (summary) => {
+  console.log('');
+  console.log('Workflow Metrics Summary');
+  console.log('═'.repeat(120));
+  console.log('');
+
+  if (!Array.isArray(summary) || summary.length === 0) {
+    console.log('No completed workflows found.');
+    console.log('');
+    console.log('Workflows are tracked when:');
+    console.log('  1. Agent calls smart_turn(start) with a task description');
+    console.log('  2. Task matches a workflow pattern (debugging, review, refactor, testing, architecture)');
+    console.log('  3. Agent calls smart_turn(end) to complete the workflow');
+    console.log('');
+    return;
+  }
+
+  const totalWorkflows = summary.reduce((sum, s) => sum + s.count, 0);
+  const totalRaw = summary.reduce((sum, s) => sum + s.total_raw_tokens, 0);
+  const totalCompressed = summary.reduce((sum, s) => sum + s.total_compressed_tokens, 0);
+  const totalSaved = summary.reduce((sum, s) => sum + s.total_saved_tokens, 0);
+  const totalBaseline = summary.reduce((sum, s) => sum + s.total_baseline_tokens, 0);
+
+  console.log(`Total Workflows: ${formatNumber(totalWorkflows)}`);
+  console.log(`Total Raw Tokens: ${formatNumber(totalRaw)}`);
+  console.log(`Total Compressed Tokens: ${formatNumber(totalCompressed)}`);
+  console.log(`Total Saved Tokens: ${formatNumber(totalSaved)} (${((totalSaved / totalRaw) * 100).toFixed(2)}%)`);
+  console.log(`Total Baseline Tokens: ${formatNumber(totalBaseline)}`);
+  console.log(`Savings vs Baseline: ${formatNumber(totalBaseline - totalCompressed)} (${(((totalBaseline - totalCompressed) / totalBaseline) * 100).toFixed(2)}%)`);
+  console.log('');
+  console.log('By Workflow Type:');
+  console.log('─'.repeat(120));
+  console.log(
+    'Type'.padEnd(20) +
+      'Count'.padStart(8) +
+      'Avg Steps'.padStart(12) +
+      'Avg Duration'.padStart(15) +
+      'Avg Savings'.padStart(15) +
+      'vs Baseline'.padStart(15),
+  );
+  console.log('─'.repeat(120));
+
+  for (const s of summary) {
+    const def = WORKFLOW_DEFINITIONS[s.workflow_type];
+    const name = def ? def.name : s.workflow_type;
+
+    console.log(
+      name.padEnd(20) +
+        formatNumber(s.count).padStart(8) +
+        s.avgStepsCount.toString().padStart(12) +
+        formatDuration(s.avgDurationMs).padStart(15) +
+        `${s.avgSavingsPct}%`.padStart(15) +
+        `${s.avgVsBaselinePct}%`.padStart(15),
+    );
+  }
+
+  console.log('─'.repeat(120));
+  console.log('');
+
+  // Show detailed breakdown for each workflow type
+  console.log('Detailed Breakdown:');
+  console.log('');
+
+  for (const s of summary) {
+    const def = WORKFLOW_DEFINITIONS[s.workflow_type];
+    const name = def ? def.name : s.workflow_type;
+
+    console.log(`${name}:`);
+    console.log(`  Workflows: ${formatNumber(s.count)}`);
+    console.log(`  Avg Steps: ${s.avgStepsCount}`);
+    console.log(`  Avg Duration: ${formatDuration(s.avgDurationMs)}`);
+    console.log(`  Total Raw Tokens: ${formatNumber(s.total_raw_tokens)}`);
+    console.log(`  Total Compressed Tokens: ${formatNumber(s.total_compressed_tokens)}`);
+    console.log(`  Total Saved Tokens: ${formatNumber(s.total_saved_tokens)} (${s.avgSavingsPct}%)`);
+    console.log(`  Baseline Tokens: ${formatNumber(s.total_baseline_tokens)}`);
+    console.log(`  Savings vs Baseline: ${formatNumber(s.total_baseline_tokens - s.total_compressed_tokens)} (${s.avgVsBaselinePct}%)`);
+    console.log('');
+  }
+};
+
+const printWorkflows = (workflows) => {
+  console.log('');
+  console.log('Recent Workflows');
+  console.log('═'.repeat(120));
+  console.log('');
+
+  if (workflows.length === 0) {
+    console.log('No workflows found.');
+    console.log('');
+    return;
+  }
+
+  for (const w of workflows) {
+    const def = WORKFLOW_DEFINITIONS[w.workflow_type];
+    const name = def ? def.name : w.workflow_type;
+    const status = w.end_time ? '✓ Completed' : '⏳ In Progress';
+
+    console.log(`${name} (${status})`);
+    console.log(`  Workflow ID: ${w.workflow_id}`);
+    console.log(`  Session ID: ${w.session_id || 'N/A'}`);
+    console.log(`  Started: ${new Date(w.start_time).toLocaleString()}`);
+
+    if (w.end_time) {
+      console.log(`  Ended: ${new Date(w.end_time).toLocaleString()}`);
+      console.log(`  Duration: ${formatDuration(w.duration_ms)}`);
+      console.log(`  Steps: ${w.steps_count}`);
+      console.log(`  Tools Used: ${w.toolsUsed.join(', ')}`);
+      console.log(`  Raw Tokens: ${formatNumber(w.raw_tokens)}`);
+      console.log(`  Compressed Tokens: ${formatNumber(w.compressed_tokens)}`);
+      console.log(`  Saved Tokens: ${formatNumber(w.saved_tokens)} (${w.savings_pct}%)`);
+      console.log(`  Baseline Tokens: ${formatNumber(w.baseline_tokens)}`);
+      console.log(`  Savings vs Baseline: ${formatNumber(w.baseline_tokens - w.compressed_tokens)} (${w.vs_baseline_pct}%)`);
+    }
+
+    console.log('');
+  }
+};
+
+const main = async () => {
+  try {
+    const options = parseArgs(process.argv.slice(2));
+
+    if (options.summary) {
+      const summary = getWorkflowSummaryByType();
+
+      if (options.json) {
+        console.log(JSON.stringify(summary, null, 2));
+        return;
+      }
+
+      printSummary(summary);
+      return;
+    }
+
+    const workflows = getWorkflowMetrics({
+      workflowType: options.type,
+      sessionId: options.sessionId,
+      limit: options.limit,
+      completed: true,
+    });
+
+    if (options.json) {
+      console.log(JSON.stringify(workflows, null, 2));
+      return;
+    }
+
+    printWorkflows(workflows);
+  } catch (error) {
+    console.error('Error:', error.message);
+    process.exit(1);
+  }
+};
+
+const isDirectExecution = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+
+if (isDirectExecution) {
+  main();
+}

package/src/analytics/adoption.js

@@ -0,0 +1,197 @@
+/**
+ * Adoption analytics - measures how often agents use devctx tools in practice
+ * 
+ * Limitations:
+ * - Complexity is inferred from operation count, not actual task complexity
+ * - Can't detect feedback shown (requires agent cooperation)
+ * - Can't detect forcing prompts (requires prompt analysis)
+ * - Can only measure when devctx IS used, not when it's ignored
+ */
+
+const DEVCTX_TOOLS = new Set([
+  'smart_read',
+  'smart_search',
+  'smart_context',
+  'smart_shell',
+  'smart_summary',
+  'smart_turn',
+  'smart_read_batch',
+  'smart_metrics',
+  'build_index',
+  'warm_cache',
+  'git_blame',
+  'cross_project',
+]);
+
+const inferComplexity = (opCount, fileCount) => {
+  if (opCount <= 2 && fileCount <= 1) return 'trivial';
+  if (opCount <= 5 && fileCount <= 3) return 'simple';
+  if (opCount <= 15 && fileCount <= 10) return 'moderate';
+  return 'complex';
+};
+
+const groupBySession = (entries) => {
+  const sessions = new Map();
+  
+  for (const entry of entries) {
+    const sessionId = entry.sessionId || entry.session_id || 'unknown';
+    
+    if (!sessions.has(sessionId)) {
+      sessions.set(sessionId, {
+        sessionId,
+        operations: [],
+        devctxTools: new Set(),
+        nativeTools: new Set(),
+        filesAccessed: new Set(),
+        totalRawTokens: 0,
+        totalCompressedTokens: 0,
+        totalSavedTokens: 0,
+      });
+    }
+    
+    const session = sessions.get(sessionId);
+    session.operations.push(entry);
+    
+    const tool = entry.tool;
+    if (DEVCTX_TOOLS.has(tool)) {
+      session.devctxTools.add(tool);
+    } else if (tool) {
+      session.nativeTools.add(tool);
+    }
+    
+    if (entry.target) {
+      session.filesAccessed.add(entry.target);
+    }
+    
+    session.totalRawTokens += Number(entry.rawTokens || entry.raw_tokens || 0);
+    session.totalCompressedTokens += Number(entry.compressedTokens || entry.compressed_tokens || 0);
+    session.totalSavedTokens += Number(entry.savedTokens || entry.saved_tokens || 0);
+  }
+  
+  return Array.from(sessions.values());
+};
+
+export const analyzeAdoption = (entries) => {
+  const sessions = groupBySession(entries);
+  
+  const sessionsWithDevctx = sessions.filter(s => s.devctxTools.size > 0);
+  const sessionsWithoutDevctx = sessions.filter(s => s.devctxTools.size === 0 && s.operations.length > 0);
+  
+  const byComplexity = {
+    trivial: { total: 0, withDevctx: 0 },
+    simple: { total: 0, withDevctx: 0 },
+    moderate: { total: 0, withDevctx: 0 },
+    complex: { total: 0, withDevctx: 0 },
+  };
+  
+  for (const session of sessions) {
+    const complexity = inferComplexity(session.operations.length, session.filesAccessed.size);
+    byComplexity[complexity].total += 1;
+    if (session.devctxTools.size > 0) {
+      byComplexity[complexity].withDevctx += 1;
+    }
+  }
+  
+  const nonTrivialSessions = sessions.filter(s => {
+    const complexity = inferComplexity(s.operations.length, s.filesAccessed.size);
+    return complexity !== 'trivial';
+  });
+  
+  const nonTrivialWithDevctx = nonTrivialSessions.filter(s => s.devctxTools.size > 0);
+  
+  const toolUsageCount = {};
+  for (const session of sessionsWithDevctx) {
+    for (const tool of session.devctxTools) {
+      toolUsageCount[tool] = (toolUsageCount[tool] || 0) + 1;
+    }
+  }
+  
+  return {
+    totalSessions: sessions.length,
+    sessionsWithDevctx: sessionsWithDevctx.length,
+    sessionsWithoutDevctx: sessionsWithoutDevctx.length,
+    adoptionRate: sessions.length > 0 
+      ? Number(((sessionsWithDevctx.length / sessions.length) * 100).toFixed(1))
+      : 0,
+    
+    nonTrivial: {
+      total: nonTrivialSessions.length,
+      withDevctx: nonTrivialWithDevctx.length,
+      adoptionRate: nonTrivialSessions.length > 0
+        ? Number(((nonTrivialWithDevctx.length / nonTrivialSessions.length) * 100).toFixed(1))
+        : 0,
+    },
+    
+    byComplexity: Object.fromEntries(
+      Object.entries(byComplexity).map(([level, stats]) => [
+        level,
+        {
+          ...stats,
+          adoptionRate: stats.total > 0 
+            ? Number(((stats.withDevctx / stats.total) * 100).toFixed(1))
+            : 0,
+        },
+      ])
+    ),
+    
+    toolUsageCount,
+    
+    avgToolsPerSession: sessionsWithDevctx.length > 0
+      ? Number((sessionsWithDevctx.reduce((sum, s) => sum + s.devctxTools.size, 0) / sessionsWithDevctx.length).toFixed(1))
+      : 0,
+    
+    avgTokenSavingsWhenUsed: sessionsWithDevctx.length > 0
+      ? Number((sessionsWithDevctx.reduce((sum, s) => sum + s.totalSavedTokens, 0) / sessionsWithDevctx.length).toFixed(0))
+      : 0,
+  };
+};
+
+export const formatAdoptionReport = (stats) => {
+  const lines = [];
+  
+  lines.push('');
+  lines.push('Adoption Analysis (Inferred from Tool Usage)');
+  lines.push('');
+  lines.push(`Total sessions:        ${stats.totalSessions}`);
+  lines.push(`Sessions with devctx:  ${stats.sessionsWithDevctx} (${stats.adoptionRate}%)`);
+  lines.push(`Sessions without:      ${stats.sessionsWithoutDevctx} (${100 - stats.adoptionRate}%)`);
+  lines.push('');
+  
+  lines.push('Non-Trivial Tasks Only:');
+  lines.push(`Total:                 ${stats.nonTrivial.total}`);
+  lines.push(`With devctx:           ${stats.nonTrivial.withDevctx} (${stats.nonTrivial.adoptionRate}%)`);
+  lines.push(`Without devctx:        ${stats.nonTrivial.total - stats.nonTrivial.withDevctx} (${100 - stats.nonTrivial.adoptionRate}%)`);
+  lines.push('');
+  
+  lines.push('By Inferred Complexity:');
+  for (const [level, data] of Object.entries(stats.byComplexity)) {
+    if (data.total === 0) continue;
+    lines.push(`- ${level.padEnd(10)} ${data.withDevctx}/${data.total} (${data.adoptionRate}%)`);
+  }
+  lines.push('');
+  
+  if (stats.sessionsWithDevctx > 0) {
+    lines.push('When devctx IS used:');
+    lines.push(`Avg tools/session:     ${stats.avgToolsPerSession}`);
+    lines.push(`Avg token savings:     ${stats.avgTokenSavingsWhenUsed.toLocaleString()} tokens`);
+    lines.push('');
+  }
+  
+  lines.push('Top Tools Used:');
+  const sortedTools = Object.entries(stats.toolUsageCount)
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 5);
+  for (const [tool, count] of sortedTools) {
+    lines.push(`- ${tool.padEnd(20)} ${count} sessions`);
+  }
+  lines.push('');
+  
+  lines.push('Limitations:');
+  lines.push('- Complexity inferred from operation count (not actual task complexity)');
+  lines.push('- Can only measure when devctx IS used (tool calls visible)');
+  lines.push('- Cannot measure feedback shown or forcing prompts (requires agent cooperation)');
+  lines.push('- Sessions without devctx may be simple tasks (not adoption failures)');
+  lines.push('');
+  
+  return lines.join('\n');
+};

package/src/decision-explainer.js

@@ -0,0 +1,168 @@
+/**
+ * Decision explainer - tracks and explains why devctx tools were used or not used
+ * 
+ * Enable with environment variable: DEVCTX_EXPLAIN=true
+ * 
+ * Provides transparency into agent decision-making:
+ * - Why was smart_read used instead of Read?
+ * - Why was smart_search chosen?
+ * - What are the expected benefits?
+ */
+
+const sessionDecisions = {
+  decisions: [],
+  enabled: false,
+};
+
+/**
+ * Check if explanations are enabled
+ */
+export const isExplainEnabled = () => {
+  const envValue = process.env.DEVCTX_EXPLAIN?.toLowerCase();
+  const enabled = envValue === 'true' || envValue === '1' || envValue === 'yes';
+  sessionDecisions.enabled = enabled;
+  return enabled;
+};
+
+/**
+ * Record a decision with explanation
+ */
+export const recordDecision = ({
+  tool,
+  action,
+  reason,
+  alternative = null,
+  expectedBenefit = null,
+  context = null,
+}) => {
+  if (!isExplainEnabled()) return;
+  
+  sessionDecisions.decisions.push({
+    tool,
+    action,
+    reason,
+    alternative,
+    expectedBenefit,
+    context,
+    timestamp: new Date().toISOString(),
+  });
+};
+
+/**
+ * Get all decisions for current session
+ */
+export const getSessionDecisions = () => {
+  return sessionDecisions.decisions;
+};
+
+/**
+ * Format decisions as markdown for display
+ */
+export const formatDecisionExplanations = () => {
+  if (!isExplainEnabled()) return '';
+  
+  const decisions = getSessionDecisions();
+  if (decisions.length === 0) return '';
+  
+  const lines = [];
+  lines.push('');
+  lines.push('---');
+  lines.push('');
+  lines.push('🤖 **Decision explanations:**');
+  lines.push('');
+  
+  for (const decision of decisions) {
+    lines.push(`**${decision.tool}** (${decision.action})`);
+    lines.push(`- **Why:** ${decision.reason}`);
+    
+    if (decision.alternative) {
+      lines.push(`- **Instead of:** ${decision.alternative}`);
+    }
+    
+    if (decision.expectedBenefit) {
+      lines.push(`- **Expected benefit:** ${decision.expectedBenefit}`);
+    }
+    
+    if (decision.context) {
+      lines.push(`- **Context:** ${decision.context}`);
+    }
+    
+    lines.push('');
+  }
+  
+  lines.push('*To disable: `export DEVCTX_EXPLAIN=false`*');
+  
+  return lines.join('\n');
+};
+
+/**
+ * Reset session decisions (for testing or manual reset)
+ */
+export const resetSessionDecisions = () => {
+  sessionDecisions.decisions = [];
+};
+
+/**
+ * Common decision reasons (for consistency)
+ */
+export const DECISION_REASONS = {
+  // smart_read reasons
+  LARGE_FILE: 'File is large (>500 lines), outline mode extracts structure only',
+  SYMBOL_EXTRACTION: 'Extracting specific symbol, smart_read can locate and extract it efficiently',
+  TOKEN_BUDGET: 'Token budget constraint, cascading to more compressed mode',
+  MULTIPLE_SYMBOLS: 'Reading multiple symbols, smart_read can batch them',
+  
+  // smart_search reasons
+  MULTIPLE_FILES: 'Query spans 50+ files, smart_search ranks by relevance',
+  INTENT_AWARE: 'Intent-aware search prioritizes relevant results (debug/implementation/tests)',
+  INDEX_BOOST: 'Symbol index available, boosting relevant matches',
+  PATTERN_SEARCH: 'Complex pattern search, smart_search handles regex efficiently',
+  
+  // smart_context reasons
+  TASK_CONTEXT: 'Building complete context for task, smart_context orchestrates multiple reads',
+  RELATED_FILES: 'Need related files (callers, tests, types), smart_context finds them',
+  ONE_CALL: 'Single call to get all context, more efficient than multiple reads',
+  DIFF_ANALYSIS: 'Analyzing git diff, smart_context expands changed symbols',
+  
+  // smart_shell reasons
+  COMMAND_OUTPUT: 'Command output needs compression (git log, npm test, etc.)',
+  RELEVANT_LINES: 'Extracting relevant lines from command output',
+  SAFE_EXECUTION: 'Using allowlist-validated command execution',
+  
+  // smart_summary reasons
+  CHECKPOINT: 'Saving task checkpoint for session recovery',
+  RESUME: 'Recovering previous task context',
+  PERSISTENCE: 'Maintaining task state across agent restarts',
+  
+  // Native tool reasons
+  SIMPLE_TASK: 'Task is simple, native tool is more direct',
+  ALREADY_CACHED: 'Content already in context, no need for compression',
+  SINGLE_LINE: 'Reading single line, native Read is sufficient',
+  SMALL_FILE: 'File is small (<100 lines), compression not needed',
+  NO_INDEX: 'No symbol index available, native search is equivalent',
+};
+
+/**
+ * Common expected benefits (for consistency)
+ */
+export const EXPECTED_BENEFITS = {
+  TOKEN_SAVINGS: (tokens) => `~${formatTokens(tokens)} saved`,
+  FASTER_RESPONSE: 'Faster response due to less data to process',
+  BETTER_RANKING: 'Better result ranking, relevant items first',
+  COMPLETE_CONTEXT: 'Complete context in single call',
+  SESSION_RECOVERY: 'Can recover task state if agent restarts',
+  FOCUSED_RESULTS: 'Focused on relevant code only',
+};
+
+/**
+ * Format token count for display
+ */
+const formatTokens = (tokens) => {
+  if (tokens >= 1000000) {
+    return `${(tokens / 1000000).toFixed(1)}M tokens`;
+  }
+  if (tokens >= 1000) {
+    return `${(tokens / 1000).toFixed(1)}K tokens`;
+  }
+  return `${tokens} tokens`;
+};