prjct-cli 0.44.1 → 0.45.0

package/core/context-tools/summary-tool.ts

@@ -0,0 +1,309 @@
+/**
+ * Summary Tool - Intelligent file summarization
+ *
+ * Combines:
+ * - Code signatures (public API)
+ * - JSDoc/docstring extraction
+ * - Key dependencies
+ *
+ * Achieves high compression by returning only public-facing elements.
+ *
+ * @module context-tools/summary-tool
+ * @version 1.0.0
+ */
+
+import fs from 'fs/promises'
+import path from 'path'
+import type { SummaryToolOutput, PublicAPIEntry, SignatureType } from './types'
+import { measureCompression, noCompression, combineMetrics } from './token-counter'
+import { extractSignatures } from './signatures-tool'
+import { analyzeImports } from './imports-tool'
+import { isNotFoundError } from '../types/fs'
+
+// =============================================================================
+// Docstring Patterns
+// =============================================================================
+
+interface DocstringPattern {
+  start: RegExp
+  end: RegExp | null
+  singleLine?: boolean
+}
+
+/**
+ * Docstring patterns by language
+ */
+const DOCSTRING_PATTERNS: Record<string, DocstringPattern[]> = {
+  typescript: [
+    { start: /\/\*\*/, end: /\*\// }, // JSDoc
+    { start: /\/\/\//, end: null, singleLine: true }, // Triple-slash
+  ],
+  javascript: [
+    { start: /\/\*\*/, end: /\*\// }, // JSDoc
+  ],
+  python: [
+    { start: /"""/, end: /"""/ }, // Triple quotes
+    { start: /'''/, end: /'''/ }, // Single quotes
+  ],
+  go: [
+    { start: /\/\//, end: null, singleLine: true }, // Line comment (Go uses these as docs)
+  ],
+  rust: [
+    { start: /\/\/\//, end: null, singleLine: true }, // Doc comment
+    { start: /\/\/!/, end: null, singleLine: true }, // Inner doc comment
+  ],
+}
+
+/**
+ * Extension to language mapping
+ */
+const EXT_TO_LANG: Record<string, string> = {
+  '.ts': 'typescript',
+  '.tsx': 'typescript',
+  '.js': 'javascript',
+  '.jsx': 'javascript',
+  '.mjs': 'javascript',
+  '.py': 'python',
+  '.go': 'go',
+  '.rs': 'rust',
+}
+
+// =============================================================================
+// Main Functions
+// =============================================================================
+
+/**
+ * Generate an intelligent summary of a file
+ *
+ * @param filePath - Path to the file
+ * @param projectPath - Project root path
+ * @returns Summary with public API and metrics
+ */
+export async function summarizeFile(
+  filePath: string,
+  projectPath: string = process.cwd()
+): Promise<SummaryToolOutput> {
+  const absolutePath = path.isAbsolute(filePath)
+    ? filePath
+    : path.join(projectPath, filePath)
+
+  // Read file content
+  let content: string
+  try {
+    content = await fs.readFile(absolutePath, 'utf-8')
+  } catch (error) {
+    if (isNotFoundError(error)) {
+      return {
+        file: filePath,
+        purpose: 'File not found',
+        publicAPI: [],
+        dependencies: [],
+        metrics: noCompression(''),
+      }
+    }
+    throw error
+  }
+
+  // Get language
+  const ext = path.extname(filePath).toLowerCase()
+  const language = EXT_TO_LANG[ext] || 'unknown'
+
+  // Extract signatures
+  const signaturesResult = await extractSignatures(filePath, projectPath)
+
+  // Extract imports for dependencies
+  const importsResult = await analyzeImports(filePath, projectPath)
+
+  // Extract file-level docstring (purpose)
+  const purpose = extractFilePurpose(content, language)
+
+  // Build public API from exported signatures
+  const publicAPI: PublicAPIEntry[] = signaturesResult.signatures
+    .filter((sig) => sig.exported)
+    .map((sig) => ({
+      name: sig.name,
+      type: sig.type,
+      signature: sig.signature,
+      description: sig.docstring
+        ? extractDescriptionFromDocstring(sig.docstring)
+        : undefined,
+    }))
+
+  // Get key dependencies (internal only, external are obvious from package.json)
+  const dependencies = importsResult.imports
+    .filter((imp) => !imp.isExternal && imp.resolved)
+    .map((imp) => imp.resolved!)
+    .slice(0, 10) // Limit to 10
+
+  // Build summary content for metrics
+  const summaryContent = buildSummaryText(purpose, publicAPI, dependencies)
+
+  return {
+    file: filePath,
+    purpose,
+    publicAPI,
+    dependencies,
+    metrics: measureCompression(content, summaryContent),
+  }
+}
+
+/**
+ * Summarize all files in a directory
+ */
+export async function summarizeDirectory(
+  dirPath: string,
+  projectPath: string = process.cwd(),
+  options: { recursive?: boolean } = {}
+): Promise<SummaryToolOutput[]> {
+  const absolutePath = path.isAbsolute(dirPath)
+    ? dirPath
+    : path.join(projectPath, dirPath)
+
+  const results: SummaryToolOutput[] = []
+
+  async function processDir(dir: string): Promise<void> {
+    const entries = await fs.readdir(dir, { withFileTypes: true })
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name)
+      const relativePath = path.relative(projectPath, fullPath)
+
+      if (entry.isDirectory()) {
+        // Skip common ignore patterns
+        if (
+          entry.name === 'node_modules' ||
+          entry.name === '.git' ||
+          entry.name.startsWith('.')
+        ) {
+          continue
+        }
+        if (options.recursive) {
+          await processDir(fullPath)
+        }
+      } else if (entry.isFile()) {
+        const ext = path.extname(entry.name).toLowerCase()
+        if (EXT_TO_LANG[ext]) {
+          const result = await summarizeFile(relativePath, projectPath)
+          results.push(result)
+        }
+      }
+    }
+  }
+
+  await processDir(absolutePath)
+  return results
+}
+
+// =============================================================================
+// Helper Functions
+// =============================================================================
+
+/**
+ * Extract file-level purpose from first docstring
+ */
+function extractFilePurpose(content: string, language: string): string {
+  const patterns = DOCSTRING_PATTERNS[language] || []
+  const lines = content.split('\n')
+
+  // Look for a file-level docstring in first 30 lines
+  for (let i = 0; i < Math.min(30, lines.length); i++) {
+    const line = lines[i].trim()
+
+    for (const pattern of patterns) {
+      if (pattern.start.test(line)) {
+        if (pattern.singleLine) {
+          // Single-line comment - grab consecutive lines
+          const commentLines: string[] = []
+          let j = i
+          while (j < lines.length && pattern.start.test(lines[j].trim())) {
+            commentLines.push(lines[j].trim().replace(pattern.start, '').trim())
+            j++
+          }
+          if (commentLines.length > 0) {
+            return commentLines.slice(0, 3).join(' ').trim()
+          }
+        } else if (pattern.end) {
+          // Multi-line comment - extract until end
+          let comment = ''
+          let j = i
+          while (j < lines.length) {
+            comment += lines[j] + '\n'
+            if (pattern.end.test(lines[j])) break
+            j++
+          }
+          // Extract first meaningful line
+          const meaningfulLines = comment
+            .replace(pattern.start, '')
+            .replace(pattern.end!, '')
+            .split('\n')
+            .map((l) => l.replace(/^\s*\*\s?/, '').trim())
+            .filter((l) => l.length > 0 && !l.startsWith('@'))
+          if (meaningfulLines.length > 0) {
+            return meaningfulLines.slice(0, 2).join(' ').trim()
+          }
+        }
+      }
+    }
+
+    // Stop if we hit code (not comments or empty lines)
+    if (line.length > 0 && !line.startsWith('//') && !line.startsWith('#') && !line.startsWith('/*') && !line.startsWith('*') && !line.startsWith("'") && !line.startsWith('"')) {
+      break
+    }
+  }
+
+  // Fallback: derive from filename
+  const fileName = content.split('\n')[0] || ''
+  return `Module: ${path.basename(fileName, path.extname(fileName))}`
+}
+
+/**
+ * Extract description from a docstring line
+ */
+function extractDescriptionFromDocstring(docstring: string): string {
+  // Remove comment markers and clean up
+  return docstring
+    .replace(/^\/\*\*\s*/, '')
+    .replace(/\*\/$/, '')
+    .replace(/^\/\/\/?\s*/, '')
+    .replace(/^#\s*/, '')
+    .replace(/^"""\s*/, '')
+    .replace(/"""\s*$/, '')
+    .trim()
+    .split('\n')[0] // First line only
+    .trim()
+}
+
+/**
+ * Build summary text for metrics calculation
+ */
+function buildSummaryText(
+  purpose: string,
+  publicAPI: PublicAPIEntry[],
+  dependencies: string[]
+): string {
+  const parts: string[] = []
+
+  parts.push(`Purpose: ${purpose}`)
+  parts.push('')
+
+  if (publicAPI.length > 0) {
+    parts.push('Public API:')
+    for (const entry of publicAPI) {
+      const desc = entry.description ? ` - ${entry.description}` : ''
+      parts.push(`  ${entry.type} ${entry.name}: ${entry.signature}${desc}`)
+    }
+    parts.push('')
+  }
+
+  if (dependencies.length > 0) {
+    parts.push(`Dependencies: ${dependencies.join(', ')}`)
+  }
+
+  return parts.join('\n')
+}
+
+// =============================================================================
+// Exports
+// =============================================================================
+
+export default { summarizeFile, summarizeDirectory }

package/core/context-tools/token-counter.ts

@@ -0,0 +1,279 @@
+/**
+ * Token Counter - REAL token measurement for context tools
+ *
+ * Provides accurate token estimation for measuring compression rates.
+ * Uses industry-standard approximation: ~4 characters per token.
+ *
+ * This is critical for the Value Dashboard to show REAL savings,
+ * not estimated ones.
+ *
+ * @module context-tools/token-counter
+ * @version 1.0.0
+ */
+
+import type { TokenMetrics } from './types'
+
+// =============================================================================
+// Constants
+// =============================================================================
+
+/**
+ * Average characters per token
+ *
+ * Based on empirical analysis of Claude/GPT tokenizers:
+ * - Code averages ~3.5-4.5 chars/token
+ * - English text averages ~4-5 chars/token
+ * - We use 4 as a conservative middle ground
+ */
+const CHARS_PER_TOKEN = 4
+
+/**
+ * Model pricing per 1000 tokens (January 2026)
+ * Sources:
+ * - Anthropic: https://docs.anthropic.com/en/docs/about-claude/models
+ * - OpenAI: https://openai.com/pricing
+ * - Google: https://ai.google.dev/pricing
+ */
+const MODEL_PRICING = {
+  // Anthropic Claude (2026)
+  'claude-opus-4.5':   { input: 0.005,   output: 0.025 },   // $5/$25 per M
+  'claude-sonnet-4.5': { input: 0.003,   output: 0.015 },   // $3/$15 per M
+  'claude-haiku-4.5':  { input: 0.001,   output: 0.005 },   // $1/$5 per M
+  'claude-opus-4':     { input: 0.015,   output: 0.075 },   // $15/$75 per M (legacy)
+  // OpenAI
+  'gpt-4o':            { input: 0.0025,  output: 0.01 },    // $2.50/$10 per M
+  'gpt-4-turbo':       { input: 0.01,    output: 0.03 },    // $10/$30 per M
+  'gpt-4o-mini':       { input: 0.00015, output: 0.0006 },  // $0.15/$0.60 per M
+  // Google
+  'gemini-1.5-pro':    { input: 0.00125, output: 0.005 },   // $1.25/$5 per M
+  'gemini-1.5-flash':  { input: 0.000075, output: 0.0003 }, // $0.075/$0.30 per M
+} as const
+
+type ModelName = keyof typeof MODEL_PRICING
+
+// Default model for cost calculations
+const DEFAULT_MODEL: ModelName = 'claude-sonnet-4.5'
+
+// =============================================================================
+// Core Functions
+// =============================================================================
+
+/**
+ * Count tokens in a text string
+ *
+ * Uses character-based estimation that's accurate enough for
+ * measuring compression rates without requiring actual tokenizer.
+ *
+ * @param text - The text to count tokens for
+ * @returns Estimated token count
+ */
+export function countTokens(text: string): number {
+  if (!text || text.length === 0) return 0
+  return Math.ceil(text.length / CHARS_PER_TOKEN)
+}
+
+/**
+ * Models to show in cost breakdown (most popular)
+ */
+const BREAKDOWN_MODELS: ModelName[] = [
+  'claude-sonnet-4.5',
+  'claude-opus-4.5',
+  'gpt-4o',
+  'gemini-1.5-pro',
+]
+
+/**
+ * Calculate cost breakdown for a model
+ * Output potential = estimated savings if response is proportionally shorter
+ */
+function calculateModelCost(tokensSaved: number, model: ModelName): {
+  inputSaved: number
+  outputPotential: number
+  total: number
+} {
+  const pricing = MODEL_PRICING[model]
+  const inputSaved = (tokensSaved / 1000) * pricing.input
+  // Conservative estimate: output savings ~30% of compression benefit
+  // (less context = more focused response, but not 1:1)
+  const outputPotential = (tokensSaved / 1000) * pricing.output * 0.3
+  return {
+    inputSaved,
+    outputPotential,
+    total: inputSaved + outputPotential,
+  }
+}
+
+/**
+ * Format cost as currency string
+ */
+function formatCostSaved(cost: number): string {
+  if (cost < 0.001) {
+    return '<$0.01'
+  }
+  if (cost < 0.01) {
+    return `$${cost.toFixed(3)}`
+  }
+  return `$${cost.toFixed(2)}`
+}
+
+/**
+ * Measure compression between original and filtered content
+ *
+ * @param original - Original content before filtering
+ * @param filtered - Filtered content after compression
+ * @returns Token metrics with compression rate and multi-model cost savings
+ */
+export function measureCompression(original: string, filtered: string): TokenMetrics {
+  const originalTokens = countTokens(original)
+  const filteredTokens = countTokens(filtered)
+  const tokensSaved = Math.max(0, originalTokens - filteredTokens)
+
+  const compression =
+    originalTokens > 0 ? (originalTokens - filteredTokens) / originalTokens : 0
+
+  // Calculate cost for default model
+  const defaultCost = calculateModelCost(tokensSaved, DEFAULT_MODEL)
+
+  // Calculate breakdown for popular models
+  const byModel = BREAKDOWN_MODELS.map(model => ({
+    model,
+    ...calculateModelCost(tokensSaved, model),
+  }))
+
+  return {
+    tokens: {
+      original: originalTokens,
+      filtered: filteredTokens,
+      saved: tokensSaved,
+    },
+    compression: Math.max(0, Math.min(1, compression)),
+    cost: {
+      saved: defaultCost.total,
+      formatted: formatCostSaved(defaultCost.total),
+      byModel,
+    },
+  }
+}
+
+/**
+ * Create metrics for a fallback (no compression) case
+ *
+ * @param content - The full content that couldn't be compressed
+ * @returns Token metrics with 0% compression
+ */
+export function noCompression(content: string): TokenMetrics {
+  const tokens = countTokens(content)
+
+  return {
+    tokens: { original: tokens, filtered: tokens, saved: 0 },
+    compression: 0,
+    cost: {
+      saved: 0,
+      formatted: '$0.00',
+      byModel: BREAKDOWN_MODELS.map(model => ({
+        model,
+        inputSaved: 0,
+        outputPotential: 0,
+        total: 0,
+      })),
+    },
+  }
+}
+
+/**
+ * Combine multiple token metrics into one
+ *
+ * Useful when processing multiple files and aggregating results.
+ *
+ * @param metrics - Array of token metrics to combine
+ * @returns Combined metrics
+ */
+export function combineMetrics(metrics: TokenMetrics[]): TokenMetrics {
+  if (metrics.length === 0) {
+    return noCompression('')
+  }
+
+  // Sum tokens
+  const totalOriginal = metrics.reduce((sum, m) => sum + m.tokens.original, 0)
+  const totalFiltered = metrics.reduce((sum, m) => sum + m.tokens.filtered, 0)
+  const totalSaved = metrics.reduce((sum, m) => sum + m.tokens.saved, 0)
+
+  // Calculate overall compression
+  const compression = totalOriginal > 0
+    ? (totalOriginal - totalFiltered) / totalOriginal
+    : 0
+
+  // Sum costs by model
+  const byModel = BREAKDOWN_MODELS.map(model => {
+    const modelMetrics = metrics.map(m =>
+      m.cost.byModel.find(b => b.model === model) || { inputSaved: 0, outputPotential: 0, total: 0 }
+    )
+    return {
+      model,
+      inputSaved: modelMetrics.reduce((sum, m) => sum + m.inputSaved, 0),
+      outputPotential: modelMetrics.reduce((sum, m) => sum + m.outputPotential, 0),
+      total: modelMetrics.reduce((sum, m) => sum + m.total, 0),
+    }
+  })
+
+  const totalCost = metrics.reduce((sum, m) => sum + m.cost.saved, 0)
+
+  return {
+    tokens: {
+      original: totalOriginal,
+      filtered: totalFiltered,
+      saved: totalSaved,
+    },
+    compression,
+    cost: {
+      saved: totalCost,
+      formatted: formatCostSaved(totalCost),
+      byModel,
+    },
+  }
+}
+
+/**
+ * Format token count for display
+ *
+ * @param tokens - Number of tokens
+ * @returns Human-readable string (e.g., "1.5K", "2.3M")
+ */
+export function formatTokenCount(tokens: number): string {
+  if (tokens >= 1_000_000) {
+    return `${(tokens / 1_000_000).toFixed(1)}M`
+  }
+  if (tokens >= 1_000) {
+    return `${(tokens / 1_000).toFixed(1)}K`
+  }
+  return tokens.toLocaleString()
+}
+
+/**
+ * Format compression rate for display
+ *
+ * @param rate - Compression rate (0-1)
+ * @returns Human-readable string (e.g., "89%")
+ */
+export function formatCompressionRate(rate: number): string {
+  return `${Math.round(rate * 100)}%`
+}
+
+// =============================================================================
+// Exports
+// =============================================================================
+
+export { formatCostSaved }
+
+export default {
+  countTokens,
+  measureCompression,
+  noCompression,
+  combineMetrics,
+  formatTokenCount,
+  formatCompressionRate,
+  formatCostSaved,
+  CHARS_PER_TOKEN,
+  MODEL_PRICING,
+  DEFAULT_MODEL,
+}