prjct-cli 1.21.0 → 1.23.0

package/core/context-tools/summary-tool.ts

@@ -1,301 +0,0 @@
-/**
- * Summary Tool - Intelligent file summarization
- *
- * Combines:
- * - Code signatures (public API)
- * - JSDoc/docstring extraction
- * - Key dependencies
- *
- * Achieves high compression by returning only public-facing elements.
- *
- * @module context-tools/summary-tool
- * @version 1.0.0
- */
-
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import { isNotFoundError } from '../types/fs'
-import { analyzeImports } from './imports-tool'
-import { extractSignatures } from './signatures-tool'
-import { measureCompression, noCompression } from './token-counter'
-import type { PublicAPIEntry, SummaryToolOutput } from './types'
-
-// =============================================================================
-// Docstring Patterns
-// =============================================================================
-
-interface DocstringPattern {
-  start: RegExp
-  end: RegExp | null
-  singleLine?: boolean
-}
-
-/**
- * Docstring patterns by language
- */
-const DOCSTRING_PATTERNS: Record<string, DocstringPattern[]> = {
-  typescript: [
-    { start: /\/\*\*/, end: /\*\// }, // JSDoc
-    { start: /\/\/\//, end: null, singleLine: true }, // Triple-slash
-  ],
-  javascript: [
-    { start: /\/\*\*/, end: /\*\// }, // JSDoc
-  ],
-  python: [
-    { start: /"""/, end: /"""/ }, // Triple quotes
-    { start: /'''/, end: /'''/ }, // Single quotes
-  ],
-  go: [
-    { start: /\/\//, end: null, singleLine: true }, // Line comment (Go uses these as docs)
-  ],
-  rust: [
-    { start: /\/\/\//, end: null, singleLine: true }, // Doc comment
-    { start: /\/\/!/, end: null, singleLine: true }, // Inner doc comment
-  ],
-}
-
-/**
- * Extension to language mapping
- */
-const EXT_TO_LANG: Record<string, string> = {
-  '.ts': 'typescript',
-  '.tsx': 'typescript',
-  '.js': 'javascript',
-  '.jsx': 'javascript',
-  '.mjs': 'javascript',
-  '.py': 'python',
-  '.go': 'go',
-  '.rs': 'rust',
-}
-
-// =============================================================================
-// Main Functions
-// =============================================================================
-
-/**
- * Generate an intelligent summary of a file
- *
- * @param filePath - Path to the file
- * @param projectPath - Project root path
- * @returns Summary with public API and metrics
- */
-export async function summarizeFile(
-  filePath: string,
-  projectPath: string = process.cwd()
-): Promise<SummaryToolOutput> {
-  const absolutePath = path.isAbsolute(filePath) ? filePath : path.join(projectPath, filePath)
-
-  // Read file content
-  let content: string
-  try {
-    content = await fs.readFile(absolutePath, 'utf-8')
-  } catch (error) {
-    if (isNotFoundError(error)) {
-      return {
-        file: filePath,
-        purpose: 'File not found',
-        publicAPI: [],
-        dependencies: [],
-        metrics: noCompression(''),
-      }
-    }
-    throw error
-  }
-
-  // Get language
-  const ext = path.extname(filePath).toLowerCase()
-  const language = EXT_TO_LANG[ext] || 'unknown'
-
-  // Extract signatures
-  const signaturesResult = await extractSignatures(filePath, projectPath)
-
-  // Extract imports for dependencies
-  const importsResult = await analyzeImports(filePath, projectPath)
-
-  // Extract file-level docstring (purpose)
-  const purpose = extractFilePurpose(content, language)
-
-  // Build public API from exported signatures
-  const publicAPI: PublicAPIEntry[] = signaturesResult.signatures
-    .filter((sig) => sig.exported)
-    .map((sig) => ({
-      name: sig.name,
-      type: sig.type,
-      signature: sig.signature,
-      description: sig.docstring ? extractDescriptionFromDocstring(sig.docstring) : undefined,
-    }))
-
-  // Get key dependencies (internal only, external are obvious from package.json)
-  const dependencies = importsResult.imports
-    .filter((imp) => !imp.isExternal && imp.resolved)
-    .map((imp) => imp.resolved!)
-    .slice(0, 10) // Limit to 10
-
-  // Build summary content for metrics
-  const summaryContent = buildSummaryText(purpose, publicAPI, dependencies)
-
-  return {
-    file: filePath,
-    purpose,
-    publicAPI,
-    dependencies,
-    metrics: measureCompression(content, summaryContent),
-  }
-}
-
-/**
- * Summarize all files in a directory
- */
-export async function summarizeDirectory(
-  dirPath: string,
-  projectPath: string = process.cwd(),
-  options: { recursive?: boolean } = {}
-): Promise<SummaryToolOutput[]> {
-  const absolutePath = path.isAbsolute(dirPath) ? dirPath : path.join(projectPath, dirPath)
-
-  const results: SummaryToolOutput[] = []
-
-  async function processDir(dir: string): Promise<void> {
-    const entries = await fs.readdir(dir, { withFileTypes: true })
-
-    for (const entry of entries) {
-      const fullPath = path.join(dir, entry.name)
-      const relativePath = path.relative(projectPath, fullPath)
-
-      if (entry.isDirectory()) {
-        // Skip common ignore patterns
-        if (entry.name === 'node_modules' || entry.name === '.git' || entry.name.startsWith('.')) {
-          continue
-        }
-        if (options.recursive) {
-          await processDir(fullPath)
-        }
-      } else if (entry.isFile()) {
-        const ext = path.extname(entry.name).toLowerCase()
-        if (EXT_TO_LANG[ext]) {
-          const result = await summarizeFile(relativePath, projectPath)
-          results.push(result)
-        }
-      }
-    }
-  }
-
-  await processDir(absolutePath)
-  return results
-}
-
-// =============================================================================
-// Helper Functions
-// =============================================================================
-
-/**
- * Extract file-level purpose from first docstring
- */
-function extractFilePurpose(content: string, language: string): string {
-  const patterns = DOCSTRING_PATTERNS[language] || []
-  const lines = content.split('\n')
-
-  // Look for a file-level docstring in first 30 lines
-  for (let i = 0; i < Math.min(30, lines.length); i++) {
-    const line = lines[i].trim()
-
-    for (const pattern of patterns) {
-      if (pattern.start.test(line)) {
-        if (pattern.singleLine) {
-          // Single-line comment - grab consecutive lines
-          const commentLines: string[] = []
-          let j = i
-          while (j < lines.length && pattern.start.test(lines[j].trim())) {
-            commentLines.push(lines[j].trim().replace(pattern.start, '').trim())
-            j++
-          }
-          if (commentLines.length > 0) {
-            return commentLines.slice(0, 3).join(' ').trim()
-          }
-        } else if (pattern.end) {
-          // Multi-line comment - extract until end
-          let comment = ''
-          let j = i
-          while (j < lines.length) {
-            comment += `${lines[j]}\n`
-            if (pattern.end.test(lines[j])) break
-            j++
-          }
-          // Extract first meaningful line
-          const meaningfulLines = comment
-            .replace(pattern.start, '')
-            .replace(pattern.end!, '')
-            .split('\n')
-            .map((l) => l.replace(/^\s*\*\s?/, '').trim())
-            .filter((l) => l.length > 0 && !l.startsWith('@'))
-          if (meaningfulLines.length > 0) {
-            return meaningfulLines.slice(0, 2).join(' ').trim()
-          }
-        }
-      }
-    }
-
-    // Stop if we hit code (not comments or empty lines)
-    if (
-      line.length > 0 &&
-      !line.startsWith('//') &&
-      !line.startsWith('#') &&
-      !line.startsWith('/*') &&
-      !line.startsWith('*') &&
-      !line.startsWith("'") &&
-      !line.startsWith('"')
-    ) {
-      break
-    }
-  }
-
-  // Fallback: derive from filename
-  const fileName = content.split('\n')[0] || ''
-  return `Module: ${path.basename(fileName, path.extname(fileName))}`
-}
-
-/**
- * Extract description from a docstring line
- */
-function extractDescriptionFromDocstring(docstring: string): string {
-  // Remove comment markers and clean up
-  return docstring
-    .replace(/^\/\*\*\s*/, '')
-    .replace(/\*\/$/, '')
-    .replace(/^\/\/\/?\s*/, '')
-    .replace(/^#\s*/, '')
-    .replace(/^"""\s*/, '')
-    .replace(/"""\s*$/, '')
-    .trim()
-    .split('\n')[0] // First line only
-    .trim()
-}
-
-/**
- * Build summary text for metrics calculation
- */
-function buildSummaryText(
-  purpose: string,
-  publicAPI: PublicAPIEntry[],
-  dependencies: string[]
-): string {
-  const parts: string[] = []
-
-  parts.push(`Purpose: ${purpose}`)
-  parts.push('')
-
-  if (publicAPI.length > 0) {
-    parts.push('Public API:')
-    for (const entry of publicAPI) {
-      const desc = entry.description ? ` - ${entry.description}` : ''
-      parts.push(`  ${entry.type} ${entry.name}: ${entry.signature}${desc}`)
-    }
-    parts.push('')
-  }
-
-  if (dependencies.length > 0) {
-    parts.push(`Dependencies: ${dependencies.join(', ')}`)
-  }
-
-  return parts.join('\n')
-}

package/core/context-tools/token-counter.ts

@@ -1,273 +0,0 @@
-/**
- * Token Counter - REAL token measurement for context tools
- *
- * Provides accurate token estimation for measuring compression rates.
- * Uses industry-standard approximation: ~4 characters per token.
- *
- * This is critical for the Value Dashboard to show REAL savings,
- * not estimated ones.
- *
- * @module context-tools/token-counter
- * @version 1.0.0
- */
-
-import type { TokenMetrics } from './types'
-
-// =============================================================================
-// Constants
-// =============================================================================
-
-/**
- * Average characters per token
- *
- * Based on empirical analysis of Claude/GPT tokenizers:
- * - Code averages ~3.5-4.5 chars/token
- * - English text averages ~4-5 chars/token
- * - We use 4 as a conservative middle ground
- */
-const CHARS_PER_TOKEN = 4
-
-/**
- * Model pricing per 1000 tokens (January 2026)
- * Sources:
- * - Anthropic: https://docs.anthropic.com/en/docs/about-claude/models
- * - OpenAI: https://openai.com/pricing
- * - Google: https://ai.google.dev/pricing
- */
-const MODEL_PRICING = {
-  // Anthropic Claude (2026)
-  'claude-opus-4.5': { input: 0.005, output: 0.025 }, // $5/$25 per M
-  'claude-sonnet-4.5': { input: 0.003, output: 0.015 }, // $3/$15 per M
-  'claude-haiku-4.5': { input: 0.001, output: 0.005 }, // $1/$5 per M
-  'claude-opus-4': { input: 0.015, output: 0.075 }, // $15/$75 per M (legacy)
-  'claude-opus-4-6': { input: 0.015, output: 0.075 }, // $15/$75 per M
-  // OpenAI
-  'gpt-4o': { input: 0.0025, output: 0.01 }, // $2.50/$10 per M
-  'gpt-4-turbo': { input: 0.01, output: 0.03 }, // $10/$30 per M
-  'gpt-4o-mini': { input: 0.00015, output: 0.0006 }, // $0.15/$0.60 per M
-  // Google
-  'gemini-1.5-pro': { input: 0.00125, output: 0.005 }, // $1.25/$5 per M
-  'gemini-1.5-flash': { input: 0.000075, output: 0.0003 }, // $0.075/$0.30 per M
-} as const
-
-type ModelName = keyof typeof MODEL_PRICING
-
-// Default model for cost calculations
-const DEFAULT_MODEL: ModelName = 'claude-sonnet-4.5'
-
-// =============================================================================
-// Core Functions
-// =============================================================================
-
-/**
- * Count tokens in a text string
- *
- * Uses character-based estimation that's accurate enough for
- * measuring compression rates without requiring actual tokenizer.
- *
- * @param text - The text to count tokens for
- * @returns Estimated token count
- */
-export function countTokens(text: string): number {
-  if (!text || text.length === 0) return 0
-  return Math.ceil(text.length / CHARS_PER_TOKEN)
-}
-
-/**
- * Models to show in cost breakdown (most popular)
- */
-const BREAKDOWN_MODELS: ModelName[] = [
-  'claude-sonnet-4.5',
-  'claude-opus-4.5',
-  'claude-opus-4-6',
-  'gpt-4o',
-  'gemini-1.5-pro',
-]
-
-/**
- * Calculate cost breakdown for a model
- * Output potential = estimated savings if response is proportionally shorter
- */
-function calculateModelCost(
-  tokensSaved: number,
-  model: ModelName
-): {
-  inputSaved: number
-  outputPotential: number
-  total: number
-} {
-  const pricing = MODEL_PRICING[model]
-  const inputSaved = (tokensSaved / 1000) * pricing.input
-  // Conservative estimate: output savings ~30% of compression benefit
-  // (less context = more focused response, but not 1:1)
-  const outputPotential = (tokensSaved / 1000) * pricing.output * 0.3
-  return {
-    inputSaved,
-    outputPotential,
-    total: inputSaved + outputPotential,
-  }
-}
-
-/**
- * Format cost as currency string
- */
-function formatCostSaved(cost: number): string {
-  if (cost < 0.001) {
-    return '<$0.01'
-  }
-  if (cost < 0.01) {
-    return `$${cost.toFixed(3)}`
-  }
-  return `$${cost.toFixed(2)}`
-}
-
-/**
- * Measure compression between original and filtered content
- *
- * @param original - Original content before filtering
- * @param filtered - Filtered content after compression
- * @returns Token metrics with compression rate and multi-model cost savings
- */
-export function measureCompression(original: string, filtered: string): TokenMetrics {
-  const originalTokens = countTokens(original)
-  const filteredTokens = countTokens(filtered)
-  const tokensSaved = Math.max(0, originalTokens - filteredTokens)
-
-  const compression = originalTokens > 0 ? (originalTokens - filteredTokens) / originalTokens : 0
-
-  // Calculate cost for default model
-  const defaultCost = calculateModelCost(tokensSaved, DEFAULT_MODEL)
-
-  // Calculate breakdown for popular models
-  const byModel = BREAKDOWN_MODELS.map((model) => ({
-    model,
-    ...calculateModelCost(tokensSaved, model),
-  }))
-
-  return {
-    tokens: {
-      original: originalTokens,
-      filtered: filteredTokens,
-      saved: tokensSaved,
-    },
-    compression: Math.max(0, Math.min(1, compression)),
-    cost: {
-      saved: defaultCost.total,
-      formatted: formatCostSaved(defaultCost.total),
-      byModel,
-    },
-  }
-}
-
-/**
- * Create metrics for a fallback (no compression) case
- *
- * @param content - The full content that couldn't be compressed
- * @returns Token metrics with 0% compression
- */
-export function noCompression(content: string): TokenMetrics {
-  const tokens = countTokens(content)
-
-  return {
-    tokens: { original: tokens, filtered: tokens, saved: 0 },
-    compression: 0,
-    cost: {
-      saved: 0,
-      formatted: '$0.00',
-      byModel: BREAKDOWN_MODELS.map((model) => ({
-        model,
-        inputSaved: 0,
-        outputPotential: 0,
-        total: 0,
-      })),
-    },
-  }
-}
-
-/**
- * Combine multiple token metrics into one
- *
- * Useful when processing multiple files and aggregating results.
- *
- * @param metrics - Array of token metrics to combine
- * @returns Combined metrics
- */
-export function combineMetrics(metrics: TokenMetrics[]): TokenMetrics {
-  if (metrics.length === 0) {
-    return noCompression('')
-  }
-
-  // Sum tokens
-  const totalOriginal = metrics.reduce((sum, m) => sum + m.tokens.original, 0)
-  const totalFiltered = metrics.reduce((sum, m) => sum + m.tokens.filtered, 0)
-  const totalSaved = metrics.reduce((sum, m) => sum + m.tokens.saved, 0)
-
-  // Calculate overall compression
-  const compression = totalOriginal > 0 ? (totalOriginal - totalFiltered) / totalOriginal : 0
-
-  // Sum costs by model
-  const byModel = BREAKDOWN_MODELS.map((model) => {
-    const modelMetrics = metrics.map(
-      (m) =>
-        m.cost.byModel.find((b) => b.model === model) || {
-          inputSaved: 0,
-          outputPotential: 0,
-          total: 0,
-        }
-    )
-    return {
-      model,
-      inputSaved: modelMetrics.reduce((sum, m) => sum + m.inputSaved, 0),
-      outputPotential: modelMetrics.reduce((sum, m) => sum + m.outputPotential, 0),
-      total: modelMetrics.reduce((sum, m) => sum + m.total, 0),
-    }
-  })
-
-  const totalCost = metrics.reduce((sum, m) => sum + m.cost.saved, 0)
-
-  return {
-    tokens: {
-      original: totalOriginal,
-      filtered: totalFiltered,
-      saved: totalSaved,
-    },
-    compression,
-    cost: {
-      saved: totalCost,
-      formatted: formatCostSaved(totalCost),
-      byModel,
-    },
-  }
-}
-
-/**
- * Format token count for display
- *
- * @param tokens - Number of tokens
- * @returns Human-readable string (e.g., "1.5K", "2.3M")
- */
-export function formatTokenCount(tokens: number): string {
-  if (tokens >= 1_000_000) {
-    return `${(tokens / 1_000_000).toFixed(1)}M`
-  }
-  if (tokens >= 1_000) {
-    return `${(tokens / 1_000).toFixed(1)}K`
-  }
-  return tokens.toLocaleString()
-}
-
-/**
- * Format compression rate for display
- *
- * @param rate - Compression rate (0-1)
- * @returns Human-readable string (e.g., "89%")
- */
-export function formatCompressionRate(rate: number): string {
-  return `${Math.round(rate * 100)}%`
-}
-
-// =============================================================================
-// Exports
-// =============================================================================
-
-export { formatCostSaved }