toonify-mcp 0.1.0

package/src/metrics/metrics-collector.ts

@@ -0,0 +1,125 @@
+/**
+ * MetricsCollector: Local-only metrics tracking
+ */
+
+import { promises as fs } from 'fs';
+import path from 'path';
+import os from 'os';
+import type { TokenStats } from '../optimizer/types.js';
+
+export interface OptimizationMetric {
+  timestamp: string;
+  toolName: string;
+  originalTokens: number;
+  optimizedTokens: number;
+  savings: number;
+  savingsPercentage: number;
+  wasOptimized: boolean;
+  format?: string;
+  reason?: string;
+}
+
+export class MetricsCollector {
+  private statsPath: string;
+
+  constructor() {
+    // Store in user's home directory, not in project
+    this.statsPath = path.join(
+      os.homedir(),
+      '.claude',
+      'token_stats.json'
+    );
+  }
+
+  /**
+   * Record an optimization attempt
+   */
+  async record(metric: OptimizationMetric): Promise<void> {
+    try {
+      const stats = await this.loadStats();
+
+      // Update aggregated stats
+      stats.totalRequests++;
+      if (metric.wasOptimized) {
+        stats.optimizedRequests++;
+      }
+
+      stats.tokensBeforeOptimization += metric.originalTokens;
+      stats.tokensAfterOptimization += metric.optimizedTokens;
+      stats.totalSavings += metric.savings;
+
+      // Recalculate average
+      stats.averageSavingsPercentage =
+        stats.optimizedRequests > 0
+          ? (stats.totalSavings / stats.tokensBeforeOptimization) * 100
+          : 0;
+
+      await this.saveStats(stats);
+    } catch (error) {
+      // Silent failure - metrics should never break functionality
+      console.error('Failed to record metrics:', error);
+    }
+  }
+
+  /**
+   * Get current statistics
+   */
+  async getStats(): Promise<TokenStats> {
+    return await this.loadStats();
+  }
+
+  /**
+   * Load stats from disk
+   */
+  private async loadStats(): Promise<TokenStats> {
+    try {
+      // Ensure directory exists
+      await fs.mkdir(path.dirname(this.statsPath), { recursive: true });
+
+      const data = await fs.readFile(this.statsPath, 'utf-8');
+      return JSON.parse(data);
+    } catch {
+      // Return empty stats if file doesn't exist
+      return {
+        totalRequests: 0,
+        optimizedRequests: 0,
+        tokensBeforeOptimization: 0,
+        tokensAfterOptimization: 0,
+        totalSavings: 0,
+        averageSavingsPercentage: 0,
+      };
+    }
+  }
+
+  /**
+   * Save stats to disk
+   */
+  private async saveStats(stats: TokenStats): Promise<void> {
+    await fs.writeFile(
+      this.statsPath,
+      JSON.stringify(stats, null, 2),
+      'utf-8'
+    );
+  }
+
+  /**
+   * Format stats as dashboard
+   */
+  async formatDashboard(): Promise<string> {
+    const stats = await this.getStats();
+
+    return `
+📊 Token Optimization Stats
+━━━━━━━━━━━━━━━━━━━━━━━━
+Total Requests: ${stats.totalRequests}
+Optimized: ${stats.optimizedRequests} (${((stats.optimizedRequests / stats.totalRequests) * 100).toFixed(1)}%)
+
+Tokens Before: ${stats.tokensBeforeOptimization.toLocaleString()}
+Tokens After: ${stats.tokensAfterOptimization.toLocaleString()}
+Total Savings: ${stats.totalSavings.toLocaleString()} (${stats.averageSavingsPercentage.toFixed(1)}%)
+
+💰 Cost Savings (at $3/1M input tokens):
+   $${((stats.totalSavings / 1_000_000) * 3).toFixed(2)} saved
+`.trim();
+  }
+}

package/src/optimizer/token-optimizer.ts

@@ -0,0 +1,214 @@
+/**
+ * TokenOptimizer: Core optimization logic using Toonify
+ */
+
+import { encode as toonEncode, decode as toonDecode } from '@toon-format/toon';
+import { encoding_for_model } from 'tiktoken';
+import yaml from 'yaml';
+import type {
+  OptimizationResult,
+  ToolMetadata,
+  StructuredData,
+  OptimizationConfig
+} from './types.js';
+
+export class TokenOptimizer {
+  private config: OptimizationConfig;
+  private tokenEncoder;
+
+  constructor(config: Partial<OptimizationConfig> = {}) {
+    this.config = {
+      enabled: true,
+      minTokensThreshold: 50,
+      minSavingsThreshold: 30,
+      maxProcessingTime: 50,
+      skipToolPatterns: [],
+      ...config
+    };
+
+    // Use Claude tokenizer
+    this.tokenEncoder = encoding_for_model('gpt-4');
+  }
+
+  /**
+   * Main optimization method
+   */
+  async optimize(
+    content: string,
+    metadata?: ToolMetadata
+  ): Promise<OptimizationResult> {
+    const startTime = Date.now();
+
+    // Quick path: skip if disabled or content too small
+    if (!this.config.enabled || content.length < 200) {
+      return {
+        optimized: false,
+        originalContent: content,
+        originalTokens: this.countTokens(content),
+        reason: 'Content too small'
+      };
+    }
+
+    // Skip if tool matches skip patterns
+    if (metadata?.toolName && this.shouldSkipTool(metadata.toolName)) {
+      return {
+        optimized: false,
+        originalContent: content,
+        originalTokens: this.countTokens(content),
+        reason: `Tool ${metadata.toolName} in skip list`
+      };
+    }
+
+    // Detect structured data
+    const structuredData = this.detectStructuredData(content);
+    if (!structuredData) {
+      return {
+        optimized: false,
+        originalContent: content,
+        originalTokens: this.countTokens(content),
+        reason: 'Not structured data'
+      };
+    }
+
+    try {
+      // Convert to TOON format
+      const toonContent = toonEncode(structuredData.data);
+
+      // Count tokens
+      const originalTokens = this.countTokens(content);
+      const optimizedTokens = this.countTokens(toonContent);
+
+      // Calculate savings
+      const tokenSavings = originalTokens - optimizedTokens;
+      const savingsPercentage = (tokenSavings / originalTokens) * 100;
+
+      // Check if worth using
+      if (savingsPercentage < this.config.minSavingsThreshold) {
+        return {
+          optimized: false,
+          originalContent: content,
+          originalTokens,
+          reason: `Savings too low: ${savingsPercentage.toFixed(1)}%`
+        };
+      }
+
+      // Check processing time
+      const elapsed = Date.now() - startTime;
+      if (elapsed > this.config.maxProcessingTime) {
+        return {
+          optimized: false,
+          originalContent: content,
+          originalTokens,
+          reason: `Processing timeout: ${elapsed}ms`
+        };
+      }
+
+      return {
+        optimized: true,
+        originalContent: content,
+        optimizedContent: toonContent,
+        originalTokens,
+        optimizedTokens,
+        savings: {
+          tokens: tokenSavings,
+          percentage: savingsPercentage
+        },
+        format: structuredData.type
+      };
+
+    } catch (error) {
+      // Silent fallback on error
+      return {
+        optimized: false,
+        originalContent: content,
+        originalTokens: this.countTokens(content),
+        reason: `Error: ${error instanceof Error ? error.message : 'Unknown'}`
+      };
+    }
+  }
+
+  /**
+   * Detect if content is structured data (JSON/CSV/YAML)
+   */
+  private detectStructuredData(content: string): StructuredData | null {
+    // Try JSON first
+    try {
+      const data = JSON.parse(content);
+      if (typeof data === 'object' && data !== null) {
+        return { type: 'json', data, confidence: 1.0 };
+      }
+    } catch {}
+
+    // Try YAML
+    try {
+      const data = yaml.parse(content);
+      if (typeof data === 'object' && data !== null) {
+        return { type: 'yaml', data, confidence: 0.9 };
+      }
+    } catch {}
+
+    // Try CSV (simple heuristic)
+    if (this.looksLikeCSV(content)) {
+      try {
+        const data = this.parseSimpleCSV(content);
+        return { type: 'csv', data, confidence: 0.8 };
+      } catch {}
+    }
+
+    return null;
+  }
+
+  /**
+   * Simple CSV detection heuristic
+   */
+  private looksLikeCSV(content: string): boolean {
+    const lines = content.split('\n').filter(l => l.trim());
+    if (lines.length < 2) return false;
+
+    const firstLineCommas = (lines[0].match(/,/g) || []).length;
+    if (firstLineCommas === 0) return false;
+
+    // Check if most lines have similar comma count
+    let matchingLines = 0;
+    for (let i = 1; i < Math.min(lines.length, 10); i++) {
+      const commas = (lines[i].match(/,/g) || []).length;
+      if (commas === firstLineCommas) matchingLines++;
+    }
+
+    return matchingLines >= Math.min(lines.length - 1, 7);
+  }
+
+  /**
+   * Parse simple CSV to array of objects
+   */
+  private parseSimpleCSV(content: string): any[] {
+    const lines = content.split('\n').filter(l => l.trim());
+    const headers = lines[0].split(',').map(h => h.trim());
+
+    return lines.slice(1).map(line => {
+      const values = line.split(',').map(v => v.trim());
+      const obj: any = {};
+      headers.forEach((header, i) => {
+        obj[header] = values[i] || '';
+      });
+      return obj;
+    });
+  }
+
+  /**
+   * Count tokens in text
+   */
+  private countTokens(text: string): number {
+    return this.tokenEncoder.encode(text).length;
+  }
+
+  /**
+   * Check if tool should be skipped
+   */
+  private shouldSkipTool(toolName: string): boolean {
+    return this.config.skipToolPatterns?.some(pattern => {
+      const regex = new RegExp(pattern);
+      return regex.test(toolName);
+    }) ?? false;
+  }
+}

package/src/optimizer/types.ts

@@ -0,0 +1,46 @@
+/**
+ * Type definitions for token optimization
+ */
+
+export interface OptimizationResult {
+  optimized: boolean;
+  originalContent: string;
+  optimizedContent?: string;
+  originalTokens: number;
+  optimizedTokens?: number;
+  savings?: {
+    tokens: number;
+    percentage: number;
+  };
+  format?: 'json' | 'csv' | 'yaml' | 'unknown';
+  reason?: string; // Why optimization was skipped
+}
+
+export interface ToolMetadata {
+  toolName: string;
+  contentType?: string;
+  size: number;
+}
+
+export interface StructuredData {
+  type: 'json' | 'csv' | 'yaml';
+  data: any;
+  confidence: number;
+}
+
+export interface OptimizationConfig {
+  enabled: boolean;
+  minTokensThreshold: number; // Only optimize if content > N tokens
+  minSavingsThreshold: number; // Only use if savings > N%
+  maxProcessingTime: number; // Max ms to spend optimizing
+  skipToolPatterns?: string[]; // Tool names to skip
+}
+
+export interface TokenStats {
+  totalRequests: number;
+  optimizedRequests: number;
+  tokensBeforeOptimization: number;
+  tokensAfterOptimization: number;
+  totalSavings: number;
+  averageSavingsPercentage: number;
+}

package/src/server/mcp-server.ts

@@ -0,0 +1,134 @@
+/**
+ * ToonifyMCPServer: MCP server that wraps tool results with token optimization
+ */
+
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import {
+  CallToolRequestSchema,
+  ListToolsRequestSchema,
+} from '@modelcontextprotocol/sdk/types.js';
+import { TokenOptimizer } from '../optimizer/token-optimizer.js';
+import { MetricsCollector } from '../metrics/metrics-collector.js';
+
+export class ToonifyMCPServer {
+  private server: Server;
+  private optimizer: TokenOptimizer;
+  private metrics: MetricsCollector;
+
+  constructor() {
+    this.server = new Server(
+      {
+        name: 'claude-code-toonify',
+        version: '0.1.0',
+      },
+      {
+        capabilities: {
+          tools: {},
+        },
+      }
+    );
+
+    this.optimizer = new TokenOptimizer();
+    this.metrics = new MetricsCollector();
+
+    this.setupHandlers();
+  }
+
+  private setupHandlers() {
+    // List available tools
+    this.server.setRequestHandler(ListToolsRequestSchema, async () => ({
+      tools: [
+        {
+          name: 'optimize_content',
+          description: 'Optimize structured data content for token efficiency using TOON format',
+          inputSchema: {
+            type: 'object',
+            properties: {
+              content: {
+                type: 'string',
+                description: 'The content to optimize (JSON, CSV, or YAML)',
+              },
+              toolName: {
+                type: 'string',
+                description: 'Name of the tool that generated this content',
+              },
+            },
+            required: ['content'],
+          },
+        },
+        {
+          name: 'get_stats',
+          description: 'Get token optimization statistics',
+          inputSchema: {
+            type: 'object',
+            properties: {},
+          },
+        },
+      ],
+    }));
+
+    // Handle tool calls
+    this.server.setRequestHandler(CallToolRequestSchema, async (request) => {
+      const { name, arguments: args } = request.params;
+
+      switch (name) {
+        case 'optimize_content': {
+          const { content, toolName } = args as {
+            content: string;
+            toolName?: string;
+          };
+
+          const result = await this.optimizer.optimize(content, {
+            toolName: toolName || 'unknown',
+            size: content.length,
+          });
+
+          // Record metrics
+          await this.metrics.record({
+            timestamp: new Date().toISOString(),
+            toolName: toolName || 'unknown',
+            originalTokens: result.originalTokens,
+            optimizedTokens: result.optimizedTokens || result.originalTokens,
+            savings: result.savings?.tokens || 0,
+            savingsPercentage: result.savings?.percentage || 0,
+            wasOptimized: result.optimized,
+            format: result.format,
+            reason: result.reason,
+          });
+
+          return {
+            content: [
+              {
+                type: 'text',
+                text: JSON.stringify(result, null, 2),
+              },
+            ],
+          };
+        }
+
+        case 'get_stats': {
+          const stats = await this.metrics.getStats();
+          return {
+            content: [
+              {
+                type: 'text',
+                text: JSON.stringify(stats, null, 2),
+              },
+            ],
+          };
+        }
+
+        default:
+          throw new Error(`Unknown tool: ${name}`);
+      }
+    });
+  }
+
+  async start() {
+    const transport = new StdioServerTransport();
+    await this.server.connect(transport);
+
+    console.error('Toonify MCP Server running on stdio');
+  }
+}

package/test-mcp.json

@@ -0,0 +1,5 @@
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "tools/list"
+}

package/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "node",
+    "lib": ["ES2022"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "**/*.test.ts"]
+}