salmon-loop 0.2.3

package/dist/core/context/token/encoding-registry.js

@@ -0,0 +1,173 @@
+/**
+ * Encoding registry using Strategy pattern.
+ *
+ * Allows registration of new encodings without modifying existing code.
+ * Follows Open/Closed Principle.
+ */
+import { get_encoding } from 'tiktoken';
+/**
+ * Base encoding implementation using tiktoken.
+ */
+class TiktokenEncoding {
+    name;
+    models;
+    encoder = null;
+    initialized = false;
+    constructor(name, models) {
+        this.name = name;
+        this.models = models;
+    }
+    async initialize() {
+        if (this.initialized)
+            return;
+        this.encoder = get_encoding(this.getTiktokenName());
+        this.initialized = true;
+    }
+    encode(text) {
+        this.ensureInitialized();
+        return Array.from(this.encoder.encode(text));
+    }
+    decode(tokens) {
+        this.ensureInitialized();
+        const uint32Tokens = new Uint32Array(tokens);
+        const decoded = this.encoder.decode(uint32Tokens);
+        return new TextDecoder().decode(decoded);
+    }
+    count(text) {
+        return this.encode(text).length;
+    }
+    dispose() {
+        if (this.encoder) {
+            this.encoder.free();
+            this.encoder = null;
+        }
+        this.initialized = false;
+    }
+    ensureInitialized() {
+        if (!this.initialized || !this.encoder) {
+            throw new Error(`Encoding ${this.name} not initialized. Call initialize() first.`);
+        }
+    }
+}
+/**
+ * cl100k_base encoding (GPT-4, GPT-3.5-turbo, Claude).
+ */
+class Cl100kEncoding extends TiktokenEncoding {
+    constructor() {
+        super('cl100k_base', ['openai-gpt4', 'openai-gpt35', 'anthropic-claude']);
+    }
+    getTiktokenName() {
+        return 'cl100k_base';
+    }
+}
+/**
+ * o200k_base encoding (GPT-4o, GPT-4o-mini).
+ */
+class O200kEncoding extends TiktokenEncoding {
+    constructor() {
+        super('o200k_base', ['openai-gpt4o']);
+    }
+    getTiktokenName() {
+        return 'o200k_base';
+    }
+}
+/**
+ * Registry for encoding strategies.
+ *
+ * Features:
+ * - Register new encodings at runtime
+ * - Auto-select encoding by model family
+ * - Default encoding fallback
+ */
+export class EncodingRegistry {
+    encodings = new Map();
+    modelToEncoding = new Map();
+    initialized = false;
+    constructor() {
+        // Register default encodings
+        this.register(new Cl100kEncoding());
+        this.register(new O200kEncoding());
+    }
+    /**
+     * Register a new encoding.
+     */
+    register(encoding) {
+        this.encodings.set(encoding.name, encoding);
+        for (const model of encoding.models) {
+            this.modelToEncoding.set(model, encoding.name);
+        }
+    }
+    /**
+     * Get encoding by type.
+     */
+    get(encoding) {
+        const enc = this.encodings.get(encoding);
+        if (!enc) {
+            throw new Error(`Unknown encoding: ${encoding}`);
+        }
+        return enc;
+    }
+    /**
+     * Get encoding for a model family.
+     * Returns default encoding if model not found.
+     */
+    getByModel(model) {
+        const encodingName = this.modelToEncoding.get(model);
+        if (!encodingName) {
+            // Default fallback
+            return this.get('cl100k_base');
+        }
+        return this.get(encodingName);
+    }
+    /**
+     * Get encoding type for a model family.
+     */
+    getEncodingTypeForModel(model) {
+        return this.modelToEncoding.get(model) ?? 'cl100k_base';
+    }
+    /**
+     * Check if encoding is registered.
+     */
+    has(encoding) {
+        return this.encodings.has(encoding);
+    }
+    /**
+     * List all registered encodings.
+     */
+    listEncodings() {
+        return Array.from(this.encodings.keys());
+    }
+    /**
+     * List all registered models.
+     */
+    listModels() {
+        return Array.from(this.modelToEncoding.keys());
+    }
+    /**
+     * Initialize all encodings.
+     * Call once at application startup.
+     */
+    async initializeAll() {
+        if (this.initialized)
+            return;
+        await Promise.all(Array.from(this.encodings.values()).map((e) => e.initialize()));
+        this.initialized = true;
+    }
+    /**
+     * Dispose all encodings.
+     * Call at application shutdown.
+     */
+    disposeAll() {
+        for (const encoding of this.encodings.values()) {
+            encoding.dispose();
+        }
+        this.initialized = false;
+    }
+    /**
+     * Check if initialized.
+     */
+    isInitialized() {
+        return this.initialized;
+    }
+}
+//# sourceMappingURL=encoding-registry.js.map

package/dist/core/context/token/index.js

@@ -0,0 +1,31 @@
+/**
+ * Token counting module.
+ *
+ * Provides accurate token counting with caching for performance optimization.
+ * Includes model-adaptive budget calculation.
+ *
+ * @example
+ * ```typescript
+ * import { TokenCounter, TokenBudgetCalculator } from './token/index.js';
+ *
+ * const counter = new TokenCounter();
+ * await counter.initialize();
+ *
+ * // Simple counting
+ * const tokens = counter.count('Hello, world!');
+ *
+ * // Count for a model
+ * const tokens = counter.countByModel('Hello', 'openai-gpt4');
+ *
+ * // Model-adaptive budget
+ * const budget = new TokenBudgetCalculator();
+ * budget.setModel('gpt-4o');
+ * const defaultBudget = budget.getDefaultBudget();
+ * ```
+ */
+export { TokenCounter } from './counter.js';
+export { TokenCache } from './cache.js';
+export { EncodingRegistry } from './encoding-registry.js';
+export { TokenBudgetCalculator, DEFAULT_TOKEN_BUDGET_CONFIG } from './token-budget.js';
+export { AdaptiveBudgetCalculator, MODEL_CONTEXT_CONFIG, getAdaptiveBudgetCalculator, resetAdaptiveBudgetCalculator, getModelRecommendedBudget, getModelMinBudget, getModelMaxPrimaryTokens, } from './adaptive-budget.js';
+//# sourceMappingURL=index.js.map

package/dist/core/context/token/token-budget.js

@@ -0,0 +1,213 @@
+/**
+ * Token budget calculator for context management.
+ *
+ * Provides token-based budget calculation as a replacement for character-based estimation.
+ * Designed for gradual migration from character-based to token-based budgeting.
+ *
+ * Supports model-adaptive budget via AdaptiveBudgetCalculator integration.
+ */
+import { LIMITS } from '../../config/limits.js';
+import { getAdaptiveBudgetCalculator, } from './adaptive-budget.js';
+import { TokenCounter } from './index.js';
+/**
+ * Default token budget configuration.
+ */
+export const DEFAULT_TOKEN_BUDGET_CONFIG = {
+    mode: 'token',
+    defaultTokenBudget: LIMITS.maxContextTokens,
+    minTokenBudget: LIMITS.minContextTokens,
+};
+/**
+ * Token budget calculator.
+ *
+ * Provides methods for calculating token counts of context components.
+ * Supports model-adaptive budget via setModel().
+ */
+export class TokenBudgetCalculator {
+    config;
+    tokenCounter = null;
+    initialized = false;
+    modelId = null;
+    modelConfig = null;
+    constructor(config = DEFAULT_TOKEN_BUDGET_CONFIG) {
+        this.config = config;
+    }
+    /**
+     * Initialize the calculator (loads tiktoken).
+     */
+    async initialize() {
+        if (this.initialized)
+            return;
+        if (this.config.mode === 'token') {
+            this.tokenCounter = new TokenCounter();
+            await this.tokenCounter.initialize();
+        }
+        this.initialized = true;
+    }
+    /**
+     * Dispose resources.
+     */
+    dispose() {
+        if (this.tokenCounter) {
+            this.tokenCounter.dispose();
+            this.tokenCounter = null;
+        }
+        this.initialized = false;
+        this.modelId = null;
+        this.modelConfig = null;
+    }
+    /**
+     * Set model for adaptive budget calculation.
+     * This updates budget limits based on model capabilities.
+     */
+    setModel(modelId) {
+        this.modelId = modelId;
+        this.modelConfig = getAdaptiveBudgetCalculator().resolveConfig(modelId);
+    }
+    /**
+     * Get current model ID.
+     */
+    getModel() {
+        return this.modelId;
+    }
+    /**
+     * Get current model context config.
+     */
+    getModelConfig() {
+        return this.modelConfig;
+    }
+    /**
+     * Set user budget configuration override.
+     */
+    setUserConfig(config) {
+        getAdaptiveBudgetCalculator().setUserConfig(config);
+        // Refresh model config if model is set
+        if (this.modelId) {
+            this.modelConfig = getAdaptiveBudgetCalculator().resolveConfig(this.modelId);
+        }
+    }
+    /**
+     * Get current budget mode.
+     */
+    get mode() {
+        return this.config.mode;
+    }
+    /**
+     * Count tokens/chars for a string based on current mode.
+     */
+    count(text) {
+        if (this.config.mode === 'char') {
+            return text.length;
+        }
+        if (!this.tokenCounter) {
+            // Fallback to char estimation if not initialized
+            return Math.ceil(text.length / 4);
+        }
+        return this.tokenCounter.count(text);
+    }
+    /**
+     * Calculate total tokens for context.
+     */
+    calculateTotalTokens(context) {
+        const primary = this.count(context.primaryText ?? '');
+        const related = context.relatedFiles?.reduce((sum, file) => sum + this.count(file.content ?? ''), 0) ?? 0;
+        const snippets = context.rgSnippets.reduce((sum, snippet) => sum + this.count(snippet.content ?? ''), 0);
+        const diff = this.count(context.gitDiff ?? '') +
+            this.count(context.stagedDiff ?? '') +
+            this.count(context.unstagedDiff ?? '') +
+            this.count(context.untrackedDiff ?? '');
+        return primary + related + snippets + diff;
+    }
+    /**
+     * Calculate section-wise tokens.
+     */
+    calculateSectionTokens(context) {
+        const primary = this.count(context.primaryText ?? '');
+        const relatedFiles = context.relatedFiles?.reduce((sum, f) => sum + this.count(f.content ?? ''), 0) ?? 0;
+        const rgSnippets = context.rgSnippets.reduce((sum, s) => sum + this.count(s.content ?? ''), 0);
+        const diffs = this.count(context.gitDiff ?? '') +
+            this.count(context.stagedDiff ?? '') +
+            this.count(context.unstagedDiff ?? '') +
+            this.count(context.untrackedDiff ?? '');
+        return {
+            primary,
+            relatedFiles,
+            rgSnippets,
+            diffs,
+            total: primary + relatedFiles + rgSnippets + diffs,
+        };
+    }
+    /**
+     * Calculate tokens for a file.
+     */
+    calculateFileTokens(file) {
+        return this.count(file.content ?? '');
+    }
+    /**
+     * Calculate tokens for a snippet.
+     */
+    calculateSnippetTokens(snippet) {
+        return this.count(snippet.content ?? '');
+    }
+    /**
+     * Calculate tokens for diffs.
+     */
+    calculateDiffTokens(context) {
+        return (this.count(context.gitDiff ?? '') +
+            this.count(context.stagedDiff ?? '') +
+            this.count(context.unstagedDiff ?? '') +
+            this.count(context.untrackedDiff ?? ''));
+    }
+    /**
+     * Get default budget for current mode.
+     * If model is set, returns model-specific recommended budget.
+     */
+    getDefaultBudget() {
+        if (this.modelConfig) {
+            return this.modelConfig.recommendedBudget;
+        }
+        return this.config.defaultTokenBudget;
+    }
+    /**
+     * Get minimum budget for current mode.
+     * If model is set, returns model-specific minimum budget.
+     */
+    getMinBudget() {
+        if (this.modelConfig) {
+            return this.modelConfig.minBudget;
+        }
+        return this.config.minTokenBudget;
+    }
+    /**
+     * Get maximum primary tokens.
+     * If model is set, returns model-specific max primary tokens.
+     */
+    getMaxPrimaryTokens() {
+        if (this.modelConfig) {
+            return this.modelConfig.maxPrimaryTokens;
+        }
+        return this.config.defaultTokenBudget;
+    }
+    /**
+     * Get output buffer for current model.
+     */
+    getOutputBuffer() {
+        if (this.modelConfig) {
+            return this.modelConfig.outputBuffer;
+        }
+        return 4096; // Default output buffer
+    }
+    /**
+     * Convert token budget to char estimate (for backwards compatibility).
+     */
+    tokenToCharEstimate(tokens) {
+        return tokens * 4; // Rough estimate: 1 token ≈ 4 chars
+    }
+    /**
+     * Convert char budget to token estimate.
+     */
+    charToTokenEstimate(chars) {
+        return Math.ceil(chars / 4);
+    }
+}
+//# sourceMappingURL=token-budget.js.map

package/dist/core/context/token/types.js

@@ -0,0 +1,10 @@
+/**
+ * Token counting types and interfaces.
+ *
+ * Design Principles:
+ * - Strategy pattern for encoding extensibility
+ * - Model-agnostic configuration
+ * - Cache-first for performance
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/core/context/truncation/index.js

@@ -0,0 +1,23 @@
+/**
+ * Semantic truncation module.
+ *
+ * Provides type-aware truncation for tool outputs.
+ *
+ * @example
+ * ```typescript
+ * import { SemanticTruncator, truncateOutput } from './truncation/index.js';
+ *
+ * // Using convenience function
+ * const result = truncateOutput(output, 4000, 'error');
+ *
+ * // Using class
+ * const truncator = new SemanticTruncator();
+ * const result = truncator.truncate(output, 4000);
+ * console.log(result.keyInfoPreserved); // ['error_messages', 'stack_trace_top']
+ * ```
+ */
+export { SemanticTruncator, getSemanticTruncator, truncateOutput } from './semantic-truncator.js';
+export { detectOutputType, detectOutputTypeWithHint } from './type-detector.js';
+export { DEFAULT_TRUNCATION_CONFIG } from './types.js';
+export { ErrorStackStrategy, JsonStrategy, GitDiffStrategy, LogStrategy, TestResultStrategy, GenericStrategy, } from './strategies/index.js';
+//# sourceMappingURL=index.js.map

package/dist/core/context/truncation/semantic-truncator.js

@@ -0,0 +1,103 @@
+/**
+ * Semantic truncator - main entry point.
+ *
+ * Analyzes output type and applies appropriate truncation strategy.
+ */
+import { ErrorStackStrategy, JsonStrategy, GitDiffStrategy, LogStrategy, TestResultStrategy, GenericStrategy, } from './strategies/index.js';
+import { detectOutputType, detectOutputTypeWithHint } from './type-detector.js';
+import { DEFAULT_TRUNCATION_CONFIG } from './types.js';
+/**
+ * Semantic truncator with type-aware strategies.
+ */
+export class SemanticTruncator {
+    config;
+    strategies;
+    constructor(config = DEFAULT_TRUNCATION_CONFIG) {
+        this.config = config;
+        // Register all strategies
+        this.strategies = [
+            new ErrorStackStrategy(config),
+            new JsonStrategy(config),
+            new GitDiffStrategy(config),
+            new LogStrategy(config),
+            new TestResultStrategy(config),
+            new GenericStrategy(config), // Must be last (fallback)
+        ];
+    }
+    /**
+     * Truncate output using appropriate strategy.
+     *
+     * @param output - Raw output string
+     * @param budget - Maximum character budget
+     * @param typeHint - Optional type hint for detection
+     * @returns Truncated output with metadata
+     */
+    truncate(output, budget, typeHint) {
+        // Detect output type
+        const detection = typeHint
+            ? detectOutputTypeWithHint(output, typeHint)
+            : detectOutputType(output);
+        // Find appropriate strategy
+        const strategy = this.findStrategy(detection.type);
+        // Apply truncation
+        return strategy.truncate(output, budget);
+    }
+    /**
+     * Truncate output with explicit type.
+     *
+     * @param output - Raw output string
+     * @param type - Explicit output type
+     * @param budget - Maximum character budget
+     * @returns Truncated output with metadata
+     */
+    truncateWithType(output, type, budget) {
+        const strategy = this.findStrategy(type);
+        return strategy.truncate(output, budget);
+    }
+    /**
+     * Detect output type without truncating.
+     *
+     * @param output - Raw output string
+     * @returns Detected type with confidence
+     */
+    detectType(output) {
+        return detectOutputType(output);
+    }
+    /**
+     * Find strategy for given type.
+     */
+    findStrategy(type) {
+        for (const strategy of this.strategies) {
+            if (strategy.supportedTypes.includes(type)) {
+                return strategy;
+            }
+        }
+        // Fallback to generic
+        return this.strategies[this.strategies.length - 1];
+    }
+}
+/**
+ * Global truncator instance.
+ */
+let globalTruncator = null;
+/**
+ * Get global truncator instance.
+ */
+export function getSemanticTruncator() {
+    if (!globalTruncator) {
+        globalTruncator = new SemanticTruncator();
+    }
+    return globalTruncator;
+}
+/**
+ * Convenience function for semantic truncation.
+ *
+ * @param output - Raw output string
+ * @param budget - Maximum character budget
+ * @param typeHint - Optional type hint
+ * @returns Truncated output
+ */
+export function truncateOutput(output, budget, typeHint) {
+    return getSemanticTruncator().truncate(output, budget, typeHint);
+}
+//# sourceMappingURL=semantic-truncator.js.map

package/dist/core/context/truncation/strategies/error-stack.js

@@ -0,0 +1,94 @@
+/**
+ * Error stack truncation strategy.
+ *
+ * Preserves complete error information - errors are critical for debugging.
+ * Only truncates if absolutely necessary, preserving the error message and
+ * top of stack trace.
+ */
+import { DEFAULT_TRUNCATION_CONFIG } from '../types.js';
+/**
+ * Error stack truncation strategy.
+ * Prioritizes preserving all error information.
+ */
+export class ErrorStackStrategy {
+    config;
+    name = 'error_stack';
+    supportedTypes = ['error_stack'];
+    constructor(config = DEFAULT_TRUNCATION_CONFIG) {
+        this.config = config;
+    }
+    canHandle(output) {
+        return /^\s*\w*Error:|^\s+at\s+.+\(/m.test(output);
+    }
+    truncate(output, budget) {
+        const lines = output.split('\n');
+        const keyInfoPreserved = [];
+        // If output fits in budget, return as-is
+        if (output.length <= budget) {
+            return {
+                content: output,
+                wasTruncated: false,
+                strategy: this.name,
+                keyInfoPreserved: ['complete_error'],
+            };
+        }
+        // Find error lines (Error: TypeError:, etc.)
+        const errorLines = [];
+        const stackLines = [];
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            if (/^\s*\w*Error:|TypeError:|ReferenceError:|SyntaxError:|RuntimeError:/i.test(line)) {
+                errorLines.push(i);
+            }
+            else if (/^\s+at\s+/.test(line)) {
+                stackLines.push(i);
+            }
+        }
+        // Preserve error messages with context
+        const preservedIndices = new Set();
+        // Always preserve first line (often the error message)
+        preservedIndices.add(0);
+        // Preserve all error lines
+        for (const idx of errorLines) {
+            preservedIndices.add(idx);
+            // Add context lines around error
+            for (let j = Math.max(0, idx - this.config.contextLines); j <= Math.min(lines.length - 1, idx + this.config.contextLines); j++) {
+                preservedIndices.add(j);
+            }
+        }
+        // Preserve top of stack trace (most relevant)
+        for (let i = 0; i < Math.min(5, stackLines.length); i++) {
+            preservedIndices.add(stackLines[i]);
+        }
+        // Build truncated output
+        const result = [];
+        let currentLength = 0;
+        let lastAdded = -2;
+        for (let i = 0; i < lines.length; i++) {
+            if (preservedIndices.has(i)) {
+                // Add truncation marker if there's a gap
+                if (i > lastAdded + 1 && lastAdded >= 0) {
+                    result.push(this.config.truncationMarker);
+                    currentLength += this.config.truncationMarker.length;
+                }
+                result.push(lines[i]);
+                currentLength += lines[i].length + 1; // +1 for newline
+                lastAdded = i;
+            }
+            // Check budget
+            if (currentLength > budget) {
+                break;
+            }
+        }
+        keyInfoPreserved.push('error_messages');
+        if (stackLines.length > 0)
+            keyInfoPreserved.push('stack_trace_top');
+        return {
+            content: result.join('\n'),
+            wasTruncated: true,
+            strategy: this.name,
+            keyInfoPreserved,
+        };
+    }
+}
+//# sourceMappingURL=error-stack.js.map

package/dist/core/context/truncation/strategies/generic.js

@@ -0,0 +1,48 @@
+/**
+ * Generic truncation strategy.
+ *
+ * Fallback strategy that uses simple head/tail truncation
+ * with a marker in the middle.
+ */
+import { DEFAULT_TRUNCATION_CONFIG } from '../types.js';
+/**
+ * Generic truncation strategy.
+ * Uses simple head/tail truncation.
+ */
+export class GenericStrategy {
+    config;
+    name = 'generic';
+    supportedTypes = ['generic'];
+    constructor(config = DEFAULT_TRUNCATION_CONFIG) {
+        this.config = config;
+    }
+    canHandle(_output) {
+        return true; // Always can handle
+    }
+    truncate(output, budget) {
+        // If output fits in budget, return as-is
+        if (output.length <= budget) {
+            return {
+                content: output,
+                wasTruncated: false,
+                strategy: this.name,
+                keyInfoPreserved: ['complete_content'],
+            };
+        }
+        // Reserve space for truncation marker
+        const markerLength = this.config.truncationMarker.length;
+        const availableBudget = budget - markerLength;
+        // Split budget between head and tail (50/50)
+        const headBudget = Math.floor(availableBudget / 2);
+        const tailBudget = availableBudget - headBudget;
+        const head = output.slice(0, headBudget);
+        const tail = output.slice(-tailBudget);
+        return {
+            content: `${head}${this.config.truncationMarker}${tail}`,
+            wasTruncated: true,
+            strategy: this.name,
+            keyInfoPreserved: ['head', 'tail'],
+        };
+    }
+}
+//# sourceMappingURL=generic.js.map