ruvector 0.1.65 → 0.1.66

package/dist/core/learning-engine.js

@@ -0,0 +1,589 @@
+"use strict";
+/**
+ * Multi-Algorithm Learning Engine
+ * Supports 9 RL algorithms for intelligent hooks optimization
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LearningEngine = void 0;
+// Default configs for each task type
+const TASK_ALGORITHM_MAP = {
+    'agent-routing': {
+        algorithm: 'double-q',
+        learningRate: 0.1,
+        discountFactor: 0.95,
+        epsilon: 0.1,
+    },
+    'error-avoidance': {
+        algorithm: 'sarsa',
+        learningRate: 0.05,
+        discountFactor: 0.99,
+        epsilon: 0.05,
+    },
+    'confidence-scoring': {
+        algorithm: 'actor-critic',
+        learningRate: 0.01,
+        discountFactor: 0.95,
+        epsilon: 0.1,
+        entropyCoef: 0.01,
+    },
+    'trajectory-learning': {
+        algorithm: 'decision-transformer',
+        learningRate: 0.001,
+        discountFactor: 0.99,
+        epsilon: 0,
+        sequenceLength: 20,
+    },
+    'context-ranking': {
+        algorithm: 'ppo',
+        learningRate: 0.0003,
+        discountFactor: 0.99,
+        epsilon: 0.2,
+        clipRange: 0.2,
+        entropyCoef: 0.01,
+    },
+    'memory-recall': {
+        algorithm: 'td-lambda',
+        learningRate: 0.1,
+        discountFactor: 0.9,
+        epsilon: 0.1,
+        lambda: 0.8,
+    },
+};
+class LearningEngine {
+    constructor() {
+        this.configs = new Map();
+        this.qTables = new Map();
+        this.qTables2 = new Map(); // For Double-Q
+        this.eligibilityTraces = new Map();
+        this.actorWeights = new Map();
+        this.criticValues = new Map();
+        this.trajectories = [];
+        this.stats = new Map();
+        this.rewardHistory = [];
+        // Initialize with default configs
+        for (const [task, config] of Object.entries(TASK_ALGORITHM_MAP)) {
+            this.configs.set(task, { ...config });
+        }
+        // Initialize stats for all algorithms
+        const algorithms = [
+            'q-learning', 'sarsa', 'double-q', 'actor-critic',
+            'ppo', 'decision-transformer', 'monte-carlo', 'td-lambda', 'dqn'
+        ];
+        for (const alg of algorithms) {
+            this.stats.set(alg, {
+                algorithm: alg,
+                updates: 0,
+                avgReward: 0,
+                convergenceScore: 0,
+                lastUpdate: Date.now(),
+            });
+        }
+    }
+    /**
+     * Configure algorithm for a specific task type
+     */
+    configure(task, config) {
+        const existing = this.configs.get(task) || TASK_ALGORITHM_MAP[task];
+        this.configs.set(task, { ...existing, ...config });
+    }
+    /**
+     * Get current configuration for a task
+     */
+    getConfig(task) {
+        return this.configs.get(task) || TASK_ALGORITHM_MAP[task];
+    }
+    /**
+     * Update Q-value using the appropriate algorithm
+     */
+    update(task, experience) {
+        const config = this.getConfig(task);
+        let delta = 0;
+        switch (config.algorithm) {
+            case 'q-learning':
+                delta = this.qLearningUpdate(experience, config);
+                break;
+            case 'sarsa':
+                delta = this.sarsaUpdate(experience, config);
+                break;
+            case 'double-q':
+                delta = this.doubleQUpdate(experience, config);
+                break;
+            case 'actor-critic':
+                delta = this.actorCriticUpdate(experience, config);
+                break;
+            case 'ppo':
+                delta = this.ppoUpdate(experience, config);
+                break;
+            case 'td-lambda':
+                delta = this.tdLambdaUpdate(experience, config);
+                break;
+            case 'monte-carlo':
+                // Monte Carlo needs full episodes
+                this.addToCurrentTrajectory(experience);
+                if (experience.done) {
+                    delta = this.monteCarloUpdate(config);
+                }
+                break;
+            case 'decision-transformer':
+                this.addToCurrentTrajectory(experience);
+                if (experience.done) {
+                    delta = this.decisionTransformerUpdate(config);
+                }
+                break;
+            case 'dqn':
+                delta = this.dqnUpdate(experience, config);
+                break;
+        }
+        // Update stats
+        this.updateStats(config.algorithm, experience.reward, Math.abs(delta));
+        return delta;
+    }
+    /**
+     * Get best action for a state
+     */
+    getBestAction(task, state, actions) {
+        const config = this.getConfig(task);
+        // Epsilon-greedy exploration
+        if (Math.random() < config.epsilon) {
+            const randomAction = actions[Math.floor(Math.random() * actions.length)];
+            return { action: randomAction, confidence: 0.5 };
+        }
+        let bestAction = actions[0];
+        let bestValue = -Infinity;
+        let values = [];
+        const qTable = this.getQTable(state);
+        for (const action of actions) {
+            const value = qTable.get(action) || 0;
+            values.push(value);
+            if (value > bestValue) {
+                bestValue = value;
+                bestAction = action;
+            }
+        }
+        // Calculate confidence using softmax
+        const confidence = this.softmaxConfidence(values, actions.indexOf(bestAction));
+        return { action: bestAction, confidence };
+    }
+    /**
+     * Get action probabilities (for Actor-Critic and PPO)
+     */
+    getActionProbabilities(state, actions) {
+        const probs = new Map();
+        const qTable = this.getQTable(state);
+        const values = actions.map(a => qTable.get(a) || 0);
+        const maxVal = Math.max(...values);
+        const expValues = values.map(v => Math.exp(v - maxVal));
+        const sumExp = expValues.reduce((a, b) => a + b, 0);
+        for (let i = 0; i < actions.length; i++) {
+            probs.set(actions[i], expValues[i] / sumExp);
+        }
+        return probs;
+    }
+    // ============ Algorithm Implementations ============
+    /**
+     * Standard Q-Learning: Q(s,a) += α * (r + γ * max_a' Q(s',a') - Q(s,a))
+     */
+    qLearningUpdate(exp, config) {
+        const { state, action, reward, nextState, done } = exp;
+        const { learningRate: α, discountFactor: γ } = config;
+        const qTable = this.getQTable(state);
+        const nextQTable = this.getQTable(nextState);
+        const currentQ = qTable.get(action) || 0;
+        const maxNextQ = done ? 0 : Math.max(0, ...Array.from(nextQTable.values()));
+        const target = reward + γ * maxNextQ;
+        const delta = target - currentQ;
+        const newQ = currentQ + α * delta;
+        qTable.set(action, newQ);
+        return delta;
+    }
+    /**
+     * SARSA: On-policy, more conservative
+     * Q(s,a) += α * (r + γ * Q(s',a') - Q(s,a))
+     */
+    sarsaUpdate(exp, config) {
+        const { state, action, reward, nextState, done } = exp;
+        const { learningRate: α, discountFactor: γ, epsilon } = config;
+        const qTable = this.getQTable(state);
+        const nextQTable = this.getQTable(nextState);
+        const currentQ = qTable.get(action) || 0;
+        // On-policy: use expected value under current policy (ε-greedy)
+        let nextQ = 0;
+        if (!done) {
+            const nextActions = Array.from(nextQTable.keys());
+            if (nextActions.length > 0) {
+                const maxQ = Math.max(...Array.from(nextQTable.values()));
+                const avgQ = Array.from(nextQTable.values()).reduce((a, b) => a + b, 0) / nextActions.length;
+                // Expected value under ε-greedy
+                nextQ = (1 - epsilon) * maxQ + epsilon * avgQ;
+            }
+        }
+        const target = reward + γ * nextQ;
+        const delta = target - currentQ;
+        const newQ = currentQ + α * delta;
+        qTable.set(action, newQ);
+        return delta;
+    }
+    /**
+     * Double Q-Learning: Reduces overestimation bias
+     * Uses two Q-tables, randomly updates one using the other for target
+     */
+    doubleQUpdate(exp, config) {
+        const { state, action, reward, nextState, done } = exp;
+        const { learningRate: α, discountFactor: γ } = config;
+        const useFirst = Math.random() < 0.5;
+        const qTable = useFirst ? this.getQTable(state) : this.getQTable2(state);
+        const otherQTable = useFirst ? this.getQTable2(nextState) : this.getQTable(nextState);
+        const nextQTable = useFirst ? this.getQTable(nextState) : this.getQTable2(nextState);
+        const currentQ = qTable.get(action) || 0;
+        let nextQ = 0;
+        if (!done) {
+            // Find best action in next state using one table
+            let bestAction = '';
+            let bestValue = -Infinity;
+            for (const [a, v] of nextQTable) {
+                if (v > bestValue) {
+                    bestValue = v;
+                    bestAction = a;
+                }
+            }
+            // Evaluate using other table
+            if (bestAction) {
+                nextQ = otherQTable.get(bestAction) || 0;
+            }
+        }
+        const target = reward + γ * nextQ;
+        const delta = target - currentQ;
+        const newQ = currentQ + α * delta;
+        qTable.set(action, newQ);
+        return delta;
+    }
+    /**
+     * Actor-Critic: Policy gradient with value baseline
+     */
+    actorCriticUpdate(exp, config) {
+        const { state, action, reward, nextState, done } = exp;
+        const { learningRate: α, discountFactor: γ } = config;
+        // Critic update (TD error)
+        const V = this.criticValues.get(state) || 0;
+        const V_next = done ? 0 : (this.criticValues.get(nextState) || 0);
+        const tdError = reward + γ * V_next - V;
+        this.criticValues.set(state, V + α * tdError);
+        // Actor update (policy gradient)
+        const qTable = this.getQTable(state);
+        const currentQ = qTable.get(action) || 0;
+        // Use TD error as advantage estimate
+        const newQ = currentQ + α * tdError;
+        qTable.set(action, newQ);
+        return tdError;
+    }
+    /**
+     * PPO: Clipped policy gradient for stable training
+     */
+    ppoUpdate(exp, config) {
+        const { state, action, reward, nextState, done } = exp;
+        const { learningRate: α, discountFactor: γ, clipRange = 0.2 } = config;
+        // Critic update
+        const V = this.criticValues.get(state) || 0;
+        const V_next = done ? 0 : (this.criticValues.get(nextState) || 0);
+        const advantage = reward + γ * V_next - V;
+        this.criticValues.set(state, V + α * advantage);
+        // Actor update with clipping
+        const qTable = this.getQTable(state);
+        const oldQ = qTable.get(action) || 0;
+        // Compute probability ratio (simplified)
+        const ratio = Math.exp(α * advantage);
+        const clippedRatio = Math.max(1 - clipRange, Math.min(1 + clipRange, ratio));
+        // PPO objective: min(ratio * A, clip(ratio) * A)
+        const update = Math.min(ratio * advantage, clippedRatio * advantage);
+        const newQ = oldQ + α * update;
+        qTable.set(action, newQ);
+        return advantage;
+    }
+    /**
+     * TD(λ): Temporal difference with eligibility traces
+     */
+    tdLambdaUpdate(exp, config) {
+        const { state, action, reward, nextState, done } = exp;
+        const { learningRate: α, discountFactor: γ, lambda = 0.8 } = config;
+        const qTable = this.getQTable(state);
+        const nextQTable = this.getQTable(nextState);
+        const currentQ = qTable.get(action) || 0;
+        const maxNextQ = done ? 0 : Math.max(0, ...Array.from(nextQTable.values()));
+        const tdError = reward + γ * maxNextQ - currentQ;
+        // Update eligibility trace for current state-action
+        const traces = this.getEligibilityTraces(state);
+        traces.set(action, (traces.get(action) || 0) + 1);
+        // Update all state-actions with eligibility traces
+        for (const [s, sTraces] of this.eligibilityTraces) {
+            const sQTable = this.getQTable(s);
+            for (const [a, trace] of sTraces) {
+                const q = sQTable.get(a) || 0;
+                sQTable.set(a, q + α * tdError * trace);
+                // Decay trace
+                sTraces.set(a, γ * lambda * trace);
+            }
+        }
+        return tdError;
+    }
+    /**
+     * Monte Carlo: Full episode learning
+     */
+    monteCarloUpdate(config) {
+        const { learningRate: α, discountFactor: γ } = config;
+        const trajectory = this.trajectories[this.trajectories.length - 1];
+        if (!trajectory || trajectory.experiences.length === 0)
+            return 0;
+        let G = 0; // Return
+        let totalDelta = 0;
+        // Work backwards through episode
+        for (let t = trajectory.experiences.length - 1; t >= 0; t--) {
+            const exp = trajectory.experiences[t];
+            G = exp.reward + γ * G;
+            const qTable = this.getQTable(exp.state);
+            const currentQ = qTable.get(exp.action) || 0;
+            const delta = G - currentQ;
+            qTable.set(exp.action, currentQ + α * delta);
+            totalDelta += Math.abs(delta);
+        }
+        trajectory.completed = true;
+        trajectory.totalReward = G;
+        return totalDelta / trajectory.experiences.length;
+    }
+    /**
+     * Decision Transformer: Sequence modeling for trajectories
+     */
+    decisionTransformerUpdate(config) {
+        const { learningRate: α, sequenceLength = 20 } = config;
+        const trajectory = this.trajectories[this.trajectories.length - 1];
+        if (!trajectory || trajectory.experiences.length === 0)
+            return 0;
+        // Decision Transformer learns to predict actions given (return, state, action) sequences
+        // Here we use a simplified version that learns state-action patterns
+        let totalDelta = 0;
+        const experiences = trajectory.experiences.slice(-sequenceLength);
+        // Calculate returns-to-go
+        const returns = [];
+        let R = 0;
+        for (let i = experiences.length - 1; i >= 0; i--) {
+            R += experiences[i].reward;
+            returns.unshift(R);
+        }
+        // Update Q-values weighted by return-to-go
+        for (let i = 0; i < experiences.length; i++) {
+            const exp = experiences[i];
+            const qTable = this.getQTable(exp.state);
+            const currentQ = qTable.get(exp.action) || 0;
+            // Weight by normalized return
+            const normalizedReturn = returns[i] / (Math.abs(returns[0]) + 1);
+            const target = currentQ + α * normalizedReturn * exp.reward;
+            const delta = target - currentQ;
+            qTable.set(exp.action, target);
+            totalDelta += Math.abs(delta);
+        }
+        trajectory.completed = true;
+        trajectory.totalReward = returns[0];
+        return totalDelta / experiences.length;
+    }
+    /**
+     * DQN: Deep Q-Network (simplified without actual neural network)
+     * Uses experience replay and target network concepts
+     */
+    dqnUpdate(exp, config) {
+        // Add to replay buffer (trajectory)
+        this.addToCurrentTrajectory(exp);
+        // Sample from replay buffer
+        const replayExp = this.sampleFromReplay();
+        if (!replayExp)
+            return this.qLearningUpdate(exp, config);
+        // Use sampled experience for update (breaks correlation)
+        return this.qLearningUpdate(replayExp, config);
+    }
+    // ============ Helper Methods ============
+    getQTable(state) {
+        if (!this.qTables.has(state)) {
+            this.qTables.set(state, new Map());
+        }
+        return this.qTables.get(state);
+    }
+    getQTable2(state) {
+        if (!this.qTables2.has(state)) {
+            this.qTables2.set(state, new Map());
+        }
+        return this.qTables2.get(state);
+    }
+    getEligibilityTraces(state) {
+        if (!this.eligibilityTraces.has(state)) {
+            this.eligibilityTraces.set(state, new Map());
+        }
+        return this.eligibilityTraces.get(state);
+    }
+    softmaxConfidence(values, selectedIdx) {
+        if (values.length === 0)
+            return 0.5;
+        const maxVal = Math.max(...values);
+        const expValues = values.map(v => Math.exp(v - maxVal));
+        const sumExp = expValues.reduce((a, b) => a + b, 0);
+        return expValues[selectedIdx] / sumExp;
+    }
+    addToCurrentTrajectory(exp) {
+        if (this.trajectories.length === 0 || this.trajectories[this.trajectories.length - 1].completed) {
+            this.trajectories.push({
+                experiences: [],
+                totalReward: 0,
+                completed: false,
+            });
+        }
+        this.trajectories[this.trajectories.length - 1].experiences.push(exp);
+    }
+    sampleFromReplay() {
+        const allExperiences = [];
+        for (const traj of this.trajectories) {
+            allExperiences.push(...traj.experiences);
+        }
+        if (allExperiences.length === 0)
+            return null;
+        return allExperiences[Math.floor(Math.random() * allExperiences.length)];
+    }
+    updateStats(algorithm, reward, delta) {
+        const stats = this.stats.get(algorithm);
+        if (!stats)
+            return;
+        stats.updates++;
+        stats.lastUpdate = Date.now();
+        // Running average reward
+        this.rewardHistory.push(reward);
+        if (this.rewardHistory.length > 1000) {
+            this.rewardHistory.shift();
+        }
+        stats.avgReward = this.rewardHistory.reduce((a, b) => a + b, 0) / this.rewardHistory.length;
+        // Convergence score (inverse of recent delta magnitude)
+        stats.convergenceScore = 1 / (1 + delta);
+    }
+    /**
+     * Get statistics for all algorithms
+     */
+    getStats() {
+        return new Map(this.stats);
+    }
+    /**
+     * Get statistics summary
+     */
+    getStatsSummary() {
+        let bestAlgorithm = 'q-learning';
+        let bestScore = -Infinity;
+        let totalUpdates = 0;
+        const algorithms = [];
+        for (const [alg, stats] of this.stats) {
+            algorithms.push(stats);
+            totalUpdates += stats.updates;
+            const score = stats.avgReward * stats.convergenceScore;
+            if (score > bestScore && stats.updates > 0) {
+                bestScore = score;
+                bestAlgorithm = alg;
+            }
+        }
+        return {
+            bestAlgorithm,
+            totalUpdates,
+            avgReward: this.rewardHistory.length > 0
+                ? this.rewardHistory.reduce((a, b) => a + b, 0) / this.rewardHistory.length
+                : 0,
+            algorithms: algorithms.filter(a => a.updates > 0),
+        };
+    }
+    /**
+     * Export state for persistence
+     */
+    export() {
+        const qTables = {};
+        for (const [state, actions] of this.qTables) {
+            qTables[state] = Object.fromEntries(actions);
+        }
+        const qTables2 = {};
+        for (const [state, actions] of this.qTables2) {
+            qTables2[state] = Object.fromEntries(actions);
+        }
+        const criticValues = Object.fromEntries(this.criticValues);
+        const stats = {};
+        for (const [alg, s] of this.stats) {
+            stats[alg] = s;
+        }
+        const configs = {};
+        for (const [task, config] of this.configs) {
+            configs[task] = config;
+        }
+        return {
+            qTables,
+            qTables2,
+            criticValues,
+            trajectories: this.trajectories.slice(-100), // Keep last 100 trajectories
+            stats,
+            configs,
+            rewardHistory: this.rewardHistory.slice(-1000),
+        };
+    }
+    /**
+     * Import state from persistence
+     */
+    import(data) {
+        // Q-tables
+        this.qTables.clear();
+        for (const [state, actions] of Object.entries(data.qTables || {})) {
+            this.qTables.set(state, new Map(Object.entries(actions)));
+        }
+        this.qTables2.clear();
+        for (const [state, actions] of Object.entries(data.qTables2 || {})) {
+            this.qTables2.set(state, new Map(Object.entries(actions)));
+        }
+        // Critic values
+        this.criticValues = new Map(Object.entries(data.criticValues || {}));
+        // Trajectories
+        this.trajectories = data.trajectories || [];
+        // Stats
+        for (const [alg, s] of Object.entries(data.stats || {})) {
+            this.stats.set(alg, s);
+        }
+        // Configs
+        for (const [task, config] of Object.entries(data.configs || {})) {
+            this.configs.set(task, config);
+        }
+        // Reward history
+        this.rewardHistory = data.rewardHistory || [];
+    }
+    /**
+     * Clear all learning data
+     */
+    clear() {
+        this.qTables.clear();
+        this.qTables2.clear();
+        this.eligibilityTraces.clear();
+        this.actorWeights.clear();
+        this.criticValues.clear();
+        this.trajectories = [];
+        this.rewardHistory = [];
+        // Reset stats
+        for (const stats of this.stats.values()) {
+            stats.updates = 0;
+            stats.avgReward = 0;
+            stats.convergenceScore = 0;
+        }
+    }
+    /**
+     * Get available algorithms
+     */
+    static getAlgorithms() {
+        return [
+            { algorithm: 'q-learning', description: 'Simple off-policy learning', bestFor: 'General routing' },
+            { algorithm: 'sarsa', description: 'On-policy, conservative', bestFor: 'Error avoidance' },
+            { algorithm: 'double-q', description: 'Reduces overestimation', bestFor: 'Precise routing' },
+            { algorithm: 'actor-critic', description: 'Policy gradient + value', bestFor: 'Confidence scoring' },
+            { algorithm: 'ppo', description: 'Stable policy updates', bestFor: 'Preference learning' },
+            { algorithm: 'decision-transformer', description: 'Sequence modeling', bestFor: 'Trajectory patterns' },
+            { algorithm: 'monte-carlo', description: 'Full episode learning', bestFor: 'Unbiased estimates' },
+            { algorithm: 'td-lambda', description: 'Eligibility traces', bestFor: 'Credit assignment' },
+            { algorithm: 'dqn', description: 'Experience replay', bestFor: 'High-dim states' },
+        ];
+    }
+}
+exports.LearningEngine = LearningEngine;
+exports.default = LearningEngine;