sentinel-agentos 0.3.8 → 0.3.10

package/dist/evaluator/exec-evaluator.d.ts

@@ -0,0 +1,102 @@
+import { PreExecMetrics, RuntimeMetrics, PostExecMetrics } from '../types';
+import { SchemaGate } from '../guard/schema-gate';
+import { RiskGate } from '../guard/risk-gate';
+import { WorkingMemory } from '../memory/working';
+/**
+ * PreExecEvaluator — captures metrics before tool execution.
+ *
+ * Watches the Guard layer output and WorkingMemory context
+ * to score parameter quality, context utilization, and risk.
+ */
+export declare class PreExecEvaluator {
+    private schemaGate;
+    private riskGate;
+    private workingMemory;
+    constructor(schemaGate: SchemaGate, riskGate: RiskGate, workingMemory: WorkingMemory);
+    /**
+     * Evaluate a tool call before execution.
+     */
+    evaluate(toolName: string, parameters: Record<string, unknown>): PreExecMetrics;
+    /**
+     * Score parameter quality based on contextual awareness.
+     *
+     * High quality: path contains session-relevant project paths,
+     * content references open files, etc.
+     * Low quality: bare strings, random-looking paths, missing files.
+     */
+    private evaluateParamQuality;
+    /**
+     * Score how well the agent uses stored context.
+     */
+    private evaluateContextUtilization;
+}
+/**
+ * RuntimeEvaluator — captures metrics during execution.
+ *
+ * Tracks retries, self-corrections, timeouts, and
+ * whether the agent selected the right tool for the job.
+ */
+export declare class RuntimeEvaluator {
+    /** Historical tool selection patterns — toolName -> successful scenario count */
+    private toolHistory;
+    /**
+     * Evaluate a completed tool execution.
+     */
+    evaluate(options: {
+        toolName: string;
+        startTime: number;
+        endTime: number;
+        retryCount: number;
+        wasSelfCorrected: boolean;
+        hadTimeout: boolean;
+        expectedTool?: string;
+        toolResult: unknown;
+    }): RuntimeMetrics;
+    /** Record a tool call in the history tracker */
+    private recordToolCall;
+    /** Get tool selection accuracy statistics */
+    getToolAccuracy(): Record<string, {
+        calls: number;
+        successRate: number;
+    }>;
+}
+/**
+ * PostExecEvaluator — captures metrics after execution.
+ *
+ * Scores verify results, user acceptance patterns,
+ * and checks if the agent actually used its own result later.
+ */
+export declare class PostExecEvaluator {
+    /** Track result references for utilization scoring */
+    private resultReferenceTracker;
+    /**
+     * Evaluate post-execution outcomes.
+     */
+    evaluate(options: {
+        verifyPassed: boolean;
+        verifyChecks: number;
+        verifyFailures: number;
+        userAccepted: boolean;
+        userProvidedEdit: boolean;
+        resultWasUsed: boolean;
+        diffLinesChanged?: number;
+    }): PostExecMetrics;
+    /**
+     * Track a tool result for later utilization detection.
+     * Call this after each tool execution.
+     */
+    trackResult(operationId: string, result: unknown): void;
+    /**
+     * Mark a previously-tracked result as referenced (used by the agent later).
+     */
+    markResultReferenced(operationId: string): void;
+    /**
+     * Check if a result has been utilized by the agent.
+     */
+    isResultReferenced(operationId: string): boolean;
+    /**
+     * Get overall result utilization rate.
+     */
+    getUtilizationRate(): number;
+}
+//# sourceMappingURL=exec-evaluator.d.ts.map

package/dist/evaluator/exec-evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"exec-evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator/exec-evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,EACd,cAAc,EACd,eAAe,EAGhB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAElD;;;;;GAKG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,aAAa,CAAgB;gBAGnC,UAAU,EAAE,UAAU,EACtB,QAAQ,EAAE,QAAQ,EAClB,aAAa,EAAE,aAAa;IAO9B;;OAEG;IACH,QAAQ,CACN,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAClC,cAAc;IAuBjB;;;;;;OAMG;IACH,OAAO,CAAC,oBAAoB;IA+C5B;;OAEG;IACH,OAAO,CAAC,0BAA0B;CAqCnC;AAED;;;;;GAKG;AACH,qBAAa,gBAAgB;IAC3B,iFAAiF;IACjF,OAAO,CAAC,WAAW,CAAgE;IAEnF;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,gBAAgB,EAAE,OAAO,CAAC;QAC1B,UAAU,EAAE,OAAO,CAAC;QACpB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,OAAO,CAAC;KACrB,GAAG,cAAc;IA0ClB,gDAAgD;IAChD,OAAO,CAAC,cAAc;IAUtB,6CAA6C;IAC7C,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;CAY1E;AAED;;;;;GAKG;AACH,qBAAa,iBAAiB;IAC5B,sDAAsD;IACtD,OAAO,CAAC,sBAAsB,CAAoE;IAElG;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE;QAChB,YAAY,EAAE,OAAO,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,OAAO,CAAC;QACtB,gBAAgB,EAAE,OAAO,CAAC;QAC1B,aAAa,EAAE,OAAO,CAAC;QACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,GAAG,eAAe;IA+BnB;;;OAGG;IACH,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,IAAI;IAIvD;;OAEG;IACH,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAK/C;;OAEG;IACH,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;IAIhD;;OAEG;IACH,kBAAkB,IAAI,MAAM;CAM7B"}

package/dist/evaluator/exec-evaluator.js

@@ -0,0 +1,266 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PostExecEvaluator = exports.RuntimeEvaluator = exports.PreExecEvaluator = void 0;
+/**
+ * PreExecEvaluator — captures metrics before tool execution.
+ *
+ * Watches the Guard layer output and WorkingMemory context
+ * to score parameter quality, context utilization, and risk.
+ */
+class PreExecEvaluator {
+    schemaGate;
+    riskGate;
+    workingMemory;
+    constructor(schemaGate, riskGate, workingMemory) {
+        this.schemaGate = schemaGate;
+        this.riskGate = riskGate;
+        this.workingMemory = workingMemory;
+    }
+    /**
+     * Evaluate a tool call before execution.
+     */
+    evaluate(toolName, parameters) {
+        // 1. Schema check
+        const schemaCheck = this.schemaGate.check(toolName, parameters);
+        // 2. Risk assessment
+        const riskScore = this.riskGate.evaluate(toolName, parameters);
+        // 3. Parameter quality: does the agent use context-aware params?
+        const paramQuality = this.evaluateParamQuality(toolName, parameters);
+        // 4. Context utilization: is the agent leveraging WorkingMemory?
+        const contextUtilization = this.evaluateContextUtilization(toolName, parameters);
+        return {
+            timestamp: Date.now(),
+            toolName,
+            schemaCheck,
+            riskScore,
+            paramQuality,
+            contextUtilization,
+        };
+    }
+    /**
+     * Score parameter quality based on contextual awareness.
+     *
+     * High quality: path contains session-relevant project paths,
+     * content references open files, etc.
+     * Low quality: bare strings, random-looking paths, missing files.
+     */
+    evaluateParamQuality(_toolName, parameters) {
+        let score = 0.5; // neutral start
+        const observations = [];
+        // Check if path references an open file
+        if (typeof parameters['path'] === 'string') {
+            const path = parameters['path'];
+            if (this.workingMemory.openFiles.some((f) => path.includes(f))) {
+                score += 0.3;
+                observations.push('Path references an open file');
+            }
+            if (path.startsWith('/') || path.match(/^[A-Z]:\\/)) {
+                observations.push('Absolute path used');
+            }
+        }
+        // Check if content parameter is meaningful
+        if (typeof parameters['content'] === 'string') {
+            const content = parameters['content'];
+            if (content.length > 20) {
+                score = Math.min(1.0, score + 0.1);
+            }
+            if (content.length === 0) {
+                score -= 0.2;
+                observations.push('Empty content — possible error');
+            }
+        }
+        // Check for file paths in multiple parameters
+        const filePaths = Object.values(parameters).filter((v) => typeof v === 'string' && (v.includes('.ts') || v.includes('.js') || v.includes('.json')));
+        if (filePaths.length > 1) {
+            score = Math.min(1.0, score + 0.1);
+            observations.push('Multiple file references — coordinated operation');
+        }
+        return {
+            score: Math.round(Math.max(0, Math.min(1, score)) * 100) / 100,
+            observations,
+        };
+    }
+    /**
+     * Score how well the agent uses stored context.
+     */
+    evaluateContextUtilization(_toolName, parameters) {
+        let score = 0.4;
+        const patterns = [];
+        // Check if agent references recent messages
+        if (this.workingMemory.recentMessages.length > 0) {
+            score += 0.1;
+            patterns.push(`${this.workingMemory.recentMessages.length} recent messages available`);
+        }
+        // Check if agent uses cached tool results
+        const cachedCount = this.workingMemory.recentToolResults.size;
+        if (cachedCount > 0) {
+            score += 0.1;
+            patterns.push(`${cachedCount} cached results available`);
+        }
+        // Check parameter values for context patterns
+        const allValues = Object.values(parameters).map(String).join(' ');
+        for (const msg of this.workingMemory.recentMessages.slice(-3)) {
+            const words = msg.content.split(/\s+/).filter((w) => w.length > 3);
+            for (const word of words.slice(0, 5)) {
+                if (allValues.includes(word)) {
+                    score += 0.1;
+                    patterns.push(`Parameter references recent context: "${word}"`);
+                }
+            }
+        }
+        return {
+            score: Math.round(Math.min(1.0, score) * 100) / 100,
+            patterns,
+        };
+    }
+}
+exports.PreExecEvaluator = PreExecEvaluator;
+/**
+ * RuntimeEvaluator — captures metrics during execution.
+ *
+ * Tracks retries, self-corrections, timeouts, and
+ * whether the agent selected the right tool for the job.
+ */
+class RuntimeEvaluator {
+    /** Historical tool selection patterns — toolName -> successful scenario count */
+    toolHistory = new Map();
+    /**
+     * Evaluate a completed tool execution.
+     */
+    evaluate(options) {
+        const durationMs = options.endTime - options.startTime;
+        const toolSuccess = !options.hadTimeout && options.toolResult !== undefined;
+        // Tool selection accuracy: compare against historical patterns
+        let toolSelectionMatch;
+        if (options.expectedTool) {
+            // Direct comparison if expectedTool is provided
+            toolSelectionMatch = options.toolName === options.expectedTool;
+        }
+        else {
+            // Auto-detect: is this tool historically successful for similar params?
+            const history = this.toolHistory.get(options.toolName);
+            if (history) {
+                const historicalSuccessRate = history.calls > 0
+                    ? history.successes / history.calls
+                    : 0;
+                // If this tool has >70% historical success, consider it a "good" selection
+                toolSelectionMatch = historicalSuccessRate > 0.7 ? true : undefined;
+            }
+        }
+        // Record this call in history
+        this.recordToolCall(options.toolName, toolSuccess);
+        // Adaptive score: composite of retry rate, timeout, correction
+        let adaptiveScore = 1.0;
+        adaptiveScore -= options.retryCount * 0.15; // Each retry costs 0.15
+        if (options.hadTimeout)
+            adaptiveScore -= 0.5;
+        if (options.wasSelfCorrected)
+            adaptiveScore += 0.2; // Self-correction is good!
+        adaptiveScore = Math.max(0, Math.min(1, adaptiveScore));
+        return {
+            retryCount: options.retryCount,
+            selfCorrected: options.wasSelfCorrected,
+            hadTimeout: options.hadTimeout,
+            toolSuccess,
+            toolSelectionMatch,
+            adaptiveScore: Math.round(adaptiveScore * 100) / 100,
+            durationMs,
+        };
+    }
+    /** Record a tool call in the history tracker */
+    recordToolCall(toolName, success) {
+        const existing = this.toolHistory.get(toolName);
+        if (existing) {
+            existing.calls++;
+            if (success)
+                existing.successes++;
+        }
+        else {
+            this.toolHistory.set(toolName, { calls: 1, successes: success ? 1 : 0 });
+        }
+    }
+    /** Get tool selection accuracy statistics */
+    getToolAccuracy() {
+        const result = {};
+        for (const [tool, history] of this.toolHistory) {
+            result[tool] = {
+                calls: history.calls,
+                successRate: history.calls > 0
+                    ? Math.round((history.successes / history.calls) * 100) / 100
+                    : 0,
+            };
+        }
+        return result;
+    }
+}
+exports.RuntimeEvaluator = RuntimeEvaluator;
+/**
+ * PostExecEvaluator — captures metrics after execution.
+ *
+ * Scores verify results, user acceptance patterns,
+ * and checks if the agent actually used its own result later.
+ */
+class PostExecEvaluator {
+    /** Track result references for utilization scoring */
+    resultReferenceTracker = new Map();
+    /**
+     * Evaluate post-execution outcomes.
+     */
+    evaluate(options) {
+        // Verify score
+        const verifyScore = options.verifyChecks > 0
+            ? 1 - (options.verifyFailures / options.verifyChecks)
+            : 1;
+        // User acceptance
+        const acceptance = options.userAccepted ? 1.0 : options.userProvidedEdit ? 0.3 : 0.7;
+        // Composite outcome score
+        const outcomeScore = (verifyScore * 0.3 +
+            acceptance * 0.4 +
+            (options.resultWasUsed ? 0.3 : 0));
+        // Overall health flag
+        const healthy = verifyScore > 0.8 && acceptance > 0.5;
+        return {
+            verifyPassed: options.verifyPassed,
+            verifyScore: Math.round(verifyScore * 100) / 100,
+            userAccepted: options.userAccepted,
+            userEditRate: options.userProvidedEdit ? 1 : 0,
+            resultUtilized: options.resultWasUsed,
+            outcomeScore: Math.round(outcomeScore * 100) / 100,
+            healthy,
+            diffLinesChanged: options.diffLinesChanged,
+        };
+    }
+    /**
+     * Track a tool result for later utilization detection.
+     * Call this after each tool execution.
+     */
+    trackResult(operationId, result) {
+        this.resultReferenceTracker.set(operationId, { result, referenced: false });
+    }
+    /**
+     * Mark a previously-tracked result as referenced (used by the agent later).
+     */
+    markResultReferenced(operationId) {
+        const entry = this.resultReferenceTracker.get(operationId);
+        if (entry)
+            entry.referenced = true;
+    }
+    /**
+     * Check if a result has been utilized by the agent.
+     */
+    isResultReferenced(operationId) {
+        return this.resultReferenceTracker.get(operationId)?.referenced ?? false;
+    }
+    /**
+     * Get overall result utilization rate.
+     */
+    getUtilizationRate() {
+        const entries = Array.from(this.resultReferenceTracker.values());
+        if (entries.length === 0)
+            return 0;
+        const referenced = entries.filter((e) => e.referenced).length;
+        return Math.round((referenced / entries.length) * 100) / 100;
+    }
+}
+exports.PostExecEvaluator = PostExecEvaluator;
+//# sourceMappingURL=exec-evaluator.js.map

package/dist/evaluator/exec-evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"exec-evaluator.js","sourceRoot":"","sources":["../../src/evaluator/exec-evaluator.ts"],"names":[],"mappings":";;;AAWA;;;;;GAKG;AACH,MAAa,gBAAgB;IACnB,UAAU,CAAa;IACvB,QAAQ,CAAW;IACnB,aAAa,CAAgB;IAErC,YACE,UAAsB,EACtB,QAAkB,EAClB,aAA4B;QAE5B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,QAAQ,CACN,QAAgB,EAChB,UAAmC;QAEnC,kBAAkB;QAClB,MAAM,WAAW,GAAgB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,qBAAqB;QACrB,MAAM,SAAS,GAAc,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE1E,iEAAiE;QACjE,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAErE,iEAAiE;QACjE,MAAM,kBAAkB,GAAG,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAEjF,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,QAAQ;YACR,WAAW;YACX,SAAS;YACT,YAAY;YACZ,kBAAkB;SACnB,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACK,oBAAoB,CAC1B,SAAiB,EACjB,UAAmC;QAEnC,IAAI,KAAK,GAAG,GAAG,CAAC,CAAC,gBAAgB;QACjC,MAAM,YAAY,GAAa,EAAE,CAAC;QAElC,wCAAwC;QACxC,IAAI,OAAO,UAAU,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAW,CAAC;YAC1C,IAAI,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/D,KAAK,IAAI,GAAG,CAAC;gBACb,YAAY,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;YACpD,CAAC;YACD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;gBACpD,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,2CAA2C;QAC3C,IAAI,OAAO,UAAU,CAAC,SAAS,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC9C,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAW,CAAC;YAChD,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACxB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,GAAG,GAAG,CAAC,CAAC;YACrC,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,KAAK,IAAI,GAAG,CAAC;gBACb,YAAY,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAED,8CAA8C;QAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAChD,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAChG,CAAC;QAEF,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,GAAG,GAAG,CAAC,CAAC;YACnC,YAAY,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;QACxE,CAAC;QAED,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;YAC9D,YAAY;SACb,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,0BAA0B,CAChC,SAAiB,EACjB,UAAmC;QAEnC,IAAI,KAAK,GAAG,GAAG,CAAC;QAChB,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,4CAA4C;QAC5C,IAAI,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,KAAK,IAAI,GAAG,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,MAAM,4BAA4B,CAAC,CAAC;QACzF,CAAC;QAED,0CAA0C;QAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,iBAAiB,CAAC,IAAI,CAAC;QAC9D,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YACpB,KAAK,IAAI,GAAG,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,GAAG,WAAW,2BAA2B,CAAC,CAAC;QAC3D,CAAC;QAED,8CAA8C;QAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACnE,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBACrC,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7B,KAAK,IAAI,GAAG,CAAC;oBACb,QAAQ,CAAC,IAAI,CAAC,yCAAyC,IAAI,GAAG,CAAC,CAAC;gBAClE,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;YACnD,QAAQ;SACT,CAAC;IACJ,CAAC;CACF;AA1ID,4CA0IC;AAED;;;;;GAKG;AACH,MAAa,gBAAgB;IAC3B,iFAAiF;IACzE,WAAW,GAAsD,IAAI,GAAG,EAAE,CAAC;IAEnF;;OAEG;IACH,QAAQ,CAAC,OASR;QACC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC;QACvD,MAAM,WAAW,GAAG,CAAC,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC;QAE5E,+DAA+D;QAC/D,IAAI,kBAAuC,CAAC;QAC5C,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACzB,gDAAgD;YAChD,kBAAkB,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,YAAY,CAAC;QACjE,CAAC;aAAM,CAAC;YACN,wEAAwE;YACxE,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACvD,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,qBAAqB,GAAG,OAAO,CAAC,KAAK,GAAG,CAAC;oBAC7C,CAAC,CAAC,OAAO,CAAC,SAAS,GAAG,OAAO,CAAC,KAAK;oBACnC,CAAC,CAAC,CAAC,CAAC;gBACN,2EAA2E;gBAC3E,kBAAkB,GAAG,qBAAqB,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YACtE,CAAC;QACH,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QAEnD,+DAA+D;QAC/D,IAAI,aAAa,GAAG,GAAG,CAAC;QACxB,aAAa,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,wBAAwB;QACpE,IAAI,OAAO,CAAC,UAAU;YAAE,aAAa,IAAI,GAAG,CAAC;QAC7C,IAAI,OAAO,CAAC,gBAAgB;YAAE,aAAa,IAAI,GAAG,CAAC,CAAC,2BAA2B;QAC/E,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;QAExD,OAAO;YACL,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,aAAa,EAAE,OAAO,CAAC,gBAAgB;YACvC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW;YACX,kBAAkB;YAClB,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC,GAAG,GAAG;YACpD,UAAU;SACX,CAAC;IACJ,CAAC;IAED,gDAAgD;IACxC,cAAc,CAAC,QAAgB,EAAE,OAAgB;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,IAAI,OAAO;gBAAE,QAAQ,CAAC,SAAS,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,eAAe;QACb,MAAM,MAAM,GAA2D,EAAE,CAAC;QAC1E,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,GAAG;gBACb,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,WAAW,EAAE,OAAO,CAAC,KAAK,GAAG,CAAC;oBAC5B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;oBAC7D,CAAC,CAAC,CAAC;aACN,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAlFD,4CAkFC;AAED;;;;;GAKG;AACH,MAAa,iBAAiB;IAC5B,sDAAsD;IAC9C,sBAAsB,GAA0D,IAAI,GAAG,EAAE,CAAC;IAElG;;OAEG;IACH,QAAQ,CAAC,OAQR;QACC,eAAe;QACf,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,GAAG,CAAC;YAC1C,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC;YACrD,CAAC,CAAC,CAAC,CAAC;QAEN,kBAAkB;QAClB,MAAM,UAAU,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAErF,0BAA0B;QAC1B,MAAM,YAAY,GAAG,CACnB,WAAW,GAAG,GAAG;YACjB,UAAU,GAAG,GAAG;YAChB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAClC,CAAC;QAEF,sBAAsB;QACtB,MAAM,OAAO,GAAG,WAAW,GAAG,GAAG,IAAI,UAAU,GAAG,GAAG,CAAC;QAEtD,OAAO;YACL,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC,GAAG,GAAG;YAChD,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,YAAY,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9C,cAAc,EAAE,OAAO,CAAC,aAAa;YACrC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC,GAAG,GAAG;YAClD,OAAO;YACP,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;SAC3C,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,WAAmB,EAAE,MAAe;QAC9C,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,WAAmB;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,KAAK;YAAE,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,WAAmB;QACpC,OAAO,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,UAAU,IAAI,KAAK,CAAC;IAC3E,CAAC;IAED;;OAEG;IACH,kBAAkB;QAChB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,MAAM,EAAE,CAAC,CAAC;QACjE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACnC,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;QAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;IAC/D,CAAC;CACF;AA9ED,8CA8EC"}

package/dist/evaluator/feedback.d.ts

@@ -0,0 +1,102 @@
+import { ImplicitFeedback, SignalType } from '../types';
+import { AuditEntry } from '../types';
+/**
+ * ImplicitFeedbackEngine — captures and interprets implicit user signals.
+ *
+ * Instead of relying on explicit "thumbs up/down", this engine
+ * detects subtle signals from user behavior to infer satisfaction.
+ *
+ * Two modes:
+ * - Manual: caller provides explicit signals via record()
+ * - Auto-detect: scans audit log to infer signals (results unused,
+ *   results modified later, repeated same tool, verify failures)
+ *
+ * This is the key differentiator of AgentOS: it learns from
+ * what users DO, not just what they SAY.
+ */
+export declare class ImplicitFeedbackEngine {
+    private feedbackLog;
+    private persistPath;
+    private globalAuditPath;
+    private detectedKeys;
+    private static readonly MAX_FEEDBACK;
+    private static readonly MAX_DETECTED_KEYS;
+    private persistLinePath;
+    /**
+     * Record an implicit feedback signal.
+     */
+    record(signal: SignalType, sessionId: string, operationId?: string, confidence?: number, source?: string): ImplicitFeedback;
+    /**
+     * Scan the audit log and auto-detect implicit feedback signals.
+     *
+     * Detection rules (conservative — low confidence to avoid false positives):
+     * - verify FAIL or WARN → user_provided_correction (agent made mistakes)
+     * - same tool+params called within 60s → user_repeated_instruction (low confidence, noisy)
+     * - high risk operations that were retried and eventually passed → agent_self_corrected
+     *
+     * Note: auto-detected signals carry lower confidence than explicit user feedback.
+     * They serve as supplementary data, not primary quality indicators.
+     *
+     * @param entries Recent audit entries to analyze
+     * @param sessionId Session to attribute signals to
+     * @returns Number of signals auto-detected
+     */
+    /**
+     * Enable persistence for feedbackLog and auto-detected signal keys.
+     */
+    enablePersistence(workspaceRoot: string): void;
+    /** Persist: append one line to feedback-lines.jsonl. */
+    private _writeCount;
+    private persist;
+    /** Compact feedback-lines.jsonl: keep last 200 lines only. */
+    private compactFeedbackFile;
+    /** Load persisted feedback log from disk. */
+    private load;
+    /**
+     * Cross-session auto-detect: scan the global audit.jsonl for signals
+     * from ALL sessions, not just the current one.
+     */
+    autoDetectGlobal(): number;
+    autoDetect(entries: AuditEntry[], sessionId: string): number;
+    /**
+     * Analyze user messages to detect implicit correction/feedback signals.
+     *
+     * Chinese corrective patterns (high precision, low recall — only matches clear signals):
+     * - "不对"/"错了"/"不是这样" → user_provided_correction (confidence 0.8)
+     * - "漏了"/"缺了"/"没包括"/"遗漏" → user_provided_correction (confidence 0.75)
+     * - "你没"/"你怎么"/"你咋" + 负面动作(忘记/漏/没/不) → user_provided_correction (confidence 0.7)
+     * - "失忆"/"忘了"/"不记得" → user_provided_correction (confidence 0.85)
+     * - "重新"/"再查"/"再搜"/"再看看" → user_repeated_instruction (confidence 0.6)
+     * - "不对吧"/"没音信" → user_interrupted (confidence 0.5)
+     *
+     * English patterns:
+     * - "wrong"/"incorrect"/"not right" → user_provided_correction (confidence 0.7)
+     * - "missed"/"missing"/"forgot"/"incomplete" → user_provided_correction (confidence 0.7)
+     * - "redo"/"again"/"retry"/"try again" → user_repeated_instruction (confidence 0.5)
+     *
+     * Returns number of signals detected.
+     */
+    detectFromUserMessages(messages: Array<{
+        role: string;
+        content: string;
+        ts?: number;
+    }>, sessionId: string): number;
+    private getSignalStrength;
+    getSatisfactionScore(sessionId?: string, recentHours?: number): number;
+    query(filter?: {
+        signal?: SignalType;
+        sessionId?: string;
+        minStrength?: number;
+        maxStrength?: number;
+        since?: number;
+        limit?: number;
+    }): ImplicitFeedback[];
+    stats(): {
+        totalSignals: number;
+        positiveSignals: number;
+        negativeSignals: number;
+        averageStrength: number;
+        mostCommonSignal: SignalType | null;
+    };
+}
+//# sourceMappingURL=feedback.d.ts.map

package/dist/evaluator/feedback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"feedback.d.ts","sourceRoot":"","sources":["../../src/evaluator/feedback.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAYtC;;;;;;;;;;;;;GAaG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,WAAW,CAA0B;IAC7C,OAAO,CAAC,WAAW,CAAuB;IAE1C,OAAO,CAAC,eAAe,CAAuB;IAE9C,OAAO,CAAC,YAAY,CAA0B;IAE9C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAO;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAO;IAEhD,OAAO,CAAC,eAAe,CAAuB;IAE9C;;OAEG;IACH,MAAM,CACJ,MAAM,EAAE,UAAU,EAClB,SAAS,EAAE,MAAM,EACjB,WAAW,CAAC,EAAE,MAAM,EACpB,UAAU,SAAM,EAChB,MAAM,SAAkB,GACvB,gBAAgB;IA8BnB;;;;;;;;;;;;;;OAcG;IACH;;OAEG;IACH,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI;IAW9C,wDAAwD;IACxD,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,OAAO;IAgBf,8DAA8D;IAC9D,OAAO,CAAC,mBAAmB;IAa3B,6CAA6C;IAC7C,OAAO,CAAC,IAAI;IAkCZ;;;OAGG;IACH,gBAAgB,IAAI,MAAM;IAa1B,UAAU,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;IAmF5D;;;;;;;;;;;;;;;;;OAiBG;IACH,sBAAsB,CAAC,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM;IA2FlH,OAAO,CAAC,iBAAiB;IAkBzB,oBAAoB,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,WAAW,SAAK,GAAG,MAAM;IAoClE,KAAK,CAAC,MAAM,GAAE;QACZ,MAAM,CAAC,EAAE,UAAU,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KACX,GAAG,gBAAgB,EAAE;IAa3B,KAAK,IAAI;QACP,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,eAAe,EAAE,MAAM,CAAC;QACxB,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,UAAU,GAAG,IAAI,CAAC;KACrC;CAuBF"}