vektor-slipstream 1.2.2 → 1.3.0

package/package.json

@@ -1,90 +1,114 @@
-{
-    "name": "vektor-slipstream",
-    "version": "1.2.2",
-    "description": "Hardware-accelerated persistent memory for AI agents. Local-first, zero cloud dependency, $0 embedding cost.",
-    "main": "slipstream-core.js",
-    "bin": {
-        "vektor": "./vektor-cli.js"
-    },
-    "exports": {
-        ".": "./slipstream-core.js",
-        "./cloak": "./cloak-index.js",
-        "./cloak-warmup": "./cloak-warmup.js",
-        "./db": "./slipstream-db.js",
-        "./embedder": "./slipstream-embedder.js",
-        "./cloak-behaviour": "./cloak-behaviour.js",
-        "./cloak-pattern-store": "./cloak-pattern-store.js",
-        "./cloak-recorder-auto": "./cloak-recorder-auto.js",
-        "./cloak-recorder-snippet": "./cloak-recorder-snippet.js",
-        "./cloak-turbo-quant": "./cloak-turbo-quant.js"
-    },
-    "keywords": [
-        "ai", "memory", "agent", "vector", "sqlite", "embeddings",
-        "langchain", "openai", "anthropic", "claude", "mcp",
-        "rag", "persistent-memory", "local-ai", "onnx", "mistral", "cloak"
-    ],
-    "author": "VEKTOR Memory <hello@vektormemory.com>",
-    "license": "SEE LICENSE IN LICENSE",
-    "homepage": "https://vektormemory.com",
-    "repository": {
-        "type": "git",
-        "url": "https://github.com/Vektor-Memory/Vektor-memory"
-    },
-    "engines": {
-        "node": ">=18.0.0"
-    },
-    "dependencies": {
-        "better-sqlite3": "^12.8.0",
-        "onnxruntime-node": "^1.17.3"
-    },
-    "optionalDependencies": {
-        "@anthropic-ai/sdk": "^0.82.0",
-        "@xenova/transformers": "^2.17.2",
-        "playwright": "^1.44.0",
-        "blessed": "^0.1.81",
-        "blessed-contrib": "^4.11.0",
-        "sqlite-vec-darwin-arm64": "^0.1.6",
-        "sqlite-vec-linux-x64": "^0.1.6",
-        "sqlite-vec-windows-x64": "^0.1.6"
-    },
-    "files": [
-        "slipstream-core.js",
-        "slipstream-db.js",
-        "slipstream-embedder.js",
-        "detect-hardware.js",
-        "vektor-licence.js",
-        "vektor-licence-prompt.js",
-        "vektor-cli.js",
-        "vektor-setup.js",
-        "vektor-banner-loader.js",
-        "vektor-tui.js",
-        "boot-screen.html",
-        "cloak-core.js",
-        "cloak-index.js",
-        "cloak-identity.js",
-        "cloak-captcha.js",
-        "cloak-llms.js",
-        "cloak-turbo-quant.js",
-        "cloak-warmup.js",
-        "cloak-behaviour.js",
-        "cloak-recorder-snippet.js",
-        "cloak-pattern-store.js",
-        "cloak-recorder-auto.js",
-        "sovereign.js",
-        "visualize.js",
-        "TENETS.md",
-        "README.md",
-        "LICENSE",
-        "models/model_quantized.onnx",
-        "models/vocab.json",
-        "examples/",
-        "mistral/"
-    ],
-    "publishConfig": {
-        "access": "public"
-    },
-    "devDependencies": {
-        "javascript-obfuscator": "^5.4.1"
-    }
-}
-
+{
+  "name": "vektor-slipstream",
+  "version": "1.3.0",
+  "description": "Hardware-accelerated persistent memory for AI agents. Local-first, zero cloud dependency, $0 embedding cost.",
+  "main": "slipstream-core.js",
+  "bin": {
+    "vektor": "./vektor-cli.js"
+  },
+  "exports": {
+    ".": "./slipstream-core.js",
+    "./cloak": "./cloak-index.js",
+    "./cloak-warmup": "./cloak-warmup.js",
+    "./db": "./slipstream-db.js",
+    "./embedder": "./slipstream-embedder.js",
+    "./cloak-behaviour": "./cloak-behaviour.js",
+    "./cloak-pattern-store": "./cloak-pattern-store.js",
+    "./cloak-recorder-auto": "./cloak-recorder-auto.js",
+    "./cloak-recorder-snippet": "./cloak-recorder-snippet.js",
+    "./cloak-turbo-quant": "./cloak-turbo-quant.js",
+    "./cloak/cortex": "./cortex.js",
+    "./cloak/axon": "./axon.js",
+    "./cloak/cerebellum": "./cerebellum.js",
+    "./cloak/token": "./token.js",
+    "./cloak/mcp": "./index.js"
+  },
+  "keywords": [
+    "ai",
+    "memory",
+    "agent",
+    "vector",
+    "sqlite",
+    "embeddings",
+    "langchain",
+    "openai",
+    "anthropic",
+    "claude",
+    "mcp",
+    "rag",
+    "persistent-memory",
+    "local-ai",
+    "onnx",
+    "mistral",
+    "cloak"
+  ],
+  "author": "VEKTOR Memory <hello@vektormemory.com>",
+  "license": "SEE LICENSE IN LICENSE",
+  "homepage": "https://vektormemory.com",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Vektor-Memory/Vektor-memory"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "better-sqlite3": "^12.8.0",
+    "onnxruntime-node": "^1.17.3"
+  },
+  "optionalDependencies": {
+    "@anthropic-ai/sdk": "^0.82.0",
+    "@xenova/transformers": "^2.17.2",
+    "playwright": "^1.44.0",
+    "blessed": "^0.1.81",
+    "blessed-contrib": "^4.11.0",
+    "sqlite-vec-darwin-arm64": "^0.1.6",
+    "sqlite-vec-linux-x64": "^0.1.6",
+    "sqlite-vec-windows-x64": "^0.1.6"
+  },
+  "files": [
+    "slipstream-core.js",
+    "slipstream-db.js",
+    "slipstream-embedder.js",
+    "detect-hardware.js",
+    "vektor-licence.js",
+    "vektor-licence-prompt.js",
+    "vektor-cli.js",
+    "vektor-setup.js",
+    "vektor-banner-loader.js",
+    "vektor-tui.js",
+    "boot-screen.html",
+    "cloak-core.js",
+    "cloak-index.js",
+    "cloak-identity.js",
+    "cloak-captcha.js",
+    "cloak-llms.js",
+    "cloak-turbo-quant.js",
+    "cloak-warmup.js",
+    "cloak-behaviour.js",
+    "cloak-recorder-snippet.js",
+    "cloak-pattern-store.js",
+    "cloak-recorder-auto.js",
+    "sovereign.js",
+    "visualize.js",
+    "TENETS.md",
+    "README.md",
+    "LICENSE",
+    "cortex.js",
+    "axon.js",
+    "cerebellum.js",
+    "token.js",
+    "index.js",
+    "models/model_quantized.onnx",
+    "models/vocab.json",
+    "examples/",
+    "mistral/",
+    "vektor-slipstream.dxt"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "javascript-obfuscator": "^5.4.1"
+  }
+}

package/token.js

@@ -0,0 +1,322 @@
+/**
+ * cloak_token.js
+ * Token ledger — aggregates context budget consumption per session.
+ * Writes ONE summary node per session end (via cloak_axon integration).
+ * Detects waste patterns: repeated reads, oversized context injections.
+ *
+ * Design principles:
+ *   - Aggregated, not per-operation. Never writes on every read/write.
+ *   - Session total written ONCE at Stop (passed to cloak_axon's tokenCount).
+ *   - Waste patterns flagged at session end, not in real time.
+ *   - Uses character-to-token ratios (OpenWolf method, ±15% accurate).
+ *
+ * Architecture: CLOAK layer → Vektor temporal graph
+ * Research: OpenWolf token-ledger pattern
+ */
+
+'use strict';
+
+// ---------------------------------------------------------------------------
+// Token estimation ratios (same as cortex.js for consistency)
+// ---------------------------------------------------------------------------
+const TOKEN_RATIOS = {
+  code : 3.5,
+  prose: 4.0,
+  mixed: 3.75,
+};
+
+const CODE_EXTS = new Set(['.js','.ts','.jsx','.tsx','.py','.go','.rs','.rb','.java','.c','.cpp']);
+
+function estimateTokens(content, filePath = '') {
+  if (!content) return 0;
+  const ext   = (filePath.match(/\.[^.]+$/) || [''])[0].toLowerCase();
+  const bytes = Buffer.byteLength(String(content), 'utf8');
+  if (CODE_EXTS.has(ext)) return Math.round(bytes / TOKEN_RATIOS.code);
+  return Math.round(bytes / TOKEN_RATIOS.mixed);
+}
+
+// ---------------------------------------------------------------------------
+// In-RAM session ledger
+// ---------------------------------------------------------------------------
+const _ledger = {
+  sessionId    : null,
+  reads        : [],   // { file, tokens, ts }
+  writes       : [],   // { file, tokens, ts }
+  wasteEvents  : [],   // { type, description, tokens_wasted }
+  totalReadTokens : 0,
+  totalWriteTokens: 0,
+};
+
+function _resetLedger() {
+  _ledger.sessionId          = null;
+  _ledger.reads              = [];
+  _ledger.writes             = [];
+  _ledger.wasteEvents        = [];
+  _ledger.totalReadTokens    = 0;
+  _ledger.totalWriteTokens   = 0;
+}
+
+// ---------------------------------------------------------------------------
+// Waste detection thresholds
+// ---------------------------------------------------------------------------
+const WASTE_RULES = {
+  REPEATED_READ_THRESHOLD  : 2,      // same file read > 2x = waste
+  LARGE_READ_THRESHOLD     : 2000,   // reading >2000 tokens when anatomy would suffice
+  REPEATED_READ_PENALTY    : 0.8,    // assume 80% of repeated read tokens are wasted
+};
+
+// ---------------------------------------------------------------------------
+// Session lifecycle
+// ---------------------------------------------------------------------------
+
+/**
+ * startSession(sessionId)
+ * Called from cloak_axon onSessionStart. Resets ledger for new session.
+ */
+function startSession(sessionId) {
+  _resetLedger();
+  _ledger.sessionId = sessionId;
+}
+
+// ---------------------------------------------------------------------------
+// Event tracking — called from hook integration
+// ---------------------------------------------------------------------------
+
+/**
+ * trackRead({ filePath, content })
+ * Record a file read. Detects repeated reads and large reads.
+ */
+function trackRead({ filePath, content } = {}) {
+  if (!_ledger.sessionId) return;
+
+  const tokens = estimateTokens(content, filePath);
+  const file   = filePath || 'unknown';
+  const ts     = Date.now();
+
+  _ledger.reads.push({ file, tokens, ts });
+  _ledger.totalReadTokens += tokens;
+
+  // Detect repeated reads of same file this session
+  const readCount = _ledger.reads.filter(r => r.file === file).length;
+  if (readCount > WASTE_RULES.REPEATED_READ_THRESHOLD) {
+    const wastedTokens = Math.round(tokens * WASTE_RULES.REPEATED_READ_PENALTY);
+    _ledger.wasteEvents.push({
+      type           : 'repeated_read',
+      description    : `${file} read ${readCount}x this session`,
+      tokens_wasted  : wastedTokens,
+      recommendation : 'Use cloak_cortex anatomy to check file before re-reading',
+    });
+  }
+
+  // Detect large reads that cortex anatomy could have avoided
+  if (tokens > WASTE_RULES.LARGE_READ_THRESHOLD) {
+    _ledger.wasteEvents.push({
+      type           : 'large_read',
+      description    : `${file} read at ~${tokens} tokens`,
+      tokens_wasted  : Math.round(tokens * 0.5), // assume 50% avoidable
+      recommendation : 'Check cloak_cortex anatomy description first',
+    });
+  }
+}
+
+/**
+ * trackWrite({ filePath, content })
+ * Record a file write.
+ */
+function trackWrite({ filePath, content } = {}) {
+  if (!_ledger.sessionId) return;
+
+  const tokens = estimateTokens(content, filePath);
+  _ledger.writes.push({ file: filePath || 'unknown', tokens, ts: Date.now() });
+  _ledger.totalWriteTokens += tokens;
+}
+
+// ---------------------------------------------------------------------------
+// Session summary — called at session end, returns total for cloak_axon
+// ---------------------------------------------------------------------------
+
+/**
+ * getSessionSummary()
+ * Returns aggregated stats for the current session.
+ * This is passed as tokenCount to cloak_axon.onSessionStop().
+ */
+function getSessionSummary() {
+  if (!_ledger.sessionId) return null;
+
+  const totalTokens    = _ledger.totalReadTokens + _ledger.totalWriteTokens;
+  const totalWasted    = _ledger.wasteEvents.reduce((sum, e) => sum + (e.tokens_wasted || 0), 0);
+  const wasteRate      = totalTokens > 0 ? Math.round((totalWasted / totalTokens) * 100) : 0;
+
+  // Top files by token consumption
+  const fileTotals = {};
+  [..._ledger.reads, ..._ledger.writes].forEach(({ file, tokens }) => {
+    fileTotals[file] = (fileTotals[file] || 0) + tokens;
+  });
+  const topFiles = Object.entries(fileTotals)
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 5)
+    .map(([file, tokens]) => `${file}:~${tokens}t`);
+
+  return {
+    sessionId        : _ledger.sessionId,
+    totalTokens,
+    readTokens       : _ledger.totalReadTokens,
+    writeTokens      : _ledger.totalWriteTokens,
+    wastedTokens     : totalWasted,
+    wasteRate        : `${wasteRate}%`,
+    wasteEvents      : _ledger.wasteEvents.length,
+    topFiles,
+    readCount        : _ledger.reads.length,
+    writeCount       : _ledger.writes.length,
+    uniqueFilesRead  : new Set(_ledger.reads.map(r => r.file)).size,
+    uniqueFilesWritten: new Set(_ledger.writes.map(w => w.file)).size,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Temporal graph node builder — ONE write per session
+// This is called by cloak_axon at session stop, not by token.js directly.
+// ---------------------------------------------------------------------------
+
+/**
+ * buildTokenNode(summary)
+ * Builds the Vektor memory string for the temporal graph.
+ * cloak_axon calls this and includes it in the MemCell, OR it can be
+ * written as a standalone temporal node. Either way: one write per session.
+ */
+function buildTokenNode(summary) {
+  if (!summary) return null;
+
+  return [
+    `[CLOAK_TOKEN_LEDGER]`,
+    `session:${summary.sessionId}`,
+    `total_tokens:~${summary.totalTokens}`,
+    `read_tokens:~${summary.readTokens}`,
+    `write_tokens:~${summary.writeTokens}`,
+    `wasted_tokens:~${summary.wastedTokens}`,
+    `waste_rate:${summary.wasteRate}`,
+    `waste_events:${summary.wasteEvents}`,
+    `top_files:${summary.topFiles.join(',')}`,
+    `logged_at:${new Date().toISOString()}`,
+  ].join(' | ');
+}
+
+// ---------------------------------------------------------------------------
+// Lifetime waste report — queries Vektor for historical patterns
+// ---------------------------------------------------------------------------
+
+/**
+ * getWasteReport({ memory, days })
+ * Queries Vektor temporal graph for token waste patterns over N days.
+ * Returns actionable recommendations. Read-only — no writes.
+ *
+ * @param {object} memory  - Vektor memory instance
+ * @param {number} days    - Lookback window (default: 30)
+ */
+async function getWasteReport({ memory, days = 30 } = {}) {
+  if (!memory) throw new Error('cloak_token: memory instance is required');
+
+  let nodes;
+  try {
+    nodes = await memory.recall(
+      `[CLOAK_TOKEN_LEDGER]`,
+      { limit: days }
+    );
+  } catch (err) {
+    return { error: err.message, sessions: [] };
+  }
+
+  if (!nodes || nodes.length === 0) {
+    return { sessions: [], totalTokens: 0, totalWasted: 0, recommendations: [] };
+  }
+
+  // Parse historical nodes
+  const sessions = nodes.map(n => {
+    const parts  = n.content.split(' | ');
+    const get    = key => {
+      const p = parts.find(p => p.startsWith(`${key}:`));
+      return p ? p.slice(key.length + 1) : null;
+    };
+    return {
+      sessionId    : get('session'),
+      totalTokens  : parseInt(get('total_tokens')?.replace('~','') || '0'),
+      wastedTokens : parseInt(get('wasted_tokens')?.replace('~','') || '0'),
+      wasteRate    : get('waste_rate'),
+      wasteEvents  : parseInt(get('waste_events') || '0'),
+    };
+  });
+
+  const totalTokens = sessions.reduce((s, n) => s + n.totalTokens, 0);
+  const totalWasted = sessions.reduce((s, n) => s + n.wastedTokens, 0);
+  const avgWasteRate = totalTokens > 0 ? Math.round((totalWasted / totalTokens) * 100) : 0;
+
+  const recommendations = [];
+  if (avgWasteRate > 30) {
+    recommendations.push(`High waste rate (${avgWasteRate}%) — run cloak_cortex scan to refresh anatomy index`);
+  }
+  if (sessions.filter(s => s.wasteEvents > 3).length > 3) {
+    recommendations.push('Repeated waste events — check most-read files and add to anatomy');
+  }
+
+  return {
+    sessions : sessions.slice(0, 10),
+    totalTokens,
+    totalWasted,
+    avgWasteRate : `${avgWasteRate}%`,
+    recommendations,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Claude Code hook integration
+// ---------------------------------------------------------------------------
+
+/**
+ * handleHookPayload({ payload, sessionId })
+ * Routes Claude Code hook events to trackRead / trackWrite.
+ * Called from the main MCP server hook handler.
+ */
+function handleHookPayload({ payload, sessionId } = {}) {
+  if (!payload) return null;
+
+  const hookName = payload?.hook_event_name || payload?.event;
+  const tool     = payload?.tool_name;
+  const input    = payload?.tool_input;
+
+  // Start ledger if we receive a session ID and haven't started yet
+  if (sessionId && !_ledger.sessionId) {
+    startSession(sessionId);
+  }
+
+  switch (hookName) {
+    case 'PostToolUse':
+    case 'post_tool_use': {
+      if (tool === 'Read') {
+        trackRead({
+          filePath: input?.file_path || input?.path,
+          content : payload?.tool_response?.content || '',
+        });
+      } else if (['Write','Edit','MultiEdit'].includes(tool)) {
+        trackWrite({
+          filePath: input?.file_path || input?.path,
+          content : input?.new_string || input?.content || '',
+        });
+      }
+      break;
+    }
+  }
+
+  return null; // token tracking never returns hook responses
+}
+
+module.exports = {
+  startSession,
+  trackRead,
+  trackWrite,
+  getSessionSummary,
+  buildTokenNode,
+  getWasteReport,
+  handleHookPayload,
+  estimateTokens,
+  _ledger, // exported for testing and cloak_axon integration
+};