smart-context-mcp 1.18.1 → 1.20.0

package/README.md

@@ -56,7 +56,7 @@ Restart your AI client. Done.
 # Check installed version
 npm list -g smart-context-mcp
 
-# Should show: smart-context-mcp@1.18.1 (or later)
+# Should show: smart-context-mcp@1.20.0 (or later)
 
 # Update to latest version
 npm update -g smart-context-mcp
@@ -278,22 +278,30 @@ Check actual usage:
 
 Provides **two key components**:
 
-### 1. Specialized Tools (12 tools)
+### 1. Specialized Tools (20 tools)
 
 | Tool | Purpose | Savings |
 |------|---------|---------|
-| `smart_read` | Read files in outline/signatures mode | 90% |
+| `smart_read` | Read files in outline / signatures / symbol / explain mode | 90% |
 | `smart_read_batch` | Read multiple files in one call | 90% |
-| `smart_search` | Intent-aware code search with ranking | 95% |
-| `smart_context` | One-call context builder | 85% |
-| `smart_summary` | Task checkpoint management | 98% |
-| `smart_turn` | Task recovery orchestration | - |
-| `smart_metrics` | Token usage inspection | - |
-| `smart_shell` | Safe command execution | 94% |
-| `build_index` | Symbol index builder | - |
+| `smart_search` | Intent-aware code search with ranking and `kinds` filter (incl. ADRs) | 95% |
+| `smart_context` | One-call context builder; `paths: { from, to }` mode traverses the import graph | 85% |
+| `smart_shell` | Safe command execution (TAP / git-log / diff compression) | 94% |
+| `smart_test` | Affected tests via graph + sandboxed runner + persisted `last_failure` | - |
+| `smart_review` | One-call review preflight: diff + callers + tests + heuristic findings | - |
+| `build_index` | Symbol index builder (incremental) | - |
 | `warm_cache` | File preloading (5x faster cold start) | - |
 | `git_blame` | Function-level code attribution | - |
 | `cross_project` | Multi-project context | - |
+| `smart_summary` | Task checkpoint management with rolling window | 98% |
+| `smart_status` | Quick session / project state inspection | - |
+| `smart_doctor` | Health checks for storage, index, hooks | - |
+| `smart_edit` | Targeted symbol-aware edits | - |
+| `smart_turn` | Turn boundary + `nextActions[]` machine-readable plan | - |
+| `smart_resume` | Lightweight alias for `smart_turn(phase: 'start', verbosity: 'minimal')` | - |
+| `smart_metrics` | Token usage inspection | - |
+| `smart_playbook` | Declarative workflows: composes `smart_*` tools in one call (preflight-merge, debug-flake, refactor-safe, doc-sync, ramp-up) | - |
+| `global_memory` | Cross-project memory in `~/.devctx/global.db` (opt-in, scrubbed for secrets, semantic recall) | - |
 
 ### 2. Agent Rules (Task-Specific Guidance)
 
@@ -646,9 +654,12 @@ npm run verify
 ## Storage
 
 Data stored in `.devctx/`:
-- `index.json` - Symbol index
-- `state.sqlite` - Task checkpoints, metrics, patterns (Node 22+)
-- `metrics.jsonl` - Legacy fallback (Node 18-20)
+- `index.json` - Symbol index (`INDEX_VERSION 7`: ADR + ADR sections, richer Python/Go)
+- `state.sqlite` - Sessions, metrics, patterns, task handoffs, test failures, explain cache (Node 22+)
+- `metrics.jsonl` - Opt-in legacy file, only when `DEVCTX_METRICS_FILE=path.jsonl` is set
+
+Cross-project (opt-in via `DEVCTX_GLOBAL_MEMORY=true`):
+- `~/.devctx/global.db` - Scrubbed decisions, patterns, playbooks, notes with semantic recall
 
 Add to `.gitignore`:
 ```

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "smart-context-mcp",
   "mcpName": "io.github.Arrayo/smart-context-mcp",
-  "version": "1.18.1",
+  "version": "1.20.0",
   "description": "MCP server that reduces agent token usage by 90% with intelligent context compression, task checkpoint persistence, and workflow-aware agent guidance.",
   "author": "Francisco Caballero Portero <fcp1978@hotmail.com>",
   "type": "module",
@@ -69,8 +69,10 @@
     "eval:context": "node ./evals/harness.js --tool=context",
     "eval:both": "node ./evals/harness.js --tool=both",
     "eval:self": "node ./evals/harness.js --root=../.. --corpus=./evals/corpus/self-tasks.json",
+    "eval:self:json": "node ./evals/harness.js --root=../.. --corpus=./evals/corpus/self-tasks.json --json",
     "eval:realworld": "node ./evals/realworld-eval.js",
     "eval:realworld:json": "node ./evals/realworld-eval.js --json",
+    "eval:kpi:baseline": "node ./evals/kpi-baseline.js",
     "eval:report": "node ./evals/report.js",
     "report:metrics": "node ./scripts/report-metrics.js",
     "report:workflows": "node ./scripts/report-workflow-metrics.js",
@@ -83,5 +85,6 @@
     "js-tiktoken": "^1.0.21",
     "typescript": "^6.0.2",
     "zod": "^4.1.5"
-  }
+  },
+  "packageManager": "pnpm@10.33.3+sha512.a19744364a7e248b92657a4ca5973f9354d21caf982579674b1c539f32c7420c47138ad8b1254df07aba9bc782d9b3029e3db34d5dbff974326eb74dac8ff489"
 }

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/Arrayo/smart-context-mcp",
     "source": "github"
   },
-  "version": "1.18.1",
+  "version": "1.20.0",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "smart-context-mcp",
-      "version": "1.18.1",
+      "version": "1.20.0",
       "transport": {
         "type": "stdio"
       },

package/src/embeddings/embedder.js

@@ -0,0 +1,28 @@
+import { embed as hashingEmbed, cosineSimilarity, buildCorpusIdf, DEFAULT_DIMENSIONS } from './hashing.js';
+
+const HASHING_EMBEDDER = {
+  id: 'hashing-v1',
+  dimensions: DEFAULT_DIMENSIONS,
+  embed: (text, options = {}) => hashingEmbed(text, options),
+  similarity: cosineSimilarity,
+  buildCorpusIdf,
+};
+
+let activeEmbedder = HASHING_EMBEDDER;
+
+export const getEmbedder = () => activeEmbedder;
+
+export const setEmbedder = (embedder) => {
+  if (!embedder || typeof embedder.embed !== 'function' || typeof embedder.similarity !== 'function') {
+    throw new Error('Embedder must implement embed(text, opts) and similarity(a, b)');
+  }
+  activeEmbedder = {
+    id: embedder.id ?? 'custom',
+    dimensions: embedder.dimensions ?? DEFAULT_DIMENSIONS,
+    embed: embedder.embed,
+    similarity: embedder.similarity,
+    buildCorpusIdf: embedder.buildCorpusIdf ?? buildCorpusIdf,
+  };
+};
+
+export const resetEmbedder = () => { activeEmbedder = HASHING_EMBEDDER; };

package/src/embeddings/hashing.js

@@ -0,0 +1,77 @@
+import { tokenize } from './tokenize.js';
+
+export const DEFAULT_DIMENSIONS = 256;
+
+const fnv1a = (str) => {
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < str.length; i += 1) {
+    hash ^= str.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193) >>> 0;
+  }
+  return hash;
+};
+
+const signedBucket = (token, dims) => {
+  const h = fnv1a(token);
+  const bucket = h % dims;
+  const sign = ((h >>> 16) & 1) === 0 ? 1 : -1;
+  return { bucket, sign };
+};
+
+const l2Normalize = (vector) => {
+  let sumSq = 0;
+  for (let i = 0; i < vector.length; i += 1) sumSq += vector[i] * vector[i];
+  if (sumSq === 0) return vector;
+  const norm = Math.sqrt(sumSq);
+  for (let i = 0; i < vector.length; i += 1) vector[i] /= norm;
+  return vector;
+};
+
+export const embed = (text, { dimensions = DEFAULT_DIMENSIONS, idf = null } = {}) => {
+  const tokens = tokenize(text);
+  const vector = new Float32Array(dimensions);
+  if (tokens.length === 0) return vector;
+
+  const counts = new Map();
+  for (const token of tokens) {
+    counts.set(token, (counts.get(token) ?? 0) + 1);
+  }
+
+  for (const [token, count] of counts) {
+    const tf = 1 + Math.log(count);
+    const weight = idf ? tf * (idf.get(token) ?? 1) : tf;
+    const { bucket, sign } = signedBucket(token, dimensions);
+    vector[bucket] += sign * weight;
+  }
+
+  return l2Normalize(vector);
+};
+
+export const cosineSimilarity = (a, b) => {
+  if (!a || !b || a.length !== b.length) return 0;
+  let dot = 0;
+  for (let i = 0; i < a.length; i += 1) dot += a[i] * b[i];
+  return dot;
+};
+
+export const buildCorpusIdf = (documents) => {
+  const df = new Map();
+  let docCount = 0;
+  for (const doc of documents) {
+    docCount += 1;
+    const seen = new Set();
+    for (const token of tokenize(doc)) {
+      if (!seen.has(token)) {
+        seen.add(token);
+        df.set(token, (df.get(token) ?? 0) + 1);
+      }
+    }
+  }
+  const idf = new Map();
+  for (const [token, freq] of df) {
+    idf.set(token, Math.log((docCount + 1) / (freq + 1)) + 1);
+  }
+  return idf;
+};
+
+export const _internal = { fnv1a, signedBucket, l2Normalize };

package/src/embeddings/index.js

@@ -0,0 +1,91 @@
+import { getEmbedder } from './embedder.js';
+
+const symbolToText = (symbol, filePath) => {
+  const parts = [];
+  parts.push(symbol.name);
+  if (symbol.parent) parts.push(symbol.parent);
+  if (symbol.kind) parts.push(symbol.kind);
+  if (symbol.signature) parts.push(symbol.signature);
+  if (symbol.snippet) parts.push(symbol.snippet);
+  if (Array.isArray(symbol.decorators)) parts.push(...symbol.decorators);
+  if (filePath) {
+    const segments = filePath.replace(/\\/g, '/').split('/');
+    parts.push(...segments);
+  }
+  return parts.join(' ');
+};
+
+const fileToText = (relPath, fileInfo) => {
+  const parts = [relPath];
+  for (const symbol of fileInfo?.symbols ?? []) {
+    parts.push(symbol.name);
+    if (symbol.signature) parts.push(symbol.signature);
+  }
+  return parts.join(' ');
+};
+
+export const buildIndexCorpusIdf = (index) => {
+  const embedder = getEmbedder();
+  const docs = [];
+  for (const [relPath, fileInfo] of Object.entries(index?.files ?? {})) {
+    docs.push(fileToText(relPath, fileInfo));
+    for (const symbol of fileInfo.symbols ?? []) {
+      docs.push(symbolToText(symbol, relPath));
+    }
+  }
+  return embedder.buildCorpusIdf(docs);
+};
+
+export const embedQuery = (query, options = {}) => {
+  const embedder = getEmbedder();
+  return embedder.embed(query, options);
+};
+
+export const embedFile = (relPath, fileInfo, options = {}) => {
+  const embedder = getEmbedder();
+  return embedder.embed(fileToText(relPath, fileInfo), options);
+};
+
+export const embedSymbol = (symbol, relPath, options = {}) => {
+  const embedder = getEmbedder();
+  return embedder.embed(symbolToText(symbol, relPath), options);
+};
+
+export const semanticRankSymbols = ({ query, index, limit = 10, idf = null }) => {
+  if (!query || !index) return [];
+  const embedder = getEmbedder();
+  const queryVec = embedder.embed(query, { idf });
+  const results = [];
+  for (const [relPath, fileInfo] of Object.entries(index.files ?? {})) {
+    for (const symbol of fileInfo.symbols ?? []) {
+      const vec = embedder.embed(symbolToText(symbol, relPath), { idf });
+      const score = embedder.similarity(queryVec, vec);
+      if (score > 0) {
+        results.push({ score, path: relPath, symbol });
+      }
+    }
+  }
+  results.sort((a, b) => b.score - a.score);
+  return results.slice(0, limit);
+};
+
+export const semanticRankFiles = ({ query, index, limit = 10, idf = null }) => {
+  if (!query || !index) return [];
+  const embedder = getEmbedder();
+  const queryVec = embedder.embed(query, { idf });
+  const results = [];
+  for (const [relPath, fileInfo] of Object.entries(index.files ?? {})) {
+    const vec = embedder.embed(fileToText(relPath, fileInfo), { idf });
+    const score = embedder.similarity(queryVec, vec);
+    if (score > 0) {
+      results.push({ score, path: relPath, symbolCount: fileInfo.symbols?.length ?? 0 });
+    }
+  }
+  results.sort((a, b) => b.score - a.score);
+  return results.slice(0, limit);
+};
+
+export { getEmbedder, setEmbedder, resetEmbedder } from './embedder.js';
+export { tokenize } from './tokenize.js';
+export { embed, cosineSimilarity, buildCorpusIdf, DEFAULT_DIMENSIONS } from './hashing.js';
+export const _internal = { symbolToText, fileToText };

package/src/embeddings/tokenize.js

@@ -0,0 +1,46 @@
+const STOP_WORDS = new Set([
+  'the', 'a', 'an', 'and', 'or', 'but', 'is', 'in', 'on', 'at', 'to', 'for', 'of', 'with',
+  'as', 'by', 'from', 'this', 'that', 'these', 'those', 'it', 'its', 'be', 'been', 'being',
+  'are', 'was', 'were', 'has', 'have', 'had', 'do', 'does', 'did', 'will', 'would', 'should',
+  'can', 'could', 'may', 'might', 'must', 'shall', 'not', 'no', 'yes', 'if', 'else', 'then',
+  'return', 'true', 'false', 'null', 'undefined', 'self', 'this', 'use', 'using', 'used', 'via',
+]);
+
+const CAMEL_RE = /([a-z0-9])([A-Z])/g;
+const SNAKE_RE = /[_-]+/g;
+const NON_WORD_RE = /[^A-Za-z0-9_]+/g;
+const NUMBER_RE = /^\d+$/;
+
+const splitIdentifier = (token) => {
+  if (!token) return [];
+  const camelExpanded = token.replace(CAMEL_RE, '$1 $2');
+  const parts = camelExpanded.replace(SNAKE_RE, ' ').split(/\s+/).filter(Boolean);
+  const out = new Set();
+  out.add(token.toLowerCase());
+  for (const part of parts) {
+    const lower = part.toLowerCase();
+    if (!lower || NUMBER_RE.test(lower)) continue;
+    if (STOP_WORDS.has(lower)) continue;
+    if (lower.length < 2) continue;
+    out.add(lower);
+  }
+  return [...out];
+};
+
+export const tokenize = (text) => {
+  if (!text || typeof text !== 'string') return [];
+  const raw = text.replace(NON_WORD_RE, ' ').split(/\s+/).filter(Boolean);
+  const out = [];
+  for (const token of raw) {
+    if (NUMBER_RE.test(token)) continue;
+    const expanded = splitIdentifier(token);
+    for (const part of expanded) {
+      if (!STOP_WORDS.has(part) && part.length >= 2) {
+        out.push(part);
+      }
+    }
+  }
+  return out;
+};
+
+export const _internal = { STOP_WORDS, splitIdentifier };

package/src/global-memory/scrub.js

@@ -0,0 +1,46 @@
+const SECRET_PATTERNS = [
+  /(api[_-]?key|secret|token|password|passwd|pwd|bearer|authorization)\s*[:=]\s*['"]?([A-Za-z0-9_\-+/=]{8,})['"]?/gi,
+  /(aws_access_key_id|aws_secret_access_key)\s*[:=]\s*['"]?([A-Za-z0-9_\-+/=]{8,})['"]?/gi,
+  /sk-[A-Za-z0-9]{20,}/g,
+  /ghp_[A-Za-z0-9]{20,}/g,
+  /xoxb-[A-Za-z0-9-]{20,}/g,
+  /AIza[A-Za-z0-9_-]{30,}/g,
+  /eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g,
+  /-----BEGIN [A-Z ]+PRIVATE KEY-----[\s\S]+?-----END [A-Z ]+PRIVATE KEY-----/g,
+  /(?:postgres|postgresql|mysql|mongodb|redis):\/\/[^\s'"]+/gi,
+  /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g,
+];
+
+const HOME_PATH_RE = /\/home\/[^/\s'"]+/g;
+const USER_PATH_RE = /\/Users\/[^/\s'"]+/g;
+const WIN_USER_RE = /C:\\Users\\[^\\/\s'"]+/g;
+
+const REDACTION = '[REDACTED]';
+
+export const scrubContent = (content) => {
+  if (typeof content !== 'string') return content;
+  let cleaned = content;
+  for (const pattern of SECRET_PATTERNS) {
+    cleaned = cleaned.replace(pattern, REDACTION);
+  }
+  cleaned = cleaned.replace(HOME_PATH_RE, '~').replace(USER_PATH_RE, '~').replace(WIN_USER_RE, '~');
+  return cleaned;
+};
+
+export const containsLikelySecret = (content) => {
+  if (typeof content !== 'string') return false;
+  return SECRET_PATTERNS.some((re) => {
+    re.lastIndex = 0;
+    return re.test(content);
+  });
+};
+
+export const hashProjectPath = (absolutePath) => {
+  if (!absolutePath || typeof absolutePath !== 'string') return null;
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < absolutePath.length; i += 1) {
+    hash ^= absolutePath.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193) >>> 0;
+  }
+  return `proj-${hash.toString(36)}`;
+};