wayfind 2.0.61 → 2.0.63

package/README.md

@@ -180,13 +180,11 @@ Auto-registered during `wayfind init`. When a team container is running, the loc
 
 | Variable | Description |
 |----------|-------------|
-| `OPENAI_API_KEY` | Semantic search embeddings (full-text works without it) |
+| `OPENAI_API_KEY` | Upgrade semantic search to OpenAI embeddings (Xenova local model is used by default — no key needed) |
 | `TEAM_CONTEXT_LLM_MODEL` | LLM for digests (default: `claude-sonnet-4-5-20250929`) |
 | `TEAM_CONTEXT_DIGEST_SCHEDULE` | Cron schedule (default: `0 8 * * 1` — Monday 8am) |
 | `TEAM_CONTEXT_EXCLUDE_REPOS` | Repos to exclude from digests |
 | `TEAM_CONTEXT_TELEMETRY` | `true` for anonymous usage telemetry |
-| `TEAM_CONTEXT_NO_SLACK` | Run container without Slack integration (set to `1`) |
-| `TEAM_CONTEXT_KEY_ROTATE_SCHEDULE` | API key rotation cron (default: `0 2 * * *`) |
 
 ---
 

package/bin/content-store.js

@@ -421,6 +421,28 @@ function cosineSimilarity(a, b) {
  * @param {boolean} [options.embeddings] - Generate embeddings (default: true if OPENAI_API_KEY or TEAM_CONTEXT_SIMULATE)
  * @returns {Promise<Object>} - Stats: { entryCount, newEntries, updatedEntries, skippedEntries, removedEntries }
  */
+
+/**
+ * Embed a batch of items concurrently.
+ * @param {Array<{id: string, content: string}>} items
+ * @param {number} [concurrency=20]
+ * @returns {Promise<Map<string, Array|null>>} Map from id to vector (or null on failure)
+ */
+async function batchEmbed(items, concurrency = 20) {
+  const results = new Map();
+  for (let i = 0; i < items.length; i += concurrency) {
+    const chunk = items.slice(i, i + concurrency);
+    await Promise.all(chunk.map(async ({ id, content }) => {
+      try {
+        results.set(id, await llm.generateEmbedding(content));
+      } catch {
+        results.set(id, null);
+      }
+    }));
+  }
+  return results;
+}
+
 async function indexJournals(options = {}) {
   const journalDir = options.journalDir || DEFAULT_JOURNAL_DIR;
   const storePath = options.storePath || resolveStorePath();
@@ -497,10 +519,11 @@ async function indexJournals(options = {}) {
     }
   }
 
-  // Compute diffs
+  // Compute diffs — collect embedding work for batching
   const stats = { entryCount: 0, newEntries: 0, updatedEntries: 0, skippedEntries: 0, removedEntries: 0 };
   const finalEntries = {};
   const finalEmbeddings = { ...existingEmbeddings };
+  const pendingEmbeds = []; // { id, content, reason: 'update'|'changed'|'new' }
 
   for (const [id, entry] of Object.entries(newEntries)) {
     const existing = existingEntries[id];
@@ -511,46 +534,38 @@ async function indexJournals(options = {}) {
       // Unchanged — but generate embedding if missing and embeddings are enabled
       entry.hasEmbedding = existing.hasEmbedding;
       if (doEmbeddings && !existing.hasEmbedding && content) {
-        try {
-          const vec = await llm.generateEmbedding(content);
-          finalEmbeddings[id] = vec;
-          entry.hasEmbedding = true;
-          stats.updatedEntries++;
-        } catch (err) {
-          stats.skippedEntries++;
-        }
+        pendingEmbeds.push({ id, content, reason: 'update' });
       } else {
         stats.skippedEntries++;
       }
       finalEntries[id] = entry;
     } else if (existing) {
       // Changed — re-embed
-      if (doEmbeddings) {
-        try {
-          const vec = await llm.generateEmbedding(content);
-          finalEmbeddings[id] = vec;
-          entry.hasEmbedding = true;
-        } catch (err) {
-          // Keep going without embedding for this entry
-          entry.hasEmbedding = false;
-          delete finalEmbeddings[id];
-        }
-      }
       finalEntries[id] = entry;
       stats.updatedEntries++;
+      if (doEmbeddings) pendingEmbeds.push({ id, content, reason: 'changed' });
     } else {
       // New entry
-      if (doEmbeddings) {
-        try {
-          const vec = await llm.generateEmbedding(content);
-          finalEmbeddings[id] = vec;
-          entry.hasEmbedding = true;
-        } catch (err) {
-          entry.hasEmbedding = false;
-        }
-      }
       finalEntries[id] = entry;
       stats.newEntries++;
+      if (doEmbeddings) pendingEmbeds.push({ id, content, reason: 'new' });
+    }
+  }
+
+  // Batch embed all pending entries concurrently
+  if (pendingEmbeds.length > 0) {
+    const embedResults = await batchEmbed(pendingEmbeds);
+    for (const { id, reason } of pendingEmbeds) {
+      const vec = embedResults.get(id);
+      if (vec) {
+        finalEmbeddings[id] = vec;
+        finalEntries[id].hasEmbedding = true;
+        if (reason === 'update') stats.updatedEntries++;
+      } else {
+        if (reason === 'update') stats.skippedEntries++;
+        else if (reason === 'changed') { finalEntries[id].hasEmbedding = false; delete finalEmbeddings[id]; }
+        // 'new': hasEmbedding stays false
+      }
     }
   }
 
@@ -1582,18 +1597,6 @@ async function indexConversations(options = {}) {
       };
       convEntry.qualityScore = computeQualityScore(convEntry);
       existingIndex.entries[id] = convEntry;
-
-      if (doEmbeddings) {
-        try {
-          const vec = await llm.generateEmbedding(content);
-          existingEmbeddings[id] = vec;
-          existingIndex.entries[id].hasEmbedding = true;
-        } catch {
-          // Continue without embedding
-        }
-      }
-
-      delete existingIndex.entries[id]._content;
       entryIds.push(id);
       stats.decisionsExtracted++;
     }
@@ -1608,6 +1611,29 @@ async function indexConversations(options = {}) {
     stats.transcriptsProcessed++;
   }
 
+  // Batch embed all new conversation entries
+  if (doEmbeddings) {
+    const pendingEmbeds = Object.entries(existingIndex.entries)
+      .filter(([, e]) => e._content && !e.hasEmbedding)
+      .map(([id, e]) => ({ id, content: e._content }));
+
+    if (pendingEmbeds.length > 0) {
+      const embedResults = await batchEmbed(pendingEmbeds);
+      for (const { id } of pendingEmbeds) {
+        const vec = embedResults.get(id);
+        if (vec) {
+          existingEmbeddings[id] = vec;
+          existingIndex.entries[id].hasEmbedding = true;
+        }
+      }
+    }
+  }
+
+  // Strip temp _content fields
+  for (const entry of Object.values(existingIndex.entries)) {
+    delete entry._content;
+  }
+
   // Save everything
   existingIndex.entryCount = Object.keys(existingIndex.entries).length;
   if (doEmbeddings) {
@@ -1876,6 +1902,7 @@ async function indexSignals(options = {}) {
   const existingEmbeddings = doEmbeddings ? (_sigModelChanged ? {} : backend.loadEmbeddings()) : {};
 
   const stats = { fileCount: 0, newEntries: 0, updatedEntries: 0, skippedEntries: 0 };
+  const signalPendingEmbeds = []; // { id, content, reason: 'update'|'changed'|'new'|'chunk-update'|'chunk-new' }
 
   // Scan channel directories
   let channels;
@@ -1960,14 +1987,7 @@ async function indexSignals(options = {}) {
       if (existing && existing.contentHash === hash) {
         // Unchanged — but generate embedding if missing
         if (doEmbeddings && !existing.hasEmbedding && content) {
-          try {
-            const vec = await llm.generateEmbedding(content);
-            existingEmbeddings[id] = vec;
-            existing.hasEmbedding = true;
-            stats.updatedEntries++;
-          } catch {
-            stats.skippedEntries++;
-          }
+          signalPendingEmbeds.push({ id, content, reason: 'update' });
         } else {
           stats.skippedEntries++;
         }
@@ -1985,16 +2005,7 @@ async function indexSignals(options = {}) {
           tags,
           hasEmbedding: false,
         };
-
-        if (doEmbeddings) {
-          try {
-            const vec = await llm.generateEmbedding(content);
-            existingEmbeddings[id] = vec;
-            existingIndex.entries[id].hasEmbedding = true;
-          } catch {
-            delete existingEmbeddings[id];
-          }
-        }
+        if (doEmbeddings) signalPendingEmbeds.push({ id, content, reason: 'changed' });
         stats.updatedEntries++;
       } else {
         // New entry
@@ -2010,16 +2021,7 @@ async function indexSignals(options = {}) {
           tags,
           hasEmbedding: false,
         };
-
-        if (doEmbeddings) {
-          try {
-            const vec = await llm.generateEmbedding(content);
-            existingEmbeddings[id] = vec;
-            existingIndex.entries[id].hasEmbedding = true;
-          } catch {
-            // Continue without embedding
-          }
-        }
+        if (doEmbeddings) signalPendingEmbeds.push({ id, content, reason: 'new' });
         stats.newEntries++;
       }
     }
@@ -2095,13 +2097,7 @@ async function indexSignals(options = {}) {
 
       if (existingChunk && existingChunk.contentHash === chunkHash) {
         if (doEmbeddings && !existingChunk.hasEmbedding) {
-          try {
-            const vec = await llm.generateEmbedding(section);
-            existingEmbeddings[chunkId] = vec;
-            existingChunk.hasEmbedding = true;
-          } catch {
-            // Skip
-          }
+          signalPendingEmbeds.push({ id: chunkId, content: section, reason: 'chunk-update' });
         }
         continue;
       }
@@ -2122,13 +2118,24 @@ async function indexSignals(options = {}) {
       };
 
       if (doEmbeddings) {
-        try {
-          const vec = await llm.generateEmbedding(section);
-          existingEmbeddings[chunkId] = vec;
-          existingIndex.entries[chunkId].hasEmbedding = true;
-        } catch {
-          // Continue without embedding
-        }
+        signalPendingEmbeds.push({ id: chunkId, content: section, reason: 'chunk-new' });
+      }
+    }
+  }
+
+  // Batch embed all pending signal entries
+  if (signalPendingEmbeds.length > 0) {
+    const embedResults = await batchEmbed(signalPendingEmbeds);
+    for (const { id, reason } of signalPendingEmbeds) {
+      const vec = embedResults.get(id);
+      if (vec) {
+        existingEmbeddings[id] = vec;
+        existingIndex.entries[id].hasEmbedding = true;
+        if (reason === 'update') stats.updatedEntries++;
+      } else {
+        if (reason === 'update') stats.skippedEntries++;
+        else if (reason === 'changed') { existingIndex.entries[id].hasEmbedding = false; delete existingEmbeddings[id]; }
+        // 'new', 'chunk-update', 'chunk-new': hasEmbedding stays false
       }
     }
   }

package/bin/storage/sqlite-backend.js

@@ -203,11 +203,13 @@ class SqliteBackend {
     for (const row of rows) {
       entries[row.id] = rowToEntry(row);
     }
+    const embeddingModelRow = this.db.prepare("SELECT value FROM metadata WHERE key = 'embedding_model'").get();
     return {
       version: INDEX_VERSION,
       lastUpdated: Date.now(),
       entryCount: rows.length,
       entries,
+      ...(embeddingModelRow ? { embedding_model: embeddingModelRow.value } : {}),
     };
   }
 
@@ -228,6 +230,9 @@ class SqliteBackend {
       for (const [id, entry] of Object.entries(entries)) {
         stmt.run(entryToRow(id, entry));
       }
+      if (index.embedding_model) {
+        this.db.prepare('INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)').run('embedding_model', index.embedding_model);
+      }
     });
     txn();
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wayfind",
-  "version": "2.0.61",
+  "version": "2.0.63",
   "description": "Team decision trail for AI-assisted development. The connective tissue between product, engineering, and strategy.",
   "bin": {
     "wayfind": "./bin/team-context.js",

package/plugin/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "wayfind",
-  "version": "2.0.61",
+  "version": "2.0.63",
   "description": "Team decision trail for AI-assisted development. Session memory, decision journals, and team digests.",
   "author": {
     "name": "Wayfind",

package/setup.sh

@@ -441,6 +441,41 @@ PYEOF
             fi
         fi
 
+        # MCP server — register wayfind-mcp in settings.json
+        MCP_REGISTERED=false
+        grep -q '"wayfind"' "$SETTINGS" 2>/dev/null && MCP_REGISTERED=true
+        if [ "$MCP_REGISTERED" = false ]; then
+            if [ "$DRY_RUN" = false ]; then
+                TMP_SETTINGS="$(mktemp)"
+                if python3 - "$SETTINGS" "$TMP_SETTINGS" <<'PYEOF' 2>/dev/null; then
+import json, sys
+settings_path, out_path = sys.argv[1], sys.argv[2]
+try:
+    with open(settings_path) as f:
+        settings = json.load(f)
+except (json.JSONDecodeError, IOError):
+    sys.exit(1)
+mcp = settings.setdefault("mcpServers", {})
+if "wayfind" not in mcp:
+    mcp["wayfind"] = {"command": "wayfind-mcp"}
+with open(out_path, "w") as f:
+    json.dump(settings, f, indent=2)
+    f.write("\n")
+PYEOF
+                    mv "$TMP_SETTINGS" "$SETTINGS"
+                    log "Registered wayfind MCP server in ~/.claude/settings.json"
+                else
+                    rm -f "$TMP_SETTINGS"
+                    warn "Could not register MCP server — add manually to ~/.claude/settings.json:"
+                    warn '  "mcpServers": { "wayfind": { "command": "wayfind-mcp" } }'
+                fi
+            else
+                info "[dry-run] Would register wayfind MCP server in $SETTINGS"
+            fi
+        else
+            info "MCP server already registered in settings.json — skipped"
+        fi
+
         ;;
 
     cursor)
@@ -470,6 +505,48 @@ PYEOF
                 warn "--repo path not found or not a directory: $REPO_DIR"
             fi
         fi
+
+        # MCP server — register wayfind-mcp in ~/.cursor/mcp.json
+        CURSOR_MCP="$HOME/.cursor/mcp.json"
+        run mkdir -p "$HOME/.cursor"
+        CURSOR_MCP_REGISTERED=false
+        grep -q '"wayfind"' "$CURSOR_MCP" 2>/dev/null && CURSOR_MCP_REGISTERED=true
+        if [ "$CURSOR_MCP_REGISTERED" = false ]; then
+            if [ "$DRY_RUN" = false ]; then
+                if [ ! -f "$CURSOR_MCP" ]; then
+                    printf '{\n  "mcpServers": {\n    "wayfind": {\n      "command": "wayfind-mcp",\n      "args": []\n    }\n  }\n}\n' > "$CURSOR_MCP"
+                    log "Created ~/.cursor/mcp.json with wayfind MCP server"
+                else
+                    TMP_MCP="$(mktemp)"
+                    if python3 - "$CURSOR_MCP" "$TMP_MCP" <<'PYEOF' 2>/dev/null; then
+import json, sys
+mcp_path, out_path = sys.argv[1], sys.argv[2]
+try:
+    with open(mcp_path) as f:
+        config = json.load(f)
+except (json.JSONDecodeError, IOError):
+    config = {}
+mcp = config.setdefault("mcpServers", {})
+if "wayfind" not in mcp:
+    mcp["wayfind"] = {"command": "wayfind-mcp", "args": []}
+with open(out_path, "w") as f:
+    json.dump(config, f, indent=2)
+    f.write("\n")
+PYEOF
+                        mv "$TMP_MCP" "$CURSOR_MCP"
+                        log "Registered wayfind MCP server in ~/.cursor/mcp.json"
+                    else
+                        rm -f "$TMP_MCP"
+                        warn "Could not register MCP server — add manually to ~/.cursor/mcp.json:"
+                        warn '  "mcpServers": { "wayfind": { "command": "wayfind-mcp", "args": [] } }'
+                    fi
+                fi
+            else
+                info "[dry-run] Would register wayfind MCP server in ~/.cursor/mcp.json"
+            fi
+        else
+            info "MCP server already registered in ~/.cursor/mcp.json — skipped"
+        fi
         ;;
 
     generic)