wayfind 2.0.49 → 2.0.51

package/bin/connectors/llm.js

@@ -356,6 +356,59 @@ async function detect() {
 
 const OPENAI_EMBEDDINGS_URL = 'https://api.openai.com/v1/embeddings';
 const DEFAULT_EMBEDDING_MODEL = 'text-embedding-3-small';
+const LOCAL_EMBEDDING_MODEL = 'Xenova/all-MiniLM-L6-v2';
+
+// Cached pipeline instance — expensive to initialize, reuse across calls.
+let _localPipeline = null;
+
+/**
+ * Try to generate an embedding using the local ONNX model (@xenova/transformers).
+ * Returns null if the package is not installed or the model fails to load.
+ * Downloads the model (~80MB) on first use into the transformers cache.
+ * @param {string} text
+ * @returns {Promise<number[]|null>}
+ */
+async function generateEmbeddingLocal(text) {
+  try {
+    if (!_localPipeline) {
+      // Dynamic require — optional dep, may not be installed
+      const { pipeline, env } = require('@xenova/transformers');
+      // Suppress progress output in non-interactive contexts
+      if (!process.stdout.isTTY) {
+        env.allowLocalModels = false;
+      }
+      process.stderr.write('[wayfind] Loading local embedding model (first use — may take a moment)...\n');
+      _localPipeline = await pipeline('feature-extraction', LOCAL_EMBEDDING_MODEL);
+    }
+    const output = await _localPipeline(text, { pooling: 'mean', normalize: true });
+    return Array.from(output.data);
+  } catch (_) {
+    return null;
+  }
+}
+
+/**
+ * Detect which embedding provider is active based on env vars and installed packages.
+ * Returns an object describing the provider so callers can surface this to users.
+ * @returns {{ provider: string, model: string, requiresKey: boolean, available: boolean }}
+ */
+function getEmbeddingProviderInfo() {
+  if (isSimulation()) {
+    return { provider: 'simulation', model: 'fake-1536d', requiresKey: false, available: true };
+  }
+  if (process.env.AZURE_OPENAI_EMBEDDING_ENDPOINT) {
+    const hasKey = !!process.env.AZURE_OPENAI_EMBEDDING_KEY;
+    return { provider: 'azure', model: process.env.AZURE_OPENAI_EMBEDDING_DEPLOYMENT || 'text-embedding-3-small', requiresKey: true, available: hasKey };
+  }
+  if (process.env.OPENAI_API_KEY) {
+    return { provider: 'openai', model: DEFAULT_EMBEDDING_MODEL, requiresKey: true, available: true };
+  }
+  try {
+    require.resolve('@xenova/transformers');
+    return { provider: 'local', model: LOCAL_EMBEDDING_MODEL, requiresKey: false, available: true };
+  } catch (_) {}
+  return { provider: 'none', model: null, requiresKey: false, available: false };
+}
 
 /**
  * Generate an embedding vector for the given text.
@@ -392,7 +445,14 @@ async function generateEmbedding(text, options = {}) {
   const apiKeyEnv = options.apiKeyEnv || 'OPENAI_API_KEY';
   const apiKey = process.env[apiKeyEnv];
   if (!apiKey) {
-    throw new Error(`Embeddings: Missing API key. Set ${apiKeyEnv} environment variable.`);
+    // No cloud key — try local model before failing
+    const localVec = await generateEmbeddingLocal(text);
+    if (localVec !== null) return localVec;
+    throw new Error(
+      'Embeddings: No provider configured.\n' +
+      '  Option 1 (cloud): set OPENAI_API_KEY or AZURE_OPENAI_EMBEDDING_ENDPOINT\n' +
+      '  Option 2 (local, no key): npm install -g @xenova/transformers'
+    );
   }
 
   const baseUrl = options.baseUrl || OPENAI_EMBEDDINGS_URL.replace('/embeddings', '');
@@ -477,5 +537,6 @@ module.exports = {
   call,
   detect,
   generateEmbedding,
+  getEmbeddingProviderInfo,
   isSimulation,
 };

package/bin/content-store.js

@@ -426,7 +426,7 @@ async function indexJournals(options = {}) {
   const storePath = options.storePath || resolveStorePath();
   const doEmbeddings = options.embeddings !== undefined
     ? options.embeddings
-    : !!(process.env.OPENAI_API_KEY || process.env.AZURE_OPENAI_EMBEDDING_ENDPOINT || llm.isSimulation());
+    : !!(process.env.OPENAI_API_KEY || process.env.AZURE_OPENAI_EMBEDDING_ENDPOINT || llm.isSimulation() || llm.getEmbeddingProviderInfo().available);
 
   if (!journalDir || !storePath) {
     throw new Error('Journal directory and store path are required.');
@@ -440,7 +440,26 @@ async function indexJournals(options = {}) {
   const backend = getBackend(storePath);
   const existingIndex = backend.loadIndex();
   const existingEntries = existingIndex ? existingIndex.entries : {};
-  const existingEmbeddings = doEmbeddings ? backend.loadEmbeddings() : {};
+
+  // Detect embedding model mismatch — if the stored model differs from the current
+  // provider, treat all entries as needing re-embedding so the index stays consistent.
+  // A silent auto-migration is better than silently returning garbage similarity scores.
+  const currentProviderInfo = llm.getEmbeddingProviderInfo();
+  const storedEmbeddingModel = existingIndex ? existingIndex.embedding_model : null;
+  const modelChanged = doEmbeddings && storedEmbeddingModel && currentProviderInfo.model &&
+    storedEmbeddingModel !== currentProviderInfo.model;
+  if (modelChanged) {
+    process.stderr.write(
+      `[wayfind] Embedding model changed: ${storedEmbeddingModel} → ${currentProviderInfo.model}\n` +
+      `[wayfind] Re-embedding all entries for consistent search results...\n`
+    );
+    // Mark all existing entries as needing re-embedding
+    for (const entry of Object.values(existingEntries)) {
+      entry.hasEmbedding = false;
+    }
+  }
+
+  const existingEmbeddings = doEmbeddings ? (modelChanged ? {} : backend.loadEmbeddings()) : {};
 
   // Parse all journal files
   const files = fs.readdirSync(journalDir).filter(f => DATE_FILE_RE.test(f)).sort();
@@ -546,11 +565,13 @@ async function indexJournals(options = {}) {
   stats.entryCount = Object.keys(finalEntries).length;
 
   // Save
+  const embeddingModel = doEmbeddings ? llm.getEmbeddingProviderInfo().model : null;
   const index = {
     version: INDEX_VERSION,
     lastUpdated: Date.now(),
     entryCount: stats.entryCount,
     entries: finalEntries,
+    ...(embeddingModel ? { embedding_model: embeddingModel } : {}),
   };
 
   backend.saveIndex(index);
@@ -563,6 +584,7 @@ async function indexJournals(options = {}) {
     entry_count: stats.entryCount,
     new_entries: stats.newEntries,
     has_embeddings: doEmbeddings,
+    embedding_model: embeddingModel,
   });
 
   return stats;
@@ -1375,7 +1397,7 @@ async function indexConversations(options = {}) {
   const storePath = options.storePath || resolveStorePath();
   const doEmbeddings = options.embeddings !== undefined
     ? options.embeddings
-    : !!(process.env.OPENAI_API_KEY || process.env.AZURE_OPENAI_EMBEDDING_ENDPOINT || llm.isSimulation());
+    : !!(process.env.OPENAI_API_KEY || process.env.AZURE_OPENAI_EMBEDDING_ENDPOINT || llm.isSimulation() || llm.getEmbeddingProviderInfo().available);
 
   if (!projectsDir || !storePath) {
     throw new Error('Projects directory and store path are required.');
@@ -1397,9 +1419,24 @@ async function indexConversations(options = {}) {
   // Load existing indexes
   const backend = getBackend(storePath);
   const existingIndex = backend.loadIndex() || { version: INDEX_VERSION, entries: {}, lastUpdated: Date.now(), entryCount: 0 };
-  const existingEmbeddings = doEmbeddings ? backend.loadEmbeddings() : {};
   const convIndex = backend.loadConversationIndex();
 
+  // Auto-migrate embeddings if model changed
+  const _convProviderInfo = llm.getEmbeddingProviderInfo();
+  const _convStoredModel = existingIndex.embedding_model || null;
+  const _convModelChanged = doEmbeddings && _convStoredModel && _convProviderInfo.model &&
+    _convStoredModel !== _convProviderInfo.model;
+  if (_convModelChanged) {
+    process.stderr.write(
+      `[wayfind] Embedding model changed: ${_convStoredModel} → ${_convProviderInfo.model}\n` +
+      `[wayfind] Re-embedding conversation entries...\n`
+    );
+    for (const entry of Object.values(existingIndex.entries)) {
+      entry.hasEmbedding = false;
+    }
+  }
+  const existingEmbeddings = doEmbeddings ? (_convModelChanged ? {} : backend.loadEmbeddings()) : {};
+
   // Compute since cutoff
   let sinceCutoff = 0;
   if (options.since) {
@@ -1566,6 +1603,10 @@ async function indexConversations(options = {}) {
 
   // Save everything
   existingIndex.entryCount = Object.keys(existingIndex.entries).length;
+  if (doEmbeddings) {
+    const model = llm.getEmbeddingProviderInfo().model;
+    if (model) existingIndex.embedding_model = model;
+  }
   backend.saveIndex(existingIndex);
   if (doEmbeddings) {
     backend.saveEmbeddings(existingEmbeddings);
@@ -1801,7 +1842,7 @@ async function indexSignals(options = {}) {
   const storePath = options.storePath || resolveStorePath();
   const doEmbeddings = options.embeddings !== undefined
     ? options.embeddings
-    : !!(process.env.OPENAI_API_KEY || process.env.AZURE_OPENAI_EMBEDDING_ENDPOINT || llm.isSimulation());
+    : !!(process.env.OPENAI_API_KEY || process.env.AZURE_OPENAI_EMBEDDING_ENDPOINT || llm.isSimulation() || llm.getEmbeddingProviderInfo().available);
 
   if (!signalsDir || !storePath) {
     throw new Error('Signals directory and store path are required.');
@@ -1814,7 +1855,18 @@ async function indexSignals(options = {}) {
   // Load existing index (contains journal + conversation entries too)
   const backend = getBackend(storePath);
   const existingIndex = backend.loadIndex() || { version: INDEX_VERSION, entries: {}, lastUpdated: Date.now(), entryCount: 0 };
-  const existingEmbeddings = doEmbeddings ? backend.loadEmbeddings() : {};
+
+  // Auto-migrate embeddings if model changed
+  const _sigProviderInfo = llm.getEmbeddingProviderInfo();
+  const _sigStoredModel = existingIndex.embedding_model || null;
+  const _sigModelChanged = doEmbeddings && _sigStoredModel && _sigProviderInfo.model &&
+    _sigStoredModel !== _sigProviderInfo.model;
+  if (_sigModelChanged) {
+    for (const entry of Object.values(existingIndex.entries)) {
+      entry.hasEmbedding = false;
+    }
+  }
+  const existingEmbeddings = doEmbeddings ? (_sigModelChanged ? {} : backend.loadEmbeddings()) : {};
 
   const stats = { fileCount: 0, newEntries: 0, updatedEntries: 0, skippedEntries: 0 };
 
@@ -2076,6 +2128,10 @@ async function indexSignals(options = {}) {
 
   // Save
   existingIndex.entryCount = Object.keys(existingIndex.entries).length;
+  if (doEmbeddings) {
+    const model = llm.getEmbeddingProviderInfo().model;
+    if (model) existingIndex.embedding_model = model;
+  }
   backend.saveIndex(existingIndex);
   if (doEmbeddings) {
     backend.saveEmbeddings(existingEmbeddings);
@@ -2376,6 +2432,12 @@ module.exports = {
   saveIndex: (storePath, index) => getBackend(storePath || resolveStorePath()).saveIndex(index),
   loadEmbeddings: (storePath) => getBackend(storePath || resolveStorePath()).loadEmbeddings(),
   saveEmbeddings: (storePath, embeddings) => getBackend(storePath || resolveStorePath()).saveEmbeddings(embeddings),
+  getStoredEmbeddingModel: (storePath) => {
+    try {
+      const idx = getBackend(storePath || resolveStorePath()).loadIndex();
+      return idx ? (idx.embedding_model || null) : null;
+    } catch { return null; }
+  },
   loadConversationIndex: (storePath) => getBackend(storePath || resolveStorePath()).loadConversationIndex(),
   saveConversationIndex: (storePath, convIndex) => getBackend(storePath || resolveStorePath()).saveConversationIndex(convIndex),
 

package/bin/team-context.js

@@ -38,6 +38,7 @@ const digest = require('./digest');
 const slack = require('./slack');
 const slackBot = require('./slack-bot');
 const contentStore = require('./content-store');
+const llm = require('./connectors/llm');
 const rebuildStatus = require('./rebuild-status');
 const telemetry = require('./telemetry');
 
@@ -1310,7 +1311,20 @@ async function runReindex(args) {
   const force = args.includes('--force');
 
   if (force) {
-    console.log('Force mode: clearing content store for full reindex...');
+    // Warn if stored embeddings used a different model than the current provider
+    const storedModel = contentStore.getStoredEmbeddingModel();
+    const currentProvider = llm.getEmbeddingProviderInfo();
+    if (storedModel && currentProvider.model && storedModel !== currentProvider.model) {
+      console.log(`⚠️  Embedding model mismatch:`);
+      console.log(`   Stored embeddings: ${storedModel}`);
+      console.log(`   Current provider:  ${currentProvider.model} (${currentProvider.provider})`);
+      console.log(`   All embeddings will be cleared and regenerated with the current provider.`);
+      console.log(`   Entries without embeddings will fall back to full-text search until reindexed.`);
+    } else if (storedModel) {
+      console.log(`Force mode: clearing content store — will regenerate ${storedModel} embeddings...`);
+    } else {
+      console.log('Force mode: clearing content store for full reindex...');
+    }
     try {
       const backend = contentStore.getBackend();
       const emptyIndex = { version: contentStore.INDEX_VERSION, entries: {}, lastUpdated: Date.now(), entryCount: 0 };
@@ -3581,7 +3595,7 @@ function contextSync() {
  *
  * @param {string[]} args - CLI arguments (--quiet suppresses output)
  */
-function contextPull(args) {
+async function contextPull(args) {
   const quiet = args.includes('--quiet');
   const background = args.includes('--background');
   const log = quiet ? () => {} : console.log;
@@ -3614,6 +3628,16 @@ function contextPull(args) {
     log('[wayfind] Pulled latest team-context');
     // Mark success — doctor checks this to warn on prolonged failures
     try { fs.writeFileSync(markerFile, new Date().toISOString()); } catch {}
+    // Index any new team journals into the local content store
+    const journalsDir = path.join(teamPath, 'journals');
+    if (fs.existsSync(journalsDir)) {
+      try {
+        const stats = await contentStore.indexJournals({ journalDir: journalsDir });
+        if (!quiet && stats.newEntries > 0) {
+          log(`[wayfind] Indexed ${stats.newEntries} new team journal entries`);
+        }
+      } catch (_) {}
+    }
   } else if (result.error && result.error.code === 'ETIMEDOUT') {
     log('[wayfind] Team-context pull timed out — using local state');
   } else {
@@ -5404,12 +5428,48 @@ async function runContainerDoctor() {
         console.log('       Run: wayfind reindex');
         issues++;
       }
+
+      // Embedding provider + model mismatch check
+      try {
+        const providerInfo = llm.getEmbeddingProviderInfo();
+        const storedModel = contentStore.getStoredEmbeddingModel(storePath);
+        if (!providerInfo.available) {
+          warn('Embedding provider: none configured — semantic search unavailable (full-text only)');
+          console.log('       Option 1 (cloud): set OPENAI_API_KEY');
+          console.log('       Option 2 (local, no key): npm install -g @xenova/transformers');
+          issues++;
+        } else {
+          const label = providerInfo.provider === 'local'
+            ? `local (${providerInfo.model}) — no API key required`
+            : `${providerInfo.provider} (${providerInfo.model})`;
+          pass(`Embedding provider: ${label}`);
+          if (storedModel && providerInfo.model && storedModel !== providerInfo.model) {
+            warn(`Embedding model mismatch: stored=${storedModel}, current=${providerInfo.model}`);
+            console.log('       Semantic search results may be degraded — stored embeddings are from a different model.');
+            console.log('       Fix: wayfind reindex --force  (clears and regenerates all embeddings)');
+            issues++;
+          }
+        }
+      } catch (_) {}
     } catch (e) {
       warn(`Embedding coverage: error — ${e.message}`);
       issues++;
     }
   }
 
+  // ── Embedding provider (standalone, when no entries yet) ───────────────────
+  if (entryCount === 0) {
+    try {
+      const providerInfo = llm.getEmbeddingProviderInfo();
+      if (!providerInfo.available) {
+        warn('Embedding provider: none — semantic search will not be available');
+        console.log('       Option 1 (cloud): set OPENAI_API_KEY');
+        console.log('       Option 2 (local, no key): npm install -g @xenova/transformers');
+        issues++;
+      }
+    } catch (_) {}
+  }
+
   // 4. Signal freshness — are there signal files from today?
   const signalsDir = path.join(EFFECTIVE_DIR, 'signals');
   const today = new Date().toISOString().slice(0, 10); // YYYY-MM-DD

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wayfind",
-  "version": "2.0.49",
+  "version": "2.0.51",
   "description": "Team decision trail for AI-assisted development. The connective tissue between product, engineering, and strategy.",
   "bin": {
     "wayfind": "./bin/team-context.js",
@@ -54,6 +54,7 @@
     "posthog-node": "^5.28.0"
   },
   "optionalDependencies": {
-    "better-sqlite3": "^11.0.0"
+    "better-sqlite3": "^11.0.0",
+    "@xenova/transformers": "^2.17.2"
   }
 }

package/plugin/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "wayfind",
-  "version": "2.0.48",
+  "version": "2.0.49",
   "description": "Team decision trail for AI-assisted development. Session memory, decision journals, and team digests.",
   "author": {
     "name": "Wayfind",
@@ -9,6 +9,14 @@
   "homepage": "https://github.com/usewayfind/wayfind",
   "repository": "https://github.com/usewayfind/wayfind",
   "license": "Apache-2.0",
-  "keywords": ["team", "context", "memory", "decisions", "digest", "journal", "session"],
+  "keywords": [
+    "team",
+    "context",
+    "memory",
+    "decisions",
+    "digest",
+    "journal",
+    "session"
+  ],
   "skills": "./skills/"
 }

package/plugin/skills/init-memory/SKILL.md

@@ -45,6 +45,53 @@ Check if `.claude/wayfind.json` already exists in the repo.
 
 **Verify `.gitignore` coverage:** `.claude/wayfind.json` must be gitignored. Step 3 already includes it in the required entries — confirm this is still the case. If someone removed it, Step 3 will restore it.
 
+## Step 1.7: Embedding provider (first-time only)
+
+Read `~/.claude/team-context/context.json`. Check for an `embedding_provider` field.
+
+**If `embedding_provider` is already set:** Skip this step silently.
+
+**If not set:** Present the following choice to the user:
+
+```
+Wayfind uses embeddings for semantic search (e.g. "find the auth refactor discussion").
+
+Choose your embedding provider:
+
+  1. Local model (recommended for getting started)
+     - No API key needed
+     - ~80MB download on first use, cached after that
+     - Works offline
+     - Good quality for most queries
+
+  2. OpenAI (higher quality)
+     - Requires OPENAI_API_KEY
+     - ~$0/month at normal usage
+     - Best retrieval quality
+
+  3. Azure OpenAI
+     - Requires AZURE_OPENAI_EMBEDDING_ENDPOINT + key
+     - For enterprise deployments
+
+⚠️  Switching providers later requires reindexing your content store.
+    Run: wayfind reindex --force
+    Embeddings are model-specific — mixing models breaks semantic search.
+
+Which provider? [1/2/3, default: 1]
+```
+
+Wait for their answer (default to 1 if they press enter). Write their choice to `~/.claude/team-context/context.json` as:
+
+```json
+{ "embedding_provider": "local" }   // for choice 1
+{ "embedding_provider": "openai" }  // for choice 2
+{ "embedding_provider": "azure" }   // for choice 3
+```
+
+(Merge into existing context.json — do not overwrite other fields.)
+
+Report: "Embedding provider set to: <name>"
+
 ## Step 2: Create state files (if missing)
 
 This repo uses TWO state files with different visibility:

package/plugin/skills/init-team/SKILL.md

@@ -45,6 +45,38 @@ Before starting, verify:
 
 If any prerequisite fails, tell the user what's needed and stop.
 
+## Step 0.5: Embedding Provider
+
+Read `~/.claude/team-context/context.json`. Check for an `embedding_provider` field.
+
+**If already set:** Report the current provider and skip this step.
+
+**If not set:** Present this choice:
+
+```
+Wayfind uses embeddings for semantic search across your team's decision trail.
+
+Choose your embedding provider:
+
+  1. Local model (recommended for getting started)
+     - No API key needed, works offline
+     - ~80MB download on first use, cached after
+     - Good quality for most queries
+
+  2. OpenAI (higher quality)
+     - Requires OPENAI_API_KEY (~$0/month at normal usage)
+
+  3. Azure OpenAI
+     - Requires AZURE_OPENAI_EMBEDDING_ENDPOINT + key
+
+⚠️  Switching providers later requires reindexing: wayfind reindex --force
+    Embeddings are model-specific — changing models after indexing breaks semantic search.
+
+Which provider? [1/2/3, default: 1]
+```
+
+Write choice to `~/.claude/team-context/context.json` as `embedding_provider: "local"|"openai"|"azure"`.
+
 ## Step 1: Team Context Repo
 
 This repo holds shared journals, strategy state, digest archives, and the GitHub