wayfind 2.0.34 → 2.0.36

package/bin/connectors/notion.js

@@ -249,11 +249,23 @@ async function configure() {
     .map((d) => d.trim())
     .filter(Boolean);
 
+  // Optional: page IDs for full content extraction
+  console.log('');
+  console.log('Optional: specific page IDs to extract full content from (comma-separated).');
+  console.log('These pages will have their body text included in signals, not just metadata.');
+  console.log('Find page IDs in the URL: notion.so/<workspace>/<page-id>');
+  const pageInput = await ask('Page IDs: ');
+  const pages = pageInput
+    .split(',')
+    .map((p) => p.trim().replace(/-/g, ''))
+    .filter(Boolean);
+
   const channelConfig = {
     transport: 'https',
     token,
     token_env: 'NOTION_TOKEN',
     databases: databases.length > 0 ? databases : null,
+    pages: pages.length > 0 ? pages : null,
     last_pull: null,
   };
 
@@ -261,8 +273,12 @@ async function configure() {
   console.log('Notion connector configured.');
   if (databases.length > 0) {
     console.log(`Monitoring ${databases.length} database(s).`);
-  } else {
-    console.log('Monitoring all shared pages.');
+  }
+  if (pages.length > 0) {
+    console.log(`Extracting content from ${pages.length} page(s).`);
+  }
+  if (databases.length === 0 && pages.length === 0) {
+    console.log('Monitoring all shared pages (metadata only).');
   }
   console.log('');
 
@@ -298,6 +314,37 @@ async function pull(config, since) {
     dbEntries.push(...entries.map((e) => ({ ...e, _databaseId: dbId })));
   }
 
+  // Fetch content for targeted pages
+  const targetedPageIds = config.pages || [];
+  const pageContents = {};
+  if (targetedPageIds.length > 0) {
+    for (const pageId of targetedPageIds) {
+      try {
+        const content = await fetchPageContent(token, pageId);
+        if (content && content.trim()) {
+          pageContents[pageId] = content;
+        }
+      } catch {
+        // Skip pages that fail — may have been deleted or unshared
+      }
+    }
+    // Also fetch targeted pages that aren't in the recent pages list
+    const recentPageIds = new Set(pages.map((p) => p.id.replace(/-/g, '')));
+    for (const pageId of targetedPageIds) {
+      if (!recentPageIds.has(pageId.replace(/-/g, ''))) {
+        try {
+          const endpoint = `/pages/${pageId}`;
+          const page = await notionGet(token, endpoint);
+          if (page && page.id) {
+            pages.push(page);
+          }
+        } catch {
+          // Skip — page may not exist
+        }
+      }
+    }
+  }
+
   // Fetch comment counts for active pages (top 20 by recency)
   const activePages = pages.slice(0, 20);
   const commentCounts = {};
@@ -314,6 +361,7 @@ async function pull(config, since) {
 
   // Analyze
   const analysis = analyzeActivity(pages, dbEntries, commentCounts, sinceDate, todayDate, userMap);
+  analysis.pageContents = pageContents;
 
   // Generate markdown
   const md = generateMarkdown(analysis, sinceDate, todayDate, timestamp, userMap);
@@ -496,6 +544,86 @@ async function fetchComments(token, pageId) {
   }
 }
 
+// ── Page content extraction ────────────────────────────────────────────────
+
+async function fetchPageContent(token, pageId, maxChars = 5000) {
+  const blocks = [];
+  let cursor = undefined;
+  const MAX_REQUESTS = 5;
+  let requests = 0;
+
+  while (requests < MAX_REQUESTS) {
+    requests++;
+    const endpoint = `/blocks/${pageId}/children?page_size=100` + (cursor ? `&start_cursor=${cursor}` : '');
+    let response;
+    try {
+      response = await notionGet(token, endpoint);
+    } catch {
+      break;
+    }
+    const results = Array.isArray(response.results) ? response.results : [];
+    blocks.push(...results);
+    if (!response.has_more) break;
+    cursor = response.next_cursor;
+  }
+
+  // Convert blocks to markdown
+  const lines = [];
+  let totalChars = 0;
+
+  for (const block of blocks) {
+    if (totalChars >= maxChars) break;
+    const line = blockToMarkdown(block);
+    if (line !== null) {
+      lines.push(line);
+      totalChars += line.length;
+    }
+  }
+
+  return lines.join('\n');
+}
+
+function blockToMarkdown(block) {
+  const type = block.type;
+  if (!type) return null;
+
+  const richTextToPlain = (rt) =>
+    Array.isArray(rt) ? rt.map((t) => t.plain_text || '').join('') : '';
+
+  const data = block[type];
+  if (!data) return null;
+
+  switch (type) {
+    case 'paragraph':
+      return richTextToPlain(data.rich_text);
+    case 'heading_1':
+      return '# ' + richTextToPlain(data.rich_text);
+    case 'heading_2':
+      return '## ' + richTextToPlain(data.rich_text);
+    case 'heading_3':
+      return '### ' + richTextToPlain(data.rich_text);
+    case 'bulleted_list_item':
+      return '- ' + richTextToPlain(data.rich_text);
+    case 'numbered_list_item':
+      return '1. ' + richTextToPlain(data.rich_text);
+    case 'to_do':
+      return (data.checked ? '- [x] ' : '- [ ] ') + richTextToPlain(data.rich_text);
+    case 'toggle':
+      return '> ' + richTextToPlain(data.rich_text);
+    case 'callout':
+      return '> ' + richTextToPlain(data.rich_text);
+    case 'quote':
+      return '> ' + richTextToPlain(data.rich_text);
+    case 'code':
+      return '```\n' + richTextToPlain(data.rich_text) + '\n```';
+    case 'divider':
+      return '---';
+    default:
+      // Skip unsupported block types (image, embed, file, etc.)
+      return null;
+  }
+}
+
 // ── Property extraction ─────────────────────────────────────────────────────
 
 function extractTitle(page) {
@@ -700,6 +828,22 @@ function generateMarkdown(analysis, sinceDate, todayDate, timestamp, userMap) {
     lines.push('');
   }
 
+  // Targeted page content
+  const pageContents = analysis.pageContents || {};
+  if (Object.keys(pageContents).length > 0) {
+    lines.push('## Page Content');
+    lines.push('');
+    for (const [pageId, content] of Object.entries(pageContents)) {
+      // Find the page title from the pages list
+      const page = analysis.pages.find((p) => p.id.replace(/-/g, '') === pageId.replace(/-/g, ''));
+      const title = page ? extractTitle(page) : `Page ${pageId.slice(0, 8)}`;
+      lines.push(`### ${sanitizeForMarkdown(title)}`);
+      lines.push('');
+      lines.push(content);
+      lines.push('');
+    }
+  }
+
   // Summary
   lines.push('## Summary');
   lines.push('');

package/bin/team-context.js

@@ -4322,17 +4322,17 @@ function ensureContainerConfig() {
     changed = true;
   }
 
-  // Backfill container-specific paths into existing digest config (fixes configs
-  // created before store_path/journal_dir/signals_dir were added)
+  // Override container-specific paths in digest config — the mounted connectors.json
+  // may have host paths that don't exist inside the container
   if (config.digest) {
-    const defaults = {
+    const containerPaths = {
       store_path: process.env.TEAM_CONTEXT_STORE_PATH || contentStore.DEFAULT_STORE_PATH,
       journal_dir: process.env.TEAM_CONTEXT_JOURNALS_DIR || '/data/journals',
       signals_dir: process.env.TEAM_CONTEXT_SIGNALS_DIR || contentStore.DEFAULT_SIGNALS_DIR,
       team_context_dir: process.env.TEAM_CONTEXT_TEAM_CONTEXT_DIR || '',
     };
-    for (const [key, val] of Object.entries(defaults)) {
-      if (!config.digest[key]) {
+    for (const [key, val] of Object.entries(containerPaths)) {
+      if (config.digest[key] !== val) {
         config.digest[key] = val;
         changed = true;
       }
@@ -4356,14 +4356,14 @@ function ensureContainerConfig() {
     changed = true;
   }
 
-  // Backfill container-specific paths into existing bot config
+  // Override container-specific paths in bot config (same reason as digest above)
   if (config.slack_bot) {
-    const botDefaults = {
+    const botPaths = {
       store_path: process.env.TEAM_CONTEXT_STORE_PATH || contentStore.DEFAULT_STORE_PATH,
       journal_dir: process.env.TEAM_CONTEXT_JOURNALS_DIR || '/data/journals',
     };
-    for (const [key, val] of Object.entries(botDefaults)) {
-      if (!config.slack_bot[key]) {
+    for (const [key, val] of Object.entries(botPaths)) {
+      if (config.slack_bot[key] !== val) {
         config.slack_bot[key] = val;
         changed = true;
       }
@@ -4414,11 +4414,13 @@ function ensureContainerConfig() {
   // Notion connector
   if (!config.notion && process.env.NOTION_TOKEN) {
     const databases = process.env.TEAM_CONTEXT_NOTION_DATABASES;
+    const pages = process.env.TEAM_CONTEXT_NOTION_PAGES;
     config.notion = {
       transport: 'https',
       token: process.env.NOTION_TOKEN,
       token_env: 'NOTION_TOKEN',
       databases: databases ? databases.split(',').map((d) => d.trim()) : null,
+      pages: pages ? pages.split(',').map((p) => p.trim().replace(/-/g, '')) : null,
       last_pull: null,
     };
     changed = true;

package/doctor.sh

@@ -466,6 +466,52 @@ check_storage_backend() {
     else
         [ "$VERBOSE" = true ] && info "No connectors.json — signal freshness check skipped"
     fi
+
+    # ── 4. Embedding coverage ───────────────────────────────────────────────
+    local EMBED_RESULT
+    EMBED_RESULT=$(node -e "
+      try {
+        const storage = require('$SCRIPT_DIR/bin/storage/index.js');
+        const storePath = '$WAYFIND_DIR/content-store';
+        const backend = storage.getBackend(storePath);
+        const idx = backend.loadIndex();
+        if (!idx || !idx.entries) { console.log('SKIP'); process.exit(0); }
+        let total = 0, embedded = 0;
+        for (const e of Object.values(idx.entries)) { total++; if (e.hasEmbedding) embedded++; }
+        if (total === 0) { console.log('SKIP'); process.exit(0); }
+        const pct = Math.round(100 * embedded / total);
+        const hasKey = !!(process.env.OPENAI_API_KEY || process.env.AZURE_OPENAI_EMBEDDING_ENDPOINT);
+        console.log(pct + ':' + embedded + ':' + total + ':' + hasKey);
+      } catch (e) {
+        console.log('SKIP:' + e.message.split('\n')[0]);
+      }
+    " 2>/dev/null) || EMBED_RESULT="SKIP"
+
+    if [[ "$EMBED_RESULT" != SKIP* ]]; then
+        IFS=':' read -r PCT EMBEDDED TOTAL HAS_KEY <<< "$EMBED_RESULT"
+        if [ "$PCT" -ge 90 ]; then
+            ok "Embeddings: $EMBEDDED/$TOTAL entries ($PCT%)"
+        elif [ "$PCT" -ge 50 ]; then
+            warn "Embeddings: only $EMBEDDED/$TOTAL entries ($PCT%) — search quality degraded"
+            info "Run: wayfind reindex  (with OPENAI_API_KEY or AZURE_OPENAI_EMBEDDING_* set)"
+            ISSUES=$((ISSUES + 1))
+        elif [ "$PCT" -gt 0 ]; then
+            err "Embeddings: $EMBEDDED/$TOTAL entries ($PCT%) — semantic search mostly broken"
+            info "Run: wayfind reindex  (with OPENAI_API_KEY or AZURE_OPENAI_EMBEDDING_* set)"
+            ISSUES=$((ISSUES + 1))
+        else
+            if [ "$HAS_KEY" = "true" ]; then
+                err "Embeddings: 0/$TOTAL entries — embedding API key is set but no embeddings generated"
+                info "Run: wayfind reindex  to generate embeddings"
+                ISSUES=$((ISSUES + 1))
+            else
+                warn "Embeddings: 0/$TOTAL entries — no embedding API key configured"
+                info "Set OPENAI_API_KEY or AZURE_OPENAI_EMBEDDING_* then run: wayfind reindex"
+                ISSUES=$((ISSUES + 1))
+            fi
+        fi
+    fi
+
     set -e  # Restore errexit
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wayfind",
-  "version": "2.0.34",
+  "version": "2.0.36",
   "description": "Team decision trail for AI-assisted development. The connective tissue between product, engineering, and strategy.",
   "bin": {
     "wayfind": "./bin/team-context.js"