wayfind 2.0.28 → 2.0.30

package/bin/connectors/github.js

@@ -303,7 +303,28 @@ async function pull(config, since) {
       highlights.push(`${failedCount} CI failure(s)`);
     }
 
-    repoHighlights.push({ repo: repoStr, openPRs, mergedPRs, highlights });
+    repoHighlights.push({
+      repo: repoStr,
+      openPRs,
+      mergedPRs,
+      highlights,
+      topPRs: data.prs.slice(0, 5).map((pr) => ({
+        number: pr.number,
+        title: pr.title,
+        author: pr.user?.login || pr.user?.name || 'unknown',
+        state: pr.merged_at ? 'merged' : pr.state,
+      })),
+      topIssues: data.issues.slice(0, 5).map((iss) => ({
+        number: iss.number,
+        title: iss.title,
+        labels: (iss.labels || []).map((l) => (typeof l === 'string' ? l : l.name)).filter(Boolean),
+        state: iss.state,
+      })),
+      failedRuns: failed.map((r) => ({
+        name: r.name || r.workflow?.name || 'unknown',
+        branch: r.head_branch || '',
+      })),
+    });
   }
 
   // Generate rollup summary
@@ -489,6 +510,21 @@ function generateSummaryMarkdown(
     for (const h of rh.highlights) {
       lines.push(`- ${h}`);
     }
+    if (rh.topPRs && rh.topPRs.length > 0) {
+      const prItems = rh.topPRs.map((pr) => `#${pr.number} "${pr.title}" (${pr.author}, ${pr.state})`);
+      lines.push(`**PRs:** ${prItems.join(' | ')}`);
+    }
+    if (rh.topIssues && rh.topIssues.length > 0) {
+      const issueItems = rh.topIssues.map((iss) => {
+        const labels = iss.labels && iss.labels.length > 0 ? ` [${iss.labels.join(', ')}]` : '';
+        return `#${iss.number} "${iss.title}"${labels} (${iss.state})`;
+      });
+      lines.push(`**Issues:** ${issueItems.join(' | ')}`);
+    }
+    if (rh.failedRuns && rh.failedRuns.length > 0) {
+      const runItems = rh.failedRuns.map((r) => `${r.name}${r.branch ? ' (' + r.branch + ')' : ''}`);
+      lines.push(`**Failed CI:** ${runItems.join(' | ')}`);
+    }
     lines.push('');
   }
 

package/bin/content-store.js

@@ -212,6 +212,22 @@ function generateEntryId(date, repo, title) {
   return crypto.createHash('sha256').update(input).digest('hex').slice(0, 12);
 }
 
+/**
+ * Compute a quality score for an entry (0-3).
+ * +1 if has reasoning (explains WHY)
+ * +1 if has alternatives (what was rejected)
+ * +1 if substantive content (>500 chars)
+ * @param {Object} entry - Entry metadata
+ * @returns {number} 0-3
+ */
+function computeQualityScore(entry) {
+  let score = 0;
+  if (entry.hasReasoning) score++;
+  if (entry.hasAlternatives) score++;
+  if ((entry.contentLength || 0) > 500) score++;
+  return score;
+}
+
 /**
  * Build the text content for embedding from an entry's fields.
  * @param {Object} entry - Entry with date, repo, title, fields
@@ -329,7 +345,7 @@ async function indexJournals(options = {}) {
       const content = buildContent({ ...entry, date, author });
       const hash = contentHash(content);
 
-      newEntries[id] = {
+      const entryMeta = {
         date,
         repo: entry.repo,
         title: entry.title,
@@ -339,8 +355,12 @@ async function indexJournals(options = {}) {
         contentLength: content.length,
         tags: extractTags(entry),
         hasEmbedding: false,
+        hasReasoning: false,
+        hasAlternatives: false,
         _content: content, // temporary, not saved to index
       };
+      entryMeta.qualityScore = computeQualityScore(entryMeta);
+      newEntries[id] = entryMeta;
     }
   }
 
@@ -744,39 +764,50 @@ function getEntryContent(entryId, options = {}) {
   // ── Signal entries ──────────────────────────────────────────────────────
   if (entry.source === 'signal') {
     if (!signalsDir) return null;
-    // entry.repo is like 'signals/github' — extract the channel
-    const channel = (entry.repo || '').replace(/^signals\//, '');
-    if (!channel) return null;
-
-    const channelDir = path.join(signalsDir, channel);
-    if (!fs.existsSync(channelDir)) return null;
-
-    // Find a matching file in the channel directory
-    // Try date-based filename first, then scan for any file containing the title
-    const dateCandidates = [
-      path.join(channelDir, `${entry.date}.md`),
-      path.join(channelDir, `${entry.date}-summary.md`),
-    ];
-    for (const filePath of dateCandidates) {
+    const repo = entry.repo || '';
+
+    // Determine file location based on repo format:
+    // - 'signals/channel' (summary files) → signalsDir/channel/
+    // - 'owner/repo' (per-repo files) → find the channel dir containing owner/repo/
+    let searchDirs = [];
+    if (repo.startsWith('signals/')) {
+      const channel = repo.replace(/^signals\//, '');
+      searchDirs = [path.join(signalsDir, channel)];
+    } else {
+      // Per-repo entry: search all channel dirs for owner/repo subdirectory
       try {
-        return fs.readFileSync(filePath, 'utf8');
-      } catch {
-        // Try next candidate
-      }
+        const channels = fs.readdirSync(signalsDir, { withFileTypes: true })
+          .filter(d => d.isDirectory()).map(d => d.name);
+        for (const ch of channels) {
+          const repoDir = path.join(signalsDir, ch, repo);
+          if (fs.existsSync(repoDir)) {
+            searchDirs.push(repoDir);
+          }
+        }
+      } catch { /* skip */ }
     }
 
-    // Scan channel dir for files matching the date
-    try {
-      const files = fs.readdirSync(channelDir).filter(f => f.endsWith('.md') && f.includes(entry.date));
-      for (const file of files) {
+    for (const dir of searchDirs) {
+      if (!fs.existsSync(dir)) continue;
+      // Try date-based filename first, then summary, then scan
+      const dateCandidates = [
+        path.join(dir, `${entry.date}.md`),
+        path.join(dir, `${entry.date}-summary.md`),
+      ];
+      for (const filePath of dateCandidates) {
         try {
-          return fs.readFileSync(path.join(channelDir, file), 'utf8');
-        } catch {
-          continue;
-        }
+          return fs.readFileSync(filePath, 'utf8');
+        } catch { /* try next */ }
       }
-    } catch {
-      // Channel dir not readable
+      // Scan for files matching the date
+      try {
+        const files = fs.readdirSync(dir).filter(f => f.endsWith('.md') && f.includes(entry.date));
+        for (const file of files) {
+          try {
+            return fs.readFileSync(path.join(dir, file), 'utf8');
+          } catch { continue; }
+        }
+      } catch { /* dir not readable */ }
     }
 
     return null;
@@ -1346,7 +1377,7 @@ async function indexConversations(options = {}) {
 
       const hash = contentHash(content);
 
-      existingIndex.entries[id] = {
+      const convEntry = {
         date,
         repo: transcript.repo,
         title: decision.title,
@@ -1361,6 +1392,8 @@ async function indexConversations(options = {}) {
         hasAlternatives: !!decision.has_alternatives,
         _content: content,
       };
+      convEntry.qualityScore = computeQualityScore(convEntry);
+      existingIndex.entries[id] = convEntry;
 
       if (doEmbeddings) {
         try {
@@ -1653,16 +1686,42 @@ async function indexSignals(options = {}) {
 
   for (const channel of channels) {
     const channelDir = path.join(signalsDir, channel);
-    let files;
+
+    // Collect all .md files: channel root + recursive owner/repo subdirectories
+    const signalFiles = [];
     try {
-      files = fs.readdirSync(channelDir).filter(f => f.endsWith('.md')).sort();
+      const entries = fs.readdirSync(channelDir, { withFileTypes: true });
+      // Channel-root .md files (summaries like YYYY-MM-DD-summary.md)
+      for (const e of entries) {
+        if (e.isFile() && e.name.endsWith('.md')) {
+          signalFiles.push({ filePath: path.join(channelDir, e.name), file: e.name, repo: 'signals/' + channel });
+        }
+      }
+      // Walk owner/repo subdirectories (e.g., github/acme-corp/web-api/)
+      for (const ownerEntry of entries) {
+        if (!ownerEntry.isDirectory()) continue;
+        const ownerDir = path.join(channelDir, ownerEntry.name);
+        let repoEntries;
+        try { repoEntries = fs.readdirSync(ownerDir, { withFileTypes: true }); } catch { continue; }
+        for (const repoEntry of repoEntries) {
+          if (!repoEntry.isDirectory()) continue;
+          const repoDir = path.join(ownerDir, repoEntry.name);
+          const repoStr = `${ownerEntry.name}/${repoEntry.name}`;
+          let repoFiles;
+          try { repoFiles = fs.readdirSync(repoDir).filter(f => f.endsWith('.md')); } catch { continue; }
+          for (const f of repoFiles) {
+            signalFiles.push({ filePath: path.join(repoDir, f), file: f, repo: repoStr });
+          }
+        }
+      }
     } catch {
       continue;
     }
 
-    for (const file of files) {
+    signalFiles.sort((a, b) => a.file.localeCompare(b.file));
+
+    for (const { filePath, file, repo } of signalFiles) {
       stats.fileCount++;
-      const filePath = path.join(channelDir, file);
       let content;
       try {
         content = fs.readFileSync(filePath, 'utf8');
@@ -1670,16 +1729,17 @@ async function indexSignals(options = {}) {
         continue;
       }
 
-      // Extract date from filename (YYYY-MM-DD.md) or fall back to filename
-      const dateMatch = file.match(/^(\d{4}-\d{2}-\d{2})\.md$/);
+      // Extract date from filename (YYYY-MM-DD.md or YYYY-MM-DD-summary.md) or fall back to filename
+      const dateMatch = file.match(/^(\d{4}-\d{2}-\d{2})/);
       const date = dateMatch ? dateMatch[1] : file.replace(/\.md$/, '');
 
       // Extract title from first # heading, or fall back to filename
       const titleMatch = content.match(/^#\s+(.+)$/m);
       const title = titleMatch ? titleMatch[1].trim() : file.replace(/\.md$/, '');
 
-      // Extract tags: channel name + any ## section headings
+      // Extract tags: channel name + repo + any ## section headings
       const tags = [channel];
+      if (repo !== 'signals/' + channel) tags.push(repo);
       const sectionRe = /^##\s+(.+)$/gm;
       let sectionMatch;
       while ((sectionMatch = sectionRe.exec(content)) !== null) {
@@ -1689,7 +1749,6 @@ async function indexSignals(options = {}) {
         }
       }
 
-      const repo = 'signals/' + channel;
       const id = generateEntryId(date, repo, file.replace(/\.md$/, ''));
       const hash = contentHash(content);
 
@@ -1975,6 +2034,32 @@ function computeQualityProfile(options = {}) {
 
 // ── Exports ─────────────────────────────────────────────────────────────────
 
+/**
+ * Deduplicate search results by removing raw entries that have been absorbed
+ * into distilled entries. If a distilled entry exists in the results, its
+ * source entries (listed in distilledFrom) are removed.
+ * @param {Array<{id: string, entry: Object, score?: number}>} results
+ * @returns {Array} Deduplicated results
+ */
+function deduplicateResults(results) {
+  if (!results || results.length === 0) return results;
+
+  // Collect all IDs that have been absorbed into distilled entries
+  const absorbedIds = new Set();
+  for (const r of results) {
+    if (r.entry && r.entry.distilledFrom && Array.isArray(r.entry.distilledFrom)) {
+      for (const id of r.entry.distilledFrom) {
+        absorbedIds.add(id);
+      }
+    }
+  }
+
+  if (absorbedIds.size === 0) return results;
+
+  // Filter out absorbed entries
+  return results.filter(r => !absorbedIds.has(r.id));
+}
+
 module.exports = {
   // Parsing
   parseJournalFile,
@@ -2005,6 +2090,10 @@ module.exports = {
   isRepoExcluded,
   applyFilters,
 
+  // Quality & dedup
+  computeQualityScore,
+  deduplicateResults,
+
   // Core operations
   indexJournals,
   indexSignals,

package/bin/digest.js

@@ -269,13 +269,18 @@ function collectFromStore(sinceDate, options = {}) {
   });
 
   if (entries.length === 0) {
-    return { journals: '', signals: '', entryCount: 0 };
+    return { journals: '', signals: '', entryCount: 0, entryMeta: [] };
   }
 
   const journalParts = [];
   const signalParts = [];
+  const journalMeta = [];
+  const signalMeta = [];
 
   for (const { id, entry } of entries) {
+    // Skip raw entries that have been absorbed into a distilled entry
+    if (entry.distilledFrom) continue;
+
     const content = contentStore.getEntryContent(id, { storePath, journalDir, signalsDir });
     if (!content) continue;
 
@@ -287,10 +292,21 @@ function collectFromStore(sinceDate, options = {}) {
     const meta = author ? `${header}\n${author}\n` : `${header}\n`;
     const formatted = `${meta}\n${content}`;
 
+    const itemMeta = {
+      date: entry.date,
+      source: entry.source,
+      qualityScore: entry.qualityScore || 0,
+      hasReasoning: entry.hasReasoning,
+      hasAlternatives: entry.hasAlternatives,
+      distillTier: entry.distillTier || 'raw',
+    };
+
     if (source === 'signal') {
       signalParts.push(formatted);
+      signalMeta.push(itemMeta);
     } else {
       journalParts.push(formatted);
+      journalMeta.push(itemMeta);
     }
   }
 
@@ -298,6 +314,7 @@ function collectFromStore(sinceDate, options = {}) {
     journals: journalParts.join('\n\n---\n\n'),
     signals: signalParts.join('\n\n---\n\n'),
     entryCount: entries.length,
+    entryMeta: { journal: journalMeta, signal: signalMeta },
   };
 }
 
@@ -618,56 +635,102 @@ function buildPrompt(personaId, signalContent, journalContent, dateRange, contex
 }
 
 /**
- * Apply token budget constraints to signal and journal content.
- * Truncates oldest journal entries first, then signal content.
+ * Apply token budget constraints with quality-weighted packing.
+ * Higher quality entries are kept preferentially over low-quality ones.
  * @param {string} signalContent
  * @param {string} journalContent
  * @param {number} maxChars
- * @returns {{ signals: string, journals: string, truncated: boolean }}
+ * @param {Object} [options] - Optional metadata for quality-weighted packing
+ * @param {Object} [options.entryMeta] - { journal: [{qualityScore, date, ...}], signal: [...] }
+ * @param {Array} [options.scores] - Intelligence scores from Haiku scoring
+ * @param {string} [options.personaId] - Current persona for score lookup
+ * @returns {{ signals: string, journals: string, truncated: boolean, stats: Object }}
  */
-function applyTokenBudget(signalContent, journalContent, maxChars) {
+function applyTokenBudget(signalContent, journalContent, maxChars, options = {}) {
   const total = signalContent.length + journalContent.length;
   if (total <= maxChars) {
-    return { signals: signalContent, journals: journalContent, truncated: false };
+    return { signals: signalContent, journals: journalContent, truncated: false, stats: { dropped: 0 } };
+  }
+
+  const { entryMeta, scores, personaId } = options;
+
+  // Split into sections
+  const signalSections = signalContent ? signalContent.split('\n\n---\n\n') : [];
+  const journalSections = journalContent ? journalContent.split('\n\n---\n\n') : [];
+  const signalMetaArr = (entryMeta && entryMeta.signal) || [];
+  const journalMetaArr = (entryMeta && entryMeta.journal) || [];
+
+  // Score each section with composite priority
+  const todayStr = today();
+  const yesterdayStr = (() => { const d = new Date(); d.setDate(d.getDate() - 1); return d.toISOString().split('T')[0]; })();
+
+  const allSections = [];
+  for (let i = 0; i < signalSections.length; i++) {
+    const meta = signalMetaArr[i] || {};
+    const quality = meta.qualityScore || 0;
+    const recency = (meta.date === todayStr || meta.date === yesterdayStr) ? 1 : 0;
+    const intel = (scores && scores[i] && personaId) ? (scores[i][personaId] || 0) : 0;
+    const distillBonus = (meta.distillTier && meta.distillTier !== 'raw') ? 1 : 0;
+    allSections.push({
+      text: signalSections[i],
+      type: 'signal',
+      priority: quality + recency + intel + distillBonus,
+      len: signalSections[i].length,
+    });
+  }
+  for (let i = 0; i < journalSections.length; i++) {
+    const meta = journalMetaArr[i] || {};
+    const quality = meta.qualityScore || 0;
+    const recency = (meta.date === todayStr || meta.date === yesterdayStr) ? 1 : 0;
+    // Journal score indices start after signal count
+    const scoreIdx = signalSections.length + i;
+    const intel = (scores && scores[scoreIdx] && personaId) ? (scores[scoreIdx][personaId] || 0) : 0;
+    const distillBonus = (meta.distillTier && meta.distillTier !== 'raw') ? 1 : 0;
+    allSections.push({
+      text: journalSections[i],
+      type: 'journal',
+      priority: quality + recency + intel + distillBonus,
+      len: journalSections[i].length,
+    });
   }
 
-  const truncationNote = '\n\n> Note: Input was truncated to fit within token budget. Some older entries may be omitted.\n';
-  const noteLen = truncationNote.length;
-  const available = maxChars - noteLen;
+  // Sort by priority descending (highest quality first)
+  allSections.sort((a, b) => b.priority - a.priority);
 
-  let trimmedJournals = journalContent;
-  let trimmedSignals = signalContent;
-  let journalsTrimmed = false;
-  let signalsTrimmed = false;
+  // Greedily pack into budget
+  const truncationNote = '\n\n> Note: Input was truncated to fit within token budget. Lower-quality entries were dropped.\n';
+  const available = maxChars - truncationNote.length;
+  const keptSignals = [];
+  const keptJournals = [];
+  let used = 0;
+  let dropped = 0;
 
-  // Strategy: drop oldest journal entries first, then trim signals
-  if (trimmedSignals.length + trimmedJournals.length > available) {
-    // Try trimming journals first (keep newest entries)
-    const journalBudget = Math.max(0, available - trimmedSignals.length);
-    if (journalBudget < trimmedJournals.length) {
-      trimmedJournals = trimmedJournals.slice(trimmedJournals.length - journalBudget);
-      journalsTrimmed = true;
+  for (const section of allSections) {
+    const sectionCost = section.len + 7; // account for '\n\n---\n\n' separator
+    if (used + sectionCost <= available) {
+      if (section.type === 'signal') {
+        keptSignals.push(section.text);
+      } else {
+        keptJournals.push(section.text);
+      }
+      used += sectionCost;
+    } else {
+      dropped++;
     }
   }
 
-  if (trimmedSignals.length + trimmedJournals.length > available) {
-    // Still over — trim signal content from the end
-    const signalBudget = Math.max(0, available - trimmedJournals.length);
-    trimmedSignals = trimmedSignals.slice(0, signalBudget);
-    signalsTrimmed = true;
-  }
-
-  // Append truncation note to whichever content was actually trimmed
-  if (signalsTrimmed) {
-    trimmedSignals += truncationNote;
-  } else if (journalsTrimmed) {
-    trimmedJournals += truncationNote;
+  const truncated = dropped > 0;
+  let finalSignals = keptSignals.join('\n\n---\n\n');
+  let finalJournals = keptJournals.join('\n\n---\n\n');
+  if (truncated) {
+    finalJournals += truncationNote;
   }
 
   return {
-    signals: trimmedSignals,
-    journals: trimmedJournals,
-    truncated: true,
+    signals: finalSignals,
+    journals: finalJournals,
+    truncated,
+    stats: { dropped, total: allSections.length, kept: allSections.length - dropped },
   };
 }
 
@@ -748,7 +811,11 @@ async function generateDigest(config, personaIds, sinceDate, onProgress) {
       ({ signals: pSignals, journals: pJournals } =
         intelligence.filterForPersona(signalContent, journalContent, scores, personaId, threshold, allPersonaIds));
     }
-    const budget = applyTokenBudget(pSignals, pJournals, maxInputChars);
+    const budget = applyTokenBudget(pSignals, pJournals, maxInputChars, {
+      entryMeta: storeResult.entryMeta,
+      scores,
+      personaId,
+    });
     pSignals = budget.signals;
     pJournals = budget.journals;
 
@@ -805,7 +872,21 @@ async function generateDigest(config, personaIds, sinceDate, onProgress) {
   fs.writeFileSync(combinedFile, combinedContent, 'utf8');
   files.push(combinedFile);
 
-  return { files, personas: personaIds, dateRange, scores };
+  // Compute input stats for preview mode
+  const entryMeta = storeResult.entryMeta || {};
+  const journalMeta = entryMeta.journal || [];
+  const signalMeta = entryMeta.signal || [];
+  const inputStats = {
+    journalEntries: journalMeta.length,
+    signalEntries: signalMeta.length,
+    qualityDistribution: {
+      rich: journalMeta.filter(m => m.qualityScore >= 2).length,
+      medium: journalMeta.filter(m => m.qualityScore === 1).length,
+      thin: journalMeta.filter(m => m.qualityScore === 0).length,
+    },
+  };
+
+  return { files, personas: personaIds, dateRange, scores, inputStats };
 }
 
 /**