pi-para 0.5.2

package/src/link-discovery.ts

@@ -0,0 +1,170 @@
+/**
+ * Link discovery — finds related wiki pages and adds [[wikilinks]].
+ *
+ * Uses BM25 search to find pages that are semantically related but
+ * not yet linked. Runs as a periodic daemon job or on-demand.
+ *
+ * Strategy:
+ * 1. For each page, search the wiki using its title + key terms
+ * 2. Filter results to pages not already linked
+ * 3. If a related page scores above threshold, add it to ## Connections
+ * 4. Sync frontmatter.links
+ */
+
+import {
+  listPages,
+  readPage,
+  writePage,
+} from "./wiki.js";
+import type { WikiPage } from "./wiki.js";
+import { searchWiki } from "./store.js";
+import type { QMDStore } from "./store.js";
+import { extractWikilinks, syncFrontmatterLinks } from "./link-utils.js";
+
+// -- Types -------------------------------------------------------------------
+
+export interface LinkDiscoveryResult {
+  pagesUpdated: number;
+  linksAdded: number;
+  discoveries: Array<{
+    page: string;
+    newLinks: string[];
+  }>;
+}
+
+export interface LinkDiscoveryOptions {
+  /** Minimum BM25 score to consider a page related (0-1). Default 0.3 */
+  minScore?: number;
+  /** Max new links to add per page per run. Default 5 */
+  maxLinksPerPage?: number;
+  /** Dry run — report but don't write. Default false */
+  dryRun?: boolean;
+}
+
+// -- Discovery ---------------------------------------------------------------
+
+/**
+ * Discover and add missing [[wikilinks]] between related pages.
+ *
+ * For each page:
+ * 1. Search wiki using the page's title as query
+ * 2. Find high-scoring pages not already linked
+ * 3. Add them to ## Connections section
+ * 4. Sync frontmatter.links
+ */
+export async function discoverLinks(
+  wikiDir: string,
+  store: QMDStore,
+  options?: LinkDiscoveryOptions,
+): Promise<LinkDiscoveryResult> {
+  const minScore = options?.minScore ?? 0.3;
+  const maxLinksPerPage = options?.maxLinksPerPage ?? 5;
+  const dryRun = options?.dryRun ?? false;
+
+  const refs = await listPages(wikiDir);
+  const pages: WikiPage[] = [];
+  for (const ref of refs) {
+    const page = await readPage(wikiDir, ref.category, ref.slug);
+    if (page) pages.push(page);
+  }
+
+  const allSlugs = new Set(pages.map(p => p.slug));
+  const result: LinkDiscoveryResult = {
+    pagesUpdated: 0,
+    linksAdded: 0,
+    discoveries: [],
+  };
+
+  for (const page of pages) {
+    // Get existing outgoing links
+    const existingLinks = new Set(extractWikilinks(page.body));
+    // Also include frontmatter links
+    for (const l of page.frontmatter.links) existingLinks.add(l);
+
+    // Search for related pages using title
+    const query = page.frontmatter.title;
+    let related;
+    try {
+      related = await searchWiki(store, query, { limit: 10 });
+    } catch {
+      continue; // search failure — skip this page
+    }
+
+    // Filter: not self, not already linked, above score threshold
+    const candidates = related
+      .filter(r =>
+        r.page.slug !== page.slug &&
+        !existingLinks.has(r.page.slug) &&
+        r.score >= minScore
+      )
+      .slice(0, maxLinksPerPage);
+
+    if (candidates.length === 0) continue;
+
+    const newLinks = candidates.map(c => c.page.slug);
+
+    // Build ## Connections addition
+    const connectionsLines = newLinks.map(slug => {
+      const targetPage = pages.find(p => p.slug === slug);
+      const title = targetPage?.frontmatter.title ?? slug;
+      return `- [[${slug}]] — ${title}`;
+    });
+
+    // Update page body
+    let newBody = page.body;
+    const hasConnections = /^## Connections/m.test(newBody);
+
+    if (hasConnections) {
+      // Append to existing Connections section
+      // Find the end of the Connections section (next ## heading or EOF)
+      const connectionsIdx = newBody.indexOf("## Connections");
+      const afterConnections = newBody.indexOf("\n## ", connectionsIdx + 1);
+      const insertPos = afterConnections === -1 ? newBody.length : afterConnections;
+
+      // Insert before the next section
+      const insertion = "\n" + connectionsLines.join("\n");
+      newBody = newBody.slice(0, insertPos) + insertion + newBody.slice(insertPos);
+    } else {
+      // Add new Connections section at the end (before ## Sources or ## Open Questions if they exist)
+      const sourcesIdx = newBody.indexOf("\n## Sources");
+      const questionsIdx = newBody.indexOf("\n## Open Questions");
+      const insertBefore = Math.min(
+        sourcesIdx === -1 ? Infinity : sourcesIdx,
+        questionsIdx === -1 ? Infinity : questionsIdx,
+      );
+
+      const section = "\n\n## Connections\n" + connectionsLines.join("\n");
+
+      if (insertBefore === Infinity) {
+        newBody = newBody.trimEnd() + section + "\n";
+      } else {
+        newBody = newBody.slice(0, insertBefore) + section + newBody.slice(insertBefore);
+      }
+    }
+
+    // Sync frontmatter.links
+    const updatedLinks = syncFrontmatterLinks(newBody);
+
+    if (!dryRun) {
+      const updatedPage: WikiPage = {
+        ...page,
+        body: newBody,
+        frontmatter: {
+          ...page.frontmatter,
+          links: updatedLinks,
+          updated: new Date().toISOString(),
+        },
+      };
+      await writePage(wikiDir, updatedPage);
+    }
+
+    result.pagesUpdated++;
+    result.linksAdded += newLinks.length;
+    result.discoveries.push({
+      page: `${page.category}/${page.slug}`,
+      newLinks,
+    });
+  }
+
+  return result;
+}

package/src/link-utils.ts

@@ -0,0 +1,169 @@
+/**
+ * Shared wikilink utilities — extraction, auto-linking, and sync.
+ *
+ * Centralizes the [[wikilink]] logic that was duplicated across
+ * tools.ts, lint.ts, processor.ts, and webui/server.ts.
+ */
+
+// -- Wikilink extraction ----------------------------------------------------
+
+const WIKILINK_RE = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
+
+/** Extract all [[wikilink]] targets from markdown body. Returns deduplicated slugs. */
+export function extractWikilinks(body: string): string[] {
+  const links: string[] = [];
+  let m: RegExpExecArray | null;
+  const re = new RegExp(WIKILINK_RE.source, WIKILINK_RE.flags);
+  while ((m = re.exec(body)) !== null) {
+    links.push(m[1].trim());
+  }
+  return [...new Set(links)];
+}
+
+/** Remove [[target]] (and optional display text) from body text. */
+export function removeWikilink(body: string, target: string): string {
+  const escaped = target.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const re = new RegExp(`\\[\\[${escaped}(?:\\|[^\\]]+)?\\]\\]`, "g");
+  return body.replace(re, "");
+}
+
+// -- Auto-linking -----------------------------------------------------------
+
+/**
+ * Identify regions in markdown body that should NOT be auto-linked.
+ * Returns array of [start, end] ranges to skip.
+ *
+ * Protected regions:
+ * - Fenced code blocks (``` ... ```)
+ * - Inline code (`...`)
+ * - Existing [[wikilinks]]
+ * - URLs (http:// https://)
+ * - YAML frontmatter lines referencing slugs
+ */
+function findProtectedRanges(body: string): Array<[number, number]> {
+  const ranges: Array<[number, number]> = [];
+
+  // Fenced code blocks
+  const fencedRe = /```[\s\S]*?```/g;
+  let m: RegExpExecArray | null;
+  while ((m = fencedRe.exec(body)) !== null) {
+    ranges.push([m.index, m.index + m[0].length]);
+  }
+
+  // Inline code
+  const inlineCodeRe = /`[^`]+`/g;
+  while ((m = inlineCodeRe.exec(body)) !== null) {
+    ranges.push([m.index, m.index + m[0].length]);
+  }
+
+  // Existing [[wikilinks]]
+  const wlRe = /\[\[[^\]]+\]\]/g;
+  while ((m = wlRe.exec(body)) !== null) {
+    ranges.push([m.index, m.index + m[0].length]);
+  }
+
+  // URLs
+  const urlRe = /https?:\/\/[^\s)>]+/g;
+  while ((m = urlRe.exec(body)) !== null) {
+    ranges.push([m.index, m.index + m[0].length]);
+  }
+
+  // Markdown headings (don't wrap slugs in heading lines)
+  const headingRe = /^#{1,6}\s+.*$/gm;
+  while ((m = headingRe.exec(body)) !== null) {
+    ranges.push([m.index, m.index + m[0].length]);
+  }
+
+  return ranges.sort((a, b) => a[0] - b[0]);
+}
+
+/**
+ * Check if a position falls inside any protected range.
+ */
+function isProtected(pos: number, end: number, ranges: Array<[number, number]>): boolean {
+  for (const [rStart, rEnd] of ranges) {
+    if (rStart > end) break; // ranges are sorted
+    if (pos >= rStart && end <= rEnd) return true;
+    if (pos < rEnd && end > rStart) return true; // overlap
+  }
+  return false;
+}
+
+/**
+ * Auto-link page slug mentions in body text with [[wikilinks]].
+ *
+ * - Wraps unlinked slug mentions in [[slug]]
+ * - Skips code blocks, URLs, existing [[wikilinks]], headings
+ * - Matches longest slugs first to avoid partial matches
+ * - Uses word-boundary matching (won't match "pi-para" inside "pi-para-daemon")
+ * - Only links each slug once per page (first occurrence) to avoid noise
+ *
+ * @param body - Page body markdown
+ * @param allSlugs - Set of all existing page slugs
+ * @param ownSlug - This page's own slug (excluded from auto-linking)
+ * @returns Updated body with [[wikilinks]] inserted
+ */
+export function autoLinkSlugs(
+  body: string,
+  allSlugs: Set<string>,
+  ownSlug?: string,
+): string {
+  if (allSlugs.size === 0) return body;
+
+  // Sort slugs longest-first so "pi-para-daemon" is matched before "pi-para"
+  const sortedSlugs = [...allSlugs]
+    .filter(s => s !== ownSlug)
+    .sort((a, b) => b.length - a.length);
+
+  const protectedRanges = findProtectedRanges(body);
+  const linkedSlugs = new Set(extractWikilinks(body));
+
+  let result = body;
+
+  for (const slug of sortedSlugs) {
+    // Skip if already linked
+    if (linkedSlugs.has(slug)) continue;
+
+    // Build regex: match slug at word boundaries
+    // Slug chars are [a-z0-9-], so word boundary = not preceded/followed by these chars
+    const escaped = slug.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    // Use negative lookbehind/lookahead for slug characters
+    const re = new RegExp(`(?<![a-z0-9-])${escaped}(?![a-z0-9-])`, "g");
+
+    // Re-compute protected ranges for current state of result
+    const currentProtected = findProtectedRanges(result);
+
+    let match: RegExpExecArray | null;
+    let linked = false;
+
+    // Reset regex
+    re.lastIndex = 0;
+    while ((match = re.exec(result)) !== null) {
+      if (linked) break; // only link first occurrence
+
+      const pos = match.index;
+      const end = pos + match[0].length;
+
+      if (isProtected(pos, end, currentProtected)) continue;
+
+      // Replace this occurrence with [[slug]]
+      const before = result.slice(0, pos);
+      const after = result.slice(end);
+      const replacement = `[[${slug}]]`;
+      result = before + replacement + after;
+      linked = true;
+
+      // Update regex index to account for replacement length difference
+      re.lastIndex = pos + replacement.length;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Sync frontmatter.links to match all [[wikilinks]] found in body text.
+ */
+export function syncFrontmatterLinks(body: string): string[] {
+  return extractWikilinks(body);
+}