pi-harness-runtime 1.1.326 → 1.1.328

package/packages/wiki/src/index.ts

@@ -5,7 +5,7 @@
  */
 
 import { existsSync, readFileSync } from "node:fs";
-import { join, resolve, relative } from "node:path";
+import { join, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
 import {
   WikiStorage,
@@ -13,7 +13,6 @@ import {
   computeHash,
   now,
   parseFrontmatter,
-  combinePage,
 } from "./storage.js";
 import type {
   WikiConfig,
@@ -43,12 +42,16 @@ export function isWikiEnabled(): boolean {
 }
 
 /**
- * Require wiki to be enabled
+ * Require wiki to be enabled and ensure storage is initialized
  */
-function requireWiki(): void {
+function requireWiki(storage: WikiStorage): void {
   if (!isWikiEnabled()) {
     throw new Error("Wiki feature is disabled. Set HARNESS_KNOWLEDGE_WIKI=1 to enable.");
   }
+  // Lazy initialization: ensure storage is ready on first use
+  if (!storage.isInitialized()) {
+    storage.initialize();
+  }
 }
 
 // ============================================================================
@@ -82,7 +85,9 @@ export class WikiManager {
    * Initialize wiki structure
    */
   initialize(): WikiInitResult {
-    requireWiki();
+    // Initialize storage before checking readiness
+    this.storage.initialize();
+    requireWiki(this.storage);
     return this.storage.initialize();
   }
 
@@ -99,12 +104,13 @@ export class WikiManager {
 
   /**
    * Ingest a source file and create snapshot
+   * Returns existing snapshot if same requestId was already processed
    */
   ingestSource(
     sourcePath: string,
     requestId?: string
-  ): { snapshot: SourceSnapshot; operation: WikiOperation } {
-    requireWiki();
+  ): { snapshot: SourceSnapshot; operation: WikiOperation; existing: boolean } {
+    requireWiki(this.storage);
 
     // Validate source path
     const resolvedPath = resolve(sourcePath);
@@ -112,6 +118,30 @@ export class WikiManager {
       throw new Error(`Source file not found: ${sourcePath}`);
     }
 
+    // Check for duplicate request
+    if (requestId) {
+      const operations = this.storage.getRecentOperations(1000);
+      const existingOp = operations.find(
+        (op) => op.requestId === requestId && op.status === "success"
+      );
+      if (existingOp) {
+        const details = existingOp.details as { snapshotId?: string; sourcePath?: string };
+        // Check if the source path matches (same requestId for different source = conflict)
+        if (details.sourcePath && details.sourcePath !== resolvedPath) {
+          throw new Error(
+            `Request ID ${requestId} was already used for a different source path. ` +
+            `Original: ${details.sourcePath}, Requested: ${sourcePath}`
+          );
+        }
+        if (details.snapshotId) {
+          const snapshot = this.storage.getSnapshot(details.snapshotId);
+          if (snapshot) {
+            return { snapshot, operation: existingOp, existing: true };
+          }
+        }
+      }
+    }
+
     // Read content
     const content = readFileSync(resolvedPath, "utf8");
 
@@ -122,10 +152,13 @@ export class WikiManager {
       // Create snapshot
       const snapshot = this.storage.createSnapshot(resolvedPath, content, operation.id);
 
+      // Update operation with snapshot ID for future deduplication
+      this.storage.updateOperationDetails(operation.id, { snapshotId: snapshot.id });
+
       // Complete operation
       this.storage.completeOperation(operation.id, "success");
 
-      return { snapshot, operation };
+      return { snapshot, operation, existing: false };
     } catch (err) {
       const error = err instanceof Error ? err.message : String(err);
       this.storage.completeOperation(operation.id, "failed", error);
@@ -137,7 +170,7 @@ export class WikiManager {
    * Get source snapshot
    */
   getSnapshot(snapshotId: string): SourceSnapshot | null {
-    requireWiki();
+    requireWiki(this.storage);
     return this.storage.getSnapshot(snapshotId);
   }
 
@@ -156,7 +189,7 @@ export class WikiManager {
     relatedPages?: string[];
     codeVersion?: string;
   }): WikiPage {
-    requireWiki();
+    requireWiki(this.storage);
 
     const id = `page-${randomUUID().slice(0, 8)}`;
     const timestamp = now();
@@ -199,7 +232,7 @@ export class WikiManager {
       verification: WikiPageFrontmatter["verification"];
     }>
   ): WikiPage {
-    requireWiki();
+    requireWiki(this.storage);
 
     const existing = this.storage.loadPage(pageId);
     if (!existing) {
@@ -227,7 +260,7 @@ export class WikiManager {
    * Load a wiki page
    */
   loadPage(pageId: string): WikiPage | null {
-    requireWiki();
+    requireWiki(this.storage);
     return this.storage.loadPage(pageId);
   }
 
@@ -235,7 +268,7 @@ export class WikiManager {
    * Delete a wiki page
    */
   deletePage(pageId: string): void {
-    requireWiki();
+    requireWiki(this.storage);
     this.storage.deletePage(pageId);
   }
 
@@ -243,7 +276,7 @@ export class WikiManager {
    * Get all pages
    */
   getAllPages(): WikiPage[] {
-    requireWiki();
+    requireWiki(this.storage);
     return this.storage.getAllPages();
   }
 
@@ -255,7 +288,7 @@ export class WikiManager {
    * Search wiki pages
    */
   search(query: string, filters?: WikiSearchFilters, limit = 10): WikiSearchResult[] {
-    requireWiki();
+    requireWiki(this.storage);
     return this.storage.search(query, filters, limit);
   }
 
@@ -263,7 +296,7 @@ export class WikiManager {
    * Get page index
    */
   getIndex() {
-    requireWiki();
+    requireWiki(this.storage);
     return this.storage.getIndex();
   }
 
@@ -275,7 +308,7 @@ export class WikiManager {
    * Stage a change for later application
    */
   stageChange(pageId: string, expectedRevision: string, newContent: string): StagedChange {
-    requireWiki();
+    requireWiki(this.storage);
 
     const existing = this.storage.loadPage(pageId);
     if (!existing) {
@@ -297,7 +330,7 @@ export class WikiManager {
    * Get staged change for a page
    */
   getStagedChange(pageId: string): StagedChange | undefined {
-    requireWiki();
+    requireWiki(this.storage);
     return this.storage.getStagedChange(pageId);
   }
 
@@ -305,7 +338,7 @@ export class WikiManager {
    * Clear staged change
    */
   clearStagedChange(pageId: string): void {
-    requireWiki();
+    requireWiki(this.storage);
     this.storage.clearStagedChange(pageId);
   }
 
@@ -316,8 +349,8 @@ export class WikiManager {
   /**
    * Apply staged change with conflict detection
    */
-  applyChange(pageId: string, requestId?: string): WikiPage {
-    requireWiki();
+  applyChange(pageId: string, _requestId?: string): WikiPage {
+    requireWiki(this.storage);
 
     const staged = this.storage.getStagedChange(pageId);
     if (!staged) {
@@ -329,8 +362,10 @@ export class WikiManager {
       throw new Error(`Page not found: ${pageId}`);
     }
 
-    // Check for conflicts
-    const currentRevision = computeHash(existing.rawFrontmatter || "");
+    // Check for conflicts - hash entire page content to detect any changes
+    const currentRevision = computeHash(
+      (existing.rawFrontmatter || "") + "\n" + existing.body
+    );
     if (staged.expectedRevision !== currentRevision) {
       throw new Error(
         `Conflict: page was modified. Expected revision ${staged.expectedRevision}, found ${currentRevision}`
@@ -368,7 +403,7 @@ export class WikiManager {
    * Run lint checks on wiki
    */
   lint(scope?: "all" | "pages" | "links" | "freshness"): LintFinding[] {
-    requireWiki();
+    requireWiki(this.storage);
 
     const findings: LintFinding[] = [];
 
@@ -543,7 +578,7 @@ export class WikiManager {
    * Get recent operations
    */
   getRecentOperations(limit = 50): WikiOperation[] {
-    requireWiki();
+    requireWiki(this.storage);
     return this.storage.getRecentOperations(limit);
   }
 }
@@ -557,7 +592,14 @@ export class WikiManager {
  */
 export function createWikiManager(workspaceRoot: string, projectId: string): WikiManager {
   const storage = createWikiStorage(workspaceRoot, projectId);
-  return new WikiManager(storage);
+  const manager = new WikiManager(storage);
+
+  // Initialize storage if feature is enabled (may be called lazily later)
+  if (process.env.HARNESS_KNOWLEDGE_WIKI === "1") {
+    storage.initialize();
+  }
+
+  return manager;
 }
 
 // Re-export storage class

package/packages/wiki/src/storage.ts

@@ -14,7 +14,7 @@ import {
   readdirSync,
   statSync,
 } from "node:fs";
-import { join, dirname, relative } from "node:path";
+import { join, dirname, relative, resolve } from "node:path";
 import { createHash, randomUUID } from "node:crypto";
 import type {
   WikiConfig,
@@ -66,7 +66,9 @@ export function parseFrontmatter(
   const frontmatter = parseYaml(yamlStr);
 
   return {
-    frontmatter: frontmatter as unknown as WikiPageFrontmatter | null,
+    // SAFETY: parseYaml returns Record<string, unknown>, cast through unknown to satisfy TS
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    frontmatter: frontmatter as any,
     body: body.trim(),
     raw: yamlStr,
   };
@@ -74,26 +76,75 @@ export function parseFrontmatter(
 
 /**
  * Simple YAML parser for frontmatter
- * Handles: strings, booleans, numbers, arrays, nested objects
+ * Handles: strings, booleans, numbers, arrays, nested objects, YAML list items
  */
 function parseYaml(yamlStr: string): Record<string, unknown> {
   const result: Record<string, unknown> = {};
   const lines = yamlStr.split("\n");
-  let currentKey = "";
-  let currentIndent = 0;
   const stack: Array<{ obj: Record<string, unknown>; indent: number }> = [];
 
-  for (const line of lines) {
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
     const trimmed = line.trim();
     if (!trimmed || trimmed.startsWith("#")) continue;
 
     const indent = line.search(/\S/);
-    const colonIdx = trimmed.indexOf(":");
 
+    // Handle YAML list items (lines starting with '-')
+    if (trimmed.startsWith("-")) {
+      const itemContent = trimmed.slice(1).trim();
+      const colonIdx = itemContent.indexOf(":");
+
+      if (colonIdx !== -1) {
+        // This is a list item with key-value pairs on same line or next lines
+        const key = itemContent.slice(0, colonIdx).trim();
+        const value = itemContent.slice(colonIdx + 1).trim();
+
+        // Pop stack while indent decreases
+        while (stack.length > 0 && indent < stack[stack.length - 1].indent) {
+          stack.pop();
+        }
+
+        // Find or create the array
+        const parent = stack.length > 0 ? stack[stack.length - 1].obj : result;
+        const arrayKey = parent ? Object.keys(parent).slice(-1)[0] : undefined;
+        let array: unknown[];
+
+        if (arrayKey && Array.isArray(parent[arrayKey])) {
+          array = parent[arrayKey] as unknown[];
+        } else {
+          // Start a new array
+          const newArray: unknown[] = [];
+          if (parent) {
+            parent[arrayKey || key] = newArray;
+          } else {
+            result[key] = newArray;
+          }
+          array = newArray;
+        }
+
+        // Create new object for this list item
+        const item: Record<string, unknown> = {};
+        array.push(item);
+
+        // Parse the key-value on this line
+        const parsedValue = parseValue(value);
+        if (parsedValue !== undefined) {
+          item[key] = parsedValue;
+        }
+
+        // Push this item to stack for nested properties
+        stack.push({ obj: item, indent });
+      }
+      continue;
+    }
+
+    // Regular key-value line
+    const colonIdx = trimmed.indexOf(":");
     if (colonIdx === -1) continue;
 
     const key = trimmed.slice(0, colonIdx).trim();
-    let value: unknown = trimmed.slice(colonIdx + 1).trim();
+    const rawValue = trimmed.slice(colonIdx + 1).trim();
 
     // Pop stack while indent decreases
     while (stack.length > 0 && indent <= stack[stack.length - 1].indent) {
@@ -102,34 +153,41 @@ function parseYaml(yamlStr: string): Record<string, unknown> {
 
     const parent = stack.length > 0 ? stack[stack.length - 1].obj : result;
 
-    // Parse value
-    if (value === "" || value === "null" || value === "~") {
-      value = null;
-    } else if (value === "true") {
-      value = true;
-    } else if (value === "false") {
-      value = false;
-    } else if (/^-?\d+(\.\d+)?$/.test(value as string)) {
-      value = parseFloat(value as string);
-    } else if ((value as string).startsWith("[") && (value as string).endsWith("]")) {
-      // Array
-      const arrContent = (value as string).slice(1, -1);
-      value = arrContent
-        ? arrContent.split(",").map((s) => s.trim().replace(/^["']|["']$/g, ""))
-        : [];
-    } else {
-      // String - remove quotes
-      value = (value as string).replace(/^["']|["']$/g, "");
+    // Check if value is empty (nested content follows)
+    if (rawValue === "" || rawValue === "~" || rawValue === "null") {
+      // Nested content follows - create placeholder
+      const nested: Record<string, unknown> = {};
+      parent[key] = nested;
+      stack.push({ obj: nested, indent });
+      continue;
     }
 
-    parent[key] = value;
-    currentKey = key;
-    currentIndent = indent;
+    parent[key] = parseValue(rawValue);
   }
 
   return result;
 }
 
+/**
+ * Parse a YAML scalar value or array
+ */
+type YamlValue = string | number | boolean | null | string[];
+function parseValue(value: string): YamlValue {
+  if (value === "" || value === "null" || value === "~") {
+    return null;
+  }
+  if (value === "true") return true;
+  if (value === "false") return false;
+  if (/^-?\d+(\.\d+)?$/.test(value)) {
+    return parseFloat(value);
+  }
+  if (value.startsWith("[") && value.endsWith("]")) {
+    const content = value.slice(1, -1);
+    return content ? content.split(",").map((s) => s.trim().replace(/^["']|["']$/g, "")) : [];
+  }
+  return value.replace(/^["']|["']$/g, "");
+}
+
 /**
  * Serialize frontmatter to YAML string
  */
@@ -144,11 +202,15 @@ export function serializeFrontmatter(
     if (Array.isArray(value)) {
       if (value.length === 0) {
         lines.push(`${key}: []`);
-      } else if (value.every((v) => typeof v === "object")) {
-        // Array of objects
+      } else if (value.every((v) => typeof v === "object" && v !== null)) {
+        // Array of objects - serialize as proper YAML
         lines.push(`${key}:`);
         for (const item of value) {
-          lines.push(`  - ${JSON.stringify(item)}`);
+          lines.push(`  - sourceId: ${(item as Record<string, unknown>).sourceId}`);
+          lines.push(`    path: ${(item as Record<string, unknown>).path}`);
+          lines.push(`    contentHash: ${(item as Record<string, unknown>).contentHash}`);
+          lines.push(`    capturedAt: ${(item as Record<string, unknown>).capturedAt}`);
+          lines.push(`    redacted: ${(item as Record<string, unknown>).redacted}`);
         }
       } else {
         lines.push(`${key}: [${value.join(", ")}]`);
@@ -195,16 +257,38 @@ export function combinePage(
 export class WikiStorage {
   private config: WikiConfig;
   private index: Map<string, PageIndexEntry> = new Map();
+  private indexMtime = 0; // Track when index was last loaded
   private operations: WikiOperation[] = [];
   private stagedChanges: Map<string, StagedChange> = new Map();
-  private dirty = false;
+  private initialized = false;
 
   constructor(config: WikiConfig) {
     this.config = config;
-    this.ensureStructure();
+    // Don't create directories until explicitly initialized
+    // This prevents writes when the feature is disabled
+  }
+
+  /**
+   * Initialize storage (call before any read/write operations)
+   * This is called automatically by WikiManager when feature is enabled
+   */
+  /**
+   * Prepare storage for use (load indexes and operations)
+   * Called after directories are ensured by initialize()
+   */
+  private prepareStorage(): void {
+    if (this.initialized) return;
     this.loadIndex();
     this.loadOperations();
     this.loadStaged();
+    this.initialized = true;
+  }
+
+  /**
+   * Check if storage is initialized
+   */
+  isInitialized(): boolean {
+    return this.initialized;
   }
 
   /**
@@ -302,20 +386,26 @@ verification:
   }
 
   /**
-   * Ensure directory structure exists
+   * Load page index from disk (if changed since last load)
    */
-  private ensureStructure(): void {
-    if (!existsSync(this.config.wikiRoot)) {
-      mkdirSync(this.config.wikiRoot, { recursive: true });
+  private loadIndex(force = false): void {
+    if (!force) {
+      try {
+        const stat = statSync(this.config.indexFile);
+        // Only reload if file was modified since last load
+        if (stat.mtimeMs <= this.indexMtime) {
+          return;
+        }
+      } catch {
+        // File doesn't exist or can't be stat'd
+      }
     }
-  }
 
-  /**
-   * Load page index from disk
-   */
-  private loadIndex(): void {
     this.index.clear();
-    if (!existsSync(this.config.indexFile)) return;
+    if (!existsSync(this.config.indexFile)) {
+      this.indexMtime = 0;
+      return;
+    }
 
     try {
       const content = readFileSync(this.config.indexFile, "utf8");
@@ -323,21 +413,51 @@ verification:
       for (const entry of entries) {
         this.index.set(entry.id, entry);
       }
+      const stat = statSync(this.config.indexFile);
+      this.indexMtime = stat.mtimeMs;
     } catch {
       // Invalid index, rebuild
+      this.indexMtime = 0;
     }
   }
 
   /**
-   * Save page index to disk (atomic)
+   * Save page index to disk (atomic, merges with existing entries)
    */
   private saveIndex(): void {
-    const entries = Array.from(this.index.values());
+    // Reload from disk to merge (handles concurrent writes)
+    const diskEntries = this.loadIndexFromDisk();
+    const merged = new Map<string, PageIndexEntry>(diskEntries);
+
+    // Apply in-memory changes
+    for (const [id, entry] of this.index) {
+      merged.set(id, entry);
+    }
+
+    const entries = Array.from(merged.values());
     const content = JSON.stringify(entries, null, 2);
     const tmpFile = this.config.indexFile + ".tmp";
     writeFileSync(tmpFile, content, "utf8");
     renameSync(tmpFile, this.config.indexFile);
-    this.dirty = false;
+  }
+
+  /**
+   * Load index from disk only (no cache)
+   */
+  private loadIndexFromDisk(): Map<string, PageIndexEntry> {
+    const result = new Map<string, PageIndexEntry>();
+    if (!existsSync(this.config.indexFile)) return result;
+
+    try {
+      const content = readFileSync(this.config.indexFile, "utf8");
+      const entries: PageIndexEntry[] = JSON.parse(content);
+      for (const entry of entries) {
+        result.set(entry.id, entry);
+      }
+    } catch {
+      // Invalid index, return empty
+    }
+    return result;
   }
 
   /**
@@ -436,7 +556,9 @@ verification:
    * Get source snapshot by ID
    */
   getSnapshot(snapshotId: string): SourceSnapshot | null {
+    this.validateSnapshotId(snapshotId);
     const snapFile = join(this.config.rawDir, `${snapshotId}.json`);
+    this.ensureWithinDirectory(snapFile, this.config.rawDir);
     if (!existsSync(snapFile)) return null;
 
     try {
@@ -455,8 +577,10 @@ verification:
    * Save a wiki page
    */
   savePage(page: WikiPage): void {
+    this.validatePageId(page.frontmatter.id);
     const content = combinePage(page.frontmatter, page.body);
     const pageFile = join(this.config.knowledgeDir, `${page.frontmatter.id}.md`);
+    this.ensureWithinDirectory(pageFile, this.config.knowledgeDir);
     const tmpFile = pageFile + ".tmp";
     writeFileSync(tmpFile, content, "utf8");
     renameSync(tmpFile, pageFile);
@@ -474,14 +598,15 @@ verification:
     };
     this.index.set(entry.id, entry);
     this.saveIndex();
-    this.dirty = true;
   }
 
   /**
    * Load a wiki page
    */
   loadPage(pageId: string): WikiPage | null {
+    this.validatePageId(pageId);
     const pageFile = join(this.config.knowledgeDir, `${pageId}.md`);
+    this.ensureWithinDirectory(pageFile, this.config.knowledgeDir);
     if (!existsSync(pageFile)) return null;
 
     const content = readFileSync(pageFile, "utf8");
@@ -492,10 +617,42 @@ verification:
     return { frontmatter, body, rawFrontmatter: raw };
   }
 
+  /**
+   * Validate page ID to prevent path traversal attacks
+   */
+  private validatePageId(pageId: string): void {
+    // Reject paths with directory separators or parent references
+    if (pageId.includes("/") || pageId.includes("\\") || pageId.includes("..")) {
+      throw new Error(`Invalid page ID: ${pageId}`);
+    }
+  }
+
+  /**
+   * Validate snapshot ID to prevent path traversal attacks
+   */
+  private validateSnapshotId(snapshotId: string): void {
+    // Reject paths with directory separators or parent references
+    if (snapshotId.includes("/") || snapshotId.includes("\\") || snapshotId.includes("..")) {
+      throw new Error(`Invalid snapshot ID: ${snapshotId}`);
+    }
+  }
+
+  /**
+   * Ensure a path is within the expected directory (defense in depth)
+   */
+  private ensureWithinDirectory(filePath: string, expectedDir: string): void {
+    const resolved = resolve(filePath);
+    const expected = resolve(expectedDir);
+    if (!resolved.startsWith(expected + "/") && resolved !== expected) {
+      throw new Error(`Path ${filePath} is outside expected directory ${expectedDir}`);
+    }
+  }
+
   /**
    * Delete a wiki page
    */
   deletePage(pageId: string): void {
+    this.validatePageId(pageId);
     const pageFile = join(this.config.knowledgeDir, `${pageId}.md`);
     if (existsSync(pageFile)) {
       unlinkSync(pageFile);
@@ -510,6 +667,9 @@ verification:
    * Get all pages
    */
   getAllPages(): WikiPage[] {
+    // Reload index to get latest changes from other processes
+    this.loadIndex();
+
     const pages: WikiPage[] = [];
     if (!existsSync(this.config.knowledgeDir)) return pages;
 
@@ -555,6 +715,8 @@ verification:
    * Get all staged changes
    */
   getAllStagedChanges(): StagedChange[] {
+    // Reload staged changes to get latest from other processes
+    this.loadStaged();
     return Array.from(this.stagedChanges.values());
   }
 
@@ -603,6 +765,19 @@ verification:
     renameSync(tmpFile, this.config.logFile);
   }
 
+  /**
+   * Update operation details (before completion)
+   */
+  updateOperationDetails(opId: string, details: Record<string, unknown>): void {
+    const opIndex = this.operations.findIndex((o) => o.id === opId);
+    if (opIndex === -1) return;
+
+    this.operations[opIndex] = {
+      ...this.operations[opIndex],
+      details: { ...this.operations[opIndex].details, ...details },
+    };
+  }
+
   /**
    * Get recent operations
    */
@@ -621,7 +796,7 @@ verification:
     const queryLower = query.toLowerCase();
     const results: WikiSearchResult[] = [];
 
-    for (const [id, entry] of this.index) {
+    for (const [_id, entry] of this.index) {
       // Apply filters
       if (filters?.kinds?.length && !filters.kinds.includes(entry.kind)) continue;
       if (filters?.statuses?.length && !filters.statuses.includes(entry.status)) continue;