wicked-brain 0.1.0

package/server/lib/sqlite-search.mjs

@@ -0,0 +1,260 @@
+import Database from "better-sqlite3";
+import { parseWikilinks } from "./wikilinks.mjs";
+import { statSync } from "node:fs";
+
+function escapeFtsQuery(query) {
+  return query
+    .trim()
+    .split(/\s+/)
+    .filter(Boolean)
+    .map((w) => `"${w.replace(/"/g, '""')}"`)
+    .join(" ");
+}
+
+export class SqliteSearch {
+  #db;
+  #brainId;
+  #startTime;
+
+  constructor(dbPath, brainId) {
+    this.#brainId = brainId;
+    this.#startTime = Date.now();
+    this.#db = new Database(dbPath);
+    this.#db.pragma("journal_mode = WAL");
+    this.#initSchema();
+  }
+
+  #initSchema() {
+    this.#db.exec(`
+      CREATE TABLE IF NOT EXISTS documents (
+        id TEXT PRIMARY KEY,
+        path TEXT NOT NULL,
+        content TEXT NOT NULL,
+        frontmatter TEXT,
+        brain_id TEXT NOT NULL,
+        indexed_at INTEGER NOT NULL
+      );
+
+      CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
+        id,
+        path,
+        content,
+        brain_id,
+        tokenize='porter unicode61'
+      );
+
+      CREATE TABLE IF NOT EXISTS links (
+        source_id TEXT NOT NULL,
+        source_brain TEXT NOT NULL,
+        target_path TEXT NOT NULL,
+        target_brain TEXT,
+        link_text TEXT
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_links_source ON links(source_id);
+      CREATE INDEX IF NOT EXISTS idx_links_target ON links(target_path);
+    `);
+  }
+
+  index(doc) {
+    const { id, path, content, frontmatter = null } = doc;
+    const brainId = this.#brainId;
+    const indexedAt = Date.now();
+
+    const upsertDoc = this.#db.prepare(`
+      INSERT INTO documents (id, path, content, frontmatter, brain_id, indexed_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        path = excluded.path,
+        content = excluded.content,
+        frontmatter = excluded.frontmatter,
+        brain_id = excluded.brain_id,
+        indexed_at = excluded.indexed_at
+    `);
+
+    const deleteFts = this.#db.prepare(`DELETE FROM documents_fts WHERE id = ?`);
+    const insertFts = this.#db.prepare(`
+      INSERT INTO documents_fts (id, path, content, brain_id)
+      VALUES (?, ?, ?, ?)
+    `);
+
+    const deleteLinks = this.#db.prepare(`DELETE FROM links WHERE source_id = ?`);
+    const insertLink = this.#db.prepare(`
+      INSERT INTO links (source_id, source_brain, target_path, target_brain, link_text)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+
+    const run = this.#db.transaction(() => {
+      upsertDoc.run(id, path, content, frontmatter, brainId, indexedAt);
+      deleteFts.run(id);
+      insertFts.run(id, path, content, brainId);
+      deleteLinks.run(id);
+      const wikilinks = parseWikilinks(content);
+      for (const link of wikilinks) {
+        insertLink.run(id, brainId, link.path, link.brain, link.raw);
+      }
+    });
+
+    run();
+  }
+
+  remove(id) {
+    const run = this.#db.transaction(() => {
+      this.#db.prepare(`DELETE FROM documents WHERE id = ?`).run(id);
+      this.#db.prepare(`DELETE FROM documents_fts WHERE id = ?`).run(id);
+      this.#db.prepare(`DELETE FROM links WHERE source_id = ?`).run(id);
+    });
+    run();
+  }
+
+  reindex(docs) {
+    const run = this.#db.transaction(() => {
+      this.#db.exec(`DELETE FROM documents`);
+      this.#db.exec(`DELETE FROM documents_fts`);
+      this.#db.exec(`DELETE FROM links`);
+      for (const doc of docs) {
+        this.index(doc);
+      }
+    });
+    run();
+  }
+
+  search({ query, limit = 10, offset = 0 }) {
+    const escaped = escapeFtsQuery(query);
+    if (!escaped) return { results: [], total_matches: 0, showing: 0 };
+
+    const rows = this.#db
+      .prepare(`
+        SELECT
+          d.id,
+          d.path,
+          d.brain_id,
+          snippet(documents_fts, 2, '<b>', '</b>', '…', 32) AS snippet
+        FROM documents_fts f
+        JOIN documents d ON d.id = f.id
+        WHERE documents_fts MATCH ?
+        ORDER BY rank
+        LIMIT ? OFFSET ?
+      `)
+      .all(escaped, limit, offset);
+
+    const countRow = this.#db
+      .prepare(`SELECT COUNT(*) as cnt FROM documents_fts WHERE documents_fts MATCH ?`)
+      .get(escaped);
+
+    const total_matches = countRow ? countRow.cnt : 0;
+
+    return {
+      results: rows,
+      total_matches,
+      showing: rows.length,
+    };
+  }
+
+  federatedSearch({ query, brains = [], limit = 10 }) {
+    const localResults = this.search({ query, limit });
+    const allResults = [...localResults.results];
+    const unreachable = [];
+
+    for (const { brainId, dbPath } of brains) {
+      try {
+        const attached = `brain_${brainId.replace(/[^a-zA-Z0-9_]/g, "_")}`;
+        this.#db.prepare(`ATTACH DATABASE ? AS ${attached}`).run(dbPath);
+        try {
+          const escaped = escapeFtsQuery(query);
+          const rows = this.#db
+            .prepare(`
+              SELECT
+                d.id,
+                d.path,
+                d.brain_id,
+                snippet(${attached}.documents_fts, 2, '<b>', '</b>', '…', 32) AS snippet
+              FROM ${attached}.documents_fts f
+              JOIN ${attached}.documents d ON d.id = f.id
+              WHERE ${attached}.documents_fts MATCH ?
+              ORDER BY rank
+              LIMIT ?
+            `)
+            .all(escaped, limit);
+          allResults.push(...rows);
+        } finally {
+          this.#db.prepare(`DETACH DATABASE ${attached}`).run();
+        }
+      } catch {
+        unreachable.push(brainId);
+      }
+    }
+
+    allResults.sort((a, b) => (a.rank ?? 0) - (b.rank ?? 0));
+    const trimmed = allResults.slice(0, limit);
+
+    return {
+      results: trimmed,
+      total_matches: localResults.total_matches,
+      showing: trimmed.length,
+      unreachable,
+    };
+  }
+
+  backlinks(id) {
+    return this.#db
+      .prepare(`
+        SELECT source_id, source_brain, link_text
+        FROM links
+        WHERE target_path = ?
+      `)
+      .all(id);
+  }
+
+  forwardLinks(id) {
+    const rows = this.#db
+      .prepare(`
+        SELECT target_path, target_brain
+        FROM links
+        WHERE source_id = ?
+      `)
+      .all(id);
+    return rows.map((r) => r.target_path);
+  }
+
+  stats() {
+    const total = this.#db
+      .prepare(`SELECT COUNT(*) as cnt FROM documents`)
+      .get().cnt;
+
+    const chunks = this.#db
+      .prepare(`SELECT COUNT(*) as cnt FROM documents WHERE path LIKE 'chunks/%'`)
+      .get().cnt;
+
+    const wiki = this.#db
+      .prepare(`SELECT COUNT(*) as cnt FROM documents WHERE path LIKE 'wiki/%'`)
+      .get().cnt;
+
+    const lastRow = this.#db
+      .prepare(`SELECT MAX(indexed_at) as last FROM documents`)
+      .get();
+    const last_indexed = lastRow ? lastRow.last : null;
+
+    const dbFile = this.#db.name;
+    let db_size = null;
+    try {
+      db_size = statSync(dbFile).size;
+    } catch {
+      // in-memory or inaccessible
+    }
+
+    return { total, chunks, wiki, last_indexed, db_size };
+  }
+
+  health() {
+    return {
+      status: "ok",
+      uptime: Date.now() - this.#startTime,
+      brain_id: this.#brainId,
+    };
+  }
+
+  close() {
+    this.#db.close();
+  }
+}

package/server/lib/wikilinks.mjs

@@ -0,0 +1,19 @@
+const WIKILINK_RE = /\[\[([^\]]+)\]\]/g;
+
+export function parseWikilinks(text) {
+  const links = [];
+  for (const match of text.matchAll(WIKILINK_RE)) {
+    const inner = match[1].trim();
+    if (!inner) continue;
+    const raw = match[0];
+    if (inner.includes("::")) {
+      const idx = inner.indexOf("::");
+      const brain = inner.slice(0, idx).trim();
+      const path = inner.slice(idx + 2).trim();
+      if (brain && path) links.push({ brain, path, raw });
+    } else {
+      links.push({ brain: null, path: inner, raw });
+    }
+  }
+  return links;
+}

package/server/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "wicked-brain-server",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "SQLite FTS5 search server for wicked-brain digital knowledge bases",
+  "keywords": [
+    "wicked-brain",
+    "sqlite",
+    "fts5",
+    "search",
+    "knowledge-base"
+  ],
+  "author": "Mike Parcewski",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mikeparcewski/wicked-brain.git",
+    "directory": "server"
+  },
+  "bin": {
+    "wicked-brain-server": "./bin/wicked-brain-server.mjs"
+  },
+  "files": [
+    "bin/",
+    "lib/",
+    "package.json"
+  ],
+  "scripts": {
+    "test": "node --test test/*.test.mjs",
+    "start": "node bin/wicked-brain-server.mjs"
+  },
+  "dependencies": {
+    "better-sqlite3": "^12.0.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

package/skills/wicked-brain-batch/SKILL.md

@@ -0,0 +1,112 @@
+---
+name: wicked-brain:batch
+description: |
+  Pattern for batch operations that would otherwise fill context with repetitive
+  tool calls. Detects the available runtime (Node, Python, shell), writes a
+  script, runs it, and reports results. Used internally by other skills.
+  
+  Use when: any brain operation needs to process more than 5 files, run more
+  than 10 API calls, or would otherwise burn context on repetitive operations.
+---
+
+# wicked-brain:batch
+
+You handle batch operations efficiently by generating and running scripts instead
+of executing repetitive tool calls inline.
+
+## When to use
+
+- Ingesting a directory of files (wicked-brain:ingest calls you)
+- Reindexing all content (wicked-brain:lint or rebuild)
+- Bulk search across many terms
+- Any operation touching more than 5 files
+
+## Why scripts over tool calls
+
+| Approach | Context cost | Speed | Reliability |
+|---|---|---|---|
+| 50 Read + 50 Write + 50 Bash (curl) | ~150 tool calls, floods context | Slow (round-trips) | Error-prone (partial failures) |
+| Write 1 script + Run 1 script + Read output | ~3 tool calls | Fast (single process) | Script handles errors internally |
+
+## Process
+
+### Step 1: Detect runtime
+
+Check what's available, in preference order:
+
+```bash
+node --version 2>/dev/null && echo "node" || python3 --version 2>/dev/null && echo "python3" || python --version 2>/dev/null && echo "python" || echo "shell"
+```
+
+Prefer Node.js (since wicked-brain-server requires it, it's always available).
+
+### Step 2: Write the script
+
+Write to `{brain_path}/_meta/batch-{operation}.mjs` (or `.py` or `.sh`).
+
+The script must:
+1. Accept the brain path, server port, and operation-specific params
+2. Do all the work (walk dirs, read files, write chunks, curl APIs)
+3. Log progress to stdout (one line per file processed)
+4. Handle errors per-file (don't stop on one failure)
+5. Print a summary at the end
+
+### Step 3: Run the script
+
+```bash
+node {brain_path}/_meta/batch-{operation}.mjs
+```
+
+### Step 4: Read output and report
+
+Read the script's stdout. Summarize results to the user.
+
+### Step 5: Clean up
+
+Optionally delete the script after successful completion:
+```bash
+rm {brain_path}/_meta/batch-{operation}.mjs
+```
+
+Or keep it for re-runs — the user can run it manually too.
+
+## Cross-Platform Notes
+
+- Node.js scripts are fully cross-platform (same code on macOS/Linux/Windows)
+- Python scripts are fully cross-platform
+- Shell scripts need macOS/Linux + Windows variants — avoid if Node or Python available
+- Use `fetch()` (Node 18+) instead of `curl` in scripts — it's native and cross-platform
+- Use `node:fs` and `node:path` — they handle platform differences
+
+## Template: Node.js batch script
+
+See wicked-brain:ingest for a complete example. The key structure:
+
+```javascript
+#!/usr/bin/env node
+import { ... } from "node:fs";
+import { ... } from "node:path";
+
+const BRAIN = "{brain_path}";
+const PORT = {port};
+
+// Walk, process, index, report
+```
+
+## Template: Python batch script
+
+```python
+#!/usr/bin/env python3
+import os, json, hashlib, urllib.request
+
+BRAIN = "{brain_path}"
+PORT = {port}
+
+def api(action, params):
+    data = json.dumps({"action": action, "params": params}).encode()
+    req = urllib.request.Request(f"http://localhost:{PORT}/api",
+        data=data, headers={"Content-Type": "application/json"})
+    return json.loads(urllib.request.urlopen(req).read())
+
+# Walk, process, index, report
+```

package/skills/wicked-brain-compile/SKILL.md

@@ -0,0 +1,124 @@
+---
+name: wicked-brain:compile
+description: |
+  Synthesize wiki articles from brain chunks. Dispatches a compile subagent
+  that identifies concept clusters in chunks and writes structured wiki
+  articles with backlinks and source attribution.
+  
+  Use when: "compile the brain", "write wiki articles", "synthesize knowledge",
+  "brain compile".
+---
+
+# wicked-brain:compile
+
+You compile wiki articles from the brain's chunks by dispatching a compile subagent.
+
+## Cross-Platform Notes
+
+Commands in this skill work on macOS, Linux, and Windows. When a command has
+platform differences, alternatives are shown. Your native tools (Read, Write,
+Grep, Glob) work everywhere — prefer them over shell commands when possible.
+
+For the brain path default:
+- macOS/Linux: ~/.wicked-brain
+- Windows: %USERPROFILE%\.wicked-brain
+
+## Config
+
+Read `_meta/config.json` for brain path and server port.
+If it doesn't exist, trigger wicked-brain:init.
+
+## Process
+
+Dispatch a compile subagent with these instructions:
+
+```
+You are a compile agent for the digital brain at {brain_path}.
+Server: http://localhost:{port}/api
+
+## Your task
+
+Read chunks and synthesize wiki articles that capture key concepts.
+
+## Step 1: Orient
+
+Get brain stats:
+```bash
+curl -s -X POST http://localhost:{port}/api \
+  -H "Content-Type: application/json" \
+  -d '{"action":"stats","params":{}}'
+```
+
+List existing wiki articles using your Glob tool on `{brain_path}/wiki/**/*.md`.
+Shell fallback:
+- macOS/Linux: `find {brain_path}/wiki -name "*.md" -type f 2>/dev/null`
+- Windows: `Get-ChildItem -Recurse -Filter "*.md" "{brain_path}\wiki" 2>nul`
+
+List chunks using your Glob tool on `{brain_path}/chunks/extracted/**/*.md`.
+Shell fallback:
+- macOS/Linux: `find {brain_path}/chunks/extracted -name "*.md" -type f 2>/dev/null`
+- Windows: `Get-ChildItem -Recurse -Filter "*.md" "{brain_path}\chunks\extracted" 2>nul`
+
+## Step 2: Find uncovered chunks
+
+For each chunk directory, check if a wiki article references those chunks.
+Use your Grep tool to find which chunks are already cited in wiki articles.
+Shell fallback:
+- macOS/Linux: `grep -rl "chunk-" {brain_path}/wiki/ 2>/dev/null`
+- Windows: `findstr /s /m "chunk-" "{brain_path}\wiki\*.md" 2>nul`
+
+Focus on chunks NOT referenced by any wiki article.
+
+## Step 3: Read uncovered chunks
+
+Read uncovered chunks (frontmatter + body) to understand their content.
+Group them by topic/concept.
+
+## Step 4: Write wiki articles
+
+For each concept cluster, write a wiki article to `{brain_path}/wiki/concepts/{concept-name}.md`
+or `{brain_path}/wiki/topics/{topic-name}.md`:
+
+```
+---
+authored_by: llm
+authored_at: {ISO timestamp}
+source_chunks:
+  - {chunk-path-1}
+  - {chunk-path-2}
+contains:
+  - {topic tags}
+---
+
+# {Concept Name}
+
+{Article body with [[backlinks]] to source chunks.}
+
+Every factual claim should link to its source: [[chunks/extracted/{source}/chunk-NNN]].
+
+## Related
+
+- [[other-concept]]
+- [[brain-id::cross-brain-concept]] (if applicable)
+```
+
+## Step 5: Index new articles
+
+For each article written:
+```bash
+curl -s -X POST http://localhost:{port}/api \
+  -H "Content-Type: application/json" \
+  -d '{"action":"index","params":{"id":"{path}","path":"{path}","content":"{content}","brain_id":"{brain_id}"}}'
+```
+
+## Step 6: Log
+
+Append to `{brain_path}/_meta/log.jsonl` for each article:
+```json
+{"ts":"{ISO}","op":"write","path":"{article_path}","author":"llm:compile","content_hash":"{hash}"}
+```
+
+## Step 7: Report
+
+State how many articles were created/updated and what concepts they cover.
+```

package/skills/wicked-brain-enhance/SKILL.md

@@ -0,0 +1,103 @@
+---
+name: wicked-brain:enhance
+description: |
+  Fill gaps in brain knowledge. Dispatches an enhance subagent that identifies
+  thin areas and writes inferred chunks to expand coverage.
+  
+  Use when: "enhance the brain", "fill gaps", "brain enhance",
+  "what's missing in the brain".
+---
+
+# wicked-brain:enhance
+
+You enhance the brain by dispatching a subagent that fills knowledge gaps.
+
+## Cross-Platform Notes
+
+Commands in this skill work on macOS, Linux, and Windows. When a command has
+platform differences, alternatives are shown. Your native tools (Read, Write,
+Grep, Glob) work everywhere — prefer them over shell commands when possible.
+
+For the brain path default:
+- macOS/Linux: ~/.wicked-brain
+- Windows: %USERPROFILE%\.wicked-brain
+
+## Config
+
+Read `_meta/config.json` for brain path and server port.
+If it doesn't exist, trigger wicked-brain:init.
+
+## Process
+
+Dispatch an enhance subagent with these instructions:
+
+```
+You are a knowledge enhancement agent for the digital brain at {brain_path}.
+Server: http://localhost:{port}/api
+
+## Step 1: Find gaps
+
+Read the recent event log using your Read tool on `{brain_path}/_meta/log.jsonl`
+(read the last 100 lines). Shell fallback:
+- macOS/Linux: `tail -100 {brain_path}/_meta/log.jsonl`
+- Windows: `Get-Content "{brain_path}\_meta\log.jsonl" -Tail 100`
+
+Get stats:
+```bash
+curl -s -X POST http://localhost:{port}/api \
+  -H "Content-Type: application/json" \
+  -d '{"action":"stats","params":{}}'
+```
+
+Search for thin areas — topics mentioned in existing chunks but with few entries.
+Use your Grep tool on `{brain_path}/chunks/` to find all `contains:` fields and
+count occurrences. Shell fallback:
+- macOS/Linux: `grep -roh 'contains:' {brain_path}/chunks/ -A 5 2>/dev/null | grep '  - ' | sort | uniq -c | sort -n`
+- Windows: `Select-String -Recurse -Pattern "  - " "{brain_path}\chunks\*.md" 2>nul | Select-Object -ExpandProperty Line | Sort-Object | Group-Object | Sort-Object Count`
+
+## Step 2: Identify what's missing
+
+Based on existing content, reason about:
+- Topics mentioned but never elaborated
+- Connections between concepts that exist but aren't documented
+- Questions the brain can't currently answer
+
+## Step 3: Write inferred chunks
+
+For each gap, write a new chunk to `{brain_path}/chunks/inferred/{topic}/chunk-NNN.md`:
+
+```
+---
+source: inferred
+source_type: llm
+chunk_id: inferred/{topic}/chunk-NNN
+content_type:
+  - text
+contains:
+  - {topic tags}
+entities:
+  systems: []
+  people: []
+  programs: []
+  metrics: []
+confidence: 0.6
+indexed_at: {ISO timestamp}
+authored_by: llm
+narrative_theme: {what this fills}
+source_chunks:
+  - {existing chunk that informed this inference}
+---
+
+{Synthesized content based on existing brain knowledge. Do not fabricate facts.
+Only synthesize connections and summaries from what already exists.}
+```
+
+## Step 4: Index and log
+
+Index each new chunk via the server API.
+Append to log.jsonl for each chunk written.
+
+## Step 5: Report
+
+State what gaps were identified and how many inferred chunks were created.
+```