wikimem 0.2.3 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +138 -29
  2. package/README.md +173 -311
  3. package/dist/cli/commands/ask.d.ts +3 -0
  4. package/dist/cli/commands/ask.d.ts.map +1 -0
  5. package/dist/cli/commands/ask.js +63 -0
  6. package/dist/cli/commands/ask.js.map +1 -0
  7. package/dist/cli/commands/export.d.ts +3 -0
  8. package/dist/cli/commands/export.d.ts.map +1 -0
  9. package/dist/cli/commands/export.js +108 -0
  10. package/dist/cli/commands/export.js.map +1 -0
  11. package/dist/cli/commands/history.d.ts +3 -0
  12. package/dist/cli/commands/history.d.ts.map +1 -0
  13. package/dist/cli/commands/history.js +61 -0
  14. package/dist/cli/commands/history.js.map +1 -0
  15. package/dist/cli/commands/improve.d.ts.map +1 -1
  16. package/dist/cli/commands/improve.js +4 -3
  17. package/dist/cli/commands/improve.js.map +1 -1
  18. package/dist/cli/commands/ingest.d.ts.map +1 -1
  19. package/dist/cli/commands/ingest.js +5 -4
  20. package/dist/cli/commands/ingest.js.map +1 -1
  21. package/dist/cli/commands/init.d.ts.map +1 -1
  22. package/dist/cli/commands/init.js +246 -79
  23. package/dist/cli/commands/init.js.map +1 -1
  24. package/dist/cli/commands/lint.d.ts.map +1 -1
  25. package/dist/cli/commands/lint.js +4 -3
  26. package/dist/cli/commands/lint.js.map +1 -1
  27. package/dist/cli/commands/mcp.d.ts +3 -0
  28. package/dist/cli/commands/mcp.d.ts.map +1 -0
  29. package/dist/cli/commands/mcp.js +11 -0
  30. package/dist/cli/commands/mcp.js.map +1 -0
  31. package/dist/cli/commands/open.d.ts +3 -0
  32. package/dist/cli/commands/open.d.ts.map +1 -0
  33. package/dist/cli/commands/open.js +36 -0
  34. package/dist/cli/commands/open.js.map +1 -0
  35. package/dist/cli/commands/query.d.ts.map +1 -1
  36. package/dist/cli/commands/query.js +5 -4
  37. package/dist/cli/commands/query.js.map +1 -1
  38. package/dist/cli/commands/search.d.ts +3 -0
  39. package/dist/cli/commands/search.d.ts.map +1 -0
  40. package/dist/cli/commands/search.js +61 -0
  41. package/dist/cli/commands/search.js.map +1 -0
  42. package/dist/cli/commands/serve.d.ts.map +1 -1
  43. package/dist/cli/commands/serve.js +41 -2
  44. package/dist/cli/commands/serve.js.map +1 -1
  45. package/dist/cli/commands/watch.d.ts.map +1 -1
  46. package/dist/cli/commands/watch.js +4 -3
  47. package/dist/cli/commands/watch.js.map +1 -1
  48. package/dist/cli/index.d.ts.map +1 -1
  49. package/dist/cli/index.js +27 -1
  50. package/dist/cli/index.js.map +1 -1
  51. package/dist/core/audit-trail.d.ts +15 -0
  52. package/dist/core/audit-trail.d.ts.map +1 -0
  53. package/dist/core/audit-trail.js +43 -0
  54. package/dist/core/audit-trail.js.map +1 -0
  55. package/dist/core/claude-code.d.ts +10 -0
  56. package/dist/core/claude-code.d.ts.map +1 -0
  57. package/dist/core/claude-code.js +81 -0
  58. package/dist/core/claude-code.js.map +1 -0
  59. package/dist/core/config.d.ts +23 -0
  60. package/dist/core/config.d.ts.map +1 -1
  61. package/dist/core/config.js.map +1 -1
  62. package/dist/core/connectors.d.ts +58 -0
  63. package/dist/core/connectors.d.ts.map +1 -0
  64. package/dist/core/connectors.js +189 -0
  65. package/dist/core/connectors.js.map +1 -0
  66. package/dist/core/folder-scanner.d.ts +10 -0
  67. package/dist/core/folder-scanner.d.ts.map +1 -0
  68. package/dist/core/folder-scanner.js +84 -0
  69. package/dist/core/folder-scanner.js.map +1 -0
  70. package/dist/core/git.d.ts +137 -0
  71. package/dist/core/git.d.ts.map +1 -0
  72. package/dist/core/git.js +520 -0
  73. package/dist/core/git.js.map +1 -0
  74. package/dist/core/history.d.ts +21 -0
  75. package/dist/core/history.d.ts.map +1 -0
  76. package/dist/core/history.js +107 -0
  77. package/dist/core/history.js.map +1 -0
  78. package/dist/core/improve.d.ts.map +1 -1
  79. package/dist/core/improve.js +9 -0
  80. package/dist/core/improve.js.map +1 -1
  81. package/dist/core/ingest.d.ts +1 -0
  82. package/dist/core/ingest.d.ts.map +1 -1
  83. package/dist/core/ingest.js +78 -5
  84. package/dist/core/ingest.js.map +1 -1
  85. package/dist/core/observer.d.ts +71 -0
  86. package/dist/core/observer.d.ts.map +1 -0
  87. package/dist/core/observer.js +350 -0
  88. package/dist/core/observer.js.map +1 -0
  89. package/dist/core/pipeline-events.d.ts +63 -0
  90. package/dist/core/pipeline-events.d.ts.map +1 -0
  91. package/dist/core/pipeline-events.js +109 -0
  92. package/dist/core/pipeline-events.js.map +1 -0
  93. package/dist/core/query.d.ts.map +1 -1
  94. package/dist/core/query.js +16 -8
  95. package/dist/core/query.js.map +1 -1
  96. package/dist/core/scraper.d.ts +41 -0
  97. package/dist/core/scraper.d.ts.map +1 -0
  98. package/dist/core/scraper.js +277 -0
  99. package/dist/core/scraper.js.map +1 -0
  100. package/dist/index.js +3 -2
  101. package/dist/index.js.map +1 -1
  102. package/dist/mcp-entry.d.ts +10 -0
  103. package/dist/mcp-entry.d.ts.map +1 -0
  104. package/dist/mcp-entry.js +21 -0
  105. package/dist/mcp-entry.js.map +1 -0
  106. package/dist/mcp-server.d.ts +15 -0
  107. package/dist/mcp-server.d.ts.map +1 -0
  108. package/dist/mcp-server.js +390 -0
  109. package/dist/mcp-server.js.map +1 -0
  110. package/dist/processors/audio.d.ts.map +1 -1
  111. package/dist/processors/audio.js +42 -4
  112. package/dist/processors/audio.js.map +1 -1
  113. package/dist/processors/pdf.d.ts.map +1 -1
  114. package/dist/processors/pdf.js +2 -3
  115. package/dist/processors/pdf.js.map +1 -1
  116. package/dist/processors/url.js +4 -1
  117. package/dist/processors/url.js.map +1 -1
  118. package/dist/providers/claude.d.ts +1 -0
  119. package/dist/providers/claude.d.ts.map +1 -1
  120. package/dist/providers/claude.js +5 -3
  121. package/dist/providers/claude.js.map +1 -1
  122. package/dist/providers/index.d.ts +17 -1
  123. package/dist/providers/index.d.ts.map +1 -1
  124. package/dist/providers/index.js +144 -0
  125. package/dist/providers/index.js.map +1 -1
  126. package/dist/providers/openai.d.ts +1 -0
  127. package/dist/providers/openai.d.ts.map +1 -1
  128. package/dist/providers/openai.js +5 -3
  129. package/dist/providers/openai.js.map +1 -1
  130. package/dist/providers/types.d.ts +18 -0
  131. package/dist/providers/types.d.ts.map +1 -1
  132. package/dist/templates/config-yaml.d.ts.map +1 -1
  133. package/dist/templates/config-yaml.js +13 -2
  134. package/dist/templates/config-yaml.js.map +1 -1
  135. package/dist/web/public/index.html +8157 -745
  136. package/dist/web/server.d.ts.map +1 -1
  137. package/dist/web/server.js +2101 -29
  138. package/dist/web/server.js.map +1 -1
  139. package/package.json +7 -4
  140. package/scripts/install.sh +54 -0
  141. package/src/web/public/index.html +8157 -745
  142. package/templates/mcp-config.json +9 -0
  143. package/dist/web/public/public/index.html +0 -946
package/README.md CHANGED
@@ -1,228 +1,194 @@
1
- # llmwiki
1
+ # `wikimem`
2
2
 
3
- [![npm version](https://img.shields.io/npm/v/llmwiki.svg)](https://www.npmjs.com/package/llmwiki)
4
- [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
5
- [![TypeScript](https://img.shields.io/badge/TypeScript-strict-blue.svg)](https://www.typescriptlang.org/)
6
- [![Tests](https://img.shields.io/badge/tests-passing-brightgreen.svg)](#tests)
7
-
8
- **Build self-improving knowledge bases with LLMs.**
3
+ **Self-improving wiki IDE. Ingest anything. Query with any LLM. Three automations.**
9
4
 
10
- Drop files in. Get a structured, interlinked wiki out. It improves itself while you sleep.
5
+ [![npm version](https://img.shields.io/npm/v/wikimem.svg)](https://www.npmjs.com/package/wikimem)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
7
+ [![Formats](https://img.shields.io/badge/formats-13%2B-blue.svg)](#13-format-ingestion)
11
8
 
12
9
  ```bash
13
- npx llmwiki init my-wiki
10
+ npx wikimem@latest
14
11
  ```
15
12
 
16
- ```
17
- raw/ wiki/
18
- 2026-04-07/ index.md ........... content catalog
19
- paper.pdf ──LLM──> sources/paper.md ... summary + citations
20
- podcast.mp3 entities/openai.md . people, orgs, tools
21
- screenshot.png concepts/rag.md .... ideas + frameworks
22
- blog-url syntheses/ ......... cross-cutting analysis
23
- ```
13
+ ## What is WikiMem?
24
14
 
25
- llmwiki processes any source (text, PDF, audio, video, images, URLs), compiles it into an interlinked markdown wiki with frontmatter and `[[wikilinks]]`, and opens directly in Obsidian.
15
+ WikiMem takes [Karpathy's LLM wiki concept](https://x.com/karpathy/status/1908625766490001799) and turns it into a full IDE. Drop any file — PDF, audio, video, slides, spreadsheet, URL — and watch it compile into structured, interlinked wiki pages via Claude, GPT-4o, or Ollama. Three automations keep your knowledge base growing and self-improving while you sleep.
26
16
 
27
- Works with **Claude, OpenAI, or Ollama** (local). Your data stays on your machine.
28
-
29
- Inspired by [Andrej Karpathy's LLM Wiki pattern](https://x.com/karpathy/status/1908625766490001799).
30
-
31
- ## Install
32
-
33
- ```bash
34
- npm install -g llmwiki
35
17
  ```
36
-
37
- **Requirements:** Node.js >= 18 · An LLM API key (or Ollama running locally)
18
+ raw/ wiki/
19
+ paper.pdf index.md ........... content catalog
20
+ podcast.mp3 ──LLM──> sources/paper.md ... summary + citations
21
+ screenshot.png entities/openai.md . people, orgs, tools
22
+ meeting.docx concepts/rag.md .... ideas + frameworks
23
+ blog-url syntheses/ ......... cross-cutting analysis
24
+ ```
38
25
 
39
26
  ## Quick Start
40
27
 
41
28
  ```bash
42
- # 1. Create a vault
43
- llmwiki init my-wiki
29
+ # Create a vault and start the IDE
30
+ npx wikimem init my-wiki
44
31
  cd my-wiki
45
-
46
- # 2. Ingest something
47
- llmwiki ingest https://en.wikipedia.org/wiki/Large_language_model
48
- llmwiki ingest ~/Documents/research-paper.pdf
49
-
50
- # 3. Ask questions
51
- llmwiki query "What are the key differences between RAG and compiled knowledge?"
32
+ npx wikimem serve
52
33
  ```
53
34
 
54
- That's it. Your wiki is now a folder of markdown files you can open in Obsidian, VS Code, or any text editor.
35
+ Open [http://localhost:3141](http://localhost:3141). That's it you have a running wiki IDE.
55
36
 
56
- ## Why llmwiki?
37
+ ```bash
38
+ # Or ingest from the CLI
39
+ wikimem ingest paper.pdf
40
+ wikimem ingest https://en.wikipedia.org/wiki/Large_language_model
41
+ wikimem query "What are the key themes across my sources?"
42
+ ```
57
43
 
58
- **The problem:** You have dozens of sources — papers, podcasts, articles, screenshots, meeting recordings. They sit in folders. You forget what's in them. When you need something, you search and re-read.
44
+ ## Features
59
45
 
60
- **RAG approach:** Chunk documents, embed them, retrieve at query time. Lossy, opaque, and the "knowledge" lives in a vector database you can't read.
46
+ ### 13+ Format Ingestion
61
47
 
62
- **llmwiki approach:** Compile sources into structured markdown pages with summaries, cross-references, and citations. The knowledge is readable, editable, version-controlled, and improves itself over time.
48
+ Drop anything. WikiMem detects the file type, runs the right processor, and produces wiki pages with cross-references and citations.
63
49
 
64
- | | RAG | llmwiki |
65
- |---|---|---|
66
- | **Storage** | Vector embeddings | Plain markdown files |
67
- | **Readable?** | No (opaque vectors) | Yes (open in any editor) |
68
- | **Editable?** | Rebuild index | Edit any page |
69
- | **Version control** | Difficult | `git diff` |
70
- | **Self-improving** | No | Yes (LLM Council) |
71
- | **Works offline** | Depends | Yes (with Ollama) |
72
- | **Obsidian** | Plugin required | Native (`[[wikilinks]]`) |
50
+ | Format | Extensions | Processor |
51
+ |--------|-----------|-----------|
52
+ | Text | `.md`, `.txt` | Direct read |
53
+ | Structured | `.json`, `.csv`, `.yaml` | Schema-aware extraction |
54
+ | PDF | `.pdf` | Built-in text extraction |
55
+ | Office | `.docx`, `.pptx`, `.xlsx` | Document parsing |
56
+ | HTML | `.html`, `.htm` | Tag stripping + content extraction |
57
+ | Image | `.png`, `.jpg`, `.gif`, `.webp` | Claude Vision description |
58
+ | Audio | `.mp3`, `.wav`, `.m4a`, `.ogg`, `.flac` | Whisper / Deepgram transcription |
59
+ | Video | `.mp4`, `.mov`, `.avi`, `.mkv`, `.webm` | ffmpeg → Whisper transcription |
60
+ | URL | `https://...` | Firecrawl / fetch → markdown |
73
61
 
74
- ## Features
62
+ ### Knowledge Graph
75
63
 
76
- - [x] **Multi-format ingestion** — text, PDF, audio, video, images, URLs, Office docs
77
- - [x] **Multi-model support** — Claude, OpenAI, Ollama (local)
78
- - [x] **Obsidian-native** — `[[wikilinks]]`, YAML frontmatter, opens directly as vault
79
- - [x] **Semantic dedup** — rejects near-duplicate sources automatically
80
- - [x] **BM25 search** — zero-dependency full-text search, no external services
81
- - [x] **Auto-indexing** — `index.md` + `log.md` maintained automatically
82
- - [x] **Watch mode** — drop files into `raw/`, auto-ingested
83
- - [x] **Self-improvement** — LLM Council scores your wiki and fixes issues
84
- - [x] **External scraping** — pull from RSS, GitHub, URLs on a schedule
85
- - [x] **Health checks** — find orphan pages, broken links, missing summaries
86
- - [x] **File-back answers** — query results saved as synthesis pages
87
- - [x] **Schema co-evolution** — AGENTS.md evolves with your wiki
88
- - [x] **Domain templates** — personal, research, business, codebase
89
- - [x] **Local-first** — everything is files. No database. No cloud dependency.
64
+ D3-powered interactive force-directed graph. Click a node to highlight its neighbors, double-click to open. Community detection clusters related pages. Hub nodes sized by connection count.
90
65
 
91
- ## Architecture
66
+ ### Time-Lapse
92
67
 
93
- ```
94
- ┌────────────────────────────────────────────────────┐
95
- │ llmwiki CLI │
96
- │ │
97
- │ llmwiki init Create a new vault │
98
- │ llmwiki ingest Process source → wiki pages │
99
- │ llmwiki query Ask questions with citations │
100
- │ llmwiki lint Health-check the wiki │
101
- │ llmwiki watch Auto-ingest on file drop │
102
- │ llmwiki scrape Fetch from external sources │
103
- │ llmwiki improve Self-improvement cycle │
104
- │ llmwiki status Vault statistics │
105
- ├──────────────────────┬─────────────────────────────┤
106
- │ Three Layers │ Three Automations │
107
- │ │ │
108
- │ raw/ │ A1: Ingest & Process │
109
- │ (immutable) <────│ file/URL → markdown │
110
- │ │ → place in wiki/ │
111
- │ wiki/ │ │
112
- │ (LLM-owned) <────│ A2: External Scrape │
113
- │ │ RSS, GitHub, web → raw/ │
114
- │ AGENTS.md │ │
115
- │ (schema) <────│ A3: Self-Improve │
116
- │ │ LLM Council → score → fix │
117
- ├──────────────────────┴─────────────────────────────┤
118
- │ LLM Providers │
119
- │ Claude (Anthropic) · OpenAI (GPT) · Ollama │
120
- ├────────────────────────────────────────────────────┤
121
- │ Processors │
122
- │ Text · PDF · Audio · Video · Image · URL · HTML │
123
- └────────────────────────────────────────────────────┘
124
- ```
68
+ Watch your knowledge base grow commit-by-commit. Every wiki change is checkpointed in git — scrub through the timeline to see pages appear, links form, and the graph densify.
125
69
 
126
- ### Three Layers
70
+ ### WYSIWYG Editing
127
71
 
128
- 1. **`raw/`** &mdash; Immutable source documents. Date-stamped subdirectories. Never modified by the LLM.
129
- 2. **`wiki/`** &mdash; LLM-generated markdown. Source summaries, entity pages, concept pages, synthesis pages. The LLM owns this entirely.
130
- 3. **`AGENTS.md`** &mdash; Schema file. Tells the LLM how the wiki is structured, what conventions to follow, how to process sources. Co-evolved by you and the LLM.
72
+ Click any wiki page to edit it inline. Markdown shortcuts, live preview, `Cmd+S` to save. Changes are auto-committed to git.
131
73
 
132
74
  ### Three Automations
133
75
 
134
- - **A1: Ingest & Process** &mdash; Detects file type, runs the appropriate processor (Whisper for audio, ffmpeg+Whisper for video, Claude Vision for images, text extraction for PDF), asks the LLM to produce wiki pages with cross-references.
135
- - **A2: External Scrape** &mdash; Fetches from RSS feeds, GitHub trending, web URLs. Deposits results in `raw/` and triggers A1 automatically.
136
- - **A3: Self-Improve** &mdash; LLM Council evaluates wiki quality across 5 dimensions (coverage, consistency, cross-linking, freshness, organization), proposes improvements, and applies them if below a configurable threshold.
137
-
138
- ## All Commands
76
+ | Automation | Trigger | What it does |
77
+ |------------|---------|--------------|
78
+ | **Ingest** | File watcher on `raw/` | New file detected process wiki pages git commit |
79
+ | **Scrape** | Cron schedule or manual | RSS feeds, GitHub trending, URLs → fetch → deposit in `raw/` → triggers Ingest |
80
+ | **Observe** | Nightly or manual | LLM Council scores wiki quality (coverage, consistency, cross-linking, freshness, organization) → proposes and applies improvements |
139
81
 
140
- ### `llmwiki init [directory]`
82
+ ### Git Checkpointing
141
83
 
142
- Create a new vault with the standard directory structure.
84
+ Every change committed automatically. Browse history, restore snapshots, see diffs. Your wiki is a git repo from day one.
143
85
 
144
- ```bash
145
- llmwiki init my-wiki # Create in my-wiki/
146
- llmwiki init . # Initialize current directory
147
- llmwiki init my-wiki --template research # Use research template
148
- llmwiki init my-wiki --force # Overwrite existing
149
- ```
86
+ ### Pipeline Visualization
150
87
 
151
- Templates: `personal` (default), `research`, `business`, `codebase`
88
+ See exactly how your document flows through the system — file detection, text extraction, LLM processing, page generation, cross-linking, indexing — step by step in the web UI.
152
89
 
153
- ### `llmwiki ingest <source>`
90
+ ### Connectors
154
91
 
155
- Process a file or URL into wiki pages.
92
+ Sync external sources into your vault automatically.
156
93
 
157
- ```bash
158
- llmwiki ingest paper.pdf # PDF → extract text → wiki pages
159
- llmwiki ingest podcast.mp3 # Audio Whisper transcription → wiki
160
- llmwiki ingest screenshot.png # Image Claude Vision description → wiki
161
- llmwiki ingest lecture.mp4 # Video ffmpeg → Whisper → wiki
162
- llmwiki ingest article.md # Markdown wiki pages
163
- llmwiki ingest data.json # JSON code block in wiki
164
- llmwiki ingest page.html # HTML strip tags → wiki
165
- llmwiki ingest report.docx # Office → basic extraction → wiki
166
- llmwiki ingest https://example.com/post # URL → Firecrawl/fetch → wiki
167
- llmwiki ingest raw/2026-04-07/file.md # Re-ingest from raw/
168
- ```
94
+ | Connector | Status |
95
+ |-----------|--------|
96
+ | Local folders | Shipped |
97
+ | Git repos | Shipped |
98
+ | GitHub | Shipped |
99
+ | Webhooks | Shipped |
100
+ | Slack | 🔜 Coming soon |
101
+ | Gmail | 🔜 Coming soon |
169
102
 
170
- Each source is auto-detected by file type, copied to `raw/{date}/`, checked for duplicates, compiled into wiki pages by the LLM, and indexed. Use `-p` to pick a provider, `-m` for a specific model, `--verbose` for detailed output.
103
+ ### MCP Server
171
104
 
172
- ### `llmwiki query <question>`
173
-
174
- Ask a question and get an answer synthesized from your wiki.
105
+ Use WikiMem as a tool inside Claude Code, Cursor, or any MCP-compatible client.
175
106
 
176
107
  ```bash
177
- llmwiki query "What are the main themes across my sources?"
178
- llmwiki query "Compare approaches to knowledge management" --file
179
- llmwiki query "Who is mentioned most frequently?" -p openai
108
+ wikimem mcp
180
109
  ```
181
110
 
182
- Use `--file` to save the answer as a synthesis page in `wiki/syntheses/`. The query engine uses BM25 search to find relevant pages, reads the top 10, and synthesizes an answer with `[[wikilink]]` citations.
183
-
184
- ### `llmwiki lint`
185
-
186
- Health-check the wiki for structural issues.
187
-
188
- ```bash
189
- llmwiki lint # Check for issues
190
- llmwiki lint --fix # Auto-fix where possible
111
+ ### Multiple LLMs
112
+
113
+ | Provider | Flag | Default Model |
114
+ |----------|------|---------------|
115
+ | Claude | `-p claude` | `claude-sonnet-4-20250514` |
116
+ | OpenAI | `-p openai` | `gpt-4o` |
117
+ | Ollama | `-p ollama` | `llama3.2` |
118
+
119
+ Ollama runs fully local no API keys, no network, no data leaves your machine.
120
+
121
+ ## CLI Reference
122
+
123
+ | Command | Description |
124
+ |---------|-------------|
125
+ | `wikimem init [dir]` | Create a new vault (`--template research\|business\|codebase`, `--from-folder`, `--from-repo`) |
126
+ | `wikimem serve` | Start the web IDE on port 3141 |
127
+ | `wikimem ingest <source>` | Process a file or URL into wiki pages |
128
+ | `wikimem search <term>` | BM25 full-text search across wiki pages |
129
+ | `wikimem ask <question>` | Ask a question, get an answer from your wiki |
130
+ | `wikimem query <question>` | Ask a question and optionally save as synthesis page (`--file`) |
131
+ | `wikimem lint` | Health-check: orphan pages, broken links, missing summaries (`--fix`) |
132
+ | `wikimem status` | Vault statistics: pages, words, sources, links, orphans |
133
+ | `wikimem watch` | Auto-ingest files dropped into `raw/` |
134
+ | `wikimem scrape` | Fetch from configured RSS/GitHub/URL sources |
135
+ | `wikimem improve` | Run self-improvement cycle (`--dry-run`, `--threshold 90`) |
136
+ | `wikimem export` | Export wiki to other formats |
137
+ | `wikimem open` | Open vault in Obsidian |
138
+ | `wikimem history` | Browse audit trail, restore snapshots |
139
+ | `wikimem mcp` | Start MCP server for Claude Code / Cursor |
140
+ | `wikimem duplicates` | Detect and manage near-duplicate sources |
141
+
142
+ ## Web UI
143
+
144
+ `wikimem serve` opens a full IDE at [localhost:3141](http://localhost:3141):
145
+
146
+ - **File tree** — browse wiki pages with collapsible folders
147
+ - **Tabbed editor** — open multiple pages, WYSIWYG markdown editing
148
+ - **Knowledge graph** — interactive D3 force-directed visualization
149
+ - **Pipeline view** — drag-and-drop file ingestion with step-by-step progress
150
+ - **Time-lapse** — scrub through git history to watch your wiki grow
151
+ - **Search** — `Cmd+K` fuzzy search across all pages
152
+ - **Command palette** — `Cmd+P` for quick actions
153
+ - **Settings** — configure API keys, models, and automations from the UI
154
+ - **Ask your knowledge** — query your wiki from the browser
155
+
156
+ ## MCP Server
157
+
158
+ WikiMem ships with a built-in MCP server so Claude Code and Cursor can read, search, and query your wiki directly.
159
+
160
+ **Add to Claude Code** (`.mcp.json`):
161
+
162
+ ```json
163
+ {
164
+ "mcpServers": {
165
+ "wikimem": {
166
+ "command": "npx",
167
+ "args": ["-y", "wikimem", "mcp"],
168
+ "env": {
169
+ "WIKIMEM_VAULT": "/path/to/your/vault"
170
+ }
171
+ }
172
+ }
173
+ }
191
174
  ```
192
175
 
193
- Checks for:
194
- - Orphan pages (no inbound `[[wikilinks]]`)
195
- - Broken wikilinks (links to non-existent pages)
196
- - Pages missing frontmatter summaries
197
- - Near-empty pages (< 10 words)
198
-
199
- Reports a quality score out of 100.
200
-
201
- ### `llmwiki watch`
202
-
203
- Watch the `raw/` directory and auto-ingest new files.
176
+ **Or run standalone:**
204
177
 
205
178
  ```bash
206
- llmwiki watch # Watch current vault
207
- llmwiki watch -v ./my-wiki # Watch a specific vault
179
+ wikimem-mcp
208
180
  ```
209
181
 
210
- Uses `chokidar` for reliable cross-platform file watching. Waits for writes to stabilize before ingesting (2-second debounce). Press `Ctrl+C` to stop.
211
-
212
- ### `llmwiki scrape`
213
-
214
- Fetch content from configured external sources and deposit in `raw/`.
215
-
216
- ```bash
217
- llmwiki scrape # Run all configured sources
218
- llmwiki scrape -s "HN Top" # Run a specific source
219
- ```
182
+ ## Configuration
220
183
 
221
- Sources are configured in `config.yaml`:
184
+ After `wikimem init`, your vault contains `config.yaml`:
222
185
 
223
186
  ```yaml
187
+ provider: claude # claude | openai | ollama
188
+ model: claude-sonnet-4-20250514
189
+
224
190
  sources:
225
- - name: "HN Top Stories"
191
+ - name: "HN Front Page"
226
192
  type: rss
227
193
  url: "https://hnrss.org/frontpage"
228
194
 
@@ -230,55 +196,11 @@ sources:
230
196
  type: github
231
197
  query: "stars:>100 created:>7d language:typescript"
232
198
 
233
- - name: "Company Blog"
234
- type: url
235
- url: "https://example.com/blog"
199
+ improvement:
200
+ threshold: 80
201
+ schedule: "0 3 * * *" # 3am nightly
236
202
  ```
237
203
 
238
- Supported source types: `rss`, `github`, `url`
239
-
240
- ### `llmwiki improve`
241
-
242
- Run the self-improvement cycle (Automation 3).
243
-
244
- ```bash
245
- llmwiki improve # Evaluate and improve
246
- llmwiki improve --dry-run # Show what would change
247
- llmwiki improve --threshold 90 # Stricter quality bar
248
- ```
249
-
250
- The improvement cycle:
251
-
252
- 1. **Score** &mdash; Evaluates 5 quality dimensions (coverage, consistency, cross-linking, freshness, organization)
253
- 2. **Decide** &mdash; If score < threshold (default 80), improvements are needed
254
- 3. **Improve** &mdash; Proposes actions: add cross-links, create missing pages, expand stubs, flag contradictions
255
- 4. **Log** &mdash; Records what changed and why in `log.md`
256
-
257
- ### `llmwiki status`
258
-
259
- Show vault statistics at a glance.
260
-
261
- ```bash
262
- llmwiki status
263
- ```
264
-
265
- ```
266
- llmwiki vault status
267
- ────────────────────────────────────
268
- Pages: 42
269
- Words: 18,340
270
- Sources: 15
271
- Wiki links: 127
272
- Orphan pages: 2
273
- Last updated: 2026-04-07
274
- ```
275
-
276
- ## Configuration
277
-
278
- After `llmwiki init`, your vault contains a `config.yaml` where you set the LLM provider, external sources, self-improvement schedule, and processing options.
279
-
280
- See [docs/configuration.md](docs/configuration.md) for the full reference.
281
-
282
204
  ### Environment Variables
283
205
 
284
206
  | Variable | Purpose |
@@ -286,113 +208,53 @@ See [docs/configuration.md](docs/configuration.md) for the full reference.
286
208
  | `ANTHROPIC_API_KEY` | Claude API access (default provider) |
287
209
  | `OPENAI_API_KEY` | OpenAI API access |
288
210
  | `OLLAMA_BASE_URL` | Ollama server URL (default: `http://localhost:11434`) |
289
- | `FIRECRAWL_API_KEY` | Enhanced URL-to-markdown (optional, falls back to fetch) |
290
- | `DEEPGRAM_API_KEY` | Audio transcription API (optional, falls back to Whisper) |
291
-
292
- ## Multi-Model Support
211
+ | `FIRECRAWL_API_KEY` | Enhanced URL-to-markdown (optional) |
212
+ | `DEEPGRAM_API_KEY` | Audio transcription (optional, falls back to Whisper) |
293
213
 
294
- llmwiki works with any major LLM provider. Choose at init time or per-command.
295
-
296
- | Provider | Flag | Default Model | Env Variable |
297
- |----------|------|---------------|-------------|
298
- | **Claude** | `-p claude` | `claude-sonnet-4-20250514` | `ANTHROPIC_API_KEY` |
299
- | **OpenAI** | `-p openai` | `gpt-4o` | `OPENAI_API_KEY` |
300
- | **Ollama** | `-p ollama` | `llama3.2` | `OLLAMA_BASE_URL` |
301
-
302
- ```bash
303
- # Use Claude (default)
304
- llmwiki ingest paper.pdf
305
-
306
- # Use OpenAI
307
- llmwiki ingest paper.pdf -p openai -m gpt-4o-mini
308
-
309
- # Use Ollama (fully local, no API keys)
310
- llmwiki ingest paper.pdf -p ollama -m llama3.2
311
- ```
312
-
313
- ## Multi-Format Support
314
-
315
- | Format | Extensions | Processor | Requirements |
316
- |--------|-----------|-----------|-------------|
317
- | **Text** | `.md`, `.txt`, `.csv` | Direct read | None |
318
- | **PDF** | `.pdf` | Built-in text extraction | None |
319
- | **Audio** | `.mp3`, `.wav`, `.m4a`, `.ogg`, `.flac`, `.aac` | Whisper / Deepgram | `whisper` CLI or `DEEPGRAM_API_KEY` |
320
- | **Video** | `.mp4`, `.mov`, `.avi`, `.mkv`, `.webm` | ffmpeg + Whisper | `ffmpeg` + `whisper` |
321
- | **Image** | `.jpg`, `.png`, `.gif`, `.webp` | Claude Vision | `ANTHROPIC_API_KEY` |
322
- | **HTML** | `.html`, `.htm` | Tag stripping | None |
323
- | **JSON** | `.json` | Code block wrapping | None |
324
- | **Office** | `.docx`, `.pptx`, `.xlsx` | Basic extraction | None (enhanced coming) |
325
- | **URL** | `https://...` | Firecrawl / fetch | Optional `FIRECRAWL_API_KEY` |
326
-
327
- When a processor's requirements are not met (e.g., Whisper not installed for audio), llmwiki creates a reference page noting the source file and suggests installing the missing tool. The raw file is always preserved.
328
-
329
- ## Obsidian Integration
330
-
331
- llmwiki vaults are Obsidian vaults. Open any llmwiki directory in Obsidian and you get:
332
-
333
- - **Graph view** showing all pages and their `[[wikilinks]]`
334
- - **YAML frontmatter** rendered as page metadata
335
- - **Backlinks** panel showing what links to each page
336
- - **Search** across all wiki content
337
- - **Tag view** from frontmatter `tags:` arrays
338
-
339
- No plugins required. No configuration. Just `Open folder as vault` in Obsidian.
214
+ ## Architecture
340
215
 
341
216
  ```
342
- # Every wiki page has frontmatter like this:
343
- ---
344
- title: "Attention Is All You Need"
345
- type: source
346
- created: "2026-04-07"
347
- updated: "2026-04-07"
348
- tags: [transformers, attention, nlp]
349
- sources: ["raw/2026-04-07/attention-paper.pdf"]
350
- summary: "Foundational transformer architecture paper introducing self-attention"
351
- ---
217
+ vault/
218
+ ├── wiki/ ← LLM-generated pages (sources/, entities/, concepts/, syntheses/)
219
+ ├── raw/ ← Immutable source documents (date-stamped subdirectories)
220
+ ├── AGENTS.md ← Schema — wiki structure + conventions
221
+ ├── config.yaml ← Configuration — provider, sources, schedules
222
+ └── index.md ← Content catalog (auto-maintained)
352
223
  ```
353
224
 
354
- ## Vault Structure
225
+ **Three layers:** `raw/` (immutable sources) → LLM processing → `wiki/` (structured knowledge). `AGENTS.md` is the schema file that tells the LLM how to structure output — it co-evolves with your wiki.
355
226
 
356
- ```
357
- my-wiki/
358
- ├── AGENTS.md # Schema — wiki structure + conventions
359
- ├── config.yaml # Configuration — provider, sources, schedules
360
- ├── .gitignore
361
- ├── raw/ # Immutable source archive
362
- │ ├── 2026-04-07/
363
- │ │ ├── paper.pdf
364
- │ │ ├── podcast.mp3
365
- │ │ └── blog-post.md
366
- │ └── 2026-04-08/
367
- │ └── meeting-notes.md
368
- └── wiki/ # LLM-generated knowledge base
369
- ├── index.md # Auto-maintained content catalog
370
- ├── log.md # Chronological operation record
371
- ├── sources/ # One summary per ingested source
372
- ├── entities/ # People, organizations, tools
373
- ├── concepts/ # Ideas, frameworks, patterns
374
- └── syntheses/ # Cross-cutting analyses, query results
375
- ```
227
+ **Three automations:** Ingest (file watcher → process → wiki pages), Scrape (RSS/GitHub/URLs → raw/), Observe (LLM Council → score → improve).
376
228
 
377
- ## Tests
229
+ ## Obsidian Integration
378
230
 
379
- ```bash
380
- cd /path/to/llmwiki && pnpm test
381
- ```
231
+ WikiMem vaults are Obsidian vaults. Open any wikimem directory in Obsidian — no plugins, no configuration:
382
232
 
383
- ## Contributing
233
+ - `[[wikilinks]]` rendered as backlinks
234
+ - YAML frontmatter as page metadata
235
+ - Graph view showing all connections
236
+ - Tag view from frontmatter `tags:` arrays
384
237
 
385
- See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
238
+ ## Privacy
386
239
 
387
- ## See Also
240
+ - Everything runs locally. Your wiki is a folder of markdown files.
241
+ - No data sent anywhere except LLM API calls (and those are optional with Ollama).
242
+ - `raw/` excluded from git by default — your source documents stay private.
243
+ - `config.yaml` excluded from git — API keys never committed.
388
244
 
389
- - **[agentgrid](https://github.com/naman10parikh/agentgrid)** &mdash; Manage grids of AI coding agents in tmux
390
- - **[agentdial](https://github.com/naman10parikh/Energy)** &mdash; Universal agent identity protocol across 8 messaging channels
245
+ | Path | Safe to commit? | Why |
246
+ |------|:-:|-----|
247
+ | `wiki/` | ✅ | LLM-generated summaries, no raw personal data |
248
+ | `AGENTS.md` | ✅ | Schema file, no personal data |
249
+ | `raw/` | ❌ | Original source files |
250
+ | `config.yaml` | ❌ | May contain API keys |
391
251
 
392
252
  ## Credits
393
253
 
394
- Inspired by [Andrej Karpathy's LLM Wiki pattern](https://x.com/karpathy/status/1908625766490001799) &mdash; the idea that LLMs should compile knowledge into structured, interlinked wikis rather than just answering questions from raw chunks.
254
+ Inspired by [Andrej Karpathy's LLM Wiki pattern](https://x.com/karpathy/status/1908625766490001799) the idea that LLMs should compile knowledge into structured, interlinked wikis rather than just answering questions from raw chunks.
255
+
256
+ Built with [Express](https://expressjs.com/), [D3](https://d3js.org/), [simple-git](https://github.com/steveukx/git-js), and the [Claude](https://docs.anthropic.com/) / [OpenAI](https://platform.openai.com/) / [Ollama](https://ollama.com/) APIs.
395
257
 
396
258
  ## License
397
259
 
398
- MIT &mdash; see [LICENSE](LICENSE).
260
+ MIT see [LICENSE](LICENSE).
@@ -0,0 +1,3 @@
1
+ import { Command } from 'commander';
2
+ export declare function registerAskCommand(program: Command): void;
3
+ //# sourceMappingURL=ask.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ask.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/ask.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiBpC,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CA2DzD"}
@@ -0,0 +1,63 @@
1
+ import chalk from 'chalk';
2
+ import ora from 'ora';
3
+ import { resolve } from 'node:path';
4
+ import { existsSync } from 'node:fs';
5
+ import { queryWiki } from '../../core/query.js';
6
+ import { getVaultConfig } from '../../core/vault.js';
7
+ import { createProviderFromUserConfig } from '../../providers/index.js';
8
+ import { loadConfig } from '../../core/config.js';
9
+ export function registerAskCommand(program) {
10
+ program
11
+ .command('ask <question>')
12
+ .description('Ask a question — LLM-powered Q&A against the wiki')
13
+ .option('-v, --vault <path>', 'Vault root directory', '.')
14
+ .option('-p, --provider <provider>', 'LLM provider (claude, openai, ollama)')
15
+ .option('-m, --model <model>', 'Model to use')
16
+ .option('--file', 'Save the answer as a new wiki page')
17
+ .action(async (question, options) => {
18
+ const vaultRoot = resolve(options.vault ?? '.');
19
+ const config = getVaultConfig(vaultRoot);
20
+ const userConfig = loadConfig(config.configPath);
21
+ if (!existsSync(config.schemaPath)) {
22
+ console.error(chalk.red('Not a wikimem vault. Run `wikimem init` first.'));
23
+ process.exit(1);
24
+ }
25
+ const provider = createProviderFromUserConfig(userConfig, {
26
+ providerOverride: options.provider,
27
+ model: options.model,
28
+ });
29
+ const spinner = ora('Thinking…').start();
30
+ try {
31
+ const result = await queryWiki(question, config, provider, {
32
+ fileBack: options.file ?? false,
33
+ });
34
+ spinner.stop();
35
+ console.log();
36
+ const formatted = result.answer
37
+ .replace(/^(#{1,3})\s+(.+)$/gm, (_m, hashes, text) => {
38
+ if (hashes === '#')
39
+ return chalk.bold.underline(text);
40
+ if (hashes === '##')
41
+ return chalk.bold(text);
42
+ return chalk.italic(text);
43
+ })
44
+ .replace(/\[\[([^\]]+)\]\]/g, (_m, link) => chalk.cyan(`[[${link}]]`))
45
+ .replace(/`([^`]+)`/g, (_m, code) => chalk.yellow(code))
46
+ .replace(/\*\*([^*]+)\*\*/g, (_m, bold) => chalk.bold(bold));
47
+ console.log(formatted);
48
+ console.log();
49
+ if (result.sourcesConsulted.length > 0) {
50
+ console.log(chalk.dim(`Sources: ${result.sourcesConsulted.join(', ')}`));
51
+ }
52
+ if (result.filedAs) {
53
+ console.log(chalk.green(`Saved to: ${result.filedAs}`));
54
+ }
55
+ }
56
+ catch (error) {
57
+ spinner.fail('Failed');
58
+ console.error(chalk.red(error instanceof Error ? error.message : String(error)));
59
+ process.exit(1);
60
+ }
61
+ });
62
+ }
63
+ //# sourceMappingURL=ask.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ask.js","sourceRoot":"","sources":["../../../src/cli/commands/ask.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,4BAA4B,EAAE,MAAM,0BAA0B,CAAC;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AASlD,MAAM,UAAU,kBAAkB,CAAC,OAAgB;IACjD,OAAO;SACJ,OAAO,CAAC,gBAAgB,CAAC;SACzB,WAAW,CAAC,mDAAmD,CAAC;SAChE,MAAM,CAAC,oBAAoB,EAAE,sBAAsB,EAAE,GAAG,CAAC;SACzD,MAAM,CAAC,2BAA2B,EAAE,uCAAuC,CAAC;SAC5E,MAAM,CAAC,qBAAqB,EAAE,cAAc,CAAC;SAC7C,MAAM,CAAC,QAAQ,EAAE,oCAAoC,CAAC;SACtD,MAAM,CAAC,KAAK,EAAE,QAAgB,EAAE,OAAmB,EAAE,EAAE;QACtD,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAEjD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC,CAAC;YAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,QAAQ,GAAG,4BAA4B,CAAC,UAAU,EAAE;YACxD,gBAAgB,EAAE,OAAO,CAAC,QAAQ;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE;gBACzD,QAAQ,EAAE,OAAO,CAAC,IAAI,IAAI,KAAK;aAChC,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,EAAE,CAAC;YAEd,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM;iBAC5B,OAAO,CAAC,qBAAqB,EAAE,CAAC,EAAE,EAAE,MAAc,EAAE,IAAY,EAAE,EAAE;gBACnE,IAAI,MAAM,KAAK,GAAG;oBAAE,OAAO,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBACtD,IAAI,MAAM,KAAK,IAAI;oBAAE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7C,OAAO,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC,CAAC;iBACD,OAAO,CAAC,mBAAmB,EAAE,CAAC,EAAE,EAAE,IAAY,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC;iBAC7E,OAAO,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,IAAY,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;iBAC/D,OAAO,CAAC,kBAAkB,EAAE,CAAC,EAAE,EAAE,IAAY,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAEvE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACvB,OAAO,CAAC,GAAG,EAAE,CAAC;YAEd,IAAI,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAC3E,CAAC;YAED,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACjF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}