wikimem 0.3.0 β†’ 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/CHANGELOG.md +138 -29
  2. package/README.md +171 -309
  3. package/dist/cli/commands/ask.d.ts +3 -0
  4. package/dist/cli/commands/ask.d.ts.map +1 -0
  5. package/dist/cli/commands/ask.js +63 -0
  6. package/dist/cli/commands/ask.js.map +1 -0
  7. package/dist/cli/commands/export.d.ts +3 -0
  8. package/dist/cli/commands/export.d.ts.map +1 -0
  9. package/dist/cli/commands/export.js +108 -0
  10. package/dist/cli/commands/export.js.map +1 -0
  11. package/dist/cli/commands/history.d.ts +3 -0
  12. package/dist/cli/commands/history.d.ts.map +1 -0
  13. package/dist/cli/commands/history.js +61 -0
  14. package/dist/cli/commands/history.js.map +1 -0
  15. package/dist/cli/commands/improve.d.ts.map +1 -1
  16. package/dist/cli/commands/improve.js +4 -3
  17. package/dist/cli/commands/improve.js.map +1 -1
  18. package/dist/cli/commands/ingest.d.ts.map +1 -1
  19. package/dist/cli/commands/ingest.js +5 -4
  20. package/dist/cli/commands/ingest.js.map +1 -1
  21. package/dist/cli/commands/init.d.ts.map +1 -1
  22. package/dist/cli/commands/init.js +337 -81
  23. package/dist/cli/commands/init.js.map +1 -1
  24. package/dist/cli/commands/lint.d.ts.map +1 -1
  25. package/dist/cli/commands/lint.js +4 -3
  26. package/dist/cli/commands/lint.js.map +1 -1
  27. package/dist/cli/commands/mcp.d.ts +3 -0
  28. package/dist/cli/commands/mcp.d.ts.map +1 -0
  29. package/dist/cli/commands/mcp.js +11 -0
  30. package/dist/cli/commands/mcp.js.map +1 -0
  31. package/dist/cli/commands/open.d.ts +3 -0
  32. package/dist/cli/commands/open.d.ts.map +1 -0
  33. package/dist/cli/commands/open.js +36 -0
  34. package/dist/cli/commands/open.js.map +1 -0
  35. package/dist/cli/commands/query.d.ts.map +1 -1
  36. package/dist/cli/commands/query.js +5 -4
  37. package/dist/cli/commands/query.js.map +1 -1
  38. package/dist/cli/commands/search.d.ts +3 -0
  39. package/dist/cli/commands/search.d.ts.map +1 -0
  40. package/dist/cli/commands/search.js +61 -0
  41. package/dist/cli/commands/search.js.map +1 -0
  42. package/dist/cli/commands/serve.d.ts.map +1 -1
  43. package/dist/cli/commands/serve.js +41 -2
  44. package/dist/cli/commands/serve.js.map +1 -1
  45. package/dist/cli/commands/watch.d.ts.map +1 -1
  46. package/dist/cli/commands/watch.js +4 -3
  47. package/dist/cli/commands/watch.js.map +1 -1
  48. package/dist/cli/index.d.ts.map +1 -1
  49. package/dist/cli/index.js +27 -1
  50. package/dist/cli/index.js.map +1 -1
  51. package/dist/core/audit-trail.d.ts +15 -0
  52. package/dist/core/audit-trail.d.ts.map +1 -0
  53. package/dist/core/audit-trail.js +43 -0
  54. package/dist/core/audit-trail.js.map +1 -0
  55. package/dist/core/claude-code.d.ts +10 -0
  56. package/dist/core/claude-code.d.ts.map +1 -0
  57. package/dist/core/claude-code.js +81 -0
  58. package/dist/core/claude-code.js.map +1 -0
  59. package/dist/core/config.d.ts +23 -0
  60. package/dist/core/config.d.ts.map +1 -1
  61. package/dist/core/config.js.map +1 -1
  62. package/dist/core/connectors.d.ts +58 -0
  63. package/dist/core/connectors.d.ts.map +1 -0
  64. package/dist/core/connectors.js +189 -0
  65. package/dist/core/connectors.js.map +1 -0
  66. package/dist/core/folder-scanner.d.ts +10 -0
  67. package/dist/core/folder-scanner.d.ts.map +1 -0
  68. package/dist/core/folder-scanner.js +84 -0
  69. package/dist/core/folder-scanner.js.map +1 -0
  70. package/dist/core/git.d.ts +137 -0
  71. package/dist/core/git.d.ts.map +1 -0
  72. package/dist/core/git.js +520 -0
  73. package/dist/core/git.js.map +1 -0
  74. package/dist/core/history.d.ts +21 -0
  75. package/dist/core/history.d.ts.map +1 -0
  76. package/dist/core/history.js +107 -0
  77. package/dist/core/history.js.map +1 -0
  78. package/dist/core/improve.d.ts.map +1 -1
  79. package/dist/core/improve.js +9 -0
  80. package/dist/core/improve.js.map +1 -1
  81. package/dist/core/ingest.d.ts +1 -0
  82. package/dist/core/ingest.d.ts.map +1 -1
  83. package/dist/core/ingest.js +151 -7
  84. package/dist/core/ingest.js.map +1 -1
  85. package/dist/core/lint.d.ts.map +1 -1
  86. package/dist/core/lint.js +23 -4
  87. package/dist/core/lint.js.map +1 -1
  88. package/dist/core/oauth-defaults.d.ts +31 -0
  89. package/dist/core/oauth-defaults.d.ts.map +1 -0
  90. package/dist/core/oauth-defaults.js +77 -0
  91. package/dist/core/oauth-defaults.js.map +1 -0
  92. package/dist/core/observer.d.ts +94 -0
  93. package/dist/core/observer.d.ts.map +1 -0
  94. package/dist/core/observer.js +492 -0
  95. package/dist/core/observer.js.map +1 -0
  96. package/dist/core/pipeline-events.d.ts +63 -0
  97. package/dist/core/pipeline-events.d.ts.map +1 -0
  98. package/dist/core/pipeline-events.js +109 -0
  99. package/dist/core/pipeline-events.js.map +1 -0
  100. package/dist/core/query.d.ts.map +1 -1
  101. package/dist/core/query.js +16 -8
  102. package/dist/core/query.js.map +1 -1
  103. package/dist/core/scraper.d.ts +41 -0
  104. package/dist/core/scraper.d.ts.map +1 -0
  105. package/dist/core/scraper.js +277 -0
  106. package/dist/core/scraper.js.map +1 -0
  107. package/dist/core/sync/gdrive.d.ts +14 -0
  108. package/dist/core/sync/gdrive.d.ts.map +1 -0
  109. package/dist/core/sync/gdrive.js +205 -0
  110. package/dist/core/sync/gdrive.js.map +1 -0
  111. package/dist/core/sync/github.d.ts +20 -0
  112. package/dist/core/sync/github.d.ts.map +1 -0
  113. package/dist/core/sync/github.js +206 -0
  114. package/dist/core/sync/github.js.map +1 -0
  115. package/dist/core/sync/gmail.d.ts +15 -0
  116. package/dist/core/sync/gmail.d.ts.map +1 -0
  117. package/dist/core/sync/gmail.js +159 -0
  118. package/dist/core/sync/gmail.js.map +1 -0
  119. package/dist/core/sync/index.d.ts +47 -0
  120. package/dist/core/sync/index.d.ts.map +1 -0
  121. package/dist/core/sync/index.js +100 -0
  122. package/dist/core/sync/index.js.map +1 -0
  123. package/dist/core/sync/jira.d.ts +15 -0
  124. package/dist/core/sync/jira.d.ts.map +1 -0
  125. package/dist/core/sync/jira.js +176 -0
  126. package/dist/core/sync/jira.js.map +1 -0
  127. package/dist/core/sync/linear.d.ts +15 -0
  128. package/dist/core/sync/linear.d.ts.map +1 -0
  129. package/dist/core/sync/linear.js +111 -0
  130. package/dist/core/sync/linear.js.map +1 -0
  131. package/dist/core/sync/notion.d.ts +14 -0
  132. package/dist/core/sync/notion.d.ts.map +1 -0
  133. package/dist/core/sync/notion.js +168 -0
  134. package/dist/core/sync/notion.js.map +1 -0
  135. package/dist/core/sync/rss.d.ts +20 -0
  136. package/dist/core/sync/rss.d.ts.map +1 -0
  137. package/dist/core/sync/rss.js +165 -0
  138. package/dist/core/sync/rss.js.map +1 -0
  139. package/dist/core/sync/scheduler.d.ts +31 -0
  140. package/dist/core/sync/scheduler.d.ts.map +1 -0
  141. package/dist/core/sync/scheduler.js +129 -0
  142. package/dist/core/sync/scheduler.js.map +1 -0
  143. package/dist/core/sync/slack.d.ts +16 -0
  144. package/dist/core/sync/slack.d.ts.map +1 -0
  145. package/dist/core/sync/slack.js +173 -0
  146. package/dist/core/sync/slack.js.map +1 -0
  147. package/dist/core/vault.d.ts +22 -0
  148. package/dist/core/vault.d.ts.map +1 -1
  149. package/dist/core/vault.js +65 -0
  150. package/dist/core/vault.js.map +1 -1
  151. package/dist/core/webhooks.d.ts +13 -0
  152. package/dist/core/webhooks.d.ts.map +1 -0
  153. package/dist/core/webhooks.js +206 -0
  154. package/dist/core/webhooks.js.map +1 -0
  155. package/dist/index.js +3 -2
  156. package/dist/index.js.map +1 -1
  157. package/dist/mcp-entry.d.ts +10 -0
  158. package/dist/mcp-entry.d.ts.map +1 -0
  159. package/dist/mcp-entry.js +21 -0
  160. package/dist/mcp-entry.js.map +1 -0
  161. package/dist/mcp-server.d.ts +20 -0
  162. package/dist/mcp-server.d.ts.map +1 -0
  163. package/dist/mcp-server.js +483 -0
  164. package/dist/mcp-server.js.map +1 -0
  165. package/dist/mcp-tools-extended.d.ts +15 -0
  166. package/dist/mcp-tools-extended.d.ts.map +1 -0
  167. package/dist/mcp-tools-extended.js +277 -0
  168. package/dist/mcp-tools-extended.js.map +1 -0
  169. package/dist/processors/audio.d.ts.map +1 -1
  170. package/dist/processors/audio.js +42 -4
  171. package/dist/processors/audio.js.map +1 -1
  172. package/dist/processors/csv.d.ts +18 -0
  173. package/dist/processors/csv.d.ts.map +1 -0
  174. package/dist/processors/csv.js +230 -0
  175. package/dist/processors/csv.js.map +1 -0
  176. package/dist/processors/image.d.ts.map +1 -1
  177. package/dist/processors/image.js +55 -27
  178. package/dist/processors/image.js.map +1 -1
  179. package/dist/processors/pdf.d.ts.map +1 -1
  180. package/dist/processors/pdf.js +6 -3
  181. package/dist/processors/pdf.js.map +1 -1
  182. package/dist/processors/pptx.d.ts +3 -1
  183. package/dist/processors/pptx.d.ts.map +1 -1
  184. package/dist/processors/pptx.js +236 -95
  185. package/dist/processors/pptx.js.map +1 -1
  186. package/dist/processors/url.js +4 -1
  187. package/dist/processors/url.js.map +1 -1
  188. package/dist/processors/xlsx.d.ts +2 -0
  189. package/dist/processors/xlsx.d.ts.map +1 -1
  190. package/dist/processors/xlsx.js +182 -46
  191. package/dist/processors/xlsx.js.map +1 -1
  192. package/dist/providers/claude.d.ts +1 -0
  193. package/dist/providers/claude.d.ts.map +1 -1
  194. package/dist/providers/claude.js +5 -3
  195. package/dist/providers/claude.js.map +1 -1
  196. package/dist/providers/index.d.ts +17 -1
  197. package/dist/providers/index.d.ts.map +1 -1
  198. package/dist/providers/index.js +144 -0
  199. package/dist/providers/index.js.map +1 -1
  200. package/dist/providers/openai.d.ts +1 -0
  201. package/dist/providers/openai.d.ts.map +1 -1
  202. package/dist/providers/openai.js +5 -3
  203. package/dist/providers/openai.js.map +1 -1
  204. package/dist/providers/types.d.ts +18 -0
  205. package/dist/providers/types.d.ts.map +1 -1
  206. package/dist/templates/config-yaml.d.ts.map +1 -1
  207. package/dist/templates/config-yaml.js +12 -1
  208. package/dist/templates/config-yaml.js.map +1 -1
  209. package/dist/templates/source-types.d.ts +33 -0
  210. package/dist/templates/source-types.d.ts.map +1 -0
  211. package/dist/templates/source-types.js +178 -0
  212. package/dist/templates/source-types.js.map +1 -0
  213. package/dist/web/public/index.html +9836 -742
  214. package/dist/web/server.d.ts.map +1 -1
  215. package/dist/web/server.js +2823 -43
  216. package/dist/web/server.js.map +1 -1
  217. package/package.json +10 -4
  218. package/scripts/install.sh +54 -0
  219. package/src/web/public/index.html +9836 -742
  220. package/templates/mcp-config.json +9 -0
  221. package/templates/source-types/article.md +21 -0
  222. package/templates/source-types/book.md +21 -0
  223. package/templates/source-types/paper.md +23 -0
  224. package/templates/source-types/podcast.md +21 -0
  225. package/templates/source-types/raw-notes.md +17 -0
  226. package/templates/source-types/tweet-thread.md +19 -0
  227. package/templates/source-types/video.md +21 -0
  228. package/dist/web/public/public/index.html +0 -946
@@ -1,8 +1,10 @@
1
1
  import express from 'express';
2
- import { existsSync, writeFileSync, mkdirSync, readdirSync, statSync } from 'node:fs';
3
- import { join, basename } from 'node:path';
2
+ import { readFileSync, existsSync, writeFileSync, mkdirSync, readdirSync, statSync, renameSync, unlinkSync as fsUnlinkSync } from 'node:fs';
3
+ import { join, resolve, extname, basename, dirname, relative } from 'node:path';
4
4
  import { fileURLToPath } from 'node:url';
5
- import { getVaultConfig, getVaultStats, listWikiPages, readWikiPage } from '../core/vault.js';
5
+ import { randomBytes } from 'node:crypto';
6
+ import { getVaultConfig, getVaultStats, listWikiPages, readWikiPage, writeWikiPage, readPageVersions } from '../core/vault.js';
7
+ import { getBundledCredentials, getBundledDeviceFlowClientId } from '../core/oauth-defaults.js';
6
8
  const __dirname = fileURLToPath(new URL('.', import.meta.url));
7
9
  function buildGraph(config) {
8
10
  const pages = listWikiPages(config.wikiDir);
@@ -71,13 +73,13 @@ function listPages(config) {
71
73
  export function createServer(vaultRoot, port) {
72
74
  const app = express();
73
75
  const config = getVaultConfig(vaultRoot);
76
+ // Load persisted pipeline runs so they survive server restarts
77
+ import('../core/pipeline-events.js').then(({ pipelineEvents }) => {
78
+ pipelineEvents.initPersistence(vaultRoot);
79
+ }).catch(() => { });
74
80
  app.use(express.json());
75
81
  app.use(express.urlencoded({ extended: true }));
76
- // Serve static files
77
82
  const publicDir = join(__dirname, 'public');
78
- if (existsSync(publicDir)) {
79
- app.use(express.static(publicDir));
80
- }
81
83
  // API: vault status
82
84
  app.get('/api/status', (_req, res) => {
83
85
  try {
@@ -98,6 +100,36 @@ export function createServer(vaultRoot, port) {
98
100
  res.status(500).json({ error: 'Failed to list pages' });
99
101
  }
100
102
  });
103
+ // API: create new page
104
+ app.post('/api/pages', (req, res) => {
105
+ try {
106
+ const { title, slug, content } = req.body;
107
+ if (!title || !slug) {
108
+ res.status(400).json({ error: 'Missing title or slug' });
109
+ return;
110
+ }
111
+ if (!/^[a-zA-Z0-9_-]+$/.test(slug)) {
112
+ res.status(400).json({ error: 'Invalid slug β€” only alphanumeric, hyphens, and underscores allowed' });
113
+ return;
114
+ }
115
+ const dest = join(config.wikiDir, `${slug}.md`);
116
+ if (!resolve(dest).startsWith(resolve(config.wikiDir))) {
117
+ res.status(403).json({ error: 'Path traversal denied' });
118
+ return;
119
+ }
120
+ if (existsSync(dest)) {
121
+ res.status(409).json({ error: 'Page already exists' });
122
+ return;
123
+ }
124
+ mkdirSync(config.wikiDir, { recursive: true });
125
+ writeFileSync(dest, content ?? `---\ntitle: "${title}"\ntype: page\n---\n\n# ${title}\n`, 'utf-8');
126
+ res.json({ status: 'created', path: dest, title });
127
+ }
128
+ catch (err) {
129
+ const msg = err instanceof Error ? err.message : String(err);
130
+ res.status(500).json({ error: `Failed to create page: ${msg}` });
131
+ }
132
+ });
101
133
  // API: read single page
102
134
  app.get('/api/pages/:title', (req, res) => {
103
135
  try {
@@ -133,6 +165,357 @@ export function createServer(vaultRoot, port) {
133
165
  res.status(500).json({ error: 'Failed to read page' });
134
166
  }
135
167
  });
168
+ // API: read raw page content (for editing)
169
+ app.get('/api/wiki/page/raw', (req, res) => {
170
+ try {
171
+ const pagePath = req.query['path'];
172
+ if (!pagePath) {
173
+ res.status(400).json({ error: 'Missing path query parameter' });
174
+ return;
175
+ }
176
+ const resolved = resolve(pagePath);
177
+ if (!resolved.startsWith(resolve(config.wikiDir))) {
178
+ res.status(403).json({ error: 'Access denied: path outside wiki directory' });
179
+ return;
180
+ }
181
+ if (!existsSync(resolved)) {
182
+ res.status(404).json({ error: 'File not found' });
183
+ return;
184
+ }
185
+ const raw = readFileSync(resolved, 'utf-8');
186
+ res.json({ path: resolved, raw });
187
+ }
188
+ catch (err) {
189
+ const msg = err instanceof Error ? err.message : String(err);
190
+ res.status(500).json({ error: `Failed to read raw page: ${msg}` });
191
+ }
192
+ });
193
+ // API: save wiki page (write to disk + auto-commit)
194
+ app.put('/api/wiki/page', async (req, res) => {
195
+ try {
196
+ const { path: pagePath, content, frontmatter } = req.body;
197
+ if (!pagePath || content === undefined) {
198
+ res.status(400).json({ error: 'Missing path or content' });
199
+ return;
200
+ }
201
+ const resolved = resolve(pagePath);
202
+ if (!resolved.startsWith(resolve(config.wikiDir))) {
203
+ res.status(403).json({ error: 'Access denied: path outside wiki directory' });
204
+ return;
205
+ }
206
+ // Parse the raw content β€” if it includes frontmatter, use gray-matter
207
+ const matter = await import('gray-matter');
208
+ const parsed = matter.default(content);
209
+ const finalFrontmatter = frontmatter ?? parsed.data;
210
+ const finalContent = frontmatter ? content : parsed.content;
211
+ writeWikiPage(resolved, finalContent, finalFrontmatter);
212
+ // Auto-commit if git-initialized
213
+ let commitResult = null;
214
+ try {
215
+ const { autoCommit, isGitRepo } = await import('../core/git.js');
216
+ if (await isGitRepo(config.root)) {
217
+ const title = finalFrontmatter['title'] ?? basename(resolved, '.md');
218
+ commitResult = await autoCommit(config.root, 'manual', `edit "${title}" via web UI`);
219
+ }
220
+ }
221
+ catch { /* non-fatal */ }
222
+ res.json({ status: 'saved', path: resolved, commit: commitResult });
223
+ }
224
+ catch (err) {
225
+ const msg = err instanceof Error ? err.message : String(err);
226
+ res.status(500).json({ error: `Failed to save page: ${msg}` });
227
+ }
228
+ });
229
+ // API: read raw page content (full markdown with frontmatter)
230
+ app.get('/api/pages/:title/raw', (req, res) => {
231
+ try {
232
+ const title = req.params['title'];
233
+ if (!title) {
234
+ res.status(400).json({ error: 'Missing title' });
235
+ return;
236
+ }
237
+ const pages = listWikiPages(config.wikiDir);
238
+ const titleLower = title.toLowerCase();
239
+ const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
240
+ const match = pages.find((p) => {
241
+ const fileSlug = basename(p, '.md');
242
+ if (fileSlug === title || fileSlug === slugified)
243
+ return true;
244
+ try {
245
+ const page = readWikiPage(p);
246
+ return page.title.toLowerCase() === titleLower;
247
+ }
248
+ catch {
249
+ return false;
250
+ }
251
+ });
252
+ if (!match) {
253
+ res.status(404).json({ error: 'Page not found' });
254
+ return;
255
+ }
256
+ const raw = readFileSync(match, 'utf-8');
257
+ res.json({ raw, path: match, slug: basename(match, '.md') });
258
+ }
259
+ catch (err) {
260
+ res.status(500).json({ error: 'Failed to read raw page' });
261
+ }
262
+ });
263
+ // API: get page version history (COMP-MP-002 temporal reasoning)
264
+ app.get('/api/pages/:title/versions', (req, res) => {
265
+ try {
266
+ const title = req.params['title'];
267
+ if (!title) {
268
+ res.status(400).json({ error: 'Missing title' });
269
+ return;
270
+ }
271
+ const slug = title.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
272
+ const versions = readPageVersions(config.root, slug);
273
+ // Also include current page as the "latest" entry
274
+ const pages = listWikiPages(config.wikiDir);
275
+ const titleLower = title.toLowerCase();
276
+ const match = pages.find((p) => {
277
+ const fileSlug = basename(p, '.md');
278
+ if (fileSlug === slug)
279
+ return true;
280
+ try {
281
+ return readWikiPage(p).title.toLowerCase() === titleLower;
282
+ }
283
+ catch {
284
+ return false;
285
+ }
286
+ });
287
+ let current = null;
288
+ if (match) {
289
+ try {
290
+ const page = readWikiPage(match);
291
+ current = {
292
+ version: page.frontmatter['fact_version'] ?? versions.length + 1,
293
+ timestamp: page.frontmatter['learned_at'] ?? page.frontmatter['updated'] ?? new Date().toISOString(),
294
+ content: page.content,
295
+ frontmatter: page.frontmatter,
296
+ source: page.frontmatter['sources']?.[0],
297
+ actor: page.frontmatter['added_by'] ?? 'unknown',
298
+ };
299
+ }
300
+ catch { /* unreadable */ }
301
+ }
302
+ res.json({ versions, current, total: versions.length + (current ? 1 : 0) });
303
+ }
304
+ catch (err) {
305
+ res.status(500).json({ error: 'Failed to read page versions' });
306
+ }
307
+ });
308
+ // API: update page content (full markdown with frontmatter)
309
+ app.put('/api/pages/:title', async (req, res) => {
310
+ try {
311
+ const title = req.params['title'];
312
+ if (!title) {
313
+ res.status(400).json({ error: 'Missing title' });
314
+ return;
315
+ }
316
+ const { content } = req.body;
317
+ if (content === undefined || content === null) {
318
+ res.status(400).json({ error: 'Missing content field' });
319
+ return;
320
+ }
321
+ const pages = listWikiPages(config.wikiDir);
322
+ const titleLower = title.toLowerCase();
323
+ const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
324
+ const match = pages.find((p) => {
325
+ const fileSlug = basename(p, '.md');
326
+ if (fileSlug === title || fileSlug === slugified)
327
+ return true;
328
+ try {
329
+ const page = readWikiPage(p);
330
+ return page.title.toLowerCase() === titleLower;
331
+ }
332
+ catch {
333
+ return false;
334
+ }
335
+ });
336
+ if (!match) {
337
+ res.status(404).json({ error: 'Page not found' });
338
+ return;
339
+ }
340
+ writeFileSync(match, content, 'utf-8');
341
+ // Extract title from new content for commit message
342
+ const matter = await import('gray-matter');
343
+ const parsed = matter.default(content);
344
+ const pageTitle = parsed.data['title'] || title;
345
+ // Auto-commit via git
346
+ try {
347
+ const { autoCommit } = await import('../core/git.js');
348
+ await autoCommit(config.root, 'manual', `edit page "${pageTitle}"`);
349
+ }
350
+ catch { /* git commit is best-effort */ }
351
+ const page = readWikiPage(match);
352
+ res.json({ status: 'saved', page });
353
+ }
354
+ catch (err) {
355
+ const msg = err instanceof Error ? err.message : String(err);
356
+ res.status(500).json({ error: `Failed to save page: ${msg}` });
357
+ }
358
+ });
359
+ // API: delete a wiki page
360
+ app.delete('/api/pages/:title', async (req, res) => {
361
+ try {
362
+ const title = req.params['title'];
363
+ if (!title) {
364
+ res.status(400).json({ error: 'Missing title' });
365
+ return;
366
+ }
367
+ const pages = listWikiPages(config.wikiDir);
368
+ const titleLower = title.toLowerCase();
369
+ const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
370
+ const match = pages.find((p) => {
371
+ const fileSlug = basename(p, '.md');
372
+ if (fileSlug === title || fileSlug === slugified)
373
+ return true;
374
+ try {
375
+ return readWikiPage(p).title.toLowerCase() === titleLower;
376
+ }
377
+ catch {
378
+ return false;
379
+ }
380
+ });
381
+ if (!match) {
382
+ res.status(404).json({ error: 'Page not found' });
383
+ return;
384
+ }
385
+ const { unlinkSync } = await import('node:fs');
386
+ unlinkSync(match);
387
+ try {
388
+ const { autoCommit } = await import('../core/git.js');
389
+ await autoCommit(config.root, 'manual', `delete page "${title}"`);
390
+ }
391
+ catch { }
392
+ res.json({ status: 'deleted' });
393
+ }
394
+ catch (err) {
395
+ const msg = err instanceof Error ? err.message : String(err);
396
+ res.status(500).json({ error: `Failed to delete: ${msg}` });
397
+ }
398
+ });
399
+ // API: update validation status / confidence on a wiki page
400
+ app.patch('/api/pages/:title/validate', async (req, res) => {
401
+ try {
402
+ const title = req.params['title'];
403
+ if (!title) {
404
+ res.status(400).json({ error: 'Missing title' });
405
+ return;
406
+ }
407
+ const { validation_status, confidence } = req.body;
408
+ const allowed = ['verified', 'outdated', 'wrong', 'unreviewed'];
409
+ if (validation_status && !allowed.includes(validation_status)) {
410
+ res.status(400).json({ error: `Invalid status. Allowed: ${allowed.join(', ')}` });
411
+ return;
412
+ }
413
+ const pages = listWikiPages(config.wikiDir);
414
+ const titleLower = title.toLowerCase();
415
+ const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
416
+ const match = pages.find((p) => {
417
+ const fileSlug = basename(p, '.md');
418
+ if (fileSlug === title || fileSlug === slugified)
419
+ return true;
420
+ try {
421
+ return readWikiPage(p).title.toLowerCase() === titleLower;
422
+ }
423
+ catch {
424
+ return false;
425
+ }
426
+ });
427
+ if (!match) {
428
+ res.status(404).json({ error: 'Page not found' });
429
+ return;
430
+ }
431
+ const matter = await import('gray-matter');
432
+ const raw = readFileSync(match, 'utf-8');
433
+ const parsed = matter.default(raw);
434
+ if (validation_status)
435
+ parsed.data['validation_status'] = validation_status;
436
+ if (confidence !== undefined)
437
+ parsed.data['confidence'] = Math.max(0, Math.min(100, confidence));
438
+ parsed.data['validated_at'] = new Date().toISOString().split('T')[0];
439
+ // Adjust confidence based on validation feedback
440
+ if (validation_status === 'verified') {
441
+ parsed.data['confidence'] = Math.max(parsed.data['confidence'] ?? 50, 85);
442
+ }
443
+ else if (validation_status === 'outdated') {
444
+ parsed.data['confidence'] = Math.min(parsed.data['confidence'] ?? 50, 40);
445
+ }
446
+ else if (validation_status === 'wrong') {
447
+ parsed.data['confidence'] = Math.min(parsed.data['confidence'] ?? 50, 15);
448
+ }
449
+ writeFileSync(match, matter.default.stringify(parsed.content, parsed.data), 'utf-8');
450
+ try {
451
+ const { autoCommit } = await import('../core/git.js');
452
+ await autoCommit(config.root, 'manual', `validate page "${title}" as ${validation_status ?? 'updated'}`);
453
+ }
454
+ catch { /* git commit is best-effort */ }
455
+ const page = readWikiPage(match);
456
+ res.json({ status: 'updated', page });
457
+ }
458
+ catch (err) {
459
+ const msg = err instanceof Error ? err.message : String(err);
460
+ res.status(500).json({ error: `Failed to validate: ${msg}` });
461
+ }
462
+ });
463
+ // API: rename a wiki page
464
+ app.post('/api/pages/:title/rename', async (req, res) => {
465
+ try {
466
+ const oldTitle = req.params['title'];
467
+ const { newTitle } = req.body;
468
+ if (!oldTitle || !newTitle) {
469
+ res.status(400).json({ error: 'Missing title' });
470
+ return;
471
+ }
472
+ const pages = listWikiPages(config.wikiDir);
473
+ const titleLower = oldTitle.toLowerCase();
474
+ const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
475
+ const match = pages.find((p) => {
476
+ const fileSlug = basename(p, '.md');
477
+ if (fileSlug === oldTitle || fileSlug === slugified)
478
+ return true;
479
+ try {
480
+ return readWikiPage(p).title.toLowerCase() === titleLower;
481
+ }
482
+ catch {
483
+ return false;
484
+ }
485
+ });
486
+ if (!match) {
487
+ res.status(404).json({ error: 'Page not found' });
488
+ return;
489
+ }
490
+ const content = readFileSync(match, 'utf-8');
491
+ const newContent = content.replace(/^title:\s*["']?.*?["']?\s*$/m, `title: "${newTitle}"`);
492
+ const newSlug = newTitle.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
493
+ if (!newSlug) {
494
+ res.status(400).json({ error: 'Invalid title β€” produces empty slug' });
495
+ return;
496
+ }
497
+ const newPath = join(match.substring(0, match.lastIndexOf('/')), newSlug + '.md');
498
+ if (!resolve(newPath).startsWith(resolve(config.wikiDir))) {
499
+ res.status(403).json({ error: 'Path traversal denied' });
500
+ return;
501
+ }
502
+ writeFileSync(newPath, newContent, 'utf-8');
503
+ if (newPath !== match) {
504
+ const { unlinkSync } = await import('node:fs');
505
+ unlinkSync(match);
506
+ }
507
+ try {
508
+ const { autoCommit } = await import('../core/git.js');
509
+ await autoCommit(config.root, 'manual', `rename "${oldTitle}" β†’ "${newTitle}"`);
510
+ }
511
+ catch { }
512
+ res.json({ status: 'renamed', newTitle, newSlug });
513
+ }
514
+ catch (err) {
515
+ const msg = err instanceof Error ? err.message : String(err);
516
+ res.status(500).json({ error: `Failed to rename: ${msg}` });
517
+ }
518
+ });
136
519
  // API: knowledge graph data
137
520
  app.get('/api/graph', (_req, res) => {
138
521
  try {
@@ -143,26 +526,43 @@ export function createServer(vaultRoot, port) {
143
526
  res.status(500).json({ error: 'Failed to build graph' });
144
527
  }
145
528
  });
146
- // API: upload raw file for ingestion
529
+ // API: upload raw file and auto-trigger ingest
147
530
  app.post('/api/upload', (req, res) => {
148
531
  const chunks = [];
149
532
  req.on('data', (chunk) => chunks.push(chunk));
150
- req.on('end', () => {
533
+ req.on('end', async () => {
151
534
  const filename = req.headers['x-filename'];
152
535
  if (!filename) {
153
536
  res.status(400).json({ error: 'Missing x-filename header' });
154
537
  return;
155
538
  }
156
- const rawDir = config.rawDir;
157
- if (!existsSync(rawDir)) {
158
- mkdirSync(rawDir, { recursive: true });
159
- }
160
- const dest = join(rawDir, basename(filename));
539
+ const now = new Date().toISOString().split('T')[0] ?? '';
540
+ const dateDir = join(config.rawDir, now);
541
+ mkdirSync(dateDir, { recursive: true });
542
+ const dest = join(dateDir, basename(filename));
161
543
  writeFileSync(dest, Buffer.concat(chunks));
162
- res.json({ status: 'uploaded', path: dest });
544
+ const autoIngest = req.headers['x-auto-ingest'] !== 'false';
545
+ if (autoIngest) {
546
+ try {
547
+ const { ingestSource } = await import('../core/ingest.js');
548
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
549
+ const { loadConfig } = await import('../core/config.js');
550
+ const userConfig = loadConfig(config.configPath);
551
+ const provider = createProviderFromUserConfig(userConfig);
552
+ const result = await ingestSource(dest, config, provider, { verbose: false });
553
+ res.json({ status: 'ingested', path: dest, ...result });
554
+ }
555
+ catch (err) {
556
+ const msg = err instanceof Error ? err.message : String(err);
557
+ res.json({ status: 'uploaded', path: dest, ingestError: msg });
558
+ }
559
+ }
560
+ else {
561
+ res.json({ status: 'uploaded', path: dest });
562
+ }
163
563
  });
164
564
  });
165
- // API: raw files list
565
+ // API: raw files list (recursive through date-stamped subdirectories)
166
566
  app.get('/api/raw', (_req, res) => {
167
567
  try {
168
568
  const rawDir = config.rawDir;
@@ -170,54 +570,412 @@ export function createServer(vaultRoot, port) {
170
570
  res.json([]);
171
571
  return;
172
572
  }
173
- const files = readdirSync(rawDir)
174
- .filter((f) => !f.startsWith('.'))
175
- .map((f) => {
176
- const full = join(rawDir, f);
177
- const stat = statSync(full);
178
- return { name: f, size: stat.size, modified: stat.mtime.toISOString() };
179
- });
573
+ const files = [];
574
+ function walkRaw(dir, prefix) {
575
+ for (const entry of readdirSync(dir)) {
576
+ if (entry.startsWith('.') || entry.endsWith('.meta.json'))
577
+ continue;
578
+ const full = join(dir, entry);
579
+ const stat = statSync(full);
580
+ if (stat.isDirectory()) {
581
+ walkRaw(full, prefix ? `${prefix}/${entry}` : entry);
582
+ }
583
+ else {
584
+ files.push({
585
+ name: prefix ? `${prefix}/${entry}` : entry,
586
+ path: full,
587
+ size: stat.size,
588
+ modified: stat.mtime.toISOString(),
589
+ });
590
+ }
591
+ }
592
+ }
593
+ walkRaw(rawDir, '');
180
594
  res.json(files);
181
595
  }
182
596
  catch (err) {
183
597
  res.status(500).json({ error: 'Failed to list raw files' });
184
598
  }
185
599
  });
186
- // API: query the wiki
187
- app.post('/api/query', async (req, res) => {
600
+ // API: read raw file content (for preview)
601
+ app.get('/api/raw/view/:filename', (req, res) => {
188
602
  try {
189
- const { question, provider: providerName } = req.body;
190
- if (!question) {
191
- res.status(400).json({ error: 'Missing question field' });
603
+ const filename = req.params['filename'];
604
+ if (!filename) {
605
+ res.status(400).json({ error: 'Missing filename' });
192
606
  return;
193
607
  }
194
- // Dynamic import to avoid circular deps
195
- const { queryWiki } = await import('../core/query.js');
196
- const { createProvider } = await import('../providers/index.js');
197
- const { loadConfig } = await import('../core/config.js');
198
- const userConfig = loadConfig(config.configPath);
199
- const provider = createProvider(providerName ?? userConfig.provider ?? 'claude');
200
- const result = await queryWiki(question, config, provider, { fileBack: false });
201
- res.json(result);
608
+ const decoded = decodeURIComponent(filename);
609
+ const fullPath = join(config.rawDir, decoded);
610
+ const resolved = resolve(fullPath);
611
+ if (!resolved.startsWith(resolve(config.rawDir))) {
612
+ res.status(403).json({ error: 'Access denied' });
613
+ return;
614
+ }
615
+ if (!existsSync(resolved)) {
616
+ res.status(404).json({ error: 'File not found' });
617
+ return;
618
+ }
619
+ const ext = extname(resolved).toLowerCase();
620
+ const imageExts = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'];
621
+ if (imageExts.includes(ext)) {
622
+ res.sendFile(resolved);
623
+ return;
624
+ }
625
+ const textExts = ['.md', '.txt', '.csv', '.json', '.yaml', '.yml', '.xml', '.html', '.htm', '.ts', '.js', '.py', '.go', '.rs'];
626
+ if (textExts.includes(ext) || ext === '') {
627
+ const content = readFileSync(resolved, 'utf-8');
628
+ res.json({ type: 'text', content, filename: decoded });
629
+ return;
630
+ }
631
+ res.json({ type: 'binary', filename: decoded, size: statSync(resolved).size, message: 'Binary file β€” extract with wikimem ingest' });
202
632
  }
203
633
  catch (err) {
204
- const msg = err instanceof Error ? err.message : String(err);
205
- res.status(500).json({ error: `Query failed: ${msg}` });
634
+ res.status(500).json({ error: 'Failed to read raw file' });
206
635
  }
207
636
  });
208
- // API: ingest a URL
209
- app.post('/api/ingest', async (req, res) => {
637
+ // API: file tree (wiki + raw hierarchy)
638
+ app.get('/api/tree', (_req, res) => {
210
639
  try {
211
- const { source } = req.body;
640
+ function buildWikiTree(dir, relPath) {
641
+ const nodes = [];
642
+ if (!existsSync(dir))
643
+ return nodes;
644
+ const entries = readdirSync(dir).sort();
645
+ for (const entry of entries) {
646
+ if (entry.startsWith('.'))
647
+ continue;
648
+ const full = join(dir, entry);
649
+ const stat = statSync(full);
650
+ const childPath = relPath ? `${relPath}/${entry}` : entry;
651
+ if (stat.isDirectory()) {
652
+ const children = buildWikiTree(full, childPath);
653
+ nodes.push({ name: entry, type: 'dir', path: childPath, children });
654
+ }
655
+ else if (entry.endsWith('.md')) {
656
+ try {
657
+ const page = readWikiPage(full);
658
+ const cat = page.frontmatter['type']
659
+ ?? page.frontmatter['category']
660
+ ?? 'page';
661
+ nodes.push({ name: entry, type: 'wiki', path: childPath, title: page.title, category: cat });
662
+ }
663
+ catch {
664
+ nodes.push({ name: entry, type: 'wiki', path: childPath, title: entry.replace('.md', '') });
665
+ }
666
+ }
667
+ }
668
+ return nodes;
669
+ }
670
+ function buildRawTree(dir, relPath) {
671
+ const nodes = [];
672
+ if (!existsSync(dir))
673
+ return nodes;
674
+ const entries = readdirSync(dir).sort();
675
+ for (const entry of entries) {
676
+ if (entry.startsWith('.') || entry.endsWith('.meta.json'))
677
+ continue;
678
+ const full = join(dir, entry);
679
+ const stat = statSync(full);
680
+ const childPath = relPath ? `${relPath}/${entry}` : entry;
681
+ if (stat.isDirectory()) {
682
+ nodes.push({ name: entry, type: 'dir', path: childPath, children: buildRawTree(full, childPath) });
683
+ }
684
+ else {
685
+ nodes.push({ name: entry, type: 'raw', path: childPath, size: stat.size });
686
+ }
687
+ }
688
+ return nodes;
689
+ }
690
+ res.json({
691
+ wiki: buildWikiTree(config.wikiDir, ''),
692
+ raw: buildRawTree(config.rawDir, ''),
693
+ });
694
+ }
695
+ catch {
696
+ res.status(500).json({ error: 'Failed to build file tree' });
697
+ }
698
+ });
699
+ // API: read config (for settings page)
700
+ app.get('/api/config', async (_req, res) => {
701
+ try {
702
+ const { loadConfig } = await import('../core/config.js');
703
+ const userConfig = loadConfig(config.configPath);
704
+ const safeConfig = { ...userConfig };
705
+ if (safeConfig.api_key)
706
+ safeConfig.api_key = safeConfig.api_key.slice(0, 8) + '…';
707
+ const sc = safeConfig;
708
+ if (sc['gemini_api_key']) {
709
+ sc['gemini_api_key'] = String(sc['gemini_api_key']).slice(0, 8) + '…';
710
+ }
711
+ res.json(safeConfig);
712
+ }
713
+ catch {
714
+ res.json({});
715
+ }
716
+ });
717
+ // API: update config (for settings page)
718
+ app.put('/api/config', async (req, res) => {
719
+ try {
720
+ const { loadConfig } = await import('../core/config.js');
721
+ const YAML = await import('yaml');
722
+ const updates = req.body;
723
+ const current = loadConfig(config.configPath);
724
+ const merged = { ...current, ...updates };
725
+ writeFileSync(config.configPath, YAML.stringify(merged), 'utf-8');
726
+ res.json({ status: 'saved' });
727
+ }
728
+ catch (err) {
729
+ const msg = err instanceof Error ? err.message : String(err);
730
+ res.status(500).json({ error: `Failed to save config: ${msg}` });
731
+ }
732
+ });
733
+ // API: test provider connection
734
+ app.post('/api/config/test-provider', async (req, res) => {
735
+ try {
736
+ const { provider: providerName, apiKey } = req.body;
737
+ if (providerName === 'claude-code') {
738
+ const { isClaudeCodeAvailable } = await import('../core/claude-code.js');
739
+ if (isClaudeCodeAvailable()) {
740
+ res.json({ status: 'ok', provider: 'claude-code' });
741
+ }
742
+ else {
743
+ res.json({ status: 'error', error: 'Claude Code CLI not found in PATH' });
744
+ }
745
+ return;
746
+ }
747
+ if (!apiKey) {
748
+ res.status(400).json({ error: 'Missing apiKey' });
749
+ return;
750
+ }
751
+ if (providerName === 'claude' || !providerName) {
752
+ const { default: Anthropic } = await import('@anthropic-ai/sdk');
753
+ const client = new Anthropic({ apiKey });
754
+ await client.messages.create({
755
+ model: 'claude-3-haiku-20240307',
756
+ max_tokens: 10,
757
+ messages: [{ role: 'user', content: 'ping' }],
758
+ });
759
+ res.json({ status: 'ok', provider: 'claude' });
760
+ }
761
+ else {
762
+ res.json({ status: 'ok', provider: providerName, note: 'Skipped validation' });
763
+ }
764
+ }
765
+ catch (err) {
766
+ const msg = err instanceof Error ? err.message : String(err);
767
+ res.json({ status: 'error', error: msg });
768
+ }
769
+ });
770
+ // API: Claude Code CLI availability
771
+ app.get('/api/claude-code/status', async (_req, res) => {
772
+ try {
773
+ const { isClaudeCodeAvailable, getClaudeCodePath } = await import('../core/claude-code.js');
774
+ res.json({ available: isClaudeCodeAvailable(), path: getClaudeCodePath() });
775
+ }
776
+ catch {
777
+ res.json({ available: false, path: null });
778
+ }
779
+ });
780
+ // API: search pages (with filters: category, tag, dateFrom, dateTo)
781
+ app.get('/api/search', (req, res) => {
782
+ try {
783
+ const q = (req.query['q'] ?? '').toLowerCase().trim();
784
+ const limit = parseInt(req.query['limit']) || 20;
785
+ const filterCategory = (req.query['category'] ?? '').toLowerCase().trim();
786
+ const filterTag = (req.query['tag'] ?? '').toLowerCase().trim();
787
+ const filterDateFrom = (req.query['dateFrom'] ?? '').trim();
788
+ const filterDateTo = (req.query['dateTo'] ?? '').trim();
789
+ if (!q && !filterCategory && !filterTag) {
790
+ res.json({ results: [] });
791
+ return;
792
+ }
793
+ const pages = listWikiPages(config.wikiDir);
794
+ const results = [];
795
+ for (const pagePath of pages) {
796
+ const page = readWikiPage(pagePath);
797
+ const category = (page.frontmatter['category'] ?? page.frontmatter['type'] ?? '').toLowerCase();
798
+ const tags = (page.frontmatter['tags'] ?? []).map((t) => t.toLowerCase());
799
+ const created = (page.frontmatter['created'] ?? page.frontmatter['date'] ?? '');
800
+ // Apply filters
801
+ if (filterCategory && category !== filterCategory)
802
+ continue;
803
+ if (filterTag && !tags.includes(filterTag))
804
+ continue;
805
+ if (filterDateFrom && created && created < filterDateFrom)
806
+ continue;
807
+ if (filterDateTo && created && created > filterDateTo)
808
+ continue;
809
+ // If no text query, include all filtered pages
810
+ if (!q) {
811
+ results.push({ title: page.title, category: category || 'uncategorized', tags, created, wordCount: page.wordCount, matchType: 'filter' });
812
+ if (results.length >= limit)
813
+ break;
814
+ continue;
815
+ }
816
+ // Text matching
817
+ const titleMatch = page.title.toLowerCase().includes(q);
818
+ const tagMatch = tags.some(t => t.includes(q));
819
+ let snippet;
820
+ let contentMatch = false;
821
+ const bodyLower = page.content.toLowerCase();
822
+ const idx = bodyLower.indexOf(q);
823
+ if (idx >= 0) {
824
+ contentMatch = true;
825
+ // Build snippet with highlighted match context
826
+ const start = Math.max(0, idx - 50);
827
+ const end = Math.min(page.content.length, idx + q.length + 80);
828
+ const raw = page.content.substring(start, end).replace(/\n/g, ' ').replace(/\s+/g, ' ');
829
+ // Wrap the matched term in <mark> for highlighting
830
+ const matchStart = idx - start;
831
+ const before = raw.substring(0, matchStart);
832
+ const match = raw.substring(matchStart, matchStart + q.length);
833
+ const after = raw.substring(matchStart + q.length);
834
+ snippet = (start > 0 ? '…' : '') + before + '<mark>' + match + '</mark>' + after + (end < page.content.length ? '…' : '');
835
+ }
836
+ if (titleMatch || tagMatch || contentMatch) {
837
+ const matchType = titleMatch ? 'title' : tagMatch ? 'tag' : 'content';
838
+ results.push({ title: page.title, category: category || 'uncategorized', tags, created, wordCount: page.wordCount, snippet, matchType });
839
+ }
840
+ if (results.length >= limit)
841
+ break;
842
+ }
843
+ // Also return available filter options for the UI
844
+ const allCategories = [...new Set(pages.map(p => {
845
+ const pg = readWikiPage(p);
846
+ return (pg.frontmatter['category'] ?? pg.frontmatter['type'] ?? '').toLowerCase();
847
+ }).filter(Boolean))].sort();
848
+ const allTags = [...new Set(pages.flatMap(p => {
849
+ const pg = readWikiPage(p);
850
+ return (pg.frontmatter['tags'] ?? []).map((t) => t.toLowerCase());
851
+ }))].sort();
852
+ res.json({ results, filters: { categories: allCategories, tags: allTags } });
853
+ }
854
+ catch {
855
+ res.json({ results: [] });
856
+ }
857
+ });
858
+ // Utility: extract keywords from text (lowercase, deduplicated, stopwords removed)
859
+ function extractKeywords(text) {
860
+ const stopwords = new Set(['the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'can', 'shall', 'and', 'or', 'but', 'if', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'from', 'as', 'into', 'about', 'that', 'this', 'it', 'its', 'not', 'no', 'so', 'up', 'out', 'then', 'than', 'more', 'also', 'very', 'just', 'only', 'each', 'any', 'all', 'both', 'few', 'many', 'some', 'such', 'too', 'own', 'same', 'other', 'most', 'much', 'what', 'when', 'where', 'which', 'who', 'how']);
861
+ return new Set(text.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').split(/\s+/)
862
+ .filter(w => w.length > 2 && !stopwords.has(w)));
863
+ }
864
+ // Utility: Jaccard similarity between two sets
865
+ function jaccardSimilarity(a, b) {
866
+ if (a.size === 0 && b.size === 0)
867
+ return 0;
868
+ let intersection = 0;
869
+ for (const item of a) {
870
+ if (b.has(item))
871
+ intersection++;
872
+ }
873
+ const union = a.size + b.size - intersection;
874
+ return union === 0 ? 0 : intersection / union;
875
+ }
876
+ // API: similar pages (Jaccard similarity on tags + wikilinks + title keywords)
877
+ app.get('/api/pages/:title/similar', (req, res) => {
878
+ try {
879
+ const targetTitle = decodeURIComponent(req.params['title'] ?? '');
880
+ const limit = parseInt(req.query['limit']) || 5;
881
+ const pages = listWikiPages(config.wikiDir);
882
+ // Find target page
883
+ let targetPage = null;
884
+ const allPages = [];
885
+ for (const p of pages) {
886
+ const page = readWikiPage(p);
887
+ allPages.push(page);
888
+ if (page.title.toLowerCase() === targetTitle.toLowerCase()) {
889
+ targetPage = page;
890
+ }
891
+ }
892
+ if (!targetPage) {
893
+ res.json({ similar: [] });
894
+ return;
895
+ }
896
+ // Build feature sets for target
897
+ const targetTags = new Set((targetPage.frontmatter['tags'] ?? []).map((t) => t.toLowerCase()));
898
+ const targetLinks = new Set((targetPage.wikilinks ?? []).map((l) => l.toLowerCase()));
899
+ const targetWords = extractKeywords(targetPage.title + ' ' + targetPage.content);
900
+ const targetCategory = (targetPage.frontmatter['category'] ?? targetPage.frontmatter['type'] ?? '').toLowerCase();
901
+ // Score each other page
902
+ const scored = [];
903
+ for (const page of allPages) {
904
+ if (page.title.toLowerCase() === targetTitle.toLowerCase())
905
+ continue;
906
+ const pageTags = new Set((page.frontmatter['tags'] ?? []).map((t) => t.toLowerCase()));
907
+ const pageLinks = new Set((page.wikilinks ?? []).map((l) => l.toLowerCase()));
908
+ const pageWords = extractKeywords(page.title + ' ' + page.content);
909
+ const pageCategory = (page.frontmatter['category'] ?? page.frontmatter['type'] ?? '').toLowerCase();
910
+ // Jaccard similarity components (weighted)
911
+ const tagSim = jaccardSimilarity(targetTags, pageTags);
912
+ const linkSim = jaccardSimilarity(targetLinks, pageLinks);
913
+ const wordSim = jaccardSimilarity(targetWords, pageWords);
914
+ const catBonus = targetCategory && targetCategory === pageCategory ? 0.15 : 0;
915
+ // Weighted composite: tags 40%, links 30%, keywords 20%, category 10%
916
+ const score = (tagSim * 0.4) + (linkSim * 0.3) + (wordSim * 0.2) + catBonus;
917
+ if (score > 0.02) {
918
+ const sharedTags = [...targetTags].filter(t => pageTags.has(t));
919
+ const sharedLinks = [...targetLinks].filter(l => pageLinks.has(l));
920
+ const category = page.frontmatter['category'] ?? page.frontmatter['type'] ?? 'uncategorized';
921
+ scored.push({ title: page.title, category, score: Math.round(score * 100) / 100, sharedTags, sharedLinks });
922
+ }
923
+ }
924
+ scored.sort((a, b) => b.score - a.score);
925
+ res.json({ similar: scored.slice(0, limit) });
926
+ }
927
+ catch {
928
+ res.json({ similar: [] });
929
+ }
930
+ });
931
+ // API: wiki history (audit trail)
932
+ app.get('/api/history', async (_req, res) => {
933
+ try {
934
+ const { listSnapshots } = await import('../core/history.js');
935
+ const entries = listSnapshots(config);
936
+ res.json(entries);
937
+ }
938
+ catch {
939
+ res.json([]);
940
+ }
941
+ });
942
+ // API: query the wiki
943
+ app.post('/api/query', async (req, res) => {
944
+ try {
945
+ const { question, provider: providerName, model: modelName } = req.body;
946
+ if (!question) {
947
+ res.status(400).json({ error: 'Missing question field' });
948
+ return;
949
+ }
950
+ const { queryWiki } = await import('../core/query.js');
951
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
952
+ const { loadConfig } = await import('../core/config.js');
953
+ const userConfig = loadConfig(config.configPath);
954
+ const provider = createProviderFromUserConfig(userConfig, {
955
+ providerOverride: providerName,
956
+ model: modelName,
957
+ });
958
+ const result = await queryWiki(question, config, provider, { fileBack: false });
959
+ res.json(result);
960
+ }
961
+ catch (err) {
962
+ const msg = err instanceof Error ? err.message : String(err);
963
+ res.status(500).json({ error: `Query failed: ${msg}` });
964
+ }
965
+ });
966
+ // API: ingest a URL
967
+ app.post('/api/ingest', async (req, res) => {
968
+ try {
969
+ const { source } = req.body;
212
970
  if (!source) {
213
971
  res.status(400).json({ error: 'Missing source field' });
214
972
  return;
215
973
  }
216
974
  const { ingestSource } = await import('../core/ingest.js');
217
- const { createProvider } = await import('../providers/index.js');
975
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
218
976
  const { loadConfig } = await import('../core/config.js');
219
977
  const userConfig = loadConfig(config.configPath);
220
- const provider = createProvider(userConfig.provider ?? 'claude');
978
+ const provider = createProviderFromUserConfig(userConfig);
221
979
  const result = await ingestSource(source, config, provider, { verbose: false });
222
980
  res.json(result);
223
981
  }
@@ -226,6 +984,2028 @@ export function createServer(vaultRoot, port) {
226
984
  res.status(500).json({ error: `Ingest failed: ${msg}` });
227
985
  }
228
986
  });
987
+ // === GIT API ENDPOINTS ===
988
+ // API: git status (lightweight by default, add ?full=true for file list)
989
+ app.get('/api/git/status', async (req, res) => {
990
+ try {
991
+ const { isGitRepo, getGitStatus, getBranches } = await import('../core/git.js');
992
+ const isRepo = await isGitRepo(config.root);
993
+ if (!isRepo) {
994
+ res.json({ initialized: false });
995
+ return;
996
+ }
997
+ const includeFull = req.query['full'] === 'true';
998
+ const status = await getGitStatus(config.root);
999
+ const branches = await getBranches(config.root);
1000
+ const result = {
1001
+ initialized: true,
1002
+ branch: branches.current,
1003
+ branches: branches.all,
1004
+ isDetached: branches.isDetached,
1005
+ changedCount: status?.files?.length ?? 0,
1006
+ isClean: status?.isClean() ?? true,
1007
+ };
1008
+ if (includeFull) {
1009
+ result['files'] = status?.files ?? [];
1010
+ }
1011
+ res.json(result);
1012
+ }
1013
+ catch (err) {
1014
+ const msg = err instanceof Error ? err.message : String(err);
1015
+ res.status(500).json({ error: `Git status failed: ${msg}` });
1016
+ }
1017
+ });
1018
+ // API: initialize git repo
1019
+ app.post('/api/git/init', async (_req, res) => {
1020
+ try {
1021
+ const { initGitRepo } = await import('../core/git.js');
1022
+ const result = await initGitRepo(config);
1023
+ res.json(result);
1024
+ }
1025
+ catch (err) {
1026
+ const msg = err instanceof Error ? err.message : String(err);
1027
+ res.status(500).json({ error: `Git init failed: ${msg}` });
1028
+ }
1029
+ });
1030
+ // API: git log (audit trail) with optional wiki-only filtering
1031
+ app.get('/api/git/log', async (req, res) => {
1032
+ try {
1033
+ const { getGitLog, isGitRepo } = await import('../core/git.js');
1034
+ if (!(await isGitRepo(config.root))) {
1035
+ res.json([]);
1036
+ return;
1037
+ }
1038
+ const limit = parseInt(req.query['limit']) || 50;
1039
+ const wikiOnly = req.query['wikiOnly'] !== 'false';
1040
+ const search = req.query['search'] || undefined;
1041
+ const log = await getGitLog(config.root, limit, { wikiOnly, search });
1042
+ res.json(log);
1043
+ }
1044
+ catch (err) {
1045
+ const msg = err instanceof Error ? err.message : String(err);
1046
+ res.status(500).json({ error: `Git log failed: ${msg}` });
1047
+ }
1048
+ });
1049
+ // API: git diff for a specific commit
1050
+ app.get('/api/git/diff/:hash', async (req, res) => {
1051
+ try {
1052
+ const hash = req.params['hash'];
1053
+ if (!hash) {
1054
+ res.status(400).json({ error: 'Missing hash' });
1055
+ return;
1056
+ }
1057
+ const { getGitDiff, isGitRepo } = await import('../core/git.js');
1058
+ if (!(await isGitRepo(config.root))) {
1059
+ res.json({ diff: '', stats: [] });
1060
+ return;
1061
+ }
1062
+ const result = await getGitDiff(config.root, hash);
1063
+ res.json(result);
1064
+ }
1065
+ catch (err) {
1066
+ const msg = err instanceof Error ? err.message : String(err);
1067
+ res.status(500).json({ error: `Git diff failed: ${msg}` });
1068
+ }
1069
+ });
1070
+ // API: create branch (optionally from a specific commit hash)
1071
+ app.post('/api/git/branch', async (req, res) => {
1072
+ try {
1073
+ const { name, fromHash } = req.body;
1074
+ if (!name) {
1075
+ res.status(400).json({ error: 'Missing branch name' });
1076
+ return;
1077
+ }
1078
+ const { createBranch } = await import('../core/git.js');
1079
+ const result = await createBranch(config.root, name, fromHash);
1080
+ res.json(result);
1081
+ }
1082
+ catch (err) {
1083
+ const msg = err instanceof Error ? err.message : String(err);
1084
+ res.status(500).json({ error: `Create branch failed: ${msg}` });
1085
+ }
1086
+ });
1087
+ // API: switch branch
1088
+ app.post('/api/git/checkout', async (req, res) => {
1089
+ try {
1090
+ const { branch } = req.body;
1091
+ if (!branch) {
1092
+ res.status(400).json({ error: 'Missing branch name' });
1093
+ return;
1094
+ }
1095
+ const { switchBranch } = await import('../core/git.js');
1096
+ const result = await switchBranch(config.root, branch);
1097
+ res.json(result);
1098
+ }
1099
+ catch (err) {
1100
+ const msg = err instanceof Error ? err.message : String(err);
1101
+ res.status(500).json({ error: `Checkout failed: ${msg}` });
1102
+ }
1103
+ });
1104
+ // API: create tag (milestone)
1105
+ app.post('/api/git/tag', async (req, res) => {
1106
+ try {
1107
+ const { name, message: tagMsg } = req.body;
1108
+ if (!name) {
1109
+ res.status(400).json({ error: 'Missing tag name' });
1110
+ return;
1111
+ }
1112
+ const { createTag } = await import('../core/git.js');
1113
+ const result = await createTag(config.root, name, tagMsg);
1114
+ res.json(result);
1115
+ }
1116
+ catch (err) {
1117
+ const msg = err instanceof Error ? err.message : String(err);
1118
+ res.status(500).json({ error: `Create tag failed: ${msg}` });
1119
+ }
1120
+ });
1121
+ // API: restore to a specific commit
1122
+ app.post('/api/git/restore', async (req, res) => {
1123
+ try {
1124
+ const { hash } = req.body;
1125
+ if (!hash) {
1126
+ res.status(400).json({ error: 'Missing commit hash' });
1127
+ return;
1128
+ }
1129
+ const { restoreToCommit } = await import('../core/git.js');
1130
+ const result = await restoreToCommit(config.root, hash);
1131
+ res.json(result);
1132
+ }
1133
+ catch (err) {
1134
+ const msg = err instanceof Error ? err.message : String(err);
1135
+ res.status(500).json({ error: `Restore failed: ${msg}` });
1136
+ }
1137
+ });
1138
+ // API: get file tree at a specific commit (for time-lapse)
1139
+ app.get('/api/git/tree/:hash', async (req, res) => {
1140
+ try {
1141
+ const hash = req.params['hash'];
1142
+ if (!hash) {
1143
+ res.status(400).json({ error: 'Missing hash' });
1144
+ return;
1145
+ }
1146
+ const { getTreeAtCommit, isGitRepo } = await import('../core/git.js');
1147
+ if (!(await isGitRepo(config.root))) {
1148
+ res.json([]);
1149
+ return;
1150
+ }
1151
+ const tree = await getTreeAtCommit(config.root, hash);
1152
+ res.json(tree);
1153
+ }
1154
+ catch (err) {
1155
+ const msg = err instanceof Error ? err.message : String(err);
1156
+ res.status(500).json({ error: `Get tree failed: ${msg}` });
1157
+ }
1158
+ });
1159
+ // API: batch fetch file trees for multiple commits (time-lapse pre-fetch)
1160
+ app.post('/api/git/trees/batch', async (req, res) => {
1161
+ try {
1162
+ const { hashes } = req.body;
1163
+ if (!hashes || !Array.isArray(hashes) || hashes.length === 0) {
1164
+ res.status(400).json({ error: 'Missing or empty hashes array' });
1165
+ return;
1166
+ }
1167
+ const capped = hashes.slice(0, 500);
1168
+ const { getTreeAtCommit, isGitRepo } = await import('../core/git.js');
1169
+ if (!(await isGitRepo(config.root))) {
1170
+ res.json({});
1171
+ return;
1172
+ }
1173
+ const result = {};
1174
+ const concurrency = 10;
1175
+ for (let i = 0; i < capped.length; i += concurrency) {
1176
+ const batch = capped.slice(i, i + concurrency);
1177
+ await Promise.all(batch.map(async (hash) => {
1178
+ try {
1179
+ result[hash] = await getTreeAtCommit(config.root, hash);
1180
+ }
1181
+ catch {
1182
+ result[hash] = [];
1183
+ }
1184
+ }));
1185
+ }
1186
+ res.json(result);
1187
+ }
1188
+ catch (err) {
1189
+ const msg = err instanceof Error ? err.message : String(err);
1190
+ res.status(500).json({ error: `Batch tree fetch failed: ${msg}` });
1191
+ }
1192
+ });
1193
+ // API: graph data at a specific commit (for time-lapse graph animation)
1194
+ app.get('/api/git/graph/:hash', async (req, res) => {
1195
+ try {
1196
+ const hash = req.params['hash'];
1197
+ if (!hash) {
1198
+ res.status(400).json({ error: 'Missing hash' });
1199
+ return;
1200
+ }
1201
+ const { getGraphAtCommit, isGitRepo } = await import('../core/git.js');
1202
+ if (!(await isGitRepo(config.root))) {
1203
+ res.json({ nodes: [], links: [] });
1204
+ return;
1205
+ }
1206
+ const graph = await getGraphAtCommit(config.root, hash);
1207
+ res.json(graph);
1208
+ }
1209
+ catch (err) {
1210
+ const msg = err instanceof Error ? err.message : String(err);
1211
+ res.status(500).json({ error: `Get graph at commit failed: ${msg}` });
1212
+ }
1213
+ });
1214
+ // API: batch graph snapshots for time-lapse (pre-fetch all commit graphs)
1215
+ app.post('/api/git/graph-batch', async (req, res) => {
1216
+ try {
1217
+ const { hashes } = req.body;
1218
+ if (!hashes?.length) {
1219
+ res.status(400).json({ error: 'Missing hashes array' });
1220
+ return;
1221
+ }
1222
+ const { getGraphAtCommit, isGitRepo } = await import('../core/git.js');
1223
+ if (!(await isGitRepo(config.root))) {
1224
+ res.json({});
1225
+ return;
1226
+ }
1227
+ const results = {};
1228
+ for (const hash of hashes.slice(0, 100)) {
1229
+ results[hash] = await getGraphAtCommit(config.root, hash);
1230
+ }
1231
+ res.json(results);
1232
+ }
1233
+ catch (err) {
1234
+ const msg = err instanceof Error ? err.message : String(err);
1235
+ res.status(500).json({ error: `Batch graph failed: ${msg}` });
1236
+ }
1237
+ });
1238
+ // API: list branches (dedicated endpoint)
1239
+ app.get('/api/git/branches', async (_req, res) => {
1240
+ try {
1241
+ const { getBranches, isGitRepo } = await import('../core/git.js');
1242
+ if (!(await isGitRepo(config.root))) {
1243
+ res.json({ current: '', branches: [], isDetached: false });
1244
+ return;
1245
+ }
1246
+ const info = await getBranches(config.root);
1247
+ res.json({ current: info.current, branches: info.all, isDetached: info.isDetached });
1248
+ }
1249
+ catch (err) {
1250
+ const msg = err instanceof Error ? err.message : String(err);
1251
+ res.status(500).json({ error: `Failed to list branches: ${msg}` });
1252
+ }
1253
+ });
1254
+ // API: create session branch (auto-naming with wiki/session-<timestamp> or custom name)
1255
+ app.post('/api/git/branches', async (req, res) => {
1256
+ try {
1257
+ const { name, session } = req.body;
1258
+ const { createBranch } = await import('../core/git.js');
1259
+ let branchName = name;
1260
+ if (!branchName || session) {
1261
+ const ts = new Date().toISOString().replace(/[:.]/g, '-').substring(0, 19);
1262
+ branchName = name ? `wiki/${name}` : `wiki/session-${ts}`;
1263
+ }
1264
+ const result = await createBranch(config.root, branchName);
1265
+ res.json(result);
1266
+ }
1267
+ catch (err) {
1268
+ const msg = err instanceof Error ? err.message : String(err);
1269
+ res.status(500).json({ error: `Create branch failed: ${msg}` });
1270
+ }
1271
+ });
1272
+ // API: push current branch to remote
1273
+ app.post('/api/git/push', async (req, res) => {
1274
+ try {
1275
+ const { remote } = req.body;
1276
+ const { pushBranch } = await import('../core/git.js');
1277
+ const result = await pushBranch(config.root, remote || 'origin');
1278
+ res.json(result);
1279
+ }
1280
+ catch (err) {
1281
+ const msg = err instanceof Error ? err.message : String(err);
1282
+ res.status(500).json({ error: `Push failed: ${msg}` });
1283
+ }
1284
+ });
1285
+ // API: submit for review (PR workflow: diff summary, auto-branch, push)
1286
+ app.post('/api/git/pr', async (req, res) => {
1287
+ try {
1288
+ const { description } = req.body;
1289
+ const { submitForReview } = await import('../core/git.js');
1290
+ const result = await submitForReview(config.root, description);
1291
+ res.json(result);
1292
+ }
1293
+ catch (err) {
1294
+ const msg = err instanceof Error ? err.message : String(err);
1295
+ res.status(500).json({ error: `Submit for review failed: ${msg}` });
1296
+ }
1297
+ });
1298
+ // API: get diff summary (working branch vs base)
1299
+ app.get('/api/git/diff-summary', async (req, res) => {
1300
+ try {
1301
+ const baseBranch = req.query['base'];
1302
+ const { getDiffSummary, isGitRepo } = await import('../core/git.js');
1303
+ if (!(await isGitRepo(config.root))) {
1304
+ res.json({ filesAdded: [], filesModified: [], filesDeleted: [], totalAdditions: 0, totalDeletions: 0 });
1305
+ return;
1306
+ }
1307
+ const summary = await getDiffSummary(config.root, baseBranch);
1308
+ res.json(summary);
1309
+ }
1310
+ catch (err) {
1311
+ const msg = err instanceof Error ? err.message : String(err);
1312
+ res.status(500).json({ error: `Diff summary failed: ${msg}` });
1313
+ }
1314
+ });
1315
+ // API: parsed diff for a commit (rich diff view with per-file hunks)
1316
+ app.get('/api/git/diff/:hash/parsed', async (req, res) => {
1317
+ try {
1318
+ const hash = req.params['hash'];
1319
+ if (!hash) {
1320
+ res.status(400).json({ error: 'Missing hash' });
1321
+ return;
1322
+ }
1323
+ const { getParsedDiff, isGitRepo } = await import('../core/git.js');
1324
+ if (!(await isGitRepo(config.root))) {
1325
+ res.json({ files: [], stats: { additions: 0, deletions: 0, filesChanged: 0 } });
1326
+ return;
1327
+ }
1328
+ const result = await getParsedDiff(config.root, hash);
1329
+ res.json(result);
1330
+ }
1331
+ catch (err) {
1332
+ const msg = err instanceof Error ? err.message : String(err);
1333
+ res.status(500).json({ error: `Parsed diff failed: ${msg}` });
1334
+ }
1335
+ });
1336
+ // API: migrate .wikimem/history snapshots to git commits
1337
+ app.post('/api/git/migrate-history', async (_req, res) => {
1338
+ try {
1339
+ const { migrateHistoryToGit } = await import('../core/git.js');
1340
+ const result = await migrateHistoryToGit(config);
1341
+ res.json(result);
1342
+ }
1343
+ catch (err) {
1344
+ const msg = err instanceof Error ? err.message : String(err);
1345
+ res.status(500).json({ error: `Migration failed: ${msg}` });
1346
+ }
1347
+ });
1348
+ // === RAW FILE PREVIEW ENDPOINTS ===
1349
+ // API: serve raw file with correct content-type (for PDF, images, video, audio)
1350
+ app.get('/api/raw/file', (req, res) => {
1351
+ try {
1352
+ let filePath = String(req.query['path'] ?? '');
1353
+ if (filePath.startsWith('/'))
1354
+ filePath = filePath.slice(1);
1355
+ if (!filePath) {
1356
+ res.status(400).json({ error: 'Missing path query param' });
1357
+ return;
1358
+ }
1359
+ const decoded = decodeURIComponent(filePath);
1360
+ const fullPath = join(config.rawDir, decoded);
1361
+ const resolved = resolve(fullPath);
1362
+ if (!resolved.startsWith(resolve(config.rawDir))) {
1363
+ res.status(403).json({ error: 'Access denied' });
1364
+ return;
1365
+ }
1366
+ if (!existsSync(resolved)) {
1367
+ res.status(404).json({ error: 'File not found' });
1368
+ return;
1369
+ }
1370
+ const ext = extname(resolved).toLowerCase();
1371
+ const mimeTypes = {
1372
+ '.pdf': 'application/pdf',
1373
+ '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
1374
+ '.png': 'image/png', '.gif': 'image/gif',
1375
+ '.webp': 'image/webp', '.svg': 'image/svg+xml',
1376
+ '.mp4': 'video/mp4', '.webm': 'video/webm', '.mov': 'video/quicktime',
1377
+ '.mp3': 'audio/mpeg', '.wav': 'audio/wav', '.ogg': 'audio/ogg',
1378
+ '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
1379
+ '.xls': 'application/vnd.ms-excel',
1380
+ '.csv': 'text/csv',
1381
+ '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
1382
+ '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
1383
+ };
1384
+ const mime = mimeTypes[ext] ?? 'application/octet-stream';
1385
+ res.setHeader('Accept-Ranges', 'bytes');
1386
+ res.setHeader('Content-Type', mime);
1387
+ res.sendFile(resolved);
1388
+ }
1389
+ catch (err) {
1390
+ res.status(500).json({ error: 'Failed to serve file' });
1391
+ }
1392
+ });
1393
+ // API: raw file metadata (for preview header)
1394
+ app.get('/api/raw/meta', (req, res) => {
1395
+ try {
1396
+ let filePath = String(req.query['path'] ?? '');
1397
+ if (filePath.startsWith('/'))
1398
+ filePath = filePath.slice(1);
1399
+ if (!filePath) {
1400
+ res.status(400).json({ error: 'Missing path query param' });
1401
+ return;
1402
+ }
1403
+ const decoded = decodeURIComponent(filePath);
1404
+ const fullPath = join(config.rawDir, decoded);
1405
+ const resolved = resolve(fullPath);
1406
+ if (!resolved.startsWith(resolve(config.rawDir))) {
1407
+ res.status(403).json({ error: 'Access denied' });
1408
+ return;
1409
+ }
1410
+ if (!existsSync(resolved)) {
1411
+ res.status(404).json({ error: 'File not found' });
1412
+ return;
1413
+ }
1414
+ const stat = statSync(resolved);
1415
+ const ext = extname(resolved).toLowerCase();
1416
+ const imageExts = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'];
1417
+ const videoExts = ['.mp4', '.webm', '.mov'];
1418
+ const audioExts = ['.mp3', '.wav', '.ogg'];
1419
+ const pdfExts = ['.pdf'];
1420
+ const spreadsheetExts = ['.csv', '.xlsx', '.xls'];
1421
+ const textExts = ['.md', '.txt', '.json', '.yaml', '.yml', '.xml', '.html', '.htm', '.ts', '.js', '.py', '.go', '.rs', '.toml', '.ini', '.cfg', '.env', '.sh', '.bash', '.zsh'];
1422
+ const documentExts = ['.docx', '.doc', '.pptx', '.ppt', '.rtf', '.odt', '.odp'];
1423
+ let previewType;
1424
+ if (imageExts.includes(ext))
1425
+ previewType = 'image';
1426
+ else if (videoExts.includes(ext))
1427
+ previewType = 'video';
1428
+ else if (audioExts.includes(ext))
1429
+ previewType = 'audio';
1430
+ else if (pdfExts.includes(ext))
1431
+ previewType = 'pdf';
1432
+ else if (spreadsheetExts.includes(ext))
1433
+ previewType = 'spreadsheet';
1434
+ else if (textExts.includes(ext))
1435
+ previewType = 'text';
1436
+ else if (documentExts.includes(ext))
1437
+ previewType = 'document';
1438
+ else
1439
+ previewType = 'binary';
1440
+ // Find wiki pages that were generated from this raw file
1441
+ const linkedPages = [];
1442
+ try {
1443
+ const pages = listWikiPages(config.wikiDir);
1444
+ for (const pagePath of pages) {
1445
+ const page = readWikiPage(pagePath);
1446
+ const sources = page.frontmatter['sources'];
1447
+ if (sources?.some(s => s.includes(decoded) || s.includes(basename(decoded)))) {
1448
+ linkedPages.push({
1449
+ title: page.title,
1450
+ slug: basename(pagePath, '.md'),
1451
+ });
1452
+ }
1453
+ }
1454
+ }
1455
+ catch { /* non-fatal */ }
1456
+ res.json({
1457
+ name: basename(decoded),
1458
+ path: decoded,
1459
+ size: stat.size,
1460
+ modified: stat.mtime.toISOString(),
1461
+ created: stat.birthtime.toISOString(),
1462
+ extension: ext,
1463
+ previewType,
1464
+ linkedWikiPages: linkedPages,
1465
+ });
1466
+ }
1467
+ catch (err) {
1468
+ res.status(500).json({ error: 'Failed to get file metadata' });
1469
+ }
1470
+ });
1471
+ // ─── Raw File Operations (Cursor-Parity) ──────────────────────────────────
1472
+ // POST /api/raw/rename β€” rename a raw file
1473
+ app.post('/api/raw/rename', async (req, res) => {
1474
+ try {
1475
+ const { oldPath, newPath } = req.body;
1476
+ if (!oldPath || !newPath) {
1477
+ res.status(400).json({ error: 'Missing oldPath or newPath' });
1478
+ return;
1479
+ }
1480
+ const resolvedOld = resolve(config.rawDir, oldPath);
1481
+ const resolvedNew = resolve(config.rawDir, newPath);
1482
+ if (!resolvedOld.startsWith(resolve(config.rawDir)) || !resolvedNew.startsWith(resolve(config.rawDir))) {
1483
+ res.status(403).json({ error: 'Access denied: path outside raw directory' });
1484
+ return;
1485
+ }
1486
+ if (!existsSync(resolvedOld)) {
1487
+ res.status(404).json({ error: 'File not found' });
1488
+ return;
1489
+ }
1490
+ mkdirSync(dirname(resolvedNew), { recursive: true });
1491
+ renameSync(resolvedOld, resolvedNew);
1492
+ try {
1493
+ const { autoCommit, isGitRepo } = await import('../core/git.js');
1494
+ if (await isGitRepo(config.root)) {
1495
+ await autoCommit(config.root, 'manual', `rename raw "${basename(oldPath)}" β†’ "${basename(newPath)}"`);
1496
+ }
1497
+ }
1498
+ catch { /* non-fatal */ }
1499
+ res.json({ status: 'renamed', oldPath, newPath });
1500
+ }
1501
+ catch (err) {
1502
+ const msg = err instanceof Error ? err.message : String(err);
1503
+ res.status(500).json({ error: `Rename failed: ${msg}` });
1504
+ }
1505
+ });
1506
+ // DELETE /api/raw/file β€” delete a raw file
1507
+ app.delete('/api/raw/file', async (req, res) => {
1508
+ try {
1509
+ const filePath = req.query['path'];
1510
+ if (!filePath) {
1511
+ res.status(400).json({ error: 'Missing path query param' });
1512
+ return;
1513
+ }
1514
+ const resolved = resolve(config.rawDir, filePath);
1515
+ if (!resolved.startsWith(resolve(config.rawDir))) {
1516
+ res.status(403).json({ error: 'Access denied: path outside raw directory' });
1517
+ return;
1518
+ }
1519
+ if (!existsSync(resolved)) {
1520
+ res.status(404).json({ error: 'File not found' });
1521
+ return;
1522
+ }
1523
+ fsUnlinkSync(resolved);
1524
+ try {
1525
+ const { autoCommit, isGitRepo } = await import('../core/git.js');
1526
+ if (await isGitRepo(config.root)) {
1527
+ await autoCommit(config.root, 'manual', `delete raw file "${basename(filePath)}"`);
1528
+ }
1529
+ }
1530
+ catch { /* non-fatal */ }
1531
+ res.json({ status: 'deleted', path: filePath });
1532
+ }
1533
+ catch (err) {
1534
+ const msg = err instanceof Error ? err.message : String(err);
1535
+ res.status(500).json({ error: `Delete failed: ${msg}` });
1536
+ }
1537
+ });
1538
+ // POST /api/raw/move β€” move a raw file to a different directory
1539
+ app.post('/api/raw/move', async (req, res) => {
1540
+ try {
1541
+ const { path: filePath, targetDir } = req.body;
1542
+ if (!filePath || !targetDir) {
1543
+ res.status(400).json({ error: 'Missing path or targetDir' });
1544
+ return;
1545
+ }
1546
+ const resolvedSrc = resolve(config.rawDir, filePath);
1547
+ const resolvedDest = resolve(config.rawDir, targetDir, basename(filePath));
1548
+ if (!resolvedSrc.startsWith(resolve(config.rawDir)) || !resolvedDest.startsWith(resolve(config.rawDir))) {
1549
+ res.status(403).json({ error: 'Access denied: path outside raw directory' });
1550
+ return;
1551
+ }
1552
+ if (!existsSync(resolvedSrc)) {
1553
+ res.status(404).json({ error: 'Source file not found' });
1554
+ return;
1555
+ }
1556
+ mkdirSync(dirname(resolvedDest), { recursive: true });
1557
+ renameSync(resolvedSrc, resolvedDest);
1558
+ try {
1559
+ const { autoCommit, isGitRepo } = await import('../core/git.js');
1560
+ if (await isGitRepo(config.root)) {
1561
+ await autoCommit(config.root, 'manual', `move raw "${basename(filePath)}" β†’ ${targetDir}/`);
1562
+ }
1563
+ }
1564
+ catch { /* non-fatal */ }
1565
+ res.json({ status: 'moved', from: filePath, to: targetDir + '/' + basename(filePath) });
1566
+ }
1567
+ catch (err) {
1568
+ const msg = err instanceof Error ? err.message : String(err);
1569
+ res.status(500).json({ error: `Move failed: ${msg}` });
1570
+ }
1571
+ });
1572
+ // POST /api/pages/:title/move β€” move a wiki page to a different category folder
1573
+ app.post('/api/pages/:title/move', async (req, res) => {
1574
+ try {
1575
+ const title = req.params['title'];
1576
+ const { targetCategory } = req.body;
1577
+ if (!title || !targetCategory) {
1578
+ res.status(400).json({ error: 'Missing title or targetCategory' });
1579
+ return;
1580
+ }
1581
+ const pages = listWikiPages(config.wikiDir);
1582
+ const titleLower = title.toLowerCase();
1583
+ const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
1584
+ const match = pages.find((p) => {
1585
+ const fileSlug = basename(p, '.md');
1586
+ if (fileSlug === title || fileSlug === slugified)
1587
+ return true;
1588
+ try {
1589
+ return readWikiPage(p).title.toLowerCase() === titleLower;
1590
+ }
1591
+ catch {
1592
+ return false;
1593
+ }
1594
+ });
1595
+ if (!match) {
1596
+ res.status(404).json({ error: 'Page not found' });
1597
+ return;
1598
+ }
1599
+ const destDir = resolve(config.wikiDir, targetCategory);
1600
+ if (!destDir.startsWith(resolve(config.wikiDir))) {
1601
+ res.status(403).json({ error: 'Access denied: category outside wiki directory' });
1602
+ return;
1603
+ }
1604
+ mkdirSync(destDir, { recursive: true });
1605
+ const destPath = join(destDir, basename(match));
1606
+ renameSync(match, destPath);
1607
+ // Update frontmatter category/type
1608
+ let content = readFileSync(destPath, 'utf-8');
1609
+ if (content.match(/^type:\s*.+$/m)) {
1610
+ content = content.replace(/^type:\s*.+$/m, `type: ${targetCategory}`);
1611
+ }
1612
+ else if (content.match(/^category:\s*.+$/m)) {
1613
+ content = content.replace(/^category:\s*.+$/m, `category: ${targetCategory}`);
1614
+ }
1615
+ writeFileSync(destPath, content, 'utf-8');
1616
+ try {
1617
+ const { autoCommit, isGitRepo } = await import('../core/git.js');
1618
+ if (await isGitRepo(config.root)) {
1619
+ await autoCommit(config.root, 'manual', `move page "${title}" β†’ ${targetCategory}/`);
1620
+ }
1621
+ }
1622
+ catch { /* non-fatal */ }
1623
+ res.json({ status: 'moved', title, targetCategory, newPath: relative(config.wikiDir, destPath) });
1624
+ }
1625
+ catch (err) {
1626
+ const msg = err instanceof Error ? err.message : String(err);
1627
+ res.status(500).json({ error: `Move failed: ${msg}` });
1628
+ }
1629
+ });
1630
+ // POST /api/folders β€” create a new folder inside wiki/ or raw/
1631
+ app.post('/api/folders', (req, res) => {
1632
+ try {
1633
+ const { path: folderPath } = req.body;
1634
+ if (!folderPath) {
1635
+ res.status(400).json({ error: 'Missing path' });
1636
+ return;
1637
+ }
1638
+ const resolved = resolve(config.root, folderPath);
1639
+ const wikiBase = resolve(config.wikiDir);
1640
+ const rawBase = resolve(config.rawDir);
1641
+ if (!resolved.startsWith(wikiBase) && !resolved.startsWith(rawBase)) {
1642
+ res.status(403).json({ error: 'Access denied: folder must be inside wiki/ or raw/' });
1643
+ return;
1644
+ }
1645
+ mkdirSync(resolved, { recursive: true });
1646
+ res.json({ status: 'created', path: folderPath });
1647
+ }
1648
+ catch (err) {
1649
+ const msg = err instanceof Error ? err.message : String(err);
1650
+ res.status(500).json({ error: `Create folder failed: ${msg}` });
1651
+ }
1652
+ });
1653
+ // ─── OAuth Connector System ───────────────────────────────────────────────
1654
+ const OAUTH_PROVIDERS = {
1655
+ github: {
1656
+ authorizeUrl: 'https://github.com/login/oauth/authorize',
1657
+ tokenUrl: 'https://github.com/login/oauth/access_token',
1658
+ scopes: 'repo read:user',
1659
+ clientIdKey: 'github_client_id',
1660
+ clientSecretKey: 'github_client_secret',
1661
+ },
1662
+ slack: {
1663
+ authorizeUrl: 'https://slack.com/oauth/v2/authorize',
1664
+ tokenUrl: 'https://slack.com/api/oauth.v2.access',
1665
+ scopes: 'channels:history channels:read users:read',
1666
+ clientIdKey: 'slack_client_id',
1667
+ clientSecretKey: 'slack_client_secret',
1668
+ },
1669
+ google: {
1670
+ authorizeUrl: 'https://accounts.google.com/o/oauth2/v2/auth',
1671
+ tokenUrl: 'https://oauth2.googleapis.com/token',
1672
+ scopes: 'https://www.googleapis.com/auth/gmail.readonly https://www.googleapis.com/auth/drive.readonly',
1673
+ clientIdKey: 'google_client_id',
1674
+ clientSecretKey: 'google_client_secret',
1675
+ },
1676
+ linear: {
1677
+ authorizeUrl: 'https://linear.app/oauth/authorize',
1678
+ tokenUrl: 'https://api.linear.app/oauth/token',
1679
+ scopes: 'read',
1680
+ clientIdKey: 'linear_client_id',
1681
+ clientSecretKey: 'linear_client_secret',
1682
+ },
1683
+ jira: {
1684
+ authorizeUrl: 'https://auth.atlassian.com/authorize',
1685
+ tokenUrl: 'https://auth.atlassian.com/oauth/token',
1686
+ scopes: 'read:jira-work read:jira-user offline_access',
1687
+ clientIdKey: 'jira_client_id',
1688
+ clientSecretKey: 'jira_client_secret',
1689
+ },
1690
+ };
1691
+ const oauthStates = new Map();
1692
+ /**
1693
+ * Resolve OAuth credentials: bundled defaults β†’ env vars β†’ config.yaml.
1694
+ * Bundled defaults ship with the npm package (pre-registered WikiMem OAuth apps).
1695
+ * Env vars override bundled defaults (for development/self-hosted).
1696
+ * config.yaml is the user-level fallback.
1697
+ */
1698
+ function resolveOAuthCredentials(providerName) {
1699
+ const providerConfig = OAUTH_PROVIDERS[providerName];
1700
+ if (!providerConfig)
1701
+ return null;
1702
+ // 1. Environment variables (highest priority β€” override bundled defaults)
1703
+ const envPrefix = `WIKIMEM_${providerName.toUpperCase()}`;
1704
+ const envId = process.env[`${envPrefix}_CLIENT_ID`];
1705
+ const envSecret = process.env[`${envPrefix}_CLIENT_SECRET`];
1706
+ if (envId && envSecret)
1707
+ return { clientId: envId, clientSecret: envSecret };
1708
+ // 2. Bundled defaults (pre-registered WikiMem OAuth apps shipped with package)
1709
+ const bundled = getBundledCredentials(providerName);
1710
+ if (bundled)
1711
+ return bundled;
1712
+ // 3. User-configured credentials in config.yaml
1713
+ try {
1714
+ const { loadConfig: lc } = require('../core/config.js');
1715
+ const userConfig = lc(config.configPath);
1716
+ const configId = userConfig[providerConfig.clientIdKey];
1717
+ const configSecret = userConfig[providerConfig.clientSecretKey];
1718
+ if (configId && configSecret)
1719
+ return { clientId: configId, clientSecret: configSecret };
1720
+ }
1721
+ catch { /* config load failure is non-fatal */ }
1722
+ return null;
1723
+ }
1724
+ function getTokenStorePath() {
1725
+ return join(vaultRoot, '.wikimem', 'tokens.json');
1726
+ }
1727
+ function loadOAuthTokens() {
1728
+ const tokenPath = getTokenStorePath();
1729
+ if (!existsSync(tokenPath))
1730
+ return {};
1731
+ try {
1732
+ return JSON.parse(readFileSync(tokenPath, 'utf-8'));
1733
+ }
1734
+ catch {
1735
+ return {};
1736
+ }
1737
+ }
1738
+ function saveOAuthToken(provider, tokenData) {
1739
+ const tokenPath = getTokenStorePath();
1740
+ mkdirSync(dirname(tokenPath), { recursive: true });
1741
+ const tokens = loadOAuthTokens();
1742
+ tokens[provider] = { ...tokenData, connectedAt: new Date().toISOString() };
1743
+ writeFileSync(tokenPath, JSON.stringify(tokens, null, 2), 'utf-8');
1744
+ }
1745
+ // GET /api/auth/tokens β€” list which providers have tokens stored + credential availability
1746
+ app.get('/api/auth/tokens', (_req, res) => {
1747
+ try {
1748
+ const tokens = loadOAuthTokens();
1749
+ const result = {};
1750
+ for (const provider of Object.keys(OAUTH_PROVIDERS)) {
1751
+ const token = tokens[provider];
1752
+ const creds = resolveOAuthCredentials(provider);
1753
+ result[provider] = {
1754
+ connected: !!token,
1755
+ connectedAt: token?.connectedAt,
1756
+ hasCredentials: !!creds,
1757
+ ...(provider === 'github' ? { hasDeviceFlow: !!resolveDeviceFlowClientId() } : {}),
1758
+ };
1759
+ }
1760
+ res.json(result);
1761
+ }
1762
+ catch {
1763
+ res.json({});
1764
+ }
1765
+ });
1766
+ // GET /api/auth/start/:provider β€” generate OAuth authorize URL
1767
+ app.get('/api/auth/start/:provider', async (req, res) => {
1768
+ try {
1769
+ const providerName = req.params['provider'];
1770
+ if (!providerName) {
1771
+ res.status(400).json({ error: 'Missing provider' });
1772
+ return;
1773
+ }
1774
+ const providerConfig = OAUTH_PROVIDERS[providerName];
1775
+ if (!providerConfig) {
1776
+ res.status(400).json({ error: `Unknown provider: ${providerName}` });
1777
+ return;
1778
+ }
1779
+ const creds = resolveOAuthCredentials(providerName);
1780
+ if (!creds) {
1781
+ res.status(400).json({ error: 'no_credentials', message: `No OAuth credentials found for ${providerName}. Add credentials in Settings or set WIKIMEM_${providerName.toUpperCase()}_CLIENT_ID / _CLIENT_SECRET env vars.` });
1782
+ return;
1783
+ }
1784
+ const state = randomBytes(24).toString('hex');
1785
+ oauthStates.set(state, { provider: providerName, createdAt: Date.now() });
1786
+ // Clean stale states (>10 min)
1787
+ for (const [key, val] of oauthStates) {
1788
+ if (Date.now() - val.createdAt > 600_000)
1789
+ oauthStates.delete(key);
1790
+ }
1791
+ const redirectUri = `http://localhost:${port}/api/auth/callback`;
1792
+ const params = new URLSearchParams({
1793
+ client_id: creds.clientId,
1794
+ redirect_uri: redirectUri,
1795
+ scope: providerConfig.scopes,
1796
+ state,
1797
+ response_type: 'code',
1798
+ ...(providerName === 'google' ? { access_type: 'offline', prompt: 'consent' } : {}),
1799
+ ...(providerName === 'jira' ? { audience: 'api.atlassian.com', prompt: 'consent' } : {}),
1800
+ });
1801
+ const authorizeUrl = `${providerConfig.authorizeUrl}?${params.toString()}`;
1802
+ res.json({ url: authorizeUrl, state });
1803
+ }
1804
+ catch (err) {
1805
+ const msg = err instanceof Error ? err.message : String(err);
1806
+ res.status(500).json({ error: `OAuth start failed: ${msg}` });
1807
+ }
1808
+ });
1809
+ // GET /api/auth/callback β€” OAuth callback handler
1810
+ app.get('/api/auth/callback', async (req, res) => {
1811
+ try {
1812
+ const code = req.query['code'];
1813
+ const state = req.query['state'];
1814
+ if (!code || !state) {
1815
+ res.status(400).send('<h2>Missing code or state</h2>');
1816
+ return;
1817
+ }
1818
+ const stateData = oauthStates.get(state);
1819
+ if (!stateData) {
1820
+ res.status(400).send('<h2>Invalid or expired state token</h2>');
1821
+ return;
1822
+ }
1823
+ oauthStates.delete(state);
1824
+ const providerConfig = OAUTH_PROVIDERS[stateData.provider];
1825
+ if (!providerConfig) {
1826
+ res.status(400).send('<h2>Unknown provider</h2>');
1827
+ return;
1828
+ }
1829
+ const creds = resolveOAuthCredentials(stateData.provider);
1830
+ if (!creds) {
1831
+ res.status(400).send('<h2>Missing client credentials</h2><p>Configure OAuth credentials in Settings or via environment variables.</p>');
1832
+ return;
1833
+ }
1834
+ const redirectUri = `http://localhost:${port}/api/auth/callback`;
1835
+ const tokenRes = await fetch(providerConfig.tokenUrl, {
1836
+ method: 'POST',
1837
+ headers: {
1838
+ 'Content-Type': 'application/x-www-form-urlencoded',
1839
+ Accept: 'application/json',
1840
+ },
1841
+ body: new URLSearchParams({
1842
+ client_id: creds.clientId,
1843
+ client_secret: creds.clientSecret,
1844
+ code,
1845
+ redirect_uri: redirectUri,
1846
+ grant_type: 'authorization_code',
1847
+ }),
1848
+ });
1849
+ const tokenBody = await tokenRes.json();
1850
+ const accessToken = tokenBody['access_token'];
1851
+ if (!accessToken) {
1852
+ res.status(400).send(`<h2>Token exchange failed</h2><pre>${JSON.stringify(tokenBody, null, 2)}</pre>`);
1853
+ return;
1854
+ }
1855
+ saveOAuthToken(stateData.provider, {
1856
+ access_token: accessToken,
1857
+ refresh_token: tokenBody['refresh_token'],
1858
+ scope: tokenBody['scope'],
1859
+ });
1860
+ // Auto-trigger sync after successful OAuth connection (fire-and-forget)
1861
+ import('../core/sync/index.js').then(({ syncProvider: sp }) => {
1862
+ sp(stateData.provider, vaultRoot).catch(() => { });
1863
+ }).catch(() => { });
1864
+ const providerDisplay = stateData.provider.charAt(0).toUpperCase() + stateData.provider.slice(1);
1865
+ res.send(`<!DOCTYPE html><html><head><style>
1866
+ body { font-family: Inter, system-ui, sans-serif; background: #1e1e1e; color: #ccc; display: flex; align-items: center; justify-content: center; min-height: 100vh; }
1867
+ .card { background: #252526; border: 1px solid #3e3e3e; border-radius: 12px; padding: 32px 40px; text-align: center; }
1868
+ h2 { color: #4ec9b0; margin: 0 0 8px; } p { color: #808080; font-size: 14px; }
1869
+ .spin { width:20px;height:20px;border:2px solid #3e3e3e;border-top-color:#4ec9b0;border-radius:50%;animation:s 1s linear infinite;margin:12px auto 0 }
1870
+ @keyframes s { to { transform:rotate(360deg) } }
1871
+ </style></head><body><div class="card"><h2>Connected!</h2><p>${providerDisplay} is now linked to WikiMem.</p><div class="spin"></div><p style="margin-top:8px"><small>Syncing your data... This window will close automatically.</small></p></div>
1872
+ <script>if(window.opener){window.opener.postMessage({type:'wikimem-oauth-connected',provider:'${stateData.provider}'},'*');}setTimeout(function(){window.close()},3000)</script>
1873
+ </body></html>`);
1874
+ }
1875
+ catch (err) {
1876
+ const msg = err instanceof Error ? err.message : String(err);
1877
+ res.status(500).send(`<h2>OAuth callback failed</h2><pre>${msg}</pre>`);
1878
+ }
1879
+ });
1880
+ // DELETE /api/auth/tokens/:provider β€” disconnect a provider
1881
+ app.delete('/api/auth/tokens/:provider', (_req, res) => {
1882
+ try {
1883
+ const provider = _req.params['provider'];
1884
+ if (!provider) {
1885
+ res.status(400).json({ error: 'Missing provider' });
1886
+ return;
1887
+ }
1888
+ const tokenPath = getTokenStorePath();
1889
+ const tokens = loadOAuthTokens();
1890
+ delete tokens[provider];
1891
+ mkdirSync(dirname(tokenPath), { recursive: true });
1892
+ writeFileSync(tokenPath, JSON.stringify(tokens, null, 2), 'utf-8');
1893
+ res.json({ status: 'disconnected', provider });
1894
+ }
1895
+ catch (err) {
1896
+ const msg = err instanceof Error ? err.message : String(err);
1897
+ res.status(500).json({ error: `Disconnect failed: ${msg}` });
1898
+ }
1899
+ });
1900
+ // POST /api/auth/tokens/:provider β€” manually store an API key (for Notion, etc.)
1901
+ app.post('/api/auth/tokens/:provider', (req, res) => {
1902
+ try {
1903
+ const provider = req.params['provider'];
1904
+ if (!provider) {
1905
+ res.status(400).json({ error: 'Missing provider' });
1906
+ return;
1907
+ }
1908
+ const { api_key } = req.body;
1909
+ if (!api_key) {
1910
+ res.status(400).json({ error: 'Missing api_key' });
1911
+ return;
1912
+ }
1913
+ saveOAuthToken(provider, { access_token: api_key });
1914
+ res.json({ status: 'connected', provider });
1915
+ }
1916
+ catch (err) {
1917
+ const msg = err instanceof Error ? err.message : String(err);
1918
+ res.status(500).json({ error: `Save token failed: ${msg}` });
1919
+ }
1920
+ });
1921
+ // ─── GitHub Device Flow (no client_secret needed) ──────────────────────────
1922
+ /**
1923
+ * Resolve the GitHub client ID for Device Flow.
1924
+ * Priority: env vars β†’ bundled defaults β†’ config.yaml
1925
+ */
1926
+ function resolveDeviceFlowClientId() {
1927
+ // 1. Env vars (highest priority)
1928
+ const deviceId = process.env['WIKIMEM_GITHUB_DEVICE_CLIENT_ID'];
1929
+ if (deviceId)
1930
+ return deviceId;
1931
+ const regularId = process.env['WIKIMEM_GITHUB_CLIENT_ID'];
1932
+ if (regularId)
1933
+ return regularId;
1934
+ // 2. Bundled defaults (shipped with package)
1935
+ const bundled = getBundledDeviceFlowClientId();
1936
+ if (bundled)
1937
+ return bundled;
1938
+ // 3. config.yaml (user-configured)
1939
+ try {
1940
+ const { loadConfig: lc } = require('../core/config.js');
1941
+ const userConfig = lc(config.configPath);
1942
+ const configId = userConfig['github_client_id'];
1943
+ if (configId)
1944
+ return configId;
1945
+ }
1946
+ catch { /* config load failure is non-fatal */ }
1947
+ return null;
1948
+ }
1949
+ // POST /api/auth/device-flow/start β€” initiate GitHub Device Flow
1950
+ app.post('/api/auth/device-flow/start', async (_req, res) => {
1951
+ try {
1952
+ const clientId = resolveDeviceFlowClientId();
1953
+ if (!clientId) {
1954
+ res.status(400).json({
1955
+ error: 'no_client_id',
1956
+ message: 'No GitHub client ID found. Set WIKIMEM_GITHUB_DEVICE_CLIENT_ID or WIKIMEM_GITHUB_CLIENT_ID env var.',
1957
+ });
1958
+ return;
1959
+ }
1960
+ const ghRes = await fetch('https://github.com/login/device/code', {
1961
+ method: 'POST',
1962
+ headers: {
1963
+ 'Content-Type': 'application/x-www-form-urlencoded',
1964
+ Accept: 'application/json',
1965
+ },
1966
+ body: new URLSearchParams({
1967
+ client_id: clientId,
1968
+ scope: 'repo read:user',
1969
+ }),
1970
+ });
1971
+ const body = await ghRes.json();
1972
+ if (body['error']) {
1973
+ res.status(400).json({ error: body['error'], message: body['error_description'] ?? 'Device flow initiation failed' });
1974
+ return;
1975
+ }
1976
+ res.json({
1977
+ device_code: body['device_code'],
1978
+ user_code: body['user_code'],
1979
+ verification_uri: body['verification_uri'],
1980
+ expires_in: body['expires_in'],
1981
+ interval: body['interval'],
1982
+ });
1983
+ }
1984
+ catch (err) {
1985
+ const msg = err instanceof Error ? err.message : String(err);
1986
+ res.status(500).json({ error: `Device flow start failed: ${msg}` });
1987
+ }
1988
+ });
1989
+ // POST /api/auth/device-flow/poll β€” poll for token after user enters code
1990
+ app.post('/api/auth/device-flow/poll', async (req, res) => {
1991
+ try {
1992
+ const { device_code } = req.body;
1993
+ if (!device_code) {
1994
+ res.status(400).json({ error: 'missing_device_code', message: 'device_code is required' });
1995
+ return;
1996
+ }
1997
+ const clientId = resolveDeviceFlowClientId();
1998
+ if (!clientId) {
1999
+ res.status(400).json({ error: 'no_client_id', message: 'No GitHub client ID configured' });
2000
+ return;
2001
+ }
2002
+ const ghRes = await fetch('https://github.com/login/oauth/access_token', {
2003
+ method: 'POST',
2004
+ headers: {
2005
+ 'Content-Type': 'application/x-www-form-urlencoded',
2006
+ Accept: 'application/json',
2007
+ },
2008
+ body: new URLSearchParams({
2009
+ client_id: clientId,
2010
+ device_code,
2011
+ grant_type: 'urn:ietf:params:oauth:grant-type:device_code',
2012
+ }),
2013
+ });
2014
+ const body = await ghRes.json();
2015
+ const error = body['error'];
2016
+ if (error === 'authorization_pending') {
2017
+ res.json({ status: 'pending' });
2018
+ return;
2019
+ }
2020
+ if (error === 'slow_down') {
2021
+ res.json({ status: 'slow_down' });
2022
+ return;
2023
+ }
2024
+ if (error === 'expired_token') {
2025
+ res.json({ status: 'expired' });
2026
+ return;
2027
+ }
2028
+ if (error) {
2029
+ res.status(400).json({ status: 'error', error, message: body['error_description'] ?? 'Token exchange failed' });
2030
+ return;
2031
+ }
2032
+ const accessToken = body['access_token'];
2033
+ if (!accessToken) {
2034
+ res.status(400).json({ status: 'error', error: 'no_token', message: 'No access_token in response' });
2035
+ return;
2036
+ }
2037
+ saveOAuthToken('github', {
2038
+ access_token: accessToken,
2039
+ scope: body['scope'],
2040
+ });
2041
+ // Auto-trigger GitHub sync after device flow connection (fire-and-forget)
2042
+ import('../core/sync/index.js').then(({ syncProvider: sp }) => {
2043
+ sp('github', vaultRoot).catch(() => { });
2044
+ }).catch(() => { });
2045
+ res.json({ status: 'complete', access_token: accessToken });
2046
+ }
2047
+ catch (err) {
2048
+ const msg = err instanceof Error ? err.message : String(err);
2049
+ res.status(500).json({ error: `Device flow poll failed: ${msg}` });
2050
+ }
2051
+ });
2052
+ // Pipeline SSE endpoint β€” real-time step updates
2053
+ app.get('/api/pipeline/events', (_req, res) => {
2054
+ res.writeHead(200, {
2055
+ 'Content-Type': 'text/event-stream',
2056
+ 'Cache-Control': 'no-cache',
2057
+ Connection: 'keep-alive',
2058
+ });
2059
+ res.write('data: {"type":"connected"}\n\n');
2060
+ const onStep = (event) => {
2061
+ res.write(`data: ${JSON.stringify(event)}\n\n`);
2062
+ };
2063
+ import('../core/pipeline-events.js').then(({ pipelineEvents }) => {
2064
+ pipelineEvents.on('step', onStep);
2065
+ _req.on('close', () => pipelineEvents.off('step', onStep));
2066
+ });
2067
+ });
2068
+ // Pipeline runs history
2069
+ app.get('/api/pipeline/runs', async (_req, res) => {
2070
+ try {
2071
+ const { pipelineEvents } = await import('../core/pipeline-events.js');
2072
+ const runs = pipelineEvents.getRecentRuns();
2073
+ const summaries = runs.map(r => ({
2074
+ id: r.id,
2075
+ source: r.source,
2076
+ startedAt: r.startedAt,
2077
+ eventCount: r.events.length,
2078
+ hasSummary: !!r.summary,
2079
+ hasLLMTrace: !!r.llmTrace,
2080
+ result: r.result,
2081
+ }));
2082
+ res.json({ runs: summaries });
2083
+ }
2084
+ catch {
2085
+ res.json({ runs: [] });
2086
+ }
2087
+ });
2088
+ // Pipeline run detail (with LLM trace and summary)
2089
+ app.get('/api/pipeline/runs/:id', async (req, res) => {
2090
+ try {
2091
+ const { pipelineEvents } = await import('../core/pipeline-events.js');
2092
+ const runs = pipelineEvents.getRecentRuns();
2093
+ const run = runs.find(r => r.id === req.params['id']);
2094
+ if (!run) {
2095
+ res.status(404).json({ error: 'Run not found' });
2096
+ return;
2097
+ }
2098
+ res.json(run);
2099
+ }
2100
+ catch {
2101
+ res.status(500).json({ error: 'Failed to get run' });
2102
+ }
2103
+ });
2104
+ // ─── Pipeline Configuration (custom steps) ──────────────────────────────────
2105
+ app.get('/api/pipeline/config', async (_req, res) => {
2106
+ try {
2107
+ const { loadConfig } = await import('../core/config.js');
2108
+ const userConfig = loadConfig(config.configPath);
2109
+ const pipelineConfig = userConfig['pipeline'] ?? {};
2110
+ const customSteps = pipelineConfig['custom_steps'] ?? [];
2111
+ const disabledSteps = pipelineConfig['disabled_steps'] ?? [];
2112
+ res.json({ custom_steps: customSteps, disabled_steps: disabledSteps });
2113
+ }
2114
+ catch {
2115
+ res.json({ custom_steps: [], disabled_steps: [] });
2116
+ }
2117
+ });
2118
+ app.put('/api/pipeline/config', async (req, res) => {
2119
+ try {
2120
+ const { loadConfig } = await import('../core/config.js');
2121
+ const YAML = await import('yaml');
2122
+ const current = loadConfig(config.configPath);
2123
+ const body = req.body;
2124
+ const pipeline = current['pipeline'] ?? {};
2125
+ if (body.custom_steps !== undefined)
2126
+ pipeline['custom_steps'] = body.custom_steps;
2127
+ if (body.disabled_steps !== undefined)
2128
+ pipeline['disabled_steps'] = body.disabled_steps;
2129
+ current['pipeline'] = pipeline;
2130
+ writeFileSync(config.configPath, YAML.stringify(current), 'utf-8');
2131
+ res.json({ status: 'saved' });
2132
+ }
2133
+ catch (err) {
2134
+ const msg = err instanceof Error ? err.message : String(err);
2135
+ res.status(500).json({ error: `Failed to save pipeline config: ${msg}` });
2136
+ }
2137
+ });
2138
+ // ─── Automation 3: Webhook Ingest ─────────────────────────────────────────
2139
+ // Track files currently being ingested by webhook to avoid double-ingest with watcher
2140
+ const webhookIngestingFiles = new Set();
2141
+ // POST /api/webhook/ingest β€” accept external content, run ingest pipeline
2142
+ app.post('/api/webhook/ingest', async (req, res) => {
2143
+ try {
2144
+ const { content, title, source, tags, metadata } = req.body;
2145
+ if (!content || content.trim().length === 0) {
2146
+ res.status(400).json({ error: 'Missing or empty content field' });
2147
+ return;
2148
+ }
2149
+ const now = new Date().toISOString().split('T')[0] ?? '';
2150
+ const pageTitle = title ?? `Webhook Ingest ${new Date().toISOString()}`;
2151
+ const markdown = `# ${pageTitle}\n\n${source ? `Source: ${source}\n` : ''}Ingested via webhook: ${new Date().toISOString()}\n\n${content}`;
2152
+ // Write to raw/ so ingest pipeline can find it
2153
+ const { mkdirSync: mkdir, writeFileSync: write } = await import('node:fs');
2154
+ const { join: joinPath } = await import('node:path');
2155
+ const { slugify } = await import('../core/vault.js');
2156
+ const dateDir = joinPath(config.rawDir, now);
2157
+ mkdir(dateDir, { recursive: true });
2158
+ const filePath = joinPath(dateDir, `${slugify(pageTitle.substring(0, 60))}-webhook.md`);
2159
+ write(filePath, markdown, 'utf-8');
2160
+ // Tell the watcher to skip this file (webhook handles ingest)
2161
+ webhookIngestingFiles.add(filePath);
2162
+ const { ingestSource } = await import('../core/ingest.js');
2163
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
2164
+ const { loadConfig } = await import('../core/config.js');
2165
+ const { appendAuditEntry } = await import('../core/audit-trail.js');
2166
+ const userConfig = loadConfig(config.configPath);
2167
+ const provider = createProviderFromUserConfig(userConfig);
2168
+ let result;
2169
+ try {
2170
+ result = await ingestSource(filePath, config, provider, {
2171
+ verbose: false,
2172
+ tags: tags ?? [],
2173
+ metadata,
2174
+ addedBy: 'webhook',
2175
+ });
2176
+ }
2177
+ catch (ingestErr) {
2178
+ const ingestMsg = ingestErr instanceof Error ? ingestErr.message : String(ingestErr);
2179
+ appendAuditEntry(vaultRoot, {
2180
+ action: 'ingest',
2181
+ actor: 'webhook',
2182
+ source: source ?? filePath,
2183
+ summary: `Webhook ingest FAILED: "${pageTitle}" β€” ${ingestMsg.substring(0, 200)}`,
2184
+ pagesAffected: [pageTitle],
2185
+ });
2186
+ webhookIngestingFiles.delete(filePath);
2187
+ res.status(500).json({ error: `Webhook ingest failed: ${ingestMsg}` });
2188
+ return;
2189
+ }
2190
+ appendAuditEntry(vaultRoot, {
2191
+ action: 'ingest',
2192
+ actor: 'webhook',
2193
+ source: source ?? filePath,
2194
+ summary: `Webhook ingest: "${pageTitle}" β€” ${result.pagesUpdated} pages created.`,
2195
+ pagesAffected: [pageTitle],
2196
+ });
2197
+ webhookIngestingFiles.delete(filePath);
2198
+ res.json({
2199
+ success: !result.rejected,
2200
+ pagesCreated: result.pagesUpdated,
2201
+ title: result.title,
2202
+ rawPath: result.rawPath,
2203
+ rejected: result.rejected ?? false,
2204
+ rejectionReason: result.rejectionReason,
2205
+ });
2206
+ }
2207
+ catch (err) {
2208
+ const msg = err instanceof Error ? err.message : String(err);
2209
+ res.status(500).json({ error: `Webhook ingest failed: ${msg}` });
2210
+ }
2211
+ });
2212
+ // ─── Centralized Ingestion Gateway ───────────────────────────────────────
2213
+ // POST /api/gateway/ingest β€” universal ingestion from any source
2214
+ app.post('/api/gateway/ingest', async (req, res) => {
2215
+ const runId = `gw-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
2216
+ try {
2217
+ const { content, url, source, title, tags, metadata } = req.body;
2218
+ const sourceLabel = source ?? 'api';
2219
+ const validSources = new Set(['email', 'slack', 'webhook', 'api', 'web']);
2220
+ if (source && !validSources.has(source)) {
2221
+ res.status(400).json({ runId, error: `Invalid source. Expected one of: ${[...validSources].join(', ')}` });
2222
+ return;
2223
+ }
2224
+ if (!content && !url) {
2225
+ res.status(400).json({ runId, error: 'Must provide at least content or url' });
2226
+ return;
2227
+ }
2228
+ const { ingestSource } = await import('../core/ingest.js');
2229
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
2230
+ const { loadConfig } = await import('../core/config.js');
2231
+ const { slugify } = await import('../core/vault.js');
2232
+ const userConfig = loadConfig(config.configPath);
2233
+ const provider = createProviderFromUserConfig(userConfig);
2234
+ const auditActor = 'webhook';
2235
+ // URL-only ingestion: pass URL directly to ingestSource
2236
+ if (url && !content) {
2237
+ const result = await ingestSource(url, config, provider, {
2238
+ verbose: false,
2239
+ tags: tags ?? [],
2240
+ metadata: { ...metadata, gateway_source: sourceLabel, gateway_run: runId },
2241
+ addedBy: 'webhook',
2242
+ });
2243
+ res.json({
2244
+ runId,
2245
+ status: result.rejected ? 'rejected' : 'ingested',
2246
+ pagesCreated: result.pagesUpdated ?? 0,
2247
+ title: result.title,
2248
+ rejected: result.rejected ?? false,
2249
+ rejectionReason: result.rejectionReason,
2250
+ });
2251
+ return;
2252
+ }
2253
+ // Content-based: write to raw/ then ingest
2254
+ const now = new Date().toISOString().split('T')[0] ?? '';
2255
+ const pageTitle = title ?? `Gateway ${sourceLabel} ${new Date().toISOString()}`;
2256
+ const frontmatter = [
2257
+ `source: ${sourceLabel}`,
2258
+ `ingested_via: gateway`,
2259
+ `run_id: ${runId}`,
2260
+ ...(tags?.length ? [`tags: [${tags.join(', ')}]`] : []),
2261
+ ...(metadata ? Object.entries(metadata).map(([k, v]) => `${k}: ${v}`) : []),
2262
+ ].join('\n');
2263
+ const markdown = `---\ntitle: "${pageTitle}"\n${frontmatter}\n---\n\n${content}`;
2264
+ const dateDir = join(config.rawDir, now);
2265
+ mkdirSync(dateDir, { recursive: true });
2266
+ const slug = slugify(pageTitle.substring(0, 60));
2267
+ const filePath = join(dateDir, `${slug}-gw-${sourceLabel}.md`);
2268
+ writeFileSync(filePath, markdown, 'utf-8');
2269
+ webhookIngestingFiles.add(filePath);
2270
+ let result;
2271
+ try {
2272
+ result = await ingestSource(filePath, config, provider, {
2273
+ verbose: false,
2274
+ tags: tags ?? [],
2275
+ metadata: { ...metadata, gateway_source: sourceLabel, gateway_run: runId },
2276
+ addedBy: 'webhook',
2277
+ });
2278
+ }
2279
+ finally {
2280
+ webhookIngestingFiles.delete(filePath);
2281
+ }
2282
+ try {
2283
+ const { appendAuditEntry } = await import('../core/audit-trail.js');
2284
+ appendAuditEntry(vaultRoot, {
2285
+ action: 'ingest',
2286
+ actor: auditActor,
2287
+ source: url ?? filePath,
2288
+ summary: `Gateway ingest (${sourceLabel}): "${pageTitle}" β€” ${result.pagesUpdated} pages created.`,
2289
+ pagesAffected: [pageTitle],
2290
+ });
2291
+ }
2292
+ catch { /* non-fatal */ }
2293
+ res.json({
2294
+ runId,
2295
+ status: result.rejected ? 'rejected' : 'ingested',
2296
+ pagesCreated: result.pagesUpdated ?? 0,
2297
+ title: result.title,
2298
+ rejected: result.rejected ?? false,
2299
+ rejectionReason: result.rejectionReason,
2300
+ });
2301
+ }
2302
+ catch (err) {
2303
+ const msg = err instanceof Error ? err.message : String(err);
2304
+ res.status(500).json({ runId, status: 'error', error: `Gateway ingest failed: ${msg}` });
2305
+ }
2306
+ });
2307
+ // ─── Audit Trail API ──────────────────────────────────────────────────────
2308
+ // GET /api/audit-trail?limit=50&actor=all&action=all
2309
+ app.get('/api/audit-trail', async (req, res) => {
2310
+ try {
2311
+ const limit = parseInt(req.query['limit']) || 50;
2312
+ const actor = req.query['actor'] || 'all';
2313
+ const action = req.query['action'] || 'all';
2314
+ const since = req.query['since'];
2315
+ const before = req.query['before'];
2316
+ const { readAuditTrail } = await import('../core/audit-trail.js');
2317
+ const entries = readAuditTrail(vaultRoot, limit, actor, action, since, before);
2318
+ res.json({ entries, total: entries.length });
2319
+ }
2320
+ catch (err) {
2321
+ res.status(500).json({ error: String(err) });
2322
+ }
2323
+ });
2324
+ // ─── Automation Settings API ─────────────────────────────────────────────
2325
+ function getAutomationSettingsPath() {
2326
+ return join(vaultRoot, '.wikimem', 'automations.json');
2327
+ }
2328
+ function loadAutomationSettings() {
2329
+ const settingsPath = getAutomationSettingsPath();
2330
+ if (!existsSync(settingsPath))
2331
+ return {};
2332
+ try {
2333
+ return JSON.parse(readFileSync(settingsPath, 'utf-8'));
2334
+ }
2335
+ catch {
2336
+ return {};
2337
+ }
2338
+ }
2339
+ function saveAutomationSettings(settings) {
2340
+ const settingsPath = getAutomationSettingsPath();
2341
+ const dir = join(vaultRoot, '.wikimem');
2342
+ mkdirSync(dir, { recursive: true });
2343
+ writeFileSync(settingsPath, JSON.stringify(settings, null, 2), 'utf-8');
2344
+ }
2345
+ // GET /api/automations/settings β€” read all automation toggle state
2346
+ app.get('/api/automations/settings', (_req, res) => {
2347
+ try {
2348
+ const settings = loadAutomationSettings();
2349
+ res.json(settings);
2350
+ }
2351
+ catch (err) {
2352
+ res.status(500).json({ error: String(err) });
2353
+ }
2354
+ });
2355
+ // PATCH /api/automations/settings β€” update a single automation setting
2356
+ app.patch('/api/automations/settings', (req, res) => {
2357
+ try {
2358
+ const { automation, key, value } = req.body;
2359
+ if (!automation || !key) {
2360
+ res.status(400).json({ error: 'Missing automation or key' });
2361
+ return;
2362
+ }
2363
+ const settings = loadAutomationSettings();
2364
+ if (!settings[automation])
2365
+ settings[automation] = {};
2366
+ settings[automation][key] = value;
2367
+ saveAutomationSettings(settings);
2368
+ res.json({ status: 'saved', automation, key, value });
2369
+ }
2370
+ catch (err) {
2371
+ res.status(500).json({ error: String(err) });
2372
+ }
2373
+ });
2374
+ // POST /api/automations/sourcing/run β€” trigger smart sourcing (alias for /api/automations/scrape)
2375
+ app.post('/api/automations/sourcing/run', async (req, res) => {
2376
+ try {
2377
+ const { runSmartScraper } = await import('../core/scraper.js');
2378
+ const { loadConfig } = await import('../core/config.js');
2379
+ const userConfig = loadConfig(config.configPath);
2380
+ const result = await runSmartScraper(config, userConfig, { dryRun: false, maxItems: 10 });
2381
+ res.json(result);
2382
+ }
2383
+ catch (err) {
2384
+ const msg = err instanceof Error ? err.message : String(err);
2385
+ res.status(500).json({ error: `Sourcing run failed: ${msg}` });
2386
+ }
2387
+ });
2388
+ // ─── Observer APIs ────────────────────────────────────────────────────────
2389
+ // POST /api/observer/run β€” trigger observer manually
2390
+ app.post('/api/observer/run', async (req, res) => {
2391
+ try {
2392
+ const { autoImprove, maxImprovements, maxBudget, model } = req.body ?? {};
2393
+ const { runObserver } = await import('../core/observer.js');
2394
+ const report = await runObserver(config, { autoImprove, maxImprovements, maxBudget, model });
2395
+ res.json({
2396
+ success: true,
2397
+ date: report.date,
2398
+ totalPages: report.totalPages,
2399
+ pagesReviewed: report.pagesReviewed,
2400
+ averageScore: report.averageScore,
2401
+ maxScore: report.maxScore,
2402
+ orphanCount: report.orphans.length,
2403
+ gapCount: report.gaps.length,
2404
+ contradictionCount: report.contradictions.length,
2405
+ contradictions: report.contradictions,
2406
+ improvementCount: report.improvements?.length ?? 0,
2407
+ improvements: report.improvements ?? [],
2408
+ topIssues: report.topIssues ?? [],
2409
+ weakestPages: report.scores.filter(s => s.score < report.maxScore).slice(0, 10).map(s => ({
2410
+ title: s.title,
2411
+ score: s.score,
2412
+ maxScore: s.maxScore,
2413
+ issues: s.issues,
2414
+ })),
2415
+ reportPath: `.wikimem/observer-reports/${report.date}.json`,
2416
+ });
2417
+ }
2418
+ catch (err) {
2419
+ const msg = err instanceof Error ? err.message : String(err);
2420
+ res.status(500).json({ error: `Observer run failed: ${msg}` });
2421
+ }
2422
+ });
2423
+ // GET /api/observer/reports β€” list all reports
2424
+ app.get('/api/observer/reports', async (_req, res) => {
2425
+ try {
2426
+ const { listObserverReports } = await import('../core/observer.js');
2427
+ const reports = listObserverReports(vaultRoot);
2428
+ res.json({ reports });
2429
+ }
2430
+ catch (err) {
2431
+ res.status(500).json({ error: String(err) });
2432
+ }
2433
+ });
2434
+ // GET /api/observer/reports/:date β€” get specific report
2435
+ app.get('/api/observer/reports/:date', async (req, res) => {
2436
+ try {
2437
+ const date = req.params['date'];
2438
+ if (!date) {
2439
+ res.status(400).json({ error: 'Missing date' });
2440
+ return;
2441
+ }
2442
+ const { readObserverReport } = await import('../core/observer.js');
2443
+ const report = readObserverReport(vaultRoot, date);
2444
+ if (!report) {
2445
+ res.status(404).json({ error: `No report found for ${date}` });
2446
+ return;
2447
+ }
2448
+ res.json(report);
2449
+ }
2450
+ catch (err) {
2451
+ res.status(500).json({ error: String(err) });
2452
+ }
2453
+ });
2454
+ // ─── Lint / Health Check ──────────────────────────────────────────────────
2455
+ // POST /api/lint β€” run wiki lint check (includes contradiction detection)
2456
+ app.post('/api/lint', async (_req, res) => {
2457
+ try {
2458
+ const { lintWiki } = await import('../core/lint.js');
2459
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
2460
+ const { loadConfig } = await import('../core/config.js');
2461
+ const userConfig = loadConfig(config.configPath);
2462
+ const provider = createProviderFromUserConfig(userConfig);
2463
+ const result = await lintWiki(config, provider, { fix: false });
2464
+ res.json(result);
2465
+ }
2466
+ catch (err) {
2467
+ res.status(500).json({ error: String(err) });
2468
+ }
2469
+ });
2470
+ // GET /api/contradictions β€” get detected contradictions with page content for side-by-side diff
2471
+ app.get('/api/contradictions', async (_req, res) => {
2472
+ try {
2473
+ const { flagContradictions } = await import('../core/observer.js');
2474
+ const pages = listWikiPages(config.wikiDir);
2475
+ const contradictions = flagContradictions(pages);
2476
+ // Enrich with page content for side-by-side display
2477
+ const enriched = contradictions.map(c => {
2478
+ const pageA = readWikiPage(c.pageA);
2479
+ const pageB = readWikiPage(c.pageB);
2480
+ return {
2481
+ ...c,
2482
+ summaryA: String(pageA.frontmatter['summary'] ?? '').slice(0, 500),
2483
+ summaryB: String(pageB.frontmatter['summary'] ?? '').slice(0, 500),
2484
+ contentSnippetA: pageA.content.slice(0, 300),
2485
+ contentSnippetB: pageB.content.slice(0, 300),
2486
+ };
2487
+ });
2488
+ res.json({ contradictions: enriched, count: enriched.length });
2489
+ }
2490
+ catch (err) {
2491
+ res.status(500).json({ error: String(err) });
2492
+ }
2493
+ });
2494
+ // POST /api/contradictions/resolve β€” resolve a detected contradiction
2495
+ app.post('/api/contradictions/resolve', async (req, res) => {
2496
+ try {
2497
+ const { pageAPath, pageBPath, resolution, reason } = req.body;
2498
+ if (!pageAPath || !pageBPath || !resolution) {
2499
+ res.status(400).json({ error: 'Missing pageAPath, pageBPath, or resolution' });
2500
+ return;
2501
+ }
2502
+ const { readFileSync, writeFileSync } = await import('node:fs');
2503
+ const matter = (await import('gray-matter')).default;
2504
+ const { basename } = await import('node:path');
2505
+ // Read both pages
2506
+ const rawA = readFileSync(pageAPath, 'utf-8');
2507
+ const rawB = readFileSync(pageBPath, 'utf-8');
2508
+ const parsedA = matter(rawA);
2509
+ const parsedB = matter(rawB);
2510
+ const now = new Date().toISOString().split('T')[0] ?? '';
2511
+ const entry = {
2512
+ date: now,
2513
+ resolution,
2514
+ opposing_page: '',
2515
+ reason: reason ?? '',
2516
+ };
2517
+ // Add conflicts_resolved to the page that "won" or both for merge/dismiss
2518
+ if (resolution === 'keep-a' || resolution === 'merge' || resolution === 'dismiss') {
2519
+ const existing = Array.isArray(parsedA.data['conflicts_resolved'])
2520
+ ? parsedA.data['conflicts_resolved']
2521
+ : [];
2522
+ entry.opposing_page = basename(pageBPath, '.md');
2523
+ existing.push({ ...entry });
2524
+ parsedA.data['conflicts_resolved'] = existing;
2525
+ parsedA.data['updated'] = now;
2526
+ writeFileSync(pageAPath, matter.stringify(parsedA.content, parsedA.data), 'utf-8');
2527
+ }
2528
+ if (resolution === 'keep-b' || resolution === 'merge' || resolution === 'dismiss') {
2529
+ const existing = Array.isArray(parsedB.data['conflicts_resolved'])
2530
+ ? parsedB.data['conflicts_resolved']
2531
+ : [];
2532
+ entry.opposing_page = basename(pageAPath, '.md');
2533
+ existing.push({ ...entry });
2534
+ parsedB.data['conflicts_resolved'] = existing;
2535
+ parsedB.data['updated'] = now;
2536
+ writeFileSync(pageBPath, matter.stringify(parsedB.content, parsedB.data), 'utf-8');
2537
+ }
2538
+ // Auto-commit the resolution
2539
+ try {
2540
+ const { autoCommit, isGitRepo } = await import('../core/git.js');
2541
+ if (await isGitRepo(vaultRoot)) {
2542
+ const titleA = String(parsedA.data['title'] ?? basename(pageAPath, '.md'));
2543
+ const titleB = String(parsedB.data['title'] ?? basename(pageBPath, '.md'));
2544
+ await autoCommit(vaultRoot, 'resolve', `conflict between "${titleA}" and "${titleB}" β†’ ${resolution}`, reason ?? '');
2545
+ }
2546
+ }
2547
+ catch { /* non-fatal */ }
2548
+ // Audit trail
2549
+ try {
2550
+ const { appendAuditEntry } = await import('../core/audit-trail.js');
2551
+ appendAuditEntry(vaultRoot, {
2552
+ action: 'edit',
2553
+ actor: 'human',
2554
+ summary: `Resolved conflict: ${resolution} (${basename(pageAPath, '.md')} vs ${basename(pageBPath, '.md')})${reason ? ': ' + reason : ''}`,
2555
+ pagesAffected: [basename(pageAPath, '.md'), basename(pageBPath, '.md')],
2556
+ });
2557
+ }
2558
+ catch { /* non-fatal */ }
2559
+ res.json({ success: true, resolution });
2560
+ }
2561
+ catch (err) {
2562
+ res.status(500).json({ error: String(err) });
2563
+ }
2564
+ });
2565
+ // ─── Automations: Scrape ──────────────────────────────────────────────────
2566
+ // POST /api/automations/scrape β€” trigger scrape for a source (or all)
2567
+ app.post('/api/automations/scrape', async (req, res) => {
2568
+ try {
2569
+ const { source: sourceName, dryRun, maxItems } = req.body;
2570
+ const { runSmartScraper } = await import('../core/scraper.js');
2571
+ const { loadConfig } = await import('../core/config.js');
2572
+ const userConfig = loadConfig(config.configPath);
2573
+ const result = await runSmartScraper(config, userConfig, { dryRun: dryRun ?? false, maxItems: maxItems ?? 10 }, sourceName);
2574
+ res.json(result);
2575
+ }
2576
+ catch (err) {
2577
+ const msg = err instanceof Error ? err.message : String(err);
2578
+ res.status(500).json({ error: `Scrape failed: ${msg}` });
2579
+ }
2580
+ });
2581
+ // POST /api/automations/observe β€” trigger observer with optional budget
2582
+ app.post('/api/automations/observe', async (req, res) => {
2583
+ try {
2584
+ const { maxPagesToReview, maxCostEstimate } = req.body;
2585
+ void maxCostEstimate;
2586
+ const { runObserver } = await import('../core/observer.js');
2587
+ const report = await runObserver(config, { maxPagesToReview });
2588
+ res.json({
2589
+ success: true,
2590
+ date: report.date,
2591
+ totalPages: report.totalPages,
2592
+ pagesReviewed: report.pagesReviewed,
2593
+ averageScore: report.averageScore,
2594
+ maxScore: report.maxScore,
2595
+ orphanCount: report.orphans.length,
2596
+ gapCount: report.gaps.length,
2597
+ contradictionCount: report.contradictions.length,
2598
+ reportPath: `.wikimem/observer-reports/${report.date}.json`,
2599
+ });
2600
+ }
2601
+ catch (err) {
2602
+ const msg = err instanceof Error ? err.message : String(err);
2603
+ res.status(500).json({ error: `Observer run failed: ${msg}` });
2604
+ }
2605
+ });
2606
+ // GET /api/automations/status β€” status of all three automations
2607
+ app.get('/api/automations/status', async (_req, res) => {
2608
+ try {
2609
+ const { readAuditTrail } = await import('../core/audit-trail.js');
2610
+ const { isObserverCronRunning } = await import('../core/observer.js');
2611
+ const { loadConfig } = await import('../core/config.js');
2612
+ const userConfig = loadConfig(config.configPath);
2613
+ const settings = loadAutomationSettings();
2614
+ const recentIngest = readAuditTrail(vaultRoot, 1, undefined, 'ingest');
2615
+ const recentScrape = readAuditTrail(vaultRoot, 1, undefined, 'scrape');
2616
+ const recentObserve = readAuditTrail(vaultRoot, 1, undefined, 'observe');
2617
+ res.json({
2618
+ ingest: {
2619
+ enabled: settings['ingest']?.['enabled'] !== false,
2620
+ lastRun: recentIngest[0]?.timestamp ?? null,
2621
+ watcherActive: true,
2622
+ },
2623
+ scrape: {
2624
+ enabled: settings['scrape']?.['enabled'] !== false,
2625
+ lastRun: recentScrape[0]?.timestamp ?? null,
2626
+ sourcesConfigured: (userConfig.sources ?? []).length,
2627
+ },
2628
+ observe: {
2629
+ enabled: settings['observe']?.['enabled'] !== false,
2630
+ lastRun: recentObserve[0]?.timestamp ?? null,
2631
+ cronActive: isObserverCronRunning(),
2632
+ schedule: '0 3 * * *',
2633
+ },
2634
+ });
2635
+ }
2636
+ catch (err) {
2637
+ const msg = err instanceof Error ? err.message : String(err);
2638
+ res.status(500).json({ error: `Failed to get automation status: ${msg}` });
2639
+ }
2640
+ });
2641
+ // ─── Connector APIs ───────────────────────────────────────────────────────
2642
+ // GET /api/connectors β€” list all configured connectors
2643
+ app.get('/api/connectors', async (_req, res) => {
2644
+ try {
2645
+ const { getConnectorManager } = await import('../core/connectors.js');
2646
+ const mgr = getConnectorManager(vaultRoot);
2647
+ const connectors = mgr.getAll().map(c => ({
2648
+ ...c,
2649
+ exists: existsSync(c.path),
2650
+ }));
2651
+ res.json({ connectors });
2652
+ }
2653
+ catch (err) {
2654
+ res.status(500).json({ error: String(err) });
2655
+ }
2656
+ });
2657
+ // POST /api/connectors β€” add a new connector
2658
+ app.post('/api/connectors', async (req, res) => {
2659
+ try {
2660
+ const { getConnectorManager } = await import('../core/connectors.js');
2661
+ const mgr = getConnectorManager(vaultRoot);
2662
+ const body = req.body;
2663
+ if (!body.type) {
2664
+ res.status(400).json({ error: 'Missing type' });
2665
+ return;
2666
+ }
2667
+ let resolvedPath = body.path || '';
2668
+ // RSS and Notion connectors don't need a filesystem path
2669
+ if (body.type === 'rss' || body.type === 'notion') {
2670
+ if (!body.url && body.type === 'rss') {
2671
+ res.status(400).json({ error: 'RSS connector requires a feed URL' });
2672
+ return;
2673
+ }
2674
+ resolvedPath = resolvedPath || join(vaultRoot, 'raw');
2675
+ mkdirSync(resolvedPath, { recursive: true });
2676
+ }
2677
+ else if (body.type === 'github' || (body.type === 'git-repo' && body.url)) {
2678
+ // For git repos, clone if URL provided
2679
+ const reposDir = join(vaultRoot, '.wikimem-repos');
2680
+ mkdirSync(reposDir, { recursive: true });
2681
+ const repoName = (body.url ?? '').split('/').pop()?.replace('.git', '') || 'repo';
2682
+ resolvedPath = join(reposDir, repoName);
2683
+ if (!existsSync(resolvedPath)) {
2684
+ const result = await mgr.cloneRepo(body.url, resolvedPath);
2685
+ if (!result.success) {
2686
+ res.status(500).json({ error: result.error });
2687
+ return;
2688
+ }
2689
+ }
2690
+ }
2691
+ if (!resolvedPath || !existsSync(resolvedPath)) {
2692
+ res.status(400).json({ error: `Path does not exist: ${resolvedPath}` });
2693
+ return;
2694
+ }
2695
+ const connector = mgr.add({
2696
+ type: body.type,
2697
+ name: body.name || (body.type === 'rss' ? (body.url ?? 'RSS Feed') : basename(resolvedPath)),
2698
+ path: resolvedPath,
2699
+ url: body.url,
2700
+ includeGlobs: body.includeGlobs,
2701
+ excludeGlobs: body.excludeGlobs,
2702
+ autoSync: body.autoSync ?? false,
2703
+ syncSchedule: body.syncSchedule,
2704
+ });
2705
+ if (connector.autoSync)
2706
+ mgr.startWatcher(connector);
2707
+ res.json({ connector });
2708
+ }
2709
+ catch (err) {
2710
+ res.status(500).json({ error: String(err) });
2711
+ }
2712
+ });
2713
+ // DELETE /api/connectors/:id β€” remove a connector
2714
+ app.delete('/api/connectors/:id', async (req, res) => {
2715
+ try {
2716
+ const { getConnectorManager } = await import('../core/connectors.js');
2717
+ const mgr = getConnectorManager(vaultRoot);
2718
+ const removed = mgr.remove(req.params['id']);
2719
+ res.json({ removed });
2720
+ }
2721
+ catch (err) {
2722
+ res.status(500).json({ error: String(err) });
2723
+ }
2724
+ });
2725
+ // POST /api/connectors/:id/sync β€” manually trigger a sync (scan + ingest all files)
2726
+ app.post('/api/connectors/:id/sync', async (req, res) => {
2727
+ try {
2728
+ const { getConnectorManager } = await import('../core/connectors.js');
2729
+ const { ingestSource } = await import('../core/ingest.js');
2730
+ const mgr = getConnectorManager(vaultRoot);
2731
+ const connector = mgr.get(req.params['id']);
2732
+ if (!connector) {
2733
+ res.status(404).json({ error: 'Connector not found' });
2734
+ return;
2735
+ }
2736
+ mgr.updateStatus(connector.id, 'syncing');
2737
+ const startMs = Date.now();
2738
+ const files = await mgr.scanFiles(connector);
2739
+ let pagesCreated = 0, linksAdded = 0, ingested = 0;
2740
+ const errors = [];
2741
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
2742
+ const { loadConfig } = await import('../core/config.js');
2743
+ const userConfig = loadConfig(config.configPath);
2744
+ const provider = createProviderFromUserConfig(userConfig);
2745
+ for (const filePath of files.slice(0, 50)) { // cap at 50 per sync
2746
+ try {
2747
+ const result = await ingestSource(filePath, config, provider, { verbose: false });
2748
+ pagesCreated += result.pagesUpdated ?? 0;
2749
+ linksAdded += result.linksAdded ?? 0;
2750
+ ingested++;
2751
+ }
2752
+ catch (e) {
2753
+ errors.push(`${basename(filePath)}: ${String(e).substring(0, 100)}`);
2754
+ }
2755
+ }
2756
+ mgr.updateStatus(connector.id, 'active', {
2757
+ lastSyncAt: new Date().toISOString(),
2758
+ totalFiles: files.length,
2759
+ });
2760
+ const result = {
2761
+ connectorId: connector.id,
2762
+ filesFound: files.length,
2763
+ filesIngested: ingested,
2764
+ pagesCreated,
2765
+ linksAdded,
2766
+ errors,
2767
+ duration: Date.now() - startMs,
2768
+ };
2769
+ res.json(result);
2770
+ }
2771
+ catch (err) {
2772
+ res.status(500).json({ error: String(err) });
2773
+ }
2774
+ });
2775
+ // POST /api/connectors/:id/scan β€” scan files in connector (no ingest)
2776
+ app.get('/api/connectors/:id/scan', async (req, res) => {
2777
+ try {
2778
+ const { getConnectorManager } = await import('../core/connectors.js');
2779
+ const mgr = getConnectorManager(vaultRoot);
2780
+ const connector = mgr.get(req.params['id']);
2781
+ if (!connector) {
2782
+ res.status(404).json({ error: 'Not found' });
2783
+ return;
2784
+ }
2785
+ const files = await mgr.scanFiles(connector);
2786
+ res.json({ files: files.slice(0, 200), total: files.length });
2787
+ }
2788
+ catch (err) {
2789
+ res.status(500).json({ error: String(err) });
2790
+ }
2791
+ });
2792
+ // ─── Platform Sync APIs ──────────────────────────────────────────────────
2793
+ // Shared scheduler singleton β€” reused across routes and startup
2794
+ let syncScheduler = null;
2795
+ async function getSyncScheduler() {
2796
+ if (!syncScheduler) {
2797
+ const { SyncScheduler } = await import('../core/sync/index.js');
2798
+ syncScheduler = new SyncScheduler(vaultRoot);
2799
+ }
2800
+ return syncScheduler;
2801
+ }
2802
+ // IMPORTANT: Specific routes MUST be registered before the generic :provider route
2803
+ // to avoid Express matching "rss" or "schedules" as a provider name.
2804
+ // GET /api/sync/schedules β€” list all active sync schedules
2805
+ app.get('/api/sync/schedules', async (_req, res) => {
2806
+ try {
2807
+ const scheduler = await getSyncScheduler();
2808
+ res.json({ schedules: scheduler.getSchedules() });
2809
+ }
2810
+ catch (err) {
2811
+ res.status(500).json({ error: String(err) });
2812
+ }
2813
+ });
2814
+ // POST /api/sync/rss/:connectorId β€” trigger RSS feed sync for a specific connector
2815
+ app.post('/api/sync/rss/:connectorId', async (req, res) => {
2816
+ try {
2817
+ const connectorId = req.params['connectorId'];
2818
+ if (!connectorId) {
2819
+ res.status(400).json({ error: 'Missing connectorId' });
2820
+ return;
2821
+ }
2822
+ const { syncRssConnector } = await import('../core/sync/index.js');
2823
+ const result = await syncRssConnector(connectorId, vaultRoot);
2824
+ res.json(result);
2825
+ }
2826
+ catch (err) {
2827
+ const msg = err instanceof Error ? err.message : String(err);
2828
+ res.status(500).json({ error: `RSS sync failed: ${msg}` });
2829
+ }
2830
+ });
2831
+ // POST /api/sync/:provider/schedule β€” set sync schedule for a provider
2832
+ app.post('/api/sync/:provider/schedule', async (req, res) => {
2833
+ try {
2834
+ const provider = req.params['provider'];
2835
+ const { schedule } = req.body;
2836
+ if (!provider || !schedule) {
2837
+ res.status(400).json({ error: 'Missing provider or schedule' });
2838
+ return;
2839
+ }
2840
+ const scheduler = await getSyncScheduler();
2841
+ scheduler.schedule(provider, schedule);
2842
+ res.json({ status: 'scheduled', provider, schedule });
2843
+ }
2844
+ catch (err) {
2845
+ res.status(500).json({ error: String(err) });
2846
+ }
2847
+ });
2848
+ // POST /api/sync/:provider β€” trigger sync for a connected OAuth provider
2849
+ // MUST be last among /api/sync/* routes (generic :provider catches everything)
2850
+ app.post('/api/sync/:provider', async (req, res) => {
2851
+ try {
2852
+ const provider = req.params['provider'];
2853
+ if (!provider) {
2854
+ res.status(400).json({ error: 'Missing provider' });
2855
+ return;
2856
+ }
2857
+ const { syncProvider } = await import('../core/sync/index.js');
2858
+ const result = await syncProvider(provider, vaultRoot);
2859
+ res.json(result);
2860
+ }
2861
+ catch (err) {
2862
+ const msg = err instanceof Error ? err.message : String(err);
2863
+ res.status(500).json({ error: `Sync failed: ${msg}` });
2864
+ }
2865
+ });
2866
+ // ─── Enhanced Webhooks ──────────────────────────────────────────────────
2867
+ // POST /api/webhook/github β€” receive GitHub push/issue/PR webhooks
2868
+ app.post('/api/webhook/github', async (req, res) => {
2869
+ try {
2870
+ const { parseGitHubWebhook, webhookToMarkdown } = await import('../core/webhooks.js');
2871
+ const headers = {};
2872
+ for (const [k, v] of Object.entries(req.headers)) {
2873
+ if (typeof v === 'string')
2874
+ headers[k] = v;
2875
+ }
2876
+ const payload = parseGitHubWebhook(headers, req.body);
2877
+ if (!payload) {
2878
+ res.status(200).json({ status: 'ignored' });
2879
+ return;
2880
+ }
2881
+ const markdown = webhookToMarkdown(payload);
2882
+ const { mkdirSync: mkdir, writeFileSync: write } = await import('node:fs');
2883
+ const { join: joinPath } = await import('node:path');
2884
+ const { slugify } = await import('../core/vault.js');
2885
+ const now = new Date().toISOString().split('T')[0] ?? '';
2886
+ const dateDir = joinPath(config.rawDir, now);
2887
+ mkdir(dateDir, { recursive: true });
2888
+ const filePath = joinPath(dateDir, `github-${payload.event}-${slugify(payload.title.substring(0, 40))}.md`);
2889
+ write(filePath, markdown, 'utf-8');
2890
+ res.json({ status: 'received', event: payload.event, title: payload.title });
2891
+ }
2892
+ catch (err) {
2893
+ res.status(500).json({ error: String(err) });
2894
+ }
2895
+ });
2896
+ // POST /api/webhook/slack β€” receive Slack events and slash commands
2897
+ app.post('/api/webhook/slack', async (req, res) => {
2898
+ try {
2899
+ const { parseSlackWebhook, webhookToMarkdown } = await import('../core/webhooks.js');
2900
+ const headers = {};
2901
+ for (const [k, v] of Object.entries(req.headers)) {
2902
+ if (typeof v === 'string')
2903
+ headers[k] = v;
2904
+ }
2905
+ const payload = parseSlackWebhook(headers, req.body);
2906
+ if (!payload) {
2907
+ res.status(200).json({ status: 'ignored' });
2908
+ return;
2909
+ }
2910
+ // Handle Slack URL verification challenge
2911
+ if (payload.event === 'url_verification') {
2912
+ res.json({ challenge: payload.content });
2913
+ return;
2914
+ }
2915
+ const markdown = webhookToMarkdown(payload);
2916
+ const { mkdirSync: mkdir, writeFileSync: write } = await import('node:fs');
2917
+ const { join: joinPath } = await import('node:path');
2918
+ const { slugify } = await import('../core/vault.js');
2919
+ const now = new Date().toISOString().split('T')[0] ?? '';
2920
+ const dateDir = joinPath(config.rawDir, now);
2921
+ mkdir(dateDir, { recursive: true });
2922
+ const filePath = joinPath(dateDir, `slack-${payload.event}-${slugify(payload.title.substring(0, 40))}.md`);
2923
+ write(filePath, markdown, 'utf-8');
2924
+ res.json({ status: 'received', event: payload.event, title: payload.title });
2925
+ }
2926
+ catch (err) {
2927
+ res.status(500).json({ error: String(err) });
2928
+ }
2929
+ });
2930
+ // Start SyncScheduler for connected providers (reuses singleton from routes)
2931
+ getSyncScheduler().then((scheduler) => {
2932
+ scheduler.startFromConfig();
2933
+ scheduler.on('sync-complete', (result) => {
2934
+ console.log(`[sync] ${result.provider}: ${result.filesWritten} files synced`);
2935
+ });
2936
+ }).catch(() => { });
2937
+ // Wire file-detected events from connectors to auto-ingest
2938
+ (async () => {
2939
+ const { getConnectorManager } = await import('../core/connectors.js');
2940
+ const { ingestSource } = await import('../core/ingest.js');
2941
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
2942
+ const { loadConfig } = await import('../core/config.js');
2943
+ const mgr = getConnectorManager(vaultRoot);
2944
+ mgr.on('file-detected', async ({ connectorId, filePath }) => {
2945
+ const connector = mgr.get(connectorId);
2946
+ if (!connector)
2947
+ return;
2948
+ mgr.updateStatus(connectorId, 'syncing');
2949
+ try {
2950
+ const userConfig = loadConfig(config.configPath);
2951
+ const provider = createProviderFromUserConfig(userConfig);
2952
+ await ingestSource(filePath, config, provider, { verbose: false });
2953
+ mgr.updateStatus(connectorId, 'active', { lastSyncAt: new Date().toISOString() });
2954
+ }
2955
+ catch {
2956
+ mgr.updateStatus(connectorId, 'error', { errorMessage: `Failed to ingest ${basename(filePath)}` });
2957
+ }
2958
+ });
2959
+ mgr.startAllWatchers();
2960
+ })().catch(() => { });
2961
+ // Start Observer nightly cron (3am)
2962
+ import('../core/observer.js').then(({ startObserverCron }) => {
2963
+ startObserverCron(config);
2964
+ }).catch(() => { });
2965
+ // AUTO-005: Watch raw/ directory for new files, auto-trigger ingest
2966
+ (async () => {
2967
+ try {
2968
+ const chokidar = await import('chokidar');
2969
+ const { ingestSource } = await import('../core/ingest.js');
2970
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
2971
+ const { loadConfig } = await import('../core/config.js');
2972
+ const rawDir = config.rawDir;
2973
+ if (!existsSync(rawDir))
2974
+ return;
2975
+ const watcher = chokidar.watch(rawDir, {
2976
+ ignored: ['**/.git/**', '**/*.tmp'],
2977
+ persistent: true,
2978
+ ignoreInitial: true,
2979
+ awaitWriteFinish: { stabilityThreshold: 1500 },
2980
+ });
2981
+ watcher.on('add', async (filePath) => {
2982
+ const INGESTIBLE = new Set(['.md', '.txt', '.pdf', '.json', '.yaml', '.yml', '.csv', '.html', '.docx', '.mp3', '.wav', '.m4a', '.mp4', '.mov']);
2983
+ const ext = filePath.split('.').pop()?.toLowerCase() || '';
2984
+ if (!INGESTIBLE.has('.' + ext))
2985
+ return;
2986
+ if (webhookIngestingFiles.has(filePath)) {
2987
+ console.log(`[watcher] Skipping ${filePath} (already being ingested by webhook)`);
2988
+ return;
2989
+ }
2990
+ console.log(`[watcher] New file detected: ${filePath}`);
2991
+ try {
2992
+ const userConfig = loadConfig(config.configPath);
2993
+ const provider = createProviderFromUserConfig(userConfig);
2994
+ await ingestSource(filePath, config, provider, { verbose: false });
2995
+ console.log(`[watcher] Ingested: ${filePath}`);
2996
+ }
2997
+ catch (err) {
2998
+ console.error(`[watcher] Failed to ingest ${filePath}:`, err);
2999
+ }
3000
+ });
3001
+ console.log(` Auto-watching raw/ for new files: ${rawDir}`);
3002
+ }
3003
+ catch { }
3004
+ })().catch(() => { });
3005
+ // Serve static files AFTER all API routes
3006
+ if (existsSync(publicDir)) {
3007
+ app.use(express.static(publicDir));
3008
+ }
229
3009
  // Serve index.html for all other routes (SPA)
230
3010
  app.get('/{*path}', (_req, res) => {
231
3011
  const indexPath = join(publicDir, 'index.html');