wikimem 0.3.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +138 -29
  2. package/README.md +171 -309
  3. package/dist/cli/commands/ask.d.ts +3 -0
  4. package/dist/cli/commands/ask.d.ts.map +1 -0
  5. package/dist/cli/commands/ask.js +63 -0
  6. package/dist/cli/commands/ask.js.map +1 -0
  7. package/dist/cli/commands/export.d.ts +3 -0
  8. package/dist/cli/commands/export.d.ts.map +1 -0
  9. package/dist/cli/commands/export.js +108 -0
  10. package/dist/cli/commands/export.js.map +1 -0
  11. package/dist/cli/commands/history.d.ts +3 -0
  12. package/dist/cli/commands/history.d.ts.map +1 -0
  13. package/dist/cli/commands/history.js +61 -0
  14. package/dist/cli/commands/history.js.map +1 -0
  15. package/dist/cli/commands/improve.d.ts.map +1 -1
  16. package/dist/cli/commands/improve.js +4 -3
  17. package/dist/cli/commands/improve.js.map +1 -1
  18. package/dist/cli/commands/ingest.d.ts.map +1 -1
  19. package/dist/cli/commands/ingest.js +5 -4
  20. package/dist/cli/commands/ingest.js.map +1 -1
  21. package/dist/cli/commands/init.d.ts.map +1 -1
  22. package/dist/cli/commands/init.js +246 -79
  23. package/dist/cli/commands/init.js.map +1 -1
  24. package/dist/cli/commands/lint.d.ts.map +1 -1
  25. package/dist/cli/commands/lint.js +4 -3
  26. package/dist/cli/commands/lint.js.map +1 -1
  27. package/dist/cli/commands/mcp.d.ts +3 -0
  28. package/dist/cli/commands/mcp.d.ts.map +1 -0
  29. package/dist/cli/commands/mcp.js +11 -0
  30. package/dist/cli/commands/mcp.js.map +1 -0
  31. package/dist/cli/commands/open.d.ts +3 -0
  32. package/dist/cli/commands/open.d.ts.map +1 -0
  33. package/dist/cli/commands/open.js +36 -0
  34. package/dist/cli/commands/open.js.map +1 -0
  35. package/dist/cli/commands/query.d.ts.map +1 -1
  36. package/dist/cli/commands/query.js +5 -4
  37. package/dist/cli/commands/query.js.map +1 -1
  38. package/dist/cli/commands/search.d.ts +3 -0
  39. package/dist/cli/commands/search.d.ts.map +1 -0
  40. package/dist/cli/commands/search.js +61 -0
  41. package/dist/cli/commands/search.js.map +1 -0
  42. package/dist/cli/commands/serve.d.ts.map +1 -1
  43. package/dist/cli/commands/serve.js +41 -2
  44. package/dist/cli/commands/serve.js.map +1 -1
  45. package/dist/cli/commands/watch.d.ts.map +1 -1
  46. package/dist/cli/commands/watch.js +4 -3
  47. package/dist/cli/commands/watch.js.map +1 -1
  48. package/dist/cli/index.d.ts.map +1 -1
  49. package/dist/cli/index.js +27 -1
  50. package/dist/cli/index.js.map +1 -1
  51. package/dist/core/audit-trail.d.ts +15 -0
  52. package/dist/core/audit-trail.d.ts.map +1 -0
  53. package/dist/core/audit-trail.js +43 -0
  54. package/dist/core/audit-trail.js.map +1 -0
  55. package/dist/core/claude-code.d.ts +10 -0
  56. package/dist/core/claude-code.d.ts.map +1 -0
  57. package/dist/core/claude-code.js +81 -0
  58. package/dist/core/claude-code.js.map +1 -0
  59. package/dist/core/config.d.ts +23 -0
  60. package/dist/core/config.d.ts.map +1 -1
  61. package/dist/core/config.js.map +1 -1
  62. package/dist/core/connectors.d.ts +58 -0
  63. package/dist/core/connectors.d.ts.map +1 -0
  64. package/dist/core/connectors.js +189 -0
  65. package/dist/core/connectors.js.map +1 -0
  66. package/dist/core/folder-scanner.d.ts +10 -0
  67. package/dist/core/folder-scanner.d.ts.map +1 -0
  68. package/dist/core/folder-scanner.js +84 -0
  69. package/dist/core/folder-scanner.js.map +1 -0
  70. package/dist/core/git.d.ts +137 -0
  71. package/dist/core/git.d.ts.map +1 -0
  72. package/dist/core/git.js +520 -0
  73. package/dist/core/git.js.map +1 -0
  74. package/dist/core/history.d.ts +21 -0
  75. package/dist/core/history.d.ts.map +1 -0
  76. package/dist/core/history.js +107 -0
  77. package/dist/core/history.js.map +1 -0
  78. package/dist/core/improve.d.ts.map +1 -1
  79. package/dist/core/improve.js +9 -0
  80. package/dist/core/improve.js.map +1 -1
  81. package/dist/core/ingest.d.ts +1 -0
  82. package/dist/core/ingest.d.ts.map +1 -1
  83. package/dist/core/ingest.js +78 -5
  84. package/dist/core/ingest.js.map +1 -1
  85. package/dist/core/observer.d.ts +71 -0
  86. package/dist/core/observer.d.ts.map +1 -0
  87. package/dist/core/observer.js +350 -0
  88. package/dist/core/observer.js.map +1 -0
  89. package/dist/core/pipeline-events.d.ts +63 -0
  90. package/dist/core/pipeline-events.d.ts.map +1 -0
  91. package/dist/core/pipeline-events.js +109 -0
  92. package/dist/core/pipeline-events.js.map +1 -0
  93. package/dist/core/query.d.ts.map +1 -1
  94. package/dist/core/query.js +16 -8
  95. package/dist/core/query.js.map +1 -1
  96. package/dist/core/scraper.d.ts +41 -0
  97. package/dist/core/scraper.d.ts.map +1 -0
  98. package/dist/core/scraper.js +277 -0
  99. package/dist/core/scraper.js.map +1 -0
  100. package/dist/index.js +3 -2
  101. package/dist/index.js.map +1 -1
  102. package/dist/mcp-entry.d.ts +10 -0
  103. package/dist/mcp-entry.d.ts.map +1 -0
  104. package/dist/mcp-entry.js +21 -0
  105. package/dist/mcp-entry.js.map +1 -0
  106. package/dist/mcp-server.d.ts +15 -0
  107. package/dist/mcp-server.d.ts.map +1 -0
  108. package/dist/mcp-server.js +390 -0
  109. package/dist/mcp-server.js.map +1 -0
  110. package/dist/processors/audio.d.ts.map +1 -1
  111. package/dist/processors/audio.js +42 -4
  112. package/dist/processors/audio.js.map +1 -1
  113. package/dist/processors/pdf.d.ts.map +1 -1
  114. package/dist/processors/pdf.js +2 -3
  115. package/dist/processors/pdf.js.map +1 -1
  116. package/dist/processors/url.js +4 -1
  117. package/dist/processors/url.js.map +1 -1
  118. package/dist/providers/claude.d.ts +1 -0
  119. package/dist/providers/claude.d.ts.map +1 -1
  120. package/dist/providers/claude.js +5 -3
  121. package/dist/providers/claude.js.map +1 -1
  122. package/dist/providers/index.d.ts +17 -1
  123. package/dist/providers/index.d.ts.map +1 -1
  124. package/dist/providers/index.js +144 -0
  125. package/dist/providers/index.js.map +1 -1
  126. package/dist/providers/openai.d.ts +1 -0
  127. package/dist/providers/openai.d.ts.map +1 -1
  128. package/dist/providers/openai.js +5 -3
  129. package/dist/providers/openai.js.map +1 -1
  130. package/dist/providers/types.d.ts +18 -0
  131. package/dist/providers/types.d.ts.map +1 -1
  132. package/dist/templates/config-yaml.d.ts.map +1 -1
  133. package/dist/templates/config-yaml.js +12 -1
  134. package/dist/templates/config-yaml.js.map +1 -1
  135. package/dist/web/public/index.html +8157 -745
  136. package/dist/web/server.d.ts.map +1 -1
  137. package/dist/web/server.js +2101 -29
  138. package/dist/web/server.js.map +1 -1
  139. package/package.json +7 -4
  140. package/scripts/install.sh +54 -0
  141. package/src/web/public/index.html +8157 -745
  142. package/templates/mcp-config.json +9 -0
  143. package/dist/web/public/public/index.html +0 -946
@@ -1,8 +1,9 @@
1
1
  import express from 'express';
2
- import { existsSync, writeFileSync, mkdirSync, readdirSync, statSync } from 'node:fs';
3
- import { join, basename } from 'node:path';
2
+ import { readFileSync, existsSync, writeFileSync, mkdirSync, readdirSync, statSync, renameSync, unlinkSync as fsUnlinkSync } from 'node:fs';
3
+ import { join, resolve, extname, basename, dirname, relative } from 'node:path';
4
4
  import { fileURLToPath } from 'node:url';
5
- import { getVaultConfig, getVaultStats, listWikiPages, readWikiPage } from '../core/vault.js';
5
+ import { randomBytes } from 'node:crypto';
6
+ import { getVaultConfig, getVaultStats, listWikiPages, readWikiPage, writeWikiPage } from '../core/vault.js';
6
7
  const __dirname = fileURLToPath(new URL('.', import.meta.url));
7
8
  function buildGraph(config) {
8
9
  const pages = listWikiPages(config.wikiDir);
@@ -71,13 +72,13 @@ function listPages(config) {
71
72
  export function createServer(vaultRoot, port) {
72
73
  const app = express();
73
74
  const config = getVaultConfig(vaultRoot);
75
+ // Load persisted pipeline runs so they survive server restarts
76
+ import('../core/pipeline-events.js').then(({ pipelineEvents }) => {
77
+ pipelineEvents.initPersistence(vaultRoot);
78
+ }).catch(() => { });
74
79
  app.use(express.json());
75
80
  app.use(express.urlencoded({ extended: true }));
76
- // Serve static files
77
81
  const publicDir = join(__dirname, 'public');
78
- if (existsSync(publicDir)) {
79
- app.use(express.static(publicDir));
80
- }
81
82
  // API: vault status
82
83
  app.get('/api/status', (_req, res) => {
83
84
  try {
@@ -98,6 +99,28 @@ export function createServer(vaultRoot, port) {
98
99
  res.status(500).json({ error: 'Failed to list pages' });
99
100
  }
100
101
  });
102
+ // API: create new page
103
+ app.post('/api/pages', (req, res) => {
104
+ try {
105
+ const { title, slug, content } = req.body;
106
+ if (!title || !slug) {
107
+ res.status(400).json({ error: 'Missing title or slug' });
108
+ return;
109
+ }
110
+ const dest = join(config.wikiDir, `${slug}.md`);
111
+ if (existsSync(dest)) {
112
+ res.status(409).json({ error: 'Page already exists' });
113
+ return;
114
+ }
115
+ mkdirSync(config.wikiDir, { recursive: true });
116
+ writeFileSync(dest, content ?? `---\ntitle: "${title}"\ntype: page\n---\n\n# ${title}\n`, 'utf-8');
117
+ res.json({ status: 'created', path: dest, title });
118
+ }
119
+ catch (err) {
120
+ const msg = err instanceof Error ? err.message : String(err);
121
+ res.status(500).json({ error: `Failed to create page: ${msg}` });
122
+ }
123
+ });
101
124
  // API: read single page
102
125
  app.get('/api/pages/:title', (req, res) => {
103
126
  try {
@@ -133,6 +156,240 @@ export function createServer(vaultRoot, port) {
133
156
  res.status(500).json({ error: 'Failed to read page' });
134
157
  }
135
158
  });
159
+ // API: read raw page content (for editing)
160
+ app.get('/api/wiki/page/raw', (req, res) => {
161
+ try {
162
+ const pagePath = req.query['path'];
163
+ if (!pagePath) {
164
+ res.status(400).json({ error: 'Missing path query parameter' });
165
+ return;
166
+ }
167
+ const resolved = resolve(pagePath);
168
+ if (!resolved.startsWith(resolve(config.wikiDir))) {
169
+ res.status(403).json({ error: 'Access denied: path outside wiki directory' });
170
+ return;
171
+ }
172
+ if (!existsSync(resolved)) {
173
+ res.status(404).json({ error: 'File not found' });
174
+ return;
175
+ }
176
+ const raw = readFileSync(resolved, 'utf-8');
177
+ res.json({ path: resolved, raw });
178
+ }
179
+ catch (err) {
180
+ const msg = err instanceof Error ? err.message : String(err);
181
+ res.status(500).json({ error: `Failed to read raw page: ${msg}` });
182
+ }
183
+ });
184
+ // API: save wiki page (write to disk + auto-commit)
185
+ app.put('/api/wiki/page', async (req, res) => {
186
+ try {
187
+ const { path: pagePath, content, frontmatter } = req.body;
188
+ if (!pagePath || content === undefined) {
189
+ res.status(400).json({ error: 'Missing path or content' });
190
+ return;
191
+ }
192
+ const resolved = resolve(pagePath);
193
+ if (!resolved.startsWith(resolve(config.wikiDir))) {
194
+ res.status(403).json({ error: 'Access denied: path outside wiki directory' });
195
+ return;
196
+ }
197
+ // Parse the raw content — if it includes frontmatter, use gray-matter
198
+ const matter = await import('gray-matter');
199
+ const parsed = matter.default(content);
200
+ const finalFrontmatter = frontmatter ?? parsed.data;
201
+ const finalContent = frontmatter ? content : parsed.content;
202
+ writeWikiPage(resolved, finalContent, finalFrontmatter);
203
+ // Auto-commit if git-initialized
204
+ let commitResult = null;
205
+ try {
206
+ const { autoCommit, isGitRepo } = await import('../core/git.js');
207
+ if (await isGitRepo(config.root)) {
208
+ const title = finalFrontmatter['title'] ?? basename(resolved, '.md');
209
+ commitResult = await autoCommit(config.root, 'manual', `edit "${title}" via web UI`);
210
+ }
211
+ }
212
+ catch { /* non-fatal */ }
213
+ res.json({ status: 'saved', path: resolved, commit: commitResult });
214
+ }
215
+ catch (err) {
216
+ const msg = err instanceof Error ? err.message : String(err);
217
+ res.status(500).json({ error: `Failed to save page: ${msg}` });
218
+ }
219
+ });
220
+ // API: read raw page content (full markdown with frontmatter)
221
+ app.get('/api/pages/:title/raw', (req, res) => {
222
+ try {
223
+ const title = req.params['title'];
224
+ if (!title) {
225
+ res.status(400).json({ error: 'Missing title' });
226
+ return;
227
+ }
228
+ const pages = listWikiPages(config.wikiDir);
229
+ const titleLower = title.toLowerCase();
230
+ const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
231
+ const match = pages.find((p) => {
232
+ const fileSlug = basename(p, '.md');
233
+ if (fileSlug === title || fileSlug === slugified)
234
+ return true;
235
+ try {
236
+ const page = readWikiPage(p);
237
+ return page.title.toLowerCase() === titleLower;
238
+ }
239
+ catch {
240
+ return false;
241
+ }
242
+ });
243
+ if (!match) {
244
+ res.status(404).json({ error: 'Page not found' });
245
+ return;
246
+ }
247
+ const raw = readFileSync(match, 'utf-8');
248
+ res.json({ raw, path: match, slug: basename(match, '.md') });
249
+ }
250
+ catch (err) {
251
+ res.status(500).json({ error: 'Failed to read raw page' });
252
+ }
253
+ });
254
+ // API: update page content (full markdown with frontmatter)
255
+ app.put('/api/pages/:title', async (req, res) => {
256
+ try {
257
+ const title = req.params['title'];
258
+ if (!title) {
259
+ res.status(400).json({ error: 'Missing title' });
260
+ return;
261
+ }
262
+ const { content } = req.body;
263
+ if (content === undefined || content === null) {
264
+ res.status(400).json({ error: 'Missing content field' });
265
+ return;
266
+ }
267
+ const pages = listWikiPages(config.wikiDir);
268
+ const titleLower = title.toLowerCase();
269
+ const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
270
+ const match = pages.find((p) => {
271
+ const fileSlug = basename(p, '.md');
272
+ if (fileSlug === title || fileSlug === slugified)
273
+ return true;
274
+ try {
275
+ const page = readWikiPage(p);
276
+ return page.title.toLowerCase() === titleLower;
277
+ }
278
+ catch {
279
+ return false;
280
+ }
281
+ });
282
+ if (!match) {
283
+ res.status(404).json({ error: 'Page not found' });
284
+ return;
285
+ }
286
+ writeFileSync(match, content, 'utf-8');
287
+ // Extract title from new content for commit message
288
+ const matter = await import('gray-matter');
289
+ const parsed = matter.default(content);
290
+ const pageTitle = parsed.data['title'] || title;
291
+ // Auto-commit via git
292
+ try {
293
+ const { autoCommit } = await import('../core/git.js');
294
+ await autoCommit(config.root, 'manual', `edit page "${pageTitle}"`);
295
+ }
296
+ catch { /* git commit is best-effort */ }
297
+ const page = readWikiPage(match);
298
+ res.json({ status: 'saved', page });
299
+ }
300
+ catch (err) {
301
+ const msg = err instanceof Error ? err.message : String(err);
302
+ res.status(500).json({ error: `Failed to save page: ${msg}` });
303
+ }
304
+ });
305
+ // API: delete a wiki page
306
+ app.delete('/api/pages/:title', async (req, res) => {
307
+ try {
308
+ const title = req.params['title'];
309
+ if (!title) {
310
+ res.status(400).json({ error: 'Missing title' });
311
+ return;
312
+ }
313
+ const pages = listWikiPages(config.wikiDir);
314
+ const titleLower = title.toLowerCase();
315
+ const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
316
+ const match = pages.find((p) => {
317
+ const fileSlug = basename(p, '.md');
318
+ if (fileSlug === title || fileSlug === slugified)
319
+ return true;
320
+ try {
321
+ return readWikiPage(p).title.toLowerCase() === titleLower;
322
+ }
323
+ catch {
324
+ return false;
325
+ }
326
+ });
327
+ if (!match) {
328
+ res.status(404).json({ error: 'Page not found' });
329
+ return;
330
+ }
331
+ const { unlinkSync } = await import('node:fs');
332
+ unlinkSync(match);
333
+ try {
334
+ const { autoCommit } = await import('../core/git.js');
335
+ await autoCommit(config.root, 'manual', `delete page "${title}"`);
336
+ }
337
+ catch { }
338
+ res.json({ status: 'deleted' });
339
+ }
340
+ catch (err) {
341
+ const msg = err instanceof Error ? err.message : String(err);
342
+ res.status(500).json({ error: `Failed to delete: ${msg}` });
343
+ }
344
+ });
345
+ // API: rename a wiki page
346
+ app.post('/api/pages/:title/rename', async (req, res) => {
347
+ try {
348
+ const oldTitle = req.params['title'];
349
+ const { newTitle } = req.body;
350
+ if (!oldTitle || !newTitle) {
351
+ res.status(400).json({ error: 'Missing title' });
352
+ return;
353
+ }
354
+ const pages = listWikiPages(config.wikiDir);
355
+ const titleLower = oldTitle.toLowerCase();
356
+ const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
357
+ const match = pages.find((p) => {
358
+ const fileSlug = basename(p, '.md');
359
+ if (fileSlug === oldTitle || fileSlug === slugified)
360
+ return true;
361
+ try {
362
+ return readWikiPage(p).title.toLowerCase() === titleLower;
363
+ }
364
+ catch {
365
+ return false;
366
+ }
367
+ });
368
+ if (!match) {
369
+ res.status(404).json({ error: 'Page not found' });
370
+ return;
371
+ }
372
+ const content = readFileSync(match, 'utf-8');
373
+ const newContent = content.replace(/^title:\s*["']?.*?["']?\s*$/m, `title: "${newTitle}"`);
374
+ const newSlug = newTitle.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
375
+ const newPath = join(match.substring(0, match.lastIndexOf('/')), newSlug + '.md');
376
+ writeFileSync(newPath, newContent, 'utf-8');
377
+ if (newPath !== match) {
378
+ const { unlinkSync } = await import('node:fs');
379
+ unlinkSync(match);
380
+ }
381
+ try {
382
+ const { autoCommit } = await import('../core/git.js');
383
+ await autoCommit(config.root, 'manual', `rename "${oldTitle}" → "${newTitle}"`);
384
+ }
385
+ catch { }
386
+ res.json({ status: 'renamed', newTitle, newSlug });
387
+ }
388
+ catch (err) {
389
+ const msg = err instanceof Error ? err.message : String(err);
390
+ res.status(500).json({ error: `Failed to rename: ${msg}` });
391
+ }
392
+ });
136
393
  // API: knowledge graph data
137
394
  app.get('/api/graph', (_req, res) => {
138
395
  try {
@@ -143,26 +400,43 @@ export function createServer(vaultRoot, port) {
143
400
  res.status(500).json({ error: 'Failed to build graph' });
144
401
  }
145
402
  });
146
- // API: upload raw file for ingestion
403
+ // API: upload raw file and auto-trigger ingest
147
404
  app.post('/api/upload', (req, res) => {
148
405
  const chunks = [];
149
406
  req.on('data', (chunk) => chunks.push(chunk));
150
- req.on('end', () => {
407
+ req.on('end', async () => {
151
408
  const filename = req.headers['x-filename'];
152
409
  if (!filename) {
153
410
  res.status(400).json({ error: 'Missing x-filename header' });
154
411
  return;
155
412
  }
156
- const rawDir = config.rawDir;
157
- if (!existsSync(rawDir)) {
158
- mkdirSync(rawDir, { recursive: true });
159
- }
160
- const dest = join(rawDir, basename(filename));
413
+ const now = new Date().toISOString().split('T')[0] ?? '';
414
+ const dateDir = join(config.rawDir, now);
415
+ mkdirSync(dateDir, { recursive: true });
416
+ const dest = join(dateDir, basename(filename));
161
417
  writeFileSync(dest, Buffer.concat(chunks));
162
- res.json({ status: 'uploaded', path: dest });
418
+ const autoIngest = req.headers['x-auto-ingest'] !== 'false';
419
+ if (autoIngest) {
420
+ try {
421
+ const { ingestSource } = await import('../core/ingest.js');
422
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
423
+ const { loadConfig } = await import('../core/config.js');
424
+ const userConfig = loadConfig(config.configPath);
425
+ const provider = createProviderFromUserConfig(userConfig);
426
+ const result = await ingestSource(dest, config, provider, { verbose: false });
427
+ res.json({ status: 'ingested', path: dest, ...result });
428
+ }
429
+ catch (err) {
430
+ const msg = err instanceof Error ? err.message : String(err);
431
+ res.json({ status: 'uploaded', path: dest, ingestError: msg });
432
+ }
433
+ }
434
+ else {
435
+ res.json({ status: 'uploaded', path: dest });
436
+ }
163
437
  });
164
438
  });
165
- // API: raw files list
439
+ // API: raw files list (recursive through date-stamped subdirectories)
166
440
  app.get('/api/raw', (_req, res) => {
167
441
  try {
168
442
  const rawDir = config.rawDir;
@@ -170,33 +444,280 @@ export function createServer(vaultRoot, port) {
170
444
  res.json([]);
171
445
  return;
172
446
  }
173
- const files = readdirSync(rawDir)
174
- .filter((f) => !f.startsWith('.'))
175
- .map((f) => {
176
- const full = join(rawDir, f);
177
- const stat = statSync(full);
178
- return { name: f, size: stat.size, modified: stat.mtime.toISOString() };
179
- });
447
+ const files = [];
448
+ function walkRaw(dir, prefix) {
449
+ for (const entry of readdirSync(dir)) {
450
+ if (entry.startsWith('.') || entry.endsWith('.meta.json'))
451
+ continue;
452
+ const full = join(dir, entry);
453
+ const stat = statSync(full);
454
+ if (stat.isDirectory()) {
455
+ walkRaw(full, prefix ? `${prefix}/${entry}` : entry);
456
+ }
457
+ else {
458
+ files.push({
459
+ name: prefix ? `${prefix}/${entry}` : entry,
460
+ path: full,
461
+ size: stat.size,
462
+ modified: stat.mtime.toISOString(),
463
+ });
464
+ }
465
+ }
466
+ }
467
+ walkRaw(rawDir, '');
180
468
  res.json(files);
181
469
  }
182
470
  catch (err) {
183
471
  res.status(500).json({ error: 'Failed to list raw files' });
184
472
  }
185
473
  });
474
+ // API: read raw file content (for preview)
475
+ app.get('/api/raw/view/:filename', (req, res) => {
476
+ try {
477
+ const filename = req.params['filename'];
478
+ if (!filename) {
479
+ res.status(400).json({ error: 'Missing filename' });
480
+ return;
481
+ }
482
+ const decoded = decodeURIComponent(filename);
483
+ const fullPath = join(config.rawDir, decoded);
484
+ const resolved = resolve(fullPath);
485
+ if (!resolved.startsWith(resolve(config.rawDir))) {
486
+ res.status(403).json({ error: 'Access denied' });
487
+ return;
488
+ }
489
+ if (!existsSync(resolved)) {
490
+ res.status(404).json({ error: 'File not found' });
491
+ return;
492
+ }
493
+ const ext = extname(resolved).toLowerCase();
494
+ const imageExts = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'];
495
+ if (imageExts.includes(ext)) {
496
+ res.sendFile(resolved);
497
+ return;
498
+ }
499
+ const textExts = ['.md', '.txt', '.csv', '.json', '.yaml', '.yml', '.xml', '.html', '.htm', '.ts', '.js', '.py', '.go', '.rs'];
500
+ if (textExts.includes(ext) || ext === '') {
501
+ const content = readFileSync(resolved, 'utf-8');
502
+ res.json({ type: 'text', content, filename: decoded });
503
+ return;
504
+ }
505
+ res.json({ type: 'binary', filename: decoded, size: statSync(resolved).size, message: 'Binary file — extract with wikimem ingest' });
506
+ }
507
+ catch (err) {
508
+ res.status(500).json({ error: 'Failed to read raw file' });
509
+ }
510
+ });
511
+ // API: file tree (wiki + raw hierarchy)
512
+ app.get('/api/tree', (_req, res) => {
513
+ try {
514
+ function buildWikiTree(dir, relPath) {
515
+ const nodes = [];
516
+ if (!existsSync(dir))
517
+ return nodes;
518
+ const entries = readdirSync(dir).sort();
519
+ for (const entry of entries) {
520
+ if (entry.startsWith('.'))
521
+ continue;
522
+ const full = join(dir, entry);
523
+ const stat = statSync(full);
524
+ const childPath = relPath ? `${relPath}/${entry}` : entry;
525
+ if (stat.isDirectory()) {
526
+ const children = buildWikiTree(full, childPath);
527
+ nodes.push({ name: entry, type: 'dir', path: childPath, children });
528
+ }
529
+ else if (entry.endsWith('.md')) {
530
+ try {
531
+ const page = readWikiPage(full);
532
+ const cat = page.frontmatter['type']
533
+ ?? page.frontmatter['category']
534
+ ?? 'page';
535
+ nodes.push({ name: entry, type: 'wiki', path: childPath, title: page.title, category: cat });
536
+ }
537
+ catch {
538
+ nodes.push({ name: entry, type: 'wiki', path: childPath, title: entry.replace('.md', '') });
539
+ }
540
+ }
541
+ }
542
+ return nodes;
543
+ }
544
+ function buildRawTree(dir, relPath) {
545
+ const nodes = [];
546
+ if (!existsSync(dir))
547
+ return nodes;
548
+ const entries = readdirSync(dir).sort();
549
+ for (const entry of entries) {
550
+ if (entry.startsWith('.') || entry.endsWith('.meta.json'))
551
+ continue;
552
+ const full = join(dir, entry);
553
+ const stat = statSync(full);
554
+ const childPath = relPath ? `${relPath}/${entry}` : entry;
555
+ if (stat.isDirectory()) {
556
+ nodes.push({ name: entry, type: 'dir', path: childPath, children: buildRawTree(full, childPath) });
557
+ }
558
+ else {
559
+ nodes.push({ name: entry, type: 'raw', path: childPath, size: stat.size });
560
+ }
561
+ }
562
+ return nodes;
563
+ }
564
+ res.json({
565
+ wiki: buildWikiTree(config.wikiDir, ''),
566
+ raw: buildRawTree(config.rawDir, ''),
567
+ });
568
+ }
569
+ catch {
570
+ res.status(500).json({ error: 'Failed to build file tree' });
571
+ }
572
+ });
573
+ // API: read config (for settings page)
574
+ app.get('/api/config', async (_req, res) => {
575
+ try {
576
+ const { loadConfig } = await import('../core/config.js');
577
+ const userConfig = loadConfig(config.configPath);
578
+ const safeConfig = { ...userConfig };
579
+ if (safeConfig.api_key)
580
+ safeConfig.api_key = safeConfig.api_key.slice(0, 8) + '…';
581
+ const sc = safeConfig;
582
+ if (sc['gemini_api_key']) {
583
+ sc['gemini_api_key'] = String(sc['gemini_api_key']).slice(0, 8) + '…';
584
+ }
585
+ res.json(safeConfig);
586
+ }
587
+ catch {
588
+ res.json({});
589
+ }
590
+ });
591
+ // API: update config (for settings page)
592
+ app.put('/api/config', async (req, res) => {
593
+ try {
594
+ const { loadConfig } = await import('../core/config.js');
595
+ const YAML = await import('yaml');
596
+ const updates = req.body;
597
+ const current = loadConfig(config.configPath);
598
+ const merged = { ...current, ...updates };
599
+ writeFileSync(config.configPath, YAML.stringify(merged), 'utf-8');
600
+ res.json({ status: 'saved' });
601
+ }
602
+ catch (err) {
603
+ const msg = err instanceof Error ? err.message : String(err);
604
+ res.status(500).json({ error: `Failed to save config: ${msg}` });
605
+ }
606
+ });
607
+ // API: test provider connection
608
+ app.post('/api/config/test-provider', async (req, res) => {
609
+ try {
610
+ const { provider: providerName, apiKey } = req.body;
611
+ if (providerName === 'claude-code') {
612
+ const { isClaudeCodeAvailable } = await import('../core/claude-code.js');
613
+ if (isClaudeCodeAvailable()) {
614
+ res.json({ status: 'ok', provider: 'claude-code' });
615
+ }
616
+ else {
617
+ res.json({ status: 'error', error: 'Claude Code CLI not found in PATH' });
618
+ }
619
+ return;
620
+ }
621
+ if (!apiKey) {
622
+ res.status(400).json({ error: 'Missing apiKey' });
623
+ return;
624
+ }
625
+ if (providerName === 'claude' || !providerName) {
626
+ const { default: Anthropic } = await import('@anthropic-ai/sdk');
627
+ const client = new Anthropic({ apiKey });
628
+ await client.messages.create({
629
+ model: 'claude-3-haiku-20240307',
630
+ max_tokens: 10,
631
+ messages: [{ role: 'user', content: 'ping' }],
632
+ });
633
+ res.json({ status: 'ok', provider: 'claude' });
634
+ }
635
+ else {
636
+ res.json({ status: 'ok', provider: providerName, note: 'Skipped validation' });
637
+ }
638
+ }
639
+ catch (err) {
640
+ const msg = err instanceof Error ? err.message : String(err);
641
+ res.json({ status: 'error', error: msg });
642
+ }
643
+ });
644
+ // API: Claude Code CLI availability
645
+ app.get('/api/claude-code/status', async (_req, res) => {
646
+ try {
647
+ const { isClaudeCodeAvailable, getClaudeCodePath } = await import('../core/claude-code.js');
648
+ res.json({ available: isClaudeCodeAvailable(), path: getClaudeCodePath() });
649
+ }
650
+ catch {
651
+ res.json({ available: false, path: null });
652
+ }
653
+ });
654
+ // API: search pages
655
+ app.get('/api/search', (req, res) => {
656
+ try {
657
+ const q = (req.query['q'] ?? '').toLowerCase().trim();
658
+ const limit = parseInt(req.query['limit']) || 20;
659
+ if (!q) {
660
+ res.json({ results: [] });
661
+ return;
662
+ }
663
+ const pages = listWikiPages(config.wikiDir);
664
+ const results = [];
665
+ for (const pagePath of pages) {
666
+ const page = readWikiPage(pagePath);
667
+ const titleMatch = page.title.toLowerCase().includes(q);
668
+ const tagMatch = (page.frontmatter['tags'] ?? []).some((t) => t.toLowerCase().includes(q));
669
+ let snippet;
670
+ let contentMatch = false;
671
+ if (!titleMatch) {
672
+ const bodyLower = page.content.toLowerCase();
673
+ const idx = bodyLower.indexOf(q);
674
+ if (idx >= 0) {
675
+ contentMatch = true;
676
+ const start = Math.max(0, idx - 40);
677
+ const end = Math.min(page.content.length, idx + q.length + 60);
678
+ snippet = (start > 0 ? '…' : '') + page.content.substring(start, end).replace(/\n/g, ' ') + (end < page.content.length ? '…' : '');
679
+ }
680
+ }
681
+ if (titleMatch || tagMatch || contentMatch) {
682
+ const category = page.frontmatter['category'] ?? 'uncategorized';
683
+ results.push({ title: page.title, category, wordCount: page.wordCount, snippet });
684
+ }
685
+ if (results.length >= limit)
686
+ break;
687
+ }
688
+ res.json({ results });
689
+ }
690
+ catch {
691
+ res.json({ results: [] });
692
+ }
693
+ });
694
+ // API: wiki history (audit trail)
695
+ app.get('/api/history', async (_req, res) => {
696
+ try {
697
+ const { listSnapshots } = await import('../core/history.js');
698
+ const entries = listSnapshots(config);
699
+ res.json(entries);
700
+ }
701
+ catch {
702
+ res.json([]);
703
+ }
704
+ });
186
705
  // API: query the wiki
187
706
  app.post('/api/query', async (req, res) => {
188
707
  try {
189
- const { question, provider: providerName } = req.body;
708
+ const { question, provider: providerName, model: modelName } = req.body;
190
709
  if (!question) {
191
710
  res.status(400).json({ error: 'Missing question field' });
192
711
  return;
193
712
  }
194
- // Dynamic import to avoid circular deps
195
713
  const { queryWiki } = await import('../core/query.js');
196
- const { createProvider } = await import('../providers/index.js');
714
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
197
715
  const { loadConfig } = await import('../core/config.js');
198
716
  const userConfig = loadConfig(config.configPath);
199
- const provider = createProvider(providerName ?? userConfig.provider ?? 'claude');
717
+ const provider = createProviderFromUserConfig(userConfig, {
718
+ providerOverride: providerName,
719
+ model: modelName,
720
+ });
200
721
  const result = await queryWiki(question, config, provider, { fileBack: false });
201
722
  res.json(result);
202
723
  }
@@ -214,10 +735,10 @@ export function createServer(vaultRoot, port) {
214
735
  return;
215
736
  }
216
737
  const { ingestSource } = await import('../core/ingest.js');
217
- const { createProvider } = await import('../providers/index.js');
738
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
218
739
  const { loadConfig } = await import('../core/config.js');
219
740
  const userConfig = loadConfig(config.configPath);
220
- const provider = createProvider(userConfig.provider ?? 'claude');
741
+ const provider = createProviderFromUserConfig(userConfig);
221
742
  const result = await ingestSource(source, config, provider, { verbose: false });
222
743
  res.json(result);
223
744
  }
@@ -226,6 +747,1557 @@ export function createServer(vaultRoot, port) {
226
747
  res.status(500).json({ error: `Ingest failed: ${msg}` });
227
748
  }
228
749
  });
750
+ // === GIT API ENDPOINTS ===
751
+ // API: git status (lightweight by default, add ?full=true for file list)
752
+ app.get('/api/git/status', async (req, res) => {
753
+ try {
754
+ const { isGitRepo, getGitStatus, getBranches } = await import('../core/git.js');
755
+ const isRepo = await isGitRepo(config.root);
756
+ if (!isRepo) {
757
+ res.json({ initialized: false });
758
+ return;
759
+ }
760
+ const includeFull = req.query['full'] === 'true';
761
+ const status = await getGitStatus(config.root);
762
+ const branches = await getBranches(config.root);
763
+ const result = {
764
+ initialized: true,
765
+ branch: branches.current,
766
+ branches: branches.all,
767
+ isDetached: branches.isDetached,
768
+ changedCount: status?.files?.length ?? 0,
769
+ isClean: status?.isClean() ?? true,
770
+ };
771
+ if (includeFull) {
772
+ result['files'] = status?.files ?? [];
773
+ }
774
+ res.json(result);
775
+ }
776
+ catch (err) {
777
+ const msg = err instanceof Error ? err.message : String(err);
778
+ res.status(500).json({ error: `Git status failed: ${msg}` });
779
+ }
780
+ });
781
+ // API: initialize git repo
782
+ app.post('/api/git/init', async (_req, res) => {
783
+ try {
784
+ const { initGitRepo } = await import('../core/git.js');
785
+ const result = await initGitRepo(config);
786
+ res.json(result);
787
+ }
788
+ catch (err) {
789
+ const msg = err instanceof Error ? err.message : String(err);
790
+ res.status(500).json({ error: `Git init failed: ${msg}` });
791
+ }
792
+ });
793
+ // API: git log (audit trail) with optional wiki-only filtering
794
+ app.get('/api/git/log', async (req, res) => {
795
+ try {
796
+ const { getGitLog, isGitRepo } = await import('../core/git.js');
797
+ if (!(await isGitRepo(config.root))) {
798
+ res.json([]);
799
+ return;
800
+ }
801
+ const limit = parseInt(req.query['limit']) || 50;
802
+ const wikiOnly = req.query['wikiOnly'] !== 'false';
803
+ const search = req.query['search'] || undefined;
804
+ const log = await getGitLog(config.root, limit, { wikiOnly, search });
805
+ res.json(log);
806
+ }
807
+ catch (err) {
808
+ const msg = err instanceof Error ? err.message : String(err);
809
+ res.status(500).json({ error: `Git log failed: ${msg}` });
810
+ }
811
+ });
812
+ // API: git diff for a specific commit
813
+ app.get('/api/git/diff/:hash', async (req, res) => {
814
+ try {
815
+ const hash = req.params['hash'];
816
+ if (!hash) {
817
+ res.status(400).json({ error: 'Missing hash' });
818
+ return;
819
+ }
820
+ const { getGitDiff, isGitRepo } = await import('../core/git.js');
821
+ if (!(await isGitRepo(config.root))) {
822
+ res.json({ diff: '', stats: [] });
823
+ return;
824
+ }
825
+ const result = await getGitDiff(config.root, hash);
826
+ res.json(result);
827
+ }
828
+ catch (err) {
829
+ const msg = err instanceof Error ? err.message : String(err);
830
+ res.status(500).json({ error: `Git diff failed: ${msg}` });
831
+ }
832
+ });
833
+ // API: create branch (optionally from a specific commit hash)
834
+ app.post('/api/git/branch', async (req, res) => {
835
+ try {
836
+ const { name, fromHash } = req.body;
837
+ if (!name) {
838
+ res.status(400).json({ error: 'Missing branch name' });
839
+ return;
840
+ }
841
+ const { createBranch } = await import('../core/git.js');
842
+ const result = await createBranch(config.root, name, fromHash);
843
+ res.json(result);
844
+ }
845
+ catch (err) {
846
+ const msg = err instanceof Error ? err.message : String(err);
847
+ res.status(500).json({ error: `Create branch failed: ${msg}` });
848
+ }
849
+ });
850
+ // API: switch branch
851
+ app.post('/api/git/checkout', async (req, res) => {
852
+ try {
853
+ const { branch } = req.body;
854
+ if (!branch) {
855
+ res.status(400).json({ error: 'Missing branch name' });
856
+ return;
857
+ }
858
+ const { switchBranch } = await import('../core/git.js');
859
+ const result = await switchBranch(config.root, branch);
860
+ res.json(result);
861
+ }
862
+ catch (err) {
863
+ const msg = err instanceof Error ? err.message : String(err);
864
+ res.status(500).json({ error: `Checkout failed: ${msg}` });
865
+ }
866
+ });
867
+ // API: create tag (milestone)
868
+ app.post('/api/git/tag', async (req, res) => {
869
+ try {
870
+ const { name, message: tagMsg } = req.body;
871
+ if (!name) {
872
+ res.status(400).json({ error: 'Missing tag name' });
873
+ return;
874
+ }
875
+ const { createTag } = await import('../core/git.js');
876
+ const result = await createTag(config.root, name, tagMsg);
877
+ res.json(result);
878
+ }
879
+ catch (err) {
880
+ const msg = err instanceof Error ? err.message : String(err);
881
+ res.status(500).json({ error: `Create tag failed: ${msg}` });
882
+ }
883
+ });
884
+ // API: restore to a specific commit
885
+ app.post('/api/git/restore', async (req, res) => {
886
+ try {
887
+ const { hash } = req.body;
888
+ if (!hash) {
889
+ res.status(400).json({ error: 'Missing commit hash' });
890
+ return;
891
+ }
892
+ const { restoreToCommit } = await import('../core/git.js');
893
+ const result = await restoreToCommit(config.root, hash);
894
+ res.json(result);
895
+ }
896
+ catch (err) {
897
+ const msg = err instanceof Error ? err.message : String(err);
898
+ res.status(500).json({ error: `Restore failed: ${msg}` });
899
+ }
900
+ });
901
+ // API: get file tree at a specific commit (for time-lapse)
902
+ app.get('/api/git/tree/:hash', async (req, res) => {
903
+ try {
904
+ const hash = req.params['hash'];
905
+ if (!hash) {
906
+ res.status(400).json({ error: 'Missing hash' });
907
+ return;
908
+ }
909
+ const { getTreeAtCommit, isGitRepo } = await import('../core/git.js');
910
+ if (!(await isGitRepo(config.root))) {
911
+ res.json([]);
912
+ return;
913
+ }
914
+ const tree = await getTreeAtCommit(config.root, hash);
915
+ res.json(tree);
916
+ }
917
+ catch (err) {
918
+ const msg = err instanceof Error ? err.message : String(err);
919
+ res.status(500).json({ error: `Get tree failed: ${msg}` });
920
+ }
921
+ });
922
+ // API: batch fetch file trees for multiple commits (time-lapse pre-fetch)
923
+ app.post('/api/git/trees/batch', async (req, res) => {
924
+ try {
925
+ const { hashes } = req.body;
926
+ if (!hashes || !Array.isArray(hashes) || hashes.length === 0) {
927
+ res.status(400).json({ error: 'Missing or empty hashes array' });
928
+ return;
929
+ }
930
+ const capped = hashes.slice(0, 500);
931
+ const { getTreeAtCommit, isGitRepo } = await import('../core/git.js');
932
+ if (!(await isGitRepo(config.root))) {
933
+ res.json({});
934
+ return;
935
+ }
936
+ const result = {};
937
+ const concurrency = 10;
938
+ for (let i = 0; i < capped.length; i += concurrency) {
939
+ const batch = capped.slice(i, i + concurrency);
940
+ await Promise.all(batch.map(async (hash) => {
941
+ try {
942
+ result[hash] = await getTreeAtCommit(config.root, hash);
943
+ }
944
+ catch {
945
+ result[hash] = [];
946
+ }
947
+ }));
948
+ }
949
+ res.json(result);
950
+ }
951
+ catch (err) {
952
+ const msg = err instanceof Error ? err.message : String(err);
953
+ res.status(500).json({ error: `Batch tree fetch failed: ${msg}` });
954
+ }
955
+ });
956
+ // API: graph data at a specific commit (for time-lapse graph animation)
957
+ app.get('/api/git/graph/:hash', async (req, res) => {
958
+ try {
959
+ const hash = req.params['hash'];
960
+ if (!hash) {
961
+ res.status(400).json({ error: 'Missing hash' });
962
+ return;
963
+ }
964
+ const { getGraphAtCommit, isGitRepo } = await import('../core/git.js');
965
+ if (!(await isGitRepo(config.root))) {
966
+ res.json({ nodes: [], links: [] });
967
+ return;
968
+ }
969
+ const graph = await getGraphAtCommit(config.root, hash);
970
+ res.json(graph);
971
+ }
972
+ catch (err) {
973
+ const msg = err instanceof Error ? err.message : String(err);
974
+ res.status(500).json({ error: `Get graph at commit failed: ${msg}` });
975
+ }
976
+ });
977
+ // API: batch graph snapshots for time-lapse (pre-fetch all commit graphs)
978
+ app.post('/api/git/graph-batch', async (req, res) => {
979
+ try {
980
+ const { hashes } = req.body;
981
+ if (!hashes?.length) {
982
+ res.status(400).json({ error: 'Missing hashes array' });
983
+ return;
984
+ }
985
+ const { getGraphAtCommit, isGitRepo } = await import('../core/git.js');
986
+ if (!(await isGitRepo(config.root))) {
987
+ res.json({});
988
+ return;
989
+ }
990
+ const results = {};
991
+ for (const hash of hashes.slice(0, 100)) {
992
+ results[hash] = await getGraphAtCommit(config.root, hash);
993
+ }
994
+ res.json(results);
995
+ }
996
+ catch (err) {
997
+ const msg = err instanceof Error ? err.message : String(err);
998
+ res.status(500).json({ error: `Batch graph failed: ${msg}` });
999
+ }
1000
+ });
1001
+ // API: list branches (dedicated endpoint)
1002
+ app.get('/api/git/branches', async (_req, res) => {
1003
+ try {
1004
+ const { getBranches, isGitRepo } = await import('../core/git.js');
1005
+ if (!(await isGitRepo(config.root))) {
1006
+ res.json({ current: '', branches: [], isDetached: false });
1007
+ return;
1008
+ }
1009
+ const info = await getBranches(config.root);
1010
+ res.json({ current: info.current, branches: info.all, isDetached: info.isDetached });
1011
+ }
1012
+ catch (err) {
1013
+ const msg = err instanceof Error ? err.message : String(err);
1014
+ res.status(500).json({ error: `Failed to list branches: ${msg}` });
1015
+ }
1016
+ });
1017
+ // API: create session branch (auto-naming with wiki/session-<timestamp> or custom name)
1018
+ app.post('/api/git/branches', async (req, res) => {
1019
+ try {
1020
+ const { name, session } = req.body;
1021
+ const { createBranch } = await import('../core/git.js');
1022
+ let branchName = name;
1023
+ if (!branchName || session) {
1024
+ const ts = new Date().toISOString().replace(/[:.]/g, '-').substring(0, 19);
1025
+ branchName = name ? `wiki/${name}` : `wiki/session-${ts}`;
1026
+ }
1027
+ const result = await createBranch(config.root, branchName);
1028
+ res.json(result);
1029
+ }
1030
+ catch (err) {
1031
+ const msg = err instanceof Error ? err.message : String(err);
1032
+ res.status(500).json({ error: `Create branch failed: ${msg}` });
1033
+ }
1034
+ });
1035
+ // API: push current branch to remote
1036
+ app.post('/api/git/push', async (req, res) => {
1037
+ try {
1038
+ const { remote } = req.body;
1039
+ const { pushBranch } = await import('../core/git.js');
1040
+ const result = await pushBranch(config.root, remote || 'origin');
1041
+ res.json(result);
1042
+ }
1043
+ catch (err) {
1044
+ const msg = err instanceof Error ? err.message : String(err);
1045
+ res.status(500).json({ error: `Push failed: ${msg}` });
1046
+ }
1047
+ });
1048
+ // API: submit for review (PR workflow: diff summary, auto-branch, push)
1049
+ app.post('/api/git/pr', async (req, res) => {
1050
+ try {
1051
+ const { description } = req.body;
1052
+ const { submitForReview } = await import('../core/git.js');
1053
+ const result = await submitForReview(config.root, description);
1054
+ res.json(result);
1055
+ }
1056
+ catch (err) {
1057
+ const msg = err instanceof Error ? err.message : String(err);
1058
+ res.status(500).json({ error: `Submit for review failed: ${msg}` });
1059
+ }
1060
+ });
1061
+ // API: get diff summary (working branch vs base)
1062
+ app.get('/api/git/diff-summary', async (req, res) => {
1063
+ try {
1064
+ const baseBranch = req.query['base'];
1065
+ const { getDiffSummary, isGitRepo } = await import('../core/git.js');
1066
+ if (!(await isGitRepo(config.root))) {
1067
+ res.json({ filesAdded: [], filesModified: [], filesDeleted: [], totalAdditions: 0, totalDeletions: 0 });
1068
+ return;
1069
+ }
1070
+ const summary = await getDiffSummary(config.root, baseBranch);
1071
+ res.json(summary);
1072
+ }
1073
+ catch (err) {
1074
+ const msg = err instanceof Error ? err.message : String(err);
1075
+ res.status(500).json({ error: `Diff summary failed: ${msg}` });
1076
+ }
1077
+ });
1078
+ // API: parsed diff for a commit (rich diff view with per-file hunks)
1079
+ app.get('/api/git/diff/:hash/parsed', async (req, res) => {
1080
+ try {
1081
+ const hash = req.params['hash'];
1082
+ if (!hash) {
1083
+ res.status(400).json({ error: 'Missing hash' });
1084
+ return;
1085
+ }
1086
+ const { getParsedDiff, isGitRepo } = await import('../core/git.js');
1087
+ if (!(await isGitRepo(config.root))) {
1088
+ res.json({ files: [], stats: { additions: 0, deletions: 0, filesChanged: 0 } });
1089
+ return;
1090
+ }
1091
+ const result = await getParsedDiff(config.root, hash);
1092
+ res.json(result);
1093
+ }
1094
+ catch (err) {
1095
+ const msg = err instanceof Error ? err.message : String(err);
1096
+ res.status(500).json({ error: `Parsed diff failed: ${msg}` });
1097
+ }
1098
+ });
1099
+ // API: migrate .wikimem/history snapshots to git commits
1100
+ app.post('/api/git/migrate-history', async (_req, res) => {
1101
+ try {
1102
+ const { migrateHistoryToGit } = await import('../core/git.js');
1103
+ const result = await migrateHistoryToGit(config);
1104
+ res.json(result);
1105
+ }
1106
+ catch (err) {
1107
+ const msg = err instanceof Error ? err.message : String(err);
1108
+ res.status(500).json({ error: `Migration failed: ${msg}` });
1109
+ }
1110
+ });
1111
+ // === RAW FILE PREVIEW ENDPOINTS ===
1112
+ // API: serve raw file with correct content-type (for PDF, images, video, audio)
1113
+ app.get('/api/raw/file', (req, res) => {
1114
+ try {
1115
+ let filePath = String(req.query['path'] ?? '');
1116
+ if (filePath.startsWith('/'))
1117
+ filePath = filePath.slice(1);
1118
+ if (!filePath) {
1119
+ res.status(400).json({ error: 'Missing path query param' });
1120
+ return;
1121
+ }
1122
+ const decoded = decodeURIComponent(filePath);
1123
+ const fullPath = join(config.rawDir, decoded);
1124
+ const resolved = resolve(fullPath);
1125
+ if (!resolved.startsWith(resolve(config.rawDir))) {
1126
+ res.status(403).json({ error: 'Access denied' });
1127
+ return;
1128
+ }
1129
+ if (!existsSync(resolved)) {
1130
+ res.status(404).json({ error: 'File not found' });
1131
+ return;
1132
+ }
1133
+ const ext = extname(resolved).toLowerCase();
1134
+ const mimeTypes = {
1135
+ '.pdf': 'application/pdf',
1136
+ '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
1137
+ '.png': 'image/png', '.gif': 'image/gif',
1138
+ '.webp': 'image/webp', '.svg': 'image/svg+xml',
1139
+ '.mp4': 'video/mp4', '.webm': 'video/webm', '.mov': 'video/quicktime',
1140
+ '.mp3': 'audio/mpeg', '.wav': 'audio/wav', '.ogg': 'audio/ogg',
1141
+ '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
1142
+ '.xls': 'application/vnd.ms-excel',
1143
+ '.csv': 'text/csv',
1144
+ '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
1145
+ '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
1146
+ };
1147
+ const mime = mimeTypes[ext] ?? 'application/octet-stream';
1148
+ res.setHeader('Accept-Ranges', 'bytes');
1149
+ res.setHeader('Content-Type', mime);
1150
+ res.sendFile(resolved);
1151
+ }
1152
+ catch (err) {
1153
+ res.status(500).json({ error: 'Failed to serve file' });
1154
+ }
1155
+ });
1156
+ // API: raw file metadata (for preview header)
1157
+ app.get('/api/raw/meta', (req, res) => {
1158
+ try {
1159
+ let filePath = String(req.query['path'] ?? '');
1160
+ if (filePath.startsWith('/'))
1161
+ filePath = filePath.slice(1);
1162
+ if (!filePath) {
1163
+ res.status(400).json({ error: 'Missing path query param' });
1164
+ return;
1165
+ }
1166
+ const decoded = decodeURIComponent(filePath);
1167
+ const fullPath = join(config.rawDir, decoded);
1168
+ const resolved = resolve(fullPath);
1169
+ if (!resolved.startsWith(resolve(config.rawDir))) {
1170
+ res.status(403).json({ error: 'Access denied' });
1171
+ return;
1172
+ }
1173
+ if (!existsSync(resolved)) {
1174
+ res.status(404).json({ error: 'File not found' });
1175
+ return;
1176
+ }
1177
+ const stat = statSync(resolved);
1178
+ const ext = extname(resolved).toLowerCase();
1179
+ const imageExts = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'];
1180
+ const videoExts = ['.mp4', '.webm', '.mov'];
1181
+ const audioExts = ['.mp3', '.wav', '.ogg'];
1182
+ const pdfExts = ['.pdf'];
1183
+ const spreadsheetExts = ['.csv', '.xlsx', '.xls'];
1184
+ const textExts = ['.md', '.txt', '.json', '.yaml', '.yml', '.xml', '.html', '.htm', '.ts', '.js', '.py', '.go', '.rs', '.toml', '.ini', '.cfg', '.env', '.sh', '.bash', '.zsh'];
1185
+ const documentExts = ['.docx', '.doc', '.pptx', '.ppt', '.rtf', '.odt', '.odp'];
1186
+ let previewType;
1187
+ if (imageExts.includes(ext))
1188
+ previewType = 'image';
1189
+ else if (videoExts.includes(ext))
1190
+ previewType = 'video';
1191
+ else if (audioExts.includes(ext))
1192
+ previewType = 'audio';
1193
+ else if (pdfExts.includes(ext))
1194
+ previewType = 'pdf';
1195
+ else if (spreadsheetExts.includes(ext))
1196
+ previewType = 'spreadsheet';
1197
+ else if (textExts.includes(ext))
1198
+ previewType = 'text';
1199
+ else if (documentExts.includes(ext))
1200
+ previewType = 'document';
1201
+ else
1202
+ previewType = 'binary';
1203
+ // Find wiki pages that were generated from this raw file
1204
+ const linkedPages = [];
1205
+ try {
1206
+ const pages = listWikiPages(config.wikiDir);
1207
+ for (const pagePath of pages) {
1208
+ const page = readWikiPage(pagePath);
1209
+ const sources = page.frontmatter['sources'];
1210
+ if (sources?.some(s => s.includes(decoded) || s.includes(basename(decoded)))) {
1211
+ linkedPages.push({
1212
+ title: page.title,
1213
+ slug: basename(pagePath, '.md'),
1214
+ });
1215
+ }
1216
+ }
1217
+ }
1218
+ catch { /* non-fatal */ }
1219
+ res.json({
1220
+ name: basename(decoded),
1221
+ path: decoded,
1222
+ size: stat.size,
1223
+ modified: stat.mtime.toISOString(),
1224
+ created: stat.birthtime.toISOString(),
1225
+ extension: ext,
1226
+ previewType,
1227
+ linkedWikiPages: linkedPages,
1228
+ });
1229
+ }
1230
+ catch (err) {
1231
+ res.status(500).json({ error: 'Failed to get file metadata' });
1232
+ }
1233
+ });
1234
+ // ─── Raw File Operations (Cursor-Parity) ──────────────────────────────────
1235
+ // POST /api/raw/rename — rename a raw file
1236
+ app.post('/api/raw/rename', async (req, res) => {
1237
+ try {
1238
+ const { oldPath, newPath } = req.body;
1239
+ if (!oldPath || !newPath) {
1240
+ res.status(400).json({ error: 'Missing oldPath or newPath' });
1241
+ return;
1242
+ }
1243
+ const resolvedOld = resolve(config.rawDir, oldPath);
1244
+ const resolvedNew = resolve(config.rawDir, newPath);
1245
+ if (!resolvedOld.startsWith(resolve(config.rawDir)) || !resolvedNew.startsWith(resolve(config.rawDir))) {
1246
+ res.status(403).json({ error: 'Access denied: path outside raw directory' });
1247
+ return;
1248
+ }
1249
+ if (!existsSync(resolvedOld)) {
1250
+ res.status(404).json({ error: 'File not found' });
1251
+ return;
1252
+ }
1253
+ mkdirSync(dirname(resolvedNew), { recursive: true });
1254
+ renameSync(resolvedOld, resolvedNew);
1255
+ try {
1256
+ const { autoCommit, isGitRepo } = await import('../core/git.js');
1257
+ if (await isGitRepo(config.root)) {
1258
+ await autoCommit(config.root, 'manual', `rename raw "${basename(oldPath)}" → "${basename(newPath)}"`);
1259
+ }
1260
+ }
1261
+ catch { /* non-fatal */ }
1262
+ res.json({ status: 'renamed', oldPath, newPath });
1263
+ }
1264
+ catch (err) {
1265
+ const msg = err instanceof Error ? err.message : String(err);
1266
+ res.status(500).json({ error: `Rename failed: ${msg}` });
1267
+ }
1268
+ });
1269
+ // DELETE /api/raw/file — delete a raw file
1270
+ app.delete('/api/raw/file', async (req, res) => {
1271
+ try {
1272
+ const filePath = req.query['path'];
1273
+ if (!filePath) {
1274
+ res.status(400).json({ error: 'Missing path query param' });
1275
+ return;
1276
+ }
1277
+ const resolved = resolve(config.rawDir, filePath);
1278
+ if (!resolved.startsWith(resolve(config.rawDir))) {
1279
+ res.status(403).json({ error: 'Access denied: path outside raw directory' });
1280
+ return;
1281
+ }
1282
+ if (!existsSync(resolved)) {
1283
+ res.status(404).json({ error: 'File not found' });
1284
+ return;
1285
+ }
1286
+ fsUnlinkSync(resolved);
1287
+ try {
1288
+ const { autoCommit, isGitRepo } = await import('../core/git.js');
1289
+ if (await isGitRepo(config.root)) {
1290
+ await autoCommit(config.root, 'manual', `delete raw file "${basename(filePath)}"`);
1291
+ }
1292
+ }
1293
+ catch { /* non-fatal */ }
1294
+ res.json({ status: 'deleted', path: filePath });
1295
+ }
1296
+ catch (err) {
1297
+ const msg = err instanceof Error ? err.message : String(err);
1298
+ res.status(500).json({ error: `Delete failed: ${msg}` });
1299
+ }
1300
+ });
1301
+ // POST /api/raw/move — move a raw file to a different directory
1302
+ app.post('/api/raw/move', async (req, res) => {
1303
+ try {
1304
+ const { path: filePath, targetDir } = req.body;
1305
+ if (!filePath || !targetDir) {
1306
+ res.status(400).json({ error: 'Missing path or targetDir' });
1307
+ return;
1308
+ }
1309
+ const resolvedSrc = resolve(config.rawDir, filePath);
1310
+ const resolvedDest = resolve(config.rawDir, targetDir, basename(filePath));
1311
+ if (!resolvedSrc.startsWith(resolve(config.rawDir)) || !resolvedDest.startsWith(resolve(config.rawDir))) {
1312
+ res.status(403).json({ error: 'Access denied: path outside raw directory' });
1313
+ return;
1314
+ }
1315
+ if (!existsSync(resolvedSrc)) {
1316
+ res.status(404).json({ error: 'Source file not found' });
1317
+ return;
1318
+ }
1319
+ mkdirSync(dirname(resolvedDest), { recursive: true });
1320
+ renameSync(resolvedSrc, resolvedDest);
1321
+ try {
1322
+ const { autoCommit, isGitRepo } = await import('../core/git.js');
1323
+ if (await isGitRepo(config.root)) {
1324
+ await autoCommit(config.root, 'manual', `move raw "${basename(filePath)}" → ${targetDir}/`);
1325
+ }
1326
+ }
1327
+ catch { /* non-fatal */ }
1328
+ res.json({ status: 'moved', from: filePath, to: targetDir + '/' + basename(filePath) });
1329
+ }
1330
+ catch (err) {
1331
+ const msg = err instanceof Error ? err.message : String(err);
1332
+ res.status(500).json({ error: `Move failed: ${msg}` });
1333
+ }
1334
+ });
1335
+ // POST /api/pages/:title/move — move a wiki page to a different category folder
1336
+ app.post('/api/pages/:title/move', async (req, res) => {
1337
+ try {
1338
+ const title = req.params['title'];
1339
+ const { targetCategory } = req.body;
1340
+ if (!title || !targetCategory) {
1341
+ res.status(400).json({ error: 'Missing title or targetCategory' });
1342
+ return;
1343
+ }
1344
+ const pages = listWikiPages(config.wikiDir);
1345
+ const titleLower = title.toLowerCase();
1346
+ const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
1347
+ const match = pages.find((p) => {
1348
+ const fileSlug = basename(p, '.md');
1349
+ if (fileSlug === title || fileSlug === slugified)
1350
+ return true;
1351
+ try {
1352
+ return readWikiPage(p).title.toLowerCase() === titleLower;
1353
+ }
1354
+ catch {
1355
+ return false;
1356
+ }
1357
+ });
1358
+ if (!match) {
1359
+ res.status(404).json({ error: 'Page not found' });
1360
+ return;
1361
+ }
1362
+ const destDir = resolve(config.wikiDir, targetCategory);
1363
+ if (!destDir.startsWith(resolve(config.wikiDir))) {
1364
+ res.status(403).json({ error: 'Access denied: category outside wiki directory' });
1365
+ return;
1366
+ }
1367
+ mkdirSync(destDir, { recursive: true });
1368
+ const destPath = join(destDir, basename(match));
1369
+ renameSync(match, destPath);
1370
+ // Update frontmatter category/type
1371
+ let content = readFileSync(destPath, 'utf-8');
1372
+ if (content.match(/^type:\s*.+$/m)) {
1373
+ content = content.replace(/^type:\s*.+$/m, `type: ${targetCategory}`);
1374
+ }
1375
+ else if (content.match(/^category:\s*.+$/m)) {
1376
+ content = content.replace(/^category:\s*.+$/m, `category: ${targetCategory}`);
1377
+ }
1378
+ writeFileSync(destPath, content, 'utf-8');
1379
+ try {
1380
+ const { autoCommit, isGitRepo } = await import('../core/git.js');
1381
+ if (await isGitRepo(config.root)) {
1382
+ await autoCommit(config.root, 'manual', `move page "${title}" → ${targetCategory}/`);
1383
+ }
1384
+ }
1385
+ catch { /* non-fatal */ }
1386
+ res.json({ status: 'moved', title, targetCategory, newPath: relative(config.wikiDir, destPath) });
1387
+ }
1388
+ catch (err) {
1389
+ const msg = err instanceof Error ? err.message : String(err);
1390
+ res.status(500).json({ error: `Move failed: ${msg}` });
1391
+ }
1392
+ });
1393
+ // POST /api/folders — create a new folder inside wiki/ or raw/
1394
+ app.post('/api/folders', (req, res) => {
1395
+ try {
1396
+ const { path: folderPath } = req.body;
1397
+ if (!folderPath) {
1398
+ res.status(400).json({ error: 'Missing path' });
1399
+ return;
1400
+ }
1401
+ const resolved = resolve(config.root, folderPath);
1402
+ const wikiBase = resolve(config.wikiDir);
1403
+ const rawBase = resolve(config.rawDir);
1404
+ if (!resolved.startsWith(wikiBase) && !resolved.startsWith(rawBase)) {
1405
+ res.status(403).json({ error: 'Access denied: folder must be inside wiki/ or raw/' });
1406
+ return;
1407
+ }
1408
+ mkdirSync(resolved, { recursive: true });
1409
+ res.json({ status: 'created', path: folderPath });
1410
+ }
1411
+ catch (err) {
1412
+ const msg = err instanceof Error ? err.message : String(err);
1413
+ res.status(500).json({ error: `Create folder failed: ${msg}` });
1414
+ }
1415
+ });
1416
+ // ─── OAuth Connector System ───────────────────────────────────────────────
1417
+ const OAUTH_PROVIDERS = {
1418
+ github: {
1419
+ authorizeUrl: 'https://github.com/login/oauth/authorize',
1420
+ tokenUrl: 'https://github.com/login/oauth/access_token',
1421
+ scopes: 'repo read:user',
1422
+ clientIdKey: 'github_client_id',
1423
+ clientSecretKey: 'github_client_secret',
1424
+ },
1425
+ slack: {
1426
+ authorizeUrl: 'https://slack.com/oauth/v2/authorize',
1427
+ tokenUrl: 'https://slack.com/api/oauth.v2.access',
1428
+ scopes: 'channels:history channels:read users:read',
1429
+ clientIdKey: 'slack_client_id',
1430
+ clientSecretKey: 'slack_client_secret',
1431
+ },
1432
+ google: {
1433
+ authorizeUrl: 'https://accounts.google.com/o/oauth2/v2/auth',
1434
+ tokenUrl: 'https://oauth2.googleapis.com/token',
1435
+ scopes: 'https://www.googleapis.com/auth/gmail.readonly https://www.googleapis.com/auth/drive.readonly',
1436
+ clientIdKey: 'google_client_id',
1437
+ clientSecretKey: 'google_client_secret',
1438
+ },
1439
+ linear: {
1440
+ authorizeUrl: 'https://linear.app/oauth/authorize',
1441
+ tokenUrl: 'https://api.linear.app/oauth/token',
1442
+ scopes: 'read',
1443
+ clientIdKey: 'linear_client_id',
1444
+ clientSecretKey: 'linear_client_secret',
1445
+ },
1446
+ jira: {
1447
+ authorizeUrl: 'https://auth.atlassian.com/authorize',
1448
+ tokenUrl: 'https://auth.atlassian.com/oauth/token',
1449
+ scopes: 'read:jira-work read:jira-user offline_access',
1450
+ clientIdKey: 'jira_client_id',
1451
+ clientSecretKey: 'jira_client_secret',
1452
+ },
1453
+ };
1454
+ const oauthStates = new Map();
1455
+ function getTokenStorePath() {
1456
+ return join(vaultRoot, '.wikimem', 'tokens.json');
1457
+ }
1458
+ function loadOAuthTokens() {
1459
+ const tokenPath = getTokenStorePath();
1460
+ if (!existsSync(tokenPath))
1461
+ return {};
1462
+ try {
1463
+ return JSON.parse(readFileSync(tokenPath, 'utf-8'));
1464
+ }
1465
+ catch {
1466
+ return {};
1467
+ }
1468
+ }
1469
+ function saveOAuthToken(provider, tokenData) {
1470
+ const tokenPath = getTokenStorePath();
1471
+ mkdirSync(dirname(tokenPath), { recursive: true });
1472
+ const tokens = loadOAuthTokens();
1473
+ tokens[provider] = { ...tokenData, connectedAt: new Date().toISOString() };
1474
+ writeFileSync(tokenPath, JSON.stringify(tokens, null, 2), 'utf-8');
1475
+ }
1476
+ // GET /api/auth/tokens — list which providers have tokens stored
1477
+ app.get('/api/auth/tokens', (_req, res) => {
1478
+ try {
1479
+ const tokens = loadOAuthTokens();
1480
+ const result = {};
1481
+ for (const provider of Object.keys(OAUTH_PROVIDERS)) {
1482
+ const token = tokens[provider];
1483
+ result[provider] = { connected: !!token, connectedAt: token?.connectedAt };
1484
+ }
1485
+ res.json(result);
1486
+ }
1487
+ catch {
1488
+ res.json({});
1489
+ }
1490
+ });
1491
+ // GET /api/auth/start/:provider — generate OAuth authorize URL
1492
+ app.get('/api/auth/start/:provider', async (req, res) => {
1493
+ try {
1494
+ const providerName = req.params['provider'];
1495
+ if (!providerName) {
1496
+ res.status(400).json({ error: 'Missing provider' });
1497
+ return;
1498
+ }
1499
+ const providerConfig = OAUTH_PROVIDERS[providerName];
1500
+ if (!providerConfig) {
1501
+ res.status(400).json({ error: `Unknown provider: ${providerName}` });
1502
+ return;
1503
+ }
1504
+ const { loadConfig } = await import('../core/config.js');
1505
+ const userConfig = loadConfig(config.configPath);
1506
+ const clientId = userConfig[providerConfig.clientIdKey];
1507
+ if (!clientId) {
1508
+ res.status(400).json({ error: `Missing ${providerConfig.clientIdKey} in config.yaml` });
1509
+ return;
1510
+ }
1511
+ const state = randomBytes(24).toString('hex');
1512
+ oauthStates.set(state, { provider: providerName, createdAt: Date.now() });
1513
+ // Clean stale states (>10 min)
1514
+ for (const [key, val] of oauthStates) {
1515
+ if (Date.now() - val.createdAt > 600_000)
1516
+ oauthStates.delete(key);
1517
+ }
1518
+ const redirectUri = `http://localhost:${port}/api/auth/callback`;
1519
+ const params = new URLSearchParams({
1520
+ client_id: clientId,
1521
+ redirect_uri: redirectUri,
1522
+ scope: providerConfig.scopes,
1523
+ state,
1524
+ response_type: 'code',
1525
+ ...(providerName === 'google' ? { access_type: 'offline', prompt: 'consent' } : {}),
1526
+ ...(providerName === 'jira' ? { audience: 'api.atlassian.com', prompt: 'consent' } : {}),
1527
+ });
1528
+ const authorizeUrl = `${providerConfig.authorizeUrl}?${params.toString()}`;
1529
+ res.json({ url: authorizeUrl, state });
1530
+ }
1531
+ catch (err) {
1532
+ const msg = err instanceof Error ? err.message : String(err);
1533
+ res.status(500).json({ error: `OAuth start failed: ${msg}` });
1534
+ }
1535
+ });
1536
+ // GET /api/auth/callback — OAuth callback handler
1537
+ app.get('/api/auth/callback', async (req, res) => {
1538
+ try {
1539
+ const code = req.query['code'];
1540
+ const state = req.query['state'];
1541
+ if (!code || !state) {
1542
+ res.status(400).send('<h2>Missing code or state</h2>');
1543
+ return;
1544
+ }
1545
+ const stateData = oauthStates.get(state);
1546
+ if (!stateData) {
1547
+ res.status(400).send('<h2>Invalid or expired state token</h2>');
1548
+ return;
1549
+ }
1550
+ oauthStates.delete(state);
1551
+ const providerConfig = OAUTH_PROVIDERS[stateData.provider];
1552
+ if (!providerConfig) {
1553
+ res.status(400).send('<h2>Unknown provider</h2>');
1554
+ return;
1555
+ }
1556
+ const { loadConfig } = await import('../core/config.js');
1557
+ const userConfig = loadConfig(config.configPath);
1558
+ const clientId = userConfig[providerConfig.clientIdKey];
1559
+ const clientSecret = userConfig[providerConfig.clientSecretKey];
1560
+ if (!clientId || !clientSecret) {
1561
+ res.status(400).send('<h2>Missing client credentials in config</h2>');
1562
+ return;
1563
+ }
1564
+ const redirectUri = `http://localhost:${port}/api/auth/callback`;
1565
+ const tokenRes = await fetch(providerConfig.tokenUrl, {
1566
+ method: 'POST',
1567
+ headers: {
1568
+ 'Content-Type': 'application/x-www-form-urlencoded',
1569
+ Accept: 'application/json',
1570
+ },
1571
+ body: new URLSearchParams({
1572
+ client_id: clientId,
1573
+ client_secret: clientSecret,
1574
+ code,
1575
+ redirect_uri: redirectUri,
1576
+ grant_type: 'authorization_code',
1577
+ }),
1578
+ });
1579
+ const tokenBody = await tokenRes.json();
1580
+ const accessToken = tokenBody['access_token'];
1581
+ if (!accessToken) {
1582
+ res.status(400).send(`<h2>Token exchange failed</h2><pre>${JSON.stringify(tokenBody, null, 2)}</pre>`);
1583
+ return;
1584
+ }
1585
+ saveOAuthToken(stateData.provider, {
1586
+ access_token: accessToken,
1587
+ refresh_token: tokenBody['refresh_token'],
1588
+ scope: tokenBody['scope'],
1589
+ });
1590
+ res.send(`<!DOCTYPE html><html><head><style>
1591
+ body { font-family: Inter, system-ui, sans-serif; background: #1e1e1e; color: #ccc; display: flex; align-items: center; justify-content: center; min-height: 100vh; }
1592
+ .card { background: #252526; border: 1px solid #3e3e3e; border-radius: 12px; padding: 32px 40px; text-align: center; }
1593
+ h2 { color: #4ec9b0; margin: 0 0 8px; } p { color: #808080; font-size: 14px; }
1594
+ </style></head><body><div class="card"><h2>Connected!</h2><p>${stateData.provider} is now linked to WikiMem.</p><p style="margin-top:12px"><small>You can close this tab.</small></p></div></body></html>`);
1595
+ }
1596
+ catch (err) {
1597
+ const msg = err instanceof Error ? err.message : String(err);
1598
+ res.status(500).send(`<h2>OAuth callback failed</h2><pre>${msg}</pre>`);
1599
+ }
1600
+ });
1601
+ // DELETE /api/auth/tokens/:provider — disconnect a provider
1602
+ app.delete('/api/auth/tokens/:provider', (_req, res) => {
1603
+ try {
1604
+ const provider = _req.params['provider'];
1605
+ if (!provider) {
1606
+ res.status(400).json({ error: 'Missing provider' });
1607
+ return;
1608
+ }
1609
+ const tokenPath = getTokenStorePath();
1610
+ const tokens = loadOAuthTokens();
1611
+ delete tokens[provider];
1612
+ mkdirSync(dirname(tokenPath), { recursive: true });
1613
+ writeFileSync(tokenPath, JSON.stringify(tokens, null, 2), 'utf-8');
1614
+ res.json({ status: 'disconnected', provider });
1615
+ }
1616
+ catch (err) {
1617
+ const msg = err instanceof Error ? err.message : String(err);
1618
+ res.status(500).json({ error: `Disconnect failed: ${msg}` });
1619
+ }
1620
+ });
1621
+ // Pipeline SSE endpoint — real-time step updates
1622
+ app.get('/api/pipeline/events', (_req, res) => {
1623
+ res.writeHead(200, {
1624
+ 'Content-Type': 'text/event-stream',
1625
+ 'Cache-Control': 'no-cache',
1626
+ Connection: 'keep-alive',
1627
+ });
1628
+ res.write('data: {"type":"connected"}\n\n');
1629
+ const onStep = (event) => {
1630
+ res.write(`data: ${JSON.stringify(event)}\n\n`);
1631
+ };
1632
+ import('../core/pipeline-events.js').then(({ pipelineEvents }) => {
1633
+ pipelineEvents.on('step', onStep);
1634
+ _req.on('close', () => pipelineEvents.off('step', onStep));
1635
+ });
1636
+ });
1637
+ // Pipeline runs history
1638
+ app.get('/api/pipeline/runs', async (_req, res) => {
1639
+ try {
1640
+ const { pipelineEvents } = await import('../core/pipeline-events.js');
1641
+ const runs = pipelineEvents.getRecentRuns();
1642
+ const summaries = runs.map(r => ({
1643
+ id: r.id,
1644
+ source: r.source,
1645
+ startedAt: r.startedAt,
1646
+ eventCount: r.events.length,
1647
+ hasSummary: !!r.summary,
1648
+ hasLLMTrace: !!r.llmTrace,
1649
+ result: r.result,
1650
+ }));
1651
+ res.json({ runs: summaries });
1652
+ }
1653
+ catch {
1654
+ res.json({ runs: [] });
1655
+ }
1656
+ });
1657
+ // Pipeline run detail (with LLM trace and summary)
1658
+ app.get('/api/pipeline/runs/:id', async (req, res) => {
1659
+ try {
1660
+ const { pipelineEvents } = await import('../core/pipeline-events.js');
1661
+ const runs = pipelineEvents.getRecentRuns();
1662
+ const run = runs.find(r => r.id === req.params['id']);
1663
+ if (!run) {
1664
+ res.status(404).json({ error: 'Run not found' });
1665
+ return;
1666
+ }
1667
+ res.json(run);
1668
+ }
1669
+ catch {
1670
+ res.status(500).json({ error: 'Failed to get run' });
1671
+ }
1672
+ });
1673
+ // ─── Pipeline Configuration (custom steps) ──────────────────────────────────
1674
+ app.get('/api/pipeline/config', async (_req, res) => {
1675
+ try {
1676
+ const { loadConfig } = await import('../core/config.js');
1677
+ const userConfig = loadConfig(config.configPath);
1678
+ const pipelineConfig = userConfig['pipeline'] ?? {};
1679
+ const customSteps = pipelineConfig['custom_steps'] ?? [];
1680
+ const disabledSteps = pipelineConfig['disabled_steps'] ?? [];
1681
+ res.json({ custom_steps: customSteps, disabled_steps: disabledSteps });
1682
+ }
1683
+ catch {
1684
+ res.json({ custom_steps: [], disabled_steps: [] });
1685
+ }
1686
+ });
1687
+ app.put('/api/pipeline/config', async (req, res) => {
1688
+ try {
1689
+ const { loadConfig } = await import('../core/config.js');
1690
+ const YAML = await import('yaml');
1691
+ const current = loadConfig(config.configPath);
1692
+ const body = req.body;
1693
+ const pipeline = current['pipeline'] ?? {};
1694
+ if (body.custom_steps !== undefined)
1695
+ pipeline['custom_steps'] = body.custom_steps;
1696
+ if (body.disabled_steps !== undefined)
1697
+ pipeline['disabled_steps'] = body.disabled_steps;
1698
+ current['pipeline'] = pipeline;
1699
+ writeFileSync(config.configPath, YAML.stringify(current), 'utf-8');
1700
+ res.json({ status: 'saved' });
1701
+ }
1702
+ catch (err) {
1703
+ const msg = err instanceof Error ? err.message : String(err);
1704
+ res.status(500).json({ error: `Failed to save pipeline config: ${msg}` });
1705
+ }
1706
+ });
1707
+ // ─── Automation 3: Webhook Ingest ─────────────────────────────────────────
1708
+ // Track files currently being ingested by webhook to avoid double-ingest with watcher
1709
+ const webhookIngestingFiles = new Set();
1710
+ // POST /api/webhook/ingest — accept external content, run ingest pipeline
1711
+ app.post('/api/webhook/ingest', async (req, res) => {
1712
+ try {
1713
+ const { content, title, source, tags, metadata } = req.body;
1714
+ if (!content || content.trim().length === 0) {
1715
+ res.status(400).json({ error: 'Missing or empty content field' });
1716
+ return;
1717
+ }
1718
+ const now = new Date().toISOString().split('T')[0] ?? '';
1719
+ const pageTitle = title ?? `Webhook Ingest ${new Date().toISOString()}`;
1720
+ const markdown = `# ${pageTitle}\n\n${source ? `Source: ${source}\n` : ''}Ingested via webhook: ${new Date().toISOString()}\n\n${content}`;
1721
+ // Write to raw/ so ingest pipeline can find it
1722
+ const { mkdirSync: mkdir, writeFileSync: write } = await import('node:fs');
1723
+ const { join: joinPath } = await import('node:path');
1724
+ const { slugify } = await import('../core/vault.js');
1725
+ const dateDir = joinPath(config.rawDir, now);
1726
+ mkdir(dateDir, { recursive: true });
1727
+ const filePath = joinPath(dateDir, `${slugify(pageTitle.substring(0, 60))}-webhook.md`);
1728
+ write(filePath, markdown, 'utf-8');
1729
+ // Tell the watcher to skip this file (webhook handles ingest)
1730
+ webhookIngestingFiles.add(filePath);
1731
+ const { ingestSource } = await import('../core/ingest.js');
1732
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
1733
+ const { loadConfig } = await import('../core/config.js');
1734
+ const { appendAuditEntry } = await import('../core/audit-trail.js');
1735
+ const userConfig = loadConfig(config.configPath);
1736
+ const provider = createProviderFromUserConfig(userConfig);
1737
+ let result;
1738
+ try {
1739
+ result = await ingestSource(filePath, config, provider, {
1740
+ verbose: false,
1741
+ tags: tags ?? [],
1742
+ metadata,
1743
+ addedBy: 'webhook',
1744
+ });
1745
+ }
1746
+ catch (ingestErr) {
1747
+ const ingestMsg = ingestErr instanceof Error ? ingestErr.message : String(ingestErr);
1748
+ appendAuditEntry(vaultRoot, {
1749
+ action: 'ingest',
1750
+ actor: 'webhook',
1751
+ source: source ?? filePath,
1752
+ summary: `Webhook ingest FAILED: "${pageTitle}" — ${ingestMsg.substring(0, 200)}`,
1753
+ pagesAffected: [pageTitle],
1754
+ });
1755
+ webhookIngestingFiles.delete(filePath);
1756
+ res.status(500).json({ error: `Webhook ingest failed: ${ingestMsg}` });
1757
+ return;
1758
+ }
1759
+ appendAuditEntry(vaultRoot, {
1760
+ action: 'ingest',
1761
+ actor: 'webhook',
1762
+ source: source ?? filePath,
1763
+ summary: `Webhook ingest: "${pageTitle}" — ${result.pagesUpdated} pages created.`,
1764
+ pagesAffected: [pageTitle],
1765
+ });
1766
+ webhookIngestingFiles.delete(filePath);
1767
+ res.json({
1768
+ success: !result.rejected,
1769
+ pagesCreated: result.pagesUpdated,
1770
+ title: result.title,
1771
+ rawPath: result.rawPath,
1772
+ rejected: result.rejected ?? false,
1773
+ rejectionReason: result.rejectionReason,
1774
+ });
1775
+ }
1776
+ catch (err) {
1777
+ const msg = err instanceof Error ? err.message : String(err);
1778
+ res.status(500).json({ error: `Webhook ingest failed: ${msg}` });
1779
+ }
1780
+ });
1781
+ // ─── Centralized Ingestion Gateway ───────────────────────────────────────
1782
+ // POST /api/gateway/ingest — universal ingestion from any source
1783
+ app.post('/api/gateway/ingest', async (req, res) => {
1784
+ const runId = `gw-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
1785
+ try {
1786
+ const { content, url, source, title, tags, metadata } = req.body;
1787
+ const sourceLabel = source ?? 'api';
1788
+ const validSources = new Set(['email', 'slack', 'webhook', 'api', 'web']);
1789
+ if (source && !validSources.has(source)) {
1790
+ res.status(400).json({ runId, error: `Invalid source. Expected one of: ${[...validSources].join(', ')}` });
1791
+ return;
1792
+ }
1793
+ if (!content && !url) {
1794
+ res.status(400).json({ runId, error: 'Must provide at least content or url' });
1795
+ return;
1796
+ }
1797
+ const { ingestSource } = await import('../core/ingest.js');
1798
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
1799
+ const { loadConfig } = await import('../core/config.js');
1800
+ const { slugify } = await import('../core/vault.js');
1801
+ const userConfig = loadConfig(config.configPath);
1802
+ const provider = createProviderFromUserConfig(userConfig);
1803
+ const auditActor = 'webhook';
1804
+ // URL-only ingestion: pass URL directly to ingestSource
1805
+ if (url && !content) {
1806
+ const result = await ingestSource(url, config, provider, {
1807
+ verbose: false,
1808
+ tags: tags ?? [],
1809
+ metadata: { ...metadata, gateway_source: sourceLabel, gateway_run: runId },
1810
+ addedBy: 'webhook',
1811
+ });
1812
+ res.json({
1813
+ runId,
1814
+ status: result.rejected ? 'rejected' : 'ingested',
1815
+ pagesCreated: result.pagesUpdated ?? 0,
1816
+ title: result.title,
1817
+ rejected: result.rejected ?? false,
1818
+ rejectionReason: result.rejectionReason,
1819
+ });
1820
+ return;
1821
+ }
1822
+ // Content-based: write to raw/ then ingest
1823
+ const now = new Date().toISOString().split('T')[0] ?? '';
1824
+ const pageTitle = title ?? `Gateway ${sourceLabel} ${new Date().toISOString()}`;
1825
+ const frontmatter = [
1826
+ `source: ${sourceLabel}`,
1827
+ `ingested_via: gateway`,
1828
+ `run_id: ${runId}`,
1829
+ ...(tags?.length ? [`tags: [${tags.join(', ')}]`] : []),
1830
+ ...(metadata ? Object.entries(metadata).map(([k, v]) => `${k}: ${v}`) : []),
1831
+ ].join('\n');
1832
+ const markdown = `---\ntitle: "${pageTitle}"\n${frontmatter}\n---\n\n${content}`;
1833
+ const dateDir = join(config.rawDir, now);
1834
+ mkdirSync(dateDir, { recursive: true });
1835
+ const slug = slugify(pageTitle.substring(0, 60));
1836
+ const filePath = join(dateDir, `${slug}-gw-${sourceLabel}.md`);
1837
+ writeFileSync(filePath, markdown, 'utf-8');
1838
+ webhookIngestingFiles.add(filePath);
1839
+ let result;
1840
+ try {
1841
+ result = await ingestSource(filePath, config, provider, {
1842
+ verbose: false,
1843
+ tags: tags ?? [],
1844
+ metadata: { ...metadata, gateway_source: sourceLabel, gateway_run: runId },
1845
+ addedBy: 'webhook',
1846
+ });
1847
+ }
1848
+ finally {
1849
+ webhookIngestingFiles.delete(filePath);
1850
+ }
1851
+ try {
1852
+ const { appendAuditEntry } = await import('../core/audit-trail.js');
1853
+ appendAuditEntry(vaultRoot, {
1854
+ action: 'ingest',
1855
+ actor: auditActor,
1856
+ source: url ?? filePath,
1857
+ summary: `Gateway ingest (${sourceLabel}): "${pageTitle}" — ${result.pagesUpdated} pages created.`,
1858
+ pagesAffected: [pageTitle],
1859
+ });
1860
+ }
1861
+ catch { /* non-fatal */ }
1862
+ res.json({
1863
+ runId,
1864
+ status: result.rejected ? 'rejected' : 'ingested',
1865
+ pagesCreated: result.pagesUpdated ?? 0,
1866
+ title: result.title,
1867
+ rejected: result.rejected ?? false,
1868
+ rejectionReason: result.rejectionReason,
1869
+ });
1870
+ }
1871
+ catch (err) {
1872
+ const msg = err instanceof Error ? err.message : String(err);
1873
+ res.status(500).json({ runId, status: 'error', error: `Gateway ingest failed: ${msg}` });
1874
+ }
1875
+ });
1876
+ // ─── Audit Trail API ──────────────────────────────────────────────────────
1877
+ // GET /api/audit-trail?limit=50&actor=all&action=all
1878
+ app.get('/api/audit-trail', async (req, res) => {
1879
+ try {
1880
+ const limit = parseInt(req.query['limit']) || 50;
1881
+ const actor = req.query['actor'] || 'all';
1882
+ const action = req.query['action'] || 'all';
1883
+ const since = req.query['since'];
1884
+ const before = req.query['before'];
1885
+ const { readAuditTrail } = await import('../core/audit-trail.js');
1886
+ const entries = readAuditTrail(vaultRoot, limit, actor, action, since, before);
1887
+ res.json({ entries, total: entries.length });
1888
+ }
1889
+ catch (err) {
1890
+ res.status(500).json({ error: String(err) });
1891
+ }
1892
+ });
1893
+ // ─── Automation Settings API ─────────────────────────────────────────────
1894
+ function getAutomationSettingsPath() {
1895
+ return join(vaultRoot, '.wikimem', 'automations.json');
1896
+ }
1897
+ function loadAutomationSettings() {
1898
+ const settingsPath = getAutomationSettingsPath();
1899
+ if (!existsSync(settingsPath))
1900
+ return {};
1901
+ try {
1902
+ return JSON.parse(readFileSync(settingsPath, 'utf-8'));
1903
+ }
1904
+ catch {
1905
+ return {};
1906
+ }
1907
+ }
1908
+ function saveAutomationSettings(settings) {
1909
+ const settingsPath = getAutomationSettingsPath();
1910
+ const dir = join(vaultRoot, '.wikimem');
1911
+ mkdirSync(dir, { recursive: true });
1912
+ writeFileSync(settingsPath, JSON.stringify(settings, null, 2), 'utf-8');
1913
+ }
1914
+ // GET /api/automations/settings — read all automation toggle state
1915
+ app.get('/api/automations/settings', (_req, res) => {
1916
+ try {
1917
+ const settings = loadAutomationSettings();
1918
+ res.json(settings);
1919
+ }
1920
+ catch (err) {
1921
+ res.status(500).json({ error: String(err) });
1922
+ }
1923
+ });
1924
+ // PATCH /api/automations/settings — update a single automation setting
1925
+ app.patch('/api/automations/settings', (req, res) => {
1926
+ try {
1927
+ const { automation, key, value } = req.body;
1928
+ if (!automation || !key) {
1929
+ res.status(400).json({ error: 'Missing automation or key' });
1930
+ return;
1931
+ }
1932
+ const settings = loadAutomationSettings();
1933
+ if (!settings[automation])
1934
+ settings[automation] = {};
1935
+ settings[automation][key] = value;
1936
+ saveAutomationSettings(settings);
1937
+ res.json({ status: 'saved', automation, key, value });
1938
+ }
1939
+ catch (err) {
1940
+ res.status(500).json({ error: String(err) });
1941
+ }
1942
+ });
1943
+ // POST /api/automations/sourcing/run — trigger smart sourcing (alias for /api/automations/scrape)
1944
+ app.post('/api/automations/sourcing/run', async (req, res) => {
1945
+ try {
1946
+ const { runSmartScraper } = await import('../core/scraper.js');
1947
+ const { loadConfig } = await import('../core/config.js');
1948
+ const userConfig = loadConfig(config.configPath);
1949
+ const result = await runSmartScraper(config, userConfig, { dryRun: false, maxItems: 10 });
1950
+ res.json(result);
1951
+ }
1952
+ catch (err) {
1953
+ const msg = err instanceof Error ? err.message : String(err);
1954
+ res.status(500).json({ error: `Sourcing run failed: ${msg}` });
1955
+ }
1956
+ });
1957
+ // ─── Observer APIs ────────────────────────────────────────────────────────
1958
+ // POST /api/observer/run — trigger observer manually
1959
+ app.post('/api/observer/run', async (_req, res) => {
1960
+ try {
1961
+ const { runObserver } = await import('../core/observer.js');
1962
+ const report = await runObserver(config);
1963
+ res.json({
1964
+ success: true,
1965
+ date: report.date,
1966
+ totalPages: report.totalPages,
1967
+ pagesReviewed: report.pagesReviewed,
1968
+ averageScore: report.averageScore,
1969
+ maxScore: report.maxScore,
1970
+ orphanCount: report.orphans.length,
1971
+ gapCount: report.gaps.length,
1972
+ contradictionCount: report.contradictions.length,
1973
+ reportPath: `.wikimem/observer-reports/${report.date}.json`,
1974
+ });
1975
+ }
1976
+ catch (err) {
1977
+ const msg = err instanceof Error ? err.message : String(err);
1978
+ res.status(500).json({ error: `Observer run failed: ${msg}` });
1979
+ }
1980
+ });
1981
+ // GET /api/observer/reports — list all reports
1982
+ app.get('/api/observer/reports', async (_req, res) => {
1983
+ try {
1984
+ const { listObserverReports } = await import('../core/observer.js');
1985
+ const reports = listObserverReports(vaultRoot);
1986
+ res.json({ reports });
1987
+ }
1988
+ catch (err) {
1989
+ res.status(500).json({ error: String(err) });
1990
+ }
1991
+ });
1992
+ // GET /api/observer/reports/:date — get specific report
1993
+ app.get('/api/observer/reports/:date', async (req, res) => {
1994
+ try {
1995
+ const date = req.params['date'];
1996
+ if (!date) {
1997
+ res.status(400).json({ error: 'Missing date' });
1998
+ return;
1999
+ }
2000
+ const { readObserverReport } = await import('../core/observer.js');
2001
+ const report = readObserverReport(vaultRoot, date);
2002
+ if (!report) {
2003
+ res.status(404).json({ error: `No report found for ${date}` });
2004
+ return;
2005
+ }
2006
+ res.json(report);
2007
+ }
2008
+ catch (err) {
2009
+ res.status(500).json({ error: String(err) });
2010
+ }
2011
+ });
2012
+ // ─── Automations: Scrape ──────────────────────────────────────────────────
2013
+ // POST /api/automations/scrape — trigger scrape for a source (or all)
2014
+ app.post('/api/automations/scrape', async (req, res) => {
2015
+ try {
2016
+ const { source: sourceName, dryRun, maxItems } = req.body;
2017
+ const { runSmartScraper } = await import('../core/scraper.js');
2018
+ const { loadConfig } = await import('../core/config.js');
2019
+ const userConfig = loadConfig(config.configPath);
2020
+ const result = await runSmartScraper(config, userConfig, { dryRun: dryRun ?? false, maxItems: maxItems ?? 10 }, sourceName);
2021
+ res.json(result);
2022
+ }
2023
+ catch (err) {
2024
+ const msg = err instanceof Error ? err.message : String(err);
2025
+ res.status(500).json({ error: `Scrape failed: ${msg}` });
2026
+ }
2027
+ });
2028
+ // POST /api/automations/observe — trigger observer with optional budget
2029
+ app.post('/api/automations/observe', async (req, res) => {
2030
+ try {
2031
+ const { maxPagesToReview, maxCostEstimate } = req.body;
2032
+ void maxCostEstimate;
2033
+ const { runObserver } = await import('../core/observer.js');
2034
+ const report = await runObserver(config, { maxPagesToReview });
2035
+ res.json({
2036
+ success: true,
2037
+ date: report.date,
2038
+ totalPages: report.totalPages,
2039
+ pagesReviewed: report.pagesReviewed,
2040
+ averageScore: report.averageScore,
2041
+ maxScore: report.maxScore,
2042
+ orphanCount: report.orphans.length,
2043
+ gapCount: report.gaps.length,
2044
+ contradictionCount: report.contradictions.length,
2045
+ reportPath: `.wikimem/observer-reports/${report.date}.json`,
2046
+ });
2047
+ }
2048
+ catch (err) {
2049
+ const msg = err instanceof Error ? err.message : String(err);
2050
+ res.status(500).json({ error: `Observer run failed: ${msg}` });
2051
+ }
2052
+ });
2053
+ // GET /api/automations/status — status of all three automations
2054
+ app.get('/api/automations/status', async (_req, res) => {
2055
+ try {
2056
+ const { readAuditTrail } = await import('../core/audit-trail.js');
2057
+ const { isObserverCronRunning } = await import('../core/observer.js');
2058
+ const { loadConfig } = await import('../core/config.js');
2059
+ const userConfig = loadConfig(config.configPath);
2060
+ const settings = loadAutomationSettings();
2061
+ const recentIngest = readAuditTrail(vaultRoot, 1, undefined, 'ingest');
2062
+ const recentScrape = readAuditTrail(vaultRoot, 1, undefined, 'scrape');
2063
+ const recentObserve = readAuditTrail(vaultRoot, 1, undefined, 'observe');
2064
+ res.json({
2065
+ ingest: {
2066
+ enabled: settings['ingest']?.['enabled'] !== false,
2067
+ lastRun: recentIngest[0]?.timestamp ?? null,
2068
+ watcherActive: true,
2069
+ },
2070
+ scrape: {
2071
+ enabled: settings['scrape']?.['enabled'] !== false,
2072
+ lastRun: recentScrape[0]?.timestamp ?? null,
2073
+ sourcesConfigured: (userConfig.sources ?? []).length,
2074
+ },
2075
+ observe: {
2076
+ enabled: settings['observe']?.['enabled'] !== false,
2077
+ lastRun: recentObserve[0]?.timestamp ?? null,
2078
+ cronActive: isObserverCronRunning(),
2079
+ schedule: '0 3 * * *',
2080
+ },
2081
+ });
2082
+ }
2083
+ catch (err) {
2084
+ const msg = err instanceof Error ? err.message : String(err);
2085
+ res.status(500).json({ error: `Failed to get automation status: ${msg}` });
2086
+ }
2087
+ });
2088
+ // ─── Connector APIs ───────────────────────────────────────────────────────
2089
+ // GET /api/connectors — list all configured connectors
2090
+ app.get('/api/connectors', async (_req, res) => {
2091
+ try {
2092
+ const { getConnectorManager } = await import('../core/connectors.js');
2093
+ const mgr = getConnectorManager(vaultRoot);
2094
+ const connectors = mgr.getAll().map(c => ({
2095
+ ...c,
2096
+ exists: existsSync(c.path),
2097
+ }));
2098
+ res.json({ connectors });
2099
+ }
2100
+ catch (err) {
2101
+ res.status(500).json({ error: String(err) });
2102
+ }
2103
+ });
2104
+ // POST /api/connectors — add a new connector
2105
+ app.post('/api/connectors', async (req, res) => {
2106
+ try {
2107
+ const { getConnectorManager } = await import('../core/connectors.js');
2108
+ const mgr = getConnectorManager(vaultRoot);
2109
+ const body = req.body;
2110
+ if (!body.type) {
2111
+ res.status(400).json({ error: 'Missing type' });
2112
+ return;
2113
+ }
2114
+ let resolvedPath = body.path || '';
2115
+ // For git repos, clone if URL provided
2116
+ if (body.type === 'github' || (body.type === 'git-repo' && body.url)) {
2117
+ const reposDir = join(vaultRoot, '.wikimem-repos');
2118
+ mkdirSync(reposDir, { recursive: true });
2119
+ const repoName = (body.url ?? '').split('/').pop()?.replace('.git', '') || 'repo';
2120
+ resolvedPath = join(reposDir, repoName);
2121
+ if (!existsSync(resolvedPath)) {
2122
+ const result = await mgr.cloneRepo(body.url, resolvedPath);
2123
+ if (!result.success) {
2124
+ res.status(500).json({ error: result.error });
2125
+ return;
2126
+ }
2127
+ }
2128
+ }
2129
+ if (!resolvedPath || !existsSync(resolvedPath)) {
2130
+ res.status(400).json({ error: `Path does not exist: ${resolvedPath}` });
2131
+ return;
2132
+ }
2133
+ const connector = mgr.add({
2134
+ type: body.type,
2135
+ name: body.name || basename(resolvedPath),
2136
+ path: resolvedPath,
2137
+ url: body.url,
2138
+ includeGlobs: body.includeGlobs,
2139
+ excludeGlobs: body.excludeGlobs,
2140
+ autoSync: body.autoSync ?? false,
2141
+ });
2142
+ if (connector.autoSync)
2143
+ mgr.startWatcher(connector);
2144
+ res.json({ connector });
2145
+ }
2146
+ catch (err) {
2147
+ res.status(500).json({ error: String(err) });
2148
+ }
2149
+ });
2150
+ // DELETE /api/connectors/:id — remove a connector
2151
+ app.delete('/api/connectors/:id', async (req, res) => {
2152
+ try {
2153
+ const { getConnectorManager } = await import('../core/connectors.js');
2154
+ const mgr = getConnectorManager(vaultRoot);
2155
+ const removed = mgr.remove(req.params['id']);
2156
+ res.json({ removed });
2157
+ }
2158
+ catch (err) {
2159
+ res.status(500).json({ error: String(err) });
2160
+ }
2161
+ });
2162
+ // POST /api/connectors/:id/sync — manually trigger a sync (scan + ingest all files)
2163
+ app.post('/api/connectors/:id/sync', async (req, res) => {
2164
+ try {
2165
+ const { getConnectorManager } = await import('../core/connectors.js');
2166
+ const { ingestSource } = await import('../core/ingest.js');
2167
+ const mgr = getConnectorManager(vaultRoot);
2168
+ const connector = mgr.get(req.params['id']);
2169
+ if (!connector) {
2170
+ res.status(404).json({ error: 'Connector not found' });
2171
+ return;
2172
+ }
2173
+ mgr.updateStatus(connector.id, 'syncing');
2174
+ const startMs = Date.now();
2175
+ const files = await mgr.scanFiles(connector);
2176
+ let pagesCreated = 0, linksAdded = 0, ingested = 0;
2177
+ const errors = [];
2178
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
2179
+ const { loadConfig } = await import('../core/config.js');
2180
+ const userConfig = loadConfig(config.configPath);
2181
+ const provider = createProviderFromUserConfig(userConfig);
2182
+ for (const filePath of files.slice(0, 50)) { // cap at 50 per sync
2183
+ try {
2184
+ const result = await ingestSource(filePath, config, provider, { verbose: false });
2185
+ pagesCreated += result.pagesUpdated ?? 0;
2186
+ linksAdded += result.linksAdded ?? 0;
2187
+ ingested++;
2188
+ }
2189
+ catch (e) {
2190
+ errors.push(`${basename(filePath)}: ${String(e).substring(0, 100)}`);
2191
+ }
2192
+ }
2193
+ mgr.updateStatus(connector.id, 'active', {
2194
+ lastSyncAt: new Date().toISOString(),
2195
+ totalFiles: files.length,
2196
+ });
2197
+ const result = {
2198
+ connectorId: connector.id,
2199
+ filesFound: files.length,
2200
+ filesIngested: ingested,
2201
+ pagesCreated,
2202
+ linksAdded,
2203
+ errors,
2204
+ duration: Date.now() - startMs,
2205
+ };
2206
+ res.json(result);
2207
+ }
2208
+ catch (err) {
2209
+ res.status(500).json({ error: String(err) });
2210
+ }
2211
+ });
2212
+ // POST /api/connectors/:id/scan — scan files in connector (no ingest)
2213
+ app.get('/api/connectors/:id/scan', async (req, res) => {
2214
+ try {
2215
+ const { getConnectorManager } = await import('../core/connectors.js');
2216
+ const mgr = getConnectorManager(vaultRoot);
2217
+ const connector = mgr.get(req.params['id']);
2218
+ if (!connector) {
2219
+ res.status(404).json({ error: 'Not found' });
2220
+ return;
2221
+ }
2222
+ const files = await mgr.scanFiles(connector);
2223
+ res.json({ files: files.slice(0, 200), total: files.length });
2224
+ }
2225
+ catch (err) {
2226
+ res.status(500).json({ error: String(err) });
2227
+ }
2228
+ });
2229
+ // Wire file-detected events from connectors to auto-ingest
2230
+ (async () => {
2231
+ const { getConnectorManager } = await import('../core/connectors.js');
2232
+ const { ingestSource } = await import('../core/ingest.js');
2233
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
2234
+ const { loadConfig } = await import('../core/config.js');
2235
+ const mgr = getConnectorManager(vaultRoot);
2236
+ mgr.on('file-detected', async ({ connectorId, filePath }) => {
2237
+ const connector = mgr.get(connectorId);
2238
+ if (!connector)
2239
+ return;
2240
+ mgr.updateStatus(connectorId, 'syncing');
2241
+ try {
2242
+ const userConfig = loadConfig(config.configPath);
2243
+ const provider = createProviderFromUserConfig(userConfig);
2244
+ await ingestSource(filePath, config, provider, { verbose: false });
2245
+ mgr.updateStatus(connectorId, 'active', { lastSyncAt: new Date().toISOString() });
2246
+ }
2247
+ catch {
2248
+ mgr.updateStatus(connectorId, 'error', { errorMessage: `Failed to ingest ${basename(filePath)}` });
2249
+ }
2250
+ });
2251
+ mgr.startAllWatchers();
2252
+ })().catch(() => { });
2253
+ // Start Observer nightly cron (3am)
2254
+ import('../core/observer.js').then(({ startObserverCron }) => {
2255
+ startObserverCron(config);
2256
+ }).catch(() => { });
2257
+ // AUTO-005: Watch raw/ directory for new files, auto-trigger ingest
2258
+ (async () => {
2259
+ try {
2260
+ const chokidar = await import('chokidar');
2261
+ const { ingestSource } = await import('../core/ingest.js');
2262
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
2263
+ const { loadConfig } = await import('../core/config.js');
2264
+ const rawDir = config.rawDir;
2265
+ if (!existsSync(rawDir))
2266
+ return;
2267
+ const watcher = chokidar.watch(rawDir, {
2268
+ ignored: ['**/.git/**', '**/*.tmp'],
2269
+ persistent: true,
2270
+ ignoreInitial: true,
2271
+ awaitWriteFinish: { stabilityThreshold: 1500 },
2272
+ });
2273
+ watcher.on('add', async (filePath) => {
2274
+ const INGESTIBLE = new Set(['.md', '.txt', '.pdf', '.json', '.yaml', '.yml', '.csv', '.html', '.docx', '.mp3', '.wav', '.m4a', '.mp4', '.mov']);
2275
+ const ext = filePath.split('.').pop()?.toLowerCase() || '';
2276
+ if (!INGESTIBLE.has('.' + ext))
2277
+ return;
2278
+ if (webhookIngestingFiles.has(filePath)) {
2279
+ console.log(`[watcher] Skipping ${filePath} (already being ingested by webhook)`);
2280
+ return;
2281
+ }
2282
+ console.log(`[watcher] New file detected: ${filePath}`);
2283
+ try {
2284
+ const userConfig = loadConfig(config.configPath);
2285
+ const provider = createProviderFromUserConfig(userConfig);
2286
+ await ingestSource(filePath, config, provider, { verbose: false });
2287
+ console.log(`[watcher] Ingested: ${filePath}`);
2288
+ }
2289
+ catch (err) {
2290
+ console.error(`[watcher] Failed to ingest ${filePath}:`, err);
2291
+ }
2292
+ });
2293
+ console.log(` Auto-watching raw/ for new files: ${rawDir}`);
2294
+ }
2295
+ catch { }
2296
+ })().catch(() => { });
2297
+ // Serve static files AFTER all API routes
2298
+ if (existsSync(publicDir)) {
2299
+ app.use(express.static(publicDir));
2300
+ }
229
2301
  // Serve index.html for all other routes (SPA)
230
2302
  app.get('/{*path}', (_req, res) => {
231
2303
  const indexPath = join(publicDir, 'index.html');