wikimem 0.2.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +138 -29
- package/README.md +173 -311
- package/dist/cli/commands/ask.d.ts +3 -0
- package/dist/cli/commands/ask.d.ts.map +1 -0
- package/dist/cli/commands/ask.js +63 -0
- package/dist/cli/commands/ask.js.map +1 -0
- package/dist/cli/commands/export.d.ts +3 -0
- package/dist/cli/commands/export.d.ts.map +1 -0
- package/dist/cli/commands/export.js +108 -0
- package/dist/cli/commands/export.js.map +1 -0
- package/dist/cli/commands/history.d.ts +3 -0
- package/dist/cli/commands/history.d.ts.map +1 -0
- package/dist/cli/commands/history.js +61 -0
- package/dist/cli/commands/history.js.map +1 -0
- package/dist/cli/commands/improve.d.ts.map +1 -1
- package/dist/cli/commands/improve.js +4 -3
- package/dist/cli/commands/improve.js.map +1 -1
- package/dist/cli/commands/ingest.d.ts.map +1 -1
- package/dist/cli/commands/ingest.js +5 -4
- package/dist/cli/commands/ingest.js.map +1 -1
- package/dist/cli/commands/init.d.ts.map +1 -1
- package/dist/cli/commands/init.js +246 -79
- package/dist/cli/commands/init.js.map +1 -1
- package/dist/cli/commands/lint.d.ts.map +1 -1
- package/dist/cli/commands/lint.js +4 -3
- package/dist/cli/commands/lint.js.map +1 -1
- package/dist/cli/commands/mcp.d.ts +3 -0
- package/dist/cli/commands/mcp.d.ts.map +1 -0
- package/dist/cli/commands/mcp.js +11 -0
- package/dist/cli/commands/mcp.js.map +1 -0
- package/dist/cli/commands/open.d.ts +3 -0
- package/dist/cli/commands/open.d.ts.map +1 -0
- package/dist/cli/commands/open.js +36 -0
- package/dist/cli/commands/open.js.map +1 -0
- package/dist/cli/commands/query.d.ts.map +1 -1
- package/dist/cli/commands/query.js +5 -4
- package/dist/cli/commands/query.js.map +1 -1
- package/dist/cli/commands/search.d.ts +3 -0
- package/dist/cli/commands/search.d.ts.map +1 -0
- package/dist/cli/commands/search.js +61 -0
- package/dist/cli/commands/search.js.map +1 -0
- package/dist/cli/commands/serve.d.ts.map +1 -1
- package/dist/cli/commands/serve.js +41 -2
- package/dist/cli/commands/serve.js.map +1 -1
- package/dist/cli/commands/watch.d.ts.map +1 -1
- package/dist/cli/commands/watch.js +4 -3
- package/dist/cli/commands/watch.js.map +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +27 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/core/audit-trail.d.ts +15 -0
- package/dist/core/audit-trail.d.ts.map +1 -0
- package/dist/core/audit-trail.js +43 -0
- package/dist/core/audit-trail.js.map +1 -0
- package/dist/core/claude-code.d.ts +10 -0
- package/dist/core/claude-code.d.ts.map +1 -0
- package/dist/core/claude-code.js +81 -0
- package/dist/core/claude-code.js.map +1 -0
- package/dist/core/config.d.ts +23 -0
- package/dist/core/config.d.ts.map +1 -1
- package/dist/core/config.js.map +1 -1
- package/dist/core/connectors.d.ts +58 -0
- package/dist/core/connectors.d.ts.map +1 -0
- package/dist/core/connectors.js +189 -0
- package/dist/core/connectors.js.map +1 -0
- package/dist/core/folder-scanner.d.ts +10 -0
- package/dist/core/folder-scanner.d.ts.map +1 -0
- package/dist/core/folder-scanner.js +84 -0
- package/dist/core/folder-scanner.js.map +1 -0
- package/dist/core/git.d.ts +137 -0
- package/dist/core/git.d.ts.map +1 -0
- package/dist/core/git.js +520 -0
- package/dist/core/git.js.map +1 -0
- package/dist/core/history.d.ts +21 -0
- package/dist/core/history.d.ts.map +1 -0
- package/dist/core/history.js +107 -0
- package/dist/core/history.js.map +1 -0
- package/dist/core/improve.d.ts.map +1 -1
- package/dist/core/improve.js +9 -0
- package/dist/core/improve.js.map +1 -1
- package/dist/core/ingest.d.ts +1 -0
- package/dist/core/ingest.d.ts.map +1 -1
- package/dist/core/ingest.js +78 -5
- package/dist/core/ingest.js.map +1 -1
- package/dist/core/observer.d.ts +71 -0
- package/dist/core/observer.d.ts.map +1 -0
- package/dist/core/observer.js +350 -0
- package/dist/core/observer.js.map +1 -0
- package/dist/core/pipeline-events.d.ts +63 -0
- package/dist/core/pipeline-events.d.ts.map +1 -0
- package/dist/core/pipeline-events.js +109 -0
- package/dist/core/pipeline-events.js.map +1 -0
- package/dist/core/query.d.ts.map +1 -1
- package/dist/core/query.js +16 -8
- package/dist/core/query.js.map +1 -1
- package/dist/core/scraper.d.ts +41 -0
- package/dist/core/scraper.d.ts.map +1 -0
- package/dist/core/scraper.js +277 -0
- package/dist/core/scraper.js.map +1 -0
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/mcp-entry.d.ts +10 -0
- package/dist/mcp-entry.d.ts.map +1 -0
- package/dist/mcp-entry.js +21 -0
- package/dist/mcp-entry.js.map +1 -0
- package/dist/mcp-server.d.ts +15 -0
- package/dist/mcp-server.d.ts.map +1 -0
- package/dist/mcp-server.js +390 -0
- package/dist/mcp-server.js.map +1 -0
- package/dist/processors/audio.d.ts.map +1 -1
- package/dist/processors/audio.js +42 -4
- package/dist/processors/audio.js.map +1 -1
- package/dist/processors/pdf.d.ts.map +1 -1
- package/dist/processors/pdf.js +2 -3
- package/dist/processors/pdf.js.map +1 -1
- package/dist/processors/url.js +4 -1
- package/dist/processors/url.js.map +1 -1
- package/dist/providers/claude.d.ts +1 -0
- package/dist/providers/claude.d.ts.map +1 -1
- package/dist/providers/claude.js +5 -3
- package/dist/providers/claude.js.map +1 -1
- package/dist/providers/index.d.ts +17 -1
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js +144 -0
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/openai.d.ts +1 -0
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +5 -3
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/types.d.ts +18 -0
- package/dist/providers/types.d.ts.map +1 -1
- package/dist/templates/config-yaml.d.ts.map +1 -1
- package/dist/templates/config-yaml.js +13 -2
- package/dist/templates/config-yaml.js.map +1 -1
- package/dist/web/public/index.html +8157 -745
- package/dist/web/server.d.ts.map +1 -1
- package/dist/web/server.js +2101 -29
- package/dist/web/server.js.map +1 -1
- package/package.json +7 -4
- package/scripts/install.sh +54 -0
- package/src/web/public/index.html +8157 -745
- package/templates/mcp-config.json +9 -0
- package/dist/web/public/public/index.html +0 -946
package/dist/web/server.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import express from 'express';
|
|
2
|
-
import { existsSync, writeFileSync, mkdirSync, readdirSync, statSync } from 'node:fs';
|
|
3
|
-
import { join, basename } from 'node:path';
|
|
2
|
+
import { readFileSync, existsSync, writeFileSync, mkdirSync, readdirSync, statSync, renameSync, unlinkSync as fsUnlinkSync } from 'node:fs';
|
|
3
|
+
import { join, resolve, extname, basename, dirname, relative } from 'node:path';
|
|
4
4
|
import { fileURLToPath } from 'node:url';
|
|
5
|
-
import {
|
|
5
|
+
import { randomBytes } from 'node:crypto';
|
|
6
|
+
import { getVaultConfig, getVaultStats, listWikiPages, readWikiPage, writeWikiPage } from '../core/vault.js';
|
|
6
7
|
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
|
7
8
|
function buildGraph(config) {
|
|
8
9
|
const pages = listWikiPages(config.wikiDir);
|
|
@@ -71,13 +72,13 @@ function listPages(config) {
|
|
|
71
72
|
export function createServer(vaultRoot, port) {
|
|
72
73
|
const app = express();
|
|
73
74
|
const config = getVaultConfig(vaultRoot);
|
|
75
|
+
// Load persisted pipeline runs so they survive server restarts
|
|
76
|
+
import('../core/pipeline-events.js').then(({ pipelineEvents }) => {
|
|
77
|
+
pipelineEvents.initPersistence(vaultRoot);
|
|
78
|
+
}).catch(() => { });
|
|
74
79
|
app.use(express.json());
|
|
75
80
|
app.use(express.urlencoded({ extended: true }));
|
|
76
|
-
// Serve static files
|
|
77
81
|
const publicDir = join(__dirname, 'public');
|
|
78
|
-
if (existsSync(publicDir)) {
|
|
79
|
-
app.use(express.static(publicDir));
|
|
80
|
-
}
|
|
81
82
|
// API: vault status
|
|
82
83
|
app.get('/api/status', (_req, res) => {
|
|
83
84
|
try {
|
|
@@ -98,6 +99,28 @@ export function createServer(vaultRoot, port) {
|
|
|
98
99
|
res.status(500).json({ error: 'Failed to list pages' });
|
|
99
100
|
}
|
|
100
101
|
});
|
|
102
|
+
// API: create new page
|
|
103
|
+
app.post('/api/pages', (req, res) => {
|
|
104
|
+
try {
|
|
105
|
+
const { title, slug, content } = req.body;
|
|
106
|
+
if (!title || !slug) {
|
|
107
|
+
res.status(400).json({ error: 'Missing title or slug' });
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
const dest = join(config.wikiDir, `${slug}.md`);
|
|
111
|
+
if (existsSync(dest)) {
|
|
112
|
+
res.status(409).json({ error: 'Page already exists' });
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
mkdirSync(config.wikiDir, { recursive: true });
|
|
116
|
+
writeFileSync(dest, content ?? `---\ntitle: "${title}"\ntype: page\n---\n\n# ${title}\n`, 'utf-8');
|
|
117
|
+
res.json({ status: 'created', path: dest, title });
|
|
118
|
+
}
|
|
119
|
+
catch (err) {
|
|
120
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
121
|
+
res.status(500).json({ error: `Failed to create page: ${msg}` });
|
|
122
|
+
}
|
|
123
|
+
});
|
|
101
124
|
// API: read single page
|
|
102
125
|
app.get('/api/pages/:title', (req, res) => {
|
|
103
126
|
try {
|
|
@@ -133,6 +156,240 @@ export function createServer(vaultRoot, port) {
|
|
|
133
156
|
res.status(500).json({ error: 'Failed to read page' });
|
|
134
157
|
}
|
|
135
158
|
});
|
|
159
|
+
// API: read raw page content (for editing)
|
|
160
|
+
app.get('/api/wiki/page/raw', (req, res) => {
|
|
161
|
+
try {
|
|
162
|
+
const pagePath = req.query['path'];
|
|
163
|
+
if (!pagePath) {
|
|
164
|
+
res.status(400).json({ error: 'Missing path query parameter' });
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
const resolved = resolve(pagePath);
|
|
168
|
+
if (!resolved.startsWith(resolve(config.wikiDir))) {
|
|
169
|
+
res.status(403).json({ error: 'Access denied: path outside wiki directory' });
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
if (!existsSync(resolved)) {
|
|
173
|
+
res.status(404).json({ error: 'File not found' });
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
const raw = readFileSync(resolved, 'utf-8');
|
|
177
|
+
res.json({ path: resolved, raw });
|
|
178
|
+
}
|
|
179
|
+
catch (err) {
|
|
180
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
181
|
+
res.status(500).json({ error: `Failed to read raw page: ${msg}` });
|
|
182
|
+
}
|
|
183
|
+
});
|
|
184
|
+
// API: save wiki page (write to disk + auto-commit)
|
|
185
|
+
app.put('/api/wiki/page', async (req, res) => {
|
|
186
|
+
try {
|
|
187
|
+
const { path: pagePath, content, frontmatter } = req.body;
|
|
188
|
+
if (!pagePath || content === undefined) {
|
|
189
|
+
res.status(400).json({ error: 'Missing path or content' });
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
const resolved = resolve(pagePath);
|
|
193
|
+
if (!resolved.startsWith(resolve(config.wikiDir))) {
|
|
194
|
+
res.status(403).json({ error: 'Access denied: path outside wiki directory' });
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
// Parse the raw content — if it includes frontmatter, use gray-matter
|
|
198
|
+
const matter = await import('gray-matter');
|
|
199
|
+
const parsed = matter.default(content);
|
|
200
|
+
const finalFrontmatter = frontmatter ?? parsed.data;
|
|
201
|
+
const finalContent = frontmatter ? content : parsed.content;
|
|
202
|
+
writeWikiPage(resolved, finalContent, finalFrontmatter);
|
|
203
|
+
// Auto-commit if git-initialized
|
|
204
|
+
let commitResult = null;
|
|
205
|
+
try {
|
|
206
|
+
const { autoCommit, isGitRepo } = await import('../core/git.js');
|
|
207
|
+
if (await isGitRepo(config.root)) {
|
|
208
|
+
const title = finalFrontmatter['title'] ?? basename(resolved, '.md');
|
|
209
|
+
commitResult = await autoCommit(config.root, 'manual', `edit "${title}" via web UI`);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
catch { /* non-fatal */ }
|
|
213
|
+
res.json({ status: 'saved', path: resolved, commit: commitResult });
|
|
214
|
+
}
|
|
215
|
+
catch (err) {
|
|
216
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
217
|
+
res.status(500).json({ error: `Failed to save page: ${msg}` });
|
|
218
|
+
}
|
|
219
|
+
});
|
|
220
|
+
// API: read raw page content (full markdown with frontmatter)
|
|
221
|
+
app.get('/api/pages/:title/raw', (req, res) => {
|
|
222
|
+
try {
|
|
223
|
+
const title = req.params['title'];
|
|
224
|
+
if (!title) {
|
|
225
|
+
res.status(400).json({ error: 'Missing title' });
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
const pages = listWikiPages(config.wikiDir);
|
|
229
|
+
const titleLower = title.toLowerCase();
|
|
230
|
+
const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
|
231
|
+
const match = pages.find((p) => {
|
|
232
|
+
const fileSlug = basename(p, '.md');
|
|
233
|
+
if (fileSlug === title || fileSlug === slugified)
|
|
234
|
+
return true;
|
|
235
|
+
try {
|
|
236
|
+
const page = readWikiPage(p);
|
|
237
|
+
return page.title.toLowerCase() === titleLower;
|
|
238
|
+
}
|
|
239
|
+
catch {
|
|
240
|
+
return false;
|
|
241
|
+
}
|
|
242
|
+
});
|
|
243
|
+
if (!match) {
|
|
244
|
+
res.status(404).json({ error: 'Page not found' });
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
const raw = readFileSync(match, 'utf-8');
|
|
248
|
+
res.json({ raw, path: match, slug: basename(match, '.md') });
|
|
249
|
+
}
|
|
250
|
+
catch (err) {
|
|
251
|
+
res.status(500).json({ error: 'Failed to read raw page' });
|
|
252
|
+
}
|
|
253
|
+
});
|
|
254
|
+
// API: update page content (full markdown with frontmatter)
|
|
255
|
+
app.put('/api/pages/:title', async (req, res) => {
|
|
256
|
+
try {
|
|
257
|
+
const title = req.params['title'];
|
|
258
|
+
if (!title) {
|
|
259
|
+
res.status(400).json({ error: 'Missing title' });
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
const { content } = req.body;
|
|
263
|
+
if (content === undefined || content === null) {
|
|
264
|
+
res.status(400).json({ error: 'Missing content field' });
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
const pages = listWikiPages(config.wikiDir);
|
|
268
|
+
const titleLower = title.toLowerCase();
|
|
269
|
+
const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
|
270
|
+
const match = pages.find((p) => {
|
|
271
|
+
const fileSlug = basename(p, '.md');
|
|
272
|
+
if (fileSlug === title || fileSlug === slugified)
|
|
273
|
+
return true;
|
|
274
|
+
try {
|
|
275
|
+
const page = readWikiPage(p);
|
|
276
|
+
return page.title.toLowerCase() === titleLower;
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
return false;
|
|
280
|
+
}
|
|
281
|
+
});
|
|
282
|
+
if (!match) {
|
|
283
|
+
res.status(404).json({ error: 'Page not found' });
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
writeFileSync(match, content, 'utf-8');
|
|
287
|
+
// Extract title from new content for commit message
|
|
288
|
+
const matter = await import('gray-matter');
|
|
289
|
+
const parsed = matter.default(content);
|
|
290
|
+
const pageTitle = parsed.data['title'] || title;
|
|
291
|
+
// Auto-commit via git
|
|
292
|
+
try {
|
|
293
|
+
const { autoCommit } = await import('../core/git.js');
|
|
294
|
+
await autoCommit(config.root, 'manual', `edit page "${pageTitle}"`);
|
|
295
|
+
}
|
|
296
|
+
catch { /* git commit is best-effort */ }
|
|
297
|
+
const page = readWikiPage(match);
|
|
298
|
+
res.json({ status: 'saved', page });
|
|
299
|
+
}
|
|
300
|
+
catch (err) {
|
|
301
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
302
|
+
res.status(500).json({ error: `Failed to save page: ${msg}` });
|
|
303
|
+
}
|
|
304
|
+
});
|
|
305
|
+
// API: delete a wiki page
|
|
306
|
+
app.delete('/api/pages/:title', async (req, res) => {
|
|
307
|
+
try {
|
|
308
|
+
const title = req.params['title'];
|
|
309
|
+
if (!title) {
|
|
310
|
+
res.status(400).json({ error: 'Missing title' });
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
const pages = listWikiPages(config.wikiDir);
|
|
314
|
+
const titleLower = title.toLowerCase();
|
|
315
|
+
const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
|
316
|
+
const match = pages.find((p) => {
|
|
317
|
+
const fileSlug = basename(p, '.md');
|
|
318
|
+
if (fileSlug === title || fileSlug === slugified)
|
|
319
|
+
return true;
|
|
320
|
+
try {
|
|
321
|
+
return readWikiPage(p).title.toLowerCase() === titleLower;
|
|
322
|
+
}
|
|
323
|
+
catch {
|
|
324
|
+
return false;
|
|
325
|
+
}
|
|
326
|
+
});
|
|
327
|
+
if (!match) {
|
|
328
|
+
res.status(404).json({ error: 'Page not found' });
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
const { unlinkSync } = await import('node:fs');
|
|
332
|
+
unlinkSync(match);
|
|
333
|
+
try {
|
|
334
|
+
const { autoCommit } = await import('../core/git.js');
|
|
335
|
+
await autoCommit(config.root, 'manual', `delete page "${title}"`);
|
|
336
|
+
}
|
|
337
|
+
catch { }
|
|
338
|
+
res.json({ status: 'deleted' });
|
|
339
|
+
}
|
|
340
|
+
catch (err) {
|
|
341
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
342
|
+
res.status(500).json({ error: `Failed to delete: ${msg}` });
|
|
343
|
+
}
|
|
344
|
+
});
|
|
345
|
+
// API: rename a wiki page
|
|
346
|
+
app.post('/api/pages/:title/rename', async (req, res) => {
|
|
347
|
+
try {
|
|
348
|
+
const oldTitle = req.params['title'];
|
|
349
|
+
const { newTitle } = req.body;
|
|
350
|
+
if (!oldTitle || !newTitle) {
|
|
351
|
+
res.status(400).json({ error: 'Missing title' });
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
const pages = listWikiPages(config.wikiDir);
|
|
355
|
+
const titleLower = oldTitle.toLowerCase();
|
|
356
|
+
const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
|
357
|
+
const match = pages.find((p) => {
|
|
358
|
+
const fileSlug = basename(p, '.md');
|
|
359
|
+
if (fileSlug === oldTitle || fileSlug === slugified)
|
|
360
|
+
return true;
|
|
361
|
+
try {
|
|
362
|
+
return readWikiPage(p).title.toLowerCase() === titleLower;
|
|
363
|
+
}
|
|
364
|
+
catch {
|
|
365
|
+
return false;
|
|
366
|
+
}
|
|
367
|
+
});
|
|
368
|
+
if (!match) {
|
|
369
|
+
res.status(404).json({ error: 'Page not found' });
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
const content = readFileSync(match, 'utf-8');
|
|
373
|
+
const newContent = content.replace(/^title:\s*["']?.*?["']?\s*$/m, `title: "${newTitle}"`);
|
|
374
|
+
const newSlug = newTitle.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
|
375
|
+
const newPath = join(match.substring(0, match.lastIndexOf('/')), newSlug + '.md');
|
|
376
|
+
writeFileSync(newPath, newContent, 'utf-8');
|
|
377
|
+
if (newPath !== match) {
|
|
378
|
+
const { unlinkSync } = await import('node:fs');
|
|
379
|
+
unlinkSync(match);
|
|
380
|
+
}
|
|
381
|
+
try {
|
|
382
|
+
const { autoCommit } = await import('../core/git.js');
|
|
383
|
+
await autoCommit(config.root, 'manual', `rename "${oldTitle}" → "${newTitle}"`);
|
|
384
|
+
}
|
|
385
|
+
catch { }
|
|
386
|
+
res.json({ status: 'renamed', newTitle, newSlug });
|
|
387
|
+
}
|
|
388
|
+
catch (err) {
|
|
389
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
390
|
+
res.status(500).json({ error: `Failed to rename: ${msg}` });
|
|
391
|
+
}
|
|
392
|
+
});
|
|
136
393
|
// API: knowledge graph data
|
|
137
394
|
app.get('/api/graph', (_req, res) => {
|
|
138
395
|
try {
|
|
@@ -143,26 +400,43 @@ export function createServer(vaultRoot, port) {
|
|
|
143
400
|
res.status(500).json({ error: 'Failed to build graph' });
|
|
144
401
|
}
|
|
145
402
|
});
|
|
146
|
-
// API: upload raw file
|
|
403
|
+
// API: upload raw file and auto-trigger ingest
|
|
147
404
|
app.post('/api/upload', (req, res) => {
|
|
148
405
|
const chunks = [];
|
|
149
406
|
req.on('data', (chunk) => chunks.push(chunk));
|
|
150
|
-
req.on('end', () => {
|
|
407
|
+
req.on('end', async () => {
|
|
151
408
|
const filename = req.headers['x-filename'];
|
|
152
409
|
if (!filename) {
|
|
153
410
|
res.status(400).json({ error: 'Missing x-filename header' });
|
|
154
411
|
return;
|
|
155
412
|
}
|
|
156
|
-
const
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
const dest = join(rawDir, basename(filename));
|
|
413
|
+
const now = new Date().toISOString().split('T')[0] ?? '';
|
|
414
|
+
const dateDir = join(config.rawDir, now);
|
|
415
|
+
mkdirSync(dateDir, { recursive: true });
|
|
416
|
+
const dest = join(dateDir, basename(filename));
|
|
161
417
|
writeFileSync(dest, Buffer.concat(chunks));
|
|
162
|
-
|
|
418
|
+
const autoIngest = req.headers['x-auto-ingest'] !== 'false';
|
|
419
|
+
if (autoIngest) {
|
|
420
|
+
try {
|
|
421
|
+
const { ingestSource } = await import('../core/ingest.js');
|
|
422
|
+
const { createProviderFromUserConfig } = await import('../providers/index.js');
|
|
423
|
+
const { loadConfig } = await import('../core/config.js');
|
|
424
|
+
const userConfig = loadConfig(config.configPath);
|
|
425
|
+
const provider = createProviderFromUserConfig(userConfig);
|
|
426
|
+
const result = await ingestSource(dest, config, provider, { verbose: false });
|
|
427
|
+
res.json({ status: 'ingested', path: dest, ...result });
|
|
428
|
+
}
|
|
429
|
+
catch (err) {
|
|
430
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
431
|
+
res.json({ status: 'uploaded', path: dest, ingestError: msg });
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
else {
|
|
435
|
+
res.json({ status: 'uploaded', path: dest });
|
|
436
|
+
}
|
|
163
437
|
});
|
|
164
438
|
});
|
|
165
|
-
// API: raw files list
|
|
439
|
+
// API: raw files list (recursive through date-stamped subdirectories)
|
|
166
440
|
app.get('/api/raw', (_req, res) => {
|
|
167
441
|
try {
|
|
168
442
|
const rawDir = config.rawDir;
|
|
@@ -170,33 +444,280 @@ export function createServer(vaultRoot, port) {
|
|
|
170
444
|
res.json([]);
|
|
171
445
|
return;
|
|
172
446
|
}
|
|
173
|
-
const files =
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
447
|
+
const files = [];
|
|
448
|
+
function walkRaw(dir, prefix) {
|
|
449
|
+
for (const entry of readdirSync(dir)) {
|
|
450
|
+
if (entry.startsWith('.') || entry.endsWith('.meta.json'))
|
|
451
|
+
continue;
|
|
452
|
+
const full = join(dir, entry);
|
|
453
|
+
const stat = statSync(full);
|
|
454
|
+
if (stat.isDirectory()) {
|
|
455
|
+
walkRaw(full, prefix ? `${prefix}/${entry}` : entry);
|
|
456
|
+
}
|
|
457
|
+
else {
|
|
458
|
+
files.push({
|
|
459
|
+
name: prefix ? `${prefix}/${entry}` : entry,
|
|
460
|
+
path: full,
|
|
461
|
+
size: stat.size,
|
|
462
|
+
modified: stat.mtime.toISOString(),
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
walkRaw(rawDir, '');
|
|
180
468
|
res.json(files);
|
|
181
469
|
}
|
|
182
470
|
catch (err) {
|
|
183
471
|
res.status(500).json({ error: 'Failed to list raw files' });
|
|
184
472
|
}
|
|
185
473
|
});
|
|
474
|
+
// API: read raw file content (for preview)
|
|
475
|
+
app.get('/api/raw/view/:filename', (req, res) => {
|
|
476
|
+
try {
|
|
477
|
+
const filename = req.params['filename'];
|
|
478
|
+
if (!filename) {
|
|
479
|
+
res.status(400).json({ error: 'Missing filename' });
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
const decoded = decodeURIComponent(filename);
|
|
483
|
+
const fullPath = join(config.rawDir, decoded);
|
|
484
|
+
const resolved = resolve(fullPath);
|
|
485
|
+
if (!resolved.startsWith(resolve(config.rawDir))) {
|
|
486
|
+
res.status(403).json({ error: 'Access denied' });
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
if (!existsSync(resolved)) {
|
|
490
|
+
res.status(404).json({ error: 'File not found' });
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
const ext = extname(resolved).toLowerCase();
|
|
494
|
+
const imageExts = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'];
|
|
495
|
+
if (imageExts.includes(ext)) {
|
|
496
|
+
res.sendFile(resolved);
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
const textExts = ['.md', '.txt', '.csv', '.json', '.yaml', '.yml', '.xml', '.html', '.htm', '.ts', '.js', '.py', '.go', '.rs'];
|
|
500
|
+
if (textExts.includes(ext) || ext === '') {
|
|
501
|
+
const content = readFileSync(resolved, 'utf-8');
|
|
502
|
+
res.json({ type: 'text', content, filename: decoded });
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
res.json({ type: 'binary', filename: decoded, size: statSync(resolved).size, message: 'Binary file — extract with wikimem ingest' });
|
|
506
|
+
}
|
|
507
|
+
catch (err) {
|
|
508
|
+
res.status(500).json({ error: 'Failed to read raw file' });
|
|
509
|
+
}
|
|
510
|
+
});
|
|
511
|
+
// API: file tree (wiki + raw hierarchy)
|
|
512
|
+
app.get('/api/tree', (_req, res) => {
|
|
513
|
+
try {
|
|
514
|
+
function buildWikiTree(dir, relPath) {
|
|
515
|
+
const nodes = [];
|
|
516
|
+
if (!existsSync(dir))
|
|
517
|
+
return nodes;
|
|
518
|
+
const entries = readdirSync(dir).sort();
|
|
519
|
+
for (const entry of entries) {
|
|
520
|
+
if (entry.startsWith('.'))
|
|
521
|
+
continue;
|
|
522
|
+
const full = join(dir, entry);
|
|
523
|
+
const stat = statSync(full);
|
|
524
|
+
const childPath = relPath ? `${relPath}/${entry}` : entry;
|
|
525
|
+
if (stat.isDirectory()) {
|
|
526
|
+
const children = buildWikiTree(full, childPath);
|
|
527
|
+
nodes.push({ name: entry, type: 'dir', path: childPath, children });
|
|
528
|
+
}
|
|
529
|
+
else if (entry.endsWith('.md')) {
|
|
530
|
+
try {
|
|
531
|
+
const page = readWikiPage(full);
|
|
532
|
+
const cat = page.frontmatter['type']
|
|
533
|
+
?? page.frontmatter['category']
|
|
534
|
+
?? 'page';
|
|
535
|
+
nodes.push({ name: entry, type: 'wiki', path: childPath, title: page.title, category: cat });
|
|
536
|
+
}
|
|
537
|
+
catch {
|
|
538
|
+
nodes.push({ name: entry, type: 'wiki', path: childPath, title: entry.replace('.md', '') });
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
return nodes;
|
|
543
|
+
}
|
|
544
|
+
function buildRawTree(dir, relPath) {
|
|
545
|
+
const nodes = [];
|
|
546
|
+
if (!existsSync(dir))
|
|
547
|
+
return nodes;
|
|
548
|
+
const entries = readdirSync(dir).sort();
|
|
549
|
+
for (const entry of entries) {
|
|
550
|
+
if (entry.startsWith('.') || entry.endsWith('.meta.json'))
|
|
551
|
+
continue;
|
|
552
|
+
const full = join(dir, entry);
|
|
553
|
+
const stat = statSync(full);
|
|
554
|
+
const childPath = relPath ? `${relPath}/${entry}` : entry;
|
|
555
|
+
if (stat.isDirectory()) {
|
|
556
|
+
nodes.push({ name: entry, type: 'dir', path: childPath, children: buildRawTree(full, childPath) });
|
|
557
|
+
}
|
|
558
|
+
else {
|
|
559
|
+
nodes.push({ name: entry, type: 'raw', path: childPath, size: stat.size });
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
return nodes;
|
|
563
|
+
}
|
|
564
|
+
res.json({
|
|
565
|
+
wiki: buildWikiTree(config.wikiDir, ''),
|
|
566
|
+
raw: buildRawTree(config.rawDir, ''),
|
|
567
|
+
});
|
|
568
|
+
}
|
|
569
|
+
catch {
|
|
570
|
+
res.status(500).json({ error: 'Failed to build file tree' });
|
|
571
|
+
}
|
|
572
|
+
});
|
|
573
|
+
// API: read config (for settings page)
|
|
574
|
+
app.get('/api/config', async (_req, res) => {
|
|
575
|
+
try {
|
|
576
|
+
const { loadConfig } = await import('../core/config.js');
|
|
577
|
+
const userConfig = loadConfig(config.configPath);
|
|
578
|
+
const safeConfig = { ...userConfig };
|
|
579
|
+
if (safeConfig.api_key)
|
|
580
|
+
safeConfig.api_key = safeConfig.api_key.slice(0, 8) + '…';
|
|
581
|
+
const sc = safeConfig;
|
|
582
|
+
if (sc['gemini_api_key']) {
|
|
583
|
+
sc['gemini_api_key'] = String(sc['gemini_api_key']).slice(0, 8) + '…';
|
|
584
|
+
}
|
|
585
|
+
res.json(safeConfig);
|
|
586
|
+
}
|
|
587
|
+
catch {
|
|
588
|
+
res.json({});
|
|
589
|
+
}
|
|
590
|
+
});
|
|
591
|
+
// API: update config (for settings page)
|
|
592
|
+
app.put('/api/config', async (req, res) => {
|
|
593
|
+
try {
|
|
594
|
+
const { loadConfig } = await import('../core/config.js');
|
|
595
|
+
const YAML = await import('yaml');
|
|
596
|
+
const updates = req.body;
|
|
597
|
+
const current = loadConfig(config.configPath);
|
|
598
|
+
const merged = { ...current, ...updates };
|
|
599
|
+
writeFileSync(config.configPath, YAML.stringify(merged), 'utf-8');
|
|
600
|
+
res.json({ status: 'saved' });
|
|
601
|
+
}
|
|
602
|
+
catch (err) {
|
|
603
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
604
|
+
res.status(500).json({ error: `Failed to save config: ${msg}` });
|
|
605
|
+
}
|
|
606
|
+
});
|
|
607
|
+
// API: test provider connection
|
|
608
|
+
app.post('/api/config/test-provider', async (req, res) => {
|
|
609
|
+
try {
|
|
610
|
+
const { provider: providerName, apiKey } = req.body;
|
|
611
|
+
if (providerName === 'claude-code') {
|
|
612
|
+
const { isClaudeCodeAvailable } = await import('../core/claude-code.js');
|
|
613
|
+
if (isClaudeCodeAvailable()) {
|
|
614
|
+
res.json({ status: 'ok', provider: 'claude-code' });
|
|
615
|
+
}
|
|
616
|
+
else {
|
|
617
|
+
res.json({ status: 'error', error: 'Claude Code CLI not found in PATH' });
|
|
618
|
+
}
|
|
619
|
+
return;
|
|
620
|
+
}
|
|
621
|
+
if (!apiKey) {
|
|
622
|
+
res.status(400).json({ error: 'Missing apiKey' });
|
|
623
|
+
return;
|
|
624
|
+
}
|
|
625
|
+
if (providerName === 'claude' || !providerName) {
|
|
626
|
+
const { default: Anthropic } = await import('@anthropic-ai/sdk');
|
|
627
|
+
const client = new Anthropic({ apiKey });
|
|
628
|
+
await client.messages.create({
|
|
629
|
+
model: 'claude-3-haiku-20240307',
|
|
630
|
+
max_tokens: 10,
|
|
631
|
+
messages: [{ role: 'user', content: 'ping' }],
|
|
632
|
+
});
|
|
633
|
+
res.json({ status: 'ok', provider: 'claude' });
|
|
634
|
+
}
|
|
635
|
+
else {
|
|
636
|
+
res.json({ status: 'ok', provider: providerName, note: 'Skipped validation' });
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
catch (err) {
|
|
640
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
641
|
+
res.json({ status: 'error', error: msg });
|
|
642
|
+
}
|
|
643
|
+
});
|
|
644
|
+
// API: Claude Code CLI availability
|
|
645
|
+
app.get('/api/claude-code/status', async (_req, res) => {
|
|
646
|
+
try {
|
|
647
|
+
const { isClaudeCodeAvailable, getClaudeCodePath } = await import('../core/claude-code.js');
|
|
648
|
+
res.json({ available: isClaudeCodeAvailable(), path: getClaudeCodePath() });
|
|
649
|
+
}
|
|
650
|
+
catch {
|
|
651
|
+
res.json({ available: false, path: null });
|
|
652
|
+
}
|
|
653
|
+
});
|
|
654
|
+
// API: search pages
|
|
655
|
+
app.get('/api/search', (req, res) => {
|
|
656
|
+
try {
|
|
657
|
+
const q = (req.query['q'] ?? '').toLowerCase().trim();
|
|
658
|
+
const limit = parseInt(req.query['limit']) || 20;
|
|
659
|
+
if (!q) {
|
|
660
|
+
res.json({ results: [] });
|
|
661
|
+
return;
|
|
662
|
+
}
|
|
663
|
+
const pages = listWikiPages(config.wikiDir);
|
|
664
|
+
const results = [];
|
|
665
|
+
for (const pagePath of pages) {
|
|
666
|
+
const page = readWikiPage(pagePath);
|
|
667
|
+
const titleMatch = page.title.toLowerCase().includes(q);
|
|
668
|
+
const tagMatch = (page.frontmatter['tags'] ?? []).some((t) => t.toLowerCase().includes(q));
|
|
669
|
+
let snippet;
|
|
670
|
+
let contentMatch = false;
|
|
671
|
+
if (!titleMatch) {
|
|
672
|
+
const bodyLower = page.content.toLowerCase();
|
|
673
|
+
const idx = bodyLower.indexOf(q);
|
|
674
|
+
if (idx >= 0) {
|
|
675
|
+
contentMatch = true;
|
|
676
|
+
const start = Math.max(0, idx - 40);
|
|
677
|
+
const end = Math.min(page.content.length, idx + q.length + 60);
|
|
678
|
+
snippet = (start > 0 ? '…' : '') + page.content.substring(start, end).replace(/\n/g, ' ') + (end < page.content.length ? '…' : '');
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
if (titleMatch || tagMatch || contentMatch) {
|
|
682
|
+
const category = page.frontmatter['category'] ?? 'uncategorized';
|
|
683
|
+
results.push({ title: page.title, category, wordCount: page.wordCount, snippet });
|
|
684
|
+
}
|
|
685
|
+
if (results.length >= limit)
|
|
686
|
+
break;
|
|
687
|
+
}
|
|
688
|
+
res.json({ results });
|
|
689
|
+
}
|
|
690
|
+
catch {
|
|
691
|
+
res.json({ results: [] });
|
|
692
|
+
}
|
|
693
|
+
});
|
|
694
|
+
// API: wiki history (audit trail)
|
|
695
|
+
app.get('/api/history', async (_req, res) => {
|
|
696
|
+
try {
|
|
697
|
+
const { listSnapshots } = await import('../core/history.js');
|
|
698
|
+
const entries = listSnapshots(config);
|
|
699
|
+
res.json(entries);
|
|
700
|
+
}
|
|
701
|
+
catch {
|
|
702
|
+
res.json([]);
|
|
703
|
+
}
|
|
704
|
+
});
|
|
186
705
|
// API: query the wiki
|
|
187
706
|
app.post('/api/query', async (req, res) => {
|
|
188
707
|
try {
|
|
189
|
-
const { question, provider: providerName } = req.body;
|
|
708
|
+
const { question, provider: providerName, model: modelName } = req.body;
|
|
190
709
|
if (!question) {
|
|
191
710
|
res.status(400).json({ error: 'Missing question field' });
|
|
192
711
|
return;
|
|
193
712
|
}
|
|
194
|
-
// Dynamic import to avoid circular deps
|
|
195
713
|
const { queryWiki } = await import('../core/query.js');
|
|
196
|
-
const {
|
|
714
|
+
const { createProviderFromUserConfig } = await import('../providers/index.js');
|
|
197
715
|
const { loadConfig } = await import('../core/config.js');
|
|
198
716
|
const userConfig = loadConfig(config.configPath);
|
|
199
|
-
const provider =
|
|
717
|
+
const provider = createProviderFromUserConfig(userConfig, {
|
|
718
|
+
providerOverride: providerName,
|
|
719
|
+
model: modelName,
|
|
720
|
+
});
|
|
200
721
|
const result = await queryWiki(question, config, provider, { fileBack: false });
|
|
201
722
|
res.json(result);
|
|
202
723
|
}
|
|
@@ -214,10 +735,10 @@ export function createServer(vaultRoot, port) {
|
|
|
214
735
|
return;
|
|
215
736
|
}
|
|
216
737
|
const { ingestSource } = await import('../core/ingest.js');
|
|
217
|
-
const {
|
|
738
|
+
const { createProviderFromUserConfig } = await import('../providers/index.js');
|
|
218
739
|
const { loadConfig } = await import('../core/config.js');
|
|
219
740
|
const userConfig = loadConfig(config.configPath);
|
|
220
|
-
const provider =
|
|
741
|
+
const provider = createProviderFromUserConfig(userConfig);
|
|
221
742
|
const result = await ingestSource(source, config, provider, { verbose: false });
|
|
222
743
|
res.json(result);
|
|
223
744
|
}
|
|
@@ -226,6 +747,1557 @@ export function createServer(vaultRoot, port) {
|
|
|
226
747
|
res.status(500).json({ error: `Ingest failed: ${msg}` });
|
|
227
748
|
}
|
|
228
749
|
});
|
|
750
|
+
// === GIT API ENDPOINTS ===
|
|
751
|
+
// API: git status (lightweight by default, add ?full=true for file list)
|
|
752
|
+
app.get('/api/git/status', async (req, res) => {
|
|
753
|
+
try {
|
|
754
|
+
const { isGitRepo, getGitStatus, getBranches } = await import('../core/git.js');
|
|
755
|
+
const isRepo = await isGitRepo(config.root);
|
|
756
|
+
if (!isRepo) {
|
|
757
|
+
res.json({ initialized: false });
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
760
|
+
const includeFull = req.query['full'] === 'true';
|
|
761
|
+
const status = await getGitStatus(config.root);
|
|
762
|
+
const branches = await getBranches(config.root);
|
|
763
|
+
const result = {
|
|
764
|
+
initialized: true,
|
|
765
|
+
branch: branches.current,
|
|
766
|
+
branches: branches.all,
|
|
767
|
+
isDetached: branches.isDetached,
|
|
768
|
+
changedCount: status?.files?.length ?? 0,
|
|
769
|
+
isClean: status?.isClean() ?? true,
|
|
770
|
+
};
|
|
771
|
+
if (includeFull) {
|
|
772
|
+
result['files'] = status?.files ?? [];
|
|
773
|
+
}
|
|
774
|
+
res.json(result);
|
|
775
|
+
}
|
|
776
|
+
catch (err) {
|
|
777
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
778
|
+
res.status(500).json({ error: `Git status failed: ${msg}` });
|
|
779
|
+
}
|
|
780
|
+
});
|
|
781
|
+
// API: initialize git repo
|
|
782
|
+
app.post('/api/git/init', async (_req, res) => {
|
|
783
|
+
try {
|
|
784
|
+
const { initGitRepo } = await import('../core/git.js');
|
|
785
|
+
const result = await initGitRepo(config);
|
|
786
|
+
res.json(result);
|
|
787
|
+
}
|
|
788
|
+
catch (err) {
|
|
789
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
790
|
+
res.status(500).json({ error: `Git init failed: ${msg}` });
|
|
791
|
+
}
|
|
792
|
+
});
|
|
793
|
+
// API: git log (audit trail) with optional wiki-only filtering
|
|
794
|
+
app.get('/api/git/log', async (req, res) => {
|
|
795
|
+
try {
|
|
796
|
+
const { getGitLog, isGitRepo } = await import('../core/git.js');
|
|
797
|
+
if (!(await isGitRepo(config.root))) {
|
|
798
|
+
res.json([]);
|
|
799
|
+
return;
|
|
800
|
+
}
|
|
801
|
+
const limit = parseInt(req.query['limit']) || 50;
|
|
802
|
+
const wikiOnly = req.query['wikiOnly'] !== 'false';
|
|
803
|
+
const search = req.query['search'] || undefined;
|
|
804
|
+
const log = await getGitLog(config.root, limit, { wikiOnly, search });
|
|
805
|
+
res.json(log);
|
|
806
|
+
}
|
|
807
|
+
catch (err) {
|
|
808
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
809
|
+
res.status(500).json({ error: `Git log failed: ${msg}` });
|
|
810
|
+
}
|
|
811
|
+
});
|
|
812
|
+
// API: git diff for a specific commit
|
|
813
|
+
app.get('/api/git/diff/:hash', async (req, res) => {
|
|
814
|
+
try {
|
|
815
|
+
const hash = req.params['hash'];
|
|
816
|
+
if (!hash) {
|
|
817
|
+
res.status(400).json({ error: 'Missing hash' });
|
|
818
|
+
return;
|
|
819
|
+
}
|
|
820
|
+
const { getGitDiff, isGitRepo } = await import('../core/git.js');
|
|
821
|
+
if (!(await isGitRepo(config.root))) {
|
|
822
|
+
res.json({ diff: '', stats: [] });
|
|
823
|
+
return;
|
|
824
|
+
}
|
|
825
|
+
const result = await getGitDiff(config.root, hash);
|
|
826
|
+
res.json(result);
|
|
827
|
+
}
|
|
828
|
+
catch (err) {
|
|
829
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
830
|
+
res.status(500).json({ error: `Git diff failed: ${msg}` });
|
|
831
|
+
}
|
|
832
|
+
});
|
|
833
|
+
// API: create branch (optionally from a specific commit hash)
|
|
834
|
+
app.post('/api/git/branch', async (req, res) => {
|
|
835
|
+
try {
|
|
836
|
+
const { name, fromHash } = req.body;
|
|
837
|
+
if (!name) {
|
|
838
|
+
res.status(400).json({ error: 'Missing branch name' });
|
|
839
|
+
return;
|
|
840
|
+
}
|
|
841
|
+
const { createBranch } = await import('../core/git.js');
|
|
842
|
+
const result = await createBranch(config.root, name, fromHash);
|
|
843
|
+
res.json(result);
|
|
844
|
+
}
|
|
845
|
+
catch (err) {
|
|
846
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
847
|
+
res.status(500).json({ error: `Create branch failed: ${msg}` });
|
|
848
|
+
}
|
|
849
|
+
});
|
|
850
|
+
// API: switch branch
|
|
851
|
+
app.post('/api/git/checkout', async (req, res) => {
|
|
852
|
+
try {
|
|
853
|
+
const { branch } = req.body;
|
|
854
|
+
if (!branch) {
|
|
855
|
+
res.status(400).json({ error: 'Missing branch name' });
|
|
856
|
+
return;
|
|
857
|
+
}
|
|
858
|
+
const { switchBranch } = await import('../core/git.js');
|
|
859
|
+
const result = await switchBranch(config.root, branch);
|
|
860
|
+
res.json(result);
|
|
861
|
+
}
|
|
862
|
+
catch (err) {
|
|
863
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
864
|
+
res.status(500).json({ error: `Checkout failed: ${msg}` });
|
|
865
|
+
}
|
|
866
|
+
});
|
|
867
|
+
// API: create tag (milestone)
|
|
868
|
+
app.post('/api/git/tag', async (req, res) => {
|
|
869
|
+
try {
|
|
870
|
+
const { name, message: tagMsg } = req.body;
|
|
871
|
+
if (!name) {
|
|
872
|
+
res.status(400).json({ error: 'Missing tag name' });
|
|
873
|
+
return;
|
|
874
|
+
}
|
|
875
|
+
const { createTag } = await import('../core/git.js');
|
|
876
|
+
const result = await createTag(config.root, name, tagMsg);
|
|
877
|
+
res.json(result);
|
|
878
|
+
}
|
|
879
|
+
catch (err) {
|
|
880
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
881
|
+
res.status(500).json({ error: `Create tag failed: ${msg}` });
|
|
882
|
+
}
|
|
883
|
+
});
|
|
884
|
+
// API: restore to a specific commit
|
|
885
|
+
app.post('/api/git/restore', async (req, res) => {
|
|
886
|
+
try {
|
|
887
|
+
const { hash } = req.body;
|
|
888
|
+
if (!hash) {
|
|
889
|
+
res.status(400).json({ error: 'Missing commit hash' });
|
|
890
|
+
return;
|
|
891
|
+
}
|
|
892
|
+
const { restoreToCommit } = await import('../core/git.js');
|
|
893
|
+
const result = await restoreToCommit(config.root, hash);
|
|
894
|
+
res.json(result);
|
|
895
|
+
}
|
|
896
|
+
catch (err) {
|
|
897
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
898
|
+
res.status(500).json({ error: `Restore failed: ${msg}` });
|
|
899
|
+
}
|
|
900
|
+
});
|
|
901
|
+
// API: get file tree at a specific commit (for time-lapse)
|
|
902
|
+
app.get('/api/git/tree/:hash', async (req, res) => {
|
|
903
|
+
try {
|
|
904
|
+
const hash = req.params['hash'];
|
|
905
|
+
if (!hash) {
|
|
906
|
+
res.status(400).json({ error: 'Missing hash' });
|
|
907
|
+
return;
|
|
908
|
+
}
|
|
909
|
+
const { getTreeAtCommit, isGitRepo } = await import('../core/git.js');
|
|
910
|
+
if (!(await isGitRepo(config.root))) {
|
|
911
|
+
res.json([]);
|
|
912
|
+
return;
|
|
913
|
+
}
|
|
914
|
+
const tree = await getTreeAtCommit(config.root, hash);
|
|
915
|
+
res.json(tree);
|
|
916
|
+
}
|
|
917
|
+
catch (err) {
|
|
918
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
919
|
+
res.status(500).json({ error: `Get tree failed: ${msg}` });
|
|
920
|
+
}
|
|
921
|
+
});
|
|
922
|
+
// API: batch fetch file trees for multiple commits (time-lapse pre-fetch)
|
|
923
|
+
app.post('/api/git/trees/batch', async (req, res) => {
|
|
924
|
+
try {
|
|
925
|
+
const { hashes } = req.body;
|
|
926
|
+
if (!hashes || !Array.isArray(hashes) || hashes.length === 0) {
|
|
927
|
+
res.status(400).json({ error: 'Missing or empty hashes array' });
|
|
928
|
+
return;
|
|
929
|
+
}
|
|
930
|
+
const capped = hashes.slice(0, 500);
|
|
931
|
+
const { getTreeAtCommit, isGitRepo } = await import('../core/git.js');
|
|
932
|
+
if (!(await isGitRepo(config.root))) {
|
|
933
|
+
res.json({});
|
|
934
|
+
return;
|
|
935
|
+
}
|
|
936
|
+
const result = {};
|
|
937
|
+
const concurrency = 10;
|
|
938
|
+
for (let i = 0; i < capped.length; i += concurrency) {
|
|
939
|
+
const batch = capped.slice(i, i + concurrency);
|
|
940
|
+
await Promise.all(batch.map(async (hash) => {
|
|
941
|
+
try {
|
|
942
|
+
result[hash] = await getTreeAtCommit(config.root, hash);
|
|
943
|
+
}
|
|
944
|
+
catch {
|
|
945
|
+
result[hash] = [];
|
|
946
|
+
}
|
|
947
|
+
}));
|
|
948
|
+
}
|
|
949
|
+
res.json(result);
|
|
950
|
+
}
|
|
951
|
+
catch (err) {
|
|
952
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
953
|
+
res.status(500).json({ error: `Batch tree fetch failed: ${msg}` });
|
|
954
|
+
}
|
|
955
|
+
});
|
|
956
|
+
// API: graph data at a specific commit (for time-lapse graph animation)
|
|
957
|
+
app.get('/api/git/graph/:hash', async (req, res) => {
|
|
958
|
+
try {
|
|
959
|
+
const hash = req.params['hash'];
|
|
960
|
+
if (!hash) {
|
|
961
|
+
res.status(400).json({ error: 'Missing hash' });
|
|
962
|
+
return;
|
|
963
|
+
}
|
|
964
|
+
const { getGraphAtCommit, isGitRepo } = await import('../core/git.js');
|
|
965
|
+
if (!(await isGitRepo(config.root))) {
|
|
966
|
+
res.json({ nodes: [], links: [] });
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
const graph = await getGraphAtCommit(config.root, hash);
|
|
970
|
+
res.json(graph);
|
|
971
|
+
}
|
|
972
|
+
catch (err) {
|
|
973
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
974
|
+
res.status(500).json({ error: `Get graph at commit failed: ${msg}` });
|
|
975
|
+
}
|
|
976
|
+
});
|
|
977
|
+
// API: batch graph snapshots for time-lapse (pre-fetch all commit graphs)
|
|
978
|
+
app.post('/api/git/graph-batch', async (req, res) => {
|
|
979
|
+
try {
|
|
980
|
+
const { hashes } = req.body;
|
|
981
|
+
if (!hashes?.length) {
|
|
982
|
+
res.status(400).json({ error: 'Missing hashes array' });
|
|
983
|
+
return;
|
|
984
|
+
}
|
|
985
|
+
const { getGraphAtCommit, isGitRepo } = await import('../core/git.js');
|
|
986
|
+
if (!(await isGitRepo(config.root))) {
|
|
987
|
+
res.json({});
|
|
988
|
+
return;
|
|
989
|
+
}
|
|
990
|
+
const results = {};
|
|
991
|
+
for (const hash of hashes.slice(0, 100)) {
|
|
992
|
+
results[hash] = await getGraphAtCommit(config.root, hash);
|
|
993
|
+
}
|
|
994
|
+
res.json(results);
|
|
995
|
+
}
|
|
996
|
+
catch (err) {
|
|
997
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
998
|
+
res.status(500).json({ error: `Batch graph failed: ${msg}` });
|
|
999
|
+
}
|
|
1000
|
+
});
|
|
1001
|
+
// API: list branches (dedicated endpoint)
|
|
1002
|
+
app.get('/api/git/branches', async (_req, res) => {
|
|
1003
|
+
try {
|
|
1004
|
+
const { getBranches, isGitRepo } = await import('../core/git.js');
|
|
1005
|
+
if (!(await isGitRepo(config.root))) {
|
|
1006
|
+
res.json({ current: '', branches: [], isDetached: false });
|
|
1007
|
+
return;
|
|
1008
|
+
}
|
|
1009
|
+
const info = await getBranches(config.root);
|
|
1010
|
+
res.json({ current: info.current, branches: info.all, isDetached: info.isDetached });
|
|
1011
|
+
}
|
|
1012
|
+
catch (err) {
|
|
1013
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1014
|
+
res.status(500).json({ error: `Failed to list branches: ${msg}` });
|
|
1015
|
+
}
|
|
1016
|
+
});
|
|
1017
|
+
// API: create session branch (auto-naming with wiki/session-<timestamp> or custom name)
|
|
1018
|
+
app.post('/api/git/branches', async (req, res) => {
|
|
1019
|
+
try {
|
|
1020
|
+
const { name, session } = req.body;
|
|
1021
|
+
const { createBranch } = await import('../core/git.js');
|
|
1022
|
+
let branchName = name;
|
|
1023
|
+
if (!branchName || session) {
|
|
1024
|
+
const ts = new Date().toISOString().replace(/[:.]/g, '-').substring(0, 19);
|
|
1025
|
+
branchName = name ? `wiki/${name}` : `wiki/session-${ts}`;
|
|
1026
|
+
}
|
|
1027
|
+
const result = await createBranch(config.root, branchName);
|
|
1028
|
+
res.json(result);
|
|
1029
|
+
}
|
|
1030
|
+
catch (err) {
|
|
1031
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1032
|
+
res.status(500).json({ error: `Create branch failed: ${msg}` });
|
|
1033
|
+
}
|
|
1034
|
+
});
|
|
1035
|
+
// API: push current branch to remote
|
|
1036
|
+
app.post('/api/git/push', async (req, res) => {
|
|
1037
|
+
try {
|
|
1038
|
+
const { remote } = req.body;
|
|
1039
|
+
const { pushBranch } = await import('../core/git.js');
|
|
1040
|
+
const result = await pushBranch(config.root, remote || 'origin');
|
|
1041
|
+
res.json(result);
|
|
1042
|
+
}
|
|
1043
|
+
catch (err) {
|
|
1044
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1045
|
+
res.status(500).json({ error: `Push failed: ${msg}` });
|
|
1046
|
+
}
|
|
1047
|
+
});
|
|
1048
|
+
// API: submit for review (PR workflow: diff summary, auto-branch, push)
|
|
1049
|
+
app.post('/api/git/pr', async (req, res) => {
|
|
1050
|
+
try {
|
|
1051
|
+
const { description } = req.body;
|
|
1052
|
+
const { submitForReview } = await import('../core/git.js');
|
|
1053
|
+
const result = await submitForReview(config.root, description);
|
|
1054
|
+
res.json(result);
|
|
1055
|
+
}
|
|
1056
|
+
catch (err) {
|
|
1057
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1058
|
+
res.status(500).json({ error: `Submit for review failed: ${msg}` });
|
|
1059
|
+
}
|
|
1060
|
+
});
|
|
1061
|
+
// API: get diff summary (working branch vs base)
|
|
1062
|
+
app.get('/api/git/diff-summary', async (req, res) => {
|
|
1063
|
+
try {
|
|
1064
|
+
const baseBranch = req.query['base'];
|
|
1065
|
+
const { getDiffSummary, isGitRepo } = await import('../core/git.js');
|
|
1066
|
+
if (!(await isGitRepo(config.root))) {
|
|
1067
|
+
res.json({ filesAdded: [], filesModified: [], filesDeleted: [], totalAdditions: 0, totalDeletions: 0 });
|
|
1068
|
+
return;
|
|
1069
|
+
}
|
|
1070
|
+
const summary = await getDiffSummary(config.root, baseBranch);
|
|
1071
|
+
res.json(summary);
|
|
1072
|
+
}
|
|
1073
|
+
catch (err) {
|
|
1074
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1075
|
+
res.status(500).json({ error: `Diff summary failed: ${msg}` });
|
|
1076
|
+
}
|
|
1077
|
+
});
|
|
1078
|
+
// API: parsed diff for a commit (rich diff view with per-file hunks)
|
|
1079
|
+
app.get('/api/git/diff/:hash/parsed', async (req, res) => {
|
|
1080
|
+
try {
|
|
1081
|
+
const hash = req.params['hash'];
|
|
1082
|
+
if (!hash) {
|
|
1083
|
+
res.status(400).json({ error: 'Missing hash' });
|
|
1084
|
+
return;
|
|
1085
|
+
}
|
|
1086
|
+
const { getParsedDiff, isGitRepo } = await import('../core/git.js');
|
|
1087
|
+
if (!(await isGitRepo(config.root))) {
|
|
1088
|
+
res.json({ files: [], stats: { additions: 0, deletions: 0, filesChanged: 0 } });
|
|
1089
|
+
return;
|
|
1090
|
+
}
|
|
1091
|
+
const result = await getParsedDiff(config.root, hash);
|
|
1092
|
+
res.json(result);
|
|
1093
|
+
}
|
|
1094
|
+
catch (err) {
|
|
1095
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1096
|
+
res.status(500).json({ error: `Parsed diff failed: ${msg}` });
|
|
1097
|
+
}
|
|
1098
|
+
});
|
|
1099
|
+
// API: migrate .wikimem/history snapshots to git commits
|
|
1100
|
+
app.post('/api/git/migrate-history', async (_req, res) => {
|
|
1101
|
+
try {
|
|
1102
|
+
const { migrateHistoryToGit } = await import('../core/git.js');
|
|
1103
|
+
const result = await migrateHistoryToGit(config);
|
|
1104
|
+
res.json(result);
|
|
1105
|
+
}
|
|
1106
|
+
catch (err) {
|
|
1107
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1108
|
+
res.status(500).json({ error: `Migration failed: ${msg}` });
|
|
1109
|
+
}
|
|
1110
|
+
});
|
|
1111
|
+
// === RAW FILE PREVIEW ENDPOINTS ===
|
|
1112
|
+
// API: serve raw file with correct content-type (for PDF, images, video, audio)
|
|
1113
|
+
app.get('/api/raw/file', (req, res) => {
|
|
1114
|
+
try {
|
|
1115
|
+
let filePath = String(req.query['path'] ?? '');
|
|
1116
|
+
if (filePath.startsWith('/'))
|
|
1117
|
+
filePath = filePath.slice(1);
|
|
1118
|
+
if (!filePath) {
|
|
1119
|
+
res.status(400).json({ error: 'Missing path query param' });
|
|
1120
|
+
return;
|
|
1121
|
+
}
|
|
1122
|
+
const decoded = decodeURIComponent(filePath);
|
|
1123
|
+
const fullPath = join(config.rawDir, decoded);
|
|
1124
|
+
const resolved = resolve(fullPath);
|
|
1125
|
+
if (!resolved.startsWith(resolve(config.rawDir))) {
|
|
1126
|
+
res.status(403).json({ error: 'Access denied' });
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
if (!existsSync(resolved)) {
|
|
1130
|
+
res.status(404).json({ error: 'File not found' });
|
|
1131
|
+
return;
|
|
1132
|
+
}
|
|
1133
|
+
const ext = extname(resolved).toLowerCase();
|
|
1134
|
+
const mimeTypes = {
|
|
1135
|
+
'.pdf': 'application/pdf',
|
|
1136
|
+
'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
|
1137
|
+
'.png': 'image/png', '.gif': 'image/gif',
|
|
1138
|
+
'.webp': 'image/webp', '.svg': 'image/svg+xml',
|
|
1139
|
+
'.mp4': 'video/mp4', '.webm': 'video/webm', '.mov': 'video/quicktime',
|
|
1140
|
+
'.mp3': 'audio/mpeg', '.wav': 'audio/wav', '.ogg': 'audio/ogg',
|
|
1141
|
+
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
1142
|
+
'.xls': 'application/vnd.ms-excel',
|
|
1143
|
+
'.csv': 'text/csv',
|
|
1144
|
+
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
1145
|
+
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
1146
|
+
};
|
|
1147
|
+
const mime = mimeTypes[ext] ?? 'application/octet-stream';
|
|
1148
|
+
res.setHeader('Accept-Ranges', 'bytes');
|
|
1149
|
+
res.setHeader('Content-Type', mime);
|
|
1150
|
+
res.sendFile(resolved);
|
|
1151
|
+
}
|
|
1152
|
+
catch (err) {
|
|
1153
|
+
res.status(500).json({ error: 'Failed to serve file' });
|
|
1154
|
+
}
|
|
1155
|
+
});
|
|
1156
|
+
// API: raw file metadata (for preview header)
|
|
1157
|
+
app.get('/api/raw/meta', (req, res) => {
|
|
1158
|
+
try {
|
|
1159
|
+
let filePath = String(req.query['path'] ?? '');
|
|
1160
|
+
if (filePath.startsWith('/'))
|
|
1161
|
+
filePath = filePath.slice(1);
|
|
1162
|
+
if (!filePath) {
|
|
1163
|
+
res.status(400).json({ error: 'Missing path query param' });
|
|
1164
|
+
return;
|
|
1165
|
+
}
|
|
1166
|
+
const decoded = decodeURIComponent(filePath);
|
|
1167
|
+
const fullPath = join(config.rawDir, decoded);
|
|
1168
|
+
const resolved = resolve(fullPath);
|
|
1169
|
+
if (!resolved.startsWith(resolve(config.rawDir))) {
|
|
1170
|
+
res.status(403).json({ error: 'Access denied' });
|
|
1171
|
+
return;
|
|
1172
|
+
}
|
|
1173
|
+
if (!existsSync(resolved)) {
|
|
1174
|
+
res.status(404).json({ error: 'File not found' });
|
|
1175
|
+
return;
|
|
1176
|
+
}
|
|
1177
|
+
const stat = statSync(resolved);
|
|
1178
|
+
const ext = extname(resolved).toLowerCase();
|
|
1179
|
+
const imageExts = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'];
|
|
1180
|
+
const videoExts = ['.mp4', '.webm', '.mov'];
|
|
1181
|
+
const audioExts = ['.mp3', '.wav', '.ogg'];
|
|
1182
|
+
const pdfExts = ['.pdf'];
|
|
1183
|
+
const spreadsheetExts = ['.csv', '.xlsx', '.xls'];
|
|
1184
|
+
const textExts = ['.md', '.txt', '.json', '.yaml', '.yml', '.xml', '.html', '.htm', '.ts', '.js', '.py', '.go', '.rs', '.toml', '.ini', '.cfg', '.env', '.sh', '.bash', '.zsh'];
|
|
1185
|
+
const documentExts = ['.docx', '.doc', '.pptx', '.ppt', '.rtf', '.odt', '.odp'];
|
|
1186
|
+
let previewType;
|
|
1187
|
+
if (imageExts.includes(ext))
|
|
1188
|
+
previewType = 'image';
|
|
1189
|
+
else if (videoExts.includes(ext))
|
|
1190
|
+
previewType = 'video';
|
|
1191
|
+
else if (audioExts.includes(ext))
|
|
1192
|
+
previewType = 'audio';
|
|
1193
|
+
else if (pdfExts.includes(ext))
|
|
1194
|
+
previewType = 'pdf';
|
|
1195
|
+
else if (spreadsheetExts.includes(ext))
|
|
1196
|
+
previewType = 'spreadsheet';
|
|
1197
|
+
else if (textExts.includes(ext))
|
|
1198
|
+
previewType = 'text';
|
|
1199
|
+
else if (documentExts.includes(ext))
|
|
1200
|
+
previewType = 'document';
|
|
1201
|
+
else
|
|
1202
|
+
previewType = 'binary';
|
|
1203
|
+
// Find wiki pages that were generated from this raw file
|
|
1204
|
+
const linkedPages = [];
|
|
1205
|
+
try {
|
|
1206
|
+
const pages = listWikiPages(config.wikiDir);
|
|
1207
|
+
for (const pagePath of pages) {
|
|
1208
|
+
const page = readWikiPage(pagePath);
|
|
1209
|
+
const sources = page.frontmatter['sources'];
|
|
1210
|
+
if (sources?.some(s => s.includes(decoded) || s.includes(basename(decoded)))) {
|
|
1211
|
+
linkedPages.push({
|
|
1212
|
+
title: page.title,
|
|
1213
|
+
slug: basename(pagePath, '.md'),
|
|
1214
|
+
});
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
catch { /* non-fatal */ }
|
|
1219
|
+
res.json({
|
|
1220
|
+
name: basename(decoded),
|
|
1221
|
+
path: decoded,
|
|
1222
|
+
size: stat.size,
|
|
1223
|
+
modified: stat.mtime.toISOString(),
|
|
1224
|
+
created: stat.birthtime.toISOString(),
|
|
1225
|
+
extension: ext,
|
|
1226
|
+
previewType,
|
|
1227
|
+
linkedWikiPages: linkedPages,
|
|
1228
|
+
});
|
|
1229
|
+
}
|
|
1230
|
+
catch (err) {
|
|
1231
|
+
res.status(500).json({ error: 'Failed to get file metadata' });
|
|
1232
|
+
}
|
|
1233
|
+
});
|
|
1234
|
+
// ─── Raw File Operations (Cursor-Parity) ──────────────────────────────────
|
|
1235
|
+
// POST /api/raw/rename — rename a raw file
|
|
1236
|
+
app.post('/api/raw/rename', async (req, res) => {
|
|
1237
|
+
try {
|
|
1238
|
+
const { oldPath, newPath } = req.body;
|
|
1239
|
+
if (!oldPath || !newPath) {
|
|
1240
|
+
res.status(400).json({ error: 'Missing oldPath or newPath' });
|
|
1241
|
+
return;
|
|
1242
|
+
}
|
|
1243
|
+
const resolvedOld = resolve(config.rawDir, oldPath);
|
|
1244
|
+
const resolvedNew = resolve(config.rawDir, newPath);
|
|
1245
|
+
if (!resolvedOld.startsWith(resolve(config.rawDir)) || !resolvedNew.startsWith(resolve(config.rawDir))) {
|
|
1246
|
+
res.status(403).json({ error: 'Access denied: path outside raw directory' });
|
|
1247
|
+
return;
|
|
1248
|
+
}
|
|
1249
|
+
if (!existsSync(resolvedOld)) {
|
|
1250
|
+
res.status(404).json({ error: 'File not found' });
|
|
1251
|
+
return;
|
|
1252
|
+
}
|
|
1253
|
+
mkdirSync(dirname(resolvedNew), { recursive: true });
|
|
1254
|
+
renameSync(resolvedOld, resolvedNew);
|
|
1255
|
+
try {
|
|
1256
|
+
const { autoCommit, isGitRepo } = await import('../core/git.js');
|
|
1257
|
+
if (await isGitRepo(config.root)) {
|
|
1258
|
+
await autoCommit(config.root, 'manual', `rename raw "${basename(oldPath)}" → "${basename(newPath)}"`);
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
catch { /* non-fatal */ }
|
|
1262
|
+
res.json({ status: 'renamed', oldPath, newPath });
|
|
1263
|
+
}
|
|
1264
|
+
catch (err) {
|
|
1265
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1266
|
+
res.status(500).json({ error: `Rename failed: ${msg}` });
|
|
1267
|
+
}
|
|
1268
|
+
});
|
|
1269
|
+
// DELETE /api/raw/file — delete a raw file
|
|
1270
|
+
app.delete('/api/raw/file', async (req, res) => {
|
|
1271
|
+
try {
|
|
1272
|
+
const filePath = req.query['path'];
|
|
1273
|
+
if (!filePath) {
|
|
1274
|
+
res.status(400).json({ error: 'Missing path query param' });
|
|
1275
|
+
return;
|
|
1276
|
+
}
|
|
1277
|
+
const resolved = resolve(config.rawDir, filePath);
|
|
1278
|
+
if (!resolved.startsWith(resolve(config.rawDir))) {
|
|
1279
|
+
res.status(403).json({ error: 'Access denied: path outside raw directory' });
|
|
1280
|
+
return;
|
|
1281
|
+
}
|
|
1282
|
+
if (!existsSync(resolved)) {
|
|
1283
|
+
res.status(404).json({ error: 'File not found' });
|
|
1284
|
+
return;
|
|
1285
|
+
}
|
|
1286
|
+
fsUnlinkSync(resolved);
|
|
1287
|
+
try {
|
|
1288
|
+
const { autoCommit, isGitRepo } = await import('../core/git.js');
|
|
1289
|
+
if (await isGitRepo(config.root)) {
|
|
1290
|
+
await autoCommit(config.root, 'manual', `delete raw file "${basename(filePath)}"`);
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
catch { /* non-fatal */ }
|
|
1294
|
+
res.json({ status: 'deleted', path: filePath });
|
|
1295
|
+
}
|
|
1296
|
+
catch (err) {
|
|
1297
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1298
|
+
res.status(500).json({ error: `Delete failed: ${msg}` });
|
|
1299
|
+
}
|
|
1300
|
+
});
|
|
1301
|
+
// POST /api/raw/move — move a raw file to a different directory
|
|
1302
|
+
app.post('/api/raw/move', async (req, res) => {
|
|
1303
|
+
try {
|
|
1304
|
+
const { path: filePath, targetDir } = req.body;
|
|
1305
|
+
if (!filePath || !targetDir) {
|
|
1306
|
+
res.status(400).json({ error: 'Missing path or targetDir' });
|
|
1307
|
+
return;
|
|
1308
|
+
}
|
|
1309
|
+
const resolvedSrc = resolve(config.rawDir, filePath);
|
|
1310
|
+
const resolvedDest = resolve(config.rawDir, targetDir, basename(filePath));
|
|
1311
|
+
if (!resolvedSrc.startsWith(resolve(config.rawDir)) || !resolvedDest.startsWith(resolve(config.rawDir))) {
|
|
1312
|
+
res.status(403).json({ error: 'Access denied: path outside raw directory' });
|
|
1313
|
+
return;
|
|
1314
|
+
}
|
|
1315
|
+
if (!existsSync(resolvedSrc)) {
|
|
1316
|
+
res.status(404).json({ error: 'Source file not found' });
|
|
1317
|
+
return;
|
|
1318
|
+
}
|
|
1319
|
+
mkdirSync(dirname(resolvedDest), { recursive: true });
|
|
1320
|
+
renameSync(resolvedSrc, resolvedDest);
|
|
1321
|
+
try {
|
|
1322
|
+
const { autoCommit, isGitRepo } = await import('../core/git.js');
|
|
1323
|
+
if (await isGitRepo(config.root)) {
|
|
1324
|
+
await autoCommit(config.root, 'manual', `move raw "${basename(filePath)}" → ${targetDir}/`);
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
catch { /* non-fatal */ }
|
|
1328
|
+
res.json({ status: 'moved', from: filePath, to: targetDir + '/' + basename(filePath) });
|
|
1329
|
+
}
|
|
1330
|
+
catch (err) {
|
|
1331
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1332
|
+
res.status(500).json({ error: `Move failed: ${msg}` });
|
|
1333
|
+
}
|
|
1334
|
+
});
|
|
1335
|
+
// POST /api/pages/:title/move — move a wiki page to a different category folder
|
|
1336
|
+
app.post('/api/pages/:title/move', async (req, res) => {
|
|
1337
|
+
try {
|
|
1338
|
+
const title = req.params['title'];
|
|
1339
|
+
const { targetCategory } = req.body;
|
|
1340
|
+
if (!title || !targetCategory) {
|
|
1341
|
+
res.status(400).json({ error: 'Missing title or targetCategory' });
|
|
1342
|
+
return;
|
|
1343
|
+
}
|
|
1344
|
+
const pages = listWikiPages(config.wikiDir);
|
|
1345
|
+
const titleLower = title.toLowerCase();
|
|
1346
|
+
const slugified = titleLower.replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
|
1347
|
+
const match = pages.find((p) => {
|
|
1348
|
+
const fileSlug = basename(p, '.md');
|
|
1349
|
+
if (fileSlug === title || fileSlug === slugified)
|
|
1350
|
+
return true;
|
|
1351
|
+
try {
|
|
1352
|
+
return readWikiPage(p).title.toLowerCase() === titleLower;
|
|
1353
|
+
}
|
|
1354
|
+
catch {
|
|
1355
|
+
return false;
|
|
1356
|
+
}
|
|
1357
|
+
});
|
|
1358
|
+
if (!match) {
|
|
1359
|
+
res.status(404).json({ error: 'Page not found' });
|
|
1360
|
+
return;
|
|
1361
|
+
}
|
|
1362
|
+
const destDir = resolve(config.wikiDir, targetCategory);
|
|
1363
|
+
if (!destDir.startsWith(resolve(config.wikiDir))) {
|
|
1364
|
+
res.status(403).json({ error: 'Access denied: category outside wiki directory' });
|
|
1365
|
+
return;
|
|
1366
|
+
}
|
|
1367
|
+
mkdirSync(destDir, { recursive: true });
|
|
1368
|
+
const destPath = join(destDir, basename(match));
|
|
1369
|
+
renameSync(match, destPath);
|
|
1370
|
+
// Update frontmatter category/type
|
|
1371
|
+
let content = readFileSync(destPath, 'utf-8');
|
|
1372
|
+
if (content.match(/^type:\s*.+$/m)) {
|
|
1373
|
+
content = content.replace(/^type:\s*.+$/m, `type: ${targetCategory}`);
|
|
1374
|
+
}
|
|
1375
|
+
else if (content.match(/^category:\s*.+$/m)) {
|
|
1376
|
+
content = content.replace(/^category:\s*.+$/m, `category: ${targetCategory}`);
|
|
1377
|
+
}
|
|
1378
|
+
writeFileSync(destPath, content, 'utf-8');
|
|
1379
|
+
try {
|
|
1380
|
+
const { autoCommit, isGitRepo } = await import('../core/git.js');
|
|
1381
|
+
if (await isGitRepo(config.root)) {
|
|
1382
|
+
await autoCommit(config.root, 'manual', `move page "${title}" → ${targetCategory}/`);
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
catch { /* non-fatal */ }
|
|
1386
|
+
res.json({ status: 'moved', title, targetCategory, newPath: relative(config.wikiDir, destPath) });
|
|
1387
|
+
}
|
|
1388
|
+
catch (err) {
|
|
1389
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1390
|
+
res.status(500).json({ error: `Move failed: ${msg}` });
|
|
1391
|
+
}
|
|
1392
|
+
});
|
|
1393
|
+
// POST /api/folders — create a new folder inside wiki/ or raw/
|
|
1394
|
+
app.post('/api/folders', (req, res) => {
|
|
1395
|
+
try {
|
|
1396
|
+
const { path: folderPath } = req.body;
|
|
1397
|
+
if (!folderPath) {
|
|
1398
|
+
res.status(400).json({ error: 'Missing path' });
|
|
1399
|
+
return;
|
|
1400
|
+
}
|
|
1401
|
+
const resolved = resolve(config.root, folderPath);
|
|
1402
|
+
const wikiBase = resolve(config.wikiDir);
|
|
1403
|
+
const rawBase = resolve(config.rawDir);
|
|
1404
|
+
if (!resolved.startsWith(wikiBase) && !resolved.startsWith(rawBase)) {
|
|
1405
|
+
res.status(403).json({ error: 'Access denied: folder must be inside wiki/ or raw/' });
|
|
1406
|
+
return;
|
|
1407
|
+
}
|
|
1408
|
+
mkdirSync(resolved, { recursive: true });
|
|
1409
|
+
res.json({ status: 'created', path: folderPath });
|
|
1410
|
+
}
|
|
1411
|
+
catch (err) {
|
|
1412
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1413
|
+
res.status(500).json({ error: `Create folder failed: ${msg}` });
|
|
1414
|
+
}
|
|
1415
|
+
});
|
|
1416
|
+
// ─── OAuth Connector System ───────────────────────────────────────────────
|
|
1417
|
+
const OAUTH_PROVIDERS = {
|
|
1418
|
+
github: {
|
|
1419
|
+
authorizeUrl: 'https://github.com/login/oauth/authorize',
|
|
1420
|
+
tokenUrl: 'https://github.com/login/oauth/access_token',
|
|
1421
|
+
scopes: 'repo read:user',
|
|
1422
|
+
clientIdKey: 'github_client_id',
|
|
1423
|
+
clientSecretKey: 'github_client_secret',
|
|
1424
|
+
},
|
|
1425
|
+
slack: {
|
|
1426
|
+
authorizeUrl: 'https://slack.com/oauth/v2/authorize',
|
|
1427
|
+
tokenUrl: 'https://slack.com/api/oauth.v2.access',
|
|
1428
|
+
scopes: 'channels:history channels:read users:read',
|
|
1429
|
+
clientIdKey: 'slack_client_id',
|
|
1430
|
+
clientSecretKey: 'slack_client_secret',
|
|
1431
|
+
},
|
|
1432
|
+
google: {
|
|
1433
|
+
authorizeUrl: 'https://accounts.google.com/o/oauth2/v2/auth',
|
|
1434
|
+
tokenUrl: 'https://oauth2.googleapis.com/token',
|
|
1435
|
+
scopes: 'https://www.googleapis.com/auth/gmail.readonly https://www.googleapis.com/auth/drive.readonly',
|
|
1436
|
+
clientIdKey: 'google_client_id',
|
|
1437
|
+
clientSecretKey: 'google_client_secret',
|
|
1438
|
+
},
|
|
1439
|
+
linear: {
|
|
1440
|
+
authorizeUrl: 'https://linear.app/oauth/authorize',
|
|
1441
|
+
tokenUrl: 'https://api.linear.app/oauth/token',
|
|
1442
|
+
scopes: 'read',
|
|
1443
|
+
clientIdKey: 'linear_client_id',
|
|
1444
|
+
clientSecretKey: 'linear_client_secret',
|
|
1445
|
+
},
|
|
1446
|
+
jira: {
|
|
1447
|
+
authorizeUrl: 'https://auth.atlassian.com/authorize',
|
|
1448
|
+
tokenUrl: 'https://auth.atlassian.com/oauth/token',
|
|
1449
|
+
scopes: 'read:jira-work read:jira-user offline_access',
|
|
1450
|
+
clientIdKey: 'jira_client_id',
|
|
1451
|
+
clientSecretKey: 'jira_client_secret',
|
|
1452
|
+
},
|
|
1453
|
+
};
|
|
1454
|
+
const oauthStates = new Map();
|
|
1455
|
+
function getTokenStorePath() {
|
|
1456
|
+
return join(vaultRoot, '.wikimem', 'tokens.json');
|
|
1457
|
+
}
|
|
1458
|
+
function loadOAuthTokens() {
|
|
1459
|
+
const tokenPath = getTokenStorePath();
|
|
1460
|
+
if (!existsSync(tokenPath))
|
|
1461
|
+
return {};
|
|
1462
|
+
try {
|
|
1463
|
+
return JSON.parse(readFileSync(tokenPath, 'utf-8'));
|
|
1464
|
+
}
|
|
1465
|
+
catch {
|
|
1466
|
+
return {};
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
function saveOAuthToken(provider, tokenData) {
|
|
1470
|
+
const tokenPath = getTokenStorePath();
|
|
1471
|
+
mkdirSync(dirname(tokenPath), { recursive: true });
|
|
1472
|
+
const tokens = loadOAuthTokens();
|
|
1473
|
+
tokens[provider] = { ...tokenData, connectedAt: new Date().toISOString() };
|
|
1474
|
+
writeFileSync(tokenPath, JSON.stringify(tokens, null, 2), 'utf-8');
|
|
1475
|
+
}
|
|
1476
|
+
// GET /api/auth/tokens — list which providers have tokens stored
|
|
1477
|
+
app.get('/api/auth/tokens', (_req, res) => {
|
|
1478
|
+
try {
|
|
1479
|
+
const tokens = loadOAuthTokens();
|
|
1480
|
+
const result = {};
|
|
1481
|
+
for (const provider of Object.keys(OAUTH_PROVIDERS)) {
|
|
1482
|
+
const token = tokens[provider];
|
|
1483
|
+
result[provider] = { connected: !!token, connectedAt: token?.connectedAt };
|
|
1484
|
+
}
|
|
1485
|
+
res.json(result);
|
|
1486
|
+
}
|
|
1487
|
+
catch {
|
|
1488
|
+
res.json({});
|
|
1489
|
+
}
|
|
1490
|
+
});
|
|
1491
|
+
// GET /api/auth/start/:provider — generate OAuth authorize URL
|
|
1492
|
+
app.get('/api/auth/start/:provider', async (req, res) => {
|
|
1493
|
+
try {
|
|
1494
|
+
const providerName = req.params['provider'];
|
|
1495
|
+
if (!providerName) {
|
|
1496
|
+
res.status(400).json({ error: 'Missing provider' });
|
|
1497
|
+
return;
|
|
1498
|
+
}
|
|
1499
|
+
const providerConfig = OAUTH_PROVIDERS[providerName];
|
|
1500
|
+
if (!providerConfig) {
|
|
1501
|
+
res.status(400).json({ error: `Unknown provider: ${providerName}` });
|
|
1502
|
+
return;
|
|
1503
|
+
}
|
|
1504
|
+
const { loadConfig } = await import('../core/config.js');
|
|
1505
|
+
const userConfig = loadConfig(config.configPath);
|
|
1506
|
+
const clientId = userConfig[providerConfig.clientIdKey];
|
|
1507
|
+
if (!clientId) {
|
|
1508
|
+
res.status(400).json({ error: `Missing ${providerConfig.clientIdKey} in config.yaml` });
|
|
1509
|
+
return;
|
|
1510
|
+
}
|
|
1511
|
+
const state = randomBytes(24).toString('hex');
|
|
1512
|
+
oauthStates.set(state, { provider: providerName, createdAt: Date.now() });
|
|
1513
|
+
// Clean stale states (>10 min)
|
|
1514
|
+
for (const [key, val] of oauthStates) {
|
|
1515
|
+
if (Date.now() - val.createdAt > 600_000)
|
|
1516
|
+
oauthStates.delete(key);
|
|
1517
|
+
}
|
|
1518
|
+
const redirectUri = `http://localhost:${port}/api/auth/callback`;
|
|
1519
|
+
const params = new URLSearchParams({
|
|
1520
|
+
client_id: clientId,
|
|
1521
|
+
redirect_uri: redirectUri,
|
|
1522
|
+
scope: providerConfig.scopes,
|
|
1523
|
+
state,
|
|
1524
|
+
response_type: 'code',
|
|
1525
|
+
...(providerName === 'google' ? { access_type: 'offline', prompt: 'consent' } : {}),
|
|
1526
|
+
...(providerName === 'jira' ? { audience: 'api.atlassian.com', prompt: 'consent' } : {}),
|
|
1527
|
+
});
|
|
1528
|
+
const authorizeUrl = `${providerConfig.authorizeUrl}?${params.toString()}`;
|
|
1529
|
+
res.json({ url: authorizeUrl, state });
|
|
1530
|
+
}
|
|
1531
|
+
catch (err) {
|
|
1532
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1533
|
+
res.status(500).json({ error: `OAuth start failed: ${msg}` });
|
|
1534
|
+
}
|
|
1535
|
+
});
|
|
1536
|
+
// GET /api/auth/callback — OAuth callback handler
|
|
1537
|
+
app.get('/api/auth/callback', async (req, res) => {
|
|
1538
|
+
try {
|
|
1539
|
+
const code = req.query['code'];
|
|
1540
|
+
const state = req.query['state'];
|
|
1541
|
+
if (!code || !state) {
|
|
1542
|
+
res.status(400).send('<h2>Missing code or state</h2>');
|
|
1543
|
+
return;
|
|
1544
|
+
}
|
|
1545
|
+
const stateData = oauthStates.get(state);
|
|
1546
|
+
if (!stateData) {
|
|
1547
|
+
res.status(400).send('<h2>Invalid or expired state token</h2>');
|
|
1548
|
+
return;
|
|
1549
|
+
}
|
|
1550
|
+
oauthStates.delete(state);
|
|
1551
|
+
const providerConfig = OAUTH_PROVIDERS[stateData.provider];
|
|
1552
|
+
if (!providerConfig) {
|
|
1553
|
+
res.status(400).send('<h2>Unknown provider</h2>');
|
|
1554
|
+
return;
|
|
1555
|
+
}
|
|
1556
|
+
const { loadConfig } = await import('../core/config.js');
|
|
1557
|
+
const userConfig = loadConfig(config.configPath);
|
|
1558
|
+
const clientId = userConfig[providerConfig.clientIdKey];
|
|
1559
|
+
const clientSecret = userConfig[providerConfig.clientSecretKey];
|
|
1560
|
+
if (!clientId || !clientSecret) {
|
|
1561
|
+
res.status(400).send('<h2>Missing client credentials in config</h2>');
|
|
1562
|
+
return;
|
|
1563
|
+
}
|
|
1564
|
+
const redirectUri = `http://localhost:${port}/api/auth/callback`;
|
|
1565
|
+
const tokenRes = await fetch(providerConfig.tokenUrl, {
|
|
1566
|
+
method: 'POST',
|
|
1567
|
+
headers: {
|
|
1568
|
+
'Content-Type': 'application/x-www-form-urlencoded',
|
|
1569
|
+
Accept: 'application/json',
|
|
1570
|
+
},
|
|
1571
|
+
body: new URLSearchParams({
|
|
1572
|
+
client_id: clientId,
|
|
1573
|
+
client_secret: clientSecret,
|
|
1574
|
+
code,
|
|
1575
|
+
redirect_uri: redirectUri,
|
|
1576
|
+
grant_type: 'authorization_code',
|
|
1577
|
+
}),
|
|
1578
|
+
});
|
|
1579
|
+
const tokenBody = await tokenRes.json();
|
|
1580
|
+
const accessToken = tokenBody['access_token'];
|
|
1581
|
+
if (!accessToken) {
|
|
1582
|
+
res.status(400).send(`<h2>Token exchange failed</h2><pre>${JSON.stringify(tokenBody, null, 2)}</pre>`);
|
|
1583
|
+
return;
|
|
1584
|
+
}
|
|
1585
|
+
saveOAuthToken(stateData.provider, {
|
|
1586
|
+
access_token: accessToken,
|
|
1587
|
+
refresh_token: tokenBody['refresh_token'],
|
|
1588
|
+
scope: tokenBody['scope'],
|
|
1589
|
+
});
|
|
1590
|
+
res.send(`<!DOCTYPE html><html><head><style>
|
|
1591
|
+
body { font-family: Inter, system-ui, sans-serif; background: #1e1e1e; color: #ccc; display: flex; align-items: center; justify-content: center; min-height: 100vh; }
|
|
1592
|
+
.card { background: #252526; border: 1px solid #3e3e3e; border-radius: 12px; padding: 32px 40px; text-align: center; }
|
|
1593
|
+
h2 { color: #4ec9b0; margin: 0 0 8px; } p { color: #808080; font-size: 14px; }
|
|
1594
|
+
</style></head><body><div class="card"><h2>Connected!</h2><p>${stateData.provider} is now linked to WikiMem.</p><p style="margin-top:12px"><small>You can close this tab.</small></p></div></body></html>`);
|
|
1595
|
+
}
|
|
1596
|
+
catch (err) {
|
|
1597
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1598
|
+
res.status(500).send(`<h2>OAuth callback failed</h2><pre>${msg}</pre>`);
|
|
1599
|
+
}
|
|
1600
|
+
});
|
|
1601
|
+
// DELETE /api/auth/tokens/:provider — disconnect a provider
|
|
1602
|
+
app.delete('/api/auth/tokens/:provider', (_req, res) => {
|
|
1603
|
+
try {
|
|
1604
|
+
const provider = _req.params['provider'];
|
|
1605
|
+
if (!provider) {
|
|
1606
|
+
res.status(400).json({ error: 'Missing provider' });
|
|
1607
|
+
return;
|
|
1608
|
+
}
|
|
1609
|
+
const tokenPath = getTokenStorePath();
|
|
1610
|
+
const tokens = loadOAuthTokens();
|
|
1611
|
+
delete tokens[provider];
|
|
1612
|
+
mkdirSync(dirname(tokenPath), { recursive: true });
|
|
1613
|
+
writeFileSync(tokenPath, JSON.stringify(tokens, null, 2), 'utf-8');
|
|
1614
|
+
res.json({ status: 'disconnected', provider });
|
|
1615
|
+
}
|
|
1616
|
+
catch (err) {
|
|
1617
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1618
|
+
res.status(500).json({ error: `Disconnect failed: ${msg}` });
|
|
1619
|
+
}
|
|
1620
|
+
});
|
|
1621
|
+
// Pipeline SSE endpoint — real-time step updates
|
|
1622
|
+
app.get('/api/pipeline/events', (_req, res) => {
|
|
1623
|
+
res.writeHead(200, {
|
|
1624
|
+
'Content-Type': 'text/event-stream',
|
|
1625
|
+
'Cache-Control': 'no-cache',
|
|
1626
|
+
Connection: 'keep-alive',
|
|
1627
|
+
});
|
|
1628
|
+
res.write('data: {"type":"connected"}\n\n');
|
|
1629
|
+
const onStep = (event) => {
|
|
1630
|
+
res.write(`data: ${JSON.stringify(event)}\n\n`);
|
|
1631
|
+
};
|
|
1632
|
+
import('../core/pipeline-events.js').then(({ pipelineEvents }) => {
|
|
1633
|
+
pipelineEvents.on('step', onStep);
|
|
1634
|
+
_req.on('close', () => pipelineEvents.off('step', onStep));
|
|
1635
|
+
});
|
|
1636
|
+
});
|
|
1637
|
+
// Pipeline runs history
|
|
1638
|
+
app.get('/api/pipeline/runs', async (_req, res) => {
|
|
1639
|
+
try {
|
|
1640
|
+
const { pipelineEvents } = await import('../core/pipeline-events.js');
|
|
1641
|
+
const runs = pipelineEvents.getRecentRuns();
|
|
1642
|
+
const summaries = runs.map(r => ({
|
|
1643
|
+
id: r.id,
|
|
1644
|
+
source: r.source,
|
|
1645
|
+
startedAt: r.startedAt,
|
|
1646
|
+
eventCount: r.events.length,
|
|
1647
|
+
hasSummary: !!r.summary,
|
|
1648
|
+
hasLLMTrace: !!r.llmTrace,
|
|
1649
|
+
result: r.result,
|
|
1650
|
+
}));
|
|
1651
|
+
res.json({ runs: summaries });
|
|
1652
|
+
}
|
|
1653
|
+
catch {
|
|
1654
|
+
res.json({ runs: [] });
|
|
1655
|
+
}
|
|
1656
|
+
});
|
|
1657
|
+
// Pipeline run detail (with LLM trace and summary)
|
|
1658
|
+
app.get('/api/pipeline/runs/:id', async (req, res) => {
|
|
1659
|
+
try {
|
|
1660
|
+
const { pipelineEvents } = await import('../core/pipeline-events.js');
|
|
1661
|
+
const runs = pipelineEvents.getRecentRuns();
|
|
1662
|
+
const run = runs.find(r => r.id === req.params['id']);
|
|
1663
|
+
if (!run) {
|
|
1664
|
+
res.status(404).json({ error: 'Run not found' });
|
|
1665
|
+
return;
|
|
1666
|
+
}
|
|
1667
|
+
res.json(run);
|
|
1668
|
+
}
|
|
1669
|
+
catch {
|
|
1670
|
+
res.status(500).json({ error: 'Failed to get run' });
|
|
1671
|
+
}
|
|
1672
|
+
});
|
|
1673
|
+
// ─── Pipeline Configuration (custom steps) ──────────────────────────────────
|
|
1674
|
+
app.get('/api/pipeline/config', async (_req, res) => {
|
|
1675
|
+
try {
|
|
1676
|
+
const { loadConfig } = await import('../core/config.js');
|
|
1677
|
+
const userConfig = loadConfig(config.configPath);
|
|
1678
|
+
const pipelineConfig = userConfig['pipeline'] ?? {};
|
|
1679
|
+
const customSteps = pipelineConfig['custom_steps'] ?? [];
|
|
1680
|
+
const disabledSteps = pipelineConfig['disabled_steps'] ?? [];
|
|
1681
|
+
res.json({ custom_steps: customSteps, disabled_steps: disabledSteps });
|
|
1682
|
+
}
|
|
1683
|
+
catch {
|
|
1684
|
+
res.json({ custom_steps: [], disabled_steps: [] });
|
|
1685
|
+
}
|
|
1686
|
+
});
|
|
1687
|
+
app.put('/api/pipeline/config', async (req, res) => {
|
|
1688
|
+
try {
|
|
1689
|
+
const { loadConfig } = await import('../core/config.js');
|
|
1690
|
+
const YAML = await import('yaml');
|
|
1691
|
+
const current = loadConfig(config.configPath);
|
|
1692
|
+
const body = req.body;
|
|
1693
|
+
const pipeline = current['pipeline'] ?? {};
|
|
1694
|
+
if (body.custom_steps !== undefined)
|
|
1695
|
+
pipeline['custom_steps'] = body.custom_steps;
|
|
1696
|
+
if (body.disabled_steps !== undefined)
|
|
1697
|
+
pipeline['disabled_steps'] = body.disabled_steps;
|
|
1698
|
+
current['pipeline'] = pipeline;
|
|
1699
|
+
writeFileSync(config.configPath, YAML.stringify(current), 'utf-8');
|
|
1700
|
+
res.json({ status: 'saved' });
|
|
1701
|
+
}
|
|
1702
|
+
catch (err) {
|
|
1703
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1704
|
+
res.status(500).json({ error: `Failed to save pipeline config: ${msg}` });
|
|
1705
|
+
}
|
|
1706
|
+
});
|
|
1707
|
+
// ─── Automation 3: Webhook Ingest ─────────────────────────────────────────
|
|
1708
|
+
// Track files currently being ingested by webhook to avoid double-ingest with watcher
|
|
1709
|
+
const webhookIngestingFiles = new Set();
|
|
1710
|
+
// POST /api/webhook/ingest — accept external content, run ingest pipeline
|
|
1711
|
+
app.post('/api/webhook/ingest', async (req, res) => {
|
|
1712
|
+
try {
|
|
1713
|
+
const { content, title, source, tags, metadata } = req.body;
|
|
1714
|
+
if (!content || content.trim().length === 0) {
|
|
1715
|
+
res.status(400).json({ error: 'Missing or empty content field' });
|
|
1716
|
+
return;
|
|
1717
|
+
}
|
|
1718
|
+
const now = new Date().toISOString().split('T')[0] ?? '';
|
|
1719
|
+
const pageTitle = title ?? `Webhook Ingest ${new Date().toISOString()}`;
|
|
1720
|
+
const markdown = `# ${pageTitle}\n\n${source ? `Source: ${source}\n` : ''}Ingested via webhook: ${new Date().toISOString()}\n\n${content}`;
|
|
1721
|
+
// Write to raw/ so ingest pipeline can find it
|
|
1722
|
+
const { mkdirSync: mkdir, writeFileSync: write } = await import('node:fs');
|
|
1723
|
+
const { join: joinPath } = await import('node:path');
|
|
1724
|
+
const { slugify } = await import('../core/vault.js');
|
|
1725
|
+
const dateDir = joinPath(config.rawDir, now);
|
|
1726
|
+
mkdir(dateDir, { recursive: true });
|
|
1727
|
+
const filePath = joinPath(dateDir, `${slugify(pageTitle.substring(0, 60))}-webhook.md`);
|
|
1728
|
+
write(filePath, markdown, 'utf-8');
|
|
1729
|
+
// Tell the watcher to skip this file (webhook handles ingest)
|
|
1730
|
+
webhookIngestingFiles.add(filePath);
|
|
1731
|
+
const { ingestSource } = await import('../core/ingest.js');
|
|
1732
|
+
const { createProviderFromUserConfig } = await import('../providers/index.js');
|
|
1733
|
+
const { loadConfig } = await import('../core/config.js');
|
|
1734
|
+
const { appendAuditEntry } = await import('../core/audit-trail.js');
|
|
1735
|
+
const userConfig = loadConfig(config.configPath);
|
|
1736
|
+
const provider = createProviderFromUserConfig(userConfig);
|
|
1737
|
+
let result;
|
|
1738
|
+
try {
|
|
1739
|
+
result = await ingestSource(filePath, config, provider, {
|
|
1740
|
+
verbose: false,
|
|
1741
|
+
tags: tags ?? [],
|
|
1742
|
+
metadata,
|
|
1743
|
+
addedBy: 'webhook',
|
|
1744
|
+
});
|
|
1745
|
+
}
|
|
1746
|
+
catch (ingestErr) {
|
|
1747
|
+
const ingestMsg = ingestErr instanceof Error ? ingestErr.message : String(ingestErr);
|
|
1748
|
+
appendAuditEntry(vaultRoot, {
|
|
1749
|
+
action: 'ingest',
|
|
1750
|
+
actor: 'webhook',
|
|
1751
|
+
source: source ?? filePath,
|
|
1752
|
+
summary: `Webhook ingest FAILED: "${pageTitle}" — ${ingestMsg.substring(0, 200)}`,
|
|
1753
|
+
pagesAffected: [pageTitle],
|
|
1754
|
+
});
|
|
1755
|
+
webhookIngestingFiles.delete(filePath);
|
|
1756
|
+
res.status(500).json({ error: `Webhook ingest failed: ${ingestMsg}` });
|
|
1757
|
+
return;
|
|
1758
|
+
}
|
|
1759
|
+
appendAuditEntry(vaultRoot, {
|
|
1760
|
+
action: 'ingest',
|
|
1761
|
+
actor: 'webhook',
|
|
1762
|
+
source: source ?? filePath,
|
|
1763
|
+
summary: `Webhook ingest: "${pageTitle}" — ${result.pagesUpdated} pages created.`,
|
|
1764
|
+
pagesAffected: [pageTitle],
|
|
1765
|
+
});
|
|
1766
|
+
webhookIngestingFiles.delete(filePath);
|
|
1767
|
+
res.json({
|
|
1768
|
+
success: !result.rejected,
|
|
1769
|
+
pagesCreated: result.pagesUpdated,
|
|
1770
|
+
title: result.title,
|
|
1771
|
+
rawPath: result.rawPath,
|
|
1772
|
+
rejected: result.rejected ?? false,
|
|
1773
|
+
rejectionReason: result.rejectionReason,
|
|
1774
|
+
});
|
|
1775
|
+
}
|
|
1776
|
+
catch (err) {
|
|
1777
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1778
|
+
res.status(500).json({ error: `Webhook ingest failed: ${msg}` });
|
|
1779
|
+
}
|
|
1780
|
+
});
|
|
1781
|
+
// ─── Centralized Ingestion Gateway ───────────────────────────────────────
|
|
1782
|
+
// POST /api/gateway/ingest — universal ingestion from any source
|
|
1783
|
+
app.post('/api/gateway/ingest', async (req, res) => {
|
|
1784
|
+
const runId = `gw-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
1785
|
+
try {
|
|
1786
|
+
const { content, url, source, title, tags, metadata } = req.body;
|
|
1787
|
+
const sourceLabel = source ?? 'api';
|
|
1788
|
+
const validSources = new Set(['email', 'slack', 'webhook', 'api', 'web']);
|
|
1789
|
+
if (source && !validSources.has(source)) {
|
|
1790
|
+
res.status(400).json({ runId, error: `Invalid source. Expected one of: ${[...validSources].join(', ')}` });
|
|
1791
|
+
return;
|
|
1792
|
+
}
|
|
1793
|
+
if (!content && !url) {
|
|
1794
|
+
res.status(400).json({ runId, error: 'Must provide at least content or url' });
|
|
1795
|
+
return;
|
|
1796
|
+
}
|
|
1797
|
+
const { ingestSource } = await import('../core/ingest.js');
|
|
1798
|
+
const { createProviderFromUserConfig } = await import('../providers/index.js');
|
|
1799
|
+
const { loadConfig } = await import('../core/config.js');
|
|
1800
|
+
const { slugify } = await import('../core/vault.js');
|
|
1801
|
+
const userConfig = loadConfig(config.configPath);
|
|
1802
|
+
const provider = createProviderFromUserConfig(userConfig);
|
|
1803
|
+
const auditActor = 'webhook';
|
|
1804
|
+
// URL-only ingestion: pass URL directly to ingestSource
|
|
1805
|
+
if (url && !content) {
|
|
1806
|
+
const result = await ingestSource(url, config, provider, {
|
|
1807
|
+
verbose: false,
|
|
1808
|
+
tags: tags ?? [],
|
|
1809
|
+
metadata: { ...metadata, gateway_source: sourceLabel, gateway_run: runId },
|
|
1810
|
+
addedBy: 'webhook',
|
|
1811
|
+
});
|
|
1812
|
+
res.json({
|
|
1813
|
+
runId,
|
|
1814
|
+
status: result.rejected ? 'rejected' : 'ingested',
|
|
1815
|
+
pagesCreated: result.pagesUpdated ?? 0,
|
|
1816
|
+
title: result.title,
|
|
1817
|
+
rejected: result.rejected ?? false,
|
|
1818
|
+
rejectionReason: result.rejectionReason,
|
|
1819
|
+
});
|
|
1820
|
+
return;
|
|
1821
|
+
}
|
|
1822
|
+
// Content-based: write to raw/ then ingest
|
|
1823
|
+
const now = new Date().toISOString().split('T')[0] ?? '';
|
|
1824
|
+
const pageTitle = title ?? `Gateway ${sourceLabel} ${new Date().toISOString()}`;
|
|
1825
|
+
const frontmatter = [
|
|
1826
|
+
`source: ${sourceLabel}`,
|
|
1827
|
+
`ingested_via: gateway`,
|
|
1828
|
+
`run_id: ${runId}`,
|
|
1829
|
+
...(tags?.length ? [`tags: [${tags.join(', ')}]`] : []),
|
|
1830
|
+
...(metadata ? Object.entries(metadata).map(([k, v]) => `${k}: ${v}`) : []),
|
|
1831
|
+
].join('\n');
|
|
1832
|
+
const markdown = `---\ntitle: "${pageTitle}"\n${frontmatter}\n---\n\n${content}`;
|
|
1833
|
+
const dateDir = join(config.rawDir, now);
|
|
1834
|
+
mkdirSync(dateDir, { recursive: true });
|
|
1835
|
+
const slug = slugify(pageTitle.substring(0, 60));
|
|
1836
|
+
const filePath = join(dateDir, `${slug}-gw-${sourceLabel}.md`);
|
|
1837
|
+
writeFileSync(filePath, markdown, 'utf-8');
|
|
1838
|
+
webhookIngestingFiles.add(filePath);
|
|
1839
|
+
let result;
|
|
1840
|
+
try {
|
|
1841
|
+
result = await ingestSource(filePath, config, provider, {
|
|
1842
|
+
verbose: false,
|
|
1843
|
+
tags: tags ?? [],
|
|
1844
|
+
metadata: { ...metadata, gateway_source: sourceLabel, gateway_run: runId },
|
|
1845
|
+
addedBy: 'webhook',
|
|
1846
|
+
});
|
|
1847
|
+
}
|
|
1848
|
+
finally {
|
|
1849
|
+
webhookIngestingFiles.delete(filePath);
|
|
1850
|
+
}
|
|
1851
|
+
try {
|
|
1852
|
+
const { appendAuditEntry } = await import('../core/audit-trail.js');
|
|
1853
|
+
appendAuditEntry(vaultRoot, {
|
|
1854
|
+
action: 'ingest',
|
|
1855
|
+
actor: auditActor,
|
|
1856
|
+
source: url ?? filePath,
|
|
1857
|
+
summary: `Gateway ingest (${sourceLabel}): "${pageTitle}" — ${result.pagesUpdated} pages created.`,
|
|
1858
|
+
pagesAffected: [pageTitle],
|
|
1859
|
+
});
|
|
1860
|
+
}
|
|
1861
|
+
catch { /* non-fatal */ }
|
|
1862
|
+
res.json({
|
|
1863
|
+
runId,
|
|
1864
|
+
status: result.rejected ? 'rejected' : 'ingested',
|
|
1865
|
+
pagesCreated: result.pagesUpdated ?? 0,
|
|
1866
|
+
title: result.title,
|
|
1867
|
+
rejected: result.rejected ?? false,
|
|
1868
|
+
rejectionReason: result.rejectionReason,
|
|
1869
|
+
});
|
|
1870
|
+
}
|
|
1871
|
+
catch (err) {
|
|
1872
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1873
|
+
res.status(500).json({ runId, status: 'error', error: `Gateway ingest failed: ${msg}` });
|
|
1874
|
+
}
|
|
1875
|
+
});
|
|
1876
|
+
// ─── Audit Trail API ──────────────────────────────────────────────────────
|
|
1877
|
+
// GET /api/audit-trail?limit=50&actor=all&action=all
|
|
1878
|
+
app.get('/api/audit-trail', async (req, res) => {
|
|
1879
|
+
try {
|
|
1880
|
+
const limit = parseInt(req.query['limit']) || 50;
|
|
1881
|
+
const actor = req.query['actor'] || 'all';
|
|
1882
|
+
const action = req.query['action'] || 'all';
|
|
1883
|
+
const since = req.query['since'];
|
|
1884
|
+
const before = req.query['before'];
|
|
1885
|
+
const { readAuditTrail } = await import('../core/audit-trail.js');
|
|
1886
|
+
const entries = readAuditTrail(vaultRoot, limit, actor, action, since, before);
|
|
1887
|
+
res.json({ entries, total: entries.length });
|
|
1888
|
+
}
|
|
1889
|
+
catch (err) {
|
|
1890
|
+
res.status(500).json({ error: String(err) });
|
|
1891
|
+
}
|
|
1892
|
+
});
|
|
1893
|
+
// ─── Automation Settings API ─────────────────────────────────────────────
|
|
1894
|
+
function getAutomationSettingsPath() {
|
|
1895
|
+
return join(vaultRoot, '.wikimem', 'automations.json');
|
|
1896
|
+
}
|
|
1897
|
+
function loadAutomationSettings() {
|
|
1898
|
+
const settingsPath = getAutomationSettingsPath();
|
|
1899
|
+
if (!existsSync(settingsPath))
|
|
1900
|
+
return {};
|
|
1901
|
+
try {
|
|
1902
|
+
return JSON.parse(readFileSync(settingsPath, 'utf-8'));
|
|
1903
|
+
}
|
|
1904
|
+
catch {
|
|
1905
|
+
return {};
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
function saveAutomationSettings(settings) {
|
|
1909
|
+
const settingsPath = getAutomationSettingsPath();
|
|
1910
|
+
const dir = join(vaultRoot, '.wikimem');
|
|
1911
|
+
mkdirSync(dir, { recursive: true });
|
|
1912
|
+
writeFileSync(settingsPath, JSON.stringify(settings, null, 2), 'utf-8');
|
|
1913
|
+
}
|
|
1914
|
+
// GET /api/automations/settings — read all automation toggle state
|
|
1915
|
+
app.get('/api/automations/settings', (_req, res) => {
|
|
1916
|
+
try {
|
|
1917
|
+
const settings = loadAutomationSettings();
|
|
1918
|
+
res.json(settings);
|
|
1919
|
+
}
|
|
1920
|
+
catch (err) {
|
|
1921
|
+
res.status(500).json({ error: String(err) });
|
|
1922
|
+
}
|
|
1923
|
+
});
|
|
1924
|
+
// PATCH /api/automations/settings — update a single automation setting
|
|
1925
|
+
app.patch('/api/automations/settings', (req, res) => {
|
|
1926
|
+
try {
|
|
1927
|
+
const { automation, key, value } = req.body;
|
|
1928
|
+
if (!automation || !key) {
|
|
1929
|
+
res.status(400).json({ error: 'Missing automation or key' });
|
|
1930
|
+
return;
|
|
1931
|
+
}
|
|
1932
|
+
const settings = loadAutomationSettings();
|
|
1933
|
+
if (!settings[automation])
|
|
1934
|
+
settings[automation] = {};
|
|
1935
|
+
settings[automation][key] = value;
|
|
1936
|
+
saveAutomationSettings(settings);
|
|
1937
|
+
res.json({ status: 'saved', automation, key, value });
|
|
1938
|
+
}
|
|
1939
|
+
catch (err) {
|
|
1940
|
+
res.status(500).json({ error: String(err) });
|
|
1941
|
+
}
|
|
1942
|
+
});
|
|
1943
|
+
// POST /api/automations/sourcing/run — trigger smart sourcing (alias for /api/automations/scrape)
|
|
1944
|
+
app.post('/api/automations/sourcing/run', async (req, res) => {
|
|
1945
|
+
try {
|
|
1946
|
+
const { runSmartScraper } = await import('../core/scraper.js');
|
|
1947
|
+
const { loadConfig } = await import('../core/config.js');
|
|
1948
|
+
const userConfig = loadConfig(config.configPath);
|
|
1949
|
+
const result = await runSmartScraper(config, userConfig, { dryRun: false, maxItems: 10 });
|
|
1950
|
+
res.json(result);
|
|
1951
|
+
}
|
|
1952
|
+
catch (err) {
|
|
1953
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1954
|
+
res.status(500).json({ error: `Sourcing run failed: ${msg}` });
|
|
1955
|
+
}
|
|
1956
|
+
});
|
|
1957
|
+
// ─── Observer APIs ────────────────────────────────────────────────────────
|
|
1958
|
+
// POST /api/observer/run — trigger observer manually
|
|
1959
|
+
app.post('/api/observer/run', async (_req, res) => {
|
|
1960
|
+
try {
|
|
1961
|
+
const { runObserver } = await import('../core/observer.js');
|
|
1962
|
+
const report = await runObserver(config);
|
|
1963
|
+
res.json({
|
|
1964
|
+
success: true,
|
|
1965
|
+
date: report.date,
|
|
1966
|
+
totalPages: report.totalPages,
|
|
1967
|
+
pagesReviewed: report.pagesReviewed,
|
|
1968
|
+
averageScore: report.averageScore,
|
|
1969
|
+
maxScore: report.maxScore,
|
|
1970
|
+
orphanCount: report.orphans.length,
|
|
1971
|
+
gapCount: report.gaps.length,
|
|
1972
|
+
contradictionCount: report.contradictions.length,
|
|
1973
|
+
reportPath: `.wikimem/observer-reports/${report.date}.json`,
|
|
1974
|
+
});
|
|
1975
|
+
}
|
|
1976
|
+
catch (err) {
|
|
1977
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1978
|
+
res.status(500).json({ error: `Observer run failed: ${msg}` });
|
|
1979
|
+
}
|
|
1980
|
+
});
|
|
1981
|
+
// GET /api/observer/reports — list all reports
|
|
1982
|
+
app.get('/api/observer/reports', async (_req, res) => {
|
|
1983
|
+
try {
|
|
1984
|
+
const { listObserverReports } = await import('../core/observer.js');
|
|
1985
|
+
const reports = listObserverReports(vaultRoot);
|
|
1986
|
+
res.json({ reports });
|
|
1987
|
+
}
|
|
1988
|
+
catch (err) {
|
|
1989
|
+
res.status(500).json({ error: String(err) });
|
|
1990
|
+
}
|
|
1991
|
+
});
|
|
1992
|
+
// GET /api/observer/reports/:date — get specific report
|
|
1993
|
+
app.get('/api/observer/reports/:date', async (req, res) => {
|
|
1994
|
+
try {
|
|
1995
|
+
const date = req.params['date'];
|
|
1996
|
+
if (!date) {
|
|
1997
|
+
res.status(400).json({ error: 'Missing date' });
|
|
1998
|
+
return;
|
|
1999
|
+
}
|
|
2000
|
+
const { readObserverReport } = await import('../core/observer.js');
|
|
2001
|
+
const report = readObserverReport(vaultRoot, date);
|
|
2002
|
+
if (!report) {
|
|
2003
|
+
res.status(404).json({ error: `No report found for ${date}` });
|
|
2004
|
+
return;
|
|
2005
|
+
}
|
|
2006
|
+
res.json(report);
|
|
2007
|
+
}
|
|
2008
|
+
catch (err) {
|
|
2009
|
+
res.status(500).json({ error: String(err) });
|
|
2010
|
+
}
|
|
2011
|
+
});
|
|
2012
|
+
// ─── Automations: Scrape ──────────────────────────────────────────────────
|
|
2013
|
+
// POST /api/automations/scrape — trigger scrape for a source (or all)
|
|
2014
|
+
app.post('/api/automations/scrape', async (req, res) => {
|
|
2015
|
+
try {
|
|
2016
|
+
const { source: sourceName, dryRun, maxItems } = req.body;
|
|
2017
|
+
const { runSmartScraper } = await import('../core/scraper.js');
|
|
2018
|
+
const { loadConfig } = await import('../core/config.js');
|
|
2019
|
+
const userConfig = loadConfig(config.configPath);
|
|
2020
|
+
const result = await runSmartScraper(config, userConfig, { dryRun: dryRun ?? false, maxItems: maxItems ?? 10 }, sourceName);
|
|
2021
|
+
res.json(result);
|
|
2022
|
+
}
|
|
2023
|
+
catch (err) {
|
|
2024
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2025
|
+
res.status(500).json({ error: `Scrape failed: ${msg}` });
|
|
2026
|
+
}
|
|
2027
|
+
});
|
|
2028
|
+
// POST /api/automations/observe — trigger observer with optional budget
|
|
2029
|
+
app.post('/api/automations/observe', async (req, res) => {
|
|
2030
|
+
try {
|
|
2031
|
+
const { maxPagesToReview, maxCostEstimate } = req.body;
|
|
2032
|
+
void maxCostEstimate;
|
|
2033
|
+
const { runObserver } = await import('../core/observer.js');
|
|
2034
|
+
const report = await runObserver(config, { maxPagesToReview });
|
|
2035
|
+
res.json({
|
|
2036
|
+
success: true,
|
|
2037
|
+
date: report.date,
|
|
2038
|
+
totalPages: report.totalPages,
|
|
2039
|
+
pagesReviewed: report.pagesReviewed,
|
|
2040
|
+
averageScore: report.averageScore,
|
|
2041
|
+
maxScore: report.maxScore,
|
|
2042
|
+
orphanCount: report.orphans.length,
|
|
2043
|
+
gapCount: report.gaps.length,
|
|
2044
|
+
contradictionCount: report.contradictions.length,
|
|
2045
|
+
reportPath: `.wikimem/observer-reports/${report.date}.json`,
|
|
2046
|
+
});
|
|
2047
|
+
}
|
|
2048
|
+
catch (err) {
|
|
2049
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2050
|
+
res.status(500).json({ error: `Observer run failed: ${msg}` });
|
|
2051
|
+
}
|
|
2052
|
+
});
|
|
2053
|
+
// GET /api/automations/status — status of all three automations
|
|
2054
|
+
app.get('/api/automations/status', async (_req, res) => {
|
|
2055
|
+
try {
|
|
2056
|
+
const { readAuditTrail } = await import('../core/audit-trail.js');
|
|
2057
|
+
const { isObserverCronRunning } = await import('../core/observer.js');
|
|
2058
|
+
const { loadConfig } = await import('../core/config.js');
|
|
2059
|
+
const userConfig = loadConfig(config.configPath);
|
|
2060
|
+
const settings = loadAutomationSettings();
|
|
2061
|
+
const recentIngest = readAuditTrail(vaultRoot, 1, undefined, 'ingest');
|
|
2062
|
+
const recentScrape = readAuditTrail(vaultRoot, 1, undefined, 'scrape');
|
|
2063
|
+
const recentObserve = readAuditTrail(vaultRoot, 1, undefined, 'observe');
|
|
2064
|
+
res.json({
|
|
2065
|
+
ingest: {
|
|
2066
|
+
enabled: settings['ingest']?.['enabled'] !== false,
|
|
2067
|
+
lastRun: recentIngest[0]?.timestamp ?? null,
|
|
2068
|
+
watcherActive: true,
|
|
2069
|
+
},
|
|
2070
|
+
scrape: {
|
|
2071
|
+
enabled: settings['scrape']?.['enabled'] !== false,
|
|
2072
|
+
lastRun: recentScrape[0]?.timestamp ?? null,
|
|
2073
|
+
sourcesConfigured: (userConfig.sources ?? []).length,
|
|
2074
|
+
},
|
|
2075
|
+
observe: {
|
|
2076
|
+
enabled: settings['observe']?.['enabled'] !== false,
|
|
2077
|
+
lastRun: recentObserve[0]?.timestamp ?? null,
|
|
2078
|
+
cronActive: isObserverCronRunning(),
|
|
2079
|
+
schedule: '0 3 * * *',
|
|
2080
|
+
},
|
|
2081
|
+
});
|
|
2082
|
+
}
|
|
2083
|
+
catch (err) {
|
|
2084
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2085
|
+
res.status(500).json({ error: `Failed to get automation status: ${msg}` });
|
|
2086
|
+
}
|
|
2087
|
+
});
|
|
2088
|
+
// ─── Connector APIs ───────────────────────────────────────────────────────
|
|
2089
|
+
// GET /api/connectors — list all configured connectors
|
|
2090
|
+
app.get('/api/connectors', async (_req, res) => {
|
|
2091
|
+
try {
|
|
2092
|
+
const { getConnectorManager } = await import('../core/connectors.js');
|
|
2093
|
+
const mgr = getConnectorManager(vaultRoot);
|
|
2094
|
+
const connectors = mgr.getAll().map(c => ({
|
|
2095
|
+
...c,
|
|
2096
|
+
exists: existsSync(c.path),
|
|
2097
|
+
}));
|
|
2098
|
+
res.json({ connectors });
|
|
2099
|
+
}
|
|
2100
|
+
catch (err) {
|
|
2101
|
+
res.status(500).json({ error: String(err) });
|
|
2102
|
+
}
|
|
2103
|
+
});
|
|
2104
|
+
// POST /api/connectors — add a new connector
|
|
2105
|
+
app.post('/api/connectors', async (req, res) => {
|
|
2106
|
+
try {
|
|
2107
|
+
const { getConnectorManager } = await import('../core/connectors.js');
|
|
2108
|
+
const mgr = getConnectorManager(vaultRoot);
|
|
2109
|
+
const body = req.body;
|
|
2110
|
+
if (!body.type) {
|
|
2111
|
+
res.status(400).json({ error: 'Missing type' });
|
|
2112
|
+
return;
|
|
2113
|
+
}
|
|
2114
|
+
let resolvedPath = body.path || '';
|
|
2115
|
+
// For git repos, clone if URL provided
|
|
2116
|
+
if (body.type === 'github' || (body.type === 'git-repo' && body.url)) {
|
|
2117
|
+
const reposDir = join(vaultRoot, '.wikimem-repos');
|
|
2118
|
+
mkdirSync(reposDir, { recursive: true });
|
|
2119
|
+
const repoName = (body.url ?? '').split('/').pop()?.replace('.git', '') || 'repo';
|
|
2120
|
+
resolvedPath = join(reposDir, repoName);
|
|
2121
|
+
if (!existsSync(resolvedPath)) {
|
|
2122
|
+
const result = await mgr.cloneRepo(body.url, resolvedPath);
|
|
2123
|
+
if (!result.success) {
|
|
2124
|
+
res.status(500).json({ error: result.error });
|
|
2125
|
+
return;
|
|
2126
|
+
}
|
|
2127
|
+
}
|
|
2128
|
+
}
|
|
2129
|
+
if (!resolvedPath || !existsSync(resolvedPath)) {
|
|
2130
|
+
res.status(400).json({ error: `Path does not exist: ${resolvedPath}` });
|
|
2131
|
+
return;
|
|
2132
|
+
}
|
|
2133
|
+
const connector = mgr.add({
|
|
2134
|
+
type: body.type,
|
|
2135
|
+
name: body.name || basename(resolvedPath),
|
|
2136
|
+
path: resolvedPath,
|
|
2137
|
+
url: body.url,
|
|
2138
|
+
includeGlobs: body.includeGlobs,
|
|
2139
|
+
excludeGlobs: body.excludeGlobs,
|
|
2140
|
+
autoSync: body.autoSync ?? false,
|
|
2141
|
+
});
|
|
2142
|
+
if (connector.autoSync)
|
|
2143
|
+
mgr.startWatcher(connector);
|
|
2144
|
+
res.json({ connector });
|
|
2145
|
+
}
|
|
2146
|
+
catch (err) {
|
|
2147
|
+
res.status(500).json({ error: String(err) });
|
|
2148
|
+
}
|
|
2149
|
+
});
|
|
2150
|
+
// DELETE /api/connectors/:id — remove a connector
|
|
2151
|
+
app.delete('/api/connectors/:id', async (req, res) => {
|
|
2152
|
+
try {
|
|
2153
|
+
const { getConnectorManager } = await import('../core/connectors.js');
|
|
2154
|
+
const mgr = getConnectorManager(vaultRoot);
|
|
2155
|
+
const removed = mgr.remove(req.params['id']);
|
|
2156
|
+
res.json({ removed });
|
|
2157
|
+
}
|
|
2158
|
+
catch (err) {
|
|
2159
|
+
res.status(500).json({ error: String(err) });
|
|
2160
|
+
}
|
|
2161
|
+
});
|
|
2162
|
+
// POST /api/connectors/:id/sync — manually trigger a sync (scan + ingest all files)
|
|
2163
|
+
app.post('/api/connectors/:id/sync', async (req, res) => {
|
|
2164
|
+
try {
|
|
2165
|
+
const { getConnectorManager } = await import('../core/connectors.js');
|
|
2166
|
+
const { ingestSource } = await import('../core/ingest.js');
|
|
2167
|
+
const mgr = getConnectorManager(vaultRoot);
|
|
2168
|
+
const connector = mgr.get(req.params['id']);
|
|
2169
|
+
if (!connector) {
|
|
2170
|
+
res.status(404).json({ error: 'Connector not found' });
|
|
2171
|
+
return;
|
|
2172
|
+
}
|
|
2173
|
+
mgr.updateStatus(connector.id, 'syncing');
|
|
2174
|
+
const startMs = Date.now();
|
|
2175
|
+
const files = await mgr.scanFiles(connector);
|
|
2176
|
+
let pagesCreated = 0, linksAdded = 0, ingested = 0;
|
|
2177
|
+
const errors = [];
|
|
2178
|
+
const { createProviderFromUserConfig } = await import('../providers/index.js');
|
|
2179
|
+
const { loadConfig } = await import('../core/config.js');
|
|
2180
|
+
const userConfig = loadConfig(config.configPath);
|
|
2181
|
+
const provider = createProviderFromUserConfig(userConfig);
|
|
2182
|
+
for (const filePath of files.slice(0, 50)) { // cap at 50 per sync
|
|
2183
|
+
try {
|
|
2184
|
+
const result = await ingestSource(filePath, config, provider, { verbose: false });
|
|
2185
|
+
pagesCreated += result.pagesUpdated ?? 0;
|
|
2186
|
+
linksAdded += result.linksAdded ?? 0;
|
|
2187
|
+
ingested++;
|
|
2188
|
+
}
|
|
2189
|
+
catch (e) {
|
|
2190
|
+
errors.push(`${basename(filePath)}: ${String(e).substring(0, 100)}`);
|
|
2191
|
+
}
|
|
2192
|
+
}
|
|
2193
|
+
mgr.updateStatus(connector.id, 'active', {
|
|
2194
|
+
lastSyncAt: new Date().toISOString(),
|
|
2195
|
+
totalFiles: files.length,
|
|
2196
|
+
});
|
|
2197
|
+
const result = {
|
|
2198
|
+
connectorId: connector.id,
|
|
2199
|
+
filesFound: files.length,
|
|
2200
|
+
filesIngested: ingested,
|
|
2201
|
+
pagesCreated,
|
|
2202
|
+
linksAdded,
|
|
2203
|
+
errors,
|
|
2204
|
+
duration: Date.now() - startMs,
|
|
2205
|
+
};
|
|
2206
|
+
res.json(result);
|
|
2207
|
+
}
|
|
2208
|
+
catch (err) {
|
|
2209
|
+
res.status(500).json({ error: String(err) });
|
|
2210
|
+
}
|
|
2211
|
+
});
|
|
2212
|
+
// POST /api/connectors/:id/scan — scan files in connector (no ingest)
|
|
2213
|
+
app.get('/api/connectors/:id/scan', async (req, res) => {
|
|
2214
|
+
try {
|
|
2215
|
+
const { getConnectorManager } = await import('../core/connectors.js');
|
|
2216
|
+
const mgr = getConnectorManager(vaultRoot);
|
|
2217
|
+
const connector = mgr.get(req.params['id']);
|
|
2218
|
+
if (!connector) {
|
|
2219
|
+
res.status(404).json({ error: 'Not found' });
|
|
2220
|
+
return;
|
|
2221
|
+
}
|
|
2222
|
+
const files = await mgr.scanFiles(connector);
|
|
2223
|
+
res.json({ files: files.slice(0, 200), total: files.length });
|
|
2224
|
+
}
|
|
2225
|
+
catch (err) {
|
|
2226
|
+
res.status(500).json({ error: String(err) });
|
|
2227
|
+
}
|
|
2228
|
+
});
|
|
2229
|
+
// Wire file-detected events from connectors to auto-ingest
|
|
2230
|
+
(async () => {
|
|
2231
|
+
const { getConnectorManager } = await import('../core/connectors.js');
|
|
2232
|
+
const { ingestSource } = await import('../core/ingest.js');
|
|
2233
|
+
const { createProviderFromUserConfig } = await import('../providers/index.js');
|
|
2234
|
+
const { loadConfig } = await import('../core/config.js');
|
|
2235
|
+
const mgr = getConnectorManager(vaultRoot);
|
|
2236
|
+
mgr.on('file-detected', async ({ connectorId, filePath }) => {
|
|
2237
|
+
const connector = mgr.get(connectorId);
|
|
2238
|
+
if (!connector)
|
|
2239
|
+
return;
|
|
2240
|
+
mgr.updateStatus(connectorId, 'syncing');
|
|
2241
|
+
try {
|
|
2242
|
+
const userConfig = loadConfig(config.configPath);
|
|
2243
|
+
const provider = createProviderFromUserConfig(userConfig);
|
|
2244
|
+
await ingestSource(filePath, config, provider, { verbose: false });
|
|
2245
|
+
mgr.updateStatus(connectorId, 'active', { lastSyncAt: new Date().toISOString() });
|
|
2246
|
+
}
|
|
2247
|
+
catch {
|
|
2248
|
+
mgr.updateStatus(connectorId, 'error', { errorMessage: `Failed to ingest ${basename(filePath)}` });
|
|
2249
|
+
}
|
|
2250
|
+
});
|
|
2251
|
+
mgr.startAllWatchers();
|
|
2252
|
+
})().catch(() => { });
|
|
2253
|
+
// Start Observer nightly cron (3am)
|
|
2254
|
+
import('../core/observer.js').then(({ startObserverCron }) => {
|
|
2255
|
+
startObserverCron(config);
|
|
2256
|
+
}).catch(() => { });
|
|
2257
|
+
// AUTO-005: Watch raw/ directory for new files, auto-trigger ingest
|
|
2258
|
+
(async () => {
|
|
2259
|
+
try {
|
|
2260
|
+
const chokidar = await import('chokidar');
|
|
2261
|
+
const { ingestSource } = await import('../core/ingest.js');
|
|
2262
|
+
const { createProviderFromUserConfig } = await import('../providers/index.js');
|
|
2263
|
+
const { loadConfig } = await import('../core/config.js');
|
|
2264
|
+
const rawDir = config.rawDir;
|
|
2265
|
+
if (!existsSync(rawDir))
|
|
2266
|
+
return;
|
|
2267
|
+
const watcher = chokidar.watch(rawDir, {
|
|
2268
|
+
ignored: ['**/.git/**', '**/*.tmp'],
|
|
2269
|
+
persistent: true,
|
|
2270
|
+
ignoreInitial: true,
|
|
2271
|
+
awaitWriteFinish: { stabilityThreshold: 1500 },
|
|
2272
|
+
});
|
|
2273
|
+
watcher.on('add', async (filePath) => {
|
|
2274
|
+
const INGESTIBLE = new Set(['.md', '.txt', '.pdf', '.json', '.yaml', '.yml', '.csv', '.html', '.docx', '.mp3', '.wav', '.m4a', '.mp4', '.mov']);
|
|
2275
|
+
const ext = filePath.split('.').pop()?.toLowerCase() || '';
|
|
2276
|
+
if (!INGESTIBLE.has('.' + ext))
|
|
2277
|
+
return;
|
|
2278
|
+
if (webhookIngestingFiles.has(filePath)) {
|
|
2279
|
+
console.log(`[watcher] Skipping ${filePath} (already being ingested by webhook)`);
|
|
2280
|
+
return;
|
|
2281
|
+
}
|
|
2282
|
+
console.log(`[watcher] New file detected: ${filePath}`);
|
|
2283
|
+
try {
|
|
2284
|
+
const userConfig = loadConfig(config.configPath);
|
|
2285
|
+
const provider = createProviderFromUserConfig(userConfig);
|
|
2286
|
+
await ingestSource(filePath, config, provider, { verbose: false });
|
|
2287
|
+
console.log(`[watcher] Ingested: ${filePath}`);
|
|
2288
|
+
}
|
|
2289
|
+
catch (err) {
|
|
2290
|
+
console.error(`[watcher] Failed to ingest ${filePath}:`, err);
|
|
2291
|
+
}
|
|
2292
|
+
});
|
|
2293
|
+
console.log(` Auto-watching raw/ for new files: ${rawDir}`);
|
|
2294
|
+
}
|
|
2295
|
+
catch { }
|
|
2296
|
+
})().catch(() => { });
|
|
2297
|
+
// Serve static files AFTER all API routes
|
|
2298
|
+
if (existsSync(publicDir)) {
|
|
2299
|
+
app.use(express.static(publicDir));
|
|
2300
|
+
}
|
|
229
2301
|
// Serve index.html for all other routes (SPA)
|
|
230
2302
|
app.get('/{*path}', (_req, res) => {
|
|
231
2303
|
const indexPath = join(publicDir, 'index.html');
|