wikimem 0.3.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +138 -29
- package/README.md +171 -309
- package/dist/cli/commands/ask.d.ts +3 -0
- package/dist/cli/commands/ask.d.ts.map +1 -0
- package/dist/cli/commands/ask.js +63 -0
- package/dist/cli/commands/ask.js.map +1 -0
- package/dist/cli/commands/export.d.ts +3 -0
- package/dist/cli/commands/export.d.ts.map +1 -0
- package/dist/cli/commands/export.js +108 -0
- package/dist/cli/commands/export.js.map +1 -0
- package/dist/cli/commands/history.d.ts +3 -0
- package/dist/cli/commands/history.d.ts.map +1 -0
- package/dist/cli/commands/history.js +61 -0
- package/dist/cli/commands/history.js.map +1 -0
- package/dist/cli/commands/improve.d.ts.map +1 -1
- package/dist/cli/commands/improve.js +4 -3
- package/dist/cli/commands/improve.js.map +1 -1
- package/dist/cli/commands/ingest.d.ts.map +1 -1
- package/dist/cli/commands/ingest.js +5 -4
- package/dist/cli/commands/ingest.js.map +1 -1
- package/dist/cli/commands/init.d.ts.map +1 -1
- package/dist/cli/commands/init.js +337 -81
- package/dist/cli/commands/init.js.map +1 -1
- package/dist/cli/commands/lint.d.ts.map +1 -1
- package/dist/cli/commands/lint.js +4 -3
- package/dist/cli/commands/lint.js.map +1 -1
- package/dist/cli/commands/mcp.d.ts +3 -0
- package/dist/cli/commands/mcp.d.ts.map +1 -0
- package/dist/cli/commands/mcp.js +11 -0
- package/dist/cli/commands/mcp.js.map +1 -0
- package/dist/cli/commands/open.d.ts +3 -0
- package/dist/cli/commands/open.d.ts.map +1 -0
- package/dist/cli/commands/open.js +36 -0
- package/dist/cli/commands/open.js.map +1 -0
- package/dist/cli/commands/query.d.ts.map +1 -1
- package/dist/cli/commands/query.js +5 -4
- package/dist/cli/commands/query.js.map +1 -1
- package/dist/cli/commands/search.d.ts +3 -0
- package/dist/cli/commands/search.d.ts.map +1 -0
- package/dist/cli/commands/search.js +61 -0
- package/dist/cli/commands/search.js.map +1 -0
- package/dist/cli/commands/serve.d.ts.map +1 -1
- package/dist/cli/commands/serve.js +41 -2
- package/dist/cli/commands/serve.js.map +1 -1
- package/dist/cli/commands/watch.d.ts.map +1 -1
- package/dist/cli/commands/watch.js +4 -3
- package/dist/cli/commands/watch.js.map +1 -1
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +27 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/core/audit-trail.d.ts +15 -0
- package/dist/core/audit-trail.d.ts.map +1 -0
- package/dist/core/audit-trail.js +43 -0
- package/dist/core/audit-trail.js.map +1 -0
- package/dist/core/claude-code.d.ts +10 -0
- package/dist/core/claude-code.d.ts.map +1 -0
- package/dist/core/claude-code.js +81 -0
- package/dist/core/claude-code.js.map +1 -0
- package/dist/core/config.d.ts +23 -0
- package/dist/core/config.d.ts.map +1 -1
- package/dist/core/config.js.map +1 -1
- package/dist/core/connectors.d.ts +58 -0
- package/dist/core/connectors.d.ts.map +1 -0
- package/dist/core/connectors.js +189 -0
- package/dist/core/connectors.js.map +1 -0
- package/dist/core/folder-scanner.d.ts +10 -0
- package/dist/core/folder-scanner.d.ts.map +1 -0
- package/dist/core/folder-scanner.js +84 -0
- package/dist/core/folder-scanner.js.map +1 -0
- package/dist/core/git.d.ts +137 -0
- package/dist/core/git.d.ts.map +1 -0
- package/dist/core/git.js +520 -0
- package/dist/core/git.js.map +1 -0
- package/dist/core/history.d.ts +21 -0
- package/dist/core/history.d.ts.map +1 -0
- package/dist/core/history.js +107 -0
- package/dist/core/history.js.map +1 -0
- package/dist/core/improve.d.ts.map +1 -1
- package/dist/core/improve.js +9 -0
- package/dist/core/improve.js.map +1 -1
- package/dist/core/ingest.d.ts +1 -0
- package/dist/core/ingest.d.ts.map +1 -1
- package/dist/core/ingest.js +151 -7
- package/dist/core/ingest.js.map +1 -1
- package/dist/core/lint.d.ts.map +1 -1
- package/dist/core/lint.js +23 -4
- package/dist/core/lint.js.map +1 -1
- package/dist/core/oauth-defaults.d.ts +31 -0
- package/dist/core/oauth-defaults.d.ts.map +1 -0
- package/dist/core/oauth-defaults.js +77 -0
- package/dist/core/oauth-defaults.js.map +1 -0
- package/dist/core/observer.d.ts +94 -0
- package/dist/core/observer.d.ts.map +1 -0
- package/dist/core/observer.js +492 -0
- package/dist/core/observer.js.map +1 -0
- package/dist/core/pipeline-events.d.ts +63 -0
- package/dist/core/pipeline-events.d.ts.map +1 -0
- package/dist/core/pipeline-events.js +109 -0
- package/dist/core/pipeline-events.js.map +1 -0
- package/dist/core/query.d.ts.map +1 -1
- package/dist/core/query.js +16 -8
- package/dist/core/query.js.map +1 -1
- package/dist/core/scraper.d.ts +41 -0
- package/dist/core/scraper.d.ts.map +1 -0
- package/dist/core/scraper.js +277 -0
- package/dist/core/scraper.js.map +1 -0
- package/dist/core/sync/gdrive.d.ts +14 -0
- package/dist/core/sync/gdrive.d.ts.map +1 -0
- package/dist/core/sync/gdrive.js +205 -0
- package/dist/core/sync/gdrive.js.map +1 -0
- package/dist/core/sync/github.d.ts +20 -0
- package/dist/core/sync/github.d.ts.map +1 -0
- package/dist/core/sync/github.js +206 -0
- package/dist/core/sync/github.js.map +1 -0
- package/dist/core/sync/gmail.d.ts +15 -0
- package/dist/core/sync/gmail.d.ts.map +1 -0
- package/dist/core/sync/gmail.js +159 -0
- package/dist/core/sync/gmail.js.map +1 -0
- package/dist/core/sync/index.d.ts +47 -0
- package/dist/core/sync/index.d.ts.map +1 -0
- package/dist/core/sync/index.js +100 -0
- package/dist/core/sync/index.js.map +1 -0
- package/dist/core/sync/jira.d.ts +15 -0
- package/dist/core/sync/jira.d.ts.map +1 -0
- package/dist/core/sync/jira.js +176 -0
- package/dist/core/sync/jira.js.map +1 -0
- package/dist/core/sync/linear.d.ts +15 -0
- package/dist/core/sync/linear.d.ts.map +1 -0
- package/dist/core/sync/linear.js +111 -0
- package/dist/core/sync/linear.js.map +1 -0
- package/dist/core/sync/notion.d.ts +14 -0
- package/dist/core/sync/notion.d.ts.map +1 -0
- package/dist/core/sync/notion.js +168 -0
- package/dist/core/sync/notion.js.map +1 -0
- package/dist/core/sync/rss.d.ts +20 -0
- package/dist/core/sync/rss.d.ts.map +1 -0
- package/dist/core/sync/rss.js +165 -0
- package/dist/core/sync/rss.js.map +1 -0
- package/dist/core/sync/scheduler.d.ts +31 -0
- package/dist/core/sync/scheduler.d.ts.map +1 -0
- package/dist/core/sync/scheduler.js +129 -0
- package/dist/core/sync/scheduler.js.map +1 -0
- package/dist/core/sync/slack.d.ts +16 -0
- package/dist/core/sync/slack.d.ts.map +1 -0
- package/dist/core/sync/slack.js +173 -0
- package/dist/core/sync/slack.js.map +1 -0
- package/dist/core/vault.d.ts +22 -0
- package/dist/core/vault.d.ts.map +1 -1
- package/dist/core/vault.js +65 -0
- package/dist/core/vault.js.map +1 -1
- package/dist/core/webhooks.d.ts +13 -0
- package/dist/core/webhooks.d.ts.map +1 -0
- package/dist/core/webhooks.js +206 -0
- package/dist/core/webhooks.js.map +1 -0
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/mcp-entry.d.ts +10 -0
- package/dist/mcp-entry.d.ts.map +1 -0
- package/dist/mcp-entry.js +21 -0
- package/dist/mcp-entry.js.map +1 -0
- package/dist/mcp-server.d.ts +20 -0
- package/dist/mcp-server.d.ts.map +1 -0
- package/dist/mcp-server.js +483 -0
- package/dist/mcp-server.js.map +1 -0
- package/dist/mcp-tools-extended.d.ts +15 -0
- package/dist/mcp-tools-extended.d.ts.map +1 -0
- package/dist/mcp-tools-extended.js +277 -0
- package/dist/mcp-tools-extended.js.map +1 -0
- package/dist/processors/audio.d.ts.map +1 -1
- package/dist/processors/audio.js +42 -4
- package/dist/processors/audio.js.map +1 -1
- package/dist/processors/csv.d.ts +18 -0
- package/dist/processors/csv.d.ts.map +1 -0
- package/dist/processors/csv.js +230 -0
- package/dist/processors/csv.js.map +1 -0
- package/dist/processors/image.d.ts.map +1 -1
- package/dist/processors/image.js +55 -27
- package/dist/processors/image.js.map +1 -1
- package/dist/processors/pdf.d.ts.map +1 -1
- package/dist/processors/pdf.js +6 -3
- package/dist/processors/pdf.js.map +1 -1
- package/dist/processors/pptx.d.ts +3 -1
- package/dist/processors/pptx.d.ts.map +1 -1
- package/dist/processors/pptx.js +236 -95
- package/dist/processors/pptx.js.map +1 -1
- package/dist/processors/url.js +4 -1
- package/dist/processors/url.js.map +1 -1
- package/dist/processors/xlsx.d.ts +2 -0
- package/dist/processors/xlsx.d.ts.map +1 -1
- package/dist/processors/xlsx.js +182 -46
- package/dist/processors/xlsx.js.map +1 -1
- package/dist/providers/claude.d.ts +1 -0
- package/dist/providers/claude.d.ts.map +1 -1
- package/dist/providers/claude.js +5 -3
- package/dist/providers/claude.js.map +1 -1
- package/dist/providers/index.d.ts +17 -1
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js +144 -0
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/openai.d.ts +1 -0
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +5 -3
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/types.d.ts +18 -0
- package/dist/providers/types.d.ts.map +1 -1
- package/dist/templates/config-yaml.d.ts.map +1 -1
- package/dist/templates/config-yaml.js +12 -1
- package/dist/templates/config-yaml.js.map +1 -1
- package/dist/templates/source-types.d.ts +33 -0
- package/dist/templates/source-types.d.ts.map +1 -0
- package/dist/templates/source-types.js +178 -0
- package/dist/templates/source-types.js.map +1 -0
- package/dist/web/public/index.html +9836 -742
- package/dist/web/server.d.ts.map +1 -1
- package/dist/web/server.js +2823 -43
- package/dist/web/server.js.map +1 -1
- package/package.json +10 -4
- package/scripts/install.sh +54 -0
- package/src/web/public/index.html +9836 -742
- package/templates/mcp-config.json +9 -0
- package/templates/source-types/article.md +21 -0
- package/templates/source-types/book.md +21 -0
- package/templates/source-types/paper.md +23 -0
- package/templates/source-types/podcast.md +21 -0
- package/templates/source-types/raw-notes.md +17 -0
- package/templates/source-types/tweet-thread.md +19 -0
- package/templates/source-types/video.md +21 -0
- package/dist/web/public/public/index.html +0 -946
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extended MCP tool handlers for WikiMem.
|
|
3
|
+
*
|
|
4
|
+
* Handles: wikimem_observe, wikimem_improve, wikimem_pipeline,
|
|
5
|
+
* wikimem_scrape, wikimem_connectors
|
|
6
|
+
*
|
|
7
|
+
* All imports are dynamic to keep MCP server startup fast.
|
|
8
|
+
*/
|
|
9
|
+
import { basename, dirname, join } from 'node:path';
|
|
10
|
+
import { fileURLToPath } from 'node:url';
|
|
11
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
12
|
+
const __dirname = dirname(__filename);
|
|
13
|
+
// ─── wikimem_observe ──────────────────────────────────────────────────────
|
|
14
|
+
export async function handleObserve(config, args) {
|
|
15
|
+
const pageArg = typeof args['page'] === 'string' ? args['page'].trim() : null;
|
|
16
|
+
const { runObserver } = await import('./core/observer.js');
|
|
17
|
+
const report = await runObserver(config, { maxPagesToReview: pageArg ? 1 : undefined });
|
|
18
|
+
if (pageArg) {
|
|
19
|
+
const target = pageArg.toLowerCase();
|
|
20
|
+
const match = report.scores.find((s) => s.title.toLowerCase() === target
|
|
21
|
+
|| basename(s.page, '.md').toLowerCase() === target);
|
|
22
|
+
if (!match) {
|
|
23
|
+
return {
|
|
24
|
+
found: false,
|
|
25
|
+
message: `No page found matching "${pageArg}". Use wikimem_list to see available pages.`,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
return {
|
|
29
|
+
found: true,
|
|
30
|
+
page: match.title,
|
|
31
|
+
score: match.score,
|
|
32
|
+
maxScore: match.maxScore,
|
|
33
|
+
breakdown: match.breakdown,
|
|
34
|
+
issues: match.issues,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
const weakPages = report.scores
|
|
38
|
+
.filter((s) => s.score < s.maxScore * 0.5)
|
|
39
|
+
.slice(0, 20)
|
|
40
|
+
.map((s) => ({ title: s.title, score: s.score, maxScore: s.maxScore, issues: s.issues }));
|
|
41
|
+
return {
|
|
42
|
+
date: report.date,
|
|
43
|
+
totalPages: report.totalPages,
|
|
44
|
+
pagesReviewed: report.pagesReviewed,
|
|
45
|
+
averageScore: report.averageScore,
|
|
46
|
+
maxScore: report.maxScore,
|
|
47
|
+
weakPages,
|
|
48
|
+
orphanCount: report.orphans.length,
|
|
49
|
+
orphans: report.orphans.slice(0, 15).map((o) => o.title),
|
|
50
|
+
gapCount: report.gaps.length,
|
|
51
|
+
gaps: report.gaps.slice(0, 10),
|
|
52
|
+
contradictions: report.contradictions.slice(0, 10),
|
|
53
|
+
topIssues: report.topIssues,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
// ─── wikimem_improve ──────────────────────────────────────────────────────
|
|
57
|
+
export async function handleImprove(config, args) {
|
|
58
|
+
const pageArg = typeof args['page'] === 'string' ? args['page'].trim() : null;
|
|
59
|
+
const autoApply = args['autoApply'] === true;
|
|
60
|
+
const { loadConfig } = await import('./core/config.js');
|
|
61
|
+
const userConfig = loadConfig(config.configPath);
|
|
62
|
+
const { createProviderFromUserConfig } = await import('./providers/index.js');
|
|
63
|
+
const provider = createProviderFromUserConfig(userConfig);
|
|
64
|
+
const { improveWiki } = await import('./core/improve.js');
|
|
65
|
+
const result = await improveWiki(config, provider, {
|
|
66
|
+
threshold: 80,
|
|
67
|
+
dryRun: !autoApply,
|
|
68
|
+
});
|
|
69
|
+
let actions = result.actions;
|
|
70
|
+
if (pageArg) {
|
|
71
|
+
const target = pageArg.toLowerCase();
|
|
72
|
+
actions = actions.filter((a) => a.description.toLowerCase().includes(target));
|
|
73
|
+
}
|
|
74
|
+
return {
|
|
75
|
+
score: result.score,
|
|
76
|
+
dimensions: result.dimensions,
|
|
77
|
+
actionCount: actions.length,
|
|
78
|
+
actions: actions.map((a) => ({
|
|
79
|
+
type: a.type,
|
|
80
|
+
description: a.description,
|
|
81
|
+
applied: a.applied,
|
|
82
|
+
})),
|
|
83
|
+
autoApply,
|
|
84
|
+
message: autoApply
|
|
85
|
+
? `Applied ${actions.filter((a) => a.applied).length}/${actions.length} improvements.`
|
|
86
|
+
: `Found ${actions.length} improvement(s). Set autoApply=true to apply them.`,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
// ─── wikimem_pipeline ─────────────────────────────────────────────────────
|
|
90
|
+
export async function handlePipeline(vaultRoot, args) {
|
|
91
|
+
const limit = typeof args['limit'] === 'number' ? args['limit'] : 10;
|
|
92
|
+
const { pipelineEvents } = await import('./core/pipeline-events.js');
|
|
93
|
+
pipelineEvents.initPersistence(vaultRoot);
|
|
94
|
+
const recentRuns = pipelineEvents.getRecentRuns().slice(0, limit);
|
|
95
|
+
const currentRun = pipelineEvents.getCurrentRun();
|
|
96
|
+
const { getConnectorManager } = await import('./core/connectors.js');
|
|
97
|
+
const cm = getConnectorManager(vaultRoot);
|
|
98
|
+
const connectors = cm.getAll();
|
|
99
|
+
return {
|
|
100
|
+
currentRun: currentRun
|
|
101
|
+
? {
|
|
102
|
+
id: currentRun.id,
|
|
103
|
+
source: currentRun.source,
|
|
104
|
+
startedAt: currentRun.startedAt,
|
|
105
|
+
stepCount: currentRun.events.length,
|
|
106
|
+
lastStep: currentRun.events[currentRun.events.length - 1] ?? null,
|
|
107
|
+
}
|
|
108
|
+
: null,
|
|
109
|
+
recentRuns: recentRuns.map((r) => ({
|
|
110
|
+
id: r.id,
|
|
111
|
+
source: r.source,
|
|
112
|
+
startedAt: r.startedAt,
|
|
113
|
+
status: r.events[r.events.length - 1]?.step ?? 'unknown',
|
|
114
|
+
pagesCreated: r.result?.pagesCreated ?? 0,
|
|
115
|
+
linksAdded: r.result?.linksAdded ?? 0,
|
|
116
|
+
title: r.result?.title ?? '',
|
|
117
|
+
})),
|
|
118
|
+
connectorHealth: connectors.map((c) => ({
|
|
119
|
+
id: c.id,
|
|
120
|
+
name: c.name,
|
|
121
|
+
type: c.type,
|
|
122
|
+
status: c.status,
|
|
123
|
+
lastSyncAt: c.lastSyncAt ?? null,
|
|
124
|
+
totalFiles: c.totalFiles ?? 0,
|
|
125
|
+
errorMessage: c.errorMessage ?? null,
|
|
126
|
+
})),
|
|
127
|
+
totalRuns: recentRuns.length,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
// ─── wikimem_scrape ───────────────────────────────────────────────────────
|
|
131
|
+
export async function handleScrape(vaultRoot, args) {
|
|
132
|
+
const url = args['url'];
|
|
133
|
+
if (typeof url !== 'string' || !url.trim()) {
|
|
134
|
+
throw { code: -32602, message: 'url must be a non-empty string' };
|
|
135
|
+
}
|
|
136
|
+
const { spawn } = await import('node:child_process');
|
|
137
|
+
const wikimemBin = join(__dirname, 'index.js');
|
|
138
|
+
const cliArgs = [wikimemBin, 'ingest', url, '--vault', vaultRoot];
|
|
139
|
+
const tags = args['tags'];
|
|
140
|
+
if (Array.isArray(tags) && tags.length > 0) {
|
|
141
|
+
cliArgs.push('--tags', tags.join(','));
|
|
142
|
+
}
|
|
143
|
+
const result = await new Promise((res) => {
|
|
144
|
+
const proc = spawn(process.execPath, cliArgs, { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
145
|
+
let output = '';
|
|
146
|
+
proc.stdout?.on('data', (d) => { output += d.toString(); });
|
|
147
|
+
proc.stderr?.on('data', (d) => { output += d.toString(); });
|
|
148
|
+
proc.on('close', (code) => res({ success: code === 0, output }));
|
|
149
|
+
proc.on('error', (err) => res({ success: false, output: err.message }));
|
|
150
|
+
});
|
|
151
|
+
return {
|
|
152
|
+
url,
|
|
153
|
+
tags: Array.isArray(tags) ? tags : [],
|
|
154
|
+
success: result.success,
|
|
155
|
+
output: result.output.trim(),
|
|
156
|
+
message: result.success
|
|
157
|
+
? `Successfully scraped and ingested: ${url}`
|
|
158
|
+
: `Scrape/ingest failed. See output for details.`,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
// ─── wikimem_connectors ───────────────────────────────────────────────────
|
|
162
|
+
export async function handleConnectors(vaultRoot, args) {
|
|
163
|
+
const action = args['action'];
|
|
164
|
+
if (typeof action !== 'string') {
|
|
165
|
+
throw { code: -32602, message: 'action is required (list | add | remove | sync)' };
|
|
166
|
+
}
|
|
167
|
+
const { getConnectorManager } = await import('./core/connectors.js');
|
|
168
|
+
const cm = getConnectorManager(vaultRoot);
|
|
169
|
+
switch (action) {
|
|
170
|
+
case 'list': {
|
|
171
|
+
const connectors = cm.getAll();
|
|
172
|
+
return {
|
|
173
|
+
count: connectors.length,
|
|
174
|
+
connectors: connectors.map((c) => ({
|
|
175
|
+
id: c.id,
|
|
176
|
+
name: c.name,
|
|
177
|
+
type: c.type,
|
|
178
|
+
path: c.path,
|
|
179
|
+
url: c.url ?? null,
|
|
180
|
+
status: c.status,
|
|
181
|
+
autoSync: c.autoSync ?? false,
|
|
182
|
+
lastSyncAt: c.lastSyncAt ?? null,
|
|
183
|
+
totalFiles: c.totalFiles ?? 0,
|
|
184
|
+
createdAt: c.createdAt,
|
|
185
|
+
})),
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
case 'add': {
|
|
189
|
+
const connName = args['name'];
|
|
190
|
+
const connType = args['type'];
|
|
191
|
+
const connPath = args['path'];
|
|
192
|
+
if (typeof connName !== 'string' || !connName.trim()) {
|
|
193
|
+
throw { code: -32602, message: 'name is required for add action' };
|
|
194
|
+
}
|
|
195
|
+
if (connType !== 'folder' && connType !== 'git-repo') {
|
|
196
|
+
throw { code: -32602, message: 'type must be "folder" or "git-repo"' };
|
|
197
|
+
}
|
|
198
|
+
if (typeof connPath !== 'string' || !connPath.trim()) {
|
|
199
|
+
throw { code: -32602, message: 'path is required for add action' };
|
|
200
|
+
}
|
|
201
|
+
const connector = cm.add({
|
|
202
|
+
name: connName.trim(),
|
|
203
|
+
type: connType,
|
|
204
|
+
path: connPath.trim(),
|
|
205
|
+
url: typeof args['url'] === 'string' ? args['url'] : undefined,
|
|
206
|
+
autoSync: args['autoSync'] === true,
|
|
207
|
+
includeGlobs: Array.isArray(args['includeGlobs']) ? args['includeGlobs'] : undefined,
|
|
208
|
+
});
|
|
209
|
+
return {
|
|
210
|
+
success: true,
|
|
211
|
+
connector: {
|
|
212
|
+
id: connector.id,
|
|
213
|
+
name: connector.name,
|
|
214
|
+
type: connector.type,
|
|
215
|
+
path: connector.path,
|
|
216
|
+
status: connector.status,
|
|
217
|
+
autoSync: connector.autoSync ?? false,
|
|
218
|
+
},
|
|
219
|
+
message: `Connector "${connector.name}" added with ID ${connector.id}.`,
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
case 'remove': {
|
|
223
|
+
const removeId = args['id'];
|
|
224
|
+
if (typeof removeId !== 'string' || !removeId.trim()) {
|
|
225
|
+
throw { code: -32602, message: 'id is required for remove action' };
|
|
226
|
+
}
|
|
227
|
+
const removed = cm.remove(removeId.trim());
|
|
228
|
+
return {
|
|
229
|
+
success: removed,
|
|
230
|
+
message: removed
|
|
231
|
+
? `Connector ${removeId} removed.`
|
|
232
|
+
: `No connector found with ID ${removeId}.`,
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
case 'sync': {
|
|
236
|
+
const syncId = args['id'];
|
|
237
|
+
if (typeof syncId !== 'string' || !syncId.trim()) {
|
|
238
|
+
throw { code: -32602, message: 'id is required for sync action' };
|
|
239
|
+
}
|
|
240
|
+
const connector = cm.get(syncId.trim());
|
|
241
|
+
if (!connector) {
|
|
242
|
+
return {
|
|
243
|
+
success: false,
|
|
244
|
+
message: `No connector found with ID ${syncId}.`,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
cm.updateStatus(connector.id, 'syncing');
|
|
248
|
+
try {
|
|
249
|
+
const files = await cm.scanFiles(connector);
|
|
250
|
+
cm.updateStatus(connector.id, 'active', {
|
|
251
|
+
lastSyncAt: new Date().toISOString(),
|
|
252
|
+
totalFiles: files.length,
|
|
253
|
+
});
|
|
254
|
+
return {
|
|
255
|
+
success: true,
|
|
256
|
+
connectorId: connector.id,
|
|
257
|
+
connectorName: connector.name,
|
|
258
|
+
filesFound: files.length,
|
|
259
|
+
message: `Sync complete for "${connector.name}": found ${files.length} ingestible files.`,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
catch (err) {
|
|
263
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
264
|
+
cm.updateStatus(connector.id, 'error', { errorMessage: errMsg });
|
|
265
|
+
return {
|
|
266
|
+
success: false,
|
|
267
|
+
connectorId: connector.id,
|
|
268
|
+
error: errMsg,
|
|
269
|
+
message: `Sync failed for "${connector.name}": ${errMsg}`,
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
default:
|
|
274
|
+
throw { code: -32602, message: `Unknown connector action: ${action}. Use list, add, remove, or sync.` };
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
//# sourceMappingURL=mcp-tools-extended.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mcp-tools-extended.js","sourceRoot":"","sources":["../src/mcp-tools-extended.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAGzC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AAEtC,6EAA6E;AAE7E,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAmB,EACnB,IAA6B;IAE7B,MAAM,OAAO,GAAG,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9E,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAE3D,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;IAExF,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,MAAM;eAClC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM,CACtD,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO;gBACL,KAAK,EAAE,KAAK;gBACZ,OAAO,EAAE,2BAA2B,OAAO,6CAA6C;aACzF,CAAC;QACJ,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,KAAK,CAAC,KAAK;YACjB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM;SAC5B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC;SACzC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAE5F,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,SAAS;QACT,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;QAClC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QACxD,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;QAC5B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QAC9B,cAAc,EAAE,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QAClD,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,6EAA6E;AAE7E,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAmB,EACnB,IAA6B;IAE7B,MAAM,OAAO,GAAG,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9E,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,IAAI,CAAC;IAE7C,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;IACxD,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjD,MAAM,EAAE,4BAA4B,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAC9E,MAAM,QAAQ,GAAG,4BAA4B,CAAC,UAAU,CAAC,CAAC;IAE1D,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE;QACjD,SAAS,EAAE,EAAE;QACb,MAAM,EAAE,CAAC,SAAS;KACnB,CAAC,CAAC;IAEH,IAAI,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC7B,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QACrC,OAAO,GAAG,OAAO,CAAC,MAAM,CACtB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CACpD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC,CAAC;QACH,SAAS;QACT,OAAO,EAAE,SAAS;YAChB,CAAC,CAAC,WAAW,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,gBAAgB;YACtF,CAAC,CAAC,SAAS,OAAO,CAAC,MAAM,oDAAoD;KAChF,CAAC;AACJ,CAAC;AAED,6EAA6E;AAE7E,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAiB,EACjB,IAA6B;IAE7B,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAErE,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;IACrE,cAAc,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;IAE1C,MAAM,UAAU,GAAG,cAAc,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAClE,MAAM,UAAU,GAAG,cAAc,CAAC,aAAa,EAAE,CAAC;IAElD,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IACrE,MAAM,EAAE,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;IAE/B,OAAO;QACL,UAAU,EAAE,UAAU;YACpB,CAAC,CAAC;gBACE,EAAE,EAAE,UAAU,CAAC,EAAE;gBACjB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,MAAM;gBACnC,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI;aAClE;YACH,CAAC,CAAC,IAAI;QACR,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACjC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,IAAI,SAAS;YACxD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,YAAY,IAAI,CAAC;YACzC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,UAAU,IAAI,CAAC;YACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE;SAC7B,CAAC,CAAC;QACH,eAAe,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACtC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,IAAI;YAChC,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,CAAC;YAC7B,YAAY,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI;SACrC,CAAC,CAAC;QACH,SAAS,EAAE,UAAU,CAAC,MAAM;KAC7B,CAAC;AACJ,CAAC;AAED,6EAA6E;AAE7E,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAiB,EACjB,IAA6B;IAE7B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IACxB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;QAC3C,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;IACpE,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAE/C,MAAM,OAAO,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAElE,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3C,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,IAAI,OAAO,CAAuC,CAAC,GAAG,EAAE,EAAE;QAC7E,MAAM,IAAI,GAAG,KAAK,CAChB,OAAO,CAAC,QAAQ,EAChB,OAAO,EACP,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACtC,CAAC;QAEF,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACjE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,GAAG;QACH,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE;QACrC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;QAC5B,OAAO,EAAE,MAAM,CAAC,OAAO;YACrB,CAAC,CAAC,sCAAsC,GAAG,EAAE;YAC7C,CAAC,CAAC,+CAA+C;KACpD,CAAC;AACJ,CAAC;AAED,6EAA6E;AAE7E,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,SAAiB,EACjB,IAA6B;IAE7B,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC9B,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,iDAAiD,EAAE,CAAC;IACrF,CAAC;IAED,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IACrE,MAAM,EAAE,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IAE1C,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,UAAU,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;YAC/B,OAAO;gBACL,KAAK,EAAE,UAAU,CAAC,MAAM;gBACxB,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACjC,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,IAAI;oBAClB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,KAAK;oBAC7B,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,IAAI;oBAChC,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,CAAC;oBAC7B,SAAS,EAAE,CAAC,CAAC,SAAS;iBACvB,CAAC,CAAC;aACJ,CAAC;QACJ,CAAC;QAED,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;gBACrD,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC;YACrE,CAAC;YACD,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;gBACrD,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC;YACzE,CAAC;YACD,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;gBACrD,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC;YACrE,CAAC;YAED,MAAM,SAAS,GAAG,EAAE,CAAC,GAAG,CAAC;gBACvB,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE;gBACrB,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE;gBACrB,GAAG,EAAE,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC9D,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI;gBACnC,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAa,CAAC,CAAC,CAAC,SAAS;aACjG,CAAC,CAAC;YAEH,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,SAAS,EAAE;oBACT,EAAE,EAAE,SAAS,CAAC,EAAE;oBAChB,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,IAAI,EAAE,SAAS,CAAC,IAAI;oBACpB,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,QAAQ,EAAE,SAAS,CAAC,QAAQ,IAAI,KAAK;iBACtC;gBACD,OAAO,EAAE,cAAc,SAAS,CAAC,IAAI,mBAAmB,SAAS,CAAC,EAAE,GAAG;aACxE,CAAC;QACJ,CAAC;QAED,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;gBACrD,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,kCAAkC,EAAE,CAAC;YACtE,CAAC;YACD,MAAM,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3C,OAAO;gBACL,OAAO,EAAE,OAAO;gBAChB,OAAO,EAAE,OAAO;oBACd,CAAC,CAAC,aAAa,QAAQ,WAAW;oBAClC,CAAC,CAAC,8BAA8B,QAAQ,GAAG;aAC9C,CAAC;QACJ,CAAC;QAED,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;gBACjD,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;YACpE,CAAC;YACD,MAAM,SAAS,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YACxC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,OAAO,EAAE,8BAA8B,MAAM,GAAG;iBACjD,CAAC;YACJ,CAAC;YAED,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;YAEzC,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAC5C,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE;oBACtC,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACpC,UAAU,EAAE,KAAK,CAAC,MAAM;iBACzB,CAAC,CAAC;gBAEH,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,aAAa,EAAE,SAAS,CAAC,IAAI;oBAC7B,UAAU,EAAE,KAAK,CAAC,MAAM;oBACxB,OAAO,EAAE,sBAAsB,SAAS,CAAC,IAAI,YAAY,KAAK,CAAC,MAAM,oBAAoB;iBAC1F,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAChE,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAC;gBACjE,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,WAAW,EAAE,SAAS,CAAC,EAAE;oBACzB,KAAK,EAAE,MAAM;oBACb,OAAO,EAAE,oBAAoB,SAAS,CAAC,IAAI,MAAM,MAAM,EAAE;iBAC1D,CAAC;YACJ,CAAC;QACH,CAAC;QAED;YACE,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,6BAA6B,MAAM,mCAAmC,EAAE,CAAC;IAC5G,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"audio.d.ts","sourceRoot":"","sources":["../../src/processors/audio.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAID,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAErD;AAED,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,
|
|
1
|
+
{"version":3,"file":"audio.d.ts","sourceRoot":"","sources":["../../src/processors/audio.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAID,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAErD;AAED,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAgCzE"}
|
package/dist/processors/audio.js
CHANGED
|
@@ -12,12 +12,17 @@ export async function processAudio(filePath) {
|
|
|
12
12
|
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
|
13
13
|
throw new Error(`Unsupported audio format: ${ext}. Supported: ${[...SUPPORTED_EXTENSIONS].join(', ')}`);
|
|
14
14
|
}
|
|
15
|
-
// Try
|
|
15
|
+
// Try OpenAI Whisper API first (recommended)
|
|
16
|
+
const openaiKey = process.env['OPENAI_API_KEY'];
|
|
17
|
+
if (openaiKey) {
|
|
18
|
+
return await transcribeWithOpenAIWhisper(filePath, title, openaiKey);
|
|
19
|
+
}
|
|
20
|
+
// Try Deepgram API
|
|
16
21
|
const deepgramKey = process.env['DEEPGRAM_API_KEY'];
|
|
17
22
|
if (deepgramKey) {
|
|
18
23
|
return await transcribeWithDeepgram(filePath, title, deepgramKey);
|
|
19
24
|
}
|
|
20
|
-
// Try local Whisper CLI
|
|
25
|
+
// Try local Whisper CLI (pip install openai-whisper)
|
|
21
26
|
if (isWhisperAvailable()) {
|
|
22
27
|
return await transcribeWithWhisper(filePath, title);
|
|
23
28
|
}
|
|
@@ -29,6 +34,39 @@ export async function processAudio(filePath) {
|
|
|
29
34
|
sourcePath: filePath,
|
|
30
35
|
};
|
|
31
36
|
}
|
|
37
|
+
async function transcribeWithOpenAIWhisper(filePath, title, apiKey) {
|
|
38
|
+
const { createReadStream } = await import('node:fs');
|
|
39
|
+
const FormData = (await import('node:buffer')).Blob ? null : null;
|
|
40
|
+
const audioData = readFileSync(filePath);
|
|
41
|
+
const ext = extname(filePath).toLowerCase();
|
|
42
|
+
const boundary = '----WikiMemBoundary' + Date.now();
|
|
43
|
+
const filename = basename(filePath);
|
|
44
|
+
const contentType = getContentType(ext);
|
|
45
|
+
const prefix = Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${filename}"\r\nContent-Type: ${contentType}\r\n\r\n`);
|
|
46
|
+
const suffix = Buffer.from(`\r\n--${boundary}\r\nContent-Disposition: form-data; name="model"\r\n\r\nwhisper-1\r\n--${boundary}\r\nContent-Disposition: form-data; name="response_format"\r\n\r\ntext\r\n--${boundary}--\r\n`);
|
|
47
|
+
const body = Buffer.concat([prefix, audioData, suffix]);
|
|
48
|
+
const response = await fetch('https://api.openai.com/v1/audio/transcriptions', {
|
|
49
|
+
method: 'POST',
|
|
50
|
+
headers: {
|
|
51
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
52
|
+
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
|
53
|
+
},
|
|
54
|
+
body,
|
|
55
|
+
});
|
|
56
|
+
if (!response.ok) {
|
|
57
|
+
const errText = await response.text().catch(() => '');
|
|
58
|
+
throw new Error(`OpenAI Whisper API error: ${response.status} ${errText.substring(0, 200)}`);
|
|
59
|
+
}
|
|
60
|
+
const transcript = (await response.text()).trim();
|
|
61
|
+
const duration = getDuration(filePath);
|
|
62
|
+
return {
|
|
63
|
+
title,
|
|
64
|
+
transcript,
|
|
65
|
+
markdown: buildMarkdown(title, filePath, transcript, duration),
|
|
66
|
+
duration,
|
|
67
|
+
sourcePath: filePath,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
32
70
|
async function transcribeWithWhisper(filePath, title) {
|
|
33
71
|
const tmpOutput = join(tmpdir(), `wikimem-whisper-${Date.now()}`);
|
|
34
72
|
const result = spawnSync('whisper', [
|
|
@@ -71,8 +109,8 @@ async function transcribeWithDeepgram(filePath, title, apiKey) {
|
|
|
71
109
|
throw new Error(`Deepgram API error: ${response.status} ${response.statusText}`);
|
|
72
110
|
}
|
|
73
111
|
const data = (await response.json());
|
|
74
|
-
const transcript = data.results
|
|
75
|
-
const durationSec = data.metadata
|
|
112
|
+
const transcript = data.results?.channels?.[0]?.alternatives?.[0]?.transcript ?? '';
|
|
113
|
+
const durationSec = data.metadata?.duration;
|
|
76
114
|
const duration = durationSec ? formatDuration(durationSec) : undefined;
|
|
77
115
|
return {
|
|
78
116
|
title,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"audio.js","sourceRoot":"","sources":["../../src/processors/audio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAUjC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAEhG,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,OAAO,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAEtC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,gBAAgB,CAAC,GAAG,oBAAoB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1G,CAAC;IAED,
|
|
1
|
+
{"version":3,"file":"audio.js","sourceRoot":"","sources":["../../src/processors/audio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAUjC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAEhG,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,OAAO,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAEtC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,gBAAgB,CAAC,GAAG,oBAAoB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1G,CAAC;IAED,6CAA6C;IAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAChD,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,MAAM,2BAA2B,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IACvE,CAAC;IAED,mBAAmB;IACnB,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IACpD,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,MAAM,sBAAsB,CAAC,QAAQ,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC;IACpE,CAAC;IAED,qDAAqD;IACrD,IAAI,kBAAkB,EAAE,EAAE,CAAC;QACzB,OAAO,MAAM,qBAAqB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACtD,CAAC;IAED,4CAA4C;IAC5C,OAAO;QACL,KAAK;QACL,UAAU,EAAE,EAAE;QACd,QAAQ,EAAE,aAAa,CAAC,KAAK,EAAE,QAAQ,EAAE,0EAA0E,CAAC;QACpH,UAAU,EAAE,QAAQ;KACrB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,2BAA2B,CAAC,QAAgB,EAAE,KAAa,EAAE,MAAc;IACxF,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACrD,MAAM,QAAQ,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAElE,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAE5C,MAAM,QAAQ,GAAG,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACpC,MAAM,WAAW,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAExC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,KAAK,QAAQ,8DAA8D,QAAQ,sBAAsB,WAAW,UAAU,CAC/H,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,SAAS,QAAQ,0EAA0E,QAAQ,+EAA+E,QAAQ,QAAQ,CACnM,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;IAExD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,gDAAgD,EAAE;QAC7E,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,eAAe,EAAE,UAAU,MAAM,EAAE;YACnC,cAAc,EAAE,iCAAiC,QAAQ,EAAE;SAC5D;QACD,IAAI;KACL,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACtD,MAAM,IAAI,KAAK,CAAC,6BAA6B,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/F,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IAEvC,OAAO;QACL,KAAK;QACL,UAAU;QACV,QAAQ,EAAE,aAAa,CAAC,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC;QAC9D,QAAQ;QACR,UAAU,EAAE,QAAQ;KACrB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,QAAgB,EAAE,KAAa;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,mBAAmB,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAElE,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE;QAClC,QAAQ;QACR,SAAS,EAAE,MAAM;QACjB,iBAAiB,EAAE,KAAK;QACxB,cAAc,EAAE,MAAM,EAAE;QACxB,QAAQ,EAAE,OAAO;KAClB,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAE3C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,4BAA4B;IAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,GAAG,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/E,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEpF,WAAW;IACX,IAAI,UAAU,CAAC,OAAO,CAAC;QAAE,UAAU,CAAC,OAAO,CAAC,CAAC;IAE7C,wCAAwC;IACxC,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IAEvC,OAAO;QACL,KAAK;QACL,UAAU;QACV,QAAQ,EAAE,aAAa,CAAC,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC;QAC9D,QAAQ;QACR,UAAU,EAAE,QAAQ;KACrB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,QAAgB,EAAE,KAAa,EAAE,MAAc;IACnF,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEzC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,mFAAmF,EAAE;QAChH,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,eAAe,EAAE,SAAS,MAAM,EAAE;YAClC,cAAc,EAAE,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;SAClD;QACD,IAAI,EAAE,SAAS;KAChB,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACnF,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAOlC,CAAC;IAEF,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,EAAE,CAAC;IACpF,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5C,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvE,OAAO;QACL,KAAK;QACL,UAAU;QACV,QAAQ,EAAE,aAAa,CAAC,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC;QAC9D,QAAQ;QACR,UAAU,EAAE,QAAQ;KACrB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,KAAa,EAAE,QAAgB,EAAE,UAAkB,EAAE,QAAiB;IAC3F,OAAO,KAAK,KAAK;;iBAEF,QAAQ,CAAC,QAAQ,CAAC,KAAK,QAAQ;mBAC7B,QAAQ,CAAC,CAAC,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;mBAC/C,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;;;;EAIvD,UAAU,IAAI,4BAA4B;CAC3C,CAAC;AACF,CAAC;AAED,SAAS,kBAAkB;IACzB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACtF,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,QAAgB;IACnC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE;YAClC,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,iBAAiB;YACjD,KAAK,EAAE,oCAAoC,EAAE,QAAQ;SACtD,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACjD,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IAC9D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,OAAe;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC;IACtC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;IAChC,OAAO,GAAG,CAAC,GAAG,CAAC;AACjB,CAAC;AAED,SAAS,cAAc,CAAC,GAAW;IACjC,QAAQ,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC,CAAC,OAAO,YAAY,CAAC;QACjC,KAAK,MAAM,CAAC,CAAC,OAAO,WAAW,CAAC;QAChC,KAAK,MAAM,CAAC,CAAC,OAAO,WAAW,CAAC;QAChC,KAAK,MAAM,CAAC,CAAC,OAAO,WAAW,CAAC;QAChC,KAAK,OAAO,CAAC,CAAC,OAAO,YAAY,CAAC;QAClC,OAAO,CAAC,CAAC,OAAO,YAAY,CAAC;IAC/B,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CSV (.csv, .tsv) processor.
|
|
3
|
+
* Pure Node.js parser — no external dependencies.
|
|
4
|
+
* Handles: quoted fields, commas in values, newlines in quotes, BOM, TSV.
|
|
5
|
+
*/
|
|
6
|
+
export interface CsvResult {
|
|
7
|
+
title: string;
|
|
8
|
+
content: string;
|
|
9
|
+
markdown: string;
|
|
10
|
+
rowCount: number;
|
|
11
|
+
columnCount: number;
|
|
12
|
+
sourcePath: string;
|
|
13
|
+
}
|
|
14
|
+
/** Check whether a file path looks like CSV/TSV. */
|
|
15
|
+
export declare function isCsvFile(filePath: string): boolean;
|
|
16
|
+
/** Process a CSV or TSV file into structured output with markdown table. */
|
|
17
|
+
export declare function processCsv(filePath: string): Promise<CsvResult>;
|
|
18
|
+
//# sourceMappingURL=csv.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/processors/csv.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;CACpB;AAKD,oDAAoD;AACpD,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAGnD;AAED,4EAA4E;AAC5E,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAkCrE"}
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CSV (.csv, .tsv) processor.
|
|
3
|
+
* Pure Node.js parser — no external dependencies.
|
|
4
|
+
* Handles: quoted fields, commas in values, newlines in quotes, BOM, TSV.
|
|
5
|
+
*/
|
|
6
|
+
import { readFileSync } from 'node:fs';
|
|
7
|
+
import { basename, extname } from 'node:path';
|
|
8
|
+
const MAX_DISPLAY_ROWS = 100;
|
|
9
|
+
const MAX_DISPLAY_COLS = 10;
|
|
10
|
+
/** Check whether a file path looks like CSV/TSV. */
|
|
11
|
+
export function isCsvFile(filePath) {
|
|
12
|
+
const ext = extname(filePath).toLowerCase();
|
|
13
|
+
return ext === '.csv' || ext === '.tsv';
|
|
14
|
+
}
|
|
15
|
+
/** Process a CSV or TSV file into structured output with markdown table. */
|
|
16
|
+
export async function processCsv(filePath) {
|
|
17
|
+
const ext = extname(filePath).toLowerCase();
|
|
18
|
+
const title = basename(filePath, ext);
|
|
19
|
+
const raw = readFileSync(filePath, 'utf-8');
|
|
20
|
+
const cleaned = stripBom(raw);
|
|
21
|
+
const delimiter = ext === '.tsv' ? '\t' : detectDelimiter(cleaned);
|
|
22
|
+
const rows = parseRows(cleaned, delimiter);
|
|
23
|
+
if (rows.length === 0) {
|
|
24
|
+
return {
|
|
25
|
+
title,
|
|
26
|
+
content: `[CSV — no data extracted from ${basename(filePath)}]`,
|
|
27
|
+
markdown: buildMarkdown(title, filePath, '', 0, 0, []),
|
|
28
|
+
rowCount: 0,
|
|
29
|
+
columnCount: 0,
|
|
30
|
+
sourcePath: filePath,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
const columnCount = Math.max(...rows.map((r) => r.length));
|
|
34
|
+
const rowCount = rows.length - 1; // exclude header row
|
|
35
|
+
const columnTypes = detectColumnTypes(rows);
|
|
36
|
+
const tableContent = buildTable(rows, columnCount);
|
|
37
|
+
return {
|
|
38
|
+
title,
|
|
39
|
+
content: tableContent,
|
|
40
|
+
markdown: buildMarkdown(title, filePath, tableContent, rowCount, columnCount, columnTypes),
|
|
41
|
+
rowCount,
|
|
42
|
+
columnCount,
|
|
43
|
+
sourcePath: filePath,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
// CSV parser — handles RFC 4180 (quoted fields, embedded commas/newlines)
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
function stripBom(text) {
|
|
50
|
+
return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
|
|
51
|
+
}
|
|
52
|
+
/** Sniff the most likely delimiter from the first few lines. */
|
|
53
|
+
function detectDelimiter(text) {
|
|
54
|
+
const sample = text.substring(0, 2000);
|
|
55
|
+
const commas = (sample.match(/,/g) ?? []).length;
|
|
56
|
+
const tabs = (sample.match(/\t/g) ?? []).length;
|
|
57
|
+
const semicolons = (sample.match(/;/g) ?? []).length;
|
|
58
|
+
const pipes = (sample.match(/\|/g) ?? []).length;
|
|
59
|
+
const counts = [
|
|
60
|
+
[',', commas],
|
|
61
|
+
['\t', tabs],
|
|
62
|
+
[';', semicolons],
|
|
63
|
+
['|', pipes],
|
|
64
|
+
];
|
|
65
|
+
counts.sort((a, b) => b[1] - a[1]);
|
|
66
|
+
const best = counts[0];
|
|
67
|
+
return best && best[1] > 0 ? best[0] : ',';
|
|
68
|
+
}
|
|
69
|
+
/** Parse CSV text into a 2-D array of strings, respecting quoted fields. */
|
|
70
|
+
function parseRows(text, delimiter) {
|
|
71
|
+
const rows = [];
|
|
72
|
+
let currentRow = [];
|
|
73
|
+
let field = '';
|
|
74
|
+
let inQuotes = false;
|
|
75
|
+
let i = 0;
|
|
76
|
+
while (i < text.length) {
|
|
77
|
+
const ch = text[i];
|
|
78
|
+
if (inQuotes) {
|
|
79
|
+
if (ch === '"') {
|
|
80
|
+
// Escaped quote ("")
|
|
81
|
+
if (i + 1 < text.length && text[i + 1] === '"') {
|
|
82
|
+
field += '"';
|
|
83
|
+
i += 2;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
// End of quoted field
|
|
87
|
+
inQuotes = false;
|
|
88
|
+
i++;
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
field += ch;
|
|
92
|
+
i++;
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
// Not inside quotes
|
|
96
|
+
if (ch === '"' && field.length === 0) {
|
|
97
|
+
inQuotes = true;
|
|
98
|
+
i++;
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
if (ch === delimiter) {
|
|
102
|
+
currentRow.push(field.trim());
|
|
103
|
+
field = '';
|
|
104
|
+
i++;
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
if (ch === '\r') {
|
|
108
|
+
// CR or CRLF
|
|
109
|
+
currentRow.push(field.trim());
|
|
110
|
+
field = '';
|
|
111
|
+
if (currentRow.some((c) => c.length > 0)) {
|
|
112
|
+
rows.push(currentRow);
|
|
113
|
+
}
|
|
114
|
+
currentRow = [];
|
|
115
|
+
i++;
|
|
116
|
+
if (i < text.length && text[i] === '\n')
|
|
117
|
+
i++;
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
if (ch === '\n') {
|
|
121
|
+
currentRow.push(field.trim());
|
|
122
|
+
field = '';
|
|
123
|
+
if (currentRow.some((c) => c.length > 0)) {
|
|
124
|
+
rows.push(currentRow);
|
|
125
|
+
}
|
|
126
|
+
currentRow = [];
|
|
127
|
+
i++;
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
field += ch;
|
|
131
|
+
i++;
|
|
132
|
+
}
|
|
133
|
+
// Flush last field/row
|
|
134
|
+
if (field.length > 0 || currentRow.length > 0) {
|
|
135
|
+
currentRow.push(field.trim());
|
|
136
|
+
if (currentRow.some((c) => c.length > 0)) {
|
|
137
|
+
rows.push(currentRow);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return rows;
|
|
141
|
+
}
|
|
142
|
+
function detectColumnTypes(rows) {
|
|
143
|
+
if (rows.length < 2)
|
|
144
|
+
return [];
|
|
145
|
+
const headers = rows[0];
|
|
146
|
+
if (!headers)
|
|
147
|
+
return [];
|
|
148
|
+
const dataRows = rows.slice(1, Math.min(rows.length, 51)); // sample up to 50 data rows
|
|
149
|
+
return headers.map((header, colIdx) => {
|
|
150
|
+
const values = dataRows
|
|
151
|
+
.map((row) => row[colIdx] ?? '')
|
|
152
|
+
.filter((v) => v.length > 0);
|
|
153
|
+
if (values.length === 0) {
|
|
154
|
+
return { name: header || `col_${colIdx + 1}`, type: 'text' };
|
|
155
|
+
}
|
|
156
|
+
// Check number
|
|
157
|
+
const numberCount = values.filter((v) => /^-?[\d,]+\.?\d*$/.test(v.replace(/,/g, ''))).length;
|
|
158
|
+
if (numberCount / values.length > 0.8) {
|
|
159
|
+
return { name: header || `col_${colIdx + 1}`, type: 'number' };
|
|
160
|
+
}
|
|
161
|
+
// Check boolean
|
|
162
|
+
const boolCount = values.filter((v) => ['true', 'false', 'yes', 'no', '0', '1'].includes(v.toLowerCase())).length;
|
|
163
|
+
if (boolCount / values.length > 0.8) {
|
|
164
|
+
return { name: header || `col_${colIdx + 1}`, type: 'boolean' };
|
|
165
|
+
}
|
|
166
|
+
// Check date
|
|
167
|
+
const dateCount = values.filter((v) => /^\d{4}[-/]\d{1,2}[-/]\d{1,2}/.test(v) || /^\d{1,2}[-/]\d{1,2}[-/]\d{2,4}/.test(v)).length;
|
|
168
|
+
if (dateCount / values.length > 0.8) {
|
|
169
|
+
return { name: header || `col_${colIdx + 1}`, type: 'date' };
|
|
170
|
+
}
|
|
171
|
+
return { name: header || `col_${colIdx + 1}`, type: 'text' };
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
// ---------------------------------------------------------------------------
|
|
175
|
+
// Markdown table builder
|
|
176
|
+
// ---------------------------------------------------------------------------
|
|
177
|
+
function buildTable(rows, totalCols) {
|
|
178
|
+
if (rows.length === 0)
|
|
179
|
+
return '';
|
|
180
|
+
const truncateCols = totalCols > MAX_DISPLAY_COLS;
|
|
181
|
+
const displayCols = truncateCols ? MAX_DISPLAY_COLS : totalCols;
|
|
182
|
+
const lines = [];
|
|
183
|
+
for (let i = 0; i < Math.min(rows.length, MAX_DISPLAY_ROWS + 1); i++) {
|
|
184
|
+
const row = rows[i];
|
|
185
|
+
if (!row)
|
|
186
|
+
continue;
|
|
187
|
+
const cells = [];
|
|
188
|
+
for (let j = 0; j < displayCols; j++) {
|
|
189
|
+
const cell = row[j] ?? '';
|
|
190
|
+
// Escape pipes and collapse newlines for table safety
|
|
191
|
+
cells.push(cell.replace(/\|/g, '\\|').replace(/\n/g, ' '));
|
|
192
|
+
}
|
|
193
|
+
if (truncateCols) {
|
|
194
|
+
cells.push('...');
|
|
195
|
+
}
|
|
196
|
+
lines.push(`| ${cells.join(' | ')} |`);
|
|
197
|
+
// Separator after header row
|
|
198
|
+
if (i === 0) {
|
|
199
|
+
const sep = cells.map(() => '---');
|
|
200
|
+
lines.push(`| ${sep.join(' | ')} |`);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
if (rows.length > MAX_DISPLAY_ROWS + 1) {
|
|
204
|
+
const remaining = rows.length - MAX_DISPLAY_ROWS - 1;
|
|
205
|
+
lines.push(`\n> _...and ${remaining} more rows (truncated)_`);
|
|
206
|
+
}
|
|
207
|
+
return lines.join('\n');
|
|
208
|
+
}
|
|
209
|
+
// ---------------------------------------------------------------------------
|
|
210
|
+
// Full markdown output
|
|
211
|
+
// ---------------------------------------------------------------------------
|
|
212
|
+
function buildMarkdown(title, filePath, content, rowCount, columnCount, columnTypes) {
|
|
213
|
+
const typesSummary = columnTypes.length > 0
|
|
214
|
+
? columnTypes.map((c) => `\`${c.name}\` (${c.type})`).join(', ')
|
|
215
|
+
: 'N/A';
|
|
216
|
+
return `# ${title}
|
|
217
|
+
|
|
218
|
+
> **Source:** [${basename(filePath)}](${filePath})
|
|
219
|
+
> **Type:** CSV
|
|
220
|
+
> **Rows:** ${rowCount}
|
|
221
|
+
> **Columns:** ${columnCount}
|
|
222
|
+
> **Column types:** ${typesSummary}
|
|
223
|
+
> **Processed:** ${new Date().toISOString().split('T')[0]}
|
|
224
|
+
|
|
225
|
+
## Data
|
|
226
|
+
|
|
227
|
+
${content}
|
|
228
|
+
`;
|
|
229
|
+
}
|
|
230
|
+
//# sourceMappingURL=csv.js.map
|