stellavault 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Autonomous_Knowledge_Foundry.pdf +0 -0
- package/CHANGELOG.md +60 -0
- package/README.md +140 -95
- package/package.json +1 -1
- package/packages/cli/dist/commands/ask-cmd.d.ts +4 -0
- package/packages/cli/dist/commands/ask-cmd.js +35 -0
- package/packages/cli/dist/commands/autopilot-cmd.d.ts +4 -0
- package/packages/cli/dist/commands/autopilot-cmd.js +76 -0
- package/packages/cli/dist/commands/compile-cmd.d.ts +6 -0
- package/packages/cli/dist/commands/compile-cmd.js +30 -0
- package/packages/cli/dist/commands/digest-cmd.d.ts +1 -0
- package/packages/cli/dist/commands/digest-cmd.js +57 -0
- package/packages/cli/dist/commands/fleeting-cmd.d.ts +4 -0
- package/packages/cli/dist/commands/fleeting-cmd.js +45 -0
- package/packages/cli/dist/commands/graph-cmd.js +13 -1
- package/packages/cli/dist/commands/ingest-cmd.d.ts +9 -0
- package/packages/cli/dist/commands/ingest-cmd.js +131 -0
- package/packages/cli/dist/commands/init-cmd.js +39 -1
- package/packages/cli/dist/commands/lint-cmd.d.ts +2 -0
- package/packages/cli/dist/commands/lint-cmd.js +61 -0
- package/packages/cli/dist/index.js +46 -1
- package/packages/cli/package.json +1 -1
- package/packages/core/dist/api/server.js +284 -0
- package/packages/core/dist/i18n/note-strings.d.ts +5 -0
- package/packages/core/dist/i18n/note-strings.js +94 -0
- package/packages/core/dist/index.d.ts +9 -0
- package/packages/core/dist/index.js +5 -0
- package/packages/core/dist/intelligence/ask-engine.d.ts +23 -0
- package/packages/core/dist/intelligence/ask-engine.js +108 -0
- package/packages/core/dist/intelligence/ingest-pipeline.d.ts +31 -0
- package/packages/core/dist/intelligence/ingest-pipeline.js +192 -0
- package/packages/core/dist/intelligence/knowledge-lint.d.ts +27 -0
- package/packages/core/dist/intelligence/knowledge-lint.js +132 -0
- package/packages/core/dist/intelligence/wiki-compiler.d.ts +30 -0
- package/packages/core/dist/intelligence/wiki-compiler.js +222 -0
- package/packages/core/dist/intelligence/youtube-extractor.d.ts +29 -0
- package/packages/core/dist/intelligence/youtube-extractor.js +311 -0
- package/packages/core/dist/intelligence/zettelkasten.d.ts +59 -0
- package/packages/core/dist/intelligence/zettelkasten.js +234 -0
- package/packages/core/dist/mcp/server.d.ts +2 -0
- package/packages/core/dist/mcp/server.js +18 -1
- package/packages/core/dist/mcp/tools/agentic-graph.d.ts +6 -0
- package/packages/core/dist/mcp/tools/agentic-graph.js +35 -7
- package/packages/core/dist/mcp/tools/ask.d.ts +29 -0
- package/packages/core/dist/mcp/tools/ask.js +40 -0
- package/packages/core/package.json +5 -1
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
// YouTube 콘텐츠 추출기 v2 — 자막(타임스탬프 보존) + 메타데이터 → 구조화 노트
|
|
2
|
+
// PM 분석 반영: HTML 엔티티, 채널명, 이중 frontmatter, 태그 품질, 요약 품질
|
|
3
|
+
import { nt } from '../i18n/note-strings.js';
|
|
4
|
+
/**
|
|
5
|
+
* YouTube URL에서 모든 콘텐츠 추출.
|
|
6
|
+
* 반환값은 데이터만 — 노트 포맷팅은 별도.
|
|
7
|
+
*/
|
|
8
|
+
export async function extractYouTubeContent(url) {
|
|
9
|
+
const html = await fetchPage(url);
|
|
10
|
+
// 메타데이터 (cleanHtml 먼저 적용)
|
|
11
|
+
const title = cleanHtml(extractMeta(html, 'og:title') ?? extractBetween(html, '<title>', '</title>') ?? 'Untitled');
|
|
12
|
+
const channelName = cleanHtml(extractBetween(html, '"ownerChannelName":"', '"')
|
|
13
|
+
?? extractBetween(html, '"author":"', '"')
|
|
14
|
+
?? 'Unknown Channel');
|
|
15
|
+
// 전체 설명 가져오기 (og:description은 ~150자로 잘림)
|
|
16
|
+
const description = cleanHtml(extractFullDescription(html));
|
|
17
|
+
const duration = extractBetween(html, '"lengthSeconds":"', '"') ?? '';
|
|
18
|
+
const publishDate = extractBetween(html, '"publishDate":"', '"') ?? '';
|
|
19
|
+
const viewCount = extractBetween(html, '"viewCount":"', '"') ?? '';
|
|
20
|
+
const thumbnail = extractMeta(html, 'og:image') ?? '';
|
|
21
|
+
const videoId = extractVideoId(url);
|
|
22
|
+
// 자막 (타임스탬프 보존)
|
|
23
|
+
let transcript = [];
|
|
24
|
+
let rawTranscript = '';
|
|
25
|
+
try {
|
|
26
|
+
transcript = await extractTimedTranscript(html, videoId);
|
|
27
|
+
rawTranscript = transcript.map(s => s.text).join(' ');
|
|
28
|
+
}
|
|
29
|
+
catch { /* 자막 없음 */ }
|
|
30
|
+
// 태그 (cleanHtml 적용된 데이터에서 추출)
|
|
31
|
+
const tags = extractSmartTags(title, description);
|
|
32
|
+
// 요약 (문장 중요도 기반)
|
|
33
|
+
const summary = generateSmartSummary(title, rawTranscript, description);
|
|
34
|
+
return {
|
|
35
|
+
title, channelName, description, transcript, rawTranscript,
|
|
36
|
+
duration: formatDuration(duration), publishDate, tags, url,
|
|
37
|
+
videoId, viewCount, thumbnail, summary,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* 추출된 콘텐츠 → Stellavault .md 노트 (frontmatter 포함하지 않음 — pipeline이 처리).
|
|
42
|
+
*/
|
|
43
|
+
export function formatYouTubeNote(content) {
|
|
44
|
+
const lines = [];
|
|
45
|
+
// Header
|
|
46
|
+
lines.push(`# ${content.title}`, '');
|
|
47
|
+
lines.push(`> **${content.channelName}** | ${content.duration} | ${content.publishDate?.split('T')[0] ?? ''}`);
|
|
48
|
+
if (content.viewCount)
|
|
49
|
+
lines.push(`> ${nt('views')}: ${Number(content.viewCount).toLocaleString()}`);
|
|
50
|
+
lines.push(`> ${content.url}`, '');
|
|
51
|
+
if (content.summary) {
|
|
52
|
+
lines.push(`## ${nt('summary')}`, '', content.summary, '');
|
|
53
|
+
}
|
|
54
|
+
if (content.description.length > 30) {
|
|
55
|
+
lines.push(`## ${nt('description')}`, '', content.description.slice(0, 3000), '');
|
|
56
|
+
}
|
|
57
|
+
if (content.transcript.length > 0) {
|
|
58
|
+
lines.push(`## ${nt('transcript')}`, '');
|
|
59
|
+
const segments = groupIntoSegments(content.transcript);
|
|
60
|
+
for (const seg of segments) {
|
|
61
|
+
const ts = formatSeconds(seg.startTime);
|
|
62
|
+
const ytLink = `https://youtu.be/${content.videoId}?t=${Math.floor(seg.startTime)}`;
|
|
63
|
+
lines.push(`### [${ts}](${ytLink})`, '', seg.text, '');
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return lines.join('\n');
|
|
67
|
+
}
|
|
68
|
+
// ─── 자막 추출 (타임스탬프 보존) ───
|
|
69
|
+
async function extractTimedTranscript(html, videoId) {
|
|
70
|
+
// 1차: HTML에서 captionTracks URL 직접 fetch
|
|
71
|
+
const segments = await extractTimedTranscriptFromHtml(html);
|
|
72
|
+
if (segments.length > 0)
|
|
73
|
+
return segments;
|
|
74
|
+
// 2차: yt-dlp fallback (YouTube bot 보호 우회)
|
|
75
|
+
if (videoId)
|
|
76
|
+
return extractTimedTranscriptViaTool(videoId);
|
|
77
|
+
return [];
|
|
78
|
+
}
|
|
79
|
+
async function extractTimedTranscriptFromHtml(html) {
|
|
80
|
+
const captionMatch = html.match(/"captionTracks":\[(.*?)\]/);
|
|
81
|
+
if (!captionMatch)
|
|
82
|
+
return [];
|
|
83
|
+
const tracks = captionMatch[1];
|
|
84
|
+
let captionUrl = '';
|
|
85
|
+
const allUrls = [...tracks.matchAll(/"baseUrl":"(.*?)"/g)].map(m => m[1]);
|
|
86
|
+
const allLangs = [...tracks.matchAll(/"languageCode":"(.*?)"/g)].map(m => m[1]);
|
|
87
|
+
for (const targetLang of ['ko', 'en']) {
|
|
88
|
+
const idx = allLangs.indexOf(targetLang);
|
|
89
|
+
if (idx >= 0 && allUrls[idx]) {
|
|
90
|
+
captionUrl = allUrls[idx];
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (!captionUrl && allUrls.length > 0)
|
|
95
|
+
captionUrl = allUrls[0];
|
|
96
|
+
if (!captionUrl)
|
|
97
|
+
return [];
|
|
98
|
+
const cleanUrl = captionUrl.replace(/\\u0026/g, '&').replace(/\\u003d/g, '=').replace(/\\\//g, '/');
|
|
99
|
+
const resp = await fetch(cleanUrl, { signal: AbortSignal.timeout(10000) });
|
|
100
|
+
const xml = await resp.text();
|
|
101
|
+
return parseTranscriptXml(xml);
|
|
102
|
+
}
|
|
103
|
+
async function extractTimedTranscriptViaTool(videoId) {
|
|
104
|
+
const { execSync } = await import('node:child_process');
|
|
105
|
+
const { readdirSync, readFileSync, unlinkSync } = await import('node:fs');
|
|
106
|
+
const { join } = await import('node:path');
|
|
107
|
+
const tmpDir = (await import('node:os')).tmpdir();
|
|
108
|
+
const tmpBase = join(tmpDir, `sv-sub-${videoId}`);
|
|
109
|
+
const url = `https://www.youtube.com/watch?v=${videoId}`;
|
|
110
|
+
// Try each language separately to avoid partial failure aborting everything
|
|
111
|
+
for (const lang of ['ko', 'en']) {
|
|
112
|
+
try {
|
|
113
|
+
execSync(`python -m yt_dlp --write-auto-sub --sub-lang ${lang} --skip-download --sub-format srv1 -o "${tmpBase}" "${url}"`, { timeout: 30000, stdio: 'pipe' });
|
|
114
|
+
}
|
|
115
|
+
catch { /* lang not available — try next */
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
// Find the subtitle file
|
|
119
|
+
const files = readdirSync(tmpDir).filter(f => f.startsWith(`sv-sub-${videoId}`) && f.endsWith('.srv1'));
|
|
120
|
+
if (files.length === 0)
|
|
121
|
+
continue;
|
|
122
|
+
const xml = readFileSync(join(tmpDir, files[0]), 'utf-8');
|
|
123
|
+
// Cleanup
|
|
124
|
+
for (const f of files) {
|
|
125
|
+
try {
|
|
126
|
+
unlinkSync(join(tmpDir, f));
|
|
127
|
+
}
|
|
128
|
+
catch { /* ok */ }
|
|
129
|
+
}
|
|
130
|
+
const segments = parseTranscriptXml(xml);
|
|
131
|
+
if (segments.length > 0)
|
|
132
|
+
return segments;
|
|
133
|
+
}
|
|
134
|
+
return [];
|
|
135
|
+
}
|
|
136
|
+
function parseTranscriptXml(xml) {
|
|
137
|
+
if (!xml || xml.length === 0)
|
|
138
|
+
return [];
|
|
139
|
+
const segments = [];
|
|
140
|
+
const matches = xml.matchAll(/<text start="([^"]+)"[^>]*>(.*?)<\/text>/g);
|
|
141
|
+
for (const m of matches) {
|
|
142
|
+
const text = cleanHtml(m[2]).trim();
|
|
143
|
+
if (text)
|
|
144
|
+
segments.push({ startTime: parseFloat(m[1]), text });
|
|
145
|
+
}
|
|
146
|
+
return segments;
|
|
147
|
+
}
|
|
148
|
+
function groupIntoSegments(timedTexts, gapThreshold = 30) {
|
|
149
|
+
const segments = [];
|
|
150
|
+
let current = null;
|
|
151
|
+
let lastStart = 0;
|
|
152
|
+
for (const item of timedTexts) {
|
|
153
|
+
if (!current || (item.startTime - lastStart > gapThreshold)) {
|
|
154
|
+
if (current)
|
|
155
|
+
segments.push(current);
|
|
156
|
+
current = { startTime: item.startTime, texts: [] };
|
|
157
|
+
}
|
|
158
|
+
current.texts.push(item.text);
|
|
159
|
+
lastStart = item.startTime;
|
|
160
|
+
}
|
|
161
|
+
if (current)
|
|
162
|
+
segments.push(current);
|
|
163
|
+
return segments.map(s => ({ startTime: s.startTime, text: s.texts.join(' ') }));
|
|
164
|
+
}
|
|
165
|
+
// ─── 태그 추출 (한국어 조사 처리) ───
|
|
166
|
+
function extractSmartTags(title, description) {
|
|
167
|
+
const text = `${title} ${title} ${description}`; // 제목 가중치 2배
|
|
168
|
+
const koSuffixes = ['에서는', '에서', '에게', '까지', '부터', '으로', '에는', '와', '과', '을', '를', '이', '가', '은', '는', '의', '에', '로', '도', '만'];
|
|
169
|
+
const stopwords = new Set([
|
|
170
|
+
'this', 'that', 'with', 'from', 'have', 'been', 'will', 'about',
|
|
171
|
+
'your', 'more', 'what', 'http', 'https', 'www', 'youtube', 'com',
|
|
172
|
+
'이번', '영상', '에서', '있는', '하는', '것을', '합니다', '있습니다',
|
|
173
|
+
'안녕하세요', '바이브랩스입니다', '됩니다', '그리고', '하지만', '이렇게',
|
|
174
|
+
]);
|
|
175
|
+
const tokens = text.split(/[\s,.\-_#|?!'"()\[\]{}:;]+/).filter(Boolean);
|
|
176
|
+
const freq = new Map();
|
|
177
|
+
for (let token of tokens) {
|
|
178
|
+
// 한국어 조사 제거
|
|
179
|
+
if (token.length > 2) {
|
|
180
|
+
for (const suffix of koSuffixes) {
|
|
181
|
+
if (token.endsWith(suffix) && token.length - suffix.length >= 2) {
|
|
182
|
+
token = token.slice(0, -suffix.length);
|
|
183
|
+
break;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
if (token.length < 2 || /^\d+$/.test(token) || stopwords.has(token.toLowerCase()))
|
|
188
|
+
continue;
|
|
189
|
+
if (/&[a-z]+;|&#\d+;/.test(token))
|
|
190
|
+
continue;
|
|
191
|
+
freq.set(token, (freq.get(token) ?? 0) + 1);
|
|
192
|
+
}
|
|
193
|
+
return [...freq.entries()]
|
|
194
|
+
.sort((a, b) => b[1] - a[1])
|
|
195
|
+
.slice(0, 6)
|
|
196
|
+
.map(([w]) => w);
|
|
197
|
+
}
|
|
198
|
+
// ─── 요약 (문장 중요도 기반) ───
|
|
199
|
+
function generateSmartSummary(title, transcript, description) {
|
|
200
|
+
const source = transcript.length > 100 ? transcript : description;
|
|
201
|
+
if (source.length < 50)
|
|
202
|
+
return `${title} — ${nt('youtubeVideo')}`;
|
|
203
|
+
const sentences = source
|
|
204
|
+
.split(/[.。!?]\s+|(?<=다)\s+|(?<=요)\s+/)
|
|
205
|
+
.map(s => s.trim())
|
|
206
|
+
.filter(s => s.length > 15 && s.length < 300);
|
|
207
|
+
if (sentences.length <= 3)
|
|
208
|
+
return sentences.join(' ');
|
|
209
|
+
const titleWords = new Set(title.split(/[\s,.\-_]+/).filter(w => w.length > 2).map(w => w.toLowerCase()));
|
|
210
|
+
const scored = sentences.map((sentence, idx) => {
|
|
211
|
+
let score = 0;
|
|
212
|
+
const words = sentence.toLowerCase().split(/\s+/);
|
|
213
|
+
for (const w of words) {
|
|
214
|
+
if (titleWords.has(w))
|
|
215
|
+
score += 3;
|
|
216
|
+
}
|
|
217
|
+
const pos = idx / sentences.length;
|
|
218
|
+
if (pos > 0.15 && pos < 0.4)
|
|
219
|
+
score += 2;
|
|
220
|
+
if (pos > 0.6 && pos < 0.85)
|
|
221
|
+
score += 1;
|
|
222
|
+
if (pos < 0.05)
|
|
223
|
+
score -= 3; // 인트로 인사 강력 패널티
|
|
224
|
+
if (/\d+/.test(sentence))
|
|
225
|
+
score += 1;
|
|
226
|
+
return { sentence, score, idx };
|
|
227
|
+
});
|
|
228
|
+
return scored
|
|
229
|
+
.sort((a, b) => b.score - a.score)
|
|
230
|
+
.slice(0, 4)
|
|
231
|
+
.sort((a, b) => a.idx - b.idx)
|
|
232
|
+
.map(s => s.sentence)
|
|
233
|
+
.join(' ');
|
|
234
|
+
}
|
|
235
|
+
// ─── 유틸 ───
|
|
236
|
+
function extractFullDescription(html) {
|
|
237
|
+
// 방법 1: JSON에서 shortDescription 추출 (이스케이프된 JSON 문자열)
|
|
238
|
+
const shortDescMatch = html.match(/"shortDescription":"((?:[^"\\]|\\.)*)"/);
|
|
239
|
+
if (shortDescMatch) {
|
|
240
|
+
try {
|
|
241
|
+
return JSON.parse(`"${shortDescMatch[1]}"`);
|
|
242
|
+
}
|
|
243
|
+
catch { /* fallback */ }
|
|
244
|
+
}
|
|
245
|
+
// 방법 2: description.simpleText
|
|
246
|
+
const simpleMatch = html.match(/"description":\{"simpleText":"((?:[^"\\]|\\.)*)"/);
|
|
247
|
+
if (simpleMatch) {
|
|
248
|
+
try {
|
|
249
|
+
return JSON.parse(`"${simpleMatch[1]}"`);
|
|
250
|
+
}
|
|
251
|
+
catch { /* fallback */ }
|
|
252
|
+
}
|
|
253
|
+
// 방법 3: og:description (짧지만 최후 수단)
|
|
254
|
+
return extractMeta(html, 'og:description') ?? '';
|
|
255
|
+
}
|
|
256
|
+
async function fetchPage(url) {
|
|
257
|
+
const resp = await fetch(url, {
|
|
258
|
+
headers: {
|
|
259
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
|
260
|
+
'Accept-Language': 'ko-KR,ko;q=0.9,en;q=0.8',
|
|
261
|
+
},
|
|
262
|
+
signal: AbortSignal.timeout(15000),
|
|
263
|
+
});
|
|
264
|
+
return resp.text();
|
|
265
|
+
}
|
|
266
|
+
function extractMeta(html, property) {
|
|
267
|
+
const match = html.match(new RegExp(`<meta[^>]*property="${property}"[^>]*content="([^"]*)"`, 'i'))
|
|
268
|
+
?? html.match(new RegExp(`<meta[^>]*name="${property}"[^>]*content="([^"]*)"`, 'i'));
|
|
269
|
+
return match?.[1];
|
|
270
|
+
}
|
|
271
|
+
function extractBetween(html, start, end) {
|
|
272
|
+
const idx = html.indexOf(start);
|
|
273
|
+
if (idx < 0)
|
|
274
|
+
return undefined;
|
|
275
|
+
const s = idx + start.length;
|
|
276
|
+
const e = html.indexOf(end, s);
|
|
277
|
+
if (e < 0)
|
|
278
|
+
return undefined;
|
|
279
|
+
return html.substring(s, e);
|
|
280
|
+
}
|
|
281
|
+
function cleanHtml(text) {
|
|
282
|
+
return text
|
|
283
|
+
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
|
284
|
+
.replace(/'/g, "'").replace(/"/g, '"').replace(/'/g, "'")
|
|
285
|
+
.replace(/ /g, ' ').replace(/\\n/g, ' ').replace(/\n/g, ' ')
|
|
286
|
+
.trim();
|
|
287
|
+
}
|
|
288
|
+
function formatDuration(seconds) {
|
|
289
|
+
const s = parseInt(seconds);
|
|
290
|
+
if (isNaN(s))
|
|
291
|
+
return '';
|
|
292
|
+
const h = Math.floor(s / 3600);
|
|
293
|
+
const m = Math.floor((s % 3600) / 60);
|
|
294
|
+
const sec = s % 60;
|
|
295
|
+
return h > 0 ? `${h}:${String(m).padStart(2, '0')}:${String(sec).padStart(2, '0')}` : `${m}:${String(sec).padStart(2, '0')}`;
|
|
296
|
+
}
|
|
297
|
+
function formatSeconds(sec) {
|
|
298
|
+
const m = Math.floor(sec / 60);
|
|
299
|
+
const s = Math.floor(sec % 60);
|
|
300
|
+
return `${m}:${String(s).padStart(2, '0')}`;
|
|
301
|
+
}
|
|
302
|
+
function extractVideoId(url) {
|
|
303
|
+
try {
|
|
304
|
+
const u = new URL(url);
|
|
305
|
+
return u.searchParams.get('v') ?? u.pathname.split('/').pop() ?? '';
|
|
306
|
+
}
|
|
307
|
+
catch {
|
|
308
|
+
return '';
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
//# sourceMappingURL=youtube-extractor.js.map
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
export interface FrontmatterEntry {
|
|
2
|
+
filePath: string;
|
|
3
|
+
title: string;
|
|
4
|
+
id?: string;
|
|
5
|
+
type?: string;
|
|
6
|
+
tags: string[];
|
|
7
|
+
summary?: string;
|
|
8
|
+
connections: string[];
|
|
9
|
+
archived?: boolean;
|
|
10
|
+
created?: string;
|
|
11
|
+
wordCount: number;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Vault의 모든 .md 파일에서 프론트메터만 추출 (본문 미읽기 → 토큰 절감).
|
|
15
|
+
* 40만 단어를 수백 줄로 압축.
|
|
16
|
+
*/
|
|
17
|
+
export declare function scanFrontmatter(vaultPath: string, options?: {
|
|
18
|
+
includeArchived?: boolean;
|
|
19
|
+
}): FrontmatterEntry[];
|
|
20
|
+
/**
|
|
21
|
+
* 기존 인덱스 코드를 분석하여 다음 사용 가능한 코드를 생성.
|
|
22
|
+
* 패턴: 1A → 1B, 1A → 1A1, 1A1 → 1A1a
|
|
23
|
+
*/
|
|
24
|
+
export declare function generateNextIndexCode(existingCodes: string[], parentCode?: string): string;
|
|
25
|
+
/**
|
|
26
|
+
* 프론트메터에 인덱스 코드가 없는 노트에 자동 부여.
|
|
27
|
+
*/
|
|
28
|
+
export declare function assignIndexCodes(entries: FrontmatterEntry[]): Map<string, string>;
|
|
29
|
+
/**
|
|
30
|
+
* raw/ 폴더에서 archive 플래그가 없는 파일만 반환 (인박스 제로).
|
|
31
|
+
*/
|
|
32
|
+
export declare function getInboxItems(vaultPath: string, rawDir?: string): FrontmatterEntry[];
|
|
33
|
+
/**
|
|
34
|
+
* 파일에 archive: true 플래그 추가.
|
|
35
|
+
*/
|
|
36
|
+
export declare function archiveFile(fullPath: string, vaultPath?: string): void;
|
|
37
|
+
/**
|
|
38
|
+
* 원자성 검증: 하나의 메모에 여러 주제가 섞여있는지 감지.
|
|
39
|
+
* - heading 3개 이상 → 분할 후보
|
|
40
|
+
* - 1500단어 초과 → 분할 후보
|
|
41
|
+
*/
|
|
42
|
+
export declare function checkAtomicity(entries: FrontmatterEntry[], vaultPath: string): Array<{
|
|
43
|
+
filePath: string;
|
|
44
|
+
title: string;
|
|
45
|
+
reason: string;
|
|
46
|
+
headingCount?: number;
|
|
47
|
+
wordCount: number;
|
|
48
|
+
}>;
|
|
49
|
+
/**
|
|
50
|
+
* 고아 노트 (연결 0개) + 끊긴 링크 탐지.
|
|
51
|
+
*/
|
|
52
|
+
export declare function detectOrphansAndBrokenLinks(entries: FrontmatterEntry[]): {
|
|
53
|
+
orphans: FrontmatterEntry[];
|
|
54
|
+
brokenLinks: Array<{
|
|
55
|
+
filePath: string;
|
|
56
|
+
brokenLink: string;
|
|
57
|
+
}>;
|
|
58
|
+
};
|
|
59
|
+
//# sourceMappingURL=zettelkasten.d.ts.map
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
// Zettelkasten System — 프론트메터 스캔 + 인덱스 코드 + 인박스 제로 + 원자성 검증
|
|
2
|
+
// Inspired by Luhmann + Karpathy "Self-Compiling Zettelkasten"
|
|
3
|
+
import { readdirSync, readFileSync, writeFileSync, existsSync, statSync } from 'node:fs';
|
|
4
|
+
import { join, resolve, extname } from 'node:path';
|
|
5
|
+
/**
|
|
6
|
+
* Vault의 모든 .md 파일에서 프론트메터만 추출 (본문 미읽기 → 토큰 절감).
|
|
7
|
+
* 40만 단어를 수백 줄로 압축.
|
|
8
|
+
*/
|
|
9
|
+
export function scanFrontmatter(vaultPath, options) {
|
|
10
|
+
const entries = [];
|
|
11
|
+
const includeArchived = options?.includeArchived ?? false;
|
|
12
|
+
function walkDir(dir, rel) {
|
|
13
|
+
if (!existsSync(dir))
|
|
14
|
+
return;
|
|
15
|
+
for (const name of readdirSync(dir)) {
|
|
16
|
+
if (name.startsWith('.') || name === 'node_modules')
|
|
17
|
+
continue;
|
|
18
|
+
const full = join(dir, name);
|
|
19
|
+
const relPath = rel ? `${rel}/${name}` : name;
|
|
20
|
+
if (statSync(full).isDirectory()) {
|
|
21
|
+
walkDir(full, relPath);
|
|
22
|
+
}
|
|
23
|
+
else if (extname(name) === '.md') {
|
|
24
|
+
const entry = extractFrontmatter(full, relPath);
|
|
25
|
+
if (entry && (includeArchived || !entry.archived)) {
|
|
26
|
+
entries.push(entry);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
walkDir(vaultPath, '');
|
|
32
|
+
return entries;
|
|
33
|
+
}
|
|
34
|
+
function extractFrontmatter(fullPath, relPath) {
|
|
35
|
+
const raw = readFileSync(fullPath, 'utf-8');
|
|
36
|
+
// Quick word count without full parse
|
|
37
|
+
const wordCount = raw.split(/\s+/).length;
|
|
38
|
+
// Parse YAML frontmatter
|
|
39
|
+
const fmMatch = raw.match(/^---\n([\s\S]*?)\n---/);
|
|
40
|
+
if (!fmMatch) {
|
|
41
|
+
// No frontmatter — extract title from first heading
|
|
42
|
+
const titleMatch = raw.match(/^#\s+(.+)$/m);
|
|
43
|
+
return {
|
|
44
|
+
filePath: relPath,
|
|
45
|
+
title: titleMatch?.[1] ?? relPath.replace(/\.md$/, ''),
|
|
46
|
+
tags: [],
|
|
47
|
+
connections: [],
|
|
48
|
+
wordCount,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
const fm = fmMatch[1];
|
|
52
|
+
const get = (key) => {
|
|
53
|
+
const m = fm.match(new RegExp(`^${key}:\\s*["']?(.+?)["']?\\s*$`, 'm'));
|
|
54
|
+
return m?.[1];
|
|
55
|
+
};
|
|
56
|
+
const getArray = (key) => {
|
|
57
|
+
const m = fm.match(new RegExp(`^${key}:\\s*\\[([^\\]]*)\\]`, 'm'));
|
|
58
|
+
if (m)
|
|
59
|
+
return m[1].split(',').map(s => s.trim().replace(/["']/g, '')).filter(Boolean);
|
|
60
|
+
return [];
|
|
61
|
+
};
|
|
62
|
+
const getBool = (key) => {
|
|
63
|
+
const v = get(key);
|
|
64
|
+
if (v === 'true')
|
|
65
|
+
return true;
|
|
66
|
+
if (v === 'false')
|
|
67
|
+
return false;
|
|
68
|
+
return undefined;
|
|
69
|
+
};
|
|
70
|
+
// Extract [[backlinks]] from content (quick scan, first 500 chars)
|
|
71
|
+
const linkSection = raw.substring(0, Math.min(raw.length, 2000));
|
|
72
|
+
const links = [...linkSection.matchAll(/\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g)].map(m => m[1]);
|
|
73
|
+
return {
|
|
74
|
+
filePath: relPath,
|
|
75
|
+
title: get('title') ?? relPath.replace(/\.md$/, ''),
|
|
76
|
+
id: get('id') ?? get('zettel_id'),
|
|
77
|
+
type: get('type') ?? get('note_type'),
|
|
78
|
+
tags: getArray('tags'),
|
|
79
|
+
summary: get('summary') ?? get('description'),
|
|
80
|
+
connections: links,
|
|
81
|
+
archived: getBool('archived') ?? getBool('archive'),
|
|
82
|
+
created: get('created') ?? get('date'),
|
|
83
|
+
wordCount,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
// ─── Luhmann Index Code ───
|
|
87
|
+
/**
|
|
88
|
+
* 기존 인덱스 코드를 분석하여 다음 사용 가능한 코드를 생성.
|
|
89
|
+
* 패턴: 1A → 1B, 1A → 1A1, 1A1 → 1A1a
|
|
90
|
+
*/
|
|
91
|
+
export function generateNextIndexCode(existingCodes, parentCode) {
|
|
92
|
+
if (existingCodes.length === 0 && !parentCode)
|
|
93
|
+
return '1A';
|
|
94
|
+
if (parentCode) {
|
|
95
|
+
// 하위 분기: parentCode의 자식 중 마지막 번호 + 1
|
|
96
|
+
const children = existingCodes.filter(c => c.startsWith(parentCode) && c.length > parentCode.length);
|
|
97
|
+
if (children.length === 0) {
|
|
98
|
+
// 첫 자식: 숫자 끝이면 알파벳 추가, 알파벳 끝이면 숫자 추가
|
|
99
|
+
const lastChar = parentCode[parentCode.length - 1];
|
|
100
|
+
return /[a-zA-Z]/.test(lastChar) ? `${parentCode}1` : `${parentCode}a`;
|
|
101
|
+
}
|
|
102
|
+
// 마지막 자식의 다음
|
|
103
|
+
const sorted = children.sort();
|
|
104
|
+
const last = sorted[sorted.length - 1];
|
|
105
|
+
return incrementCode(last);
|
|
106
|
+
}
|
|
107
|
+
// 최상위: 마지막 코드의 다음
|
|
108
|
+
const topLevel = existingCodes.filter(c => /^[0-9]+[A-Z]$/.test(c)).sort();
|
|
109
|
+
if (topLevel.length === 0)
|
|
110
|
+
return '1A';
|
|
111
|
+
return incrementCode(topLevel[topLevel.length - 1]);
|
|
112
|
+
}
|
|
113
|
+
function incrementCode(code) {
|
|
114
|
+
const lastChar = code[code.length - 1];
|
|
115
|
+
const prefix = code.slice(0, -1);
|
|
116
|
+
if (/[0-9]/.test(lastChar)) {
|
|
117
|
+
return prefix + String(parseInt(lastChar) + 1);
|
|
118
|
+
}
|
|
119
|
+
if (/[a-z]/.test(lastChar)) {
|
|
120
|
+
return prefix + String.fromCharCode(lastChar.charCodeAt(0) + 1);
|
|
121
|
+
}
|
|
122
|
+
if (/[A-Z]/.test(lastChar)) {
|
|
123
|
+
return prefix + String.fromCharCode(lastChar.charCodeAt(0) + 1);
|
|
124
|
+
}
|
|
125
|
+
return code + '1';
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* 프론트메터에 인덱스 코드가 없는 노트에 자동 부여.
|
|
129
|
+
*/
|
|
130
|
+
export function assignIndexCodes(entries) {
|
|
131
|
+
const assignments = new Map(); // filePath → indexCode
|
|
132
|
+
const existingCodes = entries.filter(e => e.id).map(e => e.id);
|
|
133
|
+
for (const entry of entries) {
|
|
134
|
+
if (entry.id)
|
|
135
|
+
continue; // 이미 있음
|
|
136
|
+
// 같은 태그를 가진 노트의 인덱스 코드를 찾아 하위로 분기
|
|
137
|
+
const sibling = entries.find(e => e.id && e.tags.some(t => entry.tags.includes(t)));
|
|
138
|
+
const newCode = generateNextIndexCode([...existingCodes, ...assignments.values()], sibling?.id);
|
|
139
|
+
assignments.set(entry.filePath, newCode);
|
|
140
|
+
}
|
|
141
|
+
return assignments;
|
|
142
|
+
}
|
|
143
|
+
// ─── Inbox Zero (Archive) ───
|
|
144
|
+
/**
|
|
145
|
+
* raw/ 폴더에서 archive 플래그가 없는 파일만 반환 (인박스 제로).
|
|
146
|
+
*/
|
|
147
|
+
export function getInboxItems(vaultPath, rawDir = 'raw') {
|
|
148
|
+
const rawPath = resolve(vaultPath, rawDir);
|
|
149
|
+
return scanFrontmatter(rawPath, { includeArchived: false });
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* 파일에 archive: true 플래그 추가.
|
|
153
|
+
*/
|
|
154
|
+
export function archiveFile(fullPath, vaultPath) {
|
|
155
|
+
// Path traversal 방지
|
|
156
|
+
const resolved = resolve(fullPath);
|
|
157
|
+
if (vaultPath && !resolved.startsWith(resolve(vaultPath))) {
|
|
158
|
+
throw new Error('Path traversal detected: file outside vault');
|
|
159
|
+
}
|
|
160
|
+
if (!existsSync(resolved))
|
|
161
|
+
throw new Error(`File not found: ${fullPath}`);
|
|
162
|
+
const content = readFileSync(resolved, 'utf-8');
|
|
163
|
+
if (content.startsWith('---\n')) {
|
|
164
|
+
const updated = content.replace('---\n', '---\narchived: true\n');
|
|
165
|
+
writeFileSync(resolved, updated, 'utf-8');
|
|
166
|
+
}
|
|
167
|
+
else {
|
|
168
|
+
writeFileSync(resolved, `---\narchived: true\n---\n\n${content}`, 'utf-8');
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
// ─── Atomicity Check ───
|
|
172
|
+
/**
|
|
173
|
+
* 원자성 검증: 하나의 메모에 여러 주제가 섞여있는지 감지.
|
|
174
|
+
* - heading 3개 이상 → 분할 후보
|
|
175
|
+
* - 1500단어 초과 → 분할 후보
|
|
176
|
+
*/
|
|
177
|
+
export function checkAtomicity(entries, vaultPath) {
|
|
178
|
+
const violations = [];
|
|
179
|
+
for (const entry of entries) {
|
|
180
|
+
// 단어 수 검사
|
|
181
|
+
if (entry.wordCount > 1500) {
|
|
182
|
+
violations.push({
|
|
183
|
+
filePath: entry.filePath,
|
|
184
|
+
title: entry.title,
|
|
185
|
+
reason: `Too long (${entry.wordCount} words). Consider splitting into smaller atomic notes.`,
|
|
186
|
+
wordCount: entry.wordCount,
|
|
187
|
+
});
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
// Heading 수 검사 (프론트메터 이후)
|
|
191
|
+
const fullPath = resolve(vaultPath, entry.filePath);
|
|
192
|
+
if (!existsSync(fullPath))
|
|
193
|
+
continue;
|
|
194
|
+
const content = readFileSync(fullPath, 'utf-8');
|
|
195
|
+
const headings = content.match(/^##\s+.+$/gm) ?? [];
|
|
196
|
+
if (headings.length >= 4) {
|
|
197
|
+
violations.push({
|
|
198
|
+
filePath: entry.filePath,
|
|
199
|
+
title: entry.title,
|
|
200
|
+
reason: `Multiple topics (${headings.length} sections). Each heading could be its own note.`,
|
|
201
|
+
headingCount: headings.length,
|
|
202
|
+
wordCount: entry.wordCount,
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return violations;
|
|
207
|
+
}
|
|
208
|
+
// ─── Orphan & Broken Link Detection ───
|
|
209
|
+
/**
|
|
210
|
+
* 고아 노트 (연결 0개) + 끊긴 링크 탐지.
|
|
211
|
+
*/
|
|
212
|
+
export function detectOrphansAndBrokenLinks(entries) {
|
|
213
|
+
const titleSet = new Set(entries.map(e => e.title));
|
|
214
|
+
const fileSet = new Set(entries.map(e => e.filePath.replace(/\.md$/, '')));
|
|
215
|
+
// 고아: 어디서도 참조되지 않고 자기도 참조 안 하는 노트
|
|
216
|
+
const referenced = new Set();
|
|
217
|
+
for (const e of entries) {
|
|
218
|
+
for (const conn of e.connections) {
|
|
219
|
+
referenced.add(conn);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
const orphans = entries.filter(e => e.connections.length === 0 && !referenced.has(e.title));
|
|
223
|
+
// 끊긴 링크: [[X]]가 있는데 X 노트가 없음
|
|
224
|
+
const brokenLinks = [];
|
|
225
|
+
for (const e of entries) {
|
|
226
|
+
for (const conn of e.connections) {
|
|
227
|
+
if (!titleSet.has(conn) && !fileSet.has(conn)) {
|
|
228
|
+
brokenLinks.push({ filePath: e.filePath, brokenLink: conn });
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return { orphans, brokenLinks };
|
|
233
|
+
}
|
|
234
|
+
//# sourceMappingURL=zettelkasten.js.map
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
|
2
2
|
import type { VectorStore } from '../store/types.js';
|
|
3
3
|
import type { SearchEngine } from '../search/index.js';
|
|
4
|
+
import type { Embedder } from '../indexer/embedder.js';
|
|
4
5
|
import type { DecayEngine } from '../intelligence/decay-engine.js';
|
|
5
6
|
export interface McpServerOptions {
|
|
6
7
|
store: VectorStore;
|
|
7
8
|
searchEngine: SearchEngine;
|
|
9
|
+
embedder?: Embedder;
|
|
8
10
|
vaultPath?: string;
|
|
9
11
|
decayEngine?: DecayEngine;
|
|
10
12
|
}
|
|
@@ -17,12 +17,16 @@ import { createLearningPathTool } from './tools/learning-path.js';
|
|
|
17
17
|
import { createDetectGapsTool } from './tools/detect-gaps.js';
|
|
18
18
|
import { createGetEvolutionTool } from './tools/get-evolution.js';
|
|
19
19
|
import { createLinkCodeTool } from './tools/link-code.js';
|
|
20
|
+
import { createAskTool } from './tools/ask.js';
|
|
21
|
+
import { createAgenticGraphTools } from './tools/agentic-graph.js';
|
|
20
22
|
export function createMcpServer(options) {
|
|
21
|
-
const { store, searchEngine, vaultPath = '', decayEngine } = options;
|
|
23
|
+
const { store, searchEngine, embedder, vaultPath = '', decayEngine } = options;
|
|
22
24
|
const learningPathTool = createLearningPathTool(store);
|
|
23
25
|
const detectGapsTool = createDetectGapsTool(store);
|
|
24
26
|
const getEvolutionTool = createGetEvolutionTool(store);
|
|
25
27
|
const linkCodeTool = createLinkCodeTool(searchEngine);
|
|
28
|
+
const askTool = createAskTool(searchEngine, vaultPath);
|
|
29
|
+
const agenticTools = embedder ? createAgenticGraphTools(store, embedder, vaultPath) : [];
|
|
26
30
|
const server = new Server({ name: 'stellavault', version: '0.2.0' }, { capabilities: { tools: {} } });
|
|
27
31
|
// List tools
|
|
28
32
|
server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
@@ -35,6 +39,8 @@ export function createMcpServer(options) {
|
|
|
35
39
|
{ name: detectGapsTool.name, description: detectGapsTool.description, inputSchema: detectGapsTool.inputSchema },
|
|
36
40
|
{ name: getEvolutionTool.name, description: getEvolutionTool.description, inputSchema: getEvolutionTool.inputSchema },
|
|
37
41
|
{ name: linkCodeTool.name, description: linkCodeTool.description, inputSchema: linkCodeTool.inputSchema },
|
|
42
|
+
{ name: askTool.name, description: askTool.description, inputSchema: askTool.inputSchema },
|
|
43
|
+
...agenticTools.map(t => ({ name: t.name, description: t.description, inputSchema: t.inputSchema })),
|
|
38
44
|
],
|
|
39
45
|
}));
|
|
40
46
|
// Call tool
|
|
@@ -111,7 +117,18 @@ export function createMcpServer(options) {
|
|
|
111
117
|
case 'link-code':
|
|
112
118
|
result = await linkCodeTool.handler(args);
|
|
113
119
|
return result;
|
|
120
|
+
case 'ask':
|
|
121
|
+
result = await askTool.handler(args);
|
|
122
|
+
return result;
|
|
114
123
|
default:
|
|
124
|
+
{
|
|
125
|
+
// Agentic graph tools (create-knowledge-node, create-knowledge-link)
|
|
126
|
+
const agenticTool = agenticTools.find(t => t.name === name);
|
|
127
|
+
if (agenticTool) {
|
|
128
|
+
result = await agenticTool.handler(args);
|
|
129
|
+
return result;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
115
132
|
return { content: [{ type: 'text', text: `Unknown tool: ${name}` }], isError: true };
|
|
116
133
|
}
|
|
117
134
|
return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
|