wikimem 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/LICENSE +21 -0
- package/README.md +398 -0
- package/dist/cli/commands/duplicates.d.ts +3 -0
- package/dist/cli/commands/duplicates.d.ts.map +1 -0
- package/dist/cli/commands/duplicates.js +38 -0
- package/dist/cli/commands/duplicates.js.map +1 -0
- package/dist/cli/commands/improve.d.ts +3 -0
- package/dist/cli/commands/improve.d.ts.map +1 -0
- package/dist/cli/commands/improve.js +69 -0
- package/dist/cli/commands/improve.js.map +1 -0
- package/dist/cli/commands/ingest.d.ts +3 -0
- package/dist/cli/commands/ingest.d.ts.map +1 -0
- package/dist/cli/commands/ingest.js +181 -0
- package/dist/cli/commands/ingest.js.map +1 -0
- package/dist/cli/commands/init.d.ts +3 -0
- package/dist/cli/commands/init.d.ts.map +1 -0
- package/dist/cli/commands/init.js +91 -0
- package/dist/cli/commands/init.js.map +1 -0
- package/dist/cli/commands/lint.d.ts +3 -0
- package/dist/cli/commands/lint.d.ts.map +1 -0
- package/dist/cli/commands/lint.js +49 -0
- package/dist/cli/commands/lint.js.map +1 -0
- package/dist/cli/commands/query.d.ts +3 -0
- package/dist/cli/commands/query.d.ts.map +1 -0
- package/dist/cli/commands/query.js +51 -0
- package/dist/cli/commands/query.js.map +1 -0
- package/dist/cli/commands/scrape.d.ts +3 -0
- package/dist/cli/commands/scrape.d.ts.map +1 -0
- package/dist/cli/commands/scrape.js +47 -0
- package/dist/cli/commands/scrape.js.map +1 -0
- package/dist/cli/commands/serve.d.ts +3 -0
- package/dist/cli/commands/serve.d.ts.map +1 -0
- package/dist/cli/commands/serve.js +24 -0
- package/dist/cli/commands/serve.js.map +1 -0
- package/dist/cli/commands/status.d.ts +3 -0
- package/dist/cli/commands/status.d.ts.map +1 -0
- package/dist/cli/commands/status.js +30 -0
- package/dist/cli/commands/status.js.map +1 -0
- package/dist/cli/commands/watch.d.ts +3 -0
- package/dist/cli/commands/watch.d.ts.map +1 -0
- package/dist/cli/commands/watch.js +29 -0
- package/dist/cli/commands/watch.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +30 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/core/config.d.ts +47 -0
- package/dist/core/config.d.ts.map +1 -0
- package/dist/core/config.js +11 -0
- package/dist/core/config.js.map +1 -0
- package/dist/core/improve.d.ts +19 -0
- package/dist/core/improve.d.ts.map +1 -0
- package/dist/core/improve.js +175 -0
- package/dist/core/improve.js.map +1 -0
- package/dist/core/index-manager.d.ts +9 -0
- package/dist/core/index-manager.d.ts.map +1 -0
- package/dist/core/index-manager.js +30 -0
- package/dist/core/index-manager.js.map +1 -0
- package/dist/core/ingest.d.ts +46 -0
- package/dist/core/ingest.d.ts.map +1 -0
- package/dist/core/ingest.js +366 -0
- package/dist/core/ingest.js.map +1 -0
- package/dist/core/lint.d.ts +19 -0
- package/dist/core/lint.d.ts.map +1 -0
- package/dist/core/lint.js +90 -0
- package/dist/core/lint.js.map +1 -0
- package/dist/core/log-manager.d.ts +2 -0
- package/dist/core/log-manager.d.ts.map +1 -0
- package/dist/core/log-manager.js +14 -0
- package/dist/core/log-manager.js.map +1 -0
- package/dist/core/obsidian.d.ts +89 -0
- package/dist/core/obsidian.d.ts.map +1 -0
- package/dist/core/obsidian.js +123 -0
- package/dist/core/obsidian.js.map +1 -0
- package/dist/core/query.d.ts +16 -0
- package/dist/core/query.d.ts.map +1 -0
- package/dist/core/query.js +77 -0
- package/dist/core/query.js.map +1 -0
- package/dist/core/scrape.d.ts +13 -0
- package/dist/core/scrape.d.ts.map +1 -0
- package/dist/core/scrape.js +103 -0
- package/dist/core/scrape.js.map +1 -0
- package/dist/core/vault.d.ts +35 -0
- package/dist/core/vault.d.ts.map +1 -0
- package/dist/core/vault.js +119 -0
- package/dist/core/vault.js.map +1 -0
- package/dist/core/watcher.d.ts +4 -0
- package/dist/core/watcher.d.ts.map +1 -0
- package/dist/core/watcher.js +34 -0
- package/dist/core/watcher.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/processors/audio.d.ts +10 -0
- package/dist/processors/audio.d.ts.map +1 -0
- package/dist/processors/audio.js +139 -0
- package/dist/processors/audio.js.map +1 -0
- package/dist/processors/docx.d.ts +12 -0
- package/dist/processors/docx.d.ts.map +1 -0
- package/dist/processors/docx.js +98 -0
- package/dist/processors/docx.js.map +1 -0
- package/dist/processors/image.d.ts +9 -0
- package/dist/processors/image.d.ts.map +1 -0
- package/dist/processors/image.js +94 -0
- package/dist/processors/image.js.map +1 -0
- package/dist/processors/pdf.d.ts +10 -0
- package/dist/processors/pdf.d.ts.map +1 -0
- package/dist/processors/pdf.js +92 -0
- package/dist/processors/pdf.js.map +1 -0
- package/dist/processors/pptx.d.ts +13 -0
- package/dist/processors/pptx.d.ts.map +1 -0
- package/dist/processors/pptx.js +165 -0
- package/dist/processors/pptx.js.map +1 -0
- package/dist/processors/text.d.ts +7 -0
- package/dist/processors/text.d.ts.map +1 -0
- package/dist/processors/text.js +9 -0
- package/dist/processors/text.js.map +1 -0
- package/dist/processors/url.d.ts +7 -0
- package/dist/processors/url.d.ts.map +1 -0
- package/dist/processors/url.js +61 -0
- package/dist/processors/url.js.map +1 -0
- package/dist/processors/video.d.ts +10 -0
- package/dist/processors/video.d.ts.map +1 -0
- package/dist/processors/video.js +115 -0
- package/dist/processors/video.js.map +1 -0
- package/dist/processors/xlsx.d.ts +13 -0
- package/dist/processors/xlsx.d.ts.map +1 -0
- package/dist/processors/xlsx.js +138 -0
- package/dist/processors/xlsx.js.map +1 -0
- package/dist/providers/claude.d.ts +10 -0
- package/dist/providers/claude.d.ts.map +1 -0
- package/dist/providers/claude.js +44 -0
- package/dist/providers/claude.js.map +1 -0
- package/dist/providers/embeddings.d.ts +62 -0
- package/dist/providers/embeddings.d.ts.map +1 -0
- package/dist/providers/embeddings.js +206 -0
- package/dist/providers/embeddings.js.map +1 -0
- package/dist/providers/index.d.ts +7 -0
- package/dist/providers/index.d.ts.map +1 -0
- package/dist/providers/index.js +19 -0
- package/dist/providers/index.js.map +1 -0
- package/dist/providers/ollama.d.ts +10 -0
- package/dist/providers/ollama.d.ts.map +1 -0
- package/dist/providers/ollama.js +48 -0
- package/dist/providers/ollama.js.map +1 -0
- package/dist/providers/openai.d.ts +10 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +38 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/providers/types.d.ts +33 -0
- package/dist/providers/types.d.ts.map +1 -0
- package/dist/providers/types.js +2 -0
- package/dist/providers/types.js.map +1 -0
- package/dist/search/bm25.d.ts +18 -0
- package/dist/search/bm25.d.ts.map +1 -0
- package/dist/search/bm25.js +52 -0
- package/dist/search/bm25.js.map +1 -0
- package/dist/search/index.d.ts +12 -0
- package/dist/search/index.d.ts.map +1 -0
- package/dist/search/index.js +64 -0
- package/dist/search/index.js.map +1 -0
- package/dist/search/semantic.d.ts +30 -0
- package/dist/search/semantic.d.ts.map +1 -0
- package/dist/search/semantic.js +162 -0
- package/dist/search/semantic.js.map +1 -0
- package/dist/templates/agents-md.d.ts +2 -0
- package/dist/templates/agents-md.d.ts.map +1 -0
- package/dist/templates/agents-md.js +85 -0
- package/dist/templates/agents-md.js.map +1 -0
- package/dist/templates/config-yaml.d.ts +2 -0
- package/dist/templates/config-yaml.d.ts.map +1 -0
- package/dist/templates/config-yaml.js +81 -0
- package/dist/templates/config-yaml.js.map +1 -0
- package/dist/web/server.d.ts +2 -0
- package/dist/web/server.d.ts.map +1 -0
- package/dist/web/server.js +170 -0
- package/dist/web/server.js.map +1 -0
- package/package.json +68 -0
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync, copyFileSync, readdirSync, statSync } from 'node:fs';
|
|
2
|
+
import { join, basename, extname, resolve } from 'node:path';
|
|
3
|
+
import { readWikiPage, writeWikiPage, listWikiPages, slugify } from './vault.js';
|
|
4
|
+
import { updateIndex } from './index-manager.js';
|
|
5
|
+
import { appendLog } from './log-manager.js';
|
|
6
|
+
import { processUrl } from '../processors/url.js';
|
|
7
|
+
import { isImageFile, processImage } from '../processors/image.js';
|
|
8
|
+
import { isAudioFile, processAudio } from '../processors/audio.js';
|
|
9
|
+
import { isVideoFile, processVideo } from '../processors/video.js';
|
|
10
|
+
import { isPdfFile, processPdf } from '../processors/pdf.js';
|
|
11
|
+
import { processDocx } from '../processors/docx.js';
|
|
12
|
+
import { processXlsx } from '../processors/xlsx.js';
|
|
13
|
+
import { processPptx } from '../processors/pptx.js';
|
|
14
|
+
import { embedPage } from '../search/semantic.js';
|
|
15
|
+
export async function ingestSource(source, config, provider, options) {
|
|
16
|
+
const isUrl = source.startsWith('http://') || source.startsWith('https://');
|
|
17
|
+
const now = new Date().toISOString().split('T')[0] ?? '';
|
|
18
|
+
// Step 1: Get content as markdown
|
|
19
|
+
let content;
|
|
20
|
+
let title;
|
|
21
|
+
let rawPath;
|
|
22
|
+
if (isUrl) {
|
|
23
|
+
const urlResult = await processUrl(source);
|
|
24
|
+
content = urlResult.content;
|
|
25
|
+
title = urlResult.title;
|
|
26
|
+
// Save to raw/ with date stamp
|
|
27
|
+
const dateDir = join(config.rawDir, now);
|
|
28
|
+
mkdirSync(dateDir, { recursive: true });
|
|
29
|
+
rawPath = join(dateDir, `${slugify(title)}.md`);
|
|
30
|
+
writeFileSync(rawPath, content, 'utf-8');
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
if (!existsSync(source)) {
|
|
34
|
+
throw new Error(`File not found: ${source}`);
|
|
35
|
+
}
|
|
36
|
+
const ext = extname(source).toLowerCase();
|
|
37
|
+
rawPath = source;
|
|
38
|
+
// Copy to raw/ if not already there
|
|
39
|
+
if (!source.startsWith(config.rawDir)) {
|
|
40
|
+
const dateDir = join(config.rawDir, now);
|
|
41
|
+
mkdirSync(dateDir, { recursive: true });
|
|
42
|
+
rawPath = join(dateDir, basename(source));
|
|
43
|
+
copyFileSync(source, rawPath);
|
|
44
|
+
}
|
|
45
|
+
// Process based on file type — supports every major format
|
|
46
|
+
if (isImageFile(source)) {
|
|
47
|
+
const result = await processImage(source);
|
|
48
|
+
content = result.markdown;
|
|
49
|
+
title = result.title;
|
|
50
|
+
}
|
|
51
|
+
else if (isAudioFile(source)) {
|
|
52
|
+
const result = await processAudio(source);
|
|
53
|
+
content = result.markdown;
|
|
54
|
+
title = result.title;
|
|
55
|
+
}
|
|
56
|
+
else if (isVideoFile(source)) {
|
|
57
|
+
const result = await processVideo(source);
|
|
58
|
+
content = result.markdown;
|
|
59
|
+
title = result.title;
|
|
60
|
+
}
|
|
61
|
+
else if (isPdfFile(source)) {
|
|
62
|
+
const result = await processPdf(source);
|
|
63
|
+
content = result.markdown;
|
|
64
|
+
title = result.title;
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
// Text-based formats: .md, .txt, .csv, .json, .yaml, .xml, .html,
|
|
68
|
+
// Office formats: .docx, .xlsx, .pptx (full extraction)
|
|
69
|
+
switch (ext) {
|
|
70
|
+
case '.docx':
|
|
71
|
+
case '.doc': {
|
|
72
|
+
const docResult = await processDocx(source);
|
|
73
|
+
content = docResult.markdown;
|
|
74
|
+
title = docResult.title;
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
case '.xlsx':
|
|
78
|
+
case '.xls': {
|
|
79
|
+
const xlsResult = await processXlsx(source);
|
|
80
|
+
content = xlsResult.markdown;
|
|
81
|
+
title = xlsResult.title;
|
|
82
|
+
break;
|
|
83
|
+
}
|
|
84
|
+
case '.pptx':
|
|
85
|
+
case '.ppt': {
|
|
86
|
+
const pptResult = await processPptx(source);
|
|
87
|
+
content = pptResult.markdown;
|
|
88
|
+
title = pptResult.title;
|
|
89
|
+
break;
|
|
90
|
+
}
|
|
91
|
+
case '.json':
|
|
92
|
+
content = `# ${basename(source, ext)}\n\n\`\`\`json\n${readFileSync(source, 'utf-8').substring(0, 20000)}\n\`\`\``;
|
|
93
|
+
title = basename(source, ext);
|
|
94
|
+
break;
|
|
95
|
+
case '.html':
|
|
96
|
+
case '.htm':
|
|
97
|
+
const html = readFileSync(source, 'utf-8');
|
|
98
|
+
const htmlTitle = html.match(/<title>(.*?)<\/title>/i)?.[1];
|
|
99
|
+
content = html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().substring(0, 20000);
|
|
100
|
+
title = htmlTitle ?? basename(source, ext);
|
|
101
|
+
break;
|
|
102
|
+
default:
|
|
103
|
+
content = readFileSync(source, 'utf-8');
|
|
104
|
+
title = basename(source, ext);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
// Step 2: Semantic dedup check (unless --force)
|
|
109
|
+
if (!options.force) {
|
|
110
|
+
const existingPages = listWikiPages(config.wikiDir);
|
|
111
|
+
const isDuplicate = await checkDuplicate(content, existingPages, provider);
|
|
112
|
+
if (isDuplicate.duplicate) {
|
|
113
|
+
// Mark in raw/ frontmatter but don't add to wiki
|
|
114
|
+
const rejectionMeta = {
|
|
115
|
+
title,
|
|
116
|
+
rejected: true,
|
|
117
|
+
rejection_reason: isDuplicate.reason,
|
|
118
|
+
similar_to: isDuplicate.similarPage,
|
|
119
|
+
similarity_score: isDuplicate.score,
|
|
120
|
+
source: rawPath,
|
|
121
|
+
date: now,
|
|
122
|
+
};
|
|
123
|
+
// Write rejection metadata alongside the raw file
|
|
124
|
+
writeFileSync(rawPath + '.meta.json', JSON.stringify(rejectionMeta, null, 2), 'utf-8');
|
|
125
|
+
return {
|
|
126
|
+
title,
|
|
127
|
+
pagesUpdated: 0,
|
|
128
|
+
linksAdded: 0,
|
|
129
|
+
rawPath,
|
|
130
|
+
rejected: true,
|
|
131
|
+
rejectionReason: isDuplicate.reason,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
// Step 3: Ask LLM to process and integrate into wiki
|
|
136
|
+
const schema = existsSync(config.schemaPath)
|
|
137
|
+
? readFileSync(config.schemaPath, 'utf-8')
|
|
138
|
+
: '';
|
|
139
|
+
const indexContent = existsSync(config.indexPath)
|
|
140
|
+
? readFileSync(config.indexPath, 'utf-8')
|
|
141
|
+
: '';
|
|
142
|
+
const prompt = buildIngestPrompt(content, title, schema, indexContent);
|
|
143
|
+
const response = await provider.chat([
|
|
144
|
+
{ role: 'user', content: prompt },
|
|
145
|
+
], {
|
|
146
|
+
systemPrompt: `You are a wiki maintainer. You process source documents and produce structured wiki pages in markdown with YAML frontmatter and [[wikilinks]]. Follow the schema in AGENTS.md exactly. Be concise, factual, and thorough in cross-referencing.`,
|
|
147
|
+
maxTokens: 8192,
|
|
148
|
+
});
|
|
149
|
+
// Step 4: Parse LLM response into wiki pages
|
|
150
|
+
const pages = parseLLMPages(response.content);
|
|
151
|
+
let pagesUpdated = 0;
|
|
152
|
+
let linksAdded = 0;
|
|
153
|
+
for (const page of pages) {
|
|
154
|
+
// User-supplied category overrides LLM-detected category
|
|
155
|
+
const pageCategory = options.category ?? page.category;
|
|
156
|
+
const pagePath = join(config.wikiDir, pageCategory, `${slugify(page.title)}.md`);
|
|
157
|
+
const dir = join(pagePath, '..');
|
|
158
|
+
if (!existsSync(dir)) {
|
|
159
|
+
mkdirSync(dir, { recursive: true });
|
|
160
|
+
}
|
|
161
|
+
// Merge user-supplied tags with LLM-detected tags (dedup)
|
|
162
|
+
const mergedTags = [...new Set([...page.tags, ...(options.tags ?? [])])];
|
|
163
|
+
const frontmatterData = {
|
|
164
|
+
title: page.title,
|
|
165
|
+
type: pageCategory,
|
|
166
|
+
created: now,
|
|
167
|
+
updated: now,
|
|
168
|
+
tags: mergedTags,
|
|
169
|
+
sources: [rawPath],
|
|
170
|
+
summary: page.summary,
|
|
171
|
+
};
|
|
172
|
+
// Merge any custom metadata from --interactive or future extensions
|
|
173
|
+
if (options.metadata) {
|
|
174
|
+
for (const [key, value] of Object.entries(options.metadata)) {
|
|
175
|
+
frontmatterData[key] = value;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
writeWikiPage(pagePath, page.content, frontmatterData);
|
|
179
|
+
pagesUpdated++;
|
|
180
|
+
linksAdded += (page.content.match(/\[\[[^\]]+\]\]/g) ?? []).length;
|
|
181
|
+
}
|
|
182
|
+
// Step 5: Generate embeddings for new pages (if provider configured)
|
|
183
|
+
if (options.embeddingProvider) {
|
|
184
|
+
for (const page of pages) {
|
|
185
|
+
const pageCategory = options.category ?? page.category;
|
|
186
|
+
const pagePath = join(config.wikiDir, pageCategory, `${slugify(page.title)}.md`);
|
|
187
|
+
try {
|
|
188
|
+
await embedPage(pagePath, page.content, options.embeddingProvider);
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
// Embedding failure is non-fatal — page is still ingested
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
// Step 6: Update index.md and log.md
|
|
196
|
+
await updateIndex(config, pages);
|
|
197
|
+
appendLog(config.logPath, `ingest | ${title}`, `Processed ${source}. Created/updated ${pagesUpdated} pages.`);
|
|
198
|
+
return { title, pagesUpdated, linksAdded, rawPath };
|
|
199
|
+
}
|
|
200
|
+
async function checkDuplicate(content, existingPages, provider) {
|
|
201
|
+
if (existingPages.length === 0) {
|
|
202
|
+
return { duplicate: false, reason: '' };
|
|
203
|
+
}
|
|
204
|
+
// Simple heuristic first: check content length overlap
|
|
205
|
+
const contentSnippet = content.substring(0, 500);
|
|
206
|
+
for (const pagePath of existingPages.slice(0, 20)) {
|
|
207
|
+
try {
|
|
208
|
+
const page = readWikiPage(pagePath);
|
|
209
|
+
const pageSnippet = page.content.substring(0, 500);
|
|
210
|
+
// Simple Jaccard similarity on word sets
|
|
211
|
+
const wordsA = new Set(contentSnippet.toLowerCase().split(/\s+/));
|
|
212
|
+
const wordsB = new Set(pageSnippet.toLowerCase().split(/\s+/));
|
|
213
|
+
const intersection = new Set([...wordsA].filter((w) => wordsB.has(w)));
|
|
214
|
+
const union = new Set([...wordsA, ...wordsB]);
|
|
215
|
+
const similarity = union.size > 0 ? intersection.size / union.size : 0;
|
|
216
|
+
if (similarity > 0.7) {
|
|
217
|
+
return {
|
|
218
|
+
duplicate: true,
|
|
219
|
+
reason: `High word overlap (${Math.round(similarity * 100)}%) with existing page "${page.title}"`,
|
|
220
|
+
similarPage: pagePath,
|
|
221
|
+
score: similarity,
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
catch {
|
|
226
|
+
// Skip unreadable pages
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return { duplicate: false, reason: '' };
|
|
230
|
+
}
|
|
231
|
+
function buildIngestPrompt(content, title, schema, indexContent) {
|
|
232
|
+
return `# Task: Ingest Source Document
|
|
233
|
+
|
|
234
|
+
## Schema (AGENTS.md)
|
|
235
|
+
${schema}
|
|
236
|
+
|
|
237
|
+
## Current Wiki Index
|
|
238
|
+
${indexContent}
|
|
239
|
+
|
|
240
|
+
## Source Document: "${title}"
|
|
241
|
+
${content.substring(0, 12000)}
|
|
242
|
+
|
|
243
|
+
## Instructions
|
|
244
|
+
|
|
245
|
+
Process this source document and produce wiki pages. For each page, output in this exact format:
|
|
246
|
+
|
|
247
|
+
\`\`\`page
|
|
248
|
+
TITLE: Page Title
|
|
249
|
+
CATEGORY: sources | entities | concepts | syntheses
|
|
250
|
+
TAGS: tag1, tag2, tag3
|
|
251
|
+
SUMMARY: One-line summary for the index
|
|
252
|
+
---
|
|
253
|
+
Page content in markdown with [[wikilinks]] to other pages.
|
|
254
|
+
\`\`\`
|
|
255
|
+
|
|
256
|
+
Produce:
|
|
257
|
+
1. A source summary page (in sources/)
|
|
258
|
+
2. Entity pages for any notable people, tools, organizations mentioned (in entities/)
|
|
259
|
+
3. Concept pages for key ideas or frameworks discussed (in concepts/)
|
|
260
|
+
|
|
261
|
+
Use [[wikilinks]] extensively to connect pages. Every claim should reference its source.`;
|
|
262
|
+
}
|
|
263
|
+
function parseLLMPages(response) {
|
|
264
|
+
const pages = [];
|
|
265
|
+
const pageBlocks = response.split('```page').slice(1);
|
|
266
|
+
for (const block of pageBlocks) {
|
|
267
|
+
const endIdx = block.indexOf('```');
|
|
268
|
+
const pageContent = endIdx >= 0 ? block.substring(0, endIdx) : block;
|
|
269
|
+
const lines = pageContent.trim().split('\n');
|
|
270
|
+
const titleLine = lines.find((l) => l.startsWith('TITLE:'));
|
|
271
|
+
const categoryLine = lines.find((l) => l.startsWith('CATEGORY:'));
|
|
272
|
+
const tagsLine = lines.find((l) => l.startsWith('TAGS:'));
|
|
273
|
+
const summaryLine = lines.find((l) => l.startsWith('SUMMARY:'));
|
|
274
|
+
const separatorIdx = lines.findIndex((l) => l.trim() === '---');
|
|
275
|
+
const content = separatorIdx >= 0 ? lines.slice(separatorIdx + 1).join('\n').trim() : '';
|
|
276
|
+
if (titleLine && categoryLine) {
|
|
277
|
+
pages.push({
|
|
278
|
+
title: titleLine.replace('TITLE:', '').trim(),
|
|
279
|
+
category: categoryLine.replace('CATEGORY:', '').trim(),
|
|
280
|
+
tags: (tagsLine?.replace('TAGS:', '').trim() ?? '').split(',').map((t) => t.trim()).filter(Boolean),
|
|
281
|
+
summary: summaryLine?.replace('SUMMARY:', '').trim() ?? '',
|
|
282
|
+
content,
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
// If no structured pages found, create a single source page
|
|
287
|
+
if (pages.length === 0 && response.trim().length > 0) {
|
|
288
|
+
pages.push({
|
|
289
|
+
title: 'Untitled Source',
|
|
290
|
+
category: 'sources',
|
|
291
|
+
tags: [],
|
|
292
|
+
summary: 'Auto-generated source page',
|
|
293
|
+
content: response,
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
return pages;
|
|
297
|
+
}
|
|
298
|
+
/** Collect ingestable files from a directory */
|
|
299
|
+
export function collectFiles(dir, recursive) {
|
|
300
|
+
const files = [];
|
|
301
|
+
const resolved = resolve(dir);
|
|
302
|
+
if (!existsSync(resolved) || !statSync(resolved).isDirectory()) {
|
|
303
|
+
return files;
|
|
304
|
+
}
|
|
305
|
+
const entries = readdirSync(resolved);
|
|
306
|
+
for (const entry of entries) {
|
|
307
|
+
// Skip hidden files and .meta.json files
|
|
308
|
+
if (entry.startsWith('.') || entry.endsWith('.meta.json'))
|
|
309
|
+
continue;
|
|
310
|
+
const full = join(resolved, entry);
|
|
311
|
+
const stat = statSync(full);
|
|
312
|
+
if (stat.isDirectory() && recursive) {
|
|
313
|
+
files.push(...collectFiles(full, true));
|
|
314
|
+
}
|
|
315
|
+
else if (stat.isFile()) {
|
|
316
|
+
files.push(full);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
return files;
|
|
320
|
+
}
|
|
321
|
+
/** Check if a file was already ingested by reading log.md */
|
|
322
|
+
export function isAlreadyIngested(logPath, filePath) {
|
|
323
|
+
if (!existsSync(logPath))
|
|
324
|
+
return false;
|
|
325
|
+
const log = readFileSync(logPath, 'utf-8');
|
|
326
|
+
const fileName = basename(filePath);
|
|
327
|
+
return log.includes(fileName);
|
|
328
|
+
}
|
|
329
|
+
/** Scan raw/ for .meta.json files indicating rejected duplicates */
|
|
330
|
+
export function listDuplicates(rawDir) {
|
|
331
|
+
const duplicates = [];
|
|
332
|
+
if (!existsSync(rawDir))
|
|
333
|
+
return duplicates;
|
|
334
|
+
function walk(dir) {
|
|
335
|
+
const entries = readdirSync(dir);
|
|
336
|
+
for (const entry of entries) {
|
|
337
|
+
const full = join(dir, entry);
|
|
338
|
+
const stat = statSync(full);
|
|
339
|
+
if (stat.isDirectory()) {
|
|
340
|
+
walk(full);
|
|
341
|
+
}
|
|
342
|
+
else if (entry.endsWith('.meta.json')) {
|
|
343
|
+
try {
|
|
344
|
+
const raw = readFileSync(full, 'utf-8');
|
|
345
|
+
const meta = JSON.parse(raw);
|
|
346
|
+
if (meta['rejected'] === true) {
|
|
347
|
+
duplicates.push({
|
|
348
|
+
file: full.replace('.meta.json', ''),
|
|
349
|
+
title: meta['title'] ?? basename(full, '.meta.json'),
|
|
350
|
+
reason: meta['rejection_reason'] ?? 'Unknown',
|
|
351
|
+
similarTo: meta['similar_to'] ?? 'Unknown',
|
|
352
|
+
score: meta['similarity_score'] ?? 0,
|
|
353
|
+
date: meta['date'] ?? '',
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
catch {
|
|
358
|
+
// Skip malformed meta files
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
walk(rawDir);
|
|
364
|
+
return duplicates;
|
|
365
|
+
}
|
|
366
|
+
//# sourceMappingURL=ingest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/core/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAClH,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAG7D,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAqBlD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAc,EACd,MAAmB,EACnB,QAAqB,EACrB,OAAsB;IAEtB,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IAC5E,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAEzD,kCAAkC;IAClC,IAAI,OAAe,CAAC;IACpB,IAAI,KAAa,CAAC;IAClB,IAAI,OAAe,CAAC;IAEpB,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;QAC5B,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;QACxB,+BAA+B;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACzC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAChD,aAAa,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,EAAE,CAAC,CAAC;QAC/C,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;QAC1C,OAAO,GAAG,MAAM,CAAC;QAEjB,oCAAoC;QACpC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YACzC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACxC,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;YAC1C,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAChC,CAAC;QAED,2DAA2D;QAC3D,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;YAC1C,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC;YAC1B,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACvB,CAAC;aAAM,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;YAC1C,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC;YAC1B,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACvB,CAAC;aAAM,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;YAC1C,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC;YAC1B,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACvB,CAAC;aAAM,IAAI,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;YACxC,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC;YAC1B,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,kEAAkE;YAClE,wDAAwD;YACxD,QAAQ,GAAG,EAAE,CAAC;gBACZ,KAAK,OAAO,CAAC;gBACb,KAAK,MAAM,CAAC,CAAC,CAAC;oBACZ,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;oBAC5C,OAAO,GAAG,SAAS,CAAC,QAAQ,CAAC;oBAC7B,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;oBACxB,MAAM;gBACR,CAAC;gBACD,KAAK,OAAO,CAAC;gBACb,KAAK,MAAM,CAAC,CAAC,CAAC;oBACZ,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;oBAC5C,OAAO,GAAG,SAAS,CAAC,QAAQ,CAAC;oBAC7B,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;oBACxB,MAAM;gBACR,CAAC;gBACD,KAAK,OAAO,CAAC;gBACb,KAAK,MAAM,CAAC,CAAC,CAAC;oBACZ,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;oBAC5C,OAAO,GAAG,SAAS,CAAC,QAAQ,CAAC;oBAC7B,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;oBACxB,MAAM;gBACR,CAAC;gBACD,KAAK,OAAO;oBACV,OAAO,GAAG,KAAK,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,mBAAmB,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,UAAU,CAAC;oBACnH,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;oBAC9B,MAAM;gBACR,KAAK,OAAO,CAAC;gBACb,KAAK,MAAM;oBACT,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;oBAC5D,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;oBACxF,KAAK,GAAG,SAAS,IAAI,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;oBAC3C,MAAM;gBACR;oBACE,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBACxC,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAED,gDAAgD;IAChD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,aAAa,GAAG,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;QAC3E,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;YAC1B,iDAAiD;YACjD,MAAM,aAAa,GAAG;gBACpB,KAAK;gBACL,QAAQ,EAAE,IAAI;gBACd,gBAAgB,EAAE,WAAW,CAAC,MAAM;gBACpC,UAAU,EAAE,WAAW,CAAC,WAAW;gBACnC,gBAAgB,EAAE,WAAW,CAAC,KAAK;gBACnC,MAAM,EAAE,OAAO;gBACf,IAAI,EAAE,GAAG;aACV,CAAC;YACF,kDAAkD;YAClD,aAAa,CACX,OAAO,GAAG,YAAY,EACtB,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,EACtC,OAAO,CACR,CAAC;YACF,OAAO;gBACL,KAAK;gBACL,YAAY,EAAE,CAAC;gBACf,UAAU,EAAE,CAAC;gBACb,OAAO;gBACP,QAAQ,EAAE,IAAI;gBACd,eAAe,EAAE,WAAW,CAAC,MAAM;aACpC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAC1C,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC;QAC1C,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC;QAC/C,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC;QACzC,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IAEvE,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC;QACnC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;KAClC,EAAE;QACD,YAAY,EAAE,gPAAgP;QAC9P,SAAS,EAAE,IAAI;KAChB,CAAC,CAAC;IAEH,6CAA6C;IAC7C,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC9C,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,yDAAyD;QACzD,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACjF,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,CAAC;QAED,0DAA0D;QAC1D,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEzE,MAAM,eAAe,GAA4B;YAC/C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,GAAG;YACZ,OAAO,EAAE,GAAG;YACZ,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,CAAC,OAAO,CAAC;YAClB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;QAEF,oEAAoE;QACpE,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5D,eAAe,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;QAEvD,YAAY,EAAE,CAAC;QACf,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACrE,CAAC;IAED,qEAAqE;IACrE,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC;YACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACjF,IAAI,CAAC;gBACH,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACrE,CAAC;YAAC,MAAM,CAAC;gBACP,0DAA0D;YAC5D,CAAC;QACH,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,MAAM,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACjC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,KAAK,EAAE,EAAE,aAAa,MAAM,qBAAqB,YAAY,SAAS,CAAC,CAAC;IAE9G,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;AACtD,CAAC;AASD,KAAK,UAAU,cAAc,CAC3B,OAAe,EACf,aAAuB,EACvB,QAAqB;IAErB,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAC1C,CAAC;IAED,uDAAuD;IACvD,MAAM,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAEjD,KAAK,MAAM,QAAQ,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;YACpC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAEnD,yCAAyC;YACzC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAClE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/D,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC;YAC9C,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAEvE,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;gBACrB,OAAO;oBACL,SAAS,EAAE,IAAI;oBACf,MAAM,EAAE,sBAAsB,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,0BAA0B,IAAI,CAAC,KAAK,GAAG;oBACjG,WAAW,EAAE,QAAQ;oBACrB,KAAK,EAAE,UAAU;iBAClB,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wBAAwB;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAe,EACf,KAAa,EACb,MAAc,EACd,YAAoB;IAEpB,OAAO;;;EAGP,MAAM;;;EAGN,YAAY;;uBAES,KAAK;EAC1B,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC;;;;;;;;;;;;;;;;;;;;yFAoB4D,CAAC;AAC1F,CAAC;AAUD,SAAS,aAAa,CAAC,QAAgB;IACrC,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEtD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAErE,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC5D,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;QAEhE,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,KAAK,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAEzF,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;gBAC7C,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE;gBACtD,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;gBACnG,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE;gBAC1D,OAAO;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,4DAA4D;IAC5D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC;YACT,KAAK,EAAE,iBAAiB;YACxB,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,EAAE;YACR,OAAO,EAAE,4BAA4B;YACrC,OAAO,EAAE,QAAQ;SAClB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAYD,gDAAgD;AAChD,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,SAAkB;IAC1D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QAC/D,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACtC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,yCAAyC;QACzC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC;YAAE,SAAS;QACpE,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,WAAW,EAAE,IAAI,SAAS,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAC1C,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,iBAAiB,CAAC,OAAe,EAAE,QAAgB;IACjE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACpC,OAAO,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAChC,CAAC;AAaD,oEAAoE;AACpE,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,UAAU,CAAC;IAE3C,SAAS,IAAI,CAAC,GAAW;QACvB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC,IAAI,CAAC,CAAC;YACb,CAAC;iBAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACxC,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;oBACxC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;oBACxD,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;wBAC9B,UAAU,CAAC,IAAI,CAAC;4BACd,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC;4BACpC,KAAK,EAAG,IAAI,CAAC,OAAO,CAAY,IAAI,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;4BAChE,MAAM,EAAG,IAAI,CAAC,kBAAkB,CAAY,IAAI,SAAS;4BACzD,SAAS,EAAG,IAAI,CAAC,YAAY,CAAY,IAAI,SAAS;4BACtD,KAAK,EAAG,IAAI,CAAC,kBAAkB,CAAY,IAAI,CAAC;4BAChD,IAAI,EAAG,IAAI,CAAC,MAAM,CAAY,IAAI,EAAE;yBACrC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,4BAA4B;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,CAAC;IACb,OAAO,UAAU,CAAC;AACpB,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { LLMProvider } from '../providers/types.js';
|
|
2
|
+
import type { VaultConfig } from './vault.js';
|
|
3
|
+
export interface LintIssue {
|
|
4
|
+
category: 'orphan' | 'contradiction' | 'stale' | 'missing-link' | 'empty' | 'no-summary';
|
|
5
|
+
severity: 'error' | 'warning';
|
|
6
|
+
message: string;
|
|
7
|
+
page?: string;
|
|
8
|
+
fixed?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export interface LintResult {
|
|
11
|
+
score: number;
|
|
12
|
+
issues: LintIssue[];
|
|
13
|
+
}
|
|
14
|
+
interface LintOptions {
|
|
15
|
+
fix: boolean;
|
|
16
|
+
}
|
|
17
|
+
export declare function lintWiki(config: VaultConfig, _provider: LLMProvider, _options: LintOptions): Promise<LintResult>;
|
|
18
|
+
export {};
|
|
19
|
+
//# sourceMappingURL=lint.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lint.d.ts","sourceRoot":"","sources":["../../src/core/lint.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAI9C,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,QAAQ,GAAG,eAAe,GAAG,OAAO,GAAG,cAAc,GAAG,OAAO,GAAG,YAAY,CAAC;IACzF,QAAQ,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,SAAS,EAAE,CAAC;CACrB;AAED,UAAU,WAAW;IACnB,GAAG,EAAE,OAAO,CAAC;CACd;AAED,wBAAsB,QAAQ,CAC5B,MAAM,EAAE,WAAW,EACnB,SAAS,EAAE,WAAW,EACtB,QAAQ,EAAE,WAAW,GACpB,OAAO,CAAC,UAAU,CAAC,CA8FrB"}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { listWikiPages, readWikiPage } from './vault.js';
|
|
2
|
+
export async function lintWiki(config, _provider, _options) {
|
|
3
|
+
const issues = [];
|
|
4
|
+
const pages = listWikiPages(config.wikiDir);
|
|
5
|
+
if (pages.length === 0) {
|
|
6
|
+
return { score: 0, issues: [{ category: 'empty', severity: 'error', message: 'Wiki has no pages. Run `llmwiki ingest` to add content.' }] };
|
|
7
|
+
}
|
|
8
|
+
// Collect all page titles and wikilinks
|
|
9
|
+
const allTitles = new Set();
|
|
10
|
+
const allLinkedTo = new Set();
|
|
11
|
+
const pageData = [];
|
|
12
|
+
for (const pagePath of pages) {
|
|
13
|
+
try {
|
|
14
|
+
const page = readWikiPage(pagePath);
|
|
15
|
+
allTitles.add(page.title);
|
|
16
|
+
for (const link of page.wikilinks) {
|
|
17
|
+
allLinkedTo.add(link);
|
|
18
|
+
}
|
|
19
|
+
pageData.push({
|
|
20
|
+
path: pagePath,
|
|
21
|
+
title: page.title,
|
|
22
|
+
wikilinks: page.wikilinks,
|
|
23
|
+
wordCount: page.wordCount,
|
|
24
|
+
frontmatter: page.frontmatter,
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
issues.push({
|
|
29
|
+
category: 'empty',
|
|
30
|
+
severity: 'error',
|
|
31
|
+
message: `Failed to read page: ${pagePath}`,
|
|
32
|
+
page: pagePath,
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
// Check 1: Orphan pages (no inbound links)
|
|
37
|
+
for (const page of pageData) {
|
|
38
|
+
if (page.title === 'index' || page.title === 'log')
|
|
39
|
+
continue;
|
|
40
|
+
if (!allLinkedTo.has(page.title)) {
|
|
41
|
+
issues.push({
|
|
42
|
+
category: 'orphan',
|
|
43
|
+
severity: 'warning',
|
|
44
|
+
message: `Orphan page: "${page.title}" has no inbound links`,
|
|
45
|
+
page: page.path,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
// Check 2: Broken wikilinks (link to non-existent page)
|
|
50
|
+
for (const page of pageData) {
|
|
51
|
+
for (const link of page.wikilinks) {
|
|
52
|
+
if (!allTitles.has(link)) {
|
|
53
|
+
issues.push({
|
|
54
|
+
category: 'missing-link',
|
|
55
|
+
severity: 'warning',
|
|
56
|
+
message: `Broken wikilink: "${page.title}" links to non-existent "[[${link}]]"`,
|
|
57
|
+
page: page.path,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
// Check 3: Pages without summary in frontmatter
|
|
63
|
+
for (const page of pageData) {
|
|
64
|
+
if (!page.frontmatter['summary'] && page.title !== 'index' && page.title !== 'log') {
|
|
65
|
+
issues.push({
|
|
66
|
+
category: 'no-summary',
|
|
67
|
+
severity: 'warning',
|
|
68
|
+
message: `Page "${page.title}" missing frontmatter summary`,
|
|
69
|
+
page: page.path,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
// Check 4: Empty pages (< 10 words)
|
|
74
|
+
for (const page of pageData) {
|
|
75
|
+
if (page.wordCount < 10 && page.title !== 'index' && page.title !== 'log') {
|
|
76
|
+
issues.push({
|
|
77
|
+
category: 'empty',
|
|
78
|
+
severity: 'warning',
|
|
79
|
+
message: `Page "${page.title}" is nearly empty (${page.wordCount} words)`,
|
|
80
|
+
page: page.path,
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
// Calculate score
|
|
85
|
+
const maxIssues = pageData.length * 3; // rough estimate
|
|
86
|
+
const issueWeight = issues.reduce((sum, i) => sum + (i.severity === 'error' ? 10 : 3), 0);
|
|
87
|
+
const score = Math.max(0, Math.min(100, Math.round(100 - (issueWeight / Math.max(maxIssues, 1)) * 100)));
|
|
88
|
+
return { score, issues };
|
|
89
|
+
}
|
|
90
|
+
//# sourceMappingURL=lint.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lint.js","sourceRoot":"","sources":["../../src/core/lint.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,YAAY,EAAiB,MAAM,YAAY,CAAC;AAoBxE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,MAAmB,EACnB,SAAsB,EACtB,QAAqB;IAErB,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE5C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,yDAAyD,EAAE,CAAC,EAAE,CAAC;IAC9I,CAAC;IAED,wCAAwC;IACxC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,MAAM,QAAQ,GAAyH,EAAE,CAAC;IAE1I,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;YACpC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,OAAO;gBACjB,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,wBAAwB,QAAQ,EAAE;gBAC3C,IAAI,EAAE,QAAQ;aACf,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,2CAA2C;IAC3C,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK;YAAE,SAAS;QAC7D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,QAAQ;gBAClB,QAAQ,EAAE,SAAS;gBACnB,OAAO,EAAE,iBAAiB,IAAI,CAAC,KAAK,wBAAwB;gBAC5D,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC;oBACV,QAAQ,EAAE,cAAc;oBACxB,QAAQ,EAAE,SAAS;oBACnB,OAAO,EAAE,qBAAqB,IAAI,CAAC,KAAK,8BAA8B,IAAI,KAAK;oBAC/E,IAAI,EAAE,IAAI,CAAC,IAAI;iBAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,gDAAgD;IAChD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;YACnF,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,YAAY;gBACtB,QAAQ,EAAE,SAAS;gBACnB,OAAO,EAAE,SAAS,IAAI,CAAC,KAAK,+BAA+B;gBAC3D,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,IAAI,IAAI,CAAC,SAAS,GAAG,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,OAAO;gBACjB,QAAQ,EAAE,SAAS;gBACnB,OAAO,EAAE,SAAS,IAAI,CAAC,KAAK,sBAAsB,IAAI,CAAC,SAAS,SAAS;gBACzE,IAAI,EAAE,IAAI,CAAC,IAAI;aAChB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,iBAAiB;IACxD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1F,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAEzG,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"log-manager.d.ts","sourceRoot":"","sources":["../../src/core/log-manager.ts"],"names":[],"mappings":"AAEA,wBAAgB,SAAS,CACvB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,IAAI,CAWN"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
|
|
2
|
+
export function appendLog(logPath, operation, details) {
|
|
3
|
+
const now = new Date().toISOString().split('T')[0] ?? '';
|
|
4
|
+
const time = new Date().toISOString().split('T')[1]?.substring(0, 5) ?? '';
|
|
5
|
+
const entry = `\n## [${now} ${time}] ${operation}\n\n${details}\n`;
|
|
6
|
+
if (existsSync(logPath)) {
|
|
7
|
+
const existing = readFileSync(logPath, 'utf-8');
|
|
8
|
+
writeFileSync(logPath, existing + entry, 'utf-8');
|
|
9
|
+
}
|
|
10
|
+
else {
|
|
11
|
+
writeFileSync(logPath, `---\ntitle: Wiki Log\ntype: log\n---\n\n# Wiki Log\n${entry}`, 'utf-8');
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=log-manager.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"log-manager.js","sourceRoot":"","sources":["../../src/core/log-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAElE,MAAM,UAAU,SAAS,CACvB,OAAe,EACf,SAAiB,EACjB,OAAe;IAEf,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3E,MAAM,KAAK,GAAG,SAAS,GAAG,IAAI,IAAI,KAAK,SAAS,OAAO,OAAO,IAAI,CAAC;IAEnE,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAChD,aAAa,CAAC,OAAO,EAAE,QAAQ,GAAG,KAAK,EAAE,OAAO,CAAC,CAAC;IACpD,CAAC;SAAM,CAAC;QACN,aAAa,CAAC,OAAO,EAAE,uDAAuD,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IAClG,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import type { VaultConfig } from './vault.js';
|
|
2
|
+
/**
|
|
3
|
+
* Obsidian integration — generates .obsidian/ config directory
|
|
4
|
+
* for optimal graph display when `llmwiki init` runs.
|
|
5
|
+
*/
|
|
6
|
+
export interface ObsidianConfig {
|
|
7
|
+
app: AppConfig;
|
|
8
|
+
graph: GraphConfig;
|
|
9
|
+
appearance: AppearanceConfig;
|
|
10
|
+
}
|
|
11
|
+
export interface AppConfig {
|
|
12
|
+
livePreview: boolean;
|
|
13
|
+
readableLineLength: boolean;
|
|
14
|
+
showFrontmatter: boolean;
|
|
15
|
+
strictLineBreaks: boolean;
|
|
16
|
+
useMarkdownLinks: boolean;
|
|
17
|
+
showLineNumber: boolean;
|
|
18
|
+
foldHeading: boolean;
|
|
19
|
+
foldIndent: boolean;
|
|
20
|
+
}
|
|
21
|
+
export interface GraphConfig {
|
|
22
|
+
collapse_filter: boolean;
|
|
23
|
+
search: string;
|
|
24
|
+
showTags: boolean;
|
|
25
|
+
showAttachments: boolean;
|
|
26
|
+
hideUnresolved: boolean;
|
|
27
|
+
showOrphans: boolean;
|
|
28
|
+
collapse_color: boolean;
|
|
29
|
+
colorGroups: GraphColorGroup[];
|
|
30
|
+
collapse_display: boolean;
|
|
31
|
+
showArrow: boolean;
|
|
32
|
+
textFadeMultiplier: number;
|
|
33
|
+
nodeSizeMultiplier: number;
|
|
34
|
+
lineSizeMultiplier: number;
|
|
35
|
+
collapse_forces: boolean;
|
|
36
|
+
centerStrength: number;
|
|
37
|
+
repelStrength: number;
|
|
38
|
+
linkStrength: number;
|
|
39
|
+
linkDistance: number;
|
|
40
|
+
}
|
|
41
|
+
export interface GraphColorGroup {
|
|
42
|
+
query: string;
|
|
43
|
+
color: {
|
|
44
|
+
a: number;
|
|
45
|
+
r: number;
|
|
46
|
+
g: number;
|
|
47
|
+
b: number;
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
export interface AppearanceConfig {
|
|
51
|
+
accentColor: string;
|
|
52
|
+
theme: string;
|
|
53
|
+
cssTheme: string;
|
|
54
|
+
}
|
|
55
|
+
export interface GraphHint {
|
|
56
|
+
nodes: GraphHintNode[];
|
|
57
|
+
edges: GraphHintEdge[];
|
|
58
|
+
stats: {
|
|
59
|
+
nodeCount: number;
|
|
60
|
+
edgeCount: number;
|
|
61
|
+
orphanCount: number;
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
export interface GraphHintNode {
|
|
65
|
+
id: string;
|
|
66
|
+
title: string;
|
|
67
|
+
type: string;
|
|
68
|
+
linkCount: number;
|
|
69
|
+
}
|
|
70
|
+
export interface GraphHintEdge {
|
|
71
|
+
source: string;
|
|
72
|
+
target: string;
|
|
73
|
+
}
|
|
74
|
+
export declare function getDefaultObsidianConfig(): ObsidianConfig;
|
|
75
|
+
/**
|
|
76
|
+
* Generate the .obsidian/ directory with optimal config files
|
|
77
|
+
* for displaying an llmwiki vault.
|
|
78
|
+
*/
|
|
79
|
+
export declare function setupObsidian(root: string): void;
|
|
80
|
+
/**
|
|
81
|
+
* Generate a graph.json hint file for quick graph rendering.
|
|
82
|
+
* Reads all wiki pages and builds a node/edge adjacency list.
|
|
83
|
+
*/
|
|
84
|
+
export declare function generateGraphHint(config: VaultConfig): GraphHint;
|
|
85
|
+
/**
|
|
86
|
+
* Write the graph hint to disk for quick graph rendering.
|
|
87
|
+
*/
|
|
88
|
+
export declare function writeGraphHint(root: string, hint: GraphHint): void;
|
|
89
|
+
//# sourceMappingURL=obsidian.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"obsidian.d.ts","sourceRoot":"","sources":["../../src/core/obsidian.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAI9C;;;GAGG;AAEH,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,EAAE,WAAW,CAAC;IACnB,UAAU,EAAE,gBAAgB,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACxB,WAAW,EAAE,OAAO,CAAC;IACrB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,eAAe,EAAE,OAAO,CAAC;IACzB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,gBAAgB,EAAE,OAAO,CAAC;IAC1B,cAAc,EAAE,OAAO,CAAC;IACxB,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,OAAO,CAAC;IAClB,eAAe,EAAE,OAAO,CAAC;IACzB,cAAc,EAAE,OAAO,CAAC;IACxB,WAAW,EAAE,OAAO,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,WAAW,EAAE,eAAe,EAAE,CAAC;IAC/B,gBAAgB,EAAE,OAAO,CAAC;IAC1B,SAAS,EAAE,OAAO,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,OAAO,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACvD;AAED,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,KAAK,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;CACtE;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAmDD,wBAAgB,wBAAwB,IAAI,cAAc,CAMzD;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAyBhD;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,CAwChE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,IAAI,CAOlE"}
|