wikimem 0.2.3 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +138 -29
  2. package/README.md +173 -311
  3. package/dist/cli/commands/ask.d.ts +3 -0
  4. package/dist/cli/commands/ask.d.ts.map +1 -0
  5. package/dist/cli/commands/ask.js +63 -0
  6. package/dist/cli/commands/ask.js.map +1 -0
  7. package/dist/cli/commands/export.d.ts +3 -0
  8. package/dist/cli/commands/export.d.ts.map +1 -0
  9. package/dist/cli/commands/export.js +108 -0
  10. package/dist/cli/commands/export.js.map +1 -0
  11. package/dist/cli/commands/history.d.ts +3 -0
  12. package/dist/cli/commands/history.d.ts.map +1 -0
  13. package/dist/cli/commands/history.js +61 -0
  14. package/dist/cli/commands/history.js.map +1 -0
  15. package/dist/cli/commands/improve.d.ts.map +1 -1
  16. package/dist/cli/commands/improve.js +4 -3
  17. package/dist/cli/commands/improve.js.map +1 -1
  18. package/dist/cli/commands/ingest.d.ts.map +1 -1
  19. package/dist/cli/commands/ingest.js +5 -4
  20. package/dist/cli/commands/ingest.js.map +1 -1
  21. package/dist/cli/commands/init.d.ts.map +1 -1
  22. package/dist/cli/commands/init.js +246 -79
  23. package/dist/cli/commands/init.js.map +1 -1
  24. package/dist/cli/commands/lint.d.ts.map +1 -1
  25. package/dist/cli/commands/lint.js +4 -3
  26. package/dist/cli/commands/lint.js.map +1 -1
  27. package/dist/cli/commands/mcp.d.ts +3 -0
  28. package/dist/cli/commands/mcp.d.ts.map +1 -0
  29. package/dist/cli/commands/mcp.js +11 -0
  30. package/dist/cli/commands/mcp.js.map +1 -0
  31. package/dist/cli/commands/open.d.ts +3 -0
  32. package/dist/cli/commands/open.d.ts.map +1 -0
  33. package/dist/cli/commands/open.js +36 -0
  34. package/dist/cli/commands/open.js.map +1 -0
  35. package/dist/cli/commands/query.d.ts.map +1 -1
  36. package/dist/cli/commands/query.js +5 -4
  37. package/dist/cli/commands/query.js.map +1 -1
  38. package/dist/cli/commands/search.d.ts +3 -0
  39. package/dist/cli/commands/search.d.ts.map +1 -0
  40. package/dist/cli/commands/search.js +61 -0
  41. package/dist/cli/commands/search.js.map +1 -0
  42. package/dist/cli/commands/serve.d.ts.map +1 -1
  43. package/dist/cli/commands/serve.js +41 -2
  44. package/dist/cli/commands/serve.js.map +1 -1
  45. package/dist/cli/commands/watch.d.ts.map +1 -1
  46. package/dist/cli/commands/watch.js +4 -3
  47. package/dist/cli/commands/watch.js.map +1 -1
  48. package/dist/cli/index.d.ts.map +1 -1
  49. package/dist/cli/index.js +27 -1
  50. package/dist/cli/index.js.map +1 -1
  51. package/dist/core/audit-trail.d.ts +15 -0
  52. package/dist/core/audit-trail.d.ts.map +1 -0
  53. package/dist/core/audit-trail.js +43 -0
  54. package/dist/core/audit-trail.js.map +1 -0
  55. package/dist/core/claude-code.d.ts +10 -0
  56. package/dist/core/claude-code.d.ts.map +1 -0
  57. package/dist/core/claude-code.js +81 -0
  58. package/dist/core/claude-code.js.map +1 -0
  59. package/dist/core/config.d.ts +23 -0
  60. package/dist/core/config.d.ts.map +1 -1
  61. package/dist/core/config.js.map +1 -1
  62. package/dist/core/connectors.d.ts +58 -0
  63. package/dist/core/connectors.d.ts.map +1 -0
  64. package/dist/core/connectors.js +189 -0
  65. package/dist/core/connectors.js.map +1 -0
  66. package/dist/core/folder-scanner.d.ts +10 -0
  67. package/dist/core/folder-scanner.d.ts.map +1 -0
  68. package/dist/core/folder-scanner.js +84 -0
  69. package/dist/core/folder-scanner.js.map +1 -0
  70. package/dist/core/git.d.ts +137 -0
  71. package/dist/core/git.d.ts.map +1 -0
  72. package/dist/core/git.js +520 -0
  73. package/dist/core/git.js.map +1 -0
  74. package/dist/core/history.d.ts +21 -0
  75. package/dist/core/history.d.ts.map +1 -0
  76. package/dist/core/history.js +107 -0
  77. package/dist/core/history.js.map +1 -0
  78. package/dist/core/improve.d.ts.map +1 -1
  79. package/dist/core/improve.js +9 -0
  80. package/dist/core/improve.js.map +1 -1
  81. package/dist/core/ingest.d.ts +1 -0
  82. package/dist/core/ingest.d.ts.map +1 -1
  83. package/dist/core/ingest.js +78 -5
  84. package/dist/core/ingest.js.map +1 -1
  85. package/dist/core/observer.d.ts +71 -0
  86. package/dist/core/observer.d.ts.map +1 -0
  87. package/dist/core/observer.js +350 -0
  88. package/dist/core/observer.js.map +1 -0
  89. package/dist/core/pipeline-events.d.ts +63 -0
  90. package/dist/core/pipeline-events.d.ts.map +1 -0
  91. package/dist/core/pipeline-events.js +109 -0
  92. package/dist/core/pipeline-events.js.map +1 -0
  93. package/dist/core/query.d.ts.map +1 -1
  94. package/dist/core/query.js +16 -8
  95. package/dist/core/query.js.map +1 -1
  96. package/dist/core/scraper.d.ts +41 -0
  97. package/dist/core/scraper.d.ts.map +1 -0
  98. package/dist/core/scraper.js +277 -0
  99. package/dist/core/scraper.js.map +1 -0
  100. package/dist/index.js +3 -2
  101. package/dist/index.js.map +1 -1
  102. package/dist/mcp-entry.d.ts +10 -0
  103. package/dist/mcp-entry.d.ts.map +1 -0
  104. package/dist/mcp-entry.js +21 -0
  105. package/dist/mcp-entry.js.map +1 -0
  106. package/dist/mcp-server.d.ts +15 -0
  107. package/dist/mcp-server.d.ts.map +1 -0
  108. package/dist/mcp-server.js +390 -0
  109. package/dist/mcp-server.js.map +1 -0
  110. package/dist/processors/audio.d.ts.map +1 -1
  111. package/dist/processors/audio.js +42 -4
  112. package/dist/processors/audio.js.map +1 -1
  113. package/dist/processors/pdf.d.ts.map +1 -1
  114. package/dist/processors/pdf.js +2 -3
  115. package/dist/processors/pdf.js.map +1 -1
  116. package/dist/processors/url.js +4 -1
  117. package/dist/processors/url.js.map +1 -1
  118. package/dist/providers/claude.d.ts +1 -0
  119. package/dist/providers/claude.d.ts.map +1 -1
  120. package/dist/providers/claude.js +5 -3
  121. package/dist/providers/claude.js.map +1 -1
  122. package/dist/providers/index.d.ts +17 -1
  123. package/dist/providers/index.d.ts.map +1 -1
  124. package/dist/providers/index.js +144 -0
  125. package/dist/providers/index.js.map +1 -1
  126. package/dist/providers/openai.d.ts +1 -0
  127. package/dist/providers/openai.d.ts.map +1 -1
  128. package/dist/providers/openai.js +5 -3
  129. package/dist/providers/openai.js.map +1 -1
  130. package/dist/providers/types.d.ts +18 -0
  131. package/dist/providers/types.d.ts.map +1 -1
  132. package/dist/templates/config-yaml.d.ts.map +1 -1
  133. package/dist/templates/config-yaml.js +13 -2
  134. package/dist/templates/config-yaml.js.map +1 -1
  135. package/dist/web/public/index.html +8157 -745
  136. package/dist/web/server.d.ts.map +1 -1
  137. package/dist/web/server.js +2101 -29
  138. package/dist/web/server.js.map +1 -1
  139. package/package.json +7 -4
  140. package/scripts/install.sh +54 -0
  141. package/src/web/public/index.html +8157 -745
  142. package/templates/mcp-config.json +9 -0
  143. package/dist/web/public/public/index.html +0 -946
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Automation 1: Smart Sourcing
3
+ *
4
+ * Full-pipeline scraper that:
5
+ * - Fetches RSS feeds and web pages
6
+ * - Applies topic guardrails (keyword matching)
7
+ * - Labels content as addedBy: 'agent'
8
+ * - Runs ingest pipeline on discovered content
9
+ * - Records every action in the audit trail
10
+ */
11
+ import type { VaultConfig } from './vault.js';
12
+ import type { UserConfig } from './config.js';
13
+ export interface ScraperOptions {
14
+ /** Skip ingest pipeline — only deposit files to raw/ */
15
+ dryRun?: boolean;
16
+ /** Limit items per source (default: 10) */
17
+ maxItems?: number;
18
+ /** Override topic guardrails for this run */
19
+ skipGuardrails?: boolean;
20
+ }
21
+ export interface ScrapeAction {
22
+ timestamp: string;
23
+ sourceUrl: string;
24
+ sourceName: string;
25
+ itemTitle: string;
26
+ itemUrl: string;
27
+ deposited: boolean;
28
+ ingested: boolean;
29
+ skippedReason?: string;
30
+ }
31
+ export interface SmartScrapeResult {
32
+ sourcesProcessed: number;
33
+ filesDeposited: number;
34
+ pagesCreated: number;
35
+ actions: ScrapeAction[];
36
+ }
37
+ export declare function runSmartScraper(config: VaultConfig, userConfig: UserConfig, options?: ScraperOptions, specificSource?: string): Promise<SmartScrapeResult>;
38
+ export declare function runAdHocScrape(config: VaultConfig, userConfig: UserConfig, urls: string[], options?: ScraperOptions & {
39
+ topics?: string[];
40
+ }): Promise<SmartScrapeResult>;
41
+ //# sourceMappingURL=scraper.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/core/scraper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,OAAO,KAAK,EAAE,UAAU,EAAgB,MAAM,aAAa,CAAC;AAG5D,MAAM,WAAW,cAAc;IAC7B,wDAAwD;IACxD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,2CAA2C;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAkGD,wBAAsB,eAAe,CACnC,MAAM,EAAE,WAAW,EACnB,UAAU,EAAE,UAAU,EACtB,OAAO,GAAE,cAAmB,EAC5B,cAAc,CAAC,EAAE,MAAM,GACtB,OAAO,CAAC,iBAAiB,CAAC,CAwC5B;AAED,wBAAsB,cAAc,CAClC,MAAM,EAAE,WAAW,EACnB,UAAU,EAAE,UAAU,EACtB,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,cAAc,GAAG;IAAE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;CAAO,GACnD,OAAO,CAAC,iBAAiB,CAAC,CAe5B"}
@@ -0,0 +1,277 @@
1
+ /**
2
+ * Automation 1: Smart Sourcing
3
+ *
4
+ * Full-pipeline scraper that:
5
+ * - Fetches RSS feeds and web pages
6
+ * - Applies topic guardrails (keyword matching)
7
+ * - Labels content as addedBy: 'agent'
8
+ * - Runs ingest pipeline on discovered content
9
+ * - Records every action in the audit trail
10
+ */
11
+ import { mkdirSync, writeFileSync } from 'node:fs';
12
+ import { join } from 'node:path';
13
+ import { slugify } from './vault.js';
14
+ import { appendAuditEntry } from './audit-trail.js';
15
+ // ─── Topic Guardrails ────────────────────────────────────────────────────────
16
+ /**
17
+ * Returns true if the content matches the source's topic keywords.
18
+ * If no topics configured, all items pass.
19
+ */
20
+ function passesGuardrails(title, content, source, skipGuardrails = false) {
21
+ if (skipGuardrails)
22
+ return true;
23
+ const topics = source.topics ?? [];
24
+ if (topics.length === 0)
25
+ return true;
26
+ const haystack = `${title} ${content}`.toLowerCase();
27
+ return topics.some((kw) => haystack.includes(kw.toLowerCase()));
28
+ }
29
+ function parseRssItems(xml) {
30
+ const items = xml.match(/<item[\s>][\s\S]*?<\/item>/g) ?? [];
31
+ return items.map((item) => ({
32
+ title: stripCdata(item.match(/<title>([\s\S]*?)<\/title>/)?.[1] ?? 'Untitled').trim(),
33
+ link: (item.match(/<link>\s*([\s\S]*?)\s*<\/link>/)?.[1] ?? '').trim(),
34
+ description: stripCdata(item.match(/<description>([\s\S]*?)<\/description>/)?.[1] ?? ''),
35
+ pubDate: item.match(/<pubDate>([\s\S]*?)<\/pubDate>/)?.[1]?.trim() ?? '',
36
+ }));
37
+ }
38
+ function parseAtomEntries(xml) {
39
+ const entries = xml.match(/<entry[\s>][\s\S]*?<\/entry>/g) ?? [];
40
+ return entries.map((entry) => ({
41
+ title: stripCdata(entry.match(/<title[^>]*>([\s\S]*?)<\/title>/)?.[1] ?? 'Untitled').trim(),
42
+ link: entry.match(/<link[^>]*href=["']([^"']+)["'][^>]*\/?>/)?.[1]?.trim()
43
+ ?? (entry.match(/<link[^>]*>([\s\S]*?)<\/link>/)?.[1] ?? '').trim(),
44
+ description: stripCdata(entry.match(/<summary[^>]*>([\s\S]*?)<\/summary>/)?.[1]
45
+ ?? entry.match(/<content[^>]*>([\s\S]*?)<\/content>/)?.[1]
46
+ ?? ''),
47
+ pubDate: entry.match(/<(?:updated|published)>([\s\S]*?)<\/(?:updated|published)>/)?.[1]?.trim() ?? '',
48
+ }));
49
+ }
50
+ function parseFeedItems(xml) {
51
+ const rssItems = parseRssItems(xml);
52
+ if (rssItems.length > 0)
53
+ return rssItems;
54
+ return parseAtomEntries(xml);
55
+ }
56
+ async function fetchRssFeed(source, options) {
57
+ if (!source.url)
58
+ return [];
59
+ const res = await fetch(source.url, { signal: AbortSignal.timeout(20000) });
60
+ if (!res.ok)
61
+ throw new Error(`HTTP ${res.status} from ${source.url}`);
62
+ const xml = await res.text();
63
+ const items = parseFeedItems(xml);
64
+ return items.slice(0, options.maxItems ?? 10);
65
+ }
66
+ // ─── Web Scraper ─────────────────────────────────────────────────────────────
67
+ async function fetchPageContent(url) {
68
+ try {
69
+ const res = await fetch(url, {
70
+ signal: AbortSignal.timeout(15000),
71
+ headers: { 'User-Agent': 'WikiMem/0.6 (knowledge-base-bot)' },
72
+ });
73
+ if (!res.ok)
74
+ return '';
75
+ const html = await res.text();
76
+ // Strip scripts/styles first, then all tags
77
+ const cleaned = html
78
+ .replace(/<script[\s\S]*?<\/script>/gi, '')
79
+ .replace(/<style[\s\S]*?<\/style>/gi, '')
80
+ .replace(/<[^>]+>/g, ' ')
81
+ .replace(/&[a-z]+;/gi, ' ')
82
+ .replace(/\s{2,}/g, ' ')
83
+ .trim()
84
+ .substring(0, 8000);
85
+ return cleaned;
86
+ }
87
+ catch {
88
+ return '';
89
+ }
90
+ }
91
+ // ─── Main Scraper Entry Point ─────────────────────────────────────────────────
92
+ export async function runSmartScraper(config, userConfig, options = {}, specificSource) {
93
+ const sources = userConfig.sources ?? [];
94
+ const toProcess = specificSource
95
+ ? sources.filter((s) => s.name === specificSource)
96
+ : sources;
97
+ const now = new Date().toISOString().split('T')[0] ?? '';
98
+ const dateDir = join(config.rawDir, now);
99
+ mkdirSync(dateDir, { recursive: true });
100
+ const allActions = [];
101
+ let totalDeposited = 0;
102
+ let totalPagesCreated = 0;
103
+ const startMs = Date.now();
104
+ for (const source of toProcess) {
105
+ const sourceActions = await processSource(source, config, userConfig, dateDir, now, options);
106
+ for (const a of sourceActions.actions) {
107
+ allActions.push(a);
108
+ if (a.deposited)
109
+ totalDeposited++;
110
+ totalPagesCreated += sourceActions.pagesCreated;
111
+ }
112
+ }
113
+ // Record in audit trail
114
+ appendAuditEntry(config.root, {
115
+ action: 'scrape',
116
+ actor: 'agent',
117
+ source: toProcess.map((s) => s.url ?? s.name).join(', '),
118
+ summary: `Smart scraper ran against ${toProcess.length} source(s). Deposited ${totalDeposited} files, created ${totalPagesCreated} pages.`,
119
+ pagesAffected: allActions.filter((a) => a.ingested).map((a) => a.itemTitle),
120
+ duration: Date.now() - startMs,
121
+ });
122
+ return {
123
+ sourcesProcessed: toProcess.length,
124
+ filesDeposited: totalDeposited,
125
+ pagesCreated: totalPagesCreated,
126
+ actions: allActions,
127
+ };
128
+ }
129
+ export async function runAdHocScrape(config, userConfig, urls, options = {}) {
130
+ const sources = urls.map((url) => {
131
+ const isRss = /\.(xml|rss)$/i.test(url) || /\/feed\b|\/rss\b|\/atom\b/i.test(url);
132
+ let hostname = 'unknown';
133
+ try {
134
+ hostname = new URL(url).hostname;
135
+ }
136
+ catch { /* invalid URL */ }
137
+ const source = {
138
+ name: hostname,
139
+ type: isRss ? 'rss' : 'url',
140
+ url,
141
+ };
142
+ if (options.topics)
143
+ source.topics = options.topics;
144
+ return source;
145
+ });
146
+ const tempUserConfig = { ...userConfig, sources };
147
+ return runSmartScraper(config, tempUserConfig, options);
148
+ }
149
+ async function processSource(source, config, userConfig, dateDir, now, options) {
150
+ const actions = [];
151
+ let pagesCreated = 0;
152
+ try {
153
+ if (source.type === 'rss') {
154
+ const items = await fetchRssFeed(source, options);
155
+ for (const item of items) {
156
+ const action = {
157
+ timestamp: new Date().toISOString(),
158
+ sourceUrl: source.url ?? '',
159
+ sourceName: source.name,
160
+ itemTitle: item.title,
161
+ itemUrl: item.link,
162
+ deposited: false,
163
+ ingested: false,
164
+ };
165
+ // Fetch full page content for guardrail check
166
+ const fullContent = item.link ? await fetchPageContent(item.link) : item.description;
167
+ const combinedText = `${item.title} ${item.description} ${fullContent}`;
168
+ if (!passesGuardrails(item.title, combinedText, source, options.skipGuardrails)) {
169
+ action.skippedReason = 'Topic guardrail: no keyword match';
170
+ actions.push(action);
171
+ continue;
172
+ }
173
+ // Build markdown content
174
+ const markdown = buildArticleMarkdown(item.title, item.link, source.name, item.description, fullContent, item.pubDate);
175
+ const fileName = `${slugify(item.title.substring(0, 60))}.md`;
176
+ const filePath = join(dateDir, fileName);
177
+ writeFileSync(filePath, markdown, 'utf-8');
178
+ action.deposited = true;
179
+ // Run ingest pipeline unless dry-run
180
+ if (!options.dryRun) {
181
+ try {
182
+ const result = await ingestFile(filePath, config, userConfig);
183
+ pagesCreated += result.pagesCreated;
184
+ action.ingested = true;
185
+ }
186
+ catch (err) {
187
+ action.skippedReason = `Ingest failed: ${err instanceof Error ? err.message : String(err)}`;
188
+ }
189
+ }
190
+ actions.push(action);
191
+ }
192
+ }
193
+ else if (source.type === 'url') {
194
+ const action = {
195
+ timestamp: new Date().toISOString(),
196
+ sourceUrl: source.url ?? '',
197
+ sourceName: source.name,
198
+ itemTitle: source.name,
199
+ itemUrl: source.url ?? '',
200
+ deposited: false,
201
+ ingested: false,
202
+ };
203
+ const content = await fetchPageContent(source.url ?? '');
204
+ if (!passesGuardrails(source.name, content, source, options.skipGuardrails)) {
205
+ action.skippedReason = 'Topic guardrail: no keyword match';
206
+ actions.push(action);
207
+ }
208
+ else {
209
+ const markdown = `# ${source.name}\n\nSource: ${source.url}\nScraped: ${new Date().toISOString()}\n\n${content}`;
210
+ const fileName = `${slugify(source.name)}.md`;
211
+ const filePath = join(dateDir, fileName);
212
+ writeFileSync(filePath, markdown, 'utf-8');
213
+ action.deposited = true;
214
+ if (!options.dryRun) {
215
+ try {
216
+ const result = await ingestFile(filePath, config, userConfig);
217
+ pagesCreated += result.pagesCreated;
218
+ action.ingested = true;
219
+ }
220
+ catch (err) {
221
+ action.skippedReason = `Ingest failed: ${err instanceof Error ? err.message : String(err)}`;
222
+ }
223
+ }
224
+ actions.push(action);
225
+ }
226
+ }
227
+ }
228
+ catch (err) {
229
+ // Source-level failure — log but don't crash
230
+ const errMsg = err instanceof Error ? err.message : String(err);
231
+ actions.push({
232
+ timestamp: new Date().toISOString(),
233
+ sourceUrl: source.url ?? '',
234
+ sourceName: source.name,
235
+ itemTitle: 'Source fetch failed',
236
+ itemUrl: source.url ?? '',
237
+ deposited: false,
238
+ ingested: false,
239
+ skippedReason: errMsg,
240
+ });
241
+ }
242
+ return { actions, pagesCreated };
243
+ }
244
+ async function ingestFile(filePath, config, userConfig) {
245
+ const { ingestSource } = await import('./ingest.js');
246
+ const { createProviderFromUserConfig } = await import('../providers/index.js');
247
+ const provider = createProviderFromUserConfig(userConfig);
248
+ const result = await ingestSource(filePath, config, provider, {
249
+ verbose: false,
250
+ addedBy: 'agent',
251
+ });
252
+ return { pagesCreated: result.pagesUpdated };
253
+ }
254
+ function buildArticleMarkdown(title, link, sourceName, description, fullContent, pubDate) {
255
+ const parts = [`# ${title}`, ''];
256
+ if (link)
257
+ parts.push(`Source: ${link}`);
258
+ if (sourceName)
259
+ parts.push(`Feed: ${sourceName}`);
260
+ if (pubDate)
261
+ parts.push(`Published: ${pubDate}`);
262
+ parts.push(`Scraped: ${new Date().toISOString()}`, '');
263
+ if (description) {
264
+ parts.push('## Summary', '', stripHtml(description).substring(0, 500), '');
265
+ }
266
+ if (fullContent) {
267
+ parts.push('## Content', '', fullContent.substring(0, 6000));
268
+ }
269
+ return parts.join('\n');
270
+ }
271
+ function stripCdata(text) {
272
+ return text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1').trim();
273
+ }
274
+ function stripHtml(html) {
275
+ return html.replace(/<[^>]*>/g, '').replace(/&[a-z]+;/gi, ' ').trim();
276
+ }
277
+ //# sourceMappingURL=scraper.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/core/scraper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AA6BpD,gFAAgF;AAEhF;;;GAGG;AACH,SAAS,gBAAgB,CACvB,KAAa,EACb,OAAe,EACf,MAAoB,EACpB,cAAc,GAAG,KAAK;IAEtB,IAAI,cAAc;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,MAAM,GAAc,MAA+C,CAAC,MAAM,IAAI,EAAE,CAAC;IACvF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAErC,MAAM,QAAQ,GAAG,GAAG,KAAK,IAAI,OAAO,EAAE,CAAC,WAAW,EAAE,CAAC;IACrD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AAClE,CAAC;AAWD,SAAS,aAAa,CAAC,GAAW;IAChC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,6BAA6B,CAAC,IAAI,EAAE,CAAC;IAC7D,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE;QACrF,IAAI,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QACtE,WAAW,EAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,wCAAwC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACxF,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE;KACzE,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW;IACnC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,+BAA+B,CAAC,IAAI,EAAE,CAAC;IACjE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC7B,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,iCAAiC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE;QAC3F,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,0CAA0C,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE;eACrE,CAAC,KAAK,CAAC,KAAK,CAAC,+BAA+B,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QACrE,WAAW,EAAE,UAAU,CACrB,KAAK,CAAC,KAAK,CAAC,qCAAqC,CAAC,EAAE,CAAC,CAAC,CAAC;eAClD,KAAK,CAAC,KAAK,CAAC,qCAAqC,CAAC,EAAE,CAAC,CAAC,CAAC;eACvD,EAAE,CACR;QACD,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,4DAA4D,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE;KACtG,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,cAAc,CAAC,GAAW;IACjC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,QAAQ,CAAC;IACzC,OAAO,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAoB,EAAE,OAAuB;IACvE,IAAI,CAAC,MAAM,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5E,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,SAAS,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IACtE,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;AAChD,CAAC;AAED,gFAAgF;AAEhF,KAAK,UAAU,gBAAgB,CAAC,GAAW;IACzC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;YAClC,OAAO,EAAE,EAAE,YAAY,EAAE,kCAAkC,EAAE;SAC9D,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,4CAA4C;QAC5C,MAAM,OAAO,GAAG,IAAI;aACjB,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC;aAC1C,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC;aACxC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;aACxB,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC;aAC1B,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,IAAI,EAAE;aACN,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtB,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,iFAAiF;AAEjF,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAmB,EACnB,UAAsB,EACtB,UAA0B,EAAE,EAC5B,cAAuB;IAEvB,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,cAAc;QAC9B,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,cAAc,CAAC;QAClD,CAAC,CAAC,OAAO,CAAC;IAEZ,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACzC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,MAAM,UAAU,GAAmB,EAAE,CAAC;IACtC,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QAC7F,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,CAAC;YACtC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnB,IAAI,CAAC,CAAC,SAAS;gBAAE,cAAc,EAAE,CAAC;YAClC,iBAAiB,IAAI,aAAa,CAAC,YAAY,CAAC;QAClD,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE;QAC5B,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,OAAO;QACd,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QACxD,OAAO,EAAE,6BAA6B,SAAS,CAAC,MAAM,yBAAyB,cAAc,mBAAmB,iBAAiB,SAAS;QAC1I,aAAa,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3E,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;KAC/B,CAAC,CAAC;IAEH,OAAO;QACL,gBAAgB,EAAE,SAAS,CAAC,MAAM;QAClC,cAAc,EAAE,cAAc;QAC9B,YAAY,EAAE,iBAAiB;QAC/B,OAAO,EAAE,UAAU;KACpB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAmB,EACnB,UAAsB,EACtB,IAAc,EACd,UAAkD,EAAE;IAEpD,MAAM,OAAO,GAAmB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC/C,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,4BAA4B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClF,IAAI,QAAQ,GAAG,SAAS,CAAC;QACzB,IAAI,CAAC;YAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;QACrE,MAAM,MAAM,GAAyC;YACnD,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK;YAC3B,GAAG;SACJ,CAAC;QACF,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QACnD,OAAO,MAAsB,CAAC;IAChC,CAAC,CAAC,CAAC;IACH,MAAM,cAAc,GAAe,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE,CAAC;IAC9D,OAAO,eAAe,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;AAC1D,CAAC;AAOD,KAAK,UAAU,aAAa,CAC1B,MAAoB,EACpB,MAAmB,EACnB,UAAsB,EACtB,OAAe,EACf,GAAW,EACX,OAAuB;IAEvB,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,IAAI,CAAC;QACH,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAElD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAiB;oBAC3B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,SAAS,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;oBAC3B,UAAU,EAAE,MAAM,CAAC,IAAI;oBACvB,SAAS,EAAE,IAAI,CAAC,KAAK;oBACrB,OAAO,EAAE,IAAI,CAAC,IAAI;oBAClB,SAAS,EAAE,KAAK;oBAChB,QAAQ,EAAE,KAAK;iBAChB,CAAC;gBAEF,8CAA8C;gBAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;gBACrF,MAAM,YAAY,GAAG,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,IAAI,WAAW,EAAE,CAAC;gBAExE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;oBAChF,MAAM,CAAC,aAAa,GAAG,mCAAmC,CAAC;oBAC3D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,yBAAyB;gBACzB,MAAM,QAAQ,GAAG,oBAAoB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvH,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;gBAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACzC,aAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC3C,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;gBAExB,qCAAqC;gBACrC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBACpB,IAAI,CAAC;wBACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;wBAC9D,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;wBACpC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;oBACzB,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,MAAM,CAAC,aAAa,GAAG,kBAAkB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC9F,CAAC;gBACH,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;aAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACjC,MAAM,MAAM,GAAiB;gBAC3B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,SAAS,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;gBAC3B,UAAU,EAAE,MAAM,CAAC,IAAI;gBACvB,SAAS,EAAE,MAAM,CAAC,IAAI;gBACtB,OAAO,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;gBACzB,SAAS,EAAE,KAAK;gBAChB,QAAQ,EAAE,KAAK;aAChB,CAAC;YAEF,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC5E,MAAM,CAAC,aAAa,GAAG,mCAAmC,CAAC;gBAC3D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,MAAM,QAAQ,GAAG,KAAK,MAAM,CAAC,IAAI,eAAe,MAAM,CAAC,GAAG,cAAc,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,OAAO,OAAO,EAAE,CAAC;gBACjH,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;gBAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACzC,aAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAC3C,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;gBAExB,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBACpB,IAAI,CAAC;wBACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;wBAC9D,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;wBACpC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;oBACzB,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,MAAM,CAAC,aAAa,GAAG,kBAAkB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC9F,CAAC;gBACH,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,6CAA6C;QAC7C,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC;YACX,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;YAC3B,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,SAAS,EAAE,qBAAqB;YAChC,OAAO,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;YACzB,SAAS,EAAE,KAAK;YAChB,QAAQ,EAAE,KAAK;YACf,aAAa,EAAE,MAAM;SACtB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;AACnC,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,QAAgB,EAChB,MAAmB,EACnB,UAAsB;IAEtB,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;IACrD,MAAM,EAAE,4BAA4B,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,4BAA4B,CAAC,UAAU,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE;QAC5D,OAAO,EAAE,KAAK;QACd,OAAO,EAAE,OAAO;KACjB,CAAC,CAAC;IACH,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;AAC/C,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAAa,EACb,IAAY,EACZ,UAAkB,EAClB,WAAmB,EACnB,WAAmB,EACnB,OAAe;IAEf,MAAM,KAAK,GAAa,CAAC,KAAK,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;IAC3C,IAAI,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IACxC,IAAI,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,UAAU,EAAE,CAAC,CAAC;IAClD,IAAI,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,cAAc,OAAO,EAAE,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IACvD,IAAI,WAAW,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,WAAW,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,6BAA6B,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACxE,CAAC"}
package/dist/index.js CHANGED
@@ -29,10 +29,11 @@ if (process.argv.length <= 2) {
29
29
  console.log(chalk.bold(' Commands'));
30
30
  console.log(chalk.dim(' ─'.padEnd(40, '─')));
31
31
  console.log(` ${chalk.cyan('wikimem ingest <file|url>')} Add knowledge`);
32
- console.log(` ${chalk.cyan('wikimem query "question"')} Ask the wiki`);
32
+ console.log(` ${chalk.cyan('wikimem search "term"')} BM25 keyword search`);
33
+ console.log(` ${chalk.cyan('wikimem ask "question"')} LLM-powered Q&A`);
33
34
  console.log(` ${chalk.cyan('wikimem status')} Full statistics`);
34
35
  console.log(` ${chalk.cyan('wikimem lint')} Check wiki health`);
35
- console.log(` ${chalk.cyan('wikimem improve')} Self-improve cycle`);
36
+ console.log(` ${chalk.cyan('wikimem export -f json')} Export graph data`);
36
37
  console.log(` ${chalk.cyan('wikimem serve')} Web UI at :3141`);
37
38
  console.log(` ${chalk.cyan('wikimem --help')} All commands`);
38
39
  console.log();
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAQ,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhE,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;AAEhC,8EAA8E;AAC9E,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;IAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAEnC,IAAI,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,iDAAiD;QACjD,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QAEpC,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACnF,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,iBAAiB,CAAC,CAAC;QAC3E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,iBAAiB,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC;QAC7E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,kCAAkC,CAAC,CAAC;QAC/E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,gCAAgC,CAAC,CAAC;QAChF,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC;QAC7E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;SAAM,CAAC;QACN,6BAA6B;QAC7B,OAAO,CAAC,UAAU,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,KAAK,EAAE,CAAC;AAClB,CAAC;AAED,+DAA+D;AAC/D,eAAe,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAQ,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhE,MAAM,OAAO,GAAG,aAAa,EAAE,CAAC;AAEhC,8EAA8E;AAC9E,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;IAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAEnC,IAAI,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,iDAAiD;QACjD,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QAEpC,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACnF,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,iBAAiB,CAAC,CAAC;QAC3E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,2BAA2B,CAAC,CAAC;QACjF,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,sBAAsB,CAAC,CAAC;QAC7E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC;QAC7E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,kCAAkC,CAAC,CAAC;QAC/E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,wBAAwB,CAAC,CAAC;QAC/E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC;QAC7E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;SAAM,CAAC;QACN,6BAA6B;QAC7B,OAAO,CAAC,UAAU,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,KAAK,EAAE,CAAC;AAClB,CAAC;AAED,+DAA+D;AAC/D,eAAe,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC"}
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Lightweight MCP server entry point.
4
+ * Bypasses the heavy CLI import chain (pdf-parse, mammoth, xlsx, etc.)
5
+ * so the server starts in <2s instead of >90s.
6
+ *
7
+ * Usage: node dist/mcp-entry.js [--vault <path>]
8
+ */
9
+ export {};
10
+ //# sourceMappingURL=mcp-entry.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mcp-entry.d.ts","sourceRoot":"","sources":["../src/mcp-entry.ts"],"names":[],"mappings":";AAEA;;;;;;GAMG"}
@@ -0,0 +1,21 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Lightweight MCP server entry point.
4
+ * Bypasses the heavy CLI import chain (pdf-parse, mammoth, xlsx, etc.)
5
+ * so the server starts in <2s instead of >90s.
6
+ *
7
+ * Usage: node dist/mcp-entry.js [--vault <path>]
8
+ */
9
+ import { resolve } from 'node:path';
10
+ import { startMcpServer } from './mcp-server.js';
11
+ const args = process.argv.slice(2);
12
+ let vault = process.cwd();
13
+ for (let i = 0; i < args.length; i++) {
14
+ const next = args[i + 1];
15
+ if (args[i] === '--vault' && next) {
16
+ vault = resolve(next);
17
+ break;
18
+ }
19
+ }
20
+ startMcpServer(vault);
21
+ //# sourceMappingURL=mcp-entry.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mcp-entry.js","sourceRoot":"","sources":["../src/mcp-entry.ts"],"names":[],"mappings":";AAEA;;;;;;GAMG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACnC,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;AAE1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,EAAE,CAAC;QAClC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QACtB,MAAM;IACR,CAAC;AACH,CAAC;AAED,cAAc,CAAC,KAAK,CAAC,CAAC"}
@@ -0,0 +1,15 @@
1
+ /**
2
+ * WikiMem MCP Server — JSON-RPC 2.0 over stdin/stdout.
3
+ *
4
+ * Implements the Model Context Protocol without the SDK dependency.
5
+ * Reads newline-delimited JSON from stdin, writes responses to stdout.
6
+ *
7
+ * Tools exposed:
8
+ * wikimem_search — BM25 keyword search over wiki pages
9
+ * wikimem_read — Read a specific wiki page by title or filename
10
+ * wikimem_list — List all wiki pages with metadata
11
+ * wikimem_status — Vault statistics (page count, words, orphans, …)
12
+ * wikimem_ingest — Ingest a file or URL into the vault (async)
13
+ */
14
+ export declare function startMcpServer(vault: string): Promise<void>;
15
+ //# sourceMappingURL=mcp-server.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"mcp-server.d.ts","sourceRoot":"","sources":["../src/mcp-server.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AA4YH,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA0CjE"}