wikimem 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/CHANGELOG.md +37 -0
  2. package/LICENSE +21 -0
  3. package/README.md +398 -0
  4. package/dist/cli/commands/duplicates.d.ts +3 -0
  5. package/dist/cli/commands/duplicates.d.ts.map +1 -0
  6. package/dist/cli/commands/duplicates.js +38 -0
  7. package/dist/cli/commands/duplicates.js.map +1 -0
  8. package/dist/cli/commands/improve.d.ts +3 -0
  9. package/dist/cli/commands/improve.d.ts.map +1 -0
  10. package/dist/cli/commands/improve.js +69 -0
  11. package/dist/cli/commands/improve.js.map +1 -0
  12. package/dist/cli/commands/ingest.d.ts +3 -0
  13. package/dist/cli/commands/ingest.d.ts.map +1 -0
  14. package/dist/cli/commands/ingest.js +181 -0
  15. package/dist/cli/commands/ingest.js.map +1 -0
  16. package/dist/cli/commands/init.d.ts +3 -0
  17. package/dist/cli/commands/init.d.ts.map +1 -0
  18. package/dist/cli/commands/init.js +91 -0
  19. package/dist/cli/commands/init.js.map +1 -0
  20. package/dist/cli/commands/lint.d.ts +3 -0
  21. package/dist/cli/commands/lint.d.ts.map +1 -0
  22. package/dist/cli/commands/lint.js +49 -0
  23. package/dist/cli/commands/lint.js.map +1 -0
  24. package/dist/cli/commands/query.d.ts +3 -0
  25. package/dist/cli/commands/query.d.ts.map +1 -0
  26. package/dist/cli/commands/query.js +51 -0
  27. package/dist/cli/commands/query.js.map +1 -0
  28. package/dist/cli/commands/scrape.d.ts +3 -0
  29. package/dist/cli/commands/scrape.d.ts.map +1 -0
  30. package/dist/cli/commands/scrape.js +47 -0
  31. package/dist/cli/commands/scrape.js.map +1 -0
  32. package/dist/cli/commands/serve.d.ts +3 -0
  33. package/dist/cli/commands/serve.d.ts.map +1 -0
  34. package/dist/cli/commands/serve.js +24 -0
  35. package/dist/cli/commands/serve.js.map +1 -0
  36. package/dist/cli/commands/status.d.ts +3 -0
  37. package/dist/cli/commands/status.d.ts.map +1 -0
  38. package/dist/cli/commands/status.js +30 -0
  39. package/dist/cli/commands/status.js.map +1 -0
  40. package/dist/cli/commands/watch.d.ts +3 -0
  41. package/dist/cli/commands/watch.d.ts.map +1 -0
  42. package/dist/cli/commands/watch.js +29 -0
  43. package/dist/cli/commands/watch.js.map +1 -0
  44. package/dist/cli/index.d.ts +3 -0
  45. package/dist/cli/index.d.ts.map +1 -0
  46. package/dist/cli/index.js +30 -0
  47. package/dist/cli/index.js.map +1 -0
  48. package/dist/core/config.d.ts +47 -0
  49. package/dist/core/config.d.ts.map +1 -0
  50. package/dist/core/config.js +11 -0
  51. package/dist/core/config.js.map +1 -0
  52. package/dist/core/improve.d.ts +19 -0
  53. package/dist/core/improve.d.ts.map +1 -0
  54. package/dist/core/improve.js +175 -0
  55. package/dist/core/improve.js.map +1 -0
  56. package/dist/core/index-manager.d.ts +9 -0
  57. package/dist/core/index-manager.d.ts.map +1 -0
  58. package/dist/core/index-manager.js +30 -0
  59. package/dist/core/index-manager.js.map +1 -0
  60. package/dist/core/ingest.d.ts +46 -0
  61. package/dist/core/ingest.d.ts.map +1 -0
  62. package/dist/core/ingest.js +366 -0
  63. package/dist/core/ingest.js.map +1 -0
  64. package/dist/core/lint.d.ts +19 -0
  65. package/dist/core/lint.d.ts.map +1 -0
  66. package/dist/core/lint.js +90 -0
  67. package/dist/core/lint.js.map +1 -0
  68. package/dist/core/log-manager.d.ts +2 -0
  69. package/dist/core/log-manager.d.ts.map +1 -0
  70. package/dist/core/log-manager.js +14 -0
  71. package/dist/core/log-manager.js.map +1 -0
  72. package/dist/core/obsidian.d.ts +89 -0
  73. package/dist/core/obsidian.d.ts.map +1 -0
  74. package/dist/core/obsidian.js +123 -0
  75. package/dist/core/obsidian.js.map +1 -0
  76. package/dist/core/query.d.ts +16 -0
  77. package/dist/core/query.d.ts.map +1 -0
  78. package/dist/core/query.js +77 -0
  79. package/dist/core/query.js.map +1 -0
  80. package/dist/core/scrape.d.ts +13 -0
  81. package/dist/core/scrape.d.ts.map +1 -0
  82. package/dist/core/scrape.js +103 -0
  83. package/dist/core/scrape.js.map +1 -0
  84. package/dist/core/vault.d.ts +35 -0
  85. package/dist/core/vault.d.ts.map +1 -0
  86. package/dist/core/vault.js +119 -0
  87. package/dist/core/vault.js.map +1 -0
  88. package/dist/core/watcher.d.ts +4 -0
  89. package/dist/core/watcher.d.ts.map +1 -0
  90. package/dist/core/watcher.js +34 -0
  91. package/dist/core/watcher.js.map +1 -0
  92. package/dist/index.d.ts +3 -0
  93. package/dist/index.d.ts.map +1 -0
  94. package/dist/index.js +5 -0
  95. package/dist/index.js.map +1 -0
  96. package/dist/processors/audio.d.ts +10 -0
  97. package/dist/processors/audio.d.ts.map +1 -0
  98. package/dist/processors/audio.js +139 -0
  99. package/dist/processors/audio.js.map +1 -0
  100. package/dist/processors/docx.d.ts +12 -0
  101. package/dist/processors/docx.d.ts.map +1 -0
  102. package/dist/processors/docx.js +98 -0
  103. package/dist/processors/docx.js.map +1 -0
  104. package/dist/processors/image.d.ts +9 -0
  105. package/dist/processors/image.d.ts.map +1 -0
  106. package/dist/processors/image.js +94 -0
  107. package/dist/processors/image.js.map +1 -0
  108. package/dist/processors/pdf.d.ts +10 -0
  109. package/dist/processors/pdf.d.ts.map +1 -0
  110. package/dist/processors/pdf.js +92 -0
  111. package/dist/processors/pdf.js.map +1 -0
  112. package/dist/processors/pptx.d.ts +13 -0
  113. package/dist/processors/pptx.d.ts.map +1 -0
  114. package/dist/processors/pptx.js +165 -0
  115. package/dist/processors/pptx.js.map +1 -0
  116. package/dist/processors/text.d.ts +7 -0
  117. package/dist/processors/text.d.ts.map +1 -0
  118. package/dist/processors/text.js +9 -0
  119. package/dist/processors/text.js.map +1 -0
  120. package/dist/processors/url.d.ts +7 -0
  121. package/dist/processors/url.d.ts.map +1 -0
  122. package/dist/processors/url.js +61 -0
  123. package/dist/processors/url.js.map +1 -0
  124. package/dist/processors/video.d.ts +10 -0
  125. package/dist/processors/video.d.ts.map +1 -0
  126. package/dist/processors/video.js +115 -0
  127. package/dist/processors/video.js.map +1 -0
  128. package/dist/processors/xlsx.d.ts +13 -0
  129. package/dist/processors/xlsx.d.ts.map +1 -0
  130. package/dist/processors/xlsx.js +138 -0
  131. package/dist/processors/xlsx.js.map +1 -0
  132. package/dist/providers/claude.d.ts +10 -0
  133. package/dist/providers/claude.d.ts.map +1 -0
  134. package/dist/providers/claude.js +44 -0
  135. package/dist/providers/claude.js.map +1 -0
  136. package/dist/providers/embeddings.d.ts +62 -0
  137. package/dist/providers/embeddings.d.ts.map +1 -0
  138. package/dist/providers/embeddings.js +206 -0
  139. package/dist/providers/embeddings.js.map +1 -0
  140. package/dist/providers/index.d.ts +7 -0
  141. package/dist/providers/index.d.ts.map +1 -0
  142. package/dist/providers/index.js +19 -0
  143. package/dist/providers/index.js.map +1 -0
  144. package/dist/providers/ollama.d.ts +10 -0
  145. package/dist/providers/ollama.d.ts.map +1 -0
  146. package/dist/providers/ollama.js +48 -0
  147. package/dist/providers/ollama.js.map +1 -0
  148. package/dist/providers/openai.d.ts +10 -0
  149. package/dist/providers/openai.d.ts.map +1 -0
  150. package/dist/providers/openai.js +38 -0
  151. package/dist/providers/openai.js.map +1 -0
  152. package/dist/providers/types.d.ts +33 -0
  153. package/dist/providers/types.d.ts.map +1 -0
  154. package/dist/providers/types.js +2 -0
  155. package/dist/providers/types.js.map +1 -0
  156. package/dist/search/bm25.d.ts +18 -0
  157. package/dist/search/bm25.d.ts.map +1 -0
  158. package/dist/search/bm25.js +52 -0
  159. package/dist/search/bm25.js.map +1 -0
  160. package/dist/search/index.d.ts +12 -0
  161. package/dist/search/index.d.ts.map +1 -0
  162. package/dist/search/index.js +64 -0
  163. package/dist/search/index.js.map +1 -0
  164. package/dist/search/semantic.d.ts +30 -0
  165. package/dist/search/semantic.d.ts.map +1 -0
  166. package/dist/search/semantic.js +162 -0
  167. package/dist/search/semantic.js.map +1 -0
  168. package/dist/templates/agents-md.d.ts +2 -0
  169. package/dist/templates/agents-md.d.ts.map +1 -0
  170. package/dist/templates/agents-md.js +85 -0
  171. package/dist/templates/agents-md.js.map +1 -0
  172. package/dist/templates/config-yaml.d.ts +2 -0
  173. package/dist/templates/config-yaml.d.ts.map +1 -0
  174. package/dist/templates/config-yaml.js +81 -0
  175. package/dist/templates/config-yaml.js.map +1 -0
  176. package/dist/web/server.d.ts +2 -0
  177. package/dist/web/server.d.ts.map +1 -0
  178. package/dist/web/server.js +170 -0
  179. package/dist/web/server.js.map +1 -0
  180. package/package.json +68 -0
@@ -0,0 +1,64 @@
1
+ import { readFileSync } from 'node:fs';
2
+ import { basename, extname } from 'node:path';
3
+ import { bm25Search } from './bm25.js';
4
+ import { semanticSearch } from './semantic.js';
5
+ /**
6
+ * Search wiki pages. Supports BM25 (keyword), semantic (embedding), and hybrid modes.
7
+ */
8
+ export async function searchPages(query, pagePaths, options) {
9
+ const mode = options?.mode ?? 'bm25';
10
+ const limit = options?.limit ?? 20;
11
+ // BM25 keyword search (always available)
12
+ const bm25Results = runBm25Search(query, pagePaths);
13
+ if (mode === 'bm25' || !options?.embeddingProvider || !options?.wikiDir) {
14
+ return bm25Results.slice(0, limit);
15
+ }
16
+ // Semantic search
17
+ if (mode === 'semantic') {
18
+ const semResults = await semanticSearch(query, options.wikiDir, options.embeddingProvider, limit);
19
+ return semResults.map((r) => r.path);
20
+ }
21
+ // Hybrid: merge BM25 + semantic with reciprocal rank fusion
22
+ const semResults = await semanticSearch(query, options.wikiDir, options.embeddingProvider, limit);
23
+ const merged = reciprocalRankFusion(bm25Results, semResults.map((r) => r.path));
24
+ return merged.slice(0, limit);
25
+ }
26
+ function runBm25Search(query, pagePaths) {
27
+ const documents = pagePaths.map((path) => {
28
+ try {
29
+ const content = readFileSync(path, 'utf-8');
30
+ const title = basename(path, extname(path));
31
+ return { path, content, title };
32
+ }
33
+ catch {
34
+ return { path, content: '', title: basename(path) };
35
+ }
36
+ });
37
+ const results = bm25Search(query, documents);
38
+ return results.map((r) => r.path);
39
+ }
40
+ /**
41
+ * Reciprocal Rank Fusion: merge two ranked lists into one.
42
+ * RRF score = sum(1 / (k + rank)) for each list, with k=60.
43
+ */
44
+ function reciprocalRankFusion(listA, listB, k = 60) {
45
+ const scores = new Map();
46
+ for (let i = 0; i < listA.length; i++) {
47
+ const item = listA[i];
48
+ if (!item)
49
+ continue;
50
+ const prev = scores.get(item) ?? 0;
51
+ scores.set(item, prev + 1 / (k + i + 1));
52
+ }
53
+ for (let i = 0; i < listB.length; i++) {
54
+ const item = listB[i];
55
+ if (!item)
56
+ continue;
57
+ const prev = scores.get(item) ?? 0;
58
+ scores.set(item, prev + 1 / (k + i + 1));
59
+ }
60
+ return [...scores.entries()]
61
+ .sort((a, b) => b[1] - a[1])
62
+ .map(([path]) => path);
63
+ }
64
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/search/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAU/C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAa,EACb,SAAmB,EACnB,OAAuB;IAEvB,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,MAAM,CAAC;IACrC,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;IAEnC,yCAAyC;IACzC,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAEpD,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,OAAO,EAAE,iBAAiB,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;QACxE,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,kBAAkB;IAClB,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,MAAM,cAAc,CACrC,KAAK,EACL,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,iBAAiB,EACzB,KAAK,CACN,CAAC;QACF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,4DAA4D;IAC5D,MAAM,UAAU,GAAG,MAAM,cAAc,CACrC,KAAK,EACL,OAAO,CAAC,OAAO,EACf,OAAO,CAAC,iBAAiB,EACzB,KAAK,CACN,CAAC;IAEF,MAAM,MAAM,GAAG,oBAAoB,CACjC,WAAW,EACX,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAC9B,CAAC;IAEF,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,aAAa,CAAC,KAAa,EAAE,SAAmB;IACvD,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACtD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC7C,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACpC,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,KAAe,EAAE,KAAe,EAAE,CAAC,GAAG,EAAE;IACpE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACzB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC"}
@@ -0,0 +1,30 @@
1
+ /**
2
+ * Semantic search using embeddings.
3
+ * Loads .embedding.json sidecar files and ranks by cosine similarity.
4
+ */
5
+ import type { EmbeddingProvider } from '../providers/embeddings.js';
6
+ export interface SemanticSearchResult {
7
+ path: string;
8
+ title: string;
9
+ score: number;
10
+ }
11
+ /**
12
+ * Search wiki pages by semantic similarity using embeddings.
13
+ */
14
+ export declare function semanticSearch(query: string, wikiDir: string, embeddingProvider: EmbeddingProvider, limit?: number): Promise<SemanticSearchResult[]>;
15
+ /**
16
+ * Generate and store an embedding for a wiki page.
17
+ */
18
+ export declare function embedPage(pagePath: string, content: string, embeddingProvider: EmbeddingProvider): Promise<void>;
19
+ /**
20
+ * Batch-embed all wiki pages that don't have embedding sidecar files yet.
21
+ */
22
+ export declare function embedAllPages(wikiDir: string, embeddingProvider: EmbeddingProvider, options?: {
23
+ force?: boolean;
24
+ batchSize?: number;
25
+ }): Promise<{
26
+ embedded: number;
27
+ skipped: number;
28
+ errors: number;
29
+ }>;
30
+ //# sourceMappingURL=semantic.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"semantic.d.ts","sourceRoot":"","sources":["../../src/search/semantic.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,EAAE,iBAAiB,EAAmB,MAAM,4BAA4B,CAAC;AAErF,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,iBAAiB,EAAE,iBAAiB,EACpC,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAqCjC;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,iBAAiB,EAAE,iBAAiB,GACnC,OAAO,CAAC,IAAI,CAAC,CAaf;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,iBAAiB,EAAE,iBAAiB,EACpC,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAChD,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAwDhE"}
@@ -0,0 +1,162 @@
1
+ /**
2
+ * Semantic search using embeddings.
3
+ * Loads .embedding.json sidecar files and ranks by cosine similarity.
4
+ */
5
+ import { readFileSync, existsSync, writeFileSync, readdirSync, statSync } from 'node:fs';
6
+ import { join, basename, extname } from 'node:path';
7
+ import { cosineSimilarity } from '../providers/embeddings.js';
8
+ /**
9
+ * Search wiki pages by semantic similarity using embeddings.
10
+ */
11
+ export async function semanticSearch(query, wikiDir, embeddingProvider, limit) {
12
+ const maxResults = limit ?? 10;
13
+ // Embed the query (use RETRIEVAL_QUERY task type for queries)
14
+ const queryEmbedding = await embeddingProvider.embed(query);
15
+ // Find all embedding sidecar files
16
+ const embeddingFiles = findEmbeddingFiles(wikiDir);
17
+ if (embeddingFiles.length === 0) {
18
+ return [];
19
+ }
20
+ // Score each page against the query
21
+ const results = [];
22
+ for (const embFile of embeddingFiles) {
23
+ try {
24
+ const raw = readFileSync(embFile, 'utf-8');
25
+ const embedding = JSON.parse(raw);
26
+ // Corresponding wiki page path
27
+ const pagePath = embFile.replace(/\.embedding\.json$/, '');
28
+ if (!existsSync(pagePath))
29
+ continue;
30
+ const score = cosineSimilarity(queryEmbedding.values, embedding.vector);
31
+ const title = basename(pagePath, extname(pagePath));
32
+ results.push({ path: pagePath, title, score });
33
+ }
34
+ catch {
35
+ // Skip malformed embedding files
36
+ }
37
+ }
38
+ return results
39
+ .sort((a, b) => b.score - a.score)
40
+ .slice(0, maxResults);
41
+ }
42
+ /**
43
+ * Generate and store an embedding for a wiki page.
44
+ */
45
+ export async function embedPage(pagePath, content, embeddingProvider) {
46
+ const embedding = await embeddingProvider.embed(content);
47
+ const result = {
48
+ vector: embedding.values,
49
+ model: embedding.model,
50
+ dimensions: embedding.dimensions,
51
+ text: content.substring(0, 500), // Store a snippet for debugging
52
+ timestamp: new Date().toISOString(),
53
+ };
54
+ const sidecarPath = pagePath + '.embedding.json';
55
+ writeFileSync(sidecarPath, JSON.stringify(result), 'utf-8');
56
+ }
57
+ /**
58
+ * Batch-embed all wiki pages that don't have embedding sidecar files yet.
59
+ */
60
+ export async function embedAllPages(wikiDir, embeddingProvider, options) {
61
+ const pages = findMarkdownFiles(wikiDir);
62
+ const batchSize = options?.batchSize ?? 20;
63
+ let embedded = 0;
64
+ let skipped = 0;
65
+ let errors = 0;
66
+ // Collect pages that need embedding
67
+ const toEmbed = [];
68
+ for (const pagePath of pages) {
69
+ const sidecarPath = pagePath + '.embedding.json';
70
+ if (!options?.force && existsSync(sidecarPath)) {
71
+ skipped++;
72
+ continue;
73
+ }
74
+ try {
75
+ const content = readFileSync(pagePath, 'utf-8');
76
+ if (content.trim().length > 0) {
77
+ toEmbed.push({ path: pagePath, content: content.substring(0, 8000) });
78
+ }
79
+ }
80
+ catch {
81
+ errors++;
82
+ }
83
+ }
84
+ // Process in batches
85
+ for (let i = 0; i < toEmbed.length; i += batchSize) {
86
+ const batch = toEmbed.slice(i, i + batchSize);
87
+ try {
88
+ const texts = batch.map((p) => p.content);
89
+ const embeddings = await embeddingProvider.embedBatch(texts);
90
+ for (let j = 0; j < batch.length; j++) {
91
+ const page = batch[j];
92
+ const emb = embeddings[j];
93
+ if (!page || !emb)
94
+ continue;
95
+ const result = {
96
+ vector: emb.values,
97
+ model: emb.model,
98
+ dimensions: emb.dimensions,
99
+ text: page.content.substring(0, 500),
100
+ timestamp: new Date().toISOString(),
101
+ };
102
+ writeFileSync(page.path + '.embedding.json', JSON.stringify(result), 'utf-8');
103
+ embedded++;
104
+ }
105
+ }
106
+ catch {
107
+ errors += batch.length;
108
+ }
109
+ }
110
+ return { embedded, skipped, errors };
111
+ }
112
+ function findEmbeddingFiles(dir) {
113
+ const files = [];
114
+ if (!existsSync(dir))
115
+ return files;
116
+ function walk(d) {
117
+ const entries = readdirSync(d);
118
+ for (const entry of entries) {
119
+ const full = join(d, entry);
120
+ try {
121
+ const stat = statSync(full);
122
+ if (stat.isDirectory()) {
123
+ walk(full);
124
+ }
125
+ else if (entry.endsWith('.embedding.json')) {
126
+ files.push(full);
127
+ }
128
+ }
129
+ catch {
130
+ // Skip inaccessible files
131
+ }
132
+ }
133
+ }
134
+ walk(dir);
135
+ return files;
136
+ }
137
+ function findMarkdownFiles(dir) {
138
+ const files = [];
139
+ if (!existsSync(dir))
140
+ return files;
141
+ function walk(d) {
142
+ const entries = readdirSync(d);
143
+ for (const entry of entries) {
144
+ const full = join(d, entry);
145
+ try {
146
+ const stat = statSync(full);
147
+ if (stat.isDirectory()) {
148
+ walk(full);
149
+ }
150
+ else if (entry.endsWith('.md')) {
151
+ files.push(full);
152
+ }
153
+ }
154
+ catch {
155
+ // Skip inaccessible files
156
+ }
157
+ }
158
+ }
159
+ walk(dir);
160
+ return files;
161
+ }
162
+ //# sourceMappingURL=semantic.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"semantic.js","sourceRoot":"","sources":["../../src/search/semantic.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACzF,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAS9D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,OAAe,EACf,iBAAoC,EACpC,KAAc;IAEd,MAAM,UAAU,GAAG,KAAK,IAAI,EAAE,CAAC;IAE/B,8DAA8D;IAC9D,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAE5D,mCAAmC;IACnC,MAAM,cAAc,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAEnD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,oCAAoC;IACpC,MAAM,OAAO,GAA2B,EAAE,CAAC;IAE3C,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC;YAErD,+BAA+B;YAC/B,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;YAC3D,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;gBAAE,SAAS;YAEpC,MAAM,KAAK,GAAG,gBAAgB,CAAC,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YACxE,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAEpD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,iCAAiC;QACnC,CAAC;IACH,CAAC;IAED,OAAO,OAAO;SACX,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAgB,EAChB,OAAe,EACf,iBAAoC;IAEpC,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAEzD,MAAM,MAAM,GAAoB;QAC9B,MAAM,EAAE,SAAS,CAAC,MAAM;QACxB,KAAK,EAAE,SAAS,CAAC,KAAK;QACtB,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,gCAAgC;QACjE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;IAEF,MAAM,WAAW,GAAG,QAAQ,GAAG,iBAAiB,CAAC;IACjD,aAAa,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AAC9D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,iBAAoC,EACpC,OAAiD;IAEjD,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC;IAC3C,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,oCAAoC;IACpC,MAAM,OAAO,GAA6C,EAAE,CAAC;IAE7D,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,WAAW,GAAG,QAAQ,GAAG,iBAAiB,CAAC;QACjD,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/C,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAChD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,EAAE,CAAC;QACX,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC9C,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC1C,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAE7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACtB,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG;oBAAE,SAAS;gBAE5B,MAAM,MAAM,GAAoB;oBAC9B,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,UAAU,EAAE,GAAG,CAAC,UAAU;oBAC1B,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC;oBACpC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACpC,CAAC;gBAEF,aAAa,CAAC,IAAI,CAAC,IAAI,GAAG,iBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;gBAC9E,QAAQ,EAAE,CAAC;YACb,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACvC,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnC,SAAS,IAAI,CAAC,CAAS;QACrB,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC5B,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;oBACvB,IAAI,CAAC,IAAI,CAAC,CAAC;gBACb,CAAC;qBAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;oBAC7C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAW;IACpC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnC,SAAS,IAAI,CAAC,CAAS;QACrB,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC5B,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;oBACvB,IAAI,CAAC,IAAI,CAAC,CAAC;gBACb,CAAC;qBAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBACjC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,KAAK,CAAC;AACf,CAAC"}
@@ -0,0 +1,2 @@
1
+ export declare function getDefaultAgentsMd(template: string): string;
2
+ //# sourceMappingURL=agents-md.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"agents-md.d.ts","sourceRoot":"","sources":["../../src/templates/agents-md.ts"],"names":[],"mappings":"AAAA,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAoF3D"}
@@ -0,0 +1,85 @@
1
+ export function getDefaultAgentsMd(template) {
2
+ const now = new Date().toISOString().split('T')[0];
3
+ return `# AGENTS.md — Wiki Schema & Conventions
4
+
5
+ > This file tells the LLM how your wiki is structured.
6
+ > You and the LLM co-evolve this over time.
7
+
8
+ ## Domain: ${template}
9
+
10
+ ## Created: ${now}
11
+
12
+ ## Directory Structure
13
+
14
+ \`\`\`
15
+ raw/ # Immutable source documents (never modified by LLM)
16
+ wiki/ # LLM-generated markdown (the knowledge base)
17
+ index.md # Content catalog — every page listed with summary
18
+ log.md # Chronological record of operations
19
+ sources/ # One summary page per ingested source
20
+ entities/ # Pages for people, organizations, tools, etc.
21
+ concepts/ # Pages for ideas, frameworks, patterns, etc.
22
+ syntheses/ # Cross-cutting analyses, comparisons, explorations
23
+ AGENTS.md # This file — wiki schema and conventions
24
+ config.yaml # Configuration (LLM provider, sources, schedules)
25
+ \`\`\`
26
+
27
+ ## Page Conventions
28
+
29
+ Every wiki page has YAML frontmatter:
30
+
31
+ \`\`\`yaml
32
+ ---
33
+ title: "Page Title"
34
+ type: source | entity | concept | synthesis | index | log
35
+ created: "YYYY-MM-DD"
36
+ updated: "YYYY-MM-DD"
37
+ tags: [tag1, tag2]
38
+ sources: ["raw/filename.md"] # Which raw sources inform this page
39
+ related: ["[[Other Page]]"] # Explicit cross-references
40
+ summary: "One-line summary" # Used in index.md
41
+ ---
42
+ \`\`\`
43
+
44
+ ## Wikilinks
45
+
46
+ Use \`[[Page Title]]\` to link between pages. The LLM maintains these links.
47
+ Orphan pages (no inbound links) are flagged by \`llmwiki lint\`.
48
+
49
+ ## Operations
50
+
51
+ ### Ingest
52
+ When a new source is added to raw/:
53
+ 1. Read the source completely
54
+ 2. Create/update a source summary page in wiki/sources/
55
+ 3. Identify entities mentioned → create/update entity pages
56
+ 4. Identify concepts discussed → create/update concept pages
57
+ 5. Update index.md with new/modified pages
58
+ 6. Append to log.md
59
+
60
+ ### Query
61
+ When answering a question:
62
+ 1. Read index.md to find relevant pages
63
+ 2. Read the relevant pages
64
+ 3. Synthesize an answer with [[wikilink]] citations
65
+ 4. Optionally file the answer as a synthesis page
66
+
67
+ ### Lint
68
+ Periodically check for:
69
+ - Contradictions between pages
70
+ - Stale claims superseded by newer sources
71
+ - Orphan pages with no inbound links
72
+ - Missing cross-references
73
+ - Important concepts mentioned but lacking their own page
74
+ - Data gaps that could be filled
75
+
76
+ ## Quality Standards
77
+
78
+ - Every claim should cite its source via wikilink
79
+ - Summaries should be concise (1-3 sentences in frontmatter)
80
+ - Pages should be interconnected (no isolated islands)
81
+ - Prefer updating existing pages over creating new ones
82
+ - Flag contradictions explicitly rather than silently overwriting
83
+ `;
84
+ }
85
+ //# sourceMappingURL=agents-md.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"agents-md.js","sourceRoot":"","sources":["../../src/templates/agents-md.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,kBAAkB,CAAC,QAAgB;IACjD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnD,OAAO;;;;;aAKI,QAAQ;;cAEP,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyEhB,CAAC;AACF,CAAC"}
@@ -0,0 +1,2 @@
1
+ export declare function getDefaultConfig(template: string): string;
2
+ //# sourceMappingURL=config-yaml.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config-yaml.d.ts","sourceRoot":"","sources":["../../src/templates/config-yaml.ts"],"names":[],"mappings":"AAAA,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CA4EzD"}
@@ -0,0 +1,81 @@
1
+ export function getDefaultConfig(template) {
2
+ return `# llmwiki configuration
3
+ # Docs: https://github.com/naman10parikh/llmwiki
4
+
5
+ # LLM Provider
6
+ provider: claude # claude | openai | ollama
7
+ model: ~ # Leave empty for provider default
8
+ # api_key: ~ # Set via ANTHROPIC_API_KEY or OPENAI_API_KEY env var
9
+
10
+ # Embeddings (for semantic search)
11
+ embeddings:
12
+ provider: auto # gemini | openai | local | auto (auto-detect from env)
13
+ # model: ~ # Leave empty for provider default
14
+ # GOOGLE_API_KEY env var for Gemini, OPENAI_API_KEY for OpenAI
15
+ # 'local' works offline with zero deps (hash-based, lower quality)
16
+
17
+ # Vault Settings
18
+ vault:
19
+ name: "My ${capitalize(template)} Wiki"
20
+ template: ${template} # personal | research | business | codebase
21
+
22
+ # Automation 2: External Source Scraping
23
+ # Configure sources to scrape on schedule
24
+ sources: []
25
+ # Example:
26
+ # sources:
27
+ # - name: "HN Top Stories"
28
+ # type: rss
29
+ # url: "https://hnrss.org/frontpage"
30
+ # schedule: "0 8 * * *" # Daily at 8am
31
+ #
32
+ # - name: "GitHub Trending"
33
+ # type: github
34
+ # query: "stars:>100 created:>7d language:typescript"
35
+ # schedule: "0 12 * * MON" # Weekly on Monday
36
+ #
37
+ # - name: "Blog Feed"
38
+ # type: rss
39
+ # url: "https://example.com/feed.xml"
40
+ # schedule: "0 */6 * * *" # Every 6 hours
41
+
42
+ # Automation 3: Self-Improvement
43
+ improve:
44
+ enabled: true
45
+ schedule: "0 3 * * *" # Daily at 3am
46
+ threshold: 80 # Score threshold (0-100)
47
+ # Set to false to require manual approval before changes
48
+ auto_apply: false
49
+
50
+ # Search
51
+ search:
52
+ engine: bm25 # bm25 | semantic | hybrid
53
+ # bm25: keyword search (built-in, no API needed)
54
+ # semantic: embedding-based similarity (requires embeddings config)
55
+ # hybrid: combines bm25 + semantic via reciprocal rank fusion
56
+
57
+ # Processing
58
+ processing:
59
+ # Audio transcription (requires Whisper or Deepgram)
60
+ audio:
61
+ enabled: false
62
+ provider: whisper # whisper (local) | deepgram (API)
63
+ # deepgram_api_key: ~
64
+
65
+ # Image description (requires vision-capable model)
66
+ image:
67
+ enabled: false
68
+
69
+ # PDF extraction
70
+ pdf:
71
+ enabled: true
72
+
73
+ # Video processing (requires ffmpeg)
74
+ video:
75
+ enabled: false
76
+ `;
77
+ }
78
+ function capitalize(s) {
79
+ return s.charAt(0).toUpperCase() + s.slice(1);
80
+ }
81
+ //# sourceMappingURL=config-yaml.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config-yaml.js","sourceRoot":"","sources":["../../src/templates/config-yaml.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,OAAO;;;;;;;;;;;;;;;;;cAiBK,UAAU,CAAC,QAAQ,CAAC;cACpB,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwDrB,CAAC;AACF,CAAC;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC"}
@@ -0,0 +1,2 @@
1
+ export declare function createServer(vaultRoot: string, port: number): void;
2
+ //# sourceMappingURL=server.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/web/server.ts"],"names":[],"mappings":"AAyFA,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAuHlE"}
@@ -0,0 +1,170 @@
1
+ import express from 'express';
2
+ import { existsSync, writeFileSync, mkdirSync, readdirSync, statSync } from 'node:fs';
3
+ import { join, basename } from 'node:path';
4
+ import { fileURLToPath } from 'node:url';
5
+ import { getVaultConfig, getVaultStats, listWikiPages, readWikiPage } from '../core/vault.js';
6
+ const __dirname = fileURLToPath(new URL('.', import.meta.url));
7
+ function buildGraph(config) {
8
+ const pages = listWikiPages(config.wikiDir);
9
+ const nodesMap = new Map();
10
+ const links = [];
11
+ const incomingCount = new Map();
12
+ for (const pagePath of pages) {
13
+ const page = readWikiPage(pagePath);
14
+ const id = basename(pagePath, '.md');
15
+ const category = page.frontmatter['category'] ?? 'uncategorized';
16
+ nodesMap.set(id, {
17
+ id,
18
+ title: page.title,
19
+ wordCount: page.wordCount,
20
+ category,
21
+ linksOut: page.wikilinks.length,
22
+ linksIn: 0,
23
+ });
24
+ for (const link of page.wikilinks) {
25
+ links.push({ source: id, target: link });
26
+ incomingCount.set(link, (incomingCount.get(link) ?? 0) + 1);
27
+ }
28
+ }
29
+ // Update incoming link counts
30
+ for (const [id, count] of incomingCount) {
31
+ const node = nodesMap.get(id);
32
+ if (node) {
33
+ node.linksIn = count;
34
+ }
35
+ }
36
+ return {
37
+ nodes: Array.from(nodesMap.values()),
38
+ links: links.filter((l) => nodesMap.has(l.source) && nodesMap.has(l.target)),
39
+ };
40
+ }
41
+ function listPages(config) {
42
+ const pages = listWikiPages(config.wikiDir);
43
+ return pages.map((p) => {
44
+ const page = readWikiPage(p);
45
+ return {
46
+ title: page.title,
47
+ path: p,
48
+ wordCount: page.wordCount,
49
+ category: page.frontmatter['category'] ?? 'uncategorized',
50
+ wikilinks: page.wikilinks,
51
+ };
52
+ });
53
+ }
54
+ export function createServer(vaultRoot, port) {
55
+ const app = express();
56
+ const config = getVaultConfig(vaultRoot);
57
+ app.use(express.json());
58
+ app.use(express.urlencoded({ extended: true }));
59
+ // Serve static files
60
+ const publicDir = join(__dirname, 'public');
61
+ if (existsSync(publicDir)) {
62
+ app.use(express.static(publicDir));
63
+ }
64
+ // API: vault status
65
+ app.get('/api/status', (_req, res) => {
66
+ try {
67
+ const stats = getVaultStats(config);
68
+ res.json(stats);
69
+ }
70
+ catch (err) {
71
+ res.status(500).json({ error: 'Failed to read vault status' });
72
+ }
73
+ });
74
+ // API: list pages
75
+ app.get('/api/pages', (_req, res) => {
76
+ try {
77
+ const pages = listPages(config);
78
+ res.json(pages);
79
+ }
80
+ catch (err) {
81
+ res.status(500).json({ error: 'Failed to list pages' });
82
+ }
83
+ });
84
+ // API: read single page
85
+ app.get('/api/pages/:title', (req, res) => {
86
+ try {
87
+ const title = req.params['title'];
88
+ if (!title) {
89
+ res.status(400).json({ error: 'Missing title' });
90
+ return;
91
+ }
92
+ const pages = listWikiPages(config.wikiDir);
93
+ const match = pages.find((p) => basename(p, '.md') === title);
94
+ if (!match) {
95
+ res.status(404).json({ error: 'Page not found' });
96
+ return;
97
+ }
98
+ const page = readWikiPage(match);
99
+ res.json(page);
100
+ }
101
+ catch (err) {
102
+ res.status(500).json({ error: 'Failed to read page' });
103
+ }
104
+ });
105
+ // API: knowledge graph data
106
+ app.get('/api/graph', (_req, res) => {
107
+ try {
108
+ const graph = buildGraph(config);
109
+ res.json(graph);
110
+ }
111
+ catch (err) {
112
+ res.status(500).json({ error: 'Failed to build graph' });
113
+ }
114
+ });
115
+ // API: upload raw file for ingestion
116
+ app.post('/api/upload', (req, res) => {
117
+ const chunks = [];
118
+ req.on('data', (chunk) => chunks.push(chunk));
119
+ req.on('end', () => {
120
+ const filename = req.headers['x-filename'];
121
+ if (!filename) {
122
+ res.status(400).json({ error: 'Missing x-filename header' });
123
+ return;
124
+ }
125
+ const rawDir = config.rawDir;
126
+ if (!existsSync(rawDir)) {
127
+ mkdirSync(rawDir, { recursive: true });
128
+ }
129
+ const dest = join(rawDir, basename(filename));
130
+ writeFileSync(dest, Buffer.concat(chunks));
131
+ res.json({ status: 'uploaded', path: dest });
132
+ });
133
+ });
134
+ // API: raw files list
135
+ app.get('/api/raw', (_req, res) => {
136
+ try {
137
+ const rawDir = config.rawDir;
138
+ if (!existsSync(rawDir)) {
139
+ res.json([]);
140
+ return;
141
+ }
142
+ const files = readdirSync(rawDir)
143
+ .filter((f) => !f.startsWith('.'))
144
+ .map((f) => {
145
+ const full = join(rawDir, f);
146
+ const stat = statSync(full);
147
+ return { name: f, size: stat.size, modified: stat.mtime.toISOString() };
148
+ });
149
+ res.json(files);
150
+ }
151
+ catch (err) {
152
+ res.status(500).json({ error: 'Failed to list raw files' });
153
+ }
154
+ });
155
+ // Serve index.html for all other routes (SPA)
156
+ app.get('/{*path}', (_req, res) => {
157
+ const indexPath = join(publicDir, 'index.html');
158
+ if (existsSync(indexPath)) {
159
+ res.sendFile(indexPath);
160
+ }
161
+ else {
162
+ res.status(404).send('Web UI not found. Rebuild with pnpm build.');
163
+ }
164
+ });
165
+ app.listen(port, () => {
166
+ console.log(`\n llmwiki web UI running at http://localhost:${port}`);
167
+ console.log(` Vault: ${vaultRoot}\n`);
168
+ });
169
+ }
170
+ //# sourceMappingURL=server.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/web/server.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAgB,UAAU,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACpG,OAAO,EAAE,IAAI,EAAoB,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAG9F,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAqB/D,SAAS,UAAU,CAAC,MAAmB;IACrC,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC9C,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEhD,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,EAAE,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAI,IAAI,CAAC,WAAW,CAAC,UAAU,CAAY,IAAI,eAAe,CAAC;QAC7E,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE;YACf,EAAE;YACF,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ;YACR,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM;YAC/B,OAAO,EAAE,CAAC;SACX,CAAC,CAAC;QAEH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9B,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;KAC7E,CAAC;AACJ,CAAC;AAUD,SAAS,SAAS,CAAC,MAAmB;IACpC,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACrB,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7B,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,CAAC;YACP,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAG,IAAI,CAAC,WAAW,CAAC,UAAU,CAAY,IAAI,eAAe;YACrE,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,SAAiB,EAAE,IAAY;IAC1D,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;IAEzC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACxB,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAEhD,qBAAqB;IACrB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC5C,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IACrC,CAAC;IAED,oBAAoB;IACpB,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;YACpC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,6BAA6B,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,kBAAkB;IAClB,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;QAClC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,sBAAsB,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,wBAAwB;IACxB,GAAG,CAAC,GAAG,CAAC,mBAAmB,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACxC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;gBACjD,OAAO;YACT,CAAC;YACD,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC;YAC9D,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAClD,OAAO;YACT,CAAC;YACD,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,4BAA4B;IAC5B,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;QAClC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,qCAAqC;IACrC,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACnC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACtD,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACjB,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,YAAY,CAAuB,CAAC;YACjE,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,2BAA2B,EAAE,CAAC,CAAC;gBAC7D,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxB,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC9C,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,sBAAsB;IACtB,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC7B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACb,OAAO;YACT,CAAC;YACD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC;iBAC9B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;iBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACT,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAC7B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC5B,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YAC1E,CAAC,CAAC,CAAC;YACL,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,0BAA0B,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,8CAA8C;IAC9C,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAChD,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1B,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;QACrE,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;QACpB,OAAO,CAAC,GAAG,CAAC,kDAAkD,IAAI,EAAE,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,IAAI,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACL,CAAC"}