somtum 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/LICENSE +663 -0
  2. package/README.md +459 -0
  3. package/dist/cli/config_cmd.d.ts +10 -0
  4. package/dist/cli/config_cmd.d.ts.map +1 -0
  5. package/dist/cli/config_cmd.js +96 -0
  6. package/dist/cli/config_cmd.js.map +1 -0
  7. package/dist/cli/doctor.d.ts +17 -0
  8. package/dist/cli/doctor.d.ts.map +1 -0
  9. package/dist/cli/doctor.js +169 -0
  10. package/dist/cli/doctor.js.map +1 -0
  11. package/dist/cli/edit.d.ts +4 -0
  12. package/dist/cli/edit.d.ts.map +1 -0
  13. package/dist/cli/edit.js +63 -0
  14. package/dist/cli/edit.js.map +1 -0
  15. package/dist/cli/export.d.ts +19 -0
  16. package/dist/cli/export.d.ts.map +1 -0
  17. package/dist/cli/export.js +84 -0
  18. package/dist/cli/export.js.map +1 -0
  19. package/dist/cli/forget.d.ts +11 -0
  20. package/dist/cli/forget.d.ts.map +1 -0
  21. package/dist/cli/forget.js +32 -0
  22. package/dist/cli/forget.js.map +1 -0
  23. package/dist/cli/hook.d.ts +2 -0
  24. package/dist/cli/hook.d.ts.map +1 -0
  25. package/dist/cli/hook.js +66 -0
  26. package/dist/cli/hook.js.map +1 -0
  27. package/dist/cli/import.d.ts +18 -0
  28. package/dist/cli/import.d.ts.map +1 -0
  29. package/dist/cli/import.js +90 -0
  30. package/dist/cli/import.js.map +1 -0
  31. package/dist/cli/index.d.ts +3 -0
  32. package/dist/cli/index.d.ts.map +1 -0
  33. package/dist/cli/index.js +223 -0
  34. package/dist/cli/index.js.map +1 -0
  35. package/dist/cli/init.d.ts +24 -0
  36. package/dist/cli/init.d.ts.map +1 -0
  37. package/dist/cli/init.js +144 -0
  38. package/dist/cli/init.js.map +1 -0
  39. package/dist/cli/purge.d.ts +16 -0
  40. package/dist/cli/purge.d.ts.map +1 -0
  41. package/dist/cli/purge.js +69 -0
  42. package/dist/cli/purge.js.map +1 -0
  43. package/dist/cli/rebuild.d.ts +15 -0
  44. package/dist/cli/rebuild.d.ts.map +1 -0
  45. package/dist/cli/rebuild.js +41 -0
  46. package/dist/cli/rebuild.js.map +1 -0
  47. package/dist/cli/reindex.d.ts +6 -0
  48. package/dist/cli/reindex.d.ts.map +1 -0
  49. package/dist/cli/reindex.js +47 -0
  50. package/dist/cli/reindex.js.map +1 -0
  51. package/dist/cli/search.d.ts +16 -0
  52. package/dist/cli/search.d.ts.map +1 -0
  53. package/dist/cli/search.js +62 -0
  54. package/dist/cli/search.js.map +1 -0
  55. package/dist/cli/show.d.ts +12 -0
  56. package/dist/cli/show.d.ts.map +1 -0
  57. package/dist/cli/show.js +49 -0
  58. package/dist/cli/show.js.map +1 -0
  59. package/dist/cli/stats.d.ts +27 -0
  60. package/dist/cli/stats.d.ts.map +1 -0
  61. package/dist/cli/stats.js +80 -0
  62. package/dist/cli/stats.js.map +1 -0
  63. package/dist/cli/sync.d.ts +18 -0
  64. package/dist/cli/sync.d.ts.map +1 -0
  65. package/dist/cli/sync.js +138 -0
  66. package/dist/cli/sync.js.map +1 -0
  67. package/dist/config.d.ts +12 -0
  68. package/dist/config.d.ts.map +1 -0
  69. package/dist/config.js +52 -0
  70. package/dist/config.js.map +1 -0
  71. package/dist/core/cache.d.ts +29 -0
  72. package/dist/core/cache.d.ts.map +1 -0
  73. package/dist/core/cache.js +137 -0
  74. package/dist/core/cache.js.map +1 -0
  75. package/dist/core/db.d.ts +12 -0
  76. package/dist/core/db.d.ts.map +1 -0
  77. package/dist/core/db.js +80 -0
  78. package/dist/core/db.js.map +1 -0
  79. package/dist/core/embeddings.d.ts +15 -0
  80. package/dist/core/embeddings.d.ts.map +1 -0
  81. package/dist/core/embeddings.js +74 -0
  82. package/dist/core/embeddings.js.map +1 -0
  83. package/dist/core/embeddings_bootstrap.d.ts +3 -0
  84. package/dist/core/embeddings_bootstrap.d.ts.map +1 -0
  85. package/dist/core/embeddings_bootstrap.js +16 -0
  86. package/dist/core/embeddings_bootstrap.js.map +1 -0
  87. package/dist/core/embeddings_xenova.d.ts +3 -0
  88. package/dist/core/embeddings_xenova.d.ts.map +1 -0
  89. package/dist/core/embeddings_xenova.js +28 -0
  90. package/dist/core/embeddings_xenova.js.map +1 -0
  91. package/dist/core/extractor.d.ts +27 -0
  92. package/dist/core/extractor.d.ts.map +1 -0
  93. package/dist/core/extractor.js +133 -0
  94. package/dist/core/extractor.js.map +1 -0
  95. package/dist/core/file_summary.d.ts +33 -0
  96. package/dist/core/file_summary.d.ts.map +1 -0
  97. package/dist/core/file_summary.js +127 -0
  98. package/dist/core/file_summary.js.map +1 -0
  99. package/dist/core/fingerprint.d.ts +13 -0
  100. package/dist/core/fingerprint.d.ts.map +1 -0
  101. package/dist/core/fingerprint.js +36 -0
  102. package/dist/core/fingerprint.js.map +1 -0
  103. package/dist/core/index_gen.d.ts +16 -0
  104. package/dist/core/index_gen.d.ts.map +1 -0
  105. package/dist/core/index_gen.js +173 -0
  106. package/dist/core/index_gen.js.map +1 -0
  107. package/dist/core/memory_files.d.ts +4 -0
  108. package/dist/core/memory_files.d.ts.map +1 -0
  109. package/dist/core/memory_files.js +36 -0
  110. package/dist/core/memory_files.js.map +1 -0
  111. package/dist/core/privacy.d.ts +9 -0
  112. package/dist/core/privacy.d.ts.map +1 -0
  113. package/dist/core/privacy.js +32 -0
  114. package/dist/core/privacy.js.map +1 -0
  115. package/dist/core/project_id.d.ts +3 -0
  116. package/dist/core/project_id.d.ts.map +1 -0
  117. package/dist/core/project_id.js +31 -0
  118. package/dist/core/project_id.js.map +1 -0
  119. package/dist/core/reindex.d.ts +10 -0
  120. package/dist/core/reindex.d.ts.map +1 -0
  121. package/dist/core/reindex.js +40 -0
  122. package/dist/core/reindex.js.map +1 -0
  123. package/dist/core/retrieval_stats.d.ts +22 -0
  124. package/dist/core/retrieval_stats.d.ts.map +1 -0
  125. package/dist/core/retrieval_stats.js +56 -0
  126. package/dist/core/retrieval_stats.js.map +1 -0
  127. package/dist/core/retriever/bm25.d.ts +15 -0
  128. package/dist/core/retriever/bm25.d.ts.map +1 -0
  129. package/dist/core/retriever/bm25.js +51 -0
  130. package/dist/core/retriever/bm25.js.map +1 -0
  131. package/dist/core/retriever/embeddings.d.ts +9 -0
  132. package/dist/core/retriever/embeddings.d.ts.map +1 -0
  133. package/dist/core/retriever/embeddings.js +34 -0
  134. package/dist/core/retriever/embeddings.js.map +1 -0
  135. package/dist/core/retriever/factory.d.ts +6 -0
  136. package/dist/core/retriever/factory.d.ts.map +1 -0
  137. package/dist/core/retriever/factory.js +53 -0
  138. package/dist/core/retriever/factory.js.map +1 -0
  139. package/dist/core/retriever/hybrid.d.ts +14 -0
  140. package/dist/core/retriever/hybrid.d.ts.map +1 -0
  141. package/dist/core/retriever/hybrid.js +52 -0
  142. package/dist/core/retriever/hybrid.js.map +1 -0
  143. package/dist/core/retriever/llm_index.d.ts +18 -0
  144. package/dist/core/retriever/llm_index.d.ts.map +1 -0
  145. package/dist/core/retriever/llm_index.js +90 -0
  146. package/dist/core/retriever/llm_index.js.map +1 -0
  147. package/dist/core/retriever/types.d.ts +16 -0
  148. package/dist/core/retriever/types.d.ts.map +1 -0
  149. package/dist/core/retriever/types.js +2 -0
  150. package/dist/core/retriever/types.js.map +1 -0
  151. package/dist/core/schema.d.ts +470 -0
  152. package/dist/core/schema.d.ts.map +1 -0
  153. package/dist/core/schema.js +153 -0
  154. package/dist/core/schema.js.map +1 -0
  155. package/dist/core/store.d.ts +35 -0
  156. package/dist/core/store.d.ts.map +1 -0
  157. package/dist/core/store.js +172 -0
  158. package/dist/core/store.js.map +1 -0
  159. package/dist/core/tokens.d.ts +3 -0
  160. package/dist/core/tokens.d.ts.map +1 -0
  161. package/dist/core/tokens.js +12 -0
  162. package/dist/core/tokens.js.map +1 -0
  163. package/dist/core/transcript.d.ts +16 -0
  164. package/dist/core/transcript.d.ts.map +1 -0
  165. package/dist/core/transcript.js +128 -0
  166. package/dist/core/transcript.js.map +1 -0
  167. package/dist/db/migrations/001_init.sql +90 -0
  168. package/dist/db/migrations/002_cache_files.sql +6 -0
  169. package/dist/db/migrations/003_retrieval_stats.sql +17 -0
  170. package/dist/hooks/post_session.d.ts +65 -0
  171. package/dist/hooks/post_session.d.ts.map +1 -0
  172. package/dist/hooks/post_session.js +249 -0
  173. package/dist/hooks/post_session.js.map +1 -0
  174. package/dist/hooks/pre_prompt.d.ts +55 -0
  175. package/dist/hooks/pre_prompt.d.ts.map +1 -0
  176. package/dist/hooks/pre_prompt.js +98 -0
  177. package/dist/hooks/pre_prompt.js.map +1 -0
  178. package/dist/hooks/pre_read.d.ts +56 -0
  179. package/dist/hooks/pre_read.d.ts.map +1 -0
  180. package/dist/hooks/pre_read.js +77 -0
  181. package/dist/hooks/pre_read.js.map +1 -0
  182. package/dist/index.d.ts +15 -0
  183. package/dist/index.d.ts.map +1 -0
  184. package/dist/index.js +15 -0
  185. package/dist/index.js.map +1 -0
  186. package/dist/mcp/server.d.ts +20 -0
  187. package/dist/mcp/server.d.ts.map +1 -0
  188. package/dist/mcp/server.js +128 -0
  189. package/dist/mcp/server.js.map +1 -0
  190. package/dist/mcp/tools.d.ts +72 -0
  191. package/dist/mcp/tools.d.ts.map +1 -0
  192. package/dist/mcp/tools.js +153 -0
  193. package/dist/mcp/tools.js.map +1 -0
  194. package/package.json +79 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retrieval_stats.d.ts","sourceRoot":"","sources":["../../src/core/retrieval_stats.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAgBrD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,qBAAa,mBAAmB;IAClB,OAAO,CAAC,QAAQ,CAAC,EAAE;gBAAF,EAAE,EAAE,EAAE;IAEnC,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,GAAG,IAAI;IAYxE,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAY1C,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAW3C,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,kBAAkB,EAAE;IAW9D,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,eAAe;CAYvD"}
@@ -0,0 +1,56 @@
1
+ export class RetrievalStatsStore {
2
+ db;
3
+ constructor(db) {
4
+ this.db = db;
5
+ }
6
+ incrementRetrieval(projectId, strategy) {
7
+ this.db
8
+ .prepare(`INSERT INTO retrieval_stats (project_id, strategy, call_count, last_called_at)
9
+ VALUES (?, ?, 1, ?)
10
+ ON CONFLICT (project_id, strategy) DO UPDATE SET
11
+ call_count = call_count + 1,
12
+ last_called_at = excluded.last_called_at`)
13
+ .run(projectId, strategy, Date.now());
14
+ }
15
+ incrementCacheHit(projectId) {
16
+ this.db
17
+ .prepare(`INSERT INTO cache_hit_stats (project_id, hit_count, miss_count, last_hit_at)
18
+ VALUES (?, 1, 0, ?)
19
+ ON CONFLICT (project_id) DO UPDATE SET
20
+ hit_count = hit_count + 1,
21
+ last_hit_at = excluded.last_hit_at`)
22
+ .run(projectId, Date.now());
23
+ }
24
+ incrementCacheMiss(projectId) {
25
+ this.db
26
+ .prepare(`INSERT INTO cache_hit_stats (project_id, hit_count, miss_count, last_hit_at)
27
+ VALUES (?, 0, 1, NULL)
28
+ ON CONFLICT (project_id) DO UPDATE SET
29
+ miss_count = miss_count + 1`)
30
+ .run(projectId);
31
+ }
32
+ getRetrievalBreakdown(projectId) {
33
+ const rows = this.db
34
+ .prepare(`SELECT * FROM retrieval_stats WHERE project_id = ? ORDER BY call_count DESC`)
35
+ .all(projectId);
36
+ return rows.map((r) => ({
37
+ strategy: r.strategy,
38
+ call_count: r.call_count,
39
+ last_called_at: r.last_called_at,
40
+ }));
41
+ }
42
+ getCacheHitSummary(projectId) {
43
+ const row = this.db
44
+ .prepare(`SELECT * FROM cache_hit_stats WHERE project_id = ?`)
45
+ .get(projectId);
46
+ if (!row)
47
+ return { hit_count: 0, miss_count: 0, hit_rate: null };
48
+ const total = row.hit_count + row.miss_count;
49
+ return {
50
+ hit_count: row.hit_count,
51
+ miss_count: row.miss_count,
52
+ hit_rate: total > 0 ? Number((row.hit_count / total).toFixed(3)) : null,
53
+ };
54
+ }
55
+ }
56
+ //# sourceMappingURL=retrieval_stats.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retrieval_stats.js","sourceRoot":"","sources":["../../src/core/retrieval_stats.ts"],"names":[],"mappings":"AA6BA,MAAM,OAAO,mBAAmB;IACD;IAA7B,YAA6B,EAAM;QAAN,OAAE,GAAF,EAAE,CAAI;IAAG,CAAC;IAEvC,kBAAkB,CAAC,SAAiB,EAAE,QAA2B;QAC/D,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;;;oDAI4C,CAC7C;aACA,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,iBAAiB,CAAC,SAAiB;QACjC,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;;;8CAIsC,CACvC;aACA,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAChC,CAAC;IAED,kBAAkB,CAAC,SAAiB;QAClC,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;;uCAG+B,CAChC;aACA,GAAG,CAAC,SAAS,CAAC,CAAC;IACpB,CAAC;IAED,qBAAqB,CAAC,SAAiB;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,6EAA6E,CAAC;aACtF,GAAG,CAAC,SAAS,CAAmB,CAAC;QACpC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACtB,QAAQ,EAAE,CAAC,CAAC,QAA6B;YACzC,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,cAAc,EAAE,CAAC,CAAC,cAAc;SACjC,CAAC,CAAC,CAAC;IACN,CAAC;IAED,kBAAkB,CAAC,SAAiB;QAClC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,oDAAoD,CAAC;aAC7D,GAAG,CAAC,SAAS,CAA4B,CAAC;QAC7C,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC;QAC7C,OAAO;YACL,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;SACxE,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,15 @@
1
+ import type { DB } from '../db.js';
2
+ import type { Observation } from '../schema.js';
3
+ import type { Retriever, RetrievalResult, SearchOptions } from './types.js';
4
+ export declare class Bm25Retriever implements Retriever {
5
+ private readonly db;
6
+ readonly name: "bm25";
7
+ private readonly store;
8
+ constructor(db: DB);
9
+ search(query: string, options: SearchOptions): Promise<RetrievalResult[]>;
10
+ }
11
+ export declare function _forTestingOnly(): {
12
+ escapeFtsQuery: (q: string) => string;
13
+ };
14
+ export type { Observation };
15
+ //# sourceMappingURL=bm25.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../../src/core/retriever/bm25.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,UAAU,CAAC;AAEnC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAoB5E,qBAAa,aAAc,YAAW,SAAS;IAIjC,OAAO,CAAC,QAAQ,CAAC,EAAE;IAH/B,QAAQ,CAAC,IAAI,EAAG,MAAM,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;gBAEP,EAAE,EAAE,EAAE;IAI7B,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CA0BhF;AAED,wBAAgB,eAAe,IAAI;IAAE,cAAc,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,CAAA;CAAE,CAE3E;AAED,YAAY,EAAE,WAAW,EAAE,CAAC"}
@@ -0,0 +1,51 @@
1
+ import { MemoryStore } from '../store.js';
2
+ // SQLite FTS5 MATCH uses an embedded query grammar; user input with bare
3
+ // punctuation can raise "fts5: syntax error". We quote each token to treat
4
+ // the input as a bag of literal terms (phrase-ish), which is the right
5
+ // behavior for natural-language queries against BM25.
6
+ function escapeFtsQuery(query) {
7
+ const tokens = query
8
+ .split(/\s+/)
9
+ .map((t) => t.replace(/["]/g, '').trim())
10
+ .filter((t) => t.length > 0);
11
+ if (tokens.length === 0)
12
+ return '""';
13
+ return tokens.map((t) => `"${t}"`).join(' OR ');
14
+ }
15
+ export class Bm25Retriever {
16
+ db;
17
+ name = 'bm25';
18
+ store;
19
+ constructor(db) {
20
+ this.db = db;
21
+ this.store = new MemoryStore(db);
22
+ }
23
+ async search(query, options) {
24
+ const ftsQuery = escapeFtsQuery(query);
25
+ if (ftsQuery === '""')
26
+ return [];
27
+ // bm25() returns lower-is-better; invert so higher = more relevant,
28
+ // and the Retriever contract's `score` reads naturally.
29
+ const rows = this.db
30
+ .prepare(`SELECT o.id AS id, -bm25(observations_fts) AS score
31
+ FROM observations_fts
32
+ JOIN observations o ON o.rowid = observations_fts.rowid
33
+ WHERE observations_fts MATCH ?
34
+ AND o.project_id = ?
35
+ AND o.deleted_at IS NULL
36
+ ORDER BY score DESC
37
+ LIMIT ?`)
38
+ .all(ftsQuery, options.projectId, options.k);
39
+ const results = [];
40
+ for (const row of rows) {
41
+ const obs = this.store.get(row.id);
42
+ if (obs)
43
+ results.push({ id: row.id, score: row.score, observation: obs, source: 'bm25' });
44
+ }
45
+ return results;
46
+ }
47
+ }
48
+ export function _forTestingOnly() {
49
+ return { escapeFtsQuery };
50
+ }
51
+ //# sourceMappingURL=bm25.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"bm25.js","sourceRoot":"","sources":["../../../src/core/retriever/bm25.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI1C,yEAAyE;AACzE,2EAA2E;AAC3E,uEAAuE;AACvE,sDAAsD;AACtD,SAAS,cAAc,CAAC,KAAa;IACnC,MAAM,MAAM,GAAG,KAAK;SACjB,KAAK,CAAC,KAAK,CAAC;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;SACxC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAClD,CAAC;AAOD,MAAM,OAAO,aAAa;IAIK;IAHpB,IAAI,GAAG,MAAe,CAAC;IACf,KAAK,CAAc;IAEpC,YAA6B,EAAM;QAAN,OAAE,GAAF,EAAE,CAAI;QACjC,IAAI,CAAC,KAAK,GAAG,IAAI,WAAW,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAsB;QAChD,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,EAAE,CAAC;QAEjC,oEAAoE;QACpE,wDAAwD;QACxD,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;;;;;;iBAOS,CACV;aACA,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAc,CAAC;QAE5D,MAAM,OAAO,GAAsB,EAAE,CAAC;QACtC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,GAAG;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAC5F,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,EAAE,cAAc,EAAE,CAAC;AAC5B,CAAC"}
@@ -0,0 +1,9 @@
1
+ import type { DB } from '../db.js';
2
+ import type { Retriever, RetrievalResult, SearchOptions } from './types.js';
3
+ export declare class EmbeddingsRetriever implements Retriever {
4
+ readonly name: "embeddings";
5
+ private readonly store;
6
+ constructor(db: DB);
7
+ search(query: string, options: SearchOptions): Promise<RetrievalResult[]>;
8
+ }
9
+ //# sourceMappingURL=embeddings.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../../src/core/retriever/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,UAAU,CAAC;AAGnC,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAI5E,qBAAa,mBAAoB,YAAW,SAAS;IACnD,QAAQ,CAAC,IAAI,EAAG,YAAY,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;gBAExB,EAAE,EAAE,EAAE;IAIZ,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAqBhF"}
@@ -0,0 +1,34 @@
1
+ import { MemoryStore } from '../store.js';
2
+ import { cosineSimilarity, decodeVector, getEmbedder } from '../embeddings.js';
3
+ // Brute-force cosine over all stored vectors for the project. Fine up to
4
+ // tens of thousands of observations; swap for an ANN index later if needed.
5
+ export class EmbeddingsRetriever {
6
+ name = 'embeddings';
7
+ store;
8
+ constructor(db) {
9
+ this.store = new MemoryStore(db);
10
+ }
11
+ async search(query, options) {
12
+ if (query.trim().length === 0)
13
+ return [];
14
+ const embedder = await getEmbedder();
15
+ const [qv] = await embedder.embed([query]);
16
+ if (!qv)
17
+ return [];
18
+ const rows = this.store.listWithEmbeddings(options.projectId);
19
+ const scored = rows.map((r) => {
20
+ const v = decodeVector(r.embedding, embedder.dim);
21
+ return { id: r.id, score: cosineSimilarity(qv, v) };
22
+ });
23
+ scored.sort((a, b) => b.score - a.score);
24
+ const top = scored.slice(0, options.k);
25
+ const results = [];
26
+ for (const s of top) {
27
+ const obs = this.store.get(s.id);
28
+ if (obs)
29
+ results.push({ id: s.id, score: s.score, observation: obs, source: 'embeddings' });
30
+ }
31
+ return results;
32
+ }
33
+ }
34
+ //# sourceMappingURL=embeddings.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../../src/core/retriever/embeddings.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAG/E,yEAAyE;AACzE,4EAA4E;AAC5E,MAAM,OAAO,mBAAmB;IACrB,IAAI,GAAG,YAAqB,CAAC;IACrB,KAAK,CAAc;IAEpC,YAAY,EAAM;QAChB,IAAI,CAAC,KAAK,GAAG,IAAI,WAAW,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAsB;QAChD,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3C,IAAI,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC5B,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YAClD,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAEzC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,OAAO,GAAsB,EAAE,CAAC;QACtC,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACjC,IAAI,GAAG;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QAC9F,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
@@ -0,0 +1,6 @@
1
+ import type { DB } from '../db.js';
2
+ import type { Config, RetrievalStrategy } from '../schema.js';
3
+ import type { Retriever } from './types.js';
4
+ export declare function makeRetriever(strategy: RetrievalStrategy, db: DB, config: Config): Retriever;
5
+ export declare function strategyAvailable(strategy: RetrievalStrategy, config?: Config): boolean;
6
+ //# sourceMappingURL=factory.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../../src/core/retriever/factory.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,UAAU,CAAC;AACnC,OAAO,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAY5C,wBAAgB,aAAa,CAAC,QAAQ,EAAE,iBAAiB,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,CAuB5F;AAED,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAUvF"}
@@ -0,0 +1,53 @@
1
+ import Anthropic from '@anthropic-ai/sdk';
2
+ import { Bm25Retriever } from './bm25.js';
3
+ import { EmbeddingsRetriever } from './embeddings.js';
4
+ import { HybridRetriever } from './hybrid.js';
5
+ import { LlmIndexRetriever } from './llm_index.js';
6
+ import { anthropicCaller } from '../extractor.js';
7
+ import { isEmbedderReady } from '../embeddings.js';
8
+ // Returns a retriever by strategy name. Falls back to BM25 when the requested
9
+ // strategy's prerequisites are missing (e.g. embeddings disabled in config or
10
+ // the model has not been loaded). Callers that care about the fallback can
11
+ // check `strategyAvailable` up front.
12
+ export function makeRetriever(strategy, db, config) {
13
+ switch (strategy) {
14
+ case 'bm25':
15
+ return new Bm25Retriever(db);
16
+ case 'embeddings':
17
+ if (!config.retrieval.embeddings.enabled)
18
+ return new Bm25Retriever(db);
19
+ return new EmbeddingsRetriever(db);
20
+ case 'hybrid':
21
+ if (!config.retrieval.embeddings.enabled)
22
+ return new Bm25Retriever(db);
23
+ return new HybridRetriever(db);
24
+ case 'index': {
25
+ if (!config.retrieval.index.enabled)
26
+ return new Bm25Retriever(db);
27
+ const apiKey = process.env['ANTHROPIC_API_KEY'];
28
+ if (!apiKey)
29
+ return new Bm25Retriever(db);
30
+ const caller = anthropicCaller(new Anthropic({ apiKey }));
31
+ return new LlmIndexRetriever(db, {
32
+ caller,
33
+ model: config.retrieval.index.model,
34
+ });
35
+ }
36
+ default:
37
+ return new Bm25Retriever(db);
38
+ }
39
+ }
40
+ export function strategyAvailable(strategy, config) {
41
+ if (strategy === 'bm25')
42
+ return true;
43
+ if (strategy === 'embeddings' || strategy === 'hybrid') {
44
+ if (!config?.retrieval.embeddings.enabled)
45
+ return false;
46
+ return isEmbedderReady() || true;
47
+ }
48
+ if (strategy === 'index') {
49
+ return Boolean(config?.retrieval.index.enabled && process.env['ANTHROPIC_API_KEY']);
50
+ }
51
+ return false;
52
+ }
53
+ //# sourceMappingURL=factory.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"factory.js","sourceRoot":"","sources":["../../../src/core/retriever/factory.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAI1C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,8EAA8E;AAC9E,8EAA8E;AAC9E,2EAA2E;AAC3E,sCAAsC;AACtC,MAAM,UAAU,aAAa,CAAC,QAA2B,EAAE,EAAM,EAAE,MAAc;IAC/E,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,MAAM;YACT,OAAO,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;QAC/B,KAAK,YAAY;YACf,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO;gBAAE,OAAO,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;YACvE,OAAO,IAAI,mBAAmB,CAAC,EAAE,CAAC,CAAC;QACrC,KAAK,QAAQ;YACX,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO;gBAAE,OAAO,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;YACvE,OAAO,IAAI,eAAe,CAAC,EAAE,CAAC,CAAC;QACjC,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO;gBAAE,OAAO,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;YAClE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YAChD,IAAI,CAAC,MAAM;gBAAE,OAAO,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;YAC1D,OAAO,IAAI,iBAAiB,CAAC,EAAE,EAAE;gBAC/B,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK;aACpC,CAAC,CAAC;QACL,CAAC;QACD;YACE,OAAO,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAA2B,EAAE,MAAe;IAC5E,IAAI,QAAQ,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACrC,IAAI,QAAQ,KAAK,YAAY,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACvD,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QACxD,OAAO,eAAe,EAAE,IAAI,IAAI,CAAC;IACnC,CAAC;IACD,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QACzB,OAAO,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
@@ -0,0 +1,14 @@
1
+ import type { DB } from '../db.js';
2
+ import type { Retriever, RetrievalResult, SearchOptions } from './types.js';
3
+ export interface HybridOptions {
4
+ poolSize?: number;
5
+ }
6
+ export declare class HybridRetriever implements Retriever {
7
+ readonly name: "hybrid";
8
+ private readonly bm25;
9
+ private readonly embeddings;
10
+ private readonly poolSize;
11
+ constructor(db: DB, options?: HybridOptions);
12
+ search(query: string, options: SearchOptions): Promise<RetrievalResult[]>;
13
+ }
14
+ //# sourceMappingURL=hybrid.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"hybrid.d.ts","sourceRoot":"","sources":["../../../src/core/retriever/hybrid.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,UAAU,CAAC;AAGnC,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAW5E,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,eAAgB,YAAW,SAAS;IAC/C,QAAQ,CAAC,IAAI,EAAG,QAAQ,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgB;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;IACjD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;gBAEtB,EAAE,EAAE,EAAE,EAAE,OAAO,GAAE,aAAkB;IAMzC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAoChF"}
@@ -0,0 +1,52 @@
1
+ import { Bm25Retriever } from './bm25.js';
2
+ import { EmbeddingsRetriever } from './embeddings.js';
3
+ // Reciprocal Rank Fusion. Widely-used, parameter-light way to blend two
4
+ // ranked lists without having to normalize scores across strategies.
5
+ // k=60 is the standard default from the original RRF paper.
6
+ const RRF_K = 60;
7
+ function rrf(rank) {
8
+ return 1 / (RRF_K + rank + 1);
9
+ }
10
+ export class HybridRetriever {
11
+ name = 'hybrid';
12
+ bm25;
13
+ embeddings;
14
+ poolSize;
15
+ constructor(db, options = {}) {
16
+ this.bm25 = new Bm25Retriever(db);
17
+ this.embeddings = new EmbeddingsRetriever(db);
18
+ this.poolSize = options.poolSize ?? 50;
19
+ }
20
+ async search(query, options) {
21
+ const poolOpts = { k: this.poolSize, projectId: options.projectId };
22
+ // Run both retrievers in parallel; if one throws (e.g. embedder not loaded)
23
+ // the other still produces a result set.
24
+ const [bm25Res, embRes] = await Promise.all([
25
+ this.bm25.search(query, poolOpts).catch(() => []),
26
+ this.embeddings.search(query, poolOpts).catch(() => []),
27
+ ]);
28
+ const merged = new Map();
29
+ const accumulate = (list) => {
30
+ list.forEach((r, i) => {
31
+ const prev = merged.get(r.id);
32
+ const contribution = rrf(i);
33
+ if (prev) {
34
+ prev.score += contribution;
35
+ }
36
+ else {
37
+ merged.set(r.id, {
38
+ result: { ...r, source: 'hybrid' },
39
+ score: contribution,
40
+ });
41
+ }
42
+ });
43
+ };
44
+ accumulate(bm25Res);
45
+ accumulate(embRes);
46
+ return [...merged.values()]
47
+ .sort((a, b) => b.score - a.score)
48
+ .slice(0, options.k)
49
+ .map((e) => ({ ...e.result, score: e.score }));
50
+ }
51
+ }
52
+ //# sourceMappingURL=hybrid.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"hybrid.js","sourceRoot":"","sources":["../../../src/core/retriever/hybrid.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAGtD,wEAAwE;AACxE,qEAAqE;AACrE,4DAA4D;AAC5D,MAAM,KAAK,GAAG,EAAE,CAAC;AAEjB,SAAS,GAAG,CAAC,IAAY;IACvB,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;AAChC,CAAC;AAMD,MAAM,OAAO,eAAe;IACjB,IAAI,GAAG,QAAiB,CAAC;IACjB,IAAI,CAAgB;IACpB,UAAU,CAAsB;IAChC,QAAQ,CAAS;IAElC,YAAY,EAAM,EAAE,UAAyB,EAAE;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,UAAU,GAAG,IAAI,mBAAmB,CAAC,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAsB;QAChD,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;QACpE,4EAA4E;QAC5E,yCAAyC;QACzC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC1C,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAuB,CAAC;YACtE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAuB,CAAC;SAC7E,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,IAAI,GAAG,EAGnB,CAAC;QAEJ,MAAM,UAAU,GAAG,CAAC,IAAuB,EAAQ,EAAE;YACnD,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACpB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9B,MAAM,YAAY,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC5B,IAAI,IAAI,EAAE,CAAC;oBACT,IAAI,CAAC,KAAK,IAAI,YAAY,CAAC;gBAC7B,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;wBACf,MAAM,EAAE,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE;wBAClC,KAAK,EAAE,YAAY;qBACpB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC;QACF,UAAU,CAAC,OAAO,CAAC,CAAC;QACpB,UAAU,CAAC,MAAM,CAAC,CAAC;QAEnB,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;aACxB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACjC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;aACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACnD,CAAC;CACF"}
@@ -0,0 +1,18 @@
1
+ import type { DB } from '../db.js';
2
+ import type { LlmCaller } from '../extractor.js';
3
+ import type { Retriever, RetrievalResult, SearchOptions } from './types.js';
4
+ export interface LlmIndexRetrieverOptions {
5
+ caller: LlmCaller;
6
+ model: string;
7
+ maxCatalog?: number;
8
+ }
9
+ export declare class LlmIndexRetriever implements Retriever {
10
+ readonly name: "index";
11
+ private readonly store;
12
+ private readonly caller;
13
+ private readonly model;
14
+ private readonly maxCatalog;
15
+ constructor(db: DB, opts: LlmIndexRetrieverOptions);
16
+ search(query: string, options: SearchOptions): Promise<RetrievalResult[]>;
17
+ }
18
+ //# sourceMappingURL=llm_index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"llm_index.d.ts","sourceRoot":"","sources":["../../../src/core/retriever/llm_index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,UAAU,CAAC;AAEnC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AA+C5E,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,SAAS,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IAId,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAID,qBAAa,iBAAkB,YAAW,SAAS;IACjD,QAAQ,CAAC,IAAI,EAAG,OAAO,CAAU;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,wBAAwB;IAO5C,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAyChF"}
@@ -0,0 +1,90 @@
1
+ import { z } from 'zod';
2
+ import { MemoryStore } from '../store.js';
3
+ // LLM-backed retriever. Feeds a compact catalog (id + title + kind + files +
4
+ // tags) to a cheap model and asks it to pick the top-K observation IDs.
5
+ // Slower and costlier than BM25, but much better at paraphrased queries
6
+ // and cross-field intent ("auth stuff" → matches an observation titled
7
+ // "JWT session rotation"). Falls back to BM25 via the factory when the
8
+ // caller or the `retrieval.index.enabled` flag is missing.
9
+ const SYSTEM_PROMPT = `You are Somtum's index retriever. You are given a user query and a catalog of stored observations (each with id, kind, title, tags, files). Pick the most relevant observations for the query.
10
+
11
+ OUTPUT: a single JSON object, no prose, no markdown fences:
12
+ {"ids": ["id1","id2",...]}
13
+
14
+ Rules:
15
+ - Only include IDs that appear verbatim in the catalog.
16
+ - Order by relevance, most relevant first.
17
+ - Return at most the number of IDs requested by the user.
18
+ - If nothing matches, return {"ids": []}.`;
19
+ const IdsSchema = z.object({ ids: z.array(z.string()) });
20
+ function toCatalog(observations) {
21
+ return observations.map((o) => ({
22
+ id: o.id,
23
+ kind: o.kind,
24
+ title: o.title,
25
+ tags: o.tags,
26
+ files: o.files,
27
+ }));
28
+ }
29
+ function extractJsonBlob(text) {
30
+ const trimmed = text.trim();
31
+ const fence = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/);
32
+ if (fence?.[1])
33
+ return fence[1].trim();
34
+ return trimmed;
35
+ }
36
+ const DEFAULT_MAX_CATALOG = 500;
37
+ export class LlmIndexRetriever {
38
+ name = 'index';
39
+ store;
40
+ caller;
41
+ model;
42
+ maxCatalog;
43
+ constructor(db, opts) {
44
+ this.store = new MemoryStore(db);
45
+ this.caller = opts.caller;
46
+ this.model = opts.model;
47
+ this.maxCatalog = opts.maxCatalog ?? DEFAULT_MAX_CATALOG;
48
+ }
49
+ async search(query, options) {
50
+ if (query.trim().length === 0)
51
+ return [];
52
+ // listByProject returns most-recent first, so truncation drops the oldest.
53
+ const all = this.store.listByProject(options.projectId, { limit: this.maxCatalog });
54
+ if (all.length === 0)
55
+ return [];
56
+ const catalog = toCatalog(all);
57
+ const user = `Query: ${query}\n\n` +
58
+ `Return up to ${options.k} IDs.\n\n` +
59
+ `Catalog:\n${JSON.stringify(catalog)}`;
60
+ const { text } = await this.caller.complete({
61
+ model: this.model,
62
+ system: SYSTEM_PROMPT,
63
+ user,
64
+ });
65
+ let parsed;
66
+ try {
67
+ parsed = JSON.parse(extractJsonBlob(text));
68
+ }
69
+ catch {
70
+ return [];
71
+ }
72
+ const validated = IdsSchema.safeParse(parsed);
73
+ if (!validated.success)
74
+ return [];
75
+ const byId = new Map(all.map((o) => [o.id, o]));
76
+ const results = [];
77
+ const top = validated.data.ids.slice(0, options.k);
78
+ top.forEach((id, idx) => {
79
+ const obs = byId.get(id);
80
+ if (!obs)
81
+ return;
82
+ // Synthetic linear score so callers that rank multiple strategies
83
+ // can still compare. Top hit = 1.0, last hit = ~1/k.
84
+ const score = (top.length - idx) / top.length;
85
+ results.push({ id, score, observation: obs, source: 'index' });
86
+ });
87
+ return results;
88
+ }
89
+ }
90
+ //# sourceMappingURL=llm_index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"llm_index.js","sourceRoot":"","sources":["../../../src/core/retriever/llm_index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAK1C,6EAA6E;AAC7E,wEAAwE;AACxE,wEAAwE;AACxE,uEAAuE;AACvE,uEAAuE;AACvE,2DAA2D;AAE3D,MAAM,aAAa,GAAG;;;;;;;;;0CASoB,CAAC;AAE3C,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;AAUzD,SAAS,SAAS,CAAC,YAA2B;IAC5C,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9B,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,KAAK,EAAE,CAAC,CAAC,KAAK;KACf,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAC5D,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,OAAO,OAAO,CAAC;AACjB,CAAC;AAWD,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,OAAgB,CAAC;IAChB,KAAK,CAAc;IACnB,MAAM,CAAY;IAClB,KAAK,CAAS;IACd,UAAU,CAAS;IAEpC,YAAY,EAAM,EAAE,IAA8B;QAChD,IAAI,CAAC,KAAK,GAAG,IAAI,WAAW,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,mBAAmB,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAsB;QAChD,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEzC,2EAA2E;QAC3E,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACpF,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEhC,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,IAAI,GACR,UAAU,KAAK,MAAM;YACrB,gBAAgB,OAAO,CAAC,CAAC,WAAW;YACpC,aAAa,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAEzC,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;YAC1C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,aAAa;YACrB,IAAI;SACL,CAAC,CAAC;QAEH,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,CAAC,SAAS,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,OAAO,GAAsB,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QACnD,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzB,IAAI,CAAC,GAAG;gBAAE,OAAO;YACjB,kEAAkE;YAClE,qDAAqD;YACrD,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC;YAC9C,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
@@ -0,0 +1,16 @@
1
+ import type { Observation } from '../schema.js';
2
+ export interface RetrievalResult {
3
+ id: string;
4
+ score: number;
5
+ observation: Observation;
6
+ source: 'bm25' | 'embeddings' | 'index' | 'hybrid';
7
+ }
8
+ export interface SearchOptions {
9
+ k: number;
10
+ projectId: string;
11
+ }
12
+ export interface Retriever {
13
+ readonly name: 'bm25' | 'embeddings' | 'index' | 'hybrid';
14
+ search(query: string, options: SearchOptions): Promise<RetrievalResult[]>;
15
+ }
16
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/core/retriever/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,WAAW,CAAC;IAEzB,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,OAAO,GAAG,QAAQ,CAAC;CACpD;AAED,MAAM,WAAW,aAAa;IAC5B,CAAC,EAAE,MAAM,CAAC;IACV,SAAS,EAAE,MAAM,CAAC;CACnB;AAID,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,GAAG,OAAO,GAAG,QAAQ,CAAC;IAC1D,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CAC3E"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/core/retriever/types.ts"],"names":[],"mappings":""}