smartcontext-proxy 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. package/PLAN.md +406 -0
  2. package/PROGRESS.md +60 -0
  3. package/README.md +99 -0
  4. package/SPEC.md +915 -0
  5. package/adapters/openclaw/embedding.d.ts +8 -0
  6. package/adapters/openclaw/embedding.js +16 -0
  7. package/adapters/openclaw/embedding.ts +15 -0
  8. package/adapters/openclaw/index.d.ts +18 -0
  9. package/adapters/openclaw/index.js +42 -0
  10. package/adapters/openclaw/index.ts +43 -0
  11. package/adapters/openclaw/session-importer.d.ts +22 -0
  12. package/adapters/openclaw/session-importer.js +99 -0
  13. package/adapters/openclaw/session-importer.ts +105 -0
  14. package/adapters/openclaw/storage.d.ts +26 -0
  15. package/adapters/openclaw/storage.js +177 -0
  16. package/adapters/openclaw/storage.ts +183 -0
  17. package/dist/adapters/openclaw/embedding.d.ts +8 -0
  18. package/dist/adapters/openclaw/embedding.js +16 -0
  19. package/dist/adapters/openclaw/index.d.ts +18 -0
  20. package/dist/adapters/openclaw/index.js +42 -0
  21. package/dist/adapters/openclaw/session-importer.d.ts +22 -0
  22. package/dist/adapters/openclaw/session-importer.js +99 -0
  23. package/dist/adapters/openclaw/storage.d.ts +26 -0
  24. package/dist/adapters/openclaw/storage.js +177 -0
  25. package/dist/config/auto-detect.d.ts +3 -0
  26. package/dist/config/auto-detect.js +48 -0
  27. package/dist/config/defaults.d.ts +2 -0
  28. package/dist/config/defaults.js +28 -0
  29. package/dist/config/schema.d.ts +30 -0
  30. package/dist/config/schema.js +3 -0
  31. package/dist/context/budget.d.ts +25 -0
  32. package/dist/context/budget.js +85 -0
  33. package/dist/context/canonical.d.ts +39 -0
  34. package/dist/context/canonical.js +12 -0
  35. package/dist/context/chunker.d.ts +9 -0
  36. package/dist/context/chunker.js +148 -0
  37. package/dist/context/optimizer.d.ts +31 -0
  38. package/dist/context/optimizer.js +163 -0
  39. package/dist/context/retriever.d.ts +29 -0
  40. package/dist/context/retriever.js +103 -0
  41. package/dist/daemon/process.d.ts +6 -0
  42. package/dist/daemon/process.js +76 -0
  43. package/dist/daemon/service.d.ts +2 -0
  44. package/dist/daemon/service.js +99 -0
  45. package/dist/embedding/ollama.d.ts +11 -0
  46. package/dist/embedding/ollama.js +72 -0
  47. package/dist/embedding/types.d.ts +6 -0
  48. package/dist/embedding/types.js +3 -0
  49. package/dist/index.d.ts +2 -0
  50. package/dist/index.js +190 -0
  51. package/dist/metrics/collector.d.ts +43 -0
  52. package/dist/metrics/collector.js +72 -0
  53. package/dist/providers/anthropic.d.ts +15 -0
  54. package/dist/providers/anthropic.js +109 -0
  55. package/dist/providers/google.d.ts +13 -0
  56. package/dist/providers/google.js +40 -0
  57. package/dist/providers/ollama.d.ts +13 -0
  58. package/dist/providers/ollama.js +82 -0
  59. package/dist/providers/openai.d.ts +15 -0
  60. package/dist/providers/openai.js +115 -0
  61. package/dist/providers/types.d.ts +18 -0
  62. package/dist/providers/types.js +3 -0
  63. package/dist/proxy/router.d.ts +12 -0
  64. package/dist/proxy/router.js +46 -0
  65. package/dist/proxy/server.d.ts +25 -0
  66. package/dist/proxy/server.js +265 -0
  67. package/dist/proxy/stream.d.ts +8 -0
  68. package/dist/proxy/stream.js +32 -0
  69. package/dist/src/config/auto-detect.d.ts +3 -0
  70. package/dist/src/config/auto-detect.js +48 -0
  71. package/dist/src/config/defaults.d.ts +2 -0
  72. package/dist/src/config/defaults.js +28 -0
  73. package/dist/src/config/schema.d.ts +30 -0
  74. package/dist/src/config/schema.js +3 -0
  75. package/dist/src/context/budget.d.ts +25 -0
  76. package/dist/src/context/budget.js +85 -0
  77. package/dist/src/context/canonical.d.ts +39 -0
  78. package/dist/src/context/canonical.js +12 -0
  79. package/dist/src/context/chunker.d.ts +9 -0
  80. package/dist/src/context/chunker.js +148 -0
  81. package/dist/src/context/optimizer.d.ts +31 -0
  82. package/dist/src/context/optimizer.js +163 -0
  83. package/dist/src/context/retriever.d.ts +29 -0
  84. package/dist/src/context/retriever.js +103 -0
  85. package/dist/src/daemon/process.d.ts +6 -0
  86. package/dist/src/daemon/process.js +76 -0
  87. package/dist/src/daemon/service.d.ts +2 -0
  88. package/dist/src/daemon/service.js +99 -0
  89. package/dist/src/embedding/ollama.d.ts +11 -0
  90. package/dist/src/embedding/ollama.js +72 -0
  91. package/dist/src/embedding/types.d.ts +6 -0
  92. package/dist/src/embedding/types.js +3 -0
  93. package/dist/src/index.d.ts +2 -0
  94. package/dist/src/index.js +190 -0
  95. package/dist/src/metrics/collector.d.ts +43 -0
  96. package/dist/src/metrics/collector.js +72 -0
  97. package/dist/src/providers/anthropic.d.ts +15 -0
  98. package/dist/src/providers/anthropic.js +109 -0
  99. package/dist/src/providers/google.d.ts +13 -0
  100. package/dist/src/providers/google.js +40 -0
  101. package/dist/src/providers/ollama.d.ts +13 -0
  102. package/dist/src/providers/ollama.js +82 -0
  103. package/dist/src/providers/openai.d.ts +15 -0
  104. package/dist/src/providers/openai.js +115 -0
  105. package/dist/src/providers/types.d.ts +18 -0
  106. package/dist/src/providers/types.js +3 -0
  107. package/dist/src/proxy/router.d.ts +12 -0
  108. package/dist/src/proxy/router.js +46 -0
  109. package/dist/src/proxy/server.d.ts +25 -0
  110. package/dist/src/proxy/server.js +265 -0
  111. package/dist/src/proxy/stream.d.ts +8 -0
  112. package/dist/src/proxy/stream.js +32 -0
  113. package/dist/src/storage/lancedb.d.ts +21 -0
  114. package/dist/src/storage/lancedb.js +158 -0
  115. package/dist/src/storage/types.d.ts +52 -0
  116. package/dist/src/storage/types.js +3 -0
  117. package/dist/src/test/context.test.d.ts +1 -0
  118. package/dist/src/test/context.test.js +141 -0
  119. package/dist/src/test/dashboard.test.d.ts +1 -0
  120. package/dist/src/test/dashboard.test.js +85 -0
  121. package/dist/src/test/proxy.test.d.ts +1 -0
  122. package/dist/src/test/proxy.test.js +188 -0
  123. package/dist/src/ui/dashboard.d.ts +2 -0
  124. package/dist/src/ui/dashboard.js +183 -0
  125. package/dist/storage/lancedb.d.ts +21 -0
  126. package/dist/storage/lancedb.js +158 -0
  127. package/dist/storage/types.d.ts +52 -0
  128. package/dist/storage/types.js +3 -0
  129. package/dist/test/context.test.d.ts +1 -0
  130. package/dist/test/context.test.js +141 -0
  131. package/dist/test/dashboard.test.d.ts +1 -0
  132. package/dist/test/dashboard.test.js +85 -0
  133. package/dist/test/proxy.test.d.ts +1 -0
  134. package/dist/test/proxy.test.js +188 -0
  135. package/dist/ui/dashboard.d.ts +2 -0
  136. package/dist/ui/dashboard.js +183 -0
  137. package/package.json +38 -0
  138. package/src/config/auto-detect.ts +51 -0
  139. package/src/config/defaults.ts +26 -0
  140. package/src/config/schema.ts +33 -0
  141. package/src/context/budget.ts +126 -0
  142. package/src/context/canonical.ts +50 -0
  143. package/src/context/chunker.ts +165 -0
  144. package/src/context/optimizer.ts +201 -0
  145. package/src/context/retriever.ts +123 -0
  146. package/src/daemon/process.ts +70 -0
  147. package/src/daemon/service.ts +103 -0
  148. package/src/embedding/ollama.ts +68 -0
  149. package/src/embedding/types.ts +6 -0
  150. package/src/index.ts +176 -0
  151. package/src/metrics/collector.ts +114 -0
  152. package/src/providers/anthropic.ts +117 -0
  153. package/src/providers/google.ts +42 -0
  154. package/src/providers/ollama.ts +87 -0
  155. package/src/providers/openai.ts +127 -0
  156. package/src/providers/types.ts +20 -0
  157. package/src/proxy/router.ts +48 -0
  158. package/src/proxy/server.ts +315 -0
  159. package/src/proxy/stream.ts +39 -0
  160. package/src/storage/lancedb.ts +169 -0
  161. package/src/storage/types.ts +47 -0
  162. package/src/test/context.test.ts +165 -0
  163. package/src/test/dashboard.test.ts +94 -0
  164. package/src/test/proxy.test.ts +218 -0
  165. package/src/ui/dashboard.ts +184 -0
  166. package/tsconfig.json +18 -0
@@ -0,0 +1,148 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.estimateTokens = estimateTokens;
4
+ exports.chunkConversation = chunkConversation;
5
+ const canonical_js_1 = require("./canonical.js");
6
+ const node_crypto_1 = require("node:crypto");
7
+ const AVG_CHARS_PER_TOKEN = 4;
8
+ const MAX_CHUNK_TOKENS = 2000;
9
+ function estimateTokens(text) {
10
+ return Math.ceil(text.length / AVG_CHARS_PER_TOKEN);
11
+ }
12
+ /** Extract file paths mentioned in text */
13
+ function extractFilePaths(text) {
14
+ const patterns = [
15
+ /(?:^|\s)((?:\/[\w.-]+)+(?:\.\w+)?)/gm,
16
+ /(?:^|\s)((?:[\w.-]+\/)+[\w.-]+(?:\.\w+)?)/gm,
17
+ /`([^`]*\/[^`]*\.\w+)`/g,
18
+ ];
19
+ const files = new Set();
20
+ for (const pattern of patterns) {
21
+ let match;
22
+ while ((match = pattern.exec(text)) !== null) {
23
+ const p = match[1].trim();
24
+ if (p.length > 3 && p.includes('/') && !p.startsWith('http')) {
25
+ files.add(p);
26
+ }
27
+ }
28
+ }
29
+ return Array.from(files);
30
+ }
31
+ /** Extract tool names from text */
32
+ function extractTools(text) {
33
+ const tools = new Set();
34
+ const patterns = [
35
+ /tool[_\s]?(?:use|call|result)[:\s]+(\w+)/gi,
36
+ ];
37
+ for (const pattern of patterns) {
38
+ let match;
39
+ while ((match = pattern.exec(text)) !== null) {
40
+ tools.add(match[1]);
41
+ }
42
+ }
43
+ return Array.from(tools);
44
+ }
45
+ /**
46
+ * Chunk a conversation into indexable units.
47
+ * Each chunk = one user-assistant exchange pair.
48
+ * Long responses are split at paragraph boundaries.
49
+ */
50
+ function chunkConversation(messages, sessionId, baseTimestamp) {
51
+ const chunks = [];
52
+ let exchangeIndex = 0;
53
+ for (let i = 0; i < messages.length; i++) {
54
+ const msg = messages[i];
55
+ if (msg.role !== 'user')
56
+ continue;
57
+ const userText = (0, canonical_js_1.getTextContent)(msg);
58
+ let assistantText = '';
59
+ for (let j = i + 1; j < messages.length; j++) {
60
+ if (messages[j].role === 'assistant') {
61
+ assistantText = (0, canonical_js_1.getTextContent)(messages[j]);
62
+ break;
63
+ }
64
+ }
65
+ const fullText = `User: ${userText}\n\nAssistant: ${assistantText}`;
66
+ const tokens = estimateTokens(fullText);
67
+ const files = extractFilePaths(fullText);
68
+ const tools = extractTools(fullText);
69
+ const summary = userText.slice(0, 100);
70
+ const timestamp = baseTimestamp ? baseTimestamp + exchangeIndex * 1000 : Date.now();
71
+ if (tokens <= MAX_CHUNK_TOKENS) {
72
+ chunks.push({
73
+ id: (0, node_crypto_1.randomUUID)(),
74
+ text: fullText,
75
+ embedding: [],
76
+ sessionId,
77
+ timestamp,
78
+ metadata: { files, tools, summary, tokenCount: tokens, exchangeIndex },
79
+ });
80
+ }
81
+ else {
82
+ const paragraphs = splitIntoParagraphs(fullText);
83
+ let current = '';
84
+ let partIndex = 0;
85
+ for (const para of paragraphs) {
86
+ if (estimateTokens(current + para) > MAX_CHUNK_TOKENS && current.length > 0) {
87
+ chunks.push({
88
+ id: (0, node_crypto_1.randomUUID)(),
89
+ text: current.trim(),
90
+ embedding: [],
91
+ sessionId,
92
+ timestamp,
93
+ metadata: {
94
+ files: extractFilePaths(current),
95
+ tools: extractTools(current),
96
+ summary: `${summary} (part ${partIndex + 1})`,
97
+ tokenCount: estimateTokens(current),
98
+ exchangeIndex,
99
+ },
100
+ });
101
+ current = '';
102
+ partIndex++;
103
+ }
104
+ current += para + '\n\n';
105
+ }
106
+ if (current.trim()) {
107
+ chunks.push({
108
+ id: (0, node_crypto_1.randomUUID)(),
109
+ text: current.trim(),
110
+ embedding: [],
111
+ sessionId,
112
+ timestamp,
113
+ metadata: {
114
+ files: extractFilePaths(current),
115
+ tools: extractTools(current),
116
+ summary: `${summary} (part ${partIndex + 1})`,
117
+ tokenCount: estimateTokens(current),
118
+ exchangeIndex,
119
+ },
120
+ });
121
+ }
122
+ }
123
+ exchangeIndex++;
124
+ }
125
+ return chunks;
126
+ }
127
+ function splitIntoParagraphs(text) {
128
+ const parts = [];
129
+ let inCodeBlock = false;
130
+ let current = '';
131
+ for (const line of text.split('\n')) {
132
+ if (line.startsWith('```')) {
133
+ inCodeBlock = !inCodeBlock;
134
+ }
135
+ if (!inCodeBlock && line.trim() === '' && current.trim()) {
136
+ parts.push(current.trim());
137
+ current = '';
138
+ }
139
+ else {
140
+ current += line + '\n';
141
+ }
142
+ }
143
+ if (current.trim()) {
144
+ parts.push(current.trim());
145
+ }
146
+ return parts;
147
+ }
148
+ //# sourceMappingURL=chunker.js.map
@@ -0,0 +1,31 @@
1
+ import type { CanonicalRequest, CanonicalMessage } from './canonical.js';
2
+ import { type RetrievalResult } from './retriever.js';
3
+ import { type PackedContext } from './budget.js';
4
+ import type { EmbeddingAdapter } from '../embedding/types.js';
5
+ import type { StorageAdapter } from '../storage/types.js';
6
+ import type { ContextConfig } from '../config/schema.js';
7
+ export interface OptimizationResult {
8
+ optimizedMessages: CanonicalMessage[];
9
+ systemPrompt?: string;
10
+ packed: PackedContext;
11
+ retrieval: RetrievalResult | null;
12
+ passThrough: boolean;
13
+ reason?: string;
14
+ }
15
+ export declare class ContextOptimizer {
16
+ private embedding;
17
+ private storage;
18
+ private config;
19
+ private retriever;
20
+ constructor(embedding: EmbeddingAdapter, storage: StorageAdapter, config: ContextConfig);
21
+ /**
22
+ * Optimize context for a request.
23
+ * Returns pass-through if optimization isn't beneficial.
24
+ */
25
+ optimize(request: CanonicalRequest): Promise<OptimizationResult>;
26
+ /**
27
+ * Index a completed exchange for future retrieval.
28
+ * Called asynchronously after response completes.
29
+ */
30
+ indexExchange(messages: CanonicalMessage[], sessionId: string): Promise<void>;
31
+ }
@@ -0,0 +1,163 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ContextOptimizer = void 0;
4
+ const canonical_js_1 = require("./canonical.js");
5
+ const chunker_js_1 = require("./chunker.js");
6
+ const retriever_js_1 = require("./retriever.js");
7
+ const budget_js_1 = require("./budget.js");
8
+ class ContextOptimizer {
9
+ embedding;
10
+ storage;
11
+ config;
12
+ retriever;
13
+ constructor(embedding, storage, config) {
14
+ this.embedding = embedding;
15
+ this.storage = storage;
16
+ this.config = config;
17
+ this.retriever = new retriever_js_1.Retriever(embedding, storage, config);
18
+ }
19
+ /**
20
+ * Optimize context for a request.
21
+ * Returns pass-through if optimization isn't beneficial.
22
+ */
23
+ async optimize(request) {
24
+ const { messages, systemPrompt, model } = request;
25
+ // Calculate original size
26
+ const originalTokens = (0, chunker_js_1.estimateTokens)(systemPrompt || '') +
27
+ messages.reduce((sum, m) => sum + (0, chunker_js_1.estimateTokens)((0, canonical_js_1.getTextContent)(m)), 0);
28
+ // Don't optimize short conversations (< tier1 threshold)
29
+ const exchangeCount = messages.filter((m) => m.role === 'user').length;
30
+ if (exchangeCount <= this.config.tier1_exchanges) {
31
+ return {
32
+ optimizedMessages: messages,
33
+ systemPrompt,
34
+ packed: {
35
+ systemPrompt,
36
+ tier1Messages: messages,
37
+ tier2Chunks: [],
38
+ allocation: {
39
+ systemPromptTokens: (0, chunker_js_1.estimateTokens)(systemPrompt || ''),
40
+ tier1Tokens: originalTokens - (0, chunker_js_1.estimateTokens)(systemPrompt || ''),
41
+ tier2Budget: 0,
42
+ tier3Reserve: 0,
43
+ responseReserve: this.config.response_reserve_tokens,
44
+ totalAvailable: originalTokens,
45
+ },
46
+ originalTokens,
47
+ optimizedTokens: originalTokens,
48
+ savingsPercent: 0,
49
+ },
50
+ retrieval: null,
51
+ passThrough: true,
52
+ reason: 'conversation too short for optimization',
53
+ };
54
+ }
55
+ // Extract query from last user message
56
+ const lastUserMsg = [...messages].reverse().find((m) => m.role === 'user');
57
+ const queryText = lastUserMsg ? (0, canonical_js_1.getTextContent)(lastUserMsg) : '';
58
+ // Extract mentioned file paths from query
59
+ const filePathPattern = /(?:[\w.-]+\/)+[\w.-]+(?:\.\w+)?/g;
60
+ const mentionedFiles = queryText.match(filePathPattern) || [];
61
+ // Generate session ID from system prompt hash (stable across requests)
62
+ const sessionId = request.rawHeaders['x-smartcontext-session'] ||
63
+ `session-${simpleHash(systemPrompt || 'default')}`;
64
+ // Retrieve relevant chunks
65
+ let retrieval;
66
+ try {
67
+ retrieval = await this.retriever.retrieve(queryText, sessionId, mentionedFiles);
68
+ }
69
+ catch (err) {
70
+ // Graceful degradation: retrieval failed, pass through
71
+ return {
72
+ optimizedMessages: messages,
73
+ systemPrompt,
74
+ packed: {
75
+ systemPrompt,
76
+ tier1Messages: messages,
77
+ tier2Chunks: [],
78
+ allocation: {
79
+ systemPromptTokens: (0, chunker_js_1.estimateTokens)(systemPrompt || ''),
80
+ tier1Tokens: originalTokens - (0, chunker_js_1.estimateTokens)(systemPrompt || ''),
81
+ tier2Budget: 0,
82
+ tier3Reserve: 0,
83
+ responseReserve: this.config.response_reserve_tokens,
84
+ totalAvailable: originalTokens,
85
+ },
86
+ originalTokens,
87
+ optimizedTokens: originalTokens,
88
+ savingsPercent: 0,
89
+ },
90
+ retrieval: null,
91
+ passThrough: true,
92
+ reason: `retrieval failed: ${err}`,
93
+ };
94
+ }
95
+ // Pack context with budget allocation
96
+ const packed = (0, budget_js_1.packContext)(systemPrompt, messages, retrieval.chunks, model, this.config.tier1_exchanges, this.config.tier3_token_reserve, this.config.response_reserve_tokens);
97
+ // If savings < 10%, not worth the risk — pass through
98
+ if (packed.savingsPercent < 10) {
99
+ return {
100
+ optimizedMessages: messages,
101
+ systemPrompt,
102
+ packed,
103
+ retrieval,
104
+ passThrough: true,
105
+ reason: `savings too low (${packed.savingsPercent}%)`,
106
+ };
107
+ }
108
+ // Assemble optimized messages
109
+ const optimizedMessages = [];
110
+ // Add retrieved context as a system-injected context block
111
+ if (packed.tier2Chunks.length > 0) {
112
+ const contextBlock = packed.tier2Chunks
113
+ .map((c) => `[Context from ${new Date(c.timestamp).toISOString().slice(0, 19)}]\n${c.text}`)
114
+ .join('\n\n---\n\n');
115
+ optimizedMessages.push({
116
+ role: 'user',
117
+ content: `[Retrieved context from previous exchanges]\n\n${contextBlock}`,
118
+ metadata: { provider: 'smartcontext' },
119
+ });
120
+ optimizedMessages.push({
121
+ role: 'assistant',
122
+ content: 'I have reviewed the retrieved context from our previous exchanges. I\'ll use this information to provide more informed responses.',
123
+ metadata: { provider: 'smartcontext' },
124
+ });
125
+ }
126
+ // Add Tier 1 messages (last N exchanges, verbatim)
127
+ optimizedMessages.push(...packed.tier1Messages);
128
+ return {
129
+ optimizedMessages,
130
+ systemPrompt,
131
+ packed,
132
+ retrieval,
133
+ passThrough: false,
134
+ };
135
+ }
136
+ /**
137
+ * Index a completed exchange for future retrieval.
138
+ * Called asynchronously after response completes.
139
+ */
140
+ async indexExchange(messages, sessionId) {
141
+ const chunks = (0, chunker_js_1.chunkConversation)(messages, sessionId);
142
+ if (chunks.length === 0)
143
+ return;
144
+ // Embed all chunks
145
+ const texts = chunks.map((c) => c.text);
146
+ const embeddings = await this.embedding.embed(texts);
147
+ for (let i = 0; i < chunks.length; i++) {
148
+ chunks[i].embedding = embeddings[i];
149
+ }
150
+ await this.storage.upsertChunks(chunks);
151
+ }
152
+ }
153
+ exports.ContextOptimizer = ContextOptimizer;
154
+ function simpleHash(str) {
155
+ let hash = 0;
156
+ for (let i = 0; i < str.length; i++) {
157
+ const char = str.charCodeAt(i);
158
+ hash = ((hash << 5) - hash) + char;
159
+ hash = hash & hash; // Convert to 32-bit integer
160
+ }
161
+ return Math.abs(hash).toString(36);
162
+ }
163
+ //# sourceMappingURL=optimizer.js.map
@@ -0,0 +1,29 @@
1
+ import type { EmbeddingAdapter } from '../embedding/types.js';
2
+ import type { StorageAdapter, ScoredChunk } from '../storage/types.js';
3
+ import type { ContextConfig } from '../config/schema.js';
4
+ export interface RetrievalResult {
5
+ chunks: ScoredChunk[];
6
+ queryEmbeddingMs: number;
7
+ searchMs: number;
8
+ candidates: number;
9
+ aboveThreshold: number;
10
+ afterDedup: number;
11
+ topScore: number;
12
+ }
13
+ /**
14
+ * Retrieval pipeline:
15
+ * 1. Embed query
16
+ * 2. Vector search (top-K candidates)
17
+ * 3. Apply boosts (recency, file-path)
18
+ * 4. Dedup near-duplicates
19
+ * 5. Confidence gate
20
+ */
21
+ export declare class Retriever {
22
+ private embedding;
23
+ private storage;
24
+ private config;
25
+ constructor(embedding: EmbeddingAdapter, storage: StorageAdapter, config: ContextConfig);
26
+ retrieve(queryText: string, currentSessionId: string, mentionedFiles: string[]): Promise<RetrievalResult>;
27
+ /** Remove near-duplicate chunks (same content, different IDs) */
28
+ private dedup;
29
+ }
@@ -0,0 +1,103 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Retriever = void 0;
4
+ /**
5
+ * Retrieval pipeline:
6
+ * 1. Embed query
7
+ * 2. Vector search (top-K candidates)
8
+ * 3. Apply boosts (recency, file-path)
9
+ * 4. Dedup near-duplicates
10
+ * 5. Confidence gate
11
+ */
12
+ class Retriever {
13
+ embedding;
14
+ storage;
15
+ config;
16
+ constructor(embedding, storage, config) {
17
+ this.embedding = embedding;
18
+ this.storage = storage;
19
+ this.config = config;
20
+ }
21
+ async retrieve(queryText, currentSessionId, mentionedFiles) {
22
+ // 1. Embed query
23
+ const embedStart = Date.now();
24
+ const [queryEmbedding] = await this.embedding.embed([queryText]);
25
+ const queryEmbeddingMs = Date.now() - embedStart;
26
+ // 2. Vector search with boosts
27
+ const searchStart = Date.now();
28
+ const candidates = await this.storage.search(queryEmbedding, {
29
+ topK: this.config.tier2_max_chunks * 2,
30
+ minScore: this.config.tier2_min_score * 0.8, // Lower threshold, filter later
31
+ sessionBoost: {
32
+ sessionId: currentSessionId,
33
+ boost: this.config.recency_boost,
34
+ },
35
+ fileBoost: mentionedFiles.length > 0
36
+ ? { patterns: mentionedFiles, boost: this.config.filepath_boost }
37
+ : undefined,
38
+ });
39
+ const searchMs = Date.now() - searchStart;
40
+ // 3. Confidence gate — if best score too low, skip retrieval
41
+ if (candidates.length === 0 || candidates[0].score < this.config.confidence_gate) {
42
+ return {
43
+ chunks: [],
44
+ queryEmbeddingMs,
45
+ searchMs,
46
+ candidates: candidates.length,
47
+ aboveThreshold: 0,
48
+ afterDedup: 0,
49
+ topScore: candidates[0]?.score || 0,
50
+ };
51
+ }
52
+ // 4. Filter by threshold
53
+ const aboveThreshold = candidates.filter((c) => c.score >= this.config.tier2_min_score);
54
+ // 5. Dedup near-duplicates (by text similarity)
55
+ const deduped = this.dedup(aboveThreshold);
56
+ // 6. Ensure minimum chunks
57
+ const result = deduped.slice(0, this.config.tier2_max_chunks);
58
+ const minChunks = Math.min(3, aboveThreshold.length);
59
+ while (result.length < minChunks && aboveThreshold.length > result.length) {
60
+ const next = aboveThreshold[result.length];
61
+ if (!result.some((r) => r.id === next.id)) {
62
+ result.push(next);
63
+ }
64
+ }
65
+ return {
66
+ chunks: result,
67
+ queryEmbeddingMs,
68
+ searchMs,
69
+ candidates: candidates.length,
70
+ aboveThreshold: aboveThreshold.length,
71
+ afterDedup: deduped.length,
72
+ topScore: result[0]?.score || 0,
73
+ };
74
+ }
75
+ /** Remove near-duplicate chunks (same content, different IDs) */
76
+ dedup(chunks) {
77
+ const kept = [];
78
+ for (const chunk of chunks) {
79
+ const isDup = kept.some((k) => {
80
+ // Simple text similarity check
81
+ const shorter = Math.min(k.text.length, chunk.text.length);
82
+ const longer = Math.max(k.text.length, chunk.text.length);
83
+ if (shorter / longer < 0.8)
84
+ return false;
85
+ // Compare first 200 chars
86
+ const a = k.text.slice(0, 200).toLowerCase();
87
+ const b = chunk.text.slice(0, 200).toLowerCase();
88
+ let matches = 0;
89
+ for (let i = 0; i < Math.min(a.length, b.length); i++) {
90
+ if (a[i] === b[i])
91
+ matches++;
92
+ }
93
+ return matches / Math.max(a.length, b.length) > this.config.dedup_threshold;
94
+ });
95
+ if (!isDup) {
96
+ kept.push(chunk);
97
+ }
98
+ }
99
+ return kept;
100
+ }
101
+ }
102
+ exports.Retriever = Retriever;
103
+ //# sourceMappingURL=retriever.js.map
@@ -0,0 +1,6 @@
1
+ export declare function getPid(): number | null;
2
+ export declare function writePid(): void;
3
+ export declare function removePid(): void;
4
+ export declare function startDaemon(args: string[]): number;
5
+ export declare function stopDaemon(): boolean;
6
+ export declare function isDaemonChild(): boolean;
@@ -0,0 +1,76 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.getPid = getPid;
7
+ exports.writePid = writePid;
8
+ exports.removePid = removePid;
9
+ exports.startDaemon = startDaemon;
10
+ exports.stopDaemon = stopDaemon;
11
+ exports.isDaemonChild = isDaemonChild;
12
+ const node_fs_1 = __importDefault(require("node:fs"));
13
+ const node_path_1 = __importDefault(require("node:path"));
14
+ const node_child_process_1 = require("node:child_process");
15
+ const PID_FILE = node_path_1.default.join(process.env['HOME'] || '.', '.smartcontext', 'smartcontext.pid');
16
+ const LOG_FILE = node_path_1.default.join(process.env['HOME'] || '.', '.smartcontext', 'logs', 'proxy.log');
17
+ function getPid() {
18
+ try {
19
+ const pid = parseInt(node_fs_1.default.readFileSync(PID_FILE, 'utf-8').trim(), 10);
20
+ // Check if process is alive
21
+ process.kill(pid, 0);
22
+ return pid;
23
+ }
24
+ catch {
25
+ // Clean up stale PID file
26
+ try {
27
+ node_fs_1.default.unlinkSync(PID_FILE);
28
+ }
29
+ catch { }
30
+ return null;
31
+ }
32
+ }
33
+ function writePid() {
34
+ const dir = node_path_1.default.dirname(PID_FILE);
35
+ node_fs_1.default.mkdirSync(dir, { recursive: true });
36
+ node_fs_1.default.writeFileSync(PID_FILE, String(process.pid));
37
+ }
38
+ function removePid() {
39
+ try {
40
+ node_fs_1.default.unlinkSync(PID_FILE);
41
+ }
42
+ catch { }
43
+ }
44
+ function startDaemon(args) {
45
+ const existingPid = getPid();
46
+ if (existingPid) {
47
+ console.log(`Already running (PID ${existingPid})`);
48
+ return existingPid;
49
+ }
50
+ const logDir = node_path_1.default.dirname(LOG_FILE);
51
+ node_fs_1.default.mkdirSync(logDir, { recursive: true });
52
+ const out = node_fs_1.default.openSync(LOG_FILE, 'a');
53
+ const err = node_fs_1.default.openSync(LOG_FILE, 'a');
54
+ const child = (0, node_child_process_1.spawn)(process.execPath, [__filename, ...args, '--daemon-child'], {
55
+ detached: true,
56
+ stdio: ['ignore', out, err],
57
+ });
58
+ child.unref();
59
+ console.log(`Started (PID ${child.pid})`);
60
+ return child.pid;
61
+ }
62
+ function stopDaemon() {
63
+ const pid = getPid();
64
+ if (!pid) {
65
+ console.log('Not running');
66
+ return false;
67
+ }
68
+ process.kill(pid, 'SIGTERM');
69
+ removePid();
70
+ console.log(`Stopped (PID ${pid})`);
71
+ return true;
72
+ }
73
+ function isDaemonChild() {
74
+ return process.argv.includes('--daemon-child');
75
+ }
76
+ //# sourceMappingURL=process.js.map
@@ -0,0 +1,2 @@
1
+ export declare function installService(port: number): string;
2
+ export declare function uninstallService(): string;
@@ -0,0 +1,99 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.installService = installService;
7
+ exports.uninstallService = uninstallService;
8
+ const node_fs_1 = __importDefault(require("node:fs"));
9
+ const node_path_1 = __importDefault(require("node:path"));
10
+ const node_os_1 = __importDefault(require("node:os"));
11
+ function installService(port) {
12
+ if (process.platform === 'darwin') {
13
+ return installLaunchAgent(port);
14
+ }
15
+ return installSystemd(port);
16
+ }
17
+ function uninstallService() {
18
+ if (process.platform === 'darwin') {
19
+ return uninstallLaunchAgent();
20
+ }
21
+ return uninstallSystemd();
22
+ }
23
+ function installLaunchAgent(port) {
24
+ const plistDir = node_path_1.default.join(node_os_1.default.homedir(), 'Library', 'LaunchAgents');
25
+ const plistPath = node_path_1.default.join(plistDir, 'com.smartcontext.proxy.plist');
26
+ const logDir = node_path_1.default.join(node_os_1.default.homedir(), '.smartcontext', 'logs');
27
+ node_fs_1.default.mkdirSync(plistDir, { recursive: true });
28
+ node_fs_1.default.mkdirSync(logDir, { recursive: true });
29
+ const nodePath = process.execPath;
30
+ const scriptPath = node_path_1.default.resolve(node_path_1.default.join(__dirname, '..', 'index.js'));
31
+ const plist = `<?xml version="1.0" encoding="UTF-8"?>
32
+ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
33
+ <plist version="1.0">
34
+ <dict>
35
+ <key>Label</key>
36
+ <string>com.smartcontext.proxy</string>
37
+ <key>ProgramArguments</key>
38
+ <array>
39
+ <string>${nodePath}</string>
40
+ <string>${scriptPath}</string>
41
+ <string>--port</string>
42
+ <string>${port}</string>
43
+ </array>
44
+ <key>RunAtLoad</key>
45
+ <true/>
46
+ <key>KeepAlive</key>
47
+ <true/>
48
+ <key>StandardOutPath</key>
49
+ <string>${logDir}/proxy.log</string>
50
+ <key>StandardErrorPath</key>
51
+ <string>${logDir}/proxy.err.log</string>
52
+ </dict>
53
+ </plist>`;
54
+ node_fs_1.default.writeFileSync(plistPath, plist);
55
+ return plistPath;
56
+ }
57
+ function uninstallLaunchAgent() {
58
+ const plistPath = node_path_1.default.join(node_os_1.default.homedir(), 'Library', 'LaunchAgents', 'com.smartcontext.proxy.plist');
59
+ try {
60
+ node_fs_1.default.unlinkSync(plistPath);
61
+ return `Removed ${plistPath}`;
62
+ }
63
+ catch {
64
+ return 'No service file found';
65
+ }
66
+ }
67
+ function installSystemd(port) {
68
+ const serviceDir = node_path_1.default.join(node_os_1.default.homedir(), '.config', 'systemd', 'user');
69
+ const servicePath = node_path_1.default.join(serviceDir, 'smartcontext-proxy.service');
70
+ node_fs_1.default.mkdirSync(serviceDir, { recursive: true });
71
+ const nodePath = process.execPath;
72
+ const scriptPath = node_path_1.default.resolve(node_path_1.default.join(__dirname, '..', 'index.js'));
73
+ const service = `[Unit]
74
+ Description=SmartContext Proxy
75
+ After=network.target
76
+
77
+ [Service]
78
+ Type=simple
79
+ ExecStart=${nodePath} ${scriptPath} --port ${port}
80
+ Restart=always
81
+ RestartSec=5
82
+
83
+ [Install]
84
+ WantedBy=default.target
85
+ `;
86
+ node_fs_1.default.writeFileSync(servicePath, service);
87
+ return servicePath;
88
+ }
89
+ function uninstallSystemd() {
90
+ const servicePath = node_path_1.default.join(node_os_1.default.homedir(), '.config', 'systemd', 'user', 'smartcontext-proxy.service');
91
+ try {
92
+ node_fs_1.default.unlinkSync(servicePath);
93
+ return `Removed ${servicePath}`;
94
+ }
95
+ catch {
96
+ return 'No service file found';
97
+ }
98
+ }
99
+ //# sourceMappingURL=service.js.map
@@ -0,0 +1,11 @@
1
+ import type { EmbeddingAdapter } from './types.js';
2
+ export declare class OllamaEmbeddingAdapter implements EmbeddingAdapter {
3
+ private url;
4
+ private model;
5
+ name: string;
6
+ dimensions: number;
7
+ constructor(url?: string, model?: string);
8
+ initialize(): Promise<void>;
9
+ embed(texts: string[]): Promise<number[][]>;
10
+ private embedSingle;
11
+ }