smartcontext-proxy 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLAN.md +406 -0
- package/PROGRESS.md +60 -0
- package/README.md +99 -0
- package/SPEC.md +915 -0
- package/adapters/openclaw/embedding.d.ts +8 -0
- package/adapters/openclaw/embedding.js +16 -0
- package/adapters/openclaw/embedding.ts +15 -0
- package/adapters/openclaw/index.d.ts +18 -0
- package/adapters/openclaw/index.js +42 -0
- package/adapters/openclaw/index.ts +43 -0
- package/adapters/openclaw/session-importer.d.ts +22 -0
- package/adapters/openclaw/session-importer.js +99 -0
- package/adapters/openclaw/session-importer.ts +105 -0
- package/adapters/openclaw/storage.d.ts +26 -0
- package/adapters/openclaw/storage.js +177 -0
- package/adapters/openclaw/storage.ts +183 -0
- package/dist/adapters/openclaw/embedding.d.ts +8 -0
- package/dist/adapters/openclaw/embedding.js +16 -0
- package/dist/adapters/openclaw/index.d.ts +18 -0
- package/dist/adapters/openclaw/index.js +42 -0
- package/dist/adapters/openclaw/session-importer.d.ts +22 -0
- package/dist/adapters/openclaw/session-importer.js +99 -0
- package/dist/adapters/openclaw/storage.d.ts +26 -0
- package/dist/adapters/openclaw/storage.js +177 -0
- package/dist/config/auto-detect.d.ts +3 -0
- package/dist/config/auto-detect.js +48 -0
- package/dist/config/defaults.d.ts +2 -0
- package/dist/config/defaults.js +28 -0
- package/dist/config/schema.d.ts +30 -0
- package/dist/config/schema.js +3 -0
- package/dist/context/budget.d.ts +25 -0
- package/dist/context/budget.js +85 -0
- package/dist/context/canonical.d.ts +39 -0
- package/dist/context/canonical.js +12 -0
- package/dist/context/chunker.d.ts +9 -0
- package/dist/context/chunker.js +148 -0
- package/dist/context/optimizer.d.ts +31 -0
- package/dist/context/optimizer.js +163 -0
- package/dist/context/retriever.d.ts +29 -0
- package/dist/context/retriever.js +103 -0
- package/dist/daemon/process.d.ts +6 -0
- package/dist/daemon/process.js +76 -0
- package/dist/daemon/service.d.ts +2 -0
- package/dist/daemon/service.js +99 -0
- package/dist/embedding/ollama.d.ts +11 -0
- package/dist/embedding/ollama.js +72 -0
- package/dist/embedding/types.d.ts +6 -0
- package/dist/embedding/types.js +3 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +190 -0
- package/dist/metrics/collector.d.ts +43 -0
- package/dist/metrics/collector.js +72 -0
- package/dist/providers/anthropic.d.ts +15 -0
- package/dist/providers/anthropic.js +109 -0
- package/dist/providers/google.d.ts +13 -0
- package/dist/providers/google.js +40 -0
- package/dist/providers/ollama.d.ts +13 -0
- package/dist/providers/ollama.js +82 -0
- package/dist/providers/openai.d.ts +15 -0
- package/dist/providers/openai.js +115 -0
- package/dist/providers/types.d.ts +18 -0
- package/dist/providers/types.js +3 -0
- package/dist/proxy/router.d.ts +12 -0
- package/dist/proxy/router.js +46 -0
- package/dist/proxy/server.d.ts +25 -0
- package/dist/proxy/server.js +265 -0
- package/dist/proxy/stream.d.ts +8 -0
- package/dist/proxy/stream.js +32 -0
- package/dist/src/config/auto-detect.d.ts +3 -0
- package/dist/src/config/auto-detect.js +48 -0
- package/dist/src/config/defaults.d.ts +2 -0
- package/dist/src/config/defaults.js +28 -0
- package/dist/src/config/schema.d.ts +30 -0
- package/dist/src/config/schema.js +3 -0
- package/dist/src/context/budget.d.ts +25 -0
- package/dist/src/context/budget.js +85 -0
- package/dist/src/context/canonical.d.ts +39 -0
- package/dist/src/context/canonical.js +12 -0
- package/dist/src/context/chunker.d.ts +9 -0
- package/dist/src/context/chunker.js +148 -0
- package/dist/src/context/optimizer.d.ts +31 -0
- package/dist/src/context/optimizer.js +163 -0
- package/dist/src/context/retriever.d.ts +29 -0
- package/dist/src/context/retriever.js +103 -0
- package/dist/src/daemon/process.d.ts +6 -0
- package/dist/src/daemon/process.js +76 -0
- package/dist/src/daemon/service.d.ts +2 -0
- package/dist/src/daemon/service.js +99 -0
- package/dist/src/embedding/ollama.d.ts +11 -0
- package/dist/src/embedding/ollama.js +72 -0
- package/dist/src/embedding/types.d.ts +6 -0
- package/dist/src/embedding/types.js +3 -0
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.js +190 -0
- package/dist/src/metrics/collector.d.ts +43 -0
- package/dist/src/metrics/collector.js +72 -0
- package/dist/src/providers/anthropic.d.ts +15 -0
- package/dist/src/providers/anthropic.js +109 -0
- package/dist/src/providers/google.d.ts +13 -0
- package/dist/src/providers/google.js +40 -0
- package/dist/src/providers/ollama.d.ts +13 -0
- package/dist/src/providers/ollama.js +82 -0
- package/dist/src/providers/openai.d.ts +15 -0
- package/dist/src/providers/openai.js +115 -0
- package/dist/src/providers/types.d.ts +18 -0
- package/dist/src/providers/types.js +3 -0
- package/dist/src/proxy/router.d.ts +12 -0
- package/dist/src/proxy/router.js +46 -0
- package/dist/src/proxy/server.d.ts +25 -0
- package/dist/src/proxy/server.js +265 -0
- package/dist/src/proxy/stream.d.ts +8 -0
- package/dist/src/proxy/stream.js +32 -0
- package/dist/src/storage/lancedb.d.ts +21 -0
- package/dist/src/storage/lancedb.js +158 -0
- package/dist/src/storage/types.d.ts +52 -0
- package/dist/src/storage/types.js +3 -0
- package/dist/src/test/context.test.d.ts +1 -0
- package/dist/src/test/context.test.js +141 -0
- package/dist/src/test/dashboard.test.d.ts +1 -0
- package/dist/src/test/dashboard.test.js +85 -0
- package/dist/src/test/proxy.test.d.ts +1 -0
- package/dist/src/test/proxy.test.js +188 -0
- package/dist/src/ui/dashboard.d.ts +2 -0
- package/dist/src/ui/dashboard.js +183 -0
- package/dist/storage/lancedb.d.ts +21 -0
- package/dist/storage/lancedb.js +158 -0
- package/dist/storage/types.d.ts +52 -0
- package/dist/storage/types.js +3 -0
- package/dist/test/context.test.d.ts +1 -0
- package/dist/test/context.test.js +141 -0
- package/dist/test/dashboard.test.d.ts +1 -0
- package/dist/test/dashboard.test.js +85 -0
- package/dist/test/proxy.test.d.ts +1 -0
- package/dist/test/proxy.test.js +188 -0
- package/dist/ui/dashboard.d.ts +2 -0
- package/dist/ui/dashboard.js +183 -0
- package/package.json +38 -0
- package/src/config/auto-detect.ts +51 -0
- package/src/config/defaults.ts +26 -0
- package/src/config/schema.ts +33 -0
- package/src/context/budget.ts +126 -0
- package/src/context/canonical.ts +50 -0
- package/src/context/chunker.ts +165 -0
- package/src/context/optimizer.ts +201 -0
- package/src/context/retriever.ts +123 -0
- package/src/daemon/process.ts +70 -0
- package/src/daemon/service.ts +103 -0
- package/src/embedding/ollama.ts +68 -0
- package/src/embedding/types.ts +6 -0
- package/src/index.ts +176 -0
- package/src/metrics/collector.ts +114 -0
- package/src/providers/anthropic.ts +117 -0
- package/src/providers/google.ts +42 -0
- package/src/providers/ollama.ts +87 -0
- package/src/providers/openai.ts +127 -0
- package/src/providers/types.ts +20 -0
- package/src/proxy/router.ts +48 -0
- package/src/proxy/server.ts +315 -0
- package/src/proxy/stream.ts +39 -0
- package/src/storage/lancedb.ts +169 -0
- package/src/storage/types.ts +47 -0
- package/src/test/context.test.ts +165 -0
- package/src/test/dashboard.test.ts +94 -0
- package/src/test/proxy.test.ts +218 -0
- package/src/ui/dashboard.ts +184 -0
- package/tsconfig.json +18 -0
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import http from 'node:http';
|
|
2
|
+
import type { StorageAdapter, Chunk, ScoredChunk, SearchOptions, Exchange } from '../../src/storage/types.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* OpenSearch storage adapter for SmartContext.
|
|
6
|
+
* Stores chunks and metrics in OpenSearch indices on Castle VPS.
|
|
7
|
+
*/
|
|
8
|
+
export class OpenClawStorageAdapter implements StorageAdapter {
|
|
9
|
+
name = 'openclaw-opensearch';
|
|
10
|
+
private baseUrl: string;
|
|
11
|
+
private chunksIndex = 'smartcontext-chunks';
|
|
12
|
+
private metricsIndex = 'smartcontext-metrics';
|
|
13
|
+
private logsIndex = 'smartcontext-logs';
|
|
14
|
+
|
|
15
|
+
constructor(esUrl: string = 'http://localhost:9201') {
|
|
16
|
+
this.baseUrl = esUrl;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
async initialize(): Promise<void> {
|
|
20
|
+
// Create indices if they don't exist
|
|
21
|
+
for (const index of [this.chunksIndex, this.metricsIndex, this.logsIndex]) {
|
|
22
|
+
try {
|
|
23
|
+
await this.esRequest('PUT', `/${index}`, {
|
|
24
|
+
settings: { number_of_shards: 1, number_of_replicas: 0 },
|
|
25
|
+
mappings: index === this.chunksIndex ? this.chunkMappings() : undefined,
|
|
26
|
+
});
|
|
27
|
+
} catch {
|
|
28
|
+
// Index may already exist
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async upsertChunks(chunks: Chunk[]): Promise<void> {
|
|
34
|
+
if (chunks.length === 0) return;
|
|
35
|
+
|
|
36
|
+
const bulkBody = chunks.flatMap((c) => [
|
|
37
|
+
JSON.stringify({ index: { _index: this.chunksIndex, _id: c.id } }),
|
|
38
|
+
JSON.stringify({
|
|
39
|
+
text: c.text,
|
|
40
|
+
embedding: c.embedding,
|
|
41
|
+
sessionId: c.sessionId,
|
|
42
|
+
timestamp: c.timestamp,
|
|
43
|
+
summary: c.metadata.summary,
|
|
44
|
+
tokenCount: c.metadata.tokenCount,
|
|
45
|
+
exchangeIndex: c.metadata.exchangeIndex,
|
|
46
|
+
files: c.metadata.files,
|
|
47
|
+
tools: c.metadata.tools,
|
|
48
|
+
}),
|
|
49
|
+
]).join('\n') + '\n';
|
|
50
|
+
|
|
51
|
+
await this.esRequest('POST', '/_bulk', bulkBody, 'application/x-ndjson');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async search(embedding: number[], options: SearchOptions): Promise<ScoredChunk[]> {
|
|
55
|
+
const query: Record<string, unknown> = {
|
|
56
|
+
size: options.topK,
|
|
57
|
+
query: {
|
|
58
|
+
script_score: {
|
|
59
|
+
query: { match_all: {} },
|
|
60
|
+
script: {
|
|
61
|
+
source: "cosineSimilarity(params.query_vector, 'embedding') + 1.0",
|
|
62
|
+
params: { query_vector: embedding },
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const result = await this.esRequest('POST', `/${this.chunksIndex}/_search`, query);
|
|
69
|
+
const parsed = JSON.parse(result);
|
|
70
|
+
|
|
71
|
+
return (parsed.hits?.hits || []).map((hit: Record<string, unknown>) => {
|
|
72
|
+
const source = hit._source as Record<string, unknown>;
|
|
73
|
+
let score = ((hit._score as number) - 1.0); // Remove the +1.0 offset
|
|
74
|
+
|
|
75
|
+
// Apply boosts
|
|
76
|
+
if (options.sessionBoost && source.sessionId === options.sessionBoost.sessionId) {
|
|
77
|
+
score += options.sessionBoost.boost;
|
|
78
|
+
}
|
|
79
|
+
if (options.fileBoost) {
|
|
80
|
+
const files = (source.files as string[]) || [];
|
|
81
|
+
if (options.fileBoost.patterns.some((p) => files.some((f) => f.includes(p)))) {
|
|
82
|
+
score += options.fileBoost.boost;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return {
|
|
87
|
+
id: hit._id as string,
|
|
88
|
+
text: source.text as string,
|
|
89
|
+
embedding: [],
|
|
90
|
+
sessionId: source.sessionId as string,
|
|
91
|
+
timestamp: source.timestamp as number,
|
|
92
|
+
score,
|
|
93
|
+
metadata: {
|
|
94
|
+
summary: source.summary as string,
|
|
95
|
+
tokenCount: source.tokenCount as number,
|
|
96
|
+
exchangeIndex: source.exchangeIndex as number,
|
|
97
|
+
files: source.files as string[],
|
|
98
|
+
tools: source.tools as string[],
|
|
99
|
+
},
|
|
100
|
+
} as ScoredChunk;
|
|
101
|
+
}).filter((c: ScoredChunk) => c.score >= options.minScore);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async appendLog(sessionId: string, exchange: Exchange): Promise<void> {
|
|
105
|
+
await this.esRequest('POST', `/${this.logsIndex}/_doc`, {
|
|
106
|
+
sessionId,
|
|
107
|
+
...exchange,
|
|
108
|
+
indexedAt: new Date().toISOString(),
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async getSessionLog(sessionId: string): Promise<Exchange[]> {
|
|
113
|
+
const result = await this.esRequest('POST', `/${this.logsIndex}/_search`, {
|
|
114
|
+
size: 1000,
|
|
115
|
+
query: { term: { sessionId } },
|
|
116
|
+
sort: [{ timestamp: 'asc' }],
|
|
117
|
+
});
|
|
118
|
+
const parsed = JSON.parse(result);
|
|
119
|
+
return (parsed.hits?.hits || []).map((h: Record<string, unknown>) => (h._source as unknown) as Exchange);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async getStats(): Promise<{ chunks: number; sessions: number; diskBytes: number }> {
|
|
123
|
+
try {
|
|
124
|
+
const result = await this.esRequest('GET', `/${this.chunksIndex}/_count`);
|
|
125
|
+
const parsed = JSON.parse(result);
|
|
126
|
+
return {
|
|
127
|
+
chunks: parsed.count || 0,
|
|
128
|
+
sessions: 0, // Would need aggregation
|
|
129
|
+
diskBytes: 0,
|
|
130
|
+
};
|
|
131
|
+
} catch {
|
|
132
|
+
return { chunks: 0, sessions: 0, diskBytes: 0 };
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async close(): Promise<void> {
|
|
137
|
+
// No cleanup needed for HTTP client
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
private chunkMappings() {
|
|
141
|
+
return {
|
|
142
|
+
properties: {
|
|
143
|
+
text: { type: 'text' },
|
|
144
|
+
embedding: { type: 'dense_vector', dims: 768 },
|
|
145
|
+
sessionId: { type: 'keyword' },
|
|
146
|
+
timestamp: { type: 'long' },
|
|
147
|
+
summary: { type: 'text' },
|
|
148
|
+
tokenCount: { type: 'integer' },
|
|
149
|
+
exchangeIndex: { type: 'integer' },
|
|
150
|
+
files: { type: 'keyword' },
|
|
151
|
+
tools: { type: 'keyword' },
|
|
152
|
+
},
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
private esRequest(method: string, path: string, body?: unknown, contentType?: string): Promise<string> {
|
|
157
|
+
return new Promise((resolve, reject) => {
|
|
158
|
+
const url = new URL(`${this.baseUrl}${path}`);
|
|
159
|
+
const bodyStr = typeof body === 'string' ? body : body ? JSON.stringify(body) : undefined;
|
|
160
|
+
|
|
161
|
+
const req = http.request(url, {
|
|
162
|
+
method,
|
|
163
|
+
headers: {
|
|
164
|
+
'Content-Type': contentType || 'application/json',
|
|
165
|
+
...(bodyStr ? { 'Content-Length': Buffer.byteLength(bodyStr).toString() } : {}),
|
|
166
|
+
},
|
|
167
|
+
}, (res) => {
|
|
168
|
+
let data = '';
|
|
169
|
+
res.on('data', (chunk) => (data += chunk));
|
|
170
|
+
res.on('end', () => {
|
|
171
|
+
if (res.statusCode && res.statusCode >= 400) {
|
|
172
|
+
reject(new Error(`ES ${method} ${path}: ${res.statusCode} ${data.slice(0, 200)}`));
|
|
173
|
+
} else {
|
|
174
|
+
resolve(data);
|
|
175
|
+
}
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
req.on('error', reject);
|
|
179
|
+
if (bodyStr) req.write(bodyStr);
|
|
180
|
+
req.end();
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { OllamaEmbeddingAdapter } from '../../src/embedding/ollama.js';
|
|
2
|
+
/**
|
|
3
|
+
* OpenClaw embedding adapter — wraps Ollama adapter with Beast PC defaults.
|
|
4
|
+
* Uses nomic-embed-text on Beast's GPU for fast embeddings.
|
|
5
|
+
*/
|
|
6
|
+
export declare class OpenClawEmbeddingAdapter extends OllamaEmbeddingAdapter {
|
|
7
|
+
constructor(ollamaUrl?: string, model?: string);
|
|
8
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.OpenClawEmbeddingAdapter = void 0;
|
|
4
|
+
const ollama_js_1 = require("../../src/embedding/ollama.js");
|
|
5
|
+
/**
|
|
6
|
+
* OpenClaw embedding adapter — wraps Ollama adapter with Beast PC defaults.
|
|
7
|
+
* Uses nomic-embed-text on Beast's GPU for fast embeddings.
|
|
8
|
+
*/
|
|
9
|
+
class OpenClawEmbeddingAdapter extends ollama_js_1.OllamaEmbeddingAdapter {
|
|
10
|
+
constructor(ollamaUrl = 'http://localhost:11434', model = 'nomic-embed-text') {
|
|
11
|
+
super(ollamaUrl, model);
|
|
12
|
+
this.name = 'openclaw-ollama';
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
exports.OpenClawEmbeddingAdapter = OpenClawEmbeddingAdapter;
|
|
16
|
+
//# sourceMappingURL=embedding.js.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* smartcontext-adapter-openclaw
|
|
3
|
+
*
|
|
4
|
+
* OpenClaw-specific adapter for SmartContext Proxy.
|
|
5
|
+
* Uses OpenSearch on Castle for storage and Beast Ollama for embeddings.
|
|
6
|
+
* Auto-discovers config from ~/.openclaw/
|
|
7
|
+
*/
|
|
8
|
+
export { OpenClawStorageAdapter } from './storage.js';
|
|
9
|
+
export { OpenClawEmbeddingAdapter } from './embedding.js';
|
|
10
|
+
export { SessionImporter } from './session-importer.js';
|
|
11
|
+
export interface OpenClawConfig {
|
|
12
|
+
ocHome: string;
|
|
13
|
+
esUrl: string;
|
|
14
|
+
ollamaUrl: string;
|
|
15
|
+
ollamaModel: string;
|
|
16
|
+
}
|
|
17
|
+
/** Auto-discover OpenClaw config from filesystem */
|
|
18
|
+
export declare function discoverConfig(ocHome?: string): OpenClawConfig;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* smartcontext-adapter-openclaw
|
|
4
|
+
*
|
|
5
|
+
* OpenClaw-specific adapter for SmartContext Proxy.
|
|
6
|
+
* Uses OpenSearch on Castle for storage and Beast Ollama for embeddings.
|
|
7
|
+
* Auto-discovers config from ~/.openclaw/
|
|
8
|
+
*/
|
|
9
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
10
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
11
|
+
};
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.SessionImporter = exports.OpenClawEmbeddingAdapter = exports.OpenClawStorageAdapter = void 0;
|
|
14
|
+
exports.discoverConfig = discoverConfig;
|
|
15
|
+
var storage_js_1 = require("./storage.js");
|
|
16
|
+
Object.defineProperty(exports, "OpenClawStorageAdapter", { enumerable: true, get: function () { return storage_js_1.OpenClawStorageAdapter; } });
|
|
17
|
+
var embedding_js_1 = require("./embedding.js");
|
|
18
|
+
Object.defineProperty(exports, "OpenClawEmbeddingAdapter", { enumerable: true, get: function () { return embedding_js_1.OpenClawEmbeddingAdapter; } });
|
|
19
|
+
var session_importer_js_1 = require("./session-importer.js");
|
|
20
|
+
Object.defineProperty(exports, "SessionImporter", { enumerable: true, get: function () { return session_importer_js_1.SessionImporter; } });
|
|
21
|
+
const node_fs_1 = __importDefault(require("node:fs"));
|
|
22
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
23
|
+
/** Auto-discover OpenClaw config from filesystem */
|
|
24
|
+
function discoverConfig(ocHome) {
|
|
25
|
+
const home = ocHome || node_path_1.default.join(process.env['HOME'] || '.', '.openclaw');
|
|
26
|
+
// Read openclaw.json for model and API settings
|
|
27
|
+
let esUrl = process.env['ES_URL'] || 'http://localhost:9201';
|
|
28
|
+
let ollamaUrl = 'http://localhost:11434';
|
|
29
|
+
let ollamaModel = 'nomic-embed-text';
|
|
30
|
+
const configPath = node_path_1.default.join(home, 'openclaw.json');
|
|
31
|
+
if (node_fs_1.default.existsSync(configPath)) {
|
|
32
|
+
try {
|
|
33
|
+
const config = JSON.parse(node_fs_1.default.readFileSync(configPath, 'utf-8'));
|
|
34
|
+
if (config.models?.providers?.ollama?.baseUrl) {
|
|
35
|
+
ollamaUrl = config.models.providers.ollama.baseUrl;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
catch { }
|
|
39
|
+
}
|
|
40
|
+
return { ocHome: home, esUrl, ollamaUrl, ollamaModel };
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { StorageAdapter } from '../../src/storage/types.js';
|
|
2
|
+
import type { EmbeddingAdapter } from '../../src/embedding/types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Import OC gateway session logs into SmartContext storage.
|
|
5
|
+
* Reads JSONL session logs, chunks them, embeds, and stores.
|
|
6
|
+
*/
|
|
7
|
+
export declare class SessionImporter {
|
|
8
|
+
private storage;
|
|
9
|
+
private embedding;
|
|
10
|
+
constructor(storage: StorageAdapter, embedding: EmbeddingAdapter);
|
|
11
|
+
/**
|
|
12
|
+
* Import sessions from OC gateway log directory.
|
|
13
|
+
* @param logDir Path to OC session logs (e.g., ~/.openclaw/logs/sessions/)
|
|
14
|
+
* @param onProgress Callback for progress reporting
|
|
15
|
+
*/
|
|
16
|
+
importFromDirectory(logDir: string, onProgress?: (msg: string) => void): Promise<{
|
|
17
|
+
imported: number;
|
|
18
|
+
chunks: number;
|
|
19
|
+
errors: number;
|
|
20
|
+
}>;
|
|
21
|
+
private parseSessionLog;
|
|
22
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.SessionImporter = void 0;
|
|
7
|
+
const node_fs_1 = __importDefault(require("node:fs"));
|
|
8
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
9
|
+
const chunker_js_1 = require("../../src/context/chunker.js");
|
|
10
|
+
/**
|
|
11
|
+
* Import OC gateway session logs into SmartContext storage.
|
|
12
|
+
* Reads JSONL session logs, chunks them, embeds, and stores.
|
|
13
|
+
*/
|
|
14
|
+
class SessionImporter {
|
|
15
|
+
storage;
|
|
16
|
+
embedding;
|
|
17
|
+
constructor(storage, embedding) {
|
|
18
|
+
this.storage = storage;
|
|
19
|
+
this.embedding = embedding;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Import sessions from OC gateway log directory.
|
|
23
|
+
* @param logDir Path to OC session logs (e.g., ~/.openclaw/logs/sessions/)
|
|
24
|
+
* @param onProgress Callback for progress reporting
|
|
25
|
+
*/
|
|
26
|
+
async importFromDirectory(logDir, onProgress) {
|
|
27
|
+
let imported = 0;
|
|
28
|
+
let totalChunks = 0;
|
|
29
|
+
let errors = 0;
|
|
30
|
+
if (!node_fs_1.default.existsSync(logDir)) {
|
|
31
|
+
onProgress?.(`Directory not found: ${logDir}`);
|
|
32
|
+
return { imported, chunks: totalChunks, errors };
|
|
33
|
+
}
|
|
34
|
+
const files = node_fs_1.default.readdirSync(logDir).filter((f) => f.endsWith('.jsonl') || f.endsWith('.json'));
|
|
35
|
+
onProgress?.(`Found ${files.length} session files`);
|
|
36
|
+
for (const file of files) {
|
|
37
|
+
try {
|
|
38
|
+
const filePath = node_path_1.default.join(logDir, file);
|
|
39
|
+
const sessionId = node_path_1.default.basename(file, node_path_1.default.extname(file));
|
|
40
|
+
const content = node_fs_1.default.readFileSync(filePath, 'utf-8');
|
|
41
|
+
const messages = this.parseSessionLog(content);
|
|
42
|
+
if (messages.length === 0)
|
|
43
|
+
continue;
|
|
44
|
+
const chunks = (0, chunker_js_1.chunkConversation)(messages, sessionId);
|
|
45
|
+
if (chunks.length === 0)
|
|
46
|
+
continue;
|
|
47
|
+
// Embed chunks in batches
|
|
48
|
+
const batchSize = 10;
|
|
49
|
+
for (let i = 0; i < chunks.length; i += batchSize) {
|
|
50
|
+
const batch = chunks.slice(i, i + batchSize);
|
|
51
|
+
const texts = batch.map((c) => c.text);
|
|
52
|
+
const embeddings = await this.embedding.embed(texts);
|
|
53
|
+
for (let j = 0; j < batch.length; j++) {
|
|
54
|
+
batch[j].embedding = embeddings[j];
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
await this.storage.upsertChunks(chunks);
|
|
58
|
+
imported++;
|
|
59
|
+
totalChunks += chunks.length;
|
|
60
|
+
onProgress?.(`Imported ${file}: ${chunks.length} chunks`);
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
errors++;
|
|
64
|
+
onProgress?.(`Error importing ${file}: ${err}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
onProgress?.(`Done: ${imported} sessions, ${totalChunks} chunks, ${errors} errors`);
|
|
68
|
+
return { imported, chunks: totalChunks, errors };
|
|
69
|
+
}
|
|
70
|
+
parseSessionLog(content) {
|
|
71
|
+
const messages = [];
|
|
72
|
+
for (const line of content.split('\n').filter(Boolean)) {
|
|
73
|
+
try {
|
|
74
|
+
const entry = JSON.parse(line);
|
|
75
|
+
// Support different OC log formats
|
|
76
|
+
if (entry.role && entry.content) {
|
|
77
|
+
messages.push({
|
|
78
|
+
role: entry.role,
|
|
79
|
+
content: typeof entry.content === 'string' ? entry.content : JSON.stringify(entry.content),
|
|
80
|
+
timestamp: entry.timestamp ? new Date(entry.timestamp).getTime() : undefined,
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
else if (entry.type === 'user_message' || entry.type === 'assistant_message') {
|
|
84
|
+
messages.push({
|
|
85
|
+
role: entry.type === 'user_message' ? 'user' : 'assistant',
|
|
86
|
+
content: entry.text || entry.content || '',
|
|
87
|
+
timestamp: entry.timestamp ? new Date(entry.timestamp).getTime() : undefined,
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
// Skip unparseable lines
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return messages;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
exports.SessionImporter = SessionImporter;
|
|
99
|
+
//# sourceMappingURL=session-importer.js.map
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { StorageAdapter, Chunk, ScoredChunk, SearchOptions, Exchange } from '../../src/storage/types.js';
|
|
2
|
+
/**
|
|
3
|
+
* OpenSearch storage adapter for SmartContext.
|
|
4
|
+
* Stores chunks and metrics in OpenSearch indices on Castle VPS.
|
|
5
|
+
*/
|
|
6
|
+
export declare class OpenClawStorageAdapter implements StorageAdapter {
|
|
7
|
+
name: string;
|
|
8
|
+
private baseUrl;
|
|
9
|
+
private chunksIndex;
|
|
10
|
+
private metricsIndex;
|
|
11
|
+
private logsIndex;
|
|
12
|
+
constructor(esUrl?: string);
|
|
13
|
+
initialize(): Promise<void>;
|
|
14
|
+
upsertChunks(chunks: Chunk[]): Promise<void>;
|
|
15
|
+
search(embedding: number[], options: SearchOptions): Promise<ScoredChunk[]>;
|
|
16
|
+
appendLog(sessionId: string, exchange: Exchange): Promise<void>;
|
|
17
|
+
getSessionLog(sessionId: string): Promise<Exchange[]>;
|
|
18
|
+
getStats(): Promise<{
|
|
19
|
+
chunks: number;
|
|
20
|
+
sessions: number;
|
|
21
|
+
diskBytes: number;
|
|
22
|
+
}>;
|
|
23
|
+
close(): Promise<void>;
|
|
24
|
+
private chunkMappings;
|
|
25
|
+
private esRequest;
|
|
26
|
+
}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.OpenClawStorageAdapter = void 0;
|
|
7
|
+
const node_http_1 = __importDefault(require("node:http"));
|
|
8
|
+
/**
|
|
9
|
+
* OpenSearch storage adapter for SmartContext.
|
|
10
|
+
* Stores chunks and metrics in OpenSearch indices on Castle VPS.
|
|
11
|
+
*/
|
|
12
|
+
class OpenClawStorageAdapter {
|
|
13
|
+
name = 'openclaw-opensearch';
|
|
14
|
+
baseUrl;
|
|
15
|
+
chunksIndex = 'smartcontext-chunks';
|
|
16
|
+
metricsIndex = 'smartcontext-metrics';
|
|
17
|
+
logsIndex = 'smartcontext-logs';
|
|
18
|
+
constructor(esUrl = 'http://localhost:9201') {
|
|
19
|
+
this.baseUrl = esUrl;
|
|
20
|
+
}
|
|
21
|
+
async initialize() {
|
|
22
|
+
// Create indices if they don't exist
|
|
23
|
+
for (const index of [this.chunksIndex, this.metricsIndex, this.logsIndex]) {
|
|
24
|
+
try {
|
|
25
|
+
await this.esRequest('PUT', `/${index}`, {
|
|
26
|
+
settings: { number_of_shards: 1, number_of_replicas: 0 },
|
|
27
|
+
mappings: index === this.chunksIndex ? this.chunkMappings() : undefined,
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
// Index may already exist
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
async upsertChunks(chunks) {
|
|
36
|
+
if (chunks.length === 0)
|
|
37
|
+
return;
|
|
38
|
+
const bulkBody = chunks.flatMap((c) => [
|
|
39
|
+
JSON.stringify({ index: { _index: this.chunksIndex, _id: c.id } }),
|
|
40
|
+
JSON.stringify({
|
|
41
|
+
text: c.text,
|
|
42
|
+
embedding: c.embedding,
|
|
43
|
+
sessionId: c.sessionId,
|
|
44
|
+
timestamp: c.timestamp,
|
|
45
|
+
summary: c.metadata.summary,
|
|
46
|
+
tokenCount: c.metadata.tokenCount,
|
|
47
|
+
exchangeIndex: c.metadata.exchangeIndex,
|
|
48
|
+
files: c.metadata.files,
|
|
49
|
+
tools: c.metadata.tools,
|
|
50
|
+
}),
|
|
51
|
+
]).join('\n') + '\n';
|
|
52
|
+
await this.esRequest('POST', '/_bulk', bulkBody, 'application/x-ndjson');
|
|
53
|
+
}
|
|
54
|
+
async search(embedding, options) {
|
|
55
|
+
const query = {
|
|
56
|
+
size: options.topK,
|
|
57
|
+
query: {
|
|
58
|
+
script_score: {
|
|
59
|
+
query: { match_all: {} },
|
|
60
|
+
script: {
|
|
61
|
+
source: "cosineSimilarity(params.query_vector, 'embedding') + 1.0",
|
|
62
|
+
params: { query_vector: embedding },
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
const result = await this.esRequest('POST', `/${this.chunksIndex}/_search`, query);
|
|
68
|
+
const parsed = JSON.parse(result);
|
|
69
|
+
return (parsed.hits?.hits || []).map((hit) => {
|
|
70
|
+
const source = hit._source;
|
|
71
|
+
let score = (hit._score - 1.0); // Remove the +1.0 offset
|
|
72
|
+
// Apply boosts
|
|
73
|
+
if (options.sessionBoost && source.sessionId === options.sessionBoost.sessionId) {
|
|
74
|
+
score += options.sessionBoost.boost;
|
|
75
|
+
}
|
|
76
|
+
if (options.fileBoost) {
|
|
77
|
+
const files = source.files || [];
|
|
78
|
+
if (options.fileBoost.patterns.some((p) => files.some((f) => f.includes(p)))) {
|
|
79
|
+
score += options.fileBoost.boost;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
id: hit._id,
|
|
84
|
+
text: source.text,
|
|
85
|
+
embedding: [],
|
|
86
|
+
sessionId: source.sessionId,
|
|
87
|
+
timestamp: source.timestamp,
|
|
88
|
+
score,
|
|
89
|
+
metadata: {
|
|
90
|
+
summary: source.summary,
|
|
91
|
+
tokenCount: source.tokenCount,
|
|
92
|
+
exchangeIndex: source.exchangeIndex,
|
|
93
|
+
files: source.files,
|
|
94
|
+
tools: source.tools,
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
}).filter((c) => c.score >= options.minScore);
|
|
98
|
+
}
|
|
99
|
+
async appendLog(sessionId, exchange) {
|
|
100
|
+
await this.esRequest('POST', `/${this.logsIndex}/_doc`, {
|
|
101
|
+
sessionId,
|
|
102
|
+
...exchange,
|
|
103
|
+
indexedAt: new Date().toISOString(),
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
async getSessionLog(sessionId) {
|
|
107
|
+
const result = await this.esRequest('POST', `/${this.logsIndex}/_search`, {
|
|
108
|
+
size: 1000,
|
|
109
|
+
query: { term: { sessionId } },
|
|
110
|
+
sort: [{ timestamp: 'asc' }],
|
|
111
|
+
});
|
|
112
|
+
const parsed = JSON.parse(result);
|
|
113
|
+
return (parsed.hits?.hits || []).map((h) => h._source);
|
|
114
|
+
}
|
|
115
|
+
async getStats() {
|
|
116
|
+
try {
|
|
117
|
+
const result = await this.esRequest('GET', `/${this.chunksIndex}/_count`);
|
|
118
|
+
const parsed = JSON.parse(result);
|
|
119
|
+
return {
|
|
120
|
+
chunks: parsed.count || 0,
|
|
121
|
+
sessions: 0, // Would need aggregation
|
|
122
|
+
diskBytes: 0,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return { chunks: 0, sessions: 0, diskBytes: 0 };
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
async close() {
|
|
130
|
+
// No cleanup needed for HTTP client
|
|
131
|
+
}
|
|
132
|
+
chunkMappings() {
|
|
133
|
+
return {
|
|
134
|
+
properties: {
|
|
135
|
+
text: { type: 'text' },
|
|
136
|
+
embedding: { type: 'dense_vector', dims: 768 },
|
|
137
|
+
sessionId: { type: 'keyword' },
|
|
138
|
+
timestamp: { type: 'long' },
|
|
139
|
+
summary: { type: 'text' },
|
|
140
|
+
tokenCount: { type: 'integer' },
|
|
141
|
+
exchangeIndex: { type: 'integer' },
|
|
142
|
+
files: { type: 'keyword' },
|
|
143
|
+
tools: { type: 'keyword' },
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
esRequest(method, path, body, contentType) {
|
|
148
|
+
return new Promise((resolve, reject) => {
|
|
149
|
+
const url = new URL(`${this.baseUrl}${path}`);
|
|
150
|
+
const bodyStr = typeof body === 'string' ? body : body ? JSON.stringify(body) : undefined;
|
|
151
|
+
const req = node_http_1.default.request(url, {
|
|
152
|
+
method,
|
|
153
|
+
headers: {
|
|
154
|
+
'Content-Type': contentType || 'application/json',
|
|
155
|
+
...(bodyStr ? { 'Content-Length': Buffer.byteLength(bodyStr).toString() } : {}),
|
|
156
|
+
},
|
|
157
|
+
}, (res) => {
|
|
158
|
+
let data = '';
|
|
159
|
+
res.on('data', (chunk) => (data += chunk));
|
|
160
|
+
res.on('end', () => {
|
|
161
|
+
if (res.statusCode && res.statusCode >= 400) {
|
|
162
|
+
reject(new Error(`ES ${method} ${path}: ${res.statusCode} ${data.slice(0, 200)}`));
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
resolve(data);
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
});
|
|
169
|
+
req.on('error', reject);
|
|
170
|
+
if (bodyStr)
|
|
171
|
+
req.write(bodyStr);
|
|
172
|
+
req.end();
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
exports.OpenClawStorageAdapter = OpenClawStorageAdapter;
|
|
177
|
+
//# sourceMappingURL=storage.js.map
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.detectProviders = detectProviders;
|
|
4
|
+
exports.buildConfig = buildConfig;
|
|
5
|
+
const defaults_js_1 = require("./defaults.js");
|
|
6
|
+
const PROVIDER_ENV_MAP = {
|
|
7
|
+
anthropic: {
|
|
8
|
+
envKey: 'ANTHROPIC_API_KEY',
|
|
9
|
+
baseUrl: 'https://api.anthropic.com',
|
|
10
|
+
},
|
|
11
|
+
openai: {
|
|
12
|
+
envKey: 'OPENAI_API_KEY',
|
|
13
|
+
baseUrl: 'https://api.openai.com',
|
|
14
|
+
},
|
|
15
|
+
google: {
|
|
16
|
+
envKey: 'GOOGLE_API_KEY',
|
|
17
|
+
baseUrl: 'https://generativelanguage.googleapis.com',
|
|
18
|
+
},
|
|
19
|
+
openrouter: {
|
|
20
|
+
envKey: 'OPENROUTER_API_KEY',
|
|
21
|
+
baseUrl: 'https://openrouter.ai/api',
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
function detectProviders() {
|
|
25
|
+
const providers = {};
|
|
26
|
+
for (const [name, { envKey, baseUrl }] of Object.entries(PROVIDER_ENV_MAP)) {
|
|
27
|
+
const apiKey = process.env[envKey];
|
|
28
|
+
if (apiKey) {
|
|
29
|
+
providers[name] = { apiKey, baseUrl };
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
// Ollama: detect by host, no API key needed
|
|
33
|
+
const ollamaHost = process.env['OLLAMA_HOST'] || 'http://localhost:11434';
|
|
34
|
+
providers['ollama'] = { apiKey: '', baseUrl: ollamaHost };
|
|
35
|
+
return providers;
|
|
36
|
+
}
|
|
37
|
+
function buildConfig(overrides) {
|
|
38
|
+
const providers = detectProviders();
|
|
39
|
+
return {
|
|
40
|
+
...defaults_js_1.DEFAULT_CONFIG,
|
|
41
|
+
providers,
|
|
42
|
+
...overrides,
|
|
43
|
+
proxy: { ...defaults_js_1.DEFAULT_CONFIG.proxy, ...overrides?.proxy },
|
|
44
|
+
context: { ...defaults_js_1.DEFAULT_CONFIG.context, ...overrides?.context },
|
|
45
|
+
logging: { ...defaults_js_1.DEFAULT_CONFIG.logging, ...overrides?.logging },
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=auto-detect.js.map
|