symmetry-cli 1.0.7 → 1.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/provider.ts DELETED
@@ -1,411 +0,0 @@
1
- import { PassThrough, Readable } from "node:stream";
2
- import { pipeline } from "stream/promises";
3
- import chalk from "chalk";
4
- import Hyperswarm from "hyperswarm";
5
- import crypto from "hypercore-crypto";
6
- import fs from "node:fs";
7
-
8
- import { ConfigManager } from "./config";
9
- import {
10
- createMessage,
11
- getChatDataFromProvider,
12
- safeParseJson,
13
- safeParseStreamResponse,
14
- } from "./utils";
15
- import { logger } from "./logger";
16
- import { Peer, ProviderMessage, InferenceRequest, Message } from "./types";
17
- import { PROVIDER_HELLO_TIMEOUT, serverMessageKeys } from "./constants";
18
-
19
- export class SymmetryProvider {
20
- private _challenge: Buffer | null = null;
21
- private _config: ConfigManager;
22
- private _conversationIndex = 0;
23
- private _discoveryKey: Buffer | null = null;
24
- private _isPublic = false;
25
- private _providerConnections: number = 0;
26
- private _providerSwarm: Hyperswarm | null = null;
27
- private _serverSwarm: Hyperswarm | null = null;
28
- private _serverPeer: Peer | null = null;
29
-
30
- constructor(configPath: string) {
31
- logger.info(`🔗 Initializing client using config file: ${configPath}`);
32
- this._config = new ConfigManager(configPath);
33
- this._isPublic = this._config.get("public");
34
- }
35
-
36
- async init(): Promise<void> {
37
- this._providerSwarm = new Hyperswarm({
38
- maxConnections: this._config.get("maxConnections"),
39
- });
40
- const keyPair = crypto.keyPair(
41
- Buffer.alloc(32).fill(this._config.get("name"))
42
- );
43
- this._discoveryKey = crypto.discoveryKey(keyPair.publicKey);
44
- const discovery = this._providerSwarm.join(this._discoveryKey, {
45
- server: true,
46
- client: true,
47
- });
48
- await discovery.flushed();
49
-
50
- this._providerSwarm.on("error", (err: Error) => {
51
- logger.error(chalk.red("🚨 Swarm Error:"), err);
52
- });
53
-
54
- this._providerSwarm.on("connection", (peer: Peer) => {
55
- logger.info(`⚡️ New connection from peer: ${peer.rawStream.remoteHost}`);
56
- this.listeners(peer);
57
- });
58
-
59
- logger.info(`📁 Symmetry client initialized.`);
60
- logger.info(`🔑 Discovery key: ${this._discoveryKey.toString("hex")}`);
61
-
62
- if (this._isPublic) {
63
- logger.info(
64
- chalk.white(`🔑 Server key: ${this._config.get("serverKey")}`)
65
- );
66
- logger.info(chalk.white("🔗 Joining server, please wait."));
67
- this.joinServer();
68
- }
69
-
70
- process.on("SIGINT", async () => {
71
- await this._providerSwarm?.destroy();
72
- process.exit(0);
73
- });
74
-
75
- process.on("uncaughtException", (err) => {
76
- if (err.message === "connection reset by peer") {
77
- this._providerConnections = Math.max(0, this._providerConnections - 1);
78
- }
79
- });
80
- }
81
-
82
- async destroySwarms() {
83
- await this._providerSwarm?.destroy();
84
- await this._serverSwarm?.destroy();
85
- }
86
-
87
- private async testProviderCall(): Promise<void> {
88
- const testCall = async () => {
89
- logger.info(chalk.white(`👋 Saying hello to your provider...`));
90
- const testMessages: Message[] = [
91
- { role: "user", content: "Hello, this is a test message." },
92
- ];
93
- const req = this.buildStreamRequest(testMessages);
94
-
95
- if (!req) {
96
- logger.error(chalk.red("❌ Failed to build test request"));
97
- throw new Error("Failed to build test request");
98
- }
99
-
100
- const { requestOptions, requestBody } = req;
101
- const { protocol, hostname, port, path, method, headers } =
102
- requestOptions;
103
- const url = `${protocol}://${hostname}:${port}${path}`;
104
-
105
- logger.info(chalk.white(`🚀 Sending test request to ${url}`));
106
-
107
- try {
108
- const response = await fetch(url, {
109
- method,
110
- headers,
111
- body: JSON.stringify(requestBody),
112
- });
113
-
114
- if (!response.ok) {
115
- logger.error(
116
- chalk.red(
117
- `❌ Server responded with status code: ${response.status}`
118
- )
119
- );
120
- this.destroySwarms();
121
- throw new Error(
122
- `Server responded with status code: ${response.status}`
123
- );
124
- }
125
-
126
- if (!response.body) {
127
- logger.error(
128
- chalk.red("❌ Failed to get a ReadableStream from the response")
129
- );
130
- this.destroySwarms();
131
- throw new Error("Failed to get a ReadableStream from the response");
132
- }
133
-
134
- logger.info(chalk.white(`📡 Got response, checking stream...`));
135
-
136
- const reader = response.body.getReader();
137
- const { done } = await reader.read();
138
- if (done) {
139
- logger.error(chalk.red("❌ Stream ended without data"));
140
- this.destroySwarms();
141
- throw new Error("Stream ended without data");
142
- }
143
-
144
- logger.info(chalk.green(`✅ Test inference call successful!`));
145
- } catch (error) {
146
- this.destroySwarms();
147
- logger.error(
148
- chalk.red(`❌ Error during test inference call: ${error}`)
149
- );
150
- throw error;
151
- }
152
-
153
- logger.info(chalk.white(`🔗 Test call successful!`));
154
- };
155
-
156
- setTimeout(() => testCall(), PROVIDER_HELLO_TIMEOUT)
157
- }
158
-
159
- async joinServer(): Promise<void> {
160
- this._serverSwarm = new Hyperswarm();
161
- const serverKey = Buffer.from(this._config.get("serverKey"));
162
- this._serverSwarm.join(crypto.discoveryKey(serverKey), {
163
- client: true,
164
- server: false,
165
- });
166
- this._serverSwarm.flush();
167
- this._serverSwarm.on("connection", (peer: Peer) => {
168
- this._serverPeer = peer;
169
- logger.info(chalk.green("🔗 Connected to server."));
170
-
171
- this.testProviderCall();
172
-
173
- this._challenge = crypto.randomBytes(32);
174
-
175
- this._serverPeer.write(
176
- createMessage(serverMessageKeys.challenge, {
177
- challenge: this._challenge,
178
- })
179
- );
180
-
181
- this._serverPeer.write(
182
- createMessage(serverMessageKeys.join, {
183
- ...this._config.getAll(),
184
- discoveryKey: this._discoveryKey?.toString("hex"),
185
- })
186
- );
187
-
188
- this._serverPeer.on("data", async (buffer: Buffer) => {
189
- if (!buffer) return;
190
-
191
- const data = safeParseJson<
192
- ProviderMessage<{ message: string; signature: { data: string } }>
193
- >(buffer.toString());
194
-
195
- if (data && data.key) {
196
- switch (data.key) {
197
- case serverMessageKeys.challenge:
198
- this.handleServerVerification(
199
- data.data as { message: string; signature: { data: string } }
200
- );
201
- break;
202
- case serverMessageKeys.ping:
203
- this._serverPeer?.write(createMessage(serverMessageKeys.pong));
204
- break;
205
- }
206
- }
207
- });
208
- });
209
- }
210
-
211
- getServerPublicKey(serverKeyHex: string): Buffer {
212
- const publicKey = Buffer.from(serverKeyHex, "hex");
213
- if (publicKey.length !== 32) {
214
- throw new Error(
215
- `Expected a 32-byte public key, but got ${publicKey.length} bytes`
216
- );
217
- }
218
- return publicKey;
219
- }
220
-
221
- handleServerVerification(data: {
222
- message: string;
223
- signature: { data: string };
224
- }) {
225
- if (!this._challenge) {
226
- console.log("No challenge set. Cannot verify.");
227
- return;
228
- }
229
-
230
- const serverKeyHex = this._config.get("serverKey");
231
- try {
232
- const publicKey = this.getServerPublicKey(serverKeyHex);
233
- const signatureBuffer = Buffer.from(data.signature.data, "base64");
234
-
235
- const verified = crypto.verify(
236
- this._challenge,
237
- signatureBuffer,
238
- publicKey
239
- );
240
-
241
- if (verified) {
242
- logger.info(chalk.greenBright(`✅ Verification successful.`));
243
- } else {
244
- logger.error(`❌ Verification failed!`);
245
- }
246
- } catch (error) {
247
- console.error("Error during verification:", error);
248
- }
249
- }
250
-
251
- private listeners(peer: Peer): void {
252
- peer.on("data", async (buffer: Buffer) => {
253
- if (!buffer) return;
254
- const data = safeParseJson<ProviderMessage<InferenceRequest>>(
255
- buffer.toString()
256
- );
257
- if (data && data.key) {
258
- switch (data.key) {
259
- case serverMessageKeys.newConversation:
260
- this._conversationIndex = this._conversationIndex + 1;
261
- break;
262
- case serverMessageKeys.inference:
263
- logger.info(
264
- `📦 Inference message received from ${peer.rawStream.remoteHost}`
265
- );
266
- await this.handleInferenceRequest(data, peer);
267
- break;
268
- }
269
- }
270
- });
271
- }
272
-
273
- private getMessagesWithSystem(messages: Message[]): Message[] {
274
- const systemMessage = this._config.get("systemMessage");
275
- if (messages.length === 2 && systemMessage) {
276
- messages.unshift({
277
- role: "system",
278
- content: systemMessage,
279
- });
280
- }
281
- return messages;
282
- }
283
-
284
- private async handleInferenceRequest(
285
- data: ProviderMessage<InferenceRequest>,
286
- peer: Peer
287
- ): Promise<void> {
288
- const emitterKey = data.data.key;
289
- const messages = this.getMessagesWithSystem(data?.data.messages);
290
- const req = this.buildStreamRequest(messages);
291
-
292
- if (!req) return;
293
-
294
- const { requestOptions, requestBody } = req;
295
- const { protocol, hostname, port, path, method, headers } = requestOptions;
296
- const url = `${protocol}://${hostname}:${port}${path}`;
297
-
298
- try {
299
- const response = await fetch(url, {
300
- method,
301
- headers,
302
- body: JSON.stringify(requestBody),
303
- });
304
-
305
- if (!response.ok) {
306
- throw new Error(
307
- `Server responded with status code: ${response.status}`
308
- );
309
- }
310
-
311
- if (!response.body) {
312
- throw new Error("Failed to get a ReadableStream from the response");
313
- }
314
-
315
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
316
- const responseStream = Readable.fromWeb(response.body as any);
317
- const peerStream = new PassThrough();
318
- responseStream.pipe(peerStream);
319
- let completion = "";
320
-
321
- const provider = this._config.get("apiProvider");
322
-
323
- peer.write(
324
- JSON.stringify({
325
- symmetryEmitterKey: emitterKey,
326
- })
327
- );
328
-
329
- const peerPipeline = pipeline(peerStream, async function (source) {
330
- for await (const chunk of source) {
331
- if (peer.writable) {
332
- completion += getChatDataFromProvider(
333
- provider,
334
- safeParseStreamResponse(chunk.toString())
335
- );
336
-
337
- const write = peer.write(chunk);
338
-
339
- if (!write) {
340
- await new Promise((resolve) => peer.once("drain", resolve));
341
- }
342
- } else {
343
- break;
344
- }
345
- }
346
- });
347
- await Promise.resolve(peerPipeline);
348
-
349
- peer.write(
350
- createMessage(serverMessageKeys.inferenceEnded, data?.data.key)
351
- );
352
-
353
- if (
354
- this._config.get("dataCollectionEnabled") &&
355
- data.data.key === serverMessageKeys.inference
356
- ) {
357
- this.saveCompletion(completion, peer, data.data.messages);
358
- }
359
- } catch (error) {
360
- let errorMessage = "An error occurred during inference";
361
- if (error instanceof Error) errorMessage = error.message;
362
- logger.error(`🚨 ${errorMessage}`);
363
- }
364
- }
365
-
366
- private async saveCompletion(
367
- completion: string,
368
- peer: Peer,
369
- messages: Message[]
370
- ) {
371
- fs.writeFile(
372
- `${this._config.get("path")}/${peer.publicKey.toString("hex")}-${
373
- this._conversationIndex
374
- }.json`,
375
- JSON.stringify([
376
- ...messages,
377
- {
378
- role: "assistant",
379
- content: completion,
380
- },
381
- ]),
382
- () => {
383
- logger.info(`📝 Completion saved to file`);
384
- }
385
- );
386
- }
387
-
388
- private buildStreamRequest(messages: Message[]) {
389
- const requestOptions = {
390
- hostname: this._config.get("apiHostname"),
391
- port: Number(this._config.get("apiPort")),
392
- path: this._config.get("apiPath"),
393
- protocol: this._config.get("apiProtocol"),
394
- method: "POST",
395
- headers: {
396
- "Content-Type": "application/json",
397
- Authorization: `Bearer ${this._config.get("apiKey")}`,
398
- },
399
- };
400
-
401
- const requestBody = {
402
- model: this._config.get("modelName"),
403
- messages: messages || undefined,
404
- stream: true,
405
- };
406
-
407
- return { requestOptions, requestBody };
408
- }
409
- }
410
-
411
- export default SymmetryProvider;
package/src/types.ts DELETED
@@ -1,243 +0,0 @@
1
- /* eslint-disable @typescript-eslint/no-explicit-any */
2
- import { serverMessageKeys } from "./constants";
3
-
4
- export interface ProviderConfig {
5
- apiHostname: string;
6
- dataCollectionEnabled: boolean;
7
- apiKey?: string;
8
- apiPath: string;
9
- apiPort: number;
10
- apiProtocol: string;
11
- apiProvider: string;
12
- discoveryKey: string;
13
- key: string;
14
- maxConnections: number;
15
- modelName: string;
16
- name: string;
17
- path: string;
18
- port: number;
19
- public: boolean;
20
- serverKey: string;
21
- systemMessage: string;
22
- }
23
-
24
- export interface ProviderMessage<T = unknown> {
25
- key: string;
26
- data: T;
27
- }
28
-
29
- export interface InferenceRequest {
30
- key: string;
31
- messages: Message[];
32
- }
33
-
34
- interface ReadableState {
35
- highWaterMark: number;
36
- buffer: any;
37
- length: number;
38
- pipes: any[];
39
- flowing: boolean | null;
40
- ended: boolean;
41
- endEmitted: boolean;
42
- reading: boolean;
43
- sync: boolean;
44
- needReadable: boolean;
45
- emittedReadable: boolean;
46
- readableListening: boolean;
47
- resumeScheduled: boolean;
48
- paused: boolean;
49
- emitClose: boolean;
50
- autoDestroy: boolean;
51
- destroyed: boolean;
52
- closed: boolean;
53
- closeEmitted: boolean;
54
- defaultEncoding: string;
55
- awaitDrainWriters: any;
56
- multiAwaitDrain: boolean;
57
- readingMore: boolean;
58
- decoder: null | any;
59
- encoding: null | string;
60
- }
61
-
62
- interface WritableState {
63
- highWaterMark: number;
64
- objectMode: boolean;
65
- finalCalled: boolean;
66
- needDrain: boolean;
67
- ending: boolean;
68
- ended: boolean;
69
- finished: boolean;
70
- destroyed: boolean;
71
- decodeStrings: boolean;
72
- defaultEncoding: string;
73
- length: number;
74
- writing: boolean;
75
- corked: number;
76
- sync: boolean;
77
- bufferProcessing: boolean;
78
- writecb: () => void;
79
- writelen: number;
80
- afterWriteTickInfo: null | any;
81
- bufferedRequest: null | any;
82
- lastBufferedRequest: null | any;
83
- pendingcb: number;
84
- prefinished: boolean;
85
- errorEmitted: boolean;
86
- emitClose: boolean;
87
- autoDestroy: boolean;
88
- bufferedRequestCount: number;
89
- corkedRequestsFree: any;
90
- }
91
-
92
- interface UDXStream {
93
- udx: UDX;
94
- socket: UDXSocket;
95
- id: number;
96
- remoteId: number;
97
- remoteHost: string;
98
- remoteFamily: number;
99
- remotePort: number;
100
- userData: any;
101
- }
102
-
103
- interface UDX {
104
- _handle: Buffer;
105
- _watchers: Set<any>;
106
- _buffer: Buffer;
107
- }
108
-
109
- interface UDXSocket {
110
- udx: UDX;
111
- _handle: Buffer;
112
- _inited: boolean;
113
- _host: string;
114
- _family: number;
115
- _ipv6Only: boolean;
116
- _port: number;
117
- _reqs: any[];
118
- _free: any[];
119
- _closing: null | any;
120
- _closed: boolean;
121
- streams: Set<any>;
122
- userData: any;
123
- }
124
-
125
- export interface Peer {
126
- publicKey: Buffer;
127
- remotePublicKey: Buffer;
128
- handshakeHash: Buffer;
129
- write: (value: string) => boolean;
130
- on: (event: string, listener: (...args: any[]) => void) => this;
131
- once: (event: string, listener: (...args: any[]) => void) => this;
132
- writable: boolean;
133
- key: string;
134
- discovery_key: string;
135
-
136
- _duplexState: number;
137
- _readableState: ReadableState;
138
- _writableState: WritableState;
139
-
140
- noiseStream: Peer;
141
- isInitiator: boolean;
142
- rawStream: UDXStream;
143
-
144
- connected: boolean;
145
- keepAlive: number;
146
- timeout: number;
147
- userData: any;
148
- opened: Promise<void>;
149
- rawBytesWritten: number;
150
- rawBytesRead: number;
151
- relay: null | any;
152
- puncher: null | any;
153
- _rawStream: UDXStream;
154
- _handshake: null | any;
155
- _handshakePattern: null | any;
156
- _handshakeDone: null | any;
157
- _state: number;
158
- _len: number;
159
- _tmp: number;
160
- _message: null | any;
161
- _openedDone: () => void;
162
- _startDone: () => void;
163
- _drainDone: () => void;
164
- _outgoingPlain: null | any;
165
- _outgoingWrapped: null | any;
166
- _utp: null | any;
167
- _setup: boolean;
168
- _ended: number;
169
- _encrypt: {
170
- key: Buffer;
171
- state: Buffer;
172
- header: Buffer;
173
- };
174
- _decrypt: {
175
- key: Buffer;
176
- state: Buffer;
177
- final: boolean;
178
- };
179
- _timeoutTimer: null | NodeJS.Timeout;
180
- _keepAliveTimer: null | NodeJS.Timeout;
181
- }
182
-
183
- export interface Session {
184
- id: string;
185
- providerId: string;
186
- createdAt: Date;
187
- expiresAt: Date;
188
- }
189
-
190
- export interface PeerSessionRequest {
191
- modelName: string;
192
- preferredProviderId?: string;
193
- }
194
-
195
- export interface PeerWithSession extends Session {
196
- peer_key: string | null;
197
- discovery_key: string | null;
198
- model_name: string | null;
199
- }
200
-
201
- export interface PeerUpsert {
202
- key: string;
203
- discoveryKey: string;
204
- config: {
205
- modelName?: string;
206
- public?: boolean;
207
- serverKey?: string;
208
- };
209
- }
210
-
211
- export interface Message {
212
- role: string;
213
- content: string | undefined;
214
- }
215
-
216
- export type ServerMessageKey = keyof typeof serverMessageKeys;
217
-
218
- export interface StreamResponse {
219
- model: string
220
- created_at: string
221
- response: string
222
- content: string
223
- message: {
224
- content: string
225
- }
226
- done: boolean
227
- context: number[]
228
- total_duration: number
229
- load_duration: number
230
- prompt_eval_count: number
231
- prompt_eval_duration: number
232
- eval_count: number
233
- eval_duration: number
234
- type? : string
235
- choices: [
236
- {
237
- text: string
238
- delta: {
239
- content: string
240
- }
241
- }
242
- ]
243
- }
package/src/utils.ts DELETED
@@ -1,52 +0,0 @@
1
- import { apiProviders } from "./constants";
2
- import { ServerMessageKey, StreamResponse } from "./types";
3
-
4
- export function safeParseJson<T>(data: string): T | undefined {
5
- try {
6
- return JSON.parse(data) as T;
7
- } catch (e) {
8
- return undefined;
9
- }
10
- }
11
-
12
- export function createMessage<T>(key: ServerMessageKey, data?: T): string {
13
- return JSON.stringify({ key, data });
14
- }
15
-
16
- export function isStreamWithDataPrefix(stringBuffer: string) {
17
- return stringBuffer.startsWith('data:')
18
- }
19
-
20
- export function safeParseStreamResponse(
21
- stringBuffer: string
22
- ): StreamResponse | undefined {
23
- try {
24
- if (isStreamWithDataPrefix(stringBuffer)) {
25
- return JSON.parse(stringBuffer.split('data:')[1])
26
- }
27
- return JSON.parse(stringBuffer)
28
- } catch (e) {
29
- return undefined
30
- }
31
- }
32
-
33
- export const getChatDataFromProvider = (
34
- provider: string,
35
- data: StreamResponse | undefined
36
- ) => {
37
- switch (provider) {
38
- case apiProviders.Ollama:
39
- case apiProviders.OpenWebUI:
40
- return data?.choices[0].delta?.content
41
- ? data?.choices[0].delta.content
42
- : ''
43
- case apiProviders.LlamaCpp:
44
- return data?.content
45
- case apiProviders.LiteLLM:
46
- default:
47
- if (data?.choices[0].delta.content === 'undefined') return ''
48
- return data?.choices[0].delta?.content
49
- ? data?.choices[0].delta.content
50
- : ''
51
- }
52
- }