recker 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/ai/adaptive-timeout.d.ts +51 -0
- package/dist/ai/adaptive-timeout.d.ts.map +1 -0
- package/dist/ai/adaptive-timeout.js +208 -0
- package/dist/ai/client.d.ts +24 -0
- package/dist/ai/client.d.ts.map +1 -0
- package/dist/ai/client.js +289 -0
- package/dist/ai/index.d.ts +10 -0
- package/dist/ai/index.d.ts.map +1 -0
- package/dist/ai/index.js +6 -0
- package/dist/ai/providers/anthropic.d.ts +64 -0
- package/dist/ai/providers/anthropic.d.ts.map +1 -0
- package/dist/ai/providers/anthropic.js +367 -0
- package/dist/ai/providers/base.d.ts +49 -0
- package/dist/ai/providers/base.d.ts.map +1 -0
- package/dist/ai/providers/base.js +145 -0
- package/dist/ai/providers/index.d.ts +7 -0
- package/dist/ai/providers/index.d.ts.map +1 -0
- package/dist/ai/providers/index.js +3 -0
- package/dist/ai/providers/openai.d.ts +65 -0
- package/dist/ai/providers/openai.d.ts.map +1 -0
- package/dist/ai/providers/openai.js +298 -0
- package/dist/ai/rate-limiter.d.ts +44 -0
- package/dist/ai/rate-limiter.d.ts.map +1 -0
- package/dist/ai/rate-limiter.js +212 -0
- package/dist/bench/generator.d.ts +19 -0
- package/dist/bench/generator.d.ts.map +1 -0
- package/dist/bench/generator.js +86 -0
- package/dist/bench/stats.d.ts +35 -0
- package/dist/bench/stats.d.ts.map +1 -0
- package/dist/bench/stats.js +60 -0
- package/dist/cli/handler.d.ts +11 -0
- package/dist/cli/handler.d.ts.map +1 -0
- package/dist/cli/handler.js +92 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +255 -0
- package/dist/cli/presets.d.ts +2 -0
- package/dist/cli/presets.d.ts.map +1 -0
- package/dist/cli/presets.js +67 -0
- package/dist/cli/tui/ai-chat.d.ts +3 -0
- package/dist/cli/tui/ai-chat.d.ts.map +1 -0
- package/dist/cli/tui/ai-chat.js +100 -0
- package/dist/cli/tui/load-dashboard.d.ts +3 -0
- package/dist/cli/tui/load-dashboard.d.ts.map +1 -0
- package/dist/cli/tui/load-dashboard.js +117 -0
- package/dist/cli/tui/shell.d.ts +27 -0
- package/dist/cli/tui/shell.d.ts.map +1 -0
- package/dist/cli/tui/shell.js +386 -0
- package/dist/cli/tui/websocket.d.ts +2 -0
- package/dist/cli/tui/websocket.d.ts.map +1 -0
- package/dist/cli/tui/websocket.js +87 -0
- package/dist/contract/index.d.ts +2 -2
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/core/client.d.ts +1 -0
- package/dist/core/client.d.ts.map +1 -1
- package/dist/core/client.js +4 -2
- package/dist/core/request-promise.d.ts +1 -1
- package/dist/core/request-promise.d.ts.map +1 -1
- package/dist/mcp/contract.d.ts +1 -1
- package/dist/mcp/contract.d.ts.map +1 -1
- package/dist/protocols/ftp.d.ts +28 -5
- package/dist/protocols/ftp.d.ts.map +1 -1
- package/dist/protocols/ftp.js +549 -136
- package/dist/protocols/sftp.d.ts +4 -2
- package/dist/protocols/sftp.d.ts.map +1 -1
- package/dist/protocols/sftp.js +16 -2
- package/dist/protocols/telnet.d.ts +37 -5
- package/dist/protocols/telnet.d.ts.map +1 -1
- package/dist/protocols/telnet.js +434 -58
- package/dist/scrape/document.d.ts.map +1 -1
- package/dist/scrape/document.js +7 -12
- package/dist/testing/index.d.ts +2 -0
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +1 -0
- package/dist/testing/mock-udp-server.d.ts +44 -0
- package/dist/testing/mock-udp-server.d.ts.map +1 -0
- package/dist/testing/mock-udp-server.js +188 -0
- package/dist/transport/base-udp.d.ts +36 -0
- package/dist/transport/base-udp.d.ts.map +1 -0
- package/dist/transport/base-udp.js +188 -0
- package/dist/transport/udp-response.d.ts +65 -0
- package/dist/transport/udp-response.d.ts.map +1 -0
- package/dist/transport/udp-response.js +269 -0
- package/dist/transport/udp.d.ts +22 -0
- package/dist/transport/udp.d.ts.map +1 -0
- package/dist/transport/udp.js +260 -0
- package/dist/types/ai.d.ts +268 -0
- package/dist/types/ai.d.ts.map +1 -0
- package/dist/types/ai.js +1 -0
- package/dist/types/udp.d.ts +138 -0
- package/dist/types/udp.d.ts.map +1 -0
- package/dist/types/udp.js +1 -0
- package/dist/udp/index.d.ts +6 -0
- package/dist/udp/index.d.ts.map +1 -0
- package/dist/udp/index.js +3 -0
- package/dist/utils/chart.d.ts +15 -0
- package/dist/utils/chart.d.ts.map +1 -0
- package/dist/utils/chart.js +94 -0
- package/dist/utils/colors.d.ts +27 -0
- package/dist/utils/colors.d.ts.map +1 -0
- package/dist/utils/colors.js +50 -0
- package/dist/utils/optional-require.d.ts +20 -0
- package/dist/utils/optional-require.d.ts.map +1 -0
- package/dist/utils/optional-require.js +105 -0
- package/package.json +53 -12
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
import { BaseAIProvider, AIError, RateLimitError, ContextLengthError, OverloadedError, AuthenticationError, } from './base.js';
|
|
2
|
+
export class OpenAIProvider extends BaseAIProvider {
|
|
3
|
+
openaiConfig;
|
|
4
|
+
constructor(config = {}) {
|
|
5
|
+
super({ ...config, name: 'openai' });
|
|
6
|
+
this.openaiConfig = config;
|
|
7
|
+
}
|
|
8
|
+
getEnvApiKey() {
|
|
9
|
+
return process.env.OPENAI_API_KEY;
|
|
10
|
+
}
|
|
11
|
+
getBaseUrl() {
|
|
12
|
+
return this.config.baseUrl || 'https://api.openai.com/v1';
|
|
13
|
+
}
|
|
14
|
+
buildHeaders() {
|
|
15
|
+
const headers = {
|
|
16
|
+
'Content-Type': 'application/json',
|
|
17
|
+
'Authorization': `Bearer ${this.getApiKey()}`,
|
|
18
|
+
...this.config.headers,
|
|
19
|
+
};
|
|
20
|
+
if (this.openaiConfig.organization) {
|
|
21
|
+
headers['OpenAI-Organization'] = this.openaiConfig.organization;
|
|
22
|
+
}
|
|
23
|
+
return headers;
|
|
24
|
+
}
|
|
25
|
+
transformMessages(messages) {
|
|
26
|
+
return messages.map((msg) => {
|
|
27
|
+
const openaiMsg = {
|
|
28
|
+
role: msg.role,
|
|
29
|
+
content: this.transformContent(msg.content),
|
|
30
|
+
};
|
|
31
|
+
if (msg.name)
|
|
32
|
+
openaiMsg.name = msg.name;
|
|
33
|
+
if (msg.tool_call_id)
|
|
34
|
+
openaiMsg.tool_call_id = msg.tool_call_id;
|
|
35
|
+
if (msg.tool_calls)
|
|
36
|
+
openaiMsg.tool_calls = msg.tool_calls;
|
|
37
|
+
return openaiMsg;
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
transformContent(content) {
|
|
41
|
+
if (typeof content === 'string')
|
|
42
|
+
return content;
|
|
43
|
+
if (!content)
|
|
44
|
+
return null;
|
|
45
|
+
return content.map((part) => {
|
|
46
|
+
if (part.type === 'text') {
|
|
47
|
+
return { type: 'text', text: part.text };
|
|
48
|
+
}
|
|
49
|
+
if (part.type === 'image_url') {
|
|
50
|
+
return {
|
|
51
|
+
type: 'image_url',
|
|
52
|
+
image_url: {
|
|
53
|
+
url: part.image_url.url,
|
|
54
|
+
detail: part.image_url.detail,
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
if (part.type === 'image') {
|
|
59
|
+
const base64 = Buffer.from(part.data).toString('base64');
|
|
60
|
+
return {
|
|
61
|
+
type: 'image_url',
|
|
62
|
+
image_url: {
|
|
63
|
+
url: `data:${part.mediaType};base64,${base64}`,
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
return { type: 'text', text: '' };
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
transformTools(tools) {
|
|
71
|
+
if (!tools)
|
|
72
|
+
return undefined;
|
|
73
|
+
return tools.map((tool) => ({
|
|
74
|
+
type: tool.type,
|
|
75
|
+
function: tool.function,
|
|
76
|
+
}));
|
|
77
|
+
}
|
|
78
|
+
async chat(options) {
|
|
79
|
+
const context = {
|
|
80
|
+
startTime: performance.now(),
|
|
81
|
+
tokenCount: 0,
|
|
82
|
+
};
|
|
83
|
+
const messages = this.prepareMessages(options);
|
|
84
|
+
const body = this.buildChatBody(options, messages, false);
|
|
85
|
+
const response = await this.makeRequest('/chat/completions', body, options.signal);
|
|
86
|
+
const data = await response.json();
|
|
87
|
+
return this.parseResponse(data, context);
|
|
88
|
+
}
|
|
89
|
+
async stream(options) {
|
|
90
|
+
const context = {
|
|
91
|
+
startTime: performance.now(),
|
|
92
|
+
tokenCount: 0,
|
|
93
|
+
};
|
|
94
|
+
const messages = this.prepareMessages(options);
|
|
95
|
+
const body = this.buildChatBody(options, messages, true);
|
|
96
|
+
const response = await this.makeRequest('/chat/completions', body, options.signal);
|
|
97
|
+
return this.parseSSEStream(response, context);
|
|
98
|
+
}
|
|
99
|
+
async embed(options) {
|
|
100
|
+
const startTime = performance.now();
|
|
101
|
+
const body = {
|
|
102
|
+
model: options.model || this.config.defaultModel || 'text-embedding-3-large',
|
|
103
|
+
input: options.input,
|
|
104
|
+
...(options.dimensions && { dimensions: options.dimensions }),
|
|
105
|
+
};
|
|
106
|
+
const response = await this.makeRequest('/embeddings', body, options.signal);
|
|
107
|
+
const data = await response.json();
|
|
108
|
+
const latency = {
|
|
109
|
+
ttft: performance.now() - startTime,
|
|
110
|
+
tps: 0,
|
|
111
|
+
total: performance.now() - startTime,
|
|
112
|
+
};
|
|
113
|
+
return {
|
|
114
|
+
embeddings: data.data.map((d) => d.embedding),
|
|
115
|
+
usage: {
|
|
116
|
+
inputTokens: data.usage.prompt_tokens,
|
|
117
|
+
outputTokens: 0,
|
|
118
|
+
totalTokens: data.usage.total_tokens,
|
|
119
|
+
},
|
|
120
|
+
model: data.model,
|
|
121
|
+
provider: 'openai',
|
|
122
|
+
latency,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
prepareMessages(options) {
|
|
126
|
+
const messages = [...options.messages];
|
|
127
|
+
if (options.systemPrompt) {
|
|
128
|
+
messages.unshift({
|
|
129
|
+
role: 'system',
|
|
130
|
+
content: options.systemPrompt,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
return messages;
|
|
134
|
+
}
|
|
135
|
+
buildChatBody(options, messages, stream) {
|
|
136
|
+
const body = {
|
|
137
|
+
model: options.model || this.config.defaultModel || 'gpt-5.1',
|
|
138
|
+
messages: this.transformMessages(messages),
|
|
139
|
+
stream,
|
|
140
|
+
};
|
|
141
|
+
if (options.temperature !== undefined)
|
|
142
|
+
body.temperature = options.temperature;
|
|
143
|
+
if (options.topP !== undefined)
|
|
144
|
+
body.top_p = options.topP;
|
|
145
|
+
if (options.maxTokens !== undefined)
|
|
146
|
+
body.max_tokens = options.maxTokens;
|
|
147
|
+
if (options.stop)
|
|
148
|
+
body.stop = options.stop;
|
|
149
|
+
if (options.tools)
|
|
150
|
+
body.tools = this.transformTools(options.tools);
|
|
151
|
+
if (options.toolChoice)
|
|
152
|
+
body.tool_choice = options.toolChoice;
|
|
153
|
+
if (options.responseFormat) {
|
|
154
|
+
if (options.responseFormat.type === 'json_object') {
|
|
155
|
+
body.response_format = { type: 'json_object' };
|
|
156
|
+
}
|
|
157
|
+
else if (options.responseFormat.type === 'json_schema') {
|
|
158
|
+
body.response_format = {
|
|
159
|
+
type: 'json_schema',
|
|
160
|
+
json_schema: options.responseFormat.schema,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (stream) {
|
|
165
|
+
body.stream_options = { include_usage: true };
|
|
166
|
+
}
|
|
167
|
+
return body;
|
|
168
|
+
}
|
|
169
|
+
async makeRequest(endpoint, body, signal) {
|
|
170
|
+
const url = `${this.getBaseUrl()}${endpoint}`;
|
|
171
|
+
const headers = this.buildHeaders();
|
|
172
|
+
const response = await fetch(url, {
|
|
173
|
+
method: 'POST',
|
|
174
|
+
headers,
|
|
175
|
+
body: JSON.stringify(body),
|
|
176
|
+
signal,
|
|
177
|
+
});
|
|
178
|
+
if (!response.ok) {
|
|
179
|
+
await this.handleError(response);
|
|
180
|
+
}
|
|
181
|
+
return response;
|
|
182
|
+
}
|
|
183
|
+
async handleError(response) {
|
|
184
|
+
let errorData = {};
|
|
185
|
+
try {
|
|
186
|
+
errorData = await response.json();
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
}
|
|
190
|
+
const message = errorData.error?.message || response.statusText;
|
|
191
|
+
const code = errorData.error?.code || errorData.error?.type;
|
|
192
|
+
switch (response.status) {
|
|
193
|
+
case 401:
|
|
194
|
+
throw new AuthenticationError('openai');
|
|
195
|
+
case 429:
|
|
196
|
+
const retryAfter = parseInt(response.headers.get('retry-after') || '0', 10);
|
|
197
|
+
throw new RateLimitError('openai', retryAfter || undefined);
|
|
198
|
+
case 400:
|
|
199
|
+
if (code === 'context_length_exceeded' || message.includes('maximum context length')) {
|
|
200
|
+
throw new ContextLengthError('openai');
|
|
201
|
+
}
|
|
202
|
+
break;
|
|
203
|
+
case 503:
|
|
204
|
+
case 529:
|
|
205
|
+
throw new OverloadedError('openai');
|
|
206
|
+
}
|
|
207
|
+
throw new AIError(message, 'openai', code, response.status, response.status >= 500);
|
|
208
|
+
}
|
|
209
|
+
parseResponse(data, context) {
|
|
210
|
+
const choice = data.choices[0];
|
|
211
|
+
const message = choice?.message;
|
|
212
|
+
const content = typeof message?.content === 'string' ? message.content : '';
|
|
213
|
+
const toolCalls = message?.tool_calls ? this.parseToolCalls(message.tool_calls) : undefined;
|
|
214
|
+
const usage = data.usage
|
|
215
|
+
? {
|
|
216
|
+
inputTokens: data.usage.prompt_tokens,
|
|
217
|
+
outputTokens: data.usage.completion_tokens,
|
|
218
|
+
totalTokens: data.usage.total_tokens,
|
|
219
|
+
}
|
|
220
|
+
: this.emptyUsage();
|
|
221
|
+
context.tokenCount = usage.outputTokens;
|
|
222
|
+
return {
|
|
223
|
+
content,
|
|
224
|
+
usage,
|
|
225
|
+
latency: this.calculateLatency(context),
|
|
226
|
+
model: data.model,
|
|
227
|
+
provider: 'openai',
|
|
228
|
+
cached: false,
|
|
229
|
+
finishReason: choice?.finish_reason,
|
|
230
|
+
toolCalls,
|
|
231
|
+
raw: data,
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
parseStreamEvent(chunk, context) {
|
|
235
|
+
const data = JSON.parse(chunk);
|
|
236
|
+
const choice = data.choices?.[0];
|
|
237
|
+
if (!choice) {
|
|
238
|
+
if (data.usage) {
|
|
239
|
+
return {
|
|
240
|
+
type: 'usage',
|
|
241
|
+
usage: {
|
|
242
|
+
inputTokens: data.usage.prompt_tokens,
|
|
243
|
+
outputTokens: data.usage.completion_tokens,
|
|
244
|
+
totalTokens: data.usage.total_tokens,
|
|
245
|
+
},
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
return null;
|
|
249
|
+
}
|
|
250
|
+
const delta = choice.delta;
|
|
251
|
+
if (delta.content) {
|
|
252
|
+
context.tokenCount++;
|
|
253
|
+
return {
|
|
254
|
+
type: 'text',
|
|
255
|
+
content: delta.content,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
if (delta.tool_calls?.length) {
|
|
259
|
+
const tc = delta.tool_calls[0];
|
|
260
|
+
if (tc.id) {
|
|
261
|
+
return {
|
|
262
|
+
type: 'tool_call',
|
|
263
|
+
toolCall: {
|
|
264
|
+
id: tc.id,
|
|
265
|
+
type: 'function',
|
|
266
|
+
function: {
|
|
267
|
+
name: tc.function?.name || '',
|
|
268
|
+
arguments: tc.function?.arguments || '',
|
|
269
|
+
},
|
|
270
|
+
},
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
else if (tc.function?.arguments) {
|
|
274
|
+
return {
|
|
275
|
+
type: 'tool_call_delta',
|
|
276
|
+
index: tc.index,
|
|
277
|
+
delta: {
|
|
278
|
+
arguments: tc.function.arguments,
|
|
279
|
+
},
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
if (choice.finish_reason) {
|
|
284
|
+
return {
|
|
285
|
+
type: 'done',
|
|
286
|
+
finishReason: choice.finish_reason,
|
|
287
|
+
usage: data.usage
|
|
288
|
+
? {
|
|
289
|
+
inputTokens: data.usage.prompt_tokens,
|
|
290
|
+
outputTokens: data.usage.completion_tokens,
|
|
291
|
+
totalTokens: data.usage.total_tokens,
|
|
292
|
+
}
|
|
293
|
+
: undefined,
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
return null;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { TokenRateLimitConfig, ChatOptions } from '../types/ai.js';
|
|
2
|
+
export declare class TokenRateLimiter {
|
|
3
|
+
private config;
|
|
4
|
+
private state;
|
|
5
|
+
private queue;
|
|
6
|
+
private processing;
|
|
7
|
+
private nextId;
|
|
8
|
+
constructor(config?: TokenRateLimitConfig);
|
|
9
|
+
execute<T>(fn: () => Promise<T>, options?: {
|
|
10
|
+
estimatedTokens?: number;
|
|
11
|
+
priority?: 'high' | 'normal' | 'low';
|
|
12
|
+
}): Promise<T>;
|
|
13
|
+
executeChat<T>(options: ChatOptions, fn: () => Promise<T>): Promise<T>;
|
|
14
|
+
recordUsage(actualTokens: number, estimatedTokens: number): void;
|
|
15
|
+
getUsage(): {
|
|
16
|
+
tokensUsed: number;
|
|
17
|
+
tokensRemaining: number;
|
|
18
|
+
requestsUsed: number;
|
|
19
|
+
requestsRemaining: number;
|
|
20
|
+
resetIn: number;
|
|
21
|
+
queueLength: number;
|
|
22
|
+
};
|
|
23
|
+
clearQueue(): void;
|
|
24
|
+
reset(): void;
|
|
25
|
+
private canExecute;
|
|
26
|
+
private executeNow;
|
|
27
|
+
private enqueue;
|
|
28
|
+
private processQueue;
|
|
29
|
+
private maybeResetWindow;
|
|
30
|
+
private getResetTime;
|
|
31
|
+
private getRetryAfter;
|
|
32
|
+
private sleep;
|
|
33
|
+
}
|
|
34
|
+
export declare class RateLimitExceededError extends Error {
|
|
35
|
+
readonly retryAfter: number;
|
|
36
|
+
constructor(retryAfter: number);
|
|
37
|
+
}
|
|
38
|
+
export declare const PROVIDER_RATE_LIMITS: Record<string, Partial<TokenRateLimitConfig>>;
|
|
39
|
+
export declare function createRateLimiter(providerTier?: string, overrides?: Partial<TokenRateLimitConfig>): TokenRateLimiter;
|
|
40
|
+
export declare const tokenEstimators: {
|
|
41
|
+
estimateMessage(content: string): number;
|
|
42
|
+
estimateChatTokens(options: ChatOptions): number;
|
|
43
|
+
};
|
|
44
|
+
//# sourceMappingURL=rate-limiter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rate-limiter.d.ts","sourceRoot":"","sources":["../../src/ai/rate-limiter.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAyFxE,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,KAAK,CAA4B;IACzC,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,MAAM,CAAa;gBAEf,MAAM,GAAE,oBAAyB;IAYvC,OAAO,CAAC,CAAC,EACb,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,OAAO,CAAC,EAAE;QAAE,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAA;KAAE,GAC3E,OAAO,CAAC,CAAC,CAAC;IA4BP,WAAW,CAAC,CAAC,EACjB,OAAO,EAAE,WAAW,EACpB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACnB,OAAO,CAAC,CAAC,CAAC;IAUb,WAAW,CAAC,YAAY,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,IAAI;IAchE,QAAQ,IAAI;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,eAAe,EAAE,MAAM,CAAC;QACxB,YAAY,EAAE,MAAM,CAAC;QACrB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;KACrB;IAeD,UAAU,IAAI,IAAI;IAUlB,KAAK,IAAI,IAAI;IAYb,OAAO,CAAC,UAAU;YAYJ,UAAU;IAUxB,OAAO,CAAC,OAAO;YAmCD,YAAY;IAkC1B,OAAO,CAAC,gBAAgB;IAcxB,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,aAAa;IAOrB,OAAO,CAAC,KAAK;CAGd;AAKD,qBAAa,sBAAuB,SAAQ,KAAK;aACnB,UAAU,EAAE,MAAM;gBAAlB,UAAU,EAAE,MAAM;CAI/C;AAKD,eAAO,MAAM,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAa9E,CAAC;AAKF,wBAAgB,iBAAiB,CAC/B,YAAY,CAAC,EAAE,MAAM,EACrB,SAAS,CAAC,EAAE,OAAO,CAAC,oBAAoB,CAAC,GACxC,gBAAgB,CAGlB;AAKD,eAAO,MAAM,eAAe;6BA/UD,MAAM,GAAG,MAAM;gCAQZ,WAAW,GAAG,MAAM;CAuUH,CAAC"}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
const DEFAULT_LIMITS = {
|
|
2
|
+
tokensPerMinute: 90000,
|
|
3
|
+
requestsPerMinute: 500,
|
|
4
|
+
strategy: 'queue',
|
|
5
|
+
priority: () => 'normal',
|
|
6
|
+
};
|
|
7
|
+
const TOKEN_ESTIMATORS = {
|
|
8
|
+
estimateMessage(content) {
|
|
9
|
+
if (!content)
|
|
10
|
+
return 0;
|
|
11
|
+
return Math.ceil(content.length / 4);
|
|
12
|
+
},
|
|
13
|
+
estimateChatTokens(options) {
|
|
14
|
+
let tokens = 0;
|
|
15
|
+
for (const msg of options.messages) {
|
|
16
|
+
if (typeof msg.content === 'string') {
|
|
17
|
+
tokens += this.estimateMessage(msg.content);
|
|
18
|
+
}
|
|
19
|
+
else if (Array.isArray(msg.content)) {
|
|
20
|
+
for (const part of msg.content) {
|
|
21
|
+
if (part.type === 'text') {
|
|
22
|
+
tokens += this.estimateMessage(part.text);
|
|
23
|
+
}
|
|
24
|
+
else if (part.type === 'image' || part.type === 'image_url') {
|
|
25
|
+
tokens += 1000;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
if (options.systemPrompt) {
|
|
31
|
+
tokens += this.estimateMessage(options.systemPrompt);
|
|
32
|
+
}
|
|
33
|
+
tokens += options.maxTokens || 1000;
|
|
34
|
+
if (options.tools?.length) {
|
|
35
|
+
tokens += options.tools.length * 100;
|
|
36
|
+
}
|
|
37
|
+
return tokens;
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
export class TokenRateLimiter {
|
|
41
|
+
config;
|
|
42
|
+
state;
|
|
43
|
+
queue = [];
|
|
44
|
+
processing = false;
|
|
45
|
+
nextId = 0;
|
|
46
|
+
constructor(config = {}) {
|
|
47
|
+
this.config = { ...DEFAULT_LIMITS, ...config };
|
|
48
|
+
this.state = {
|
|
49
|
+
tokenCount: 0,
|
|
50
|
+
requestCount: 0,
|
|
51
|
+
windowStart: Date.now(),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
async execute(fn, options) {
|
|
55
|
+
const estimatedTokens = options?.estimatedTokens || 1000;
|
|
56
|
+
const priority = options?.priority || 'normal';
|
|
57
|
+
if (this.canExecute(estimatedTokens)) {
|
|
58
|
+
return this.executeNow(fn, estimatedTokens);
|
|
59
|
+
}
|
|
60
|
+
switch (this.config.strategy) {
|
|
61
|
+
case 'throw':
|
|
62
|
+
throw new RateLimitExceededError(this.getRetryAfter());
|
|
63
|
+
case 'retry-after':
|
|
64
|
+
const retryAfter = this.getRetryAfter();
|
|
65
|
+
await this.sleep(retryAfter);
|
|
66
|
+
return this.execute(fn, options);
|
|
67
|
+
case 'queue':
|
|
68
|
+
default:
|
|
69
|
+
return this.enqueue(fn, estimatedTokens, priority);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
async executeChat(options, fn) {
|
|
73
|
+
const estimatedTokens = TOKEN_ESTIMATORS.estimateChatTokens(options);
|
|
74
|
+
const priority = this.config.priority(options);
|
|
75
|
+
return this.execute(fn, { estimatedTokens, priority });
|
|
76
|
+
}
|
|
77
|
+
recordUsage(actualTokens, estimatedTokens) {
|
|
78
|
+
const diff = actualTokens - estimatedTokens;
|
|
79
|
+
this.state.tokenCount += diff;
|
|
80
|
+
if (this.state.tokenCount < 0) {
|
|
81
|
+
this.state.tokenCount = 0;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
getUsage() {
|
|
85
|
+
this.maybeResetWindow();
|
|
86
|
+
return {
|
|
87
|
+
tokensUsed: this.state.tokenCount,
|
|
88
|
+
tokensRemaining: Math.max(0, this.config.tokensPerMinute - this.state.tokenCount),
|
|
89
|
+
requestsUsed: this.state.requestCount,
|
|
90
|
+
requestsRemaining: Math.max(0, this.config.requestsPerMinute - this.state.requestCount),
|
|
91
|
+
resetIn: this.getResetTime(),
|
|
92
|
+
queueLength: this.queue.length,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
clearQueue() {
|
|
96
|
+
for (const req of this.queue) {
|
|
97
|
+
req.reject(new Error('Queue cleared'));
|
|
98
|
+
}
|
|
99
|
+
this.queue = [];
|
|
100
|
+
}
|
|
101
|
+
reset() {
|
|
102
|
+
this.state = {
|
|
103
|
+
tokenCount: 0,
|
|
104
|
+
requestCount: 0,
|
|
105
|
+
windowStart: Date.now(),
|
|
106
|
+
};
|
|
107
|
+
this.clearQueue();
|
|
108
|
+
}
|
|
109
|
+
canExecute(estimatedTokens) {
|
|
110
|
+
this.maybeResetWindow();
|
|
111
|
+
const hasTokenBudget = this.state.tokenCount + estimatedTokens <= this.config.tokensPerMinute;
|
|
112
|
+
const hasRequestBudget = this.state.requestCount + 1 <= this.config.requestsPerMinute;
|
|
113
|
+
return hasTokenBudget && hasRequestBudget;
|
|
114
|
+
}
|
|
115
|
+
async executeNow(fn, estimatedTokens) {
|
|
116
|
+
this.state.tokenCount += estimatedTokens;
|
|
117
|
+
this.state.requestCount++;
|
|
118
|
+
return fn();
|
|
119
|
+
}
|
|
120
|
+
enqueue(fn, estimatedTokens, priority) {
|
|
121
|
+
return new Promise((resolve, reject) => {
|
|
122
|
+
const request = {
|
|
123
|
+
id: `req-${this.nextId++}`,
|
|
124
|
+
priority,
|
|
125
|
+
estimatedTokens,
|
|
126
|
+
execute: fn,
|
|
127
|
+
resolve,
|
|
128
|
+
reject,
|
|
129
|
+
enqueueTime: Date.now(),
|
|
130
|
+
};
|
|
131
|
+
let insertIndex = this.queue.length;
|
|
132
|
+
const priorityOrder = { high: 0, normal: 1, low: 2 };
|
|
133
|
+
for (let i = 0; i < this.queue.length; i++) {
|
|
134
|
+
if (priorityOrder[priority] < priorityOrder[this.queue[i].priority]) {
|
|
135
|
+
insertIndex = i;
|
|
136
|
+
break;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
this.queue.splice(insertIndex, 0, request);
|
|
140
|
+
this.processQueue();
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
async processQueue() {
|
|
144
|
+
if (this.processing)
|
|
145
|
+
return;
|
|
146
|
+
this.processing = true;
|
|
147
|
+
try {
|
|
148
|
+
while (this.queue.length > 0) {
|
|
149
|
+
const next = this.queue[0];
|
|
150
|
+
while (!this.canExecute(next.estimatedTokens)) {
|
|
151
|
+
const waitTime = this.getRetryAfter();
|
|
152
|
+
await this.sleep(Math.min(waitTime, 1000));
|
|
153
|
+
this.maybeResetWindow();
|
|
154
|
+
}
|
|
155
|
+
this.queue.shift();
|
|
156
|
+
try {
|
|
157
|
+
const result = await this.executeNow(next.execute, next.estimatedTokens);
|
|
158
|
+
next.resolve(result);
|
|
159
|
+
}
|
|
160
|
+
catch (error) {
|
|
161
|
+
next.reject(error);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
finally {
|
|
166
|
+
this.processing = false;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
maybeResetWindow() {
|
|
170
|
+
const now = Date.now();
|
|
171
|
+
const windowDuration = 60000;
|
|
172
|
+
if (now - this.state.windowStart >= windowDuration) {
|
|
173
|
+
this.state.tokenCount = 0;
|
|
174
|
+
this.state.requestCount = 0;
|
|
175
|
+
this.state.windowStart = now;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
getResetTime() {
|
|
179
|
+
const windowDuration = 60000;
|
|
180
|
+
return Math.max(0, windowDuration - (Date.now() - this.state.windowStart));
|
|
181
|
+
}
|
|
182
|
+
getRetryAfter() {
|
|
183
|
+
return this.getResetTime();
|
|
184
|
+
}
|
|
185
|
+
sleep(ms) {
|
|
186
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
export class RateLimitExceededError extends Error {
|
|
190
|
+
retryAfter;
|
|
191
|
+
constructor(retryAfter) {
|
|
192
|
+
super(`Rate limit exceeded. Retry after ${retryAfter}ms`);
|
|
193
|
+
this.retryAfter = retryAfter;
|
|
194
|
+
this.name = 'RateLimitExceededError';
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
export const PROVIDER_RATE_LIMITS = {
|
|
198
|
+
'openai-tier1': { tokensPerMinute: 200000, requestsPerMinute: 500 },
|
|
199
|
+
'openai-tier2': { tokensPerMinute: 450000, requestsPerMinute: 5000 },
|
|
200
|
+
'openai-tier3': { tokensPerMinute: 600000, requestsPerMinute: 5000 },
|
|
201
|
+
'openai-tier4': { tokensPerMinute: 800000, requestsPerMinute: 10000 },
|
|
202
|
+
'openai-tier5': { tokensPerMinute: 10000000, requestsPerMinute: 10000 },
|
|
203
|
+
'anthropic-tier1': { tokensPerMinute: 80000, requestsPerMinute: 50 },
|
|
204
|
+
'anthropic-tier2': { tokensPerMinute: 160000, requestsPerMinute: 1000 },
|
|
205
|
+
'anthropic-tier3': { tokensPerMinute: 400000, requestsPerMinute: 2000 },
|
|
206
|
+
'anthropic-tier4': { tokensPerMinute: 800000, requestsPerMinute: 4000 },
|
|
207
|
+
};
|
|
208
|
+
export function createRateLimiter(providerTier, overrides) {
|
|
209
|
+
const baseConfig = providerTier ? PROVIDER_RATE_LIMITS[providerTier] || {} : {};
|
|
210
|
+
return new TokenRateLimiter({ ...baseConfig, ...overrides });
|
|
211
|
+
}
|
|
212
|
+
export const tokenEstimators = TOKEN_ESTIMATORS;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { LoadStats } from './stats.js';
|
|
2
|
+
export type LoadMode = 'throughput' | 'stress' | 'realistic';
|
|
3
|
+
export interface LoadConfig {
|
|
4
|
+
url: string;
|
|
5
|
+
users: number;
|
|
6
|
+
duration: number;
|
|
7
|
+
mode: LoadMode;
|
|
8
|
+
http2?: boolean;
|
|
9
|
+
rampUp?: number;
|
|
10
|
+
}
|
|
11
|
+
export declare class LoadGenerator {
|
|
12
|
+
private config;
|
|
13
|
+
stats: LoadStats;
|
|
14
|
+
private running;
|
|
15
|
+
constructor(config: LoadConfig);
|
|
16
|
+
start(): Promise<void[]>;
|
|
17
|
+
stop(): void;
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=generator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../../src/bench/generator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,MAAM,MAAM,QAAQ,GAAG,YAAY,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE7D,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAa;IACpB,KAAK,EAAE,SAAS,CAAC;IACxB,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,EAAE,UAAU;IAKxB,KAAK;IA4FX,IAAI;CAGL"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { createClient } from '../core/client.js';
|
|
2
|
+
import { LoadStats } from './stats.js';
|
|
3
|
+
export class LoadGenerator {
|
|
4
|
+
config;
|
|
5
|
+
stats;
|
|
6
|
+
running = false;
|
|
7
|
+
constructor(config) {
|
|
8
|
+
this.config = config;
|
|
9
|
+
this.stats = new LoadStats();
|
|
10
|
+
}
|
|
11
|
+
async start() {
|
|
12
|
+
this.running = true;
|
|
13
|
+
const startTime = Date.now();
|
|
14
|
+
const client = createClient({
|
|
15
|
+
baseUrl: new URL(this.config.url).origin,
|
|
16
|
+
observability: false,
|
|
17
|
+
http2: this.config.http2,
|
|
18
|
+
concurrency: {
|
|
19
|
+
max: this.config.users * 2,
|
|
20
|
+
requestsPerInterval: Infinity,
|
|
21
|
+
interval: 1000,
|
|
22
|
+
agent: {
|
|
23
|
+
connections: this.config.users,
|
|
24
|
+
pipelining: this.config.mode === 'throughput' ? 2 : 1,
|
|
25
|
+
keepAlive: false
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
retry: {
|
|
29
|
+
maxAttempts: this.config.mode === 'stress' ? 0 : 2
|
|
30
|
+
},
|
|
31
|
+
timeout: 5000
|
|
32
|
+
});
|
|
33
|
+
const path = new URL(this.config.url).pathname;
|
|
34
|
+
const userLoop = async () => {
|
|
35
|
+
this.stats.activeUsers++;
|
|
36
|
+
while (this.running) {
|
|
37
|
+
const start = performance.now();
|
|
38
|
+
const controller = new AbortController();
|
|
39
|
+
try {
|
|
40
|
+
if (this.config.mode === 'realistic') {
|
|
41
|
+
await new Promise(r => setTimeout(r, 50 + Math.random() * 450));
|
|
42
|
+
}
|
|
43
|
+
const res = await client.get(path, { signal: controller.signal });
|
|
44
|
+
const duration = performance.now() - start;
|
|
45
|
+
const bytes = Number(res.headers.get('content-length') || 0);
|
|
46
|
+
this.stats.addResult(duration, res.status, bytes);
|
|
47
|
+
}
|
|
48
|
+
catch (err) {
|
|
49
|
+
if (err.name === 'AbortError' || err.code === 'UND_ERR_ABORTED') {
|
|
50
|
+
this.stats.addResult(performance.now() - start, 0, 0, new Error('Request Aborted (timeout)'));
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
this.stats.addResult(performance.now() - start, 0, 0, err);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
finally {
|
|
57
|
+
controller.abort();
|
|
58
|
+
}
|
|
59
|
+
if (this.config.mode === 'stress') {
|
|
60
|
+
await new Promise(r => setImmediate(r));
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
this.stats.activeUsers--;
|
|
64
|
+
};
|
|
65
|
+
const users = [];
|
|
66
|
+
const rampUpMs = (this.config.rampUp || 0) * 1000;
|
|
67
|
+
for (let i = 0; i < this.config.users; i++) {
|
|
68
|
+
const delay = rampUpMs > 0 ? (i / this.config.users) * rampUpMs : 0;
|
|
69
|
+
const userSession = async () => {
|
|
70
|
+
if (delay > 0)
|
|
71
|
+
await new Promise(r => setTimeout(r, delay));
|
|
72
|
+
if (!this.running)
|
|
73
|
+
return;
|
|
74
|
+
await userLoop();
|
|
75
|
+
};
|
|
76
|
+
users.push(userSession());
|
|
77
|
+
}
|
|
78
|
+
setTimeout(() => {
|
|
79
|
+
this.running = false;
|
|
80
|
+
}, this.config.duration * 1000);
|
|
81
|
+
return Promise.all(users);
|
|
82
|
+
}
|
|
83
|
+
stop() {
|
|
84
|
+
this.running = false;
|
|
85
|
+
}
|
|
86
|
+
}
|