testronaut 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +194 -0
- package/bin/cli.js +1 -1
- package/bin/init.js +141 -57
- package/bin/initHelpers.js +140 -0
- package/core/agent.js +1 -1
- package/core/turnLoop.js +170 -107
- package/llm/gemini/geminiProvider.js +188 -0
- package/llm/llmFactory.js +50 -0
- package/llm/modelResolver.js +65 -0
- package/llm/openAI/openaiProvider.js +54 -0
- package/package.json +9 -3
- package/tests/binTests/init.test.js +189 -0
- package/tests/binTests/initHelpersTests/defaultConfig.test.js +12 -0
- package/tests/binTests/initHelpersTests/makeEnvTemplate.test.js +14 -0
- package/tests/binTests/initHelpersTests/migrateLegacyConfig.test.js +19 -0
- package/tests/binTests/initHelpersTests/models.test.js +21 -0
- package/tests/coreTests/turnLoop.test.js +177 -0
- package/tests/llmTests/geminiProvider.test.js +191 -0
- package/tests/llmTests/llmFactory.test.js +78 -0
- package/tests/llmTests/modelResolver.test.js +119 -0
- package/tests/llmTests/openAIProvider.test.js +117 -0
- package/tests/toolsTests/tokenControl.test.js +150 -0
- package/tools/tokenControl.js +107 -17
- package/vitest.config.js +8 -0
- package/openAI/modelResolver.js +0 -33
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// tests/llmTests/geminiProvider.test.js
|
|
2
|
+
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
|
3
|
+
|
|
4
|
+
// Hoisted shared so the mock factory can access/record state
|
|
5
|
+
const { shared } = vi.hoisted(() => ({
|
|
6
|
+
shared: {
|
|
7
|
+
apiKey: null,
|
|
8
|
+
getModelOpts: null,
|
|
9
|
+
lastContents: null,
|
|
10
|
+
// default response generator; tests can override
|
|
11
|
+
generateResponse: () => ({
|
|
12
|
+
response: {
|
|
13
|
+
candidates: [
|
|
14
|
+
{ content: { parts: [{ text: 'hello' }] } }
|
|
15
|
+
],
|
|
16
|
+
usageMetadata: { totalTokenCount: 123 },
|
|
17
|
+
},
|
|
18
|
+
}),
|
|
19
|
+
},
|
|
20
|
+
}));
|
|
21
|
+
|
|
22
|
+
// Mock the Google SDK
|
|
23
|
+
vi.mock('@google/generative-ai', () => {
|
|
24
|
+
class FakeModel {
|
|
25
|
+
constructor(opts) { shared.getModelOpts = opts; }
|
|
26
|
+
async generateContent({ contents }) {
|
|
27
|
+
shared.lastContents = contents;
|
|
28
|
+
return shared.generateResponse();
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
class GoogleGenerativeAI {
|
|
32
|
+
constructor(key) { shared.apiKey = key; }
|
|
33
|
+
getGenerativeModel(opts) { shared.getModelOpts = opts; return new FakeModel(opts); }
|
|
34
|
+
}
|
|
35
|
+
return { GoogleGenerativeAI };
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
// Import SUT after mocks
|
|
39
|
+
import { GeminiProvider } from '../../llm/gemini/geminiProvider.js';
|
|
40
|
+
|
|
41
|
+
describe('GeminiProvider', () => {
|
|
42
|
+
beforeEach(() => {
|
|
43
|
+
shared.apiKey = null;
|
|
44
|
+
shared.getModelOpts = null;
|
|
45
|
+
shared.lastContents = null;
|
|
46
|
+
// reset default response
|
|
47
|
+
shared.generateResponse = () => ({
|
|
48
|
+
response: {
|
|
49
|
+
candidates: [{ content: { parts: [{ text: 'hello' }] } }],
|
|
50
|
+
usageMetadata: { totalTokenCount: 123 },
|
|
51
|
+
},
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it('throws if constructed without apiKey', () => {
|
|
56
|
+
expect(() => new GeminiProvider({})).toThrow(/GEMINI_API_KEY/i);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('passes apiKey and model/tools into the SDK correctly', async () => {
|
|
60
|
+
const prov = new GeminiProvider({ apiKey: 'gk-123' });
|
|
61
|
+
|
|
62
|
+
const tools = [
|
|
63
|
+
{ type: 'function', function: { name: 'foo', description: 'Foo', parameters: { type: 'object' } } },
|
|
64
|
+
{ type: 'function', function: { name: 'bar', parameters: { type: 'object', properties: { x: { type: 'string' } } } } },
|
|
65
|
+
];
|
|
66
|
+
|
|
67
|
+
await prov.chat({
|
|
68
|
+
model: 'gemini-2.5-flash',
|
|
69
|
+
messages: [{ role: 'user', content: 'hi' }],
|
|
70
|
+
tools,
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
expect(shared.apiKey).toBe('gk-123');
|
|
74
|
+
expect(shared.getModelOpts?.model).toBe('gemini-2.5-flash');
|
|
75
|
+
// Tools converted to Gemini functionDeclarations wrapper
|
|
76
|
+
expect(Array.isArray(shared.getModelOpts?.tools)).toBe(true);
|
|
77
|
+
expect(shared.getModelOpts.tools[0].functionDeclarations.map(d => d.name)).toEqual(['foo', 'bar']);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('converts OpenAI-like messages → Gemini contents (system prefix + tool calls + tool results)', async () => {
|
|
81
|
+
const prov = new GeminiProvider({ apiKey: 'gk-xyz' });
|
|
82
|
+
|
|
83
|
+
// Build a conversation:
|
|
84
|
+
// - system
|
|
85
|
+
// - user (should get system prefix injected)
|
|
86
|
+
// - assistant with tool_calls
|
|
87
|
+
// - tool result
|
|
88
|
+
// - assistant normal text
|
|
89
|
+
const messages = [
|
|
90
|
+
{ role: 'system', content: 'You are helpful.' },
|
|
91
|
+
{ role: 'user', content: 'search for x' },
|
|
92
|
+
{
|
|
93
|
+
role: 'assistant',
|
|
94
|
+
tool_calls: [
|
|
95
|
+
{ id: '1', type: 'function', function: { name: 'search', arguments: JSON.stringify({ q: 'x' }) } },
|
|
96
|
+
],
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
role: 'tool',
|
|
100
|
+
tool_call_id: '1',
|
|
101
|
+
name: 'search',
|
|
102
|
+
type: 'function',
|
|
103
|
+
content: JSON.stringify({ results: [1, 2, 3] }),
|
|
104
|
+
},
|
|
105
|
+
{ role: 'assistant', content: 'Here are results.' },
|
|
106
|
+
];
|
|
107
|
+
|
|
108
|
+
await prov.chat({ model: 'gemini-2.5-pro', messages });
|
|
109
|
+
|
|
110
|
+
const c = shared.lastContents;
|
|
111
|
+
expect(Array.isArray(c)).toBe(true);
|
|
112
|
+
|
|
113
|
+
// 1) First item: user with injected system prefix
|
|
114
|
+
expect(c[0].role).toBe('user');
|
|
115
|
+
expect(c[0].parts[0].text).toMatch(/^\[system]\nYou are helpful\./);
|
|
116
|
+
expect(c[0].parts[1].text).toBe('search for x');
|
|
117
|
+
|
|
118
|
+
// 2) Assistant tool call → model role with functionCall
|
|
119
|
+
expect(c[1].role).toBe('model');
|
|
120
|
+
expect(c[1].parts[0].functionCall).toEqual({ name: 'search', args: { q: 'x' } });
|
|
121
|
+
|
|
122
|
+
// 3) Tool result → user role with JSON marker
|
|
123
|
+
expect(c[2].role).toBe('user');
|
|
124
|
+
const toolJson = JSON.parse(c[2].parts[0].text);
|
|
125
|
+
expect(toolJson._tool_result).toBe(true);
|
|
126
|
+
expect(toolJson.tool_call_id).toBe('1');
|
|
127
|
+
expect(toolJson.name).toBe('search');
|
|
128
|
+
|
|
129
|
+
// 4) Final assistant text → model role with text
|
|
130
|
+
expect(c[3].role).toBe('model');
|
|
131
|
+
expect(c[3].parts[0].text).toBe('Here are results.');
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
it('maps Gemini candidate parts → OpenAI-like assistant message and usage', async () => {
|
|
135
|
+
// Make the fake SDK return a functionCall + text
|
|
136
|
+
shared.generateResponse = () => ({
|
|
137
|
+
response: {
|
|
138
|
+
candidates: [
|
|
139
|
+
{
|
|
140
|
+
content: {
|
|
141
|
+
parts: [
|
|
142
|
+
{ functionCall: { name: 'open', args: { url: 'https://example.com' } } },
|
|
143
|
+
{ text: 'Ok!' },
|
|
144
|
+
],
|
|
145
|
+
},
|
|
146
|
+
},
|
|
147
|
+
],
|
|
148
|
+
usageMetadata: { totalTokenCount: 999, other: 'meta' },
|
|
149
|
+
},
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
const prov = new GeminiProvider({ apiKey: 'gk-abc' });
|
|
153
|
+
const { message, usage } = await prov.chat({
|
|
154
|
+
model: 'gemini-2.5-flash',
|
|
155
|
+
messages: [{ role: 'user', content: 'go' }],
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
expect(message.role).toBe('assistant');
|
|
159
|
+
expect(message.content).toBe('Ok!');
|
|
160
|
+
expect(Array.isArray(message.tool_calls)).toBe(true);
|
|
161
|
+
expect(message.tool_calls[0].function.name).toBe('open');
|
|
162
|
+
expect(JSON.parse(message.tool_calls[0].function.arguments)).toEqual({ url: 'https://example.com' });
|
|
163
|
+
|
|
164
|
+
expect(usage.total_tokens).toBe(999);
|
|
165
|
+
expect(usage.providerRaw).toEqual({ totalTokenCount: 999, other: 'meta' });
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it('supports image parts (encoded to inlineData)', async () => {
|
|
169
|
+
const prov = new GeminiProvider({ apiKey: 'gk-xyz' });
|
|
170
|
+
|
|
171
|
+
const imageBytes = Buffer.from([1, 2, 3, 4]);
|
|
172
|
+
await prov.chat({
|
|
173
|
+
model: 'gemini-2.5-flash',
|
|
174
|
+
messages: [
|
|
175
|
+
{
|
|
176
|
+
role: 'user',
|
|
177
|
+
content: [
|
|
178
|
+
{ type: 'text', text: 'see image' },
|
|
179
|
+
{ type: 'image', mimeType: 'image/png', data: imageBytes },
|
|
180
|
+
],
|
|
181
|
+
},
|
|
182
|
+
],
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
const c = shared.lastContents;
|
|
186
|
+
expect(c[0].role).toBe('user');
|
|
187
|
+
expect(c[0].parts[0].text).toBe('see image');
|
|
188
|
+
expect(c[0].parts[1].inlineData.mimeType).toBe('image/png');
|
|
189
|
+
expect(typeof c[0].parts[1].inlineData.data).toBe('string'); // base64
|
|
190
|
+
});
|
|
191
|
+
});
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// tests/llmTests/llmFactory.test.js
|
|
2
|
+
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
|
3
|
+
|
|
4
|
+
// Hoisted shared spies so mock factories can access them
|
|
5
|
+
const { shared } = vi.hoisted(() => ({
|
|
6
|
+
shared: {
|
|
7
|
+
openAICtor: vi.fn(),
|
|
8
|
+
geminiCtor: vi.fn(),
|
|
9
|
+
},
|
|
10
|
+
}));
|
|
11
|
+
|
|
12
|
+
// Mock provider classes; constructors record received options
|
|
13
|
+
vi.mock('../../llm/openAI/openaiProvider.js', () => ({
|
|
14
|
+
OpenAIProvider: function (opts) {
|
|
15
|
+
shared.openAICtor(opts);
|
|
16
|
+
// minimal adapter surface used by callers
|
|
17
|
+
this.chat = vi.fn(async () => ({ message: { role: 'assistant', content: 'ok' }, usage: {} }));
|
|
18
|
+
},
|
|
19
|
+
}));
|
|
20
|
+
|
|
21
|
+
vi.mock('../../llm/gemini/geminiProvider.js', () => ({
|
|
22
|
+
GeminiProvider: function (opts) {
|
|
23
|
+
shared.geminiCtor(opts);
|
|
24
|
+
this.chat = vi.fn(async () => ({ message: { role: 'assistant', content: 'ok' }, usage: {} }));
|
|
25
|
+
},
|
|
26
|
+
}));
|
|
27
|
+
|
|
28
|
+
// Import SUT after mocks
|
|
29
|
+
import { getLLM } from '../../llm/llmFactory.js';
|
|
30
|
+
|
|
31
|
+
const ORIGINAL_ENV = { ...process.env };
|
|
32
|
+
|
|
33
|
+
describe('getLLM', () => {
|
|
34
|
+
beforeEach(() => {
|
|
35
|
+
process.env = { ...ORIGINAL_ENV };
|
|
36
|
+
shared.openAICtor.mockClear();
|
|
37
|
+
shared.geminiCtor.mockClear();
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
afterEach(() => {
|
|
41
|
+
process.env = { ...ORIGINAL_ENV };
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('creates OpenAI provider (case-insensitive) and passes env key', () => {
|
|
45
|
+
process.env.OPENAI_API_KEY = 'sk-openai-123';
|
|
46
|
+
const llm = getLLM('OpEnAi');
|
|
47
|
+
expect(typeof llm.chat).toBe('function');
|
|
48
|
+
expect(shared.openAICtor).toHaveBeenCalledTimes(1);
|
|
49
|
+
expect(shared.openAICtor).toHaveBeenCalledWith(
|
|
50
|
+
expect.objectContaining({ apiKey: 'sk-openai-123' })
|
|
51
|
+
);
|
|
52
|
+
expect(shared.geminiCtor).not.toHaveBeenCalled();
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it('creates Gemini provider and passes env key', () => {
|
|
56
|
+
process.env.GEMINI_API_KEY = 'gk-gemini-abc';
|
|
57
|
+
const llm = getLLM('gemini');
|
|
58
|
+
expect(typeof llm.chat).toBe('function');
|
|
59
|
+
expect(shared.geminiCtor).toHaveBeenCalledTimes(1);
|
|
60
|
+
expect(shared.geminiCtor).toHaveBeenCalledWith(
|
|
61
|
+
expect.objectContaining({ apiKey: 'gk-gemini-abc' })
|
|
62
|
+
);
|
|
63
|
+
expect(shared.openAICtor).not.toHaveBeenCalled();
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('merges opts after env so explicit opts.apiKey overrides env', () => {
|
|
67
|
+
process.env.OPENAI_API_KEY = 'sk-env-will-be-overridden';
|
|
68
|
+
const llm = getLLM('openai', { apiKey: 'sk-passed-in' });
|
|
69
|
+
expect(shared.openAICtor).toHaveBeenCalledWith(
|
|
70
|
+
expect.objectContaining({ apiKey: 'sk-passed-in' })
|
|
71
|
+
);
|
|
72
|
+
expect(typeof llm.chat).toBe('function');
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it('throws on unsupported provider', () => {
|
|
76
|
+
expect(() => getLLM('anthropic')).toThrow(/Unsupported LLM provider/i);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
// tests/llmTests/modelResolver.test.js
|
|
2
|
+
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
3
|
+
import fs from 'fs';
|
|
4
|
+
import os from 'os';
|
|
5
|
+
import path from 'path';
|
|
6
|
+
|
|
7
|
+
import { resolveProviderModel } from '../../llm/modelResolver.js';
|
|
8
|
+
|
|
9
|
+
const ORIGINAL_ENV = { ...process.env };
|
|
10
|
+
const originalCwd = process.cwd();
|
|
11
|
+
|
|
12
|
+
function makeTempProject() {
|
|
13
|
+
return fs.mkdtempSync(path.join(os.tmpdir(), 'testronaut-'));
|
|
14
|
+
}
|
|
15
|
+
function writeConfig(dir, obj) {
|
|
16
|
+
fs.writeFileSync(path.join(dir, 'testronaut-config.json'), JSON.stringify(obj, null, 2), 'utf8');
|
|
17
|
+
}
|
|
18
|
+
function clearResolverEnv() {
|
|
19
|
+
delete process.env.TESTRONAUT_PROVIDER;
|
|
20
|
+
delete process.env.TESTRONAUT_MODEL;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
beforeEach(() => {
|
|
24
|
+
process.env = { ...ORIGINAL_ENV };
|
|
25
|
+
clearResolverEnv();
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
afterEach(() => {
|
|
29
|
+
process.env = { ...ORIGINAL_ENV };
|
|
30
|
+
process.chdir(originalCwd);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
describe('resolveProviderModel', () => {
|
|
34
|
+
it('returns env provider/model when both set', () => {
|
|
35
|
+
process.env.TESTRONAUT_PROVIDER = 'gemini';
|
|
36
|
+
process.env.TESTRONAUT_MODEL = 'gemini-2.5-flash';
|
|
37
|
+
const res = resolveProviderModel();
|
|
38
|
+
expect(res).toEqual({ provider: 'gemini', model: 'gemini-2.5-flash' });
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it('reads provider/model from config file when present', () => {
|
|
42
|
+
const temp = makeTempProject();
|
|
43
|
+
writeConfig(temp, { provider: 'openai', model: 'gpt-5' });
|
|
44
|
+
|
|
45
|
+
const res = resolveProviderModel({ cwd: temp }); // 👈 pass cwd
|
|
46
|
+
expect(res.provider).toBe('openai');
|
|
47
|
+
expect(res.model).toBe('gpt-5');
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it('applies env override for only one value (env model overrides config model)', () => {
|
|
51
|
+
const temp = makeTempProject();
|
|
52
|
+
writeConfig(temp, { provider: 'openai', model: 'gpt-4o' });
|
|
53
|
+
process.env.TESTRONAUT_MODEL = 'gpt-5-mini';
|
|
54
|
+
|
|
55
|
+
const res = resolveProviderModel({ cwd: temp });
|
|
56
|
+
expect(res.provider).toBe('openai');
|
|
57
|
+
expect(res.model).toBe('gpt-5-mini');
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it('handles legacy config with model="openai"', () => {
|
|
61
|
+
const temp = makeTempProject();
|
|
62
|
+
writeConfig(temp, { model: 'openai' });
|
|
63
|
+
|
|
64
|
+
const res = resolveProviderModel({ cwd: temp });
|
|
65
|
+
expect(res).toEqual({ provider: 'openai', model: 'gpt-4o' });
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it('returns defaults when config missing', () => {
|
|
69
|
+
const temp = makeTempProject();
|
|
70
|
+
const res = resolveProviderModel({ cwd: temp });
|
|
71
|
+
expect(res).toEqual({ provider: 'openai', model: 'gpt-4o' });
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('returns defaults when JSON parsing fails', () => {
|
|
75
|
+
const temp = makeTempProject();
|
|
76
|
+
fs.writeFileSync(path.join(temp, 'testronaut-config.json'), '{bad json', 'utf8');
|
|
77
|
+
|
|
78
|
+
const res = resolveProviderModel({ cwd: temp });
|
|
79
|
+
expect(res.provider).toBe('openai');
|
|
80
|
+
expect(res.model).toBe('gpt-4o');
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it('trims env vars and treats empty strings as absent', () => {
|
|
84
|
+
process.env.TESTRONAUT_PROVIDER = ' openai ';
|
|
85
|
+
process.env.TESTRONAUT_MODEL = ' gpt-5 ';
|
|
86
|
+
let res = resolveProviderModel();
|
|
87
|
+
expect(res).toEqual({ provider: 'openai', model: 'gpt-5' });
|
|
88
|
+
|
|
89
|
+
process.env.TESTRONAUT_PROVIDER = ' ';
|
|
90
|
+
process.env.TESTRONAUT_MODEL = ' ';
|
|
91
|
+
res = resolveProviderModel();
|
|
92
|
+
// falls back to defaults when env vars are just whitespace
|
|
93
|
+
expect(res).toEqual({ provider: 'openai', model: 'gpt-4o' });
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it('prefers config when only provider is in env and model only in config', () => {
|
|
97
|
+
process.env.TESTRONAUT_PROVIDER = 'gemini';
|
|
98
|
+
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'testronaut-'));
|
|
99
|
+
fs.writeFileSync(
|
|
100
|
+
path.join(temp, 'testronaut-config.json'),
|
|
101
|
+
JSON.stringify({ provider: 'openai', model: 'gemini-2.5-flash' }),
|
|
102
|
+
'utf8'
|
|
103
|
+
);
|
|
104
|
+
const res = resolveProviderModel({ cwd: temp });
|
|
105
|
+
expect(res).toEqual({ provider: 'gemini', model: 'gemini-2.5-flash' });
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it('ignores unrelated keys in config and still resolves', () => {
|
|
109
|
+
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'testronaut-'));
|
|
110
|
+
fs.writeFileSync(
|
|
111
|
+
path.join(temp, 'testronaut-config.json'),
|
|
112
|
+
JSON.stringify({ provider: 'openai', model: 'gpt-4o', extra: { foo: 1 } }),
|
|
113
|
+
'utf8'
|
|
114
|
+
);
|
|
115
|
+
const res = resolveProviderModel({ cwd: temp });
|
|
116
|
+
expect(res).toEqual({ provider: 'openai', model: 'gpt-4o' });
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
});
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
// tests/llmTests/openAIProvider.test.js
|
|
2
|
+
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
|
3
|
+
|
|
4
|
+
// Hoisted shared so the mock factory can access/record state
|
|
5
|
+
const { shared } = vi.hoisted(() => ({
|
|
6
|
+
shared: {
|
|
7
|
+
ctorOpts: null,
|
|
8
|
+
lastCreateArgs: null,
|
|
9
|
+
// default response; tests can override
|
|
10
|
+
responseFactory: () => ({
|
|
11
|
+
choices: [{ message: { role: 'assistant', content: 'hello' } }],
|
|
12
|
+
usage: { total_tokens: 321, prompt_tokens: 100, completion_tokens: 221 },
|
|
13
|
+
headers: { 'x-ratelimit-limit-tokens': '90000' },
|
|
14
|
+
}),
|
|
15
|
+
},
|
|
16
|
+
}));
|
|
17
|
+
|
|
18
|
+
// Mock the OpenAI SDK surface we use
|
|
19
|
+
vi.mock('openai', () => {
|
|
20
|
+
class FakeChatCompletions {
|
|
21
|
+
async create(args) {
|
|
22
|
+
shared.lastCreateArgs = args;
|
|
23
|
+
return shared.responseFactory();
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
class FakeChat {
|
|
27
|
+
constructor() {
|
|
28
|
+
this.completions = new FakeChatCompletions();
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
class OpenAI {
|
|
32
|
+
constructor(opts) {
|
|
33
|
+
shared.ctorOpts = opts;
|
|
34
|
+
this.chat = new FakeChat();
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return { default: OpenAI };
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
// Import SUT after mocks
|
|
41
|
+
import { OpenAIProvider } from '../../llm/openAI/openaiProvider.js';
|
|
42
|
+
|
|
43
|
+
describe('OpenAIProvider', () => {
|
|
44
|
+
beforeEach(() => {
|
|
45
|
+
shared.ctorOpts = null;
|
|
46
|
+
shared.lastCreateArgs = null;
|
|
47
|
+
shared.responseFactory = () => ({
|
|
48
|
+
choices: [{ message: { role: 'assistant', content: 'hello' } }],
|
|
49
|
+
usage: { total_tokens: 321, prompt_tokens: 100, completion_tokens: 221 },
|
|
50
|
+
headers: { 'x-ratelimit-limit-tokens': '90000' },
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it('throws without apiKey', () => {
|
|
55
|
+
expect(() => new OpenAIProvider({})).toThrow(/OPENAI_API_KEY/i);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it('passes apiKey to OpenAI constructor', () => {
|
|
59
|
+
const p = new OpenAIProvider({ apiKey: 'sk-test-123' });
|
|
60
|
+
expect(shared.ctorOpts).toEqual({ apiKey: 'sk-test-123' });
|
|
61
|
+
expect(typeof p.chat).toBe('function');
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it('calls chat.completions.create with model/messages/tools', async () => {
|
|
65
|
+
const prov = new OpenAIProvider({ apiKey: 'sk-abc' });
|
|
66
|
+
const tools = [{ type: 'function', function: { name: 'doThing', parameters: { type: 'object' } } }];
|
|
67
|
+
|
|
68
|
+
await prov.chat({
|
|
69
|
+
model: 'gpt-4o',
|
|
70
|
+
messages: [{ role: 'user', content: 'hi' }],
|
|
71
|
+
tools,
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
expect(shared.lastCreateArgs).toEqual({
|
|
75
|
+
model: 'gpt-4o',
|
|
76
|
+
messages: [{ role: 'user', content: 'hi' }],
|
|
77
|
+
tools,
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('returns normalized message, usage, and headers', async () => {
|
|
82
|
+
const prov = new OpenAIProvider({ apiKey: 'sk-abc' });
|
|
83
|
+
const { message, usage, headers } = await prov.chat({
|
|
84
|
+
model: 'gpt-4o',
|
|
85
|
+
messages: [{ role: 'user', content: 'go' }],
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
expect(message).toEqual({ role: 'assistant', content: 'hello' });
|
|
89
|
+
expect(usage.total_tokens).toBe(321);
|
|
90
|
+
expect(usage.providerRaw).toEqual({ total_tokens: 321, prompt_tokens: 100, completion_tokens: 221 });
|
|
91
|
+
expect(headers).toEqual({ 'x-ratelimit-limit-tokens': '90000' });
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it('handles SDK variants with headers on response.response.headers', async () => {
|
|
95
|
+
shared.responseFactory = () => ({
|
|
96
|
+
choices: [{ message: { role: 'assistant', content: 'ok' } }],
|
|
97
|
+
usage: { total_tokens: 11 },
|
|
98
|
+
response: { headers: { 'x-ratelimit-limit-tokens': '12345' } },
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
const prov = new OpenAIProvider({ apiKey: 'sk-abc' });
|
|
102
|
+
const { headers } = await prov.chat({ model: 'gpt-4o', messages: [] });
|
|
103
|
+
expect(headers).toEqual({ 'x-ratelimit-limit-tokens': '12345' });
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it('provides a default assistant message when choices are empty', async () => {
|
|
107
|
+
shared.responseFactory = () => ({
|
|
108
|
+
choices: [],
|
|
109
|
+
usage: {},
|
|
110
|
+
headers: {},
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
const prov = new OpenAIProvider({ apiKey: 'sk-abc' });
|
|
114
|
+
const { message } = await prov.chat({ model: 'gpt-4o', messages: [] });
|
|
115
|
+
expect(message).toEqual({ role: 'assistant', content: '' });
|
|
116
|
+
});
|
|
117
|
+
});
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// tests/tokenControlTests/tokenControl.test.js
|
|
2
|
+
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
|
3
|
+
|
|
4
|
+
// Hoist env so we can safely mutate per test
|
|
5
|
+
const ORIGINAL_ENV = { ...process.env };
|
|
6
|
+
|
|
7
|
+
// Mock wait so tests are instant
|
|
8
|
+
vi.mock('../../tools/turnLoopUtils.js', () => ({
|
|
9
|
+
wait: vi.fn(async () => {}),
|
|
10
|
+
}));
|
|
11
|
+
|
|
12
|
+
// Mock tiktoken in a controllable way
|
|
13
|
+
const encMock = {
|
|
14
|
+
encode: vi.fn((s) => Array.from(String(s)).map(() => 1)), // 1 token per char
|
|
15
|
+
free: vi.fn(() => {}),
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
const tiktokenMocks = vi.hoisted(() => ({
|
|
19
|
+
encoding_for_model_impl: vi.fn(),
|
|
20
|
+
get_encoding_impl: vi.fn(() => encMock),
|
|
21
|
+
}));
|
|
22
|
+
|
|
23
|
+
vi.mock('@dqbd/tiktoken', () => ({
|
|
24
|
+
encoding_for_model: (model) => tiktokenMocks.encoding_for_model_impl(model),
|
|
25
|
+
get_encoding: (name) => tiktokenMocks.get_encoding_impl(name),
|
|
26
|
+
}));
|
|
27
|
+
|
|
28
|
+
// Import after mocks
|
|
29
|
+
import {
|
|
30
|
+
tokenEstimate,
|
|
31
|
+
getCurrentTokenLimit,
|
|
32
|
+
updateLimitsFromHeaders,
|
|
33
|
+
tokenUseCoolOff,
|
|
34
|
+
recordTokenUsage,
|
|
35
|
+
pruneOldTokenUsage,
|
|
36
|
+
__resetTokenControlForTests,
|
|
37
|
+
} from '../../tools/tokenControl.js';
|
|
38
|
+
|
|
39
|
+
describe('tokenControl', () => {
|
|
40
|
+
beforeEach(() => {
|
|
41
|
+
// reset env and mocks
|
|
42
|
+
process.env = { ...ORIGINAL_ENV };
|
|
43
|
+
__resetTokenControlForTests();
|
|
44
|
+
tiktokenMocks.encoding_for_model_impl.mockReset();
|
|
45
|
+
tiktokenMocks.get_encoding_impl.mockReset().mockReturnValue(encMock);
|
|
46
|
+
encMock.encode.mockClear();
|
|
47
|
+
encMock.free.mockClear();
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
afterEach(() => {
|
|
51
|
+
process.env = { ...ORIGINAL_ENV };
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
describe('tokenEstimate', () => {
|
|
55
|
+
it('uses encoding_for_model when available', async () => {
|
|
56
|
+
tiktokenMocks.encoding_for_model_impl.mockReturnValue(encMock);
|
|
57
|
+
|
|
58
|
+
const n = await tokenEstimate('gpt-4o', 'abcd'); // 4 chars → 4 tokens via mock
|
|
59
|
+
expect(n).toBe(4);
|
|
60
|
+
expect(encMock.encode).toHaveBeenCalledOnce();
|
|
61
|
+
expect(encMock.free).toHaveBeenCalledOnce();
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it('falls back to get_encoding when encoding_for_model throws', async () => {
|
|
65
|
+
tiktokenMocks.encoding_for_model_impl.mockImplementation(() => { throw new Error('no direct encoding'); });
|
|
66
|
+
tiktokenMocks.get_encoding_impl.mockReturnValue(encMock);
|
|
67
|
+
|
|
68
|
+
const n = await tokenEstimate('gemini-2.5-flash', 'abc'); // 3
|
|
69
|
+
expect(n).toBe(3);
|
|
70
|
+
expect(encMock.encode).toHaveBeenCalledOnce();
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it('falls back to bytes/4 when both tokenizers fail', async () => {
|
|
74
|
+
tiktokenMocks.encoding_for_model_impl.mockImplementation(() => { throw new Error('no direct encoding'); });
|
|
75
|
+
tiktokenMocks.get_encoding_impl.mockImplementation(() => { throw new Error('no base encoding'); });
|
|
76
|
+
|
|
77
|
+
const n = await tokenEstimate('unknown-model', 'abcdefgh'); // 8 bytes → 8/4 = 2
|
|
78
|
+
expect(n).toBe(2);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('stringifies non-string input', async () => {
|
|
82
|
+
tiktokenMocks.encoding_for_model_impl.mockReturnValue(encMock);
|
|
83
|
+
const n = await tokenEstimate('gpt-4o', { a: 1, b: 2 });
|
|
84
|
+
expect(encMock.encode).toHaveBeenCalled();
|
|
85
|
+
expect(typeof n).toBe('number');
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
describe('getCurrentTokenLimit / updateLimitsFromHeaders', () => {
|
|
90
|
+
it('uses defaults by model pattern', () => {
|
|
91
|
+
const openai = getCurrentTokenLimit('gpt-4o');
|
|
92
|
+
const gemini = getCurrentTokenLimit('gemini-2.5-flash');
|
|
93
|
+
expect(openai.tpm).toBeGreaterThan(0);
|
|
94
|
+
expect(gemini.tpm).toBeGreaterThan(0);
|
|
95
|
+
expect(['default', 'env', 'header']).toContain(openai.source);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it('honors ENV override TESTRONAUT_TOKENS_PER_MIN', () => {
|
|
99
|
+
process.env.TESTRONAUT_TOKENS_PER_MIN = '999';
|
|
100
|
+
const anyModel = getCurrentTokenLimit('what-ever-model');
|
|
101
|
+
expect(anyModel.tpm).toBe(999);
|
|
102
|
+
expect(anyModel.source).toBe('env');
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it('updates limits from headers (header wins)', () => {
|
|
106
|
+
const before = getCurrentTokenLimit('gpt-4o');
|
|
107
|
+
updateLimitsFromHeaders('gpt-4o', { 'x-ratelimit-limit-tokens': '1234' });
|
|
108
|
+
const after = getCurrentTokenLimit('gpt-4o');
|
|
109
|
+
expect(after.tpm).toBe(1234);
|
|
110
|
+
expect(after.source).toBe('header');
|
|
111
|
+
// sanity check it actually changed (unless defaults already 1234)
|
|
112
|
+
if (before.tpm !== 1234) expect(after.tpm).not.toBe(before.tpm);
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
describe('rolling window + cooldown', () => {
|
|
117
|
+
it('prunes old entries outside the window', () => {
|
|
118
|
+
const now = Date.now();
|
|
119
|
+
const entries = [
|
|
120
|
+
[now - 70000, 100], // old → pruned
|
|
121
|
+
[now - 10000, 50], // kept
|
|
122
|
+
[now - 5000, 30], // kept
|
|
123
|
+
];
|
|
124
|
+
const { turnTimestamps, totalTokensUsed } = pruneOldTokenUsage(entries, 60000);
|
|
125
|
+
expect(turnTimestamps.length).toBe(2);
|
|
126
|
+
expect(totalTokensUsed).toBe(80);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it('triggers backoff when usage exceeds TPM', async () => {
|
|
130
|
+
// small ENV cap to force backoff
|
|
131
|
+
process.env.TESTRONAUT_TOKENS_PER_MIN = '50';
|
|
132
|
+
const entries = [];
|
|
133
|
+
recordTokenUsage(entries, 60);
|
|
134
|
+
|
|
135
|
+
const result = await tokenUseCoolOff(60, entries, 'any-model');
|
|
136
|
+
expect(result.shouldBackoff).toBe(true);
|
|
137
|
+
expect(result.totalTokensUsed).toBe(0);
|
|
138
|
+
expect(Array.isArray(result.turnTimestamps)).toBe(true);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it('does not backoff when under TPM', async () => {
|
|
142
|
+
process.env.TESTRONAUT_TOKENS_PER_MIN = '100';
|
|
143
|
+
const entries = [];
|
|
144
|
+
recordTokenUsage(entries, 60);
|
|
145
|
+
const result = await tokenUseCoolOff(60, entries, 'any-model');
|
|
146
|
+
expect(result.shouldBackoff).toBe(false);
|
|
147
|
+
expect(result.totalTokensUsed).toBe(60);
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
});
|