wayfind 2.0.80 → 2.0.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/connectors/llm.js
CHANGED
|
@@ -144,7 +144,7 @@ async function callAnthropic(config, systemPrompt, userContent) {
|
|
|
144
144
|
const payload = JSON.stringify({
|
|
145
145
|
model: config.model,
|
|
146
146
|
max_tokens: config.max_tokens || DEFAULT_MAX_TOKENS,
|
|
147
|
-
system: systemPrompt,
|
|
147
|
+
system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }],
|
|
148
148
|
messages: [{ role: 'user', content: userContent }],
|
|
149
149
|
});
|
|
150
150
|
|
|
@@ -327,13 +327,35 @@ async function callWithTools(config, systemPrompt, userContent, tools, handleToo
|
|
|
327
327
|
const MAX_ITERATIONS = 10;
|
|
328
328
|
let messages = [{ role: 'user', content: userContent }];
|
|
329
329
|
|
|
330
|
+
// Breakpoint on system prompt and last tool definition: the system+tools prefix
|
|
331
|
+
// is stable across all iterations, so these hits are what pay for the loop.
|
|
332
|
+
const cachedSystem = [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }];
|
|
333
|
+
const cachedTools = tools.map((t, idx) =>
|
|
334
|
+
idx === tools.length - 1 ? { ...t, cache_control: { type: 'ephemeral' } } : t
|
|
335
|
+
);
|
|
336
|
+
|
|
330
337
|
for (let i = 0; i < MAX_ITERATIONS; i++) {
|
|
338
|
+
// Breakpoint on the last message so the growing transcript also caches
|
|
339
|
+
// across iterations after the first.
|
|
340
|
+
const cachedMessages = messages.map((m, idx) => {
|
|
341
|
+
if (idx !== messages.length - 1) return m;
|
|
342
|
+
if (typeof m.content === 'string') {
|
|
343
|
+
return { ...m, content: [{ type: 'text', text: m.content, cache_control: { type: 'ephemeral' } }] };
|
|
344
|
+
}
|
|
345
|
+
if (Array.isArray(m.content) && m.content.length > 0) {
|
|
346
|
+
const last = m.content[m.content.length - 1];
|
|
347
|
+
const patched = { ...last, cache_control: { type: 'ephemeral' } };
|
|
348
|
+
return { ...m, content: [...m.content.slice(0, -1), patched] };
|
|
349
|
+
}
|
|
350
|
+
return m;
|
|
351
|
+
});
|
|
352
|
+
|
|
331
353
|
const payload = JSON.stringify({
|
|
332
354
|
model: config.model,
|
|
333
355
|
max_tokens: config.max_tokens || DEFAULT_MAX_TOKENS,
|
|
334
|
-
system:
|
|
335
|
-
messages,
|
|
336
|
-
tools,
|
|
356
|
+
system: cachedSystem,
|
|
357
|
+
messages: cachedMessages,
|
|
358
|
+
tools: cachedTools,
|
|
337
359
|
});
|
|
338
360
|
|
|
339
361
|
const res = await httpPost(ANTHROPIC_API_URL, headers, payload);
|
package/package.json
CHANGED