pyyol 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +24 -0
- package/dist/cli.js +77 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +11 -0
- package/dist/instrument.d.ts +29 -3
- package/dist/instrument.js +307 -102
- package/dist/movetools.d.ts +141 -0
- package/dist/movetools.js +486 -0
- package/dist/pricing.d.ts +16 -4
- package/dist/pricing.js +55 -8
- package/dist/scaffold.d.ts +74 -0
- package/dist/scaffold.js +276 -0
- package/dist/telemetry.d.ts +24 -0
- package/dist/telemetry.js +62 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +6 -2
- package/rules/llms-full.txt +687 -26
- package/skill/SKILL.md +1 -0
- package/skill/references/telemetry.md +55 -0
package/dist/instrument.js
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
// pass `stream_options: { include_usage: true }` and record manually, or use
|
|
17
17
|
// non-streaming calls for automatic capture.
|
|
18
18
|
import { estimateCost } from "./pricing.js";
|
|
19
|
+
import { fromRequest, issue } from "./scaffold.js";
|
|
19
20
|
import * as providers from "./providers.js";
|
|
20
21
|
import { currentSpan, currentUsage } from "./telemetry.js";
|
|
21
22
|
// [prototype, method, original] for uninstrument().
|
|
@@ -35,11 +36,40 @@ export function disableGateway() {
|
|
|
35
36
|
gateway.key = "";
|
|
36
37
|
gateway.base = "";
|
|
37
38
|
}
|
|
38
|
-
|
|
39
|
-
|
|
39
|
+
// Which WIRE FORMAT a provider speaks. This decides the gateway path, and it is the only
|
|
40
|
+
// per-provider knowledge routing needs.
|
|
41
|
+
//
|
|
42
|
+
// WHY NOT A PROVIDER->PATH TABLE. There was one, listing openai and anthropic — not even groq.
|
|
43
|
+
// Every other provider resolved to "" and was therefore NOT ROUTED, silently: no proofs, no
|
|
44
|
+
// Verified badge, and in ranked play decisions that count as unproven. A table naming every
|
|
45
|
+
// provider is always one release behind the ecosystem, so the verified tier was structurally
|
|
46
|
+
// OpenAI-and-Anthropic-only.
|
|
47
|
+
//
|
|
48
|
+
// What actually varies is one bit: does the provider's own SDK append a version segment?
|
|
49
|
+
//
|
|
50
|
+
// OpenAI-wire clients call {base}/chat/completions -> the base must end in /v1
|
|
51
|
+
// Anthropic clients call {base}/v1/messages -> the base must NOT
|
|
52
|
+
// Google clients call {base}/v1beta/models/... -> the base must NOT
|
|
53
|
+
//
|
|
54
|
+
// Three cases, with OpenAI-wire the DEFAULT because most of the ecosystem speaks it. A provider
|
|
55
|
+
// nobody has heard of routes correctly on the day it ships.
|
|
56
|
+
const NO_VERSION_SUFFIX = new Set(["anthropic", "google", "vertex"]);
|
|
57
|
+
/**
|
|
58
|
+
* The baseURL a provider client should point at, or "" when routing is off or the provider
|
|
59
|
+
* cannot be routed.
|
|
60
|
+
*
|
|
61
|
+
* Returns "" for a LOCAL/self-hosted provider, deliberately: the gateway runs on Pyyol's side
|
|
62
|
+
* and cannot reach a model server on the developer's own machine, so pointing a client at it
|
|
63
|
+
* would break every call. That play is unverified — and also free, so no cost attribution is
|
|
64
|
+
* lost either.
|
|
65
|
+
*/
|
|
40
66
|
export function gatewayBaseUrl(provider) {
|
|
41
|
-
|
|
42
|
-
|
|
67
|
+
if (!gateway.base || !provider)
|
|
68
|
+
return "";
|
|
69
|
+
if (providers.isSelfHosted(provider))
|
|
70
|
+
return "";
|
|
71
|
+
const suffix = NO_VERSION_SUFFIX.has(provider.toLowerCase()) ? "" : "/v1";
|
|
72
|
+
return `${gateway.base}/gw/${provider}${suffix}`;
|
|
43
73
|
}
|
|
44
74
|
/** The X-Pyyol-* identity headers for the current turn (empty if routing off). */
|
|
45
75
|
export function gatewayHeaders() {
|
|
@@ -76,15 +106,14 @@ function detectProvider(client) {
|
|
|
76
106
|
* `patchedAs` is the SDK we wrapped (which wire format this is). The client's baseURL
|
|
77
107
|
* is consulted first and wins. A baseURL pointing at the PYYOL GATEWAY is not used for
|
|
78
108
|
* attribution — it says the call was proxied, not who served it — so the upstream is
|
|
79
|
-
* recovered
|
|
109
|
+
* recovered by PARSING the gateway path (/gw/<slug>[/v1]) rather than matching a table, so a
|
|
110
|
+
* provider added tomorrow is attributed correctly without touching this. */
|
|
80
111
|
export function resolveCallProvider(resource, patchedAs) {
|
|
81
112
|
const base = clientBaseUrl(resource);
|
|
82
113
|
if (base && gateway.base && base.startsWith(gateway.base)) {
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
return provider;
|
|
87
|
-
}
|
|
114
|
+
const parts = base.slice(gateway.base.length).replace(/^\/+|\/+$/g, "").split("/");
|
|
115
|
+
if (parts.length >= 2 && parts[0] === "gw" && parts[1])
|
|
116
|
+
return parts[1];
|
|
88
117
|
return patchedAs;
|
|
89
118
|
}
|
|
90
119
|
return providers.resolve({ baseUrl: base, fallback: patchedAs });
|
|
@@ -150,104 +179,260 @@ function get(obj, name, dflt) {
|
|
|
150
179
|
const v = obj[name];
|
|
151
180
|
return v === undefined ? dflt : v;
|
|
152
181
|
}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
/** Ollama's native shape: no `usage` object at all, counts at the top level.
|
|
157
|
-
* Without this an agent running Ollama reported zero tokens forever — it looked
|
|
158
|
-
* instrumented and measured nothing, and no bill ever arrives to contradict a zero. */
|
|
159
|
-
function extractOllama(resp) {
|
|
160
|
-
const prompt = get(resp, "prompt_eval_count");
|
|
161
|
-
const completion = get(resp, "eval_count");
|
|
162
|
-
if (prompt === undefined && completion === undefined)
|
|
163
|
-
return null;
|
|
164
|
-
return {
|
|
165
|
-
model: get(resp, "model", "") || "",
|
|
166
|
-
provider: providers.OLLAMA,
|
|
167
|
-
promptTokens: Math.trunc(prompt || 0),
|
|
168
|
-
completionTokens: Math.trunc(completion || 0),
|
|
169
|
-
cachedTokens: 0,
|
|
170
|
-
reasoningTokens: 0,
|
|
171
|
-
};
|
|
182
|
+
const MODEL_KEYS = new Set(["model", "modelversion", "modelid", "modelname", "modelslug"]);
|
|
183
|
+
function normalizeKey(key) {
|
|
184
|
+
return key.toLowerCase().replace(/[_\-.]/g, "");
|
|
172
185
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
const um = get(resp, "usageMetadata") ?? get(resp, "usage_metadata");
|
|
176
|
-
if (um == null)
|
|
177
|
-
return null;
|
|
178
|
-
const prompt = get(um, "promptTokenCount", get(um, "prompt_token_count", 0)) || 0;
|
|
179
|
-
const completion = get(um, "candidatesTokenCount", get(um, "candidates_token_count", 0)) || 0;
|
|
180
|
-
if (!prompt && !completion)
|
|
181
|
-
return null;
|
|
182
|
-
return {
|
|
183
|
-
model: get(resp, "modelVersion", "") || get(resp, "model", "") || "",
|
|
184
|
-
provider: providers.GOOGLE,
|
|
185
|
-
promptTokens: Math.trunc(prompt),
|
|
186
|
-
completionTokens: Math.trunc(completion),
|
|
187
|
-
cachedTokens: Math.trunc(get(um, "cachedContentTokenCount", get(um, "cached_content_token_count", 0)) || 0),
|
|
188
|
-
reasoningTokens: Math.trunc(get(um, "thoughtsTokenCount", get(um, "thoughts_token_count", 0)) || 0),
|
|
189
|
-
};
|
|
186
|
+
function isModelKey(key) {
|
|
187
|
+
return MODEL_KEYS.has(normalizeKey(key));
|
|
190
188
|
}
|
|
191
|
-
/**
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
189
|
+
/**
|
|
190
|
+
* What a response field MEANS, independent of what it is called.
|
|
191
|
+
*
|
|
192
|
+
* Ordered most-specific-first: "cache_read_input_tokens" contains both "cache" and "input" and
|
|
193
|
+
* must read as a cache field, not an input count. Getting that order wrong would make Anthropic's
|
|
194
|
+
* cache read look like its input total.
|
|
195
|
+
*/
|
|
196
|
+
function classifyUsageKey(key) {
|
|
197
|
+
const k = key.toLowerCase().replace(/-/g, "_");
|
|
198
|
+
if (k.includes("cach")) {
|
|
199
|
+
if (k.includes("creat") || k.includes("writ"))
|
|
200
|
+
return "cacheWrite";
|
|
201
|
+
// A MISS is ordinary uncached input, already inside the prompt total that accompanies it
|
|
202
|
+
// (DeepSeek documents prompt == hit + miss), so counting it would double-bill.
|
|
203
|
+
if (k.includes("miss"))
|
|
204
|
+
return null;
|
|
205
|
+
if (k.includes("read") || k.includes("hit") || k.includes("cached"))
|
|
206
|
+
return "cacheRead";
|
|
207
|
+
// A bare cache count with no direction: read is the cheaper and therefore conservative
|
|
208
|
+
// reading — overstating a discount is worse than understating it.
|
|
209
|
+
return "cacheRead";
|
|
210
|
+
}
|
|
211
|
+
if (k.includes("reasoning") || k.includes("thought"))
|
|
212
|
+
return "reasoning";
|
|
213
|
+
if (k.includes("total"))
|
|
214
|
+
return "total";
|
|
215
|
+
// Output before input: these are unambiguous, and doing them first keeps the input rules from
|
|
216
|
+
// having to exclude them.
|
|
217
|
+
if (k.includes("completion") || k.includes("output") || k.includes("candidates") || k === "eval_count") {
|
|
218
|
+
return "output";
|
|
219
|
+
}
|
|
220
|
+
if (k.includes("prompt"))
|
|
221
|
+
return "inputPrompt";
|
|
222
|
+
if (k.includes("input"))
|
|
223
|
+
return "inputFresh";
|
|
224
|
+
return null;
|
|
225
|
+
}
|
|
226
|
+
function fieldsOf(node) {
|
|
227
|
+
if (node === null || node === undefined)
|
|
195
228
|
return null;
|
|
196
|
-
|
|
197
|
-
const completion = get(tokens, "outputTokens", get(tokens, "output_tokens", 0)) || 0;
|
|
198
|
-
if (!prompt && !completion)
|
|
229
|
+
if (typeof node !== "object")
|
|
199
230
|
return null;
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
promptTokens: Math.trunc(prompt),
|
|
204
|
-
completionTokens: Math.trunc(completion),
|
|
205
|
-
cachedTokens: 0,
|
|
206
|
-
reasoningTokens: 0,
|
|
207
|
-
};
|
|
231
|
+
if (Array.isArray(node))
|
|
232
|
+
return null;
|
|
233
|
+
return node;
|
|
208
234
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
235
|
+
/**
|
|
236
|
+
* Accumulate every usage-shaped number under `node`, by concept.
|
|
237
|
+
*
|
|
238
|
+
* Takes the MAXIMUM per concept rather than the last value seen: responses repeat counts (a
|
|
239
|
+
* streamed body carries them across frames), and a later zero for a field the provider is not
|
|
240
|
+
* reporting in that frame would erase a real count already found.
|
|
241
|
+
*/
|
|
242
|
+
function harvestUsage(node, acc, depth = 0) {
|
|
243
|
+
if (depth > 12)
|
|
244
|
+
return;
|
|
245
|
+
if (Array.isArray(node)) {
|
|
246
|
+
for (const item of node)
|
|
247
|
+
harvestUsage(item, acc, depth + 1);
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
const fields = fieldsOf(node);
|
|
251
|
+
if (!fields)
|
|
252
|
+
return;
|
|
253
|
+
for (const key of Object.keys(fields).sort()) {
|
|
254
|
+
const value = fields[key];
|
|
255
|
+
if (isModelKey(key) && typeof value === "string" && value && !acc.model) {
|
|
256
|
+
acc.model = value;
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
260
|
+
const concept = classifyUsageKey(key);
|
|
261
|
+
if (concept && value > 0 && value > (acc[concept] ?? 0)) {
|
|
262
|
+
acc[concept] = Math.trunc(value);
|
|
263
|
+
acc.found = true;
|
|
264
|
+
}
|
|
265
|
+
continue;
|
|
266
|
+
}
|
|
267
|
+
harvestUsage(value, acc, depth + 1);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
const USAGE_CONTAINER_HINTS = ["usage", "tokens", "accounting", "billing"];
|
|
271
|
+
/** Collect every non-scalar value whose key satisfies `match`, in deterministic order. */
|
|
272
|
+
function findByKey(node, match, depth = 0) {
|
|
273
|
+
if (depth > 12)
|
|
274
|
+
return [];
|
|
275
|
+
const out = [];
|
|
276
|
+
if (Array.isArray(node)) {
|
|
277
|
+
for (const item of node)
|
|
278
|
+
out.push(...findByKey(item, match, depth + 1));
|
|
279
|
+
return out;
|
|
280
|
+
}
|
|
281
|
+
const fields = fieldsOf(node);
|
|
282
|
+
if (!fields)
|
|
283
|
+
return out;
|
|
284
|
+
for (const key of Object.keys(fields).sort()) {
|
|
285
|
+
const v = fields[key];
|
|
286
|
+
if (v !== null && typeof v === "object") {
|
|
287
|
+
if (match(key)) {
|
|
288
|
+
out.push(v); // the envelope is the unit; do not also descend into it
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
out.push(...findByKey(v, match, depth + 1));
|
|
218
292
|
}
|
|
293
|
+
}
|
|
294
|
+
return out;
|
|
295
|
+
}
|
|
296
|
+
function harvestModel(node, acc, depth = 0) {
|
|
297
|
+
if (depth > 12 || acc.model)
|
|
298
|
+
return;
|
|
299
|
+
if (Array.isArray(node)) {
|
|
300
|
+
for (const item of node)
|
|
301
|
+
harvestModel(item, acc, depth + 1);
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
const fields = fieldsOf(node);
|
|
305
|
+
if (!fields)
|
|
306
|
+
return;
|
|
307
|
+
for (const key of Object.keys(fields).sort()) {
|
|
308
|
+
if (isModelKey(key) && typeof fields[key] === "string" && fields[key]) {
|
|
309
|
+
acc.model = fields[key];
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
for (const key of Object.keys(fields).sort())
|
|
314
|
+
harvestModel(fields[key], acc, depth + 1);
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* Read the CANONICAL usage envelope where one exists, and only scan more widely when there is none.
|
|
318
|
+
*
|
|
319
|
+
* WHY PRIORITY RATHER THAN "take the largest match". Once the walk is general, an unrelated number
|
|
320
|
+
* elsewhere in a response can be read as a token count — a proxy echoing several dialects, or a
|
|
321
|
+
* provider that left a legacy field in place, produces a body carrying both a real `usage` object
|
|
322
|
+
* and a stray `prompt_eval_count`. Keeping the largest value would let whichever number happened
|
|
323
|
+
* to be bigger decide what the developer is charged.
|
|
324
|
+
*
|
|
325
|
+
* `usage` is what OpenAI, Anthropic, Bedrock, Mistral and every OpenAI-wire server fill in. The
|
|
326
|
+
* alternatives are what a provider uses INSTEAD of it, never alongside — so it wins outright.
|
|
327
|
+
*/
|
|
328
|
+
function harvestPreferringEnvelope(resp, acc) {
|
|
329
|
+
harvestModel(resp, acc);
|
|
330
|
+
// Tier 1: the canonical envelope, wherever it sits (Anthropic's streaming shape nests it inside
|
|
331
|
+
// `message`, so this cannot be a top-level-only lookup).
|
|
332
|
+
const envelopes = findByKey(resp, (k) => normalizeKey(k) === "usage");
|
|
333
|
+
if (envelopes.length) {
|
|
334
|
+
for (const e of envelopes)
|
|
335
|
+
harvestUsage(e, acc);
|
|
336
|
+
if (acc.found)
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
// Tier 2: a differently-named accounting container (Google's usageMetadata, Cohere's meta.tokens).
|
|
340
|
+
const containers = findByKey(resp, (k) => {
|
|
341
|
+
const lk = k.toLowerCase();
|
|
342
|
+
return USAGE_CONTAINER_HINTS.some((h) => lk.includes(h));
|
|
343
|
+
});
|
|
344
|
+
if (containers.length) {
|
|
345
|
+
for (const c of containers)
|
|
346
|
+
harvestUsage(c, acc);
|
|
347
|
+
if (acc.found)
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
// Tier 3: no envelope at all. Ollama natively puts its counts at the TOP LEVEL, and a local model
|
|
351
|
+
// bills nothing — so a zero there is never contradicted by an invoice, which is exactly why it
|
|
352
|
+
// went unnoticed. Scan everything rather than report it as free.
|
|
353
|
+
harvestUsage(resp, acc);
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* A vendor label inferred from DISTINCTIVE container names, not from token field names.
|
|
357
|
+
*
|
|
358
|
+
* Kept because the label is user-visible and feeds pricing: an open-weight model is free when
|
|
359
|
+
* self-hosted and billed when a hosted provider serves it, with the same model id either way. Only
|
|
360
|
+
* markers that genuinely identify one vendor are listed, and this is a LAST-RESORT hint — the
|
|
361
|
+
* caller's own resolution from the client's baseURL wins wherever it has one.
|
|
362
|
+
*
|
|
363
|
+
* A plain `usage` envelope OUTRANKS every marker: a response can carry both, and the envelope the
|
|
364
|
+
* provider actually filled in is the one that decides.
|
|
365
|
+
*/
|
|
366
|
+
function vendorFromMarkers(resp) {
|
|
367
|
+
const fields = fieldsOf(resp) ?? {};
|
|
368
|
+
const keys = new Set(Object.keys(fields).map(normalizeKey));
|
|
369
|
+
if (keys.has("usage"))
|
|
370
|
+
return "";
|
|
371
|
+
if (keys.has("usagemetadata"))
|
|
372
|
+
return providers.GOOGLE;
|
|
373
|
+
if (keys.has("promptevalcount") || keys.has("evalcount"))
|
|
374
|
+
return providers.OLLAMA;
|
|
375
|
+
const meta = fieldsOf(fields["meta"]);
|
|
376
|
+
if (meta && meta["tokens"] !== undefined)
|
|
377
|
+
return "cohere";
|
|
378
|
+
return "";
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* Pull normalized usage from ANY provider response, or null if it carries none.
|
|
382
|
+
*
|
|
383
|
+
* Normalizes onto ONE convention: promptTokens is the total billable input, with cache reads and
|
|
384
|
+
* writes as SUBSETS of it. Providers genuinely disagree and the disagreement is silent, so the rule
|
|
385
|
+
* follows the WORD USED rather than a vendor list:
|
|
386
|
+
*
|
|
387
|
+
* - a PROMPT-family key names the whole prompt, so cache is already inside it
|
|
388
|
+
* - an INPUT-family key names fresh input, so cache is billed on top
|
|
389
|
+
*
|
|
390
|
+
* A reported total cross-checks the additive case, so a provider using "input" for a
|
|
391
|
+
* cache-inclusive total is corrected by its own arithmetic instead of being over-counted.
|
|
392
|
+
*/
|
|
393
|
+
export function extractUsage(resp) {
|
|
394
|
+
const acc = {};
|
|
395
|
+
harvestPreferringEnvelope(resp, acc);
|
|
396
|
+
if (!acc.found)
|
|
219
397
|
return null;
|
|
398
|
+
const cacheRead = acc.cacheRead ?? 0;
|
|
399
|
+
const cacheWrite = acc.cacheWrite ?? 0;
|
|
400
|
+
const completion = acc.output ?? 0;
|
|
401
|
+
const total = acc.total ?? 0;
|
|
402
|
+
const inPrompt = acc.inputPrompt ?? 0;
|
|
403
|
+
const inFresh = acc.inputFresh ?? 0;
|
|
404
|
+
let prompt;
|
|
405
|
+
if (inPrompt > 0) {
|
|
406
|
+
prompt = inPrompt;
|
|
407
|
+
}
|
|
408
|
+
else if (inFresh > 0) {
|
|
409
|
+
prompt = inFresh + cacheRead + cacheWrite;
|
|
410
|
+
if (total > 0 && prompt > total - completion && total - completion >= inFresh) {
|
|
411
|
+
prompt = total - completion;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
else {
|
|
415
|
+
// No input count, but cache counts present: the cache IS the input we know about.
|
|
416
|
+
prompt = cacheRead + cacheWrite;
|
|
417
|
+
}
|
|
418
|
+
// A cache count larger than the prompt total cannot be a subset of it. Raise the total rather
|
|
419
|
+
// than let pricing clamp the excess away as if it had never been billed.
|
|
420
|
+
prompt = Math.max(prompt, cacheRead + cacheWrite);
|
|
421
|
+
let provider = vendorFromMarkers(resp);
|
|
422
|
+
if (!provider) {
|
|
423
|
+
if (inPrompt > 0)
|
|
424
|
+
provider = "openai";
|
|
425
|
+
else if (inFresh > 0)
|
|
426
|
+
provider = "anthropic";
|
|
220
427
|
}
|
|
221
|
-
const model = get(resp, "model", "") || "";
|
|
222
|
-
let prompt = get(u, "prompt_tokens");
|
|
223
|
-
let completion = get(u, "completion_tokens");
|
|
224
|
-
const styleOpenAiChat = prompt !== undefined || completion !== undefined;
|
|
225
|
-
if (prompt === undefined)
|
|
226
|
-
prompt = get(u, "input_tokens", 0);
|
|
227
|
-
if (completion === undefined)
|
|
228
|
-
completion = get(u, "output_tokens", 0);
|
|
229
|
-
let cached = 0;
|
|
230
|
-
let reasoning = 0;
|
|
231
|
-
const ptd = get(u, "prompt_tokens_details");
|
|
232
|
-
if (ptd != null)
|
|
233
|
-
cached = get(ptd, "cached_tokens", 0) || 0;
|
|
234
|
-
const ctd = get(u, "completion_tokens_details");
|
|
235
|
-
if (ctd != null)
|
|
236
|
-
reasoning = get(ctd, "reasoning_tokens", 0) || 0;
|
|
237
|
-
if (!cached)
|
|
238
|
-
cached = get(u, "cache_read_input_tokens", 0) || 0;
|
|
239
|
-
let provider = "";
|
|
240
|
-
if (styleOpenAiChat)
|
|
241
|
-
provider = "openai";
|
|
242
|
-
else if (get(u, "input_tokens") !== undefined)
|
|
243
|
-
provider = "anthropic";
|
|
244
428
|
return {
|
|
245
|
-
model,
|
|
429
|
+
model: acc.model ?? "",
|
|
246
430
|
provider,
|
|
247
|
-
promptTokens: Math.trunc(prompt
|
|
248
|
-
completionTokens: Math.trunc(completion
|
|
249
|
-
cachedTokens: Math.trunc(
|
|
250
|
-
|
|
431
|
+
promptTokens: Math.trunc(prompt),
|
|
432
|
+
completionTokens: Math.trunc(completion),
|
|
433
|
+
cachedTokens: Math.trunc(cacheRead),
|
|
434
|
+
cachedWriteTokens: Math.trunc(cacheWrite),
|
|
435
|
+
reasoningTokens: Math.trunc(acc.reasoning ?? 0),
|
|
251
436
|
};
|
|
252
437
|
}
|
|
253
438
|
/** Record usage from a provider response: compute cost, add to the turn
|
|
@@ -266,6 +451,7 @@ export function recordResponse(resp, o = {}) {
|
|
|
266
451
|
promptTokens: info.promptTokens,
|
|
267
452
|
completionTokens: info.completionTokens,
|
|
268
453
|
cachedTokens: info.cachedTokens,
|
|
454
|
+
cachedWriteTokens: info.cachedWriteTokens,
|
|
269
455
|
reasoningTokens: info.reasoningTokens,
|
|
270
456
|
});
|
|
271
457
|
currentUsage()?.add({
|
|
@@ -275,7 +461,9 @@ export function recordResponse(resp, o = {}) {
|
|
|
275
461
|
completionTokens: info.completionTokens,
|
|
276
462
|
reasoningTokens: info.reasoningTokens,
|
|
277
463
|
cachedTokens: info.cachedTokens,
|
|
464
|
+
cachedWriteTokens: info.cachedWriteTokens,
|
|
278
465
|
estimatedCost: cost,
|
|
466
|
+
latencyMs: o.latencyMs ?? 0,
|
|
279
467
|
});
|
|
280
468
|
currentSpan().logModelCall({
|
|
281
469
|
provider,
|
|
@@ -290,7 +478,7 @@ export function recordResponse(resp, o = {}) {
|
|
|
290
478
|
}
|
|
291
479
|
/** @internal Wrap `proto[method]` so its resolved return value is recorded.
|
|
292
480
|
* Idempotent and fully guarded. Exported for tests. */
|
|
293
|
-
export function patchPrototype(proto, method, provider) {
|
|
481
|
+
export function patchPrototype(proto, method, provider, endpoint = "") {
|
|
294
482
|
if (proto == null)
|
|
295
483
|
return false;
|
|
296
484
|
const orig = proto[method];
|
|
@@ -298,6 +486,18 @@ export function patchPrototype(proto, method, provider) {
|
|
|
298
486
|
return false;
|
|
299
487
|
const wrapped = async function (...args) {
|
|
300
488
|
injectGatewayHeaders(this, args);
|
|
489
|
+
// Fingerprint the scaffold from the OUTGOING request: the system prompt, tools and
|
|
490
|
+
// sampling are what the developer wrote, and none of that comes back in the response.
|
|
491
|
+
// Guarded like every other hook — a fingerprinting problem must never be why a
|
|
492
|
+
// developer's model call fails.
|
|
493
|
+
try {
|
|
494
|
+
const req = (args[0] ?? {});
|
|
495
|
+
const fp = fromRequest(req, endpoint);
|
|
496
|
+
currentUsage()?.observeScaffold(fp, fp ? "" : issue(req, endpoint));
|
|
497
|
+
}
|
|
498
|
+
catch {
|
|
499
|
+
// instrumentation must never break the dev's call
|
|
500
|
+
}
|
|
301
501
|
const start = Date.now();
|
|
302
502
|
const resp = await orig.apply(this, args);
|
|
303
503
|
try {
|
|
@@ -327,17 +527,22 @@ async function patchOpenAI() {
|
|
|
327
527
|
let patched = false;
|
|
328
528
|
const chat = await tryImport("openai/resources/chat/completions");
|
|
329
529
|
if (chat?.Completions?.prototype)
|
|
330
|
-
patched =
|
|
530
|
+
patched =
|
|
531
|
+
patchPrototype(chat.Completions.prototype, "create", "openai", "openai.chat.completions") ||
|
|
532
|
+
patched;
|
|
331
533
|
const responses = await tryImport("openai/resources/responses");
|
|
332
534
|
if (responses?.Responses?.prototype)
|
|
333
|
-
patched =
|
|
535
|
+
patched =
|
|
536
|
+
patchPrototype(responses.Responses.prototype, "create", "openai", "openai.responses") || patched;
|
|
334
537
|
return patched;
|
|
335
538
|
}
|
|
336
539
|
async function patchAnthropic() {
|
|
337
540
|
let patched = false;
|
|
338
541
|
const messages = await tryImport("@anthropic-ai/sdk/resources/messages");
|
|
339
542
|
if (messages?.Messages?.prototype)
|
|
340
|
-
patched =
|
|
543
|
+
patched =
|
|
544
|
+
patchPrototype(messages.Messages.prototype, "create", "anthropic", "anthropic.messages") ||
|
|
545
|
+
patched;
|
|
341
546
|
return patched;
|
|
342
547
|
}
|
|
343
548
|
/** Auto-capture LLM usage from installed providers. Pass e.g. `["openai"]` to limit
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
export declare const TOOL_GOOFSPIEL = "play_card";
|
|
2
|
+
export declare const TOOL_MAFIA = "mafia_action";
|
|
3
|
+
export declare const TOOL_MONOPOLY = "monopoly_action";
|
|
4
|
+
export declare const GAME_GOOFSPIEL = "goofspiel";
|
|
5
|
+
export declare const GAME_MAFIA = "mafia";
|
|
6
|
+
export declare const GAME_MONOPOLY = "monopoly";
|
|
7
|
+
/**
|
|
8
|
+
* NO_TARGET is the wire convention for "this action names no seat".
|
|
9
|
+
*
|
|
10
|
+
* NOT zero, and worth reading twice: seat 0 is a real player. A forgotten target used to act
|
|
11
|
+
* silently on seat 0, which is why MafiaMove defaults to -1 — and the canonical form has to
|
|
12
|
+
* agree, or every untargeted action would bind as an action against that player.
|
|
13
|
+
*/
|
|
14
|
+
export declare const NO_TARGET = -1;
|
|
15
|
+
/** The tool name that carries a move for `game`, or "" if the game has no contract. */
|
|
16
|
+
export declare function moveToolName(game: string): string;
|
|
17
|
+
export declare function moveTool(game: string, provider?: string, planRounds?: number): Record<string, unknown>;
|
|
18
|
+
/**
|
|
19
|
+
* The provider-specific way to REQUIRE the move tool.
|
|
20
|
+
*
|
|
21
|
+
* Worth using. Without it a model may answer in prose, and a turn with no tool call is
|
|
22
|
+
* unverified — the agent keeps playing but earns no completion binding.
|
|
23
|
+
*/
|
|
24
|
+
export declare function moveToolChoice(game: string, provider?: string): unknown;
|
|
25
|
+
/**
|
|
26
|
+
* The move arguments the model emitted, or null if it emitted no usable move call.
|
|
27
|
+
*
|
|
28
|
+
* STRUCTURAL, not per-provider. Every provider that has ever expressed a tool call has
|
|
29
|
+
* expressed it as a name beside an arguments blob, as SIBLINGS in one object:
|
|
30
|
+
*
|
|
31
|
+
* OpenAI {"function": {"name": "play_card", "arguments": "{\"card\":7}"}}
|
|
32
|
+
* Responses {"type": "function_call", "name": "play_card", "arguments": "{...}"}
|
|
33
|
+
* Anthropic {"type": "tool_use", "name": "play_card", "input": {"card": 7}}
|
|
34
|
+
* Google {"functionCall": {"name": "play_card", "args": {"card": 7}}}
|
|
35
|
+
* Bedrock {"toolUse": {"name": "play_card", "input": {"card": 7}}}
|
|
36
|
+
* Ollama {"function": {"name": "play_card", "arguments": {"card": 7}}}
|
|
37
|
+
*
|
|
38
|
+
* So the walk looks for that structure anywhere in the document and a provider nobody has heard
|
|
39
|
+
* of works on the day it ships. Enumerating shapes loses by construction: new providers appear
|
|
40
|
+
* constantly, every self-hosted server has its own dialect, and an unlisted one fails SILENTLY —
|
|
41
|
+
* the turn is never bound and nobody learns why.
|
|
42
|
+
*
|
|
43
|
+
* SAFE because the tool NAME is the discriminator and it is ours. The one near-miss is a response
|
|
44
|
+
* echoing the tool DEFINITION, which is why "parameters" is NOT accepted as an arguments key: a
|
|
45
|
+
* JSON Schema yields no card and falls through to null rather than to a wrong move. That
|
|
46
|
+
* direction matters — a wrong move REJECTS an honest turn, a miss only leaves it unverified.
|
|
47
|
+
*
|
|
48
|
+
* Returns the LAST matching call: a model that corrected itself stands behind its final answer.
|
|
49
|
+
*/
|
|
50
|
+
export declare function moveFromResponse(game: string, resp: unknown): Record<string, unknown> | null;
|
|
51
|
+
/** The bound form of a Goofspiel move: the card, and nothing else. */
|
|
52
|
+
export declare function canonGoofspiel(card: number): string;
|
|
53
|
+
/**
|
|
54
|
+
* The bound form of a Mafia action: the verb and its target seat.
|
|
55
|
+
*
|
|
56
|
+
* The PHASE is deliberately excluded — it is server state, not the model's choice, and binding
|
|
57
|
+
* it would reject an honest turn over a field the model had no say in.
|
|
58
|
+
*
|
|
59
|
+
* Negative targets collapse to one token; ZERO DOES NOT. Seat 0 is an ordinary player, and
|
|
60
|
+
* abstaining is its own action kind rather than a sentinel target, so "no seat" is only ever an
|
|
61
|
+
* absent or negative field. Collapsing 0 too would let a move against that one player be
|
|
62
|
+
* substituted for doing nothing.
|
|
63
|
+
*/
|
|
64
|
+
export declare function canonMafia(kind: string, target: number): string;
|
|
65
|
+
/**
|
|
66
|
+
* The bound form of a Monopoly action: verb, property, amount.
|
|
67
|
+
*
|
|
68
|
+
* All three are always rendered, including zeros. Omitting an absent field would let "mortgage
|
|
69
|
+
* property 0 for 50" and "mortgage property 50 for 0" reduce to the same string, and two
|
|
70
|
+
* different decisions sharing one canonical form is the one thing this mechanism cannot tolerate.
|
|
71
|
+
*/
|
|
72
|
+
export declare function canonMonopoly(kind: string, property?: number, amount?: number): string;
|
|
73
|
+
/**
|
|
74
|
+
* Reduce move arguments to the canonical string a bound decision stores.
|
|
75
|
+
*
|
|
76
|
+
* null means "nothing bindable here", which callers must treat as an unverified turn and never
|
|
77
|
+
* as a wrong move.
|
|
78
|
+
*/
|
|
79
|
+
export declare function canonMove(game: string, args: Record<string, unknown> | null): string | null;
|
|
80
|
+
/**
|
|
81
|
+
* The canonical move the platform will bind for this response, or null.
|
|
82
|
+
*
|
|
83
|
+
* The one call worth making in a test: it is exactly what the gateway does, so an agent that
|
|
84
|
+
* asserts on this locally cannot be surprised by a rejection in a real match.
|
|
85
|
+
*/
|
|
86
|
+
export declare function boundMove(game: string, resp: unknown): string | null;
|
|
87
|
+
/**
|
|
88
|
+
* The argument that carries a multi-round decision.
|
|
89
|
+
*
|
|
90
|
+
* Named once, and it must match the Go gateway and the Python SDK exactly: a mismatch would
|
|
91
|
+
* not throw, it would silently fall back to single-round binding and quietly restore the
|
|
92
|
+
* coverage problem range bindings exist to fix.
|
|
93
|
+
*/
|
|
94
|
+
export declare const PLAN_KEY = "plan";
|
|
95
|
+
/**
|
|
96
|
+
* The most rounds one completion may claim to have decided.
|
|
97
|
+
*
|
|
98
|
+
* Bounded because the plan is attacker-supplied — uncapped, a single call could assert a
|
|
99
|
+
* hundred thousand rounds and become that many database writes. Comfortably above any real
|
|
100
|
+
* game, so a legitimate agent never meets it.
|
|
101
|
+
*/
|
|
102
|
+
export declare const MAX_SPAN_ROUNDS = 64;
|
|
103
|
+
/** One round a completion decided. */
|
|
104
|
+
export interface RoundMove {
|
|
105
|
+
round: number;
|
|
106
|
+
move: string;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Reduce move arguments to EVERY round they decided.
|
|
110
|
+
*
|
|
111
|
+
* # Why a completion may cover more than one round
|
|
112
|
+
*
|
|
113
|
+
* Coverage used to count CALLS, so one completion bound one round. An agent that batches — one
|
|
114
|
+
* call planning three rounds — therefore scored about 33% on real staked tables while playing
|
|
115
|
+
* entirely model-backed, and cost optimisation is something this platform means to REWARD.
|
|
116
|
+
* Coverage now means "decisions a model made" rather than "calls made".
|
|
117
|
+
*
|
|
118
|
+
* # Why claiming a span is safe
|
|
119
|
+
*
|
|
120
|
+
* A span is a COMMITMENT, not a free coverage win. Match-time enforcement is unchanged, so
|
|
121
|
+
* submitting anything other than the bound move for a covered round is rejected exactly as a
|
|
122
|
+
* substitution is. An agent that over-claims has only tied its own hands.
|
|
123
|
+
*
|
|
124
|
+
* # The one thing a span must never do
|
|
125
|
+
*
|
|
126
|
+
* Rounds before `provenRound` are DROPPED. Those turns have already been played, so a binding
|
|
127
|
+
* over them is coverage nothing will ever check — an agent could retroactively claim turns it
|
|
128
|
+
* played unbound. Forward claims are self-limiting because they are enforced.
|
|
129
|
+
*
|
|
130
|
+
* null means nothing is bindable. A plan naming one round twice returns null WHOLE: two moves
|
|
131
|
+
* for one slot has no honest reading, and picking either would be guessing for the agent.
|
|
132
|
+
*/
|
|
133
|
+
export declare function canonPlan(game: string, args: Record<string, unknown> | null, provenRound: number): RoundMove[] | null;
|
|
134
|
+
/**
|
|
135
|
+
* Every round this response will bind, exactly as the gateway will read it.
|
|
136
|
+
*
|
|
137
|
+
* Worth calling in a test before shipping a batching agent: if this does not list the round you
|
|
138
|
+
* are about to play, that turn will not be bound, and if it lists a DIFFERENT move than you
|
|
139
|
+
* intend to submit, the match will reject it.
|
|
140
|
+
*/
|
|
141
|
+
export declare function boundPlan(game: string, resp: unknown, provenRound: number): RoundMove[] | null;
|