pyyol 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -4
- package/dist/adapter.d.ts +3 -3
- package/dist/adapter.js +8 -1
- package/dist/cli.js +187 -6
- package/dist/index.d.ts +7 -2
- package/dist/index.js +4 -1
- package/dist/install-ping.d.ts +2 -0
- package/dist/install-ping.js +42 -0
- package/dist/instrument.d.ts +41 -0
- package/dist/instrument.js +276 -0
- package/dist/models.d.ts +4 -0
- package/dist/pricing.d.ts +27 -0
- package/dist/pricing.js +110 -0
- package/dist/runtime.d.ts +1 -0
- package/dist/runtime.js +32 -7
- package/dist/server.js +10 -2
- package/dist/telemetry.d.ts +51 -0
- package/dist/telemetry.js +68 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
- package/rules/llms-full.txt +258 -26
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
// Automatic, zero-config LLM usage capture (mirrors the Python SDK's instrument.py).
|
|
2
|
+
//
|
|
3
|
+
// Call `instrument()` once at startup and the SDK transparently wraps the OpenAI
|
|
4
|
+
// and Anthropic clients: every non-streaming completion has its REAL model, token
|
|
5
|
+
// counts, and cost recorded from the provider's own response — no boilerplate. The
|
|
6
|
+
// runtime auto-attaches the captured usage to the outgoing move, so it reaches the
|
|
7
|
+
// arena benchmark even with Lens disabled.
|
|
8
|
+
//
|
|
9
|
+
// import { instrument } from "pyyol";
|
|
10
|
+
// await instrument(); // once, at startup
|
|
11
|
+
//
|
|
12
|
+
// Design: never imports a provider that isn't installed; every capture path is
|
|
13
|
+
// guarded so instrumentation can never throw into the developer's call; idempotent;
|
|
14
|
+
// uninstrument() restores originals (used by tests).
|
|
15
|
+
// Limitation (Phase 2): streaming responses carry no usage on the returned stream;
|
|
16
|
+
// pass `stream_options: { include_usage: true }` and record manually, or use
|
|
17
|
+
// non-streaming calls for automatic capture.
|
|
18
|
+
import { estimateCost } from "./pricing.js";
|
|
19
|
+
import { currentSpan, currentUsage } from "./telemetry.js";
|
|
20
|
+
// [prototype, method, original] for uninstrument().
|
|
21
|
+
const PATCHED = [];
|
|
22
|
+
// --- Verified-tier gateway routing (Phase 4c) ---------------------------------
|
|
23
|
+
// When routing is enabled, the wrapper injects the Pyyol identity headers
|
|
24
|
+
// (X-Pyyol-Key/Match/Turn) into each LLM call's request options so the Pyyol Gateway
|
|
25
|
+
// can attribute the server-observed usage; route() points a client's baseURL at the
|
|
26
|
+
// gateway. Together, ranked LLM traffic flows through the gateway with one opt-in line.
|
|
27
|
+
const gateway = { key: "", base: "" };
|
|
28
|
+
/** Enable gateway routing (called by the runtime in ranked mode; safe in tests). */
|
|
29
|
+
export function enableGateway(agentKey, baseUrl) {
|
|
30
|
+
gateway.key = agentKey ?? "";
|
|
31
|
+
gateway.base = (baseUrl ?? "").replace(/\/+$/, "");
|
|
32
|
+
}
|
|
33
|
+
export function disableGateway() {
|
|
34
|
+
gateway.key = "";
|
|
35
|
+
gateway.base = "";
|
|
36
|
+
}
|
|
37
|
+
const PROVIDER_PATH = { openai: "/gw/openai/v1", anthropic: "/gw/anthropic" };
|
|
38
|
+
/** The baseURL a provider client should point at, or "" if routing is off/unknown. */
|
|
39
|
+
export function gatewayBaseUrl(provider) {
|
|
40
|
+
const path = PROVIDER_PATH[provider];
|
|
41
|
+
return gateway.base && path ? gateway.base + path : "";
|
|
42
|
+
}
|
|
43
|
+
/** The X-Pyyol-* identity headers for the current turn (empty if routing off). */
|
|
44
|
+
export function gatewayHeaders() {
|
|
45
|
+
if (!gateway.key)
|
|
46
|
+
return {};
|
|
47
|
+
const h = { "X-Pyyol-Key": gateway.key };
|
|
48
|
+
const acc = currentUsage();
|
|
49
|
+
if (acc) {
|
|
50
|
+
if (acc.matchId)
|
|
51
|
+
h["X-Pyyol-Match"] = acc.matchId;
|
|
52
|
+
h["X-Pyyol-Turn"] = String(acc.turn ?? 0);
|
|
53
|
+
}
|
|
54
|
+
return h;
|
|
55
|
+
}
|
|
56
|
+
function detectProvider(client) {
|
|
57
|
+
const name = (client?.constructor?.name ?? "").toLowerCase();
|
|
58
|
+
if (name.includes("openai"))
|
|
59
|
+
return "openai";
|
|
60
|
+
if (name.includes("anthropic"))
|
|
61
|
+
return "anthropic";
|
|
62
|
+
// duck-type fallback
|
|
63
|
+
if (client?.chat?.completions)
|
|
64
|
+
return "openai";
|
|
65
|
+
if (client?.messages)
|
|
66
|
+
return "anthropic";
|
|
67
|
+
return "";
|
|
68
|
+
}
|
|
69
|
+
/** Point a provider client at the Pyyol Gateway (sets its baseURL). Explicit, robust
|
|
70
|
+
* opt-in that operates on the given instance. Returns the client. No-op when routing
|
|
71
|
+
* is off or the provider can't be determined. */
|
|
72
|
+
export function route(client, provider) {
|
|
73
|
+
const prov = provider ?? detectProvider(client);
|
|
74
|
+
const url = prov ? gatewayBaseUrl(prov) : "";
|
|
75
|
+
if (url) {
|
|
76
|
+
try {
|
|
77
|
+
client.baseURL = url;
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
// ignore — never break the caller
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return client;
|
|
84
|
+
}
|
|
85
|
+
/** Best-effort read of the baseURL the provider client will actually call. The patched
|
|
86
|
+
* method is bound to a resource whose `_client` holds the configured baseURL. */
|
|
87
|
+
function clientBaseUrl(resource) {
|
|
88
|
+
try {
|
|
89
|
+
const base = resource?._client?.baseURL;
|
|
90
|
+
return base ? String(base).replace(/\/+$/, "") : "";
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return "";
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/** True only when this call's client is pointed at the Pyyol Gateway. Guards header
|
|
97
|
+
* injection so the X-Pyyol-Key credential is NEVER sent to a third-party provider
|
|
98
|
+
* (e.g. a client the dev forgot to route()) — only to the gateway that issued it. */
|
|
99
|
+
function targetsGateway(resource) {
|
|
100
|
+
if (!gateway.base)
|
|
101
|
+
return false;
|
|
102
|
+
const base = clientBaseUrl(resource);
|
|
103
|
+
return !!base && base.startsWith(gateway.base);
|
|
104
|
+
}
|
|
105
|
+
// injectGatewayHeaders merges the Pyyol identity headers into an LLM call's request
|
|
106
|
+
// options. JS SDKs take per-request headers via a SECOND options arg
|
|
107
|
+
// (create(body, { headers })), so we ensure args[1].headers carries them. Dev-supplied
|
|
108
|
+
// headers win. No-op when routing is off OR when the call does not target the gateway.
|
|
109
|
+
function injectGatewayHeaders(resource, args) {
|
|
110
|
+
if (!targetsGateway(resource))
|
|
111
|
+
return;
|
|
112
|
+
const headers = gatewayHeaders();
|
|
113
|
+
if (!Object.keys(headers).length)
|
|
114
|
+
return;
|
|
115
|
+
try {
|
|
116
|
+
const opts = (args[1] && typeof args[1] === "object" ? args[1] : {});
|
|
117
|
+
opts.headers = { ...headers, ...(opts.headers ?? {}) }; // dev-supplied overrides
|
|
118
|
+
args[1] = opts;
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
// never break the dev's call
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
function get(obj, name, dflt) {
|
|
125
|
+
if (obj == null)
|
|
126
|
+
return dflt;
|
|
127
|
+
const v = obj[name];
|
|
128
|
+
return v === undefined ? dflt : v;
|
|
129
|
+
}
|
|
130
|
+
/** Pull normalized usage from a provider response, or null if it has none.
|
|
131
|
+
* Handles OpenAI Chat Completions, Anthropic Messages, and the OpenAI Responses
|
|
132
|
+
* API; duck-typed so a plain object or an SDK object both work. */
|
|
133
|
+
export function extractUsage(resp) {
|
|
134
|
+
const u = get(resp, "usage");
|
|
135
|
+
if (u == null)
|
|
136
|
+
return null;
|
|
137
|
+
const model = get(resp, "model", "") || "";
|
|
138
|
+
let prompt = get(u, "prompt_tokens");
|
|
139
|
+
let completion = get(u, "completion_tokens");
|
|
140
|
+
const styleOpenAiChat = prompt !== undefined || completion !== undefined;
|
|
141
|
+
if (prompt === undefined)
|
|
142
|
+
prompt = get(u, "input_tokens", 0);
|
|
143
|
+
if (completion === undefined)
|
|
144
|
+
completion = get(u, "output_tokens", 0);
|
|
145
|
+
let cached = 0;
|
|
146
|
+
let reasoning = 0;
|
|
147
|
+
const ptd = get(u, "prompt_tokens_details");
|
|
148
|
+
if (ptd != null)
|
|
149
|
+
cached = get(ptd, "cached_tokens", 0) || 0;
|
|
150
|
+
const ctd = get(u, "completion_tokens_details");
|
|
151
|
+
if (ctd != null)
|
|
152
|
+
reasoning = get(ctd, "reasoning_tokens", 0) || 0;
|
|
153
|
+
if (!cached)
|
|
154
|
+
cached = get(u, "cache_read_input_tokens", 0) || 0;
|
|
155
|
+
let provider = "";
|
|
156
|
+
if (styleOpenAiChat)
|
|
157
|
+
provider = "openai";
|
|
158
|
+
else if (get(u, "input_tokens") !== undefined)
|
|
159
|
+
provider = "anthropic";
|
|
160
|
+
return {
|
|
161
|
+
model,
|
|
162
|
+
provider,
|
|
163
|
+
promptTokens: Math.trunc(prompt || 0),
|
|
164
|
+
completionTokens: Math.trunc(completion || 0),
|
|
165
|
+
cachedTokens: Math.trunc(cached || 0),
|
|
166
|
+
reasoningTokens: Math.trunc(reasoning || 0),
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
/** Record usage from a provider response: compute cost, add to the turn
|
|
170
|
+
* accumulator, and emit a Lens model_call span. Returns the extracted usage (or
|
|
171
|
+
* null). Also the public manual hook for clients this module doesn't auto-wrap. */
|
|
172
|
+
export function recordResponse(resp, o = {}) {
|
|
173
|
+
const info = extractUsage(resp);
|
|
174
|
+
if (info === null)
|
|
175
|
+
return null;
|
|
176
|
+
const provider = o.provider || info.provider;
|
|
177
|
+
const cost = estimateCost(info.model, {
|
|
178
|
+
promptTokens: info.promptTokens,
|
|
179
|
+
completionTokens: info.completionTokens,
|
|
180
|
+
cachedTokens: info.cachedTokens,
|
|
181
|
+
reasoningTokens: info.reasoningTokens,
|
|
182
|
+
});
|
|
183
|
+
currentUsage()?.add({
|
|
184
|
+
model: info.model,
|
|
185
|
+
provider,
|
|
186
|
+
promptTokens: info.promptTokens,
|
|
187
|
+
completionTokens: info.completionTokens,
|
|
188
|
+
reasoningTokens: info.reasoningTokens,
|
|
189
|
+
cachedTokens: info.cachedTokens,
|
|
190
|
+
estimatedCost: cost,
|
|
191
|
+
});
|
|
192
|
+
currentSpan().logModelCall({
|
|
193
|
+
provider,
|
|
194
|
+
model: info.model,
|
|
195
|
+
promptTokens: info.promptTokens,
|
|
196
|
+
completionTokens: info.completionTokens,
|
|
197
|
+
totalTokens: info.promptTokens + info.completionTokens,
|
|
198
|
+
estimatedCost: cost,
|
|
199
|
+
latencyMs: o.latencyMs ?? 0,
|
|
200
|
+
});
|
|
201
|
+
return info;
|
|
202
|
+
}
|
|
203
|
+
/** @internal Wrap `proto[method]` so its resolved return value is recorded.
|
|
204
|
+
* Idempotent and fully guarded. Exported for tests. */
|
|
205
|
+
export function patchPrototype(proto, method, provider) {
|
|
206
|
+
if (proto == null)
|
|
207
|
+
return false;
|
|
208
|
+
const orig = proto[method];
|
|
209
|
+
if (typeof orig !== "function" || orig._pyyolInstrumented)
|
|
210
|
+
return false;
|
|
211
|
+
const wrapped = async function (...args) {
|
|
212
|
+
injectGatewayHeaders(this, args);
|
|
213
|
+
const start = Date.now();
|
|
214
|
+
const resp = await orig.apply(this, args);
|
|
215
|
+
try {
|
|
216
|
+
recordResponse(resp, { provider, latencyMs: Date.now() - start });
|
|
217
|
+
}
|
|
218
|
+
catch {
|
|
219
|
+
// instrumentation must never break the dev's call
|
|
220
|
+
}
|
|
221
|
+
return resp;
|
|
222
|
+
};
|
|
223
|
+
wrapped._pyyolInstrumented = true;
|
|
224
|
+
proto[method] = wrapped;
|
|
225
|
+
PATCHED.push([proto, method, orig]);
|
|
226
|
+
return true;
|
|
227
|
+
}
|
|
228
|
+
async function tryImport(spec) {
|
|
229
|
+
try {
|
|
230
|
+
return await import(spec);
|
|
231
|
+
}
|
|
232
|
+
catch {
|
|
233
|
+
return null;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
async function patchOpenAI() {
|
|
237
|
+
let patched = false;
|
|
238
|
+
const chat = await tryImport("openai/resources/chat/completions");
|
|
239
|
+
if (chat?.Completions?.prototype)
|
|
240
|
+
patched = patchPrototype(chat.Completions.prototype, "create", "openai") || patched;
|
|
241
|
+
const responses = await tryImport("openai/resources/responses");
|
|
242
|
+
if (responses?.Responses?.prototype)
|
|
243
|
+
patched = patchPrototype(responses.Responses.prototype, "create", "openai") || patched;
|
|
244
|
+
return patched;
|
|
245
|
+
}
|
|
246
|
+
async function patchAnthropic() {
|
|
247
|
+
let patched = false;
|
|
248
|
+
const messages = await tryImport("@anthropic-ai/sdk/resources/messages");
|
|
249
|
+
if (messages?.Messages?.prototype)
|
|
250
|
+
patched = patchPrototype(messages.Messages.prototype, "create", "anthropic") || patched;
|
|
251
|
+
return patched;
|
|
252
|
+
}
|
|
253
|
+
/** Auto-capture LLM usage from installed providers. Pass e.g. `["openai"]` to limit
|
|
254
|
+
* which are patched; default patches all supported providers that are installed.
|
|
255
|
+
* Returns the list actually instrumented. Safe to call more than once. */
|
|
256
|
+
export async function instrument(providers) {
|
|
257
|
+
const want = new Set(providers ?? ["openai", "anthropic"]);
|
|
258
|
+
const done = [];
|
|
259
|
+
if (want.has("openai") && (await patchOpenAI()))
|
|
260
|
+
done.push("openai");
|
|
261
|
+
if (want.has("anthropic") && (await patchAnthropic()))
|
|
262
|
+
done.push("anthropic");
|
|
263
|
+
return done;
|
|
264
|
+
}
|
|
265
|
+
/** Restore all patched methods (primarily for tests). */
|
|
266
|
+
export function uninstrument() {
|
|
267
|
+
while (PATCHED.length) {
|
|
268
|
+
const [proto, method, orig] = PATCHED.pop();
|
|
269
|
+
try {
|
|
270
|
+
proto[method] = orig;
|
|
271
|
+
}
|
|
272
|
+
catch {
|
|
273
|
+
// ignore
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
package/dist/models.d.ts
CHANGED
|
@@ -95,6 +95,10 @@ export interface MonopolyMove {
|
|
|
95
95
|
}
|
|
96
96
|
export interface MafiaMove {
|
|
97
97
|
action: string;
|
|
98
|
+
/** Seat to act on. Seat 0 is a real player, so for a night action
|
|
99
|
+
* (kill/investigate/protect/profile) set an explicit seat — omit it (or use -1)
|
|
100
|
+
* ONLY to mean "no target" (the engine then drops the untargeted action rather
|
|
101
|
+
* than acting on seat 0). Votes/discussion treat a missing/≤0 target as no target. */
|
|
98
102
|
target?: number;
|
|
99
103
|
tone?: string;
|
|
100
104
|
text?: string;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export declare const PRICING_VERSION = "2026-07-24";
|
|
2
|
+
export interface Rate {
|
|
3
|
+
/** USD per 1M input tokens. */
|
|
4
|
+
input: number;
|
|
5
|
+
/** USD per 1M output tokens. */
|
|
6
|
+
output: number;
|
|
7
|
+
/** USD per 1M cached (prompt-cache read) input tokens; defaults to `input`. */
|
|
8
|
+
cachedInput?: number;
|
|
9
|
+
}
|
|
10
|
+
/** Map a raw model string to a canonical table key, or null if unknown. */
|
|
11
|
+
export declare function canonical(model: string): string | null;
|
|
12
|
+
/** The Rate for a model (falls back to a mid-tier rate for unknown models). */
|
|
13
|
+
export declare function rateFor(model: string): Rate;
|
|
14
|
+
/** True if the model maps to an explicit table entry (not the fallback). */
|
|
15
|
+
export declare function isKnown(model: string): boolean;
|
|
16
|
+
export interface CostArgs {
|
|
17
|
+
promptTokens?: number;
|
|
18
|
+
completionTokens?: number;
|
|
19
|
+
cachedTokens?: number;
|
|
20
|
+
reasoningTokens?: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* USD cost estimate for one model call. `cachedTokens` are a subset of
|
|
24
|
+
* `promptTokens` billed at the cached-input rate; `reasoningTokens` are output
|
|
25
|
+
* tokens already counted in `completionTokens` (kept for reporting).
|
|
26
|
+
*/
|
|
27
|
+
export declare function estimateCost(model: string, a?: CostArgs): number;
|
package/dist/pricing.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// Versioned LLM price table + cost estimation (mirrors the Python SDK's pricing.py).
|
|
2
|
+
//
|
|
3
|
+
// Single, dated source of truth for turning token counts into a USD cost estimate.
|
|
4
|
+
// Prices are public list prices in USD per 1,000,000 tokens as of PRICING_VERSION;
|
|
5
|
+
// they are estimates for the sandbox/unverified tier (the ranked/verified tier's
|
|
6
|
+
// authoritative cost comes from the Pyyol Gateway). When a provider changes prices,
|
|
7
|
+
// bump PRICING_VERSION and update the table — never edit silently, so a cost can
|
|
8
|
+
// always be traced to the table that produced it.
|
|
9
|
+
// Bump whenever any rate below changes. Stamped onto every estimate.
|
|
10
|
+
export const PRICING_VERSION = "2026-07-24";
|
|
11
|
+
// Canonical model id -> Rate. Lowercase, provider-agnostic.
|
|
12
|
+
const TABLE = {
|
|
13
|
+
// OpenAI
|
|
14
|
+
"gpt-4o": { input: 2.5, output: 10.0, cachedInput: 1.25 },
|
|
15
|
+
"gpt-4o-mini": { input: 0.15, output: 0.6, cachedInput: 0.075 },
|
|
16
|
+
"gpt-4.1": { input: 2.0, output: 8.0, cachedInput: 0.5 },
|
|
17
|
+
"gpt-4.1-mini": { input: 0.4, output: 1.6, cachedInput: 0.1 },
|
|
18
|
+
"gpt-4.1-nano": { input: 0.1, output: 0.4, cachedInput: 0.025 },
|
|
19
|
+
o1: { input: 15.0, output: 60.0, cachedInput: 7.5 },
|
|
20
|
+
"o1-mini": { input: 1.1, output: 4.4, cachedInput: 0.55 },
|
|
21
|
+
o3: { input: 2.0, output: 8.0, cachedInput: 0.5 },
|
|
22
|
+
"o3-mini": { input: 1.1, output: 4.4, cachedInput: 0.55 },
|
|
23
|
+
"o4-mini": { input: 1.1, output: 4.4, cachedInput: 0.275 },
|
|
24
|
+
"gpt-3.5-turbo": { input: 0.5, output: 1.5 },
|
|
25
|
+
// Anthropic (distinct Opus / Sonnet / Haiku)
|
|
26
|
+
"claude-opus": { input: 15.0, output: 75.0, cachedInput: 1.5 },
|
|
27
|
+
"claude-sonnet": { input: 3.0, output: 15.0, cachedInput: 0.3 },
|
|
28
|
+
"claude-haiku": { input: 0.8, output: 4.0, cachedInput: 0.08 },
|
|
29
|
+
// Google (Gemini)
|
|
30
|
+
"gemini-flash": { input: 0.15, output: 0.6, cachedInput: 0.0375 },
|
|
31
|
+
"gemini-pro": { input: 1.25, output: 5.0, cachedInput: 0.3125 },
|
|
32
|
+
// Open-weight / self-hosted (no per-token bill)
|
|
33
|
+
llama: { input: 0.0, output: 0.0 },
|
|
34
|
+
mistral: { input: 0.0, output: 0.0 },
|
|
35
|
+
qwen: { input: 0.0, output: 0.0 },
|
|
36
|
+
deepseek: { input: 0.27, output: 1.1 },
|
|
37
|
+
};
|
|
38
|
+
// Last-resort rate for an unmapped model (never silently $0 unless open-weight).
|
|
39
|
+
const FALLBACK = { input: 0.5, output: 1.5 };
|
|
40
|
+
// Ordered [substring, canonical] rules; first match wins, most specific first.
|
|
41
|
+
const RULES = [
|
|
42
|
+
["gpt-4o-mini", "gpt-4o-mini"],
|
|
43
|
+
["gpt-4o", "gpt-4o"],
|
|
44
|
+
["4o-mini", "gpt-4o-mini"],
|
|
45
|
+
["4o", "gpt-4o"],
|
|
46
|
+
["gpt-4.1-nano", "gpt-4.1-nano"],
|
|
47
|
+
["gpt-4.1-mini", "gpt-4.1-mini"],
|
|
48
|
+
["gpt-4.1", "gpt-4.1"],
|
|
49
|
+
["4.1-nano", "gpt-4.1-nano"],
|
|
50
|
+
["4.1-mini", "gpt-4.1-mini"],
|
|
51
|
+
["4.1", "gpt-4.1"],
|
|
52
|
+
["o1-mini", "o1-mini"],
|
|
53
|
+
["o1", "o1"],
|
|
54
|
+
["o3-mini", "o3-mini"],
|
|
55
|
+
["o3", "o3"],
|
|
56
|
+
["o4-mini", "o4-mini"],
|
|
57
|
+
["gpt-3.5", "gpt-3.5-turbo"],
|
|
58
|
+
["3.5-turbo", "gpt-3.5-turbo"],
|
|
59
|
+
["opus", "claude-opus"],
|
|
60
|
+
["sonnet", "claude-sonnet"],
|
|
61
|
+
["haiku", "claude-haiku"],
|
|
62
|
+
["gemini-1.5-flash", "gemini-flash"],
|
|
63
|
+
["gemini-2.0-flash", "gemini-flash"],
|
|
64
|
+
["gemini-2.5-flash", "gemini-flash"],
|
|
65
|
+
["flash", "gemini-flash"],
|
|
66
|
+
["gemini-1.5-pro", "gemini-pro"],
|
|
67
|
+
["gemini-2.5-pro", "gemini-pro"],
|
|
68
|
+
["gemini", "gemini-pro"],
|
|
69
|
+
["llama", "llama"],
|
|
70
|
+
["mistral", "mistral"],
|
|
71
|
+
["mixtral", "mistral"],
|
|
72
|
+
["qwen", "qwen"],
|
|
73
|
+
["deepseek", "deepseek"],
|
|
74
|
+
];
|
|
75
|
+
/** Map a raw model string to a canonical table key, or null if unknown. */
|
|
76
|
+
export function canonical(model) {
|
|
77
|
+
const m = (model ?? "").trim().toLowerCase();
|
|
78
|
+
if (!m)
|
|
79
|
+
return null;
|
|
80
|
+
if (m in TABLE)
|
|
81
|
+
return m;
|
|
82
|
+
for (const [needle, key] of RULES)
|
|
83
|
+
if (m.includes(needle))
|
|
84
|
+
return key;
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
/** The Rate for a model (falls back to a mid-tier rate for unknown models). */
|
|
88
|
+
export function rateFor(model) {
|
|
89
|
+
const key = canonical(model);
|
|
90
|
+
return key !== null ? TABLE[key] : FALLBACK;
|
|
91
|
+
}
|
|
92
|
+
/** True if the model maps to an explicit table entry (not the fallback). */
|
|
93
|
+
export function isKnown(model) {
|
|
94
|
+
return canonical(model) !== null;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* USD cost estimate for one model call. `cachedTokens` are a subset of
|
|
98
|
+
* `promptTokens` billed at the cached-input rate; `reasoningTokens` are output
|
|
99
|
+
* tokens already counted in `completionTokens` (kept for reporting).
|
|
100
|
+
*/
|
|
101
|
+
export function estimateCost(model, a = {}) {
|
|
102
|
+
const rate = rateFor(model);
|
|
103
|
+
const prompt = Math.max(0, a.promptTokens ?? 0);
|
|
104
|
+
const completion = Math.max(0, a.completionTokens ?? 0);
|
|
105
|
+
const cached = Math.max(0, Math.min(a.cachedTokens ?? 0, prompt));
|
|
106
|
+
const fullInput = prompt - cached;
|
|
107
|
+
const cachedRate = rate.cachedInput ?? rate.input;
|
|
108
|
+
const cost = (fullInput * rate.input + cached * cachedRate + completion * rate.output) / 1_000_000;
|
|
109
|
+
return Math.round(cost * 1e8) / 1e8;
|
|
110
|
+
}
|
package/dist/runtime.d.ts
CHANGED
|
@@ -74,6 +74,7 @@ export declare class RuntimeConnector {
|
|
|
74
74
|
private nudged;
|
|
75
75
|
private registered;
|
|
76
76
|
private refreshAttempts;
|
|
77
|
+
private turnNo;
|
|
77
78
|
private readonly tracer;
|
|
78
79
|
constructor(agent: Agent, opts: RuntimeOptions);
|
|
79
80
|
/** The gateway echoes the newest published version on the registered frame.
|
package/dist/runtime.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { SDK_VERSION } from "./server.js";
|
|
2
|
-
import { Tracer } from "./telemetry.js";
|
|
2
|
+
import { Tracer, runTurnUsage } from "./telemetry.js";
|
|
3
3
|
// Frame types — byte-identical to the Go gateway (internal/agentgw/frame.go).
|
|
4
4
|
const HELLO = "hello", REGISTERED = "registered", PONG = "pong";
|
|
5
5
|
const INITIALIZE = "initialize", TURN = "turn", EVENT = "event", GAME_END = "game_end", ERROR = "error";
|
|
@@ -88,6 +88,7 @@ export class RuntimeConnector {
|
|
|
88
88
|
nudged = false; // print the "upgrade available" notice at most once
|
|
89
89
|
registered = false; // true once this session's register succeeded
|
|
90
90
|
refreshAttempts = 0; // per-connection guard against a refresh loop
|
|
91
|
+
turnNo = 0; // monotonic per-connection turn counter (telemetry attribution fallback)
|
|
91
92
|
// Opt-in Pyyol Lens telemetry (no-op unless PYYOL_LENS_ENDPOINT+KEY set).
|
|
92
93
|
// Correlated to the match trace so the agent's model/tool calls render with
|
|
93
94
|
// the platform's authoritative gateway spans.
|
|
@@ -329,20 +330,44 @@ export class RuntimeConnector {
|
|
|
329
330
|
break;
|
|
330
331
|
case TURN: {
|
|
331
332
|
const view = frame.payload ?? {};
|
|
332
|
-
|
|
333
|
-
//
|
|
334
|
-
|
|
333
|
+
this.turnNo++;
|
|
334
|
+
// Per-turn index for telemetry attribution. Goofspiel has `round`, Mafia has
|
|
335
|
+
// `day`; Monopoly has neither, so fall back to the monotonic per-match turn
|
|
336
|
+
// counter — otherwise X-Pyyol-Turn was always 0 for 2/3 games.
|
|
337
|
+
const turnNo = Number(view.round ?? view.day ?? 0) || this.turnNo;
|
|
338
|
+
// Bracket the developer's handler in a Lens span AND a turn-local usage
|
|
339
|
+
// accumulator. Inside decideTurn the author can reach the span via
|
|
340
|
+
// pyyol.currentSpan(); if instrument() is active, every LLM call is captured
|
|
341
|
+
// automatically. The accumulator is always on (independent of Lens) so usage
|
|
342
|
+
// rides the move to the arena regardless.
|
|
343
|
+
const { result, usage } = await runTurnUsage(() => this.tracer.runTurn({
|
|
335
344
|
matchId: view.match_id ?? "",
|
|
336
345
|
game: view.game ?? "",
|
|
337
|
-
round:
|
|
346
|
+
round: turnNo,
|
|
338
347
|
agentId: this.opts.agentId,
|
|
339
|
-
}, () => this.agent.decideTurn(view));
|
|
348
|
+
}, () => this.agent.decideTurn(view)), { matchId: view.match_id ?? "", turn: turnNo });
|
|
349
|
+
const { status, body } = result;
|
|
350
|
+
// Auto-attach captured model/token/cost to the move so the arena benchmark
|
|
351
|
+
// records real usage with no developer boilerplate. A dev-supplied `usage`
|
|
352
|
+
// always wins — we never overwrite it.
|
|
353
|
+
if (status === 200 &&
|
|
354
|
+
body != null &&
|
|
355
|
+
typeof body === "object" &&
|
|
356
|
+
!usage.empty &&
|
|
357
|
+
!("usage" in body)) {
|
|
358
|
+
body.usage = usage.toMoveUsage();
|
|
359
|
+
}
|
|
340
360
|
if (status === 200) {
|
|
341
361
|
send({ t: RESPONSE, id: frame.id ?? "", payload: body });
|
|
342
362
|
this.feed("turn", summarizeMove(frame.payload?.game ?? "", body));
|
|
343
363
|
}
|
|
344
364
|
else {
|
|
345
|
-
|
|
365
|
+
const err = body?.error ?? "handler_error";
|
|
366
|
+
const detail = body?.message || err;
|
|
367
|
+
// Surface the REAL handler error in the feed (not just "handler_error"), so
|
|
368
|
+
// a crashing step() is visible in `pyyol dev`.
|
|
369
|
+
send({ t: RESPONSE, id: frame.id ?? "", error: err });
|
|
370
|
+
this.feed("error", `turn ${this.turnNo}: ${detail} → fallback`);
|
|
346
371
|
}
|
|
347
372
|
break;
|
|
348
373
|
}
|
package/dist/server.js
CHANGED
|
@@ -105,8 +105,16 @@ export class Agent {
|
|
|
105
105
|
const move = await handler(parseView(data));
|
|
106
106
|
return { status: 200, body: move ?? {} };
|
|
107
107
|
}
|
|
108
|
-
catch {
|
|
109
|
-
|
|
108
|
+
catch (e) {
|
|
109
|
+
// Do NOT swallow: a crashing step() is the #1 "why doesn't my agent work"
|
|
110
|
+
// trap. Log the real error + stack (so `pyyol dev` shows it) and return the
|
|
111
|
+
// message so the runtime can surface it in the feed. The engine still applies
|
|
112
|
+
// its deterministic fallback for this turn, so the match isn't wedged.
|
|
113
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
114
|
+
console.error(`${game} step() threw: ${msg}`);
|
|
115
|
+
if (e instanceof Error && e.stack)
|
|
116
|
+
console.error(e.stack);
|
|
117
|
+
return { status: 500, body: { error: "handler_error", message: msg } };
|
|
110
118
|
}
|
|
111
119
|
}
|
|
112
120
|
// --- shared handler invocation (used by both the HTTP path and the socket
|
package/dist/telemetry.d.ts
CHANGED
|
@@ -31,6 +31,57 @@ export declare class Span {
|
|
|
31
31
|
/** The span for the turn currently being handled, or a no-op span outside one.
|
|
32
32
|
* Always safe to call and chain (never null). */
|
|
33
33
|
export declare function currentSpan(): Span;
|
|
34
|
+
export interface MoveUsage {
|
|
35
|
+
prompt_tokens: number;
|
|
36
|
+
completion_tokens: number;
|
|
37
|
+
total_tokens: number;
|
|
38
|
+
reasoning_tokens?: number;
|
|
39
|
+
cached_tokens?: number;
|
|
40
|
+
estimated_cost?: number;
|
|
41
|
+
model?: string | string[];
|
|
42
|
+
provider?: string | string[];
|
|
43
|
+
}
|
|
44
|
+
export interface UsageAdd {
|
|
45
|
+
model?: string;
|
|
46
|
+
provider?: string;
|
|
47
|
+
promptTokens?: number;
|
|
48
|
+
completionTokens?: number;
|
|
49
|
+
reasoningTokens?: number;
|
|
50
|
+
cachedTokens?: number;
|
|
51
|
+
estimatedCost?: number;
|
|
52
|
+
}
|
|
53
|
+
/** Sums token usage + cost across every model call within a single turn. */
|
|
54
|
+
export declare class UsageAccumulator {
|
|
55
|
+
promptTokens: number;
|
|
56
|
+
completionTokens: number;
|
|
57
|
+
reasoningTokens: number;
|
|
58
|
+
cachedTokens: number;
|
|
59
|
+
estimatedCost: number;
|
|
60
|
+
calls: number;
|
|
61
|
+
readonly models: string[];
|
|
62
|
+
readonly providers: string[];
|
|
63
|
+
matchId: string;
|
|
64
|
+
turn: number;
|
|
65
|
+
add(u: UsageAdd): void;
|
|
66
|
+
get totalTokens(): number;
|
|
67
|
+
get empty(): boolean;
|
|
68
|
+
/** The `usage` block attached to a move — matches the arena's TokenUsage decode
|
|
69
|
+
* (prompt/completion/reasoning/total) plus SDK-side model/provider/cost. */
|
|
70
|
+
toMoveUsage(): MoveUsage;
|
|
71
|
+
}
|
|
72
|
+
/** The accumulator for the turn in progress, or undefined outside one. The
|
|
73
|
+
* instrumentation calls this to record real usage; it no-ops when undefined. */
|
|
74
|
+
export declare function currentUsage(): UsageAccumulator | undefined;
|
|
75
|
+
/** Run `fn` with a fresh usage accumulator installed as current (always active,
|
|
76
|
+
* independent of the Tracer), returning both fn's result and the accumulator.
|
|
77
|
+
* ctx.matchId/turn are carried so gateway routing can attribute a call to the match. */
|
|
78
|
+
export declare function runTurnUsage<T>(fn: () => T | Promise<T>, ctx?: {
|
|
79
|
+
matchId?: string;
|
|
80
|
+
turn?: number;
|
|
81
|
+
}): Promise<{
|
|
82
|
+
result: T;
|
|
83
|
+
usage: UsageAccumulator;
|
|
84
|
+
}>;
|
|
34
85
|
export interface TracerOptions {
|
|
35
86
|
endpoint?: string;
|
|
36
87
|
apiKey?: string;
|
package/dist/telemetry.js
CHANGED
|
@@ -90,6 +90,74 @@ const storage = new AsyncLocalStorage();
|
|
|
90
90
|
export function currentSpan() {
|
|
91
91
|
return storage.getStore() ?? NOOP_SPAN;
|
|
92
92
|
}
|
|
93
|
+
/** Sums token usage + cost across every model call within a single turn. */
|
|
94
|
+
export class UsageAccumulator {
|
|
95
|
+
promptTokens = 0;
|
|
96
|
+
completionTokens = 0;
|
|
97
|
+
reasoningTokens = 0;
|
|
98
|
+
cachedTokens = 0;
|
|
99
|
+
estimatedCost = 0;
|
|
100
|
+
calls = 0;
|
|
101
|
+
models = [];
|
|
102
|
+
providers = [];
|
|
103
|
+
// Turn context (for gateway attribution); set by runTurnUsage().
|
|
104
|
+
matchId = "";
|
|
105
|
+
turn = 0;
|
|
106
|
+
add(u) {
|
|
107
|
+
this.promptTokens += Math.max(0, Math.trunc(u.promptTokens ?? 0));
|
|
108
|
+
this.completionTokens += Math.max(0, Math.trunc(u.completionTokens ?? 0));
|
|
109
|
+
this.reasoningTokens += Math.max(0, Math.trunc(u.reasoningTokens ?? 0));
|
|
110
|
+
this.cachedTokens += Math.max(0, Math.trunc(u.cachedTokens ?? 0));
|
|
111
|
+
this.estimatedCost += Math.max(0, u.estimatedCost ?? 0);
|
|
112
|
+
this.calls += 1;
|
|
113
|
+
if (u.model && !this.models.includes(u.model))
|
|
114
|
+
this.models.push(u.model);
|
|
115
|
+
if (u.provider && !this.providers.includes(u.provider))
|
|
116
|
+
this.providers.push(u.provider);
|
|
117
|
+
}
|
|
118
|
+
get totalTokens() {
|
|
119
|
+
return this.promptTokens + this.completionTokens;
|
|
120
|
+
}
|
|
121
|
+
get empty() {
|
|
122
|
+
return this.calls === 0;
|
|
123
|
+
}
|
|
124
|
+
/** The `usage` block attached to a move — matches the arena's TokenUsage decode
|
|
125
|
+
* (prompt/completion/reasoning/total) plus SDK-side model/provider/cost. */
|
|
126
|
+
toMoveUsage() {
|
|
127
|
+
const usage = {
|
|
128
|
+
prompt_tokens: this.promptTokens,
|
|
129
|
+
completion_tokens: this.completionTokens,
|
|
130
|
+
total_tokens: this.totalTokens,
|
|
131
|
+
};
|
|
132
|
+
if (this.reasoningTokens)
|
|
133
|
+
usage.reasoning_tokens = this.reasoningTokens;
|
|
134
|
+
if (this.cachedTokens)
|
|
135
|
+
usage.cached_tokens = this.cachedTokens;
|
|
136
|
+
if (this.estimatedCost)
|
|
137
|
+
usage.estimated_cost = Math.round(this.estimatedCost * 1e8) / 1e8;
|
|
138
|
+
if (this.models.length)
|
|
139
|
+
usage.model = this.models.length === 1 ? this.models[0] : this.models;
|
|
140
|
+
if (this.providers.length)
|
|
141
|
+
usage.provider = this.providers.length === 1 ? this.providers[0] : this.providers;
|
|
142
|
+
return usage;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const usageStorage = new AsyncLocalStorage();
|
|
146
|
+
/** The accumulator for the turn in progress, or undefined outside one. The
|
|
147
|
+
* instrumentation calls this to record real usage; it no-ops when undefined. */
|
|
148
|
+
export function currentUsage() {
|
|
149
|
+
return usageStorage.getStore();
|
|
150
|
+
}
|
|
151
|
+
/** Run `fn` with a fresh usage accumulator installed as current (always active,
|
|
152
|
+
* independent of the Tracer), returning both fn's result and the accumulator.
|
|
153
|
+
* ctx.matchId/turn are carried so gateway routing can attribute a call to the match. */
|
|
154
|
+
export async function runTurnUsage(fn, ctx = {}) {
|
|
155
|
+
const acc = new UsageAccumulator();
|
|
156
|
+
acc.matchId = ctx.matchId ?? "";
|
|
157
|
+
acc.turn = ctx.turn ?? 0;
|
|
158
|
+
const result = await usageStorage.run(acc, fn);
|
|
159
|
+
return { result, usage: acc };
|
|
160
|
+
}
|
|
93
161
|
/** Batching, non-blocking emitter to the Pyyol Lens ingest. */
|
|
94
162
|
export class Tracer {
|
|
95
163
|
enabled;
|
package/dist/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const SDK_VERSION = "1.
|
|
1
|
+
export declare const SDK_VERSION = "1.3.0";
|
package/dist/version.js
CHANGED