ravensight-playtest 0.2.9 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -8
- package/docs/ADAPTERS.md +2 -0
- package/docs/NATIVE_RUNNER.md +243 -0
- package/package.json +2 -1
- package/src/cli.js +6 -0
- package/src/commands/check.js +330 -106
- package/src/commands/run.js +55 -2
- package/src/run/aggregate.js +97 -33
- package/src/run/args.js +2 -0
- package/src/run/drivers/driver.js +7 -0
- package/src/run/drivers/web.js +376 -41
- package/src/run/drivers/webdriver-probe.js +353 -0
- package/src/run/dropped.js +7 -7
- package/src/run/personaLoop.js +247 -37
- package/src/run/regenerate.js +24 -4
- package/src/run/resume.js +7 -1
- package/src/run/tools.js +31 -10
- package/src/run/usage.js +19 -1
package/src/run/usage.js
CHANGED
|
@@ -29,6 +29,7 @@ export function createUsageLedger() {
|
|
|
29
29
|
const byModel = new Map();
|
|
30
30
|
const started = Date.now();
|
|
31
31
|
let calls = 0;
|
|
32
|
+
let modelMs = 0;
|
|
32
33
|
let serverTotalUsd = null;
|
|
33
34
|
let serverCalls = null;
|
|
34
35
|
|
|
@@ -36,8 +37,10 @@ export function createUsageLedger() {
|
|
|
36
37
|
/**
|
|
37
38
|
* @param {{model: string|null, usage: Object}} turn
|
|
38
39
|
*/
|
|
39
|
-
record({ model, usage }) {
|
|
40
|
+
record({ model, usage, ms }) {
|
|
40
41
|
calls += 1;
|
|
42
|
+
// Time spent waiting on the model, when the caller measured it.
|
|
43
|
+
if (Number.isFinite(Number(ms)) && Number(ms) > 0) modelMs += Number(ms);
|
|
41
44
|
const key = model || 'unknown';
|
|
42
45
|
const entry = byModel.get(key) || { model: key, input_tokens: 0, cache_read_tokens: 0, output_tokens: 0, usd: 0 };
|
|
43
46
|
// The report schema has no cache-creation field, so creation tokens
|
|
@@ -64,6 +67,20 @@ export function createUsageLedger() {
|
|
|
64
67
|
return calls;
|
|
65
68
|
},
|
|
66
69
|
|
|
70
|
+
/**
|
|
71
|
+
* Wall time since the ledger was made, and time spent waiting on model
|
|
72
|
+
* calls, in seconds. What `complete` sends as `duration`, so the run card
|
|
73
|
+
* does not read 0s for a run that played for minutes.
|
|
74
|
+
*
|
|
75
|
+
* @returns {{wall_seconds: number, model_seconds: number}}
|
|
76
|
+
*/
|
|
77
|
+
duration() {
|
|
78
|
+
return {
|
|
79
|
+
wall_seconds: Number(((Date.now() - started) / 1000).toFixed(3)),
|
|
80
|
+
model_seconds: Number((modelMs / 1000).toFixed(3))
|
|
81
|
+
};
|
|
82
|
+
},
|
|
83
|
+
|
|
67
84
|
/**
|
|
68
85
|
* The `usage` object as the report schema wants it.
|
|
69
86
|
*
|
|
@@ -87,6 +104,7 @@ export function createUsageLedger() {
|
|
|
87
104
|
const report = this.toReportUsage();
|
|
88
105
|
return {
|
|
89
106
|
...report,
|
|
107
|
+
model_seconds: Number((modelMs / 1000).toFixed(3)),
|
|
90
108
|
local_model_calls: calls,
|
|
91
109
|
server_model_calls: serverCalls,
|
|
92
110
|
usd_source: serverTotalUsd === null ? 'unavailable' : 'server',
|