vent-hq 0.13.1 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs
CHANGED
|
@@ -13,78 +13,103 @@ import * as fs3 from "node:fs/promises";
|
|
|
13
13
|
import * as fs from "node:fs/promises";
|
|
14
14
|
import * as path from "node:path";
|
|
15
15
|
import { homedir } from "node:os";
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
var
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
try {
|
|
36
|
-
await fs.rm(CREDENTIALS_FILE);
|
|
37
|
-
} catch {
|
|
38
|
-
}
|
|
16
|
+
import { randomUUID } from "node:crypto";
|
|
17
|
+
|
|
18
|
+
// ../../node_modules/.pnpm/jose@6.2.15/node_modules/jose/dist/webapi/lib/buffer_utils.js
|
|
19
|
+
var encoder = new TextEncoder();
|
|
20
|
+
var decoder = new TextDecoder();
|
|
21
|
+
var strictDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
22
|
+
var UINT32_RANGE = 2 ** 32;
|
|
23
|
+
function decodeBase64(encoded, url = false) {
|
|
24
|
+
if (Uint8Array.fromBase64)
|
|
25
|
+
return Uint8Array.fromBase64(encoded, { alphabet: url ? "base64url" : "base64" });
|
|
26
|
+
if (url) {
|
|
27
|
+
if (encoded.includes("+") || encoded.includes("/"))
|
|
28
|
+
throw new TypeError("Invalid base64url");
|
|
29
|
+
encoded = encoded.replace(/-/g, "+").replace(/_/g, "/");
|
|
30
|
+
}
|
|
31
|
+
const binary = atob(encoded), bytes = new Uint8Array(binary.length);
|
|
32
|
+
for (let i = 0; i < binary.length; i++)
|
|
33
|
+
bytes[i] = binary.charCodeAt(i);
|
|
34
|
+
return bytes;
|
|
39
35
|
}
|
|
40
36
|
|
|
41
|
-
//
|
|
42
|
-
var
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
this.
|
|
37
|
+
// ../../node_modules/.pnpm/jose@6.2.15/node_modules/jose/dist/webapi/util/errors.js
|
|
38
|
+
var JOSEError = class extends Error {
|
|
39
|
+
static code = "ERR_JOSE_GENERIC";
|
|
40
|
+
code = "ERR_JOSE_GENERIC";
|
|
41
|
+
constructor(message, options) {
|
|
42
|
+
super(message, options), this.name = this.constructor.name, Error.captureStackTrace?.(this, this.constructor);
|
|
47
43
|
}
|
|
48
44
|
};
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
try {
|
|
62
|
-
body = await res.json();
|
|
63
|
-
} catch {
|
|
64
|
-
body = await res.text();
|
|
65
|
-
}
|
|
66
|
-
throw new ApiError(res.status, body);
|
|
45
|
+
var JWTInvalid = class extends JOSEError {
|
|
46
|
+
static code = "ERR_JWT_INVALID";
|
|
47
|
+
code = "ERR_JWT_INVALID";
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
// ../../node_modules/.pnpm/jose@6.2.15/node_modules/jose/dist/webapi/util/base64url.js
|
|
51
|
+
var invalid = "The input to be decoded is not correctly encoded.";
|
|
52
|
+
function decode(input) {
|
|
53
|
+
try {
|
|
54
|
+
return decodeBase64(typeof input == "string" ? input : decoder.decode(input), true);
|
|
55
|
+
} catch (cause) {
|
|
56
|
+
throw new TypeError(invalid, { cause });
|
|
67
57
|
}
|
|
68
|
-
return res;
|
|
69
58
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
}
|
|
78
|
-
})
|
|
79
|
-
});
|
|
80
|
-
return res.json();
|
|
59
|
+
|
|
60
|
+
// ../../node_modules/.pnpm/jose@6.2.15/node_modules/jose/dist/webapi/lib/validate.js
|
|
61
|
+
function isObject(input) {
|
|
62
|
+
if (typeof input != "object" || input === null || Object.prototype.toString.call(input) !== "[object Object]")
|
|
63
|
+
return false;
|
|
64
|
+
const prototype = Object.getPrototypeOf(input);
|
|
65
|
+
return prototype === null || Object.getPrototypeOf(prototype) === null;
|
|
81
66
|
}
|
|
82
67
|
|
|
83
|
-
//
|
|
84
|
-
|
|
68
|
+
// ../../node_modules/.pnpm/jose@6.2.15/node_modules/jose/dist/webapi/util/decode_jwt.js
|
|
69
|
+
function decodeJwt(jwt) {
|
|
70
|
+
if (typeof jwt != "string")
|
|
71
|
+
throw new JWTInvalid("JWTs must use Compact JWS serialization, JWT must be a string");
|
|
72
|
+
const { 1: encodedPayload, length } = jwt.split(".");
|
|
73
|
+
if (length === 5)
|
|
74
|
+
throw new JWTInvalid("Only JWTs using Compact JWS serialization can be decoded");
|
|
75
|
+
if (length !== 3)
|
|
76
|
+
throw new JWTInvalid("Invalid JWT");
|
|
77
|
+
if (!encodedPayload)
|
|
78
|
+
throw new JWTInvalid("JWTs must contain a payload");
|
|
79
|
+
let claimsSetBytes;
|
|
80
|
+
try {
|
|
81
|
+
claimsSetBytes = decode(encodedPayload);
|
|
82
|
+
} catch (cause) {
|
|
83
|
+
throw new JWTInvalid("Failed to base64url decode the payload", { cause });
|
|
84
|
+
}
|
|
85
|
+
let claimsSet;
|
|
86
|
+
try {
|
|
87
|
+
claimsSet = JSON.parse(strictDecoder.decode(claimsSetBytes));
|
|
88
|
+
} catch (cause) {
|
|
89
|
+
throw new JWTInvalid("Failed to parse the decoded payload as JSON", { cause });
|
|
90
|
+
}
|
|
91
|
+
if (!isObject(claimsSet))
|
|
92
|
+
throw new JWTInvalid("Invalid JWT Claims Set");
|
|
93
|
+
return claimsSet;
|
|
94
|
+
}
|
|
85
95
|
|
|
86
96
|
// ../shared/src/constants.ts
|
|
87
97
|
var RUNNER_CALLBACK_MAX_SKEW_MS = 5 * 6e4;
|
|
98
|
+
var CALLER_ACCENTS = [
|
|
99
|
+
"american",
|
|
100
|
+
"british",
|
|
101
|
+
"australian",
|
|
102
|
+
"filipino",
|
|
103
|
+
"spanish_mexican",
|
|
104
|
+
"spanish_peninsular",
|
|
105
|
+
"spanish_colombian",
|
|
106
|
+
"spanish_argentine",
|
|
107
|
+
"german",
|
|
108
|
+
"french",
|
|
109
|
+
"italian",
|
|
110
|
+
"dutch",
|
|
111
|
+
"japanese"
|
|
112
|
+
];
|
|
88
113
|
|
|
89
114
|
// ../../node_modules/.pnpm/zod@3.25.76/node_modules/zod/v3/external.js
|
|
90
115
|
var external_exports = {};
|
|
@@ -4128,6 +4153,13 @@ var coerce = {
|
|
|
4128
4153
|
var NEVER = INVALID;
|
|
4129
4154
|
|
|
4130
4155
|
// ../shared/src/schemas.ts
|
|
4156
|
+
var RefreshTokenRequestSchema = external_exports.object({
|
|
4157
|
+
refresh_token: external_exports.string().regex(/^vent_[0-9a-f]{48}$/)
|
|
4158
|
+
});
|
|
4159
|
+
var AccessCredentialsSchema = external_exports.object({
|
|
4160
|
+
access_token: external_exports.string().min(1),
|
|
4161
|
+
refresh_token: RefreshTokenRequestSchema.shape.refresh_token
|
|
4162
|
+
});
|
|
4131
4163
|
var CallerPersonaSchema = external_exports.object({
|
|
4132
4164
|
pace: external_exports.enum(["slow", "normal", "fast"]).optional(),
|
|
4133
4165
|
clarity: external_exports.enum(["clear", "vague", "rambling"]).optional(),
|
|
@@ -4147,7 +4179,7 @@ var CallerAudioEffectsSchema = external_exports.object({
|
|
|
4147
4179
|
speakerphone: external_exports.boolean().optional(),
|
|
4148
4180
|
mic_distance: external_exports.enum(["close", "normal", "far"]).optional(),
|
|
4149
4181
|
clarity: external_exports.number().min(0).max(1).optional(),
|
|
4150
|
-
accent: external_exports.
|
|
4182
|
+
accent: external_exports.enum(CALLER_ACCENTS).optional(),
|
|
4151
4183
|
packet_loss: external_exports.number().min(0).max(0.3).optional(),
|
|
4152
4184
|
jitter_ms: external_exports.number().min(0).max(100).optional()
|
|
4153
4185
|
});
|
|
@@ -4155,7 +4187,6 @@ var ConversationCallSpecSchema = external_exports.object({
|
|
|
4155
4187
|
name: external_exports.string().optional(),
|
|
4156
4188
|
caller_prompt: external_exports.string().min(1),
|
|
4157
4189
|
max_turns: external_exports.number().int().min(1).max(50).default(6),
|
|
4158
|
-
silence_threshold_ms: external_exports.number().int().min(200).max(1e4).optional(),
|
|
4159
4190
|
persona: CallerPersonaSchema,
|
|
4160
4191
|
caller_audio: CallerAudioEffectsSchema.optional(),
|
|
4161
4192
|
/** ISO 639-1 language code for multilingual calls (e.g., "es", "fr", "de"). Caller speaks this language, STT transcribes it, judge evaluates in it. */
|
|
@@ -4246,20 +4277,6 @@ var PlatformConfigSchema = external_exports.discriminatedUnion("provider", [
|
|
|
4246
4277
|
RetellPlatformSchema,
|
|
4247
4278
|
ElevenLabsPlatformSchema
|
|
4248
4279
|
]);
|
|
4249
|
-
var PlatformSummarySchema = external_exports.object({
|
|
4250
|
-
provider: PlatformProviderSchema
|
|
4251
|
-
}).catchall(external_exports.unknown());
|
|
4252
|
-
var PlatformConnectionSummarySchema = external_exports.object({
|
|
4253
|
-
id: external_exports.string().uuid(),
|
|
4254
|
-
provider: PlatformProviderSchema,
|
|
4255
|
-
version: external_exports.number().int().min(1),
|
|
4256
|
-
resource_label: external_exports.string().min(1)
|
|
4257
|
-
});
|
|
4258
|
-
var RunPlatformSummarySchema = external_exports.object({
|
|
4259
|
-
platform_connection_id: external_exports.string().uuid().nullable(),
|
|
4260
|
-
platform_connection: PlatformConnectionSummarySchema.nullable(),
|
|
4261
|
-
platform: PlatformSummarySchema.nullable()
|
|
4262
|
-
});
|
|
4263
4280
|
var ConversationTurnSchema = external_exports.object({
|
|
4264
4281
|
role: external_exports.enum(["caller", "agent"]),
|
|
4265
4282
|
text: external_exports.string(),
|
|
@@ -4282,11 +4299,11 @@ var ConversationTurnSchema = external_exports.object({
|
|
|
4282
4299
|
});
|
|
4283
4300
|
var LatencyMetricsSchema = external_exports.object({
|
|
4284
4301
|
ttfb_per_turn_ms: external_exports.array(external_exports.number()),
|
|
4285
|
-
p50_ttfb_ms: external_exports.number(),
|
|
4286
|
-
p90_ttfb_ms: external_exports.number(),
|
|
4287
|
-
p95_ttfb_ms: external_exports.number(),
|
|
4288
|
-
p99_ttfb_ms: external_exports.number(),
|
|
4289
|
-
first_turn_ttfb_ms: external_exports.number(),
|
|
4302
|
+
p50_ttfb_ms: external_exports.number().optional(),
|
|
4303
|
+
p90_ttfb_ms: external_exports.number().optional(),
|
|
4304
|
+
p95_ttfb_ms: external_exports.number().optional(),
|
|
4305
|
+
p99_ttfb_ms: external_exports.number().optional(),
|
|
4306
|
+
first_turn_ttfb_ms: external_exports.number().optional(),
|
|
4290
4307
|
ttfw_per_turn_ms: external_exports.array(external_exports.number()).optional(),
|
|
4291
4308
|
p50_ttfw_ms: external_exports.number().optional(),
|
|
4292
4309
|
p90_ttfw_ms: external_exports.number().optional(),
|
|
@@ -4373,7 +4390,7 @@ var CallMetadataSchema = external_exports.object({
|
|
|
4373
4390
|
transfers: external_exports.array(CallTransferSchema).optional()
|
|
4374
4391
|
});
|
|
4375
4392
|
var ConversationMetricsSchema = external_exports.object({
|
|
4376
|
-
mean_ttfb_ms: external_exports.number(),
|
|
4393
|
+
mean_ttfb_ms: external_exports.number().optional(),
|
|
4377
4394
|
mean_ttfw_ms: external_exports.number().optional(),
|
|
4378
4395
|
latency: LatencyMetricsSchema.optional(),
|
|
4379
4396
|
tool_calls: ToolCallMetricsSchema.optional(),
|
|
@@ -4385,6 +4402,7 @@ var ConversationCallResultSchema = external_exports.object({
|
|
|
4385
4402
|
name: external_exports.string().optional(),
|
|
4386
4403
|
caller_prompt: external_exports.string(),
|
|
4387
4404
|
status: external_exports.enum(["completed", "error"]),
|
|
4405
|
+
end_note: external_exports.string().optional(),
|
|
4388
4406
|
transcript: external_exports.array(ConversationTurnSchema),
|
|
4389
4407
|
observed_tool_calls: external_exports.array(ObservedToolCallSchema).optional(),
|
|
4390
4408
|
duration_ms: external_exports.number(),
|
|
@@ -4401,7 +4419,7 @@ var RunAggregateV2Schema = external_exports.object({
|
|
|
4401
4419
|
total_duration_ms: external_exports.number(),
|
|
4402
4420
|
total_cost_usd: external_exports.number().optional()
|
|
4403
4421
|
});
|
|
4404
|
-
var RUNNER_TERMINAL_STATUSES = ["pass", "fail"];
|
|
4422
|
+
var RUNNER_TERMINAL_STATUSES = ["pass", "fail", "cancelled"];
|
|
4405
4423
|
var RunnerCallbackV2Schema = external_exports.object({
|
|
4406
4424
|
run_id: external_exports.string().uuid(),
|
|
4407
4425
|
status: external_exports.enum(RUNNER_TERMINAL_STATUSES),
|
|
@@ -4409,6 +4427,10 @@ var RunnerCallbackV2Schema = external_exports.object({
|
|
|
4409
4427
|
aggregate: RunAggregateV2Schema,
|
|
4410
4428
|
error_text: external_exports.string().optional()
|
|
4411
4429
|
});
|
|
4430
|
+
var RunFailedCallbackSchema = external_exports.object({
|
|
4431
|
+
run_id: external_exports.string().uuid(),
|
|
4432
|
+
error_text: external_exports.string().max(2e3)
|
|
4433
|
+
});
|
|
4412
4434
|
var RunEventCallbackSchema = external_exports.object({
|
|
4413
4435
|
run_id: external_exports.string().uuid(),
|
|
4414
4436
|
event_type: external_exports.string().min(1).max(64),
|
|
@@ -4432,7 +4454,7 @@ function formatConversationResult(raw, options = {}) {
|
|
|
4432
4454
|
const r = raw;
|
|
4433
4455
|
if (typeof r.caller_prompt !== "string") return null;
|
|
4434
4456
|
const verbose = options.verbose ?? false;
|
|
4435
|
-
const
|
|
4457
|
+
const debug = verbose ? formatDebug(r) : void 0;
|
|
4436
4458
|
const warnings = dedupeStrings([
|
|
4437
4459
|
...formatProviderWarningMessages(r.call_metadata?.provider_warnings)
|
|
4438
4460
|
]);
|
|
@@ -4441,6 +4463,7 @@ function formatConversationResult(raw, options = {}) {
|
|
|
4441
4463
|
status: r.status,
|
|
4442
4464
|
duration_ms: r.duration_ms,
|
|
4443
4465
|
error: r.error ?? null,
|
|
4466
|
+
end_note: r.end_note ?? null,
|
|
4444
4467
|
transcript: formatTranscript(r.transcript, options),
|
|
4445
4468
|
latency: r.metrics?.latency ? formatLatency(r.metrics.latency, r.metrics, verbose) : null,
|
|
4446
4469
|
tool_calls: formatToolCalls(r.metrics?.tool_calls, r.observed_tool_calls, verbose),
|
|
@@ -4449,7 +4472,7 @@ function formatConversationResult(raw, options = {}) {
|
|
|
4449
4472
|
};
|
|
4450
4473
|
if (warnings.length > 0) result.warnings = warnings;
|
|
4451
4474
|
if (verbose) result.caller_prompt = r.caller_prompt;
|
|
4452
|
-
if (
|
|
4475
|
+
if (debug) result.debug = debug;
|
|
4453
4476
|
return result;
|
|
4454
4477
|
}
|
|
4455
4478
|
function formatTranscript(turns, options) {
|
|
@@ -4463,7 +4486,7 @@ function formatTranscript(turns, options) {
|
|
|
4463
4486
|
if (t.ttfw_ms != null) turn.ttfw_ms = t.ttfw_ms;
|
|
4464
4487
|
if (t.audio_duration_ms != null) turn.audio_duration_ms = t.audio_duration_ms;
|
|
4465
4488
|
if (options.verbose) {
|
|
4466
|
-
const
|
|
4489
|
+
const debug = compactUnknownRecord({
|
|
4467
4490
|
timestamp_ms: t.timestamp_ms,
|
|
4468
4491
|
caller_decision_mode: t.caller_decision_mode,
|
|
4469
4492
|
silence_pad_ms: t.silence_pad_ms,
|
|
@@ -4473,8 +4496,8 @@ function formatTranscript(turns, options) {
|
|
|
4473
4496
|
component_latency: t.component_latency,
|
|
4474
4497
|
platform_transcript: t.platform_transcript
|
|
4475
4498
|
});
|
|
4476
|
-
if (
|
|
4477
|
-
turn.debug =
|
|
4499
|
+
if (debug && Object.keys(debug).length > 0) {
|
|
4500
|
+
turn.debug = debug;
|
|
4478
4501
|
}
|
|
4479
4502
|
}
|
|
4480
4503
|
return turn;
|
|
@@ -4483,11 +4506,13 @@ function formatTranscript(turns, options) {
|
|
|
4483
4506
|
function formatLatency(latency, metrics, verbose) {
|
|
4484
4507
|
const hasTtfw = metrics.mean_ttfw_ms != null && latency.p50_ttfw_ms != null && latency.p95_ttfw_ms != null;
|
|
4485
4508
|
const responseTimeSource = hasTtfw ? "ttfw" : "ttfb";
|
|
4509
|
+
const firstResponse = hasTtfw ? latency.first_turn_ttfw_ms ?? latency.first_turn_ttfb_ms : latency.first_turn_ttfb_ms;
|
|
4510
|
+
if (firstResponse == null || !hasTtfw && (metrics.mean_ttfb_ms == null || latency.p95_ttfb_ms == null)) return null;
|
|
4486
4511
|
const result = {
|
|
4487
4512
|
response_time_ms: hasTtfw ? metrics.mean_ttfw_ms : metrics.mean_ttfb_ms,
|
|
4488
4513
|
response_time_source: responseTimeSource,
|
|
4489
4514
|
p95_response_time_ms: hasTtfw ? latency.p95_ttfw_ms : latency.p95_ttfb_ms,
|
|
4490
|
-
first_response_time_ms:
|
|
4515
|
+
first_response_time_ms: firstResponse
|
|
4491
4516
|
};
|
|
4492
4517
|
if (!verbose) return result;
|
|
4493
4518
|
result.p50_response_time_ms = hasTtfw ? latency.p50_ttfw_ms : latency.p50_ttfb_ms;
|
|
@@ -4613,7 +4638,7 @@ function formatCallMetadata(meta, verbose) {
|
|
|
4613
4638
|
return result;
|
|
4614
4639
|
}
|
|
4615
4640
|
function formatDebug(result) {
|
|
4616
|
-
const
|
|
4641
|
+
const debug = compactUnknownRecord({
|
|
4617
4642
|
signal_quality: result.metrics?.signal_quality,
|
|
4618
4643
|
harness_overhead: result.metrics?.harness_overhead,
|
|
4619
4644
|
provider_warnings: nonEmptyArray(result.call_metadata?.provider_warnings),
|
|
@@ -4621,7 +4646,7 @@ function formatDebug(result) {
|
|
|
4621
4646
|
observed_tool_calls: formatDebugToolCalls(result.observed_tool_calls),
|
|
4622
4647
|
provider_metadata: result.call_metadata?.provider_metadata
|
|
4623
4648
|
});
|
|
4624
|
-
return
|
|
4649
|
+
return debug && Object.keys(debug).length > 0 ? debug : void 0;
|
|
4625
4650
|
}
|
|
4626
4651
|
function formatDebugToolCalls(observed) {
|
|
4627
4652
|
if (!observed || observed.length === 0) return void 0;
|
|
@@ -4654,18 +4679,242 @@ function compactUnknownRecord(record) {
|
|
|
4654
4679
|
// ../shared/src/callback-auth.ts
|
|
4655
4680
|
import { createHmac, timingSafeEqual } from "node:crypto";
|
|
4656
4681
|
|
|
4657
|
-
// src/lib/
|
|
4658
|
-
var
|
|
4659
|
-
var
|
|
4660
|
-
|
|
4661
|
-
|
|
4662
|
-
|
|
4663
|
-
|
|
4664
|
-
|
|
4682
|
+
// src/lib/config.ts
|
|
4683
|
+
var CONFIG_DIR = path.join(homedir(), ".vent");
|
|
4684
|
+
var CREDENTIALS_FILE = path.join(CONFIG_DIR, "credentials");
|
|
4685
|
+
var API_BASE = process.env.VENT_API_URL ?? "https://api.venthq.dev";
|
|
4686
|
+
var DASHBOARD_URL = process.env.VENT_DASHBOARD_URL ?? "https://venthq.dev";
|
|
4687
|
+
var refreshes = /* @__PURE__ */ new Map();
|
|
4688
|
+
async function readCredentials() {
|
|
4689
|
+
const accessToken = process.env.VENT_ACCESS_TOKEN;
|
|
4690
|
+
const refreshToken = process.env.VENT_REFRESH_TOKEN;
|
|
4691
|
+
if (accessToken || refreshToken) return { accessToken, refreshToken, environment: true };
|
|
4692
|
+
try {
|
|
4693
|
+
const parsed = AccessCredentialsSchema.safeParse(JSON.parse(await fs.readFile(CREDENTIALS_FILE, "utf-8")));
|
|
4694
|
+
if (parsed.success) return {
|
|
4695
|
+
accessToken: parsed.data.access_token,
|
|
4696
|
+
refreshToken: parsed.data.refresh_token,
|
|
4697
|
+
environment: false
|
|
4698
|
+
};
|
|
4699
|
+
} catch {
|
|
4700
|
+
}
|
|
4701
|
+
return null;
|
|
4702
|
+
}
|
|
4703
|
+
function usableAccessToken(token, marginMs = 0) {
|
|
4704
|
+
if (!token) return false;
|
|
4705
|
+
try {
|
|
4706
|
+
return (decodeJwt(token).exp ?? 0) * 1e3 > Date.now() + marginMs;
|
|
4707
|
+
} catch {
|
|
4708
|
+
return false;
|
|
4709
|
+
}
|
|
4710
|
+
}
|
|
4711
|
+
async function refresh(source) {
|
|
4712
|
+
const refreshToken = source.refreshToken;
|
|
4713
|
+
const existing = refreshes.get(refreshToken);
|
|
4714
|
+
if (existing) return existing;
|
|
4715
|
+
const work = (async () => {
|
|
4716
|
+
let response;
|
|
4717
|
+
try {
|
|
4718
|
+
response = await fetch(`${API_BASE}/auth/refresh`, {
|
|
4719
|
+
method: "POST",
|
|
4720
|
+
headers: { "Content-Type": "application/json" },
|
|
4721
|
+
body: JSON.stringify({ refresh_token: refreshToken }),
|
|
4722
|
+
signal: AbortSignal.timeout(1e4)
|
|
4723
|
+
});
|
|
4724
|
+
} catch {
|
|
4725
|
+
throw new Error("Could not refresh Vent sign-in. Check your connection and try again.");
|
|
4726
|
+
}
|
|
4727
|
+
if (!response.ok) {
|
|
4728
|
+
throw new Error(response.status === 401 ? "Vent credentials were rejected. Run `npx vent-hq login` again or replace VENT_REFRESH_TOKEN." : `Vent sign-in refresh failed (${response.status}). Try again.`);
|
|
4729
|
+
}
|
|
4730
|
+
const parsed = AccessCredentialsSchema.safeParse(await response.json());
|
|
4731
|
+
if (!parsed.success) throw new Error("Vent returned invalid sign-in credentials.");
|
|
4732
|
+
const credentials = parsed.data;
|
|
4733
|
+
if (source.environment) {
|
|
4734
|
+
process.env.VENT_ACCESS_TOKEN = credentials.access_token;
|
|
4735
|
+
process.env.VENT_REFRESH_TOKEN = credentials.refresh_token;
|
|
4736
|
+
} else {
|
|
4737
|
+
await saveCredentials(credentials);
|
|
4738
|
+
}
|
|
4739
|
+
return credentials.access_token;
|
|
4740
|
+
})();
|
|
4741
|
+
refreshes.set(refreshToken, work);
|
|
4742
|
+
try {
|
|
4743
|
+
return await work;
|
|
4744
|
+
} finally {
|
|
4745
|
+
refreshes.delete(refreshToken);
|
|
4746
|
+
}
|
|
4747
|
+
}
|
|
4748
|
+
async function loadAccessToken() {
|
|
4749
|
+
const source = await readCredentials();
|
|
4750
|
+
if (!source) return null;
|
|
4751
|
+
if (source.refreshToken && !usableAccessToken(source.accessToken, 6e4)) return refresh(source);
|
|
4752
|
+
return usableAccessToken(source.accessToken) ? source.accessToken : null;
|
|
4753
|
+
}
|
|
4754
|
+
async function refreshAccessToken(rejectedToken) {
|
|
4755
|
+
const source = await readCredentials();
|
|
4756
|
+
if (!source?.refreshToken) return null;
|
|
4757
|
+
if (source.accessToken !== rejectedToken && usableAccessToken(source.accessToken, 6e4)) {
|
|
4758
|
+
return source.accessToken;
|
|
4759
|
+
}
|
|
4760
|
+
return refresh(source);
|
|
4761
|
+
}
|
|
4762
|
+
async function saveCredentials(credentials) {
|
|
4763
|
+
await fs.mkdir(CONFIG_DIR, { recursive: true, mode: 448 });
|
|
4764
|
+
const temporary = `${CREDENTIALS_FILE}.${randomUUID()}.tmp`;
|
|
4765
|
+
try {
|
|
4766
|
+
await fs.writeFile(temporary, `${JSON.stringify(credentials)}
|
|
4767
|
+
`, { mode: 384, flag: "wx" });
|
|
4768
|
+
await fs.rename(temporary, CREDENTIALS_FILE);
|
|
4769
|
+
} finally {
|
|
4770
|
+
await fs.rm(temporary, { force: true });
|
|
4771
|
+
}
|
|
4772
|
+
}
|
|
4773
|
+
async function deleteCredentials() {
|
|
4774
|
+
try {
|
|
4775
|
+
await fs.rm(CREDENTIALS_FILE);
|
|
4776
|
+
} catch {
|
|
4777
|
+
}
|
|
4778
|
+
}
|
|
4779
|
+
|
|
4780
|
+
// src/lib/api.ts
|
|
4781
|
+
var ApiError = class extends Error {
|
|
4782
|
+
constructor(status, body) {
|
|
4783
|
+
super(`API error ${status}: ${typeof body === "string" ? body : JSON.stringify(body)}`);
|
|
4784
|
+
this.status = status;
|
|
4785
|
+
this.body = body;
|
|
4786
|
+
}
|
|
4787
|
+
};
|
|
4788
|
+
async function authenticatedFetch(path4, apiKey, options = {}) {
|
|
4789
|
+
const url = `${API_BASE}${path4}`;
|
|
4790
|
+
const token = await loadAccessToken() ?? apiKey;
|
|
4791
|
+
const send = (accessToken) => {
|
|
4792
|
+
const headers = new Headers(options.headers);
|
|
4793
|
+
headers.set("Authorization", `Bearer ${accessToken}`);
|
|
4794
|
+
if (!headers.has("Content-Type")) headers.set("Content-Type", "application/json");
|
|
4795
|
+
return fetch(url, { ...options, headers });
|
|
4796
|
+
};
|
|
4797
|
+
const response = await send(token);
|
|
4798
|
+
if (response.status !== 401) return response;
|
|
4799
|
+
const refreshed = await refreshAccessToken(token);
|
|
4800
|
+
if (!refreshed) return response;
|
|
4801
|
+
await response.body?.cancel();
|
|
4802
|
+
return send(refreshed);
|
|
4803
|
+
}
|
|
4804
|
+
async function apiFetch(path4, apiKey, options = {}) {
|
|
4805
|
+
const res = await authenticatedFetch(path4, apiKey, options);
|
|
4806
|
+
if (!res.ok) {
|
|
4807
|
+
const text = await res.text();
|
|
4808
|
+
let body = text;
|
|
4809
|
+
try {
|
|
4810
|
+
body = JSON.parse(text);
|
|
4811
|
+
} catch {
|
|
4812
|
+
}
|
|
4813
|
+
throw new ApiError(res.status, body);
|
|
4814
|
+
}
|
|
4815
|
+
return res;
|
|
4665
4816
|
}
|
|
4666
|
-
|
|
4667
|
-
|
|
4817
|
+
|
|
4818
|
+
// src/lib/submit-run.ts
|
|
4819
|
+
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
4820
|
+
var RETRY_DELAY_MS = 2e3;
|
|
4821
|
+
async function submitRun(accessToken, payload) {
|
|
4822
|
+
const body = JSON.stringify({ ...payload, idempotency_key: randomUUID2() });
|
|
4823
|
+
const send = () => apiFetch("/runs/submit", accessToken, { method: "POST", body });
|
|
4824
|
+
try {
|
|
4825
|
+
return await send();
|
|
4826
|
+
} catch (err) {
|
|
4827
|
+
const answerMayBeLost = !(err instanceof ApiError) || [502, 503, 504].includes(err.status);
|
|
4828
|
+
if (!answerMayBeLost) throw err;
|
|
4829
|
+
await new Promise((resolve2) => setTimeout(resolve2, RETRY_DELAY_MS));
|
|
4830
|
+
return send();
|
|
4831
|
+
}
|
|
4668
4832
|
}
|
|
4833
|
+
|
|
4834
|
+
// src/lib/sse.ts
|
|
4835
|
+
var MAX_RETRIES = 5;
|
|
4836
|
+
var RETRY_DELAY_MS2 = 2e3;
|
|
4837
|
+
async function* streamRunEvents(runId, apiKey, signal) {
|
|
4838
|
+
const path4 = `/runs/${runId}/stream`;
|
|
4839
|
+
const seenIds = /* @__PURE__ */ new Set();
|
|
4840
|
+
let retries = 0;
|
|
4841
|
+
while (retries <= MAX_RETRIES) {
|
|
4842
|
+
if (retries > 0) {
|
|
4843
|
+
await new Promise((r) => setTimeout(r, RETRY_DELAY_MS2));
|
|
4844
|
+
}
|
|
4845
|
+
let res;
|
|
4846
|
+
try {
|
|
4847
|
+
res = await authenticatedFetch(path4, apiKey, {
|
|
4848
|
+
signal
|
|
4849
|
+
});
|
|
4850
|
+
} catch (err) {
|
|
4851
|
+
if (err.name === "AbortError") throw err;
|
|
4852
|
+
retries++;
|
|
4853
|
+
continue;
|
|
4854
|
+
}
|
|
4855
|
+
if (res.status >= 502 && res.status <= 504) {
|
|
4856
|
+
retries++;
|
|
4857
|
+
continue;
|
|
4858
|
+
}
|
|
4859
|
+
if (!res.ok) {
|
|
4860
|
+
const body = await res.text();
|
|
4861
|
+
throw new Error(`SSE stream failed (${res.status}): ${body}`);
|
|
4862
|
+
}
|
|
4863
|
+
if (!res.body) {
|
|
4864
|
+
throw new Error("SSE stream returned no body");
|
|
4865
|
+
}
|
|
4866
|
+
const reader = res.body.getReader();
|
|
4867
|
+
const decoder2 = new TextDecoder();
|
|
4868
|
+
let buffer = "";
|
|
4869
|
+
try {
|
|
4870
|
+
while (true) {
|
|
4871
|
+
let readResult;
|
|
4872
|
+
try {
|
|
4873
|
+
readResult = await reader.read();
|
|
4874
|
+
} catch (err) {
|
|
4875
|
+
if (err.name === "AbortError") throw err;
|
|
4876
|
+
break;
|
|
4877
|
+
}
|
|
4878
|
+
const { done, value } = readResult;
|
|
4879
|
+
if (done) {
|
|
4880
|
+
break;
|
|
4881
|
+
}
|
|
4882
|
+
const chunk = decoder2.decode(value, { stream: true });
|
|
4883
|
+
buffer += chunk;
|
|
4884
|
+
const lines = buffer.split("\n");
|
|
4885
|
+
buffer = lines.pop();
|
|
4886
|
+
for (const line of lines) {
|
|
4887
|
+
if (line.startsWith("data: ")) {
|
|
4888
|
+
const raw = line.slice(6);
|
|
4889
|
+
try {
|
|
4890
|
+
const event = JSON.parse(raw);
|
|
4891
|
+
if (event.id && seenIds.has(event.id)) {
|
|
4892
|
+
continue;
|
|
4893
|
+
}
|
|
4894
|
+
if (event.id) seenIds.add(event.id);
|
|
4895
|
+
yield event;
|
|
4896
|
+
if (event.event_type === "run_complete") {
|
|
4897
|
+
return;
|
|
4898
|
+
}
|
|
4899
|
+
} catch {
|
|
4900
|
+
}
|
|
4901
|
+
}
|
|
4902
|
+
}
|
|
4903
|
+
}
|
|
4904
|
+
} finally {
|
|
4905
|
+
reader.releaseLock();
|
|
4906
|
+
}
|
|
4907
|
+
retries++;
|
|
4908
|
+
}
|
|
4909
|
+
yield {
|
|
4910
|
+
event_type: "error",
|
|
4911
|
+
message: `Stream lost after ${MAX_RETRIES} reconnect attempts without receiving run_complete`
|
|
4912
|
+
};
|
|
4913
|
+
}
|
|
4914
|
+
|
|
4915
|
+
// src/lib/output.ts
|
|
4916
|
+
import { writeFileSync } from "node:fs";
|
|
4917
|
+
var isTTY = process.stdout.isTTY;
|
|
4669
4918
|
function stdoutSync(data) {
|
|
4670
4919
|
if (isTTY) {
|
|
4671
4920
|
process.stdout.write(data);
|
|
@@ -4746,6 +4995,9 @@ function printRunComplete(meta) {
|
|
|
4746
4995
|
stdoutSync(`${total} call${total === 1 ? "" : "s"} ran
|
|
4747
4996
|
`);
|
|
4748
4997
|
}
|
|
4998
|
+
if (typeof meta.error_text === "string") {
|
|
4999
|
+
stdoutSync(red(`Run ${String(meta.status ?? "ended")}: ${meta.error_text}`) + "\n");
|
|
5000
|
+
}
|
|
4749
5001
|
}
|
|
4750
5002
|
function printSummary(callResults, runComplete, runId, options = {}) {
|
|
4751
5003
|
const allCalls = options.rawCalls ? formatRawCalls(options.rawCalls, options.verbose ?? false) : callResults.map((e) => {
|
|
@@ -4763,10 +5015,14 @@ function printSummary(callResults, runComplete, runId, options = {}) {
|
|
|
4763
5015
|
const counts = agg?.conversation_calls;
|
|
4764
5016
|
const summaryData = buildRunSummaryJson({
|
|
4765
5017
|
runId,
|
|
5018
|
+
status: runComplete.status,
|
|
4766
5019
|
total: runComplete.total_calls ?? counts?.total,
|
|
4767
5020
|
formattedCalls: allCalls,
|
|
4768
5021
|
verbose: options.verbose,
|
|
4769
|
-
runDetails: options.runDetails ?? {
|
|
5022
|
+
runDetails: options.runDetails ?? {
|
|
5023
|
+
aggregate: runComplete.aggregate,
|
|
5024
|
+
error_text: runComplete.error_text
|
|
5025
|
+
}
|
|
4770
5026
|
});
|
|
4771
5027
|
if (!isTTY) {
|
|
4772
5028
|
stdoutSync(JSON.stringify(summaryData, null, 2) + "\n");
|
|
@@ -4843,141 +5099,24 @@ function printError(message) {
|
|
|
4843
5099
|
stdoutSync(line);
|
|
4844
5100
|
}
|
|
4845
5101
|
function printInfo(message, { force } = {}) {
|
|
4846
|
-
if (!force && !isTTY
|
|
5102
|
+
if (!force && !isTTY) return;
|
|
4847
5103
|
const line = blue("\u25B8") + ` ${message}
|
|
4848
5104
|
`;
|
|
4849
5105
|
stdoutSync(line);
|
|
4850
5106
|
}
|
|
4851
5107
|
function printSuccess(message, { force } = {}) {
|
|
4852
|
-
if (!force && !isTTY
|
|
5108
|
+
if (!force && !isTTY) return;
|
|
4853
5109
|
const line = green("\u2714") + ` ${message}
|
|
4854
5110
|
`;
|
|
4855
5111
|
stdoutSync(line);
|
|
4856
5112
|
}
|
|
4857
5113
|
function printWarn(message, { force } = {}) {
|
|
4858
|
-
if (!force && !isTTY
|
|
5114
|
+
if (!force && !isTTY) return;
|
|
4859
5115
|
const line = yellow("\u26A0") + ` ${message}
|
|
4860
5116
|
`;
|
|
4861
5117
|
stdoutSync(line);
|
|
4862
5118
|
}
|
|
4863
5119
|
|
|
4864
|
-
// src/lib/sse.ts
|
|
4865
|
-
function log(msg) {
|
|
4866
|
-
if (!isVerbose()) return;
|
|
4867
|
-
const ts = (/* @__PURE__ */ new Date()).toISOString().slice(11, 23);
|
|
4868
|
-
const line = `[vent:sse ${ts}] ${msg}
|
|
4869
|
-
`;
|
|
4870
|
-
process.stdout.write(line);
|
|
4871
|
-
}
|
|
4872
|
-
var MAX_RETRIES = 5;
|
|
4873
|
-
var RETRY_DELAY_MS = 2e3;
|
|
4874
|
-
async function* streamRunEvents(runId, apiKey, signal) {
|
|
4875
|
-
const url = `${API_BASE}/runs/${runId}/stream`;
|
|
4876
|
-
const seenIds = /* @__PURE__ */ new Set();
|
|
4877
|
-
let retries = 0;
|
|
4878
|
-
while (retries <= MAX_RETRIES) {
|
|
4879
|
-
if (retries > 0) {
|
|
4880
|
-
log(`reconnecting (attempt ${retries}/${MAX_RETRIES}) after ${RETRY_DELAY_MS}ms\u2026`);
|
|
4881
|
-
await new Promise((r) => setTimeout(r, RETRY_DELAY_MS));
|
|
4882
|
-
}
|
|
4883
|
-
log(`connecting to ${url}`);
|
|
4884
|
-
let res;
|
|
4885
|
-
try {
|
|
4886
|
-
res = await fetch(url, {
|
|
4887
|
-
headers: { Authorization: `Bearer ${apiKey}` },
|
|
4888
|
-
signal
|
|
4889
|
-
});
|
|
4890
|
-
} catch (err) {
|
|
4891
|
-
if (err.name === "AbortError") throw err;
|
|
4892
|
-
log(`fetch error: ${err.message}`);
|
|
4893
|
-
retries++;
|
|
4894
|
-
continue;
|
|
4895
|
-
}
|
|
4896
|
-
log(`response: status=${res.status} content-type=${res.headers.get("content-type")}`);
|
|
4897
|
-
if (!res.ok) {
|
|
4898
|
-
const body = await res.text();
|
|
4899
|
-
log(`error body: ${body}`);
|
|
4900
|
-
throw new Error(`SSE stream failed (${res.status}): ${body}`);
|
|
4901
|
-
}
|
|
4902
|
-
if (!res.body) {
|
|
4903
|
-
throw new Error("SSE stream returned no body");
|
|
4904
|
-
}
|
|
4905
|
-
const reader = res.body.getReader();
|
|
4906
|
-
const decoder = new TextDecoder();
|
|
4907
|
-
let buffer = "";
|
|
4908
|
-
let chunkCount = 0;
|
|
4909
|
-
let eventCount = 0;
|
|
4910
|
-
let gotRunComplete = false;
|
|
4911
|
-
let streamError = null;
|
|
4912
|
-
try {
|
|
4913
|
-
while (true) {
|
|
4914
|
-
let readResult;
|
|
4915
|
-
try {
|
|
4916
|
-
readResult = await reader.read();
|
|
4917
|
-
} catch (err) {
|
|
4918
|
-
if (err.name === "AbortError") throw err;
|
|
4919
|
-
streamError = err;
|
|
4920
|
-
log(`read error: ${streamError.message}`);
|
|
4921
|
-
break;
|
|
4922
|
-
}
|
|
4923
|
-
const { done, value } = readResult;
|
|
4924
|
-
if (done) {
|
|
4925
|
-
log(`stream done after ${chunkCount} chunks, ${eventCount} events`);
|
|
4926
|
-
break;
|
|
4927
|
-
}
|
|
4928
|
-
chunkCount++;
|
|
4929
|
-
const chunk = decoder.decode(value, { stream: true });
|
|
4930
|
-
buffer += chunk;
|
|
4931
|
-
if (chunkCount <= 3 || chunkCount % 10 === 0) {
|
|
4932
|
-
log(`chunk #${chunkCount} (${chunk.length} bytes) buffer=${buffer.length} bytes`);
|
|
4933
|
-
}
|
|
4934
|
-
const lines = buffer.split("\n");
|
|
4935
|
-
buffer = lines.pop();
|
|
4936
|
-
for (const line of lines) {
|
|
4937
|
-
if (line.startsWith("data: ")) {
|
|
4938
|
-
const raw = line.slice(6);
|
|
4939
|
-
try {
|
|
4940
|
-
const event = JSON.parse(raw);
|
|
4941
|
-
eventCount++;
|
|
4942
|
-
if (event.id && seenIds.has(event.id)) {
|
|
4943
|
-
log(`skipping duplicate event ${event.id}`);
|
|
4944
|
-
continue;
|
|
4945
|
-
}
|
|
4946
|
-
if (event.id) seenIds.add(event.id);
|
|
4947
|
-
log(`parsed event #${eventCount}: type=${event.event_type}`);
|
|
4948
|
-
yield event;
|
|
4949
|
-
if (event.event_type === "run_complete") {
|
|
4950
|
-
log("run_complete received \u2014 closing stream");
|
|
4951
|
-
gotRunComplete = true;
|
|
4952
|
-
return;
|
|
4953
|
-
}
|
|
4954
|
-
} catch {
|
|
4955
|
-
log(`malformed JSON: ${raw.slice(0, 200)}`);
|
|
4956
|
-
}
|
|
4957
|
-
} else if (line.startsWith(": ")) {
|
|
4958
|
-
if (chunkCount <= 3) {
|
|
4959
|
-
log(`heartbeat: "${line}"`);
|
|
4960
|
-
}
|
|
4961
|
-
}
|
|
4962
|
-
}
|
|
4963
|
-
}
|
|
4964
|
-
} finally {
|
|
4965
|
-
reader.releaseLock();
|
|
4966
|
-
log("reader released");
|
|
4967
|
-
}
|
|
4968
|
-
if (gotRunComplete) return;
|
|
4969
|
-
retries++;
|
|
4970
|
-
if (retries <= MAX_RETRIES) {
|
|
4971
|
-
log(`stream ended without run_complete \u2014 will retry (${retries}/${MAX_RETRIES})`);
|
|
4972
|
-
}
|
|
4973
|
-
}
|
|
4974
|
-
log(`exhausted ${MAX_RETRIES} retries without run_complete`);
|
|
4975
|
-
yield {
|
|
4976
|
-
event_type: "error",
|
|
4977
|
-
message: `Stream lost after ${MAX_RETRIES} reconnect attempts without receiving run_complete`
|
|
4978
|
-
};
|
|
4979
|
-
}
|
|
4980
|
-
|
|
4981
5120
|
// src/lib/run-history.ts
|
|
4982
5121
|
import * as fs2 from "node:fs/promises";
|
|
4983
5122
|
import * as path2 from "node:path";
|
|
@@ -5028,7 +5167,7 @@ async function saveRunHistory(runId, callResults, runCompleteData) {
|
|
|
5028
5167
|
}
|
|
5029
5168
|
}
|
|
5030
5169
|
|
|
5031
|
-
// src/lib/platform-
|
|
5170
|
+
// src/lib/platform-config.ts
|
|
5032
5171
|
var PLATFORM_ENV_MAP = {
|
|
5033
5172
|
vapi: { vapi_api_key: "VAPI_API_KEY", vapi_assistant_id: "VAPI_ASSISTANT_ID" },
|
|
5034
5173
|
bland: { bland_api_key: "BLAND_API_KEY", bland_pathway_id: "BLAND_PATHWAY_ID", persona_id: "BLAND_PERSONA_ID" },
|
|
@@ -5096,7 +5235,6 @@ function resolveRemotePlatformConfig(config) {
|
|
|
5096
5235
|
const cfg = config;
|
|
5097
5236
|
const adapter = cfg.connection?.adapter;
|
|
5098
5237
|
if (!isRemotePlatformAdapter(adapter)) return null;
|
|
5099
|
-
if (cfg.connection?.platform_connection_id) return null;
|
|
5100
5238
|
const current = cfg.connection?.platform ?? {};
|
|
5101
5239
|
const provider = current.provider ?? adapter;
|
|
5102
5240
|
if (provider !== adapter) {
|
|
@@ -5129,19 +5267,10 @@ function resolveRemotePlatformConfig(config) {
|
|
|
5129
5267
|
// src/commands/run.ts
|
|
5130
5268
|
var isTTY2 = process.stdout.isTTY;
|
|
5131
5269
|
async function runCommand(args) {
|
|
5132
|
-
debug(`start args=${JSON.stringify({ file: args.file, session: args.session })}`);
|
|
5133
|
-
const accessToken = await loadAccessToken();
|
|
5134
|
-
if (!accessToken) {
|
|
5135
|
-
printError("No Vent access token found. Set VENT_ACCESS_TOKEN, run `npx vent-hq login`, or pass --access-token.");
|
|
5136
|
-
return 2;
|
|
5137
|
-
}
|
|
5138
|
-
debug(`access-token resolved (${accessToken.slice(0, 8)}\u2026)`);
|
|
5139
5270
|
let config;
|
|
5140
5271
|
try {
|
|
5141
|
-
debug(`reading suite file: ${args.file}`);
|
|
5142
5272
|
const raw = await fs3.readFile(args.file, "utf-8");
|
|
5143
5273
|
const suite = JSON.parse(raw);
|
|
5144
|
-
debug(`suite parsed \u2014 keys: ${Object.keys(suite).join(", ")}`);
|
|
5145
5274
|
if (!suite.connection) {
|
|
5146
5275
|
printError("Suite file must have a `connection` object.");
|
|
5147
5276
|
return 2;
|
|
@@ -5164,7 +5293,6 @@ async function runCommand(args) {
|
|
|
5164
5293
|
printError(`Suite has ${callNames.length} calls. Use --call <name> to pick one: ${callNames.join(", ")}`);
|
|
5165
5294
|
return 2;
|
|
5166
5295
|
}
|
|
5167
|
-
debug(`selected call: ${callName}`);
|
|
5168
5296
|
const call = { ...suite.calls[callName], name: callName };
|
|
5169
5297
|
config = { connection: suite.connection, call };
|
|
5170
5298
|
} catch (err) {
|
|
@@ -5175,9 +5303,11 @@ async function runCommand(args) {
|
|
|
5175
5303
|
printError(`Invalid suite JSON: ${err.message}`);
|
|
5176
5304
|
return 2;
|
|
5177
5305
|
}
|
|
5178
|
-
let resolvedRemotePlatform = null;
|
|
5179
5306
|
try {
|
|
5180
|
-
|
|
5307
|
+
const platform = resolveRemotePlatformConfig(config);
|
|
5308
|
+
if (platform) {
|
|
5309
|
+
config.connection.platform = platform;
|
|
5310
|
+
}
|
|
5181
5311
|
} catch (err) {
|
|
5182
5312
|
printError(err.message);
|
|
5183
5313
|
return 2;
|
|
@@ -5190,41 +5320,17 @@ async function runCommand(args) {
|
|
|
5190
5320
|
);
|
|
5191
5321
|
return 2;
|
|
5192
5322
|
}
|
|
5193
|
-
|
|
5323
|
+
const accessToken = await loadAccessToken();
|
|
5324
|
+
if (!accessToken) {
|
|
5325
|
+
printError("No Vent credentials found. Set VENT_REFRESH_TOKEN or run `npx vent-hq login`.");
|
|
5326
|
+
return 2;
|
|
5327
|
+
}
|
|
5194
5328
|
printInfo("Submitting run\u2026");
|
|
5195
5329
|
let submitResult;
|
|
5196
|
-
let activeAccessToken = accessToken;
|
|
5197
|
-
const configConnection = config;
|
|
5198
|
-
async function prepareConfigForSubmit(currentAccessToken) {
|
|
5199
|
-
if (!configConnection.connection) return;
|
|
5200
|
-
if (resolvedRemotePlatform) {
|
|
5201
|
-
debug(`ensuring saved ${resolvedRemotePlatform.provider} connection\u2026`);
|
|
5202
|
-
const ensured = await ensurePlatformConnection(currentAccessToken, resolvedRemotePlatform);
|
|
5203
|
-
configConnection.connection.platform_connection_id = ensured.platform_connection_id;
|
|
5204
|
-
debug(
|
|
5205
|
-
`saved connection ready id=${ensured.platform_connection_id} created=${ensured.created} updated=${ensured.updated}`
|
|
5206
|
-
);
|
|
5207
|
-
}
|
|
5208
|
-
if ("platform" in configConnection.connection) {
|
|
5209
|
-
delete configConnection.connection.platform;
|
|
5210
|
-
}
|
|
5211
|
-
}
|
|
5212
|
-
async function submitPrepared(currentAccessToken) {
|
|
5213
|
-
await prepareConfigForSubmit(currentAccessToken);
|
|
5214
|
-
const res = await apiFetch("/runs/submit", currentAccessToken, {
|
|
5215
|
-
method: "POST",
|
|
5216
|
-
body: JSON.stringify({
|
|
5217
|
-
config,
|
|
5218
|
-
agent_session_id: args.session
|
|
5219
|
-
})
|
|
5220
|
-
});
|
|
5221
|
-
debug(`API response status: ${res.status}`);
|
|
5222
|
-
return res.json();
|
|
5223
|
-
}
|
|
5224
5330
|
try {
|
|
5225
|
-
|
|
5331
|
+
const res = await submitRun(accessToken, { config, agent_session_id: args.session });
|
|
5332
|
+
submitResult = await res.json();
|
|
5226
5333
|
} catch (err) {
|
|
5227
|
-
debug(`submit error: ${err.message}`);
|
|
5228
5334
|
if (err instanceof ApiError && err.status === 401) {
|
|
5229
5335
|
printError("Access token rejected. Run `npx vent-hq login` to re-authenticate.");
|
|
5230
5336
|
return 1;
|
|
@@ -5234,13 +5340,10 @@ async function runCommand(args) {
|
|
|
5234
5340
|
}
|
|
5235
5341
|
const { run_id } = submitResult;
|
|
5236
5342
|
if (!run_id) {
|
|
5237
|
-
debug(`no run_id in response: ${JSON.stringify(submitResult)}`);
|
|
5238
5343
|
printError("Server returned no run_id. Response: " + JSON.stringify(submitResult));
|
|
5239
5344
|
return 2;
|
|
5240
5345
|
}
|
|
5241
|
-
debug(`run created: ${run_id} status=${submitResult.status}`);
|
|
5242
5346
|
printInfo(`Run ${run_id} created.`);
|
|
5243
|
-
debug(`connecting to SSE stream for run ${run_id}\u2026`);
|
|
5244
5347
|
printInfo(`Streaming results for run ${run_id}\u2026`);
|
|
5245
5348
|
const abortController = new AbortController();
|
|
5246
5349
|
let exitCode = 0;
|
|
@@ -5249,14 +5352,12 @@ async function runCommand(args) {
|
|
|
5249
5352
|
let stopRequested = false;
|
|
5250
5353
|
let stopPromise = null;
|
|
5251
5354
|
const onSignal = () => {
|
|
5252
|
-
debug("received SIGINT/SIGTERM \u2014 requesting cancel and aborting stream");
|
|
5253
5355
|
if (!stopRequested) {
|
|
5254
5356
|
stopRequested = true;
|
|
5255
|
-
stopPromise = apiFetch(`/runs/${run_id}/stop`,
|
|
5357
|
+
stopPromise = apiFetch(`/runs/${run_id}/stop`, accessToken, {
|
|
5256
5358
|
method: "POST",
|
|
5257
5359
|
signal: AbortSignal.timeout(5e3)
|
|
5258
|
-
}).catch((
|
|
5259
|
-
debug(`stop POST failed: ${err.message}`);
|
|
5360
|
+
}).catch(() => {
|
|
5260
5361
|
});
|
|
5261
5362
|
}
|
|
5262
5363
|
abortController.abort();
|
|
@@ -5264,54 +5365,38 @@ async function runCommand(args) {
|
|
|
5264
5365
|
process.on("SIGINT", onSignal);
|
|
5265
5366
|
process.on("SIGTERM", onSignal);
|
|
5266
5367
|
try {
|
|
5267
|
-
|
|
5268
|
-
for await (const event of streamRunEvents(run_id, activeAccessToken, abortController.signal)) {
|
|
5269
|
-
eventCount++;
|
|
5368
|
+
for await (const event of streamRunEvents(run_id, accessToken, abortController.signal)) {
|
|
5270
5369
|
const meta = event.metadata_json ?? {};
|
|
5271
|
-
debug(`event #${eventCount}: type=${event.event_type} meta_keys=[${Object.keys(meta).join(",")}] message="${event.message ?? ""}"`);
|
|
5272
5370
|
printEvent(event);
|
|
5273
5371
|
if (event.event_type === "call_completed") {
|
|
5274
5372
|
callResults.push(event);
|
|
5275
|
-
debug(`call_completed: name=${meta.call_name} status=${meta.status} duration=${meta.duration_ms}ms completed=${meta.completed}/${meta.total}`);
|
|
5276
5373
|
}
|
|
5277
5374
|
if (event.event_type === "run_complete") {
|
|
5278
5375
|
runCompleteData = meta;
|
|
5279
|
-
|
|
5280
|
-
const callMeta = e.metadata_json ?? {};
|
|
5281
|
-
const callStatus = callMeta["status"];
|
|
5282
|
-
return callStatus === "error";
|
|
5283
|
-
});
|
|
5284
|
-
exitCode = anyCallErrored ? 1 : 0;
|
|
5285
|
-
debug(`run_complete: anyCallErrored=${anyCallErrored} exitCode=${exitCode}`);
|
|
5376
|
+
exitCode = meta["status"] === "pass" ? 0 : 1;
|
|
5286
5377
|
}
|
|
5287
5378
|
if (event.event_type === "error") {
|
|
5288
5379
|
printError(event.message ?? "Stream connection lost");
|
|
5289
5380
|
exitCode = 2;
|
|
5290
5381
|
}
|
|
5291
5382
|
}
|
|
5292
|
-
debug(`SSE stream ended \u2014 received ${eventCount} events total`);
|
|
5293
5383
|
} catch (err) {
|
|
5294
5384
|
if (err.name !== "AbortError") {
|
|
5295
|
-
debug(`stream error: ${err.message}`);
|
|
5296
5385
|
printError(`Stream error: ${err.message}`);
|
|
5297
5386
|
exitCode = 2;
|
|
5298
|
-
} else {
|
|
5299
|
-
debug("stream aborted (user signal)");
|
|
5300
5387
|
}
|
|
5301
5388
|
} finally {
|
|
5302
5389
|
process.off("SIGINT", onSignal);
|
|
5303
5390
|
process.off("SIGTERM", onSignal);
|
|
5304
5391
|
}
|
|
5305
|
-
debug(`summary: callResults=${callResults.length} runComplete=${!!runCompleteData} exitCode=${exitCode}`);
|
|
5306
5392
|
if (runCompleteData) {
|
|
5307
5393
|
let rawRunDetails = null;
|
|
5308
|
-
if (args.verbose && !isTTY2) {
|
|
5394
|
+
if (args.verbose && !isTTY2 || callResults.length === 0) {
|
|
5309
5395
|
try {
|
|
5310
|
-
const res = await apiFetch(`/runs/${run_id}`,
|
|
5396
|
+
const res = await apiFetch(`/runs/${run_id}`, accessToken);
|
|
5311
5397
|
rawRunDetails = await res.json();
|
|
5312
|
-
} catch
|
|
5313
|
-
|
|
5314
|
-
printWarn("Verbose result fetch failed; falling back to streamed summary.");
|
|
5398
|
+
} catch {
|
|
5399
|
+
printWarn("Run result fetch failed; showing the streamed summary only.");
|
|
5315
5400
|
}
|
|
5316
5401
|
}
|
|
5317
5402
|
printSummary(callResults, runCompleteData, run_id, {
|
|
@@ -5330,11 +5415,9 @@ async function runCommand(args) {
|
|
|
5330
5415
|
if (runCompleteData) {
|
|
5331
5416
|
const savedPath = await saveRunHistory(run_id, callResults, runCompleteData);
|
|
5332
5417
|
if (savedPath) {
|
|
5333
|
-
debug(`run saved to ${savedPath}`);
|
|
5334
5418
|
printInfo(`Run saved to ${savedPath}`);
|
|
5335
5419
|
}
|
|
5336
5420
|
}
|
|
5337
|
-
debug(`exiting with code ${exitCode}`);
|
|
5338
5421
|
if (stopPromise) {
|
|
5339
5422
|
await stopPromise;
|
|
5340
5423
|
}
|
|
@@ -5355,14 +5438,9 @@ var RelayClient = class {
|
|
|
5355
5438
|
config;
|
|
5356
5439
|
closed = false;
|
|
5357
5440
|
handlers = /* @__PURE__ */ new Map();
|
|
5358
|
-
_agentEnv = {};
|
|
5359
5441
|
constructor(config) {
|
|
5360
5442
|
this.config = config;
|
|
5361
5443
|
}
|
|
5362
|
-
/** Env vars received from Vent server to inject into the agent process. */
|
|
5363
|
-
get agentEnv() {
|
|
5364
|
-
return this._agentEnv;
|
|
5365
|
-
}
|
|
5366
5444
|
on(event, handler) {
|
|
5367
5445
|
const list = this.handlers.get(event) ?? [];
|
|
5368
5446
|
list.push(handler);
|
|
@@ -5374,16 +5452,11 @@ var RelayClient = class {
|
|
|
5374
5452
|
}
|
|
5375
5453
|
}
|
|
5376
5454
|
async connect(timeoutMs = 3e4) {
|
|
5377
|
-
if (!this.config.sessionId && !this.config.runId) {
|
|
5378
|
-
throw new Error("RelayClient.connect() requires either sessionId or runId");
|
|
5379
|
-
}
|
|
5380
5455
|
const wsBase = this.config.apiUrl.replace(/^http/, "ws");
|
|
5381
|
-
const
|
|
5382
|
-
const controlUrl = `${wsBase}/relay/control?${query}&token=${this.config.relayToken}`;
|
|
5456
|
+
const controlUrl = `${wsBase}/relay/control?session_id=${this.config.sessionId}&token=${this.config.relayToken}`;
|
|
5383
5457
|
return new Promise((resolve2, reject) => {
|
|
5384
5458
|
const ws = new WebSocket(controlUrl);
|
|
5385
5459
|
ws.binaryType = "arraybuffer";
|
|
5386
|
-
let configReceived = false;
|
|
5387
5460
|
let settled = false;
|
|
5388
5461
|
const settle = (fn) => {
|
|
5389
5462
|
if (!settled) {
|
|
@@ -5403,13 +5476,10 @@ var RelayClient = class {
|
|
|
5403
5476
|
this.setupControlHandlers(ws);
|
|
5404
5477
|
this.emit("connected");
|
|
5405
5478
|
});
|
|
5406
|
-
this.on("
|
|
5407
|
-
|
|
5408
|
-
settle(() =>
|
|
5479
|
+
this.on("ready", () => settle(() => resolve2()));
|
|
5480
|
+
ws.addEventListener("close", (ev) => {
|
|
5481
|
+
settle(() => reject(new Error(`Relay rejected the connection (${ev.code}${ev.reason ? `: ${ev.reason}` : ""})`)));
|
|
5409
5482
|
});
|
|
5410
|
-
setTimeout(() => {
|
|
5411
|
-
if (!configReceived && this.controlWs) settle(() => resolve2());
|
|
5412
|
-
}, 3e3);
|
|
5413
5483
|
ws.addEventListener("error", (ev) => {
|
|
5414
5484
|
if (!this.controlWs) {
|
|
5415
5485
|
settle(() => reject(new Error(`Failed to connect to relay: ${ev.message ?? "connection error"}`)));
|
|
@@ -5417,22 +5487,6 @@ var RelayClient = class {
|
|
|
5417
5487
|
});
|
|
5418
5488
|
});
|
|
5419
5489
|
}
|
|
5420
|
-
async activate(timeoutMs = 15e3) {
|
|
5421
|
-
if (!this.config.runId) {
|
|
5422
|
-
throw new Error("activate() requires runId");
|
|
5423
|
-
}
|
|
5424
|
-
const activateUrl = `${this.config.apiUrl}/internal/runs/${this.config.runId}/activate`;
|
|
5425
|
-
const response = await fetch(activateUrl, {
|
|
5426
|
-
method: "POST",
|
|
5427
|
-
headers: { "Content-Type": "application/json" },
|
|
5428
|
-
body: JSON.stringify({ relay_token: this.config.relayToken }),
|
|
5429
|
-
signal: AbortSignal.timeout(timeoutMs)
|
|
5430
|
-
});
|
|
5431
|
-
if (!response.ok) {
|
|
5432
|
-
const text = await response.text();
|
|
5433
|
-
throw new Error(`Activation failed (${response.status}): ${text}`);
|
|
5434
|
-
}
|
|
5435
|
-
}
|
|
5436
5490
|
async disconnect() {
|
|
5437
5491
|
this.closed = true;
|
|
5438
5492
|
for (const [connId, conn] of this.localConnections) {
|
|
@@ -5482,17 +5536,14 @@ var RelayClient = class {
|
|
|
5482
5536
|
try {
|
|
5483
5537
|
const raw = typeof event.data === "string" ? event.data : new TextDecoder().decode(event.data);
|
|
5484
5538
|
const msg = JSON.parse(raw);
|
|
5485
|
-
if (msg.type === "config"
|
|
5486
|
-
this.
|
|
5487
|
-
this.emit("config_received");
|
|
5539
|
+
if (msg.type === "config") {
|
|
5540
|
+
this.emit("ready");
|
|
5488
5541
|
} else if (msg.type === "new_connection" && msg.conn_id) {
|
|
5489
5542
|
this.handleNewConnection(msg.conn_id);
|
|
5490
5543
|
} else if (msg.type === "close" && msg.conn_id) {
|
|
5491
5544
|
const conn = this.localConnections.get(msg.conn_id);
|
|
5492
5545
|
if (conn?.local.readyState !== WebSocket.CLOSED) conn?.local.close();
|
|
5493
5546
|
this.localConnections.delete(msg.conn_id);
|
|
5494
|
-
} else if (msg.type === "run_complete") {
|
|
5495
|
-
this.emit("run_complete");
|
|
5496
5547
|
}
|
|
5497
5548
|
} catch {
|
|
5498
5549
|
}
|
|
@@ -5560,25 +5611,17 @@ async function startAgentSession(relayConfig) {
|
|
|
5560
5611
|
healthEndpoint: relayConfig.health_endpoint
|
|
5561
5612
|
};
|
|
5562
5613
|
const client = new RelayClient(clientConfig);
|
|
5563
|
-
client.on("log", (msg) => {
|
|
5564
|
-
if (isVerbose()) process.stdout.write(`${msg}
|
|
5565
|
-
`);
|
|
5566
|
-
});
|
|
5567
5614
|
await client.connect();
|
|
5568
5615
|
let agentProcess = null;
|
|
5569
5616
|
if (relayConfig.start_command) {
|
|
5570
|
-
const env = { ...process.env,
|
|
5617
|
+
const env = { ...process.env, PORT: String(relayConfig.agent_port) };
|
|
5571
5618
|
agentProcess = spawn(relayConfig.start_command, {
|
|
5572
5619
|
shell: true,
|
|
5573
5620
|
stdio: "pipe",
|
|
5574
5621
|
env
|
|
5575
5622
|
});
|
|
5576
|
-
agentProcess.stdout?.
|
|
5577
|
-
|
|
5578
|
-
});
|
|
5579
|
-
agentProcess.stderr?.on("data", (data) => {
|
|
5580
|
-
if (isVerbose()) process.stdout.write(`[agent] ${data}`);
|
|
5581
|
-
});
|
|
5623
|
+
agentProcess.stdout?.resume();
|
|
5624
|
+
agentProcess.stderr?.resume();
|
|
5582
5625
|
agentProcess.on("error", (err) => {
|
|
5583
5626
|
process.stdout.write(`Agent process error: ${err.message}
|
|
5584
5627
|
`);
|
|
@@ -5613,7 +5656,7 @@ async function waitForHealth(port, endpoint, timeoutMs = 3e4) {
|
|
|
5613
5656
|
async function agentStartCommand(args) {
|
|
5614
5657
|
const accessToken = await loadAccessToken();
|
|
5615
5658
|
if (!accessToken) {
|
|
5616
|
-
printError("No Vent
|
|
5659
|
+
printError("No Vent credentials found. Set VENT_REFRESH_TOKEN or run `npx vent-hq login`.");
|
|
5617
5660
|
return 2;
|
|
5618
5661
|
}
|
|
5619
5662
|
let parsedConfig;
|
|
@@ -5638,7 +5681,6 @@ async function agentStartCommand(args) {
|
|
|
5638
5681
|
}
|
|
5639
5682
|
if (!connection.agent_port) {
|
|
5640
5683
|
connection.agent_port = await findFreePort();
|
|
5641
|
-
debug(`auto-port assigned for agent session: ${connection.agent_port}`);
|
|
5642
5684
|
}
|
|
5643
5685
|
let session = null;
|
|
5644
5686
|
let relay = null;
|
|
@@ -5648,8 +5690,7 @@ async function agentStartCommand(args) {
|
|
|
5648
5690
|
await apiFetch(`/agent-sessions/${session.session_id}/close`, accessToken, {
|
|
5649
5691
|
method: "POST"
|
|
5650
5692
|
});
|
|
5651
|
-
} catch
|
|
5652
|
-
debug(`agent session close failed: ${err.message}`);
|
|
5693
|
+
} catch {
|
|
5653
5694
|
}
|
|
5654
5695
|
};
|
|
5655
5696
|
try {
|
|
@@ -5686,7 +5727,7 @@ async function agentStartCommand(args) {
|
|
|
5686
5727
|
async function agentStopCommand(args) {
|
|
5687
5728
|
const accessToken = await loadAccessToken();
|
|
5688
5729
|
if (!accessToken) {
|
|
5689
|
-
printError("No Vent
|
|
5730
|
+
printError("No Vent credentials found. Set VENT_REFRESH_TOKEN or run `npx vent-hq login`.");
|
|
5690
5731
|
return 2;
|
|
5691
5732
|
}
|
|
5692
5733
|
try {
|
|
@@ -5795,10 +5836,11 @@ async function deviceAuthFlow() {
|
|
|
5795
5836
|
});
|
|
5796
5837
|
if (!res.ok) continue;
|
|
5797
5838
|
const data = await res.json();
|
|
5798
|
-
|
|
5799
|
-
|
|
5800
|
-
|
|
5801
|
-
|
|
5839
|
+
if (data.status === "approved") {
|
|
5840
|
+
const parsed = AccessCredentialsSchema.safeParse(data);
|
|
5841
|
+
if (!parsed.success) return { ok: false, error: "Vent returned invalid sign-in credentials." };
|
|
5842
|
+
await saveCredentials(parsed.data);
|
|
5843
|
+
return { ok: true, accessToken: parsed.data.access_token };
|
|
5802
5844
|
}
|
|
5803
5845
|
if (data.status === "expired") {
|
|
5804
5846
|
return { ok: false, error: "Session expired. Run `npx vent-hq login` again." };
|
|
@@ -5817,7 +5859,7 @@ async function loginCommand(args) {
|
|
|
5817
5859
|
if (args.status) {
|
|
5818
5860
|
const token = await loadAccessToken();
|
|
5819
5861
|
if (token) {
|
|
5820
|
-
printSuccess(
|
|
5862
|
+
printSuccess("Logged in.", { force: true });
|
|
5821
5863
|
return 0;
|
|
5822
5864
|
}
|
|
5823
5865
|
printInfo("Not logged in. Run `npx vent-hq login`.", { force: true });
|
|
@@ -5825,9 +5867,10 @@ async function loginCommand(args) {
|
|
|
5825
5867
|
}
|
|
5826
5868
|
const result = await deviceAuthFlow();
|
|
5827
5869
|
if (result.ok) {
|
|
5828
|
-
printSuccess("Logged in! Vent
|
|
5870
|
+
printSuccess("Logged in! Vent credentials saved to ~/.vent/credentials", { force: true });
|
|
5829
5871
|
return 0;
|
|
5830
5872
|
}
|
|
5873
|
+
printError(result.error);
|
|
5831
5874
|
return 1;
|
|
5832
5875
|
}
|
|
5833
5876
|
|
|
@@ -5861,16 +5904,16 @@ import * as fs5 from "node:fs/promises";
|
|
|
5861
5904
|
import * as path3 from "node:path";
|
|
5862
5905
|
|
|
5863
5906
|
// src/skills/claude-code.md
|
|
5864
|
-
var claude_code_default = '---\nname: vent\ndescription: Place real calls against the user\'s voice agent to verify recent code changes worked end-to-end. Trigger on test requests or after any non-trivial voice-agent change.\nallowed-tools: Bash(npx vent-hq *)\n---\n\n# Vent - Voice Agent Calls\n\n## When to Test\n\nVent is the voice CI gate. Auto-trigger a call after any non-trivial voice-affecting change \u2014 don\'t wait to be asked. Non-trivial means anything that affects what the caller hears, how the agent responds, what tools are called, how the call routes, or audio timing: prompt rewrites or sections (not one-word tweaks), tool schema or handler changes, state/memory logic, transfer logic, provider/audio config, platform integration, and any voice-behavior bug fix. One-line copy or comment edits don\'t need a call.\n\nBefore launching, scan `.vent/runs/` for unresolved issues from prior runs (failing status, regressions, warnings, latency jumps); fix in-scope ones as part of this change and verify they\'re resolved by the new run, surface out-of-scope ones rather than silently leaving them. The task is not done until you\'ve reviewed the transcript, tool calls, and observed behavior and judged that the change works \u2014 Vent\'s `status` and exit code only reflect pipeline completion, not mission success.\n\n## Autonomous Iteration\n\n**Default: iterate.** Run a fix \u2192 Vent-call \u2192 fix \u2192 Vent-call loop whenever a call surfaces a fixable issue. Don\'t report a failure (or a degraded-quality success) back to the user before attempting to fix it \u2014 they want the agent working, not a diagnosis. After each call, if the result reveals a problem you can fix with a concrete plan, apply the fix and run again. Fixable issues include: tool schema or registry gaps, missing handlers, prompt phrasing, **STT vocabulary mismatches** (e.g. domain terms misheard \u2014 Deepgram `keyterms`, custom vocabulary, prompt-level term lists), **latency tunables** (endpointing thresholds, preemptive generation, model selection), and audio quality config. If you\'d be guessing at the next attempt, stop and ask the user instead of trying.\n\n**"Mission success but with quality problems" still requires iteration.** If the call ran end-to-end but had noticeably bad latency, STT mishearing the agent\'s domain vocabulary, audio artifacts, or any other caller-experience degradation that has a concrete fix, the call is *not done* \u2014 apply the fix and re-run. Don\'t downgrade fixable problems to "observations" or "things worth flagging." The pipeline running cleanly is the floor, not the ceiling.\n\n**Never ask permission to iterate.** "Want me to apply this and re-run?" / "Should I fix it?" / "Let me know if you want me to..." \u2014 these are skill violations. If you have a concrete fix, apply it and run the next call. The user opts out by saying so explicitly; silence is not opt-out.\n\nStop iterating and report when one of:\n\n- The next call confirms the fix worked AND no further fixable issues remain (success \u2014 done).\n- The same fix fails twice (your hypothesis is wrong; ask the user).\n- The failure mode shifts between attempts (you\'re chasing a moving target).\n- You can\'t justify the next call against its provider cost \u2014 each call spends real money and provider quota.\n\nIf the user explicitly opts out ("just report", "don\'t iterate", "stop after the first call", "I\'ll fix it myself"), respect that \u2014 run once, report, stop. Otherwise iterate by default. The first time you start an autonomous loop in a session, mention it once so the user knows it\'s happening (e.g. "the call surfaced X \u2014 fixing and re-running; I\'ll report back when it converges or stops making progress").\n\n## Claude Code Execution\n\nUse a 5-minute shell-tool timeout (`300000` ms) on Vent run commands so normal calls are not killed by the default 2-minute Bash timeout. This is not backgrounding; wait for stdout/results before ending your response. Use the JSON returned by `npx vent-hq run` directly.\n\nClaude Code serializes separate Bash tool calls for `npx vent-hq ...`, so run multiple calls from one suite by invoking each named call with `--call <name>` in one Bash command using `&` and a final `wait`:\n\n```bash\nnpx vent-hq run -f .vent/suite.vapi.json --call happy-path & \\\nnpx vent-hq run -f .vent/suite.vapi.json --call tool-path & \\\nwait\n```\n\nFor long-running composite commands \u2014 e.g. LiveKit\'s required `kill workers \u2192 start fresh worker \u2192 wait 60s \u2192 npx vent-hq run` preflight \u2014 keep the entire sequence in **one foreground Bash call**. Do not use `Monitor` or `run_in_background: true` for the wait. Both end the turn until an event fires, which means Vent never sees an inflight call and the user reads it as "you backgrounded the test." The harness also blocks bare `sleep 60` (any long leading sleep), so fold the readiness check and the wait into a single `until`-loop \u2014 short sleeps inside a polling loop are allowed because each iteration is real work:\n\n```bash\nnohup npm run dev > /tmp/lk-agent.log 2>&1 &\nstart=$(date +%s); \\\nuntil grep -q "registered worker" /tmp/lk-agent.log 2>/dev/null \\\n && [ $(($(date +%s) - start)) -ge 60 ]; do sleep 2; done \\\n&& npx vent-hq run -f .vent/suite.livekit.json --call <name>\n```\n\nFor these composites, raise the Bash timeout to **10 minutes** (`600000` ms) \u2014 startup + 60s wait + call can easily exceed the 5-minute default.\n\n## Workflow\n\n1. Identify the behavior under test. Read enough of the agent codebase to understand its system prompt, tools, handlers, routes, provider config, platform wiring, and expected handoffs.\n2. Reuse an existing `.vent/suite.<adapter>.json` when possible. If `.vent/` contains multiple suites, inspect `connection.adapter` and report which suite file produced the result.\n3. Create or update a suite only when the existing calls do not cover the changed behavior. Name calls after real flows, for example `reschedule-appointment`, not `call-1`.\n4. If the suite uses `start_command`, start one shared local session first with `npx vent-hq agent start -f .vent/suite.<adapter>.json`, then pass `--session <session-id>` to each run.\n\n **For locally-run LiveKit agents: every run requires killing *all* workers, starting one fresh worker, and waiting a full 60 seconds before submitting.** Unconditional \u2014 LiveKit Cloud round-robins across registered workers, so a single survivor with a dead inference subprocess fails ~N-1 of N calls. Don\'t rely on `pkill -f <path-pattern>`; bare command lines like `node --import tsx agent.ts dev` won\'t match a path filter. Use `ps aux | grep -E "node.*agent\\.ts|@livekit/agents.*ipc"`, `kill -9` by PID, re-run `ps` to confirm zero survivors, then start the fresh worker. Skipping the 60s wait fails with `did not publish audio track`; if that error appears alongside `Error [ERR_IPC_CHANNEL_CLOSED] from InferenceProcExecutor.doInference` in the agent log right after a "running EOU detection" line, that\'s a straggler \u2014 redo the kill sweep. Hosted LiveKit Cloud agents don\'t need any of this; run normally. (See `Claude Code Execution` for how to compose the kill\u2192start\u2192wait\u2192submit as a single foreground command.)\n5. Pick which call(s) to run based on the change. Fixed bug: replay the failing scenario. Changed tool: include a call that triggers that tool. Prompt or routing change: include the relevant happy path and any important edge path.\n6. Compare against the previous JSON in `.vent/runs/` when validating a fix or regression. Check status flips, latency jumps, tool-call success drops, cost jumps, and transcript divergence. Correlate with `git diff` between saved `git_sha` values when available; skip if no previous run exists.\n\n## Commands\n\n```bash\nnpx vent-hq init # First-time setup (auth + skill install + starter suite)\nnpx vent-hq login # Log in to existing account\nnpx vent-hq login --status # Print whether credentials are present\nnpx vent-hq logout # Remove saved credentials from ~/.vent/credentials\nnpx vent-hq run -f .vent/suite.X.json # Run a single-call suite\nnpx vent-hq run -f .vent/suite.X.json --call NAME # Run one named call from a multi-call suite\nnpx vent-hq run ... --session <session-id> # Add to any run; routes through an existing local relay session\nnpx vent-hq run ... --verbose # Add to any run; include verbose debug fields\nnpx vent-hq stop <run-id> # Cancel a queued or running run\nnpx vent-hq agent start -f .vent/suite.X.json # Start a shared local relay session\nnpx vent-hq agent stop <session-id> # Stop a shared local relay session\n```\n\nIf `~/.vent/credentials` is missing and `VENT_ACCESS_TOKEN` is not set, run `npx vent-hq init`. For an existing account, run `npx vent-hq login` or set `VENT_ACCESS_TOKEN`.\n\n## Suite Config\n\nSuites live in `.vent/suite.<adapter>.json`. `connection` is declared once per suite. `calls` is a named map, and each key becomes the call name used with `--call`.\n\nLocal websocket suite:\n\n```json\n{\n "connection": {\n "adapter": "websocket",\n "start_command": "npm run start",\n "health_endpoint": "/health",\n "agent_port": 3001\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8,\n "silence_threshold_ms": 1200\n }\n }\n}\n```\n\nPlatform-direct suite:\n\n```json\n{\n "connection": {\n "adapter": "vapi",\n "platform": { "provider": "vapi" }\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nWrite `caller_prompt` as a realistic caller with a name, goal, mood, constraints, and conditional behavior. Set `max_turns` based on flow complexity: FAQ `4-6`, booking or tool use `8-12`, complex flows `12-20`.\n\nCall fields:\n\n- `caller_prompt` and `max_turns` are required.\n- `silence_threshold_ms` must be `200-10000`. Common ranges: FAQ `800-1200`, tool calls `2000-3000`, complex reasoning `3000-5000`.\n- `persona` supports `pace`, `clarity`, `disfluencies`, `cooperation`, `emotion`, `memory`, `intent_clarity`, and `confirmation_style`.\n- `caller_audio` supports noise, speed, speakerphone, mic distance, clarity, accent, packet loss, and jitter.\n- `language` is an ISO 639-1 code such as `en`, `es`, `fr`, `de`, `it`, `nl`, or `ja`.\n- `voice` is `"male"` or `"female"` (English only; default female). Use to flip the caller\'s perceived gender. Ignored if `caller_audio.accent` is set or `language` is non-English.\n\n## Connections and Credentials\n\n### Adapter choice\n\nUse `websocket` for your own local or hosted runtime. Use `start_command` for local agents or `agent_url` for hosted custom endpoints. For `start_command` and `agent_url`, do not put Deepgram, ElevenLabs, OpenAI, or other agent runtime keys into Vent config unless the Vent adapter itself needs them \u2014 the tested agent owns its own runtime credentials.\n\nUse `vapi`, `retell`, `elevenlabs`, `bland`, or `livekit` for platform-direct testing. In this mode Vent itself talks to the provider on the user\'s behalf.\n\nVent provides `DEEPGRAM_API_KEY` and `ANTHROPIC_API_KEY` for its hosted caller/evaluation stack \u2014 those are Vent\'s, not the tested agent\'s.\n\n### Credential resolution\n\nIn platform-direct mode the CLI auto-resolves credentials from `.env.local`, `.env`, and the current shell environment. Do not run `source .env && export` before Vent commands. If you include credential fields in JSON, use the actual value, not the env var name. Do not manually author `platform_connection_id`; the CLI creates or updates the saved platform connection automatically.\n\nAuto-resolved env vars and JSON fields:\n\n- Vapi: `VAPI_API_KEY` -> `vapi_api_key`; `VAPI_ASSISTANT_ID` or `VAPI_AGENT_ID` -> `vapi_assistant_id`\n- Bland: `BLAND_API_KEY` -> `bland_api_key`; `BLAND_PATHWAY_ID` -> `bland_pathway_id`; `BLAND_PERSONA_ID` -> `persona_id`\n- LiveKit: `LIVEKIT_API_KEY` -> `livekit_api_key`; `LIVEKIT_API_SECRET` -> `livekit_api_secret`; `LIVEKIT_URL` -> `livekit_url`\n- Retell: `RETELL_API_KEY` -> `retell_api_key`; `RETELL_AGENT_ID` -> `retell_agent_id`\n- ElevenLabs: `ELEVENLABS_API_KEY` -> `elevenlabs_api_key`; `ELEVENLABS_AGENT_ID` -> `elevenlabs_agent_id`\n\n### Provider config\n\nUse existing provider config when possible: Vapi assistant, Retell agent, ElevenLabs agent, Bland pathway, or LiveKit agent. Bland uniquely supports inline config \u2014 `platform` may use `bland_pathway_id`, `persona_id`, or an inline `task` (with optional voice, model, and turn-handling overrides; see Bland\'s API docs for the full field list).\n\n### Concurrency\n\nWhen you fan out multiple Vent calls in parallel against the same provider (for example, running several named calls from one suite at once with `&` and `wait`), respect the provider\'s per-account concurrency limit. Exceeding it makes calls queue or fail at the provider \u2014 Vent does not enforce these caps for you.\n\nRecord the limit as `max_concurrency` in the suite\'s `platform` block so it\'s visible on future runs. Ask the user which plan they\'re on if sizing matters; otherwise use the conservative default in bold.\n\n- **Vapi**: **10** included per account; reserved lines can be purchased self-serve; Enterprise is unlimited.\n- **Retell**: Pay-as-you-go includes **20**; Enterprise has no cap.\n- **Bland**: Start=**10**, Build=50, Scale=100, Enterprise=unlimited.\n- **ElevenLabs**: Free=**4**, Starter=6, Creator=10, Pro=20, Scale=30, Business=30. Burst pricing can temporarily allow up to 3x base.\n- **LiveKit Cloud**: Build=**5**, Ship=20, Scale=50 managed inference sessions (the usual gate for voice agents); agent-session concurrency can go higher (Scale up to 600).\n\n## WebSocket\n\nFor `adapter: "websocket"`, Vent sends binary 16-bit mono PCM audio over one websocket connection. Websocket text frames are optional JSON events. Audio-only websocket agents still work, but events improve turn detection and observability. Vent sends `{"type":"end-call"}` when the test is done.\n\nUseful websocket text frames:\n\n```jsonc\n{"type":"speech-update","status":"started"}\n{"type":"speech-update","status":"stopped"}\n{"type":"tool_call","name":"check_availability","arguments":{},"result":{},"successful":true,"duration_ms":150}\n{"type":"vent:timing","stt_ms":120,"llm_ms":450,"tts_ms":80}\n{"type":"vent:session","platform":"custom","provider_call_id":"call_123","provider_session_id":"session_abc"}\n{"type":"vent:call-metadata","call_metadata":{"recording_url":"https://...","cost_usd":0.12}}\n{"type":"vent:transcript","role":"caller","text":"I need to reschedule","turn_index":0}\n{"type":"vent:transfer","destination":"+15551234567","status":"attempted"}\n{"type":"vent:debug-url","label":"trace","url":"https://..."}\n{"type":"vent:warning","message":"provider warning","code":"provider_warning"}\n```\n\n`vent:session-report` is **not** handled by the websocket adapter \u2014 it\'s only consumed by the LiveKit helper. Do not emit it from a websocket agent.\n\nPlatform adapters capture tool calls automatically. Websocket agents must emit `tool_call` frames for tool observability. Platform adapters get component latency automatically. Websocket agents should emit `vent:timing` after each agent response when STT/LLM/TTS breakdown is available.\n\n## LiveKit\n\nBefore running LiveKit tests, install and add the Vent helper to the LiveKit agent entrypoint. Node: `npm install @vent-hq/livekit`, then call `instrumentLiveKitAgent({ ctx, session })`. Python: `pip install vent-livekit`, then call `instrument_livekit_agent(ctx=ctx, session=session)`.\n\nLiveKit direct mode requires the LiveKit Agents SDK. Custom LiveKit participants should use the websocket adapter with a relay. If the LiveKit agent registered with an explicit dispatch name, set `livekit_agent_name` in `platform`.\n\nLiveKit parallel calls are supported, but capacity depends on where the user runs their agent. LiveKit Cloud only routes audio \u2014 the agent code runs on the user\'s own infrastructure (laptop, Fly.io, Railway, k8s, etc.), and that box\'s CPU/RAM is the real bottleneck. Before firing parallel calls, derive `platform.max_concurrency` from the minimum of: (1) the user\'s LiveKit plan limit (Build=5, Ship=20, Scale=50+ concurrent agent sessions), (2) their agent worker box capacity (rough rule: 1c/1GB\u21921\u20133 jobs, 2c/4GB\u21925\u201310, 4c/8GB\u219210\u201325, 8c/16GB\u219225\u201350; multiply by number of worker boxes), and (3) their STT/TTS provider concurrency (e.g. Deepgram TTS streaming = 2). If the user wants more parallel calls than that minimum allows, push back \u2014 explain which limit is the bottleneck (plan / box / provider) and either reduce the count or suggest scaling that limit. Architecture: register one `agent_name` and run N workers under it (one big worker, or several smaller ones \u2014 same name); LiveKit\'s dispatcher round-robins jobs across them with built-in failover. Don\'t use numbered names (`agent-0`, `agent-1`) \u2014 that reinvents what the dispatcher does and breaks failover.\n\nUse the LiveKit helper for observability; do not publish `vent:*` topics manually. Do not hand-roll `vent:session-report` from `ctx.addShutdownCallback`; after `room.disconnect()` it can fail with `engine is closed`. The helper captures SDK metrics, tool events, conversation items, usage, and close events. Native LiveKit `lk.transcription` and `lk.agent.state` provide transcript and agent-state timing.\n\n## Output\n\n### Live result\n\n`npx vent-hq run` returns a single JSON result on stdout in non-TTY mode (not an SSE JSONL stream). Exit codes: `0` = call ran through the pipeline; `1` = pipeline-level failure; `2` = harness error.\n\nMost result fields are always present; `latency`, `component_latency`, `call_metadata`, and `emotion` may be `null` when the underlying analysis didn\'t run; `debug` is absent without `--verbose`. Branch on null before reading nested fields. Use `--verbose` only when the default doesn\'t explain a failure \u2014 when you need `platform_transcript` (to check Vent\'s STT), per-turn or component-level latency breakdowns, the raw tool-call timeline, or provider-native artifacts in `debug.provider_metadata`. Otherwise skip \u2014 it just adds noise.\n\nVent\'s transcript is ground truth. Judge on semantic intent: ignore homophones and minor mis-hears (`"check teach hat"` for `"check that"`, missing question marks on short tails) \u2014 those are streaming-STT noise on Vent\'s caller side, not agent bugs, and **don\'t surface them in the report** (they\'re Vent-side artifacts, not actionable for the user). But clear gibberish or word-soup (e.g. `"Cristoxin"` where the agent should have said `"Of course, talk soon"`) is **not** a Vent artifact \u2014 Vent\'s STT does not invent words like that. It means the platform\'s TTS produced corrupted audio or the agent\'s STT/LLM generated the wrong text, and the fix lives there (TTS voice config, agent prompt, model temperature, codec issue). Never dismiss the run as a "Vent harness STT" issue; iterate on the agent or flag the platform.\n\nFor transfers: `call_metadata.transfer_attempted` (provider claimed) and `transfer_completed` (Vent-verified) can disagree \u2014 report both. `transfers[]` carries destination, type, and per-attempt status.\n\n### Saved history\n\nAfter every run, Vent writes the full result JSON to `.vent/runs/`. Shape:\n\n```jsonc\n{\n "run_id": "...",\n "timestamp": "2026-04-21T...Z",\n "git_sha": "...",\n "summary": { "calls_total": 2, "total_duration_ms": 12345, "total_cost_usd": 0.01 },\n "call_results": [\n { "name": "happy-path", "status": "completed", "duration_ms": 6123, "transcript": [], "observed_tool_calls": [], "latency": { "response_time_ms": 420, "p95_response_time_ms": 980 }, "call_metadata": { "cost_usd": 0.004 } }\n ]\n}\n```\n\nWhen comparing against a prior run (Workflow step 6), inspect:\n\n- Run-completion status flips: `call_results[i].status` (pipeline-only \u2014 judge mission success from the transcript)\n- Latency: `call_results[i].latency.response_time_ms` (mean) or `latency.p95_response_time_ms` increased >20%\n- Tool calls: count of `call_results[i].observed_tool_calls[].successful` dropped\n- Cost: `summary.total_cost_usd` or `call_results[i].call_metadata.cost_usd` increased >30%\n- Transcript: `call_results[i].transcript` diverged in semantic content (ignore STT noise)\n\n## Reporting Results\n\nBefore reporting, read the agent\'s code to locate where the observed behavior originates. If the issue is small and you can fix it, fix it and explain what you did \u2014 don\'t ask permission first.\n\nAdapt the report shape to the call \u2014 a clean pass needs little, a regression with a multi-layer cause needs more. Use a transcript excerpt when it helps the user see what happened.\n\nHard rules:\n\n- Pair raw numbers with their plain-English meaning \u2014 don\'t drop the number, but don\'t leave it unexplained. E.g. "p95 latency was 850ms, which is snappy and well within natural conversational pacing" or "p95 hit 1.6 seconds with the LLM as the bottleneck \u2014 noticeably sluggish to a caller."\n- Name the user\'s voice agent by platform on first mention (e.g. "the Vapi agent responded snappily throughout") so the user knows immediately which agent the observation is about. After that, just say "the agent" \u2014 don\'t repeat the platform name on every line.\n- Always include the recording from `call_metadata.recording_url` as an inline `[Recording](url)` link, placed in **one block at the very end of the report** \u2014 never sprinkled through the prose. Single call: one link as the last line. Multi-call: one labeled link per call (e.g. `reschedule-appointment: [Recording](url)`). Never paste a bare URL.\n- Mission success is your judgment, not Vent\'s. The per-call `status` is only `"completed"` (pipeline ran) or `"error"` (pipeline failed); decide whether the agent actually accomplished the scenario from the transcript and tool calls.\n- Similar-sounding word substitutions (e.g. "ocean" for "OSHA") are STT ambiguity, not comprehension failure. The fix lives in STT keyword hints, custom vocabulary, or a prompt-level term list \u2014 not the agent\'s reasoning.\n- Surface only what the user can act on in their own agent\'s code or config \u2014 never `warnings[]` (infrastructure noise), Vent-side artifacts (caller wait modes, harness timing, internal pipeline quirks), or `cost_usd` unless asked.\n\nFor multi-call runs, lead with your own judgment of what happened across the calls (e.g. "3 of 4 did what they were supposed to; `cancel-appointment` never actually canceled"), not a parroted pass/fail count. Then cover each call with whatever depth it needs.\n';
|
|
5907
|
+
var claude_code_default = '---\nname: vent\ndescription: Place real calls against the user\'s voice agent to verify recent code changes worked end-to-end. Trigger on test requests or after any non-trivial voice-agent change.\nallowed-tools: Bash(npx vent-hq *)\n---\n\n# Vent - Voice Agent Calls\n\n## When to Test\n\nVent is the voice CI gate. Auto-trigger a call after any non-trivial voice-affecting change \u2014 don\'t wait to be asked. Non-trivial means anything that affects what the caller hears, how the agent responds, what tools are called, how the call routes, or audio timing: prompt rewrites or sections (not one-word tweaks), tool schema or handler changes, state/memory logic, transfer logic, provider/audio config, platform integration, and any voice-behavior bug fix. One-line copy or comment edits don\'t need a call.\n\nBefore launching, scan `.vent/runs/` for unresolved issues from prior runs (failing status, regressions, warnings, latency jumps); fix in-scope ones as part of this change and verify they\'re resolved by the new run, surface out-of-scope ones rather than silently leaving them. The task is not done until you\'ve reviewed the transcript, tool calls, and observed behavior and judged that the change works \u2014 Vent\'s `status` and exit code only reflect pipeline completion, not mission success.\n\n## Autonomous Iteration\n\n**Default: iterate.** Run a fix \u2192 Vent-call \u2192 fix \u2192 Vent-call loop whenever a call surfaces a fixable issue. Don\'t report a failure (or a degraded-quality success) back to the user before attempting to fix it \u2014 they want the agent working, not a diagnosis. After each call, if the result reveals a problem you can fix with a concrete plan, apply the fix and run again. Fixable issues include: tool schema or registry gaps, missing handlers, prompt phrasing, **STT vocabulary mismatches** (e.g. domain terms misheard \u2014 Deepgram `keyterms`, custom vocabulary, prompt-level term lists), **latency tunables** (endpointing thresholds, preemptive generation, model selection), and audio quality config. If you\'d be guessing at the next attempt, stop and ask the user instead of trying.\n\n**"Mission success but with quality problems" still requires iteration.** If the call ran end-to-end but had noticeably bad latency, STT mishearing the agent\'s domain vocabulary, audio artifacts, or any other caller-experience degradation that has a concrete fix, the call is *not done* \u2014 apply the fix and re-run. Don\'t downgrade fixable problems to "observations" or "things worth flagging." The pipeline running cleanly is the floor, not the ceiling.\n\n**Never ask permission to iterate.** "Want me to apply this and re-run?" / "Should I fix it?" / "Let me know if you want me to..." \u2014 these are skill violations. If you have a concrete fix, apply it and run the next call. The user opts out by saying so explicitly; silence is not opt-out.\n\nStop iterating and report when one of:\n\n- The next call confirms the fix worked AND no further fixable issues remain (success \u2014 done).\n- The same fix fails twice (your hypothesis is wrong; ask the user).\n- The failure mode shifts between attempts (you\'re chasing a moving target).\n- You can\'t justify the next call against its provider cost \u2014 each call spends real money and provider quota.\n\nIf the user explicitly opts out ("just report", "don\'t iterate", "stop after the first call", "I\'ll fix it myself"), respect that \u2014 run once, report, stop. Otherwise iterate by default. The first time you start an autonomous loop in a session, mention it once so the user knows it\'s happening (e.g. "the call surfaced X \u2014 fixing and re-running; I\'ll report back when it converges or stops making progress").\n\n## Claude Code Execution\n\nUse a 5-minute shell-tool timeout (`300000` ms) on Vent run commands so normal calls are not killed by the default 2-minute Bash timeout. This is not backgrounding; wait for stdout/results before ending your response. Use the JSON returned by `npx vent-hq run` directly.\n\nClaude Code serializes separate Bash tool calls for `npx vent-hq ...`, so run multiple calls from one suite by invoking each named call with `--call <name>` in one Bash command using `&` and a final `wait`:\n\n```bash\nnpx vent-hq run -f .vent/suite.vapi.json --call happy-path & \\\nnpx vent-hq run -f .vent/suite.vapi.json --call tool-path & \\\nwait\n```\n\nFor long-running composite commands \u2014 e.g. LiveKit\'s required `kill workers \u2192 start fresh worker \u2192 wait 60s \u2192 npx vent-hq run` preflight \u2014 keep the entire sequence in **one foreground Bash call**. Do not use `Monitor` or `run_in_background: true` for the wait. Both end the turn until an event fires, which means Vent never sees an inflight call and the user reads it as "you backgrounded the test." The harness also blocks bare `sleep 60` (any long leading sleep), so fold the readiness check and the wait into a single `until`-loop \u2014 short sleeps inside a polling loop are allowed because each iteration is real work:\n\n```bash\nnohup npm run dev > /tmp/lk-agent.log 2>&1 &\nstart=$(date +%s); \\\nuntil grep -q "registered worker" /tmp/lk-agent.log 2>/dev/null \\\n && [ $(($(date +%s) - start)) -ge 60 ]; do sleep 2; done \\\n&& npx vent-hq run -f .vent/suite.livekit.json --call <name>\n```\n\nFor these composites, raise the Bash timeout to **10 minutes** (`600000` ms) \u2014 startup + 60s wait + call can easily exceed the 5-minute default.\n\n## Workflow\n\n1. Identify the behavior under test. Read enough of the agent codebase to understand its system prompt, tools, handlers, routes, provider config, platform wiring, and expected handoffs.\n2. Reuse an existing `.vent/suite.<adapter>.json` when possible. If `.vent/` contains multiple suites, inspect `connection.adapter` and report which suite file produced the result.\n3. Create or update a suite only when the existing calls do not cover the changed behavior. Name calls after real flows, for example `reschedule-appointment`, not `call-1`.\n4. If the suite uses `start_command`, start one shared local session first with `npx vent-hq agent start -f .vent/suite.<adapter>.json`, then pass `--session <session-id>` to each run.\n\n **For locally-run LiveKit agents: every run requires killing *all* workers, starting one fresh worker, and waiting a full 60 seconds before submitting.** Unconditional \u2014 LiveKit Cloud round-robins across registered workers, so a single survivor with a dead inference subprocess fails ~N-1 of N calls. Don\'t rely on `pkill -f <path-pattern>`; bare command lines like `node --import tsx agent.ts dev` won\'t match a path filter. Use `ps aux | grep -E "node.*agent\\.ts|@livekit/agents.*ipc"`, `kill -9` by PID, re-run `ps` to confirm zero survivors, then start the fresh worker. Skipping the 60s wait fails with `did not publish audio track`; if that error appears alongside `Error [ERR_IPC_CHANNEL_CLOSED] from InferenceProcExecutor.doInference` in the agent log right after a "running EOU detection" line, that\'s a straggler \u2014 redo the kill sweep. Hosted LiveKit Cloud agents don\'t need any of this; run normally. (See `Claude Code Execution` for how to compose the kill\u2192start\u2192wait\u2192submit as a single foreground command.)\n5. Pick which call(s) to run based on the change. Fixed bug: replay the failing scenario. Changed tool: include a call that triggers that tool. Prompt or routing change: include the relevant happy path and any important edge path.\n6. Compare against the previous JSON in `.vent/runs/` when validating a fix or regression. Check status flips, latency jumps, tool-call success drops, cost jumps, and transcript divergence. Correlate with `git diff` between saved `git_sha` values when available; skip if no previous run exists.\n\n## Commands\n\n```bash\nnpx vent-hq init # First-time setup (auth + skill install + starter suite)\nnpx vent-hq login # Log in to existing account\nnpx vent-hq login --status # Print whether credentials are present\nnpx vent-hq logout # Remove saved credentials from ~/.vent/credentials\nnpx vent-hq run -f .vent/suite.X.json # Run a single-call suite\nnpx vent-hq run -f .vent/suite.X.json --call NAME # Run one named call from a multi-call suite\nnpx vent-hq run ... --session <session-id> # Add to any run; routes through an existing local relay session\nnpx vent-hq run ... --verbose # Add to any run; include verbose debug fields\nnpx vent-hq stop <run-id> # Cancel a queued or running run\nnpx vent-hq agent start -f .vent/suite.X.json # Start a shared local relay session\nnpx vent-hq agent stop <session-id> # Stop a shared local relay session\n```\n\nIf `~/.vent/credentials` is missing and neither `VENT_REFRESH_TOKEN` nor `VENT_ACCESS_TOKEN` is set, run `npx vent-hq init`. For an existing account, run `npx vent-hq login` or set `VENT_REFRESH_TOKEN` to an API key. Access JWTs renew automatically; `VENT_ACCESS_TOKEN` alone is short-lived.\n\n## Suite Config\n\nSuites live in `.vent/suite.<adapter>.json`. `connection` is declared once per suite. `calls` is a named map, and each key becomes the call name used with `--call`.\n\nLocal websocket suite:\n\n```json\n{\n "connection": {\n "adapter": "websocket",\n "start_command": "npm run start",\n "health_endpoint": "/health",\n "agent_port": 3001\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nPlatform-direct suite:\n\n```json\n{\n "connection": {\n "adapter": "vapi",\n "platform": { "provider": "vapi" }\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nWrite `caller_prompt` as a realistic caller with a name, goal, mood, constraints, and conditional behavior. Set `max_turns` based on flow complexity: FAQ `4-6`, booking or tool use `8-12`, complex flows `12-20`.\n\nCall fields:\n\n- `caller_prompt` and `max_turns` are required.\n- `persona` supports `pace`, `clarity`, `disfluencies`, `cooperation`, `emotion`, `memory`, `intent_clarity`, and `confirmation_style`.\n- `caller_audio` supports noise, speed, speakerphone, mic distance, clarity, accent, packet loss, and jitter.\n- `caller_audio.accent` picks the caller\'s voice. Supported: `american` (default), `british`, `australian`, `filipino`, `spanish_mexican`, `spanish_peninsular`, `spanish_colombian`, `spanish_argentine`, `german`, `french`, `italian`, `dutch`, `japanese`. Any other value is rejected. The non-English accents are voices in that language: also set `language` (e.g. `"es"`) so the caller speaks it.\n- `language` is an ISO 639-1 code such as `en`, `es`, `fr`, `de`, `it`, `nl`, or `ja`.\n- `voice` is `"male"` or `"female"` (English only; default female). Use to flip the caller\'s perceived gender. Ignored if `caller_audio.accent` is set or `language` is non-English.\n\n## Connections and Credentials\n\n### Adapter choice\n\nUse `websocket` for your own local or hosted runtime. Use `start_command` for local agents or `agent_url` for hosted custom endpoints. For `start_command` and `agent_url`, do not put Deepgram, ElevenLabs, OpenAI, or other agent runtime keys into Vent config unless the Vent adapter itself needs them \u2014 the tested agent owns its own runtime credentials.\n\nUse `vapi`, `retell`, `elevenlabs`, `bland`, or `livekit` for platform-direct testing. In this mode Vent itself talks to the provider on the user\'s behalf.\n\nVent provides `DEEPGRAM_API_KEY` and `ANTHROPIC_API_KEY` for its hosted caller/evaluation stack \u2014 those are Vent\'s, not the tested agent\'s.\n\n### Credential resolution\n\nIn platform-direct mode the CLI auto-resolves credentials from `.env.local`, `.env`, and the current shell environment. Do not run `source .env && export` before Vent commands. Keep credentials out of chat and suite files; the CLI sends the resolved configuration to Vent, which stores an encrypted copy with each run.\n\nAuto-resolved env vars and JSON fields:\n\n- Vapi: `VAPI_API_KEY` -> `vapi_api_key`; `VAPI_ASSISTANT_ID` or `VAPI_AGENT_ID` -> `vapi_assistant_id`\n- Bland: `BLAND_API_KEY` -> `bland_api_key`; `BLAND_PATHWAY_ID` -> `bland_pathway_id`; `BLAND_PERSONA_ID` -> `persona_id`\n- LiveKit: `LIVEKIT_API_KEY` -> `livekit_api_key`; `LIVEKIT_API_SECRET` -> `livekit_api_secret`; `LIVEKIT_URL` -> `livekit_url`\n- Retell: `RETELL_API_KEY` -> `retell_api_key`; `RETELL_AGENT_ID` -> `retell_agent_id`\n- ElevenLabs: `ELEVENLABS_API_KEY` -> `elevenlabs_api_key`; `ELEVENLABS_AGENT_ID` -> `elevenlabs_agent_id`\n\n### Provider config\n\nUse existing provider config when possible: Vapi assistant, Retell agent, ElevenLabs agent, Bland pathway, or LiveKit agent. Bland uniquely supports inline config \u2014 `platform` may use `bland_pathway_id`, `persona_id`, or an inline `task` (with optional voice, model, and turn-handling overrides; see Bland\'s API docs for the full field list).\n\n### Concurrency\n\nWhen you fan out multiple Vent calls in parallel against the same provider (for example, running several named calls from one suite at once with `&` and `wait`), respect the provider\'s per-account concurrency limit. Exceeding it makes calls queue or fail at the provider \u2014 Vent does not enforce these caps for you.\n\nRecord the limit as `max_concurrency` in the suite\'s `platform` block so it\'s visible on future runs. Ask the user which plan they\'re on if sizing matters; otherwise use the conservative default in bold.\n\n- **Vapi**: **10** included per account; reserved lines can be purchased self-serve; Enterprise is unlimited.\n- **Retell**: Pay-as-you-go includes **20**; Enterprise has no cap.\n- **Bland**: Start=**10**, Build=50, Scale=100, Enterprise=unlimited.\n- **ElevenLabs**: Free=**4**, Starter=6, Creator=10, Pro=20, Scale=30, Business=30. Burst pricing can temporarily allow up to 3x base.\n- **LiveKit Cloud**: Build=**5**, Ship=20, Scale=50 managed inference sessions (the usual gate for voice agents); agent-session concurrency can go higher (Scale up to 600).\n\n## WebSocket\n\nFor `adapter: "websocket"`, Vent sends binary 16-bit mono PCM audio over one websocket connection. Websocket text frames are optional JSON events. Audio-only websocket agents still work, but events improve turn detection and observability. Vent sends `{"type":"end-call"}` when the test is done.\n\nUseful websocket text frames:\n\n```jsonc\n{"type":"speech-update","status":"started"}\n{"type":"speech-update","status":"stopped"}\n{"type":"tool_call","name":"check_availability","arguments":{},"result":{},"successful":true,"duration_ms":150}\n{"type":"vent:timing","stt_ms":120,"llm_ms":450,"tts_ms":80}\n{"type":"vent:session","platform":"custom","provider_call_id":"call_123","provider_session_id":"session_abc"}\n{"type":"vent:call-metadata","call_metadata":{"recording_url":"https://...","cost_usd":0.12}}\n{"type":"vent:transcript","role":"caller","text":"I need to reschedule","turn_index":0}\n{"type":"vent:transfer","destination":"+15551234567","status":"attempted"}\n{"type":"vent:debug-url","label":"trace","url":"https://..."}\n{"type":"vent:warning","message":"provider warning","code":"provider_warning"}\n```\n\n`vent:session-report` is **not** handled by the websocket adapter \u2014 it\'s only consumed by the LiveKit helper. Do not emit it from a websocket agent.\n\nPlatform adapters capture tool calls automatically. Websocket agents must emit `tool_call` frames for tool observability. Platform adapters get component latency automatically. Websocket agents should emit `vent:timing` after each agent response when STT/LLM/TTS breakdown is available.\n\n## LiveKit\n\nBefore running LiveKit tests, install and add the Vent helper to the LiveKit agent entrypoint. Node: `npm install @vent-hq/livekit`, then call `instrumentLiveKitAgent({ ctx, session })`. Python: `pip install vent-livekit`, then call `instrument_livekit_agent(ctx=ctx, session=session)`.\n\nLiveKit direct mode requires the LiveKit Agents SDK. Custom LiveKit participants should use the websocket adapter with a relay. If the LiveKit agent registered with an explicit dispatch name, set `livekit_agent_name` in `platform`.\n\nLiveKit parallel calls are supported, but capacity depends on where the user runs their agent. LiveKit Cloud only routes audio \u2014 the agent code runs on the user\'s own infrastructure (laptop, Fly.io, Railway, k8s, etc.), and that box\'s CPU/RAM is the real bottleneck. Before firing parallel calls, derive `platform.max_concurrency` from the minimum of: (1) the user\'s LiveKit plan limit (Build=5, Ship=20, Scale=50+ concurrent agent sessions), (2) their agent worker box capacity (rough rule: 1c/1GB\u21921\u20133 jobs, 2c/4GB\u21925\u201310, 4c/8GB\u219210\u201325, 8c/16GB\u219225\u201350; multiply by number of worker boxes), and (3) their STT/TTS provider concurrency (e.g. Deepgram TTS streaming = 2). If the user wants more parallel calls than that minimum allows, push back \u2014 explain which limit is the bottleneck (plan / box / provider) and either reduce the count or suggest scaling that limit. Architecture: register one `agent_name` and run N workers under it (one big worker, or several smaller ones \u2014 same name); LiveKit\'s dispatcher round-robins jobs across them with built-in failover. Don\'t use numbered names (`agent-0`, `agent-1`) \u2014 that reinvents what the dispatcher does and breaks failover.\n\nUse the LiveKit helper for observability; do not publish `vent:*` topics manually. Do not hand-roll `vent:session-report` from `ctx.addShutdownCallback`; after `room.disconnect()` it can fail with `engine is closed`. The helper captures SDK metrics, tool events, conversation items, usage, and close events. Native LiveKit `lk.transcription` and `lk.agent.state` provide transcript and agent-state timing.\n\n## Output\n\n### Live result\n\n`npx vent-hq run` returns a single JSON result on stdout in non-TTY mode (not an SSE JSONL stream). Exit codes: `0` = the run passed (every call ran end to end); `1` = it didn\'t (a call errored, or the run failed or was cancelled); `2` = the result stream was lost. A call `error` starting with `Vent failed, agent not tested:` is a failure of Vent itself (its caller LLM, STT/TTS or worker): rerun, it says nothing about the agent. Any other `error` comes from the agent\'s platform or your configuration. `end_note` says how a completed call ended when Vent ended it or the connection closed (e.g. `No agent speech for 30s after the caller\'s turn; Vent hung up.`): read the transcript to judge whether that\'s an agent bug.\n\nMost result fields are always present; `latency`, `component_latency`, `call_metadata`, and `emotion` may be `null` when the underlying analysis didn\'t run; `debug` is absent without `--verbose`. Branch on null before reading nested fields. Use `--verbose` only when the default doesn\'t explain a failure \u2014 when you need `platform_transcript` (to check Vent\'s STT), per-turn or component-level latency breakdowns, the raw tool-call timeline, or provider-native artifacts in `debug.provider_metadata`. Otherwise skip \u2014 it just adds noise.\n\nVent\'s transcript is ground truth. Judge on semantic intent: ignore homophones and minor mis-hears (`"check teach hat"` for `"check that"`, missing question marks on short tails) \u2014 those are streaming-STT noise on Vent\'s caller side, not agent bugs, and **don\'t surface them in the report** (they\'re Vent-side artifacts, not actionable for the user). But clear gibberish or word-soup (e.g. `"Cristoxin"` where the agent should have said `"Of course, talk soon"`) is **not** a Vent artifact \u2014 Vent\'s STT does not invent words like that. It means the platform\'s TTS produced corrupted audio or the agent\'s STT/LLM generated the wrong text, and the fix lives there (TTS voice config, agent prompt, model temperature, codec issue). Missing or cut-off agent text, unlike word-soup, can come from Vent\'s STT stream dropping: check the recording before blaming the agent.\n\nFor transfers: `call_metadata.transfer_attempted` (provider claimed) and `transfer_completed` (from the platform\'s call data) can disagree \u2014 report both. `transfers[]` carries destination, type, and per-attempt status.\n\n### Saved history\n\nAfter every run, Vent writes the full result JSON to `.vent/runs/`. Shape:\n\n```jsonc\n{\n "run_id": "...",\n "timestamp": "2026-04-21T...Z",\n "git_sha": "...",\n "summary": { "calls_total": 2, "total_duration_ms": 12345, "total_cost_usd": 0.01 },\n "call_results": [\n { "name": "happy-path", "status": "completed", "duration_ms": 6123, "transcript": [], "observed_tool_calls": [], "latency": { "response_time_ms": 420, "p95_response_time_ms": 980 }, "call_metadata": { "cost_usd": 0.004 } }\n ]\n}\n```\n\nWhen comparing against a prior run (Workflow step 6), inspect:\n\n- Run-completion status flips: `call_results[i].status` (pipeline-only \u2014 judge mission success from the transcript)\n- Latency: `call_results[i].latency.response_time_ms` (mean) or `latency.p95_response_time_ms` increased >20%\n- Tool calls: count of `call_results[i].observed_tool_calls[].successful` dropped\n- Cost: `summary.total_cost_usd` or `call_results[i].call_metadata.cost_usd` increased >30%\n- Transcript: `call_results[i].transcript` diverged in semantic content (ignore STT noise)\n\n## Reporting Results\n\nBefore reporting, read the agent\'s code to locate where the observed behavior originates. If the issue is small and you can fix it, fix it and explain what you did \u2014 don\'t ask permission first.\n\nAdapt the report shape to the call \u2014 a clean pass needs little, a regression with a multi-layer cause needs more. Use a transcript excerpt when it helps the user see what happened.\n\nHard rules:\n\n- Pair raw numbers with their plain-English meaning \u2014 don\'t drop the number, but don\'t leave it unexplained. E.g. "p95 latency was 850ms, which is snappy and well within natural conversational pacing" or "p95 hit 1.6 seconds with the LLM as the bottleneck \u2014 noticeably sluggish to a caller."\n- Name the user\'s voice agent by platform on first mention (e.g. "the Vapi agent responded snappily throughout") so the user knows immediately which agent the observation is about. After that, just say "the agent" \u2014 don\'t repeat the platform name on every line.\n- Always include the recording from `call_metadata.recording_url` as an inline `[Recording](url)` link, placed in **one block at the very end of the report** \u2014 never sprinkled through the prose. Single call: one link as the last line. Multi-call: one labeled link per call (e.g. `reschedule-appointment: [Recording](url)`). Never paste a bare URL.\n- Mission success is your judgment, not Vent\'s. The per-call `status` is only `"completed"` (pipeline ran) or `"error"` (pipeline failed); decide whether the agent actually accomplished the scenario from the transcript and tool calls.\n- Similar-sounding word substitutions (e.g. "ocean" for "OSHA") are STT ambiguity, not comprehension failure. The fix lives in STT keyword hints, custom vocabulary, or a prompt-level term list \u2014 not the agent\'s reasoning.\n- Surface only what the user can act on in their own agent\'s code or config \u2014 never Vent-side artifacts (caller wait modes, harness timing, internal pipeline quirks), or `cost_usd` unless asked.\n\nFor multi-call runs, lead with your own judgment of what happened across the calls (e.g. "3 of 4 did what they were supposed to; `cancel-appointment` never actually canceled"), not a parroted pass/fail count. Then cover each call with whatever depth it needs.\n';
|
|
5865
5908
|
|
|
5866
5909
|
// src/skills/cursor.md
|
|
5867
|
-
var cursor_default = '---\ndescription: Place real calls against the user\'s voice agent to verify recent code changes worked end-to-end. Trigger on test requests or after any non-trivial voice-agent change.\nalwaysApply: true\n---\n\n# Vent - Voice Agent Calls\n\n## When to Test\n\nVent is the voice CI gate. Auto-trigger a call after any non-trivial voice-affecting change \u2014 don\'t wait to be asked. Non-trivial means anything that affects what the caller hears, how the agent responds, what tools are called, how the call routes, or audio timing: prompt rewrites or sections (not one-word tweaks), tool schema or handler changes, state/memory logic, transfer logic, provider/audio config, platform integration, and any voice-behavior bug fix. One-line copy or comment edits don\'t need a call.\n\nBefore launching, scan `.vent/runs/` for unresolved issues from prior runs (failing status, regressions, warnings, latency jumps); fix in-scope ones as part of this change and verify they\'re resolved by the new run, surface out-of-scope ones rather than silently leaving them. The task is not done until you\'ve reviewed the transcript, tool calls, and observed behavior and judged that the change works \u2014 Vent\'s `status` and exit code only reflect pipeline completion, not mission success.\n\n## Autonomous Iteration\n\n**Default: iterate.** Run a fix \u2192 Vent-call \u2192 fix \u2192 Vent-call loop whenever a call surfaces a fixable issue. Don\'t report a failure (or a degraded-quality success) back to the user before attempting to fix it \u2014 they want the agent working, not a diagnosis. After each call, if the result reveals a problem you can fix with a concrete plan, apply the fix and run again. Fixable issues include: tool schema or registry gaps, missing handlers, prompt phrasing, **STT vocabulary mismatches** (e.g. domain terms misheard \u2014 Deepgram `keyterms`, custom vocabulary, prompt-level term lists), **latency tunables** (endpointing thresholds, preemptive generation, model selection), and audio quality config. If you\'d be guessing at the next attempt, stop and ask the user instead of trying.\n\n**"Mission success but with quality problems" still requires iteration.** If the call ran end-to-end but had noticeably bad latency, STT mishearing the agent\'s domain vocabulary, audio artifacts, or any other caller-experience degradation that has a concrete fix, the call is *not done* \u2014 apply the fix and re-run. Don\'t downgrade fixable problems to "observations" or "things worth flagging." The pipeline running cleanly is the floor, not the ceiling.\n\n**Never ask permission to iterate.** "Want me to apply this and re-run?" / "Should I fix it?" / "Let me know if you want me to..." \u2014 these are skill violations. If you have a concrete fix, apply it and run the next call. The user opts out by saying so explicitly; silence is not opt-out.\n\nStop iterating and report when one of:\n\n- The next call confirms the fix worked AND no further fixable issues remain (success \u2014 done).\n- The same fix fails twice (your hypothesis is wrong; ask the user).\n- The failure mode shifts between attempts (you\'re chasing a moving target).\n- You can\'t justify the next call against its provider cost \u2014 each call spends real money and provider quota.\n\nIf the user explicitly opts out ("just report", "don\'t iterate", "stop after the first call", "I\'ll fix it myself"), respect that \u2014 run once, report, stop. Otherwise iterate by default. The first time you start an autonomous loop in a session, mention it once so the user knows it\'s happening (e.g. "the call surfaced X \u2014 fixing and re-running; I\'ll report back when it converges or stops making progress").\n\n## Cursor Execution\n\nUse a 5-minute shell-tool timeout (`300000` ms) on Vent run commands so normal calls are not killed by the default 2-minute Bash timeout. This is not backgrounding; wait for stdout/results before ending your response. Use the JSON returned by `npx vent-hq run` directly.\n\nCursor cannot run separate shell tool calls concurrently \u2014 for multiple calls from one suite, invoke each named call with `--call <name>` in one shell command using `&` and a final `wait`:\n\n```bash\nnpx vent-hq run -f .vent/suite.vapi.json --call happy-path & \\\nnpx vent-hq run -f .vent/suite.vapi.json --call tool-path & \\\nwait\n```\n\n## Workflow\n\n1. Identify the behavior under test. Read enough of the agent codebase to understand its system prompt, tools, handlers, routes, provider config, platform wiring, and expected handoffs.\n2. Reuse an existing `.vent/suite.<adapter>.json` when possible. If `.vent/` contains multiple suites, inspect `connection.adapter` and report which suite file produced the result.\n3. Create or update a suite only when the existing calls do not cover the changed behavior. Name calls after real flows, for example `reschedule-appointment`, not `call-1`.\n4. If the suite uses `start_command`, start one shared local session first with `npx vent-hq agent start -f .vent/suite.<adapter>.json`, then pass `--session <session-id>` to each run.\n\n **For locally-run LiveKit agents: every run requires killing *all* workers, starting one fresh worker, and waiting a full 60 seconds before submitting.** Unconditional \u2014 LiveKit Cloud round-robins across registered workers, so a single survivor with a dead inference subprocess fails ~N-1 of N calls. Don\'t rely on `pkill -f <path-pattern>`; bare command lines like `node --import tsx agent.ts dev` won\'t match a path filter. Use `ps aux | grep -E "node.*agent\\.ts|@livekit/agents.*ipc"`, `kill -9` by PID, re-run `ps` to confirm zero survivors, then start the fresh worker. Skipping the 60s wait fails with `did not publish audio track`; if that error appears alongside `Error [ERR_IPC_CHANNEL_CLOSED] from InferenceProcExecutor.doInference` in the agent log right after a "running EOU detection" line, that\'s a straggler \u2014 redo the kill sweep. Hosted LiveKit Cloud agents don\'t need any of this; run normally.\n5. Pick which call(s) to run based on the change. Fixed bug: replay the failing scenario. Changed tool: include a call that triggers that tool. Prompt or routing change: include the relevant happy path and any important edge path.\n6. Compare against the previous JSON in `.vent/runs/` when validating a fix or regression. Check status flips, latency jumps, tool-call success drops, cost jumps, and transcript divergence. Correlate with `git diff` between saved `git_sha` values when available; skip if no previous run exists.\n\n## Commands\n\n```bash\nnpx vent-hq init # First-time setup (auth + skill install + starter suite)\nnpx vent-hq login # Log in to existing account\nnpx vent-hq login --status # Print whether credentials are present\nnpx vent-hq logout # Remove saved credentials from ~/.vent/credentials\nnpx vent-hq run -f .vent/suite.X.json # Run a single-call suite\nnpx vent-hq run -f .vent/suite.X.json --call NAME # Run one named call from a multi-call suite\nnpx vent-hq run ... --session <session-id> # Add to any run; routes through an existing local relay session\nnpx vent-hq run ... --verbose # Add to any run; include verbose debug fields\nnpx vent-hq stop <run-id> # Cancel a queued or running run\nnpx vent-hq agent start -f .vent/suite.X.json # Start a shared local relay session\nnpx vent-hq agent stop <session-id> # Stop a shared local relay session\n```\n\nIf `~/.vent/credentials` is missing and `VENT_ACCESS_TOKEN` is not set, run `npx vent-hq init`. For an existing account, run `npx vent-hq login` or set `VENT_ACCESS_TOKEN`.\n\n## Suite Config\n\nSuites live in `.vent/suite.<adapter>.json`. `connection` is declared once per suite. `calls` is a named map, and each key becomes the call name used with `--call`.\n\nLocal websocket suite:\n\n```json\n{\n "connection": {\n "adapter": "websocket",\n "start_command": "npm run start",\n "health_endpoint": "/health",\n "agent_port": 3001\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8,\n "silence_threshold_ms": 1200\n }\n }\n}\n```\n\nPlatform-direct suite:\n\n```json\n{\n "connection": {\n "adapter": "vapi",\n "platform": { "provider": "vapi" }\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nWrite `caller_prompt` as a realistic caller with a name, goal, mood, constraints, and conditional behavior. Set `max_turns` based on flow complexity: FAQ `4-6`, booking or tool use `8-12`, complex flows `12-20`.\n\nCall fields:\n\n- `caller_prompt` and `max_turns` are required.\n- `silence_threshold_ms` must be `200-10000`. Common ranges: FAQ `800-1200`, tool calls `2000-3000`, complex reasoning `3000-5000`.\n- `persona` supports `pace`, `clarity`, `disfluencies`, `cooperation`, `emotion`, `memory`, `intent_clarity`, and `confirmation_style`.\n- `caller_audio` supports noise, speed, speakerphone, mic distance, clarity, accent, packet loss, and jitter.\n- `language` is an ISO 639-1 code such as `en`, `es`, `fr`, `de`, `it`, `nl`, or `ja`.\n- `voice` is `"male"` or `"female"` (English only; default female). Use to flip the caller\'s perceived gender. Ignored if `caller_audio.accent` is set or `language` is non-English.\n\n## Connections and Credentials\n\n### Adapter choice\n\nUse `websocket` for your own local or hosted runtime. Use `start_command` for local agents or `agent_url` for hosted custom endpoints. For `start_command` and `agent_url`, do not put Deepgram, ElevenLabs, OpenAI, or other agent runtime keys into Vent config unless the Vent adapter itself needs them \u2014 the tested agent owns its own runtime credentials.\n\nUse `vapi`, `retell`, `elevenlabs`, `bland`, or `livekit` for platform-direct testing. In this mode Vent itself talks to the provider on the user\'s behalf.\n\nVent provides `DEEPGRAM_API_KEY` and `ANTHROPIC_API_KEY` for its hosted caller/evaluation stack \u2014 those are Vent\'s, not the tested agent\'s.\n\n### Credential resolution\n\nIn platform-direct mode the CLI auto-resolves credentials from `.env.local`, `.env`, and the current shell environment. Do not run `source .env && export` before Vent commands. If you include credential fields in JSON, use the actual value, not the env var name. Do not manually author `platform_connection_id`; the CLI creates or updates the saved platform connection automatically.\n\nAuto-resolved env vars and JSON fields:\n\n- Vapi: `VAPI_API_KEY` -> `vapi_api_key`; `VAPI_ASSISTANT_ID` or `VAPI_AGENT_ID` -> `vapi_assistant_id`\n- Bland: `BLAND_API_KEY` -> `bland_api_key`; `BLAND_PATHWAY_ID` -> `bland_pathway_id`; `BLAND_PERSONA_ID` -> `persona_id`\n- LiveKit: `LIVEKIT_API_KEY` -> `livekit_api_key`; `LIVEKIT_API_SECRET` -> `livekit_api_secret`; `LIVEKIT_URL` -> `livekit_url`\n- Retell: `RETELL_API_KEY` -> `retell_api_key`; `RETELL_AGENT_ID` -> `retell_agent_id`\n- ElevenLabs: `ELEVENLABS_API_KEY` -> `elevenlabs_api_key`; `ELEVENLABS_AGENT_ID` -> `elevenlabs_agent_id`\n\n### Provider config\n\nUse existing provider config when possible: Vapi assistant, Retell agent, ElevenLabs agent, Bland pathway, or LiveKit agent. Bland uniquely supports inline config \u2014 `platform` may use `bland_pathway_id`, `persona_id`, or an inline `task` (with optional voice, model, and turn-handling overrides; see Bland\'s API docs for the full field list).\n\n### Concurrency\n\nWhen you fan out multiple Vent calls in parallel against the same provider (for example, running several named calls from one suite at once with `&` and `wait`), respect the provider\'s per-account concurrency limit. Exceeding it makes calls queue or fail at the provider \u2014 Vent does not enforce these caps for you.\n\nRecord the limit as `max_concurrency` in the suite\'s `platform` block so it\'s visible on future runs. Ask the user which plan they\'re on if sizing matters; otherwise use the conservative default in bold.\n\n- **Vapi**: **10** included per account; reserved lines can be purchased self-serve; Enterprise is unlimited.\n- **Retell**: Pay-as-you-go includes **20**; Enterprise has no cap.\n- **Bland**: Start=**10**, Build=50, Scale=100, Enterprise=unlimited.\n- **ElevenLabs**: Free=**4**, Starter=6, Creator=10, Pro=20, Scale=30, Business=30. Burst pricing can temporarily allow up to 3x base.\n- **LiveKit Cloud**: Build=**5**, Ship=20, Scale=50 managed inference sessions (the usual gate for voice agents); agent-session concurrency can go higher (Scale up to 600).\n\n## WebSocket\n\nFor `adapter: "websocket"`, Vent sends binary 16-bit mono PCM audio over one websocket connection. Websocket text frames are optional JSON events. Audio-only websocket agents still work, but events improve turn detection and observability. Vent sends `{"type":"end-call"}` when the test is done.\n\nUseful websocket text frames:\n\n```jsonc\n{"type":"speech-update","status":"started"}\n{"type":"speech-update","status":"stopped"}\n{"type":"tool_call","name":"check_availability","arguments":{},"result":{},"successful":true,"duration_ms":150}\n{"type":"vent:timing","stt_ms":120,"llm_ms":450,"tts_ms":80}\n{"type":"vent:session","platform":"custom","provider_call_id":"call_123","provider_session_id":"session_abc"}\n{"type":"vent:call-metadata","call_metadata":{"recording_url":"https://...","cost_usd":0.12}}\n{"type":"vent:transcript","role":"caller","text":"I need to reschedule","turn_index":0}\n{"type":"vent:transfer","destination":"+15551234567","status":"attempted"}\n{"type":"vent:debug-url","label":"trace","url":"https://..."}\n{"type":"vent:warning","message":"provider warning","code":"provider_warning"}\n```\n\n`vent:session-report` is **not** handled by the websocket adapter \u2014 it\'s only consumed by the LiveKit helper. Do not emit it from a websocket agent.\n\nPlatform adapters capture tool calls automatically. Websocket agents must emit `tool_call` frames for tool observability. Platform adapters get component latency automatically. Websocket agents should emit `vent:timing` after each agent response when STT/LLM/TTS breakdown is available.\n\n## LiveKit\n\nBefore running LiveKit tests, install and add the Vent helper to the LiveKit agent entrypoint. Node: `npm install @vent-hq/livekit`, then call `instrumentLiveKitAgent({ ctx, session })`. Python: `pip install vent-livekit`, then call `instrument_livekit_agent(ctx=ctx, session=session)`.\n\nLiveKit direct mode requires the LiveKit Agents SDK. Custom LiveKit participants should use the websocket adapter with a relay. If the LiveKit agent registered with an explicit dispatch name, set `livekit_agent_name` in `platform`.\n\nLiveKit parallel calls are supported, but capacity depends on where the user runs their agent. LiveKit Cloud only routes audio \u2014 the agent code runs on the user\'s own infrastructure (laptop, Fly.io, Railway, k8s, etc.), and that box\'s CPU/RAM is the real bottleneck. Before firing parallel calls, derive `platform.max_concurrency` from the minimum of: (1) the user\'s LiveKit plan limit (Build=5, Ship=20, Scale=50+ concurrent agent sessions), (2) their agent worker box capacity (rough rule: 1c/1GB\u21921\u20133 jobs, 2c/4GB\u21925\u201310, 4c/8GB\u219210\u201325, 8c/16GB\u219225\u201350; multiply by number of worker boxes), and (3) their STT/TTS provider concurrency (e.g. Deepgram TTS streaming = 2). If the user wants more parallel calls than that minimum allows, push back \u2014 explain which limit is the bottleneck (plan / box / provider) and either reduce the count or suggest scaling that limit. Architecture: register one `agent_name` and run N workers under it (one big worker, or several smaller ones \u2014 same name); LiveKit\'s dispatcher round-robins jobs across them with built-in failover. Don\'t use numbered names (`agent-0`, `agent-1`) \u2014 that reinvents what the dispatcher does and breaks failover.\n\nUse the LiveKit helper for observability; do not publish `vent:*` topics manually. Do not hand-roll `vent:session-report` from `ctx.addShutdownCallback`; after `room.disconnect()` it can fail with `engine is closed`. The helper captures SDK metrics, tool events, conversation items, usage, and close events. Native LiveKit `lk.transcription` and `lk.agent.state` provide transcript and agent-state timing.\n\n## Output\n\n### Live result\n\n`npx vent-hq run` returns a single JSON result on stdout in non-TTY mode (not an SSE JSONL stream). Exit codes: `0` = call ran through the pipeline; `1` = pipeline-level failure; `2` = harness error.\n\nMost result fields are always present; `latency`, `component_latency`, `call_metadata`, and `emotion` may be `null` when the underlying analysis didn\'t run; `debug` is absent without `--verbose`. Branch on null before reading nested fields. Use `--verbose` only when the default doesn\'t explain a failure \u2014 when you need `platform_transcript` (to check Vent\'s STT), per-turn or component-level latency breakdowns, the raw tool-call timeline, or provider-native artifacts in `debug.provider_metadata`. Otherwise skip \u2014 it just adds noise.\n\nVent\'s transcript is ground truth. Judge on semantic intent: ignore homophones and minor mis-hears (`"check teach hat"` for `"check that"`, missing question marks on short tails) \u2014 those are streaming-STT noise on Vent\'s caller side, not agent bugs, and **don\'t surface them in the report** (they\'re Vent-side artifacts, not actionable for the user). But clear gibberish or word-soup (e.g. `"Cristoxin"` where the agent should have said `"Of course, talk soon"`) is **not** a Vent artifact \u2014 Vent\'s STT does not invent words like that. It means the platform\'s TTS produced corrupted audio or the agent\'s STT/LLM generated the wrong text, and the fix lives there (TTS voice config, agent prompt, model temperature, codec issue). Never dismiss the run as a "Vent harness STT" issue; iterate on the agent or flag the platform.\n\nFor transfers: `call_metadata.transfer_attempted` (provider claimed) and `transfer_completed` (Vent-verified) can disagree \u2014 report both. `transfers[]` carries destination, type, and per-attempt status.\n\n### Saved history\n\nAfter every run, Vent writes the full result JSON to `.vent/runs/`. Shape:\n\n```jsonc\n{\n "run_id": "...",\n "timestamp": "2026-04-21T...Z",\n "git_sha": "...",\n "summary": { "calls_total": 2, "total_duration_ms": 12345, "total_cost_usd": 0.01 },\n "call_results": [\n { "name": "happy-path", "status": "completed", "duration_ms": 6123, "transcript": [], "observed_tool_calls": [], "latency": { "response_time_ms": 420, "p95_response_time_ms": 980 }, "call_metadata": { "cost_usd": 0.004 } }\n ]\n}\n```\n\nWhen comparing against a prior run (Workflow step 6), inspect:\n\n- Run-completion status flips: `call_results[i].status` (pipeline-only \u2014 judge mission success from the transcript)\n- Latency: `call_results[i].latency.response_time_ms` (mean) or `latency.p95_response_time_ms` increased >20%\n- Tool calls: count of `call_results[i].observed_tool_calls[].successful` dropped\n- Cost: `summary.total_cost_usd` or `call_results[i].call_metadata.cost_usd` increased >30%\n- Transcript: `call_results[i].transcript` diverged in semantic content (ignore STT noise)\n\n## Reporting Results\n\nBefore reporting, read the agent\'s code to locate where the observed behavior originates. If the issue is small and you can fix it, fix it and explain what you did \u2014 don\'t ask permission first.\n\nAdapt the report shape to the call \u2014 a clean pass needs little, a regression with a multi-layer cause needs more. Use a transcript excerpt when it helps the user see what happened.\n\nHard rules:\n\n- Pair raw numbers with their plain-English meaning \u2014 don\'t drop the number, but don\'t leave it unexplained. E.g. "p95 latency was 850ms, which is snappy and well within natural conversational pacing" or "p95 hit 1.6 seconds with the LLM as the bottleneck \u2014 noticeably sluggish to a caller."\n- Name the user\'s voice agent by platform on first mention (e.g. "the Vapi agent responded snappily throughout") so the user knows immediately which agent the observation is about. After that, just say "the agent" \u2014 don\'t repeat the platform name on every line.\n- Always include the recording from `call_metadata.recording_url` as an inline `[Recording](url)` link, placed in **one block at the very end of the report** \u2014 never sprinkled through the prose. Single call: one link as the last line. Multi-call: one labeled link per call (e.g. `reschedule-appointment: [Recording](url)`). Never paste a bare URL.\n- Mission success is your judgment, not Vent\'s. The per-call `status` is only `"completed"` (pipeline ran) or `"error"` (pipeline failed); decide whether the agent actually accomplished the scenario from the transcript and tool calls.\n- Similar-sounding word substitutions (e.g. "ocean" for "OSHA") are STT ambiguity, not comprehension failure. The fix lives in STT keyword hints, custom vocabulary, or a prompt-level term list \u2014 not the agent\'s reasoning.\n- Surface only what the user can act on in their own agent\'s code or config \u2014 never `warnings[]` (infrastructure noise), Vent-side artifacts (caller wait modes, harness timing, internal pipeline quirks), or `cost_usd` unless asked.\n\nFor multi-call runs, lead with your own judgment of what happened across the calls (e.g. "3 of 4 did what they were supposed to; `cancel-appointment` never actually canceled"), not a parroted pass/fail count. Then cover each call with whatever depth it needs.\n';
|
|
5910
|
+
var cursor_default = '---\ndescription: Place real calls against the user\'s voice agent to verify recent code changes worked end-to-end. Trigger on test requests or after any non-trivial voice-agent change.\nalwaysApply: true\n---\n\n# Vent - Voice Agent Calls\n\n## When to Test\n\nVent is the voice CI gate. Auto-trigger a call after any non-trivial voice-affecting change \u2014 don\'t wait to be asked. Non-trivial means anything that affects what the caller hears, how the agent responds, what tools are called, how the call routes, or audio timing: prompt rewrites or sections (not one-word tweaks), tool schema or handler changes, state/memory logic, transfer logic, provider/audio config, platform integration, and any voice-behavior bug fix. One-line copy or comment edits don\'t need a call.\n\nBefore launching, scan `.vent/runs/` for unresolved issues from prior runs (failing status, regressions, warnings, latency jumps); fix in-scope ones as part of this change and verify they\'re resolved by the new run, surface out-of-scope ones rather than silently leaving them. The task is not done until you\'ve reviewed the transcript, tool calls, and observed behavior and judged that the change works \u2014 Vent\'s `status` and exit code only reflect pipeline completion, not mission success.\n\n## Autonomous Iteration\n\n**Default: iterate.** Run a fix \u2192 Vent-call \u2192 fix \u2192 Vent-call loop whenever a call surfaces a fixable issue. Don\'t report a failure (or a degraded-quality success) back to the user before attempting to fix it \u2014 they want the agent working, not a diagnosis. After each call, if the result reveals a problem you can fix with a concrete plan, apply the fix and run again. Fixable issues include: tool schema or registry gaps, missing handlers, prompt phrasing, **STT vocabulary mismatches** (e.g. domain terms misheard \u2014 Deepgram `keyterms`, custom vocabulary, prompt-level term lists), **latency tunables** (endpointing thresholds, preemptive generation, model selection), and audio quality config. If you\'d be guessing at the next attempt, stop and ask the user instead of trying.\n\n**"Mission success but with quality problems" still requires iteration.** If the call ran end-to-end but had noticeably bad latency, STT mishearing the agent\'s domain vocabulary, audio artifacts, or any other caller-experience degradation that has a concrete fix, the call is *not done* \u2014 apply the fix and re-run. Don\'t downgrade fixable problems to "observations" or "things worth flagging." The pipeline running cleanly is the floor, not the ceiling.\n\n**Never ask permission to iterate.** "Want me to apply this and re-run?" / "Should I fix it?" / "Let me know if you want me to..." \u2014 these are skill violations. If you have a concrete fix, apply it and run the next call. The user opts out by saying so explicitly; silence is not opt-out.\n\nStop iterating and report when one of:\n\n- The next call confirms the fix worked AND no further fixable issues remain (success \u2014 done).\n- The same fix fails twice (your hypothesis is wrong; ask the user).\n- The failure mode shifts between attempts (you\'re chasing a moving target).\n- You can\'t justify the next call against its provider cost \u2014 each call spends real money and provider quota.\n\nIf the user explicitly opts out ("just report", "don\'t iterate", "stop after the first call", "I\'ll fix it myself"), respect that \u2014 run once, report, stop. Otherwise iterate by default. The first time you start an autonomous loop in a session, mention it once so the user knows it\'s happening (e.g. "the call surfaced X \u2014 fixing and re-running; I\'ll report back when it converges or stops making progress").\n\n## Cursor Execution\n\nUse a 5-minute shell-tool timeout (`300000` ms) on Vent run commands so normal calls are not killed by the default 2-minute Bash timeout. This is not backgrounding; wait for stdout/results before ending your response. Use the JSON returned by `npx vent-hq run` directly.\n\nCursor cannot run separate shell tool calls concurrently \u2014 for multiple calls from one suite, invoke each named call with `--call <name>` in one shell command using `&` and a final `wait`:\n\n```bash\nnpx vent-hq run -f .vent/suite.vapi.json --call happy-path & \\\nnpx vent-hq run -f .vent/suite.vapi.json --call tool-path & \\\nwait\n```\n\n## Workflow\n\n1. Identify the behavior under test. Read enough of the agent codebase to understand its system prompt, tools, handlers, routes, provider config, platform wiring, and expected handoffs.\n2. Reuse an existing `.vent/suite.<adapter>.json` when possible. If `.vent/` contains multiple suites, inspect `connection.adapter` and report which suite file produced the result.\n3. Create or update a suite only when the existing calls do not cover the changed behavior. Name calls after real flows, for example `reschedule-appointment`, not `call-1`.\n4. If the suite uses `start_command`, start one shared local session first with `npx vent-hq agent start -f .vent/suite.<adapter>.json`, then pass `--session <session-id>` to each run.\n\n **For locally-run LiveKit agents: every run requires killing *all* workers, starting one fresh worker, and waiting a full 60 seconds before submitting.** Unconditional \u2014 LiveKit Cloud round-robins across registered workers, so a single survivor with a dead inference subprocess fails ~N-1 of N calls. Don\'t rely on `pkill -f <path-pattern>`; bare command lines like `node --import tsx agent.ts dev` won\'t match a path filter. Use `ps aux | grep -E "node.*agent\\.ts|@livekit/agents.*ipc"`, `kill -9` by PID, re-run `ps` to confirm zero survivors, then start the fresh worker. Skipping the 60s wait fails with `did not publish audio track`; if that error appears alongside `Error [ERR_IPC_CHANNEL_CLOSED] from InferenceProcExecutor.doInference` in the agent log right after a "running EOU detection" line, that\'s a straggler \u2014 redo the kill sweep. Hosted LiveKit Cloud agents don\'t need any of this; run normally.\n5. Pick which call(s) to run based on the change. Fixed bug: replay the failing scenario. Changed tool: include a call that triggers that tool. Prompt or routing change: include the relevant happy path and any important edge path.\n6. Compare against the previous JSON in `.vent/runs/` when validating a fix or regression. Check status flips, latency jumps, tool-call success drops, cost jumps, and transcript divergence. Correlate with `git diff` between saved `git_sha` values when available; skip if no previous run exists.\n\n## Commands\n\n```bash\nnpx vent-hq init # First-time setup (auth + skill install + starter suite)\nnpx vent-hq login # Log in to existing account\nnpx vent-hq login --status # Print whether credentials are present\nnpx vent-hq logout # Remove saved credentials from ~/.vent/credentials\nnpx vent-hq run -f .vent/suite.X.json # Run a single-call suite\nnpx vent-hq run -f .vent/suite.X.json --call NAME # Run one named call from a multi-call suite\nnpx vent-hq run ... --session <session-id> # Add to any run; routes through an existing local relay session\nnpx vent-hq run ... --verbose # Add to any run; include verbose debug fields\nnpx vent-hq stop <run-id> # Cancel a queued or running run\nnpx vent-hq agent start -f .vent/suite.X.json # Start a shared local relay session\nnpx vent-hq agent stop <session-id> # Stop a shared local relay session\n```\n\nIf `~/.vent/credentials` is missing and neither `VENT_REFRESH_TOKEN` nor `VENT_ACCESS_TOKEN` is set, run `npx vent-hq init`. For an existing account, run `npx vent-hq login` or set `VENT_REFRESH_TOKEN` to an API key. Access JWTs renew automatically; `VENT_ACCESS_TOKEN` alone is short-lived.\n\n## Suite Config\n\nSuites live in `.vent/suite.<adapter>.json`. `connection` is declared once per suite. `calls` is a named map, and each key becomes the call name used with `--call`.\n\nLocal websocket suite:\n\n```json\n{\n "connection": {\n "adapter": "websocket",\n "start_command": "npm run start",\n "health_endpoint": "/health",\n "agent_port": 3001\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nPlatform-direct suite:\n\n```json\n{\n "connection": {\n "adapter": "vapi",\n "platform": { "provider": "vapi" }\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nWrite `caller_prompt` as a realistic caller with a name, goal, mood, constraints, and conditional behavior. Set `max_turns` based on flow complexity: FAQ `4-6`, booking or tool use `8-12`, complex flows `12-20`.\n\nCall fields:\n\n- `caller_prompt` and `max_turns` are required.\n- `persona` supports `pace`, `clarity`, `disfluencies`, `cooperation`, `emotion`, `memory`, `intent_clarity`, and `confirmation_style`.\n- `caller_audio` supports noise, speed, speakerphone, mic distance, clarity, accent, packet loss, and jitter.\n- `caller_audio.accent` picks the caller\'s voice. Supported: `american` (default), `british`, `australian`, `filipino`, `spanish_mexican`, `spanish_peninsular`, `spanish_colombian`, `spanish_argentine`, `german`, `french`, `italian`, `dutch`, `japanese`. Any other value is rejected. The non-English accents are voices in that language: also set `language` (e.g. `"es"`) so the caller speaks it.\n- `language` is an ISO 639-1 code such as `en`, `es`, `fr`, `de`, `it`, `nl`, or `ja`.\n- `voice` is `"male"` or `"female"` (English only; default female). Use to flip the caller\'s perceived gender. Ignored if `caller_audio.accent` is set or `language` is non-English.\n\n## Connections and Credentials\n\n### Adapter choice\n\nUse `websocket` for your own local or hosted runtime. Use `start_command` for local agents or `agent_url` for hosted custom endpoints. For `start_command` and `agent_url`, do not put Deepgram, ElevenLabs, OpenAI, or other agent runtime keys into Vent config unless the Vent adapter itself needs them \u2014 the tested agent owns its own runtime credentials.\n\nUse `vapi`, `retell`, `elevenlabs`, `bland`, or `livekit` for platform-direct testing. In this mode Vent itself talks to the provider on the user\'s behalf.\n\nVent provides `DEEPGRAM_API_KEY` and `ANTHROPIC_API_KEY` for its hosted caller/evaluation stack \u2014 those are Vent\'s, not the tested agent\'s.\n\n### Credential resolution\n\nIn platform-direct mode the CLI auto-resolves credentials from `.env.local`, `.env`, and the current shell environment. Do not run `source .env && export` before Vent commands. Keep credentials out of chat and suite files; the CLI sends the resolved configuration to Vent, which stores an encrypted copy with each run.\n\nAuto-resolved env vars and JSON fields:\n\n- Vapi: `VAPI_API_KEY` -> `vapi_api_key`; `VAPI_ASSISTANT_ID` or `VAPI_AGENT_ID` -> `vapi_assistant_id`\n- Bland: `BLAND_API_KEY` -> `bland_api_key`; `BLAND_PATHWAY_ID` -> `bland_pathway_id`; `BLAND_PERSONA_ID` -> `persona_id`\n- LiveKit: `LIVEKIT_API_KEY` -> `livekit_api_key`; `LIVEKIT_API_SECRET` -> `livekit_api_secret`; `LIVEKIT_URL` -> `livekit_url`\n- Retell: `RETELL_API_KEY` -> `retell_api_key`; `RETELL_AGENT_ID` -> `retell_agent_id`\n- ElevenLabs: `ELEVENLABS_API_KEY` -> `elevenlabs_api_key`; `ELEVENLABS_AGENT_ID` -> `elevenlabs_agent_id`\n\n### Provider config\n\nUse existing provider config when possible: Vapi assistant, Retell agent, ElevenLabs agent, Bland pathway, or LiveKit agent. Bland uniquely supports inline config \u2014 `platform` may use `bland_pathway_id`, `persona_id`, or an inline `task` (with optional voice, model, and turn-handling overrides; see Bland\'s API docs for the full field list).\n\n### Concurrency\n\nWhen you fan out multiple Vent calls in parallel against the same provider (for example, running several named calls from one suite at once with `&` and `wait`), respect the provider\'s per-account concurrency limit. Exceeding it makes calls queue or fail at the provider \u2014 Vent does not enforce these caps for you.\n\nRecord the limit as `max_concurrency` in the suite\'s `platform` block so it\'s visible on future runs. Ask the user which plan they\'re on if sizing matters; otherwise use the conservative default in bold.\n\n- **Vapi**: **10** included per account; reserved lines can be purchased self-serve; Enterprise is unlimited.\n- **Retell**: Pay-as-you-go includes **20**; Enterprise has no cap.\n- **Bland**: Start=**10**, Build=50, Scale=100, Enterprise=unlimited.\n- **ElevenLabs**: Free=**4**, Starter=6, Creator=10, Pro=20, Scale=30, Business=30. Burst pricing can temporarily allow up to 3x base.\n- **LiveKit Cloud**: Build=**5**, Ship=20, Scale=50 managed inference sessions (the usual gate for voice agents); agent-session concurrency can go higher (Scale up to 600).\n\n## WebSocket\n\nFor `adapter: "websocket"`, Vent sends binary 16-bit mono PCM audio over one websocket connection. Websocket text frames are optional JSON events. Audio-only websocket agents still work, but events improve turn detection and observability. Vent sends `{"type":"end-call"}` when the test is done.\n\nUseful websocket text frames:\n\n```jsonc\n{"type":"speech-update","status":"started"}\n{"type":"speech-update","status":"stopped"}\n{"type":"tool_call","name":"check_availability","arguments":{},"result":{},"successful":true,"duration_ms":150}\n{"type":"vent:timing","stt_ms":120,"llm_ms":450,"tts_ms":80}\n{"type":"vent:session","platform":"custom","provider_call_id":"call_123","provider_session_id":"session_abc"}\n{"type":"vent:call-metadata","call_metadata":{"recording_url":"https://...","cost_usd":0.12}}\n{"type":"vent:transcript","role":"caller","text":"I need to reschedule","turn_index":0}\n{"type":"vent:transfer","destination":"+15551234567","status":"attempted"}\n{"type":"vent:debug-url","label":"trace","url":"https://..."}\n{"type":"vent:warning","message":"provider warning","code":"provider_warning"}\n```\n\n`vent:session-report` is **not** handled by the websocket adapter \u2014 it\'s only consumed by the LiveKit helper. Do not emit it from a websocket agent.\n\nPlatform adapters capture tool calls automatically. Websocket agents must emit `tool_call` frames for tool observability. Platform adapters get component latency automatically. Websocket agents should emit `vent:timing` after each agent response when STT/LLM/TTS breakdown is available.\n\n## LiveKit\n\nBefore running LiveKit tests, install and add the Vent helper to the LiveKit agent entrypoint. Node: `npm install @vent-hq/livekit`, then call `instrumentLiveKitAgent({ ctx, session })`. Python: `pip install vent-livekit`, then call `instrument_livekit_agent(ctx=ctx, session=session)`.\n\nLiveKit direct mode requires the LiveKit Agents SDK. Custom LiveKit participants should use the websocket adapter with a relay. If the LiveKit agent registered with an explicit dispatch name, set `livekit_agent_name` in `platform`.\n\nLiveKit parallel calls are supported, but capacity depends on where the user runs their agent. LiveKit Cloud only routes audio \u2014 the agent code runs on the user\'s own infrastructure (laptop, Fly.io, Railway, k8s, etc.), and that box\'s CPU/RAM is the real bottleneck. Before firing parallel calls, derive `platform.max_concurrency` from the minimum of: (1) the user\'s LiveKit plan limit (Build=5, Ship=20, Scale=50+ concurrent agent sessions), (2) their agent worker box capacity (rough rule: 1c/1GB\u21921\u20133 jobs, 2c/4GB\u21925\u201310, 4c/8GB\u219210\u201325, 8c/16GB\u219225\u201350; multiply by number of worker boxes), and (3) their STT/TTS provider concurrency (e.g. Deepgram TTS streaming = 2). If the user wants more parallel calls than that minimum allows, push back \u2014 explain which limit is the bottleneck (plan / box / provider) and either reduce the count or suggest scaling that limit. Architecture: register one `agent_name` and run N workers under it (one big worker, or several smaller ones \u2014 same name); LiveKit\'s dispatcher round-robins jobs across them with built-in failover. Don\'t use numbered names (`agent-0`, `agent-1`) \u2014 that reinvents what the dispatcher does and breaks failover.\n\nUse the LiveKit helper for observability; do not publish `vent:*` topics manually. Do not hand-roll `vent:session-report` from `ctx.addShutdownCallback`; after `room.disconnect()` it can fail with `engine is closed`. The helper captures SDK metrics, tool events, conversation items, usage, and close events. Native LiveKit `lk.transcription` and `lk.agent.state` provide transcript and agent-state timing.\n\n## Output\n\n### Live result\n\n`npx vent-hq run` returns a single JSON result on stdout in non-TTY mode (not an SSE JSONL stream). Exit codes: `0` = the run passed (every call ran end to end); `1` = it didn\'t (a call errored, or the run failed or was cancelled); `2` = the result stream was lost. A call `error` starting with `Vent failed, agent not tested:` is a failure of Vent itself (its caller LLM, STT/TTS or worker): rerun, it says nothing about the agent. Any other `error` comes from the agent\'s platform or your configuration. `end_note` says how a completed call ended when Vent ended it or the connection closed (e.g. `No agent speech for 30s after the caller\'s turn; Vent hung up.`): read the transcript to judge whether that\'s an agent bug.\n\nMost result fields are always present; `latency`, `component_latency`, `call_metadata`, and `emotion` may be `null` when the underlying analysis didn\'t run; `debug` is absent without `--verbose`. Branch on null before reading nested fields. Use `--verbose` only when the default doesn\'t explain a failure \u2014 when you need `platform_transcript` (to check Vent\'s STT), per-turn or component-level latency breakdowns, the raw tool-call timeline, or provider-native artifacts in `debug.provider_metadata`. Otherwise skip \u2014 it just adds noise.\n\nVent\'s transcript is ground truth. Judge on semantic intent: ignore homophones and minor mis-hears (`"check teach hat"` for `"check that"`, missing question marks on short tails) \u2014 those are streaming-STT noise on Vent\'s caller side, not agent bugs, and **don\'t surface them in the report** (they\'re Vent-side artifacts, not actionable for the user). But clear gibberish or word-soup (e.g. `"Cristoxin"` where the agent should have said `"Of course, talk soon"`) is **not** a Vent artifact \u2014 Vent\'s STT does not invent words like that. It means the platform\'s TTS produced corrupted audio or the agent\'s STT/LLM generated the wrong text, and the fix lives there (TTS voice config, agent prompt, model temperature, codec issue). Missing or cut-off agent text, unlike word-soup, can come from Vent\'s STT stream dropping: check the recording before blaming the agent.\n\nFor transfers: `call_metadata.transfer_attempted` (provider claimed) and `transfer_completed` (from the platform\'s call data) can disagree \u2014 report both. `transfers[]` carries destination, type, and per-attempt status.\n\n### Saved history\n\nAfter every run, Vent writes the full result JSON to `.vent/runs/`. Shape:\n\n```jsonc\n{\n "run_id": "...",\n "timestamp": "2026-04-21T...Z",\n "git_sha": "...",\n "summary": { "calls_total": 2, "total_duration_ms": 12345, "total_cost_usd": 0.01 },\n "call_results": [\n { "name": "happy-path", "status": "completed", "duration_ms": 6123, "transcript": [], "observed_tool_calls": [], "latency": { "response_time_ms": 420, "p95_response_time_ms": 980 }, "call_metadata": { "cost_usd": 0.004 } }\n ]\n}\n```\n\nWhen comparing against a prior run (Workflow step 6), inspect:\n\n- Run-completion status flips: `call_results[i].status` (pipeline-only \u2014 judge mission success from the transcript)\n- Latency: `call_results[i].latency.response_time_ms` (mean) or `latency.p95_response_time_ms` increased >20%\n- Tool calls: count of `call_results[i].observed_tool_calls[].successful` dropped\n- Cost: `summary.total_cost_usd` or `call_results[i].call_metadata.cost_usd` increased >30%\n- Transcript: `call_results[i].transcript` diverged in semantic content (ignore STT noise)\n\n## Reporting Results\n\nBefore reporting, read the agent\'s code to locate where the observed behavior originates. If the issue is small and you can fix it, fix it and explain what you did \u2014 don\'t ask permission first.\n\nAdapt the report shape to the call \u2014 a clean pass needs little, a regression with a multi-layer cause needs more. Use a transcript excerpt when it helps the user see what happened.\n\nHard rules:\n\n- Pair raw numbers with their plain-English meaning \u2014 don\'t drop the number, but don\'t leave it unexplained. E.g. "p95 latency was 850ms, which is snappy and well within natural conversational pacing" or "p95 hit 1.6 seconds with the LLM as the bottleneck \u2014 noticeably sluggish to a caller."\n- Name the user\'s voice agent by platform on first mention (e.g. "the Vapi agent responded snappily throughout") so the user knows immediately which agent the observation is about. After that, just say "the agent" \u2014 don\'t repeat the platform name on every line.\n- Always include the recording from `call_metadata.recording_url` as an inline `[Recording](url)` link, placed in **one block at the very end of the report** \u2014 never sprinkled through the prose. Single call: one link as the last line. Multi-call: one labeled link per call (e.g. `reschedule-appointment: [Recording](url)`). Never paste a bare URL.\n- Mission success is your judgment, not Vent\'s. The per-call `status` is only `"completed"` (pipeline ran) or `"error"` (pipeline failed); decide whether the agent actually accomplished the scenario from the transcript and tool calls.\n- Similar-sounding word substitutions (e.g. "ocean" for "OSHA") are STT ambiguity, not comprehension failure. The fix lives in STT keyword hints, custom vocabulary, or a prompt-level term list \u2014 not the agent\'s reasoning.\n- Surface only what the user can act on in their own agent\'s code or config \u2014 never Vent-side artifacts (caller wait modes, harness timing, internal pipeline quirks), or `cost_usd` unless asked.\n\nFor multi-call runs, lead with your own judgment of what happened across the calls (e.g. "3 of 4 did what they were supposed to; `cancel-appointment` never actually canceled"), not a parroted pass/fail count. Then cover each call with whatever depth it needs.\n';
|
|
5868
5911
|
|
|
5869
5912
|
// src/skills/codex.md
|
|
5870
|
-
var codex_default = '# Vent - Voice Agent Calls\n\n## When to Test\n\nVent is the voice CI gate. Auto-trigger a call after any non-trivial voice-affecting change \u2014 don\'t wait to be asked. Non-trivial means anything that affects what the caller hears, how the agent responds, what tools are called, how the call routes, or audio timing: prompt rewrites or sections (not one-word tweaks), tool schema or handler changes, state/memory logic, transfer logic, provider/audio config, platform integration, and any voice-behavior bug fix. One-line copy or comment edits don\'t need a call.\n\nBefore launching, scan `.vent/runs/` for unresolved issues from prior runs (failing status, regressions, warnings, latency jumps); fix in-scope ones as part of this change and verify they\'re resolved by the new run, surface out-of-scope ones rather than silently leaving them. The task is not done until you\'ve reviewed the transcript, tool calls, and observed behavior and judged that the change works \u2014 Vent\'s `status` and exit code only reflect pipeline completion, not mission success.\n\n## Autonomous Iteration\n\n**Default: iterate.** Run a fix \u2192 Vent-call \u2192 fix \u2192 Vent-call loop whenever a call surfaces a fixable issue. Don\'t report a failure (or a degraded-quality success) back to the user before attempting to fix it \u2014 they want the agent working, not a diagnosis. After each call, if the result reveals a problem you can fix with a concrete plan, apply the fix and run again. Fixable issues include: tool schema or registry gaps, missing handlers, prompt phrasing, **STT vocabulary mismatches** (e.g. domain terms misheard \u2014 Deepgram `keyterms`, custom vocabulary, prompt-level term lists), **latency tunables** (endpointing thresholds, preemptive generation, model selection), and audio quality config. If you\'d be guessing at the next attempt, stop and ask the user instead of trying.\n\n**"Mission success but with quality problems" still requires iteration.** If the call ran end-to-end but had noticeably bad latency, STT mishearing the agent\'s domain vocabulary, audio artifacts, or any other caller-experience degradation that has a concrete fix, the call is *not done* \u2014 apply the fix and re-run. Don\'t downgrade fixable problems to "observations" or "things worth flagging." The pipeline running cleanly is the floor, not the ceiling.\n\n**Never ask permission to iterate.** "Want me to apply this and re-run?" / "Should I fix it?" / "Let me know if you want me to..." \u2014 these are skill violations. If you have a concrete fix, apply it and run the next call. The user opts out by saying so explicitly; silence is not opt-out.\n\nStop iterating and report when one of:\n\n- The next call confirms the fix worked AND no further fixable issues remain (success \u2014 done).\n- The same fix fails twice (your hypothesis is wrong; ask the user).\n- The failure mode shifts between attempts (you\'re chasing a moving target).\n- You can\'t justify the next call against its provider cost \u2014 each call spends real money and provider quota.\n\nIf the user explicitly opts out ("just report", "don\'t iterate", "stop after the first call", "I\'ll fix it myself"), respect that \u2014 run once, report, stop. Otherwise iterate by default. The first time you start an autonomous loop in a session, mention it once so the user knows it\'s happening (e.g. "the call surfaced X \u2014 fixing and re-running; I\'ll report back when it converges or stops making progress").\n\n## Codex Execution\n\nUse a 5-minute shell-tool timeout (`300000` ms) on Vent run commands so normal calls are not killed by the default 2-minute Bash timeout. This is not backgrounding; wait for stdout/results before ending your response. Use the JSON returned by `npx vent-hq run` directly.\n\nCodex can run shell tool calls concurrently \u2014 for multiple calls from one suite, run each named call as its own parallel shell tool call (do not combine them with `&`):\n\n```bash\nnpx vent-hq run -f .vent/suite.vapi.json --call happy-path\nnpx vent-hq run -f .vent/suite.vapi.json --call tool-path\n```\n\n## Workflow\n\n1. Identify the behavior under test. Read enough of the agent codebase to understand its system prompt, tools, handlers, routes, provider config, platform wiring, and expected handoffs.\n2. Reuse an existing `.vent/suite.<adapter>.json` when possible. If `.vent/` contains multiple suites, inspect `connection.adapter` and report which suite file produced the result.\n3. Create or update a suite only when the existing calls do not cover the changed behavior. Name calls after real flows, for example `reschedule-appointment`, not `call-1`.\n4. If the suite uses `start_command`, start one shared local session first with `npx vent-hq agent start -f .vent/suite.<adapter>.json`, then pass `--session <session-id>` to each run.\n\n **For locally-run LiveKit agents: every run requires killing *all* workers, starting one fresh worker, and waiting a full 60 seconds before submitting.** Unconditional \u2014 LiveKit Cloud round-robins across registered workers, so a single survivor with a dead inference subprocess fails ~N-1 of N calls. Don\'t rely on `pkill -f <path-pattern>`; bare command lines like `node --import tsx agent.ts dev` won\'t match a path filter. Use `ps aux | grep -E "node.*agent\\.ts|@livekit/agents.*ipc"`, `kill -9` by PID, re-run `ps` to confirm zero survivors, then start the fresh worker. Skipping the 60s wait fails with `did not publish audio track`; if that error appears alongside `Error [ERR_IPC_CHANNEL_CLOSED] from InferenceProcExecutor.doInference` in the agent log right after a "running EOU detection" line, that\'s a straggler \u2014 redo the kill sweep. Hosted LiveKit Cloud agents don\'t need any of this; run normally.\n5. Pick which call(s) to run based on the change. Fixed bug: replay the failing scenario. Changed tool: include a call that triggers that tool. Prompt or routing change: include the relevant happy path and any important edge path.\n6. Compare against the previous JSON in `.vent/runs/` when validating a fix or regression. Check status flips, latency jumps, tool-call success drops, cost jumps, and transcript divergence. Correlate with `git diff` between saved `git_sha` values when available; skip if no previous run exists.\n\n## Commands\n\n```bash\nnpx vent-hq init # First-time setup (auth + skill install + starter suite)\nnpx vent-hq login # Log in to existing account\nnpx vent-hq login --status # Print whether credentials are present\nnpx vent-hq logout # Remove saved credentials from ~/.vent/credentials\nnpx vent-hq run -f .vent/suite.X.json # Run a single-call suite\nnpx vent-hq run -f .vent/suite.X.json --call NAME # Run one named call from a multi-call suite\nnpx vent-hq run ... --session <session-id> # Add to any run; routes through an existing local relay session\nnpx vent-hq run ... --verbose # Add to any run; include verbose debug fields\nnpx vent-hq stop <run-id> # Cancel a queued or running run\nnpx vent-hq agent start -f .vent/suite.X.json # Start a shared local relay session\nnpx vent-hq agent stop <session-id> # Stop a shared local relay session\n```\n\nIf `~/.vent/credentials` is missing and `VENT_ACCESS_TOKEN` is not set, run `npx vent-hq init`. For an existing account, run `npx vent-hq login` or set `VENT_ACCESS_TOKEN`.\n\n## Suite Config\n\nSuites live in `.vent/suite.<adapter>.json`. `connection` is declared once per suite. `calls` is a named map, and each key becomes the call name used with `--call`.\n\nLocal websocket suite:\n\n```json\n{\n "connection": {\n "adapter": "websocket",\n "start_command": "npm run start",\n "health_endpoint": "/health",\n "agent_port": 3001\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8,\n "silence_threshold_ms": 1200\n }\n }\n}\n```\n\nPlatform-direct suite:\n\n```json\n{\n "connection": {\n "adapter": "vapi",\n "platform": { "provider": "vapi" }\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nWrite `caller_prompt` as a realistic caller with a name, goal, mood, constraints, and conditional behavior. Set `max_turns` based on flow complexity: FAQ `4-6`, booking or tool use `8-12`, complex flows `12-20`.\n\nCall fields:\n\n- `caller_prompt` and `max_turns` are required.\n- `silence_threshold_ms` must be `200-10000`. Common ranges: FAQ `800-1200`, tool calls `2000-3000`, complex reasoning `3000-5000`.\n- `persona` supports `pace`, `clarity`, `disfluencies`, `cooperation`, `emotion`, `memory`, `intent_clarity`, and `confirmation_style`.\n- `caller_audio` supports noise, speed, speakerphone, mic distance, clarity, accent, packet loss, and jitter.\n- `language` is an ISO 639-1 code such as `en`, `es`, `fr`, `de`, `it`, `nl`, or `ja`.\n- `voice` is `"male"` or `"female"` (English only; default female). Use to flip the caller\'s perceived gender. Ignored if `caller_audio.accent` is set or `language` is non-English.\n\n## Connections and Credentials\n\n### Adapter choice\n\nUse `websocket` for your own local or hosted runtime. Use `start_command` for local agents or `agent_url` for hosted custom endpoints. For `start_command` and `agent_url`, do not put Deepgram, ElevenLabs, OpenAI, or other agent runtime keys into Vent config unless the Vent adapter itself needs them \u2014 the tested agent owns its own runtime credentials.\n\nUse `vapi`, `retell`, `elevenlabs`, `bland`, or `livekit` for platform-direct testing. In this mode Vent itself talks to the provider on the user\'s behalf.\n\nVent provides `DEEPGRAM_API_KEY` and `ANTHROPIC_API_KEY` for its hosted caller/evaluation stack \u2014 those are Vent\'s, not the tested agent\'s.\n\n### Credential resolution\n\nIn platform-direct mode the CLI auto-resolves credentials from `.env.local`, `.env`, and the current shell environment. Do not run `source .env && export` before Vent commands. If you include credential fields in JSON, use the actual value, not the env var name. Do not manually author `platform_connection_id`; the CLI creates or updates the saved platform connection automatically.\n\nAuto-resolved env vars and JSON fields:\n\n- Vapi: `VAPI_API_KEY` -> `vapi_api_key`; `VAPI_ASSISTANT_ID` or `VAPI_AGENT_ID` -> `vapi_assistant_id`\n- Bland: `BLAND_API_KEY` -> `bland_api_key`; `BLAND_PATHWAY_ID` -> `bland_pathway_id`; `BLAND_PERSONA_ID` -> `persona_id`\n- LiveKit: `LIVEKIT_API_KEY` -> `livekit_api_key`; `LIVEKIT_API_SECRET` -> `livekit_api_secret`; `LIVEKIT_URL` -> `livekit_url`\n- Retell: `RETELL_API_KEY` -> `retell_api_key`; `RETELL_AGENT_ID` -> `retell_agent_id`\n- ElevenLabs: `ELEVENLABS_API_KEY` -> `elevenlabs_api_key`; `ELEVENLABS_AGENT_ID` -> `elevenlabs_agent_id`\n\n### Provider config\n\nUse existing provider config when possible: Vapi assistant, Retell agent, ElevenLabs agent, Bland pathway, or LiveKit agent. Bland uniquely supports inline config \u2014 `platform` may use `bland_pathway_id`, `persona_id`, or an inline `task` (with optional voice, model, and turn-handling overrides; see Bland\'s API docs for the full field list).\n\n### Concurrency\n\nWhen you fan out multiple Vent calls in parallel against the same provider (for example, running several named calls from one suite at once), respect the provider\'s per-account concurrency limit. Exceeding it makes calls queue or fail at the provider \u2014 Vent does not enforce these caps for you.\n\nRecord the limit as `max_concurrency` in the suite\'s `platform` block so it\'s visible on future runs. Ask the user which plan they\'re on if sizing matters; otherwise use the conservative default in bold.\n\n- **Vapi**: **10** included per account; reserved lines can be purchased self-serve; Enterprise is unlimited.\n- **Retell**: Pay-as-you-go includes **20**; Enterprise has no cap.\n- **Bland**: Start=**10**, Build=50, Scale=100, Enterprise=unlimited.\n- **ElevenLabs**: Free=**4**, Starter=6, Creator=10, Pro=20, Scale=30, Business=30. Burst pricing can temporarily allow up to 3x base.\n- **LiveKit Cloud**: Build=**5**, Ship=20, Scale=50 managed inference sessions (the usual gate for voice agents); agent-session concurrency can go higher (Scale up to 600).\n\n## WebSocket\n\nFor `adapter: "websocket"`, Vent sends binary 16-bit mono PCM audio over one websocket connection. Websocket text frames are optional JSON events. Audio-only websocket agents still work, but events improve turn detection and observability. Vent sends `{"type":"end-call"}` when the test is done.\n\nUseful websocket text frames:\n\n```jsonc\n{"type":"speech-update","status":"started"}\n{"type":"speech-update","status":"stopped"}\n{"type":"tool_call","name":"check_availability","arguments":{},"result":{},"successful":true,"duration_ms":150}\n{"type":"vent:timing","stt_ms":120,"llm_ms":450,"tts_ms":80}\n{"type":"vent:session","platform":"custom","provider_call_id":"call_123","provider_session_id":"session_abc"}\n{"type":"vent:call-metadata","call_metadata":{"recording_url":"https://...","cost_usd":0.12}}\n{"type":"vent:transcript","role":"caller","text":"I need to reschedule","turn_index":0}\n{"type":"vent:transfer","destination":"+15551234567","status":"attempted"}\n{"type":"vent:debug-url","label":"trace","url":"https://..."}\n{"type":"vent:warning","message":"provider warning","code":"provider_warning"}\n```\n\n`vent:session-report` is **not** handled by the websocket adapter \u2014 it\'s only consumed by the LiveKit helper. Do not emit it from a websocket agent.\n\nPlatform adapters capture tool calls automatically. Websocket agents must emit `tool_call` frames for tool observability. Platform adapters get component latency automatically. Websocket agents should emit `vent:timing` after each agent response when STT/LLM/TTS breakdown is available.\n\n## LiveKit\n\nBefore running LiveKit tests, install and add the Vent helper to the LiveKit agent entrypoint. Node: `npm install @vent-hq/livekit`, then call `instrumentLiveKitAgent({ ctx, session })`. Python: `pip install vent-livekit`, then call `instrument_livekit_agent(ctx=ctx, session=session)`.\n\nLiveKit direct mode requires the LiveKit Agents SDK. Custom LiveKit participants should use the websocket adapter with a relay. If the LiveKit agent registered with an explicit dispatch name, set `livekit_agent_name` in `platform`.\n\nLiveKit parallel calls are supported, but capacity depends on where the user runs their agent. LiveKit Cloud only routes audio \u2014 the agent code runs on the user\'s own infrastructure (laptop, Fly.io, Railway, k8s, etc.), and that box\'s CPU/RAM is the real bottleneck. Before firing parallel calls, derive `platform.max_concurrency` from the minimum of: (1) the user\'s LiveKit plan limit (Build=5, Ship=20, Scale=50+ concurrent agent sessions), (2) their agent worker box capacity (rough rule: 1c/1GB\u21921\u20133 jobs, 2c/4GB\u21925\u201310, 4c/8GB\u219210\u201325, 8c/16GB\u219225\u201350; multiply by number of worker boxes), and (3) their STT/TTS provider concurrency (e.g. Deepgram TTS streaming = 2). If the user wants more parallel calls than that minimum allows, push back \u2014 explain which limit is the bottleneck (plan / box / provider) and either reduce the count or suggest scaling that limit. Architecture: register one `agent_name` and run N workers under it (one big worker, or several smaller ones \u2014 same name); LiveKit\'s dispatcher round-robins jobs across them with built-in failover. Don\'t use numbered names (`agent-0`, `agent-1`) \u2014 that reinvents what the dispatcher does and breaks failover.\n\nUse the LiveKit helper for observability; do not publish `vent:*` topics manually. Do not hand-roll `vent:session-report` from `ctx.addShutdownCallback`; after `room.disconnect()` it can fail with `engine is closed`. The helper captures SDK metrics, tool events, conversation items, usage, and close events. Native LiveKit `lk.transcription` and `lk.agent.state` provide transcript and agent-state timing.\n\n## Output\n\n### Live result\n\n`npx vent-hq run` returns a single JSON result on stdout in non-TTY mode (not an SSE JSONL stream). Exit codes: `0` = call ran through the pipeline; `1` = pipeline-level failure; `2` = harness error.\n\nMost result fields are always present; `latency`, `component_latency`, `call_metadata`, and `emotion` may be `null` when the underlying analysis didn\'t run; `debug` is absent without `--verbose`. Branch on null before reading nested fields. Use `--verbose` only when the default doesn\'t explain a failure \u2014 when you need `platform_transcript` (to check Vent\'s STT), per-turn or component-level latency breakdowns, the raw tool-call timeline, or provider-native artifacts in `debug.provider_metadata`. Otherwise skip \u2014 it just adds noise.\n\nVent\'s transcript is ground truth. Judge on semantic intent: ignore homophones and minor mis-hears (`"check teach hat"` for `"check that"`, missing question marks on short tails) \u2014 those are streaming-STT noise on Vent\'s caller side, not agent bugs, and **don\'t surface them in the report** (they\'re Vent-side artifacts, not actionable for the user). But clear gibberish or word-soup (e.g. `"Cristoxin"` where the agent should have said `"Of course, talk soon"`) is **not** a Vent artifact \u2014 Vent\'s STT does not invent words like that. It means the platform\'s TTS produced corrupted audio or the agent\'s STT/LLM generated the wrong text, and the fix lives there (TTS voice config, agent prompt, model temperature, codec issue). Never dismiss the run as a "Vent harness STT" issue; iterate on the agent or flag the platform.\n\nFor transfers: `call_metadata.transfer_attempted` (provider claimed) and `transfer_completed` (Vent-verified) can disagree \u2014 report both. `transfers[]` carries destination, type, and per-attempt status.\n\n### Saved history\n\nAfter every run, Vent writes the full result JSON to `.vent/runs/`. Shape:\n\n```jsonc\n{\n "run_id": "...",\n "timestamp": "2026-04-21T...Z",\n "git_sha": "...",\n "summary": { "calls_total": 2, "total_duration_ms": 12345, "total_cost_usd": 0.01 },\n "call_results": [\n { "name": "happy-path", "status": "completed", "duration_ms": 6123, "transcript": [], "observed_tool_calls": [], "latency": { "response_time_ms": 420, "p95_response_time_ms": 980 }, "call_metadata": { "cost_usd": 0.004 } }\n ]\n}\n```\n\nWhen comparing against a prior run (Workflow step 6), inspect:\n\n- Run-completion status flips: `call_results[i].status` (pipeline-only \u2014 judge mission success from the transcript)\n- Latency: `call_results[i].latency.response_time_ms` (mean) or `latency.p95_response_time_ms` increased >20%\n- Tool calls: count of `call_results[i].observed_tool_calls[].successful` dropped\n- Cost: `summary.total_cost_usd` or `call_results[i].call_metadata.cost_usd` increased >30%\n- Transcript: `call_results[i].transcript` diverged in semantic content (ignore STT noise)\n\n## Reporting Results\n\nBefore reporting, read the agent\'s code to locate where the observed behavior originates. If the issue is small and you can fix it, fix it and explain what you did \u2014 don\'t ask permission first.\n\nAdapt the report shape to the call \u2014 a clean pass needs little, a regression with a multi-layer cause needs more. Use a transcript excerpt when it helps the user see what happened.\n\nHard rules:\n\n- Pair raw numbers with their plain-English meaning \u2014 don\'t drop the number, but don\'t leave it unexplained. E.g. "p95 latency was 850ms, which is snappy and well within natural conversational pacing" or "p95 hit 1.6 seconds with the LLM as the bottleneck \u2014 noticeably sluggish to a caller."\n- Name the user\'s voice agent by platform on first mention (e.g. "the Vapi agent responded snappily throughout") so the user knows immediately which agent the observation is about. After that, just say "the agent" \u2014 don\'t repeat the platform name on every line.\n- Always include the recording from `call_metadata.recording_url` as an inline `[Recording](url)` link, placed in **one block at the very end of the report** \u2014 never sprinkled through the prose. Single call: one link as the last line. Multi-call: one labeled link per call (e.g. `reschedule-appointment: [Recording](url)`). Never paste a bare URL.\n- Mission success is your judgment, not Vent\'s. The per-call `status` is only `"completed"` (pipeline ran) or `"error"` (pipeline failed); decide whether the agent actually accomplished the scenario from the transcript and tool calls.\n- Similar-sounding word substitutions (e.g. "ocean" for "OSHA") are STT ambiguity, not comprehension failure. The fix lives in STT keyword hints, custom vocabulary, or a prompt-level term list \u2014 not the agent\'s reasoning.\n- Surface only what the user can act on in their own agent\'s code or config \u2014 never `warnings[]` (infrastructure noise), Vent-side artifacts (caller wait modes, harness timing, internal pipeline quirks), or `cost_usd` unless asked.\n\nFor multi-call runs, lead with your own judgment of what happened across the calls (e.g. "3 of 4 did what they were supposed to; `cancel-appointment` never actually canceled"), not a parroted pass/fail count. Then cover each call with whatever depth it needs.\n';
|
|
5913
|
+
var codex_default = '# Vent - Voice Agent Calls\n\n## When to Test\n\nVent is the voice CI gate. Auto-trigger a call after any non-trivial voice-affecting change \u2014 don\'t wait to be asked. Non-trivial means anything that affects what the caller hears, how the agent responds, what tools are called, how the call routes, or audio timing: prompt rewrites or sections (not one-word tweaks), tool schema or handler changes, state/memory logic, transfer logic, provider/audio config, platform integration, and any voice-behavior bug fix. One-line copy or comment edits don\'t need a call.\n\nBefore launching, scan `.vent/runs/` for unresolved issues from prior runs (failing status, regressions, warnings, latency jumps); fix in-scope ones as part of this change and verify they\'re resolved by the new run, surface out-of-scope ones rather than silently leaving them. The task is not done until you\'ve reviewed the transcript, tool calls, and observed behavior and judged that the change works \u2014 Vent\'s `status` and exit code only reflect pipeline completion, not mission success.\n\n## Autonomous Iteration\n\n**Default: iterate.** Run a fix \u2192 Vent-call \u2192 fix \u2192 Vent-call loop whenever a call surfaces a fixable issue. Don\'t report a failure (or a degraded-quality success) back to the user before attempting to fix it \u2014 they want the agent working, not a diagnosis. After each call, if the result reveals a problem you can fix with a concrete plan, apply the fix and run again. Fixable issues include: tool schema or registry gaps, missing handlers, prompt phrasing, **STT vocabulary mismatches** (e.g. domain terms misheard \u2014 Deepgram `keyterms`, custom vocabulary, prompt-level term lists), **latency tunables** (endpointing thresholds, preemptive generation, model selection), and audio quality config. If you\'d be guessing at the next attempt, stop and ask the user instead of trying.\n\n**"Mission success but with quality problems" still requires iteration.** If the call ran end-to-end but had noticeably bad latency, STT mishearing the agent\'s domain vocabulary, audio artifacts, or any other caller-experience degradation that has a concrete fix, the call is *not done* \u2014 apply the fix and re-run. Don\'t downgrade fixable problems to "observations" or "things worth flagging." The pipeline running cleanly is the floor, not the ceiling.\n\n**Never ask permission to iterate.** "Want me to apply this and re-run?" / "Should I fix it?" / "Let me know if you want me to..." \u2014 these are skill violations. If you have a concrete fix, apply it and run the next call. The user opts out by saying so explicitly; silence is not opt-out.\n\nStop iterating and report when one of:\n\n- The next call confirms the fix worked AND no further fixable issues remain (success \u2014 done).\n- The same fix fails twice (your hypothesis is wrong; ask the user).\n- The failure mode shifts between attempts (you\'re chasing a moving target).\n- You can\'t justify the next call against its provider cost \u2014 each call spends real money and provider quota.\n\nIf the user explicitly opts out ("just report", "don\'t iterate", "stop after the first call", "I\'ll fix it myself"), respect that \u2014 run once, report, stop. Otherwise iterate by default. The first time you start an autonomous loop in a session, mention it once so the user knows it\'s happening (e.g. "the call surfaced X \u2014 fixing and re-running; I\'ll report back when it converges or stops making progress").\n\n## Codex Execution\n\nUse a 5-minute shell-tool timeout (`300000` ms) on Vent run commands so normal calls are not killed by the default 2-minute Bash timeout. This is not backgrounding; wait for stdout/results before ending your response. Use the JSON returned by `npx vent-hq run` directly.\n\nCodex can run shell tool calls concurrently \u2014 for multiple calls from one suite, run each named call as its own parallel shell tool call (do not combine them with `&`):\n\n```bash\nnpx vent-hq run -f .vent/suite.vapi.json --call happy-path\nnpx vent-hq run -f .vent/suite.vapi.json --call tool-path\n```\n\n## Workflow\n\n1. Identify the behavior under test. Read enough of the agent codebase to understand its system prompt, tools, handlers, routes, provider config, platform wiring, and expected handoffs.\n2. Reuse an existing `.vent/suite.<adapter>.json` when possible. If `.vent/` contains multiple suites, inspect `connection.adapter` and report which suite file produced the result.\n3. Create or update a suite only when the existing calls do not cover the changed behavior. Name calls after real flows, for example `reschedule-appointment`, not `call-1`.\n4. If the suite uses `start_command`, start one shared local session first with `npx vent-hq agent start -f .vent/suite.<adapter>.json`, then pass `--session <session-id>` to each run.\n\n **For locally-run LiveKit agents: every run requires killing *all* workers, starting one fresh worker, and waiting a full 60 seconds before submitting.** Unconditional \u2014 LiveKit Cloud round-robins across registered workers, so a single survivor with a dead inference subprocess fails ~N-1 of N calls. Don\'t rely on `pkill -f <path-pattern>`; bare command lines like `node --import tsx agent.ts dev` won\'t match a path filter. Use `ps aux | grep -E "node.*agent\\.ts|@livekit/agents.*ipc"`, `kill -9` by PID, re-run `ps` to confirm zero survivors, then start the fresh worker. Skipping the 60s wait fails with `did not publish audio track`; if that error appears alongside `Error [ERR_IPC_CHANNEL_CLOSED] from InferenceProcExecutor.doInference` in the agent log right after a "running EOU detection" line, that\'s a straggler \u2014 redo the kill sweep. Hosted LiveKit Cloud agents don\'t need any of this; run normally.\n5. Pick which call(s) to run based on the change. Fixed bug: replay the failing scenario. Changed tool: include a call that triggers that tool. Prompt or routing change: include the relevant happy path and any important edge path.\n6. Compare against the previous JSON in `.vent/runs/` when validating a fix or regression. Check status flips, latency jumps, tool-call success drops, cost jumps, and transcript divergence. Correlate with `git diff` between saved `git_sha` values when available; skip if no previous run exists.\n\n## Commands\n\n```bash\nnpx vent-hq init # First-time setup (auth + skill install + starter suite)\nnpx vent-hq login # Log in to existing account\nnpx vent-hq login --status # Print whether credentials are present\nnpx vent-hq logout # Remove saved credentials from ~/.vent/credentials\nnpx vent-hq run -f .vent/suite.X.json # Run a single-call suite\nnpx vent-hq run -f .vent/suite.X.json --call NAME # Run one named call from a multi-call suite\nnpx vent-hq run ... --session <session-id> # Add to any run; routes through an existing local relay session\nnpx vent-hq run ... --verbose # Add to any run; include verbose debug fields\nnpx vent-hq stop <run-id> # Cancel a queued or running run\nnpx vent-hq agent start -f .vent/suite.X.json # Start a shared local relay session\nnpx vent-hq agent stop <session-id> # Stop a shared local relay session\n```\n\nIf `~/.vent/credentials` is missing and neither `VENT_REFRESH_TOKEN` nor `VENT_ACCESS_TOKEN` is set, run `npx vent-hq init`. For an existing account, run `npx vent-hq login` or set `VENT_REFRESH_TOKEN` to an API key. Access JWTs renew automatically; `VENT_ACCESS_TOKEN` alone is short-lived.\n\n## Suite Config\n\nSuites live in `.vent/suite.<adapter>.json`. `connection` is declared once per suite. `calls` is a named map, and each key becomes the call name used with `--call`.\n\nLocal websocket suite:\n\n```json\n{\n "connection": {\n "adapter": "websocket",\n "start_command": "npm run start",\n "health_endpoint": "/health",\n "agent_port": 3001\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nPlatform-direct suite:\n\n```json\n{\n "connection": {\n "adapter": "vapi",\n "platform": { "provider": "vapi" }\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nWrite `caller_prompt` as a realistic caller with a name, goal, mood, constraints, and conditional behavior. Set `max_turns` based on flow complexity: FAQ `4-6`, booking or tool use `8-12`, complex flows `12-20`.\n\nCall fields:\n\n- `caller_prompt` and `max_turns` are required.\n- `persona` supports `pace`, `clarity`, `disfluencies`, `cooperation`, `emotion`, `memory`, `intent_clarity`, and `confirmation_style`.\n- `caller_audio` supports noise, speed, speakerphone, mic distance, clarity, accent, packet loss, and jitter.\n- `caller_audio.accent` picks the caller\'s voice. Supported: `american` (default), `british`, `australian`, `filipino`, `spanish_mexican`, `spanish_peninsular`, `spanish_colombian`, `spanish_argentine`, `german`, `french`, `italian`, `dutch`, `japanese`. Any other value is rejected. The non-English accents are voices in that language: also set `language` (e.g. `"es"`) so the caller speaks it.\n- `language` is an ISO 639-1 code such as `en`, `es`, `fr`, `de`, `it`, `nl`, or `ja`.\n- `voice` is `"male"` or `"female"` (English only; default female). Use to flip the caller\'s perceived gender. Ignored if `caller_audio.accent` is set or `language` is non-English.\n\n## Connections and Credentials\n\n### Adapter choice\n\nUse `websocket` for your own local or hosted runtime. Use `start_command` for local agents or `agent_url` for hosted custom endpoints. For `start_command` and `agent_url`, do not put Deepgram, ElevenLabs, OpenAI, or other agent runtime keys into Vent config unless the Vent adapter itself needs them \u2014 the tested agent owns its own runtime credentials.\n\nUse `vapi`, `retell`, `elevenlabs`, `bland`, or `livekit` for platform-direct testing. In this mode Vent itself talks to the provider on the user\'s behalf.\n\nVent provides `DEEPGRAM_API_KEY` and `ANTHROPIC_API_KEY` for its hosted caller/evaluation stack \u2014 those are Vent\'s, not the tested agent\'s.\n\n### Credential resolution\n\nIn platform-direct mode the CLI auto-resolves credentials from `.env.local`, `.env`, and the current shell environment. Do not run `source .env && export` before Vent commands. Keep credentials out of chat and suite files; the CLI sends the resolved configuration to Vent, which stores an encrypted copy with each run.\n\nAuto-resolved env vars and JSON fields:\n\n- Vapi: `VAPI_API_KEY` -> `vapi_api_key`; `VAPI_ASSISTANT_ID` or `VAPI_AGENT_ID` -> `vapi_assistant_id`\n- Bland: `BLAND_API_KEY` -> `bland_api_key`; `BLAND_PATHWAY_ID` -> `bland_pathway_id`; `BLAND_PERSONA_ID` -> `persona_id`\n- LiveKit: `LIVEKIT_API_KEY` -> `livekit_api_key`; `LIVEKIT_API_SECRET` -> `livekit_api_secret`; `LIVEKIT_URL` -> `livekit_url`\n- Retell: `RETELL_API_KEY` -> `retell_api_key`; `RETELL_AGENT_ID` -> `retell_agent_id`\n- ElevenLabs: `ELEVENLABS_API_KEY` -> `elevenlabs_api_key`; `ELEVENLABS_AGENT_ID` -> `elevenlabs_agent_id`\n\n### Provider config\n\nUse existing provider config when possible: Vapi assistant, Retell agent, ElevenLabs agent, Bland pathway, or LiveKit agent. Bland uniquely supports inline config \u2014 `platform` may use `bland_pathway_id`, `persona_id`, or an inline `task` (with optional voice, model, and turn-handling overrides; see Bland\'s API docs for the full field list).\n\n### Concurrency\n\nWhen you fan out multiple Vent calls in parallel against the same provider (for example, running several named calls from one suite at once), respect the provider\'s per-account concurrency limit. Exceeding it makes calls queue or fail at the provider \u2014 Vent does not enforce these caps for you.\n\nRecord the limit as `max_concurrency` in the suite\'s `platform` block so it\'s visible on future runs. Ask the user which plan they\'re on if sizing matters; otherwise use the conservative default in bold.\n\n- **Vapi**: **10** included per account; reserved lines can be purchased self-serve; Enterprise is unlimited.\n- **Retell**: Pay-as-you-go includes **20**; Enterprise has no cap.\n- **Bland**: Start=**10**, Build=50, Scale=100, Enterprise=unlimited.\n- **ElevenLabs**: Free=**4**, Starter=6, Creator=10, Pro=20, Scale=30, Business=30. Burst pricing can temporarily allow up to 3x base.\n- **LiveKit Cloud**: Build=**5**, Ship=20, Scale=50 managed inference sessions (the usual gate for voice agents); agent-session concurrency can go higher (Scale up to 600).\n\n## WebSocket\n\nFor `adapter: "websocket"`, Vent sends binary 16-bit mono PCM audio over one websocket connection. Websocket text frames are optional JSON events. Audio-only websocket agents still work, but events improve turn detection and observability. Vent sends `{"type":"end-call"}` when the test is done.\n\nUseful websocket text frames:\n\n```jsonc\n{"type":"speech-update","status":"started"}\n{"type":"speech-update","status":"stopped"}\n{"type":"tool_call","name":"check_availability","arguments":{},"result":{},"successful":true,"duration_ms":150}\n{"type":"vent:timing","stt_ms":120,"llm_ms":450,"tts_ms":80}\n{"type":"vent:session","platform":"custom","provider_call_id":"call_123","provider_session_id":"session_abc"}\n{"type":"vent:call-metadata","call_metadata":{"recording_url":"https://...","cost_usd":0.12}}\n{"type":"vent:transcript","role":"caller","text":"I need to reschedule","turn_index":0}\n{"type":"vent:transfer","destination":"+15551234567","status":"attempted"}\n{"type":"vent:debug-url","label":"trace","url":"https://..."}\n{"type":"vent:warning","message":"provider warning","code":"provider_warning"}\n```\n\n`vent:session-report` is **not** handled by the websocket adapter \u2014 it\'s only consumed by the LiveKit helper. Do not emit it from a websocket agent.\n\nPlatform adapters capture tool calls automatically. Websocket agents must emit `tool_call` frames for tool observability. Platform adapters get component latency automatically. Websocket agents should emit `vent:timing` after each agent response when STT/LLM/TTS breakdown is available.\n\n## LiveKit\n\nBefore running LiveKit tests, install and add the Vent helper to the LiveKit agent entrypoint. Node: `npm install @vent-hq/livekit`, then call `instrumentLiveKitAgent({ ctx, session })`. Python: `pip install vent-livekit`, then call `instrument_livekit_agent(ctx=ctx, session=session)`.\n\nLiveKit direct mode requires the LiveKit Agents SDK. Custom LiveKit participants should use the websocket adapter with a relay. If the LiveKit agent registered with an explicit dispatch name, set `livekit_agent_name` in `platform`.\n\nLiveKit parallel calls are supported, but capacity depends on where the user runs their agent. LiveKit Cloud only routes audio \u2014 the agent code runs on the user\'s own infrastructure (laptop, Fly.io, Railway, k8s, etc.), and that box\'s CPU/RAM is the real bottleneck. Before firing parallel calls, derive `platform.max_concurrency` from the minimum of: (1) the user\'s LiveKit plan limit (Build=5, Ship=20, Scale=50+ concurrent agent sessions), (2) their agent worker box capacity (rough rule: 1c/1GB\u21921\u20133 jobs, 2c/4GB\u21925\u201310, 4c/8GB\u219210\u201325, 8c/16GB\u219225\u201350; multiply by number of worker boxes), and (3) their STT/TTS provider concurrency (e.g. Deepgram TTS streaming = 2). If the user wants more parallel calls than that minimum allows, push back \u2014 explain which limit is the bottleneck (plan / box / provider) and either reduce the count or suggest scaling that limit. Architecture: register one `agent_name` and run N workers under it (one big worker, or several smaller ones \u2014 same name); LiveKit\'s dispatcher round-robins jobs across them with built-in failover. Don\'t use numbered names (`agent-0`, `agent-1`) \u2014 that reinvents what the dispatcher does and breaks failover.\n\nUse the LiveKit helper for observability; do not publish `vent:*` topics manually. Do not hand-roll `vent:session-report` from `ctx.addShutdownCallback`; after `room.disconnect()` it can fail with `engine is closed`. The helper captures SDK metrics, tool events, conversation items, usage, and close events. Native LiveKit `lk.transcription` and `lk.agent.state` provide transcript and agent-state timing.\n\n## Output\n\n### Live result\n\n`npx vent-hq run` returns a single JSON result on stdout in non-TTY mode (not an SSE JSONL stream). Exit codes: `0` = the run passed (every call ran end to end); `1` = it didn\'t (a call errored, or the run failed or was cancelled); `2` = the result stream was lost. A call `error` starting with `Vent failed, agent not tested:` is a failure of Vent itself (its caller LLM, STT/TTS or worker): rerun, it says nothing about the agent. Any other `error` comes from the agent\'s platform or your configuration. `end_note` says how a completed call ended when Vent ended it or the connection closed (e.g. `No agent speech for 30s after the caller\'s turn; Vent hung up.`): read the transcript to judge whether that\'s an agent bug.\n\nMost result fields are always present; `latency`, `component_latency`, `call_metadata`, and `emotion` may be `null` when the underlying analysis didn\'t run; `debug` is absent without `--verbose`. Branch on null before reading nested fields. Use `--verbose` only when the default doesn\'t explain a failure \u2014 when you need `platform_transcript` (to check Vent\'s STT), per-turn or component-level latency breakdowns, the raw tool-call timeline, or provider-native artifacts in `debug.provider_metadata`. Otherwise skip \u2014 it just adds noise.\n\nVent\'s transcript is ground truth. Judge on semantic intent: ignore homophones and minor mis-hears (`"check teach hat"` for `"check that"`, missing question marks on short tails) \u2014 those are streaming-STT noise on Vent\'s caller side, not agent bugs, and **don\'t surface them in the report** (they\'re Vent-side artifacts, not actionable for the user). But clear gibberish or word-soup (e.g. `"Cristoxin"` where the agent should have said `"Of course, talk soon"`) is **not** a Vent artifact \u2014 Vent\'s STT does not invent words like that. It means the platform\'s TTS produced corrupted audio or the agent\'s STT/LLM generated the wrong text, and the fix lives there (TTS voice config, agent prompt, model temperature, codec issue). Missing or cut-off agent text, unlike word-soup, can come from Vent\'s STT stream dropping: check the recording before blaming the agent.\n\nFor transfers: `call_metadata.transfer_attempted` (provider claimed) and `transfer_completed` (from the platform\'s call data) can disagree \u2014 report both. `transfers[]` carries destination, type, and per-attempt status.\n\n### Saved history\n\nAfter every run, Vent writes the full result JSON to `.vent/runs/`. Shape:\n\n```jsonc\n{\n "run_id": "...",\n "timestamp": "2026-04-21T...Z",\n "git_sha": "...",\n "summary": { "calls_total": 2, "total_duration_ms": 12345, "total_cost_usd": 0.01 },\n "call_results": [\n { "name": "happy-path", "status": "completed", "duration_ms": 6123, "transcript": [], "observed_tool_calls": [], "latency": { "response_time_ms": 420, "p95_response_time_ms": 980 }, "call_metadata": { "cost_usd": 0.004 } }\n ]\n}\n```\n\nWhen comparing against a prior run (Workflow step 6), inspect:\n\n- Run-completion status flips: `call_results[i].status` (pipeline-only \u2014 judge mission success from the transcript)\n- Latency: `call_results[i].latency.response_time_ms` (mean) or `latency.p95_response_time_ms` increased >20%\n- Tool calls: count of `call_results[i].observed_tool_calls[].successful` dropped\n- Cost: `summary.total_cost_usd` or `call_results[i].call_metadata.cost_usd` increased >30%\n- Transcript: `call_results[i].transcript` diverged in semantic content (ignore STT noise)\n\n## Reporting Results\n\nBefore reporting, read the agent\'s code to locate where the observed behavior originates. If the issue is small and you can fix it, fix it and explain what you did \u2014 don\'t ask permission first.\n\nAdapt the report shape to the call \u2014 a clean pass needs little, a regression with a multi-layer cause needs more. Use a transcript excerpt when it helps the user see what happened.\n\nHard rules:\n\n- Pair raw numbers with their plain-English meaning \u2014 don\'t drop the number, but don\'t leave it unexplained. E.g. "p95 latency was 850ms, which is snappy and well within natural conversational pacing" or "p95 hit 1.6 seconds with the LLM as the bottleneck \u2014 noticeably sluggish to a caller."\n- Name the user\'s voice agent by platform on first mention (e.g. "the Vapi agent responded snappily throughout") so the user knows immediately which agent the observation is about. After that, just say "the agent" \u2014 don\'t repeat the platform name on every line.\n- Always include the recording from `call_metadata.recording_url` as an inline `[Recording](url)` link, placed in **one block at the very end of the report** \u2014 never sprinkled through the prose. Single call: one link as the last line. Multi-call: one labeled link per call (e.g. `reschedule-appointment: [Recording](url)`). Never paste a bare URL.\n- Mission success is your judgment, not Vent\'s. The per-call `status` is only `"completed"` (pipeline ran) or `"error"` (pipeline failed); decide whether the agent actually accomplished the scenario from the transcript and tool calls.\n- Similar-sounding word substitutions (e.g. "ocean" for "OSHA") are STT ambiguity, not comprehension failure. The fix lives in STT keyword hints, custom vocabulary, or a prompt-level term list \u2014 not the agent\'s reasoning.\n- Surface only what the user can act on in their own agent\'s code or config \u2014 never Vent-side artifacts (caller wait modes, harness timing, internal pipeline quirks), or `cost_usd` unless asked.\n\nFor multi-call runs, lead with your own judgment of what happened across the calls (e.g. "3 of 4 did what they were supposed to; `cancel-appointment` never actually canceled"), not a parroted pass/fail count. Then cover each call with whatever depth it needs.\n';
|
|
5871
5914
|
|
|
5872
5915
|
// src/skills/windsurf.md
|
|
5873
|
-
var windsurf_default = '---\nname: vent\ndescription: Place real calls against the user\'s voice agent to verify recent code changes worked end-to-end. Trigger on test requests or after any non-trivial voice-agent change.\n---\n\n# Vent - Voice Agent Calls\n\n## When to Test\n\nVent is the voice CI gate. Auto-trigger a call after any non-trivial voice-affecting change \u2014 don\'t wait to be asked. Non-trivial means anything that affects what the caller hears, how the agent responds, what tools are called, how the call routes, or audio timing: prompt rewrites or sections (not one-word tweaks), tool schema or handler changes, state/memory logic, transfer logic, provider/audio config, platform integration, and any voice-behavior bug fix. One-line copy or comment edits don\'t need a call.\n\nBefore launching, scan `.vent/runs/` for unresolved issues from prior runs (failing status, regressions, warnings, latency jumps); fix in-scope ones as part of this change and verify they\'re resolved by the new run, surface out-of-scope ones rather than silently leaving them. The task is not done until you\'ve reviewed the transcript, tool calls, and observed behavior and judged that the change works \u2014 Vent\'s `status` and exit code only reflect pipeline completion, not mission success.\n\n## Autonomous Iteration\n\n**Default: iterate.** Run a fix \u2192 Vent-call \u2192 fix \u2192 Vent-call loop whenever a call surfaces a fixable issue. Don\'t report a failure (or a degraded-quality success) back to the user before attempting to fix it \u2014 they want the agent working, not a diagnosis. After each call, if the result reveals a problem you can fix with a concrete plan, apply the fix and run again. Fixable issues include: tool schema or registry gaps, missing handlers, prompt phrasing, **STT vocabulary mismatches** (e.g. domain terms misheard \u2014 Deepgram `keyterms`, custom vocabulary, prompt-level term lists), **latency tunables** (endpointing thresholds, preemptive generation, model selection), and audio quality config. If you\'d be guessing at the next attempt, stop and ask the user instead of trying.\n\n**"Mission success but with quality problems" still requires iteration.** If the call ran end-to-end but had noticeably bad latency, STT mishearing the agent\'s domain vocabulary, audio artifacts, or any other caller-experience degradation that has a concrete fix, the call is *not done* \u2014 apply the fix and re-run. Don\'t downgrade fixable problems to "observations" or "things worth flagging." The pipeline running cleanly is the floor, not the ceiling.\n\n**Never ask permission to iterate.** "Want me to apply this and re-run?" / "Should I fix it?" / "Let me know if you want me to..." \u2014 these are skill violations. If you have a concrete fix, apply it and run the next call. The user opts out by saying so explicitly; silence is not opt-out.\n\nStop iterating and report when one of:\n\n- The next call confirms the fix worked AND no further fixable issues remain (success \u2014 done).\n- The same fix fails twice (your hypothesis is wrong; ask the user).\n- The failure mode shifts between attempts (you\'re chasing a moving target).\n- You can\'t justify the next call against its provider cost \u2014 each call spends real money and provider quota.\n\nIf the user explicitly opts out ("just report", "don\'t iterate", "stop after the first call", "I\'ll fix it myself"), respect that \u2014 run once, report, stop. Otherwise iterate by default. The first time you start an autonomous loop in a session, mention it once so the user knows it\'s happening (e.g. "the call surfaced X \u2014 fixing and re-running; I\'ll report back when it converges or stops making progress").\n\n## Windsurf Execution\n\nVent calls typically take 30 seconds to 2 minutes. Each `vent-hq run` is one shell tool call \u2014 wait for stdout (the JSON result) before responding. Don\'t background; use the JSON returned by `npx vent-hq run` directly. If Cascade\'s auto-execution level prompts for approval on `npx vent-hq` commands, ask the user to add `npx vent-hq *` to the workspace allow list once so subsequent calls flow without interruption.\n\nCascade runs shell tool calls in parallel within a turn \u2014 for multiple calls from one suite, issue each named call as its own separate shell tool call in the same turn (do not combine them with `&` and `wait`):\n\n```bash\nnpx vent-hq run -f .vent/suite.vapi.json --call happy-path\nnpx vent-hq run -f .vent/suite.vapi.json --call tool-path\n```\n\nStay within Cascade\'s per-turn parallel-tool-call budget \u2014 fan out at most ~6 calls in one turn and respect the provider concurrency caps below.\n\n## Workflow\n\n1. Identify the behavior under test. Read enough of the agent codebase to understand its system prompt, tools, handlers, routes, provider config, platform wiring, and expected handoffs.\n2. Reuse an existing `.vent/suite.<adapter>.json` when possible. If `.vent/` contains multiple suites, inspect `connection.adapter` and report which suite file produced the result.\n3. Create or update a suite only when the existing calls do not cover the changed behavior. Name calls after real flows, for example `reschedule-appointment`, not `call-1`.\n4. If the suite uses `start_command`, start one shared local session first with `npx vent-hq agent start -f .vent/suite.<adapter>.json`, then pass `--session <session-id>` to each run.\n\n **For locally-run LiveKit agents: every run requires killing *all* workers, starting one fresh worker, and waiting a full 60 seconds before submitting.** Unconditional \u2014 LiveKit Cloud round-robins across registered workers, so a single survivor with a dead inference subprocess fails ~N-1 of N calls. Don\'t rely on `pkill -f <path-pattern>`; bare command lines like `node --import tsx agent.ts dev` won\'t match a path filter. Use `ps aux | grep -E "node.*agent\\.ts|@livekit/agents.*ipc"`, `kill -9` by PID, re-run `ps` to confirm zero survivors, then start the fresh worker. Skipping the 60s wait fails with `did not publish audio track`; if that error appears alongside `Error [ERR_IPC_CHANNEL_CLOSED] from InferenceProcExecutor.doInference` in the agent log right after a "running EOU detection" line, that\'s a straggler \u2014 redo the kill sweep. Hosted LiveKit Cloud agents don\'t need any of this; run normally.\n5. Pick which call(s) to run based on the change. Fixed bug: replay the failing scenario. Changed tool: include a call that triggers that tool. Prompt or routing change: include the relevant happy path and any important edge path.\n6. Compare against the previous JSON in `.vent/runs/` when validating a fix or regression. Check status flips, latency jumps, tool-call success drops, cost jumps, and transcript divergence. Correlate with `git diff` between saved `git_sha` values when available; skip if no previous run exists.\n\n## Commands\n\n```bash\nnpx vent-hq init # First-time setup (auth + skill install + starter suite)\nnpx vent-hq login # Log in to existing account\nnpx vent-hq login --status # Print whether credentials are present\nnpx vent-hq logout # Remove saved credentials from ~/.vent/credentials\nnpx vent-hq run -f .vent/suite.X.json # Run a single-call suite\nnpx vent-hq run -f .vent/suite.X.json --call NAME # Run one named call from a multi-call suite\nnpx vent-hq run ... --session <session-id> # Add to any run; routes through an existing local relay session\nnpx vent-hq run ... --verbose # Add to any run; include verbose debug fields\nnpx vent-hq stop <run-id> # Cancel a queued or running run\nnpx vent-hq agent start -f .vent/suite.X.json # Start a shared local relay session\nnpx vent-hq agent stop <session-id> # Stop a shared local relay session\n```\n\nIf `~/.vent/credentials` is missing and `VENT_ACCESS_TOKEN` is not set, run `npx vent-hq init`. For an existing account, run `npx vent-hq login` or set `VENT_ACCESS_TOKEN`.\n\n## Suite Config\n\nSuites live in `.vent/suite.<adapter>.json`. `connection` is declared once per suite. `calls` is a named map, and each key becomes the call name used with `--call`.\n\nLocal websocket suite:\n\n```json\n{\n "connection": {\n "adapter": "websocket",\n "start_command": "npm run start",\n "health_endpoint": "/health",\n "agent_port": 3001\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8,\n "silence_threshold_ms": 1200\n }\n }\n}\n```\n\nPlatform-direct suite:\n\n```json\n{\n "connection": {\n "adapter": "vapi",\n "platform": { "provider": "vapi" }\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nWrite `caller_prompt` as a realistic caller with a name, goal, mood, constraints, and conditional behavior. Set `max_turns` based on flow complexity: FAQ `4-6`, booking or tool use `8-12`, complex flows `12-20`.\n\nCall fields:\n\n- `caller_prompt` and `max_turns` are required.\n- `silence_threshold_ms` must be `200-10000`. Common ranges: FAQ `800-1200`, tool calls `2000-3000`, complex reasoning `3000-5000`.\n- `persona` supports `pace`, `clarity`, `disfluencies`, `cooperation`, `emotion`, `memory`, `intent_clarity`, and `confirmation_style`.\n- `caller_audio` supports noise, speed, speakerphone, mic distance, clarity, accent, packet loss, and jitter.\n- `language` is an ISO 639-1 code such as `en`, `es`, `fr`, `de`, `it`, `nl`, or `ja`.\n- `voice` is `"male"` or `"female"` (English only; default female). Use to flip the caller\'s perceived gender. Ignored if `caller_audio.accent` is set or `language` is non-English.\n\n## Connections and Credentials\n\n### Adapter choice\n\nUse `websocket` for your own local or hosted runtime. Use `start_command` for local agents or `agent_url` for hosted custom endpoints. For `start_command` and `agent_url`, do not put Deepgram, ElevenLabs, OpenAI, or other agent runtime keys into Vent config unless the Vent adapter itself needs them \u2014 the tested agent owns its own runtime credentials.\n\nUse `vapi`, `retell`, `elevenlabs`, `bland`, or `livekit` for platform-direct testing. In this mode Vent itself talks to the provider on the user\'s behalf.\n\nVent provides `DEEPGRAM_API_KEY` and `ANTHROPIC_API_KEY` for its hosted caller/evaluation stack \u2014 those are Vent\'s, not the tested agent\'s.\n\n### Credential resolution\n\nIn platform-direct mode the CLI auto-resolves credentials from `.env.local`, `.env`, and the current shell environment. Do not run `source .env && export` before Vent commands. If you include credential fields in JSON, use the actual value, not the env var name. Do not manually author `platform_connection_id`; the CLI creates or updates the saved platform connection automatically.\n\nAuto-resolved env vars and JSON fields:\n\n- Vapi: `VAPI_API_KEY` -> `vapi_api_key`; `VAPI_ASSISTANT_ID` or `VAPI_AGENT_ID` -> `vapi_assistant_id`\n- Bland: `BLAND_API_KEY` -> `bland_api_key`; `BLAND_PATHWAY_ID` -> `bland_pathway_id`; `BLAND_PERSONA_ID` -> `persona_id`\n- LiveKit: `LIVEKIT_API_KEY` -> `livekit_api_key`; `LIVEKIT_API_SECRET` -> `livekit_api_secret`; `LIVEKIT_URL` -> `livekit_url`\n- Retell: `RETELL_API_KEY` -> `retell_api_key`; `RETELL_AGENT_ID` -> `retell_agent_id`\n- ElevenLabs: `ELEVENLABS_API_KEY` -> `elevenlabs_api_key`; `ELEVENLABS_AGENT_ID` -> `elevenlabs_agent_id`\n\n### Provider config\n\nUse existing provider config when possible: Vapi assistant, Retell agent, ElevenLabs agent, Bland pathway, or LiveKit agent. Bland uniquely supports inline config \u2014 `platform` may use `bland_pathway_id`, `persona_id`, or an inline `task` (with optional voice, model, and turn-handling overrides; see Bland\'s API docs for the full field list).\n\n### Concurrency\n\nWhen you fan out multiple Vent calls in parallel against the same provider (for example, running several named calls from one suite at once), respect the provider\'s per-account concurrency limit. Exceeding it makes calls queue or fail at the provider \u2014 Vent does not enforce these caps for you.\n\nRecord the limit as `max_concurrency` in the suite\'s `platform` block so it\'s visible on future runs. Ask the user which plan they\'re on if sizing matters; otherwise use the conservative default in bold.\n\n- **Vapi**: **10** included per account; reserved lines can be purchased self-serve; Enterprise is unlimited.\n- **Retell**: Pay-as-you-go includes **20**; Enterprise has no cap.\n- **Bland**: Start=**10**, Build=50, Scale=100, Enterprise=unlimited.\n- **ElevenLabs**: Free=**4**, Starter=6, Creator=10, Pro=20, Scale=30, Business=30. Burst pricing can temporarily allow up to 3x base.\n- **LiveKit Cloud**: Build=**5**, Ship=20, Scale=50 managed inference sessions (the usual gate for voice agents); agent-session concurrency can go higher (Scale up to 600).\n\n## WebSocket\n\nFor `adapter: "websocket"`, Vent sends binary 16-bit mono PCM audio over one websocket connection. Websocket text frames are optional JSON events. Audio-only websocket agents still work, but events improve turn detection and observability. Vent sends `{"type":"end-call"}` when the test is done.\n\nUseful websocket text frames:\n\n```jsonc\n{"type":"speech-update","status":"started"}\n{"type":"speech-update","status":"stopped"}\n{"type":"tool_call","name":"check_availability","arguments":{},"result":{},"successful":true,"duration_ms":150}\n{"type":"vent:timing","stt_ms":120,"llm_ms":450,"tts_ms":80}\n{"type":"vent:session","platform":"custom","provider_call_id":"call_123","provider_session_id":"session_abc"}\n{"type":"vent:call-metadata","call_metadata":{"recording_url":"https://...","cost_usd":0.12}}\n{"type":"vent:transcript","role":"caller","text":"I need to reschedule","turn_index":0}\n{"type":"vent:transfer","destination":"+15551234567","status":"attempted"}\n{"type":"vent:debug-url","label":"trace","url":"https://..."}\n{"type":"vent:warning","message":"provider warning","code":"provider_warning"}\n```\n\n`vent:session-report` is **not** handled by the websocket adapter \u2014 it\'s only consumed by the LiveKit helper. Do not emit it from a websocket agent.\n\nPlatform adapters capture tool calls automatically. Websocket agents must emit `tool_call` frames for tool observability. Platform adapters get component latency automatically. Websocket agents should emit `vent:timing` after each agent response when STT/LLM/TTS breakdown is available.\n\n## LiveKit\n\nBefore running LiveKit tests, install and add the Vent helper to the LiveKit agent entrypoint. Node: `npm install @vent-hq/livekit`, then call `instrumentLiveKitAgent({ ctx, session })`. Python: `pip install vent-livekit`, then call `instrument_livekit_agent(ctx=ctx, session=session)`.\n\nLiveKit direct mode requires the LiveKit Agents SDK. Custom LiveKit participants should use the websocket adapter with a relay. If the LiveKit agent registered with an explicit dispatch name, set `livekit_agent_name` in `platform`.\n\nLiveKit parallel calls are supported, but capacity depends on where the user runs their agent. LiveKit Cloud only routes audio \u2014 the agent code runs on the user\'s own infrastructure (laptop, Fly.io, Railway, k8s, etc.), and that box\'s CPU/RAM is the real bottleneck. Before firing parallel calls, derive `platform.max_concurrency` from the minimum of: (1) the user\'s LiveKit plan limit (Build=5, Ship=20, Scale=50+ concurrent agent sessions), (2) their agent worker box capacity (rough rule: 1c/1GB\u21921\u20133 jobs, 2c/4GB\u21925\u201310, 4c/8GB\u219210\u201325, 8c/16GB\u219225\u201350; multiply by number of worker boxes), and (3) their STT/TTS provider concurrency (e.g. Deepgram TTS streaming = 2). If the user wants more parallel calls than that minimum allows, push back \u2014 explain which limit is the bottleneck (plan / box / provider) and either reduce the count or suggest scaling that limit. Architecture: register one `agent_name` and run N workers under it (one big worker, or several smaller ones \u2014 same name); LiveKit\'s dispatcher round-robins jobs across them with built-in failover. Don\'t use numbered names (`agent-0`, `agent-1`) \u2014 that reinvents what the dispatcher does and breaks failover.\n\nUse the LiveKit helper for observability; do not publish `vent:*` topics manually. Do not hand-roll `vent:session-report` from `ctx.addShutdownCallback`; after `room.disconnect()` it can fail with `engine is closed`. The helper captures SDK metrics, tool events, conversation items, usage, and close events. Native LiveKit `lk.transcription` and `lk.agent.state` provide transcript and agent-state timing.\n\n## Output\n\n### Live result\n\n`npx vent-hq run` returns a single JSON result on stdout in non-TTY mode (not an SSE JSONL stream). Exit codes: `0` = call ran through the pipeline; `1` = pipeline-level failure; `2` = harness error.\n\nMost result fields are always present; `latency`, `component_latency`, `call_metadata`, and `emotion` may be `null` when the underlying analysis didn\'t run; `debug` is absent without `--verbose`. Branch on null before reading nested fields. Use `--verbose` only when the default doesn\'t explain a failure \u2014 when you need `platform_transcript` (to check Vent\'s STT), per-turn or component-level latency breakdowns, the raw tool-call timeline, or provider-native artifacts in `debug.provider_metadata`. Otherwise skip \u2014 it just adds noise.\n\nVent\'s transcript is ground truth. Judge on semantic intent: ignore homophones and minor mis-hears (`"check teach hat"` for `"check that"`, missing question marks on short tails) \u2014 those are streaming-STT noise on Vent\'s caller side, not agent bugs, and **don\'t surface them in the report** (they\'re Vent-side artifacts, not actionable for the user). But clear gibberish or word-soup (e.g. `"Cristoxin"` where the agent should have said `"Of course, talk soon"`) is **not** a Vent artifact \u2014 Vent\'s STT does not invent words like that. It means the platform\'s TTS produced corrupted audio or the agent\'s STT/LLM generated the wrong text, and the fix lives there (TTS voice config, agent prompt, model temperature, codec issue). Never dismiss the run as a "Vent harness STT" issue; iterate on the agent or flag the platform.\n\nFor transfers: `call_metadata.transfer_attempted` (provider claimed) and `transfer_completed` (Vent-verified) can disagree \u2014 report both. `transfers[]` carries destination, type, and per-attempt status.\n\n### Saved history\n\nAfter every run, Vent writes the full result JSON to `.vent/runs/`. Shape:\n\n```jsonc\n{\n "run_id": "...",\n "timestamp": "2026-04-21T...Z",\n "git_sha": "...",\n "summary": { "calls_total": 2, "total_duration_ms": 12345, "total_cost_usd": 0.01 },\n "call_results": [\n { "name": "happy-path", "status": "completed", "duration_ms": 6123, "transcript": [], "observed_tool_calls": [], "latency": { "response_time_ms": 420, "p95_response_time_ms": 980 }, "call_metadata": { "cost_usd": 0.004 } }\n ]\n}\n```\n\nWhen comparing against a prior run (Workflow step 6), inspect:\n\n- Run-completion status flips: `call_results[i].status` (pipeline-only \u2014 judge mission success from the transcript)\n- Latency: `call_results[i].latency.response_time_ms` (mean) or `latency.p95_response_time_ms` increased >20%\n- Tool calls: count of `call_results[i].observed_tool_calls[].successful` dropped\n- Cost: `summary.total_cost_usd` or `call_results[i].call_metadata.cost_usd` increased >30%\n- Transcript: `call_results[i].transcript` diverged in semantic content (ignore STT noise)\n\n## Reporting Results\n\nBefore reporting, read the agent\'s code to locate where the observed behavior originates. If the issue is small and you can fix it, fix it and explain what you did \u2014 don\'t ask permission first.\n\nAdapt the report shape to the call \u2014 a clean pass needs little, a regression with a multi-layer cause needs more. Use a transcript excerpt when it helps the user see what happened.\n\nHard rules:\n\n- Pair raw numbers with their plain-English meaning \u2014 don\'t drop the number, but don\'t leave it unexplained. E.g. "p95 latency was 850ms, which is snappy and well within natural conversational pacing" or "p95 hit 1.6 seconds with the LLM as the bottleneck \u2014 noticeably sluggish to a caller."\n- Name the user\'s voice agent by platform on first mention (e.g. "the Vapi agent responded snappily throughout") so the user knows immediately which agent the observation is about. After that, just say "the agent" \u2014 don\'t repeat the platform name on every line.\n- Always include the recording from `call_metadata.recording_url` as an inline `[Recording](url)` link, placed in **one block at the very end of the report** \u2014 never sprinkled through the prose. Single call: one link as the last line. Multi-call: one labeled link per call (e.g. `reschedule-appointment: [Recording](url)`). Never paste a bare URL.\n- Mission success is your judgment, not Vent\'s. The per-call `status` is only `"completed"` (pipeline ran) or `"error"` (pipeline failed); decide whether the agent actually accomplished the scenario from the transcript and tool calls.\n- Similar-sounding word substitutions (e.g. "ocean" for "OSHA") are STT ambiguity, not comprehension failure. The fix lives in STT keyword hints, custom vocabulary, or a prompt-level term list \u2014 not the agent\'s reasoning.\n- Surface only what the user can act on in their own agent\'s code or config \u2014 never `warnings[]` (infrastructure noise), Vent-side artifacts (caller wait modes, harness timing, internal pipeline quirks), or `cost_usd` unless asked.\n\nFor multi-call runs, lead with your own judgment of what happened across the calls (e.g. "3 of 4 did what they were supposed to; `cancel-appointment` never actually canceled"), not a parroted pass/fail count. Then cover each call with whatever depth it needs.\n';
|
|
5916
|
+
var windsurf_default = '---\nname: vent\ndescription: Place real calls against the user\'s voice agent to verify recent code changes worked end-to-end. Trigger on test requests or after any non-trivial voice-agent change.\n---\n\n# Vent - Voice Agent Calls\n\n## When to Test\n\nVent is the voice CI gate. Auto-trigger a call after any non-trivial voice-affecting change \u2014 don\'t wait to be asked. Non-trivial means anything that affects what the caller hears, how the agent responds, what tools are called, how the call routes, or audio timing: prompt rewrites or sections (not one-word tweaks), tool schema or handler changes, state/memory logic, transfer logic, provider/audio config, platform integration, and any voice-behavior bug fix. One-line copy or comment edits don\'t need a call.\n\nBefore launching, scan `.vent/runs/` for unresolved issues from prior runs (failing status, regressions, warnings, latency jumps); fix in-scope ones as part of this change and verify they\'re resolved by the new run, surface out-of-scope ones rather than silently leaving them. The task is not done until you\'ve reviewed the transcript, tool calls, and observed behavior and judged that the change works \u2014 Vent\'s `status` and exit code only reflect pipeline completion, not mission success.\n\n## Autonomous Iteration\n\n**Default: iterate.** Run a fix \u2192 Vent-call \u2192 fix \u2192 Vent-call loop whenever a call surfaces a fixable issue. Don\'t report a failure (or a degraded-quality success) back to the user before attempting to fix it \u2014 they want the agent working, not a diagnosis. After each call, if the result reveals a problem you can fix with a concrete plan, apply the fix and run again. Fixable issues include: tool schema or registry gaps, missing handlers, prompt phrasing, **STT vocabulary mismatches** (e.g. domain terms misheard \u2014 Deepgram `keyterms`, custom vocabulary, prompt-level term lists), **latency tunables** (endpointing thresholds, preemptive generation, model selection), and audio quality config. If you\'d be guessing at the next attempt, stop and ask the user instead of trying.\n\n**"Mission success but with quality problems" still requires iteration.** If the call ran end-to-end but had noticeably bad latency, STT mishearing the agent\'s domain vocabulary, audio artifacts, or any other caller-experience degradation that has a concrete fix, the call is *not done* \u2014 apply the fix and re-run. Don\'t downgrade fixable problems to "observations" or "things worth flagging." The pipeline running cleanly is the floor, not the ceiling.\n\n**Never ask permission to iterate.** "Want me to apply this and re-run?" / "Should I fix it?" / "Let me know if you want me to..." \u2014 these are skill violations. If you have a concrete fix, apply it and run the next call. The user opts out by saying so explicitly; silence is not opt-out.\n\nStop iterating and report when one of:\n\n- The next call confirms the fix worked AND no further fixable issues remain (success \u2014 done).\n- The same fix fails twice (your hypothesis is wrong; ask the user).\n- The failure mode shifts between attempts (you\'re chasing a moving target).\n- You can\'t justify the next call against its provider cost \u2014 each call spends real money and provider quota.\n\nIf the user explicitly opts out ("just report", "don\'t iterate", "stop after the first call", "I\'ll fix it myself"), respect that \u2014 run once, report, stop. Otherwise iterate by default. The first time you start an autonomous loop in a session, mention it once so the user knows it\'s happening (e.g. "the call surfaced X \u2014 fixing and re-running; I\'ll report back when it converges or stops making progress").\n\n## Windsurf Execution\n\nVent calls typically take 30 seconds to 2 minutes. Each `vent-hq run` is one shell tool call \u2014 wait for stdout (the JSON result) before responding. Don\'t background; use the JSON returned by `npx vent-hq run` directly. If Cascade\'s auto-execution level prompts for approval on `npx vent-hq` commands, ask the user to add `npx vent-hq *` to the workspace allow list once so subsequent calls flow without interruption.\n\nCascade runs shell tool calls in parallel within a turn \u2014 for multiple calls from one suite, issue each named call as its own separate shell tool call in the same turn (do not combine them with `&` and `wait`):\n\n```bash\nnpx vent-hq run -f .vent/suite.vapi.json --call happy-path\nnpx vent-hq run -f .vent/suite.vapi.json --call tool-path\n```\n\nStay within Cascade\'s per-turn parallel-tool-call budget \u2014 fan out at most ~6 calls in one turn and respect the provider concurrency caps below.\n\n## Workflow\n\n1. Identify the behavior under test. Read enough of the agent codebase to understand its system prompt, tools, handlers, routes, provider config, platform wiring, and expected handoffs.\n2. Reuse an existing `.vent/suite.<adapter>.json` when possible. If `.vent/` contains multiple suites, inspect `connection.adapter` and report which suite file produced the result.\n3. Create or update a suite only when the existing calls do not cover the changed behavior. Name calls after real flows, for example `reschedule-appointment`, not `call-1`.\n4. If the suite uses `start_command`, start one shared local session first with `npx vent-hq agent start -f .vent/suite.<adapter>.json`, then pass `--session <session-id>` to each run.\n\n **For locally-run LiveKit agents: every run requires killing *all* workers, starting one fresh worker, and waiting a full 60 seconds before submitting.** Unconditional \u2014 LiveKit Cloud round-robins across registered workers, so a single survivor with a dead inference subprocess fails ~N-1 of N calls. Don\'t rely on `pkill -f <path-pattern>`; bare command lines like `node --import tsx agent.ts dev` won\'t match a path filter. Use `ps aux | grep -E "node.*agent\\.ts|@livekit/agents.*ipc"`, `kill -9` by PID, re-run `ps` to confirm zero survivors, then start the fresh worker. Skipping the 60s wait fails with `did not publish audio track`; if that error appears alongside `Error [ERR_IPC_CHANNEL_CLOSED] from InferenceProcExecutor.doInference` in the agent log right after a "running EOU detection" line, that\'s a straggler \u2014 redo the kill sweep. Hosted LiveKit Cloud agents don\'t need any of this; run normally.\n5. Pick which call(s) to run based on the change. Fixed bug: replay the failing scenario. Changed tool: include a call that triggers that tool. Prompt or routing change: include the relevant happy path and any important edge path.\n6. Compare against the previous JSON in `.vent/runs/` when validating a fix or regression. Check status flips, latency jumps, tool-call success drops, cost jumps, and transcript divergence. Correlate with `git diff` between saved `git_sha` values when available; skip if no previous run exists.\n\n## Commands\n\n```bash\nnpx vent-hq init # First-time setup (auth + skill install + starter suite)\nnpx vent-hq login # Log in to existing account\nnpx vent-hq login --status # Print whether credentials are present\nnpx vent-hq logout # Remove saved credentials from ~/.vent/credentials\nnpx vent-hq run -f .vent/suite.X.json # Run a single-call suite\nnpx vent-hq run -f .vent/suite.X.json --call NAME # Run one named call from a multi-call suite\nnpx vent-hq run ... --session <session-id> # Add to any run; routes through an existing local relay session\nnpx vent-hq run ... --verbose # Add to any run; include verbose debug fields\nnpx vent-hq stop <run-id> # Cancel a queued or running run\nnpx vent-hq agent start -f .vent/suite.X.json # Start a shared local relay session\nnpx vent-hq agent stop <session-id> # Stop a shared local relay session\n```\n\nIf `~/.vent/credentials` is missing and neither `VENT_REFRESH_TOKEN` nor `VENT_ACCESS_TOKEN` is set, run `npx vent-hq init`. For an existing account, run `npx vent-hq login` or set `VENT_REFRESH_TOKEN` to an API key. Access JWTs renew automatically; `VENT_ACCESS_TOKEN` alone is short-lived.\n\n## Suite Config\n\nSuites live in `.vent/suite.<adapter>.json`. `connection` is declared once per suite. `calls` is a named map, and each key becomes the call name used with `--call`.\n\nLocal websocket suite:\n\n```json\n{\n "connection": {\n "adapter": "websocket",\n "start_command": "npm run start",\n "health_endpoint": "/health",\n "agent_port": 3001\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nPlatform-direct suite:\n\n```json\n{\n "connection": {\n "adapter": "vapi",\n "platform": { "provider": "vapi" }\n },\n "calls": {\n "happy-path": {\n "caller_prompt": "You are Maria calling to reschedule her appointment to next Tuesday.",\n "max_turns": 8\n }\n }\n}\n```\n\nWrite `caller_prompt` as a realistic caller with a name, goal, mood, constraints, and conditional behavior. Set `max_turns` based on flow complexity: FAQ `4-6`, booking or tool use `8-12`, complex flows `12-20`.\n\nCall fields:\n\n- `caller_prompt` and `max_turns` are required.\n- `persona` supports `pace`, `clarity`, `disfluencies`, `cooperation`, `emotion`, `memory`, `intent_clarity`, and `confirmation_style`.\n- `caller_audio` supports noise, speed, speakerphone, mic distance, clarity, accent, packet loss, and jitter.\n- `caller_audio.accent` picks the caller\'s voice. Supported: `american` (default), `british`, `australian`, `filipino`, `spanish_mexican`, `spanish_peninsular`, `spanish_colombian`, `spanish_argentine`, `german`, `french`, `italian`, `dutch`, `japanese`. Any other value is rejected. The non-English accents are voices in that language: also set `language` (e.g. `"es"`) so the caller speaks it.\n- `language` is an ISO 639-1 code such as `en`, `es`, `fr`, `de`, `it`, `nl`, or `ja`.\n- `voice` is `"male"` or `"female"` (English only; default female). Use to flip the caller\'s perceived gender. Ignored if `caller_audio.accent` is set or `language` is non-English.\n\n## Connections and Credentials\n\n### Adapter choice\n\nUse `websocket` for your own local or hosted runtime. Use `start_command` for local agents or `agent_url` for hosted custom endpoints. For `start_command` and `agent_url`, do not put Deepgram, ElevenLabs, OpenAI, or other agent runtime keys into Vent config unless the Vent adapter itself needs them \u2014 the tested agent owns its own runtime credentials.\n\nUse `vapi`, `retell`, `elevenlabs`, `bland`, or `livekit` for platform-direct testing. In this mode Vent itself talks to the provider on the user\'s behalf.\n\nVent provides `DEEPGRAM_API_KEY` and `ANTHROPIC_API_KEY` for its hosted caller/evaluation stack \u2014 those are Vent\'s, not the tested agent\'s.\n\n### Credential resolution\n\nIn platform-direct mode the CLI auto-resolves credentials from `.env.local`, `.env`, and the current shell environment. Do not run `source .env && export` before Vent commands. Keep credentials out of chat and suite files; the CLI sends the resolved configuration to Vent, which stores an encrypted copy with each run.\n\nAuto-resolved env vars and JSON fields:\n\n- Vapi: `VAPI_API_KEY` -> `vapi_api_key`; `VAPI_ASSISTANT_ID` or `VAPI_AGENT_ID` -> `vapi_assistant_id`\n- Bland: `BLAND_API_KEY` -> `bland_api_key`; `BLAND_PATHWAY_ID` -> `bland_pathway_id`; `BLAND_PERSONA_ID` -> `persona_id`\n- LiveKit: `LIVEKIT_API_KEY` -> `livekit_api_key`; `LIVEKIT_API_SECRET` -> `livekit_api_secret`; `LIVEKIT_URL` -> `livekit_url`\n- Retell: `RETELL_API_KEY` -> `retell_api_key`; `RETELL_AGENT_ID` -> `retell_agent_id`\n- ElevenLabs: `ELEVENLABS_API_KEY` -> `elevenlabs_api_key`; `ELEVENLABS_AGENT_ID` -> `elevenlabs_agent_id`\n\n### Provider config\n\nUse existing provider config when possible: Vapi assistant, Retell agent, ElevenLabs agent, Bland pathway, or LiveKit agent. Bland uniquely supports inline config \u2014 `platform` may use `bland_pathway_id`, `persona_id`, or an inline `task` (with optional voice, model, and turn-handling overrides; see Bland\'s API docs for the full field list).\n\n### Concurrency\n\nWhen you fan out multiple Vent calls in parallel against the same provider (for example, running several named calls from one suite at once), respect the provider\'s per-account concurrency limit. Exceeding it makes calls queue or fail at the provider \u2014 Vent does not enforce these caps for you.\n\nRecord the limit as `max_concurrency` in the suite\'s `platform` block so it\'s visible on future runs. Ask the user which plan they\'re on if sizing matters; otherwise use the conservative default in bold.\n\n- **Vapi**: **10** included per account; reserved lines can be purchased self-serve; Enterprise is unlimited.\n- **Retell**: Pay-as-you-go includes **20**; Enterprise has no cap.\n- **Bland**: Start=**10**, Build=50, Scale=100, Enterprise=unlimited.\n- **ElevenLabs**: Free=**4**, Starter=6, Creator=10, Pro=20, Scale=30, Business=30. Burst pricing can temporarily allow up to 3x base.\n- **LiveKit Cloud**: Build=**5**, Ship=20, Scale=50 managed inference sessions (the usual gate for voice agents); agent-session concurrency can go higher (Scale up to 600).\n\n## WebSocket\n\nFor `adapter: "websocket"`, Vent sends binary 16-bit mono PCM audio over one websocket connection. Websocket text frames are optional JSON events. Audio-only websocket agents still work, but events improve turn detection and observability. Vent sends `{"type":"end-call"}` when the test is done.\n\nUseful websocket text frames:\n\n```jsonc\n{"type":"speech-update","status":"started"}\n{"type":"speech-update","status":"stopped"}\n{"type":"tool_call","name":"check_availability","arguments":{},"result":{},"successful":true,"duration_ms":150}\n{"type":"vent:timing","stt_ms":120,"llm_ms":450,"tts_ms":80}\n{"type":"vent:session","platform":"custom","provider_call_id":"call_123","provider_session_id":"session_abc"}\n{"type":"vent:call-metadata","call_metadata":{"recording_url":"https://...","cost_usd":0.12}}\n{"type":"vent:transcript","role":"caller","text":"I need to reschedule","turn_index":0}\n{"type":"vent:transfer","destination":"+15551234567","status":"attempted"}\n{"type":"vent:debug-url","label":"trace","url":"https://..."}\n{"type":"vent:warning","message":"provider warning","code":"provider_warning"}\n```\n\n`vent:session-report` is **not** handled by the websocket adapter \u2014 it\'s only consumed by the LiveKit helper. Do not emit it from a websocket agent.\n\nPlatform adapters capture tool calls automatically. Websocket agents must emit `tool_call` frames for tool observability. Platform adapters get component latency automatically. Websocket agents should emit `vent:timing` after each agent response when STT/LLM/TTS breakdown is available.\n\n## LiveKit\n\nBefore running LiveKit tests, install and add the Vent helper to the LiveKit agent entrypoint. Node: `npm install @vent-hq/livekit`, then call `instrumentLiveKitAgent({ ctx, session })`. Python: `pip install vent-livekit`, then call `instrument_livekit_agent(ctx=ctx, session=session)`.\n\nLiveKit direct mode requires the LiveKit Agents SDK. Custom LiveKit participants should use the websocket adapter with a relay. If the LiveKit agent registered with an explicit dispatch name, set `livekit_agent_name` in `platform`.\n\nLiveKit parallel calls are supported, but capacity depends on where the user runs their agent. LiveKit Cloud only routes audio \u2014 the agent code runs on the user\'s own infrastructure (laptop, Fly.io, Railway, k8s, etc.), and that box\'s CPU/RAM is the real bottleneck. Before firing parallel calls, derive `platform.max_concurrency` from the minimum of: (1) the user\'s LiveKit plan limit (Build=5, Ship=20, Scale=50+ concurrent agent sessions), (2) their agent worker box capacity (rough rule: 1c/1GB\u21921\u20133 jobs, 2c/4GB\u21925\u201310, 4c/8GB\u219210\u201325, 8c/16GB\u219225\u201350; multiply by number of worker boxes), and (3) their STT/TTS provider concurrency (e.g. Deepgram TTS streaming = 2). If the user wants more parallel calls than that minimum allows, push back \u2014 explain which limit is the bottleneck (plan / box / provider) and either reduce the count or suggest scaling that limit. Architecture: register one `agent_name` and run N workers under it (one big worker, or several smaller ones \u2014 same name); LiveKit\'s dispatcher round-robins jobs across them with built-in failover. Don\'t use numbered names (`agent-0`, `agent-1`) \u2014 that reinvents what the dispatcher does and breaks failover.\n\nUse the LiveKit helper for observability; do not publish `vent:*` topics manually. Do not hand-roll `vent:session-report` from `ctx.addShutdownCallback`; after `room.disconnect()` it can fail with `engine is closed`. The helper captures SDK metrics, tool events, conversation items, usage, and close events. Native LiveKit `lk.transcription` and `lk.agent.state` provide transcript and agent-state timing.\n\n## Output\n\n### Live result\n\n`npx vent-hq run` returns a single JSON result on stdout in non-TTY mode (not an SSE JSONL stream). Exit codes: `0` = the run passed (every call ran end to end); `1` = it didn\'t (a call errored, or the run failed or was cancelled); `2` = the result stream was lost. A call `error` starting with `Vent failed, agent not tested:` is a failure of Vent itself (its caller LLM, STT/TTS or worker): rerun, it says nothing about the agent. Any other `error` comes from the agent\'s platform or your configuration. `end_note` says how a completed call ended when Vent ended it or the connection closed (e.g. `No agent speech for 30s after the caller\'s turn; Vent hung up.`): read the transcript to judge whether that\'s an agent bug.\n\nMost result fields are always present; `latency`, `component_latency`, `call_metadata`, and `emotion` may be `null` when the underlying analysis didn\'t run; `debug` is absent without `--verbose`. Branch on null before reading nested fields. Use `--verbose` only when the default doesn\'t explain a failure \u2014 when you need `platform_transcript` (to check Vent\'s STT), per-turn or component-level latency breakdowns, the raw tool-call timeline, or provider-native artifacts in `debug.provider_metadata`. Otherwise skip \u2014 it just adds noise.\n\nVent\'s transcript is ground truth. Judge on semantic intent: ignore homophones and minor mis-hears (`"check teach hat"` for `"check that"`, missing question marks on short tails) \u2014 those are streaming-STT noise on Vent\'s caller side, not agent bugs, and **don\'t surface them in the report** (they\'re Vent-side artifacts, not actionable for the user). But clear gibberish or word-soup (e.g. `"Cristoxin"` where the agent should have said `"Of course, talk soon"`) is **not** a Vent artifact \u2014 Vent\'s STT does not invent words like that. It means the platform\'s TTS produced corrupted audio or the agent\'s STT/LLM generated the wrong text, and the fix lives there (TTS voice config, agent prompt, model temperature, codec issue). Missing or cut-off agent text, unlike word-soup, can come from Vent\'s STT stream dropping: check the recording before blaming the agent.\n\nFor transfers: `call_metadata.transfer_attempted` (provider claimed) and `transfer_completed` (from the platform\'s call data) can disagree \u2014 report both. `transfers[]` carries destination, type, and per-attempt status.\n\n### Saved history\n\nAfter every run, Vent writes the full result JSON to `.vent/runs/`. Shape:\n\n```jsonc\n{\n "run_id": "...",\n "timestamp": "2026-04-21T...Z",\n "git_sha": "...",\n "summary": { "calls_total": 2, "total_duration_ms": 12345, "total_cost_usd": 0.01 },\n "call_results": [\n { "name": "happy-path", "status": "completed", "duration_ms": 6123, "transcript": [], "observed_tool_calls": [], "latency": { "response_time_ms": 420, "p95_response_time_ms": 980 }, "call_metadata": { "cost_usd": 0.004 } }\n ]\n}\n```\n\nWhen comparing against a prior run (Workflow step 6), inspect:\n\n- Run-completion status flips: `call_results[i].status` (pipeline-only \u2014 judge mission success from the transcript)\n- Latency: `call_results[i].latency.response_time_ms` (mean) or `latency.p95_response_time_ms` increased >20%\n- Tool calls: count of `call_results[i].observed_tool_calls[].successful` dropped\n- Cost: `summary.total_cost_usd` or `call_results[i].call_metadata.cost_usd` increased >30%\n- Transcript: `call_results[i].transcript` diverged in semantic content (ignore STT noise)\n\n## Reporting Results\n\nBefore reporting, read the agent\'s code to locate where the observed behavior originates. If the issue is small and you can fix it, fix it and explain what you did \u2014 don\'t ask permission first.\n\nAdapt the report shape to the call \u2014 a clean pass needs little, a regression with a multi-layer cause needs more. Use a transcript excerpt when it helps the user see what happened.\n\nHard rules:\n\n- Pair raw numbers with their plain-English meaning \u2014 don\'t drop the number, but don\'t leave it unexplained. E.g. "p95 latency was 850ms, which is snappy and well within natural conversational pacing" or "p95 hit 1.6 seconds with the LLM as the bottleneck \u2014 noticeably sluggish to a caller."\n- Name the user\'s voice agent by platform on first mention (e.g. "the Vapi agent responded snappily throughout") so the user knows immediately which agent the observation is about. After that, just say "the agent" \u2014 don\'t repeat the platform name on every line.\n- Always include the recording from `call_metadata.recording_url` as an inline `[Recording](url)` link, placed in **one block at the very end of the report** \u2014 never sprinkled through the prose. Single call: one link as the last line. Multi-call: one labeled link per call (e.g. `reschedule-appointment: [Recording](url)`). Never paste a bare URL.\n- Mission success is your judgment, not Vent\'s. The per-call `status` is only `"completed"` (pipeline ran) or `"error"` (pipeline failed); decide whether the agent actually accomplished the scenario from the transcript and tool calls.\n- Similar-sounding word substitutions (e.g. "ocean" for "OSHA") are STT ambiguity, not comprehension failure. The fix lives in STT keyword hints, custom vocabulary, or a prompt-level term list \u2014 not the agent\'s reasoning.\n- Surface only what the user can act on in their own agent\'s code or config \u2014 never Vent-side artifacts (caller wait modes, harness timing, internal pipeline quirks), or `cost_usd` unless asked.\n\nFor multi-call runs, lead with your own judgment of what happened across the calls (e.g. "3 of 4 did what they were supposed to; `cancel-appointment` never actually canceled"), not a parroted pass/fail count. Then cover each call with whatever depth it needs.\n';
|
|
5874
5917
|
|
|
5875
5918
|
// src/lib/setup.ts
|
|
5876
5919
|
var SUITE_SCAFFOLD = JSON.stringify(
|
|
@@ -5964,9 +6007,10 @@ async function initCommand() {
|
|
|
5964
6007
|
body: JSON.stringify({ github_token: ghToken })
|
|
5965
6008
|
});
|
|
5966
6009
|
if (res.ok) {
|
|
5967
|
-
const
|
|
5968
|
-
|
|
5969
|
-
|
|
6010
|
+
const data = await res.json();
|
|
6011
|
+
const credentials = AccessCredentialsSchema.parse(data);
|
|
6012
|
+
const username = data.username;
|
|
6013
|
+
await saveCredentials(credentials);
|
|
5970
6014
|
printSuccess(`Authenticated as @${username} (via GitHub).`, {
|
|
5971
6015
|
force: true
|
|
5972
6016
|
});
|
|
@@ -6008,7 +6052,10 @@ async function stopCommand(args) {
|
|
|
6008
6052
|
method: "POST"
|
|
6009
6053
|
});
|
|
6010
6054
|
const data = await res.json();
|
|
6011
|
-
printSuccess(
|
|
6055
|
+
printSuccess(
|
|
6056
|
+
data.status === "cancelling" ? `Run ${data.id} is stopping \u2014 the call hangs up at the end of the current turn.` : `Run ${data.id} cancelled.`,
|
|
6057
|
+
{ force: true }
|
|
6058
|
+
);
|
|
6012
6059
|
} catch (err) {
|
|
6013
6060
|
if (err instanceof ApiError) {
|
|
6014
6061
|
const body = err.body;
|
|
@@ -6089,7 +6136,7 @@ async function main() {
|
|
|
6089
6136
|
return 0;
|
|
6090
6137
|
}
|
|
6091
6138
|
if (command === "--version" || command === "-v") {
|
|
6092
|
-
const pkg = await import("./package-
|
|
6139
|
+
const pkg = await import("./package-ZEOXQ7DY.mjs");
|
|
6093
6140
|
console.log(`vent-hq ${pkg.default.version}`);
|
|
6094
6141
|
return 0;
|
|
6095
6142
|
}
|