pi-harness-runtime 1.1.385 → 1.1.387
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.ts +42 -24
- package/package.json +3 -2
- package/packages/a2a-adapter/package.json +1 -1
- package/packages/agent-policy/package.json +1 -1
- package/packages/architecture-generator/package.json +1 -1
- package/packages/auth/package.json +1 -1
- package/packages/autonomous-refactor/package.json +1 -1
- package/packages/autonomous-runtime/package.json +1 -1
- package/packages/cache-strategy/package.json +1 -1
- package/packages/capability-registry/package.json +1 -1
- package/packages/checkpoint/package.json +1 -1
- package/packages/cli-plugin-sdk/package.json +1 -1
- package/packages/clipboard/package.json +1 -1
- package/packages/clipboard-plugin/package.json +1 -1
- package/packages/code-generation/package.json +1 -1
- package/packages/code-review/package.json +1 -1
- package/packages/codex-adapter/package.json +1 -1
- package/packages/codex-watcher/package.json +1 -1
- package/packages/config-capture/package.json +1 -1
- package/packages/connection-runtime/package.json +1 -1
- package/packages/context-compiler/package.json +1 -1
- package/packages/context-discovery/package.json +1 -1
- package/packages/context-manager/package.json +1 -1
- package/packages/contracts/package.json +1 -1
- package/packages/cookie-sanitizer/package.json +1 -1
- package/packages/cost-optimizer/package.json +1 -1
- package/packages/dependency-analyzer/package.json +1 -1
- package/packages/django-plugin/package.json +1 -1
- package/packages/doc-generator/package.json +1 -1
- package/packages/durable-review/package.json +1 -1
- package/packages/e2e-tester/package.json +1 -1
- package/packages/evaluation-engine/package.json +1 -1
- package/packages/evaluation-runner/package.json +1 -1
- package/packages/event-bus/package.json +1 -1
- package/packages/event-sink/package.json +1 -1
- package/packages/event-store/package.json +1 -1
- package/packages/experience-replay/package.json +1 -1
- package/packages/feedback-collector/package.json +1 -1
- package/packages/file-copy-helper/package.json +1 -1
- package/packages/framework-detector/package.json +1 -1
- package/packages/framework-plugin-sdk/package.json +1 -1
- package/packages/frappe-plugin/package.json +1 -1
- package/packages/generic-web-plugin/package.json +1 -1
- package/packages/health-monitor/package.json +1 -1
- package/packages/intent-analyzer/package.json +1 -1
- package/packages/jev-judge/dist/auto-continue.d.ts.map +1 -1
- package/packages/jev-judge/dist/auto-continue.js +26 -0
- package/packages/jev-judge/dist/auto-continue.js.map +1 -1
- package/packages/jev-judge/dist/environment-judge.d.ts.map +1 -1
- package/packages/jev-judge/dist/environment-judge.js +7 -4
- package/packages/jev-judge/dist/environment-judge.js.map +1 -1
- package/packages/jev-judge/dist/index.d.ts +87 -12
- package/packages/jev-judge/dist/index.d.ts.map +1 -1
- package/packages/jev-judge/dist/index.js +291 -75
- package/packages/jev-judge/dist/index.js.map +1 -1
- package/packages/jev-judge/dist/provider-error.d.ts +61 -0
- package/packages/jev-judge/dist/provider-error.d.ts.map +1 -0
- package/packages/jev-judge/dist/provider-error.js +165 -0
- package/packages/jev-judge/dist/provider-error.js.map +1 -0
- package/packages/jev-judge/dist/task-intake.d.ts.map +1 -1
- package/packages/jev-judge/dist/task-intake.js +3 -2
- package/packages/jev-judge/dist/task-intake.js.map +1 -1
- package/packages/jev-judge/dist/types.d.ts +56 -2
- package/packages/jev-judge/dist/types.d.ts.map +1 -1
- package/packages/jev-judge/dist/typesafe-provider.d.ts +26 -1
- package/packages/jev-judge/dist/typesafe-provider.d.ts.map +1 -1
- package/packages/jev-judge/dist/typesafe-provider.js +153 -62
- package/packages/jev-judge/dist/typesafe-provider.js.map +1 -1
- package/packages/jev-judge/dist/typesafe-request.d.ts +65 -0
- package/packages/jev-judge/dist/typesafe-request.d.ts.map +1 -0
- package/packages/jev-judge/dist/typesafe-request.js +138 -0
- package/packages/jev-judge/dist/typesafe-request.js.map +1 -0
- package/packages/jev-judge/package.json +1 -1
- package/packages/jev-judge/src/auto-continue.ts +26 -0
- package/packages/jev-judge/src/environment-judge.ts +7 -4
- package/packages/jev-judge/src/index.ts +617 -333
- package/packages/jev-judge/src/provider-error.ts +190 -0
- package/packages/jev-judge/src/task-intake.ts +3 -2
- package/packages/jev-judge/src/types.ts +62 -5
- package/packages/jev-judge/src/typesafe-provider.ts +452 -318
- package/packages/jev-judge/src/typesafe-request.ts +181 -0
- package/packages/jev-judge/tests/jev-provider-chain.test.ts +536 -0
- package/packages/jev-judge/tests/typesafe-provider.test.ts +504 -44
- package/packages/knowledge-graph/package.json +1 -1
- package/packages/knowledge-retrieval/package.json +1 -1
- package/packages/laravel-plugin/package.json +1 -1
- package/packages/learning-engine/package.json +1 -1
- package/packages/mcp-adapter/package.json +1 -1
- package/packages/memory-engine/package.json +1 -1
- package/packages/milestone-manager/package.json +1 -1
- package/packages/model-registry/package.json +1 -1
- package/packages/nextjs-plugin/package.json +1 -1
- package/packages/notification/package.json +1 -1
- package/packages/observability/package.json +1 -1
- package/packages/okf-indexer/package.json +1 -1
- package/packages/paperclip-integration/package.json +1 -1
- package/packages/performance-optimizer/package.json +1 -1
- package/packages/permission-mode/package.json +1 -1
- package/packages/privilege-broker/package.json +1 -1
- package/packages/project-analyzer/package.json +1 -1
- package/packages/project-bootstrap/package.json +1 -1
- package/packages/projection-engine/package.json +1 -1
- package/packages/prompt-compiler/package.json +1 -1
- package/packages/prompt-versioning/package.json +1 -1
- package/packages/provider-adapter-sdk/package.json +1 -1
- package/packages/provider-router/package.json +1 -1
- package/packages/provider-selector/package.json +1 -1
- package/packages/providers/package.json +1 -1
- package/packages/qdrant-skills/package.json +1 -1
- package/packages/quota-manager/package.json +1 -1
- package/packages/rate-limiter/package.json +1 -1
- package/packages/react-vite-plugin/package.json +1 -1
- package/packages/relay/package.json +1 -1
- package/packages/release-manager/package.json +1 -1
- package/packages/requirement-compiler/package.json +1 -1
- package/packages/runtime/package.json +1 -1
- package/packages/scheduler/package.json +1 -1
- package/packages/scheduler-adapter/package.json +1 -1
- package/packages/session/package.json +1 -1
- package/packages/session-api/package.json +1 -1
- package/packages/session-export/package.json +1 -1
- package/packages/shared-context/package.json +1 -1
- package/packages/skill-mcp-client/package.json +1 -1
- package/packages/skill-registry/package.json +1 -1
- package/packages/skills/package.json +1 -1
- package/packages/sprint-planner/package.json +1 -1
- package/packages/subscription-engine/package.json +1 -1
- package/packages/task-compiler/package.json +1 -1
- package/packages/telegram/package.json +1 -1
- package/packages/tencentdb-memory/package.json +1 -1
- package/packages/tencentdb-sync/package.json +1 -1
- package/packages/test-data-generator/package.json +1 -1
- package/packages/test-generator/package.json +1 -1
- package/packages/todo-bd-sync/package.json +1 -1
- package/packages/token-estimation/package.json +1 -1
- package/packages/token-optimizer/package.json +1 -1
- package/packages/trajectory/package.json +1 -1
- package/packages/tui/package.json +1 -1
- package/packages/types/package.json +1 -1
- package/packages/voice-api/package.json +1 -1
- package/packages/webmcp-tester/package.json +1 -1
- package/packages/wiki/package.json +1 -1
- package/packages/workflow-events/package.json +1 -1
- package/packages/workspace-scanner/package.json +1 -1
- package/packages/worktree/package.json +1 -1
- package/packages/write-review/package.json +1 -1
- package/scripts/run-root-tests.mjs +207 -0
|
@@ -6,84 +6,102 @@
|
|
|
6
6
|
*
|
|
7
7
|
* API: POST https://api.typesafe.ai/v1/systemone
|
|
8
8
|
* Auth: Bearer token (TYPESAFE_API_KEY)
|
|
9
|
+
*
|
|
10
|
+
* The request contract (choice criteria as a dictionary, score questions with
|
|
11
|
+
* a mandatory rubric) is implemented in ./typesafe-request.ts; the mapping was
|
|
12
|
+
* verified against the live endpoint.
|
|
9
13
|
*/
|
|
10
14
|
|
|
11
15
|
import type {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
JevQuestion,
|
|
17
|
+
JevQuestions,
|
|
18
|
+
JevResponse,
|
|
19
|
+
JevDecision,
|
|
20
|
+
JevEvaluationResult,
|
|
21
|
+
JevNoulResponse,
|
|
22
|
+
JevChoiceResponse,
|
|
23
|
+
JevScoreResponse,
|
|
20
24
|
} from "./types.js";
|
|
25
|
+
import {
|
|
26
|
+
buildChoiceCriteria,
|
|
27
|
+
buildScoreScale,
|
|
28
|
+
scorePositionToProbability,
|
|
29
|
+
scorePositionToValue,
|
|
30
|
+
validateQuestion,
|
|
31
|
+
type ScoreScale,
|
|
32
|
+
} from "./typesafe-request.js";
|
|
33
|
+
import { JevProviderError, isRetryableStatus, isTransportError } from "./provider-error.js";
|
|
21
34
|
|
|
22
35
|
/**
|
|
23
36
|
* TypeSafe API request format
|
|
24
37
|
*/
|
|
25
38
|
interface TypeSafeRequest {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
39
|
+
/** The model that handles the request. Required. */
|
|
40
|
+
model: string;
|
|
41
|
+
/** The content to evaluate. */
|
|
42
|
+
state: string | object | unknown[];
|
|
43
|
+
/** A map of typed questions. Keys are question IDs. */
|
|
44
|
+
questions: Record<string, TypeSafeQuestion>;
|
|
32
45
|
}
|
|
33
46
|
|
|
34
47
|
/**
|
|
35
48
|
* TypeSafe question format (per API spec, id not needed in question object - it's the map key)
|
|
36
49
|
*/
|
|
37
50
|
interface TypeSafeQuestion {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
51
|
+
type: "noul" | "choice" | "score";
|
|
52
|
+
instructions: string;
|
|
53
|
+
/**
|
|
54
|
+
* Choice questions: a dictionary of `name -> description`.
|
|
55
|
+
* Score questions: an ordered list of level descriptions.
|
|
56
|
+
*/
|
|
57
|
+
criteria?: Record<string, string> | string[];
|
|
44
58
|
}
|
|
45
59
|
|
|
46
60
|
/**
|
|
47
61
|
* TypeSafe API response format
|
|
48
62
|
*/
|
|
49
63
|
interface TypeSafeResponse {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
64
|
+
answers: Record<string, TypeSafeAnswer>;
|
|
65
|
+
model?: string;
|
|
66
|
+
latency_ms?: number;
|
|
53
67
|
}
|
|
54
68
|
|
|
55
69
|
/**
|
|
56
70
|
* TypeSafe answer format
|
|
57
71
|
*/
|
|
58
72
|
interface TypeSafeAnswer {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
73
|
+
type: "noul" | "choice" | "score";
|
|
74
|
+
/** For noul type: probability of yes (0-1). */
|
|
75
|
+
noul?: number;
|
|
76
|
+
/** For choice type: the selected option. */
|
|
77
|
+
choice?: string;
|
|
78
|
+
/** For score type: the probability-weighted position in the rubric. */
|
|
79
|
+
score?: number;
|
|
80
|
+
/** For choice/score: confidence 0-1. */
|
|
81
|
+
confidence?: number;
|
|
82
|
+
/** For choice type: the probability of every choice. */
|
|
83
|
+
probabilities?: Record<string, number>;
|
|
84
|
+
/** For score type: the rubric levels mapped to their level index. */
|
|
85
|
+
legend?: Record<string, string>;
|
|
68
86
|
}
|
|
69
87
|
|
|
70
88
|
/**
|
|
71
89
|
* TypeSafe provider configuration
|
|
72
90
|
*/
|
|
73
91
|
export interface TypeSafeConfig {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
92
|
+
apiKey: string;
|
|
93
|
+
model?: string; // Defaults to jev-latest
|
|
94
|
+
baseUrl?: string; // Defaults to https://api.typesafe.ai
|
|
95
|
+
timeoutMs?: number; // Defaults to 30000
|
|
78
96
|
}
|
|
79
97
|
|
|
80
98
|
/**
|
|
81
99
|
* Default configuration
|
|
82
100
|
*/
|
|
83
101
|
const DEFAULT_CONFIG = {
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
102
|
+
model: "jev-latest",
|
|
103
|
+
baseUrl: "https://api.typesafe.ai",
|
|
104
|
+
timeoutMs: 30000,
|
|
87
105
|
};
|
|
88
106
|
|
|
89
107
|
/**
|
|
@@ -93,290 +111,406 @@ const DEFAULT_CONFIG = {
|
|
|
93
111
|
* More reliable than OpenRouter proxy and supports full feature set.
|
|
94
112
|
*/
|
|
95
113
|
export class TypeSafeJudge {
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
114
|
+
private config: Required<TypeSafeConfig>;
|
|
115
|
+
|
|
116
|
+
constructor(config: TypeSafeConfig) {
|
|
117
|
+
this.config = {
|
|
118
|
+
...DEFAULT_CONFIG,
|
|
119
|
+
...config,
|
|
120
|
+
} as Required<TypeSafeConfig>;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Evaluate state against questions using TypeSafe API
|
|
125
|
+
*/
|
|
126
|
+
async evaluate<T extends string>(
|
|
127
|
+
state: string | object,
|
|
128
|
+
questions: JevQuestions<T>
|
|
129
|
+
): Promise<JevEvaluationResult<T>> {
|
|
130
|
+
const startTime = Date.now();
|
|
131
|
+
|
|
132
|
+
// Convert state to string if object
|
|
133
|
+
const stateStr = typeof state === "string" ? state : JSON.stringify(state);
|
|
134
|
+
|
|
135
|
+
// Build and validate the request. One invalid question fails the whole
|
|
136
|
+
// request on the API side, so contract problems are caught locally.
|
|
137
|
+
const { request, scoreScales } = this.buildRequest(stateStr, questions);
|
|
138
|
+
|
|
139
|
+
// Call TypeSafe API directly
|
|
140
|
+
let response: Response;
|
|
141
|
+
try {
|
|
142
|
+
response = await fetch(`${this.config.baseUrl}/v1/systemone`, {
|
|
143
|
+
method: "POST",
|
|
144
|
+
headers: {
|
|
145
|
+
"Content-Type": "application/json",
|
|
146
|
+
Authorization: `Bearer ${this.config.apiKey}`,
|
|
147
|
+
},
|
|
148
|
+
// Only the wire format is sent: the API rejects unknown fields.
|
|
149
|
+
body: JSON.stringify(request),
|
|
150
|
+
signal: AbortSignal.timeout(this.config.timeoutMs),
|
|
151
|
+
});
|
|
152
|
+
} catch (error) {
|
|
153
|
+
// A network failure, a timeout or an abort never reached a server
|
|
154
|
+
// that rejected the request: another provider can still serve it.
|
|
155
|
+
// Anything else (a malformed URL, a bug in our own code) is NOT
|
|
156
|
+
// retryable - it must not disappear behind a fallback.
|
|
157
|
+
throw new JevProviderError(
|
|
158
|
+
`TypeSafe request failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
159
|
+
{ provider: "typesafe", retryable: isTransportError(error), cause: error }
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const latencyMs = Date.now() - startTime;
|
|
164
|
+
|
|
165
|
+
if (!response.ok) {
|
|
166
|
+
const errorText = await response.text().catch(() => "Unknown error");
|
|
167
|
+
throw new JevProviderError(
|
|
168
|
+
`TypeSafe API error ${response.status}: ${errorText}`,
|
|
169
|
+
{
|
|
170
|
+
provider: "typesafe",
|
|
171
|
+
status: response.status,
|
|
172
|
+
retryable: isRetryableStatus(response.status),
|
|
173
|
+
}
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const data = await response.json() as TypeSafeResponse;
|
|
178
|
+
|
|
179
|
+
// Build decisions map
|
|
180
|
+
const decisions = {} as Record<T, JevDecision<T>>;
|
|
181
|
+
|
|
182
|
+
for (const [questionId, question] of Object.entries(questions)) {
|
|
183
|
+
const questionData = question as JevQuestion;
|
|
184
|
+
const answer = data.answers?.[questionId] as TypeSafeAnswer | undefined;
|
|
185
|
+
|
|
186
|
+
// Reject missing required answers
|
|
187
|
+
if (!answer) {
|
|
188
|
+
throw new JevProviderError(
|
|
189
|
+
`TypeSafe API: missing answer for required question "${questionId}"`,
|
|
190
|
+
{ provider: "typesafe", retryable: false }
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const decision = this.buildDecision(
|
|
195
|
+
questionId,
|
|
196
|
+
questionData,
|
|
197
|
+
answer,
|
|
198
|
+
scoreScales[questionId]
|
|
199
|
+
);
|
|
200
|
+
(decisions as Record<string, JevDecision>)[questionId] = decision;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
return {
|
|
204
|
+
decisions,
|
|
205
|
+
state: stateStr,
|
|
206
|
+
latencyMs,
|
|
207
|
+
model: data.model ?? this.config.model,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Build the API request and validate it against the API contract.
|
|
213
|
+
*
|
|
214
|
+
* The returned `request` is the wire format; `scoreScales` is local
|
|
215
|
+
* bookkeeping and is never sent to the API (it rejects unknown fields).
|
|
216
|
+
*/
|
|
217
|
+
private buildRequest<T extends string>(
|
|
218
|
+
stateStr: string,
|
|
219
|
+
questions: JevQuestions<T>
|
|
220
|
+
): { request: TypeSafeRequest; scoreScales: Record<string, ScoreScale> } {
|
|
221
|
+
const questionsMap: Record<string, TypeSafeQuestion> = {};
|
|
222
|
+
const scoreScales: Record<string, ScoreScale> = {};
|
|
223
|
+
|
|
224
|
+
for (const [id, raw] of Object.entries(questions)) {
|
|
225
|
+
const questionObj = raw as JevQuestion;
|
|
226
|
+
const scoreScale = validateQuestion(id, questionObj);
|
|
227
|
+
|
|
228
|
+
if (questionObj.type === "choice") {
|
|
229
|
+
questionsMap[id] = {
|
|
230
|
+
type: "choice",
|
|
231
|
+
instructions: questionObj.instructions,
|
|
232
|
+
criteria: buildChoiceCriteria(id, questionObj),
|
|
233
|
+
};
|
|
234
|
+
} else if (questionObj.type === "score") {
|
|
235
|
+
const { criteria } = buildScoreScale(id, questionObj);
|
|
236
|
+
questionsMap[id] = {
|
|
237
|
+
type: "score",
|
|
238
|
+
instructions: questionObj.instructions,
|
|
239
|
+
criteria,
|
|
240
|
+
};
|
|
241
|
+
if (scoreScale) scoreScales[id] = scoreScale;
|
|
242
|
+
} else {
|
|
243
|
+
questionsMap[id] = {
|
|
244
|
+
type: "noul",
|
|
245
|
+
instructions: questionObj.instructions,
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
if (Object.keys(questionsMap).length === 0) {
|
|
251
|
+
throw new JevProviderError("TypeSafe request invalid: no questions to ask", {
|
|
252
|
+
provider: "typesafe",
|
|
253
|
+
retryable: false,
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
return {
|
|
258
|
+
request: {
|
|
259
|
+
model: this.config.model,
|
|
260
|
+
state: stateStr,
|
|
261
|
+
questions: questionsMap,
|
|
262
|
+
},
|
|
263
|
+
scoreScales,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Quick noul (yes/no) evaluation
|
|
269
|
+
*/
|
|
270
|
+
async noul(
|
|
271
|
+
state: string | object,
|
|
272
|
+
question: string
|
|
273
|
+
): Promise<{ noul: number; confidence: "high" | "medium" | "low" }> {
|
|
274
|
+
const result = await this.evaluate(state, {
|
|
275
|
+
answer: { type: "noul", instructions: question },
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
const decision = result.decisions.answer as JevDecision<"answer">;
|
|
279
|
+
const noulResponse = decision.response as JevNoulResponse;
|
|
280
|
+
return {
|
|
281
|
+
noul: noulResponse.noul,
|
|
282
|
+
confidence: decision.confidence,
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Quick choice evaluation
|
|
288
|
+
*/
|
|
289
|
+
async choice<T extends string>(
|
|
290
|
+
state: string | object,
|
|
291
|
+
question: string,
|
|
292
|
+
options: T[]
|
|
293
|
+
): Promise<{ choice: T; confidence: number }> {
|
|
294
|
+
const result = await this.evaluate(state, {
|
|
295
|
+
answer: { type: "choice", instructions: question, options },
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
const decision = result.decisions.answer as JevDecision<"answer">;
|
|
299
|
+
const choiceResponse = decision.response as JevChoiceResponse<T>;
|
|
300
|
+
return {
|
|
301
|
+
choice: choiceResponse.choice,
|
|
302
|
+
confidence: choiceResponse.confidence,
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Quick score evaluation
|
|
308
|
+
*/
|
|
309
|
+
async score(
|
|
310
|
+
state: string | object,
|
|
311
|
+
question: string,
|
|
312
|
+
min = 0,
|
|
313
|
+
max = 100
|
|
314
|
+
): Promise<{ score: number; confidence: number }> {
|
|
315
|
+
const result = await this.evaluate(state, {
|
|
316
|
+
answer: { type: "score", instructions: question, min, max },
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
const decision = result.decisions.answer as JevDecision<"answer">;
|
|
320
|
+
const scoreResponse = decision.response as JevScoreResponse;
|
|
321
|
+
return {
|
|
322
|
+
score: scoreResponse.score,
|
|
323
|
+
confidence: scoreResponse.confidence,
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Build JevDecision from TypeSafe answer
|
|
329
|
+
*/
|
|
330
|
+
private buildDecision<T extends string>(
|
|
331
|
+
questionId: T,
|
|
332
|
+
question: JevQuestion,
|
|
333
|
+
answer: TypeSafeAnswer,
|
|
334
|
+
scoreScale?: ScoreScale
|
|
335
|
+
): JevDecision<T> {
|
|
336
|
+
let response: JevResponse;
|
|
337
|
+
let probability: number;
|
|
338
|
+
|
|
339
|
+
switch (answer.type) {
|
|
340
|
+
case "noul":
|
|
341
|
+
// FIX: Validate noul value is present and in range [0, 1]
|
|
342
|
+
if (answer.noul === undefined || answer.noul === null) {
|
|
343
|
+
throw new JevProviderError(
|
|
344
|
+
`TypeSafe API: missing noul value for question "${questionId}"`,
|
|
345
|
+
{ provider: "typesafe", retryable: false }
|
|
346
|
+
);
|
|
347
|
+
}
|
|
348
|
+
if (typeof answer.noul !== "number" || isNaN(answer.noul)) {
|
|
349
|
+
throw new JevProviderError(
|
|
350
|
+
`TypeSafe API: invalid noul value for question "${questionId}": ${JSON.stringify(answer.noul)}`,
|
|
351
|
+
{ provider: "typesafe", retryable: false }
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
if (answer.noul < 0 || answer.noul > 1) {
|
|
355
|
+
throw new JevProviderError(
|
|
356
|
+
`TypeSafe API: noul value out of range [0,1] for question "${questionId}": ${answer.noul}`,
|
|
357
|
+
{ provider: "typesafe", retryable: false }
|
|
358
|
+
);
|
|
359
|
+
}
|
|
360
|
+
response = {
|
|
361
|
+
type: "noul",
|
|
362
|
+
noul: answer.noul,
|
|
363
|
+
};
|
|
364
|
+
probability = answer.noul;
|
|
365
|
+
break;
|
|
366
|
+
|
|
367
|
+
case "choice": {
|
|
368
|
+
const options = choiceCandidates(question);
|
|
369
|
+
// The model's choice MUST win. The previous expression
|
|
370
|
+
// (`answer.choice ?? question.type === "choice" ? options[0] : ""`)
|
|
371
|
+
// was parsed as `(answer.choice ?? (question.type === "choice")) ? options[0] : ""`,
|
|
372
|
+
// so a non-empty returned choice was DISCARDED and the first option was
|
|
373
|
+
// always selected.
|
|
374
|
+
const returned =
|
|
375
|
+
typeof answer.choice === "string" ? answer.choice.trim() : "";
|
|
376
|
+
let selected: string;
|
|
377
|
+
if (returned.length > 0) {
|
|
378
|
+
if (options.includes(returned)) {
|
|
379
|
+
selected = returned;
|
|
380
|
+
} else {
|
|
381
|
+
// Case-insensitive match: keep the canonical option spelling.
|
|
382
|
+
const canonical = options.find(
|
|
383
|
+
(option) => option.toLowerCase() === returned.toLowerCase(),
|
|
384
|
+
);
|
|
385
|
+
// No match: pass the model's value through unchanged instead of
|
|
386
|
+
// silently substituting a different option.
|
|
387
|
+
selected = canonical ?? returned;
|
|
388
|
+
}
|
|
389
|
+
} else {
|
|
390
|
+
// The model returned no usable choice: fall back to the first option.
|
|
391
|
+
selected = options[0] ?? "";
|
|
392
|
+
}
|
|
393
|
+
response = {
|
|
394
|
+
type: "choice",
|
|
395
|
+
choice: selected,
|
|
396
|
+
confidence: answer.confidence ?? 0.5,
|
|
397
|
+
};
|
|
398
|
+
probability = answer.confidence ?? 0.5;
|
|
399
|
+
break;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
case "score": {
|
|
403
|
+
if (answer.score === undefined || answer.score === null) {
|
|
404
|
+
throw new JevProviderError(
|
|
405
|
+
`TypeSafe API: missing score value for question "${questionId}"`,
|
|
406
|
+
{ provider: "typesafe", retryable: false }
|
|
407
|
+
);
|
|
408
|
+
}
|
|
409
|
+
if (typeof answer.score !== "number" || isNaN(answer.score)) {
|
|
410
|
+
throw new JevProviderError(
|
|
411
|
+
`TypeSafe API: invalid score value for question "${questionId}": ${JSON.stringify(answer.score)}`,
|
|
412
|
+
{ provider: "typesafe", retryable: false }
|
|
413
|
+
);
|
|
414
|
+
}
|
|
415
|
+
// The API answers with the probability-weighted POSITION in the
|
|
416
|
+
// rubric (0 .. levels-1), not with a value on the min/max scale.
|
|
417
|
+
const scale: ScoreScale = scoreScale ?? {
|
|
418
|
+
levels: 2,
|
|
419
|
+
min: question.type === "score" ? question.min ?? 0 : 0,
|
|
420
|
+
max: question.type === "score" ? question.max ?? 100 : 100,
|
|
421
|
+
};
|
|
422
|
+
const value = scorePositionToValue(answer.score, scale);
|
|
423
|
+
response = {
|
|
424
|
+
type: "score",
|
|
425
|
+
score: value,
|
|
426
|
+
confidence: answer.confidence ?? 0.5,
|
|
427
|
+
};
|
|
428
|
+
probability = scorePositionToProbability(answer.score, scale);
|
|
429
|
+
break;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
default:
|
|
433
|
+
// Fallback for unknown types
|
|
434
|
+
response = {
|
|
435
|
+
type: "noul",
|
|
436
|
+
noul: 0.5,
|
|
437
|
+
};
|
|
438
|
+
probability = 0.5;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
const confidence = this.getConfidence(answer.confidence ?? probability);
|
|
442
|
+
|
|
443
|
+
return {
|
|
444
|
+
questionId,
|
|
445
|
+
question,
|
|
446
|
+
response,
|
|
447
|
+
probability,
|
|
448
|
+
confidence,
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/**
|
|
453
|
+
* Determine confidence level from probability
|
|
454
|
+
*/
|
|
455
|
+
private getConfidence(probability: number): "high" | "medium" | "low" {
|
|
456
|
+
// Confidence is based on distance from 0.5 (uncertain)
|
|
457
|
+
const distance = Math.abs(probability - 0.5);
|
|
458
|
+
if (distance > 0.35) return "high";
|
|
459
|
+
if (distance > 0.2) return "medium";
|
|
460
|
+
return "low";
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
/**
|
|
465
|
+
* The candidate names of a choice question.
|
|
466
|
+
*
|
|
467
|
+
* `options` is the ergonomic way to name the candidates; a question that only
|
|
468
|
+
* carries `criteria` is read from the criteria itself.
|
|
469
|
+
*/
|
|
470
|
+
function choiceCandidates(
|
|
471
|
+
question: JevQuestion
|
|
472
|
+
): string[] {
|
|
473
|
+
if (question.type !== "choice") return [];
|
|
474
|
+
const options = question.options;
|
|
475
|
+
if (options && options.length > 0) return [...options];
|
|
476
|
+
const criteria = question.criteria;
|
|
477
|
+
if (Array.isArray(criteria)) return [...criteria];
|
|
478
|
+
if (criteria) return Object.keys(criteria);
|
|
479
|
+
return [];
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Determine the provider that owns a base URL.
|
|
484
|
+
*
|
|
485
|
+
* The provider is derived from the ENDPOINT, never from the shape of a key:
|
|
486
|
+
* guessing from a key is how an OpenRouter key ended up being sent to the
|
|
487
|
+
* TypeSafe API.
|
|
488
|
+
*/
|
|
489
|
+
export function providerForBaseUrl(baseUrl: string): "typesafe" | "openrouter" {
|
|
490
|
+
if (baseUrl.includes("typesafe.ai")) return "typesafe";
|
|
491
|
+
if (baseUrl.includes("openrouter")) return "openrouter";
|
|
492
|
+
throw new Error(
|
|
493
|
+
`Jev: unknown provider base URL "${baseUrl}". Use resolveJevProviders() or pass an explicit provider chain.`
|
|
494
|
+
);
|
|
358
495
|
}
|
|
359
496
|
|
|
360
497
|
/**
|
|
361
498
|
* Detect which provider to use based on API key or config
|
|
499
|
+
*
|
|
500
|
+
* @deprecated The provider cannot be derived from a key - a bare key says
|
|
501
|
+
* nothing about the endpoint it belongs to, and guessing from its length is
|
|
502
|
+
* how an OpenRouter key ended up being sent to the TypeSafe API. Pass the
|
|
503
|
+
* `baseUrl` of the provider, or use `resolveJevProviders()` (see ./index.ts),
|
|
504
|
+
* which binds every credential to its own provider.
|
|
362
505
|
*/
|
|
363
506
|
export function detectProvider(
|
|
364
|
-
|
|
365
|
-
|
|
507
|
+
_apiKey: string,
|
|
508
|
+
baseUrl?: string
|
|
366
509
|
): "typesafe" | "openrouter" {
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
return "openrouter";
|
|
374
|
-
}
|
|
375
|
-
// If key looks like TypeSafe (no specific pattern known), prefer TypeSafe
|
|
376
|
-
// TypeSafe keys are typically longer than OpenAI-compatible keys
|
|
377
|
-
if (apiKey.length > 50) {
|
|
378
|
-
return "typesafe";
|
|
379
|
-
}
|
|
380
|
-
// Default to OpenRouter for short keys (OpenAI-compatible)
|
|
381
|
-
return "openrouter";
|
|
510
|
+
if (!baseUrl) {
|
|
511
|
+
throw new Error(
|
|
512
|
+
"Jev: detectProvider() cannot guess the provider from the key. Pass the baseUrl of the provider or use resolveJevProviders()."
|
|
513
|
+
);
|
|
514
|
+
}
|
|
515
|
+
return providerForBaseUrl(baseUrl);
|
|
382
516
|
}
|