pi-harness-runtime 1.1.385 → 1.1.387

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/index.ts +42 -24
  2. package/package.json +3 -2
  3. package/packages/a2a-adapter/package.json +1 -1
  4. package/packages/agent-policy/package.json +1 -1
  5. package/packages/architecture-generator/package.json +1 -1
  6. package/packages/auth/package.json +1 -1
  7. package/packages/autonomous-refactor/package.json +1 -1
  8. package/packages/autonomous-runtime/package.json +1 -1
  9. package/packages/cache-strategy/package.json +1 -1
  10. package/packages/capability-registry/package.json +1 -1
  11. package/packages/checkpoint/package.json +1 -1
  12. package/packages/cli-plugin-sdk/package.json +1 -1
  13. package/packages/clipboard/package.json +1 -1
  14. package/packages/clipboard-plugin/package.json +1 -1
  15. package/packages/code-generation/package.json +1 -1
  16. package/packages/code-review/package.json +1 -1
  17. package/packages/codex-adapter/package.json +1 -1
  18. package/packages/codex-watcher/package.json +1 -1
  19. package/packages/config-capture/package.json +1 -1
  20. package/packages/connection-runtime/package.json +1 -1
  21. package/packages/context-compiler/package.json +1 -1
  22. package/packages/context-discovery/package.json +1 -1
  23. package/packages/context-manager/package.json +1 -1
  24. package/packages/contracts/package.json +1 -1
  25. package/packages/cookie-sanitizer/package.json +1 -1
  26. package/packages/cost-optimizer/package.json +1 -1
  27. package/packages/dependency-analyzer/package.json +1 -1
  28. package/packages/django-plugin/package.json +1 -1
  29. package/packages/doc-generator/package.json +1 -1
  30. package/packages/durable-review/package.json +1 -1
  31. package/packages/e2e-tester/package.json +1 -1
  32. package/packages/evaluation-engine/package.json +1 -1
  33. package/packages/evaluation-runner/package.json +1 -1
  34. package/packages/event-bus/package.json +1 -1
  35. package/packages/event-sink/package.json +1 -1
  36. package/packages/event-store/package.json +1 -1
  37. package/packages/experience-replay/package.json +1 -1
  38. package/packages/feedback-collector/package.json +1 -1
  39. package/packages/file-copy-helper/package.json +1 -1
  40. package/packages/framework-detector/package.json +1 -1
  41. package/packages/framework-plugin-sdk/package.json +1 -1
  42. package/packages/frappe-plugin/package.json +1 -1
  43. package/packages/generic-web-plugin/package.json +1 -1
  44. package/packages/health-monitor/package.json +1 -1
  45. package/packages/intent-analyzer/package.json +1 -1
  46. package/packages/jev-judge/dist/auto-continue.d.ts.map +1 -1
  47. package/packages/jev-judge/dist/auto-continue.js +26 -0
  48. package/packages/jev-judge/dist/auto-continue.js.map +1 -1
  49. package/packages/jev-judge/dist/environment-judge.d.ts.map +1 -1
  50. package/packages/jev-judge/dist/environment-judge.js +7 -4
  51. package/packages/jev-judge/dist/environment-judge.js.map +1 -1
  52. package/packages/jev-judge/dist/index.d.ts +87 -12
  53. package/packages/jev-judge/dist/index.d.ts.map +1 -1
  54. package/packages/jev-judge/dist/index.js +291 -75
  55. package/packages/jev-judge/dist/index.js.map +1 -1
  56. package/packages/jev-judge/dist/provider-error.d.ts +61 -0
  57. package/packages/jev-judge/dist/provider-error.d.ts.map +1 -0
  58. package/packages/jev-judge/dist/provider-error.js +165 -0
  59. package/packages/jev-judge/dist/provider-error.js.map +1 -0
  60. package/packages/jev-judge/dist/task-intake.d.ts.map +1 -1
  61. package/packages/jev-judge/dist/task-intake.js +3 -2
  62. package/packages/jev-judge/dist/task-intake.js.map +1 -1
  63. package/packages/jev-judge/dist/types.d.ts +56 -2
  64. package/packages/jev-judge/dist/types.d.ts.map +1 -1
  65. package/packages/jev-judge/dist/typesafe-provider.d.ts +26 -1
  66. package/packages/jev-judge/dist/typesafe-provider.d.ts.map +1 -1
  67. package/packages/jev-judge/dist/typesafe-provider.js +153 -62
  68. package/packages/jev-judge/dist/typesafe-provider.js.map +1 -1
  69. package/packages/jev-judge/dist/typesafe-request.d.ts +65 -0
  70. package/packages/jev-judge/dist/typesafe-request.d.ts.map +1 -0
  71. package/packages/jev-judge/dist/typesafe-request.js +138 -0
  72. package/packages/jev-judge/dist/typesafe-request.js.map +1 -0
  73. package/packages/jev-judge/package.json +1 -1
  74. package/packages/jev-judge/src/auto-continue.ts +26 -0
  75. package/packages/jev-judge/src/environment-judge.ts +7 -4
  76. package/packages/jev-judge/src/index.ts +617 -333
  77. package/packages/jev-judge/src/provider-error.ts +190 -0
  78. package/packages/jev-judge/src/task-intake.ts +3 -2
  79. package/packages/jev-judge/src/types.ts +62 -5
  80. package/packages/jev-judge/src/typesafe-provider.ts +452 -318
  81. package/packages/jev-judge/src/typesafe-request.ts +181 -0
  82. package/packages/jev-judge/tests/jev-provider-chain.test.ts +536 -0
  83. package/packages/jev-judge/tests/typesafe-provider.test.ts +504 -44
  84. package/packages/knowledge-graph/package.json +1 -1
  85. package/packages/knowledge-retrieval/package.json +1 -1
  86. package/packages/laravel-plugin/package.json +1 -1
  87. package/packages/learning-engine/package.json +1 -1
  88. package/packages/mcp-adapter/package.json +1 -1
  89. package/packages/memory-engine/package.json +1 -1
  90. package/packages/milestone-manager/package.json +1 -1
  91. package/packages/model-registry/package.json +1 -1
  92. package/packages/nextjs-plugin/package.json +1 -1
  93. package/packages/notification/package.json +1 -1
  94. package/packages/observability/package.json +1 -1
  95. package/packages/okf-indexer/package.json +1 -1
  96. package/packages/paperclip-integration/package.json +1 -1
  97. package/packages/performance-optimizer/package.json +1 -1
  98. package/packages/permission-mode/package.json +1 -1
  99. package/packages/privilege-broker/package.json +1 -1
  100. package/packages/project-analyzer/package.json +1 -1
  101. package/packages/project-bootstrap/package.json +1 -1
  102. package/packages/projection-engine/package.json +1 -1
  103. package/packages/prompt-compiler/package.json +1 -1
  104. package/packages/prompt-versioning/package.json +1 -1
  105. package/packages/provider-adapter-sdk/package.json +1 -1
  106. package/packages/provider-router/package.json +1 -1
  107. package/packages/provider-selector/package.json +1 -1
  108. package/packages/providers/package.json +1 -1
  109. package/packages/qdrant-skills/package.json +1 -1
  110. package/packages/quota-manager/package.json +1 -1
  111. package/packages/rate-limiter/package.json +1 -1
  112. package/packages/react-vite-plugin/package.json +1 -1
  113. package/packages/relay/package.json +1 -1
  114. package/packages/release-manager/package.json +1 -1
  115. package/packages/requirement-compiler/package.json +1 -1
  116. package/packages/runtime/package.json +1 -1
  117. package/packages/scheduler/package.json +1 -1
  118. package/packages/scheduler-adapter/package.json +1 -1
  119. package/packages/session/package.json +1 -1
  120. package/packages/session-api/package.json +1 -1
  121. package/packages/session-export/package.json +1 -1
  122. package/packages/shared-context/package.json +1 -1
  123. package/packages/skill-mcp-client/package.json +1 -1
  124. package/packages/skill-registry/package.json +1 -1
  125. package/packages/skills/package.json +1 -1
  126. package/packages/sprint-planner/package.json +1 -1
  127. package/packages/subscription-engine/package.json +1 -1
  128. package/packages/task-compiler/package.json +1 -1
  129. package/packages/telegram/package.json +1 -1
  130. package/packages/tencentdb-memory/package.json +1 -1
  131. package/packages/tencentdb-sync/package.json +1 -1
  132. package/packages/test-data-generator/package.json +1 -1
  133. package/packages/test-generator/package.json +1 -1
  134. package/packages/todo-bd-sync/package.json +1 -1
  135. package/packages/token-estimation/package.json +1 -1
  136. package/packages/token-optimizer/package.json +1 -1
  137. package/packages/trajectory/package.json +1 -1
  138. package/packages/tui/package.json +1 -1
  139. package/packages/types/package.json +1 -1
  140. package/packages/voice-api/package.json +1 -1
  141. package/packages/webmcp-tester/package.json +1 -1
  142. package/packages/wiki/package.json +1 -1
  143. package/packages/workflow-events/package.json +1 -1
  144. package/packages/workspace-scanner/package.json +1 -1
  145. package/packages/worktree/package.json +1 -1
  146. package/packages/write-review/package.json +1 -1
  147. package/scripts/run-root-tests.mjs +207 -0
@@ -6,84 +6,102 @@
6
6
  *
7
7
  * API: POST https://api.typesafe.ai/v1/systemone
8
8
  * Auth: Bearer token (TYPESAFE_API_KEY)
9
+ *
10
+ * The request contract (choice criteria as a dictionary, score questions with
11
+ * a mandatory rubric) is implemented in ./typesafe-request.ts; the mapping was
12
+ * verified against the live endpoint.
9
13
  */
10
14
 
11
15
  import type {
12
- JevQuestion,
13
- JevQuestions,
14
- JevResponse,
15
- JevDecision,
16
- JevEvaluationResult,
17
- JevNoulResponse,
18
- JevChoiceResponse,
19
- JevScoreResponse,
16
+ JevQuestion,
17
+ JevQuestions,
18
+ JevResponse,
19
+ JevDecision,
20
+ JevEvaluationResult,
21
+ JevNoulResponse,
22
+ JevChoiceResponse,
23
+ JevScoreResponse,
20
24
  } from "./types.js";
25
+ import {
26
+ buildChoiceCriteria,
27
+ buildScoreScale,
28
+ scorePositionToProbability,
29
+ scorePositionToValue,
30
+ validateQuestion,
31
+ type ScoreScale,
32
+ } from "./typesafe-request.js";
33
+ import { JevProviderError, isRetryableStatus, isTransportError } from "./provider-error.js";
21
34
 
22
35
  /**
23
36
  * TypeSafe API request format
24
37
  */
25
38
  interface TypeSafeRequest {
26
- /** The model that handles the request. Required. */
27
- model: string;
28
- /** The content to evaluate. */
29
- state: string | object | unknown[];
30
- /** A map of typed questions. Keys are question IDs. */
31
- questions: Record<string, TypeSafeQuestion>;
39
+ /** The model that handles the request. Required. */
40
+ model: string;
41
+ /** The content to evaluate. */
42
+ state: string | object | unknown[];
43
+ /** A map of typed questions. Keys are question IDs. */
44
+ questions: Record<string, TypeSafeQuestion>;
32
45
  }
33
46
 
34
47
  /**
35
48
  * TypeSafe question format (per API spec, id not needed in question object - it's the map key)
36
49
  */
37
50
  interface TypeSafeQuestion {
38
- type: "noul" | "choice" | "score";
39
- instructions: string;
40
- options?: string[]; // For choice type
41
- criteria?: string[]; // For choice type - ordered criteria
42
- min?: number; // For score type
43
- max?: number; // For score type
51
+ type: "noul" | "choice" | "score";
52
+ instructions: string;
53
+ /**
54
+ * Choice questions: a dictionary of `name -> description`.
55
+ * Score questions: an ordered list of level descriptions.
56
+ */
57
+ criteria?: Record<string, string> | string[];
44
58
  }
45
59
 
46
60
  /**
47
61
  * TypeSafe API response format
48
62
  */
49
63
  interface TypeSafeResponse {
50
- answers: Record<string, TypeSafeAnswer>;
51
- model?: string;
52
- latency_ms?: number;
64
+ answers: Record<string, TypeSafeAnswer>;
65
+ model?: string;
66
+ latency_ms?: number;
53
67
  }
54
68
 
55
69
  /**
56
70
  * TypeSafe answer format
57
71
  */
58
72
  interface TypeSafeAnswer {
59
- type: "noul" | "choice" | "score";
60
- /** For noul type: probability of yes (0-1). */
61
- noul?: number;
62
- /** For choice type: the selected option. */
63
- choice?: string;
64
- /** For score type: the probability-weighted score. */
65
- score?: number;
66
- /** For choice/score: confidence 0-1. */
67
- confidence?: number;
73
+ type: "noul" | "choice" | "score";
74
+ /** For noul type: probability of yes (0-1). */
75
+ noul?: number;
76
+ /** For choice type: the selected option. */
77
+ choice?: string;
78
+ /** For score type: the probability-weighted position in the rubric. */
79
+ score?: number;
80
+ /** For choice/score: confidence 0-1. */
81
+ confidence?: number;
82
+ /** For choice type: the probability of every choice. */
83
+ probabilities?: Record<string, number>;
84
+ /** For score type: the rubric levels mapped to their level index. */
85
+ legend?: Record<string, string>;
68
86
  }
69
87
 
70
88
  /**
71
89
  * TypeSafe provider configuration
72
90
  */
73
91
  export interface TypeSafeConfig {
74
- apiKey: string;
75
- model?: string; // Defaults to jev-latest
76
- baseUrl?: string; // Defaults to https://api.typesafe.ai
77
- timeoutMs?: number; // Defaults to 30000
92
+ apiKey: string;
93
+ model?: string; // Defaults to jev-latest
94
+ baseUrl?: string; // Defaults to https://api.typesafe.ai
95
+ timeoutMs?: number; // Defaults to 30000
78
96
  }
79
97
 
80
98
  /**
81
99
  * Default configuration
82
100
  */
83
101
  const DEFAULT_CONFIG = {
84
- model: "jev-latest",
85
- baseUrl: "https://api.typesafe.ai",
86
- timeoutMs: 30000,
102
+ model: "jev-latest",
103
+ baseUrl: "https://api.typesafe.ai",
104
+ timeoutMs: 30000,
87
105
  };
88
106
 
89
107
  /**
@@ -93,290 +111,406 @@ const DEFAULT_CONFIG = {
93
111
  * More reliable than OpenRouter proxy and supports full feature set.
94
112
  */
95
113
  export class TypeSafeJudge {
96
- private config: Required<TypeSafeConfig>;
97
-
98
- constructor(config: TypeSafeConfig) {
99
- this.config = {
100
- ...DEFAULT_CONFIG,
101
- ...config,
102
- } as Required<TypeSafeConfig>;
103
- }
104
-
105
- /**
106
- * Evaluate state against questions using TypeSafe API
107
- */
108
- async evaluate<T extends string>(
109
- state: string | object,
110
- questions: JevQuestions<T>
111
- ): Promise<JevEvaluationResult<T>> {
112
- const startTime = Date.now();
113
-
114
- // Convert state to string if object
115
- const stateStr = typeof state === "string" ? state : JSON.stringify(state);
116
-
117
- // Build TypeSafe question format as a map (not array)
118
- const questionsMap: Record<string, TypeSafeQuestion> = {};
119
- for (const [id, q] of Object.entries(questions)) {
120
- const questionObj = q as JevQuestion;
121
- questionsMap[id] = {
122
- type: questionObj.type,
123
- instructions: questionObj.instructions,
124
- };
125
-
126
- if (questionObj.type === "choice") {
127
- questionsMap[id].options = questionObj.options;
128
- // For choice questions, criteria defines ordered preference levels
129
- questionsMap[id].criteria = questionObj.options;
130
- } else if (questionObj.type === "score") {
131
- questionsMap[id].min = questionObj.min ?? 0;
132
- questionsMap[id].max = questionObj.max ?? 100;
133
- }
134
- }
135
-
136
- // Call TypeSafe API directly
137
- const response = await fetch(`${this.config.baseUrl}/v1/systemone`, {
138
- method: "POST",
139
- headers: {
140
- "Content-Type": "application/json",
141
- Authorization: `Bearer ${this.config.apiKey}`,
142
- },
143
- body: JSON.stringify({
144
- model: this.config.model,
145
- questions: questionsMap,
146
- state: stateStr,
147
- } satisfies TypeSafeRequest),
148
- signal: AbortSignal.timeout(this.config.timeoutMs),
149
- });
150
-
151
- const latencyMs = Date.now() - startTime;
152
-
153
- if (!response.ok) {
154
- const errorText = await response.text().catch(() => "Unknown error");
155
- throw new Error(`TypeSafe API error ${response.status}: ${errorText}`);
156
- }
157
-
158
- const data = await response.json() as TypeSafeResponse;
159
-
160
- // Build decisions map
161
- const decisions = {} as Record<T, JevDecision<T>>;
162
-
163
- for (const [questionId, question] of Object.entries(questions)) {
164
- const questionData = question as JevQuestion;
165
- const answer = data.answers?.[questionId] as TypeSafeAnswer | undefined;
166
-
167
- // Reject missing required answers
168
- if (!answer) {
169
- throw new Error(`TypeSafe API: missing answer for required question "${questionId}"`);
170
- }
171
-
172
- const decision = this.buildDecision(
173
- questionId,
174
- questionData,
175
- answer
176
- );
177
- (decisions as Record<string, JevDecision>)[questionId] = decision;
178
- }
179
-
180
- return {
181
- decisions,
182
- state: stateStr,
183
- latencyMs,
184
- model: data.model ?? "typesafe/systemone",
185
- };
186
- }
187
-
188
- /**
189
- * Quick noul (yes/no) evaluation
190
- */
191
- async noul(
192
- state: string | object,
193
- question: string
194
- ): Promise<{ noul: number; confidence: "high" | "medium" | "low" }> {
195
- const result = await this.evaluate(state, {
196
- answer: { type: "noul", instructions: question },
197
- });
198
-
199
- const decision = result.decisions.answer as JevDecision<"answer">;
200
- const noulResponse = decision.response as JevNoulResponse;
201
- return {
202
- noul: noulResponse.noul,
203
- confidence: decision.confidence,
204
- };
205
- }
206
-
207
- /**
208
- * Quick choice evaluation
209
- */
210
- async choice<T extends string>(
211
- state: string | object,
212
- question: string,
213
- options: T[]
214
- ): Promise<{ choice: T; confidence: number }> {
215
- const result = await this.evaluate(state, {
216
- answer: { type: "choice", instructions: question, options },
217
- });
218
-
219
- const decision = result.decisions.answer as JevDecision<"answer">;
220
- const choiceResponse = decision.response as JevChoiceResponse<T>;
221
- return {
222
- choice: choiceResponse.choice,
223
- confidence: choiceResponse.confidence,
224
- };
225
- }
226
-
227
- /**
228
- * Quick score evaluation
229
- */
230
- async score(
231
- state: string | object,
232
- question: string,
233
- min = 0,
234
- max = 100
235
- ): Promise<{ score: number; confidence: number }> {
236
- const result = await this.evaluate(state, {
237
- answer: { type: "score", instructions: question, min, max },
238
- });
239
-
240
- const decision = result.decisions.answer as JevDecision<"answer">;
241
- const scoreResponse = decision.response as JevScoreResponse;
242
- return {
243
- score: scoreResponse.score,
244
- confidence: scoreResponse.confidence,
245
- };
246
- }
247
-
248
- /**
249
- * Build JevDecision from TypeSafe answer
250
- */
251
- private buildDecision<T extends string>(
252
- questionId: T,
253
- question: JevQuestion,
254
- answer: TypeSafeAnswer
255
- ): JevDecision<T> {
256
- let response: JevResponse;
257
- let probability: number;
258
-
259
- switch (answer.type) {
260
- case "noul":
261
- // FIX: Validate noul value is present and in range [0, 1]
262
- if (answer.noul === undefined || answer.noul === null) {
263
- throw new Error(`TypeSafe API: missing noul value for question "${questionId}"`);
264
- }
265
- if (typeof answer.noul !== "number" || isNaN(answer.noul)) {
266
- throw new Error(`TypeSafe API: invalid noul value for question "${questionId}": ${JSON.stringify(answer.noul)}`);
267
- }
268
- if (answer.noul < 0 || answer.noul > 1) {
269
- throw new Error(`TypeSafe API: noul value out of range [0,1] for question "${questionId}": ${answer.noul}`);
270
- }
271
- response = {
272
- type: "noul",
273
- noul: answer.noul,
274
- };
275
- probability = answer.noul;
276
- break;
277
-
278
- case "choice": {
279
- const options =
280
- question.type === "choice"
281
- ? ((question as { options?: string[] }).options ?? [])
282
- : [];
283
- // The model's choice MUST win. The previous expression
284
- // (`answer.choice ?? question.type === "choice" ? options[0] : ""`)
285
- // was parsed as `(answer.choice ?? (question.type === "choice")) ? options[0] : ""`,
286
- // so a non-empty returned choice was DISCARDED and the first option was
287
- // always selected.
288
- const returned =
289
- typeof answer.choice === "string" ? answer.choice.trim() : "";
290
- let selected: string;
291
- if (returned.length > 0) {
292
- if (options.includes(returned)) {
293
- selected = returned;
294
- } else {
295
- // Case-insensitive match: keep the canonical option spelling.
296
- const canonical = options.find(
297
- (option) => option.toLowerCase() === returned.toLowerCase(),
298
- );
299
- // No match: pass the model's value through unchanged instead of
300
- // silently substituting a different option.
301
- selected = canonical ?? returned;
302
- }
303
- } else {
304
- // The model returned no usable choice: fall back to the first option.
305
- selected = options[0] ?? "";
306
- }
307
- response = {
308
- type: "choice",
309
- choice: selected,
310
- confidence: answer.confidence ?? 0.5,
311
- };
312
- probability = answer.confidence ?? 0.5;
313
- break;
314
- }
315
-
316
- case "score":
317
- const min = question.type === "score" ? (question as { min?: number }).min ?? 0 : 0;
318
- const max = question.type === "score" ? (question as { max?: number }).max ?? 100 : 100;
319
- response = {
320
- type: "score",
321
- score: answer.score ?? (min + max) / 2,
322
- confidence: answer.confidence ?? 0.5,
323
- };
324
- // Normalize score to 0-1
325
- probability = ((answer.score ?? (min + max) / 2) - min) / (max - min);
326
- break;
327
-
328
- default:
329
- // Fallback for unknown types
330
- response = {
331
- type: "noul",
332
- noul: 0.5,
333
- };
334
- probability = 0.5;
335
- }
336
-
337
- const confidence = this.getConfidence(answer.confidence ?? probability);
338
-
339
- return {
340
- questionId,
341
- question,
342
- response,
343
- probability,
344
- confidence,
345
- };
346
- }
347
-
348
- /**
349
- * Determine confidence level from probability
350
- */
351
- private getConfidence(probability: number): "high" | "medium" | "low" {
352
- // Confidence is based on distance from 0.5 (uncertain)
353
- const distance = Math.abs(probability - 0.5);
354
- if (distance > 0.35) return "high";
355
- if (distance > 0.2) return "medium";
356
- return "low";
357
- }
114
+ private config: Required<TypeSafeConfig>;
115
+
116
+ constructor(config: TypeSafeConfig) {
117
+ this.config = {
118
+ ...DEFAULT_CONFIG,
119
+ ...config,
120
+ } as Required<TypeSafeConfig>;
121
+ }
122
+
123
+ /**
124
+ * Evaluate state against questions using TypeSafe API
125
+ */
126
+ async evaluate<T extends string>(
127
+ state: string | object,
128
+ questions: JevQuestions<T>
129
+ ): Promise<JevEvaluationResult<T>> {
130
+ const startTime = Date.now();
131
+
132
+ // Convert state to string if object
133
+ const stateStr = typeof state === "string" ? state : JSON.stringify(state);
134
+
135
+ // Build and validate the request. One invalid question fails the whole
136
+ // request on the API side, so contract problems are caught locally.
137
+ const { request, scoreScales } = this.buildRequest(stateStr, questions);
138
+
139
+ // Call TypeSafe API directly
140
+ let response: Response;
141
+ try {
142
+ response = await fetch(`${this.config.baseUrl}/v1/systemone`, {
143
+ method: "POST",
144
+ headers: {
145
+ "Content-Type": "application/json",
146
+ Authorization: `Bearer ${this.config.apiKey}`,
147
+ },
148
+ // Only the wire format is sent: the API rejects unknown fields.
149
+ body: JSON.stringify(request),
150
+ signal: AbortSignal.timeout(this.config.timeoutMs),
151
+ });
152
+ } catch (error) {
153
+ // A network failure, a timeout or an abort never reached a server
154
+ // that rejected the request: another provider can still serve it.
155
+ // Anything else (a malformed URL, a bug in our own code) is NOT
156
+ // retryable - it must not disappear behind a fallback.
157
+ throw new JevProviderError(
158
+ `TypeSafe request failed: ${error instanceof Error ? error.message : String(error)}`,
159
+ { provider: "typesafe", retryable: isTransportError(error), cause: error }
160
+ );
161
+ }
162
+
163
+ const latencyMs = Date.now() - startTime;
164
+
165
+ if (!response.ok) {
166
+ const errorText = await response.text().catch(() => "Unknown error");
167
+ throw new JevProviderError(
168
+ `TypeSafe API error ${response.status}: ${errorText}`,
169
+ {
170
+ provider: "typesafe",
171
+ status: response.status,
172
+ retryable: isRetryableStatus(response.status),
173
+ }
174
+ );
175
+ }
176
+
177
+ const data = await response.json() as TypeSafeResponse;
178
+
179
+ // Build decisions map
180
+ const decisions = {} as Record<T, JevDecision<T>>;
181
+
182
+ for (const [questionId, question] of Object.entries(questions)) {
183
+ const questionData = question as JevQuestion;
184
+ const answer = data.answers?.[questionId] as TypeSafeAnswer | undefined;
185
+
186
+ // Reject missing required answers
187
+ if (!answer) {
188
+ throw new JevProviderError(
189
+ `TypeSafe API: missing answer for required question "${questionId}"`,
190
+ { provider: "typesafe", retryable: false }
191
+ );
192
+ }
193
+
194
+ const decision = this.buildDecision(
195
+ questionId,
196
+ questionData,
197
+ answer,
198
+ scoreScales[questionId]
199
+ );
200
+ (decisions as Record<string, JevDecision>)[questionId] = decision;
201
+ }
202
+
203
+ return {
204
+ decisions,
205
+ state: stateStr,
206
+ latencyMs,
207
+ model: data.model ?? this.config.model,
208
+ };
209
+ }
210
+
211
+ /**
212
+ * Build the API request and validate it against the API contract.
213
+ *
214
+ * The returned `request` is the wire format; `scoreScales` is local
215
+ * bookkeeping and is never sent to the API (it rejects unknown fields).
216
+ */
217
+ private buildRequest<T extends string>(
218
+ stateStr: string,
219
+ questions: JevQuestions<T>
220
+ ): { request: TypeSafeRequest; scoreScales: Record<string, ScoreScale> } {
221
+ const questionsMap: Record<string, TypeSafeQuestion> = {};
222
+ const scoreScales: Record<string, ScoreScale> = {};
223
+
224
+ for (const [id, raw] of Object.entries(questions)) {
225
+ const questionObj = raw as JevQuestion;
226
+ const scoreScale = validateQuestion(id, questionObj);
227
+
228
+ if (questionObj.type === "choice") {
229
+ questionsMap[id] = {
230
+ type: "choice",
231
+ instructions: questionObj.instructions,
232
+ criteria: buildChoiceCriteria(id, questionObj),
233
+ };
234
+ } else if (questionObj.type === "score") {
235
+ const { criteria } = buildScoreScale(id, questionObj);
236
+ questionsMap[id] = {
237
+ type: "score",
238
+ instructions: questionObj.instructions,
239
+ criteria,
240
+ };
241
+ if (scoreScale) scoreScales[id] = scoreScale;
242
+ } else {
243
+ questionsMap[id] = {
244
+ type: "noul",
245
+ instructions: questionObj.instructions,
246
+ };
247
+ }
248
+ }
249
+
250
+ if (Object.keys(questionsMap).length === 0) {
251
+ throw new JevProviderError("TypeSafe request invalid: no questions to ask", {
252
+ provider: "typesafe",
253
+ retryable: false,
254
+ });
255
+ }
256
+
257
+ return {
258
+ request: {
259
+ model: this.config.model,
260
+ state: stateStr,
261
+ questions: questionsMap,
262
+ },
263
+ scoreScales,
264
+ };
265
+ }
266
+
267
+ /**
268
+ * Quick noul (yes/no) evaluation
269
+ */
270
+ async noul(
271
+ state: string | object,
272
+ question: string
273
+ ): Promise<{ noul: number; confidence: "high" | "medium" | "low" }> {
274
+ const result = await this.evaluate(state, {
275
+ answer: { type: "noul", instructions: question },
276
+ });
277
+
278
+ const decision = result.decisions.answer as JevDecision<"answer">;
279
+ const noulResponse = decision.response as JevNoulResponse;
280
+ return {
281
+ noul: noulResponse.noul,
282
+ confidence: decision.confidence,
283
+ };
284
+ }
285
+
286
+ /**
287
+ * Quick choice evaluation
288
+ */
289
+ async choice<T extends string>(
290
+ state: string | object,
291
+ question: string,
292
+ options: T[]
293
+ ): Promise<{ choice: T; confidence: number }> {
294
+ const result = await this.evaluate(state, {
295
+ answer: { type: "choice", instructions: question, options },
296
+ });
297
+
298
+ const decision = result.decisions.answer as JevDecision<"answer">;
299
+ const choiceResponse = decision.response as JevChoiceResponse<T>;
300
+ return {
301
+ choice: choiceResponse.choice,
302
+ confidence: choiceResponse.confidence,
303
+ };
304
+ }
305
+
306
+ /**
307
+ * Quick score evaluation
308
+ */
309
+ async score(
310
+ state: string | object,
311
+ question: string,
312
+ min = 0,
313
+ max = 100
314
+ ): Promise<{ score: number; confidence: number }> {
315
+ const result = await this.evaluate(state, {
316
+ answer: { type: "score", instructions: question, min, max },
317
+ });
318
+
319
+ const decision = result.decisions.answer as JevDecision<"answer">;
320
+ const scoreResponse = decision.response as JevScoreResponse;
321
+ return {
322
+ score: scoreResponse.score,
323
+ confidence: scoreResponse.confidence,
324
+ };
325
+ }
326
+
327
+ /**
328
+ * Build JevDecision from TypeSafe answer
329
+ */
330
+ private buildDecision<T extends string>(
331
+ questionId: T,
332
+ question: JevQuestion,
333
+ answer: TypeSafeAnswer,
334
+ scoreScale?: ScoreScale
335
+ ): JevDecision<T> {
336
+ let response: JevResponse;
337
+ let probability: number;
338
+
339
+ switch (answer.type) {
340
+ case "noul":
341
+ // FIX: Validate noul value is present and in range [0, 1]
342
+ if (answer.noul === undefined || answer.noul === null) {
343
+ throw new JevProviderError(
344
+ `TypeSafe API: missing noul value for question "${questionId}"`,
345
+ { provider: "typesafe", retryable: false }
346
+ );
347
+ }
348
+ if (typeof answer.noul !== "number" || isNaN(answer.noul)) {
349
+ throw new JevProviderError(
350
+ `TypeSafe API: invalid noul value for question "${questionId}": ${JSON.stringify(answer.noul)}`,
351
+ { provider: "typesafe", retryable: false }
352
+ );
353
+ }
354
+ if (answer.noul < 0 || answer.noul > 1) {
355
+ throw new JevProviderError(
356
+ `TypeSafe API: noul value out of range [0,1] for question "${questionId}": ${answer.noul}`,
357
+ { provider: "typesafe", retryable: false }
358
+ );
359
+ }
360
+ response = {
361
+ type: "noul",
362
+ noul: answer.noul,
363
+ };
364
+ probability = answer.noul;
365
+ break;
366
+
367
+ case "choice": {
368
+ const options = choiceCandidates(question);
369
+ // The model's choice MUST win. The previous expression
370
+ // (`answer.choice ?? question.type === "choice" ? options[0] : ""`)
371
+ // was parsed as `(answer.choice ?? (question.type === "choice")) ? options[0] : ""`,
372
+ // so a non-empty returned choice was DISCARDED and the first option was
373
+ // always selected.
374
+ const returned =
375
+ typeof answer.choice === "string" ? answer.choice.trim() : "";
376
+ let selected: string;
377
+ if (returned.length > 0) {
378
+ if (options.includes(returned)) {
379
+ selected = returned;
380
+ } else {
381
+ // Case-insensitive match: keep the canonical option spelling.
382
+ const canonical = options.find(
383
+ (option) => option.toLowerCase() === returned.toLowerCase(),
384
+ );
385
+ // No match: pass the model's value through unchanged instead of
386
+ // silently substituting a different option.
387
+ selected = canonical ?? returned;
388
+ }
389
+ } else {
390
+ // The model returned no usable choice: fall back to the first option.
391
+ selected = options[0] ?? "";
392
+ }
393
+ response = {
394
+ type: "choice",
395
+ choice: selected,
396
+ confidence: answer.confidence ?? 0.5,
397
+ };
398
+ probability = answer.confidence ?? 0.5;
399
+ break;
400
+ }
401
+
402
+ case "score": {
403
+ if (answer.score === undefined || answer.score === null) {
404
+ throw new JevProviderError(
405
+ `TypeSafe API: missing score value for question "${questionId}"`,
406
+ { provider: "typesafe", retryable: false }
407
+ );
408
+ }
409
+ if (typeof answer.score !== "number" || isNaN(answer.score)) {
410
+ throw new JevProviderError(
411
+ `TypeSafe API: invalid score value for question "${questionId}": ${JSON.stringify(answer.score)}`,
412
+ { provider: "typesafe", retryable: false }
413
+ );
414
+ }
415
+ // The API answers with the probability-weighted POSITION in the
416
+ // rubric (0 .. levels-1), not with a value on the min/max scale.
417
+ const scale: ScoreScale = scoreScale ?? {
418
+ levels: 2,
419
+ min: question.type === "score" ? question.min ?? 0 : 0,
420
+ max: question.type === "score" ? question.max ?? 100 : 100,
421
+ };
422
+ const value = scorePositionToValue(answer.score, scale);
423
+ response = {
424
+ type: "score",
425
+ score: value,
426
+ confidence: answer.confidence ?? 0.5,
427
+ };
428
+ probability = scorePositionToProbability(answer.score, scale);
429
+ break;
430
+ }
431
+
432
+ default:
433
+ // Fallback for unknown types
434
+ response = {
435
+ type: "noul",
436
+ noul: 0.5,
437
+ };
438
+ probability = 0.5;
439
+ }
440
+
441
+ const confidence = this.getConfidence(answer.confidence ?? probability);
442
+
443
+ return {
444
+ questionId,
445
+ question,
446
+ response,
447
+ probability,
448
+ confidence,
449
+ };
450
+ }
451
+
452
+ /**
453
+ * Determine confidence level from probability
454
+ */
455
+ private getConfidence(probability: number): "high" | "medium" | "low" {
456
+ // Confidence is based on distance from 0.5 (uncertain)
457
+ const distance = Math.abs(probability - 0.5);
458
+ if (distance > 0.35) return "high";
459
+ if (distance > 0.2) return "medium";
460
+ return "low";
461
+ }
462
+ }
463
+
464
+ /**
465
+ * The candidate names of a choice question.
466
+ *
467
+ * `options` is the ergonomic way to name the candidates; a question that only
468
+ * carries `criteria` is read from the criteria itself.
469
+ */
470
+ function choiceCandidates(
471
+ question: JevQuestion
472
+ ): string[] {
473
+ if (question.type !== "choice") return [];
474
+ const options = question.options;
475
+ if (options && options.length > 0) return [...options];
476
+ const criteria = question.criteria;
477
+ if (Array.isArray(criteria)) return [...criteria];
478
+ if (criteria) return Object.keys(criteria);
479
+ return [];
480
+ }
481
+
482
+ /**
483
+ * Determine the provider that owns a base URL.
484
+ *
485
+ * The provider is derived from the ENDPOINT, never from the shape of a key:
486
+ * guessing from a key is how an OpenRouter key ended up being sent to the
487
+ * TypeSafe API.
488
+ */
489
+ export function providerForBaseUrl(baseUrl: string): "typesafe" | "openrouter" {
490
+ if (baseUrl.includes("typesafe.ai")) return "typesafe";
491
+ if (baseUrl.includes("openrouter")) return "openrouter";
492
+ throw new Error(
493
+ `Jev: unknown provider base URL "${baseUrl}". Use resolveJevProviders() or pass an explicit provider chain.`
494
+ );
358
495
  }
359
496
 
360
497
  /**
361
498
  * Detect which provider to use based on API key or config
499
+ *
500
+ * @deprecated The provider cannot be derived from a key - a bare key says
501
+ * nothing about the endpoint it belongs to, and guessing from its length is
502
+ * how an OpenRouter key ended up being sent to the TypeSafe API. Pass the
503
+ * `baseUrl` of the provider, or use `resolveJevProviders()` (see ./index.ts),
504
+ * which binds every credential to its own provider.
362
505
  */
363
506
  export function detectProvider(
364
- apiKey: string,
365
- baseUrl?: string
507
+ _apiKey: string,
508
+ baseUrl?: string
366
509
  ): "typesafe" | "openrouter" {
367
- // If baseUrl is explicitly set to TypeSafe, use TypeSafe
368
- if (baseUrl?.includes("typesafe.ai")) {
369
- return "typesafe";
370
- }
371
- // If using OpenRouter URL, use OpenRouter
372
- if (baseUrl?.includes("openrouter")) {
373
- return "openrouter";
374
- }
375
- // If key looks like TypeSafe (no specific pattern known), prefer TypeSafe
376
- // TypeSafe keys are typically longer than OpenAI-compatible keys
377
- if (apiKey.length > 50) {
378
- return "typesafe";
379
- }
380
- // Default to OpenRouter for short keys (OpenAI-compatible)
381
- return "openrouter";
510
+ if (!baseUrl) {
511
+ throw new Error(
512
+ "Jev: detectProvider() cannot guess the provider from the key. Pass the baseUrl of the provider or use resolveJevProviders()."
513
+ );
514
+ }
515
+ return providerForBaseUrl(baseUrl);
382
516
  }