pareta 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.cjs +80 -43
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +55 -38
- package/dist/index.d.ts +55 -38
- package/dist/index.mjs +80 -43
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -47,6 +47,17 @@ declare class ChatCompletion extends BaseModel {
|
|
|
47
47
|
get created(): number | null;
|
|
48
48
|
get choices(): Choice[];
|
|
49
49
|
get usage(): Usage;
|
|
50
|
+
/** #164 receipt, held OFF `.raw` so `toDict()` stays the raw server JSON. */
|
|
51
|
+
private _cost?;
|
|
52
|
+
/** #164: attach the per-call receipt from the response headers (billed +
|
|
53
|
+
* the frontier counterfactual, micro-USD ints). Chainable. */
|
|
54
|
+
withCost(headers?: Headers): ChatCompletion;
|
|
55
|
+
/** What Pareta charged, micro-USD (0 on an idempotent replay); null if absent. */
|
|
56
|
+
get billedMicroUsd(): number | null;
|
|
57
|
+
/** What one list-priced frontier call would have cost, micro-USD. */
|
|
58
|
+
get frontierWouldHaveCostMicroUsd(): number | null;
|
|
59
|
+
/** frontier / billed — e.g. 17 = 17× cheaper; null if either is missing or billed is 0. */
|
|
60
|
+
get savingsFactor(): number | null;
|
|
50
61
|
}
|
|
51
62
|
/** One SSE delta. `chunk.choices[0].delta.content` is the incremental text. */
|
|
52
63
|
declare class ChatCompletionChunk extends ChatCompletion {
|
|
@@ -88,13 +99,13 @@ declare class EvalSet extends BaseModel {
|
|
|
88
99
|
get itemCount(): number | null;
|
|
89
100
|
get scoringStrategy(): string | null;
|
|
90
101
|
/** The one-sentence success criterion this set was created with (CB1:
|
|
91
|
-
* DATA +
|
|
92
|
-
get
|
|
102
|
+
* DATA + PROMPT). Required at create. */
|
|
103
|
+
get prompt(): string | null;
|
|
93
104
|
}
|
|
94
105
|
/**
|
|
95
|
-
* One proposed
|
|
96
|
-
* `ProposalResult.proposals`). `warning` is set on
|
|
97
|
-
* the data looks extraction-shaped (judge grading is weaker there).
|
|
106
|
+
* One proposed way to score an uploaded dataset (a row of
|
|
107
|
+
* `ProposalResult.proposals`). `warning` is set on a "custom-eval" proposal
|
|
108
|
+
* when the data looks extraction-shaped (judge grading is weaker there).
|
|
98
109
|
*/
|
|
99
110
|
declare class ContractProposal extends BaseModel {
|
|
100
111
|
get taskId(): string | null;
|
|
@@ -103,10 +114,10 @@ declare class ContractProposal extends BaseModel {
|
|
|
103
114
|
get warning(): string | null;
|
|
104
115
|
}
|
|
105
116
|
/**
|
|
106
|
-
*
|
|
107
|
-
* /v1/eval-sets/propose-contract):
|
|
108
|
-
*
|
|
109
|
-
*
|
|
117
|
+
* Pareta's answer for `evals.proposeContract` (POST
|
|
118
|
+
* /v1/eval-sets/propose-contract): how your data could be scored under your
|
|
119
|
+
* stated prompt. Nothing is persisted — confirm by passing the chosen
|
|
120
|
+
* `taskId` (or letting a task-less `create` use a clean single proposal).
|
|
110
121
|
*/
|
|
111
122
|
declare class ProposalResult extends BaseModel {
|
|
112
123
|
get proposals(): ContractProposal[];
|
|
@@ -114,16 +125,17 @@ declare class ProposalResult extends BaseModel {
|
|
|
114
125
|
get split(): Raw | null;
|
|
115
126
|
get conflict(): Raw | null;
|
|
116
127
|
get closestTask(): string | null;
|
|
117
|
-
get
|
|
128
|
+
get prompt(): string | null;
|
|
118
129
|
get message(): string | null;
|
|
119
|
-
/** True when
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
* `task: "custom-eval"`, never
|
|
130
|
+
/** True when exactly ONE homogeneous SPECIFIC task matched at high/medium
|
|
131
|
+
* confidence with no conflict/split — the case a task-less `create`
|
|
132
|
+
* proceeds with, no human confirming. `custom-eval` (a judge panel grading
|
|
133
|
+
* each answer against your prompt) is EXCLUDED: it is offered only when
|
|
134
|
+
* nothing specific fits, and using it is the user's CHOICE — opt in with
|
|
135
|
+
* `task: "custom-eval"`, never silently. */
|
|
125
136
|
get isClean(): boolean;
|
|
126
|
-
/**
|
|
137
|
+
/** How a task-less `create` would score this set, or null if the result
|
|
138
|
+
* isn't clean (pass `task` yourself). */
|
|
127
139
|
get boundTask(): string | null;
|
|
128
140
|
}
|
|
129
141
|
/**
|
|
@@ -319,16 +331,17 @@ declare class Tasks {
|
|
|
319
331
|
/**
|
|
320
332
|
* `client.evals` — eval sets + runs (bring-your-own-data evaluation).
|
|
321
333
|
*
|
|
322
|
-
* // An eval set is DATA +
|
|
323
|
-
* // OPTIONAL —
|
|
324
|
-
*
|
|
325
|
-
*
|
|
334
|
+
* // An eval set is DATA + PROMPT (v3, breaking): prompt REQUIRED, task
|
|
335
|
+
* // OPTIONAL — Pareta works out how to score the results from your
|
|
336
|
+
* // prompt + the data's shape.
|
|
337
|
+
* await pa.evals.proposeContract({ items: [...], prompt: "…" }); // preview
|
|
338
|
+
* const set = await pa.evals.sets.create({ items: [...], prompt: "…" }); // clean match used automatically
|
|
326
339
|
* await pa.evals.sets.uploadDocument(set.id, file, { idx, fieldName }); // blob tasks
|
|
327
340
|
* const run = await pa.evals.runs.create({ evalSet: set.id, models: [...], wait: true });
|
|
328
|
-
* // or, in one call: runs.create({ items,
|
|
341
|
+
* // or, in one call: runs.create({ items, prompt: "…", models, wait: true })
|
|
329
342
|
*
|
|
330
|
-
* `create` (and the inline `runs.create` sugar) require `
|
|
331
|
-
* `task` they call `proposeContract` and
|
|
343
|
+
* `create` (and the inline `runs.create` sugar) require `prompt`; with no
|
|
344
|
+
* `task` they call `proposeContract` and accept ONLY a clean single
|
|
332
345
|
* high/medium match — a conflict, split, or ambiguity throws with the
|
|
333
346
|
* proposals so you pin `task`.
|
|
334
347
|
*
|
|
@@ -346,21 +359,21 @@ declare class EvalSets {
|
|
|
346
359
|
private readonly client;
|
|
347
360
|
constructor(client: Transport);
|
|
348
361
|
/**
|
|
349
|
-
* Persist an eval set from your rows. `
|
|
350
|
-
* on what the model should do with each item. `task` is OPTIONAL — omit it
|
|
351
|
-
*
|
|
352
|
-
* (
|
|
353
|
-
* `task` to pin one explicitly.
|
|
362
|
+
* Persist an eval set from your rows. `prompt` is REQUIRED (v3): one sentence
|
|
363
|
+
* on what the model should do with each item. `task` is OPTIONAL — omit it
|
|
364
|
+
* and Pareta works out how to score the results from your prompt + the
|
|
365
|
+
* data's shape (a clean single match is used; anything ambiguous throws with
|
|
366
|
+
* the proposals). Pass `task` to pin one explicitly.
|
|
354
367
|
*/
|
|
355
368
|
create(params: {
|
|
356
369
|
items: Array<Record<string, unknown>>;
|
|
357
|
-
|
|
370
|
+
prompt: string;
|
|
358
371
|
task?: string;
|
|
359
372
|
name?: string;
|
|
360
373
|
}): Promise<EvalSet>;
|
|
361
|
-
/** Internal: the
|
|
374
|
+
/** Internal: the propose call `create` uses when no task is pinned. Public on
|
|
362
375
|
* `Evals` as `proposeContract`; kept here so `sets.create` can reach it. */
|
|
363
|
-
proposeContract(items: Array<Record<string, unknown>>,
|
|
376
|
+
proposeContract(items: Array<Record<string, unknown>>, prompt: string): Promise<ProposalResult>;
|
|
364
377
|
list(): Promise<EvalSet[]>;
|
|
365
378
|
retrieve(evalSetId: string): Promise<EvalSet>;
|
|
366
379
|
delete(evalSetId: string): Promise<void>;
|
|
@@ -379,7 +392,7 @@ interface EvalRunCreateParams {
|
|
|
379
392
|
evalSet?: string;
|
|
380
393
|
task?: string;
|
|
381
394
|
items?: Array<Record<string, unknown>>;
|
|
382
|
-
|
|
395
|
+
prompt?: string;
|
|
383
396
|
models: string[];
|
|
384
397
|
frontier?: FrontierSpec;
|
|
385
398
|
name?: string;
|
|
@@ -405,14 +418,15 @@ declare class Evals {
|
|
|
405
418
|
constructor(client: Transport);
|
|
406
419
|
private readonly client;
|
|
407
420
|
/**
|
|
408
|
-
*
|
|
421
|
+
* How would your data be scored under your stated `prompt`? Stateless
|
|
409
422
|
* discovery — nothing is persisted. Returns a ProposalResult (ranked
|
|
410
|
-
* proposals, the
|
|
411
|
-
* calls this under the hood; use it directly to
|
|
423
|
+
* proposals, the task a task-less create would use, conflict/split
|
|
424
|
+
* reporting). `sets.create` calls this under the hood; use it directly to
|
|
425
|
+
* preview the scoring first.
|
|
412
426
|
*/
|
|
413
427
|
proposeContract(params: {
|
|
414
428
|
items: Array<Record<string, unknown>>;
|
|
415
|
-
|
|
429
|
+
prompt: string;
|
|
416
430
|
}): Promise<ProposalResult>;
|
|
417
431
|
/**
|
|
418
432
|
* The frontier (vendor) roster you can evaluate against. With `task`, each is
|
|
@@ -632,6 +646,9 @@ interface RequestOptions {
|
|
|
632
646
|
files?: Record<string, FilePart>;
|
|
633
647
|
data?: Record<string, string>;
|
|
634
648
|
cast?: (raw: unknown) => unknown;
|
|
649
|
+
/** Invoked with the response headers on success (before the body is cast) —
|
|
650
|
+
* lets a caller read receipt headers (#164: X-Pareta-Billed + counterfactual). */
|
|
651
|
+
onHeaders?: (headers: Headers) => void;
|
|
635
652
|
}
|
|
636
653
|
interface StreamOptions {
|
|
637
654
|
body?: unknown;
|
|
@@ -698,7 +715,7 @@ declare class Pareta implements Transport {
|
|
|
698
715
|
stream<T = unknown>(method: string, path: string, opts?: StreamOptions): AsyncGenerator<T>;
|
|
699
716
|
}
|
|
700
717
|
|
|
701
|
-
declare const VERSION = "
|
|
718
|
+
declare const VERSION = "3.0.0";
|
|
702
719
|
|
|
703
720
|
/**
|
|
704
721
|
* Typed error hierarchy for the Pareta SDK — a faithful port of the Python
|
package/dist/index.d.ts
CHANGED
|
@@ -47,6 +47,17 @@ declare class ChatCompletion extends BaseModel {
|
|
|
47
47
|
get created(): number | null;
|
|
48
48
|
get choices(): Choice[];
|
|
49
49
|
get usage(): Usage;
|
|
50
|
+
/** #164 receipt, held OFF `.raw` so `toDict()` stays the raw server JSON. */
|
|
51
|
+
private _cost?;
|
|
52
|
+
/** #164: attach the per-call receipt from the response headers (billed +
|
|
53
|
+
* the frontier counterfactual, micro-USD ints). Chainable. */
|
|
54
|
+
withCost(headers?: Headers): ChatCompletion;
|
|
55
|
+
/** What Pareta charged, micro-USD (0 on an idempotent replay); null if absent. */
|
|
56
|
+
get billedMicroUsd(): number | null;
|
|
57
|
+
/** What one list-priced frontier call would have cost, micro-USD. */
|
|
58
|
+
get frontierWouldHaveCostMicroUsd(): number | null;
|
|
59
|
+
/** frontier / billed — e.g. 17 = 17× cheaper; null if either is missing or billed is 0. */
|
|
60
|
+
get savingsFactor(): number | null;
|
|
50
61
|
}
|
|
51
62
|
/** One SSE delta. `chunk.choices[0].delta.content` is the incremental text. */
|
|
52
63
|
declare class ChatCompletionChunk extends ChatCompletion {
|
|
@@ -88,13 +99,13 @@ declare class EvalSet extends BaseModel {
|
|
|
88
99
|
get itemCount(): number | null;
|
|
89
100
|
get scoringStrategy(): string | null;
|
|
90
101
|
/** The one-sentence success criterion this set was created with (CB1:
|
|
91
|
-
* DATA +
|
|
92
|
-
get
|
|
102
|
+
* DATA + PROMPT). Required at create. */
|
|
103
|
+
get prompt(): string | null;
|
|
93
104
|
}
|
|
94
105
|
/**
|
|
95
|
-
* One proposed
|
|
96
|
-
* `ProposalResult.proposals`). `warning` is set on
|
|
97
|
-
* the data looks extraction-shaped (judge grading is weaker there).
|
|
106
|
+
* One proposed way to score an uploaded dataset (a row of
|
|
107
|
+
* `ProposalResult.proposals`). `warning` is set on a "custom-eval" proposal
|
|
108
|
+
* when the data looks extraction-shaped (judge grading is weaker there).
|
|
98
109
|
*/
|
|
99
110
|
declare class ContractProposal extends BaseModel {
|
|
100
111
|
get taskId(): string | null;
|
|
@@ -103,10 +114,10 @@ declare class ContractProposal extends BaseModel {
|
|
|
103
114
|
get warning(): string | null;
|
|
104
115
|
}
|
|
105
116
|
/**
|
|
106
|
-
*
|
|
107
|
-
* /v1/eval-sets/propose-contract):
|
|
108
|
-
*
|
|
109
|
-
*
|
|
117
|
+
* Pareta's answer for `evals.proposeContract` (POST
|
|
118
|
+
* /v1/eval-sets/propose-contract): how your data could be scored under your
|
|
119
|
+
* stated prompt. Nothing is persisted — confirm by passing the chosen
|
|
120
|
+
* `taskId` (or letting a task-less `create` use a clean single proposal).
|
|
110
121
|
*/
|
|
111
122
|
declare class ProposalResult extends BaseModel {
|
|
112
123
|
get proposals(): ContractProposal[];
|
|
@@ -114,16 +125,17 @@ declare class ProposalResult extends BaseModel {
|
|
|
114
125
|
get split(): Raw | null;
|
|
115
126
|
get conflict(): Raw | null;
|
|
116
127
|
get closestTask(): string | null;
|
|
117
|
-
get
|
|
128
|
+
get prompt(): string | null;
|
|
118
129
|
get message(): string | null;
|
|
119
|
-
/** True when
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
* `task: "custom-eval"`, never
|
|
130
|
+
/** True when exactly ONE homogeneous SPECIFIC task matched at high/medium
|
|
131
|
+
* confidence with no conflict/split — the case a task-less `create`
|
|
132
|
+
* proceeds with, no human confirming. `custom-eval` (a judge panel grading
|
|
133
|
+
* each answer against your prompt) is EXCLUDED: it is offered only when
|
|
134
|
+
* nothing specific fits, and using it is the user's CHOICE — opt in with
|
|
135
|
+
* `task: "custom-eval"`, never silently. */
|
|
125
136
|
get isClean(): boolean;
|
|
126
|
-
/**
|
|
137
|
+
/** How a task-less `create` would score this set, or null if the result
|
|
138
|
+
* isn't clean (pass `task` yourself). */
|
|
127
139
|
get boundTask(): string | null;
|
|
128
140
|
}
|
|
129
141
|
/**
|
|
@@ -319,16 +331,17 @@ declare class Tasks {
|
|
|
319
331
|
/**
|
|
320
332
|
* `client.evals` — eval sets + runs (bring-your-own-data evaluation).
|
|
321
333
|
*
|
|
322
|
-
* // An eval set is DATA +
|
|
323
|
-
* // OPTIONAL —
|
|
324
|
-
*
|
|
325
|
-
*
|
|
334
|
+
* // An eval set is DATA + PROMPT (v3, breaking): prompt REQUIRED, task
|
|
335
|
+
* // OPTIONAL — Pareta works out how to score the results from your
|
|
336
|
+
* // prompt + the data's shape.
|
|
337
|
+
* await pa.evals.proposeContract({ items: [...], prompt: "…" }); // preview
|
|
338
|
+
* const set = await pa.evals.sets.create({ items: [...], prompt: "…" }); // clean match used automatically
|
|
326
339
|
* await pa.evals.sets.uploadDocument(set.id, file, { idx, fieldName }); // blob tasks
|
|
327
340
|
* const run = await pa.evals.runs.create({ evalSet: set.id, models: [...], wait: true });
|
|
328
|
-
* // or, in one call: runs.create({ items,
|
|
341
|
+
* // or, in one call: runs.create({ items, prompt: "…", models, wait: true })
|
|
329
342
|
*
|
|
330
|
-
* `create` (and the inline `runs.create` sugar) require `
|
|
331
|
-
* `task` they call `proposeContract` and
|
|
343
|
+
* `create` (and the inline `runs.create` sugar) require `prompt`; with no
|
|
344
|
+
* `task` they call `proposeContract` and accept ONLY a clean single
|
|
332
345
|
* high/medium match — a conflict, split, or ambiguity throws with the
|
|
333
346
|
* proposals so you pin `task`.
|
|
334
347
|
*
|
|
@@ -346,21 +359,21 @@ declare class EvalSets {
|
|
|
346
359
|
private readonly client;
|
|
347
360
|
constructor(client: Transport);
|
|
348
361
|
/**
|
|
349
|
-
* Persist an eval set from your rows. `
|
|
350
|
-
* on what the model should do with each item. `task` is OPTIONAL — omit it
|
|
351
|
-
*
|
|
352
|
-
* (
|
|
353
|
-
* `task` to pin one explicitly.
|
|
362
|
+
* Persist an eval set from your rows. `prompt` is REQUIRED (v3): one sentence
|
|
363
|
+
* on what the model should do with each item. `task` is OPTIONAL — omit it
|
|
364
|
+
* and Pareta works out how to score the results from your prompt + the
|
|
365
|
+
* data's shape (a clean single match is used; anything ambiguous throws with
|
|
366
|
+
* the proposals). Pass `task` to pin one explicitly.
|
|
354
367
|
*/
|
|
355
368
|
create(params: {
|
|
356
369
|
items: Array<Record<string, unknown>>;
|
|
357
|
-
|
|
370
|
+
prompt: string;
|
|
358
371
|
task?: string;
|
|
359
372
|
name?: string;
|
|
360
373
|
}): Promise<EvalSet>;
|
|
361
|
-
/** Internal: the
|
|
374
|
+
/** Internal: the propose call `create` uses when no task is pinned. Public on
|
|
362
375
|
* `Evals` as `proposeContract`; kept here so `sets.create` can reach it. */
|
|
363
|
-
proposeContract(items: Array<Record<string, unknown>>,
|
|
376
|
+
proposeContract(items: Array<Record<string, unknown>>, prompt: string): Promise<ProposalResult>;
|
|
364
377
|
list(): Promise<EvalSet[]>;
|
|
365
378
|
retrieve(evalSetId: string): Promise<EvalSet>;
|
|
366
379
|
delete(evalSetId: string): Promise<void>;
|
|
@@ -379,7 +392,7 @@ interface EvalRunCreateParams {
|
|
|
379
392
|
evalSet?: string;
|
|
380
393
|
task?: string;
|
|
381
394
|
items?: Array<Record<string, unknown>>;
|
|
382
|
-
|
|
395
|
+
prompt?: string;
|
|
383
396
|
models: string[];
|
|
384
397
|
frontier?: FrontierSpec;
|
|
385
398
|
name?: string;
|
|
@@ -405,14 +418,15 @@ declare class Evals {
|
|
|
405
418
|
constructor(client: Transport);
|
|
406
419
|
private readonly client;
|
|
407
420
|
/**
|
|
408
|
-
*
|
|
421
|
+
* How would your data be scored under your stated `prompt`? Stateless
|
|
409
422
|
* discovery — nothing is persisted. Returns a ProposalResult (ranked
|
|
410
|
-
* proposals, the
|
|
411
|
-
* calls this under the hood; use it directly to
|
|
423
|
+
* proposals, the task a task-less create would use, conflict/split
|
|
424
|
+
* reporting). `sets.create` calls this under the hood; use it directly to
|
|
425
|
+
* preview the scoring first.
|
|
412
426
|
*/
|
|
413
427
|
proposeContract(params: {
|
|
414
428
|
items: Array<Record<string, unknown>>;
|
|
415
|
-
|
|
429
|
+
prompt: string;
|
|
416
430
|
}): Promise<ProposalResult>;
|
|
417
431
|
/**
|
|
418
432
|
* The frontier (vendor) roster you can evaluate against. With `task`, each is
|
|
@@ -632,6 +646,9 @@ interface RequestOptions {
|
|
|
632
646
|
files?: Record<string, FilePart>;
|
|
633
647
|
data?: Record<string, string>;
|
|
634
648
|
cast?: (raw: unknown) => unknown;
|
|
649
|
+
/** Invoked with the response headers on success (before the body is cast) —
|
|
650
|
+
* lets a caller read receipt headers (#164: X-Pareta-Billed + counterfactual). */
|
|
651
|
+
onHeaders?: (headers: Headers) => void;
|
|
635
652
|
}
|
|
636
653
|
interface StreamOptions {
|
|
637
654
|
body?: unknown;
|
|
@@ -698,7 +715,7 @@ declare class Pareta implements Transport {
|
|
|
698
715
|
stream<T = unknown>(method: string, path: string, opts?: StreamOptions): AsyncGenerator<T>;
|
|
699
716
|
}
|
|
700
717
|
|
|
701
|
-
declare const VERSION = "
|
|
718
|
+
declare const VERSION = "3.0.0";
|
|
702
719
|
|
|
703
720
|
/**
|
|
704
721
|
* Typed error hierarchy for the Pareta SDK — a faithful port of the Python
|
package/dist/index.mjs
CHANGED
|
@@ -172,7 +172,7 @@ function parseSSE(reader, opts) {
|
|
|
172
172
|
}
|
|
173
173
|
|
|
174
174
|
// src/version.ts
|
|
175
|
-
var VERSION = "
|
|
175
|
+
var VERSION = "3.0.0";
|
|
176
176
|
|
|
177
177
|
// src/money.ts
|
|
178
178
|
var MICRO_PER_CENT = 10000n;
|
|
@@ -253,6 +253,36 @@ var ChatCompletion = class extends BaseModel {
|
|
|
253
253
|
get usage() {
|
|
254
254
|
return new Usage(this.raw.usage ?? {});
|
|
255
255
|
}
|
|
256
|
+
/** #164 receipt, held OFF `.raw` so `toDict()` stays the raw server JSON. */
|
|
257
|
+
_cost;
|
|
258
|
+
/** #164: attach the per-call receipt from the response headers (billed +
|
|
259
|
+
* the frontier counterfactual, micro-USD ints). Chainable. */
|
|
260
|
+
withCost(headers) {
|
|
261
|
+
const int = (name) => {
|
|
262
|
+
const v = headers?.get(name);
|
|
263
|
+
if (v == null) return null;
|
|
264
|
+
const n = parseInt(v, 10);
|
|
265
|
+
return Number.isNaN(n) ? null : n;
|
|
266
|
+
};
|
|
267
|
+
this._cost = {
|
|
268
|
+
billed: int("X-Pareta-Billed"),
|
|
269
|
+
frontier: int("X-Pareta-Frontier-Would-Have-Cost")
|
|
270
|
+
};
|
|
271
|
+
return this;
|
|
272
|
+
}
|
|
273
|
+
/** What Pareta charged, micro-USD (0 on an idempotent replay); null if absent. */
|
|
274
|
+
get billedMicroUsd() {
|
|
275
|
+
return this._cost?.billed ?? null;
|
|
276
|
+
}
|
|
277
|
+
/** What one list-priced frontier call would have cost, micro-USD. */
|
|
278
|
+
get frontierWouldHaveCostMicroUsd() {
|
|
279
|
+
return this._cost?.frontier ?? null;
|
|
280
|
+
}
|
|
281
|
+
/** frontier / billed — e.g. 17 = 17× cheaper; null if either is missing or billed is 0. */
|
|
282
|
+
get savingsFactor() {
|
|
283
|
+
const b = this.billedMicroUsd, f = this.frontierWouldHaveCostMicroUsd;
|
|
284
|
+
return b && f && b > 0 ? Math.round(f / b * 10) / 10 : null;
|
|
285
|
+
}
|
|
256
286
|
};
|
|
257
287
|
var ChatCompletionChunk = class extends ChatCompletion {
|
|
258
288
|
};
|
|
@@ -340,9 +370,9 @@ var EvalSet = class extends BaseModel {
|
|
|
340
370
|
return this.raw.scoring_strategy ?? null;
|
|
341
371
|
}
|
|
342
372
|
/** The one-sentence success criterion this set was created with (CB1:
|
|
343
|
-
* DATA +
|
|
344
|
-
get
|
|
345
|
-
return this.raw.
|
|
373
|
+
* DATA + PROMPT). Required at create. */
|
|
374
|
+
get prompt() {
|
|
375
|
+
return this.raw.prompt ?? null;
|
|
346
376
|
}
|
|
347
377
|
};
|
|
348
378
|
function evalSetFromCreate(raw) {
|
|
@@ -381,23 +411,24 @@ var ProposalResult = class extends BaseModel {
|
|
|
381
411
|
get closestTask() {
|
|
382
412
|
return this.raw.closest_task ?? null;
|
|
383
413
|
}
|
|
384
|
-
get
|
|
385
|
-
return this.raw.
|
|
414
|
+
get prompt() {
|
|
415
|
+
return this.raw.prompt ?? null;
|
|
386
416
|
}
|
|
387
417
|
get message() {
|
|
388
418
|
return this.raw.message ?? null;
|
|
389
419
|
}
|
|
390
|
-
/** True when
|
|
391
|
-
*
|
|
392
|
-
*
|
|
393
|
-
*
|
|
394
|
-
*
|
|
395
|
-
* `task: "custom-eval"`, never
|
|
420
|
+
/** True when exactly ONE homogeneous SPECIFIC task matched at high/medium
|
|
421
|
+
* confidence with no conflict/split — the case a task-less `create`
|
|
422
|
+
* proceeds with, no human confirming. `custom-eval` (a judge panel grading
|
|
423
|
+
* each answer against your prompt) is EXCLUDED: it is offered only when
|
|
424
|
+
* nothing specific fits, and using it is the user's CHOICE — opt in with
|
|
425
|
+
* `task: "custom-eval"`, never silently. */
|
|
396
426
|
get isClean() {
|
|
397
427
|
const props = this.proposals;
|
|
398
428
|
return this.homogeneous && this.conflict == null && this.split == null && props.length === 1 && (props[0].confidence === "high" || props[0].confidence === "medium") && Boolean(props[0].taskId) && props[0].taskId !== "custom-eval";
|
|
399
429
|
}
|
|
400
|
-
/**
|
|
430
|
+
/** How a task-less `create` would score this set, or null if the result
|
|
431
|
+
* isn't clean (pass `task` yourself). */
|
|
401
432
|
get boundTask() {
|
|
402
433
|
return this.isClean ? this.proposals[0].taskId : null;
|
|
403
434
|
}
|
|
@@ -642,10 +673,14 @@ var Completions = class {
|
|
|
642
673
|
cast: (raw) => new ChatCompletionChunk(raw)
|
|
643
674
|
});
|
|
644
675
|
}
|
|
676
|
+
let headers;
|
|
645
677
|
return this.client.request("POST", PATH, {
|
|
646
678
|
body,
|
|
647
|
-
cast: (raw) => new ChatCompletion(raw)
|
|
648
|
-
|
|
679
|
+
cast: (raw) => new ChatCompletion(raw),
|
|
680
|
+
onHeaders: (h) => {
|
|
681
|
+
headers = h;
|
|
682
|
+
}
|
|
683
|
+
}).then((completion) => completion.withCost(headers));
|
|
649
684
|
}
|
|
650
685
|
};
|
|
651
686
|
var Chat = class {
|
|
@@ -742,37 +777,37 @@ async function toBlob(file, mimeOverride) {
|
|
|
742
777
|
const mime = mimeOverride || "application/octet-stream";
|
|
743
778
|
return { blob: new Blob([bytes], { type: mime }), filename: "upload", size: bytes.byteLength, mime };
|
|
744
779
|
}
|
|
745
|
-
function
|
|
746
|
-
const s = (
|
|
780
|
+
function requirePrompt(prompt) {
|
|
781
|
+
const s = (prompt ?? "").trim();
|
|
747
782
|
if (!s) {
|
|
748
783
|
throw new ParetaError(
|
|
749
|
-
'
|
|
784
|
+
'prompt is required: one sentence describing what the model should do with each item (e.g. "extract vendor, total and date from each invoice"). Pass prompt to evals.create / proposeContract.'
|
|
750
785
|
);
|
|
751
786
|
}
|
|
752
787
|
return s.slice(0, 500);
|
|
753
788
|
}
|
|
754
|
-
function itemsJsonl(task, items,
|
|
789
|
+
function itemsJsonl(task, items, prompt, name) {
|
|
755
790
|
if (!items || items.length === 0) throw new ParetaError("items is required and must be non-empty");
|
|
756
791
|
const jsonl = items.map((it) => JSON.stringify(it)).join("\n");
|
|
757
792
|
return {
|
|
758
793
|
files: { items: { filename: `items.${task}.jsonl`, content: jsonl, contentType: "application/jsonl" } },
|
|
759
|
-
data: { task_id: task,
|
|
794
|
+
data: { task_id: task, prompt, name: name || `sdk eval set (${items.length} items)` }
|
|
760
795
|
};
|
|
761
796
|
}
|
|
762
|
-
function proposeMultipart(items,
|
|
797
|
+
function proposeMultipart(items, prompt) {
|
|
763
798
|
if (!items || items.length === 0) throw new ParetaError("items is required and must be non-empty");
|
|
764
799
|
const jsonl = items.map((it) => JSON.stringify(it)).join("\n");
|
|
765
800
|
return {
|
|
766
801
|
files: { items: { filename: "items.jsonl", content: jsonl, contentType: "application/jsonl" } },
|
|
767
|
-
data: {
|
|
802
|
+
data: { prompt }
|
|
768
803
|
};
|
|
769
804
|
}
|
|
770
805
|
function bindError(result) {
|
|
771
|
-
const
|
|
806
|
+
const prompt = result.prompt ?? "";
|
|
772
807
|
if (result.conflict) {
|
|
773
808
|
const c = result.conflict;
|
|
774
809
|
return new ParetaError(
|
|
775
|
-
`
|
|
810
|
+
`prompt '${prompt}' describes a different job than the data's shape supports (reads as '${c.intended_task}': ${c.reasoning}). Pass a task to pin the task, or revise the prompt.`
|
|
776
811
|
);
|
|
777
812
|
}
|
|
778
813
|
if (result.split) {
|
|
@@ -785,14 +820,14 @@ function bindError(result) {
|
|
|
785
820
|
if (props.length === 1 && props[0].taskId === "custom-eval") {
|
|
786
821
|
const warn = props[0].warning ? ` (${props[0].warning})` : "";
|
|
787
822
|
return new ParetaError(
|
|
788
|
-
`no
|
|
823
|
+
`no ready-made scorer fits this data for prompt '${prompt}'. A judge panel can grade each answer against what you asked for (win rate vs gpt-5.5) \u2014 pass task: "custom-eval" to use it.${warn} Or revise the data/prompt so a specific task fits.`
|
|
789
824
|
);
|
|
790
825
|
}
|
|
791
826
|
const options = props.map((p) => p.taskId).filter(Boolean);
|
|
792
827
|
if (options.length === 0 && result.closestTask) options.push(result.closestTask);
|
|
793
828
|
const hint = options.length ? ` Candidates: ${options.join(", ")}.` : "";
|
|
794
829
|
return new ParetaError(
|
|
795
|
-
`
|
|
830
|
+
`couldn't work out how to score this for prompt '${prompt}'.${hint} Pass a task to choose yourself, or inspect evals.proposeContract({ items, prompt }).`
|
|
796
831
|
);
|
|
797
832
|
}
|
|
798
833
|
function mergeCandidates(models, frontierIds) {
|
|
@@ -816,27 +851,27 @@ var EvalSets = class {
|
|
|
816
851
|
}
|
|
817
852
|
client;
|
|
818
853
|
/**
|
|
819
|
-
* Persist an eval set from your rows. `
|
|
820
|
-
* on what the model should do with each item. `task` is OPTIONAL — omit it
|
|
821
|
-
*
|
|
822
|
-
* (
|
|
823
|
-
* `task` to pin one explicitly.
|
|
854
|
+
* Persist an eval set from your rows. `prompt` is REQUIRED (v3): one sentence
|
|
855
|
+
* on what the model should do with each item. `task` is OPTIONAL — omit it
|
|
856
|
+
* and Pareta works out how to score the results from your prompt + the
|
|
857
|
+
* data's shape (a clean single match is used; anything ambiguous throws with
|
|
858
|
+
* the proposals). Pass `task` to pin one explicitly.
|
|
824
859
|
*/
|
|
825
860
|
async create(params) {
|
|
826
|
-
const
|
|
861
|
+
const prompt = requirePrompt(params.prompt);
|
|
827
862
|
let task = params.task;
|
|
828
863
|
if (task == null) {
|
|
829
|
-
const proposal = await this.proposeContract(params.items,
|
|
864
|
+
const proposal = await this.proposeContract(params.items, prompt);
|
|
830
865
|
task = proposal.boundTask ?? void 0;
|
|
831
866
|
if (task == null) throw bindError(proposal);
|
|
832
867
|
}
|
|
833
|
-
const { files, data } = itemsJsonl(task, params.items,
|
|
868
|
+
const { files, data } = itemsJsonl(task, params.items, prompt, params.name);
|
|
834
869
|
return this.client.request("POST", BASE2, { files, data, cast: evalSetFromCreate });
|
|
835
870
|
}
|
|
836
|
-
/** Internal: the
|
|
871
|
+
/** Internal: the propose call `create` uses when no task is pinned. Public on
|
|
837
872
|
* `Evals` as `proposeContract`; kept here so `sets.create` can reach it. */
|
|
838
|
-
proposeContract(items,
|
|
839
|
-
const { files, data } = proposeMultipart(items,
|
|
873
|
+
proposeContract(items, prompt) {
|
|
874
|
+
const { files, data } = proposeMultipart(items, requirePrompt(prompt));
|
|
840
875
|
return this.client.request("POST", PROPOSE, { files, data, cast: proposalResult });
|
|
841
876
|
}
|
|
842
877
|
list() {
|
|
@@ -907,11 +942,11 @@ var EvalRuns = class {
|
|
|
907
942
|
let evalSet = params.evalSet;
|
|
908
943
|
if (evalSet == null) {
|
|
909
944
|
if (!params.items) {
|
|
910
|
-
throw new ParetaError("pass evalSet: <id>, or items (+
|
|
945
|
+
throw new ParetaError("pass evalSet: <id>, or items (+ prompt) to create one");
|
|
911
946
|
}
|
|
912
947
|
evalSet = (await this.sets.create({
|
|
913
948
|
items: params.items,
|
|
914
|
-
|
|
949
|
+
prompt: params.prompt,
|
|
915
950
|
task: params.task,
|
|
916
951
|
name: params.name
|
|
917
952
|
})).id ?? void 0;
|
|
@@ -955,13 +990,14 @@ var Evals = class {
|
|
|
955
990
|
}
|
|
956
991
|
client;
|
|
957
992
|
/**
|
|
958
|
-
*
|
|
993
|
+
* How would your data be scored under your stated `prompt`? Stateless
|
|
959
994
|
* discovery — nothing is persisted. Returns a ProposalResult (ranked
|
|
960
|
-
* proposals, the
|
|
961
|
-
* calls this under the hood; use it directly to
|
|
995
|
+
* proposals, the task a task-less create would use, conflict/split
|
|
996
|
+
* reporting). `sets.create` calls this under the hood; use it directly to
|
|
997
|
+
* preview the scoring first.
|
|
962
998
|
*/
|
|
963
999
|
proposeContract(params) {
|
|
964
|
-
return this.sets.proposeContract(params.items, params.
|
|
1000
|
+
return this.sets.proposeContract(params.items, params.prompt);
|
|
965
1001
|
}
|
|
966
1002
|
/**
|
|
967
1003
|
* The frontier (vendor) roster you can evaluate against. With `task`, each is
|
|
@@ -1308,6 +1344,7 @@ var Pareta = class _Pareta {
|
|
|
1308
1344
|
break;
|
|
1309
1345
|
}
|
|
1310
1346
|
if (res.ok) {
|
|
1347
|
+
if (opts.onHeaders) opts.onHeaders(res.headers);
|
|
1311
1348
|
const text = await res.text();
|
|
1312
1349
|
const raw = text ? JSON.parse(text) : {};
|
|
1313
1350
|
return cast ? cast(raw) : raw;
|