pareta 1.4.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/index.cjs +149 -8
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +73 -5
- package/dist/index.d.ts +73 -5
- package/dist/index.mjs +148 -9
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -87,6 +87,44 @@ declare class EvalSet extends BaseModel {
|
|
|
87
87
|
get name(): string | null;
|
|
88
88
|
get itemCount(): number | null;
|
|
89
89
|
get scoringStrategy(): string | null;
|
|
90
|
+
/** The one-sentence success criterion this set was created with (CB1:
|
|
91
|
+
* DATA + INTENT). Required at create. */
|
|
92
|
+
get intent(): string | null;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* One proposed grading contract for an uploaded dataset (a row of
|
|
96
|
+
* `ProposalResult.proposals`). `warning` is set on the custom-eval floor when
|
|
97
|
+
* the data looks extraction-shaped (judge grading is weaker there).
|
|
98
|
+
*/
|
|
99
|
+
declare class ContractProposal extends BaseModel {
|
|
100
|
+
get taskId(): string | null;
|
|
101
|
+
get confidence(): string | null;
|
|
102
|
+
get evidence(): Raw;
|
|
103
|
+
get warning(): string | null;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* The binder's answer for `evals.proposeContract` (POST
|
|
107
|
+
* /v1/eval-sets/propose-contract): which grading contract(s) fit your data
|
|
108
|
+
* under your stated intent. Nothing is persisted — confirm by passing the
|
|
109
|
+
* chosen `taskId` (or letting `create` auto-bind a clean single proposal).
|
|
110
|
+
*/
|
|
111
|
+
declare class ProposalResult extends BaseModel {
|
|
112
|
+
get proposals(): ContractProposal[];
|
|
113
|
+
get homogeneous(): boolean;
|
|
114
|
+
get split(): Raw | null;
|
|
115
|
+
get conflict(): Raw | null;
|
|
116
|
+
get closestTask(): string | null;
|
|
117
|
+
get intent(): string | null;
|
|
118
|
+
get message(): string | null;
|
|
119
|
+
/** True when the binder matched exactly ONE homogeneous SPECIFIC contract at
|
|
120
|
+
* high/medium confidence and no conflict/split — the case `create` auto-binds
|
|
121
|
+
* without a human confirming. The `custom-eval` universal FLOOR is EXCLUDED:
|
|
122
|
+
* the binder offers it only when no specific contract fits, and per the
|
|
123
|
+
* precision ladder the floor is a CHOICE — the user opts in with
|
|
124
|
+
* `task: "custom-eval"`, never a silent auto-bind. */
|
|
125
|
+
get isClean(): boolean;
|
|
126
|
+
/** The task `create` would auto-bind, or null if the result isn't clean. */
|
|
127
|
+
get boundTask(): string | null;
|
|
90
128
|
}
|
|
91
129
|
/**
|
|
92
130
|
* One scored item inside `EvalResult.perItem`. `prediction` is the model's raw
|
|
@@ -281,10 +319,18 @@ declare class Tasks {
|
|
|
281
319
|
/**
|
|
282
320
|
* `client.evals` — eval sets + runs (bring-your-own-data evaluation).
|
|
283
321
|
*
|
|
284
|
-
*
|
|
322
|
+
* // An eval set is DATA + INTENT (v2, breaking): intent REQUIRED, task
|
|
323
|
+
* // OPTIONAL — the binder resolves your intent + the data's shape.
|
|
324
|
+
* await pa.evals.proposeContract({ items: [...], intent: "…" }); // preview
|
|
325
|
+
* const set = await pa.evals.sets.create({ items: [...], intent: "…" }); // auto-binds
|
|
285
326
|
* await pa.evals.sets.uploadDocument(set.id, file, { idx, fieldName }); // blob tasks
|
|
286
327
|
* const run = await pa.evals.runs.create({ evalSet: set.id, models: [...], wait: true });
|
|
287
|
-
* // or, in one call: runs.create({
|
|
328
|
+
* // or, in one call: runs.create({ items, intent: "…", models, wait: true })
|
|
329
|
+
*
|
|
330
|
+
* `create` (and the inline `runs.create` sugar) require `intent`; with no
|
|
331
|
+
* `task` they call `proposeContract` and auto-bind ONLY a clean single
|
|
332
|
+
* high/medium match — a conflict, split, or ambiguity throws with the
|
|
333
|
+
* proposals so you pin `task`.
|
|
288
334
|
*
|
|
289
335
|
* Runs are metered (the org balance is debited for compute); `run.cost` is the
|
|
290
336
|
* billed total in dollars (floored to cents). `frontier` accepts an explicit
|
|
@@ -299,11 +345,22 @@ type FileInput = string | Blob | ArrayBuffer | Uint8Array;
|
|
|
299
345
|
declare class EvalSets {
|
|
300
346
|
private readonly client;
|
|
301
347
|
constructor(client: Transport);
|
|
348
|
+
/**
|
|
349
|
+
* Persist an eval set from your rows. `intent` is REQUIRED (v2): one sentence
|
|
350
|
+
* on what the model should do with each item. `task` is OPTIONAL — omit it and
|
|
351
|
+
* the binder resolves your intent + the data's shape to a grading contract
|
|
352
|
+
* (auto-binds a clean single match; throws with the proposals otherwise). Pass
|
|
353
|
+
* `task` to pin one explicitly.
|
|
354
|
+
*/
|
|
302
355
|
create(params: {
|
|
303
|
-
task: string;
|
|
304
356
|
items: Array<Record<string, unknown>>;
|
|
357
|
+
intent: string;
|
|
358
|
+
task?: string;
|
|
305
359
|
name?: string;
|
|
306
360
|
}): Promise<EvalSet>;
|
|
361
|
+
/** Internal: the binder call `create` uses when no task is pinned. Public on
|
|
362
|
+
* `Evals` as `proposeContract`; kept here so `sets.create` can reach it. */
|
|
363
|
+
proposeContract(items: Array<Record<string, unknown>>, intent: string): Promise<ProposalResult>;
|
|
307
364
|
list(): Promise<EvalSet[]>;
|
|
308
365
|
retrieve(evalSetId: string): Promise<EvalSet>;
|
|
309
366
|
delete(evalSetId: string): Promise<void>;
|
|
@@ -322,6 +379,7 @@ interface EvalRunCreateParams {
|
|
|
322
379
|
evalSet?: string;
|
|
323
380
|
task?: string;
|
|
324
381
|
items?: Array<Record<string, unknown>>;
|
|
382
|
+
intent?: string;
|
|
325
383
|
models: string[];
|
|
326
384
|
frontier?: FrontierSpec;
|
|
327
385
|
name?: string;
|
|
@@ -346,6 +404,16 @@ declare class Evals {
|
|
|
346
404
|
readonly runs: EvalRuns;
|
|
347
405
|
constructor(client: Transport);
|
|
348
406
|
private readonly client;
|
|
407
|
+
/**
|
|
408
|
+
* Which grading contract fits your data under your stated `intent`? Stateless
|
|
409
|
+
* discovery — nothing is persisted. Returns a ProposalResult (ranked
|
|
410
|
+
* proposals, the auto-bind decision, conflict/split reporting). `sets.create`
|
|
411
|
+
* calls this under the hood; use it directly to preview the binding first.
|
|
412
|
+
*/
|
|
413
|
+
proposeContract(params: {
|
|
414
|
+
items: Array<Record<string, unknown>>;
|
|
415
|
+
intent: string;
|
|
416
|
+
}): Promise<ProposalResult>;
|
|
349
417
|
/**
|
|
350
418
|
* The frontier (vendor) roster you can evaluate against. With `task`, each is
|
|
351
419
|
* annotated `benchmarked` + the roster is vision-filtered for document tasks.
|
|
@@ -630,7 +698,7 @@ declare class Pareta implements Transport {
|
|
|
630
698
|
stream<T = unknown>(method: string, path: string, opts?: StreamOptions): AsyncGenerator<T>;
|
|
631
699
|
}
|
|
632
700
|
|
|
633
|
-
declare const VERSION = "
|
|
701
|
+
declare const VERSION = "2.0.0";
|
|
634
702
|
|
|
635
703
|
/**
|
|
636
704
|
* Typed error hierarchy for the Pareta SDK — a faithful port of the Python
|
|
@@ -722,4 +790,4 @@ declare class EndpointNotReadyError extends APIStatusError {
|
|
|
722
790
|
constructor(message: string, init: APIStatusErrorInit);
|
|
723
791
|
}
|
|
724
792
|
|
|
725
|
-
export { APIConnectionError, APIStatusError, APITimeoutError, Audio, type AudioInput, AuthenticationError, BadRequestError, BaseModel, ChatCompletion, ChatCompletionChunk, type ChatCompletionCreateParams, type ChatMessage, Choice, ConflictError, Embeddings, type EmbeddingsFn, type EmbeddingsOptions, EndpointNotReadyError, type ErrorDetail, EvalItemResult, EvalResult, EvalRun, type EvalRunCreateParams, EvalRuns, EvalSet, EvalSets, type FileInput, type FilePart, FrontierModel, type FrontierSpec, type ImageEditOptions, type ImageGenerateOptions, ImageGeneration, type ImageInput, Images, InsufficientCreditsError, Message, Model, ModelList, NotFoundError, Pareta, ParetaError, type ParetaOptions, PermissionDeniedError, RateLimitError, Rerank, type RerankFn, type RerankOptions, RerankResult, type SSEEvent, Speech, type SpeechOptions, Task, TaskMatch, TaskMatchCandidate, Transcription, type TranscriptionOptions, type Transport, Usage, VERSION, type ValidationErrorItem };
|
|
793
|
+
export { APIConnectionError, APIStatusError, APITimeoutError, Audio, type AudioInput, AuthenticationError, BadRequestError, BaseModel, ChatCompletion, ChatCompletionChunk, type ChatCompletionCreateParams, type ChatMessage, Choice, ConflictError, ContractProposal, Embeddings, type EmbeddingsFn, type EmbeddingsOptions, EndpointNotReadyError, type ErrorDetail, EvalItemResult, EvalResult, EvalRun, type EvalRunCreateParams, EvalRuns, EvalSet, EvalSets, type FileInput, type FilePart, FrontierModel, type FrontierSpec, type ImageEditOptions, type ImageGenerateOptions, ImageGeneration, type ImageInput, Images, InsufficientCreditsError, Message, Model, ModelList, NotFoundError, Pareta, ParetaError, type ParetaOptions, PermissionDeniedError, ProposalResult, RateLimitError, Rerank, type RerankFn, type RerankOptions, RerankResult, type SSEEvent, Speech, type SpeechOptions, Task, TaskMatch, TaskMatchCandidate, Transcription, type TranscriptionOptions, type Transport, Usage, VERSION, type ValidationErrorItem };
|
package/dist/index.d.ts
CHANGED
|
@@ -87,6 +87,44 @@ declare class EvalSet extends BaseModel {
|
|
|
87
87
|
get name(): string | null;
|
|
88
88
|
get itemCount(): number | null;
|
|
89
89
|
get scoringStrategy(): string | null;
|
|
90
|
+
/** The one-sentence success criterion this set was created with (CB1:
|
|
91
|
+
* DATA + INTENT). Required at create. */
|
|
92
|
+
get intent(): string | null;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* One proposed grading contract for an uploaded dataset (a row of
|
|
96
|
+
* `ProposalResult.proposals`). `warning` is set on the custom-eval floor when
|
|
97
|
+
* the data looks extraction-shaped (judge grading is weaker there).
|
|
98
|
+
*/
|
|
99
|
+
declare class ContractProposal extends BaseModel {
|
|
100
|
+
get taskId(): string | null;
|
|
101
|
+
get confidence(): string | null;
|
|
102
|
+
get evidence(): Raw;
|
|
103
|
+
get warning(): string | null;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* The binder's answer for `evals.proposeContract` (POST
|
|
107
|
+
* /v1/eval-sets/propose-contract): which grading contract(s) fit your data
|
|
108
|
+
* under your stated intent. Nothing is persisted — confirm by passing the
|
|
109
|
+
* chosen `taskId` (or letting `create` auto-bind a clean single proposal).
|
|
110
|
+
*/
|
|
111
|
+
declare class ProposalResult extends BaseModel {
|
|
112
|
+
get proposals(): ContractProposal[];
|
|
113
|
+
get homogeneous(): boolean;
|
|
114
|
+
get split(): Raw | null;
|
|
115
|
+
get conflict(): Raw | null;
|
|
116
|
+
get closestTask(): string | null;
|
|
117
|
+
get intent(): string | null;
|
|
118
|
+
get message(): string | null;
|
|
119
|
+
/** True when the binder matched exactly ONE homogeneous SPECIFIC contract at
|
|
120
|
+
* high/medium confidence and no conflict/split — the case `create` auto-binds
|
|
121
|
+
* without a human confirming. The `custom-eval` universal FLOOR is EXCLUDED:
|
|
122
|
+
* the binder offers it only when no specific contract fits, and per the
|
|
123
|
+
* precision ladder the floor is a CHOICE — the user opts in with
|
|
124
|
+
* `task: "custom-eval"`, never a silent auto-bind. */
|
|
125
|
+
get isClean(): boolean;
|
|
126
|
+
/** The task `create` would auto-bind, or null if the result isn't clean. */
|
|
127
|
+
get boundTask(): string | null;
|
|
90
128
|
}
|
|
91
129
|
/**
|
|
92
130
|
* One scored item inside `EvalResult.perItem`. `prediction` is the model's raw
|
|
@@ -281,10 +319,18 @@ declare class Tasks {
|
|
|
281
319
|
/**
|
|
282
320
|
* `client.evals` — eval sets + runs (bring-your-own-data evaluation).
|
|
283
321
|
*
|
|
284
|
-
*
|
|
322
|
+
* // An eval set is DATA + INTENT (v2, breaking): intent REQUIRED, task
|
|
323
|
+
* // OPTIONAL — the binder resolves your intent + the data's shape.
|
|
324
|
+
* await pa.evals.proposeContract({ items: [...], intent: "…" }); // preview
|
|
325
|
+
* const set = await pa.evals.sets.create({ items: [...], intent: "…" }); // auto-binds
|
|
285
326
|
* await pa.evals.sets.uploadDocument(set.id, file, { idx, fieldName }); // blob tasks
|
|
286
327
|
* const run = await pa.evals.runs.create({ evalSet: set.id, models: [...], wait: true });
|
|
287
|
-
* // or, in one call: runs.create({
|
|
328
|
+
* // or, in one call: runs.create({ items, intent: "…", models, wait: true })
|
|
329
|
+
*
|
|
330
|
+
* `create` (and the inline `runs.create` sugar) require `intent`; with no
|
|
331
|
+
* `task` they call `proposeContract` and auto-bind ONLY a clean single
|
|
332
|
+
* high/medium match — a conflict, split, or ambiguity throws with the
|
|
333
|
+
* proposals so you pin `task`.
|
|
288
334
|
*
|
|
289
335
|
* Runs are metered (the org balance is debited for compute); `run.cost` is the
|
|
290
336
|
* billed total in dollars (floored to cents). `frontier` accepts an explicit
|
|
@@ -299,11 +345,22 @@ type FileInput = string | Blob | ArrayBuffer | Uint8Array;
|
|
|
299
345
|
declare class EvalSets {
|
|
300
346
|
private readonly client;
|
|
301
347
|
constructor(client: Transport);
|
|
348
|
+
/**
|
|
349
|
+
* Persist an eval set from your rows. `intent` is REQUIRED (v2): one sentence
|
|
350
|
+
* on what the model should do with each item. `task` is OPTIONAL — omit it and
|
|
351
|
+
* the binder resolves your intent + the data's shape to a grading contract
|
|
352
|
+
* (auto-binds a clean single match; throws with the proposals otherwise). Pass
|
|
353
|
+
* `task` to pin one explicitly.
|
|
354
|
+
*/
|
|
302
355
|
create(params: {
|
|
303
|
-
task: string;
|
|
304
356
|
items: Array<Record<string, unknown>>;
|
|
357
|
+
intent: string;
|
|
358
|
+
task?: string;
|
|
305
359
|
name?: string;
|
|
306
360
|
}): Promise<EvalSet>;
|
|
361
|
+
/** Internal: the binder call `create` uses when no task is pinned. Public on
|
|
362
|
+
* `Evals` as `proposeContract`; kept here so `sets.create` can reach it. */
|
|
363
|
+
proposeContract(items: Array<Record<string, unknown>>, intent: string): Promise<ProposalResult>;
|
|
307
364
|
list(): Promise<EvalSet[]>;
|
|
308
365
|
retrieve(evalSetId: string): Promise<EvalSet>;
|
|
309
366
|
delete(evalSetId: string): Promise<void>;
|
|
@@ -322,6 +379,7 @@ interface EvalRunCreateParams {
|
|
|
322
379
|
evalSet?: string;
|
|
323
380
|
task?: string;
|
|
324
381
|
items?: Array<Record<string, unknown>>;
|
|
382
|
+
intent?: string;
|
|
325
383
|
models: string[];
|
|
326
384
|
frontier?: FrontierSpec;
|
|
327
385
|
name?: string;
|
|
@@ -346,6 +404,16 @@ declare class Evals {
|
|
|
346
404
|
readonly runs: EvalRuns;
|
|
347
405
|
constructor(client: Transport);
|
|
348
406
|
private readonly client;
|
|
407
|
+
/**
|
|
408
|
+
* Which grading contract fits your data under your stated `intent`? Stateless
|
|
409
|
+
* discovery — nothing is persisted. Returns a ProposalResult (ranked
|
|
410
|
+
* proposals, the auto-bind decision, conflict/split reporting). `sets.create`
|
|
411
|
+
* calls this under the hood; use it directly to preview the binding first.
|
|
412
|
+
*/
|
|
413
|
+
proposeContract(params: {
|
|
414
|
+
items: Array<Record<string, unknown>>;
|
|
415
|
+
intent: string;
|
|
416
|
+
}): Promise<ProposalResult>;
|
|
349
417
|
/**
|
|
350
418
|
* The frontier (vendor) roster you can evaluate against. With `task`, each is
|
|
351
419
|
* annotated `benchmarked` + the roster is vision-filtered for document tasks.
|
|
@@ -630,7 +698,7 @@ declare class Pareta implements Transport {
|
|
|
630
698
|
stream<T = unknown>(method: string, path: string, opts?: StreamOptions): AsyncGenerator<T>;
|
|
631
699
|
}
|
|
632
700
|
|
|
633
|
-
declare const VERSION = "
|
|
701
|
+
declare const VERSION = "2.0.0";
|
|
634
702
|
|
|
635
703
|
/**
|
|
636
704
|
* Typed error hierarchy for the Pareta SDK — a faithful port of the Python
|
|
@@ -722,4 +790,4 @@ declare class EndpointNotReadyError extends APIStatusError {
|
|
|
722
790
|
constructor(message: string, init: APIStatusErrorInit);
|
|
723
791
|
}
|
|
724
792
|
|
|
725
|
-
export { APIConnectionError, APIStatusError, APITimeoutError, Audio, type AudioInput, AuthenticationError, BadRequestError, BaseModel, ChatCompletion, ChatCompletionChunk, type ChatCompletionCreateParams, type ChatMessage, Choice, ConflictError, Embeddings, type EmbeddingsFn, type EmbeddingsOptions, EndpointNotReadyError, type ErrorDetail, EvalItemResult, EvalResult, EvalRun, type EvalRunCreateParams, EvalRuns, EvalSet, EvalSets, type FileInput, type FilePart, FrontierModel, type FrontierSpec, type ImageEditOptions, type ImageGenerateOptions, ImageGeneration, type ImageInput, Images, InsufficientCreditsError, Message, Model, ModelList, NotFoundError, Pareta, ParetaError, type ParetaOptions, PermissionDeniedError, RateLimitError, Rerank, type RerankFn, type RerankOptions, RerankResult, type SSEEvent, Speech, type SpeechOptions, Task, TaskMatch, TaskMatchCandidate, Transcription, type TranscriptionOptions, type Transport, Usage, VERSION, type ValidationErrorItem };
|
|
793
|
+
export { APIConnectionError, APIStatusError, APITimeoutError, Audio, type AudioInput, AuthenticationError, BadRequestError, BaseModel, ChatCompletion, ChatCompletionChunk, type ChatCompletionCreateParams, type ChatMessage, Choice, ConflictError, ContractProposal, Embeddings, type EmbeddingsFn, type EmbeddingsOptions, EndpointNotReadyError, type ErrorDetail, EvalItemResult, EvalResult, EvalRun, type EvalRunCreateParams, EvalRuns, EvalSet, EvalSets, type FileInput, type FilePart, FrontierModel, type FrontierSpec, type ImageEditOptions, type ImageGenerateOptions, ImageGeneration, type ImageInput, Images, InsufficientCreditsError, Message, Model, ModelList, NotFoundError, Pareta, ParetaError, type ParetaOptions, PermissionDeniedError, ProposalResult, RateLimitError, Rerank, type RerankFn, type RerankOptions, RerankResult, type SSEEvent, Speech, type SpeechOptions, Task, TaskMatch, TaskMatchCandidate, Transcription, type TranscriptionOptions, type Transport, Usage, VERSION, type ValidationErrorItem };
|
package/dist/index.mjs
CHANGED
|
@@ -172,7 +172,7 @@ function parseSSE(reader, opts) {
|
|
|
172
172
|
}
|
|
173
173
|
|
|
174
174
|
// src/version.ts
|
|
175
|
-
var VERSION = "
|
|
175
|
+
var VERSION = "2.0.0";
|
|
176
176
|
|
|
177
177
|
// src/money.ts
|
|
178
178
|
var MICRO_PER_CENT = 10000n;
|
|
@@ -339,6 +339,11 @@ var EvalSet = class extends BaseModel {
|
|
|
339
339
|
get scoringStrategy() {
|
|
340
340
|
return this.raw.scoring_strategy ?? null;
|
|
341
341
|
}
|
|
342
|
+
/** The one-sentence success criterion this set was created with (CB1:
|
|
343
|
+
* DATA + INTENT). Required at create. */
|
|
344
|
+
get intent() {
|
|
345
|
+
return this.raw.intent ?? null;
|
|
346
|
+
}
|
|
342
347
|
};
|
|
343
348
|
function evalSetFromCreate(raw) {
|
|
344
349
|
return new EvalSet(raw?.eval_set ?? {});
|
|
@@ -346,6 +351,60 @@ function evalSetFromCreate(raw) {
|
|
|
346
351
|
function evalSetList(raw) {
|
|
347
352
|
return (raw?.eval_sets ?? []).map((e) => new EvalSet(e));
|
|
348
353
|
}
|
|
354
|
+
var ContractProposal = class extends BaseModel {
|
|
355
|
+
get taskId() {
|
|
356
|
+
return this.raw.task_id ?? null;
|
|
357
|
+
}
|
|
358
|
+
get confidence() {
|
|
359
|
+
return this.raw.confidence ?? null;
|
|
360
|
+
}
|
|
361
|
+
get evidence() {
|
|
362
|
+
return this.raw.evidence ?? {};
|
|
363
|
+
}
|
|
364
|
+
get warning() {
|
|
365
|
+
return this.raw.warning ?? null;
|
|
366
|
+
}
|
|
367
|
+
};
|
|
368
|
+
var ProposalResult = class extends BaseModel {
|
|
369
|
+
get proposals() {
|
|
370
|
+
return (this.raw.proposals ?? []).map((p) => new ContractProposal(p));
|
|
371
|
+
}
|
|
372
|
+
get homogeneous() {
|
|
373
|
+
return Boolean(this.raw.homogeneous);
|
|
374
|
+
}
|
|
375
|
+
get split() {
|
|
376
|
+
return this.raw.split ?? null;
|
|
377
|
+
}
|
|
378
|
+
get conflict() {
|
|
379
|
+
return this.raw.conflict ?? null;
|
|
380
|
+
}
|
|
381
|
+
get closestTask() {
|
|
382
|
+
return this.raw.closest_task ?? null;
|
|
383
|
+
}
|
|
384
|
+
get intent() {
|
|
385
|
+
return this.raw.intent ?? null;
|
|
386
|
+
}
|
|
387
|
+
get message() {
|
|
388
|
+
return this.raw.message ?? null;
|
|
389
|
+
}
|
|
390
|
+
/** True when the binder matched exactly ONE homogeneous SPECIFIC contract at
|
|
391
|
+
* high/medium confidence and no conflict/split — the case `create` auto-binds
|
|
392
|
+
* without a human confirming. The `custom-eval` universal FLOOR is EXCLUDED:
|
|
393
|
+
* the binder offers it only when no specific contract fits, and per the
|
|
394
|
+
* precision ladder the floor is a CHOICE — the user opts in with
|
|
395
|
+
* `task: "custom-eval"`, never a silent auto-bind. */
|
|
396
|
+
get isClean() {
|
|
397
|
+
const props = this.proposals;
|
|
398
|
+
return this.homogeneous && this.conflict == null && this.split == null && props.length === 1 && (props[0].confidence === "high" || props[0].confidence === "medium") && Boolean(props[0].taskId) && props[0].taskId !== "custom-eval";
|
|
399
|
+
}
|
|
400
|
+
/** The task `create` would auto-bind, or null if the result isn't clean. */
|
|
401
|
+
get boundTask() {
|
|
402
|
+
return this.isClean ? this.proposals[0].taskId : null;
|
|
403
|
+
}
|
|
404
|
+
};
|
|
405
|
+
function proposalResult(raw) {
|
|
406
|
+
return new ProposalResult(raw ?? {});
|
|
407
|
+
}
|
|
349
408
|
var EvalItemResult = class extends BaseModel {
|
|
350
409
|
get idx() {
|
|
351
410
|
return this.raw.idx ?? null;
|
|
@@ -639,6 +698,7 @@ var Tasks = class {
|
|
|
639
698
|
|
|
640
699
|
// src/resources/evals.ts
|
|
641
700
|
var BASE2 = "/v1/eval-sets";
|
|
701
|
+
var PROPOSE = "/v1/eval-sets/propose-contract";
|
|
642
702
|
var RUNS = "/v1/eval-runs";
|
|
643
703
|
var FRONTIER = "/v1/eval/frontier-models";
|
|
644
704
|
var INLINE_MAX = 5 * 1024 * 1024;
|
|
@@ -682,14 +742,59 @@ async function toBlob(file, mimeOverride) {
|
|
|
682
742
|
const mime = mimeOverride || "application/octet-stream";
|
|
683
743
|
return { blob: new Blob([bytes], { type: mime }), filename: "upload", size: bytes.byteLength, mime };
|
|
684
744
|
}
|
|
685
|
-
function
|
|
745
|
+
function requireIntent(intent) {
|
|
746
|
+
const s = (intent ?? "").trim();
|
|
747
|
+
if (!s) {
|
|
748
|
+
throw new ParetaError(
|
|
749
|
+
'intent is required: one sentence describing what the model should do with each item (e.g. "extract vendor, total and date from each invoice"). Pass intent to evals.create / proposeContract.'
|
|
750
|
+
);
|
|
751
|
+
}
|
|
752
|
+
return s.slice(0, 500);
|
|
753
|
+
}
|
|
754
|
+
function itemsJsonl(task, items, intent, name) {
|
|
686
755
|
if (!items || items.length === 0) throw new ParetaError("items is required and must be non-empty");
|
|
687
756
|
const jsonl = items.map((it) => JSON.stringify(it)).join("\n");
|
|
688
757
|
return {
|
|
689
758
|
files: { items: { filename: `items.${task}.jsonl`, content: jsonl, contentType: "application/jsonl" } },
|
|
690
|
-
data: { task_id: task, name: name || `sdk eval set (${items.length} items)` }
|
|
759
|
+
data: { task_id: task, intent, name: name || `sdk eval set (${items.length} items)` }
|
|
691
760
|
};
|
|
692
761
|
}
|
|
762
|
+
function proposeMultipart(items, intent) {
|
|
763
|
+
if (!items || items.length === 0) throw new ParetaError("items is required and must be non-empty");
|
|
764
|
+
const jsonl = items.map((it) => JSON.stringify(it)).join("\n");
|
|
765
|
+
return {
|
|
766
|
+
files: { items: { filename: "items.jsonl", content: jsonl, contentType: "application/jsonl" } },
|
|
767
|
+
data: { intent }
|
|
768
|
+
};
|
|
769
|
+
}
|
|
770
|
+
function bindError(result) {
|
|
771
|
+
const intent = result.intent ?? "";
|
|
772
|
+
if (result.conflict) {
|
|
773
|
+
const c = result.conflict;
|
|
774
|
+
return new ParetaError(
|
|
775
|
+
`intent '${intent}' describes a different job than the data's shape supports (reads as '${c.intended_task}': ${c.reasoning}). Pass a task to pin a contract, or revise the intent.`
|
|
776
|
+
);
|
|
777
|
+
}
|
|
778
|
+
if (result.split) {
|
|
779
|
+
const s = result.split;
|
|
780
|
+
return new ParetaError(
|
|
781
|
+
`the dataset looks MIXED \u2014 ${s.validated_n}/${s.total_n} items fit '${s.closest_task}', the rest a different shape. Split the set or pass a task.`
|
|
782
|
+
);
|
|
783
|
+
}
|
|
784
|
+
const props = result.proposals;
|
|
785
|
+
if (props.length === 1 && props[0].taskId === "custom-eval") {
|
|
786
|
+
const warn = props[0].warning ? ` (${props[0].warning})` : "";
|
|
787
|
+
return new ParetaError(
|
|
788
|
+
`no specific grading contract fits this data for intent '${intent}'. The custom-eval universal floor is available \u2014 it grades by judged win-rate vs the frontier anchor.${warn} Pass task: "custom-eval" to use it, or revise the data/intent for a specific contract.`
|
|
789
|
+
);
|
|
790
|
+
}
|
|
791
|
+
const options = props.map((p) => p.taskId).filter(Boolean);
|
|
792
|
+
if (options.length === 0 && result.closestTask) options.push(result.closestTask);
|
|
793
|
+
const hint = options.length ? ` Candidates: ${options.join(", ")}.` : "";
|
|
794
|
+
return new ParetaError(
|
|
795
|
+
`could not confidently bind a grading contract for intent '${intent}'.${hint} Pass a task to pin one, or inspect evals.proposeContract({ items, intent }).`
|
|
796
|
+
);
|
|
797
|
+
}
|
|
693
798
|
function mergeCandidates(models, frontierIds) {
|
|
694
799
|
const cands = [...models ?? [], ...frontierIds ?? []];
|
|
695
800
|
if (cands.length === 0) throw new ParetaError("models is required (the open candidates to evaluate)");
|
|
@@ -710,10 +815,30 @@ var EvalSets = class {
|
|
|
710
815
|
this.client = client;
|
|
711
816
|
}
|
|
712
817
|
client;
|
|
713
|
-
|
|
714
|
-
|
|
818
|
+
/**
|
|
819
|
+
* Persist an eval set from your rows. `intent` is REQUIRED (v2): one sentence
|
|
820
|
+
* on what the model should do with each item. `task` is OPTIONAL — omit it and
|
|
821
|
+
* the binder resolves your intent + the data's shape to a grading contract
|
|
822
|
+
* (auto-binds a clean single match; throws with the proposals otherwise). Pass
|
|
823
|
+
* `task` to pin one explicitly.
|
|
824
|
+
*/
|
|
825
|
+
async create(params) {
|
|
826
|
+
const intent = requireIntent(params.intent);
|
|
827
|
+
let task = params.task;
|
|
828
|
+
if (task == null) {
|
|
829
|
+
const proposal = await this.proposeContract(params.items, intent);
|
|
830
|
+
task = proposal.boundTask ?? void 0;
|
|
831
|
+
if (task == null) throw bindError(proposal);
|
|
832
|
+
}
|
|
833
|
+
const { files, data } = itemsJsonl(task, params.items, intent, params.name);
|
|
715
834
|
return this.client.request("POST", BASE2, { files, data, cast: evalSetFromCreate });
|
|
716
835
|
}
|
|
836
|
+
/** Internal: the binder call `create` uses when no task is pinned. Public on
|
|
837
|
+
* `Evals` as `proposeContract`; kept here so `sets.create` can reach it. */
|
|
838
|
+
proposeContract(items, intent) {
|
|
839
|
+
const { files, data } = proposeMultipart(items, requireIntent(intent));
|
|
840
|
+
return this.client.request("POST", PROPOSE, { files, data, cast: proposalResult });
|
|
841
|
+
}
|
|
717
842
|
list() {
|
|
718
843
|
return this.client.request("GET", BASE2, { cast: evalSetList });
|
|
719
844
|
}
|
|
@@ -781,10 +906,15 @@ var EvalRuns = class {
|
|
|
781
906
|
async create(params) {
|
|
782
907
|
let evalSet = params.evalSet;
|
|
783
908
|
if (evalSet == null) {
|
|
784
|
-
if (!
|
|
785
|
-
throw new ParetaError("pass evalSet: <id>, or
|
|
909
|
+
if (!params.items) {
|
|
910
|
+
throw new ParetaError("pass evalSet: <id>, or items (+ intent) to create one");
|
|
786
911
|
}
|
|
787
|
-
evalSet = (await this.sets.create({
|
|
912
|
+
evalSet = (await this.sets.create({
|
|
913
|
+
items: params.items,
|
|
914
|
+
intent: params.intent,
|
|
915
|
+
task: params.task,
|
|
916
|
+
name: params.name
|
|
917
|
+
})).id ?? void 0;
|
|
788
918
|
}
|
|
789
919
|
const frontierIds = await this.frontierIds(params.frontier, evalSet, params.task);
|
|
790
920
|
const candidateModelIds = mergeCandidates(params.models, frontierIds);
|
|
@@ -824,6 +954,15 @@ var Evals = class {
|
|
|
824
954
|
this.client = client;
|
|
825
955
|
}
|
|
826
956
|
client;
|
|
957
|
+
/**
|
|
958
|
+
* Which grading contract fits your data under your stated `intent`? Stateless
|
|
959
|
+
* discovery — nothing is persisted. Returns a ProposalResult (ranked
|
|
960
|
+
* proposals, the auto-bind decision, conflict/split reporting). `sets.create`
|
|
961
|
+
* calls this under the hood; use it directly to preview the binding first.
|
|
962
|
+
*/
|
|
963
|
+
proposeContract(params) {
|
|
964
|
+
return this.sets.proposeContract(params.items, params.intent);
|
|
965
|
+
}
|
|
827
966
|
/**
|
|
828
967
|
* The frontier (vendor) roster you can evaluate against. With `task`, each is
|
|
829
968
|
* annotated `benchmarked` + the roster is vision-filtered for document tasks.
|
|
@@ -1235,6 +1374,6 @@ var Pareta = class _Pareta {
|
|
|
1235
1374
|
}
|
|
1236
1375
|
};
|
|
1237
1376
|
|
|
1238
|
-
export { APIConnectionError, APIStatusError, APITimeoutError, Audio, AuthenticationError, BadRequestError, BaseModel, ChatCompletion, ChatCompletionChunk, Choice, ConflictError, Embeddings, EndpointNotReadyError, EvalItemResult, EvalResult, EvalRun, EvalRuns, EvalSet, EvalSets, FrontierModel, ImageGeneration, Images, InsufficientCreditsError, Message, Model, ModelList, NotFoundError, Pareta, ParetaError, PermissionDeniedError, RateLimitError, Rerank, RerankResult, Speech, Task, TaskMatch, TaskMatchCandidate, Transcription, Usage, VERSION };
|
|
1377
|
+
export { APIConnectionError, APIStatusError, APITimeoutError, Audio, AuthenticationError, BadRequestError, BaseModel, ChatCompletion, ChatCompletionChunk, Choice, ConflictError, ContractProposal, Embeddings, EndpointNotReadyError, EvalItemResult, EvalResult, EvalRun, EvalRuns, EvalSet, EvalSets, FrontierModel, ImageGeneration, Images, InsufficientCreditsError, Message, Model, ModelList, NotFoundError, Pareta, ParetaError, PermissionDeniedError, ProposalResult, RateLimitError, Rerank, RerankResult, Speech, Task, TaskMatch, TaskMatchCandidate, Transcription, Usage, VERSION };
|
|
1239
1378
|
//# sourceMappingURL=index.mjs.map
|
|
1240
1379
|
//# sourceMappingURL=index.mjs.map
|