pareta 1.4.0 → 2.0.0

package/README.md

@@ -40,7 +40,9 @@ head-to-head against frontier models on your own ground truth and prices every
 contender honestly:
 
 ```ts
-const set = await pa.evals.sets.create({ task: "invoice-extraction", items: [...] });
+const set = await pa.evals.sets.create({
+  items: [...], intent: "extract vendor, total and date from each invoice",
+});
 const run = await pa.evals.runs.create({
   evalSet: set.id, models: ["auto"], frontier: ["claude-opus-4-7"], wait: true,
 });

package/dist/index.cjs

@@ -174,7 +174,7 @@ function parseSSE(reader, opts) {
 }
 
 // src/version.ts
-var VERSION = "1.4.0";
+var VERSION = "2.0.0";
 
 // src/money.ts
 var MICRO_PER_CENT = 10000n;
@@ -341,6 +341,11 @@ var EvalSet = class extends BaseModel {
   get scoringStrategy() {
     return this.raw.scoring_strategy ?? null;
   }
+  /** The one-sentence success criterion this set was created with (CB1:
+   * DATA + INTENT). Required at create. */
+  get intent() {
+    return this.raw.intent ?? null;
+  }
 };
 function evalSetFromCreate(raw) {
   return new EvalSet(raw?.eval_set ?? {});
@@ -348,6 +353,60 @@ function evalSetFromCreate(raw) {
 function evalSetList(raw) {
   return (raw?.eval_sets ?? []).map((e) => new EvalSet(e));
 }
+var ContractProposal = class extends BaseModel {
+  get taskId() {
+    return this.raw.task_id ?? null;
+  }
+  get confidence() {
+    return this.raw.confidence ?? null;
+  }
+  get evidence() {
+    return this.raw.evidence ?? {};
+  }
+  get warning() {
+    return this.raw.warning ?? null;
+  }
+};
+var ProposalResult = class extends BaseModel {
+  get proposals() {
+    return (this.raw.proposals ?? []).map((p) => new ContractProposal(p));
+  }
+  get homogeneous() {
+    return Boolean(this.raw.homogeneous);
+  }
+  get split() {
+    return this.raw.split ?? null;
+  }
+  get conflict() {
+    return this.raw.conflict ?? null;
+  }
+  get closestTask() {
+    return this.raw.closest_task ?? null;
+  }
+  get intent() {
+    return this.raw.intent ?? null;
+  }
+  get message() {
+    return this.raw.message ?? null;
+  }
+  /** True when the binder matched exactly ONE homogeneous SPECIFIC contract at
+   * high/medium confidence and no conflict/split — the case `create` auto-binds
+   * without a human confirming. The `custom-eval` universal FLOOR is EXCLUDED:
+   * the binder offers it only when no specific contract fits, and per the
+   * precision ladder the floor is a CHOICE — the user opts in with
+   * `task: "custom-eval"`, never a silent auto-bind. */
+  get isClean() {
+    const props = this.proposals;
+    return this.homogeneous && this.conflict == null && this.split == null && props.length === 1 && (props[0].confidence === "high" || props[0].confidence === "medium") && Boolean(props[0].taskId) && props[0].taskId !== "custom-eval";
+  }
+  /** The task `create` would auto-bind, or null if the result isn't clean. */
+  get boundTask() {
+    return this.isClean ? this.proposals[0].taskId : null;
+  }
+};
+function proposalResult(raw) {
+  return new ProposalResult(raw ?? {});
+}
 var EvalItemResult = class extends BaseModel {
   get idx() {
     return this.raw.idx ?? null;
@@ -641,6 +700,7 @@ var Tasks = class {
 
 // src/resources/evals.ts
 var BASE2 = "/v1/eval-sets";
+var PROPOSE = "/v1/eval-sets/propose-contract";
 var RUNS = "/v1/eval-runs";
 var FRONTIER = "/v1/eval/frontier-models";
 var INLINE_MAX = 5 * 1024 * 1024;
@@ -684,14 +744,59 @@ async function toBlob(file, mimeOverride) {
   const mime = mimeOverride || "application/octet-stream";
   return { blob: new Blob([bytes], { type: mime }), filename: "upload", size: bytes.byteLength, mime };
 }
-function itemsJsonl(task, items, name) {
+function requireIntent(intent) {
+  const s = (intent ?? "").trim();
+  if (!s) {
+    throw new ParetaError(
+      'intent is required: one sentence describing what the model should do with each item (e.g. "extract vendor, total and date from each invoice"). Pass intent to evals.create / proposeContract.'
+    );
+  }
+  return s.slice(0, 500);
+}
+function itemsJsonl(task, items, intent, name) {
   if (!items || items.length === 0) throw new ParetaError("items is required and must be non-empty");
   const jsonl = items.map((it) => JSON.stringify(it)).join("\n");
   return {
     files: { items: { filename: `items.${task}.jsonl`, content: jsonl, contentType: "application/jsonl" } },
-    data: { task_id: task, name: name || `sdk eval set (${items.length} items)` }
+    data: { task_id: task, intent, name: name || `sdk eval set (${items.length} items)` }
   };
 }
+function proposeMultipart(items, intent) {
+  if (!items || items.length === 0) throw new ParetaError("items is required and must be non-empty");
+  const jsonl = items.map((it) => JSON.stringify(it)).join("\n");
+  return {
+    files: { items: { filename: "items.jsonl", content: jsonl, contentType: "application/jsonl" } },
+    data: { intent }
+  };
+}
+function bindError(result) {
+  const intent = result.intent ?? "";
+  if (result.conflict) {
+    const c = result.conflict;
+    return new ParetaError(
+      `intent '${intent}' describes a different job than the data's shape supports (reads as '${c.intended_task}': ${c.reasoning}). Pass a task to pin a contract, or revise the intent.`
+    );
+  }
+  if (result.split) {
+    const s = result.split;
+    return new ParetaError(
+      `the dataset looks MIXED \u2014 ${s.validated_n}/${s.total_n} items fit '${s.closest_task}', the rest a different shape. Split the set or pass a task.`
+    );
+  }
+  const props = result.proposals;
+  if (props.length === 1 && props[0].taskId === "custom-eval") {
+    const warn = props[0].warning ? ` (${props[0].warning})` : "";
+    return new ParetaError(
+      `no specific grading contract fits this data for intent '${intent}'. The custom-eval universal floor is available \u2014 it grades by judged win-rate vs the frontier anchor.${warn} Pass task: "custom-eval" to use it, or revise the data/intent for a specific contract.`
+    );
+  }
+  const options = props.map((p) => p.taskId).filter(Boolean);
+  if (options.length === 0 && result.closestTask) options.push(result.closestTask);
+  const hint = options.length ? ` Candidates: ${options.join(", ")}.` : "";
+  return new ParetaError(
+    `could not confidently bind a grading contract for intent '${intent}'.${hint} Pass a task to pin one, or inspect evals.proposeContract({ items, intent }).`
+  );
+}
 function mergeCandidates(models, frontierIds) {
   const cands = [...models ?? [], ...frontierIds ?? []];
   if (cands.length === 0) throw new ParetaError("models is required (the open candidates to evaluate)");
@@ -712,10 +817,30 @@ var EvalSets = class {
     this.client = client;
   }
   client;
-  create(params) {
-    const { files, data } = itemsJsonl(params.task, params.items, params.name);
+  /**
+   * Persist an eval set from your rows. `intent` is REQUIRED (v2): one sentence
+   * on what the model should do with each item. `task` is OPTIONAL — omit it and
+   * the binder resolves your intent + the data's shape to a grading contract
+   * (auto-binds a clean single match; throws with the proposals otherwise). Pass
+   * `task` to pin one explicitly.
+   */
+  async create(params) {
+    const intent = requireIntent(params.intent);
+    let task = params.task;
+    if (task == null) {
+      const proposal = await this.proposeContract(params.items, intent);
+      task = proposal.boundTask ?? void 0;
+      if (task == null) throw bindError(proposal);
+    }
+    const { files, data } = itemsJsonl(task, params.items, intent, params.name);
     return this.client.request("POST", BASE2, { files, data, cast: evalSetFromCreate });
   }
+  /** Internal: the binder call `create` uses when no task is pinned. Public on
+   * `Evals` as `proposeContract`; kept here so `sets.create` can reach it. */
+  proposeContract(items, intent) {
+    const { files, data } = proposeMultipart(items, requireIntent(intent));
+    return this.client.request("POST", PROPOSE, { files, data, cast: proposalResult });
+  }
   list() {
     return this.client.request("GET", BASE2, { cast: evalSetList });
   }
@@ -783,10 +908,15 @@ var EvalRuns = class {
   async create(params) {
     let evalSet = params.evalSet;
     if (evalSet == null) {
-      if (!(params.task && params.items)) {
-        throw new ParetaError("pass evalSet: <id>, or task + items to create one");
+      if (!params.items) {
+        throw new ParetaError("pass evalSet: <id>, or items (+ intent) to create one");
       }
-      evalSet = (await this.sets.create({ task: params.task, items: params.items, name: params.name })).id ?? void 0;
+      evalSet = (await this.sets.create({
+        items: params.items,
+        intent: params.intent,
+        task: params.task,
+        name: params.name
+      })).id ?? void 0;
     }
     const frontierIds = await this.frontierIds(params.frontier, evalSet, params.task);
     const candidateModelIds = mergeCandidates(params.models, frontierIds);
@@ -826,6 +956,15 @@ var Evals = class {
     this.client = client;
   }
   client;
+  /**
+   * Which grading contract fits your data under your stated `intent`? Stateless
+   * discovery — nothing is persisted. Returns a ProposalResult (ranked
+   * proposals, the auto-bind decision, conflict/split reporting). `sets.create`
+   * calls this under the hood; use it directly to preview the binding first.
+   */
+  proposeContract(params) {
+    return this.sets.proposeContract(params.items, params.intent);
+  }
   /**
    * The frontier (vendor) roster you can evaluate against. With `task`, each is
    * annotated `benchmarked` + the roster is vision-filtered for document tasks.
@@ -1248,6 +1387,7 @@ exports.ChatCompletion = ChatCompletion;
 exports.ChatCompletionChunk = ChatCompletionChunk;
 exports.Choice = Choice;
 exports.ConflictError = ConflictError;
+exports.ContractProposal = ContractProposal;
 exports.Embeddings = Embeddings;
 exports.EndpointNotReadyError = EndpointNotReadyError;
 exports.EvalItemResult = EvalItemResult;
@@ -1267,6 +1407,7 @@ exports.NotFoundError = NotFoundError;
 exports.Pareta = Pareta;
 exports.ParetaError = ParetaError;
 exports.PermissionDeniedError = PermissionDeniedError;
+exports.ProposalResult = ProposalResult;
 exports.RateLimitError = RateLimitError;
 exports.Rerank = Rerank;
 exports.RerankResult = RerankResult;