pareta 0.1.0 → 0.2.1

package/dist/index.d.ts

@@ -75,6 +75,16 @@ declare class Endpoint extends BaseModel {
     get task(): string | null;
     get url(): string | null;
     get isLive(): boolean;
+    /** Structured-extraction endpoints (contract / SEC / ICD …): the system prompt
+     *  the benchmark used. The proxy applies it automatically when you send no
+     *  `system` message (so quality matches the leaderboard by default); returned
+     *  here so you can read it, or override it with your own `system` message.
+     *  `null` when the task has no injectable prompt (see `promptScaffold`). */
+    get recommendedSystemPrompt(): string | null;
+    /** Fixed-label classification endpoints: a copy-and-customize `system` prompt
+     *  template. NEVER auto-applied — the categories are yours, not the benchmark's
+     *  — so fill them in and send it as your `system` message. `null` otherwise. */
+    get promptScaffold(): string | null;
 }
 declare class Task extends BaseModel {
     get id(): string | null;
@@ -102,6 +112,17 @@ declare class EvalSet extends BaseModel {
     get itemCount(): number | null;
     get scoringStrategy(): string | null;
 }
+/**
+ * One scored item inside `EvalResult.perItem`. `prediction` is the model's raw
+ * output, truncated server-side — present only on items that reached scoring
+ * (not pool/build errors), there to debug a 0.0 `score` without re-running.
+ */
+declare class EvalItemResult extends BaseModel {
+    get idx(): number | null;
+    get score(): number | null;
+    get prediction(): string | null;
+    get error(): string | null;
+}
 /**
  * One model's aggregate on an eval run. `modelId` is the per-task public alias;
  * `kind` ('open' | 'frontier') is populated by the Slice-4 result schema.
@@ -115,6 +136,8 @@ declare class EvalResult extends BaseModel {
     get meanCostMicroUsd(): number | null;
     get nSucceeded(): number | null;
     get errorCount(): number | null;
+    /** Per-item rows (idx/score/prediction/error); empty when not persisted. */
+    get perItem(): EvalItemResult[];
 }
 declare class LeaderboardEntry extends BaseModel {
     get name(): string | null;
@@ -378,6 +401,79 @@ declare class Evals {
     frontierModels(task?: string): Promise<FrontierModel[]>;
 }
 
+/**
+ * `client.auto` — the routing brain's surrounding surfaces.
+ *
+ * Calling the brain itself is plain chat with `model: "auto"`:
+ *
+ *   client.chat.completions.create({
+ *     model: "auto",
+ *     messages: [{ role: "user", content: "…" }],
+ *   })
+ *
+ * This resource carries the org-level metrics rollup (requests, success,
+ * spend, PROJECTED savings vs frontier) and the metered frontier comparison
+ * the Playground uses.
+ */
+
+interface AutoMetrics {
+    requests_30d: number;
+    requests_today: number;
+    success_rate_30d: number | null;
+    billed_micro_usd_30d: number;
+    billed_micro_usd_today: number;
+    cost_to_serve_micro_usd_30d: number;
+    /** PROJECTED (frontier list-priced counterfactual) — labeled so until
+     * dual-run calibration lands. */
+    savings_vs_frontier_micro_usd_30d: number | null;
+    savings_multiple_30d: number | null;
+    performance_hourly_7d: Array<{
+        hour: string;
+        requests: number;
+        error_rate: number;
+        p50_ms: number | null;
+        p95_ms: number | null;
+    }>;
+    days_30d: Array<{
+        day: string;
+        n: number;
+        ok: number;
+        success_rate: number;
+    }>;
+    last_request: {
+        created_at: string;
+        status_code: number;
+        duration_ms: number;
+        billed_micro_usd: number | null;
+        cost_to_serve_micro_usd: number | null;
+    } | null;
+}
+interface FrontierComparison {
+    model: string;
+    content: string;
+    cost_micro_usd: number;
+    latency_ms: number;
+}
+declare class Auto {
+    private readonly client;
+    constructor(client: Transport);
+    /** Your org's `model: "auto"` traffic, rolled up. Read-only, free. */
+    metrics(): Promise<AutoMetrics>;
+    /**
+     * One prompt against a frontier vendor for a side-by-side with
+     * `model: "auto"` — METERED at the vendor's actual token cost (a failed
+     * vendor call bills $0). Allowed models: gpt-5.5, gemini-3-5-flash,
+     * gemini-3-1-pro, claude-sonnet-4-6.
+     */
+    compareFrontier(params: {
+        model: string;
+        messages: Array<{
+            role: string;
+            content: string;
+        }>;
+    }): Promise<FrontierComparison>;
+}
+
 /**
  * The `Pareta` client — one class (no sync/async split; JS is Promise-only).
  * A faithful port of the Python `_client.py` transport: header construction,
@@ -432,6 +528,7 @@ declare class Pareta implements Transport {
     readonly endpoints: Endpoints;
     readonly tasks: Tasks;
     readonly evals: Evals;
+    readonly auto: Auto;
     constructor(options?: ParetaOptions);
     /** Build from PARETA_API_KEY (+ optional PARETA_BASE_URL); explicit opts win. */
     static fromEnv(options?: ParetaOptions): Pareta;
@@ -547,4 +644,4 @@ declare class EndpointNotReadyError extends APIStatusError {
     constructor(message: string, init: APIStatusErrorInit);
 }
 
-export { APIConnectionError, APIStatusError, APITimeoutError, AuthenticationError, BadRequestError, BaseModel, ChatCompletion, ChatCompletionChunk, type ChatCompletionCreateParams, type ChatMessage, Choice, ConflictError, type DeployParams, Endpoint, EndpointMetrics, EndpointNotReadyError, type ErrorDetail, EvalResult, EvalRun, type EvalRunCreateParams, EvalRuns, EvalSet, EvalSets, type FileInput, type FilePart, FrontierModel, type FrontierSpec, InsufficientCreditsError, Leaderboard, LeaderboardEntry, Message, Model, ModelList, NotFoundError, Pareta, ParetaError, type ParetaOptions, PermissionDeniedError, RateLimitError, type SSEEvent, Task, TaskMatch, TaskMatchCandidate, type Transport, Usage, VERSION, type ValidationErrorItem };
+export { APIConnectionError, APIStatusError, APITimeoutError, AuthenticationError, BadRequestError, BaseModel, ChatCompletion, ChatCompletionChunk, type ChatCompletionCreateParams, type ChatMessage, Choice, ConflictError, type DeployParams, Endpoint, EndpointMetrics, EndpointNotReadyError, type ErrorDetail, EvalItemResult, EvalResult, EvalRun, type EvalRunCreateParams, EvalRuns, EvalSet, EvalSets, type FileInput, type FilePart, FrontierModel, type FrontierSpec, InsufficientCreditsError, Leaderboard, LeaderboardEntry, Message, Model, ModelList, NotFoundError, Pareta, ParetaError, type ParetaOptions, PermissionDeniedError, RateLimitError, type SSEEvent, Task, TaskMatch, TaskMatchCandidate, type Transport, Usage, VERSION, type ValidationErrorItem };

package/dist/index.mjs

@@ -301,6 +301,20 @@ var Endpoint = class extends BaseModel {
   get isLive() {
     return this.raw.status === "live";
   }
+  /** Structured-extraction endpoints (contract / SEC / ICD …): the system prompt
+   *  the benchmark used. The proxy applies it automatically when you send no
+   *  `system` message (so quality matches the leaderboard by default); returned
+   *  here so you can read it, or override it with your own `system` message.
+   *  `null` when the task has no injectable prompt (see `promptScaffold`). */
+  get recommendedSystemPrompt() {
+    return this.raw.recommendedSystemPrompt ?? null;
+  }
+  /** Fixed-label classification endpoints: a copy-and-customize `system` prompt
+   *  template. NEVER auto-applied — the categories are yours, not the benchmark's
+   *  — so fill them in and send it as your `system` message. `null` otherwise. */
+  get promptScaffold() {
+    return this.raw.promptScaffold ?? null;
+  }
 };
 function endpointList(raw) {
   return (raw ?? []).map((e) => new Endpoint(e));
@@ -373,6 +387,20 @@ function evalSetFromCreate(raw) {
 function evalSetList(raw) {
   return (raw?.eval_sets ?? []).map((e) => new EvalSet(e));
 }
+var EvalItemResult = class extends BaseModel {
+  get idx() {
+    return this.raw.idx ?? null;
+  }
+  get score() {
+    return this.raw.score ?? null;
+  }
+  get prediction() {
+    return this.raw.prediction ?? null;
+  }
+  get error() {
+    return this.raw.error ?? null;
+  }
+};
 var EvalResult = class extends BaseModel {
   get modelId() {
     return this.raw.model_id ?? null;
@@ -399,6 +427,10 @@ var EvalResult = class extends BaseModel {
   get errorCount() {
     return this.raw.error_count ?? null;
   }
+  /** Per-item rows (idx/score/prediction/error); empty when not persisted. */
+  get perItem() {
+    return (this.raw.per_item ?? []).map((it) => new EvalItemResult(it));
+  }
 };
 var LeaderboardEntry = class extends BaseModel {
   get name() {
@@ -863,6 +895,31 @@ var Evals = class {
   }
 };
 
+// src/resources/auto.ts
+var Auto = class {
+  constructor(client) {
+    this.client = client;
+  }
+  client;
+  /** Your org's `model: "auto"` traffic, rolled up. Read-only, free. */
+  metrics() {
+    return this.client.request("GET", "/v1/auto/metrics");
+  }
+  /**
+   * One prompt against a frontier vendor for a side-by-side with
+   * `model: "auto"` — METERED at the vendor's actual token cost (a failed
+   * vendor call bills $0). Allowed models: gpt-5.5, gemini-3-5-flash,
+   * gemini-3-1-pro, claude-sonnet-4-6.
+   */
+  compareFrontier(params) {
+    return this.client.request(
+      "POST",
+      "/v1/playground/frontier",
+      { body: params }
+    );
+  }
+};
+
 // src/client.ts
 var DEFAULT_BASE_URL = "https://api.pareta.ai";
 var DEFAULT_TIMEOUT_MS = 6e4;
@@ -883,6 +940,7 @@ var Pareta = class _Pareta {
   endpoints;
   tasks;
   evals;
+  auto;
   constructor(options = {}) {
     if (!options.apiKey) {
       throw new ParetaError(
@@ -903,6 +961,7 @@ var Pareta = class _Pareta {
     this.endpoints = new Endpoints(this);
     this.tasks = new Tasks(this);
     this.evals = new Evals(this);
+    this.auto = new Auto(this);
   }
   /** Build from PARETA_API_KEY (+ optional PARETA_BASE_URL); explicit opts win. */
   static fromEnv(options = {}) {
@@ -1077,6 +1136,6 @@ var Pareta = class _Pareta {
   }
 };
 
-export { APIConnectionError, APIStatusError, APITimeoutError, AuthenticationError, BadRequestError, BaseModel, ChatCompletion, ChatCompletionChunk, Choice, ConflictError, Endpoint, EndpointMetrics, EndpointNotReadyError, EvalResult, EvalRun, EvalRuns, EvalSet, EvalSets, FrontierModel, InsufficientCreditsError, Leaderboard, LeaderboardEntry, Message, Model, ModelList, NotFoundError, Pareta, ParetaError, PermissionDeniedError, RateLimitError, Task, TaskMatch, TaskMatchCandidate, Usage, VERSION };
+export { APIConnectionError, APIStatusError, APITimeoutError, AuthenticationError, BadRequestError, BaseModel, ChatCompletion, ChatCompletionChunk, Choice, ConflictError, Endpoint, EndpointMetrics, EndpointNotReadyError, EvalItemResult, EvalResult, EvalRun, EvalRuns, EvalSet, EvalSets, FrontierModel, InsufficientCreditsError, Leaderboard, LeaderboardEntry, Message, Model, ModelList, NotFoundError, Pareta, ParetaError, PermissionDeniedError, RateLimitError, Task, TaskMatch, TaskMatchCandidate, Usage, VERSION };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map