scorecard-ai 1.0.0-alpha.9 → 1.1.0

package/resources/metrics.mjs

@@ -0,0 +1,48 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from "../core/resource.mjs";
+import { path } from "../internal/utils/path.mjs";
+export class Metrics extends APIResource {
+    /**
+     * Create a new Metric for evaluating system outputs. The structure of a metric
+     * depends on the evalType and outputType of the metric.
+     *
+     * @example
+     * ```ts
+     * const metric = await client.metrics.create('314', {
+     *   evalType: 'ai',
+     *   name: 'Response Accuracy',
+     *   outputType: 'boolean',
+     *   promptTemplate:
+     *     'Please evaluate if the following response is factually accurate: {{outputs.response}}',
+     *   description:
+     *     'Evaluates if the response is factually accurate',
+     *   evalModelName: 'gpt-4o',
+     *   guidelines:
+     *     'Check if the response contains factually correct information',
+     *   temperature: 0.1,
+     * });
+     * ```
+     */
+    create(projectID, body, options) {
+        return this._client.post(path `/projects/${projectID}/metrics`, { body, ...options });
+    }
+    /**
+     * Update an existing Metric. You must specify the evalType and outputType of the
+     * metric. The structure of a metric depends on the evalType and outputType of the
+     * metric.
+     *
+     * @example
+     * ```ts
+     * const metric = await client.metrics.update('321', {
+     *   evalType: 'ai',
+     *   outputType: 'boolean',
+     *   promptTemplate:
+     *     'Using the following guidelines, evaluate the response: {{ guidelines }}\n\nResponse: {{ outputs.response }}\n\nIdeal answer: {{ expected.idealResponse }}',
+     * });
+     * ```
+     */
+    update(metricID, body, options) {
+        return this._client.patch(path `/metrics/${metricID}`, { body, ...options });
+    }
+}
+//# sourceMappingURL=metrics.mjs.map

package/resources/metrics.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.mjs","sourceRoot":"","sources":["../src/resources/metrics.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAGf,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,MAAM,CAAC,SAAiB,EAAE,IAAwB,EAAE,OAAwB;QAC1E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,aAAa,SAAS,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,MAAM,CAAC,QAAgB,EAAE,IAAwB,EAAE,OAAwB;QACzE,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAA,YAAY,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;CACF"}

package/resources/runs.d.mts

@@ -9,7 +9,7 @@ export declare class Runs extends APIResource {
      * ```ts
      * const run = await client.runs.create('314', {
      *   metricIds: ['789', '101'],
-     *   systemConfigId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
+     *   systemVersionId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
      *   testsetId: '246',
      * });
      * ```
@@ -37,9 +37,9 @@ export interface Run {
      */
     testsetId: string | null;
     /**
-     * The ID of the system configuration this Run is using.
+     * The ID of the system version this Run is using.
      */
-    systemConfigId?: string;
+    systemVersionId?: string;
 }
 export interface RunCreateParams {
     /**
@@ -47,9 +47,9 @@ export interface RunCreateParams {
      */
     metricIds: Array<string>;
     /**
-     * The ID of the system configuration this Run is using.
+     * The ID of the system version this Run is using.
      */
-    systemConfigId?: string;
+    systemVersionId?: string;
     /**
      * The ID of the Testset this Run is testing.
      */

package/resources/runs.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"runs.d.mts","sourceRoot":"","sources":["../src/resources/runs.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAGzB,qBAAa,IAAK,SAAQ,WAAW;IACnC;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,GAAG,CAAC;CAG5F;AAED;;GAEG;AACH,MAAM,WAAW,GAAG;IAClB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEzB;;OAEG;IACH,MAAM,EACF,SAAS,GACT,oBAAoB,GACpB,mBAAmB,GACnB,kBAAkB,GAClB,iBAAiB,GACjB,wBAAwB,GACxB,WAAW,CAAC;IAEhB;;OAEG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEzB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,CAAC,OAAO,WAAW,IAAI,CAAC;IAC5B,OAAO,EAAE,KAAK,GAAG,IAAI,GAAG,EAAE,KAAK,eAAe,IAAI,eAAe,EAAE,CAAC;CACrE"}
+{"version":3,"file":"runs.d.mts","sourceRoot":"","sources":["../src/resources/runs.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAGzB,qBAAa,IAAK,SAAQ,WAAW;IACnC;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,GAAG,CAAC;CAG5F;AAED;;GAEG;AACH,MAAM,WAAW,GAAG;IAClB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEzB;;OAEG;IACH,MAAM,EACF,SAAS,GACT,oBAAoB,GACpB,mBAAmB,GACnB,kBAAkB,GAClB,iBAAiB,GACjB,wBAAwB,GACxB,WAAW,CAAC;IAEhB;;OAEG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEzB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,CAAC,OAAO,WAAW,IAAI,CAAC;IAC5B,OAAO,EAAE,KAAK,GAAG,IAAI,GAAG,EAAE,KAAK,eAAe,IAAI,eAAe,EAAE,CAAC;CACrE"}

package/resources/runs.d.ts

@@ -9,7 +9,7 @@ export declare class Runs extends APIResource {
      * ```ts
      * const run = await client.runs.create('314', {
      *   metricIds: ['789', '101'],
-     *   systemConfigId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
+     *   systemVersionId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
      *   testsetId: '246',
      * });
      * ```
@@ -37,9 +37,9 @@ export interface Run {
      */
     testsetId: string | null;
     /**
-     * The ID of the system configuration this Run is using.
+     * The ID of the system version this Run is using.
      */
-    systemConfigId?: string;
+    systemVersionId?: string;
 }
 export interface RunCreateParams {
     /**
@@ -47,9 +47,9 @@ export interface RunCreateParams {
      */
     metricIds: Array<string>;
     /**
-     * The ID of the system configuration this Run is using.
+     * The ID of the system version this Run is using.
      */
-    systemConfigId?: string;
+    systemVersionId?: string;
     /**
      * The ID of the Testset this Run is testing.
      */

package/resources/runs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../src/resources/runs.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAGzB,qBAAa,IAAK,SAAQ,WAAW;IACnC;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,GAAG,CAAC;CAG5F;AAED;;GAEG;AACH,MAAM,WAAW,GAAG;IAClB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEzB;;OAEG;IACH,MAAM,EACF,SAAS,GACT,oBAAoB,GACpB,mBAAmB,GACnB,kBAAkB,GAClB,iBAAiB,GACjB,wBAAwB,GACxB,WAAW,CAAC;IAEhB;;OAEG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEzB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,CAAC,OAAO,WAAW,IAAI,CAAC;IAC5B,OAAO,EAAE,KAAK,GAAG,IAAI,GAAG,EAAE,KAAK,eAAe,IAAI,eAAe,EAAE,CAAC;CACrE"}
+{"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../src/resources/runs.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAGzB,qBAAa,IAAK,SAAQ,WAAW;IACnC;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,GAAG,CAAC;CAG5F;AAED;;GAEG;AACH,MAAM,WAAW,GAAG;IAClB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEzB;;OAEG;IACH,MAAM,EACF,SAAS,GACT,oBAAoB,GACpB,mBAAmB,GACnB,kBAAkB,GAClB,iBAAiB,GACjB,wBAAwB,GACxB,WAAW,CAAC;IAEhB;;OAEG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEzB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEzB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,CAAC,OAAO,WAAW,IAAI,CAAC;IAC5B,OAAO,EAAE,KAAK,GAAG,IAAI,GAAG,EAAE,KAAK,eAAe,IAAI,eAAe,EAAE,CAAC;CACrE"}

package/resources/runs.js

@@ -12,7 +12,7 @@ class Runs extends resource_1.APIResource {
      * ```ts
      * const run = await client.runs.create('314', {
      *   metricIds: ['789', '101'],
-     *   systemConfigId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
+     *   systemVersionId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
      *   testsetId: '246',
      * });
      * ```

package/resources/runs.mjs

@@ -9,7 +9,7 @@ export class Runs extends APIResource {
      * ```ts
      * const run = await client.runs.create('314', {
      *   metricIds: ['789', '101'],
-     *   systemConfigId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
+     *   systemVersionId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
      *   testsetId: '246',
      * });
      * ```

package/resources/systems/index.d.mts

@@ -0,0 +1,3 @@
+export { Systems, type System, type SystemDeleteResponse, type SystemCreateParams, type SystemUpdateParams, type SystemListParams, type SystemsPaginatedResponse, } from "./systems.mjs";
+export { Versions, type SystemVersion, type VersionCreateParams, type VersionListParams, type SystemVersionsPaginatedResponse, } from "./versions.mjs";
+//# sourceMappingURL=index.d.mts.map

package/resources/systems/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../../src/resources/systems/index.ts"],"names":[],"mappings":"OAEO,EACL,OAAO,EACP,KAAK,MAAM,EACX,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,GAC9B;OACM,EACL,QAAQ,EACR,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,+BAA+B,GACrC"}

package/resources/systems/index.d.ts

@@ -0,0 +1,3 @@
+export { Systems, type System, type SystemDeleteResponse, type SystemCreateParams, type SystemUpdateParams, type SystemListParams, type SystemsPaginatedResponse, } from "./systems.js";
+export { Versions, type SystemVersion, type VersionCreateParams, type VersionListParams, type SystemVersionsPaginatedResponse, } from "./versions.js";
+//# sourceMappingURL=index.d.ts.map

package/resources/systems/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/resources/systems/index.ts"],"names":[],"mappings":"OAEO,EACL,OAAO,EACP,KAAK,MAAM,EACX,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,GAC9B;OACM,EACL,QAAQ,EACR,KAAK,aAAa,EAClB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,+BAA+B,GACrC"}

package/resources/systems/index.js

@@ -0,0 +1,9 @@
+"use strict";
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Versions = exports.Systems = void 0;
+var systems_1 = require("./systems.js");
+Object.defineProperty(exports, "Systems", { enumerable: true, get: function () { return systems_1.Systems; } });
+var versions_1 = require("./versions.js");
+Object.defineProperty(exports, "Versions", { enumerable: true, get: function () { return versions_1.Versions; } });
+//# sourceMappingURL=index.js.map

package/resources/systems/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/resources/systems/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,wCAQmB;AAPjB,kGAAA,OAAO,OAAA;AAQT,0CAMoB;AALlB,oGAAA,QAAQ,OAAA"}

package/resources/systems/index.mjs

@@ -0,0 +1,4 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+export { Systems, } from "./systems.mjs";
+export { Versions, } from "./versions.mjs";
+//# sourceMappingURL=index.mjs.map

package/resources/systems/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../../src/resources/systems/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EACL,OAAO,GAOR;OACM,EACL,QAAQ,GAKT"}

package/resources/systems/systems.d.mts

@@ -0,0 +1,229 @@
+import { APIResource } from "../../core/resource.mjs";
+import * as VersionsAPI from "./versions.mjs";
+import { SystemVersion, SystemVersionsPaginatedResponse, VersionCreateParams, VersionListParams, Versions } from "./versions.mjs";
+import { APIPromise } from "../../core/api-promise.mjs";
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../../core/pagination.mjs";
+import { RequestOptions } from "../../internal/request-options.mjs";
+export declare class Systems extends APIResource {
+    versions: VersionsAPI.Versions;
+    /**
+     * Create a new system definition that specifies the interface contracts for a
+     * component you want to evaluate.
+     *
+     * A system acts as a template that defines three key contracts through JSON
+     * Schemas:
+     *
+     * 1. Input Schema: What data your system accepts (e.g., user queries, context
+     *    documents)
+     * 2. Output Schema: What data your system produces (e.g., responses, confidence
+     *    scores)
+     * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+     *    temperature)
+     *
+     * This separation lets you evaluate any system as a black box, focusing on its
+     * interface rather than implementation details.
+     *
+     * @example
+     * ```ts
+     * const system = await client.systems.create('314', {
+     *   configSchema: {
+     *     type: 'object',
+     *     properties: {
+     *       temperature: { type: 'number' },
+     *       maxTokens: { type: 'integer' },
+     *       model: { type: 'string', enum: ['gpt-4', 'gpt-4-turbo'] },
+     *     },
+     *     required: ['model'],
+     *   },
+     *   description: 'Production chatbot powered by GPT-4',
+     *   inputSchema: {
+     *     type: 'object',
+     *     properties: {
+     *       messages: {
+     *         type: 'array',
+     *         items: {
+     *           type: 'object',
+     *           properties: {
+     *             role: { type: 'string', enum: ['system', 'user', 'assistant'] },
+     *             content: { type: 'string' },
+     *           },
+     *           required: ['role', 'content'],
+     *         },
+     *       },
+     *     },
+     *     required: ['messages'],
+     *   },
+     *   name: 'GPT-4 Chatbot',
+     *   outputSchema: {
+     *     type: 'object',
+     *     properties: { response: { type: 'string' } },
+     *     required: ['response'],
+     *   },
+     * });
+     * ```
+     */
+    create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Update an existing system definition. Only the fields provided in the request
+     * body will be updated. If a field is provided, the new content will replace the
+     * existing content. If a field is not provided, the existing content will remain
+     * unchanged.
+     *
+     * When updating schemas:
+     *
+     * - The system will accept your changes regardless of compatibility with existing
+     *   configurations
+     * - Schema updates won't invalidate existing evaluations or configurations
+     * - For significant redesigns, creating a new system definition provides a cleaner
+     *   separation
+     *
+     * @example
+     * ```ts
+     * const system = await client.systems.update(
+     *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
+     *   {
+     *     description:
+     *       'Updated production chatbot powered by GPT-4 Turbo',
+     *     name: 'GPT-4 Turbo Chatbot',
+     *   },
+     * );
+     * ```
+     */
+    update(systemID: string, body?: SystemUpdateParams | null | undefined, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+     *
+     * @example
+     * ```ts
+     * // Automatically fetches more pages as needed.
+     * for await (const system of client.systems.list('314')) {
+     *   // ...
+     * }
+     * ```
+     */
+    list(projectID: string, query?: SystemListParams | null | undefined, options?: RequestOptions): PagePromise<SystemsPaginatedResponse, System>;
+    /**
+     * Delete a system definition by ID. This will not delete associated system
+     * versions.
+     *
+     * @example
+     * ```ts
+     * const system = await client.systems.delete(
+     *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
+     * );
+     * ```
+     */
+    delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse>;
+    /**
+     * Retrieve a specific system by ID.
+     *
+     * @example
+     * ```ts
+     * const system = await client.systems.get(
+     *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
+     * );
+     * ```
+     */
+    get(systemID: string, options?: RequestOptions): APIPromise<System>;
+}
+export type SystemsPaginatedResponse = PaginatedResponse<System>;
+/**
+ * A System Under Test (SUT) defines the interface to a component or service you
+ * want to evaluate.
+ *
+ * It specifies three contracts through schemas:
+ *
+ * - inputSchema: The structure of data the system accepts.
+ * - outputSchema: The structure of data the system produces.
+ * - configSchema: The parameters that modify system behavior.
+ *
+ * This abstraction lets you evaluate any system as a black box, focusing on its
+ * interface rather than implementation details. It's particularly useful for
+ * systems with variable outputs or complex internal state.
+ *
+ * Systems are templates - to run evaluations, pair them with a SystemVersion that
+ * provides specific parameter values.
+ */
+export interface System {
+    /**
+     * The ID of the system.
+     */
+    id: string;
+    /**
+     * The schema of the system's configuration.
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system.
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs.
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system.
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs.
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemDeleteResponse {
+    /**
+     * Whether the deletion was successful.
+     */
+    success: boolean;
+}
+export interface SystemCreateParams {
+    /**
+     * The schema of the system's configuration.
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system.
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs.
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system.
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs.
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemUpdateParams {
+    /**
+     * The schema of the system's configuration.
+     */
+    configSchema?: Record<string, unknown>;
+    /**
+     * The description of the system.
+     */
+    description?: string;
+    /**
+     * The schema of the system's inputs.
+     */
+    inputSchema?: Record<string, unknown>;
+    /**
+     * The name of the system.
+     */
+    name?: string;
+    /**
+     * The schema of the system's outputs.
+     */
+    outputSchema?: Record<string, unknown>;
+}
+export interface SystemListParams extends PaginatedResponseParams {
+}
+export declare namespace Systems {
+    export { type System as System, type SystemDeleteResponse as SystemDeleteResponse, type SystemsPaginatedResponse as SystemsPaginatedResponse, type SystemCreateParams as SystemCreateParams, type SystemUpdateParams as SystemUpdateParams, type SystemListParams as SystemListParams, };
+    export { Versions as Versions, type SystemVersion as SystemVersion, type SystemVersionsPaginatedResponse as SystemVersionsPaginatedResponse, type VersionCreateParams as VersionCreateParams, type VersionListParams as VersionListParams, };
+}
+//# sourceMappingURL=systems.d.mts.map

package/resources/systems/systems.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"systems.d.mts","sourceRoot":"","sources":["../../src/resources/systems/systems.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,KAAK,WAAW;OAChB,EACL,aAAa,EACb,+BAA+B,EAC/B,mBAAmB,EACnB,iBAAiB,EACjB,QAAQ,EACT;OACM,EAAE,UAAU,EAAE;OACd,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE;OAChE,EAAE,cAAc,EAAE;AAGzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAA0C;IAExE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuDG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;IAIjG;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,MAAM,CACJ,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE,kBAAkB,GAAG,IAAI,GAAG,SAAc,EAChD,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,MAAM,CAAC;IAIrB;;;;;;;;;;OAUG;IACH,IAAI,CACF,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,gBAAgB,GAAG,IAAI,GAAG,SAAc,EAC/C,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,wBAAwB,EAAE,MAAM,CAAC;IAOhD;;;;;;;;;;OAUG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,oBAAoB,CAAC;IAIpF;;;;;;;;;OASG;IACH,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;CAGpE;AAED,MAAM,MAAM,wBAAwB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;AAEjE;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,MAAM;IACrB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,gBAAiB,SAAQ,uBAAuB;CAAG;AAIpE,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,MAAM,IAAI,MAAM,EACrB,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,gBAAgB,IAAI,gBAAgB,GAC1C,CAAC;IAEF,OAAO,EACL,QAAQ,IAAI,QAAQ,EACpB,KAAK,aAAa,IAAI,aAAa,EACnC,KAAK,+BAA+B,IAAI,+BAA+B,EACvE,KAAK,mBAAmB,IAAI,mBAAmB,EAC/C,KAAK,iBAAiB,IAAI,iBAAiB,GAC5C,CAAC;CACH"}

package/resources/systems/systems.d.ts

@@ -0,0 +1,229 @@
+import { APIResource } from "../../core/resource.js";
+import * as VersionsAPI from "./versions.js";
+import { SystemVersion, SystemVersionsPaginatedResponse, VersionCreateParams, VersionListParams, Versions } from "./versions.js";
+import { APIPromise } from "../../core/api-promise.js";
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../../core/pagination.js";
+import { RequestOptions } from "../../internal/request-options.js";
+export declare class Systems extends APIResource {
+    versions: VersionsAPI.Versions;
+    /**
+     * Create a new system definition that specifies the interface contracts for a
+     * component you want to evaluate.
+     *
+     * A system acts as a template that defines three key contracts through JSON
+     * Schemas:
+     *
+     * 1. Input Schema: What data your system accepts (e.g., user queries, context
+     *    documents)
+     * 2. Output Schema: What data your system produces (e.g., responses, confidence
+     *    scores)
+     * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+     *    temperature)
+     *
+     * This separation lets you evaluate any system as a black box, focusing on its
+     * interface rather than implementation details.
+     *
+     * @example
+     * ```ts
+     * const system = await client.systems.create('314', {
+     *   configSchema: {
+     *     type: 'object',
+     *     properties: {
+     *       temperature: { type: 'number' },
+     *       maxTokens: { type: 'integer' },
+     *       model: { type: 'string', enum: ['gpt-4', 'gpt-4-turbo'] },
+     *     },
+     *     required: ['model'],
+     *   },
+     *   description: 'Production chatbot powered by GPT-4',
+     *   inputSchema: {
+     *     type: 'object',
+     *     properties: {
+     *       messages: {
+     *         type: 'array',
+     *         items: {
+     *           type: 'object',
+     *           properties: {
+     *             role: { type: 'string', enum: ['system', 'user', 'assistant'] },
+     *             content: { type: 'string' },
+     *           },
+     *           required: ['role', 'content'],
+     *         },
+     *       },
+     *     },
+     *     required: ['messages'],
+     *   },
+     *   name: 'GPT-4 Chatbot',
+     *   outputSchema: {
+     *     type: 'object',
+     *     properties: { response: { type: 'string' } },
+     *     required: ['response'],
+     *   },
+     * });
+     * ```
+     */
+    create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Update an existing system definition. Only the fields provided in the request
+     * body will be updated. If a field is provided, the new content will replace the
+     * existing content. If a field is not provided, the existing content will remain
+     * unchanged.
+     *
+     * When updating schemas:
+     *
+     * - The system will accept your changes regardless of compatibility with existing
+     *   configurations
+     * - Schema updates won't invalidate existing evaluations or configurations
+     * - For significant redesigns, creating a new system definition provides a cleaner
+     *   separation
+     *
+     * @example
+     * ```ts
+     * const system = await client.systems.update(
+     *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
+     *   {
+     *     description:
+     *       'Updated production chatbot powered by GPT-4 Turbo',
+     *     name: 'GPT-4 Turbo Chatbot',
+     *   },
+     * );
+     * ```
+     */
+    update(systemID: string, body?: SystemUpdateParams | null | undefined, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+     *
+     * @example
+     * ```ts
+     * // Automatically fetches more pages as needed.
+     * for await (const system of client.systems.list('314')) {
+     *   // ...
+     * }
+     * ```
+     */
+    list(projectID: string, query?: SystemListParams | null | undefined, options?: RequestOptions): PagePromise<SystemsPaginatedResponse, System>;
+    /**
+     * Delete a system definition by ID. This will not delete associated system
+     * versions.
+     *
+     * @example
+     * ```ts
+     * const system = await client.systems.delete(
+     *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
+     * );
+     * ```
+     */
+    delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse>;
+    /**
+     * Retrieve a specific system by ID.
+     *
+     * @example
+     * ```ts
+     * const system = await client.systems.get(
+     *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
+     * );
+     * ```
+     */
+    get(systemID: string, options?: RequestOptions): APIPromise<System>;
+}
+export type SystemsPaginatedResponse = PaginatedResponse<System>;
+/**
+ * A System Under Test (SUT) defines the interface to a component or service you
+ * want to evaluate.
+ *
+ * It specifies three contracts through schemas:
+ *
+ * - inputSchema: The structure of data the system accepts.
+ * - outputSchema: The structure of data the system produces.
+ * - configSchema: The parameters that modify system behavior.
+ *
+ * This abstraction lets you evaluate any system as a black box, focusing on its
+ * interface rather than implementation details. It's particularly useful for
+ * systems with variable outputs or complex internal state.
+ *
+ * Systems are templates - to run evaluations, pair them with a SystemVersion that
+ * provides specific parameter values.
+ */
+export interface System {
+    /**
+     * The ID of the system.
+     */
+    id: string;
+    /**
+     * The schema of the system's configuration.
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system.
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs.
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system.
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs.
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemDeleteResponse {
+    /**
+     * Whether the deletion was successful.
+     */
+    success: boolean;
+}
+export interface SystemCreateParams {
+    /**
+     * The schema of the system's configuration.
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system.
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs.
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system.
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs.
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemUpdateParams {
+    /**
+     * The schema of the system's configuration.
+     */
+    configSchema?: Record<string, unknown>;
+    /**
+     * The description of the system.
+     */
+    description?: string;
+    /**
+     * The schema of the system's inputs.
+     */
+    inputSchema?: Record<string, unknown>;
+    /**
+     * The name of the system.
+     */
+    name?: string;
+    /**
+     * The schema of the system's outputs.
+     */
+    outputSchema?: Record<string, unknown>;
+}
+export interface SystemListParams extends PaginatedResponseParams {
+}
+export declare namespace Systems {
+    export { type System as System, type SystemDeleteResponse as SystemDeleteResponse, type SystemsPaginatedResponse as SystemsPaginatedResponse, type SystemCreateParams as SystemCreateParams, type SystemUpdateParams as SystemUpdateParams, type SystemListParams as SystemListParams, };
+    export { Versions as Versions, type SystemVersion as SystemVersion, type SystemVersionsPaginatedResponse as SystemVersionsPaginatedResponse, type VersionCreateParams as VersionCreateParams, type VersionListParams as VersionListParams, };
+}
+//# sourceMappingURL=systems.d.ts.map

package/resources/systems/systems.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"systems.d.ts","sourceRoot":"","sources":["../../src/resources/systems/systems.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,KAAK,WAAW;OAChB,EACL,aAAa,EACb,+BAA+B,EAC/B,mBAAmB,EACnB,iBAAiB,EACjB,QAAQ,EACT;OACM,EAAE,UAAU,EAAE;OACd,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE;OAChE,EAAE,cAAc,EAAE;AAGzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAA0C;IAExE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuDG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;IAIjG;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,MAAM,CACJ,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE,kBAAkB,GAAG,IAAI,GAAG,SAAc,EAChD,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,MAAM,CAAC;IAIrB;;;;;;;;;;OAUG;IACH,IAAI,CACF,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,gBAAgB,GAAG,IAAI,GAAG,SAAc,EAC/C,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,wBAAwB,EAAE,MAAM,CAAC;IAOhD;;;;;;;;;;OAUG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,oBAAoB,CAAC;IAIpF;;;;;;;;;OASG;IACH,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;CAGpE;AAED,MAAM,MAAM,wBAAwB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;AAEjE;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,MAAM;IACrB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,gBAAiB,SAAQ,uBAAuB;CAAG;AAIpE,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,MAAM,IAAI,MAAM,EACrB,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,gBAAgB,IAAI,gBAAgB,GAC1C,CAAC;IAEF,OAAO,EACL,QAAQ,IAAI,QAAQ,EACpB,KAAK,aAAa,IAAI,aAAa,EACnC,KAAK,+BAA+B,IAAI,+BAA+B,EACvE,KAAK,mBAAmB,IAAI,mBAAmB,EAC/C,KAAK,iBAAiB,IAAI,iBAAiB,GAC5C,CAAC;CACH"}