scorecard-ai 1.0.0-alpha.0 → 1.0.0-alpha.1

package/resources/system-configs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"system-configs.d.ts","sourceRoot":"","sources":["../src/resources/system-configs.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE;OAChE,EAAE,cAAc,EAAE;AAGzB,qBAAa,aAAc,SAAQ,WAAW;IAC5C;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CACJ,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,wBAAwB,EAC9B,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,YAAY,CAAC;IAI3B;;;;;OAKG;IACH,IAAI,CACF,QAAQ,EAAE,MAAM,EAChB,KAAK,GAAE,sBAAsB,GAAG,IAAI,GAAG,SAAc,EACrD,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,8BAA8B,EAAE,YAAY,CAAC;IAO5D;;OAEG;IACH,GAAG,CACD,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,qBAAqB,EAC7B,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,YAAY,CAAC;CAI5B;AAED,MAAM,MAAM,8BAA8B,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;AAE7E;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,YAAY;IAC3B;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;CACxD;AAED,yBAAiB,YAAY,CAAC;IAC5B,UAAiB,eAAe;QAC9B;;WAEG;QACH,OAAO,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;KACd;CACF;AAED,MAAM,WAAW,wBAAwB;IACvC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,gBAAgB,CAAC,EAAE,KAAK,CAAC,wBAAwB,CAAC,eAAe,CAAC,CAAC;CACpE;AAED,yBAAiB,wBAAwB,CAAC;IACxC,UAAiB,eAAe;QAC9B;;WAEG;QACH,OAAO,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;KACd;CACF;AAED,MAAM,WAAW,sBAAuB,SAAQ,uBAAuB;CAAG;AAE1E,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,CAAC,OAAO,WAAW,aAAa,CAAC;IACrC,OAAO,EACL,KAAK,YAAY,IAAI,YAAY,EACjC,KAAK,8BAA8B,IAAI,8BAA8B,EACrE,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,sBAAsB,IAAI,sBAAsB,EACrD,KAAK,qBAAqB,IAAI,qBAAqB,GACpD,CAAC;CACH"}

package/resources/system-configs.js

@@ -0,0 +1,50 @@
+"use strict";
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SystemConfigs = void 0;
+const resource_1 = require("../core/resource.js");
+const pagination_1 = require("../core/pagination.js");
+const path_1 = require("../internal/utils/path.js");
+class SystemConfigs extends resource_1.APIResource {
+    /**
+     * Create a new configuration for a system.
+     *
+     * Each configuration contains specific parameter values that match the system's
+     * configSchema - things like model parameters, thresholds, or processing options.
+     * Once created, configurations cannot be modified, ensuring stable reference
+     * points for evaluations.
+     *
+     * When creating a configuration:
+     *
+     * - The 'config' object is validated against the parent system's configSchema
+     * - Configurations with validation errors are still stored, with errors included
+     *   in the response
+     * - Validation errors indicate fields that don't match the schema but don't
+     *   prevent creation
+     * - Having validation errors may affect how some evaluation metrics are calculated
+     */
+    create(systemID, body, options) {
+        return this._client.post((0, path_1.path) `/systems/${systemID}/configs`, { body, ...options });
+    }
+    /**
+     * Retrieve a paginated list of configurations for a specific system.
+     *
+     * System configurations provide concrete parameter values for a System Under Test,
+     * defining exactly how the system should be configured during an evaluation run.
+     */
+    list(systemID, query = {}, options) {
+        return this._client.getAPIList((0, path_1.path) `/systems/${systemID}/configs`, (pagination_1.PaginatedResponse), {
+            query,
+            ...options,
+        });
+    }
+    /**
+     * Retrieve a specific system configuration by ID.
+     */
+    get(systemConfigID, params, options) {
+        const { systemId } = params;
+        return this._client.get((0, path_1.path) `/systems/${systemId}/configs/${systemConfigID}`, options);
+    }
+}
+exports.SystemConfigs = SystemConfigs;
+//# sourceMappingURL=system-configs.js.map

package/resources/system-configs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"system-configs.js","sourceRoot":"","sources":["../src/resources/system-configs.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAA+C;AAE/C,sDAAkG;AAElG,oDAA8C;AAE9C,MAAa,aAAc,SAAQ,sBAAW;IAC5C;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CACJ,QAAgB,EAChB,IAA8B,EAC9B,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAA,WAAI,EAAA,YAAY,QAAQ,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACrF,CAAC;IAED;;;;;OAKG;IACH,IAAI,CACF,QAAgB,EAChB,QAAmD,EAAE,EACrD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAA,WAAI,EAAA,YAAY,QAAQ,UAAU,EAAE,CAAA,8BAA+B,CAAA,EAAE;YAClG,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,GAAG,CACD,cAAsB,EACtB,MAA6B,EAC7B,OAAwB;QAExB,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,YAAY,QAAQ,YAAY,cAAc,EAAE,EAAE,OAAO,CAAC,CAAC;IACzF,CAAC;CACF;AAtDD,sCAsDC"}

package/resources/system-configs.mjs

@@ -0,0 +1,46 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from "../core/resource.mjs";
+import { PaginatedResponse } from "../core/pagination.mjs";
+import { path } from "../internal/utils/path.mjs";
+export class SystemConfigs extends APIResource {
+    /**
+     * Create a new configuration for a system.
+     *
+     * Each configuration contains specific parameter values that match the system's
+     * configSchema - things like model parameters, thresholds, or processing options.
+     * Once created, configurations cannot be modified, ensuring stable reference
+     * points for evaluations.
+     *
+     * When creating a configuration:
+     *
+     * - The 'config' object is validated against the parent system's configSchema
+     * - Configurations with validation errors are still stored, with errors included
+     *   in the response
+     * - Validation errors indicate fields that don't match the schema but don't
+     *   prevent creation
+     * - Having validation errors may affect how some evaluation metrics are calculated
+     */
+    create(systemID, body, options) {
+        return this._client.post(path `/systems/${systemID}/configs`, { body, ...options });
+    }
+    /**
+     * Retrieve a paginated list of configurations for a specific system.
+     *
+     * System configurations provide concrete parameter values for a System Under Test,
+     * defining exactly how the system should be configured during an evaluation run.
+     */
+    list(systemID, query = {}, options) {
+        return this._client.getAPIList(path `/systems/${systemID}/configs`, (PaginatedResponse), {
+            query,
+            ...options,
+        });
+    }
+    /**
+     * Retrieve a specific system configuration by ID.
+     */
+    get(systemConfigID, params, options) {
+        const { systemId } = params;
+        return this._client.get(path `/systems/${systemId}/configs/${systemConfigID}`, options);
+    }
+}
+//# sourceMappingURL=system-configs.mjs.map

package/resources/system-configs.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"system-configs.mjs","sourceRoot":"","sources":["../src/resources/system-configs.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAEf,EAAe,iBAAiB,EAAgC;OAEhE,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,aAAc,SAAQ,WAAW;IAC5C;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CACJ,QAAgB,EAChB,IAA8B,EAC9B,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,YAAY,QAAQ,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACrF,CAAC;IAED;;;;;OAKG;IACH,IAAI,CACF,QAAgB,EAChB,QAAmD,EAAE,EACrD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAA,YAAY,QAAQ,UAAU,EAAE,CAAA,iBAA+B,CAAA,EAAE;YAClG,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,GAAG,CACD,cAAsB,EACtB,MAA6B,EAC7B,OAAwB;QAExB,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,YAAY,QAAQ,YAAY,cAAc,EAAE,EAAE,OAAO,CAAC,CAAC;IACzF,CAAC;CACF"}

package/resources/systems.d.mts

@@ -0,0 +1,152 @@
+import { APIResource } from "../core/resource.mjs";
+import { APIPromise } from "../core/api-promise.mjs";
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../core/pagination.mjs";
+import { RequestOptions } from "../internal/request-options.mjs";
+export declare class Systems extends APIResource {
+    /**
+     * Create a new system definition that specifies the interface contracts for a
+     * component you want to evaluate.
+     *
+     * A system acts as a template that defines three key contracts through JSON
+     * Schemas:
+     *
+     * 1. Input Schema: What data your system accepts (e.g., user queries, context
+     *    documents)
+     * 2. Output Schema: What data your system produces (e.g., responses, confidence
+     *    scores)
+     * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+     *    temperature)
+     *
+     * This separation lets you evaluate any system as a black box, focusing on its
+     * interface rather than implementation details.
+     */
+    create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Update an existing system definition. Only the fields provided in the request
+     * body will be updated. If a field is provided, the new content will replace the
+     * existing content. If a field is not provided, the existing content will remain
+     * unchanged.
+     *
+     * When updating schemas:
+     *
+     * - The system will accept your changes regardless of compatibility with existing
+     *   configurations
+     * - Schema updates won't invalidate existing evaluations or configurations
+     * - For significant redesigns, creating a new system definition provides a cleaner
+     *   separation
+     */
+    update(systemID: string, body?: SystemUpdateParams | null | undefined, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+     */
+    list(projectID: string, query?: SystemListParams | null | undefined, options?: RequestOptions): PagePromise<SystemsPaginatedResponse, System>;
+    /**
+     * Delete a system definition by ID. This will not delete associated system
+     * configurations.
+     */
+    delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse>;
+    /**
+     * Retrieve a specific system by ID.
+     */
+    get(systemID: string, options?: RequestOptions): APIPromise<System>;
+}
+export type SystemsPaginatedResponse = PaginatedResponse<System>;
+/**
+ * A System Under Test (SUT) defines the interface to a component or service you
+ * want to evaluate.
+ *
+ * It specifies three contracts through schemas:
+ *
+ * - inputSchema: The structure of data the system accepts
+ * - outputSchema: The structure of data the system produces
+ * - configSchema: The parameters that modify system behavior
+ *
+ * This abstraction lets you evaluate any system as a black box, focusing on its
+ * interface rather than implementation details. It's particularly useful for
+ * systems with variable outputs or complex internal state.
+ *
+ * Systems are templates - to run evaluations, pair them with a SystemConfig that
+ * provides specific parameter values.
+ */
+export interface System {
+    /**
+     * The ID of the system
+     */
+    id: string;
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemDeleteResponse {
+    /**
+     * Whether the deletion was successful
+     */
+    success: boolean;
+}
+export interface SystemCreateParams {
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemUpdateParams {
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema?: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description?: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema?: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name?: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema?: Record<string, unknown>;
+}
+export interface SystemListParams extends PaginatedResponseParams {
+}
+export declare namespace Systems {
+    export { type System as System, type SystemDeleteResponse as SystemDeleteResponse, type SystemsPaginatedResponse as SystemsPaginatedResponse, type SystemCreateParams as SystemCreateParams, type SystemUpdateParams as SystemUpdateParams, type SystemListParams as SystemListParams, };
+}
+//# sourceMappingURL=systems.d.mts.map

package/resources/systems.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"systems.d.mts","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE;OAChE,EAAE,cAAc,EAAE;AAGzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;IAIjG;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE,kBAAkB,GAAG,IAAI,GAAG,SAAc,EAChD,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,MAAM,CAAC;IAIrB;;OAEG;IACH,IAAI,CACF,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,gBAAgB,GAAG,IAAI,GAAG,SAAc,EAC/C,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,wBAAwB,EAAE,MAAM,CAAC;IAOhD;;;OAGG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,oBAAoB,CAAC;IAIpF;;OAEG;IACH,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;CAGpE;AAED,MAAM,MAAM,wBAAwB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;AAEjE;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,MAAM;IACrB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,gBAAiB,SAAQ,uBAAuB;CAAG;AAEpE,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,MAAM,IAAI,MAAM,EACrB,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,gBAAgB,IAAI,gBAAgB,GAC1C,CAAC;CACH"}

package/resources/systems.d.ts

@@ -0,0 +1,152 @@
+import { APIResource } from "../core/resource.js";
+import { APIPromise } from "../core/api-promise.js";
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../core/pagination.js";
+import { RequestOptions } from "../internal/request-options.js";
+export declare class Systems extends APIResource {
+    /**
+     * Create a new system definition that specifies the interface contracts for a
+     * component you want to evaluate.
+     *
+     * A system acts as a template that defines three key contracts through JSON
+     * Schemas:
+     *
+     * 1. Input Schema: What data your system accepts (e.g., user queries, context
+     *    documents)
+     * 2. Output Schema: What data your system produces (e.g., responses, confidence
+     *    scores)
+     * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+     *    temperature)
+     *
+     * This separation lets you evaluate any system as a black box, focusing on its
+     * interface rather than implementation details.
+     */
+    create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Update an existing system definition. Only the fields provided in the request
+     * body will be updated. If a field is provided, the new content will replace the
+     * existing content. If a field is not provided, the existing content will remain
+     * unchanged.
+     *
+     * When updating schemas:
+     *
+     * - The system will accept your changes regardless of compatibility with existing
+     *   configurations
+     * - Schema updates won't invalidate existing evaluations or configurations
+     * - For significant redesigns, creating a new system definition provides a cleaner
+     *   separation
+     */
+    update(systemID: string, body?: SystemUpdateParams | null | undefined, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+     */
+    list(projectID: string, query?: SystemListParams | null | undefined, options?: RequestOptions): PagePromise<SystemsPaginatedResponse, System>;
+    /**
+     * Delete a system definition by ID. This will not delete associated system
+     * configurations.
+     */
+    delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse>;
+    /**
+     * Retrieve a specific system by ID.
+     */
+    get(systemID: string, options?: RequestOptions): APIPromise<System>;
+}
+export type SystemsPaginatedResponse = PaginatedResponse<System>;
+/**
+ * A System Under Test (SUT) defines the interface to a component or service you
+ * want to evaluate.
+ *
+ * It specifies three contracts through schemas:
+ *
+ * - inputSchema: The structure of data the system accepts
+ * - outputSchema: The structure of data the system produces
+ * - configSchema: The parameters that modify system behavior
+ *
+ * This abstraction lets you evaluate any system as a black box, focusing on its
+ * interface rather than implementation details. It's particularly useful for
+ * systems with variable outputs or complex internal state.
+ *
+ * Systems are templates - to run evaluations, pair them with a SystemConfig that
+ * provides specific parameter values.
+ */
+export interface System {
+    /**
+     * The ID of the system
+     */
+    id: string;
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemDeleteResponse {
+    /**
+     * Whether the deletion was successful
+     */
+    success: boolean;
+}
+export interface SystemCreateParams {
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemUpdateParams {
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema?: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description?: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema?: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name?: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema?: Record<string, unknown>;
+}
+export interface SystemListParams extends PaginatedResponseParams {
+}
+export declare namespace Systems {
+    export { type System as System, type SystemDeleteResponse as SystemDeleteResponse, type SystemsPaginatedResponse as SystemsPaginatedResponse, type SystemCreateParams as SystemCreateParams, type SystemUpdateParams as SystemUpdateParams, type SystemListParams as SystemListParams, };
+}
+//# sourceMappingURL=systems.d.ts.map

package/resources/systems.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"systems.d.ts","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE;OAChE,EAAE,cAAc,EAAE;AAGzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;IAIjG;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE,kBAAkB,GAAG,IAAI,GAAG,SAAc,EAChD,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,MAAM,CAAC;IAIrB;;OAEG;IACH,IAAI,CACF,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,gBAAgB,GAAG,IAAI,GAAG,SAAc,EAC/C,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,wBAAwB,EAAE,MAAM,CAAC;IAOhD;;;OAGG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,oBAAoB,CAAC;IAIpF;;OAEG;IACH,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;CAGpE;AAED,MAAM,MAAM,wBAAwB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;AAEjE;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,MAAM;IACrB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,gBAAiB,SAAQ,uBAAuB;CAAG;AAEpE,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,MAAM,IAAI,MAAM,EACrB,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,gBAAgB,IAAI,gBAAgB,GAC1C,CAAC;CACH"}

package/resources/systems.js

@@ -0,0 +1,70 @@
+"use strict";
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Systems = void 0;
+const resource_1 = require("../core/resource.js");
+const pagination_1 = require("../core/pagination.js");
+const path_1 = require("../internal/utils/path.js");
+class Systems extends resource_1.APIResource {
+    /**
+     * Create a new system definition that specifies the interface contracts for a
+     * component you want to evaluate.
+     *
+     * A system acts as a template that defines three key contracts through JSON
+     * Schemas:
+     *
+     * 1. Input Schema: What data your system accepts (e.g., user queries, context
+     *    documents)
+     * 2. Output Schema: What data your system produces (e.g., responses, confidence
+     *    scores)
+     * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+     *    temperature)
+     *
+     * This separation lets you evaluate any system as a black box, focusing on its
+     * interface rather than implementation details.
+     */
+    create(projectID, body, options) {
+        return this._client.post((0, path_1.path) `/projects/${projectID}/systems`, { body, ...options });
+    }
+    /**
+     * Update an existing system definition. Only the fields provided in the request
+     * body will be updated. If a field is provided, the new content will replace the
+     * existing content. If a field is not provided, the existing content will remain
+     * unchanged.
+     *
+     * When updating schemas:
+     *
+     * - The system will accept your changes regardless of compatibility with existing
+     *   configurations
+     * - Schema updates won't invalidate existing evaluations or configurations
+     * - For significant redesigns, creating a new system definition provides a cleaner
+     *   separation
+     */
+    update(systemID, body = {}, options) {
+        return this._client.patch((0, path_1.path) `/systems/${systemID}`, { body, ...options });
+    }
+    /**
+     * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+     */
+    list(projectID, query = {}, options) {
+        return this._client.getAPIList((0, path_1.path) `/projects/${projectID}/systems`, (pagination_1.PaginatedResponse), {
+            query,
+            ...options,
+        });
+    }
+    /**
+     * Delete a system definition by ID. This will not delete associated system
+     * configurations.
+     */
+    delete(systemID, options) {
+        return this._client.delete((0, path_1.path) `/systems/${systemID}`, options);
+    }
+    /**
+     * Retrieve a specific system by ID.
+     */
+    get(systemID, options) {
+        return this._client.get((0, path_1.path) `/systems/${systemID}`, options);
+    }
+}
+exports.Systems = Systems;
+//# sourceMappingURL=systems.js.map

package/resources/systems.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"systems.js","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAA+C;AAE/C,sDAAkG;AAElG,oDAA8C;AAE9C,MAAa,OAAQ,SAAQ,sBAAW;IACtC;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CAAC,SAAiB,EAAE,IAAwB,EAAE,OAAwB;QAC1E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAA,WAAI,EAAA,aAAa,SAAS,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,QAAgB,EAChB,OAA8C,EAAE,EAChD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAA,WAAI,EAAA,YAAY,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED;;OAEG;IACH,IAAI,CACF,SAAiB,EACjB,QAA6C,EAAE,EAC/C,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAA,WAAI,EAAA,aAAa,SAAS,UAAU,EAAE,CAAA,8BAAyB,CAAA,EAAE;YAC9F,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,QAAgB,EAAE,OAAwB;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAA,WAAI,EAAA,YAAY,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,QAAgB,EAAE,OAAwB;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,YAAY,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IAC/D,CAAC;CACF;AAxED,0BAwEC"}

package/resources/systems.mjs

@@ -0,0 +1,66 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from "../core/resource.mjs";
+import { PaginatedResponse } from "../core/pagination.mjs";
+import { path } from "../internal/utils/path.mjs";
+export class Systems extends APIResource {
+    /**
+     * Create a new system definition that specifies the interface contracts for a
+     * component you want to evaluate.
+     *
+     * A system acts as a template that defines three key contracts through JSON
+     * Schemas:
+     *
+     * 1. Input Schema: What data your system accepts (e.g., user queries, context
+     *    documents)
+     * 2. Output Schema: What data your system produces (e.g., responses, confidence
+     *    scores)
+     * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+     *    temperature)
+     *
+     * This separation lets you evaluate any system as a black box, focusing on its
+     * interface rather than implementation details.
+     */
+    create(projectID, body, options) {
+        return this._client.post(path `/projects/${projectID}/systems`, { body, ...options });
+    }
+    /**
+     * Update an existing system definition. Only the fields provided in the request
+     * body will be updated. If a field is provided, the new content will replace the
+     * existing content. If a field is not provided, the existing content will remain
+     * unchanged.
+     *
+     * When updating schemas:
+     *
+     * - The system will accept your changes regardless of compatibility with existing
+     *   configurations
+     * - Schema updates won't invalidate existing evaluations or configurations
+     * - For significant redesigns, creating a new system definition provides a cleaner
+     *   separation
+     */
+    update(systemID, body = {}, options) {
+        return this._client.patch(path `/systems/${systemID}`, { body, ...options });
+    }
+    /**
+     * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+     */
+    list(projectID, query = {}, options) {
+        return this._client.getAPIList(path `/projects/${projectID}/systems`, (PaginatedResponse), {
+            query,
+            ...options,
+        });
+    }
+    /**
+     * Delete a system definition by ID. This will not delete associated system
+     * configurations.
+     */
+    delete(systemID, options) {
+        return this._client.delete(path `/systems/${systemID}`, options);
+    }
+    /**
+     * Retrieve a specific system by ID.
+     */
+    get(systemID, options) {
+        return this._client.get(path `/systems/${systemID}`, options);
+    }
+}
+//# sourceMappingURL=systems.mjs.map

package/resources/systems.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"systems.mjs","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAEf,EAAe,iBAAiB,EAAgC;OAEhE,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CAAC,SAAiB,EAAE,IAAwB,EAAE,OAAwB;QAC1E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,aAAa,SAAS,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,QAAgB,EAChB,OAA8C,EAAE,EAChD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAA,YAAY,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED;;OAEG;IACH,IAAI,CACF,SAAiB,EACjB,QAA6C,EAAE,EAC/C,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAA,aAAa,SAAS,UAAU,EAAE,CAAA,iBAAyB,CAAA,EAAE;YAC9F,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,QAAgB,EAAE,OAAwB;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAA,YAAY,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,QAAgB,EAAE,OAAwB;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,YAAY,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IAC/D,CAAC;CACF"}

package/resources/testcases.d.mts

@@ -16,9 +16,9 @@ export declare class Testcases extends APIResource {
      */
     list(testsetID: string, query?: TestcaseListParams | null | undefined, options?: RequestOptions): PagePromise<TestcasesPaginatedResponse, Testcase>;
     /**
-     * Delete multiple testcases from the specified testset.
+     * Delete multiple testcases by their IDs.
      */
-    delete(testsetID: string, body: TestcaseDeleteParams, options?: RequestOptions): APIPromise<TestcaseDeleteResponse>;
+    delete(body: TestcaseDeleteParams, options?: RequestOptions): APIPromise<TestcaseDeleteResponse>;
     /**
      * Retrieve a specific testcase by ID.
      */
@@ -38,15 +38,15 @@ export interface Testcase {
      * The ID of the testcase
      */
     id: string;
-    /**
-     * The JSON data of the testcase, which is validated against the testset's schema.
-     */
-    data: Record<string, unknown>;
     /**
      * Derived from data based on the testset's fieldMapping. Contains all fields
      * marked as inputs, including those with validation errors.
      */
     inputs: Record<string, unknown>;
+    /**
+     * The JSON data of the testcase, which is validated against the testset's schema.
+     */
+    jsonData: Record<string, unknown>;
     /**
      * Derived from data based on the testset's fieldMapping. Contains all fields
      * marked as labels, including those with validation errors.
@@ -79,25 +79,9 @@ export interface TestcaseCreateResponse {
 }
 export interface TestcaseDeleteResponse {
     /**
-     * Number of testcases successfully deleted
-     */
-    deletedCount: number;
-    /**
-     * List of errors encountered during deletion, if any
+     * Whether the deletion was successful
      */
-    errors: Array<TestcaseDeleteResponse.Error>;
-}
-export declare namespace TestcaseDeleteResponse {
-    interface Error {
-        /**
-         * ID of the testcase that failed to be deleted
-         */
-        id: string;
-        /**
-         * Error message explaining why the deletion failed
-         */
-        message: string;
-    }
+    success: boolean;
 }
 export interface TestcaseCreateParams {
     /**
@@ -118,20 +102,20 @@ export declare namespace TestcaseCreateParams {
         /**
          * The JSON data of the testcase, which is validated against the testset's schema.
          */
-        data: Record<string, unknown>;
+        jsonData: Record<string, unknown>;
     }
 }
 export interface TestcaseUpdateParams {
     /**
      * The JSON data of the testcase, which is validated against the testset's schema.
      */
-    data: Record<string, unknown>;
+    jsonData: Record<string, unknown>;
 }
 export interface TestcaseListParams extends PaginatedResponseParams {
 }
 export interface TestcaseDeleteParams {
     /**
-     * IDs of testcases to delete (max 100)
+     * IDs of testcases to delete
      */
     ids: Array<string>;
 }