scorecard-ai 0.6.1 → 1.0.0-alpha.1

package/resources/system-configs.mjs

@@ -0,0 +1,46 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from "../core/resource.mjs";
+import { PaginatedResponse } from "../core/pagination.mjs";
+import { path } from "../internal/utils/path.mjs";
+export class SystemConfigs extends APIResource {
+    /**
+     * Create a new configuration for a system.
+     *
+     * Each configuration contains specific parameter values that match the system's
+     * configSchema - things like model parameters, thresholds, or processing options.
+     * Once created, configurations cannot be modified, ensuring stable reference
+     * points for evaluations.
+     *
+     * When creating a configuration:
+     *
+     * - The 'config' object is validated against the parent system's configSchema
+     * - Configurations with validation errors are still stored, with errors included
+     *   in the response
+     * - Validation errors indicate fields that don't match the schema but don't
+     *   prevent creation
+     * - Having validation errors may affect how some evaluation metrics are calculated
+     */
+    create(systemID, body, options) {
+        return this._client.post(path `/systems/${systemID}/configs`, { body, ...options });
+    }
+    /**
+     * Retrieve a paginated list of configurations for a specific system.
+     *
+     * System configurations provide concrete parameter values for a System Under Test,
+     * defining exactly how the system should be configured during an evaluation run.
+     */
+    list(systemID, query = {}, options) {
+        return this._client.getAPIList(path `/systems/${systemID}/configs`, (PaginatedResponse), {
+            query,
+            ...options,
+        });
+    }
+    /**
+     * Retrieve a specific system configuration by ID.
+     */
+    get(systemConfigID, params, options) {
+        const { systemId } = params;
+        return this._client.get(path `/systems/${systemId}/configs/${systemConfigID}`, options);
+    }
+}
+//# sourceMappingURL=system-configs.mjs.map

package/resources/system-configs.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"system-configs.mjs","sourceRoot":"","sources":["../src/resources/system-configs.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAEf,EAAe,iBAAiB,EAAgC;OAEhE,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,aAAc,SAAQ,WAAW;IAC5C;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CACJ,QAAgB,EAChB,IAA8B,EAC9B,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,YAAY,QAAQ,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACrF,CAAC;IAED;;;;;OAKG;IACH,IAAI,CACF,QAAgB,EAChB,QAAmD,EAAE,EACrD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAA,YAAY,QAAQ,UAAU,EAAE,CAAA,iBAA+B,CAAA,EAAE;YAClG,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,GAAG,CACD,cAAsB,EACtB,MAA6B,EAC7B,OAAwB;QAExB,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,YAAY,QAAQ,YAAY,cAAc,EAAE,EAAE,OAAO,CAAC,CAAC;IACzF,CAAC;CACF"}

package/resources/systems.d.mts

@@ -0,0 +1,152 @@
+import { APIResource } from "../core/resource.mjs";
+import { APIPromise } from "../core/api-promise.mjs";
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../core/pagination.mjs";
+import { RequestOptions } from "../internal/request-options.mjs";
+export declare class Systems extends APIResource {
+    /**
+     * Create a new system definition that specifies the interface contracts for a
+     * component you want to evaluate.
+     *
+     * A system acts as a template that defines three key contracts through JSON
+     * Schemas:
+     *
+     * 1. Input Schema: What data your system accepts (e.g., user queries, context
+     *    documents)
+     * 2. Output Schema: What data your system produces (e.g., responses, confidence
+     *    scores)
+     * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+     *    temperature)
+     *
+     * This separation lets you evaluate any system as a black box, focusing on its
+     * interface rather than implementation details.
+     */
+    create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Update an existing system definition. Only the fields provided in the request
+     * body will be updated. If a field is provided, the new content will replace the
+     * existing content. If a field is not provided, the existing content will remain
+     * unchanged.
+     *
+     * When updating schemas:
+     *
+     * - The system will accept your changes regardless of compatibility with existing
+     *   configurations
+     * - Schema updates won't invalidate existing evaluations or configurations
+     * - For significant redesigns, creating a new system definition provides a cleaner
+     *   separation
+     */
+    update(systemID: string, body?: SystemUpdateParams | null | undefined, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+     */
+    list(projectID: string, query?: SystemListParams | null | undefined, options?: RequestOptions): PagePromise<SystemsPaginatedResponse, System>;
+    /**
+     * Delete a system definition by ID. This will not delete associated system
+     * configurations.
+     */
+    delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse>;
+    /**
+     * Retrieve a specific system by ID.
+     */
+    get(systemID: string, options?: RequestOptions): APIPromise<System>;
+}
+export type SystemsPaginatedResponse = PaginatedResponse<System>;
+/**
+ * A System Under Test (SUT) defines the interface to a component or service you
+ * want to evaluate.
+ *
+ * It specifies three contracts through schemas:
+ *
+ * - inputSchema: The structure of data the system accepts
+ * - outputSchema: The structure of data the system produces
+ * - configSchema: The parameters that modify system behavior
+ *
+ * This abstraction lets you evaluate any system as a black box, focusing on its
+ * interface rather than implementation details. It's particularly useful for
+ * systems with variable outputs or complex internal state.
+ *
+ * Systems are templates - to run evaluations, pair them with a SystemConfig that
+ * provides specific parameter values.
+ */
+export interface System {
+    /**
+     * The ID of the system
+     */
+    id: string;
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemDeleteResponse {
+    /**
+     * Whether the deletion was successful
+     */
+    success: boolean;
+}
+export interface SystemCreateParams {
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemUpdateParams {
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema?: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description?: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema?: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name?: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema?: Record<string, unknown>;
+}
+export interface SystemListParams extends PaginatedResponseParams {
+}
+export declare namespace Systems {
+    export { type System as System, type SystemDeleteResponse as SystemDeleteResponse, type SystemsPaginatedResponse as SystemsPaginatedResponse, type SystemCreateParams as SystemCreateParams, type SystemUpdateParams as SystemUpdateParams, type SystemListParams as SystemListParams, };
+}
+//# sourceMappingURL=systems.d.mts.map

package/resources/systems.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"systems.d.mts","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE;OAChE,EAAE,cAAc,EAAE;AAGzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;IAIjG;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE,kBAAkB,GAAG,IAAI,GAAG,SAAc,EAChD,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,MAAM,CAAC;IAIrB;;OAEG;IACH,IAAI,CACF,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,gBAAgB,GAAG,IAAI,GAAG,SAAc,EAC/C,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,wBAAwB,EAAE,MAAM,CAAC;IAOhD;;;OAGG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,oBAAoB,CAAC;IAIpF;;OAEG;IACH,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;CAGpE;AAED,MAAM,MAAM,wBAAwB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;AAEjE;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,MAAM;IACrB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,gBAAiB,SAAQ,uBAAuB;CAAG;AAEpE,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,MAAM,IAAI,MAAM,EACrB,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,gBAAgB,IAAI,gBAAgB,GAC1C,CAAC;CACH"}

package/resources/systems.d.ts

@@ -0,0 +1,152 @@
+import { APIResource } from "../core/resource.js";
+import { APIPromise } from "../core/api-promise.js";
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../core/pagination.js";
+import { RequestOptions } from "../internal/request-options.js";
+export declare class Systems extends APIResource {
+    /**
+     * Create a new system definition that specifies the interface contracts for a
+     * component you want to evaluate.
+     *
+     * A system acts as a template that defines three key contracts through JSON
+     * Schemas:
+     *
+     * 1. Input Schema: What data your system accepts (e.g., user queries, context
+     *    documents)
+     * 2. Output Schema: What data your system produces (e.g., responses, confidence
+     *    scores)
+     * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+     *    temperature)
+     *
+     * This separation lets you evaluate any system as a black box, focusing on its
+     * interface rather than implementation details.
+     */
+    create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Update an existing system definition. Only the fields provided in the request
+     * body will be updated. If a field is provided, the new content will replace the
+     * existing content. If a field is not provided, the existing content will remain
+     * unchanged.
+     *
+     * When updating schemas:
+     *
+     * - The system will accept your changes regardless of compatibility with existing
+     *   configurations
+     * - Schema updates won't invalidate existing evaluations or configurations
+     * - For significant redesigns, creating a new system definition provides a cleaner
+     *   separation
+     */
+    update(systemID: string, body?: SystemUpdateParams | null | undefined, options?: RequestOptions): APIPromise<System>;
+    /**
+     * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+     */
+    list(projectID: string, query?: SystemListParams | null | undefined, options?: RequestOptions): PagePromise<SystemsPaginatedResponse, System>;
+    /**
+     * Delete a system definition by ID. This will not delete associated system
+     * configurations.
+     */
+    delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse>;
+    /**
+     * Retrieve a specific system by ID.
+     */
+    get(systemID: string, options?: RequestOptions): APIPromise<System>;
+}
+export type SystemsPaginatedResponse = PaginatedResponse<System>;
+/**
+ * A System Under Test (SUT) defines the interface to a component or service you
+ * want to evaluate.
+ *
+ * It specifies three contracts through schemas:
+ *
+ * - inputSchema: The structure of data the system accepts
+ * - outputSchema: The structure of data the system produces
+ * - configSchema: The parameters that modify system behavior
+ *
+ * This abstraction lets you evaluate any system as a black box, focusing on its
+ * interface rather than implementation details. It's particularly useful for
+ * systems with variable outputs or complex internal state.
+ *
+ * Systems are templates - to run evaluations, pair them with a SystemConfig that
+ * provides specific parameter values.
+ */
+export interface System {
+    /**
+     * The ID of the system
+     */
+    id: string;
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemDeleteResponse {
+    /**
+     * Whether the deletion was successful
+     */
+    success: boolean;
+}
+export interface SystemCreateParams {
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema: Record<string, unknown>;
+}
+export interface SystemUpdateParams {
+    /**
+     * The schema of the system's configuration
+     */
+    configSchema?: Record<string, unknown>;
+    /**
+     * The description of the system
+     */
+    description?: string;
+    /**
+     * The schema of the system's inputs
+     */
+    inputSchema?: Record<string, unknown>;
+    /**
+     * The name of the system
+     */
+    name?: string;
+    /**
+     * The schema of the system's outputs
+     */
+    outputSchema?: Record<string, unknown>;
+}
+export interface SystemListParams extends PaginatedResponseParams {
+}
+export declare namespace Systems {
+    export { type System as System, type SystemDeleteResponse as SystemDeleteResponse, type SystemsPaginatedResponse as SystemsPaginatedResponse, type SystemCreateParams as SystemCreateParams, type SystemUpdateParams as SystemUpdateParams, type SystemListParams as SystemListParams, };
+}
+//# sourceMappingURL=systems.d.ts.map

package/resources/systems.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"systems.d.ts","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE;OAChE,EAAE,cAAc,EAAE;AAGzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;IAIjG;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE,kBAAkB,GAAG,IAAI,GAAG,SAAc,EAChD,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,MAAM,CAAC;IAIrB;;OAEG;IACH,IAAI,CACF,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,gBAAgB,GAAG,IAAI,GAAG,SAAc,EAC/C,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,wBAAwB,EAAE,MAAM,CAAC;IAOhD;;;OAGG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,oBAAoB,CAAC;IAIpF;;OAEG;IACH,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;CAGpE;AAED,MAAM,MAAM,wBAAwB,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;AAEjE;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,MAAM;IACrB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAErC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEtC;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,gBAAiB,SAAQ,uBAAuB;CAAG;AAEpE,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,MAAM,IAAI,MAAM,EACrB,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,gBAAgB,IAAI,gBAAgB,GAC1C,CAAC;CACH"}

package/resources/systems.js

@@ -0,0 +1,70 @@
+"use strict";
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Systems = void 0;
+const resource_1 = require("../core/resource.js");
+const pagination_1 = require("../core/pagination.js");
+const path_1 = require("../internal/utils/path.js");
+class Systems extends resource_1.APIResource {
+    /**
+     * Create a new system definition that specifies the interface contracts for a
+     * component you want to evaluate.
+     *
+     * A system acts as a template that defines three key contracts through JSON
+     * Schemas:
+     *
+     * 1. Input Schema: What data your system accepts (e.g., user queries, context
+     *    documents)
+     * 2. Output Schema: What data your system produces (e.g., responses, confidence
+     *    scores)
+     * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+     *    temperature)
+     *
+     * This separation lets you evaluate any system as a black box, focusing on its
+     * interface rather than implementation details.
+     */
+    create(projectID, body, options) {
+        return this._client.post((0, path_1.path) `/projects/${projectID}/systems`, { body, ...options });
+    }
+    /**
+     * Update an existing system definition. Only the fields provided in the request
+     * body will be updated. If a field is provided, the new content will replace the
+     * existing content. If a field is not provided, the existing content will remain
+     * unchanged.
+     *
+     * When updating schemas:
+     *
+     * - The system will accept your changes regardless of compatibility with existing
+     *   configurations
+     * - Schema updates won't invalidate existing evaluations or configurations
+     * - For significant redesigns, creating a new system definition provides a cleaner
+     *   separation
+     */
+    update(systemID, body = {}, options) {
+        return this._client.patch((0, path_1.path) `/systems/${systemID}`, { body, ...options });
+    }
+    /**
+     * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+     */
+    list(projectID, query = {}, options) {
+        return this._client.getAPIList((0, path_1.path) `/projects/${projectID}/systems`, (pagination_1.PaginatedResponse), {
+            query,
+            ...options,
+        });
+    }
+    /**
+     * Delete a system definition by ID. This will not delete associated system
+     * configurations.
+     */
+    delete(systemID, options) {
+        return this._client.delete((0, path_1.path) `/systems/${systemID}`, options);
+    }
+    /**
+     * Retrieve a specific system by ID.
+     */
+    get(systemID, options) {
+        return this._client.get((0, path_1.path) `/systems/${systemID}`, options);
+    }
+}
+exports.Systems = Systems;
+//# sourceMappingURL=systems.js.map

package/resources/systems.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"systems.js","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAA+C;AAE/C,sDAAkG;AAElG,oDAA8C;AAE9C,MAAa,OAAQ,SAAQ,sBAAW;IACtC;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CAAC,SAAiB,EAAE,IAAwB,EAAE,OAAwB;QAC1E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAA,WAAI,EAAA,aAAa,SAAS,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,QAAgB,EAChB,OAA8C,EAAE,EAChD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAA,WAAI,EAAA,YAAY,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED;;OAEG;IACH,IAAI,CACF,SAAiB,EACjB,QAA6C,EAAE,EAC/C,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAA,WAAI,EAAA,aAAa,SAAS,UAAU,EAAE,CAAA,8BAAyB,CAAA,EAAE;YAC9F,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,QAAgB,EAAE,OAAwB;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAA,WAAI,EAAA,YAAY,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,QAAgB,EAAE,OAAwB;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,YAAY,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IAC/D,CAAC;CACF;AAxED,0BAwEC"}

package/resources/systems.mjs

@@ -0,0 +1,66 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from "../core/resource.mjs";
+import { PaginatedResponse } from "../core/pagination.mjs";
+import { path } from "../internal/utils/path.mjs";
+export class Systems extends APIResource {
+    /**
+     * Create a new system definition that specifies the interface contracts for a
+     * component you want to evaluate.
+     *
+     * A system acts as a template that defines three key contracts through JSON
+     * Schemas:
+     *
+     * 1. Input Schema: What data your system accepts (e.g., user queries, context
+     *    documents)
+     * 2. Output Schema: What data your system produces (e.g., responses, confidence
+     *    scores)
+     * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+     *    temperature)
+     *
+     * This separation lets you evaluate any system as a black box, focusing on its
+     * interface rather than implementation details.
+     */
+    create(projectID, body, options) {
+        return this._client.post(path `/projects/${projectID}/systems`, { body, ...options });
+    }
+    /**
+     * Update an existing system definition. Only the fields provided in the request
+     * body will be updated. If a field is provided, the new content will replace the
+     * existing content. If a field is not provided, the existing content will remain
+     * unchanged.
+     *
+     * When updating schemas:
+     *
+     * - The system will accept your changes regardless of compatibility with existing
+     *   configurations
+     * - Schema updates won't invalidate existing evaluations or configurations
+     * - For significant redesigns, creating a new system definition provides a cleaner
+     *   separation
+     */
+    update(systemID, body = {}, options) {
+        return this._client.patch(path `/systems/${systemID}`, { body, ...options });
+    }
+    /**
+     * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+     */
+    list(projectID, query = {}, options) {
+        return this._client.getAPIList(path `/projects/${projectID}/systems`, (PaginatedResponse), {
+            query,
+            ...options,
+        });
+    }
+    /**
+     * Delete a system definition by ID. This will not delete associated system
+     * configurations.
+     */
+    delete(systemID, options) {
+        return this._client.delete(path `/systems/${systemID}`, options);
+    }
+    /**
+     * Retrieve a specific system by ID.
+     */
+    get(systemID, options) {
+        return this._client.get(path `/systems/${systemID}`, options);
+    }
+}
+//# sourceMappingURL=systems.mjs.map

package/resources/systems.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"systems.mjs","sourceRoot":"","sources":["../src/resources/systems.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAEf,EAAe,iBAAiB,EAAgC;OAEhE,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CAAC,SAAiB,EAAE,IAAwB,EAAE,OAAwB;QAC1E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,aAAa,SAAS,UAAU,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACvF,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,QAAgB,EAChB,OAA8C,EAAE,EAChD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAA,YAAY,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED;;OAEG;IACH,IAAI,CACF,SAAiB,EACjB,QAA6C,EAAE,EAC/C,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAA,aAAa,SAAS,UAAU,EAAE,CAAA,iBAAyB,CAAA,EAAE;YAC9F,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,QAAgB,EAAE,OAAwB;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAA,YAAY,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,QAAgB,EAAE,OAAwB;QAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,YAAY,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IAC/D,CAAC;CACF"}

package/resources/testcases.d.mts

@@ -0,0 +1,125 @@
+import { APIResource } from "../core/resource.mjs";
+import { APIPromise } from "../core/api-promise.mjs";
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../core/pagination.mjs";
+import { RequestOptions } from "../internal/request-options.mjs";
+export declare class Testcases extends APIResource {
+    /**
+     * Create multiple testcases in the specified testset.
+     */
+    create(testsetID: string, body: TestcaseCreateParams, options?: RequestOptions): APIPromise<TestcaseCreateResponse>;
+    /**
+     * Replace the data of an existing testcase while keeping its ID.
+     */
+    update(testcaseID: string, body: TestcaseUpdateParams, options?: RequestOptions): APIPromise<Testcase>;
+    /**
+     * Retrieve a paginated list of testcases belonging to a testset.
+     */
+    list(testsetID: string, query?: TestcaseListParams | null | undefined, options?: RequestOptions): PagePromise<TestcasesPaginatedResponse, Testcase>;
+    /**
+     * Delete multiple testcases by their IDs.
+     */
+    delete(body: TestcaseDeleteParams, options?: RequestOptions): APIPromise<TestcaseDeleteResponse>;
+    /**
+     * Retrieve a specific testcase by ID.
+     */
+    get(testcaseID: string, options?: RequestOptions): APIPromise<Testcase>;
+}
+export type TestcasesPaginatedResponse = PaginatedResponse<Testcase>;
+/**
+ * A test case in the Scorecard system. Contains JSON data that is validated
+ * against the schema defined by its testset. The `inputs` and `labels` fields are
+ * derived from the `data` field based on the testset's `fieldMapping`, and include
+ * all mapped fields, including those with validation errors. Testcases are stored
+ * regardless of validation results, with any validation errors included in the
+ * `validationErrors` field.
+ */
+export interface Testcase {
+    /**
+     * The ID of the testcase
+     */
+    id: string;
+    /**
+     * Derived from data based on the testset's fieldMapping. Contains all fields
+     * marked as inputs, including those with validation errors.
+     */
+    inputs: Record<string, unknown>;
+    /**
+     * The JSON data of the testcase, which is validated against the testset's schema.
+     */
+    jsonData: Record<string, unknown>;
+    /**
+     * Derived from data based on the testset's fieldMapping. Contains all fields
+     * marked as labels, including those with validation errors.
+     */
+    labels: Record<string, unknown>;
+    /**
+     * The ID of the testset this testcase belongs to
+     */
+    testsetId: string;
+    /**
+     * Validation errors found in the testcase data. If present, the testcase doesn't
+     * fully conform to its testset's schema.
+     */
+    validationErrors?: Array<Testcase.ValidationError>;
+}
+export declare namespace Testcase {
+    interface ValidationError {
+        /**
+         * Human-readable error description
+         */
+        message: string;
+        /**
+         * JSON Pointer to the field with the validation error
+         */
+        path: string;
+    }
+}
+export interface TestcaseCreateResponse {
+    items: Array<Testcase>;
+}
+export interface TestcaseDeleteResponse {
+    /**
+     * Whether the deletion was successful
+     */
+    success: boolean;
+}
+export interface TestcaseCreateParams {
+    /**
+     * Testcases to create (max 100)
+     */
+    items: Array<TestcaseCreateParams.Item>;
+}
+export declare namespace TestcaseCreateParams {
+    /**
+     * A test case in the Scorecard system. Contains JSON data that is validated
+     * against the schema defined by its testset. The `inputs` and `labels` fields are
+     * derived from the `data` field based on the testset's `fieldMapping`, and include
+     * all mapped fields, including those with validation errors. Testcases are stored
+     * regardless of validation results, with any validation errors included in the
+     * `validationErrors` field.
+     */
+    interface Item {
+        /**
+         * The JSON data of the testcase, which is validated against the testset's schema.
+         */
+        jsonData: Record<string, unknown>;
+    }
+}
+export interface TestcaseUpdateParams {
+    /**
+     * The JSON data of the testcase, which is validated against the testset's schema.
+     */
+    jsonData: Record<string, unknown>;
+}
+export interface TestcaseListParams extends PaginatedResponseParams {
+}
+export interface TestcaseDeleteParams {
+    /**
+     * IDs of testcases to delete
+     */
+    ids: Array<string>;
+}
+export declare namespace Testcases {
+    export { type Testcase as Testcase, type TestcaseCreateResponse as TestcaseCreateResponse, type TestcaseDeleteResponse as TestcaseDeleteResponse, type TestcasesPaginatedResponse as TestcasesPaginatedResponse, type TestcaseCreateParams as TestcaseCreateParams, type TestcaseUpdateParams as TestcaseUpdateParams, type TestcaseListParams as TestcaseListParams, type TestcaseDeleteParams as TestcaseDeleteParams, };
+}
+//# sourceMappingURL=testcases.d.mts.map

package/resources/testcases.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"testcases.d.mts","sourceRoot":"","sources":["../src/resources/testcases.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE;OAChE,EAAE,cAAc,EAAE;AAGzB,qBAAa,SAAU,SAAQ,WAAW;IACxC;;OAEG;IACH,MAAM,CACJ,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,oBAAoB,EAC1B,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,sBAAsB,CAAC;IAIrC;;OAEG;IACH,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,QAAQ,CAAC;IAItG;;OAEG;IACH,IAAI,CACF,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,kBAAkB,GAAG,IAAI,GAAG,SAAc,EACjD,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,0BAA0B,EAAE,QAAQ,CAAC;IAOpD;;OAEG;IACH,MAAM,CAAC,IAAI,EAAE,oBAAoB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,sBAAsB,CAAC;IAIhG;;OAEG;IACH,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,QAAQ,CAAC;CAGxE;AAED,MAAM,MAAM,0BAA0B,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;AAErE;;;;;;;GAOG;AACH,MAAM,WAAW,QAAQ;IACvB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAElC;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;CACpD;AAED,yBAAiB,QAAQ,CAAC;IACxB,UAAiB,eAAe;QAC9B;;WAEG;QACH,OAAO,EAAE,MAAM,CAAC;QAEhB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;KACd;CACF;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,sBAAsB;IACrC;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,KAAK,EAAE,KAAK,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;CACzC;AAED,yBAAiB,oBAAoB,CAAC;IACpC;;;;;;;OAOG;IACH,UAAiB,IAAI;QACnB;;WAEG;QACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC;CACF;AAED,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,kBAAmB,SAAQ,uBAAuB;CAAG;AAEtE,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACpB;AAED,MAAM,CAAC,OAAO,WAAW,SAAS,CAAC;IACjC,OAAO,EACL,KAAK,QAAQ,IAAI,QAAQ,EACzB,KAAK,sBAAsB,IAAI,sBAAsB,EACrD,KAAK,sBAAsB,IAAI,sBAAsB,EACrD,KAAK,0BAA0B,IAAI,0BAA0B,EAC7D,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,oBAAoB,IAAI,oBAAoB,EACjD,KAAK,kBAAkB,IAAI,kBAAkB,EAC7C,KAAK,oBAAoB,IAAI,oBAAoB,GAClD,CAAC;CACH"}