scorecard-ai 1.2.0 → 2.0.0-alpha.1

package/src/resources/systems/systems.ts

@@ -2,13 +2,7 @@
 
 import { APIResource } from '../../core/resource';
 import * as VersionsAPI from './versions';
-import {
-  SystemVersion,
-  SystemVersionsPaginatedResponse,
-  VersionCreateParams,
-  VersionListParams,
-  Versions,
-} from './versions';
+import { SystemVersion, VersionUpsertParams, Versions } from './versions';
 import { APIPromise } from '../../core/api-promise';
 import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../../core/pagination';
 import { RequestOptions } from '../../internal/request-options';
@@ -18,87 +12,17 @@ export class Systems extends APIResource {
   versions: VersionsAPI.Versions = new VersionsAPI.Versions(this._client);
 
   /**
-   * Create a new system definition that specifies the interface contracts for a
-   * component you want to evaluate.
-   *
-   * A system acts as a template that defines three key contracts through JSON
-   * Schemas:
-   *
-   * 1. Input Schema: What data your system accepts (e.g., user queries, context
-   *    documents)
-   * 2. Output Schema: What data your system produces (e.g., responses, confidence
-   *    scores)
-   * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
-   *    temperature)
-   *
-   * This separation lets you evaluate any system as a black box, focusing on its
-   * interface rather than implementation details.
-   *
-   * @example
-   * ```ts
-   * const system = await client.systems.create('314', {
-   *   configSchema: {
-   *     type: 'object',
-   *     properties: {
-   *       temperature: { type: 'number' },
-   *       maxTokens: { type: 'integer' },
-   *       model: { type: 'string', enum: ['gpt-4', 'gpt-4-turbo'] },
-   *     },
-   *     required: ['model'],
-   *   },
-   *   description: 'Production chatbot powered by GPT-4',
-   *   inputSchema: {
-   *     type: 'object',
-   *     properties: {
-   *       messages: {
-   *         type: 'array',
-   *         items: {
-   *           type: 'object',
-   *           properties: {
-   *             role: { type: 'string', enum: ['system', 'user', 'assistant'] },
-   *             content: { type: 'string' },
-   *           },
-   *           required: ['role', 'content'],
-   *         },
-   *       },
-   *     },
-   *     required: ['messages'],
-   *   },
-   *   name: 'GPT-4 Chatbot',
-   *   outputSchema: {
-   *     type: 'object',
-   *     properties: { response: { type: 'string' } },
-   *     required: ['response'],
-   *   },
-   * });
-   * ```
-   */
-  create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System> {
-    return this._client.post(path`/projects/${projectID}/systems`, { body, ...options });
-  }
-
-  /**
-   * Update an existing system definition. Only the fields provided in the request
-   * body will be updated. If a field is provided, the new content will replace the
-   * existing content. If a field is not provided, the existing content will remain
-   * unchanged.
-   *
-   * When updating schemas:
-   *
-   * - The system will accept your changes regardless of compatibility with existing
-   *   configurations
-   * - Schema updates won't invalidate existing evaluations or configurations
-   * - For significant redesigns, creating a new system definition provides a cleaner
-   *   separation
+   * Update an existing system. Only the fields provided in the request body will be
+   * updated. If a field is provided, the new content will replace the existing
+   * content. If a field is not provided, the existing content will remain unchanged.
    *
    * @example
    * ```ts
    * const system = await client.systems.update(
    *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
    *   {
-   *     description:
-   *       'Updated production chatbot powered by GPT-4 Turbo',
-   *     name: 'GPT-4 Turbo Chatbot',
+   *     productionVersionId:
+   *       '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3',
    *   },
    * );
    * ```
@@ -161,23 +85,29 @@ export class Systems extends APIResource {
   get(systemID: string, options?: RequestOptions): APIPromise<System> {
     return this._client.get(path`/systems/${systemID}`, options);
   }
+
+  /**
+   * Create a new system. If one with the same name in the project exists, it updates
+   * it instead.
+   *
+   * @example
+   * ```ts
+   * const system = await client.systems.upsert('314', {
+   *   config: { temperature: 0.1, maxTokens: 1024 },
+   *   description: 'Production chatbot powered by GPT-4',
+   *   name: 'GPT-4 Chatbot',
+   * });
+   * ```
+   */
+  upsert(projectID: string, body: SystemUpsertParams, options?: RequestOptions): APIPromise<System> {
+    return this._client.post(path`/projects/${projectID}/systems`, { body, ...options });
+  }
 }
 
 export type SystemsPaginatedResponse = PaginatedResponse<System>;
 
 /**
- * A System Under Test (SUT) defines the interface to a component or service you
- * want to evaluate.
- *
- * It specifies three contracts through schemas:
- *
- * - inputSchema: The structure of data the system accepts.
- * - outputSchema: The structure of data the system produces.
- * - configSchema: The parameters that modify system behavior.
- *
- * This abstraction lets you evaluate any system as a black box, focusing on its
- * interface rather than implementation details. It's particularly useful for
- * systems with variable outputs or complex internal state.
+ * A System Under Test (SUT).
  *
  * Systems are templates - to run evaluations, pair them with a SystemVersion that
  * provides specific parameter values.
@@ -188,30 +118,48 @@ export interface System {
    */
   id: string;
 
-  /**
-   * The schema of the system's configuration.
-   */
-  configSchema: { [key: string]: unknown };
-
   /**
    * The description of the system.
    */
   description: string;
 
   /**
-   * The schema of the system's inputs.
+   * The name of the system. Unique within the project.
    */
-  inputSchema: { [key: string]: unknown };
+  name: string;
 
   /**
-   * The name of the system.
+   * The production version of the system.
    */
-  name: string;
+  productionVersion: VersionsAPI.SystemVersion;
 
   /**
-   * The schema of the system's outputs.
+   * The versions of the system.
    */
-  outputSchema: { [key: string]: unknown };
+  versions: Array<System.Version>;
+}
+
+export namespace System {
+  /**
+   * A SystemVersion defines the specific settings for a System Under Test.
+   *
+   * System versions contain parameter values that determine system behavior during
+   * evaluation. They are immutable snapshots - once created, they never change.
+   *
+   * When running evaluations, you reference a specific systemVersionId to establish
+   * which system version to test.
+   */
+  export interface Version {
+    /**
+     * The ID of the system version.
+     */
+    id: string;
+
+    /**
+     * The name of the system version.
+     */
+    name: string;
+  }
 }
 
 export interface SystemDeleteResponse {
@@ -221,38 +169,30 @@ export interface SystemDeleteResponse {
   success: boolean;
 }
 
-export interface SystemCreateParams {
-  /**
-   * The schema of the system's configuration.
-   */
-  configSchema: { [key: string]: unknown };
-
+export interface SystemUpdateParams {
   /**
    * The description of the system.
    */
-  description: string;
-
-  /**
-   * The schema of the system's inputs.
-   */
-  inputSchema: { [key: string]: unknown };
+  description?: string;
 
   /**
-   * The name of the system.
+   * The name of the system. Unique within the project.
    */
-  name: string;
+  name?: string;
 
   /**
-   * The schema of the system's outputs.
+   * The ID of the production version of the system.
    */
-  outputSchema: { [key: string]: unknown };
+  productionVersionId?: string;
 }
 
-export interface SystemUpdateParams {
+export interface SystemListParams extends PaginatedResponseParams {}
+
+export interface SystemUpsertParams {
   /**
-   * The schema of the system's configuration.
+   * The configuration of the system.
    */
-  configSchema?: { [key: string]: unknown };
+  config: Record<string, unknown>;
 
   /**
    * The description of the system.
@@ -260,23 +200,12 @@ export interface SystemUpdateParams {
   description?: string;
 
   /**
-   * The schema of the system's inputs.
-   */
-  inputSchema?: { [key: string]: unknown };
-
-  /**
-   * The name of the system.
+   * The name of the system. Should be unique within the project. Default is "Default
+   * system"
    */
   name?: string;
-
-  /**
-   * The schema of the system's outputs.
-   */
-  outputSchema?: { [key: string]: unknown };
 }
 
-export interface SystemListParams extends PaginatedResponseParams {}
-
 Systems.Versions = Versions;
 
 export declare namespace Systems {
@@ -284,16 +213,14 @@ export declare namespace Systems {
     type System as System,
     type SystemDeleteResponse as SystemDeleteResponse,
     type SystemsPaginatedResponse as SystemsPaginatedResponse,
-    type SystemCreateParams as SystemCreateParams,
     type SystemUpdateParams as SystemUpdateParams,
     type SystemListParams as SystemListParams,
+    type SystemUpsertParams as SystemUpsertParams,
   };
 
   export {
     Versions as Versions,
     type SystemVersion as SystemVersion,
-    type SystemVersionsPaginatedResponse as SystemVersionsPaginatedResponse,
-    type VersionCreateParams as VersionCreateParams,
-    type VersionListParams as VersionListParams,
+    type VersionUpsertParams as VersionUpsertParams,
   };
 }

package/src/resources/systems/versions.ts

@@ -2,92 +2,51 @@
 
 import { APIResource } from '../../core/resource';
 import { APIPromise } from '../../core/api-promise';
-import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../../core/pagination';
 import { RequestOptions } from '../../internal/request-options';
 import { path } from '../../internal/utils/path';
 
 export class Versions extends APIResource {
   /**
-   * Create a new version for a system.
-   *
-   * Each version contains specific parameter values that match the system's
-   * `configSchema` - things like model parameters, thresholds, or processing
-   * options. Once created, versions cannot be modified, ensuring stable reference
-   * points for evaluations.
-   *
-   * When creating a system version:
-   *
-   * - The `config` object is validated against the parent system's `configSchema`.
-   * - System versions with validation errors are still stored, with errors included
-   *   in the response.
-   * - Validation errors indicate fields that don't match the schema but don't
-   *   prevent creation.
-   * - Having validation errors may affect how some evaluation metrics are
-   *   calculated.
+   * Retrieve a specific system version by ID.
    *
    * @example
    * ```ts
-   * const systemVersion = await client.systems.versions.create(
-   *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
-   *   {
-   *     config: {
-   *       temperature: 0.1,
-   *       maxTokens: 1024,
-   *       model: 'gpt-4-turbo',
-   *     },
-   *     name: 'Production (Low Temperature)',
-   *   },
+   * const systemVersion = await client.systems.versions.get(
+   *   '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
    * );
    * ```
    */
-  create(systemID: string, body: VersionCreateParams, options?: RequestOptions): APIPromise<SystemVersion> {
-    return this._client.post(path`/systems/${systemID}/configs`, { body, ...options });
+  get(systemVersionID: string, options?: RequestOptions): APIPromise<SystemVersion> {
+    return this._client.get(path`/systems/versions/${systemVersionID}`, options);
   }
 
   /**
-   * Retrieve a paginated list of system versions for a specific system.
+   * Create a new system version if it does not already exist. Does **not** set the
+   * created version to be the system's production version.
    *
-   * System versions provide concrete parameter values for a System Under Test,
-   * defining exactly how the system should be configured during an evaluation run.
+   * If there is already a system version with the same config, its name will be
+   * updated.
    *
    * @example
    * ```ts
-   * // Automatically fetches more pages as needed.
-   * for await (const systemVersion of client.systems.versions.list(
+   * const systemVersion = await client.systems.versions.upsert(
    *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
-   * )) {
-   *   // ...
-   * }
-   * ```
-   */
-  list(
-    systemID: string,
-    query: VersionListParams | null | undefined = {},
-    options?: RequestOptions,
-  ): PagePromise<SystemVersionsPaginatedResponse, SystemVersion> {
-    return this._client.getAPIList(path`/systems/${systemID}/configs`, PaginatedResponse<SystemVersion>, {
-      query,
-      ...options,
-    });
-  }
-
-  /**
-   * Retrieve a specific system version by ID.
-   *
-   * @example
-   * ```ts
-   * const systemVersion = await client.systems.versions.get(
-   *   '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0',
+   *   {
+   *     config: {
+   *       temperature: 0.5,
+   *       maxTokens: 1024,
+   *       model: 'gemini-2.0-flash',
+   *     },
+   *     name: 'Test model: Gemini',
+   *   },
    * );
    * ```
    */
-  get(systemVersionID: string, options?: RequestOptions): APIPromise<SystemVersion> {
-    return this._client.get(path`/systems/configs/${systemVersionID}`, options);
+  upsert(systemID: string, body: VersionUpsertParams, options?: RequestOptions): APIPromise<SystemVersion> {
+    return this._client.post(path`/systems/${systemID}/versions`, { body, ...options });
   }
 }
 
-export type SystemVersionsPaginatedResponse = PaginatedResponse<SystemVersion>;
-
 /**
  * A SystemVersion defines the specific settings for a System Under Test.
  *
@@ -96,9 +55,6 @@ export type SystemVersionsPaginatedResponse = PaginatedResponse<SystemVersion>;
  *
  * When running evaluations, you reference a specific systemVersionId to establish
  * which system version to test.
- *
- * System versions will be validated against the system's configSchema, with
- * non-conforming values generating warnings.
  */
 export interface SystemVersion {
   /**
@@ -109,7 +65,7 @@ export interface SystemVersion {
   /**
    * The configuration of the system version.
    */
-  config: { [key: string]: unknown };
+  config: Record<string, unknown>;
 
   /**
    * The name of the system version.
@@ -120,47 +76,21 @@ export interface SystemVersion {
    * The ID of the system the system version belongs to.
    */
   systemId: string;
-
-  /**
-   * Validation errors found in the system version. If present, the system version
-   * doesn't fully conform to its system's configSchema.
-   */
-  validationErrors?: Array<SystemVersion.ValidationError>;
-}
-
-export namespace SystemVersion {
-  export interface ValidationError {
-    /**
-     * Human-readable error description.
-     */
-    message: string;
-
-    /**
-     * JSON Pointer to the field with the validation error.
-     */
-    path: string;
-  }
 }
 
-export interface VersionCreateParams {
+export interface VersionUpsertParams {
   /**
    * The configuration of the system version.
    */
-  config: { [key: string]: unknown };
+  config: Record<string, unknown>;
 
   /**
-   * The name of the system version.
+   * The name of the system version. If creating a new system version and the name
+   * isn't provided, it will be autogenerated.
    */
-  name: string;
+  name?: string;
 }
 
-export interface VersionListParams extends PaginatedResponseParams {}
-
 export declare namespace Versions {
-  export {
-    type SystemVersion as SystemVersion,
-    type SystemVersionsPaginatedResponse as SystemVersionsPaginatedResponse,
-    type VersionCreateParams as VersionCreateParams,
-    type VersionListParams as VersionListParams,
-  };
+  export { type SystemVersion as SystemVersion, type VersionUpsertParams as VersionUpsertParams };
 }

package/src/resources/testcases.ts

@@ -136,18 +136,18 @@ export interface Testcase {
    * Derived from data based on the Testset's fieldMapping. Contains all fields
    * marked as expected outputs, including those with validation errors.
    */
-  expected: { [key: string]: unknown };
+  expected: Record<string, unknown>;
 
   /**
    * Derived from data based on the Testset's fieldMapping. Contains all fields
    * marked as inputs, including those with validation errors.
    */
-  inputs: { [key: string]: unknown };
+  inputs: Record<string, unknown>;
 
   /**
    * The JSON data of the Testcase, which is validated against the Testset's schema.
    */
-  jsonData: { [key: string]: unknown };
+  jsonData: Record<string, unknown>;
 
   /**
    * The ID of the Testset this Testcase belongs to.
@@ -198,7 +198,7 @@ export namespace TestcaseCreateParams {
     /**
      * The JSON data of the Testcase, which is validated against the Testset's schema.
      */
-    jsonData: { [key: string]: unknown };
+    jsonData: Record<string, unknown>;
   }
 }
 
@@ -206,7 +206,7 @@ export interface TestcaseUpdateParams {
   /**
    * The JSON data of the Testcase, which is validated against the Testset's schema.
    */
-  jsonData: { [key: string]: unknown };
+  jsonData: Record<string, unknown>;
 }
 
 export interface TestcaseListParams extends PaginatedResponseParams {}

package/src/resources/testsets.ts

@@ -162,7 +162,7 @@ export interface Testset {
   /**
    * The JSON schema for each Testcase in the Testset.
    */
-  jsonSchema: { [key: string]: unknown };
+  jsonSchema: Record<string, unknown>;
 
   /**
    * The name of the Testset.
@@ -215,7 +215,7 @@ export interface TestsetCreateParams {
   /**
    * The JSON schema for each Testcase in the Testset.
    */
-  jsonSchema: { [key: string]: unknown };
+  jsonSchema: Record<string, unknown>;
 
   /**
    * The name of the Testset.
@@ -261,7 +261,7 @@ export interface TestsetUpdateParams {
   /**
    * The JSON schema for each Testcase in the Testset.
    */
-  jsonSchema?: { [key: string]: unknown };
+  jsonSchema?: Record<string, unknown>;
 
   /**
    * The name of the Testset.

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '1.2.0'; // x-release-please-version
+export const VERSION = '2.0.0-alpha.1'; // x-release-please-version

package/version.d.mts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.2.0";
+export declare const VERSION = "2.0.0-alpha.1";
 //# sourceMappingURL=version.d.mts.map

package/version.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"version.d.mts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC"}
+{"version":3,"file":"version.d.mts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,kBAAkB,CAAC"}

package/version.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.2.0";
+export declare const VERSION = "2.0.0-alpha.1";
 //# sourceMappingURL=version.d.ts.map

package/version.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC"}
+{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,kBAAkB,CAAC"}

package/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '1.2.0'; // x-release-please-version
+exports.VERSION = '2.0.0-alpha.1'; // x-release-please-version
 //# sourceMappingURL=version.js.map

package/version.js.map

@@ -1 +1 @@
-{"version":3,"file":"version.js","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":";;;AAAa,QAAA,OAAO,GAAG,OAAO,CAAC,CAAC,2BAA2B"}
+{"version":3,"file":"version.js","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":";;;AAAa,QAAA,OAAO,GAAG,eAAe,CAAC,CAAC,2BAA2B"}

package/version.mjs

@@ -1,2 +1,2 @@
-export const VERSION = '1.2.0'; // x-release-please-version
+export const VERSION = '2.0.0-alpha.1'; // x-release-please-version
 //# sourceMappingURL=version.mjs.map

package/version.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"version.mjs","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,2BAA2B"}
+{"version":3,"file":"version.mjs","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,eAAe,CAAC,CAAC,2BAA2B"}