scorecard-ai 1.0.0-alpha.0 → 1.0.0-alpha.2

package/src/resources/execution-records.ts

@@ -0,0 +1,85 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class ExecutionRecords extends APIResource {
+  /**
+   * Create a new execution record.
+   */
+  create(
+    runID: string,
+    body: ExecutionRecordCreateParams,
+    options?: RequestOptions,
+  ): APIPromise<ExecutionRecord> {
+    return this._client.post(path`/runs/${runID}/executionrecords`, { body, ...options });
+  }
+}
+
+/**
+ * An execution record in the Scorecard system.
+ */
+export interface ExecutionRecord {
+  /**
+   * The ID of the execution record
+   */
+  id: string;
+
+  /**
+   * The actual inputs sent to the system, which should match the system's input
+   * schema
+   */
+  inputs: Record<string, unknown>;
+
+  /**
+   * The expected outputs for the testcase
+   */
+  labels: Record<string, unknown>;
+
+  /**
+   * The actual outputs from the system
+   */
+  outputs: Record<string, unknown>;
+
+  /**
+   * The ID of the run containing this execution record
+   */
+  runId: string;
+
+  /**
+   * The ID of the testcase
+   */
+  testcaseId?: string;
+}
+
+export interface ExecutionRecordCreateParams {
+  /**
+   * The actual inputs sent to the system, which should match the system's input
+   * schema
+   */
+  inputs: Record<string, unknown>;
+
+  /**
+   * The expected outputs for the testcase
+   */
+  labels: Record<string, unknown>;
+
+  /**
+   * The actual outputs from the system
+   */
+  outputs: Record<string, unknown>;
+
+  /**
+   * The ID of the testcase
+   */
+  testcaseId?: string;
+}
+
+export declare namespace ExecutionRecords {
+  export {
+    type ExecutionRecord as ExecutionRecord,
+    type ExecutionRecordCreateParams as ExecutionRecordCreateParams,
+  };
+}

package/src/resources/index.ts

@@ -1,12 +1,35 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 export * from './shared';
+export {
+  ExecutionRecords,
+  type ExecutionRecord,
+  type ExecutionRecordCreateParams,
+} from './execution-records';
 export {
   Projects,
   type ProjectListResponse,
   type ProjectListParams,
   type ProjectListResponsesPaginatedResponse,
 } from './projects';
+export { Runs, type Run, type RunUpdateResponse, type RunCreateParams, type RunUpdateParams } from './runs';
+export {
+  SystemConfigs,
+  type SystemConfig,
+  type SystemConfigCreateParams,
+  type SystemConfigListParams,
+  type SystemConfigGetParams,
+  type SystemConfigsPaginatedResponse,
+} from './system-configs';
+export {
+  Systems,
+  type System,
+  type SystemDeleteResponse,
+  type SystemCreateParams,
+  type SystemUpdateParams,
+  type SystemListParams,
+  type SystemsPaginatedResponse,
+} from './systems';
 export {
   Testcases,
   type Testcase,

package/src/resources/runs.ts

@@ -0,0 +1,118 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class Runs extends APIResource {
+  /**
+   * Create a new run.
+   */
+  create(projectID: string, body: RunCreateParams, options?: RequestOptions): APIPromise<Run> {
+    return this._client.post(path`/projects/${projectID}/runs`, { body, ...options });
+  }
+
+  /**
+   * Update the status of a run.
+   */
+  update(runID: string, body: RunUpdateParams, options?: RequestOptions): APIPromise<RunUpdateResponse> {
+    return this._client.patch(path`/runs/${runID}`, { body, ...options });
+  }
+}
+
+/**
+ * A Run in the Scorecard system.
+ */
+export interface Run {
+  /**
+   * The ID of the Run
+   */
+  id: string;
+
+  /**
+   * The IDs of the metrics this Run is using
+   */
+  metricIds: Array<string>;
+
+  /**
+   * The status of the Run
+   */
+  status:
+    | 'pending'
+    | 'awaiting_execution'
+    | 'running_execution'
+    | 'awaiting_scoring'
+    | 'running_scoring'
+    | 'awaiting_human_scoring'
+    | 'completed';
+
+  /**
+   * The ID of the Testset this Run is testing
+   */
+  testsetId: string;
+
+  /**
+   * The ID of the system configuration this Run is using
+   */
+  systemConfigId?: string;
+}
+
+export interface RunUpdateResponse {
+  /**
+   * The ID of the Run
+   */
+  id: string;
+
+  /**
+   * The status of the Run
+   */
+  status:
+    | 'pending'
+    | 'awaiting_execution'
+    | 'running_execution'
+    | 'awaiting_scoring'
+    | 'running_scoring'
+    | 'awaiting_human_scoring'
+    | 'completed';
+}
+
+export interface RunCreateParams {
+  /**
+   * The IDs of the metrics this Run is using
+   */
+  metricIds: Array<string>;
+
+  /**
+   * The ID of the Testset this Run is testing
+   */
+  testsetId: string;
+
+  /**
+   * The ID of the system configuration this Run is using
+   */
+  systemConfigId?: string;
+}
+
+export interface RunUpdateParams {
+  /**
+   * The status of the Run
+   */
+  status:
+    | 'pending'
+    | 'awaiting_execution'
+    | 'running_execution'
+    | 'awaiting_scoring'
+    | 'running_scoring'
+    | 'awaiting_human_scoring'
+    | 'completed';
+}
+
+export declare namespace Runs {
+  export {
+    type Run as Run,
+    type RunUpdateResponse as RunUpdateResponse,
+    type RunCreateParams as RunCreateParams,
+    type RunUpdateParams as RunUpdateParams,
+  };
+}

package/src/resources/system-configs.ts

@@ -0,0 +1,170 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class SystemConfigs extends APIResource {
+  /**
+   * Create a new configuration for a system.
+   *
+   * Each configuration contains specific parameter values that match the system's
+   * configSchema - things like model parameters, thresholds, or processing options.
+   * Once created, configurations cannot be modified, ensuring stable reference
+   * points for evaluations.
+   *
+   * When creating a configuration:
+   *
+   * - The 'config' object is validated against the parent system's configSchema
+   * - Configurations with validation errors are still stored, with errors included
+   *   in the response
+   * - Validation errors indicate fields that don't match the schema but don't
+   *   prevent creation
+   * - Having validation errors may affect how some evaluation metrics are calculated
+   */
+  create(
+    systemID: string,
+    body: SystemConfigCreateParams,
+    options?: RequestOptions,
+  ): APIPromise<SystemConfig> {
+    return this._client.post(path`/systems/${systemID}/configs`, { body, ...options });
+  }
+
+  /**
+   * Retrieve a paginated list of configurations for a specific system.
+   *
+   * System configurations provide concrete parameter values for a System Under Test,
+   * defining exactly how the system should be configured during an evaluation run.
+   */
+  list(
+    systemID: string,
+    query: SystemConfigListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<SystemConfigsPaginatedResponse, SystemConfig> {
+    return this._client.getAPIList(path`/systems/${systemID}/configs`, PaginatedResponse<SystemConfig>, {
+      query,
+      ...options,
+    });
+  }
+
+  /**
+   * Retrieve a specific system configuration by ID.
+   */
+  get(
+    systemConfigID: string,
+    params: SystemConfigGetParams,
+    options?: RequestOptions,
+  ): APIPromise<SystemConfig> {
+    const { systemId } = params;
+    return this._client.get(path`/systems/${systemId}/configs/${systemConfigID}`, options);
+  }
+}
+
+export type SystemConfigsPaginatedResponse = PaginatedResponse<SystemConfig>;
+
+/**
+ * A SystemConfig defines the specific settings for a System Under Test.
+ *
+ * Configurations contain parameter values that determine system behavior during
+ * evaluation. They are immutable snapshots - once created, they never change.
+ *
+ * When running evaluations, you reference a specific configId to establish which
+ * configuration to test.
+ *
+ * Configurations will be validated against the system's configSchema, with
+ * non-conforming values generating warnings.
+ */
+export interface SystemConfig {
+  /**
+   * The ID of the system configuration
+   */
+  id: string;
+
+  /**
+   * The configuration of the system
+   */
+  config: Record<string, unknown>;
+
+  /**
+   * The name of the system configuration
+   */
+  name: string;
+
+  /**
+   * The ID of the system the configuration belongs to
+   */
+  systemId: string;
+
+  /**
+   * Validation errors found in the configuration. If present, the configuration
+   * doesn't fully conform to its system's configSchema.
+   */
+  validationErrors?: Array<SystemConfig.ValidationError>;
+}
+
+export namespace SystemConfig {
+  export interface ValidationError {
+    /**
+     * Human-readable error description
+     */
+    message: string;
+
+    /**
+     * JSON Pointer to the field with the validation error
+     */
+    path: string;
+  }
+}
+
+export interface SystemConfigCreateParams {
+  /**
+   * The configuration of the system
+   */
+  config: Record<string, unknown>;
+
+  /**
+   * The name of the system configuration
+   */
+  name: string;
+
+  /**
+   * Validation errors found in the configuration. If present, the configuration
+   * doesn't fully conform to its system's configSchema.
+   */
+  validationErrors?: Array<SystemConfigCreateParams.ValidationError>;
+}
+
+export namespace SystemConfigCreateParams {
+  export interface ValidationError {
+    /**
+     * Human-readable error description
+     */
+    message: string;
+
+    /**
+     * JSON Pointer to the field with the validation error
+     */
+    path: string;
+  }
+}
+
+export interface SystemConfigListParams extends PaginatedResponseParams {}
+
+export interface SystemConfigGetParams {
+  /**
+   * The ID of the system the configuration belongs to
+   */
+  systemId: string;
+}
+
+export declare namespace SystemConfigs {
+  export {
+    type SystemConfig as SystemConfig,
+    type SystemConfigsPaginatedResponse as SystemConfigsPaginatedResponse,
+    type SystemConfigCreateParams as SystemConfigCreateParams,
+    type SystemConfigListParams as SystemConfigListParams,
+    type SystemConfigGetParams as SystemConfigGetParams,
+  };
+}

package/src/resources/systems.ts

@@ -0,0 +1,206 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class Systems extends APIResource {
+  /**
+   * Create a new system definition that specifies the interface contracts for a
+   * component you want to evaluate.
+   *
+   * A system acts as a template that defines three key contracts through JSON
+   * Schemas:
+   *
+   * 1. Input Schema: What data your system accepts (e.g., user queries, context
+   *    documents)
+   * 2. Output Schema: What data your system produces (e.g., responses, confidence
+   *    scores)
+   * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+   *    temperature)
+   *
+   * This separation lets you evaluate any system as a black box, focusing on its
+   * interface rather than implementation details.
+   */
+  create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System> {
+    return this._client.post(path`/projects/${projectID}/systems`, { body, ...options });
+  }
+
+  /**
+   * Update an existing system definition. Only the fields provided in the request
+   * body will be updated. If a field is provided, the new content will replace the
+   * existing content. If a field is not provided, the existing content will remain
+   * unchanged.
+   *
+   * When updating schemas:
+   *
+   * - The system will accept your changes regardless of compatibility with existing
+   *   configurations
+   * - Schema updates won't invalidate existing evaluations or configurations
+   * - For significant redesigns, creating a new system definition provides a cleaner
+   *   separation
+   */
+  update(
+    systemID: string,
+    body: SystemUpdateParams | null | undefined = {},
+    options?: RequestOptions,
+  ): APIPromise<System> {
+    return this._client.patch(path`/systems/${systemID}`, { body, ...options });
+  }
+
+  /**
+   * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+   */
+  list(
+    projectID: string,
+    query: SystemListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<SystemsPaginatedResponse, System> {
+    return this._client.getAPIList(path`/projects/${projectID}/systems`, PaginatedResponse<System>, {
+      query,
+      ...options,
+    });
+  }
+
+  /**
+   * Delete a system definition by ID. This will not delete associated system
+   * configurations.
+   */
+  delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse> {
+    return this._client.delete(path`/systems/${systemID}`, options);
+  }
+
+  /**
+   * Retrieve a specific system by ID.
+   */
+  get(systemID: string, options?: RequestOptions): APIPromise<System> {
+    return this._client.get(path`/systems/${systemID}`, options);
+  }
+}
+
+export type SystemsPaginatedResponse = PaginatedResponse<System>;
+
+/**
+ * A System Under Test (SUT) defines the interface to a component or service you
+ * want to evaluate.
+ *
+ * It specifies three contracts through schemas:
+ *
+ * - inputSchema: The structure of data the system accepts
+ * - outputSchema: The structure of data the system produces
+ * - configSchema: The parameters that modify system behavior
+ *
+ * This abstraction lets you evaluate any system as a black box, focusing on its
+ * interface rather than implementation details. It's particularly useful for
+ * systems with variable outputs or complex internal state.
+ *
+ * Systems are templates - to run evaluations, pair them with a SystemConfig that
+ * provides specific parameter values.
+ */
+export interface System {
+  /**
+   * The ID of the system
+   */
+  id: string;
+
+  /**
+   * The schema of the system's configuration
+   */
+  configSchema: Record<string, unknown>;
+
+  /**
+   * The description of the system
+   */
+  description: string;
+
+  /**
+   * The schema of the system's inputs
+   */
+  inputSchema: Record<string, unknown>;
+
+  /**
+   * The name of the system
+   */
+  name: string;
+
+  /**
+   * The schema of the system's outputs
+   */
+  outputSchema: Record<string, unknown>;
+}
+
+export interface SystemDeleteResponse {
+  /**
+   * Whether the deletion was successful
+   */
+  success: boolean;
+}
+
+export interface SystemCreateParams {
+  /**
+   * The schema of the system's configuration
+   */
+  configSchema: Record<string, unknown>;
+
+  /**
+   * The description of the system
+   */
+  description: string;
+
+  /**
+   * The schema of the system's inputs
+   */
+  inputSchema: Record<string, unknown>;
+
+  /**
+   * The name of the system
+   */
+  name: string;
+
+  /**
+   * The schema of the system's outputs
+   */
+  outputSchema: Record<string, unknown>;
+}
+
+export interface SystemUpdateParams {
+  /**
+   * The schema of the system's configuration
+   */
+  configSchema?: Record<string, unknown>;
+
+  /**
+   * The description of the system
+   */
+  description?: string;
+
+  /**
+   * The schema of the system's inputs
+   */
+  inputSchema?: Record<string, unknown>;
+
+  /**
+   * The name of the system
+   */
+  name?: string;
+
+  /**
+   * The schema of the system's outputs
+   */
+  outputSchema?: Record<string, unknown>;
+}
+
+export interface SystemListParams extends PaginatedResponseParams {}
+
+export declare namespace Systems {
+  export {
+    type System as System,
+    type SystemDeleteResponse as SystemDeleteResponse,
+    type SystemsPaginatedResponse as SystemsPaginatedResponse,
+    type SystemCreateParams as SystemCreateParams,
+    type SystemUpdateParams as SystemUpdateParams,
+    type SystemListParams as SystemListParams,
+  };
+}

package/src/resources/testcases.ts

@@ -40,14 +40,10 @@ export class Testcases extends APIResource {
   }
 
   /**
-   * Delete multiple testcases from the specified testset.
+   * Delete multiple testcases by their IDs.
    */
-  delete(
-    testsetID: string,
-    body: TestcaseDeleteParams,
-    options?: RequestOptions,
-  ): APIPromise<TestcaseDeleteResponse> {
-    return this._client.delete(path`/testsets/${testsetID}/testcases`, { body, ...options });
+  delete(body: TestcaseDeleteParams, options?: RequestOptions): APIPromise<TestcaseDeleteResponse> {
+    return this._client.post('/testcases/bulk-delete', { body, ...options });
   }
 
   /**
@@ -74,17 +70,17 @@ export interface Testcase {
    */
   id: string;
 
-  /**
-   * The JSON data of the testcase, which is validated against the testset's schema.
-   */
-  data: Record<string, unknown>;
-
   /**
    * Derived from data based on the testset's fieldMapping. Contains all fields
    * marked as inputs, including those with validation errors.
    */
   inputs: Record<string, unknown>;
 
+  /**
+   * The JSON data of the testcase, which is validated against the testset's schema.
+   */
+  jsonData: Record<string, unknown>;
+
   /**
    * Derived from data based on the testset's fieldMapping. Contains all fields
    * marked as labels, including those with validation errors.
@@ -123,28 +119,9 @@ export interface TestcaseCreateResponse {
 
 export interface TestcaseDeleteResponse {
   /**
-   * Number of testcases successfully deleted
-   */
-  deletedCount: number;
-
-  /**
-   * List of errors encountered during deletion, if any
+   * Whether the deletion was successful
    */
-  errors: Array<TestcaseDeleteResponse.Error>;
-}
-
-export namespace TestcaseDeleteResponse {
-  export interface Error {
-    /**
-     * ID of the testcase that failed to be deleted
-     */
-    id: string;
-
-    /**
-     * Error message explaining why the deletion failed
-     */
-    message: string;
-  }
+  success: boolean;
 }
 
 export interface TestcaseCreateParams {
@@ -167,7 +144,7 @@ export namespace TestcaseCreateParams {
     /**
      * The JSON data of the testcase, which is validated against the testset's schema.
      */
-    data: Record<string, unknown>;
+    jsonData: Record<string, unknown>;
   }
 }
 
@@ -175,14 +152,14 @@ export interface TestcaseUpdateParams {
   /**
    * The JSON data of the testcase, which is validated against the testset's schema.
    */
-  data: Record<string, unknown>;
+  jsonData: Record<string, unknown>;
 }
 
 export interface TestcaseListParams extends PaginatedResponseParams {}
 
 export interface TestcaseDeleteParams {
   /**
-   * IDs of testcases to delete (max 100)
+   * IDs of testcases to delete
    */
   ids: Array<string>;
 }