scorecard-ai 1.0.0-alpha.0 → 1.0.0-alpha.1

package/src/resources/system-configs.ts

@@ -0,0 +1,170 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class SystemConfigs extends APIResource {
+  /**
+   * Create a new configuration for a system.
+   *
+   * Each configuration contains specific parameter values that match the system's
+   * configSchema - things like model parameters, thresholds, or processing options.
+   * Once created, configurations cannot be modified, ensuring stable reference
+   * points for evaluations.
+   *
+   * When creating a configuration:
+   *
+   * - The 'config' object is validated against the parent system's configSchema
+   * - Configurations with validation errors are still stored, with errors included
+   *   in the response
+   * - Validation errors indicate fields that don't match the schema but don't
+   *   prevent creation
+   * - Having validation errors may affect how some evaluation metrics are calculated
+   */
+  create(
+    systemID: string,
+    body: SystemConfigCreateParams,
+    options?: RequestOptions,
+  ): APIPromise<SystemConfig> {
+    return this._client.post(path`/systems/${systemID}/configs`, { body, ...options });
+  }
+
+  /**
+   * Retrieve a paginated list of configurations for a specific system.
+   *
+   * System configurations provide concrete parameter values for a System Under Test,
+   * defining exactly how the system should be configured during an evaluation run.
+   */
+  list(
+    systemID: string,
+    query: SystemConfigListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<SystemConfigsPaginatedResponse, SystemConfig> {
+    return this._client.getAPIList(path`/systems/${systemID}/configs`, PaginatedResponse<SystemConfig>, {
+      query,
+      ...options,
+    });
+  }
+
+  /**
+   * Retrieve a specific system configuration by ID.
+   */
+  get(
+    systemConfigID: string,
+    params: SystemConfigGetParams,
+    options?: RequestOptions,
+  ): APIPromise<SystemConfig> {
+    const { systemId } = params;
+    return this._client.get(path`/systems/${systemId}/configs/${systemConfigID}`, options);
+  }
+}
+
+export type SystemConfigsPaginatedResponse = PaginatedResponse<SystemConfig>;
+
+/**
+ * A SystemConfig defines the specific settings for a System Under Test.
+ *
+ * Configurations contain parameter values that determine system behavior during
+ * evaluation. They are immutable snapshots - once created, they never change.
+ *
+ * When running evaluations, you reference a specific configId to establish which
+ * configuration to test.
+ *
+ * Configurations will be validated against the system's configSchema, with
+ * non-conforming values generating warnings.
+ */
+export interface SystemConfig {
+  /**
+   * The ID of the system configuration
+   */
+  id: string;
+
+  /**
+   * The configuration of the system
+   */
+  config: Record<string, unknown>;
+
+  /**
+   * The name of the system configuration
+   */
+  name: string;
+
+  /**
+   * The ID of the system the configuration belongs to
+   */
+  systemId: string;
+
+  /**
+   * Validation errors found in the configuration. If present, the configuration
+   * doesn't fully conform to its system's configSchema.
+   */
+  validationErrors?: Array<SystemConfig.ValidationError>;
+}
+
+export namespace SystemConfig {
+  export interface ValidationError {
+    /**
+     * Human-readable error description
+     */
+    message: string;
+
+    /**
+     * JSON Pointer to the field with the validation error
+     */
+    path: string;
+  }
+}
+
+export interface SystemConfigCreateParams {
+  /**
+   * The configuration of the system
+   */
+  config: Record<string, unknown>;
+
+  /**
+   * The name of the system configuration
+   */
+  name: string;
+
+  /**
+   * Validation errors found in the configuration. If present, the configuration
+   * doesn't fully conform to its system's configSchema.
+   */
+  validationErrors?: Array<SystemConfigCreateParams.ValidationError>;
+}
+
+export namespace SystemConfigCreateParams {
+  export interface ValidationError {
+    /**
+     * Human-readable error description
+     */
+    message: string;
+
+    /**
+     * JSON Pointer to the field with the validation error
+     */
+    path: string;
+  }
+}
+
+export interface SystemConfigListParams extends PaginatedResponseParams {}
+
+export interface SystemConfigGetParams {
+  /**
+   * The ID of the system the configuration belongs to
+   */
+  systemId: string;
+}
+
+export declare namespace SystemConfigs {
+  export {
+    type SystemConfig as SystemConfig,
+    type SystemConfigsPaginatedResponse as SystemConfigsPaginatedResponse,
+    type SystemConfigCreateParams as SystemConfigCreateParams,
+    type SystemConfigListParams as SystemConfigListParams,
+    type SystemConfigGetParams as SystemConfigGetParams,
+  };
+}

package/src/resources/systems.ts

@@ -0,0 +1,206 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class Systems extends APIResource {
+  /**
+   * Create a new system definition that specifies the interface contracts for a
+   * component you want to evaluate.
+   *
+   * A system acts as a template that defines three key contracts through JSON
+   * Schemas:
+   *
+   * 1. Input Schema: What data your system accepts (e.g., user queries, context
+   *    documents)
+   * 2. Output Schema: What data your system produces (e.g., responses, confidence
+   *    scores)
+   * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+   *    temperature)
+   *
+   * This separation lets you evaluate any system as a black box, focusing on its
+   * interface rather than implementation details.
+   */
+  create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System> {
+    return this._client.post(path`/projects/${projectID}/systems`, { body, ...options });
+  }
+
+  /**
+   * Update an existing system definition. Only the fields provided in the request
+   * body will be updated. If a field is provided, the new content will replace the
+   * existing content. If a field is not provided, the existing content will remain
+   * unchanged.
+   *
+   * When updating schemas:
+   *
+   * - The system will accept your changes regardless of compatibility with existing
+   *   configurations
+   * - Schema updates won't invalidate existing evaluations or configurations
+   * - For significant redesigns, creating a new system definition provides a cleaner
+   *   separation
+   */
+  update(
+    systemID: string,
+    body: SystemUpdateParams | null | undefined = {},
+    options?: RequestOptions,
+  ): APIPromise<System> {
+    return this._client.patch(path`/systems/${systemID}`, { body, ...options });
+  }
+
+  /**
+   * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+   */
+  list(
+    projectID: string,
+    query: SystemListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<SystemsPaginatedResponse, System> {
+    return this._client.getAPIList(path`/projects/${projectID}/systems`, PaginatedResponse<System>, {
+      query,
+      ...options,
+    });
+  }
+
+  /**
+   * Delete a system definition by ID. This will not delete associated system
+   * configurations.
+   */
+  delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse> {
+    return this._client.delete(path`/systems/${systemID}`, options);
+  }
+
+  /**
+   * Retrieve a specific system by ID.
+   */
+  get(systemID: string, options?: RequestOptions): APIPromise<System> {
+    return this._client.get(path`/systems/${systemID}`, options);
+  }
+}
+
+export type SystemsPaginatedResponse = PaginatedResponse<System>;
+
+/**
+ * A System Under Test (SUT) defines the interface to a component or service you
+ * want to evaluate.
+ *
+ * It specifies three contracts through schemas:
+ *
+ * - inputSchema: The structure of data the system accepts
+ * - outputSchema: The structure of data the system produces
+ * - configSchema: The parameters that modify system behavior
+ *
+ * This abstraction lets you evaluate any system as a black box, focusing on its
+ * interface rather than implementation details. It's particularly useful for
+ * systems with variable outputs or complex internal state.
+ *
+ * Systems are templates - to run evaluations, pair them with a SystemConfig that
+ * provides specific parameter values.
+ */
+export interface System {
+  /**
+   * The ID of the system
+   */
+  id: string;
+
+  /**
+   * The schema of the system's configuration
+   */
+  configSchema: Record<string, unknown>;
+
+  /**
+   * The description of the system
+   */
+  description: string;
+
+  /**
+   * The schema of the system's inputs
+   */
+  inputSchema: Record<string, unknown>;
+
+  /**
+   * The name of the system
+   */
+  name: string;
+
+  /**
+   * The schema of the system's outputs
+   */
+  outputSchema: Record<string, unknown>;
+}
+
+export interface SystemDeleteResponse {
+  /**
+   * Whether the deletion was successful
+   */
+  success: boolean;
+}
+
+export interface SystemCreateParams {
+  /**
+   * The schema of the system's configuration
+   */
+  configSchema: Record<string, unknown>;
+
+  /**
+   * The description of the system
+   */
+  description: string;
+
+  /**
+   * The schema of the system's inputs
+   */
+  inputSchema: Record<string, unknown>;
+
+  /**
+   * The name of the system
+   */
+  name: string;
+
+  /**
+   * The schema of the system's outputs
+   */
+  outputSchema: Record<string, unknown>;
+}
+
+export interface SystemUpdateParams {
+  /**
+   * The schema of the system's configuration
+   */
+  configSchema?: Record<string, unknown>;
+
+  /**
+   * The description of the system
+   */
+  description?: string;
+
+  /**
+   * The schema of the system's inputs
+   */
+  inputSchema?: Record<string, unknown>;
+
+  /**
+   * The name of the system
+   */
+  name?: string;
+
+  /**
+   * The schema of the system's outputs
+   */
+  outputSchema?: Record<string, unknown>;
+}
+
+export interface SystemListParams extends PaginatedResponseParams {}
+
+export declare namespace Systems {
+  export {
+    type System as System,
+    type SystemDeleteResponse as SystemDeleteResponse,
+    type SystemsPaginatedResponse as SystemsPaginatedResponse,
+    type SystemCreateParams as SystemCreateParams,
+    type SystemUpdateParams as SystemUpdateParams,
+    type SystemListParams as SystemListParams,
+  };
+}

package/src/resources/testcases.ts

@@ -40,14 +40,10 @@ export class Testcases extends APIResource {
   }
 
   /**
-   * Delete multiple testcases from the specified testset.
+   * Delete multiple testcases by their IDs.
    */
-  delete(
-    testsetID: string,
-    body: TestcaseDeleteParams,
-    options?: RequestOptions,
-  ): APIPromise<TestcaseDeleteResponse> {
-    return this._client.delete(path`/testsets/${testsetID}/testcases`, { body, ...options });
+  delete(body: TestcaseDeleteParams, options?: RequestOptions): APIPromise<TestcaseDeleteResponse> {
+    return this._client.post('/testcases/bulk-delete', { body, ...options });
   }
 
   /**
@@ -74,17 +70,17 @@ export interface Testcase {
    */
   id: string;
 
-  /**
-   * The JSON data of the testcase, which is validated against the testset's schema.
-   */
-  data: Record<string, unknown>;
-
   /**
    * Derived from data based on the testset's fieldMapping. Contains all fields
    * marked as inputs, including those with validation errors.
    */
   inputs: Record<string, unknown>;
 
+  /**
+   * The JSON data of the testcase, which is validated against the testset's schema.
+   */
+  jsonData: Record<string, unknown>;
+
   /**
    * Derived from data based on the testset's fieldMapping. Contains all fields
    * marked as labels, including those with validation errors.
@@ -123,28 +119,9 @@ export interface TestcaseCreateResponse {
 
 export interface TestcaseDeleteResponse {
   /**
-   * Number of testcases successfully deleted
-   */
-  deletedCount: number;
-
-  /**
-   * List of errors encountered during deletion, if any
+   * Whether the deletion was successful
    */
-  errors: Array<TestcaseDeleteResponse.Error>;
-}
-
-export namespace TestcaseDeleteResponse {
-  export interface Error {
-    /**
-     * ID of the testcase that failed to be deleted
-     */
-    id: string;
-
-    /**
-     * Error message explaining why the deletion failed
-     */
-    message: string;
-  }
+  success: boolean;
 }
 
 export interface TestcaseCreateParams {
@@ -167,7 +144,7 @@ export namespace TestcaseCreateParams {
     /**
      * The JSON data of the testcase, which is validated against the testset's schema.
      */
-    data: Record<string, unknown>;
+    jsonData: Record<string, unknown>;
   }
 }
 
@@ -175,14 +152,14 @@ export interface TestcaseUpdateParams {
   /**
    * The JSON data of the testcase, which is validated against the testset's schema.
    */
-  data: Record<string, unknown>;
+  jsonData: Record<string, unknown>;
 }
 
 export interface TestcaseListParams extends PaginatedResponseParams {}
 
 export interface TestcaseDeleteParams {
   /**
-   * IDs of testcases to delete (max 100)
+   * IDs of testcases to delete
    */
   ids: Array<string>;
 }

package/src/resources/testsets.ts

@@ -110,12 +110,15 @@ export interface Testset {
    */
   fieldMapping: Testset.FieldMapping;
 
+  /**
+   * The JSON schema for each testcase in the testset
+   */
+  jsonSchema: Record<string, unknown>;
+
   /**
    * The name of the testset
    */
   name: string;
-
-  schema: Record<string, unknown>;
 }
 
 export namespace Testset {
@@ -161,14 +164,14 @@ export interface TestsetCreateParams {
   fieldMapping: TestsetCreateParams.FieldMapping;
 
   /**
-   * The name of the testset
+   * The JSON schema for each testcase in the testset
    */
-  name: string;
+  jsonSchema: Record<string, unknown>;
 
   /**
-   * The JSON schema for each testcase in the testset
+   * The name of the testset
    */
-  schema: Record<string, unknown>;
+  name: string;
 }
 
 export namespace TestsetCreateParams {
@@ -207,14 +210,14 @@ export interface TestsetUpdateParams {
   fieldMapping?: TestsetUpdateParams.FieldMapping;
 
   /**
-   * The name of the testset
+   * The JSON schema for each testcase in the testset
    */
-  name?: string;
+  jsonSchema?: Record<string, unknown>;
 
   /**
-   * The JSON schema for each testcase in the testset
+   * The name of the testset
    */
-  schema?: Record<string, unknown>;
+  name?: string;
 }
 
 export namespace TestsetUpdateParams {

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '1.0.0-alpha.0'; // x-release-please-version
+export const VERSION = '1.0.0-alpha.1'; // x-release-please-version

package/version.d.mts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.0.0-alpha.0";
+export declare const VERSION = "1.0.0-alpha.1";
 //# sourceMappingURL=version.d.mts.map

package/version.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "1.0.0-alpha.0";
+export declare const VERSION = "1.0.0-alpha.1";
 //# sourceMappingURL=version.d.ts.map

package/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '1.0.0-alpha.0'; // x-release-please-version
+exports.VERSION = '1.0.0-alpha.1'; // x-release-please-version
 //# sourceMappingURL=version.js.map

package/version.mjs

@@ -1,2 +1,2 @@
-export const VERSION = '1.0.0-alpha.0'; // x-release-please-version
+export const VERSION = '1.0.0-alpha.1'; // x-release-please-version
 //# sourceMappingURL=version.mjs.map