scorecard-ai 0.6.1 → 1.0.0-alpha.1

package/src/resources/systems.ts

@@ -0,0 +1,206 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class Systems extends APIResource {
+  /**
+   * Create a new system definition that specifies the interface contracts for a
+   * component you want to evaluate.
+   *
+   * A system acts as a template that defines three key contracts through JSON
+   * Schemas:
+   *
+   * 1. Input Schema: What data your system accepts (e.g., user queries, context
+   *    documents)
+   * 2. Output Schema: What data your system produces (e.g., responses, confidence
+   *    scores)
+   * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
+   *    temperature)
+   *
+   * This separation lets you evaluate any system as a black box, focusing on its
+   * interface rather than implementation details.
+   */
+  create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System> {
+    return this._client.post(path`/projects/${projectID}/systems`, { body, ...options });
+  }
+
+  /**
+   * Update an existing system definition. Only the fields provided in the request
+   * body will be updated. If a field is provided, the new content will replace the
+   * existing content. If a field is not provided, the existing content will remain
+   * unchanged.
+   *
+   * When updating schemas:
+   *
+   * - The system will accept your changes regardless of compatibility with existing
+   *   configurations
+   * - Schema updates won't invalidate existing evaluations or configurations
+   * - For significant redesigns, creating a new system definition provides a cleaner
+   *   separation
+   */
+  update(
+    systemID: string,
+    body: SystemUpdateParams | null | undefined = {},
+    options?: RequestOptions,
+  ): APIPromise<System> {
+    return this._client.patch(path`/systems/${systemID}`, { body, ...options });
+  }
+
+  /**
+   * Retrieve a paginated list of all systems. Systems are ordered by creation date.
+   */
+  list(
+    projectID: string,
+    query: SystemListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<SystemsPaginatedResponse, System> {
+    return this._client.getAPIList(path`/projects/${projectID}/systems`, PaginatedResponse<System>, {
+      query,
+      ...options,
+    });
+  }
+
+  /**
+   * Delete a system definition by ID. This will not delete associated system
+   * configurations.
+   */
+  delete(systemID: string, options?: RequestOptions): APIPromise<SystemDeleteResponse> {
+    return this._client.delete(path`/systems/${systemID}`, options);
+  }
+
+  /**
+   * Retrieve a specific system by ID.
+   */
+  get(systemID: string, options?: RequestOptions): APIPromise<System> {
+    return this._client.get(path`/systems/${systemID}`, options);
+  }
+}
+
+export type SystemsPaginatedResponse = PaginatedResponse<System>;
+
+/**
+ * A System Under Test (SUT) defines the interface to a component or service you
+ * want to evaluate.
+ *
+ * It specifies three contracts through schemas:
+ *
+ * - inputSchema: The structure of data the system accepts
+ * - outputSchema: The structure of data the system produces
+ * - configSchema: The parameters that modify system behavior
+ *
+ * This abstraction lets you evaluate any system as a black box, focusing on its
+ * interface rather than implementation details. It's particularly useful for
+ * systems with variable outputs or complex internal state.
+ *
+ * Systems are templates - to run evaluations, pair them with a SystemConfig that
+ * provides specific parameter values.
+ */
+export interface System {
+  /**
+   * The ID of the system
+   */
+  id: string;
+
+  /**
+   * The schema of the system's configuration
+   */
+  configSchema: Record<string, unknown>;
+
+  /**
+   * The description of the system
+   */
+  description: string;
+
+  /**
+   * The schema of the system's inputs
+   */
+  inputSchema: Record<string, unknown>;
+
+  /**
+   * The name of the system
+   */
+  name: string;
+
+  /**
+   * The schema of the system's outputs
+   */
+  outputSchema: Record<string, unknown>;
+}
+
+export interface SystemDeleteResponse {
+  /**
+   * Whether the deletion was successful
+   */
+  success: boolean;
+}
+
+export interface SystemCreateParams {
+  /**
+   * The schema of the system's configuration
+   */
+  configSchema: Record<string, unknown>;
+
+  /**
+   * The description of the system
+   */
+  description: string;
+
+  /**
+   * The schema of the system's inputs
+   */
+  inputSchema: Record<string, unknown>;
+
+  /**
+   * The name of the system
+   */
+  name: string;
+
+  /**
+   * The schema of the system's outputs
+   */
+  outputSchema: Record<string, unknown>;
+}
+
+export interface SystemUpdateParams {
+  /**
+   * The schema of the system's configuration
+   */
+  configSchema?: Record<string, unknown>;
+
+  /**
+   * The description of the system
+   */
+  description?: string;
+
+  /**
+   * The schema of the system's inputs
+   */
+  inputSchema?: Record<string, unknown>;
+
+  /**
+   * The name of the system
+   */
+  name?: string;
+
+  /**
+   * The schema of the system's outputs
+   */
+  outputSchema?: Record<string, unknown>;
+}
+
+export interface SystemListParams extends PaginatedResponseParams {}
+
+export declare namespace Systems {
+  export {
+    type System as System,
+    type SystemDeleteResponse as SystemDeleteResponse,
+    type SystemsPaginatedResponse as SystemsPaginatedResponse,
+    type SystemCreateParams as SystemCreateParams,
+    type SystemUpdateParams as SystemUpdateParams,
+    type SystemListParams as SystemListParams,
+  };
+}

package/src/resources/testcases.ts

@@ -0,0 +1,178 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class Testcases extends APIResource {
+  /**
+   * Create multiple testcases in the specified testset.
+   */
+  create(
+    testsetID: string,
+    body: TestcaseCreateParams,
+    options?: RequestOptions,
+  ): APIPromise<TestcaseCreateResponse> {
+    return this._client.post(path`/testsets/${testsetID}/testcases`, { body, ...options });
+  }
+
+  /**
+   * Replace the data of an existing testcase while keeping its ID.
+   */
+  update(testcaseID: string, body: TestcaseUpdateParams, options?: RequestOptions): APIPromise<Testcase> {
+    return this._client.put(path`/testcases/${testcaseID}`, { body, ...options });
+  }
+
+  /**
+   * Retrieve a paginated list of testcases belonging to a testset.
+   */
+  list(
+    testsetID: string,
+    query: TestcaseListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<TestcasesPaginatedResponse, Testcase> {
+    return this._client.getAPIList(path`/testsets/${testsetID}/testcases`, PaginatedResponse<Testcase>, {
+      query,
+      ...options,
+    });
+  }
+
+  /**
+   * Delete multiple testcases by their IDs.
+   */
+  delete(body: TestcaseDeleteParams, options?: RequestOptions): APIPromise<TestcaseDeleteResponse> {
+    return this._client.post('/testcases/bulk-delete', { body, ...options });
+  }
+
+  /**
+   * Retrieve a specific testcase by ID.
+   */
+  get(testcaseID: string, options?: RequestOptions): APIPromise<Testcase> {
+    return this._client.get(path`/testcases/${testcaseID}`, options);
+  }
+}
+
+export type TestcasesPaginatedResponse = PaginatedResponse<Testcase>;
+
+/**
+ * A test case in the Scorecard system. Contains JSON data that is validated
+ * against the schema defined by its testset. The `inputs` and `labels` fields are
+ * derived from the `data` field based on the testset's `fieldMapping`, and include
+ * all mapped fields, including those with validation errors. Testcases are stored
+ * regardless of validation results, with any validation errors included in the
+ * `validationErrors` field.
+ */
+export interface Testcase {
+  /**
+   * The ID of the testcase
+   */
+  id: string;
+
+  /**
+   * Derived from data based on the testset's fieldMapping. Contains all fields
+   * marked as inputs, including those with validation errors.
+   */
+  inputs: Record<string, unknown>;
+
+  /**
+   * The JSON data of the testcase, which is validated against the testset's schema.
+   */
+  jsonData: Record<string, unknown>;
+
+  /**
+   * Derived from data based on the testset's fieldMapping. Contains all fields
+   * marked as labels, including those with validation errors.
+   */
+  labels: Record<string, unknown>;
+
+  /**
+   * The ID of the testset this testcase belongs to
+   */
+  testsetId: string;
+
+  /**
+   * Validation errors found in the testcase data. If present, the testcase doesn't
+   * fully conform to its testset's schema.
+   */
+  validationErrors?: Array<Testcase.ValidationError>;
+}
+
+export namespace Testcase {
+  export interface ValidationError {
+    /**
+     * Human-readable error description
+     */
+    message: string;
+
+    /**
+     * JSON Pointer to the field with the validation error
+     */
+    path: string;
+  }
+}
+
+export interface TestcaseCreateResponse {
+  items: Array<Testcase>;
+}
+
+export interface TestcaseDeleteResponse {
+  /**
+   * Whether the deletion was successful
+   */
+  success: boolean;
+}
+
+export interface TestcaseCreateParams {
+  /**
+   * Testcases to create (max 100)
+   */
+  items: Array<TestcaseCreateParams.Item>;
+}
+
+export namespace TestcaseCreateParams {
+  /**
+   * A test case in the Scorecard system. Contains JSON data that is validated
+   * against the schema defined by its testset. The `inputs` and `labels` fields are
+   * derived from the `data` field based on the testset's `fieldMapping`, and include
+   * all mapped fields, including those with validation errors. Testcases are stored
+   * regardless of validation results, with any validation errors included in the
+   * `validationErrors` field.
+   */
+  export interface Item {
+    /**
+     * The JSON data of the testcase, which is validated against the testset's schema.
+     */
+    jsonData: Record<string, unknown>;
+  }
+}
+
+export interface TestcaseUpdateParams {
+  /**
+   * The JSON data of the testcase, which is validated against the testset's schema.
+   */
+  jsonData: Record<string, unknown>;
+}
+
+export interface TestcaseListParams extends PaginatedResponseParams {}
+
+export interface TestcaseDeleteParams {
+  /**
+   * IDs of testcases to delete
+   */
+  ids: Array<string>;
+}
+
+export declare namespace Testcases {
+  export {
+    type Testcase as Testcase,
+    type TestcaseCreateResponse as TestcaseCreateResponse,
+    type TestcaseDeleteResponse as TestcaseDeleteResponse,
+    type TestcasesPaginatedResponse as TestcasesPaginatedResponse,
+    type TestcaseCreateParams as TestcaseCreateParams,
+    type TestcaseUpdateParams as TestcaseUpdateParams,
+    type TestcaseListParams as TestcaseListParams,
+    type TestcaseDeleteParams as TestcaseDeleteParams,
+  };
+}

package/src/resources/testsets.ts

@@ -0,0 +1,257 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class Testsets extends APIResource {
+  /**
+   * Create a new testset for a project. The testset will be created in the project
+   * specified in the path.
+   */
+  create(projectID: string, body: TestsetCreateParams, options?: RequestOptions): APIPromise<Testset> {
+    return this._client.post(path`/projects/${projectID}/testsets`, { body, ...options });
+  }
+
+  /**
+   * Update a testset. Only the fields provided in the request body will be updated.
+   * If a field is provided, the new content will replace the existing content. If a
+   * field is not provided, the existing content will remain unchanged.
+   *
+   * When updating the schema:
+   *
+   * - If field mappings are not provided and existing mappings reference fields that
+   *   no longer exist, those mappings will be automatically removed
+   * - To preserve all existing mappings, ensure all referenced fields remain in the
+   *   updated schema
+   * - For complete control, provide both schema and fieldMapping when updating the
+   *   schema
+   */
+  update(
+    testsetID: string,
+    body: TestsetUpdateParams | null | undefined = {},
+    options?: RequestOptions,
+  ): APIPromise<Testset> {
+    return this._client.patch(path`/testsets/${testsetID}`, { body, ...options });
+  }
+
+  /**
+   * Retrieve a paginated list of testsets belonging to a project.
+   */
+  list(
+    projectID: string,
+    query: TestsetListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<TestsetsPaginatedResponse, Testset> {
+    return this._client.getAPIList(path`/projects/${projectID}/testsets`, PaginatedResponse<Testset>, {
+      query,
+      ...options,
+    });
+  }
+
+  /**
+   * Delete testset
+   */
+  delete(testsetID: string, options?: RequestOptions): APIPromise<TestsetDeleteResponse> {
+    return this._client.delete(path`/testsets/${testsetID}`, options);
+  }
+
+  /**
+   * Get testset by ID
+   */
+  get(testsetID: string, options?: RequestOptions): APIPromise<Testset> {
+    return this._client.get(path`/testsets/${testsetID}`, options);
+  }
+}
+
+export type TestsetsPaginatedResponse = PaginatedResponse<Testset>;
+
+/**
+ * A collection of test cases that share the same schema. Each testset defines the
+ * structure of its test cases through a JSON schema. The `fieldMapping` object
+ * maps top-level keys of the testcase schema to their roles (input/label). Fields
+ * not mentioned in the `fieldMapping` during creation or update are treated as
+ * metadata.
+ *
+ * ## JSON Schema validation constraints supported:
+ *
+ * - **Required fields** - Fields listed in the schema's `required` array must be
+ *   present in testcases
+ * - **Type validation** - Values must match the specified type (string, number,
+ *   boolean, null, integer, object, array)
+ * - **Enum validation** - Values must be one of the options specified in the
+ *   `enum` array
+ * - **Object property validation** - Properties of objects must conform to their
+ *   defined schemas
+ * - **Array item validation** - Items in arrays must conform to the `items` schema
+ * - **Logical composition** - Values must conform to at least one schema in the
+ *   `anyOf` array
+ *
+ * Testcases that fail validation will still be stored, but will include
+ * `validationErrors` detailing the issues. Extra fields in the testcase data that
+ * are not in the schema will be stored but are ignored during validation.
+ */
+export interface Testset {
+  /**
+   * The ID of the testset
+   */
+  id: string;
+
+  /**
+   * The description of the testset
+   */
+  description: string;
+
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  fieldMapping: Testset.FieldMapping;
+
+  /**
+   * The JSON schema for each testcase in the testset
+   */
+  jsonSchema: Record<string, unknown>;
+
+  /**
+   * The name of the testset
+   */
+  name: string;
+}
+
+export namespace Testset {
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  export interface FieldMapping {
+    /**
+     * Fields that represent inputs to the AI system
+     */
+    inputs: Array<string>;
+
+    /**
+     * Fields that represent expected outputs/labels
+     */
+    labels: Array<string>;
+
+    /**
+     * Fields that are not inputs or labels
+     */
+    metadata: Array<string>;
+  }
+}
+
+export interface TestsetDeleteResponse {
+  /**
+   * Whether the deletion was successful
+   */
+  success: boolean;
+}
+
+export interface TestsetCreateParams {
+  /**
+   * The description of the testset
+   */
+  description: string;
+
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  fieldMapping: TestsetCreateParams.FieldMapping;
+
+  /**
+   * The JSON schema for each testcase in the testset
+   */
+  jsonSchema: Record<string, unknown>;
+
+  /**
+   * The name of the testset
+   */
+  name: string;
+}
+
+export namespace TestsetCreateParams {
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  export interface FieldMapping {
+    /**
+     * Fields that represent inputs to the AI system
+     */
+    inputs: Array<string>;
+
+    /**
+     * Fields that represent expected outputs/labels
+     */
+    labels: Array<string>;
+
+    /**
+     * Fields that are not inputs or labels
+     */
+    metadata: Array<string>;
+  }
+}
+
+export interface TestsetUpdateParams {
+  /**
+   * The description of the testset
+   */
+  description?: string;
+
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  fieldMapping?: TestsetUpdateParams.FieldMapping;
+
+  /**
+   * The JSON schema for each testcase in the testset
+   */
+  jsonSchema?: Record<string, unknown>;
+
+  /**
+   * The name of the testset
+   */
+  name?: string;
+}
+
+export namespace TestsetUpdateParams {
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  export interface FieldMapping {
+    /**
+     * Fields that represent inputs to the AI system
+     */
+    inputs: Array<string>;
+
+    /**
+     * Fields that represent expected outputs/labels
+     */
+    labels: Array<string>;
+
+    /**
+     * Fields that are not inputs or labels
+     */
+    metadata: Array<string>;
+  }
+}
+
+export interface TestsetListParams extends PaginatedResponseParams {}
+
+export declare namespace Testsets {
+  export {
+    type Testset as Testset,
+    type TestsetDeleteResponse as TestsetDeleteResponse,
+    type TestsetsPaginatedResponse as TestsetsPaginatedResponse,
+    type TestsetCreateParams as TestsetCreateParams,
+    type TestsetUpdateParams as TestsetUpdateParams,
+    type TestsetListParams as TestsetListParams,
+  };
+}

package/src/resources.ts

@@ -0,0 +1 @@
+export * from './resources/index';

package/src/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  // this config is included in the published src directory to prevent TS errors
+  // from appearing when users go to source, and VSCode opens the source .ts file
+  // via declaration maps
+  "include": ["index.ts"],
+  "compilerOptions": {
+    "target": "ES2015",
+    "lib": ["DOM", "DOM.Iterable", "ES2018"],
+    "moduleResolution": "node"
+  }
+}

package/src/uploads.ts

@@ -0,0 +1,2 @@
+/** @deprecated Import from ./core/uploads instead */
+export * from './core/uploads';

package/src/version.ts

@@ -0,0 +1 @@
+export const VERSION = '1.0.0-alpha.1'; // x-release-please-version

package/uploads.d.mts

@@ -0,0 +1,2 @@
+export * from "./core/uploads.mjs";
+//# sourceMappingURL=uploads.d.mts.map

package/uploads.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"uploads.d.mts","sourceRoot":"","sources":["src/uploads.ts"],"names":[],"mappings":""}

package/uploads.d.ts

@@ -0,0 +1,2 @@
+export * from "./core/uploads.js";
+//# sourceMappingURL=uploads.d.ts.map

package/uploads.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"uploads.d.ts","sourceRoot":"","sources":["src/uploads.ts"],"names":[],"mappings":""}

package/uploads.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+/** @deprecated Import from ./core/uploads instead */
+__exportStar(require("./core/uploads.js"), exports);
+//# sourceMappingURL=uploads.js.map

package/uploads.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"uploads.js","sourceRoot":"","sources":["src/uploads.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,qDAAqD;AACrD,oDAA+B"}