scorecard-ai 0.6.1 → 1.0.0-alpha.0

package/src/internal/utils/uuid.ts

@@ -0,0 +1,19 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { getCrypto } from '../shims/crypto';
+
+/**
+ * https://stackoverflow.com/a/2117523
+ */
+export let uuid4 = function () {
+  const crypto = getCrypto();
+  if (crypto?.randomUUID) {
+    uuid4 = crypto.randomUUID.bind(crypto);
+    return crypto.randomUUID();
+  }
+  const u8 = new Uint8Array(1);
+  const randomByte = crypto ? () => crypto.getRandomValues(u8)[0]! : () => (Math.random() * 0xff) & 0xff;
+  return '10000000-1000-4000-8000-100000000000'.replace(/[018]/g, (c) =>
+    (+c ^ (randomByte() & (15 >> (+c / 4)))).toString(16),
+  );
+};

package/src/internal/utils/values.ts

@@ -0,0 +1,102 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { ScorecardError } from '../../core/error';
+
+// https://url.spec.whatwg.org/#url-scheme-string
+const startsWithSchemeRegexp = /^[a-z][a-z0-9+.-]*:/i;
+
+export const isAbsoluteURL = (url: string): boolean => {
+  return startsWithSchemeRegexp.test(url);
+};
+
+/** Returns an object if the given value isn't an object, otherwise returns as-is */
+export function maybeObj(x: unknown): object {
+  if (typeof x !== 'object') {
+    return {};
+  }
+
+  return x ?? {};
+}
+
+// https://stackoverflow.com/a/34491287
+export function isEmptyObj(obj: Object | null | undefined): boolean {
+  if (!obj) return true;
+  for (const _k in obj) return false;
+  return true;
+}
+
+// https://eslint.org/docs/latest/rules/no-prototype-builtins
+export function hasOwn<T extends object = object>(obj: T, key: PropertyKey): key is keyof T {
+  return Object.prototype.hasOwnProperty.call(obj, key);
+}
+
+export function isObj(obj: unknown): obj is Record<string, unknown> {
+  return obj != null && typeof obj === 'object' && !Array.isArray(obj);
+}
+
+export const ensurePresent = <T>(value: T | null | undefined): T => {
+  if (value == null) {
+    throw new ScorecardError(`Expected a value to be given but received ${value} instead.`);
+  }
+
+  return value;
+};
+
+export const validatePositiveInteger = (name: string, n: unknown): number => {
+  if (typeof n !== 'number' || !Number.isInteger(n)) {
+    throw new ScorecardError(`${name} must be an integer`);
+  }
+  if (n < 0) {
+    throw new ScorecardError(`${name} must be a positive integer`);
+  }
+  return n;
+};
+
+export const coerceInteger = (value: unknown): number => {
+  if (typeof value === 'number') return Math.round(value);
+  if (typeof value === 'string') return parseInt(value, 10);
+
+  throw new ScorecardError(`Could not coerce ${value} (type: ${typeof value}) into a number`);
+};
+
+export const coerceFloat = (value: unknown): number => {
+  if (typeof value === 'number') return value;
+  if (typeof value === 'string') return parseFloat(value);
+
+  throw new ScorecardError(`Could not coerce ${value} (type: ${typeof value}) into a number`);
+};
+
+export const coerceBoolean = (value: unknown): boolean => {
+  if (typeof value === 'boolean') return value;
+  if (typeof value === 'string') return value === 'true';
+  return Boolean(value);
+};
+
+export const maybeCoerceInteger = (value: unknown): number | undefined => {
+  if (value === undefined) {
+    return undefined;
+  }
+  return coerceInteger(value);
+};
+
+export const maybeCoerceFloat = (value: unknown): number | undefined => {
+  if (value === undefined) {
+    return undefined;
+  }
+  return coerceFloat(value);
+};
+
+export const maybeCoerceBoolean = (value: unknown): boolean | undefined => {
+  if (value === undefined) {
+    return undefined;
+  }
+  return coerceBoolean(value);
+};
+
+export const safeJSON = (text: string) => {
+  try {
+    return JSON.parse(text);
+  } catch (err) {
+    return undefined;
+  }
+};

package/src/internal/utils.ts

@@ -0,0 +1,8 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export * from './utils/values';
+export * from './utils/base64';
+export * from './utils/env';
+export * from './utils/log';
+export * from './utils/uuid';
+export * from './utils/sleep';

package/src/lib/.keep

@@ -0,0 +1,4 @@
+File generated from our OpenAPI spec by Stainless.
+
+This directory can be used to store custom files to expand the SDK.
+It is ignored by Stainless code generation and its content (other than this keep file) won't be touched.

package/src/pagination.ts

@@ -0,0 +1,2 @@
+/** @deprecated Import from ./core/pagination instead */
+export * from './core/pagination';

package/src/resource.ts

@@ -0,0 +1,2 @@
+/** @deprecated Import from ./core/resource instead */
+export * from './core/resource';

package/src/resources/index.ts

@@ -0,0 +1,29 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export * from './shared';
+export {
+  Projects,
+  type ProjectListResponse,
+  type ProjectListParams,
+  type ProjectListResponsesPaginatedResponse,
+} from './projects';
+export {
+  Testcases,
+  type Testcase,
+  type TestcaseCreateResponse,
+  type TestcaseDeleteResponse,
+  type TestcaseCreateParams,
+  type TestcaseUpdateParams,
+  type TestcaseListParams,
+  type TestcaseDeleteParams,
+  type TestcasesPaginatedResponse,
+} from './testcases';
+export {
+  Testsets,
+  type Testset,
+  type TestsetDeleteResponse,
+  type TestsetCreateParams,
+  type TestsetUpdateParams,
+  type TestsetListParams,
+  type TestsetsPaginatedResponse,
+} from './testsets';

package/src/resources/projects.ts

@@ -0,0 +1,48 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+
+export class Projects extends APIResource {
+  /**
+   * Retrieve a paginated list of all projects. Projects are ordered by creation
+   * date, with oldest projects first.
+   */
+  list(
+    query: ProjectListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<ProjectListResponsesPaginatedResponse, ProjectListResponse> {
+    return this._client.getAPIList('/projects', PaginatedResponse<ProjectListResponse>, {
+      query,
+      ...options,
+    });
+  }
+}
+
+export type ProjectListResponsesPaginatedResponse = PaginatedResponse<ProjectListResponse>;
+
+/**
+ * A project in the Scorecard system.
+ */
+export interface ProjectListResponse {
+  /**
+   * The ID of the project
+   */
+  id: string;
+
+  /**
+   * The name of the project
+   */
+  name: string | null;
+}
+
+export interface ProjectListParams extends PaginatedResponseParams {}
+
+export declare namespace Projects {
+  export {
+    type ProjectListResponse as ProjectListResponse,
+    type ProjectListResponsesPaginatedResponse as ProjectListResponsesPaginatedResponse,
+    type ProjectListParams as ProjectListParams,
+  };
+}

package/src/resources/shared.ts

@@ -0,0 +1,12 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+/**
+ * An API error
+ */
+export interface APIError {
+  code: string;
+
+  details: Record<string, unknown>;
+
+  message: string;
+}

package/src/resources/testcases.ts

@@ -0,0 +1,201 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class Testcases extends APIResource {
+  /**
+   * Create multiple testcases in the specified testset.
+   */
+  create(
+    testsetID: string,
+    body: TestcaseCreateParams,
+    options?: RequestOptions,
+  ): APIPromise<TestcaseCreateResponse> {
+    return this._client.post(path`/testsets/${testsetID}/testcases`, { body, ...options });
+  }
+
+  /**
+   * Replace the data of an existing testcase while keeping its ID.
+   */
+  update(testcaseID: string, body: TestcaseUpdateParams, options?: RequestOptions): APIPromise<Testcase> {
+    return this._client.put(path`/testcases/${testcaseID}`, { body, ...options });
+  }
+
+  /**
+   * Retrieve a paginated list of testcases belonging to a testset.
+   */
+  list(
+    testsetID: string,
+    query: TestcaseListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<TestcasesPaginatedResponse, Testcase> {
+    return this._client.getAPIList(path`/testsets/${testsetID}/testcases`, PaginatedResponse<Testcase>, {
+      query,
+      ...options,
+    });
+  }
+
+  /**
+   * Delete multiple testcases from the specified testset.
+   */
+  delete(
+    testsetID: string,
+    body: TestcaseDeleteParams,
+    options?: RequestOptions,
+  ): APIPromise<TestcaseDeleteResponse> {
+    return this._client.delete(path`/testsets/${testsetID}/testcases`, { body, ...options });
+  }
+
+  /**
+   * Retrieve a specific testcase by ID.
+   */
+  get(testcaseID: string, options?: RequestOptions): APIPromise<Testcase> {
+    return this._client.get(path`/testcases/${testcaseID}`, options);
+  }
+}
+
+export type TestcasesPaginatedResponse = PaginatedResponse<Testcase>;
+
+/**
+ * A test case in the Scorecard system. Contains JSON data that is validated
+ * against the schema defined by its testset. The `inputs` and `labels` fields are
+ * derived from the `data` field based on the testset's `fieldMapping`, and include
+ * all mapped fields, including those with validation errors. Testcases are stored
+ * regardless of validation results, with any validation errors included in the
+ * `validationErrors` field.
+ */
+export interface Testcase {
+  /**
+   * The ID of the testcase
+   */
+  id: string;
+
+  /**
+   * The JSON data of the testcase, which is validated against the testset's schema.
+   */
+  data: Record<string, unknown>;
+
+  /**
+   * Derived from data based on the testset's fieldMapping. Contains all fields
+   * marked as inputs, including those with validation errors.
+   */
+  inputs: Record<string, unknown>;
+
+  /**
+   * Derived from data based on the testset's fieldMapping. Contains all fields
+   * marked as labels, including those with validation errors.
+   */
+  labels: Record<string, unknown>;
+
+  /**
+   * The ID of the testset this testcase belongs to
+   */
+  testsetId: string;
+
+  /**
+   * Validation errors found in the testcase data. If present, the testcase doesn't
+   * fully conform to its testset's schema.
+   */
+  validationErrors?: Array<Testcase.ValidationError>;
+}
+
+export namespace Testcase {
+  export interface ValidationError {
+    /**
+     * Human-readable error description
+     */
+    message: string;
+
+    /**
+     * JSON Pointer to the field with the validation error
+     */
+    path: string;
+  }
+}
+
+export interface TestcaseCreateResponse {
+  items: Array<Testcase>;
+}
+
+export interface TestcaseDeleteResponse {
+  /**
+   * Number of testcases successfully deleted
+   */
+  deletedCount: number;
+
+  /**
+   * List of errors encountered during deletion, if any
+   */
+  errors: Array<TestcaseDeleteResponse.Error>;
+}
+
+export namespace TestcaseDeleteResponse {
+  export interface Error {
+    /**
+     * ID of the testcase that failed to be deleted
+     */
+    id: string;
+
+    /**
+     * Error message explaining why the deletion failed
+     */
+    message: string;
+  }
+}
+
+export interface TestcaseCreateParams {
+  /**
+   * Testcases to create (max 100)
+   */
+  items: Array<TestcaseCreateParams.Item>;
+}
+
+export namespace TestcaseCreateParams {
+  /**
+   * A test case in the Scorecard system. Contains JSON data that is validated
+   * against the schema defined by its testset. The `inputs` and `labels` fields are
+   * derived from the `data` field based on the testset's `fieldMapping`, and include
+   * all mapped fields, including those with validation errors. Testcases are stored
+   * regardless of validation results, with any validation errors included in the
+   * `validationErrors` field.
+   */
+  export interface Item {
+    /**
+     * The JSON data of the testcase, which is validated against the testset's schema.
+     */
+    data: Record<string, unknown>;
+  }
+}
+
+export interface TestcaseUpdateParams {
+  /**
+   * The JSON data of the testcase, which is validated against the testset's schema.
+   */
+  data: Record<string, unknown>;
+}
+
+export interface TestcaseListParams extends PaginatedResponseParams {}
+
+export interface TestcaseDeleteParams {
+  /**
+   * IDs of testcases to delete (max 100)
+   */
+  ids: Array<string>;
+}
+
+export declare namespace Testcases {
+  export {
+    type Testcase as Testcase,
+    type TestcaseCreateResponse as TestcaseCreateResponse,
+    type TestcaseDeleteResponse as TestcaseDeleteResponse,
+    type TestcasesPaginatedResponse as TestcasesPaginatedResponse,
+    type TestcaseCreateParams as TestcaseCreateParams,
+    type TestcaseUpdateParams as TestcaseUpdateParams,
+    type TestcaseListParams as TestcaseListParams,
+    type TestcaseDeleteParams as TestcaseDeleteParams,
+  };
+}

package/src/resources/testsets.ts

@@ -0,0 +1,254 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class Testsets extends APIResource {
+  /**
+   * Create a new testset for a project. The testset will be created in the project
+   * specified in the path.
+   */
+  create(projectID: string, body: TestsetCreateParams, options?: RequestOptions): APIPromise<Testset> {
+    return this._client.post(path`/projects/${projectID}/testsets`, { body, ...options });
+  }
+
+  /**
+   * Update a testset. Only the fields provided in the request body will be updated.
+   * If a field is provided, the new content will replace the existing content. If a
+   * field is not provided, the existing content will remain unchanged.
+   *
+   * When updating the schema:
+   *
+   * - If field mappings are not provided and existing mappings reference fields that
+   *   no longer exist, those mappings will be automatically removed
+   * - To preserve all existing mappings, ensure all referenced fields remain in the
+   *   updated schema
+   * - For complete control, provide both schema and fieldMapping when updating the
+   *   schema
+   */
+  update(
+    testsetID: string,
+    body: TestsetUpdateParams | null | undefined = {},
+    options?: RequestOptions,
+  ): APIPromise<Testset> {
+    return this._client.patch(path`/testsets/${testsetID}`, { body, ...options });
+  }
+
+  /**
+   * Retrieve a paginated list of testsets belonging to a project.
+   */
+  list(
+    projectID: string,
+    query: TestsetListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<TestsetsPaginatedResponse, Testset> {
+    return this._client.getAPIList(path`/projects/${projectID}/testsets`, PaginatedResponse<Testset>, {
+      query,
+      ...options,
+    });
+  }
+
+  /**
+   * Delete testset
+   */
+  delete(testsetID: string, options?: RequestOptions): APIPromise<TestsetDeleteResponse> {
+    return this._client.delete(path`/testsets/${testsetID}`, options);
+  }
+
+  /**
+   * Get testset by ID
+   */
+  get(testsetID: string, options?: RequestOptions): APIPromise<Testset> {
+    return this._client.get(path`/testsets/${testsetID}`, options);
+  }
+}
+
+export type TestsetsPaginatedResponse = PaginatedResponse<Testset>;
+
+/**
+ * A collection of test cases that share the same schema. Each testset defines the
+ * structure of its test cases through a JSON schema. The `fieldMapping` object
+ * maps top-level keys of the testcase schema to their roles (input/label). Fields
+ * not mentioned in the `fieldMapping` during creation or update are treated as
+ * metadata.
+ *
+ * ## JSON Schema validation constraints supported:
+ *
+ * - **Required fields** - Fields listed in the schema's `required` array must be
+ *   present in testcases
+ * - **Type validation** - Values must match the specified type (string, number,
+ *   boolean, null, integer, object, array)
+ * - **Enum validation** - Values must be one of the options specified in the
+ *   `enum` array
+ * - **Object property validation** - Properties of objects must conform to their
+ *   defined schemas
+ * - **Array item validation** - Items in arrays must conform to the `items` schema
+ * - **Logical composition** - Values must conform to at least one schema in the
+ *   `anyOf` array
+ *
+ * Testcases that fail validation will still be stored, but will include
+ * `validationErrors` detailing the issues. Extra fields in the testcase data that
+ * are not in the schema will be stored but are ignored during validation.
+ */
+export interface Testset {
+  /**
+   * The ID of the testset
+   */
+  id: string;
+
+  /**
+   * The description of the testset
+   */
+  description: string;
+
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  fieldMapping: Testset.FieldMapping;
+
+  /**
+   * The name of the testset
+   */
+  name: string;
+
+  schema: Record<string, unknown>;
+}
+
+export namespace Testset {
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  export interface FieldMapping {
+    /**
+     * Fields that represent inputs to the AI system
+     */
+    inputs: Array<string>;
+
+    /**
+     * Fields that represent expected outputs/labels
+     */
+    labels: Array<string>;
+
+    /**
+     * Fields that are not inputs or labels
+     */
+    metadata: Array<string>;
+  }
+}
+
+export interface TestsetDeleteResponse {
+  /**
+   * Whether the deletion was successful
+   */
+  success: boolean;
+}
+
+export interface TestsetCreateParams {
+  /**
+   * The description of the testset
+   */
+  description: string;
+
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  fieldMapping: TestsetCreateParams.FieldMapping;
+
+  /**
+   * The name of the testset
+   */
+  name: string;
+
+  /**
+   * The JSON schema for each testcase in the testset
+   */
+  schema: Record<string, unknown>;
+}
+
+export namespace TestsetCreateParams {
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  export interface FieldMapping {
+    /**
+     * Fields that represent inputs to the AI system
+     */
+    inputs: Array<string>;
+
+    /**
+     * Fields that represent expected outputs/labels
+     */
+    labels: Array<string>;
+
+    /**
+     * Fields that are not inputs or labels
+     */
+    metadata: Array<string>;
+  }
+}
+
+export interface TestsetUpdateParams {
+  /**
+   * The description of the testset
+   */
+  description?: string;
+
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  fieldMapping?: TestsetUpdateParams.FieldMapping;
+
+  /**
+   * The name of the testset
+   */
+  name?: string;
+
+  /**
+   * The JSON schema for each testcase in the testset
+   */
+  schema?: Record<string, unknown>;
+}
+
+export namespace TestsetUpdateParams {
+  /**
+   * Maps top-level keys of the testcase schema to their roles (input/label).
+   * Unmapped fields are treated as metadata.
+   */
+  export interface FieldMapping {
+    /**
+     * Fields that represent inputs to the AI system
+     */
+    inputs: Array<string>;
+
+    /**
+     * Fields that represent expected outputs/labels
+     */
+    labels: Array<string>;
+
+    /**
+     * Fields that are not inputs or labels
+     */
+    metadata: Array<string>;
+  }
+}
+
+export interface TestsetListParams extends PaginatedResponseParams {}
+
+export declare namespace Testsets {
+  export {
+    type Testset as Testset,
+    type TestsetDeleteResponse as TestsetDeleteResponse,
+    type TestsetsPaginatedResponse as TestsetsPaginatedResponse,
+    type TestsetCreateParams as TestsetCreateParams,
+    type TestsetUpdateParams as TestsetUpdateParams,
+    type TestsetListParams as TestsetListParams,
+  };
+}

package/src/resources.ts

@@ -0,0 +1 @@
+export * from './resources/index';