scorecard-ai 0.6.1 → 1.0.0-alpha.1

package/src/internal/utils/uuid.ts

@@ -0,0 +1,19 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { getCrypto } from '../shims/crypto';
+
+/**
+ * https://stackoverflow.com/a/2117523
+ */
+export let uuid4 = function () {
+  const crypto = getCrypto();
+  if (crypto?.randomUUID) {
+    uuid4 = crypto.randomUUID.bind(crypto);
+    return crypto.randomUUID();
+  }
+  const u8 = new Uint8Array(1);
+  const randomByte = crypto ? () => crypto.getRandomValues(u8)[0]! : () => (Math.random() * 0xff) & 0xff;
+  return '10000000-1000-4000-8000-100000000000'.replace(/[018]/g, (c) =>
+    (+c ^ (randomByte() & (15 >> (+c / 4)))).toString(16),
+  );
+};

package/src/internal/utils/values.ts

@@ -0,0 +1,102 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { ScorecardError } from '../../core/error';
+
+// https://url.spec.whatwg.org/#url-scheme-string
+const startsWithSchemeRegexp = /^[a-z][a-z0-9+.-]*:/i;
+
+export const isAbsoluteURL = (url: string): boolean => {
+  return startsWithSchemeRegexp.test(url);
+};
+
+/** Returns an object if the given value isn't an object, otherwise returns as-is */
+export function maybeObj(x: unknown): object {
+  if (typeof x !== 'object') {
+    return {};
+  }
+
+  return x ?? {};
+}
+
+// https://stackoverflow.com/a/34491287
+export function isEmptyObj(obj: Object | null | undefined): boolean {
+  if (!obj) return true;
+  for (const _k in obj) return false;
+  return true;
+}
+
+// https://eslint.org/docs/latest/rules/no-prototype-builtins
+export function hasOwn<T extends object = object>(obj: T, key: PropertyKey): key is keyof T {
+  return Object.prototype.hasOwnProperty.call(obj, key);
+}
+
+export function isObj(obj: unknown): obj is Record<string, unknown> {
+  return obj != null && typeof obj === 'object' && !Array.isArray(obj);
+}
+
+export const ensurePresent = <T>(value: T | null | undefined): T => {
+  if (value == null) {
+    throw new ScorecardError(`Expected a value to be given but received ${value} instead.`);
+  }
+
+  return value;
+};
+
+export const validatePositiveInteger = (name: string, n: unknown): number => {
+  if (typeof n !== 'number' || !Number.isInteger(n)) {
+    throw new ScorecardError(`${name} must be an integer`);
+  }
+  if (n < 0) {
+    throw new ScorecardError(`${name} must be a positive integer`);
+  }
+  return n;
+};
+
+export const coerceInteger = (value: unknown): number => {
+  if (typeof value === 'number') return Math.round(value);
+  if (typeof value === 'string') return parseInt(value, 10);
+
+  throw new ScorecardError(`Could not coerce ${value} (type: ${typeof value}) into a number`);
+};
+
+export const coerceFloat = (value: unknown): number => {
+  if (typeof value === 'number') return value;
+  if (typeof value === 'string') return parseFloat(value);
+
+  throw new ScorecardError(`Could not coerce ${value} (type: ${typeof value}) into a number`);
+};
+
+export const coerceBoolean = (value: unknown): boolean => {
+  if (typeof value === 'boolean') return value;
+  if (typeof value === 'string') return value === 'true';
+  return Boolean(value);
+};
+
+export const maybeCoerceInteger = (value: unknown): number | undefined => {
+  if (value === undefined) {
+    return undefined;
+  }
+  return coerceInteger(value);
+};
+
+export const maybeCoerceFloat = (value: unknown): number | undefined => {
+  if (value === undefined) {
+    return undefined;
+  }
+  return coerceFloat(value);
+};
+
+export const maybeCoerceBoolean = (value: unknown): boolean | undefined => {
+  if (value === undefined) {
+    return undefined;
+  }
+  return coerceBoolean(value);
+};
+
+export const safeJSON = (text: string) => {
+  try {
+    return JSON.parse(text);
+  } catch (err) {
+    return undefined;
+  }
+};

package/src/internal/utils.ts

@@ -0,0 +1,8 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export * from './utils/values';
+export * from './utils/base64';
+export * from './utils/env';
+export * from './utils/log';
+export * from './utils/uuid';
+export * from './utils/sleep';

package/src/lib/.keep

@@ -0,0 +1,4 @@
+File generated from our OpenAPI spec by Stainless.
+
+This directory can be used to store custom files to expand the SDK.
+It is ignored by Stainless code generation and its content (other than this keep file) won't be touched.

package/src/pagination.ts

@@ -0,0 +1,2 @@
+/** @deprecated Import from ./core/pagination instead */
+export * from './core/pagination';

package/src/resource.ts

@@ -0,0 +1,2 @@
+/** @deprecated Import from ./core/resource instead */
+export * from './core/resource';

package/src/resources/execution-records.ts

@@ -0,0 +1,85 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class ExecutionRecords extends APIResource {
+  /**
+   * Create a new execution record.
+   */
+  create(
+    runID: string,
+    body: ExecutionRecordCreateParams,
+    options?: RequestOptions,
+  ): APIPromise<ExecutionRecord> {
+    return this._client.post(path`/runs/${runID}/executionrecords`, { body, ...options });
+  }
+}
+
+/**
+ * An execution record in the Scorecard system.
+ */
+export interface ExecutionRecord {
+  /**
+   * The ID of the execution record
+   */
+  id: string;
+
+  /**
+   * The actual inputs sent to the system, which should match the system's input
+   * schema
+   */
+  inputs: Record<string, unknown>;
+
+  /**
+   * The expected outputs for the testcase
+   */
+  labels: Record<string, unknown>;
+
+  /**
+   * The actual outputs from the system
+   */
+  outputs: Record<string, unknown>;
+
+  /**
+   * The ID of the run containing this execution record
+   */
+  runId: string;
+
+  /**
+   * The ID of the testcase
+   */
+  testcaseId?: string;
+}
+
+export interface ExecutionRecordCreateParams {
+  /**
+   * The actual inputs sent to the system, which should match the system's input
+   * schema
+   */
+  inputs: Record<string, unknown>;
+
+  /**
+   * The expected outputs for the testcase
+   */
+  labels: Record<string, unknown>;
+
+  /**
+   * The actual outputs from the system
+   */
+  outputs: Record<string, unknown>;
+
+  /**
+   * The ID of the testcase
+   */
+  testcaseId?: string;
+}
+
+export declare namespace ExecutionRecords {
+  export {
+    type ExecutionRecord as ExecutionRecord,
+    type ExecutionRecordCreateParams as ExecutionRecordCreateParams,
+  };
+}

package/src/resources/index.ts

@@ -0,0 +1,52 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export * from './shared';
+export {
+  ExecutionRecords,
+  type ExecutionRecord,
+  type ExecutionRecordCreateParams,
+} from './execution-records';
+export {
+  Projects,
+  type ProjectListResponse,
+  type ProjectListParams,
+  type ProjectListResponsesPaginatedResponse,
+} from './projects';
+export { Runs, type Run, type RunCreateParams } from './runs';
+export {
+  SystemConfigs,
+  type SystemConfig,
+  type SystemConfigCreateParams,
+  type SystemConfigListParams,
+  type SystemConfigGetParams,
+  type SystemConfigsPaginatedResponse,
+} from './system-configs';
+export {
+  Systems,
+  type System,
+  type SystemDeleteResponse,
+  type SystemCreateParams,
+  type SystemUpdateParams,
+  type SystemListParams,
+  type SystemsPaginatedResponse,
+} from './systems';
+export {
+  Testcases,
+  type Testcase,
+  type TestcaseCreateResponse,
+  type TestcaseDeleteResponse,
+  type TestcaseCreateParams,
+  type TestcaseUpdateParams,
+  type TestcaseListParams,
+  type TestcaseDeleteParams,
+  type TestcasesPaginatedResponse,
+} from './testcases';
+export {
+  Testsets,
+  type Testset,
+  type TestsetDeleteResponse,
+  type TestsetCreateParams,
+  type TestsetUpdateParams,
+  type TestsetListParams,
+  type TestsetsPaginatedResponse,
+} from './testsets';

package/src/resources/projects.ts

@@ -0,0 +1,48 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+
+export class Projects extends APIResource {
+  /**
+   * Retrieve a paginated list of all projects. Projects are ordered by creation
+   * date, with oldest projects first.
+   */
+  list(
+    query: ProjectListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<ProjectListResponsesPaginatedResponse, ProjectListResponse> {
+    return this._client.getAPIList('/projects', PaginatedResponse<ProjectListResponse>, {
+      query,
+      ...options,
+    });
+  }
+}
+
+export type ProjectListResponsesPaginatedResponse = PaginatedResponse<ProjectListResponse>;
+
+/**
+ * A project in the Scorecard system.
+ */
+export interface ProjectListResponse {
+  /**
+   * The ID of the project
+   */
+  id: string;
+
+  /**
+   * The name of the project
+   */
+  name: string | null;
+}
+
+export interface ProjectListParams extends PaginatedResponseParams {}
+
+export declare namespace Projects {
+  export {
+    type ProjectListResponse as ProjectListResponse,
+    type ProjectListResponsesPaginatedResponse as ProjectListResponsesPaginatedResponse,
+    type ProjectListParams as ProjectListParams,
+  };
+}

package/src/resources/runs.ts

@@ -0,0 +1,61 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class Runs extends APIResource {
+  /**
+   * Create a new run.
+   */
+  create(projectID: string, body: RunCreateParams, options?: RequestOptions): APIPromise<Run> {
+    return this._client.post(path`/projects/${projectID}/runs`, { body, ...options });
+  }
+}
+
+/**
+ * A Run in the Scorecard system.
+ */
+export interface Run {
+  /**
+   * The ID of the Run
+   */
+  id: string;
+
+  /**
+   * The IDs of the metrics this Run is using
+   */
+  metricIds: Array<string>;
+
+  /**
+   * The ID of the Testset this Run is testing
+   */
+  testsetId: string;
+
+  /**
+   * The ID of the system configuration this Run is using
+   */
+  systemConfigId?: string;
+}
+
+export interface RunCreateParams {
+  /**
+   * The IDs of the metrics this Run is using
+   */
+  metricIds: Array<string>;
+
+  /**
+   * The ID of the Testset this Run is testing
+   */
+  testsetId: string;
+
+  /**
+   * The ID of the system configuration this Run is using
+   */
+  systemConfigId?: string;
+}
+
+export declare namespace Runs {
+  export { type Run as Run, type RunCreateParams as RunCreateParams };
+}

package/src/resources/shared.ts

@@ -0,0 +1,12 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+/**
+ * An API error
+ */
+export interface APIError {
+  code: string;
+
+  details: Record<string, unknown>;
+
+  message: string;
+}

package/src/resources/system-configs.ts

@@ -0,0 +1,170 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import { APIResource } from '../core/resource';
+import { APIPromise } from '../core/api-promise';
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../core/pagination';
+import { RequestOptions } from '../internal/request-options';
+import { path } from '../internal/utils/path';
+
+export class SystemConfigs extends APIResource {
+  /**
+   * Create a new configuration for a system.
+   *
+   * Each configuration contains specific parameter values that match the system's
+   * configSchema - things like model parameters, thresholds, or processing options.
+   * Once created, configurations cannot be modified, ensuring stable reference
+   * points for evaluations.
+   *
+   * When creating a configuration:
+   *
+   * - The 'config' object is validated against the parent system's configSchema
+   * - Configurations with validation errors are still stored, with errors included
+   *   in the response
+   * - Validation errors indicate fields that don't match the schema but don't
+   *   prevent creation
+   * - Having validation errors may affect how some evaluation metrics are calculated
+   */
+  create(
+    systemID: string,
+    body: SystemConfigCreateParams,
+    options?: RequestOptions,
+  ): APIPromise<SystemConfig> {
+    return this._client.post(path`/systems/${systemID}/configs`, { body, ...options });
+  }
+
+  /**
+   * Retrieve a paginated list of configurations for a specific system.
+   *
+   * System configurations provide concrete parameter values for a System Under Test,
+   * defining exactly how the system should be configured during an evaluation run.
+   */
+  list(
+    systemID: string,
+    query: SystemConfigListParams | null | undefined = {},
+    options?: RequestOptions,
+  ): PagePromise<SystemConfigsPaginatedResponse, SystemConfig> {
+    return this._client.getAPIList(path`/systems/${systemID}/configs`, PaginatedResponse<SystemConfig>, {
+      query,
+      ...options,
+    });
+  }
+
+  /**
+   * Retrieve a specific system configuration by ID.
+   */
+  get(
+    systemConfigID: string,
+    params: SystemConfigGetParams,
+    options?: RequestOptions,
+  ): APIPromise<SystemConfig> {
+    const { systemId } = params;
+    return this._client.get(path`/systems/${systemId}/configs/${systemConfigID}`, options);
+  }
+}
+
+export type SystemConfigsPaginatedResponse = PaginatedResponse<SystemConfig>;
+
+/**
+ * A SystemConfig defines the specific settings for a System Under Test.
+ *
+ * Configurations contain parameter values that determine system behavior during
+ * evaluation. They are immutable snapshots - once created, they never change.
+ *
+ * When running evaluations, you reference a specific configId to establish which
+ * configuration to test.
+ *
+ * Configurations will be validated against the system's configSchema, with
+ * non-conforming values generating warnings.
+ */
+export interface SystemConfig {
+  /**
+   * The ID of the system configuration
+   */
+  id: string;
+
+  /**
+   * The configuration of the system
+   */
+  config: Record<string, unknown>;
+
+  /**
+   * The name of the system configuration
+   */
+  name: string;
+
+  /**
+   * The ID of the system the configuration belongs to
+   */
+  systemId: string;
+
+  /**
+   * Validation errors found in the configuration. If present, the configuration
+   * doesn't fully conform to its system's configSchema.
+   */
+  validationErrors?: Array<SystemConfig.ValidationError>;
+}
+
+export namespace SystemConfig {
+  export interface ValidationError {
+    /**
+     * Human-readable error description
+     */
+    message: string;
+
+    /**
+     * JSON Pointer to the field with the validation error
+     */
+    path: string;
+  }
+}
+
+export interface SystemConfigCreateParams {
+  /**
+   * The configuration of the system
+   */
+  config: Record<string, unknown>;
+
+  /**
+   * The name of the system configuration
+   */
+  name: string;
+
+  /**
+   * Validation errors found in the configuration. If present, the configuration
+   * doesn't fully conform to its system's configSchema.
+   */
+  validationErrors?: Array<SystemConfigCreateParams.ValidationError>;
+}
+
+export namespace SystemConfigCreateParams {
+  export interface ValidationError {
+    /**
+     * Human-readable error description
+     */
+    message: string;
+
+    /**
+     * JSON Pointer to the field with the validation error
+     */
+    path: string;
+  }
+}
+
+export interface SystemConfigListParams extends PaginatedResponseParams {}
+
+export interface SystemConfigGetParams {
+  /**
+   * The ID of the system the configuration belongs to
+   */
+  systemId: string;
+}
+
+export declare namespace SystemConfigs {
+  export {
+    type SystemConfig as SystemConfig,
+    type SystemConfigsPaginatedResponse as SystemConfigsPaginatedResponse,
+    type SystemConfigCreateParams as SystemConfigCreateParams,
+    type SystemConfigListParams as SystemConfigListParams,
+    type SystemConfigGetParams as SystemConfigGetParams,
+  };
+}