scorecard-ai 1.1.0 → 2.0.0-alpha.1

package/src/client.ts

@@ -5,7 +5,6 @@ import type { HTTPMethod, PromiseOrValue, MergedRequestInit, FinalizedRequestIni
 import { uuid4 } from './internal/utils/uuid';
 import { validatePositiveInteger, isAbsoluteURL, safeJSON } from './internal/utils/values';
 import { sleep } from './internal/utils/sleep';
-import { type Logger, type LogLevel, parseLogLevel } from './internal/utils/log';
 export type { Logger, LogLevel } from './internal/utils/log';
 import { castToError, isAbortError } from './internal/errors';
 import type { APIResponseProps } from './internal/parse';
@@ -19,9 +18,6 @@ import { AbstractPage, type PaginatedResponseParams, PaginatedResponseResponse }
 import * as Uploads from './core/uploads';
 import * as API from './resources/index';
 import { APIPromise } from './core/api-promise';
-import { type Fetch } from './internal/builtin-types';
-import { HeadersLike, NullableHeaders, buildHeaders } from './internal/headers';
-import { FinalRequestOptions, RequestOptions } from './internal/request-options';
 import { Metric, MetricCreateParams, MetricUpdateParams, Metrics } from './resources/metrics';
 import {
   Project,
@@ -30,7 +26,7 @@ import {
   Projects,
   ProjectsPaginatedResponse,
 } from './resources/projects';
-import { Record as RecordsAPIRecord, RecordCreateParams, Records } from './resources/records';
+import { Record, RecordCreateParams, Records } from './resources/records';
 import { Run, RunCreateParams, Runs } from './resources/runs';
 import { Score, ScoreUpsertParams, Scores } from './resources/scores';
 import {
@@ -53,18 +49,27 @@ import {
   Testsets,
   TestsetsPaginatedResponse,
 } from './resources/testsets';
-import { readEnv } from './internal/utils/env';
-import { formatRequestDetails, loggerFor } from './internal/utils/log';
-import { isEmptyObj } from './internal/utils/values';
 import {
   System,
-  SystemCreateParams,
   SystemDeleteResponse,
   SystemListParams,
   SystemUpdateParams,
+  SystemUpsertParams,
   Systems,
   SystemsPaginatedResponse,
 } from './resources/systems/systems';
+import { type Fetch, type Record as BuiltinRecord } from './internal/builtin-types';
+import { HeadersLike, NullableHeaders, buildHeaders } from './internal/headers';
+import { FinalRequestOptions, RequestOptions } from './internal/request-options';
+import { readEnv } from './internal/utils/env';
+import {
+  type LogLevel,
+  type Logger,
+  formatRequestDetails,
+  loggerFor,
+  parseLogLevel,
+} from './internal/utils/log';
+import { isEmptyObj } from './internal/utils/values';
 
 const environments = {
   production: 'https://api2.scorecard.io/api/v2',
@@ -151,7 +156,7 @@ export interface ClientOptions {
    * These can be removed in individual requests by explicitly setting the
    * param to `undefined` in request options.
    */
-  defaultQuery?: Record<string, string | undefined> | undefined;
+  defaultQuery?: BuiltinRecord<string, string | undefined> | undefined;
 
   /**
    * Set the log level.
@@ -198,7 +203,7 @@ export class Scorecard {
    * @param {Fetch} [opts.fetch] - Specify a custom `fetch` function implementation.
    * @param {number} [opts.maxRetries=2] - The maximum number of times the client will retry a request.
    * @param {HeadersLike} opts.defaultHeaders - Default headers to include with every request to the API.
-   * @param {Record<string, string | undefined>} opts.defaultQuery - Default query parameters to include with every request to the API.
+   * @param {BuiltinRecord<string, string | undefined>} opts.defaultQuery - Default query parameters to include with every request to the API.
    */
   constructor({
     baseURL = readEnv('SCORECARD_BASE_URL'),
@@ -210,8 +215,6 @@ export class Scorecard {
         "The SCORECARD_API_KEY environment variable is missing or empty; either provide it, or instantiate the Scorecard client with an apiKey option, like new Scorecard({ apiKey: 'My API Key' }).",
       );
     }
-    // Support both API keys (which start with 'ak_') and legacy JWT bearer tokens
-    apiKey = !apiKey || apiKey.startsWith('ak_') ? apiKey : `Bearer ${apiKey}`;
 
     const options: ClientOptions = {
       apiKey,
@@ -259,13 +262,21 @@ export class Scorecard {
       timeout: this.timeout,
       logger: this.logger,
       logLevel: this.logLevel,
+      fetch: this.fetch,
       fetchOptions: this.fetchOptions,
       apiKey: this.apiKey,
       ...options,
     });
   }
 
-  protected defaultQuery(): Record<string, string | undefined> | undefined {
+  /**
+   * Check whether the base URL is set to its default.
+   */
+  #baseURLOverridden(): boolean {
+    return this.baseURL !== environments[this._options.environment || 'production'];
+  }
+
+  protected defaultQuery(): BuiltinRecord<string, string | undefined> | undefined {
     return this._options.defaultQuery;
   }
 
@@ -274,13 +285,13 @@ export class Scorecard {
   }
 
   protected authHeaders(opts: FinalRequestOptions): NullableHeaders | undefined {
-    return buildHeaders([{ Authorization: this.apiKey }]);
+    return buildHeaders([{ Authorization: `Bearer ${this.apiKey}` }]);
   }
 
   /**
    * Basic re-implementation of `qs.stringify` for primitive types.
    */
-  protected stringifyQuery(query: Record<string, unknown>): string {
+  protected stringifyQuery(query: BuiltinRecord<string, unknown>): string {
     return Object.entries(query)
       .filter(([_, value]) => typeof value !== 'undefined')
       .map(([key, value]) => {
@@ -314,11 +325,16 @@ export class Scorecard {
     return Errors.APIError.generate(status, error, message, headers);
   }
 
-  buildURL(path: string, query: Record<string, unknown> | null | undefined): string {
+  buildURL(
+    path: string,
+    query: BuiltinRecord<string, unknown> | null | undefined,
+    defaultBaseURL?: string | undefined,
+  ): string {
+    const baseURL = (!this.#baseURLOverridden() && defaultBaseURL) || this.baseURL;
     const url =
       isAbsoluteURL(path) ?
         new URL(path)
-      : new URL(this.baseURL + (this.baseURL.endsWith('/') && path.startsWith('/') ? path.slice(1) : path));
+      : new URL(baseURL + (baseURL.endsWith('/') && path.startsWith('/') ? path.slice(1) : path));
 
     const defaultQuery = this.defaultQuery();
     if (!isEmptyObj(defaultQuery)) {
@@ -326,7 +342,7 @@ export class Scorecard {
     }
 
     if (typeof query === 'object' && query && !Array.isArray(query)) {
-      url.search = this.stringifyQuery(query as Record<string, unknown>);
+      url.search = this.stringifyQuery(query as BuiltinRecord<string, unknown>);
     }
 
     return url.toString();
@@ -678,9 +694,9 @@ export class Scorecard {
     { retryCount = 0 }: { retryCount?: number } = {},
   ): { req: FinalizedRequestInit; url: string; timeout: number } {
     const options = { ...inputOptions };
-    const { method, path, query } = options;
+    const { method, path, query, defaultBaseURL } = options;
 
-    const url = this.buildURL(path!, query as Record<string, unknown>);
+    const url = this.buildURL(path!, query as BuiltinRecord<string, unknown>, defaultBaseURL);
     if ('timeout' in options) validatePositiveInteger('timeout', options.timeout);
     options.timeout = options.timeout ?? this.timeout;
     const { bodyHeaders, body } = this.buildBody({ options });
@@ -858,11 +874,7 @@ export declare namespace Scorecard {
     type MetricUpdateParams as MetricUpdateParams,
   };
 
-  export {
-    Records as Records,
-    type RecordsAPIRecord as Record,
-    type RecordCreateParams as RecordCreateParams,
-  };
+  export { Records as Records, type Record as Record, type RecordCreateParams as RecordCreateParams };
 
   export { Scores as Scores, type Score as Score, type ScoreUpsertParams as ScoreUpsertParams };
 
@@ -871,9 +883,9 @@ export declare namespace Scorecard {
     type System as System,
     type SystemDeleteResponse as SystemDeleteResponse,
     type SystemsPaginatedResponse as SystemsPaginatedResponse,
-    type SystemCreateParams as SystemCreateParams,
     type SystemUpdateParams as SystemUpdateParams,
     type SystemListParams as SystemListParams,
+    type SystemUpsertParams as SystemUpsertParams,
   };
 
   export type APIError = API.APIError;

package/src/core/resource.ts

@@ -2,7 +2,7 @@
 
 import type { Scorecard } from '../client';
 
-export class APIResource {
+export abstract class APIResource {
   protected _client: Scorecard;
 
   constructor(client: Scorecard) {

package/src/internal/headers.ts

@@ -1,5 +1,7 @@
 // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+import { isReadonlyArray } from './utils/values';
+
 type HeaderValue = string | undefined | null;
 export type HeadersLike =
   | Headers
@@ -9,7 +11,7 @@ export type HeadersLike =
   | null
   | NullableHeaders;
 
-const brand_privateNullableHeaders = Symbol('brand.privateNullableHeaders');
+const brand_privateNullableHeaders = /* @__PURE__ */ Symbol('brand.privateNullableHeaders');
 
 /**
  * @internal
@@ -25,8 +27,6 @@ export type NullableHeaders = {
   nulls: Set<string>;
 };
 
-const isArray = Array.isArray as (val: unknown) => val is readonly unknown[];
-
 function* iterateHeaders(headers: HeadersLike): IterableIterator<readonly [string, string | null]> {
   if (!headers) return;
 
@@ -43,7 +43,7 @@ function* iterateHeaders(headers: HeadersLike): IterableIterator<readonly [strin
   let iter: Iterable<readonly (HeaderValue | readonly HeaderValue[])[]>;
   if (headers instanceof Headers) {
     iter = headers.entries();
-  } else if (isArray(headers)) {
+  } else if (isReadonlyArray(headers)) {
     iter = headers;
   } else {
     shouldClear = true;
@@ -52,7 +52,7 @@ function* iterateHeaders(headers: HeadersLike): IterableIterator<readonly [strin
   for (let row of iter) {
     const name = row[0];
     if (typeof name !== 'string') throw new TypeError('expected header name to be a string');
-    const values = isArray(row[1]) ? row[1] : [row[1]];
+    const values = isReadonlyArray(row[1]) ? row[1] : [row[1]];
     let didClear = false;
     for (const value of values) {
       if (value === undefined) continue;

package/src/internal/request-options.ts

@@ -20,6 +20,7 @@ export type RequestOptions = {
   fetchOptions?: MergedRequestInit;
   signal?: AbortSignal | undefined | null;
   idempotencyKey?: string;
+  defaultBaseURL?: string | undefined;
 
   __binaryResponse?: boolean | undefined;
 };

package/src/internal/uploads.ts

@@ -90,7 +90,7 @@ export const multipartFormRequestOptions = async (
   return { ...opts, body: await createForm(opts.body, fetch) };
 };
 
-const supportsFormDataMap = new WeakMap<Fetch, Promise<boolean>>();
+const supportsFormDataMap = /** @__PURE__ */ new WeakMap<Fetch, Promise<boolean>>();
 
 /**
  * node-fetch doesn't support the global FormData object in recent node versions. Instead of sending

package/src/internal/utils/log.ts

@@ -58,7 +58,7 @@ const noopLogger = {
   debug: noop,
 };
 
-let cachedLoggers = new WeakMap<Logger, [LogLevel, Logger]>();
+let cachedLoggers = /** @__PURE__ */ new WeakMap<Logger, [LogLevel, Logger]>();
 
 export function loggerFor(client: Scorecard): Logger {
   const logger = client.logger;

package/src/internal/utils/path.ts

@@ -62,4 +62,4 @@ export const createPathTagFunction = (pathEncoder = encodeURIPath) =>
 /**
  * URI-encodes path params and ensures no unsafe /./ or /../ path segments are introduced.
  */
-export const path = createPathTagFunction(encodeURIPath);
+export const path = /* @__PURE__ */ createPathTagFunction(encodeURIPath);

package/src/internal/utils/values.ts

@@ -9,6 +9,9 @@ export const isAbsoluteURL = (url: string): boolean => {
   return startsWithSchemeRegexp.test(url);
 };
 
+export let isArray = (val: unknown): val is unknown[] => ((isArray = Array.isArray), isArray(val));
+export let isReadonlyArray = isArray as (val: unknown) => val is readonly unknown[];
+
 /** Returns an object if the given value isn't an object, otherwise returns as-is */
 export function maybeObj(x: unknown): object {
   if (typeof x !== 'object') {

package/src/resources/index.ts

@@ -16,9 +16,9 @@ export {
   Systems,
   type System,
   type SystemDeleteResponse,
-  type SystemCreateParams,
   type SystemUpdateParams,
   type SystemListParams,
+  type SystemUpsertParams,
   type SystemsPaginatedResponse,
 } from './systems/systems';
 export {

package/src/resources/systems/index.ts

@@ -4,15 +4,9 @@ export {
   Systems,
   type System,
   type SystemDeleteResponse,
-  type SystemCreateParams,
   type SystemUpdateParams,
   type SystemListParams,
+  type SystemUpsertParams,
   type SystemsPaginatedResponse,
 } from './systems';
-export {
-  Versions,
-  type SystemVersion,
-  type VersionCreateParams,
-  type VersionListParams,
-  type SystemVersionsPaginatedResponse,
-} from './versions';
+export { Versions, type SystemVersion, type VersionUpsertParams } from './versions';

package/src/resources/systems/systems.ts

@@ -2,13 +2,7 @@
 
 import { APIResource } from '../../core/resource';
 import * as VersionsAPI from './versions';
-import {
-  SystemVersion,
-  SystemVersionsPaginatedResponse,
-  VersionCreateParams,
-  VersionListParams,
-  Versions,
-} from './versions';
+import { SystemVersion, VersionUpsertParams, Versions } from './versions';
 import { APIPromise } from '../../core/api-promise';
 import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from '../../core/pagination';
 import { RequestOptions } from '../../internal/request-options';
@@ -18,87 +12,17 @@ export class Systems extends APIResource {
   versions: VersionsAPI.Versions = new VersionsAPI.Versions(this._client);
 
   /**
-   * Create a new system definition that specifies the interface contracts for a
-   * component you want to evaluate.
-   *
-   * A system acts as a template that defines three key contracts through JSON
-   * Schemas:
-   *
-   * 1. Input Schema: What data your system accepts (e.g., user queries, context
-   *    documents)
-   * 2. Output Schema: What data your system produces (e.g., responses, confidence
-   *    scores)
-   * 3. Config Schema: What parameters can be adjusted (e.g., model selection,
-   *    temperature)
-   *
-   * This separation lets you evaluate any system as a black box, focusing on its
-   * interface rather than implementation details.
-   *
-   * @example
-   * ```ts
-   * const system = await client.systems.create('314', {
-   *   configSchema: {
-   *     type: 'object',
-   *     properties: {
-   *       temperature: { type: 'number' },
-   *       maxTokens: { type: 'integer' },
-   *       model: { type: 'string', enum: ['gpt-4', 'gpt-4-turbo'] },
-   *     },
-   *     required: ['model'],
-   *   },
-   *   description: 'Production chatbot powered by GPT-4',
-   *   inputSchema: {
-   *     type: 'object',
-   *     properties: {
-   *       messages: {
-   *         type: 'array',
-   *         items: {
-   *           type: 'object',
-   *           properties: {
-   *             role: { type: 'string', enum: ['system', 'user', 'assistant'] },
-   *             content: { type: 'string' },
-   *           },
-   *           required: ['role', 'content'],
-   *         },
-   *       },
-   *     },
-   *     required: ['messages'],
-   *   },
-   *   name: 'GPT-4 Chatbot',
-   *   outputSchema: {
-   *     type: 'object',
-   *     properties: { response: { type: 'string' } },
-   *     required: ['response'],
-   *   },
-   * });
-   * ```
-   */
-  create(projectID: string, body: SystemCreateParams, options?: RequestOptions): APIPromise<System> {
-    return this._client.post(path`/projects/${projectID}/systems`, { body, ...options });
-  }
-
-  /**
-   * Update an existing system definition. Only the fields provided in the request
-   * body will be updated. If a field is provided, the new content will replace the
-   * existing content. If a field is not provided, the existing content will remain
-   * unchanged.
-   *
-   * When updating schemas:
-   *
-   * - The system will accept your changes regardless of compatibility with existing
-   *   configurations
-   * - Schema updates won't invalidate existing evaluations or configurations
-   * - For significant redesigns, creating a new system definition provides a cleaner
-   *   separation
+   * Update an existing system. Only the fields provided in the request body will be
+   * updated. If a field is provided, the new content will replace the existing
+   * content. If a field is not provided, the existing content will remain unchanged.
    *
    * @example
    * ```ts
    * const system = await client.systems.update(
    *   '12345678-0a8b-4f66-b6f3-2ddcfa097257',
    *   {
-   *     description:
-   *       'Updated production chatbot powered by GPT-4 Turbo',
-   *     name: 'GPT-4 Turbo Chatbot',
+   *     productionVersionId:
+   *       '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3',
    *   },
    * );
    * ```
@@ -161,23 +85,29 @@ export class Systems extends APIResource {
   get(systemID: string, options?: RequestOptions): APIPromise<System> {
     return this._client.get(path`/systems/${systemID}`, options);
   }
+
+  /**
+   * Create a new system. If one with the same name in the project exists, it updates
+   * it instead.
+   *
+   * @example
+   * ```ts
+   * const system = await client.systems.upsert('314', {
+   *   config: { temperature: 0.1, maxTokens: 1024 },
+   *   description: 'Production chatbot powered by GPT-4',
+   *   name: 'GPT-4 Chatbot',
+   * });
+   * ```
+   */
+  upsert(projectID: string, body: SystemUpsertParams, options?: RequestOptions): APIPromise<System> {
+    return this._client.post(path`/projects/${projectID}/systems`, { body, ...options });
+  }
 }
 
 export type SystemsPaginatedResponse = PaginatedResponse<System>;
 
 /**
- * A System Under Test (SUT) defines the interface to a component or service you
- * want to evaluate.
- *
- * It specifies three contracts through schemas:
- *
- * - inputSchema: The structure of data the system accepts.
- * - outputSchema: The structure of data the system produces.
- * - configSchema: The parameters that modify system behavior.
- *
- * This abstraction lets you evaluate any system as a black box, focusing on its
- * interface rather than implementation details. It's particularly useful for
- * systems with variable outputs or complex internal state.
+ * A System Under Test (SUT).
  *
  * Systems are templates - to run evaluations, pair them with a SystemVersion that
  * provides specific parameter values.
@@ -188,30 +118,48 @@ export interface System {
    */
   id: string;
 
-  /**
-   * The schema of the system's configuration.
-   */
-  configSchema: Record<string, unknown>;
-
   /**
    * The description of the system.
    */
   description: string;
 
   /**
-   * The schema of the system's inputs.
+   * The name of the system. Unique within the project.
    */
-  inputSchema: Record<string, unknown>;
+  name: string;
 
   /**
-   * The name of the system.
+   * The production version of the system.
    */
-  name: string;
+  productionVersion: VersionsAPI.SystemVersion;
 
   /**
-   * The schema of the system's outputs.
+   * The versions of the system.
    */
-  outputSchema: Record<string, unknown>;
+  versions: Array<System.Version>;
+}
+
+export namespace System {
+  /**
+   * A SystemVersion defines the specific settings for a System Under Test.
+   *
+   * System versions contain parameter values that determine system behavior during
+   * evaluation. They are immutable snapshots - once created, they never change.
+   *
+   * When running evaluations, you reference a specific systemVersionId to establish
+   * which system version to test.
+   */
+  export interface Version {
+    /**
+     * The ID of the system version.
+     */
+    id: string;
+
+    /**
+     * The name of the system version.
+     */
+    name: string;
+  }
 }
 
 export interface SystemDeleteResponse {
@@ -221,38 +169,30 @@ export interface SystemDeleteResponse {
   success: boolean;
 }
 
-export interface SystemCreateParams {
-  /**
-   * The schema of the system's configuration.
-   */
-  configSchema: Record<string, unknown>;
-
+export interface SystemUpdateParams {
   /**
    * The description of the system.
    */
-  description: string;
-
-  /**
-   * The schema of the system's inputs.
-   */
-  inputSchema: Record<string, unknown>;
+  description?: string;
 
   /**
-   * The name of the system.
+   * The name of the system. Unique within the project.
    */
-  name: string;
+  name?: string;
 
   /**
-   * The schema of the system's outputs.
+   * The ID of the production version of the system.
    */
-  outputSchema: Record<string, unknown>;
+  productionVersionId?: string;
 }
 
-export interface SystemUpdateParams {
+export interface SystemListParams extends PaginatedResponseParams {}
+
+export interface SystemUpsertParams {
   /**
-   * The schema of the system's configuration.
+   * The configuration of the system.
    */
-  configSchema?: Record<string, unknown>;
+  config: Record<string, unknown>;
 
   /**
    * The description of the system.
@@ -260,23 +200,12 @@ export interface SystemUpdateParams {
   description?: string;
 
   /**
-   * The schema of the system's inputs.
-   */
-  inputSchema?: Record<string, unknown>;
-
-  /**
-   * The name of the system.
+   * The name of the system. Should be unique within the project. Default is "Default
+   * system"
    */
   name?: string;
-
-  /**
-   * The schema of the system's outputs.
-   */
-  outputSchema?: Record<string, unknown>;
 }
 
-export interface SystemListParams extends PaginatedResponseParams {}
-
 Systems.Versions = Versions;
 
 export declare namespace Systems {
@@ -284,16 +213,14 @@ export declare namespace Systems {
     type System as System,
     type SystemDeleteResponse as SystemDeleteResponse,
     type SystemsPaginatedResponse as SystemsPaginatedResponse,
-    type SystemCreateParams as SystemCreateParams,
     type SystemUpdateParams as SystemUpdateParams,
     type SystemListParams as SystemListParams,
+    type SystemUpsertParams as SystemUpsertParams,
   };
 
   export {
     Versions as Versions,
     type SystemVersion as SystemVersion,
-    type SystemVersionsPaginatedResponse as SystemVersionsPaginatedResponse,
-    type VersionCreateParams as VersionCreateParams,
-    type VersionListParams as VersionListParams,
+    type VersionUpsertParams as VersionUpsertParams,
   };
 }