scorecard-ai 0.6.1 → 1.0.0-alpha.0

package/resources/testsets.d.ts

@@ -0,0 +1,194 @@
+import { APIResource } from "../core/resource.js";
+import { APIPromise } from "../core/api-promise.js";
+import { PagePromise, PaginatedResponse, type PaginatedResponseParams } from "../core/pagination.js";
+import { RequestOptions } from "../internal/request-options.js";
+export declare class Testsets extends APIResource {
+    /**
+     * Create a new testset for a project. The testset will be created in the project
+     * specified in the path.
+     */
+    create(projectID: string, body: TestsetCreateParams, options?: RequestOptions): APIPromise<Testset>;
+    /**
+     * Update a testset. Only the fields provided in the request body will be updated.
+     * If a field is provided, the new content will replace the existing content. If a
+     * field is not provided, the existing content will remain unchanged.
+     *
+     * When updating the schema:
+     *
+     * - If field mappings are not provided and existing mappings reference fields that
+     *   no longer exist, those mappings will be automatically removed
+     * - To preserve all existing mappings, ensure all referenced fields remain in the
+     *   updated schema
+     * - For complete control, provide both schema and fieldMapping when updating the
+     *   schema
+     */
+    update(testsetID: string, body?: TestsetUpdateParams | null | undefined, options?: RequestOptions): APIPromise<Testset>;
+    /**
+     * Retrieve a paginated list of testsets belonging to a project.
+     */
+    list(projectID: string, query?: TestsetListParams | null | undefined, options?: RequestOptions): PagePromise<TestsetsPaginatedResponse, Testset>;
+    /**
+     * Delete testset
+     */
+    delete(testsetID: string, options?: RequestOptions): APIPromise<TestsetDeleteResponse>;
+    /**
+     * Get testset by ID
+     */
+    get(testsetID: string, options?: RequestOptions): APIPromise<Testset>;
+}
+export type TestsetsPaginatedResponse = PaginatedResponse<Testset>;
+/**
+ * A collection of test cases that share the same schema. Each testset defines the
+ * structure of its test cases through a JSON schema. The `fieldMapping` object
+ * maps top-level keys of the testcase schema to their roles (input/label). Fields
+ * not mentioned in the `fieldMapping` during creation or update are treated as
+ * metadata.
+ *
+ * ## JSON Schema validation constraints supported:
+ *
+ * - **Required fields** - Fields listed in the schema's `required` array must be
+ *   present in testcases
+ * - **Type validation** - Values must match the specified type (string, number,
+ *   boolean, null, integer, object, array)
+ * - **Enum validation** - Values must be one of the options specified in the
+ *   `enum` array
+ * - **Object property validation** - Properties of objects must conform to their
+ *   defined schemas
+ * - **Array item validation** - Items in arrays must conform to the `items` schema
+ * - **Logical composition** - Values must conform to at least one schema in the
+ *   `anyOf` array
+ *
+ * Testcases that fail validation will still be stored, but will include
+ * `validationErrors` detailing the issues. Extra fields in the testcase data that
+ * are not in the schema will be stored but are ignored during validation.
+ */
+export interface Testset {
+    /**
+     * The ID of the testset
+     */
+    id: string;
+    /**
+     * The description of the testset
+     */
+    description: string;
+    /**
+     * Maps top-level keys of the testcase schema to their roles (input/label).
+     * Unmapped fields are treated as metadata.
+     */
+    fieldMapping: Testset.FieldMapping;
+    /**
+     * The name of the testset
+     */
+    name: string;
+    schema: Record<string, unknown>;
+}
+export declare namespace Testset {
+    /**
+     * Maps top-level keys of the testcase schema to their roles (input/label).
+     * Unmapped fields are treated as metadata.
+     */
+    interface FieldMapping {
+        /**
+         * Fields that represent inputs to the AI system
+         */
+        inputs: Array<string>;
+        /**
+         * Fields that represent expected outputs/labels
+         */
+        labels: Array<string>;
+        /**
+         * Fields that are not inputs or labels
+         */
+        metadata: Array<string>;
+    }
+}
+export interface TestsetDeleteResponse {
+    /**
+     * Whether the deletion was successful
+     */
+    success: boolean;
+}
+export interface TestsetCreateParams {
+    /**
+     * The description of the testset
+     */
+    description: string;
+    /**
+     * Maps top-level keys of the testcase schema to their roles (input/label).
+     * Unmapped fields are treated as metadata.
+     */
+    fieldMapping: TestsetCreateParams.FieldMapping;
+    /**
+     * The name of the testset
+     */
+    name: string;
+    /**
+     * The JSON schema for each testcase in the testset
+     */
+    schema: Record<string, unknown>;
+}
+export declare namespace TestsetCreateParams {
+    /**
+     * Maps top-level keys of the testcase schema to their roles (input/label).
+     * Unmapped fields are treated as metadata.
+     */
+    interface FieldMapping {
+        /**
+         * Fields that represent inputs to the AI system
+         */
+        inputs: Array<string>;
+        /**
+         * Fields that represent expected outputs/labels
+         */
+        labels: Array<string>;
+        /**
+         * Fields that are not inputs or labels
+         */
+        metadata: Array<string>;
+    }
+}
+export interface TestsetUpdateParams {
+    /**
+     * The description of the testset
+     */
+    description?: string;
+    /**
+     * Maps top-level keys of the testcase schema to their roles (input/label).
+     * Unmapped fields are treated as metadata.
+     */
+    fieldMapping?: TestsetUpdateParams.FieldMapping;
+    /**
+     * The name of the testset
+     */
+    name?: string;
+    /**
+     * The JSON schema for each testcase in the testset
+     */
+    schema?: Record<string, unknown>;
+}
+export declare namespace TestsetUpdateParams {
+    /**
+     * Maps top-level keys of the testcase schema to their roles (input/label).
+     * Unmapped fields are treated as metadata.
+     */
+    interface FieldMapping {
+        /**
+         * Fields that represent inputs to the AI system
+         */
+        inputs: Array<string>;
+        /**
+         * Fields that represent expected outputs/labels
+         */
+        labels: Array<string>;
+        /**
+         * Fields that are not inputs or labels
+         */
+        metadata: Array<string>;
+    }
+}
+export interface TestsetListParams extends PaginatedResponseParams {
+}
+export declare namespace Testsets {
+    export { type Testset as Testset, type TestsetDeleteResponse as TestsetDeleteResponse, type TestsetsPaginatedResponse as TestsetsPaginatedResponse, type TestsetCreateParams as TestsetCreateParams, type TestsetUpdateParams as TestsetUpdateParams, type TestsetListParams as TestsetListParams, };
+}
+//# sourceMappingURL=testsets.d.ts.map

package/resources/testsets.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"testsets.d.ts","sourceRoot":"","sources":["../src/resources/testsets.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,EAAE,UAAU,EAAE;OACd,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,uBAAuB,EAAE;OAChE,EAAE,cAAc,EAAE;AAGzB,qBAAa,QAAS,SAAQ,WAAW;IACvC;;;OAGG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC;IAInG;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE,mBAAmB,GAAG,IAAI,GAAG,SAAc,EACjD,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,OAAO,CAAC;IAItB;;OAEG;IACH,IAAI,CACF,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,iBAAiB,GAAG,IAAI,GAAG,SAAc,EAChD,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,yBAAyB,EAAE,OAAO,CAAC;IAOlD;;OAEG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,qBAAqB,CAAC;IAItF;;OAEG;IACH,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC;CAGtE;AAED,MAAM,MAAM,yBAAyB,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;AAEnE;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,WAAW,OAAO;IACtB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,YAAY,EAAE,OAAO,CAAC,YAAY,CAAC;IAEnC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,yBAAiB,OAAO,CAAC;IACvB;;;OAGG;IACH,UAAiB,YAAY;QAC3B;;WAEG;QACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAEtB;;WAEG;QACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAEtB;;WAEG;QACH,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;KACzB;CACF;AAED,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,YAAY,EAAE,mBAAmB,CAAC,YAAY,CAAC;IAE/C;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,yBAAiB,mBAAmB,CAAC;IACnC;;;OAGG;IACH,UAAiB,YAAY;QAC3B;;WAEG;QACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAEtB;;WAEG;QACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAEtB;;WAEG;QACH,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;KACzB;CACF;AAED,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,YAAY,CAAC,EAAE,mBAAmB,CAAC,YAAY,CAAC;IAEhD;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,yBAAiB,mBAAmB,CAAC;IACnC;;;OAGG;IACH,UAAiB,YAAY;QAC3B;;WAEG;QACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAEtB;;WAEG;QACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAEtB;;WAEG;QACH,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;KACzB;CACF;AAED,MAAM,WAAW,iBAAkB,SAAQ,uBAAuB;CAAG;AAErE,MAAM,CAAC,OAAO,WAAW,QAAQ,CAAC;IAChC,OAAO,EACL,KAAK,OAAO,IAAI,OAAO,EACvB,KAAK,qBAAqB,IAAI,qBAAqB,EACnD,KAAK,yBAAyB,IAAI,yBAAyB,EAC3D,KAAK,mBAAmB,IAAI,mBAAmB,EAC/C,KAAK,mBAAmB,IAAI,mBAAmB,EAC/C,KAAK,iBAAiB,IAAI,iBAAiB,GAC5C,CAAC;CACH"}

package/resources/testsets.js

@@ -0,0 +1,56 @@
+"use strict";
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Testsets = void 0;
+const resource_1 = require("../core/resource.js");
+const pagination_1 = require("../core/pagination.js");
+const path_1 = require("../internal/utils/path.js");
+class Testsets extends resource_1.APIResource {
+    /**
+     * Create a new testset for a project. The testset will be created in the project
+     * specified in the path.
+     */
+    create(projectID, body, options) {
+        return this._client.post((0, path_1.path) `/projects/${projectID}/testsets`, { body, ...options });
+    }
+    /**
+     * Update a testset. Only the fields provided in the request body will be updated.
+     * If a field is provided, the new content will replace the existing content. If a
+     * field is not provided, the existing content will remain unchanged.
+     *
+     * When updating the schema:
+     *
+     * - If field mappings are not provided and existing mappings reference fields that
+     *   no longer exist, those mappings will be automatically removed
+     * - To preserve all existing mappings, ensure all referenced fields remain in the
+     *   updated schema
+     * - For complete control, provide both schema and fieldMapping when updating the
+     *   schema
+     */
+    update(testsetID, body = {}, options) {
+        return this._client.patch((0, path_1.path) `/testsets/${testsetID}`, { body, ...options });
+    }
+    /**
+     * Retrieve a paginated list of testsets belonging to a project.
+     */
+    list(projectID, query = {}, options) {
+        return this._client.getAPIList((0, path_1.path) `/projects/${projectID}/testsets`, (pagination_1.PaginatedResponse), {
+            query,
+            ...options,
+        });
+    }
+    /**
+     * Delete testset
+     */
+    delete(testsetID, options) {
+        return this._client.delete((0, path_1.path) `/testsets/${testsetID}`, options);
+    }
+    /**
+     * Get testset by ID
+     */
+    get(testsetID, options) {
+        return this._client.get((0, path_1.path) `/testsets/${testsetID}`, options);
+    }
+}
+exports.Testsets = Testsets;
+//# sourceMappingURL=testsets.js.map

package/resources/testsets.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testsets.js","sourceRoot":"","sources":["../src/resources/testsets.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAA+C;AAE/C,sDAAkG;AAElG,oDAA8C;AAE9C,MAAa,QAAS,SAAQ,sBAAW;IACvC;;;OAGG;IACH,MAAM,CAAC,SAAiB,EAAE,IAAyB,EAAE,OAAwB;QAC3E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAA,WAAI,EAAA,aAAa,SAAS,WAAW,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,SAAiB,EACjB,OAA+C,EAAE,EACjD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAA,WAAI,EAAA,aAAa,SAAS,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAChF,CAAC;IAED;;OAEG;IACH,IAAI,CACF,SAAiB,EACjB,QAA8C,EAAE,EAChD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAA,WAAI,EAAA,aAAa,SAAS,WAAW,EAAE,CAAA,8BAA0B,CAAA,EAAE;YAChG,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,SAAiB,EAAE,OAAwB;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAA,WAAI,EAAA,aAAa,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,SAAiB,EAAE,OAAwB;QAC7C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,aAAa,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;IACjE,CAAC;CACF;AA1DD,4BA0DC"}

package/resources/testsets.mjs

@@ -0,0 +1,52 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+import { APIResource } from "../core/resource.mjs";
+import { PaginatedResponse } from "../core/pagination.mjs";
+import { path } from "../internal/utils/path.mjs";
+export class Testsets extends APIResource {
+    /**
+     * Create a new testset for a project. The testset will be created in the project
+     * specified in the path.
+     */
+    create(projectID, body, options) {
+        return this._client.post(path `/projects/${projectID}/testsets`, { body, ...options });
+    }
+    /**
+     * Update a testset. Only the fields provided in the request body will be updated.
+     * If a field is provided, the new content will replace the existing content. If a
+     * field is not provided, the existing content will remain unchanged.
+     *
+     * When updating the schema:
+     *
+     * - If field mappings are not provided and existing mappings reference fields that
+     *   no longer exist, those mappings will be automatically removed
+     * - To preserve all existing mappings, ensure all referenced fields remain in the
+     *   updated schema
+     * - For complete control, provide both schema and fieldMapping when updating the
+     *   schema
+     */
+    update(testsetID, body = {}, options) {
+        return this._client.patch(path `/testsets/${testsetID}`, { body, ...options });
+    }
+    /**
+     * Retrieve a paginated list of testsets belonging to a project.
+     */
+    list(projectID, query = {}, options) {
+        return this._client.getAPIList(path `/projects/${projectID}/testsets`, (PaginatedResponse), {
+            query,
+            ...options,
+        });
+    }
+    /**
+     * Delete testset
+     */
+    delete(testsetID, options) {
+        return this._client.delete(path `/testsets/${testsetID}`, options);
+    }
+    /**
+     * Get testset by ID
+     */
+    get(testsetID, options) {
+        return this._client.get(path `/testsets/${testsetID}`, options);
+    }
+}
+//# sourceMappingURL=testsets.mjs.map

package/resources/testsets.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"testsets.mjs","sourceRoot":"","sources":["../src/resources/testsets.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAEf,EAAe,iBAAiB,EAAgC;OAEhE,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,QAAS,SAAQ,WAAW;IACvC;;;OAGG;IACH,MAAM,CAAC,SAAiB,EAAE,IAAyB,EAAE,OAAwB;QAC3E,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,aAAa,SAAS,WAAW,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CACJ,SAAiB,EACjB,OAA+C,EAAE,EACjD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAA,aAAa,SAAS,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAChF,CAAC;IAED;;OAEG;IACH,IAAI,CACF,SAAiB,EACjB,QAA8C,EAAE,EAChD,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAA,aAAa,SAAS,WAAW,EAAE,CAAA,iBAA0B,CAAA,EAAE;YAChG,KAAK;YACL,GAAG,OAAO;SACX,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,SAAiB,EAAE,OAAwB;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAA,aAAa,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,SAAiB,EAAE,OAAwB;QAC7C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,aAAa,SAAS,EAAE,EAAE,OAAO,CAAC,CAAC;IACjE,CAAC;CACF"}

package/resources.d.mts

@@ -0,0 +1,2 @@
+export * from "./resources/index.mjs";
+//# sourceMappingURL=resources.d.mts.map

package/resources.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resources.d.mts","sourceRoot":"","sources":["src/resources.ts"],"names":[],"mappings":""}

package/resources.d.ts

@@ -0,0 +1,2 @@
+export * from "./resources/index.js";
+//# sourceMappingURL=resources.d.ts.map

package/resources.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resources.d.ts","sourceRoot":"","sources":["src/resources.ts"],"names":[],"mappings":""}

package/resources.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./resources/index.js"), exports);
+//# sourceMappingURL=resources.js.map

package/resources.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resources.js","sourceRoot":"","sources":["src/resources.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,uDAAkC"}

package/resources.mjs

@@ -0,0 +1,2 @@
+export * from "./resources/index.mjs";
+//# sourceMappingURL=resources.mjs.map

package/resources.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"resources.mjs","sourceRoot":"","sources":["src/resources.ts"],"names":[],"mappings":""}

package/src/api-promise.ts

@@ -0,0 +1,2 @@
+/** @deprecated Import from ./core/api-promise instead */
+export * from './core/api-promise';