scorecard-ai 0.1.10

package/Client.d.ts

@@ -0,0 +1,31 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+import * as environments from "./environments";
+import * as core from "./core";
+import { Testset } from "./api/resources/testset/client/Client";
+import { Testcase } from "./api/resources/testcase/client/Client";
+import { Testrecord } from "./api/resources/testrecord/client/Client";
+import { Run } from "./api/resources/run/client/Client";
+export declare namespace ScorecardClient {
+    interface Options {
+        environment?: core.Supplier<environments.ScorecardEnvironment | string>;
+        apiKey: core.Supplier<string>;
+    }
+    interface RequestOptions {
+        timeoutInSeconds?: number;
+        maxRetries?: number;
+    }
+}
+export declare class ScorecardClient {
+    protected readonly _options: ScorecardClient.Options;
+    constructor(_options: ScorecardClient.Options);
+    protected _testset: Testset | undefined;
+    get testset(): Testset;
+    protected _testcase: Testcase | undefined;
+    get testcase(): Testcase;
+    protected _testrecord: Testrecord | undefined;
+    get testrecord(): Testrecord;
+    protected _run: Run | undefined;
+    get run(): Run;
+}

package/Client.js

@@ -0,0 +1,32 @@
+"use strict";
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScorecardClient = void 0;
+const Client_1 = require("./api/resources/testset/client/Client");
+const Client_2 = require("./api/resources/testcase/client/Client");
+const Client_3 = require("./api/resources/testrecord/client/Client");
+const Client_4 = require("./api/resources/run/client/Client");
+class ScorecardClient {
+    constructor(_options) {
+        this._options = _options;
+    }
+    get testset() {
+        var _a;
+        return ((_a = this._testset) !== null && _a !== void 0 ? _a : (this._testset = new Client_1.Testset(this._options)));
+    }
+    get testcase() {
+        var _a;
+        return ((_a = this._testcase) !== null && _a !== void 0 ? _a : (this._testcase = new Client_2.Testcase(this._options)));
+    }
+    get testrecord() {
+        var _a;
+        return ((_a = this._testrecord) !== null && _a !== void 0 ? _a : (this._testrecord = new Client_3.Testrecord(this._options)));
+    }
+    get run() {
+        var _a;
+        return ((_a = this._run) !== null && _a !== void 0 ? _a : (this._run = new Client_4.Run(this._options)));
+    }
+}
+exports.ScorecardClient = ScorecardClient;

package/README.md

@@ -0,0 +1,99 @@
+# Scorecard AI Node Library
+
+[![fern shield](https://img.shields.io/badge/%F0%9F%8C%BF-SDK%20generated%20by%20Fern-brightgreen)](https://github.com/fern-api/fern)
+
+## Installation
+
+```bash
+npm install scorecard-ai
+# or
+yarn add scorecard-ai
+```
+
+## Usage
+Instantiate our client and start making calls to the API. 
+The SDK uses fetch and is compatible in multiple environments
+such as Node, Vercel, Cloudflare Workers. 
+
+```typescript
+import { ScorecardClient } from "scorecard";
+
+const scorecard = new ScorecardClient(
+  api_key="YOUR_API_KEY" // Defaults to SCORECARD_API_KEY
+)
+
+const testset = await scorecard.testset.create({
+  testsetId=1234, 
+  userQuery="Your prompt...", 
+  modelParams: {
+    "param": "value"
+  }
+})
+
+console.log("Created testset", testset)
+```
+
+## Running Tests
+The `runTests` method will load your test cases, 
+invoke your models with saved prompts, 
+and record success and failure. This is 
+available on both the sync and async client.  
+
+```typescript
+import { callModel } from "../app.ts";
+import { ScorecardClient } from "scorecard";
+
+const scorecard = new ScorecardClient(
+  apiKey="YOUR_API_KEY"
+)
+
+client.runTests({
+  inputTestsetId: 123,
+  scoringConfigId: 456,
+  // The model invocation that you would like to test
+  model_invocation: () => callModel(prompt),
+})
+```
+
+## Timeouts
+By default, the client is configured to have 
+a timeout of 60 seconds. You can customize 
+this value by passing in request options. 
+
+```typescript
+await scorecard.testset.create({
+    testsetId=1234, 
+    ...
+  }, 
+  {
+    timeoutInSeconds: 30, // override timeout to 30s
+  });
+```
+
+## Handling Exceptions
+All exceptions thrown by the SDK will 
+sublcass [ScorecardError](./src/errors/ScorecardError.ts). 
+
+```typescript
+import { Scorecard, ScorecardError } from "scorecard";
+
+try {
+  await client.testset.get({...});
+} catch(err) {
+  if (err instanceof Scorecard.UnprocessableEntityError) {
+    // handle unprocessable entity error
+  } else if (err instanceof ScorecardError) {
+    // handle any sdk related error
+  }
+}
+```
+
+## Beta status
+
+This SDK is in beta, and there may be breaking changes between versions without a major version update. Therefore, we recommend pinning the package version to a specific version in your pyproject.toml file. This way, you can install the same version each time without breaking changes unless you are intentionally looking for the latest version.
+
+## Contributing
+
+While we value open-source contributions to this SDK, this library is generated programmatically. Additions made directly to this library would have to be moved over to our generation code, otherwise they would be overwritten upon the next generated release. Feel free to open a PR as a proof of concept, but know that we will not be able to merge it as-is. We suggest opening an issue first to discuss with us!
+
+On the other hand, contributions to the README are always very welcome!

package/api/errors/ForbiddenError.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+import * as errors from "../../errors";
+import * as Scorecard from "..";
+export declare class ForbiddenError extends errors.ScorecardError {
+    constructor(body: Scorecard.UnauthorizedErrorBody);
+}

package/api/errors/ForbiddenError.js

@@ -0,0 +1,41 @@
+"use strict";
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ForbiddenError = void 0;
+const errors = __importStar(require("../../errors"));
+class ForbiddenError extends errors.ScorecardError {
+    constructor(body) {
+        super({
+            message: "ForbiddenError",
+            statusCode: 403,
+            body: body,
+        });
+        Object.setPrototypeOf(this, ForbiddenError.prototype);
+    }
+}
+exports.ForbiddenError = ForbiddenError;

package/api/errors/NotFoundError.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+import * as errors from "../../errors";
+import * as Scorecard from "..";
+export declare class NotFoundError extends errors.ScorecardError {
+    constructor(body: Scorecard.NotFoundErrorBody);
+}

package/api/errors/NotFoundError.js

@@ -0,0 +1,41 @@
+"use strict";
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NotFoundError = void 0;
+const errors = __importStar(require("../../errors"));
+class NotFoundError extends errors.ScorecardError {
+    constructor(body) {
+        super({
+            message: "NotFoundError",
+            statusCode: 404,
+            body: body,
+        });
+        Object.setPrototypeOf(this, NotFoundError.prototype);
+    }
+}
+exports.NotFoundError = NotFoundError;

package/api/errors/UnauthorizedError.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+import * as errors from "../../errors";
+import * as Scorecard from "..";
+export declare class UnauthorizedError extends errors.ScorecardError {
+    constructor(body: Scorecard.UnauthenticatedError);
+}

package/api/errors/UnauthorizedError.js

@@ -0,0 +1,41 @@
+"use strict";
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UnauthorizedError = void 0;
+const errors = __importStar(require("../../errors"));
+class UnauthorizedError extends errors.ScorecardError {
+    constructor(body) {
+        super({
+            message: "UnauthorizedError",
+            statusCode: 401,
+            body: body,
+        });
+        Object.setPrototypeOf(this, UnauthorizedError.prototype);
+    }
+}
+exports.UnauthorizedError = UnauthorizedError;

package/api/errors/UnprocessableEntityError.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+import * as errors from "../../errors";
+import * as Scorecard from "..";
+export declare class UnprocessableEntityError extends errors.ScorecardError {
+    constructor(body: Scorecard.HttpValidationError);
+}

package/api/errors/UnprocessableEntityError.js

@@ -0,0 +1,41 @@
+"use strict";
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UnprocessableEntityError = void 0;
+const errors = __importStar(require("../../errors"));
+class UnprocessableEntityError extends errors.ScorecardError {
+    constructor(body) {
+        super({
+            message: "UnprocessableEntityError",
+            statusCode: 422,
+            body: body,
+        });
+        Object.setPrototypeOf(this, UnprocessableEntityError.prototype);
+    }
+}
+exports.UnprocessableEntityError = UnprocessableEntityError;

package/api/errors/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./UnauthorizedError";
+export * from "./ForbiddenError";
+export * from "./NotFoundError";
+export * from "./UnprocessableEntityError";

package/api/errors/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./UnauthorizedError"), exports);
+__exportStar(require("./ForbiddenError"), exports);
+__exportStar(require("./NotFoundError"), exports);
+__exportStar(require("./UnprocessableEntityError"), exports);

package/api/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./resources";
+export * from "./types";
+export * from "./errors";

package/api/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./resources"), exports);
+__exportStar(require("./types"), exports);
+__exportStar(require("./errors"), exports);

package/api/resources/index.d.ts

@@ -0,0 +1,10 @@
+export * as testcase from "./testcase";
+export * from "./testcase/types";
+export * as testrecord from "./testrecord";
+export * from "./testrecord/types";
+export * as testset from "./testset";
+export * as run from "./run";
+export * from "./testset/client/requests";
+export * from "./testcase/client/requests";
+export * from "./testrecord/client/requests";
+export * from "./run/client/requests";

package/api/resources/index.js

@@ -0,0 +1,39 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.run = exports.testset = exports.testrecord = exports.testcase = void 0;
+exports.testcase = __importStar(require("./testcase"));
+__exportStar(require("./testcase/types"), exports);
+exports.testrecord = __importStar(require("./testrecord"));
+__exportStar(require("./testrecord/types"), exports);
+exports.testset = __importStar(require("./testset"));
+exports.run = __importStar(require("./run"));
+__exportStar(require("./testset/client/requests"), exports);
+__exportStar(require("./testcase/client/requests"), exports);
+__exportStar(require("./testrecord/client/requests"), exports);
+__exportStar(require("./run/client/requests"), exports);

package/api/resources/run/client/Client.d.ts

@@ -0,0 +1,52 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+import * as environments from "../../../../environments";
+import * as core from "../../../../core";
+import * as Scorecard from "../../..";
+export declare namespace Run {
+    interface Options {
+        environment?: core.Supplier<environments.ScorecardEnvironment | string>;
+        apiKey: core.Supplier<string>;
+    }
+    interface RequestOptions {
+        timeoutInSeconds?: number;
+        maxRetries?: number;
+    }
+}
+export declare class Run {
+    protected readonly _options: Run.Options;
+    constructor(_options: Run.Options);
+    /**
+     * Create a new Run
+     * @throws {@link Scorecard.UnauthorizedError}
+     * @throws {@link Scorecard.ForbiddenError}
+     * @throws {@link Scorecard.NotFoundError}
+     * @throws {@link Scorecard.UnprocessableEntityError}
+     */
+    create(request: Scorecard.RunCreateParams, requestOptions?: Run.RequestOptions): Promise<Scorecard.RunExternal>;
+    /**
+     * Retrieve a Run metadata
+     * @throws {@link Scorecard.UnauthorizedError}
+     * @throws {@link Scorecard.ForbiddenError}
+     * @throws {@link Scorecard.NotFoundError}
+     * @throws {@link Scorecard.UnprocessableEntityError}
+     *
+     * @example
+     *     await scorecard.run.get(1)
+     */
+    get(runId: number, requestOptions?: Run.RequestOptions): Promise<Scorecard.RunExternal>;
+    /**
+     * Update the status of a run.
+     * @throws {@link Scorecard.UnauthorizedError}
+     * @throws {@link Scorecard.ForbiddenError}
+     * @throws {@link Scorecard.NotFoundError}
+     * @throws {@link Scorecard.UnprocessableEntityError}
+     *
+     * @example
+     *     await scorecard.run.updateStatus(1, {
+     *         status: Scorecard.RunStatus.Pending
+     *     })
+     */
+    updateStatus(runId: number, request: Scorecard.UpdateStatusParams, requestOptions?: Run.RequestOptions): Promise<Scorecard.RunExternal>;
+}