scorecard-ai 0.1.10

package/serialization/types/UnauthorizedErrorBody.js

@@ -0,0 +1,35 @@
+"use strict";
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UnauthorizedErrorBody = void 0;
+const core = __importStar(require("../../core"));
+exports.UnauthorizedErrorBody = core.serialization.object({
+    error: core.serialization.string().optional(),
+    errorDescription: core.serialization.property("error_description", core.serialization.string().optional()),
+    statusCode: core.serialization.property("status_code", core.serialization.number().optional()),
+});

package/serialization/types/ValidationError.d.ts

@@ -0,0 +1,14 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+import * as serializers from "..";
+import * as Scorecard from "../../api";
+import * as core from "../../core";
+export declare const ValidationError: core.serialization.ObjectSchema<serializers.ValidationError.Raw, Scorecard.ValidationError>;
+export declare namespace ValidationError {
+    interface Raw {
+        loc: serializers.ValidationErrorLocItem.Raw[];
+        msg: string;
+        type: string;
+    }
+}

package/serialization/types/ValidationError.js

@@ -0,0 +1,44 @@
+"use strict";
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ValidationError = void 0;
+const core = __importStar(require("../../core"));
+exports.ValidationError = core.serialization.object({
+    loc: core.serialization.list(core.serialization.lazy(() => __awaiter(void 0, void 0, void 0, function* () { return (yield Promise.resolve().then(() => __importStar(require("..")))).ValidationErrorLocItem; }))),
+    msg: core.serialization.string(),
+    type: core.serialization.string(),
+});

package/serialization/types/ValidationErrorLocItem.d.ts

@@ -0,0 +1,10 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+import * as serializers from "..";
+import * as Scorecard from "../../api";
+import * as core from "../../core";
+export declare const ValidationErrorLocItem: core.serialization.Schema<serializers.ValidationErrorLocItem.Raw, Scorecard.ValidationErrorLocItem>;
+export declare namespace ValidationErrorLocItem {
+    type Raw = string | number;
+}

package/serialization/types/ValidationErrorLocItem.js

@@ -0,0 +1,31 @@
+"use strict";
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ValidationErrorLocItem = void 0;
+const core = __importStar(require("../../core"));
+exports.ValidationErrorLocItem = core.serialization.undiscriminatedUnion([core.serialization.string(), core.serialization.number()]);

package/serialization/types/index.d.ts

@@ -0,0 +1,39 @@
+export * from "./CreateRunParams";
+export * from "./CustomSchemaInput";
+export * from "./CustomSchemaOutput";
+export * from "./CustomVariable";
+export * from "./DataTypeEnum";
+export * from "./FileUrl";
+export * from "./HttpValidationError";
+export * from "./NotFoundErrorBody";
+export * from "./PaginatedTestcaseResponse";
+export * from "./RoleEnum";
+export * from "./RunExternal";
+export * from "./RunStatus";
+export * from "./TestCaseCustomInputsValue";
+export * from "./TestCaseCustomLabelsValue";
+export * from "./TestCase";
+export * from "./TestCaseCreate";
+export * from "./TestRecordCreate";
+export * from "./TestSetCreate";
+export * from "./TestrecordCustomInputsValue";
+export * from "./TestrecordCustomLabelsValue";
+export * from "./TestrecordCustomOutputsValue";
+export * from "./Testrecord";
+export * from "./Testset";
+export * from "./UnauthenticatedError";
+export * from "./UnauthorizedErrorBody";
+export * from "./ValidationErrorLocItem";
+export * from "./ValidationError";
+export * from "./BodyStartGithubWorkflowStartGithubWorkflowPost";
+export * from "./ExecutionParams";
+export * from "./ModelParams";
+export * from "./ScoreExecutionParams";
+export * from "./ScoringParams";
+export * from "./TestCaseCreateInput";
+export * from "./AppCreateRunParams";
+export * from "./AppTestRecordCreate";
+export * from "./AppTestSetCreate";
+export * from "./LibDtoTypesCreateRunParams";
+export * from "./LibDtoTypesTestRecordCreate";
+export * from "./LibDtoTypesTestSetCreate";

package/serialization/types/index.js

@@ -0,0 +1,55 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./CreateRunParams"), exports);
+__exportStar(require("./CustomSchemaInput"), exports);
+__exportStar(require("./CustomSchemaOutput"), exports);
+__exportStar(require("./CustomVariable"), exports);
+__exportStar(require("./DataTypeEnum"), exports);
+__exportStar(require("./FileUrl"), exports);
+__exportStar(require("./HttpValidationError"), exports);
+__exportStar(require("./NotFoundErrorBody"), exports);
+__exportStar(require("./PaginatedTestcaseResponse"), exports);
+__exportStar(require("./RoleEnum"), exports);
+__exportStar(require("./RunExternal"), exports);
+__exportStar(require("./RunStatus"), exports);
+__exportStar(require("./TestCaseCustomInputsValue"), exports);
+__exportStar(require("./TestCaseCustomLabelsValue"), exports);
+__exportStar(require("./TestCase"), exports);
+__exportStar(require("./TestCaseCreate"), exports);
+__exportStar(require("./TestRecordCreate"), exports);
+__exportStar(require("./TestSetCreate"), exports);
+__exportStar(require("./TestrecordCustomInputsValue"), exports);
+__exportStar(require("./TestrecordCustomLabelsValue"), exports);
+__exportStar(require("./TestrecordCustomOutputsValue"), exports);
+__exportStar(require("./Testrecord"), exports);
+__exportStar(require("./Testset"), exports);
+__exportStar(require("./UnauthenticatedError"), exports);
+__exportStar(require("./UnauthorizedErrorBody"), exports);
+__exportStar(require("./ValidationErrorLocItem"), exports);
+__exportStar(require("./ValidationError"), exports);
+__exportStar(require("./BodyStartGithubWorkflowStartGithubWorkflowPost"), exports);
+__exportStar(require("./ExecutionParams"), exports);
+__exportStar(require("./ModelParams"), exports);
+__exportStar(require("./ScoreExecutionParams"), exports);
+__exportStar(require("./ScoringParams"), exports);
+__exportStar(require("./TestCaseCreateInput"), exports);
+__exportStar(require("./AppCreateRunParams"), exports);
+__exportStar(require("./AppTestRecordCreate"), exports);
+__exportStar(require("./AppTestSetCreate"), exports);
+__exportStar(require("./LibDtoTypesCreateRunParams"), exports);
+__exportStar(require("./LibDtoTypesTestRecordCreate"), exports);
+__exportStar(require("./LibDtoTypesTestSetCreate"), exports);

package/wrapper/ScorecardClient.d.ts

@@ -0,0 +1,32 @@
+import { ScorecardClient as FernClient } from "../Client";
+import * as core from "../core";
+export declare namespace ScorecardClient {
+    interface Options {
+        /**
+         * Your Scorecard API Key. Defaults to the environment
+         * variable SCORECARD_API_KEY.
+         */
+        apiKey?: core.Supplier<string>;
+    }
+    interface RunTestArgs {
+        /**
+         * The ID of the testset you want to run.
+         */
+        inputTestsetId: number;
+        /**
+         * The ID of the scoring config you want to use.
+         */
+        scoringConfigId: number;
+        /**
+         * A function that will call your AI model with a prompt.
+         */
+        modelInvocation: (prompt: string) => Promise<any>;
+    }
+}
+export declare class ScorecardClient extends FernClient {
+    constructor(options?: ScorecardClient.Options);
+    /**
+     * Runs all tests within a testset
+     */
+    runTests({ inputTestsetId, scoringConfigId, modelInvocation, }: ScorecardClient.RunTestArgs): Promise<void>;
+}

package/wrapper/ScorecardClient.js

@@ -0,0 +1,86 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScorecardClient = void 0;
+const Client_1 = require("../Client");
+const errors = __importStar(require("../errors"));
+class ScorecardClient extends Client_1.ScorecardClient {
+    constructor(options = {}) {
+        var _a;
+        const apiKey = (_a = options.apiKey) !== null && _a !== void 0 ? _a : process.env["SCORECARD_API_KEY"];
+        if (apiKey == null) {
+            throw new errors.ScorecardError({
+                message: "Please pass in your Scorecard API Key or export SCORECARD_API_KEY in your environment.",
+            });
+        }
+        super({
+            apiKey,
+        });
+    }
+    /**
+     * Runs all tests within a testset
+     */
+    runTests({ inputTestsetId, scoringConfigId, modelInvocation, }) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const run = yield this.run.create({
+                testsetId: inputTestsetId,
+                scoringConfigId,
+            });
+            if (run.id == null) {
+                throw new errors.ScorecardError({
+                    message: `Didn't receive run id after creating run for testid=${inputTestsetId}`,
+                });
+            }
+            const testcases = yield this.testset.getTestcases(inputTestsetId);
+            for (const testcase of testcases.results) {
+                if (testcase.id == null) {
+                    continue;
+                }
+                console.log(`Running testcase ${testcase.id}...`);
+                console.log(`User query: ${testcase.userQuery}...`);
+                const modelResponse = yield modelInvocation(testcase.userQuery);
+                this.testrecord.create(run.id, {
+                    testcaseId: testcase.id,
+                    testsetId: inputTestsetId,
+                    userQuery: testcase.userQuery,
+                    context: testcase.context,
+                    ideal: testcase.ideal,
+                    response: modelResponse,
+                });
+            }
+            console.log("Finished running testcases...");
+        });
+    }
+}
+exports.ScorecardClient = ScorecardClient;