scorecard-ai 0.6.1 → 1.0.0-alpha.0

package/serialization/types/UnauthenticatedError.js

@@ -1,35 +0,0 @@
-"use strict";
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.UnauthenticatedError = void 0;
-const core = __importStar(require("../../core"));
-exports.UnauthenticatedError = core.serialization.object({
-    error: core.serialization.string().optional(),
-    errorDescription: core.serialization.property("error_description", core.serialization.string().optional()),
-    statusCode: core.serialization.property("status_code", core.serialization.number().optional()),
-});

package/serialization/types/UnauthorizedErrorBody.d.ts

@@ -1,14 +0,0 @@
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-import * as serializers from "../index";
-import * as Scorecard from "../../api/index";
-import * as core from "../../core";
-export declare const UnauthorizedErrorBody: core.serialization.ObjectSchema<serializers.UnauthorizedErrorBody.Raw, Scorecard.UnauthorizedErrorBody>;
-export declare namespace UnauthorizedErrorBody {
-    interface Raw {
-        error?: string | null;
-        error_description?: string | null;
-        status_code?: number | null;
-    }
-}

package/serialization/types/UnauthorizedErrorBody.js

@@ -1,35 +0,0 @@
-"use strict";
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.UnauthorizedErrorBody = void 0;
-const core = __importStar(require("../../core"));
-exports.UnauthorizedErrorBody = core.serialization.object({
-    error: core.serialization.string().optional(),
-    errorDescription: core.serialization.property("error_description", core.serialization.string().optional()),
-    statusCode: core.serialization.property("status_code", core.serialization.number().optional()),
-});

package/serialization/types/ValidationError.d.ts

@@ -1,15 +0,0 @@
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-import * as serializers from "../index";
-import * as Scorecard from "../../api/index";
-import * as core from "../../core";
-import { ValidationErrorLocItem } from "./ValidationErrorLocItem";
-export declare const ValidationError: core.serialization.ObjectSchema<serializers.ValidationError.Raw, Scorecard.ValidationError>;
-export declare namespace ValidationError {
-    interface Raw {
-        loc: ValidationErrorLocItem.Raw[];
-        msg: string;
-        type: string;
-    }
-}

package/serialization/types/ValidationError.js

@@ -1,36 +0,0 @@
-"use strict";
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ValidationError = void 0;
-const core = __importStar(require("../../core"));
-const ValidationErrorLocItem_1 = require("./ValidationErrorLocItem");
-exports.ValidationError = core.serialization.object({
-    loc: core.serialization.list(ValidationErrorLocItem_1.ValidationErrorLocItem),
-    msg: core.serialization.string(),
-    type: core.serialization.string(),
-});

package/serialization/types/ValidationErrorLocItem.d.ts

@@ -1,10 +0,0 @@
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-import * as serializers from "../index";
-import * as Scorecard from "../../api/index";
-import * as core from "../../core";
-export declare const ValidationErrorLocItem: core.serialization.Schema<serializers.ValidationErrorLocItem.Raw, Scorecard.ValidationErrorLocItem>;
-export declare namespace ValidationErrorLocItem {
-    type Raw = string | number;
-}

package/serialization/types/ValidationErrorLocItem.js

@@ -1,31 +0,0 @@
-"use strict";
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ValidationErrorLocItem = void 0;
-const core = __importStar(require("../../core"));
-exports.ValidationErrorLocItem = core.serialization.undiscriminatedUnion([core.serialization.string(), core.serialization.number()]);

package/serialization/types/index.d.ts

@@ -1,44 +0,0 @@
-export * from "./CustomSchema";
-export * from "./CustomVariable";
-export * from "./DataTypeEnum";
-export * from "./FileUrl";
-export * from "./Grade";
-export * from "./HttpValidationError";
-export * from "./IngestionMethod";
-export * from "./JsonObjectInputValue";
-export * from "./JsonObject";
-export * from "./JsonObjectOutputValue";
-export * from "./NotFoundErrorBody";
-export * from "./PaginatedTestcaseResponse";
-export * from "./PromptModelParamsValue";
-export * from "./Prompt";
-export * from "./PromptCursorPage";
-export * from "./RoleEnum";
-export * from "./Run";
-export * from "./RunMetric";
-export * from "./RunStatus";
-export * from "./ScoreStatus";
-export * from "./ScoringConfig";
-export * from "./Span";
-export * from "./TestCaseCustomInputsValue";
-export * from "./TestCaseCustomLabelsValue";
-export * from "./TestCase";
-export * from "./TestcaseBatchDeletionResponse";
-export * from "./TestcaseDeletionResponse";
-export * from "./TestrecordCustomInputsValue";
-export * from "./TestrecordCustomLabelsValue";
-export * from "./TestrecordCustomOutputsValue";
-export * from "./TestrecordModelParamsValue";
-export * from "./TestrecordModelDebugInfoValue";
-export * from "./Testrecord";
-export * from "./Testset";
-export * from "./TestsetCursorPage";
-export * from "./Trace";
-export * from "./UnauthenticatedError";
-export * from "./UnauthorizedErrorBody";
-export * from "./ValidationErrorLocItem";
-export * from "./ValidationError";
-export * from "./CreateGithubWorkflowParams";
-export * from "./CreateRunParams";
-export * from "./TestRecordCreate";
-export * from "./TestSetCreate";

package/serialization/types/index.js

@@ -1,60 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./CustomSchema"), exports);
-__exportStar(require("./CustomVariable"), exports);
-__exportStar(require("./DataTypeEnum"), exports);
-__exportStar(require("./FileUrl"), exports);
-__exportStar(require("./Grade"), exports);
-__exportStar(require("./HttpValidationError"), exports);
-__exportStar(require("./IngestionMethod"), exports);
-__exportStar(require("./JsonObjectInputValue"), exports);
-__exportStar(require("./JsonObject"), exports);
-__exportStar(require("./JsonObjectOutputValue"), exports);
-__exportStar(require("./NotFoundErrorBody"), exports);
-__exportStar(require("./PaginatedTestcaseResponse"), exports);
-__exportStar(require("./PromptModelParamsValue"), exports);
-__exportStar(require("./Prompt"), exports);
-__exportStar(require("./PromptCursorPage"), exports);
-__exportStar(require("./RoleEnum"), exports);
-__exportStar(require("./Run"), exports);
-__exportStar(require("./RunMetric"), exports);
-__exportStar(require("./RunStatus"), exports);
-__exportStar(require("./ScoreStatus"), exports);
-__exportStar(require("./ScoringConfig"), exports);
-__exportStar(require("./Span"), exports);
-__exportStar(require("./TestCaseCustomInputsValue"), exports);
-__exportStar(require("./TestCaseCustomLabelsValue"), exports);
-__exportStar(require("./TestCase"), exports);
-__exportStar(require("./TestcaseBatchDeletionResponse"), exports);
-__exportStar(require("./TestcaseDeletionResponse"), exports);
-__exportStar(require("./TestrecordCustomInputsValue"), exports);
-__exportStar(require("./TestrecordCustomLabelsValue"), exports);
-__exportStar(require("./TestrecordCustomOutputsValue"), exports);
-__exportStar(require("./TestrecordModelParamsValue"), exports);
-__exportStar(require("./TestrecordModelDebugInfoValue"), exports);
-__exportStar(require("./Testrecord"), exports);
-__exportStar(require("./Testset"), exports);
-__exportStar(require("./TestsetCursorPage"), exports);
-__exportStar(require("./Trace"), exports);
-__exportStar(require("./UnauthenticatedError"), exports);
-__exportStar(require("./UnauthorizedErrorBody"), exports);
-__exportStar(require("./ValidationErrorLocItem"), exports);
-__exportStar(require("./ValidationError"), exports);
-__exportStar(require("./CreateGithubWorkflowParams"), exports);
-__exportStar(require("./CreateRunParams"), exports);
-__exportStar(require("./TestRecordCreate"), exports);
-__exportStar(require("./TestSetCreate"), exports);

package/telemetry.d.ts

@@ -1 +0,0 @@
-declare function checkInstalled(...modules: string[]): boolean;

package/telemetry.js

@@ -1,83 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-function checkInstalled(...modules) {
-    for (let i = 0; i < modules.length; i++) {
-        let module = modules[i];
-        try {
-            require.resolve(module);
-        }
-        catch (e) {
-            return false;
-        }
-    }
-    return true;
-}
-module.exports = {
-    setup(name, scorecardConfig, debug = false) {
-        const required = [
-            '@opentelemetry/api',
-            '@opentelemetry/core',
-            '@opentelemetry/exporter-trace-otlp-http',
-            '@opentelemetry/resources',
-            '@opentelemetry/sdk-trace-node',
-            '@opentelemetry/semantic-conventions'
-        ];
-        if (!checkInstalled(...required)) {
-            throw new Error(`Required OpenTelemetry packages not installed. Please ensure that all necessary peerDependencies are installed:\n${required.map(module => `- ${module}`).join('\n')}`);
-        }
-        const { propagation, trace } = require('@opentelemetry/api');
-        const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = require('@opentelemetry/core');
-        const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
-        const { Resource } = require('@opentelemetry/resources');
-        const { BatchSpanProcessor, ConsoleSpanExporter, NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
-        const { SEMRESATTRS_SERVICE_NAME } = require('@opentelemetry/semantic-conventions');
-        const resource = Resource.default().merge(new Resource({
-            [SEMRESATTRS_SERVICE_NAME]: name,
-        }));
-        const provider = new NodeTracerProvider({
-            resource,
-        });
-        if (debug) {
-            const consoleExporter = new ConsoleSpanExporter();
-            const processor = new BatchSpanProcessor(consoleExporter);
-            provider.addSpanProcessor(processor);
-        }
-        propagation.setGlobalPropagator(new CompositePropagator({
-            propagators: [
-                new W3CTraceContextPropagator(),
-                new W3CBaggagePropagator()
-            ]
-        }));
-        const url = scorecardConfig.telemetryUrl ? scorecardConfig.telemetryUrl : 'https://telemetry.getscorecard.ai';
-        const otlpExporter = new OTLPTraceExporter({
-            url: `${url}/v1/traces`,
-            headers: {
-                'Authorization': `Bearer ${scorecardConfig.telemetryKey}`
-            },
-        });
-        const processor = new BatchSpanProcessor(otlpExporter);
-        provider.addSpanProcessor(processor);
-        provider.register();
-        if (checkInstalled('@langchain/core', '@arizeai/openinference-instrumentation-langchain', '@opentelemetry/instrumentation')) {
-            const { LangChainInstrumentation } = require('@arizeai/openinference-instrumentation-langchain');
-            new LangChainInstrumentation().manuallyInstrument(require('@langchain/core/callbacks/manager'));
-        }
-        if (checkInstalled('openai', '@arizeai/openinference-instrumentation-openai', '@opentelemetry/instrumentation')) {
-            const { OpenAIInstrumentation } = require('@arizeai/openinference-instrumentation-openai');
-            new OpenAIInstrumentation().manuallyInstrument(require('openai'));
-        }
-        const tracer = trace.getTracer();
-        process.on('beforeExit', () => __awaiter(this, void 0, void 0, function* () {
-            yield provider.shutdown();
-        }));
-        return tracer;
-    }
-};

package/wrapper/ScorecardClient.d.ts

@@ -1,28 +0,0 @@
-import { ScorecardClient as FernClient } from "../Client";
-export declare namespace ScorecardClient {
-    interface Options extends Omit<FernClient.Options, "fetcher"> {
-    }
-    interface RequestOptions extends FernClient.RequestOptions {
-    }
-    interface RunTestArgs {
-        /**
-         * The ID of the testset you want to run.
-         */
-        inputTestsetId: number;
-        /**
-         * The ID of the scoring config you want to use.
-         */
-        scoringConfigId: number;
-        /**
-         * A function that will call your AI model with a prompt.
-         */
-        modelInvocation: (prompt: string) => Promise<any>;
-    }
-}
-export declare class ScorecardClient extends FernClient {
-    constructor(options: ScorecardClient.Options);
-    /**
-     * Runs all tests within a testset
-     */
-    runTests({ inputTestsetId, scoringConfigId, modelInvocation, }: ScorecardClient.RunTestArgs): Promise<void>;
-}

package/wrapper/ScorecardClient.js

@@ -1,85 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ScorecardClient = void 0;
-const api_1 = require("@opentelemetry/api");
-const Client_1 = require("../Client");
-const core_1 = require("../core");
-const errors = __importStar(require("../errors"));
-class ScorecardClient extends Client_1.ScorecardClient {
-    constructor(options) {
-        const fetcher = (args) => {
-            var _a;
-            args.headers = (_a = args.headers) !== null && _a !== void 0 ? _a : {};
-            api_1.propagation.inject(api_1.context.active(), args.headers);
-            return (0, core_1.fetcher)(args);
-        };
-        super(Object.assign(Object.assign({}, options), { fetcher }));
-    }
-    /**
-     * Runs all tests within a testset
-     */
-    runTests({ inputTestsetId, scoringConfigId, modelInvocation, }) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const run = yield this.run.create({
-                testsetId: inputTestsetId,
-                scoringConfigId,
-            });
-            if (run.id == null) {
-                throw new errors.ScorecardError({
-                    message: `Didn't receive run id after creating run for testid=${inputTestsetId}`,
-                });
-            }
-            const testcases = yield this.testset.getTestcases(inputTestsetId);
-            for (const testcase of testcases.results) {
-                if (testcase.id == null) {
-                    continue;
-                }
-                console.log(`Running testcase ${testcase.id}...`);
-                console.log(`User query: ${testcase.userQuery}...`);
-                const modelResponse = yield modelInvocation(testcase.userQuery);
-                this.testrecord.create(run.id, {
-                    testcaseId: testcase.id,
-                    testsetId: inputTestsetId,
-                    userQuery: testcase.userQuery,
-                    context: testcase.context,
-                    ideal: testcase.ideal,
-                    response: modelResponse,
-                });
-            }
-            console.log("Finished running testcases...");
-        });
-    }
-}
-exports.ScorecardClient = ScorecardClient;