scorecard-ai 0.5.3 → 0.6.0

package/dist/serialization/types/TestCaseCustomLabelsValue.js

@@ -25,22 +25,14 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TestCaseCustomLabelsValue = void 0;
+const serializers = __importStar(require("../index"));
 const core = __importStar(require("../../core"));
 const FileUrl_1 = require("./FileUrl");
 exports.TestCaseCustomLabelsValue = core.serialization.undiscriminatedUnion([
     FileUrl_1.FileUrl,
-    core.serialization.lazyObject(() => __awaiter(void 0, void 0, void 0, function* () { return (yield Promise.resolve().then(() => __importStar(require("..")))).JsonObject; })),
+    core.serialization.lazyObject(() => serializers.JsonObject),
     core.serialization.string(),
     core.serialization.number(),
     core.serialization.number(),

package/dist/serialization/types/TestcaseBatchDeletionResponse.d.ts

@@ -0,0 +1,13 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+import * as serializers from "../index";
+import * as Scorecard from "../../api/index";
+import * as core from "../../core";
+export declare const TestcaseBatchDeletionResponse: core.serialization.ObjectSchema<serializers.TestcaseBatchDeletionResponse.Raw, Scorecard.TestcaseBatchDeletionResponse>;
+export declare namespace TestcaseBatchDeletionResponse {
+    interface Raw {
+        ids: number[];
+        detail: string;
+    }
+}

package/dist/serialization/types/{TestCaseCreate.js → TestcaseBatchDeletionResponse.js}

@@ -26,12 +26,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TestCaseCreate = void 0;
+exports.TestcaseBatchDeletionResponse = void 0;
 const core = __importStar(require("../../core"));
-exports.TestCaseCreate = core.serialization.object({
-    testsetId: core.serialization.property("testset_id", core.serialization.number()),
-    userQuery: core.serialization.property("user_query", core.serialization.string()),
-    context: core.serialization.string().optional(),
-    response: core.serialization.string().optional(),
-    ideal: core.serialization.string().optional(),
+exports.TestcaseBatchDeletionResponse = core.serialization.object({
+    ids: core.serialization.list(core.serialization.number()),
+    detail: core.serialization.string(),
 });

package/dist/serialization/types/Testrecord.d.ts

@@ -28,5 +28,6 @@ export declare namespace Testrecord {
         prompt?: string | null;
         model_params?: Record<string, TestrecordModelParamsValue.Raw | null | undefined> | null;
         model_debug_info?: Record<string, TestrecordModelDebugInfoValue.Raw | null | undefined> | null;
+        error_message?: string | null;
     }
 }

package/dist/serialization/types/Testrecord.js

@@ -50,4 +50,5 @@ exports.Testrecord = core.serialization.object({
     prompt: core.serialization.string().optional(),
     modelParams: core.serialization.property("model_params", core.serialization.record(core.serialization.string(), TestrecordModelParamsValue_1.TestrecordModelParamsValue.optional()).optional()),
     modelDebugInfo: core.serialization.property("model_debug_info", core.serialization.record(core.serialization.string(), TestrecordModelDebugInfoValue_1.TestrecordModelDebugInfoValue.optional()).optional()),
+    errorMessage: core.serialization.property("error_message", core.serialization.string().optional()),
 });

package/dist/serialization/types/TestrecordCustomInputsValue.js

@@ -25,21 +25,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TestrecordCustomInputsValue = void 0;
+const serializers = __importStar(require("../index"));
 const core = __importStar(require("../../core"));
 const FileUrl_1 = require("./FileUrl");
 exports.TestrecordCustomInputsValue = core.serialization.undiscriminatedUnion([
     core.serialization.string(),
     FileUrl_1.FileUrl,
-    core.serialization.lazyObject(() => __awaiter(void 0, void 0, void 0, function* () { return (yield Promise.resolve().then(() => __importStar(require("..")))).JsonObject; })),
+    core.serialization.lazyObject(() => serializers.JsonObject),
 ]);

package/dist/serialization/types/TestrecordCustomLabelsValue.js

@@ -25,21 +25,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TestrecordCustomLabelsValue = void 0;
+const serializers = __importStar(require("../index"));
 const core = __importStar(require("../../core"));
 const FileUrl_1 = require("./FileUrl");
 exports.TestrecordCustomLabelsValue = core.serialization.undiscriminatedUnion([
     core.serialization.string(),
     FileUrl_1.FileUrl,
-    core.serialization.lazyObject(() => __awaiter(void 0, void 0, void 0, function* () { return (yield Promise.resolve().then(() => __importStar(require("..")))).JsonObject; })),
+    core.serialization.lazyObject(() => serializers.JsonObject),
 ]);

package/dist/serialization/types/TestrecordCustomOutputsValue.js

@@ -25,21 +25,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TestrecordCustomOutputsValue = void 0;
+const serializers = __importStar(require("../index"));
 const core = __importStar(require("../../core"));
 const FileUrl_1 = require("./FileUrl");
 exports.TestrecordCustomOutputsValue = core.serialization.undiscriminatedUnion([
     core.serialization.string(),
     FileUrl_1.FileUrl,
-    core.serialization.lazyObject(() => __awaiter(void 0, void 0, void 0, function* () { return (yield Promise.resolve().then(() => __importStar(require("..")))).JsonObject; })),
+    core.serialization.lazyObject(() => serializers.JsonObject),
 ]);

package/dist/serialization/types/Trace.d.ts

@@ -10,5 +10,6 @@ export declare namespace Trace {
         TraceId: string;
         Start: string;
         End: string;
+        Spans?: serializers.Span.Raw[] | null;
     }
 }

package/dist/serialization/types/Trace.js

@@ -27,9 +27,11 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Trace = void 0;
+const serializers = __importStar(require("../index"));
 const core = __importStar(require("../../core"));
 exports.Trace = core.serialization.object({
     traceId: core.serialization.property("TraceId", core.serialization.string()),
     start: core.serialization.property("Start", core.serialization.date()),
     end: core.serialization.property("End", core.serialization.date()),
+    spans: core.serialization.property("Spans", core.serialization.list(core.serialization.lazyObject(() => serializers.Span)).optional()),
 });

package/dist/serialization/types/index.d.ts

@@ -1,10 +1,10 @@
-export * from "./CreateRunParams";
 export * from "./CustomSchema";
 export * from "./CustomVariable";
 export * from "./DataTypeEnum";
 export * from "./FileUrl";
 export * from "./Grade";
 export * from "./HttpValidationError";
+export * from "./IngestionMethod";
 export * from "./JsonObjectInputValue";
 export * from "./JsonObject";
 export * from "./JsonObjectOutputValue";
@@ -17,13 +17,12 @@ export * from "./Run";
 export * from "./RunMetric";
 export * from "./RunStatus";
 export * from "./ScoreStatus";
+export * from "./ScoringConfig";
 export * from "./Span";
 export * from "./TestCaseCustomInputsValue";
 export * from "./TestCaseCustomLabelsValue";
 export * from "./TestCase";
-export * from "./TestCaseCreate";
-export * from "./TestRecordCreate";
-export * from "./TestSetCreate";
+export * from "./TestcaseBatchDeletionResponse";
 export * from "./TestcaseDeletionResponse";
 export * from "./TestrecordCustomInputsValue";
 export * from "./TestrecordCustomLabelsValue";
@@ -38,3 +37,6 @@ export * from "./UnauthorizedErrorBody";
 export * from "./ValidationErrorLocItem";
 export * from "./ValidationError";
 export * from "./CreateGithubWorkflowParams";
+export * from "./CreateRunParams";
+export * from "./TestRecordCreate";
+export * from "./TestSetCreate";

package/dist/serialization/types/index.js

@@ -14,13 +14,13 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./CreateRunParams"), exports);
 __exportStar(require("./CustomSchema"), exports);
 __exportStar(require("./CustomVariable"), exports);
 __exportStar(require("./DataTypeEnum"), exports);
 __exportStar(require("./FileUrl"), exports);
 __exportStar(require("./Grade"), exports);
 __exportStar(require("./HttpValidationError"), exports);
+__exportStar(require("./IngestionMethod"), exports);
 __exportStar(require("./JsonObjectInputValue"), exports);
 __exportStar(require("./JsonObject"), exports);
 __exportStar(require("./JsonObjectOutputValue"), exports);
@@ -33,13 +33,12 @@ __exportStar(require("./Run"), exports);
 __exportStar(require("./RunMetric"), exports);
 __exportStar(require("./RunStatus"), exports);
 __exportStar(require("./ScoreStatus"), exports);
+__exportStar(require("./ScoringConfig"), exports);
 __exportStar(require("./Span"), exports);
 __exportStar(require("./TestCaseCustomInputsValue"), exports);
 __exportStar(require("./TestCaseCustomLabelsValue"), exports);
 __exportStar(require("./TestCase"), exports);
-__exportStar(require("./TestCaseCreate"), exports);
-__exportStar(require("./TestRecordCreate"), exports);
-__exportStar(require("./TestSetCreate"), exports);
+__exportStar(require("./TestcaseBatchDeletionResponse"), exports);
 __exportStar(require("./TestcaseDeletionResponse"), exports);
 __exportStar(require("./TestrecordCustomInputsValue"), exports);
 __exportStar(require("./TestrecordCustomLabelsValue"), exports);
@@ -54,3 +53,6 @@ __exportStar(require("./UnauthorizedErrorBody"), exports);
 __exportStar(require("./ValidationErrorLocItem"), exports);
 __exportStar(require("./ValidationError"), exports);
 __exportStar(require("./CreateGithubWorkflowParams"), exports);
+__exportStar(require("./CreateRunParams"), exports);
+__exportStar(require("./TestRecordCreate"), exports);
+__exportStar(require("./TestSetCreate"), exports);

package/dist/telemetry.d.ts

@@ -1,6 +1 @@
-declare const trace: any;
-declare const OTLPTraceExporter: any;
-declare const Resource: any;
-declare const BatchSpanProcessor: any, ConsoleSpanExporter: any, NodeTracerProvider: any;
-declare const SEMRESATTRS_SERVICE_NAME: any;
 declare function checkInstalled(...modules: string[]): boolean;

package/dist/telemetry.js

@@ -8,11 +8,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const { trace } = require('@opentelemetry/api');
-const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
-const { Resource } = require('@opentelemetry/resources');
-const { BatchSpanProcessor, ConsoleSpanExporter, NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
-const { SEMRESATTRS_SERVICE_NAME } = require('@opentelemetry/semantic-conventions');
 function checkInstalled(...modules) {
     for (let i = 0; i < modules.length; i++) {
         let module = modules[i];
@@ -27,6 +22,23 @@ function checkInstalled(...modules) {
 }
 module.exports = {
     setup(name, scorecardConfig, debug = false) {
+        const required = [
+            '@opentelemetry/api',
+            '@opentelemetry/core',
+            '@opentelemetry/exporter-trace-otlp-http',
+            '@opentelemetry/resources',
+            '@opentelemetry/sdk-trace-node',
+            '@opentelemetry/semantic-conventions'
+        ];
+        if (!checkInstalled(...required)) {
+            throw new Error(`Required OpenTelemetry packages not installed. Please ensure that all necessary peerDependencies are installed:\n${required.map(module => `- ${module}`).join('\n')}`);
+        }
+        const { propagation, trace } = require('@opentelemetry/api');
+        const { CompositePropagator, W3CBaggagePropagator, W3CTraceContextPropagator } = require('@opentelemetry/core');
+        const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
+        const { Resource } = require('@opentelemetry/resources');
+        const { BatchSpanProcessor, ConsoleSpanExporter, NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
+        const { SEMRESATTRS_SERVICE_NAME } = require('@opentelemetry/semantic-conventions');
         const resource = Resource.default().merge(new Resource({
             [SEMRESATTRS_SERVICE_NAME]: name,
         }));
@@ -38,6 +50,12 @@ module.exports = {
             const processor = new BatchSpanProcessor(consoleExporter);
             provider.addSpanProcessor(processor);
         }
+        propagation.setGlobalPropagator(new CompositePropagator({
+            propagators: [
+                new W3CTraceContextPropagator(),
+                new W3CBaggagePropagator()
+            ]
+        }));
         const url = scorecardConfig.telemetryUrl ? scorecardConfig.telemetryUrl : 'https://telemetry.getscorecard.ai';
         const otlpExporter = new OTLPTraceExporter({
             url: `${url}/v1/traces`,
@@ -48,11 +66,11 @@ module.exports = {
         const processor = new BatchSpanProcessor(otlpExporter);
         provider.addSpanProcessor(processor);
         provider.register();
-        if (checkInstalled('@langchain/core')) {
+        if (checkInstalled('@langchain/core', '@arizeai/openinference-instrumentation-langchain', '@opentelemetry/instrumentation')) {
             const { LangChainInstrumentation } = require('@arizeai/openinference-instrumentation-langchain');
             new LangChainInstrumentation().manuallyInstrument(require('@langchain/core/callbacks/manager'));
         }
-        if (checkInstalled('openai')) {
+        if (checkInstalled('openai', '@arizeai/openinference-instrumentation-openai', '@opentelemetry/instrumentation')) {
             const { OpenAIInstrumentation } = require('@arizeai/openinference-instrumentation-openai');
             new OpenAIInstrumentation().manuallyInstrument(require('openai'));
         }

package/dist/wrapper/ScorecardClient.d.ts

@@ -1,5 +1,7 @@
 import { ScorecardClient as FernClient } from "../Client";
 export declare namespace ScorecardClient {
+    interface Options extends Omit<FernClient.Options, "fetcher"> {
+    }
     interface RequestOptions extends FernClient.RequestOptions {
     }
     interface RunTestArgs {
@@ -18,6 +20,7 @@ export declare namespace ScorecardClient {
     }
 }
 export declare class ScorecardClient extends FernClient {
+    constructor(options: ScorecardClient.Options);
     /**
      * Runs all tests within a testset
      */

package/dist/wrapper/ScorecardClient.js

@@ -33,9 +33,20 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ScorecardClient = void 0;
+const api_1 = require("@opentelemetry/api");
 const Client_1 = require("../Client");
+const core_1 = require("../core");
 const errors = __importStar(require("../errors"));
 class ScorecardClient extends Client_1.ScorecardClient {
+    constructor(options) {
+        const fetcher = (args) => {
+            var _a;
+            args.headers = (_a = args.headers) !== null && _a !== void 0 ? _a : {};
+            api_1.propagation.inject(api_1.context.active(), args.headers);
+            return (0, core_1.fetcher)(args);
+        };
+        super(Object.assign(Object.assign({}, options), { fetcher }));
+    }
     /**
      * Runs all tests within a testset
      */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scorecard-ai",
-    "version": "0.5.3",
+    "version": "0.6.0",
     "private": false,
     "repository": "https://github.com/scorecard-ai/scorecard-node",
     "license": "See LICENSE",
@@ -14,7 +14,7 @@
     },
     "dependencies": {
         "url-join": "4.0.1",
-        "form-data": "4.0.0",
+        "form-data": "^4.0.0",
         "formdata-node": "^6.0.3",
         "node-fetch": "2.7.0",
         "qs": "6.11.2"
@@ -24,6 +24,7 @@
         "@arizeai/openinference-instrumentation-openai": "^0.2.0",
         "@langchain/core": "^0.1.63",
         "@opentelemetry/api": "*",
+        "@opentelemetry/core": "*",
         "@opentelemetry/exporter-trace-otlp-http": "*",
         "@opentelemetry/instrumentation": ">=0.40.0",
         "@opentelemetry/resources": "*",
@@ -44,6 +45,9 @@
         "@opentelemetry/api": {
             "optional": false
         },
+        "@opentelemetry/core": {
+            "optional": false
+        },
         "@opentelemetry/exporter-trace-otlp-http": {
             "optional": false
         },
@@ -67,6 +71,7 @@
         "@types/url-join": "4.0.1",
         "@types/qs": "6.9.8",
         "@types/node-fetch": "2.6.9",
+        "fetch-mock-jest": "^1.5.1",
         "jest": "29.7.0",
         "@types/jest": "29.5.5",
         "ts-jest": "29.1.1",
@@ -78,11 +83,17 @@
         "@arizeai/openinference-instrumentation-openai": "^0.2.0",
         "@langchain/core": "^0.1.63",
         "@opentelemetry/api": "latest",
+        "@opentelemetry/core": "latest",
         "@opentelemetry/exporter-trace-otlp-http": "latest",
         "@opentelemetry/instrumentation": "latest",
         "@opentelemetry/resources": "latest",
         "@opentelemetry/sdk-trace-node": "latest",
         "@opentelemetry/semantic-conventions": "latest",
         "openai": "^4.47.1"
+    },
+    "browser": {
+        "fs": false,
+        "os": false,
+        "path": false
     }
 }