scorecard-ai 0.3.0 → 0.5.2

package/serialization/types/TestCaseCustomLabelsValue.js

@@ -39,7 +39,10 @@ exports.TestCaseCustomLabelsValue = void 0;
 const core = __importStar(require("../../core"));
 const FileUrl_1 = require("./FileUrl");
 exports.TestCaseCustomLabelsValue = core.serialization.undiscriminatedUnion([
-    core.serialization.string(),
     FileUrl_1.FileUrl,
     core.serialization.lazyObject(() => __awaiter(void 0, void 0, void 0, function* () { return (yield Promise.resolve().then(() => __importStar(require("..")))).JsonObject; })),
+    core.serialization.string(),
+    core.serialization.number(),
+    core.serialization.number(),
+    core.serialization.boolean(),
 ]);

package/serialization/types/TestcaseDeletionResponse.d.ts

@@ -0,0 +1,13 @@
+/**
+ * This file was auto-generated by Fern from our API Definition.
+ */
+import * as serializers from "../index";
+import * as Scorecard from "../../api/index";
+import * as core from "../../core";
+export declare const TestcaseDeletionResponse: core.serialization.ObjectSchema<serializers.TestcaseDeletionResponse.Raw, Scorecard.TestcaseDeletionResponse>;
+export declare namespace TestcaseDeletionResponse {
+    interface Raw {
+        id: number;
+        detail: string;
+    }
+}

package/{dist/serialization/resources/prompt/client/getGraph.js → serialization/types/TestcaseDeletionResponse.js}

@@ -26,7 +26,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Response = void 0;
-const core = __importStar(require("../../../../core"));
-const Prompt_1 = require("../../../types/Prompt");
-exports.Response = core.serialization.list(Prompt_1.Prompt);
+exports.TestcaseDeletionResponse = void 0;
+const core = __importStar(require("../../core"));
+exports.TestcaseDeletionResponse = core.serialization.object({
+    id: core.serialization.number(),
+    detail: core.serialization.string(),
+});

package/serialization/types/Testrecord.d.ts

@@ -21,7 +21,6 @@ export declare namespace Testrecord {
         context?: string | null;
         model_response?: string | null;
         ideal?: string | null;
-        full_prompt?: string | null;
         custom_inputs?: Record<string, TestrecordCustomInputsValue.Raw | null | undefined> | null;
         custom_labels?: Record<string, TestrecordCustomLabelsValue.Raw | null | undefined> | null;
         custom_outputs?: Record<string, TestrecordCustomOutputsValue.Raw | null | undefined> | null;

package/serialization/types/Testrecord.js

@@ -43,7 +43,6 @@ exports.Testrecord = core.serialization.object({
     context: core.serialization.string().optional(),
     modelResponse: core.serialization.property("model_response", core.serialization.string().optional()),
     ideal: core.serialization.string().optional(),
-    fullPrompt: core.serialization.property("full_prompt", core.serialization.string().optional()),
     customInputs: core.serialization.property("custom_inputs", core.serialization.record(core.serialization.string(), TestrecordCustomInputsValue_1.TestrecordCustomInputsValue.optional()).optional()),
     customLabels: core.serialization.property("custom_labels", core.serialization.record(core.serialization.string(), TestrecordCustomLabelsValue_1.TestrecordCustomLabelsValue.optional()).optional()),
     customOutputs: core.serialization.property("custom_outputs", core.serialization.record(core.serialization.string(), TestrecordCustomOutputsValue_1.TestrecordCustomOutputsValue.optional()).optional()),

package/serialization/types/index.d.ts

@@ -24,6 +24,7 @@ export * from "./TestCase";
 export * from "./TestCaseCreate";
 export * from "./TestRecordCreate";
 export * from "./TestSetCreate";
+export * from "./TestcaseDeletionResponse";
 export * from "./TestrecordCustomInputsValue";
 export * from "./TestrecordCustomLabelsValue";
 export * from "./TestrecordCustomOutputsValue";

package/serialization/types/index.js

@@ -40,6 +40,7 @@ __exportStar(require("./TestCase"), exports);
 __exportStar(require("./TestCaseCreate"), exports);
 __exportStar(require("./TestRecordCreate"), exports);
 __exportStar(require("./TestSetCreate"), exports);
+__exportStar(require("./TestcaseDeletionResponse"), exports);
 __exportStar(require("./TestrecordCustomInputsValue"), exports);
 __exportStar(require("./TestrecordCustomLabelsValue"), exports);
 __exportStar(require("./TestrecordCustomOutputsValue"), exports);

package/telemetry.d.ts

@@ -0,0 +1,6 @@
+declare const trace: any;
+declare const OTLPTraceExporter: any;
+declare const Resource: any;
+declare const BatchSpanProcessor: any, ConsoleSpanExporter: any, NodeTracerProvider: any;
+declare const SEMRESATTRS_SERVICE_NAME: any;
+declare function checkInstalled(...modules: string[]): boolean;

package/telemetry.js

@@ -0,0 +1,65 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const { trace } = require('@opentelemetry/api');
+const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
+const { Resource } = require('@opentelemetry/resources');
+const { BatchSpanProcessor, ConsoleSpanExporter, NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
+const { SEMRESATTRS_SERVICE_NAME } = require('@opentelemetry/semantic-conventions');
+function checkInstalled(...modules) {
+    for (let i = 0; i < modules.length; i++) {
+        let module = modules[i];
+        try {
+            require.resolve(module);
+        }
+        catch (e) {
+            return false;
+        }
+    }
+    return true;
+}
+module.exports = {
+    setup(name, scorecardConfig, debug = false) {
+        const resource = Resource.default().merge(new Resource({
+            [SEMRESATTRS_SERVICE_NAME]: name,
+        }));
+        const provider = new NodeTracerProvider({
+            resource,
+        });
+        if (debug) {
+            const consoleExporter = new ConsoleSpanExporter();
+            const processor = new BatchSpanProcessor(consoleExporter);
+            provider.addSpanProcessor(processor);
+        }
+        const url = scorecardConfig.telemetryUrl ? scorecardConfig.telemetryUrl : 'https://telemetry.getscorecard.ai';
+        const otlpExporter = new OTLPTraceExporter({
+            url: `${url}/v1/traces`,
+            headers: {
+                'Authorization': `Bearer ${scorecardConfig.telemetryKey}`
+            },
+        });
+        const processor = new BatchSpanProcessor(otlpExporter);
+        provider.addSpanProcessor(processor);
+        provider.register();
+        if (checkInstalled('@langchain/core')) {
+            const { LangChainInstrumentation } = require('@arizeai/openinference-instrumentation-langchain');
+            new LangChainInstrumentation().manuallyInstrument(require('@langchain/core/callbacks/manager'));
+        }
+        if (checkInstalled('openai')) {
+            const { OpenAIInstrumentation } = require('@arizeai/openinference-instrumentation-openai');
+            new OpenAIInstrumentation().manuallyInstrument(require('openai'));
+        }
+        const tracer = trace.getTracer();
+        process.on('beforeExit', () => __awaiter(this, void 0, void 0, function* () {
+            yield provider.shutdown();
+        }));
+        return tracer;
+    }
+};

package/wrapper/ScorecardClient.d.ts

@@ -1,4 +1,5 @@
 import { ScorecardClient as FernClient } from "../Client";
+import { ScorecardEnvironment } from "../environments";
 import * as core from "../core";
 export declare namespace ScorecardClient {
     interface Options {
@@ -7,6 +8,9 @@ export declare namespace ScorecardClient {
          * variable SCORECARD_API_KEY.
          */
         apiKey?: core.Supplier<string>;
+        environment?: core.Supplier<ScorecardEnvironment | string>;
+    }
+    interface RequestOptions extends FernClient.RequestOptions {
     }
     interface RunTestArgs {
         /**

package/wrapper/ScorecardClient.js

@@ -39,13 +39,11 @@ class ScorecardClient extends Client_1.ScorecardClient {
     constructor(options = {}) {
         var _a;
         const apiKey = (_a = options.apiKey) !== null && _a !== void 0 ? _a : process.env["SCORECARD_API_KEY"];
-        if (apiKey == null) {
-            throw new errors.ScorecardError({
-                message: "Please pass in your Scorecard API Key or export SCORECARD_API_KEY in your environment.",
-            });
+        if (apiKey !== undefined) {
+            super(Object.assign(Object.assign({}, options), { apiKey }));
         }
-        super({
-            apiKey,
+        throw new errors.ScorecardError({
+            message: "Please pass in your Scorecard API Key or export SCORECARD_API_KEY in your environment.",
         });
     }
     /**

package/dist/serialization/resources/prompt/client/getGraph.d.ts

@@ -1,11 +0,0 @@
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-import * as serializers from "../../../index";
-import * as Scorecard from "../../../../api/index";
-import * as core from "../../../../core";
-import { Prompt } from "../../../types/Prompt";
-export declare const Response: core.serialization.Schema<serializers.prompt.getGraph.Response.Raw, Scorecard.Prompt[]>;
-export declare namespace Response {
-    type Raw = Prompt.Raw[];
-}

package/dist/serialization/resources/prompt/client/getProds.d.ts

@@ -1,11 +0,0 @@
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-import * as serializers from "../../../index";
-import * as Scorecard from "../../../../api/index";
-import * as core from "../../../../core";
-import { Prompt } from "../../../types/Prompt";
-export declare const Response: core.serialization.Schema<serializers.prompt.getProds.Response.Raw, Scorecard.Prompt[]>;
-export declare namespace Response {
-    type Raw = Prompt.Raw[];
-}

package/dist/serialization/resources/prompt/client/getProds.js

@@ -1,32 +0,0 @@
-"use strict";
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Response = void 0;
-const core = __importStar(require("../../../../core"));
-const Prompt_1 = require("../../../types/Prompt");
-exports.Response = core.serialization.list(Prompt_1.Prompt);

package/dist/serialization/resources/prompt/client/getRoots.d.ts

@@ -1,11 +0,0 @@
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-import * as serializers from "../../../index";
-import * as Scorecard from "../../../../api/index";
-import * as core from "../../../../core";
-import { Prompt } from "../../../types/Prompt";
-export declare const Response: core.serialization.Schema<serializers.prompt.getRoots.Response.Raw, Scorecard.Prompt[]>;
-export declare namespace Response {
-    type Raw = Prompt.Raw[];
-}

package/serialization/resources/prompt/client/getGraph.d.ts

@@ -1,11 +0,0 @@
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-import * as serializers from "../../../index";
-import * as Scorecard from "../../../../api/index";
-import * as core from "../../../../core";
-import { Prompt } from "../../../types/Prompt";
-export declare const Response: core.serialization.Schema<serializers.prompt.getGraph.Response.Raw, Scorecard.Prompt[]>;
-export declare namespace Response {
-    type Raw = Prompt.Raw[];
-}

package/serialization/resources/prompt/client/getGraph.js

@@ -1,32 +0,0 @@
-"use strict";
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Response = void 0;
-const core = __importStar(require("../../../../core"));
-const Prompt_1 = require("../../../types/Prompt");
-exports.Response = core.serialization.list(Prompt_1.Prompt);

package/serialization/resources/prompt/client/getProds.d.ts

@@ -1,11 +0,0 @@
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-import * as serializers from "../../../index";
-import * as Scorecard from "../../../../api/index";
-import * as core from "../../../../core";
-import { Prompt } from "../../../types/Prompt";
-export declare const Response: core.serialization.Schema<serializers.prompt.getProds.Response.Raw, Scorecard.Prompt[]>;
-export declare namespace Response {
-    type Raw = Prompt.Raw[];
-}

package/serialization/resources/prompt/client/getProds.js

@@ -1,32 +0,0 @@
-"use strict";
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Response = void 0;
-const core = __importStar(require("../../../../core"));
-const Prompt_1 = require("../../../types/Prompt");
-exports.Response = core.serialization.list(Prompt_1.Prompt);

package/serialization/resources/prompt/client/getRoots.d.ts

@@ -1,11 +0,0 @@
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-import * as serializers from "../../../index";
-import * as Scorecard from "../../../../api/index";
-import * as core from "../../../../core";
-import { Prompt } from "../../../types/Prompt";
-export declare const Response: core.serialization.Schema<serializers.prompt.getRoots.Response.Raw, Scorecard.Prompt[]>;
-export declare namespace Response {
-    type Raw = Prompt.Raw[];
-}

package/serialization/resources/prompt/client/getRoots.js

@@ -1,32 +0,0 @@
-"use strict";
-/**
- * This file was auto-generated by Fern from our API Definition.
- */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Response = void 0;
-const core = __importStar(require("../../../../core"));
-const Prompt_1 = require("../../../types/Prompt");
-exports.Response = core.serialization.list(Prompt_1.Prompt);