scorecard-ai 0.3.0 → 0.5.2

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "scorecard-ai",
-    "version": "0.3.0",
+    "version": "0.5.2",
     "private": false,
     "repository": "https://github.com/scorecard-ai/scorecard-node",
     "license": "See LICENSE",
@@ -19,6 +19,50 @@
         "node-fetch": "2.7.0",
         "qs": "6.11.2"
     },
+    "peerDependencies": {
+        "@arizeai/openinference-instrumentation-langchain": "^0.0.5",
+        "@arizeai/openinference-instrumentation-openai": "^0.2.0",
+        "@langchain/core": "^0.1.63",
+        "@opentelemetry/api": "*",
+        "@opentelemetry/exporter-trace-otlp-http": "*",
+        "@opentelemetry/instrumentation": ">=0.40.0",
+        "@opentelemetry/resources": "*",
+        "@opentelemetry/sdk-trace-node": "*",
+        "@opentelemetry/semantic-conventions": "*",
+        "openai": "^4.47.1"
+    },
+    "peerDependenciesMeta": {
+        "@arizeai/openinference-instrumentation-langchain": {
+            "optional": false
+        },
+        "@arizeai/openinference-instrumentation-openai": {
+            "optional": false
+        },
+        "@langchain/core": {
+            "optional": true
+        },
+        "@opentelemetry/api": {
+            "optional": false
+        },
+        "@opentelemetry/exporter-trace-otlp-http": {
+            "optional": false
+        },
+        "@opentelemetry/instrumentation": {
+            "optional": false
+        },
+        "@opentelemetry/resources": {
+            "optional": false
+        },
+        "@opentelemetry/sdk-trace-node": {
+            "optional": false
+        },
+        "@opentelemetry/semantic-conventions": {
+            "optional": false
+        },
+        "openai": {
+            "optional": true
+        }
+    },
     "devDependencies": {
         "@types/url-join": "4.0.1",
         "@types/qs": "6.9.8",
@@ -29,6 +73,16 @@
         "jest-environment-jsdom": "29.7.0",
         "@types/node": "17.0.33",
         "prettier": "2.7.1",
-        "typescript": "4.6.4"
+        "typescript": "4.6.4",
+        "@arizeai/openinference-instrumentation-langchain": "^0.0.5",
+        "@arizeai/openinference-instrumentation-openai": "^0.2.0",
+        "@langchain/core": "^0.1.63",
+        "@opentelemetry/api": "latest",
+        "@opentelemetry/exporter-trace-otlp-http": "latest",
+        "@opentelemetry/instrumentation": "latest",
+        "@opentelemetry/resources": "latest",
+        "@opentelemetry/sdk-trace-node": "latest",
+        "@opentelemetry/semantic-conventions": "latest",
+        "openai": "^4.47.1"
     }
 }

package/reference.md

@@ -58,7 +58,7 @@ await scorecard.testset.get(1);
 
 <dd>
 
-**testsetId: `number`**
+**testsetId: `number`** — The ID of the Testset to retrieve.
 
 </dd>
 
@@ -141,7 +141,7 @@ await scorecard.testset.delete(1);
 
 <dd>
 
-**testsetId: `number`**
+**testsetId: `number`** — The ID of the Testset to delete.
 
 </dd>
 
@@ -309,7 +309,7 @@ await scorecard.testset.readSchema(1);
 
 <dd>
 
-**testsetId: `number`**
+**testsetId: `number`** — The ID of the Testset to retrieve the schema from.
 
 </dd>
 
@@ -392,7 +392,7 @@ await scorecard.testset.getTestcases(1);
 
 <dd>
 
-**testsetId: `number`**
+**testsetId: `number`** — The Testset ID to retrieve testcases from.
 
 </dd>
 
@@ -466,9 +466,7 @@ Create a new Testcase
 <dd>
 
 ```ts
-await scorecard.testcase.create(1, {
-    userQuery: "user_query",
-});
+await scorecard.testcase.create(1);
 ```
 
 </dd>
@@ -489,7 +487,7 @@ await scorecard.testcase.create(1, {
 
 <dd>
 
-**testsetId: `number`**
+**testsetId: `number`** — The ID of the Testset to create the Testcase in.
 
 </dd>
 
@@ -582,7 +580,7 @@ await scorecard.testcase.get(1, 1);
 
 <dd>
 
-**testcaseId: `number`**
+**testcaseId: `number`** — The ID of the Testcase to retrieve.
 
 </dd>
 
@@ -592,7 +590,7 @@ await scorecard.testcase.get(1, 1);
 
 <dd>
 
-**testsetId: `number`**
+**testsetId: `number`** — The ID of the Testset to retrieve the Testcase from.
 
 </dd>
 
@@ -617,7 +615,7 @@ await scorecard.testcase.get(1, 1);
 </dl>
 </details>
 
-<details><summary> <code>scorecard.testcase.<a href="./src/api/resources/testcase/client/Client.ts">delete</a>(testcaseId, testsetId) -> unknown</code> </summary>
+<details><summary> <code>scorecard.testcase.<a href="./src/api/resources/testcase/client/Client.ts">delete</a>(testcaseId, testsetId) -> Scorecard.TestcaseDeletionResponse</code> </summary>
 
 <dl>
 
@@ -675,7 +673,7 @@ await scorecard.testcase.delete(1, 1);
 
 <dd>
 
-**testcaseId: `number`**
+**testcaseId: `number`** — The ID of the Testcase to delete.
 
 </dd>
 
@@ -685,7 +683,7 @@ await scorecard.testcase.delete(1, 1);
 
 <dd>
 
-**testsetId: `number`**
+**testsetId: `number`** — The ID of the Testset to delete the Testcase from.
 
 </dd>
 
@@ -863,7 +861,7 @@ await scorecard.testrecord.create(1);
 
 <dd>
 
-**runId: `number`**
+**runId: `number`** — The ID of the Run to create the Testrecord in.
 
 </dd>
 
@@ -937,15 +935,7 @@ Create a new Run
 <dd>
 
 ```ts
-await scorecard.run.create({
-    testsetId: 1,
-    status: "RUNNING_EXECUTION",
-    modelParams: {
-        param1: "value1",
-        param2: "value2",
-    },
-    metrics: [1, 2],
-});
+await scorecard.run.create();
 ```
 
 </dd>
@@ -1111,9 +1101,7 @@ Update the status of a run.
 <dd>
 
 ```ts
-await scorecard.run.updateStatus(1, {
-    status: Scorecard.RunStatus.Pending,
-});
+await scorecard.run.updateStatus(1);
 ```
 
 </dd>
@@ -1231,7 +1219,7 @@ await scorecard.score.create(1, 1, {
 
 <dd>
 
-**runId: `number`**
+**runId: `number`** — The ID of the run that created the testrecord to be scored.
 
 </dd>
 
@@ -1241,7 +1229,7 @@ await scorecard.score.create(1, 1, {
 
 <dd>
 
-**testrecordId: `number`**
+**testrecordId: `number`** — The ID of the testrecord to be scored.
 
 </dd>
 
@@ -1334,7 +1322,7 @@ await scorecard.score.update(1, 1, 1);
 
 <dd>
 
-**runId: `number`**
+**runId: `number`** — The run ID that created the test record to be scored.
 
 </dd>
 
@@ -1344,7 +1332,7 @@ await scorecard.score.update(1, 1, 1);
 
 <dd>
 
-**testrecordId: `number`**
+**testrecordId: `number`** — The ID of the testrecord whose score will be updated.
 
 </dd>
 
@@ -1354,7 +1342,7 @@ await scorecard.score.update(1, 1, 1);
 
 <dd>
 
-**scoreId: `number`**
+**scoreId: `number`** — The ID of the score to be updated.
 
 </dd>
 
@@ -1577,7 +1565,11 @@ await scorecard.traces.get("trace_id");
 
 <dd>
 
-Create a new prompt
+Two types of prompts can be created - a root prompt or a child prompt (aka Prompt Version in app).
+
+        A root prompt can be created by providing the `name` param, and it will always be tagged as prod.
+
+        A child prompt can be created by providing the `parent_id` param. Note that the `name` param in this case will be ignored as all descendents from a root prompt would share the root's name. `is_prod` can also be provided to configure whether a child should be tagged as prod.
 
 </dd>
 
@@ -1599,7 +1591,8 @@ Create a new prompt
 
 ```ts
 await scorecard.prompt.create({
-    promptTemplate: "You are a virtual assistant",
+    promptTemplate:
+        "<system>\nYou are a helpful assistant. Use the provided context to answer the user's query.\n\nContext: {context}\n</system>\n\n<user>\n{user_query}\n</user>",
     name: "Prompt Name",
     description: "Description of the prompt",
     modelParams: {
@@ -1654,7 +1647,7 @@ await scorecard.prompt.create({
 </dl>
 </details>
 
-<details><summary> <code>scorecard.prompt.<a href="./src/api/resources/prompt/client/Client.ts">getRoots</a>() -> Scorecard.Prompt[]</code> </summary>
+<details><summary> <code>scorecard.prompt.<a href="./src/api/resources/prompt/client/Client.ts">get</a>(id) -> Scorecard.Prompt</code> </summary>
 
 <dl>
 
@@ -1670,7 +1663,7 @@ await scorecard.prompt.create({
 
 <dd>
 
-Retrieve active root prompts for the org
+Retrieve a prompt by id
 
 </dd>
 
@@ -1691,7 +1684,7 @@ Retrieve active root prompts for the org
 <dd>
 
 ```ts
-await scorecard.prompt.getRoots();
+await scorecard.prompt.get("id");
 ```
 
 </dd>
@@ -1712,75 +1705,12 @@ await scorecard.prompt.getRoots();
 
 <dd>
 
-**requestOptions: `Prompt.RequestOptions`**
-
-</dd>
-
-</dl>
-
-</dd>
-
-</dl>
-
-</dd>
-
-</dl>
-</details>
-
-<details><summary> <code>scorecard.prompt.<a href="./src/api/resources/prompt/client/Client.ts">getProds</a>() -> Scorecard.Prompt[]</code> </summary>
-
-<dl>
-
-<dd>
-
-#### 📝 Description
-
-<dl>
-
-<dd>
-
-<dl>
-
-<dd>
-
-Retrieve prod prompts for the org
-
-</dd>
-
-</dl>
-
-</dd>
-
-</dl>
-
-#### 🔌 Usage
-
-<dl>
-
-<dd>
-
-<dl>
-
-<dd>
-
-```ts
-await scorecard.prompt.getProds();
-```
-
-</dd>
-
-</dl>
+**id: `string`** — The id of the prompt to get.
 
 </dd>
 
 </dl>
 
-#### ⚙️ Parameters
-
-<dl>
-
-<dd>
-
 <dl>
 
 <dd>
@@ -1800,7 +1730,7 @@ await scorecard.prompt.getProds();
 </dl>
 </details>
 
-<details><summary> <code>scorecard.prompt.<a href="./src/api/resources/prompt/client/Client.ts">get</a>(id) -> Scorecard.Prompt</code> </summary>
+<details><summary> <code>scorecard.prompt.<a href="./src/api/resources/prompt/client/Client.ts">delete</a>(id) -> unknown</code> </summary>
 
 <dl>
 
@@ -1816,7 +1746,7 @@ await scorecard.prompt.getProds();
 
 <dd>
 
-Retrieve a prompt by id
+Delete a root prompt and all of its children.
 
 </dd>
 
@@ -1837,7 +1767,7 @@ Retrieve a prompt by id
 <dd>
 
 ```ts
-await scorecard.prompt.get("id");
+await scorecard.prompt.delete("id");
 ```
 
 </dd>
@@ -1858,7 +1788,7 @@ await scorecard.prompt.get("id");
 
 <dd>
 
-**id: `string`** — The id of the prompt to get.
+**id: `string`** — The id of the root prompt to delete.
 
 </dd>
 
@@ -1899,7 +1829,9 @@ await scorecard.prompt.get("id");
 
 <dd>
 
-Update a prompt
+Update a prompt.
+
+        `is_prod` tags the provided prompt as the production prompt within the prompt graph.
 
 </dd>
 
@@ -1921,7 +1853,7 @@ Update a prompt
 
 ```ts
 await scorecard.prompt.update("id", {
-    isArchived: true,
+    isProd: true,
 });
 ```
 
@@ -2060,86 +1992,3 @@ await scorecard.prompt.getByName("name");
 
 </dl>
 </details>
-
-<details><summary> <code>scorecard.prompt.<a href="./src/api/resources/prompt/client/Client.ts">getGraph</a>(id) -> Scorecard.Prompt[]</code> </summary>
-
-<dl>
-
-<dd>
-
-#### 📝 Description
-
-<dl>
-
-<dd>
-
-<dl>
-
-<dd>
-
-Retrieve all nodes within the same graph as the prompt, sorted by created_at desc
-
-</dd>
-
-</dl>
-
-</dd>
-
-</dl>
-
-#### 🔌 Usage
-
-<dl>
-
-<dd>
-
-<dl>
-
-<dd>
-
-```ts
-await scorecard.prompt.getGraph("id");
-```
-
-</dd>
-
-</dl>
-
-</dd>
-
-</dl>
-
-#### ⚙️ Parameters
-
-<dl>
-
-<dd>
-
-<dl>
-
-<dd>
-
-**id: `string`** — The id of the prompt.
-
-</dd>
-
-</dl>
-
-<dl>
-
-<dd>
-
-**requestOptions: `Prompt.RequestOptions`**
-
-</dd>
-
-</dl>
-
-</dd>
-
-</dl>
-
-</dd>
-
-</dl>
-</details>

package/serialization/resources/prompt/client/index.d.ts

@@ -1,4 +1 @@
-export * as getRoots from "./getRoots";
-export * as getProds from "./getProds";
-export * as getGraph from "./getGraph";
 export * from "./requests";

package/serialization/resources/prompt/client/index.js

@@ -10,24 +10,8 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
 }));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getGraph = exports.getProds = exports.getRoots = void 0;
-exports.getRoots = __importStar(require("./getRoots"));
-exports.getProds = __importStar(require("./getProds"));
-exports.getGraph = __importStar(require("./getGraph"));
 __exportStar(require("./requests"), exports);

package/serialization/resources/prompt/client/requests/PromptUpdateParams.d.ts

@@ -7,7 +7,6 @@ import * as core from "../../../../../core";
 export declare const PromptUpdateParams: core.serialization.Schema<serializers.PromptUpdateParams.Raw, Scorecard.PromptUpdateParams>;
 export declare namespace PromptUpdateParams {
     interface Raw {
-        is_archived?: boolean | null;
         is_prod?: boolean | null;
     }
 }

package/serialization/resources/prompt/client/requests/PromptUpdateParams.js

@@ -29,6 +29,5 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.PromptUpdateParams = void 0;
 const core = __importStar(require("../../../../../core"));
 exports.PromptUpdateParams = core.serialization.object({
-    isArchived: core.serialization.property("is_archived", core.serialization.boolean().optional()),
     isProd: core.serialization.property("is_prod", core.serialization.boolean().optional()),
 });

package/serialization/resources/run/client/requests/UpdateStatusParams.d.ts

@@ -8,6 +8,6 @@ import { RunStatus } from "../../../../types/RunStatus";
 export declare const UpdateStatusParams: core.serialization.Schema<serializers.UpdateStatusParams.Raw, Scorecard.UpdateStatusParams>;
 export declare namespace UpdateStatusParams {
     interface Raw {
-        status: RunStatus.Raw;
+        status?: RunStatus.Raw | null;
     }
 }

package/serialization/resources/run/client/requests/UpdateStatusParams.js

@@ -30,5 +30,5 @@ exports.UpdateStatusParams = void 0;
 const core = __importStar(require("../../../../../core"));
 const RunStatus_1 = require("../../../../types/RunStatus");
 exports.UpdateStatusParams = core.serialization.object({
-    status: RunStatus_1.RunStatus,
+    status: RunStatus_1.RunStatus.optional(),
 });

package/serialization/resources/testcase/client/requests/TestcaseCreateParams.d.ts

@@ -9,7 +9,7 @@ import { TestcaseCreateParamsCustomLabelsValue } from "../../types/TestcaseCreat
 export declare const TestcaseCreateParams: core.serialization.Schema<serializers.TestcaseCreateParams.Raw, Scorecard.TestcaseCreateParams>;
 export declare namespace TestcaseCreateParams {
     interface Raw {
-        user_query: string;
+        user_query?: string | null;
         context?: string | null;
         ideal?: string | null;
         custom_inputs?: Record<string, TestcaseCreateParamsCustomInputsValue.Raw | null | undefined> | null;

package/serialization/resources/testcase/client/requests/TestcaseCreateParams.js

@@ -31,7 +31,7 @@ const core = __importStar(require("../../../../../core"));
 const TestcaseCreateParamsCustomInputsValue_1 = require("../../types/TestcaseCreateParamsCustomInputsValue");
 const TestcaseCreateParamsCustomLabelsValue_1 = require("../../types/TestcaseCreateParamsCustomLabelsValue");
 exports.TestcaseCreateParams = core.serialization.object({
-    userQuery: core.serialization.property("user_query", core.serialization.string()),
+    userQuery: core.serialization.property("user_query", core.serialization.string().optional()),
     context: core.serialization.string().optional(),
     ideal: core.serialization.string().optional(),
     customInputs: core.serialization.property("custom_inputs", core.serialization

package/serialization/resources/testcase/types/TestcaseCreateParamsCustomInputsValue.d.ts

@@ -7,5 +7,5 @@ import * as core from "../../../../core";
 import { FileUrl } from "../../../types/FileUrl";
 export declare const TestcaseCreateParamsCustomInputsValue: core.serialization.Schema<serializers.TestcaseCreateParamsCustomInputsValue.Raw, Scorecard.TestcaseCreateParamsCustomInputsValue>;
 export declare namespace TestcaseCreateParamsCustomInputsValue {
-    type Raw = string | FileUrl.Raw | serializers.JsonObject.Raw;
+    type Raw = FileUrl.Raw | serializers.JsonObject.Raw | string | number | number | boolean;
 }

package/serialization/resources/testcase/types/TestcaseCreateParamsCustomInputsValue.js

@@ -39,7 +39,10 @@ exports.TestcaseCreateParamsCustomInputsValue = void 0;
 const core = __importStar(require("../../../../core"));
 const FileUrl_1 = require("../../../types/FileUrl");
 exports.TestcaseCreateParamsCustomInputsValue = core.serialization.undiscriminatedUnion([
-    core.serialization.string(),
     FileUrl_1.FileUrl,
     core.serialization.lazyObject(() => __awaiter(void 0, void 0, void 0, function* () { return (yield Promise.resolve().then(() => __importStar(require("../../..")))).JsonObject; })),
+    core.serialization.string(),
+    core.serialization.number(),
+    core.serialization.number(),
+    core.serialization.boolean(),
 ]);

package/serialization/resources/testcase/types/TestcaseCreateParamsCustomLabelsValue.d.ts

@@ -7,5 +7,5 @@ import * as core from "../../../../core";
 import { FileUrl } from "../../../types/FileUrl";
 export declare const TestcaseCreateParamsCustomLabelsValue: core.serialization.Schema<serializers.TestcaseCreateParamsCustomLabelsValue.Raw, Scorecard.TestcaseCreateParamsCustomLabelsValue>;
 export declare namespace TestcaseCreateParamsCustomLabelsValue {
-    type Raw = string | FileUrl.Raw | serializers.JsonObject.Raw;
+    type Raw = FileUrl.Raw | serializers.JsonObject.Raw | string | number | number | boolean;
 }

package/serialization/resources/testcase/types/TestcaseCreateParamsCustomLabelsValue.js

@@ -39,7 +39,10 @@ exports.TestcaseCreateParamsCustomLabelsValue = void 0;
 const core = __importStar(require("../../../../core"));
 const FileUrl_1 = require("../../../types/FileUrl");
 exports.TestcaseCreateParamsCustomLabelsValue = core.serialization.undiscriminatedUnion([
-    core.serialization.string(),
     FileUrl_1.FileUrl,
     core.serialization.lazyObject(() => __awaiter(void 0, void 0, void 0, function* () { return (yield Promise.resolve().then(() => __importStar(require("../../..")))).JsonObject; })),
+    core.serialization.string(),
+    core.serialization.number(),
+    core.serialization.number(),
+    core.serialization.boolean(),
 ]);

package/serialization/types/TestCaseCustomInputsValue.d.ts

@@ -7,5 +7,5 @@ import * as core from "../../core";
 import { FileUrl } from "./FileUrl";
 export declare const TestCaseCustomInputsValue: core.serialization.Schema<serializers.TestCaseCustomInputsValue.Raw, Scorecard.TestCaseCustomInputsValue>;
 export declare namespace TestCaseCustomInputsValue {
-    type Raw = string | FileUrl.Raw | serializers.JsonObject.Raw;
+    type Raw = FileUrl.Raw | serializers.JsonObject.Raw | string | number | number | boolean;
 }

package/serialization/types/TestCaseCustomInputsValue.js

@@ -39,7 +39,10 @@ exports.TestCaseCustomInputsValue = void 0;
 const core = __importStar(require("../../core"));
 const FileUrl_1 = require("./FileUrl");
 exports.TestCaseCustomInputsValue = core.serialization.undiscriminatedUnion([
-    core.serialization.string(),
     FileUrl_1.FileUrl,
     core.serialization.lazyObject(() => __awaiter(void 0, void 0, void 0, function* () { return (yield Promise.resolve().then(() => __importStar(require("..")))).JsonObject; })),
+    core.serialization.string(),
+    core.serialization.number(),
+    core.serialization.number(),
+    core.serialization.boolean(),
 ]);

package/serialization/types/TestCaseCustomLabelsValue.d.ts

@@ -7,5 +7,5 @@ import * as core from "../../core";
 import { FileUrl } from "./FileUrl";
 export declare const TestCaseCustomLabelsValue: core.serialization.Schema<serializers.TestCaseCustomLabelsValue.Raw, Scorecard.TestCaseCustomLabelsValue>;
 export declare namespace TestCaseCustomLabelsValue {
-    type Raw = string | FileUrl.Raw | serializers.JsonObject.Raw;
+    type Raw = FileUrl.Raw | serializers.JsonObject.Raw | string | number | number | boolean;
 }