vellum-ai 0.0.8 → 0.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/Client.d.ts +3 -0
  2. package/Client.js +5 -0
  3. package/api/resources/index.d.ts +2 -0
  4. package/api/resources/index.js +3 -1
  5. package/api/resources/sandboxes/client/requests/UpsertSandboxScenarioRequestRequest.d.ts +1 -1
  6. package/api/resources/testSuites/client/Client.d.ts +17 -0
  7. package/api/resources/testSuites/client/Client.js +81 -0
  8. package/api/resources/testSuites/client/index.d.ts +1 -0
  9. package/api/resources/testSuites/client/index.js +17 -0
  10. package/api/resources/testSuites/client/requests/TestSuiteTestCaseRequest.d.ts +14 -0
  11. package/api/resources/testSuites/client/requests/index.d.ts +1 -0
  12. package/api/resources/testSuites/client/requests/index.js +2 -0
  13. package/api/resources/testSuites/index.d.ts +1 -0
  14. package/api/resources/testSuites/index.js +17 -0
  15. package/api/types/EvaluationParams.d.ts +7 -0
  16. package/api/types/EvaluationParamsRequest.d.ts +7 -0
  17. package/api/types/ProcessingFailureReasonEnum.d.ts +7 -0
  18. package/api/types/ProcessingFailureReasonEnum.js +9 -0
  19. package/api/types/SandboxMetricInputParams.d.ts +2 -1
  20. package/api/types/SandboxMetricInputParamsRequest.d.ts +2 -1
  21. package/api/types/SandboxScenario.d.ts +1 -1
  22. package/api/types/{SandboxInput.d.ts → ScenarioInput.d.ts} +1 -1
  23. package/api/types/{SandboxInputRequest.d.ts → ScenarioInputRequest.d.ts} +1 -1
  24. package/api/types/ScenarioInputRequest.js +5 -0
  25. package/api/types/SlimDocument.d.ts +3 -1
  26. package/api/types/TestSuiteTestCase.d.ts +14 -0
  27. package/api/types/TestSuiteTestCase.js +5 -0
  28. package/api/types/index.d.ts +6 -2
  29. package/api/types/index.js +6 -2
  30. package/dist/Client.d.ts +3 -0
  31. package/dist/Client.js +5 -0
  32. package/dist/api/resources/index.d.ts +2 -0
  33. package/dist/api/resources/index.js +3 -1
  34. package/dist/api/resources/sandboxes/client/requests/UpsertSandboxScenarioRequestRequest.d.ts +1 -1
  35. package/dist/api/resources/testSuites/client/Client.d.ts +17 -0
  36. package/dist/api/resources/testSuites/client/Client.js +81 -0
  37. package/dist/api/resources/testSuites/client/index.d.ts +1 -0
  38. package/dist/api/resources/testSuites/client/index.js +17 -0
  39. package/dist/api/resources/testSuites/client/requests/TestSuiteTestCaseRequest.d.ts +14 -0
  40. package/dist/api/resources/testSuites/client/requests/TestSuiteTestCaseRequest.js +5 -0
  41. package/dist/api/resources/testSuites/client/requests/index.d.ts +1 -0
  42. package/dist/api/resources/testSuites/client/requests/index.js +2 -0
  43. package/dist/api/resources/testSuites/index.d.ts +1 -0
  44. package/dist/api/resources/testSuites/index.js +17 -0
  45. package/dist/api/types/EvaluationParams.d.ts +7 -0
  46. package/dist/api/types/EvaluationParams.js +5 -0
  47. package/dist/api/types/EvaluationParamsRequest.d.ts +7 -0
  48. package/dist/api/types/EvaluationParamsRequest.js +5 -0
  49. package/dist/api/types/ProcessingFailureReasonEnum.d.ts +7 -0
  50. package/dist/api/types/ProcessingFailureReasonEnum.js +9 -0
  51. package/dist/api/types/SandboxMetricInputParams.d.ts +2 -1
  52. package/dist/api/types/SandboxMetricInputParamsRequest.d.ts +2 -1
  53. package/dist/api/types/SandboxScenario.d.ts +1 -1
  54. package/dist/api/types/{SandboxInput.d.ts → ScenarioInput.d.ts} +1 -1
  55. package/dist/api/types/ScenarioInput.js +5 -0
  56. package/dist/api/types/{SandboxInputRequest.d.ts → ScenarioInputRequest.d.ts} +1 -1
  57. package/dist/api/types/ScenarioInputRequest.js +5 -0
  58. package/dist/api/types/SlimDocument.d.ts +3 -1
  59. package/dist/api/types/TestSuiteTestCase.d.ts +14 -0
  60. package/dist/api/types/TestSuiteTestCase.js +5 -0
  61. package/dist/api/types/index.d.ts +6 -2
  62. package/dist/api/types/index.js +6 -2
  63. package/dist/serialization/resources/index.d.ts +2 -0
  64. package/dist/serialization/resources/index.js +3 -1
  65. package/dist/serialization/resources/sandboxes/client/requests/UpsertSandboxScenarioRequestRequest.d.ts +1 -1
  66. package/dist/serialization/resources/sandboxes/client/requests/UpsertSandboxScenarioRequestRequest.js +1 -1
  67. package/dist/serialization/resources/testSuites/client/index.d.ts +1 -0
  68. package/dist/serialization/resources/testSuites/client/index.js +17 -0
  69. package/dist/serialization/resources/testSuites/client/requests/TestSuiteTestCaseRequest.d.ts +15 -0
  70. package/dist/serialization/resources/testSuites/client/requests/TestSuiteTestCaseRequest.js +36 -0
  71. package/dist/serialization/resources/testSuites/client/requests/index.d.ts +1 -0
  72. package/dist/serialization/resources/testSuites/client/requests/index.js +5 -0
  73. package/dist/serialization/resources/testSuites/index.d.ts +1 -0
  74. package/dist/serialization/resources/testSuites/index.js +17 -0
  75. package/dist/serialization/types/EvaluationParams.d.ts +12 -0
  76. package/dist/serialization/types/EvaluationParams.js +33 -0
  77. package/dist/serialization/types/EvaluationParamsRequest.d.ts +12 -0
  78. package/dist/serialization/types/EvaluationParamsRequest.js +33 -0
  79. package/dist/serialization/types/ProcessingFailureReasonEnum.d.ts +10 -0
  80. package/dist/serialization/types/ProcessingFailureReasonEnum.js +31 -0
  81. package/dist/serialization/types/SandboxMetricInputParams.d.ts +1 -1
  82. package/dist/serialization/types/SandboxMetricInputParams.js +1 -1
  83. package/dist/serialization/types/SandboxMetricInputParamsRequest.d.ts +1 -1
  84. package/dist/serialization/types/SandboxMetricInputParamsRequest.js +1 -1
  85. package/dist/serialization/types/SandboxScenario.d.ts +1 -1
  86. package/dist/serialization/types/SandboxScenario.js +1 -1
  87. package/dist/serialization/types/{SandboxInput.d.ts → ScenarioInput.d.ts} +2 -2
  88. package/dist/serialization/types/{SandboxInput.js → ScenarioInput.js} +2 -2
  89. package/dist/serialization/types/{SandboxInputRequest.d.ts → ScenarioInputRequest.d.ts} +2 -2
  90. package/dist/serialization/types/{SandboxInputRequest.js → ScenarioInputRequest.js} +2 -2
  91. package/dist/serialization/types/SlimDocument.d.ts +1 -0
  92. package/dist/serialization/types/SlimDocument.js +1 -0
  93. package/dist/serialization/types/TestSuiteTestCase.d.ts +15 -0
  94. package/dist/serialization/types/TestSuiteTestCase.js +36 -0
  95. package/dist/serialization/types/index.d.ts +6 -2
  96. package/dist/serialization/types/index.js +6 -2
  97. package/package.json +1 -1
  98. package/serialization/resources/index.d.ts +2 -0
  99. package/serialization/resources/index.js +3 -1
  100. package/serialization/resources/sandboxes/client/requests/UpsertSandboxScenarioRequestRequest.d.ts +1 -1
  101. package/serialization/resources/sandboxes/client/requests/UpsertSandboxScenarioRequestRequest.js +1 -1
  102. package/serialization/resources/testSuites/client/index.d.ts +1 -0
  103. package/serialization/resources/testSuites/client/index.js +17 -0
  104. package/serialization/resources/testSuites/client/requests/TestSuiteTestCaseRequest.d.ts +15 -0
  105. package/serialization/resources/testSuites/client/requests/TestSuiteTestCaseRequest.js +36 -0
  106. package/serialization/resources/testSuites/client/requests/index.d.ts +1 -0
  107. package/serialization/resources/testSuites/client/requests/index.js +5 -0
  108. package/serialization/resources/testSuites/index.d.ts +1 -0
  109. package/serialization/resources/testSuites/index.js +17 -0
  110. package/serialization/types/EvaluationParams.d.ts +12 -0
  111. package/serialization/types/EvaluationParams.js +33 -0
  112. package/serialization/types/EvaluationParamsRequest.d.ts +12 -0
  113. package/serialization/types/EvaluationParamsRequest.js +33 -0
  114. package/serialization/types/ProcessingFailureReasonEnum.d.ts +10 -0
  115. package/serialization/types/ProcessingFailureReasonEnum.js +31 -0
  116. package/serialization/types/SandboxMetricInputParams.d.ts +1 -1
  117. package/serialization/types/SandboxMetricInputParams.js +1 -1
  118. package/serialization/types/SandboxMetricInputParamsRequest.d.ts +1 -1
  119. package/serialization/types/SandboxMetricInputParamsRequest.js +1 -1
  120. package/serialization/types/SandboxScenario.d.ts +1 -1
  121. package/serialization/types/SandboxScenario.js +1 -1
  122. package/serialization/types/{SandboxInput.d.ts → ScenarioInput.d.ts} +2 -2
  123. package/serialization/types/{SandboxInput.js → ScenarioInput.js} +2 -2
  124. package/serialization/types/{SandboxInputRequest.d.ts → ScenarioInputRequest.d.ts} +2 -2
  125. package/serialization/types/{SandboxInputRequest.js → ScenarioInputRequest.js} +2 -2
  126. package/serialization/types/SlimDocument.d.ts +1 -0
  127. package/serialization/types/SlimDocument.js +1 -0
  128. package/serialization/types/TestSuiteTestCase.d.ts +15 -0
  129. package/serialization/types/TestSuiteTestCase.js +36 -0
  130. package/serialization/types/index.d.ts +6 -2
  131. package/serialization/types/index.js +6 -2
  132. /package/api/{types/SandboxInput.js → resources/testSuites/client/requests/TestSuiteTestCaseRequest.js} +0 -0
  133. /package/api/types/{SandboxInputRequest.js → EvaluationParams.js} +0 -0
  134. /package/{dist/api/types/SandboxInput.js → api/types/EvaluationParamsRequest.js} +0 -0
  135. /package/{dist/api/types/SandboxInputRequest.js → api/types/ScenarioInput.js} +0 -0
package/Client.d.ts CHANGED
@@ -7,6 +7,7 @@ import * as Vellum from "./api";
7
7
  import { Documents } from "./api/resources/documents/client/Client";
8
8
  import { ModelVersions } from "./api/resources/modelVersions/client/Client";
9
9
  import { Sandboxes } from "./api/resources/sandboxes/client/Client";
10
+ import { TestSuites } from "./api/resources/testSuites/client/Client";
10
11
  export declare namespace VellumClient {
11
12
  interface Options {
12
13
  environment?: environments.VellumEnvironment | environments.VellumEnvironmentUrls;
@@ -25,4 +26,6 @@ export declare class VellumClient {
25
26
  get modelVersions(): ModelVersions;
26
27
  protected _sandboxes: Sandboxes | undefined;
27
28
  get sandboxes(): Sandboxes;
29
+ protected _testSuites: TestSuites | undefined;
30
+ get testSuites(): TestSuites;
28
31
  }
package/Client.js CHANGED
@@ -38,6 +38,7 @@ const errors = __importStar(require("./errors"));
38
38
  const Client_1 = require("./api/resources/documents/client/Client");
39
39
  const Client_2 = require("./api/resources/modelVersions/client/Client");
40
40
  const Client_3 = require("./api/resources/sandboxes/client/Client");
41
+ const Client_4 = require("./api/resources/testSuites/client/Client");
41
42
  class VellumClient {
42
43
  options;
43
44
  constructor(options) {
@@ -164,5 +165,9 @@ class VellumClient {
164
165
  get sandboxes() {
165
166
  return (this._sandboxes ??= new Client_3.Sandboxes(this.options));
166
167
  }
168
+ _testSuites;
169
+ get testSuites() {
170
+ return (this._testSuites ??= new Client_4.TestSuites(this.options));
171
+ }
167
172
  }
168
173
  exports.VellumClient = VellumClient;
@@ -1,5 +1,7 @@
1
1
  export * as documents from "./documents";
2
2
  export * as modelVersions from "./modelVersions";
3
3
  export * as sandboxes from "./sandboxes";
4
+ export * as testSuites from "./testSuites";
4
5
  export * from "./documents/client/requests";
5
6
  export * from "./sandboxes/client/requests";
7
+ export * from "./testSuites/client/requests";
@@ -26,9 +26,11 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
26
26
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
27
27
  };
28
28
  Object.defineProperty(exports, "__esModule", { value: true });
29
- exports.sandboxes = exports.modelVersions = exports.documents = void 0;
29
+ exports.testSuites = exports.sandboxes = exports.modelVersions = exports.documents = void 0;
30
30
  exports.documents = __importStar(require("./documents"));
31
31
  exports.modelVersions = __importStar(require("./modelVersions"));
32
32
  exports.sandboxes = __importStar(require("./sandboxes"));
33
+ exports.testSuites = __importStar(require("./testSuites"));
33
34
  __exportStar(require("./documents/client/requests"), exports);
34
35
  __exportStar(require("./sandboxes/client/requests"), exports);
36
+ __exportStar(require("./testSuites/client/requests"), exports);
@@ -5,7 +5,7 @@ import * as Vellum from "../../../..";
5
5
  export interface UpsertSandboxScenarioRequestRequest {
6
6
  label?: string;
7
7
  /** The inputs for the scenario */
8
- inputs: Vellum.SandboxInputRequest[];
8
+ inputs: Vellum.ScenarioInputRequest[];
9
9
  /** The id of the scenario to update. If none is provided, an id will be generated and a new scenario will be appended. */
10
10
  scenarioId?: string;
11
11
  metricInputParams?: Vellum.SandboxMetricInputParamsRequest;
@@ -0,0 +1,17 @@
1
+ /**
2
+ * This file was auto-generated by Fern from our API Definition.
3
+ */
4
+ import * as environments from "../../../../environments";
5
+ import * as core from "../../../../core";
6
+ import * as Vellum from "../../..";
7
+ export declare namespace TestSuites {
8
+ interface Options {
9
+ environment?: environments.VellumEnvironment | environments.VellumEnvironmentUrls;
10
+ apiKey: core.Supplier<string>;
11
+ }
12
+ }
13
+ export declare class TestSuites {
14
+ protected readonly options: TestSuites.Options;
15
+ constructor(options: TestSuites.Options);
16
+ upsertTestSuiteTestCase(id: string, request: Vellum.TestSuiteTestCaseRequest): Promise<Vellum.TestSuiteTestCase>;
17
+ }
@@ -0,0 +1,81 @@
1
+ "use strict";
2
+ /**
3
+ * This file was auto-generated by Fern from our API Definition.
4
+ */
5
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
6
+ if (k2 === undefined) k2 = k;
7
+ var desc = Object.getOwnPropertyDescriptor(m, k);
8
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
9
+ desc = { enumerable: true, get: function() { return m[k]; } };
10
+ }
11
+ Object.defineProperty(o, k2, desc);
12
+ }) : (function(o, m, k, k2) {
13
+ if (k2 === undefined) k2 = k;
14
+ o[k2] = m[k];
15
+ }));
16
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
17
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
18
+ }) : function(o, v) {
19
+ o["default"] = v;
20
+ });
21
+ var __importStar = (this && this.__importStar) || function (mod) {
22
+ if (mod && mod.__esModule) return mod;
23
+ var result = {};
24
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
25
+ __setModuleDefault(result, mod);
26
+ return result;
27
+ };
28
+ var __importDefault = (this && this.__importDefault) || function (mod) {
29
+ return (mod && mod.__esModule) ? mod : { "default": mod };
30
+ };
31
+ Object.defineProperty(exports, "__esModule", { value: true });
32
+ exports.TestSuites = void 0;
33
+ const environments = __importStar(require("../../../../environments"));
34
+ const core = __importStar(require("../../../../core"));
35
+ const url_join_1 = __importDefault(require("url-join"));
36
+ const serializers = __importStar(require("../../../../serialization"));
37
+ const errors = __importStar(require("../../../../errors"));
38
+ class TestSuites {
39
+ options;
40
+ constructor(options) {
41
+ this.options = options;
42
+ }
43
+ async upsertTestSuiteTestCase(id, request) {
44
+ const _response = await core.fetcher({
45
+ url: (0, url_join_1.default)((this.options.environment ?? environments.VellumEnvironment.Production).default, `v1/test-suites/${id}/test-cases`),
46
+ method: "POST",
47
+ headers: {
48
+ X_API_KEY: await core.Supplier.get(this.options.apiKey),
49
+ },
50
+ contentType: "application/json",
51
+ body: await serializers.TestSuiteTestCaseRequest.jsonOrThrow(request, { unrecognizedObjectKeys: "strip" }),
52
+ });
53
+ if (_response.ok) {
54
+ return await serializers.TestSuiteTestCase.parseOrThrow(_response.body, {
55
+ unrecognizedObjectKeys: "passthrough",
56
+ allowUnrecognizedUnionMembers: true,
57
+ allowUnrecognizedEnumValues: true,
58
+ });
59
+ }
60
+ if (_response.error.reason === "status-code") {
61
+ throw new errors.VellumError({
62
+ statusCode: _response.error.statusCode,
63
+ body: _response.error.body,
64
+ });
65
+ }
66
+ switch (_response.error.reason) {
67
+ case "non-json":
68
+ throw new errors.VellumError({
69
+ statusCode: _response.error.statusCode,
70
+ body: _response.error.rawBody,
71
+ });
72
+ case "timeout":
73
+ throw new errors.VellumTimeoutError();
74
+ case "unknown":
75
+ throw new errors.VellumError({
76
+ message: _response.error.errorMessage,
77
+ });
78
+ }
79
+ }
80
+ }
81
+ exports.TestSuites = TestSuites;
@@ -0,0 +1 @@
1
+ export * from "./requests";
@@ -0,0 +1,17 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./requests"), exports);
@@ -0,0 +1,14 @@
1
+ /**
2
+ * This file was auto-generated by Fern from our API Definition.
3
+ */
4
+ import * as Vellum from "../../../..";
5
+ export interface TestSuiteTestCaseRequest {
6
+ /** The id of the test case to update. If none is provided, an id will be generated and a new test case will be appended. */
7
+ testCaseId?: string;
8
+ /** A human-friendly label for the test case. */
9
+ label?: string;
10
+ /** Key/value pairs for each input variable that the Test Suite expects. */
11
+ inputValues: Record<string, unknown>;
12
+ /** Parameters to use when evaluating the test case, specific to the test suite's evaluation metric. */
13
+ evaluationParams: Vellum.EvaluationParamsRequest;
14
+ }
@@ -0,0 +1 @@
1
+ export { TestSuiteTestCaseRequest } from "./TestSuiteTestCaseRequest";
@@ -0,0 +1,2 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
@@ -0,0 +1 @@
1
+ export * from "./client";
@@ -0,0 +1,17 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./client"), exports);
@@ -0,0 +1,7 @@
1
+ /**
2
+ * This file was auto-generated by Fern from our API Definition.
3
+ */
4
+ export interface EvaluationParams {
5
+ /** The target value to compare the LLM output against. Typically what you expect or desire the LLM output to be. */
6
+ target?: string;
7
+ }
@@ -0,0 +1,7 @@
1
+ /**
2
+ * This file was auto-generated by Fern from our API Definition.
3
+ */
4
+ export interface EvaluationParamsRequest {
5
+ /** The target value to compare the LLM output against. Typically what you expect or desire the LLM output to be. */
6
+ target?: string;
7
+ }
@@ -0,0 +1,7 @@
1
+ /**
2
+ * This file was auto-generated by Fern from our API Definition.
3
+ */
4
+ export declare type ProcessingFailureReasonEnum = "EXCEEDED_CHARACTER_LIMIT";
5
+ export declare const ProcessingFailureReasonEnum: {
6
+ readonly ExceededCharacterLimit: "EXCEEDED_CHARACTER_LIMIT";
7
+ };
@@ -0,0 +1,9 @@
1
+ "use strict";
2
+ /**
3
+ * This file was auto-generated by Fern from our API Definition.
4
+ */
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.ProcessingFailureReasonEnum = void 0;
7
+ exports.ProcessingFailureReasonEnum = {
8
+ ExceededCharacterLimit: "EXCEEDED_CHARACTER_LIMIT",
9
+ };
@@ -1,6 +1,7 @@
1
1
  /**
2
2
  * This file was auto-generated by Fern from our API Definition.
3
3
  */
4
+ import * as Vellum from "..";
4
5
  export interface SandboxMetricInputParams {
5
- params?: Record<string, unknown>;
6
+ params?: Vellum.EvaluationParams;
6
7
  }
@@ -1,6 +1,7 @@
1
1
  /**
2
2
  * This file was auto-generated by Fern from our API Definition.
3
3
  */
4
+ import * as Vellum from "..";
4
5
  export interface SandboxMetricInputParamsRequest {
5
- params?: Record<string, unknown>;
6
+ params?: Vellum.EvaluationParamsRequest;
6
7
  }
@@ -5,7 +5,7 @@ import * as Vellum from "..";
5
5
  export interface SandboxScenario {
6
6
  label?: string;
7
7
  /** The inputs for the scenario */
8
- inputs: Vellum.SandboxInput[];
8
+ inputs: Vellum.ScenarioInput[];
9
9
  /** The id of the scenario */
10
10
  id: string;
11
11
  metricInputParams: Vellum.SandboxMetricInputParams;
@@ -2,7 +2,7 @@
2
2
  * This file was auto-generated by Fern from our API Definition.
3
3
  */
4
4
  import * as Vellum from "..";
5
- export interface SandboxInput {
5
+ export interface ScenarioInput {
6
6
  key: string;
7
7
  type?: Vellum.TypeEnum;
8
8
  value?: string;
@@ -2,7 +2,7 @@
2
2
  * This file was auto-generated by Fern from our API Definition.
3
3
  */
4
4
  import * as Vellum from "..";
5
- export interface SandboxInputRequest {
5
+ export interface ScenarioInputRequest {
6
6
  key: string;
7
7
  type?: Vellum.TypeEnum;
8
8
  value?: string;
@@ -0,0 +1,5 @@
1
+ "use strict";
2
+ /**
3
+ * This file was auto-generated by Fern from our API Definition.
4
+ */
5
+ Object.defineProperty(exports, "__esModule", { value: true });
@@ -11,8 +11,10 @@ export interface SlimDocument {
11
11
  lastUploadedAt: string;
12
12
  /** Human-friendly name for this document. */
13
13
  label: string;
14
- /** The current processing state of the document */
14
+ /** An enum value representing where this document is along its processing lifecycle. Note that this is different than its indexing lifecycle. */
15
15
  processingState?: Vellum.ProcessingStateEnum;
16
+ /** An enum value representing why the document could not be processed. Is null unless processing_state is FAILED. */
17
+ processingFailureReason?: Vellum.ProcessingFailureReasonEnum;
16
18
  /** The document's current status. */
17
19
  status?: Vellum.SlimDocumentStatusEnum;
18
20
  /** A list of keywords associated with this document. Originally provided when uploading the document. */
@@ -0,0 +1,14 @@
1
+ /**
2
+ * This file was auto-generated by Fern from our API Definition.
3
+ */
4
+ import * as Vellum from "..";
5
+ export interface TestSuiteTestCase {
6
+ /** The id of the test case to update. If none is provided, an id will be generated and a new test case will be appended. */
7
+ testCaseId?: string;
8
+ /** A human-friendly label for the test case. */
9
+ label?: string;
10
+ /** Key/value pairs for each input variable that the Test Suite expects. */
11
+ inputValues: Record<string, unknown>;
12
+ /** Parameters to use when evaluating the test case, specific to the test suite's evaluation metric. */
13
+ evaluationParams: Vellum.EvaluationParams;
14
+ }
@@ -0,0 +1,5 @@
1
+ "use strict";
2
+ /**
3
+ * This file was auto-generated by Fern from our API Definition.
4
+ */
5
+ Object.defineProperty(exports, "__esModule", { value: true });
@@ -5,6 +5,8 @@ export * from "./ChatMessageRole";
5
5
  export * from "./Document";
6
6
  export * from "./DocumentDocumentToDocumentIndex";
7
7
  export * from "./EnrichedNormalizedCompletion";
8
+ export * from "./EvaluationParams";
9
+ export * from "./EvaluationParamsRequest";
8
10
  export * from "./FinishReasonEnum";
9
11
  export * from "./GenerateErrorResponse";
10
12
  export * from "./GenerateOptionsRequest";
@@ -25,16 +27,17 @@ export * from "./ModelVersionSandboxSnapshot";
25
27
  export * from "./NormalizedLogProbs";
26
28
  export * from "./NormalizedTokenLogProbs";
27
29
  export * from "./PaginatedSlimDocumentList";
30
+ export * from "./ProcessingFailureReasonEnum";
28
31
  export * from "./ProcessingStateEnum";
29
32
  export * from "./PromptTemplateBlock";
30
33
  export * from "./PromptTemplateBlockData";
31
34
  export * from "./PromptTemplateBlockProperties";
32
35
  export * from "./ProviderEnum";
33
- export * from "./SandboxInput";
34
- export * from "./SandboxInputRequest";
35
36
  export * from "./SandboxMetricInputParams";
36
37
  export * from "./SandboxMetricInputParamsRequest";
37
38
  export * from "./SandboxScenario";
39
+ export * from "./ScenarioInput";
40
+ export * from "./ScenarioInputRequest";
38
41
  export * from "./SearchErrorResponse";
39
42
  export * from "./SearchFiltersRequest";
40
43
  export * from "./SearchRequestOptionsRequest";
@@ -46,6 +49,7 @@ export * from "./SlimDocument";
46
49
  export * from "./SlimDocumentStatusEnum";
47
50
  export * from "./SubmitCompletionActualRequest";
48
51
  export * from "./SubmitCompletionActualsErrorResponse";
52
+ export * from "./TestSuiteTestCase";
49
53
  export * from "./TypeEnum";
50
54
  export * from "./UploadDocumentErrorResponse";
51
55
  export * from "./UploadDocumentResponse";
@@ -21,6 +21,8 @@ __exportStar(require("./ChatMessageRole"), exports);
21
21
  __exportStar(require("./Document"), exports);
22
22
  __exportStar(require("./DocumentDocumentToDocumentIndex"), exports);
23
23
  __exportStar(require("./EnrichedNormalizedCompletion"), exports);
24
+ __exportStar(require("./EvaluationParams"), exports);
25
+ __exportStar(require("./EvaluationParamsRequest"), exports);
24
26
  __exportStar(require("./FinishReasonEnum"), exports);
25
27
  __exportStar(require("./GenerateErrorResponse"), exports);
26
28
  __exportStar(require("./GenerateOptionsRequest"), exports);
@@ -41,16 +43,17 @@ __exportStar(require("./ModelVersionSandboxSnapshot"), exports);
41
43
  __exportStar(require("./NormalizedLogProbs"), exports);
42
44
  __exportStar(require("./NormalizedTokenLogProbs"), exports);
43
45
  __exportStar(require("./PaginatedSlimDocumentList"), exports);
46
+ __exportStar(require("./ProcessingFailureReasonEnum"), exports);
44
47
  __exportStar(require("./ProcessingStateEnum"), exports);
45
48
  __exportStar(require("./PromptTemplateBlock"), exports);
46
49
  __exportStar(require("./PromptTemplateBlockData"), exports);
47
50
  __exportStar(require("./PromptTemplateBlockProperties"), exports);
48
51
  __exportStar(require("./ProviderEnum"), exports);
49
- __exportStar(require("./SandboxInput"), exports);
50
- __exportStar(require("./SandboxInputRequest"), exports);
51
52
  __exportStar(require("./SandboxMetricInputParams"), exports);
52
53
  __exportStar(require("./SandboxMetricInputParamsRequest"), exports);
53
54
  __exportStar(require("./SandboxScenario"), exports);
55
+ __exportStar(require("./ScenarioInput"), exports);
56
+ __exportStar(require("./ScenarioInputRequest"), exports);
54
57
  __exportStar(require("./SearchErrorResponse"), exports);
55
58
  __exportStar(require("./SearchFiltersRequest"), exports);
56
59
  __exportStar(require("./SearchRequestOptionsRequest"), exports);
@@ -62,6 +65,7 @@ __exportStar(require("./SlimDocument"), exports);
62
65
  __exportStar(require("./SlimDocumentStatusEnum"), exports);
63
66
  __exportStar(require("./SubmitCompletionActualRequest"), exports);
64
67
  __exportStar(require("./SubmitCompletionActualsErrorResponse"), exports);
68
+ __exportStar(require("./TestSuiteTestCase"), exports);
65
69
  __exportStar(require("./TypeEnum"), exports);
66
70
  __exportStar(require("./UploadDocumentErrorResponse"), exports);
67
71
  __exportStar(require("./UploadDocumentResponse"), exports);
package/dist/Client.d.ts CHANGED
@@ -7,6 +7,7 @@ import * as Vellum from "./api";
7
7
  import { Documents } from "./api/resources/documents/client/Client";
8
8
  import { ModelVersions } from "./api/resources/modelVersions/client/Client";
9
9
  import { Sandboxes } from "./api/resources/sandboxes/client/Client";
10
+ import { TestSuites } from "./api/resources/testSuites/client/Client";
10
11
  export declare namespace VellumClient {
11
12
  interface Options {
12
13
  environment?: environments.VellumEnvironment | environments.VellumEnvironmentUrls;
@@ -25,4 +26,6 @@ export declare class VellumClient {
25
26
  get modelVersions(): ModelVersions;
26
27
  protected _sandboxes: Sandboxes | undefined;
27
28
  get sandboxes(): Sandboxes;
29
+ protected _testSuites: TestSuites | undefined;
30
+ get testSuites(): TestSuites;
28
31
  }
package/dist/Client.js CHANGED
@@ -38,6 +38,7 @@ const errors = __importStar(require("./errors"));
38
38
  const Client_1 = require("./api/resources/documents/client/Client");
39
39
  const Client_2 = require("./api/resources/modelVersions/client/Client");
40
40
  const Client_3 = require("./api/resources/sandboxes/client/Client");
41
+ const Client_4 = require("./api/resources/testSuites/client/Client");
41
42
  class VellumClient {
42
43
  options;
43
44
  constructor(options) {
@@ -164,5 +165,9 @@ class VellumClient {
164
165
  get sandboxes() {
165
166
  return (this._sandboxes ??= new Client_3.Sandboxes(this.options));
166
167
  }
168
+ _testSuites;
169
+ get testSuites() {
170
+ return (this._testSuites ??= new Client_4.TestSuites(this.options));
171
+ }
167
172
  }
168
173
  exports.VellumClient = VellumClient;
@@ -1,5 +1,7 @@
1
1
  export * as documents from "./documents";
2
2
  export * as modelVersions from "./modelVersions";
3
3
  export * as sandboxes from "./sandboxes";
4
+ export * as testSuites from "./testSuites";
4
5
  export * from "./documents/client/requests";
5
6
  export * from "./sandboxes/client/requests";
7
+ export * from "./testSuites/client/requests";
@@ -26,9 +26,11 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
26
26
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
27
27
  };
28
28
  Object.defineProperty(exports, "__esModule", { value: true });
29
- exports.sandboxes = exports.modelVersions = exports.documents = void 0;
29
+ exports.testSuites = exports.sandboxes = exports.modelVersions = exports.documents = void 0;
30
30
  exports.documents = __importStar(require("./documents"));
31
31
  exports.modelVersions = __importStar(require("./modelVersions"));
32
32
  exports.sandboxes = __importStar(require("./sandboxes"));
33
+ exports.testSuites = __importStar(require("./testSuites"));
33
34
  __exportStar(require("./documents/client/requests"), exports);
34
35
  __exportStar(require("./sandboxes/client/requests"), exports);
36
+ __exportStar(require("./testSuites/client/requests"), exports);
@@ -5,7 +5,7 @@ import * as Vellum from "../../../..";
5
5
  export interface UpsertSandboxScenarioRequestRequest {
6
6
  label?: string;
7
7
  /** The inputs for the scenario */
8
- inputs: Vellum.SandboxInputRequest[];
8
+ inputs: Vellum.ScenarioInputRequest[];
9
9
  /** The id of the scenario to update. If none is provided, an id will be generated and a new scenario will be appended. */
10
10
  scenarioId?: string;
11
11
  metricInputParams?: Vellum.SandboxMetricInputParamsRequest;
@@ -0,0 +1,17 @@
1
+ /**
2
+ * This file was auto-generated by Fern from our API Definition.
3
+ */
4
+ import * as environments from "../../../../environments";
5
+ import * as core from "../../../../core";
6
+ import * as Vellum from "../../..";
7
+ export declare namespace TestSuites {
8
+ interface Options {
9
+ environment?: environments.VellumEnvironment | environments.VellumEnvironmentUrls;
10
+ apiKey: core.Supplier<string>;
11
+ }
12
+ }
13
+ export declare class TestSuites {
14
+ protected readonly options: TestSuites.Options;
15
+ constructor(options: TestSuites.Options);
16
+ upsertTestSuiteTestCase(id: string, request: Vellum.TestSuiteTestCaseRequest): Promise<Vellum.TestSuiteTestCase>;
17
+ }
@@ -0,0 +1,81 @@
1
+ "use strict";
2
+ /**
3
+ * This file was auto-generated by Fern from our API Definition.
4
+ */
5
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
6
+ if (k2 === undefined) k2 = k;
7
+ var desc = Object.getOwnPropertyDescriptor(m, k);
8
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
9
+ desc = { enumerable: true, get: function() { return m[k]; } };
10
+ }
11
+ Object.defineProperty(o, k2, desc);
12
+ }) : (function(o, m, k, k2) {
13
+ if (k2 === undefined) k2 = k;
14
+ o[k2] = m[k];
15
+ }));
16
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
17
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
18
+ }) : function(o, v) {
19
+ o["default"] = v;
20
+ });
21
+ var __importStar = (this && this.__importStar) || function (mod) {
22
+ if (mod && mod.__esModule) return mod;
23
+ var result = {};
24
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
25
+ __setModuleDefault(result, mod);
26
+ return result;
27
+ };
28
+ var __importDefault = (this && this.__importDefault) || function (mod) {
29
+ return (mod && mod.__esModule) ? mod : { "default": mod };
30
+ };
31
+ Object.defineProperty(exports, "__esModule", { value: true });
32
+ exports.TestSuites = void 0;
33
+ const environments = __importStar(require("../../../../environments"));
34
+ const core = __importStar(require("../../../../core"));
35
+ const url_join_1 = __importDefault(require("url-join"));
36
+ const serializers = __importStar(require("../../../../serialization"));
37
+ const errors = __importStar(require("../../../../errors"));
38
+ class TestSuites {
39
+ options;
40
+ constructor(options) {
41
+ this.options = options;
42
+ }
43
+ async upsertTestSuiteTestCase(id, request) {
44
+ const _response = await core.fetcher({
45
+ url: (0, url_join_1.default)((this.options.environment ?? environments.VellumEnvironment.Production).default, `v1/test-suites/${id}/test-cases`),
46
+ method: "POST",
47
+ headers: {
48
+ X_API_KEY: await core.Supplier.get(this.options.apiKey),
49
+ },
50
+ contentType: "application/json",
51
+ body: await serializers.TestSuiteTestCaseRequest.jsonOrThrow(request, { unrecognizedObjectKeys: "strip" }),
52
+ });
53
+ if (_response.ok) {
54
+ return await serializers.TestSuiteTestCase.parseOrThrow(_response.body, {
55
+ unrecognizedObjectKeys: "passthrough",
56
+ allowUnrecognizedUnionMembers: true,
57
+ allowUnrecognizedEnumValues: true,
58
+ });
59
+ }
60
+ if (_response.error.reason === "status-code") {
61
+ throw new errors.VellumError({
62
+ statusCode: _response.error.statusCode,
63
+ body: _response.error.body,
64
+ });
65
+ }
66
+ switch (_response.error.reason) {
67
+ case "non-json":
68
+ throw new errors.VellumError({
69
+ statusCode: _response.error.statusCode,
70
+ body: _response.error.rawBody,
71
+ });
72
+ case "timeout":
73
+ throw new errors.VellumTimeoutError();
74
+ case "unknown":
75
+ throw new errors.VellumError({
76
+ message: _response.error.errorMessage,
77
+ });
78
+ }
79
+ }
80
+ }
81
+ exports.TestSuites = TestSuites;
@@ -0,0 +1 @@
1
+ export * from "./requests";
@@ -0,0 +1,17 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./requests"), exports);