vellum-ai 0.0.21__py3-none-any.whl → 0.0.22__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. vellum/__init__.py +19 -19
  2. vellum/client.py +270 -101
  3. vellum/core/__init__.py +11 -2
  4. vellum/core/client_wrapper.py +27 -0
  5. vellum/core/remove_none_from_dict.py +11 -0
  6. vellum/resources/deployments/client.py +35 -15
  7. vellum/resources/document_indexes/client.py +64 -16
  8. vellum/resources/documents/client.py +110 -35
  9. vellum/resources/model_versions/client.py +67 -25
  10. vellum/resources/registered_prompts/client.py +80 -16
  11. vellum/resources/sandboxes/client.py +90 -25
  12. vellum/resources/test_suites/client.py +90 -25
  13. vellum/types/deployment_read.py +2 -6
  14. vellum/types/document.py +3 -7
  15. vellum/types/document_document_to_document_index.py +2 -2
  16. vellum/types/document_index_read.py +3 -7
  17. vellum/types/enriched_normalized_completion.py +5 -9
  18. vellum/types/evaluation_params.py +1 -3
  19. vellum/types/evaluation_params_request.py +1 -3
  20. vellum/types/generate_error_response.py +1 -1
  21. vellum/types/generate_request.py +3 -7
  22. vellum/types/generate_result.py +2 -6
  23. vellum/types/generate_result_data.py +1 -1
  24. vellum/types/generate_result_error.py +1 -1
  25. vellum/types/model_version_build_config.py +2 -6
  26. vellum/types/model_version_compile_prompt_response.py +1 -1
  27. vellum/types/model_version_compiled_prompt.py +2 -4
  28. vellum/types/model_version_exec_config.py +3 -3
  29. vellum/types/model_version_read.py +6 -10
  30. vellum/types/model_version_sandbox_snapshot.py +3 -5
  31. vellum/types/prompt_template_block_properties_request.py +2 -2
  32. vellum/types/prompt_template_block_request.py +1 -1
  33. vellum/types/prompt_template_input_variable.py +1 -1
  34. vellum/types/prompt_template_input_variable_request.py +1 -1
  35. vellum/types/register_prompt_error_response.py +1 -1
  36. vellum/types/register_prompt_prompt.py +2 -2
  37. vellum/types/register_prompt_prompt_info_request.py +1 -1
  38. vellum/types/register_prompt_response.py +5 -7
  39. vellum/types/registered_prompt_deployment.py +3 -3
  40. vellum/types/registered_prompt_model_version.py +2 -2
  41. vellum/types/registered_prompt_sandbox.py +2 -2
  42. vellum/types/registered_prompt_sandbox_snapshot.py +1 -1
  43. vellum/types/sandbox_scenario.py +2 -2
  44. vellum/types/scenario_input_request.py +1 -1
  45. vellum/types/search_error_response.py +1 -1
  46. vellum/types/search_filters_request.py +1 -1
  47. vellum/types/search_request_options_request.py +4 -6
  48. vellum/types/search_response.py +1 -1
  49. vellum/types/search_result.py +3 -3
  50. vellum/types/search_result_merging_request.py +1 -1
  51. vellum/types/search_weights_request.py +2 -2
  52. vellum/types/slim_document.py +5 -9
  53. vellum/types/submit_completion_actual_request.py +5 -15
  54. vellum/types/terminal_node_chat_history_result.py +1 -1
  55. vellum/types/terminal_node_json_result.py +1 -1
  56. vellum/types/terminal_node_result_output.py +2 -4
  57. vellum/types/terminal_node_string_result.py +1 -1
  58. vellum/types/test_suite_test_case.py +4 -8
  59. vellum/types/upload_document_response.py +1 -1
  60. vellum/types/workflow_node_result_data.py +7 -11
  61. vellum/types/workflow_request_chat_history_input_request.py +1 -3
  62. vellum/types/workflow_request_input_request.py +2 -6
  63. vellum/types/workflow_request_json_input_request.py +1 -3
  64. vellum/types/workflow_request_string_input_request.py +1 -3
  65. vellum/types/workflow_result_event_output_data.py +2 -8
  66. vellum/types/workflow_result_event_output_data_chat_history.py +3 -0
  67. vellum/types/workflow_result_event_output_data_json.py +3 -0
  68. vellum/types/workflow_result_event_output_data_string.py +6 -1
  69. vellum/types/workflow_stream_event.py +1 -4
  70. {vellum_ai-0.0.21.dist-info → vellum_ai-0.0.22.dist-info}/METADATA +1 -1
  71. {vellum_ai-0.0.21.dist-info → vellum_ai-0.0.22.dist-info}/RECORD +72 -71
  72. vellum/core/remove_none_from_headers.py +0 -11
  73. {vellum_ai-0.0.21.dist-info → vellum_ai-0.0.22.dist-info}/WHEEL +0 -0
@@ -4,12 +4,11 @@ import typing
4
4
  import urllib.parse
5
5
  from json.decoder import JSONDecodeError
6
6
 
7
- import httpx
8
7
  import pydantic
9
8
 
10
9
  from ...core.api_error import ApiError
10
+ from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
11
11
  from ...core.jsonable_encoder import jsonable_encoder
12
- from ...core.remove_none_from_headers import remove_none_from_headers
13
12
  from ...environment import VellumEnvironment
14
13
  from ...errors.conflict_error import ConflictError
15
14
  from ...types.provider_enum import ProviderEnum
@@ -23,9 +22,11 @@ OMIT = typing.cast(typing.Any, ...)
23
22
 
24
23
 
25
24
  class RegisteredPromptsClient:
26
- def __init__(self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, api_key: str):
25
+ def __init__(
26
+ self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, client_wrapper: SyncClientWrapper
27
+ ):
27
28
  self._environment = environment
28
- self.api_key = api_key
29
+ self._client_wrapper = client_wrapper
29
30
 
30
31
  def register_prompt(
31
32
  self,
@@ -38,6 +39,37 @@ class RegisteredPromptsClient:
38
39
  parameters: RegisterPromptModelParametersRequest,
39
40
  meta: typing.Optional[typing.Dict[str, typing.Any]] = OMIT,
40
41
  ) -> RegisterPromptResponse:
42
+ """
43
+ <strong style="background-color:#ffc107; color:white; padding:4px; border-radius:4px">Unstable</strong>
44
+
45
+ Registers a prompt within Vellum and creates associated Vellum entities. Intended to be used by integration
46
+ partners, not directly by Vellum users.
47
+
48
+ Under the hood, this endpoint creates a new sandbox, a new model version, and a new deployment.
49
+
50
+ Parameters:
51
+ - label: str. A human-friendly label for corresponding entities created in Vellum. <span style="white-space: nowrap">`non-empty`</span>
52
+
53
+ - name: str. A uniquely-identifying name for corresponding entities created in Vellum. <span style="white-space: nowrap">`non-empty`</span>
54
+
55
+ - prompt: RegisterPromptPromptInfoRequest. Information about how to execute the prompt template.
56
+
57
+ - provider: ProviderEnum. The initial LLM provider to use for this prompt
58
+
59
+ * `ANTHROPIC` - Anthropic
60
+ * `COHERE` - Cohere
61
+ * `GOOGLE` - Google
62
+ * `HOSTED` - Hosted
63
+ * `MOSAICML` - MosaicML
64
+ * `MYSTIC` - Mystic
65
+ * `OPENAI` - OpenAI
66
+ * `PYQ` - Pyq
67
+ - model: str. The initial model to use for this prompt <span style="white-space: nowrap">`non-empty`</span>
68
+
69
+ - parameters: RegisterPromptModelParametersRequest. The initial model parameters to use for this prompt
70
+
71
+ - meta: typing.Optional[typing.Dict[str, typing.Any]]. Optionally include additional metadata to store along with the prompt.
72
+ """
41
73
  _request: typing.Dict[str, typing.Any] = {
42
74
  "label": label,
43
75
  "name": name,
@@ -48,11 +80,11 @@ class RegisteredPromptsClient:
48
80
  }
49
81
  if meta is not OMIT:
50
82
  _request["meta"] = meta
51
- _response = httpx.request(
83
+ _response = self._client_wrapper.httpx_client.request(
52
84
  "POST",
53
85
  urllib.parse.urljoin(f"{self._environment.default}/", "v1/registered-prompts/register"),
54
86
  json=jsonable_encoder(_request),
55
- headers=remove_none_from_headers({"X_API_KEY": self.api_key}),
87
+ headers=self._client_wrapper.get_headers(),
56
88
  timeout=None,
57
89
  )
58
90
  if 200 <= _response.status_code < 300:
@@ -67,9 +99,11 @@ class RegisteredPromptsClient:
67
99
 
68
100
 
69
101
  class AsyncRegisteredPromptsClient:
70
- def __init__(self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, api_key: str):
102
+ def __init__(
103
+ self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, client_wrapper: AsyncClientWrapper
104
+ ):
71
105
  self._environment = environment
72
- self.api_key = api_key
106
+ self._client_wrapper = client_wrapper
73
107
 
74
108
  async def register_prompt(
75
109
  self,
@@ -82,6 +116,37 @@ class AsyncRegisteredPromptsClient:
82
116
  parameters: RegisterPromptModelParametersRequest,
83
117
  meta: typing.Optional[typing.Dict[str, typing.Any]] = OMIT,
84
118
  ) -> RegisterPromptResponse:
119
+ """
120
+ <strong style="background-color:#ffc107; color:white; padding:4px; border-radius:4px">Unstable</strong>
121
+
122
+ Registers a prompt within Vellum and creates associated Vellum entities. Intended to be used by integration
123
+ partners, not directly by Vellum users.
124
+
125
+ Under the hood, this endpoint creates a new sandbox, a new model version, and a new deployment.
126
+
127
+ Parameters:
128
+ - label: str. A human-friendly label for corresponding entities created in Vellum. <span style="white-space: nowrap">`non-empty`</span>
129
+
130
+ - name: str. A uniquely-identifying name for corresponding entities created in Vellum. <span style="white-space: nowrap">`non-empty`</span>
131
+
132
+ - prompt: RegisterPromptPromptInfoRequest. Information about how to execute the prompt template.
133
+
134
+ - provider: ProviderEnum. The initial LLM provider to use for this prompt
135
+
136
+ * `ANTHROPIC` - Anthropic
137
+ * `COHERE` - Cohere
138
+ * `GOOGLE` - Google
139
+ * `HOSTED` - Hosted
140
+ * `MOSAICML` - MosaicML
141
+ * `MYSTIC` - Mystic
142
+ * `OPENAI` - OpenAI
143
+ * `PYQ` - Pyq
144
+ - model: str. The initial model to use for this prompt <span style="white-space: nowrap">`non-empty`</span>
145
+
146
+ - parameters: RegisterPromptModelParametersRequest. The initial model parameters to use for this prompt
147
+
148
+ - meta: typing.Optional[typing.Dict[str, typing.Any]]. Optionally include additional metadata to store along with the prompt.
149
+ """
85
150
  _request: typing.Dict[str, typing.Any] = {
86
151
  "label": label,
87
152
  "name": name,
@@ -92,14 +157,13 @@ class AsyncRegisteredPromptsClient:
92
157
  }
93
158
  if meta is not OMIT:
94
159
  _request["meta"] = meta
95
- async with httpx.AsyncClient() as _client:
96
- _response = await _client.request(
97
- "POST",
98
- urllib.parse.urljoin(f"{self._environment.default}/", "v1/registered-prompts/register"),
99
- json=jsonable_encoder(_request),
100
- headers=remove_none_from_headers({"X_API_KEY": self.api_key}),
101
- timeout=None,
102
- )
160
+ _response = await self._client_wrapper.httpx_client.request(
161
+ "POST",
162
+ urllib.parse.urljoin(f"{self._environment.default}/", "v1/registered-prompts/register"),
163
+ json=jsonable_encoder(_request),
164
+ headers=self._client_wrapper.get_headers(),
165
+ timeout=None,
166
+ )
103
167
  if 200 <= _response.status_code < 300:
104
168
  return pydantic.parse_obj_as(RegisterPromptResponse, _response.json()) # type: ignore
105
169
  if _response.status_code == 409:
@@ -4,12 +4,11 @@ import typing
4
4
  import urllib.parse
5
5
  from json.decoder import JSONDecodeError
6
6
 
7
- import httpx
8
7
  import pydantic
9
8
 
10
9
  from ...core.api_error import ApiError
10
+ from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
11
11
  from ...core.jsonable_encoder import jsonable_encoder
12
- from ...core.remove_none_from_headers import remove_none_from_headers
13
12
  from ...environment import VellumEnvironment
14
13
  from ...types.sandbox_metric_input_params_request import SandboxMetricInputParamsRequest
15
14
  from ...types.sandbox_scenario import SandboxScenario
@@ -20,9 +19,11 @@ OMIT = typing.cast(typing.Any, ...)
20
19
 
21
20
 
22
21
  class SandboxesClient:
23
- def __init__(self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, api_key: str):
22
+ def __init__(
23
+ self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, client_wrapper: SyncClientWrapper
24
+ ):
24
25
  self._environment = environment
25
- self.api_key = api_key
26
+ self._client_wrapper = client_wrapper
26
27
 
27
28
  def upsert_sandbox_scenario(
28
29
  self,
@@ -33,6 +34,28 @@ class SandboxesClient:
33
34
  scenario_id: typing.Optional[str] = OMIT,
34
35
  metric_input_params: typing.Optional[SandboxMetricInputParamsRequest] = OMIT,
35
36
  ) -> SandboxScenario:
37
+ """
38
+ <strong style="background-color:#ffc107; color:white; padding:4px; border-radius:4px">Unstable</strong>
39
+
40
+ Upserts a new scenario for a sandbox, keying off of the optionally provided scenario id.
41
+
42
+ If an id is provided and has a match, the scenario will be updated. If no id is provided or no match
43
+ is found, a new scenario will be appended to the end.
44
+
45
+ Note that a full replacement of the scenario is performed, so any fields not provided will be removed
46
+ or overwritten with default values.
47
+
48
+ Parameters:
49
+ - id: str. A UUID string identifying this sandbox.
50
+
51
+ - label: typing.Optional[str].
52
+
53
+ - inputs: typing.List[ScenarioInputRequest]. The inputs for the scenario
54
+
55
+ - scenario_id: typing.Optional[str]. The id of the scenario to update. If none is provided, an id will be generated and a new scenario will be appended. <span style="white-space: nowrap">`non-empty`</span>
56
+
57
+ - metric_input_params: typing.Optional[SandboxMetricInputParamsRequest].
58
+ """
36
59
  _request: typing.Dict[str, typing.Any] = {"inputs": inputs}
37
60
  if label is not OMIT:
38
61
  _request["label"] = label
@@ -40,11 +63,11 @@ class SandboxesClient:
40
63
  _request["scenario_id"] = scenario_id
41
64
  if metric_input_params is not OMIT:
42
65
  _request["metric_input_params"] = metric_input_params
43
- _response = httpx.request(
66
+ _response = self._client_wrapper.httpx_client.request(
44
67
  "POST",
45
68
  urllib.parse.urljoin(f"{self._environment.default}/", f"v1/sandboxes/{id}/scenarios"),
46
69
  json=jsonable_encoder(_request),
47
- headers=remove_none_from_headers({"X_API_KEY": self.api_key}),
70
+ headers=self._client_wrapper.get_headers(),
48
71
  timeout=None,
49
72
  )
50
73
  if 200 <= _response.status_code < 300:
@@ -56,10 +79,20 @@ class SandboxesClient:
56
79
  raise ApiError(status_code=_response.status_code, body=_response_json)
57
80
 
58
81
  def delete_sandbox_scenario(self, id: str, scenario_id: str) -> None:
59
- _response = httpx.request(
82
+ """
83
+ <strong style="background-color:#ffc107; color:white; padding:4px; border-radius:4px">Unstable</strong>
84
+
85
+ Deletes an existing scenario from a sandbox, keying off of the provided scenario id.
86
+
87
+ Parameters:
88
+ - id: str. A UUID string identifying this sandbox.
89
+
90
+ - scenario_id: str. An id identifying the scenario that you'd like to delete
91
+ """
92
+ _response = self._client_wrapper.httpx_client.request(
60
93
  "DELETE",
61
94
  urllib.parse.urljoin(f"{self._environment.default}/", f"v1/sandboxes/{id}/scenarios/{scenario_id}"),
62
- headers=remove_none_from_headers({"X_API_KEY": self.api_key}),
95
+ headers=self._client_wrapper.get_headers(),
63
96
  timeout=None,
64
97
  )
65
98
  if 200 <= _response.status_code < 300:
@@ -72,9 +105,11 @@ class SandboxesClient:
72
105
 
73
106
 
74
107
  class AsyncSandboxesClient:
75
- def __init__(self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, api_key: str):
108
+ def __init__(
109
+ self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, client_wrapper: AsyncClientWrapper
110
+ ):
76
111
  self._environment = environment
77
- self.api_key = api_key
112
+ self._client_wrapper = client_wrapper
78
113
 
79
114
  async def upsert_sandbox_scenario(
80
115
  self,
@@ -85,6 +120,28 @@ class AsyncSandboxesClient:
85
120
  scenario_id: typing.Optional[str] = OMIT,
86
121
  metric_input_params: typing.Optional[SandboxMetricInputParamsRequest] = OMIT,
87
122
  ) -> SandboxScenario:
123
+ """
124
+ <strong style="background-color:#ffc107; color:white; padding:4px; border-radius:4px">Unstable</strong>
125
+
126
+ Upserts a new scenario for a sandbox, keying off of the optionally provided scenario id.
127
+
128
+ If an id is provided and has a match, the scenario will be updated. If no id is provided or no match
129
+ is found, a new scenario will be appended to the end.
130
+
131
+ Note that a full replacement of the scenario is performed, so any fields not provided will be removed
132
+ or overwritten with default values.
133
+
134
+ Parameters:
135
+ - id: str. A UUID string identifying this sandbox.
136
+
137
+ - label: typing.Optional[str].
138
+
139
+ - inputs: typing.List[ScenarioInputRequest]. The inputs for the scenario
140
+
141
+ - scenario_id: typing.Optional[str]. The id of the scenario to update. If none is provided, an id will be generated and a new scenario will be appended. <span style="white-space: nowrap">`non-empty`</span>
142
+
143
+ - metric_input_params: typing.Optional[SandboxMetricInputParamsRequest].
144
+ """
88
145
  _request: typing.Dict[str, typing.Any] = {"inputs": inputs}
89
146
  if label is not OMIT:
90
147
  _request["label"] = label
@@ -92,14 +149,13 @@ class AsyncSandboxesClient:
92
149
  _request["scenario_id"] = scenario_id
93
150
  if metric_input_params is not OMIT:
94
151
  _request["metric_input_params"] = metric_input_params
95
- async with httpx.AsyncClient() as _client:
96
- _response = await _client.request(
97
- "POST",
98
- urllib.parse.urljoin(f"{self._environment.default}/", f"v1/sandboxes/{id}/scenarios"),
99
- json=jsonable_encoder(_request),
100
- headers=remove_none_from_headers({"X_API_KEY": self.api_key}),
101
- timeout=None,
102
- )
152
+ _response = await self._client_wrapper.httpx_client.request(
153
+ "POST",
154
+ urllib.parse.urljoin(f"{self._environment.default}/", f"v1/sandboxes/{id}/scenarios"),
155
+ json=jsonable_encoder(_request),
156
+ headers=self._client_wrapper.get_headers(),
157
+ timeout=None,
158
+ )
103
159
  if 200 <= _response.status_code < 300:
104
160
  return pydantic.parse_obj_as(SandboxScenario, _response.json()) # type: ignore
105
161
  try:
@@ -109,13 +165,22 @@ class AsyncSandboxesClient:
109
165
  raise ApiError(status_code=_response.status_code, body=_response_json)
110
166
 
111
167
  async def delete_sandbox_scenario(self, id: str, scenario_id: str) -> None:
112
- async with httpx.AsyncClient() as _client:
113
- _response = await _client.request(
114
- "DELETE",
115
- urllib.parse.urljoin(f"{self._environment.default}/", f"v1/sandboxes/{id}/scenarios/{scenario_id}"),
116
- headers=remove_none_from_headers({"X_API_KEY": self.api_key}),
117
- timeout=None,
118
- )
168
+ """
169
+ <strong style="background-color:#ffc107; color:white; padding:4px; border-radius:4px">Unstable</strong>
170
+
171
+ Deletes an existing scenario from a sandbox, keying off of the provided scenario id.
172
+
173
+ Parameters:
174
+ - id: str. A UUID string identifying this sandbox.
175
+
176
+ - scenario_id: str. An id identifying the scenario that you'd like to delete
177
+ """
178
+ _response = await self._client_wrapper.httpx_client.request(
179
+ "DELETE",
180
+ urllib.parse.urljoin(f"{self._environment.default}/", f"v1/sandboxes/{id}/scenarios/{scenario_id}"),
181
+ headers=self._client_wrapper.get_headers(),
182
+ timeout=None,
183
+ )
119
184
  if 200 <= _response.status_code < 300:
120
185
  return
121
186
  try:
@@ -4,12 +4,11 @@ import typing
4
4
  import urllib.parse
5
5
  from json.decoder import JSONDecodeError
6
6
 
7
- import httpx
8
7
  import pydantic
9
8
 
10
9
  from ...core.api_error import ApiError
10
+ from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
11
11
  from ...core.jsonable_encoder import jsonable_encoder
12
- from ...core.remove_none_from_headers import remove_none_from_headers
13
12
  from ...environment import VellumEnvironment
14
13
  from ...types.evaluation_params_request import EvaluationParamsRequest
15
14
  from ...types.test_suite_test_case import TestSuiteTestCase
@@ -19,9 +18,11 @@ OMIT = typing.cast(typing.Any, ...)
19
18
 
20
19
 
21
20
  class TestSuitesClient:
22
- def __init__(self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, api_key: str):
21
+ def __init__(
22
+ self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, client_wrapper: SyncClientWrapper
23
+ ):
23
24
  self._environment = environment
24
- self.api_key = api_key
25
+ self._client_wrapper = client_wrapper
25
26
 
26
27
  def upsert_test_suite_test_case(
27
28
  self,
@@ -32,16 +33,38 @@ class TestSuitesClient:
32
33
  input_values: typing.Dict[str, typing.Any],
33
34
  evaluation_params: EvaluationParamsRequest,
34
35
  ) -> TestSuiteTestCase:
36
+ """
37
+ <strong style="background-color:#ffc107; color:white; padding:4px; border-radius:4px">Unstable</strong>
38
+
39
+ Upserts a new test case for a test suite, keying off of the optionally provided test case id.
40
+
41
+ If an id is provided and has a match, the test case will be updated. If no id is provided or no match
42
+ is found, a new test case will be appended to the end.
43
+
44
+ Note that a full replacement of the test case is performed, so any fields not provided will be removed
45
+ or overwritten with default values.
46
+
47
+ Parameters:
48
+ - id: str. A UUID string identifying this test suite.
49
+
50
+ - test_case_id: typing.Optional[str]. The id of the test case to update. If none is provided, an id will be generated and a new test case will be appended. <span style="white-space: nowrap">`non-empty`</span>
51
+
52
+ - label: typing.Optional[str]. A human-friendly label for the test case.
53
+
54
+ - input_values: typing.Dict[str, typing.Any]. Key/value pairs for each input variable that the Test Suite expects.
55
+
56
+ - evaluation_params: EvaluationParamsRequest. Parameters to use when evaluating the test case, specific to the test suite's evaluation metric.
57
+ """
35
58
  _request: typing.Dict[str, typing.Any] = {"input_values": input_values, "evaluation_params": evaluation_params}
36
59
  if test_case_id is not OMIT:
37
60
  _request["test_case_id"] = test_case_id
38
61
  if label is not OMIT:
39
62
  _request["label"] = label
40
- _response = httpx.request(
63
+ _response = self._client_wrapper.httpx_client.request(
41
64
  "POST",
42
65
  urllib.parse.urljoin(f"{self._environment.default}/", f"v1/test-suites/{id}/test-cases"),
43
66
  json=jsonable_encoder(_request),
44
- headers=remove_none_from_headers({"X_API_KEY": self.api_key}),
67
+ headers=self._client_wrapper.get_headers(),
45
68
  timeout=None,
46
69
  )
47
70
  if 200 <= _response.status_code < 300:
@@ -53,10 +76,20 @@ class TestSuitesClient:
53
76
  raise ApiError(status_code=_response.status_code, body=_response_json)
54
77
 
55
78
  def delete_test_suite_test_case(self, id: str, test_case_id: str) -> None:
56
- _response = httpx.request(
79
+ """
80
+ <strong style="background-color:#ffc107; color:white; padding:4px; border-radius:4px">Unstable</strong>
81
+
82
+ Deletes an existing test case for a test suite, keying off of the test case id.
83
+
84
+ Parameters:
85
+ - id: str. A UUID string identifying this test suite.
86
+
87
+ - test_case_id: str. An id identifying the test case that you'd like to delete
88
+ """
89
+ _response = self._client_wrapper.httpx_client.request(
57
90
  "DELETE",
58
91
  urllib.parse.urljoin(f"{self._environment.default}/", f"v1/test-suites/{id}/test-cases/{test_case_id}"),
59
- headers=remove_none_from_headers({"X_API_KEY": self.api_key}),
92
+ headers=self._client_wrapper.get_headers(),
60
93
  timeout=None,
61
94
  )
62
95
  if 200 <= _response.status_code < 300:
@@ -69,9 +102,11 @@ class TestSuitesClient:
69
102
 
70
103
 
71
104
  class AsyncTestSuitesClient:
72
- def __init__(self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, api_key: str):
105
+ def __init__(
106
+ self, *, environment: VellumEnvironment = VellumEnvironment.PRODUCTION, client_wrapper: AsyncClientWrapper
107
+ ):
73
108
  self._environment = environment
74
- self.api_key = api_key
109
+ self._client_wrapper = client_wrapper
75
110
 
76
111
  async def upsert_test_suite_test_case(
77
112
  self,
@@ -82,19 +117,40 @@ class AsyncTestSuitesClient:
82
117
  input_values: typing.Dict[str, typing.Any],
83
118
  evaluation_params: EvaluationParamsRequest,
84
119
  ) -> TestSuiteTestCase:
120
+ """
121
+ <strong style="background-color:#ffc107; color:white; padding:4px; border-radius:4px">Unstable</strong>
122
+
123
+ Upserts a new test case for a test suite, keying off of the optionally provided test case id.
124
+
125
+ If an id is provided and has a match, the test case will be updated. If no id is provided or no match
126
+ is found, a new test case will be appended to the end.
127
+
128
+ Note that a full replacement of the test case is performed, so any fields not provided will be removed
129
+ or overwritten with default values.
130
+
131
+ Parameters:
132
+ - id: str. A UUID string identifying this test suite.
133
+
134
+ - test_case_id: typing.Optional[str]. The id of the test case to update. If none is provided, an id will be generated and a new test case will be appended. <span style="white-space: nowrap">`non-empty`</span>
135
+
136
+ - label: typing.Optional[str]. A human-friendly label for the test case.
137
+
138
+ - input_values: typing.Dict[str, typing.Any]. Key/value pairs for each input variable that the Test Suite expects.
139
+
140
+ - evaluation_params: EvaluationParamsRequest. Parameters to use when evaluating the test case, specific to the test suite's evaluation metric.
141
+ """
85
142
  _request: typing.Dict[str, typing.Any] = {"input_values": input_values, "evaluation_params": evaluation_params}
86
143
  if test_case_id is not OMIT:
87
144
  _request["test_case_id"] = test_case_id
88
145
  if label is not OMIT:
89
146
  _request["label"] = label
90
- async with httpx.AsyncClient() as _client:
91
- _response = await _client.request(
92
- "POST",
93
- urllib.parse.urljoin(f"{self._environment.default}/", f"v1/test-suites/{id}/test-cases"),
94
- json=jsonable_encoder(_request),
95
- headers=remove_none_from_headers({"X_API_KEY": self.api_key}),
96
- timeout=None,
97
- )
147
+ _response = await self._client_wrapper.httpx_client.request(
148
+ "POST",
149
+ urllib.parse.urljoin(f"{self._environment.default}/", f"v1/test-suites/{id}/test-cases"),
150
+ json=jsonable_encoder(_request),
151
+ headers=self._client_wrapper.get_headers(),
152
+ timeout=None,
153
+ )
98
154
  if 200 <= _response.status_code < 300:
99
155
  return pydantic.parse_obj_as(TestSuiteTestCase, _response.json()) # type: ignore
100
156
  try:
@@ -104,13 +160,22 @@ class AsyncTestSuitesClient:
104
160
  raise ApiError(status_code=_response.status_code, body=_response_json)
105
161
 
106
162
  async def delete_test_suite_test_case(self, id: str, test_case_id: str) -> None:
107
- async with httpx.AsyncClient() as _client:
108
- _response = await _client.request(
109
- "DELETE",
110
- urllib.parse.urljoin(f"{self._environment.default}/", f"v1/test-suites/{id}/test-cases/{test_case_id}"),
111
- headers=remove_none_from_headers({"X_API_KEY": self.api_key}),
112
- timeout=None,
113
- )
163
+ """
164
+ <strong style="background-color:#ffc107; color:white; padding:4px; border-radius:4px">Unstable</strong>
165
+
166
+ Deletes an existing test case for a test suite, keying off of the test case id.
167
+
168
+ Parameters:
169
+ - id: str. A UUID string identifying this test suite.
170
+
171
+ - test_case_id: str. An id identifying the test case that you'd like to delete
172
+ """
173
+ _response = await self._client_wrapper.httpx_client.request(
174
+ "DELETE",
175
+ urllib.parse.urljoin(f"{self._environment.default}/", f"v1/test-suites/{id}/test-cases/{test_case_id}"),
176
+ headers=self._client_wrapper.get_headers(),
177
+ timeout=None,
178
+ )
114
179
  if 200 <= _response.status_code < 300:
115
180
  return
116
181
  try:
@@ -15,14 +15,10 @@ class DeploymentRead(pydantic.BaseModel):
15
15
  id: str
16
16
  created: str
17
17
  label: str = pydantic.Field(
18
- description=(
19
- 'A human-readable label for the deployment <span style="white-space: nowrap">`<= 150 characters`</span> \n'
20
- )
18
+ description='A human-readable label for the deployment <span style="white-space: nowrap">`<= 150 characters`</span> '
21
19
  )
22
20
  name: str = pydantic.Field(
23
- description=(
24
- 'A name that uniquely identifies this deployment within its workspace <span style="white-space: nowrap">`<= 150 characters`</span> \n'
25
- )
21
+ description='A name that uniquely identifies this deployment within its workspace <span style="white-space: nowrap">`<= 150 characters`</span> '
26
22
  )
27
23
  status: typing.Optional[DeploymentReadStatusEnum] = pydantic.Field(
28
24
  description=(
vellum/types/document.py CHANGED
@@ -9,16 +9,12 @@ from ..core.datetime_utils import serialize_datetime
9
9
 
10
10
 
11
11
  class Document(pydantic.BaseModel):
12
- id: str = pydantic.Field(description=("The ID of the document.\n"))
12
+ id: str = pydantic.Field(description="The ID of the document.")
13
13
  label: str = pydantic.Field(
14
- description=(
15
- 'The human-readable name for the document. <span style="white-space: nowrap">`<= 1000 characters`</span> \n'
16
- )
14
+ description='The human-readable name for the document. <span style="white-space: nowrap">`<= 1000 characters`</span> '
17
15
  )
18
16
  external_id: typing.Optional[str] = pydantic.Field(
19
- description=(
20
- "The unique ID of the document as represented in an external system and specified when it was originally uploaded.\n"
21
- )
17
+ description="The unique ID of the document as represented in an external system and specified when it was originally uploaded."
22
18
  )
23
19
 
24
20
  def json(self, **kwargs: typing.Any) -> str:
@@ -10,9 +10,9 @@ from .indexing_state_enum import IndexingStateEnum
10
10
 
11
11
 
12
12
  class DocumentDocumentToDocumentIndex(pydantic.BaseModel):
13
- id: str = pydantic.Field(description=("Vellum-generated ID that uniquely identifies this link.\n"))
13
+ id: str = pydantic.Field(description="Vellum-generated ID that uniquely identifies this link.")
14
14
  document_index_id: str = pydantic.Field(
15
- description=("Vellum-generated ID that uniquely identifies the index this document is included in.\n")
15
+ description="Vellum-generated ID that uniquely identifies the index this document is included in."
16
16
  )
17
17
  indexing_state: typing.Optional[IndexingStateEnum] = pydantic.Field(
18
18
  description=(
@@ -14,14 +14,10 @@ class DocumentIndexRead(pydantic.BaseModel):
14
14
  id: str
15
15
  created: str
16
16
  label: str = pydantic.Field(
17
- description=(
18
- 'A human-readable label for the document index <span style="white-space: nowrap">`<= 150 characters`</span> \n'
19
- )
17
+ description='A human-readable label for the document index <span style="white-space: nowrap">`<= 150 characters`</span> '
20
18
  )
21
19
  name: str = pydantic.Field(
22
- description=(
23
- 'A name that uniquely identifies this index within its workspace <span style="white-space: nowrap">`<= 150 characters`</span> \n'
24
- )
20
+ description='A name that uniquely identifies this index within its workspace <span style="white-space: nowrap">`<= 150 characters`</span> '
25
21
  )
26
22
  status: typing.Optional[DocumentIndexStatus] = pydantic.Field(
27
23
  description=(
@@ -38,7 +34,7 @@ class DocumentIndexRead(pydantic.BaseModel):
38
34
  )
39
35
  )
40
36
  indexing_config: typing.Dict[str, typing.Any] = pydantic.Field(
41
- description=("Configuration representing how documents should be indexed\n")
37
+ description="Configuration representing how documents should be indexed"
42
38
  )
43
39
 
44
40
  def json(self, **kwargs: typing.Any) -> str:
@@ -12,13 +12,11 @@ from .normalized_log_probs import NormalizedLogProbs
12
12
 
13
13
 
14
14
  class EnrichedNormalizedCompletion(pydantic.BaseModel):
15
- id: str = pydantic.Field(description=("The Vellum-generated ID of the completion.\n"))
15
+ id: str = pydantic.Field(description="The Vellum-generated ID of the completion.")
16
16
  external_id: typing.Optional[str] = pydantic.Field(
17
- description=(
18
- "The external ID that was originally provided along with the generation request, which uniquely identifies this generation in an external system.\n"
19
- )
17
+ description="The external ID that was originally provided along with the generation request, which uniquely identifies this generation in an external system."
20
18
  )
21
- text: str = pydantic.Field(description=("The text generated by the LLM.\n"))
19
+ text: str = pydantic.Field(description="The text generated by the LLM.")
22
20
  finish_reason: typing.Optional[FinishReasonEnum] = pydantic.Field(
23
21
  description=(
24
22
  "The reason the generation finished.\n"
@@ -29,11 +27,9 @@ class EnrichedNormalizedCompletion(pydantic.BaseModel):
29
27
  )
30
28
  )
31
29
  logprobs: typing.Optional[NormalizedLogProbs] = pydantic.Field(
32
- description=("The logprobs of the completion. Only present if specified in the original request options.\n")
33
- )
34
- model_version_id: str = pydantic.Field(
35
- description=("The ID of the model version used to generate this completion.\n")
30
+ description="The logprobs of the completion. Only present if specified in the original request options."
36
31
  )
32
+ model_version_id: str = pydantic.Field(description="The ID of the model version used to generate this completion.")
37
33
  type: typing.Optional[ContentType]
38
34
 
39
35
  def json(self, **kwargs: typing.Any) -> str:
@@ -10,9 +10,7 @@ from ..core.datetime_utils import serialize_datetime
10
10
 
11
11
  class EvaluationParams(pydantic.BaseModel):
12
12
  target: typing.Optional[str] = pydantic.Field(
13
- description=(
14
- "The target value to compare the LLM output against. Typically what you expect or desire the LLM output to be.\n"
15
- )
13
+ description="The target value to compare the LLM output against. Typically what you expect or desire the LLM output to be."
16
14
  )
17
15
 
18
16
  def json(self, **kwargs: typing.Any) -> str:
@@ -10,9 +10,7 @@ from ..core.datetime_utils import serialize_datetime
10
10
 
11
11
  class EvaluationParamsRequest(pydantic.BaseModel):
12
12
  target: typing.Optional[str] = pydantic.Field(
13
- description=(
14
- "The target value to compare the LLM output against. Typically what you expect or desire the LLM output to be.\n"
15
- )
13
+ description="The target value to compare the LLM output against. Typically what you expect or desire the LLM output to be."
16
14
  )
17
15
 
18
16
  def json(self, **kwargs: typing.Any) -> str: