sambanova 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sambanova might be problematic. Click here for more details.
sambanova-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: sambanova
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: llama-index llms sambanova integration
|
|
5
|
+
Author: Your Name
|
|
6
|
+
Author-email: you@example.com
|
|
7
|
+
Requires-Python: >=3.8.1,<4.0.0
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Requires-Dist: llama-index-core (>=0.10.1,<0.11.0)
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# LlamaIndex LLM Integration: SambaNova LLM
|
|
17
|
+
|
|
18
|
+
SambaNovaLLM is a custom LLM (Language Model) interface that allows you to interact with AI models hosted on SambaNova's offerings - Sambaverse and SambaStudio
|
|
19
|
+
|
|
20
|
+
## Key Features:
|
|
21
|
+
|
|
22
|
+
- Integration with SambaNova-hosted AI models
|
|
23
|
+
- Integration two SambaNova offerings - Sambaverse and SambaStudio
|
|
24
|
+
- Support for completion based interactions
|
|
25
|
+
- Streaming support for completion responses
|
|
26
|
+
- Seamless integration with the LlamaIndex ecosystem
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install llama-index-llms-sambanova
|
|
32
|
+
```
|
|
33
|
+
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# LlamaIndex LLM Integration: SambaNova LLM
|
|
2
|
+
|
|
3
|
+
SambaNovaLLM is a custom LLM (Language Model) interface that allows you to interact with AI models hosted on SambaNova's offerings - Sambaverse and SambaStudio
|
|
4
|
+
|
|
5
|
+
## Key Features:
|
|
6
|
+
|
|
7
|
+
- Integration with SambaNova-hosted AI models
|
|
8
|
+
- Integration two SambaNova offerings - Sambaverse and SambaStudio
|
|
9
|
+
- Support for completion based interactions
|
|
10
|
+
- Streaming support for completion responses
|
|
11
|
+
- Seamless integration with the LlamaIndex ecosystem
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install llama-index-llms-sambanova
|
|
17
|
+
```
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
python_sources()
|
|
@@ -0,0 +1,575 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import Any, Dict, Optional, List
|
|
3
|
+
|
|
4
|
+
import requests
|
|
5
|
+
from llama_index.core.base.llms.types import (
|
|
6
|
+
CompletionResponse,
|
|
7
|
+
CompletionResponseGen,
|
|
8
|
+
LLMMetadata,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
from llama_index.core.llms.callbacks import llm_completion_callback
|
|
12
|
+
from llama_index.core.llms.custom import CustomLLM
|
|
13
|
+
from llama_index.core.bridge.pydantic import Field, PrivateAttr
|
|
14
|
+
import json
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Sambaverse(CustomLLM):
|
|
18
|
+
"""
|
|
19
|
+
Sambaverse LLM.
|
|
20
|
+
|
|
21
|
+
Examples:
|
|
22
|
+
`pip install llama-index-llms-sambanova`
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
from llama_index.llms.sambanova import Sambaverse
|
|
26
|
+
|
|
27
|
+
llm = Sambaverse(...)
|
|
28
|
+
|
|
29
|
+
response = llm.complete("What is the meaning of life?")
|
|
30
|
+
|
|
31
|
+
print(response)
|
|
32
|
+
```
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
sambaverse_url: str = Field(
|
|
36
|
+
default="https://sambaverse.sambanova.ai",
|
|
37
|
+
description="URL of the Sambaverse server",
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
sambaverse_api_key: str = Field(
|
|
41
|
+
default="",
|
|
42
|
+
description="API key for the Sambaverse server",
|
|
43
|
+
)
|
|
44
|
+
sambaverse_model_name: str = Field(
|
|
45
|
+
default="", description="Name of the Sambaverse model to use"
|
|
46
|
+
)
|
|
47
|
+
model_kwargs: Dict[str, Any] = Field(
|
|
48
|
+
default_factory=dict,
|
|
49
|
+
description="Additional keyword arguments to pass to the model",
|
|
50
|
+
)
|
|
51
|
+
streaming: bool = Field(
|
|
52
|
+
default=False,
|
|
53
|
+
description="Boolean to state whether to stream response or not",
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
_client: requests.Session = PrivateAttr()
|
|
57
|
+
|
|
58
|
+
def __init__(
|
|
59
|
+
self,
|
|
60
|
+
sambaverse_url: str = "https://sambaverse.sambanova.ai",
|
|
61
|
+
sambaverse_api_key: str = "",
|
|
62
|
+
sambaverse_model_name: str = "",
|
|
63
|
+
model_kwargs: Dict[str, Any] = {},
|
|
64
|
+
streaming: bool = False,
|
|
65
|
+
client: Optional[requests.Session] = None,
|
|
66
|
+
) -> None:
|
|
67
|
+
super().__init__()
|
|
68
|
+
|
|
69
|
+
self.sambaverse_url = sambaverse_url
|
|
70
|
+
self.sambaverse_api_key = sambaverse_api_key
|
|
71
|
+
self.sambaverse_model_name = sambaverse_model_name
|
|
72
|
+
self.model_kwargs = model_kwargs
|
|
73
|
+
self.streaming = streaming
|
|
74
|
+
self._client = client or requests.Session()
|
|
75
|
+
self._validate_env_vars()
|
|
76
|
+
|
|
77
|
+
def _validate_env_vars(self):
|
|
78
|
+
if not self.sambaverse_model_name:
|
|
79
|
+
self.sambaverse_model_name = os.getenv("SAMBAVERSE_MODEL_NAME")
|
|
80
|
+
if not self.sambaverse_api_key:
|
|
81
|
+
self.sambaverse_api_key = os.getenv("SAMBAVERSE_API_KEY")
|
|
82
|
+
|
|
83
|
+
if not self.sambaverse_model_name:
|
|
84
|
+
raise ValueError(
|
|
85
|
+
"Sambaverse model name must be provided either as an argument or set in the environment variable 'SAMBAVERSE_MODEL_NAME'."
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
if not self.sambaverse_api_key:
|
|
89
|
+
raise ValueError(
|
|
90
|
+
"Sambaverse API key must be provided either as an argument or set in the environment variable 'SAMBAVERSE_API_KEY'."
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
def _get_full_url(self, endpoint: str) -> str:
|
|
94
|
+
return f"{self.sambaverse_url}/{endpoint}"
|
|
95
|
+
|
|
96
|
+
def _get_model_kwargs(self, stop: Optional[List[str]]) -> str:
|
|
97
|
+
try:
|
|
98
|
+
_model_kwargs = self.model_kwargs or {}
|
|
99
|
+
_kwarg_stop_sequences = set(_model_kwargs.get("stop_sequences", []))
|
|
100
|
+
_stop_sequences = set(stop or _kwarg_stop_sequences)
|
|
101
|
+
|
|
102
|
+
if not _kwarg_stop_sequences:
|
|
103
|
+
_model_kwargs["stop_sequences"] = ",".join(
|
|
104
|
+
f'"{x}"' for x in _stop_sequences
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
tuning_params_dict = {
|
|
108
|
+
k: {"type": type(v).__name__, "value": str(v)}
|
|
109
|
+
for k, v in _model_kwargs.items()
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
return json.dumps(tuning_params_dict)
|
|
113
|
+
|
|
114
|
+
except Exception as e:
|
|
115
|
+
raise ValueError(f"Error getting model kwargs: {e}")
|
|
116
|
+
|
|
117
|
+
def _process_api_response(self, response: requests.Response) -> Dict:
|
|
118
|
+
result: Dict[str, Any] = {}
|
|
119
|
+
if response.status_code != 200:
|
|
120
|
+
raise ValueError(
|
|
121
|
+
f"Received unexpected status code {response.status_code}: {response.text}"
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
try:
|
|
125
|
+
lines_result = response.text.strip().split("\n")
|
|
126
|
+
text_result = lines_result[-1]
|
|
127
|
+
if response.status_code == 200 and json.loads(text_result).get("error"):
|
|
128
|
+
completion = ""
|
|
129
|
+
for line in lines_result[:-1]:
|
|
130
|
+
completion += json.loads(line)["result"]["responses"][0][
|
|
131
|
+
"stream_token"
|
|
132
|
+
]
|
|
133
|
+
text_result = lines_result[-2]
|
|
134
|
+
result = json.loads(text_result)
|
|
135
|
+
result["result"]["responses"][0]["completion"] = completion
|
|
136
|
+
else:
|
|
137
|
+
result = json.loads(text_result)
|
|
138
|
+
except Exception as e:
|
|
139
|
+
result["detail"] = str(e)
|
|
140
|
+
if "status_code" not in result:
|
|
141
|
+
result["status_code"] = response.status_code
|
|
142
|
+
return result
|
|
143
|
+
|
|
144
|
+
def _process_api_stream_response(self, response: requests.Response) -> Any:
|
|
145
|
+
try:
|
|
146
|
+
for line in response.iter_lines():
|
|
147
|
+
chunk = json.loads(line)
|
|
148
|
+
if "status_code" not in chunk:
|
|
149
|
+
chunk["status_code"] = response.status_code
|
|
150
|
+
if chunk["status_code"] == 200 and chunk.get("error"):
|
|
151
|
+
chunk["result"] = {"responses": [{"stream_token": ""}]}
|
|
152
|
+
return chunk
|
|
153
|
+
yield chunk
|
|
154
|
+
except Exception as e:
|
|
155
|
+
raise RuntimeError(f"Error processing streaming response: {e}")
|
|
156
|
+
|
|
157
|
+
def _send_sambaverse_request(
|
|
158
|
+
self, endpoint: str, data: Dict[str, Any], stream: bool = False
|
|
159
|
+
) -> requests.Response:
|
|
160
|
+
url = self._get_full_url(endpoint)
|
|
161
|
+
headers = {
|
|
162
|
+
"key": self.sambaverse_api_key,
|
|
163
|
+
"Content-Type": "application/json",
|
|
164
|
+
"modelName": self.sambaverse_model_name,
|
|
165
|
+
}
|
|
166
|
+
try:
|
|
167
|
+
return self._client.post(url, headers=headers, json=data, stream=stream)
|
|
168
|
+
|
|
169
|
+
except Exception as e:
|
|
170
|
+
raise ValueError(f"Error sending request to Sambaverse: {e}")
|
|
171
|
+
|
|
172
|
+
def _prepare_request_data(self, prompt: str) -> Dict[str, Any]:
|
|
173
|
+
try:
|
|
174
|
+
model_params = self._get_model_kwargs(stop=None)
|
|
175
|
+
return {"instance": prompt, "params": json.loads(model_params)}
|
|
176
|
+
|
|
177
|
+
except Exception as e:
|
|
178
|
+
raise ValueError(f"Error preparing request data: {e}")
|
|
179
|
+
|
|
180
|
+
def _get_completion_from_response(self, response: Dict) -> str:
|
|
181
|
+
try:
|
|
182
|
+
return (
|
|
183
|
+
response.get("result", {})
|
|
184
|
+
.get("responses", [{}])[0]
|
|
185
|
+
.get("completion", "")
|
|
186
|
+
)
|
|
187
|
+
except Exception as e:
|
|
188
|
+
raise ValueError(f"Error processing response: {e}")
|
|
189
|
+
|
|
190
|
+
@classmethod
|
|
191
|
+
def class_name(cls) -> str:
|
|
192
|
+
return "Samabaverse"
|
|
193
|
+
|
|
194
|
+
@property
|
|
195
|
+
def metadata(self) -> LLMMetadata:
|
|
196
|
+
"""LLM metadata."""
|
|
197
|
+
return LLMMetadata(
|
|
198
|
+
model_name=self.sambaverse_model_name,
|
|
199
|
+
model_kwargs=self.model_kwargs,
|
|
200
|
+
description="Sambanova LLM",
|
|
201
|
+
is_streaming=self.streaming,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
@llm_completion_callback()
|
|
205
|
+
def complete(self, prompt: str) -> CompletionResponse:
|
|
206
|
+
"""
|
|
207
|
+
Complete the given prompt using the Sambaverse model.
|
|
208
|
+
|
|
209
|
+
Args:
|
|
210
|
+
prompt (str): The input prompt to complete.
|
|
211
|
+
|
|
212
|
+
Returns:
|
|
213
|
+
CompletionResponse: The completed text generated by the model.
|
|
214
|
+
"""
|
|
215
|
+
data = self._prepare_request_data(prompt)
|
|
216
|
+
response = self._send_sambaverse_request("api/predict", data)
|
|
217
|
+
processed_response = self._process_api_response(response)
|
|
218
|
+
completion_text = self._get_completion_from_response(processed_response)
|
|
219
|
+
|
|
220
|
+
return CompletionResponse(text=completion_text)
|
|
221
|
+
|
|
222
|
+
@llm_completion_callback()
|
|
223
|
+
def stream_complete(self, prompt: str) -> CompletionResponseGen:
|
|
224
|
+
"""
|
|
225
|
+
Stream the completion of the given prompt using the Sambaverse model.
|
|
226
|
+
|
|
227
|
+
Args:
|
|
228
|
+
prompt (str): The input prompt to complete.
|
|
229
|
+
|
|
230
|
+
Yields:
|
|
231
|
+
CompletionResponseGen: Streamed completion text generated by the model.
|
|
232
|
+
"""
|
|
233
|
+
print("In stream_complete")
|
|
234
|
+
data = self._prepare_request_data(prompt)
|
|
235
|
+
response = self._send_sambaverse_request("api/predict", data, stream=True)
|
|
236
|
+
|
|
237
|
+
for token in self._process_api_stream_response(response):
|
|
238
|
+
processed_token = token["result"]["responses"][0]["stream_token"]
|
|
239
|
+
yield CompletionResponse(text=processed_token)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
class SambaStudio(CustomLLM):
|
|
243
|
+
"""
|
|
244
|
+
SambaStudio LLM.
|
|
245
|
+
|
|
246
|
+
Examples:
|
|
247
|
+
`pip install llama-index-llms-sambanova`
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
from llama_index.llms.sambanova import SambaStudio
|
|
251
|
+
|
|
252
|
+
llm = Sambaverse(...)
|
|
253
|
+
|
|
254
|
+
response = llm.complete("What is the meaning of life?")
|
|
255
|
+
|
|
256
|
+
print(response)
|
|
257
|
+
```
|
|
258
|
+
"""
|
|
259
|
+
|
|
260
|
+
sambastudio_base_url: str = Field(
|
|
261
|
+
default="",
|
|
262
|
+
description="URL of the SambaStudio server",
|
|
263
|
+
)
|
|
264
|
+
sambastudio_base_uri: str = Field(
|
|
265
|
+
default="",
|
|
266
|
+
description="Base URI of the SambaStudio server",
|
|
267
|
+
)
|
|
268
|
+
sambastudio_project_id: str = Field(
|
|
269
|
+
default="",
|
|
270
|
+
description="Project ID of the SambaStudio server",
|
|
271
|
+
)
|
|
272
|
+
sambastudio_endpoint_id: str = Field(
|
|
273
|
+
default="",
|
|
274
|
+
description="Endpoint ID of the SambaStudio server",
|
|
275
|
+
)
|
|
276
|
+
sambastudio_api_key: str = Field(
|
|
277
|
+
default="",
|
|
278
|
+
description="API key for the SambaStudio server",
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
model_kwargs: Dict[str, Any] = Field(
|
|
282
|
+
default_factory=dict,
|
|
283
|
+
description="Additional keyword arguments to pass to the model",
|
|
284
|
+
)
|
|
285
|
+
streaming: bool = Field(
|
|
286
|
+
default=False,
|
|
287
|
+
description="Boolean to state whether to stream response or not",
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
_client: requests.Session = PrivateAttr()
|
|
291
|
+
|
|
292
|
+
def __init__(
|
|
293
|
+
self,
|
|
294
|
+
sambastudio_base_url: str = "",
|
|
295
|
+
sambastudio_base_uri: str = "",
|
|
296
|
+
sambastudio_project_id: str = "",
|
|
297
|
+
sambastudio_endpoint_id: str = "",
|
|
298
|
+
model_kwargs: Dict[str, Any] = {},
|
|
299
|
+
streaming: bool = False,
|
|
300
|
+
client: Optional[requests.Session] = None,
|
|
301
|
+
) -> None:
|
|
302
|
+
super().__init__()
|
|
303
|
+
|
|
304
|
+
self.sambastudio_base_url = sambastudio_base_url
|
|
305
|
+
self.sambastudio_base_uri = sambastudio_base_uri
|
|
306
|
+
self.sambastudio_project_id = sambastudio_project_id
|
|
307
|
+
self.sambastudio_endpoint_id = sambastudio_endpoint_id
|
|
308
|
+
self.model_kwargs = model_kwargs
|
|
309
|
+
self.streaming = streaming
|
|
310
|
+
self._client = client or requests.Session()
|
|
311
|
+
self._validate_env_vars()
|
|
312
|
+
|
|
313
|
+
def _validate_env_vars(self):
|
|
314
|
+
if not self.sambaverse_api_key:
|
|
315
|
+
self.sambaverse_api_key = os.getenv("SAMBAVERSE_API_KEY")
|
|
316
|
+
if not self.sambastudio_base_url:
|
|
317
|
+
self.sambastudio_base_url = os.getenv("SAMBASTUDIO_BASE_URL")
|
|
318
|
+
if not self.sambastudio_base_uri:
|
|
319
|
+
self.sambastudio_base_uri = os.getenv("SAMBASTUDIO_BASE_URI")
|
|
320
|
+
if not self.sambastudio_project_id:
|
|
321
|
+
self.sambastudio_project_id = os.getenv("SAMBASTUDIO_PROJECT_ID")
|
|
322
|
+
if not self.sambastudio_endpoint_id:
|
|
323
|
+
self.sambastudio_endpoint_id = os.getenv("SAMBASTUDIO_ENDPOINT_ID")
|
|
324
|
+
|
|
325
|
+
if not self.sambaverse_api_key:
|
|
326
|
+
raise ValueError(
|
|
327
|
+
"Sambaverse API key must be provided either as an argument or set in the environment variable 'SAMBAVERSE_API_KEY'."
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
if not self.sambastudio_base_url:
|
|
331
|
+
raise ValueError(
|
|
332
|
+
"Sambastudio base URL must be provided either as an argument or set in the environment variable 'SAMBASTUDIO_BASE_URL'."
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
if not self.sambastudio_base_uri:
|
|
336
|
+
raise ValueError(
|
|
337
|
+
"Sambastudio base URI must be provided either as an argument or set in the environment variable 'SAMBASTUDIO_BASE_URI'."
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
if not self.sambastudio_project_id:
|
|
341
|
+
raise ValueError(
|
|
342
|
+
"Sambastudio project ID must be provided either as an argument or set in the environment variable 'SAMBASTUDIO_PROJECT_ID'."
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
if not self.sambastudio_endpoint_id:
|
|
346
|
+
raise ValueError(
|
|
347
|
+
"Sambastudio endpoint ID must be provided either as an argument or set in the environment variable 'SAMBASTUDIO_ENDPOINT_ID'."
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
def _get_full_url(self, path: str) -> str:
|
|
351
|
+
return f"{self.sambastudio_base_url}/{self.sambastudio_base_uri}/{path}"
|
|
352
|
+
|
|
353
|
+
def _get_model_kwargs(self, stop: Optional[List[str]]) -> str:
|
|
354
|
+
try:
|
|
355
|
+
_model_kwargs = self.model_kwargs or {}
|
|
356
|
+
_kwarg_stop_sequences = set(_model_kwargs.get("stop_sequences", []))
|
|
357
|
+
_stop_sequences = set(stop or _kwarg_stop_sequences)
|
|
358
|
+
|
|
359
|
+
if not _kwarg_stop_sequences:
|
|
360
|
+
_model_kwargs["stop_sequences"] = ",".join(
|
|
361
|
+
f'"{x}"' for x in _stop_sequences
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
tuning_params_dict = {
|
|
365
|
+
k: {"type": type(v).__name__, "value": str(v)}
|
|
366
|
+
for k, v in _model_kwargs.items()
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
return json.dumps(tuning_params_dict)
|
|
370
|
+
|
|
371
|
+
except Exception as e:
|
|
372
|
+
raise ValueError(f"Error getting model kwargs: {e}")
|
|
373
|
+
|
|
374
|
+
def _process_api_response(self, response: requests.Response) -> Dict:
|
|
375
|
+
result: Dict[str, Any] = {}
|
|
376
|
+
try:
|
|
377
|
+
result = response.json()
|
|
378
|
+
except Exception as e:
|
|
379
|
+
result["detail"] = str(e)
|
|
380
|
+
if "status_code" not in result:
|
|
381
|
+
result["status_code"] = response.status_code
|
|
382
|
+
return result
|
|
383
|
+
|
|
384
|
+
def _process_api_stream_response(self, response: requests.Response) -> Any:
|
|
385
|
+
"""Process the streaming response."""
|
|
386
|
+
if "nlp" in self.sambastudio_base_uri:
|
|
387
|
+
try:
|
|
388
|
+
import sseclient
|
|
389
|
+
except ImportError:
|
|
390
|
+
raise ImportError(
|
|
391
|
+
"could not import sseclient library"
|
|
392
|
+
"Please install it with `pip install sseclient-py`."
|
|
393
|
+
)
|
|
394
|
+
client = sseclient.SSEClient(response)
|
|
395
|
+
close_conn = False
|
|
396
|
+
for event in client.events():
|
|
397
|
+
if event.event == "error_event":
|
|
398
|
+
close_conn = True
|
|
399
|
+
chunk = {
|
|
400
|
+
"event": event.event,
|
|
401
|
+
"data": event.data,
|
|
402
|
+
"status_code": response.status_code,
|
|
403
|
+
}
|
|
404
|
+
yield chunk
|
|
405
|
+
if close_conn:
|
|
406
|
+
client.close()
|
|
407
|
+
elif "generic" in self.sambastudio_base_uri:
|
|
408
|
+
try:
|
|
409
|
+
for line in response.iter_lines():
|
|
410
|
+
chunk = json.loads(line)
|
|
411
|
+
if "status_code" not in chunk:
|
|
412
|
+
chunk["status_code"] = response.status_code
|
|
413
|
+
if chunk["status_code"] == 200 and chunk.get("error"):
|
|
414
|
+
chunk["result"] = {"responses": [{"stream_token": ""}]}
|
|
415
|
+
yield chunk
|
|
416
|
+
except Exception as e:
|
|
417
|
+
raise RuntimeError(f"Error processing streaming response: {e}")
|
|
418
|
+
else:
|
|
419
|
+
raise ValueError(
|
|
420
|
+
f"handling of endpoint uri: {self.api_base_uri} not implemented"
|
|
421
|
+
)
|
|
422
|
+
|
|
423
|
+
def _send_sambaverse_request(
|
|
424
|
+
self, data: Dict[str, Any], stream: bool = False
|
|
425
|
+
) -> requests.Response:
|
|
426
|
+
try:
|
|
427
|
+
if stream:
|
|
428
|
+
url = self._get_full_url(
|
|
429
|
+
f"stream/{self.sambastudio_project_id}/{self.sambastudio_endpoint_id}"
|
|
430
|
+
)
|
|
431
|
+
headers = {
|
|
432
|
+
"key": self.sambaverse_api_key,
|
|
433
|
+
"Content-Type": "application/json",
|
|
434
|
+
}
|
|
435
|
+
return self._client.post(url, headers=headers, json=data, stream=True)
|
|
436
|
+
else:
|
|
437
|
+
url = self._get_full_url(
|
|
438
|
+
f"{self.sambastudio_project_id}/{self.sambastudio_endpoint_id}"
|
|
439
|
+
)
|
|
440
|
+
headers = {
|
|
441
|
+
"key": self.sambaverse_api_key,
|
|
442
|
+
"Content-Type": "application/json",
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
return self._client.post(url, headers=headers, json=data, stream=stream)
|
|
446
|
+
except Exception as e:
|
|
447
|
+
raise ValueError(f"Error sending request to Sambaverse: {e}")
|
|
448
|
+
|
|
449
|
+
def _prepare_request_data(self, prompt: str) -> Dict[str, Any]:
|
|
450
|
+
try:
|
|
451
|
+
data = {}
|
|
452
|
+
if isinstance(prompt, str):
|
|
453
|
+
input = [prompt]
|
|
454
|
+
if "nlp" in self.api_base_uri:
|
|
455
|
+
model_params = self._get_model_kwargs(stop=None)
|
|
456
|
+
if model_params:
|
|
457
|
+
data = {"inputs": input, "params": json.loads(model_params)}
|
|
458
|
+
else:
|
|
459
|
+
data = {"inputs": input}
|
|
460
|
+
elif "generic" in self.api_base_uri:
|
|
461
|
+
model_params = self._get_model_kwargs(stop=None)
|
|
462
|
+
if model_params:
|
|
463
|
+
data = {"instance": input, "params": json.loads(model_params)}
|
|
464
|
+
else:
|
|
465
|
+
data = {"instance": input}
|
|
466
|
+
else:
|
|
467
|
+
raise ValueError(
|
|
468
|
+
f"handling of endpoint uri: {self.api_base_uri} not implemented"
|
|
469
|
+
)
|
|
470
|
+
return data
|
|
471
|
+
|
|
472
|
+
except Exception as e:
|
|
473
|
+
raise ValueError(f"Error preparing request data: {e}")
|
|
474
|
+
|
|
475
|
+
def _get_completion_from_response(self, response: Dict) -> str:
|
|
476
|
+
try:
|
|
477
|
+
if "nlp" in self.sambastudio_base_uri:
|
|
478
|
+
return response["data"][0]["completion"]
|
|
479
|
+
elif "generic" in self.sambastudio_base_uri:
|
|
480
|
+
return response["predictions"][0]["completion"]
|
|
481
|
+
else:
|
|
482
|
+
raise ValueError(
|
|
483
|
+
f"handling of endpoint uri: {self.sambastudio_base_uri} not implemented"
|
|
484
|
+
)
|
|
485
|
+
except Exception as e:
|
|
486
|
+
raise ValueError(f"Error processing response: {e}")
|
|
487
|
+
|
|
488
|
+
def _get_stream_token_from_response(self, response: Dict) -> str:
|
|
489
|
+
try:
|
|
490
|
+
if "nlp" in self.sambastudio_base_uri:
|
|
491
|
+
return response["data"]["stream_token"]
|
|
492
|
+
elif "generic" in self.sambastudio_base_uri:
|
|
493
|
+
return response["result"]["responses"][0]["stream_token"]
|
|
494
|
+
else:
|
|
495
|
+
raise ValueError(
|
|
496
|
+
f"handling of endpoint uri: {self.sambastudio_base_uri} not implemented"
|
|
497
|
+
)
|
|
498
|
+
except Exception as e:
|
|
499
|
+
raise ValueError(f"Error processing response: {e}")
|
|
500
|
+
|
|
501
|
+
@classmethod
|
|
502
|
+
def class_name(cls) -> str:
|
|
503
|
+
return "SambaStudio"
|
|
504
|
+
|
|
505
|
+
@property
|
|
506
|
+
def metadata(self) -> LLMMetadata:
|
|
507
|
+
"""LLM metadata."""
|
|
508
|
+
return LLMMetadata(
|
|
509
|
+
model_kwargs=self.model_kwargs,
|
|
510
|
+
description="Sambanova LLM",
|
|
511
|
+
is_streaming=self.streaming,
|
|
512
|
+
)
|
|
513
|
+
|
|
514
|
+
@llm_completion_callback()
|
|
515
|
+
def complete(self, prompt: str) -> CompletionResponse:
|
|
516
|
+
"""
|
|
517
|
+
Complete the given prompt using the SambaStudio model.
|
|
518
|
+
|
|
519
|
+
Args:
|
|
520
|
+
prompt (str): The input prompt to complete.
|
|
521
|
+
|
|
522
|
+
Returns:
|
|
523
|
+
CompletionResponse: The completed text generated by the model.
|
|
524
|
+
"""
|
|
525
|
+
data = self._prepare_request_data(prompt)
|
|
526
|
+
response = self._send_sambaverse_request(data)
|
|
527
|
+
processed_response = self._process_api_response(response)
|
|
528
|
+
completion_text = self._get_completion_from_response(processed_response)
|
|
529
|
+
|
|
530
|
+
return CompletionResponse(text=completion_text)
|
|
531
|
+
|
|
532
|
+
@llm_completion_callback()
|
|
533
|
+
def stream_complete(self, prompt: str) -> CompletionResponseGen:
|
|
534
|
+
"""
|
|
535
|
+
Stream the completion of the given prompt using the SambaStudio model.
|
|
536
|
+
|
|
537
|
+
Args:
|
|
538
|
+
prompt (str): The input prompt to complete.
|
|
539
|
+
|
|
540
|
+
Yields:
|
|
541
|
+
CompletionResponseGen: Streamed completion text generated by the model.
|
|
542
|
+
"""
|
|
543
|
+
print("In stream_complete")
|
|
544
|
+
data = self._prepare_request_data(prompt)
|
|
545
|
+
response = self._send_sambaverse_request(data, stream=True)
|
|
546
|
+
|
|
547
|
+
for token in self._process_api_stream_response(response):
|
|
548
|
+
processed_token = self._get_stream_token_from_response(token)
|
|
549
|
+
yield CompletionResponse(text=processed_token)
|
|
550
|
+
|
|
551
|
+
@llm_completion_callback()
|
|
552
|
+
def nlp_prediction(self, prompt: str) -> CompletionResponse:
|
|
553
|
+
"""
|
|
554
|
+
Perform NLP prediction for the given prompt using the SambaStudio model.
|
|
555
|
+
|
|
556
|
+
Args:
|
|
557
|
+
prompt (str): The input prompt to predict.
|
|
558
|
+
|
|
559
|
+
Returns:
|
|
560
|
+
CompletionResponse: The prediction result generated by the model.
|
|
561
|
+
"""
|
|
562
|
+
return self.complete(prompt)
|
|
563
|
+
|
|
564
|
+
@llm_completion_callback()
|
|
565
|
+
def nlp_prediction_stream(self, prompt: str) -> CompletionResponseGen:
|
|
566
|
+
"""
|
|
567
|
+
Stream NLP prediction for the given prompt using the SambaStudio model.
|
|
568
|
+
|
|
569
|
+
Args:
|
|
570
|
+
prompt (str): The input prompt to predict.
|
|
571
|
+
|
|
572
|
+
Yields:
|
|
573
|
+
CompletionResponseGen: Streamed prediction result generated by the model.
|
|
574
|
+
"""
|
|
575
|
+
return self.stream_complete(prompt)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
build-backend = "poetry.core.masonry.api"
|
|
3
|
+
requires = ["poetry-core"]
|
|
4
|
+
|
|
5
|
+
[tool.codespell]
|
|
6
|
+
check-filenames = true
|
|
7
|
+
check-hidden = true
|
|
8
|
+
skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb"
|
|
9
|
+
|
|
10
|
+
[tool.llamahub]
|
|
11
|
+
contains_example = false
|
|
12
|
+
import_path = "llama_index.llms.sambanova"
|
|
13
|
+
|
|
14
|
+
[tool.llamahub.class_authors]
|
|
15
|
+
SambaStudio = "Pradeep"
|
|
16
|
+
Sambaverse = "Pradeep"
|
|
17
|
+
|
|
18
|
+
[tool.mypy]
|
|
19
|
+
disallow_untyped_defs = true
|
|
20
|
+
exclude = ["_static", "build", "examples", "notebooks", "venv"]
|
|
21
|
+
ignore_missing_imports = true
|
|
22
|
+
python_version = "3.8"
|
|
23
|
+
|
|
24
|
+
[tool.poetry]
|
|
25
|
+
authors = ["Your Name <you@example.com>"]
|
|
26
|
+
description = "llama-index llms sambanova integration"
|
|
27
|
+
name = "sambanova"
|
|
28
|
+
readme = "README.md"
|
|
29
|
+
version = "0.1.0"
|
|
30
|
+
|
|
31
|
+
[tool.poetry.dependencies]
|
|
32
|
+
python = ">=3.8.1,<4.0.0"
|
|
33
|
+
llama-index-core = "^0.10.1"
|
|
34
|
+
|
|
35
|
+
[tool.poetry.group.dev.dependencies]
|
|
36
|
+
ipython = "8.10.0"
|
|
37
|
+
jupyter = "^1.0.0"
|
|
38
|
+
mypy = "0.991"
|
|
39
|
+
pre-commit = "3.2.0"
|
|
40
|
+
pylint = "2.15.10"
|
|
41
|
+
pytest = "7.2.1"
|
|
42
|
+
pytest-mock = "3.11.1"
|
|
43
|
+
ruff = "0.0.292"
|
|
44
|
+
tree-sitter-languages = "^1.8.0"
|
|
45
|
+
types-Deprecated = ">=0.1.0"
|
|
46
|
+
types-PyYAML = "^6.0.12.12"
|
|
47
|
+
types-protobuf = "^4.24.0.4"
|
|
48
|
+
types-redis = "4.5.5.0"
|
|
49
|
+
types-requests = "2.28.11.8"
|
|
50
|
+
types-setuptools = "67.1.0.0"
|
|
51
|
+
|
|
52
|
+
[tool.poetry.group.dev.dependencies.codespell]
|
|
53
|
+
extras = ["toml"]
|
|
54
|
+
version = ">=v2.2.6"
|
|
55
|
+
|
|
56
|
+
[[tool.poetry.packages]]
|
|
57
|
+
include = "llama_index/"
|