ragaai-catalyst 2.0.7__py3-none-any.whl → 2.0.7.2b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ragaai_catalyst/evaluation.py +2 -2
- ragaai_catalyst/tracers/agentic_tracing/Untitled-1.json +660 -0
- ragaai_catalyst/tracers/agentic_tracing/__init__.py +3 -0
- ragaai_catalyst/tracers/agentic_tracing/agent_tracer.py +311 -0
- ragaai_catalyst/tracers/agentic_tracing/agentic_tracing.py +212 -0
- ragaai_catalyst/tracers/agentic_tracing/base.py +270 -0
- ragaai_catalyst/tracers/agentic_tracing/data_structure.py +239 -0
- ragaai_catalyst/tracers/agentic_tracing/llm_tracer.py +906 -0
- ragaai_catalyst/tracers/agentic_tracing/network_tracer.py +286 -0
- ragaai_catalyst/tracers/agentic_tracing/sample.py +197 -0
- ragaai_catalyst/tracers/agentic_tracing/tool_tracer.py +235 -0
- ragaai_catalyst/tracers/agentic_tracing/unique_decorator.py +221 -0
- ragaai_catalyst/tracers/agentic_tracing/unique_decorator_test.py +172 -0
- ragaai_catalyst/tracers/agentic_tracing/user_interaction_tracer.py +67 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/__init__.py +3 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/api_utils.py +18 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/data_classes.py +61 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/generic.py +32 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py +181 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json +5946 -0
- ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py +74 -0
- ragaai_catalyst/tracers/llamaindex_callback.py +60 -56
- ragaai_catalyst/tracers/tracer.py +26 -4
- ragaai_catalyst/tracers/upload_traces.py +138 -0
- ragaai_catalyst-2.0.7.2b1.dist-info/METADATA +39 -0
- ragaai_catalyst-2.0.7.2b1.dist-info/RECORD +50 -0
- ragaai_catalyst-2.0.7.dist-info/METADATA +0 -386
- ragaai_catalyst-2.0.7.dist-info/RECORD +0 -29
- {ragaai_catalyst-2.0.7.dist-info → ragaai_catalyst-2.0.7.2b1.dist-info}/WHEEL +0 -0
- {ragaai_catalyst-2.0.7.dist-info → ragaai_catalyst-2.0.7.2b1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,74 @@
|
|
1
|
+
import json
|
2
|
+
import os
|
3
|
+
from importlib import resources
|
4
|
+
from dataclasses import asdict
|
5
|
+
|
6
|
+
|
7
|
+
def convert_usage_to_dict(usage):
|
8
|
+
# Initialize the token_usage dictionary with default values
|
9
|
+
token_usage = {
|
10
|
+
"input": 0,
|
11
|
+
"completion": 0,
|
12
|
+
"reasoning": 0, # Default reasoning tokens to 0 unless specified
|
13
|
+
}
|
14
|
+
|
15
|
+
if usage:
|
16
|
+
if isinstance(usage, dict):
|
17
|
+
# Access usage data as dictionary keys
|
18
|
+
token_usage["input"] = usage.get("prompt_tokens", 0)
|
19
|
+
token_usage["completion"] = usage.get("completion_tokens", 0)
|
20
|
+
# If reasoning tokens are provided, adjust accordingly
|
21
|
+
token_usage["reasoning"] = usage.get("reasoning_tokens", 0)
|
22
|
+
else:
|
23
|
+
# Handle the case where usage is not a dictionary
|
24
|
+
# This could be an object with attributes, or something else
|
25
|
+
try:
|
26
|
+
token_usage["input"] = getattr(usage, "prompt_tokens", 0)
|
27
|
+
token_usage["completion"] = getattr(usage, "completion_tokens", 0)
|
28
|
+
token_usage["reasoning"] = getattr(usage, "reasoning_tokens", 0)
|
29
|
+
except AttributeError:
|
30
|
+
# If attributes are not found, log or handle the error as needed
|
31
|
+
print(f"Warning: Unexpected usage type: {type(usage)}")
|
32
|
+
|
33
|
+
return token_usage
|
34
|
+
|
35
|
+
|
36
|
+
def calculate_cost(
|
37
|
+
token_usage,
|
38
|
+
input_cost_per_token=0.0,
|
39
|
+
output_cost_per_token=0.0,
|
40
|
+
reasoning_cost_per_token=0.0,
|
41
|
+
):
|
42
|
+
input_tokens = token_usage.get("prompt_tokens", 0)
|
43
|
+
output_tokens = token_usage.get("completion_tokens", 0)
|
44
|
+
reasoning_tokens = token_usage.get("reasoning_tokens", 0)
|
45
|
+
|
46
|
+
input_cost = input_tokens * input_cost_per_token
|
47
|
+
output_cost = output_tokens * output_cost_per_token
|
48
|
+
reasoning_cost = reasoning_tokens * reasoning_cost_per_token
|
49
|
+
|
50
|
+
total_cost = input_cost + output_cost + reasoning_cost
|
51
|
+
|
52
|
+
return {
|
53
|
+
"input": input_cost,
|
54
|
+
"completion": output_cost,
|
55
|
+
"reasoning": reasoning_cost,
|
56
|
+
"total": total_cost,
|
57
|
+
}
|
58
|
+
|
59
|
+
|
60
|
+
def load_model_costs():
|
61
|
+
try:
|
62
|
+
current_dir = os.path.dirname(os.path.abspath(__file__))
|
63
|
+
model_costs_path = os.path.join(current_dir, "model_costs.json")
|
64
|
+
with open(model_costs_path, "r") as file:
|
65
|
+
return json.load(file)
|
66
|
+
except FileNotFoundError:
|
67
|
+
with resources.open_text("utils", "model_costs.json") as file:
|
68
|
+
return json.load(file)
|
69
|
+
|
70
|
+
|
71
|
+
def log_event(event_data, log_file_path):
|
72
|
+
event_data = asdict(event_data)
|
73
|
+
with open(log_file_path, "a") as f:
|
74
|
+
f.write(json.dumps(event_data) + "\n")
|
@@ -11,6 +11,7 @@ import tempfile
|
|
11
11
|
|
12
12
|
from ..ragaai_catalyst import RagaAICatalyst
|
13
13
|
|
14
|
+
|
14
15
|
class CustomEncoder(json.JSONEncoder):
|
15
16
|
def default(self, obj):
|
16
17
|
if isinstance(obj, Enum):
|
@@ -54,7 +55,7 @@ class LlamaIndexTracer:
|
|
54
55
|
payload: Optional[Dict[str, Any]] = None,
|
55
56
|
event_id: str = "",
|
56
57
|
parent_id: str = "",
|
57
|
-
**kwargs: Any
|
58
|
+
**kwargs: Any,
|
58
59
|
) -> None:
|
59
60
|
trace = {
|
60
61
|
"event_type": event_type,
|
@@ -68,7 +69,7 @@ class LlamaIndexTracer:
|
|
68
69
|
self.in_query = True
|
69
70
|
self.query_event_id = event_id
|
70
71
|
self.current_query_traces = []
|
71
|
-
|
72
|
+
|
72
73
|
if self.in_query:
|
73
74
|
self.current_query_traces.append(trace)
|
74
75
|
self.traces.append(trace)
|
@@ -78,7 +79,7 @@ class LlamaIndexTracer:
|
|
78
79
|
event_type: Optional[str],
|
79
80
|
payload: Optional[Dict[str, Any]] = None,
|
80
81
|
event_id: str = "",
|
81
|
-
**kwargs: Any
|
82
|
+
**kwargs: Any,
|
82
83
|
) -> None:
|
83
84
|
trace = {
|
84
85
|
"event_type": event_type,
|
@@ -90,24 +91,21 @@ class LlamaIndexTracer:
|
|
90
91
|
if self.in_query:
|
91
92
|
self.current_query_traces.append(trace)
|
92
93
|
self.traces.append(trace)
|
93
|
-
|
94
|
+
|
94
95
|
# If this is the end of a query event, automatically save the traces
|
95
96
|
if event_type == "query" and event_id == self.query_event_id:
|
96
97
|
self.in_query = False
|
97
98
|
outer_self._save_current_query_traces(self.current_query_traces)
|
98
99
|
self.current_query_traces = []
|
99
|
-
|
100
100
|
|
101
101
|
self.trace_handler = CustomTraceHandler()
|
102
102
|
self.callback_manager.add_handler(self.trace_handler)
|
103
103
|
Settings.callback_manager = self.callback_manager
|
104
104
|
|
105
|
-
|
106
105
|
# Monkey-patch LlamaIndex components
|
107
106
|
self._monkey_patch()
|
108
107
|
return self # Return self to allow method chaining
|
109
108
|
|
110
|
-
|
111
109
|
def _save_current_query_traces(self, query_traces):
|
112
110
|
"""Save traces for the current query"""
|
113
111
|
self.query_count += 1
|
@@ -131,7 +129,6 @@ class LlamaIndexTracer:
|
|
131
129
|
self._insert_traces(presignedUrl)
|
132
130
|
# print(f"Query {self.query_count} traces uploaded")
|
133
131
|
|
134
|
-
|
135
132
|
def _monkey_patch(self):
|
136
133
|
"""Monkey-patch LlamaIndex components to automatically include the callback manager"""
|
137
134
|
from llama_index.core import VectorStoreIndex, ServiceContext
|
@@ -181,7 +178,7 @@ class LlamaIndexTracer:
|
|
181
178
|
# self._upload_traces(save_json_to_pwd=True)
|
182
179
|
self.callback_manager.remove_handler(self.trace_handler)
|
183
180
|
self._restore_original_inits()
|
184
|
-
print("Traces
|
181
|
+
print("Traces uploaded")
|
185
182
|
self._upload_task = True
|
186
183
|
|
187
184
|
def _restore_original_inits(self):
|
@@ -213,17 +210,17 @@ class LlamaIndexTracer:
|
|
213
210
|
Generate a random trace ID using UUID4.
|
214
211
|
Returns a string representation of the UUID with no hyphens.
|
215
212
|
"""
|
216
|
-
return
|
213
|
+
return "0x" + str(uuid.uuid4()).replace("-", "")
|
217
214
|
|
218
215
|
def _get_user_passed_detail(self):
|
219
216
|
user_detail = self.user_detail
|
220
217
|
user_detail["trace_id"] = self._generate_trace_id()
|
221
218
|
metadata = user_detail["metadata"]
|
222
219
|
metadata["log_source"] = "llamaindex_tracer"
|
223
|
-
metadata["recorded_on"] = datetime.utcnow().isoformat().replace(
|
220
|
+
metadata["recorded_on"] = datetime.utcnow().isoformat().replace("T", " ")
|
224
221
|
user_detail["metadata"] = metadata
|
225
222
|
return user_detail
|
226
|
-
|
223
|
+
|
227
224
|
def _add_traces_in_data(self, traces=None):
|
228
225
|
"""Add traces to user detail"""
|
229
226
|
user_detail = self._get_user_passed_detail()
|
@@ -234,37 +231,40 @@ class LlamaIndexTracer:
|
|
234
231
|
user_detail["traces"] = traces
|
235
232
|
return user_detail
|
236
233
|
|
237
|
-
|
238
234
|
def _create_dataset_schema_with_trace(self):
|
239
235
|
SCHEMA_MAPPING_NEW = {
|
240
236
|
"trace_id": {"columnType": "traceId"},
|
241
237
|
"trace_uri": {"columnType": "traceUri"},
|
242
238
|
"prompt": {"columnType": "prompt"},
|
243
|
-
"response":{"columnType": "response"},
|
239
|
+
"response": {"columnType": "response"},
|
244
240
|
"context": {"columnType": "context"},
|
245
|
-
"llm_model": {"columnType":"pipeline"},
|
241
|
+
"llm_model": {"columnType": "pipeline"},
|
246
242
|
"recorded_on": {"columnType": "metadata"},
|
247
|
-
"embed_model": {"columnType":"pipeline"},
|
243
|
+
"embed_model": {"columnType": "pipeline"},
|
248
244
|
"log_source": {"columnType": "metadata"},
|
249
|
-
"vector_store":{"columnType":"pipeline"},
|
250
|
-
"feedback": {"columnType":"feedBack"}
|
245
|
+
"vector_store": {"columnType": "pipeline"},
|
246
|
+
"feedback": {"columnType": "feedBack"},
|
251
247
|
}
|
248
|
+
|
252
249
|
def make_request():
|
253
250
|
headers = {
|
254
251
|
"Content-Type": "application/json",
|
255
252
|
"Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
|
256
253
|
"X-Project-Name": self.project_name,
|
257
254
|
}
|
258
|
-
payload = json.dumps(
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
255
|
+
payload = json.dumps(
|
256
|
+
{
|
257
|
+
"datasetName": self.dataset_name,
|
258
|
+
"schemaMapping": SCHEMA_MAPPING_NEW,
|
259
|
+
"traceFolderUrl": None,
|
260
|
+
}
|
261
|
+
)
|
262
|
+
response = requests.request(
|
263
|
+
"POST",
|
264
264
|
f"{self.base_url}/v1/llm/dataset/logs",
|
265
265
|
headers=headers,
|
266
266
|
data=payload,
|
267
|
-
timeout=self.timeout
|
267
|
+
timeout=self.timeout,
|
268
268
|
)
|
269
269
|
|
270
270
|
return response
|
@@ -277,31 +277,35 @@ class LlamaIndexTracer:
|
|
277
277
|
if response.status_code != 200:
|
278
278
|
return response.status_code
|
279
279
|
return response.status_code
|
280
|
-
|
280
|
+
|
281
281
|
def _get_presigned_url(self):
|
282
|
-
payload = json.dumps(
|
282
|
+
payload = json.dumps(
|
283
|
+
{
|
283
284
|
"datasetName": self.dataset_name,
|
284
285
|
"numFiles": 1,
|
285
|
-
}
|
286
|
+
}
|
287
|
+
)
|
286
288
|
headers = {
|
287
289
|
"Content-Type": "application/json",
|
288
290
|
"Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
|
289
291
|
"X-Project-Name": self.project_name,
|
290
292
|
}
|
291
293
|
|
292
|
-
response = requests.request(
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
294
|
+
response = requests.request(
|
295
|
+
"GET",
|
296
|
+
f"{self.base_url}/v1/llm/presigned-url",
|
297
|
+
headers=headers,
|
298
|
+
data=payload,
|
299
|
+
timeout=self.timeout,
|
300
|
+
)
|
297
301
|
if response.status_code == 200:
|
298
302
|
presignedUrls = response.json()["data"]["presignedUrls"][0]
|
299
303
|
return presignedUrls
|
300
|
-
|
304
|
+
|
301
305
|
def _put_presigned_url(self, presignedUrl, filename):
|
302
306
|
headers = {
|
303
|
-
|
304
|
-
|
307
|
+
"Content-Type": "application/json",
|
308
|
+
}
|
305
309
|
|
306
310
|
if "blob.core.windows.net" in presignedUrl: # Azure
|
307
311
|
headers["x-ms-blob-type"] = "BlockBlob"
|
@@ -309,31 +313,31 @@ class LlamaIndexTracer:
|
|
309
313
|
with open(filename) as f:
|
310
314
|
payload = f.read().replace("\n", "").replace("\r", "").encode()
|
311
315
|
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
headers=headers,
|
316
|
-
data=payload,
|
317
|
-
timeout=self.timeout)
|
316
|
+
response = requests.request(
|
317
|
+
"PUT", presignedUrl, headers=headers, data=payload, timeout=self.timeout
|
318
|
+
)
|
318
319
|
if response.status_code != 200 or response.status_code != 201:
|
319
320
|
return response, response.status_code
|
320
|
-
|
321
|
+
|
321
322
|
def _insert_traces(self, presignedUrl):
|
322
323
|
headers = {
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
payload = json.dumps(
|
324
|
+
"Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
|
325
|
+
"Content-Type": "application/json",
|
326
|
+
"X-Project-Name": self.project_name,
|
327
|
+
}
|
328
|
+
payload = json.dumps(
|
329
|
+
{
|
328
330
|
"datasetName": self.dataset_name,
|
329
331
|
"presignedUrl": presignedUrl,
|
330
|
-
}
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
332
|
+
}
|
333
|
+
)
|
334
|
+
response = requests.request(
|
335
|
+
"POST",
|
336
|
+
f"{self.base_url}/v1/llm/insert/trace",
|
337
|
+
headers=headers,
|
338
|
+
data=payload,
|
339
|
+
timeout=self.timeout,
|
340
|
+
)
|
337
341
|
|
338
342
|
def _upload_traces(self, save_json_to_pwd=None):
|
339
343
|
"""Save traces to a file"""
|
@@ -351,7 +355,7 @@ class LlamaIndexTracer:
|
|
351
355
|
presignedUrl = self._get_presigned_url()
|
352
356
|
self._put_presigned_url(presignedUrl, filename)
|
353
357
|
self._insert_traces(presignedUrl)
|
354
|
-
print("Traces
|
358
|
+
print("Traces uploaded")
|
355
359
|
|
356
360
|
def get_upload_status(self):
|
357
361
|
"""Check the status of the trace upload."""
|
@@ -19,11 +19,13 @@ from .instrumentators import (
|
|
19
19
|
from .utils import get_unique_key
|
20
20
|
# from .llamaindex_callback import LlamaIndexTracer
|
21
21
|
from ..ragaai_catalyst import RagaAICatalyst
|
22
|
+
from .agentic_tracing.agentic_tracing import AgenticTracing
|
23
|
+
from .agentic_tracing.llm_tracer import LLMTracerMixin
|
22
24
|
|
23
25
|
logger = logging.getLogger(__name__)
|
24
26
|
|
25
27
|
|
26
|
-
class Tracer:
|
28
|
+
class Tracer(AgenticTracing):
|
27
29
|
NUM_PROJECTS = 100
|
28
30
|
TIMEOUT = 10
|
29
31
|
def __init__(
|
@@ -41,6 +43,7 @@ class Tracer:
|
|
41
43
|
|
42
44
|
Args:
|
43
45
|
project_name (str): The name of the project.
|
46
|
+
dataset_name (str): The name of the dataset.
|
44
47
|
tracer_type (str, optional): The type of tracer. Defaults to None.
|
45
48
|
pipeline (dict, optional): The pipeline configuration. Defaults to None.
|
46
49
|
metadata (dict, optional): The metadata. Defaults to None.
|
@@ -50,16 +53,28 @@ class Tracer:
|
|
50
53
|
Returns:
|
51
54
|
None
|
52
55
|
"""
|
56
|
+
# Set auto_instrument_llm to True to enable automatic LLM tracing
|
57
|
+
user_detail = {
|
58
|
+
"project_name": project_name,
|
59
|
+
"project_id": None, # Will be set after project validation
|
60
|
+
"dataset_name": dataset_name,
|
61
|
+
"trace_user_detail": {"metadata": metadata} if metadata else {}
|
62
|
+
}
|
63
|
+
super().__init__(user_detail=user_detail, auto_instrument_llm=True)
|
64
|
+
self.is_active = True
|
53
65
|
self.project_name = project_name
|
54
66
|
self.dataset_name = dataset_name
|
55
67
|
self.tracer_type = tracer_type
|
56
68
|
self.metadata = self._improve_metadata(metadata, tracer_type)
|
69
|
+
self.metadata["total_cost"] = 0.0
|
70
|
+
self.metadata["total_tokens"] = 0
|
57
71
|
self.pipeline = pipeline
|
58
72
|
self.description = description
|
59
73
|
self.upload_timeout = upload_timeout
|
60
74
|
self.base_url = f"{RagaAICatalyst.BASE_URL}"
|
61
75
|
self.timeout = 10
|
62
76
|
self.num_projects = 100
|
77
|
+
self.start_time = datetime.datetime.now(datetime.timezone.utc)
|
63
78
|
|
64
79
|
try:
|
65
80
|
response = requests.get(
|
@@ -81,6 +96,7 @@ class Tracer:
|
|
81
96
|
self.project_id = [
|
82
97
|
project["id"] for project in response.json()["data"]["content"] if project["name"] == project_name
|
83
98
|
][0]
|
99
|
+
self._pass_user_data()
|
84
100
|
|
85
101
|
except requests.exceptions.RequestException as e:
|
86
102
|
logger.error(f"Failed to retrieve projects list: {e}")
|
@@ -98,7 +114,8 @@ class Tracer:
|
|
98
114
|
from .llamaindex_callback import LlamaIndexTracer
|
99
115
|
|
100
116
|
else:
|
101
|
-
|
117
|
+
self._upload_task = None
|
118
|
+
# raise ValueError (f"Currently supported tracer types are 'langchain' and 'llamaindex'.")
|
102
119
|
|
103
120
|
def _improve_metadata(self, metadata, tracer_type):
|
104
121
|
if metadata is None:
|
@@ -157,7 +174,9 @@ class Tracer:
|
|
157
174
|
elif self.tracer_type == "llamaindex":
|
158
175
|
from .llamaindex_callback import LlamaIndexTracer
|
159
176
|
return LlamaIndexTracer(self._pass_user_data()).start()
|
160
|
-
|
177
|
+
else:
|
178
|
+
super().start()
|
179
|
+
return self
|
161
180
|
|
162
181
|
def stop(self):
|
163
182
|
"""Stop the tracer and initiate trace upload."""
|
@@ -172,7 +191,9 @@ class Tracer:
|
|
172
191
|
return "Trace upload initiated. Use get_upload_status() to check the status."
|
173
192
|
elif self.tracer_type == "llamaindex":
|
174
193
|
from .llamaindex_callback import LlamaIndexTracer
|
175
|
-
return LlamaIndexTracer().stop()
|
194
|
+
return LlamaIndexTracer(self._pass_user_data()).stop()
|
195
|
+
else:
|
196
|
+
super().stop()
|
176
197
|
|
177
198
|
def get_upload_status(self):
|
178
199
|
"""Check the status of the trace upload."""
|
@@ -262,6 +283,7 @@ class Tracer:
|
|
262
283
|
# Reset instrumentation flag
|
263
284
|
self.is_instrumented = False
|
264
285
|
# Note: We're not resetting all attributes here to allow for upload status checking
|
286
|
+
|
265
287
|
def _pass_user_data(self):
|
266
288
|
return {"project_name":self.project_name,
|
267
289
|
"project_id": self.project_id,
|
@@ -0,0 +1,138 @@
|
|
1
|
+
import requests
|
2
|
+
import json
|
3
|
+
import os
|
4
|
+
from datetime import datetime
|
5
|
+
|
6
|
+
|
7
|
+
class UploadTraces:
|
8
|
+
def __init__(
|
9
|
+
self,
|
10
|
+
json_file_path,
|
11
|
+
project_name,
|
12
|
+
project_id,
|
13
|
+
dataset_name,
|
14
|
+
user_detail,
|
15
|
+
base_url,
|
16
|
+
):
|
17
|
+
self.json_file_path = json_file_path
|
18
|
+
self.project_name = project_name
|
19
|
+
self.project_id = project_id
|
20
|
+
self.dataset_name = dataset_name
|
21
|
+
self.user_detail = user_detail
|
22
|
+
self.base_url = base_url
|
23
|
+
self.timeout = 10
|
24
|
+
|
25
|
+
def _create_dataset_schema_with_trace(self):
|
26
|
+
SCHEMA_MAPPING_NEW = {
|
27
|
+
"trace_id": {"columnType": "traceId"},
|
28
|
+
"trace_uri": {"columnType": "traceUri"},
|
29
|
+
"prompt": {"columnType": "prompt"},
|
30
|
+
"response": {"columnType": "response"},
|
31
|
+
"context": {"columnType": "context"},
|
32
|
+
"llm_model": {"columnType": "pipeline"},
|
33
|
+
"recorded_on": {"columnType": "metadata"},
|
34
|
+
"embed_model": {"columnType": "pipeline"},
|
35
|
+
"log_source": {"columnType": "metadata"},
|
36
|
+
"vector_store": {"columnType": "pipeline"},
|
37
|
+
"feedback": {"columnType": "feedBack"},
|
38
|
+
}
|
39
|
+
|
40
|
+
def make_request():
|
41
|
+
headers = {
|
42
|
+
"Content-Type": "application/json",
|
43
|
+
"Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
|
44
|
+
"X-Project-Name": self.project_name,
|
45
|
+
}
|
46
|
+
payload = json.dumps(
|
47
|
+
{
|
48
|
+
"datasetName": self.dataset_name,
|
49
|
+
"schemaMapping": SCHEMA_MAPPING_NEW,
|
50
|
+
"traceFolderUrl": None,
|
51
|
+
}
|
52
|
+
)
|
53
|
+
response = requests.request(
|
54
|
+
"POST",
|
55
|
+
f"{self.base_url}/v1/llm/dataset/logs",
|
56
|
+
headers=headers,
|
57
|
+
data=payload,
|
58
|
+
timeout=self.timeout,
|
59
|
+
)
|
60
|
+
|
61
|
+
return response
|
62
|
+
|
63
|
+
response = make_request()
|
64
|
+
|
65
|
+
if response.status_code == 401:
|
66
|
+
# get_token() # Fetch a new token and set it in the environment
|
67
|
+
response = make_request() # Retry the request
|
68
|
+
if response.status_code != 200:
|
69
|
+
return response.status_code
|
70
|
+
return response.status_code
|
71
|
+
|
72
|
+
def _get_presigned_url(self):
|
73
|
+
payload = json.dumps(
|
74
|
+
{
|
75
|
+
"datasetName": self.dataset_name,
|
76
|
+
"numFiles": 1,
|
77
|
+
}
|
78
|
+
)
|
79
|
+
headers = {
|
80
|
+
"Content-Type": "application/json",
|
81
|
+
"Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
|
82
|
+
"X-Project-Name": self.project_name,
|
83
|
+
}
|
84
|
+
|
85
|
+
response = requests.request(
|
86
|
+
"GET",
|
87
|
+
f"{self.base_url}/v1/llm/presigned-url",
|
88
|
+
headers=headers,
|
89
|
+
data=payload,
|
90
|
+
timeout=self.timeout,
|
91
|
+
)
|
92
|
+
if response.status_code == 200:
|
93
|
+
presignedUrls = response.json()["data"]["presignedUrls"][0]
|
94
|
+
return presignedUrls
|
95
|
+
|
96
|
+
def _put_presigned_url(self, presignedUrl, filename):
|
97
|
+
headers = {
|
98
|
+
"Content-Type": "application/json",
|
99
|
+
}
|
100
|
+
|
101
|
+
if "blob.core.windows.net" in presignedUrl: # Azure
|
102
|
+
headers["x-ms-blob-type"] = "BlockBlob"
|
103
|
+
print(f"Uploading traces...")
|
104
|
+
with open(filename) as f:
|
105
|
+
payload = f.read().replace("\n", "").replace("\r", "").encode()
|
106
|
+
|
107
|
+
response = requests.request(
|
108
|
+
"PUT", presignedUrl, headers=headers, data=payload, timeout=self.timeout
|
109
|
+
)
|
110
|
+
if response.status_code != 200 or response.status_code != 201:
|
111
|
+
return response, response.status_code
|
112
|
+
|
113
|
+
def _insert_traces(self, presignedUrl):
|
114
|
+
headers = {
|
115
|
+
"Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
|
116
|
+
"Content-Type": "application/json",
|
117
|
+
"X-Project-Name": self.project_name,
|
118
|
+
}
|
119
|
+
payload = json.dumps(
|
120
|
+
{
|
121
|
+
"datasetName": self.dataset_name,
|
122
|
+
"presignedUrl": presignedUrl,
|
123
|
+
}
|
124
|
+
)
|
125
|
+
response = requests.request(
|
126
|
+
"POST",
|
127
|
+
f"{self.base_url}/v1/llm/insert/trace",
|
128
|
+
headers=headers,
|
129
|
+
data=payload,
|
130
|
+
timeout=self.timeout,
|
131
|
+
)
|
132
|
+
|
133
|
+
def upload_traces(self):
|
134
|
+
self._create_dataset_schema_with_trace()
|
135
|
+
presignedUrl = self._get_presigned_url()
|
136
|
+
self._put_presigned_url(presignedUrl, self.json_file_path)
|
137
|
+
self._insert_traces(presignedUrl)
|
138
|
+
print("Traces uploaded")
|
@@ -0,0 +1,39 @@
|
|
1
|
+
Metadata-Version: 2.1
|
2
|
+
Name: ragaai_catalyst
|
3
|
+
Version: 2.0.7.2b1
|
4
|
+
Summary: RAGA AI CATALYST
|
5
|
+
Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
|
6
|
+
Requires-Python: >=3.9
|
7
|
+
Description-Content-Type: text/markdown
|
8
|
+
Requires-Dist: aiohttp>=3.10.2
|
9
|
+
Requires-Dist: opentelemetry-api==1.25.0
|
10
|
+
Requires-Dist: opentelemetry-sdk==1.25.0
|
11
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc==1.25.0
|
12
|
+
Requires-Dist: opentelemetry-instrumentation==0.46b0
|
13
|
+
Requires-Dist: opentelemetry-instrumentation-fastapi==0.46b0
|
14
|
+
Requires-Dist: opentelemetry-instrumentation-asgi==0.46b0
|
15
|
+
Requires-Dist: opentelemetry-semantic-conventions==0.46b0
|
16
|
+
Requires-Dist: opentelemetry-util-http==0.46b0
|
17
|
+
Requires-Dist: opentelemetry-instrumentation-langchain~=0.24.0
|
18
|
+
Requires-Dist: opentelemetry-instrumentation-openai~=0.24.0
|
19
|
+
Requires-Dist: langchain-core>=0.2.11
|
20
|
+
Requires-Dist: langchain>=0.2.11
|
21
|
+
Requires-Dist: openai>=1.57.0
|
22
|
+
Requires-Dist: pandas>=2.1.1
|
23
|
+
Requires-Dist: groq>=0.11.0
|
24
|
+
Requires-Dist: PyPDF2>=3.0.1
|
25
|
+
Requires-Dist: google-generativeai>=0.8.2
|
26
|
+
Requires-Dist: Markdown>=3.7
|
27
|
+
Requires-Dist: litellm==1.51.1
|
28
|
+
Requires-Dist: tenacity==8.3.0
|
29
|
+
Requires-Dist: tqdm>=4.66.5
|
30
|
+
Requires-Dist: llama-index==0.10.0
|
31
|
+
Requires-Dist: pyopenssl==24.2.1
|
32
|
+
Requires-Dist: psutil>=5.9.5
|
33
|
+
Provides-Extra: dev
|
34
|
+
Requires-Dist: pytest; extra == "dev"
|
35
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
36
|
+
Requires-Dist: black; extra == "dev"
|
37
|
+
Requires-Dist: isort; extra == "dev"
|
38
|
+
Requires-Dist: mypy; extra == "dev"
|
39
|
+
Requires-Dist: flake8; extra == "dev"
|
@@ -0,0 +1,50 @@
|
|
1
|
+
ragaai_catalyst/__init__.py,sha256=BdIJ_UUre0uEnRTsLw_hE0C0muWk6XWNZqdVOel22R4,537
|
2
|
+
ragaai_catalyst/_version.py,sha256=JKt9KaVNOMVeGs8ojO6LvIZr7ZkMzNN-gCcvryy4x8E,460
|
3
|
+
ragaai_catalyst/dataset.py,sha256=j_vu3Xkp_8qYW0J9Qnn53Uyh98MsugqYl5zxhOv9EOg,10731
|
4
|
+
ragaai_catalyst/evaluation.py,sha256=xrJ8kxw-fGNJgHsBTJCdH4jrxwzI6jpzUIQ9po-V4kc,20327
|
5
|
+
ragaai_catalyst/experiment.py,sha256=8KvqgJg5JVnt9ghhGDJvdb4mN7ETBX_E5gNxBT0Nsn8,19010
|
6
|
+
ragaai_catalyst/guard_executor.py,sha256=llPbE3DyVtrybojXknzBZj8-dtUrGBQwi9-ZiPJxGRo,3762
|
7
|
+
ragaai_catalyst/guardrails_manager.py,sha256=DILMOAASK57FH9BLq_8yC1AQzRJ8McMFLwCXgYwNAd4,11904
|
8
|
+
ragaai_catalyst/internal_api_completion.py,sha256=DdICI5yfEudiOAIC8L4oxH0Qz7kX-BZCdo9IWsi2gNo,2965
|
9
|
+
ragaai_catalyst/prompt_manager.py,sha256=W8ypramzOprrJ7-22d5vkBXIuIQ8v9XAzKDGxKsTK28,16550
|
10
|
+
ragaai_catalyst/proxy_call.py,sha256=CHxldeceZUaLU-to_hs_Kf1z_b2vHMssLS_cOBedu78,5499
|
11
|
+
ragaai_catalyst/ragaai_catalyst.py,sha256=FdqMzwuQLqS2-3JJDsTQ8uh2itllOxfPrRUjb8Kwmn0,17428
|
12
|
+
ragaai_catalyst/synthetic_data_generation.py,sha256=uDV9tNwto2xSkWg5XHXUvjErW-4P34CTrxaJpRfezyA,19250
|
13
|
+
ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
|
14
|
+
ragaai_catalyst/tracers/__init__.py,sha256=NppmJhD3sQ5R1q6teaZLS7rULj08Gb6JT8XiPRIe_B0,49
|
15
|
+
ragaai_catalyst/tracers/llamaindex_callback.py,sha256=WanVQNhVJZtw2xxRVMKb9Gfut3LWsUD6O3hQlM2HRYM,13760
|
16
|
+
ragaai_catalyst/tracers/tracer.py,sha256=F7LiIUKj1mCl9tl_xwJOWLP-80ICuw5KVBsismLpOM0,12280
|
17
|
+
ragaai_catalyst/tracers/upload_traces.py,sha256=gtLovS56rw_xTYrLSN3s8kKdf9J2R8fti1t3t-QAVxU,4559
|
18
|
+
ragaai_catalyst/tracers/agentic_tracing/Untitled-1.json,sha256=kqwtLzflk342U7Tvjf5Bari9m3I0dVMVU7wNShDwuW4,43639
|
19
|
+
ragaai_catalyst/tracers/agentic_tracing/__init__.py,sha256=6QyQI8P7aNFHTantNJOP1ZdSNrDKBLhlg_gGNj7tm1c,73
|
20
|
+
ragaai_catalyst/tracers/agentic_tracing/agent_tracer.py,sha256=n0fuZOguYScuRoRijz-16okfh54HWlpbTmt133msi64,11815
|
21
|
+
ragaai_catalyst/tracers/agentic_tracing/agentic_tracing.py,sha256=XvoOqEJjqzhE5Xa8mJ2knik6qrecxd1-kRZAFl7P0FI,8680
|
22
|
+
ragaai_catalyst/tracers/agentic_tracing/base.py,sha256=4YWt9Sdrs4IZoJldBN_tl0DCu4zLLEYyDoLvs36oFiw,9955
|
23
|
+
ragaai_catalyst/tracers/agentic_tracing/data_structure.py,sha256=YRZFTzenIIPJ-eDblnUUjY1QeECu6DozfhCgmj38n9s,4696
|
24
|
+
ragaai_catalyst/tracers/agentic_tracing/llm_tracer.py,sha256=hxvUc4CYeVncgzJThykyg8Dd98lgogSGbMch8pjF5C8,36766
|
25
|
+
ragaai_catalyst/tracers/agentic_tracing/network_tracer.py,sha256=6FTA15xMnum9omM_0Jd9cMIuWdKu1gR5Tc8fOXAkP8E,10068
|
26
|
+
ragaai_catalyst/tracers/agentic_tracing/sample.py,sha256=S4rCcKzU_5SB62BYEbNn_1VbbTdG4396N8rdZ3ZNGcE,5654
|
27
|
+
ragaai_catalyst/tracers/agentic_tracing/tool_tracer.py,sha256=O9HJ-lOTK5WYyv6u27W0pvQp4vfsuZGYDua6Ev2zzFc,8407
|
28
|
+
ragaai_catalyst/tracers/agentic_tracing/unique_decorator.py,sha256=WtcALkwS8dsJhqc1641toyH1lVwWyzFwRLBKY56XKJI,7448
|
29
|
+
ragaai_catalyst/tracers/agentic_tracing/unique_decorator_test.py,sha256=Xk1cLzs-2A3dgyBwRRnCWs7Eubki40FVonwd433hPN8,4805
|
30
|
+
ragaai_catalyst/tracers/agentic_tracing/user_interaction_tracer.py,sha256=7Y-31rcoJ4YeZn9gSnqJgujGpRX6rKAkq6Y0KDIMTOo,2085
|
31
|
+
ragaai_catalyst/tracers/agentic_tracing/utils/__init__.py,sha256=XdB3X_ufe4RVvGorxSqAiB9dYv4UD7Hvvuw3bsDUppY,60
|
32
|
+
ragaai_catalyst/tracers/agentic_tracing/utils/api_utils.py,sha256=JyNCbfpW-w4O9CjtemTqmor2Rh1WGpQwhRaDSRmBxw8,689
|
33
|
+
ragaai_catalyst/tracers/agentic_tracing/utils/data_classes.py,sha256=gFOadWzUORETaH3Y_HerHXs_aVFEt5ry2NQ36domW58,1267
|
34
|
+
ragaai_catalyst/tracers/agentic_tracing/utils/generic.py,sha256=WwXT01xmp8MSr7KinuDCSK9a1ifpLcT7ajFkvYviG_A,1190
|
35
|
+
ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py,sha256=noCDrF1tG-n1LSdooq_-kkBw8z35YvsiGGhcy1-yL4M,5810
|
36
|
+
ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json,sha256=5jvhriJJkWT-nz8S-KsgsjOn9g7D2hKYob8URl9FHOk,215176
|
37
|
+
ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py,sha256=9cFzfFqIA968bUG7LNTjdN7zbdEXUtcvRKg883ade2c,2586
|
38
|
+
ragaai_catalyst/tracers/exporters/__init__.py,sha256=kVA8zp05h3phu4e-iHSlnznp_PzMRczB7LphSsZgUjg,138
|
39
|
+
ragaai_catalyst/tracers/exporters/file_span_exporter.py,sha256=RgGteu-NVGprXKkynvyIO5yOjpbtA41R3W_NzCjnkwE,6445
|
40
|
+
ragaai_catalyst/tracers/exporters/raga_exporter.py,sha256=rQ5Wj71f2Ke3qLlV8KiWCskbGBR-ia_hlzDx86rPrEo,18188
|
41
|
+
ragaai_catalyst/tracers/instrumentators/__init__.py,sha256=FgnMQupoRTzmVsG9YKsLQera2Pfs-AluZv8CxwavoyQ,253
|
42
|
+
ragaai_catalyst/tracers/instrumentators/langchain.py,sha256=yMN0qVF0pUVk6R5M1vJoUXezDo1ejs4klCFRlE8x4vE,574
|
43
|
+
ragaai_catalyst/tracers/instrumentators/llamaindex.py,sha256=SMrRlR4xM7k9HK43hakE8rkrWHxMlmtmWD-AX6TeByc,416
|
44
|
+
ragaai_catalyst/tracers/instrumentators/openai.py,sha256=14R4KW9wQCR1xysLfsP_nxS7cqXrTPoD8En4MBAaZUU,379
|
45
|
+
ragaai_catalyst/tracers/utils/__init__.py,sha256=KeMaZtYaTojilpLv65qH08QmpYclfpacDA0U3wg6Ybw,64
|
46
|
+
ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
|
47
|
+
ragaai_catalyst-2.0.7.2b1.dist-info/METADATA,sha256=HWc-Jm90W0_CmYO7jEHWz8oUTDdJOkYSXkyPo5tEFD4,1677
|
48
|
+
ragaai_catalyst-2.0.7.2b1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
49
|
+
ragaai_catalyst-2.0.7.2b1.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
|
50
|
+
ragaai_catalyst-2.0.7.2b1.dist-info/RECORD,,
|