ragaai-catalyst 2.1.5b3__py3-none-any.whl → 2.1.5b5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json +26 -8
- ragaai_catalyst/tracers/langchain_callback.py +568 -0
- ragaai_catalyst/tracers/tracer.py +10 -10
- {ragaai_catalyst-2.1.5b3.dist-info → ragaai_catalyst-2.1.5b5.dist-info}/METADATA +1 -1
- {ragaai_catalyst-2.1.5b3.dist-info → ragaai_catalyst-2.1.5b5.dist-info}/RECORD +8 -7
- {ragaai_catalyst-2.1.5b3.dist-info → ragaai_catalyst-2.1.5b5.dist-info}/LICENSE +0 -0
- {ragaai_catalyst-2.1.5b3.dist-info → ragaai_catalyst-2.1.5b5.dist-info}/WHEEL +0 -0
- {ragaai_catalyst-2.1.5b3.dist-info → ragaai_catalyst-2.1.5b5.dist-info}/top_level.txt +0 -0
@@ -5223,6 +5223,24 @@
|
|
5223
5223
|
"mode": "chat",
|
5224
5224
|
"supports_system_messages": true
|
5225
5225
|
},
|
5226
|
+
"ai21.jamba-1-5-large-v1:0": {
|
5227
|
+
"max_tokens": 256000,
|
5228
|
+
"max_input_tokens": 256000,
|
5229
|
+
"max_output_tokens": 256000,
|
5230
|
+
"input_cost_per_token": 2e-06,
|
5231
|
+
"output_cost_per_token": 8e-06,
|
5232
|
+
"litellm_provider": "bedrock",
|
5233
|
+
"mode": "chat"
|
5234
|
+
},
|
5235
|
+
"ai21.jamba-1-5-mini-v1:0": {
|
5236
|
+
"max_tokens": 256000,
|
5237
|
+
"max_input_tokens": 256000,
|
5238
|
+
"max_output_tokens": 256000,
|
5239
|
+
"input_cost_per_token": 2e-07,
|
5240
|
+
"output_cost_per_token": 4e-07,
|
5241
|
+
"litellm_provider": "bedrock",
|
5242
|
+
"mode": "chat"
|
5243
|
+
},
|
5226
5244
|
"amazon.titan-text-lite-v1": {
|
5227
5245
|
"max_tokens": 4000,
|
5228
5246
|
"max_input_tokens": 42000,
|
@@ -5541,8 +5559,8 @@
|
|
5541
5559
|
"max_tokens": 8192,
|
5542
5560
|
"max_input_tokens": 200000,
|
5543
5561
|
"max_output_tokens": 8192,
|
5544
|
-
"input_cost_per_token":
|
5545
|
-
"output_cost_per_token":
|
5562
|
+
"input_cost_per_token": 8e-07,
|
5563
|
+
"output_cost_per_token": 4e-06,
|
5546
5564
|
"litellm_provider": "bedrock",
|
5547
5565
|
"mode": "chat",
|
5548
5566
|
"supports_assistant_prefill": true,
|
@@ -5611,8 +5629,8 @@
|
|
5611
5629
|
"max_tokens": 8192,
|
5612
5630
|
"max_input_tokens": 200000,
|
5613
5631
|
"max_output_tokens": 8192,
|
5614
|
-
"input_cost_per_token":
|
5615
|
-
"output_cost_per_token":
|
5632
|
+
"input_cost_per_token": 8e-07,
|
5633
|
+
"output_cost_per_token": 4e-06,
|
5616
5634
|
"litellm_provider": "bedrock",
|
5617
5635
|
"mode": "chat",
|
5618
5636
|
"supports_assistant_prefill": true,
|
@@ -5681,8 +5699,8 @@
|
|
5681
5699
|
"max_tokens": 8192,
|
5682
5700
|
"max_input_tokens": 200000,
|
5683
5701
|
"max_output_tokens": 8192,
|
5684
|
-
"input_cost_per_token":
|
5685
|
-
"output_cost_per_token":
|
5702
|
+
"input_cost_per_token": 2.5e-07,
|
5703
|
+
"output_cost_per_token": 1.25e-06,
|
5686
5704
|
"litellm_provider": "bedrock",
|
5687
5705
|
"mode": "chat",
|
5688
5706
|
"supports_function_calling": true,
|
@@ -6056,8 +6074,8 @@
|
|
6056
6074
|
"max_tokens": 8191,
|
6057
6075
|
"max_input_tokens": 100000,
|
6058
6076
|
"max_output_tokens": 8191,
|
6059
|
-
"input_cost_per_token":
|
6060
|
-
"output_cost_per_token":
|
6077
|
+
"input_cost_per_token": 8e-07,
|
6078
|
+
"output_cost_per_token": 2.4e-06,
|
6061
6079
|
"litellm_provider": "bedrock",
|
6062
6080
|
"mode": "chat"
|
6063
6081
|
},
|
@@ -0,0 +1,568 @@
|
|
1
|
+
from typing import Any, Dict, List, Optional, Union, Sequence
|
2
|
+
|
3
|
+
import attr
|
4
|
+
from langchain.callbacks.base import BaseCallbackHandler
|
5
|
+
from langchain.schema import LLMResult, AgentAction, AgentFinish, BaseMessage
|
6
|
+
from datetime import datetime
|
7
|
+
import json
|
8
|
+
import os
|
9
|
+
from uuid import UUID
|
10
|
+
from functools import wraps
|
11
|
+
import asyncio
|
12
|
+
from langchain_core.documents import Document
|
13
|
+
import logging
|
14
|
+
import tempfile
|
15
|
+
|
16
|
+
logging.basicConfig(level=logging.INFO)
|
17
|
+
logger = logging.getLogger(__name__)
|
18
|
+
|
19
|
+
|
20
|
+
class LangchainTracer(BaseCallbackHandler):
|
21
|
+
"""
|
22
|
+
An enhanced callback handler for LangChain that traces all actions and saves them to a JSON file.
|
23
|
+
Includes improved error handling, async support, and configuration options.
|
24
|
+
"""
|
25
|
+
|
26
|
+
def __init__(
|
27
|
+
self,
|
28
|
+
output_path: str = tempfile.gettempdir(),
|
29
|
+
trace_all: bool = True,
|
30
|
+
save_interval: Optional[int] = None,
|
31
|
+
log_level: int = logging.INFO,
|
32
|
+
):
|
33
|
+
"""
|
34
|
+
Initialize the tracer with enhanced configuration options.
|
35
|
+
|
36
|
+
Args:
|
37
|
+
output_path (str): Directory where trace files will be saved
|
38
|
+
trace_all (bool): Whether to trace all components or only specific ones
|
39
|
+
save_interval (Optional[int]): Interval in seconds to auto-save traces
|
40
|
+
log_level (int): Logging level for the tracer
|
41
|
+
"""
|
42
|
+
super().__init__()
|
43
|
+
self.output_path = output_path
|
44
|
+
self.trace_all = trace_all
|
45
|
+
self.save_interval = save_interval
|
46
|
+
self._active = False
|
47
|
+
self._original_inits = {}
|
48
|
+
self._original_methods = {}
|
49
|
+
self.additional_metadata = {}
|
50
|
+
self._save_task = None
|
51
|
+
self._current_query = None # Add this line to track the current query
|
52
|
+
self.filepath = None
|
53
|
+
logger.setLevel(log_level)
|
54
|
+
|
55
|
+
if not os.path.exists(output_path):
|
56
|
+
os.makedirs(output_path)
|
57
|
+
|
58
|
+
self.reset_trace()
|
59
|
+
|
60
|
+
|
61
|
+
def __enter__(self):
|
62
|
+
"""Context manager entry"""
|
63
|
+
self.start()
|
64
|
+
return self
|
65
|
+
|
66
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
67
|
+
"""Context manager exit"""
|
68
|
+
|
69
|
+
self.stop()
|
70
|
+
if exc_type:
|
71
|
+
logger.error(f"Error in context manager: {exc_val}")
|
72
|
+
return False
|
73
|
+
return True
|
74
|
+
|
75
|
+
def reset_trace(self):
|
76
|
+
"""Reset the current trace to initial state with enhanced structure"""
|
77
|
+
self.current_trace: Dict[str, Any] = {
|
78
|
+
"start_time": None,
|
79
|
+
"end_time": None,
|
80
|
+
"actions": [],
|
81
|
+
"llm_calls": [],
|
82
|
+
"chain_starts": [],
|
83
|
+
"chain_ends": [],
|
84
|
+
"agent_actions": [],
|
85
|
+
"chat_model_calls": [],
|
86
|
+
"retriever_actions": [],
|
87
|
+
"tokens": [],
|
88
|
+
"errors": [],
|
89
|
+
"query": self._current_query, # Add this line to include the query in the trace
|
90
|
+
"metadata": {
|
91
|
+
"version": "2.0",
|
92
|
+
"trace_all": self.trace_all,
|
93
|
+
"save_interval": self.save_interval,
|
94
|
+
},
|
95
|
+
}
|
96
|
+
|
97
|
+
async def _periodic_save(self):
|
98
|
+
"""Periodically save traces if save_interval is set"""
|
99
|
+
while self._active and self.save_interval:
|
100
|
+
await asyncio.sleep(self.save_interval)
|
101
|
+
await self._async_save_trace()
|
102
|
+
|
103
|
+
async def _async_save_trace(self, force: bool = False):
|
104
|
+
"""Asynchronously save the current trace to a JSON file"""
|
105
|
+
if not self.current_trace["start_time"] and not force:
|
106
|
+
return
|
107
|
+
|
108
|
+
try:
|
109
|
+
self.current_trace["end_time"] = datetime.now()
|
110
|
+
|
111
|
+
# Use the query from the trace or fallback to a default
|
112
|
+
safe_query = self._current_query or "unknown"
|
113
|
+
|
114
|
+
# Sanitize the query for filename
|
115
|
+
safe_query = ''.join(c for c in safe_query if c.isalnum() or c.isspace())[:50].strip()
|
116
|
+
|
117
|
+
# Add a timestamp to ensure unique filenames
|
118
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
119
|
+
filename = f"langchain_callback_traces.json"
|
120
|
+
filepath = os.path.join(self.output_path, filename)
|
121
|
+
self.filepath = filepath
|
122
|
+
|
123
|
+
trace_to_save = self.current_trace.copy()
|
124
|
+
trace_to_save["start_time"] = str(trace_to_save["start_time"])
|
125
|
+
trace_to_save["end_time"] = str(trace_to_save["end_time"])
|
126
|
+
|
127
|
+
# Save if there are meaningful events or if force is True
|
128
|
+
if (
|
129
|
+
len(trace_to_save["llm_calls"]) > 0
|
130
|
+
or len(trace_to_save["chain_starts"]) > 0
|
131
|
+
or len(trace_to_save["chain_ends"]) > 0
|
132
|
+
or len(trace_to_save["errors"]) > 0
|
133
|
+
or force
|
134
|
+
):
|
135
|
+
async with asyncio.Lock():
|
136
|
+
with open(filepath, "w", encoding="utf-8") as f:
|
137
|
+
json.dump(trace_to_save, f, indent=2, default=str)
|
138
|
+
|
139
|
+
logger.info(f"Trace saved to: {filepath}")
|
140
|
+
|
141
|
+
# Reset the current query after saving
|
142
|
+
self._current_query = None
|
143
|
+
|
144
|
+
# Reset the trace
|
145
|
+
self.reset_trace()
|
146
|
+
|
147
|
+
except Exception as e:
|
148
|
+
logger.error(f"Error saving trace: {e}")
|
149
|
+
self.on_error(e, context="save_trace")
|
150
|
+
|
151
|
+
def _save_trace(self, force: bool = False):
|
152
|
+
"""Synchronous version of trace saving"""
|
153
|
+
if asyncio.get_event_loop().is_running():
|
154
|
+
asyncio.create_task(self._async_save_trace(force))
|
155
|
+
else:
|
156
|
+
asyncio.run(self._async_save_trace(force))
|
157
|
+
|
158
|
+
def _create_safe_wrapper(self, original_func, component_name):
|
159
|
+
"""Create a safely wrapped version of an original function with enhanced error handling"""
|
160
|
+
|
161
|
+
@wraps(original_func)
|
162
|
+
def wrapped(*args, **kwargs):
|
163
|
+
if not self._active:
|
164
|
+
return original_func(*args, **kwargs)
|
165
|
+
|
166
|
+
try:
|
167
|
+
# Deep copy kwargs to avoid modifying the original
|
168
|
+
kwargs_copy = kwargs.copy() if kwargs is not None else {}
|
169
|
+
|
170
|
+
# Handle different calling conventions
|
171
|
+
if 'callbacks' not in kwargs_copy:
|
172
|
+
kwargs_copy['callbacks'] = [self]
|
173
|
+
elif self not in kwargs_copy['callbacks']:
|
174
|
+
kwargs_copy['callbacks'].append(self)
|
175
|
+
|
176
|
+
# Try different method signatures
|
177
|
+
try:
|
178
|
+
# First, try calling with modified kwargs
|
179
|
+
return original_func(*args, **kwargs_copy)
|
180
|
+
except TypeError:
|
181
|
+
# If that fails, try without kwargs
|
182
|
+
try:
|
183
|
+
return original_func(*args)
|
184
|
+
except Exception as e:
|
185
|
+
# If all else fails, use original call
|
186
|
+
logger.error(f"Failed to invoke {component_name} with modified callbacks: {e}")
|
187
|
+
return original_func(*args, **kwargs)
|
188
|
+
|
189
|
+
except Exception as e:
|
190
|
+
# Log any errors that occur during the function call
|
191
|
+
logger.error(f"Error in {component_name} wrapper: {e}")
|
192
|
+
|
193
|
+
# Record the error using the tracer's error handling method
|
194
|
+
self.on_error(e, context=f"wrapper_{component_name}")
|
195
|
+
|
196
|
+
# Fallback to calling the original function without modifications
|
197
|
+
return original_func(*args, **kwargs)
|
198
|
+
|
199
|
+
return wrapped
|
200
|
+
|
201
|
+
|
202
|
+
def _monkey_patch(self):
|
203
|
+
"""Enhanced monkey-patching with comprehensive component support"""
|
204
|
+
from langchain.llms import OpenAI
|
205
|
+
# from langchain_groq import ChatGroq
|
206
|
+
# from langchain_google_genai import ChatGoogleGenerativeAI
|
207
|
+
# from langchain_anthropic import ChatAnthropic
|
208
|
+
from langchain_community.chat_models import ChatLiteLLM
|
209
|
+
# from langchain_cohere import ChatCohere
|
210
|
+
from langchain_openai import ChatOpenAI as ChatOpenAI_LangchainOpenAI
|
211
|
+
from langchain.chat_models import ChatOpenAI as ChatOpenAI_ChatModels
|
212
|
+
from langchain.chains import create_retrieval_chain, RetrievalQA
|
213
|
+
|
214
|
+
components_to_patch = {
|
215
|
+
"OpenAI": (OpenAI, "__init__"),
|
216
|
+
# "ChatGroq": (ChatGroq, "__init__"),
|
217
|
+
# "ChatGoogleGenerativeAI": (ChatGoogleGenerativeAI, "__init__"),
|
218
|
+
# "ChatAnthropic": (ChatAnthropic, "__init__"),
|
219
|
+
"ChatLiteLLM": (ChatLiteLLM, "__init__"),
|
220
|
+
# "ChatCohere": (ChatCohere, "__init__"),
|
221
|
+
"ChatOpenAI_LangchainOpenAI": (ChatOpenAI_LangchainOpenAI, "__init__"),
|
222
|
+
"ChatOpenAI_ChatModels": (ChatOpenAI_ChatModels, "__init__"),
|
223
|
+
"RetrievalQA": (RetrievalQA, "from_chain_type"),
|
224
|
+
"create_retrieval_chain": (create_retrieval_chain, None),
|
225
|
+
}
|
226
|
+
|
227
|
+
for name, (component, method_name) in components_to_patch.items():
|
228
|
+
try:
|
229
|
+
if method_name == "__init__":
|
230
|
+
original = component.__init__
|
231
|
+
self._original_inits[name] = original
|
232
|
+
component.__init__ = self._create_safe_wrapper(original, name)
|
233
|
+
elif method_name:
|
234
|
+
original = getattr(component, method_name)
|
235
|
+
self._original_methods[name] = original
|
236
|
+
if isinstance(original, classmethod):
|
237
|
+
wrapped = classmethod(
|
238
|
+
self._create_safe_wrapper(original.__func__, name)
|
239
|
+
)
|
240
|
+
else:
|
241
|
+
wrapped = self._create_safe_wrapper(original, name)
|
242
|
+
setattr(component, method_name, wrapped)
|
243
|
+
else:
|
244
|
+
self._original_methods[name] = component
|
245
|
+
globals()[name] = self._create_safe_wrapper(component, name)
|
246
|
+
except Exception as e:
|
247
|
+
logger.error(f"Error patching {name}: {e}")
|
248
|
+
self.on_error(e, context=f"patch_{name}")
|
249
|
+
|
250
|
+
def _restore_original_methods(self):
|
251
|
+
"""Restore all original methods and functions with enhanced error handling"""
|
252
|
+
from langchain.llms import OpenAI
|
253
|
+
# from langchain_groq import ChatGroq
|
254
|
+
# from langchain_google_genai import ChatGoogleGenerativeAI
|
255
|
+
# from langchain_anthropic import ChatAnthropic
|
256
|
+
from langchain_community.chat_models import ChatLiteLLM
|
257
|
+
# from langchain_cohere import ChatCohere
|
258
|
+
from langchain_openai import ChatOpenAI as ChatOpenAI_LangchainOpenAI
|
259
|
+
from langchain.chat_models import ChatOpenAI as ChatOpenAI_ChatModels
|
260
|
+
from langchain.chains import create_retrieval_chain, RetrievalQA
|
261
|
+
|
262
|
+
|
263
|
+
for name, original in self._original_inits.items():
|
264
|
+
try:
|
265
|
+
component = eval(name)
|
266
|
+
component.__init__ = original
|
267
|
+
except Exception as e:
|
268
|
+
logger.error(f"Error restoring {name}: {e}")
|
269
|
+
self.on_error(e, context=f"restore_{name}")
|
270
|
+
|
271
|
+
for name, original in self._original_methods.items():
|
272
|
+
try:
|
273
|
+
if "." in name:
|
274
|
+
module_name, method_name = name.rsplit(".", 1)
|
275
|
+
module = eval(module_name)
|
276
|
+
setattr(module, method_name, original)
|
277
|
+
else:
|
278
|
+
globals()[name] = original
|
279
|
+
except Exception as e:
|
280
|
+
logger.error(f"Error restoring {name}: {e}")
|
281
|
+
self.on_error(e, context=f"restore_{name}")
|
282
|
+
|
283
|
+
def start(self):
|
284
|
+
"""Start tracing with enhanced error handling and async support"""
|
285
|
+
try:
|
286
|
+
self.reset_trace()
|
287
|
+
self.current_trace["start_time"] = datetime.now()
|
288
|
+
self._active = True
|
289
|
+
self._monkey_patch()
|
290
|
+
|
291
|
+
if self.save_interval:
|
292
|
+
loop = asyncio.get_event_loop()
|
293
|
+
self._save_task = loop.create_task(self._periodic_save())
|
294
|
+
|
295
|
+
logger.info("Tracing started")
|
296
|
+
except Exception as e:
|
297
|
+
logger.error(f"Error starting tracer: {e}")
|
298
|
+
self.on_error(e, context="start")
|
299
|
+
raise
|
300
|
+
|
301
|
+
def stop(self):
|
302
|
+
"""Stop tracing with enhanced cleanup"""
|
303
|
+
try:
|
304
|
+
self._active = False
|
305
|
+
if self._save_task:
|
306
|
+
self._save_task.cancel()
|
307
|
+
self._restore_original_methods()
|
308
|
+
# self._save_trace(force=True)
|
309
|
+
|
310
|
+
return self.current_trace.copy(), self.additional_metadata
|
311
|
+
|
312
|
+
logger.info("Tracing stopped")
|
313
|
+
except Exception as e:
|
314
|
+
logger.error(f"Error stopping tracer: {e}")
|
315
|
+
self.on_error(e, context="stop")
|
316
|
+
raise
|
317
|
+
finally:
|
318
|
+
self._original_inits.clear()
|
319
|
+
self._original_methods.clear()
|
320
|
+
|
321
|
+
def force_save(self):
|
322
|
+
"""Force save the current trace"""
|
323
|
+
self._save_trace(force=True)
|
324
|
+
|
325
|
+
# Callback methods with enhanced error handling and logging
|
326
|
+
def on_llm_start(
|
327
|
+
self,
|
328
|
+
serialized: Dict[str, Any],
|
329
|
+
prompts: List[str],
|
330
|
+
run_id: UUID,
|
331
|
+
**kwargs: Any,
|
332
|
+
) -> None:
|
333
|
+
try:
|
334
|
+
if not self.current_trace["start_time"]:
|
335
|
+
self.current_trace["start_time"] = datetime.now()
|
336
|
+
|
337
|
+
self.current_trace["llm_calls"].append(
|
338
|
+
{
|
339
|
+
"timestamp": datetime.now(),
|
340
|
+
"event": "llm_start",
|
341
|
+
"serialized": serialized,
|
342
|
+
"prompts": prompts,
|
343
|
+
"run_id": str(run_id),
|
344
|
+
"additional_kwargs": kwargs,
|
345
|
+
}
|
346
|
+
)
|
347
|
+
except Exception as e:
|
348
|
+
self.on_error(e, context="llm_start")
|
349
|
+
|
350
|
+
def on_llm_end(self, response: LLMResult, *, run_id: UUID, **kwargs: Any) -> None:
|
351
|
+
try:
|
352
|
+
self.current_trace["llm_calls"].append(
|
353
|
+
{
|
354
|
+
"timestamp": datetime.now(),
|
355
|
+
"event": "llm_end",
|
356
|
+
"response": response.dict(),
|
357
|
+
"run_id": str(run_id),
|
358
|
+
"additional_kwargs": kwargs,
|
359
|
+
}
|
360
|
+
)
|
361
|
+
|
362
|
+
end_time = datetime.now()
|
363
|
+
self.additional_metadata["latency"] = (end_time - self.current_trace["start_time"]).total_seconds()
|
364
|
+
|
365
|
+
if response and response.llm_output:
|
366
|
+
self.additional_metadata["model_name"] = response.llm_output.get("model_name", "")
|
367
|
+
self.additional_metadata["tokens"] = {}
|
368
|
+
if response.llm_output.get("token_usage"):
|
369
|
+
self.additional_metadata["tokens"]["total"] = response.llm_output["token_usage"].get("total_tokens", 0)
|
370
|
+
self.additional_metadata["tokens"]["prompt"] = response.llm_output["token_usage"].get("prompt_tokens", 0)
|
371
|
+
self.additional_metadata["tokens"]["completion"] = response.llm_output["token_usage"].get("completion_tokens", 0)
|
372
|
+
except Exception as e:
|
373
|
+
self.on_error(e, context="llm_end")
|
374
|
+
|
375
|
+
def on_chat_model_start(
|
376
|
+
self,
|
377
|
+
serialized: Dict[str, Any],
|
378
|
+
messages: List[List[BaseMessage]],
|
379
|
+
*,
|
380
|
+
run_id: UUID,
|
381
|
+
**kwargs: Any,
|
382
|
+
) -> None:
|
383
|
+
try:
|
384
|
+
messages_dict = [
|
385
|
+
[
|
386
|
+
{
|
387
|
+
"type": msg.type,
|
388
|
+
"content": msg.content,
|
389
|
+
"additional_kwargs": msg.additional_kwargs,
|
390
|
+
}
|
391
|
+
for msg in batch
|
392
|
+
]
|
393
|
+
for batch in messages
|
394
|
+
]
|
395
|
+
|
396
|
+
self.current_trace["chat_model_calls"].append(
|
397
|
+
{
|
398
|
+
"timestamp": datetime.now(),
|
399
|
+
"event": "chat_model_start",
|
400
|
+
"serialized": serialized,
|
401
|
+
"messages": messages_dict,
|
402
|
+
"run_id": str(run_id),
|
403
|
+
"additional_kwargs": kwargs,
|
404
|
+
}
|
405
|
+
)
|
406
|
+
except Exception as e:
|
407
|
+
self.on_error(e, context="chat_model_start")
|
408
|
+
|
409
|
+
def on_chain_start(
|
410
|
+
self,
|
411
|
+
serialized: Dict[str, Any],
|
412
|
+
inputs: Dict[str, Any],
|
413
|
+
*,
|
414
|
+
run_id: UUID,
|
415
|
+
**kwargs: Any,
|
416
|
+
) -> None:
|
417
|
+
try:
|
418
|
+
context = ""
|
419
|
+
query = ""
|
420
|
+
if isinstance(inputs, dict):
|
421
|
+
if "context" in inputs:
|
422
|
+
if isinstance(inputs["context"], Document):
|
423
|
+
context = inputs["context"].page_content
|
424
|
+
elif isinstance(inputs["context"], list):
|
425
|
+
context = "\n".join(
|
426
|
+
doc.page_content if isinstance(doc, Document) else str(doc)
|
427
|
+
for doc in inputs["context"]
|
428
|
+
)
|
429
|
+
elif isinstance(inputs["context"], str):
|
430
|
+
context = inputs["context"]
|
431
|
+
|
432
|
+
query = inputs.get("question", inputs.get("input", ""))
|
433
|
+
|
434
|
+
# Set the current query
|
435
|
+
self._current_query = query
|
436
|
+
|
437
|
+
chain_event = {
|
438
|
+
"timestamp": datetime.now(),
|
439
|
+
"serialized": serialized,
|
440
|
+
"context": context,
|
441
|
+
"query": inputs.get("question", inputs.get("input", "")),
|
442
|
+
"run_id": str(run_id),
|
443
|
+
"additional_kwargs": kwargs,
|
444
|
+
}
|
445
|
+
|
446
|
+
self.current_trace["chain_starts"].append(chain_event)
|
447
|
+
except Exception as e:
|
448
|
+
self.on_error(e, context="chain_start")
|
449
|
+
|
450
|
+
def on_chain_end(
|
451
|
+
self, outputs: Dict[str, Any], *, run_id: UUID, **kwargs: Any
|
452
|
+
) -> None:
|
453
|
+
try:
|
454
|
+
self.current_trace["chain_ends"].append(
|
455
|
+
{
|
456
|
+
"timestamp": datetime.now(),
|
457
|
+
"outputs": outputs,
|
458
|
+
"run_id": str(run_id),
|
459
|
+
"additional_kwargs": kwargs,
|
460
|
+
}
|
461
|
+
)
|
462
|
+
except Exception as e:
|
463
|
+
self.on_error(e, context="chain_end")
|
464
|
+
|
465
|
+
def on_agent_action(self, action: AgentAction, run_id: UUID, **kwargs: Any) -> None:
|
466
|
+
try:
|
467
|
+
self.current_trace["agent_actions"].append(
|
468
|
+
{
|
469
|
+
"timestamp": datetime.now(),
|
470
|
+
"action": action.dict(),
|
471
|
+
"run_id": str(run_id),
|
472
|
+
"additional_kwargs": kwargs,
|
473
|
+
}
|
474
|
+
)
|
475
|
+
except Exception as e:
|
476
|
+
self.on_error(e, context="agent_action")
|
477
|
+
|
478
|
+
def on_agent_finish(self, finish: AgentFinish, run_id: UUID, **kwargs: Any) -> None:
|
479
|
+
try:
|
480
|
+
self.current_trace["agent_actions"].append(
|
481
|
+
{
|
482
|
+
"timestamp": datetime.now(),
|
483
|
+
"event": "agent_finish",
|
484
|
+
"finish": finish.dict(),
|
485
|
+
"run_id": str(run_id),
|
486
|
+
"additional_kwargs": kwargs,
|
487
|
+
}
|
488
|
+
)
|
489
|
+
except Exception as e:
|
490
|
+
self.on_error(e, context="agent_finish")
|
491
|
+
|
492
|
+
def on_retriever_start(
|
493
|
+
self, serialized: Dict[str, Any], query: str, *, run_id: UUID, **kwargs: Any
|
494
|
+
) -> None:
|
495
|
+
try:
|
496
|
+
retriever_event = {
|
497
|
+
"timestamp": datetime.now(),
|
498
|
+
"event": "retriever_start",
|
499
|
+
"serialized": serialized,
|
500
|
+
"query": query,
|
501
|
+
"run_id": str(run_id),
|
502
|
+
"additional_kwargs": kwargs,
|
503
|
+
}
|
504
|
+
|
505
|
+
self.current_trace["retriever_actions"].append(retriever_event)
|
506
|
+
except Exception as e:
|
507
|
+
self.on_error(e, context="retriever_start")
|
508
|
+
|
509
|
+
def on_retriever_end(
|
510
|
+
self, documents: Sequence[Document], *, run_id: UUID, **kwargs: Any
|
511
|
+
) -> None:
|
512
|
+
try:
|
513
|
+
processed_documents = [
|
514
|
+
{"page_content": doc.page_content, "metadata": doc.metadata}
|
515
|
+
for doc in documents
|
516
|
+
]
|
517
|
+
|
518
|
+
retriever_event = {
|
519
|
+
"timestamp": datetime.now(),
|
520
|
+
"event": "retriever_end",
|
521
|
+
"documents": processed_documents,
|
522
|
+
"run_id": str(run_id),
|
523
|
+
"additional_kwargs": kwargs,
|
524
|
+
}
|
525
|
+
|
526
|
+
self.current_trace["retriever_actions"].append(retriever_event)
|
527
|
+
except Exception as e:
|
528
|
+
self.on_error(e, context="retriever_end")
|
529
|
+
|
530
|
+
def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
|
531
|
+
try:
|
532
|
+
self.current_trace["tokens"].append(
|
533
|
+
{
|
534
|
+
"timestamp": datetime.now(),
|
535
|
+
"event": "new_token",
|
536
|
+
"token": token,
|
537
|
+
"additional_kwargs": kwargs,
|
538
|
+
}
|
539
|
+
)
|
540
|
+
except Exception as e:
|
541
|
+
self.on_error(e, context="llm_new_token")
|
542
|
+
|
543
|
+
def on_error(self, error: Exception, context: str = "", **kwargs: Any) -> None:
|
544
|
+
"""Enhanced error handling with context"""
|
545
|
+
try:
|
546
|
+
error_event = {
|
547
|
+
"timestamp": datetime.now(),
|
548
|
+
"error": str(error),
|
549
|
+
"error_type": type(error).__name__,
|
550
|
+
"context": context,
|
551
|
+
"additional_kwargs": kwargs,
|
552
|
+
}
|
553
|
+
self.current_trace["errors"].append(error_event)
|
554
|
+
logger.error(f"Error in {context}: {error}")
|
555
|
+
except Exception as e:
|
556
|
+
logger.critical(f"Error in error handler: {e}")
|
557
|
+
|
558
|
+
def on_chain_error(self, error: Exception, **kwargs: Any) -> None:
|
559
|
+
self.on_error(error, context="chain", **kwargs)
|
560
|
+
|
561
|
+
def on_llm_error(self, error: Exception, **kwargs: Any) -> None:
|
562
|
+
self.on_error(error, context="llm", **kwargs)
|
563
|
+
|
564
|
+
def on_tool_error(self, error: Exception, **kwargs: Any) -> None:
|
565
|
+
self.on_error(error, context="tool", **kwargs)
|
566
|
+
|
567
|
+
def on_retriever_error(self, error: Exception, **kwargs: Any) -> None:
|
568
|
+
self.on_error(error, context="retriever", **kwargs)
|
@@ -283,20 +283,20 @@ class Tracer(AgenticTracing):
|
|
283
283
|
data, additional_metadata = self.langchain_tracer.stop()
|
284
284
|
|
285
285
|
# Add cost if possible
|
286
|
-
|
287
|
-
if additional_metadata['model_name']:
|
286
|
+
if additional_metadata.get('model_name'):
|
288
287
|
try:
|
289
288
|
model_cost_data = self.model_cost_dict[additional_metadata['model_name']]
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
289
|
+
if 'tokens' in additional_metadata and all(k in additional_metadata['tokens'] for k in ['prompt', 'completion']):
|
290
|
+
prompt_cost = additional_metadata["tokens"]["prompt"]*model_cost_data["input_cost_per_token"]
|
291
|
+
completion_cost = additional_metadata["tokens"]["completion"]*model_cost_data["output_cost_per_token"]
|
292
|
+
additional_metadata.setdefault('cost', {})["total_cost"] = prompt_cost + completion_cost
|
293
|
+
else:
|
294
|
+
logger.warning("Token information missing in additional_metadata")
|
295
295
|
except Exception as e:
|
296
296
|
logger.warning(f"Error adding cost: {e}")
|
297
|
-
|
298
|
-
|
299
|
-
|
297
|
+
else:
|
298
|
+
logger.debug("Model name not available in additional_metadata, skipping cost calculation")
|
299
|
+
|
300
300
|
additional_metadata["total_tokens"] = additional_metadata["tokens"]["total"]
|
301
301
|
additional_metadata["total_cost"] = additional_metadata["cost"]["total_cost"]
|
302
302
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.2
|
2
2
|
Name: ragaai_catalyst
|
3
|
-
Version: 2.1.
|
3
|
+
Version: 2.1.5b5
|
4
4
|
Summary: RAGA AI CATALYST
|
5
5
|
Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>
|
6
6
|
Requires-Python: <3.13,>=3.9
|
@@ -13,8 +13,9 @@ ragaai_catalyst/synthetic_data_generation.py,sha256=uDV9tNwto2xSkWg5XHXUvjErW-4P
|
|
13
13
|
ragaai_catalyst/utils.py,sha256=TlhEFwLyRU690HvANbyoRycR3nQ67lxVUQoUOfTPYQ0,3772
|
14
14
|
ragaai_catalyst/tracers/__init__.py,sha256=LfgTes-nHpazssbGKnn8kyLZNr49kIPrlkrqqoTFTfc,301
|
15
15
|
ragaai_catalyst/tracers/distributed.py,sha256=AIRvS5Ur4jbFDXsUkYuCTmtGoHHx3LOG4n5tWOh610U,10330
|
16
|
+
ragaai_catalyst/tracers/langchain_callback.py,sha256=LvMBhgvAX8ftyBQ9Naeui46EoDa2nHQZq48Ra6nL-Qg,21991
|
16
17
|
ragaai_catalyst/tracers/llamaindex_callback.py,sha256=ZY0BJrrlz-P9Mg2dX-ZkVKG3gSvzwqBtk7JL_05MiYA,14028
|
17
|
-
ragaai_catalyst/tracers/tracer.py,sha256=
|
18
|
+
ragaai_catalyst/tracers/tracer.py,sha256=spAiNQ85vpMntckHIgn0w7h33gxqWg5WoxOEcvT-e7U,19179
|
18
19
|
ragaai_catalyst/tracers/upload_traces.py,sha256=2TWdRTN6FMaX-dqDv8BJWQS0xrCGYKkXEYOi2kK3Z3Y,5487
|
19
20
|
ragaai_catalyst/tracers/agentic_tracing/README.md,sha256=X4QwLb7-Jg7GQMIXj-SerZIgDETfw-7VgYlczOR8ZeQ,4508
|
20
21
|
ragaai_catalyst/tracers/agentic_tracing/__init__.py,sha256=yf6SKvOPSpH-9LiKaoLKXwqj5sez8F_5wkOb91yp0oE,260
|
@@ -47,7 +48,7 @@ ragaai_catalyst/tracers/agentic_tracing/utils/file_name_tracker.py,sha256=515NND
|
|
47
48
|
ragaai_catalyst/tracers/agentic_tracing/utils/generic.py,sha256=WwXT01xmp8MSr7KinuDCSK9a1ifpLcT7ajFkvYviG_A,1190
|
48
49
|
ragaai_catalyst/tracers/agentic_tracing/utils/get_user_trace_metrics.py,sha256=vPZ4dn4EHFW0kqd1GyRpsYXbfrRrd0DXCmh-pzsDBNE,1109
|
49
50
|
ragaai_catalyst/tracers/agentic_tracing/utils/llm_utils.py,sha256=wlXCuaRe81s-7FWdJ_MquXFGRZZfNrZxLIIxl-Ohbqk,15541
|
50
|
-
ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json,sha256=
|
51
|
+
ragaai_catalyst/tracers/agentic_tracing/utils/model_costs.json,sha256=E_uKa1SSrigaorCiAShZr4inKNMc54jcEy4B_7pT4DA,295002
|
51
52
|
ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py,sha256=MqeRNGxzeuh9qTK0NbYMftl9V9Z0V7gMgBoHkrXP56k,1592
|
52
53
|
ragaai_catalyst/tracers/agentic_tracing/utils/system_monitor.py,sha256=H8WNsk4v_5T6OUw4TFOzlDLjQhJwjh1nAMyMAoqMEi4,6946
|
53
54
|
ragaai_catalyst/tracers/agentic_tracing/utils/trace_utils.py,sha256=RciiDdo2riibEoM8X0FKHaXi78y3bWwNkV8U0leqigk,3508
|
@@ -64,8 +65,8 @@ ragaai_catalyst/tracers/utils/__init__.py,sha256=KeMaZtYaTojilpLv65qH08QmpYclfpa
|
|
64
65
|
ragaai_catalyst/tracers/utils/convert_langchain_callbacks_output.py,sha256=ofrNrxf2b1hpjDh_zeaxiYq86azn1MF3kW8-ViYPEg0,1641
|
65
66
|
ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py,sha256=cghjCuUe8w-2MZdh9xgtRGe3y219u26GGzpnuY4Wt6Q,3047
|
66
67
|
ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
|
67
|
-
ragaai_catalyst-2.1.
|
68
|
-
ragaai_catalyst-2.1.
|
69
|
-
ragaai_catalyst-2.1.
|
70
|
-
ragaai_catalyst-2.1.
|
71
|
-
ragaai_catalyst-2.1.
|
68
|
+
ragaai_catalyst-2.1.5b5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
69
|
+
ragaai_catalyst-2.1.5b5.dist-info/METADATA,sha256=ShN_vaYhKMLho8Q49eBzlWYUNEQD3U6w9qgRtiMYsiM,12764
|
70
|
+
ragaai_catalyst-2.1.5b5.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
71
|
+
ragaai_catalyst-2.1.5b5.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
|
72
|
+
ragaai_catalyst-2.1.5b5.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|