ragaai-catalyst 2.1.5b35__py3-none-any.whl → 2.1.5b36__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ragaai_catalyst/tracers/agentic_tracing/tracers/base.py +20 -20
- ragaai_catalyst/tracers/agentic_tracing/upload/trace_uploader.py +219 -548
- {ragaai_catalyst-2.1.5b35.dist-info → ragaai_catalyst-2.1.5b36.dist-info}/METADATA +1 -1
- {ragaai_catalyst-2.1.5b35.dist-info → ragaai_catalyst-2.1.5b36.dist-info}/RECORD +7 -7
- {ragaai_catalyst-2.1.5b35.dist-info → ragaai_catalyst-2.1.5b36.dist-info}/LICENSE +0 -0
- {ragaai_catalyst-2.1.5b35.dist-info → ragaai_catalyst-2.1.5b36.dist-info}/WHEEL +0 -0
- {ragaai_catalyst-2.1.5b35.dist-info → ragaai_catalyst-2.1.5b36.dist-info}/top_level.txt +0 -0
@@ -305,7 +305,7 @@ class BaseTracer:
|
|
305
305
|
|
306
306
|
logger.debug("Base URL used for uploading: {}".format(self.base_url))
|
307
307
|
|
308
|
-
# Submit to background process for uploading
|
308
|
+
# Submit to background process for uploading using futures
|
309
309
|
self.upload_task_id = submit_upload_task(
|
310
310
|
filepath=filepath,
|
311
311
|
hash_id=hash_id,
|
@@ -317,28 +317,28 @@ class BaseTracer:
|
|
317
317
|
base_url=self.base_url
|
318
318
|
)
|
319
319
|
|
320
|
-
#
|
321
|
-
|
320
|
+
# For backward compatibility
|
321
|
+
self._is_uploading = True
|
322
322
|
|
323
|
-
#
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
323
|
+
# Start checking for completion if a callback is registered
|
324
|
+
if self._upload_completed_callback:
|
325
|
+
# Start a thread to check status and call callback when complete
|
326
|
+
def check_status_and_callback():
|
327
|
+
status = self.get_upload_status()
|
328
|
+
if status.get("status") in ["completed", "failed"]:
|
329
|
+
self._is_uploading = False
|
330
|
+
# Execute callback
|
331
|
+
try:
|
332
|
+
self._upload_completed_callback(self)
|
333
|
+
except Exception as e:
|
334
|
+
logger.error(f"Error in upload completion callback: {e}")
|
335
|
+
return
|
336
336
|
|
337
|
-
|
338
|
-
|
337
|
+
# Check again after a delay
|
338
|
+
threading.Timer(5.0, check_status_and_callback).start()
|
339
339
|
|
340
|
-
|
341
|
-
|
340
|
+
# Start checking
|
341
|
+
threading.Timer(5.0, check_status_and_callback).start()
|
342
342
|
|
343
343
|
logger.info(f"Submitted upload task with ID: {self.upload_task_id}")
|
344
344
|
|
@@ -17,6 +17,8 @@ from datetime import datetime
|
|
17
17
|
import atexit
|
18
18
|
import glob
|
19
19
|
from logging.handlers import RotatingFileHandler
|
20
|
+
import concurrent.futures
|
21
|
+
from typing import Dict, Any, Optional
|
20
22
|
|
21
23
|
# Set up logging
|
22
24
|
log_dir = os.path.join(tempfile.gettempdir(), "ragaai_logs")
|
@@ -55,320 +57,175 @@ except ImportError:
|
|
55
57
|
QUEUE_DIR = os.path.join(tempfile.gettempdir(), "ragaai_tasks")
|
56
58
|
os.makedirs(QUEUE_DIR, exist_ok=True)
|
57
59
|
|
58
|
-
# Clean up any stale processes
|
59
|
-
def cleanup_stale_processes():
|
60
|
-
"""Check for stale processes but allow active uploads to complete"""
|
61
|
-
pid_file = os.path.join(tempfile.gettempdir(), "trace_uploader.pid")
|
62
|
-
if os.path.exists(pid_file):
|
63
|
-
try:
|
64
|
-
with open(pid_file, "r") as f:
|
65
|
-
old_pid = int(f.read().strip())
|
66
|
-
try:
|
67
|
-
import psutil
|
68
|
-
if psutil.pid_exists(old_pid):
|
69
|
-
p = psutil.Process(old_pid)
|
70
|
-
if "trace_uploader.py" in " ".join(p.cmdline()):
|
71
|
-
# Instead of terminating, just remove the PID file
|
72
|
-
# This allows the process to finish its current uploads
|
73
|
-
logger.info(f"Removing PID file for process {old_pid}")
|
74
|
-
os.remove(pid_file)
|
75
|
-
return
|
76
|
-
except Exception as e:
|
77
|
-
logger.warning(f"Error checking stale process: {e}")
|
78
|
-
os.remove(pid_file)
|
79
|
-
except Exception as e:
|
80
|
-
logger.warning(f"Error reading PID file: {e}")
|
81
|
-
|
82
|
-
cleanup_stale_processes()
|
83
|
-
|
84
60
|
# Status codes
|
85
61
|
STATUS_PENDING = "pending"
|
86
62
|
STATUS_PROCESSING = "processing"
|
87
63
|
STATUS_COMPLETED = "completed"
|
88
64
|
STATUS_FAILED = "failed"
|
89
65
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
self.task_id = task_id or f"task_{int(time.time())}_{os.getpid()}_{hash(str(time.time()))}"
|
95
|
-
self.status = STATUS_PENDING
|
96
|
-
self.attempts = 0
|
97
|
-
self.max_attempts = 3
|
98
|
-
self.created_at = datetime.now().isoformat()
|
99
|
-
self.updated_at = self.created_at
|
100
|
-
self.error = None
|
101
|
-
|
102
|
-
# Task details
|
103
|
-
self.filepath = kwargs.get("filepath")
|
104
|
-
self.hash_id = kwargs.get("hash_id")
|
105
|
-
self.zip_path = kwargs.get("zip_path")
|
106
|
-
self.project_name = kwargs.get("project_name")
|
107
|
-
self.project_id = kwargs.get("project_id")
|
108
|
-
self.dataset_name = kwargs.get("dataset_name")
|
109
|
-
self.user_details = kwargs.get("user_details", {})
|
110
|
-
self.base_url = kwargs.get("base_url")
|
111
|
-
|
112
|
-
def to_dict(self):
|
113
|
-
"""Convert task to dictionary for serialization"""
|
114
|
-
return {
|
115
|
-
"task_id": self.task_id,
|
116
|
-
"status": self.status,
|
117
|
-
"attempts": self.attempts,
|
118
|
-
"max_attempts": self.max_attempts,
|
119
|
-
"created_at": self.created_at,
|
120
|
-
"updated_at": self.updated_at,
|
121
|
-
"error": self.error,
|
122
|
-
"filepath": self.filepath,
|
123
|
-
"hash_id": self.hash_id,
|
124
|
-
"zip_path": self.zip_path,
|
125
|
-
"project_name": self.project_name,
|
126
|
-
"project_id": self.project_id,
|
127
|
-
"dataset_name": self.dataset_name,
|
128
|
-
"user_details": self.user_details,
|
129
|
-
"base_url": self.base_url
|
130
|
-
}
|
131
|
-
|
132
|
-
@classmethod
|
133
|
-
def from_dict(cls, data):
|
134
|
-
"""Create task from dictionary"""
|
135
|
-
task = cls(task_id=data.get("task_id"))
|
136
|
-
task.status = data.get("status", STATUS_PENDING)
|
137
|
-
task.attempts = data.get("attempts", 0)
|
138
|
-
task.max_attempts = data.get("max_attempts", 3)
|
139
|
-
task.created_at = data.get("created_at")
|
140
|
-
task.updated_at = data.get("updated_at")
|
141
|
-
task.error = data.get("error")
|
142
|
-
task.filepath = data.get("filepath")
|
143
|
-
task.hash_id = data.get("hash_id")
|
144
|
-
task.zip_path = data.get("zip_path")
|
145
|
-
task.project_name = data.get("project_name")
|
146
|
-
task.project_id = data.get("project_id")
|
147
|
-
task.dataset_name = data.get("dataset_name")
|
148
|
-
task.user_details = data.get("user_details", {})
|
149
|
-
task.base_url = data.get("base_url")
|
150
|
-
return task
|
151
|
-
|
152
|
-
def update_status(self, status, error=None):
|
153
|
-
"""Update task status"""
|
154
|
-
self.status = status
|
155
|
-
self.updated_at = datetime.now().isoformat()
|
156
|
-
if error:
|
157
|
-
self.error = str(error)
|
158
|
-
self.save()
|
159
|
-
|
160
|
-
def increment_attempts(self):
|
161
|
-
"""Increment the attempt counter"""
|
162
|
-
self.attempts += 1
|
163
|
-
self.updated_at = datetime.now().isoformat()
|
164
|
-
self.save()
|
165
|
-
|
166
|
-
def save(self):
|
167
|
-
"""Save task to disk"""
|
168
|
-
task_path = os.path.join(QUEUE_DIR, f"{self.task_id}.json")
|
169
|
-
with open(task_path, "w") as f:
|
170
|
-
json.dump(self.to_dict(), f, indent=2)
|
171
|
-
|
172
|
-
def delete(self):
|
173
|
-
"""Delete task file from disk"""
|
174
|
-
task_path = os.path.join(QUEUE_DIR, f"{self.task_id}.json")
|
175
|
-
if os.path.exists(task_path):
|
176
|
-
os.remove(task_path)
|
177
|
-
|
178
|
-
@staticmethod
|
179
|
-
def list_pending_tasks():
|
180
|
-
"""List all pending tasks"""
|
181
|
-
tasks = []
|
182
|
-
#logger.info("Listing pending tasks from queue directory: {}".format(QUEUE_DIR))
|
183
|
-
for filename in os.listdir(QUEUE_DIR):
|
184
|
-
if filename.endswith(".json"):
|
185
|
-
try:
|
186
|
-
with open(os.path.join(QUEUE_DIR, filename), "r") as f:
|
187
|
-
task_data = json.load(f)
|
188
|
-
task = UploadTask.from_dict(task_data)
|
189
|
-
if task.status in [STATUS_PENDING, STATUS_FAILED] and task.attempts < task.max_attempts:
|
190
|
-
# Verify files still exist
|
191
|
-
if (not task.filepath or os.path.exists(task.filepath)) and \
|
192
|
-
(not task.zip_path or os.path.exists(task.zip_path)):
|
193
|
-
tasks.append(task)
|
194
|
-
else:
|
195
|
-
# Files missing, mark as failed
|
196
|
-
task.update_status(STATUS_FAILED, "Required files missing")
|
197
|
-
except Exception as e:
|
198
|
-
logger.error(f"Error loading task {filename}: {e}")
|
199
|
-
return tasks
|
66
|
+
# Global executor for handling uploads
|
67
|
+
_executor = None
|
68
|
+
# Dictionary to track futures and their associated task IDs
|
69
|
+
_futures: Dict[str, Any] = {}
|
200
70
|
|
71
|
+
def get_executor():
|
72
|
+
"""Get or create the thread pool executor"""
|
73
|
+
global _executor
|
74
|
+
if _executor is None:
|
75
|
+
_executor = concurrent.futures.ThreadPoolExecutor(max_workers=8, thread_name_prefix="trace_uploader")
|
76
|
+
return _executor
|
201
77
|
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
Handles the actual upload work in a separate process
|
78
|
+
def process_upload(task_id: str, filepath: str, hash_id: str, zip_path: str,
|
79
|
+
project_name: str, project_id: str, dataset_name: str,
|
80
|
+
user_details: Dict[str, Any], base_url: str) -> Dict[str, Any]:
|
206
81
|
"""
|
82
|
+
Process a single upload task
|
207
83
|
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
signal.signal(signal.SIGINT, self.handle_signal)
|
84
|
+
Args:
|
85
|
+
task_id: Unique identifier for the task
|
86
|
+
filepath: Path to the trace file
|
87
|
+
hash_id: Hash ID for the code
|
88
|
+
zip_path: Path to the code zip file
|
89
|
+
project_name: Project name
|
90
|
+
project_id: Project ID
|
91
|
+
dataset_name: Dataset name
|
92
|
+
user_details: User details dictionary
|
93
|
+
base_url: Base URL for API calls
|
219
94
|
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
# No tasks, sleep before checking again
|
236
|
-
time.sleep(5)
|
237
|
-
except Exception as e:
|
238
|
-
logger.error(f"Error in uploader loop: {e}")
|
239
|
-
time.sleep(5)
|
240
|
-
|
241
|
-
logger.info("Trace uploader stopped")
|
242
|
-
|
243
|
-
def process_task(self, task):
|
244
|
-
"""Process a single upload task"""
|
245
|
-
logger.info(f"Starting to process task {task.task_id}")
|
246
|
-
logger.debug(f"Task details: {task.to_dict()}")
|
247
|
-
|
95
|
+
Returns:
|
96
|
+
Dict containing status and any error information
|
97
|
+
"""
|
98
|
+
logger.info(f"Processing upload task {task_id}")
|
99
|
+
result = {
|
100
|
+
"task_id": task_id,
|
101
|
+
"status": STATUS_PROCESSING,
|
102
|
+
"error": None,
|
103
|
+
"start_time": datetime.now().isoformat()
|
104
|
+
}
|
105
|
+
|
106
|
+
# Save initial status to file
|
107
|
+
save_task_status(result)
|
108
|
+
|
109
|
+
try:
|
248
110
|
# Check if file exists
|
249
|
-
if not os.path.exists(
|
250
|
-
error_msg = f"Task filepath does not exist: {
|
111
|
+
if not os.path.exists(filepath):
|
112
|
+
error_msg = f"Task filepath does not exist: {filepath}"
|
251
113
|
logger.error(error_msg)
|
252
|
-
|
253
|
-
|
114
|
+
result["status"] = STATUS_FAILED
|
115
|
+
result["error"] = error_msg
|
116
|
+
save_task_status(result)
|
117
|
+
return result
|
254
118
|
|
255
119
|
if not IMPORTS_AVAILABLE:
|
256
|
-
logger.warning(f"Test mode: Simulating processing of task {
|
257
|
-
|
258
|
-
|
259
|
-
|
120
|
+
logger.warning(f"Test mode: Simulating processing of task {task_id}")
|
121
|
+
time.sleep(2) # Simulate work
|
122
|
+
result["status"] = STATUS_COMPLETED
|
123
|
+
save_task_status(result)
|
124
|
+
return result
|
260
125
|
|
261
|
-
|
262
|
-
|
263
|
-
task.update_status(STATUS_PROCESSING)
|
264
|
-
task.increment_attempts()
|
265
|
-
|
266
|
-
# Log memory state for debugging
|
267
|
-
try:
|
268
|
-
import psutil
|
269
|
-
process = psutil.Process()
|
270
|
-
logger.debug(f"Memory usage before processing: {process.memory_info().rss / 1024 / 1024:.2f} MB")
|
271
|
-
except ImportError:
|
272
|
-
pass
|
273
|
-
|
126
|
+
# Step 1: Create dataset schema
|
127
|
+
logger.info(f"Creating dataset schema for {dataset_name} with base_url: {base_url}")
|
274
128
|
try:
|
275
|
-
# Step 1: Create dataset schema
|
276
|
-
logger.info(f"Creating dataset schema for {task.dataset_name} with base_url: {task.base_url}")
|
277
129
|
response = create_dataset_schema_with_trace(
|
278
|
-
dataset_name=
|
279
|
-
project_name=
|
280
|
-
base_url=
|
130
|
+
dataset_name=dataset_name,
|
131
|
+
project_name=project_name,
|
132
|
+
base_url=base_url
|
281
133
|
)
|
282
134
|
logger.info(f"Dataset schema created: {response}")
|
283
|
-
|
284
|
-
# Step 2: Upload trace metrics
|
285
|
-
if task.filepath and os.path.exists(task.filepath):
|
286
|
-
logger.info(f"Uploading trace metrics for {task.filepath}")
|
287
|
-
try:
|
288
|
-
response = upload_trace_metric(
|
289
|
-
json_file_path=task.filepath,
|
290
|
-
dataset_name=task.dataset_name,
|
291
|
-
project_name=task.project_name,
|
292
|
-
base_url=task.base_url
|
293
|
-
)
|
294
|
-
logger.info(f"Trace metrics uploaded: {response}")
|
295
|
-
except Exception as e:
|
296
|
-
logger.error(f"Error uploading trace metrics: {e}")
|
297
|
-
# Continue with other uploads
|
298
|
-
else:
|
299
|
-
logger.warning(f"Trace file {task.filepath} not found, skipping metrics upload")
|
300
|
-
|
301
|
-
# Step 3: Upload agentic traces
|
302
|
-
if task.filepath and os.path.exists(task.filepath):
|
303
|
-
logger.info(f"Uploading agentic traces for {task.filepath}")
|
304
|
-
try:
|
305
|
-
upload_traces = UploadAgenticTraces(
|
306
|
-
json_file_path=task.filepath,
|
307
|
-
project_name=task.project_name,
|
308
|
-
project_id=task.project_id,
|
309
|
-
dataset_name=task.dataset_name,
|
310
|
-
user_detail=task.user_details,
|
311
|
-
base_url=task.base_url,
|
312
|
-
)
|
313
|
-
upload_traces.upload_agentic_traces()
|
314
|
-
logger.info("Agentic traces uploaded successfully")
|
315
|
-
except Exception as e:
|
316
|
-
logger.error(f"Error uploading agentic traces: {e}")
|
317
|
-
# Continue with code upload
|
318
|
-
else:
|
319
|
-
logger.warning(f"Trace file {task.filepath} not found, skipping traces upload")
|
320
|
-
|
321
|
-
# Step 4: Upload code hash
|
322
|
-
if task.hash_id and task.zip_path and os.path.exists(task.zip_path):
|
323
|
-
logger.info(f"Uploading code hash {task.hash_id}")
|
324
|
-
try:
|
325
|
-
response = upload_code(
|
326
|
-
hash_id=task.hash_id,
|
327
|
-
zip_path=task.zip_path,
|
328
|
-
project_name=task.project_name,
|
329
|
-
dataset_name=task.dataset_name,
|
330
|
-
base_url=task.base_url
|
331
|
-
)
|
332
|
-
logger.info(f"Code hash uploaded: {response}")
|
333
|
-
except Exception as e:
|
334
|
-
logger.error(f"Error uploading code hash: {e}")
|
335
|
-
else:
|
336
|
-
logger.warning(f"Code zip {task.zip_path} not found, skipping code upload")
|
337
|
-
|
338
|
-
# Mark task as completed
|
339
|
-
task.update_status(STATUS_COMPLETED)
|
340
|
-
logger.info(f"Task {task.task_id} completed successfully")
|
341
|
-
|
342
|
-
# Clean up task file
|
343
|
-
task.delete()
|
344
|
-
|
345
135
|
except Exception as e:
|
346
|
-
logger.error(f"Error
|
347
|
-
|
348
|
-
task.update_status(STATUS_FAILED, str(e))
|
349
|
-
logger.error(f"Task {task.task_id} failed after {task.attempts} attempts")
|
350
|
-
else:
|
351
|
-
task.update_status(STATUS_PENDING, str(e))
|
352
|
-
logger.warning(f"Task {task.task_id} will be retried (attempt {task.attempts}/{task.max_attempts})")
|
353
|
-
finally:
|
354
|
-
self.processing = False
|
136
|
+
logger.error(f"Error creating dataset schema: {e}")
|
137
|
+
# Continue with other steps
|
355
138
|
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
139
|
+
# Step 2: Upload trace metrics
|
140
|
+
if filepath and os.path.exists(filepath):
|
141
|
+
logger.info(f"Uploading trace metrics for {filepath}")
|
142
|
+
try:
|
143
|
+
response = upload_trace_metric(
|
144
|
+
json_file_path=filepath,
|
145
|
+
dataset_name=dataset_name,
|
146
|
+
project_name=project_name,
|
147
|
+
base_url=base_url
|
148
|
+
)
|
149
|
+
logger.info(f"Trace metrics uploaded: {response}")
|
150
|
+
except Exception as e:
|
151
|
+
logger.error(f"Error uploading trace metrics: {e}")
|
152
|
+
# Continue with other uploads
|
153
|
+
else:
|
154
|
+
logger.warning(f"Trace file {filepath} not found, skipping metrics upload")
|
360
155
|
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
156
|
+
# Step 3: Upload agentic traces
|
157
|
+
if filepath and os.path.exists(filepath):
|
158
|
+
logger.info(f"Uploading agentic traces for {filepath}")
|
159
|
+
try:
|
160
|
+
upload_traces = UploadAgenticTraces(
|
161
|
+
json_file_path=filepath,
|
162
|
+
project_name=project_name,
|
163
|
+
project_id=project_id,
|
164
|
+
dataset_name=dataset_name,
|
165
|
+
user_detail=user_details,
|
166
|
+
base_url=base_url,
|
167
|
+
)
|
168
|
+
upload_traces.upload_agentic_traces()
|
169
|
+
logger.info("Agentic traces uploaded successfully")
|
170
|
+
except Exception as e:
|
171
|
+
logger.error(f"Error uploading agentic traces: {e}")
|
172
|
+
# Continue with code upload
|
173
|
+
else:
|
174
|
+
logger.warning(f"Trace file {filepath} not found, skipping traces upload")
|
175
|
+
|
176
|
+
# Step 4: Upload code hash
|
177
|
+
if hash_id and zip_path and os.path.exists(zip_path):
|
178
|
+
logger.info(f"Uploading code hash {hash_id}")
|
179
|
+
try:
|
180
|
+
response = upload_code(
|
181
|
+
hash_id=hash_id,
|
182
|
+
zip_path=zip_path,
|
183
|
+
project_name=project_name,
|
184
|
+
dataset_name=dataset_name,
|
185
|
+
base_url=base_url
|
186
|
+
)
|
187
|
+
logger.info(f"Code hash uploaded: {response}")
|
188
|
+
except Exception as e:
|
189
|
+
logger.error(f"Error uploading code hash: {e}")
|
190
|
+
else:
|
191
|
+
logger.warning(f"Code zip {zip_path} not found, skipping code upload")
|
365
192
|
|
193
|
+
# Mark task as completed
|
194
|
+
result["status"] = STATUS_COMPLETED
|
195
|
+
result["end_time"] = datetime.now().isoformat()
|
196
|
+
logger.info(f"Task {task_id} completed successfully")
|
197
|
+
|
198
|
+
except Exception as e:
|
199
|
+
logger.error(f"Error processing task {task_id}: {e}")
|
200
|
+
result["status"] = STATUS_FAILED
|
201
|
+
result["error"] = str(e)
|
202
|
+
result["end_time"] = datetime.now().isoformat()
|
203
|
+
|
204
|
+
# Save final status
|
205
|
+
save_task_status(result)
|
206
|
+
return result
|
207
|
+
|
208
|
+
def save_task_status(task_status: Dict[str, Any]):
|
209
|
+
"""Save task status to a file"""
|
210
|
+
task_id = task_status["task_id"]
|
211
|
+
status_path = os.path.join(QUEUE_DIR, f"{task_id}_status.json")
|
212
|
+
with open(status_path, "w") as f:
|
213
|
+
json.dump(task_status, f, indent=2)
|
366
214
|
|
367
215
|
def submit_upload_task(filepath, hash_id, zip_path, project_name, project_id, dataset_name, user_details, base_url):
|
368
216
|
"""
|
369
|
-
Submit a new upload task
|
370
|
-
This function can be called from the main application.
|
217
|
+
Submit a new upload task using futures.
|
371
218
|
|
219
|
+
Args:
|
220
|
+
filepath: Path to the trace file
|
221
|
+
hash_id: Hash ID for the code
|
222
|
+
zip_path: Path to the code zip file
|
223
|
+
project_name: Project name
|
224
|
+
project_id: Project ID
|
225
|
+
dataset_name: Dataset name
|
226
|
+
user_details: User details dictionary
|
227
|
+
base_url: Base URL for API calls
|
228
|
+
|
372
229
|
Returns:
|
373
230
|
str: Task ID
|
374
231
|
"""
|
@@ -380,11 +237,18 @@ def submit_upload_task(filepath, hash_id, zip_path, project_name, project_id, da
|
|
380
237
|
logger.error(f"Trace file not found: {filepath}")
|
381
238
|
return None
|
382
239
|
|
383
|
-
# Create
|
240
|
+
# Create absolute path to the trace file
|
384
241
|
filepath = os.path.abspath(filepath)
|
385
242
|
logger.debug(f"Using absolute filepath: {filepath}")
|
386
243
|
|
387
|
-
task
|
244
|
+
# Generate a unique task ID
|
245
|
+
task_id = f"task_{int(time.time())}_{os.getpid()}_{hash(str(time.time()))}"
|
246
|
+
|
247
|
+
# Submit the task to the executor
|
248
|
+
executor = get_executor()
|
249
|
+
future = executor.submit(
|
250
|
+
process_upload,
|
251
|
+
task_id=task_id,
|
388
252
|
filepath=filepath,
|
389
253
|
hash_id=hash_id,
|
390
254
|
zip_path=zip_path,
|
@@ -395,290 +259,97 @@ def submit_upload_task(filepath, hash_id, zip_path, project_name, project_id, da
|
|
395
259
|
base_url=base_url
|
396
260
|
)
|
397
261
|
|
398
|
-
#
|
399
|
-
|
400
|
-
logger.debug(f"Saving task to: {task_path}")
|
262
|
+
# Store the future for later status checks
|
263
|
+
_futures[task_id] = future
|
401
264
|
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
except Exception as e:
|
411
|
-
logger.error(f"Error creating task file: {e}", exc_info=True)
|
412
|
-
return None
|
413
|
-
|
414
|
-
# Ensure uploader process is running
|
415
|
-
logger.info("Starting uploader process...")
|
416
|
-
pid = ensure_uploader_running()
|
417
|
-
if pid:
|
418
|
-
logger.info(f"Uploader process running with PID {pid}")
|
419
|
-
else:
|
420
|
-
logger.warning("Failed to start uploader process, but task was queued")
|
265
|
+
# Create initial status
|
266
|
+
initial_status = {
|
267
|
+
"task_id": task_id,
|
268
|
+
"status": STATUS_PENDING,
|
269
|
+
"error": None,
|
270
|
+
"start_time": datetime.now().isoformat()
|
271
|
+
}
|
272
|
+
save_task_status(initial_status)
|
421
273
|
|
422
|
-
return
|
423
|
-
|
274
|
+
return task_id
|
424
275
|
|
425
276
|
def get_task_status(task_id):
|
426
277
|
"""
|
427
278
|
Get the status of a task by ID.
|
428
|
-
This function can be called from the main application.
|
429
279
|
|
280
|
+
Args:
|
281
|
+
task_id: Task ID to check
|
282
|
+
|
430
283
|
Returns:
|
431
284
|
dict: Task status information
|
432
285
|
"""
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
286
|
+
logger.debug(f"Getting status for task {task_id}")
|
287
|
+
|
288
|
+
# Check if we have a future for this task
|
289
|
+
future = _futures.get(task_id)
|
290
|
+
|
291
|
+
# If we have a future, check its status
|
292
|
+
if future:
|
293
|
+
if future.done():
|
294
|
+
try:
|
295
|
+
# Get the result (this will re-raise any exception that occurred)
|
296
|
+
result = future.result(timeout=0)
|
297
|
+
return result
|
298
|
+
except concurrent.futures.TimeoutError:
|
299
|
+
return {"status": STATUS_PROCESSING, "error": None}
|
300
|
+
except Exception as e:
|
301
|
+
logger.error(f"Error retrieving future result for task {task_id}: {e}")
|
302
|
+
return {"status": STATUS_FAILED, "error": str(e)}
|
303
|
+
else:
|
304
|
+
return {"status": STATUS_PROCESSING, "error": None}
|
305
|
+
|
306
|
+
# If we don't have a future, try to read from the status file
|
307
|
+
status_path = os.path.join(QUEUE_DIR, f"{task_id}_status.json")
|
308
|
+
if os.path.exists(status_path):
|
309
|
+
try:
|
310
|
+
with open(status_path, "r") as f:
|
439
311
|
return json.load(f)
|
440
|
-
|
312
|
+
except Exception as e:
|
313
|
+
logger.error(f"Error reading status file for task {task_id}: {e}")
|
314
|
+
return {"status": "unknown", "error": f"Error reading status: {e}"}
|
441
315
|
|
442
|
-
|
443
|
-
return json.load(f)
|
316
|
+
return {"status": "unknown", "error": "Task not found"}
|
444
317
|
|
318
|
+
def shutdown():
|
319
|
+
"""Shutdown the executor"""
|
320
|
+
global _executor
|
321
|
+
if _executor:
|
322
|
+
logger.info("Shutting down executor")
|
323
|
+
_executor.shutdown(wait=False)
|
324
|
+
_executor = None
|
445
325
|
|
326
|
+
# Register shutdown handler
|
327
|
+
atexit.register(shutdown)
|
328
|
+
|
329
|
+
# For backward compatibility
|
446
330
|
def ensure_uploader_running():
|
447
331
|
"""
|
448
|
-
Ensure the uploader
|
449
|
-
|
332
|
+
Ensure the uploader is running.
|
333
|
+
This is a no-op in the futures implementation, but kept for API compatibility.
|
450
334
|
"""
|
451
|
-
|
452
|
-
|
453
|
-
# Check if we can find a running process
|
454
|
-
pid_file = os.path.join(tempfile.gettempdir(), "trace_uploader.pid")
|
455
|
-
logger.debug(f"PID file location: {pid_file}")
|
456
|
-
|
457
|
-
if os.path.exists(pid_file):
|
458
|
-
try:
|
459
|
-
with open(pid_file, "r") as f:
|
460
|
-
pid_str = f.read().strip()
|
461
|
-
logger.debug(f"Read PID from file: {pid_str}")
|
462
|
-
pid = int(pid_str)
|
463
|
-
|
464
|
-
# Check if process is actually running
|
465
|
-
# Use platform-specific process check
|
466
|
-
is_running = False
|
467
|
-
try:
|
468
|
-
if os.name == 'posix': # Unix/Linux/Mac
|
469
|
-
logger.debug(f"Checking process {pid} on Unix/Mac")
|
470
|
-
os.kill(pid, 0) # This raises an exception if process doesn't exist
|
471
|
-
is_running = True
|
472
|
-
else: # Windows
|
473
|
-
logger.debug(f"Checking process {pid} on Windows")
|
474
|
-
import ctypes
|
475
|
-
kernel32 = ctypes.windll.kernel32
|
476
|
-
SYNCHRONIZE = 0x00100000
|
477
|
-
process = kernel32.OpenProcess(SYNCHRONIZE, False, pid)
|
478
|
-
if process:
|
479
|
-
kernel32.CloseHandle(process)
|
480
|
-
is_running = True
|
481
|
-
except (ImportError, AttributeError) as e:
|
482
|
-
logger.debug(f"Platform-specific check failed: {e}, falling back to cross-platform check")
|
483
|
-
# Fall back to cross-platform check
|
484
|
-
try:
|
485
|
-
import psutil
|
486
|
-
is_running = psutil.pid_exists(pid)
|
487
|
-
logger.debug(f"psutil check result: {is_running}")
|
488
|
-
except ImportError:
|
489
|
-
logger.debug("psutil not available, using basic process check")
|
490
|
-
# If psutil is not available, make a best guess
|
491
|
-
try:
|
492
|
-
os.kill(pid, 0)
|
493
|
-
is_running = True
|
494
|
-
except Exception as e:
|
495
|
-
logger.debug(f"Basic process check failed: {e}")
|
496
|
-
is_running = False
|
497
|
-
|
498
|
-
if is_running:
|
499
|
-
logger.debug(f"Uploader process already running with PID {pid}")
|
500
|
-
return pid
|
501
|
-
except (ProcessLookupError, ValueError, PermissionError):
|
502
|
-
# Process not running or other error, remove stale PID file
|
503
|
-
try:
|
504
|
-
os.remove(pid_file)
|
505
|
-
except:
|
506
|
-
pass
|
507
|
-
|
508
|
-
# Start new process
|
509
|
-
logger.info("Starting new uploader process")
|
510
|
-
|
511
|
-
# Get the path to this script
|
512
|
-
script_path = os.path.abspath(__file__)
|
513
|
-
|
514
|
-
# Start detached process in a platform-specific way
|
515
|
-
try:
|
516
|
-
# First, try the preferred method for each platform
|
517
|
-
if os.name == 'posix': # Unix/Linux/Mac
|
518
|
-
import subprocess
|
519
|
-
|
520
|
-
# Use double fork method on Unix systems
|
521
|
-
try:
|
522
|
-
# First fork
|
523
|
-
pid = os.fork()
|
524
|
-
if pid > 0:
|
525
|
-
# Parent process, return
|
526
|
-
return pid
|
527
|
-
|
528
|
-
# Decouple from parent environment
|
529
|
-
os.chdir('/')
|
530
|
-
os.setsid()
|
531
|
-
os.umask(0)
|
532
|
-
|
533
|
-
# Second fork
|
534
|
-
pid = os.fork()
|
535
|
-
if pid > 0:
|
536
|
-
# Exit from second parent
|
537
|
-
os._exit(0)
|
538
|
-
|
539
|
-
# Redirect standard file descriptors
|
540
|
-
sys.stdout.flush()
|
541
|
-
sys.stderr.flush()
|
542
|
-
si = open(os.devnull, 'r')
|
543
|
-
so = open(os.path.join(tempfile.gettempdir(), 'trace_uploader_stdout.log'), 'a+')
|
544
|
-
se = open(os.path.join(tempfile.gettempdir(), 'trace_uploader_stderr.log'), 'a+')
|
545
|
-
os.dup2(si.fileno(), sys.stdin.fileno())
|
546
|
-
os.dup2(so.fileno(), sys.stdout.fileno())
|
547
|
-
os.dup2(se.fileno(), sys.stderr.fileno())
|
548
|
-
|
549
|
-
# Execute the daemon process
|
550
|
-
os.execl(sys.executable, sys.executable, script_path, '--daemon')
|
551
|
-
|
552
|
-
except (AttributeError, OSError):
|
553
|
-
# Fork not available, try subprocess
|
554
|
-
process = subprocess.Popen(
|
555
|
-
[sys.executable, script_path, "--daemon"],
|
556
|
-
stdout=subprocess.PIPE,
|
557
|
-
stderr=subprocess.PIPE,
|
558
|
-
stdin=subprocess.PIPE,
|
559
|
-
start_new_session=True # Detach from parent
|
560
|
-
)
|
561
|
-
pid = process.pid
|
562
|
-
|
563
|
-
else: # Windows
|
564
|
-
import subprocess
|
565
|
-
# Use the DETACHED_PROCESS flag on Windows
|
566
|
-
startupinfo = subprocess.STARTUPINFO()
|
567
|
-
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
568
|
-
startupinfo.wShowWindow = 0 # SW_HIDE
|
569
|
-
|
570
|
-
# Windows-specific flags
|
571
|
-
DETACHED_PROCESS = 0x00000008
|
572
|
-
CREATE_NO_WINDOW = 0x08000000
|
573
|
-
|
574
|
-
process = subprocess.Popen(
|
575
|
-
[sys.executable, script_path, "--daemon"],
|
576
|
-
stdout=subprocess.PIPE,
|
577
|
-
stderr=subprocess.PIPE,
|
578
|
-
stdin=subprocess.PIPE,
|
579
|
-
startupinfo=startupinfo,
|
580
|
-
creationflags=DETACHED_PROCESS | CREATE_NO_WINDOW
|
581
|
-
)
|
582
|
-
pid = process.pid
|
583
|
-
|
584
|
-
# Write PID to file
|
585
|
-
with open(pid_file, "w") as f:
|
586
|
-
f.write(str(pid))
|
587
|
-
|
588
|
-
logger.info(f"Started uploader process with PID {pid}")
|
589
|
-
return pid
|
590
|
-
|
591
|
-
except Exception as e:
|
592
|
-
logger.error(f"Error starting uploader process using primary method: {e}")
|
593
|
-
|
594
|
-
# Fallback method using multiprocessing (works on most platforms)
|
595
|
-
try:
|
596
|
-
logger.info("Trying fallback method with multiprocessing")
|
597
|
-
import multiprocessing
|
598
|
-
|
599
|
-
def run_uploader():
|
600
|
-
"""Run the uploader in a separate process"""
|
601
|
-
# Redirect output
|
602
|
-
sys.stdout = open(os.path.join(tempfile.gettempdir(), 'trace_uploader_stdout.log'), 'a+')
|
603
|
-
sys.stderr = open(os.path.join(tempfile.gettempdir(), 'trace_uploader_stderr.log'), 'a+')
|
604
|
-
|
605
|
-
# Run daemon
|
606
|
-
run_daemon()
|
607
|
-
|
608
|
-
# Start process
|
609
|
-
process = multiprocessing.Process(target=run_uploader)
|
610
|
-
process.daemon = True # Daemonize it
|
611
|
-
process.start()
|
612
|
-
pid = process.pid
|
613
|
-
|
614
|
-
# Write PID to file
|
615
|
-
with open(pid_file, "w") as f:
|
616
|
-
f.write(str(pid))
|
617
|
-
|
618
|
-
logger.info(f"Started uploader process with fallback method, PID {pid}")
|
619
|
-
return pid
|
620
|
-
|
621
|
-
except Exception as e2:
|
622
|
-
logger.error(f"Error starting uploader process using fallback method: {e2}")
|
623
|
-
|
624
|
-
# Super fallback - run in the current process if all else fails
|
625
|
-
# This is not ideal but better than failing completely
|
626
|
-
try:
|
627
|
-
logger.warning("Using emergency fallback - running in current process thread")
|
628
|
-
import threading
|
629
|
-
|
630
|
-
thread = threading.Thread(target=run_daemon, daemon=True)
|
631
|
-
thread.start()
|
632
|
-
|
633
|
-
# No real PID since it's a thread, but we'll create a marker file
|
634
|
-
with open(pid_file, "w") as f:
|
635
|
-
f.write(f"thread_{id(thread)}")
|
636
|
-
|
637
|
-
return None
|
638
|
-
except Exception as e3:
|
639
|
-
logger.error(f"All methods failed to start uploader: {e3}")
|
640
|
-
return None
|
641
|
-
|
335
|
+
get_executor() # Just ensure the executor is created
|
336
|
+
return True
|
642
337
|
|
338
|
+
# For backward compatibility with the old daemon mode
|
643
339
|
def run_daemon():
|
644
|
-
"""
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
try:
|
651
|
-
uploader = TraceUploader()
|
652
|
-
uploader.start()
|
653
|
-
finally:
|
654
|
-
# Clean up PID file
|
655
|
-
if os.path.exists(pid_file):
|
656
|
-
os.remove(pid_file)
|
657
|
-
|
340
|
+
"""
|
341
|
+
Run the uploader as a daemon process.
|
342
|
+
This is a no-op in the futures implementation, but kept for API compatibility.
|
343
|
+
"""
|
344
|
+
logger.info("Daemon mode not needed in futures implementation")
|
345
|
+
return
|
658
346
|
|
659
347
|
if __name__ == "__main__":
|
660
348
|
parser = argparse.ArgumentParser(description="Trace uploader process")
|
661
349
|
parser.add_argument("--daemon", action="store_true", help="Run as daemon process")
|
662
|
-
parser.add_argument("--test", action="store_true", help="Submit a test task")
|
663
350
|
args = parser.parse_args()
|
664
351
|
|
665
352
|
if args.daemon:
|
666
|
-
|
667
|
-
elif args.test:
|
668
|
-
# Submit a test task
|
669
|
-
test_file = os.path.join(tempfile.gettempdir(), "test_trace.json")
|
670
|
-
with open(test_file, "w") as f:
|
671
|
-
f.write("{}")
|
672
|
-
|
673
|
-
task_id = submit_upload_task(
|
674
|
-
filepath=test_file,
|
675
|
-
hash_id="test_hash",
|
676
|
-
zip_path=test_file,
|
677
|
-
project_name="test_project",
|
678
|
-
project_id="test_id",
|
679
|
-
dataset_name="test_dataset",
|
680
|
-
user_details={"id": "test_user"}
|
681
|
-
)
|
682
|
-
print(f"Submitted test task with ID: {task_id}")
|
353
|
+
logger.info("Daemon mode not needed in futures implementation")
|
683
354
|
else:
|
684
|
-
|
355
|
+
logger.info("Interactive mode not needed in futures implementation")
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.2
|
2
2
|
Name: ragaai_catalyst
|
3
|
-
Version: 2.1.
|
3
|
+
Version: 2.1.5b36
|
4
4
|
Summary: RAGA AI CATALYST
|
5
5
|
Author-email: Kiran Scaria <kiran.scaria@raga.ai>, Kedar Gaikwad <kedar.gaikwad@raga.ai>, Dushyant Mahajan <dushyant.mahajan@raga.ai>, Siddhartha Kosti <siddhartha.kosti@raga.ai>, Ritika Goel <ritika.goel@raga.ai>, Vijay Chaurasia <vijay.chaurasia@raga.ai>, Tushar Kumar <tushar.kumar@raga.ai>
|
6
6
|
Requires-Python: <3.13,>=3.9
|
@@ -45,7 +45,7 @@ ragaai_catalyst/tracers/agentic_tracing/tests/ai_travel_agent.py,sha256=S4rCcKzU
|
|
45
45
|
ragaai_catalyst/tracers/agentic_tracing/tests/unique_decorator_test.py,sha256=Xk1cLzs-2A3dgyBwRRnCWs7Eubki40FVonwd433hPN8,4805
|
46
46
|
ragaai_catalyst/tracers/agentic_tracing/tracers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
47
47
|
ragaai_catalyst/tracers/agentic_tracing/tracers/agent_tracer.py,sha256=LzbsHvELwBmH8ObFomJRhiQ98b6MEi18irm0DPiplt0,29743
|
48
|
-
ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=
|
48
|
+
ragaai_catalyst/tracers/agentic_tracing/tracers/base.py,sha256=Kmy1kgwy19e7MuMMq9GPUq9VXpJV2bXeaIhx8UxX5Sc,54251
|
49
49
|
ragaai_catalyst/tracers/agentic_tracing/tracers/custom_tracer.py,sha256=OBJJjFSvwRjCGNJyqX3yIfC1W05ZN2QUXasCJ4gmCjQ,13930
|
50
50
|
ragaai_catalyst/tracers/agentic_tracing/tracers/langgraph_tracer.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
51
51
|
ragaai_catalyst/tracers/agentic_tracing/tracers/llm_tracer.py,sha256=z-qzmCQCkhyW0aLDUR_rNq4pmxhAaVhNY-kZQsox-Ws,50221
|
@@ -54,7 +54,7 @@ ragaai_catalyst/tracers/agentic_tracing/tracers/network_tracer.py,sha256=m8CxYkl
|
|
54
54
|
ragaai_catalyst/tracers/agentic_tracing/tracers/tool_tracer.py,sha256=xxrliKPfdfbIZRZqMnUewsaTD8_Hv0dbuoBivNZGD4U,21674
|
55
55
|
ragaai_catalyst/tracers/agentic_tracing/tracers/user_interaction_tracer.py,sha256=bhSUhNQCuJXKjgJAXhjKEYjnHMpYN90FSZdR84fNIKU,4614
|
56
56
|
ragaai_catalyst/tracers/agentic_tracing/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
57
|
-
ragaai_catalyst/tracers/agentic_tracing/upload/trace_uploader.py,sha256=
|
57
|
+
ragaai_catalyst/tracers/agentic_tracing/upload/trace_uploader.py,sha256=uRuZlo5XFoBK-a9tYI5ckhSRiXCaUf9fsE2Plu_W1XI,12282
|
58
58
|
ragaai_catalyst/tracers/agentic_tracing/upload/upload_agentic_traces.py,sha256=icycLgfA0734xxoM1rTMG_iIrI3iM94th8RQggJ7sSw,8541
|
59
59
|
ragaai_catalyst/tracers/agentic_tracing/upload/upload_code.py,sha256=aw_eHhUYRbR_9IbIkNjYb7NOsmETD3k1p4a6gxaGI7Q,6462
|
60
60
|
ragaai_catalyst/tracers/agentic_tracing/upload/upload_local_metric.py,sha256=m1O8lKpxKwtHofXLW3fTHX5yfqDW5GxoveARlg5cTw4,2571
|
@@ -87,8 +87,8 @@ ragaai_catalyst/tracers/utils/extraction_logic_llama_index.py,sha256=ZhPs0YhVtB8
|
|
87
87
|
ragaai_catalyst/tracers/utils/langchain_tracer_extraction_logic.py,sha256=XS2_x2qneqEx9oAighLg-LRiueWcESLwIC2r7eJT-Ww,3117
|
88
88
|
ragaai_catalyst/tracers/utils/model_prices_and_context_window_backup.json,sha256=C3uwkibJ08C9sOX-54kulZYmJlIpZ-SQpfE6HNGrjbM,343502
|
89
89
|
ragaai_catalyst/tracers/utils/utils.py,sha256=ViygfJ7vZ7U0CTSA1lbxVloHp4NSlmfDzBRNCJuMhis,2374
|
90
|
-
ragaai_catalyst-2.1.
|
91
|
-
ragaai_catalyst-2.1.
|
92
|
-
ragaai_catalyst-2.1.
|
93
|
-
ragaai_catalyst-2.1.
|
94
|
-
ragaai_catalyst-2.1.
|
90
|
+
ragaai_catalyst-2.1.5b36.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
91
|
+
ragaai_catalyst-2.1.5b36.dist-info/METADATA,sha256=zhMfu6uQAQsus_PGtduzzY8JtrbH_ZEaNStdUJs80VQ,21884
|
92
|
+
ragaai_catalyst-2.1.5b36.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
|
93
|
+
ragaai_catalyst-2.1.5b36.dist-info/top_level.txt,sha256=HpgsdRgEJMk8nqrU6qdCYk3di7MJkDL0B19lkc7dLfM,16
|
94
|
+
ragaai_catalyst-2.1.5b36.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|