ragaai-catalyst 2.2.4b5__py3-none-any.whl → 2.2.5b2__py3-none-any.whl

ragaai_catalyst/tracers/agentic_tracing/upload/upload_local_metric.py

@@ -1,74 +0,0 @@
-import logging
-import os
-import requests
-
-from ragaai_catalyst import RagaAICatalyst
-
-logger = logging.getLogger(__name__)
-logging_level = (
-    logger.setLevel(logging.DEBUG)
-    if os.getenv("DEBUG")
-    else logger.setLevel(logging.INFO)
-)
-
-
-def calculate_metric(project_id, metric_name, model, provider, **kwargs):
-    user_id = "1"
-    org_domain = "raga"
-
-    headers = {
-        "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-        "X-Project-Id": str(project_id),
-        "Content-Type": "application/json"
-    }
-
-    payload = {
-        "data": [
-            {
-                "metric_name": metric_name,
-                "metric_config": {
-                    "threshold": {
-                        "isEditable": True,
-                        "lte": 0.3
-                    },
-                    "model": model,
-                    "orgDomain": org_domain,
-                    "provider": provider,
-                    "user_id": user_id,
-                    "job_id": 1,
-                    "metric_name": metric_name,
-                    "request_id": 1
-                },
-                "variable_mapping": kwargs,
-                "trace_object": {
-                    "Data": {
-                        "DocId": "doc-1",
-                        "Prompt": kwargs.get("prompt"),
-                        "Response": kwargs.get("response"),
-                        "Context": kwargs.get("context"),
-                        "ExpectedResponse": kwargs.get("expected_response"),
-                        "ExpectedContext": kwargs.get("expected_context"),
-                        "Chat": kwargs.get("chat"),
-                        "Instructions": kwargs.get("instructions"),
-                        "SystemPrompt": kwargs.get("system_prompt"),
-                        "Text": kwargs.get("text")
-                    },
-                    "claims": {},
-                    "last_computed_metrics": {
-                        metric_name: {
-                        }
-                    }
-                }
-            }
-        ]
-    }
-
-    try:
-        BASE_URL = RagaAICatalyst.BASE_URL
-        response = requests.post(f"{BASE_URL}/v1/llm/calculate-metric", headers=headers, json=payload, timeout=30)
-        logger.debug(f"Metric calculation response status {response.status_code}")
-        response.raise_for_status()
-        return response.json()
-    except requests.exceptions.RequestException as e:
-        logger.debug(f"Error in calculate-metric api: {e}, payload: {payload}")
-        raise Exception(f"Error in calculate-metric: {e}")

ragaai_catalyst/tracers/agentic_tracing/utils/api_utils.py

@@ -1,21 +0,0 @@
-import requests
-import logging
-
-logger = logging.getLogger(__name__)
-
-def fetch_analysis_trace(base_url, trace_id):
-    """
-    Fetches the analysis trace data from the server.
-
-    :param base_url: The base URL of the server (e.g., "http://localhost:3000").
-    :param trace_id: The ID of the trace to fetch.
-    :return: The JSON response from the server if successful, otherwise None.
-    """
-    try:
-        url = f"{base_url}/api/analysis_traces/{trace_id}"
-        response = requests.get(url)
-        response.raise_for_status()  # Raise an error for bad responses (4xx, 5xx)
-        return response.json()
-    except requests.exceptions.RequestException as e:
-        logger.error(f"Error fetching analysis trace: {e}")
-        return None

ragaai_catalyst/tracers/agentic_tracing/utils/generic.py

@@ -1,32 +0,0 @@
-import os
-import logging
-
-
-def get_db_path():
-    db_filename = "trace_data.db"
-
-    # First, try the package directory
-    package_dir = os.path.dirname(os.path.abspath(__file__))
-    public_dir = os.path.join(package_dir, "..", "ui", "dist")
-    package_db_path = os.path.join(public_dir, db_filename)
-
-    # Ensure the directory exists
-    os.makedirs(os.path.dirname(package_db_path), exist_ok=True)
-
-    if os.path.exists(os.path.dirname(package_db_path)):
-        logging.debug(f"Using package database: {package_db_path}")
-        return f"sqlite:///{package_db_path}"
-
-    # Then, try the local directory
-    local_db_path = os.path.join(os.getcwd(), "agentneo", "ui", "dist", db_filename)
-    if os.path.exists(os.path.dirname(local_db_path)):
-        logging.debug(f"Using local database: {local_db_path}")
-        return f"sqlite:///{local_db_path}"
-
-    # Finally, try the local "/dist" directory
-    local_dist_path = os.path.join(os.getcwd(), "dist", db_filename)
-    if os.path.exists(os.path.dirname(local_dist_path)):
-        logging.debug(f"Using local database: {local_dist_path}")
-        return f"sqlite:///{local_dist_path}"
-
-    return f"sqlite:///{package_db_path}"

ragaai_catalyst/tracers/agentic_tracing/utils/get_user_trace_metrics.py

@@ -1,28 +0,0 @@
-import requests
-import os
-from ....ragaai_catalyst import RagaAICatalyst
-from ....dataset import Dataset
-
-def get_user_trace_metrics(project_name, dataset_name):
-    try:
-        list_datasets = Dataset(project_name=project_name).list_datasets()
-        if not list_datasets:
-            return []
-        elif dataset_name not in list_datasets:
-            return []
-        else:
-            headers = {
-                "Authorization": f"Bearer {os.getenv('RAGAAI_CATALYST_TOKEN')}",
-                "X-Project-Name": project_name,
-            }
-            response = requests.request("GET", 
-                                        f"{RagaAICatalyst.BASE_URL}/v1/llm/trace/metrics?datasetName={dataset_name}", 
-                                        headers=headers, timeout=10)
-            if response.status_code != 200:
-                print(f"Error fetching traces metrics: {response.json()['message']}")
-                return None
-            
-            return response.json()["data"]["columns"]
-    except Exception as e:
-        print(f"Error fetching traces metrics: {e}")
-        return None

ragaai_catalyst/tracers/agentic_tracing/utils/span_attributes.py

@@ -1,133 +0,0 @@
-import os
-from typing import List, Dict, Any, Optional
-import logging
-
-logger = logging.getLogger(__name__)
-logging_level = (
-    logger.setLevel(logging.DEBUG)
-    if os.getenv("DEBUG")
-    else logger.setLevel(logging.INFO)
-)
-
-
-class SpanAttributes:
-    def __init__(self, name, project_id: Optional[int] = None):
-        self.name = name
-        self.tags = []
-        self.metadata = {}
-        self.metrics = []
-        self.local_metrics = []
-        self.feedback = None
-        self.project_id = project_id
-        self.trace_attributes = ["tags", "metadata", "metrics"]
-        self.gt = None
-        self.context = None
-
-    def add_tags(self, tags: str | List[str]):
-        if isinstance(tags, str):
-            tags = [tags]
-        self.tags.extend(tags)
-        logger.debug(f"Added tags: {tags}")
-
-    def add_metadata(self, metadata):
-        self.metadata.update(metadata)
-        logger.debug(f"Added metadata: {metadata}")
-
-    def add_metrics(
-            self,
-            name: str,
-            score: float | int,
-            reasoning: str = "",
-            cost: float = None,
-            latency: float = None,
-            metadata: Dict[str, Any] = {},
-            config: Dict[str, Any] = {},
-    ):
-        self.metrics.append(
-            {
-                "name": name,
-                "score": score,
-                "reason": reasoning,
-                "source": "user",
-                "cost": cost,
-                "latency": latency,
-                "metadata": metadata,
-                "mappings": [],
-                "config": config,
-            }
-        )
-        logger.debug(f"Added metrics: {self.metrics}")
-
-    def add_feedback(self, feedback: Any):
-        self.feedback = feedback
-        logger.debug(f"Added feedback: {self.feedback}")
-
-    # TODO: Add validation to check if all the required parameters are present
-    def execute_metrics(self, **kwargs: Any):
-        name = kwargs.get("name")
-        model = kwargs.get("model")
-        provider = kwargs.get("provider")
-        display_name = kwargs.get("display_name", None)
-        mapping = kwargs.get("mapping", None)
-
-        if isinstance(name, str):
-            metrics = [{
-                "name": name
-            }]
-        else:
-            metrics = name if isinstance(name, list) else [name] if isinstance(name, dict) else []
-
-        for metric in metrics:
-            if not isinstance(metric, dict):
-                logger.error(f"Expected dict, got {type(metric)}")
-                continue
-
-            if "name" not in metric:
-                logger.error("Metric must contain 'name'")
-                continue
-
-            metric_name = metric["name"]
-            if metric_name in self.local_metrics:
-                count = sum(1 for m in self.local_metrics if m.startswith(metric_name))
-                metric_name = f"{metric_name}_{count + 1}"
-
-            prompt =None
-            context = None
-            response = None
-            # if mapping is not None:
-            #     prompt = mapping['prompt']
-            #     context = mapping['context']
-            #     response = mapping['response']
-            new_metric = {
-                "name": metric_name,
-                "model": model,
-                "provider": provider,
-                "project_id": self.project_id,
-                # "prompt": prompt,
-                # "context": context,
-                # "response": response,
-                "displayName": display_name,
-                "mapping": mapping
-            }
-            self.local_metrics.append(new_metric)
-
-    def add_gt(self, gt: Any):
-        if not isinstance(gt, (str, int, float, bool, list, dict)):
-            logger.error(f"Unsupported type for gt: {type(gt)}")
-            return
-        if self.gt:
-            logger.warning(f"GT already exists: {self.gt} \n Overwriting...")
-        self.gt = gt
-        logger.debug(f"Added gt: {self.gt}")
-
-    def add_context(self, context: Any):
-        if isinstance(context, str):
-            if not context.strip():
-                logger.warning("Empty or whitespace-only context string provided")
-            self.context = str(context)
-        else:
-            try:
-                self.context = str(context)
-            except Exception as e:
-                logger.warning('Cannot cast the context to string... Skipping')
-        logger.debug(f"Added context: {self.context}")

ragaai_catalyst/tracers/agentic_tracing/utils/supported_llm_provider.toml

@@ -1,34 +0,0 @@
-# List of all supported LLM method calls
-
-supported_llm_calls = [
-    # OpenAI
-    "OpenAI.chat.completions.create()",
-    "AsyncOpenAI.chat.completions.create()",
-    
-    # OpenAI Beta
-    "OpenAI.beta.threads.create()",
-    "OpenAI.beta.threads.messages.create()",
-    "OpenAI.beta.threads.runs.create()",
-    
-    # Anthropic
-    "Anthropic.messages.create()",
-    "Anthropic.messages.acreate()",
-    
-    # Google VertexAI/PaLM
-    "GenerativeModel.generate_content()",
-    "GenerativeModel.generate_content_async()",
-    "ChatVertexAI._generate()",
-    "ChatVertexAI._agenerate()",
-    "ChatVertexAI.complete()",
-    "ChatVertexAI.acomplete()",
-    
-    # Google GenerativeAI
-    "ChatGoogleGenerativeAI._generate()",
-    "ChatGoogleGenerativeAI._agenerate()",
-    "ChatGoogleGenerativeAI.complete()",
-    "ChatGoogleGenerativeAI.acomplete()",
-    
-    # LiteLLM
-    "litellm.completion()",
-    "litellm.acompletion()"
-]