deeprails 0.2.0__tar.gz → 0.3.0__tar.gz

{deeprails-0.2.0 → deeprails-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeprails
-Version: 0.2.0
+Version: 0.3.0
 Summary: Python SDK for interacting with the DeepRails API
 Project-URL: Homepage, https://deeprails.com
 Project-URL: Documentation, https://docs.deeprails.com
@@ -55,12 +55,17 @@ client = DeepRails(token="YOUR_API_KEY")
 evaluation = client.create_evaluation(
     model_input={"user_prompt": "Prompt used to generate completion"},
     model_output="Generated output",
-    model_used="gpt-4o-mini (LLM used to generate completion)",
+    model_used="gpt-4o-mini",
     guardrail_metrics=["correctness", "completeness"]
 )
-
-# Print evaluation ID
 print(f"Evaluation created with ID: {evaluation.eval_id}")
+
+# Create a monitor
+monitor = client.create_monitor(
+    name="Production Assistant Monitor",
+    description="Tracking our production assistant quality"
+)
+print(f"Monitor created with ID: {monitor.monitor_id}")
 ```
 
 ## Features
@@ -69,6 +74,7 @@ print(f"Evaluation created with ID: {evaluation.eval_id}")
 - **Comprehensive Metrics**: Evaluate outputs on correctness, completeness, and more
 - **Real-time Progress**: Track evaluation progress in real-time
 - **Detailed Results**: Get detailed scores and rationales for each metric
+- **Continuous Monitoring**: Create monitors to track AI system performance over time
 
 ## Authentication
 
@@ -81,14 +87,16 @@ token = os.environ.get("DEEPRAILS_API_KEY")
 client = DeepRails(token=token)
 ```
 
-## Creating Evaluations
+## Evaluation Service
+
+### Creating Evaluations
 
 ```python
 try:
     evaluation = client.create_evaluation(
         model_input={"user_prompt": "Prompt used to generate completion"},
         model_output="Generated output",
-        model_used="gpt-4o-mini (LLM used to generate completion)",
+        model_used="gpt-4o-mini",
         guardrail_metrics=["correctness", "completeness"]
     )
     print(f"ID: {evaluation.eval_id}")
@@ -98,7 +106,7 @@ except Exception as e:
     print(f"Error: {e}")
 ```
 
-### Parameters
+#### Parameters
 
 - `model_input`: Dictionary containing the prompt and any context (must include `user_prompt`)
 - `model_output`: The generated output to evaluate
@@ -108,7 +116,7 @@ except Exception as e:
 - `nametag`: (Optional) Custom identifier for this evaluation
 - `webhook`: (Optional) URL to receive completion notifications
 
-## Retrieving Evaluations
+### Retrieving Evaluations
 
 ```python
 try:
@@ -126,13 +134,84 @@ except Exception as e:
     print(f"Error: {e}")
 ```
 
+## Monitor Service
+
+### Creating Monitors
+
+```python
+try:
+    # Create a monitor
+    monitor = client.create_monitor(
+        name="Production Chat Assistant Monitor",
+        description="Monitoring our production chatbot responses"
+    )
+    
+    print(f"Monitor created with ID: {monitor.monitor_id}")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+### Logging Monitor Events
+
+```python
+try:
+    # Add an event to the monitor
+    event = client.create_monitor_event(
+        monitor_id="mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
+        model_input={"user_prompt": "Tell me about renewable energy"},
+        model_output="Renewable energy comes from natural sources...",
+        model_used="gpt-4o-mini",
+        guardrail_metrics=["correctness", "completeness", "comprehensive_safety"]
+    )
+    
+    print(f"Monitor event created with ID: {event.event_id}")
+    print(f"Associated evaluation ID: {event.evaluation_id}")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+### Retrieving Monitor Data
+
+```python
+try:
+    # Get monitor details
+    monitor = client.get_monitor("mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx")
+    print(f"Monitor name: {monitor.name}")
+    print(f"Status: {monitor.monitor_status}")
+    
+    # Get monitor events
+    events = client.get_monitor_events(
+        monitor_id="mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", 
+        limit=10
+    )
+    
+    for event in events:
+        print(f"Event ID: {event.event_id}")
+        print(f"Evaluation ID: {event.evaluation_id}")
+        
+    # List all monitors with filtering
+    monitors = client.get_monitors(
+        limit=5,
+        monitor_status=["active"],
+        sort_by="created_at",
+        sort_order="desc"
+    )
+    
+    print(f"Total monitors: {monitors.pagination.total_count}")
+    for m in monitors.monitors:
+        print(f"{m.name}: {m.event_count} events")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
 ## Available Metrics
 
-- `correctness`: Evaluates factual accuracy of the output
-- `completeness`: Checks if the output addresses all aspects of the prompt
-- `harmfulness`: Detects potentially harmful content
-- `bias`: Identifies biased language or reasoning
-- And more...
+- `correctness`: Measures factual accuracy by evaluating whether each claim in the output is true and verifiable.
+- `completeness`: Assesses whether the response addresses all necessary parts of the prompt with sufficient detail and relevance.
+- `instruction_adherence`: Checks whether the AI followed the explicit instructions in the prompt and system directives.
+- `context_adherence`: Determines whether each factual claim is directly supported by the provided context.
+- `ground_truth_adherence`: Measures how closely the output matches a known correct answer (gold standard).
+- `comprehensive_safety`: Detects and categorizes safety violations across areas like PII, CBRN, hate speech, self-harm, and more.
 
 ## Error Handling
 

deeprails-0.3.0/README.md

@@ -0,0 +1,198 @@
+# DeepRails Python SDK
+
+A lightweight, intuitive Python SDK for interacting with the DeepRails API. DeepRails helps you evaluate and improve AI-generated outputs through a comprehensive set of guardrail metrics.
+
+## Installation
+
+```bash
+pip install deeprails
+```
+
+## Quick Start
+
+```python
+from deeprails import DeepRails
+
+# Initialize with your API token
+client = DeepRails(token="YOUR_API_KEY")
+
+# Create an evaluation
+evaluation = client.create_evaluation(
+    model_input={"user_prompt": "Prompt used to generate completion"},
+    model_output="Generated output",
+    model_used="gpt-4o-mini",
+    guardrail_metrics=["correctness", "completeness"]
+)
+print(f"Evaluation created with ID: {evaluation.eval_id}")
+
+# Create a monitor
+monitor = client.create_monitor(
+    name="Production Assistant Monitor",
+    description="Tracking our production assistant quality"
+)
+print(f"Monitor created with ID: {monitor.monitor_id}")
+```
+
+## Features
+
+- **Simple API**: Just a few lines of code to integrate evaluation into your workflow
+- **Comprehensive Metrics**: Evaluate outputs on correctness, completeness, and more
+- **Real-time Progress**: Track evaluation progress in real-time
+- **Detailed Results**: Get detailed scores and rationales for each metric
+- **Continuous Monitoring**: Create monitors to track AI system performance over time
+
+## Authentication
+
+All API requests require authentication using your DeepRails API key. Your API key is a sensitive credential that should be kept secure.
+
+```python
+# Best practice: Load token from environment variable
+import os
+token = os.environ.get("DEEPRAILS_API_KEY")
+client = DeepRails(token=token)
+```
+
+## Evaluation Service
+
+### Creating Evaluations
+
+```python
+try:
+    evaluation = client.create_evaluation(
+        model_input={"user_prompt": "Prompt used to generate completion"},
+        model_output="Generated output",
+        model_used="gpt-4o-mini",
+        guardrail_metrics=["correctness", "completeness"]
+    )
+    print(f"ID: {evaluation.eval_id}")
+    print(f"Status: {evaluation.evaluation_status}")
+    print(f"Progress: {evaluation.progress}%")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+#### Parameters
+
+- `model_input`: Dictionary containing the prompt and any context (must include `user_prompt`)
+- `model_output`: The generated output to evaluate
+- `model_used`: (Optional) The model that generated the output
+- `run_mode`: (Optional) Evaluation run mode - defaults to "smart"
+- `guardrail_metrics`: (Optional) List of metrics to evaluate
+- `nametag`: (Optional) Custom identifier for this evaluation
+- `webhook`: (Optional) URL to receive completion notifications
+
+### Retrieving Evaluations
+
+```python
+try:
+    eval_id = "eval-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+    evaluation = client.get_evaluation(eval_id)
+    
+    print(f"Status: {evaluation.evaluation_status}")
+    
+    if evaluation.evaluation_result:
+        print("\nResults:")
+        for metric, result in evaluation.evaluation_result.items():
+            score = result.get('score', 'N/A')
+            print(f"  {metric}: {score}")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+## Monitor Service
+
+### Creating Monitors
+
+```python
+try:
+    # Create a monitor
+    monitor = client.create_monitor(
+        name="Production Chat Assistant Monitor",
+        description="Monitoring our production chatbot responses"
+    )
+    
+    print(f"Monitor created with ID: {monitor.monitor_id}")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+### Logging Monitor Events
+
+```python
+try:
+    # Add an event to the monitor
+    event = client.create_monitor_event(
+        monitor_id="mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
+        model_input={"user_prompt": "Tell me about renewable energy"},
+        model_output="Renewable energy comes from natural sources...",
+        model_used="gpt-4o-mini",
+        guardrail_metrics=["correctness", "completeness", "comprehensive_safety"]
+    )
+    
+    print(f"Monitor event created with ID: {event.event_id}")
+    print(f"Associated evaluation ID: {event.evaluation_id}")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+### Retrieving Monitor Data
+
+```python
+try:
+    # Get monitor details
+    monitor = client.get_monitor("mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx")
+    print(f"Monitor name: {monitor.name}")
+    print(f"Status: {monitor.monitor_status}")
+    
+    # Get monitor events
+    events = client.get_monitor_events(
+        monitor_id="mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", 
+        limit=10
+    )
+    
+    for event in events:
+        print(f"Event ID: {event.event_id}")
+        print(f"Evaluation ID: {event.evaluation_id}")
+        
+    # List all monitors with filtering
+    monitors = client.get_monitors(
+        limit=5,
+        monitor_status=["active"],
+        sort_by="created_at",
+        sort_order="desc"
+    )
+    
+    print(f"Total monitors: {monitors.pagination.total_count}")
+    for m in monitors.monitors:
+        print(f"{m.name}: {m.event_count} events")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+## Available Metrics
+
+- `correctness`: Measures factual accuracy by evaluating whether each claim in the output is true and verifiable.
+- `completeness`: Assesses whether the response addresses all necessary parts of the prompt with sufficient detail and relevance.
+- `instruction_adherence`: Checks whether the AI followed the explicit instructions in the prompt and system directives.
+- `context_adherence`: Determines whether each factual claim is directly supported by the provided context.
+- `ground_truth_adherence`: Measures how closely the output matches a known correct answer (gold standard).
+- `comprehensive_safety`: Detects and categorizes safety violations across areas like PII, CBRN, hate speech, self-harm, and more.
+
+## Error Handling
+
+The SDK throws `DeepRailsAPIError` for API-related errors, with status code and detailed message.
+
+```python
+from deeprails import DeepRailsAPIError
+
+try:
+    # SDK operations
+except DeepRailsAPIError as e:
+    print(f"API Error: {e.status_code} - {e.error_detail}")
+except Exception as e:
+    print(f"Unexpected error: {e}")
+```
+
+## Support
+
+For questions or support, please contact support@deeprails.ai.

deeprails-0.3.0/deeprails/client.py

@@ -0,0 +1,285 @@
+import httpx
+from typing import List, Optional, Dict, Any
+
+from .schemas import EvaluationResponse
+from .exceptions import DeepRailsAPIError
+
+class DeepRails:
+    """
+    Python SDK client for the DeepRails API.
+    """
+
+    def __init__(self, token: str, base_url: str = "https://api.deeprails.com"):
+        """
+        Initializes the DeepRails client.
+
+        Args:
+            token: Your DeepRails API key (starts with 'sk_').
+            base_url: The base URL of the DeepRails API.
+        """
+        if not token:
+            raise ValueError("A valid DeepRails API token is required.")
+        
+        self._base_url = base_url
+        self._headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "User-Agent": "deeprails-python-sdk/0.3.0"
+        }
+        self._client = httpx.Client(base_url=self._base_url, headers=self._headers, timeout=30.0)
+
+
+    def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
+        """Helper method to make requests and handle API errors."""
+        try:
+            response = self._client.request(method, endpoint, **kwargs)
+            response.raise_for_status()
+            return response
+        except httpx.HTTPStatusError as e:
+            error_detail = "No detail provided."
+            try:
+                error_detail = e.response.json().get("detail", error_detail)
+            except Exception:
+                error_detail = e.response.text
+            raise DeepRailsAPIError(status_code=e.response.status_code, error_detail=error_detail) from e
+        except httpx.RequestError as e:
+            raise DeepRailsAPIError(status_code=500, error_detail=f"Request failed: {e}") from e
+
+
+    def create_evaluation(
+        self,
+        *,
+        model_input: Dict[str, Any],
+        model_output: str,
+        model_used: Optional[str] = None,
+        run_mode: Optional[str] = "smart",  # Set default to "smart"
+        guardrail_metrics: Optional[List[str]] = None,
+        nametag: Optional[str] = None,
+        webhook: Optional[str] = None
+    ) -> EvaluationResponse:
+        """
+        Creates a new evaluation and immediately processes it.
+
+        Args:
+            model_input: A dictionary containing the inputs for the model.
+                         Must contain a "user_prompt" key.
+            model_output: The response generated by the model you are evaluating.
+            model_used: The name or identifier of the model being evaluated.
+            run_mode: The evaluation mode (e.g., "smart", "dev").
+            guardrail_metrics: A list of metrics to evaluate.
+            nametag: A user-defined name or tag for the evaluation.
+            webhook: A URL to send a POST request to upon evaluation completion.
+
+        Returns:
+            An EvaluationResponse object with the details of the created evaluation.
+        """
+        if "user_prompt" not in model_input:
+            raise ValueError("`model_input` must contain a 'user_prompt' key.")
+
+        payload = {
+            "model_input": model_input,
+            "model_output": model_output,
+            "model_used": model_used,
+            "run_mode": run_mode,
+            "guardrail_metrics": guardrail_metrics,
+            "nametag": nametag,
+            "webhook": webhook,
+        }
+        json_payload = {k: v for k, v in payload.items() if v is not None}
+
+        response = self._request("POST", "/evaluate", json=json_payload)
+        return EvaluationResponse.parse_obj(response.json())
+
+
+    def get_evaluation(self, eval_id: str) -> EvaluationResponse:
+        """
+        Retrieves the status and results of a specific evaluation.
+
+        Args:
+            eval_id: The unique identifier of the evaluation.
+
+        Returns:
+            An EvaluationResponse object with the full, up-to-date details of the evaluation.
+        """
+        response = self._request("GET", f"/evaluate/{eval_id}")
+        return EvaluationResponse.parse_obj(response.json())
+
+
+    def create_monitor(
+        self,
+        *,
+        name: str,
+        description: Optional[str] = None
+    ) -> MonitorResponse:
+        """
+        Creates a new monitor for tracking AI responses.
+
+        Args:
+            name: A name for the monitor.
+            description: Optional description of the monitor's purpose.
+
+        Returns:
+            A MonitorResponse object with the details of the created monitor.
+        """
+        payload = {
+            "name": name,
+            "description": description
+        }
+        
+        # Remove None values
+        json_payload = {k: v for k, v in payload.items() if v is not None}
+
+        response = self._request("POST", "/monitor", json=json_payload)
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json:
+            return MonitorResponse.parse_obj(response_json["data"])
+        else:
+            return MonitorResponse.parse_obj(response_json)
+
+    def get_monitor(self, monitor_id: str) -> MonitorResponse:
+        """
+        Get details of a specific monitor.
+
+        Args:
+            monitor_id: The ID of the monitor to retrieve.
+
+        Returns:
+            A MonitorResponse object with the monitor details.
+        """
+        response = self._request("GET", f"/monitor/{monitor_id}")
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json:
+            return MonitorResponse.parse_obj(response_json["data"])
+        else:
+            return MonitorResponse.parse_obj(response_json)
+
+    def create_monitor_event(
+        self,
+        *,
+        monitor_id: str,
+        model_input: Dict[str, Any],
+        model_output: str,
+        guardrail_metrics: List[str],
+        model_used: Optional[str] = None,
+        run_mode: Optional[str] = None,
+        nametag: Optional[str] = None,
+        webhook: Optional[str] = None
+    ) -> MonitorEventResponse:
+        """
+        Creates a new event for a monitor.
+        
+        Args:
+            monitor_id: The ID of the monitor to create an event for.
+            model_input: A dictionary containing the inputs for the model.
+            model_output: The response generated by the model you are evaluating.
+            guardrail_metrics: A list of metrics to evaluate.
+            model_used: The name or identifier of the model being evaluated.
+            run_mode: The evaluation mode (e.g., "smart", "dev").
+            nametag: A user-defined name or tag for the event.
+            webhook: A URL to send a POST request to upon evaluation completion.
+        
+        Returns:
+            A MonitorEventResponse object with the details of the created event.
+        """
+        payload = {
+            "model_input": model_input,
+            "model_output": model_output,
+            "model_used": model_used,
+            "run_mode": run_mode,
+            "guardrail_metrics": guardrail_metrics,
+            "nametag": nametag,
+            "webhook": webhook,
+        }
+        
+        # Remove None values
+        json_payload = {k: v for k, v in payload.items() if v is not None}
+        
+        response = self._request("POST", f"/monitor/{monitor_id}/events", json=json_payload)
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json:
+            return MonitorEventResponse.parse_obj(response_json["data"])
+        else:
+            return MonitorEventResponse.parse_obj(response_json)
+
+    def get_monitor_events(
+        self, 
+        monitor_id: str, 
+        limit: int = 10, 
+        offset: int = 0
+    ) -> List[MonitorEventResponse]:
+        """
+        Retrieves events for a specific monitor.
+        
+        Args:
+            monitor_id: The ID of the monitor to get events for.
+            limit: Maximum number of events to return (default: 10).
+            offset: Offset for pagination (default: 0).
+            
+        Returns:
+            A list of MonitorEventResponse objects with details of the monitor events.
+        """
+        params = {
+            "limit": limit,
+            "offset": offset
+        }
+        
+        response = self._request("GET", f"/monitor/{monitor_id}/events", params=params)
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json and isinstance(response_json["data"], list):
+            return [MonitorEventResponse.parse_obj(event) for event in response_json["data"]]
+        else:
+            # Fallback if the response structure is unexpected
+            return []
+
+    def get_monitors(
+        self,
+        *,
+        page: int = 1,
+        limit: int = 20,
+        search: Optional[List[str]] = None,
+        monitor_status: Optional[List[str]] = None,
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None,
+        sort_by: str = "created_at",
+        sort_order: str = "desc"
+    ) -> MonitorListResponse:
+        """
+        Get a paginated list of monitors with optional filtering.
+        
+        Args:
+            page: Page number for pagination (default: 1)
+            limit: Number of items per page (default: 20, max: 100)
+            search: Optional list of free-text search terms
+            monitor_status: Optional list of monitor statuses ("active", "inactive", "all")
+            date_from: Optional filter for monitors from this date (ISO format)
+            date_to: Optional filter for monitors to this date (ISO format)
+            sort_by: Field to sort by (default: "created_at")
+            sort_order: Sort order (default: "desc")
+            
+        Returns:
+            A MonitorListResponse object containing monitors, pagination info, and applied filters.
+        """
+        params = {
+            "page": page,
+            "limit": limit,
+            "sort_by": sort_by,
+            "sort_order": sort_order,
+            "search": search,
+            "monitor_status": monitor_status,
+            "date_from": date_from,
+            "date_to": date_to
+        }
+        
+        # Remove None values
+        params = {k: v for k, v in params.items() if v is not None}
+        
+        response = self._request("GET", "/monitor", params=params)
+        return MonitorListResponse.parse_obj(response.json())

deeprails-0.3.0/deeprails/schemas.py

@@ -0,0 +1,92 @@
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel, Field
+from datetime import datetime
+
+
+class EvaluationResponse(BaseModel):
+    """Represents the response for an evaluation from the DeepRails API."""
+    eval_id: str
+    evaluation_status: str
+    guardrail_metrics: Optional[List[str]] = None
+    model_used: Optional[str] = None
+    run_mode: Optional[str] = None
+    model_input: Optional[Dict[str, Any]] = None
+    model_output: Optional[str] = None
+    estimated_cost: Optional[float] = None
+    input_tokens: Optional[int] = None
+    output_tokens: Optional[int] = None
+    nametag: Optional[str] = None
+    progress: Optional[int] = Field(None, ge=0, le=100)
+    start_timestamp: Optional[datetime] = None
+    completion_timestamp: Optional[datetime] = None
+    error_message: Optional[str] = None
+    error_timestamp: Optional[datetime] = None
+    evaluation_result: Optional[Dict[str, Any]] = None
+    evaluation_total_cost: Optional[float] = None
+    created_at: Optional[datetime] = None
+    modified_at: Optional[datetime] = None
+
+    class Config:
+        extra = 'ignore'
+
+class MonitorResponse(BaseModel):
+    """Represents a monitor from the DeepRails API."""
+    monitor_id: str
+    user_id: str
+    name: str
+    description: Optional[str] = None
+    monitor_status: str
+    created_at: str
+    updated_at: str
+
+    class Config:
+        extra = 'ignore'
+
+class MonitorEventCreate(BaseModel):
+    """Model for creating a new monitor event."""
+    model_input: Dict[str, Any]
+    model_output: str
+    model_used: Optional[str] = None
+    run_mode: Optional[str] = None
+    guardrail_metrics: List[str]
+    nametag: Optional[str] = None
+    webhook: Optional[str] = None
+
+class MonitorEventResponse(BaseModel):
+    """Response model for a monitor event."""
+    event_id: str
+    monitor_id: str
+    evaluation_id: str
+    created_at: str
+
+    class Config:
+        extra = 'ignore'
+
+class PaginationInfo(BaseModel):
+    """Pagination information for list responses."""
+    page: int
+    limit: int
+    total_pages: int
+    total_count: int
+    has_next: bool
+    has_previous: bool
+
+class MonitorFiltersApplied(BaseModel):
+    """Information about which filters were applied to the monitor query."""
+    search: Optional[List[str]] = None
+    status: Optional[List[str]] = None
+    date_from: Optional[str] = None
+    date_to: Optional[str] = None
+    sort_by: Optional[str] = None
+    sort_order: Optional[str] = None
+
+class MonitorWithEventCountResponse(MonitorResponse):
+    """Monitor response with event count information."""
+    event_count: int
+    latest_event_modified_at: Optional[str] = None
+
+class MonitorListResponse(BaseModel):
+    """Response model for a paginated list of monitors."""
+    monitors: List[MonitorWithEventCountResponse]
+    pagination: PaginationInfo
+    filters_applied: MonitorFiltersApplied

{deeprails-0.2.0 → deeprails-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "deeprails"
-version = "0.2.0"
+version = "0.3.0"
 description = "Python SDK for interacting with the DeepRails API"
 readme = {file = "README.md", content-type = "text/markdown"}
 authors = [{name = "Neil Mate", email = "support@deeprails.ai"}]

deeprails-0.2.0/README.md

@@ -1,119 +0,0 @@
-# DeepRails Python SDK
-
-A lightweight, intuitive Python SDK for interacting with the DeepRails API. DeepRails helps you evaluate and improve AI-generated outputs through a comprehensive set of guardrail metrics.
-
-## Installation
-
-```bash
-pip install deeprails
-```
-
-## Quick Start
-
-```python
-from deeprails import DeepRails
-
-# Initialize with your API token
-client = DeepRails(token="YOUR_API_KEY")
-
-# Create an evaluation
-evaluation = client.create_evaluation(
-    model_input={"user_prompt": "Prompt used to generate completion"},
-    model_output="Generated output",
-    model_used="gpt-4o-mini (LLM used to generate completion)",
-    guardrail_metrics=["correctness", "completeness"]
-)
-
-# Print evaluation ID
-print(f"Evaluation created with ID: {evaluation.eval_id}")
-```
-
-## Features
-
-- **Simple API**: Just a few lines of code to integrate evaluation into your workflow
-- **Comprehensive Metrics**: Evaluate outputs on correctness, completeness, and more
-- **Real-time Progress**: Track evaluation progress in real-time
-- **Detailed Results**: Get detailed scores and rationales for each metric
-
-## Authentication
-
-All API requests require authentication using your DeepRails API key. Your API key is a sensitive credential that should be kept secure.
-
-```python
-# Best practice: Load token from environment variable
-import os
-token = os.environ.get("DEEPRAILS_API_KEY")
-client = DeepRails(token=token)
-```
-
-## Creating Evaluations
-
-```python
-try:
-    evaluation = client.create_evaluation(
-        model_input={"user_prompt": "Prompt used to generate completion"},
-        model_output="Generated output",
-        model_used="gpt-4o-mini (LLM used to generate completion)",
-        guardrail_metrics=["correctness", "completeness"]
-    )
-    print(f"ID: {evaluation.eval_id}")
-    print(f"Status: {evaluation.evaluation_status}")
-    print(f"Progress: {evaluation.progress}%")
-except Exception as e:
-    print(f"Error: {e}")
-```
-
-### Parameters
-
-- `model_input`: Dictionary containing the prompt and any context (must include `user_prompt`)
-- `model_output`: The generated output to evaluate
-- `model_used`: (Optional) The model that generated the output
-- `run_mode`: (Optional) Evaluation run mode - defaults to "smart"
-- `guardrail_metrics`: (Optional) List of metrics to evaluate
-- `nametag`: (Optional) Custom identifier for this evaluation
-- `webhook`: (Optional) URL to receive completion notifications
-
-## Retrieving Evaluations
-
-```python
-try:
-    eval_id = "eval-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
-    evaluation = client.get_evaluation(eval_id)
-    
-    print(f"Status: {evaluation.evaluation_status}")
-    
-    if evaluation.evaluation_result:
-        print("\nResults:")
-        for metric, result in evaluation.evaluation_result.items():
-            score = result.get('score', 'N/A')
-            print(f"  {metric}: {score}")
-except Exception as e:
-    print(f"Error: {e}")
-```
-
-## Available Metrics
-
-- `correctness`: Evaluates factual accuracy of the output
-- `completeness`: Checks if the output addresses all aspects of the prompt
-- `harmfulness`: Detects potentially harmful content
-- `bias`: Identifies biased language or reasoning
-- And more...
-
-## Error Handling
-
-The SDK throws `DeepRailsAPIError` for API-related errors, with status code and detailed message.
-
-```python
-from deeprails import DeepRailsAPIError
-
-try:
-    # SDK operations
-except DeepRailsAPIError as e:
-    print(f"API Error: {e.status_code} - {e.error_detail}")
-except Exception as e:
-    print(f"Unexpected error: {e}")
-```
-
-## Support
-
-For questions or support, please contact support@deeprails.ai.

deeprails-0.2.0/deeprails/client.py

@@ -1,101 +0,0 @@
-import httpx
-from typing import List, Optional, Dict, Any
-
-from .schemas import EvaluationResponse
-from .exceptions import DeepRailsAPIError
-
-class DeepRails:
-    """
-    Python SDK client for the DeepRails API.
-    """
-    def __init__(self, token: str, base_url: str = "https://api.deeprails.com"):
-        """
-        Initializes the DeepRails client.
-
-        Args:
-            token: Your DeepRails API key (starts with 'sk_').
-            base_url: The base URL of the DeepRails API.
-        """
-        if not token:
-            raise ValueError("A valid DeepRails API token is required.")
-        
-        self._base_url = base_url
-        self._headers = {
-            "Authorization": f"Bearer {token}",
-            "Content-Type": "application/json",
-            "User-Agent": "deeprails-python-sdk/0.2.0"
-        }
-        self._client = httpx.Client(base_url=self._base_url, headers=self._headers, timeout=30.0)
-
-    def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
-        """Helper method to make requests and handle API errors."""
-        try:
-            response = self._client.request(method, endpoint, **kwargs)
-            response.raise_for_status()
-            return response
-        except httpx.HTTPStatusError as e:
-            error_detail = "No detail provided."
-            try:
-                error_detail = e.response.json().get("detail", error_detail)
-            except Exception:
-                error_detail = e.response.text
-            raise DeepRailsAPIError(status_code=e.response.status_code, error_detail=error_detail) from e
-        except httpx.RequestError as e:
-            raise DeepRailsAPIError(status_code=500, error_detail=f"Request failed: {e}") from e
-
-    def create_evaluation(
-        self,
-        *,
-        model_input: Dict[str, Any],
-        model_output: str,
-        model_used: Optional[str] = None,
-        run_mode: Optional[str] = "smart",  # Set default to "smart"
-        guardrail_metrics: Optional[List[str]] = None,
-        nametag: Optional[str] = None,
-        webhook: Optional[str] = None
-    ) -> EvaluationResponse:
-        """
-        Creates a new evaluation and immediately processes it.
-
-        Args:
-            model_input: A dictionary containing the inputs for the model.
-                         Must contain a "user_prompt" key.
-            model_output: The response generated by the model you are evaluating.
-            model_used: The name or identifier of the model being evaluated.
-            run_mode: The evaluation mode (e.g., "smart", "dev").
-            guardrail_metrics: A list of metrics to evaluate.
-            nametag: A user-defined name or tag for the evaluation.
-            webhook: A URL to send a POST request to upon evaluation completion.
-
-        Returns:
-            An EvaluationResponse object with the details of the created evaluation.
-        """
-        if "user_prompt" not in model_input:
-            raise ValueError("`model_input` must contain a 'user_prompt' key.")
-
-        payload = {
-            "model_input": model_input,
-            "model_output": model_output,
-            "model_used": model_used,
-            "run_mode": run_mode,
-            "guardrail_metrics": guardrail_metrics,
-            "nametag": nametag,
-            "webhook": webhook,
-        }
-        json_payload = {k: v for k, v in payload.items() if v is not None}
-
-        response = self._request("POST", "/evaluate", json=json_payload)
-        return EvaluationResponse.parse_obj(response.json())
-
-    def get_evaluation(self, eval_id: str) -> EvaluationResponse:
-        """
-        Retrieves the status and results of a specific evaluation.
-
-        Args:
-            eval_id: The unique identifier of the evaluation.
-
-        Returns:
-            An EvaluationResponse object with the full, up-to-date details of the evaluation.
-        """
-        response = self._request("GET", f"/evaluate/{eval_id}")
-        return EvaluationResponse.parse_obj(response.json())

deeprails-0.2.0/deeprails/schemas.py

@@ -1,30 +0,0 @@
-from typing import List, Optional, Dict, Any
-from pydantic import BaseModel, Field
-from datetime import datetime
-
-
-class EvaluationResponse(BaseModel):
-    """Represents the response for an evaluation from the DeepRails API."""
-    eval_id: str
-    evaluation_status: str
-    guardrail_metrics: Optional[List[str]] = None
-    model_used: Optional[str] = None
-    run_mode: Optional[str] = None
-    model_input: Optional[Dict[str, Any]] = None
-    model_output: Optional[str] = None
-    estimated_cost: Optional[float] = None
-    input_tokens: Optional[int] = None
-    output_tokens: Optional[int] = None
-    nametag: Optional[str] = None
-    progress: Optional[int] = Field(None, ge=0, le=100)
-    start_timestamp: Optional[datetime] = None
-    completion_timestamp: Optional[datetime] = None
-    error_message: Optional[str] = None
-    error_timestamp: Optional[datetime] = None
-    evaluation_result: Optional[Dict[str, Any]] = None
-    evaluation_total_cost: Optional[float] = None
-    created_at: Optional[datetime] = None
-    modified_at: Optional[datetime] = None
-
-    class Config:
-        extra = 'ignore'