deeprails 0.2.1__tar.gz → 0.3.0__tar.gz

{deeprails-0.2.1 → deeprails-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeprails
-Version: 0.2.1
+Version: 0.3.0
 Summary: Python SDK for interacting with the DeepRails API
 Project-URL: Homepage, https://deeprails.com
 Project-URL: Documentation, https://docs.deeprails.com
@@ -55,12 +55,17 @@ client = DeepRails(token="YOUR_API_KEY")
 evaluation = client.create_evaluation(
     model_input={"user_prompt": "Prompt used to generate completion"},
     model_output="Generated output",
-    model_used="gpt-4o-mini (LLM used to generate completion)",
+    model_used="gpt-4o-mini",
     guardrail_metrics=["correctness", "completeness"]
 )
-
-# Print evaluation ID
 print(f"Evaluation created with ID: {evaluation.eval_id}")
+
+# Create a monitor
+monitor = client.create_monitor(
+    name="Production Assistant Monitor",
+    description="Tracking our production assistant quality"
+)
+print(f"Monitor created with ID: {monitor.monitor_id}")
 ```
 
 ## Features
@@ -69,6 +74,7 @@ print(f"Evaluation created with ID: {evaluation.eval_id}")
 - **Comprehensive Metrics**: Evaluate outputs on correctness, completeness, and more
 - **Real-time Progress**: Track evaluation progress in real-time
 - **Detailed Results**: Get detailed scores and rationales for each metric
+- **Continuous Monitoring**: Create monitors to track AI system performance over time
 
 ## Authentication
 
@@ -81,14 +87,16 @@ token = os.environ.get("DEEPRAILS_API_KEY")
 client = DeepRails(token=token)
 ```
 
-## Creating Evaluations
+## Evaluation Service
+
+### Creating Evaluations
 
 ```python
 try:
     evaluation = client.create_evaluation(
         model_input={"user_prompt": "Prompt used to generate completion"},
         model_output="Generated output",
-        model_used="gpt-4o-mini (LLM used to generate completion)",
+        model_used="gpt-4o-mini",
         guardrail_metrics=["correctness", "completeness"]
     )
     print(f"ID: {evaluation.eval_id}")
@@ -98,7 +106,7 @@ except Exception as e:
     print(f"Error: {e}")
 ```
 
-### Parameters
+#### Parameters
 
 - `model_input`: Dictionary containing the prompt and any context (must include `user_prompt`)
 - `model_output`: The generated output to evaluate
@@ -108,7 +116,7 @@ except Exception as e:
 - `nametag`: (Optional) Custom identifier for this evaluation
 - `webhook`: (Optional) URL to receive completion notifications
 
-## Retrieving Evaluations
+### Retrieving Evaluations
 
 ```python
 try:
@@ -126,6 +134,76 @@ except Exception as e:
     print(f"Error: {e}")
 ```
 
+## Monitor Service
+
+### Creating Monitors
+
+```python
+try:
+    # Create a monitor
+    monitor = client.create_monitor(
+        name="Production Chat Assistant Monitor",
+        description="Monitoring our production chatbot responses"
+    )
+    
+    print(f"Monitor created with ID: {monitor.monitor_id}")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+### Logging Monitor Events
+
+```python
+try:
+    # Add an event to the monitor
+    event = client.create_monitor_event(
+        monitor_id="mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
+        model_input={"user_prompt": "Tell me about renewable energy"},
+        model_output="Renewable energy comes from natural sources...",
+        model_used="gpt-4o-mini",
+        guardrail_metrics=["correctness", "completeness", "comprehensive_safety"]
+    )
+    
+    print(f"Monitor event created with ID: {event.event_id}")
+    print(f"Associated evaluation ID: {event.evaluation_id}")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+### Retrieving Monitor Data
+
+```python
+try:
+    # Get monitor details
+    monitor = client.get_monitor("mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx")
+    print(f"Monitor name: {monitor.name}")
+    print(f"Status: {monitor.monitor_status}")
+    
+    # Get monitor events
+    events = client.get_monitor_events(
+        monitor_id="mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", 
+        limit=10
+    )
+    
+    for event in events:
+        print(f"Event ID: {event.event_id}")
+        print(f"Evaluation ID: {event.evaluation_id}")
+        
+    # List all monitors with filtering
+    monitors = client.get_monitors(
+        limit=5,
+        monitor_status=["active"],
+        sort_by="created_at",
+        sort_order="desc"
+    )
+    
+    print(f"Total monitors: {monitors.pagination.total_count}")
+    for m in monitors.monitors:
+        print(f"{m.name}: {m.event_count} events")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
 ## Available Metrics
 
 - `correctness`: Measures factual accuracy by evaluating whether each claim in the output is true and verifiable.
@@ -135,7 +213,6 @@ except Exception as e:
 - `ground_truth_adherence`: Measures how closely the output matches a known correct answer (gold standard).
 - `comprehensive_safety`: Detects and categorizes safety violations across areas like PII, CBRN, hate speech, self-harm, and more.
 
-
 ## Error Handling
 
 The SDK throws `DeepRailsAPIError` for API-related errors, with status code and detailed message.

{deeprails-0.2.1 → deeprails-0.3.0}/README.md

@@ -20,12 +20,17 @@ client = DeepRails(token="YOUR_API_KEY")
 evaluation = client.create_evaluation(
     model_input={"user_prompt": "Prompt used to generate completion"},
     model_output="Generated output",
-    model_used="gpt-4o-mini (LLM used to generate completion)",
+    model_used="gpt-4o-mini",
     guardrail_metrics=["correctness", "completeness"]
 )
-
-# Print evaluation ID
 print(f"Evaluation created with ID: {evaluation.eval_id}")
+
+# Create a monitor
+monitor = client.create_monitor(
+    name="Production Assistant Monitor",
+    description="Tracking our production assistant quality"
+)
+print(f"Monitor created with ID: {monitor.monitor_id}")
 ```
 
 ## Features
@@ -34,6 +39,7 @@ print(f"Evaluation created with ID: {evaluation.eval_id}")
 - **Comprehensive Metrics**: Evaluate outputs on correctness, completeness, and more
 - **Real-time Progress**: Track evaluation progress in real-time
 - **Detailed Results**: Get detailed scores and rationales for each metric
+- **Continuous Monitoring**: Create monitors to track AI system performance over time
 
 ## Authentication
 
@@ -46,14 +52,16 @@ token = os.environ.get("DEEPRAILS_API_KEY")
 client = DeepRails(token=token)
 ```
 
-## Creating Evaluations
+## Evaluation Service
+
+### Creating Evaluations
 
 ```python
 try:
     evaluation = client.create_evaluation(
         model_input={"user_prompt": "Prompt used to generate completion"},
         model_output="Generated output",
-        model_used="gpt-4o-mini (LLM used to generate completion)",
+        model_used="gpt-4o-mini",
         guardrail_metrics=["correctness", "completeness"]
     )
     print(f"ID: {evaluation.eval_id}")
@@ -63,7 +71,7 @@ except Exception as e:
     print(f"Error: {e}")
 ```
 
-### Parameters
+#### Parameters
 
 - `model_input`: Dictionary containing the prompt and any context (must include `user_prompt`)
 - `model_output`: The generated output to evaluate
@@ -73,7 +81,7 @@ except Exception as e:
 - `nametag`: (Optional) Custom identifier for this evaluation
 - `webhook`: (Optional) URL to receive completion notifications
 
-## Retrieving Evaluations
+### Retrieving Evaluations
 
 ```python
 try:
@@ -91,6 +99,76 @@ except Exception as e:
     print(f"Error: {e}")
 ```
 
+## Monitor Service
+
+### Creating Monitors
+
+```python
+try:
+    # Create a monitor
+    monitor = client.create_monitor(
+        name="Production Chat Assistant Monitor",
+        description="Monitoring our production chatbot responses"
+    )
+    
+    print(f"Monitor created with ID: {monitor.monitor_id}")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+### Logging Monitor Events
+
+```python
+try:
+    # Add an event to the monitor
+    event = client.create_monitor_event(
+        monitor_id="mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
+        model_input={"user_prompt": "Tell me about renewable energy"},
+        model_output="Renewable energy comes from natural sources...",
+        model_used="gpt-4o-mini",
+        guardrail_metrics=["correctness", "completeness", "comprehensive_safety"]
+    )
+    
+    print(f"Monitor event created with ID: {event.event_id}")
+    print(f"Associated evaluation ID: {event.evaluation_id}")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+### Retrieving Monitor Data
+
+```python
+try:
+    # Get monitor details
+    monitor = client.get_monitor("mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx")
+    print(f"Monitor name: {monitor.name}")
+    print(f"Status: {monitor.monitor_status}")
+    
+    # Get monitor events
+    events = client.get_monitor_events(
+        monitor_id="mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", 
+        limit=10
+    )
+    
+    for event in events:
+        print(f"Event ID: {event.event_id}")
+        print(f"Evaluation ID: {event.evaluation_id}")
+        
+    # List all monitors with filtering
+    monitors = client.get_monitors(
+        limit=5,
+        monitor_status=["active"],
+        sort_by="created_at",
+        sort_order="desc"
+    )
+    
+    print(f"Total monitors: {monitors.pagination.total_count}")
+    for m in monitors.monitors:
+        print(f"{m.name}: {m.event_count} events")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
 ## Available Metrics
 
 - `correctness`: Measures factual accuracy by evaluating whether each claim in the output is true and verifiable.
@@ -100,7 +178,6 @@ except Exception as e:
 - `ground_truth_adherence`: Measures how closely the output matches a known correct answer (gold standard).
 - `comprehensive_safety`: Detects and categorizes safety violations across areas like PII, CBRN, hate speech, self-harm, and more.
 
-
 ## Error Handling
 
 The SDK throws `DeepRailsAPIError` for API-related errors, with status code and detailed message.

deeprails-0.3.0/deeprails/client.py

@@ -0,0 +1,285 @@
+import httpx
+from typing import List, Optional, Dict, Any
+
+from .schemas import EvaluationResponse
+from .exceptions import DeepRailsAPIError
+
+class DeepRails:
+    """
+    Python SDK client for the DeepRails API.
+    """
+
+    def __init__(self, token: str, base_url: str = "https://api.deeprails.com"):
+        """
+        Initializes the DeepRails client.
+
+        Args:
+            token: Your DeepRails API key (starts with 'sk_').
+            base_url: The base URL of the DeepRails API.
+        """
+        if not token:
+            raise ValueError("A valid DeepRails API token is required.")
+        
+        self._base_url = base_url
+        self._headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "User-Agent": "deeprails-python-sdk/0.3.0"
+        }
+        self._client = httpx.Client(base_url=self._base_url, headers=self._headers, timeout=30.0)
+
+
+    def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
+        """Helper method to make requests and handle API errors."""
+        try:
+            response = self._client.request(method, endpoint, **kwargs)
+            response.raise_for_status()
+            return response
+        except httpx.HTTPStatusError as e:
+            error_detail = "No detail provided."
+            try:
+                error_detail = e.response.json().get("detail", error_detail)
+            except Exception:
+                error_detail = e.response.text
+            raise DeepRailsAPIError(status_code=e.response.status_code, error_detail=error_detail) from e
+        except httpx.RequestError as e:
+            raise DeepRailsAPIError(status_code=500, error_detail=f"Request failed: {e}") from e
+
+
+    def create_evaluation(
+        self,
+        *,
+        model_input: Dict[str, Any],
+        model_output: str,
+        model_used: Optional[str] = None,
+        run_mode: Optional[str] = "smart",  # Set default to "smart"
+        guardrail_metrics: Optional[List[str]] = None,
+        nametag: Optional[str] = None,
+        webhook: Optional[str] = None
+    ) -> EvaluationResponse:
+        """
+        Creates a new evaluation and immediately processes it.
+
+        Args:
+            model_input: A dictionary containing the inputs for the model.
+                         Must contain a "user_prompt" key.
+            model_output: The response generated by the model you are evaluating.
+            model_used: The name or identifier of the model being evaluated.
+            run_mode: The evaluation mode (e.g., "smart", "dev").
+            guardrail_metrics: A list of metrics to evaluate.
+            nametag: A user-defined name or tag for the evaluation.
+            webhook: A URL to send a POST request to upon evaluation completion.
+
+        Returns:
+            An EvaluationResponse object with the details of the created evaluation.
+        """
+        if "user_prompt" not in model_input:
+            raise ValueError("`model_input` must contain a 'user_prompt' key.")
+
+        payload = {
+            "model_input": model_input,
+            "model_output": model_output,
+            "model_used": model_used,
+            "run_mode": run_mode,
+            "guardrail_metrics": guardrail_metrics,
+            "nametag": nametag,
+            "webhook": webhook,
+        }
+        json_payload = {k: v for k, v in payload.items() if v is not None}
+
+        response = self._request("POST", "/evaluate", json=json_payload)
+        return EvaluationResponse.parse_obj(response.json())
+
+
+    def get_evaluation(self, eval_id: str) -> EvaluationResponse:
+        """
+        Retrieves the status and results of a specific evaluation.
+
+        Args:
+            eval_id: The unique identifier of the evaluation.
+
+        Returns:
+            An EvaluationResponse object with the full, up-to-date details of the evaluation.
+        """
+        response = self._request("GET", f"/evaluate/{eval_id}")
+        return EvaluationResponse.parse_obj(response.json())
+
+
+    def create_monitor(
+        self,
+        *,
+        name: str,
+        description: Optional[str] = None
+    ) -> MonitorResponse:
+        """
+        Creates a new monitor for tracking AI responses.
+
+        Args:
+            name: A name for the monitor.
+            description: Optional description of the monitor's purpose.
+
+        Returns:
+            A MonitorResponse object with the details of the created monitor.
+        """
+        payload = {
+            "name": name,
+            "description": description
+        }
+        
+        # Remove None values
+        json_payload = {k: v for k, v in payload.items() if v is not None}
+
+        response = self._request("POST", "/monitor", json=json_payload)
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json:
+            return MonitorResponse.parse_obj(response_json["data"])
+        else:
+            return MonitorResponse.parse_obj(response_json)
+
+    def get_monitor(self, monitor_id: str) -> MonitorResponse:
+        """
+        Get details of a specific monitor.
+
+        Args:
+            monitor_id: The ID of the monitor to retrieve.
+
+        Returns:
+            A MonitorResponse object with the monitor details.
+        """
+        response = self._request("GET", f"/monitor/{monitor_id}")
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json:
+            return MonitorResponse.parse_obj(response_json["data"])
+        else:
+            return MonitorResponse.parse_obj(response_json)
+
+    def create_monitor_event(
+        self,
+        *,
+        monitor_id: str,
+        model_input: Dict[str, Any],
+        model_output: str,
+        guardrail_metrics: List[str],
+        model_used: Optional[str] = None,
+        run_mode: Optional[str] = None,
+        nametag: Optional[str] = None,
+        webhook: Optional[str] = None
+    ) -> MonitorEventResponse:
+        """
+        Creates a new event for a monitor.
+        
+        Args:
+            monitor_id: The ID of the monitor to create an event for.
+            model_input: A dictionary containing the inputs for the model.
+            model_output: The response generated by the model you are evaluating.
+            guardrail_metrics: A list of metrics to evaluate.
+            model_used: The name or identifier of the model being evaluated.
+            run_mode: The evaluation mode (e.g., "smart", "dev").
+            nametag: A user-defined name or tag for the event.
+            webhook: A URL to send a POST request to upon evaluation completion.
+        
+        Returns:
+            A MonitorEventResponse object with the details of the created event.
+        """
+        payload = {
+            "model_input": model_input,
+            "model_output": model_output,
+            "model_used": model_used,
+            "run_mode": run_mode,
+            "guardrail_metrics": guardrail_metrics,
+            "nametag": nametag,
+            "webhook": webhook,
+        }
+        
+        # Remove None values
+        json_payload = {k: v for k, v in payload.items() if v is not None}
+        
+        response = self._request("POST", f"/monitor/{monitor_id}/events", json=json_payload)
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json:
+            return MonitorEventResponse.parse_obj(response_json["data"])
+        else:
+            return MonitorEventResponse.parse_obj(response_json)
+
+    def get_monitor_events(
+        self, 
+        monitor_id: str, 
+        limit: int = 10, 
+        offset: int = 0
+    ) -> List[MonitorEventResponse]:
+        """
+        Retrieves events for a specific monitor.
+        
+        Args:
+            monitor_id: The ID of the monitor to get events for.
+            limit: Maximum number of events to return (default: 10).
+            offset: Offset for pagination (default: 0).
+            
+        Returns:
+            A list of MonitorEventResponse objects with details of the monitor events.
+        """
+        params = {
+            "limit": limit,
+            "offset": offset
+        }
+        
+        response = self._request("GET", f"/monitor/{monitor_id}/events", params=params)
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json and isinstance(response_json["data"], list):
+            return [MonitorEventResponse.parse_obj(event) for event in response_json["data"]]
+        else:
+            # Fallback if the response structure is unexpected
+            return []
+
+    def get_monitors(
+        self,
+        *,
+        page: int = 1,
+        limit: int = 20,
+        search: Optional[List[str]] = None,
+        monitor_status: Optional[List[str]] = None,
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None,
+        sort_by: str = "created_at",
+        sort_order: str = "desc"
+    ) -> MonitorListResponse:
+        """
+        Get a paginated list of monitors with optional filtering.
+        
+        Args:
+            page: Page number for pagination (default: 1)
+            limit: Number of items per page (default: 20, max: 100)
+            search: Optional list of free-text search terms
+            monitor_status: Optional list of monitor statuses ("active", "inactive", "all")
+            date_from: Optional filter for monitors from this date (ISO format)
+            date_to: Optional filter for monitors to this date (ISO format)
+            sort_by: Field to sort by (default: "created_at")
+            sort_order: Sort order (default: "desc")
+            
+        Returns:
+            A MonitorListResponse object containing monitors, pagination info, and applied filters.
+        """
+        params = {
+            "page": page,
+            "limit": limit,
+            "sort_by": sort_by,
+            "sort_order": sort_order,
+            "search": search,
+            "monitor_status": monitor_status,
+            "date_from": date_from,
+            "date_to": date_to
+        }
+        
+        # Remove None values
+        params = {k: v for k, v in params.items() if v is not None}
+        
+        response = self._request("GET", "/monitor", params=params)
+        return MonitorListResponse.parse_obj(response.json())

deeprails-0.3.0/deeprails/schemas.py

@@ -0,0 +1,92 @@
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel, Field
+from datetime import datetime
+
+
+class EvaluationResponse(BaseModel):
+    """Represents the response for an evaluation from the DeepRails API."""
+    eval_id: str
+    evaluation_status: str
+    guardrail_metrics: Optional[List[str]] = None
+    model_used: Optional[str] = None
+    run_mode: Optional[str] = None
+    model_input: Optional[Dict[str, Any]] = None
+    model_output: Optional[str] = None
+    estimated_cost: Optional[float] = None
+    input_tokens: Optional[int] = None
+    output_tokens: Optional[int] = None
+    nametag: Optional[str] = None
+    progress: Optional[int] = Field(None, ge=0, le=100)
+    start_timestamp: Optional[datetime] = None
+    completion_timestamp: Optional[datetime] = None
+    error_message: Optional[str] = None
+    error_timestamp: Optional[datetime] = None
+    evaluation_result: Optional[Dict[str, Any]] = None
+    evaluation_total_cost: Optional[float] = None
+    created_at: Optional[datetime] = None
+    modified_at: Optional[datetime] = None
+
+    class Config:
+        extra = 'ignore'
+
+class MonitorResponse(BaseModel):
+    """Represents a monitor from the DeepRails API."""
+    monitor_id: str
+    user_id: str
+    name: str
+    description: Optional[str] = None
+    monitor_status: str
+    created_at: str
+    updated_at: str
+
+    class Config:
+        extra = 'ignore'
+
+class MonitorEventCreate(BaseModel):
+    """Model for creating a new monitor event."""
+    model_input: Dict[str, Any]
+    model_output: str
+    model_used: Optional[str] = None
+    run_mode: Optional[str] = None
+    guardrail_metrics: List[str]
+    nametag: Optional[str] = None
+    webhook: Optional[str] = None
+
+class MonitorEventResponse(BaseModel):
+    """Response model for a monitor event."""
+    event_id: str
+    monitor_id: str
+    evaluation_id: str
+    created_at: str
+
+    class Config:
+        extra = 'ignore'
+
+class PaginationInfo(BaseModel):
+    """Pagination information for list responses."""
+    page: int
+    limit: int
+    total_pages: int
+    total_count: int
+    has_next: bool
+    has_previous: bool
+
+class MonitorFiltersApplied(BaseModel):
+    """Information about which filters were applied to the monitor query."""
+    search: Optional[List[str]] = None
+    status: Optional[List[str]] = None
+    date_from: Optional[str] = None
+    date_to: Optional[str] = None
+    sort_by: Optional[str] = None
+    sort_order: Optional[str] = None
+
+class MonitorWithEventCountResponse(MonitorResponse):
+    """Monitor response with event count information."""
+    event_count: int
+    latest_event_modified_at: Optional[str] = None
+
+class MonitorListResponse(BaseModel):
+    """Response model for a paginated list of monitors."""
+    monitors: List[MonitorWithEventCountResponse]
+    pagination: PaginationInfo
+    filters_applied: MonitorFiltersApplied

{deeprails-0.2.1 → deeprails-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "deeprails"
-version = "0.2.1"
+version = "0.3.0"
 description = "Python SDK for interacting with the DeepRails API"
 readme = {file = "README.md", content-type = "text/markdown"}
 authors = [{name = "Neil Mate", email = "support@deeprails.ai"}]

deeprails-0.2.1/deeprails/client.py

@@ -1,101 +0,0 @@
-import httpx
-from typing import List, Optional, Dict, Any
-
-from .schemas import EvaluationResponse
-from .exceptions import DeepRailsAPIError
-
-class DeepRails:
-    """
-    Python SDK client for the DeepRails API.
-    """
-    def __init__(self, token: str, base_url: str = "https://api.deeprails.com"):
-        """
-        Initializes the DeepRails client.
-
-        Args:
-            token: Your DeepRails API key (starts with 'sk_').
-            base_url: The base URL of the DeepRails API.
-        """
-        if not token:
-            raise ValueError("A valid DeepRails API token is required.")
-        
-        self._base_url = base_url
-        self._headers = {
-            "Authorization": f"Bearer {token}",
-            "Content-Type": "application/json",
-            "User-Agent": "deeprails-python-sdk/0.2.0"
-        }
-        self._client = httpx.Client(base_url=self._base_url, headers=self._headers, timeout=30.0)
-
-    def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
-        """Helper method to make requests and handle API errors."""
-        try:
-            response = self._client.request(method, endpoint, **kwargs)
-            response.raise_for_status()
-            return response
-        except httpx.HTTPStatusError as e:
-            error_detail = "No detail provided."
-            try:
-                error_detail = e.response.json().get("detail", error_detail)
-            except Exception:
-                error_detail = e.response.text
-            raise DeepRailsAPIError(status_code=e.response.status_code, error_detail=error_detail) from e
-        except httpx.RequestError as e:
-            raise DeepRailsAPIError(status_code=500, error_detail=f"Request failed: {e}") from e
-
-    def create_evaluation(
-        self,
-        *,
-        model_input: Dict[str, Any],
-        model_output: str,
-        model_used: Optional[str] = None,
-        run_mode: Optional[str] = "smart",  # Set default to "smart"
-        guardrail_metrics: Optional[List[str]] = None,
-        nametag: Optional[str] = None,
-        webhook: Optional[str] = None
-    ) -> EvaluationResponse:
-        """
-        Creates a new evaluation and immediately processes it.
-
-        Args:
-            model_input: A dictionary containing the inputs for the model.
-                         Must contain a "user_prompt" key.
-            model_output: The response generated by the model you are evaluating.
-            model_used: The name or identifier of the model being evaluated.
-            run_mode: The evaluation mode (e.g., "smart", "dev").
-            guardrail_metrics: A list of metrics to evaluate.
-            nametag: A user-defined name or tag for the evaluation.
-            webhook: A URL to send a POST request to upon evaluation completion.
-
-        Returns:
-            An EvaluationResponse object with the details of the created evaluation.
-        """
-        if "user_prompt" not in model_input:
-            raise ValueError("`model_input` must contain a 'user_prompt' key.")
-
-        payload = {
-            "model_input": model_input,
-            "model_output": model_output,
-            "model_used": model_used,
-            "run_mode": run_mode,
-            "guardrail_metrics": guardrail_metrics,
-            "nametag": nametag,
-            "webhook": webhook,
-        }
-        json_payload = {k: v for k, v in payload.items() if v is not None}
-
-        response = self._request("POST", "/evaluate", json=json_payload)
-        return EvaluationResponse.parse_obj(response.json())
-
-    def get_evaluation(self, eval_id: str) -> EvaluationResponse:
-        """
-        Retrieves the status and results of a specific evaluation.
-
-        Args:
-            eval_id: The unique identifier of the evaluation.
-
-        Returns:
-            An EvaluationResponse object with the full, up-to-date details of the evaluation.
-        """
-        response = self._request("GET", f"/evaluate/{eval_id}")
-        return EvaluationResponse.parse_obj(response.json())

deeprails-0.2.1/deeprails/schemas.py

@@ -1,30 +0,0 @@
-from typing import List, Optional, Dict, Any
-from pydantic import BaseModel, Field
-from datetime import datetime
-
-
-class EvaluationResponse(BaseModel):
-    """Represents the response for an evaluation from the DeepRails API."""
-    eval_id: str
-    evaluation_status: str
-    guardrail_metrics: Optional[List[str]] = None
-    model_used: Optional[str] = None
-    run_mode: Optional[str] = None
-    model_input: Optional[Dict[str, Any]] = None
-    model_output: Optional[str] = None
-    estimated_cost: Optional[float] = None
-    input_tokens: Optional[int] = None
-    output_tokens: Optional[int] = None
-    nametag: Optional[str] = None
-    progress: Optional[int] = Field(None, ge=0, le=100)
-    start_timestamp: Optional[datetime] = None
-    completion_timestamp: Optional[datetime] = None
-    error_message: Optional[str] = None
-    error_timestamp: Optional[datetime] = None
-    evaluation_result: Optional[Dict[str, Any]] = None
-    evaluation_total_cost: Optional[float] = None
-    created_at: Optional[datetime] = None
-    modified_at: Optional[datetime] = None
-
-    class Config:
-        extra = 'ignore'