deeprails 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

deeprails/client.py

@@ -8,6 +8,7 @@ class DeepRails:
     """
     Python SDK client for the DeepRails API.
     """
+
     def __init__(self, token: str, base_url: str = "https://api.deeprails.com"):
         """
         Initializes the DeepRails client.
@@ -23,10 +24,11 @@ class DeepRails:
         self._headers = {
             "Authorization": f"Bearer {token}",
             "Content-Type": "application/json",
-            "User-Agent": "deeprails-python-sdk/0.2.0"
+            "User-Agent": "deeprails-python-sdk/0.3.0"
         }
         self._client = httpx.Client(base_url=self._base_url, headers=self._headers, timeout=30.0)
 
+
     def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
         """Helper method to make requests and handle API errors."""
         try:
@@ -43,6 +45,7 @@ class DeepRails:
         except httpx.RequestError as e:
             raise DeepRailsAPIError(status_code=500, error_detail=f"Request failed: {e}") from e
 
+
     def create_evaluation(
         self,
         *,
@@ -87,6 +90,7 @@ class DeepRails:
         response = self._request("POST", "/evaluate", json=json_payload)
         return EvaluationResponse.parse_obj(response.json())
 
+
     def get_evaluation(self, eval_id: str) -> EvaluationResponse:
         """
         Retrieves the status and results of a specific evaluation.
@@ -98,4 +102,184 @@ class DeepRails:
             An EvaluationResponse object with the full, up-to-date details of the evaluation.
         """
         response = self._request("GET", f"/evaluate/{eval_id}")
-        return EvaluationResponse.parse_obj(response.json())
+        return EvaluationResponse.parse_obj(response.json())
+
+
+    def create_monitor(
+        self,
+        *,
+        name: str,
+        description: Optional[str] = None
+    ) -> MonitorResponse:
+        """
+        Creates a new monitor for tracking AI responses.
+
+        Args:
+            name: A name for the monitor.
+            description: Optional description of the monitor's purpose.
+
+        Returns:
+            A MonitorResponse object with the details of the created monitor.
+        """
+        payload = {
+            "name": name,
+            "description": description
+        }
+        
+        # Remove None values
+        json_payload = {k: v for k, v in payload.items() if v is not None}
+
+        response = self._request("POST", "/monitor", json=json_payload)
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json:
+            return MonitorResponse.parse_obj(response_json["data"])
+        else:
+            return MonitorResponse.parse_obj(response_json)
+
+    def get_monitor(self, monitor_id: str) -> MonitorResponse:
+        """
+        Get details of a specific monitor.
+
+        Args:
+            monitor_id: The ID of the monitor to retrieve.
+
+        Returns:
+            A MonitorResponse object with the monitor details.
+        """
+        response = self._request("GET", f"/monitor/{monitor_id}")
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json:
+            return MonitorResponse.parse_obj(response_json["data"])
+        else:
+            return MonitorResponse.parse_obj(response_json)
+
+    def create_monitor_event(
+        self,
+        *,
+        monitor_id: str,
+        model_input: Dict[str, Any],
+        model_output: str,
+        guardrail_metrics: List[str],
+        model_used: Optional[str] = None,
+        run_mode: Optional[str] = None,
+        nametag: Optional[str] = None,
+        webhook: Optional[str] = None
+    ) -> MonitorEventResponse:
+        """
+        Creates a new event for a monitor.
+        
+        Args:
+            monitor_id: The ID of the monitor to create an event for.
+            model_input: A dictionary containing the inputs for the model.
+            model_output: The response generated by the model you are evaluating.
+            guardrail_metrics: A list of metrics to evaluate.
+            model_used: The name or identifier of the model being evaluated.
+            run_mode: The evaluation mode (e.g., "smart", "dev").
+            nametag: A user-defined name or tag for the event.
+            webhook: A URL to send a POST request to upon evaluation completion.
+        
+        Returns:
+            A MonitorEventResponse object with the details of the created event.
+        """
+        payload = {
+            "model_input": model_input,
+            "model_output": model_output,
+            "model_used": model_used,
+            "run_mode": run_mode,
+            "guardrail_metrics": guardrail_metrics,
+            "nametag": nametag,
+            "webhook": webhook,
+        }
+        
+        # Remove None values
+        json_payload = {k: v for k, v in payload.items() if v is not None}
+        
+        response = self._request("POST", f"/monitor/{monitor_id}/events", json=json_payload)
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json:
+            return MonitorEventResponse.parse_obj(response_json["data"])
+        else:
+            return MonitorEventResponse.parse_obj(response_json)
+
+    def get_monitor_events(
+        self, 
+        monitor_id: str, 
+        limit: int = 10, 
+        offset: int = 0
+    ) -> List[MonitorEventResponse]:
+        """
+        Retrieves events for a specific monitor.
+        
+        Args:
+            monitor_id: The ID of the monitor to get events for.
+            limit: Maximum number of events to return (default: 10).
+            offset: Offset for pagination (default: 0).
+            
+        Returns:
+            A list of MonitorEventResponse objects with details of the monitor events.
+        """
+        params = {
+            "limit": limit,
+            "offset": offset
+        }
+        
+        response = self._request("GET", f"/monitor/{monitor_id}/events", params=params)
+        response_json = response.json()
+        
+        # Handle DeepRails API response structure
+        if "data" in response_json and isinstance(response_json["data"], list):
+            return [MonitorEventResponse.parse_obj(event) for event in response_json["data"]]
+        else:
+            # Fallback if the response structure is unexpected
+            return []
+
+    def get_monitors(
+        self,
+        *,
+        page: int = 1,
+        limit: int = 20,
+        search: Optional[List[str]] = None,
+        monitor_status: Optional[List[str]] = None,
+        date_from: Optional[str] = None,
+        date_to: Optional[str] = None,
+        sort_by: str = "created_at",
+        sort_order: str = "desc"
+    ) -> MonitorListResponse:
+        """
+        Get a paginated list of monitors with optional filtering.
+        
+        Args:
+            page: Page number for pagination (default: 1)
+            limit: Number of items per page (default: 20, max: 100)
+            search: Optional list of free-text search terms
+            monitor_status: Optional list of monitor statuses ("active", "inactive", "all")
+            date_from: Optional filter for monitors from this date (ISO format)
+            date_to: Optional filter for monitors to this date (ISO format)
+            sort_by: Field to sort by (default: "created_at")
+            sort_order: Sort order (default: "desc")
+            
+        Returns:
+            A MonitorListResponse object containing monitors, pagination info, and applied filters.
+        """
+        params = {
+            "page": page,
+            "limit": limit,
+            "sort_by": sort_by,
+            "sort_order": sort_order,
+            "search": search,
+            "monitor_status": monitor_status,
+            "date_from": date_from,
+            "date_to": date_to
+        }
+        
+        # Remove None values
+        params = {k: v for k, v in params.items() if v is not None}
+        
+        response = self._request("GET", "/monitor", params=params)
+        return MonitorListResponse.parse_obj(response.json())

deeprails/schemas.py

@@ -27,4 +27,66 @@ class EvaluationResponse(BaseModel):
     modified_at: Optional[datetime] = None
 
     class Config:
-        extra = 'ignore'
+        extra = 'ignore'
+
+class MonitorResponse(BaseModel):
+    """Represents a monitor from the DeepRails API."""
+    monitor_id: str
+    user_id: str
+    name: str
+    description: Optional[str] = None
+    monitor_status: str
+    created_at: str
+    updated_at: str
+
+    class Config:
+        extra = 'ignore'
+
+class MonitorEventCreate(BaseModel):
+    """Model for creating a new monitor event."""
+    model_input: Dict[str, Any]
+    model_output: str
+    model_used: Optional[str] = None
+    run_mode: Optional[str] = None
+    guardrail_metrics: List[str]
+    nametag: Optional[str] = None
+    webhook: Optional[str] = None
+
+class MonitorEventResponse(BaseModel):
+    """Response model for a monitor event."""
+    event_id: str
+    monitor_id: str
+    evaluation_id: str
+    created_at: str
+
+    class Config:
+        extra = 'ignore'
+
+class PaginationInfo(BaseModel):
+    """Pagination information for list responses."""
+    page: int
+    limit: int
+    total_pages: int
+    total_count: int
+    has_next: bool
+    has_previous: bool
+
+class MonitorFiltersApplied(BaseModel):
+    """Information about which filters were applied to the monitor query."""
+    search: Optional[List[str]] = None
+    status: Optional[List[str]] = None
+    date_from: Optional[str] = None
+    date_to: Optional[str] = None
+    sort_by: Optional[str] = None
+    sort_order: Optional[str] = None
+
+class MonitorWithEventCountResponse(MonitorResponse):
+    """Monitor response with event count information."""
+    event_count: int
+    latest_event_modified_at: Optional[str] = None
+
+class MonitorListResponse(BaseModel):
+    """Response model for a paginated list of monitors."""
+    monitors: List[MonitorWithEventCountResponse]
+    pagination: PaginationInfo
+    filters_applied: MonitorFiltersApplied

{deeprails-0.2.0.dist-info → deeprails-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deeprails
-Version: 0.2.0
+Version: 0.3.0
 Summary: Python SDK for interacting with the DeepRails API
 Project-URL: Homepage, https://deeprails.com
 Project-URL: Documentation, https://docs.deeprails.com
@@ -55,12 +55,17 @@ client = DeepRails(token="YOUR_API_KEY")
 evaluation = client.create_evaluation(
     model_input={"user_prompt": "Prompt used to generate completion"},
     model_output="Generated output",
-    model_used="gpt-4o-mini (LLM used to generate completion)",
+    model_used="gpt-4o-mini",
     guardrail_metrics=["correctness", "completeness"]
 )
-
-# Print evaluation ID
 print(f"Evaluation created with ID: {evaluation.eval_id}")
+
+# Create a monitor
+monitor = client.create_monitor(
+    name="Production Assistant Monitor",
+    description="Tracking our production assistant quality"
+)
+print(f"Monitor created with ID: {monitor.monitor_id}")
 ```
 
 ## Features
@@ -69,6 +74,7 @@ print(f"Evaluation created with ID: {evaluation.eval_id}")
 - **Comprehensive Metrics**: Evaluate outputs on correctness, completeness, and more
 - **Real-time Progress**: Track evaluation progress in real-time
 - **Detailed Results**: Get detailed scores and rationales for each metric
+- **Continuous Monitoring**: Create monitors to track AI system performance over time
 
 ## Authentication
 
@@ -81,14 +87,16 @@ token = os.environ.get("DEEPRAILS_API_KEY")
 client = DeepRails(token=token)
 ```
 
-## Creating Evaluations
+## Evaluation Service
+
+### Creating Evaluations
 
 ```python
 try:
     evaluation = client.create_evaluation(
         model_input={"user_prompt": "Prompt used to generate completion"},
         model_output="Generated output",
-        model_used="gpt-4o-mini (LLM used to generate completion)",
+        model_used="gpt-4o-mini",
         guardrail_metrics=["correctness", "completeness"]
     )
     print(f"ID: {evaluation.eval_id}")
@@ -98,7 +106,7 @@ except Exception as e:
     print(f"Error: {e}")
 ```
 
-### Parameters
+#### Parameters
 
 - `model_input`: Dictionary containing the prompt and any context (must include `user_prompt`)
 - `model_output`: The generated output to evaluate
@@ -108,7 +116,7 @@ except Exception as e:
 - `nametag`: (Optional) Custom identifier for this evaluation
 - `webhook`: (Optional) URL to receive completion notifications
 
-## Retrieving Evaluations
+### Retrieving Evaluations
 
 ```python
 try:
@@ -126,13 +134,84 @@ except Exception as e:
     print(f"Error: {e}")
 ```
 
+## Monitor Service
+
+### Creating Monitors
+
+```python
+try:
+    # Create a monitor
+    monitor = client.create_monitor(
+        name="Production Chat Assistant Monitor",
+        description="Monitoring our production chatbot responses"
+    )
+    
+    print(f"Monitor created with ID: {monitor.monitor_id}")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+### Logging Monitor Events
+
+```python
+try:
+    # Add an event to the monitor
+    event = client.create_monitor_event(
+        monitor_id="mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
+        model_input={"user_prompt": "Tell me about renewable energy"},
+        model_output="Renewable energy comes from natural sources...",
+        model_used="gpt-4o-mini",
+        guardrail_metrics=["correctness", "completeness", "comprehensive_safety"]
+    )
+    
+    print(f"Monitor event created with ID: {event.event_id}")
+    print(f"Associated evaluation ID: {event.evaluation_id}")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
+### Retrieving Monitor Data
+
+```python
+try:
+    # Get monitor details
+    monitor = client.get_monitor("mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx")
+    print(f"Monitor name: {monitor.name}")
+    print(f"Status: {monitor.monitor_status}")
+    
+    # Get monitor events
+    events = client.get_monitor_events(
+        monitor_id="mon-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", 
+        limit=10
+    )
+    
+    for event in events:
+        print(f"Event ID: {event.event_id}")
+        print(f"Evaluation ID: {event.evaluation_id}")
+        
+    # List all monitors with filtering
+    monitors = client.get_monitors(
+        limit=5,
+        monitor_status=["active"],
+        sort_by="created_at",
+        sort_order="desc"
+    )
+    
+    print(f"Total monitors: {monitors.pagination.total_count}")
+    for m in monitors.monitors:
+        print(f"{m.name}: {m.event_count} events")
+except Exception as e:
+    print(f"Error: {e}")
+```
+
 ## Available Metrics
 
-- `correctness`: Evaluates factual accuracy of the output
-- `completeness`: Checks if the output addresses all aspects of the prompt
-- `harmfulness`: Detects potentially harmful content
-- `bias`: Identifies biased language or reasoning
-- And more...
+- `correctness`: Measures factual accuracy by evaluating whether each claim in the output is true and verifiable.
+- `completeness`: Assesses whether the response addresses all necessary parts of the prompt with sufficient detail and relevance.
+- `instruction_adherence`: Checks whether the AI followed the explicit instructions in the prompt and system directives.
+- `context_adherence`: Determines whether each factual claim is directly supported by the provided context.
+- `ground_truth_adherence`: Measures how closely the output matches a known correct answer (gold standard).
+- `comprehensive_safety`: Detects and categorizes safety violations across areas like PII, CBRN, hate speech, self-harm, and more.
 
 ## Error Handling
 

deeprails-0.3.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+deeprails/__init__.py,sha256=7ccTz1heYcCd3DIH3wmHc67FD6CUzM8_J4WmDeq0RZ0,29
+deeprails/client.py,sha256=Jv4Z4LR3g_eYO4L86sabOAibHJGV9p8naDOl1Yw-2L0,10406
+deeprails/exceptions.py,sha256=ipwFq4lROv7XpcBC5h9cGqPf6f68zeOMEyKPVy7H0co,405
+deeprails/schemas.py,sha256=XqBUFNEW4nwxm53YIUWVk4fEPsDGC6gzzXGL9lPuRAU,2870
+deeprails-0.3.0.dist-info/METADATA,sha256=qWrjZCxrzK1ar8I9Q8xzoDt_QB1kq_JWRLlJKeV_oOQ,8366
+deeprails-0.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+deeprails-0.3.0.dist-info/licenses/LICENSE,sha256=GsV7lN6fihCcDgkJbfs0rq1q9d6IyB0TFQ8HLKUpSXM,1077
+deeprails-0.3.0.dist-info/RECORD,,

deeprails-0.2.0.dist-info/RECORD

@@ -1,8 +0,0 @@
-deeprails/__init__.py,sha256=7ccTz1heYcCd3DIH3wmHc67FD6CUzM8_J4WmDeq0RZ0,29
-deeprails/client.py,sha256=gm3aq5FQj7qpLbWeyKkew3gbRI0l9QyGhvk7m0N7JxU,4004
-deeprails/exceptions.py,sha256=ipwFq4lROv7XpcBC5h9cGqPf6f68zeOMEyKPVy7H0co,405
-deeprails/schemas.py,sha256=pqZ-J7mIVdYe-q8sPSw07fe3H4YFCYb-PrshA_qU8RU,1113
-deeprails-0.2.0.dist-info/METADATA,sha256=Evweth9DLw-iHgbfZQdhfFG0i6MMXnm0GRkPyhscJe0,5759
-deeprails-0.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-deeprails-0.2.0.dist-info/licenses/LICENSE,sha256=GsV7lN6fihCcDgkJbfs0rq1q9d6IyB0TFQ8HLKUpSXM,1077
-deeprails-0.2.0.dist-info/RECORD,,