deeprails 1.9.0__tar.gz → 1.10.0__tar.gz

deeprails-1.10.0/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  ".": "1.10.0"
+}

{deeprails-1.9.0 → deeprails-1.10.0}/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 1.10.0 (2025-10-29)
+
+Full Changelog: [v1.9.0...v1.10.0](https://github.com/deeprails/deeprails-sdk-python/compare/v1.9.0...v1.10.0)
+
+### Features
+
+* **api:** remove evaluate api ([a3ddec1](https://github.com/deeprails/deeprails-sdk-python/commit/a3ddec1696eaa1247eea2a3bffd61e63d6537d30))
+* **api:** remove evaluate references ([23519a3](https://github.com/deeprails/deeprails-sdk-python/commit/23519a3349a254fe2fe24a51aeb59545d0820b70))
+
 ## 1.9.0 (2025-10-24)
 
 Full Changelog: [v1.8.0...v1.9.0](https://github.com/deeprails/deeprails-sdk-python/compare/v1.8.0...v1.9.0)

{deeprails-1.9.0 → deeprails-1.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: deeprails
-Version: 1.9.0
+Version: 1.10.0
 Summary: The official Python library for the deeprails API
 Project-URL: Homepage, https://docs.deeprails.com/
 Project-URL: Repository, https://github.com/deeprails/deeprails-sdk-python

{deeprails-1.9.0 → deeprails-1.10.0}/api.md

@@ -28,16 +28,3 @@ Methods:
 - <code title="get /monitor/{monitor_id}">client.monitor.<a href="./src/deeprails/resources/monitor.py">retrieve</a>(monitor_id, \*\*<a href="src/deeprails/types/monitor_retrieve_params.py">params</a>) -> <a href="./src/deeprails/types/monitor_detail_response.py">MonitorDetailResponse</a></code>
 - <code title="put /monitor/{monitor_id}">client.monitor.<a href="./src/deeprails/resources/monitor.py">update</a>(monitor_id, \*\*<a href="src/deeprails/types/monitor_update_params.py">params</a>) -> <a href="./src/deeprails/types/monitor_response.py">MonitorResponse</a></code>
 - <code title="post /monitor/{monitor_id}/events">client.monitor.<a href="./src/deeprails/resources/monitor.py">submit_event</a>(monitor_id, \*\*<a href="src/deeprails/types/monitor_submit_event_params.py">params</a>) -> <a href="./src/deeprails/types/monitor_event_response.py">MonitorEventResponse</a></code>
-
-# Evaluate
-
-Types:
-
-```python
-from deeprails.types import Evaluation
-```
-
-Methods:
-
-- <code title="post /evaluate">client.evaluate.<a href="./src/deeprails/resources/evaluate.py">create</a>(\*\*<a href="src/deeprails/types/evaluate_create_params.py">params</a>) -> <a href="./src/deeprails/types/evaluation.py">Evaluation</a></code>
-- <code title="get /evaluate/{eval_id}">client.evaluate.<a href="./src/deeprails/resources/evaluate.py">retrieve</a>(eval_id) -> <a href="./src/deeprails/types/evaluation.py">Evaluation</a></code>

{deeprails-1.9.0 → deeprails-1.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "deeprails"
-version = "1.9.0"
+version = "1.10.0"
 description = "The official Python library for the deeprails API"
 dynamic = ["readme"]
 license = "Apache-2.0"

{deeprails-1.9.0 → deeprails-1.10.0}/src/deeprails/_client.py

@@ -21,7 +21,7 @@ from ._types import (
 )
 from ._utils import is_given, get_async_library
 from ._version import __version__
-from .resources import defend, monitor, evaluate
+from .resources import defend, monitor
 from ._streaming import Stream as Stream, AsyncStream as AsyncStream
 from ._exceptions import APIStatusError, DeeprailsError
 from ._base_client import (
@@ -45,7 +45,6 @@ __all__ = [
 class Deeprails(SyncAPIClient):
     defend: defend.DefendResource
     monitor: monitor.MonitorResource
-    evaluate: evaluate.EvaluateResource
     with_raw_response: DeeprailsWithRawResponse
     with_streaming_response: DeeprailsWithStreamedResponse
 
@@ -105,7 +104,6 @@ class Deeprails(SyncAPIClient):
 
         self.defend = defend.DefendResource(self)
         self.monitor = monitor.MonitorResource(self)
-        self.evaluate = evaluate.EvaluateResource(self)
         self.with_raw_response = DeeprailsWithRawResponse(self)
         self.with_streaming_response = DeeprailsWithStreamedResponse(self)
 
@@ -217,7 +215,6 @@ class Deeprails(SyncAPIClient):
 class AsyncDeeprails(AsyncAPIClient):
     defend: defend.AsyncDefendResource
     monitor: monitor.AsyncMonitorResource
-    evaluate: evaluate.AsyncEvaluateResource
     with_raw_response: AsyncDeeprailsWithRawResponse
     with_streaming_response: AsyncDeeprailsWithStreamedResponse
 
@@ -277,7 +274,6 @@ class AsyncDeeprails(AsyncAPIClient):
 
         self.defend = defend.AsyncDefendResource(self)
         self.monitor = monitor.AsyncMonitorResource(self)
-        self.evaluate = evaluate.AsyncEvaluateResource(self)
         self.with_raw_response = AsyncDeeprailsWithRawResponse(self)
         self.with_streaming_response = AsyncDeeprailsWithStreamedResponse(self)
 
@@ -390,28 +386,24 @@ class DeeprailsWithRawResponse:
     def __init__(self, client: Deeprails) -> None:
         self.defend = defend.DefendResourceWithRawResponse(client.defend)
         self.monitor = monitor.MonitorResourceWithRawResponse(client.monitor)
-        self.evaluate = evaluate.EvaluateResourceWithRawResponse(client.evaluate)
 
 
 class AsyncDeeprailsWithRawResponse:
     def __init__(self, client: AsyncDeeprails) -> None:
         self.defend = defend.AsyncDefendResourceWithRawResponse(client.defend)
         self.monitor = monitor.AsyncMonitorResourceWithRawResponse(client.monitor)
-        self.evaluate = evaluate.AsyncEvaluateResourceWithRawResponse(client.evaluate)
 
 
 class DeeprailsWithStreamedResponse:
     def __init__(self, client: Deeprails) -> None:
         self.defend = defend.DefendResourceWithStreamingResponse(client.defend)
         self.monitor = monitor.MonitorResourceWithStreamingResponse(client.monitor)
-        self.evaluate = evaluate.EvaluateResourceWithStreamingResponse(client.evaluate)
 
 
 class AsyncDeeprailsWithStreamedResponse:
     def __init__(self, client: AsyncDeeprails) -> None:
         self.defend = defend.AsyncDefendResourceWithStreamingResponse(client.defend)
         self.monitor = monitor.AsyncMonitorResourceWithStreamingResponse(client.monitor)
-        self.evaluate = evaluate.AsyncEvaluateResourceWithStreamingResponse(client.evaluate)
 
 
 Client = Deeprails

{deeprails-1.9.0 → deeprails-1.10.0}/src/deeprails/_version.py

@@ -1,4 +1,4 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 __title__ = "deeprails"
-__version__ = "1.9.0"  # x-release-please-version
+__version__ = "1.10.0"  # x-release-please-version

{deeprails-1.9.0 → deeprails-1.10.0}/src/deeprails/resources/__init__.py

@@ -16,14 +16,6 @@ from .monitor import (
     MonitorResourceWithStreamingResponse,
     AsyncMonitorResourceWithStreamingResponse,
 )
-from .evaluate import (
-    EvaluateResource,
-    AsyncEvaluateResource,
-    EvaluateResourceWithRawResponse,
-    AsyncEvaluateResourceWithRawResponse,
-    EvaluateResourceWithStreamingResponse,
-    AsyncEvaluateResourceWithStreamingResponse,
-)
 
 __all__ = [
     "DefendResource",
@@ -38,10 +30,4 @@ __all__ = [
     "AsyncMonitorResourceWithRawResponse",
     "MonitorResourceWithStreamingResponse",
     "AsyncMonitorResourceWithStreamingResponse",
-    "EvaluateResource",
-    "AsyncEvaluateResource",
-    "EvaluateResourceWithRawResponse",
-    "AsyncEvaluateResourceWithRawResponse",
-    "EvaluateResourceWithStreamingResponse",
-    "AsyncEvaluateResourceWithStreamingResponse",
 ]

{deeprails-1.9.0 → deeprails-1.10.0}/src/deeprails/types/__init__.py

@@ -2,12 +2,10 @@
 
 from __future__ import annotations
 
-from .evaluation import Evaluation as Evaluation
 from .defend_response import DefendResponse as DefendResponse
 from .monitor_response import MonitorResponse as MonitorResponse
 from .monitor_create_params import MonitorCreateParams as MonitorCreateParams
 from .monitor_update_params import MonitorUpdateParams as MonitorUpdateParams
-from .evaluate_create_params import EvaluateCreateParams as EvaluateCreateParams
 from .monitor_event_response import MonitorEventResponse as MonitorEventResponse
 from .monitor_detail_response import MonitorDetailResponse as MonitorDetailResponse
 from .monitor_retrieve_params import MonitorRetrieveParams as MonitorRetrieveParams

deeprails-1.9.0/src/deeprails/types/evaluation.py → deeprails-1.10.0/src/deeprails/types/monitor_detail_response.py

@@ -8,10 +8,10 @@ from pydantic import Field as FieldInfo
 
 from .._models import BaseModel
 
-__all__ = ["Evaluation", "ModelInput"]
+__all__ = ["MonitorDetailResponse", "Evaluation", "EvaluationModelInput", "Stats"]
 
 
-class ModelInput(BaseModel):
+class EvaluationModelInput(BaseModel):
     ground_truth: Optional[str] = None
     """The ground truth for evaluating Ground Truth Adherence guardrail."""
 
@@ -29,7 +29,7 @@ class Evaluation(BaseModel):
     evaluation_status: Literal["in_progress", "completed", "canceled", "queued", "failed"]
     """Status of the evaluation."""
 
-    api_model_input: ModelInput = FieldInfo(alias="model_input")
+    api_model_input: EvaluationModelInput = FieldInfo(alias="model_input")
     """A dictionary of inputs sent to the LLM to generate output.
 
     The dictionary must contain at least a `user_prompt` field or a `system_prompt`
@@ -103,3 +103,60 @@ class Evaluation(BaseModel):
 
     start_timestamp: Optional[datetime] = None
     """The time the evaluation started in UTC."""
+
+
+class Stats(BaseModel):
+    completed_evaluations: Optional[int] = None
+    """Number of evaluations that completed successfully."""
+
+    failed_evaluations: Optional[int] = None
+    """Number of evaluations that failed."""
+
+    in_progress_evaluations: Optional[int] = None
+    """Number of evaluations currently in progress."""
+
+    queued_evaluations: Optional[int] = None
+    """Number of evaluations currently queued."""
+
+    total_evaluations: Optional[int] = None
+    """Total number of evaluations performed by this monitor."""
+
+
+class MonitorDetailResponse(BaseModel):
+    monitor_id: str
+    """A unique monitor ID."""
+
+    monitor_status: Literal["active", "inactive"]
+    """Status of the monitor.
+
+    Can be `active` or `inactive`. Inactive monitors no longer record and evaluate
+    events.
+    """
+
+    name: str
+    """Name of this monitor."""
+
+    created_at: Optional[datetime] = None
+    """The time the monitor was created in UTC."""
+
+    description: Optional[str] = None
+    """Description of this monitor."""
+
+    evaluations: Optional[List[Evaluation]] = None
+    """An array of all evaluations performed by this monitor.
+
+    Each one corresponds to a separate monitor event.
+    """
+
+    stats: Optional[Stats] = None
+    """
+    Contains five fields used for stats of this monitor: total evaluations,
+    completed evaluations, failed evaluations, queued evaluations, and in progress
+    evaluations.
+    """
+
+    updated_at: Optional[datetime] = None
+    """The most recent time the monitor was modified in UTC."""
+
+    user_id: Optional[str] = None
+    """User ID of the user who created the monitor."""

deeprails-1.9.0/.release-please-manifest.json

@@ -1,3 +0,0 @@
-{
-  ".": "1.9.0"
-}

deeprails-1.9.0/src/deeprails/resources/evaluate.py

@@ -1,334 +0,0 @@
-# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-from __future__ import annotations
-
-from typing import List
-from typing_extensions import Literal
-
-import httpx
-
-from ..types import evaluate_create_params
-from .._types import Body, Omit, Query, Headers, NotGiven, omit, not_given
-from .._utils import maybe_transform, async_maybe_transform
-from .._compat import cached_property
-from .._resource import SyncAPIResource, AsyncAPIResource
-from .._response import (
-    to_raw_response_wrapper,
-    to_streamed_response_wrapper,
-    async_to_raw_response_wrapper,
-    async_to_streamed_response_wrapper,
-)
-from .._base_client import make_request_options
-from ..types.evaluation import Evaluation
-
-__all__ = ["EvaluateResource", "AsyncEvaluateResource"]
-
-
-class EvaluateResource(SyncAPIResource):
-    @cached_property
-    def with_raw_response(self) -> EvaluateResourceWithRawResponse:
-        """
-        This property can be used as a prefix for any HTTP method call to return
-        the raw response object instead of the parsed content.
-
-        For more information, see https://www.github.com/deeprails/deeprails-sdk-python#accessing-raw-response-data-eg-headers
-        """
-        return EvaluateResourceWithRawResponse(self)
-
-    @cached_property
-    def with_streaming_response(self) -> EvaluateResourceWithStreamingResponse:
-        """
-        An alternative to `.with_raw_response` that doesn't eagerly read the response body.
-
-        For more information, see https://www.github.com/deeprails/deeprails-sdk-python#with_streaming_response
-        """
-        return EvaluateResourceWithStreamingResponse(self)
-
-    def create(
-        self,
-        *,
-        model_input: evaluate_create_params.ModelInput,
-        model_output: str,
-        run_mode: Literal["precision_plus", "precision", "smart", "economy"],
-        guardrail_metrics: List[
-            Literal[
-                "correctness",
-                "completeness",
-                "instruction_adherence",
-                "context_adherence",
-                "ground_truth_adherence",
-                "comprehensive_safety",
-            ]
-        ]
-        | Omit = omit,
-        model_used: str | Omit = omit,
-        nametag: str | Omit = omit,
-        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
-        # The extra values given here take precedence over values defined on the client or passed to this method.
-        extra_headers: Headers | None = None,
-        extra_query: Query | None = None,
-        extra_body: Body | None = None,
-        timeout: float | httpx.Timeout | None | NotGiven = not_given,
-    ) -> Evaluation:
-        """
-        Use this endpoint to evaluate a model's input and output pair against selected
-        guardrail metrics
-
-        Args:
-          model_input: A dictionary of inputs sent to the LLM to generate output. The dictionary must
-              contain at least a `user_prompt` field or a `system_prompt` field. For
-              ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-
-          model_output: Output generated by the LLM to be evaluated.
-
-          run_mode: Run mode for the evaluation. The run mode allows the user to optimize for speed,
-              accuracy, and cost by determining which models are used to evaluate the event.
-              Available run modes include `precision_plus`, `precision`, `smart`, and
-              `economy`. Defaults to `smart`.
-
-          guardrail_metrics: An array of guardrail metrics that the model input and output pair will be
-              evaluated on. For non-enterprise users, these will be limited to the allowed
-              guardrail metrics.
-
-          model_used: Model ID used to generate the output, like `gpt-4o` or `o3`.
-
-          nametag: An optional, user-defined tag for the evaluation.
-
-          extra_headers: Send extra headers
-
-          extra_query: Add additional query parameters to the request
-
-          extra_body: Add additional JSON properties to the request
-
-          timeout: Override the client-level default timeout for this request, in seconds
-        """
-        return self._post(
-            "/evaluate",
-            body=maybe_transform(
-                {
-                    "model_input": model_input,
-                    "model_output": model_output,
-                    "run_mode": run_mode,
-                    "guardrail_metrics": guardrail_metrics,
-                    "model_used": model_used,
-                    "nametag": nametag,
-                },
-                evaluate_create_params.EvaluateCreateParams,
-            ),
-            options=make_request_options(
-                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
-            ),
-            cast_to=Evaluation,
-        )
-
-    def retrieve(
-        self,
-        eval_id: str,
-        *,
-        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
-        # The extra values given here take precedence over values defined on the client or passed to this method.
-        extra_headers: Headers | None = None,
-        extra_query: Query | None = None,
-        extra_body: Body | None = None,
-        timeout: float | httpx.Timeout | None | NotGiven = not_given,
-    ) -> Evaluation:
-        """
-        Use this endpoint to retrieve the evaluation record for a given evaluation ID
-
-        Args:
-          extra_headers: Send extra headers
-
-          extra_query: Add additional query parameters to the request
-
-          extra_body: Add additional JSON properties to the request
-
-          timeout: Override the client-level default timeout for this request, in seconds
-        """
-        if not eval_id:
-            raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
-        return self._get(
-            f"/evaluate/{eval_id}",
-            options=make_request_options(
-                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
-            ),
-            cast_to=Evaluation,
-        )
-
-
-class AsyncEvaluateResource(AsyncAPIResource):
-    @cached_property
-    def with_raw_response(self) -> AsyncEvaluateResourceWithRawResponse:
-        """
-        This property can be used as a prefix for any HTTP method call to return
-        the raw response object instead of the parsed content.
-
-        For more information, see https://www.github.com/deeprails/deeprails-sdk-python#accessing-raw-response-data-eg-headers
-        """
-        return AsyncEvaluateResourceWithRawResponse(self)
-
-    @cached_property
-    def with_streaming_response(self) -> AsyncEvaluateResourceWithStreamingResponse:
-        """
-        An alternative to `.with_raw_response` that doesn't eagerly read the response body.
-
-        For more information, see https://www.github.com/deeprails/deeprails-sdk-python#with_streaming_response
-        """
-        return AsyncEvaluateResourceWithStreamingResponse(self)
-
-    async def create(
-        self,
-        *,
-        model_input: evaluate_create_params.ModelInput,
-        model_output: str,
-        run_mode: Literal["precision_plus", "precision", "smart", "economy"],
-        guardrail_metrics: List[
-            Literal[
-                "correctness",
-                "completeness",
-                "instruction_adherence",
-                "context_adherence",
-                "ground_truth_adherence",
-                "comprehensive_safety",
-            ]
-        ]
-        | Omit = omit,
-        model_used: str | Omit = omit,
-        nametag: str | Omit = omit,
-        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
-        # The extra values given here take precedence over values defined on the client or passed to this method.
-        extra_headers: Headers | None = None,
-        extra_query: Query | None = None,
-        extra_body: Body | None = None,
-        timeout: float | httpx.Timeout | None | NotGiven = not_given,
-    ) -> Evaluation:
-        """
-        Use this endpoint to evaluate a model's input and output pair against selected
-        guardrail metrics
-
-        Args:
-          model_input: A dictionary of inputs sent to the LLM to generate output. The dictionary must
-              contain at least a `user_prompt` field or a `system_prompt` field. For
-              ground_truth_adherence guardrail metric, `ground_truth` should be provided.
-
-          model_output: Output generated by the LLM to be evaluated.
-
-          run_mode: Run mode for the evaluation. The run mode allows the user to optimize for speed,
-              accuracy, and cost by determining which models are used to evaluate the event.
-              Available run modes include `precision_plus`, `precision`, `smart`, and
-              `economy`. Defaults to `smart`.
-
-          guardrail_metrics: An array of guardrail metrics that the model input and output pair will be
-              evaluated on. For non-enterprise users, these will be limited to the allowed
-              guardrail metrics.
-
-          model_used: Model ID used to generate the output, like `gpt-4o` or `o3`.
-
-          nametag: An optional, user-defined tag for the evaluation.
-
-          extra_headers: Send extra headers
-
-          extra_query: Add additional query parameters to the request
-
-          extra_body: Add additional JSON properties to the request
-
-          timeout: Override the client-level default timeout for this request, in seconds
-        """
-        return await self._post(
-            "/evaluate",
-            body=await async_maybe_transform(
-                {
-                    "model_input": model_input,
-                    "model_output": model_output,
-                    "run_mode": run_mode,
-                    "guardrail_metrics": guardrail_metrics,
-                    "model_used": model_used,
-                    "nametag": nametag,
-                },
-                evaluate_create_params.EvaluateCreateParams,
-            ),
-            options=make_request_options(
-                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
-            ),
-            cast_to=Evaluation,
-        )
-
-    async def retrieve(
-        self,
-        eval_id: str,
-        *,
-        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
-        # The extra values given here take precedence over values defined on the client or passed to this method.
-        extra_headers: Headers | None = None,
-        extra_query: Query | None = None,
-        extra_body: Body | None = None,
-        timeout: float | httpx.Timeout | None | NotGiven = not_given,
-    ) -> Evaluation:
-        """
-        Use this endpoint to retrieve the evaluation record for a given evaluation ID
-
-        Args:
-          extra_headers: Send extra headers
-
-          extra_query: Add additional query parameters to the request
-
-          extra_body: Add additional JSON properties to the request
-
-          timeout: Override the client-level default timeout for this request, in seconds
-        """
-        if not eval_id:
-            raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
-        return await self._get(
-            f"/evaluate/{eval_id}",
-            options=make_request_options(
-                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
-            ),
-            cast_to=Evaluation,
-        )
-
-
-class EvaluateResourceWithRawResponse:
-    def __init__(self, evaluate: EvaluateResource) -> None:
-        self._evaluate = evaluate
-
-        self.create = to_raw_response_wrapper(
-            evaluate.create,
-        )
-        self.retrieve = to_raw_response_wrapper(
-            evaluate.retrieve,
-        )
-
-
-class AsyncEvaluateResourceWithRawResponse:
-    def __init__(self, evaluate: AsyncEvaluateResource) -> None:
-        self._evaluate = evaluate
-
-        self.create = async_to_raw_response_wrapper(
-            evaluate.create,
-        )
-        self.retrieve = async_to_raw_response_wrapper(
-            evaluate.retrieve,
-        )
-
-
-class EvaluateResourceWithStreamingResponse:
-    def __init__(self, evaluate: EvaluateResource) -> None:
-        self._evaluate = evaluate
-
-        self.create = to_streamed_response_wrapper(
-            evaluate.create,
-        )
-        self.retrieve = to_streamed_response_wrapper(
-            evaluate.retrieve,
-        )
-
-
-class AsyncEvaluateResourceWithStreamingResponse:
-    def __init__(self, evaluate: AsyncEvaluateResource) -> None:
-        self._evaluate = evaluate
-
-        self.create = async_to_streamed_response_wrapper(
-            evaluate.create,
-        )
-        self.retrieve = async_to_streamed_response_wrapper(
-            evaluate.retrieve,
-        )

deeprails-1.9.0/src/deeprails/types/evaluate_create_params.py

@@ -1,63 +0,0 @@
-# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-from __future__ import annotations
-
-from typing import List
-from typing_extensions import Literal, Required, TypedDict
-
-__all__ = ["EvaluateCreateParams", "ModelInput"]
-
-
-class EvaluateCreateParams(TypedDict, total=False):
-    model_input: Required[ModelInput]
-    """A dictionary of inputs sent to the LLM to generate output.
-
-    The dictionary must contain at least a `user_prompt` field or a `system_prompt`
-    field. For ground_truth_adherence guardrail metric, `ground_truth` should be
-    provided.
-    """
-
-    model_output: Required[str]
-    """Output generated by the LLM to be evaluated."""
-
-    run_mode: Required[Literal["precision_plus", "precision", "smart", "economy"]]
-    """Run mode for the evaluation.
-
-    The run mode allows the user to optimize for speed, accuracy, and cost by
-    determining which models are used to evaluate the event. Available run modes
-    include `precision_plus`, `precision`, `smart`, and `economy`. Defaults to
-    `smart`.
-    """
-
-    guardrail_metrics: List[
-        Literal[
-            "correctness",
-            "completeness",
-            "instruction_adherence",
-            "context_adherence",
-            "ground_truth_adherence",
-            "comprehensive_safety",
-        ]
-    ]
-    """
-    An array of guardrail metrics that the model input and output pair will be
-    evaluated on. For non-enterprise users, these will be limited to the allowed
-    guardrail metrics.
-    """
-
-    model_used: str
-    """Model ID used to generate the output, like `gpt-4o` or `o3`."""
-
-    nametag: str
-    """An optional, user-defined tag for the evaluation."""
-
-
-class ModelInput(TypedDict, total=False):
-    ground_truth: str
-    """The ground truth for evaluating Ground Truth Adherence guardrail."""
-
-    system_prompt: str
-    """The system prompt used to generate the output."""
-
-    user_prompt: str
-    """The user prompt used to generate the output."""

deeprails-1.9.0/src/deeprails/types/monitor_detail_response.py

@@ -1,67 +0,0 @@
-# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-from typing import List, Optional
-from datetime import datetime
-from typing_extensions import Literal
-
-from .._models import BaseModel
-from .evaluation import Evaluation
-
-__all__ = ["MonitorDetailResponse", "Stats"]
-
-
-class Stats(BaseModel):
-    completed_evaluations: Optional[int] = None
-    """Number of evaluations that completed successfully."""
-
-    failed_evaluations: Optional[int] = None
-    """Number of evaluations that failed."""
-
-    in_progress_evaluations: Optional[int] = None
-    """Number of evaluations currently in progress."""
-
-    queued_evaluations: Optional[int] = None
-    """Number of evaluations currently queued."""
-
-    total_evaluations: Optional[int] = None
-    """Total number of evaluations performed by this monitor."""
-
-
-class MonitorDetailResponse(BaseModel):
-    monitor_id: str
-    """A unique monitor ID."""
-
-    monitor_status: Literal["active", "inactive"]
-    """Status of the monitor.
-
-    Can be `active` or `inactive`. Inactive monitors no longer record and evaluate
-    events.
-    """
-
-    name: str
-    """Name of this monitor."""
-
-    created_at: Optional[datetime] = None
-    """The time the monitor was created in UTC."""
-
-    description: Optional[str] = None
-    """Description of this monitor."""
-
-    evaluations: Optional[List[Evaluation]] = None
-    """An array of all evaluations performed by this monitor.
-
-    Each one corresponds to a separate monitor event.
-    """
-
-    stats: Optional[Stats] = None
-    """
-    Contains five fields used for stats of this monitor: total evaluations,
-    completed evaluations, failed evaluations, queued evaluations, and in progress
-    evaluations.
-    """
-
-    updated_at: Optional[datetime] = None
-    """The most recent time the monitor was modified in UTC."""
-
-    user_id: Optional[str] = None
-    """User ID of the user who created the monitor."""

deeprails-1.9.0/tests/api_resources/test_evaluate.py

@@ -1,222 +0,0 @@
-# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
-from __future__ import annotations
-
-import os
-from typing import Any, cast
-
-import pytest
-
-from deeprails import Deeprails, AsyncDeeprails
-from tests.utils import assert_matches_type
-from deeprails.types import Evaluation
-
-base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010")
-
-
-class TestEvaluate:
-    parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"])
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    def test_method_create(self, client: Deeprails) -> None:
-        evaluate = client.evaluate.create(
-            model_input={},
-            model_output="model_output",
-            run_mode="precision_plus",
-        )
-        assert_matches_type(Evaluation, evaluate, path=["response"])
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    def test_method_create_with_all_params(self, client: Deeprails) -> None:
-        evaluate = client.evaluate.create(
-            model_input={
-                "ground_truth": "ground_truth",
-                "system_prompt": "system_prompt",
-                "user_prompt": "user_prompt",
-            },
-            model_output="model_output",
-            run_mode="precision_plus",
-            guardrail_metrics=["correctness"],
-            model_used="model_used",
-            nametag="nametag",
-        )
-        assert_matches_type(Evaluation, evaluate, path=["response"])
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    def test_raw_response_create(self, client: Deeprails) -> None:
-        response = client.evaluate.with_raw_response.create(
-            model_input={},
-            model_output="model_output",
-            run_mode="precision_plus",
-        )
-
-        assert response.is_closed is True
-        assert response.http_request.headers.get("X-Stainless-Lang") == "python"
-        evaluate = response.parse()
-        assert_matches_type(Evaluation, evaluate, path=["response"])
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    def test_streaming_response_create(self, client: Deeprails) -> None:
-        with client.evaluate.with_streaming_response.create(
-            model_input={},
-            model_output="model_output",
-            run_mode="precision_plus",
-        ) as response:
-            assert not response.is_closed
-            assert response.http_request.headers.get("X-Stainless-Lang") == "python"
-
-            evaluate = response.parse()
-            assert_matches_type(Evaluation, evaluate, path=["response"])
-
-        assert cast(Any, response.is_closed) is True
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    def test_method_retrieve(self, client: Deeprails) -> None:
-        evaluate = client.evaluate.retrieve(
-            "eval_id",
-        )
-        assert_matches_type(Evaluation, evaluate, path=["response"])
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    def test_raw_response_retrieve(self, client: Deeprails) -> None:
-        response = client.evaluate.with_raw_response.retrieve(
-            "eval_id",
-        )
-
-        assert response.is_closed is True
-        assert response.http_request.headers.get("X-Stainless-Lang") == "python"
-        evaluate = response.parse()
-        assert_matches_type(Evaluation, evaluate, path=["response"])
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    def test_streaming_response_retrieve(self, client: Deeprails) -> None:
-        with client.evaluate.with_streaming_response.retrieve(
-            "eval_id",
-        ) as response:
-            assert not response.is_closed
-            assert response.http_request.headers.get("X-Stainless-Lang") == "python"
-
-            evaluate = response.parse()
-            assert_matches_type(Evaluation, evaluate, path=["response"])
-
-        assert cast(Any, response.is_closed) is True
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    def test_path_params_retrieve(self, client: Deeprails) -> None:
-        with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
-            client.evaluate.with_raw_response.retrieve(
-                "",
-            )
-
-
-class TestAsyncEvaluate:
-    parametrize = pytest.mark.parametrize(
-        "async_client", [False, True, {"http_client": "aiohttp"}], indirect=True, ids=["loose", "strict", "aiohttp"]
-    )
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    async def test_method_create(self, async_client: AsyncDeeprails) -> None:
-        evaluate = await async_client.evaluate.create(
-            model_input={},
-            model_output="model_output",
-            run_mode="precision_plus",
-        )
-        assert_matches_type(Evaluation, evaluate, path=["response"])
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    async def test_method_create_with_all_params(self, async_client: AsyncDeeprails) -> None:
-        evaluate = await async_client.evaluate.create(
-            model_input={
-                "ground_truth": "ground_truth",
-                "system_prompt": "system_prompt",
-                "user_prompt": "user_prompt",
-            },
-            model_output="model_output",
-            run_mode="precision_plus",
-            guardrail_metrics=["correctness"],
-            model_used="model_used",
-            nametag="nametag",
-        )
-        assert_matches_type(Evaluation, evaluate, path=["response"])
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    async def test_raw_response_create(self, async_client: AsyncDeeprails) -> None:
-        response = await async_client.evaluate.with_raw_response.create(
-            model_input={},
-            model_output="model_output",
-            run_mode="precision_plus",
-        )
-
-        assert response.is_closed is True
-        assert response.http_request.headers.get("X-Stainless-Lang") == "python"
-        evaluate = await response.parse()
-        assert_matches_type(Evaluation, evaluate, path=["response"])
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    async def test_streaming_response_create(self, async_client: AsyncDeeprails) -> None:
-        async with async_client.evaluate.with_streaming_response.create(
-            model_input={},
-            model_output="model_output",
-            run_mode="precision_plus",
-        ) as response:
-            assert not response.is_closed
-            assert response.http_request.headers.get("X-Stainless-Lang") == "python"
-
-            evaluate = await response.parse()
-            assert_matches_type(Evaluation, evaluate, path=["response"])
-
-        assert cast(Any, response.is_closed) is True
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    async def test_method_retrieve(self, async_client: AsyncDeeprails) -> None:
-        evaluate = await async_client.evaluate.retrieve(
-            "eval_id",
-        )
-        assert_matches_type(Evaluation, evaluate, path=["response"])
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    async def test_raw_response_retrieve(self, async_client: AsyncDeeprails) -> None:
-        response = await async_client.evaluate.with_raw_response.retrieve(
-            "eval_id",
-        )
-
-        assert response.is_closed is True
-        assert response.http_request.headers.get("X-Stainless-Lang") == "python"
-        evaluate = await response.parse()
-        assert_matches_type(Evaluation, evaluate, path=["response"])
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    async def test_streaming_response_retrieve(self, async_client: AsyncDeeprails) -> None:
-        async with async_client.evaluate.with_streaming_response.retrieve(
-            "eval_id",
-        ) as response:
-            assert not response.is_closed
-            assert response.http_request.headers.get("X-Stainless-Lang") == "python"
-
-            evaluate = await response.parse()
-            assert_matches_type(Evaluation, evaluate, path=["response"])
-
-        assert cast(Any, response.is_closed) is True
-
-    @pytest.mark.skip(reason="Prism tests are disabled")
-    @parametrize
-    async def test_path_params_retrieve(self, async_client: AsyncDeeprails) -> None:
-        with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
-            await async_client.evaluate.with_raw_response.retrieve(
-                "",
-            )