deeprails 0.3.2__py3-none-any.whl → 1.2.0__py3-none-any.whl

deeprails/resources/evaluate.py

@@ -0,0 +1,334 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import List
+from typing_extensions import Literal
+
+import httpx
+
+from ..types import evaluate_create_params
+from .._types import Body, Omit, Query, Headers, NotGiven, omit, not_given
+from .._utils import maybe_transform, async_maybe_transform
+from .._compat import cached_property
+from .._resource import SyncAPIResource, AsyncAPIResource
+from .._response import (
+    to_raw_response_wrapper,
+    to_streamed_response_wrapper,
+    async_to_raw_response_wrapper,
+    async_to_streamed_response_wrapper,
+)
+from .._base_client import make_request_options
+from ..types.evaluation import Evaluation
+
+__all__ = ["EvaluateResource", "AsyncEvaluateResource"]
+
+
+class EvaluateResource(SyncAPIResource):
+    @cached_property
+    def with_raw_response(self) -> EvaluateResourceWithRawResponse:
+        """
+        This property can be used as a prefix for any HTTP method call to return
+        the raw response object instead of the parsed content.
+
+        For more information, see https://www.github.com/deeprails/deeprails-sdk-python#accessing-raw-response-data-eg-headers
+        """
+        return EvaluateResourceWithRawResponse(self)
+
+    @cached_property
+    def with_streaming_response(self) -> EvaluateResourceWithStreamingResponse:
+        """
+        An alternative to `.with_raw_response` that doesn't eagerly read the response body.
+
+        For more information, see https://www.github.com/deeprails/deeprails-sdk-python#with_streaming_response
+        """
+        return EvaluateResourceWithStreamingResponse(self)
+
+    def create(
+        self,
+        *,
+        model_input: evaluate_create_params.ModelInput,
+        model_output: str,
+        run_mode: Literal["precision_plus", "precision", "smart", "economy"],
+        guardrail_metrics: List[
+            Literal[
+                "correctness",
+                "completeness",
+                "instruction_adherence",
+                "context_adherence",
+                "ground_truth_adherence",
+                "comprehensive_safety",
+            ]
+        ]
+        | Omit = omit,
+        model_used: str | Omit = omit,
+        nametag: str | Omit = omit,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = not_given,
+    ) -> Evaluation:
+        """
+        Use this endpoint to evaluate a model's input and output pair against selected
+        guardrail metrics
+
+        Args:
+          model_input: A dictionary of inputs sent to the LLM to generate output. This must contain a
+              `user_prompt` field and an optional `context` field. Additional properties are
+              allowed.
+
+          model_output: Output generated by the LLM to be evaluated.
+
+          run_mode: Run mode for the evaluation. The run mode allows the user to optimize for speed,
+              accuracy, and cost by determining which models are used to evaluate the event.
+              Available run modes include `precision_plus`, `precision`, `smart`, and
+              `economy`. Defaults to `smart`.
+
+          guardrail_metrics: An array of guardrail metrics that the model input and output pair will be
+              evaluated on. For non-enterprise users, these will be limited to the allowed
+              guardrail metrics.
+
+          model_used: Model ID used to generate the output, like `gpt-4o` or `o3`.
+
+          nametag: An optional, user-defined tag for the evaluation.
+
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        return self._post(
+            "/evaluate",
+            body=maybe_transform(
+                {
+                    "model_input": model_input,
+                    "model_output": model_output,
+                    "run_mode": run_mode,
+                    "guardrail_metrics": guardrail_metrics,
+                    "model_used": model_used,
+                    "nametag": nametag,
+                },
+                evaluate_create_params.EvaluateCreateParams,
+            ),
+            options=make_request_options(
+                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
+            ),
+            cast_to=Evaluation,
+        )
+
+    def retrieve(
+        self,
+        eval_id: str,
+        *,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = not_given,
+    ) -> Evaluation:
+        """
+        Retrieve the evaluation record for a given evaluation ID.
+
+        Args:
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        if not eval_id:
+            raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
+        return self._get(
+            f"/evaluate/{eval_id}",
+            options=make_request_options(
+                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
+            ),
+            cast_to=Evaluation,
+        )
+
+
+class AsyncEvaluateResource(AsyncAPIResource):
+    @cached_property
+    def with_raw_response(self) -> AsyncEvaluateResourceWithRawResponse:
+        """
+        This property can be used as a prefix for any HTTP method call to return
+        the raw response object instead of the parsed content.
+
+        For more information, see https://www.github.com/deeprails/deeprails-sdk-python#accessing-raw-response-data-eg-headers
+        """
+        return AsyncEvaluateResourceWithRawResponse(self)
+
+    @cached_property
+    def with_streaming_response(self) -> AsyncEvaluateResourceWithStreamingResponse:
+        """
+        An alternative to `.with_raw_response` that doesn't eagerly read the response body.
+
+        For more information, see https://www.github.com/deeprails/deeprails-sdk-python#with_streaming_response
+        """
+        return AsyncEvaluateResourceWithStreamingResponse(self)
+
+    async def create(
+        self,
+        *,
+        model_input: evaluate_create_params.ModelInput,
+        model_output: str,
+        run_mode: Literal["precision_plus", "precision", "smart", "economy"],
+        guardrail_metrics: List[
+            Literal[
+                "correctness",
+                "completeness",
+                "instruction_adherence",
+                "context_adherence",
+                "ground_truth_adherence",
+                "comprehensive_safety",
+            ]
+        ]
+        | Omit = omit,
+        model_used: str | Omit = omit,
+        nametag: str | Omit = omit,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = not_given,
+    ) -> Evaluation:
+        """
+        Use this endpoint to evaluate a model's input and output pair against selected
+        guardrail metrics
+
+        Args:
+          model_input: A dictionary of inputs sent to the LLM to generate output. This must contain a
+              `user_prompt` field and an optional `context` field. Additional properties are
+              allowed.
+
+          model_output: Output generated by the LLM to be evaluated.
+
+          run_mode: Run mode for the evaluation. The run mode allows the user to optimize for speed,
+              accuracy, and cost by determining which models are used to evaluate the event.
+              Available run modes include `precision_plus`, `precision`, `smart`, and
+              `economy`. Defaults to `smart`.
+
+          guardrail_metrics: An array of guardrail metrics that the model input and output pair will be
+              evaluated on. For non-enterprise users, these will be limited to the allowed
+              guardrail metrics.
+
+          model_used: Model ID used to generate the output, like `gpt-4o` or `o3`.
+
+          nametag: An optional, user-defined tag for the evaluation.
+
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        return await self._post(
+            "/evaluate",
+            body=await async_maybe_transform(
+                {
+                    "model_input": model_input,
+                    "model_output": model_output,
+                    "run_mode": run_mode,
+                    "guardrail_metrics": guardrail_metrics,
+                    "model_used": model_used,
+                    "nametag": nametag,
+                },
+                evaluate_create_params.EvaluateCreateParams,
+            ),
+            options=make_request_options(
+                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
+            ),
+            cast_to=Evaluation,
+        )
+
+    async def retrieve(
+        self,
+        eval_id: str,
+        *,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = not_given,
+    ) -> Evaluation:
+        """
+        Retrieve the evaluation record for a given evaluation ID.
+
+        Args:
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        if not eval_id:
+            raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
+        return await self._get(
+            f"/evaluate/{eval_id}",
+            options=make_request_options(
+                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
+            ),
+            cast_to=Evaluation,
+        )
+
+
+class EvaluateResourceWithRawResponse:
+    def __init__(self, evaluate: EvaluateResource) -> None:
+        self._evaluate = evaluate
+
+        self.create = to_raw_response_wrapper(
+            evaluate.create,
+        )
+        self.retrieve = to_raw_response_wrapper(
+            evaluate.retrieve,
+        )
+
+
+class AsyncEvaluateResourceWithRawResponse:
+    def __init__(self, evaluate: AsyncEvaluateResource) -> None:
+        self._evaluate = evaluate
+
+        self.create = async_to_raw_response_wrapper(
+            evaluate.create,
+        )
+        self.retrieve = async_to_raw_response_wrapper(
+            evaluate.retrieve,
+        )
+
+
+class EvaluateResourceWithStreamingResponse:
+    def __init__(self, evaluate: EvaluateResource) -> None:
+        self._evaluate = evaluate
+
+        self.create = to_streamed_response_wrapper(
+            evaluate.create,
+        )
+        self.retrieve = to_streamed_response_wrapper(
+            evaluate.retrieve,
+        )
+
+
+class AsyncEvaluateResourceWithStreamingResponse:
+    def __init__(self, evaluate: AsyncEvaluateResource) -> None:
+        self._evaluate = evaluate
+
+        self.create = async_to_streamed_response_wrapper(
+            evaluate.create,
+        )
+        self.retrieve = async_to_streamed_response_wrapper(
+            evaluate.retrieve,
+        )