deeprails 1.11.0__tar.gz → 1.12.0__tar.gz

deeprails-1.12.0/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  ".": "1.12.0"
+}

{deeprails-1.11.0 → deeprails-1.12.0}/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 1.12.0 (2025-11-02)
+
+Full Changelog: [v1.11.0...v1.12.0](https://github.com/deeprails/deeprails-sdk-python/compare/v1.11.0...v1.12.0)
+
+### Features
+
+* **api:** restructure monitor service ([47cac0a](https://github.com/deeprails/deeprails-sdk-python/commit/47cac0a5568f3f73619ccb34e701901468dffbd0))
+
+
+### Chores
+
+* **internal/tests:** avoid race condition with implicit client cleanup ([0413192](https://github.com/deeprails/deeprails-sdk-python/commit/0413192c31932a21cd34cc4fe63669914a1d1de3))
+
 ## 1.11.0 (2025-10-30)
 
 Full Changelog: [v1.10.0...v1.11.0](https://github.com/deeprails/deeprails-sdk-python/compare/v1.10.0...v1.11.0)

{deeprails-1.11.0 → deeprails-1.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: deeprails
-Version: 1.11.0
+Version: 1.12.0
 Summary: The official Python library for the deeprails API
 Project-URL: Homepage, https://docs.deeprails.com/
 Project-URL: Repository, https://github.com/deeprails/deeprails-sdk-python

{deeprails-1.11.0 → deeprails-1.12.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "deeprails"
-version = "1.11.0"
+version = "1.12.0"
 description = "The official Python library for the deeprails API"
 dynamic = ["readme"]
 license = "Apache-2.0"

{deeprails-1.11.0 → deeprails-1.12.0}/src/deeprails/_version.py

@@ -1,4 +1,4 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 __title__ = "deeprails"
-__version__ = "1.11.0"  # x-release-please-version
+__version__ = "1.12.0"  # x-release-please-version

{deeprails-1.11.0 → deeprails-1.12.0}/src/deeprails/resources/monitor.py

@@ -8,7 +8,7 @@ from typing_extensions import Literal
 import httpx
 
 from ..types import monitor_create_params, monitor_update_params, monitor_retrieve_params, monitor_submit_event_params
-from .._types import Body, Omit, Query, Headers, NotGiven, omit, not_given
+from .._types import Body, Omit, Query, Headers, NotGiven, SequenceNotStr, omit, not_given
 from .._utils import maybe_transform, async_maybe_transform
 from .._compat import cached_property
 from .._resource import SyncAPIResource, AsyncAPIResource
@@ -49,8 +49,20 @@ class MonitorResource(SyncAPIResource):
     def create(
         self,
         *,
+        guardrail_metrics: List[
+            Literal[
+                "correctness",
+                "completeness",
+                "instruction_adherence",
+                "context_adherence",
+                "ground_truth_adherence",
+                "comprehensive_safety",
+            ]
+        ],
         name: str,
         description: str | Omit = omit,
+        file_search: SequenceNotStr[str] | Omit = omit,
+        web_search: bool | Omit = omit,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
         extra_headers: Headers | None = None,
@@ -63,10 +75,20 @@ class MonitorResource(SyncAPIResource):
         using guardrails
 
         Args:
+          guardrail_metrics: An array of guardrail metrics that the model input and output pair will be
+              evaluated on. For non-enterprise users, these will be limited to `correctness`,
+              `completeness`, `instruction_adherence`, `context_adherence`,
+              `ground_truth_adherence`, and/or `comprehensive_safety`.
+
           name: Name of the new monitor.
 
           description: Description of the new monitor.
 
+          file_search: An array of file IDs to search in the monitor's evaluations. Files must be
+              uploaded via the DeepRails API first.
+
+          web_search: Whether to enable web search for this monitor's evaluations. Defaults to false.
+
           extra_headers: Send extra headers
 
           extra_query: Add additional query parameters to the request
@@ -79,8 +101,11 @@ class MonitorResource(SyncAPIResource):
             "/monitor",
             body=maybe_transform(
                 {
+                    "guardrail_metrics": guardrail_metrics,
                     "name": name,
                     "description": description,
+                    "file_search": file_search,
+                    "web_search": web_search,
                 },
                 monitor_create_params.MonitorCreateParams,
             ),
@@ -187,19 +212,8 @@ class MonitorResource(SyncAPIResource):
         self,
         monitor_id: str,
         *,
-        guardrail_metrics: List[
-            Literal[
-                "correctness",
-                "completeness",
-                "instruction_adherence",
-                "context_adherence",
-                "ground_truth_adherence",
-                "comprehensive_safety",
-            ]
-        ],
         model_input: monitor_submit_event_params.ModelInput,
         model_output: str,
-        model_used: str | Omit = omit,
         nametag: str | Omit = omit,
         run_mode: Literal["precision_plus", "precision", "smart", "economy"] | Omit = omit,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
@@ -214,19 +228,12 @@ class MonitorResource(SyncAPIResource):
         evaluation
 
         Args:
-          guardrail_metrics: An array of guardrail metrics that the model input and output pair will be
-              evaluated on. For non-enterprise users, these will be limited to `correctness`,
-              `completeness`, `instruction_adherence`, `context_adherence`,
-              `ground_truth_adherence`, and/or `comprehensive_safety`.
-
           model_input: A dictionary of inputs sent to the LLM to generate output. The dictionary must
               contain at least a `user_prompt` field or a `system_prompt` field. For
               ground_truth_adherence guardrail metric, `ground_truth` should be provided.
 
           model_output: Output generated by the LLM to be evaluated.
 
-          model_used: Model ID used to generate the output, like `gpt-4o` or `o3`.
-
           nametag: An optional, user-defined tag for the event.
 
           run_mode: Run mode for the monitor event. The run mode allows the user to optimize for
@@ -248,10 +255,8 @@ class MonitorResource(SyncAPIResource):
             f"/monitor/{monitor_id}/events",
             body=maybe_transform(
                 {
-                    "guardrail_metrics": guardrail_metrics,
                     "model_input": model_input,
                     "model_output": model_output,
-                    "model_used": model_used,
                     "nametag": nametag,
                     "run_mode": run_mode,
                 },
@@ -287,8 +292,20 @@ class AsyncMonitorResource(AsyncAPIResource):
     async def create(
         self,
         *,
+        guardrail_metrics: List[
+            Literal[
+                "correctness",
+                "completeness",
+                "instruction_adherence",
+                "context_adherence",
+                "ground_truth_adherence",
+                "comprehensive_safety",
+            ]
+        ],
         name: str,
         description: str | Omit = omit,
+        file_search: SequenceNotStr[str] | Omit = omit,
+        web_search: bool | Omit = omit,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
         extra_headers: Headers | None = None,
@@ -301,10 +318,20 @@ class AsyncMonitorResource(AsyncAPIResource):
         using guardrails
 
         Args:
+          guardrail_metrics: An array of guardrail metrics that the model input and output pair will be
+              evaluated on. For non-enterprise users, these will be limited to `correctness`,
+              `completeness`, `instruction_adherence`, `context_adherence`,
+              `ground_truth_adherence`, and/or `comprehensive_safety`.
+
           name: Name of the new monitor.
 
           description: Description of the new monitor.
 
+          file_search: An array of file IDs to search in the monitor's evaluations. Files must be
+              uploaded via the DeepRails API first.
+
+          web_search: Whether to enable web search for this monitor's evaluations. Defaults to false.
+
           extra_headers: Send extra headers
 
           extra_query: Add additional query parameters to the request
@@ -317,8 +344,11 @@ class AsyncMonitorResource(AsyncAPIResource):
             "/monitor",
             body=await async_maybe_transform(
                 {
+                    "guardrail_metrics": guardrail_metrics,
                     "name": name,
                     "description": description,
+                    "file_search": file_search,
+                    "web_search": web_search,
                 },
                 monitor_create_params.MonitorCreateParams,
             ),
@@ -425,19 +455,8 @@ class AsyncMonitorResource(AsyncAPIResource):
         self,
         monitor_id: str,
         *,
-        guardrail_metrics: List[
-            Literal[
-                "correctness",
-                "completeness",
-                "instruction_adherence",
-                "context_adherence",
-                "ground_truth_adherence",
-                "comprehensive_safety",
-            ]
-        ],
         model_input: monitor_submit_event_params.ModelInput,
         model_output: str,
-        model_used: str | Omit = omit,
         nametag: str | Omit = omit,
         run_mode: Literal["precision_plus", "precision", "smart", "economy"] | Omit = omit,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
@@ -452,19 +471,12 @@ class AsyncMonitorResource(AsyncAPIResource):
         evaluation
 
         Args:
-          guardrail_metrics: An array of guardrail metrics that the model input and output pair will be
-              evaluated on. For non-enterprise users, these will be limited to `correctness`,
-              `completeness`, `instruction_adherence`, `context_adherence`,
-              `ground_truth_adherence`, and/or `comprehensive_safety`.
-
           model_input: A dictionary of inputs sent to the LLM to generate output. The dictionary must
               contain at least a `user_prompt` field or a `system_prompt` field. For
               ground_truth_adherence guardrail metric, `ground_truth` should be provided.
 
           model_output: Output generated by the LLM to be evaluated.
 
-          model_used: Model ID used to generate the output, like `gpt-4o` or `o3`.
-
           nametag: An optional, user-defined tag for the event.
 
           run_mode: Run mode for the monitor event. The run mode allows the user to optimize for
@@ -486,10 +498,8 @@ class AsyncMonitorResource(AsyncAPIResource):
             f"/monitor/{monitor_id}/events",
             body=await async_maybe_transform(
                 {
-                    "guardrail_metrics": guardrail_metrics,
                     "model_input": model_input,
                     "model_output": model_output,
-                    "model_used": model_used,
                     "nametag": nametag,
                     "run_mode": run_mode,
                 },

deeprails-1.12.0/src/deeprails/types/monitor_create_params.py

@@ -0,0 +1,46 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import List
+from typing_extensions import Literal, Required, TypedDict
+
+from .._types import SequenceNotStr
+
+__all__ = ["MonitorCreateParams"]
+
+
+class MonitorCreateParams(TypedDict, total=False):
+    guardrail_metrics: Required[
+        List[
+            Literal[
+                "correctness",
+                "completeness",
+                "instruction_adherence",
+                "context_adherence",
+                "ground_truth_adherence",
+                "comprehensive_safety",
+            ]
+        ]
+    ]
+    """
+    An array of guardrail metrics that the model input and output pair will be
+    evaluated on. For non-enterprise users, these will be limited to `correctness`,
+    `completeness`, `instruction_adherence`, `context_adherence`,
+    `ground_truth_adherence`, and/or `comprehensive_safety`.
+    """
+
+    name: Required[str]
+    """Name of the new monitor."""
+
+    description: str
+    """Description of the new monitor."""
+
+    file_search: SequenceNotStr[str]
+    """An array of file IDs to search in the monitor's evaluations.
+
+    Files must be uploaded via the DeepRails API first.
+    """
+
+    web_search: bool
+    """Whether to enable web search for this monitor's evaluations. Defaults to false."""

{deeprails-1.11.0 → deeprails-1.12.0}/src/deeprails/types/monitor_submit_event_params.py

@@ -2,32 +2,12 @@
 
 from __future__ import annotations
 
-from typing import List
 from typing_extensions import Literal, Required, TypedDict
 
 __all__ = ["MonitorSubmitEventParams", "ModelInput"]
 
 
 class MonitorSubmitEventParams(TypedDict, total=False):
-    guardrail_metrics: Required[
-        List[
-            Literal[
-                "correctness",
-                "completeness",
-                "instruction_adherence",
-                "context_adherence",
-                "ground_truth_adherence",
-                "comprehensive_safety",
-            ]
-        ]
-    ]
-    """
-    An array of guardrail metrics that the model input and output pair will be
-    evaluated on. For non-enterprise users, these will be limited to `correctness`,
-    `completeness`, `instruction_adherence`, `context_adherence`,
-    `ground_truth_adherence`, and/or `comprehensive_safety`.
-    """
-
     model_input: Required[ModelInput]
     """A dictionary of inputs sent to the LLM to generate output.
 
@@ -39,9 +19,6 @@ class MonitorSubmitEventParams(TypedDict, total=False):
     model_output: Required[str]
     """Output generated by the LLM to be evaluated."""
 
-    model_used: str
-    """Model ID used to generate the output, like `gpt-4o` or `o3`."""
-
     nametag: str
     """An optional, user-defined tag for the event."""
 

{deeprails-1.11.0 → deeprails-1.12.0}/tests/api_resources/test_monitor.py

@@ -25,6 +25,7 @@ class TestMonitor:
     @parametrize
     def test_method_create(self, client: Deeprails) -> None:
         monitor = client.monitor.create(
+            guardrail_metrics=["correctness"],
             name="name",
         )
         assert_matches_type(MonitorResponse, monitor, path=["response"])
@@ -33,8 +34,11 @@ class TestMonitor:
     @parametrize
     def test_method_create_with_all_params(self, client: Deeprails) -> None:
         monitor = client.monitor.create(
+            guardrail_metrics=["correctness"],
             name="name",
             description="description",
+            file_search=["string"],
+            web_search=True,
         )
         assert_matches_type(MonitorResponse, monitor, path=["response"])
 
@@ -42,6 +46,7 @@ class TestMonitor:
     @parametrize
     def test_raw_response_create(self, client: Deeprails) -> None:
         response = client.monitor.with_raw_response.create(
+            guardrail_metrics=["correctness"],
             name="name",
         )
 
@@ -54,6 +59,7 @@ class TestMonitor:
     @parametrize
     def test_streaming_response_create(self, client: Deeprails) -> None:
         with client.monitor.with_streaming_response.create(
+            guardrail_metrics=["correctness"],
             name="name",
         ) as response:
             assert not response.is_closed
@@ -173,7 +179,6 @@ class TestMonitor:
     def test_method_submit_event(self, client: Deeprails) -> None:
         monitor = client.monitor.submit_event(
             monitor_id="monitor_id",
-            guardrail_metrics=["correctness"],
             model_input={},
             model_output="model_output",
         )
@@ -184,14 +189,12 @@ class TestMonitor:
     def test_method_submit_event_with_all_params(self, client: Deeprails) -> None:
         monitor = client.monitor.submit_event(
             monitor_id="monitor_id",
-            guardrail_metrics=["correctness"],
             model_input={
                 "ground_truth": "ground_truth",
                 "system_prompt": "system_prompt",
                 "user_prompt": "user_prompt",
             },
             model_output="model_output",
-            model_used="model_used",
             nametag="nametag",
             run_mode="precision_plus",
         )
@@ -202,7 +205,6 @@ class TestMonitor:
     def test_raw_response_submit_event(self, client: Deeprails) -> None:
         response = client.monitor.with_raw_response.submit_event(
             monitor_id="monitor_id",
-            guardrail_metrics=["correctness"],
             model_input={},
             model_output="model_output",
         )
@@ -217,7 +219,6 @@ class TestMonitor:
     def test_streaming_response_submit_event(self, client: Deeprails) -> None:
         with client.monitor.with_streaming_response.submit_event(
             monitor_id="monitor_id",
-            guardrail_metrics=["correctness"],
             model_input={},
             model_output="model_output",
         ) as response:
@@ -235,7 +236,6 @@ class TestMonitor:
         with pytest.raises(ValueError, match=r"Expected a non-empty value for `monitor_id` but received ''"):
             client.monitor.with_raw_response.submit_event(
                 monitor_id="",
-                guardrail_metrics=["correctness"],
                 model_input={},
                 model_output="model_output",
             )
@@ -250,6 +250,7 @@ class TestAsyncMonitor:
     @parametrize
     async def test_method_create(self, async_client: AsyncDeeprails) -> None:
         monitor = await async_client.monitor.create(
+            guardrail_metrics=["correctness"],
             name="name",
         )
         assert_matches_type(MonitorResponse, monitor, path=["response"])
@@ -258,8 +259,11 @@ class TestAsyncMonitor:
     @parametrize
     async def test_method_create_with_all_params(self, async_client: AsyncDeeprails) -> None:
         monitor = await async_client.monitor.create(
+            guardrail_metrics=["correctness"],
             name="name",
             description="description",
+            file_search=["string"],
+            web_search=True,
         )
         assert_matches_type(MonitorResponse, monitor, path=["response"])
 
@@ -267,6 +271,7 @@ class TestAsyncMonitor:
     @parametrize
     async def test_raw_response_create(self, async_client: AsyncDeeprails) -> None:
         response = await async_client.monitor.with_raw_response.create(
+            guardrail_metrics=["correctness"],
             name="name",
         )
 
@@ -279,6 +284,7 @@ class TestAsyncMonitor:
     @parametrize
     async def test_streaming_response_create(self, async_client: AsyncDeeprails) -> None:
         async with async_client.monitor.with_streaming_response.create(
+            guardrail_metrics=["correctness"],
             name="name",
         ) as response:
             assert not response.is_closed
@@ -398,7 +404,6 @@ class TestAsyncMonitor:
     async def test_method_submit_event(self, async_client: AsyncDeeprails) -> None:
         monitor = await async_client.monitor.submit_event(
             monitor_id="monitor_id",
-            guardrail_metrics=["correctness"],
             model_input={},
             model_output="model_output",
         )
@@ -409,14 +414,12 @@ class TestAsyncMonitor:
     async def test_method_submit_event_with_all_params(self, async_client: AsyncDeeprails) -> None:
         monitor = await async_client.monitor.submit_event(
             monitor_id="monitor_id",
-            guardrail_metrics=["correctness"],
             model_input={
                 "ground_truth": "ground_truth",
                 "system_prompt": "system_prompt",
                 "user_prompt": "user_prompt",
             },
             model_output="model_output",
-            model_used="model_used",
             nametag="nametag",
             run_mode="precision_plus",
         )
@@ -427,7 +430,6 @@ class TestAsyncMonitor:
     async def test_raw_response_submit_event(self, async_client: AsyncDeeprails) -> None:
         response = await async_client.monitor.with_raw_response.submit_event(
             monitor_id="monitor_id",
-            guardrail_metrics=["correctness"],
             model_input={},
             model_output="model_output",
         )
@@ -442,7 +444,6 @@ class TestAsyncMonitor:
     async def test_streaming_response_submit_event(self, async_client: AsyncDeeprails) -> None:
         async with async_client.monitor.with_streaming_response.submit_event(
             monitor_id="monitor_id",
-            guardrail_metrics=["correctness"],
             model_input={},
             model_output="model_output",
         ) as response:
@@ -460,7 +461,6 @@ class TestAsyncMonitor:
         with pytest.raises(ValueError, match=r"Expected a non-empty value for `monitor_id` but received ''"):
             await async_client.monitor.with_raw_response.submit_event(
                 monitor_id="",
-                guardrail_metrics=["correctness"],
                 model_input={},
                 model_output="model_output",
             )