runloop-api-client 0.18.0__tar.gz → 0.19.0__tar.gz

runloop_api_client-0.19.0/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  ".": "0.19.0"
+}

{runloop_api_client-0.18.0 → runloop_api_client-0.19.0}/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.19.0 (2025-02-03)
+
+Full Changelog: [v0.18.0...v0.19.0](https://github.com/runloopai/api-client-python/compare/v0.18.0...v0.19.0)
+
+### Features
+
+* add helper methods for scenarios ([551fcfc](https://github.com/runloopai/api-client-python/commit/551fcfc1dc5b8b67c0c6179d49c3eef2c0a38676))
+* **api:** api update ([#520](https://github.com/runloopai/api-client-python/issues/520)) ([750b4ba](https://github.com/runloopai/api-client-python/commit/750b4ba51cc0c848258113af8bf882ceb679bf9e))
+
 ## 0.18.0 (2025-02-01)
 
 Full Changelog: [v0.17.0...v0.18.0](https://github.com/runloopai/api-client-python/compare/v0.17.0...v0.18.0)

{runloop_api_client-0.18.0 → runloop_api_client-0.19.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runloop_api_client
-Version: 0.18.0
+Version: 0.19.0
 Summary: The official Python library for the runloop API
 Project-URL: Homepage, https://github.com/runloopai/api-client-python
 Project-URL: Repository, https://github.com/runloopai/api-client-python

{runloop_api_client-0.18.0 → runloop_api_client-0.19.0}/api.md

@@ -24,6 +24,7 @@ Methods:
 - <code title="post /v1/benchmarks">client.benchmarks.<a href="./src/runloop_api_client/resources/benchmarks/benchmarks.py">create</a>(\*\*<a href="src/runloop_api_client/types/benchmark_create_params.py">params</a>) -> <a href="./src/runloop_api_client/types/benchmark_view.py">BenchmarkView</a></code>
 - <code title="get /v1/benchmarks/{id}">client.benchmarks.<a href="./src/runloop_api_client/resources/benchmarks/benchmarks.py">retrieve</a>(id) -> <a href="./src/runloop_api_client/types/benchmark_view.py">BenchmarkView</a></code>
 - <code title="get /v1/benchmarks">client.benchmarks.<a href="./src/runloop_api_client/resources/benchmarks/benchmarks.py">list</a>(\*\*<a href="src/runloop_api_client/types/benchmark_list_params.py">params</a>) -> <a href="./src/runloop_api_client/types/benchmark_view.py">SyncBenchmarksCursorIDPage[BenchmarkView]</a></code>
+- <code title="get /v1/benchmarks/list_public">client.benchmarks.<a href="./src/runloop_api_client/resources/benchmarks/benchmarks.py">list_public</a>(\*\*<a href="src/runloop_api_client/types/benchmark_list_public_params.py">params</a>) -> <a href="./src/runloop_api_client/types/benchmark_view.py">SyncBenchmarksCursorIDPage[BenchmarkView]</a></code>
 - <code title="post /v1/benchmarks/start_run">client.benchmarks.<a href="./src/runloop_api_client/resources/benchmarks/benchmarks.py">start_run</a>(\*\*<a href="src/runloop_api_client/types/benchmark_start_run_params.py">params</a>) -> <a href="./src/runloop_api_client/types/benchmark_run_view.py">BenchmarkRunView</a></code>
 
 ## Runs
@@ -261,6 +262,7 @@ Methods:
 - <code title="post /v1/scenarios">client.scenarios.<a href="./src/runloop_api_client/resources/scenarios/scenarios.py">create</a>(\*\*<a href="src/runloop_api_client/types/scenario_create_params.py">params</a>) -> <a href="./src/runloop_api_client/types/scenario_view.py">ScenarioView</a></code>
 - <code title="get /v1/scenarios/{id}">client.scenarios.<a href="./src/runloop_api_client/resources/scenarios/scenarios.py">retrieve</a>(id) -> <a href="./src/runloop_api_client/types/scenario_view.py">ScenarioView</a></code>
 - <code title="get /v1/scenarios">client.scenarios.<a href="./src/runloop_api_client/resources/scenarios/scenarios.py">list</a>(\*\*<a href="src/runloop_api_client/types/scenario_list_params.py">params</a>) -> <a href="./src/runloop_api_client/types/scenario_view.py">SyncScenariosCursorIDPage[ScenarioView]</a></code>
+- <code title="get /v1/scenarios/list_public">client.scenarios.<a href="./src/runloop_api_client/resources/scenarios/scenarios.py">list_public</a>(\*\*<a href="src/runloop_api_client/types/scenario_list_public_params.py">params</a>) -> <a href="./src/runloop_api_client/types/scenario_view.py">SyncScenariosCursorIDPage[ScenarioView]</a></code>
 - <code title="post /v1/scenarios/start_run">client.scenarios.<a href="./src/runloop_api_client/resources/scenarios/scenarios.py">start_run</a>(\*\*<a href="src/runloop_api_client/types/scenario_start_run_params.py">params</a>) -> <a href="./src/runloop_api_client/types/scenario_run_view.py">ScenarioRunView</a></code>
 
 ## Runs

{runloop_api_client-0.18.0 → runloop_api_client-0.19.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runloop_api_client"
-version = "0.18.0"
+version = "0.19.0"
 description = "The official Python library for the runloop API"
 dynamic = ["readme"]
 license = "MIT"

{runloop_api_client-0.18.0 → runloop_api_client-0.19.0}/src/runloop_api_client/_version.py

@@ -1,4 +1,4 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 __title__ = "runloop_api_client"
-__version__ = "0.18.0"  # x-release-please-version
+__version__ = "0.19.0"  # x-release-please-version

{runloop_api_client-0.18.0 → runloop_api_client-0.19.0}/src/runloop_api_client/resources/benchmarks/benchmarks.py

@@ -14,7 +14,12 @@ from .runs import (
     RunsResourceWithStreamingResponse,
     AsyncRunsResourceWithStreamingResponse,
 )
-from ...types import benchmark_list_params, benchmark_create_params, benchmark_start_run_params
+from ...types import (
+    benchmark_list_params,
+    benchmark_create_params,
+    benchmark_start_run_params,
+    benchmark_list_public_params,
+)
 from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven
 from ..._utils import (
     maybe_transform,
@@ -147,7 +152,6 @@ class BenchmarksResource(SyncAPIResource):
         self,
         *,
         limit: int | NotGiven = NOT_GIVEN,
-        public: bool | NotGiven = NOT_GIVEN,
         starting_after: str | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -162,9 +166,6 @@ class BenchmarksResource(SyncAPIResource):
         Args:
           limit: The limit of items to return. Default is 20.
 
-          public: List public benchmarks, e.g. SWE-bench. Defaults to false, i.e. only
-              user-defined benchmarks are listed.
-
           starting_after: Load the next page of data starting after the item with the given ID.
 
           extra_headers: Send extra headers
@@ -186,7 +187,6 @@ class BenchmarksResource(SyncAPIResource):
                 query=maybe_transform(
                     {
                         "limit": limit,
-                        "public": public,
                         "starting_after": starting_after,
                     },
                     benchmark_list_params.BenchmarkListParams,
@@ -195,6 +195,53 @@ class BenchmarksResource(SyncAPIResource):
             model=BenchmarkView,
         )
 
+    def list_public(
+        self,
+        *,
+        limit: int | NotGiven = NOT_GIVEN,
+        starting_after: str | NotGiven = NOT_GIVEN,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> SyncBenchmarksCursorIDPage[BenchmarkView]:
+        """
+        List all public benchmarks matching filter.
+
+        Args:
+          limit: The limit of items to return. Default is 20.
+
+          starting_after: Load the next page of data starting after the item with the given ID.
+
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        return self._get_api_list(
+            "/v1/benchmarks/list_public",
+            page=SyncBenchmarksCursorIDPage[BenchmarkView],
+            options=make_request_options(
+                extra_headers=extra_headers,
+                extra_query=extra_query,
+                extra_body=extra_body,
+                timeout=timeout,
+                query=maybe_transform(
+                    {
+                        "limit": limit,
+                        "starting_after": starting_after,
+                    },
+                    benchmark_list_public_params.BenchmarkListPublicParams,
+                ),
+            ),
+            model=BenchmarkView,
+        )
+
     def start_run(
         self,
         *,
@@ -357,7 +404,6 @@ class AsyncBenchmarksResource(AsyncAPIResource):
         self,
         *,
         limit: int | NotGiven = NOT_GIVEN,
-        public: bool | NotGiven = NOT_GIVEN,
         starting_after: str | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -372,9 +418,6 @@ class AsyncBenchmarksResource(AsyncAPIResource):
         Args:
           limit: The limit of items to return. Default is 20.
 
-          public: List public benchmarks, e.g. SWE-bench. Defaults to false, i.e. only
-              user-defined benchmarks are listed.
-
           starting_after: Load the next page of data starting after the item with the given ID.
 
           extra_headers: Send extra headers
@@ -396,7 +439,6 @@ class AsyncBenchmarksResource(AsyncAPIResource):
                 query=maybe_transform(
                     {
                         "limit": limit,
-                        "public": public,
                         "starting_after": starting_after,
                     },
                     benchmark_list_params.BenchmarkListParams,
@@ -405,6 +447,53 @@ class AsyncBenchmarksResource(AsyncAPIResource):
             model=BenchmarkView,
         )
 
+    def list_public(
+        self,
+        *,
+        limit: int | NotGiven = NOT_GIVEN,
+        starting_after: str | NotGiven = NOT_GIVEN,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> AsyncPaginator[BenchmarkView, AsyncBenchmarksCursorIDPage[BenchmarkView]]:
+        """
+        List all public benchmarks matching filter.
+
+        Args:
+          limit: The limit of items to return. Default is 20.
+
+          starting_after: Load the next page of data starting after the item with the given ID.
+
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        return self._get_api_list(
+            "/v1/benchmarks/list_public",
+            page=AsyncBenchmarksCursorIDPage[BenchmarkView],
+            options=make_request_options(
+                extra_headers=extra_headers,
+                extra_query=extra_query,
+                extra_body=extra_body,
+                timeout=timeout,
+                query=maybe_transform(
+                    {
+                        "limit": limit,
+                        "starting_after": starting_after,
+                    },
+                    benchmark_list_public_params.BenchmarkListPublicParams,
+                ),
+            ),
+            model=BenchmarkView,
+        )
+
     async def start_run(
         self,
         *,
@@ -469,6 +558,9 @@ class BenchmarksResourceWithRawResponse:
         self.list = to_raw_response_wrapper(
             benchmarks.list,
         )
+        self.list_public = to_raw_response_wrapper(
+            benchmarks.list_public,
+        )
         self.start_run = to_raw_response_wrapper(
             benchmarks.start_run,
         )
@@ -491,6 +583,9 @@ class AsyncBenchmarksResourceWithRawResponse:
         self.list = async_to_raw_response_wrapper(
             benchmarks.list,
         )
+        self.list_public = async_to_raw_response_wrapper(
+            benchmarks.list_public,
+        )
         self.start_run = async_to_raw_response_wrapper(
             benchmarks.start_run,
         )
@@ -513,6 +608,9 @@ class BenchmarksResourceWithStreamingResponse:
         self.list = to_streamed_response_wrapper(
             benchmarks.list,
         )
+        self.list_public = to_streamed_response_wrapper(
+            benchmarks.list_public,
+        )
         self.start_run = to_streamed_response_wrapper(
             benchmarks.start_run,
         )
@@ -535,6 +633,9 @@ class AsyncBenchmarksResourceWithStreamingResponse:
         self.list = async_to_streamed_response_wrapper(
             benchmarks.list,
         )
+        self.list_public = async_to_streamed_response_wrapper(
+            benchmarks.list_public,
+        )
         self.start_run = async_to_streamed_response_wrapper(
             benchmarks.start_run,
         )

{runloop_api_client-0.18.0 → runloop_api_client-0.19.0}/src/runloop_api_client/resources/scenarios/runs.py

@@ -15,8 +15,11 @@ from ..._response import (
     async_to_streamed_response_wrapper,
 )
 from ...pagination import SyncBenchmarkRunsCursorIDPage, AsyncBenchmarkRunsCursorIDPage
+from ..._exceptions import RunloopError
+from ...lib.polling import PollingConfig, poll_until
 from ..._base_client import AsyncPaginator, make_request_options
 from ...types.scenarios import run_list_params
+from ...lib.polling_async import async_poll_until
 from ...types.scenario_run_view import ScenarioRunView
 
 __all__ = ["RunsResource", "AsyncRunsResource"]
@@ -206,6 +209,147 @@ class RunsResource(SyncAPIResource):
             cast_to=ScenarioRunView,
         )
 
+    def await_scored(
+        self,
+        id: str,
+        *,
+        polling_config: PollingConfig | None = None,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> ScenarioRunView:
+        """Wait for a scenario run to be scored.
+        
+        Args:
+            id: The ID of the scenario run to wait for
+            polling_config: Optional polling configuration
+            extra_headers: Send extra headers
+            extra_query: Add additional query parameters to the request
+            extra_body: Add additional JSON properties to the request
+            timeout: Override the client-level default timeout for this request, in seconds
+
+        Returns:
+            The scored scenario run
+
+        Raises:
+            PollingTimeout: If polling times out before scenario run is scored
+            RunloopError: If scenario run enters a non-scored terminal state
+        """
+        def retrieve_run() -> ScenarioRunView:
+            return self.retrieve(
+                id,
+                extra_headers=extra_headers,
+                extra_query=extra_query,
+                extra_body=extra_body,
+                timeout=timeout
+            )
+
+        def is_done_scoring(run: ScenarioRunView) -> bool:
+            return run.state not in ["scoring"]
+
+        run = poll_until(retrieve_run, is_done_scoring, polling_config)
+
+        if run.state != "scored":
+            raise RunloopError(
+                f"Scenario run entered non-scored state unexpectedly: {run.state}"
+            )
+
+        return run
+
+    def score_and_await(
+        self,
+        id: str,
+        *,
+        polling_config: PollingConfig | None = None,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> ScenarioRunView:
+        """Score a scenario run and wait for it to be scored.
+        
+        Args:
+            id: The ID of the scenario run to score and wait for
+            polling_config: Optional polling configuration
+            extra_headers: Send extra headers
+            extra_query: Add additional query parameters to the request
+            extra_body: Add additional JSON properties to the request
+            timeout: Override the client-level default timeout for this request, in seconds
+
+        Returns:
+            The scored scenario run
+
+        Raises:
+            PollingTimeout: If polling times out before scenario run is scored
+            RunloopError: If scenario run enters a non-scored terminal state
+        """
+        self.score(
+            id,
+            extra_headers=extra_headers,
+            extra_query=extra_query,
+            extra_body=extra_body,
+            timeout=timeout,
+        )
+
+        return self.await_scored(
+            id,
+            polling_config=polling_config,
+            extra_headers=extra_headers,
+            extra_query=extra_query,
+            extra_body=extra_body,
+            timeout=timeout,
+        )
+
+    def score_and_complete(
+        self,
+        id: str,
+        *,
+        polling_config: PollingConfig | None = None,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> ScenarioRunView:
+        """Score a scenario run, wait for it to be scored, then complete it.
+        
+        Args:
+            id: The ID of the scenario run to score, wait for, and complete
+            polling_config: Optional polling configuration
+            extra_headers: Send extra headers
+            extra_query: Add additional query parameters to the request
+            extra_body: Add additional JSON properties to the request
+            timeout: Override the client-level default timeout for this request, in seconds
+
+        Returns:
+            The completed scenario run
+
+        Raises:
+            PollingTimeout: If polling times out before scenario run is scored
+            RunloopError: If scenario run enters a non-scored terminal state
+        """
+        self.score_and_await(
+            id,
+            polling_config=polling_config,
+            extra_headers=extra_headers,
+            extra_query=extra_query,
+            extra_body=extra_body,
+            timeout=timeout,
+        )
+
+        return self.complete(
+            id,
+            extra_headers=extra_headers,
+            extra_query=extra_query,
+            extra_body=extra_body,
+            timeout=timeout,
+        )
 
 class AsyncRunsResource(AsyncAPIResource):
     @cached_property
@@ -391,6 +535,148 @@ class AsyncRunsResource(AsyncAPIResource):
             cast_to=ScenarioRunView,
         )
 
+    async def await_scored(
+        self,
+        id: str,
+        *,
+        polling_config: PollingConfig | None = None,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> ScenarioRunView:
+        """Wait for a scenario run to be scored.
+        
+        Args:
+            id: The ID of the scenario run to wait for
+            polling_config: Optional polling configuration
+            extra_headers: Send extra headers
+            extra_query: Add additional query parameters to the request
+            extra_body: Add additional JSON properties to the request
+            timeout: Override the client-level default timeout for this request, in seconds
+
+        Returns:
+            The scored scenario run
+
+        Raises:
+            PollingTimeout: If polling times out before scenario run is scored
+            RunloopError: If scenario run enters a non-scored terminal state
+        """
+        async def retrieve_run() -> ScenarioRunView:
+            return await self.retrieve(
+                id,
+                extra_headers=extra_headers,
+                extra_query=extra_query,
+                extra_body=extra_body,
+                timeout=timeout
+            )
+
+        def is_done_scoring(run: ScenarioRunView) -> bool:
+            return run.state not in ["scoring"]
+
+        run = await async_poll_until(retrieve_run, is_done_scoring, polling_config)
+
+        if run.state != "scored":
+            raise RunloopError(
+                f"Scenario run entered non-scored state unexpectedly: {run.state}"
+            )
+
+        return run
+
+    async def score_and_await(
+        self,
+        id: str,
+        *,
+        polling_config: PollingConfig | None = None,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> ScenarioRunView:
+        """Score a scenario run and wait for it to be scored.
+        
+        Args:
+            id: The ID of the scenario run to score and wait for
+            polling_config: Optional polling configuration
+            extra_headers: Send extra headers
+            extra_query: Add additional query parameters to the request
+            extra_body: Add additional JSON properties to the request
+            timeout: Override the client-level default timeout for this request, in seconds
+
+        Returns:
+            The scored scenario run
+
+        Raises:
+            PollingTimeout: If polling times out before scenario run is scored
+            RunloopError: If scenario run enters a non-scored terminal state
+        """
+        await self.score(
+            id,
+            extra_headers=extra_headers,
+            extra_query=extra_query,
+            extra_body=extra_body,
+            timeout=timeout,
+        )
+
+        return await self.await_scored(
+            id,
+            polling_config=polling_config,
+            extra_headers=extra_headers,
+            extra_query=extra_query,
+            extra_body=extra_body,
+            timeout=timeout,
+        )
+
+    async def score_and_complete(
+        self,
+        id: str,
+        *,
+        polling_config: PollingConfig | None = None,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> ScenarioRunView:
+        """Score a scenario run, wait for it to be scored, then complete it.
+        
+        Args:
+            id: The ID of the scenario run to score, wait for, and complete
+            polling_config: Optional polling configuration
+            extra_headers: Send extra headers
+            extra_query: Add additional query parameters to the request
+            extra_body: Add additional JSON properties to the request
+            timeout: Override the client-level default timeout for this request, in seconds
+
+        Returns:
+            The completed scenario run
+
+        Raises:
+            PollingTimeout: If polling times out before scenario run is scored
+            RunloopError: If scenario run enters a non-scored terminal state
+        """
+        await self.score_and_await(
+            id,
+            polling_config=polling_config,
+            extra_headers=extra_headers,
+            extra_query=extra_query,
+            extra_body=extra_body,
+            timeout=timeout,
+        )
+
+        return await self.complete(
+            id,
+            extra_headers=extra_headers,
+            extra_query=extra_query,
+            extra_body=extra_body,
+            timeout=timeout,
+        )
+
 
 class RunsResourceWithRawResponse:
     def __init__(self, runs: RunsResource) -> None: