runloop-api-client 0.23.0__tar.gz → 0.24.0__tar.gz

runloop_api_client-0.24.0/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  ".": "0.24.0"
+}

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## 0.24.0 (2025-02-19)
+
+Full Changelog: [v0.23.0...v0.24.0](https://github.com/runloopai/api-client-python/compare/v0.23.0...v0.24.0)
+
+### Features
+
+* **api:** api update ([#549](https://github.com/runloopai/api-client-python/issues/549)) ([3e19f41](https://github.com/runloopai/api-client-python/commit/3e19f41cf8e2b9cd225439d9df90f80b34d89660))
+
+
+### Bug Fixes
+
+* asyncify on non-asyncio runtimes ([#547](https://github.com/runloopai/api-client-python/issues/547)) ([8ce7003](https://github.com/runloopai/api-client-python/commit/8ce700397fc5d6755f0a95b637efad74d6dc3fe4))
+
+
+### Chores
+
+* **internal:** codegen related update ([#543](https://github.com/runloopai/api-client-python/issues/543)) ([5c44e84](https://github.com/runloopai/api-client-python/commit/5c44e84a8a029b22461df339c280885dc972fa96))
+* **internal:** codegen related update ([#546](https://github.com/runloopai/api-client-python/issues/546)) ([d8b620d](https://github.com/runloopai/api-client-python/commit/d8b620d93c687d37a843262d5ac60fd4a9b815e8))
+* **internal:** update client tests ([#545](https://github.com/runloopai/api-client-python/issues/545)) ([30307f4](https://github.com/runloopai/api-client-python/commit/30307f4b78ba76240793b851dd735a0beb835b75))
+* **internal:** update client tests ([#548](https://github.com/runloopai/api-client-python/issues/548)) ([4e782f1](https://github.com/runloopai/api-client-python/commit/4e782f19bfa7e08ae318eca6f00fb4d5ffe5c9af))
+
 ## 0.23.0 (2025-02-11)
 
 Full Changelog: [v0.22.0...v0.23.0](https://github.com/runloopai/api-client-python/compare/v0.22.0...v0.23.0)

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: runloop_api_client
-Version: 0.23.0
+Version: 0.24.0
 Summary: The official Python library for the runloop API
 Project-URL: Homepage, https://github.com/runloopai/api-client-python
 Project-URL: Repository, https://github.com/runloopai/api-client-python

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "runloop_api_client"
-version = "0.23.0"
+version = "0.24.0"
 description = "The official Python library for the runloop API"
 dynamic = ["readme"]
 license = "MIT"

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/_utils/_sync.py

@@ -7,16 +7,20 @@ import contextvars
 from typing import Any, TypeVar, Callable, Awaitable
 from typing_extensions import ParamSpec
 
+import anyio
+import sniffio
+import anyio.to_thread
+
 T_Retval = TypeVar("T_Retval")
 T_ParamSpec = ParamSpec("T_ParamSpec")
 
 
 if sys.version_info >= (3, 9):
-    to_thread = asyncio.to_thread
+    _asyncio_to_thread = asyncio.to_thread
 else:
     # backport of https://docs.python.org/3/library/asyncio-task.html#asyncio.to_thread
     # for Python 3.8 support
-    async def to_thread(
+    async def _asyncio_to_thread(
         func: Callable[T_ParamSpec, T_Retval], /, *args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs
     ) -> Any:
         """Asynchronously run function *func* in a separate thread.
@@ -34,6 +38,17 @@ else:
         return await loop.run_in_executor(None, func_call)
 
 
+async def to_thread(
+    func: Callable[T_ParamSpec, T_Retval], /, *args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs
+) -> T_Retval:
+    if sniffio.current_async_library() == "asyncio":
+        return await _asyncio_to_thread(func, *args, **kwargs)
+
+    return await anyio.to_thread.run_sync(
+        functools.partial(func, *args, **kwargs),
+    )
+
+
 # inspired by `asyncer`, https://github.com/tiangolo/asyncer
 def asyncify(function: Callable[T_ParamSpec, T_Retval]) -> Callable[T_ParamSpec, Awaitable[T_Retval]]:
     """

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/_version.py

@@ -1,4 +1,4 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
 __title__ = "runloop_api_client"
-__version__ = "0.23.0"  # x-release-please-version
+__version__ = "0.24.0"  # x-release-please-version

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/resources/benchmarks/benchmarks.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 import httpx
 
@@ -69,6 +69,7 @@ class BenchmarksResource(SyncAPIResource):
         self,
         *,
         name: str,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         scenario_ids: Optional[List[str]] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -84,6 +85,8 @@ class BenchmarksResource(SyncAPIResource):
         Args:
           name: The name of the Benchmark.
 
+          metadata: User defined metadata to attach to the benchmark for organization.
+
           scenario_ids: The Scenario IDs that make up the Benchmark.
 
           extra_headers: Send extra headers
@@ -101,6 +104,7 @@ class BenchmarksResource(SyncAPIResource):
             body=maybe_transform(
                 {
                     "name": name,
+                    "metadata": metadata,
                     "scenario_ids": scenario_ids,
                 },
                 benchmark_create_params.BenchmarkCreateParams,
@@ -246,6 +250,7 @@ class BenchmarksResource(SyncAPIResource):
         self,
         *,
         benchmark_id: str,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         run_name: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -261,6 +266,8 @@ class BenchmarksResource(SyncAPIResource):
         Args:
           benchmark_id: ID of the Benchmark to run.
 
+          metadata: User defined metadata to attach to the benchmark run for organization.
+
           run_name: Display name of the run.
 
           extra_headers: Send extra headers
@@ -278,6 +285,7 @@ class BenchmarksResource(SyncAPIResource):
             body=maybe_transform(
                 {
                     "benchmark_id": benchmark_id,
+                    "metadata": metadata,
                     "run_name": run_name,
                 },
                 benchmark_start_run_params.BenchmarkStartRunParams,
@@ -321,6 +329,7 @@ class AsyncBenchmarksResource(AsyncAPIResource):
         self,
         *,
         name: str,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         scenario_ids: Optional[List[str]] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -336,6 +345,8 @@ class AsyncBenchmarksResource(AsyncAPIResource):
         Args:
           name: The name of the Benchmark.
 
+          metadata: User defined metadata to attach to the benchmark for organization.
+
           scenario_ids: The Scenario IDs that make up the Benchmark.
 
           extra_headers: Send extra headers
@@ -353,6 +364,7 @@ class AsyncBenchmarksResource(AsyncAPIResource):
             body=await async_maybe_transform(
                 {
                     "name": name,
+                    "metadata": metadata,
                     "scenario_ids": scenario_ids,
                 },
                 benchmark_create_params.BenchmarkCreateParams,
@@ -498,6 +510,7 @@ class AsyncBenchmarksResource(AsyncAPIResource):
         self,
         *,
         benchmark_id: str,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         run_name: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -513,6 +526,8 @@ class AsyncBenchmarksResource(AsyncAPIResource):
         Args:
           benchmark_id: ID of the Benchmark to run.
 
+          metadata: User defined metadata to attach to the benchmark run for organization.
+
           run_name: Display name of the run.
 
           extra_headers: Send extra headers
@@ -530,6 +545,7 @@ class AsyncBenchmarksResource(AsyncAPIResource):
             body=await async_maybe_transform(
                 {
                     "benchmark_id": benchmark_id,
+                    "metadata": metadata,
                     "run_name": run_name,
                 },
                 benchmark_start_run_params.BenchmarkStartRunParams,

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/resources/scenarios/scenarios.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import Optional
+from typing import Dict, Optional
 
 import httpx
 
@@ -88,6 +88,8 @@ class ScenariosResource(SyncAPIResource):
         name: str,
         scoring_contract: ScoringContractParam,
         environment_parameters: Optional[ScenarioEnvironmentParam] | NotGiven = NOT_GIVEN,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
+        reference_output: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
         extra_headers: Headers | None = None,
@@ -109,6 +111,12 @@ class ScenariosResource(SyncAPIResource):
 
           environment_parameters: The Environment in which the Scenario will run.
 
+          metadata: User defined metadata to attach to the scenario for organization.
+
+          reference_output: A string representation of the reference output to solve the scenario. Commonly
+              can be the result of a git diff or a sequence of command actions to apply to the
+              environment.
+
           extra_headers: Send extra headers
 
           extra_query: Add additional query parameters to the request
@@ -127,6 +135,8 @@ class ScenariosResource(SyncAPIResource):
                     "name": name,
                     "scoring_contract": scoring_contract,
                     "environment_parameters": environment_parameters,
+                    "metadata": metadata,
+                    "reference_output": reference_output,
                 },
                 scenario_create_params.ScenarioCreateParams,
             ),
@@ -281,6 +291,7 @@ class ScenariosResource(SyncAPIResource):
         *,
         scenario_id: str,
         benchmark_run_id: Optional[str] | NotGiven = NOT_GIVEN,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         run_name: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -298,6 +309,8 @@ class ScenariosResource(SyncAPIResource):
 
           benchmark_run_id: Benchmark to associate the run.
 
+          metadata: User defined metadata to attach to the run for organization.
+
           run_name: Display name of the run.
 
           extra_headers: Send extra headers
@@ -316,6 +329,7 @@ class ScenariosResource(SyncAPIResource):
                 {
                     "scenario_id": scenario_id,
                     "benchmark_run_id": benchmark_run_id,
+                    "metadata": metadata,
                     "run_name": run_name,
                 },
                 scenario_start_run_params.ScenarioStartRunParams,
@@ -423,6 +437,8 @@ class AsyncScenariosResource(AsyncAPIResource):
         name: str,
         scoring_contract: ScoringContractParam,
         environment_parameters: Optional[ScenarioEnvironmentParam] | NotGiven = NOT_GIVEN,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
+        reference_output: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
         extra_headers: Headers | None = None,
@@ -444,6 +460,12 @@ class AsyncScenariosResource(AsyncAPIResource):
 
           environment_parameters: The Environment in which the Scenario will run.
 
+          metadata: User defined metadata to attach to the scenario for organization.
+
+          reference_output: A string representation of the reference output to solve the scenario. Commonly
+              can be the result of a git diff or a sequence of command actions to apply to the
+              environment.
+
           extra_headers: Send extra headers
 
           extra_query: Add additional query parameters to the request
@@ -462,6 +484,8 @@ class AsyncScenariosResource(AsyncAPIResource):
                     "name": name,
                     "scoring_contract": scoring_contract,
                     "environment_parameters": environment_parameters,
+                    "metadata": metadata,
+                    "reference_output": reference_output,
                 },
                 scenario_create_params.ScenarioCreateParams,
             ),
@@ -616,6 +640,7 @@ class AsyncScenariosResource(AsyncAPIResource):
         *,
         scenario_id: str,
         benchmark_run_id: Optional[str] | NotGiven = NOT_GIVEN,
+        metadata: Optional[Dict[str, str]] | NotGiven = NOT_GIVEN,
         run_name: Optional[str] | NotGiven = NOT_GIVEN,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -633,6 +658,8 @@ class AsyncScenariosResource(AsyncAPIResource):
 
           benchmark_run_id: Benchmark to associate the run.
 
+          metadata: User defined metadata to attach to the run for organization.
+
           run_name: Display name of the run.
 
           extra_headers: Send extra headers
@@ -651,6 +678,7 @@ class AsyncScenariosResource(AsyncAPIResource):
                 {
                     "scenario_id": scenario_id,
                     "benchmark_run_id": benchmark_run_id,
+                    "metadata": metadata,
                     "run_name": run_name,
                 },
                 scenario_start_run_params.ScenarioStartRunParams,

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/benchmark_create_params.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import List, Optional
+from typing import Dict, List, Optional
 from typing_extensions import Required, TypedDict
 
 __all__ = ["BenchmarkCreateParams"]
@@ -12,5 +12,8 @@ class BenchmarkCreateParams(TypedDict, total=False):
     name: Required[str]
     """The name of the Benchmark."""
 
+    metadata: Optional[Dict[str, str]]
+    """User defined metadata to attach to the benchmark for organization."""
+
     scenario_ids: Optional[List[str]]
     """The Scenario IDs that make up the Benchmark."""

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/benchmark_run_view.py

@@ -1,6 +1,6 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-from typing import List, Optional
+from typing import Dict, List, Optional
 from typing_extensions import Literal
 
 from .._models import BaseModel
@@ -15,6 +15,12 @@ class BenchmarkRunView(BaseModel):
     benchmark_id: str
     """The ID of the Benchmark."""
 
+    metadata: Dict[str, str]
+    """User defined metadata to attach to the benchmark run for organization."""
+
+    pending_scenarios: List[str]
+    """List of Scenarios that need to be completed before benchmark can be completed."""
+
     start_time_ms: int
     """The time the benchmark run execution started (Unix timestamp milliseconds)."""
 
@@ -27,9 +33,6 @@ class BenchmarkRunView(BaseModel):
     name: Optional[str] = None
     """The name of the BenchmarkRun."""
 
-    pending_scenarios: Optional[List[str]] = None
-    """List of Scenarios that need to be completed before benchmark can be completed."""
-
     score: Optional[float] = None
     """The final score across the BenchmarkRun, present once completed.
 

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/benchmark_start_run_params.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import Optional
+from typing import Dict, Optional
 from typing_extensions import Required, TypedDict
 
 __all__ = ["BenchmarkStartRunParams"]
@@ -12,5 +12,8 @@ class BenchmarkStartRunParams(TypedDict, total=False):
     benchmark_id: Required[str]
     """ID of the Benchmark to run."""
 
+    metadata: Optional[Dict[str, str]]
+    """User defined metadata to attach to the benchmark run for organization."""
+
     run_name: Optional[str]
     """Display name of the run."""

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/benchmark_view.py

@@ -1,6 +1,6 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-from typing import List
+from typing import Dict, List
 
 from pydantic import Field as FieldInfo
 
@@ -13,6 +13,9 @@ class BenchmarkView(BaseModel):
     id: str
     """The ID of the Benchmark."""
 
+    metadata: Dict[str, str]
+    """User defined metadata to attach to the benchmark for organization."""
+
     name: str
     """The name of the Benchmark."""
 

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/scenario_create_params.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import Optional
+from typing import Dict, Optional
 from typing_extensions import Required, TypedDict
 
 from .input_context_param import InputContextParam
@@ -24,3 +24,13 @@ class ScenarioCreateParams(TypedDict, total=False):
 
     environment_parameters: Optional[ScenarioEnvironmentParam]
     """The Environment in which the Scenario will run."""
+
+    metadata: Optional[Dict[str, str]]
+    """User defined metadata to attach to the scenario for organization."""
+
+    reference_output: Optional[str]
+    """A string representation of the reference output to solve the scenario.
+
+    Commonly can be the result of a git diff or a sequence of command actions to
+    apply to the environment.
+    """

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/scenario_environment.py

@@ -3,6 +3,7 @@
 from typing import Optional
 
 from .._models import BaseModel
+from .shared.launch_parameters import LaunchParameters
 
 __all__ = ["ScenarioEnvironment"]
 
@@ -11,8 +12,17 @@ class ScenarioEnvironment(BaseModel):
     blueprint_id: Optional[str] = None
     """Use the blueprint with matching ID."""
 
+    launch_parameters: Optional[LaunchParameters] = None
+    """Optional launch parameters to apply to the devbox environment at launch."""
+
     prebuilt_id: Optional[str] = None
     """Use the prebuilt with matching ID."""
 
     snapshot_id: Optional[str] = None
     """Use the snapshot with matching ID."""
+
+    working_directory: Optional[str] = None
+    """The working directory where the agent is expected to fulfill the scenario.
+
+    Scoring functions also run from the working directory.
+    """

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/scenario_environment_param.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 from typing import Optional
 from typing_extensions import TypedDict
 
+from .shared_params.launch_parameters import LaunchParameters
+
 __all__ = ["ScenarioEnvironmentParam"]
 
 
@@ -12,8 +14,17 @@ class ScenarioEnvironmentParam(TypedDict, total=False):
     blueprint_id: Optional[str]
     """Use the blueprint with matching ID."""
 
+    launch_parameters: Optional[LaunchParameters]
+    """Optional launch parameters to apply to the devbox environment at launch."""
+
     prebuilt_id: Optional[str]
     """Use the prebuilt with matching ID."""
 
     snapshot_id: Optional[str]
     """Use the snapshot with matching ID."""
+
+    working_directory: Optional[str]
+    """The working directory where the agent is expected to fulfill the scenario.
+
+    Scoring functions also run from the working directory.
+    """

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/scenario_run_view.py

@@ -1,6 +1,6 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-from typing import Optional
+from typing import Dict, Optional
 from typing_extensions import Literal
 
 from .._models import BaseModel
@@ -16,6 +16,9 @@ class ScenarioRunView(BaseModel):
     devbox_id: str
     """ID of the Devbox on which the Scenario is running."""
 
+    metadata: Dict[str, str]
+    """User defined metadata to attach to the scenario run for organization."""
+
     scenario_id: str
     """ID of the Scenario that has been run."""
 

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/scenario_start_run_params.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import Optional
+from typing import Dict, Optional
 from typing_extensions import Required, TypedDict
 
 __all__ = ["ScenarioStartRunParams"]
@@ -15,5 +15,8 @@ class ScenarioStartRunParams(TypedDict, total=False):
     benchmark_run_id: Optional[str]
     """Benchmark to associate the run."""
 
+    metadata: Optional[Dict[str, str]]
+    """User defined metadata to attach to the run for organization."""
+
     run_name: Optional[str]
     """Display name of the run."""

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/scenario_view.py

@@ -1,6 +1,6 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-from typing import Optional
+from typing import Dict, Optional
 
 from .._models import BaseModel
 from .input_context import InputContext
@@ -17,6 +17,9 @@ class ScenarioView(BaseModel):
     input_context: InputContext
     """The input context for the Scenario."""
 
+    metadata: Dict[str, str]
+    """User defined metadata to attach to the scenario for organization."""
+
     name: str
     """The name of the Scenario."""
 
@@ -25,3 +28,10 @@ class ScenarioView(BaseModel):
 
     environment: Optional[ScenarioEnvironment] = None
     """The Environment in which the Scenario is run."""
+
+    reference_output: Optional[str] = None
+    """A string representation of the reference output to solve the scenario.
+
+    Commonly can be the result of a git diff or a sequence of command actions to
+    apply to the environment.
+    """

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/scoring_function.py

@@ -9,10 +9,15 @@ __all__ = ["ScoringFunction"]
 
 class ScoringFunction(BaseModel):
     name: str
-    """Name of scoring function."""
+    """Name of scoring function. Names must only contain [a-zA-Z0-9_-]."""
 
     type: str
-    """Type of the scoring function. Defaults to bash script."""
+    """Type of the scoring function.
+
+    Use 'bash' as type and fill out 'bash_script' field for scoring via custom bash
+    scripts. Otherwise use a type corresponding to a custom scorer function or a
+    public Runloop scorer type.
+    """
 
     weight: float
     """Wight to apply to scoring function score.
@@ -23,8 +28,8 @@ class ScoringFunction(BaseModel):
     bash_script: Optional[str] = None
     """
     A single bash script that sets up the environment, scores, and prints the final
-    score to standard out. Score should be an integer between 0 and 100, and look
-    like "score=[0..100].
+    score to standard out. Score should be a float between 0.0 and 1.0, and look
+    like "score=[0.0..1.0].
     """
 
     scorer_params: Optional[object] = None

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/scoring_function_param.py

@@ -10,10 +10,15 @@ __all__ = ["ScoringFunctionParam"]
 
 class ScoringFunctionParam(TypedDict, total=False):
     name: Required[str]
-    """Name of scoring function."""
+    """Name of scoring function. Names must only contain [a-zA-Z0-9_-]."""
 
     type: Required[str]
-    """Type of the scoring function. Defaults to bash script."""
+    """Type of the scoring function.
+
+    Use 'bash' as type and fill out 'bash_script' field for scoring via custom bash
+    scripts. Otherwise use a type corresponding to a custom scorer function or a
+    public Runloop scorer type.
+    """
 
     weight: Required[float]
     """Wight to apply to scoring function score.
@@ -24,8 +29,8 @@ class ScoringFunctionParam(TypedDict, total=False):
     bash_script: Optional[str]
     """
     A single bash script that sets up the environment, scores, and prints the final
-    score to standard out. Score should be an integer between 0 and 100, and look
-    like "score=[0..100].
+    score to standard out. Score should be a float between 0.0 and 1.0, and look
+    like "score=[0.0..1.0].
     """
 
     scorer_params: Optional[object]

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/src/runloop_api_client/types/scoring_function_result_view.py

@@ -1,5 +1,6 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
+from typing_extensions import Literal
 
 from .._models import BaseModel
 
@@ -15,3 +16,6 @@ class ScoringFunctionResultView(BaseModel):
 
     scoring_function_name: str
     """Scoring function name that ran."""
+
+    state: Literal["unknown", "complete", "error"]
+    """The state of the scoring function application."""

{runloop_api_client-0.23.0 → runloop_api_client-0.24.0}/tests/api_resources/scenarios/test_scorers.py

@@ -190,8 +190,19 @@ class TestScorers:
             scoring_context={},
             environment_parameters={
                 "blueprint_id": "blueprint_id",
+                "launch_parameters": {
+                    "after_idle": {
+                        "idle_time_seconds": 0,
+                        "on_idle": "shutdown",
+                    },
+                    "available_ports": [0],
+                    "keep_alive_time_seconds": 0,
+                    "launch_commands": ["string"],
+                    "resource_size_request": "SMALL",
+                },
                 "prebuilt_id": "prebuilt_id",
                 "snapshot_id": "snapshot_id",
+                "working_directory": "working_directory",
             },
         )
         assert_matches_type(ScorerValidateResponse, scorer, path=["response"])
@@ -400,8 +411,19 @@ class TestAsyncScorers:
             scoring_context={},
             environment_parameters={
                 "blueprint_id": "blueprint_id",
+                "launch_parameters": {
+                    "after_idle": {
+                        "idle_time_seconds": 0,
+                        "on_idle": "shutdown",
+                    },
+                    "available_ports": [0],
+                    "keep_alive_time_seconds": 0,
+                    "launch_commands": ["string"],
+                    "resource_size_request": "SMALL",
+                },
                 "prebuilt_id": "prebuilt_id",
                 "snapshot_id": "snapshot_id",
+                "working_directory": "working_directory",
             },
         )
         assert_matches_type(ScorerValidateResponse, scorer, path=["response"])