runloop_api_client 1.5.1__py3-none-any.whl

runloop_api_client/types/scenario_update_params.py

@@ -0,0 +1,49 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict, Optional
+from typing_extensions import Literal, TypedDict
+
+from .._types import SequenceNotStr
+from .input_context_update_param import InputContextUpdateParam
+from .scenario_environment_param import ScenarioEnvironmentParam
+from .scoring_contract_update_param import ScoringContractUpdateParam
+
+__all__ = ["ScenarioUpdateParams"]
+
+
+class ScenarioUpdateParams(TypedDict, total=False):
+    environment_parameters: Optional[ScenarioEnvironmentParam]
+    """The Environment in which the Scenario will run."""
+
+    input_context: Optional[InputContextUpdateParam]
+    """The input context for the Scenario."""
+
+    metadata: Optional[Dict[str, str]]
+    """User defined metadata to attach to the scenario. Pass in empty map to clear."""
+
+    name: Optional[str]
+    """Name of the scenario. Cannot be blank."""
+
+    reference_output: Optional[str]
+    """A string representation of the reference output to solve the scenario.
+
+    Commonly can be the result of a git diff or a sequence of command actions to
+    apply to the environment. Pass in empty string to clear.
+    """
+
+    required_environment_variables: Optional[SequenceNotStr[str]]
+    """Environment variables required to run the scenario.
+
+    Pass in empty list to clear.
+    """
+
+    required_secret_names: Optional[SequenceNotStr[str]]
+    """Secrets required to run the scenario. Pass in empty list to clear."""
+
+    scoring_contract: Optional[ScoringContractUpdateParam]
+    """The scoring contract for the Scenario."""
+
+    validation_type: Optional[Literal["UNSPECIFIED", "FORWARD", "REVERSE", "EVALUATION"]]
+    """Validation strategy. Pass in empty string to clear."""

runloop_api_client/types/scenario_view.py

@@ -0,0 +1,61 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Optional
+from typing_extensions import Literal
+
+from .._models import BaseModel
+from .input_context import InputContext
+from .scoring_contract import ScoringContract
+from .scenario_environment import ScenarioEnvironment
+
+__all__ = ["ScenarioView"]
+
+
+class ScenarioView(BaseModel):
+    """
+    A ScenarioDefinitionView represents a repeatable AI coding evaluation test, complete with initial environment and scoring contract.
+    """
+
+    id: str
+    """The ID of the Scenario."""
+
+    input_context: InputContext
+    """The input context for the Scenario."""
+
+    metadata: Dict[str, str]
+    """User defined metadata to attach to the scenario for organization."""
+
+    name: str
+    """The name of the Scenario."""
+
+    scoring_contract: ScoringContract
+    """The scoring contract for the Scenario."""
+
+    environment: Optional[ScenarioEnvironment] = None
+    """The Environment in which the Scenario is run."""
+
+    is_public: Optional[bool] = None
+    """Whether this scenario is public."""
+
+    reference_output: Optional[str] = None
+    """A string representation of the reference output to solve the scenario.
+
+    Commonly can be the result of a git diff or a sequence of command actions to
+    apply to the environment.
+    """
+
+    required_environment_variables: Optional[List[str]] = None
+    """Environment variables required to run the scenario.
+
+    If any required environment variables are missing, the scenario will fail to
+    start.
+    """
+
+    required_secret_names: Optional[List[str]] = None
+    """Environment variables required to run the scenario.
+
+    If any required secrets are missing, the scenario will fail to start.
+    """
+
+    validation_type: Optional[Literal["UNSPECIFIED", "FORWARD", "REVERSE", "EVALUATION"]] = None
+    """Validation strategy."""

runloop_api_client/types/scenarios/__init__.py

@@ -0,0 +1,14 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from .run_list_params import RunListParams as RunListParams
+from .scorer_list_params import ScorerListParams as ScorerListParams
+from .scorer_create_params import ScorerCreateParams as ScorerCreateParams
+from .scorer_list_response import ScorerListResponse as ScorerListResponse
+from .scorer_update_params import ScorerUpdateParams as ScorerUpdateParams
+from .scorer_create_response import ScorerCreateResponse as ScorerCreateResponse
+from .scorer_update_response import ScorerUpdateResponse as ScorerUpdateResponse
+from .scorer_validate_params import ScorerValidateParams as ScorerValidateParams
+from .scorer_retrieve_response import ScorerRetrieveResponse as ScorerRetrieveResponse
+from .scorer_validate_response import ScorerValidateResponse as ScorerValidateResponse

runloop_api_client/types/scenarios/run_list_params.py

@@ -0,0 +1,27 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import TypedDict
+
+__all__ = ["RunListParams"]
+
+
+class RunListParams(TypedDict, total=False):
+    benchmark_run_id: str
+    """Filter by benchmark run ID"""
+
+    limit: int
+    """The limit of items to return. Default is 20. Max is 5000."""
+
+    name: str
+    """Filter by name"""
+
+    scenario_id: str
+    """Filter runs associated to Scenario given ID"""
+
+    starting_after: str
+    """Load the next page of data starting after the item with the given ID."""
+
+    state: str
+    """Filter by state"""

runloop_api_client/types/scenarios/scorer_create_params.py

@@ -0,0 +1,18 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import Required, TypedDict
+
+__all__ = ["ScorerCreateParams"]
+
+
+class ScorerCreateParams(TypedDict, total=False):
+    bash_script: Required[str]
+    """
+    Bash script for the custom scorer taking context as a json object
+    $RL_SCORER_CONTEXT.
+    """
+
+    type: Required[str]
+    """Name of the type of custom scorer."""

runloop_api_client/types/scenarios/scorer_create_response.py

@@ -0,0 +1,18 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from ..._models import BaseModel
+
+__all__ = ["ScorerCreateResponse"]
+
+
+class ScorerCreateResponse(BaseModel):
+    """A ScenarioScorerView represents a custom scoring function for a Scenario."""
+
+    id: str
+    """ID for the scenario scorer."""
+
+    bash_script: str
+    """Bash script that takes in $RL_SCORER_CONTEXT as env variable and runs scoring."""
+
+    type: str
+    """Name of the type of scenario scorer."""

runloop_api_client/types/scenarios/scorer_list_params.py

@@ -0,0 +1,15 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import TypedDict
+
+__all__ = ["ScorerListParams"]
+
+
+class ScorerListParams(TypedDict, total=False):
+    limit: int
+    """The limit of items to return. Default is 20. Max is 5000."""
+
+    starting_after: str
+    """Load the next page of data starting after the item with the given ID."""

runloop_api_client/types/scenarios/scorer_list_response.py

@@ -0,0 +1,18 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from ..._models import BaseModel
+
+__all__ = ["ScorerListResponse"]
+
+
+class ScorerListResponse(BaseModel):
+    """A ScenarioScorerView represents a custom scoring function for a Scenario."""
+
+    id: str
+    """ID for the scenario scorer."""
+
+    bash_script: str
+    """Bash script that takes in $RL_SCORER_CONTEXT as env variable and runs scoring."""
+
+    type: str
+    """Name of the type of scenario scorer."""

runloop_api_client/types/scenarios/scorer_retrieve_response.py

@@ -0,0 +1,18 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from ..._models import BaseModel
+
+__all__ = ["ScorerRetrieveResponse"]
+
+
+class ScorerRetrieveResponse(BaseModel):
+    """A ScenarioScorerView represents a custom scoring function for a Scenario."""
+
+    id: str
+    """ID for the scenario scorer."""
+
+    bash_script: str
+    """Bash script that takes in $RL_SCORER_CONTEXT as env variable and runs scoring."""
+
+    type: str
+    """Name of the type of scenario scorer."""

runloop_api_client/types/scenarios/scorer_update_params.py

@@ -0,0 +1,18 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import Required, TypedDict
+
+__all__ = ["ScorerUpdateParams"]
+
+
+class ScorerUpdateParams(TypedDict, total=False):
+    bash_script: Required[str]
+    """
+    Bash script for the custom scorer taking context as a json object
+    $RL_SCORER_CONTEXT.
+    """
+
+    type: Required[str]
+    """Name of the type of custom scorer."""

runloop_api_client/types/scenarios/scorer_update_response.py

@@ -0,0 +1,18 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from ..._models import BaseModel
+
+__all__ = ["ScorerUpdateResponse"]
+
+
+class ScorerUpdateResponse(BaseModel):
+    """A ScenarioScorerView represents a custom scoring function for a Scenario."""
+
+    id: str
+    """ID for the scenario scorer."""
+
+    bash_script: str
+    """Bash script that takes in $RL_SCORER_CONTEXT as env variable and runs scoring."""
+
+    type: str
+    """Name of the type of scenario scorer."""

runloop_api_client/types/scenarios/scorer_validate_params.py

@@ -0,0 +1,17 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import Required, TypedDict
+
+from ..scenario_environment_param import ScenarioEnvironmentParam
+
+__all__ = ["ScorerValidateParams"]
+
+
+class ScorerValidateParams(TypedDict, total=False):
+    scoring_context: Required[object]
+    """Json context that gets passed to the custom scorer"""
+
+    environment_parameters: ScenarioEnvironmentParam
+    """The Environment in which the Scenario will run."""

runloop_api_client/types/scenarios/scorer_validate_response.py

@@ -0,0 +1,23 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Optional
+
+from ..._models import BaseModel
+from ..scenario_environment import ScenarioEnvironment
+from ..scoring_function_result_view import ScoringFunctionResultView
+
+__all__ = ["ScorerValidateResponse"]
+
+
+class ScorerValidateResponse(BaseModel):
+    name: str
+    """Name of the custom scorer."""
+
+    scoring_context: object
+    """Json context that gets passed to the custom scorer"""
+
+    scoring_result: ScoringFunctionResultView
+    """Result of the scoring function."""
+
+    environment_parameters: Optional[ScenarioEnvironment] = None
+    """The Environment in which the Scenario will run."""

runloop_api_client/types/scoring_contract.py

@@ -0,0 +1,17 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import List
+
+from .._models import BaseModel
+from .scoring_function import ScoringFunction
+
+__all__ = ["ScoringContract"]
+
+
+class ScoringContract(BaseModel):
+    """
+    InputContextView specifies the problem statement along with all additional context for a Scenario.
+    """
+
+    scoring_function_parameters: List[ScoringFunction]
+    """A list of scoring functions used to evaluate the Scenario."""

runloop_api_client/types/scoring_contract_param.py

@@ -0,0 +1,19 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Iterable
+from typing_extensions import Required, TypedDict
+
+from .scoring_function_param import ScoringFunctionParam
+
+__all__ = ["ScoringContractParam"]
+
+
+class ScoringContractParam(TypedDict, total=False):
+    """
+    InputContextView specifies the problem statement along with all additional context for a Scenario.
+    """
+
+    scoring_function_parameters: Required[Iterable[ScoringFunctionParam]]
+    """A list of scoring functions used to evaluate the Scenario."""

runloop_api_client/types/scoring_contract_result_view.py

@@ -0,0 +1,20 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import List
+
+from .._models import BaseModel
+from .scoring_function_result_view import ScoringFunctionResultView
+
+__all__ = ["ScoringContractResultView"]
+
+
+class ScoringContractResultView(BaseModel):
+    """
+    A ScoringContractResultView represents the result of running all scoring functions on a given input context.
+    """
+
+    score: float
+    """Total score for all scoring contracts. This will be a value between 0 and 1."""
+
+    scoring_function_results: List[ScoringFunctionResultView]
+    """List of all individual scoring function results."""

runloop_api_client/types/scoring_contract_update_param.py

@@ -0,0 +1,15 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Iterable, Optional
+from typing_extensions import TypedDict
+
+from .scoring_function_param import ScoringFunctionParam
+
+__all__ = ["ScoringContractUpdateParam"]
+
+
+class ScoringContractUpdateParam(TypedDict, total=False):
+    scoring_function_parameters: Optional[Iterable[ScoringFunctionParam]]
+    """A list of scoring functions used to evaluate the Scenario."""

runloop_api_client/types/scoring_function.py

@@ -0,0 +1,157 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import List, Union, Optional
+from typing_extensions import Literal, Annotated, TypeAlias
+
+from .._utils import PropertyInfo
+from .._models import BaseModel
+
+__all__ = [
+    "ScoringFunction",
+    "Scorer",
+    "ScorerAstGrepScoringFunction",
+    "ScorerBashScriptScoringFunction",
+    "ScorerCommandScoringFunction",
+    "ScorerCustomScoringFunction",
+    "ScorerPythonScriptScoringFunction",
+    "ScorerTestBasedScoringFunction",
+    "ScorerTestBasedScoringFunctionTestFile",
+]
+
+
+class ScorerAstGrepScoringFunction(BaseModel):
+    """AstGrepScoringFunction utilizes structured coach search for scoring."""
+
+    pattern: str
+    """AST pattern to match.
+
+    Pattern will be passed to ast-grep using the commandline surround by double
+    quotes ("), so make sure to use proper escaping (for example, \\$$\\$$\\$$).
+    """
+
+    search_directory: str
+    """The path to search."""
+
+    type: Literal["ast_grep_scorer"]
+
+    lang: Optional[str] = None
+    """The language of the pattern."""
+
+
+class ScorerBashScriptScoringFunction(BaseModel):
+    """
+    BashScriptScoringFunction is a scoring function specified by a bash script that will be run in the context of your environment.
+    """
+
+    type: Literal["bash_script_scorer"]
+
+    bash_script: Optional[str] = None
+    """
+    A single bash script that sets up the environment, scores, and prints the final
+    score to standard out. Score should be a float between 0.0 and 1.0, and look
+    like "score=[0.0..1.0].
+    """
+
+
+class ScorerCommandScoringFunction(BaseModel):
+    """
+    CommandScoringFunction executes a single command and checks the result.The output of the command will be printed. Scoring will passed if the command returns status code 0, otherwise it will be failed.
+    """
+
+    type: Literal["command_scorer"]
+
+    command: Optional[str] = None
+    """The command to execute."""
+
+
+class ScorerCustomScoringFunction(BaseModel):
+    """CustomScoringFunction is a custom, user defined scoring function."""
+
+    custom_scorer_type: str
+    """Type of the scoring function, previously registered with Runloop."""
+
+    type: Literal["custom_scorer"]
+
+    scorer_params: Optional[object] = None
+    """Additional JSON structured context to pass to the scoring function."""
+
+
+class ScorerPythonScriptScoringFunction(BaseModel):
+    """
+    PythonScriptScoringFunction will run a python script in the context of your environment as a ScoringFunction.
+    """
+
+    python_script: str
+    """Python script to be run.
+
+    The script should output the score to standard out as a float between 0.0 and
+    1.0.
+    """
+
+    type: Literal["python_script_scorer"]
+
+    python_version_constraint: Optional[str] = None
+    """Python version to run scoring. Default is "==3.12.10" """
+
+    requirements_contents: Optional[str] = None
+    """Package dependencies to be installed.
+
+    The requirements should be a valid requirements.txt file.
+    """
+
+
+class ScorerTestBasedScoringFunctionTestFile(BaseModel):
+    file_contents: Optional[str] = None
+    """Content of the test file"""
+
+    file_path: Optional[str] = None
+    """
+    Path to write content of the test file, relative to your environment's working
+    directory
+    """
+
+
+class ScorerTestBasedScoringFunction(BaseModel):
+    """
+    TestBasedScoringFunction writes test files to disk and executes a test command to verify the solution.
+    """
+
+    type: Literal["test_based_scorer"]
+
+    test_command: Optional[str] = None
+    """The command to execute for running the tests"""
+
+    test_files: Optional[List[ScorerTestBasedScoringFunctionTestFile]] = None
+    """List of test files to create"""
+
+
+Scorer: TypeAlias = Annotated[
+    Union[
+        ScorerAstGrepScoringFunction,
+        ScorerBashScriptScoringFunction,
+        ScorerCommandScoringFunction,
+        ScorerCustomScoringFunction,
+        ScorerPythonScriptScoringFunction,
+        ScorerTestBasedScoringFunction,
+    ],
+    PropertyInfo(discriminator="type"),
+]
+
+
+class ScoringFunction(BaseModel):
+    """ScoringFunction specifies a method of scoring a Scenario."""
+
+    name: str
+    """Name of scoring function. Names must only contain ``[a-zA-Z0-9_-]``."""
+
+    scorer: Scorer
+    """The scoring function to use for evaluating this scenario.
+
+    The type field determines which built-in function to use.
+    """
+
+    weight: float
+    """Weight to apply to scoring function score.
+
+    Weights of all scoring functions should sum to 1.0.
+    """