plato-sdk-v2 2.0.50__py3-none-any.whl → 2.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- plato/__init__.py +7 -6
- plato/_generated/__init__.py +1 -1
- plato/_generated/api/v1/env/evaluate_session.py +3 -3
- plato/_generated/api/v1/env/log_state_mutation.py +4 -4
- plato/_generated/api/v1/sandbox/checkpoint_vm.py +3 -3
- plato/_generated/api/v1/sandbox/save_vm_snapshot.py +3 -3
- plato/_generated/api/v1/sandbox/setup_sandbox.py +8 -8
- plato/_generated/api/v1/session/__init__.py +2 -0
- plato/_generated/api/v1/session/get_sessions_for_archival.py +100 -0
- plato/_generated/api/v1/testcases/__init__.py +6 -2
- plato/_generated/api/v1/testcases/get_mutation_groups_for_testcase.py +98 -0
- plato/_generated/api/v1/testcases/{get_next_output_testcase_for_scoring.py → get_next_testcase_for_scoring.py} +23 -10
- plato/_generated/api/v1/testcases/get_testcase_metadata_for_scoring.py +74 -0
- plato/_generated/api/v2/__init__.py +2 -1
- plato/_generated/api/v2/jobs/__init__.py +4 -0
- plato/_generated/api/v2/jobs/checkpoint.py +3 -3
- plato/_generated/api/v2/jobs/disk_snapshot.py +3 -3
- plato/_generated/api/v2/jobs/log_for_job.py +4 -39
- plato/_generated/api/v2/jobs/make.py +4 -4
- plato/_generated/api/v2/jobs/setup_sandbox.py +97 -0
- plato/_generated/api/v2/jobs/snapshot.py +3 -3
- plato/_generated/api/v2/jobs/snapshot_store.py +91 -0
- plato/_generated/api/v2/sessions/__init__.py +4 -0
- plato/_generated/api/v2/sessions/checkpoint.py +3 -3
- plato/_generated/api/v2/sessions/disk_snapshot.py +3 -3
- plato/_generated/api/v2/sessions/evaluate.py +3 -3
- plato/_generated/api/v2/sessions/log_job_mutation.py +4 -39
- plato/_generated/api/v2/sessions/make.py +4 -4
- plato/_generated/api/v2/sessions/setup_sandbox.py +98 -0
- plato/_generated/api/v2/sessions/snapshot.py +3 -3
- plato/_generated/api/v2/sessions/snapshot_store.py +94 -0
- plato/_generated/api/v2/user/__init__.py +7 -0
- plato/_generated/api/v2/user/get_current_user.py +76 -0
- plato/_generated/models/__init__.py +174 -23
- plato/_sims_generator/__init__.py +19 -4
- plato/_sims_generator/instruction.py +203 -0
- plato/_sims_generator/templates/instruction/helpers.py.jinja +161 -0
- plato/_sims_generator/templates/instruction/init.py.jinja +43 -0
- plato/agents/__init__.py +107 -517
- plato/agents/base.py +145 -0
- plato/agents/build.py +61 -0
- plato/agents/config.py +160 -0
- plato/agents/logging.py +401 -0
- plato/agents/runner.py +161 -0
- plato/agents/trajectory.py +266 -0
- plato/chronos/__init__.py +37 -0
- plato/chronos/api/__init__.py +3 -0
- plato/chronos/api/agents/__init__.py +13 -0
- plato/chronos/api/agents/create_agent.py +63 -0
- plato/chronos/api/agents/delete_agent.py +61 -0
- plato/chronos/api/agents/get_agent.py +62 -0
- plato/chronos/api/agents/get_agent_schema.py +72 -0
- plato/chronos/api/agents/get_agent_versions.py +62 -0
- plato/chronos/api/agents/list_agents.py +57 -0
- plato/chronos/api/agents/lookup_agent.py +74 -0
- plato/chronos/api/auth/__init__.py +9 -0
- plato/chronos/api/auth/debug_auth_api_auth_debug_get.py +43 -0
- plato/chronos/api/auth/get_auth_status_api_auth_status_get.py +61 -0
- plato/chronos/api/auth/get_current_user_route_api_auth_me_get.py +60 -0
- plato/chronos/api/callback/__init__.py +11 -0
- plato/chronos/api/callback/push_agent_logs.py +61 -0
- plato/chronos/api/callback/update_agent_status.py +57 -0
- plato/chronos/api/callback/upload_artifacts.py +59 -0
- plato/chronos/api/callback/upload_logs_zip.py +57 -0
- plato/chronos/api/callback/upload_trajectory.py +57 -0
- plato/chronos/api/default/__init__.py +7 -0
- plato/chronos/api/default/health.py +43 -0
- plato/chronos/api/jobs/__init__.py +7 -0
- plato/chronos/api/jobs/launch_job.py +63 -0
- plato/chronos/api/registry/__init__.py +19 -0
- plato/chronos/api/registry/get_agent_schema_api_registry_agents__agent_name__schema_get.py +62 -0
- plato/chronos/api/registry/get_agent_versions_api_registry_agents__agent_name__versions_get.py +52 -0
- plato/chronos/api/registry/get_world_schema_api_registry_worlds__package_name__schema_get.py +68 -0
- plato/chronos/api/registry/get_world_versions_api_registry_worlds__package_name__versions_get.py +52 -0
- plato/chronos/api/registry/list_registry_agents_api_registry_agents_get.py +44 -0
- plato/chronos/api/registry/list_registry_worlds_api_registry_worlds_get.py +44 -0
- plato/chronos/api/runtimes/__init__.py +11 -0
- plato/chronos/api/runtimes/create_runtime.py +63 -0
- plato/chronos/api/runtimes/delete_runtime.py +61 -0
- plato/chronos/api/runtimes/get_runtime.py +62 -0
- plato/chronos/api/runtimes/list_runtimes.py +57 -0
- plato/chronos/api/runtimes/test_runtime.py +67 -0
- plato/chronos/api/secrets/__init__.py +11 -0
- plato/chronos/api/secrets/create_secret.py +63 -0
- plato/chronos/api/secrets/delete_secret.py +61 -0
- plato/chronos/api/secrets/get_secret.py +62 -0
- plato/chronos/api/secrets/list_secrets.py +57 -0
- plato/chronos/api/secrets/update_secret.py +68 -0
- plato/chronos/api/sessions/__init__.py +10 -0
- plato/chronos/api/sessions/get_session.py +62 -0
- plato/chronos/api/sessions/get_session_logs.py +72 -0
- plato/chronos/api/sessions/get_session_logs_download.py +62 -0
- plato/chronos/api/sessions/list_sessions.py +57 -0
- plato/chronos/api/status/__init__.py +8 -0
- plato/chronos/api/status/get_status_api_status_get.py +44 -0
- plato/chronos/api/status/get_version_info_api_version_get.py +44 -0
- plato/chronos/api/templates/__init__.py +11 -0
- plato/chronos/api/templates/create_template.py +63 -0
- plato/chronos/api/templates/delete_template.py +61 -0
- plato/chronos/api/templates/get_template.py +62 -0
- plato/chronos/api/templates/list_templates.py +57 -0
- plato/chronos/api/templates/update_template.py +68 -0
- plato/chronos/api/trajectories/__init__.py +8 -0
- plato/chronos/api/trajectories/get_trajectory.py +62 -0
- plato/chronos/api/trajectories/list_trajectories.py +62 -0
- plato/chronos/api/worlds/__init__.py +10 -0
- plato/chronos/api/worlds/create_world.py +63 -0
- plato/chronos/api/worlds/delete_world.py +61 -0
- plato/chronos/api/worlds/get_world.py +62 -0
- plato/chronos/api/worlds/list_worlds.py +57 -0
- plato/chronos/client.py +171 -0
- plato/chronos/errors.py +141 -0
- plato/chronos/models/__init__.py +647 -0
- plato/chronos/py.typed +0 -0
- plato/sims/cli.py +299 -123
- plato/sims/registry.py +77 -4
- plato/v1/cli/agent.py +88 -84
- plato/v1/cli/main.py +2 -0
- plato/v1/cli/pm.py +441 -119
- plato/v1/cli/sandbox.py +747 -191
- plato/v1/cli/sim.py +11 -0
- plato/v1/cli/verify.py +1269 -0
- plato/v1/cli/world.py +3 -0
- plato/v1/flow_executor.py +21 -17
- plato/v1/models/env.py +11 -11
- plato/v1/sdk.py +2 -2
- plato/v1/sync_env.py +11 -11
- plato/v1/sync_flow_executor.py +21 -17
- plato/v1/sync_sdk.py +4 -2
- plato/v2/__init__.py +2 -0
- plato/v2/async_/environment.py +20 -1
- plato/v2/async_/session.py +54 -3
- plato/v2/sync/environment.py +2 -1
- plato/v2/sync/session.py +52 -2
- plato/worlds/README.md +218 -0
- plato/worlds/__init__.py +54 -18
- plato/worlds/base.py +304 -93
- plato/worlds/config.py +239 -73
- plato/worlds/runner.py +391 -80
- {plato_sdk_v2-2.0.50.dist-info → plato_sdk_v2-2.2.4.dist-info}/METADATA +1 -3
- {plato_sdk_v2-2.0.50.dist-info → plato_sdk_v2-2.2.4.dist-info}/RECORD +143 -68
- {plato_sdk_v2-2.0.50.dist-info → plato_sdk_v2-2.2.4.dist-info}/entry_points.txt +1 -0
- plato/_generated/api/v2/interfaces/__init__.py +0 -27
- plato/_generated/api/v2/interfaces/v2_interface_browser_create.py +0 -68
- plato/_generated/api/v2/interfaces/v2_interface_cdp_url.py +0 -65
- plato/_generated/api/v2/interfaces/v2_interface_click.py +0 -64
- plato/_generated/api/v2/interfaces/v2_interface_close.py +0 -59
- plato/_generated/api/v2/interfaces/v2_interface_computer_create.py +0 -68
- plato/_generated/api/v2/interfaces/v2_interface_cursor.py +0 -64
- plato/_generated/api/v2/interfaces/v2_interface_key.py +0 -68
- plato/_generated/api/v2/interfaces/v2_interface_screenshot.py +0 -65
- plato/_generated/api/v2/interfaces/v2_interface_scroll.py +0 -70
- plato/_generated/api/v2/interfaces/v2_interface_type.py +0 -64
- plato/world/__init__.py +0 -44
- plato/world/base.py +0 -267
- plato/world/config.py +0 -139
- plato/world/types.py +0 -47
- {plato_sdk_v2-2.0.50.dist-info → plato_sdk_v2-2.2.4.dist-info}/WHEEL +0 -0
plato/__init__.py
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
# v2: New SDK with separate sync/async modules
|
|
5
5
|
# agents: Harbor agent framework re-exports (ClaudeCode, OpenHands, Codex, etc.)
|
|
6
6
|
# sims: Simulation clients (Spree, Firefly, etc.)
|
|
7
|
-
#
|
|
7
|
+
# chronos: Chronos agent evaluation platform SDK
|
|
8
8
|
#
|
|
9
9
|
# Usage (v2 - recommended):
|
|
10
10
|
# from plato.v2 import AsyncPlato, Plato, Env
|
|
@@ -16,8 +16,9 @@
|
|
|
16
16
|
# Usage (sims):
|
|
17
17
|
# from plato.sims import SpreeClient, FireflyClient
|
|
18
18
|
#
|
|
19
|
-
# Usage (
|
|
20
|
-
# from plato.
|
|
19
|
+
# Usage (chronos):
|
|
20
|
+
# from plato.chronos import Client, AsyncClient
|
|
21
|
+
# from plato.chronos.models import AgentCreate, WorldCreate, LaunchJobRequest
|
|
21
22
|
|
|
22
23
|
from importlib.metadata import PackageNotFoundError, version
|
|
23
24
|
|
|
@@ -56,9 +57,9 @@ def __getattr__(name: str):
|
|
|
56
57
|
|
|
57
58
|
return sims
|
|
58
59
|
|
|
59
|
-
if name == "
|
|
60
|
-
|
|
60
|
+
if name == "chronos":
|
|
61
|
+
import importlib
|
|
61
62
|
|
|
62
|
-
return
|
|
63
|
+
return importlib.import_module("plato.chronos")
|
|
63
64
|
|
|
64
65
|
raise AttributeError(f"module 'plato' has no attribute '{name}'")
|
plato/_generated/__init__.py
CHANGED
|
@@ -12,7 +12,7 @@ from plato._generated.models import AppSchemasEnvRequestsEvaluateRequest, AppSch
|
|
|
12
12
|
|
|
13
13
|
def _build_request_args(
|
|
14
14
|
session_id: str,
|
|
15
|
-
body: AppSchemasEnvRequestsEvaluateRequest
|
|
15
|
+
body: AppSchemasEnvRequestsEvaluateRequest,
|
|
16
16
|
authorization: str | None = None,
|
|
17
17
|
x_api_key: str | None = None,
|
|
18
18
|
) -> dict[str, Any]:
|
|
@@ -36,7 +36,7 @@ def _build_request_args(
|
|
|
36
36
|
def sync(
|
|
37
37
|
client: httpx.Client,
|
|
38
38
|
session_id: str,
|
|
39
|
-
body: AppSchemasEnvRequestsEvaluateRequest
|
|
39
|
+
body: AppSchemasEnvRequestsEvaluateRequest,
|
|
40
40
|
authorization: str | None = None,
|
|
41
41
|
x_api_key: str | None = None,
|
|
42
42
|
) -> AppSchemasEnvResponsesEvaluateResponse:
|
|
@@ -57,7 +57,7 @@ def sync(
|
|
|
57
57
|
async def asyncio(
|
|
58
58
|
client: httpx.AsyncClient,
|
|
59
59
|
session_id: str,
|
|
60
|
-
body: AppSchemasEnvRequestsEvaluateRequest
|
|
60
|
+
body: AppSchemasEnvRequestsEvaluateRequest,
|
|
61
61
|
authorization: str | None = None,
|
|
62
62
|
x_api_key: str | None = None,
|
|
63
63
|
) -> AppSchemasEnvResponsesEvaluateResponse:
|
|
@@ -7,12 +7,12 @@ from typing import Any
|
|
|
7
7
|
import httpx
|
|
8
8
|
|
|
9
9
|
from plato._generated.errors import raise_for_status
|
|
10
|
-
from plato._generated.models import
|
|
10
|
+
from plato._generated.models import BaseStructuredRunLog, LogResponse
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def _build_request_args(
|
|
14
14
|
session_id: str,
|
|
15
|
-
body: BaseStructuredRunLog
|
|
15
|
+
body: BaseStructuredRunLog,
|
|
16
16
|
) -> dict[str, Any]:
|
|
17
17
|
"""Build request arguments."""
|
|
18
18
|
url = f"/api/v1/env/{session_id}/log"
|
|
@@ -27,7 +27,7 @@ def _build_request_args(
|
|
|
27
27
|
def sync(
|
|
28
28
|
client: httpx.Client,
|
|
29
29
|
session_id: str,
|
|
30
|
-
body: BaseStructuredRunLog
|
|
30
|
+
body: BaseStructuredRunLog,
|
|
31
31
|
) -> LogResponse:
|
|
32
32
|
"""Log a state mutation or batch of mutations.
|
|
33
33
|
|
|
@@ -47,7 +47,7 @@ def sync(
|
|
|
47
47
|
async def asyncio(
|
|
48
48
|
client: httpx.AsyncClient,
|
|
49
49
|
session_id: str,
|
|
50
|
-
body: BaseStructuredRunLog
|
|
50
|
+
body: BaseStructuredRunLog,
|
|
51
51
|
) -> LogResponse:
|
|
52
52
|
"""Log a state mutation or batch of mutations.
|
|
53
53
|
|
|
@@ -15,7 +15,7 @@ from plato._generated.models import (
|
|
|
15
15
|
|
|
16
16
|
def _build_request_args(
|
|
17
17
|
public_id: str,
|
|
18
|
-
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest
|
|
18
|
+
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest,
|
|
19
19
|
authorization: str | None = None,
|
|
20
20
|
x_api_key: str | None = None,
|
|
21
21
|
) -> dict[str, Any]:
|
|
@@ -39,7 +39,7 @@ def _build_request_args(
|
|
|
39
39
|
def sync(
|
|
40
40
|
client: httpx.Client,
|
|
41
41
|
public_id: str,
|
|
42
|
-
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest
|
|
42
|
+
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest,
|
|
43
43
|
authorization: str | None = None,
|
|
44
44
|
x_api_key: str | None = None,
|
|
45
45
|
) -> AppApiV1PublicBuildRoutesCreateSnapshotResponse:
|
|
@@ -66,7 +66,7 @@ def sync(
|
|
|
66
66
|
async def asyncio(
|
|
67
67
|
client: httpx.AsyncClient,
|
|
68
68
|
public_id: str,
|
|
69
|
-
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest
|
|
69
|
+
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest,
|
|
70
70
|
authorization: str | None = None,
|
|
71
71
|
x_api_key: str | None = None,
|
|
72
72
|
) -> AppApiV1PublicBuildRoutesCreateSnapshotResponse:
|
|
@@ -15,7 +15,7 @@ from plato._generated.models import (
|
|
|
15
15
|
|
|
16
16
|
def _build_request_args(
|
|
17
17
|
public_id: str,
|
|
18
|
-
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest
|
|
18
|
+
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest,
|
|
19
19
|
authorization: str | None = None,
|
|
20
20
|
x_api_key: str | None = None,
|
|
21
21
|
) -> dict[str, Any]:
|
|
@@ -39,7 +39,7 @@ def _build_request_args(
|
|
|
39
39
|
def sync(
|
|
40
40
|
client: httpx.Client,
|
|
41
41
|
public_id: str,
|
|
42
|
-
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest
|
|
42
|
+
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest,
|
|
43
43
|
authorization: str | None = None,
|
|
44
44
|
x_api_key: str | None = None,
|
|
45
45
|
) -> AppApiV1PublicBuildRoutesCreateSnapshotResponse:
|
|
@@ -65,7 +65,7 @@ def sync(
|
|
|
65
65
|
async def asyncio(
|
|
66
66
|
client: httpx.AsyncClient,
|
|
67
67
|
public_id: str,
|
|
68
|
-
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest
|
|
68
|
+
body: AppApiV1PublicBuildRoutesCreateSnapshotRequest,
|
|
69
69
|
authorization: str | None = None,
|
|
70
70
|
x_api_key: str | None = None,
|
|
71
71
|
) -> AppApiV1PublicBuildRoutesCreateSnapshotResponse:
|
|
@@ -7,12 +7,12 @@ from typing import Any
|
|
|
7
7
|
import httpx
|
|
8
8
|
|
|
9
9
|
from plato._generated.errors import raise_for_status
|
|
10
|
-
from plato._generated.models import
|
|
10
|
+
from plato._generated.models import AppSchemasBuildModelsSetupSandboxRequest, AppSchemasBuildModelsSetupSandboxResponse
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def _build_request_args(
|
|
14
14
|
public_id: str,
|
|
15
|
-
body:
|
|
15
|
+
body: AppSchemasBuildModelsSetupSandboxRequest,
|
|
16
16
|
authorization: str | None = None,
|
|
17
17
|
x_api_key: str | None = None,
|
|
18
18
|
) -> dict[str, Any]:
|
|
@@ -36,10 +36,10 @@ def _build_request_args(
|
|
|
36
36
|
def sync(
|
|
37
37
|
client: httpx.Client,
|
|
38
38
|
public_id: str,
|
|
39
|
-
body:
|
|
39
|
+
body: AppSchemasBuildModelsSetupSandboxRequest,
|
|
40
40
|
authorization: str | None = None,
|
|
41
41
|
x_api_key: str | None = None,
|
|
42
|
-
) ->
|
|
42
|
+
) -> AppSchemasBuildModelsSetupSandboxResponse:
|
|
43
43
|
"""Setup sandbox development environment with git clone."""
|
|
44
44
|
|
|
45
45
|
request_args = _build_request_args(
|
|
@@ -51,16 +51,16 @@ def sync(
|
|
|
51
51
|
|
|
52
52
|
response = client.request(**request_args)
|
|
53
53
|
raise_for_status(response)
|
|
54
|
-
return
|
|
54
|
+
return AppSchemasBuildModelsSetupSandboxResponse.model_validate(response.json())
|
|
55
55
|
|
|
56
56
|
|
|
57
57
|
async def asyncio(
|
|
58
58
|
client: httpx.AsyncClient,
|
|
59
59
|
public_id: str,
|
|
60
|
-
body:
|
|
60
|
+
body: AppSchemasBuildModelsSetupSandboxRequest,
|
|
61
61
|
authorization: str | None = None,
|
|
62
62
|
x_api_key: str | None = None,
|
|
63
|
-
) ->
|
|
63
|
+
) -> AppSchemasBuildModelsSetupSandboxResponse:
|
|
64
64
|
"""Setup sandbox development environment with git clone."""
|
|
65
65
|
|
|
66
66
|
request_args = _build_request_args(
|
|
@@ -72,4 +72,4 @@ async def asyncio(
|
|
|
72
72
|
|
|
73
73
|
response = await client.request(**request_args)
|
|
74
74
|
raise_for_status(response)
|
|
75
|
-
return
|
|
75
|
+
return AppSchemasBuildModelsSetupSandboxResponse.model_validate(response.json())
|
|
@@ -21,6 +21,7 @@ from . import (
|
|
|
21
21
|
get_session,
|
|
22
22
|
get_session_replays,
|
|
23
23
|
get_sessions,
|
|
24
|
+
get_sessions_for_archival,
|
|
24
25
|
get_ui_events,
|
|
25
26
|
restore_session_logs,
|
|
26
27
|
score_session_with_config,
|
|
@@ -57,5 +58,6 @@ __all__ = [
|
|
|
57
58
|
"restore_session_logs",
|
|
58
59
|
"get_logs_status",
|
|
59
60
|
"archive_session_logs_endpoint",
|
|
61
|
+
"get_sessions_for_archival",
|
|
60
62
|
"set_session_output",
|
|
61
63
|
]
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Get Sessions For Archival"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from plato._generated.errors import raise_for_status
|
|
10
|
+
from plato._generated.models import SessionArchivalListResponse
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _build_request_args(
|
|
14
|
+
page: int | None = 1,
|
|
15
|
+
page_size: int | None = 10000,
|
|
16
|
+
only_unarchived: bool | None = False,
|
|
17
|
+
authorization: str | None = None,
|
|
18
|
+
x_api_key: str | None = None,
|
|
19
|
+
) -> dict[str, Any]:
|
|
20
|
+
"""Build request arguments."""
|
|
21
|
+
url = "/api/v1/session/archival/list"
|
|
22
|
+
|
|
23
|
+
params: dict[str, Any] = {}
|
|
24
|
+
if page is not None:
|
|
25
|
+
params["page"] = page
|
|
26
|
+
if page_size is not None:
|
|
27
|
+
params["page_size"] = page_size
|
|
28
|
+
if only_unarchived is not None:
|
|
29
|
+
params["only_unarchived"] = only_unarchived
|
|
30
|
+
|
|
31
|
+
headers: dict[str, str] = {}
|
|
32
|
+
if authorization is not None:
|
|
33
|
+
headers["authorization"] = authorization
|
|
34
|
+
if x_api_key is not None:
|
|
35
|
+
headers["X-API-Key"] = x_api_key
|
|
36
|
+
|
|
37
|
+
return {
|
|
38
|
+
"method": "GET",
|
|
39
|
+
"url": url,
|
|
40
|
+
"params": params,
|
|
41
|
+
"headers": headers,
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def sync(
|
|
46
|
+
client: httpx.Client,
|
|
47
|
+
page: int | None = 1,
|
|
48
|
+
page_size: int | None = 10000,
|
|
49
|
+
only_unarchived: bool | None = False,
|
|
50
|
+
authorization: str | None = None,
|
|
51
|
+
x_api_key: str | None = None,
|
|
52
|
+
) -> SessionArchivalListResponse:
|
|
53
|
+
"""Lightweight endpoint to list sessions with archival-relevant fields only.
|
|
54
|
+
Requires admin access.
|
|
55
|
+
|
|
56
|
+
Returns minimal data (no joins) for fast bulk queries:
|
|
57
|
+
- session_id, created_at, logs_archived_at, organization_id, simulator_id
|
|
58
|
+
|
|
59
|
+
Use only_unarchived=true to filter to sessions that haven't been archived yet."""
|
|
60
|
+
|
|
61
|
+
request_args = _build_request_args(
|
|
62
|
+
page=page,
|
|
63
|
+
page_size=page_size,
|
|
64
|
+
only_unarchived=only_unarchived,
|
|
65
|
+
authorization=authorization,
|
|
66
|
+
x_api_key=x_api_key,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
response = client.request(**request_args)
|
|
70
|
+
raise_for_status(response)
|
|
71
|
+
return SessionArchivalListResponse.model_validate(response.json())
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
async def asyncio(
|
|
75
|
+
client: httpx.AsyncClient,
|
|
76
|
+
page: int | None = 1,
|
|
77
|
+
page_size: int | None = 10000,
|
|
78
|
+
only_unarchived: bool | None = False,
|
|
79
|
+
authorization: str | None = None,
|
|
80
|
+
x_api_key: str | None = None,
|
|
81
|
+
) -> SessionArchivalListResponse:
|
|
82
|
+
"""Lightweight endpoint to list sessions with archival-relevant fields only.
|
|
83
|
+
Requires admin access.
|
|
84
|
+
|
|
85
|
+
Returns minimal data (no joins) for fast bulk queries:
|
|
86
|
+
- session_id, created_at, logs_archived_at, organization_id, simulator_id
|
|
87
|
+
|
|
88
|
+
Use only_unarchived=true to filter to sessions that haven't been archived yet."""
|
|
89
|
+
|
|
90
|
+
request_args = _build_request_args(
|
|
91
|
+
page=page,
|
|
92
|
+
page_size=page_size,
|
|
93
|
+
only_unarchived=only_unarchived,
|
|
94
|
+
authorization=authorization,
|
|
95
|
+
x_api_key=x_api_key,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
response = await client.request(**request_args)
|
|
99
|
+
raise_for_status(response)
|
|
100
|
+
return SessionArchivalListResponse.model_validate(response.json())
|
|
@@ -20,9 +20,11 @@ from . import (
|
|
|
20
20
|
get_human_recorder_tab_counts,
|
|
21
21
|
get_human_recorder_testcases,
|
|
22
22
|
get_merged_mutations,
|
|
23
|
-
|
|
23
|
+
get_mutation_groups_for_testcase,
|
|
24
|
+
get_next_testcase_for_scoring,
|
|
24
25
|
get_organization_test_case_assignments,
|
|
25
26
|
get_testcase,
|
|
27
|
+
get_testcase_metadata_for_scoring,
|
|
26
28
|
get_testcase_sets,
|
|
27
29
|
get_testcases,
|
|
28
30
|
get_testcases_in_set,
|
|
@@ -35,12 +37,14 @@ __all__ = [
|
|
|
35
37
|
"get_testcases",
|
|
36
38
|
"create_testcase",
|
|
37
39
|
"get_merged_mutations",
|
|
40
|
+
"get_testcase_metadata_for_scoring",
|
|
38
41
|
"get_testcase_sets",
|
|
39
42
|
"create_testcase_set",
|
|
40
43
|
"get_organization_test_case_assignments",
|
|
41
44
|
"archive_organization_test_case_assignments",
|
|
42
45
|
"get_human_recorder_testcases",
|
|
43
|
-
"
|
|
46
|
+
"get_next_testcase_for_scoring",
|
|
47
|
+
"get_mutation_groups_for_testcase",
|
|
44
48
|
"filter_human_recorder_testcases",
|
|
45
49
|
"get_testcase",
|
|
46
50
|
"update_testcase",
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Get Mutation Groups For Testcase"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from plato._generated.errors import raise_for_status
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _build_request_args(
|
|
13
|
+
public_id: str,
|
|
14
|
+
session_ids: str | None = None,
|
|
15
|
+
authorization: str | None = None,
|
|
16
|
+
x_api_key: str | None = None,
|
|
17
|
+
) -> dict[str, Any]:
|
|
18
|
+
"""Build request arguments."""
|
|
19
|
+
url = f"/api/v1/testcases/{public_id}/mutation-groups"
|
|
20
|
+
|
|
21
|
+
params: dict[str, Any] = {}
|
|
22
|
+
if session_ids is not None:
|
|
23
|
+
params["session_ids"] = session_ids
|
|
24
|
+
|
|
25
|
+
headers: dict[str, str] = {}
|
|
26
|
+
if authorization is not None:
|
|
27
|
+
headers["authorization"] = authorization
|
|
28
|
+
if x_api_key is not None:
|
|
29
|
+
headers["X-API-Key"] = x_api_key
|
|
30
|
+
|
|
31
|
+
return {
|
|
32
|
+
"method": "GET",
|
|
33
|
+
"url": url,
|
|
34
|
+
"params": params,
|
|
35
|
+
"headers": headers,
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def sync(
|
|
40
|
+
client: httpx.Client,
|
|
41
|
+
public_id: str,
|
|
42
|
+
session_ids: str | None = None,
|
|
43
|
+
authorization: str | None = None,
|
|
44
|
+
x_api_key: str | None = None,
|
|
45
|
+
) -> Any:
|
|
46
|
+
"""Get mutation groups for a test case using generated scoring configs.
|
|
47
|
+
|
|
48
|
+
This uses AI-based config generation to group sessions by their mutation patterns,
|
|
49
|
+
properly handling things like random IDs (mutation_variables) that differ between
|
|
50
|
+
sessions but represent the same logical mutation.
|
|
51
|
+
|
|
52
|
+
Returns groups of session IDs where sessions in each group have equivalent mutations.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
session_ids: Optional comma-separated list of session IDs to process.
|
|
56
|
+
If not provided, will process up to 50 most recent sessions."""
|
|
57
|
+
|
|
58
|
+
request_args = _build_request_args(
|
|
59
|
+
public_id=public_id,
|
|
60
|
+
session_ids=session_ids,
|
|
61
|
+
authorization=authorization,
|
|
62
|
+
x_api_key=x_api_key,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
response = client.request(**request_args)
|
|
66
|
+
raise_for_status(response)
|
|
67
|
+
return response.json()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
async def asyncio(
|
|
71
|
+
client: httpx.AsyncClient,
|
|
72
|
+
public_id: str,
|
|
73
|
+
session_ids: str | None = None,
|
|
74
|
+
authorization: str | None = None,
|
|
75
|
+
x_api_key: str | None = None,
|
|
76
|
+
) -> Any:
|
|
77
|
+
"""Get mutation groups for a test case using generated scoring configs.
|
|
78
|
+
|
|
79
|
+
This uses AI-based config generation to group sessions by their mutation patterns,
|
|
80
|
+
properly handling things like random IDs (mutation_variables) that differ between
|
|
81
|
+
sessions but represent the same logical mutation.
|
|
82
|
+
|
|
83
|
+
Returns groups of session IDs where sessions in each group have equivalent mutations.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
session_ids: Optional comma-separated list of session IDs to process.
|
|
87
|
+
If not provided, will process up to 50 most recent sessions."""
|
|
88
|
+
|
|
89
|
+
request_args = _build_request_args(
|
|
90
|
+
public_id=public_id,
|
|
91
|
+
session_ids=session_ids,
|
|
92
|
+
authorization=authorization,
|
|
93
|
+
x_api_key=x_api_key,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
response = await client.request(**request_args)
|
|
97
|
+
raise_for_status(response)
|
|
98
|
+
return response.json()
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Get Next
|
|
1
|
+
"""Get Next Testcase For Scoring"""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -10,14 +10,17 @@ from plato._generated.errors import raise_for_status
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
def _build_request_args(
|
|
13
|
+
scoring_type: str,
|
|
13
14
|
exclude_test_case_id: str | None = None,
|
|
14
15
|
authorization: str | None = None,
|
|
15
16
|
x_api_key: str | None = None,
|
|
16
17
|
) -> dict[str, Any]:
|
|
17
18
|
"""Build request arguments."""
|
|
18
|
-
url = "/api/v1/testcases/next-for-
|
|
19
|
+
url = "/api/v1/testcases/next-for-scoring"
|
|
19
20
|
|
|
20
21
|
params: dict[str, Any] = {}
|
|
22
|
+
if scoring_type is not None:
|
|
23
|
+
params["scoring_type"] = scoring_type
|
|
21
24
|
if exclude_test_case_id is not None:
|
|
22
25
|
params["exclude_test_case_id"] = exclude_test_case_id
|
|
23
26
|
|
|
@@ -37,16 +40,21 @@ def _build_request_args(
|
|
|
37
40
|
|
|
38
41
|
def sync(
|
|
39
42
|
client: httpx.Client,
|
|
43
|
+
scoring_type: str,
|
|
40
44
|
exclude_test_case_id: str | None = None,
|
|
41
45
|
authorization: str | None = None,
|
|
42
46
|
x_api_key: str | None = None,
|
|
43
47
|
) -> Any:
|
|
44
|
-
"""Get the next
|
|
45
|
-
Returns test cases in NEEDS_SCORING stage with
|
|
46
|
-
Excludes test cases where the user has already scored
|
|
47
|
-
Optionally excludes a specific test case (e.g., the current one).
|
|
48
|
+
"""Get the next test case that needs scoring for the current annotator.
|
|
49
|
+
Returns test cases in NEEDS_SCORING stage with the specified scoring_type.
|
|
50
|
+
Excludes test cases where the user has already scored or created sessions.
|
|
51
|
+
Optionally excludes a specific test case (e.g., the current one).
|
|
52
|
+
|
|
53
|
+
Ordering: Prioritizes test cases with higher average scores per session (closer to completion).
|
|
54
|
+
Selection: Random from batch of 50 eligible test cases."""
|
|
48
55
|
|
|
49
56
|
request_args = _build_request_args(
|
|
57
|
+
scoring_type=scoring_type,
|
|
50
58
|
exclude_test_case_id=exclude_test_case_id,
|
|
51
59
|
authorization=authorization,
|
|
52
60
|
x_api_key=x_api_key,
|
|
@@ -59,16 +67,21 @@ def sync(
|
|
|
59
67
|
|
|
60
68
|
async def asyncio(
|
|
61
69
|
client: httpx.AsyncClient,
|
|
70
|
+
scoring_type: str,
|
|
62
71
|
exclude_test_case_id: str | None = None,
|
|
63
72
|
authorization: str | None = None,
|
|
64
73
|
x_api_key: str | None = None,
|
|
65
74
|
) -> Any:
|
|
66
|
-
"""Get the next
|
|
67
|
-
Returns test cases in NEEDS_SCORING stage with
|
|
68
|
-
Excludes test cases where the user has already scored
|
|
69
|
-
Optionally excludes a specific test case (e.g., the current one).
|
|
75
|
+
"""Get the next test case that needs scoring for the current annotator.
|
|
76
|
+
Returns test cases in NEEDS_SCORING stage with the specified scoring_type.
|
|
77
|
+
Excludes test cases where the user has already scored or created sessions.
|
|
78
|
+
Optionally excludes a specific test case (e.g., the current one).
|
|
79
|
+
|
|
80
|
+
Ordering: Prioritizes test cases with higher average scores per session (closer to completion).
|
|
81
|
+
Selection: Random from batch of 50 eligible test cases."""
|
|
70
82
|
|
|
71
83
|
request_args = _build_request_args(
|
|
84
|
+
scoring_type=scoring_type,
|
|
72
85
|
exclude_test_case_id=exclude_test_case_id,
|
|
73
86
|
authorization=authorization,
|
|
74
87
|
x_api_key=x_api_key,
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Get Testcase Metadata For Scoring"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from plato._generated.errors import raise_for_status
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _build_request_args(
|
|
13
|
+
public_id: str,
|
|
14
|
+
authorization: str | None = None,
|
|
15
|
+
x_api_key: str | None = None,
|
|
16
|
+
) -> dict[str, Any]:
|
|
17
|
+
"""Build request arguments."""
|
|
18
|
+
url = f"/api/v1/testcases/metadata-for-scoring/{public_id}"
|
|
19
|
+
|
|
20
|
+
headers: dict[str, str] = {}
|
|
21
|
+
if authorization is not None:
|
|
22
|
+
headers["authorization"] = authorization
|
|
23
|
+
if x_api_key is not None:
|
|
24
|
+
headers["X-API-Key"] = x_api_key
|
|
25
|
+
|
|
26
|
+
return {
|
|
27
|
+
"method": "GET",
|
|
28
|
+
"url": url,
|
|
29
|
+
"headers": headers,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def sync(
|
|
34
|
+
client: httpx.Client,
|
|
35
|
+
public_id: str,
|
|
36
|
+
authorization: str | None = None,
|
|
37
|
+
x_api_key: str | None = None,
|
|
38
|
+
) -> Any:
|
|
39
|
+
"""Get limited test case metadata for the scoring page.
|
|
40
|
+
Only returns: publicId, name, scoringTypes.
|
|
41
|
+
This endpoint is designed for annotators who should not see sensitive test case data
|
|
42
|
+
like prompts, hints, or expected outputs."""
|
|
43
|
+
|
|
44
|
+
request_args = _build_request_args(
|
|
45
|
+
public_id=public_id,
|
|
46
|
+
authorization=authorization,
|
|
47
|
+
x_api_key=x_api_key,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
response = client.request(**request_args)
|
|
51
|
+
raise_for_status(response)
|
|
52
|
+
return response.json()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
async def asyncio(
|
|
56
|
+
client: httpx.AsyncClient,
|
|
57
|
+
public_id: str,
|
|
58
|
+
authorization: str | None = None,
|
|
59
|
+
x_api_key: str | None = None,
|
|
60
|
+
) -> Any:
|
|
61
|
+
"""Get limited test case metadata for the scoring page.
|
|
62
|
+
Only returns: publicId, name, scoringTypes.
|
|
63
|
+
This endpoint is designed for annotators who should not see sensitive test case data
|
|
64
|
+
like prompts, hints, or expected outputs."""
|
|
65
|
+
|
|
66
|
+
request_args = _build_request_args(
|
|
67
|
+
public_id=public_id,
|
|
68
|
+
authorization=authorization,
|
|
69
|
+
x_api_key=x_api_key,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
response = await client.request(**request_args)
|
|
73
|
+
raise_for_status(response)
|
|
74
|
+
return response.json()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""API version module."""
|
|
2
2
|
|
|
3
|
-
from . import agents, artifacts, chronos, chronos_packages, cluster, jobs, pypi, releases, sessions
|
|
3
|
+
from . import agents, artifacts, chronos, chronos_packages, cluster, jobs, pypi, releases, sessions, user
|
|
4
4
|
|
|
5
5
|
__all__ = [
|
|
6
6
|
"agents",
|
|
@@ -12,4 +12,5 @@ __all__ = [
|
|
|
12
12
|
"pypi",
|
|
13
13
|
"releases",
|
|
14
14
|
"sessions",
|
|
15
|
+
"user",
|
|
15
16
|
]
|
|
@@ -14,7 +14,9 @@ from . import (
|
|
|
14
14
|
public_url,
|
|
15
15
|
reset,
|
|
16
16
|
set_date,
|
|
17
|
+
setup_sandbox,
|
|
17
18
|
snapshot,
|
|
19
|
+
snapshot_store,
|
|
18
20
|
state,
|
|
19
21
|
wait_for_ready,
|
|
20
22
|
)
|
|
@@ -25,7 +27,9 @@ __all__ = [
|
|
|
25
27
|
"close",
|
|
26
28
|
"execute",
|
|
27
29
|
"set_date",
|
|
30
|
+
"setup_sandbox",
|
|
28
31
|
"snapshot",
|
|
32
|
+
"snapshot_store",
|
|
29
33
|
"checkpoint",
|
|
30
34
|
"disk_snapshot",
|
|
31
35
|
"state",
|
|
@@ -12,7 +12,7 @@ from plato._generated.models import CreateCheckpointRequest, CreateCheckpointRes
|
|
|
12
12
|
|
|
13
13
|
def _build_request_args(
|
|
14
14
|
job_id: str,
|
|
15
|
-
body: CreateCheckpointRequest
|
|
15
|
+
body: CreateCheckpointRequest,
|
|
16
16
|
authorization: str | None = None,
|
|
17
17
|
x_api_key: str | None = None,
|
|
18
18
|
) -> dict[str, Any]:
|
|
@@ -36,7 +36,7 @@ def _build_request_args(
|
|
|
36
36
|
def sync(
|
|
37
37
|
client: httpx.Client,
|
|
38
38
|
job_id: str,
|
|
39
|
-
body: CreateCheckpointRequest
|
|
39
|
+
body: CreateCheckpointRequest,
|
|
40
40
|
authorization: str | None = None,
|
|
41
41
|
x_api_key: str | None = None,
|
|
42
42
|
) -> CreateCheckpointResult:
|
|
@@ -66,7 +66,7 @@ def sync(
|
|
|
66
66
|
async def asyncio(
|
|
67
67
|
client: httpx.AsyncClient,
|
|
68
68
|
job_id: str,
|
|
69
|
-
body: CreateCheckpointRequest
|
|
69
|
+
body: CreateCheckpointRequest,
|
|
70
70
|
authorization: str | None = None,
|
|
71
71
|
x_api_key: str | None = None,
|
|
72
72
|
) -> CreateCheckpointResult:
|