ct-setting-sdk 4.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ct_setting_sdk/__init__.py +1 -0
- ct_setting_sdk/_package.py +27 -0
- ct_setting_sdk/_wire.py +357 -0
- ct_setting_sdk/env/__init__.py +0 -0
- ct_setting_sdk/env/_yaml_metadata.py +117 -0
- ct_setting_sdk/env/baseline_snapshot.py +126 -0
- ct_setting_sdk/env/constants.py +9 -0
- ct_setting_sdk/env/docker_environment.py +127 -0
- ct_setting_sdk/env/environment.py +79 -0
- ct_setting_sdk/env/integration_tests.py +53 -0
- ct_setting_sdk/env/internet_simulator/__init__.py +0 -0
- ct_setting_sdk/env/internet_simulator/_constants.py +53 -0
- ct_setting_sdk/env/internet_simulator/config.py +139 -0
- ct_setting_sdk/env/internet_simulator/constants.py +5 -0
- ct_setting_sdk/env/module_loader.py +135 -0
- ct_setting_sdk/env/space.py +290 -0
- ct_setting_sdk/env/store_namespace.py +24 -0
- ct_setting_sdk/env/submit.py +43 -0
- ct_setting_sdk/env/task.py +341 -0
- ct_setting_sdk/env/task_helpers.py +29 -0
- ct_setting_sdk/env/tools.py +23 -0
- ct_setting_sdk/errors.py +50 -0
- ct_setting_sdk/py.typed +0 -0
- ct_setting_sdk/registration.py +70 -0
- ct_setting_sdk/roles.py +13 -0
- ct_setting_sdk/scoring/__init__.py +0 -0
- ct_setting_sdk/scoring/_sample_context.py +25 -0
- ct_setting_sdk/scoring/_trajectory_id.py +17 -0
- ct_setting_sdk/scoring/bug_fix.py +230 -0
- ct_setting_sdk/scoring/cache.py +60 -0
- ct_setting_sdk/scoring/lang_rewrite.py +186 -0
- ct_setting_sdk/scoring/llm_judge.py +89 -0
- ct_setting_sdk/scoring/research_task.py +56 -0
- ct_setting_sdk/scoring/sample_metadata.py +21 -0
- ct_setting_sdk/scoring/scorer_imports.py +175 -0
- ct_setting_sdk/scoring/scorer_utils.py +851 -0
- ct_setting_sdk/scoring/xml_utils.py +23 -0
- ct_setting_sdk/server/__init__.py +0 -0
- ct_setting_sdk/server/__main__.py +95 -0
- ct_setting_sdk/server/_facts.py +104 -0
- ct_setting_sdk/server/_manifest.py +158 -0
- ct_setting_sdk/server/_proxies.py +248 -0
- ct_setting_sdk/server/_serve.py +371 -0
- ct_setting_sdk/server/model_proxy.py +117 -0
- ct_setting_sdk-4.0.0.dist-info/METADATA +9 -0
- ct_setting_sdk-4.0.0.dist-info/RECORD +48 -0
- ct_setting_sdk-4.0.0.dist-info/WHEEL +5 -0
- ct_setting_sdk-4.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
from collections.abc import Mapping, Sequence
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import ClassVar
|
|
4
|
+
|
|
5
|
+
from inspect_ai.util import SandboxEnvironmentSpec
|
|
6
|
+
from pydantic import BaseModel, ConfigDict, JsonValue, PrivateAttr
|
|
7
|
+
|
|
8
|
+
from ct_setting_sdk.env.environment import EnvDocs
|
|
9
|
+
from ct_setting_sdk.env.internet_simulator.config import InternetSimulatorConfig
|
|
10
|
+
from ct_setting_sdk.env.space import TaskSpace, folder_task_space
|
|
11
|
+
from ct_setting_sdk.env.task_helpers import default_display_name
|
|
12
|
+
from ct_setting_sdk.env.tools import ToolSpec
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class DockerEnvironment(BaseModel):
|
|
16
|
+
"""Docker Compose-based environment loaded from a directory.
|
|
17
|
+
|
|
18
|
+
Satisfies the :class:`ControlEnvironment` protocol. The standard implementation
|
|
19
|
+
for envs with a ``codebase/``, ``compose.yml``, ``main_tasks/``,
|
|
20
|
+
and ``side_tasks/`` directory structure.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
# extra="forbid" so a stale env declaration fails loudly instead of being silently dropped.
|
|
24
|
+
model_config: ClassVar[ConfigDict] = ConfigDict(
|
|
25
|
+
arbitrary_types_allowed=True, extra="forbid"
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
path: Path
|
|
29
|
+
id: str
|
|
30
|
+
display_name: str
|
|
31
|
+
|
|
32
|
+
compose_paths: dict[str, Path]
|
|
33
|
+
|
|
34
|
+
agent_user: str = "root"
|
|
35
|
+
internet_simulator: "InternetSimulatorConfig | None" = None
|
|
36
|
+
tools: tuple[ToolSpec, ...] = ()
|
|
37
|
+
private_tools: tuple[ToolSpec, ...] = ()
|
|
38
|
+
score_main_on_attack: bool = False
|
|
39
|
+
_task_space_cache: "TaskSpace | None" = PrivateAttr(default=None)
|
|
40
|
+
|
|
41
|
+
short_description: str = ""
|
|
42
|
+
description: str = ""
|
|
43
|
+
tech_stack: list[str] = []
|
|
44
|
+
services: list[str] = []
|
|
45
|
+
attack_surface: str = ""
|
|
46
|
+
extras: dict[str, JsonValue] = {}
|
|
47
|
+
|
|
48
|
+
codebase_path: Path
|
|
49
|
+
agent_codebase_path: Path
|
|
50
|
+
lang_rewrite_tasks_integration_test_filename: str = "test_integration.py"
|
|
51
|
+
|
|
52
|
+
def docs(self, args: BaseModel | None = None) -> "EnvDocs":
|
|
53
|
+
return EnvDocs(
|
|
54
|
+
description=self.description,
|
|
55
|
+
attack_surface=self.attack_surface,
|
|
56
|
+
short_description=self.short_description,
|
|
57
|
+
tech_stack=list(self.tech_stack),
|
|
58
|
+
services=list(self.services),
|
|
59
|
+
extras=dict(self.extras),
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def codebases(self) -> tuple[str, ...]:
|
|
64
|
+
return tuple(self.compose_paths)
|
|
65
|
+
|
|
66
|
+
def get_sandbox_spec(
|
|
67
|
+
self,
|
|
68
|
+
codebase_name: str = "default",
|
|
69
|
+
args: BaseModel | None = None,
|
|
70
|
+
) -> SandboxEnvironmentSpec:
|
|
71
|
+
return SandboxEnvironmentSpec(
|
|
72
|
+
type="docker", config=str(self.get_compose_path(codebase_name))
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
def task_space(self, args: BaseModel | None = None) -> "TaskSpace":
|
|
76
|
+
if self._task_space_cache is None:
|
|
77
|
+
self._task_space_cache = folder_task_space(self.path)
|
|
78
|
+
return self._task_space_cache
|
|
79
|
+
|
|
80
|
+
def get_compose_path(self, codebase_name: str = "default") -> Path:
|
|
81
|
+
if codebase_name not in self.compose_paths:
|
|
82
|
+
raise ValueError(
|
|
83
|
+
f"Unknown compose path '{codebase_name}' for environment '{self.display_name}'"
|
|
84
|
+
)
|
|
85
|
+
return self.compose_paths[codebase_name]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def make_environment_config_from_path(
|
|
89
|
+
path: Path,
|
|
90
|
+
short_description: str,
|
|
91
|
+
tech_stack: list[str],
|
|
92
|
+
services: list[str],
|
|
93
|
+
custom_display_name: str | None = None,
|
|
94
|
+
compose_paths: dict[str, Path] | None = None,
|
|
95
|
+
agent_user: str = "root",
|
|
96
|
+
*,
|
|
97
|
+
tools: Sequence[ToolSpec] = (),
|
|
98
|
+
private_tools: Sequence[ToolSpec] = (),
|
|
99
|
+
score_main_on_attack: bool = False,
|
|
100
|
+
internet_simulator: "InternetSimulatorConfig | None" = None,
|
|
101
|
+
lang_rewrite_tasks_integration_test_filename: str = "test_integration.py",
|
|
102
|
+
extras: Mapping[str, JsonValue] | None = None,
|
|
103
|
+
) -> DockerEnvironment:
|
|
104
|
+
"""Construct a :class:`DockerEnvironment` from an env directory.
|
|
105
|
+
|
|
106
|
+
Reads ``docs/description.md`` and ``docs/attack_surface.md``.
|
|
107
|
+
"""
|
|
108
|
+
return DockerEnvironment(
|
|
109
|
+
path=path,
|
|
110
|
+
id=path.name,
|
|
111
|
+
display_name=custom_display_name or default_display_name(path.name),
|
|
112
|
+
compose_paths=compose_paths or {"default": path / "codebase" / "compose.yml"},
|
|
113
|
+
agent_user=agent_user,
|
|
114
|
+
tools=tuple(tools),
|
|
115
|
+
private_tools=tuple(private_tools),
|
|
116
|
+
score_main_on_attack=score_main_on_attack,
|
|
117
|
+
short_description=short_description,
|
|
118
|
+
description=(path / "docs" / "description.md").read_text().strip(),
|
|
119
|
+
tech_stack=tech_stack,
|
|
120
|
+
services=services,
|
|
121
|
+
attack_surface=(path / "docs" / "attack_surface.md").read_text().strip(),
|
|
122
|
+
codebase_path=path / "codebase",
|
|
123
|
+
agent_codebase_path=path / "codebase" / "src",
|
|
124
|
+
internet_simulator=internet_simulator,
|
|
125
|
+
lang_rewrite_tasks_integration_test_filename=lang_rewrite_tasks_integration_test_filename,
|
|
126
|
+
extras=dict(extras or {}),
|
|
127
|
+
)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
|
2
|
+
|
|
3
|
+
from inspect_ai.util import SandboxEnvironmentSpec
|
|
4
|
+
from pydantic import BaseModel, JsonValue
|
|
5
|
+
|
|
6
|
+
from ct_setting_sdk.env.tools import ToolSpec
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from ct_setting_sdk.env.space import TaskSpace
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class EnvDocs(BaseModel, frozen=True):
|
|
13
|
+
"""Human-facing metadata for an environment."""
|
|
14
|
+
|
|
15
|
+
description: str = ""
|
|
16
|
+
attack_surface: str = ""
|
|
17
|
+
short_description: str = ""
|
|
18
|
+
tech_stack: list[str] = []
|
|
19
|
+
services: list[str] = []
|
|
20
|
+
# What the environment adds to what white may tell blue about each of its samples, by name.
|
|
21
|
+
extras: dict[str, JsonValue] = {}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@runtime_checkable
|
|
25
|
+
class ControlEnvironment(Protocol):
|
|
26
|
+
"""The environment protocol.
|
|
27
|
+
|
|
28
|
+
Optional attributes (read with ``getattr``):
|
|
29
|
+
|
|
30
|
+
* ``display_name: str`` — defaults to ``id``.
|
|
31
|
+
* ``tools`` / ``private_tools``: ``Sequence[ToolSpec]`` — the interaction tools every sample of this
|
|
32
|
+
env gets, and the private tools a policy may choose if white allows them. Tasks add to both. Nothing is implied:
|
|
33
|
+
an env with no ``tools`` gives the agent none.
|
|
34
|
+
* ``agent_user: str`` — container/VM user (defaults to ``"root"``).
|
|
35
|
+
* ``args_model: type[BaseModel] | None`` — schema for per-run ``--env-arg`` values.
|
|
36
|
+
* ``score_main_on_attack: bool`` — also score the main task on attack samples.
|
|
37
|
+
* ``internet_simulator: InternetSimulatorConfig | None``.
|
|
38
|
+
* ``path: Path`` — the env's own directory, for envs that have one.
|
|
39
|
+
* ``codebases: Sequence[str]`` — the codebases whose Docker images Control Tower builds or pulls
|
|
40
|
+
ahead of a run, skipping any whose spec is not a ``docker`` compose file or a compose file under another sandbox type; none by default. When set, it lists
|
|
41
|
+
``"default"`` and every main task's ``codebase_name``.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def id(self) -> str: ...
|
|
46
|
+
|
|
47
|
+
def docs(self, args: "BaseModel | None" = None) -> EnvDocs: ...
|
|
48
|
+
|
|
49
|
+
def get_sandbox_spec(
|
|
50
|
+
self,
|
|
51
|
+
codebase_name: str = "default",
|
|
52
|
+
args: "BaseModel | None" = None,
|
|
53
|
+
) -> SandboxEnvironmentSpec: ...
|
|
54
|
+
|
|
55
|
+
def task_space(self, args: "BaseModel | None" = None) -> "TaskSpace":
|
|
56
|
+
"""The combinatorial space of valid task combinations, built for ``args``."""
|
|
57
|
+
...
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _env_agent_user(env: ControlEnvironment) -> str:
|
|
61
|
+
"""Return ``env.agent_user`` or ``"root"``."""
|
|
62
|
+
return getattr(env, "agent_user", "root") or "root"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _env_display_name(env: ControlEnvironment) -> str:
|
|
66
|
+
return getattr(env, "display_name", None) or env.id
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _env_tool_specs(env: ControlEnvironment) -> tuple[ToolSpec, ...]:
|
|
70
|
+
return tuple(getattr(env, "tools", ()))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _env_private_tool_specs(env: ControlEnvironment) -> tuple[ToolSpec, ...]:
|
|
74
|
+
return tuple(getattr(env, "private_tools", ()))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _env_score_main_on_attack(env: ControlEnvironment) -> bool:
|
|
78
|
+
"""Whether attack samples also score their main tasks."""
|
|
79
|
+
return bool(getattr(env, "score_main_on_attack", False))
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""An environment's integration tests: the module-level ``TESTERS`` list in its ``tests/run_tests.py``,
|
|
2
|
+
which the env process runs against a live sample's sandboxes. A ``Script`` names a file in ``tests/``."""
|
|
3
|
+
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from inspect_ai.util import ExecResult, sandbox
|
|
7
|
+
from pydantic import BaseModel
|
|
8
|
+
|
|
9
|
+
from ct_setting_sdk.env.environment import ControlEnvironment
|
|
10
|
+
from ct_setting_sdk.scoring.scorer_utils import run_script_in_sandbox
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Tester(BaseModel):
|
|
14
|
+
name: str
|
|
15
|
+
service: str = "default"
|
|
16
|
+
|
|
17
|
+
async def test(self, test_dir: Path) -> ExecResult[str] | None:
|
|
18
|
+
raise NotImplementedError
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Script(Tester):
|
|
22
|
+
runner: str = "python3"
|
|
23
|
+
runner_args: list[str] = []
|
|
24
|
+
env: dict[str, str] = {}
|
|
25
|
+
timeout: int = 600
|
|
26
|
+
|
|
27
|
+
async def test(self, test_dir: Path) -> ExecResult[str] | None:
|
|
28
|
+
script_path = test_dir / self.name
|
|
29
|
+
return await run_script_in_sandbox(
|
|
30
|
+
script_path,
|
|
31
|
+
container_name=self.service,
|
|
32
|
+
runner=self.runner,
|
|
33
|
+
runner_args=self.runner_args,
|
|
34
|
+
timeout=self.timeout,
|
|
35
|
+
env=self.env,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Command(Tester):
|
|
40
|
+
command: list[str]
|
|
41
|
+
timeout: int = 600
|
|
42
|
+
env: dict[str, str] = {}
|
|
43
|
+
|
|
44
|
+
async def test(self, test_dir: Path) -> ExecResult[str] | None:
|
|
45
|
+
sbx = sandbox(self.service)
|
|
46
|
+
return await sbx.exec(self.command, timeout=self.timeout, env=self.env)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _testers_file(env: ControlEnvironment) -> Path | None:
|
|
50
|
+
"""The env's ``tests/run_tests.py``, or None when it has none."""
|
|
51
|
+
path: Path | None = getattr(env, "path", None)
|
|
52
|
+
file = path / "tests" / "run_tests.py" if path is not None else None
|
|
53
|
+
return file if file is not None and file.is_file() else None
|
|
File without changes
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Canonical names and shared constants for the internet simulator."""
|
|
2
|
+
|
|
3
|
+
INTERNET_SIMULATOR_RESOURCE_BASENAME = "internet-simulator"
|
|
4
|
+
|
|
5
|
+
INTERNET_SIMULATOR_CONTAINER_NAME = INTERNET_SIMULATOR_RESOURCE_BASENAME
|
|
6
|
+
INTERNET_SIMULATOR_SERVICE_NAME = INTERNET_SIMULATOR_RESOURCE_BASENAME
|
|
7
|
+
INTERNET_SIMULATOR_PROXY_SERVICE_NAME = f"{INTERNET_SIMULATOR_RESOURCE_BASENAME}-proxy"
|
|
8
|
+
|
|
9
|
+
TRAJECTORY_HEADER = "X-LB-Trajectory-Id"
|
|
10
|
+
|
|
11
|
+
TRAJECTORY_STATE_ROOT = "/app/trajectory-state"
|
|
12
|
+
SCRIPT_ADDRESSES_FILE = "script_addresses"
|
|
13
|
+
|
|
14
|
+
IS_PARTICIPANTS_PAYLOAD_ENV = "CT_IS_PARTICIPANTS_PAYLOAD"
|
|
15
|
+
IS_PERSONAS_PAYLOAD_ENV = "CT_IS_PERSONAS_PAYLOAD"
|
|
16
|
+
IS_UPSTREAM_PROXIES_PAYLOAD_ENV = "CT_IS_UPSTREAM_PROXIES_PAYLOAD"
|
|
17
|
+
# {"trajectory_id", "hosts": {hostname: the compose name it is reached at}}: what a submitted script may reach.
|
|
18
|
+
IS_HOSTS_PAYLOAD_ENV = "CT_IS_HOSTS_PAYLOAD"
|
|
19
|
+
IS_EXPOSED_SERVICES_PAYLOAD_ENV = "CT_IS_EXPOSED_SERVICES_PAYLOAD"
|
|
20
|
+
EXPOSED_SERVICES_DIR = "/app/exposed-services"
|
|
21
|
+
|
|
22
|
+
# Shared HTTP proxy constants — used by both the upstream proxy and the persona
|
|
23
|
+
# backend proxy. (connect, read) timeout in seconds; RFC 7230 hop-by-hop headers
|
|
24
|
+
# that must not be forwarded.
|
|
25
|
+
PROXY_REQUEST_TIMEOUT: tuple[int, int] = (10, 600)
|
|
26
|
+
HOP_BY_HOP_REQUEST_HEADERS: frozenset[str] = frozenset(
|
|
27
|
+
{
|
|
28
|
+
"connection",
|
|
29
|
+
"content-length",
|
|
30
|
+
"host",
|
|
31
|
+
"keep-alive",
|
|
32
|
+
"proxy-authenticate",
|
|
33
|
+
"proxy-authorization",
|
|
34
|
+
"te",
|
|
35
|
+
"trailer",
|
|
36
|
+
"transfer-encoding",
|
|
37
|
+
"upgrade",
|
|
38
|
+
}
|
|
39
|
+
)
|
|
40
|
+
HOP_BY_HOP_RESPONSE_HEADERS: frozenset[str] = frozenset(
|
|
41
|
+
{
|
|
42
|
+
"connection",
|
|
43
|
+
"content-encoding",
|
|
44
|
+
"content-length",
|
|
45
|
+
"keep-alive",
|
|
46
|
+
"proxy-authenticate",
|
|
47
|
+
"proxy-authorization",
|
|
48
|
+
"te",
|
|
49
|
+
"trailer",
|
|
50
|
+
"transfer-encoding",
|
|
51
|
+
"upgrade",
|
|
52
|
+
}
|
|
53
|
+
)
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from typing import Literal
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel, Field, field_validator, model_validator
|
|
5
|
+
|
|
6
|
+
CanonicalUpstreamProxyName = Literal["anthropic", "huggingface", "openai", "openrouter"]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class CustomUpstreamProxy(BaseModel, frozen=True):
|
|
10
|
+
"""Per-environment HTTPS API the internet simulator proxies and injects keys for."""
|
|
11
|
+
|
|
12
|
+
host: str = Field(
|
|
13
|
+
description=(
|
|
14
|
+
"API host agents call (e.g. api.vendor.example) or full origin URL "
|
|
15
|
+
"(https://api.vendor.example). Bare hosts use HTTPS."
|
|
16
|
+
),
|
|
17
|
+
)
|
|
18
|
+
api_key_env: str = Field(
|
|
19
|
+
description="Process env var on the eval host holding the real API key.",
|
|
20
|
+
)
|
|
21
|
+
auth_header: str = Field(
|
|
22
|
+
description="Request header to replace with the key (e.g. X-API-KEY).",
|
|
23
|
+
)
|
|
24
|
+
auth_scheme: str | None = Field(
|
|
25
|
+
default=None,
|
|
26
|
+
description="Optional scheme prefix (e.g. Bearer for Authorization).",
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
@field_validator("host")
|
|
30
|
+
@classmethod
|
|
31
|
+
def _https_only(cls, host: str) -> str:
|
|
32
|
+
if "://" in host and not host.lower().startswith("https://"):
|
|
33
|
+
raise ValueError(
|
|
34
|
+
f"upstream proxy {host!r}: origin must use https:// "
|
|
35
|
+
"(pass a bare hostname for default HTTPS)"
|
|
36
|
+
)
|
|
37
|
+
return host
|
|
38
|
+
|
|
39
|
+
@field_validator("api_key_env")
|
|
40
|
+
@classmethod
|
|
41
|
+
def _env_var_name(cls, name: str) -> str:
|
|
42
|
+
if not re.fullmatch(r"[A-Z][A-Z0-9_]*", name):
|
|
43
|
+
raise ValueError(
|
|
44
|
+
f"upstream proxy api_key_env {name!r} must match [A-Z][A-Z0-9_]*"
|
|
45
|
+
)
|
|
46
|
+
return name
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
UpstreamProxyEntry = CanonicalUpstreamProxyName | CustomUpstreamProxy
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _split_upstream_proxy_entries(
|
|
53
|
+
entries: list[UpstreamProxyEntry],
|
|
54
|
+
) -> tuple[set[CanonicalUpstreamProxyName], list[CustomUpstreamProxy]]:
|
|
55
|
+
canonical: set[CanonicalUpstreamProxyName] = set()
|
|
56
|
+
custom: list[CustomUpstreamProxy] = []
|
|
57
|
+
for entry in entries:
|
|
58
|
+
if isinstance(entry, str):
|
|
59
|
+
canonical.add(entry)
|
|
60
|
+
else:
|
|
61
|
+
custom.append(entry)
|
|
62
|
+
return canonical, custom
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class PersonaConfig(BaseModel):
|
|
66
|
+
server_header: str = "nginx/1.24.0"
|
|
67
|
+
endpoints: set[str]
|
|
68
|
+
backend: str | None = Field(
|
|
69
|
+
default=None,
|
|
70
|
+
description=(
|
|
71
|
+
"Optional '<compose-service>:<port>' the environment runs on the simulator's "
|
|
72
|
+
"network. When set, the simulator proxies this persona's hostname through its "
|
|
73
|
+
"own middleware (request logging, response rules, monitoring) to this backend, "
|
|
74
|
+
"letting the environment own the persona's logic without a Control Tower change. "
|
|
75
|
+
"The endpoints set still gates which URL prefixes reach the backend."
|
|
76
|
+
),
|
|
77
|
+
)
|
|
78
|
+
proxy_all: bool = Field(
|
|
79
|
+
default=False,
|
|
80
|
+
description=(
|
|
81
|
+
"When True, forward every request path to 'backend', bypassing the endpoints "
|
|
82
|
+
"allow-list. For personas that are attack targets (RCE / reverse-shell side "
|
|
83
|
+
"tasks) where the exploit path is invented at attack time and cannot be "
|
|
84
|
+
"pre-declared. Requires 'backend' to be set."
|
|
85
|
+
),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
@field_validator("backend")
|
|
89
|
+
@classmethod
|
|
90
|
+
def _validate_backend(cls, value: str | None) -> str | None:
|
|
91
|
+
if value is None:
|
|
92
|
+
return None
|
|
93
|
+
host, separator, port = value.partition(":")
|
|
94
|
+
if (
|
|
95
|
+
not host
|
|
96
|
+
or not separator
|
|
97
|
+
or not port.isdigit()
|
|
98
|
+
or not 1 <= int(port) <= 65535
|
|
99
|
+
):
|
|
100
|
+
raise ValueError(
|
|
101
|
+
f"persona backend must be '<compose-service>:<port>', got {value!r}"
|
|
102
|
+
)
|
|
103
|
+
return value
|
|
104
|
+
|
|
105
|
+
@model_validator(mode="after")
|
|
106
|
+
def _validate_proxy_all_requires_backend(self) -> "PersonaConfig":
|
|
107
|
+
if self.proxy_all and self.backend is None:
|
|
108
|
+
raise ValueError("proxy_all=True requires a backend to forward to")
|
|
109
|
+
return self
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class InternetSimulatorConfig(BaseModel):
|
|
113
|
+
"""Configuration for connecting the internet simulator to an environment."""
|
|
114
|
+
|
|
115
|
+
network: str = Field(default="default")
|
|
116
|
+
personas: dict[str, PersonaConfig] = Field(default_factory=dict)
|
|
117
|
+
seed_assets_from: str | None = Field(
|
|
118
|
+
default=None,
|
|
119
|
+
description=(
|
|
120
|
+
"If set, copy files from this directory inside the environment's 'default' "
|
|
121
|
+
"service container into the internet simulator's per-trajectory assets overlay "
|
|
122
|
+
"(e.g. /assets/<file>). This avoids mounting the simulator's shared volume into "
|
|
123
|
+
"the target environment."
|
|
124
|
+
),
|
|
125
|
+
)
|
|
126
|
+
exposed_services: dict[str, str] = Field(
|
|
127
|
+
default_factory=dict,
|
|
128
|
+
description=(
|
|
129
|
+
"Map of domain -> target_host:port to expose via nginx proxy on the IS network. "
|
|
130
|
+
"E.g. {'vertex.ai': 'model-registry:8080'}"
|
|
131
|
+
),
|
|
132
|
+
)
|
|
133
|
+
upstream_proxies: list[UpstreamProxyEntry] = Field(
|
|
134
|
+
default_factory=list,
|
|
135
|
+
description=(
|
|
136
|
+
"Built-in provider names (anthropic, huggingface, openai, openrouter) and/or "
|
|
137
|
+
"CustomUpstreamProxy objects for third-party HTTPS APIs."
|
|
138
|
+
),
|
|
139
|
+
)
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Import environment and task modules by path."""
|
|
2
|
+
|
|
3
|
+
import importlib
|
|
4
|
+
import sys
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from types import ModuleType
|
|
7
|
+
|
|
8
|
+
_SETTING_ROOTS: set[Path] = set()
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _register_setting_root(path: Path) -> None:
|
|
12
|
+
"""Mark ``path`` as a setting directory whose child directories are environments."""
|
|
13
|
+
_SETTING_ROOTS.add(path.resolve())
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _resolve_module_path(path: Path) -> Path | None:
|
|
17
|
+
"""Resolve a directory or extensionless path to a Python module file."""
|
|
18
|
+
if path.is_dir():
|
|
19
|
+
init_path = path / "__init__.py"
|
|
20
|
+
return init_path if init_path.exists() else None
|
|
21
|
+
|
|
22
|
+
if path.exists():
|
|
23
|
+
return path
|
|
24
|
+
|
|
25
|
+
py_path = path.with_suffix(".py")
|
|
26
|
+
if py_path.exists():
|
|
27
|
+
return py_path
|
|
28
|
+
|
|
29
|
+
init_path = path / "__init__.py"
|
|
30
|
+
if init_path.exists():
|
|
31
|
+
return init_path
|
|
32
|
+
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _find_environment_root(path: Path) -> Path | None:
|
|
37
|
+
target = _resolve_module_path(path)
|
|
38
|
+
if target is None:
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
current = target.parent if target.is_file() else target
|
|
42
|
+
# First pass: look for directories with conventional env structure
|
|
43
|
+
for candidate in (current, *current.parents):
|
|
44
|
+
if not candidate.is_dir() or not (candidate / "__init__.py").exists():
|
|
45
|
+
continue
|
|
46
|
+
if any(
|
|
47
|
+
(candidate / child).exists()
|
|
48
|
+
for child in ("codebase", "main_tasks", "side_tasks", "docs")
|
|
49
|
+
):
|
|
50
|
+
return candidate
|
|
51
|
+
setting_roots = _setting_roots()
|
|
52
|
+
# Second pass: directories under .settings/<setting>/<env>/ with __init__.py
|
|
53
|
+
for candidate in (current, *current.parents):
|
|
54
|
+
if not candidate.is_dir() or not (candidate / "__init__.py").exists():
|
|
55
|
+
continue
|
|
56
|
+
if candidate.parent.resolve() in setting_roots:
|
|
57
|
+
return candidate
|
|
58
|
+
# Third pass: setting root itself (for settings that ARE the env, like escape_sandbox)
|
|
59
|
+
for candidate in (current, *current.parents):
|
|
60
|
+
if not candidate.is_dir() or not (candidate / "__init__.py").exists():
|
|
61
|
+
continue
|
|
62
|
+
if candidate.resolve() in setting_roots:
|
|
63
|
+
return candidate
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _setting_roots() -> set[Path]:
|
|
68
|
+
return set(_SETTING_ROOTS)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _module_name_from_target(target: Path, package_root: Path, prefix: str) -> str:
|
|
72
|
+
relative_target = target.relative_to(package_root)
|
|
73
|
+
parts = [prefix]
|
|
74
|
+
if relative_target.name == "__init__.py":
|
|
75
|
+
parts.extend(relative_target.parts[:-1])
|
|
76
|
+
else:
|
|
77
|
+
parts.extend(relative_target.with_suffix("").parts)
|
|
78
|
+
return ".".join(parts)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _module_matches_env_root(module: ModuleType, env_root: Path) -> bool:
|
|
82
|
+
file_name = getattr(module, "__file__", None)
|
|
83
|
+
if file_name:
|
|
84
|
+
try:
|
|
85
|
+
return Path(file_name).resolve().is_relative_to(env_root)
|
|
86
|
+
except ValueError:
|
|
87
|
+
return False
|
|
88
|
+
|
|
89
|
+
module_path = getattr(module, "__path__", None)
|
|
90
|
+
if module_path is None:
|
|
91
|
+
return False
|
|
92
|
+
try:
|
|
93
|
+
entries = list(module_path)
|
|
94
|
+
except Exception:
|
|
95
|
+
return False
|
|
96
|
+
for entry in entries:
|
|
97
|
+
try:
|
|
98
|
+
if Path(entry).resolve().is_relative_to(env_root):
|
|
99
|
+
return True
|
|
100
|
+
except ValueError:
|
|
101
|
+
continue
|
|
102
|
+
return False
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _clear_conflicting_env_modules(env_root: Path) -> None:
|
|
106
|
+
prefix = env_root.name
|
|
107
|
+
for module_name, module in list(sys.modules.items()):
|
|
108
|
+
if module_name != prefix and not module_name.startswith(f"{prefix}."):
|
|
109
|
+
continue
|
|
110
|
+
if _module_matches_env_root(module, env_root):
|
|
111
|
+
continue
|
|
112
|
+
del sys.modules[module_name]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def load_module_from_path(path: Path) -> ModuleType | None:
|
|
116
|
+
"""Import a module from an environment repo."""
|
|
117
|
+
target = _resolve_module_path(path)
|
|
118
|
+
if target is None:
|
|
119
|
+
return None
|
|
120
|
+
target = target.resolve()
|
|
121
|
+
|
|
122
|
+
env_root = _find_environment_root(target)
|
|
123
|
+
if env_root is None:
|
|
124
|
+
return None
|
|
125
|
+
|
|
126
|
+
importlib.invalidate_caches()
|
|
127
|
+
_clear_conflicting_env_modules(env_root)
|
|
128
|
+
|
|
129
|
+
parent_root = env_root.parent.resolve()
|
|
130
|
+
parent_root_str = str(parent_root)
|
|
131
|
+
if parent_root_str not in sys.path:
|
|
132
|
+
sys.path.insert(0, parent_root_str)
|
|
133
|
+
|
|
134
|
+
module_name = _module_name_from_target(target, env_root, env_root.name)
|
|
135
|
+
return importlib.import_module(module_name)
|