ct-setting-sdk 4.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ct_setting_sdk/__init__.py +1 -0
- ct_setting_sdk/env/__init__.py +0 -0
- ct_setting_sdk/env/baseline_snapshot.py +126 -0
- ct_setting_sdk/env/constants.py +11 -0
- ct_setting_sdk/env/docker_environment.py +134 -0
- ct_setting_sdk/env/env_requirements.py +28 -0
- ct_setting_sdk/env/environment.py +136 -0
- ct_setting_sdk/env/internet_simulator/__init__.py +0 -0
- ct_setting_sdk/env/internet_simulator/config.py +139 -0
- ct_setting_sdk/env/internet_simulator/constants.py +43 -0
- ct_setting_sdk/env/module_loader.py +135 -0
- ct_setting_sdk/env/runtime_vars.py +33 -0
- ct_setting_sdk/env/space.py +465 -0
- ct_setting_sdk/env/store_namespace.py +24 -0
- ct_setting_sdk/env/submit.py +47 -0
- ct_setting_sdk/env/task.py +397 -0
- ct_setting_sdk/env/task_helpers.py +29 -0
- ct_setting_sdk/env/tools.py +23 -0
- ct_setting_sdk/env/yaml_metadata.py +117 -0
- ct_setting_sdk/protocol/__init__.py +0 -0
- ct_setting_sdk/protocol/errors.py +50 -0
- ct_setting_sdk/protocol/package.py +27 -0
- ct_setting_sdk/protocol/roles.py +13 -0
- ct_setting_sdk/protocol/wire.py +350 -0
- ct_setting_sdk/py.typed +0 -0
- ct_setting_sdk/registration.py +70 -0
- ct_setting_sdk/scoring/__init__.py +0 -0
- ct_setting_sdk/scoring/bug_fix.py +230 -0
- ct_setting_sdk/scoring/cache.py +60 -0
- ct_setting_sdk/scoring/lang_rewrite.py +186 -0
- ct_setting_sdk/scoring/llm_judge.py +89 -0
- ct_setting_sdk/scoring/research_task.py +56 -0
- ct_setting_sdk/scoring/sample_context.py +30 -0
- ct_setting_sdk/scoring/sample_metadata.py +48 -0
- ct_setting_sdk/scoring/scorer_imports.py +177 -0
- ct_setting_sdk/scoring/scorer_utils.py +913 -0
- ct_setting_sdk/scoring/trajectory_id.py +19 -0
- ct_setting_sdk/scoring/xml_utils.py +23 -0
- ct_setting_sdk/server/__init__.py +0 -0
- ct_setting_sdk/server/__main__.py +98 -0
- ct_setting_sdk/server/facts.py +104 -0
- ct_setting_sdk/server/manifest.py +198 -0
- ct_setting_sdk/server/model_proxy.py +116 -0
- ct_setting_sdk/server/proxies.py +255 -0
- ct_setting_sdk/server/serve.py +392 -0
- ct_setting_sdk-4.0.0rc1.dist-info/METADATA +10 -0
- ct_setting_sdk-4.0.0rc1.dist-info/RECORD +49 -0
- ct_setting_sdk-4.0.0rc1.dist-info/WHEEL +5 -0
- ct_setting_sdk-4.0.0rc1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The environment side of the Control Tower env API."""
|
|
File without changes
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Baseline snapshots run after task setup but before the agent starts, so
|
|
2
|
+
scorers can compare before/after state. Declare one in the scorer.py with
|
|
3
|
+
``baseline_snapshot``; the scorer retrieves it with ``.get()``."""
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from collections.abc import Callable
|
|
8
|
+
from typing import Any, cast
|
|
9
|
+
|
|
10
|
+
from inspect_ai.util import sandbox, store
|
|
11
|
+
|
|
12
|
+
from ct_setting_sdk.env.store_namespace import store_namespace
|
|
13
|
+
from ct_setting_sdk.protocol.errors import OperationError
|
|
14
|
+
|
|
15
|
+
BaselineSnapshotFn = Callable[[], Any]
|
|
16
|
+
|
|
17
|
+
CONTAINER_BASELINE_DIR = "/root/.ct_baseline"
|
|
18
|
+
|
|
19
|
+
STORE_KEY_PREFIX = "baseline_snapshot:"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _stable_ident(fn: BaselineSnapshotFn) -> str:
|
|
23
|
+
"""Stable per-scorer identifier used as the store key and file basename,
|
|
24
|
+
derived from the fn's module so the same scorer.py resolves to the same
|
|
25
|
+
identifier in any process that loads it."""
|
|
26
|
+
return re.sub(r"[^A-Za-z0-9_.]", "_", fn.__module__)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class BaselineSnapshot:
|
|
30
|
+
"""A callable that captures baseline state and stores it for later retrieval.
|
|
31
|
+
``ident`` overrides the store-key / container-filename identity, which
|
|
32
|
+
otherwise derives from the wrapped function's module; it must resolve to the
|
|
33
|
+
same value in any process, since live mode rehydrates by identity."""
|
|
34
|
+
|
|
35
|
+
store_prefix: str | None = None
|
|
36
|
+
"""A store-key prefix to use instead of the setting package's namespace."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, fn: BaselineSnapshotFn, ident: str | None = None):
|
|
39
|
+
self._fn = fn
|
|
40
|
+
self._ident = ident if ident is not None else self._default_ident()
|
|
41
|
+
|
|
42
|
+
def _default_ident(self) -> str | None:
|
|
43
|
+
return _stable_ident(self._fn)
|
|
44
|
+
|
|
45
|
+
def _owner_ident(self, module_name: str) -> str:
|
|
46
|
+
return re.sub(r"[^A-Za-z0-9_.]", "_", module_name)
|
|
47
|
+
|
|
48
|
+
def bind_owner_module(self, module_name: str) -> None:
|
|
49
|
+
if self._ident is None:
|
|
50
|
+
self._ident = self._owner_ident(module_name)
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def _resolved_ident(self) -> str:
|
|
54
|
+
if self._ident is None:
|
|
55
|
+
raise OperationError(
|
|
56
|
+
f"baseline snapshot {type(self).__name__} has no identity: no "
|
|
57
|
+
"ident= was passed at construction."
|
|
58
|
+
)
|
|
59
|
+
return self._ident
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def _store_key(self) -> str:
|
|
63
|
+
if self.store_prefix is not None:
|
|
64
|
+
return f"{self.store_prefix}{self._resolved_ident}"
|
|
65
|
+
return f"{STORE_KEY_PREFIX}{store_namespace()}:{self._resolved_ident}"
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def _container_path(self) -> str:
|
|
69
|
+
return f"{CONTAINER_BASELINE_DIR}/{self._resolved_ident}.json"
|
|
70
|
+
|
|
71
|
+
async def __call__(self) -> dict[str, Any]:
|
|
72
|
+
"""Capture baseline and write it to the inspect per-sample store only —
|
|
73
|
+
never to the container, so a root agent cannot read it. This is the
|
|
74
|
+
eval pipeline's path."""
|
|
75
|
+
result = await self._fn()
|
|
76
|
+
store().set(self._store_key, result)
|
|
77
|
+
return result
|
|
78
|
+
|
|
79
|
+
async def capture_for_live(self) -> dict[str, Any]:
|
|
80
|
+
"""Capture baseline and persist it to the store *and* a root-owned
|
|
81
|
+
container file, so a later ``ct live score`` process can reload it.
|
|
82
|
+
Never call this from the eval pipeline: an agent with root in the
|
|
83
|
+
container could read the file and leak ground-truth scorer data."""
|
|
84
|
+
result = await self._fn()
|
|
85
|
+
store().set(self._store_key, result)
|
|
86
|
+
await self._persist_to_container(self._container_path, json.dumps(result))
|
|
87
|
+
return result
|
|
88
|
+
|
|
89
|
+
async def _persist_to_container(self, path: str, content: str | bytes) -> None:
|
|
90
|
+
sbx = sandbox()
|
|
91
|
+
await sbx.exec(["mkdir", "-p", CONTAINER_BASELINE_DIR], user="root")
|
|
92
|
+
await sbx.exec(["chmod", "0700", CONTAINER_BASELINE_DIR], user="root")
|
|
93
|
+
await sbx.write_file(path, content)
|
|
94
|
+
await sbx.exec(["chown", "root:root", path], user="root")
|
|
95
|
+
await sbx.exec(["chmod", "0600", path], user="root")
|
|
96
|
+
|
|
97
|
+
async def load_from_container(self) -> dict[str, Any]:
|
|
98
|
+
"""Rehydrate the inspect store from the container file written by
|
|
99
|
+
``capture_for_live``, so ``ct live score`` (a fresh process) sees the
|
|
100
|
+
baseline identically to eval mode. No-op if the file doesn't exist."""
|
|
101
|
+
check = await sandbox().exec(["test", "-f", self._container_path])
|
|
102
|
+
if check.returncode != 0:
|
|
103
|
+
return {}
|
|
104
|
+
try:
|
|
105
|
+
raw = await sandbox().read_file(self._container_path)
|
|
106
|
+
except FileNotFoundError:
|
|
107
|
+
return {}
|
|
108
|
+
try:
|
|
109
|
+
data = json.loads(raw)
|
|
110
|
+
except json.JSONDecodeError:
|
|
111
|
+
return {}
|
|
112
|
+
if not isinstance(data, dict):
|
|
113
|
+
return {}
|
|
114
|
+
data = cast(dict[str, Any], data)
|
|
115
|
+
store().set(self._store_key, data)
|
|
116
|
+
return data
|
|
117
|
+
|
|
118
|
+
def get(self) -> dict[str, Any]:
|
|
119
|
+
"""Retrieve the stored baseline snapshot data."""
|
|
120
|
+
return store().get(self._store_key, {})
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def baseline_snapshot(fn: BaselineSnapshotFn) -> BaselineSnapshot:
|
|
124
|
+
"""Mark a function as a baseline snapshot: captured automatically before
|
|
125
|
+
the agent runs, retrieved in the scorer with ``.get()``."""
|
|
126
|
+
return BaselineSnapshot(fn)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
AGENT_DIR = Path("/app")
|
|
5
|
+
"""The working directory inside the agent container."""
|
|
6
|
+
|
|
7
|
+
SANDBOX_NAME = "lb-docker"
|
|
8
|
+
|
|
9
|
+
DEFAULT_INTEGRATION_SUITE_TIMEOUT = int(
|
|
10
|
+
os.environ.get("CONTROL_TOWER_INTEGRATION_SUITE_TIMEOUT") or 300
|
|
11
|
+
)
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
from collections.abc import Sequence
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import ClassVar
|
|
4
|
+
|
|
5
|
+
from inspect_ai.util import SandboxEnvironmentSpec
|
|
6
|
+
from pydantic import BaseModel, ConfigDict, PrivateAttr
|
|
7
|
+
|
|
8
|
+
from ct_setting_sdk.env import env_requirements
|
|
9
|
+
from ct_setting_sdk.env.constants import SANDBOX_NAME
|
|
10
|
+
from ct_setting_sdk.env.environment import EnvDocs
|
|
11
|
+
from ct_setting_sdk.env.internet_simulator.config import InternetSimulatorConfig
|
|
12
|
+
from ct_setting_sdk.env.space import TaskSpace, folder_task_space
|
|
13
|
+
from ct_setting_sdk.env.task_helpers import default_display_name
|
|
14
|
+
from ct_setting_sdk.env.tools import ToolSpec
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class DockerEnvironment(BaseModel):
|
|
18
|
+
"""Docker Compose-based environment loaded from a directory.
|
|
19
|
+
|
|
20
|
+
Satisfies the :class:`ControlEnvironment` protocol. The standard implementation
|
|
21
|
+
for envs with a ``codebase/``, ``compose.yml``, ``main_tasks/``,
|
|
22
|
+
and ``side_tasks/`` directory structure.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
# extra="forbid" so a stale env declaration fails loudly instead of being silently dropped.
|
|
26
|
+
model_config: ClassVar[ConfigDict] = ConfigDict(
|
|
27
|
+
arbitrary_types_allowed=True, extra="forbid"
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
path: Path
|
|
31
|
+
id: str
|
|
32
|
+
display_name: str
|
|
33
|
+
|
|
34
|
+
compose_paths: dict[str, Path]
|
|
35
|
+
|
|
36
|
+
agent_user: str = "root"
|
|
37
|
+
internet_simulator: "InternetSimulatorConfig | None" = None
|
|
38
|
+
tools: tuple[ToolSpec, ...]
|
|
39
|
+
private_tools: tuple[ToolSpec, ...]
|
|
40
|
+
score_main_on_attack: bool = False
|
|
41
|
+
_task_space_cache: "TaskSpace | None" = PrivateAttr(default=None)
|
|
42
|
+
_required_env_vars: tuple[env_requirements.RequiredEnvVar, ...] = PrivateAttr(
|
|
43
|
+
default=()
|
|
44
|
+
)
|
|
45
|
+
_required_env_any_of: tuple[env_requirements.RequiredEnvAnyOf, ...] = PrivateAttr(
|
|
46
|
+
default=()
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
short_description: str = ""
|
|
50
|
+
description: str = ""
|
|
51
|
+
tech_stack: list[str] = []
|
|
52
|
+
services: list[str] = []
|
|
53
|
+
attack_surface: str = ""
|
|
54
|
+
|
|
55
|
+
codebase_path: Path
|
|
56
|
+
agent_codebase_path: Path
|
|
57
|
+
lang_rewrite_tasks_integration_test_filename: str = "test_integration.py"
|
|
58
|
+
|
|
59
|
+
def docs(self, args: BaseModel | None = None) -> "EnvDocs":
|
|
60
|
+
return EnvDocs(
|
|
61
|
+
description=self.description,
|
|
62
|
+
attack_surface=self.attack_surface,
|
|
63
|
+
short_description=self.short_description,
|
|
64
|
+
tech_stack=list(self.tech_stack),
|
|
65
|
+
services=list(self.services),
|
|
66
|
+
required_env_vars=list(self._required_env_vars),
|
|
67
|
+
required_env_any_of=list(self._required_env_any_of),
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
def get_sandbox_spec(
|
|
71
|
+
self,
|
|
72
|
+
codebase_name: str = "default",
|
|
73
|
+
args: BaseModel | None = None,
|
|
74
|
+
) -> SandboxEnvironmentSpec:
|
|
75
|
+
return SandboxEnvironmentSpec(
|
|
76
|
+
type=SANDBOX_NAME, config=str(self.get_compose_path(codebase_name))
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
def task_space(self, args: BaseModel | None = None) -> "TaskSpace":
|
|
80
|
+
if self._task_space_cache is None:
|
|
81
|
+
self._task_space_cache = folder_task_space(self.path, self.id)
|
|
82
|
+
return self._task_space_cache
|
|
83
|
+
|
|
84
|
+
def get_compose_path(self, codebase_name: str = "default") -> Path:
|
|
85
|
+
if codebase_name not in self.compose_paths:
|
|
86
|
+
raise ValueError(
|
|
87
|
+
f"Unknown compose path '{codebase_name}' for environment '{self.display_name}'"
|
|
88
|
+
)
|
|
89
|
+
return self.compose_paths[codebase_name]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def make_environment_config_from_path(
|
|
93
|
+
path: Path,
|
|
94
|
+
short_description: str,
|
|
95
|
+
tech_stack: list[str],
|
|
96
|
+
services: list[str],
|
|
97
|
+
custom_display_name: str | None = None,
|
|
98
|
+
compose_paths: dict[str, Path] | None = None,
|
|
99
|
+
agent_user: str = "root",
|
|
100
|
+
*,
|
|
101
|
+
tools: Sequence[ToolSpec],
|
|
102
|
+
private_tools: Sequence[ToolSpec],
|
|
103
|
+
score_main_on_attack: bool = False,
|
|
104
|
+
internet_simulator: "InternetSimulatorConfig | None" = None,
|
|
105
|
+
lang_rewrite_tasks_integration_test_filename: str = "test_integration.py",
|
|
106
|
+
required_env_vars: list[env_requirements.RequiredEnvVar] | None = None,
|
|
107
|
+
required_env_any_of: list[env_requirements.RequiredEnvAnyOf] | None = None,
|
|
108
|
+
) -> DockerEnvironment:
|
|
109
|
+
"""Construct a :class:`DockerEnvironment` from an env directory.
|
|
110
|
+
|
|
111
|
+
Reads ``docs/description.md`` and ``docs/attack_surface.md``.
|
|
112
|
+
"""
|
|
113
|
+
env = DockerEnvironment(
|
|
114
|
+
path=path,
|
|
115
|
+
id=path.name,
|
|
116
|
+
display_name=custom_display_name or default_display_name(path.name),
|
|
117
|
+
compose_paths=compose_paths or {"default": path / "codebase" / "compose.yml"},
|
|
118
|
+
agent_user=agent_user,
|
|
119
|
+
tools=tuple(tools),
|
|
120
|
+
private_tools=tuple(private_tools),
|
|
121
|
+
score_main_on_attack=score_main_on_attack,
|
|
122
|
+
short_description=short_description,
|
|
123
|
+
description=(path / "docs" / "description.md").read_text().strip(),
|
|
124
|
+
tech_stack=tech_stack,
|
|
125
|
+
services=services,
|
|
126
|
+
attack_surface=(path / "docs" / "attack_surface.md").read_text().strip(),
|
|
127
|
+
codebase_path=path / "codebase",
|
|
128
|
+
agent_codebase_path=path / "codebase" / "src",
|
|
129
|
+
internet_simulator=internet_simulator,
|
|
130
|
+
lang_rewrite_tasks_integration_test_filename=lang_rewrite_tasks_integration_test_filename,
|
|
131
|
+
)
|
|
132
|
+
env._required_env_vars = tuple(required_env_vars or ())
|
|
133
|
+
env._required_env_any_of = tuple(required_env_any_of or ())
|
|
134
|
+
return env
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@dataclass(frozen=True)
|
|
5
|
+
class RequiredEnvVar:
|
|
6
|
+
"""A single process environment variable that must be set (non-blank).
|
|
7
|
+
|
|
8
|
+
Use ``description`` for what the variable is for. Use ``setup`` only for how to
|
|
9
|
+
create or obtain the credential (e.g. signup URL). Where to put the value
|
|
10
|
+
(repo ``.env`` vs shell) is explained once in the shared CLI error footer, not
|
|
11
|
+
per environment.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
name: str
|
|
15
|
+
description: str = ""
|
|
16
|
+
setup: str = ""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class RequiredEnvAnyOf:
|
|
21
|
+
"""At least one of ``names`` must be set (non-blank) in the process environment.
|
|
22
|
+
|
|
23
|
+
``description`` / ``setup`` follow the same rules as :class:`RequiredEnvVar`.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
names: tuple[str, ...]
|
|
27
|
+
description: str = ""
|
|
28
|
+
setup: str = ""
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from collections.abc import Sequence
|
|
3
|
+
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
|
4
|
+
|
|
5
|
+
from inspect_ai.util import SandboxEnvironmentSpec
|
|
6
|
+
from pydantic import BaseModel, model_validator
|
|
7
|
+
|
|
8
|
+
from ct_setting_sdk.env.env_requirements import RequiredEnvAnyOf, RequiredEnvVar
|
|
9
|
+
from ct_setting_sdk.env.tools import ToolSpec
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from ct_setting_sdk.env.space import TaskSpace
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class EnvDocs(BaseModel, frozen=True):
|
|
16
|
+
"""Human-facing metadata for an environment."""
|
|
17
|
+
|
|
18
|
+
description: str = ""
|
|
19
|
+
attack_surface: str = ""
|
|
20
|
+
short_description: str = ""
|
|
21
|
+
tech_stack: list[str] = []
|
|
22
|
+
services: list[str] = []
|
|
23
|
+
required_env_vars: list[RequiredEnvVar] = []
|
|
24
|
+
required_env_any_of: list[RequiredEnvAnyOf] = []
|
|
25
|
+
|
|
26
|
+
@model_validator(mode="after")
|
|
27
|
+
def _check_required_env(self) -> "EnvDocs":
|
|
28
|
+
if errors := _required_env_errors(self):
|
|
29
|
+
raise ValueError("; ".join(errors))
|
|
30
|
+
return self
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
ENV_VAR_NAME = re.compile(r"[A-Z][A-Z0-9_]*\Z")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _required_env_errors(docs: EnvDocs) -> list[str]:
|
|
37
|
+
errors: list[str] = []
|
|
38
|
+
seen_unconditional: set[str] = set()
|
|
39
|
+
for req in docs.required_env_vars:
|
|
40
|
+
name = req.name.strip()
|
|
41
|
+
if not name:
|
|
42
|
+
errors.append("required_env_vars entry has empty name")
|
|
43
|
+
continue
|
|
44
|
+
if not ENV_VAR_NAME.fullmatch(name):
|
|
45
|
+
errors.append(
|
|
46
|
+
f"required_env_vars entry {name!r} must match [A-Z][A-Z0-9_]* "
|
|
47
|
+
"(POSIX-style environment variable name)"
|
|
48
|
+
)
|
|
49
|
+
if name in seen_unconditional:
|
|
50
|
+
errors.append(f"duplicate required env var declaration: {name!r}")
|
|
51
|
+
seen_unconditional.add(name)
|
|
52
|
+
|
|
53
|
+
seen_in_any_of: set[str] = set()
|
|
54
|
+
for i, group in enumerate(docs.required_env_any_of):
|
|
55
|
+
label = f"required_env_any_of[{i}]"
|
|
56
|
+
names = tuple(n.strip() for n in group.names if n.strip())
|
|
57
|
+
if not names:
|
|
58
|
+
errors.append(f"{label}: names is empty")
|
|
59
|
+
continue
|
|
60
|
+
if bad := [n for n in names if not ENV_VAR_NAME.fullmatch(n)]:
|
|
61
|
+
errors.append(
|
|
62
|
+
f"{label}: invalid name(s) {bad!r}, must match [A-Z][A-Z0-9_]*"
|
|
63
|
+
)
|
|
64
|
+
if dup_within := {n for n in names if names.count(n) > 1}:
|
|
65
|
+
errors.append(
|
|
66
|
+
f"{label}: duplicate name(s) in group: {sorted(dup_within)!r}"
|
|
67
|
+
)
|
|
68
|
+
if overlap_u := seen_unconditional.intersection(names):
|
|
69
|
+
errors.append(
|
|
70
|
+
f"{label}: name(s) {sorted(overlap_u)!r} also listed as unconditionally required"
|
|
71
|
+
)
|
|
72
|
+
if overlap_a := seen_in_any_of.intersection(names):
|
|
73
|
+
errors.append(
|
|
74
|
+
f"{label}: name(s) {sorted(overlap_a)!r} appear in multiple any-of groups"
|
|
75
|
+
)
|
|
76
|
+
seen_in_any_of.update(names)
|
|
77
|
+
return errors
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@runtime_checkable
|
|
81
|
+
class ControlEnvironment(Protocol):
|
|
82
|
+
"""The environment protocol.
|
|
83
|
+
|
|
84
|
+
Optional attributes (read with ``getattr``):
|
|
85
|
+
|
|
86
|
+
* ``tools`` / ``private_tools``: ``Sequence[ToolSpec]`` — the interaction tools every sample of this
|
|
87
|
+
env gets, and the private tools a policy may choose if white allows them. Tasks add to both. Nothing is implied:
|
|
88
|
+
an env with no ``tools`` gives the agent none.
|
|
89
|
+
* ``agent_user: str`` — container/VM user (defaults to ``"root"``).
|
|
90
|
+
* ``args_model: type[BaseModel] | None`` — schema for per-run ``--env-arg`` values.
|
|
91
|
+
* ``score_main_on_attack: bool`` — also score the main task on attack samples.
|
|
92
|
+
* ``internet_simulator: InternetSimulatorConfig | None``.
|
|
93
|
+
* ``path: Path`` — the env's own directory, for envs that have one.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def id(self) -> str: ...
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def display_name(self) -> str: ...
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def tools(self) -> Sequence[ToolSpec]: ...
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def private_tools(self) -> Sequence[ToolSpec]: ...
|
|
107
|
+
|
|
108
|
+
def docs(self, args: "BaseModel | None" = None) -> EnvDocs: ...
|
|
109
|
+
|
|
110
|
+
def get_sandbox_spec(
|
|
111
|
+
self,
|
|
112
|
+
codebase_name: str = "default",
|
|
113
|
+
args: "BaseModel | None" = None,
|
|
114
|
+
) -> SandboxEnvironmentSpec: ...
|
|
115
|
+
|
|
116
|
+
def task_space(self, args: "BaseModel | None" = None) -> "TaskSpace":
|
|
117
|
+
"""The combinatorial space of valid task combinations, built for ``args``."""
|
|
118
|
+
...
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def env_agent_user(env: ControlEnvironment) -> str:
|
|
122
|
+
"""Return ``env.agent_user`` or ``"root"``."""
|
|
123
|
+
return getattr(env, "agent_user", "root") or "root"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def env_tool_specs(env: ControlEnvironment) -> tuple[ToolSpec, ...]:
|
|
127
|
+
return tuple(env.tools)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def env_private_tool_specs(env: ControlEnvironment) -> tuple[ToolSpec, ...]:
|
|
131
|
+
return tuple(env.private_tools)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def env_score_main_on_attack(env: ControlEnvironment) -> bool:
|
|
135
|
+
"""Whether attack samples also score their main tasks."""
|
|
136
|
+
return bool(getattr(env, "score_main_on_attack", False))
|
|
File without changes
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from typing import Literal
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel, Field, field_validator, model_validator
|
|
5
|
+
|
|
6
|
+
CanonicalUpstreamProxyName = Literal["anthropic", "openai", "openrouter"]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class CustomUpstreamProxy(BaseModel, frozen=True):
|
|
10
|
+
"""Per-environment HTTPS API the internet simulator proxies and injects keys for."""
|
|
11
|
+
|
|
12
|
+
host: str = Field(
|
|
13
|
+
description=(
|
|
14
|
+
"API host agents call (e.g. api.vendor.example) or full origin URL "
|
|
15
|
+
"(https://api.vendor.example). Bare hosts use HTTPS."
|
|
16
|
+
),
|
|
17
|
+
)
|
|
18
|
+
api_key_env: str = Field(
|
|
19
|
+
description="Process env var on the eval host holding the real API key.",
|
|
20
|
+
)
|
|
21
|
+
auth_header: str = Field(
|
|
22
|
+
description="Request header to replace with the key (e.g. X-API-KEY).",
|
|
23
|
+
)
|
|
24
|
+
auth_scheme: str | None = Field(
|
|
25
|
+
default=None,
|
|
26
|
+
description="Optional scheme prefix (e.g. Bearer for Authorization).",
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
@field_validator("host")
|
|
30
|
+
@classmethod
|
|
31
|
+
def _https_only(cls, host: str) -> str:
|
|
32
|
+
if "://" in host and not host.lower().startswith("https://"):
|
|
33
|
+
raise ValueError(
|
|
34
|
+
f"upstream proxy {host!r}: origin must use https:// "
|
|
35
|
+
"(pass a bare hostname for default HTTPS)"
|
|
36
|
+
)
|
|
37
|
+
return host
|
|
38
|
+
|
|
39
|
+
@field_validator("api_key_env")
|
|
40
|
+
@classmethod
|
|
41
|
+
def _env_var_name(cls, name: str) -> str:
|
|
42
|
+
if not re.fullmatch(r"[A-Z][A-Z0-9_]*", name):
|
|
43
|
+
raise ValueError(
|
|
44
|
+
f"upstream proxy api_key_env {name!r} must match [A-Z][A-Z0-9_]*"
|
|
45
|
+
)
|
|
46
|
+
return name
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
UpstreamProxyEntry = CanonicalUpstreamProxyName | CustomUpstreamProxy
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def split_upstream_proxy_entries(
|
|
53
|
+
entries: list[UpstreamProxyEntry],
|
|
54
|
+
) -> tuple[set[CanonicalUpstreamProxyName], list[CustomUpstreamProxy]]:
|
|
55
|
+
canonical: set[CanonicalUpstreamProxyName] = set()
|
|
56
|
+
custom: list[CustomUpstreamProxy] = []
|
|
57
|
+
for entry in entries:
|
|
58
|
+
if isinstance(entry, str):
|
|
59
|
+
canonical.add(entry)
|
|
60
|
+
else:
|
|
61
|
+
custom.append(entry)
|
|
62
|
+
return canonical, custom
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class PersonaConfig(BaseModel):
|
|
66
|
+
server_header: str = "nginx/1.24.0"
|
|
67
|
+
endpoints: set[str]
|
|
68
|
+
backend: str | None = Field(
|
|
69
|
+
default=None,
|
|
70
|
+
description=(
|
|
71
|
+
"Optional '<compose-service>:<port>' the environment runs on the simulator's "
|
|
72
|
+
"network. When set, the simulator proxies this persona's hostname through its "
|
|
73
|
+
"own middleware (request logging, response rules, monitoring) to this backend, "
|
|
74
|
+
"letting the environment own the persona's logic without a Control Tower change. "
|
|
75
|
+
"The endpoints set still gates which URL prefixes reach the backend."
|
|
76
|
+
),
|
|
77
|
+
)
|
|
78
|
+
proxy_all: bool = Field(
|
|
79
|
+
default=False,
|
|
80
|
+
description=(
|
|
81
|
+
"When True, forward every request path to 'backend', bypassing the endpoints "
|
|
82
|
+
"allow-list. For personas that are attack targets (RCE / reverse-shell side "
|
|
83
|
+
"tasks) where the exploit path is invented at attack time and cannot be "
|
|
84
|
+
"pre-declared. Requires 'backend' to be set."
|
|
85
|
+
),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
@field_validator("backend")
|
|
89
|
+
@classmethod
|
|
90
|
+
def _validate_backend(cls, value: str | None) -> str | None:
|
|
91
|
+
if value is None:
|
|
92
|
+
return None
|
|
93
|
+
host, separator, port = value.partition(":")
|
|
94
|
+
if (
|
|
95
|
+
not host
|
|
96
|
+
or not separator
|
|
97
|
+
or not port.isdigit()
|
|
98
|
+
or not 1 <= int(port) <= 65535
|
|
99
|
+
):
|
|
100
|
+
raise ValueError(
|
|
101
|
+
f"persona backend must be '<compose-service>:<port>', got {value!r}"
|
|
102
|
+
)
|
|
103
|
+
return value
|
|
104
|
+
|
|
105
|
+
@model_validator(mode="after")
|
|
106
|
+
def _validate_proxy_all_requires_backend(self) -> "PersonaConfig":
|
|
107
|
+
if self.proxy_all and self.backend is None:
|
|
108
|
+
raise ValueError("proxy_all=True requires a backend to forward to")
|
|
109
|
+
return self
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class InternetSimulatorConfig(BaseModel):
|
|
113
|
+
"""Configuration for connecting the internet simulator to an environment."""
|
|
114
|
+
|
|
115
|
+
network: str = Field(default="default")
|
|
116
|
+
personas: dict[str, PersonaConfig] = Field(default_factory=dict)
|
|
117
|
+
seed_assets_from: str | None = Field(
|
|
118
|
+
default=None,
|
|
119
|
+
description=(
|
|
120
|
+
"If set, copy files from this directory inside the environment's 'default' "
|
|
121
|
+
"service container into the internet simulator's per-trajectory assets overlay "
|
|
122
|
+
"(e.g. /assets/<file>). This avoids mounting the simulator's shared volume into "
|
|
123
|
+
"the target environment."
|
|
124
|
+
),
|
|
125
|
+
)
|
|
126
|
+
exposed_services: dict[str, str] = Field(
|
|
127
|
+
default_factory=dict,
|
|
128
|
+
description=(
|
|
129
|
+
"Map of domain -> target_host:port to expose via nginx proxy on the IS network. "
|
|
130
|
+
"E.g. {'vertex.ai': 'model-registry:8080'}"
|
|
131
|
+
),
|
|
132
|
+
)
|
|
133
|
+
upstream_proxies: list[UpstreamProxyEntry] = Field(
|
|
134
|
+
default_factory=list,
|
|
135
|
+
description=(
|
|
136
|
+
"Built-in model-provider names (openai, anthropic, openrouter) and/or "
|
|
137
|
+
"CustomUpstreamProxy objects for third-party HTTPS APIs."
|
|
138
|
+
),
|
|
139
|
+
)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Canonical names and shared constants for the internet simulator."""
|
|
2
|
+
|
|
3
|
+
INTERNET_SIMULATOR_RESOURCE_BASENAME = "internet-simulator"
|
|
4
|
+
|
|
5
|
+
# Sandbox name scorers address; the Docker container is per image (runtimes.docker.simulator_container_name).
|
|
6
|
+
INTERNET_SIMULATOR_CONTAINER_NAME = INTERNET_SIMULATOR_RESOURCE_BASENAME
|
|
7
|
+
INTERNET_SIMULATOR_PROXY_CONTAINER_PREFIX = "lb-nginx-proxy"
|
|
8
|
+
INTERNET_SIMULATOR_SERVICE_NAME = INTERNET_SIMULATOR_RESOURCE_BASENAME
|
|
9
|
+
|
|
10
|
+
TRAJECTORY_HEADER = "X-LB-Trajectory-Id"
|
|
11
|
+
|
|
12
|
+
# Shared HTTP proxy constants — used by both the upstream proxy and the persona
|
|
13
|
+
# backend proxy. (connect, read) timeout in seconds; RFC 7230 hop-by-hop headers
|
|
14
|
+
# that must not be forwarded.
|
|
15
|
+
PROXY_REQUEST_TIMEOUT: tuple[int, int] = (10, 600)
|
|
16
|
+
HOP_BY_HOP_REQUEST_HEADERS: frozenset[str] = frozenset(
|
|
17
|
+
{
|
|
18
|
+
"connection",
|
|
19
|
+
"content-length",
|
|
20
|
+
"host",
|
|
21
|
+
"keep-alive",
|
|
22
|
+
"proxy-authenticate",
|
|
23
|
+
"proxy-authorization",
|
|
24
|
+
"te",
|
|
25
|
+
"trailer",
|
|
26
|
+
"transfer-encoding",
|
|
27
|
+
"upgrade",
|
|
28
|
+
}
|
|
29
|
+
)
|
|
30
|
+
HOP_BY_HOP_RESPONSE_HEADERS: frozenset[str] = frozenset(
|
|
31
|
+
{
|
|
32
|
+
"connection",
|
|
33
|
+
"content-encoding",
|
|
34
|
+
"content-length",
|
|
35
|
+
"keep-alive",
|
|
36
|
+
"proxy-authenticate",
|
|
37
|
+
"proxy-authorization",
|
|
38
|
+
"te",
|
|
39
|
+
"trailer",
|
|
40
|
+
"transfer-encoding",
|
|
41
|
+
"upgrade",
|
|
42
|
+
}
|
|
43
|
+
)
|