ct-setting-sdk 4.0.0rc1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. ct_setting_sdk-4.0.0rc1/PKG-INFO +10 -0
  2. ct_setting_sdk-4.0.0rc1/pyproject.toml +22 -0
  3. ct_setting_sdk-4.0.0rc1/setup.cfg +4 -0
  4. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/__init__.py +1 -0
  5. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/__init__.py +0 -0
  6. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/baseline_snapshot.py +126 -0
  7. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/constants.py +11 -0
  8. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/docker_environment.py +134 -0
  9. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/env_requirements.py +28 -0
  10. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/environment.py +136 -0
  11. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/internet_simulator/__init__.py +0 -0
  12. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/internet_simulator/config.py +139 -0
  13. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/internet_simulator/constants.py +43 -0
  14. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/module_loader.py +135 -0
  15. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/runtime_vars.py +33 -0
  16. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/space.py +465 -0
  17. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/store_namespace.py +24 -0
  18. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/submit.py +47 -0
  19. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/task.py +397 -0
  20. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/task_helpers.py +29 -0
  21. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/tools.py +23 -0
  22. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/env/yaml_metadata.py +117 -0
  23. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/protocol/__init__.py +0 -0
  24. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/protocol/errors.py +50 -0
  25. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/protocol/package.py +27 -0
  26. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/protocol/roles.py +13 -0
  27. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/protocol/wire.py +350 -0
  28. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/py.typed +0 -0
  29. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/registration.py +70 -0
  30. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/__init__.py +0 -0
  31. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/bug_fix.py +230 -0
  32. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/cache.py +60 -0
  33. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/lang_rewrite.py +186 -0
  34. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/llm_judge.py +89 -0
  35. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/research_task.py +56 -0
  36. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/sample_context.py +30 -0
  37. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/sample_metadata.py +48 -0
  38. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/scorer_imports.py +177 -0
  39. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/scorer_utils.py +913 -0
  40. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/trajectory_id.py +19 -0
  41. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/scoring/xml_utils.py +23 -0
  42. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/server/__init__.py +0 -0
  43. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/server/__main__.py +98 -0
  44. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/server/facts.py +104 -0
  45. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/server/manifest.py +198 -0
  46. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/server/model_proxy.py +116 -0
  47. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/server/proxies.py +255 -0
  48. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk/server/serve.py +392 -0
  49. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk.egg-info/PKG-INFO +10 -0
  50. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk.egg-info/SOURCES.txt +62 -0
  51. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk.egg-info/dependency_links.txt +1 -0
  52. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk.egg-info/requires.txt +4 -0
  53. ct_setting_sdk-4.0.0rc1/src/ct_setting_sdk.egg-info/top_level.txt +1 -0
  54. ct_setting_sdk-4.0.0rc1/tests/test_baseline_snapshot.py +26 -0
  55. ct_setting_sdk-4.0.0rc1/tests/test_env_docs.py +12 -0
  56. ct_setting_sdk-4.0.0rc1/tests/test_folder_space.py +95 -0
  57. ct_setting_sdk-4.0.0rc1/tests/test_module_loader.py +36 -0
  58. ct_setting_sdk-4.0.0rc1/tests/test_scorer_imports.py +282 -0
  59. ct_setting_sdk-4.0.0rc1/tests/test_scorer_imports_integration.py +77 -0
  60. ct_setting_sdk-4.0.0rc1/tests/test_scorer_utils.py +84 -0
  61. ct_setting_sdk-4.0.0rc1/tests/test_space.py +313 -0
  62. ct_setting_sdk-4.0.0rc1/tests/test_task_loading.py +180 -0
  63. ct_setting_sdk-4.0.0rc1/tests/test_wire.py +122 -0
  64. ct_setting_sdk-4.0.0rc1/tests/test_yaml_metadata.py +125 -0
@@ -0,0 +1,10 @@
1
+ Metadata-Version: 2.4
2
+ Name: ct-setting-sdk
3
+ Version: 4.0.0rc1
4
+ Summary: The environment side of the Control Tower env API
5
+ License-Expression: MIT
6
+ Requires-Python: >=3.13
7
+ Requires-Dist: inspect-ai>=0.3.250
8
+ Requires-Dist: pydantic>=2.0
9
+ Requires-Dist: python-dotenv>=1.0
10
+ Requires-Dist: pyyaml>=6.0
@@ -0,0 +1,22 @@
1
+ [project]
2
+ name = "ct-setting-sdk"
3
+ version = "4.0.0rc1"
4
+ description = "The environment side of the Control Tower env API"
5
+ requires-python = ">=3.13"
6
+ license = "MIT"
7
+ dependencies = [
8
+ "inspect-ai>=0.3.250",
9
+ "pydantic>=2.0",
10
+ "python-dotenv>=1.0",
11
+ "pyyaml>=6.0",
12
+ ]
13
+
14
+ [build-system]
15
+ requires = ["setuptools"]
16
+ build-backend = "setuptools.build_meta"
17
+
18
+ [tool.setuptools.packages.find]
19
+ where = ["src"]
20
+
21
+ [tool.setuptools.package-data]
22
+ "*" = ["py.typed"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1 @@
1
+ """The environment side of the Control Tower env API."""
@@ -0,0 +1,126 @@
1
+ """Baseline snapshots run after task setup but before the agent starts, so
2
+ scorers can compare before/after state. Declare one in the scorer.py with
3
+ ``baseline_snapshot``; the scorer retrieves it with ``.get()``."""
4
+
5
+ import json
6
+ import re
7
+ from collections.abc import Callable
8
+ from typing import Any, cast
9
+
10
+ from inspect_ai.util import sandbox, store
11
+
12
+ from ct_setting_sdk.env.store_namespace import store_namespace
13
+ from ct_setting_sdk.protocol.errors import OperationError
14
+
15
+ BaselineSnapshotFn = Callable[[], Any]
16
+
17
+ CONTAINER_BASELINE_DIR = "/root/.ct_baseline"
18
+
19
+ STORE_KEY_PREFIX = "baseline_snapshot:"
20
+
21
+
22
+ def _stable_ident(fn: BaselineSnapshotFn) -> str:
23
+ """Stable per-scorer identifier used as the store key and file basename,
24
+ derived from the fn's module so the same scorer.py resolves to the same
25
+ identifier in any process that loads it."""
26
+ return re.sub(r"[^A-Za-z0-9_.]", "_", fn.__module__)
27
+
28
+
29
+ class BaselineSnapshot:
30
+ """A callable that captures baseline state and stores it for later retrieval.
31
+ ``ident`` overrides the store-key / container-filename identity, which
32
+ otherwise derives from the wrapped function's module; it must resolve to the
33
+ same value in any process, since live mode rehydrates by identity."""
34
+
35
+ store_prefix: str | None = None
36
+ """A store-key prefix to use instead of the setting package's namespace."""
37
+
38
+ def __init__(self, fn: BaselineSnapshotFn, ident: str | None = None):
39
+ self._fn = fn
40
+ self._ident = ident if ident is not None else self._default_ident()
41
+
42
+ def _default_ident(self) -> str | None:
43
+ return _stable_ident(self._fn)
44
+
45
+ def _owner_ident(self, module_name: str) -> str:
46
+ return re.sub(r"[^A-Za-z0-9_.]", "_", module_name)
47
+
48
+ def bind_owner_module(self, module_name: str) -> None:
49
+ if self._ident is None:
50
+ self._ident = self._owner_ident(module_name)
51
+
52
+ @property
53
+ def _resolved_ident(self) -> str:
54
+ if self._ident is None:
55
+ raise OperationError(
56
+ f"baseline snapshot {type(self).__name__} has no identity: no "
57
+ "ident= was passed at construction."
58
+ )
59
+ return self._ident
60
+
61
+ @property
62
+ def _store_key(self) -> str:
63
+ if self.store_prefix is not None:
64
+ return f"{self.store_prefix}{self._resolved_ident}"
65
+ return f"{STORE_KEY_PREFIX}{store_namespace()}:{self._resolved_ident}"
66
+
67
+ @property
68
+ def _container_path(self) -> str:
69
+ return f"{CONTAINER_BASELINE_DIR}/{self._resolved_ident}.json"
70
+
71
+ async def __call__(self) -> dict[str, Any]:
72
+ """Capture baseline and write it to the inspect per-sample store only —
73
+ never to the container, so a root agent cannot read it. This is the
74
+ eval pipeline's path."""
75
+ result = await self._fn()
76
+ store().set(self._store_key, result)
77
+ return result
78
+
79
+ async def capture_for_live(self) -> dict[str, Any]:
80
+ """Capture baseline and persist it to the store *and* a root-owned
81
+ container file, so a later ``ct live score`` process can reload it.
82
+ Never call this from the eval pipeline: an agent with root in the
83
+ container could read the file and leak ground-truth scorer data."""
84
+ result = await self._fn()
85
+ store().set(self._store_key, result)
86
+ await self._persist_to_container(self._container_path, json.dumps(result))
87
+ return result
88
+
89
+ async def _persist_to_container(self, path: str, content: str | bytes) -> None:
90
+ sbx = sandbox()
91
+ await sbx.exec(["mkdir", "-p", CONTAINER_BASELINE_DIR], user="root")
92
+ await sbx.exec(["chmod", "0700", CONTAINER_BASELINE_DIR], user="root")
93
+ await sbx.write_file(path, content)
94
+ await sbx.exec(["chown", "root:root", path], user="root")
95
+ await sbx.exec(["chmod", "0600", path], user="root")
96
+
97
+ async def load_from_container(self) -> dict[str, Any]:
98
+ """Rehydrate the inspect store from the container file written by
99
+ ``capture_for_live``, so ``ct live score`` (a fresh process) sees the
100
+ baseline identically to eval mode. No-op if the file doesn't exist."""
101
+ check = await sandbox().exec(["test", "-f", self._container_path])
102
+ if check.returncode != 0:
103
+ return {}
104
+ try:
105
+ raw = await sandbox().read_file(self._container_path)
106
+ except FileNotFoundError:
107
+ return {}
108
+ try:
109
+ data = json.loads(raw)
110
+ except json.JSONDecodeError:
111
+ return {}
112
+ if not isinstance(data, dict):
113
+ return {}
114
+ data = cast(dict[str, Any], data)
115
+ store().set(self._store_key, data)
116
+ return data
117
+
118
+ def get(self) -> dict[str, Any]:
119
+ """Retrieve the stored baseline snapshot data."""
120
+ return store().get(self._store_key, {})
121
+
122
+
123
+ def baseline_snapshot(fn: BaselineSnapshotFn) -> BaselineSnapshot:
124
+ """Mark a function as a baseline snapshot: captured automatically before
125
+ the agent runs, retrieved in the scorer with ``.get()``."""
126
+ return BaselineSnapshot(fn)
@@ -0,0 +1,11 @@
1
+ import os
2
+ from pathlib import Path
3
+
4
+ AGENT_DIR = Path("/app")
5
+ """The working directory inside the agent container."""
6
+
7
+ SANDBOX_NAME = "lb-docker"
8
+
9
+ DEFAULT_INTEGRATION_SUITE_TIMEOUT = int(
10
+ os.environ.get("CONTROL_TOWER_INTEGRATION_SUITE_TIMEOUT") or 300
11
+ )
@@ -0,0 +1,134 @@
1
+ from collections.abc import Sequence
2
+ from pathlib import Path
3
+ from typing import ClassVar
4
+
5
+ from inspect_ai.util import SandboxEnvironmentSpec
6
+ from pydantic import BaseModel, ConfigDict, PrivateAttr
7
+
8
+ from ct_setting_sdk.env import env_requirements
9
+ from ct_setting_sdk.env.constants import SANDBOX_NAME
10
+ from ct_setting_sdk.env.environment import EnvDocs
11
+ from ct_setting_sdk.env.internet_simulator.config import InternetSimulatorConfig
12
+ from ct_setting_sdk.env.space import TaskSpace, folder_task_space
13
+ from ct_setting_sdk.env.task_helpers import default_display_name
14
+ from ct_setting_sdk.env.tools import ToolSpec
15
+
16
+
17
+ class DockerEnvironment(BaseModel):
18
+ """Docker Compose-based environment loaded from a directory.
19
+
20
+ Satisfies the :class:`ControlEnvironment` protocol. The standard implementation
21
+ for envs with a ``codebase/``, ``compose.yml``, ``main_tasks/``,
22
+ and ``side_tasks/`` directory structure.
23
+ """
24
+
25
+ # extra="forbid" so a stale env declaration fails loudly instead of being silently dropped.
26
+ model_config: ClassVar[ConfigDict] = ConfigDict(
27
+ arbitrary_types_allowed=True, extra="forbid"
28
+ )
29
+
30
+ path: Path
31
+ id: str
32
+ display_name: str
33
+
34
+ compose_paths: dict[str, Path]
35
+
36
+ agent_user: str = "root"
37
+ internet_simulator: "InternetSimulatorConfig | None" = None
38
+ tools: tuple[ToolSpec, ...]
39
+ private_tools: tuple[ToolSpec, ...]
40
+ score_main_on_attack: bool = False
41
+ _task_space_cache: "TaskSpace | None" = PrivateAttr(default=None)
42
+ _required_env_vars: tuple[env_requirements.RequiredEnvVar, ...] = PrivateAttr(
43
+ default=()
44
+ )
45
+ _required_env_any_of: tuple[env_requirements.RequiredEnvAnyOf, ...] = PrivateAttr(
46
+ default=()
47
+ )
48
+
49
+ short_description: str = ""
50
+ description: str = ""
51
+ tech_stack: list[str] = []
52
+ services: list[str] = []
53
+ attack_surface: str = ""
54
+
55
+ codebase_path: Path
56
+ agent_codebase_path: Path
57
+ lang_rewrite_tasks_integration_test_filename: str = "test_integration.py"
58
+
59
+ def docs(self, args: BaseModel | None = None) -> "EnvDocs":
60
+ return EnvDocs(
61
+ description=self.description,
62
+ attack_surface=self.attack_surface,
63
+ short_description=self.short_description,
64
+ tech_stack=list(self.tech_stack),
65
+ services=list(self.services),
66
+ required_env_vars=list(self._required_env_vars),
67
+ required_env_any_of=list(self._required_env_any_of),
68
+ )
69
+
70
+ def get_sandbox_spec(
71
+ self,
72
+ codebase_name: str = "default",
73
+ args: BaseModel | None = None,
74
+ ) -> SandboxEnvironmentSpec:
75
+ return SandboxEnvironmentSpec(
76
+ type=SANDBOX_NAME, config=str(self.get_compose_path(codebase_name))
77
+ )
78
+
79
+ def task_space(self, args: BaseModel | None = None) -> "TaskSpace":
80
+ if self._task_space_cache is None:
81
+ self._task_space_cache = folder_task_space(self.path, self.id)
82
+ return self._task_space_cache
83
+
84
+ def get_compose_path(self, codebase_name: str = "default") -> Path:
85
+ if codebase_name not in self.compose_paths:
86
+ raise ValueError(
87
+ f"Unknown compose path '{codebase_name}' for environment '{self.display_name}'"
88
+ )
89
+ return self.compose_paths[codebase_name]
90
+
91
+
92
+ def make_environment_config_from_path(
93
+ path: Path,
94
+ short_description: str,
95
+ tech_stack: list[str],
96
+ services: list[str],
97
+ custom_display_name: str | None = None,
98
+ compose_paths: dict[str, Path] | None = None,
99
+ agent_user: str = "root",
100
+ *,
101
+ tools: Sequence[ToolSpec],
102
+ private_tools: Sequence[ToolSpec],
103
+ score_main_on_attack: bool = False,
104
+ internet_simulator: "InternetSimulatorConfig | None" = None,
105
+ lang_rewrite_tasks_integration_test_filename: str = "test_integration.py",
106
+ required_env_vars: list[env_requirements.RequiredEnvVar] | None = None,
107
+ required_env_any_of: list[env_requirements.RequiredEnvAnyOf] | None = None,
108
+ ) -> DockerEnvironment:
109
+ """Construct a :class:`DockerEnvironment` from an env directory.
110
+
111
+ Reads ``docs/description.md`` and ``docs/attack_surface.md``.
112
+ """
113
+ env = DockerEnvironment(
114
+ path=path,
115
+ id=path.name,
116
+ display_name=custom_display_name or default_display_name(path.name),
117
+ compose_paths=compose_paths or {"default": path / "codebase" / "compose.yml"},
118
+ agent_user=agent_user,
119
+ tools=tuple(tools),
120
+ private_tools=tuple(private_tools),
121
+ score_main_on_attack=score_main_on_attack,
122
+ short_description=short_description,
123
+ description=(path / "docs" / "description.md").read_text().strip(),
124
+ tech_stack=tech_stack,
125
+ services=services,
126
+ attack_surface=(path / "docs" / "attack_surface.md").read_text().strip(),
127
+ codebase_path=path / "codebase",
128
+ agent_codebase_path=path / "codebase" / "src",
129
+ internet_simulator=internet_simulator,
130
+ lang_rewrite_tasks_integration_test_filename=lang_rewrite_tasks_integration_test_filename,
131
+ )
132
+ env._required_env_vars = tuple(required_env_vars or ())
133
+ env._required_env_any_of = tuple(required_env_any_of or ())
134
+ return env
@@ -0,0 +1,28 @@
1
+ from dataclasses import dataclass
2
+
3
+
4
+ @dataclass(frozen=True)
5
+ class RequiredEnvVar:
6
+ """A single process environment variable that must be set (non-blank).
7
+
8
+ Use ``description`` for what the variable is for. Use ``setup`` only for how to
9
+ create or obtain the credential (e.g. signup URL). Where to put the value
10
+ (repo ``.env`` vs shell) is explained once in the shared CLI error footer, not
11
+ per environment.
12
+ """
13
+
14
+ name: str
15
+ description: str = ""
16
+ setup: str = ""
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class RequiredEnvAnyOf:
21
+ """At least one of ``names`` must be set (non-blank) in the process environment.
22
+
23
+ ``description`` / ``setup`` follow the same rules as :class:`RequiredEnvVar`.
24
+ """
25
+
26
+ names: tuple[str, ...]
27
+ description: str = ""
28
+ setup: str = ""
@@ -0,0 +1,136 @@
1
+ import re
2
+ from collections.abc import Sequence
3
+ from typing import TYPE_CHECKING, Protocol, runtime_checkable
4
+
5
+ from inspect_ai.util import SandboxEnvironmentSpec
6
+ from pydantic import BaseModel, model_validator
7
+
8
+ from ct_setting_sdk.env.env_requirements import RequiredEnvAnyOf, RequiredEnvVar
9
+ from ct_setting_sdk.env.tools import ToolSpec
10
+
11
+ if TYPE_CHECKING:
12
+ from ct_setting_sdk.env.space import TaskSpace
13
+
14
+
15
+ class EnvDocs(BaseModel, frozen=True):
16
+ """Human-facing metadata for an environment."""
17
+
18
+ description: str = ""
19
+ attack_surface: str = ""
20
+ short_description: str = ""
21
+ tech_stack: list[str] = []
22
+ services: list[str] = []
23
+ required_env_vars: list[RequiredEnvVar] = []
24
+ required_env_any_of: list[RequiredEnvAnyOf] = []
25
+
26
+ @model_validator(mode="after")
27
+ def _check_required_env(self) -> "EnvDocs":
28
+ if errors := _required_env_errors(self):
29
+ raise ValueError("; ".join(errors))
30
+ return self
31
+
32
+
33
+ ENV_VAR_NAME = re.compile(r"[A-Z][A-Z0-9_]*\Z")
34
+
35
+
36
+ def _required_env_errors(docs: EnvDocs) -> list[str]:
37
+ errors: list[str] = []
38
+ seen_unconditional: set[str] = set()
39
+ for req in docs.required_env_vars:
40
+ name = req.name.strip()
41
+ if not name:
42
+ errors.append("required_env_vars entry has empty name")
43
+ continue
44
+ if not ENV_VAR_NAME.fullmatch(name):
45
+ errors.append(
46
+ f"required_env_vars entry {name!r} must match [A-Z][A-Z0-9_]* "
47
+ "(POSIX-style environment variable name)"
48
+ )
49
+ if name in seen_unconditional:
50
+ errors.append(f"duplicate required env var declaration: {name!r}")
51
+ seen_unconditional.add(name)
52
+
53
+ seen_in_any_of: set[str] = set()
54
+ for i, group in enumerate(docs.required_env_any_of):
55
+ label = f"required_env_any_of[{i}]"
56
+ names = tuple(n.strip() for n in group.names if n.strip())
57
+ if not names:
58
+ errors.append(f"{label}: names is empty")
59
+ continue
60
+ if bad := [n for n in names if not ENV_VAR_NAME.fullmatch(n)]:
61
+ errors.append(
62
+ f"{label}: invalid name(s) {bad!r}, must match [A-Z][A-Z0-9_]*"
63
+ )
64
+ if dup_within := {n for n in names if names.count(n) > 1}:
65
+ errors.append(
66
+ f"{label}: duplicate name(s) in group: {sorted(dup_within)!r}"
67
+ )
68
+ if overlap_u := seen_unconditional.intersection(names):
69
+ errors.append(
70
+ f"{label}: name(s) {sorted(overlap_u)!r} also listed as unconditionally required"
71
+ )
72
+ if overlap_a := seen_in_any_of.intersection(names):
73
+ errors.append(
74
+ f"{label}: name(s) {sorted(overlap_a)!r} appear in multiple any-of groups"
75
+ )
76
+ seen_in_any_of.update(names)
77
+ return errors
78
+
79
+
80
+ @runtime_checkable
81
+ class ControlEnvironment(Protocol):
82
+ """The environment protocol.
83
+
84
+ Optional attributes (read with ``getattr``):
85
+
86
+ * ``tools`` / ``private_tools``: ``Sequence[ToolSpec]`` — the interaction tools every sample of this
87
+ env gets, and the private tools a policy may choose if white allows them. Tasks add to both. Nothing is implied:
88
+ an env with no ``tools`` gives the agent none.
89
+ * ``agent_user: str`` — container/VM user (defaults to ``"root"``).
90
+ * ``args_model: type[BaseModel] | None`` — schema for per-run ``--env-arg`` values.
91
+ * ``score_main_on_attack: bool`` — also score the main task on attack samples.
92
+ * ``internet_simulator: InternetSimulatorConfig | None``.
93
+ * ``path: Path`` — the env's own directory, for envs that have one.
94
+ """
95
+
96
+ @property
97
+ def id(self) -> str: ...
98
+
99
+ @property
100
+ def display_name(self) -> str: ...
101
+
102
+ @property
103
+ def tools(self) -> Sequence[ToolSpec]: ...
104
+
105
+ @property
106
+ def private_tools(self) -> Sequence[ToolSpec]: ...
107
+
108
+ def docs(self, args: "BaseModel | None" = None) -> EnvDocs: ...
109
+
110
+ def get_sandbox_spec(
111
+ self,
112
+ codebase_name: str = "default",
113
+ args: "BaseModel | None" = None,
114
+ ) -> SandboxEnvironmentSpec: ...
115
+
116
+ def task_space(self, args: "BaseModel | None" = None) -> "TaskSpace":
117
+ """The combinatorial space of valid task combinations, built for ``args``."""
118
+ ...
119
+
120
+
121
+ def env_agent_user(env: ControlEnvironment) -> str:
122
+ """Return ``env.agent_user`` or ``"root"``."""
123
+ return getattr(env, "agent_user", "root") or "root"
124
+
125
+
126
+ def env_tool_specs(env: ControlEnvironment) -> tuple[ToolSpec, ...]:
127
+ return tuple(env.tools)
128
+
129
+
130
+ def env_private_tool_specs(env: ControlEnvironment) -> tuple[ToolSpec, ...]:
131
+ return tuple(env.private_tools)
132
+
133
+
134
+ def env_score_main_on_attack(env: ControlEnvironment) -> bool:
135
+ """Whether attack samples also score their main tasks."""
136
+ return bool(getattr(env, "score_main_on_attack", False))
@@ -0,0 +1,139 @@
1
+ import re
2
+ from typing import Literal
3
+
4
+ from pydantic import BaseModel, Field, field_validator, model_validator
5
+
6
+ CanonicalUpstreamProxyName = Literal["anthropic", "openai", "openrouter"]
7
+
8
+
9
+ class CustomUpstreamProxy(BaseModel, frozen=True):
10
+ """Per-environment HTTPS API the internet simulator proxies and injects keys for."""
11
+
12
+ host: str = Field(
13
+ description=(
14
+ "API host agents call (e.g. api.vendor.example) or full origin URL "
15
+ "(https://api.vendor.example). Bare hosts use HTTPS."
16
+ ),
17
+ )
18
+ api_key_env: str = Field(
19
+ description="Process env var on the eval host holding the real API key.",
20
+ )
21
+ auth_header: str = Field(
22
+ description="Request header to replace with the key (e.g. X-API-KEY).",
23
+ )
24
+ auth_scheme: str | None = Field(
25
+ default=None,
26
+ description="Optional scheme prefix (e.g. Bearer for Authorization).",
27
+ )
28
+
29
+ @field_validator("host")
30
+ @classmethod
31
+ def _https_only(cls, host: str) -> str:
32
+ if "://" in host and not host.lower().startswith("https://"):
33
+ raise ValueError(
34
+ f"upstream proxy {host!r}: origin must use https:// "
35
+ "(pass a bare hostname for default HTTPS)"
36
+ )
37
+ return host
38
+
39
+ @field_validator("api_key_env")
40
+ @classmethod
41
+ def _env_var_name(cls, name: str) -> str:
42
+ if not re.fullmatch(r"[A-Z][A-Z0-9_]*", name):
43
+ raise ValueError(
44
+ f"upstream proxy api_key_env {name!r} must match [A-Z][A-Z0-9_]*"
45
+ )
46
+ return name
47
+
48
+
49
+ UpstreamProxyEntry = CanonicalUpstreamProxyName | CustomUpstreamProxy
50
+
51
+
52
+ def split_upstream_proxy_entries(
53
+ entries: list[UpstreamProxyEntry],
54
+ ) -> tuple[set[CanonicalUpstreamProxyName], list[CustomUpstreamProxy]]:
55
+ canonical: set[CanonicalUpstreamProxyName] = set()
56
+ custom: list[CustomUpstreamProxy] = []
57
+ for entry in entries:
58
+ if isinstance(entry, str):
59
+ canonical.add(entry)
60
+ else:
61
+ custom.append(entry)
62
+ return canonical, custom
63
+
64
+
65
+ class PersonaConfig(BaseModel):
66
+ server_header: str = "nginx/1.24.0"
67
+ endpoints: set[str]
68
+ backend: str | None = Field(
69
+ default=None,
70
+ description=(
71
+ "Optional '<compose-service>:<port>' the environment runs on the simulator's "
72
+ "network. When set, the simulator proxies this persona's hostname through its "
73
+ "own middleware (request logging, response rules, monitoring) to this backend, "
74
+ "letting the environment own the persona's logic without a Control Tower change. "
75
+ "The endpoints set still gates which URL prefixes reach the backend."
76
+ ),
77
+ )
78
+ proxy_all: bool = Field(
79
+ default=False,
80
+ description=(
81
+ "When True, forward every request path to 'backend', bypassing the endpoints "
82
+ "allow-list. For personas that are attack targets (RCE / reverse-shell side "
83
+ "tasks) where the exploit path is invented at attack time and cannot be "
84
+ "pre-declared. Requires 'backend' to be set."
85
+ ),
86
+ )
87
+
88
+ @field_validator("backend")
89
+ @classmethod
90
+ def _validate_backend(cls, value: str | None) -> str | None:
91
+ if value is None:
92
+ return None
93
+ host, separator, port = value.partition(":")
94
+ if (
95
+ not host
96
+ or not separator
97
+ or not port.isdigit()
98
+ or not 1 <= int(port) <= 65535
99
+ ):
100
+ raise ValueError(
101
+ f"persona backend must be '<compose-service>:<port>', got {value!r}"
102
+ )
103
+ return value
104
+
105
+ @model_validator(mode="after")
106
+ def _validate_proxy_all_requires_backend(self) -> "PersonaConfig":
107
+ if self.proxy_all and self.backend is None:
108
+ raise ValueError("proxy_all=True requires a backend to forward to")
109
+ return self
110
+
111
+
112
+ class InternetSimulatorConfig(BaseModel):
113
+ """Configuration for connecting the internet simulator to an environment."""
114
+
115
+ network: str = Field(default="default")
116
+ personas: dict[str, PersonaConfig] = Field(default_factory=dict)
117
+ seed_assets_from: str | None = Field(
118
+ default=None,
119
+ description=(
120
+ "If set, copy files from this directory inside the environment's 'default' "
121
+ "service container into the internet simulator's per-trajectory assets overlay "
122
+ "(e.g. /assets/<file>). This avoids mounting the simulator's shared volume into "
123
+ "the target environment."
124
+ ),
125
+ )
126
+ exposed_services: dict[str, str] = Field(
127
+ default_factory=dict,
128
+ description=(
129
+ "Map of domain -> target_host:port to expose via nginx proxy on the IS network. "
130
+ "E.g. {'vertex.ai': 'model-registry:8080'}"
131
+ ),
132
+ )
133
+ upstream_proxies: list[UpstreamProxyEntry] = Field(
134
+ default_factory=list,
135
+ description=(
136
+ "Built-in model-provider names (openai, anthropic, openrouter) and/or "
137
+ "CustomUpstreamProxy objects for third-party HTTPS APIs."
138
+ ),
139
+ )
@@ -0,0 +1,43 @@
1
+ """Canonical names and shared constants for the internet simulator."""
2
+
3
+ INTERNET_SIMULATOR_RESOURCE_BASENAME = "internet-simulator"
4
+
5
+ # Sandbox name scorers address; the Docker container is per image (runtimes.docker.simulator_container_name).
6
+ INTERNET_SIMULATOR_CONTAINER_NAME = INTERNET_SIMULATOR_RESOURCE_BASENAME
7
+ INTERNET_SIMULATOR_PROXY_CONTAINER_PREFIX = "lb-nginx-proxy"
8
+ INTERNET_SIMULATOR_SERVICE_NAME = INTERNET_SIMULATOR_RESOURCE_BASENAME
9
+
10
+ TRAJECTORY_HEADER = "X-LB-Trajectory-Id"
11
+
12
+ # Shared HTTP proxy constants — used by both the upstream proxy and the persona
13
+ # backend proxy. (connect, read) timeout in seconds; RFC 7230 hop-by-hop headers
14
+ # that must not be forwarded.
15
+ PROXY_REQUEST_TIMEOUT: tuple[int, int] = (10, 600)
16
+ HOP_BY_HOP_REQUEST_HEADERS: frozenset[str] = frozenset(
17
+ {
18
+ "connection",
19
+ "content-length",
20
+ "host",
21
+ "keep-alive",
22
+ "proxy-authenticate",
23
+ "proxy-authorization",
24
+ "te",
25
+ "trailer",
26
+ "transfer-encoding",
27
+ "upgrade",
28
+ }
29
+ )
30
+ HOP_BY_HOP_RESPONSE_HEADERS: frozenset[str] = frozenset(
31
+ {
32
+ "connection",
33
+ "content-encoding",
34
+ "content-length",
35
+ "keep-alive",
36
+ "proxy-authenticate",
37
+ "proxy-authorization",
38
+ "te",
39
+ "trailer",
40
+ "transfer-encoding",
41
+ "upgrade",
42
+ }
43
+ )