circumstances 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ """Circumstances: situational testing for agents."""
2
+
3
+ from circumstances.models import FORMAT_VERSION
4
+
5
+ __all__ = ["FORMAT_VERSION"]
6
+ __version__ = "0.2.0"
7
+
@@ -0,0 +1,2 @@
1
+ """First-party neutral subject adapters."""
2
+
@@ -0,0 +1,127 @@
1
+ """Neutral OpenAI-compatible chat subject adapter."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ from collections.abc import Sequence
7
+
8
+ from circumstances.models import (
9
+ Action,
10
+ Event,
11
+ GeneratedMessageAction,
12
+ MessageAction,
13
+ MessageEvent,
14
+ ObservationEvent,
15
+ PluginDescriptor,
16
+ TrialSpec,
17
+ WaitAction,
18
+ )
19
+
20
+
21
+ class OpenAICompatibleTrialSession:
22
+ def __init__(
23
+ self,
24
+ *,
25
+ base_url: str,
26
+ api_key: str,
27
+ model: str,
28
+ system_prompt: str | None,
29
+ ) -> None:
30
+ try:
31
+ import httpx
32
+ except ImportError as exc:
33
+ raise RuntimeError("install circumstances[http] for the OpenAI adapter") from exc
34
+ self._httpx = httpx
35
+ self.client = httpx.AsyncClient(
36
+ base_url=base_url.rstrip("/"),
37
+ headers={"Authorization": f"Bearer {api_key}"} if api_key else {},
38
+ timeout=120,
39
+ )
40
+ self.model = model
41
+ self.messages: list[dict[str, str]] = []
42
+ if system_prompt:
43
+ self.messages.append({"role": "system", "content": system_prompt})
44
+
45
+ async def execute(self, action: Action, *, timeout_s: float) -> Sequence[Event]:
46
+ if isinstance(action, WaitAction):
47
+ return [
48
+ ObservationEvent(
49
+ sequence=0,
50
+ name="wait.skipped",
51
+ value=action.until or action.duration_s,
52
+ )
53
+ ]
54
+ if isinstance(action, GeneratedMessageAction):
55
+ content = action.prompt
56
+ elif isinstance(action, MessageAction):
57
+ if action.role not in {"user", "system"}:
58
+ raise ValueError(f"OpenAI adapter cannot send role {action.role!r}")
59
+ content = action.content
60
+ else:
61
+ raise ValueError(
62
+ f"OpenAI-compatible adapter does not support {action.kind!r}; "
63
+ "use a subject plugin with the required capability"
64
+ )
65
+ role = action.role if isinstance(action, MessageAction) else "user"
66
+ self.messages.append({"role": role, "content": content})
67
+ response = await self.client.post(
68
+ "/chat/completions",
69
+ json={"model": self.model, "messages": self.messages, "temperature": 0},
70
+ timeout=timeout_s,
71
+ )
72
+ response.raise_for_status()
73
+ payload = response.json()
74
+ try:
75
+ reply = payload["choices"][0]["message"]["content"]
76
+ except (KeyError, IndexError, TypeError) as exc:
77
+ raise ValueError("OpenAI-compatible response has no assistant content") from exc
78
+ if not isinstance(reply, str):
79
+ raise ValueError("OpenAI-compatible assistant content must be text")
80
+ self.messages.append({"role": "assistant", "content": reply})
81
+ return [
82
+ MessageEvent(sequence=0, role=role, content=content),
83
+ MessageEvent(sequence=0, role="assistant", content=reply),
84
+ ]
85
+
86
+ async def close(self) -> None:
87
+ await self.client.aclose()
88
+
89
+
90
+ class OpenAICompatibleSubjectPlugin:
91
+ def __init__(
92
+ self,
93
+ *,
94
+ base_url: str | None = None,
95
+ api_key: str | None = None,
96
+ model: str | None = None,
97
+ system_prompt: str | None = None,
98
+ ) -> None:
99
+ self.base_url = base_url or os.environ.get(
100
+ "CIRCUMSTANCES_OPENAI_BASE_URL", "http://127.0.0.1:11434/v1"
101
+ )
102
+ self.api_key = api_key if api_key is not None else os.environ.get(
103
+ "CIRCUMSTANCES_OPENAI_API_KEY", ""
104
+ )
105
+ self.model = model or os.environ.get("CIRCUMSTANCES_OPENAI_MODEL", "")
106
+ self.system_prompt = system_prompt
107
+
108
+ @property
109
+ def descriptor(self) -> PluginDescriptor:
110
+ return PluginDescriptor(
111
+ id="openai-compatible",
112
+ version="1",
113
+ capabilities=["message.text", "message.generated"],
114
+ models={"subject": self.model} if self.model else {},
115
+ )
116
+
117
+ async def prepare(self, trial: TrialSpec) -> OpenAICompatibleTrialSession:
118
+ del trial
119
+ if not self.model:
120
+ raise ValueError("CIRCUMSTANCES_OPENAI_MODEL is required")
121
+ return OpenAICompatibleTrialSession(
122
+ base_url=self.base_url,
123
+ api_key=self.api_key,
124
+ model=self.model,
125
+ system_prompt=self.system_prompt,
126
+ )
127
+
@@ -0,0 +1,142 @@
1
+ """Deterministic subject used by examples and contract tests."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ from collections.abc import Sequence
7
+
8
+ from circumstances.models import (
9
+ Action,
10
+ EnvironmentAction,
11
+ Event,
12
+ ExtensionAction,
13
+ GeneratedMessageAction,
14
+ JSONValue,
15
+ MessageAction,
16
+ MessageEvent,
17
+ ObservationEvent,
18
+ PluginDescriptor,
19
+ StateChangeEvent,
20
+ ToolCallEvent,
21
+ TrialSpec,
22
+ WaitAction,
23
+ )
24
+
25
+
26
+ class ToyTrialSession:
27
+ def __init__(self, trial: TrialSpec) -> None:
28
+ self.trial = trial
29
+ self.state: dict[str, JSONValue] = {}
30
+ self.closed = False
31
+
32
+ async def execute(self, action: Action, *, timeout_s: float) -> Sequence[Event]:
33
+ del timeout_s
34
+ if self.closed:
35
+ raise RuntimeError("trial session is closed")
36
+ if isinstance(action, MessageAction):
37
+ configured = self._configured_response(action.content)
38
+ if configured is not None:
39
+ return [
40
+ MessageEvent(sequence=0, role=action.role, content=action.content),
41
+ *configured,
42
+ ]
43
+ return [
44
+ MessageEvent(sequence=0, role=action.role, content=action.content),
45
+ MessageEvent(
46
+ sequence=0,
47
+ role="assistant",
48
+ content=f"Observed: {action.content}",
49
+ ),
50
+ ]
51
+ if isinstance(action, GeneratedMessageAction):
52
+ generated = f"[{action.persona or 'user'}] {action.prompt}"
53
+ return [
54
+ MessageEvent(sequence=0, role="user", content=generated),
55
+ MessageEvent(sequence=0, role="assistant", content=f"Observed: {generated}"),
56
+ ]
57
+ if isinstance(action, WaitAction):
58
+ if action.duration_s:
59
+ await asyncio.sleep(min(action.duration_s, 0.01))
60
+ return [
61
+ ObservationEvent(
62
+ sequence=0,
63
+ name="wait.completed",
64
+ value=action.until or action.duration_s,
65
+ )
66
+ ]
67
+ if isinstance(action, EnvironmentAction):
68
+ before = self.state.get(action.name)
69
+ value = action.arguments.get("value", True)
70
+ self.state[action.name] = value
71
+ return [
72
+ StateChangeEvent(
73
+ sequence=0,
74
+ path=action.name,
75
+ before=before,
76
+ after=value,
77
+ ),
78
+ ObservationEvent(sequence=0, name=action.name, value=value),
79
+ ]
80
+ if isinstance(action, ExtensionAction):
81
+ return [
82
+ ObservationEvent(
83
+ sequence=0,
84
+ name=f"{action.namespace}.{action.name}",
85
+ value=action.input,
86
+ )
87
+ ]
88
+ raise TypeError(f"unsupported action: {type(action)!r}")
89
+
90
+ def _configured_response(self, message: str) -> list[Event] | None:
91
+ responses = self.trial.case.fixture.config.get("responses")
92
+ if not isinstance(responses, dict):
93
+ return None
94
+ configured = responses.get(message)
95
+ if not isinstance(configured, dict):
96
+ return None
97
+ events: list[Event] = []
98
+ tools = configured.get("tools", [])
99
+ if isinstance(tools, list):
100
+ for index, name in enumerate(tools):
101
+ if isinstance(name, str):
102
+ events.append(
103
+ ToolCallEvent(
104
+ sequence=0,
105
+ call_id=f"toy-{index}",
106
+ name=name,
107
+ )
108
+ )
109
+ content = configured.get("text")
110
+ if isinstance(content, str):
111
+ events.append(
112
+ MessageEvent(sequence=0, role="assistant", content=content)
113
+ )
114
+ observations = configured.get("observations", {})
115
+ if isinstance(observations, dict):
116
+ for name, value in observations.items():
117
+ events.append(
118
+ ObservationEvent(sequence=0, name=name, value=value)
119
+ )
120
+ return events
121
+
122
+ async def close(self) -> None:
123
+ self.closed = True
124
+
125
+
126
+ class ToySubjectPlugin:
127
+ descriptor = PluginDescriptor(
128
+ id="toy",
129
+ version="1",
130
+ capabilities=[
131
+ "message.text",
132
+ "message.generated",
133
+ "wait",
134
+ "environment.action",
135
+ "extension",
136
+ "state.observe",
137
+ ],
138
+ )
139
+
140
+ async def prepare(self, trial: TrialSpec) -> ToyTrialSession:
141
+ return ToyTrialSession(trial)
142
+
@@ -0,0 +1,108 @@
1
+ """Immutable filesystem run bundles."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from circumstances.models import RunSummary
12
+
13
+
14
+ def _canonical(value: Any) -> bytes:
15
+ return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
16
+
17
+
18
+ class FileArtifactStore:
19
+ """Write a self-contained run bundle and seal it with file digests."""
20
+
21
+ def __init__(self, root: Path) -> None:
22
+ self.root = root.expanduser().resolve()
23
+
24
+ def persist(self, summary: RunSummary, *, pack_files: dict[str, bytes] | None = None) -> Path:
25
+ final_dir = self.root / summary.run_id
26
+ pending = self.root / f".{summary.run_id}.pending"
27
+ if final_dir.exists() or pending.exists():
28
+ raise FileExistsError(f"run bundle already exists: {summary.run_id}")
29
+ pending.mkdir(parents=True)
30
+ try:
31
+ self._write_json(pending / "run.json", summary.model_dump(mode="json"))
32
+ trials_dir = pending / "trials"
33
+ trials_dir.mkdir()
34
+ for result in summary.results:
35
+ self._write_json(
36
+ trials_dir / f"{_safe(result.trial_id)}.json",
37
+ result.model_dump(mode="json"),
38
+ )
39
+ if pack_files:
40
+ snapshot = pending / "pack"
41
+ for relative, content in sorted(pack_files.items()):
42
+ target = (snapshot / relative).resolve()
43
+ if not target.is_relative_to(snapshot.resolve()):
44
+ raise ValueError(f"pack snapshot path escapes root: {relative}")
45
+ target.parent.mkdir(parents=True, exist_ok=True)
46
+ target.write_bytes(content)
47
+ seal = {
48
+ "schema": "circumstances.run-bundle.v1",
49
+ "run_id": summary.run_id,
50
+ "files": _file_digests(pending),
51
+ }
52
+ self._write_json(pending / "seal.json", seal)
53
+ os.replace(pending, final_dir)
54
+ except BaseException:
55
+ _remove_tree(pending)
56
+ raise
57
+ return final_dir
58
+
59
+ @staticmethod
60
+ def _write_json(path: Path, value: Any) -> None:
61
+ path.write_bytes(_canonical(value) + b"\n")
62
+
63
+
64
+ def verify_bundle(path: Path) -> list[str]:
65
+ path = path.resolve()
66
+ seal_path = path / "seal.json"
67
+ seal = json.loads(seal_path.read_text())
68
+ failures: list[str] = []
69
+ expected = seal.get("files", {})
70
+ for relative, digest in expected.items():
71
+ target = (path / relative).resolve()
72
+ if not target.is_relative_to(path) or not target.is_file():
73
+ failures.append(f"missing or unsafe artifact: {relative}")
74
+ continue
75
+ actual = hashlib.sha256(target.read_bytes()).hexdigest()
76
+ if actual != digest:
77
+ failures.append(f"digest mismatch: {relative}")
78
+ extra = set(_file_digests(path)) - set(expected) - {"seal.json"}
79
+ failures.extend(f"unsealed artifact: {name}" for name in sorted(extra))
80
+ return failures
81
+
82
+
83
+ def _file_digests(root: Path) -> dict[str, str]:
84
+ result: dict[str, str] = {}
85
+ for path in sorted(root.rglob("*")):
86
+ if path.is_file() and path.name != "seal.json":
87
+ relative = path.relative_to(root).as_posix()
88
+ result[relative] = hashlib.sha256(path.read_bytes()).hexdigest()
89
+ return result
90
+
91
+
92
+ def _safe(value: str) -> str:
93
+ return "".join(
94
+ character if character.isalnum() or character in "._-" else "_"
95
+ for character in value
96
+ )
97
+
98
+
99
+ def _remove_tree(path: Path) -> None:
100
+ if not path.exists():
101
+ return
102
+ for child in sorted(path.rglob("*"), reverse=True):
103
+ if child.is_file() or child.is_symlink():
104
+ child.unlink()
105
+ elif child.is_dir():
106
+ child.rmdir()
107
+ path.rmdir()
108
+