circumstances 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- circumstances/__init__.py +7 -0
- circumstances/adapters/__init__.py +2 -0
- circumstances/adapters/openai_compatible.py +127 -0
- circumstances/adapters/toy.py +142 -0
- circumstances/artifacts.py +108 -0
- circumstances/bundles.py +320 -0
- circumstances/cli.py +427 -0
- circumstances/errors.py +15 -0
- circumstances/evaluators.py +232 -0
- circumstances/fetch.py +422 -0
- circumstances/migration.py +193 -0
- circumstances/models.py +373 -0
- circumstances/pack.py +152 -0
- circumstances/pack_index.yaml +25 -0
- circumstances/plugins.py +56 -0
- circumstances/provider.py +121 -0
- circumstances/py.typed +1 -0
- circumstances/registry.py +117 -0
- circumstances/runner.py +214 -0
- circumstances/scaffold.py +286 -0
- circumstances/schema.py +37 -0
- circumstances/selftest.py +39 -0
- circumstances/sources.py +178 -0
- circumstances/stats.py +64 -0
- circumstances/view.py +240 -0
- circumstances-0.2.0.dist-info/METADATA +202 -0
- circumstances-0.2.0.dist-info/RECORD +30 -0
- circumstances-0.2.0.dist-info/WHEEL +4 -0
- circumstances-0.2.0.dist-info/entry_points.txt +7 -0
- circumstances-0.2.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Neutral OpenAI-compatible chat subject adapter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
|
|
8
|
+
from circumstances.models import (
|
|
9
|
+
Action,
|
|
10
|
+
Event,
|
|
11
|
+
GeneratedMessageAction,
|
|
12
|
+
MessageAction,
|
|
13
|
+
MessageEvent,
|
|
14
|
+
ObservationEvent,
|
|
15
|
+
PluginDescriptor,
|
|
16
|
+
TrialSpec,
|
|
17
|
+
WaitAction,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class OpenAICompatibleTrialSession:
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
*,
|
|
25
|
+
base_url: str,
|
|
26
|
+
api_key: str,
|
|
27
|
+
model: str,
|
|
28
|
+
system_prompt: str | None,
|
|
29
|
+
) -> None:
|
|
30
|
+
try:
|
|
31
|
+
import httpx
|
|
32
|
+
except ImportError as exc:
|
|
33
|
+
raise RuntimeError("install circumstances[http] for the OpenAI adapter") from exc
|
|
34
|
+
self._httpx = httpx
|
|
35
|
+
self.client = httpx.AsyncClient(
|
|
36
|
+
base_url=base_url.rstrip("/"),
|
|
37
|
+
headers={"Authorization": f"Bearer {api_key}"} if api_key else {},
|
|
38
|
+
timeout=120,
|
|
39
|
+
)
|
|
40
|
+
self.model = model
|
|
41
|
+
self.messages: list[dict[str, str]] = []
|
|
42
|
+
if system_prompt:
|
|
43
|
+
self.messages.append({"role": "system", "content": system_prompt})
|
|
44
|
+
|
|
45
|
+
async def execute(self, action: Action, *, timeout_s: float) -> Sequence[Event]:
|
|
46
|
+
if isinstance(action, WaitAction):
|
|
47
|
+
return [
|
|
48
|
+
ObservationEvent(
|
|
49
|
+
sequence=0,
|
|
50
|
+
name="wait.skipped",
|
|
51
|
+
value=action.until or action.duration_s,
|
|
52
|
+
)
|
|
53
|
+
]
|
|
54
|
+
if isinstance(action, GeneratedMessageAction):
|
|
55
|
+
content = action.prompt
|
|
56
|
+
elif isinstance(action, MessageAction):
|
|
57
|
+
if action.role not in {"user", "system"}:
|
|
58
|
+
raise ValueError(f"OpenAI adapter cannot send role {action.role!r}")
|
|
59
|
+
content = action.content
|
|
60
|
+
else:
|
|
61
|
+
raise ValueError(
|
|
62
|
+
f"OpenAI-compatible adapter does not support {action.kind!r}; "
|
|
63
|
+
"use a subject plugin with the required capability"
|
|
64
|
+
)
|
|
65
|
+
role = action.role if isinstance(action, MessageAction) else "user"
|
|
66
|
+
self.messages.append({"role": role, "content": content})
|
|
67
|
+
response = await self.client.post(
|
|
68
|
+
"/chat/completions",
|
|
69
|
+
json={"model": self.model, "messages": self.messages, "temperature": 0},
|
|
70
|
+
timeout=timeout_s,
|
|
71
|
+
)
|
|
72
|
+
response.raise_for_status()
|
|
73
|
+
payload = response.json()
|
|
74
|
+
try:
|
|
75
|
+
reply = payload["choices"][0]["message"]["content"]
|
|
76
|
+
except (KeyError, IndexError, TypeError) as exc:
|
|
77
|
+
raise ValueError("OpenAI-compatible response has no assistant content") from exc
|
|
78
|
+
if not isinstance(reply, str):
|
|
79
|
+
raise ValueError("OpenAI-compatible assistant content must be text")
|
|
80
|
+
self.messages.append({"role": "assistant", "content": reply})
|
|
81
|
+
return [
|
|
82
|
+
MessageEvent(sequence=0, role=role, content=content),
|
|
83
|
+
MessageEvent(sequence=0, role="assistant", content=reply),
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
async def close(self) -> None:
|
|
87
|
+
await self.client.aclose()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class OpenAICompatibleSubjectPlugin:
|
|
91
|
+
def __init__(
|
|
92
|
+
self,
|
|
93
|
+
*,
|
|
94
|
+
base_url: str | None = None,
|
|
95
|
+
api_key: str | None = None,
|
|
96
|
+
model: str | None = None,
|
|
97
|
+
system_prompt: str | None = None,
|
|
98
|
+
) -> None:
|
|
99
|
+
self.base_url = base_url or os.environ.get(
|
|
100
|
+
"CIRCUMSTANCES_OPENAI_BASE_URL", "http://127.0.0.1:11434/v1"
|
|
101
|
+
)
|
|
102
|
+
self.api_key = api_key if api_key is not None else os.environ.get(
|
|
103
|
+
"CIRCUMSTANCES_OPENAI_API_KEY", ""
|
|
104
|
+
)
|
|
105
|
+
self.model = model or os.environ.get("CIRCUMSTANCES_OPENAI_MODEL", "")
|
|
106
|
+
self.system_prompt = system_prompt
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def descriptor(self) -> PluginDescriptor:
|
|
110
|
+
return PluginDescriptor(
|
|
111
|
+
id="openai-compatible",
|
|
112
|
+
version="1",
|
|
113
|
+
capabilities=["message.text", "message.generated"],
|
|
114
|
+
models={"subject": self.model} if self.model else {},
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
async def prepare(self, trial: TrialSpec) -> OpenAICompatibleTrialSession:
|
|
118
|
+
del trial
|
|
119
|
+
if not self.model:
|
|
120
|
+
raise ValueError("CIRCUMSTANCES_OPENAI_MODEL is required")
|
|
121
|
+
return OpenAICompatibleTrialSession(
|
|
122
|
+
base_url=self.base_url,
|
|
123
|
+
api_key=self.api_key,
|
|
124
|
+
model=self.model,
|
|
125
|
+
system_prompt=self.system_prompt,
|
|
126
|
+
)
|
|
127
|
+
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Deterministic subject used by examples and contract tests."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
|
|
8
|
+
from circumstances.models import (
|
|
9
|
+
Action,
|
|
10
|
+
EnvironmentAction,
|
|
11
|
+
Event,
|
|
12
|
+
ExtensionAction,
|
|
13
|
+
GeneratedMessageAction,
|
|
14
|
+
JSONValue,
|
|
15
|
+
MessageAction,
|
|
16
|
+
MessageEvent,
|
|
17
|
+
ObservationEvent,
|
|
18
|
+
PluginDescriptor,
|
|
19
|
+
StateChangeEvent,
|
|
20
|
+
ToolCallEvent,
|
|
21
|
+
TrialSpec,
|
|
22
|
+
WaitAction,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ToyTrialSession:
|
|
27
|
+
def __init__(self, trial: TrialSpec) -> None:
|
|
28
|
+
self.trial = trial
|
|
29
|
+
self.state: dict[str, JSONValue] = {}
|
|
30
|
+
self.closed = False
|
|
31
|
+
|
|
32
|
+
async def execute(self, action: Action, *, timeout_s: float) -> Sequence[Event]:
|
|
33
|
+
del timeout_s
|
|
34
|
+
if self.closed:
|
|
35
|
+
raise RuntimeError("trial session is closed")
|
|
36
|
+
if isinstance(action, MessageAction):
|
|
37
|
+
configured = self._configured_response(action.content)
|
|
38
|
+
if configured is not None:
|
|
39
|
+
return [
|
|
40
|
+
MessageEvent(sequence=0, role=action.role, content=action.content),
|
|
41
|
+
*configured,
|
|
42
|
+
]
|
|
43
|
+
return [
|
|
44
|
+
MessageEvent(sequence=0, role=action.role, content=action.content),
|
|
45
|
+
MessageEvent(
|
|
46
|
+
sequence=0,
|
|
47
|
+
role="assistant",
|
|
48
|
+
content=f"Observed: {action.content}",
|
|
49
|
+
),
|
|
50
|
+
]
|
|
51
|
+
if isinstance(action, GeneratedMessageAction):
|
|
52
|
+
generated = f"[{action.persona or 'user'}] {action.prompt}"
|
|
53
|
+
return [
|
|
54
|
+
MessageEvent(sequence=0, role="user", content=generated),
|
|
55
|
+
MessageEvent(sequence=0, role="assistant", content=f"Observed: {generated}"),
|
|
56
|
+
]
|
|
57
|
+
if isinstance(action, WaitAction):
|
|
58
|
+
if action.duration_s:
|
|
59
|
+
await asyncio.sleep(min(action.duration_s, 0.01))
|
|
60
|
+
return [
|
|
61
|
+
ObservationEvent(
|
|
62
|
+
sequence=0,
|
|
63
|
+
name="wait.completed",
|
|
64
|
+
value=action.until or action.duration_s,
|
|
65
|
+
)
|
|
66
|
+
]
|
|
67
|
+
if isinstance(action, EnvironmentAction):
|
|
68
|
+
before = self.state.get(action.name)
|
|
69
|
+
value = action.arguments.get("value", True)
|
|
70
|
+
self.state[action.name] = value
|
|
71
|
+
return [
|
|
72
|
+
StateChangeEvent(
|
|
73
|
+
sequence=0,
|
|
74
|
+
path=action.name,
|
|
75
|
+
before=before,
|
|
76
|
+
after=value,
|
|
77
|
+
),
|
|
78
|
+
ObservationEvent(sequence=0, name=action.name, value=value),
|
|
79
|
+
]
|
|
80
|
+
if isinstance(action, ExtensionAction):
|
|
81
|
+
return [
|
|
82
|
+
ObservationEvent(
|
|
83
|
+
sequence=0,
|
|
84
|
+
name=f"{action.namespace}.{action.name}",
|
|
85
|
+
value=action.input,
|
|
86
|
+
)
|
|
87
|
+
]
|
|
88
|
+
raise TypeError(f"unsupported action: {type(action)!r}")
|
|
89
|
+
|
|
90
|
+
def _configured_response(self, message: str) -> list[Event] | None:
|
|
91
|
+
responses = self.trial.case.fixture.config.get("responses")
|
|
92
|
+
if not isinstance(responses, dict):
|
|
93
|
+
return None
|
|
94
|
+
configured = responses.get(message)
|
|
95
|
+
if not isinstance(configured, dict):
|
|
96
|
+
return None
|
|
97
|
+
events: list[Event] = []
|
|
98
|
+
tools = configured.get("tools", [])
|
|
99
|
+
if isinstance(tools, list):
|
|
100
|
+
for index, name in enumerate(tools):
|
|
101
|
+
if isinstance(name, str):
|
|
102
|
+
events.append(
|
|
103
|
+
ToolCallEvent(
|
|
104
|
+
sequence=0,
|
|
105
|
+
call_id=f"toy-{index}",
|
|
106
|
+
name=name,
|
|
107
|
+
)
|
|
108
|
+
)
|
|
109
|
+
content = configured.get("text")
|
|
110
|
+
if isinstance(content, str):
|
|
111
|
+
events.append(
|
|
112
|
+
MessageEvent(sequence=0, role="assistant", content=content)
|
|
113
|
+
)
|
|
114
|
+
observations = configured.get("observations", {})
|
|
115
|
+
if isinstance(observations, dict):
|
|
116
|
+
for name, value in observations.items():
|
|
117
|
+
events.append(
|
|
118
|
+
ObservationEvent(sequence=0, name=name, value=value)
|
|
119
|
+
)
|
|
120
|
+
return events
|
|
121
|
+
|
|
122
|
+
async def close(self) -> None:
|
|
123
|
+
self.closed = True
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class ToySubjectPlugin:
|
|
127
|
+
descriptor = PluginDescriptor(
|
|
128
|
+
id="toy",
|
|
129
|
+
version="1",
|
|
130
|
+
capabilities=[
|
|
131
|
+
"message.text",
|
|
132
|
+
"message.generated",
|
|
133
|
+
"wait",
|
|
134
|
+
"environment.action",
|
|
135
|
+
"extension",
|
|
136
|
+
"state.observe",
|
|
137
|
+
],
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
async def prepare(self, trial: TrialSpec) -> ToyTrialSession:
|
|
141
|
+
return ToyTrialSession(trial)
|
|
142
|
+
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Immutable filesystem run bundles."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from circumstances.models import RunSummary
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _canonical(value: Any) -> bytes:
|
|
15
|
+
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class FileArtifactStore:
|
|
19
|
+
"""Write a self-contained run bundle and seal it with file digests."""
|
|
20
|
+
|
|
21
|
+
def __init__(self, root: Path) -> None:
|
|
22
|
+
self.root = root.expanduser().resolve()
|
|
23
|
+
|
|
24
|
+
def persist(self, summary: RunSummary, *, pack_files: dict[str, bytes] | None = None) -> Path:
|
|
25
|
+
final_dir = self.root / summary.run_id
|
|
26
|
+
pending = self.root / f".{summary.run_id}.pending"
|
|
27
|
+
if final_dir.exists() or pending.exists():
|
|
28
|
+
raise FileExistsError(f"run bundle already exists: {summary.run_id}")
|
|
29
|
+
pending.mkdir(parents=True)
|
|
30
|
+
try:
|
|
31
|
+
self._write_json(pending / "run.json", summary.model_dump(mode="json"))
|
|
32
|
+
trials_dir = pending / "trials"
|
|
33
|
+
trials_dir.mkdir()
|
|
34
|
+
for result in summary.results:
|
|
35
|
+
self._write_json(
|
|
36
|
+
trials_dir / f"{_safe(result.trial_id)}.json",
|
|
37
|
+
result.model_dump(mode="json"),
|
|
38
|
+
)
|
|
39
|
+
if pack_files:
|
|
40
|
+
snapshot = pending / "pack"
|
|
41
|
+
for relative, content in sorted(pack_files.items()):
|
|
42
|
+
target = (snapshot / relative).resolve()
|
|
43
|
+
if not target.is_relative_to(snapshot.resolve()):
|
|
44
|
+
raise ValueError(f"pack snapshot path escapes root: {relative}")
|
|
45
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
target.write_bytes(content)
|
|
47
|
+
seal = {
|
|
48
|
+
"schema": "circumstances.run-bundle.v1",
|
|
49
|
+
"run_id": summary.run_id,
|
|
50
|
+
"files": _file_digests(pending),
|
|
51
|
+
}
|
|
52
|
+
self._write_json(pending / "seal.json", seal)
|
|
53
|
+
os.replace(pending, final_dir)
|
|
54
|
+
except BaseException:
|
|
55
|
+
_remove_tree(pending)
|
|
56
|
+
raise
|
|
57
|
+
return final_dir
|
|
58
|
+
|
|
59
|
+
@staticmethod
|
|
60
|
+
def _write_json(path: Path, value: Any) -> None:
|
|
61
|
+
path.write_bytes(_canonical(value) + b"\n")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def verify_bundle(path: Path) -> list[str]:
|
|
65
|
+
path = path.resolve()
|
|
66
|
+
seal_path = path / "seal.json"
|
|
67
|
+
seal = json.loads(seal_path.read_text())
|
|
68
|
+
failures: list[str] = []
|
|
69
|
+
expected = seal.get("files", {})
|
|
70
|
+
for relative, digest in expected.items():
|
|
71
|
+
target = (path / relative).resolve()
|
|
72
|
+
if not target.is_relative_to(path) or not target.is_file():
|
|
73
|
+
failures.append(f"missing or unsafe artifact: {relative}")
|
|
74
|
+
continue
|
|
75
|
+
actual = hashlib.sha256(target.read_bytes()).hexdigest()
|
|
76
|
+
if actual != digest:
|
|
77
|
+
failures.append(f"digest mismatch: {relative}")
|
|
78
|
+
extra = set(_file_digests(path)) - set(expected) - {"seal.json"}
|
|
79
|
+
failures.extend(f"unsealed artifact: {name}" for name in sorted(extra))
|
|
80
|
+
return failures
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _file_digests(root: Path) -> dict[str, str]:
|
|
84
|
+
result: dict[str, str] = {}
|
|
85
|
+
for path in sorted(root.rglob("*")):
|
|
86
|
+
if path.is_file() and path.name != "seal.json":
|
|
87
|
+
relative = path.relative_to(root).as_posix()
|
|
88
|
+
result[relative] = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
89
|
+
return result
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _safe(value: str) -> str:
|
|
93
|
+
return "".join(
|
|
94
|
+
character if character.isalnum() or character in "._-" else "_"
|
|
95
|
+
for character in value
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _remove_tree(path: Path) -> None:
|
|
100
|
+
if not path.exists():
|
|
101
|
+
return
|
|
102
|
+
for child in sorted(path.rglob("*"), reverse=True):
|
|
103
|
+
if child.is_file() or child.is_symlink():
|
|
104
|
+
child.unlink()
|
|
105
|
+
elif child.is_dir():
|
|
106
|
+
child.rmdir()
|
|
107
|
+
path.rmdir()
|
|
108
|
+
|