idemcore 0.1.2__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- idemcore-0.2.0/CHANGELOG.md +34 -0
- {idemcore-0.1.2/src/idemcore.egg-info → idemcore-0.2.0}/PKG-INFO +21 -1
- {idemcore-0.1.2 → idemcore-0.2.0}/README.md +20 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/pyproject.toml +3 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/src/idemcore/__init__.py +4 -0
- idemcore-0.2.0/src/idemcore/__main__.py +17 -0
- idemcore-0.2.0/src/idemcore/_version.py +1 -0
- idemcore-0.2.0/src/idemcore/trace.py +695 -0
- {idemcore-0.1.2 → idemcore-0.2.0/src/idemcore.egg-info}/PKG-INFO +21 -1
- {idemcore-0.1.2 → idemcore-0.2.0}/src/idemcore.egg-info/SOURCES.txt +5 -0
- idemcore-0.2.0/src/idemcore.egg-info/entry_points.txt +2 -0
- idemcore-0.2.0/tests/make_fixture.py +18 -0
- idemcore-0.2.0/tests/test_trace.py +261 -0
- idemcore-0.1.2/CHANGELOG.md +0 -19
- idemcore-0.1.2/src/idemcore/_version.py +0 -1
- {idemcore-0.1.2 → idemcore-0.2.0}/LICENSE +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/MANIFEST.in +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/setup.cfg +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/src/idemcore/_types.py +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/src/idemcore/client.py +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/src/idemcore/errors.py +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/src/idemcore/py.typed +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/src/idemcore/verify.py +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/src/idemcore.egg-info/dependency_links.txt +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/src/idemcore.egg-info/requires.txt +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/src/idemcore.egg-info/top_level.txt +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/tests/mock_api.py +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/tests/test_client.py +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/tests/test_verify.py +0 -0
- {idemcore-0.1.2 → idemcore-0.2.0}/tests/vectors.json +0 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.2.0 (2026-11-01)
|
|
4
|
+
|
|
5
|
+
- New `idemcore.trace`: reproducible agents. Record a whole agent run (model calls with
|
|
6
|
+
their signed receipts, tool calls, approvals and other notes) as a hash-chained trace
|
|
7
|
+
(`idemcore-trace/r1`), optionally signed with your own Ed25519 key.
|
|
8
|
+
- `verify_trace()` checks a trace offline: chain, step hashes, receipt signatures and
|
|
9
|
+
commitments, and trace signatures (pin signers with `trusted_signers`).
|
|
10
|
+
- `replay()` re-sends the recorded model requests and checks that every answer is
|
|
11
|
+
bit-identical (token ids, text, chain digest). With `agent=`, your agent code runs again
|
|
12
|
+
and recorded tool results are fed back, so tools are never executed twice; any change
|
|
13
|
+
in calls or arguments is reported as a divergence.
|
|
14
|
+
- CLI: `python -m idemcore trace verify|show|replay FILE` (also `idemcore-trace`).
|
|
15
|
+
- Traces are interchangeable with the JavaScript SDK (`@idemcore/sdk` 0.2.0).
|
|
16
|
+
- Docs: `docs/agent-replay.md`, example `examples/agent_replay.py`.
|
|
17
|
+
|
|
18
|
+
## 0.1.2 (2026-11-01)
|
|
19
|
+
|
|
20
|
+
- Source repository is now public. Package metadata lists Source, Issues, Documentation
|
|
21
|
+
and Changelog links.
|
|
22
|
+
- Added SECURITY.md and CONTRIBUTING.md.
|
|
23
|
+
- No code changes.
|
|
24
|
+
|
|
25
|
+
## 0.1.1 (2026-11-01)
|
|
26
|
+
|
|
27
|
+
- README: the quickstart, streaming and async examples now pass `model` explicitly
|
|
28
|
+
(read from `client.models()`), since the API requires a model id.
|
|
29
|
+
- Release published from GitHub Actions with PyPI Trusted Publishing.
|
|
30
|
+
|
|
31
|
+
## 0.1.0 (2026-11-01)
|
|
32
|
+
|
|
33
|
+
- First release: sync and async clients, streaming, signed receipts and local
|
|
34
|
+
receipt verification.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: idemcore
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Python client for the Idemcore API: chat completions with signed receipts, and local receipt verification.
|
|
5
5
|
Author-email: Don Wick <don.wick@idemcore.net>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -143,6 +143,26 @@ The production receipt-signing public key is bundled
|
|
|
143
143
|
(`idemcore.verify.DEFAULT_TRUSTED_KEYS`); pass `keys=client.keys()` to use the live
|
|
144
144
|
key set instead.
|
|
145
145
|
|
|
146
|
+
## Reproducible agents (record, replay, verify)
|
|
147
|
+
|
|
148
|
+
Record a whole agent run (model calls with their receipts, tool calls, approvals) as a hash-chained trace,
|
|
149
|
+
verify it offline, and replay it with a bit-identical check. Recorded tool results are fed back, so tools never run twice.
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
from idemcore import trace
|
|
153
|
+
s = trace.Session(client)
|
|
154
|
+
city = s.chat("Which city is the Eiffel Tower in? One word.", max_tokens=8).text
|
|
155
|
+
w = s.tool("get_weather", {"city": city}, get_weather)
|
|
156
|
+
s.save("run.trace.json")
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
python -m idemcore trace verify run.trace.json
|
|
161
|
+
python -m idemcore trace replay run.trace.json
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
See [docs/agent-replay.md](docs/agent-replay.md) ([Tiếng Việt](docs/agent-replay.vi.md)) and `examples/agent_replay.py`.
|
|
165
|
+
|
|
146
166
|
## Errors
|
|
147
167
|
|
|
148
168
|
All errors derive from `idemcore.IdemcoreError` and carry `status`, `type` and
|
|
@@ -111,6 +111,26 @@ The production receipt-signing public key is bundled
|
|
|
111
111
|
(`idemcore.verify.DEFAULT_TRUSTED_KEYS`); pass `keys=client.keys()` to use the live
|
|
112
112
|
key set instead.
|
|
113
113
|
|
|
114
|
+
## Reproducible agents (record, replay, verify)
|
|
115
|
+
|
|
116
|
+
Record a whole agent run (model calls with their receipts, tool calls, approvals) as a hash-chained trace,
|
|
117
|
+
verify it offline, and replay it with a bit-identical check. Recorded tool results are fed back, so tools never run twice.
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from idemcore import trace
|
|
121
|
+
s = trace.Session(client)
|
|
122
|
+
city = s.chat("Which city is the Eiffel Tower in? One word.", max_tokens=8).text
|
|
123
|
+
w = s.tool("get_weather", {"city": city}, get_weather)
|
|
124
|
+
s.save("run.trace.json")
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
python -m idemcore trace verify run.trace.json
|
|
129
|
+
python -m idemcore trace replay run.trace.json
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
See [docs/agent-replay.md](docs/agent-replay.md) ([Tiếng Việt](docs/agent-replay.vi.md)) and `examples/agent_replay.py`.
|
|
133
|
+
|
|
114
134
|
## Errors
|
|
115
135
|
|
|
116
136
|
All errors derive from `idemcore.IdemcoreError` and carry `status`, `type` and
|
|
@@ -7,6 +7,8 @@ from .errors import (APIConnectionError, AuthenticationError, ConflictError, Ide
|
|
|
7
7
|
InternalServerError, InvalidRequestError, NotFoundError, PermissionDeniedError,
|
|
8
8
|
RateLimitError, ServiceUnavailableError)
|
|
9
9
|
from .verify import DEFAULT_TRUSTED_KEYS, VerificationResult, verify_receipt
|
|
10
|
+
from .trace import (ReplayDivergence, ReplayReport, Session, TraceVerification, load_trace, record, replay,
|
|
11
|
+
save_trace, verify_trace)
|
|
10
12
|
|
|
11
13
|
__all__ = [
|
|
12
14
|
"__version__", "Idemcore", "AsyncIdemcore", "ChatStream", "AsyncChatStream", "DEFAULT_BASE_URL",
|
|
@@ -15,4 +17,6 @@ __all__ = [
|
|
|
15
17
|
"PermissionDeniedError", "NotFoundError", "ConflictError", "RateLimitError", "InsufficientQuotaError",
|
|
16
18
|
"ServiceUnavailableError", "InternalServerError",
|
|
17
19
|
"verify_receipt", "VerificationResult", "DEFAULT_TRUSTED_KEYS",
|
|
20
|
+
"Session", "record", "replay", "verify_trace", "load_trace", "save_trace", "ReplayReport", "ReplayDivergence",
|
|
21
|
+
"TraceVerification",
|
|
18
22
|
]
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""``python -m idemcore trace verify|show|replay FILE``."""
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
from .trace import _cli
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def main() -> int:
|
|
9
|
+
args = sys.argv[1:]
|
|
10
|
+
if not args or args[0] != "trace":
|
|
11
|
+
print("usage: python -m idemcore trace {verify,show,replay} FILE", file=sys.stderr)
|
|
12
|
+
return 2
|
|
13
|
+
return _cli(args[1:])
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
if __name__ == "__main__":
|
|
17
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.0"
|
|
@@ -0,0 +1,695 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Reproducible agent sessions: record, replay and verify (trace format ``idemcore-trace/r1``).
|
|
3
|
+
|
|
4
|
+
A *trace* is an ordered, hash-chained list of steps:
|
|
5
|
+
|
|
6
|
+
* ``model``: one chat completion: the exact request, the signed receipt, the salt,
|
|
7
|
+
the output token ids and the text.
|
|
8
|
+
* ``tool``: one tool call made by your code: name, arguments and result, each kept
|
|
9
|
+
as canonical JSON text with its hash.
|
|
10
|
+
* ``note``: free-form annotation (for example a human approval).
|
|
11
|
+
|
|
12
|
+
Every step commits to the previous step's hash, and the manifest commits to the
|
|
13
|
+
session id, the step count and the last hash (``head``). A trace can be signed
|
|
14
|
+
with an Ed25519 key you own. Three operations:
|
|
15
|
+
|
|
16
|
+
Command line: ``python -m idemcore trace verify|show|replay FILE`` (or ``idemcore-trace``).
|
|
17
|
+
|
|
18
|
+
* :func:`verify_trace`: offline. Recomputes the chain and the manifest hash, checks
|
|
19
|
+
each receipt signature and its commitments (prompt, output tokens, text) and each
|
|
20
|
+
tool hash. No network.
|
|
21
|
+
* :func:`replay`: re-runs the model calls against the API and checks that the output
|
|
22
|
+
tokens are bit-identical. With ``agent=`` it re-runs *your agent code* and feeds
|
|
23
|
+
it the recorded tool results instead of calling the tools again.
|
|
24
|
+
* :class:`Session`: records a trace while your agent runs.
|
|
25
|
+
|
|
26
|
+
Treat a trace like the conversation itself: it holds the prompts, the answers and the
|
|
27
|
+
salts that open the receipt commitments.
|
|
28
|
+
"""
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import base64
|
|
32
|
+
import datetime as _dt
|
|
33
|
+
import hashlib
|
|
34
|
+
import json
|
|
35
|
+
import os
|
|
36
|
+
import secrets
|
|
37
|
+
from dataclasses import dataclass, field
|
|
38
|
+
from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Sequence, Union
|
|
39
|
+
|
|
40
|
+
from ._types import ChatResult
|
|
41
|
+
from ._version import __version__
|
|
42
|
+
from . import verify as _verify
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
"TRACE_FORMAT", "Session", "ReplaySession", "ReplayDivergence", "StepReport", "ReplayReport",
|
|
46
|
+
"TraceVerification", "record", "replay", "verify_trace", "load_trace", "save_trace", "dumps_trace",
|
|
47
|
+
"trace_json", "generate_signing_key",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
TRACE_FORMAT = "idemcore-trace/r1"
|
|
51
|
+
SIG_DOMAIN = b"idemcore-trace/r1\n"
|
|
52
|
+
STEP_TYPES = ("model", "tool", "note")
|
|
53
|
+
# Receipt fields that must match between a recorded answer and its replay.
|
|
54
|
+
_SAME = ("model_id", "weights_hash", "decoding", "seed", "prompt_tokens", "output_tokens",
|
|
55
|
+
"chain_digest", "chain_fnv", "finish_reason")
|
|
56
|
+
# Fields that may legitimately differ (new engine build); reported as notes.
|
|
57
|
+
_ENV = ("engine_build", "kernel_version")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# ------------------------------------------------------------------ hashing
|
|
61
|
+
def trace_json(value: Any, *, sort_keys: bool = True) -> str:
|
|
62
|
+
"""JSON text for tool arguments, results and notes: no whitespace, non-ASCII
|
|
63
|
+
literal, finite floats allowed. Keys are sorted, except for tool results,
|
|
64
|
+
which keep the tool's key order so a replayed agent sees exactly the object
|
|
65
|
+
the tool returned. The text is stored as is and hashed as bytes, so
|
|
66
|
+
verification never re-serialises it."""
|
|
67
|
+
try:
|
|
68
|
+
return json.dumps(value, sort_keys=sort_keys, separators=(",", ":"), ensure_ascii=False, allow_nan=False)
|
|
69
|
+
except (TypeError, ValueError) as e:
|
|
70
|
+
raise TypeError(f"tool arguments and results must be JSON-serialisable: {e}") from None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _h(tag: str, *parts: bytes) -> str:
|
|
74
|
+
return _verify.tagged_hash(tag, *parts)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _u(s: str) -> bytes:
|
|
78
|
+
return s.encode("utf-8")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def genesis_hash(session_id: str) -> str:
|
|
82
|
+
return _h("trace-genesis", _u(session_id))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def step_hash(step: Mapping[str, Any]) -> str:
|
|
86
|
+
body = {k: v for k, v in step.items() if k != "hash"}
|
|
87
|
+
return _h("trace-step", _verify.canonical_json(body))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def manifest_hash(manifest: Mapping[str, Any]) -> str:
|
|
91
|
+
return _h("trace-manifest", _verify.canonical_json(dict(manifest)))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def request_hash(request: Mapping[str, Any]) -> str:
|
|
95
|
+
return _h("trace-request", _verify.canonical_json(dict(request)))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def text_sha(text: str) -> str:
|
|
99
|
+
return _h("trace-text", _u(text))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _now() -> str:
|
|
103
|
+
return _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _int_dict(d: Optional[Mapping[str, Any]]) -> Dict[str, int]:
|
|
107
|
+
return {k: v for k, v in (d or {}).items() if isinstance(v, int) and not isinstance(v, bool)}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _b64u(b: bytes) -> str:
|
|
111
|
+
return base64.urlsafe_b64encode(b).rstrip(b"=").decode()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# ------------------------------------------------------------------ signing
|
|
115
|
+
def generate_signing_key() -> Dict[str, str]:
|
|
116
|
+
"""A new Ed25519 key as a private JWK (``d`` is secret; keep it out of the trace)."""
|
|
117
|
+
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
|
118
|
+
from cryptography.hazmat.primitives import serialization as s
|
|
119
|
+
k = Ed25519PrivateKey.generate()
|
|
120
|
+
d = k.private_bytes(s.Encoding.Raw, s.PrivateFormat.Raw, s.NoEncryption())
|
|
121
|
+
x = k.public_key().public_bytes(s.Encoding.Raw, s.PublicFormat.Raw)
|
|
122
|
+
return {"kty": "OKP", "crv": "Ed25519", "x": _b64u(x), "d": _b64u(d),
|
|
123
|
+
"kid": "ed25519:" + hashlib.sha256(x).hexdigest()[:16]}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _sign(manifest_h: str, jwk: Mapping[str, str]) -> Dict[str, str]:
|
|
127
|
+
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
|
128
|
+
from cryptography.hazmat.primitives import serialization as s
|
|
129
|
+
k = Ed25519PrivateKey.from_private_bytes(_verify.b64u_decode(jwk["d"]))
|
|
130
|
+
x = k.public_key().public_bytes(s.Encoding.Raw, s.PublicFormat.Raw)
|
|
131
|
+
sig = k.sign(SIG_DOMAIN + _u(manifest_h))
|
|
132
|
+
return {"alg": "Ed25519", "kid": "ed25519:" + hashlib.sha256(x).hexdigest()[:16], "x": _b64u(x),
|
|
133
|
+
"signature": _b64u(sig)}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# ------------------------------------------------------------------ recording
|
|
137
|
+
class Session:
|
|
138
|
+
"""Record an agent session.
|
|
139
|
+
|
|
140
|
+
>>> s = Session(client, meta={"task": "weather"})
|
|
141
|
+
>>> r = s.chat([{"role": "user", "content": "..."}], model="qwen2.5-7b-instruct-q4")
|
|
142
|
+
>>> temp = s.tool("get_weather", {"city": "Paris"}, get_weather)
|
|
143
|
+
>>> trace = s.finish(sign_with=my_private_jwk) # dict, or s.save("run.trace.json")
|
|
144
|
+
|
|
145
|
+
Model calls go through :meth:`chat` (or :meth:`record_model` for a streamed
|
|
146
|
+
answer). Tool calls go through :meth:`tool` (runs the function) or
|
|
147
|
+
:meth:`record_tool` (you ran it yourself).
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
def __init__(self, client: Any = None, *, meta: Optional[Mapping[str, Any]] = None,
|
|
151
|
+
session_id: Optional[str] = None):
|
|
152
|
+
self.client = client
|
|
153
|
+
self.session_id = session_id or "ses_" + secrets.token_hex(12)
|
|
154
|
+
self.created_at = _now()
|
|
155
|
+
self.meta_json = trace_json(dict(meta or {}))
|
|
156
|
+
self.steps: List[Dict[str, Any]] = []
|
|
157
|
+
self._head = genesis_hash(self.session_id)
|
|
158
|
+
self._finished: Optional[Dict[str, Any]] = None
|
|
159
|
+
|
|
160
|
+
def __enter__(self) -> "Session":
|
|
161
|
+
return self
|
|
162
|
+
|
|
163
|
+
def __exit__(self, *exc: Any) -> None:
|
|
164
|
+
pass
|
|
165
|
+
|
|
166
|
+
@property
|
|
167
|
+
def head(self) -> str:
|
|
168
|
+
return self._head
|
|
169
|
+
|
|
170
|
+
def _append(self, typ: str, body: Dict[str, Any]) -> Dict[str, Any]:
|
|
171
|
+
if self._finished is not None:
|
|
172
|
+
raise RuntimeError("session already finished")
|
|
173
|
+
step = {"index": len(self.steps), "type": typ, "prev": self._head, "body": body}
|
|
174
|
+
step["hash"] = step_hash(step)
|
|
175
|
+
self.steps.append(step)
|
|
176
|
+
self._head = step["hash"]
|
|
177
|
+
return step
|
|
178
|
+
|
|
179
|
+
# -- model calls
|
|
180
|
+
def chat(self, messages: Any, *, model: Optional[str] = None, max_tokens: Optional[int] = None,
|
|
181
|
+
privacy: Optional[str] = None, **kw: Any) -> ChatResult:
|
|
182
|
+
if self.client is None:
|
|
183
|
+
raise RuntimeError("Session has no client; pass one or use record_model()")
|
|
184
|
+
r = self.client.chat(messages, model=model, max_tokens=max_tokens, privacy=privacy, **kw)
|
|
185
|
+
self.record_model(r, model=model, max_tokens=max_tokens, privacy=privacy)
|
|
186
|
+
return r
|
|
187
|
+
|
|
188
|
+
def record_model(self, result: ChatResult, *, model: Optional[str] = None, max_tokens: Optional[int] = None,
|
|
189
|
+
privacy: Optional[str] = None, streamed: bool = False) -> Dict[str, Any]:
|
|
190
|
+
"""Add a finished :class:`ChatResult` (for example ``stream.until_done()``).
|
|
191
|
+
``model``/``max_tokens``/``privacy`` must be what you sent."""
|
|
192
|
+
request = {"messages": [dict(m) for m in result.messages], "model": model,
|
|
193
|
+
"max_tokens": max_tokens, "privacy": privacy}
|
|
194
|
+
receipt = dict(result.receipt)
|
|
195
|
+
body = {
|
|
196
|
+
"request": request,
|
|
197
|
+
"request_hash": request_hash(request),
|
|
198
|
+
"receipt_id": result.receipt_id,
|
|
199
|
+
"receipt": receipt,
|
|
200
|
+
"receipt_hash": _h("trace-receipt", _verify.canonical_json(receipt)),
|
|
201
|
+
"salt": result.salt,
|
|
202
|
+
"output_token_ids": list(result.output_token_ids) if result.output_token_ids is not None else None,
|
|
203
|
+
"output_text": result.text,
|
|
204
|
+
"output_text_sha": text_sha(result.text),
|
|
205
|
+
"finish_reason": result.finish_reason,
|
|
206
|
+
"usage": _int_dict(result.usage),
|
|
207
|
+
"streamed": bool(streamed),
|
|
208
|
+
}
|
|
209
|
+
return self._append("model", body)
|
|
210
|
+
|
|
211
|
+
# -- tool calls
|
|
212
|
+
def tool(self, name: str, args: Any, fn: Callable[..., Any], *, call_id: Optional[str] = None) -> Any:
|
|
213
|
+
"""Run ``fn`` (``fn(**args)`` for a dict, else ``fn(args)``) and record the call.
|
|
214
|
+
An exception is recorded and re-raised."""
|
|
215
|
+
args_json = trace_json(args)
|
|
216
|
+
try:
|
|
217
|
+
result = fn(**args) if isinstance(args, dict) else fn(args)
|
|
218
|
+
except Exception as e:
|
|
219
|
+
self.record_tool(name, args, error=f"{type(e).__name__}: {e}", call_id=call_id, _args_json=args_json)
|
|
220
|
+
raise
|
|
221
|
+
self.record_tool(name, args, result, call_id=call_id, _args_json=args_json)
|
|
222
|
+
return result
|
|
223
|
+
|
|
224
|
+
def record_tool(self, name: str, args: Any, result: Any = None, *, error: Optional[str] = None,
|
|
225
|
+
call_id: Optional[str] = None, _args_json: Optional[str] = None) -> Dict[str, Any]:
|
|
226
|
+
"""Record a tool call you executed yourself."""
|
|
227
|
+
if not isinstance(name, str) or not name:
|
|
228
|
+
raise ValueError("tool name must be a non-empty string")
|
|
229
|
+
args_json = _args_json if _args_json is not None else trace_json(args)
|
|
230
|
+
result_json = None if error is not None else trace_json(result, sort_keys=False)
|
|
231
|
+
body = {
|
|
232
|
+
"name": name,
|
|
233
|
+
"call_id": call_id or f"call_{len(self.steps)}",
|
|
234
|
+
"args_json": args_json,
|
|
235
|
+
"args_hash": _h("trace-tool-args", _u(args_json)),
|
|
236
|
+
"result_json": result_json,
|
|
237
|
+
"result_hash": _h("trace-tool-result", _u(result_json if result_json is not None else "")),
|
|
238
|
+
"error": error,
|
|
239
|
+
}
|
|
240
|
+
return self._append("tool", body)
|
|
241
|
+
|
|
242
|
+
def note(self, kind: str, data: Any = None) -> Dict[str, Any]:
|
|
243
|
+
"""Free-form step, e.g. ``note("approval", {"by": "ops", "decision": "allow"})``."""
|
|
244
|
+
data_json = trace_json(data)
|
|
245
|
+
return self._append("note", {"kind": str(kind), "data_json": data_json,
|
|
246
|
+
"data_hash": _h("trace-note", _u(data_json))})
|
|
247
|
+
|
|
248
|
+
# -- finish
|
|
249
|
+
def manifest(self) -> Dict[str, Any]:
|
|
250
|
+
return {"format": TRACE_FORMAT, "session_id": self.session_id, "created_at": self.created_at,
|
|
251
|
+
"sdk": f"idemcore-python/{__version__}", "meta_json": self.meta_json,
|
|
252
|
+
"genesis": genesis_hash(self.session_id), "head": self._head, "step_count": len(self.steps)}
|
|
253
|
+
|
|
254
|
+
def finish(self, *, sign_with: Optional[Mapping[str, str]] = None) -> Dict[str, Any]:
|
|
255
|
+
"""Close the session and return the trace (a JSON-ready dict)."""
|
|
256
|
+
if self._finished is None:
|
|
257
|
+
man = self.manifest()
|
|
258
|
+
mh = manifest_hash(man)
|
|
259
|
+
self._finished = {"manifest": man, "manifest_hash": mh, "steps": self.steps, "signatures": []}
|
|
260
|
+
if sign_with is not None:
|
|
261
|
+
self._finished["signatures"].append(_sign(self._finished["manifest_hash"], sign_with))
|
|
262
|
+
return self._finished
|
|
263
|
+
|
|
264
|
+
def save(self, path: str, **kw: Any) -> str:
|
|
265
|
+
save_trace(self.finish(**kw), path)
|
|
266
|
+
return path
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def record(client: Any, **kw: Any) -> Session:
|
|
270
|
+
"""Start recording: ``with record(client) as s: ...``."""
|
|
271
|
+
return Session(client, **kw)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def dumps_trace(trace: Mapping[str, Any]) -> str:
|
|
275
|
+
return json.dumps(trace, ensure_ascii=False, indent=1) + "\n"
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def save_trace(trace: Mapping[str, Any], path: str) -> None:
|
|
279
|
+
tmp = path + ".tmp"
|
|
280
|
+
with open(tmp, "w", encoding="utf-8") as f:
|
|
281
|
+
f.write(dumps_trace(trace))
|
|
282
|
+
os.replace(tmp, path)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def load_trace(path_or_text: str) -> Dict[str, Any]:
|
|
286
|
+
if path_or_text.lstrip().startswith("{"):
|
|
287
|
+
return json.loads(path_or_text)
|
|
288
|
+
with open(path_or_text, encoding="utf-8") as f:
|
|
289
|
+
return json.load(f)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
# ------------------------------------------------------------------ verification
|
|
293
|
+
@dataclass
|
|
294
|
+
class TraceVerification:
|
|
295
|
+
ok: bool
|
|
296
|
+
errors: List[str] = field(default_factory=list)
|
|
297
|
+
warnings: List[str] = field(default_factory=list)
|
|
298
|
+
steps: int = 0
|
|
299
|
+
model_steps: int = 0
|
|
300
|
+
tool_steps: int = 0
|
|
301
|
+
signers: List[str] = field(default_factory=list)
|
|
302
|
+
|
|
303
|
+
def __bool__(self) -> bool:
|
|
304
|
+
return self.ok
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def verify_trace(trace: Mapping[str, Any], *, keys: _verify.KeysLike = None, check_receipts: bool = True,
|
|
308
|
+
trusted_signers: Optional[Sequence[Union[str, Mapping[str, str]]]] = None,
|
|
309
|
+
require_signature: bool = False) -> TraceVerification:
|
|
310
|
+
"""Verify a trace offline (no network).
|
|
311
|
+
|
|
312
|
+
Checks the step chain, step hashes, the manifest hash, each tool/note hash, each
|
|
313
|
+
model step's request hash and text hash, and (``check_receipts``) each receipt's
|
|
314
|
+
Ed25519 signature against ``keys`` (default: bundled Idemcore keys) plus its
|
|
315
|
+
prompt / output-token / text commitments. ``trusted_signers`` is a list of key
|
|
316
|
+
ids (``ed25519:...``) or public JWKs; with it, at least one valid signature from
|
|
317
|
+
that list is required."""
|
|
318
|
+
errs: List[str] = []
|
|
319
|
+
warns: List[str] = []
|
|
320
|
+
out = TraceVerification(False, errs, warns)
|
|
321
|
+
try:
|
|
322
|
+
man = trace["manifest"]
|
|
323
|
+
steps = trace["steps"]
|
|
324
|
+
except (KeyError, TypeError):
|
|
325
|
+
errs.append("not a trace: manifest and steps are required")
|
|
326
|
+
return out
|
|
327
|
+
if man.get("format") != TRACE_FORMAT:
|
|
328
|
+
errs.append(f"unsupported format {man.get('format')!r}")
|
|
329
|
+
return out
|
|
330
|
+
try:
|
|
331
|
+
mh = manifest_hash(man)
|
|
332
|
+
except ValueError as e:
|
|
333
|
+
errs.append(f"manifest is not canonical: {e}")
|
|
334
|
+
return out
|
|
335
|
+
if mh != trace.get("manifest_hash"):
|
|
336
|
+
errs.append("manifest_hash mismatch")
|
|
337
|
+
sid = man.get("session_id")
|
|
338
|
+
if not isinstance(sid, str) or genesis_hash(sid) != man.get("genesis"):
|
|
339
|
+
errs.append("genesis does not match session_id")
|
|
340
|
+
if man.get("step_count") != len(steps):
|
|
341
|
+
errs.append(f"step_count {man.get('step_count')} != {len(steps)} steps")
|
|
342
|
+
prev = genesis_hash(sid) if isinstance(sid, str) else ""
|
|
343
|
+
trusted_keys = _verify.load_keys(keys) if check_receipts else []
|
|
344
|
+
for i, st in enumerate(steps):
|
|
345
|
+
p = f"step {i}"
|
|
346
|
+
if not isinstance(st, Mapping):
|
|
347
|
+
errs.append(f"{p}: not an object")
|
|
348
|
+
continue
|
|
349
|
+
if st.get("index") != i:
|
|
350
|
+
errs.append(f"{p}: index {st.get('index')}")
|
|
351
|
+
if st.get("prev") != prev:
|
|
352
|
+
errs.append(f"{p}: prev does not link to the previous step")
|
|
353
|
+
try:
|
|
354
|
+
h = step_hash(st)
|
|
355
|
+
except ValueError as e:
|
|
356
|
+
errs.append(f"{p}: not canonical ({e})")
|
|
357
|
+
h = None
|
|
358
|
+
if h != st.get("hash"):
|
|
359
|
+
errs.append(f"{p}: hash mismatch")
|
|
360
|
+
prev = st.get("hash")
|
|
361
|
+
typ, b = st.get("type"), st.get("body") or {}
|
|
362
|
+
if typ == "model":
|
|
363
|
+
out.model_steps += 1
|
|
364
|
+
errs.extend(f"{p}: {e}" for e in _verify_model_step(b, check_receipts, trusted_keys))
|
|
365
|
+
elif typ == "tool":
|
|
366
|
+
out.tool_steps += 1
|
|
367
|
+
if b.get("args_hash") != _h("trace-tool-args", _u(str(b.get("args_json")))):
|
|
368
|
+
errs.append(f"{p}: args_hash mismatch")
|
|
369
|
+
rj = b.get("result_json")
|
|
370
|
+
if b.get("result_hash") != _h("trace-tool-result", _u(rj if isinstance(rj, str) else "")):
|
|
371
|
+
errs.append(f"{p}: result_hash mismatch")
|
|
372
|
+
for k in ("args_json", "result_json"):
|
|
373
|
+
if isinstance(b.get(k), str):
|
|
374
|
+
try:
|
|
375
|
+
json.loads(b[k])
|
|
376
|
+
except ValueError:
|
|
377
|
+
errs.append(f"{p}: {k} is not JSON")
|
|
378
|
+
elif typ == "note":
|
|
379
|
+
if b.get("data_hash") != _h("trace-note", _u(str(b.get("data_json")))):
|
|
380
|
+
errs.append(f"{p}: data_hash mismatch")
|
|
381
|
+
else:
|
|
382
|
+
errs.append(f"{p}: unknown step type {typ!r}")
|
|
383
|
+
if prev != man.get("head") and steps:
|
|
384
|
+
errs.append("head does not match the last step")
|
|
385
|
+
if not steps and man.get("head") != man.get("genesis"):
|
|
386
|
+
errs.append("empty trace: head must equal genesis")
|
|
387
|
+
out.steps = len(steps)
|
|
388
|
+
# signatures
|
|
389
|
+
valid: List[str] = []
|
|
390
|
+
for sg in trace.get("signatures") or []:
|
|
391
|
+
try:
|
|
392
|
+
raw = _verify.b64u_decode(sg["x"])
|
|
393
|
+
kid = "ed25519:" + hashlib.sha256(raw).hexdigest()[:16]
|
|
394
|
+
ok = sg.get("kid") == kid and _verify.TrustedKey(kid, raw).verify(
|
|
395
|
+
_verify.b64u_decode(sg["signature"]), SIG_DOMAIN + _u(str(trace.get("manifest_hash"))))
|
|
396
|
+
except Exception:
|
|
397
|
+
ok, kid = False, str(sg.get("kid"))
|
|
398
|
+
if ok:
|
|
399
|
+
valid.append(kid)
|
|
400
|
+
else:
|
|
401
|
+
errs.append(f"signature {kid} invalid")
|
|
402
|
+
out.signers = valid
|
|
403
|
+
if trusted_signers is not None:
|
|
404
|
+
pinned = {s if isinstance(s, str) else "ed25519:" + hashlib.sha256(_verify.b64u_decode(s["x"])).hexdigest()[:16]
|
|
405
|
+
for s in trusted_signers}
|
|
406
|
+
if not pinned.intersection(valid):
|
|
407
|
+
errs.append("no valid signature from a trusted signer")
|
|
408
|
+
elif require_signature and not valid:
|
|
409
|
+
errs.append("trace is not signed")
|
|
410
|
+
elif valid:
|
|
411
|
+
warns.append("signature valid but signer not pinned (pass trusted_signers)")
|
|
412
|
+
out.ok = not errs
|
|
413
|
+
return out
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _verify_model_step(b: Mapping[str, Any], check_receipts: bool, keys: List[Any]) -> List[str]:
|
|
417
|
+
errs: List[str] = []
|
|
418
|
+
req = b.get("request") or {}
|
|
419
|
+
try:
|
|
420
|
+
if b.get("request_hash") != request_hash(req):
|
|
421
|
+
errs.append("request_hash mismatch")
|
|
422
|
+
except ValueError as e:
|
|
423
|
+
errs.append(f"request not canonical ({e})")
|
|
424
|
+
rc = b.get("receipt") or {}
|
|
425
|
+
if rc.get("receipt_id") != b.get("receipt_id"):
|
|
426
|
+
errs.append("receipt_id mismatch")
|
|
427
|
+
try:
|
|
428
|
+
if b.get("receipt_hash") != _h("trace-receipt", _verify.canonical_json(dict(rc))):
|
|
429
|
+
errs.append("receipt_hash mismatch")
|
|
430
|
+
except ValueError as e:
|
|
431
|
+
errs.append(f"receipt not canonical ({e})")
|
|
432
|
+
text = b.get("output_text")
|
|
433
|
+
if not isinstance(text, str) or b.get("output_text_sha") != text_sha(text):
|
|
434
|
+
errs.append("output_text_sha mismatch")
|
|
435
|
+
if check_receipts:
|
|
436
|
+
ids = b.get("output_token_ids")
|
|
437
|
+
res = _verify.verify_receipt(
|
|
438
|
+
rc, keys=keys, messages=req.get("messages"), salt=b.get("salt"),
|
|
439
|
+
output_token_ids=ids, output_text=text if "output_text_hash" in rc else None)
|
|
440
|
+
for lv in res.levels.values():
|
|
441
|
+
errs.extend(f"receipt {lv.level}: {e}" for e in lv.errors)
|
|
442
|
+
if ids is None and rc.get("privacy") != "full":
|
|
443
|
+
pass # token ids not disclosed: commitment checked by replay only
|
|
444
|
+
return errs
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
# ------------------------------------------------------------------ replay
|
|
448
|
+
@dataclass
|
|
449
|
+
class StepReport:
|
|
450
|
+
index: int
|
|
451
|
+
type: str
|
|
452
|
+
identical: bool
|
|
453
|
+
diffs: List[str] = field(default_factory=list)
|
|
454
|
+
notes: List[str] = field(default_factory=list)
|
|
455
|
+
receipt_id: Optional[str] = None
|
|
456
|
+
replay_receipt_id: Optional[str] = None
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
@dataclass
|
|
460
|
+
class ReplayReport:
|
|
461
|
+
ok: bool
|
|
462
|
+
steps: List[StepReport] = field(default_factory=list)
|
|
463
|
+
errors: List[str] = field(default_factory=list)
|
|
464
|
+
model_calls: int = 0
|
|
465
|
+
replay_trace: Optional[Dict[str, Any]] = None
|
|
466
|
+
|
|
467
|
+
def __bool__(self) -> bool:
|
|
468
|
+
return self.ok
|
|
469
|
+
|
|
470
|
+
def summary(self) -> str:
|
|
471
|
+
n = sum(1 for s in self.steps if s.type == "model")
|
|
472
|
+
same = sum(1 for s in self.steps if s.type == "model" and s.identical)
|
|
473
|
+
return (f"{'OK' if self.ok else 'DIVERGED'}: {same}/{n} model steps bit-identical, "
|
|
474
|
+
f"{len(self.steps) - n} tool/note steps fed from the trace"
|
|
475
|
+
+ (f"; errors: {'; '.join(self.errors)}" if self.errors else ""))
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
class ReplayDivergence(Exception):
|
|
479
|
+
def __init__(self, report: ReplayReport, message: str):
|
|
480
|
+
super().__init__(message)
|
|
481
|
+
self.report = report
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def compare_model(rec: Mapping[str, Any], new: ChatResult) -> StepReport:
|
|
485
|
+
"""Compare a recorded model step body with a fresh :class:`ChatResult`."""
|
|
486
|
+
a, b = rec.get("receipt") or {}, new.receipt or {}
|
|
487
|
+
rep = StepReport(-1, "model", True, receipt_id=rec.get("receipt_id"), replay_receipt_id=new.receipt_id)
|
|
488
|
+
ids_a, ids_b = rec.get("output_token_ids"), new.output_token_ids
|
|
489
|
+
if ids_a is not None and ids_b is not None:
|
|
490
|
+
if list(ids_a) != list(ids_b):
|
|
491
|
+
n = next((i for i, (x, y) in enumerate(zip(ids_a, ids_b)) if x != y), min(len(ids_a), len(ids_b)))
|
|
492
|
+
rep.diffs.append(f"output tokens differ at position {n} ({len(ids_a)} vs {len(ids_b)} tokens)")
|
|
493
|
+
else:
|
|
494
|
+
rep.notes.append("token ids not disclosed on one side; compared chain digest and text")
|
|
495
|
+
if rec.get("output_text") != new.text:
|
|
496
|
+
rep.diffs.append("output text differs")
|
|
497
|
+
for k in _SAME:
|
|
498
|
+
if a.get(k) != b.get(k):
|
|
499
|
+
rep.diffs.append(f"receipt {k} differs")
|
|
500
|
+
if a.get("privacy") == b.get("privacy") == "full" and a.get("output_token_hash") != b.get("output_token_hash"):
|
|
501
|
+
rep.diffs.append("receipt output_token_hash differs")
|
|
502
|
+
for k in _ENV:
|
|
503
|
+
if a.get(k) != b.get(k):
|
|
504
|
+
rep.notes.append(f"{k} changed: {a.get(k)} -> {b.get(k)}")
|
|
505
|
+
rep.identical = not rep.diffs
|
|
506
|
+
return rep
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
class ReplaySession(Session):
|
|
510
|
+
"""What your agent function receives during :func:`replay`. ``chat`` calls the
|
|
511
|
+
API and checks the answer against the trace; ``tool`` returns the recorded
|
|
512
|
+
result without running the tool."""
|
|
513
|
+
|
|
514
|
+
def __init__(self, client: Any, trace: Mapping[str, Any], report: ReplayReport, strict: bool):
|
|
515
|
+
super().__init__(client, meta={"replay_of": trace["manifest"]["session_id"],
|
|
516
|
+
"replay_of_manifest_hash": trace["manifest_hash"]})
|
|
517
|
+
self._trace_steps = list(trace["steps"])
|
|
518
|
+
self._pos = 0
|
|
519
|
+
self._report = report
|
|
520
|
+
self._strict = strict
|
|
521
|
+
|
|
522
|
+
def _next(self, typ: str) -> Mapping[str, Any]:
|
|
523
|
+
if self._pos >= len(self._trace_steps):
|
|
524
|
+
self._fail(f"agent made an extra {typ} call after the recorded session ended")
|
|
525
|
+
st = self._trace_steps[self._pos]
|
|
526
|
+
if st["type"] != typ:
|
|
527
|
+
self._fail(f"step {self._pos}: agent made a {typ} call, trace has a {st['type']} step")
|
|
528
|
+
self._pos += 1
|
|
529
|
+
return st
|
|
530
|
+
|
|
531
|
+
def _fail(self, msg: str) -> None:
|
|
532
|
+
self._report.errors.append(msg)
|
|
533
|
+
self._report.ok = False
|
|
534
|
+
raise ReplayDivergence(self._report, msg)
|
|
535
|
+
|
|
536
|
+
def chat(self, messages: Any, *, model: Optional[str] = None, max_tokens: Optional[int] = None,
|
|
537
|
+
privacy: Optional[str] = None, **kw: Any) -> ChatResult:
|
|
538
|
+
st = self._next("model")
|
|
539
|
+
from .client import _messages
|
|
540
|
+
req = {"messages": [dict(m) for m in _messages(messages)], "model": model,
|
|
541
|
+
"max_tokens": max_tokens, "privacy": privacy}
|
|
542
|
+
if request_hash(req) != st["body"]["request_hash"]:
|
|
543
|
+
self._fail(f"step {st['index']}: the agent sent a different request than the trace")
|
|
544
|
+
r = self.client.chat(messages, model=model, max_tokens=max_tokens, privacy=privacy, **kw)
|
|
545
|
+
self.record_model(r, model=model, max_tokens=max_tokens, privacy=privacy)
|
|
546
|
+
rep = compare_model(st["body"], r)
|
|
547
|
+
rep.index = st["index"]
|
|
548
|
+
self._report.steps.append(rep)
|
|
549
|
+
self._report.model_calls += 1
|
|
550
|
+
if not rep.identical:
|
|
551
|
+
self._report.ok = False
|
|
552
|
+
if self._strict:
|
|
553
|
+
raise ReplayDivergence(self._report, f"step {st['index']}: " + "; ".join(rep.diffs))
|
|
554
|
+
return r
|
|
555
|
+
|
|
556
|
+
def tool(self, name: str, args: Any, fn: Optional[Callable[..., Any]] = None, *,
|
|
557
|
+
call_id: Optional[str] = None) -> Any:
|
|
558
|
+
st = self._next("tool")
|
|
559
|
+
b = st["body"]
|
|
560
|
+
aj = trace_json(args)
|
|
561
|
+
same = b["args_hash"] == _h("trace-tool-args", _u(aj))
|
|
562
|
+
if not same: # a trace from another SDK may format numbers differently (1e-07 vs 1e-7)
|
|
563
|
+
try:
|
|
564
|
+
same = json.loads(b["args_json"]) == json.loads(aj)
|
|
565
|
+
except ValueError:
|
|
566
|
+
same = False
|
|
567
|
+
if b["name"] != name or not same:
|
|
568
|
+
self._fail(f"step {st['index']}: tool call {name} differs from the trace ({b['name']})")
|
|
569
|
+
self._report.steps.append(StepReport(st["index"], "tool", True, notes=["result fed from trace"]))
|
|
570
|
+
if b.get("error") is not None:
|
|
571
|
+
self.record_tool(name, args, error=b["error"], call_id=b.get("call_id"), _args_json=aj)
|
|
572
|
+
raise RuntimeError(f"recorded tool error: {b['error']}")
|
|
573
|
+
result = json.loads(b["result_json"])
|
|
574
|
+
self.record_tool(name, args, result, call_id=b.get("call_id"), _args_json=aj)
|
|
575
|
+
return result
|
|
576
|
+
|
|
577
|
+
def record_tool(self, name: str, args: Any, result: Any = None, **kw: Any) -> Dict[str, Any]:
|
|
578
|
+
return Session.record_tool(self, name, args, result, **kw)
|
|
579
|
+
|
|
580
|
+
def note(self, kind: str, data: Any = None) -> Dict[str, Any]:
|
|
581
|
+
st = self._next("note")
|
|
582
|
+
self._report.steps.append(StepReport(st["index"], "note", True))
|
|
583
|
+
return super().note(kind, data)
|
|
584
|
+
|
|
585
|
+
def finish_check(self) -> None:
|
|
586
|
+
if self._pos != len(self._trace_steps):
|
|
587
|
+
self._report.errors.append(f"agent stopped after {self._pos} of {len(self._trace_steps)} steps")
|
|
588
|
+
self._report.ok = False
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def replay(trace: Mapping[str, Any], client: Any, *, agent: Optional[Callable[[ReplaySession], Any]] = None,
|
|
592
|
+
strict: bool = False, verify_first: bool = True, keys: _verify.KeysLike = None) -> ReplayReport:
|
|
593
|
+
"""Replay a recorded session against the API.
|
|
594
|
+
|
|
595
|
+
Without ``agent``: re-send every recorded model request in order and check the
|
|
596
|
+
answers are bit-identical (token ids, text, chain digest). Tool and note steps
|
|
597
|
+
are taken from the trace.
|
|
598
|
+
|
|
599
|
+
With ``agent``: call ``agent(session)`` with a :class:`ReplaySession`. Your code
|
|
600
|
+
runs again; model calls go to the API and are checked, tool calls return the
|
|
601
|
+
recorded results. Any change in the order or arguments of calls is a divergence.
|
|
602
|
+
|
|
603
|
+
``strict=True`` stops at the first divergence (raises :class:`ReplayDivergence`
|
|
604
|
+
for agent replays). The new run is recorded in ``report.replay_trace``."""
|
|
605
|
+
report = ReplayReport(True)
|
|
606
|
+
if verify_first:
|
|
607
|
+
v = verify_trace(trace, keys=keys)
|
|
608
|
+
if not v.ok:
|
|
609
|
+
report.ok = False
|
|
610
|
+
report.errors.append("trace failed verification: " + "; ".join(v.errors[:5]))
|
|
611
|
+
return report
|
|
612
|
+
rs = ReplaySession(client, trace, report, strict)
|
|
613
|
+
if agent is not None:
|
|
614
|
+
try:
|
|
615
|
+
agent(rs)
|
|
616
|
+
except ReplayDivergence:
|
|
617
|
+
pass
|
|
618
|
+
else:
|
|
619
|
+
rs.finish_check()
|
|
620
|
+
else:
|
|
621
|
+
for st in trace["steps"]:
|
|
622
|
+
b = st["body"]
|
|
623
|
+
if st["type"] == "model":
|
|
624
|
+
q = b["request"]
|
|
625
|
+
try:
|
|
626
|
+
rs.chat(q["messages"], model=q.get("model"), max_tokens=q.get("max_tokens"), privacy=q.get("privacy"))
|
|
627
|
+
except ReplayDivergence:
|
|
628
|
+
break
|
|
629
|
+
elif st["type"] == "tool":
|
|
630
|
+
try:
|
|
631
|
+
rs.tool(b["name"], json.loads(b["args_json"]))
|
|
632
|
+
except RuntimeError:
|
|
633
|
+
pass
|
|
634
|
+
else:
|
|
635
|
+
rs.note(b["kind"], json.loads(b["data_json"]))
|
|
636
|
+
report.replay_trace = rs.finish()
|
|
637
|
+
return report
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
# ------------------------------------------------------------------ CLI
|
|
641
|
+
def _cli(argv: Optional[Sequence[str]] = None) -> int:
|
|
642
|
+
import argparse
|
|
643
|
+
ap = argparse.ArgumentParser(prog="idemcore-trace",
|
|
644
|
+
description="Verify, inspect or replay an Idemcore agent trace.")
|
|
645
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
646
|
+
v = sub.add_parser("verify", help="offline check (no network)")
|
|
647
|
+
v.add_argument("trace")
|
|
648
|
+
v.add_argument("--signer", action="append", help="trusted signer key id (ed25519:...)")
|
|
649
|
+
v.add_argument("--no-receipts", action="store_true", help="skip receipt signature checks")
|
|
650
|
+
v.add_argument("--keys", help="JWKS file of trusted receipt keys (default: bundled Idemcore keys)")
|
|
651
|
+
s = sub.add_parser("show", help="print the steps")
|
|
652
|
+
s.add_argument("trace")
|
|
653
|
+
r = sub.add_parser("replay", help="re-run the model calls (needs IDEMCORE_API_KEY)")
|
|
654
|
+
r.add_argument("trace")
|
|
655
|
+
r.add_argument("--out", help="write the replay's own trace here")
|
|
656
|
+
a = ap.parse_args(argv)
|
|
657
|
+
t = load_trace(a.trace)
|
|
658
|
+
if a.cmd == "verify":
|
|
659
|
+
keys = None
|
|
660
|
+
if a.keys:
|
|
661
|
+
with open(a.keys, encoding="utf-8") as f:
|
|
662
|
+
keys = json.load(f)
|
|
663
|
+
res = verify_trace(t, keys=keys, check_receipts=not a.no_receipts, trusted_signers=a.signer)
|
|
664
|
+
print(("OK" if res.ok else "FAILED") + f": {res.steps} steps ({res.model_steps} model, {res.tool_steps} tool)"
|
|
665
|
+
+ (f", signed by {', '.join(res.signers)}" if res.signers else ""))
|
|
666
|
+
for e in res.errors:
|
|
667
|
+
print(" error:", e)
|
|
668
|
+
for w in res.warnings:
|
|
669
|
+
print(" note:", w)
|
|
670
|
+
return 0 if res.ok else 1
|
|
671
|
+
if a.cmd == "show":
|
|
672
|
+
m = t["manifest"]
|
|
673
|
+
print(f"{m['session_id']} {m['created_at']} {m['step_count']} steps head {m['head'][:16]}")
|
|
674
|
+
for st in t["steps"]:
|
|
675
|
+
b = st["body"]
|
|
676
|
+
if st["type"] == "model":
|
|
677
|
+
txt = b["output_text"].replace("\n", " ")
|
|
678
|
+
print(f" #{st['index']} model {b['receipt_id']} {b['receipt'].get('output_tokens')} tok: {txt[:70]}")
|
|
679
|
+
elif st["type"] == "tool":
|
|
680
|
+
print(f" #{st['index']} tool {b['name']}({b['args_json'][:50]}) -> {(b['result_json'] or b['error'] or '')[:50]}")
|
|
681
|
+
else:
|
|
682
|
+
print(f" #{st['index']} note {b['kind']} {b['data_json'][:60]}")
|
|
683
|
+
return 0
|
|
684
|
+
from .client import Idemcore
|
|
685
|
+
rep = replay(t, Idemcore())
|
|
686
|
+
print(rep.summary())
|
|
687
|
+
for s_ in rep.steps:
|
|
688
|
+
if s_.type == "model":
|
|
689
|
+
print(f" #{s_.index} {'identical' if s_.identical else 'DIFFERENT'} {s_.receipt_id} -> {s_.replay_receipt_id}"
|
|
690
|
+
+ "".join(f"\n {d}" for d in s_.diffs + s_.notes))
|
|
691
|
+
if a.out and rep.replay_trace:
|
|
692
|
+
save_trace(rep.replay_trace, a.out)
|
|
693
|
+
return 0 if rep.ok else 1
|
|
694
|
+
|
|
695
|
+
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: idemcore
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Python client for the Idemcore API: chat completions with signed receipts, and local receipt verification.
|
|
5
5
|
Author-email: Don Wick <don.wick@idemcore.net>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -143,6 +143,26 @@ The production receipt-signing public key is bundled
|
|
|
143
143
|
(`idemcore.verify.DEFAULT_TRUSTED_KEYS`); pass `keys=client.keys()` to use the live
|
|
144
144
|
key set instead.
|
|
145
145
|
|
|
146
|
+
## Reproducible agents (record, replay, verify)
|
|
147
|
+
|
|
148
|
+
Record a whole agent run (model calls with their receipts, tool calls, approvals) as a hash-chained trace,
|
|
149
|
+
verify it offline, and replay it with a bit-identical check. Recorded tool results are fed back, so tools never run twice.
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
from idemcore import trace
|
|
153
|
+
s = trace.Session(client)
|
|
154
|
+
city = s.chat("Which city is the Eiffel Tower in? One word.", max_tokens=8).text
|
|
155
|
+
w = s.tool("get_weather", {"city": city}, get_weather)
|
|
156
|
+
s.save("run.trace.json")
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
python -m idemcore trace verify run.trace.json
|
|
161
|
+
python -m idemcore trace replay run.trace.json
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
See [docs/agent-replay.md](docs/agent-replay.md) ([Tiếng Việt](docs/agent-replay.vi.md)) and `examples/agent_replay.py`.
|
|
165
|
+
|
|
146
166
|
## Errors
|
|
147
167
|
|
|
148
168
|
All errors derive from `idemcore.IdemcoreError` and carry `status`, `type` and
|
|
@@ -4,18 +4,23 @@ MANIFEST.in
|
|
|
4
4
|
README.md
|
|
5
5
|
pyproject.toml
|
|
6
6
|
src/idemcore/__init__.py
|
|
7
|
+
src/idemcore/__main__.py
|
|
7
8
|
src/idemcore/_types.py
|
|
8
9
|
src/idemcore/_version.py
|
|
9
10
|
src/idemcore/client.py
|
|
10
11
|
src/idemcore/errors.py
|
|
11
12
|
src/idemcore/py.typed
|
|
13
|
+
src/idemcore/trace.py
|
|
12
14
|
src/idemcore/verify.py
|
|
13
15
|
src/idemcore.egg-info/PKG-INFO
|
|
14
16
|
src/idemcore.egg-info/SOURCES.txt
|
|
15
17
|
src/idemcore.egg-info/dependency_links.txt
|
|
18
|
+
src/idemcore.egg-info/entry_points.txt
|
|
16
19
|
src/idemcore.egg-info/requires.txt
|
|
17
20
|
src/idemcore.egg-info/top_level.txt
|
|
21
|
+
tests/make_fixture.py
|
|
18
22
|
tests/mock_api.py
|
|
19
23
|
tests/test_client.py
|
|
24
|
+
tests/test_trace.py
|
|
20
25
|
tests/test_verify.py
|
|
21
26
|
tests/vectors.json
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Regenerate tests/fixtures/py.trace.json (used by the JS SDK's cross-language test)."""
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
6
|
+
sys.path.insert(0, HERE)
|
|
7
|
+
from mock_api import KEY, MockAPI # noqa: E402
|
|
8
|
+
from test_trace import agent # noqa: E402
|
|
9
|
+
from idemcore import Idemcore, trace # noqa: E402
|
|
10
|
+
|
|
11
|
+
with MockAPI() as api:
|
|
12
|
+
s = trace.Session(Idemcore(KEY, base_url=api.url + "/v1"), meta={"fixture": "py", "unicode": "Hà Nội ✓"},
|
|
13
|
+
session_id="ses_000000000000000000000001")
|
|
14
|
+
agent(s)
|
|
15
|
+
s.record_tool("floats", {"x": 0.1, "y": [1e-7, 2.5]}, {"ok": True})
|
|
16
|
+
s.created_at = "2026-11-01T00:00:00.000Z"
|
|
17
|
+
trace.save_trace(s.finish(), os.path.join(HERE, "fixtures", "py.trace.json"))
|
|
18
|
+
print("ok")
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
import copy
|
|
3
|
+
import io
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
import tempfile
|
|
8
|
+
import unittest
|
|
9
|
+
from contextlib import redirect_stdout
|
|
10
|
+
|
|
11
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
12
|
+
sys.path.insert(0, HERE)
|
|
13
|
+
from mock_api import KEY, V, MockAPI # noqa: E402
|
|
14
|
+
|
|
15
|
+
from idemcore import Idemcore # noqa: E402
|
|
16
|
+
from idemcore import trace as T # noqa: E402
|
|
17
|
+
|
|
18
|
+
KEYS = {"keys": V["keys"]}
|
|
19
|
+
Q = V["messages"] # the mock's receipt commits to these messages
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def weather(city):
|
|
23
|
+
return {"city": city, "temp_c": 18.5, "sky": "cloudy"}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def agent(s):
|
|
27
|
+
"""Tiny agent: ask, call a tool, ask again."""
|
|
28
|
+
a = s.chat(Q, max_tokens=16)
|
|
29
|
+
w = s.tool("get_weather", {"city": "Paris"}, weather)
|
|
30
|
+
s.note("approval", {"by": "ops", "ok": True})
|
|
31
|
+
b = s.chat(Q, max_tokens=16)
|
|
32
|
+
return a.text, w, b.text
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TraceTests(unittest.TestCase):
|
|
36
|
+
def setUp(self):
|
|
37
|
+
self.api = MockAPI().__enter__()
|
|
38
|
+
self.c = Idemcore(KEY, base_url=self.api.url + "/v1", max_retries=1)
|
|
39
|
+
|
|
40
|
+
def tearDown(self):
|
|
41
|
+
self.api.__exit__()
|
|
42
|
+
|
|
43
|
+
def rec(self, **kw):
|
|
44
|
+
s = T.Session(self.c, meta={"task": "demo", "n": 1})
|
|
45
|
+
agent(s)
|
|
46
|
+
return s.finish(**kw)
|
|
47
|
+
|
|
48
|
+
def test_tool_result_key_order_survives_replay(self):
|
|
49
|
+
# A prompt built from json.dumps(result) must be byte-identical on replay.
|
|
50
|
+
seen = []
|
|
51
|
+
|
|
52
|
+
def ag(s):
|
|
53
|
+
s.chat(Q, max_tokens=16)
|
|
54
|
+
r = s.tool("t", {"q": 1}, lambda q: {"z": 1, "a": {"y": 2, "b": 3}})
|
|
55
|
+
seen.append(json.dumps(r))
|
|
56
|
+
s = T.Session(self.c)
|
|
57
|
+
ag(s)
|
|
58
|
+
t = s.finish()
|
|
59
|
+
self.assertEqual(t["steps"][1]["body"]["result_json"], '{"z":1,"a":{"y":2,"b":3}}')
|
|
60
|
+
self.assertEqual(t["steps"][1]["body"]["args_json"], '{"q":1}')
|
|
61
|
+
rep = T.replay(t, self.c, agent=ag, keys=KEYS)
|
|
62
|
+
self.assertTrue(rep.ok, rep.errors)
|
|
63
|
+
self.assertEqual(seen[0], seen[1])
|
|
64
|
+
|
|
65
|
+
def test_record_and_verify(self):
|
|
66
|
+
t = self.rec()
|
|
67
|
+
self.assertEqual(t["manifest"]["step_count"], 4)
|
|
68
|
+
self.assertEqual([x["type"] for x in t["steps"]], ["model", "tool", "note", "model"])
|
|
69
|
+
v = T.verify_trace(t, keys=KEYS)
|
|
70
|
+
self.assertTrue(v.ok, v.errors)
|
|
71
|
+
self.assertEqual((v.model_steps, v.tool_steps), (2, 1))
|
|
72
|
+
# round trip through JSON text
|
|
73
|
+
v2 = T.verify_trace(json.loads(T.dumps_trace(t)), keys=KEYS)
|
|
74
|
+
self.assertTrue(v2.ok, v2.errors)
|
|
75
|
+
# default (production) keys reject the test-key receipts
|
|
76
|
+
self.assertFalse(T.verify_trace(t).ok)
|
|
77
|
+
self.assertTrue(T.verify_trace(t, check_receipts=False).ok)
|
|
78
|
+
|
|
79
|
+
def test_tamper_detection(self):
|
|
80
|
+
t = self.rec()
|
|
81
|
+
cases = {
|
|
82
|
+
"tool result": lambda x: x["steps"][1]["body"].__setitem__("result_json", '{"temp_c":30}'),
|
|
83
|
+
"output text": lambda x: x["steps"][0]["body"].__setitem__("output_text", "Lyon."),
|
|
84
|
+
"drop step": lambda x: x["steps"].pop(2),
|
|
85
|
+
"reorder": lambda x: x["steps"].reverse(),
|
|
86
|
+
"receipt": lambda x: x["steps"][3]["body"]["receipt"].__setitem__("output_tokens", 8),
|
|
87
|
+
"manifest meta": lambda x: x["manifest"].__setitem__("meta_json", "{}"),
|
|
88
|
+
"truncate": lambda x: x["steps"].pop(),
|
|
89
|
+
}
|
|
90
|
+
for name, f in cases.items():
|
|
91
|
+
bad = copy.deepcopy(t)
|
|
92
|
+
f(bad)
|
|
93
|
+
self.assertFalse(T.verify_trace(bad, keys=KEYS).ok, name)
|
|
94
|
+
# an attacker who rehashes the edited tool step still breaks the chain
|
|
95
|
+
bad = copy.deepcopy(t)
|
|
96
|
+
b = bad["steps"][1]["body"]
|
|
97
|
+
b["result_json"] = '{"temp_c":30}'
|
|
98
|
+
b["result_hash"] = T._h("trace-tool-result", b["result_json"].encode())
|
|
99
|
+
bad["steps"][1]["hash"] = T.step_hash(bad["steps"][1])
|
|
100
|
+
v = T.verify_trace(bad, keys=KEYS)
|
|
101
|
+
self.assertFalse(v.ok)
|
|
102
|
+
self.assertTrue(any("prev" in e for e in v.errors), v.errors)
|
|
103
|
+
|
|
104
|
+
def test_signatures(self):
|
|
105
|
+
k = T.generate_signing_key()
|
|
106
|
+
other = T.generate_signing_key()
|
|
107
|
+
t = self.rec(sign_with=k)
|
|
108
|
+
self.assertNotIn(k["d"], json.dumps(t))
|
|
109
|
+
v = T.verify_trace(t, keys=KEYS, trusted_signers=[k["kid"]])
|
|
110
|
+
self.assertTrue(v.ok, v.errors)
|
|
111
|
+
self.assertEqual(v.signers, [k["kid"]])
|
|
112
|
+
self.assertFalse(T.verify_trace(t, keys=KEYS, trusted_signers=[other["kid"]]).ok)
|
|
113
|
+
self.assertTrue(T.verify_trace(t, keys=KEYS, trusted_signers=[{"x": k["x"]}]).ok)
|
|
114
|
+
bad = copy.deepcopy(t)
|
|
115
|
+
bad["manifest"]["created_at"] = "2020-01-01T00:00:00.000Z"
|
|
116
|
+
bad["manifest_hash"] = T.manifest_hash(bad["manifest"])
|
|
117
|
+
self.assertFalse(T.verify_trace(bad, keys=KEYS, trusted_signers=[k["kid"]]).ok)
|
|
118
|
+
unsigned = self.rec()
|
|
119
|
+
self.assertFalse(T.verify_trace(unsigned, keys=KEYS, require_signature=True).ok)
|
|
120
|
+
|
|
121
|
+
def test_replay_model_only(self):
|
|
122
|
+
t = self.rec()
|
|
123
|
+
n = len(self.api.state.requests)
|
|
124
|
+
r = T.replay(t, self.c, keys=KEYS)
|
|
125
|
+
self.assertTrue(r.ok, r.summary())
|
|
126
|
+
self.assertEqual(r.model_calls, 2)
|
|
127
|
+
self.assertEqual(len(self.api.state.requests) - n, 2) # tools are not re-run against anything
|
|
128
|
+
self.assertTrue(T.verify_trace(r.replay_trace, keys=KEYS).ok)
|
|
129
|
+
self.assertEqual(json.loads(r.replay_trace["manifest"]["meta_json"])["replay_of"], t["manifest"]["session_id"])
|
|
130
|
+
|
|
131
|
+
def test_replay_agent_feeds_recorded_tools(self):
|
|
132
|
+
t = self.rec()
|
|
133
|
+
calls = []
|
|
134
|
+
|
|
135
|
+
def agent2(s):
|
|
136
|
+
a = s.chat(Q, max_tokens=16)
|
|
137
|
+
w = s.tool("get_weather", {"city": "Paris"}, lambda city: calls.append(city))
|
|
138
|
+
s.note("approval", {"by": "ops", "ok": True})
|
|
139
|
+
s.chat(Q, max_tokens=16)
|
|
140
|
+
self.assertEqual(w["temp_c"], 18.5)
|
|
141
|
+
r = T.replay(t, self.c, agent=agent2, keys=KEYS)
|
|
142
|
+
self.assertTrue(r.ok, r.summary())
|
|
143
|
+
self.assertEqual(calls, []) # the real tool never ran
|
|
144
|
+
|
|
145
|
+
def test_replay_detects_agent_divergence(self):
|
|
146
|
+
t = self.rec()
|
|
147
|
+
|
|
148
|
+
def wrong_args(s):
|
|
149
|
+
s.chat(Q, max_tokens=16)
|
|
150
|
+
s.tool("get_weather", {"city": "Rome"})
|
|
151
|
+
r = T.replay(t, self.c, agent=wrong_args, keys=KEYS)
|
|
152
|
+
self.assertFalse(r.ok)
|
|
153
|
+
self.assertIn("differs", r.errors[0])
|
|
154
|
+
|
|
155
|
+
def stops_early(s):
|
|
156
|
+
s.chat(Q, max_tokens=16)
|
|
157
|
+
r = T.replay(t, self.c, agent=stops_early, keys=KEYS)
|
|
158
|
+
self.assertFalse(r.ok)
|
|
159
|
+
|
|
160
|
+
def other_prompt(s):
|
|
161
|
+
s.chat([{"role": "user", "content": "hi"}], max_tokens=16)
|
|
162
|
+
r = T.replay(t, self.c, agent=other_prompt, keys=KEYS)
|
|
163
|
+
self.assertFalse(r.ok)
|
|
164
|
+
|
|
165
|
+
def test_replay_detects_output_divergence(self):
|
|
166
|
+
t = self.rec()
|
|
167
|
+
real = self.c.chat
|
|
168
|
+
|
|
169
|
+
def drift(*a, **k):
|
|
170
|
+
r = real(*a, **k)
|
|
171
|
+
r.output_token_ids = r.output_token_ids[:-1] + [r.output_token_ids[-1] + 1]
|
|
172
|
+
r.receipt = dict(r.receipt, chain_digest="0" * 64, engine_build="other")
|
|
173
|
+
return r
|
|
174
|
+
self.c.chat = drift
|
|
175
|
+
r = T.replay(t, self.c, keys=KEYS)
|
|
176
|
+
self.assertFalse(r.ok)
|
|
177
|
+
d = r.steps[0]
|
|
178
|
+
self.assertFalse(d.identical)
|
|
179
|
+
self.assertTrue(any("position 6" in x for x in d.diffs), d.diffs)
|
|
180
|
+
self.assertTrue(any("chain_digest" in x for x in d.diffs))
|
|
181
|
+
self.assertTrue(any("engine_build" in x for x in d.notes))
|
|
182
|
+
r = T.replay(t, self.c, keys=KEYS, agent=agent, strict=True)
|
|
183
|
+
self.assertFalse(r.ok)
|
|
184
|
+
self.assertEqual(r.model_calls, 1)
|
|
185
|
+
|
|
186
|
+
def test_replay_refuses_tampered_trace(self):
|
|
187
|
+
t = self.rec()
|
|
188
|
+
t["steps"][1]["body"]["result_json"] = "{}"
|
|
189
|
+
r = T.replay(t, self.c, keys=KEYS)
|
|
190
|
+
self.assertFalse(r.ok)
|
|
191
|
+
self.assertEqual(r.model_calls, 0)
|
|
192
|
+
|
|
193
|
+
def test_tool_errors_and_values(self):
|
|
194
|
+
s = T.Session(self.c)
|
|
195
|
+
|
|
196
|
+
def boom(**_):
|
|
197
|
+
raise ValueError("nope")
|
|
198
|
+
with self.assertRaises(ValueError):
|
|
199
|
+
s.tool("boom", {"a": 1}, boom)
|
|
200
|
+
s.record_tool("ext", [1, 2.5, "é"], {"ok": True})
|
|
201
|
+
with self.assertRaises(TypeError):
|
|
202
|
+
s.record_tool("bad", {"x": object()}, None)
|
|
203
|
+
with self.assertRaises(TypeError):
|
|
204
|
+
s.record_tool("nan", {"x": float("nan")}, None)
|
|
205
|
+
t = s.finish()
|
|
206
|
+
self.assertEqual(t["steps"][0]["body"]["error"], "ValueError: nope")
|
|
207
|
+
self.assertEqual(t["steps"][1]["body"]["args_json"], '[1,2.5,"é"]')
|
|
208
|
+
self.assertTrue(T.verify_trace(t, keys=KEYS).ok)
|
|
209
|
+
with self.assertRaises(RuntimeError):
|
|
210
|
+
s.note("late")
|
|
211
|
+
|
|
212
|
+
def test_stream_record_model(self):
|
|
213
|
+
s = T.Session()
|
|
214
|
+
st = self.c.chat_stream(Q, max_tokens=16)
|
|
215
|
+
res = st.until_done()
|
|
216
|
+
s.record_model(res, max_tokens=16, streamed=True)
|
|
217
|
+
t = s.finish()
|
|
218
|
+
self.assertTrue(T.verify_trace(t, keys=KEYS).ok)
|
|
219
|
+
self.assertTrue(T.replay(t, self.c, keys=KEYS).ok)
|
|
220
|
+
|
|
221
|
+
def test_empty_and_garbage(self):
|
|
222
|
+
t = T.Session().finish()
|
|
223
|
+
self.assertTrue(T.verify_trace(t).ok)
|
|
224
|
+
self.assertFalse(T.verify_trace({}).ok)
|
|
225
|
+
self.assertFalse(T.verify_trace({"manifest": {"format": "x"}, "steps": []}).ok)
|
|
226
|
+
|
|
227
|
+
def test_cli(self):
|
|
228
|
+
t = self.rec(sign_with=T.generate_signing_key())
|
|
229
|
+
with tempfile.TemporaryDirectory() as d:
|
|
230
|
+
p = os.path.join(d, "run.trace.json")
|
|
231
|
+
T.save_trace(t, p)
|
|
232
|
+
buf = io.StringIO()
|
|
233
|
+
with redirect_stdout(buf):
|
|
234
|
+
rc = T._cli(["show", p])
|
|
235
|
+
self.assertEqual(rc, 0)
|
|
236
|
+
self.assertIn("tool get_weather", buf.getvalue())
|
|
237
|
+
with redirect_stdout(io.StringIO()):
|
|
238
|
+
self.assertEqual(T._cli(["verify", p, "--no-receipts"]), 0)
|
|
239
|
+
t["steps"][1]["body"]["result_json"] = "{}"
|
|
240
|
+
T.save_trace(t, p)
|
|
241
|
+
with redirect_stdout(io.StringIO()):
|
|
242
|
+
self.assertEqual(T._cli(["verify", p, "--no-receipts"]), 1)
|
|
243
|
+
|
|
244
|
+
def test_cross_language_fixture(self):
|
|
245
|
+
"""A trace recorded by the JS SDK (test/fixtures) verifies here."""
|
|
246
|
+
p = os.path.join(HERE, "fixtures", "js.trace.json")
|
|
247
|
+
if not os.path.exists(p):
|
|
248
|
+
self.skipTest("no JS fixture")
|
|
249
|
+
t = T.load_trace(p)
|
|
250
|
+
v = T.verify_trace(t, keys=KEYS)
|
|
251
|
+
self.assertTrue(v.ok, v.errors)
|
|
252
|
+
|
|
253
|
+
def agent_js(s):
|
|
254
|
+
agent(s)
|
|
255
|
+
s.tool("floats", {"x": 0.1, "y": [1e-7, 2.5]}) # JS wrote 1e-7: compared by value
|
|
256
|
+
r = T.replay(t, self.c, agent=agent_js, keys=KEYS)
|
|
257
|
+
self.assertTrue(r.ok, r.summary())
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
if __name__ == "__main__":
|
|
261
|
+
unittest.main()
|
idemcore-0.1.2/CHANGELOG.md
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
# Changelog
|
|
2
|
-
|
|
3
|
-
## 0.1.2 (2026-11-01)
|
|
4
|
-
|
|
5
|
-
- Source repository is now public. Package metadata lists Source, Issues, Documentation
|
|
6
|
-
and Changelog links.
|
|
7
|
-
- Added SECURITY.md and CONTRIBUTING.md.
|
|
8
|
-
- No code changes.
|
|
9
|
-
|
|
10
|
-
## 0.1.1 (2026-11-01)
|
|
11
|
-
|
|
12
|
-
- README: the quickstart, streaming and async examples now pass `model` explicitly
|
|
13
|
-
(read from `client.models()`), since the API requires a model id.
|
|
14
|
-
- Release published from GitHub Actions with PyPI Trusted Publishing.
|
|
15
|
-
|
|
16
|
-
## 0.1.0 (2026-11-01)
|
|
17
|
-
|
|
18
|
-
- First release: sync and async clients, streaming, signed receipts and local
|
|
19
|
-
receipt verification.
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.1.2"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|