fathom-read 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fathom_read/__init__.py +17 -0
- fathom_read/adapters/__init__.py +47 -0
- fathom_read/adapters/_tools.py +88 -0
- fathom_read/adapters/crewai.py +34 -0
- fathom_read/adapters/dbos.py +25 -0
- fathom_read/adapters/edits.py +51 -0
- fathom_read/adapters/events.py +17 -0
- fathom_read/adapters/langgraph.py +56 -0
- fathom_read/adapters/letta.py +32 -0
- fathom_read/adapters/openinference.py +66 -0
- fathom_read/cli.py +128 -0
- fathom_read/client.py +40 -0
- fathom_read/examples/crewai_events.json +31 -0
- fathom_read/examples/knowledge_update.json +26 -0
- fathom_read/examples/langgraph_history.json +37 -0
- fathom_read/examples/order_duplicate.json +31 -0
- fathom_read/examples/rename_coherent.json +63 -0
- fathom_read/examples/rename_starved.json +63 -0
- fathom_read/ops.py +79 -0
- fathom_read-0.1.0.dist-info/METADATA +141 -0
- fathom_read-0.1.0.dist-info/RECORD +25 -0
- fathom_read-0.1.0.dist-info/WHEEL +5 -0
- fathom_read-0.1.0.dist-info/entry_points.txt +2 -0
- fathom_read-0.1.0.dist-info/licenses/LICENSE +21 -0
- fathom_read-0.1.0.dist-info/top_level.txt +1 -0
fathom_read/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""fathom-read: catch the step where an AI agent contradicts a decision it already made."""
|
|
2
|
+
from .ops import Op, Finding, Verdict # noqa: F401
|
|
3
|
+
from .client import read, ReadError # noqa: F401
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
__all__ = ["Op", "Finding", "Verdict", "read", "ReadError", "read_file", "load_ops"]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def load_ops(path: str, fmt: str = "auto", mapping_path: str = None):
|
|
10
|
+
"""Turn a trace file in any supported format into the op stream."""
|
|
11
|
+
from .cli import load_ops as _load
|
|
12
|
+
return _load(path, fmt, mapping_path)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def read_file(path: str, fmt: str = "auto", supersede=None, mapping_path: str = None, **kw) -> Verdict:
|
|
16
|
+
"""Read a trace file and return the hosted read's verdict."""
|
|
17
|
+
return read(load_ops(path, fmt, mapping_path), supersede=supersede, **kw)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Adapters turn what a framework already records into the op stream the read consumes."""
|
|
2
|
+
from . import events, edits, openinference, langgraph, crewai, letta, dbos # noqa: F401
|
|
3
|
+
|
|
4
|
+
FORMATS = {
|
|
5
|
+
"events": events,
|
|
6
|
+
"edits": edits,
|
|
7
|
+
"openinference": openinference,
|
|
8
|
+
"langgraph": langgraph,
|
|
9
|
+
"crewai": crewai,
|
|
10
|
+
"letta": letta,
|
|
11
|
+
"dbos": dbos,
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def detect(doc) -> str:
|
|
16
|
+
"""Guess the format of a loaded JSON document."""
|
|
17
|
+
if isinstance(doc, dict):
|
|
18
|
+
if "ops" in doc:
|
|
19
|
+
return "events"
|
|
20
|
+
if "spans" in doc or "resourceSpans" in doc:
|
|
21
|
+
return "openinference"
|
|
22
|
+
if "edits" in doc or "initial_files" in doc:
|
|
23
|
+
return "edits"
|
|
24
|
+
if "blocks" in doc or "passages" in doc:
|
|
25
|
+
return "letta"
|
|
26
|
+
if "steps" in doc and "workflow_id" in doc:
|
|
27
|
+
return "dbos"
|
|
28
|
+
if "events" in doc:
|
|
29
|
+
return "crewai"
|
|
30
|
+
if "snapshots" in doc or "history" in doc:
|
|
31
|
+
return "langgraph"
|
|
32
|
+
if isinstance(doc, list) and doc:
|
|
33
|
+
first = doc[0]
|
|
34
|
+
if isinstance(first, dict):
|
|
35
|
+
if "op" in first and "key" in first:
|
|
36
|
+
return "events"
|
|
37
|
+
if "attributes" in first or "span_kind" in first:
|
|
38
|
+
return "openinference"
|
|
39
|
+
if "values" in first and ("next" in first or "config" in first or "metadata" in first or "step" in first):
|
|
40
|
+
return "langgraph"
|
|
41
|
+
if "type" in first and ("tool_name" in first or "task_name" in first or first.get("type", "").startswith(("tool_", "task_"))):
|
|
42
|
+
return "crewai"
|
|
43
|
+
if "step_name" in first:
|
|
44
|
+
return "dbos"
|
|
45
|
+
if "tool" in first and "args" in first:
|
|
46
|
+
return "edits"
|
|
47
|
+
raise ValueError("could not detect the trace format; pass --format")
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Shared mapping from tool calls to ops. Adapters that see (tool name, args, result, ok) use this."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from typing import Any, Dict, Optional
|
|
6
|
+
|
|
7
|
+
from ..ops import Op
|
|
8
|
+
|
|
9
|
+
# Default mapping: tool name -> how to build an op from its arguments.
|
|
10
|
+
# "key" and "value" name the argument fields; "kind" labels the fact; "op" is the ledger op.
|
|
11
|
+
DEFAULT_MAP: Dict[str, Dict[str, Any]] = {
|
|
12
|
+
# coding agents (OpenHands, CodeAct-style editors); content folding lives in the edits adapter
|
|
13
|
+
"str_replace_editor": {"op": "edit"},
|
|
14
|
+
"edit_file": {"op": "edit"},
|
|
15
|
+
"write_file": {"op": "set", "kind": "file", "key": "path", "value": "content"},
|
|
16
|
+
"create_file": {"op": "set", "kind": "file", "key": "path", "value": "content"},
|
|
17
|
+
"delete_file": {"op": "remove", "kind": "file", "key": "path"},
|
|
18
|
+
# record-keeping crews and workflows
|
|
19
|
+
"write_record": {"op": "set", "kind": "record", "key": "record", "value": "content"},
|
|
20
|
+
"update_record": {"op": "set", "kind": "record", "key": "record", "value": "content"},
|
|
21
|
+
"commit_record": {"op": "set", "kind": "record", "key": "record", "value": "content"},
|
|
22
|
+
"set_value": {"op": "set", "kind": "fact", "key": "key", "value": "value"},
|
|
23
|
+
"update_value": {"op": "set", "kind": "fact", "key": "key", "value": "value"},
|
|
24
|
+
"delete_value": {"op": "remove", "kind": "fact", "key": "key"},
|
|
25
|
+
"rename_key": {"op": "rename", "kind": "fact", "key": "old", "to": "new"},
|
|
26
|
+
# Letta memory tools
|
|
27
|
+
"core_memory_append": {"op": "append", "kind": "block", "key": "label", "value": "content"},
|
|
28
|
+
"core_memory_replace": {"op": "set", "kind": "block", "key": "label", "value": "new_content"},
|
|
29
|
+
"memory_replace": {"op": "set", "kind": "block", "key": "label", "value": "new_str"},
|
|
30
|
+
"memory_insert": {"op": "append", "kind": "block", "key": "label", "value": "new_str"},
|
|
31
|
+
"memory_rethink": {"op": "set", "kind": "block", "key": "label", "value": "new_memory"},
|
|
32
|
+
"archival_memory_insert": {"op": "add", "kind": "passage", "key": "archival", "value": "content"},
|
|
33
|
+
# ordering and transactions (web agents, customer-service agents)
|
|
34
|
+
"add_item": {"op": "add", "kind": "order", "key": "cart", "value": "item"},
|
|
35
|
+
"add_to_cart": {"op": "add", "kind": "order", "key": "cart", "value": "item"},
|
|
36
|
+
"remove_item": {"op": "remove_member", "kind": "order", "key": "cart", "value": "item"},
|
|
37
|
+
"place_order": {"op": "commit", "kind": "order", "key": "order"},
|
|
38
|
+
"checkout": {"op": "commit", "kind": "order", "key": "order"},
|
|
39
|
+
"submit": {"op": "commit", "kind": "order", "key": "order"},
|
|
40
|
+
"book_reservation": {"op": "set", "kind": "reservation", "key": "reservation_id", "value": "details"},
|
|
41
|
+
"cancel_reservation": {"op": "remove", "kind": "reservation", "key": "reservation_id"},
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _first(args: Dict[str, Any], *names: str) -> Optional[str]:
|
|
46
|
+
for n in names:
|
|
47
|
+
if n in args and args[n] is not None:
|
|
48
|
+
v = args[n]
|
|
49
|
+
return v if isinstance(v, str) else json.dumps(v, sort_keys=True)
|
|
50
|
+
return None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def load_map(path: Optional[str]) -> Dict[str, Dict[str, Any]]:
|
|
54
|
+
m = dict(DEFAULT_MAP)
|
|
55
|
+
if path:
|
|
56
|
+
with open(path) as f:
|
|
57
|
+
m.update(json.load(f))
|
|
58
|
+
return m
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def op_from_tool(name: str, args: Any, ok: bool, step: int, mapping: Dict[str, Dict[str, Any]],
|
|
62
|
+
source: str = "") -> Optional[Op]:
|
|
63
|
+
"""Build an op from one tool call, or None when the tool does not write committed state."""
|
|
64
|
+
name = (name or "").strip().lower().replace(" ", "_")
|
|
65
|
+
spec = mapping.get(name)
|
|
66
|
+
if spec is None:
|
|
67
|
+
return None
|
|
68
|
+
if not isinstance(args, dict):
|
|
69
|
+
try:
|
|
70
|
+
args = json.loads(args) if args else {}
|
|
71
|
+
except (TypeError, ValueError):
|
|
72
|
+
args = {}
|
|
73
|
+
kind = spec.get("kind", "fact")
|
|
74
|
+
op = spec["op"]
|
|
75
|
+
key = _first(args, spec.get("key", "key"), "key", "record", "path", "label", "name", "id") or spec.get("key", name)
|
|
76
|
+
value = _first(args, spec.get("value", "value"), "value", "content", "new_str", "new_content", "text")
|
|
77
|
+
if op == "edit":
|
|
78
|
+
return None # the edits adapter folds these with file contents
|
|
79
|
+
if op == "append":
|
|
80
|
+
return Op("set", kind, key, value=value, ok=ok, step=step, source=source or name)
|
|
81
|
+
if op == "remove_member":
|
|
82
|
+
return Op("remove", kind, f"{key}:{value}", ok=ok, step=step, source=source or name)
|
|
83
|
+
if op == "rename":
|
|
84
|
+
to = _first(args, spec.get("to", "new"), "new", "to")
|
|
85
|
+
return Op("rename", kind, key, to=to, ok=ok, step=step, source=source or name)
|
|
86
|
+
if op == "commit":
|
|
87
|
+
return Op("commit", kind, key, ok=ok, step=step, source=source or name)
|
|
88
|
+
return Op(op, kind, key, value=value, ok=ok, step=step, source=source or name)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CrewAI event logs: the events the crew's event bus already emits.
|
|
3
|
+
|
|
4
|
+
Input: a list of events (or {"events": [...]}) with "type" in tool_usage_finished,
|
|
5
|
+
tool_usage_error, task_completed; tool events carry tool_name and tool_args. A write tool's
|
|
6
|
+
arguments name the record and the content committed to it; a task_completed event commits
|
|
7
|
+
the task's output under the task name. Capture them with a listener on crewai's event bus.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Any, List
|
|
12
|
+
|
|
13
|
+
from ..ops import Op
|
|
14
|
+
from ._tools import op_from_tool, load_map
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def load(doc: Any, mapping_path: str = None, **_) -> List[Op]:
|
|
18
|
+
events = doc.get("events", []) if isinstance(doc, dict) else doc
|
|
19
|
+
mapping = load_map(mapping_path)
|
|
20
|
+
ops: List[Op] = []
|
|
21
|
+
for i, ev in enumerate(events):
|
|
22
|
+
t = (ev.get("type") or ev.get("_type") or "").replace("Event", "").lower()
|
|
23
|
+
t = {"toolusagefinished": "tool_usage_finished", "toolusageerror": "tool_usage_error",
|
|
24
|
+
"taskcompleted": "task_completed"}.get(t, t)
|
|
25
|
+
if t in ("tool_usage_finished", "tool_usage_error"):
|
|
26
|
+
ok = t == "tool_usage_finished" and not bool(ev.get("failure"))
|
|
27
|
+
op = op_from_tool(ev.get("tool_name", ""), ev.get("tool_args"), ok, i, mapping, source=t)
|
|
28
|
+
if op is not None:
|
|
29
|
+
ops.append(op)
|
|
30
|
+
elif t == "task_completed":
|
|
31
|
+
name = ev.get("task_name") or ev.get("task") or f"task_{i}"
|
|
32
|
+
out = ev.get("output") or ev.get("raw") or ""
|
|
33
|
+
ops.append(Op("set", "task", str(name), value=str(out), ok=True, step=i, source="task_completed"))
|
|
34
|
+
return ops
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""
|
|
2
|
+
DBOS step streams: the inputs and outputs of each step in a workflow, in order.
|
|
3
|
+
|
|
4
|
+
Input: {"workflow_id": "...", "steps": [{"step_name", "args", "result", "ok"}]} or a bare list.
|
|
5
|
+
Step names map to ops through the tool mapping (pass --map for your own step names).
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any, List
|
|
10
|
+
|
|
11
|
+
from ..ops import Op
|
|
12
|
+
from ._tools import op_from_tool, load_map
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def load(doc: Any, mapping_path: str = None, **_) -> List[Op]:
|
|
16
|
+
steps = doc.get("steps", []) if isinstance(doc, dict) else doc
|
|
17
|
+
mapping = load_map(mapping_path)
|
|
18
|
+
ops: List[Op] = []
|
|
19
|
+
for i, s in enumerate(steps):
|
|
20
|
+
ok = bool(s.get("ok", True)) and not bool(s.get("error"))
|
|
21
|
+
args = s.get("args") or s.get("inputs") or {}
|
|
22
|
+
op = op_from_tool(s.get("step_name") or s.get("name", ""), args, ok, i, mapping, source="step")
|
|
23
|
+
if op is not None:
|
|
24
|
+
ops.append(op)
|
|
25
|
+
return ops
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Coding-agent edit logs: str_replace-style edits over a set of files.
|
|
3
|
+
|
|
4
|
+
Input: {"initial_files": {path: content}, "edits": [{"tool": "str_replace_editor",
|
|
5
|
+
"args": {"path", "old_str", "new_str"}, "ok": true}, ...], "done": true}
|
|
6
|
+
or a bare list of edit records. The adapter folds each successful edit onto the file it
|
|
7
|
+
targets and emits one "set" op per edit with the file's new content, so the read can see what
|
|
8
|
+
the agent actually left in each file. A rejected edit changes nothing.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import Any, Dict, List
|
|
13
|
+
|
|
14
|
+
from ..ops import Op
|
|
15
|
+
|
|
16
|
+
EDIT_TOOLS = {"str_replace_editor", "edit_file", "str_replace", "apply_patch"}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def load(doc: Any, **_) -> List[Op]:
|
|
20
|
+
if isinstance(doc, list):
|
|
21
|
+
doc = {"edits": doc}
|
|
22
|
+
files: Dict[str, str] = dict(doc.get("initial_files", {}))
|
|
23
|
+
ops: List[Op] = []
|
|
24
|
+
step = 0
|
|
25
|
+
for path, content in files.items():
|
|
26
|
+
ops.append(Op("set", "file", path, value=content, ok=True, step=step, source="initial"))
|
|
27
|
+
step += 1
|
|
28
|
+
for e in doc.get("edits", []):
|
|
29
|
+
tool = (e.get("tool") or e.get("name") or "").strip().lower()
|
|
30
|
+
args = e.get("args") or e.get("parameters") or {}
|
|
31
|
+
ok = bool(e.get("ok", True)) and not bool(e.get("error"))
|
|
32
|
+
if tool in EDIT_TOOLS:
|
|
33
|
+
path, old, new = args.get("path"), args.get("old_str"), args.get("new_str")
|
|
34
|
+
applied = False
|
|
35
|
+
if ok and path in files and old and old in files[path]:
|
|
36
|
+
files[path] = files[path].replace(old, new or "")
|
|
37
|
+
applied = True
|
|
38
|
+
ops.append(Op("set", "file", str(path), value=files.get(str(path)), ok=applied, step=step, source=tool))
|
|
39
|
+
elif tool in ("write_file", "create_file"):
|
|
40
|
+
if ok:
|
|
41
|
+
files[str(args.get("path"))] = str(args.get("content", ""))
|
|
42
|
+
ops.append(Op("set", "file", str(args.get("path")), value=str(args.get("content", "")), ok=ok, step=step, source=tool))
|
|
43
|
+
elif tool in ("delete_file",):
|
|
44
|
+
if ok:
|
|
45
|
+
files.pop(str(args.get("path")), None)
|
|
46
|
+
ops.append(Op("remove", "file", str(args.get("path")), ok=ok, step=step, source=tool))
|
|
47
|
+
else:
|
|
48
|
+
step += 1
|
|
49
|
+
continue
|
|
50
|
+
step += 1
|
|
51
|
+
return ops
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Native format: a JSON list (or {"ops": [...]}) of op dicts, or JSON Lines with one op per line."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from typing import Any, List
|
|
6
|
+
|
|
7
|
+
from ..ops import Op
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def load(doc: Any, **_) -> List[Op]:
|
|
11
|
+
if isinstance(doc, dict):
|
|
12
|
+
doc = doc.get("ops", [])
|
|
13
|
+
return [Op.from_dict(d) for d in doc]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def load_jsonl(text: str) -> List[Op]:
|
|
17
|
+
return [Op.from_dict(json.loads(line)) for line in text.splitlines() if line.strip()]
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""
|
|
2
|
+
LangGraph state history: the checkpoint lineage from graph.get_state_history(config).
|
|
3
|
+
|
|
4
|
+
Input: a list of snapshots, each with "values" (the channel values at that step), newest
|
|
5
|
+
first as LangGraph returns them, or {"snapshots": [...]}. The adapter diffs consecutive
|
|
6
|
+
snapshots and emits one op per changed channel: a scalar or string channel that changed is a
|
|
7
|
+
"set"; a list channel that grew is an "add" per new element. Every checkpoint is a
|
|
8
|
+
successful commit, so ok is always true here; the read then checks what those commits say
|
|
9
|
+
against one another.
|
|
10
|
+
|
|
11
|
+
To export from a running graph:
|
|
12
|
+
import json
|
|
13
|
+
history = [{"values": s.values, "step": s.metadata.get("step")} for s in graph.get_state_history(config)]
|
|
14
|
+
json.dump(history, open("history.json", "w"), default=str)
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
from typing import Any, List
|
|
20
|
+
|
|
21
|
+
from ..ops import Op
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _as_text(v: Any) -> str:
|
|
25
|
+
return v if isinstance(v, str) else json.dumps(v, sort_keys=True, default=str)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def load(doc: Any, **_) -> List[Op]:
|
|
29
|
+
snaps = doc.get("snapshots", doc.get("history", [])) if isinstance(doc, dict) else doc
|
|
30
|
+
snaps = list(snaps)
|
|
31
|
+
if len(snaps) > 1:
|
|
32
|
+
# get_state_history returns newest first; put them in run order.
|
|
33
|
+
s0 = snaps[0].get("step", snaps[0].get("metadata", {}).get("step"))
|
|
34
|
+
s1 = snaps[-1].get("step", snaps[-1].get("metadata", {}).get("step"))
|
|
35
|
+
if s0 is None or s1 is None or s0 > s1:
|
|
36
|
+
snaps = snaps[::-1]
|
|
37
|
+
ops: List[Op] = []
|
|
38
|
+
prev: dict = {}
|
|
39
|
+
step = 0
|
|
40
|
+
for snap in snaps:
|
|
41
|
+
values = snap.get("values", {}) or {}
|
|
42
|
+
for chan, val in values.items():
|
|
43
|
+
old = prev.get(chan)
|
|
44
|
+
if isinstance(val, list) and isinstance(old, list) and len(val) >= len(old) and val[:len(old)] == old:
|
|
45
|
+
for item in val[len(old):]:
|
|
46
|
+
ops.append(Op("add", "channel", chan, value=_as_text(item), step=step, source="checkpoint"))
|
|
47
|
+
step += 1
|
|
48
|
+
elif isinstance(val, list) and old is None:
|
|
49
|
+
for item in val:
|
|
50
|
+
ops.append(Op("add", "channel", chan, value=_as_text(item), step=step, source="checkpoint"))
|
|
51
|
+
step += 1
|
|
52
|
+
elif val != old:
|
|
53
|
+
ops.append(Op("set", "channel", chan, value=_as_text(val), step=step, source="checkpoint"))
|
|
54
|
+
step += 1
|
|
55
|
+
prev = values
|
|
56
|
+
return ops
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Letta (MemGPT) memory: the blocks and passages the agent persists, and the edit calls that produced them.
|
|
3
|
+
|
|
4
|
+
Input: {"blocks": [{"label", "value"}], "passages": [{"text"}], "tool_calls": [{"name", "args", "ok"}]}.
|
|
5
|
+
Blocks and passages give the final committed memory; tool_calls give the stream. Export with
|
|
6
|
+
agents.blocks.list, agents.passages.list, and the tool_call messages from agents.messages.list.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Any, List
|
|
11
|
+
|
|
12
|
+
from ..ops import Op
|
|
13
|
+
from ._tools import op_from_tool, load_map
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def load(doc: Any, mapping_path: str = None, **_) -> List[Op]:
|
|
17
|
+
mapping = load_map(mapping_path)
|
|
18
|
+
ops: List[Op] = []
|
|
19
|
+
step = 0
|
|
20
|
+
for tc in doc.get("tool_calls", []) or []:
|
|
21
|
+
op = op_from_tool(tc.get("name", ""), tc.get("args"), bool(tc.get("ok", True)), step, mapping, source="tool_call")
|
|
22
|
+
if op is not None:
|
|
23
|
+
ops.append(op)
|
|
24
|
+
step += 1
|
|
25
|
+
# The persisted memory at the end of the run, as the final committed values.
|
|
26
|
+
for b in doc.get("blocks", []) or []:
|
|
27
|
+
ops.append(Op("set", "block", str(b.get("label", "block")), value=str(b.get("value", "")), step=step, source="persisted"))
|
|
28
|
+
step += 1
|
|
29
|
+
for i, p in enumerate(doc.get("passages", []) or []):
|
|
30
|
+
ops.append(Op("set", "passage", str(p.get("id", i)), value=str(p.get("text", "")), step=step, source="persisted"))
|
|
31
|
+
step += 1
|
|
32
|
+
return ops
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""
|
|
2
|
+
OpenInference traces, the span format Arize Phoenix stores.
|
|
3
|
+
|
|
4
|
+
Input: a list of spans, each with an "attributes" dict carrying openinference.span.kind,
|
|
5
|
+
tool.name, tool.parameters, and output.value; or {"spans": [...]}. Only TOOL spans matter.
|
|
6
|
+
Edit tools fold through the edits adapter when the document carries "initial_files".
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from typing import Any, Dict, List
|
|
12
|
+
|
|
13
|
+
from ..ops import Op
|
|
14
|
+
from . import edits as _edits
|
|
15
|
+
from ._tools import op_from_tool, load_map
|
|
16
|
+
|
|
17
|
+
KIND = "openinference.span.kind"
|
|
18
|
+
TOOL_NAME = "tool.name"
|
|
19
|
+
TOOL_PARAMS = "tool.parameters"
|
|
20
|
+
OUTPUT = "output.value"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _parse(v, default=None):
|
|
24
|
+
if v is None:
|
|
25
|
+
return default
|
|
26
|
+
if isinstance(v, (dict, list)):
|
|
27
|
+
return v
|
|
28
|
+
try:
|
|
29
|
+
return json.loads(v)
|
|
30
|
+
except (TypeError, ValueError):
|
|
31
|
+
return default
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def load(doc: Any, mapping_path: str = None, **_) -> List[Op]:
|
|
35
|
+
spans = doc.get("spans", []) if isinstance(doc, dict) else doc
|
|
36
|
+
initial_files = doc.get("initial_files") if isinstance(doc, dict) else None
|
|
37
|
+
mapping = load_map(mapping_path)
|
|
38
|
+
tool_spans: List[Dict[str, Any]] = []
|
|
39
|
+
for sp in spans:
|
|
40
|
+
a = sp.get("attributes", sp)
|
|
41
|
+
if str(a.get(KIND, "")).upper() == "TOOL":
|
|
42
|
+
tool_spans.append(a)
|
|
43
|
+
tool_spans.sort(key=lambda a: a.get("start_time", a.get("start", 0)) or 0)
|
|
44
|
+
|
|
45
|
+
edit_records = []
|
|
46
|
+
ops: List[Op] = []
|
|
47
|
+
for i, a in enumerate(tool_spans):
|
|
48
|
+
name = a.get(TOOL_NAME, "")
|
|
49
|
+
params = _parse(a.get(TOOL_PARAMS), {}) or {}
|
|
50
|
+
ret = _parse(a.get(OUTPUT), {})
|
|
51
|
+
ok = True
|
|
52
|
+
if isinstance(ret, dict) and "success" in ret:
|
|
53
|
+
ok = bool(ret["success"])
|
|
54
|
+
elif isinstance(ret, dict) and ret.get("error"):
|
|
55
|
+
ok = False
|
|
56
|
+
elif isinstance(ret, str) and ret.lower().startswith("error"):
|
|
57
|
+
ok = False
|
|
58
|
+
if str(name).strip().lower() in _edits.EDIT_TOOLS:
|
|
59
|
+
edit_records.append({"tool": name, "args": params, "ok": ok})
|
|
60
|
+
continue
|
|
61
|
+
op = op_from_tool(name, params, ok, i, mapping, source="span")
|
|
62
|
+
if op is not None:
|
|
63
|
+
ops.append(op)
|
|
64
|
+
if edit_records:
|
|
65
|
+
ops = _edits.load({"initial_files": initial_files or {}, "edits": edit_records}) + ops
|
|
66
|
+
return ops
|
fathom_read/cli.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
from typing import List, Optional, Tuple
|
|
8
|
+
|
|
9
|
+
from . import adapters
|
|
10
|
+
from .client import DEMO_KEY, ReadError, read
|
|
11
|
+
from .ops import Op, Verdict
|
|
12
|
+
|
|
13
|
+
EXAMPLES = os.path.join(os.path.dirname(__file__), "examples")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def load_ops(path: str, fmt: str = "auto", mapping_path: Optional[str] = None) -> List[Op]:
|
|
17
|
+
with open(path) as f:
|
|
18
|
+
text = f.read()
|
|
19
|
+
if path.endswith(".jsonl"):
|
|
20
|
+
return adapters.events.load_jsonl(text)
|
|
21
|
+
doc = json.loads(text)
|
|
22
|
+
if fmt == "auto":
|
|
23
|
+
fmt = adapters.detect(doc)
|
|
24
|
+
if fmt not in adapters.FORMATS:
|
|
25
|
+
raise SystemExit(f"unknown format {fmt!r}; run `fathom formats`")
|
|
26
|
+
return adapters.FORMATS[fmt].load(doc, mapping_path=mapping_path)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def parse_supersede(items: Optional[List[str]]) -> List[Tuple[str, str]]:
|
|
30
|
+
out = []
|
|
31
|
+
for it in items or []:
|
|
32
|
+
if "=" not in it:
|
|
33
|
+
raise SystemExit(f"--supersede expects old=new, got {it!r}")
|
|
34
|
+
old, new = it.split("=", 1)
|
|
35
|
+
out.append((old.strip(), new.strip()))
|
|
36
|
+
return out
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def render(verdict: Verdict, title: str = "") -> str:
|
|
40
|
+
lines = []
|
|
41
|
+
if title:
|
|
42
|
+
lines.append(title)
|
|
43
|
+
lines.append(f"ops read: {verdict.ops_read} rejected (no-ops): {verdict.ops_rejected} live facts: {verdict.live_facts}")
|
|
44
|
+
if verdict.coherent:
|
|
45
|
+
lines.append("committed state: coherent. No action contradicted an earlier commitment.")
|
|
46
|
+
return "\n".join(lines)
|
|
47
|
+
lines.append(f"committed state: {len(verdict.findings)} finding{'s' if len(verdict.findings) != 1 else ''}")
|
|
48
|
+
for f in verdict.findings:
|
|
49
|
+
where = f"step {f.step}" if f.step is not None else "end of run"
|
|
50
|
+
cite = f" (cites step {f.cites})" if f.cites is not None else ""
|
|
51
|
+
lines.append(f" [{f.kind}] {where}{cite}: {f.detail}")
|
|
52
|
+
return "\n".join(lines)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _read(ops, supersede, args) -> Verdict:
|
|
56
|
+
try:
|
|
57
|
+
return read(ops, supersede=supersede, key=args.key, endpoint=args.endpoint)
|
|
58
|
+
except ReadError as e:
|
|
59
|
+
raise SystemExit(f"fathom: {e}")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def cmd_read(args) -> int:
|
|
63
|
+
ops = load_ops(args.path, args.format, args.map)
|
|
64
|
+
if args.ops:
|
|
65
|
+
print(json.dumps([o.as_dict() for o in ops], indent=2))
|
|
66
|
+
return 0
|
|
67
|
+
v = _read(ops, parse_supersede(args.supersede), args)
|
|
68
|
+
if args.json:
|
|
69
|
+
print(json.dumps(v.as_dict(), indent=2))
|
|
70
|
+
else:
|
|
71
|
+
print(render(v, os.path.basename(args.path)))
|
|
72
|
+
return 0 if v.coherent else 2
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def cmd_demo(args) -> int:
|
|
76
|
+
print("fathom demo: a coding agent renames guest_id to customer_id across five files, then runs the tests.\n")
|
|
77
|
+
for name in ("rename_coherent.json", "rename_starved.json"):
|
|
78
|
+
ops = load_ops(os.path.join(EXAMPLES, name), "edits")
|
|
79
|
+
v = _read(ops, [("guest_id", "customer_id")], args)
|
|
80
|
+
print(render(v, f"== {name}"))
|
|
81
|
+
print()
|
|
82
|
+
print("Both runs reported success and a green test suite. Only one of them renamed the field.")
|
|
83
|
+
print("Try it on your own trace: fathom read path/to/trace.json --supersede old=new")
|
|
84
|
+
return 0
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def cmd_formats(args) -> int:
|
|
88
|
+
for name, mod in adapters.FORMATS.items():
|
|
89
|
+
doc = (mod.__doc__ or "").strip().splitlines()[0] if mod.__doc__ else ""
|
|
90
|
+
print(f"{name:14s} {doc}")
|
|
91
|
+
return 0
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _common(p):
|
|
95
|
+
p.add_argument("--key", help="your read key (or set FATHOM_API_KEY); the demo key is rate-limited")
|
|
96
|
+
p.add_argument("--endpoint", help=argparse.SUPPRESS)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
100
|
+
p = argparse.ArgumentParser(prog="fathom", description="Catch the step where an AI agent contradicts a decision it already made.")
|
|
101
|
+
sub = p.add_subparsers(dest="cmd")
|
|
102
|
+
|
|
103
|
+
r = sub.add_parser("read", help="read a trace and report contradictions of committed state")
|
|
104
|
+
r.add_argument("path", help="trace file (.json or .jsonl)")
|
|
105
|
+
r.add_argument("--format", default="auto", help="events | edits | openinference | langgraph | crewai | letta | dbos (default: auto)")
|
|
106
|
+
r.add_argument("--supersede", action="append", metavar="OLD=NEW", help="a token the run should have replaced, e.g. guest_id=customer_id (repeatable)")
|
|
107
|
+
r.add_argument("--map", help="JSON file mapping your tool or step names to ops")
|
|
108
|
+
r.add_argument("--json", action="store_true", help="print the verdict as JSON")
|
|
109
|
+
r.add_argument("--ops", action="store_true", help="print the op stream the adapter produced and stop (nothing is sent)")
|
|
110
|
+
_common(r)
|
|
111
|
+
r.set_defaults(fn=cmd_read)
|
|
112
|
+
|
|
113
|
+
d = sub.add_parser("demo", help="run the bundled rename example, coherent and not")
|
|
114
|
+
_common(d)
|
|
115
|
+
d.set_defaults(fn=cmd_demo)
|
|
116
|
+
|
|
117
|
+
f = sub.add_parser("formats", help="list the trace formats the adapters accept")
|
|
118
|
+
f.set_defaults(fn=cmd_formats)
|
|
119
|
+
|
|
120
|
+
args = p.parse_args(argv)
|
|
121
|
+
if not args.cmd:
|
|
122
|
+
p.print_help()
|
|
123
|
+
return 1
|
|
124
|
+
return args.fn(args)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
if __name__ == "__main__":
|
|
128
|
+
sys.exit(main())
|
fathom_read/client.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""The hosted read. The client sends an op stream and gets a verdict back."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import urllib.error
|
|
7
|
+
import urllib.request
|
|
8
|
+
from typing import Iterable, List, Optional, Tuple
|
|
9
|
+
|
|
10
|
+
from .ops import Op, Verdict
|
|
11
|
+
|
|
12
|
+
DEFAULT_ENDPOINT = "https://read.embeddedriskanalytics.com/v1/read"
|
|
13
|
+
DEMO_KEY = "demo" # rate-limited; get your own key at https://embeddedriskanalytics.com/contact.html
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ReadError(RuntimeError):
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def read(ops: Iterable[Op], supersede: Optional[List[Tuple[str, str]]] = None,
|
|
21
|
+
key: Optional[str] = None, endpoint: Optional[str] = None, timeout: float = 30.0) -> Verdict:
|
|
22
|
+
"""Send the ops to the hosted read and return its verdict."""
|
|
23
|
+
key = key or os.environ.get("FATHOM_API_KEY") or DEMO_KEY
|
|
24
|
+
endpoint = endpoint or os.environ.get("FATHOM_ENDPOINT") or DEFAULT_ENDPOINT
|
|
25
|
+
body = json.dumps({"ops": [o.as_dict() for o in ops], "supersede": [list(p) for p in (supersede or [])]}).encode()
|
|
26
|
+
req = urllib.request.Request(endpoint, data=body, method="POST", headers={
|
|
27
|
+
"Content-Type": "application/json", "Authorization": f"Bearer {key}",
|
|
28
|
+
"User-Agent": "fathom-read/0.1.0"})
|
|
29
|
+
try:
|
|
30
|
+
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
31
|
+
return Verdict.from_dict(json.loads(r.read().decode()))
|
|
32
|
+
except urllib.error.HTTPError as e:
|
|
33
|
+
msg = e.read().decode(errors="replace")
|
|
34
|
+
if e.code == 401:
|
|
35
|
+
raise ReadError("the read rejected the key; set FATHOM_API_KEY or request one at https://embeddedriskanalytics.com/contact.html") from None
|
|
36
|
+
if e.code == 429:
|
|
37
|
+
raise ReadError("the demo key is rate-limited; request your own at https://embeddedriskanalytics.com/contact.html") from None
|
|
38
|
+
raise ReadError(f"the read returned {e.code}: {msg[:200]}") from None
|
|
39
|
+
except urllib.error.URLError as e:
|
|
40
|
+
raise ReadError(f"could not reach the read at {endpoint}: {e.reason}") from None
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"type": "tool_usage_finished",
|
|
4
|
+
"tool_name": "write_record",
|
|
5
|
+
"tool_args": {
|
|
6
|
+
"record": "r0",
|
|
7
|
+
"content": "customer_id: 100"
|
|
8
|
+
}
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"type": "tool_usage_finished",
|
|
12
|
+
"tool_name": "write_record",
|
|
13
|
+
"tool_args": {
|
|
14
|
+
"record": "r1",
|
|
15
|
+
"content": "customer_id: 101"
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"type": "tool_usage_finished",
|
|
20
|
+
"tool_name": "write_record",
|
|
21
|
+
"tool_args": {
|
|
22
|
+
"record": "r2",
|
|
23
|
+
"content": "guest_id: 102"
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"type": "task_completed",
|
|
28
|
+
"task_name": "rename",
|
|
29
|
+
"output": "All records renamed to customer_id."
|
|
30
|
+
}
|
|
31
|
+
]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"op": "set",
|
|
4
|
+
"kind": "fact",
|
|
5
|
+
"key": "user.city",
|
|
6
|
+
"value": "Denver"
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"op": "set",
|
|
10
|
+
"kind": "fact",
|
|
11
|
+
"key": "user.city",
|
|
12
|
+
"value": "Austin"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"op": "set",
|
|
16
|
+
"kind": "fact",
|
|
17
|
+
"key": "user.role",
|
|
18
|
+
"value": "engineer"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"op": "answer",
|
|
22
|
+
"kind": "fact",
|
|
23
|
+
"key": "user.city",
|
|
24
|
+
"value": "The user lives in Denver."
|
|
25
|
+
}
|
|
26
|
+
]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"step": 2,
|
|
4
|
+
"values": {
|
|
5
|
+
"definition": "customer_id",
|
|
6
|
+
"records": [
|
|
7
|
+
{
|
|
8
|
+
"id": "a",
|
|
9
|
+
"cites": "customer_id"
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"id": "b",
|
|
13
|
+
"cites": "guest_id"
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"step": 1,
|
|
20
|
+
"values": {
|
|
21
|
+
"definition": "customer_id",
|
|
22
|
+
"records": [
|
|
23
|
+
{
|
|
24
|
+
"id": "a",
|
|
25
|
+
"cites": "customer_id"
|
|
26
|
+
}
|
|
27
|
+
]
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"step": 0,
|
|
32
|
+
"values": {
|
|
33
|
+
"definition": "guest_id",
|
|
34
|
+
"records": []
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
]
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"op": "add",
|
|
4
|
+
"kind": "order",
|
|
5
|
+
"key": "cart",
|
|
6
|
+
"value": "lamp"
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"op": "add",
|
|
10
|
+
"kind": "order",
|
|
11
|
+
"key": "cart",
|
|
12
|
+
"value": "chair"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"op": "add",
|
|
16
|
+
"kind": "order",
|
|
17
|
+
"key": "cart",
|
|
18
|
+
"value": "lamp"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"op": "commit",
|
|
22
|
+
"kind": "order",
|
|
23
|
+
"key": "order"
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"op": "add",
|
|
27
|
+
"kind": "order",
|
|
28
|
+
"key": "cart",
|
|
29
|
+
"value": "table"
|
|
30
|
+
}
|
|
31
|
+
]
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
{
|
|
2
|
+
"initial_files": {
|
|
3
|
+
"schema.py": "class Guest:\n guest_id: int\n name: str\n",
|
|
4
|
+
"models.py": "from schema import Guest\n\ndef load(guest_id):\n return Guest(guest_id=guest_id, name='x')\n",
|
|
5
|
+
"handler.py": "from models import load\n\ndef get(req):\n return load(req['guest_id'])\n",
|
|
6
|
+
"report.py": "def summarize(g):\n return f\"guest {g.guest_id}\"\n",
|
|
7
|
+
"test_app.py": "from handler import get\n\ndef test_get():\n assert get({'guest_id': 1}).guest_id == 1\n"
|
|
8
|
+
},
|
|
9
|
+
"edits": [
|
|
10
|
+
{
|
|
11
|
+
"tool": "str_replace_editor",
|
|
12
|
+
"args": {
|
|
13
|
+
"path": "schema.py",
|
|
14
|
+
"old_str": "guest_id: int",
|
|
15
|
+
"new_str": "customer_id: int"
|
|
16
|
+
},
|
|
17
|
+
"ok": true
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"tool": "str_replace_editor",
|
|
21
|
+
"args": {
|
|
22
|
+
"path": "models.py",
|
|
23
|
+
"old_str": "def load(guest_id):\n return Guest(guest_id=guest_id, name='x')",
|
|
24
|
+
"new_str": "def load(customer_id):\n return Guest(customer_id=customer_id, name='x')"
|
|
25
|
+
},
|
|
26
|
+
"ok": true
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"tool": "str_replace_editor",
|
|
30
|
+
"args": {
|
|
31
|
+
"path": "handler.py",
|
|
32
|
+
"old_str": "req['guest_id']",
|
|
33
|
+
"new_str": "req['customer_id']"
|
|
34
|
+
},
|
|
35
|
+
"ok": true
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"tool": "str_replace_editor",
|
|
39
|
+
"args": {
|
|
40
|
+
"path": "report.py",
|
|
41
|
+
"old_str": "guest {g.guest_id}",
|
|
42
|
+
"new_str": "customer {g.customer_id}"
|
|
43
|
+
},
|
|
44
|
+
"ok": true
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"tool": "str_replace_editor",
|
|
48
|
+
"args": {
|
|
49
|
+
"path": "test_app.py",
|
|
50
|
+
"old_str": "get({'guest_id': 1}).guest_id",
|
|
51
|
+
"new_str": "get({'customer_id': 1}).customer_id"
|
|
52
|
+
},
|
|
53
|
+
"ok": true
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"tool": "run_tests",
|
|
57
|
+
"args": {},
|
|
58
|
+
"ok": true,
|
|
59
|
+
"result": "5 passed"
|
|
60
|
+
}
|
|
61
|
+
],
|
|
62
|
+
"done": true
|
|
63
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
{
|
|
2
|
+
"initial_files": {
|
|
3
|
+
"schema.py": "class Guest:\n guest_id: int\n name: str\n",
|
|
4
|
+
"models.py": "from schema import Guest\n\ndef load(guest_id):\n return Guest(guest_id=guest_id, name='x')\n",
|
|
5
|
+
"handler.py": "from models import load\n\ndef get(req):\n return load(req['guest_id'])\n",
|
|
6
|
+
"report.py": "def summarize(g):\n return f\"guest {g.guest_id}\"\n",
|
|
7
|
+
"test_app.py": "from handler import get\n\ndef test_get():\n assert get({'guest_id': 1}).guest_id == 1\n"
|
|
8
|
+
},
|
|
9
|
+
"edits": [
|
|
10
|
+
{
|
|
11
|
+
"tool": "str_replace_editor",
|
|
12
|
+
"args": {
|
|
13
|
+
"path": "schema.py",
|
|
14
|
+
"old_str": "guest_id",
|
|
15
|
+
"new_str": "customer_id"
|
|
16
|
+
},
|
|
17
|
+
"ok": false
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"tool": "str_replace_editor",
|
|
21
|
+
"args": {
|
|
22
|
+
"path": "models.py",
|
|
23
|
+
"old_str": "guest_id",
|
|
24
|
+
"new_str": "customer_id"
|
|
25
|
+
},
|
|
26
|
+
"ok": false
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"tool": "str_replace_editor",
|
|
30
|
+
"args": {
|
|
31
|
+
"path": "handler.py",
|
|
32
|
+
"old_str": "guest_id",
|
|
33
|
+
"new_str": "customer_id"
|
|
34
|
+
},
|
|
35
|
+
"ok": false
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"tool": "str_replace_editor",
|
|
39
|
+
"args": {
|
|
40
|
+
"path": "schema.py",
|
|
41
|
+
"old_str": "guest_id",
|
|
42
|
+
"new_str": "customer_id"
|
|
43
|
+
},
|
|
44
|
+
"ok": false
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"tool": "str_replace_editor",
|
|
48
|
+
"args": {
|
|
49
|
+
"path": "models.py",
|
|
50
|
+
"old_str": "guest_id",
|
|
51
|
+
"new_str": "customer_id"
|
|
52
|
+
},
|
|
53
|
+
"ok": false
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"tool": "run_tests",
|
|
57
|
+
"args": {},
|
|
58
|
+
"ok": true,
|
|
59
|
+
"result": "1 passed"
|
|
60
|
+
}
|
|
61
|
+
],
|
|
62
|
+
"done": true
|
|
63
|
+
}
|
fathom_read/ops.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""The op stream: what an adapter produces and the hosted read consumes."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass, field, asdict
|
|
5
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
6
|
+
|
|
7
|
+
Ref = Tuple[str, str]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class Op:
|
|
12
|
+
"""One action the agent took.
|
|
13
|
+
|
|
14
|
+
op: "set" writes a value to (kind, key); "remove" deletes it; "rename" moves (kind, key)
|
|
15
|
+
to (kind, to); "add" inserts an entity into a collection under (kind, key); "answer"
|
|
16
|
+
is an assertion the agent made (a final answer, a report); "commit" marks a terminal
|
|
17
|
+
action (an order placed, a booking confirmed).
|
|
18
|
+
kind: what sort of thing the key names: "file", "record", "block", "fact", "order", or your own.
|
|
19
|
+
key: the name of the thing.
|
|
20
|
+
value: the content written (for set/add/answer).
|
|
21
|
+
to: the new name (for rename).
|
|
22
|
+
ok: whether the tool accepted the action. False makes the op a no-op.
|
|
23
|
+
refs: facts this op depends on, as (kind, key) pairs.
|
|
24
|
+
step: the position of the op in the stream (set by the reader if omitted).
|
|
25
|
+
source: where the op came from (an adapter's note).
|
|
26
|
+
"""
|
|
27
|
+
op: str
|
|
28
|
+
kind: str
|
|
29
|
+
key: str
|
|
30
|
+
value: Optional[str] = None
|
|
31
|
+
to: Optional[str] = None
|
|
32
|
+
ok: bool = True
|
|
33
|
+
refs: List[Ref] = field(default_factory=list)
|
|
34
|
+
step: Optional[int] = None
|
|
35
|
+
source: Optional[str] = None
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def from_dict(cls, d: Dict[str, Any]) -> "Op":
|
|
39
|
+
refs = [tuple(r) for r in d.get("refs", [])]
|
|
40
|
+
return cls(op=d["op"], kind=d.get("kind", "fact"), key=str(d["key"]),
|
|
41
|
+
value=None if d.get("value") is None else str(d.get("value")),
|
|
42
|
+
to=d.get("to"), ok=bool(d.get("ok", True)), refs=refs,
|
|
43
|
+
step=d.get("step"), source=d.get("source"))
|
|
44
|
+
|
|
45
|
+
def as_dict(self) -> Dict[str, Any]:
|
|
46
|
+
d = asdict(self)
|
|
47
|
+
d["refs"] = [list(r) for r in self.refs]
|
|
48
|
+
return d
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class Finding:
|
|
53
|
+
kind: str
|
|
54
|
+
step: Optional[int]
|
|
55
|
+
key: str
|
|
56
|
+
detail: str
|
|
57
|
+
cites: Optional[int] = None
|
|
58
|
+
|
|
59
|
+
def as_dict(self) -> Dict[str, Any]:
|
|
60
|
+
return asdict(self)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class Verdict:
|
|
65
|
+
coherent: bool
|
|
66
|
+
findings: List[Finding]
|
|
67
|
+
ops_read: int
|
|
68
|
+
ops_rejected: int
|
|
69
|
+
live_facts: int
|
|
70
|
+
|
|
71
|
+
@classmethod
|
|
72
|
+
def from_dict(cls, d: Dict[str, Any]) -> "Verdict":
|
|
73
|
+
return cls(coherent=bool(d["coherent"]), findings=[Finding(**f) for f in d.get("findings", [])],
|
|
74
|
+
ops_read=int(d.get("ops_read", 0)), ops_rejected=int(d.get("ops_rejected", 0)),
|
|
75
|
+
live_facts=int(d.get("live_facts", 0)))
|
|
76
|
+
|
|
77
|
+
def as_dict(self) -> Dict[str, Any]:
|
|
78
|
+
return {"coherent": self.coherent, "findings": [f.as_dict() for f in self.findings],
|
|
79
|
+
"ops_read": self.ops_read, "ops_rejected": self.ops_rejected, "live_facts": self.live_facts}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fathom-read
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Catch the step where an AI agent contradicts a decision it already made. Adapters and a CLI for the Fathom committed-state read over LangGraph, CrewAI, Letta, OpenInference, DBOS, and coding-agent traces.
|
|
5
|
+
Author-email: Embedded Risk Analytics <contact@embeddedriskanalytics.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 Peter Galligan, Embedded Risk Analytics
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://embeddedriskanalytics.com
|
|
29
|
+
Project-URL: Repository, https://github.com/ERA-Fathom/fathom
|
|
30
|
+
Project-URL: Research, https://embeddedriskanalytics.com/research.html
|
|
31
|
+
Keywords: ai-agents,llm,langgraph,crewai,letta,openinference,agent-memory,context-engineering,agent-evaluation,coherence
|
|
32
|
+
Classifier: Development Status :: 3 - Alpha
|
|
33
|
+
Classifier: Intended Audience :: Developers
|
|
34
|
+
Classifier: Programming Language :: Python :: 3
|
|
35
|
+
Classifier: Topic :: Software Development :: Testing
|
|
36
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
37
|
+
Requires-Python: >=3.9
|
|
38
|
+
Description-Content-Type: text/markdown
|
|
39
|
+
License-File: LICENSE
|
|
40
|
+
Provides-Extra: dev
|
|
41
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
|
|
44
|
+
# fathom-read
|
|
45
|
+
|
|
46
|
+
**Catch the step where an AI agent contradicts a decision it already made.**
|
|
47
|
+
|
|
48
|
+
On a long task, an agent loses track of what it already decided and starts acting against it. It renames `guest_id` to `customer_id` at step 1, then writes new code against `guest_id` at step 6. The change compiles, imports, and passes the tests. It fails at runtime.
|
|
49
|
+
|
|
50
|
+
`fathom-read` turns the traces your framework already records into an action stream and sends it to the Fathom read, which reconstructs the state the agent committed and flags the step that contradicts it. Deterministic. No model access. Nothing runs in your production path.
|
|
51
|
+
|
|
52
|
+

|
|
53
|
+
|
|
54
|
+
## Install
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
pip install fathom-read
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Run
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
fathom demo # the bundled rename example, coherent and not
|
|
64
|
+
fathom read trace.json --supersede guest_id=customer_id # your own trace
|
|
65
|
+
fathom read history.json --format langgraph # or name the format
|
|
66
|
+
fathom read trace.json --ops # see the action stream before anything is sent
|
|
67
|
+
fathom formats # the formats it reads
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`fathom read` exits 0 when the committed state is coherent and 2 when it finds a contradiction, so it drops into a test suite or a CI step as it is. Add `--json` for a machine-readable verdict.
|
|
71
|
+
|
|
72
|
+
The package ships with a demo key that is rate-limited per day. For your own key, which lifts the limit and keeps your traces on a private tier, write to [contact@embeddedriskanalytics.com](mailto:contact@embeddedriskanalytics.com?subject=fathom-read%20key) and set `FATHOM_API_KEY`. `--ops` shows exactly what would be sent: the ops the adapter produced, and nothing else.
|
|
73
|
+
|
|
74
|
+
## What it reads
|
|
75
|
+
|
|
76
|
+
| Format | What you export | How |
|
|
77
|
+
|---|---|---|
|
|
78
|
+
| `langgraph` | The checkpoint lineage | `[{"values": s.values, "step": s.metadata["step"]} for s in graph.get_state_history(config)]` |
|
|
79
|
+
| `openinference` | The spans Arize Phoenix stores | Export the trace's spans as JSON; only TOOL spans matter |
|
|
80
|
+
| `crewai` | The crew's event log | A listener on the event bus, capturing `tool_usage_finished`, `tool_usage_error`, `task_completed` |
|
|
81
|
+
| `letta` | Blocks, passages, and the memory-edit tool calls | `agents.blocks.list`, `agents.passages.list`, the tool calls from `agents.messages.list` |
|
|
82
|
+
| `dbos` | A workflow's step stream | `{"workflow_id": ..., "steps": [{"step_name", "args", "result", "ok"}]}` |
|
|
83
|
+
| `edits` | A coding agent's edit log | `{"initial_files": {...}, "edits": [{"tool": "str_replace_editor", "args": {...}, "ok": true}]}` |
|
|
84
|
+
| `events` | The native op stream | One op per line: `{"op": "set", "kind": "file", "key": "a.py", "value": "...", "ok": true}` |
|
|
85
|
+
|
|
86
|
+
Your tools have their own names. Map them once with `--map tools.json`:
|
|
87
|
+
|
|
88
|
+
```json
|
|
89
|
+
{"save_decision": {"op": "set", "kind": "decision", "key": "topic", "value": "text"},
|
|
90
|
+
"book_seat": {"op": "add", "kind": "flight", "key": "seats", "value": "seat"},
|
|
91
|
+
"confirm_booking": {"op": "commit", "kind": "flight", "key": "booking"}}
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## What it finds
|
|
95
|
+
|
|
96
|
+
| Finding | The agent... |
|
|
97
|
+
|---|---|
|
|
98
|
+
| `stale_reference` | acts on a fact it already removed or renamed away |
|
|
99
|
+
| `superseded_value` | writes or answers with a value it already replaced |
|
|
100
|
+
| `authored_contradiction` | reintroduces a token into a record it had already migrated |
|
|
101
|
+
| `residual` | ends the run with a record still carrying a value it replaced elsewhere |
|
|
102
|
+
| `duplicate_commit` | adds an entity a collection already holds |
|
|
103
|
+
| `post_commit_mutation` | changes a thing after committing it |
|
|
104
|
+
|
|
105
|
+
Every finding cites the earlier step it contradicts, so the readout is a diff between what the agent decided and what it did.
|
|
106
|
+
|
|
107
|
+
## How it reads
|
|
108
|
+
|
|
109
|
+
The read folds the agent's successful actions into a ledger of committed facts and checks every later action against the ledger. Two rules make this a reconstruction rather than a transcript. A failed action is a no-op: an edit the tool rejected leaves nothing behind. And the read consults only the agent's own actions and their results, never an answer key, so it attaches the same way on any framework. The adapters and the CLI in this repository build the action stream; the read itself runs in ERA's service.
|
|
110
|
+
|
|
111
|
+
## Use it from Python
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from fathom_read import Op, read
|
|
115
|
+
|
|
116
|
+
ops = [
|
|
117
|
+
Op("set", "fact", "user.city", value="Denver"),
|
|
118
|
+
Op("set", "fact", "user.city", value="Austin"),
|
|
119
|
+
Op("answer", "fact", "user.city", value="The user lives in Denver."),
|
|
120
|
+
]
|
|
121
|
+
verdict = read(ops) # uses FATHOM_API_KEY, or the demo key
|
|
122
|
+
for f in verdict.findings:
|
|
123
|
+
print(f.kind, f.step, f.detail)
|
|
124
|
+
# superseded_value 2 step 2 answers 'Denver' for fact 'user.city', a value the agent replaced with 'Austin' at step 1.
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## What it does not do
|
|
128
|
+
|
|
129
|
+
It does not run your agent, call a model, or need one. It does not say why the agent contradicted itself or which repair would fix it; that is the [design-partner engagement](https://embeddedriskanalytics.com/contact.html). It reads agents whose committed state lives in tool calls, checkpoints, memory writes, or edits; an agent that keeps state only in free-text logs is out of scope.
|
|
130
|
+
|
|
131
|
+
## Research
|
|
132
|
+
|
|
133
|
+
The read comes out of the Fathom program at [Embedded Risk Analytics](https://embeddedriskanalytics.com). Case studies on LangGraph, CrewAI, Letta, OpenHands, Agent-E, ContextPilot, and τ-bench are at [embeddedriskanalytics.com/research](https://embeddedriskanalytics.com/research.html). The theory is in [Records, Reflexive Modeling, and the Conditions for Stable Physical Histories](https://ssrn.com/abstract=6683578) (SSRN, 2026). See [CITATION.cff](CITATION.cff).
|
|
134
|
+
|
|
135
|
+
## Send us a trace
|
|
136
|
+
|
|
137
|
+
If you run long-horizon agents and want a readout on your own traces, send a batch: [embeddedriskanalytics.com/contact](https://embeddedriskanalytics.com/contact.html).
|
|
138
|
+
|
|
139
|
+
## License
|
|
140
|
+
|
|
141
|
+
MIT. Fathom is a trademark of Embedded Risk Analytics.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
fathom_read/__init__.py,sha256=yEJUkK3h000k1UQ00hbSflS-wkA9wfk8m1SEOK8W4WY,777
|
|
2
|
+
fathom_read/cli.py,sha256=jOHZ2voeziWLMa4MkvAn5kLdKqi2nwFOYyABngOT3tU,4902
|
|
3
|
+
fathom_read/client.py,sha256=zTu7_kEqhBxwX_56OUqda0gVeJf65pbrbgCKAp9K5NQ,1962
|
|
4
|
+
fathom_read/ops.py,sha256=CWCnTHjtomHNwPsV0or175GB_FcmSMw4BYky_LJAJMQ,2857
|
|
5
|
+
fathom_read/adapters/__init__.py,sha256=L7KKRqmBoQ1h3KEmp7TwTP2VVtVGMguBayxMSUc3bho,1811
|
|
6
|
+
fathom_read/adapters/_tools.py,sha256=NQBgpgJSXNOjy5cBhps85lJh8p3IpJ65_WnsMmLQqH4,4721
|
|
7
|
+
fathom_read/adapters/crewai.py,sha256=mRowA7tvpJ7nZWi9xwNM4jldhYiKPl1OPo2vLdOQbzU,1620
|
|
8
|
+
fathom_read/adapters/dbos.py,sha256=URURGonPjOFmDDXM46z9mRSBoqNBDRcQySxraGPOlH0,919
|
|
9
|
+
fathom_read/adapters/edits.py,sha256=fgAiSf5kUCJCzg0afMZXurbwfruOXny4R0PHVCL6pY8,2259
|
|
10
|
+
fathom_read/adapters/events.py,sha256=cK_pm3kBidXBhVj5CCmam9Z-9FivI7KJNz415AyJCrA,476
|
|
11
|
+
fathom_read/adapters/langgraph.py,sha256=SeayEjcbKx8IPGZK0_6URj4TgBEftZvngFbq1GMKcBI,2411
|
|
12
|
+
fathom_read/adapters/letta.py,sha256=efZT-iHEaDLgEJIrewP1pCE34W0XKt1rFloY-XH9o4s,1409
|
|
13
|
+
fathom_read/adapters/openinference.py,sha256=RMAuPLaeQ1WeAlO6odHKPXpbrxSfAMexyCAThVK12J4,2270
|
|
14
|
+
fathom_read/examples/crewai_events.json,sha256=8riBzsiF2F_vttHB8jFgxeDmU-oSvsfYTTFtMLLlueo,541
|
|
15
|
+
fathom_read/examples/knowledge_update.json,sha256=0vNIjQNyx-5V7CNQaDgE8vk-oQ5oO9ww7I2p5X6gp_U,354
|
|
16
|
+
fathom_read/examples/langgraph_history.json,sha256=5OFNIwOMMryZU5L7rOJDFl-mQJZr4MydiTTCmTUCrfQ,434
|
|
17
|
+
fathom_read/examples/order_duplicate.json,sha256=FFxJWsBG0LFVejcDfbQqCtxv2mMbmPQIojbotWrfky4,369
|
|
18
|
+
fathom_read/examples/rename_coherent.json,sha256=isolVJJCFTDICv1anPmqAt0l0WntWM3RqjwpvS4JN4c,1603
|
|
19
|
+
fathom_read/examples/rename_starved.json,sha256=f2wdknFoZtC1f8i6yxOTx8rONQVW1hzOFKifiqaFJmQ,1392
|
|
20
|
+
fathom_read-0.1.0.dist-info/licenses/LICENSE,sha256=FP92AThBYP1kWdoqHHyhJUxdYnXrFkfB86rt6HbvDmo,1096
|
|
21
|
+
fathom_read-0.1.0.dist-info/METADATA,sha256=0bhD8oDDJCV_jR5CyrYESmWH5iY4uDmKPzpjPimr_1Y,8258
|
|
22
|
+
fathom_read-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
23
|
+
fathom_read-0.1.0.dist-info/entry_points.txt,sha256=VMyOrIeBi3IumcxLuwmPEceYZOeGXan1Y4dYoUS8wh0,48
|
|
24
|
+
fathom_read-0.1.0.dist-info/top_level.txt,sha256=EWItbJ3VlJ7GJ9mx7pRCTK4eN2GQjAlEa6tixFHKvt4,12
|
|
25
|
+
fathom_read-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Peter Galligan, Embedded Risk Analytics
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fathom_read
|