onetrace-explain 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- onetrace_explain-0.1.0/MANIFEST.in +1 -0
- onetrace_explain-0.1.0/PKG-INFO +50 -0
- onetrace_explain-0.1.0/README.md +34 -0
- onetrace_explain-0.1.0/_reproducible_sdist.py +61 -0
- onetrace_explain-0.1.0/pyproject.toml +33 -0
- onetrace_explain-0.1.0/setup.cfg +4 -0
- onetrace_explain-0.1.0/src/onetrace_explain/__init__.py +9 -0
- onetrace_explain-0.1.0/src/onetrace_explain/__main__.py +5 -0
- onetrace_explain-0.1.0/src/onetrace_explain/cli.py +167 -0
- onetrace_explain-0.1.0/src/onetrace_explain/packet.py +200 -0
- onetrace_explain-0.1.0/src/onetrace_explain/postcheck.py +168 -0
- onetrace_explain-0.1.0/src/onetrace_explain/providers.py +128 -0
- onetrace_explain-0.1.0/src/onetrace_explain/record.py +70 -0
- onetrace_explain-0.1.0/src/onetrace_explain.egg-info/PKG-INFO +50 -0
- onetrace_explain-0.1.0/src/onetrace_explain.egg-info/SOURCES.txt +17 -0
- onetrace_explain-0.1.0/src/onetrace_explain.egg-info/dependency_links.txt +1 -0
- onetrace_explain-0.1.0/src/onetrace_explain.egg-info/entry_points.txt +2 -0
- onetrace_explain-0.1.0/src/onetrace_explain.egg-info/requires.txt +1 -0
- onetrace_explain-0.1.0/src/onetrace_explain.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
include _reproducible_sdist.py
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: onetrace-explain
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A suggested explanation of a onetrace comparison: AI-generated, checked against the record, never a verdict
|
|
5
|
+
Author-email: Shamik Saha <shamik.saha.rcciit@gmail.com>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Issues, https://github.com/shamiksaharcciit-oss/onetrace-issues
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
Requires-Dist: onetrace<0.5,>=0.4
|
|
16
|
+
|
|
17
|
+
# onetrace-explain
|
|
18
|
+
|
|
19
|
+
A suggested explanation of a onetrace comparison: **AI-generated, may be wrong**, and checked
|
|
20
|
+
against the record before anyone reads it. Never a verdict: onetrace's verdicts and exit codes are
|
|
21
|
+
what they are without it, and its exit codes say nothing about the runs. Don't use it as a CI gate.
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
onetrace localize runs/a runs/b --for answer --out report
|
|
25
|
+
onetrace-explain report --dry-run # the packet exactly as it would be sent; nothing sent
|
|
26
|
+
onetrace-explain report --model anthropic:<model name>
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
- **What is sent:** a packet built from the report and the two runs' receipts: verdicts, stage
|
|
30
|
+
names and edges, outcome classes, digests and sizes, instrument names and versions, and the
|
|
31
|
+
constants that differ with both values (`--redact NAME` sends a name and digest only). Kept text
|
|
32
|
+
only with `--include-text`. A constant's value goes as the run recorded it, a path or a token
|
|
33
|
+
included; `--redact NAME` keeps it out. Never this machine's own paths, an environment variable,
|
|
34
|
+
the model key, a signing key or an anchor's proof. The first send of a kind of item to an endpoint asks first (`--yes` for scripts).
|
|
35
|
+
- **The model:** yours, named in `~/.config/onetrace/explain.toml`:
|
|
36
|
+
|
|
37
|
+
```toml
|
|
38
|
+
[model]
|
|
39
|
+
provider = "anthropic" # or "openai" (any OpenAI-compatible endpoint, local ones too)
|
|
40
|
+
name = "<model name>"
|
|
41
|
+
key_env = "ANTHROPIC_API_KEY" # the variable holding the key; the key is never written anywhere
|
|
42
|
+
# base_url = "http://localhost:8000/v1" # for an openai-compatible server
|
|
43
|
+
```
|
|
44
|
+
- **The post-check,** with no model: a claim is dropped when it cites nothing, cites what the packet
|
|
45
|
+
doesn't hold, quotes a value its cites don't hold, or says what caused a difference. What's shown
|
|
46
|
+
says how many were dropped, and why.
|
|
47
|
+
- **Checks, not conclusions:** each suggestion comes with onetrace commands to run. A lead is
|
|
48
|
+
confirmed or ruled out only by what those report.
|
|
49
|
+
- **Recorded:** every explanation is a onetrace run of its own, under `<out>/explain/<run id>/`,
|
|
50
|
+
which `onetrace-verify` checks like any run.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# onetrace-explain
|
|
2
|
+
|
|
3
|
+
A suggested explanation of a onetrace comparison: **AI-generated, may be wrong**, and checked
|
|
4
|
+
against the record before anyone reads it. Never a verdict: onetrace's verdicts and exit codes are
|
|
5
|
+
what they are without it, and its exit codes say nothing about the runs. Don't use it as a CI gate.
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
onetrace localize runs/a runs/b --for answer --out report
|
|
9
|
+
onetrace-explain report --dry-run # the packet exactly as it would be sent; nothing sent
|
|
10
|
+
onetrace-explain report --model anthropic:<model name>
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
- **What is sent:** a packet built from the report and the two runs' receipts: verdicts, stage
|
|
14
|
+
names and edges, outcome classes, digests and sizes, instrument names and versions, and the
|
|
15
|
+
constants that differ with both values (`--redact NAME` sends a name and digest only). Kept text
|
|
16
|
+
only with `--include-text`. A constant's value goes as the run recorded it, a path or a token
|
|
17
|
+
included; `--redact NAME` keeps it out. Never this machine's own paths, an environment variable,
|
|
18
|
+
the model key, a signing key or an anchor's proof. The first send of a kind of item to an endpoint asks first (`--yes` for scripts).
|
|
19
|
+
- **The model:** yours, named in `~/.config/onetrace/explain.toml`:
|
|
20
|
+
|
|
21
|
+
```toml
|
|
22
|
+
[model]
|
|
23
|
+
provider = "anthropic" # or "openai" (any OpenAI-compatible endpoint, local ones too)
|
|
24
|
+
name = "<model name>"
|
|
25
|
+
key_env = "ANTHROPIC_API_KEY" # the variable holding the key; the key is never written anywhere
|
|
26
|
+
# base_url = "http://localhost:8000/v1" # for an openai-compatible server
|
|
27
|
+
```
|
|
28
|
+
- **The post-check,** with no model: a claim is dropped when it cites nothing, cites what the packet
|
|
29
|
+
doesn't hold, quotes a value its cites don't hold, or says what caused a difference. What's shown
|
|
30
|
+
says how many were dropped, and why.
|
|
31
|
+
- **Checks, not conclusions:** each suggestion comes with onetrace commands to run. A lead is
|
|
32
|
+
confirmed or ruled out only by what those report.
|
|
33
|
+
- **Recorded:** every explanation is a onetrace run of its own, under `<out>/explain/<run id>/`,
|
|
34
|
+
which `onetrace-verify` checks like any run.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""The build backend: setuptools' own (setuptools.build_meta), with its sdist archived so that two
|
|
2
|
+
builds of the same tree are the same bytes. Wheels are built exactly as setuptools builds them.
|
|
3
|
+
|
|
4
|
+
setuptools writes each sdist member with the file's mtime and owner, and the gzip header with the
|
|
5
|
+
build clock, so two builds of one tree differ in those bytes alone. Here the release tree setuptools
|
|
6
|
+
has already made is archived in sorted order, every member's time set to SOURCE_DATE_EPOCH (0 when it
|
|
7
|
+
is unset), owner and group 0 with no names, modes normalised (0644 for files, 0755 for directories
|
|
8
|
+
and executables), and the gzip header's time set the same way. The members and their contents are
|
|
9
|
+
setuptools' own. It runs inside the build, before the archive exists; nothing is repacked afterwards.
|
|
10
|
+
"""
|
|
11
|
+
import gzip
|
|
12
|
+
import os
|
|
13
|
+
import tarfile
|
|
14
|
+
|
|
15
|
+
from setuptools.command import sdist as _sdist_module
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _epoch() -> int:
|
|
19
|
+
v = os.environ.get("SOURCE_DATE_EPOCH", "")
|
|
20
|
+
return int(v) if v.isdigit() else 0
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
_setuptools_make_archive = _sdist_module.sdist.make_archive
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _make_archive(self, base_name, format, root_dir=None, base_dir=None, owner=None, group=None):
|
|
27
|
+
if format != "gztar":
|
|
28
|
+
return _setuptools_make_archive(self, base_name, format, root_dir, base_dir, owner, group)
|
|
29
|
+
when = _epoch()
|
|
30
|
+
root = os.path.abspath(root_dir or os.curdir)
|
|
31
|
+
top = base_dir or os.curdir
|
|
32
|
+
paths = []
|
|
33
|
+
for d, dirs, files in os.walk(os.path.join(root, top)):
|
|
34
|
+
dirs.sort()
|
|
35
|
+
rel_d = os.path.relpath(d, root)
|
|
36
|
+
paths.append(rel_d)
|
|
37
|
+
paths.extend(os.path.join(rel_d, f) for f in sorted(files))
|
|
38
|
+
target = os.path.abspath(base_name + ".tar.gz")
|
|
39
|
+
|
|
40
|
+
def norm(ti):
|
|
41
|
+
ti.mtime = when
|
|
42
|
+
ti.uid = ti.gid = 0
|
|
43
|
+
ti.uname = ti.gname = ""
|
|
44
|
+
if ti.isdir():
|
|
45
|
+
ti.mode = 0o755
|
|
46
|
+
elif ti.isfile():
|
|
47
|
+
ti.mode = 0o755 if ti.mode & 0o111 else 0o644
|
|
48
|
+
return ti
|
|
49
|
+
|
|
50
|
+
with open(target, "wb") as raw, \
|
|
51
|
+
gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=when, compresslevel=9) as gz, \
|
|
52
|
+
tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as tar:
|
|
53
|
+
for rel in paths:
|
|
54
|
+
tar.add(os.path.join(root, rel), arcname=os.path.normpath(rel), recursive=False, filter=norm)
|
|
55
|
+
return target
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
_sdist_module.sdist.make_archive = _make_archive
|
|
59
|
+
|
|
60
|
+
from setuptools.build_meta import * # noqa: E402,F401,F403 -- the hooks, unchanged
|
|
61
|
+
from setuptools.build_meta import __all__ # noqa: E402,F401
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
# setuptools' own backend, with its sdist archived reproducibly (_reproducible_sdist.py says how).
|
|
4
|
+
build-backend = "_reproducible_sdist"
|
|
5
|
+
backend-path = ["."]
|
|
6
|
+
|
|
7
|
+
[project]
|
|
8
|
+
name = "onetrace-explain"
|
|
9
|
+
version = "0.1.0"
|
|
10
|
+
description = "A suggested explanation of a onetrace comparison: AI-generated, checked against the record, never a verdict"
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
dependencies = ["onetrace>=0.4,<0.5"]
|
|
14
|
+
license = "Apache-2.0"
|
|
15
|
+
authors = [
|
|
16
|
+
{name = "Shamik Saha", email = "shamik.saha.rcciit@gmail.com"},
|
|
17
|
+
]
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Development Status :: 3 - Alpha",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Issues = "https://github.com/shamiksaharcciit-oss/onetrace-issues"
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
onetrace-explain = "onetrace_explain.cli:main"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages.find]
|
|
33
|
+
where = ["src"]
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""onetrace-explain: a suggested explanation of a onetrace comparison, AI-generated and checked
|
|
2
|
+
against the record. A separate package, installed on purpose: onetrace, onetrace-verify and the
|
|
3
|
+
console neither depend on it nor import it."""
|
|
4
|
+
import importlib.metadata
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
__version__ = importlib.metadata.version("onetrace-explain")
|
|
8
|
+
except importlib.metadata.PackageNotFoundError: # run from its source tree
|
|
9
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""`onetrace-explain REPORT_DIR [--model PROVIDER:NAME] [--dry-run] [--include-text] [--redact NAME]...
|
|
2
|
+
[--yes] [--out DIR]`: a suggested explanation of a comparison, AI-generated and checked against the
|
|
3
|
+
record. Never a verdict: onetrace's verdicts and exit codes are what they are without it.
|
|
4
|
+
|
|
5
|
+
Exit codes are this command's own and say nothing about the runs: 0 an explanation or a dry run was
|
|
6
|
+
written; 2 a usage error or the comparison can't be explained; 3 the model couldn't be reached or
|
|
7
|
+
replied with no usable JSON. Never use it as a CI gate.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import sys
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from . import packet as pk
|
|
18
|
+
from . import postcheck, providers
|
|
19
|
+
from .record import record
|
|
20
|
+
|
|
21
|
+
LABEL = "Suggested explanation — AI-generated, may be wrong"
|
|
22
|
+
SENT = providers.CONFIG.parent / "explain-sent.json"
|
|
23
|
+
WORDS = {"lead": "possible culprit", "ruled out": "ruled out", "note": "note"}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _sent() -> dict:
|
|
27
|
+
try:
|
|
28
|
+
return json.loads(SENT.read_text(encoding="utf-8"))
|
|
29
|
+
except (OSError, ValueError):
|
|
30
|
+
return {}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _remember(endpoint: str, kinds) -> None:
|
|
34
|
+
got = _sent()
|
|
35
|
+
got[endpoint] = sorted(set(got.get(endpoint, [])) | set(kinds))
|
|
36
|
+
SENT.parent.mkdir(parents=True, exist_ok=True)
|
|
37
|
+
SENT.write_text(json.dumps(got, indent=1, sort_keys=True), encoding="utf-8")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _command(check: dict, runs: dict) -> str:
|
|
41
|
+
a, b = runs["baseline"], runs["candidate"]
|
|
42
|
+
c = check["check"]
|
|
43
|
+
if c == "text":
|
|
44
|
+
return f"onetrace diff {a} {b} --text --stage {check['node']}"
|
|
45
|
+
if c == "for":
|
|
46
|
+
return f"onetrace localize {a} {b} --for {check['node']}"
|
|
47
|
+
if c == "mask":
|
|
48
|
+
return f"onetrace diff {a} {b} --mask <a mask naming {check['setting']}>"
|
|
49
|
+
if c == "reproduce":
|
|
50
|
+
return f"onetrace reproduce {runs[check['run']]} {check['node']} --runner <your runner>"
|
|
51
|
+
return f"onetrace match {runs[check['run']]}"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _fixes(claim: dict, items: dict) -> list:
|
|
55
|
+
"""For a lead citing a recorded setting or version that changed, the one suggestion version 1
|
|
56
|
+
makes: set it back and run again. Built from the cited items, never from the model's words."""
|
|
57
|
+
out = []
|
|
58
|
+
for cid in claim["cites"]:
|
|
59
|
+
it = items[cid]
|
|
60
|
+
node = cid.split("/")[1] if cid.startswith("node/") else None
|
|
61
|
+
if it["kind"] == "constant" and not it.get("redacted"):
|
|
62
|
+
out.append(f"`{it['name']}` changed from `{it['baseline']}` to `{it['candidate']}` at `{node}`; "
|
|
63
|
+
f"set it back and run again to check")
|
|
64
|
+
elif it["kind"] == "instrument":
|
|
65
|
+
va, vb = (it.get("baseline") or {}).get("version"), (it.get("candidate") or {}).get("version")
|
|
66
|
+
if va != vb:
|
|
67
|
+
out.append(f"`{(it.get('candidate') or {}).get('id')}` version changed from `{va}` to `{vb}` at "
|
|
68
|
+
f"`{node}`; set it back and run again to check")
|
|
69
|
+
return out
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def text(model_label: str, checked: dict, packet: dict, runs: dict) -> str:
|
|
73
|
+
items = pk.by_id(packet)
|
|
74
|
+
L = [LABEL, f"model: {model_label}", ""]
|
|
75
|
+
if not checked["claims"]:
|
|
76
|
+
L.append("No suggestion could be tied to the record.")
|
|
77
|
+
for c in checked["claims"]:
|
|
78
|
+
L.append(f"- {WORDS[c['kind']]}: {c['text']}")
|
|
79
|
+
L.append(f" shown by: {', '.join(c['cites'])}")
|
|
80
|
+
if c["kind"] == "lead":
|
|
81
|
+
for f in _fixes(c, items):
|
|
82
|
+
L.append(f" to check: {f}")
|
|
83
|
+
if checked["dropped"]:
|
|
84
|
+
L.append("")
|
|
85
|
+
L.append("dropped: " + "; ".join(f"{n} {why}" for why, n in sorted(checked["dropped"].items())))
|
|
86
|
+
if checked["checks"]:
|
|
87
|
+
L.append("")
|
|
88
|
+
L.append("checks onetrace can run (a lead is confirmed or ruled out only by what these report):")
|
|
89
|
+
for k in checked["checks"]:
|
|
90
|
+
L.append(f" {_command(k, runs)}")
|
|
91
|
+
if checked["checks_dropped"]:
|
|
92
|
+
L.append("checks dropped: " + "; ".join(f"{n} {why}" for why, n in sorted(checked["checks_dropped"].items())))
|
|
93
|
+
return "\n".join(L) + "\n"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def main(argv=None) -> int:
|
|
97
|
+
ap = argparse.ArgumentParser(prog="onetrace-explain", description=__doc__.split("\n\n")[0])
|
|
98
|
+
ap.add_argument("report_dir")
|
|
99
|
+
ap.add_argument("--model")
|
|
100
|
+
ap.add_argument("--dry-run", action="store_true")
|
|
101
|
+
ap.add_argument("--include-text", action="store_true")
|
|
102
|
+
ap.add_argument("--redact", action="append", default=[])
|
|
103
|
+
ap.add_argument("--yes", action="store_true")
|
|
104
|
+
ap.add_argument("--out")
|
|
105
|
+
try:
|
|
106
|
+
args = ap.parse_args(argv)
|
|
107
|
+
except SystemExit as e:
|
|
108
|
+
return 2 if e.code else 0
|
|
109
|
+
folder = Path(args.report_dir)
|
|
110
|
+
try:
|
|
111
|
+
packet = pk.build(folder, include_text=args.include_text, redact=args.redact)
|
|
112
|
+
except pk.PacketError as e:
|
|
113
|
+
print(f"onetrace-explain: {e}", file=sys.stderr)
|
|
114
|
+
return 2
|
|
115
|
+
raw = pk.dumps(packet)
|
|
116
|
+
if args.dry_run:
|
|
117
|
+
sys.stdout.buffer.write(raw + b"\n")
|
|
118
|
+
print(f"{len(raw)} bytes, sha256 {hashlib.sha256(raw).hexdigest()}; dry run: nothing sent")
|
|
119
|
+
return 0
|
|
120
|
+
try:
|
|
121
|
+
model = providers.load(args.model)
|
|
122
|
+
except providers.ModelUnavailable as e:
|
|
123
|
+
print(f"onetrace-explain: {e}", file=sys.stderr)
|
|
124
|
+
return 2
|
|
125
|
+
new = sorted(set(packet["kinds"]) - set(_sent().get(model.endpoint, [])))
|
|
126
|
+
if new and model.provider != "fake":
|
|
127
|
+
print(f"onetrace-explain: this sends to {model.endpoint} kinds of item it never sent there before: "
|
|
128
|
+
f"{', '.join(new)}")
|
|
129
|
+
if "constant" in new and any(i.get("kind") == "constant" and not i.get("redacted")
|
|
130
|
+
for i in packet.get("items") or []):
|
|
131
|
+
print("onetrace-explain: constants go with their values as the runs recorded them, whatever "
|
|
132
|
+
"they hold; --redact NAME sends a constant's name and digests only")
|
|
133
|
+
if not args.yes:
|
|
134
|
+
if not sys.stdin.isatty():
|
|
135
|
+
print("onetrace-explain: not confirmed; run it in a terminal, or pass --yes", file=sys.stderr)
|
|
136
|
+
return 2
|
|
137
|
+
if input("Send? [y/N] ").strip().lower() not in ("y", "yes"):
|
|
138
|
+
print("onetrace-explain: nothing sent")
|
|
139
|
+
return 2
|
|
140
|
+
rep = json.loads((folder / packet["report"]["name"]).read_bytes())
|
|
141
|
+
runs = {s: rep["inputs"][s]["path"] for s in ("baseline", "candidate")}
|
|
142
|
+
|
|
143
|
+
def check(reply: bytes) -> dict:
|
|
144
|
+
return postcheck.check(packet, postcheck.parse(reply.decode("utf-8", "replace")))
|
|
145
|
+
out = Path(args.out) if args.out else folder
|
|
146
|
+
try:
|
|
147
|
+
run, checked = record(out, folder, packet, raw, model, model.ask, check)
|
|
148
|
+
except providers.ModelUnavailable as e:
|
|
149
|
+
print(f"onetrace-explain: the model couldn't be reached: {e}; no explanation written", file=sys.stderr)
|
|
150
|
+
return 3
|
|
151
|
+
except postcheck.ReplyError as e:
|
|
152
|
+
print(f"onetrace-explain: {e}; no explanation written", file=sys.stderr)
|
|
153
|
+
return 3
|
|
154
|
+
if model.provider != "fake":
|
|
155
|
+
_remember(model.endpoint, packet["kinds"])
|
|
156
|
+
shown = text(model.label, checked, packet, runs)
|
|
157
|
+
(run.parent / f"{run.name}.explanation.json").write_text(
|
|
158
|
+
json.dumps({"label": LABEL, "model": model.label, "record": run.name, **checked}, indent=1, sort_keys=True),
|
|
159
|
+
encoding="utf-8")
|
|
160
|
+
(run.parent / f"{run.name}.explanation.txt").write_text(shown, encoding="utf-8")
|
|
161
|
+
sys.stdout.write(shown)
|
|
162
|
+
print(f"recorded: {run}")
|
|
163
|
+
return 0
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
if __name__ == "__main__":
|
|
167
|
+
sys.exit(main())
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""The evidence packet, `onetrace-explain-evidence/0.1`: what a model is shown, and all it is shown.
|
|
2
|
+
|
|
3
|
+
Built deterministically from what a `diff` or `localize` run wrote in its report folder and the two
|
|
4
|
+
runs that report names: the same report gives the same bytes. Every item carries an `id`, the
|
|
5
|
+
place it came from, which is what a claim must cite. By default the packet holds the verdicts,
|
|
6
|
+
stage names and edges, outcome classes, digests and byte counts, instrument names and versions,
|
|
7
|
+
and the constants that differ with both values. A constant's value goes as the run recorded it,
|
|
8
|
+
whatever it holds (a path, a token): `redact` sends that constant's name and digests only. The same
|
|
9
|
+
holds for the report's annotations, which may name an input's folders. Nothing is read from this
|
|
10
|
+
machine beyond the report and the two runs: no path of its own folders, no environment variable, no
|
|
11
|
+
model key, no signing key, no anchor's proof. Kept text goes in only with `include_text=True`.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
FORMAT = "onetrace-explain-evidence/0.1"
|
|
20
|
+
REPORTS = ("localize.json", "diff.json")
|
|
21
|
+
TEXT_TYPES = ("text/", "application/json", "application/x-ndjson", "application/jsonl")
|
|
22
|
+
TEXT_CAP = 8192 # bytes of kept text per output and run, when asked for
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class PacketError(ValueError):
|
|
26
|
+
"""The report folder can't be explained: no report, a run that changed since it, or a report
|
|
27
|
+
whose verdict was not reached on checked records."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def dumps(obj) -> bytes:
|
|
31
|
+
"""The packet's bytes: sorted keys, no spaces, UTF-8. Built twice from one report, the same."""
|
|
32
|
+
return json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _sha(raw: bytes) -> str:
|
|
36
|
+
return "sha256:" + hashlib.sha256(raw).hexdigest()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _report(folder: Path):
|
|
40
|
+
for name in REPORTS:
|
|
41
|
+
p = folder / name
|
|
42
|
+
if p.is_file():
|
|
43
|
+
raw = p.read_bytes()
|
|
44
|
+
return name, raw, json.loads(raw)
|
|
45
|
+
raise PacketError(f"{folder}: no localize.json or diff.json; run onetrace diff or localize with --out first")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _receipts(run: Path) -> tuple[dict, list]:
|
|
49
|
+
man_raw = (run / "MANIFEST.json").read_bytes()
|
|
50
|
+
man = json.loads(man_raw)
|
|
51
|
+
out = []
|
|
52
|
+
for e in man.get("chain") or []:
|
|
53
|
+
r = json.loads((run / e["file"]).read_bytes())
|
|
54
|
+
out.append((Path(e["file"]).stem, r))
|
|
55
|
+
return man, out
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _node_id(r: dict) -> str:
|
|
59
|
+
st = r.get("stage") or {}
|
|
60
|
+
return st.get("name", "") + ("#" + st["instance"] if st.get("instance") else "")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _files(items) -> dict:
|
|
64
|
+
return {i.get("name"): i for i in items or [] if isinstance(i, dict) and isinstance(i.get("name"), str)}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _kept_text(run: Path, stem: str, item: dict):
|
|
68
|
+
p = run / "artifacts" / stem / str(item.get("name"))
|
|
69
|
+
if not p.is_file():
|
|
70
|
+
return None
|
|
71
|
+
raw = p.read_bytes()
|
|
72
|
+
if _sha(raw) != item.get("digest") or not str(item.get("media_type", "")).startswith(TEXT_TYPES):
|
|
73
|
+
return None
|
|
74
|
+
try:
|
|
75
|
+
text = raw[:TEXT_CAP].decode("utf-8")
|
|
76
|
+
except UnicodeDecodeError:
|
|
77
|
+
return None
|
|
78
|
+
return text + ("" if len(raw) <= TEXT_CAP else f"\n[... {len(raw) - TEXT_CAP} more bytes not sent]")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def build(report_dir, include_text: bool = False, redact=()) -> dict:
|
|
82
|
+
"""The packet for the comparison in `report_dir`. Raises PacketError when it can't be built."""
|
|
83
|
+
folder = Path(report_dir)
|
|
84
|
+
rname, rraw, rep = _report(folder)
|
|
85
|
+
if rep.get("envelope") in ("refused", "could not check") or "ladder" not in rep:
|
|
86
|
+
raise PacketError(f"{folder}: the report's verdict is {rep.get('envelope')!r}: there is no checked "
|
|
87
|
+
f"comparison to explain")
|
|
88
|
+
runs, receipts = {}, {}
|
|
89
|
+
for side in ("baseline", "candidate"):
|
|
90
|
+
inp = (rep.get("inputs") or {}).get(side) or {}
|
|
91
|
+
run = Path(inp.get("path", ""))
|
|
92
|
+
try:
|
|
93
|
+
man, recs = _receipts(run)
|
|
94
|
+
except (OSError, ValueError) as e:
|
|
95
|
+
raise PacketError(f"{side} run {run}: can't be read ({type(e).__name__})") from None
|
|
96
|
+
want = (inp.get("record_digests") or {}).get("chain_head")
|
|
97
|
+
if want is not None and man.get("chain_head") != want:
|
|
98
|
+
raise PacketError(f"{side} run {run}: its chain head is not the one the report checked; run the "
|
|
99
|
+
f"comparison again")
|
|
100
|
+
runs[side] = {"run_id": inp.get("run_id"), "chain_head": man.get("chain_head"),
|
|
101
|
+
"format": man.get("format"), "path": run}
|
|
102
|
+
receipts[side] = {_node_id(r): (stem, r) for stem, r in recs}
|
|
103
|
+
redact = set(redact or ())
|
|
104
|
+
items, kinds = [], set()
|
|
105
|
+
|
|
106
|
+
def add(kind, id_, **body):
|
|
107
|
+
kinds.add(kind)
|
|
108
|
+
items.append({"id": id_, "kind": kind, **body})
|
|
109
|
+
|
|
110
|
+
add("verdict", "comparison/verdict", verb=rep.get("verb"), envelope=rep.get("envelope"), exit=rep.get("exit"),
|
|
111
|
+
first_difference=list(rep.get("first_difference_frontier") or
|
|
112
|
+
([rep["first_difference"]["stage"]] if isinstance(rep.get("first_difference"), dict) else [])))
|
|
113
|
+
if isinstance(rep.get("for"), dict):
|
|
114
|
+
f = rep["for"]
|
|
115
|
+
add("for", "comparison/for", node=f.get("stage"), result=f.get("result"),
|
|
116
|
+
origins=list(f.get("origins") or []), not_reaching=list(f.get("not_reaching") or []),
|
|
117
|
+
path=[{"node": p.get("stage"), "verdict": p.get("verdict")}
|
|
118
|
+
for p in f.get("path") or [] if isinstance(p, dict)])
|
|
119
|
+
if rep.get("end_outputs"):
|
|
120
|
+
add("end_outputs", "comparison/end_outputs", outputs=rep["end_outputs"])
|
|
121
|
+
if rep.get("inputs_differ"):
|
|
122
|
+
add("inputs_differ", "comparison/inputs_differ", inputs_differ=rep["inputs_differ"],
|
|
123
|
+
on_different_inputs=rep.get("on_different_inputs", "false"))
|
|
124
|
+
if isinstance(rep.get("mask"), dict):
|
|
125
|
+
add("mask", "comparison/mask", sha256=rep["mask"].get("sha256"),
|
|
126
|
+
settings_masked=len(rep.get("settings_masked") or []))
|
|
127
|
+
for row in rep.get("ladder") or []:
|
|
128
|
+
node = row.get("node") or row.get("stage")
|
|
129
|
+
if not isinstance(node, str):
|
|
130
|
+
continue
|
|
131
|
+
pre = f"node/{node}"
|
|
132
|
+
add("node", pre, node=node, verdict=row.get("verdict"), annotations=list(row.get("annotations") or []))
|
|
133
|
+
ra = (receipts["baseline"].get(node) or (None, None))
|
|
134
|
+
rb = (receipts["candidate"].get(node) or (None, None))
|
|
135
|
+
a, b = ra[1] or {}, rb[1] or {}
|
|
136
|
+
ia, ib = a.get("instrument") or {}, b.get("instrument") or {}
|
|
137
|
+
pick = lambda i: {k: i.get(k) for k in ("id", "kind", "version", "config_digest") if k in i}
|
|
138
|
+
add("instrument", f"{pre}/instrument", baseline=pick(ia), candidate=pick(ib))
|
|
139
|
+
add("outcome", f"{pre}/outcome", baseline=(a.get("outcome") or {}).get("class"),
|
|
140
|
+
candidate=(b.get("outcome") or {}).get("class"))
|
|
141
|
+
ups = sorted({i["name"][len("receipt:"):] for r in (a, b) for i in r.get("inputs") or []
|
|
142
|
+
if isinstance(i, dict) and str(i.get("name", "")).startswith("receipt:")})
|
|
143
|
+
if ups:
|
|
144
|
+
add("upstream", f"{pre}/upstream", nodes=ups)
|
|
145
|
+
ca = ((a.get("assertions") or {}).get("constants") or {})
|
|
146
|
+
cb = ((b.get("assertions") or {}).get("constants") or {})
|
|
147
|
+
for k in sorted(set(ca) | set(cb)):
|
|
148
|
+
if ca.get(k) == cb.get(k):
|
|
149
|
+
continue
|
|
150
|
+
if k in redact:
|
|
151
|
+
add("constant", f"{pre}/constant/{k}", name=k, redacted=True,
|
|
152
|
+
baseline=_sha(str(ca.get(k)).encode()) if k in ca else None,
|
|
153
|
+
candidate=_sha(str(cb.get(k)).encode()) if k in cb else None)
|
|
154
|
+
else:
|
|
155
|
+
add("constant", f"{pre}/constant/{k}", name=k, baseline=ca.get(k), candidate=cb.get(k))
|
|
156
|
+
for which in ("inputs", "outputs"):
|
|
157
|
+
fa, fb = _files(a.get(which)), _files(b.get(which))
|
|
158
|
+
for n in sorted(set(fa) | set(fb)):
|
|
159
|
+
if n.startswith("receipt:"):
|
|
160
|
+
continue # structure, given above as upstream
|
|
161
|
+
x, y = fa.get(n) or {}, fb.get(n) or {}
|
|
162
|
+
body = {"name": n, "baseline": {"digest": x.get("digest"), "bytes": x.get("bytes")} if x else None,
|
|
163
|
+
"candidate": {"digest": y.get("digest"), "bytes": y.get("bytes")} if y else None,
|
|
164
|
+
"differs": x.get("digest") != y.get("digest")}
|
|
165
|
+
add(which[:-1], f"{pre}/{which[:-1]}/{n}", **body)
|
|
166
|
+
if include_text and body["differs"] and which == "outputs" and x and y:
|
|
167
|
+
ta = _kept_text(runs["baseline"]["path"], ra[0], x)
|
|
168
|
+
tb = _kept_text(runs["candidate"]["path"], rb[0], y)
|
|
169
|
+
if ta is not None and tb is not None:
|
|
170
|
+
add("text", f"{pre}/output/{n}/text", name=n, baseline=ta, candidate=tb)
|
|
171
|
+
packet = {"format": FORMAT,
|
|
172
|
+
"report": {"name": rname, "sha256": _sha(rraw)},
|
|
173
|
+
"runs": {s: {k: v for k, v in r.items() if k != "path"} for s, r in runs.items()},
|
|
174
|
+
"items": items, "kinds": sorted(kinds)}
|
|
175
|
+
return packet
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def by_id(packet: dict) -> dict:
|
|
179
|
+
return {i["id"]: i for i in packet.get("items") or []}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def values(item: dict) -> set:
|
|
183
|
+
"""Every string an item holds, its own and inside its values: what a claim may quote from it."""
|
|
184
|
+
out = set()
|
|
185
|
+
|
|
186
|
+
def walk(v):
|
|
187
|
+
if isinstance(v, str):
|
|
188
|
+
out.add(v)
|
|
189
|
+
elif isinstance(v, (int, float)) and not isinstance(v, bool):
|
|
190
|
+
out.add(str(v))
|
|
191
|
+
elif isinstance(v, dict):
|
|
192
|
+
for x in v.values():
|
|
193
|
+
walk(x)
|
|
194
|
+
elif isinstance(v, list):
|
|
195
|
+
for x in v:
|
|
196
|
+
walk(x)
|
|
197
|
+
for k, v in item.items():
|
|
198
|
+
if k not in ("id", "kind"):
|
|
199
|
+
walk(v)
|
|
200
|
+
return out
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""The post-check: mechanical, no model. What a reply keeps, and what it loses and why.
|
|
2
|
+
|
|
3
|
+
A reply is JSON: {"claims": [{"text", "kind", "cites": [item id, ...]}, ...], "checks": [{"check",
|
|
4
|
+
...}, ...]}. A claim is kept only when its kind is one of `lead`, `ruled out` or `note`; it cites at
|
|
5
|
+
least one item; every cite is an item of the packet; every value it quotes (in one of the quoting forms QUOTED lists; a form it doesn't list is read as unquoted text),
|
|
6
|
+
and every unquoted token holding a digit or a slash, is a value or a name of an item it cites; and it says nothing is the cause of a difference
|
|
7
|
+
("cause", "because", "due to", "led to", "resulted in", "the reason", "responsible for" and their
|
|
8
|
+
like: a list of words, never a reading). A check is kept only when it is on the menu and every
|
|
9
|
+
argument names something the packet holds. Nothing here reads the model's words for meaning.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import re
|
|
15
|
+
import unicodedata
|
|
16
|
+
|
|
17
|
+
from .packet import by_id, values
|
|
18
|
+
|
|
19
|
+
KINDS = ("lead", "ruled out", "note")
|
|
20
|
+
CAUSAL = re.compile(
|
|
21
|
+
r"\b(cause[sd]?|causing|causal(ly)?|because|due\s+to|owing\s+to|thanks\s+to"
|
|
22
|
+
r"|lead(s|ing)?\s+to|led\s+to|result(s|ed|ing)?\s+in|as\s+a\s+result|the\s+reasons?"
|
|
23
|
+
r"|reasons?\s+(for|why)|responsible\s+for|explains?\s+why|attributable|trigger(s|ed)?)\b",
|
|
24
|
+
re.IGNORECASE)
|
|
25
|
+
# every quoting form a reply may use: backticks, straight and curly double and single quotes,
|
|
26
|
+
# guillemets, low-high quotes; a straight single quote only between non-word characters, so an
|
|
27
|
+
# apostrophe ("the run's") is not read as one
|
|
28
|
+
QUOTED = re.compile("|".join([
|
|
29
|
+
r"`([^`]+)`", r"\"([^\"]+)\"", r"\u201c([^\u201d]+)\u201d", r"\u2018([^\u2019]+)\u2019",
|
|
30
|
+
r"\u00ab\s?([^\u00bb]+?)\s?\u00bb", r"\u00bb\s?([^\u00ab]+?)\s?\u00ab",
|
|
31
|
+
r"\u201e([^\u201c\u201d]+)[\u201c\u201d]", r"\u201d([^\u201d]+)\u201d", r"\u201f([^\u201d]+)\u201d",
|
|
32
|
+
r"\u2039\s?([^\u203a]+?)\s?\u203a", r"\u203a\s?([^\u2039]+?)\s?\u2039",
|
|
33
|
+
r"\u300c([^\u300d]+)\u300d", r"\u300e([^\u300f]+)\u300f", r"\uff02([^\uff02]+)\uff02",
|
|
34
|
+
r"\u2033([^\u2033]+)\u2033", r"\u301d([^\u301e\u301f]+)[\u301e\u301f]", r"\uff07([^\uff07]+)\uff07",
|
|
35
|
+
r"(?<!\w)'([^'\n]+?)'(?!\w)"]))
|
|
36
|
+
# an unquoted token that names a value: one holding a digit or a slash (a number, a version, a
|
|
37
|
+
# size, a path); it must be a value or a name of an item the claim cites
|
|
38
|
+
VALUE_TOKEN = re.compile(r"[\w.:/+@#-]*[\d/][\w.:/+@#-]*")
|
|
39
|
+
# letters that look like Latin ones, folded before the causal words are looked for
|
|
40
|
+
CONFUSABLE = str.maketrans({
|
|
41
|
+
# Cyrillic
|
|
42
|
+
"\u0430": "a", "\u0435": "e", "\u043e": "o", "\u0440": "p", "\u0441": "c", "\u0443": "y",
|
|
43
|
+
"\u0445": "x", "\u0456": "i", "\u0455": "s", "\u0458": "j", "\u04bb": "h", "\u0501": "d",
|
|
44
|
+
"\u051b": "q", "\u051d": "w", "\u0491": "r", "\u0410": "A", "\u0415": "E", "\u041e": "O",
|
|
45
|
+
"\u0420": "P", "\u0421": "C", "\u0425": "X", "\u0405": "S", "\u0406": "I", "\u0408": "J",
|
|
46
|
+
# Greek
|
|
47
|
+
"\u03bf": "o", "\u03b1": "a", "\u03b5": "e", "\u03c1": "p", "\u03c5": "u", "\u03bd": "v",
|
|
48
|
+
"\u03ba": "k", "\u03b9": "i", "\u03c4": "t", "\u039f": "O", "\u0391": "A", "\u0395": "E",
|
|
49
|
+
# Armenian
|
|
50
|
+
"\u0585": "o", "\u057d": "u", "\u0570": "h", "\u0578": "n", "\u0581": "g", "\u0566": "q",
|
|
51
|
+
# Latin letters that read as others: alpha, script g, dotless i, small capitals
|
|
52
|
+
"\u0251": "a", "\u0261": "g", "\u0131": "i", "\u1d00": "a", "\u0299": "b", "\u1d04": "c",
|
|
53
|
+
"\u1d05": "d", "\u1d07": "e", "\ua730": "f", "\u0262": "g", "\u029c": "h", "\u026a": "i",
|
|
54
|
+
"\u1d0a": "j", "\u1d0b": "k", "\u029f": "l", "\u1d0d": "m", "\u0274": "n", "\u1d0f": "o",
|
|
55
|
+
"\u1d18": "p", "\u0280": "r", "\ua731": "s", "\u1d1b": "t", "\u1d1c": "u", "\u1d20": "v",
|
|
56
|
+
"\u1d21": "w", "\u028f": "y", "\u1d22": "z"})
|
|
57
|
+
CHECKS = {
|
|
58
|
+
"text": ("node",),
|
|
59
|
+
"for": ("node",),
|
|
60
|
+
"mask": ("setting",),
|
|
61
|
+
"reproduce": ("run", "node"),
|
|
62
|
+
"match": ("run",),
|
|
63
|
+
}
|
|
64
|
+
REASONS = ("not JSON", "unknown kind", "cites nothing", "cites an item the packet doesn't hold",
|
|
65
|
+
"quotes a value its cites don't hold", "says what caused a difference")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class ReplyError(ValueError):
|
|
69
|
+
"""The reply has no JSON object with claims: nothing in it can be checked."""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def parse(raw: str) -> dict:
|
|
73
|
+
"""The reply's JSON object: the whole reply, or the first {...} block in it."""
|
|
74
|
+
try:
|
|
75
|
+
obj = json.loads(raw)
|
|
76
|
+
except ValueError:
|
|
77
|
+
m = re.search(r"\{.*\}", raw, re.DOTALL)
|
|
78
|
+
if not m:
|
|
79
|
+
raise ReplyError("the reply holds no JSON object") from None
|
|
80
|
+
try:
|
|
81
|
+
obj = json.loads(m.group(0))
|
|
82
|
+
except ValueError:
|
|
83
|
+
raise ReplyError("the reply's JSON doesn't parse") from None
|
|
84
|
+
if not isinstance(obj, dict) or not isinstance(obj.get("claims", []), list):
|
|
85
|
+
raise ReplyError("the reply's JSON has no list of claims")
|
|
86
|
+
return obj
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _claim_problem(c, items) -> str | None:
|
|
90
|
+
if not isinstance(c, dict) or not isinstance(c.get("text"), str):
|
|
91
|
+
return "not JSON"
|
|
92
|
+
if c.get("kind") not in KINDS:
|
|
93
|
+
return "unknown kind"
|
|
94
|
+
cites = c.get("cites")
|
|
95
|
+
if not isinstance(cites, list) or not cites or not all(isinstance(x, str) for x in cites):
|
|
96
|
+
return "cites nothing"
|
|
97
|
+
if any(x not in items for x in cites):
|
|
98
|
+
return "cites an item the packet doesn't hold"
|
|
99
|
+
held = set().union(*(values(items[x]) for x in cites))
|
|
100
|
+
names = (held | set(cites) | {part for x in cites for part in x.split("/")}
|
|
101
|
+
| {part for v in held for part in re.split(r"[\s/]+", v) if part})
|
|
102
|
+
text = c["text"]
|
|
103
|
+
for m in QUOTED.finditer(text):
|
|
104
|
+
q = next(g for g in m.groups() if g is not None)
|
|
105
|
+
if q not in held and q not in cites:
|
|
106
|
+
return "quotes a value its cites don't hold"
|
|
107
|
+
for tok in VALUE_TOKEN.findall(QUOTED.sub(" ", text)):
|
|
108
|
+
tok = tok.strip(".:,;-")
|
|
109
|
+
if tok and tok not in names:
|
|
110
|
+
return "quotes a value its cites don't hold"
|
|
111
|
+
if CAUSAL.search(_folded(text)):
|
|
112
|
+
return "says what caused a difference"
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _folded(text: str) -> str:
|
|
117
|
+
"""The text as the causal words are looked for in it: decomposed, with nonspacing combining marks
|
|
118
|
+
(Unicode category Mn) and format characters (Cf) (a soft hyphen, a zero-width space) removed, the
|
|
119
|
+
look-alike letters CONFUSABLE lists folded to Latin ones (the table above, which names the scripts
|
|
120
|
+
it covers), then NFKC."""
|
|
121
|
+
t = unicodedata.normalize("NFKD", text)
|
|
122
|
+
t = "".join(ch for ch in t if unicodedata.category(ch) not in ("Cf", "Mn"))
|
|
123
|
+
return unicodedata.normalize("NFKC", t.translate(CONFUSABLE))
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _nodes(packet) -> set:
|
|
127
|
+
return {i["node"] for i in packet.get("items") or [] if i.get("kind") == "node"}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _settings(packet) -> set:
|
|
131
|
+
return {i["name"] for i in packet.get("items") or [] if i.get("kind") == "constant"}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _check_problem(k, packet) -> str | None:
|
|
135
|
+
if not isinstance(k, dict) or k.get("check") not in CHECKS:
|
|
136
|
+
return "not on the menu"
|
|
137
|
+
for arg in CHECKS[k["check"]]:
|
|
138
|
+
v = k.get(arg)
|
|
139
|
+
if arg == "node" and v not in _nodes(packet):
|
|
140
|
+
return "names a node the packet doesn't hold"
|
|
141
|
+
if arg == "setting" and v not in _settings(packet):
|
|
142
|
+
return "names a setting the packet doesn't hold"
|
|
143
|
+
if arg == "run" and v not in ("baseline", "candidate"):
|
|
144
|
+
return "names a run other than baseline or candidate"
|
|
145
|
+
return None
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def check(packet: dict, reply: dict) -> dict:
|
|
149
|
+
"""{"claims": kept, "checks": kept, "dropped": {reason: count}, "checks_dropped": {reason: count}},
|
|
150
|
+
each count a decimal string."""
|
|
151
|
+
items = by_id(packet)
|
|
152
|
+
kept, dropped = [], {}
|
|
153
|
+
for c in reply.get("claims") or []:
|
|
154
|
+
why = _claim_problem(c, items)
|
|
155
|
+
if why is None:
|
|
156
|
+
kept.append({"text": c["text"], "kind": c["kind"], "cites": list(c["cites"])})
|
|
157
|
+
else:
|
|
158
|
+
dropped[why] = dropped.get(why, 0) + 1
|
|
159
|
+
checks, cdropped = [], {}
|
|
160
|
+
for k in reply.get("checks") or []:
|
|
161
|
+
why = _check_problem(k, packet)
|
|
162
|
+
if why is None:
|
|
163
|
+
checks.append({a: k[a] for a in ("check",) + CHECKS[k["check"]]})
|
|
164
|
+
else:
|
|
165
|
+
cdropped[why] = cdropped.get(why, 0) + 1
|
|
166
|
+
# counts as decimal strings: the record keeps no JSON numbers
|
|
167
|
+
return {"claims": kept, "checks": checks, "dropped": {k: str(v) for k, v in dropped.items()},
|
|
168
|
+
"checks_dropped": {k: str(v) for k, v in cdropped.items()}}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""The model: an OpenAI-compatible chat endpoint (which covers local servers) or the Anthropic
|
|
2
|
+
Messages API, named in ~/.config/onetrace/explain.toml or with --model. The key is read from the
|
|
3
|
+
environment variable the config names; it is never written anywhere. `fake:PATH` replies with a
|
|
4
|
+
file's bytes and makes no network call (tests, and seeing the flow without a model).
|
|
5
|
+
|
|
6
|
+
The one network call this package makes is `Model.ask`, and only to the endpoint configured.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import urllib.error
|
|
13
|
+
import urllib.parse
|
|
14
|
+
import urllib.request
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
CONFIG = Path(os.environ.get("ONETRACE_EXPLAIN_CONFIG", Path.home() / ".config" / "onetrace" / "explain.toml"))
|
|
19
|
+
TIMEOUT = 120
|
|
20
|
+
|
|
21
|
+
SYSTEM = """You are shown an evidence packet from onetrace, a tool that records each stage of a pipeline
|
|
22
|
+
and compares two runs. Suggest where a person should look. Reply with one JSON object only:
|
|
23
|
+
{"claims": [{"text": "...", "kind": "lead" | "ruled out" | "note", "cites": ["<item id>", ...]}],
|
|
24
|
+
"checks": [{"check": "text" | "for" | "mask" | "reproduce" | "match", "node": "...", "setting": "...",
|
|
25
|
+
"run": "baseline" | "candidate"}]}
|
|
26
|
+
Rules: every claim cites the ids of the packet items it rests on; quote values only as they appear
|
|
27
|
+
in those items, in `backticks`; a lead is a possible culprit, never a certainty; never say that
|
|
28
|
+
anything caused a difference. Claims that break a rule are dropped before anyone reads them."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ModelUnavailable(RuntimeError):
|
|
32
|
+
"""The model couldn't be reached, or replied with nothing to read."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class Model:
|
|
37
|
+
provider: str # "openai", "anthropic" or "fake"
|
|
38
|
+
name: str # the model's own name, or the fake reply's path
|
|
39
|
+
base_url: str = ""
|
|
40
|
+
key_env: str = ""
|
|
41
|
+
params: dict = field(default_factory=lambda: {"temperature": "0", "max_tokens": "2000"})
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def label(self) -> str:
|
|
45
|
+
return f"{self.provider}:{self.name}"
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def endpoint(self) -> str:
|
|
49
|
+
"""Where a packet goes, as the first-send confirmation names it: never the key."""
|
|
50
|
+
if self.provider == "fake":
|
|
51
|
+
return "fake (no network)"
|
|
52
|
+
if not self.base_url:
|
|
53
|
+
return "https://api.anthropic.com"
|
|
54
|
+
u = urllib.parse.urlsplit(self.base_url)
|
|
55
|
+
host = u.hostname or ""
|
|
56
|
+
if u.port:
|
|
57
|
+
host += f":{u.port}"
|
|
58
|
+
return urllib.parse.urlunsplit((u.scheme, host, u.path, "", ""))
|
|
59
|
+
|
|
60
|
+
def ask(self, packet_bytes: bytes) -> bytes:
|
|
61
|
+
"""The model's raw reply text, as bytes, exactly as it came back."""
|
|
62
|
+
if self.provider == "fake":
|
|
63
|
+
try:
|
|
64
|
+
return Path(self.name).read_bytes()
|
|
65
|
+
except OSError as e:
|
|
66
|
+
raise ModelUnavailable(f"fake reply {self.name}: {type(e).__name__}") from None
|
|
67
|
+
key = os.environ.get(self.key_env, "") if self.key_env else ""
|
|
68
|
+
if self.key_env and not key:
|
|
69
|
+
raise ModelUnavailable(f"the environment variable {self.key_env} the config names is not set")
|
|
70
|
+
user = packet_bytes.decode("utf-8")
|
|
71
|
+
mt = int(self.params.get("max_tokens", "2000"))
|
|
72
|
+
temp = float(self.params.get("temperature", "0"))
|
|
73
|
+
if self.provider == "anthropic":
|
|
74
|
+
url = _join(self.base_url or "https://api.anthropic.com", "/v1/messages")
|
|
75
|
+
body = {"model": self.name, "max_tokens": mt, "temperature": temp, "system": SYSTEM,
|
|
76
|
+
"messages": [{"role": "user", "content": user}]}
|
|
77
|
+
headers = {"content-type": "application/json", "anthropic-version": "2023-06-01", "x-api-key": key}
|
|
78
|
+
elif self.provider == "openai":
|
|
79
|
+
url = _join(self.base_url, "/chat/completions")
|
|
80
|
+
body = {"model": self.name, "max_tokens": mt, "temperature": temp,
|
|
81
|
+
"messages": [{"role": "system", "content": SYSTEM}, {"role": "user", "content": user}]}
|
|
82
|
+
headers = {"content-type": "application/json"}
|
|
83
|
+
if key:
|
|
84
|
+
headers["authorization"] = f"Bearer {key}"
|
|
85
|
+
else:
|
|
86
|
+
raise ModelUnavailable(f"provider {self.provider!r} is not one of openai, anthropic, fake")
|
|
87
|
+
req = urllib.request.Request(url, data=json.dumps(body).encode(), headers=headers, method="POST")
|
|
88
|
+
try:
|
|
89
|
+
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
|
90
|
+
reply = json.loads(r.read())
|
|
91
|
+
except (urllib.error.URLError, OSError, ValueError) as e:
|
|
92
|
+
raise ModelUnavailable(f"{self.endpoint}: {type(e).__name__}: {getattr(e, 'reason', e)}") from None
|
|
93
|
+
try:
|
|
94
|
+
if self.provider == "anthropic":
|
|
95
|
+
text = "".join(b.get("text", "") for b in reply["content"] if b.get("type") == "text")
|
|
96
|
+
else:
|
|
97
|
+
text = reply["choices"][0]["message"]["content"]
|
|
98
|
+
except (KeyError, IndexError, TypeError):
|
|
99
|
+
raise ModelUnavailable(f"{self.endpoint}: the reply has no message text") from None
|
|
100
|
+
return text.encode("utf-8")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _join(base: str, suffix: str) -> str:
|
|
104
|
+
"""The endpoint's path with the suffix added, its query kept after it, never inside it."""
|
|
105
|
+
u = urllib.parse.urlsplit(base)
|
|
106
|
+
return urllib.parse.urlunsplit((u.scheme, u.netloc, u.path.rstrip("/") + suffix, u.query, ""))
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def load(spec: str | None) -> Model:
|
|
110
|
+
"""The model: from `--model` (`fake:PATH`, `openai:NAME`, `anthropic:NAME`), else the config
|
|
111
|
+
file's [model] table (provider, name, base_url, key_env, and optional params)."""
|
|
112
|
+
cfg = {}
|
|
113
|
+
if CONFIG.is_file():
|
|
114
|
+
import tomllib
|
|
115
|
+
cfg = tomllib.loads(CONFIG.read_text(encoding="utf-8")).get("model", {})
|
|
116
|
+
if spec:
|
|
117
|
+
provider, _, name = spec.partition(":")
|
|
118
|
+
if not name:
|
|
119
|
+
raise ModelUnavailable(f"--model {spec!r}: give it as provider:name (fake:PATH, openai:NAME, "
|
|
120
|
+
f"anthropic:NAME)")
|
|
121
|
+
cfg = dict(cfg, provider=provider, name=name)
|
|
122
|
+
if not cfg.get("provider") or not cfg.get("name"):
|
|
123
|
+
raise ModelUnavailable(f"no model is configured: pass --model, or write [model] in {CONFIG}")
|
|
124
|
+
if cfg["provider"] == "openai" and not cfg.get("base_url"):
|
|
125
|
+
raise ModelUnavailable(f"an openai-compatible model needs base_url in {CONFIG}")
|
|
126
|
+
params = {k: str(v) for k, v in (cfg.get("params") or {}).items()}
|
|
127
|
+
return Model(cfg["provider"], cfg["name"], cfg.get("base_url", ""), cfg.get("key_env", ""),
|
|
128
|
+
dict({"temperature": "0", "max_tokens": "2000"}, **params))
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Every explanation is itself a onetrace run (EX-6), written under OUT/explain/<run id>/, with
|
|
2
|
+
three stages: `evidence` (the report and both runs' receipts, by digest, in; the packet out),
|
|
3
|
+
`model` (the provider and model as its instrument, the request parameters as its settings; the raw
|
|
4
|
+
reply out, `model-generated`) and `post_check` (what was kept and dropped). onetrace-verify reads it
|
|
5
|
+
like any run.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import datetime
|
|
10
|
+
import json
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from onetrace.emit import Instrument, Recorder
|
|
14
|
+
|
|
15
|
+
from . import __version__
|
|
16
|
+
|
|
17
|
+
STAGES = ["evidence", "model", "post_check"]
|
|
18
|
+
OA = "operator-authored"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _run_id() -> str:
|
|
22
|
+
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%fZ")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def record(out: Path, report_dir: Path, packet: dict, packet_bytes: bytes, model, ask, check) -> tuple[Path, dict]:
|
|
26
|
+
"""Run the three stages, recording each; returns (the run folder, the post-check's result).
|
|
27
|
+
`ask(packet_bytes) -> raw reply bytes` and `check(raw) -> result` are called inside their stages,
|
|
28
|
+
so a model that can't be reached leaves its stage's error receipt, and nothing that looks like
|
|
29
|
+
an explanation."""
|
|
30
|
+
run_id = _run_id()
|
|
31
|
+
folder = Path(out) / "explain" / run_id
|
|
32
|
+
rec = Recorder(folder, run_id=run_id, declared_stages=STAGES, manifest=Path(__file__),
|
|
33
|
+
policy="fail-closed", anchor_reason="an explanation is a suggestion, kept for review",
|
|
34
|
+
topology=True)
|
|
35
|
+
tool = lambda i, cfg=None: Instrument(i, "explain", __version__, dict(cfg or {}), "true")
|
|
36
|
+
result = {}
|
|
37
|
+
|
|
38
|
+
@rec.stage("evidence", tool("onetrace-explain-evidence"))
|
|
39
|
+
def evidence(ctx):
|
|
40
|
+
rep = report_dir / packet["report"]["name"]
|
|
41
|
+
ctx.reference(rep, "application/json", name=f"report/{rep.name}", trust_class=OA)
|
|
42
|
+
for side in ("baseline", "candidate"):
|
|
43
|
+
run = Path(json.loads(rep.read_bytes())["inputs"][side]["path"])
|
|
44
|
+
man = json.loads((run / "MANIFEST.json").read_bytes())
|
|
45
|
+
ctx.reference(run / "MANIFEST.json", "application/json", name=f"{side}/MANIFEST.json", trust_class=OA)
|
|
46
|
+
for e in man.get("chain") or []:
|
|
47
|
+
ctx.reference(run / e["file"], "application/json", name=f"{side}/{e['file']}", trust_class=OA)
|
|
48
|
+
ctx.write("packet.json", packet_bytes, "application/json", trust_class=OA)
|
|
49
|
+
|
|
50
|
+
@rec.stage("model", Instrument(model.provider, "model", model.name, dict(model.params), "false",
|
|
51
|
+
"a model's reply is not rederived"))
|
|
52
|
+
def ask_model(ctx):
|
|
53
|
+
ctx.read_receipt("evidence")
|
|
54
|
+
raw = ask(packet_bytes)
|
|
55
|
+
result["raw"] = raw
|
|
56
|
+
ctx.write("reply.txt", raw, "text/plain", trust_class="model-generated")
|
|
57
|
+
|
|
58
|
+
@rec.stage("post_check", tool("onetrace-explain-post-check"))
|
|
59
|
+
def post_check(ctx):
|
|
60
|
+
ctx.read_receipt("model")
|
|
61
|
+
result["checked"] = check(result["raw"])
|
|
62
|
+
ctx.write_json("checked.json", result["checked"], trust_class=OA)
|
|
63
|
+
|
|
64
|
+
try:
|
|
65
|
+
evidence()
|
|
66
|
+
ask_model()
|
|
67
|
+
post_check()
|
|
68
|
+
finally:
|
|
69
|
+
rec.close()
|
|
70
|
+
return folder, result.get("checked")
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: onetrace-explain
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A suggested explanation of a onetrace comparison: AI-generated, checked against the record, never a verdict
|
|
5
|
+
Author-email: Shamik Saha <shamik.saha.rcciit@gmail.com>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Issues, https://github.com/shamiksaharcciit-oss/onetrace-issues
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
Requires-Dist: onetrace<0.5,>=0.4
|
|
16
|
+
|
|
17
|
+
# onetrace-explain
|
|
18
|
+
|
|
19
|
+
A suggested explanation of a onetrace comparison: **AI-generated, may be wrong**, and checked
|
|
20
|
+
against the record before anyone reads it. Never a verdict: onetrace's verdicts and exit codes are
|
|
21
|
+
what they are without it, and its exit codes say nothing about the runs. Don't use it as a CI gate.
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
onetrace localize runs/a runs/b --for answer --out report
|
|
25
|
+
onetrace-explain report --dry-run # the packet exactly as it would be sent; nothing sent
|
|
26
|
+
onetrace-explain report --model anthropic:<model name>
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
- **What is sent:** a packet built from the report and the two runs' receipts: verdicts, stage
|
|
30
|
+
names and edges, outcome classes, digests and sizes, instrument names and versions, and the
|
|
31
|
+
constants that differ with both values (`--redact NAME` sends a name and digest only). Kept text
|
|
32
|
+
only with `--include-text`. A constant's value goes as the run recorded it, a path or a token
|
|
33
|
+
included; `--redact NAME` keeps it out. Never this machine's own paths, an environment variable,
|
|
34
|
+
the model key, a signing key or an anchor's proof. The first send of a kind of item to an endpoint asks first (`--yes` for scripts).
|
|
35
|
+
- **The model:** yours, named in `~/.config/onetrace/explain.toml`:
|
|
36
|
+
|
|
37
|
+
```toml
|
|
38
|
+
[model]
|
|
39
|
+
provider = "anthropic" # or "openai" (any OpenAI-compatible endpoint, local ones too)
|
|
40
|
+
name = "<model name>"
|
|
41
|
+
key_env = "ANTHROPIC_API_KEY" # the variable holding the key; the key is never written anywhere
|
|
42
|
+
# base_url = "http://localhost:8000/v1" # for an openai-compatible server
|
|
43
|
+
```
|
|
44
|
+
- **The post-check,** with no model: a claim is dropped when it cites nothing, cites what the packet
|
|
45
|
+
doesn't hold, quotes a value its cites don't hold, or says what caused a difference. What's shown
|
|
46
|
+
says how many were dropped, and why.
|
|
47
|
+
- **Checks, not conclusions:** each suggestion comes with onetrace commands to run. A lead is
|
|
48
|
+
confirmed or ruled out only by what those report.
|
|
49
|
+
- **Recorded:** every explanation is a onetrace run of its own, under `<out>/explain/<run id>/`,
|
|
50
|
+
which `onetrace-verify` checks like any run.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
MANIFEST.in
|
|
2
|
+
README.md
|
|
3
|
+
_reproducible_sdist.py
|
|
4
|
+
pyproject.toml
|
|
5
|
+
src/onetrace_explain/__init__.py
|
|
6
|
+
src/onetrace_explain/__main__.py
|
|
7
|
+
src/onetrace_explain/cli.py
|
|
8
|
+
src/onetrace_explain/packet.py
|
|
9
|
+
src/onetrace_explain/postcheck.py
|
|
10
|
+
src/onetrace_explain/providers.py
|
|
11
|
+
src/onetrace_explain/record.py
|
|
12
|
+
src/onetrace_explain.egg-info/PKG-INFO
|
|
13
|
+
src/onetrace_explain.egg-info/SOURCES.txt
|
|
14
|
+
src/onetrace_explain.egg-info/dependency_links.txt
|
|
15
|
+
src/onetrace_explain.egg-info/entry_points.txt
|
|
16
|
+
src/onetrace_explain.egg-info/requires.txt
|
|
17
|
+
src/onetrace_explain.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
onetrace<0.5,>=0.4
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
onetrace_explain
|