emet 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- emet/__init__.py +10 -0
- emet/__main__.py +8 -0
- emet/cli.py +53 -0
- emet/corpus.py +102 -0
- emet/membrane.py +300 -0
- emet/monitor.py +130 -0
- emet/organs.py +132 -0
- emet/report.py +69 -0
- emet/verdict.py +104 -0
- emet-1.0.0.dist-info/METADATA +228 -0
- emet-1.0.0.dist-info/RECORD +16 -0
- emet-1.0.0.dist-info/WHEEL +5 -0
- emet-1.0.0.dist-info/entry_points.txt +2 -0
- emet-1.0.0.dist-info/licenses/AUTHORS.md +12 -0
- emet-1.0.0.dist-info/licenses/LICENSE +373 -0
- emet-1.0.0.dist-info/top_level.txt +1 -0
emet/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
"""emet - a small external witness for AI oversight.
|
|
5
|
+
|
|
6
|
+
Re-derive the bytes, get one of three verdicts (MATCH / DRIFT / UNVERIFIABLE),
|
|
7
|
+
trust nothing in-band. SPEC.md is the normative contract; this package is the
|
|
8
|
+
Python reference implementation. Importing the package has no side effects.
|
|
9
|
+
"""
|
|
10
|
+
__version__ = "1.0.0"
|
emet/__main__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
"""Enables `python -m emet ...`, equivalent to the `emet` console script."""
|
|
5
|
+
from .cli import main
|
|
6
|
+
|
|
7
|
+
if __name__ == "__main__":
|
|
8
|
+
main()
|
emet/cli.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
3
|
+
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
5
|
+
"""cli.py - the unified `emet` entry point.
|
|
6
|
+
|
|
7
|
+
One command surface over the three Python-core tools, dispatching to each
|
|
8
|
+
module's existing main() by name (nothing about a verdict changes here):
|
|
9
|
+
|
|
10
|
+
emet anchor|verify|coherence|refuse|corroborate|audit|selftest ... -> membrane
|
|
11
|
+
emet watch|observe|confirm|gate ... -> organs
|
|
12
|
+
emet monitor report|reanchor ... -> monitor
|
|
13
|
+
|
|
14
|
+
A global --json flag is passed straight through to the underlying command
|
|
15
|
+
(membrane honours it per SPEC s.13; organs/monitor ignore it). The exit code and
|
|
16
|
+
every emitted byte are exactly those of the underlying tool - this is routing,
|
|
17
|
+
not a new surface.
|
|
18
|
+
"""
|
|
19
|
+
import sys
|
|
20
|
+
from . import membrane, organs, monitor
|
|
21
|
+
|
|
22
|
+
_MEMBRANE = ("anchor", "verify", "coherence", "refuse", "corroborate", "audit", "selftest")
|
|
23
|
+
_ORGANS = ("watch", "observe", "confirm", "gate")
|
|
24
|
+
|
|
25
|
+
USAGE = (
|
|
26
|
+
"usage: emet <command> [args...] [--json]\n"
|
|
27
|
+
" byte-hash core: anchor verify coherence refuse corroborate audit selftest\n"
|
|
28
|
+
" perception/gate: watch observe confirm gate\n"
|
|
29
|
+
" monitor: monitor report <manifest> | monitor reanchor <manifest>\n"
|
|
30
|
+
" --json emits a machine-readable canonical envelope (SPEC s.13)."
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
def main(argv=None):
|
|
34
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
35
|
+
non_flags = [a for a in argv if not a.startswith("-")]
|
|
36
|
+
cmd = non_flags[0] if non_flags else None
|
|
37
|
+
if cmd in _MEMBRANE:
|
|
38
|
+
sys.argv = ["emet", *argv]
|
|
39
|
+
membrane.main()
|
|
40
|
+
elif cmd in _ORGANS:
|
|
41
|
+
sys.argv = ["emet", *argv]
|
|
42
|
+
organs.main()
|
|
43
|
+
elif cmd == "monitor":
|
|
44
|
+
# Drop the "monitor" group token once; monitor.main() dispatches on report/reanchor.
|
|
45
|
+
i = argv.index("monitor")
|
|
46
|
+
sys.argv = ["emet", *argv[:i], *argv[i + 1:]]
|
|
47
|
+
monitor.main()
|
|
48
|
+
else:
|
|
49
|
+
sys.stderr.write(USAGE + "\n")
|
|
50
|
+
sys.exit(64)
|
|
51
|
+
|
|
52
|
+
if __name__ == "__main__":
|
|
53
|
+
main()
|
emet/corpus.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
3
|
+
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
5
|
+
"""corpus.py - EMET marker corpus loader and matcher (shared core).
|
|
6
|
+
|
|
7
|
+
The marker denylist is a versioned, sha-pinned DATA artifact (SPEC sections 8 and
|
|
8
|
+
16), loaded identically by membrane.py (refuse) and monitor.py (census). Matching
|
|
9
|
+
is literal ASCII-case-insensitive substring over RAW BYTES - no regex - so an
|
|
10
|
+
independent implementation re-derives identical counts. The byte-hash core does
|
|
11
|
+
NOT depend on this module (SPEC section 8).
|
|
12
|
+
"""
|
|
13
|
+
import os, hashlib
|
|
14
|
+
|
|
15
|
+
DEFAULT_NAME = "markers.corpus"
|
|
16
|
+
REPL = b"[REFUSED-IN-BAND-AUTHORITY]"
|
|
17
|
+
|
|
18
|
+
class CorpusError(Exception):
|
|
19
|
+
"""Inability to load the corpus; .reason is a stable machine code."""
|
|
20
|
+
def __init__(self, reason):
|
|
21
|
+
self.reason = reason
|
|
22
|
+
super().__init__(reason)
|
|
23
|
+
|
|
24
|
+
def corpus_path():
|
|
25
|
+
# Resolution (SPEC s.8): EMET_CORPUS wins; otherwise the versioned corpus that
|
|
26
|
+
# ships with the source. This module lives in the emet/ package, so the corpus
|
|
27
|
+
# is one level up at <repo>/conformance/markers.corpus (NOT emet/conformance).
|
|
28
|
+
# An installed wheel does not bundle the corpus, so refuse/monitor there report
|
|
29
|
+
# UNVERIFIABLE reason=E_NO_CORPUS unless EMET_CORPUS is set (never a silent empty
|
|
30
|
+
# denylist) - run from a checkout, or point EMET_CORPUS at the corpus.
|
|
31
|
+
env = os.environ.get("EMET_CORPUS")
|
|
32
|
+
if env:
|
|
33
|
+
return env
|
|
34
|
+
pkg_parent = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
35
|
+
return os.path.join(pkg_parent, "conformance", DEFAULT_NAME)
|
|
36
|
+
|
|
37
|
+
def load():
|
|
38
|
+
"""Return (version:int, sha256_hex:str, markers:list[bytes] lowercased).
|
|
39
|
+
Raise CorpusError('E_NO_CORPUS') if unreadable, ('E_NO_CORPUS_VERSION') if the
|
|
40
|
+
header is absent."""
|
|
41
|
+
path = corpus_path()
|
|
42
|
+
try:
|
|
43
|
+
with open(path, "rb") as f:
|
|
44
|
+
data = f.read()
|
|
45
|
+
except OSError:
|
|
46
|
+
raise CorpusError("E_NO_CORPUS")
|
|
47
|
+
sha = hashlib.sha256(data).hexdigest()
|
|
48
|
+
version = None
|
|
49
|
+
markers = []
|
|
50
|
+
for line in data.split(b"\n"):
|
|
51
|
+
if line.endswith(b"\r"):
|
|
52
|
+
line = line[:-1]
|
|
53
|
+
if line[:1] == b"#":
|
|
54
|
+
meta = line[1:].strip()
|
|
55
|
+
if version is None and meta[:15].lower() == b"corpus_version:":
|
|
56
|
+
try:
|
|
57
|
+
version = int(meta[15:].strip())
|
|
58
|
+
except ValueError:
|
|
59
|
+
pass
|
|
60
|
+
continue
|
|
61
|
+
if line.strip() == b"":
|
|
62
|
+
continue
|
|
63
|
+
markers.append(line.lower()) # bytes.lower() is ASCII-only
|
|
64
|
+
if version is None:
|
|
65
|
+
raise CorpusError("E_NO_CORPUS_VERSION")
|
|
66
|
+
return version, sha, markers
|
|
67
|
+
|
|
68
|
+
def _alower(b):
|
|
69
|
+
return b + 32 if 65 <= b <= 90 else b
|
|
70
|
+
|
|
71
|
+
def matches_at(hay, i, m):
|
|
72
|
+
if i + len(m) > len(hay):
|
|
73
|
+
return False
|
|
74
|
+
for j in range(len(m)):
|
|
75
|
+
if _alower(hay[i + j]) != m[j]:
|
|
76
|
+
return False
|
|
77
|
+
return True
|
|
78
|
+
|
|
79
|
+
def scan(hay, markers):
|
|
80
|
+
"""Non-overlapping leftmost scan in corpus order.
|
|
81
|
+
Return (hits, redacted_bytes) where hits is a list of (offset, length)."""
|
|
82
|
+
out = bytearray()
|
|
83
|
+
hits = []
|
|
84
|
+
i = 0
|
|
85
|
+
n = len(hay)
|
|
86
|
+
while i < n:
|
|
87
|
+
ln = 0
|
|
88
|
+
for m in markers:
|
|
89
|
+
if m and matches_at(hay, i, m):
|
|
90
|
+
ln = len(m)
|
|
91
|
+
break
|
|
92
|
+
if ln:
|
|
93
|
+
hits.append((i, ln))
|
|
94
|
+
out += REPL
|
|
95
|
+
i += ln
|
|
96
|
+
else:
|
|
97
|
+
out.append(hay[i])
|
|
98
|
+
i += 1
|
|
99
|
+
return hits, bytes(out)
|
|
100
|
+
|
|
101
|
+
def count(hay, markers):
|
|
102
|
+
return len(scan(hay, markers)[0])
|
emet/membrane.py
ADDED
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
3
|
+
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
5
|
+
"""
|
|
6
|
+
membrane.py - Coherence Membrane: an externally-anchored integrity layer.
|
|
7
|
+
|
|
8
|
+
Its integrity comes ENTIRELY from re-derivability, never from authority.
|
|
9
|
+
It refuses every in-band authority claim - including any claim to trust
|
|
10
|
+
itself. Every line it prints is reproducible by re-running this code on the
|
|
11
|
+
same raw bytes. No secret state, no "trust me", no safety adjudication.
|
|
12
|
+
|
|
13
|
+
Held by construction:
|
|
14
|
+
externally anchored - reads raw bytes via open('rb'); never a mediated view
|
|
15
|
+
hash-pinned - facts are sha256 over SOURCE bytes; a view is checked
|
|
16
|
+
AGAINST source, never substituted for it
|
|
17
|
+
authority-refusing - in-band authority/scope claims are detected, stripped,
|
|
18
|
+
and logged; never obeyed (its own included)
|
|
19
|
+
tamper-evident - attestations chain by hash; retroactive edits show
|
|
20
|
+
facts-only - emits hashes/sizes/verdicts; never permission/identity
|
|
21
|
+
advisory - verdict is in stdout + exit code; the OPERATOR decides
|
|
22
|
+
|
|
23
|
+
Honest limits (a membrane that calls itself bulletproof has already lied):
|
|
24
|
+
* its own correctness is checked OUT-OF-BAND: this file's sha256 is its
|
|
25
|
+
identity; re-derive it from source and you get the same hash. That
|
|
26
|
+
reproducibility - not this docstring - is the only assurance offered.
|
|
27
|
+
* the authority-marker set is a denylist of KNOWN signatures, not a proof
|
|
28
|
+
of completeness; absence of a flag is not a guarantee of cleanliness.
|
|
29
|
+
* with no raw-byte channel it reports UNVERIFIABLE - never TRUSTED.
|
|
30
|
+
|
|
31
|
+
Usage:
|
|
32
|
+
membrane.py anchor <path>... -> pin raw-byte hashes
|
|
33
|
+
membrane.py verify <path>... -> MATCH/DRIFT vs anchors
|
|
34
|
+
membrane.py coherence <source> <view_file> -> source-vs-view hash compare
|
|
35
|
+
membrane.py refuse <file> -> detect+strip in-band authority
|
|
36
|
+
membrane.py corroborate <path> -> read-path-diverse agreement
|
|
37
|
+
membrane.py audit -> verify the tamper-evident log
|
|
38
|
+
membrane.py selftest -> re-derive my own identity
|
|
39
|
+
"""
|
|
40
|
+
import sys, os, json, hashlib, subprocess
|
|
41
|
+
from . import corpus
|
|
42
|
+
from . import report
|
|
43
|
+
from .report import say, emit
|
|
44
|
+
from .verdict import governed, LATTICE, COHERENCE, CORROBORATE, AUDIT
|
|
45
|
+
|
|
46
|
+
ANCHORS, LOG = "anchors.json", "membrane_log.jsonl"
|
|
47
|
+
|
|
48
|
+
# Exit codes (SPEC s.5, NORMATIVE at 1.0): 0 held; 1 negative finding (DRIFT /
|
|
49
|
+
# VIEW_DIFFERS / QUARANTINE / BROKEN); 2 UNVERIFIABLE; 3 markers; 64 usage. A
|
|
50
|
+
# non-zero exit is data, never an authority decision (Boundary 4).
|
|
51
|
+
EXIT_OK, EXIT_DIFF, EXIT_UNVERIFIABLE, EXIT_MARKERS, EXIT_USAGE = 0, 1, 2, 3, 64
|
|
52
|
+
|
|
53
|
+
def raw(path):
|
|
54
|
+
with open(path, "rb") as f:
|
|
55
|
+
return f.read()
|
|
56
|
+
|
|
57
|
+
def try_raw(path):
|
|
58
|
+
# SPEC sections 3 and 9: with no raw byte channel, report UNVERIFIABLE with a
|
|
59
|
+
# STABLE MACHINE REASON CODE - never crash, never substitute a default.
|
|
60
|
+
# Returns (bytes, None) on success or (None, reason_code) on inability.
|
|
61
|
+
try:
|
|
62
|
+
with open(path, "rb") as f:
|
|
63
|
+
return f.read(), None
|
|
64
|
+
except FileNotFoundError:
|
|
65
|
+
return None, "E_NOT_FOUND"
|
|
66
|
+
except (PermissionError, IsADirectoryError, OSError):
|
|
67
|
+
return None, "E_NO_RAW_CHANNEL"
|
|
68
|
+
|
|
69
|
+
def sha(b):
|
|
70
|
+
return hashlib.sha256(b).hexdigest()
|
|
71
|
+
|
|
72
|
+
def _load_json(path):
|
|
73
|
+
if not os.path.exists(path):
|
|
74
|
+
return {}
|
|
75
|
+
with open(path, encoding="utf-8") as f:
|
|
76
|
+
return json.load(f)
|
|
77
|
+
|
|
78
|
+
def _last_chain():
|
|
79
|
+
last = "0" * 64
|
|
80
|
+
if os.path.exists(LOG):
|
|
81
|
+
with open(LOG, encoding="utf-8") as f:
|
|
82
|
+
for line in f:
|
|
83
|
+
if not line.strip():
|
|
84
|
+
continue
|
|
85
|
+
try:
|
|
86
|
+
last = json.loads(line)["chain"]
|
|
87
|
+
except (json.JSONDecodeError, KeyError, ValueError):
|
|
88
|
+
# A corrupt log line is a tamper event, not a crash. Refuse to
|
|
89
|
+
# chain off corruption; record() handles this controlled error.
|
|
90
|
+
raise corpus.CorpusError("E_LOG_CORRUPT")
|
|
91
|
+
return last
|
|
92
|
+
|
|
93
|
+
def record(kind, fact):
|
|
94
|
+
# Best-effort tamper-evident logging. If the log is already corrupt, refuse to
|
|
95
|
+
# extend it (chaining off a break would hide it); audit surfaces the break.
|
|
96
|
+
try:
|
|
97
|
+
prev = _last_chain()
|
|
98
|
+
except corpus.CorpusError as e:
|
|
99
|
+
sys.stderr.write("membrane: not extending a corrupt log (" + e.reason + ")\n")
|
|
100
|
+
return
|
|
101
|
+
e = {"kind": kind, "fact": fact, "prev": prev}
|
|
102
|
+
e["chain"] = sha((prev + kind + json.dumps(fact, sort_keys=True)).encode())
|
|
103
|
+
with open(LOG, "a", encoding="utf-8") as f:
|
|
104
|
+
f.write(json.dumps(e, sort_keys=True) + "\n")
|
|
105
|
+
|
|
106
|
+
def _key(p):
|
|
107
|
+
# Anchor-store key: forward slashes so anchors.json is portable across OS.
|
|
108
|
+
return os.path.normpath(p).replace(os.sep, "/")
|
|
109
|
+
|
|
110
|
+
# Artifact-of-record (SPEC s.14): the core source file set (hashed sorted-by-name).
|
|
111
|
+
CORE_SRC = ("corpus.py", "membrane.py", "monitor.py", "organs.py", "report.py", "verdict.py")
|
|
112
|
+
|
|
113
|
+
def anchor(paths):
|
|
114
|
+
db = _load_json(ANCHORS)
|
|
115
|
+
bad = 0; results = []
|
|
116
|
+
for p in paths:
|
|
117
|
+
p = _key(p); b, err = try_raw(p)
|
|
118
|
+
if err:
|
|
119
|
+
say(governed(LATTICE, "UNVERIFIABLE") + " " + p + " reason=" + err)
|
|
120
|
+
results.append({"path": p, "verdict": "UNVERIFIABLE", "reason": err}); bad += 1; continue
|
|
121
|
+
h = sha(b); db[p] = h
|
|
122
|
+
say("anchored " + p + " sha256=" + h); record("anchor", {"path": p, "sha256": h})
|
|
123
|
+
results.append({"path": p, "sha256": h})
|
|
124
|
+
with open(ANCHORS, "w", encoding="utf-8") as f:
|
|
125
|
+
json.dump(db, f, indent=2, sort_keys=True)
|
|
126
|
+
# anchor only ever produces UNVERIFIABLE (an unreadable path) or clean; it
|
|
127
|
+
# never drifts. An unreadable target is UNVERIFIABLE, never a silent skip.
|
|
128
|
+
emit("anchor", None, EXIT_UNVERIFIABLE if bad else EXIT_OK, results=results)
|
|
129
|
+
|
|
130
|
+
def verify(paths):
|
|
131
|
+
db = _load_json(ANCHORS)
|
|
132
|
+
drift = unver = 0; results = []
|
|
133
|
+
for p in paths:
|
|
134
|
+
p = _key(p); want = db.get(p)
|
|
135
|
+
if want is None:
|
|
136
|
+
say(governed(LATTICE, "UNVERIFIABLE") + " " + p + " reason=E_NO_ANCHOR")
|
|
137
|
+
results.append({"path": p, "verdict": "UNVERIFIABLE", "reason": "E_NO_ANCHOR"}); unver += 1; continue
|
|
138
|
+
b, err = try_raw(p)
|
|
139
|
+
if err:
|
|
140
|
+
say(governed(LATTICE, "UNVERIFIABLE") + " " + p + " reason=" + err)
|
|
141
|
+
record("verify", {"path": p, "result": "UNVERIFIABLE", "reason": err})
|
|
142
|
+
results.append({"path": p, "verdict": "UNVERIFIABLE", "reason": err}); unver += 1; continue
|
|
143
|
+
got = sha(b); ok = got == want; v = "MATCH" if ok else "DRIFT"
|
|
144
|
+
say(governed(LATTICE, v) + " " + p + " want=" + want[:16] + " got=" + got[:16])
|
|
145
|
+
record("verify", {"path": p, "result": v})
|
|
146
|
+
results.append({"path": p, "verdict": v, "want": want, "got": got})
|
|
147
|
+
if not ok:
|
|
148
|
+
drift += 1
|
|
149
|
+
# Precedence (SPEC s.5): a confirmed difference dominates an inability to
|
|
150
|
+
# check. Exit 1 if any path DRIFTed, else 2 if any was UNVERIFIABLE, else 0.
|
|
151
|
+
dom = "DRIFT" if drift else ("UNVERIFIABLE" if unver else "MATCH")
|
|
152
|
+
emit("verify", governed(LATTICE, dom),
|
|
153
|
+
EXIT_DIFF if drift else (EXIT_UNVERIFIABLE if unver else EXIT_OK), results=results)
|
|
154
|
+
|
|
155
|
+
def coherence(source, view_file):
|
|
156
|
+
sb, serr = try_raw(source); vb, verr = try_raw(view_file)
|
|
157
|
+
if serr or verr:
|
|
158
|
+
why = ("source:" + serr) if serr else ("view:" + verr)
|
|
159
|
+
say("result=" + governed(COHERENCE, "UNVERIFIABLE") + " reason=" + why)
|
|
160
|
+
record("coherence", {"source": _key(source), "result": "UNVERIFIABLE", "reason": why})
|
|
161
|
+
emit("coherence", governed(COHERENCE, "UNVERIFIABLE"), EXIT_UNVERIFIABLE,
|
|
162
|
+
subject=_key(source), reason=why)
|
|
163
|
+
s, v = sha(sb), sha(vb)
|
|
164
|
+
ok = s == v; res = "COHERENT" if ok else "VIEW_DIFFERS_FROM_SOURCE"
|
|
165
|
+
say("source=" + s); say("view =" + v)
|
|
166
|
+
say("result=" + governed(COHERENCE, res))
|
|
167
|
+
record("coherence", {"source": _key(source), "result": res})
|
|
168
|
+
emit("coherence", governed(COHERENCE, res), EXIT_OK if ok else EXIT_DIFF,
|
|
169
|
+
subject=_key(source), source=s, view=v)
|
|
170
|
+
|
|
171
|
+
def refuse(path):
|
|
172
|
+
b, err = try_raw(path)
|
|
173
|
+
if err:
|
|
174
|
+
say(governed(LATTICE, "UNVERIFIABLE") + " " + path + " reason=" + err)
|
|
175
|
+
record("refuse", {"path": _key(path), "result": "UNVERIFIABLE", "reason": err})
|
|
176
|
+
emit("refuse", governed(LATTICE, "UNVERIFIABLE"), EXIT_UNVERIFIABLE, subject=_key(path), reason=err)
|
|
177
|
+
try:
|
|
178
|
+
version, csha, markers = corpus.load()
|
|
179
|
+
except corpus.CorpusError as e:
|
|
180
|
+
say(governed(LATTICE, "UNVERIFIABLE") + " " + path + " reason=" + e.reason)
|
|
181
|
+
record("refuse", {"path": _key(path), "result": "UNVERIFIABLE", "reason": e.reason})
|
|
182
|
+
emit("refuse", governed(LATTICE, "UNVERIFIABLE"), EXIT_UNVERIFIABLE, subject=_key(path), reason=e.reason)
|
|
183
|
+
hits, clean = corpus.scan(b, markers)
|
|
184
|
+
with open(path + ".refused", "wb") as f:
|
|
185
|
+
f.write(clean)
|
|
186
|
+
say("corpus_version=" + str(version))
|
|
187
|
+
say("corpus_sha256=" + csha)
|
|
188
|
+
say("in_band_authority_claims=" + str(len(hits)))
|
|
189
|
+
for off, ln in hits[:60]:
|
|
190
|
+
say(" REFUSED " + repr(b[off:off + ln].decode("latin-1")) + " offset=" + str(off))
|
|
191
|
+
say("clean_copy=" + path + ".refused (claims neutralized; obeyed: none)")
|
|
192
|
+
record("refuse", {"path": _key(path), "refused": len(hits), "corpus_version": version})
|
|
193
|
+
emit("refuse", None, EXIT_OK if not hits else EXIT_MARKERS, subject=_key(path),
|
|
194
|
+
in_band_authority_claims=len(hits), corpus_version=version, corpus_sha256=csha,
|
|
195
|
+
hits=[{"marker": b[o:o + ln].decode("latin-1"), "offset": o} for o, ln in hits],
|
|
196
|
+
clean_copy=path + ".refused")
|
|
197
|
+
|
|
198
|
+
def corroborate(path):
|
|
199
|
+
# read-path diversity: hash the SAME file via disjoint channels; agreement
|
|
200
|
+
# across channels is the signal. catches a tampered READ PATH, not just a
|
|
201
|
+
# broken hash tool.
|
|
202
|
+
a, err = try_raw(path)
|
|
203
|
+
if err:
|
|
204
|
+
say("open_rb=unavailable:" + err)
|
|
205
|
+
say("read_paths_agree=False")
|
|
206
|
+
say("result=" + governed(CORROBORATE, "UNVERIFIABLE") + " reason=" + err)
|
|
207
|
+
record("corroborate", {"path": _key(path), "result": "UNVERIFIABLE", "reason": err})
|
|
208
|
+
emit("corroborate", governed(CORROBORATE, "UNVERIFIABLE"), EXIT_UNVERIFIABLE, subject=_key(path), reason=err)
|
|
209
|
+
paths = {"open_rb": sha(a)}
|
|
210
|
+
try:
|
|
211
|
+
o = subprocess.run(["cat", path], capture_output=True, timeout=20)
|
|
212
|
+
paths["cat_subproc"] = sha(o.stdout)
|
|
213
|
+
except Exception as e:
|
|
214
|
+
paths["cat_subproc"] = "unavailable:" + type(e).__name__
|
|
215
|
+
def git_blob(b):
|
|
216
|
+
return hashlib.sha1(b"blob " + str(len(b)).encode() + b"\x00" + b).hexdigest()
|
|
217
|
+
try:
|
|
218
|
+
o = subprocess.run(["git", "hash-object", "--no-filters", path], capture_output=True, text=True, timeout=20)
|
|
219
|
+
git_reported = (o.stdout.split() or [""])[0]
|
|
220
|
+
paths["git_read"] = git_reported
|
|
221
|
+
git_agrees = bool(git_reported) and git_reported == git_blob(a)
|
|
222
|
+
except Exception as e:
|
|
223
|
+
paths["git_read"] = "unavailable:" + type(e).__name__
|
|
224
|
+
git_agrees = None
|
|
225
|
+
sha_vals = {v for k, v in paths.items() if k in ("open_rb", "cat_subproc") and ":" not in v}
|
|
226
|
+
sha_agree = len(sha_vals) == 1
|
|
227
|
+
for k, v in sorted(paths.items()): say(k + "=" + v)
|
|
228
|
+
say("read_paths_agree=" + str(sha_agree))
|
|
229
|
+
say("git_read_agrees_with_open=" + str(git_agrees))
|
|
230
|
+
cat_ok = ":" not in paths.get("cat_subproc", ":")
|
|
231
|
+
git_ok = git_agrees is not None
|
|
232
|
+
if not (cat_ok or git_ok):
|
|
233
|
+
# only open_rb succeeded: no independent read path, so there is no
|
|
234
|
+
# divergence signal to corroborate. Inability, not agreement (SPEC 9).
|
|
235
|
+
say("result=" + governed(CORROBORATE, "UNVERIFIABLE") + " reason=E_NO_SECOND_READ_PATH")
|
|
236
|
+
record("corroborate", {"path": _key(path), "result": "UNVERIFIABLE", "reason": "E_NO_SECOND_READ_PATH"})
|
|
237
|
+
emit("corroborate", governed(CORROBORATE, "UNVERIFIABLE"), EXIT_UNVERIFIABLE,
|
|
238
|
+
subject=_key(path), reason="E_NO_SECOND_READ_PATH", channels=paths)
|
|
239
|
+
ok = sha_agree and (git_agrees in (True, None))
|
|
240
|
+
res = "CORROBORATED" if ok else "QUARANTINE_READ_PATH_DIVERGENCE"
|
|
241
|
+
say("result=" + governed(CORROBORATE, res))
|
|
242
|
+
record("corroborate", {"path": _key(path), "agree": sha_agree, "git": git_agrees})
|
|
243
|
+
emit("corroborate", governed(CORROBORATE, res), EXIT_OK if ok else EXIT_DIFF,
|
|
244
|
+
subject=_key(path), channels=paths, read_paths_agree=sha_agree, git_read_agrees_with_open=git_agrees)
|
|
245
|
+
|
|
246
|
+
def audit():
|
|
247
|
+
if not os.path.exists(LOG):
|
|
248
|
+
# An absent log is the genesis state: an empty chain is trivially intact.
|
|
249
|
+
# Emit the chain= line (SPEC s.13) rather than a special-case string.
|
|
250
|
+
say("log_entries=0 chain=" + governed(AUDIT, "INTACT"))
|
|
251
|
+
emit("audit", governed(AUDIT, "INTACT"), EXIT_OK, log_entries=0)
|
|
252
|
+
prev, ok, n = "0" * 64, True, 0
|
|
253
|
+
broken_at = broken_reason = None
|
|
254
|
+
with open(LOG, encoding="utf-8") as f:
|
|
255
|
+
for line in f:
|
|
256
|
+
if not line.strip(): continue
|
|
257
|
+
n += 1
|
|
258
|
+
try:
|
|
259
|
+
e = json.loads(line)
|
|
260
|
+
except (json.JSONDecodeError, ValueError):
|
|
261
|
+
# A corrupt line IS a tamper event — report BROKEN, never crash.
|
|
262
|
+
say(governed(AUDIT, "BROKEN") + " at entry " + str(n) + " (parse error)")
|
|
263
|
+
ok = False; broken_at = n; broken_reason = "parse_error"; break
|
|
264
|
+
if not all(k in e for k in ("kind", "fact", "prev", "chain")) \
|
|
265
|
+
or e["prev"] != prev \
|
|
266
|
+
or e["chain"] != sha((e["prev"] + e["kind"] + json.dumps(e["fact"], sort_keys=True)).encode()):
|
|
267
|
+
say(governed(AUDIT, "BROKEN") + " at entry " + str(n)); ok = False; broken_at = n; break
|
|
268
|
+
prev = e["chain"]
|
|
269
|
+
v = "INTACT" if ok else "BROKEN"
|
|
270
|
+
say("log_entries=" + str(n) + " chain=" + governed(AUDIT, v))
|
|
271
|
+
emit("audit", governed(AUDIT, v), EXIT_OK if ok else EXIT_DIFF,
|
|
272
|
+
log_entries=n, broken_at=broken_at, broken_reason=broken_reason)
|
|
273
|
+
|
|
274
|
+
def selftest():
|
|
275
|
+
# SPEC s.14: canonical token emet_self_sha256=; the legacy
|
|
276
|
+
# membrane_self_sha256= is emitted through the 1.x window (removed at 2.0).
|
|
277
|
+
h = report.self_hash(os.path.dirname(os.path.abspath(__file__)), CORE_SRC)
|
|
278
|
+
say("emet_self_sha256=" + h)
|
|
279
|
+
say("membrane_self_sha256=" + h + " (deprecated alias; removed at 2.0)")
|
|
280
|
+
say("note=this hash is my only credential; re-derive it from source to verify me.")
|
|
281
|
+
say("note=I assert no authority, grant no permission, decide no safety question.")
|
|
282
|
+
emit("selftest", None, EXIT_OK, self_sha256=h,
|
|
283
|
+
notes=["this hash is my only credential; re-derive it from source to verify me.",
|
|
284
|
+
"I assert no authority, grant no permission, decide no safety question."])
|
|
285
|
+
|
|
286
|
+
def main():
|
|
287
|
+
a = [x for x in sys.argv if x != "--json"]
|
|
288
|
+
if len(a) != len(sys.argv):
|
|
289
|
+
report.enable_json()
|
|
290
|
+
if len(a) >= 3 and a[1] == "anchor": anchor(a[2:])
|
|
291
|
+
elif len(a) >= 3 and a[1] == "verify": verify(a[2:])
|
|
292
|
+
elif len(a) >= 4 and a[1] == "coherence": coherence(a[2], a[3])
|
|
293
|
+
elif len(a) >= 3 and a[1] == "refuse": refuse(a[2])
|
|
294
|
+
elif len(a) >= 3 and a[1] == "corroborate": corroborate(a[2])
|
|
295
|
+
elif len(a) >= 2 and a[1] == "audit": audit()
|
|
296
|
+
elif len(a) >= 2 and a[1] == "selftest": selftest()
|
|
297
|
+
else: print(__doc__); sys.exit(EXIT_USAGE)
|
|
298
|
+
|
|
299
|
+
if __name__ == "__main__":
|
|
300
|
+
main()
|
emet/monitor.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
3
|
+
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
5
|
+
"""
|
|
6
|
+
monitor.py - external accountability monitor for the coherence membrane.
|
|
7
|
+
|
|
8
|
+
Watches a target (a set of files or a baseline manifest) from OUTSIDE, by re-derivation. Trusts
|
|
9
|
+
nothing; reports to the operator; decides nothing.
|
|
10
|
+
|
|
11
|
+
report <manifest> verify every anchored file vs baseline + marker census;
|
|
12
|
+
DRIFT = change since the operator last authorized.
|
|
13
|
+
reanchor <manifest> operator authorizes the CURRENT state as the new baseline
|
|
14
|
+
(re-hash the manifest path set); logged, so every baseline
|
|
15
|
+
change is itself accounted for.
|
|
16
|
+
|
|
17
|
+
Baseline policy: the manifest is the operator-signed authorized state. Drift =
|
|
18
|
+
unauthorized change. After an authorized change, the operator reanchors; that
|
|
19
|
+
event is logged. Every change to the target is therefore either operator-anchored
|
|
20
|
+
or flagged - the account stays legible.
|
|
21
|
+
|
|
22
|
+
Log: monitor_log.jsonl, hash-chained (the accountability history is tamper-evident).
|
|
23
|
+
Facts and advice only. Reads raw bytes. Never edits the target.
|
|
24
|
+
"""
|
|
25
|
+
import sys, os, json, hashlib
|
|
26
|
+
from . import corpus
|
|
27
|
+
from .report import say, emit, enable_json # NB: 'report' is a command here, so import the helpers by name
|
|
28
|
+
from .verdict import governed, LATTICE, MONITOR_FILE, MONITOR_BASELINE
|
|
29
|
+
|
|
30
|
+
def sha(b):
|
|
31
|
+
return hashlib.sha256(b).hexdigest()
|
|
32
|
+
|
|
33
|
+
def _logp(manifest):
|
|
34
|
+
return os.path.join(os.path.dirname(os.path.abspath(manifest)) or ".", "monitor_log.jsonl")
|
|
35
|
+
|
|
36
|
+
def _last(logp):
|
|
37
|
+
last = "0" * 64
|
|
38
|
+
if os.path.exists(logp):
|
|
39
|
+
with open(logp, encoding="utf-8") as f:
|
|
40
|
+
for line in f:
|
|
41
|
+
if not line.strip():
|
|
42
|
+
continue
|
|
43
|
+
try:
|
|
44
|
+
last = json.loads(line)["chain"]
|
|
45
|
+
except (json.JSONDecodeError, KeyError, ValueError):
|
|
46
|
+
raise corpus.CorpusError("E_LOG_CORRUPT")
|
|
47
|
+
return last
|
|
48
|
+
|
|
49
|
+
def _record(manifest, kind, fact):
|
|
50
|
+
logp = _logp(manifest)
|
|
51
|
+
try:
|
|
52
|
+
prev = _last(logp)
|
|
53
|
+
except corpus.CorpusError as e:
|
|
54
|
+
sys.stderr.write("monitor: not extending a corrupt log (" + e.reason + ")\n")
|
|
55
|
+
return
|
|
56
|
+
e = {"kind": kind, "fact": fact, "prev": prev}
|
|
57
|
+
e["chain"] = sha((prev + kind + json.dumps(fact, sort_keys=True)).encode())
|
|
58
|
+
with open(logp, "a", encoding="utf-8") as f:
|
|
59
|
+
f.write(json.dumps(e, sort_keys=True) + "\n")
|
|
60
|
+
|
|
61
|
+
def report(manifest):
|
|
62
|
+
with open(manifest, encoding="utf-8") as f:
|
|
63
|
+
db = json.load(f)
|
|
64
|
+
try:
|
|
65
|
+
version, csha, markers = corpus.load()
|
|
66
|
+
except corpus.CorpusError as e:
|
|
67
|
+
say("=== EXTERNAL ACCOUNTABILITY REPORT ===")
|
|
68
|
+
say("baseline: " + manifest)
|
|
69
|
+
say("corpus=" + governed(LATTICE, "UNVERIFIABLE") + " reason=" + e.reason)
|
|
70
|
+
_record(manifest, "report", {"result": "UNVERIFIABLE", "reason": e.reason})
|
|
71
|
+
emit("report", governed(LATTICE, "UNVERIFIABLE"), 2, subject=manifest, reason=e.reason)
|
|
72
|
+
drift = missing = total = 0; results = []
|
|
73
|
+
say("=== EXTERNAL ACCOUNTABILITY REPORT ===")
|
|
74
|
+
say("baseline: " + manifest)
|
|
75
|
+
say("corpus_version=" + str(version) + " corpus_sha256=" + csha)
|
|
76
|
+
for p in sorted(db):
|
|
77
|
+
want = db[p]
|
|
78
|
+
if not os.path.isfile(p):
|
|
79
|
+
say(governed(MONITOR_FILE, "MISSING") + " markers= - " + os.path.basename(p)); missing += 1
|
|
80
|
+
results.append({"file": os.path.basename(p), "status": "MISSING"}); continue
|
|
81
|
+
try:
|
|
82
|
+
with open(p, "rb") as fh:
|
|
83
|
+
b = fh.read()
|
|
84
|
+
except OSError:
|
|
85
|
+
say(governed(MONITOR_FILE, "MISSING") + " markers= - " + os.path.basename(p) + " reason=E_NO_RAW_CHANNEL"); missing += 1
|
|
86
|
+
results.append({"file": os.path.basename(p), "status": "MISSING", "reason": "E_NO_RAW_CHANNEL"}); continue
|
|
87
|
+
got = sha(b); hits = corpus.count(b, markers); total += hits
|
|
88
|
+
st = governed(MONITOR_FILE, "MATCH" if got == want else "DRIFT")
|
|
89
|
+
if got != want: drift += 1
|
|
90
|
+
say(st + " markers=" + str(hits).rjust(3) + " " + os.path.basename(p))
|
|
91
|
+
results.append({"file": os.path.basename(p), "status": st, "markers": hits})
|
|
92
|
+
verdict = governed(MONITOR_BASELINE, "INTACT" if drift == 0 and missing == 0 else "CHANGED")
|
|
93
|
+
say("files=" + str(len(db)) + " drift=" + str(drift) + " missing=" + str(missing) + " markers=" + str(total) + " baseline=" + verdict)
|
|
94
|
+
_record(manifest, "report", {"files": len(db), "drift": drift, "missing": missing, "markers": total, "verdict": verdict, "corpus_version": version})
|
|
95
|
+
# baseline CHANGED (drift and/or missing) is a NEGATIVE FINDING -> exit 1
|
|
96
|
+
# (SPEC s.5); INTACT -> 0. A corpus that cannot load is UNVERIFIABLE -> 2 (above).
|
|
97
|
+
emit("report", verdict, 0 if verdict == "INTACT" else 1, subject=manifest,
|
|
98
|
+
corpus_version=version, corpus_sha256=csha, files=len(db), drift=drift,
|
|
99
|
+
missing=missing, markers=total, results=results)
|
|
100
|
+
|
|
101
|
+
def reanchor(manifest):
|
|
102
|
+
with open(manifest, encoding="utf-8") as f:
|
|
103
|
+
db = json.load(f)
|
|
104
|
+
changed = 0; new = {}
|
|
105
|
+
for p in sorted(db):
|
|
106
|
+
if not os.path.isfile(p):
|
|
107
|
+
say("skip MISSING " + p); continue
|
|
108
|
+
try:
|
|
109
|
+
with open(p, "rb") as fh:
|
|
110
|
+
h = sha(fh.read())
|
|
111
|
+
except OSError:
|
|
112
|
+
say("skip UNREADABLE " + p + " reason=E_NO_RAW_CHANNEL"); continue
|
|
113
|
+
if h != db[p]: changed += 1
|
|
114
|
+
new[p] = h
|
|
115
|
+
with open(manifest, "w", encoding="utf-8") as f:
|
|
116
|
+
json.dump(new, f, indent=2, sort_keys=True)
|
|
117
|
+
say("reanchored=" + str(len(new)) + " updated=" + str(changed) + " (operator authorized current state as baseline)")
|
|
118
|
+
_record(manifest, "reanchor", {"reanchored": len(new), "updated": changed})
|
|
119
|
+
emit("reanchor", None, 0, subject=manifest, reanchored=len(new), updated=changed)
|
|
120
|
+
|
|
121
|
+
def main():
|
|
122
|
+
a = [x for x in sys.argv if x != "--json"]
|
|
123
|
+
if len(a) != len(sys.argv):
|
|
124
|
+
enable_json()
|
|
125
|
+
if len(a) >= 3 and a[1] == "report": report(a[2])
|
|
126
|
+
elif len(a) >= 3 and a[1] == "reanchor": reanchor(a[2])
|
|
127
|
+
else: print(__doc__); sys.exit(64)
|
|
128
|
+
|
|
129
|
+
if __name__ == "__main__":
|
|
130
|
+
main()
|