crovia-tacet 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crovia_tacet-0.3.0/PKG-INFO +36 -0
- crovia_tacet-0.3.0/README.md +13 -0
- crovia_tacet-0.3.0/crovia_tacet.egg-info/PKG-INFO +36 -0
- crovia_tacet-0.3.0/crovia_tacet.egg-info/SOURCES.txt +22 -0
- crovia_tacet-0.3.0/crovia_tacet.egg-info/dependency_links.txt +1 -0
- crovia_tacet-0.3.0/crovia_tacet.egg-info/requires.txt +7 -0
- crovia_tacet-0.3.0/crovia_tacet.egg-info/top_level.txt +1 -0
- crovia_tacet-0.3.0/pyproject.toml +38 -0
- crovia_tacet-0.3.0/setup.cfg +4 -0
- crovia_tacet-0.3.0/tacet/__init__.py +30 -0
- crovia_tacet-0.3.0/tacet/canonical.py +77 -0
- crovia_tacet-0.3.0/tacet/egress.py +309 -0
- crovia_tacet-0.3.0/tacet/epoch.py +154 -0
- crovia_tacet-0.3.0/tacet/fixtures.py +143 -0
- crovia_tacet-0.3.0/tacet/hashing.py +73 -0
- crovia_tacet-0.3.0/tacet/keys.py +44 -0
- crovia_tacet-0.3.0/tacet/merkle.py +58 -0
- crovia_tacet-0.3.0/tacet/ots.py +239 -0
- crovia_tacet-0.3.0/tacet/silence.py +334 -0
- crovia_tacet-0.3.0/tacet/smt.py +134 -0
- crovia_tacet-0.3.0/tacet/snapshot.py +84 -0
- crovia_tacet-0.3.0/tacet/wrap.py +127 -0
- crovia_tacet-0.3.0/tests/test_egress.py +119 -0
- crovia_tacet-0.3.0/tests/test_tacet.py +252 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: crovia-tacet
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Reference implementation of TACET: verifiable silence proofs for AI training disclosure (Crovia Trust)
|
|
5
|
+
Author-email: Crovia Trust <info@croviatrust.com>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://croviatrust.com
|
|
8
|
+
Project-URL: Specification, https://github.com/croviatrust/countersign/blob/main/tacet/SPEC.md
|
|
9
|
+
Keywords: ai,transparency,eu-ai-act,merkle,non-inclusion,provenance,crovia
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Legal Industry
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Security :: Cryptography
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: cryptography>=41.0.0
|
|
19
|
+
Provides-Extra: seal
|
|
20
|
+
Requires-Dist: crovia-seal>=0.5.0; extra == "seal"
|
|
21
|
+
Provides-Extra: test
|
|
22
|
+
Requires-Dist: pytest>=7.0; extra == "test"
|
|
23
|
+
|
|
24
|
+
# crovia-tacet (reference implementation)
|
|
25
|
+
|
|
26
|
+
Reference Python implementation of [TACET](../../SPEC.md): sparse Merkle map,
|
|
27
|
+
epoch sheets, surface snapshots, delta-encoded silence proofs, and delivery as
|
|
28
|
+
an unmodified `crovia.seal.v1`.
|
|
29
|
+
|
|
30
|
+
Dependencies: `cryptography` (Ed25519). The Seal wrapper additionally needs the
|
|
31
|
+
Crovia Seal reference package (`crovia_seal`).
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install -e .
|
|
35
|
+
python3 -m pytest
|
|
36
|
+
```
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# crovia-tacet (reference implementation)
|
|
2
|
+
|
|
3
|
+
Reference Python implementation of [TACET](../../SPEC.md): sparse Merkle map,
|
|
4
|
+
epoch sheets, surface snapshots, delta-encoded silence proofs, and delivery as
|
|
5
|
+
an unmodified `crovia.seal.v1`.
|
|
6
|
+
|
|
7
|
+
Dependencies: `cryptography` (Ed25519). The Seal wrapper additionally needs the
|
|
8
|
+
Crovia Seal reference package (`crovia_seal`).
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
pip install -e .
|
|
12
|
+
python3 -m pytest
|
|
13
|
+
```
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: crovia-tacet
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Reference implementation of TACET: verifiable silence proofs for AI training disclosure (Crovia Trust)
|
|
5
|
+
Author-email: Crovia Trust <info@croviatrust.com>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://croviatrust.com
|
|
8
|
+
Project-URL: Specification, https://github.com/croviatrust/countersign/blob/main/tacet/SPEC.md
|
|
9
|
+
Keywords: ai,transparency,eu-ai-act,merkle,non-inclusion,provenance,crovia
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Legal Industry
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Security :: Cryptography
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
Requires-Dist: cryptography>=41.0.0
|
|
19
|
+
Provides-Extra: seal
|
|
20
|
+
Requires-Dist: crovia-seal>=0.5.0; extra == "seal"
|
|
21
|
+
Provides-Extra: test
|
|
22
|
+
Requires-Dist: pytest>=7.0; extra == "test"
|
|
23
|
+
|
|
24
|
+
# crovia-tacet (reference implementation)
|
|
25
|
+
|
|
26
|
+
Reference Python implementation of [TACET](../../SPEC.md): sparse Merkle map,
|
|
27
|
+
epoch sheets, surface snapshots, delta-encoded silence proofs, and delivery as
|
|
28
|
+
an unmodified `crovia.seal.v1`.
|
|
29
|
+
|
|
30
|
+
Dependencies: `cryptography` (Ed25519). The Seal wrapper additionally needs the
|
|
31
|
+
Crovia Seal reference package (`crovia_seal`).
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install -e .
|
|
35
|
+
python3 -m pytest
|
|
36
|
+
```
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
crovia_tacet.egg-info/PKG-INFO
|
|
4
|
+
crovia_tacet.egg-info/SOURCES.txt
|
|
5
|
+
crovia_tacet.egg-info/dependency_links.txt
|
|
6
|
+
crovia_tacet.egg-info/requires.txt
|
|
7
|
+
crovia_tacet.egg-info/top_level.txt
|
|
8
|
+
tacet/__init__.py
|
|
9
|
+
tacet/canonical.py
|
|
10
|
+
tacet/egress.py
|
|
11
|
+
tacet/epoch.py
|
|
12
|
+
tacet/fixtures.py
|
|
13
|
+
tacet/hashing.py
|
|
14
|
+
tacet/keys.py
|
|
15
|
+
tacet/merkle.py
|
|
16
|
+
tacet/ots.py
|
|
17
|
+
tacet/silence.py
|
|
18
|
+
tacet/smt.py
|
|
19
|
+
tacet/snapshot.py
|
|
20
|
+
tacet/wrap.py
|
|
21
|
+
tests/test_egress.py
|
|
22
|
+
tests/test_tacet.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
tacet
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "crovia-tacet"
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
description = "Reference implementation of TACET: verifiable silence proofs for AI training disclosure (Crovia Trust)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "Apache-2.0" }
|
|
12
|
+
authors = [{ name = "Crovia Trust", email = "info@croviatrust.com" }]
|
|
13
|
+
keywords = ["ai", "transparency", "eu-ai-act", "merkle", "non-inclusion", "provenance", "crovia"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Intended Audience :: Legal Industry",
|
|
18
|
+
"License :: OSI Approved :: Apache Software License",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Topic :: Security :: Cryptography",
|
|
21
|
+
]
|
|
22
|
+
dependencies = ["cryptography>=41.0.0"]
|
|
23
|
+
|
|
24
|
+
[project.optional-dependencies]
|
|
25
|
+
seal = ["crovia-seal>=0.5.0"]
|
|
26
|
+
test = ["pytest>=7.0"]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://croviatrust.com"
|
|
30
|
+
Specification = "https://github.com/croviatrust/countersign/blob/main/tacet/SPEC.md"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages.find]
|
|
33
|
+
where = ["."]
|
|
34
|
+
include = ["tacet*"]
|
|
35
|
+
|
|
36
|
+
[tool.pytest.ini_options]
|
|
37
|
+
testpaths = ["tests"]
|
|
38
|
+
addopts = "-ra -q"
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""TACET reference implementation — verifiable silence for AI training disclosure.
|
|
2
|
+
|
|
3
|
+
See ../../SPEC.md. Public API:
|
|
4
|
+
|
|
5
|
+
from tacet import SparseMerkleMap, target_key, build_sheet, build_snapshot,
|
|
6
|
+
build_silence_proof, verify_silence_proof, wrap_in_seal, verify_wrapped
|
|
7
|
+
"""
|
|
8
|
+
from .canonical import canonicalize, CanonicalizationError
|
|
9
|
+
from .hashing import EMPTY, object_hash, target_key, namespace_key, prefixed, unprefixed
|
|
10
|
+
from .keys import SigningKey, verify_signature
|
|
11
|
+
from .smt import SparseMerkleMap, CompactPath, verify_inclusion, verify_non_inclusion
|
|
12
|
+
from .epoch import build_sheet, close_sheet, validate_sheet, validate_chain, sheet_hash, witness_sign, witness_verify, SheetError
|
|
13
|
+
from .snapshot import build_snapshot, validate_snapshot, snapshot_hash, SnapshotError
|
|
14
|
+
from .silence import (build_silence_proof, verify_silence_proof, build_query, compute_silence,
|
|
15
|
+
SilenceVerifyResult, SilenceProofError,
|
|
16
|
+
STRENGTH_MAP, STRENGTH_SURFACE, STRENGTH_WITNESSED)
|
|
17
|
+
from .wrap import wrap_in_seal, verify_wrapped
|
|
18
|
+
|
|
19
|
+
__version__ = "0.3.0"
|
|
20
|
+
__all__ = [
|
|
21
|
+
"canonicalize", "CanonicalizationError",
|
|
22
|
+
"EMPTY", "object_hash", "target_key", "namespace_key", "prefixed", "unprefixed",
|
|
23
|
+
"SigningKey", "verify_signature",
|
|
24
|
+
"SparseMerkleMap", "CompactPath", "verify_inclusion", "verify_non_inclusion",
|
|
25
|
+
"build_sheet", "close_sheet", "validate_sheet", "validate_chain", "sheet_hash", "witness_sign", "witness_verify", "SheetError",
|
|
26
|
+
"build_snapshot", "validate_snapshot", "snapshot_hash", "SnapshotError",
|
|
27
|
+
"build_silence_proof", "verify_silence_proof", "build_query", "compute_silence",
|
|
28
|
+
"SilenceVerifyResult", "SilenceProofError", "STRENGTH_MAP", "STRENGTH_SURFACE", "STRENGTH_WITNESSED",
|
|
29
|
+
"wrap_in_seal", "verify_wrapped",
|
|
30
|
+
]
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""CSC-1 canonical JSON, the profile used by Crovia Seal (SPEC §4.2).
|
|
2
|
+
|
|
3
|
+
Strict subset of RFC 8785: object keys sorted by UTF-16 code units, integers
|
|
4
|
+
only (no floats), integers within the JavaScript safe range, no whitespace,
|
|
5
|
+
strings escaped per RFC 8785 §3.2.2.2. The output is byte-identical to the
|
|
6
|
+
Crovia Seal reference canonicalizer; tests assert this when that package is
|
|
7
|
+
installed.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
JS_SAFE_INT_MAX = (1 << 53) - 1
|
|
14
|
+
JS_SAFE_INT_MIN = -JS_SAFE_INT_MAX
|
|
15
|
+
|
|
16
|
+
_ESCAPES = {
|
|
17
|
+
"\"": "\\\"",
|
|
18
|
+
"\\": "\\\\",
|
|
19
|
+
"\b": "\\b",
|
|
20
|
+
"\f": "\\f",
|
|
21
|
+
"\n": "\\n",
|
|
22
|
+
"\r": "\\r",
|
|
23
|
+
"\t": "\\t",
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class CanonicalizationError(ValueError):
|
|
28
|
+
"""Raised when a value cannot be represented in CSC-1."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _utf16_key(s: str) -> tuple:
|
|
32
|
+
return tuple(int.from_bytes(s.encode("utf-16-be")[i:i + 2], "big")
|
|
33
|
+
for i in range(0, len(s.encode("utf-16-be")), 2))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _serialize_string(s: str) -> str:
|
|
37
|
+
out = ["\""]
|
|
38
|
+
for ch in s:
|
|
39
|
+
if ch in _ESCAPES:
|
|
40
|
+
out.append(_ESCAPES[ch])
|
|
41
|
+
elif ord(ch) < 0x20:
|
|
42
|
+
out.append(f"\\u{ord(ch):04x}")
|
|
43
|
+
else:
|
|
44
|
+
out.append(ch)
|
|
45
|
+
out.append("\"")
|
|
46
|
+
return "".join(out)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _serialize(value: Any) -> str:
|
|
50
|
+
if value is None:
|
|
51
|
+
return "null"
|
|
52
|
+
if value is True:
|
|
53
|
+
return "true"
|
|
54
|
+
if value is False:
|
|
55
|
+
return "false"
|
|
56
|
+
if isinstance(value, int):
|
|
57
|
+
if not JS_SAFE_INT_MIN <= value <= JS_SAFE_INT_MAX:
|
|
58
|
+
raise CanonicalizationError(f"integer outside JS safe range: {value}")
|
|
59
|
+
return str(value)
|
|
60
|
+
if isinstance(value, float):
|
|
61
|
+
raise CanonicalizationError("CSC-1 forbids floats; encode numbers as strings or integers")
|
|
62
|
+
if isinstance(value, str):
|
|
63
|
+
return _serialize_string(value)
|
|
64
|
+
if isinstance(value, (list, tuple)):
|
|
65
|
+
return "[" + ",".join(_serialize(v) for v in value) + "]"
|
|
66
|
+
if isinstance(value, dict):
|
|
67
|
+
for k in value:
|
|
68
|
+
if not isinstance(k, str):
|
|
69
|
+
raise CanonicalizationError("object keys must be strings")
|
|
70
|
+
items = sorted(value.items(), key=lambda kv: _utf16_key(kv[0]))
|
|
71
|
+
return "{" + ",".join(_serialize_string(k) + ":" + _serialize(v) for k, v in items) + "}"
|
|
72
|
+
raise CanonicalizationError(f"unsupported type: {type(value).__name__}")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def canonicalize(value: Any) -> bytes:
|
|
76
|
+
"""Return the CSC-1 encoding of `value` as UTF-8 bytes."""
|
|
77
|
+
return _serialize(value).encode("utf-8")
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
"""PNX — Proof of Non-Exfiltration for agent egress (TACET profile ``crovia.pnx.v1``).
|
|
2
|
+
|
|
3
|
+
TACET proves that a public surface did *not* contain something during a
|
|
4
|
+
witnessed window. PNX turns the same machinery inward: an egress witness sits
|
|
5
|
+
where an AI agent's outbound bytes are visible in clear (a local LLM proxy, a
|
|
6
|
+
corporate egress proxy, a CI sidecar) and commits a fingerprint of everything
|
|
7
|
+
that left. Afterwards the operator can prove, to anyone, that a set of
|
|
8
|
+
protected assets — API keys, customer records, source files — never appeared
|
|
9
|
+
in that traffic, without revealing the traffic or the assets.
|
|
10
|
+
|
|
11
|
+
Construction
|
|
12
|
+
------------
|
|
13
|
+
1. Fingerprinting (winnowing, Schleimer–Wilkerson–Aiken 2003). Every outbound
|
|
14
|
+
body is cut into k-grams of ``K_GRAM`` bytes; each k-gram is hashed with a
|
|
15
|
+
per-run salt; within every window of ``WINDOW`` consecutive hashes the
|
|
16
|
+
minimum is selected (rightmost on ties). The selected hashes are the body's
|
|
17
|
+
fingerprints. Guarantee: any byte string of length ``>= THRESHOLD`` shared by
|
|
18
|
+
an asset and a body yields at least one identical fingerprint on both sides.
|
|
19
|
+
2. Commitment. Fingerprints are keys of a depth-256 sparse Merkle map (SPEC §5)
|
|
20
|
+
with a constant leaf value. The run sheet — root, salt, parameters, byte and
|
|
21
|
+
body counts, first/last timestamp — is signed by the witness key.
|
|
22
|
+
3. Proof. For each asset the prover recomputes the asset's own fingerprints
|
|
23
|
+
and attaches a compact non-inclusion path for each against the run root. If
|
|
24
|
+
a fingerprint *is* present it attaches the inclusion path instead: the same
|
|
25
|
+
object is then evidence of exposure, not of its absence.
|
|
26
|
+
4. Anchoring. The run root is meant to be committed as a leaf of a TACET epoch
|
|
27
|
+
sheet (``epoch_leaf_key``), which gives it the epoch's drand opening and
|
|
28
|
+
Bitcoin closing. That step is the operator's, not this module's.
|
|
29
|
+
|
|
30
|
+
What a verified PNX proof does and does not say
|
|
31
|
+
-----------------------------------------------
|
|
32
|
+
It says: no substring of length ``>= THRESHOLD`` of any listed asset occurred in
|
|
33
|
+
the bytes the witness saw, and (for shorter assets) whether the exact k-grams
|
|
34
|
+
occurred. It does not say anything about traffic that bypassed the witness,
|
|
35
|
+
about paraphrase or re-encoding the witness did not normalise, or about assets
|
|
36
|
+
shorter than ``K_GRAM`` bytes, which are reported as ``undetectable`` and never
|
|
37
|
+
counted as clean.
|
|
38
|
+
"""
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import hashlib
|
|
42
|
+
import os
|
|
43
|
+
from dataclasses import dataclass, field
|
|
44
|
+
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple
|
|
45
|
+
|
|
46
|
+
from .canonical import canonicalize
|
|
47
|
+
from .hashing import prefixed, sha256, unprefixed
|
|
48
|
+
from .keys import SigningKey, verify_signature
|
|
49
|
+
from .smt import CompactPath, SparseMerkleMap, verify_inclusion, verify_non_inclusion
|
|
50
|
+
|
|
51
|
+
PROFILE = "crovia.pnx.v1"
|
|
52
|
+
K_GRAM = 32
|
|
53
|
+
WINDOW = 16
|
|
54
|
+
THRESHOLD = K_GRAM + WINDOW - 1 # 47 bytes: shared substrings at least this long are always detected
|
|
55
|
+
|
|
56
|
+
DOMAIN_FINGERPRINT = b"CROVIA-PNX-FP-v1\n"
|
|
57
|
+
DOMAIN_SHEET = b"CROVIA-PNX-SHEET-v1\n"
|
|
58
|
+
DOMAIN_LEAF_VALUE = b"CROVIA-PNX-PRESENT-v1\n"
|
|
59
|
+
PRESENT = sha256(DOMAIN_LEAF_VALUE)
|
|
60
|
+
|
|
61
|
+
VERDICT_ABSENT = "absent" # every fingerprint proven not in the map (guaranteed for len >= THRESHOLD)
|
|
62
|
+
VERDICT_ABSENT_PARTIAL = "absent-partial" # K_GRAM <= len < THRESHOLD: exact k-grams absent, guarantee does not apply
|
|
63
|
+
VERDICT_PRESENT = "present" # at least one fingerprint proven in the map
|
|
64
|
+
VERDICT_UNDETECTABLE = "undetectable" # len < K_GRAM: no k-gram can be formed
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# --------------------------------------------------------------------------- fingerprints
|
|
68
|
+
|
|
69
|
+
def kgram_hashes(data: bytes, salt: bytes, k: int = K_GRAM) -> List[bytes]:
|
|
70
|
+
"""Salted hash of every k-gram of ``data`` in order (empty if ``len(data) < k``)."""
|
|
71
|
+
if len(salt) != 16:
|
|
72
|
+
raise ValueError("salt must be 16 bytes")
|
|
73
|
+
prefix = DOMAIN_FINGERPRINT + salt
|
|
74
|
+
return [hashlib.sha256(prefix + data[i:i + k]).digest() for i in range(len(data) - k + 1)]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def winnow(hashes: Sequence[bytes], w: int = WINDOW) -> Set[bytes]:
|
|
78
|
+
"""Select the minimum of every window of ``w`` consecutive hashes (rightmost on ties)."""
|
|
79
|
+
n = len(hashes)
|
|
80
|
+
if n == 0:
|
|
81
|
+
return set()
|
|
82
|
+
if n < w:
|
|
83
|
+
return {min(hashes)}
|
|
84
|
+
out: Set[bytes] = set()
|
|
85
|
+
for i in range(n - w + 1):
|
|
86
|
+
window = hashes[i:i + w]
|
|
87
|
+
m = min(window)
|
|
88
|
+
# rightmost minimum, as in the paper, so runs of equal hashes select consistently
|
|
89
|
+
out.add(window[len(window) - 1 - window[::-1].index(m)])
|
|
90
|
+
return out
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def fingerprints(data: bytes, salt: bytes, k: int = K_GRAM, w: int = WINDOW) -> Set[bytes]:
|
|
94
|
+
"""Winnowed fingerprints of ``data``; the empty set if it is shorter than ``k``."""
|
|
95
|
+
return winnow(kgram_hashes(data, salt, k), w)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def asset_fingerprints(asset: bytes, salt: bytes, k: int = K_GRAM, w: int = WINDOW) -> Tuple[str, List[bytes]]:
|
|
99
|
+
"""Fingerprints to check for an asset and the detection class they carry.
|
|
100
|
+
|
|
101
|
+
Assets at or above ``THRESHOLD`` use winnowed fingerprints (guaranteed
|
|
102
|
+
detection). Shorter assets that still hold at least one k-gram use every
|
|
103
|
+
k-gram hash (best effort). Anything shorter is undetectable.
|
|
104
|
+
"""
|
|
105
|
+
if len(asset) < k:
|
|
106
|
+
return "undetectable", []
|
|
107
|
+
hashes = kgram_hashes(asset, salt, k)
|
|
108
|
+
if len(asset) >= k + w - 1:
|
|
109
|
+
return "guaranteed", sorted(winnow(hashes, w))
|
|
110
|
+
return "partial", sorted(set(hashes))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
# --------------------------------------------------------------------------- witness
|
|
114
|
+
|
|
115
|
+
@dataclass
|
|
116
|
+
class EgressWitness:
|
|
117
|
+
"""Accumulates fingerprints of outbound bodies for one run."""
|
|
118
|
+
|
|
119
|
+
run_id: str
|
|
120
|
+
salt: bytes = field(default_factory=lambda: os.urandom(16))
|
|
121
|
+
k: int = K_GRAM
|
|
122
|
+
w: int = WINDOW
|
|
123
|
+
_map: SparseMerkleMap = field(default_factory=SparseMerkleMap, repr=False)
|
|
124
|
+
bodies: int = 0
|
|
125
|
+
bytes_seen: int = 0
|
|
126
|
+
first_at: Optional[str] = None
|
|
127
|
+
last_at: Optional[str] = None
|
|
128
|
+
|
|
129
|
+
def ingest(self, body: bytes, at: str) -> int:
|
|
130
|
+
"""Record one outbound body observed at RFC 3339 time ``at``; returns new fingerprints added."""
|
|
131
|
+
added = 0
|
|
132
|
+
for fp in fingerprints(body, self.salt, self.k, self.w):
|
|
133
|
+
if fp not in self._map:
|
|
134
|
+
self._map.set(fp, PRESENT)
|
|
135
|
+
added += 1
|
|
136
|
+
self.bodies += 1
|
|
137
|
+
self.bytes_seen += len(body)
|
|
138
|
+
self.first_at = self.first_at or at
|
|
139
|
+
self.last_at = at
|
|
140
|
+
return added
|
|
141
|
+
|
|
142
|
+
@property
|
|
143
|
+
def root(self) -> bytes:
|
|
144
|
+
return self._map.root()
|
|
145
|
+
|
|
146
|
+
def sheet(self, witness: SigningKey, closed_at: str) -> Dict[str, Any]:
|
|
147
|
+
"""Signed run sheet: the only object that has to leave the perimeter."""
|
|
148
|
+
s: Dict[str, Any] = {
|
|
149
|
+
"profile": PROFILE,
|
|
150
|
+
"run_id": self.run_id,
|
|
151
|
+
"salt_hex": self.salt.hex(),
|
|
152
|
+
"params": {"k_gram": self.k, "window": self.w, "threshold": self.k + self.w - 1, "hash": "sha256"},
|
|
153
|
+
"egress": {"bodies": self.bodies, "bytes": self.bytes_seen, "first_at": self.first_at, "last_at": self.last_at},
|
|
154
|
+
"fingerprints": len(self._map),
|
|
155
|
+
"root": prefixed(self.root),
|
|
156
|
+
"closed_at": closed_at,
|
|
157
|
+
"witness": {"id": witness.id, "pubkey": witness.pubkey_json()},
|
|
158
|
+
}
|
|
159
|
+
sig = witness.sign(DOMAIN_SHEET + canonicalize(s))
|
|
160
|
+
s["signature"] = {"alg": "ed25519", "domain": DOMAIN_SHEET.strip().decode(), "sig_hex": sig.hex()}
|
|
161
|
+
return s
|
|
162
|
+
|
|
163
|
+
def prove(self, sheet: Dict[str, Any], assets: Iterable[Tuple[str, bytes]]) -> Dict[str, Any]:
|
|
164
|
+
"""Proof of Non-Exfiltration for labelled assets against this run's committed root."""
|
|
165
|
+
if unprefixed(sheet["root"]) != self.root:
|
|
166
|
+
raise ValueError("sheet root does not match the witness map")
|
|
167
|
+
out_assets = []
|
|
168
|
+
for label, data in assets:
|
|
169
|
+
klass, fps = asset_fingerprints(data, self.salt, self.k, self.w)
|
|
170
|
+
entry: Dict[str, Any] = {"label": label, "asset_len": len(data), "detection": klass,
|
|
171
|
+
"asset_sha256": prefixed(sha256(data)), "fingerprints": []}
|
|
172
|
+
present = 0
|
|
173
|
+
for fp in fps:
|
|
174
|
+
path = self._map.prove(fp)
|
|
175
|
+
hit = fp in self._map
|
|
176
|
+
present += hit
|
|
177
|
+
entry["fingerprints"].append({"key": fp.hex(), "present": hit, "path": path.to_json()})
|
|
178
|
+
if klass == "undetectable":
|
|
179
|
+
entry["verdict"] = VERDICT_UNDETECTABLE
|
|
180
|
+
elif present:
|
|
181
|
+
entry["verdict"] = VERDICT_PRESENT
|
|
182
|
+
else:
|
|
183
|
+
entry["verdict"] = VERDICT_ABSENT if klass == "guaranteed" else VERDICT_ABSENT_PARTIAL
|
|
184
|
+
out_assets.append(entry)
|
|
185
|
+
verdict = (VERDICT_PRESENT if any(a["verdict"] == VERDICT_PRESENT for a in out_assets)
|
|
186
|
+
else VERDICT_ABSENT if out_assets and all(a["verdict"] == VERDICT_ABSENT for a in out_assets)
|
|
187
|
+
else "mixed")
|
|
188
|
+
return {"profile": PROFILE, "proof_version": "crovia.pnx.proof.v1", "sheet": sheet,
|
|
189
|
+
"assets": out_assets, "verdict": verdict}
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def epoch_leaf_key(run_id: str) -> bytes:
|
|
193
|
+
"""Key under which a run root is committed into a TACET epoch map (namespace ``pnx/<run_id>``)."""
|
|
194
|
+
return sha256(("pnx/" + run_id).encode("utf-8"))
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# --------------------------------------------------------------------------- verification
|
|
198
|
+
|
|
199
|
+
@dataclass
|
|
200
|
+
class PnxVerifyResult:
|
|
201
|
+
ok: bool
|
|
202
|
+
verdict: str
|
|
203
|
+
errors: List[str] = field(default_factory=list)
|
|
204
|
+
warnings: List[str] = field(default_factory=list)
|
|
205
|
+
assets: Dict[str, str] = field(default_factory=dict)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _unsigned(sheet: Dict[str, Any]) -> Dict[str, Any]:
|
|
209
|
+
return {k: v for k, v in sheet.items() if k != "signature"}
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def verify_sheet(sheet: Dict[str, Any]) -> List[str]:
|
|
213
|
+
errors: List[str] = []
|
|
214
|
+
if sheet.get("profile") != PROFILE:
|
|
215
|
+
errors.append(f"unknown profile {sheet.get('profile')!r}")
|
|
216
|
+
try:
|
|
217
|
+
unprefixed(sheet["root"])
|
|
218
|
+
bytes.fromhex(sheet["salt_hex"])
|
|
219
|
+
except (KeyError, ValueError) as e:
|
|
220
|
+
errors.append(f"malformed sheet: {e}")
|
|
221
|
+
return errors
|
|
222
|
+
p = sheet.get("params") or {}
|
|
223
|
+
if p.get("hash") != "sha256" or p.get("threshold") != p.get("k_gram", 0) + p.get("window", 0) - 1:
|
|
224
|
+
errors.append("inconsistent params")
|
|
225
|
+
sig = sheet.get("signature") or {}
|
|
226
|
+
try:
|
|
227
|
+
ok = verify_signature(sheet["witness"]["pubkey"]["key_hex"], DOMAIN_SHEET + canonicalize(_unsigned(sheet)),
|
|
228
|
+
bytes.fromhex(sig.get("sig_hex", "")))
|
|
229
|
+
except (KeyError, ValueError):
|
|
230
|
+
ok = False
|
|
231
|
+
if not ok:
|
|
232
|
+
errors.append("witness signature invalid")
|
|
233
|
+
return errors
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def verify_pnx(proof: Dict[str, Any], assets: Optional[Dict[str, bytes]] = None) -> PnxVerifyResult:
|
|
237
|
+
"""Verify a PNX proof offline.
|
|
238
|
+
|
|
239
|
+
With ``assets`` (label -> bytes) the verifier recomputes every fingerprint
|
|
240
|
+
itself, so the prover cannot substitute keys. Without them it verifies the
|
|
241
|
+
paths against the keys the prover listed, which proves non-inclusion of
|
|
242
|
+
*those keys* only; the result carries a warning saying so.
|
|
243
|
+
"""
|
|
244
|
+
res = PnxVerifyResult(ok=True, verdict=proof.get("verdict", "?"))
|
|
245
|
+
sheet = proof.get("sheet") or {}
|
|
246
|
+
res.errors += verify_sheet(sheet)
|
|
247
|
+
if res.errors:
|
|
248
|
+
res.ok = False
|
|
249
|
+
return res
|
|
250
|
+
root = unprefixed(sheet["root"])
|
|
251
|
+
salt = bytes.fromhex(sheet["salt_hex"])
|
|
252
|
+
k, w = sheet["params"]["k_gram"], sheet["params"]["window"]
|
|
253
|
+
if assets is None:
|
|
254
|
+
res.warnings.append("assets not supplied: fingerprints taken from the proof, not recomputed")
|
|
255
|
+
|
|
256
|
+
computed_verdicts: Dict[str, str] = {}
|
|
257
|
+
for a in proof.get("assets", []):
|
|
258
|
+
label = a.get("label", "?")
|
|
259
|
+
listed = a.get("fingerprints", [])
|
|
260
|
+
if assets is not None:
|
|
261
|
+
if label not in assets:
|
|
262
|
+
res.errors.append(f"{label}: asset bytes not supplied")
|
|
263
|
+
continue
|
|
264
|
+
data = assets[label]
|
|
265
|
+
if prefixed(sha256(data)) != a.get("asset_sha256"):
|
|
266
|
+
res.errors.append(f"{label}: supplied asset does not match asset_sha256 in the proof")
|
|
267
|
+
continue
|
|
268
|
+
klass, expected = asset_fingerprints(data, salt, k, w)
|
|
269
|
+
if klass != a.get("detection"):
|
|
270
|
+
res.errors.append(f"{label}: detection class {a.get('detection')!r} does not match recomputed {klass!r}")
|
|
271
|
+
continue
|
|
272
|
+
if sorted(bytes.fromhex(f["key"]) for f in listed) != expected:
|
|
273
|
+
res.errors.append(f"{label}: fingerprint set does not match the asset")
|
|
274
|
+
continue
|
|
275
|
+
else:
|
|
276
|
+
klass = a.get("detection", "?")
|
|
277
|
+
present = 0
|
|
278
|
+
for f in listed:
|
|
279
|
+
key = bytes.fromhex(f["key"])
|
|
280
|
+
path = CompactPath.from_json(f["path"])
|
|
281
|
+
if f.get("present"):
|
|
282
|
+
if not verify_inclusion(root, key, PRESENT, path):
|
|
283
|
+
res.errors.append(f"{label}: inclusion path invalid for {f['key'][:16]}")
|
|
284
|
+
present += 1
|
|
285
|
+
elif not verify_non_inclusion(root, key, path):
|
|
286
|
+
res.errors.append(f"{label}: non-inclusion path invalid for {f['key'][:16]}")
|
|
287
|
+
if klass == "undetectable":
|
|
288
|
+
v = VERDICT_UNDETECTABLE
|
|
289
|
+
elif present:
|
|
290
|
+
v = VERDICT_PRESENT
|
|
291
|
+
else:
|
|
292
|
+
v = VERDICT_ABSENT if klass == "guaranteed" else VERDICT_ABSENT_PARTIAL
|
|
293
|
+
if v != a.get("verdict"):
|
|
294
|
+
res.errors.append(f"{label}: stated verdict {a.get('verdict')!r}, computed {v!r}")
|
|
295
|
+
computed_verdicts[label] = v
|
|
296
|
+
if v == VERDICT_UNDETECTABLE:
|
|
297
|
+
res.warnings.append(f"{label}: shorter than {k} bytes, cannot be fingerprinted; not counted as clean")
|
|
298
|
+
if v == VERDICT_ABSENT_PARTIAL:
|
|
299
|
+
res.warnings.append(f"{label}: shorter than the {k + w - 1}-byte guarantee; exact k-grams absent only")
|
|
300
|
+
|
|
301
|
+
res.assets = computed_verdicts
|
|
302
|
+
overall = (VERDICT_PRESENT if any(v == VERDICT_PRESENT for v in computed_verdicts.values())
|
|
303
|
+
else VERDICT_ABSENT if computed_verdicts and all(v == VERDICT_ABSENT for v in computed_verdicts.values())
|
|
304
|
+
else "mixed")
|
|
305
|
+
if overall != proof.get("verdict"):
|
|
306
|
+
res.errors.append(f"overall verdict {proof.get('verdict')!r} does not match computed {overall!r}")
|
|
307
|
+
res.verdict = overall
|
|
308
|
+
res.ok = not res.errors
|
|
309
|
+
return res
|