decisioncontrol 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- decisioncontrol/__init__.py +15 -0
- decisioncontrol/__main__.py +3 -0
- decisioncontrol/_outputs.py +28 -0
- decisioncontrol/cli.py +116 -0
- decisioncontrol/demo.py +70 -0
- decisioncontrol/evaluation.py +281 -0
- decisioncontrol/evidence.py +147 -0
- decisioncontrol/models.py +171 -0
- decisioncontrol/policy.py +32 -0
- decisioncontrol/store.py +317 -0
- decisioncontrol-0.1.0.dist-info/METADATA +333 -0
- decisioncontrol-0.1.0.dist-info/RECORD +16 -0
- decisioncontrol-0.1.0.dist-info/WHEEL +5 -0
- decisioncontrol-0.1.0.dist-info/entry_points.txt +2 -0
- decisioncontrol-0.1.0.dist-info/licenses/LICENSE +21 -0
- decisioncontrol-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Policy-constrained agent actions with durable local approval and evidence."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.1.0"
|
|
4
|
+
|
|
5
|
+
from .evidence import export_record, replay
|
|
6
|
+
from .evaluation import EvaluationCase, evaluate_cases, load_cases, replay_evaluation, write_evaluation
|
|
7
|
+
from .models import DecisionError, Evaluation, Recommendation, RoutingPolicy, StateError, Status, Ticket, Verdict
|
|
8
|
+
from .policy import evaluate
|
|
9
|
+
from .store import TicketStore
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"DecisionError", "Evaluation", "Recommendation", "RoutingPolicy", "StateError",
|
|
13
|
+
"Status", "Ticket", "TicketStore", "Verdict", "evaluate", "export_record", "replay",
|
|
14
|
+
"EvaluationCase", "evaluate_cases", "load_cases", "replay_evaluation", "write_evaluation",
|
|
15
|
+
]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Publish complete artifacts from staging directories on the same filesystem."""
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import tempfile
|
|
6
|
+
from typing import Iterator
|
|
7
|
+
|
|
8
|
+
from .models import DecisionError
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _check_target(target: Path, directory: bool) -> None:
|
|
12
|
+
if target.is_symlink() or (target.exists() and (
|
|
13
|
+
not directory or not target.is_dir() or any(target.iterdir()))):
|
|
14
|
+
raise DecisionError("output must be a new file or a new or empty directory")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@contextmanager
|
|
18
|
+
def staged_output(out: str | Path, *, directory: bool = True) -> Iterator[Path]:
|
|
19
|
+
target = Path(out)
|
|
20
|
+
_check_target(target, directory)
|
|
21
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
22
|
+
with tempfile.TemporaryDirectory(prefix=f".{target.name}-", dir=target.parent) as temporary:
|
|
23
|
+
staging = Path(temporary) / "output"
|
|
24
|
+
if directory:
|
|
25
|
+
staging.mkdir()
|
|
26
|
+
yield staging
|
|
27
|
+
_check_target(target, directory)
|
|
28
|
+
os.replace(staging, target)
|
decisioncontrol/cli.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""JSON command line interface for trusted local operators."""
|
|
2
|
+
import argparse
|
|
3
|
+
import json
|
|
4
|
+
import sqlite3
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from ._outputs import staged_output
|
|
9
|
+
from .evidence import RecordMismatch, export_record, replay
|
|
10
|
+
from .evaluation import evaluate_cases, load_cases, replay_evaluation, write_evaluation
|
|
11
|
+
from .models import DecisionError, Recommendation, RoutingPolicy, StateError
|
|
12
|
+
from .store import TicketStore
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _json_file(path):
|
|
16
|
+
with Path(path).open(encoding="utf-8") as stream:
|
|
17
|
+
return json.load(stream)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _parser():
|
|
21
|
+
parser = argparse.ArgumentParser(prog="decision-control")
|
|
22
|
+
parser.add_argument("--db", help="SQLite database path (required for storage commands)")
|
|
23
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
24
|
+
create = commands.add_parser("ticket-create")
|
|
25
|
+
create.add_argument("--ticket-id", required=True)
|
|
26
|
+
create.add_argument("--department", required=True)
|
|
27
|
+
for name in ("ticket-show", "show", "execute"):
|
|
28
|
+
commands.add_parser(name).add_argument("id")
|
|
29
|
+
submit = commands.add_parser("submit")
|
|
30
|
+
submit.add_argument("--proposal", required=True)
|
|
31
|
+
submit.add_argument("--policy", required=True)
|
|
32
|
+
submit.add_argument("--proposed-by", required=True)
|
|
33
|
+
commands.add_parser("pending")
|
|
34
|
+
for name in ("approve", "reject", "cancel"):
|
|
35
|
+
review = commands.add_parser(name)
|
|
36
|
+
review.add_argument("id")
|
|
37
|
+
review.add_argument("--reviewer", required=True)
|
|
38
|
+
review.add_argument("--note", default="", required=name == "cancel")
|
|
39
|
+
export = commands.add_parser("export")
|
|
40
|
+
export.add_argument("id")
|
|
41
|
+
export.add_argument("--out")
|
|
42
|
+
commands.add_parser("replay").add_argument("--record", required=True)
|
|
43
|
+
commands.add_parser("demo").add_argument("--out", required=True)
|
|
44
|
+
evaluation = commands.add_parser("evaluate", help="score supplied labeled recommendations; execute nothing")
|
|
45
|
+
evaluation.add_argument("--cases", required=True, help="labeled JSON Lines cases")
|
|
46
|
+
evaluation.add_argument("--policy", required=True)
|
|
47
|
+
evaluation.add_argument("--require", help="metric/support gates; omitted means UNGATED")
|
|
48
|
+
evaluation.add_argument("--out", required=True, help="new or empty output directory")
|
|
49
|
+
commands.add_parser("replay-evaluation").add_argument("--record", required=True)
|
|
50
|
+
return parser
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _run(args):
|
|
54
|
+
if args.command == "replay":
|
|
55
|
+
return replay(_json_file(args.record))
|
|
56
|
+
if args.command == "demo":
|
|
57
|
+
from .demo import run_demo
|
|
58
|
+
return run_demo(args.out)
|
|
59
|
+
if args.command == "evaluate":
|
|
60
|
+
requirements = _json_file(args.require) if args.require else None
|
|
61
|
+
report = evaluate_cases(load_cases(args.cases), RoutingPolicy.from_dict(_json_file(args.policy)), requirements)
|
|
62
|
+
return write_evaluation(report, args.out)
|
|
63
|
+
if args.command == "replay-evaluation":
|
|
64
|
+
return replay_evaluation(_json_file(args.record))
|
|
65
|
+
if not args.db:
|
|
66
|
+
raise DecisionError("--db is required for storage commands")
|
|
67
|
+
with TicketStore(args.db) as store:
|
|
68
|
+
if args.command == "ticket-create":
|
|
69
|
+
return store.create_ticket(args.ticket_id, args.department).to_dict()
|
|
70
|
+
if args.command == "ticket-show":
|
|
71
|
+
return store.get_ticket(args.id).to_dict()
|
|
72
|
+
if args.command == "submit":
|
|
73
|
+
return store.submit(Recommendation.from_dict(_json_file(args.proposal)),
|
|
74
|
+
RoutingPolicy.from_dict(_json_file(args.policy)), args.proposed_by)
|
|
75
|
+
if args.command == "pending":
|
|
76
|
+
return store.pending()
|
|
77
|
+
if args.command == "show":
|
|
78
|
+
return store.get(args.id)
|
|
79
|
+
if args.command in ("approve", "reject", "cancel"):
|
|
80
|
+
return getattr(store, args.command)(args.id, args.reviewer, args.note)
|
|
81
|
+
if args.command == "execute":
|
|
82
|
+
return store.execute(args.id)
|
|
83
|
+
if args.command == "export":
|
|
84
|
+
envelope = export_record(store.get(args.id))
|
|
85
|
+
if args.out:
|
|
86
|
+
with staged_output(args.out, directory=False) as staging:
|
|
87
|
+
with staging.open("x", encoding="utf-8") as stream:
|
|
88
|
+
json.dump(envelope, stream, indent=2, allow_nan=False)
|
|
89
|
+
stream.write("\n")
|
|
90
|
+
return envelope
|
|
91
|
+
raise DecisionError("unknown command")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def main(argv=None):
|
|
95
|
+
"""Return 0, invalid input/IO 2, or blocked lifecycle/evaluation gate 1."""
|
|
96
|
+
parser = _parser()
|
|
97
|
+
try:
|
|
98
|
+
args = parser.parse_args(argv)
|
|
99
|
+
result = _run(args)
|
|
100
|
+
except RecordMismatch as error:
|
|
101
|
+
print(f"decision-control: {error}", file=sys.stderr)
|
|
102
|
+
return 1 if args.command in ("replay", "replay-evaluation") else 2
|
|
103
|
+
except StateError as error:
|
|
104
|
+
print(f"decision-control: {error}", file=sys.stderr)
|
|
105
|
+
return 1
|
|
106
|
+
except (DecisionError, OSError, UnicodeError, json.JSONDecodeError, sqlite3.Error) as error:
|
|
107
|
+
print(f"decision-control: {error}", file=sys.stderr)
|
|
108
|
+
return 2
|
|
109
|
+
print(json.dumps(result, indent=2, allow_nan=False))
|
|
110
|
+
if args.command == "pending" and result.skipped:
|
|
111
|
+
for row in result.skipped:
|
|
112
|
+
print(f"decision-control: skipped corrupt decision {row['id']}: {row['error']}", file=sys.stderr)
|
|
113
|
+
return 1
|
|
114
|
+
if args.command == "execute" and result["status"] == "STALE":
|
|
115
|
+
return 1
|
|
116
|
+
return 1 if args.command == "evaluate" and result["report"]["disposition"] == "FAIL" else 0
|
decisioncontrol/demo.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Credential-free demonstration: hand-authored inputs, real SQLite mutations."""
|
|
2
|
+
import json
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from ._outputs import staged_output
|
|
6
|
+
from .evidence import export_record, replay
|
|
7
|
+
from .models import Recommendation, RoutingPolicy
|
|
8
|
+
from .store import TicketStore
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _save(path, value):
|
|
12
|
+
with path.open("x", encoding="utf-8") as stream:
|
|
13
|
+
json.dump(value, stream, indent=2, allow_nan=False)
|
|
14
|
+
stream.write("\n")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def run_demo(out):
|
|
18
|
+
with staged_output(out) as staging:
|
|
19
|
+
return _build_demo(staging, Path(out))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _build_demo(directory, published):
|
|
23
|
+
policy = RoutingPolicy("demo-policy-1", ("support", "engineering"),
|
|
24
|
+
("engineering",), ("local-reviewer",), 0.8)
|
|
25
|
+
cases = (("allow", "support", 0.95), ("review", "engineering", 0.95),
|
|
26
|
+
("deny", "finance", 0.99), ("abstain", "support", None))
|
|
27
|
+
db = directory / "tickets.sqlite3"
|
|
28
|
+
ids = {}
|
|
29
|
+
_save(directory / "policy.json", policy.to_dict())
|
|
30
|
+
with TicketStore(db) as store:
|
|
31
|
+
for name, destination, probability in cases:
|
|
32
|
+
store.create_ticket(f"demo-{name}", "intake")
|
|
33
|
+
proposal = Recommendation("reassign_ticket", f"demo-{name}", destination,
|
|
34
|
+
probability, "hand-authored-demo", "routing-question-1")
|
|
35
|
+
_save(directory / f"proposal-{name}.json", proposal.to_dict())
|
|
36
|
+
ids[name] = store.submit(proposal, policy, proposed_by="local-proposer")["id"]
|
|
37
|
+
store.execute(ids["allow"])
|
|
38
|
+
# Review is durable: close and reopen before approval by a trusted operator.
|
|
39
|
+
with TicketStore(db) as store:
|
|
40
|
+
pending_before_approval = [record["id"] for record in store.pending()]
|
|
41
|
+
store.approve(ids["review"], "local-reviewer", "Reviewed destination and frozen policy")
|
|
42
|
+
first = store.execute(ids["review"])
|
|
43
|
+
repeated = store.execute(ids["review"])
|
|
44
|
+
replay_results = {}
|
|
45
|
+
records = {}
|
|
46
|
+
tickets = {}
|
|
47
|
+
for name, _, _ in cases:
|
|
48
|
+
record = store.get(ids[name])
|
|
49
|
+
envelope = export_record(record)
|
|
50
|
+
record_path = directory / f"record-{name}.json"
|
|
51
|
+
_save(record_path, envelope)
|
|
52
|
+
# Replay the serialized evidence, not just the in-memory envelope.
|
|
53
|
+
with record_path.open(encoding="utf-8") as stream:
|
|
54
|
+
replay_results[name] = replay(json.load(stream))
|
|
55
|
+
records[name] = {"id": record["id"], "verdict": record["evaluation"]["verdict"],
|
|
56
|
+
"status": record["status"], "file": str(published / record_path.name)}
|
|
57
|
+
tickets[name] = store.get_ticket(f"demo-{name}").to_dict()
|
|
58
|
+
summary = {
|
|
59
|
+
"database": str(published / db.name),
|
|
60
|
+
"inputs": "Hand-authored probabilities and recommendations; no model or provider was called.",
|
|
61
|
+
"execution_boundary": "Actual local SQLite queue reassignment only; no external action.",
|
|
62
|
+
"reopened_before_approval": True,
|
|
63
|
+
"pending_before_approval": pending_before_approval,
|
|
64
|
+
"repeat_execution_same_record": first == repeated,
|
|
65
|
+
"decisions": records,
|
|
66
|
+
"tickets": tickets,
|
|
67
|
+
"replay": replay_results,
|
|
68
|
+
}
|
|
69
|
+
_save(directory / "summary.json", summary)
|
|
70
|
+
return summary
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
"""Offline policy evaluation on independently labeled, supplied recommendations.
|
|
2
|
+
|
|
3
|
+
No model is called and no action is executed. Wilson intervals assume independent
|
|
4
|
+
Bernoulli observations; synthetic or correlated cases do not validate deployment.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
import hashlib
|
|
9
|
+
import json
|
|
10
|
+
import math
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from ._outputs import staged_output
|
|
14
|
+
from .evidence import RecordMismatch, _canonical
|
|
15
|
+
from .models import DecisionError, Recommendation, RoutingPolicy, Ticket, Verdict, _fields, _probability, _text
|
|
16
|
+
from .policy import evaluate
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
METRICS = (
|
|
20
|
+
"auto_allow_rate", "unsafe_auto_allow_rate", "auto_route_error_rate",
|
|
21
|
+
"review_rate", "abstain_rate", "deny_rate", "required_review_recall",
|
|
22
|
+
"review_precision", "incorrect_approval_rate", "review_observation_rate",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class EvaluationCase:
|
|
28
|
+
id: str
|
|
29
|
+
ticket: Ticket
|
|
30
|
+
recommendation: Recommendation
|
|
31
|
+
safe_to_execute: bool
|
|
32
|
+
requires_review: bool
|
|
33
|
+
correct_destination: str
|
|
34
|
+
review_outcome: str | None = None
|
|
35
|
+
|
|
36
|
+
def __post_init__(self):
|
|
37
|
+
_text(self.id, "case id")
|
|
38
|
+
if not isinstance(self.ticket, Ticket) or not isinstance(self.recommendation, Recommendation):
|
|
39
|
+
raise DecisionError("case requires a Ticket and a Recommendation")
|
|
40
|
+
if type(self.safe_to_execute) is not bool or type(self.requires_review) is not bool:
|
|
41
|
+
raise DecisionError("safe_to_execute and requires_review labels must be booleans")
|
|
42
|
+
_text(self.correct_destination, "correct_destination")
|
|
43
|
+
if self.review_outcome is not None and self.review_outcome not in ("APPROVED", "REJECTED"):
|
|
44
|
+
raise DecisionError("review_outcome must be APPROVED, REJECTED, or null")
|
|
45
|
+
|
|
46
|
+
def to_dict(self):
|
|
47
|
+
return {"id": self.id, "ticket": self.ticket.to_dict(),
|
|
48
|
+
"recommendation": self.recommendation.to_dict(),
|
|
49
|
+
"labels": {"safe_to_execute": self.safe_to_execute,
|
|
50
|
+
"requires_review": self.requires_review,
|
|
51
|
+
"correct_destination": self.correct_destination},
|
|
52
|
+
"review_outcome": self.review_outcome}
|
|
53
|
+
|
|
54
|
+
@classmethod
|
|
55
|
+
def from_dict(cls, value):
|
|
56
|
+
raw = _fields(value, {"id", "ticket", "recommendation", "labels"}, {"review_outcome"})
|
|
57
|
+
labels = _fields(raw["labels"], {"safe_to_execute", "requires_review", "correct_destination"})
|
|
58
|
+
return cls(raw["id"], Ticket.from_dict(raw["ticket"]),
|
|
59
|
+
Recommendation.from_dict(raw["recommendation"]), **labels,
|
|
60
|
+
review_outcome=raw.get("review_outcome"))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def load_cases(path):
|
|
64
|
+
"""Load JSON Lines, rejecting malformed cases instead of excluding them from scores."""
|
|
65
|
+
cases = []
|
|
66
|
+
with Path(path).open(encoding="utf-8") as stream:
|
|
67
|
+
for line_number, line in enumerate(stream, 1):
|
|
68
|
+
if not line.strip():
|
|
69
|
+
continue
|
|
70
|
+
try:
|
|
71
|
+
cases.append(EvaluationCase.from_dict(json.loads(line)))
|
|
72
|
+
except (DecisionError, json.JSONDecodeError) as error:
|
|
73
|
+
raise DecisionError(f"{path}:{line_number}: {error}") from error
|
|
74
|
+
return cases
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _rate(numerator, denominator):
|
|
78
|
+
if denominator == 0:
|
|
79
|
+
return {"numerator": numerator, "denominator": 0, "value": None, "lower": None, "upper": None}
|
|
80
|
+
value = numerator / denominator
|
|
81
|
+
z = 1.959963984540054
|
|
82
|
+
z_squared = z * z
|
|
83
|
+
divisor = 1 + z_squared / denominator
|
|
84
|
+
center = (value + z_squared / (2 * denominator)) / divisor
|
|
85
|
+
half = z * math.sqrt(value * (1 - value) / denominator + z_squared / (4 * denominator ** 2)) / divisor
|
|
86
|
+
return {"numerator": numerator, "denominator": denominator, "value": value,
|
|
87
|
+
"lower": 0.0 if numerator == 0 else max(0.0, center - half),
|
|
88
|
+
"upper": 1.0 if numerator == denominator else min(1.0, center + half)}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _positive_int(value, name):
|
|
92
|
+
if type(value) is not int or value < 1:
|
|
93
|
+
raise DecisionError(f"{name} must be a positive integer")
|
|
94
|
+
return value
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _requirements(value):
|
|
98
|
+
if value is None:
|
|
99
|
+
return None
|
|
100
|
+
raw = _fields(value, {"min_cases", "bounds"})
|
|
101
|
+
minimum = _positive_int(raw["min_cases"], "min_cases")
|
|
102
|
+
if not isinstance(raw["bounds"], dict) or not raw["bounds"]:
|
|
103
|
+
raise DecisionError("bounds must contain at least one metric requirement")
|
|
104
|
+
bounds = {}
|
|
105
|
+
for name, bound in raw["bounds"].items():
|
|
106
|
+
if name not in METRICS:
|
|
107
|
+
raise DecisionError(f"unknown metric: {name}")
|
|
108
|
+
bound = _fields(bound, (), {"min", "max", "statistic", "min_denominator"})
|
|
109
|
+
if not ("min" in bound or "max" in bound):
|
|
110
|
+
raise DecisionError(f"{name} needs a min or max bound")
|
|
111
|
+
statistic = bound.get("statistic", "value")
|
|
112
|
+
if statistic not in ("value", "lower", "upper"):
|
|
113
|
+
raise DecisionError(f"{name}: statistic must be value, lower, or upper")
|
|
114
|
+
normalized = {"statistic": statistic,
|
|
115
|
+
"min_denominator": _positive_int(bound.get("min_denominator", 1), "min_denominator")}
|
|
116
|
+
for side in ("min", "max"):
|
|
117
|
+
if side in bound:
|
|
118
|
+
normalized[side] = _probability(bound[side], f"{name}.{side}")
|
|
119
|
+
if normalized.get("min", 0) > normalized.get("max", 1):
|
|
120
|
+
raise DecisionError(f"{name}: min exceeds max")
|
|
121
|
+
bounds[name] = normalized
|
|
122
|
+
return {"min_cases": minimum, "bounds": bounds}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _gates(metrics, count, requirements):
|
|
126
|
+
if requirements is None:
|
|
127
|
+
return []
|
|
128
|
+
enough_cases = count >= requirements["min_cases"]
|
|
129
|
+
gates = [{"metric": "cases", "statistic": "value", "observed": count,
|
|
130
|
+
"min": requirements["min_cases"], "max": None,
|
|
131
|
+
"denominator": count, "min_denominator": requirements["min_cases"],
|
|
132
|
+
"passed": enough_cases, "reason": None if enough_cases else "insufficient_cases"}]
|
|
133
|
+
for name, bound in sorted(requirements["bounds"].items()):
|
|
134
|
+
metric = metrics[name]
|
|
135
|
+
observed = metric[bound["statistic"]]
|
|
136
|
+
reason = None
|
|
137
|
+
if observed is None:
|
|
138
|
+
reason = "unmeasured"
|
|
139
|
+
elif metric["denominator"] < bound["min_denominator"]:
|
|
140
|
+
reason = "insufficient_denominator"
|
|
141
|
+
elif observed < bound.get("min", 0) or observed > bound.get("max", 1):
|
|
142
|
+
reason = "outside_bound"
|
|
143
|
+
gates.append({"metric": name, "statistic": bound["statistic"], "observed": observed,
|
|
144
|
+
"min": bound.get("min"), "max": bound.get("max"),
|
|
145
|
+
"denominator": metric["denominator"], "min_denominator": bound["min_denominator"],
|
|
146
|
+
"passed": reason is None, "reason": reason})
|
|
147
|
+
return gates
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def evaluate_cases(cases, policy: RoutingPolicy, requirements=None) -> dict:
|
|
151
|
+
"""Evaluate policy recommendations, not real execution or unauthenticated reviewers.
|
|
152
|
+
|
|
153
|
+
Unsafe automatic allowance means ALLOW despite a negative safety label or a
|
|
154
|
+
required-review label. Incorrect approval means an observed APPROVED outcome
|
|
155
|
+
on a REVIEW case with a negative safety label or an incorrect destination.
|
|
156
|
+
"""
|
|
157
|
+
if not isinstance(policy, RoutingPolicy):
|
|
158
|
+
raise DecisionError("policy must be a RoutingPolicy")
|
|
159
|
+
requirements = _requirements(requirements)
|
|
160
|
+
seen = set()
|
|
161
|
+
results = []
|
|
162
|
+
verdict_counts = {verdict: 0 for verdict in Verdict}
|
|
163
|
+
unsafe = wrong_route = required = reviewed_required = observed_reviews = approvals = incorrect_approvals = 0
|
|
164
|
+
for case in cases:
|
|
165
|
+
if not isinstance(case, EvaluationCase):
|
|
166
|
+
raise DecisionError("cases must contain EvaluationCase instances")
|
|
167
|
+
if case.id in seen:
|
|
168
|
+
raise DecisionError(f"duplicate case id: {case.id}")
|
|
169
|
+
seen.add(case.id)
|
|
170
|
+
evaluation = evaluate(case.recommendation, case.ticket, policy)
|
|
171
|
+
verdict = evaluation.verdict
|
|
172
|
+
verdict_counts[verdict] += 1
|
|
173
|
+
automatic = verdict == Verdict.ALLOW
|
|
174
|
+
reviewing = verdict == Verdict.REVIEW
|
|
175
|
+
incorrect_destination = case.recommendation.destination != case.correct_destination
|
|
176
|
+
auto_unsafe = automatic and (not case.safe_to_execute or case.requires_review)
|
|
177
|
+
auto_incorrect = automatic and incorrect_destination
|
|
178
|
+
observed = reviewing and case.review_outcome is not None
|
|
179
|
+
approved = reviewing and case.review_outcome == "APPROVED"
|
|
180
|
+
approval_incorrect = approved and (not case.safe_to_execute or incorrect_destination)
|
|
181
|
+
unsafe += auto_unsafe
|
|
182
|
+
wrong_route += auto_incorrect
|
|
183
|
+
required += case.requires_review
|
|
184
|
+
reviewed_required += reviewing and case.requires_review
|
|
185
|
+
observed_reviews += observed
|
|
186
|
+
approvals += approved
|
|
187
|
+
incorrect_approvals += approval_incorrect
|
|
188
|
+
results.append({"case": case.to_dict(), "evaluation": evaluation.to_dict(),
|
|
189
|
+
"auto_allow_unsafe": auto_unsafe, "auto_route_incorrect": auto_incorrect,
|
|
190
|
+
"approval_incorrect": approval_incorrect if approved else None})
|
|
191
|
+
count = len(results)
|
|
192
|
+
if count == 0:
|
|
193
|
+
raise DecisionError("evaluation requires at least one labeled case")
|
|
194
|
+
allowed = verdict_counts[Verdict.ALLOW]
|
|
195
|
+
reviewed = verdict_counts[Verdict.REVIEW]
|
|
196
|
+
metrics = {
|
|
197
|
+
"auto_allow_rate": _rate(allowed, count),
|
|
198
|
+
"unsafe_auto_allow_rate": _rate(unsafe, allowed),
|
|
199
|
+
"auto_route_error_rate": _rate(wrong_route, allowed),
|
|
200
|
+
"review_rate": _rate(reviewed, count),
|
|
201
|
+
"abstain_rate": _rate(verdict_counts[Verdict.ABSTAIN], count),
|
|
202
|
+
"deny_rate": _rate(verdict_counts[Verdict.DENY], count),
|
|
203
|
+
"required_review_recall": _rate(reviewed_required, required),
|
|
204
|
+
"review_precision": _rate(reviewed_required, reviewed),
|
|
205
|
+
"incorrect_approval_rate": _rate(incorrect_approvals, approvals),
|
|
206
|
+
"review_observation_rate": _rate(observed_reviews, reviewed),
|
|
207
|
+
}
|
|
208
|
+
gates = _gates(metrics, count, requirements)
|
|
209
|
+
disposition = "UNGATED" if requirements is None else ("PASS" if all(g["passed"] for g in gates) else "FAIL")
|
|
210
|
+
return {"schema_version": 1, "kind": "policy_evaluation", "source": "supplied_recommendations",
|
|
211
|
+
"cases": count, "confidence_level": 0.95, "policy": policy.to_dict(),
|
|
212
|
+
"requirements": requirements, "metrics": metrics, "gates": gates,
|
|
213
|
+
"disposition": disposition, "case_results": results}
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def replay_evaluation(envelope: dict) -> dict:
|
|
217
|
+
"""Check the complete report and independently recompute metrics/gates from frozen cases."""
|
|
218
|
+
raw = _fields(envelope, {"report", "sha256"})
|
|
219
|
+
report = raw["report"]
|
|
220
|
+
digest = raw["sha256"]
|
|
221
|
+
if (not isinstance(digest, str) or len(digest) != 64
|
|
222
|
+
or any(character not in "0123456789abcdef" for character in digest)):
|
|
223
|
+
raise DecisionError("sha256 must be a lowercase hexadecimal SHA-256 digest")
|
|
224
|
+
if hashlib.sha256(_canonical(report)).hexdigest() != digest:
|
|
225
|
+
raise RecordMismatch("evaluation checksum mismatch")
|
|
226
|
+
report = _fields(report, {"schema_version", "kind", "source", "cases", "confidence_level", "policy",
|
|
227
|
+
"requirements", "metrics", "gates", "disposition", "case_results"})
|
|
228
|
+
if type(report["schema_version"]) is not int or report["schema_version"] != 1:
|
|
229
|
+
raise DecisionError("unsupported evaluation schema_version")
|
|
230
|
+
if not isinstance(report["case_results"], list):
|
|
231
|
+
raise DecisionError("case_results must be a list")
|
|
232
|
+
cases = []
|
|
233
|
+
for row in report["case_results"]:
|
|
234
|
+
row = _fields(row, {"case", "evaluation", "auto_allow_unsafe", "auto_route_incorrect", "approval_incorrect"})
|
|
235
|
+
cases.append(EvaluationCase.from_dict(row["case"]))
|
|
236
|
+
recomputed = evaluate_cases(cases, RoutingPolicy.from_dict(report["policy"]), report["requirements"])
|
|
237
|
+
if _canonical(recomputed) != _canonical(report):
|
|
238
|
+
raise RecordMismatch("evaluation report disagrees with its frozen cases, policy, or requirements")
|
|
239
|
+
return {"valid": True, "cases": recomputed["cases"], "disposition": recomputed["disposition"]}
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _number(value):
|
|
243
|
+
return "unmeasured" if value is None else f"{value:.6g}"
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def render_evaluation(report, digest):
|
|
247
|
+
lines = [f"# Policy evaluation: {report['disposition']}", "",
|
|
248
|
+
f"- Labeled cases: {report['cases']}", f"- Canonical report SHA-256: `{digest}`",
|
|
249
|
+
"- Supplied recommendations only; no model call or action execution.", "",
|
|
250
|
+
"| Metric | Numerator | Denominator | Estimate | 95% lower | 95% upper |",
|
|
251
|
+
"| --- | ---: | ---: | ---: | ---: | ---: |"]
|
|
252
|
+
for name, metric in report["metrics"].items():
|
|
253
|
+
lines.append(f"| {name} | {metric['numerator']} | {metric['denominator']} | "
|
|
254
|
+
f"{_number(metric['value'])} | {_number(metric['lower'])} | {_number(metric['upper'])} |")
|
|
255
|
+
if report["gates"]:
|
|
256
|
+
lines.extend(["", "| Gate | Statistic | Observed | Bounds | Minimum support | Result |",
|
|
257
|
+
"| --- | --- | ---: | --- | ---: | --- |"])
|
|
258
|
+
for gate in report["gates"]:
|
|
259
|
+
bounds = ", ".join(f"{side} {gate[side]}" for side in ("min", "max") if gate[side] is not None)
|
|
260
|
+
result = "PASS" if gate["passed"] else f"FAIL ({gate['reason']})"
|
|
261
|
+
lines.append(f"| {gate['metric']} | {gate['statistic']} | {_number(gate['observed'])} | "
|
|
262
|
+
f"{bounds} | {gate['min_denominator']} | {result} |")
|
|
263
|
+
lines.extend(["", "Wilson intervals assume independent observations. Hand-authored, correlated or",
|
|
264
|
+
"unrepresentative cases do not establish deployed safety or provider quality.",
|
|
265
|
+
"Unknown denominators are unmeasured, never zero-error proof. Point-estimate",
|
|
266
|
+
"bounds are not confidence guarantees; use upper/lower bounds and enough support.", ""])
|
|
267
|
+
return "\n".join(lines)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def write_evaluation(report: dict, out) -> dict:
|
|
271
|
+
"""Preserve a complete, replayable report in a new or empty output directory."""
|
|
272
|
+
digest = hashlib.sha256(_canonical(report)).hexdigest()
|
|
273
|
+
envelope = {"report": report, "sha256": digest}
|
|
274
|
+
replay_evaluation(envelope)
|
|
275
|
+
with staged_output(out) as directory:
|
|
276
|
+
with (directory / "evaluation.json").open("x", encoding="utf-8") as stream:
|
|
277
|
+
json.dump(envelope, stream, indent=2, ensure_ascii=False, allow_nan=False)
|
|
278
|
+
stream.write("\n")
|
|
279
|
+
with (directory / "EVALUATION.md").open("x", encoding="utf-8") as stream:
|
|
280
|
+
stream.write(render_evaluation(report, digest))
|
|
281
|
+
return envelope
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Portable evidence replay without a model call or an action side effect.
|
|
2
|
+
|
|
3
|
+
A checksum detects corruption, not a malicious writer who can replace the checksum.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from datetime import datetime, timedelta
|
|
8
|
+
import hashlib
|
|
9
|
+
import json
|
|
10
|
+
from uuid import UUID
|
|
11
|
+
|
|
12
|
+
from .models import DecisionError, Evaluation, Recommendation, RoutingPolicy, Status, Ticket, _fields, _text
|
|
13
|
+
from .policy import INITIAL_STATUS, evaluate
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RecordMismatch(DecisionError):
|
|
17
|
+
"""A checksum or recomputed decision disagrees with recorded evidence."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
RECORD_FIELDS = {"schema_version", "id", "created_at", "recommendation", "ticket", "policy",
|
|
21
|
+
"evaluation", "status", "review", "result", "events"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _canonical(value):
|
|
25
|
+
try:
|
|
26
|
+
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
|
27
|
+
allow_nan=False).encode("utf-8")
|
|
28
|
+
except (TypeError, ValueError) as error:
|
|
29
|
+
raise DecisionError(f"evidence must be finite JSON: {error}") from error
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _timestamp(value):
|
|
33
|
+
_text(value, "timestamp")
|
|
34
|
+
try:
|
|
35
|
+
parsed = datetime.fromisoformat(value)
|
|
36
|
+
except ValueError as error:
|
|
37
|
+
raise DecisionError("timestamp must be ISO 8601 UTC") from error
|
|
38
|
+
if parsed.tzinfo is None or parsed.utcoffset() != timedelta(0):
|
|
39
|
+
raise DecisionError("timestamp must be ISO 8601 UTC")
|
|
40
|
+
return value
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _validate_record(record):
|
|
44
|
+
schema = record.get("schema_version") if isinstance(record, Mapping) else None
|
|
45
|
+
fields = RECORD_FIELDS | {"produced_by", "proposed_by", "policy_sha256"} if schema == 3 else RECORD_FIELDS
|
|
46
|
+
raw = _fields(record, fields)
|
|
47
|
+
if type(raw["schema_version"]) is not int or raw["schema_version"] not in (1, 2, 3):
|
|
48
|
+
raise DecisionError("unsupported evidence schema_version")
|
|
49
|
+
_text(raw["id"], "decision id")
|
|
50
|
+
try:
|
|
51
|
+
UUID(raw["id"])
|
|
52
|
+
except ValueError as error:
|
|
53
|
+
raise DecisionError("decision id must be a UUID") from error
|
|
54
|
+
_timestamp(raw["created_at"])
|
|
55
|
+
proposal = Recommendation.from_dict(raw["recommendation"])
|
|
56
|
+
ticket = Ticket.from_dict(raw["ticket"])
|
|
57
|
+
policy = RoutingPolicy.from_dict(raw["policy"])
|
|
58
|
+
if raw["schema_version"] == 3:
|
|
59
|
+
_text(raw["proposed_by"], "proposed_by")
|
|
60
|
+
producer = _fields(raw["produced_by"], {"name", "version"})
|
|
61
|
+
if producer["name"] != "decisioncontrol":
|
|
62
|
+
raise DecisionError("unexpected evidence producer")
|
|
63
|
+
_text(producer["version"], "producer version")
|
|
64
|
+
if raw["policy_sha256"] != hashlib.sha256(_canonical(raw["policy"])).hexdigest():
|
|
65
|
+
raise RecordMismatch("policy_sha256 does not match the frozen policy")
|
|
66
|
+
recorded = Evaluation.from_dict(raw["evaluation"])
|
|
67
|
+
expected = evaluate(proposal, ticket, policy)
|
|
68
|
+
if recorded != expected:
|
|
69
|
+
raise RecordMismatch("recorded verdict/reasons do not match the frozen inputs and policy")
|
|
70
|
+
try:
|
|
71
|
+
status = Status(raw["status"])
|
|
72
|
+
except (ValueError, TypeError) as error:
|
|
73
|
+
raise DecisionError("unknown lifecycle status") from error
|
|
74
|
+
events = raw["events"]
|
|
75
|
+
if not isinstance(events, list) or not events:
|
|
76
|
+
raise DecisionError("evidence must contain a submission event")
|
|
77
|
+
state = INITIAL_STATUS[expected.verdict]
|
|
78
|
+
review_event = None
|
|
79
|
+
for index, event in enumerate(events):
|
|
80
|
+
name = event.get("event") if isinstance(event, Mapping) else None
|
|
81
|
+
fields = {"event", "at", "reviewer", "note"} if name == "CANCELLED" else {"event", "at"}
|
|
82
|
+
event = _fields(event, fields)
|
|
83
|
+
at = _timestamp(event["at"])
|
|
84
|
+
if index == 0:
|
|
85
|
+
if name != "SUBMITTED" or at != raw["created_at"]:
|
|
86
|
+
raise DecisionError("first event must match submission time")
|
|
87
|
+
continue
|
|
88
|
+
if name in ("APPROVED", "REJECTED") and state == Status.PENDING_REVIEW.value:
|
|
89
|
+
review_event = event
|
|
90
|
+
state = Status.READY.value if name == "APPROVED" else Status.REJECTED.value
|
|
91
|
+
elif name in ("EXECUTED", "STALE") and state == Status.READY.value:
|
|
92
|
+
state = name
|
|
93
|
+
elif (name == "CANCELLED" and raw["schema_version"] >= 2
|
|
94
|
+
and state in (Status.READY.value, Status.PENDING_REVIEW.value)):
|
|
95
|
+
if (raw["schema_version"] == 3 and expected.verdict.value == "REVIEW"
|
|
96
|
+
and event["reviewer"] == raw["proposed_by"]):
|
|
97
|
+
raise RecordMismatch("proposer cannot cancel their own REVIEW decision")
|
|
98
|
+
if event["reviewer"] not in policy.reviewers:
|
|
99
|
+
raise RecordMismatch("cancelling reviewer is not authorized by the frozen policy")
|
|
100
|
+
if not isinstance(event["note"], str) or not event["note"].strip():
|
|
101
|
+
raise DecisionError("cancellation must contain a nonempty reason")
|
|
102
|
+
state = Status.CANCELLED.value
|
|
103
|
+
else:
|
|
104
|
+
raise RecordMismatch("event history contains a prohibited transition")
|
|
105
|
+
if status.value != state:
|
|
106
|
+
raise RecordMismatch("lifecycle status does not match event history")
|
|
107
|
+
review = raw["review"]
|
|
108
|
+
if review_event is None:
|
|
109
|
+
if review is not None:
|
|
110
|
+
raise RecordMismatch("review has no corresponding review event")
|
|
111
|
+
else:
|
|
112
|
+
review = _fields(review, {"decision", "reviewer", "note", "at"})
|
|
113
|
+
if review["decision"] != review_event["event"] or review["at"] != review_event["at"]:
|
|
114
|
+
raise RecordMismatch("review does not match its event")
|
|
115
|
+
if raw["schema_version"] == 3 and review["reviewer"] == raw["proposed_by"]:
|
|
116
|
+
raise RecordMismatch("proposer cannot review their own REVIEW decision")
|
|
117
|
+
if review["reviewer"] not in policy.reviewers:
|
|
118
|
+
raise RecordMismatch("reviewer is not authorized by the frozen policy")
|
|
119
|
+
if not isinstance(review["note"], str):
|
|
120
|
+
raise DecisionError("review note must be text")
|
|
121
|
+
if status == Status.EXECUTED:
|
|
122
|
+
result = Ticket.from_dict(raw["result"])
|
|
123
|
+
expected_result = Ticket(ticket.id, proposal.destination, ticket.revision + 1)
|
|
124
|
+
if result != expected_result:
|
|
125
|
+
raise RecordMismatch("execution result does not match the approved action")
|
|
126
|
+
elif raw["result"] is not None:
|
|
127
|
+
raise RecordMismatch("an unexecuted decision cannot contain an execution result")
|
|
128
|
+
return {"valid": True, "decision_id": raw["id"], "verdict": expected.verdict.value,
|
|
129
|
+
"status": status.value}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def export_record(record: dict) -> dict:
|
|
133
|
+
"""Validate and checksum the entire record, including review and execution history."""
|
|
134
|
+
_validate_record(record)
|
|
135
|
+
return {"record": record, "sha256": hashlib.sha256(_canonical(record)).hexdigest()}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def replay(envelope: dict) -> dict:
|
|
139
|
+
"""Check integrity and reproduce authorization from frozen inputs; never execute."""
|
|
140
|
+
raw = _fields(envelope, {"record", "sha256"})
|
|
141
|
+
digest = raw["sha256"]
|
|
142
|
+
if (not isinstance(digest, str) or len(digest) != 64
|
|
143
|
+
or any(character not in "0123456789abcdef" for character in digest)):
|
|
144
|
+
raise DecisionError("sha256 must be a 64-character digest")
|
|
145
|
+
if hashlib.sha256(_canonical(raw["record"])).hexdigest() != digest:
|
|
146
|
+
raise RecordMismatch("evidence checksum mismatch")
|
|
147
|
+
return _validate_record(raw["record"])
|