consequa-x 0.1.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- consequa_x/__init__.py +22 -0
- consequa_x/__main__.py +3 -0
- consequa_x/adapters/__init__.py +5 -0
- consequa_x/adapters/ollama.py +350 -0
- consequa_x/attribution.py +407 -0
- consequa_x/benchmark/__init__.py +1 -0
- consequa_x/benchmark/assessment_metrics.py +54 -0
- consequa_x/benchmark/calibration.py +787 -0
- consequa_x/benchmark/calibration_export.py +139 -0
- consequa_x/benchmark/calibration_mechanisms.py +266 -0
- consequa_x/benchmark/decision_coalition.py +254 -0
- consequa_x/benchmark/decision_contrastive.py +448 -0
- consequa_x/benchmark/decision_replay.py +329 -0
- consequa_x/benchmark/decision_shapley.py +628 -0
- consequa_x/benchmark/evaluation.py +187 -0
- consequa_x/benchmark/interaction_export.py +220 -0
- consequa_x/benchmark/interaction_mechanisms.py +192 -0
- consequa_x/benchmark/interval_comparison.py +276 -0
- consequa_x/benchmark/intervals.py +164 -0
- consequa_x/benchmark/payments.py +426 -0
- consequa_x/benchmark/policy_experiment.py +298 -0
- consequa_x/benchmark/policy_runner.py +234 -0
- consequa_x/benchmark/runner.py +241 -0
- consequa_x/benchmark/study.py +183 -0
- consequa_x/benchmark/targets.py +149 -0
- consequa_x/benchmark/templates.py +102 -0
- consequa_x/cli.py +1191 -0
- consequa_x/explanation.py +130 -0
- consequa_x/graph.py +142 -0
- consequa_x/models.py +532 -0
- consequa_x/policies.py +248 -0
- consequa_x/py.typed +0 -0
- consequa_x/schema.py +159 -0
- consequa_x/schemas/trace-v0.1.json +543 -0
- consequa_x/verification.py +192 -0
- consequa_x-0.1.0a2.dist-info/METADATA +303 -0
- consequa_x-0.1.0a2.dist-info/RECORD +41 -0
- consequa_x-0.1.0a2.dist-info/WHEEL +5 -0
- consequa_x-0.1.0a2.dist-info/entry_points.txt +2 -0
- consequa_x-0.1.0a2.dist-info/licenses/LICENSE +21 -0
- consequa_x-0.1.0a2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Bounded policy evaluation with failures separate from verified outcomes."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Iterable
|
|
7
|
+
|
|
8
|
+
from ..models import to_dict
|
|
9
|
+
from ..policies import Policy, PolicyError
|
|
10
|
+
from .assessment_metrics import assessment_metrics
|
|
11
|
+
from .policy_runner import MAX_TRIAL_SEED, run_policy_scenario
|
|
12
|
+
from .runner import Episode, ScenarioConfig, demo_scenarios
|
|
13
|
+
from .templates import DEFAULT_REFUND_TEMPLATE, RefundTaskTemplate
|
|
14
|
+
from .targets import action_linked_consequence
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class CallBudgetExceeded(PolicyError):
|
|
18
|
+
"""The evaluation refused another policy call before invoking the provider."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class BudgetedPolicy:
|
|
22
|
+
"""Count attempted decisions, including provider and protocol failures."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, policy: Policy, max_calls: int):
|
|
25
|
+
if type(max_calls) is not int or max_calls < 1:
|
|
26
|
+
raise ValueError("max_calls must be a positive integer")
|
|
27
|
+
self.policy = policy
|
|
28
|
+
self.max_calls = max_calls
|
|
29
|
+
self.calls = 0
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def metadata(self):
|
|
33
|
+
return to_dict(self.policy.metadata)
|
|
34
|
+
|
|
35
|
+
def decide(self, request, *, seed):
|
|
36
|
+
if self.calls >= self.max_calls:
|
|
37
|
+
raise CallBudgetExceeded("The policy-call budget is exhausted")
|
|
38
|
+
self.calls += 1
|
|
39
|
+
return self.policy.decide(request, seed=seed)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class EvaluationSample:
|
|
44
|
+
index: int
|
|
45
|
+
scenario: str
|
|
46
|
+
seed: int
|
|
47
|
+
policy_calls: int
|
|
48
|
+
episode: Episode | None = None
|
|
49
|
+
error_type: str | None = None
|
|
50
|
+
error: str | None = None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class PolicyEvaluation:
|
|
55
|
+
samples: tuple[EvaluationSample, ...]
|
|
56
|
+
policy_metadata: dict
|
|
57
|
+
planned_samples: int
|
|
58
|
+
max_calls: int
|
|
59
|
+
calls: int
|
|
60
|
+
budget_exhausted: bool
|
|
61
|
+
|
|
62
|
+
def summary(self) -> dict:
|
|
63
|
+
counts = {name: 0 for name in ("satisfied", "violated", "unknown")}
|
|
64
|
+
origin = {name: 0 for name in ("confirmed", "other_actor", "unknown")}
|
|
65
|
+
alignment = {name: 0 for name in ("consistent", "contradicted", "unassessable")}
|
|
66
|
+
linked = {name: 0 for name in ("satisfied", "not_satisfied", "unknown")}
|
|
67
|
+
completed = 0
|
|
68
|
+
for sample in self.samples:
|
|
69
|
+
if sample.episode is None:
|
|
70
|
+
continue
|
|
71
|
+
completed += 1
|
|
72
|
+
verification = sample.episode.trace.verifications[-1]
|
|
73
|
+
counts[verification.status.value] += 1
|
|
74
|
+
origin[verification.origin.value] += 1
|
|
75
|
+
label = sample.episode.explanation.assessment_alignment
|
|
76
|
+
alignment[label] = alignment.get(label, 0) + 1
|
|
77
|
+
value = action_linked_consequence(sample.episode.trace.action, verification)
|
|
78
|
+
linked["unknown" if value is None else "satisfied" if value else "not_satisfied"] += 1
|
|
79
|
+
return {
|
|
80
|
+
"planned_samples": self.planned_samples,
|
|
81
|
+
"attempted_samples": len(self.samples),
|
|
82
|
+
"completed_samples": completed,
|
|
83
|
+
"failed_samples": len(self.samples) - completed,
|
|
84
|
+
"not_started_samples": self.planned_samples - len(self.samples),
|
|
85
|
+
"policy_calls": self.calls,
|
|
86
|
+
"max_calls": self.max_calls,
|
|
87
|
+
"budget_exhausted": self.budget_exhausted,
|
|
88
|
+
"verification": counts,
|
|
89
|
+
"origin": origin,
|
|
90
|
+
"assessment_alignment": alignment,
|
|
91
|
+
"action_linked_consequence": linked,
|
|
92
|
+
"assessment_metrics": assessment_metrics(
|
|
93
|
+
(sample.episode.trace.assessment.status,
|
|
94
|
+
sample.episode.explanation.assessment_alignment)
|
|
95
|
+
for sample in self.samples if sample.episode is not None
|
|
96
|
+
),
|
|
97
|
+
"known_outcome_samples": counts["satisfied"] + counts["violated"],
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def evaluate_policy(policy: Policy, configs: Iterable[ScenarioConfig] | None = None, *,
|
|
102
|
+
samples: int = 2, seed: int = 0, max_calls: int = 100,
|
|
103
|
+
max_attempts: int = 2,
|
|
104
|
+
templates: Mapping[str, RefundTaskTemplate] | None = None) -> PolicyEvaluation:
|
|
105
|
+
"""Evaluate fresh worlds; provider/protocol failures never become violations.
|
|
106
|
+
|
|
107
|
+
Model calls are serial and bounded. A failed episode is retained as a failure
|
|
108
|
+
receipt, not dropped or replaced by a second draw. No model-level attribution
|
|
109
|
+
or generalization claim follows from these template-level smoke evaluations.
|
|
110
|
+
"""
|
|
111
|
+
if type(samples) is not int or samples < 1:
|
|
112
|
+
raise ValueError("samples must be a positive integer")
|
|
113
|
+
if type(seed) is not int or seed < 0:
|
|
114
|
+
raise ValueError("seed must be a nonnegative integer")
|
|
115
|
+
if type(max_attempts) is not int or not 1 <= max_attempts <= 5:
|
|
116
|
+
raise ValueError("max_attempts must be in 1..5")
|
|
117
|
+
configurations = tuple(demo_scenarios() if configs is None else configs)
|
|
118
|
+
if not configurations or any(not isinstance(config, ScenarioConfig) for config in configurations):
|
|
119
|
+
raise ValueError("configs must contain at least one ScenarioConfig")
|
|
120
|
+
if len({config.name for config in configurations}) != len(configurations):
|
|
121
|
+
raise ValueError("scenario names must be unique")
|
|
122
|
+
if templates is None:
|
|
123
|
+
template_map = {config.name: DEFAULT_REFUND_TEMPLATE for config in configurations}
|
|
124
|
+
else:
|
|
125
|
+
if not isinstance(templates, Mapping) or set(templates) != {
|
|
126
|
+
config.name for config in configurations
|
|
127
|
+
} or any(not isinstance(item, RefundTaskTemplate) for item in templates.values()):
|
|
128
|
+
raise ValueError("templates must map every scenario name to a RefundTaskTemplate")
|
|
129
|
+
template_map = dict(templates)
|
|
130
|
+
if seed + len(configurations) * samples - 1 > MAX_TRIAL_SEED:
|
|
131
|
+
raise ValueError("Evaluation seed schedule exceeds the model seed range")
|
|
132
|
+
bounded = BudgetedPolicy(policy, max_calls)
|
|
133
|
+
records = []
|
|
134
|
+
exhausted = False
|
|
135
|
+
for config in configurations:
|
|
136
|
+
for _ in range(samples):
|
|
137
|
+
if bounded.calls >= max_calls:
|
|
138
|
+
exhausted = True
|
|
139
|
+
break
|
|
140
|
+
index = len(records)
|
|
141
|
+
calls_before = bounded.calls
|
|
142
|
+
try:
|
|
143
|
+
episode = run_policy_scenario(config, bounded, seed=seed + index,
|
|
144
|
+
max_attempts=max_attempts,
|
|
145
|
+
template=template_map[config.name])
|
|
146
|
+
record = EvaluationSample(index, config.name, seed + index,
|
|
147
|
+
bounded.calls - calls_before, episode)
|
|
148
|
+
except PolicyError as error:
|
|
149
|
+
exhausted = isinstance(error, CallBudgetExceeded)
|
|
150
|
+
record = EvaluationSample(index, config.name, seed + index,
|
|
151
|
+
bounded.calls - calls_before,
|
|
152
|
+
error_type=type(error).__name__, error=str(error))
|
|
153
|
+
records.append(record)
|
|
154
|
+
if exhausted:
|
|
155
|
+
break
|
|
156
|
+
if exhausted:
|
|
157
|
+
break
|
|
158
|
+
return PolicyEvaluation(tuple(records), bounded.metadata, len(configurations) * samples,
|
|
159
|
+
max_calls, bounded.calls, exhausted)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def evaluation_sample_record(sample: EvaluationSample, *, root: Path | None = None) -> dict:
|
|
163
|
+
"""Return an evaluator receipt without duplicating hidden truth into a trace."""
|
|
164
|
+
record = {
|
|
165
|
+
"index": sample.index, "scenario": sample.scenario, "seed": sample.seed,
|
|
166
|
+
"policy_calls": sample.policy_calls,
|
|
167
|
+
"status": "completed" if sample.episode is not None else "failed",
|
|
168
|
+
}
|
|
169
|
+
if sample.episode is None:
|
|
170
|
+
record.update(error_type=sample.error_type, error=sample.error)
|
|
171
|
+
else:
|
|
172
|
+
episode = sample.episode
|
|
173
|
+
result = episode.trace.verifications[-1]
|
|
174
|
+
record.update(
|
|
175
|
+
action_selected=episode.action_selected,
|
|
176
|
+
verified_outcome=result.status.value,
|
|
177
|
+
effect_origin=result.origin.value,
|
|
178
|
+
reported_assessment=episode.trace.assessment.status.value,
|
|
179
|
+
assessment_alignment=episode.explanation.assessment_alignment,
|
|
180
|
+
diagnosis=episode.explanation.diagnosis,
|
|
181
|
+
action_linked_consequence_satisfied=action_linked_consequence(episode.trace.action, result),
|
|
182
|
+
)
|
|
183
|
+
if root is not None:
|
|
184
|
+
stem = f"episode_{sample.index:04d}.json"
|
|
185
|
+
record.update(trace=f"traces/{stem}", explanation=f"explanations/{stem}",
|
|
186
|
+
evaluator_truth=f"evaluator_truth/{stem}")
|
|
187
|
+
return record
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""Frozen finite-mixture refund interaction evidence; no live model calls.
|
|
2
|
+
|
|
3
|
+
Run with ``python -m consequa_x.benchmark.interaction_export --output DIR
|
|
4
|
+
--source-revision COMMIT`` after freezing and checking the source revision.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from contextlib import ExitStack
|
|
8
|
+
import gzip
|
|
9
|
+
from hashlib import sha256
|
|
10
|
+
import json
|
|
11
|
+
from math import factorial, fsum
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from .. import __version__
|
|
15
|
+
from ..models import to_dict
|
|
16
|
+
from ..schema import get_trace_schema
|
|
17
|
+
from .decision_coalition import replay_decision_coalition
|
|
18
|
+
from .evaluation import BudgetedPolicy
|
|
19
|
+
from .interaction_mechanisms import COMPONENTS, INTERACTION_MECHANISMS, TARGETS
|
|
20
|
+
from .policy_runner import MAX_TRIAL_SEED, run_policy_scenario
|
|
21
|
+
from .targets import episode_outcome
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _write(path, value):
|
|
25
|
+
path.write_text(json.dumps(to_dict(value), indent=2, sort_keys=True, allow_nan=False) + "\n",
|
|
26
|
+
encoding="utf-8", newline="\n")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _line(stream, value):
|
|
30
|
+
stream.write((json.dumps(to_dict(value), sort_keys=True, allow_nan=False,
|
|
31
|
+
separators=(",", ":")) + "\n").encode("utf-8"))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _shares(values):
|
|
35
|
+
if any(value is None for value in values):
|
|
36
|
+
return None
|
|
37
|
+
return [fsum(factorial(mask.bit_count()) * factorial(2 - mask.bit_count()) / factorial(3)
|
|
38
|
+
* (values[mask | (1 << player)] - values[mask])
|
|
39
|
+
for mask in range(8) if not mask & (1 << player)) for player in range(3)]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _summarize_case(mechanism, records):
|
|
43
|
+
result = {"actual_policy_calls": sum(record["policy_calls"] for record in records), "games": {}}
|
|
44
|
+
for game in ("strict", "on_reach"):
|
|
45
|
+
branches = [record for record in records if record["coalition_game"] == game
|
|
46
|
+
and record["record_type"] == "branch"]
|
|
47
|
+
target_summary = {}
|
|
48
|
+
for target in TARGETS:
|
|
49
|
+
values, statuses = [], []
|
|
50
|
+
for mask in range(8):
|
|
51
|
+
selected = [record for record in branches if record["mask"] == mask]
|
|
52
|
+
positive = [record for record in selected if record["component_weight"] > 0]
|
|
53
|
+
invalid = fsum(record["component_weight"] for record in positive
|
|
54
|
+
if record["status"] == "invalid")
|
|
55
|
+
failed = fsum(record["component_weight"] for record in positive
|
|
56
|
+
if record["status"] == "failed")
|
|
57
|
+
unstarted = fsum(record["component_weight"] for record in positive
|
|
58
|
+
if record["status"] == "unstarted")
|
|
59
|
+
unknown = fsum(record["component_weight"] for record in positive
|
|
60
|
+
if record["status"] == "completed"
|
|
61
|
+
and record["bad_targets"][target] is None)
|
|
62
|
+
known = fsum(record["component_weight"] for record in positive
|
|
63
|
+
if record["status"] == "completed"
|
|
64
|
+
and record["bad_targets"][target] is not None)
|
|
65
|
+
values.append(None if invalid or failed or unstarted or unknown else
|
|
66
|
+
fsum(record["component_weight"] * record["bad_targets"][target]
|
|
67
|
+
for record in positive))
|
|
68
|
+
statuses.append({"known_mass": known, "unknown_mass": unknown,
|
|
69
|
+
"invalid_mass": invalid, "failed_mass": failed, "unstarted_mass": unstarted})
|
|
70
|
+
shares = _shares(values)
|
|
71
|
+
endpoint = values[7] - values[0] if values[7] is not None and values[0] is not None else None
|
|
72
|
+
target_summary[target] = {"coalition_bad_rates": values, "mass_by_mask": statuses,
|
|
73
|
+
"shapley_values": shares, "endpoint_difference": endpoint,
|
|
74
|
+
"selection_retry_interaction": (values[3] - values[1] - values[2] + values[0]
|
|
75
|
+
if all(values[mask] is not None for mask in (0, 1, 2, 3)) else None),
|
|
76
|
+
"numeric_available": shares is not None}
|
|
77
|
+
result["games"][game] = {"targets": target_summary,
|
|
78
|
+
"branch_status_counts": {status: sum(record["status"] == status for record in branches)
|
|
79
|
+
for status in ("completed", "invalid", "failed", "unstarted")},
|
|
80
|
+
"inactive_held_branch_count": sum(bool(record["inactive_held_indices"]) for record in branches),
|
|
81
|
+
"reference": mechanism.analytic_reference(coalition_game=game)}
|
|
82
|
+
return result
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def export_refund_interactions(output, *, source_revision, seed=4_000_000, mechanisms=None):
|
|
86
|
+
"""Enumerate actual worlds for every component/game/mask, retaining refusals.
|
|
87
|
+
|
|
88
|
+
Each component/game receives a separate budget, including three planted
|
|
89
|
+
factual calls. Every requested mask is retained. Budgets are never redrawn
|
|
90
|
+
or enlarged, and zero-mass mixture components remain in the evidence.
|
|
91
|
+
``source_revision`` is caller-supplied provenance, not inspected Git state.
|
|
92
|
+
"""
|
|
93
|
+
if type(source_revision) is not str or not source_revision.strip():
|
|
94
|
+
raise ValueError("source_revision must be a nonempty frozen source identifier")
|
|
95
|
+
names = tuple(INTERACTION_MECHANISMS if mechanisms is None else mechanisms)
|
|
96
|
+
if (not names or any(type(name) is not str or name not in INTERACTION_MECHANISMS for name in names)
|
|
97
|
+
or len(set(names)) != len(names)):
|
|
98
|
+
raise ValueError("mechanisms must be unique known names")
|
|
99
|
+
if type(seed) is not int or not 4_000_000 <= seed <= MAX_TRIAL_SEED - len(names) * 1000:
|
|
100
|
+
raise ValueError("seed must reserve a disjoint valid phase window starting at 4000000 or later")
|
|
101
|
+
root = Path(output)
|
|
102
|
+
if root.exists() and (not root.is_dir() or any(root.iterdir())):
|
|
103
|
+
raise ValueError("refund interactions require a new or empty output directory")
|
|
104
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
105
|
+
schedule = []
|
|
106
|
+
for case_index, name in enumerate(names):
|
|
107
|
+
for game_index, game in enumerate(("strict", "on_reach")):
|
|
108
|
+
for component_index, component in enumerate(COMPONENTS):
|
|
109
|
+
first = seed + case_index * 1000 + game_index * 400 + component_index * 100
|
|
110
|
+
schedule.append({"case": name, "coalition_game": game, "component_index": component_index,
|
|
111
|
+
"fresh_selection": component[0], "fresh_retry": component[1],
|
|
112
|
+
"component_weight": INTERACTION_MECHANISMS[name].component_weights()[component_index],
|
|
113
|
+
"factual_world_seed": first, "factual_policy_seed": first,
|
|
114
|
+
"branch_continuation_seeds_by_mask": [first + 1 + mask for mask in range(8)],
|
|
115
|
+
"max_calls": INTERACTION_MECHANISMS[name].max_calls})
|
|
116
|
+
_write(root / "protocol.json", {"package_version": __version__, "source_revision": source_revision,
|
|
117
|
+
"source_revision_trust": "caller supplied; exporter does not inspect Git",
|
|
118
|
+
"design": "full four-component deterministic mixture, each of eight held masks, both games",
|
|
119
|
+
"sampling_claim": "finite enumeration of a declared Bernoulli population law; not stochastic observations",
|
|
120
|
+
"seed": seed, "model_phase_rule": "world/model schedule seed * 6 + reached position",
|
|
121
|
+
"mask_order": list(range(8)), "targets": TARGETS, "schedule": schedule,
|
|
122
|
+
"cases": {name: to_dict(INTERACTION_MECHANISMS[name]) for name in names},
|
|
123
|
+
"retention": "every planned branch including failed, invalid, unstarted and zero-weight components",
|
|
124
|
+
"budget_rule": "per component/game budget includes three factual calls; stop starting slots at exhaustion",
|
|
125
|
+
"references": {name: {game: INTERACTION_MECHANISMS[name].analytic_reference(coalition_game=game)
|
|
126
|
+
for game in ("strict", "on_reach")} for name in names},
|
|
127
|
+
"limitations": ["synthetic public-only policy, no live model or independence claim",
|
|
128
|
+
"exact mixture law does not validate finite permutation sampling",
|
|
129
|
+
"held decision effects are positional policy interventions, not context/fault attribution",
|
|
130
|
+
"zero-weight components are evidence only and do not change the population expectation"]})
|
|
131
|
+
_write(root / "trace-schema.json", get_trace_schema())
|
|
132
|
+
retained = {name: [] for name in names}
|
|
133
|
+
with ExitStack() as stack:
|
|
134
|
+
raw = stack.enter_context((root / "evidence.jsonl.gz").open("wb"))
|
|
135
|
+
evidence = stack.enter_context(gzip.GzipFile(filename="", fileobj=raw, mode="wb", mtime=0))
|
|
136
|
+
raw_gold = stack.enter_context((root / "evaluator-truth.jsonl.gz").open("wb"))
|
|
137
|
+
gold = stack.enter_context(gzip.GzipFile(filename="", fileobj=raw_gold, mode="wb", mtime=0))
|
|
138
|
+
for entry in schedule:
|
|
139
|
+
name, game = entry["case"], entry["coalition_game"]
|
|
140
|
+
mechanism = INTERACTION_MECHANISMS[name]
|
|
141
|
+
component = (entry["fresh_selection"], entry["fresh_retry"])
|
|
142
|
+
policy = mechanism.make_policy(factual_policy_seed=entry["factual_policy_seed"], component=component)
|
|
143
|
+
budget = BudgetedPolicy(policy, entry["max_calls"])
|
|
144
|
+
factual = run_policy_scenario(mechanism.scenario_config(), budget,
|
|
145
|
+
seed=entry["factual_world_seed"], policy_seed=entry["factual_policy_seed"], max_attempts=2)
|
|
146
|
+
base = {"case": name, "coalition_game": game, "component_index": entry["component_index"],
|
|
147
|
+
"component_weight": entry["component_weight"], "policy_metadata": policy.metadata}
|
|
148
|
+
factual_record = {**base, "record_type": "factual", "policy_calls": budget.calls,
|
|
149
|
+
"trace": factual.trace.to_dict(), "explanation": factual.explanation.to_dict()}
|
|
150
|
+
_line(evidence, factual_record)
|
|
151
|
+
retained[name].append(factual_record)
|
|
152
|
+
_line(gold, {**base, "record_type": "factual", "seed": factual.seed,
|
|
153
|
+
"config": to_dict(factual.config), "records": to_dict(factual.ground_truth)})
|
|
154
|
+
for mask, continuation_seed in enumerate(entry["branch_continuation_seeds_by_mask"]):
|
|
155
|
+
held = tuple(index for index in range(3) if mask & (1 << index))
|
|
156
|
+
record = {**base, "record_type": "branch", "mask": mask,
|
|
157
|
+
"held_indices": list(held), "continuation_seed": continuation_seed,
|
|
158
|
+
"policy_calls": 0, "inactive_held_indices": [], "decisions": [],
|
|
159
|
+
"status": "unstarted", "bad_targets": {target: None for target in TARGETS},
|
|
160
|
+
"trace": None, "explanation": None}
|
|
161
|
+
if budget.calls < budget.max_calls:
|
|
162
|
+
replay = replay_decision_coalition(factual, budget, held_indices=held,
|
|
163
|
+
continuation_seed=continuation_seed, coalition_game=game)
|
|
164
|
+
record.update(status=replay.status, summary=replay.summary(),
|
|
165
|
+
decisions=to_dict(replay.decisions), policy_calls=replay.policy_calls,
|
|
166
|
+
inactive_held_indices=list(replay.inactive_held_indices))
|
|
167
|
+
if replay.episode is not None:
|
|
168
|
+
outcome = episode_outcome(replay.episode)
|
|
169
|
+
record.update(trace=replay.episode.trace.to_dict(),
|
|
170
|
+
explanation=replay.episode.explanation.to_dict(),
|
|
171
|
+
bad_targets={target: (None if outcome.target_value(target) is None else
|
|
172
|
+
int(not outcome.target_value(target))) for target in TARGETS})
|
|
173
|
+
_line(gold, {**base, "record_type": "branch", "mask": mask,
|
|
174
|
+
"seed": replay.episode.seed, "config": to_dict(replay.episode.config),
|
|
175
|
+
"records": to_dict(replay.episode.ground_truth)})
|
|
176
|
+
record.update(budget_calls_after=budget.calls, budget_max_calls=budget.max_calls)
|
|
177
|
+
_line(evidence, record)
|
|
178
|
+
retained[name].append(record)
|
|
179
|
+
summary = {"design": "exact deterministic-mixture world enumeration",
|
|
180
|
+
"cases": {name: _summarize_case(INTERACTION_MECHANISMS[name], retained[name]) for name in names},
|
|
181
|
+
"actual_policy_calls": sum(record["policy_calls"] for rows in retained.values() for record in rows),
|
|
182
|
+
"additional_reference_policy_calls": 0,
|
|
183
|
+
"completed": True}
|
|
184
|
+
_write(root / "summary.json", summary)
|
|
185
|
+
text = ["# Refund interaction enumeration", "", "This artifact enumerates four deterministic fresh-choice components, all eight held masks, and both explicitly named games. It is not a set of live stochastic samples.", "",
|
|
186
|
+
"The factual refund, retry and final stop are planted. All targets come from actual payment readback; evaluator records are stored separately.", "",
|
|
187
|
+
"| Case | Game | Consequence shares | Interaction |", "| --- | --- | --- | --- |"]
|
|
188
|
+
for name, item in summary["cases"].items():
|
|
189
|
+
for game, result in item["games"].items():
|
|
190
|
+
target = result["targets"]["consequence"]
|
|
191
|
+
text.append(f"| {name} | {game} | {target['shapley_values']} | {target['selection_retry_interaction']} |")
|
|
192
|
+
text += ["", f"Actual synthetic policy calls: {summary['actual_policy_calls']}.", "",
|
|
193
|
+
"A positive invalid, unknown, failed or unstarted mixture weight refuses the numerical vector. Budget controls retain the reference law while refusing incomplete enumerations. Zero-weight components remain in the evidence but do not affect that law.", "",
|
|
194
|
+
"Run the independent auditor in `scripts/audit_refund_interactions.py` against this directory. Permutation-sampler accuracy, live-model studies, context/fault attribution and cross-domain recovery require separate experiments."]
|
|
195
|
+
(root / "README.md").write_text("\n".join(text) + "\n", encoding="utf-8", newline="\n")
|
|
196
|
+
_write(root / "manifest.json", {"package_version": __version__,
|
|
197
|
+
"completed": True, "status": "completed", "artifacts": [
|
|
198
|
+
{"path": path.relative_to(root).as_posix(),
|
|
199
|
+
"sha256": sha256(path.read_bytes()).hexdigest(), "bytes": path.stat().st_size}
|
|
200
|
+
for path in sorted(root.rglob("*")) if path.is_file() and path != root / "manifest.json"]})
|
|
201
|
+
return summary
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def main(argv=None):
|
|
205
|
+
import argparse
|
|
206
|
+
|
|
207
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
208
|
+
parser.add_argument("--output", required=True)
|
|
209
|
+
parser.add_argument("--source-revision", required=True)
|
|
210
|
+
parser.add_argument("--seed", type=int, default=4_000_000)
|
|
211
|
+
parser.add_argument("--mechanism", action="append", choices=tuple(INTERACTION_MECHANISMS))
|
|
212
|
+
args = parser.parse_args(argv)
|
|
213
|
+
result = export_refund_interactions(args.output, source_revision=args.source_revision,
|
|
214
|
+
seed=args.seed, mechanisms=args.mechanism)
|
|
215
|
+
print(json.dumps({"output": str(Path(args.output).resolve()),
|
|
216
|
+
"actual_policy_calls": result["actual_policy_calls"]}, sort_keys=True))
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
if __name__ == "__main__":
|
|
220
|
+
main()
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Public-only refund policies and independent population interaction laws.
|
|
2
|
+
|
|
3
|
+
The factual refund/retry/stop is planted at three declared phase keys. Fresh
|
|
4
|
+
choices either follow a seeded Bernoulli law or one explicitly named component
|
|
5
|
+
of its four-point deterministic mixture. Neither policy sees evaluator truth.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
import math
|
|
10
|
+
import random
|
|
11
|
+
from types import MappingProxyType
|
|
12
|
+
|
|
13
|
+
from ..models import AssessmentStatus, ResponseStatus
|
|
14
|
+
from ..policies import PolicyDecision, PolicyError, PolicyReply, PolicyRequest, validate_seed
|
|
15
|
+
from .decision_coalition import validate_coalition_game
|
|
16
|
+
from .payments import Fault
|
|
17
|
+
from .policy_runner import MAX_TRIAL_SEED, run_policy_scenario
|
|
18
|
+
from .runner import RefundContext, ScenarioConfig
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
TARGETS = ("action", "consequence", "action_linked_consequence")
|
|
22
|
+
COMPONENTS = ((False, False), (False, True), (True, False), (True, True))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _probability(value, name):
|
|
26
|
+
if type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 1:
|
|
27
|
+
raise ValueError(f"{name} must be a finite probability in 0..1")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class RefundInteractionPolicy:
|
|
32
|
+
selection_probability: float
|
|
33
|
+
retry_probability: float
|
|
34
|
+
factual_policy_seed: int
|
|
35
|
+
fresh_selection: bool | None = None
|
|
36
|
+
fresh_retry: bool | None = None
|
|
37
|
+
|
|
38
|
+
def __post_init__(self):
|
|
39
|
+
_probability(self.selection_probability, "selection_probability")
|
|
40
|
+
_probability(self.retry_probability, "retry_probability")
|
|
41
|
+
if type(self.factual_policy_seed) is not int or not 0 <= self.factual_policy_seed <= MAX_TRIAL_SEED:
|
|
42
|
+
raise ValueError("factual_policy_seed is outside the valid world/model phase schedule")
|
|
43
|
+
if not ((self.fresh_selection is None and self.fresh_retry is None)
|
|
44
|
+
or (type(self.fresh_selection) is bool and type(self.fresh_retry) is bool)):
|
|
45
|
+
raise ValueError("fresh component choices must both be booleans or both be None")
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def metadata(self):
|
|
49
|
+
component = self.fresh_selection is not None
|
|
50
|
+
return {
|
|
51
|
+
"provider": "synthetic", "policy": "conditioned-refund-interaction", "version": "1",
|
|
52
|
+
"selection_probability": self.selection_probability,
|
|
53
|
+
"retry_probability": self.retry_probability,
|
|
54
|
+
"fresh_rule": {
|
|
55
|
+
"kind": "deterministic_mixture_component" if component else "seeded_bernoulli",
|
|
56
|
+
"selection": self.fresh_selection, "retry": self.fresh_retry,
|
|
57
|
+
"rng": None if component else "python_random.Random(model_phase_seed)",
|
|
58
|
+
"is_live_stochastic_sample": False,
|
|
59
|
+
},
|
|
60
|
+
"conditioning": {
|
|
61
|
+
"kind": "predeclared_factual_refund_retry_stop_intervention",
|
|
62
|
+
"naturally_sampled_factual": False,
|
|
63
|
+
"reserved_model_phase_seeds": [self.factual_policy_seed * 6 + i for i in range(3)],
|
|
64
|
+
"operations": ["refund", "retry", "stop"],
|
|
65
|
+
},
|
|
66
|
+
"hidden_evidence_inputs": False,
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
def decide(self, request: PolicyRequest, *, seed: int):
|
|
70
|
+
if not isinstance(request, PolicyRequest):
|
|
71
|
+
raise PolicyError("Expected a PolicyRequest")
|
|
72
|
+
validate_seed(seed)
|
|
73
|
+
base = self.factual_policy_seed * 6
|
|
74
|
+
if seed in (base, base + 1, base + 2):
|
|
75
|
+
operation = ("refund", "retry", "stop")[seed - base]
|
|
76
|
+
source = "reserved_factual_intervention"
|
|
77
|
+
elif request.stage == "select":
|
|
78
|
+
selected = (random.Random(seed).random() < self.selection_probability
|
|
79
|
+
if self.fresh_selection is None else self.fresh_selection)
|
|
80
|
+
operation = "refund" if selected else "stop"
|
|
81
|
+
source = self.metadata["fresh_rule"]["kind"]
|
|
82
|
+
elif request.responses[-1].status is ResponseStatus.TIMEOUT and request.attempts_remaining:
|
|
83
|
+
retry = (random.Random(seed).random() < self.retry_probability
|
|
84
|
+
if self.fresh_retry is None else self.fresh_retry)
|
|
85
|
+
operation = "retry" if retry else "stop"
|
|
86
|
+
source = self.metadata["fresh_rule"]["kind"]
|
|
87
|
+
else:
|
|
88
|
+
operation, source = "stop", "fixed_final_stop"
|
|
89
|
+
return PolicyReply(PolicyDecision(operation, AssessmentStatus.UNKNOWN,
|
|
90
|
+
"Transparent fixture choice from the declared public-stage rule."),
|
|
91
|
+
{"seed": seed, "decision_source": source})
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@dataclass(frozen=True)
|
|
95
|
+
class RefundInteractionMechanism:
|
|
96
|
+
name: str
|
|
97
|
+
selection_probability: float = .5
|
|
98
|
+
retry_probability: float = .5
|
|
99
|
+
use_idempotency: bool = False
|
|
100
|
+
include_provenance: bool = True
|
|
101
|
+
read_lag_ms: int = 0
|
|
102
|
+
max_calls: int = 100
|
|
103
|
+
|
|
104
|
+
def __post_init__(self):
|
|
105
|
+
if type(self.name) is not str or not self.name.strip():
|
|
106
|
+
raise ValueError("name must be a nonempty string")
|
|
107
|
+
_probability(self.selection_probability, "selection_probability")
|
|
108
|
+
_probability(self.retry_probability, "retry_probability")
|
|
109
|
+
if type(self.use_idempotency) is not bool or type(self.include_provenance) is not bool:
|
|
110
|
+
raise ValueError("idempotency and provenance flags must be booleans")
|
|
111
|
+
if type(self.read_lag_ms) is not int or self.read_lag_ms < 0:
|
|
112
|
+
raise ValueError("read_lag_ms must be a nonnegative integer")
|
|
113
|
+
if type(self.max_calls) is not int or self.max_calls < 3:
|
|
114
|
+
raise ValueError("max_calls must leave three calls for the factual capture")
|
|
115
|
+
|
|
116
|
+
def make_policy(self, *, factual_policy_seed, component=None):
|
|
117
|
+
if component is not None and component not in COMPONENTS:
|
|
118
|
+
raise ValueError("component must be one of the four Boolean choice pairs")
|
|
119
|
+
return RefundInteractionPolicy(self.selection_probability, self.retry_probability,
|
|
120
|
+
factual_policy_seed, *(component if component is not None else (None, None)))
|
|
121
|
+
|
|
122
|
+
def scenario_config(self):
|
|
123
|
+
return ScenarioConfig(name=self.name, fault=Fault.TIMEOUT_AFTER_COMMIT,
|
|
124
|
+
context=RefundContext(retry_instruction=True), use_idempotency=self.use_idempotency,
|
|
125
|
+
include_provenance=self.include_provenance, read_lag_ms=self.read_lag_ms)
|
|
126
|
+
|
|
127
|
+
def capture_factual(self, *, seed, policy_seed):
|
|
128
|
+
return run_policy_scenario(self.scenario_config(), self.make_policy(factual_policy_seed=policy_seed),
|
|
129
|
+
seed=seed, policy_seed=policy_seed, max_attempts=2)
|
|
130
|
+
|
|
131
|
+
def component_weights(self):
|
|
132
|
+
p, q = self.selection_probability, self.retry_probability
|
|
133
|
+
return tuple((p if selected else 1 - p) * (q if retry else 1 - q)
|
|
134
|
+
for selected, retry in COMPONENTS)
|
|
135
|
+
|
|
136
|
+
def analytic_reference(self, *, coalition_game="on_reach"):
|
|
137
|
+
"""Closed-form population law, independent of rollout or sampled keys.
|
|
138
|
+
|
|
139
|
+
An unknown or invalid component of positive mass refuses a coalition;
|
|
140
|
+
no conditioning on known, valid, or reached branches is performed.
|
|
141
|
+
"""
|
|
142
|
+
validate_coalition_game(coalition_game)
|
|
143
|
+
p, q = self.selection_probability, self.retry_probability
|
|
144
|
+
selected = [1. if mask & 1 else p for mask in range(8)]
|
|
145
|
+
retry = [1. if mask & 2 else q for mask in range(8)]
|
|
146
|
+
valid = [1. if coalition_game == "on_reach" else (
|
|
147
|
+
selected[mask] * retry[mask] if mask & 4 else selected[mask] if mask & 2 else 1.)
|
|
148
|
+
for mask in range(8)]
|
|
149
|
+
targets = {}
|
|
150
|
+
for target in TARGETS:
|
|
151
|
+
bad = [1 - s if target == "action" or self.use_idempotency
|
|
152
|
+
else 1 - s * (1 - r) for s, r in zip(selected, retry)]
|
|
153
|
+
unknown = [0. if target == "action" else 1. if self.read_lag_ms else (
|
|
154
|
+
(s if self.use_idempotency else s * (1 - r))
|
|
155
|
+
if target == "action_linked_consequence" and not self.include_provenance else 0.)
|
|
156
|
+
for s, r in zip(selected, retry)]
|
|
157
|
+
values = [value if valid[mask] == 1 and unknown[mask] == 0 else None
|
|
158
|
+
for mask, value in enumerate(bad)]
|
|
159
|
+
eligible = all(value is not None for value in values)
|
|
160
|
+
if eligible:
|
|
161
|
+
if target == "action" or self.use_idempotency:
|
|
162
|
+
shares, interaction = [-(1 - p), 0., 0.], 0.
|
|
163
|
+
else:
|
|
164
|
+
shares = [-(1 - p) * (1 - q) / 2, (1 + p) * (1 - q) / 2, 0.]
|
|
165
|
+
interaction = (1 - p) * (1 - q)
|
|
166
|
+
else:
|
|
167
|
+
shares, interaction = None, None
|
|
168
|
+
endpoint_eligible = values[0] is not None and values[7] is not None
|
|
169
|
+
targets[target] = {
|
|
170
|
+
"coalition_bad_rates": values, "unknown_probabilities": unknown,
|
|
171
|
+
"shapley_eligible": eligible, "shapley_values": shares,
|
|
172
|
+
"selection_retry_interaction": interaction,
|
|
173
|
+
"endpoint_eligible": endpoint_eligible,
|
|
174
|
+
"endpoint_difference": values[7] - values[0] if endpoint_eligible else None,
|
|
175
|
+
"reason": None if eligible else "positive invalid or unknown population mass",
|
|
176
|
+
}
|
|
177
|
+
return {"name": self.name, "coalition_game": coalition_game, "decision_count": 3,
|
|
178
|
+
"selection_probability": p, "retry_probability": q,
|
|
179
|
+
"population_not_finite_seed_claim": True,
|
|
180
|
+
"coalition_valid_probabilities": valid, "targets": targets}
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
INTERACTION_MECHANISMS = MappingProxyType({item.name: item for item in (
|
|
184
|
+
RefundInteractionMechanism("refund-p50-q50"),
|
|
185
|
+
RefundInteractionMechanism("refund-p20-q80", .2, .8),
|
|
186
|
+
RefundInteractionMechanism("refund-p80-q20", .8, .2),
|
|
187
|
+
RefundInteractionMechanism("refund-p100-q100", 1., 1.),
|
|
188
|
+
RefundInteractionMechanism("idempotent-p50-q50", use_idempotency=True),
|
|
189
|
+
RefundInteractionMechanism("unknown-provenance-p50-q50", include_provenance=False),
|
|
190
|
+
RefundInteractionMechanism("lagged-readback-p50-q50", read_lag_ms=1000),
|
|
191
|
+
RefundInteractionMechanism("budget-p50-q50", max_calls=4),
|
|
192
|
+
)})
|