playbook-engine 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- playbook_engine/__init__.py +3 -0
- playbook_engine/absence.py +538 -0
- playbook_engine/agent_judge.py +1325 -0
- playbook_engine/agent_segmenter.py +140 -0
- playbook_engine/artifact_store.py +314 -0
- playbook_engine/cache_format.py +266 -0
- playbook_engine/canonicalize.py +142 -0
- playbook_engine/citation_resolver.py +274 -0
- playbook_engine/clause_aligner.py +860 -0
- playbook_engine/clause_classifier.py +900 -0
- playbook_engine/clause_differ.py +619 -0
- playbook_engine/clause_position_compiler.py +382 -0
- playbook_engine/clause_tree.py +428 -0
- playbook_engine/cli.py +4266 -0
- playbook_engine/config.py +802 -0
- playbook_engine/corpus_linter.py +969 -0
- playbook_engine/deal_dates.py +814 -0
- playbook_engine/deviation_classifier.py +210 -0
- playbook_engine/digest.py +854 -0
- playbook_engine/document_renderer.py +1134 -0
- playbook_engine/docx_ingester.py +659 -0
- playbook_engine/docx_normalizer.py +279 -0
- playbook_engine/dossiers.py +742 -0
- playbook_engine/entity_registry.py +461 -0
- playbook_engine/environment.py +226 -0
- playbook_engine/equivalence.py +1013 -0
- playbook_engine/equivalence_check.py +591 -0
- playbook_engine/extraction.py +2561 -0
- playbook_engine/floor_candidates.py +1832 -0
- playbook_engine/floor_judge.py +328 -0
- playbook_engine/index_page_assets.py +695 -0
- playbook_engine/induction_version_selector.py +163 -0
- playbook_engine/inspection_report.py +483 -0
- playbook_engine/instrument.py +178 -0
- playbook_engine/intake_plan.py +769 -0
- playbook_engine/judgment.py +387 -0
- playbook_engine/llm_segmentation_stage.py +296 -0
- playbook_engine/llm_segmenter.py +367 -0
- playbook_engine/llm_segmenter_batch.py +762 -0
- playbook_engine/natural_sort.py +22 -0
- playbook_engine/observation_builder.py +2009 -0
- playbook_engine/opf_accessors.py +459 -0
- playbook_engine/overrides.py +805 -0
- playbook_engine/pdf_ingester.py +385 -0
- playbook_engine/pipeline.py +6190 -0
- playbook_engine/playbook_assembler.py +633 -0
- playbook_engine/posture.py +837 -0
- playbook_engine/precedent.py +669 -0
- playbook_engine/prior_form_origin_check.py +270 -0
- playbook_engine/prior_reconstruction.py +1015 -0
- playbook_engine/prior_standards.py +218 -0
- playbook_engine/provenance_detector.py +523 -0
- playbook_engine/resegmentation.py +1312 -0
- playbook_engine/reversal_detector.py +304 -0
- playbook_engine/review_rows.py +320 -0
- playbook_engine/rtf_ingester.py +248 -0
- playbook_engine/rubric.py +532 -0
- playbook_engine/run_manifest.py +1262 -0
- playbook_engine/scope_gate.py +392 -0
- playbook_engine/scorecard.py +1444 -0
- playbook_engine/segmentation_grounding.py +258 -0
- playbook_engine/segmentation_qa.py +581 -0
- playbook_engine/segmenter.py +330 -0
- playbook_engine/signed_detector.py +778 -0
- playbook_engine/staging.py +933 -0
- playbook_engine/subsplit.py +603 -0
- playbook_engine/subsplit_check.py +323 -0
- playbook_engine/taxonomy.py +208 -0
- playbook_engine/taxonomy_inductor.py +529 -0
- playbook_engine/template_standards.py +221 -0
- playbook_engine/toaster_install.py +40 -0
- playbook_engine/tracked_changes_overlay.py +363 -0
- playbook_engine/validator.py +2057 -0
- playbook_engine/version_orderer.py +874 -0
- playbook_engine-2.0.0.dist-info/METADATA +322 -0
- playbook_engine-2.0.0.dist-info/RECORD +105 -0
- playbook_engine-2.0.0.dist-info/WHEEL +4 -0
- playbook_engine-2.0.0.dist-info/entry_points.txt +3 -0
- playbook_engine-2.0.0.dist-info/licenses/LICENSE +224 -0
- playbook_engine-2.0.0.dist-info/licenses/NOTICE +15 -0
- spec/CHANGELOG.md +755 -0
- spec/clause-tree.schema.json +86 -0
- spec/conformance/0.6/README.md +44 -0
- spec/conformance/0.6/manifest.json +85 -0
- spec/conformance/0.6/vectors/001-minimal-no-perspective.json +104 -0
- spec/conformance/0.6/vectors/002-variants-refused-and-exclusions.json +682 -0
- spec/conformance/0.6/vectors/003-cap-totals-and-summary.json +638 -0
- spec/conformance/0.6/vectors/004-party-alias-grouping.json +542 -0
- spec/conformance/0.6/vectors/005-opening-evidence.json +621 -0
- spec/conformance/0.6/vectors/006-opening-rules.json +849 -0
- spec/conformance/0.6/vectors/007-equivalence-label-and-coverage.json +1035 -0
- spec/conformance/0.6/vectors/008-hard-rules-and-dossiers.json +1791 -0
- spec/conformance/0.6/vectors/009-standard-parts-and-owner-label.json +788 -0
- spec/conformance/0.6/vectors/010-signing-dates.json +646 -0
- spec/conformance/0.6/vectors/011-settled-outcomes.json +1117 -0
- spec/conformance/0.6/vectors/012-prior-standards.json +943 -0
- spec/conformance/0.6/vectors/013-reconstructed-prior-form.json +568 -0
- spec/conformance/0.6/vectors/014-verified-absence.json +763 -0
- spec/conformance/0.6/vectors/015-number-and-key-rules.json +114 -0
- spec/conformance/README.md +64 -0
- spec/playbook.schema-0.6.json +3208 -0
- spec/taxonomy/affiliation-agreement.yaml +62 -0
- spec/taxonomy/cuad-base.yaml +67 -0
- spec/taxonomy/general-commercial.yaml +32 -0
- spec/taxonomy/nda.yaml +93 -0
|
@@ -0,0 +1,1325 @@
|
|
|
1
|
+
"""Agent-as-judge bridge core — issue #64.
|
|
2
|
+
|
|
3
|
+
Provides a store-backed "agent-as-judge" layer that lets an external caller
|
|
4
|
+
supply **real verdicts** into the L1–L4 pipeline through the existing judge
|
|
5
|
+
dependency-injection seam.
|
|
6
|
+
|
|
7
|
+
On a payload it has seen before (store hit), replays the stored verdict as the
|
|
8
|
+
correct dataclass. On a new payload (store miss), records the **full**
|
|
9
|
+
untruncated payload to a pending queue and returns the engine's needs-review
|
|
10
|
+
sentinel.
|
|
11
|
+
|
|
12
|
+
Components:
|
|
13
|
+
|
|
14
|
+
- ``VerdictStore`` — persistent JSONL keyed by a stable SHA-256 content hash
|
|
15
|
+
of the full payload. Mirrors the style of ``JudgmentCache`` in
|
|
16
|
+
``judgment.py`` but in its own namespace and without text truncation.
|
|
17
|
+
Default file: ``<out>/judge/verdicts.jsonl``. Each record also carries the
|
|
18
|
+
**rubric stamp** the verdict was produced under (optional on read — absent
|
|
19
|
+
on records banked before rubric versioning existed).
|
|
20
|
+
|
|
21
|
+
- ``PendingQueue`` — appends unique full payloads (deduplicated by key) to
|
|
22
|
+
``<out>/judge/pending.jsonl``. Each record carries the payload key, the
|
|
23
|
+
judge kind, the full payload dict, and the rubric version in force when
|
|
24
|
+
the item was queued.
|
|
25
|
+
|
|
26
|
+
- ``StoreBackedClassificationJudge`` — implements ``ClassificationJudge``.
|
|
27
|
+
- ``StoreBackedProvenanceJudge`` — implements ``ProvenanceJudge``.
|
|
28
|
+
- ``StoreBackedScopeJudge`` — implements ``ScopeJudge``.
|
|
29
|
+
- ``StoreBackedEquivalenceJudge`` — the ``equivalence`` kind (issue #240):
|
|
30
|
+
one verdict per distinct text against our standard, queued after mining
|
|
31
|
+
(it needs the precedent record, not a seam inside ``mine_corpus``).
|
|
32
|
+
- ``StoreBackedDealDateJudge`` — the ``deal_date`` kind (issue #247):
|
|
33
|
+
one verdict per distinct set of date passages of an executed copy, queued
|
|
34
|
+
after mining when the deterministic readers found no date.
|
|
35
|
+
- ``StoreBackedResegmentCheckJudge`` — the ``resegment_check`` kind (issue
|
|
36
|
+
#255): one verdict per sampled split or join of the re-segmentation pass,
|
|
37
|
+
queued after mining. It measures the pass; it changes nothing and gates
|
|
38
|
+
nothing.
|
|
39
|
+
- ``StoreBackedAbsenceCheckJudge`` — the ``absence_check`` kind (issue #251):
|
|
40
|
+
one verdict per sampled verified absence, "is this clause type really absent
|
|
41
|
+
from this signed text?", queued after mining. It measures the absence guard;
|
|
42
|
+
it changes nothing and gates nothing.
|
|
43
|
+
- ``StoreBackedSubsplitJudge`` — the ``subsplit`` kind (issue #257): one
|
|
44
|
+
verdict per distinct multi-topic node text, "partition these sentences into
|
|
45
|
+
clause types, or no split", asked WHILE mining (the re-segmentation stage).
|
|
46
|
+
Only a settled verdict (an owner's, or blind-checked) is applied.
|
|
47
|
+
- ``StoreBackedPriorFormOriginJudge`` — the ``prior_form_origin`` kind (issue
|
|
48
|
+
#250): one verdict per recurring first-draft wording on our paper, "is this
|
|
49
|
+
OUR former language?", queued after mining. Only a checked ``ours: true``
|
|
50
|
+
makes the cluster a reconstructed prior standard.
|
|
51
|
+
|
|
52
|
+
These are drop-in replacements for the judge parameters of
|
|
53
|
+
``mine_corpus(scope_judge=…, classification_judge=…, provenance_judge=…)``.
|
|
54
|
+
There is no deviation judge: every deviation is the deterministic standard
|
|
55
|
+
check, so no deviation item is ever queued.
|
|
56
|
+
|
|
57
|
+
Adding a judge kind (the seam the ``equivalence`` kind plugs into): write the
|
|
58
|
+
store-backed judge beside the three above, register its verdict shape with
|
|
59
|
+
:func:`register_verdict_kind` (apply-time validation and kind inference for
|
|
60
|
+
``playbook judge-apply``) and its rubric with
|
|
61
|
+
:func:`playbook_engine.rubric.register_judge_kind`. Everything generic —
|
|
62
|
+
``VerdictStore``, ``PendingQueue``, the rubric stamp, ``judge`` /
|
|
63
|
+
``judge-apply`` and the plan output — already works per kind.
|
|
64
|
+
|
|
65
|
+
Rubric versioning (see :mod:`playbook_engine.rubric`): because the store key
|
|
66
|
+
is purely content-derived, a change to the *judging criteria* — the taxonomy,
|
|
67
|
+
the prose rubric in the ``playbook-from-corpus`` skill — would otherwise
|
|
68
|
+
replay every banked verdict forever, since the clause text never moved. Each store hit is therefore checked against the rubric
|
|
69
|
+
currently in force: ``current`` replays, ``stale`` re-queues, and ``legacy``
|
|
70
|
+
(unstamped) replays but is counted and reported. ``RubricPolicy`` carries
|
|
71
|
+
both the policy knobs and the run tally the CLI reports from.
|
|
72
|
+
|
|
73
|
+
Note on ``StoreBackedScopeJudge`` (issue #87): unlike the other three,
|
|
74
|
+
``ScopeJudge.judge()`` may only return ``ScopeDecision(basis="judge")`` —
|
|
75
|
+
``scope_gate()`` raises ``ValueError`` on any other basis — so a store miss
|
|
76
|
+
cannot be expressed as a ``basis="needs_review"`` return value the way the
|
|
77
|
+
other judges do it. Instead it raises ``ScopeNeedsReviewError`` after queuing
|
|
78
|
+
the payload; ``scope_gate()`` catches that and converts it into a retained,
|
|
79
|
+
zero-confidence ``basis="judge_error"`` decision, never the stub default's
|
|
80
|
+
blind ``in_scope=True`` at confidence 0.5.
|
|
81
|
+
|
|
82
|
+
Security: full clause text IS stored in the pending queue (by design — the
|
|
83
|
+
external caller needs it to render the verdict). The store itself stores the
|
|
84
|
+
verdict dict plus key; it does NOT re-store the payload.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
from __future__ import annotations
|
|
88
|
+
|
|
89
|
+
import hashlib
|
|
90
|
+
import json
|
|
91
|
+
import logging
|
|
92
|
+
from collections.abc import Callable, Iterator
|
|
93
|
+
from contextlib import contextmanager
|
|
94
|
+
from dataclasses import dataclass, field
|
|
95
|
+
from pathlib import Path
|
|
96
|
+
from typing import Any
|
|
97
|
+
|
|
98
|
+
from playbook_engine import absence, deal_dates, prior_reconstruction, resegmentation, subsplit
|
|
99
|
+
from playbook_engine.clause_classifier import ClauseClassification
|
|
100
|
+
from playbook_engine.clause_tree import ClauseNode, ClauseTree
|
|
101
|
+
from playbook_engine.config import AgreementType
|
|
102
|
+
from playbook_engine.equivalence import (
|
|
103
|
+
EQUIVALENCE_KIND,
|
|
104
|
+
EquivalenceSubject,
|
|
105
|
+
validate_draft_verdict,
|
|
106
|
+
vs_standard_of,
|
|
107
|
+
)
|
|
108
|
+
from playbook_engine.provenance_detector import (
|
|
109
|
+
_PROVENANCE_VALUES,
|
|
110
|
+
PROVENANCE_UNKNOWN,
|
|
111
|
+
ProvenanceResult,
|
|
112
|
+
)
|
|
113
|
+
from playbook_engine.rubric import (
|
|
114
|
+
RubricPolicy,
|
|
115
|
+
RubricStamp,
|
|
116
|
+
classifier_eligible_ids,
|
|
117
|
+
rubric_version,
|
|
118
|
+
)
|
|
119
|
+
from playbook_engine.scope_gate import ScopeDecision
|
|
120
|
+
|
|
121
|
+
_log = logging.getLogger(__name__)
|
|
122
|
+
|
|
123
|
+
# ---------------------------------------------------------------------------
|
|
124
|
+
# Key construction — full payload, no truncation
|
|
125
|
+
# ---------------------------------------------------------------------------
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _payload_key(payload: Any) -> str:
|
|
129
|
+
"""SHA-256 of the JSON-serialised payload.
|
|
130
|
+
|
|
131
|
+
Unlike ``judgment._payload_key``, this function does NOT include a
|
|
132
|
+
``model_id`` component, and it deliberately does NOT include the rubric
|
|
133
|
+
version either: the key stays purely content-derived so that cross-
|
|
134
|
+
document dedup of identical clause text keeps working and so that a
|
|
135
|
+
rubric bump does not silently orphan the entire banked verdict store
|
|
136
|
+
(a key that never matches again is indistinguishable from an empty
|
|
137
|
+
store). Rubric identity is carried *beside* the verdict instead — see
|
|
138
|
+
:class:`~playbook_engine.rubric.RubricStamp` and
|
|
139
|
+
:class:`~playbook_engine.rubric.RubricPolicy` — so a bump is a counted,
|
|
140
|
+
reported, reversible event rather than a vanishing act.
|
|
141
|
+
|
|
142
|
+
Also unlike the judgment cache, text is NOT truncated: the full payload
|
|
143
|
+
is hashed to prevent false collisions across clauses that share a long
|
|
144
|
+
prefix but differ later.
|
|
145
|
+
"""
|
|
146
|
+
raw = json.dumps(payload, sort_keys=True, ensure_ascii=False)
|
|
147
|
+
return hashlib.sha256(raw.encode()).hexdigest()
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# ---------------------------------------------------------------------------
|
|
151
|
+
# VerdictStore
|
|
152
|
+
# ---------------------------------------------------------------------------
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@dataclass(frozen=True)
|
|
156
|
+
class StoredVerdict:
|
|
157
|
+
"""One record read back out of a :class:`VerdictStore`.
|
|
158
|
+
|
|
159
|
+
``rubric`` is ``None`` for every record written before rubric versioning
|
|
160
|
+
existed (the "legacy" state) — see :mod:`playbook_engine.rubric`.
|
|
161
|
+
"""
|
|
162
|
+
|
|
163
|
+
verdict: dict[str, Any]
|
|
164
|
+
rubric: RubricStamp | None = None
|
|
165
|
+
|
|
166
|
+
@property
|
|
167
|
+
def rubric_version(self) -> str | None:
|
|
168
|
+
return self.rubric.version if self.rubric else None
|
|
169
|
+
|
|
170
|
+
@property
|
|
171
|
+
def rubric_kind(self) -> str | None:
|
|
172
|
+
return self.rubric.kind if self.rubric else None
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class VerdictStore:
|
|
176
|
+
"""Persistent JSONL store: content hash → verdict dict (+ rubric stamp).
|
|
177
|
+
|
|
178
|
+
Each record: ``{"key": "<sha256>", "verdict": {…}, "rubric": {"kind": …,
|
|
179
|
+
"version": …}}``. The ``rubric`` member is optional on read — records
|
|
180
|
+
written before rubric versioning carry no stamp and load as
|
|
181
|
+
``StoredVerdict(rubric=None)``, so an existing store keeps working
|
|
182
|
+
untouched and no banked judgment is discarded on upgrade.
|
|
183
|
+
|
|
184
|
+
``get(payload) -> dict | None`` — stored verdict dict, or None.
|
|
185
|
+
``get_record(payload)`` — verdict + rubric stamp, or None.
|
|
186
|
+
``put(payload, verdict, rubric=…)`` — append; update in-memory.
|
|
187
|
+
|
|
188
|
+
Load-on-init: reads the JSONL file into memory on construction.
|
|
189
|
+
Corrupt lines are silently skipped (same contract as ``JudgmentCache``).
|
|
190
|
+
Later lines for the same key win, so a re-applied verdict is an append
|
|
191
|
+
rather than a rewrite: the original record stays on disk as an audit
|
|
192
|
+
trail of what the verdict was banked under.
|
|
193
|
+
"""
|
|
194
|
+
|
|
195
|
+
def __init__(self, store_path: Path) -> None:
|
|
196
|
+
self._store_path = store_path
|
|
197
|
+
self._store: dict[str, StoredVerdict] = {} # key -> record
|
|
198
|
+
# Open lookup captures (issue #219) — see capture_lookups().
|
|
199
|
+
self._captures: list[set[str]] = []
|
|
200
|
+
self._load()
|
|
201
|
+
|
|
202
|
+
# ------------------------------------------------------------------
|
|
203
|
+
# Public API
|
|
204
|
+
# ------------------------------------------------------------------
|
|
205
|
+
|
|
206
|
+
def get(self, payload: Any) -> dict[str, Any] | None:
|
|
207
|
+
"""Return the stored verdict dict for *payload*, or ``None`` on miss."""
|
|
208
|
+
record = self._lookup(_payload_key(payload))
|
|
209
|
+
return record.verdict if record is not None else None
|
|
210
|
+
|
|
211
|
+
def get_record(self, payload: Any) -> StoredVerdict | None:
|
|
212
|
+
"""Return the full stored record (verdict + rubric stamp), or ``None``."""
|
|
213
|
+
return self._lookup(_payload_key(payload))
|
|
214
|
+
|
|
215
|
+
def all_verdicts(self) -> list[dict[str, Any]]:
|
|
216
|
+
"""Every stored verdict dict (last write per key), in no particular order."""
|
|
217
|
+
return [record.verdict for record in self._store.values()]
|
|
218
|
+
|
|
219
|
+
@contextmanager
|
|
220
|
+
def capture_lookups(self) -> Iterator[set[str]]:
|
|
221
|
+
"""Collect every key looked up through ``get``/``get_record`` meanwhile.
|
|
222
|
+
|
|
223
|
+
Issue #219: the L1-L4 stage cache stays on under store-backed judges
|
|
224
|
+
by recording, with each cached document result, the verdict keys that
|
|
225
|
+
result was built from — then refusing to replay it when any of them
|
|
226
|
+
has since changed (see :meth:`fingerprint`). Nestable; a lookup is
|
|
227
|
+
recorded into every open capture.
|
|
228
|
+
"""
|
|
229
|
+
seen: set[str] = set()
|
|
230
|
+
self._captures.append(seen)
|
|
231
|
+
try:
|
|
232
|
+
yield seen
|
|
233
|
+
finally:
|
|
234
|
+
self._captures.remove(seen)
|
|
235
|
+
|
|
236
|
+
def fingerprint(self, key: str) -> str | None:
|
|
237
|
+
"""Opaque digest of what *key* currently holds — ``None`` when absent.
|
|
238
|
+
|
|
239
|
+
Covers the verdict AND its rubric stamp: a ``judge-apply`` that
|
|
240
|
+
overwrites a verdict moves it, so a cached result built on the old
|
|
241
|
+
record is not replayed.
|
|
242
|
+
"""
|
|
243
|
+
record = self._store.get(key)
|
|
244
|
+
if record is None:
|
|
245
|
+
return None
|
|
246
|
+
raw = json.dumps(
|
|
247
|
+
{
|
|
248
|
+
"verdict": record.verdict,
|
|
249
|
+
"rubric": record.rubric.to_dict() if record.rubric is not None else None,
|
|
250
|
+
},
|
|
251
|
+
sort_keys=True,
|
|
252
|
+
ensure_ascii=False,
|
|
253
|
+
)
|
|
254
|
+
return hashlib.sha256(raw.encode()).hexdigest()
|
|
255
|
+
|
|
256
|
+
def put(
|
|
257
|
+
self, payload: Any, verdict: dict[str, Any], *, rubric: RubricStamp | None = None
|
|
258
|
+
) -> None:
|
|
259
|
+
"""Store *verdict* for *payload* (JSON-serialisable dicts required)."""
|
|
260
|
+
self.put_by_key(_payload_key(payload), verdict, rubric=rubric)
|
|
261
|
+
|
|
262
|
+
def get_by_key(self, key: str) -> dict[str, Any] | None:
|
|
263
|
+
"""Return the stored verdict dict for a pre-computed *key*, or ``None``.
|
|
264
|
+
|
|
265
|
+
Counterpart of ``put_by_key`` — used by ``playbook judge`` to detect
|
|
266
|
+
pending items that are re-queues of stored-but-malformed verdicts.
|
|
267
|
+
"""
|
|
268
|
+
record = self._store.get(key)
|
|
269
|
+
return record.verdict if record is not None else None
|
|
270
|
+
|
|
271
|
+
def get_record_by_key(self, key: str) -> StoredVerdict | None:
|
|
272
|
+
"""Return the full stored record for a pre-computed *key*, or ``None``."""
|
|
273
|
+
return self._store.get(key)
|
|
274
|
+
|
|
275
|
+
def put_by_key(
|
|
276
|
+
self, key: str, verdict: dict[str, Any], *, rubric: RubricStamp | None = None
|
|
277
|
+
) -> None:
|
|
278
|
+
"""Store *verdict* directly by its pre-computed *key*.
|
|
279
|
+
|
|
280
|
+
Used by ``playbook judge-apply`` to import verdicts whose keys were
|
|
281
|
+
computed by the producer (e.g. from a ``pending.jsonl`` export) without
|
|
282
|
+
re-hashing the original payload.
|
|
283
|
+
|
|
284
|
+
Args:
|
|
285
|
+
key: SHA-256 hex string (as produced by ``_payload_key``).
|
|
286
|
+
verdict: JSON-serialisable verdict dict.
|
|
287
|
+
rubric: Rubric the verdict was produced under. ``None`` records
|
|
288
|
+
the verdict unstamped (legacy) — reported on every
|
|
289
|
+
subsequent judge run.
|
|
290
|
+
"""
|
|
291
|
+
self._store[key] = StoredVerdict(verdict=verdict, rubric=rubric)
|
|
292
|
+
self._append(key, verdict, rubric)
|
|
293
|
+
|
|
294
|
+
def records(self) -> list[tuple[str, StoredVerdict]]:
|
|
295
|
+
"""Return ``(key, record)`` for every stored verdict, key-sorted."""
|
|
296
|
+
return sorted(self._store.items())
|
|
297
|
+
|
|
298
|
+
def __len__(self) -> int:
|
|
299
|
+
return len(self._store)
|
|
300
|
+
|
|
301
|
+
# ------------------------------------------------------------------
|
|
302
|
+
# Internal
|
|
303
|
+
# ------------------------------------------------------------------
|
|
304
|
+
|
|
305
|
+
def _lookup(self, key: str) -> StoredVerdict | None:
|
|
306
|
+
for seen in self._captures:
|
|
307
|
+
seen.add(key)
|
|
308
|
+
return self._store.get(key)
|
|
309
|
+
|
|
310
|
+
def _load(self) -> None:
|
|
311
|
+
"""Read the store file into memory (best-effort; corrupt lines skipped)."""
|
|
312
|
+
if not self._store_path.exists():
|
|
313
|
+
return
|
|
314
|
+
try:
|
|
315
|
+
for line in self._store_path.read_text(encoding="utf-8").splitlines():
|
|
316
|
+
line = line.strip()
|
|
317
|
+
if not line:
|
|
318
|
+
continue
|
|
319
|
+
try:
|
|
320
|
+
entry = json.loads(line)
|
|
321
|
+
self._store[entry["key"]] = StoredVerdict(
|
|
322
|
+
verdict=entry["verdict"],
|
|
323
|
+
rubric=RubricStamp.from_dict(entry.get("rubric")),
|
|
324
|
+
)
|
|
325
|
+
except Exception: # noqa: BLE001
|
|
326
|
+
pass # corrupt line — skip; do not crash startup
|
|
327
|
+
except Exception: # noqa: BLE001
|
|
328
|
+
pass # unreadable file — start with empty store
|
|
329
|
+
|
|
330
|
+
def _append(self, key: str, verdict: dict[str, Any], rubric: RubricStamp | None) -> None:
|
|
331
|
+
"""Append a single entry to the JSONL file."""
|
|
332
|
+
self._store_path.parent.mkdir(parents=True, exist_ok=True)
|
|
333
|
+
record: dict[str, Any] = {"key": key, "verdict": verdict}
|
|
334
|
+
if rubric is not None:
|
|
335
|
+
record["rubric"] = rubric.to_dict()
|
|
336
|
+
entry = json.dumps(record, ensure_ascii=False) + "\n"
|
|
337
|
+
with self._store_path.open("a", encoding="utf-8") as fh:
|
|
338
|
+
fh.write(entry)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
# ---------------------------------------------------------------------------
|
|
342
|
+
# PendingQueue
|
|
343
|
+
# ---------------------------------------------------------------------------
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
class PendingQueue:
|
|
347
|
+
"""Append-only queue of pending payloads awaiting external verdict.
|
|
348
|
+
|
|
349
|
+
Each record: ``{"key": "<sha256>", "kind": "classify"|"provenance"|"scope"|
|
|
350
|
+
"segment", "payload": {…}}`` (plus the registered kinds).
|
|
351
|
+
|
|
352
|
+
Deduplication: payloads with the same key are recorded at most once,
|
|
353
|
+
even across multiple ``add()`` calls on the same instance. This is the
|
|
354
|
+
within-instance dedup (within-batch + cross-batch for the same object);
|
|
355
|
+
persistence does not deduplicate across runs (the external caller is
|
|
356
|
+
responsible for that).
|
|
357
|
+
"""
|
|
358
|
+
|
|
359
|
+
def __init__(self, queue_path: Path) -> None:
|
|
360
|
+
self._queue_path = queue_path
|
|
361
|
+
self._seen_keys: set[str] = set()
|
|
362
|
+
# Open add captures (issue #219) — see capture_adds().
|
|
363
|
+
self._captures: list[set[str]] = []
|
|
364
|
+
|
|
365
|
+
@contextmanager
|
|
366
|
+
def capture_adds(self) -> Iterator[set[str]]:
|
|
367
|
+
"""Collect the key of every ``add`` call meanwhile — deduplicated or not.
|
|
368
|
+
|
|
369
|
+
Issue #219: a document result that queued anything rests on an
|
|
370
|
+
unresolved verdict and must never be replayed from the stage cache —
|
|
371
|
+
replaying it would also drop its items from this round's queue. The
|
|
372
|
+
within-instance dedup in :meth:`add` is exactly why the attempt, not
|
|
373
|
+
the write, is what gets recorded: a second document asking the same
|
|
374
|
+
question is just as unresolved as the first.
|
|
375
|
+
"""
|
|
376
|
+
seen: set[str] = set()
|
|
377
|
+
self._captures.append(seen)
|
|
378
|
+
try:
|
|
379
|
+
yield seen
|
|
380
|
+
finally:
|
|
381
|
+
self._captures.remove(seen)
|
|
382
|
+
|
|
383
|
+
def add(self, key: str, kind: str, payload: Any, rubric_version: str | None = None) -> bool:
|
|
384
|
+
"""Append *payload* to the queue if *key* has not been seen before.
|
|
385
|
+
|
|
386
|
+
Args:
|
|
387
|
+
key: Content hash from ``_payload_key(payload)``.
|
|
388
|
+
kind: One of ``"classify"``, ``"provenance"``,
|
|
389
|
+
``"scope"``, ``"segment"`` (or a kind
|
|
390
|
+
registered through :func:`register_verdict_kind`).
|
|
391
|
+
payload: The full, untruncated judge payload dict.
|
|
392
|
+
rubric_version: Rubric in force when this item was queued. Written
|
|
393
|
+
to the record so ``playbook judge-apply`` can stamp
|
|
394
|
+
the returned verdict with the rubric the question
|
|
395
|
+
was actually asked under — without needing to
|
|
396
|
+
re-derive it (and without needing the config at
|
|
397
|
+
apply time).
|
|
398
|
+
|
|
399
|
+
Returns:
|
|
400
|
+
``True`` if a new entry was written; ``False`` if *key* was already
|
|
401
|
+
seen (deduplicated).
|
|
402
|
+
"""
|
|
403
|
+
for seen in self._captures:
|
|
404
|
+
seen.add(key)
|
|
405
|
+
if key in self._seen_keys:
|
|
406
|
+
return False
|
|
407
|
+
self._seen_keys.add(key)
|
|
408
|
+
self._append(key, kind, payload, rubric_version)
|
|
409
|
+
return True
|
|
410
|
+
|
|
411
|
+
def _append(self, key: str, kind: str, payload: Any, rubric_version: str | None = None) -> None:
|
|
412
|
+
"""Write a single record to the JSONL file."""
|
|
413
|
+
self._queue_path.parent.mkdir(parents=True, exist_ok=True)
|
|
414
|
+
entry: dict[str, Any] = {"key": key, "kind": kind, "payload": payload}
|
|
415
|
+
if rubric_version is not None:
|
|
416
|
+
entry["rubric_version"] = rubric_version
|
|
417
|
+
record = json.dumps(entry, ensure_ascii=False) + "\n"
|
|
418
|
+
with self._queue_path.open("a", encoding="utf-8") as fh:
|
|
419
|
+
fh.write(record)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
# ---------------------------------------------------------------------------
|
|
423
|
+
# Needs-review sentinels
|
|
424
|
+
# ---------------------------------------------------------------------------
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _classification_needs_review() -> ClauseClassification:
|
|
428
|
+
"""Sentinel returned when no stored verdict is available for a classify payload."""
|
|
429
|
+
return ClauseClassification(taxonomy_id=None, confidence=0.0, basis="needs_review")
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _provenance_needs_review() -> ProvenanceResult:
|
|
433
|
+
"""Sentinel returned when no stored verdict is available for a provenance payload.
|
|
434
|
+
|
|
435
|
+
No side was determined, so the result says so: ``"unknown"`` at 0.0
|
|
436
|
+
(always ambiguous), recorded as paper ``"unknown"`` — never coerced to
|
|
437
|
+
``counterparty_paper`` (issue #225).
|
|
438
|
+
"""
|
|
439
|
+
return ProvenanceResult(
|
|
440
|
+
provenance=PROVENANCE_UNKNOWN,
|
|
441
|
+
confidence=0.0,
|
|
442
|
+
basis="needs_review",
|
|
443
|
+
)
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _require_provenance_side(provenance: Any) -> None:
|
|
447
|
+
"""Raise ``ValueError`` unless a stored provenance verdict names a side.
|
|
448
|
+
|
|
449
|
+
A stored verdict names a side (the judge's two-valued answer vocabulary,
|
|
450
|
+
rubric._PROVENANCE_ANSWERS). ``"unknown"`` is reserved for the engine's
|
|
451
|
+
own no-verdict sentinel (issue #225), so it is rejected here — at apply
|
|
452
|
+
time and on store replay alike — before ``ProvenanceResult``, which
|
|
453
|
+
accepts it.
|
|
454
|
+
"""
|
|
455
|
+
if provenance not in _PROVENANCE_VALUES:
|
|
456
|
+
raise ValueError(
|
|
457
|
+
f"Unknown provenance: {provenance!r}. Must be one of {sorted(_PROVENANCE_VALUES)}"
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
# ---------------------------------------------------------------------------
|
|
462
|
+
# Apply-time verdict validation (used by `playbook judge-apply`)
|
|
463
|
+
# ---------------------------------------------------------------------------
|
|
464
|
+
|
|
465
|
+
#: Basis values that mean "no real judgment happened". A verdict *file*
|
|
466
|
+
#: (producer-supplied) must never carry one: on replay it either loops the
|
|
467
|
+
#: item forever or, worse, replays as a permanently-unjudged result while the
|
|
468
|
+
#: pending queue looks drained. The store-backed judges set these themselves.
|
|
469
|
+
_UNRESOLVED_VERDICT_BASES = frozenset({"needs_review", "judge_error", "stub"})
|
|
470
|
+
|
|
471
|
+
#: The only basis values a *producer-supplied* classify verdict may carry.
|
|
472
|
+
#: Mirrors the whitelist ``clause_classifier.classify_tree`` enforces on
|
|
473
|
+
#: replay (``"judge"``, ``"judge_error"``, ``"needs_review"``, ``"unclassified"``),
|
|
474
|
+
#: minus the two engine-internal bases already rejected above.
|
|
475
|
+
_CLASSIFY_REPLAYABLE_BASES = frozenset({"judge", "unclassified"})
|
|
476
|
+
|
|
477
|
+
#: The only basis value a *producer-supplied* provenance verdict may carry.
|
|
478
|
+
#: ``provenance_detector.ProvenanceResult`` accepts a wide ``_BASIS_VALUES``
|
|
479
|
+
#: set (``template_similarity``, ``alias_first_party``, ``hint``, ...) because
|
|
480
|
+
#: the engine's own deterministic detectors construct it with those bases —
|
|
481
|
+
#: but a producer verdict file only ever represents LLM judgment, so only
|
|
482
|
+
#: ``"llm"`` is replayable here (REFERENCE.md: "this is the one kind where
|
|
483
|
+
#: 'llm' is correct").
|
|
484
|
+
_PROVENANCE_REPLAYABLE_BASES = frozenset({"llm"})
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _validate_confidence_field(
|
|
488
|
+
verdict: dict[str, Any], field: str, *, allow_none: bool = False
|
|
489
|
+
) -> None:
|
|
490
|
+
"""Raise ``ValueError`` if ``verdict[field]`` is not a valid confidence.
|
|
491
|
+
|
|
492
|
+
Runs *before* the corresponding dataclass is constructed so a
|
|
493
|
+
producer-supplied stringified number (e.g. ``"0.8"``) never reaches the
|
|
494
|
+
dataclass's ``0.0 <= confidence <= 1.0`` comparison and raises a bare,
|
|
495
|
+
line-number-free ``TypeError`` instead of an actionable ``ValueError``
|
|
496
|
+
(issue #161). Absent fields are left to the caller's ``dict.get(...,
|
|
497
|
+
default)`` fallback and are not validated here.
|
|
498
|
+
|
|
499
|
+
Args:
|
|
500
|
+
verdict: The verdict dict from the producer's JSONL line.
|
|
501
|
+
field: Field name to check (e.g. ``"confidence"``,
|
|
502
|
+
``"scope_confidence"``).
|
|
503
|
+
allow_none: Whether an explicit ``null`` is acceptable.
|
|
504
|
+
"""
|
|
505
|
+
if field not in verdict:
|
|
506
|
+
return
|
|
507
|
+
value = verdict[field]
|
|
508
|
+
if value is None:
|
|
509
|
+
if allow_none:
|
|
510
|
+
return
|
|
511
|
+
raise ValueError(f"'{field}' must not be null")
|
|
512
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
513
|
+
raise ValueError(f"'{field}' must be a number in [0, 1], got {value!r}")
|
|
514
|
+
if not 0.0 <= value <= 1.0:
|
|
515
|
+
raise ValueError(f"'{field}' must be in [0, 1], got {value!r}")
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _validate_classify(verdict: dict[str, Any]) -> None:
|
|
519
|
+
classify_basis = verdict.get("basis", "judge")
|
|
520
|
+
if classify_basis not in _CLASSIFY_REPLAYABLE_BASES:
|
|
521
|
+
raise ValueError(
|
|
522
|
+
f"basis {classify_basis!r} is not replayable for a classify verdict; "
|
|
523
|
+
f"a supplied verdict must carry basis in "
|
|
524
|
+
f"{sorted(_CLASSIFY_REPLAYABLE_BASES)!r} (clause_classifier.classify_tree "
|
|
525
|
+
"rejects anything else on replay)"
|
|
526
|
+
)
|
|
527
|
+
_validate_confidence_field(verdict, "confidence")
|
|
528
|
+
ClauseClassification(
|
|
529
|
+
taxonomy_id=verdict.get("taxonomy_id"),
|
|
530
|
+
confidence=verdict.get("confidence", 0.0),
|
|
531
|
+
basis=classify_basis,
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _validate_provenance(verdict: dict[str, Any]) -> None:
|
|
536
|
+
if "provenance" not in verdict:
|
|
537
|
+
raise ValueError("missing 'provenance' field")
|
|
538
|
+
provenance_basis = verdict.get("basis", "llm")
|
|
539
|
+
if provenance_basis not in _PROVENANCE_REPLAYABLE_BASES:
|
|
540
|
+
raise ValueError(
|
|
541
|
+
f"basis {provenance_basis!r} is not replayable for a provenance verdict; "
|
|
542
|
+
f"a supplied verdict must carry basis in "
|
|
543
|
+
f"{sorted(_PROVENANCE_REPLAYABLE_BASES)!r} (a producer verdict file only "
|
|
544
|
+
"ever represents LLM judgment; deterministic bases are set by the engine "
|
|
545
|
+
"itself, not by you)"
|
|
546
|
+
)
|
|
547
|
+
_require_provenance_side(verdict["provenance"])
|
|
548
|
+
_validate_confidence_field(verdict, "confidence")
|
|
549
|
+
ProvenanceResult(
|
|
550
|
+
provenance=verdict["provenance"],
|
|
551
|
+
confidence=verdict.get("confidence", 0.0),
|
|
552
|
+
basis=provenance_basis,
|
|
553
|
+
)
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _validate_equivalence(verdict: dict[str, Any]) -> None:
|
|
557
|
+
validate_draft_verdict(verdict)
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def _validate_deal_date(verdict: dict[str, Any]) -> None:
|
|
561
|
+
deal_dates.validate_verdict(verdict)
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _validate_resegment_check(verdict: dict[str, Any]) -> None:
|
|
565
|
+
resegmentation.validate_check_verdict(verdict)
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def _validate_absence_check(verdict: dict[str, Any]) -> None:
|
|
569
|
+
absence.validate_check_verdict(verdict)
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def _validate_subsplit(verdict: dict[str, Any]) -> None:
|
|
573
|
+
subsplit.validate_draft(verdict)
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _validate_prior_form_origin(verdict: dict[str, Any]) -> None:
|
|
577
|
+
prior_reconstruction.validate_origin_draft(verdict)
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def _validate_scope(verdict: dict[str, Any]) -> None:
|
|
581
|
+
if not isinstance(verdict.get("in_scope"), bool):
|
|
582
|
+
raise ValueError("'in_scope' must be a JSON boolean")
|
|
583
|
+
_validate_confidence_field(verdict, "scope_confidence")
|
|
584
|
+
ScopeDecision(
|
|
585
|
+
in_scope=verdict["in_scope"],
|
|
586
|
+
scope_rationale=verdict.get("scope_rationale") or "Replayed from stored verdict.",
|
|
587
|
+
scope_confidence=verdict.get("scope_confidence", 0.0),
|
|
588
|
+
basis="judge",
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
#: ``kind -> (validate, matches)`` for every pending-item kind ``judge-apply``
|
|
593
|
+
#: accepts. ``validate(verdict)`` raises ``ValueError`` on a verdict the
|
|
594
|
+
#: kind's store-backed judge could not replay; ``matches(verdict)`` says
|
|
595
|
+
#: whether a verdict's field shape belongs to the kind (used when the key is
|
|
596
|
+
#: not in ``pending.jsonl``). Extend with :func:`register_verdict_kind`.
|
|
597
|
+
_VERDICT_KINDS: dict[
|
|
598
|
+
str, tuple[Callable[[dict[str, Any]], None], Callable[[dict[str, Any]], bool]]
|
|
599
|
+
] = {
|
|
600
|
+
"provenance": (_validate_provenance, lambda v: "provenance" in v),
|
|
601
|
+
"scope": (_validate_scope, lambda v: "in_scope" in v),
|
|
602
|
+
"classify": (_validate_classify, lambda v: "taxonomy_id" in v),
|
|
603
|
+
EQUIVALENCE_KIND: (_validate_equivalence, lambda v: "parts" in v),
|
|
604
|
+
deal_dates.DEAL_DATE_KIND: (_validate_deal_date, lambda v: "signed_on" in v),
|
|
605
|
+
resegmentation.CHECK_KIND: (_validate_resegment_check, lambda v: "resegment_ok" in v),
|
|
606
|
+
prior_reconstruction.ORIGIN_KIND: (_validate_prior_form_origin, lambda v: "ours" in v),
|
|
607
|
+
absence.CHECK_KIND: (_validate_absence_check, lambda v: "absent" in v),
|
|
608
|
+
subsplit.SUBSPLIT_KIND: (_validate_subsplit, lambda v: "split" in v),
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def register_verdict_kind(
|
|
613
|
+
kind: str,
|
|
614
|
+
validate: Callable[[dict[str, Any]], None],
|
|
615
|
+
matches: Callable[[dict[str, Any]], bool],
|
|
616
|
+
) -> None:
|
|
617
|
+
"""Teach ``judge-apply`` a new pending-item *kind*.
|
|
618
|
+
|
|
619
|
+
*validate* must reconstruct what the kind's store-backed judge builds on
|
|
620
|
+
replay and raise ``ValueError`` with an actionable message otherwise, so a
|
|
621
|
+
verdict accepted here always replays. *matches* is the kind's field-shape
|
|
622
|
+
test for :func:`infer_verdict_kind`; field names must be mutually
|
|
623
|
+
exclusive with every other registered kind. Register the kind's rubric
|
|
624
|
+
with :func:`playbook_engine.rubric.register_judge_kind` too.
|
|
625
|
+
"""
|
|
626
|
+
_VERDICT_KINDS[kind] = (validate, matches)
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def validate_verdict(kind: str, verdict: dict[str, Any]) -> None:
|
|
630
|
+
"""Validate a producer-supplied *verdict* for *kind* at apply time.
|
|
631
|
+
|
|
632
|
+
Reconstructs the exact dataclass the store-backed judge would build on
|
|
633
|
+
replay, so any verdict accepted here is guaranteed to replay instead of
|
|
634
|
+
silently re-queueing (the issue #182 malformed-verdict loop). Confidence
|
|
635
|
+
fields are type/range-checked up front (issue #161) so a stringified or
|
|
636
|
+
out-of-range confidence raises an actionable ``ValueError`` here instead
|
|
637
|
+
of a bare ``TypeError`` from dataclass construction. Raises
|
|
638
|
+
``ValueError`` with an actionable message on the first problem.
|
|
639
|
+
|
|
640
|
+
Args:
|
|
641
|
+
kind: Pending-item kind: ``classify`` / ``provenance`` / ``scope`` /
|
|
642
|
+
``equivalence`` (or a kind added with
|
|
643
|
+
:func:`register_verdict_kind`).
|
|
644
|
+
verdict: The verdict dict from the producer's JSONL line.
|
|
645
|
+
"""
|
|
646
|
+
basis = verdict.get("basis")
|
|
647
|
+
if basis in _UNRESOLVED_VERDICT_BASES:
|
|
648
|
+
raise ValueError(
|
|
649
|
+
f"basis {basis!r} is engine-internal (means 'not judged'); a "
|
|
650
|
+
"supplied verdict must carry a real basis — use 'judge' "
|
|
651
|
+
"('llm' for provenance)"
|
|
652
|
+
)
|
|
653
|
+
registered = _VERDICT_KINDS.get(kind)
|
|
654
|
+
if registered is None:
|
|
655
|
+
raise ValueError(f"unknown pending-item kind {kind!r}")
|
|
656
|
+
registered[0](verdict)
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def infer_verdict_kind(verdict: dict[str, Any]) -> str | None:
|
|
660
|
+
"""Best-effort kind inference for a verdict whose key is not in pending.
|
|
661
|
+
|
|
662
|
+
Field names are mutually exclusive across the verdict shapes, so this is
|
|
663
|
+
unambiguous when it returns at all; ``None`` means undecidable.
|
|
664
|
+
"""
|
|
665
|
+
for kind, (_validate, matches) in _VERDICT_KINDS.items():
|
|
666
|
+
if matches(verdict):
|
|
667
|
+
return kind
|
|
668
|
+
return None
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def count_judged_items(store: VerdictStore) -> dict[str, int]:
|
|
672
|
+
"""Count the verdicts held in *store* by kind (``"other"`` when undecidable).
|
|
673
|
+
|
|
674
|
+
Judging is done by the invoking agent, so the engine cannot measure tokens or
|
|
675
|
+
dollars; the number of items judged is the honest measure of the work done
|
|
676
|
+
(QA-2026-11-03 #41). Blind checks and adjudications live on the
|
|
677
|
+
verdict (``check``/``adjudicated``) and count under their own keys, on top of it.
|
|
678
|
+
"""
|
|
679
|
+
counts: dict[str, int] = {}
|
|
680
|
+
for verdict in store.all_verdicts():
|
|
681
|
+
if not isinstance(verdict, dict):
|
|
682
|
+
continue
|
|
683
|
+
kind = infer_verdict_kind(verdict) or "other"
|
|
684
|
+
counts[kind] = counts.get(kind, 0) + 1
|
|
685
|
+
check = verdict.get("check")
|
|
686
|
+
if isinstance(check, dict):
|
|
687
|
+
counts["blind_checked"] = counts.get("blind_checked", 0) + 1
|
|
688
|
+
if check.get("adjudicated") is True:
|
|
689
|
+
counts["adjudicated"] = counts.get("adjudicated", 0) + 1
|
|
690
|
+
return counts
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
# ---------------------------------------------------------------------------
|
|
694
|
+
# StoreBackedClassificationJudge
|
|
695
|
+
# ---------------------------------------------------------------------------
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
@dataclass
|
|
699
|
+
class StoreBackedClassificationJudge:
|
|
700
|
+
"""``ClassificationJudge`` that replays stored verdicts or queues new payloads.
|
|
701
|
+
|
|
702
|
+
Implements ``ClassificationJudge.classify_batch`` and is a drop-in
|
|
703
|
+
replacement for the ``classification_judge`` parameter of ``mine_corpus``.
|
|
704
|
+
|
|
705
|
+
On a store hit: returns ``ClauseClassification(basis="judge")`` reconstructed
|
|
706
|
+
from the stored verdict dict.
|
|
707
|
+
|
|
708
|
+
On a store miss: appends the full clause payload (text + heading +
|
|
709
|
+
taxonomy ids) to the pending queue and returns
|
|
710
|
+
``ClauseClassification(basis="needs_review")``.
|
|
711
|
+
|
|
712
|
+
Duplicate payloads within a single ``classify_batch`` call produce exactly
|
|
713
|
+
one pending-queue entry (deduplicated by key).
|
|
714
|
+
"""
|
|
715
|
+
|
|
716
|
+
store: VerdictStore
|
|
717
|
+
pending: PendingQueue
|
|
718
|
+
#: Staleness policy + shared run tally (see :mod:`playbook_engine.rubric`).
|
|
719
|
+
#: The default instance replays legacy verdicts and re-queues stale ones.
|
|
720
|
+
rubric: RubricPolicy = field(default_factory=RubricPolicy)
|
|
721
|
+
_seen_keys: set[str] = field(default_factory=set, init=False, repr=False)
|
|
722
|
+
|
|
723
|
+
def classify_batch(
|
|
724
|
+
self,
|
|
725
|
+
nodes: list[ClauseNode],
|
|
726
|
+
taxonomy: Any,
|
|
727
|
+
hints: Any = None,
|
|
728
|
+
) -> list[ClauseClassification]:
|
|
729
|
+
"""Classify *nodes* from the store or queue them for external review.
|
|
730
|
+
|
|
731
|
+
Args:
|
|
732
|
+
nodes: Clause nodes to classify.
|
|
733
|
+
taxonomy: The full taxonomy (used to extract taxonomy ids for payload).
|
|
734
|
+
hints: Ignored (pass-through for protocol compatibility).
|
|
735
|
+
|
|
736
|
+
Returns:
|
|
737
|
+
One ``ClauseClassification`` per node in the same order.
|
|
738
|
+
"""
|
|
739
|
+
# Active/custom entries only (issue #151) — matches cli.py's
|
|
740
|
+
# segment_cmd (taxonomy.classifier_entries()) and OPF §5 ("a
|
|
741
|
+
# compiler MUST only classify clauses into active or custom
|
|
742
|
+
# entries"). An inactive entry must never appear in the "allowed
|
|
743
|
+
# ids" a judge is shown, or a verdict naming it passes
|
|
744
|
+
# validate_verdict at apply time and then crashes classify_tree on
|
|
745
|
+
# replay.
|
|
746
|
+
tax_labels = classifier_eligible_ids(taxonomy)
|
|
747
|
+
# Computed once per batch from the taxonomy actually in force, so an
|
|
748
|
+
# edit to spec/taxonomy/*.yaml (a re-worded label or description that
|
|
749
|
+
# leaves the id set — and therefore the payload key — untouched)
|
|
750
|
+
# invalidates the classify verdicts it should.
|
|
751
|
+
current_rubric = rubric_version("classify", taxonomy=taxonomy)
|
|
752
|
+
|
|
753
|
+
results: list[ClauseClassification] = []
|
|
754
|
+
for node in nodes:
|
|
755
|
+
# Full text — NOT truncated (contrast with judgment.py text[:500]).
|
|
756
|
+
payload = {
|
|
757
|
+
"stage": "classify",
|
|
758
|
+
"text": node.text or "",
|
|
759
|
+
"heading": node.heading or "",
|
|
760
|
+
"taxonomy_ids": tax_labels,
|
|
761
|
+
}
|
|
762
|
+
key = _payload_key(payload)
|
|
763
|
+
|
|
764
|
+
record = self.store.get_record(payload)
|
|
765
|
+
if (
|
|
766
|
+
record is not None
|
|
767
|
+
and not self.rubric.evaluate(
|
|
768
|
+
"classify", record.rubric_version, current_rubric
|
|
769
|
+
).replay
|
|
770
|
+
):
|
|
771
|
+
# Stored under a rubric that has since moved (or unstamped
|
|
772
|
+
# under --strict-rubric): the banked answer is an answer to a
|
|
773
|
+
# different question. Re-queue instead of replaying silently.
|
|
774
|
+
self.pending.add(key, "classify", payload, current_rubric)
|
|
775
|
+
results.append(_classification_needs_review())
|
|
776
|
+
continue
|
|
777
|
+
cached = record.verdict if record is not None else None
|
|
778
|
+
if cached is not None:
|
|
779
|
+
try:
|
|
780
|
+
results.append(
|
|
781
|
+
ClauseClassification(
|
|
782
|
+
taxonomy_id=cached.get("taxonomy_id"),
|
|
783
|
+
confidence=cached.get("confidence", 0.0),
|
|
784
|
+
basis=cached.get("basis", "judge"),
|
|
785
|
+
)
|
|
786
|
+
)
|
|
787
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
788
|
+
# Isolate one malformed stored verdict (issue #182): must
|
|
789
|
+
# not raise out of classify_batch and get the whole
|
|
790
|
+
# taxonomy batch quarantined as basis='judge_error' by the
|
|
791
|
+
# caller's blanket except.
|
|
792
|
+
_log.warning(
|
|
793
|
+
"StoreBackedClassificationJudge: malformed stored "
|
|
794
|
+
"verdict for key %s (%s); re-queuing for review",
|
|
795
|
+
key,
|
|
796
|
+
exc,
|
|
797
|
+
)
|
|
798
|
+
self.pending.add(key, "classify", payload, current_rubric)
|
|
799
|
+
results.append(_classification_needs_review())
|
|
800
|
+
else:
|
|
801
|
+
# Queue for external verdict (deduplicated by key).
|
|
802
|
+
self.pending.add(key, "classify", payload, current_rubric)
|
|
803
|
+
results.append(_classification_needs_review())
|
|
804
|
+
|
|
805
|
+
return results
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
# ---------------------------------------------------------------------------
|
|
809
|
+
# StoreBackedProvenanceJudge
|
|
810
|
+
# ---------------------------------------------------------------------------
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
@dataclass
|
|
814
|
+
class StoreBackedProvenanceJudge:
|
|
815
|
+
"""``ProvenanceJudge`` that replays stored verdicts or queues new payloads.
|
|
816
|
+
|
|
817
|
+
Implements ``ProvenanceJudge.judge`` and is a drop-in replacement for the
|
|
818
|
+
``provenance_judge`` parameter of ``mine_corpus``.
|
|
819
|
+
|
|
820
|
+
On a store hit: returns ``ProvenanceResult(basis="llm")`` reconstructed from
|
|
821
|
+
the stored verdict dict. (The store may hold any ``_BASIS_VALUES``-valid
|
|
822
|
+
basis; "llm" is the canonical basis for a judge-supplied result per the
|
|
823
|
+
``ProvenanceJudge`` protocol contract.)
|
|
824
|
+
|
|
825
|
+
On a store miss: appends the full provenance payload (preamble + letterhead +
|
|
826
|
+
agreement_type + candidate aliases) to the pending queue and returns
|
|
827
|
+
``ProvenanceResult(provenance="unknown", confidence=0.0,
|
|
828
|
+
basis="needs_review")`` — no side is claimed until a verdict exists, and
|
|
829
|
+
the deal's paper is recorded ``"unknown"`` (issue #225).
|
|
830
|
+
"""
|
|
831
|
+
|
|
832
|
+
store: VerdictStore
|
|
833
|
+
pending: PendingQueue
|
|
834
|
+
#: See ``StoreBackedClassificationJudge.rubric``. Provenance has no
|
|
835
|
+
#: derived input at this seam (``our_party_aliases`` never reaches the
|
|
836
|
+
#: ``ProvenanceJudge`` protocol), so only the manual half moves.
|
|
837
|
+
rubric: RubricPolicy = field(default_factory=RubricPolicy)
|
|
838
|
+
|
|
839
|
+
def judge(
|
|
840
|
+
self,
|
|
841
|
+
preamble: str,
|
|
842
|
+
letterhead: str,
|
|
843
|
+
agreement_type: str,
|
|
844
|
+
) -> ProvenanceResult:
|
|
845
|
+
"""Return a provenance determination from the store or queue for review.
|
|
846
|
+
|
|
847
|
+
Args:
|
|
848
|
+
preamble: First few lines of document body (recital block).
|
|
849
|
+
letterhead: Document title / heading block.
|
|
850
|
+
agreement_type: Human-readable agreement type label.
|
|
851
|
+
|
|
852
|
+
Returns:
|
|
853
|
+
``ProvenanceResult`` with ``basis="llm"`` on a store hit, or
|
|
854
|
+
``ProvenanceResult(basis="needs_review")`` on a miss.
|
|
855
|
+
"""
|
|
856
|
+
payload = {
|
|
857
|
+
"stage": "provenance",
|
|
858
|
+
"preamble": preamble,
|
|
859
|
+
"letterhead": letterhead,
|
|
860
|
+
"agreement_type": agreement_type,
|
|
861
|
+
}
|
|
862
|
+
key = _payload_key(payload)
|
|
863
|
+
|
|
864
|
+
current_rubric = rubric_version("provenance")
|
|
865
|
+
|
|
866
|
+
record = self.store.get_record(payload)
|
|
867
|
+
if (
|
|
868
|
+
record is not None
|
|
869
|
+
and not self.rubric.evaluate("provenance", record.rubric_version, current_rubric).replay
|
|
870
|
+
):
|
|
871
|
+
self.pending.add(key, "provenance", payload, current_rubric)
|
|
872
|
+
return _provenance_needs_review()
|
|
873
|
+
cached = record.verdict if record is not None else None
|
|
874
|
+
if cached is not None:
|
|
875
|
+
try:
|
|
876
|
+
# Same two-sides check as validate_verdict: a stored
|
|
877
|
+
# "unknown" (e.g. a hand-written store row) is malformed and
|
|
878
|
+
# re-queued, never replayed as a verdict (issue #225).
|
|
879
|
+
_require_provenance_side(cached["provenance"])
|
|
880
|
+
return ProvenanceResult(
|
|
881
|
+
provenance=cached["provenance"],
|
|
882
|
+
confidence=cached.get("confidence", 0.0),
|
|
883
|
+
basis=cached.get("basis", "llm"),
|
|
884
|
+
)
|
|
885
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
886
|
+
# Isolate one malformed stored verdict (issue #182) — same
|
|
887
|
+
# pattern as StoreBackedClassificationJudge.
|
|
888
|
+
_log.warning(
|
|
889
|
+
"StoreBackedProvenanceJudge: malformed stored verdict "
|
|
890
|
+
"for key %s (%s); re-queuing for review",
|
|
891
|
+
key,
|
|
892
|
+
exc,
|
|
893
|
+
)
|
|
894
|
+
self.pending.add(key, "provenance", payload, current_rubric)
|
|
895
|
+
return _provenance_needs_review()
|
|
896
|
+
self.pending.add(key, "provenance", payload, current_rubric)
|
|
897
|
+
return _provenance_needs_review()
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
# ---------------------------------------------------------------------------
|
|
901
|
+
# StoreBackedScopeJudge
|
|
902
|
+
# ---------------------------------------------------------------------------
|
|
903
|
+
|
|
904
|
+
|
|
905
|
+
class ScopeNeedsReviewError(Exception):
|
|
906
|
+
"""Raised by ``StoreBackedScopeJudge.judge()`` on a store miss.
|
|
907
|
+
|
|
908
|
+
``ScopeJudge.judge()`` is contractually restricted to returning
|
|
909
|
+
``ScopeDecision(basis="judge")`` — ``scope_gate()`` raises ``ValueError``
|
|
910
|
+
on any other basis returned from a successful call — so "no verdict yet"
|
|
911
|
+
cannot be expressed as a sentinel return value the way the classify and
|
|
912
|
+
provenance store-backed judges use ``basis="needs_review"``.
|
|
913
|
+
|
|
914
|
+
Raising instead lets ``scope_gate()``'s existing exception handling do
|
|
915
|
+
the right thing: it converts this into ``ScopeDecision(basis=
|
|
916
|
+
"judge_error", in_scope=True, scope_confidence=0.0)`` — the document is
|
|
917
|
+
retained and flagged for review, never auto-accepted at the stub
|
|
918
|
+
default's confidence 0.5.
|
|
919
|
+
"""
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
@dataclass
|
|
923
|
+
class StoreBackedScopeJudge:
|
|
924
|
+
"""``ScopeJudge`` that replays stored verdicts or queues new payloads.
|
|
925
|
+
|
|
926
|
+
Implements ``ScopeJudge.judge`` and is a drop-in replacement for the
|
|
927
|
+
``scope_judge`` parameter of ``mine_corpus``. Closes the issue #87 hole
|
|
928
|
+
where every CLI path fell back to ``_AllInScopeJudge`` (every document
|
|
929
|
+
auto-accepted as in-scope at confidence 0.5, regardless of content).
|
|
930
|
+
|
|
931
|
+
On a store hit: returns ``ScopeDecision(basis="judge")`` reconstructed
|
|
932
|
+
from the stored verdict dict — including out-of-scope verdicts, which
|
|
933
|
+
the stub could never produce.
|
|
934
|
+
|
|
935
|
+
On a store miss: appends the full scope payload (agreement type id and
|
|
936
|
+
every clause heading in the document — not capped, unlike the headings-
|
|
937
|
+
only cache key in ``judgment.BatchedScopeJudge``) to the pending queue
|
|
938
|
+
and raises ``ScopeNeedsReviewError``. See that error's docstring for why
|
|
939
|
+
raising (rather than returning a sentinel) is required here.
|
|
940
|
+
|
|
941
|
+
Duplicate payloads across calls produce exactly one pending-queue entry.
|
|
942
|
+
"""
|
|
943
|
+
|
|
944
|
+
store: VerdictStore
|
|
945
|
+
pending: PendingQueue
|
|
946
|
+
#: See ``StoreBackedClassificationJudge.rubric``. The scope rubric's
|
|
947
|
+
#: derived half is the agreement-type definition being gated on, so
|
|
948
|
+
#: editing its description/aliases in the config re-queues scope verdicts.
|
|
949
|
+
rubric: RubricPolicy = field(default_factory=RubricPolicy)
|
|
950
|
+
|
|
951
|
+
def judge(
|
|
952
|
+
self,
|
|
953
|
+
tree: ClauseTree,
|
|
954
|
+
agreement_type: AgreementType,
|
|
955
|
+
) -> ScopeDecision:
|
|
956
|
+
"""Return a scope decision from the store, or queue it for review.
|
|
957
|
+
|
|
958
|
+
Args:
|
|
959
|
+
tree: Segmented clause tree of the document to evaluate.
|
|
960
|
+
agreement_type: Target agreement type from the engine config.
|
|
961
|
+
|
|
962
|
+
Returns:
|
|
963
|
+
``ScopeDecision`` with ``basis="judge"`` on a store hit.
|
|
964
|
+
|
|
965
|
+
Raises:
|
|
966
|
+
ScopeNeedsReviewError: on a store miss, after the payload has
|
|
967
|
+
been queued to the pending queue.
|
|
968
|
+
"""
|
|
969
|
+
payload = {
|
|
970
|
+
"stage": "scope",
|
|
971
|
+
"agreement_type_id": agreement_type.id,
|
|
972
|
+
"document_id": tree.document_id,
|
|
973
|
+
"clause_heads": [node.heading or "" for node in tree.all_nodes()],
|
|
974
|
+
}
|
|
975
|
+
key = _payload_key(payload)
|
|
976
|
+
|
|
977
|
+
current_rubric = rubric_version("scope", agreement_type=agreement_type)
|
|
978
|
+
|
|
979
|
+
record = self.store.get_record(payload)
|
|
980
|
+
if (
|
|
981
|
+
record is not None
|
|
982
|
+
and not self.rubric.evaluate("scope", record.rubric_version, current_rubric).replay
|
|
983
|
+
):
|
|
984
|
+
self.pending.add(key, "scope", payload, current_rubric)
|
|
985
|
+
raise ScopeNeedsReviewError(
|
|
986
|
+
f"Stored scope verdict for document {tree.document_id!r} was made "
|
|
987
|
+
"under an older rubric — re-queued for re-judgement."
|
|
988
|
+
)
|
|
989
|
+
cached = record.verdict if record is not None else None
|
|
990
|
+
if cached is not None:
|
|
991
|
+
try:
|
|
992
|
+
return ScopeDecision(
|
|
993
|
+
in_scope=cached["in_scope"],
|
|
994
|
+
scope_rationale=cached.get("scope_rationale")
|
|
995
|
+
or "Replayed from stored verdict.",
|
|
996
|
+
scope_confidence=cached.get("scope_confidence", 0.0),
|
|
997
|
+
basis="judge",
|
|
998
|
+
)
|
|
999
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
1000
|
+
# Isolate one malformed stored verdict (issue #182) — same
|
|
1001
|
+
# pattern as the other store-backed judges. Scope has
|
|
1002
|
+
# no needs_review sentinel to return (see class docstring),
|
|
1003
|
+
# so re-queue and raise exactly as the miss path below does.
|
|
1004
|
+
_log.warning(
|
|
1005
|
+
"StoreBackedScopeJudge: malformed stored verdict for "
|
|
1006
|
+
"document %s, key %s (%s); re-queuing for review",
|
|
1007
|
+
tree.document_id,
|
|
1008
|
+
key,
|
|
1009
|
+
exc,
|
|
1010
|
+
)
|
|
1011
|
+
self.pending.add(key, "scope", payload, current_rubric)
|
|
1012
|
+
raise ScopeNeedsReviewError(
|
|
1013
|
+
f"Malformed stored scope verdict for document {tree.document_id!r} — "
|
|
1014
|
+
"re-queued for external review."
|
|
1015
|
+
) from exc
|
|
1016
|
+
|
|
1017
|
+
self.pending.add(key, "scope", payload, current_rubric)
|
|
1018
|
+
raise ScopeNeedsReviewError(
|
|
1019
|
+
f"No stored scope verdict for document {tree.document_id!r} — "
|
|
1020
|
+
"queued for external review."
|
|
1021
|
+
)
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
# ---------------------------------------------------------------------------
|
|
1025
|
+
# StoreBackedEquivalenceJudge
|
|
1026
|
+
# ---------------------------------------------------------------------------
|
|
1027
|
+
|
|
1028
|
+
|
|
1029
|
+
@dataclass
|
|
1030
|
+
class StoreBackedEquivalenceJudge:
|
|
1031
|
+
"""Replays stored ``vs_standard`` verdicts or queues new equivalence payloads.
|
|
1032
|
+
|
|
1033
|
+
The ``equivalence`` kind (issue #240). Unlike the three judges above it
|
|
1034
|
+
has no seam inside ``mine_corpus``: the questions come from the precedent
|
|
1035
|
+
record (a deal's signed text, a non-standard opening, a refused ask,
|
|
1036
|
+
each against our standard), which exists only once the observation store
|
|
1037
|
+
is projected. ``playbook judge`` therefore calls :meth:`judge` for every
|
|
1038
|
+
subject after mining, and ``playbook project`` calls it with
|
|
1039
|
+
``pending=None`` to read labels without queueing anything.
|
|
1040
|
+
|
|
1041
|
+
A hit whose rubric moved (stale, or legacy under ``--strict-rubric``) or
|
|
1042
|
+
whose stored record is malformed is not replayed: it is re-queued (when a
|
|
1043
|
+
queue is attached) and the text stays ``null``. A miss is queued, never
|
|
1044
|
+
guessed.
|
|
1045
|
+
"""
|
|
1046
|
+
|
|
1047
|
+
store: VerdictStore
|
|
1048
|
+
pending: PendingQueue | None = None
|
|
1049
|
+
rubric: RubricPolicy = field(default_factory=RubricPolicy)
|
|
1050
|
+
|
|
1051
|
+
def judge(self, subject: EquivalenceSubject) -> dict[str, Any] | None:
|
|
1052
|
+
"""The ``vs_standard`` object for *subject*, or ``None`` when unjudged."""
|
|
1053
|
+
current_rubric = rubric_version(EQUIVALENCE_KIND)
|
|
1054
|
+
record = self.store.get_record_by_key(subject.key)
|
|
1055
|
+
if (
|
|
1056
|
+
record is not None
|
|
1057
|
+
and self.rubric.evaluate(EQUIVALENCE_KIND, record.rubric_version, current_rubric).replay
|
|
1058
|
+
):
|
|
1059
|
+
try:
|
|
1060
|
+
# Against THIS standard's parts: a judgment about other parts
|
|
1061
|
+
# (a hand-edited store) is malformed, never replayed.
|
|
1062
|
+
return vs_standard_of(record.verdict, subject.part_ids)
|
|
1063
|
+
except ValueError as exc:
|
|
1064
|
+
_log.warning(
|
|
1065
|
+
"StoreBackedEquivalenceJudge: malformed stored verdict for key %s (%s); "
|
|
1066
|
+
"treating as unjudged",
|
|
1067
|
+
subject.key,
|
|
1068
|
+
exc,
|
|
1069
|
+
)
|
|
1070
|
+
if self.pending is not None:
|
|
1071
|
+
self.pending.add(subject.key, EQUIVALENCE_KIND, subject.payload, current_rubric)
|
|
1072
|
+
return None
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
# ---------------------------------------------------------------------------
|
|
1076
|
+
# StoreBackedDealDateJudge
|
|
1077
|
+
# ---------------------------------------------------------------------------
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
@dataclass
|
|
1081
|
+
class StoreBackedDealDateJudge:
|
|
1082
|
+
"""Replays stored ``deal_date`` verdicts or queues new date-passage payloads.
|
|
1083
|
+
|
|
1084
|
+
The ``deal_date`` kind (issue #247). Like the equivalence judge it has no
|
|
1085
|
+
seam inside ``mine_corpus``: the question (the date passages of an executed
|
|
1086
|
+
copy that no deterministic reader could date) is recorded in the corpus
|
|
1087
|
+
manifest at mining, so ``playbook judge`` asks it after mining and
|
|
1088
|
+
``playbook project`` reads it with ``pending=None``.
|
|
1089
|
+
|
|
1090
|
+
A hit whose rubric moved, or whose stored record does not validate against
|
|
1091
|
+
the payload (a quote that does not occur in the passages shown, a bad
|
|
1092
|
+
date), is not replayed: it is re-queued (when a queue is attached) and the
|
|
1093
|
+
deal's date falls through to the file name / metadata. A miss is queued,
|
|
1094
|
+
never guessed.
|
|
1095
|
+
"""
|
|
1096
|
+
|
|
1097
|
+
store: VerdictStore
|
|
1098
|
+
pending: PendingQueue | None = None
|
|
1099
|
+
rubric: RubricPolicy = field(default_factory=RubricPolicy)
|
|
1100
|
+
|
|
1101
|
+
def judge(self, subject: deal_dates.DealDateSubject) -> deal_dates.DealDate | None:
|
|
1102
|
+
"""The stored answer for *subject*, or ``None`` when it is unjudged."""
|
|
1103
|
+
current_rubric = rubric_version(deal_dates.DEAL_DATE_KIND)
|
|
1104
|
+
key = _payload_key(subject.payload)
|
|
1105
|
+
record = self.store.get_record_by_key(key)
|
|
1106
|
+
if (
|
|
1107
|
+
record is not None
|
|
1108
|
+
and self.rubric.evaluate(
|
|
1109
|
+
deal_dates.DEAL_DATE_KIND, record.rubric_version, current_rubric
|
|
1110
|
+
).replay
|
|
1111
|
+
):
|
|
1112
|
+
answer = deal_dates.verdict_of(record.verdict, subject.payload)
|
|
1113
|
+
if answer is not None:
|
|
1114
|
+
return answer
|
|
1115
|
+
_log.warning(
|
|
1116
|
+
"StoreBackedDealDateJudge: malformed stored verdict for key %s; "
|
|
1117
|
+
"treating as unjudged",
|
|
1118
|
+
key,
|
|
1119
|
+
)
|
|
1120
|
+
if self.pending is not None:
|
|
1121
|
+
self.pending.add(key, deal_dates.DEAL_DATE_KIND, subject.payload, current_rubric)
|
|
1122
|
+
return None
|
|
1123
|
+
|
|
1124
|
+
|
|
1125
|
+
# ---------------------------------------------------------------------------
|
|
1126
|
+
# StoreBackedResegmentCheckJudge
|
|
1127
|
+
# ---------------------------------------------------------------------------
|
|
1128
|
+
|
|
1129
|
+
|
|
1130
|
+
@dataclass
|
|
1131
|
+
class StoreBackedResegmentCheckJudge:
|
|
1132
|
+
"""Replays stored ``resegment_check`` verdicts or queues new questions.
|
|
1133
|
+
|
|
1134
|
+
The ``resegment_check`` kind (issue #255). A question is one sampled split
|
|
1135
|
+
("is this text really a provision of that clause type?") or join ("do
|
|
1136
|
+
these passages form one clause?") of the re-segmentation pass, read from
|
|
1137
|
+
the mined ``normalized/`` trees (:func:`playbook_engine.resegmentation.check_subjects`).
|
|
1138
|
+
Like the other post-mining kinds it has no seam inside ``mine_corpus``,
|
|
1139
|
+
and its answer feeds nothing back: the verdicts are the measurement the
|
|
1140
|
+
pass's thresholds are tuned against.
|
|
1141
|
+
|
|
1142
|
+
A hit whose rubric moved, or whose stored record is not a valid verdict,
|
|
1143
|
+
is not replayed: it is re-queued (when a queue is attached) and reads as
|
|
1144
|
+
unchecked. A miss is queued, never guessed.
|
|
1145
|
+
"""
|
|
1146
|
+
|
|
1147
|
+
store: VerdictStore
|
|
1148
|
+
pending: PendingQueue | None = None
|
|
1149
|
+
rubric: RubricPolicy = field(default_factory=RubricPolicy)
|
|
1150
|
+
|
|
1151
|
+
def judge(self, subject: resegmentation.ResegmentCheckSubject) -> bool | None:
|
|
1152
|
+
"""Whether the pass was right about *subject*, or ``None`` when unchecked."""
|
|
1153
|
+
current_rubric = rubric_version(resegmentation.CHECK_KIND)
|
|
1154
|
+
record = self.store.get_record_by_key(subject.key)
|
|
1155
|
+
if (
|
|
1156
|
+
record is not None
|
|
1157
|
+
and self.rubric.evaluate(
|
|
1158
|
+
resegmentation.CHECK_KIND, record.rubric_version, current_rubric
|
|
1159
|
+
).replay
|
|
1160
|
+
):
|
|
1161
|
+
try:
|
|
1162
|
+
resegmentation.validate_check_verdict(record.verdict)
|
|
1163
|
+
return bool(record.verdict["resegment_ok"])
|
|
1164
|
+
except ValueError:
|
|
1165
|
+
_log.warning(
|
|
1166
|
+
"StoreBackedResegmentCheckJudge: malformed stored verdict for key %s; "
|
|
1167
|
+
"treating as unchecked",
|
|
1168
|
+
subject.key,
|
|
1169
|
+
)
|
|
1170
|
+
if self.pending is not None:
|
|
1171
|
+
self.pending.add(
|
|
1172
|
+
subject.key, resegmentation.CHECK_KIND, subject.payload, current_rubric
|
|
1173
|
+
)
|
|
1174
|
+
return None
|
|
1175
|
+
|
|
1176
|
+
|
|
1177
|
+
# ---------------------------------------------------------------------------
|
|
1178
|
+
# StoreBackedPriorFormOriginJudge
|
|
1179
|
+
# ---------------------------------------------------------------------------
|
|
1180
|
+
|
|
1181
|
+
|
|
1182
|
+
@dataclass
|
|
1183
|
+
class StoreBackedPriorFormOriginJudge:
|
|
1184
|
+
"""Replays stored ``prior_form_origin`` verdicts or queues new questions.
|
|
1185
|
+
|
|
1186
|
+
The ``prior_form_origin`` kind (issue #250). A question is one recurring
|
|
1187
|
+
first-draft wording on our paper that a corpus-reconstructed prior form
|
|
1188
|
+
would be built from: "is this really OUR former language, not a
|
|
1189
|
+
counterparty form that recurs?"
|
|
1190
|
+
(:func:`playbook_engine.prior_reconstruction.origin_subjects`). Like the
|
|
1191
|
+
other post-mining kinds it has no seam inside ``mine_corpus``: ``playbook
|
|
1192
|
+
judge`` asks it after mining and ``playbook project`` reads it with
|
|
1193
|
+
``pending=None``. Whether the verdict has been blind-checked is the
|
|
1194
|
+
cluster's business (``prior_reconstruction.outcome_of``), not the judge's.
|
|
1195
|
+
|
|
1196
|
+
A hit whose rubric moved, or whose stored record is not a valid verdict, is
|
|
1197
|
+
not replayed: it is re-queued (when a queue is attached) and reads as
|
|
1198
|
+
unjudged. A miss is queued, never guessed.
|
|
1199
|
+
"""
|
|
1200
|
+
|
|
1201
|
+
store: VerdictStore
|
|
1202
|
+
pending: PendingQueue | None = None
|
|
1203
|
+
rubric: RubricPolicy = field(default_factory=RubricPolicy)
|
|
1204
|
+
|
|
1205
|
+
def judge(self, subject: prior_reconstruction.OriginSubject) -> dict[str, Any] | None:
|
|
1206
|
+
"""The stored verdict for *subject*, or ``None`` when it is unjudged."""
|
|
1207
|
+
kind = prior_reconstruction.ORIGIN_KIND
|
|
1208
|
+
current_rubric = rubric_version(kind)
|
|
1209
|
+
record = self.store.get_record_by_key(subject.key)
|
|
1210
|
+
if (
|
|
1211
|
+
record is not None
|
|
1212
|
+
and self.rubric.evaluate(kind, record.rubric_version, current_rubric).replay
|
|
1213
|
+
):
|
|
1214
|
+
verdict = prior_reconstruction.verdict_of(record.verdict)
|
|
1215
|
+
if verdict is not None:
|
|
1216
|
+
return verdict
|
|
1217
|
+
_log.warning(
|
|
1218
|
+
"StoreBackedPriorFormOriginJudge: malformed stored verdict for key %s; "
|
|
1219
|
+
"treating as unjudged",
|
|
1220
|
+
subject.key,
|
|
1221
|
+
)
|
|
1222
|
+
if self.pending is not None:
|
|
1223
|
+
self.pending.add(subject.key, kind, subject.payload, current_rubric)
|
|
1224
|
+
return None
|
|
1225
|
+
|
|
1226
|
+
|
|
1227
|
+
# ---------------------------------------------------------------------------
|
|
1228
|
+
# StoreBackedAbsenceCheckJudge
|
|
1229
|
+
# ---------------------------------------------------------------------------
|
|
1230
|
+
|
|
1231
|
+
|
|
1232
|
+
@dataclass
|
|
1233
|
+
class StoreBackedAbsenceCheckJudge:
|
|
1234
|
+
"""Replays stored ``absence_check`` verdicts or queues new questions.
|
|
1235
|
+
|
|
1236
|
+
The ``absence_check`` kind (issue #251). A question is one sampled verified
|
|
1237
|
+
absence: "is this clause type really absent from this signed text?"
|
|
1238
|
+
(:func:`playbook_engine.pipeline.absence_check_subjects`). Like the other
|
|
1239
|
+
post-mining kinds it has no seam inside ``mine_corpus``, and its answer feeds
|
|
1240
|
+
nothing back into the playbook: the verdicts are the measurement the absence
|
|
1241
|
+
guard (:data:`playbook_engine.absence.ABSENCE_GUARD`) is tuned against, and
|
|
1242
|
+
an item judged present (``absent: false``) means the guard is too high.
|
|
1243
|
+
|
|
1244
|
+
A hit whose rubric moved, or whose stored record is not a valid verdict, is
|
|
1245
|
+
not replayed: it is re-queued (when a queue is attached) and reads as
|
|
1246
|
+
unchecked. A miss is queued, never guessed.
|
|
1247
|
+
"""
|
|
1248
|
+
|
|
1249
|
+
store: VerdictStore
|
|
1250
|
+
pending: PendingQueue | None = None
|
|
1251
|
+
rubric: RubricPolicy = field(default_factory=RubricPolicy)
|
|
1252
|
+
|
|
1253
|
+
def judge(self, subject: absence.AbsenceSubject) -> bool | None:
|
|
1254
|
+
"""Whether the clause is really absent from *subject*'s text, or ``None`` when unchecked."""
|
|
1255
|
+
current_rubric = rubric_version(absence.CHECK_KIND)
|
|
1256
|
+
record = self.store.get_record_by_key(subject.key)
|
|
1257
|
+
if (
|
|
1258
|
+
record is not None
|
|
1259
|
+
and self.rubric.evaluate(
|
|
1260
|
+
absence.CHECK_KIND, record.rubric_version, current_rubric
|
|
1261
|
+
).replay
|
|
1262
|
+
):
|
|
1263
|
+
try:
|
|
1264
|
+
absence.validate_check_verdict(record.verdict)
|
|
1265
|
+
return bool(record.verdict["absent"])
|
|
1266
|
+
except ValueError:
|
|
1267
|
+
_log.warning(
|
|
1268
|
+
"StoreBackedAbsenceCheckJudge: malformed stored verdict for key %s; "
|
|
1269
|
+
"treating as unchecked",
|
|
1270
|
+
subject.key,
|
|
1271
|
+
)
|
|
1272
|
+
if self.pending is not None:
|
|
1273
|
+
self.pending.add(subject.key, absence.CHECK_KIND, subject.payload, current_rubric)
|
|
1274
|
+
return None
|
|
1275
|
+
|
|
1276
|
+
|
|
1277
|
+
# ---------------------------------------------------------------------------
|
|
1278
|
+
# StoreBackedSubsplitJudge
|
|
1279
|
+
# ---------------------------------------------------------------------------
|
|
1280
|
+
|
|
1281
|
+
|
|
1282
|
+
@dataclass
|
|
1283
|
+
class StoreBackedSubsplitJudge:
|
|
1284
|
+
"""Replays settled ``subsplit`` verdicts or queues new questions (issue #257).
|
|
1285
|
+
|
|
1286
|
+
Asked while mining, in the re-segmentation stage
|
|
1287
|
+
(:func:`playbook_engine.subsplit.apply_subsplits`), once per candidate
|
|
1288
|
+
node. Returns the stored verdict only when it is current under the
|
|
1289
|
+
subject's rubric, replayable and SETTLED (an owner's, or a draft whose
|
|
1290
|
+
blind check agreed or that was adjudicated:
|
|
1291
|
+
:func:`~playbook_engine.subsplit.is_settled`); a false clause boundary is
|
|
1292
|
+
worse than a mixed node, so a draft nobody checked yet, or a disputed one,
|
|
1293
|
+
changes nothing and is not re-queued (it waits on ``playbook judge
|
|
1294
|
+
--check subsplit``). A miss, a moved rubric or a malformed stored verdict
|
|
1295
|
+
is queued, never guessed.
|
|
1296
|
+
|
|
1297
|
+
The lookup goes through :meth:`VerdictStore.get_record` so the stage cache
|
|
1298
|
+
records the dependency: a cached document is recomputed when its verdict
|
|
1299
|
+
is checked or re-drafted.
|
|
1300
|
+
"""
|
|
1301
|
+
|
|
1302
|
+
store: VerdictStore
|
|
1303
|
+
pending: PendingQueue | None = None
|
|
1304
|
+
rubric: RubricPolicy = field(default_factory=RubricPolicy)
|
|
1305
|
+
|
|
1306
|
+
def judge(self, subject: subsplit.SubsplitSubject) -> dict[str, Any] | None:
|
|
1307
|
+
"""The settled verdict for *subject*, or ``None`` (pending, unchecked or disputed)."""
|
|
1308
|
+
record = self.store.get_record(subject.payload)
|
|
1309
|
+
current = subject.rubric_version
|
|
1310
|
+
if record is not None and (
|
|
1311
|
+
current is None
|
|
1312
|
+
or self.rubric.evaluate(subsplit.SUBSPLIT_KIND, record.rubric_version, current).replay
|
|
1313
|
+
):
|
|
1314
|
+
try:
|
|
1315
|
+
subsplit.validate_replayable(record.verdict)
|
|
1316
|
+
except ValueError:
|
|
1317
|
+
_log.warning(
|
|
1318
|
+
"StoreBackedSubsplitJudge: malformed stored verdict for key %s; re-queuing",
|
|
1319
|
+
subject.key,
|
|
1320
|
+
)
|
|
1321
|
+
else:
|
|
1322
|
+
return dict(record.verdict) if subsplit.is_settled(record.verdict) else None
|
|
1323
|
+
if self.pending is not None:
|
|
1324
|
+
self.pending.add(subject.key, subsplit.SUBSPLIT_KIND, subject.payload, current)
|
|
1325
|
+
return None
|