cjm-transcript-correction-core 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cjm_transcript_correction_core/__init__.py +1 -0
- cjm_transcript_correction_core/cli.py +193 -0
- cjm_transcript_correction_core/graph.py +783 -0
- cjm_transcript_correction_core/models.py +171 -0
- cjm_transcript_correction_core/pipeline.py +489 -0
- cjm_transcript_correction_core/signals.py +208 -0
- cjm_transcript_correction_core-0.0.1.dist-info/METADATA +135 -0
- cjm_transcript_correction_core-0.0.1.dist-info/RECORD +12 -0
- cjm_transcript_correction_core-0.0.1.dist-info/WHEEL +5 -0
- cjm_transcript_correction_core-0.0.1.dist-info/entry_points.txt +2 -0
- cjm_transcript_correction_core-0.0.1.dist-info/licenses/LICENSE +201 -0
- cjm_transcript_correction_core-0.0.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Overlay data shapes for the transcript-correction workflow: the Correction / CorrectionSession graph nodes + their relation registry, the read view of a committed spine segment, the worklist item, run configuration, and the correction run manifest (proto-bundle that chains decomp -> correction)."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import time
|
|
5
|
+
import uuid
|
|
6
|
+
from dataclasses import asdict, dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Dict, List, Optional, Union
|
|
9
|
+
from uuid import uuid4
|
|
10
|
+
|
|
11
|
+
from cjm_context_graph_primitives.graph import GraphNode
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class CorrectionRelations:
|
|
15
|
+
"""Registry of edge types the correction overlay adds to the spine graph."""
|
|
16
|
+
CORRECTS = "CORRECTS" # Correction -> the layer-0 Segment(s) it corrects
|
|
17
|
+
SUPERSEDES = "SUPERSEDES" # Correction -> the prior Correction it replaces (undo/update chain)
|
|
18
|
+
DERIVED_FROM = "DERIVED_FROM" # grouping Correction -> the layer-0 Segments it regroups/prunes
|
|
19
|
+
REVIEWED = "REVIEWED" # CorrectionSession -> Segment (carries a `decision` property)
|
|
20
|
+
|
|
21
|
+
@classmethod
|
|
22
|
+
def all(cls) -> list: # All relation type strings
|
|
23
|
+
"""Return all defined relation types."""
|
|
24
|
+
return [v for k, v in cls.__dict__.items()
|
|
25
|
+
if not k.startswith('_') and isinstance(v, str)]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class Correction:
|
|
30
|
+
"""A single non-destructive correction over the committed spine (overlay node).
|
|
31
|
+
|
|
32
|
+
Layer-0 spine nodes are immutable; every correction is a supersede-able
|
|
33
|
+
overlay. Defined IN-CORE (the C6 pattern, kept at stage 5 after
|
|
34
|
+
cjm-graph-domains dissolved): a plain dataclass mapping itself onto the
|
|
35
|
+
generic GraphNode. Corrections are DECISIONS (asserted events) — they keep
|
|
36
|
+
GENERATED ids, the FLIP-TRIGGER-protected class.
|
|
37
|
+
"""
|
|
38
|
+
correction_type: str # "text_content" | "punctuation" | "grouping" | "review"
|
|
39
|
+
status: str = "applied" # "proposed" | "applied" | "superseded"
|
|
40
|
+
session_id: str = "" # Owning CorrectionSession id
|
|
41
|
+
payload: Dict[str, Any] = field(default_factory=dict) # Type-specific data (new text, prune set, ...)
|
|
42
|
+
actor: str = "human" # "human" | "agent:<id>" | "capability:<name>"
|
|
43
|
+
canonical_form: Optional[str] = None # Optional entity key (cross-transcript matching)
|
|
44
|
+
rationale: Optional[str] = None # Optional human/agent note
|
|
45
|
+
created_at: float = field(default_factory=time.time) # Unix timestamp
|
|
46
|
+
id: str = field(default_factory=lambda: str(uuid4())) # Generated node id (decision = event)
|
|
47
|
+
|
|
48
|
+
def to_graph_node(self) -> GraphNode: # Generic graph node (label = class name)
|
|
49
|
+
"""Map onto a generic GraphNode (None-valued fields excluded from properties)."""
|
|
50
|
+
props = {k: v for k, v in asdict(self).items() if k != "id" and v is not None}
|
|
51
|
+
return GraphNode(id=self.id, label="Correction", properties=props, sources=[])
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class CorrectionSession:
|
|
56
|
+
"""A resumable, reopen-able correction review over one or more sources."""
|
|
57
|
+
status: str = "in_progress" # "in_progress" | "completed" | "reopened"
|
|
58
|
+
scope: List[str] = field(default_factory=list) # Source node ids in scope
|
|
59
|
+
started_at: float = field(default_factory=time.time) # Unix timestamp at session start
|
|
60
|
+
updated_at: float = field(default_factory=time.time) # Unix timestamp of last activity
|
|
61
|
+
id: str = field(default_factory=lambda: str(uuid4())) # Generated node id (session = event)
|
|
62
|
+
|
|
63
|
+
def to_graph_node(self) -> GraphNode: # Generic graph node
|
|
64
|
+
"""Map onto a generic GraphNode (None-valued fields excluded from properties)."""
|
|
65
|
+
props = {k: v for k, v in asdict(self).items() if k != "id" and v is not None}
|
|
66
|
+
return GraphNode(id=self.id, label="CorrectionSession", properties=props, sources=[])
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass
|
|
70
|
+
class SpineSegment:
|
|
71
|
+
"""A committed layer-0 Segment loaded from the graph (read view).
|
|
72
|
+
|
|
73
|
+
Stage 5 (Source-rooted schema): segments carry an audio `TimeSlice` ref
|
|
74
|
+
(the stable anchor) + per-transcriber `CharSlice` refs into Transcript
|
|
75
|
+
nodes; `content_hash` is the AUTHORITATIVE text's hash (the `text_from`
|
|
76
|
+
slice) — the cross-transcript cache key."""
|
|
77
|
+
id: str # Graph Segment node id
|
|
78
|
+
index: int # 0-based position in the source spine
|
|
79
|
+
text: str # Layer-0 text (may be empty for silence VAD chunks)
|
|
80
|
+
start_time: Optional[float] = None # Source-coordinate start (seconds)
|
|
81
|
+
end_time: Optional[float] = None # Source-coordinate end (seconds)
|
|
82
|
+
source_locator: Optional[str] = None # Audio SourceRef locator URI (the stable provenance anchor)
|
|
83
|
+
content_hash: Optional[str] = None # Authoritative text slice's content_hash (None when empty)
|
|
84
|
+
text_from: Optional[str] = None # Authoritative Transcript node id (provenance designation)
|
|
85
|
+
text_slices: List[Dict[str, Any]] = field(default_factory=list) # [{transcript, start, end, content_hash}]
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def is_empty(self) -> bool: # True when the segment has no non-whitespace text
|
|
89
|
+
"""Empty-text segment (silence VAD chunk with no aligned words; decomp D14)."""
|
|
90
|
+
return not (self.text or "").strip()
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@dataclass
|
|
94
|
+
class WorklistItem:
|
|
95
|
+
"""One spine segment surfaced for review, with its deterministic Tier-1 flags."""
|
|
96
|
+
segment: SpineSegment # The segment under review
|
|
97
|
+
flags: List[str] = field(default_factory=list) # Tier-1 signal flags (empty, boundary, divergence, ...)
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def index(self) -> int: # Segment spine index
|
|
101
|
+
"""Spine index of the underlying segment."""
|
|
102
|
+
return self.segment.index
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass
|
|
106
|
+
class CorrectionConfig:
|
|
107
|
+
"""Configuration for one correction run."""
|
|
108
|
+
graph_capability: str = "cjm-capability-graph-sqlite" # Graph-storage capability id
|
|
109
|
+
graph_db_path: Optional[str] = None # Graph DB the spine lives in (from the decomp manifest)
|
|
110
|
+
actor: str = "human" # Actor recorded on corrections + review markers
|
|
111
|
+
assume_yes: bool = False # Auto-accept HITL seams (headless mode)
|
|
112
|
+
prune_empty: bool = True # Run the D14 empty-segment prune as the first operation
|
|
113
|
+
rendition_selector: Optional[str] = None # Which AudioRendition spine to correct ("raw" | preprocessing substring); None = auto-select the populated one (error if ambiguous)
|
|
114
|
+
|
|
115
|
+
def to_dict(self) -> Dict[str, Any]: # Plain-dict snapshot for the manifest
|
|
116
|
+
"""Serialize to a plain dict."""
|
|
117
|
+
return asdict(self)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclass
|
|
121
|
+
class CorrectionManifest:
|
|
122
|
+
"""Durable record of one correction run (proto-bundle; chainable, CR-20).
|
|
123
|
+
|
|
124
|
+
Schema 0.2.0 (stage 5): `documents` became `sources` (Document dissolved
|
|
125
|
+
into Source); the cross-transcriber diff is intra-graph now, so the
|
|
126
|
+
secondary-manifest pointer is gone."""
|
|
127
|
+
run_id: str # Unique run identifier
|
|
128
|
+
created_at: float # Unix timestamp at run start
|
|
129
|
+
config: Dict[str, Any] # CorrectionConfig snapshot
|
|
130
|
+
decomp_manifest: str # Path to the consumed decomp run manifest
|
|
131
|
+
graph_db_path: str # The shared graph DB the spine + overlay live in
|
|
132
|
+
session_id: str # CorrectionSession node id this run used
|
|
133
|
+
source_format: str = "" # Upstream manifest format tag (interchange contract)
|
|
134
|
+
source_version: str = "" # Upstream manifest schema version
|
|
135
|
+
signals_used: List[str] = field(default_factory=list) # Deterministic signals active this run
|
|
136
|
+
sources: List[Dict[str, Any]] = field(default_factory=list) # Per-source outcome records
|
|
137
|
+
|
|
138
|
+
FORMAT: str = field(default="cjm-transcript-correction-core/run-manifest", repr=False) # Format tag
|
|
139
|
+
VERSION: str = field(default="0.2.0", repr=False) # Schema version
|
|
140
|
+
|
|
141
|
+
def to_dict(self) -> Dict[str, Any]: # Plain-dict form for JSON serialization
|
|
142
|
+
"""Serialize to a plain dict."""
|
|
143
|
+
return {
|
|
144
|
+
"format": self.FORMAT,
|
|
145
|
+
"version": self.VERSION,
|
|
146
|
+
"run_id": self.run_id,
|
|
147
|
+
"created_at": self.created_at,
|
|
148
|
+
"config": self.config,
|
|
149
|
+
"decomp_manifest": self.decomp_manifest,
|
|
150
|
+
"graph_db_path": self.graph_db_path,
|
|
151
|
+
"session_id": self.session_id,
|
|
152
|
+
"source_format": self.source_format,
|
|
153
|
+
"source_version": self.source_version,
|
|
154
|
+
"signals_used": list(self.signals_used),
|
|
155
|
+
"sources": list(self.sources),
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
def save(
|
|
159
|
+
self,
|
|
160
|
+
path: Union[str, Path], # Destination JSON file (parent dirs created)
|
|
161
|
+
) -> Path: # The written path
|
|
162
|
+
"""Write the manifest as pretty-printed JSON."""
|
|
163
|
+
out = Path(path)
|
|
164
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
165
|
+
out.write_text(json.dumps(self.to_dict(), indent=2))
|
|
166
|
+
return out
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def new_run_id() -> str: # e.g. "correct_20260608_153000_1a2b3c4d"
|
|
170
|
+
"""Generate a unique, sortable correction run id."""
|
|
171
|
+
return f"correct_{time.strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}"
|
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
"""The headless correction workflow: load a decomp run manifest, resolve the shared graph DB, start/resume/reopen a CorrectionSession, recompute the worklist from deterministic signals + persisted review state, run the D14 empty-segment prune (first operation), and record a chainable correction run manifest — with a cheapest-form HITL approval seam."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import time
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Dict, List, Optional
|
|
8
|
+
|
|
9
|
+
from cjm_substrate.core.journal_store import JournalEvent, SubstrateEventType
|
|
10
|
+
from cjm_substrate.core.manager import CapabilityManager
|
|
11
|
+
from cjm_substrate.core.queue import JobQueue
|
|
12
|
+
from cjm_transcript_correction_core.graph import (active_corrections, build_prune_correction,
|
|
13
|
+
commit_nodes_edges, commit_text_correction,
|
|
14
|
+
count_source_segments,
|
|
15
|
+
find_active_text_corrections_batch,
|
|
16
|
+
find_corrections_for_session,
|
|
17
|
+
find_prior_corrections_by_hash, get_session,
|
|
18
|
+
load_empty_segments, load_review_markers,
|
|
19
|
+
load_source_corrections, load_source_segments,
|
|
20
|
+
load_variant_texts, project_effective_spine,
|
|
21
|
+
record_review_markers, set_session_status,
|
|
22
|
+
start_session)
|
|
23
|
+
from cjm_transcript_correction_core.models import (CorrectionConfig, CorrectionManifest, new_run_id,
|
|
24
|
+
SpineSegment, WorklistItem)
|
|
25
|
+
from cjm_transcript_correction_core.signals import (compute_signal_flags, detect_empty_segments,
|
|
26
|
+
variant_divergence)
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def load_decomp_manifest(
|
|
32
|
+
path: str, # Path to a decomp-core run manifest JSON
|
|
33
|
+
) -> Dict[str, Any]: # Parsed manifest dict
|
|
34
|
+
"""Load + lightly validate a decomp-core run manifest (untyped JSON; CR-20 interchange)."""
|
|
35
|
+
data = json.loads(Path(path).read_text())
|
|
36
|
+
fmt = data.get("format", "")
|
|
37
|
+
if "decomp-core" not in fmt:
|
|
38
|
+
logger.warning(f"unexpected decomp manifest format: {fmt!r} (continuing)")
|
|
39
|
+
return data
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def resolve_graph_db_path(
|
|
43
|
+
manifest: Dict[str, Any], # Decomp manifest dict
|
|
44
|
+
graph_capability: str, # Graph-storage capability id
|
|
45
|
+
override: Optional[str] = None, # Explicit override (wins)
|
|
46
|
+
) -> Optional[str]: # Absolute DB path the spine lives in
|
|
47
|
+
"""Resolve the graph DB path: explicit override > the decomp manifest's recorded db_path.
|
|
48
|
+
|
|
49
|
+
The spine lives in the decomp-core graph DB; this core writes its overlay into
|
|
50
|
+
the SAME DB (a shared substrate resource owned by no single core -- pass-2
|
|
51
|
+
graph-DB-ownership evidence).
|
|
52
|
+
"""
|
|
53
|
+
if override:
|
|
54
|
+
return override
|
|
55
|
+
rec = (manifest.get("capabilities", {}) or {}).get(graph_capability, {}) or {}
|
|
56
|
+
return rec.get("db_path")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def compute_worklist(
|
|
60
|
+
segments: List[SpineSegment], # Ordered layer-0 spine
|
|
61
|
+
review_markers: Dict[str, str], # segment_id -> decision (persisted)
|
|
62
|
+
variants: Optional[Dict[str, Dict[str, str]]] = None, # segment_id -> {transcriber: text} (intra-graph)
|
|
63
|
+
) -> List[WorklistItem]: # Items still needing review, flagged
|
|
64
|
+
"""Recompute the worklist from layer-0 + signals + review state (only decisions persist).
|
|
65
|
+
|
|
66
|
+
Segments already decided in this session (reviewed/corrected/skipped) drop out;
|
|
67
|
+
everything flagged by a deterministic signal and not yet decided surfaces.
|
|
68
|
+
"""
|
|
69
|
+
flags = compute_signal_flags(segments, variants=variants)
|
|
70
|
+
items: List[WorklistItem] = []
|
|
71
|
+
for i, s in enumerate(segments):
|
|
72
|
+
if review_markers.get(s.id): # already decided this session
|
|
73
|
+
continue
|
|
74
|
+
fl = flags.get(i, [])
|
|
75
|
+
if fl:
|
|
76
|
+
items.append(WorklistItem(segment=s, flags=fl))
|
|
77
|
+
return items
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def confirm_seam(
|
|
81
|
+
seam: str, # Seam label
|
|
82
|
+
summary_lines: List[str], # What the operator is accepting
|
|
83
|
+
warnings: List[str], # Tier-1 warnings
|
|
84
|
+
assume_yes: bool = False, # Headless: accept without prompting
|
|
85
|
+
) -> bool: # True = proceed, False = aborted
|
|
86
|
+
"""HITL approval seam in its cheapest viable form (log + optional CLI prompt).
|
|
87
|
+
|
|
88
|
+
Per-seam HITL-assist annotation (5 fields):
|
|
89
|
+
1. signal: per-document summary + Tier-1 worklist flags
|
|
90
|
+
2. deterministic pre-filter: compute_signal_flags (no AI)
|
|
91
|
+
3. modality-bridge candidate: spectrogram / audio review (future Tier 2)
|
|
92
|
+
4. authoritative verifier: re-transcribe-and-compare (future Tier 3; Gemini)
|
|
93
|
+
5. flywheel capture: decisions persist as graph nodes/edges (DURABLE, unlike
|
|
94
|
+
decomp's log-only seam -- the correction overlay IS the captured decision)
|
|
95
|
+
input() blocks the loop; safe between stages with no jobs in flight (pass-2).
|
|
96
|
+
"""
|
|
97
|
+
for line in summary_lines:
|
|
98
|
+
logger.info(f"[{seam}] {line}")
|
|
99
|
+
for w in warnings:
|
|
100
|
+
logger.warning(f"[{seam}] {w}")
|
|
101
|
+
if assume_yes:
|
|
102
|
+
logger.info(f"[{seam}] auto-accepted (assume_yes)")
|
|
103
|
+
return True
|
|
104
|
+
reply = input(f"[{seam}] proceed? [Y/n] ").strip().lower()
|
|
105
|
+
accepted = reply in ("", "y", "yes")
|
|
106
|
+
logger.info(f"[{seam}] {'accepted' if accepted else 'ABORTED'} by operator")
|
|
107
|
+
return accepted
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
async def prune_empty_segments(
|
|
111
|
+
queue: JobQueue, # Started job queue
|
|
112
|
+
cfg: CorrectionConfig, # Run configuration
|
|
113
|
+
graph_id: str, # Graph-storage capability id
|
|
114
|
+
source_id: str, # Source being corrected
|
|
115
|
+
total_count: int, # Total segment count (for the summary)
|
|
116
|
+
session_id: str, # Owning session id
|
|
117
|
+
) -> Dict[str, Any]: # {"pruned": n, "correction_id": id|None}
|
|
118
|
+
"""First operation: prune empty (silence) segments as one grouping correction (D14).
|
|
119
|
+
|
|
120
|
+
Deterministic, no-human restructure proof: loads ONLY the empty segments
|
|
121
|
+
(server-side filter -- scale-shaped) under the chosen rendition spine, builds
|
|
122
|
+
a batch grouping Correction + DERIVED_FROM edges, commits via the queue, and
|
|
123
|
+
records REVIEWED markers (decision=corrected). Layer-0 untouched; reversible
|
|
124
|
+
by superseding.
|
|
125
|
+
"""
|
|
126
|
+
empties = await load_empty_segments(queue, graph_id, source_id,
|
|
127
|
+
rendition_selector=cfg.rendition_selector)
|
|
128
|
+
if not empties:
|
|
129
|
+
logger.info(f"[prune] {source_id}: no empty segments")
|
|
130
|
+
return {"pruned": 0, "correction_id": None}
|
|
131
|
+
if not confirm_seam("prune-review",
|
|
132
|
+
[f"{source_id}: prune {len(empties)}/{total_count} empty segment(s)"],
|
|
133
|
+
[], assume_yes=cfg.assume_yes):
|
|
134
|
+
logger.warning(f"[prune] {source_id}: declined by operator")
|
|
135
|
+
return {"pruned": 0, "correction_id": None}
|
|
136
|
+
node, edges = build_prune_correction(source_id, empties, session_id, actor=cfg.actor)
|
|
137
|
+
await commit_nodes_edges(queue, graph_id, [node], edges)
|
|
138
|
+
await record_review_markers(queue, graph_id, session_id,
|
|
139
|
+
[(s.id, "corrected") for s in empties])
|
|
140
|
+
logger.info(f"[prune] {source_id}: grouping correction {node['id']} pruned {len(empties)} segment(s)")
|
|
141
|
+
return {"pruned": len(empties), "correction_id": node["id"]}
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def collect_capability_info(
|
|
145
|
+
manager: CapabilityManager, # Manager holding the loaded capabilities
|
|
146
|
+
instance_ids: List[str], # Instance ids to record
|
|
147
|
+
) -> Dict[str, Dict[str, Any]]: # instance_id -> {name, version, db_path}
|
|
148
|
+
"""Record capability identity + data-DB pointers for the run manifest (provenance)."""
|
|
149
|
+
info: Dict[str, Dict[str, Any]] = {}
|
|
150
|
+
for iid in instance_ids:
|
|
151
|
+
meta = (getattr(manager, "capabilities", {}) or {}).get(iid)
|
|
152
|
+
if meta is None:
|
|
153
|
+
continue
|
|
154
|
+
manifest = getattr(meta, "manifest", {}) or {}
|
|
155
|
+
info[iid] = {"name": meta.name, "version": getattr(meta, "version", None),
|
|
156
|
+
"db_path": manifest.get("db_path")}
|
|
157
|
+
return info
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _journal_run_event(
|
|
161
|
+
manager: CapabilityManager, # Manager owning the journal store
|
|
162
|
+
event_type: str, # SubstrateEventType value (run_started / run_finished)
|
|
163
|
+
run_id: str, # This run's manifest id
|
|
164
|
+
actor: Optional[str], # Who/what initiated the run (cfg.actor)
|
|
165
|
+
payload: Dict[str, Any], # Run-level structured detail
|
|
166
|
+
) -> None:
|
|
167
|
+
"""Append a host-tier run event to the journal (CR-14 follow-up).
|
|
168
|
+
|
|
169
|
+
The cores are the trusted host writer class: RUN_STARTED/RUN_FINISHED
|
|
170
|
+
bracket the run so the run manifest (same run_id) links to every job row
|
|
171
|
+
the run produced. No-op when the manager has no journal store; append
|
|
172
|
+
failures stay LOUD (journal contract)."""
|
|
173
|
+
journal = getattr(manager, "journal_store", None)
|
|
174
|
+
if journal is None:
|
|
175
|
+
return
|
|
176
|
+
journal.append(JournalEvent(
|
|
177
|
+
event_type=event_type, run_id=run_id, actor=actor, payload=payload))
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
async def run_correction(
|
|
181
|
+
manager: CapabilityManager, # Manager with the graph capability loaded
|
|
182
|
+
queue: JobQueue, # Started job queue
|
|
183
|
+
cfg: CorrectionConfig, # Run configuration
|
|
184
|
+
decomp_manifest_path: str, # Decomp run manifest to correct
|
|
185
|
+
graph_db_path: str, # Resolved graph DB path (shared with decomp)
|
|
186
|
+
run_id: Optional[str] = None, # Override run id
|
|
187
|
+
session_id: Optional[str] = None, # Resume/reopen an existing session
|
|
188
|
+
reopen: bool = False, # Reopen a completed session
|
|
189
|
+
) -> CorrectionManifest: # Manifest of the correction run
|
|
190
|
+
"""Correct every source in a decomp run manifest (prune + worklist surfacing).
|
|
191
|
+
|
|
192
|
+
Per source: load spine (with variant slices) -> recompute worklist -> prune
|
|
193
|
+
empty segments [prune-review seam] -> project effective spine -> record
|
|
194
|
+
outcome. The cross-transcriber diff is INTRA-GRAPH (stage 5): variant texts
|
|
195
|
+
come from the segments' own Transcript slice refs — no second manifest. The
|
|
196
|
+
fine spine is scoped to the chosen AudioRendition (cfg.rendition_selector;
|
|
197
|
+
auto-selected when a source has one decomposed rendition). Resumable: a prior
|
|
198
|
+
session's review markers drop decided segments.
|
|
199
|
+
"""
|
|
200
|
+
run_id = run_id or new_run_id()
|
|
201
|
+
# CR-14 follow-up: queue-scoped run context — every job submitted in this
|
|
202
|
+
# run carries run_id/actor into its journal rows + worker diagnostics
|
|
203
|
+
# (run-manifest <-> journal linkage); the run itself is bracketed by
|
|
204
|
+
# RUN_STARTED/RUN_FINISHED host-tier rows. Actor = cfg.actor (the same
|
|
205
|
+
# attribution recorded on Corrections in the graph).
|
|
206
|
+
queue.set_run_context(run_id=run_id, actor=cfg.actor)
|
|
207
|
+
_journal_run_event(manager, SubstrateEventType.RUN_STARTED.value, run_id, cfg.actor, {
|
|
208
|
+
"core": "cjm-transcript-correction-core", "mode": "correction",
|
|
209
|
+
"decomp_manifest": str(decomp_manifest_path),
|
|
210
|
+
"graph_capability": cfg.graph_capability,
|
|
211
|
+
})
|
|
212
|
+
try:
|
|
213
|
+
decomp = load_decomp_manifest(decomp_manifest_path)
|
|
214
|
+
source_ids = [s.get("source_node_id") for s in (decomp.get("sources", []) or [])
|
|
215
|
+
if s.get("source_node_id")]
|
|
216
|
+
if not source_ids:
|
|
217
|
+
raise SystemExit("decomp manifest lists no sources (pre-0.2.0 manifest? re-run decomp)")
|
|
218
|
+
|
|
219
|
+
# Session: start fresh, or resume/reopen an existing one.
|
|
220
|
+
if session_id:
|
|
221
|
+
if await get_session(queue, cfg.graph_capability, session_id) is None:
|
|
222
|
+
raise SystemExit(f"session {session_id} not found in graph")
|
|
223
|
+
if reopen:
|
|
224
|
+
await set_session_status(queue, cfg.graph_capability, session_id, "reopened")
|
|
225
|
+
sess_id = session_id
|
|
226
|
+
logger.info(f"resumed session {sess_id}")
|
|
227
|
+
else:
|
|
228
|
+
sess = await start_session(queue, cfg.graph_capability, source_ids)
|
|
229
|
+
sess_id = sess.id
|
|
230
|
+
logger.info(f"started session {sess_id} over {len(source_ids)} source(s)")
|
|
231
|
+
|
|
232
|
+
manifest = CorrectionManifest(
|
|
233
|
+
run_id=run_id, created_at=time.time(), config=cfg.to_dict(),
|
|
234
|
+
decomp_manifest=str(decomp_manifest_path),
|
|
235
|
+
graph_db_path=graph_db_path, session_id=sess_id,
|
|
236
|
+
source_format=decomp.get("format", ""), source_version=decomp.get("version", ""),
|
|
237
|
+
signals_used=["empty-text", "missing-timing", "boundary-missing-terminal",
|
|
238
|
+
"boundary-terminal-then-lowercase", "transcriber-divergence"],
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
for sid in source_ids:
|
|
242
|
+
n = await count_source_segments(queue, cfg.graph_capability, sid,
|
|
243
|
+
rendition_selector=cfg.rendition_selector)
|
|
244
|
+
segments = await load_source_segments(queue, cfg.graph_capability, sid,
|
|
245
|
+
rendition_selector=cfg.rendition_selector)
|
|
246
|
+
variants = await load_variant_texts(queue, cfg.graph_capability, segments)
|
|
247
|
+
markers = await load_review_markers(queue, cfg.graph_capability, sess_id)
|
|
248
|
+
worklist = compute_worklist(segments, markers, variants=variants)
|
|
249
|
+
divergences = sum(1 for it in worklist if "transcriber-divergence" in it.flags)
|
|
250
|
+
logger.info(f"[src {sid[:8]}] {n} segment(s); worklist {len(worklist)} flagged; "
|
|
251
|
+
f"{len(detect_empty_segments(segments))} empty; "
|
|
252
|
+
f"{divergences} transcriber-divergence (intra-graph)")
|
|
253
|
+
|
|
254
|
+
prune = {"pruned": 0, "correction_id": None}
|
|
255
|
+
if cfg.prune_empty:
|
|
256
|
+
prune = await prune_empty_segments(queue, cfg, cfg.graph_capability, sid, n, sess_id)
|
|
257
|
+
|
|
258
|
+
corrections = await find_corrections_for_session(queue, cfg.graph_capability, sess_id)
|
|
259
|
+
effective = project_effective_spine(segments, corrections)
|
|
260
|
+
manifest.sources.append({
|
|
261
|
+
"source_node_id": sid,
|
|
262
|
+
"segment_count": n,
|
|
263
|
+
"worklist_flagged": len(worklist),
|
|
264
|
+
"empty_segments": len(detect_empty_segments(segments)),
|
|
265
|
+
"transcriber_divergences": divergences,
|
|
266
|
+
"pruned": prune["pruned"],
|
|
267
|
+
"prune_correction_id": prune["correction_id"],
|
|
268
|
+
"effective_segment_count": len(effective),
|
|
269
|
+
})
|
|
270
|
+
|
|
271
|
+
await set_session_status(queue, cfg.graph_capability, sess_id, "completed")
|
|
272
|
+
except BaseException as e:
|
|
273
|
+
# The journal exists for exactly this row: a run that DIED records
|
|
274
|
+
# that it was attempted (failures stop being the unattributed case).
|
|
275
|
+
_journal_run_event(manager, SubstrateEventType.RUN_FINISHED.value, run_id, cfg.actor, {
|
|
276
|
+
"core": "cjm-transcript-correction-core", "mode": "correction",
|
|
277
|
+
"status": "failed", "error": repr(e),
|
|
278
|
+
})
|
|
279
|
+
raise
|
|
280
|
+
_journal_run_event(manager, SubstrateEventType.RUN_FINISHED.value, run_id, cfg.actor, {
|
|
281
|
+
"core": "cjm-transcript-correction-core", "mode": "correction",
|
|
282
|
+
"status": "completed", "session_id": sess_id,
|
|
283
|
+
"sources": len(manifest.sources),
|
|
284
|
+
})
|
|
285
|
+
return manifest
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _format_worklist_item(
|
|
289
|
+
item: WorklistItem, # The item to present
|
|
290
|
+
effective_text: str, # Current effective text (layer-0 or latest correction)
|
|
291
|
+
variant_text: Optional[str] = None, # Divergent variant text (another transcriber's reading)
|
|
292
|
+
prior_correction: Optional[str] = None, # Suggested text from the cross-transcript cache
|
|
293
|
+
) -> str: # Multi-line presentation block
|
|
294
|
+
"""Render a worklist item for the CLI review seam (text + timing + flags + hints)."""
|
|
295
|
+
s = item.segment
|
|
296
|
+
t = f"[{s.start_time:.1f}-{s.end_time:.1f}s]" if s.start_time is not None else "[--]"
|
|
297
|
+
lines = [f" #{s.index} {t} flags={','.join(item.flags)}",
|
|
298
|
+
f" text: {effective_text!r}"]
|
|
299
|
+
if variant_text is not None:
|
|
300
|
+
lines.append(f" variant: {variant_text!r}")
|
|
301
|
+
if prior_correction is not None:
|
|
302
|
+
lines.append(f" cache-hit: {prior_correction!r}")
|
|
303
|
+
return "\n".join(lines)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
async def review_worklist(
|
|
307
|
+
queue: JobQueue, # Started job queue
|
|
308
|
+
cfg: CorrectionConfig, # Run configuration
|
|
309
|
+
source_id: str, # Source under review
|
|
310
|
+
worklist: List[WorklistItem], # Flagged, undecided items
|
|
311
|
+
session_id: str, # Owning session id
|
|
312
|
+
variant_by_segment: Optional[Dict[str, str]] = None, # segment_id -> divergent variant text
|
|
313
|
+
max_items: int = 0, # Cap (0 = all)
|
|
314
|
+
) -> Dict[str, int]: # {"corrected": n, "skipped": n, "reviewed": n}
|
|
315
|
+
"""Interactive review loop -> text_content corrections (cheapest HITL seam).
|
|
316
|
+
|
|
317
|
+
Per item: present text + timing + flags (+ the divergent variant text +
|
|
318
|
+
cross-transcript cache hit), then read a decision from stdin:
|
|
319
|
+
[a]ccept (mark reviewed) / [e]dit (commit a text_content correction;
|
|
320
|
+
auto-supersedes any prior correction on the segment) / [s]kip / [q]uit.
|
|
321
|
+
On 'e', the next stdin line is the new text (blank -> adopt the variant) —
|
|
322
|
+
adopting a variant IS the "extract the superior lightweight reading" move,
|
|
323
|
+
and its provenance is the variant slice already on the segment.
|
|
324
|
+
Drivable headless via a stdin pipe (E9-companion); cfg.assume_yes marks every
|
|
325
|
+
item reviewed with no edits.
|
|
326
|
+
"""
|
|
327
|
+
variant_by_segment = variant_by_segment or {}
|
|
328
|
+
counts = {"corrected": 0, "skipped": 0, "reviewed": 0}
|
|
329
|
+
items = worklist[:max_items] if max_items > 0 else worklist
|
|
330
|
+
# C17 (stage 4): ONE batched far-end read replaces the per-item lookup —
|
|
331
|
+
# 2 graph round-trips for the whole worklist instead of 1 per item. (The
|
|
332
|
+
# per-item hash-cache lookup below stays lazy/per-item: batching it needs
|
|
333
|
+
# a hash-LIST far-end constraint — recorded promotion candidate.)
|
|
334
|
+
active_by_segment = await find_active_text_corrections_batch(
|
|
335
|
+
queue, cfg.graph_capability, [it.segment.id for it in items])
|
|
336
|
+
for item in items:
|
|
337
|
+
seg = item.segment
|
|
338
|
+
active = active_by_segment.get(seg.id)
|
|
339
|
+
effective_text = (active.get("payload", {}) or {}).get("new_text", seg.text) if active else seg.text
|
|
340
|
+
var = variant_by_segment.get(seg.id)
|
|
341
|
+
prior = None
|
|
342
|
+
if seg.content_hash:
|
|
343
|
+
hits = await find_prior_corrections_by_hash(queue, cfg.graph_capability, seg.content_hash)
|
|
344
|
+
hits = [h for h in hits if (h.get("payload") or {}).get("segment_id") != seg.id]
|
|
345
|
+
if hits:
|
|
346
|
+
prior = (hits[0].get("payload") or {}).get("new_text")
|
|
347
|
+
logger.info("review item:\n" + _format_worklist_item(item, effective_text, var, prior))
|
|
348
|
+
if cfg.assume_yes:
|
|
349
|
+
await record_review_markers(queue, cfg.graph_capability, session_id, [(seg.id, "reviewed")])
|
|
350
|
+
counts["reviewed"] += 1
|
|
351
|
+
continue
|
|
352
|
+
try:
|
|
353
|
+
decision = input(f" #{seg.index} [a]ccept/[e]dit/[s]kip/[q]uit: ").strip().lower()
|
|
354
|
+
except EOFError:
|
|
355
|
+
break
|
|
356
|
+
if decision in ("q", "quit"):
|
|
357
|
+
break
|
|
358
|
+
if decision in ("e", "edit"):
|
|
359
|
+
try:
|
|
360
|
+
new_text = input(" new text (blank = adopt variant): ").rstrip("\n")
|
|
361
|
+
except EOFError:
|
|
362
|
+
new_text = ""
|
|
363
|
+
if not new_text and var:
|
|
364
|
+
new_text = var
|
|
365
|
+
if not new_text:
|
|
366
|
+
await record_review_markers(queue, cfg.graph_capability, session_id, [(seg.id, "skipped")])
|
|
367
|
+
counts["skipped"] += 1
|
|
368
|
+
continue
|
|
369
|
+
cid = await commit_text_correction(
|
|
370
|
+
queue, cfg.graph_capability, source_id, seg.id, new_text, session_id,
|
|
371
|
+
old_text=effective_text, supersedes_id=(active.get("id") if active else None),
|
|
372
|
+
actor=cfg.actor)
|
|
373
|
+
logger.info(f" #{seg.index}: text correction {cid}"
|
|
374
|
+
+ (f" supersedes {active['id']}" if active else ""))
|
|
375
|
+
counts["corrected"] += 1
|
|
376
|
+
elif decision in ("s", "skip"):
|
|
377
|
+
await record_review_markers(queue, cfg.graph_capability, session_id, [(seg.id, "skipped")])
|
|
378
|
+
counts["skipped"] += 1
|
|
379
|
+
else:
|
|
380
|
+
await record_review_markers(queue, cfg.graph_capability, session_id, [(seg.id, "reviewed")])
|
|
381
|
+
counts["reviewed"] += 1
|
|
382
|
+
return counts
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
async def run_review(
|
|
386
|
+
manager: CapabilityManager, # Manager with the graph capability loaded
|
|
387
|
+
queue: JobQueue, # Started job queue
|
|
388
|
+
cfg: CorrectionConfig, # Run configuration
|
|
389
|
+
decomp_manifest_path: str, # Decomp run manifest to review
|
|
390
|
+
graph_db_path: str, # Resolved graph DB path (shared with decomp)
|
|
391
|
+
run_id: Optional[str] = None, # Override run id
|
|
392
|
+
session_id: Optional[str] = None, # Resume/reopen an existing session
|
|
393
|
+
reopen: bool = False, # Reopen a completed session
|
|
394
|
+
max_items: int = 0, # Max worklist items to review per source (0 = all)
|
|
395
|
+
) -> CorrectionManifest: # Manifest of the review run
|
|
396
|
+
"""Interactive review pass over a decomp manifest's flagged worklist (text corrections).
|
|
397
|
+
|
|
398
|
+
Like run_correction but enters the text-correction review loop instead of the
|
|
399
|
+
prune. Empty segments are excluded from the review worklist (they belong to the
|
|
400
|
+
prune); the effective spine is projected from the SOURCE's corrections (across
|
|
401
|
+
sessions), so a resumed/reopened session sees prior prune + text corrections.
|
|
402
|
+
The variant hints come from the segments' own slice refs (intra-graph). The
|
|
403
|
+
fine spine is scoped to the chosen AudioRendition (cfg.rendition_selector).
|
|
404
|
+
"""
|
|
405
|
+
run_id = run_id or new_run_id()
|
|
406
|
+
# CR-14 follow-up: queue-scoped run context — every job submitted in this
|
|
407
|
+
# run carries run_id/actor into its journal rows + worker diagnostics
|
|
408
|
+
# (run-manifest <-> journal linkage); the run itself is bracketed by
|
|
409
|
+
# RUN_STARTED/RUN_FINISHED host-tier rows. Actor = cfg.actor (the same
|
|
410
|
+
# attribution recorded on Corrections in the graph).
|
|
411
|
+
queue.set_run_context(run_id=run_id, actor=cfg.actor)
|
|
412
|
+
_journal_run_event(manager, SubstrateEventType.RUN_STARTED.value, run_id, cfg.actor, {
|
|
413
|
+
"core": "cjm-transcript-correction-core", "mode": "review",
|
|
414
|
+
"decomp_manifest": str(decomp_manifest_path),
|
|
415
|
+
"graph_capability": cfg.graph_capability,
|
|
416
|
+
})
|
|
417
|
+
try:
|
|
418
|
+
decomp = load_decomp_manifest(decomp_manifest_path)
|
|
419
|
+
source_ids = [s.get("source_node_id") for s in (decomp.get("sources", []) or [])
|
|
420
|
+
if s.get("source_node_id")]
|
|
421
|
+
if not source_ids:
|
|
422
|
+
raise SystemExit("decomp manifest lists no sources (pre-0.2.0 manifest? re-run decomp)")
|
|
423
|
+
|
|
424
|
+
if session_id:
|
|
425
|
+
if await get_session(queue, cfg.graph_capability, session_id) is None:
|
|
426
|
+
raise SystemExit(f"session {session_id} not found in graph")
|
|
427
|
+
if reopen:
|
|
428
|
+
await set_session_status(queue, cfg.graph_capability, session_id, "reopened")
|
|
429
|
+
sess_id = session_id
|
|
430
|
+
logger.info(f"resumed session {sess_id}")
|
|
431
|
+
else:
|
|
432
|
+
sess = await start_session(queue, cfg.graph_capability, source_ids)
|
|
433
|
+
sess_id = sess.id
|
|
434
|
+
logger.info(f"started review session {sess_id} over {len(source_ids)} source(s)")
|
|
435
|
+
|
|
436
|
+
manifest = CorrectionManifest(
|
|
437
|
+
run_id=run_id, created_at=time.time(), config=cfg.to_dict(),
|
|
438
|
+
decomp_manifest=str(decomp_manifest_path),
|
|
439
|
+
graph_db_path=graph_db_path, session_id=sess_id,
|
|
440
|
+
source_format=decomp.get("format", ""), source_version=decomp.get("version", ""),
|
|
441
|
+
signals_used=["boundary-missing-terminal", "boundary-terminal-then-lowercase",
|
|
442
|
+
"transcriber-divergence"],
|
|
443
|
+
)
|
|
444
|
+
|
|
445
|
+
for sid in source_ids:
|
|
446
|
+
n = await count_source_segments(queue, cfg.graph_capability, sid,
|
|
447
|
+
rendition_selector=cfg.rendition_selector)
|
|
448
|
+
segments = await load_source_segments(queue, cfg.graph_capability, sid,
|
|
449
|
+
rendition_selector=cfg.rendition_selector)
|
|
450
|
+
variants = await load_variant_texts(queue, cfg.graph_capability, segments)
|
|
451
|
+
variant_by_segment: Dict[str, str] = {}
|
|
452
|
+
for i, (auth, var) in variant_divergence(segments, variants).items():
|
|
453
|
+
variant_by_segment[segments[i].id] = var
|
|
454
|
+
markers = await load_review_markers(queue, cfg.graph_capability, sess_id)
|
|
455
|
+
worklist = [it for it in compute_worklist(segments, markers, variants=variants)
|
|
456
|
+
if not it.segment.is_empty]
|
|
457
|
+
logger.info(f"[src {sid[:8]}] {n} segment(s); review worklist {len(worklist)} "
|
|
458
|
+
f"(reviewing up to {max_items or len(worklist)})")
|
|
459
|
+
counts = await review_worklist(queue, cfg, sid, worklist, sess_id,
|
|
460
|
+
variant_by_segment=variant_by_segment, max_items=max_items)
|
|
461
|
+
|
|
462
|
+
corrections, superseded = await load_source_corrections(queue, cfg.graph_capability, sid)
|
|
463
|
+
active = active_corrections(corrections, superseded)
|
|
464
|
+
effective = project_effective_spine(segments, active)
|
|
465
|
+
manifest.sources.append({
|
|
466
|
+
"source_node_id": sid, "segment_count": n, "review_worklist": len(worklist),
|
|
467
|
+
"corrected": counts["corrected"], "skipped": counts["skipped"],
|
|
468
|
+
"reviewed": counts["reviewed"], "active_corrections": len(active),
|
|
469
|
+
"superseded_corrections": len(superseded), "effective_segment_count": len(effective),
|
|
470
|
+
})
|
|
471
|
+
logger.info(f"[src {sid[:8]}] corrected={counts['corrected']} skipped={counts['skipped']} "
|
|
472
|
+
f"reviewed={counts['reviewed']}; active corrections={len(active)}; "
|
|
473
|
+
f"effective spine={len(effective)}")
|
|
474
|
+
|
|
475
|
+
await set_session_status(queue, cfg.graph_capability, sess_id, "completed")
|
|
476
|
+
except BaseException as e:
|
|
477
|
+
# The journal exists for exactly this row: a run that DIED records
|
|
478
|
+
# that it was attempted (failures stop being the unattributed case).
|
|
479
|
+
_journal_run_event(manager, SubstrateEventType.RUN_FINISHED.value, run_id, cfg.actor, {
|
|
480
|
+
"core": "cjm-transcript-correction-core", "mode": "review",
|
|
481
|
+
"status": "failed", "error": repr(e),
|
|
482
|
+
})
|
|
483
|
+
raise
|
|
484
|
+
_journal_run_event(manager, SubstrateEventType.RUN_FINISHED.value, run_id, cfg.actor, {
|
|
485
|
+
"core": "cjm-transcript-correction-core", "mode": "review",
|
|
486
|
+
"status": "completed", "session_id": sess_id,
|
|
487
|
+
"sources": len(manifest.sources),
|
|
488
|
+
})
|
|
489
|
+
return manifest
|