cjm-transcript-correction-core 0.0.10__tar.gz → 0.0.30__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {cjm_transcript_correction_core-0.0.10/cjm_transcript_correction_core.egg-info → cjm_transcript_correction_core-0.0.30}/PKG-INFO +72 -3
  2. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/README.md +67 -1
  3. cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core/__init__.py +1 -0
  4. cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core/cleanread.py +220 -0
  5. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/cli.py +976 -33
  6. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/graph.py +243 -26
  7. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/journal.py +2 -1
  8. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/models.py +51 -7
  9. cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core/respine_transfer.py +243 -0
  10. cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core/signals.py +684 -0
  11. cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core/spans.py +868 -0
  12. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/spine.py +74 -0
  13. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/state.py +3 -2
  14. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/strata.py +454 -51
  15. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core.egg-info}/PKG-INFO +72 -3
  16. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core.egg-info/SOURCES.txt +7 -0
  17. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core.egg-info/entry_points.txt +3 -0
  18. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core.egg-info/requires.txt +5 -1
  19. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/pyproject.toml +7 -1
  20. cjm_transcript_correction_core-0.0.30/tests/test_cleanread.py +134 -0
  21. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_cli.py +6 -0
  22. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_graph.py +153 -0
  23. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_journal.py +1 -0
  24. cjm_transcript_correction_core-0.0.30/tests/test_respine_transfer.py +175 -0
  25. cjm_transcript_correction_core-0.0.30/tests/test_signals.py +190 -0
  26. cjm_transcript_correction_core-0.0.30/tests/test_spans.py +408 -0
  27. cjm_transcript_correction_core-0.0.30/tests/test_speaker_picker.py +106 -0
  28. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_strata.py +247 -5
  29. cjm_transcript_correction_core-0.0.10/cjm_transcript_correction_core/__init__.py +0 -1
  30. cjm_transcript_correction_core-0.0.10/cjm_transcript_correction_core/signals.py +0 -341
  31. cjm_transcript_correction_core-0.0.10/tests/test_signals.py +0 -90
  32. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/LICENSE +0 -0
  33. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/MANIFEST.in +0 -0
  34. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/launch.py +0 -0
  35. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/pipeline.py +0 -0
  36. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core.egg-info/dependency_links.txt +0 -0
  37. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core.egg-info/top_level.txt +0 -0
  38. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/setup.cfg +0 -0
  39. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_bench.py +0 -0
  40. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_gestures.py +0 -0
  41. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_models.py +0 -0
  42. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_pipeline.py +0 -0
  43. {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_state.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cjm-transcript-correction-core
3
- Version: 0.0.10
3
+ Version: 0.0.30
4
4
  Summary: A frontend-agnostic core for the transcript correction workflow — the first downstream graph-extension core; composes the graph-storage capability worker into a headless pipeline that applies unified, non-destructive corrections (text, punctuation, segmentation) as a supersede-able overlay on a committed decomposition spine, recomputes the review worklist from deterministic signals, and exposes a CLI as its first driver.
5
5
  Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
6
6
  License: Apache-2.0
@@ -14,8 +14,11 @@ Requires-Dist: cjm_substrate>=0.0.61
14
14
  Requires-Dist: cjm_context_graph_primitives>=0.0.15
15
15
  Requires-Dist: cjm-capability-primitives>=0.0.12
16
16
  Requires-Dist: cjm-context-graph-layer>=0.0.11
17
- Requires-Dist: cjm-transcript-graph-schema>=0.0.7
17
+ Requires-Dist: cjm-transcript-graph-schema>=0.0.11
18
18
  Requires-Dist: numpy
19
+ Provides-Extra: test
20
+ Requires-Dist: soundfile; extra == "test"
21
+ Requires-Dist: pytest; extra == "test"
19
22
  Dynamic: license-file
20
23
 
21
24
  # cjm-transcript-correction-core
@@ -27,21 +30,34 @@ A frontend-agnostic core for the transcript correction workflow — the first do
27
30
  ## Modules
28
31
 
29
32
  - **`cjm_transcript_correction_core.__init__`**
33
+ - **`cjm_transcript_correction_core.cleanread`** — The clean read — L1 of the reading ladder (design 6752db0a; the subtraction
30
34
  - **`cjm_transcript_correction_core.cli`** — The CLI driver — the correction core's first (and currently only) frontend. run <decomp-manifest> corrects the committed spine in the decomp graph DB, pointing the graph worker at that shared DB via load-time config, with optional session resume/reopen; review runs the interactive text-correction loop (the cross-transcriber diff is intra-graph since stage 5).
31
35
  - **`cjm_transcript_correction_core.graph`** — The correction overlay's graph I/O: targeted (scale-shaped) reads of a committed spine via the graph-storage query action, construction of Correction / CorrectionSession nodes + CORRECTS / SUPERSEDES / DERIVED_FROM / REVIEWED edges, the in-core effective-spine projection (layer-0 + applied corrections), and commit through the job queue. Hand-rolled (revolution-1) = direct CR-18 spec material; append-only on layer-0 (never update/delete a Segment).
32
36
  - **`cjm_transcript_correction_core.journal`** — Live append-through for the correction verbs — the workflow journal's domain half.
33
37
  - **`cjm_transcript_correction_core.launch`** — The shared launch surface every correction shell drives through: the
34
38
  - **`cjm_transcript_correction_core.models`** — Overlay data shapes for the transcript-correction workflow: the Correction / CorrectionSession graph nodes + their relation registry, the read view of a committed spine segment, the worklist item, run configuration, and the correction run manifest (proto-bundle that chains decomp -> correction).
35
39
  - **`cjm_transcript_correction_core.pipeline`** — The headless correction workflow: load a decomp run manifest, resolve the shared graph DB, start/resume/reopen a CorrectionSession, recompute the worklist from deterministic signals + persisted review state, run the D14 empty-segment prune (first operation), and record a chainable correction run manifest — with a cheapest-form HITL approval seam.
40
+ - **`cjm_transcript_correction_core.respine_transfer`** — The chunk-scoped transfer a chunk respine calls (work item 7a5e9c84; ruling 0b4d5cfa (5),
36
41
  - **`cjm_transcript_correction_core.signals`** — Pure deterministic Tier-1 signal functions (no capability calls): empty-segment detection, bidirectional boundary punctuation/capitalization heuristics, forced-alignment coverage flags, positional cross-transcriber diff, phonetic + edit-distance variant clustering, and the event-proposal overlay (leg 4: the finetuned detector's spans anchored onto the spine). The worklist is recomputed from these each session; revolution-1 builds ZERO new capabilities.
42
+ - **`cjm_transcript_correction_core.spans`** — Spans — the overlay-proposal lane's domain half (design bbf8bafd, extending the
37
43
  - **`cjm_transcript_correction_core.spine`**
38
44
  - **`cjm_transcript_correction_core.state`** — Sidecar view-state + spine-picker helpers — the correction TUI's pure
39
45
  - **`cjm_transcript_correction_core.strata`** — Strata — the filtering lane's domain half (DECs 304fd984 + 9d4c0a38; work items
40
46
 
41
47
  ## API
42
48
 
49
+ ### `cjm_transcript_correction_core.cleanread`
50
+
51
+ - `clean_read` _function_ — L1 — the clean read (pure). Kept lines in spine order; each carries its
52
+ - `clean_read_pack_read` _function_ — The `read` record an L1 pack carries: the layer, the marker, the
53
+ - `clean_read_segments` _function_ — Project the clean read back onto SpineSegments (clean text, same
54
+ - `clean_read_summary` _function_
55
+ - `repeat_survivor` _function_ — Locate the last repeating unit of a word-repeat span (pure): the
56
+ - `subtract_spans` _function_ — Cut the ranges out of the text, leaving `marker` at each seam (pure).
57
+
43
58
  ### `cjm_transcript_correction_core.cli`
44
59
 
60
+ - `attention_command` _function_ — Execute `attention`: the walk-lane ATTENTION TIER (item 3758f6cb, ruling f400d2c3)
45
61
  - `bench_command` _function_ — Execute the `bench` subcommand: the reserved-tail verdict join.
46
62
  - `build_parser` _function_ — Build the CLI parser (subcommands: run, review).
47
63
  - `commit_wordless_transfer` _function_ — COMMIT a planned transfer — the engine's second half. One
@@ -49,6 +65,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
49
65
  - `extract_command` _function_ — Execute the `extract` subcommand: fold the gated overlay into a manifested dataset.
50
66
  - `filter_confirm_command` _function_ — Execute `filter-confirm`: the HEADLESS HITL worklist (bc8dbbdd pass-1
51
67
  - `filter_ingest_command` _function_ — Execute `filter-ingest`: validate proposer rows against their pack and
68
+ - `filter_merge_command` _function_ — Execute `filter-merge`: fold several proposal sets over one spine into
52
69
  - `filter_pack_command` _function_ — Execute `filter-pack`: write one spine window's effective text-bearing
53
70
  - `gate_command` _function_ — Execute the `gate` subcommand: show or assert per-spine extraction gates.
54
71
  - `load_capabilities` _function_ — Discover manifests + load each capability, passing per-capability config (CR-2 caller-wins).
@@ -62,6 +79,11 @@ A frontend-agnostic core for the transcript correction workflow — the first do
62
79
  - `run_command` _function_ — Execute the `run` subcommand: correct a decomp manifest's committed spine.
63
80
  - `run_extract` _function_ — The extract fold on an ALREADY-OPEN graph seat (flywheel build leg 2,
64
81
  - `scan_command` _function_ — Execute `scan-mishomed`: flag authoritative FA words stranded outside
82
+ - `span_confirm_command` _function_ — Execute `span-confirm`: the HEADLESS HITL worklist over one span
83
+ - `span_ingest_command` _function_ — Execute `span-ingest`: resolve a proposer's quoted-word rows against
84
+ - `span_lexicon_command` _function_ — Execute `span-lexicon`: the pattern tier (design bbf8bafd (g)) — bare
85
+ - `span_merge_command` _function_ — Execute `span-merge`: fold several span sets over one spine into ONE
86
+ - `span_pack_command` _function_ — Execute `span-pack`: the filter pack's lines as a SPEECH-OVERLAY pack
65
87
  - `stats_command` _function_ — Execute the `stats` subcommand: flywheel accounting over the shared graph.
66
88
  - `transfer_command` _function_ — Execute `transfer-wordless`: replay wordless event inserts (and speaker-
67
89
  - `wordless_donors` _function_ — The EFFECTIVE wordless layer of a spine: labeled, effectively wordless
@@ -73,6 +95,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
73
95
  - `active_speaker_assignments` _function_ — Project the ACTIVE speaker assignment per segment (latest-wins).
74
96
  - `active_speech_overlays` _function_ — The surviving speech-overlay corrections (supersession applied; pure).
75
97
  - `aggregate_session_purposes` _function_ — Fold sessions into a per-source purpose mix (pure; d915d545 picker rung).
98
+ - `annotate_retired` _function_ — Mark spine rows with their retirement state (pure).
76
99
  - `apply_chunk_inserts` _function_ — Synthesize inserted chunks into the effective spine (DEC 3d3fa2a8).
77
100
  - `apply_time_nudges` _function_ — Apply timing corrections onto segment times (latest-wins per edge).
78
101
  - `bench_event_proposals` _function_ — The reserved-tail bench join (leg 4, DECs 8e05b87b + 8cf12c22) — pure.
@@ -111,6 +134,8 @@ A frontend-agnostic core for the transcript correction workflow — the first do
111
134
  - `correction_stats` _function_ — Fold one Source's ACTIVE overlay into flywheel-accounting counts (pure).
112
135
  - `corrections_to_edits` _function_ — Map this core's Correction payloads onto the layer's spine-edit vocabulary.
113
136
  - `count_source_segments` _function_ — Count a Source's segments server-side under its chosen rendition + skeleton (typed count mode).
137
+ - `default_spine` _function_ — Which spine opens by default (pure; ruling a7617bd4 rule (c)): preference is a
138
+ - `discharge_mark` _function_ — Discharge an open mark BY THE CORRECTION THAT ADDRESSED IT — the closure
114
139
  - `extract_spine_dataset` _function_ — Fold ONE spine's overlay into its v1 insert-span dataset slice (pure).
115
140
  - `fa_words_for_transcript` _function_ — One transcript's FA words in source coordinates (the scan-mishomed join,
116
141
  - `find_active_text_correction` _function_ — Single-segment convenience over the batch read (cross-session; latest wins).
@@ -143,6 +168,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
143
168
  - `set_session_status` _function_ — Update a session's status + updated_at.
144
169
  - `skeleton_hash_for` _function_ — Resolve a skeleton selector to the chosen spine's HASH (pure) — the gate's
145
170
  - `source_audio_segment_ids` _function_ — The Source's coarse spine (one small typed read; ordered by index).
171
+ - `speaker_assignment_sources` _function_ — Where each speaker Entity has been assigned — the picker's COLLECTION
146
172
  - `speech_overlay_spans` _function_ — Fold the ACTIVE speech overlays into extraction-facing span records (pure).
147
173
  - `spine_where_for` _function_ — Resolve a skeleton selector against the observed spine set (pure).
148
174
  - `start_session` _function_ — Create + commit a new CorrectionSession node.
@@ -186,8 +212,22 @@ A frontend-agnostic core for the transcript correction workflow — the first do
186
212
  - `run_correction` _function_ — Correct every source in a decomp run manifest (prune + worklist surfacing).
187
213
  - `run_review` _function_ — Interactive review pass over a decomp manifest's flagged worklist (text corrections).
188
214
 
215
+ ### `cjm_transcript_correction_core.respine_transfer`
216
+
217
+ - `chunk_respine_transfer` _function_ — The registered chunk-scoped transfer (the entry point decomp-core discovers).
218
+ - `group_speaker_rows` _function_ — Fold per-segment rows into turn-shaped commits (pure): contiguous same-entity
219
+ - `live_neighbour` _function_ — The live segment right before the chunk — the flank a donor that starts before
220
+ - `load_segments_by_ids` _function_ — Segments by id — the ONE read that bypasses the live predicate on purpose: the
221
+ - `plan_speaker_carry` _function_ — Which new segment takes which speaker (pure; 4a7ec4f8 (2)/(3)): a segment whose
222
+ - `speaker_spans` _function_ — The per-span speaker map the carry reads (pure).
223
+
189
224
  ### `cjm_transcript_correction_core.signals`
190
225
 
226
+ - `attention_boundary_marks` _function_ — The boundary signals of the attention tier (item 3758f6cb signals 1+3 and the folded
227
+ - `attention_divergence_marks` _function_ — The two-transcriber disagreement signal of the attention tier (item 3758f6cb signal
228
+ - `attention_fa_marks` _function_ — The forced-alignment signals of the attention tier (item 3758f6cb signal 1), pure.
229
+ - `attention_marks` _function_ — Compose the attention tier (item 3758f6cb): every enabled signal's mark rows,
230
+ - `attention_speaker_marks` _function_ — The next-speaker-change landmark of the folded item 65791933, pure: `speaker-change`
191
231
  - `boundary_punct_caps_flags` _function_ — Bidirectional boundary punctuation/capitalization heuristics (in-segment only).
192
232
  - `cluster_variants` _function_ — Cluster word variants by phonetic key + edit distance (fix-one-fix-all).
193
233
  - `compute_signal_flags` _function_ — Combine all deterministic Tier-1 signals into per-segment flags.
@@ -200,14 +240,36 @@ A frontend-agnostic core for the transcript correction workflow — the first do
200
240
  - `speaker_turn_proposals` _function_ — Dominant diarization cluster per segment — the assign lane's proposal paint.
201
241
  - `variant_divergence` _function_ — Within-segment cross-transcriber divergence (stage 5: intra-graph).
202
242
 
243
+ ### `cjm_transcript_correction_core.spans`
244
+
245
+ - `bench_span_proposals` _function_ — Derive the span verdicts (design bbf8bafd (f), the bench_filter_proposals
246
+ - `build_span_pack` _function_ — Build the span proposer's input: the filter pack's numbered lines with
247
+ - `find_token_span` _function_ — Resolve quoted words to WHOLE word tokens of a line (pure): the quote is
248
+ - `is_span_pack` _function_
249
+ - `lexicon_span_rows` _function_ — The LEXICON TIER (design bbf8bafd (g)): a bare um / uh token is a
250
+ - `load_span_proposal_sets` _function_ — Every SPAN proposal set for a source (and optionally one spine), newest
251
+ - `locate_span_tokens` _function_ — Where a proposal's words sit on the CURRENT line, as a character range
252
+ - `merge_span_proposals` _function_ — Fold several SPAN sets over one spine into one walkable set — the
253
+ - `overlay_label_slate` _function_ — The slate a span pack carries: every label with its gloss and whether a
254
+ - `pending_span_proposals` _function_ — The headless worklist: proposals with NO active overlay carrying their id
255
+ - `render_span_pack` _function_ — Render a span pack as the brief a proposer reads: identity + window, the
256
+ - `render_span_propset_markdown` _function_ — Project a span set for a human: one line per proposal — tier, label,
257
+ - `segment_word_tokens` _function_
258
+ - `snap_span_proposal` _function_ — Resolve a proposal against the CURRENT line at accept (design bbf8bafd
259
+ - `span_proposals_from_rows` _function_ — Resolve validated rows to span proposal-set rows: proposal id, label
260
+ - `validate_span_rows` _function_ — Validate + resolve proposer rows against their span pack — loud on the
261
+ - `write_span_propset` _function_ — Write one SPAN proposal set — the filter set's layout under the span
262
+
203
263
  ### `cjm_transcript_correction_core.spine`
204
264
 
205
265
  - `ChunkRef` _class_ — Where one Segment's VAD-chunk audio lives: the model-input WAV + the chunk-local span.
206
266
  - `SeamRef` _class_ — A source-coordinate audio span across one fine-spine boundary (the g/G
207
267
  - `SpineView` _class_ — One Source's effective correction spine, cursor-windowed for the TUI.
268
+ - `layer_speaker_menu` _function_ — Layer the assign-lane digit menu (pure; DEC 774dbe40). Tier 1 = the
208
269
  - `list_sources` _function_ — Enumerate the graph's Source nodes (the discovery corpus, 2ce81638).
209
270
  - `load_source_slice` _function_ — Decode a source-coordinate slice of the ORIGINAL media to playable samples.
210
271
  - `match_sources` _function_ — The --source selector (pure; shared by direct open and the picker's seed).
272
+ - `match_speaker_query` _function_ — Narrow the registry by a typed query (pure; DEC 774dbe40 (4)). Rank 0 =
211
273
  - `neighbor_word_bound` _function_ — The adjacent word's FA boundary facing an overlay span (pure).
212
274
  - `open_stack` _function_ — Bootstrap the graph capability stack, resolving the db path (2ce81638).
213
275
  - `parse_entity_input` _function_ — Parse the new-speaker editor line (pure). A leading `?` marks the entity
@@ -238,11 +300,18 @@ A frontend-agnostic core for the transcript correction workflow — the first do
238
300
  - `build_filter_pack` _function_ — Build the proposer's input: one source window's text-bearing effective
239
301
  - `exclude_strata` _function_ — The per-consumer filtered projection (DEC 9d4c0a38): "filtered" is a
240
302
  - `load_filter_proposal_sets` _function_ — Every filtering proposal set for a source (and optionally one spine),
303
+ - `materialized_fix_ids` _function_ — The fidelity-edit apply path's materialization: a proposer row whose
304
+ - `materialized_mark_ids` _function_ — Class-family routing's other half: a proposer's mark-family row (an ASR
305
+ - `merge_filter_proposals` _function_ — Fold several proposal sets over ONE spine into one walkable set (design
241
306
  - `new_pack_id` _function_ — Generate a unique, sortable pack id.
242
307
  - `pack_digest` _function_ — Digest the READ content (source binding + window + numbered segments) —
243
308
  - `pending_filter_proposals` _function_ — The headless worklist: proposals with NO live stratum carrying their id and
309
+ - `plan_pack_windows` _function_ — Cut a spine into `count` windows of near-equal TEXT-segment count at
244
310
  - `proposals_from_rows` _function_ — Resolve validated rows to proposal-set rows: proposal id, category, source
245
311
  - `render_filter_pack` _function_ — Render a pack as the brief a proposer reads: identity + window, the class
312
+ - `render_filter_propset_markdown` _function_ — Project a proposal set for a HUMAN to check against the source in the
313
+ - `resolve_slate` _function_ — Resolve a NAMED class slate (ruling c5b6cf42) to the two vocabularies a
314
+ - `select_span_segments` _function_ — The span-EDIT gesture's resolver (agreed 2026-09-01 on the ch04 confirm
246
315
  - `strata_index` _function_ — Segment-keyed view of the live strata (the consumer query's index).
247
316
  - `validate_proposal_rows` _function_ — Validate + normalize proposer rows against their pack — loud on the first
248
317
  - `write_filter_propset` _function_ — Write one filtering proposal set: `<out_root>/<set_id>/manifest.json` +
@@ -250,4 +319,4 @@ A frontend-agnostic core for the transcript correction workflow — the first do
250
319
  ## Dependencies
251
320
 
252
321
  **Depends on:** `cjm-capability-primitives`, `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-substrate`, `cjm-transcript-graph-schema`, `numpy`
253
- **Used by:** `cjm-transcript-correction-qt`, `cjm-transcript-correction-tui`, `cjm-transcription-core`, `cjm-workflow-hub-qt`, `cjm-workflow-hub-tui`
322
+ **Used by:** `cjm-context-graph-projection`, `cjm-transcript-correction-qt`, `cjm-transcription-core`, `cjm-workflow-hub-qt`
@@ -7,21 +7,34 @@ A frontend-agnostic core for the transcript correction workflow — the first do
7
7
  ## Modules
8
8
 
9
9
  - **`cjm_transcript_correction_core.__init__`**
10
+ - **`cjm_transcript_correction_core.cleanread`** — The clean read — L1 of the reading ladder (design 6752db0a; the subtraction
10
11
  - **`cjm_transcript_correction_core.cli`** — The CLI driver — the correction core's first (and currently only) frontend. run <decomp-manifest> corrects the committed spine in the decomp graph DB, pointing the graph worker at that shared DB via load-time config, with optional session resume/reopen; review runs the interactive text-correction loop (the cross-transcriber diff is intra-graph since stage 5).
11
12
  - **`cjm_transcript_correction_core.graph`** — The correction overlay's graph I/O: targeted (scale-shaped) reads of a committed spine via the graph-storage query action, construction of Correction / CorrectionSession nodes + CORRECTS / SUPERSEDES / DERIVED_FROM / REVIEWED edges, the in-core effective-spine projection (layer-0 + applied corrections), and commit through the job queue. Hand-rolled (revolution-1) = direct CR-18 spec material; append-only on layer-0 (never update/delete a Segment).
12
13
  - **`cjm_transcript_correction_core.journal`** — Live append-through for the correction verbs — the workflow journal's domain half.
13
14
  - **`cjm_transcript_correction_core.launch`** — The shared launch surface every correction shell drives through: the
14
15
  - **`cjm_transcript_correction_core.models`** — Overlay data shapes for the transcript-correction workflow: the Correction / CorrectionSession graph nodes + their relation registry, the read view of a committed spine segment, the worklist item, run configuration, and the correction run manifest (proto-bundle that chains decomp -> correction).
15
16
  - **`cjm_transcript_correction_core.pipeline`** — The headless correction workflow: load a decomp run manifest, resolve the shared graph DB, start/resume/reopen a CorrectionSession, recompute the worklist from deterministic signals + persisted review state, run the D14 empty-segment prune (first operation), and record a chainable correction run manifest — with a cheapest-form HITL approval seam.
17
+ - **`cjm_transcript_correction_core.respine_transfer`** — The chunk-scoped transfer a chunk respine calls (work item 7a5e9c84; ruling 0b4d5cfa (5),
16
18
  - **`cjm_transcript_correction_core.signals`** — Pure deterministic Tier-1 signal functions (no capability calls): empty-segment detection, bidirectional boundary punctuation/capitalization heuristics, forced-alignment coverage flags, positional cross-transcriber diff, phonetic + edit-distance variant clustering, and the event-proposal overlay (leg 4: the finetuned detector's spans anchored onto the spine). The worklist is recomputed from these each session; revolution-1 builds ZERO new capabilities.
19
+ - **`cjm_transcript_correction_core.spans`** — Spans — the overlay-proposal lane's domain half (design bbf8bafd, extending the
17
20
  - **`cjm_transcript_correction_core.spine`**
18
21
  - **`cjm_transcript_correction_core.state`** — Sidecar view-state + spine-picker helpers — the correction TUI's pure
19
22
  - **`cjm_transcript_correction_core.strata`** — Strata — the filtering lane's domain half (DECs 304fd984 + 9d4c0a38; work items
20
23
 
21
24
  ## API
22
25
 
26
+ ### `cjm_transcript_correction_core.cleanread`
27
+
28
+ - `clean_read` _function_ — L1 — the clean read (pure). Kept lines in spine order; each carries its
29
+ - `clean_read_pack_read` _function_ — The `read` record an L1 pack carries: the layer, the marker, the
30
+ - `clean_read_segments` _function_ — Project the clean read back onto SpineSegments (clean text, same
31
+ - `clean_read_summary` _function_
32
+ - `repeat_survivor` _function_ — Locate the last repeating unit of a word-repeat span (pure): the
33
+ - `subtract_spans` _function_ — Cut the ranges out of the text, leaving `marker` at each seam (pure).
34
+
23
35
  ### `cjm_transcript_correction_core.cli`
24
36
 
37
+ - `attention_command` _function_ — Execute `attention`: the walk-lane ATTENTION TIER (item 3758f6cb, ruling f400d2c3)
25
38
  - `bench_command` _function_ — Execute the `bench` subcommand: the reserved-tail verdict join.
26
39
  - `build_parser` _function_ — Build the CLI parser (subcommands: run, review).
27
40
  - `commit_wordless_transfer` _function_ — COMMIT a planned transfer — the engine's second half. One
@@ -29,6 +42,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
29
42
  - `extract_command` _function_ — Execute the `extract` subcommand: fold the gated overlay into a manifested dataset.
30
43
  - `filter_confirm_command` _function_ — Execute `filter-confirm`: the HEADLESS HITL worklist (bc8dbbdd pass-1
31
44
  - `filter_ingest_command` _function_ — Execute `filter-ingest`: validate proposer rows against their pack and
45
+ - `filter_merge_command` _function_ — Execute `filter-merge`: fold several proposal sets over one spine into
32
46
  - `filter_pack_command` _function_ — Execute `filter-pack`: write one spine window's effective text-bearing
33
47
  - `gate_command` _function_ — Execute the `gate` subcommand: show or assert per-spine extraction gates.
34
48
  - `load_capabilities` _function_ — Discover manifests + load each capability, passing per-capability config (CR-2 caller-wins).
@@ -42,6 +56,11 @@ A frontend-agnostic core for the transcript correction workflow — the first do
42
56
  - `run_command` _function_ — Execute the `run` subcommand: correct a decomp manifest's committed spine.
43
57
  - `run_extract` _function_ — The extract fold on an ALREADY-OPEN graph seat (flywheel build leg 2,
44
58
  - `scan_command` _function_ — Execute `scan-mishomed`: flag authoritative FA words stranded outside
59
+ - `span_confirm_command` _function_ — Execute `span-confirm`: the HEADLESS HITL worklist over one span
60
+ - `span_ingest_command` _function_ — Execute `span-ingest`: resolve a proposer's quoted-word rows against
61
+ - `span_lexicon_command` _function_ — Execute `span-lexicon`: the pattern tier (design bbf8bafd (g)) — bare
62
+ - `span_merge_command` _function_ — Execute `span-merge`: fold several span sets over one spine into ONE
63
+ - `span_pack_command` _function_ — Execute `span-pack`: the filter pack's lines as a SPEECH-OVERLAY pack
45
64
  - `stats_command` _function_ — Execute the `stats` subcommand: flywheel accounting over the shared graph.
46
65
  - `transfer_command` _function_ — Execute `transfer-wordless`: replay wordless event inserts (and speaker-
47
66
  - `wordless_donors` _function_ — The EFFECTIVE wordless layer of a spine: labeled, effectively wordless
@@ -53,6 +72,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
53
72
  - `active_speaker_assignments` _function_ — Project the ACTIVE speaker assignment per segment (latest-wins).
54
73
  - `active_speech_overlays` _function_ — The surviving speech-overlay corrections (supersession applied; pure).
55
74
  - `aggregate_session_purposes` _function_ — Fold sessions into a per-source purpose mix (pure; d915d545 picker rung).
75
+ - `annotate_retired` _function_ — Mark spine rows with their retirement state (pure).
56
76
  - `apply_chunk_inserts` _function_ — Synthesize inserted chunks into the effective spine (DEC 3d3fa2a8).
57
77
  - `apply_time_nudges` _function_ — Apply timing corrections onto segment times (latest-wins per edge).
58
78
  - `bench_event_proposals` _function_ — The reserved-tail bench join (leg 4, DECs 8e05b87b + 8cf12c22) — pure.
@@ -91,6 +111,8 @@ A frontend-agnostic core for the transcript correction workflow — the first do
91
111
  - `correction_stats` _function_ — Fold one Source's ACTIVE overlay into flywheel-accounting counts (pure).
92
112
  - `corrections_to_edits` _function_ — Map this core's Correction payloads onto the layer's spine-edit vocabulary.
93
113
  - `count_source_segments` _function_ — Count a Source's segments server-side under its chosen rendition + skeleton (typed count mode).
114
+ - `default_spine` _function_ — Which spine opens by default (pure; ruling a7617bd4 rule (c)): preference is a
115
+ - `discharge_mark` _function_ — Discharge an open mark BY THE CORRECTION THAT ADDRESSED IT — the closure
94
116
  - `extract_spine_dataset` _function_ — Fold ONE spine's overlay into its v1 insert-span dataset slice (pure).
95
117
  - `fa_words_for_transcript` _function_ — One transcript's FA words in source coordinates (the scan-mishomed join,
96
118
  - `find_active_text_correction` _function_ — Single-segment convenience over the batch read (cross-session; latest wins).
@@ -123,6 +145,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
123
145
  - `set_session_status` _function_ — Update a session's status + updated_at.
124
146
  - `skeleton_hash_for` _function_ — Resolve a skeleton selector to the chosen spine's HASH (pure) — the gate's
125
147
  - `source_audio_segment_ids` _function_ — The Source's coarse spine (one small typed read; ordered by index).
148
+ - `speaker_assignment_sources` _function_ — Where each speaker Entity has been assigned — the picker's COLLECTION
126
149
  - `speech_overlay_spans` _function_ — Fold the ACTIVE speech overlays into extraction-facing span records (pure).
127
150
  - `spine_where_for` _function_ — Resolve a skeleton selector against the observed spine set (pure).
128
151
  - `start_session` _function_ — Create + commit a new CorrectionSession node.
@@ -166,8 +189,22 @@ A frontend-agnostic core for the transcript correction workflow — the first do
166
189
  - `run_correction` _function_ — Correct every source in a decomp run manifest (prune + worklist surfacing).
167
190
  - `run_review` _function_ — Interactive review pass over a decomp manifest's flagged worklist (text corrections).
168
191
 
192
+ ### `cjm_transcript_correction_core.respine_transfer`
193
+
194
+ - `chunk_respine_transfer` _function_ — The registered chunk-scoped transfer (the entry point decomp-core discovers).
195
+ - `group_speaker_rows` _function_ — Fold per-segment rows into turn-shaped commits (pure): contiguous same-entity
196
+ - `live_neighbour` _function_ — The live segment right before the chunk — the flank a donor that starts before
197
+ - `load_segments_by_ids` _function_ — Segments by id — the ONE read that bypasses the live predicate on purpose: the
198
+ - `plan_speaker_carry` _function_ — Which new segment takes which speaker (pure; 4a7ec4f8 (2)/(3)): a segment whose
199
+ - `speaker_spans` _function_ — The per-span speaker map the carry reads (pure).
200
+
169
201
  ### `cjm_transcript_correction_core.signals`
170
202
 
203
+ - `attention_boundary_marks` _function_ — The boundary signals of the attention tier (item 3758f6cb signals 1+3 and the folded
204
+ - `attention_divergence_marks` _function_ — The two-transcriber disagreement signal of the attention tier (item 3758f6cb signal
205
+ - `attention_fa_marks` _function_ — The forced-alignment signals of the attention tier (item 3758f6cb signal 1), pure.
206
+ - `attention_marks` _function_ — Compose the attention tier (item 3758f6cb): every enabled signal's mark rows,
207
+ - `attention_speaker_marks` _function_ — The next-speaker-change landmark of the folded item 65791933, pure: `speaker-change`
171
208
  - `boundary_punct_caps_flags` _function_ — Bidirectional boundary punctuation/capitalization heuristics (in-segment only).
172
209
  - `cluster_variants` _function_ — Cluster word variants by phonetic key + edit distance (fix-one-fix-all).
173
210
  - `compute_signal_flags` _function_ — Combine all deterministic Tier-1 signals into per-segment flags.
@@ -180,14 +217,36 @@ A frontend-agnostic core for the transcript correction workflow — the first do
180
217
  - `speaker_turn_proposals` _function_ — Dominant diarization cluster per segment — the assign lane's proposal paint.
181
218
  - `variant_divergence` _function_ — Within-segment cross-transcriber divergence (stage 5: intra-graph).
182
219
 
220
+ ### `cjm_transcript_correction_core.spans`
221
+
222
+ - `bench_span_proposals` _function_ — Derive the span verdicts (design bbf8bafd (f), the bench_filter_proposals
223
+ - `build_span_pack` _function_ — Build the span proposer's input: the filter pack's numbered lines with
224
+ - `find_token_span` _function_ — Resolve quoted words to WHOLE word tokens of a line (pure): the quote is
225
+ - `is_span_pack` _function_
226
+ - `lexicon_span_rows` _function_ — The LEXICON TIER (design bbf8bafd (g)): a bare um / uh token is a
227
+ - `load_span_proposal_sets` _function_ — Every SPAN proposal set for a source (and optionally one spine), newest
228
+ - `locate_span_tokens` _function_ — Where a proposal's words sit on the CURRENT line, as a character range
229
+ - `merge_span_proposals` _function_ — Fold several SPAN sets over one spine into one walkable set — the
230
+ - `overlay_label_slate` _function_ — The slate a span pack carries: every label with its gloss and whether a
231
+ - `pending_span_proposals` _function_ — The headless worklist: proposals with NO active overlay carrying their id
232
+ - `render_span_pack` _function_ — Render a span pack as the brief a proposer reads: identity + window, the
233
+ - `render_span_propset_markdown` _function_ — Project a span set for a human: one line per proposal — tier, label,
234
+ - `segment_word_tokens` _function_
235
+ - `snap_span_proposal` _function_ — Resolve a proposal against the CURRENT line at accept (design bbf8bafd
236
+ - `span_proposals_from_rows` _function_ — Resolve validated rows to span proposal-set rows: proposal id, label
237
+ - `validate_span_rows` _function_ — Validate + resolve proposer rows against their span pack — loud on the
238
+ - `write_span_propset` _function_ — Write one SPAN proposal set — the filter set's layout under the span
239
+
183
240
  ### `cjm_transcript_correction_core.spine`
184
241
 
185
242
  - `ChunkRef` _class_ — Where one Segment's VAD-chunk audio lives: the model-input WAV + the chunk-local span.
186
243
  - `SeamRef` _class_ — A source-coordinate audio span across one fine-spine boundary (the g/G
187
244
  - `SpineView` _class_ — One Source's effective correction spine, cursor-windowed for the TUI.
245
+ - `layer_speaker_menu` _function_ — Layer the assign-lane digit menu (pure; DEC 774dbe40). Tier 1 = the
188
246
  - `list_sources` _function_ — Enumerate the graph's Source nodes (the discovery corpus, 2ce81638).
189
247
  - `load_source_slice` _function_ — Decode a source-coordinate slice of the ORIGINAL media to playable samples.
190
248
  - `match_sources` _function_ — The --source selector (pure; shared by direct open and the picker's seed).
249
+ - `match_speaker_query` _function_ — Narrow the registry by a typed query (pure; DEC 774dbe40 (4)). Rank 0 =
191
250
  - `neighbor_word_bound` _function_ — The adjacent word's FA boundary facing an overlay span (pure).
192
251
  - `open_stack` _function_ — Bootstrap the graph capability stack, resolving the db path (2ce81638).
193
252
  - `parse_entity_input` _function_ — Parse the new-speaker editor line (pure). A leading `?` marks the entity
@@ -218,11 +277,18 @@ A frontend-agnostic core for the transcript correction workflow — the first do
218
277
  - `build_filter_pack` _function_ — Build the proposer's input: one source window's text-bearing effective
219
278
  - `exclude_strata` _function_ — The per-consumer filtered projection (DEC 9d4c0a38): "filtered" is a
220
279
  - `load_filter_proposal_sets` _function_ — Every filtering proposal set for a source (and optionally one spine),
280
+ - `materialized_fix_ids` _function_ — The fidelity-edit apply path's materialization: a proposer row whose
281
+ - `materialized_mark_ids` _function_ — Class-family routing's other half: a proposer's mark-family row (an ASR
282
+ - `merge_filter_proposals` _function_ — Fold several proposal sets over ONE spine into one walkable set (design
221
283
  - `new_pack_id` _function_ — Generate a unique, sortable pack id.
222
284
  - `pack_digest` _function_ — Digest the READ content (source binding + window + numbered segments) —
223
285
  - `pending_filter_proposals` _function_ — The headless worklist: proposals with NO live stratum carrying their id and
286
+ - `plan_pack_windows` _function_ — Cut a spine into `count` windows of near-equal TEXT-segment count at
224
287
  - `proposals_from_rows` _function_ — Resolve validated rows to proposal-set rows: proposal id, category, source
225
288
  - `render_filter_pack` _function_ — Render a pack as the brief a proposer reads: identity + window, the class
289
+ - `render_filter_propset_markdown` _function_ — Project a proposal set for a HUMAN to check against the source in the
290
+ - `resolve_slate` _function_ — Resolve a NAMED class slate (ruling c5b6cf42) to the two vocabularies a
291
+ - `select_span_segments` _function_ — The span-EDIT gesture's resolver (agreed 2026-09-01 on the ch04 confirm
226
292
  - `strata_index` _function_ — Segment-keyed view of the live strata (the consumer query's index).
227
293
  - `validate_proposal_rows` _function_ — Validate + normalize proposer rows against their pack — loud on the first
228
294
  - `write_filter_propset` _function_ — Write one filtering proposal set: `<out_root>/<set_id>/manifest.json` +
@@ -230,4 +296,4 @@ A frontend-agnostic core for the transcript correction workflow — the first do
230
296
  ## Dependencies
231
297
 
232
298
  **Depends on:** `cjm-capability-primitives`, `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-substrate`, `cjm-transcript-graph-schema`, `numpy`
233
- **Used by:** `cjm-transcript-correction-qt`, `cjm-transcript-correction-tui`, `cjm-transcription-core`, `cjm-workflow-hub-qt`, `cjm-workflow-hub-tui`
299
+ **Used by:** `cjm-context-graph-projection`, `cjm-transcript-correction-qt`, `cjm-transcription-core`, `cjm-workflow-hub-qt`
@@ -0,0 +1,220 @@
1
+ """The clean read — L1 of the reading ladder (design 6752db0a; the subtraction
2
+ shape from bbf8bafd (a)).
3
+
4
+ L0 is the spine as spoken. L1 is the same spine MINUS what a reader never
5
+ needs, as a MECHANICAL projection with VISIBLE elisions — it subtracts twice:
6
+
7
+ 1. FILLER LINES first: every text segment under an excluded stratum (default
8
+ `filler` — a wholly elidable line or run; the notes types' exclude_strata
9
+ name more) drops as a line;
10
+ 2. then the FILTERABLE OVERLAY SPANS inside the lines that remain — the
11
+ annotate lane's speech overlays whose label a reader filters (hesitation-
12
+ marker / discourse-marker / word-repeat / false-start); the KEEP labels
13
+ (emphasis-repeat / coincidental-repeat) never subtract. A line emptied by
14
+ its overlays drops too.
15
+
16
+ The projection is pure over the effective spine + the active strata + the
17
+ active overlays; nothing is stored. Every elision is visible: a subtracted
18
+ span leaves `marker` in the line (default `[…]`; "" for a clean-verbatim
19
+ consumer), a dropped line is recorded in the gap before the next kept line,
20
+ and each kept line carries what was cut. A wrong exclusion must never HIDE
21
+ content silently (ruling 6752db0a (9)) — the reader can always see that
22
+ something was elided and audit it against L0.
23
+
24
+ WORD-REPEAT KEEPS ITS LAST UNIT: the overlay quotes the whole run ('the the',
25
+ 'we, we', 'and and and') because the run is the detector sample; the clean
26
+ read keeps the speaker's final instance — the span's tokens are split into
27
+ the shortest repeating unit and only the last unit survives; a run that is
28
+ not a clean repetition keeps its last token (a mechanical rule, auditable
29
+ through the marker; refine per label as drives show).
30
+ """
31
+
32
+ import re
33
+ from dataclasses import replace
34
+ from typing import Any, Dict, List, Sequence, Tuple
35
+
36
+ from cjm_transcript_correction_core.graph import reanchor_span
37
+ from cjm_transcript_correction_core.models import SpineSegment
38
+ from cjm_transcript_correction_core.strata import strata_index
39
+
40
+ CLEAN_READ_EXCLUDE_STRATA = ("filler",) # The default line-level subtraction (the notes types name more)
41
+ CLEAN_READ_FILTER_LABELS = ("hesitation-marker", "discourse-marker", "word-repeat", "false-start")
42
+ CLEAN_READ_KEEP_LAST_UNIT = ("word-repeat",) # Labels whose span keeps its final repeated unit
43
+ ELISION_MARKER = "[…]"
44
+
45
+
46
+ def _norm(t: str) -> str: # Case/punctuation-insensitive token form (the span lane's rule)
47
+ return "".join(ch for ch in t.lower() if ch.isalnum())
48
+
49
+
50
+ def repeat_survivor(
51
+ span_text: str, # The overlay's words (a whole repeated run)
52
+ ) -> Tuple[int, int]: # (char offset, char end) INSIDE span_text of the surviving last unit
53
+ """Locate the last repeating unit of a word-repeat span (pure): the
54
+ shortest unit u with the span = u × n (normalized) wins; no clean
55
+ repetition = the last token. Returned offsets are relative to the span."""
56
+ toks = [(m.start(), m.end(), m.group()) for m in re.finditer(r"\S+", span_text or "")]
57
+ if not toks:
58
+ return 0, len(span_text or "")
59
+ norm = [_norm(t) for _, _, t in toks]
60
+ k = len(norm)
61
+ for unit in range(1, k // 2 + 1):
62
+ if k % unit == 0 and all(norm[i:i + unit] == norm[:unit] for i in range(0, k, unit)):
63
+ return toks[k - unit][0], toks[-1][1]
64
+ return toks[-1][0], toks[-1][1]
65
+
66
+
67
+ def _cut_ranges(
68
+ text: str, # The line's effective text
69
+ overlays: List[Dict[str, Any]], # Active overlay correction dicts anchored on this line
70
+ labels: Sequence[str], # Filterable labels
71
+ keep_last: Sequence[str], # Labels that keep their last repeated unit
72
+ ) -> Tuple[List[Tuple[int, int]], List[Dict[str, Any]]]: # (merged char ranges to cut, the cuts made)
73
+ cuts: List[Tuple[int, int]] = []
74
+ made: List[Dict[str, Any]] = []
75
+ for c in overlays:
76
+ p = c.get("payload") or {}
77
+ lab = p.get("label")
78
+ if lab not in labels:
79
+ continue
80
+ a = p.get("anchor") or {}
81
+ located = reanchor_span(a, text)
82
+ if located is None:
83
+ made.append({"overlay_id": c.get("id"), "label": lab, "text": a.get("text_snapshot"),
84
+ "cut": None, "note": "words no longer on the line"})
85
+ continue
86
+ cs, ce = located
87
+ if lab in keep_last:
88
+ ks, ke = repeat_survivor(text[cs:ce])
89
+ rng = (cs, cs + ks) # everything before the surviving unit
90
+ tail = (cs + ke, ce) # trailing punctuation after it, if any
91
+ cuts.append(rng)
92
+ if tail[1] > tail[0]:
93
+ cuts.append(tail)
94
+ made.append({"overlay_id": c.get("id"), "label": lab, "text": text[cs:ce],
95
+ "cut": text[cs:cs + ks], "kept": text[cs + ks:cs + ke]})
96
+ else:
97
+ cuts.append((cs, ce))
98
+ made.append({"overlay_id": c.get("id"), "label": lab, "text": text[cs:ce],
99
+ "cut": text[cs:ce]})
100
+ cuts.sort()
101
+ merged: List[Tuple[int, int]] = []
102
+ for s, e in cuts:
103
+ if e <= s:
104
+ continue
105
+ if merged and s <= merged[-1][1]:
106
+ merged[-1] = (merged[-1][0], max(merged[-1][1], e))
107
+ else:
108
+ merged.append((s, e))
109
+ return merged, made
110
+
111
+
112
+ def _tidy(s: str) -> str: # Whitespace + punctuation seams after a cut
113
+ s = re.sub(r"\s+", " ", s).strip()
114
+ # ", ," -> "," — a cut's seam always leaves WHITESPACE between the two marks, so the rule
115
+ # demands it and runs BEFORE the space-before-punctuation rule closes that gap: a `::`
116
+ # the source wrote (thrust::reduce) is adjacent and never a seam (finding: the clean
117
+ # read handed a proposer `thrust:reduce`).
118
+ s = re.sub(r"([,;:])(?:\s+[,;:])+", r"\1", s)
119
+ s = re.sub(r"\s+([,.;:?!])", r"\1", s) # "larger, than" — a space before punctuation
120
+ s = re.sub(r"^[,;:](?![,;:])\s*", "", s) # a leading comma left by a cut at the line start (never a leading `::`)
121
+ return s
122
+
123
+
124
+ def subtract_spans(
125
+ text: str, # The line's effective text
126
+ ranges: List[Tuple[int, int]], # Merged char ranges to cut (from _cut_ranges)
127
+ marker: str = ELISION_MARKER, # What a cut leaves behind ("" = nothing)
128
+ ) -> str: # The clean text (tidied)
129
+ """Cut the ranges out of the text, leaving `marker` at each seam (pure)."""
130
+ if not ranges:
131
+ return text
132
+ out: List[str] = []
133
+ at = 0
134
+ for s, e in ranges:
135
+ out.append(text[at:s])
136
+ if marker:
137
+ out.append(f" {marker} ")
138
+ at = e
139
+ out.append(text[at:])
140
+ joined = "".join(out)
141
+ if marker: # never let the marker glue to punctuation-less neighbours: "we [...] have"
142
+ joined = re.sub(r"\s*" + re.escape(marker) + r"\s*", f" {marker} ", joined)
143
+ return _tidy(joined)
144
+
145
+
146
+ def clean_read(
147
+ segments: Sequence[SpineSegment], # The EFFECTIVE spine (corrections applied), index order
148
+ strata: List[Dict[str, Any]], # active_strata output
149
+ overlays: List[Dict[str, Any]], # active_speech_overlays output
150
+ *,
151
+ exclude_strata: Sequence[str] = CLEAN_READ_EXCLUDE_STRATA, # Line-level subtraction classes
152
+ filter_labels: Sequence[str] = CLEAN_READ_FILTER_LABELS, # Overlay labels a reader filters
153
+ keep_last_unit: Sequence[str] = CLEAN_READ_KEEP_LAST_UNIT, # Labels whose span keeps its last unit
154
+ marker: str = ELISION_MARKER, # In-line elision marker ("" = clean-verbatim)
155
+ ) -> List[Dict[str, Any]]: # Kept lines: {id, index, start, end, text, raw_text, elisions, dropped_before}
156
+ """L1 — the clean read (pure). Kept lines in spine order; each carries its
157
+ clean `text`, the L0 `raw_text`, the `elisions` cut inside it, and
158
+ `dropped_before`: the lines elided between the previous kept line and
159
+ this one (segment id, index, why). Wordless segments are not lines and
160
+ are never counted as dropped."""
161
+ idx = strata_index(strata)
162
+ drop = set(exclude_strata)
163
+ by_seg: Dict[str, List[Dict[str, Any]]] = {}
164
+ for c in overlays:
165
+ sid = ((c.get("payload") or {}).get("anchor") or {}).get("segment_id")
166
+ if sid:
167
+ by_seg.setdefault(sid, []).append(c)
168
+ out: List[Dict[str, Any]] = []
169
+ pending_drops: List[Dict[str, Any]] = []
170
+ for s in sorted(segments, key=lambda x: x.index):
171
+ if s.is_empty:
172
+ continue
173
+ cats = drop & set(idx.get(s.id, []))
174
+ if cats:
175
+ pending_drops.append({"id": s.id, "index": s.index, "why": sorted(cats)})
176
+ continue
177
+ text = s.text or ""
178
+ ranges, made = _cut_ranges(text, by_seg.get(s.id, []), filter_labels, keep_last_unit)
179
+ clean = subtract_spans(text, ranges, marker) if ranges else text
180
+ bare = clean.replace(marker, "").strip() if marker else clean.strip()
181
+ if not bare:
182
+ pending_drops.append({"id": s.id, "index": s.index,
183
+ "why": ["emptied by " + ", ".join(sorted({m["label"] for m in made}))]})
184
+ continue
185
+ out.append({"id": s.id, "index": s.index,
186
+ "start": float(s.start_time) if s.start_time is not None else None,
187
+ "end": float(s.end_time) if s.end_time is not None else None,
188
+ "text": clean, "raw_text": text, "elisions": made,
189
+ "dropped_before": pending_drops})
190
+ pending_drops = []
191
+ if pending_drops and out: # a trailing run of dropped lines hangs off the last kept line
192
+ out[-1]["dropped_after"] = pending_drops
193
+ return out
194
+
195
+
196
+ def clean_read_summary(lines: List[Dict[str, Any]]) -> Dict[str, int]: # Counts for a pack / a report
197
+ dropped = sum(len(ln.get("dropped_before") or []) + len(ln.get("dropped_after") or []) for ln in lines)
198
+ cut = sum(1 for ln in lines for m in (ln.get("elisions") or []) if m.get("cut") is not None)
199
+ return {"lines_kept": len(lines), "lines_dropped": dropped, "spans_cut": cut}
200
+
201
+
202
+ def clean_read_pack_read(
203
+ lines: List[Dict[str, Any]], # clean_read output
204
+ marker: str = ELISION_MARKER, # The marker the lines carry
205
+ ) -> Dict[str, Any]: # build_filter_pack's `read` argument for an L1 pack
206
+ """The `read` record an L1 pack carries: the layer, the marker, the
207
+ counts, and the gaps (segment id -> the lines dropped before it) the
208
+ pack builder attaches to its rows so the brief shows every elision."""
209
+ return {"layer": "clean", "marker": marker, **clean_read_summary(lines),
210
+ "gaps": {ln["id"]: ln["dropped_before"] for ln in lines if ln.get("dropped_before")}}
211
+
212
+
213
+ def clean_read_segments(
214
+ lines: List[Dict[str, Any]], # clean_read output
215
+ segments: Sequence[SpineSegment], # The same effective spine (for the fields a line does not carry)
216
+ ) -> List[SpineSegment]: # SpineSegments carrying the CLEAN text — what a pack builder reads as L1
217
+ """Project the clean read back onto SpineSegments (clean text, same
218
+ identity and times), so every pack builder reads L1 unchanged."""
219
+ by_id = {s.id: s for s in segments}
220
+ return [replace(by_id[ln["id"]], text=ln["text"]) for ln in lines]