cjm-transcript-correction-core 0.0.10__tar.gz → 0.0.30__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cjm_transcript_correction_core-0.0.10/cjm_transcript_correction_core.egg-info → cjm_transcript_correction_core-0.0.30}/PKG-INFO +72 -3
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/README.md +67 -1
- cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core/__init__.py +1 -0
- cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core/cleanread.py +220 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/cli.py +976 -33
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/graph.py +243 -26
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/journal.py +2 -1
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/models.py +51 -7
- cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core/respine_transfer.py +243 -0
- cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core/signals.py +684 -0
- cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core/spans.py +868 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/spine.py +74 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/state.py +3 -2
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/strata.py +454 -51
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30/cjm_transcript_correction_core.egg-info}/PKG-INFO +72 -3
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core.egg-info/SOURCES.txt +7 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core.egg-info/entry_points.txt +3 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core.egg-info/requires.txt +5 -1
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/pyproject.toml +7 -1
- cjm_transcript_correction_core-0.0.30/tests/test_cleanread.py +134 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_cli.py +6 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_graph.py +153 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_journal.py +1 -0
- cjm_transcript_correction_core-0.0.30/tests/test_respine_transfer.py +175 -0
- cjm_transcript_correction_core-0.0.30/tests/test_signals.py +190 -0
- cjm_transcript_correction_core-0.0.30/tests/test_spans.py +408 -0
- cjm_transcript_correction_core-0.0.30/tests/test_speaker_picker.py +106 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_strata.py +247 -5
- cjm_transcript_correction_core-0.0.10/cjm_transcript_correction_core/__init__.py +0 -1
- cjm_transcript_correction_core-0.0.10/cjm_transcript_correction_core/signals.py +0 -341
- cjm_transcript_correction_core-0.0.10/tests/test_signals.py +0 -90
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/LICENSE +0 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/MANIFEST.in +0 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/launch.py +0 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core/pipeline.py +0 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core.egg-info/dependency_links.txt +0 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/cjm_transcript_correction_core.egg-info/top_level.txt +0 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/setup.cfg +0 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_bench.py +0 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_gestures.py +0 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_models.py +0 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_pipeline.py +0 -0
- {cjm_transcript_correction_core-0.0.10 → cjm_transcript_correction_core-0.0.30}/tests/test_state.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cjm-transcript-correction-core
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.30
|
|
4
4
|
Summary: A frontend-agnostic core for the transcript correction workflow — the first downstream graph-extension core; composes the graph-storage capability worker into a headless pipeline that applies unified, non-destructive corrections (text, punctuation, segmentation) as a supersede-able overlay on a committed decomposition spine, recomputes the review worklist from deterministic signals, and exposes a CLI as its first driver.
|
|
5
5
|
Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -14,8 +14,11 @@ Requires-Dist: cjm_substrate>=0.0.61
|
|
|
14
14
|
Requires-Dist: cjm_context_graph_primitives>=0.0.15
|
|
15
15
|
Requires-Dist: cjm-capability-primitives>=0.0.12
|
|
16
16
|
Requires-Dist: cjm-context-graph-layer>=0.0.11
|
|
17
|
-
Requires-Dist: cjm-transcript-graph-schema>=0.0.
|
|
17
|
+
Requires-Dist: cjm-transcript-graph-schema>=0.0.11
|
|
18
18
|
Requires-Dist: numpy
|
|
19
|
+
Provides-Extra: test
|
|
20
|
+
Requires-Dist: soundfile; extra == "test"
|
|
21
|
+
Requires-Dist: pytest; extra == "test"
|
|
19
22
|
Dynamic: license-file
|
|
20
23
|
|
|
21
24
|
# cjm-transcript-correction-core
|
|
@@ -27,21 +30,34 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
27
30
|
## Modules
|
|
28
31
|
|
|
29
32
|
- **`cjm_transcript_correction_core.__init__`**
|
|
33
|
+
- **`cjm_transcript_correction_core.cleanread`** — The clean read — L1 of the reading ladder (design 6752db0a; the subtraction
|
|
30
34
|
- **`cjm_transcript_correction_core.cli`** — The CLI driver — the correction core's first (and currently only) frontend. run <decomp-manifest> corrects the committed spine in the decomp graph DB, pointing the graph worker at that shared DB via load-time config, with optional session resume/reopen; review runs the interactive text-correction loop (the cross-transcriber diff is intra-graph since stage 5).
|
|
31
35
|
- **`cjm_transcript_correction_core.graph`** — The correction overlay's graph I/O: targeted (scale-shaped) reads of a committed spine via the graph-storage query action, construction of Correction / CorrectionSession nodes + CORRECTS / SUPERSEDES / DERIVED_FROM / REVIEWED edges, the in-core effective-spine projection (layer-0 + applied corrections), and commit through the job queue. Hand-rolled (revolution-1) = direct CR-18 spec material; append-only on layer-0 (never update/delete a Segment).
|
|
32
36
|
- **`cjm_transcript_correction_core.journal`** — Live append-through for the correction verbs — the workflow journal's domain half.
|
|
33
37
|
- **`cjm_transcript_correction_core.launch`** — The shared launch surface every correction shell drives through: the
|
|
34
38
|
- **`cjm_transcript_correction_core.models`** — Overlay data shapes for the transcript-correction workflow: the Correction / CorrectionSession graph nodes + their relation registry, the read view of a committed spine segment, the worklist item, run configuration, and the correction run manifest (proto-bundle that chains decomp -> correction).
|
|
35
39
|
- **`cjm_transcript_correction_core.pipeline`** — The headless correction workflow: load a decomp run manifest, resolve the shared graph DB, start/resume/reopen a CorrectionSession, recompute the worklist from deterministic signals + persisted review state, run the D14 empty-segment prune (first operation), and record a chainable correction run manifest — with a cheapest-form HITL approval seam.
|
|
40
|
+
- **`cjm_transcript_correction_core.respine_transfer`** — The chunk-scoped transfer a chunk respine calls (work item 7a5e9c84; ruling 0b4d5cfa (5),
|
|
36
41
|
- **`cjm_transcript_correction_core.signals`** — Pure deterministic Tier-1 signal functions (no capability calls): empty-segment detection, bidirectional boundary punctuation/capitalization heuristics, forced-alignment coverage flags, positional cross-transcriber diff, phonetic + edit-distance variant clustering, and the event-proposal overlay (leg 4: the finetuned detector's spans anchored onto the spine). The worklist is recomputed from these each session; revolution-1 builds ZERO new capabilities.
|
|
42
|
+
- **`cjm_transcript_correction_core.spans`** — Spans — the overlay-proposal lane's domain half (design bbf8bafd, extending the
|
|
37
43
|
- **`cjm_transcript_correction_core.spine`**
|
|
38
44
|
- **`cjm_transcript_correction_core.state`** — Sidecar view-state + spine-picker helpers — the correction TUI's pure
|
|
39
45
|
- **`cjm_transcript_correction_core.strata`** — Strata — the filtering lane's domain half (DECs 304fd984 + 9d4c0a38; work items
|
|
40
46
|
|
|
41
47
|
## API
|
|
42
48
|
|
|
49
|
+
### `cjm_transcript_correction_core.cleanread`
|
|
50
|
+
|
|
51
|
+
- `clean_read` _function_ — L1 — the clean read (pure). Kept lines in spine order; each carries its
|
|
52
|
+
- `clean_read_pack_read` _function_ — The `read` record an L1 pack carries: the layer, the marker, the
|
|
53
|
+
- `clean_read_segments` _function_ — Project the clean read back onto SpineSegments (clean text, same
|
|
54
|
+
- `clean_read_summary` _function_
|
|
55
|
+
- `repeat_survivor` _function_ — Locate the last repeating unit of a word-repeat span (pure): the
|
|
56
|
+
- `subtract_spans` _function_ — Cut the ranges out of the text, leaving `marker` at each seam (pure).
|
|
57
|
+
|
|
43
58
|
### `cjm_transcript_correction_core.cli`
|
|
44
59
|
|
|
60
|
+
- `attention_command` _function_ — Execute `attention`: the walk-lane ATTENTION TIER (item 3758f6cb, ruling f400d2c3)
|
|
45
61
|
- `bench_command` _function_ — Execute the `bench` subcommand: the reserved-tail verdict join.
|
|
46
62
|
- `build_parser` _function_ — Build the CLI parser (subcommands: run, review).
|
|
47
63
|
- `commit_wordless_transfer` _function_ — COMMIT a planned transfer — the engine's second half. One
|
|
@@ -49,6 +65,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
49
65
|
- `extract_command` _function_ — Execute the `extract` subcommand: fold the gated overlay into a manifested dataset.
|
|
50
66
|
- `filter_confirm_command` _function_ — Execute `filter-confirm`: the HEADLESS HITL worklist (bc8dbbdd pass-1
|
|
51
67
|
- `filter_ingest_command` _function_ — Execute `filter-ingest`: validate proposer rows against their pack and
|
|
68
|
+
- `filter_merge_command` _function_ — Execute `filter-merge`: fold several proposal sets over one spine into
|
|
52
69
|
- `filter_pack_command` _function_ — Execute `filter-pack`: write one spine window's effective text-bearing
|
|
53
70
|
- `gate_command` _function_ — Execute the `gate` subcommand: show or assert per-spine extraction gates.
|
|
54
71
|
- `load_capabilities` _function_ — Discover manifests + load each capability, passing per-capability config (CR-2 caller-wins).
|
|
@@ -62,6 +79,11 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
62
79
|
- `run_command` _function_ — Execute the `run` subcommand: correct a decomp manifest's committed spine.
|
|
63
80
|
- `run_extract` _function_ — The extract fold on an ALREADY-OPEN graph seat (flywheel build leg 2,
|
|
64
81
|
- `scan_command` _function_ — Execute `scan-mishomed`: flag authoritative FA words stranded outside
|
|
82
|
+
- `span_confirm_command` _function_ — Execute `span-confirm`: the HEADLESS HITL worklist over one span
|
|
83
|
+
- `span_ingest_command` _function_ — Execute `span-ingest`: resolve a proposer's quoted-word rows against
|
|
84
|
+
- `span_lexicon_command` _function_ — Execute `span-lexicon`: the pattern tier (design bbf8bafd (g)) — bare
|
|
85
|
+
- `span_merge_command` _function_ — Execute `span-merge`: fold several span sets over one spine into ONE
|
|
86
|
+
- `span_pack_command` _function_ — Execute `span-pack`: the filter pack's lines as a SPEECH-OVERLAY pack
|
|
65
87
|
- `stats_command` _function_ — Execute the `stats` subcommand: flywheel accounting over the shared graph.
|
|
66
88
|
- `transfer_command` _function_ — Execute `transfer-wordless`: replay wordless event inserts (and speaker-
|
|
67
89
|
- `wordless_donors` _function_ — The EFFECTIVE wordless layer of a spine: labeled, effectively wordless
|
|
@@ -73,6 +95,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
73
95
|
- `active_speaker_assignments` _function_ — Project the ACTIVE speaker assignment per segment (latest-wins).
|
|
74
96
|
- `active_speech_overlays` _function_ — The surviving speech-overlay corrections (supersession applied; pure).
|
|
75
97
|
- `aggregate_session_purposes` _function_ — Fold sessions into a per-source purpose mix (pure; d915d545 picker rung).
|
|
98
|
+
- `annotate_retired` _function_ — Mark spine rows with their retirement state (pure).
|
|
76
99
|
- `apply_chunk_inserts` _function_ — Synthesize inserted chunks into the effective spine (DEC 3d3fa2a8).
|
|
77
100
|
- `apply_time_nudges` _function_ — Apply timing corrections onto segment times (latest-wins per edge).
|
|
78
101
|
- `bench_event_proposals` _function_ — The reserved-tail bench join (leg 4, DECs 8e05b87b + 8cf12c22) — pure.
|
|
@@ -111,6 +134,8 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
111
134
|
- `correction_stats` _function_ — Fold one Source's ACTIVE overlay into flywheel-accounting counts (pure).
|
|
112
135
|
- `corrections_to_edits` _function_ — Map this core's Correction payloads onto the layer's spine-edit vocabulary.
|
|
113
136
|
- `count_source_segments` _function_ — Count a Source's segments server-side under its chosen rendition + skeleton (typed count mode).
|
|
137
|
+
- `default_spine` _function_ — Which spine opens by default (pure; ruling a7617bd4 rule (c)): preference is a
|
|
138
|
+
- `discharge_mark` _function_ — Discharge an open mark BY THE CORRECTION THAT ADDRESSED IT — the closure
|
|
114
139
|
- `extract_spine_dataset` _function_ — Fold ONE spine's overlay into its v1 insert-span dataset slice (pure).
|
|
115
140
|
- `fa_words_for_transcript` _function_ — One transcript's FA words in source coordinates (the scan-mishomed join,
|
|
116
141
|
- `find_active_text_correction` _function_ — Single-segment convenience over the batch read (cross-session; latest wins).
|
|
@@ -143,6 +168,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
143
168
|
- `set_session_status` _function_ — Update a session's status + updated_at.
|
|
144
169
|
- `skeleton_hash_for` _function_ — Resolve a skeleton selector to the chosen spine's HASH (pure) — the gate's
|
|
145
170
|
- `source_audio_segment_ids` _function_ — The Source's coarse spine (one small typed read; ordered by index).
|
|
171
|
+
- `speaker_assignment_sources` _function_ — Where each speaker Entity has been assigned — the picker's COLLECTION
|
|
146
172
|
- `speech_overlay_spans` _function_ — Fold the ACTIVE speech overlays into extraction-facing span records (pure).
|
|
147
173
|
- `spine_where_for` _function_ — Resolve a skeleton selector against the observed spine set (pure).
|
|
148
174
|
- `start_session` _function_ — Create + commit a new CorrectionSession node.
|
|
@@ -186,8 +212,22 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
186
212
|
- `run_correction` _function_ — Correct every source in a decomp run manifest (prune + worklist surfacing).
|
|
187
213
|
- `run_review` _function_ — Interactive review pass over a decomp manifest's flagged worklist (text corrections).
|
|
188
214
|
|
|
215
|
+
### `cjm_transcript_correction_core.respine_transfer`
|
|
216
|
+
|
|
217
|
+
- `chunk_respine_transfer` _function_ — The registered chunk-scoped transfer (the entry point decomp-core discovers).
|
|
218
|
+
- `group_speaker_rows` _function_ — Fold per-segment rows into turn-shaped commits (pure): contiguous same-entity
|
|
219
|
+
- `live_neighbour` _function_ — The live segment right before the chunk — the flank a donor that starts before
|
|
220
|
+
- `load_segments_by_ids` _function_ — Segments by id — the ONE read that bypasses the live predicate on purpose: the
|
|
221
|
+
- `plan_speaker_carry` _function_ — Which new segment takes which speaker (pure; 4a7ec4f8 (2)/(3)): a segment whose
|
|
222
|
+
- `speaker_spans` _function_ — The per-span speaker map the carry reads (pure).
|
|
223
|
+
|
|
189
224
|
### `cjm_transcript_correction_core.signals`
|
|
190
225
|
|
|
226
|
+
- `attention_boundary_marks` _function_ — The boundary signals of the attention tier (item 3758f6cb signals 1+3 and the folded
|
|
227
|
+
- `attention_divergence_marks` _function_ — The two-transcriber disagreement signal of the attention tier (item 3758f6cb signal
|
|
228
|
+
- `attention_fa_marks` _function_ — The forced-alignment signals of the attention tier (item 3758f6cb signal 1), pure.
|
|
229
|
+
- `attention_marks` _function_ — Compose the attention tier (item 3758f6cb): every enabled signal's mark rows,
|
|
230
|
+
- `attention_speaker_marks` _function_ — The next-speaker-change landmark of the folded item 65791933, pure: `speaker-change`
|
|
191
231
|
- `boundary_punct_caps_flags` _function_ — Bidirectional boundary punctuation/capitalization heuristics (in-segment only).
|
|
192
232
|
- `cluster_variants` _function_ — Cluster word variants by phonetic key + edit distance (fix-one-fix-all).
|
|
193
233
|
- `compute_signal_flags` _function_ — Combine all deterministic Tier-1 signals into per-segment flags.
|
|
@@ -200,14 +240,36 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
200
240
|
- `speaker_turn_proposals` _function_ — Dominant diarization cluster per segment — the assign lane's proposal paint.
|
|
201
241
|
- `variant_divergence` _function_ — Within-segment cross-transcriber divergence (stage 5: intra-graph).
|
|
202
242
|
|
|
243
|
+
### `cjm_transcript_correction_core.spans`
|
|
244
|
+
|
|
245
|
+
- `bench_span_proposals` _function_ — Derive the span verdicts (design bbf8bafd (f), the bench_filter_proposals
|
|
246
|
+
- `build_span_pack` _function_ — Build the span proposer's input: the filter pack's numbered lines with
|
|
247
|
+
- `find_token_span` _function_ — Resolve quoted words to WHOLE word tokens of a line (pure): the quote is
|
|
248
|
+
- `is_span_pack` _function_
|
|
249
|
+
- `lexicon_span_rows` _function_ — The LEXICON TIER (design bbf8bafd (g)): a bare um / uh token is a
|
|
250
|
+
- `load_span_proposal_sets` _function_ — Every SPAN proposal set for a source (and optionally one spine), newest
|
|
251
|
+
- `locate_span_tokens` _function_ — Where a proposal's words sit on the CURRENT line, as a character range
|
|
252
|
+
- `merge_span_proposals` _function_ — Fold several SPAN sets over one spine into one walkable set — the
|
|
253
|
+
- `overlay_label_slate` _function_ — The slate a span pack carries: every label with its gloss and whether a
|
|
254
|
+
- `pending_span_proposals` _function_ — The headless worklist: proposals with NO active overlay carrying their id
|
|
255
|
+
- `render_span_pack` _function_ — Render a span pack as the brief a proposer reads: identity + window, the
|
|
256
|
+
- `render_span_propset_markdown` _function_ — Project a span set for a human: one line per proposal — tier, label,
|
|
257
|
+
- `segment_word_tokens` _function_
|
|
258
|
+
- `snap_span_proposal` _function_ — Resolve a proposal against the CURRENT line at accept (design bbf8bafd
|
|
259
|
+
- `span_proposals_from_rows` _function_ — Resolve validated rows to span proposal-set rows: proposal id, label
|
|
260
|
+
- `validate_span_rows` _function_ — Validate + resolve proposer rows against their span pack — loud on the
|
|
261
|
+
- `write_span_propset` _function_ — Write one SPAN proposal set — the filter set's layout under the span
|
|
262
|
+
|
|
203
263
|
### `cjm_transcript_correction_core.spine`
|
|
204
264
|
|
|
205
265
|
- `ChunkRef` _class_ — Where one Segment's VAD-chunk audio lives: the model-input WAV + the chunk-local span.
|
|
206
266
|
- `SeamRef` _class_ — A source-coordinate audio span across one fine-spine boundary (the g/G
|
|
207
267
|
- `SpineView` _class_ — One Source's effective correction spine, cursor-windowed for the TUI.
|
|
268
|
+
- `layer_speaker_menu` _function_ — Layer the assign-lane digit menu (pure; DEC 774dbe40). Tier 1 = the
|
|
208
269
|
- `list_sources` _function_ — Enumerate the graph's Source nodes (the discovery corpus, 2ce81638).
|
|
209
270
|
- `load_source_slice` _function_ — Decode a source-coordinate slice of the ORIGINAL media to playable samples.
|
|
210
271
|
- `match_sources` _function_ — The --source selector (pure; shared by direct open and the picker's seed).
|
|
272
|
+
- `match_speaker_query` _function_ — Narrow the registry by a typed query (pure; DEC 774dbe40 (4)). Rank 0 =
|
|
211
273
|
- `neighbor_word_bound` _function_ — The adjacent word's FA boundary facing an overlay span (pure).
|
|
212
274
|
- `open_stack` _function_ — Bootstrap the graph capability stack, resolving the db path (2ce81638).
|
|
213
275
|
- `parse_entity_input` _function_ — Parse the new-speaker editor line (pure). A leading `?` marks the entity
|
|
@@ -238,11 +300,18 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
238
300
|
- `build_filter_pack` _function_ — Build the proposer's input: one source window's text-bearing effective
|
|
239
301
|
- `exclude_strata` _function_ — The per-consumer filtered projection (DEC 9d4c0a38): "filtered" is a
|
|
240
302
|
- `load_filter_proposal_sets` _function_ — Every filtering proposal set for a source (and optionally one spine),
|
|
303
|
+
- `materialized_fix_ids` _function_ — The fidelity-edit apply path's materialization: a proposer row whose
|
|
304
|
+
- `materialized_mark_ids` _function_ — Class-family routing's other half: a proposer's mark-family row (an ASR
|
|
305
|
+
- `merge_filter_proposals` _function_ — Fold several proposal sets over ONE spine into one walkable set (design
|
|
241
306
|
- `new_pack_id` _function_ — Generate a unique, sortable pack id.
|
|
242
307
|
- `pack_digest` _function_ — Digest the READ content (source binding + window + numbered segments) —
|
|
243
308
|
- `pending_filter_proposals` _function_ — The headless worklist: proposals with NO live stratum carrying their id and
|
|
309
|
+
- `plan_pack_windows` _function_ — Cut a spine into `count` windows of near-equal TEXT-segment count at
|
|
244
310
|
- `proposals_from_rows` _function_ — Resolve validated rows to proposal-set rows: proposal id, category, source
|
|
245
311
|
- `render_filter_pack` _function_ — Render a pack as the brief a proposer reads: identity + window, the class
|
|
312
|
+
- `render_filter_propset_markdown` _function_ — Project a proposal set for a HUMAN to check against the source in the
|
|
313
|
+
- `resolve_slate` _function_ — Resolve a NAMED class slate (ruling c5b6cf42) to the two vocabularies a
|
|
314
|
+
- `select_span_segments` _function_ — The span-EDIT gesture's resolver (agreed 2026-09-01 on the ch04 confirm
|
|
246
315
|
- `strata_index` _function_ — Segment-keyed view of the live strata (the consumer query's index).
|
|
247
316
|
- `validate_proposal_rows` _function_ — Validate + normalize proposer rows against their pack — loud on the first
|
|
248
317
|
- `write_filter_propset` _function_ — Write one filtering proposal set: `<out_root>/<set_id>/manifest.json` +
|
|
@@ -250,4 +319,4 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
250
319
|
## Dependencies
|
|
251
320
|
|
|
252
321
|
**Depends on:** `cjm-capability-primitives`, `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-substrate`, `cjm-transcript-graph-schema`, `numpy`
|
|
253
|
-
**Used by:** `cjm-
|
|
322
|
+
**Used by:** `cjm-context-graph-projection`, `cjm-transcript-correction-qt`, `cjm-transcription-core`, `cjm-workflow-hub-qt`
|
|
@@ -7,21 +7,34 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
7
7
|
## Modules
|
|
8
8
|
|
|
9
9
|
- **`cjm_transcript_correction_core.__init__`**
|
|
10
|
+
- **`cjm_transcript_correction_core.cleanread`** — The clean read — L1 of the reading ladder (design 6752db0a; the subtraction
|
|
10
11
|
- **`cjm_transcript_correction_core.cli`** — The CLI driver — the correction core's first (and currently only) frontend. run <decomp-manifest> corrects the committed spine in the decomp graph DB, pointing the graph worker at that shared DB via load-time config, with optional session resume/reopen; review runs the interactive text-correction loop (the cross-transcriber diff is intra-graph since stage 5).
|
|
11
12
|
- **`cjm_transcript_correction_core.graph`** — The correction overlay's graph I/O: targeted (scale-shaped) reads of a committed spine via the graph-storage query action, construction of Correction / CorrectionSession nodes + CORRECTS / SUPERSEDES / DERIVED_FROM / REVIEWED edges, the in-core effective-spine projection (layer-0 + applied corrections), and commit through the job queue. Hand-rolled (revolution-1) = direct CR-18 spec material; append-only on layer-0 (never update/delete a Segment).
|
|
12
13
|
- **`cjm_transcript_correction_core.journal`** — Live append-through for the correction verbs — the workflow journal's domain half.
|
|
13
14
|
- **`cjm_transcript_correction_core.launch`** — The shared launch surface every correction shell drives through: the
|
|
14
15
|
- **`cjm_transcript_correction_core.models`** — Overlay data shapes for the transcript-correction workflow: the Correction / CorrectionSession graph nodes + their relation registry, the read view of a committed spine segment, the worklist item, run configuration, and the correction run manifest (proto-bundle that chains decomp -> correction).
|
|
15
16
|
- **`cjm_transcript_correction_core.pipeline`** — The headless correction workflow: load a decomp run manifest, resolve the shared graph DB, start/resume/reopen a CorrectionSession, recompute the worklist from deterministic signals + persisted review state, run the D14 empty-segment prune (first operation), and record a chainable correction run manifest — with a cheapest-form HITL approval seam.
|
|
17
|
+
- **`cjm_transcript_correction_core.respine_transfer`** — The chunk-scoped transfer a chunk respine calls (work item 7a5e9c84; ruling 0b4d5cfa (5),
|
|
16
18
|
- **`cjm_transcript_correction_core.signals`** — Pure deterministic Tier-1 signal functions (no capability calls): empty-segment detection, bidirectional boundary punctuation/capitalization heuristics, forced-alignment coverage flags, positional cross-transcriber diff, phonetic + edit-distance variant clustering, and the event-proposal overlay (leg 4: the finetuned detector's spans anchored onto the spine). The worklist is recomputed from these each session; revolution-1 builds ZERO new capabilities.
|
|
19
|
+
- **`cjm_transcript_correction_core.spans`** — Spans — the overlay-proposal lane's domain half (design bbf8bafd, extending the
|
|
17
20
|
- **`cjm_transcript_correction_core.spine`**
|
|
18
21
|
- **`cjm_transcript_correction_core.state`** — Sidecar view-state + spine-picker helpers — the correction TUI's pure
|
|
19
22
|
- **`cjm_transcript_correction_core.strata`** — Strata — the filtering lane's domain half (DECs 304fd984 + 9d4c0a38; work items
|
|
20
23
|
|
|
21
24
|
## API
|
|
22
25
|
|
|
26
|
+
### `cjm_transcript_correction_core.cleanread`
|
|
27
|
+
|
|
28
|
+
- `clean_read` _function_ — L1 — the clean read (pure). Kept lines in spine order; each carries its
|
|
29
|
+
- `clean_read_pack_read` _function_ — The `read` record an L1 pack carries: the layer, the marker, the
|
|
30
|
+
- `clean_read_segments` _function_ — Project the clean read back onto SpineSegments (clean text, same
|
|
31
|
+
- `clean_read_summary` _function_
|
|
32
|
+
- `repeat_survivor` _function_ — Locate the last repeating unit of a word-repeat span (pure): the
|
|
33
|
+
- `subtract_spans` _function_ — Cut the ranges out of the text, leaving `marker` at each seam (pure).
|
|
34
|
+
|
|
23
35
|
### `cjm_transcript_correction_core.cli`
|
|
24
36
|
|
|
37
|
+
- `attention_command` _function_ — Execute `attention`: the walk-lane ATTENTION TIER (item 3758f6cb, ruling f400d2c3)
|
|
25
38
|
- `bench_command` _function_ — Execute the `bench` subcommand: the reserved-tail verdict join.
|
|
26
39
|
- `build_parser` _function_ — Build the CLI parser (subcommands: run, review).
|
|
27
40
|
- `commit_wordless_transfer` _function_ — COMMIT a planned transfer — the engine's second half. One
|
|
@@ -29,6 +42,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
29
42
|
- `extract_command` _function_ — Execute the `extract` subcommand: fold the gated overlay into a manifested dataset.
|
|
30
43
|
- `filter_confirm_command` _function_ — Execute `filter-confirm`: the HEADLESS HITL worklist (bc8dbbdd pass-1
|
|
31
44
|
- `filter_ingest_command` _function_ — Execute `filter-ingest`: validate proposer rows against their pack and
|
|
45
|
+
- `filter_merge_command` _function_ — Execute `filter-merge`: fold several proposal sets over one spine into
|
|
32
46
|
- `filter_pack_command` _function_ — Execute `filter-pack`: write one spine window's effective text-bearing
|
|
33
47
|
- `gate_command` _function_ — Execute the `gate` subcommand: show or assert per-spine extraction gates.
|
|
34
48
|
- `load_capabilities` _function_ — Discover manifests + load each capability, passing per-capability config (CR-2 caller-wins).
|
|
@@ -42,6 +56,11 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
42
56
|
- `run_command` _function_ — Execute the `run` subcommand: correct a decomp manifest's committed spine.
|
|
43
57
|
- `run_extract` _function_ — The extract fold on an ALREADY-OPEN graph seat (flywheel build leg 2,
|
|
44
58
|
- `scan_command` _function_ — Execute `scan-mishomed`: flag authoritative FA words stranded outside
|
|
59
|
+
- `span_confirm_command` _function_ — Execute `span-confirm`: the HEADLESS HITL worklist over one span
|
|
60
|
+
- `span_ingest_command` _function_ — Execute `span-ingest`: resolve a proposer's quoted-word rows against
|
|
61
|
+
- `span_lexicon_command` _function_ — Execute `span-lexicon`: the pattern tier (design bbf8bafd (g)) — bare
|
|
62
|
+
- `span_merge_command` _function_ — Execute `span-merge`: fold several span sets over one spine into ONE
|
|
63
|
+
- `span_pack_command` _function_ — Execute `span-pack`: the filter pack's lines as a SPEECH-OVERLAY pack
|
|
45
64
|
- `stats_command` _function_ — Execute the `stats` subcommand: flywheel accounting over the shared graph.
|
|
46
65
|
- `transfer_command` _function_ — Execute `transfer-wordless`: replay wordless event inserts (and speaker-
|
|
47
66
|
- `wordless_donors` _function_ — The EFFECTIVE wordless layer of a spine: labeled, effectively wordless
|
|
@@ -53,6 +72,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
53
72
|
- `active_speaker_assignments` _function_ — Project the ACTIVE speaker assignment per segment (latest-wins).
|
|
54
73
|
- `active_speech_overlays` _function_ — The surviving speech-overlay corrections (supersession applied; pure).
|
|
55
74
|
- `aggregate_session_purposes` _function_ — Fold sessions into a per-source purpose mix (pure; d915d545 picker rung).
|
|
75
|
+
- `annotate_retired` _function_ — Mark spine rows with their retirement state (pure).
|
|
56
76
|
- `apply_chunk_inserts` _function_ — Synthesize inserted chunks into the effective spine (DEC 3d3fa2a8).
|
|
57
77
|
- `apply_time_nudges` _function_ — Apply timing corrections onto segment times (latest-wins per edge).
|
|
58
78
|
- `bench_event_proposals` _function_ — The reserved-tail bench join (leg 4, DECs 8e05b87b + 8cf12c22) — pure.
|
|
@@ -91,6 +111,8 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
91
111
|
- `correction_stats` _function_ — Fold one Source's ACTIVE overlay into flywheel-accounting counts (pure).
|
|
92
112
|
- `corrections_to_edits` _function_ — Map this core's Correction payloads onto the layer's spine-edit vocabulary.
|
|
93
113
|
- `count_source_segments` _function_ — Count a Source's segments server-side under its chosen rendition + skeleton (typed count mode).
|
|
114
|
+
- `default_spine` _function_ — Which spine opens by default (pure; ruling a7617bd4 rule (c)): preference is a
|
|
115
|
+
- `discharge_mark` _function_ — Discharge an open mark BY THE CORRECTION THAT ADDRESSED IT — the closure
|
|
94
116
|
- `extract_spine_dataset` _function_ — Fold ONE spine's overlay into its v1 insert-span dataset slice (pure).
|
|
95
117
|
- `fa_words_for_transcript` _function_ — One transcript's FA words in source coordinates (the scan-mishomed join,
|
|
96
118
|
- `find_active_text_correction` _function_ — Single-segment convenience over the batch read (cross-session; latest wins).
|
|
@@ -123,6 +145,7 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
123
145
|
- `set_session_status` _function_ — Update a session's status + updated_at.
|
|
124
146
|
- `skeleton_hash_for` _function_ — Resolve a skeleton selector to the chosen spine's HASH (pure) — the gate's
|
|
125
147
|
- `source_audio_segment_ids` _function_ — The Source's coarse spine (one small typed read; ordered by index).
|
|
148
|
+
- `speaker_assignment_sources` _function_ — Where each speaker Entity has been assigned — the picker's COLLECTION
|
|
126
149
|
- `speech_overlay_spans` _function_ — Fold the ACTIVE speech overlays into extraction-facing span records (pure).
|
|
127
150
|
- `spine_where_for` _function_ — Resolve a skeleton selector against the observed spine set (pure).
|
|
128
151
|
- `start_session` _function_ — Create + commit a new CorrectionSession node.
|
|
@@ -166,8 +189,22 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
166
189
|
- `run_correction` _function_ — Correct every source in a decomp run manifest (prune + worklist surfacing).
|
|
167
190
|
- `run_review` _function_ — Interactive review pass over a decomp manifest's flagged worklist (text corrections).
|
|
168
191
|
|
|
192
|
+
### `cjm_transcript_correction_core.respine_transfer`
|
|
193
|
+
|
|
194
|
+
- `chunk_respine_transfer` _function_ — The registered chunk-scoped transfer (the entry point decomp-core discovers).
|
|
195
|
+
- `group_speaker_rows` _function_ — Fold per-segment rows into turn-shaped commits (pure): contiguous same-entity
|
|
196
|
+
- `live_neighbour` _function_ — The live segment right before the chunk — the flank a donor that starts before
|
|
197
|
+
- `load_segments_by_ids` _function_ — Segments by id — the ONE read that bypasses the live predicate on purpose: the
|
|
198
|
+
- `plan_speaker_carry` _function_ — Which new segment takes which speaker (pure; 4a7ec4f8 (2)/(3)): a segment whose
|
|
199
|
+
- `speaker_spans` _function_ — The per-span speaker map the carry reads (pure).
|
|
200
|
+
|
|
169
201
|
### `cjm_transcript_correction_core.signals`
|
|
170
202
|
|
|
203
|
+
- `attention_boundary_marks` _function_ — The boundary signals of the attention tier (item 3758f6cb signals 1+3 and the folded
|
|
204
|
+
- `attention_divergence_marks` _function_ — The two-transcriber disagreement signal of the attention tier (item 3758f6cb signal
|
|
205
|
+
- `attention_fa_marks` _function_ — The forced-alignment signals of the attention tier (item 3758f6cb signal 1), pure.
|
|
206
|
+
- `attention_marks` _function_ — Compose the attention tier (item 3758f6cb): every enabled signal's mark rows,
|
|
207
|
+
- `attention_speaker_marks` _function_ — The next-speaker-change landmark of the folded item 65791933, pure: `speaker-change`
|
|
171
208
|
- `boundary_punct_caps_flags` _function_ — Bidirectional boundary punctuation/capitalization heuristics (in-segment only).
|
|
172
209
|
- `cluster_variants` _function_ — Cluster word variants by phonetic key + edit distance (fix-one-fix-all).
|
|
173
210
|
- `compute_signal_flags` _function_ — Combine all deterministic Tier-1 signals into per-segment flags.
|
|
@@ -180,14 +217,36 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
180
217
|
- `speaker_turn_proposals` _function_ — Dominant diarization cluster per segment — the assign lane's proposal paint.
|
|
181
218
|
- `variant_divergence` _function_ — Within-segment cross-transcriber divergence (stage 5: intra-graph).
|
|
182
219
|
|
|
220
|
+
### `cjm_transcript_correction_core.spans`
|
|
221
|
+
|
|
222
|
+
- `bench_span_proposals` _function_ — Derive the span verdicts (design bbf8bafd (f), the bench_filter_proposals
|
|
223
|
+
- `build_span_pack` _function_ — Build the span proposer's input: the filter pack's numbered lines with
|
|
224
|
+
- `find_token_span` _function_ — Resolve quoted words to WHOLE word tokens of a line (pure): the quote is
|
|
225
|
+
- `is_span_pack` _function_
|
|
226
|
+
- `lexicon_span_rows` _function_ — The LEXICON TIER (design bbf8bafd (g)): a bare um / uh token is a
|
|
227
|
+
- `load_span_proposal_sets` _function_ — Every SPAN proposal set for a source (and optionally one spine), newest
|
|
228
|
+
- `locate_span_tokens` _function_ — Where a proposal's words sit on the CURRENT line, as a character range
|
|
229
|
+
- `merge_span_proposals` _function_ — Fold several SPAN sets over one spine into one walkable set — the
|
|
230
|
+
- `overlay_label_slate` _function_ — The slate a span pack carries: every label with its gloss and whether a
|
|
231
|
+
- `pending_span_proposals` _function_ — The headless worklist: proposals with NO active overlay carrying their id
|
|
232
|
+
- `render_span_pack` _function_ — Render a span pack as the brief a proposer reads: identity + window, the
|
|
233
|
+
- `render_span_propset_markdown` _function_ — Project a span set for a human: one line per proposal — tier, label,
|
|
234
|
+
- `segment_word_tokens` _function_
|
|
235
|
+
- `snap_span_proposal` _function_ — Resolve a proposal against the CURRENT line at accept (design bbf8bafd
|
|
236
|
+
- `span_proposals_from_rows` _function_ — Resolve validated rows to span proposal-set rows: proposal id, label
|
|
237
|
+
- `validate_span_rows` _function_ — Validate + resolve proposer rows against their span pack — loud on the
|
|
238
|
+
- `write_span_propset` _function_ — Write one SPAN proposal set — the filter set's layout under the span
|
|
239
|
+
|
|
183
240
|
### `cjm_transcript_correction_core.spine`
|
|
184
241
|
|
|
185
242
|
- `ChunkRef` _class_ — Where one Segment's VAD-chunk audio lives: the model-input WAV + the chunk-local span.
|
|
186
243
|
- `SeamRef` _class_ — A source-coordinate audio span across one fine-spine boundary (the g/G
|
|
187
244
|
- `SpineView` _class_ — One Source's effective correction spine, cursor-windowed for the TUI.
|
|
245
|
+
- `layer_speaker_menu` _function_ — Layer the assign-lane digit menu (pure; DEC 774dbe40). Tier 1 = the
|
|
188
246
|
- `list_sources` _function_ — Enumerate the graph's Source nodes (the discovery corpus, 2ce81638).
|
|
189
247
|
- `load_source_slice` _function_ — Decode a source-coordinate slice of the ORIGINAL media to playable samples.
|
|
190
248
|
- `match_sources` _function_ — The --source selector (pure; shared by direct open and the picker's seed).
|
|
249
|
+
- `match_speaker_query` _function_ — Narrow the registry by a typed query (pure; DEC 774dbe40 (4)). Rank 0 =
|
|
191
250
|
- `neighbor_word_bound` _function_ — The adjacent word's FA boundary facing an overlay span (pure).
|
|
192
251
|
- `open_stack` _function_ — Bootstrap the graph capability stack, resolving the db path (2ce81638).
|
|
193
252
|
- `parse_entity_input` _function_ — Parse the new-speaker editor line (pure). A leading `?` marks the entity
|
|
@@ -218,11 +277,18 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
218
277
|
- `build_filter_pack` _function_ — Build the proposer's input: one source window's text-bearing effective
|
|
219
278
|
- `exclude_strata` _function_ — The per-consumer filtered projection (DEC 9d4c0a38): "filtered" is a
|
|
220
279
|
- `load_filter_proposal_sets` _function_ — Every filtering proposal set for a source (and optionally one spine),
|
|
280
|
+
- `materialized_fix_ids` _function_ — The fidelity-edit apply path's materialization: a proposer row whose
|
|
281
|
+
- `materialized_mark_ids` _function_ — Class-family routing's other half: a proposer's mark-family row (an ASR
|
|
282
|
+
- `merge_filter_proposals` _function_ — Fold several proposal sets over ONE spine into one walkable set (design
|
|
221
283
|
- `new_pack_id` _function_ — Generate a unique, sortable pack id.
|
|
222
284
|
- `pack_digest` _function_ — Digest the READ content (source binding + window + numbered segments) —
|
|
223
285
|
- `pending_filter_proposals` _function_ — The headless worklist: proposals with NO live stratum carrying their id and
|
|
286
|
+
- `plan_pack_windows` _function_ — Cut a spine into `count` windows of near-equal TEXT-segment count at
|
|
224
287
|
- `proposals_from_rows` _function_ — Resolve validated rows to proposal-set rows: proposal id, category, source
|
|
225
288
|
- `render_filter_pack` _function_ — Render a pack as the brief a proposer reads: identity + window, the class
|
|
289
|
+
- `render_filter_propset_markdown` _function_ — Project a proposal set for a HUMAN to check against the source in the
|
|
290
|
+
- `resolve_slate` _function_ — Resolve a NAMED class slate (ruling c5b6cf42) to the two vocabularies a
|
|
291
|
+
- `select_span_segments` _function_ — The span-EDIT gesture's resolver (agreed 2026-09-01 on the ch04 confirm
|
|
226
292
|
- `strata_index` _function_ — Segment-keyed view of the live strata (the consumer query's index).
|
|
227
293
|
- `validate_proposal_rows` _function_ — Validate + normalize proposer rows against their pack — loud on the first
|
|
228
294
|
- `write_filter_propset` _function_ — Write one filtering proposal set: `<out_root>/<set_id>/manifest.json` +
|
|
@@ -230,4 +296,4 @@ A frontend-agnostic core for the transcript correction workflow — the first do
|
|
|
230
296
|
## Dependencies
|
|
231
297
|
|
|
232
298
|
**Depends on:** `cjm-capability-primitives`, `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-substrate`, `cjm-transcript-graph-schema`, `numpy`
|
|
233
|
-
**Used by:** `cjm-
|
|
299
|
+
**Used by:** `cjm-context-graph-projection`, `cjm-transcript-correction-qt`, `cjm-transcription-core`, `cjm-workflow-hub-qt`
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.30"
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""The clean read — L1 of the reading ladder (design 6752db0a; the subtraction
|
|
2
|
+
shape from bbf8bafd (a)).
|
|
3
|
+
|
|
4
|
+
L0 is the spine as spoken. L1 is the same spine MINUS what a reader never
|
|
5
|
+
needs, as a MECHANICAL projection with VISIBLE elisions — it subtracts twice:
|
|
6
|
+
|
|
7
|
+
1. FILLER LINES first: every text segment under an excluded stratum (default
|
|
8
|
+
`filler` — a wholly elidable line or run; the notes types' exclude_strata
|
|
9
|
+
name more) drops as a line;
|
|
10
|
+
2. then the FILTERABLE OVERLAY SPANS inside the lines that remain — the
|
|
11
|
+
annotate lane's speech overlays whose label a reader filters (hesitation-
|
|
12
|
+
marker / discourse-marker / word-repeat / false-start); the KEEP labels
|
|
13
|
+
(emphasis-repeat / coincidental-repeat) never subtract. A line emptied by
|
|
14
|
+
its overlays drops too.
|
|
15
|
+
|
|
16
|
+
The projection is pure over the effective spine + the active strata + the
|
|
17
|
+
active overlays; nothing is stored. Every elision is visible: a subtracted
|
|
18
|
+
span leaves `marker` in the line (default `[…]`; "" for a clean-verbatim
|
|
19
|
+
consumer), a dropped line is recorded in the gap before the next kept line,
|
|
20
|
+
and each kept line carries what was cut. A wrong exclusion must never HIDE
|
|
21
|
+
content silently (ruling 6752db0a (9)) — the reader can always see that
|
|
22
|
+
something was elided and audit it against L0.
|
|
23
|
+
|
|
24
|
+
WORD-REPEAT KEEPS ITS LAST UNIT: the overlay quotes the whole run ('the the',
|
|
25
|
+
'we, we', 'and and and') because the run is the detector sample; the clean
|
|
26
|
+
read keeps the speaker's final instance — the span's tokens are split into
|
|
27
|
+
the shortest repeating unit and only the last unit survives; a run that is
|
|
28
|
+
not a clean repetition keeps its last token (a mechanical rule, auditable
|
|
29
|
+
through the marker; refine per label as drives show).
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
import re
|
|
33
|
+
from dataclasses import replace
|
|
34
|
+
from typing import Any, Dict, List, Sequence, Tuple
|
|
35
|
+
|
|
36
|
+
from cjm_transcript_correction_core.graph import reanchor_span
|
|
37
|
+
from cjm_transcript_correction_core.models import SpineSegment
|
|
38
|
+
from cjm_transcript_correction_core.strata import strata_index
|
|
39
|
+
|
|
40
|
+
CLEAN_READ_EXCLUDE_STRATA = ("filler",) # The default line-level subtraction (the notes types name more)
|
|
41
|
+
CLEAN_READ_FILTER_LABELS = ("hesitation-marker", "discourse-marker", "word-repeat", "false-start")
|
|
42
|
+
CLEAN_READ_KEEP_LAST_UNIT = ("word-repeat",) # Labels whose span keeps its final repeated unit
|
|
43
|
+
ELISION_MARKER = "[…]"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _norm(t: str) -> str: # Case/punctuation-insensitive token form (the span lane's rule)
|
|
47
|
+
return "".join(ch for ch in t.lower() if ch.isalnum())
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def repeat_survivor(
|
|
51
|
+
span_text: str, # The overlay's words (a whole repeated run)
|
|
52
|
+
) -> Tuple[int, int]: # (char offset, char end) INSIDE span_text of the surviving last unit
|
|
53
|
+
"""Locate the last repeating unit of a word-repeat span (pure): the
|
|
54
|
+
shortest unit u with the span = u × n (normalized) wins; no clean
|
|
55
|
+
repetition = the last token. Returned offsets are relative to the span."""
|
|
56
|
+
toks = [(m.start(), m.end(), m.group()) for m in re.finditer(r"\S+", span_text or "")]
|
|
57
|
+
if not toks:
|
|
58
|
+
return 0, len(span_text or "")
|
|
59
|
+
norm = [_norm(t) for _, _, t in toks]
|
|
60
|
+
k = len(norm)
|
|
61
|
+
for unit in range(1, k // 2 + 1):
|
|
62
|
+
if k % unit == 0 and all(norm[i:i + unit] == norm[:unit] for i in range(0, k, unit)):
|
|
63
|
+
return toks[k - unit][0], toks[-1][1]
|
|
64
|
+
return toks[-1][0], toks[-1][1]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _cut_ranges(
|
|
68
|
+
text: str, # The line's effective text
|
|
69
|
+
overlays: List[Dict[str, Any]], # Active overlay correction dicts anchored on this line
|
|
70
|
+
labels: Sequence[str], # Filterable labels
|
|
71
|
+
keep_last: Sequence[str], # Labels that keep their last repeated unit
|
|
72
|
+
) -> Tuple[List[Tuple[int, int]], List[Dict[str, Any]]]: # (merged char ranges to cut, the cuts made)
|
|
73
|
+
cuts: List[Tuple[int, int]] = []
|
|
74
|
+
made: List[Dict[str, Any]] = []
|
|
75
|
+
for c in overlays:
|
|
76
|
+
p = c.get("payload") or {}
|
|
77
|
+
lab = p.get("label")
|
|
78
|
+
if lab not in labels:
|
|
79
|
+
continue
|
|
80
|
+
a = p.get("anchor") or {}
|
|
81
|
+
located = reanchor_span(a, text)
|
|
82
|
+
if located is None:
|
|
83
|
+
made.append({"overlay_id": c.get("id"), "label": lab, "text": a.get("text_snapshot"),
|
|
84
|
+
"cut": None, "note": "words no longer on the line"})
|
|
85
|
+
continue
|
|
86
|
+
cs, ce = located
|
|
87
|
+
if lab in keep_last:
|
|
88
|
+
ks, ke = repeat_survivor(text[cs:ce])
|
|
89
|
+
rng = (cs, cs + ks) # everything before the surviving unit
|
|
90
|
+
tail = (cs + ke, ce) # trailing punctuation after it, if any
|
|
91
|
+
cuts.append(rng)
|
|
92
|
+
if tail[1] > tail[0]:
|
|
93
|
+
cuts.append(tail)
|
|
94
|
+
made.append({"overlay_id": c.get("id"), "label": lab, "text": text[cs:ce],
|
|
95
|
+
"cut": text[cs:cs + ks], "kept": text[cs + ks:cs + ke]})
|
|
96
|
+
else:
|
|
97
|
+
cuts.append((cs, ce))
|
|
98
|
+
made.append({"overlay_id": c.get("id"), "label": lab, "text": text[cs:ce],
|
|
99
|
+
"cut": text[cs:ce]})
|
|
100
|
+
cuts.sort()
|
|
101
|
+
merged: List[Tuple[int, int]] = []
|
|
102
|
+
for s, e in cuts:
|
|
103
|
+
if e <= s:
|
|
104
|
+
continue
|
|
105
|
+
if merged and s <= merged[-1][1]:
|
|
106
|
+
merged[-1] = (merged[-1][0], max(merged[-1][1], e))
|
|
107
|
+
else:
|
|
108
|
+
merged.append((s, e))
|
|
109
|
+
return merged, made
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _tidy(s: str) -> str: # Whitespace + punctuation seams after a cut
|
|
113
|
+
s = re.sub(r"\s+", " ", s).strip()
|
|
114
|
+
# ", ," -> "," — a cut's seam always leaves WHITESPACE between the two marks, so the rule
|
|
115
|
+
# demands it and runs BEFORE the space-before-punctuation rule closes that gap: a `::`
|
|
116
|
+
# the source wrote (thrust::reduce) is adjacent and never a seam (finding: the clean
|
|
117
|
+
# read handed a proposer `thrust:reduce`).
|
|
118
|
+
s = re.sub(r"([,;:])(?:\s+[,;:])+", r"\1", s)
|
|
119
|
+
s = re.sub(r"\s+([,.;:?!])", r"\1", s) # "larger, than" — a space before punctuation
|
|
120
|
+
s = re.sub(r"^[,;:](?![,;:])\s*", "", s) # a leading comma left by a cut at the line start (never a leading `::`)
|
|
121
|
+
return s
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def subtract_spans(
|
|
125
|
+
text: str, # The line's effective text
|
|
126
|
+
ranges: List[Tuple[int, int]], # Merged char ranges to cut (from _cut_ranges)
|
|
127
|
+
marker: str = ELISION_MARKER, # What a cut leaves behind ("" = nothing)
|
|
128
|
+
) -> str: # The clean text (tidied)
|
|
129
|
+
"""Cut the ranges out of the text, leaving `marker` at each seam (pure)."""
|
|
130
|
+
if not ranges:
|
|
131
|
+
return text
|
|
132
|
+
out: List[str] = []
|
|
133
|
+
at = 0
|
|
134
|
+
for s, e in ranges:
|
|
135
|
+
out.append(text[at:s])
|
|
136
|
+
if marker:
|
|
137
|
+
out.append(f" {marker} ")
|
|
138
|
+
at = e
|
|
139
|
+
out.append(text[at:])
|
|
140
|
+
joined = "".join(out)
|
|
141
|
+
if marker: # never let the marker glue to punctuation-less neighbours: "we [...] have"
|
|
142
|
+
joined = re.sub(r"\s*" + re.escape(marker) + r"\s*", f" {marker} ", joined)
|
|
143
|
+
return _tidy(joined)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def clean_read(
|
|
147
|
+
segments: Sequence[SpineSegment], # The EFFECTIVE spine (corrections applied), index order
|
|
148
|
+
strata: List[Dict[str, Any]], # active_strata output
|
|
149
|
+
overlays: List[Dict[str, Any]], # active_speech_overlays output
|
|
150
|
+
*,
|
|
151
|
+
exclude_strata: Sequence[str] = CLEAN_READ_EXCLUDE_STRATA, # Line-level subtraction classes
|
|
152
|
+
filter_labels: Sequence[str] = CLEAN_READ_FILTER_LABELS, # Overlay labels a reader filters
|
|
153
|
+
keep_last_unit: Sequence[str] = CLEAN_READ_KEEP_LAST_UNIT, # Labels whose span keeps its last unit
|
|
154
|
+
marker: str = ELISION_MARKER, # In-line elision marker ("" = clean-verbatim)
|
|
155
|
+
) -> List[Dict[str, Any]]: # Kept lines: {id, index, start, end, text, raw_text, elisions, dropped_before}
|
|
156
|
+
"""L1 — the clean read (pure). Kept lines in spine order; each carries its
|
|
157
|
+
clean `text`, the L0 `raw_text`, the `elisions` cut inside it, and
|
|
158
|
+
`dropped_before`: the lines elided between the previous kept line and
|
|
159
|
+
this one (segment id, index, why). Wordless segments are not lines and
|
|
160
|
+
are never counted as dropped."""
|
|
161
|
+
idx = strata_index(strata)
|
|
162
|
+
drop = set(exclude_strata)
|
|
163
|
+
by_seg: Dict[str, List[Dict[str, Any]]] = {}
|
|
164
|
+
for c in overlays:
|
|
165
|
+
sid = ((c.get("payload") or {}).get("anchor") or {}).get("segment_id")
|
|
166
|
+
if sid:
|
|
167
|
+
by_seg.setdefault(sid, []).append(c)
|
|
168
|
+
out: List[Dict[str, Any]] = []
|
|
169
|
+
pending_drops: List[Dict[str, Any]] = []
|
|
170
|
+
for s in sorted(segments, key=lambda x: x.index):
|
|
171
|
+
if s.is_empty:
|
|
172
|
+
continue
|
|
173
|
+
cats = drop & set(idx.get(s.id, []))
|
|
174
|
+
if cats:
|
|
175
|
+
pending_drops.append({"id": s.id, "index": s.index, "why": sorted(cats)})
|
|
176
|
+
continue
|
|
177
|
+
text = s.text or ""
|
|
178
|
+
ranges, made = _cut_ranges(text, by_seg.get(s.id, []), filter_labels, keep_last_unit)
|
|
179
|
+
clean = subtract_spans(text, ranges, marker) if ranges else text
|
|
180
|
+
bare = clean.replace(marker, "").strip() if marker else clean.strip()
|
|
181
|
+
if not bare:
|
|
182
|
+
pending_drops.append({"id": s.id, "index": s.index,
|
|
183
|
+
"why": ["emptied by " + ", ".join(sorted({m["label"] for m in made}))]})
|
|
184
|
+
continue
|
|
185
|
+
out.append({"id": s.id, "index": s.index,
|
|
186
|
+
"start": float(s.start_time) if s.start_time is not None else None,
|
|
187
|
+
"end": float(s.end_time) if s.end_time is not None else None,
|
|
188
|
+
"text": clean, "raw_text": text, "elisions": made,
|
|
189
|
+
"dropped_before": pending_drops})
|
|
190
|
+
pending_drops = []
|
|
191
|
+
if pending_drops and out: # a trailing run of dropped lines hangs off the last kept line
|
|
192
|
+
out[-1]["dropped_after"] = pending_drops
|
|
193
|
+
return out
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def clean_read_summary(lines: List[Dict[str, Any]]) -> Dict[str, int]: # Counts for a pack / a report
|
|
197
|
+
dropped = sum(len(ln.get("dropped_before") or []) + len(ln.get("dropped_after") or []) for ln in lines)
|
|
198
|
+
cut = sum(1 for ln in lines for m in (ln.get("elisions") or []) if m.get("cut") is not None)
|
|
199
|
+
return {"lines_kept": len(lines), "lines_dropped": dropped, "spans_cut": cut}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def clean_read_pack_read(
|
|
203
|
+
lines: List[Dict[str, Any]], # clean_read output
|
|
204
|
+
marker: str = ELISION_MARKER, # The marker the lines carry
|
|
205
|
+
) -> Dict[str, Any]: # build_filter_pack's `read` argument for an L1 pack
|
|
206
|
+
"""The `read` record an L1 pack carries: the layer, the marker, the
|
|
207
|
+
counts, and the gaps (segment id -> the lines dropped before it) the
|
|
208
|
+
pack builder attaches to its rows so the brief shows every elision."""
|
|
209
|
+
return {"layer": "clean", "marker": marker, **clean_read_summary(lines),
|
|
210
|
+
"gaps": {ln["id"]: ln["dropped_before"] for ln in lines if ln.get("dropped_before")}}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def clean_read_segments(
|
|
214
|
+
lines: List[Dict[str, Any]], # clean_read output
|
|
215
|
+
segments: Sequence[SpineSegment], # The same effective spine (for the fields a line does not carry)
|
|
216
|
+
) -> List[SpineSegment]: # SpineSegments carrying the CLEAN text — what a pack builder reads as L1
|
|
217
|
+
"""Project the clean read back onto SpineSegments (clean text, same
|
|
218
|
+
identity and times), so every pack builder reads L1 unchanged."""
|
|
219
|
+
by_id = {s.id: s for s in segments}
|
|
220
|
+
return [replace(by_id[ln["id"]], text=ln["text"]) for ln in lines]
|