cjm-transcript-decomp-core 0.0.6__tar.gz → 0.0.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cjm_transcript_decomp_core-0.0.6/cjm_transcript_decomp_core.egg-info → cjm_transcript_decomp_core-0.0.9}/PKG-INFO +3 -2
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/README.md +1 -0
- cjm_transcript_decomp_core-0.0.9/cjm_transcript_decomp_core/__init__.py +1 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core/cli.py +13 -6
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core/launch.py +5 -1
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core/models.py +6 -2
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core/pipeline.py +89 -21
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core/runs.py +100 -6
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9/cjm_transcript_decomp_core.egg-info}/PKG-INFO +3 -2
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core.egg-info/requires.txt +1 -1
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/pyproject.toml +1 -1
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/tests/test_batch_logic.py +131 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/tests/test_models.py +7 -1
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/tests/test_pipeline.py +51 -0
- cjm_transcript_decomp_core-0.0.6/cjm_transcript_decomp_core/__init__.py +0 -1
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/LICENSE +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/MANIFEST.in +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core/alignment.py +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core/discovery.py +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core/graph.py +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core/segments.py +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core/state.py +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core.egg-info/SOURCES.txt +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core.egg-info/dependency_links.txt +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core.egg-info/entry_points.txt +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/cjm_transcript_decomp_core.egg-info/top_level.txt +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/setup.cfg +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/tests/test_alignment.py +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/tests/test_cli.py +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/tests/test_graph.py +0 -0
- {cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/tests/test_segment_logic.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cjm-transcript-decomp-core
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.9
|
|
4
4
|
Summary: A frontend-agnostic core for the transcript decomposition workflow — composes isolated capability workers (forced alignment, VAD, graph storage) into a headless pipeline that decomposes transcription run manifests into a VAD-aligned context-graph spine with traceable provenance, and a CLI as its first driver.
|
|
5
5
|
Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
10
10
|
Requires-Python: >=3.12
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
License-File: LICENSE
|
|
13
|
-
Requires-Dist: cjm_substrate>=0.0.
|
|
13
|
+
Requires-Dist: cjm_substrate>=0.0.63
|
|
14
14
|
Requires-Dist: pyyaml
|
|
15
15
|
Requires-Dist: cjm_context_graph_primitives>=0.0.15
|
|
16
16
|
Requires-Dist: cjm_capability_primitives>=0.0.11
|
|
@@ -102,6 +102,7 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
|
|
|
102
102
|
- `event_spans_from_propset` _function_ — Load a proposal set BY POINTER and select its carve spans (respine trial
|
|
103
103
|
- `fa_words_from_result` _function_ — Normalize a typed forced-alignment result into FA words (pure; stage 3).
|
|
104
104
|
- `load_source_manifest` _function_ — Load + lightly validate a transcription-core run manifest.
|
|
105
|
+
- `resolve_event_propsets` _function_ — Join each proposal set to ITS source by the set manifest's own source
|
|
105
106
|
- `run_decomp` _function_ — Extend every source in a transcription run manifest with its fine spine.
|
|
106
107
|
- `sentence_spans_from_result` _function_ — Normalize a typed sentence-segmentation result into char-span tuples (pure; B.5).
|
|
107
108
|
- `submit_and_wait` _function_ — Submit one capability job, wait for it, and return its result (raise on failure).
|
|
@@ -82,6 +82,7 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
|
|
|
82
82
|
- `event_spans_from_propset` _function_ — Load a proposal set BY POINTER and select its carve spans (respine trial
|
|
83
83
|
- `fa_words_from_result` _function_ — Normalize a typed forced-alignment result into FA words (pure; stage 3).
|
|
84
84
|
- `load_source_manifest` _function_ — Load + lightly validate a transcription-core run manifest.
|
|
85
|
+
- `resolve_event_propsets` _function_ — Join each proposal set to ITS source by the set manifest's own source
|
|
85
86
|
- `run_decomp` _function_ — Extend every source in a transcription run manifest with its fine spine.
|
|
86
87
|
- `sentence_spans_from_result` _function_ — Normalize a typed sentence-segmentation result into char-span tuples (pure; B.5).
|
|
87
88
|
- `submit_and_wait` _function_ — Submit one capability job, wait for it, and return its result (raise on failure).
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.9"
|
|
@@ -78,9 +78,12 @@ def build_parser() -> argparse.ArgumentParser: # Configured CLI parser
|
|
|
78
78
|
help="Run the post-FA event-carve stage (respine trial DEC 6cc10fb7): model "
|
|
79
79
|
"event spans from --event-propset become gaps between chunks "
|
|
80
80
|
"(cut-don't-label); typically paired with --no-sentence-split")
|
|
81
|
-
run.add_argument("--event-propset", default=None,
|
|
82
|
-
help="ProposalSetManifest pointer (manifest json or its set dir); "
|
|
83
|
-
"REQUIRED with --event-split"
|
|
81
|
+
run.add_argument("--event-propset", nargs="+", default=None, metavar="POINTER",
|
|
82
|
+
help="ProposalSetManifest pointer(s) (manifest json or its set dir); "
|
|
83
|
+
"REQUIRED with --event-split. ONE per source: each set carves "
|
|
84
|
+
"the source its own manifest names (content hash, else path) — "
|
|
85
|
+
"a multi-source run takes one pointer per source, any order; "
|
|
86
|
+
"an uncovered source or a stray pointer refuses loudly")
|
|
84
87
|
run.add_argument("--event-classes", nargs="+", default=["inhale"],
|
|
85
88
|
help="Proposal classes that carve (default: inhale; word-bearing "
|
|
86
89
|
"classes must never cut)")
|
|
@@ -168,12 +171,16 @@ async def run_command(
|
|
|
168
171
|
seg_capability=args.seg_capability,
|
|
169
172
|
respine=args.respine,
|
|
170
173
|
event_split=args.event_split,
|
|
171
|
-
|
|
174
|
+
# One pointer per source (0.2.6): the single form stays the config
|
|
175
|
+
# snapshot's `event_propset` so 0.2.5 consumers read it unchanged.
|
|
176
|
+
event_propset=(args.event_propset[0] if args.event_propset
|
|
177
|
+
and len(args.event_propset) == 1 else ""),
|
|
178
|
+
event_propsets=list(args.event_propset or []),
|
|
172
179
|
event_classes=list(args.event_classes),
|
|
173
180
|
word_rescue=args.word_rescue,
|
|
174
181
|
)
|
|
175
|
-
if cfg.event_split and not cfg.
|
|
176
|
-
raise SystemExit("error: --event-split requires --event-propset")
|
|
182
|
+
if cfg.event_split and not cfg.event_propsets:
|
|
183
|
+
raise SystemExit("error: --event-split requires --event-propset (one per source)")
|
|
177
184
|
|
|
178
185
|
# CR-7 GPU subtree attribution is opt-in: --sysmon-capability threads the monitor
|
|
179
186
|
# name into BOTH the manager and the queue; the monitor loads FIRST so GPU
|
|
@@ -158,8 +158,12 @@ def batch_argv(
|
|
|
158
158
|
# classes render explicitly — the printed argv is the full contract.
|
|
159
159
|
# The GROUP's resolved propset wins (per-source picker, DEC ae450551);
|
|
160
160
|
# the CLI flag is the scripted path's pin.
|
|
161
|
+
# A group's propset is ONE pointer per source (0.2.6): a str for a
|
|
162
|
+
# single-source group, a list/tuple for a multi-source run.
|
|
163
|
+
ps = batch.get("event_propset") or args.event_propset
|
|
164
|
+
pointers = [ps] if isinstance(ps, str) else list(ps or [])
|
|
161
165
|
argv += ["--event-split",
|
|
162
|
-
"--event-propset",
|
|
166
|
+
"--event-propset", *pointers,
|
|
163
167
|
"--event-classes", *args.event_classes]
|
|
164
168
|
if args.actor:
|
|
165
169
|
argv += ["--actor", args.actor]
|
|
@@ -122,7 +122,8 @@ class DecompConfig:
|
|
|
122
122
|
seg_capability: str = "cjm-capability-pysbd" # Sentence-segmentation capability id (B.5; loaded only when sentence_split)
|
|
123
123
|
respine: bool = False # Mint a FRESH skeleton under the SAME config (DEC 9241564f): the run id joins the identity input, so the new spine never collides with the config-identical one (post-upgrade re-runs, finding e8458f6e)
|
|
124
124
|
event_split: bool = False # Run the post-FA event-carve stage (respine trial DEC 6cc10fb7): model event spans from `event_propset` become gaps between chunks (cut-don't-label); composable with sentence_split, but the trial runs it INSTEAD
|
|
125
|
-
event_propset: str = "" # ProposalSetManifest pointer (manifest json or its set dir) — the model-cut authority consumed BY POINTER; REQUIRED when event_split
|
|
125
|
+
event_propset: str = "" # ProposalSetManifest pointer (manifest json or its set dir) — the model-cut authority consumed BY POINTER; the SINGLE-source form (kept for config-snapshot compatibility; REQUIRED when event_split and event_propsets is empty)
|
|
126
|
+
event_propsets: List[str] = field(default_factory=list) # Per-source ProposalSetManifest pointers (0.2.6, multi-source runs): each set joins ITS source by the manifest's own source binding (content hash, else path) — resolve_event_propsets refuses an uncovered source or a stray pointer loudly
|
|
126
127
|
event_classes: List[str] = field(default_factory=lambda: ["inhale"]) # Proposal classes that carve (word-bearing classes like hesitation-marker must never cut)
|
|
127
128
|
word_rescue: bool = True # Run the post-carve word-rescue stage (96edc646 verdict bc7ece7b): authoritative FA words stranded outside every chunk get chunks minted — DEFAULT-ON (mis-homing is silent data corruption; opting out hides it)
|
|
128
129
|
|
|
@@ -141,6 +142,9 @@ class DecompSourceRecord:
|
|
|
141
142
|
title: str # Source display title
|
|
142
143
|
segment_count: int # Number of fine Segment nodes committed
|
|
143
144
|
segment_ids: List[str] = field(default_factory=list) # Graph Segment node ids, in order
|
|
145
|
+
skeleton_config_hash: str = "" # THIS source's Segment identity input (0.2.6): equals the run-level value unless the event stage carved per source
|
|
146
|
+
event_propset_id: str = "" # The ProposalSetManifest THIS source's carve consumed (0.2.6; per-source propset -> skeleton chain join key)
|
|
147
|
+
event_propset: str = "" # Recorded pointer of that set (0.2.6)
|
|
144
148
|
|
|
145
149
|
def to_dict(self) -> Dict[str, Any]: # Plain-dict form
|
|
146
150
|
"""Serialize to a plain dict."""
|
|
@@ -169,7 +173,7 @@ class DecompManifest:
|
|
|
169
173
|
event_propset: str = "" # Recorded path of the consumed ProposalSetManifest (0.2.4; the pointer)
|
|
170
174
|
|
|
171
175
|
FORMAT: str = field(default="cjm-transcript-decomp-core/run-manifest", repr=False) # Format tag
|
|
172
|
-
VERSION: str = field(default="0.2.
|
|
176
|
+
VERSION: str = field(default="0.2.6", repr=False) # Schema version (0.2.6: per-source skeleton hash + consumed propset on each source record — multi-source event carve; the run-level fields hold the value only when every source shares it; 0.2.5: word-rescue policy recorded; 0.2.4: event-carve respine — propset pointer + id + event policy recorded; 0.2.3: capability-driven sentence split)
|
|
173
177
|
|
|
174
178
|
def to_dict(self) -> Dict[str, Any]: # Plain-dict form for JSON serialization
|
|
175
179
|
"""Serialize to a plain dict with nested sources."""
|
|
@@ -481,27 +481,47 @@ async def run_decomp(
|
|
|
481
481
|
# (the set IS the cut authority) and the pointer + id land in the manifest
|
|
482
482
|
# (propset -> skeleton chain, DEC 16159e09 manifest-kinds-chainable).
|
|
483
483
|
event_policy = EVENT_SPLIT_POLICY if cfg.event_split else None
|
|
484
|
-
|
|
484
|
+
# Per-source carve authority (manifest 0.2.6): every source resolves ITS
|
|
485
|
+
# set by the set manifest's own source binding (resolve_event_propsets —
|
|
486
|
+
# the single-pointer config form is the one-element list), and each
|
|
487
|
+
# source's set id joins THAT source's skeleton identity. The run-level
|
|
488
|
+
# fields hold a value only when every source shares it, so single-source
|
|
489
|
+
# runs are byte-identical to 0.2.5 and multi-source runs never pretend
|
|
490
|
+
# one authority carved them all.
|
|
491
|
+
run_sources = src.get("sources", []) or []
|
|
492
|
+
per_source_propsets: List[Tuple[Dict[str, Any], List[Tuple[float, float]], str]] = []
|
|
485
493
|
if cfg.event_split:
|
|
486
|
-
|
|
494
|
+
pointers = list(cfg.event_propsets) or ([cfg.event_propset] if cfg.event_propset else [])
|
|
495
|
+
if not pointers:
|
|
487
496
|
raise RuntimeError("event_split requires event_propset (the ProposalSetManifest pointer)")
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
manifest.
|
|
491
|
-
|
|
492
|
-
|
|
497
|
+
per_source_propsets = resolve_event_propsets(pointers, run_sources, cfg.event_classes)
|
|
498
|
+
set_ids = sorted({str(m.get("proposal_set_id") or "") for m, _, _ in per_source_propsets})
|
|
499
|
+
manifest.event_propset_id = set_ids[0] if len(set_ids) == 1 else ""
|
|
500
|
+
manifest.event_propset = per_source_propsets[0][2] if len(set_ids) == 1 else ""
|
|
501
|
+
for i, (m, spans, _ptr) in enumerate(per_source_propsets):
|
|
502
|
+
logger.info(f"run {run_id}: event-carve source {i} consuming "
|
|
503
|
+
f"{m.get('proposal_set_id')} ({len(spans)} span(s) across "
|
|
504
|
+
f"classes {cfg.event_classes})")
|
|
493
505
|
word_rescue_policy = WORD_RESCUE_POLICY if cfg.word_rescue else None
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
506
|
+
|
|
507
|
+
def _skeleton_hash(event_propset_id: str) -> str:
|
|
508
|
+
return compute_skeleton_hash(
|
|
509
|
+
vad_config_hash, split_policy=split_policy,
|
|
510
|
+
split_min_chunk_s=cfg.split_min_chunk_s,
|
|
511
|
+
seg_capability=cfg.seg_capability, seg_config_hash=seg_config_hash,
|
|
512
|
+
respine_token=run_id if cfg.respine else None,
|
|
513
|
+
event_policy=event_policy,
|
|
514
|
+
event_propset_id=(event_propset_id if cfg.event_split else ""),
|
|
515
|
+
event_classes=(cfg.event_classes if cfg.event_split else None),
|
|
516
|
+
text_from_capability=str(text_from or ""),
|
|
517
|
+
text_from_config_hash=str((src_capabilities.get(text_from) or {}).get("config_hash") or ""),
|
|
518
|
+
word_rescue_policy=word_rescue_policy)
|
|
519
|
+
source_skeletons = ([_skeleton_hash(str(m.get("proposal_set_id") or ""))
|
|
520
|
+
for m, _, _ in per_source_propsets] if cfg.event_split
|
|
521
|
+
else [_skeleton_hash("")] * len(run_sources))
|
|
522
|
+
distinct_skeletons = sorted(set(source_skeletons))
|
|
523
|
+
skeleton_config_hash = (distinct_skeletons[0] if len(distinct_skeletons) == 1
|
|
524
|
+
else "" if distinct_skeletons else _skeleton_hash(""))
|
|
505
525
|
manifest.skeleton_config_hash = skeleton_config_hash
|
|
506
526
|
manifest.split_policy = split_policy
|
|
507
527
|
manifest.event_split_policy = event_policy
|
|
@@ -526,7 +546,7 @@ async def run_decomp(
|
|
|
526
546
|
|
|
527
547
|
source_path, aligned = await decompose_source(
|
|
528
548
|
queue, cfg, source, i, transcribers, text_from,
|
|
529
|
-
event_spans=
|
|
549
|
+
event_spans=(per_source_propsets[i][1] if cfg.event_split else None))
|
|
530
550
|
title = Path(source_path).stem or f"source-{i}"
|
|
531
551
|
|
|
532
552
|
empty = sum(1 for a in aligned if not a.text.strip())
|
|
@@ -543,7 +563,7 @@ async def run_decomp(
|
|
|
543
563
|
policy_label = "+".join(
|
|
544
564
|
p for p in (split_policy, event_policy, word_rescue_policy) if p) or None
|
|
545
565
|
nodes, edges, ids = build_extension_payload(
|
|
546
|
-
source, src_capabilities,
|
|
566
|
+
source, src_capabilities, source_skeletons[i], text_from, aligned,
|
|
547
567
|
split_policy=policy_label)
|
|
548
568
|
|
|
549
569
|
if not confirm_seam("commit-review",
|
|
@@ -605,7 +625,11 @@ async def run_decomp(
|
|
|
605
625
|
|
|
606
626
|
manifest.sources.append(DecompSourceRecord(
|
|
607
627
|
source_node_id=ids["source"], source_path=source_path, title=title,
|
|
608
|
-
segment_count=len(ids["segments"]), segment_ids=ids["segments"]
|
|
628
|
+
segment_count=len(ids["segments"]), segment_ids=ids["segments"],
|
|
629
|
+
skeleton_config_hash=source_skeletons[i],
|
|
630
|
+
event_propset_id=(str(per_source_propsets[i][0].get("proposal_set_id") or "")
|
|
631
|
+
if cfg.event_split else ""),
|
|
632
|
+
event_propset=(per_source_propsets[i][2] if cfg.event_split else "")))
|
|
609
633
|
except BaseException as e:
|
|
610
634
|
# The journal exists for exactly this row: a run that DIED records
|
|
611
635
|
# how far it got (failures stop being the unattributed case).
|
|
@@ -749,3 +773,47 @@ def event_spans_from_propset(
|
|
|
749
773
|
spans.append((s, e))
|
|
750
774
|
spans.sort()
|
|
751
775
|
return manifest, spans
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def resolve_event_propsets(
|
|
779
|
+
pointers: List[Union[str, Path]], # ProposalSetManifest pointers (manifest json or set dir), any order
|
|
780
|
+
sources: List[Dict[str, Any]], # The transcription run's sources, manifest order
|
|
781
|
+
classes: List[str], # Proposal classes that carve
|
|
782
|
+
) -> List[Tuple[Dict[str, Any], List[Tuple[float, float]], str]]: # Per source, manifest order: (set manifest, carve spans, the pointer as given)
|
|
783
|
+
"""Join each proposal set to ITS source by the set manifest's own source
|
|
784
|
+
binding (multi-source event carve, manifest 0.2.6): a set matches a run
|
|
785
|
+
source when its source.content_hash equals the source's content_hash,
|
|
786
|
+
else when its source.path equals the source's source_path. One set
|
|
787
|
+
carves ONE source, so every refusal is LOUD: a source no pointer covers
|
|
788
|
+
(propose over it first), a pointer claiming none of the run's sources
|
|
789
|
+
(a wrong pick), two pointers claiming one source (ambiguous authority).
|
|
790
|
+
A lone pointer over a lone source with NEITHER key matching is refused
|
|
791
|
+
too — that is exactly the silent-wrong-spans class the batch guard
|
|
792
|
+
existed for (a set whose manifest names a different media file)."""
|
|
793
|
+
loaded = [(str(p),) + event_spans_from_propset(p, classes) for p in pointers]
|
|
794
|
+
out: List[Tuple[Dict[str, Any], List[Tuple[float, float]], str]] = []
|
|
795
|
+
claimed: Dict[str, int] = {}
|
|
796
|
+
for i, src in enumerate(sources):
|
|
797
|
+
h = str(src.get("content_hash") or "")
|
|
798
|
+
path = str(src.get("source_path") or "")
|
|
799
|
+
hits = [(ptr, m, spans) for ptr, m, spans in loaded
|
|
800
|
+
if (h and str((m.get("source") or {}).get("content_hash") or "") == h)
|
|
801
|
+
or (path and str((m.get("source") or {}).get("path") or "") == path)]
|
|
802
|
+
name = Path(path).name if path else (h or f"source-{i}")
|
|
803
|
+
if not hits:
|
|
804
|
+
raise RuntimeError(
|
|
805
|
+
f"event-split: no proposal set covers source {i} ({name}) — "
|
|
806
|
+
f"propose over it first (pointers given: {[p for p, _, _ in loaded]})")
|
|
807
|
+
if len(hits) > 1:
|
|
808
|
+
raise RuntimeError(
|
|
809
|
+
f"event-split: {len(hits)} proposal sets claim source {i} ({name}): "
|
|
810
|
+
f"{[p for p, _, _ in hits]} — pass exactly one set per source")
|
|
811
|
+
ptr, m, spans = hits[0]
|
|
812
|
+
claimed[ptr] = i
|
|
813
|
+
out.append((m, spans, ptr))
|
|
814
|
+
stray = [ptr for ptr, _, _ in loaded if ptr not in claimed]
|
|
815
|
+
if stray:
|
|
816
|
+
raise RuntimeError(
|
|
817
|
+
f"event-split: proposal set(s) {stray} bind to none of this run's "
|
|
818
|
+
f"{len(sources)} source(s) — a set carves only the source its manifest names")
|
|
819
|
+
return out
|
|
@@ -9,10 +9,12 @@ tag — the manifest-as-interchange contract (CR-20) — is what separates
|
|
|
9
9
|
transcription runs from decomp runs living in one directory."""
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
|
+
import time
|
|
12
13
|
from pathlib import Path
|
|
13
14
|
from typing import Any, Dict, List, Optional, Tuple
|
|
14
15
|
|
|
15
16
|
from cjm_substrate.core.workspace import resolve_recorded_tree
|
|
17
|
+
from cjm_substrate.utils.lifecycle import partition_lifecycle
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
def _load_manifests(
|
|
@@ -186,7 +188,8 @@ class PropsetIndex:
|
|
|
186
188
|
|
|
187
189
|
def __init__(self, proposals_dir: str = "proposals"): # Workspace proposals/ (else cwd-relative)
|
|
188
190
|
self.proposals_dir = Path(proposals_dir)
|
|
189
|
-
self.sets: List[Dict[str, Any]] = [] #
|
|
191
|
+
self.sets: List[Dict[str, Any]] = [] # ACTIVE manifest dicts, newest first (+ "_path", "_lifecycle")
|
|
192
|
+
self.archived: List[Dict[str, Any]] = [] # Archived sets (lifecycle sidecar, b20cb911) — off every ring, still on disk
|
|
190
193
|
|
|
191
194
|
def load(self) -> int: # Number of proposal sets loaded
|
|
192
195
|
"""(Re)read every readable proposal-set manifest, newest first (the
|
|
@@ -206,8 +209,11 @@ class PropsetIndex:
|
|
|
206
209
|
m["_path"] = str(f)
|
|
207
210
|
rows.append(m)
|
|
208
211
|
rows.sort(key=lambda m: float(m.get("created_at") or 0.0), reverse=True)
|
|
209
|
-
|
|
210
|
-
|
|
212
|
+
# Lifecycle (b20cb911, DEC 14678fc9): an archived set leaves every
|
|
213
|
+
# ring and default (for_source reads `sets`) but stays on disk under
|
|
214
|
+
# `archived` — the sidecar seam in cjm_substrate.utils.lifecycle.
|
|
215
|
+
self.sets, self.archived = partition_lifecycle(rows)
|
|
216
|
+
return len(self.sets)
|
|
211
217
|
|
|
212
218
|
def for_source(
|
|
213
219
|
self,
|
|
@@ -251,7 +257,8 @@ class TrainingRunIndex:
|
|
|
251
257
|
|
|
252
258
|
def __init__(self, training_runs_dir: str = "training-runs"): # Workspace training-runs/ (else cwd-relative)
|
|
253
259
|
self.training_runs_dir = Path(training_runs_dir)
|
|
254
|
-
self.runs: List[Dict[str, Any]] = [] #
|
|
260
|
+
self.runs: List[Dict[str, Any]] = [] # ACTIVE manifest dicts, newest first (+ "_path", "_lifecycle")
|
|
261
|
+
self.archived: List[Dict[str, Any]] = [] # Archived runs (lifecycle sidecar, b20cb911) — off the ladder, shown on demand
|
|
255
262
|
|
|
256
263
|
def load(self) -> int: # Number of training runs loaded
|
|
257
264
|
"""(Re)read every readable training-run manifest, newest first (run
|
|
@@ -271,8 +278,11 @@ class TrainingRunIndex:
|
|
|
271
278
|
m["_path"] = str(f)
|
|
272
279
|
rows.append(m)
|
|
273
280
|
rows.sort(key=lambda m: str(m.get("run_id") or ""), reverse=True)
|
|
274
|
-
|
|
275
|
-
|
|
281
|
+
# Lifecycle (b20cb911, DEC 14678fc9): an archived run leaves the
|
|
282
|
+
# ladder and the picker's default list but stays on disk under
|
|
283
|
+
# `archived` — the m picker's h toggle lists it dimmed, a/x act on it.
|
|
284
|
+
self.runs, self.archived = partition_lifecycle(rows)
|
|
285
|
+
return len(self.runs)
|
|
276
286
|
|
|
277
287
|
def resolve(
|
|
278
288
|
self,
|
|
@@ -294,3 +304,87 @@ class TrainingRunIndex:
|
|
|
294
304
|
"""Footer text for the two-step confirm: run id tail + trained classes."""
|
|
295
305
|
classes = ",".join(m.get("classes") or []) or "?"
|
|
296
306
|
return f"{str(m.get('run_id') or '?')[-8:]} [{classes}]"
|
|
307
|
+
|
|
308
|
+
def resolve_pick(
|
|
309
|
+
self,
|
|
310
|
+
pick: Optional[str] = None, # In-session picker choice: a run's _path or run id (tail ok)
|
|
311
|
+
pin: Optional[str] = None, # Workspace-marker pin (the standing default)
|
|
312
|
+
) -> Tuple[Optional[Dict[str, Any]], str]: # (run or None, the reason the confirm names)
|
|
313
|
+
"""The propose-model ladder (work item a13ebc66): PICK > PIN > NEWEST.
|
|
314
|
+
A picker choice made in this sitting outranks the workspace marker's
|
|
315
|
+
flywheel.event_training_run pin — the pin demotes from the only
|
|
316
|
+
control to the fallback default, so swapping to a test run and back
|
|
317
|
+
no longer means hand-editing cjm-workspace.yaml (and the yaml's
|
|
318
|
+
provenance comments are never rewritten). Every miss is LOUD (None +
|
|
319
|
+
the reason): a pick whose run vanished from disk, or a pin matching
|
|
320
|
+
nothing, never silently falls through to a different model (the
|
|
321
|
+
b9717422 field-failure class the register retired overlay-v1 for)."""
|
|
322
|
+
if pick:
|
|
323
|
+
for m in self.runs:
|
|
324
|
+
rid = str(m.get("run_id") or "")
|
|
325
|
+
if m.get("_path") == pick or rid == pick or rid.endswith(pick):
|
|
326
|
+
return m, "picked"
|
|
327
|
+
gone = self._archived_match(pick)
|
|
328
|
+
if gone is not None:
|
|
329
|
+
return None, (f"picked run …{str(gone.get('run_id') or '?')[-8:]} "
|
|
330
|
+
f"is ARCHIVED — m picks another (h shows archived)")
|
|
331
|
+
return None, f"picked run {pick!r} no longer on disk"
|
|
332
|
+
if pin:
|
|
333
|
+
run = self.resolve(pin)
|
|
334
|
+
if run is not None:
|
|
335
|
+
return run, "workspace pin"
|
|
336
|
+
if self._archived_match(pin) is not None:
|
|
337
|
+
return None, (f"training-run pin {pin!r} matches an ARCHIVED "
|
|
338
|
+
f"run — m picks another, or unarchive it")
|
|
339
|
+
return None, (f"training-run pin {pin!r} matches nothing under "
|
|
340
|
+
f"{self.training_runs_dir}")
|
|
341
|
+
if self.runs:
|
|
342
|
+
return self.runs[0], "newest run, no pin"
|
|
343
|
+
return None, f"no training runs under {self.training_runs_dir}"
|
|
344
|
+
|
|
345
|
+
def _archived_match(
|
|
346
|
+
self,
|
|
347
|
+
handle: str, # A run's _path, run id, or id tail
|
|
348
|
+
) -> Optional[Dict[str, Any]]: # The ARCHIVED run the handle names, or None
|
|
349
|
+
"""The loud-miss lookup: a pick or pin that resolves to nothing in
|
|
350
|
+
`runs` may name a run that was ARCHIVED (b20cb911) — the reason
|
|
351
|
+
says so instead of 'no longer on disk' / 'matches nothing'."""
|
|
352
|
+
for m in self.archived:
|
|
353
|
+
rid = str(m.get("run_id") or "")
|
|
354
|
+
if m.get("_path") == handle or rid == handle or rid.endswith(handle):
|
|
355
|
+
return m
|
|
356
|
+
return None
|
|
357
|
+
|
|
358
|
+
@staticmethod
|
|
359
|
+
def describe(m: Dict[str, Any]) -> str: # One picker-row line of manifest facts
|
|
360
|
+
"""A picker row's facts, read off the manifest alone (capability-
|
|
361
|
+
generic): run id · base model · [classes] · the headline eval
|
|
362
|
+
(macro AUROC when the holdout block carries it, else the first
|
|
363
|
+
metric) · consumed dataset tail · mint date."""
|
|
364
|
+
parts = [str(m.get("run_id") or "?")]
|
|
365
|
+
base = str((m.get("base_model") or {}).get("model_id")
|
|
366
|
+
or (m.get("config") or {}).get("base_model_id") or "")
|
|
367
|
+
if base:
|
|
368
|
+
segs = base.rstrip("/").split("/")
|
|
369
|
+
# a hub id ends in the model name; a local checkpoint path ends
|
|
370
|
+
# in a weights FILE — name its directory instead
|
|
371
|
+
ext = segs[-1].rsplit(".", 1)[-1].lower()
|
|
372
|
+
parts.append(segs[-2] if len(segs) > 1 and ext in
|
|
373
|
+
("bin", "ckpt", "pt", "pth", "safetensors")
|
|
374
|
+
else segs[-1])
|
|
375
|
+
parts.append("[" + (",".join(m.get("classes") or []) or "?") + "]")
|
|
376
|
+
ev = m.get("eval") or {}
|
|
377
|
+
macro = ((ev.get("holdout") or {}).get("auroc") or {}).get("macro")
|
|
378
|
+
metrics = ev.get("metrics") or {}
|
|
379
|
+
if isinstance(macro, (int, float)):
|
|
380
|
+
parts.append(f"auroc {macro:.3f}")
|
|
381
|
+
elif metrics:
|
|
382
|
+
k, v = next(iter(metrics.items()))
|
|
383
|
+
parts.append(f"{k} {v:.3f}" if isinstance(v, float) else f"{k} {v}")
|
|
384
|
+
ds = str(m.get("dataset_id") or "")
|
|
385
|
+
if ds:
|
|
386
|
+
parts.append(f"ds …{ds[-8:]}")
|
|
387
|
+
created = m.get("created_at")
|
|
388
|
+
if created:
|
|
389
|
+
parts.append(time.strftime("%Y-%m-%d", time.localtime(float(created))))
|
|
390
|
+
return " · ".join(parts)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cjm-transcript-decomp-core
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.9
|
|
4
4
|
Summary: A frontend-agnostic core for the transcript decomposition workflow — composes isolated capability workers (forced alignment, VAD, graph storage) into a headless pipeline that decomposes transcription run manifests into a VAD-aligned context-graph spine with traceable provenance, and a CLI as its first driver.
|
|
5
5
|
Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
10
10
|
Requires-Python: >=3.12
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
License-File: LICENSE
|
|
13
|
-
Requires-Dist: cjm_substrate>=0.0.
|
|
13
|
+
Requires-Dist: cjm_substrate>=0.0.63
|
|
14
14
|
Requires-Dist: pyyaml
|
|
15
15
|
Requires-Dist: cjm_context_graph_primitives>=0.0.15
|
|
16
16
|
Requires-Dist: cjm_capability_primitives>=0.0.11
|
|
@@ -102,6 +102,7 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
|
|
|
102
102
|
- `event_spans_from_propset` _function_ — Load a proposal set BY POINTER and select its carve spans (respine trial
|
|
103
103
|
- `fa_words_from_result` _function_ — Normalize a typed forced-alignment result into FA words (pure; stage 3).
|
|
104
104
|
- `load_source_manifest` _function_ — Load + lightly validate a transcription-core run manifest.
|
|
105
|
+
- `resolve_event_propsets` _function_ — Join each proposal set to ITS source by the set manifest's own source
|
|
105
106
|
- `run_decomp` _function_ — Extend every source in a transcription run manifest with its fine spine.
|
|
106
107
|
- `sentence_spans_from_result` _function_ — Normalize a typed sentence-segmentation result into char-span tuples (pure; B.5).
|
|
107
108
|
- `submit_and_wait` _function_ — Submit one capability job, wait for it, and return its result (raise on failure).
|
|
@@ -14,7 +14,7 @@ classifiers = [
|
|
|
14
14
|
"Programming Language :: Python :: 3",
|
|
15
15
|
"Programming Language :: Python :: 3 :: Only",
|
|
16
16
|
]
|
|
17
|
-
dependencies = ['cjm_substrate>=0.0.
|
|
17
|
+
dependencies = ['cjm_substrate>=0.0.63', 'pyyaml', 'cjm_context_graph_primitives>=0.0.15', 'cjm_capability_primitives>=0.0.11', 'cjm-context-graph-layer>=0.0.11', 'cjm-transcript-graph-schema>=0.0.7']
|
|
18
18
|
|
|
19
19
|
[project.urls]
|
|
20
20
|
Repository = "https://github.com/cj-mills/cjm-transcript-decomp-core"
|
{cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/tests/test_batch_logic.py
RENAMED
|
@@ -293,3 +293,134 @@ def test_training_run_index_discovery_and_resolve(tmp_path):
|
|
|
293
293
|
# Empty dir: no runs, loud None
|
|
294
294
|
empty = TrainingRunIndex(str(tmp_path / "nowhere"))
|
|
295
295
|
assert empty.load() == 0 and empty.resolve() is None
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def test_training_run_resolve_pick_ladder_and_describe(tmp_path):
|
|
299
|
+
"""Work item a13ebc66 (the propose training-run picker): the model
|
|
300
|
+
ladder is PICK > PIN > NEWEST with the reason named alongside, and every
|
|
301
|
+
miss is a loud None (a vanished pick, a pin matching nothing) — never a
|
|
302
|
+
silent fall-through to a different model. describe() reads a picker
|
|
303
|
+
row off the manifest alone: id · base · [classes] · headline eval ·
|
|
304
|
+
dataset tail · date."""
|
|
305
|
+
from cjm_transcript_decomp_core.runs import TrainingRunIndex
|
|
306
|
+
d = tmp_path / "training-runs"
|
|
307
|
+
|
|
308
|
+
def _run(run_id, classes, extra=None):
|
|
309
|
+
rd = d / run_id
|
|
310
|
+
rd.mkdir(parents=True)
|
|
311
|
+
m = {"format": TrainingRunIndex.FORMAT, "run_id": run_id, "classes": classes}
|
|
312
|
+
m.update(extra or {})
|
|
313
|
+
(rd / "manifest.json").write_text(json.dumps(m))
|
|
314
|
+
|
|
315
|
+
_run("trainrun_20260731_002436_6f803b12", ["speech", "inhale"],
|
|
316
|
+
{"base_model": {"model_id": "pyannote/segmentation-3.0"},
|
|
317
|
+
"dataset_id": "dataset_20260730_e53b8daa", "created_at": 1785000000.0,
|
|
318
|
+
"eval": {"holdout": {"auroc": {"macro": 0.99659}},
|
|
319
|
+
"metrics": {"loss/val": 0.0634}}})
|
|
320
|
+
_run("trainrun_20260805_104926_73000552", ["click", "inhale"],
|
|
321
|
+
{"eval": {"metrics": {"loss/val": 0.0812}}})
|
|
322
|
+
idx = TrainingRunIndex(str(d))
|
|
323
|
+
assert idx.load() == 2
|
|
324
|
+
# No pick, no pin: newest is only a default, and the reason says so
|
|
325
|
+
run, why = idx.resolve_pick(None, None)
|
|
326
|
+
assert run["run_id"].endswith("73000552") and why == "newest run, no pin"
|
|
327
|
+
# The pin is the standing default over recency
|
|
328
|
+
run, why = idx.resolve_pick(None, "6f803b12")
|
|
329
|
+
assert run["run_id"].endswith("6f803b12") and why == "workspace pin"
|
|
330
|
+
# A pick outranks the pin — by _path, full id, or tail
|
|
331
|
+
older = idx.runs[1]
|
|
332
|
+
for pick in (older["_path"], older["run_id"], "6f803b12"):
|
|
333
|
+
run, why = idx.resolve_pick(pick, "73000552")
|
|
334
|
+
assert run["run_id"].endswith("6f803b12") and why == "picked"
|
|
335
|
+
run, why = idx.resolve_pick("73000552", "6f803b12")
|
|
336
|
+
assert run["run_id"].endswith("73000552") and why == "picked"
|
|
337
|
+
# Misses are LOUD: a vanished pick never falls through to the pin
|
|
338
|
+
run, why = idx.resolve_pick("deadbeef", "6f803b12")
|
|
339
|
+
assert run is None and "deadbeef" in why and "no longer on disk" in why
|
|
340
|
+
run, why = idx.resolve_pick(None, "deadbeef")
|
|
341
|
+
assert run is None and "matches nothing" in why
|
|
342
|
+
empty = TrainingRunIndex(str(tmp_path / "nowhere"))
|
|
343
|
+
empty.load()
|
|
344
|
+
assert empty.resolve_pick(None, None) == (None, f"no training runs under {empty.training_runs_dir}")
|
|
345
|
+
# describe(): the row facts a picker paints
|
|
346
|
+
line = TrainingRunIndex.describe(older)
|
|
347
|
+
assert line.startswith("trainrun_20260731_002436_6f803b12 · segmentation-3.0 · [speech,inhale]")
|
|
348
|
+
assert "auroc 0.997" in line and "ds …e53b8daa" in line and "2026-07-" in line
|
|
349
|
+
# No holdout block: the first metric is the headline; no base/ds: elided
|
|
350
|
+
line = TrainingRunIndex.describe(idx.runs[0])
|
|
351
|
+
assert "loss/val 0.081" in line and "ds …" not in line and "segmentation" not in line
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def test_batch_argv_renders_one_pointer_per_source():
|
|
355
|
+
"""Multi-source event carve (decomp manifest 0.2.6): a group's resolved
|
|
356
|
+
propset may be a LIST/TUPLE (one pointer per source of a multi-source
|
|
357
|
+
run) — batch_argv renders every pointer under one --event-propset, in
|
|
358
|
+
order, and the single-source str form renders exactly as before."""
|
|
359
|
+
args = resolve_split_flags(build_parser().parse_args(
|
|
360
|
+
["--event-split", "--event-classes", "inhale"]))
|
|
361
|
+
multi = batch_argv({"text_from": None, "graph_db_path": None,
|
|
362
|
+
"event_propset": ("/sets/p_src0", "/sets/p_src1"),
|
|
363
|
+
"manifests": ["ab.json"]}, args, None)
|
|
364
|
+
i = multi.index("--event-propset")
|
|
365
|
+
assert multi[i + 1:i + 3] == ["/sets/p_src0", "/sets/p_src1"]
|
|
366
|
+
assert multi[i + 3] == "--event-classes"
|
|
367
|
+
single = batch_argv({"text_from": None, "graph_db_path": None,
|
|
368
|
+
"event_propset": "/sets/p_only",
|
|
369
|
+
"manifests": ["a.json"]}, args, None)
|
|
370
|
+
j = single.index("--event-propset")
|
|
371
|
+
assert single[j + 1] == "/sets/p_only" and single[j + 2] == "--event-classes"
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def test_lifecycle_archived_artifacts_leave_pickers_and_misses_are_loud(tmp_path):
|
|
375
|
+
"""Work item b20cb911 (DEC 14678fc9): an archived training run / proposal
|
|
376
|
+
set (a lifecycle.json sidecar beside the manifest, the substrate seam)
|
|
377
|
+
leaves the index's default list — the ladder never resolves to it and
|
|
378
|
+
for_source skips it — but stays reachable via `archived`, and a pick or
|
|
379
|
+
pin that lands on one is a LOUD miss naming the archive, never a silent
|
|
380
|
+
fall-through to another model."""
|
|
381
|
+
from cjm_substrate.utils.lifecycle import ArtifactLifecycle
|
|
382
|
+
from cjm_transcript_decomp_core.runs import PropsetIndex, TrainingRunIndex
|
|
383
|
+
trs = tmp_path / "training-runs"
|
|
384
|
+
for rid in ("trainrun_1_6f803b12", "trainrun_2_fa3035c6"):
|
|
385
|
+
(trs / rid).mkdir(parents=True)
|
|
386
|
+
(trs / rid / "manifest.json").write_text(json.dumps(
|
|
387
|
+
{"format": TrainingRunIndex.FORMAT, "run_id": rid, "classes": ["speech"]}))
|
|
388
|
+
idx = TrainingRunIndex(str(trs))
|
|
389
|
+
assert idx.load() == 2 and idx.archived == []
|
|
390
|
+
ArtifactLifecycle(trs / "trainrun_2_fa3035c6").archive(reason="test run")
|
|
391
|
+
assert idx.load() == 1
|
|
392
|
+
assert [m["run_id"][-8:] for m in idx.runs] == ["6f803b12"]
|
|
393
|
+
assert [m["run_id"][-8:] for m in idx.archived] == ["fa3035c6"]
|
|
394
|
+
assert idx.runs[0]["_lifecycle"] == "active"
|
|
395
|
+
assert idx.archived[0]["_lifecycle"] == "archived"
|
|
396
|
+
# newest-no-pin skips the archived head; a pin or pick on it is loud
|
|
397
|
+
run, why = idx.resolve_pick(None, None)
|
|
398
|
+
assert run["run_id"].endswith("6f803b12") and why == "newest run, no pin"
|
|
399
|
+
run, why = idx.resolve_pick(None, "fa3035c6")
|
|
400
|
+
assert run is None and "ARCHIVED" in why and "fa3035c6" in why
|
|
401
|
+
run, why = idx.resolve_pick(idx.archived[0]["_path"], "6f803b12")
|
|
402
|
+
assert run is None and "ARCHIVED" in why and "fa3035c6" in why
|
|
403
|
+
run, why = idx.resolve_pick(None, "deadbeef")
|
|
404
|
+
assert run is None and "matches nothing" in why
|
|
405
|
+
# unarchive restores it as the newest default
|
|
406
|
+
ArtifactLifecycle(trs / "trainrun_2_fa3035c6").unarchive()
|
|
407
|
+
assert idx.load() == 2
|
|
408
|
+
assert idx.resolve_pick(None, None)[0]["run_id"].endswith("fa3035c6")
|
|
409
|
+
# proposal sets: the archived set leaves its source's ring, the older
|
|
410
|
+
# generation heads it again, and `archived` keeps it inspectable
|
|
411
|
+
props = tmp_path / "proposals"
|
|
412
|
+
for pid, created in (("propset_1_aaaaaaaa", 1.0), ("propset_2_bbbbbbbb", 2.0)):
|
|
413
|
+
(props / pid).mkdir(parents=True)
|
|
414
|
+
(props / pid / "manifest.json").write_text(json.dumps(
|
|
415
|
+
{"format": PropsetIndex.FORMAT, "proposal_set_id": pid,
|
|
416
|
+
"created_at": created, "counts": {"inhale": 1},
|
|
417
|
+
"source": {"content_hash": "sha256:xx", "path": "/x.wav"}}))
|
|
418
|
+
pidx = PropsetIndex(str(props))
|
|
419
|
+
assert pidx.load() == 2
|
|
420
|
+
assert [m["proposal_set_id"][-8:] for m in pidx.for_source("sha256:xx")] \
|
|
421
|
+
== ["bbbbbbbb", "aaaaaaaa"]
|
|
422
|
+
ArtifactLifecycle(props / "propset_2_bbbbbbbb").archive()
|
|
423
|
+
assert pidx.load() == 1
|
|
424
|
+
assert [m["proposal_set_id"][-8:] for m in pidx.for_source("sha256:xx")] \
|
|
425
|
+
== ["aaaaaaaa"]
|
|
426
|
+
assert [m["proposal_set_id"][-8:] for m in pidx.archived] == ["bbbbbbbb"]
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"""Tests for cjm_transcript_decomp_core.models — decomposition data shapes.
|
|
2
2
|
|
|
3
3
|
Projected from the models notebook's smoke-check cell at the golden-reference flip."""
|
|
4
|
+
import re
|
|
5
|
+
|
|
4
6
|
from cjm_transcript_decomp_core.models import (
|
|
5
7
|
DecompManifest,
|
|
6
8
|
DecompSegment,
|
|
@@ -38,7 +40,11 @@ def test_manifest_shape_and_run_id():
|
|
|
38
40
|
m = DecompManifest(run_id="r", created_at=0.0, config={}, source_manifest="/tmp/s.json")
|
|
39
41
|
md = m.to_dict()
|
|
40
42
|
assert md["format"] == "cjm-transcript-decomp-core/run-manifest"
|
|
41
|
-
|
|
43
|
+
# Compare to the dataclass default, not a literal: the literal pin went RED on the first
|
|
44
|
+
# run after every manifest bump (0.2.3 -> 0.2.6), training agents to ignore a red test
|
|
45
|
+
# (housekeeping 042b7805). The shape check below keeps the version a real semver.
|
|
46
|
+
assert md["version"] == m.VERSION and md["sources"] == []
|
|
47
|
+
assert re.fullmatch(r"\d+\.\d+\.\d+", m.VERSION)
|
|
42
48
|
# 0.2.2: skeleton identity rides the manifest (empty/None until run_decomp fills them)
|
|
43
49
|
assert md["skeleton_config_hash"] == "" and md["split_policy"] is None
|
|
44
50
|
# 0.2.4: event-carve chain fields (empty/None until an event_split run fills them)
|
{cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/tests/test_pipeline.py
RENAMED
|
@@ -194,3 +194,54 @@ def test_event_carve_identity_and_propset_loading(tmp_path):
|
|
|
194
194
|
import pytest
|
|
195
195
|
with pytest.raises(RuntimeError):
|
|
196
196
|
event_spans_from_propset(tmp_path / "missing", ["inhale"])
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def test_resolve_event_propsets_joins_each_set_to_its_source(tmp_path):
|
|
200
|
+
"""Multi-source event carve (manifest 0.2.6): resolve_event_propsets joins
|
|
201
|
+
each proposal set to ITS source by the set manifest's own binding —
|
|
202
|
+
content hash first, path as the fallback — in any pointer order, and
|
|
203
|
+
every refusal is loud: an uncovered source, a stray pointer, two sets
|
|
204
|
+
claiming one source, and a lone mismatched pointer over a lone source
|
|
205
|
+
(the silent-wrong-spans class)."""
|
|
206
|
+
import json
|
|
207
|
+
import pytest
|
|
208
|
+
from cjm_transcript_decomp_core.pipeline import resolve_event_propsets
|
|
209
|
+
|
|
210
|
+
def _set(name, source, spans):
|
|
211
|
+
d = tmp_path / name
|
|
212
|
+
d.mkdir()
|
|
213
|
+
(d / "manifest.json").write_text(json.dumps({
|
|
214
|
+
"format": "cjm-capability-pyannote/proposal-set-manifest",
|
|
215
|
+
"proposal_set_id": name, "source": source,
|
|
216
|
+
"files": {"proposals": "proposals.jsonl"}}))
|
|
217
|
+
(d / "proposals.jsonl").write_text("\n".join(
|
|
218
|
+
json.dumps({"label": "inhale", "tier": 1, "start_time": s, "end_time": e})
|
|
219
|
+
for s, e in spans))
|
|
220
|
+
return str(d)
|
|
221
|
+
|
|
222
|
+
a = _set("propset_a", {"content_hash": "sha256:aaa", "path": "/media/a.wav"}, [(1.0, 1.5)])
|
|
223
|
+
b = _set("propset_b", {"content_hash": "sha256:bbb", "path": "/media/b.wav"}, [(2.0, 2.5), (3.0, 3.2)])
|
|
224
|
+
c_path_only = _set("propset_c", {"path": "/media/c.wav"}, [(4.0, 4.1)])
|
|
225
|
+
sources = [{"source_path": "/media/a.wav", "content_hash": "sha256:aaa"},
|
|
226
|
+
{"source_path": "/media/b.wav", "content_hash": "sha256:bbb"},
|
|
227
|
+
{"source_path": "/media/c.wav", "content_hash": "sha256:ccc"}]
|
|
228
|
+
# any pointer order; hash join for a/b, path fallback for c
|
|
229
|
+
out = resolve_event_propsets([c_path_only, b, a], sources, ["inhale"])
|
|
230
|
+
assert [m["proposal_set_id"] for m, _, _ in out] == ["propset_a", "propset_b", "propset_c"]
|
|
231
|
+
assert [spans for _, spans, _ in out] == [[(1.0, 1.5)], [(2.0, 2.5), (3.0, 3.2)], [(4.0, 4.1)]]
|
|
232
|
+
assert [ptr for _, _, ptr in out] == [a, b, c_path_only]
|
|
233
|
+
# single-source, single pointer: the 0.2.5 shape, unchanged
|
|
234
|
+
assert resolve_event_propsets([a], sources[:1], ["inhale"])[0][0]["proposal_set_id"] == "propset_a"
|
|
235
|
+
# uncovered source
|
|
236
|
+
with pytest.raises(RuntimeError, match="no proposal set covers source 2"):
|
|
237
|
+
resolve_event_propsets([a, b], sources, ["inhale"])
|
|
238
|
+
# stray pointer (binds to none of the run's sources)
|
|
239
|
+
with pytest.raises(RuntimeError, match="bind to none"):
|
|
240
|
+
resolve_event_propsets([a, b], sources[:1], ["inhale"])
|
|
241
|
+
# two sets claiming one source
|
|
242
|
+
a2 = _set("propset_a2", {"content_hash": "sha256:aaa", "path": "/media/a.wav"}, [(1.2, 1.4)])
|
|
243
|
+
with pytest.raises(RuntimeError, match="2 proposal sets claim source 0"):
|
|
244
|
+
resolve_event_propsets([a, a2], sources[:1], ["inhale"])
|
|
245
|
+
# lone mismatched pointer over a lone source is NOT tolerated
|
|
246
|
+
with pytest.raises(RuntimeError, match="no proposal set covers source 0"):
|
|
247
|
+
resolve_event_propsets([b], sources[:1], ["inhale"])
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.0.6"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/tests/test_alignment.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cjm_transcript_decomp_core-0.0.6 → cjm_transcript_decomp_core-0.0.9}/tests/test_segment_logic.py
RENAMED
|
File without changes
|