cjm-transcript-decomp-core 0.0.9__tar.gz → 0.0.10__tar.gz

{cjm_transcript_decomp_core-0.0.9/cjm_transcript_decomp_core.egg-info → cjm_transcript_decomp_core-0.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjm-transcript-decomp-core
-Version: 0.0.9
+Version: 0.0.10
 Summary: A frontend-agnostic core for the transcript decomposition workflow — composes isolated capability workers (forced alignment, VAD, graph storage) into a headless pipeline that decomposes transcription run manifests into a VAD-aligned context-graph spine with traceable provenance, and a CLI as its first driver.
 Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
 License: Apache-2.0
@@ -26,7 +26,7 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 
 ## Modules
 
-- **`cjm_transcript_decomp_core`**
+- **`cjm_transcript_decomp_core.__init__`**
 - **`cjm_transcript_decomp_core.alignment`** — Pure forced-alignment logic (no capability calls): map FA words back to character spans in the original text, assign words to VAD chunks by timestamp, and build one text segment per VAD chunk. Extracted from the page-centric ForcedAlignmentService (Tier-1 logic).
 - **`cjm_transcript_decomp_core.cli`** — The CLI driver — the decomposition core's first (and currently only) frontend.
 - **`cjm_transcript_decomp_core.discovery`** — Capability-role discovery by manifest surface match — the journaling-by-
@@ -53,6 +53,7 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 
 ### `cjm_transcript_decomp_core.cli`
 
+- `backfill_provenance_command` _function_ — Execute `backfill-provenance`: mint the Segment -> Transcript DERIVED_FROM
 - `build_parser` _function_ — Build the CLI parser (subcommands: run).
 - `load_capabilities` _function_ — Discover manifests + load each requested capability (default instance).
 - `main` _function_ — CLI entry point (console script: `cjm-transcript-decomp-core`).
@@ -67,7 +68,9 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 
 - `SourceVerification` _class_ — Skeptical-lens verification of one Source's fine-spine extension under a
 - `build_extension_payload` _function_ — Build the fine-spine EXTENSION payload (pure; no capability calls).
+- `provenance_edges_from_segment_wire` _function_ — Derive a Segment's Transcript provenance edges from the node itself:
 - `resolve_root_ids` _function_ — Recompute the transcription-emitted root node ids from manifest data.
+- `segment_provenance_edges` _function_ — Text provenance as EDGES (finding 89b16be6, the references-must-be-edges
 - `verify_source` _function_ — Verify a Source's committed extension via server-side AGGREGATES (D13/D19).
 
 ### `cjm_transcript_decomp_core.launch`
@@ -141,4 +144,4 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 ## Dependencies
 
 **Depends on:** `cjm-capability-primitives`, `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-substrate`, `cjm-transcript-graph-schema`, `pyyaml`
-**Used by:** `cjm-transcript-decomp-qt`, `cjm-transcript-decomp-tui`
+**Used by:** `cjm-transcript-decomp-qt`

{cjm_transcript_decomp_core-0.0.9 → cjm_transcript_decomp_core-0.0.10}/README.md

@@ -6,7 +6,7 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 
 ## Modules
 
-- **`cjm_transcript_decomp_core`**
+- **`cjm_transcript_decomp_core.__init__`**
 - **`cjm_transcript_decomp_core.alignment`** — Pure forced-alignment logic (no capability calls): map FA words back to character spans in the original text, assign words to VAD chunks by timestamp, and build one text segment per VAD chunk. Extracted from the page-centric ForcedAlignmentService (Tier-1 logic).
 - **`cjm_transcript_decomp_core.cli`** — The CLI driver — the decomposition core's first (and currently only) frontend.
 - **`cjm_transcript_decomp_core.discovery`** — Capability-role discovery by manifest surface match — the journaling-by-
@@ -33,6 +33,7 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 
 ### `cjm_transcript_decomp_core.cli`
 
+- `backfill_provenance_command` _function_ — Execute `backfill-provenance`: mint the Segment -> Transcript DERIVED_FROM
 - `build_parser` _function_ — Build the CLI parser (subcommands: run).
 - `load_capabilities` _function_ — Discover manifests + load each requested capability (default instance).
 - `main` _function_ — CLI entry point (console script: `cjm-transcript-decomp-core`).
@@ -47,7 +48,9 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 
 - `SourceVerification` _class_ — Skeptical-lens verification of one Source's fine-spine extension under a
 - `build_extension_payload` _function_ — Build the fine-spine EXTENSION payload (pure; no capability calls).
+- `provenance_edges_from_segment_wire` _function_ — Derive a Segment's Transcript provenance edges from the node itself:
 - `resolve_root_ids` _function_ — Recompute the transcription-emitted root node ids from manifest data.
+- `segment_provenance_edges` _function_ — Text provenance as EDGES (finding 89b16be6, the references-must-be-edges
 - `verify_source` _function_ — Verify a Source's committed extension via server-side AGGREGATES (D13/D19).
 
 ### `cjm_transcript_decomp_core.launch`
@@ -121,4 +124,4 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 ## Dependencies
 
 **Depends on:** `cjm-capability-primitives`, `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-substrate`, `cjm-transcript-graph-schema`, `pyyaml`
-**Used by:** `cjm-transcript-decomp-qt`, `cjm-transcript-decomp-tui`
+**Used by:** `cjm-transcript-decomp-qt`

cjm_transcript_decomp_core-0.0.10/cjm_transcript_decomp_core/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.10"

{cjm_transcript_decomp_core-0.0.9 → cjm_transcript_decomp_core-0.0.10}/cjm_transcript_decomp_core/cli.py

@@ -27,9 +27,13 @@ import os
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
+from cjm_context_graph_layer.journal import journal_extend, sidecar_journal_path
+from cjm_context_graph_layer.ops import graph_task
+from cjm_context_graph_primitives.query import NodeQuery
 from cjm_substrate.core.manager import CapabilityManager
 from cjm_substrate.core.queue import JobQueue
 from cjm_substrate.core.workspace import resolve_workspace
+from cjm_transcript_decomp_core.graph import provenance_edges_from_segment_wire
 from cjm_transcript_decomp_core.models import DecompConfig
 from cjm_transcript_decomp_core.pipeline import load_source_manifest, run_decomp
 
@@ -107,6 +111,29 @@ def build_parser() -> argparse.ArgumentParser:  # Configured CLI parser
     run.add_argument("--actor", default=None,
                      help="Journal attribution for who/what initiated this run (default: cli:<username>)")
     run.add_argument("-v", "--verbose", action="store_true", help="DEBUG-level logging")
+
+    # ---- backfill-provenance: Segment -> Transcript DERIVED_FROM edges (finding 89b16be6) ----
+    bp = sub.add_parser(
+        "backfill-provenance",
+        help="Mint Segment -> Transcript DERIVED_FROM edges for Segments already on the graph "
+             "(text provenance as an edge, not only text_from metadata); journaled per source "
+             "under spine-extension, idempotent on re-run")
+    bp.add_argument("--source", default=None,
+                    help="Restrict to one Source node id (prefix ok; default: every Segment)")
+    bp.add_argument("--page-size", type=int, default=2000,
+                    help="Segments per query page (default 2000)")
+    bp.add_argument("--manifests-dir", default=".cjm/manifests", help="Capability manifests directory")
+    bp.add_argument("--graph-capability", default="cjm-capability-graph-sqlite",
+                    help="Graph-storage capability name")
+    bp.add_argument("--graph-db-path", default=None,
+                    help="Graph db path (default: the workspace capability config)")
+    bp.add_argument("--workspace", default=None,
+                    help="Workspace root (default: CJM_WORKSPACE env, else upward walk from cwd)")
+    bp.add_argument("--actor", default=None,
+                    help="Journal attribution (default: cli:<username>)")
+    bp.add_argument("--dry-run", action="store_true",
+                    help="Count the edges per source; touch neither graph nor journal")
+    bp.add_argument("-v", "--verbose", action="store_true", help="DEBUG-level logging")
     return parser
 
 
@@ -263,4 +290,80 @@ def main(
     )
     if args.command == "run":
         return asyncio.run(run_command(args))
+    if args.command == "backfill-provenance":
+        return asyncio.run(backfill_provenance_command(args))
     raise SystemExit(f"unknown command: {args.command}")
+
+
+async def backfill_provenance_command(
+    args: argparse.Namespace,  # Parsed CLI arguments for the `backfill-provenance` subcommand
+) -> int:  # Process exit code
+    """Execute `backfill-provenance`: mint the Segment -> Transcript DERIVED_FROM
+    edges (finding 89b16be6) for Segments already on the graph, deriving each
+    segment's edges from its own refs (`provenance_edges_from_segment_wire`,
+    the same derivation a fresh decomposition uses). Journaled per SOURCE under
+    the existing `spine-extension` verb (edges-only wires; the replay handler
+    is already registered, and `journal_extend`'s added-delta trim keeps a
+    re-run silent), args carry act=provenance-backfill so the ops read as what
+    they are. Pages the Segment table so a 100k-segment graph never sits in one
+    query; --source narrows to one Source id (prefix ok); --dry-run counts."""
+    ws = resolve_workspace(explicit=getattr(args, "workspace", None))
+    if ws is not None:
+        os.environ["CJM_WORKSPACE"] = str(ws.root)
+    manager = CapabilityManager(search_paths=[Path(args.manifests_dir)])
+    configs = ({args.graph_capability: {"db_path": args.graph_db_path}}
+               if args.graph_db_path else None)
+    load_capabilities(manager, [args.graph_capability], configs=configs)
+    effective = args.graph_db_path or (
+        (manager.instances[args.graph_capability].config or {}).get("db_path"))
+    if not effective:
+        raise SystemExit("no graph db path: pass --graph-db-path, or persist one on the "
+                         f"{args.graph_capability} instance in the active workspace's config store")
+    journal_path = sidecar_journal_path(str(effective))
+    actor = args.actor or f"cli:{getpass.getuser()}"
+    queue = JobQueue(deps=manager)
+    await queue.start()
+    try:
+        per_source: Dict[str, List[Dict[str, Any]]] = {}
+        seg_counts: Dict[str, int] = {}
+        offset = 0
+        while True:
+            nq = NodeQuery(label="Segment", project=["text_from", "source_id", "sources"],
+                           limit=args.page_size, offset=offset)
+            res = await graph_task(queue, args.graph_capability, "query_nodes", query=nq.to_dict())
+            rows = list(res.rows or [])
+            for row in rows:
+                sid = row.get("source_id") or "?"
+                if args.source and not sid.startswith(args.source):
+                    continue
+                seg_counts[sid] = seg_counts.get(sid, 0) + 1
+                per_source.setdefault(sid, []).extend(provenance_edges_from_segment_wire(row))
+            if len(rows) < args.page_size:
+                break
+            offset += len(rows)
+        total_edges = sum(len(v) for v in per_source.values())
+        if args.dry_run:
+            for sid in sorted(per_source):
+                print(f"{sid[:8]}  segments={seg_counts[sid]}  edges={len(per_source[sid])}")
+            print(f"dry run: {len(per_source)} sources, {sum(seg_counts.values())} segments, "
+                  f"{total_edges} DERIVED_FROM edges (nothing written)")
+            return 0
+        added = 0
+        for sid in sorted(per_source):
+            edges = per_source[sid]
+            if not edges:
+                continue
+            r = await journal_extend(queue, args.graph_capability, [], edges,
+                                     journal_path=journal_path, verb="spine-extension",
+                                     actor=actor,
+                                     args={"act": "provenance-backfill", "source_id": sid,
+                                           "segments": seg_counts[sid], "edges": len(edges)})
+            added += r.edges_added
+            print(f"{sid[:8]}  segments={seg_counts[sid]}  edges={len(edges)}  added={r.edges_added}")
+    finally:
+        await queue.stop()
+        manager.unload_capability(args.graph_capability)
+    print(f"backfilled DERIVED_FROM edges: {added} added of {total_edges} derived "
+          f"across {len(per_source)} sources (actor {actor})")
+    print(f"journal: {journal_path}")
+    return 0

{cjm_transcript_decomp_core-0.0.9 → cjm_transcript_decomp_core-0.0.10}/cjm_transcript_decomp_core/graph.py

@@ -3,7 +3,8 @@
 from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional, Tuple
 
-from cjm_context_graph_layer.grammar import grouped_spine_edges, SpineRelations
+from cjm_context_graph_layer.grammar import (grouped_spine_edges, make_edge, OverlayRelations,
+                                             SpineRelations)
 from cjm_context_graph_layer.ops import graph_task
 from cjm_context_graph_primitives.graph import GraphNode
 from cjm_context_graph_primitives.provenance import SourceRef
@@ -80,7 +81,9 @@ def build_extension_payload(
     transcriber variant; `text_from` is recorded per segment as provenance
     (which Transcript the layer-0 text came from), never as global config. Edges
     come from `grouped_spine_edges`: PART_OF to the OWNING AudioRendition,
-    STARTS_WITH per rendition, NEXT chained source-wide across coarse boundaries.
+    STARTS_WITH per rendition, NEXT chained source-wide across coarse boundaries —
+    plus one DERIVED_FROM per sliced Transcript (text provenance as an edge,
+    finding 89b16be6; role=text_from marks the authoritative one).
     """
     roots = resolve_root_ids(source_entry, capabilities_info)
     source_id = roots["source"]
@@ -114,6 +117,13 @@ def build_extension_payload(
 
     # The fine spine hangs under each segment's RENDITION (PART_OF rendition).
     edges = grouped_spine_edges([(asegs[i]["rendition"], groups[i]) for i in sorted(groups)])
+    # Text provenance is an EDGE, not only a property (finding 89b16be6): each
+    # Segment DERIVED_FROM every Transcript it slices (role=text_from on the
+    # authoritative one), so Segments are reachable from their Transcript by
+    # traversal the way they already were from their AudioRendition. Derived
+    # from the node's own refs — the same derivation the backfill verb uses.
+    for node in nodes:
+        edges.extend(provenance_edges_from_segment_wire(node))
     ids = {"source": source_id, "segments": seg_ids,
            "audio_segments": [a["audio_segment"] for a in asegs],
            "renditions": [a["rendition"] for a in asegs],
@@ -273,3 +283,48 @@ async def verify_source(
         all_have_sources=missing_sources == 0,
         source_locators=sorted(locators),
     )
+
+
+def segment_provenance_edges(
+    segment_id: str,                   # The Segment node id
+    transcript_ids: List[str],         # Transcript node ids the segment slices text from (any order; de-duplicated)
+    text_from: Optional[str] = None,   # The authoritative Transcript id (None when the segment carries no text)
+) -> List[Dict[str, Any]]:  # DERIVED_FROM edge wires, Segment -> Transcript, one per distinct transcript
+    """Text provenance as EDGES (finding 89b16be6, the references-must-be-edges
+    instance): a Segment's text comes from Transcript nodes, and until now that
+    link lived only in `text_from` metadata + the CharSlice source refs, so a
+    Segment was unreachable from its Transcript by traversal (the only
+    traceability edge went to the AudioRendition). One DERIVED_FROM per sliced
+    Transcript; the authoritative one carries role=text_from, the rest
+    role=variant. Ids are deterministic from the triple, so a backfill over
+    existing Segments and a fresh decomposition converge on the same edges."""
+    seen: set = set()
+    edges: List[Dict[str, Any]] = []
+    for tid in transcript_ids:
+        if not tid or tid in seen:
+            continue
+        seen.add(tid)
+        role = "text_from" if tid == text_from else "variant"
+        edges.append(make_edge(segment_id, tid, OverlayRelations.DERIVED_FROM, {"role": role}))
+    return edges
+
+
+def provenance_edges_from_segment_wire(
+    node: Dict[str, Any],  # A Segment node wire (or projected row): {"id", "properties"?, "sources", "text_from"?}
+) -> List[Dict[str, Any]]:  # The segment's DERIVED_FROM edges, derived from its own refs
+    """Derive a Segment's Transcript provenance edges from the node itself:
+    every `char`-sliced source ref's graph-node locator is a Transcript the
+    segment slices text from, and `text_from` (a property on a wire, a
+    projected field on a query row) names the authoritative one. ONE derivation
+    for both callers — `build_extension_payload` at decomposition time and the
+    `backfill-provenance` verb over Segments already on the graph — so the
+    backfill reproduces exactly what a fresh run would emit."""
+    props = node.get("properties") or {}
+    text_from = props.get("text_from", node.get("text_from"))
+    tids: List[str] = []
+    for ref in node.get("sources") or []:
+        sl = ref.get("slice") or {}
+        loc = ref.get("locator") or {}
+        if sl.get("kind") == "char" and loc.get("node_id"):
+            tids.append(loc["node_id"])
+    return segment_provenance_edges(node["id"], tids, text_from)

{cjm_transcript_decomp_core-0.0.9 → cjm_transcript_decomp_core-0.0.10/cjm_transcript_decomp_core.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjm-transcript-decomp-core
-Version: 0.0.9
+Version: 0.0.10
 Summary: A frontend-agnostic core for the transcript decomposition workflow — composes isolated capability workers (forced alignment, VAD, graph storage) into a headless pipeline that decomposes transcription run manifests into a VAD-aligned context-graph spine with traceable provenance, and a CLI as its first driver.
 Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
 License: Apache-2.0
@@ -26,7 +26,7 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 
 ## Modules
 
-- **`cjm_transcript_decomp_core`**
+- **`cjm_transcript_decomp_core.__init__`**
 - **`cjm_transcript_decomp_core.alignment`** — Pure forced-alignment logic (no capability calls): map FA words back to character spans in the original text, assign words to VAD chunks by timestamp, and build one text segment per VAD chunk. Extracted from the page-centric ForcedAlignmentService (Tier-1 logic).
 - **`cjm_transcript_decomp_core.cli`** — The CLI driver — the decomposition core's first (and currently only) frontend.
 - **`cjm_transcript_decomp_core.discovery`** — Capability-role discovery by manifest surface match — the journaling-by-
@@ -53,6 +53,7 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 
 ### `cjm_transcript_decomp_core.cli`
 
+- `backfill_provenance_command` _function_ — Execute `backfill-provenance`: mint the Segment -> Transcript DERIVED_FROM
 - `build_parser` _function_ — Build the CLI parser (subcommands: run).
 - `load_capabilities` _function_ — Discover manifests + load each requested capability (default instance).
 - `main` _function_ — CLI entry point (console script: `cjm-transcript-decomp-core`).
@@ -67,7 +68,9 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 
 - `SourceVerification` _class_ — Skeptical-lens verification of one Source's fine-spine extension under a
 - `build_extension_payload` _function_ — Build the fine-spine EXTENSION payload (pure; no capability calls).
+- `provenance_edges_from_segment_wire` _function_ — Derive a Segment's Transcript provenance edges from the node itself:
 - `resolve_root_ids` _function_ — Recompute the transcription-emitted root node ids from manifest data.
+- `segment_provenance_edges` _function_ — Text provenance as EDGES (finding 89b16be6, the references-must-be-edges
 - `verify_source` _function_ — Verify a Source's committed extension via server-side AGGREGATES (D13/D19).
 
 ### `cjm_transcript_decomp_core.launch`
@@ -141,4 +144,4 @@ A frontend-agnostic core for the transcript decomposition workflow — composes
 ## Dependencies
 
 **Depends on:** `cjm-capability-primitives`, `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-substrate`, `cjm-transcript-graph-schema`, `pyyaml`
-**Used by:** `cjm-transcript-decomp-qt`, `cjm-transcript-decomp-tui`
+**Used by:** `cjm-transcript-decomp-qt`

{cjm_transcript_decomp_core-0.0.9 → cjm_transcript_decomp_core-0.0.10}/tests/test_graph.py

@@ -123,3 +123,35 @@ def test_parallel_spines_disjoint_ids_and_split_metadata():
     assert all("split_policy" not in n["properties"] for n in base_nodes)
     assert all(n["properties"]["skeleton_hash"] == "sha256:skel-split" for n in split_nodes)
     assert all(n["properties"]["split_policy"] == "sentence-split/v1" for n in split_nodes)
+
+
+def test_segment_transcript_provenance_edges():
+    # finding 89b16be6: a Segment's text provenance is an EDGE, not only metadata —
+    # DERIVED_FROM per sliced Transcript, the authoritative one role=text_from, the
+    # others role=variant; an empty (audio-only) segment derives no text edge; and the
+    # wire-derived form (the backfill's input) reproduces the builder's edges exactly.
+    from cjm_transcript_decomp_core.graph import (provenance_edges_from_segment_wire,
+                                                  segment_provenance_edges)
+    roots = resolve_root_ids(SOURCE_ENTRY, CAPABILITIES)
+    tv = roots["audio_segments"][0]["transcripts"]["voxtral"]
+    tw = roots["audio_segments"][0]["transcripts"]["whisper"]
+    nodes, edges, ids = build_extension_payload(SOURCE_ENTRY, CAPABILITIES, "sha256:vad", "voxtral", _segments())
+    derived = [e for e in edges if e["relation_type"] == "DERIVED_FROM"]
+    # seg 0 slices voxtral + whisper (2 edges), seg 1 voxtral only (1), seg 2 empty (0)
+    assert len(derived) == 3
+    by_src = {}
+    for e in derived:
+        by_src.setdefault(e["source_id"], []).append(e)
+    s0 = {e["target_id"]: e["properties"]["role"] for e in by_src[ids["segments"][0]]}
+    assert s0 == {tv: "text_from", tw: "variant"}
+    assert [(e["target_id"], e["properties"]["role"]) for e in by_src[ids["segments"][1]]] == [(tv, "text_from")]
+    assert ids["segments"][2] not in by_src
+    # Deterministic ids from the triple: a rebuilt edge collides into the same id.
+    again = segment_provenance_edges(ids["segments"][0], [tw, tv], tv)
+    assert {e["id"] for e in again} == {e["id"] for e in by_src[ids["segments"][0]]}
+    # The backfill derivation (from the node's own refs) == the builder's edges.
+    for n in nodes:
+        assert provenance_edges_from_segment_wire(n) == by_src.get(n["id"], [])
+    # A projected query row carries text_from as a field, not under properties.
+    row = {"id": nodes[0]["id"], "text_from": tv, "sources": nodes[0]["sources"]}
+    assert provenance_edges_from_segment_wire(row) == by_src[nodes[0]["id"]]

cjm_transcript_decomp_core-0.0.9/cjm_transcript_decomp_core/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.9"