chunksmith-multimodal 0.3.0__py3-none-any.whl

chunksmith_multimodal/Mapper/__init__.py

@@ -0,0 +1,37 @@
+"""Mapper output helpers for ChunkSmith TitleIndexer."""
+
+from chunksmith_multimodal.Mapper.anchor_mapping_options import (
+    AnchorMappingOptions,
+    apply_anchor_mapping_options,
+)
+from chunksmith_multimodal.Mapper.combined_index import (
+    MAPPING_METHODS,
+    build_combined_index,
+    print_combined_index_summary,
+    title_chunks_to_chunk_nodes,
+)
+from chunksmith_multimodal.Mapper.mapper import (
+    assign_mapper_node_ids,
+    build_mapper,
+    format_mapper_structure,
+    format_mapper_mapping_report_text,
+    print_mapper_mapping_report,
+    save_mapper_txt,
+    summarize_anchor_mapping,
+)
+
+__all__ = [
+    "AnchorMappingOptions",
+    "apply_anchor_mapping_options",
+    "MAPPING_METHODS",
+    "assign_mapper_node_ids",
+    "build_combined_index",
+    "build_mapper",
+    "format_mapper_structure",
+    "print_combined_index_summary",
+    "format_mapper_mapping_report_text",
+    "print_mapper_mapping_report",
+    "save_mapper_txt",
+    "summarize_anchor_mapping",
+    "title_chunks_to_chunk_nodes",
+]

chunksmith_multimodal/Mapper/anchor_mapping_options.py

@@ -0,0 +1,68 @@
+"""Mapper anchor-slice settings (start/end boundary mode), separate from page-index LLM options."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from chunksmith_multimodal.indexer.options import (
+    anchor_mapping_mode_label,
+    validate_anchor_mapping_flags,
+)
+
+__all__ = [
+    "AnchorMappingOptions",
+    "apply_anchor_mapping_options",
+]
+
+
+@dataclass
+class AnchorMappingOptions:
+    """
+    How the mapper slices raw elements per outline node (flat preorder).
+
+    - ``both``: node start anchor + node end anchor
+    - ``start_only``: node start; end at next flat node's start (``use_end=False``)
+    - ``end_only``: start at previous node's end; node end (``use_start=False``)
+    """
+
+    use_start_anchor_mapping: bool = True
+    use_end_anchor_mapping: bool = False
+
+    def mode_label(self) -> str:
+        return anchor_mapping_mode_label(
+            use_start=self.use_start_anchor_mapping,
+            use_end=self.use_end_anchor_mapping,
+        )
+
+    def validate(self) -> None:
+        validate_anchor_mapping_flags(
+            use_start=self.use_start_anchor_mapping,
+            use_end=self.use_end_anchor_mapping,
+        )
+
+    def to_indexer_options_dict(self) -> dict[str, bool | str]:
+        """Patch for ``title_outline[\"indexer_options\"]`` / mapper."""
+        return {
+            "use_start_anchor_mapping": self.use_start_anchor_mapping,
+            "use_end_anchor_mapping": self.use_end_anchor_mapping,
+            "anchor_mapping_mode": self.mode_label(),
+        }
+
+
+def apply_anchor_mapping_options(
+    title_outline: dict[str, Any],
+    options: AnchorMappingOptions | None,
+) -> dict[str, Any]:
+    """
+    Merge anchor mapping flags into ``title_outline[\"indexer_options\"]`` (in place).
+
+    Call before ``build_mapper`` when re-running mapper only, or let the pipeline do it.
+    """
+    if options is None:
+        return title_outline
+    options.validate()
+    merged = dict(title_outline.get("indexer_options") or {})
+    merged.update(options.to_indexer_options_dict())
+    title_outline["indexer_options"] = merged
+    return title_outline

chunksmith_multimodal/Mapper/anchor_slice.py

@@ -0,0 +1,200 @@
+"""O(n) element slicing by start/end anchor needles with page and next-start fallbacks."""
+
+from __future__ import annotations
+
+import copy
+from typing import Any
+
+from chunksmith_multimodal.indexer.anchor_fields import (
+    anchor_end_from_row,
+    anchor_start_from_row,
+)
+from chunksmith_multimodal.Mapper.anchor_slice_find import (
+    find_end_exclusive_index,
+    find_start_element_index,
+    trim_first_element_from_offset,
+    trim_last_element_to_anchor_end,
+)
+from chunksmith_multimodal.Mapper.anchor_slice_meta import (
+    anchor_mapping_detail_from_meta,
+    anchor_mapping_status_from_meta,
+)
+
+__all__ = [
+    "node_anchor_start",
+    "node_anchor_end",
+    "effective_boundary_needles",
+    "boundary_needles_for_node",
+    "slice_elements_by_anchor_range",
+    "anchor_mapping_status_from_meta",
+    "anchor_mapping_detail_from_meta",
+]
+
+
+def node_anchor_start(node: dict[str, Any]) -> str:
+    return anchor_start_from_row(node)
+
+
+def node_anchor_end(node: dict[str, Any]) -> str:
+    return anchor_end_from_row(node)
+
+
+def effective_boundary_needles(
+    flat_nodes: list[dict[str, Any]],
+    index: int,
+    *,
+    use_start_mapping: bool,
+    use_end_mapping: bool,
+) -> tuple[str, str, str, str, str, str, str | None, str | None]:
+    """
+    Declared effective boundaries (for metadata) and slice inputs (for element search).
+
+    Returns (
+        declared_start, declared_end,
+        slice_start, slice_end,
+        next_start, prev_end,
+        next_node_id, prev_node_id,
+    ).
+
+    - both: this node's start + end anchors
+    - start_only: this start; declared end = next flat node's start
+    - end_only: declared start = previous flat node's end; this end
+    """
+    node = flat_nodes[index]
+    node_start = node_anchor_start(node)
+    node_end = node_anchor_end(node)
+
+    next_start = ""
+    next_nid: str | None = None
+    if index + 1 < len(flat_nodes):
+        nxt = flat_nodes[index + 1]
+        next_start = node_anchor_start(nxt)
+        next_nid = str(nxt.get("node_id") or "") or None
+
+    prev_end = ""
+    prev_nid: str | None = None
+    if index > 0:
+        prev = flat_nodes[index - 1]
+        prev_end = node_anchor_end(prev)
+        prev_nid = str(prev.get("node_id") or "") or None
+
+    if use_start_mapping and use_end_mapping:
+        return node_start, node_end, node_start, node_end, next_start, prev_end, next_nid, prev_nid
+    if use_start_mapping and not use_end_mapping:
+        return node_start, next_start, node_start, "", next_start, prev_end, next_nid, prev_nid
+    if not use_start_mapping and use_end_mapping:
+        return prev_end, node_end, "", node_end, next_start, prev_end, next_nid, prev_nid
+    return node_start, node_end, node_start, node_end, next_start, prev_end, next_nid, prev_nid
+
+
+def boundary_needles_for_node(
+    flat_nodes: list[dict[str, Any]],
+    index: int,
+    *,
+    use_start_mapping: bool,
+    use_end_mapping: bool,
+) -> tuple[str, str, str, str, str | None, str | None]:
+    """
+    Slice needles for one flat-preorder node.
+
+    Returns (slice_start, slice_end, next_start, prev_end, next_node_id, prev_node_id).
+    """
+    _decl_start, _decl_end, slice_start, slice_end, next_start, prev_end, next_nid, prev_nid = (
+        effective_boundary_needles(
+            flat_nodes, index, use_start_mapping=use_start_mapping, use_end_mapping=use_end_mapping
+        )
+    )
+    return slice_start, slice_end, next_start, prev_end, next_nid, prev_nid
+
+
+def slice_elements_by_anchor_range(
+    elements: list[dict[str, Any]],
+    *,
+    anchor_start: str,
+    anchor_end: str,
+    start_page: int | None,
+    end_page: int | None,
+    next_start_needle: str = "",
+    prev_end_needle: str = "",
+    anchor_mapping_mode: str = "both",
+    next_node_id: str | None = None,
+    prev_node_id: str | None = None,
+    effective_start_needle: str = "",
+    effective_end_needle: str = "",
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    """Slice ordered elements for one outline node; metadata for debugging."""
+    mode = anchor_mapping_mode or "both"
+    end_requires_next = mode == "start_only"
+    allow_next_start = mode == "start_only"
+
+    meta: dict[str, Any] = {
+        "mapping_anchor_mode": mode,
+        "mapping_anchor_start": anchor_start,
+        "mapping_anchor_end": anchor_end,
+        "mapping_effective_start_needle": effective_start_needle,
+        "mapping_effective_end_needle": effective_end_needle,
+        "mapping_anchor_start_found": False,
+        "mapping_anchor_end_found": False,
+        "mapping_start_strategy": "",
+        "mapping_end_strategy": "",
+        "mapping_slice_start_strategy": "",
+        "mapping_slice_end_strategy": "",
+        "mapping_next_node_id": next_node_id,
+        "mapping_prev_node_id": prev_node_id,
+    }
+    if not elements:
+        meta["mapping_anchor_status"] = anchor_mapping_status_from_meta(meta)
+        meta["mapping_anchor_detail"] = anchor_mapping_detail_from_meta(meta)
+        return [], meta
+
+    start_i, start_found, start_strategy = find_start_element_index(
+        elements,
+        anchor_start,
+        page_fallback=start_page,
+        prev_end_needle=prev_end_needle,
+    )
+    meta["mapping_anchor_start_found"] = start_found
+    meta["mapping_start_strategy"] = start_strategy
+
+    end_i, end_found, strategy = find_end_exclusive_index(
+        elements,
+        anchor_end,
+        from_index=start_i,
+        next_start_needle=next_start_needle if allow_next_start else "",
+        page_end_fallback=end_page,
+        end_requires_next_start=end_requires_next,
+        allow_next_start=allow_next_start,
+    )
+    meta["mapping_anchor_end_found"] = end_found
+    meta["mapping_end_strategy"] = strategy
+    meta["mapping_slice_start_strategy"] = start_strategy
+    meta["mapping_slice_end_strategy"] = strategy
+
+    sliced = [copy.deepcopy(el) for el in elements[start_i:end_i]]
+    trim_start = (
+        anchor_start if start_strategy == "anchor_start" else (prev_end_needle if start_strategy == "prev_end" else "")
+    )
+    if trim_start and sliced:
+        sliced = trim_first_element_from_offset(sliced, trim_start)
+    if anchor_end and sliced and end_found and strategy.startswith("anchor_end"):
+        sliced = trim_last_element_to_anchor_end(sliced, anchor_end)
+    if start_strategy == "prev_end":
+        meta["mapping_effective_start_boundary"] = "prev_end"
+    elif start_strategy == "anchor_start":
+        meta["mapping_effective_start_boundary"] = "anchor_start"
+    elif start_strategy == "page_start":
+        meta["mapping_effective_start_boundary"] = "page_start"
+
+    if strategy == "next_start":
+        meta["mapping_effective_end_boundary"] = "next_start"
+    elif strategy.startswith("anchor_end"):
+        meta["mapping_effective_end_boundary"] = "anchor_end"
+    elif strategy == "page_end":
+        meta["mapping_effective_end_boundary"] = "page_end"
+    elif strategy == "span_end":
+        meta["mapping_effective_end_boundary"] = "span_end"
+    else:
+        meta["mapping_effective_end_boundary"] = strategy or "unknown"
+    meta["mapping_anchor_status"] = anchor_mapping_status_from_meta(meta)
+    meta["mapping_anchor_detail"] = anchor_mapping_detail_from_meta(meta)
+    return sliced, meta

chunksmith_multimodal/Mapper/anchor_slice_find.py

@@ -0,0 +1,160 @@
+"""Needle search helpers for anchor-based element slicing."""
+
+from __future__ import annotations
+
+import copy
+import re
+from typing import Any
+
+_WS = re.compile(r"\s+")
+
+
+def element_page_number(element: dict[str, Any]) -> int | None:
+    value = element.get("page_number")
+    if isinstance(value, int):
+        return value
+    metadata = element.get("metadata")
+    if isinstance(metadata, dict) and isinstance(metadata.get("page_number"), int):
+        return metadata["page_number"]
+    return None
+
+
+def element_text(element: dict[str, Any]) -> str:
+    return str(element.get("text") or "")
+
+
+def fold_text(text: str) -> str:
+    return _WS.sub(" ", text or "").strip().casefold()
+
+
+def find_needle_span(text: str, needle: str, *, start_at: int = 0) -> tuple[int, int] | None:
+    """
+    Find ``needle`` in ``text``; return ``(start, end_exclusive)`` char offsets.
+
+    Tolerates citation brackets before a trailing period (e.g. ``loss.`` vs ``loss [11].``).
+    """
+    if not needle or not text:
+        return None
+
+    folded = text.casefold()
+    nfold = fold_text(needle)
+    if not nfold:
+        return None
+
+    off = folded.find(nfold, start_at)
+    if off >= 0:
+        return off, off + len(nfold)
+
+    core = needle.rstrip(" .!?…").strip()
+    if len(core) < 12:
+        return None
+
+    core_fold = core.casefold()
+    pattern = re.escape(core_fold) + r"(?:\s*\[\d+\])?\s*\.?"
+    match = re.search(pattern, folded[start_at:])
+    if match:
+        start = start_at + match.start()
+        return start, start_at + match.end()
+
+    return None
+
+
+def find_needle_in_text(text: str, needle: str, *, start_at: int = 0) -> int | None:
+    span = find_needle_span(text, needle, start_at=start_at)
+    return span[0] if span else None
+
+
+def find_start_element_index(
+    elements: list[dict[str, Any]],
+    anchor_start: str,
+    *,
+    page_fallback: int | None,
+    prev_end_needle: str = "",
+) -> tuple[int, bool, str]:
+    """First element at/after start match; prev_end then anchor_start; else page fallback."""
+    if prev_end_needle and not anchor_start:
+        for i, el in enumerate(elements):
+            if find_needle_in_text(element_text(el), prev_end_needle) is not None:
+                return i, True, "prev_end"
+    if anchor_start:
+        for i, el in enumerate(elements):
+            if find_needle_in_text(element_text(el), anchor_start) is not None:
+                return i, True, "anchor_start"
+    if page_fallback is not None:
+        for i, el in enumerate(elements):
+            if element_page_number(el) == page_fallback:
+                return i, False, "page_start"
+    return 0, False, "page_start"
+
+
+def find_end_exclusive_index(
+    elements: list[dict[str, Any]],
+    anchor_end: str,
+    *,
+    from_index: int,
+    next_start_needle: str,
+    page_end_fallback: int | None,
+    end_requires_next_start: bool = False,
+    allow_next_start: bool = False,
+) -> tuple[int, bool, str]:
+    """
+    Exclusive end index for slice [from_index, end).
+
+    When ``end_requires_next_start`` (start_only mode), skip node end anchor.
+    ``next_start`` is only used when ``allow_next_start`` is True (start_only).
+    """
+    n = len(elements)
+    if from_index >= n:
+        return n, False, "empty"
+
+    if anchor_end and not end_requires_next_start:
+        for i in range(from_index, n):
+            text = element_text(elements[i])
+            span = find_needle_span(text, anchor_end)
+            if span is not None:
+                _start, end_char = span
+                if end_char >= len(text.rstrip()):
+                    return i + 1, True, "anchor_end"
+                return i + 1, True, "anchor_end_partial"
+
+    if allow_next_start and next_start_needle:
+        for i in range(from_index + 1, n):
+            text = element_text(elements[i])
+            if find_needle_in_text(text, next_start_needle) is not None:
+                return i, True, "next_start"
+
+    if page_end_fallback is not None:
+        last_on_page = from_index
+        for i in range(from_index, n):
+            if element_page_number(elements[i]) == page_end_fallback:
+                last_on_page = i
+            elif element_page_number(elements[i]) is not None and element_page_number(elements[i]) > page_end_fallback:
+                break
+        return last_on_page + 1, False, "page_end"
+
+    return n, False, "span_end"
+
+
+def trim_first_element_from_offset(
+    elements: list[dict[str, Any]],
+    needle: str,
+) -> list[dict[str, Any]]:
+    if not elements or not needle:
+        return elements
+    first = copy.deepcopy(elements[0])
+    text = element_text(first)
+    span = find_needle_span(text, needle)
+    if span is not None and span[0] > 0:
+        first["text"] = text[span[0] :].strip()
+    return [first, *elements[1:]]
+
+
+def trim_last_element_to_anchor_end(elements: list[dict[str, Any]], anchor_end: str) -> list[dict[str, Any]]:
+    if not elements or not anchor_end:
+        return elements
+    last = copy.deepcopy(elements[-1])
+    text = element_text(last)
+    span = find_needle_span(text, anchor_end)
+    if span is not None:
+        last["text"] = text[: span[1]].strip()
+    return [*elements[:-1], last]

chunksmith_multimodal/Mapper/anchor_slice_meta.py

@@ -0,0 +1,75 @@
+"""Anchor mapping status and detail strings from slice metadata."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def anchor_mapping_status_from_meta(meta: dict[str, Any]) -> str:
+    """Per-node pass/failed by anchor_mapping_mode."""
+    mode = str(meta.get("mapping_anchor_mode") or "both")
+    start_found = bool(meta.get("mapping_anchor_start_found"))
+    start_strategy = str(meta.get("mapping_start_strategy") or "")
+    end_strategy = str(meta.get("mapping_end_strategy") or "")
+    end_found = bool(meta.get("mapping_anchor_end_found"))
+
+    if mode == "start_only":
+        if not start_found or start_strategy == "page_start":
+            return "failed"
+        if end_strategy == "page_end":
+            return "failed"
+        return "pass"
+
+    if mode == "end_only":
+        if not start_found or start_strategy == "page_start":
+            return "failed"
+        if not end_found or end_strategy == "page_end":
+            return "failed"
+        return "pass"
+
+    if not start_found or start_strategy == "page_start":
+        return "failed"
+    if not end_found or not end_strategy.startswith("anchor_end"):
+        return "failed"
+    return "pass"
+
+
+def anchor_mapping_detail_from_meta(meta: dict[str, Any]) -> str:
+    """Short explanation of which fallbacks were used."""
+    mode = str(meta.get("mapping_anchor_mode") or "both")
+    start_found = bool(meta.get("mapping_anchor_start_found"))
+    start_strategy = str(meta.get("mapping_start_strategy") or "")
+    end_found = bool(meta.get("mapping_anchor_end_found"))
+    end_strategy = str(meta.get("mapping_end_strategy") or "") or "unknown"
+    next_nid = meta.get("mapping_next_node_id")
+    prev_nid = meta.get("mapping_prev_node_id")
+
+    parts: list[str] = [f"mode: {mode}"]
+    if start_strategy == "prev_end":
+        label = "start: prev node end"
+        if prev_nid:
+            label += f" (node {prev_nid})"
+        parts.append(label)
+    elif start_found:
+        parts.append("start: anchor")
+    else:
+        parts.append("start: page fallback")
+
+    if end_strategy == "next_start":
+        label = "end: next node start"
+        if next_nid:
+            label += f" (node {next_nid})"
+        parts.append(label)
+    elif end_found and end_strategy.startswith("anchor_end"):
+        label = "end: anchor"
+        if end_strategy == "anchor_end_partial":
+            label = "end: anchor (trimmed at mid-element)"
+        parts.append(label)
+    elif end_strategy == "page_end":
+        parts.append("end: page fallback")
+    elif end_strategy == "span_end":
+        parts.append("end: span end")
+    else:
+        parts.append(f"end: {end_strategy}")
+
+    return "; ".join(parts)