chunksmith-cli 0.4.0__py3-none-any.whl

chunksmith_cli/core/prefs_mapper.py

@@ -0,0 +1,97 @@
+"""Map CLI ``PdfPartitionPreferences`` → ``MultiIndexingPreferences`` (PyPI library)."""
+
+from __future__ import annotations
+
+import os
+from typing import Literal
+
+from chunksmith_core.preferences import MultiIndexingPreferences
+from chunksmith_multimodal.config.config import default_anchor_mapping_options
+
+from chunksmith_cli.partition_prefs import PdfPartitionPreferences
+from chunksmith_cli.pipelines.mapping_validation import resolve_multi_indexing_mapping
+from chunksmith_cli.pipelines.multi_indexing_runtime import MultiIndexingModules
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.getenv(name)
+    if not raw:
+        return default
+    try:
+        return int(str(raw).split("#", 1)[0].strip())
+    except ValueError:
+        return default
+
+
+def _element_xml_from_mode(
+    mode: str,
+    custom: dict[str, bool] | None,
+    *,
+    element_xml_types: tuple[str, ...],
+) -> dict[str, bool]:
+    m = (mode or "all").strip().lower()
+    if m == "none":
+        return dict.fromkeys(element_xml_types, False)
+    if m == "minimal":
+        keep = {"Title", "NarrativeText", "Table", "Image"}
+        return {t: (t in keep) for t in element_xml_types}
+    if m == "core":
+        keep = {"Title", "NarrativeText", "Text", "ListItem", "Table", "Image", "Header", "Footer"}
+        return {t: (t in keep) for t in element_xml_types}
+    if m == "custom":
+        custom_map = custom or {}
+        return {t: bool(custom_map.get(t, False)) for t in element_xml_types}
+    return dict.fromkeys(element_xml_types, True)
+
+
+def multi_indexing_preferences_from_partition(
+    prefs: PdfPartitionPreferences,
+    mods: MultiIndexingModules,
+    *,
+    llm_context_format: Literal["json", "toon"],
+    partition_pdf: bool,
+    elements_json_path: str | None = None,
+    stream_llm_tokens: bool = False,
+) -> MultiIndexingPreferences:
+    """Build library preferences from wizard partition prefs."""
+    anchor = default_anchor_mapping_options()
+    mapping = resolve_multi_indexing_mapping(prefs, mapping_methods=mods.MAPPING_METHODS)
+    gen_doc = prefs.chunksmith_pageindex_generate_doc_summary
+
+    return MultiIndexingPreferences(
+        languages=prefs.languages,
+        extract_images=bool(prefs.extract_images),
+        pages_per_chunk=max(0, int(prefs.pages_per_chunk)),
+        max_concurrent=max(1, int(prefs.max_concurrent)),
+        partition_pdf=partition_pdf,
+        use_group_by_title=bool(prefs.chunksmith_use_group_by_title),
+        max_characters=int(prefs.max_characters),
+        new_after_n_chars=int(prefs.new_after_n_chars),
+        combine_text_under_n_chars=int(prefs.combine_text_under_n_chars),
+        group_multipage_sections=bool(prefs.chunksmith_group_multipage_sections),
+        mapping_method=mapping,  # type: ignore[arg-type]
+        llm_context_format=llm_context_format,
+        include_empty_mapper_fields=bool(prefs.chunksmith_include_empty_mapper_fields),
+        use_start_anchor_mapping=bool(anchor.use_start_anchor_mapping),
+        use_end_anchor_mapping=bool(anchor.use_end_anchor_mapping),
+        coded_add_page_xml=bool(prefs.chunksmith_coded_add_page_xml),
+        coded_add_group_by_title_xml=bool(prefs.chunksmith_coded_add_group_by_title_xml),
+        coded_element_xml=_element_xml_from_mode(
+            prefs.chunksmith_coded_element_xml_mode,
+            prefs.chunksmith_coded_element_xml_custom,
+            element_xml_types=mods.ELEMENT_XML_TYPES,
+        ),
+        add_outline_summaries=bool(prefs.effective_pageindex_add_outline_summaries()),
+        add_anchor=bool(prefs.chunksmith_pageindex_add_anchor),
+        generate_doc_summary=bool(gen_doc) if gen_doc is not None else True,
+        use_embedded_pdf_toc=bool(prefs.chunksmith_pageindex_use_embedded_pdf_toc),
+        off_llm_parsing=bool(prefs.chunksmith_pageindex_off_llm_parsing),
+        probe_first_excerpt=bool(prefs.chunksmith_probe_first_excerpt or False),
+        probe_merged_reflection=bool(prefs.chunksmith_probe_merged_reflection or False),
+        stream_llm_tokens=bool(stream_llm_tokens),
+        pageindex_model=os.getenv("PAGEINDEX_MODEL"),
+        llm_model=os.getenv("LLM_MODEL") or os.getenv("CHATGPT_MODEL"),
+        max_tokens_per_chunk=_env_int("CHUNKSMITH_MAX_TOKENS_PER_CHUNK", 3000),
+        overlap_pages=_env_int("CHUNKSMITH_OVERLAP_PAGES", 1),
+        elements_json_path=elements_json_path,
+    )

chunksmith_cli/core/saved_catalog.py

@@ -0,0 +1,180 @@
+"""Discover recent saved outlines and element indexes under cli/data (and logs)."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from chunksmith_cli.core.paths import cli_data_dir, default_project_logs_dir, ensure_cli_storage
+
+
+@dataclass(frozen=True)
+class SavedOutlineRow:
+    stem: str
+    path: Path
+    kind: str  # pageindex | v3 | outline
+    source: str
+
+
+@dataclass(frozen=True)
+class SavedElementIndexRow:
+    stem: str
+    path: Path
+    kind: str  # multimodal | v3
+    pageindex_path: Path | None
+    source: str
+
+
+def _index_has_queryable_elements(data: dict[str, object], *, kind: str) -> bool:
+    if kind == "multi_index":
+        return isinstance(data.get("n_elements"), int) and data["n_elements"] > 0
+    return isinstance(data.get("elements"), list)
+
+
+def _has_structure(path: Path) -> bool:
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return False
+    return isinstance(data, dict) and isinstance(data.get("structure"), list)
+
+
+def _glob_by_mtime(directory: Path, pattern: str) -> list[Path]:
+    hits: list[Path] = []
+    seen: set[Path] = set()
+    for p in directory.glob(pattern):
+        key = p.resolve()
+        if key not in seen:
+            seen.add(key)
+            hits.append(p)
+    if directory.name != "json":
+        for p in directory.glob(f"**/json/{pattern}"):
+            key = p.resolve()
+            if key not in seen:
+                seen.add(key)
+                hits.append(p)
+    hits.sort(key=lambda p: p.stat().st_mtime, reverse=True)
+    return hits
+
+
+def _collect_outline_files(directory: Path, *, source: str, limit: int) -> list[SavedOutlineRow]:
+    rows: list[SavedOutlineRow] = []
+    seen: set[str] = set()
+
+    hits: list[Path] = []
+    for pat in ("*_pageindex.json", "*_multi_index.json", "*_v3_index.json"):
+        for p in _glob_by_mtime(directory, pat):
+            if p not in hits:
+                hits.append(p)
+    hits.sort(key=lambda p: p.stat().st_mtime, reverse=True)
+
+    for p in hits:
+        if len(rows) >= limit:
+            break
+        stem = p.name
+        if stem.endswith("_pageindex.json"):
+            label = stem[: -len("_pageindex.json")]
+            kind = "pageindex"
+        elif stem.endswith("_multi_index.json"):
+            label = stem[: -len("_multi_index.json")]
+            kind = "multi_index"
+        elif stem.endswith("_v3_index.json"):
+            label = stem[: -len("_v3_index.json")]
+            kind = "v3"
+        else:
+            continue
+        if label in seen:
+            continue
+        if not _has_structure(p):
+            continue
+        seen.add(label)
+        rows.append(SavedOutlineRow(stem=label, path=p.resolve(), kind=kind, source=source))
+    return rows
+
+
+def list_saved_outlines(*, limit: int = 10) -> list[SavedOutlineRow]:
+    ensure_cli_storage()
+    rows = _collect_outline_files(cli_data_dir(), source="cli/data (+ artifact folders)", limit=limit)
+    logs = default_project_logs_dir()
+    if logs is not None:
+        json_dir = logs / "json"
+        if json_dir.is_dir():
+            for row in _collect_outline_files(json_dir, source="logs/json", limit=limit):
+                if len(rows) >= limit:
+                    break
+                if not any(r.stem == row.stem for r in rows):
+                    rows.append(row)
+    return rows[:limit]
+
+
+def _pageindex_sibling(index_path: Path) -> Path | None:
+    name = index_path.name
+    parent = index_path.parent
+    if name.endswith("_canonical_bundle.json"):
+        alt = parent / name.replace("_canonical_bundle.json", "_pageindex.json")
+        return alt if alt.is_file() else None
+    if name.endswith("_v3_index.json"):
+        alt = parent / name.replace("_v3_index.json", "_pageindex.json")
+        return alt if alt.is_file() else None
+    return None
+
+
+def _collect_element_indexes(directory: Path, *, source: str, limit: int) -> list[SavedElementIndexRow]:
+    rows: list[SavedElementIndexRow] = []
+    seen: set[str] = set()
+
+    hits: list[Path] = []
+    for pat in ("*_canonical_bundle.json", "*_v3_index.json"):
+        for p in _glob_by_mtime(directory, pat):
+            if p not in hits:
+                hits.append(p)
+    hits.sort(key=lambda p: p.stat().st_mtime, reverse=True)
+
+    for p in hits:
+        if len(rows) >= limit:
+            break
+        name = p.name
+        if name.endswith("_canonical_bundle.json"):
+            stem = name[: -len("_canonical_bundle.json")]
+            kind = "multimodal"
+        elif name.endswith("_v3_index.json"):
+            stem = name[: -len("_v3_index.json")]
+            kind = "v3"
+        else:
+            continue
+        if stem in seen:
+            continue
+        try:
+            data = json.loads(p.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            continue
+        if not isinstance(data, dict) or not _index_has_queryable_elements(data, kind=kind):
+            continue
+        seen.add(stem)
+        pi = _pageindex_sibling(p)
+        rows.append(
+            SavedElementIndexRow(
+                stem=stem,
+                path=p.resolve(),
+                kind=kind,
+                pageindex_path=pi.resolve() if pi else None,
+                source=source,
+            )
+        )
+    return rows
+
+
+def list_saved_element_indexes(*, limit: int = 10) -> list[SavedElementIndexRow]:
+    ensure_cli_storage()
+    rows = _collect_element_indexes(cli_data_dir(), source="cli/data (+ artifact folders)", limit=limit)
+    logs = default_project_logs_dir()
+    if logs is not None:
+        json_dir = logs / "json"
+        if json_dir.is_dir():
+            for row in _collect_element_indexes(json_dir, source="logs/json", limit=limit):
+                if len(rows) >= limit:
+                    break
+                if not any(r.stem == row.stem for r in rows):
+                    rows.append(row)
+    return rows[:limit]

chunksmith_cli/core/theme.py

@@ -0,0 +1,38 @@
+"""Shared terminal theme and console factory."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+from rich.console import Console
+from rich.theme import Theme
+
+VERSION = "0.4.0"
+try:
+    VERSION = version("chunksmith-cli")
+except PackageNotFoundError:
+    pass
+
+NAVY = "#1e3a5f"
+NAVY_LIGHT = "#2d5a87"
+TEAL = "#00b4a6"
+TEAL_BRIGHT = "#2dd4bf"
+TEAL_DIM = "#5ec4bc"
+MUTED = "#94a3b8"
+
+CHUNKSMITH_THEME = Theme(
+    {
+        "brand.navy": NAVY,
+        "brand.teal": TEAL,
+        "brand.muted": MUTED,
+        "info": TEAL_BRIGHT,
+        "success": "green",
+        "warning": "yellow",
+        "error": "bold red",
+    }
+)
+
+
+def make_console(**kwargs) -> Console:
+    """Application console with ChunkSmith colors."""
+    return Console(theme=CHUNKSMITH_THEME, highlight=True, **kwargs)

chunksmith_cli/elements_json_prompt.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.prompts.elements_json_prompt`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.prompts.elements_json_prompt")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/json_view.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.views.json_view`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.views.json_view")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/media_preview.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.core.media_preview`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.core.media_preview")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/menu.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.core.menu`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.core.menu")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/menus.py

@@ -0,0 +1,55 @@
+"""Menu definitions for the interactive CLI."""
+
+from __future__ import annotations
+
+from chunksmith_cli.menu import MenuOption
+
+ROOT_MENU: tuple[MenuOption, ...] = (
+    MenuOption(
+        "chunking",
+        "Multi-indexing",
+        "Unstructured → group-by-title → coded markup → LLM outline → mapper",
+        aliases=("c", "index"),
+    ),
+    MenuOption(
+        "view",
+        "View saved runs",
+        "Inspect outline JSON from saved runs",
+        aliases=("v", "browse"),
+    ),
+    MenuOption(
+        "agent",
+        "Document Q&A",
+        "Load a saved index and chat (legacy multimodal bundles)",
+        aliases=("a", "chat"),
+    ),
+)
+
+CHUNKING_MENU: tuple[MenuOption, ...] = (
+    MenuOption(
+        "1",
+        "Multi-indexing",
+        "Partition → group → coded markup → LLM outline → mapper (JSON or TOON)",
+        aliases=("multi", "indexing"),
+    ),
+    MenuOption(
+        "2",
+        "PageIndexer",
+        "PDF parser + LLM outline (no Unstructured partition)",
+        aliases=("pageindex", "pi"),
+    ),
+)
+
+VIEW_MENU: tuple[MenuOption, ...] = (
+    MenuOption(
+        "1",
+        "View outlines",
+        "Section tree, summary, raw JSON (read-only)",
+        aliases=("outline", "json", "tree"),
+    ),
+)
+
+AGENT_MENU: tuple[MenuOption, ...] = (
+    MenuOption("1", "Open saved index", "Enter a path or stem from saved runs"),
+    MenuOption("2", "Chat", "Ask questions about the loaded document"),
+)

chunksmith_cli/multi_indexing_wizard.py

@@ -0,0 +1,3 @@
+"""Shim — canonical implementation in ``cli.pipelines.multi_indexing_wizard``."""
+
+from chunksmith_cli.pipelines.multi_indexing_wizard import *  # noqa: F403

chunksmith_cli/outline_browser.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.views.outline_browser`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.views.outline_browser")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/panels.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.core.panels`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.core.panels")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/partition_prefs.py

@@ -0,0 +1,74 @@
+"""CLI process settings — mirrors MVL ``PdfPartitionPreferences`` / ``ChunkSmithProcessOptions``."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict
+
+
+@dataclass
+class PdfPartitionPreferences:
+    """Same fields as MVL-AgneticAI ``app.services.indexing.partition_prefs``."""
+
+    languages: str = "eng"
+    extract_images: bool = True
+    extract_tables: bool = True
+    max_characters: int = 3000
+    new_after_n_chars: int = 3800
+    combine_text_under_n_chars: int = 200
+    pages_per_chunk: int = 10
+    max_concurrent: int = 2
+    summarize_outline_nodes: bool = True
+
+    chunksmith_pageindex_add_text: bool = False
+    chunksmith_pageindex_node_text_source: str = "pdf_parser"
+    chunksmith_pageindex_add_anchor: bool = False
+    chunksmith_pageindex_assign_node_ids: bool = True
+    chunksmith_pageindex_add_outline_summaries: bool | None = None
+    chunksmith_pageindex_generate_doc_summary: bool | None = None
+
+    chunksmith_partition_backend: str = "api"
+    chunksmith_probe_first_excerpt: bool | None = None
+    chunksmith_probe_merged_reflection: bool | None = None
+
+    chunksmith_use_group_by_title: bool = True
+    chunksmith_mapping_method: str | None = None
+    chunksmith_include_empty_mapper_fields: bool = False
+    chunksmith_group_multipage_sections: bool = True
+    chunksmith_coded_add_page_xml: bool = True
+    chunksmith_coded_add_group_by_title_xml: bool = True
+    chunksmith_coded_element_xml_mode: str = "all"
+    chunksmith_coded_element_xml_custom: Dict[str, bool] | None = None
+    chunksmith_pageindex_use_embedded_pdf_toc: bool = True
+    chunksmith_pageindex_off_llm_parsing: bool = False
+
+    def effective_pageindex_add_outline_summaries(self) -> bool:
+        if self.chunksmith_pageindex_add_outline_summaries is not None:
+            return bool(self.chunksmith_pageindex_add_outline_summaries)
+        return bool(self.summarize_outline_nodes)
+
+    def to_stored_dict(self) -> Dict[str, Any]:
+        return {
+            "languages": self.languages,
+            "extract_images": self.extract_images,
+            "extract_tables": self.extract_tables,
+            "max_characters": self.max_characters,
+            "new_after_n_chars": self.new_after_n_chars,
+            "combine_text_under_n_chars": self.combine_text_under_n_chars,
+            "pages_per_chunk": self.pages_per_chunk,
+            "max_concurrent": self.max_concurrent,
+            "summarize_outline_nodes": self.summarize_outline_nodes,
+            "chunksmith_use_group_by_title": self.chunksmith_use_group_by_title,
+            "chunksmith_mapping_method": self.chunksmith_mapping_method,
+            "chunksmith_include_empty_mapper_fields": self.chunksmith_include_empty_mapper_fields,
+            "chunksmith_group_multipage_sections": self.chunksmith_group_multipage_sections,
+            "chunksmith_coded_add_page_xml": self.chunksmith_coded_add_page_xml,
+            "chunksmith_coded_add_group_by_title_xml": self.chunksmith_coded_add_group_by_title_xml,
+            "chunksmith_coded_element_xml_mode": self.chunksmith_coded_element_xml_mode,
+            "chunksmith_coded_element_xml_custom": self.chunksmith_coded_element_xml_custom,
+            "chunksmith_pageindex_add_anchor": self.chunksmith_pageindex_add_anchor,
+            "chunksmith_probe_first_excerpt": self.chunksmith_probe_first_excerpt,
+            "chunksmith_probe_merged_reflection": self.chunksmith_probe_merged_reflection,
+            "chunksmith_pageindex_use_embedded_pdf_toc": self.chunksmith_pageindex_use_embedded_pdf_toc,
+            "chunksmith_pageindex_off_llm_parsing": self.chunksmith_pageindex_off_llm_parsing,
+        }

chunksmith_cli/paths.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.core.paths`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.core.paths")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/pdf_prompt.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.prompts.pdf_prompt`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.prompts.pdf_prompt")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/pipelines/__init__.py

@@ -0,0 +1 @@
+"""Multi-indexing ingest wizards and artifact persistence."""

chunksmith_cli/pipelines/mapping_validation.py

@@ -0,0 +1,31 @@
+"""Validate MultiIndexing ``mapping_method`` (aligned with MVL app)."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from chunksmith_cli.partition_prefs import PdfPartitionPreferences  # noqa: TC001
+
+
+def require_mapping_method(value: str | None, *, mapping_methods: frozenset[str]) -> str:
+    raw = (value or "").strip().lower()
+    if raw not in mapping_methods:
+        raise ValueError(f"chunksmith_mapping_method is required; use one of: {', '.join(sorted(mapping_methods))}")
+    return raw
+
+
+def resolve_multi_indexing_mapping(
+    prefs: PdfPartitionPreferences,
+    *,
+    mapping_methods: frozenset[str],
+) -> str:
+    mapping_method = require_mapping_method(prefs.chunksmith_mapping_method, mapping_methods=mapping_methods)
+    use_group = bool(prefs.chunksmith_use_group_by_title)
+    if mapping_method in ("group_indexing", "title_indexing") and not use_group:
+        raise ValueError(
+            f"{mapping_method} requires chunksmith_use_group_by_title=true (or choose page_indexing / anchor_indexing)."
+        )
+    if mapping_method == "anchor_indexing" and not prefs.chunksmith_pageindex_add_anchor:
+        raise ValueError("anchor_indexing requires chunksmith_pageindex_add_anchor=true.")
+    return mapping_method

chunksmith_cli/pipelines/multi_indexing_config.py

@@ -0,0 +1,35 @@
+"""Build ``MultiIndexingPreferences`` from CLI prefs (MVL app + notebook parity)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+from chunksmith_core.preferences import MultiIndexingPreferences
+
+from chunksmith_cli.core.prefs_mapper import multi_indexing_preferences_from_partition
+from chunksmith_cli.partition_prefs import PdfPartitionPreferences
+from chunksmith_cli.pipelines.multi_indexing_runtime import MultiIndexingModules
+
+
+def build_multi_indexing_preferences(
+    mods: MultiIndexingModules,
+    prefs: PdfPartitionPreferences,
+    *,
+    llm_context_format: Literal["json", "toon"],
+    pdf_path: Path,
+    elements_json_path: Path | None,
+    partition_pdf: bool,
+    stream_llm_tokens: bool,
+) -> MultiIndexingPreferences:
+    """Map wizard answers to ``MultiIndexingPreferences`` for the PyPI pipeline."""
+    del pdf_path  # path is passed separately to the runner
+    elements_path = str(elements_json_path) if elements_json_path is not None else None
+    return multi_indexing_preferences_from_partition(
+        prefs,
+        mods,
+        llm_context_format=llm_context_format,
+        partition_pdf=partition_pdf,
+        elements_json_path=elements_path,
+        stream_llm_tokens=stream_llm_tokens,
+    )