chunksmith-cli 0.4.0__py3-none-any.whl

chunksmith_cli/pipelines/multi_indexing_wizard.py

@@ -0,0 +1,218 @@
+"""Unified wizard for multi-indexing (JSON or TOON via ``llm_context_format``)."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+from chunksmith_core.preferences import MultiIndexingPreferences
+from rich.console import Console
+from rich.prompt import Confirm, Prompt
+from rich.table import Table
+
+from chunksmith_cli.partition_prefs import PdfPartitionPreferences
+from chunksmith_cli.pipelines.multi_indexing_config import build_multi_indexing_preferences
+from chunksmith_cli.pipelines.multi_indexing_prompts import (
+    apply_coded_prefs,
+    apply_indexer_prefs,
+    preflight_unstructured_api,
+    prompt_artifact_root,
+    prompt_coded_formate_options,
+    prompt_element_source,
+    prompt_group_by_title_prefs,
+    prompt_indexer_options,
+    prompt_mapping_method,
+    prompt_partition_api_settings,
+    prompt_partition_prefs_base,
+    prompt_runs_artifact_default,
+)
+from chunksmith_cli.pipelines.multi_indexing_runtime import Encoding, load_multi_indexing
+from chunksmith_cli.prompts.elements_json_prompt import prompt_for_elements_json
+from chunksmith_cli.prompts.pdf_prompt import prompt_for_existing_pdf
+from chunksmith_cli.core.paths import ensure_cli_storage
+
+
+@dataclass
+class MultiIndexingCliInputs:
+    encoding: Encoding
+    pdf_path: Path
+    storage_root: Path
+    artifact_root: Path | None
+    mi_prefs: MultiIndexingPreferences
+    prefs: PdfPartitionPreferences
+    mods: Any
+    stream_events: bool
+    stream_llm_tokens: bool
+    show_toon_after: bool
+
+
+def _prompt_llm_context_format(console: Console) -> Literal["json", "toon"]:
+    console.print(
+        "\n[bold]LLM context format[/bold]\n"
+        "  [cyan]1[/cyan] — [bold]JSON[/bold] (default)\n"
+        "  [cyan]2[/cyan] — [bold]TOON[/bold] (compact LLM context)\n"
+    )
+    choice = Prompt.ask("Format", choices=["1", "2"], default="1", show_choices=True)
+    return "toon" if choice == "2" else "json"
+
+
+def _review_table(rows: list[tuple[str, str]]) -> Table:
+    from rich import box
+
+    t = Table(title="Run summary", box=box.ROUNDED, header_style="bold magenta")
+    t.add_column("Option", style="cyan", no_wrap=True)
+    t.add_column("Value", style="white")
+    for k, v in rows:
+        t.add_row(k, v)
+    return t
+
+
+def gather_multi_indexing_inputs(
+    console: Console,
+    *,
+    encoding: Encoding | None = None,
+) -> tuple[int, MultiIndexingCliInputs | None]:
+    if encoding is None:
+        encoding = _prompt_llm_context_format(console)
+
+    mods = load_multi_indexing(encoding)
+    label = "TOON" if encoding == "toon" else "JSON"
+    console.print(
+        f"\n[bold]Multi-indexing ({label} LLM context)[/bold]\n"
+        f"[dim]Library:[/dim] [cyan]{mods.package}[/cyan]\n"
+        "[dim]partition → group → coded → page index → mapper[/dim]\n"
+    )
+
+    prefs = prompt_partition_prefs_base(console)
+
+    console.print("[bold cyan]Step 1 — PDF[/bold cyan]\n")
+    pdf_path = prompt_for_existing_pdf(console)
+    if pdf_path is None:
+        console.print("[dim]Cancelled.[/dim]")
+        return 0, None
+
+    mode, partition = prompt_element_source(console)
+    elements_json_path: Path | None = None
+    element_label: str
+
+    if mode == "json":
+        elements_json_path = prompt_for_elements_json(console, show_banner=False)
+        if elements_json_path is None:
+            console.print("[dim]Cancelled.[/dim]")
+            return 0, None
+        element_label = f"JSON: {elements_json_path}"
+        artifact_root = prompt_artifact_root(console, pdf_path)
+    else:
+        if not preflight_unstructured_api(console, mods.preflight_api_partition):
+            return 1, None
+        prompt_partition_api_settings(console, prefs)
+        element_label = "Unstructured API partition"
+        if Confirm.ask("Save under chunksmith_cli/data/runs/{stem}/?", default=True):
+            artifact_root = prompt_runs_artifact_default(console, pdf_path)
+        else:
+            artifact_root = prompt_artifact_root(console, pdf_path)
+
+    console.print("\n[bold cyan]Step 3 — Grouping & coded markup[/bold cyan]\n")
+    prefs.chunksmith_use_group_by_title = Confirm.ask(
+        "Group elements by title before coding?", default=prefs.chunksmith_use_group_by_title
+    )
+    if prefs.chunksmith_use_group_by_title and Confirm.ask(
+        "[bold]Customize group-by-title chunk limits?[/bold]", default=False
+    ):
+        prompt_group_by_title_prefs(console, mods.GroupByTitlePrefs, mods.default_group_by_title_prefs, prefs)
+
+    coded, xml_mode, xml_custom = prompt_coded_formate_options(
+        console,
+        mods.CodedFormateOptions,
+        mods.default_coded_formate_options,
+        mods.ELEMENT_XML_TYPES,
+        use_group_by_title=prefs.chunksmith_use_group_by_title,
+    )
+    apply_coded_prefs(prefs, coded, xml_mode, xml_custom)
+
+    console.print("\n[bold cyan]Step 4 — Page index & mapper[/bold cyan]\n")
+    indexer_options = prompt_indexer_options(console, mods.IndexerOptions, mods.default_indexer_options)
+    prefs.summarize_outline_nodes = Confirm.ask(
+        "[bold]Summarize outline nodes[/bold] [dim](MVL post-step; CLI saves outline only)[/dim]",
+        default=prefs.summarize_outline_nodes,
+    )
+    apply_indexer_prefs(prefs, indexer_options, summarize_outline_nodes=prefs.summarize_outline_nodes)
+
+    mapping_method = prompt_mapping_method(
+        console,
+        mapping_methods=mods.MAPPING_METHODS,
+        use_group_by_title=prefs.chunksmith_use_group_by_title,
+        add_anchor=prefs.chunksmith_pageindex_add_anchor,
+    )
+    prefs.chunksmith_mapping_method = mapping_method
+    prefs.chunksmith_include_empty_mapper_fields = Confirm.ask(
+        "Include empty mapper fields?", default=prefs.chunksmith_include_empty_mapper_fields
+    )
+
+    storage_root = ensure_cli_storage()
+    stream_events = Confirm.ask("[bold]Show pipeline progress events?[/bold]", default=True)
+    stream_llm_tokens = Confirm.ask("[bold]Stream LLM tokens[/bold] to this terminal?", default=True)
+    show_toon_after = encoding == "toon" and Confirm.ask(
+        "[bold]After run, print TOON outline snippet?[/bold]", default=False
+    )
+
+    try:
+        mi_prefs = build_multi_indexing_preferences(
+            mods,
+            prefs,
+            llm_context_format=encoding,
+            pdf_path=pdf_path,
+            elements_json_path=elements_json_path,
+            partition_pdf=partition,
+            stream_llm_tokens=stream_llm_tokens,
+        )
+    except ValueError as e:
+        console.print(f"[bold red]Config error:[/bold red] {e}")
+        return 1, None
+
+    summary_rows = [
+        ("LLM context", label),
+        ("Library", mods.package),
+        ("PDF", str(pdf_path)),
+        ("Elements", element_label),
+        ("Group by title", str(prefs.chunksmith_use_group_by_title)),
+        ("Mapping", mapping_method),
+        ("element_xml mode", prefs.chunksmith_coded_element_xml_mode),
+        ("add_page_xml", str(prefs.chunksmith_coded_add_page_xml)),
+        ("add_group_by_title_xml", str(prefs.chunksmith_coded_add_group_by_title_xml)),
+        ("Partition pages/chunk", str(prefs.pages_per_chunk) if partition else "n/a"),
+        ("Max concurrent", str(prefs.max_concurrent) if partition else "n/a"),
+        ("Extract images", str(prefs.extract_images) if partition else "n/a"),
+        ("Extract tables", str(prefs.extract_tables) if partition else "n/a"),
+        ("Languages", prefs.languages if partition else "n/a"),
+        ("add_summary", str(prefs.effective_pageindex_add_outline_summaries())),
+        ("add_anchor", str(prefs.chunksmith_pageindex_add_anchor)),
+        ("probe_first_excerpt", str(prefs.chunksmith_probe_first_excerpt)),
+        ("probe_merged_reflection", str(prefs.chunksmith_probe_merged_reflection)),
+        ("Artifact root", str(artifact_root) if artifact_root else "(legacy flat save)"),
+        ("Storage", str(storage_root.resolve())),
+        ("PAGEINDEX_MODEL", os.getenv("PAGEINDEX_MODEL", "(default)")),
+        ("Stream LLM tokens", str(stream_llm_tokens)),
+        ("Stream pipeline events", str(stream_events)),
+    ]
+    console.print()
+    console.print(_review_table(summary_rows))
+
+    if not Confirm.ask("\n[bold]Start run[/bold]?", default=True):
+        console.print("[dim]Cancelled.[/dim]")
+        return 0, None
+
+    return 0, MultiIndexingCliInputs(
+        encoding=encoding,
+        pdf_path=pdf_path,
+        storage_root=storage_root,
+        artifact_root=artifact_root,
+        mi_prefs=mi_prefs,
+        prefs=prefs,
+        mods=mods,
+        stream_events=stream_events,
+        stream_llm_tokens=stream_llm_tokens,
+        show_toon_after=show_toon_after,
+    )

chunksmith_cli/pipelines/pageindex_wizard.py

@@ -0,0 +1,140 @@
+"""Interactive wizard for PageIndexer (PDF parser + LLM outline)."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from chunksmith_pageindex import load_settings
+from rich.console import Console
+from rich.prompt import Confirm, Prompt
+from rich.table import Table
+
+from chunksmith_cli.core.paths import ensure_cli_storage
+from chunksmith_cli.partition_prefs import PdfPartitionPreferences
+from chunksmith_cli.pipelines.multi_indexing_prompts import prompt_artifact_root, prompt_runs_artifact_default
+from chunksmith_cli.prompts.pdf_prompt import prompt_for_existing_pdf
+
+
+@dataclass
+class PageIndexCliInputs:
+    pdf_path: Path
+    storage_root: Path
+    artifact_root: Path | None
+    settings: object
+    add_summary: bool
+    add_word_range: bool
+    add_text: bool
+    assign_node_ids: bool
+    generate_doc_summary: bool | None
+    pdf_parser: str | None
+    stream_progress: bool
+
+
+def _review_table(rows: list[tuple[str, str]]) -> Table:
+    from rich import box
+
+    t = Table(title="PageIndexer run summary", box=box.ROUNDED, header_style="bold magenta")
+    t.add_column("Option", style="cyan", no_wrap=True)
+    t.add_column("Value", style="white")
+    for k, v in rows:
+        t.add_row(k, v)
+    return t
+
+
+def gather_pageindex_inputs(console: Console) -> tuple[int, PageIndexCliInputs | None]:
+    console.print(
+        "\n[bold]PageIndexer[/bold]\n"
+        "[dim]Library:[/dim] [cyan]chunksmith-pageindex[/cyan]\n"
+        "[dim]PDF parser → tagged pages → LLM outline (no Unstructured partition)[/dim]\n"
+    )
+
+    console.print("[bold cyan]Step 1 — PDF[/bold cyan]\n")
+    pdf_path = prompt_for_existing_pdf(console)
+    if pdf_path is None:
+        console.print("[dim]Cancelled.[/dim]")
+        return 0, None
+
+    console.print("\n[bold cyan]Step 2 — Outline options[/bold cyan]\n")
+    prefs = PdfPartitionPreferences()
+    add_summary = Confirm.ask(
+        "[bold]Per-section summaries[/bold] [dim](add_summary)[/dim]",
+        default=prefs.effective_pageindex_add_outline_summaries(),
+    )
+    add_word_range = Confirm.ask(
+        "[bold]Split document anchors[/bold] [dim](add_word_range / anchors)[/dim]",
+        default=prefs.chunksmith_pageindex_add_anchor,
+    )
+    add_text = Confirm.ask(
+        "[bold]Attach section text to nodes[/bold] [dim](add_text)[/dim]",
+        default=prefs.chunksmith_pageindex_add_text,
+    )
+    assign_node_ids = Confirm.ask(
+        "[bold]Assign node IDs[/bold] [dim](for retriever / agent)[/dim]",
+        default=prefs.chunksmith_pageindex_assign_node_ids,
+    )
+    gen_doc: bool | None = None
+    if Confirm.ask("[bold]Generate document description?[/bold] [dim](extra LLM call)[/dim]", default=False):
+        gen_doc = True
+
+    parser_default = os.getenv("CHUNKSMITH_PDF_PARSER", "PyPDF2")
+    pdf_parser = Prompt.ask(
+        "PDF parser",
+        choices=["PyPDF2", "PyMuPDF"],
+        default=parser_default if parser_default in ("PyPDF2", "PyMuPDF") else "PyPDF2",
+    )
+
+    console.print("\n[bold cyan]Step 3 — Save location[/bold cyan]\n")
+    if Confirm.ask("Save under chunksmith_cli/data/runs/{stem}/?", default=True):
+        artifact_root = prompt_runs_artifact_default(console, pdf_path)
+    else:
+        artifact_root = prompt_artifact_root(console, pdf_path)
+
+    storage_root = ensure_cli_storage()
+    stream_progress = Confirm.ask("[bold]Show progress messages?[/bold]", default=True)
+
+    overrides: dict[str, object] = {"pdf_parser": pdf_parser}
+    if gen_doc is not None:
+        overrides["generate_doc_summary"] = gen_doc
+    try:
+        settings = load_settings(defaults=overrides)
+    except ValueError as e:
+        console.print(f"[bold red]Settings error:[/bold red] {e}")
+        return 1, None
+
+    summary_rows = [
+        ("Pipeline", "PageIndexer"),
+        ("PDF", str(pdf_path)),
+        ("PDF parser", pdf_parser),
+        ("add_summary", str(add_summary)),
+        ("add_word_range", str(add_word_range)),
+        ("add_text", str(add_text)),
+        ("assign_node_ids", str(assign_node_ids)),
+        ("generate_doc_summary", str(gen_doc if gen_doc is not None else settings.generate_doc_summary)),
+        ("Artifact root", str(artifact_root) if artifact_root else "(legacy flat save)"),
+        ("Storage", str(storage_root.resolve())),
+        ("PAGEINDEX_MODEL", os.getenv("PAGEINDEX_MODEL", settings.pageindex_model)),
+        ("Max tokens/chunk", str(settings.max_tokens_per_chunk)),
+        ("Overlap pages", str(settings.overlap_pages)),
+    ]
+    console.print()
+    console.print(_review_table(summary_rows))
+
+    if not Confirm.ask("\n[bold]Start run[/bold]?", default=True):
+        console.print("[dim]Cancelled.[/dim]")
+        return 0, None
+
+    return 0, PageIndexCliInputs(
+        pdf_path=pdf_path,
+        storage_root=storage_root,
+        artifact_root=artifact_root,
+        settings=settings,
+        add_summary=add_summary,
+        add_word_range=add_word_range,
+        add_text=add_text,
+        assign_node_ids=assign_node_ids,
+        generate_doc_summary=gen_doc,
+        pdf_parser=pdf_parser,
+        stream_progress=stream_progress,
+    )

chunksmith_cli/pipelines/run_multi.py

@@ -0,0 +1,21 @@
+"""Run multi-indexing via published ``chunksmith-multimodal`` APIs."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Callable
+
+from chunksmith_core.preferences import MultiIndexingPreferences
+from chunksmith_multimodal import run_multi_indexing_with_preferences
+
+ProgressCallback = Callable[[str, dict[str, Any]], None]
+
+
+def run_multi_indexing(
+    pdf_path: Path,
+    prefs: MultiIndexingPreferences,
+    *,
+    progress: ProgressCallback | None = None,
+) -> Any:
+    """Execute the library pipeline with optional Rich progress callback."""
+    return run_multi_indexing_with_preferences(pdf_path, prefs, progress=progress)

chunksmith_cli/prompts/__init__.py

@@ -0,0 +1 @@
+"""Shared Rich prompts for pipeline wizards."""

chunksmith_cli/prompts/elements_json_prompt.py

@@ -0,0 +1,49 @@
+"""Prompt for an Unstructured partition JSON file (array of element dicts)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from rich.console import Console
+from rich.prompt import Confirm, Prompt
+
+from chunksmith_cli.paths import normalize_path_string, resolve_pdf_candidate
+
+
+def prompt_for_elements_json(console: Console, *, show_banner: bool = True) -> Path | None:
+    """
+    Path to a JSON file whose root is a list of Unstructured-like element dicts.
+
+    Cancel with an empty line.
+    """
+    if show_banner:
+        console.print(
+            "\n[bold green]Unstructured JSON[/bold green] — file with a top-level array of elements "
+            "(``type``, ``text``, ``metadata.page_number``, …).\n"
+            "[dim]Same path rules as the PDF prompt.[/dim]\n"
+        )
+    while True:
+        raw = Prompt.ask("[bold]Elements JSON path[/bold]", default="", show_default=False)
+        if not (raw or "").strip():
+            return None
+        p = resolve_pdf_candidate(normalize_path_string(raw))
+        if p.is_file():
+            if p.suffix.lower() == ".pdf":
+                console.print(
+                    "[bold red]That file is a PDF.[/bold red] This step needs the [bold]Unstructured partition JSON[/bold] "
+                    "(a ``.json`` file whose root is an array of elements), not the PDF.\n"
+                    "[dim]Partition the same PDF with Unstructured, save the element list to JSON, then enter that path.[/dim]\n"
+                )
+                continue
+            if p.suffix.lower() != ".json":
+                if not Confirm.ask(
+                    f"[yellow]Extension is {p.suffix!r}, not .json.[/yellow] Use this file anyway? "
+                    f"[dim](It must still be UTF-8 JSON with a top-level array.)[/dim]",
+                    default=False,
+                ):
+                    continue
+            return p
+        console.print(
+            f"[yellow]Could not open as a file:[/yellow] {p}\n"
+            "[dim]Try again, or Enter on an empty line to cancel.[/dim]\n"
+        )

chunksmith_cli/prompts/pdf_prompt.py

@@ -0,0 +1,37 @@
+"""Prompt for an existing PDF path (ChunkSmith PDF CLI flow)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from rich.console import Console
+from rich.prompt import Confirm, Prompt
+
+from chunksmith_cli.paths import normalize_path_string, resolve_pdf_candidate
+
+
+def prompt_for_existing_pdf(console: Console) -> Path | None:
+    """
+    Accept flexible path input; re-prompt until a readable file is chosen or the user cancels.
+    """
+    console.print(
+        '[dim]Paths: plain, in "quotes", ~ home, relative to this folder, or file:///… URLs. '
+        "Press Enter on an empty line to cancel.[/dim]\n"
+    )
+    while True:
+        raw = Prompt.ask("[bold]PDF path[/bold]", default="", show_default=False)
+        if not (raw or "").strip():
+            return None
+        p = resolve_pdf_candidate(normalize_path_string(raw))
+        if p.is_file():
+            if p.suffix.lower() != ".pdf":
+                if not Confirm.ask(
+                    f"[yellow]Extension is {p.suffix!r}, not .pdf.[/yellow] Use this file anyway?",
+                    default=False,
+                ):
+                    continue
+            return p
+        console.print(
+            f"[yellow]Could not open as a file:[/yellow] {p}\n"
+            "[dim]Check the path and try again, or Enter on an empty line to cancel.[/dim]\n"
+        )

chunksmith_cli/saved_catalog.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.core.saved_catalog`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.core.saved_catalog")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/theme.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.core.theme`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.core.theme")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/tree_view.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.views.tree_view`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.views.tree_view")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/view_session.py

@@ -0,0 +1,6 @@
+"""Backward-compatible import path. Prefer `cli.views.view_session`."""
+
+import importlib
+
+_mod = importlib.import_module("chunksmith_cli.views.view_session")
+globals().update({k: v for k, v in _mod.__dict__.items() if not k.startswith("__")})

chunksmith_cli/views/__init__.py

@@ -0,0 +1 @@
+"""Read-only viewers: outline tree, element lookup, explorer."""

chunksmith_cli/views/json_view.py

@@ -0,0 +1,11 @@
+"""View saved outline JSON — delegates to :func:`outline_browser.run_outline_browser_session`."""
+
+from __future__ import annotations
+
+from rich.console import Console
+
+from chunksmith_cli.outline_browser import run_outline_browser_session
+
+
+def run_saved_json_view(console: Console) -> int:
+    return run_outline_browser_session(console)