lumberjack-py 0.4.0__py3-none-any.whl

lumberjack/__init__.py

@@ -0,0 +1,4 @@
+from .lumberjack import Lumberjack
+from .models import Document
+
+__all__ = ["Document", "Lumberjack"]

lumberjack/_internal/__init__.py

@@ -0,0 +1 @@
+"""Private implementation helpers for Lumberjack's public components."""

lumberjack/_internal/block_splitter.py

@@ -0,0 +1,504 @@
+"""Internal oversized-block splitting helpers."""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from lumberjack.block import (
+    BlockOption,
+    HTMLTableConfig,
+    MarkdownTableConfig,
+    default_block_config,
+)
+
+from ..parser.html.table_parser import HTMLTableParser, HTMLTableRow
+
+if TYPE_CHECKING:
+    from ..models import DocumentBlock
+    from ..protocols import TokenizerProtocol
+
+SENTENCE_BREAK_RE = re.compile(r"(?<=[.!?\u3002\uff01\uff1f])\s+")
+PROTECTED_SPAN_RE = re.compile(r"<https?://[^\s>]+>|https?://[^\s)>\]]+")
+TABLE_DELIMITER_CELL_RE = re.compile(r":?-+(:?-+)*:?")
+
+
+class BlockSplitter:
+    """Splits oversized text blocks into token-bounded pieces."""
+
+    def __init__(
+        self,
+        tokenizer: TokenizerProtocol,
+        *,
+        max_tokens: int,
+        block_options: dict[str, BlockOption],
+    ) -> None:
+        self.tokenizer = tokenizer
+        self.max_tokens = max_tokens
+        self.block_options = block_options
+        self._html_table_parser = HTMLTableParser()
+
+    def split_oversized_block(
+        self,
+        block: DocumentBlock,
+        *,
+        default_budget: int,
+    ) -> list[tuple[str, int]] | None:
+        config = self._block_config(block.kind)
+        if config is None or not config.split:
+            return None
+
+        budget = self._block_budget(block.kind, default_budget)
+
+        if block.kind in {"code_block", "code_fence"}:
+            return self.split_code_block(block, max_tokens=budget)
+
+        if block.kind == "list" and block.children:
+            return self.split_list_block(block, max_tokens=budget)
+
+        if block.kind == "table":
+            return self.split_table_block(block, default_budget=default_budget)
+
+        if block.kind == "html_table":
+            return self.split_html_table_block(
+                block,
+                default_budget=default_budget,
+            )
+
+        return self.split_text(block.text, max_tokens=budget)
+
+    def _block_config(self, kind: str) -> BlockOption:
+        return self.block_options.get(kind.lower(), default_block_config(kind))
+
+    def _block_budget(self, kind: str, default_budget: int | None = None) -> int:
+        config = self._block_config(kind)
+        if config and config.max_tokens:
+            return config.max_tokens
+        if default_budget is not None:
+            return default_budget
+        return self.max_tokens
+
+    def _repeat_header(self, kind: str) -> bool:
+        config = self._block_config(kind)
+        return (
+            not isinstance(config, MarkdownTableConfig | HTMLTableConfig)
+            or config.repeat_header
+        )
+
+    def split_code_block(
+        self,
+        block: DocumentBlock,
+        *,
+        max_tokens: int,
+    ) -> list[tuple[str, int]]:
+        info = str(block.attrs.get("info") or block.attrs.get("language") or "").strip()
+        literal = str(block.attrs.get("literal") or "")
+        open_fence = f"```{info}".rstrip()
+        close_fence = "```"
+        empty_render = f"{open_fence}\n\n{close_fence}"
+        wrapper_tokens = self.tokenizer.count(empty_render, cache=True)
+        if wrapper_tokens >= max_tokens:
+            return [(block.text, self.tokenizer.count(block.text, cache=True))]
+
+        code_budget = max_tokens - wrapper_tokens
+        pieces = self.split_text(literal, max_tokens=code_budget)
+        result: list[tuple[str, int]] = []
+        for piece, _piece_tokens in pieces:
+            wrapped = f"{open_fence}\n{piece}\n{close_fence}"
+            # Fences add tokens beyond the literal, so recount the wrapped text.
+            result.append((wrapped, self.tokenizer.count(wrapped, cache=True)))
+        return result
+
+    def split_table_block(
+        self,
+        block: DocumentBlock,
+        *,
+        default_budget: int | None = None,
+    ) -> list[tuple[str, int]]:
+        # Handle markdown table
+        lines = [line.rstrip() for line in block.text.splitlines() if line.strip()]
+        max_tokens = self._block_budget(block.kind, default_budget)
+        repeat_header = self._repeat_header(block.kind)
+        if len(lines) < 3 or not self.is_table_delimiter_row(lines[1]):
+            return self.split_text(block.text, max_tokens=max_tokens)
+
+        def emit_piece(piece_header: list[str], rows: list[str]) -> tuple[str, int]:
+            piece = self.render_table_piece(piece_header, rows)
+            return (piece, self.tokenizer.count(piece, cache=True))
+
+        header = lines[:2]
+        rows = lines[2:]
+        pieces: list[tuple[str, int]] = []
+        current_rows: list[str] = []
+
+        for row in rows:
+            candidate_rows = [*current_rows, row]
+            candidate_header = header if repeat_header or not pieces else []
+            candidate = self.render_table_piece(candidate_header, candidate_rows)
+            candidate_tokens = self.tokenizer.count(candidate, cache=True)
+            if current_rows and candidate_tokens > max_tokens:
+                piece_header = header if repeat_header or not pieces else []
+                pieces.append(emit_piece(piece_header, current_rows))
+                current_rows = [row]
+                single_header = header if repeat_header or not pieces else []
+                single_row = self.render_table_piece(single_header, current_rows)
+                single_tokens = self.tokenizer.count(single_row, cache=True)
+                if single_tokens > max_tokens:
+                    pieces.append((single_row, single_tokens))
+                    current_rows = []
+                continue
+
+            if not current_rows and candidate_tokens > max_tokens:
+                pieces.append((candidate, candidate_tokens))
+                continue
+
+            current_rows = candidate_rows
+
+        if current_rows:
+            piece_header = header if repeat_header or not pieces else []
+            pieces.append(emit_piece(piece_header, current_rows))
+
+        return pieces or [(block.text, self.tokenizer.count(block.text, cache=True))]
+
+    def is_table_delimiter_row(self, line: str) -> bool:
+        cells = [cell.strip() for cell in line.strip().strip("|").split("|")]
+        return bool(cells) and all(
+            cell and TABLE_DELIMITER_CELL_RE.fullmatch(cell) for cell in cells
+        )
+
+    def render_table_piece(self, header: list[str], rows: list[str]) -> str:
+        return "\n".join([*header, *rows])
+
+    def split_html_table_block(
+        self,
+        block: DocumentBlock,
+        *,
+        default_budget: int | None = None,
+    ) -> list[tuple[str, int]]:
+        """Split an HTML table while preserving its original HTML format.
+
+        This method extracts HTML tables and splits them by rows while keeping
+        the HTML structure intact, without converting to markdown format.
+        """
+        max_tokens = self._block_budget(block.kind, default_budget)
+        repeat_header = self._repeat_header(block.kind)
+        tables = self._html_table_parser.extract_tables(block.text)
+        if not tables:
+            return [(block.text, self.tokenizer.count(block.text, cache=True))]
+
+        def emit(html: str) -> tuple[str, int]:
+            return (html, self.tokenizer.count(html, cache=True))
+
+        pieces: list[tuple[str, int]] = []
+        for html_table in tables:
+            # Get the raw HTML content
+            table_html = html_table.raw_html
+
+            # Extract the opening <table> tag with all attributes
+            table_open_tag = ""
+            table_match = re.search(r"<table\b[^>]*>", table_html, re.IGNORECASE)
+            if table_match:
+                table_open_tag = table_match.group(0)
+
+            # Extract caption if present
+            caption_html = ""
+            if html_table.caption:
+                caption_match = self._html_table_parser.CAPTION_RE.search(table_html)
+                if caption_match:
+                    caption_html = caption_match.group(0)
+
+            # Split by rows while preserving HTML structure
+            header_rows = list(html_table.headers)
+            data_rows = list(html_table.rows)
+
+            if not data_rows:
+                pieces.append(emit(table_html))
+                continue
+
+            # Group rows by token budget
+            current_rows: list[HTMLTableRow] = []
+            pieces_count = 0
+
+            for row in data_rows:
+                test_rows = [*current_rows, row]
+                # Build test HTML to check token count
+                candidate_headers = (
+                    header_rows if repeat_header or pieces_count == 0 else []
+                )
+                test_html = self._build_html_table_piece(
+                    table_open_tag, caption_html, candidate_headers, test_rows
+                )
+                test_tokens = self.tokenizer.count(test_html, cache=True)
+
+                if current_rows and test_tokens > max_tokens:
+                    # Emit current group. ``piece_html`` differs from
+                    # ``test_html`` (it drops the candidate row), so it needs
+                    # its own count — but its headers share the same
+                    # ``pieces_count == 0`` decision as ``candidate_headers``.
+                    piece_headers = (
+                        header_rows if repeat_header or pieces_count == 0 else []
+                    )
+                    piece_html = self._build_html_table_piece(
+                        table_open_tag, caption_html, piece_headers, current_rows
+                    )
+                    pieces.append(emit(piece_html))
+                    current_rows = [row]
+                    pieces_count += 1
+                elif not current_rows and test_tokens > max_tokens:
+                    # Single row exceeds budget, emit ``test_html`` as is —
+                    # reuse the count we just computed for the budget check.
+                    pieces.append((test_html, test_tokens))
+                    current_rows = []
+                    pieces_count += 1
+                else:
+                    current_rows.append(row)
+
+            # Don't forget remaining rows
+            if current_rows:
+                piece_headers = (
+                    header_rows if repeat_header or pieces_count == 0 else []
+                )
+                piece_html = self._build_html_table_piece(
+                    table_open_tag, caption_html, piece_headers, current_rows
+                )
+                pieces.append(emit(piece_html))
+                pieces_count += 1
+
+        return (
+            pieces
+            if pieces
+            else [(block.text, self.tokenizer.count(block.text, cache=True))]
+        )
+
+    def _build_html_table_piece(
+        self,
+        table_open_tag: str,
+        caption_html: str,
+        header_rows: list[HTMLTableRow],
+        data_rows: list[HTMLTableRow],
+    ) -> str:
+        """Build a complete HTML table piece from components.
+
+        Args:
+            table_open_tag: Complete opening <table> tag with attributes.
+            caption_html: Raw HTML caption string.
+            header_rows: List of header row objects.
+            data_rows: List of data row objects to include.
+
+        Returns:
+            Complete HTML table string.
+        """
+        lines: list[str] = [table_open_tag if table_open_tag else "<table>"]
+
+        # Add caption
+        if caption_html:
+            lines.append(caption_html)
+
+        # Add header rows
+        for header_row in header_rows:
+            lines.append(header_row.raw_html)
+
+        # Add data rows
+        for data_row in data_rows:
+            lines.append(data_row.raw_html)
+
+        lines.append("</table>")
+        return "\n".join(lines)
+
+    def split_list_block(
+        self,
+        block: DocumentBlock,
+        *,
+        max_tokens: int,
+    ) -> list[tuple[str, int]]:
+        items = [child.text for child in block.children if child.text]
+        if len(items) <= 1:
+            return self.split_text(
+                block.text,
+                max_tokens=max_tokens,
+            )
+
+        packed = self.pack_parts(
+            items,
+            max_tokens,
+            separator="\n",
+        )
+        if all(tokens <= max_tokens for _, tokens in packed):
+            return packed
+
+        pieces: list[tuple[str, int]] = []
+        for item in items:
+            item_tokens = self.tokenizer.count(item, cache=True)
+            if item_tokens <= max_tokens:
+                pieces.append((item, item_tokens))
+                continue
+            pieces.extend(
+                self.split_text(
+                    item,
+                    max_tokens=max_tokens,
+                )
+            )
+        return pieces
+
+    def split_text(
+        self,
+        text: str,
+        *,
+        max_tokens: int,
+    ) -> list[tuple[str, int]]:
+        text_tokens = self.tokenizer.count(text, cache=True)
+        if text_tokens <= max_tokens:
+            return [(text, text_tokens)]
+
+        if any(
+            self.tokenizer.count(m.group(0), cache=True) > max_tokens
+            for m in PROTECTED_SPAN_RE.finditer(text)
+        ):
+            return [(text, text_tokens)]
+
+        for separator in ("\n\n", "\n"):
+            parts = [part.strip() for part in text.split(separator) if part.strip()]
+            if len(parts) > 1:
+                packed = self._pack_fitting_parts(
+                    parts,
+                    max_tokens,
+                    separator=separator,
+                )
+                if packed is not None:
+                    return packed
+
+        sentence_parts = [
+            part.strip() for part in SENTENCE_BREAK_RE.split(text) if part.strip()
+        ]
+        if len(sentence_parts) > 1:
+            packed = self._pack_fitting_parts(
+                sentence_parts,
+                max_tokens,
+                separator=" ",
+            )
+            if packed is not None:
+                return packed
+
+        word_parts = [part for part in text.split(" ") if part]
+        if len(word_parts) > 1:
+            packed = self._pack_fitting_parts(
+                word_parts,
+                max_tokens,
+                separator=" ",
+            )
+            if packed is not None:
+                return packed
+
+        return self.hard_split(text, max_tokens)
+
+    def _pack_fitting_parts(
+        self,
+        parts: list[str],
+        max_tokens: int,
+        *,
+        separator: str,
+    ) -> list[tuple[str, int]] | None:
+        """Pack one fallback level only when every atomic part fits."""
+        part_token_counts = [self.tokenizer.count(part, cache=True) for part in parts]
+        if any(tokens > max_tokens for tokens in part_token_counts):
+            return None
+        return self.pack_parts(
+            parts,
+            max_tokens,
+            separator=separator,
+            part_token_counts=part_token_counts,
+        )
+
+    def pack_parts(
+        self,
+        parts: list[str],
+        max_tokens: int,
+        *,
+        separator: str,
+        part_token_counts: list[int] | None = None,
+    ) -> list[tuple[str, int]]:
+        if part_token_counts is not None and len(part_token_counts) != len(parts):
+            raise ValueError("part_token_counts must match parts")
+
+        packed: list[tuple[str, int]] = []
+        current_parts: list[str] = []
+        current_joined = ""
+        current_tokens = 0
+        for index, part in enumerate(parts):
+            candidate_text = (
+                current_joined + separator + part if current_joined else part
+            )
+            candidate_tokens = self.tokenizer.count(candidate_text, cache=True)
+            if current_parts and candidate_tokens > max_tokens:
+                packed.append((current_joined, current_tokens))
+                current_parts = [part]
+                current_joined = part
+                current_tokens = (
+                    part_token_counts[index]
+                    if part_token_counts is not None
+                    else self.tokenizer.count(part, cache=True)
+                )
+            else:
+                current_parts.append(part)
+                current_joined = candidate_text
+                current_tokens = candidate_tokens
+        if current_parts:
+            packed.append((current_joined, current_tokens))
+        return packed
+
+    def hard_split(
+        self,
+        text: str,
+        max_tokens: int,
+    ) -> list[tuple[str, int]]:
+        """Split with bounded prefix searches instead of growing recounts."""
+        result: list[tuple[str, int]] = []
+        start = 0
+
+        while start < len(text):
+            lower = start
+            step = 1
+            upper = min(len(text), start + step)
+
+            while upper < len(text):
+                if self.tokenizer.count(text[start:upper], cache=True) > max_tokens:
+                    break
+                lower = upper
+                step *= 2
+                upper = min(len(text), start + step)
+
+            if upper == len(text) and (
+                self.tokenizer.count(text[start:upper], cache=True) <= max_tokens
+            ):
+                lower = upper
+
+            if lower == start and (
+                self.tokenizer.count(text[start : start + 1], cache=True) > max_tokens
+            ):
+                lower = start + 1
+
+            if lower < upper and lower < len(text):
+                left = max(start + 1, lower + 1)
+                right = upper
+                best = lower
+                while left <= right:
+                    middle = (left + right) // 2
+                    if (
+                        self.tokenizer.count(text[start:middle], cache=True)
+                        <= max_tokens
+                    ):
+                        best = middle
+                        left = middle + 1
+                    else:
+                        right = middle - 1
+                lower = best
+
+            if lower == start:
+                lower = start + 1
+
+            piece = text[start:lower].strip()
+            if piece:
+                result.append((piece, self.tokenizer.count(piece, cache=True)))
+            start = lower
+
+        return result

lumberjack/_internal/formats.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+SUPPORTED_FORMATS = frozenset({"auto", "markdown", "html", "docx"})
+TEXT_FORMATS = frozenset({"markdown", "html"})
+
+
+def detect_format(source: str | bytes | Path, format: str) -> str:
+    """Resolve the input format from an explicit hint and source shape."""
+    if format not in SUPPORTED_FORMATS:
+        msg = f"Unsupported input format: {format}"
+        raise ValueError(msg)
+
+    if format != "auto":
+        return format
+
+    if isinstance(source, bytes):
+        return "docx"
+
+    if isinstance(source, Path):
+        return detect_format_from_filename(source.name)
+
+    return "markdown"
+
+
+def detect_format_from_filename(filename: str) -> str:
+    """Detect an input format from a filename extension."""
+    suffix = Path(filename).suffix.lower()
+    if suffix == ".docx":
+        return "docx"
+    if suffix in {".html", ".htm"}:
+        return "html"
+    return "markdown"
+
+
+def read_text_input(source: str | bytes | Path) -> str:
+    """Read Markdown or HTML text from any supported source shape."""
+    if isinstance(source, Path):
+        return source.read_text(encoding="utf-8")
+    if isinstance(source, bytes):
+        return source.decode("utf-8")
+    return source
+
+
+def read_docx_input(source: str | bytes | Path) -> bytes:
+    """Read DOCX binary content from any supported source shape."""
+    if isinstance(source, Path):
+        return source.read_bytes()
+    if isinstance(source, str):
+        raise TypeError(
+            "Expected bytes or a .docx file path for DOCX format, got a text string. "
+            "Pass a Path or bytes instead."
+        )
+    return source

lumberjack/_internal/options.py

@@ -0,0 +1,112 @@
+"""CLI and Web adapters for public block configuration objects."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from typing import Any
+
+from lumberjack.block import (
+    BlockConfig,
+    BlockKind,
+    BlockOption,
+    CustomBlockConfig,
+    HTMLTableConfig,
+    MarkdownTableConfig,
+)
+
+BASE_FIELDS = frozenset({"isolated", "split", "max_tokens"})
+TABLE_FIELDS = frozenset({"repeat_header"})
+
+
+def block_config_from_mapping(kind: str, config: Mapping[str, Any]) -> BlockOption:
+    """Convert one external mapping into a typed public block config."""
+    normalized = kind.strip().lower()
+    is_table = normalized in {BlockKind.TABLE, BlockKind.HTML_TABLE}
+    valid_fields = BASE_FIELDS | (TABLE_FIELDS if is_table else frozenset())
+    unknown = set(config) - valid_fields
+    if unknown:
+        names = ", ".join(sorted(unknown))
+        valid = ", ".join(sorted(valid_fields))
+        raise ValueError(
+            f"Unknown block config field(s) for {kind!r}: {names}. Valid fields: {valid}"
+        )
+    base = {name: config[name] for name in BASE_FIELDS if name in config}
+    if normalized == BlockKind.TABLE:
+        return MarkdownTableConfig(
+            **base, repeat_header=config.get("repeat_header", True)
+        )
+    if normalized == BlockKind.HTML_TABLE:
+        return HTMLTableConfig(**base, repeat_header=config.get("repeat_header", True))
+    try:
+        return BlockConfig(BlockKind(normalized), **base)
+    except ValueError:
+        return CustomBlockConfig(normalized, **base)
+
+
+def parse_block_config_mapping(
+    raw: Mapping[str, Any] | None,
+) -> list[BlockOption] | None:
+    if raw is None:
+        return None
+    result: list[BlockOption] = []
+    for kind, config in raw.items():
+        if not isinstance(config, Mapping):
+            raise TypeError(f"block_configs[{kind!r}] must be an object")
+        result.append(block_config_from_mapping(kind, config))
+    return result
+
+
+def parse_block_config_json(raw: str) -> list[BlockOption] | None:
+    if not raw or not raw.strip():
+        return None
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise ValueError("Invalid block_configs JSON") from exc
+    if not isinstance(parsed, Mapping):
+        raise TypeError("block_configs must be a JSON object")
+    return parse_block_config_mapping(parsed)
+
+
+def _parse_cli_entry(entry: str, block_kinds: frozenset[str]) -> BlockOption:
+    parts = [part.strip() for part in entry.split(":")]
+    kind = parts[0].lower() if parts else ""
+    if not kind:
+        raise ValueError("block config kind cannot be empty")
+    if kind not in block_kinds:
+        valid = ", ".join(sorted(block_kinds))
+        raise ValueError(f"Unknown block kind: {kind!r} (valid: {valid})")
+
+    config: dict[str, object] = {}
+    for token in parts[1:]:
+        lowered = token.lower()
+        if not token:
+            continue
+        if lowered == "isolated":
+            config["isolated"] = True
+        elif lowered == "nosplit":
+            config["split"] = False
+        else:
+            try:
+                config["max_tokens"] = int(token)
+            except ValueError as exc:
+                raise ValueError(f"Unknown block config token: {token!r}") from exc
+    return block_config_from_mapping(kind, config)
+
+
+def parse_cli_block_configs(
+    entries: list[str],
+    *,
+    block_kinds: frozenset[str],
+    json_config: str = "",
+) -> list[BlockOption]:
+    """Parse CLI config, with JSON entries overriding short-form entries."""
+    indexed = {
+        str(config.kind): config
+        for config in (_parse_cli_entry(entry, block_kinds) for entry in entries)
+    }
+    json_options = parse_block_config_json(json_config)
+    if json_options:
+        indexed.update((str(config.kind), config) for config in json_options)
+    return list(indexed.values())