langparse 0.1.0__py3-none-any.whl

langparse/__init__.py

@@ -0,0 +1,55 @@
+from importlib.metadata import version as _distribution_version
+
+__version__ = _distribution_version("langparse")
+
+from langparse.autoparser import AutoParser
+from langparse.chunkers import (
+    FixedTokenChunker,
+    SemanticChunker,
+    SlidingWindowChunker,
+    available_chunkers,
+    create_chunker,
+    register_chunker,
+)
+from langparse.core.chunker import BaseChunker
+from langparse.core.parser import BaseParser
+from langparse.metrics import BatchItemResult, BatchRunResult, ParseMetrics
+from langparse.parsers.docx_parser import DocxParser
+from langparse.parsers.excel_parser import ExcelParser
+from langparse.parsers.markdown_parser import MarkdownParser
+from langparse.parsers.pdf_parser import PDFParser
+from langparse.progress import ProgressCallback, ProgressEvent
+from langparse.types import (
+    Chunk,
+    Document,
+    ParsedDocumentResult,
+    ParsedElement,
+    ParsedPageResult,
+)
+
+__all__ = [
+    "__version__",
+    "Document",
+    "Chunk",
+    "ParsedDocumentResult",
+    "ParsedPageResult",
+    "ParsedElement",
+    "BaseParser",
+    "BaseChunker",
+    "AutoParser",
+    "PDFParser",
+    "MarkdownParser",
+    "DocxParser",
+    "ExcelParser",
+    "SemanticChunker",
+    "FixedTokenChunker",
+    "SlidingWindowChunker",
+    "available_chunkers",
+    "create_chunker",
+    "register_chunker",
+    "ParseMetrics",
+    "ProgressCallback",
+    "ProgressEvent",
+    "BatchItemResult",
+    "BatchRunResult",
+]

langparse/autoparser.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+from langparse.core.rendering import document_from_result
+from langparse.types import Document, ParsedDocumentResult
+
+
+class AutoParser:
+    """
+    Facade that parses any supported file without the caller picking a parser.
+
+    Extension routing lives in `ParseService`, driven by
+    `langparse.parsers.registry`, so this stays a convenience wrapper rather
+    than a second place formats can be registered and drift.
+    """
+
+    @staticmethod
+    def parse_result(file_path: str | Path, **kwargs) -> ParsedDocumentResult:
+        from langparse.services.parse_service import ParseService
+
+        engine_name = kwargs.pop("engine", None) or "simple"
+        return ParseService().parse_result(file_path, engine_name=engine_name, **kwargs)
+
+    @staticmethod
+    def parse(file_path: str | Path, **kwargs) -> Document:
+        return document_from_result(AutoParser.parse_result(file_path, **kwargs))

langparse/chunkers/__init__.py

@@ -0,0 +1,12 @@
+from langparse.chunkers.registry import available_chunkers, create_chunker, register_chunker
+from langparse.chunkers.semantic import SemanticChunker
+from langparse.chunkers.text import FixedTokenChunker, SlidingWindowChunker
+
+__all__ = [
+    "FixedTokenChunker",
+    "SlidingWindowChunker",
+    "SemanticChunker",
+    "available_chunkers",
+    "create_chunker",
+    "register_chunker",
+]

langparse/chunkers/blocks.py

@@ -0,0 +1,151 @@
+"""
+Scan Markdown into typed blocks.
+
+Chunking used to run a heading regex over the whole document, which cannot tell
+a real heading from a `#` comment inside a fenced code block -- recognising a
+fence requires tracking state across lines. Scanning into typed blocks fixes
+that structurally, and the same block types are what let the packer treat
+tables and code differently when they overflow a chunk.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
+FENCE_RE = re.compile(r"^(\s*)(`{3,}|~{3,})(.*)$")
+PAGE_MARKER_RE = re.compile(r"^\s*<!--\s*page_number:\s*(\d+)\s*-->\s*$")
+TABLE_SEPARATOR_RE = re.compile(r"^\s*\|?\s*:?-{3,}:?\s*(\|\s*:?-{3,}:?\s*)*\|?\s*$")
+
+HEADING = "heading"
+TABLE = "table"
+CODE = "code"
+PARAGRAPH = "paragraph"
+PAGE_MARKER = "page_marker"
+
+
+@dataclass
+class Block:
+    kind: str
+    text: str
+    #: heading only
+    level: int = 0
+    title: str = ""
+    #: table only
+    rows: list[list[str]] = field(default_factory=list)
+    has_header: bool = False
+    #: page_marker only
+    page_number: int = 0
+
+
+def scan_blocks(markdown: str) -> list[Block]:
+    """Split Markdown into an ordered list of typed blocks."""
+    if not markdown:
+        return []
+
+    lines = markdown.splitlines()
+    blocks: list[Block] = []
+    index = 0
+
+    while index < len(lines):
+        line = lines[index]
+
+        if not line.strip():
+            index += 1
+            continue
+
+        fence = FENCE_RE.match(line)
+        if fence:
+            index = _consume_fence(lines, index, fence.group(2), blocks)
+            continue
+
+        page_marker = PAGE_MARKER_RE.match(line)
+        if page_marker:
+            blocks.append(Block(kind=PAGE_MARKER, text=line, page_number=int(page_marker.group(1))))
+            index += 1
+            continue
+
+        heading = HEADING_RE.match(line)
+        if heading:
+            blocks.append(
+                Block(
+                    kind=HEADING,
+                    text=line,
+                    level=len(heading.group(1)),
+                    title=heading.group(2),
+                )
+            )
+            index += 1
+            continue
+
+        if _starts_table(lines, index):
+            index = _consume_table(lines, index, blocks)
+            continue
+
+        index = _consume_paragraph(lines, index, blocks)
+
+    return blocks
+
+
+def _consume_fence(lines: list[str], index: int, marker: str, blocks: list[Block]) -> int:
+    """Consume a fenced block. An unclosed fence runs to end of input."""
+    fence_char = marker[0]
+    collected = [lines[index]]
+    index += 1
+
+    while index < len(lines):
+        collected.append(lines[index])
+        closing = FENCE_RE.match(lines[index])
+        index += 1
+        if closing and closing.group(2)[0] == fence_char and len(closing.group(2)) >= len(marker):
+            break
+
+    blocks.append(Block(kind=CODE, text="\n".join(collected)))
+    return index
+
+
+def _starts_table(lines: list[str], index: int) -> bool:
+    """A table needs a pipe row followed by a separator row; pipes alone are prose."""
+    if not lines[index].lstrip().startswith("|"):
+        return False
+    return index + 1 < len(lines) and bool(TABLE_SEPARATOR_RE.match(lines[index + 1]))
+
+
+def _consume_table(lines: list[str], index: int, blocks: list[Block]) -> int:
+    collected: list[str] = []
+    while index < len(lines) and lines[index].lstrip().startswith("|"):
+        collected.append(lines[index])
+        index += 1
+
+    rows = [_split_table_row(line) for line in collected if not TABLE_SEPARATOR_RE.match(line)]
+    blocks.append(Block(kind=TABLE, text="\n".join(collected), rows=rows, has_header=bool(rows)))
+    return index
+
+
+def _split_table_row(line: str) -> list[str]:
+    stripped = line.strip()
+    if stripped.startswith("|"):
+        stripped = stripped[1:]
+    if stripped.endswith("|"):
+        stripped = stripped[:-1]
+    return [cell.strip() for cell in stripped.split("|")]
+
+
+def _consume_paragraph(lines: list[str], index: int, blocks: list[Block]) -> int:
+    collected: list[str] = []
+    while index < len(lines) and lines[index].strip():
+        line = lines[index]
+        if (
+            FENCE_RE.match(line)
+            or HEADING_RE.match(line)
+            or PAGE_MARKER_RE.match(line)
+            or _starts_table(lines, index)
+        ):
+            break
+        collected.append(line)
+        index += 1
+
+    if collected:
+        blocks.append(Block(kind=PARAGRAPH, text="\n".join(collected)))
+    return index

langparse/chunkers/profiles.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class WorkbookChunkProfile(str, Enum):
+    RETRIEVAL = "retrieval"
+    ANALYSIS = "analysis"
+
+
+@dataclass(frozen=True)
+class WorkbookChunkPolicy:
+    name: WorkbookChunkProfile
+    version: int
+    default_max_chunk_size: int
+    analysis_records: bool
+
+
+class ChunkProfileNotSupportedError(ValueError):
+    """Raised when a valid chunk profile cannot represent the parsed input."""
+
+
+_POLICIES = {
+    WorkbookChunkProfile.RETRIEVAL: WorkbookChunkPolicy(
+        name=WorkbookChunkProfile.RETRIEVAL,
+        version=1,
+        default_max_chunk_size=1000,
+        analysis_records=False,
+    ),
+    WorkbookChunkProfile.ANALYSIS: WorkbookChunkPolicy(
+        name=WorkbookChunkProfile.ANALYSIS,
+        version=1,
+        default_max_chunk_size=4000,
+        analysis_records=True,
+    ),
+}
+
+
+def resolve_workbook_chunk_policy(
+    profile: str | WorkbookChunkProfile | None,
+) -> WorkbookChunkPolicy:
+    if profile is None:
+        selected = WorkbookChunkProfile.RETRIEVAL
+    else:
+        try:
+            selected = WorkbookChunkProfile(profile)
+        except ValueError:
+            available = ", ".join(sorted(item.value for item in WorkbookChunkProfile))
+            raise ValueError(
+                f"Unknown workbook chunk profile {profile!r}. Available: {available}"
+            ) from None
+    return _POLICIES[selected]

langparse/chunkers/registry.py

@@ -0,0 +1,38 @@
+"""Explicit registry for document-text chunkers; workbook routing stays structural."""
+
+from collections.abc import Callable
+
+from langparse.chunkers.semantic import SemanticChunker
+from langparse.chunkers.text import FixedTokenChunker, SlidingWindowChunker, _validate_budget
+from langparse.core.chunker import BaseChunker
+
+_FACTORIES: dict[str, Callable[..., BaseChunker]] = {
+    "semantic": SemanticChunker,
+    "fixed-token": FixedTokenChunker,
+    "sliding-window": SlidingWindowChunker,
+}
+
+
+def register_chunker(name: str, factory: Callable[..., BaseChunker]) -> None:
+    if not isinstance(name, str) or not name.strip() or not callable(factory):
+        raise ValueError("A non-empty name and callable chunker factory are required")
+    if name in _FACTORIES:
+        raise ValueError(f"Chunk strategy {name!r} is already registered")
+    _FACTORIES[name] = factory
+
+
+def available_chunkers() -> tuple[str, ...]:
+    return tuple(sorted(_FACTORIES))
+
+
+def create_chunker(name: str = "semantic", **options) -> BaseChunker:
+    if name not in _FACTORIES:
+        raise ValueError(
+            f"Unknown chunk strategy {name!r}. Available: {', '.join(available_chunkers())}"
+        )
+    if name == "semantic":
+        _validate_budget(options.get("max_chunk_size", 1000), options.get("overlap", 0))
+    result = _FACTORIES[name](**options)
+    if not isinstance(result, BaseChunker):
+        raise TypeError("Chunker factory must return a BaseChunker")
+    return result

langparse/chunkers/semantic.py

@@ -0,0 +1,242 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+from langparse.chunkers.blocks import CODE, HEADING, PAGE_MARKER, TABLE, Block, scan_blocks
+from langparse.core.chunker import BaseChunker
+from langparse.types import Chunk, Document
+
+SENTENCE_END_RE = re.compile(r"(?<=[.!?。！？])\s+")
+JOIN = "\n\n"
+
+
+@dataclass
+class _Section:
+    header: str | None = None
+    header_level: int = 0
+    header_path: str = ""
+    blocks: list[Block] = field(default_factory=list)
+    page_numbers: set = field(default_factory=set)
+
+
+@dataclass
+class _Unit:
+    """One packable piece of content. Oversized units get a chunk to themselves."""
+
+    text: str
+    oversized: bool = False
+
+
+class SemanticChunker(BaseChunker):
+    """
+    Chunks text on Markdown structure, keeping chunks within a size budget.
+
+    Sections come from heading structure; within a section, blocks are packed
+    greedily up to `max_chunk_size`. Size is measured by `length_function`, so
+    callers embedding against a token budget can pass a tokenizer's encoder
+    instead of the default character count.
+    """
+
+    def __init__(
+        self,
+        max_chunk_size: int = 1000,
+        overlap: int = 0,
+        length_function: Callable[[str], int] = len,
+    ):
+        if overlap >= max_chunk_size:
+            raise ValueError("overlap must be smaller than max_chunk_size")
+        self.max_chunk_size = max_chunk_size
+        self.overlap = overlap
+        self.length_function = length_function
+
+    def chunk(self, document: Document, **kwargs) -> list[Chunk]:
+        sections = self._sections(scan_blocks(document.content))
+
+        chunks: list[Chunk] = []
+        for section in sections:
+            units = self._units_for(section.blocks)
+            for text, oversized in self._pack(units):
+                metadata = document.metadata.copy()
+                metadata.update(
+                    {
+                        "header": section.header,
+                        "header_level": section.header_level,
+                        "header_path": section.header_path,
+                        "page_numbers": sorted(section.page_numbers) or [1],
+                        "chunk_index": len(chunks),
+                    }
+                )
+                if oversized:
+                    metadata["oversized"] = True
+                chunks.append(Chunk(content=text, metadata=metadata))
+
+        return chunks
+
+    # -- sectioning ---------------------------------------------------------
+
+    def _sections(self, blocks: list[Block]) -> list[_Section]:
+        sections: list[_Section] = []
+        header_stack: list[tuple[int, str]] = []
+        current_page = 1
+        current = _Section(page_numbers={current_page})
+
+        for block in blocks:
+            if block.kind == PAGE_MARKER:
+                current_page = block.page_number
+                current.page_numbers.add(current_page)
+                continue
+
+            if block.kind == HEADING:
+                if current.blocks:
+                    sections.append(current)
+                while header_stack and header_stack[-1][0] >= block.level:
+                    header_stack.pop()
+                header_stack.append((block.level, block.title))
+                current = _Section(
+                    header=block.title,
+                    header_level=block.level,
+                    header_path=" > ".join(title for _, title in header_stack),
+                    blocks=[block],
+                    page_numbers={current_page},
+                )
+                continue
+
+            current.blocks.append(block)
+
+        if current.blocks:
+            sections.append(current)
+        return sections
+
+    # -- unit expansion -----------------------------------------------------
+
+    def _units_for(self, blocks: list[Block]) -> list[_Unit]:
+        units: list[_Unit] = []
+        for block in blocks:
+            if self._fits(block.text):
+                units.append(_Unit(block.text))
+            elif block.kind == TABLE:
+                units.extend(_Unit(part) for part in self._split_table(block))
+            elif block.kind == CODE:
+                # Splitting a fenced block would leave unterminated fences, so it
+                # travels whole and is flagged for the caller to notice.
+                units.append(_Unit(block.text, oversized=True))
+            else:
+                units.extend(_Unit(part) for part in self._split_prose(block.text))
+        return units
+
+    def _split_table(self, block: Block) -> list[str]:
+        """Split by row, repeating the header so each part reads on its own."""
+        if not block.rows:
+            return [block.text]
+
+        header, *data_rows = block.rows
+        header_markdown = [_row_markdown(header), _separator_markdown(len(header))]
+
+        parts: list[str] = []
+        pending: list[str] = []
+
+        for row in data_rows:
+            candidate = pending + [_row_markdown(row)]
+            # Measure the rendered candidate rather than summing row sizes: a
+            # token counter does not charge exactly one unit per newline.
+            if pending and not self._fits("\n".join(header_markdown + candidate)):
+                parts.append("\n".join(header_markdown + pending))
+                pending = [_row_markdown(row)]
+            else:
+                pending = candidate
+
+        if pending:
+            parts.append("\n".join(header_markdown + pending))
+        return parts or [block.text]
+
+    def _split_prose(self, text: str) -> list[str]:
+        parts: list[str] = []
+        pending = ""
+        for sentence in SENTENCE_END_RE.split(text):
+            if not sentence:
+                continue
+            candidate = f"{pending} {sentence}".strip() if pending else sentence
+            if pending and not self._fits(candidate):
+                parts.append(pending)
+                pending = sentence
+            else:
+                pending = candidate
+
+            while not self._fits(pending):
+                head, pending = self._cut_to_fit(pending)
+                parts.append(head)
+
+        if pending:
+            parts.append(pending)
+        return parts
+
+    def _cut_to_fit(self, text: str) -> tuple[str, str]:
+        """Hard-split a run that no sentence boundary can bring under budget."""
+        low, high, best = 1, len(text), 1
+        while low <= high:
+            middle = (low + high) // 2
+            if self.length_function(text[:middle]) <= self.max_chunk_size:
+                best = middle
+                low = middle + 1
+            else:
+                high = middle - 1
+        return text[:best], text[best:]
+
+    # -- packing ------------------------------------------------------------
+
+    def _pack(self, units: list[_Unit]) -> list[tuple[str, bool]]:
+        packed: list[tuple[str, bool]] = []
+        pending: list[str] = []
+
+        def flush():
+            if pending:
+                packed.append((JOIN.join(pending), False))
+                pending.clear()
+
+        for unit in units:
+            if unit.oversized:
+                flush()
+                packed.append((unit.text, True))
+                continue
+
+            if pending and not self._fits(JOIN.join(pending + [unit.text])):
+                flush()
+            pending.append(unit.text)
+
+        flush()
+        return self._apply_overlap(packed)
+
+    def _apply_overlap(self, packed: list[tuple[str, bool]]) -> list[tuple[str, bool]]:
+        if self.overlap <= 0 or len(packed) < 2:
+            return packed
+
+        result = [packed[0]]
+        for index in range(1, len(packed)):
+            text, oversized = packed[index]
+            tail = self._tail(packed[index - 1][0])
+            result.append((f"{tail}{JOIN}{text}" if tail else text, oversized))
+        return result
+
+    def _tail(self, text: str) -> str:
+        """Longest whitespace-aligned suffix that fits the overlap budget."""
+        words = text.split()
+        tail = ""
+        for count in range(1, len(words) + 1):
+            candidate = " ".join(words[-count:])
+            if self.length_function(candidate) > self.overlap:
+                break
+            tail = candidate
+        return tail
+
+    def _fits(self, text: str) -> bool:
+        return self.length_function(text) <= self.max_chunk_size
+
+
+def _row_markdown(row: list[str]) -> str:
+    return f"| {' | '.join(row)} |"
+
+
+def _separator_markdown(width: int) -> str:
+    return f"| {' | '.join(['---'] * width)} |"

langparse/chunkers/text.py

@@ -0,0 +1,96 @@
+"""Deterministic text windows with explicit measurement units."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from langparse.core.chunker import BaseChunker
+from langparse.types import Chunk, Document
+
+
+def _validate_budget(size: int, overlap: int) -> None:
+    if not isinstance(size, int) or isinstance(size, bool) or size <= 0:
+        raise ValueError("max_chunk_size must be a positive integer")
+    if not isinstance(overlap, int) or isinstance(overlap, bool) or not 0 <= overlap < size:
+        raise ValueError("overlap must be an integer between 0 and max_chunk_size - 1")
+
+
+class FixedTokenChunker(BaseChunker):
+    """Split encoded tokens. Default lexical units are NOT model tokens.
+
+    The default codec preserves whitespace and groups Unicode words, whitespace,
+    and individual punctuation. Supply both encoder and decoder to use a model's
+    tokenizer; the caller owns that codec's lossless decoding contract.
+    """
+
+    def __init__(
+        self,
+        max_chunk_size: int = 1000,
+        overlap: int = 0,
+        *,
+        encoder: Callable[[str], Sequence[Any]] | None = None,
+        decoder: Callable[[Sequence[Any]], str] | None = None,
+    ):
+        _validate_budget(max_chunk_size, overlap)
+        if (encoder is None) != (decoder is None):
+            raise ValueError("encoder and decoder must be supplied together")
+        self.max_chunk_size = max_chunk_size
+        self.overlap = overlap
+        self.encoder = encoder or (lambda text: re.findall(r"\s+|\w+|[^\w\s]", text))
+        self.decoder = decoder or "".join
+        self.tokenizer = "custom" if encoder is not None else "lexical"
+
+    def chunk(self, document: Document, **kwargs) -> list[Chunk]:
+        tokens = list(self.encoder(document.content))
+        chunks = []
+        for start in range(0, len(tokens), self.max_chunk_size - self.overlap):
+            end = min(start + self.max_chunk_size, len(tokens))
+            chunks.append(
+                Chunk(
+                    content=self.decoder(tokens[start:end]),
+                    metadata={
+                        **document.metadata,
+                        "chunk_strategy": "fixed-token",
+                        "chunk_index": len(chunks),
+                        "tokenizer": self.tokenizer,
+                        "token_start": start,
+                        "token_end": end,
+                        "token_count": end - start,
+                    },
+                )
+            )
+            if end == len(tokens):
+                break
+        return chunks
+
+
+class SlidingWindowChunker(BaseChunker):
+    """Character windows over the complete text, including block separators."""
+
+    def __init__(self, max_chunk_size: int = 1000, overlap: int = 0):
+        _validate_budget(max_chunk_size, overlap)
+        self.max_chunk_size = max_chunk_size
+        self.overlap = overlap
+
+    def chunk(self, document: Document, **kwargs) -> list[Chunk]:
+        chunks = []
+        for start in range(0, len(document.content), self.max_chunk_size - self.overlap):
+            end = min(start + self.max_chunk_size, len(document.content))
+            chunks.append(
+                Chunk(
+                    content=document.content[start:end],
+                    metadata={
+                        **document.metadata,
+                        "chunk_strategy": "sliding-window",
+                        "chunk_index": len(chunks),
+                        "size_unit": "character",
+                        "char_start": start,
+                        "char_end": end,
+                    },
+                )
+            )
+            if end == len(document.content):
+                break
+        return chunks