viparse 0.1.0__py3-none-any.whl

viparse/__init__.py

@@ -0,0 +1,106 @@
+"""viparse — a Vietnamese-first document loader for RAG."""
+
+from __future__ import annotations
+
+from viparse.api import load, load_batch
+from viparse.cache import Cache, DiskCache, MemoryCache
+from viparse.chunk import ChunkOptions, chunk_document
+from viparse.config import Settings, load_settings
+from viparse.detect import DetectedFormat, detect_format
+from viparse.engines.docx import DocxEngine
+from viparse.engines.legacy import LegacyOfficeEngine
+from viparse.engines.ocr import OcrEngine
+from viparse.engines.pdf import PdfEngine
+from viparse.engines.xlsx import XlsxEngine
+from viparse.errors import (
+    ConfigError,
+    EncodingError,
+    EngineUnavailable,
+    ExtractionError,
+    MissingDependency,
+    UnsafeInput,
+    UnsupportedFormat,
+    ViparseError,
+)
+from viparse.integrations import to_langchain_documents, to_llamaindex_documents
+from viparse.model import (
+    SCHEMA_VERSION,
+    Block,
+    Chunk,
+    Document,
+    DocumentMetadata,
+    Heading,
+    NormalizedDoc,
+    Paragraph,
+    RawExtraction,
+    Table,
+)
+from viparse.normalize.normalizer import VietnameseNormalizer
+from viparse.observability import MetricsHook, PipelineMetrics
+from viparse.options import LoadOptions, NormalizeForm, OutputFormat
+from viparse.pipeline import Pipeline
+from viparse.protocols import (
+    DEFAULT_PRIORITY,
+    Engine,
+    Normalizer,
+    Renderer,
+    Source,
+)
+from viparse.registry import EngineRegistry
+from viparse.structure import DocumentRenderer
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "DEFAULT_PRIORITY",
+    "Block",
+    "Cache",
+    "Chunk",
+    "ChunkOptions",
+    "chunk_document",
+    "ConfigError",
+    "DetectedFormat",
+    "DiskCache",
+    "MemoryCache",
+    "DocumentRenderer",
+    "DocxEngine",
+    "Document",
+    "DocumentMetadata",
+    "EncodingError",
+    "Engine",
+    "EngineRegistry",
+    "EngineUnavailable",
+    "ExtractionError",
+    "Heading",
+    "LegacyOfficeEngine",
+    "LoadOptions",
+    "load",
+    "load_batch",
+    "MetricsHook",
+    "MissingDependency",
+    "NormalizeForm",
+    "NormalizedDoc",
+    "Normalizer",
+    "OcrEngine",
+    "OutputFormat",
+    "Paragraph",
+    "PdfEngine",
+    "Pipeline",
+    "PipelineMetrics",
+    "RawExtraction",
+    "Renderer",
+    "SCHEMA_VERSION",
+    "Settings",
+    "Source",
+    "Table",
+    "load_settings",
+    "to_langchain_documents",
+    "to_llamaindex_documents",
+    "UnsafeInput",
+    "UnsupportedFormat",
+    "VietnameseNormalizer",
+    "ViparseError",
+    "XlsxEngine",
+    "__version__",
+    "detect_format",
+]

viparse/api.py

@@ -0,0 +1,250 @@
+"""The public :func:`load` / :func:`load_batch` API — the surface most callers use.
+
+These are thin ergonomic wrappers over :class:`~viparse.pipeline.Pipeline`, assembled
+with the built-in engines, the Vietnamese normalizer, and the document renderer. The
+keyword-only parameters map onto :class:`~viparse.options.LoadOptions`; keeping them
+keyword-only lets the surface grow (SemVer-safely) without breaking callers.
+
+``load`` returns a ``list[Document]`` — one entry today, but the list shape reserves
+room for sources that fan out into several documents (e.g. one per spreadsheet sheet)
+without a signature change.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from collections.abc import Iterable, Iterator
+from concurrent.futures import Future, ThreadPoolExecutor
+
+from viparse.cache import Cache, cache_key
+from viparse.chunk import ChunkOptions
+from viparse.config import UNSET, Settings, _Unset, load_settings
+from viparse.engines.docx import DocxEngine
+from viparse.engines.legacy import LegacyOfficeEngine
+from viparse.engines.ocr import OcrEngine
+from viparse.engines.pdf import PdfEngine
+from viparse.engines.xlsx import XlsxEngine
+from viparse.errors import ViparseError
+from viparse.model import Document, NormalizedDoc
+from viparse.normalize.normalizer import VietnameseNormalizer
+from viparse.options import LoadOptions, NormalizeForm, OutputFormat
+from viparse.pipeline import Pipeline
+from viparse.protocols import Engine, Source
+from viparse.registry import EngineRegistry
+from viparse.structure import DocumentRenderer
+
+# Content type for a batch error Document whose real type was never determined (matches
+# the pipeline's own unknown-type sentinel).
+_UNKNOWN_CONTENT_TYPE = "application/octet-stream"
+
+
+def _default_engines() -> list[Engine]:
+    """The engines registered by default.
+
+    Each is a thin adapter whose heavy parse library is imported lazily at extraction
+    time, so listing one here never pulls its dependency at import — ``core`` stays light.
+    """
+    return [DocxEngine(), XlsxEngine(), PdfEngine(), OcrEngine(), LegacyOfficeEngine()]
+
+
+def _build_pipeline() -> Pipeline:
+    """Assemble the default pipeline: built-in engines → normalizer → renderer."""
+    registry = EngineRegistry()
+    for engine in _default_engines():
+        registry.register(engine)
+    return Pipeline(registry, VietnameseNormalizer(), DocumentRenderer())
+
+
+def _resolve_options(
+    settings: Settings | None,
+    output: OutputFormat | _Unset,
+    encoding: str | None | _Unset,
+    ocr: bool | None | _Unset,
+    normalize: NormalizeForm | _Unset,
+    max_bytes: int | _Unset,
+    chunk: ChunkOptions | None,
+) -> LoadOptions:
+    """Layer the public keyword parameters over the resolved :class:`Settings`.
+
+    An argument left as :data:`~viparse.config.UNSET` falls back to ``settings`` (which,
+    when not supplied, is resolved from ``VIPARSE_*`` env vars and ``viparse.toml``); an
+    explicit argument always wins. ``chunk`` is per-call only and never layered.
+    """
+    base = settings if settings is not None else load_settings()
+    return LoadOptions(
+        fmt=base.output if output is UNSET else output,
+        encoding=base.encoding if encoding is UNSET else encoding,
+        ocr=base.ocr if ocr is UNSET else ocr,
+        normalize_form=base.normalize if normalize is UNSET else normalize,
+        max_bytes=base.max_bytes if max_bytes is UNSET else max_bytes,
+        chunk=chunk,
+    )
+
+
+def load(
+    source: Source,
+    *,
+    output: OutputFormat | _Unset = UNSET,
+    encoding: str | None | _Unset = UNSET,
+    ocr: bool | None | _Unset = UNSET,
+    normalize: NormalizeForm | _Unset = UNSET,
+    max_bytes: int | _Unset = UNSET,
+    cache: Cache | None = None,
+    chunk: ChunkOptions | None = None,
+    settings: Settings | None = None,
+) -> list[Document]:
+    """Parse ``source`` into a list of Unicode-**NFC** :class:`Document` objects.
+
+    The ``output`` / ``encoding`` / ``ocr`` / ``normalize`` / ``max_bytes`` options are
+    **layered** (SPEC-5 E5.4): an argument left unset falls back to ``VIPARSE_*`` env vars,
+    then ``viparse.toml``, then the built-in default; an explicit argument always wins.
+
+    :param source: path to the document (``str`` or :class:`~pathlib.Path`).
+    :param output: rendered output format — ``"markdown"`` (default), ``"text"``, or ``"json"``.
+    :param encoding: force a legacy source encoding (e.g. ``"tcvn3"``), or ``"auto"`` to
+        opt into content-based detection when the source carries no font signal.
+    :param ocr: force OCR on/off; the router otherwise decides from the file.
+    :param normalize: target Unicode normalization form (default ``"NFC"``).
+    :param max_bytes: reject an input larger than this many bytes (default 100 MiB).
+    :param cache: optional content-hash :class:`~viparse.cache.Cache`; a cache hit skips
+        re-parsing an unchanged file (SPEC-7 E7.3).
+    :param chunk: optional :class:`~viparse.chunk.ChunkOptions`; when given, the returned
+        Document's ``chunks`` are populated with retrieval-sized pieces (SPEC-4 E4.2).
+    :param settings: pre-resolved :class:`~viparse.config.Settings` to layer under the
+        arguments; when omitted, settings are resolved from the environment and config file
+        **on every call** (a ``viparse.toml`` stat/read + an env scan). When loading many
+        files, resolve once with :func:`~viparse.load_settings` and pass ``settings=`` here
+        (or use :func:`load_batch`, which resolves once for the whole batch).
+    """
+    options = _resolve_options(settings, output, encoding, ocr, normalize, max_bytes, chunk)
+    return [_load_one(_build_pipeline(), source, options, cache)]
+
+
+def _load_one(
+    pipeline: Pipeline, source: Source, options: LoadOptions, cache: Cache | None
+) -> Document:
+    """Run one source through the pipeline, consulting/populating ``cache`` if given."""
+    if cache is None:
+        return pipeline.run(source, options)
+    key = cache_key(source, options)
+    cached = cache.get(key)
+    if cached is not None:
+        return cached
+    document = pipeline.run(source, options)
+    cache.set(key, document)
+    return document
+
+
+def _error_document(source: Source, error: Exception, fmt: OutputFormat) -> Document:
+    """A best-effort Document for a source that failed to load in a batch.
+
+    Rendered through the normal renderer so its ``text`` is valid for the requested format
+    (e.g. a parseable JSON payload under ``output="json"``, not an empty string), with the
+    failure in ``metadata.warnings`` and the unknown-content sentinel as the content type.
+    """
+    normalized = NormalizedDoc(
+        source=str(source),
+        content_type=_UNKNOWN_CONTENT_TYPE,
+        text="",
+        warnings=[f"failed to load: {error}"],
+    )
+    return DocumentRenderer().render(normalized, fmt)
+
+
+def _batch_load(
+    pipeline: Pipeline, source: Source, options: LoadOptions, cache: Cache | None
+) -> Document:
+    """Load one source for a batch, isolating a per-source *failure* into an error Document.
+
+    Only extraction/format failures (:class:`~viparse.errors.ViparseError`) and file-access
+    errors (:class:`OSError`) are isolated so one bad file can't sink the batch. Programming
+    bugs (``TypeError`` etc.) deliberately propagate — matching the pipeline's rule that
+    they must never be silently downgraded to a warning.
+    """
+    try:
+        return _load_one(pipeline, source, options, cache)
+    except (ViparseError, OSError) as error:
+        return _error_document(source, error, options.fmt)
+
+
+def load_batch(
+    sources: Iterable[Source],
+    *,
+    output: OutputFormat | _Unset = UNSET,
+    encoding: str | None | _Unset = UNSET,
+    ocr: bool | None | _Unset = UNSET,
+    normalize: NormalizeForm | _Unset = UNSET,
+    max_bytes: int | _Unset = UNSET,
+    cache: Cache | None = None,
+    workers: int | None = None,
+    chunk: ChunkOptions | None = None,
+    settings: Settings | None = None,
+) -> Iterator[list[Document]]:
+    """Lazily load each source, yielding its result list **in input order**.
+
+    A generator (not a materialized list), so results stream out as they're ready and the
+    caller controls consumption. Unlike :func:`load`, a batch **isolates failures**: a
+    source that errors yields a best-effort Document recording the failure (its
+    ``metadata.warnings``) instead of sinking the whole batch (SPEC-7 T7.2.3).
+
+    ``workers`` (>1) parses up to that many sources concurrently on a thread pool —
+    extraction is largely IO / subprocess bound (LibreOffice, Tesseract, file reads). Only
+    ``workers`` sources are ever in flight at once (backpressure), and output order still
+    matches the input. ``None``/1 runs sequentially. Abandoning the iterator early waits for
+    the (at most ``workers``) in-flight parses to finish before the pool shuts down.
+
+    ``chunk`` behaves as in :func:`load`: when set, each result Document's ``chunks`` are
+    populated with retrieval-sized pieces. ``output`` / ``encoding`` / ``ocr`` / ``normalize`` /
+    ``max_bytes`` are layered over ``settings`` exactly as in :func:`load`.
+
+    Option resolution (and any :class:`~viparse.errors.ConfigError` from bad env/config) happens
+    **eagerly at call time**, not on first iteration — so a misconfiguration surfaces to the
+    caller here rather than mid-stream, past the per-source error isolation.
+    """
+    options = _resolve_options(settings, output, encoding, ocr, normalize, max_bytes, chunk)
+    pipeline = _build_pipeline()
+    return _iter_batch(pipeline, sources, options, cache, workers)
+
+
+def _iter_batch(
+    pipeline: Pipeline,
+    sources: Iterable[Source],
+    options: LoadOptions,
+    cache: Cache | None,
+    workers: int | None,
+) -> Iterator[list[Document]]:
+    """The lazy per-source generator behind :func:`load_batch` (options already resolved)."""
+    if workers is None or workers <= 1:
+        for source in sources:
+            yield [_batch_load(pipeline, source, options, cache)]
+        return
+    yield from _parallel_load_batch(pipeline, sources, options, cache, workers)
+
+
+def _parallel_load_batch(
+    pipeline: Pipeline,
+    sources: Iterable[Source],
+    options: LoadOptions,
+    cache: Cache | None,
+    workers: int,
+) -> Iterator[list[Document]]:
+    """Parse sources on a bounded thread pool, yielding results in input order."""
+    pending: deque[Future[Document]] = deque()
+    source_iter = iter(sources)
+    with ThreadPoolExecutor(max_workers=workers) as executor:
+
+        def submit_next() -> bool:
+            try:
+                source = next(source_iter)
+            except StopIteration:
+                return False
+            pending.append(executor.submit(_batch_load, pipeline, source, options, cache))
+            return True
+
+        for _ in range(workers):  # prime the in-flight window
+            if not submit_next():
+                break
+        while pending:
+            document = pending.popleft().result()  # oldest first → input order
+            submit_next()  # keep the window full as each result drains
+            yield [document]

viparse/cache.py

@@ -0,0 +1,120 @@
+"""Content-hash caching so unchanged files are not re-parsed (SPEC-7 E7.3).
+
+A cache is keyed by the file's content hash *plus* the options that affect the output
+(format, encoding, OCR, normalization) and the :data:`~viparse.model.SCHEMA_VERSION`, so
+a changed file, different options, or a new pipeline version all miss the cache and never
+serve a stale result. Caching is opt-in — pass a :class:`Cache` to :func:`viparse.load`.
+
+Two stores ship: :class:`MemoryCache` (per-process) and :class:`DiskCache` (persists
+across runs). Both satisfy the small :class:`Cache` Protocol, so callers can supply their
+own (Redis, S3, …).
+
+.. note::
+   The key is computed by hashing the file *before* the pipeline re-reads it to parse, so
+   rewriting a file in place *during* a load is a TOCTOU race that can cache the new bytes
+   under the old hash. Don't mutate files in place while loading them; write a new file
+   (atomic rename) instead.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import pickle
+import tempfile
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+from viparse.model import SCHEMA_VERSION, Document
+from viparse.options import LoadOptions
+from viparse.protocols import Source
+
+_HASH_CHUNK = 1024 * 1024  # hash the file in 1 MiB chunks (bounded memory)
+
+
+@runtime_checkable
+class Cache(Protocol):
+    """A content-addressed store of parsed :class:`Document` results."""
+
+    def get(self, key: str) -> Document | None:
+        """Return the cached document for ``key``, or ``None`` on a miss."""
+        ...
+
+    def set(self, key: str, document: Document) -> None:
+        """Store ``document`` under ``key``."""
+        ...
+
+
+class MemoryCache:
+    """A simple in-process cache backed by a dict."""
+
+    def __init__(self) -> None:
+        self._store: dict[str, Document] = {}
+
+    def get(self, key: str) -> Document | None:
+        return self._store.get(key)
+
+    def set(self, key: str, document: Document) -> None:
+        self._store[key] = document
+
+
+class DiskCache:
+    """A cache that persists pickled documents under a directory, one file per key."""
+
+    def __init__(self, directory: Source) -> None:
+        self._dir = Path(directory)
+        self._dir.mkdir(parents=True, exist_ok=True)
+
+    def _path(self, key: str) -> Path:
+        # Hash the key into the filename so an arbitrary key can never traverse out of the
+        # cache directory, and any string is a valid (fixed-length, filesystem-safe) name.
+        name = hashlib.sha256(key.encode("utf-8")).hexdigest()
+        return self._dir / f"{name}.pickle"
+
+    def get(self, key: str) -> Document | None:
+        try:
+            with self._path(key).open("rb") as fh:
+                # Trusted input: the cache directory holds only this library's own writes.
+                document: Document = pickle.load(fh)
+        except FileNotFoundError:
+            return None  # a plain miss
+        except Exception:  # noqa: BLE001 — any corrupt/unreadable entry is treated as a miss
+            return None
+        return document
+
+    def set(self, key: str, document: Document) -> None:
+        # Write to a temp file in the same directory, then atomically rename into place, so
+        # a crash or a concurrent writer never leaves a half-written entry to be read back.
+        fd, tmp = tempfile.mkstemp(dir=self._dir, suffix=".tmp")
+        with os.fdopen(fd, "wb") as fh:
+            pickle.dump(document, fh)
+        os.replace(tmp, self._path(key))
+
+
+def cache_key(source: Source, options: LoadOptions) -> str:
+    """A stable key from the file's content hash, its path, and the output-affecting options.
+
+    The path is part of the key so a hit only ever returns the document parsed from *that*
+    source (its ``metadata.source`` stays correct — two byte-identical files never share an
+    entry). The options are fingerprinted with :func:`repr` of a tuple so ``None`` and the
+    string ``"None"`` (and any field containing a delimiter) can never collide. ``max_bytes``
+    and ``strict`` are omitted because they do not change the parsed output; ``chunk`` is
+    included because it changes the document (its ``chunks``), and its frozen ``repr`` is stable.
+    """
+    digest = hashlib.sha256()
+    with Path(source).open("rb") as fh:
+        for block in iter(lambda: fh.read(_HASH_CHUNK), b""):
+            digest.update(block)
+    fingerprint = repr(
+        (
+            str(source),
+            SCHEMA_VERSION,
+            options.fmt,
+            options.encoding,
+            options.ocr,
+            options.normalize_form,
+            options.chunk,
+        )
+    )
+    digest.update(fingerprint.encode("utf-8"))
+    return digest.hexdigest()

viparse/chunk.py

@@ -0,0 +1,128 @@
+"""RAG chunking: split a normalized document into retrieval-sized pieces (SPEC-4 E4.2).
+
+Chunking works on the **block structure** (headings / paragraphs / table rows), not the
+flat text, so it can:
+
+- keep every chunk within a single section — any section change (a heading, or an empty
+  heading that merely resets the section) starts a new chunk, so a chunk's ``section``
+  metadata is unambiguous (it never straddles a section boundary);
+- **never split a table row** — each row is an atomic unit, and a large table is split at
+  row boundaries;
+- carry per-chunk metadata (``section`` and inherited ``page`` / ``sheet``) plus the
+  chunk's ordinal :attr:`~viparse.model.Chunk.index` for downstream provenance.
+
+Token counts are **approximate and tiktoken-free** (whitespace-delimited words), which is
+enough to bound chunk size without pinning to any one model's tokenizer. A single block
+larger than the target is emitted whole (sub-splitting long paragraphs is a future
+refinement); table rows are always kept intact regardless of size.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+
+from viparse.model import Block, Chunk, Heading, NormalizedDoc, Table, blocks_of
+
+
+@dataclass(frozen=True, slots=True)
+class ChunkOptions:
+    """Chunking knobs: target size and overlap, both in approximate tokens (words)."""
+
+    max_tokens: int = 512
+    overlap_tokens: int = 64
+
+
+def estimate_tokens(text: str) -> int:
+    """Approximate token count for ``text`` (tiktoken-free: whitespace-delimited words)."""
+    return len(text.split())
+
+
+@dataclass(frozen=True, slots=True)
+class _Unit:
+    """An atomic chunkable piece — a heading, a paragraph, or a single table row."""
+
+    text: str
+    section: str
+    is_heading: bool
+
+
+def chunk_document(document: NormalizedDoc, options: ChunkOptions) -> list[Chunk]:
+    """Split ``document`` into overlapping, single-section :class:`~viparse.model.Chunk`s."""
+    units = list(_iter_units(blocks_of(document)))
+    if not units:
+        return []
+
+    chunks: list[Chunk] = []
+    start = 0
+    count = len(units)
+    while True:  # always terminates via the `end >= count` break below (count >= 1 here)
+        end = start
+        tokens = 0
+        while end < count:
+            if end > start and _crosses_section(units, start, end):
+                break  # a new section starts here — never split one across chunks
+            unit_tokens = estimate_tokens(units[end].text)
+            if end > start and tokens + unit_tokens > options.max_tokens:
+                break  # this unit would overflow — leave it for the next chunk
+            tokens += unit_tokens
+            end += 1
+        chunks.append(_make_chunk(units[start:end], len(chunks), document))
+        if end >= count:
+            break
+        # Overlap stays within a section: a section boundary starts the next chunk fresh.
+        if _crosses_section(units, start, end):
+            start = end
+        else:
+            start = _next_start(units, start, end, options)
+    return chunks
+
+
+def _crosses_section(units: list[_Unit], start: int, end: int) -> bool:
+    """Whether ``units[end]`` opens a new section relative to the chunk started at ``start``.
+
+    True at an explicit heading unit *and* when the section label merely changes (an
+    empty-text heading resets the section without emitting a unit of its own), so a chunk
+    can never straddle a section boundary regardless of how it was introduced.
+    """
+    return units[end].is_heading or units[end].section != units[start].section
+
+
+def _iter_units(blocks: list[Block]) -> Iterator[_Unit]:
+    """Flatten blocks into atomic units, tracking the section each unit belongs to."""
+    section = ""
+    for block in blocks:
+        if isinstance(block, Heading):
+            section = block.text
+            if block.text:
+                yield _Unit(text=block.text, section=section, is_heading=True)
+        elif isinstance(block, Table):
+            for row in block.rows:
+                if any(cell.strip() for cell in row):  # skip a row whose every cell is blank
+                    yield _Unit(text="\t".join(row), section=section, is_heading=False)
+        elif block.text:  # Paragraph
+            yield _Unit(text=block.text, section=section, is_heading=False)
+
+
+def _make_chunk(group: list[_Unit], index: int, document: NormalizedDoc) -> Chunk:
+    metadata: dict[str, object] = {
+        "section": group[0].section,  # every unit in the group shares one section
+        "page": document.page,
+        "sheet": document.sheet,
+    }
+    return Chunk(text="\n".join(unit.text for unit in group), metadata=metadata, index=index)
+
+
+def _next_start(units: list[_Unit], start: int, end: int, options: ChunkOptions) -> int:
+    """Where the next chunk begins: back up from ``end`` to repeat ~``overlap_tokens``.
+
+    Never returns ``<= start``, so the scan always makes progress (an overlap larger than
+    the chunk simply repeats everything but the first unit). ``units[start:end]`` share one
+    section, so this overlap never crosses a heading boundary.
+    """
+    cursor = end
+    tokens = 0
+    while cursor > start + 1 and tokens < options.overlap_tokens:
+        cursor -= 1
+        tokens += estimate_tokens(units[cursor].text)
+    return cursor