kreuzberg 1.4.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

kreuzberg/_extractors.py

@@ -4,16 +4,15 @@ import re
 from contextlib import suppress
 from html import escape
 from io import BytesIO
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING
 
 import html_to_markdown
 import pptx
-import pypandoc
 import pypdfium2
 from anyio import Path as AsyncPath
-from charset_normalizer import detect
+from pptx.enum.shapes import MSO_SHAPE_TYPE
 
-from kreuzberg._mime_types import PANDOC_MIME_TYPE_EXT_MAP
+from kreuzberg._pandoc import process_content, process_file
 from kreuzberg._string import normalize_spaces, safe_decode
 from kreuzberg._sync import run_sync
 from kreuzberg._tesseract import batch_process_images
@@ -98,32 +97,18 @@ async def extract_pdf_file(file_path: Path, force_ocr: bool = False) -> str:
     return await extract_pdf_with_tesseract(file_path)
 
 
-async def extract_content_with_pandoc(file_data: bytes, mime_type: str, encoding: str | None = None) -> str:
+async def extract_content_with_pandoc(file_data: bytes, mime_type: str) -> str:
     """Extract text using pandoc.
 
     Args:
         file_data: The content of the file.
         mime_type: The mime type of the file.
-        encoding: An optional encoding to use when decoding the string.
-
-    Raises:
-        ParsingError: If the text could not be extracted from the file using pandoc.
 
     Returns:
         The extracted text.
     """
-    ext = PANDOC_MIME_TYPE_EXT_MAP[mime_type]
-    encoding = encoding or detect(file_data)["encoding"] or "utf-8"
-    try:
-        return normalize_spaces(
-            cast(str, await run_sync(pypandoc.convert_text, file_data, to="md", format=ext, encoding=encoding))
-        )
-    except RuntimeError as e:
-        # TODO: add test case
-        raise ParsingError(
-            f"Could not extract text from {PANDOC_MIME_TYPE_EXT_MAP[mime_type]} file contents",
-            context={"error": str(e)},
-        ) from e
+    result = await process_content(file_data, mime_type=mime_type)
+    return normalize_spaces(result.content)
 
 
 async def extract_file_with_pandoc(file_path: Path | str, mime_type: str) -> str:
@@ -133,20 +118,11 @@ async def extract_file_with_pandoc(file_path: Path | str, mime_type: str) -> str
         file_path: The path to the file.
         mime_type: The mime type of the file.
 
-    Raises:
-        ParsingError: If the text could not be extracted from the file using pandoc.
-
     Returns:
         The extracted text.
     """
-    ext = PANDOC_MIME_TYPE_EXT_MAP[mime_type]
-    try:
-        return normalize_spaces(cast(str, await run_sync(pypandoc.convert_file, file_path, to="md", format=ext)))
-    except RuntimeError as e:
-        raise ParsingError(
-            f"Could not extract text from {PANDOC_MIME_TYPE_EXT_MAP[mime_type]} file",
-            context={"file_path": str(file_path), "error": str(e)},
-        ) from e
+    result = await process_file(file_path, mime_type=mime_type)
+    return normalize_spaces(result.content)
 
 
 async def extract_pptx_file(file_path_or_contents: Path | bytes) -> str:
@@ -161,8 +137,6 @@ async def extract_pptx_file(file_path_or_contents: Path | bytes) -> str:
     Returns:
         The extracted text content
     """
-    from pptx.enum.shapes import MSO_SHAPE_TYPE
-
     md_content = ""
     file_contents = (
         file_path_or_contents

kreuzberg/_mime_types.py

@@ -54,44 +54,35 @@ IMAGE_MIME_TYPE_EXT_MAP: Final[Mapping[str, str]] = {
     "image/x-portable-pixmap": "ppm",
 }
 PANDOC_SUPPORTED_MIME_TYPES: Final[set[str]] = {
-    "application/csv",
-    "application/latex",
+    "application/csl+json",
+    "application/docbook+xml",
+    "application/epub+zip",
     "application/rtf",
     "application/vnd.oasis.opendocument.text",
     "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-    "application/x-csv",
+    "application/x-biblatex",
+    "application/x-bibtex",
+    "application/x-endnote+xml",
+    "application/x-fictionbook+xml",
+    "application/x-ipynb+json",
+    "application/x-jats+xml",
     "application/x-latex",
-    "application/x-rtf",
-    "application/x-vnd.oasis.opendocument.text",
+    "application/x-opml+xml",
+    "application/x-research-info-systems",
+    "application/x-typst",
     "text/csv",
-    "text/latex",
-    "text/rst",
-    "text/rtf",
     "text/tab-separated-values",
-    "text/x-csv",
-    "text/x-latex",
+    "text/troff",
+    "text/x-commonmark",
+    "text/x-dokuwiki",
+    "text/x-gfm",
+    "text/x-markdown",
+    "text/x-markdown-extra",
+    "text/x-mdoc",
+    "text/x-multimarkdown",
+    "text/x-org",
+    "text/x-pod",
     "text/x-rst",
-    "text/x-tsv",
-}
-PANDOC_MIME_TYPE_EXT_MAP: Final[Mapping[str, str]] = {
-    "application/csv": "csv",
-    "application/latex": "latex",
-    "application/rtf": "rtf",
-    "application/vnd.oasis.opendocument.text": "odt",
-    "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
-    "application/x-csv": "csv",
-    "application/x-latex": "latex",
-    "application/x-rtf": "rtf",
-    "application/x-vnd.oasis.opendocument.text": "odt",
-    "text/csv": "csv",
-    "text/latex": "latex",
-    "text/rst": "rst",
-    "text/rtf": "rtf",
-    "text/tab-separated-values": "tsv",
-    "text/x-csv": "csv",
-    "text/x-latex": "latex",
-    "text/x-rst": "rst",
-    "text/x-tsv": "tsv",
 }
 
 SUPPORTED_MIME_TYPES: Final[set[str]] = (

kreuzberg/_pandoc.py

@@ -0,0 +1,416 @@
+from __future__ import annotations
+
+import json
+import subprocess
+from asyncio import gather
+from dataclasses import dataclass
+from tempfile import NamedTemporaryFile
+from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, cast
+
+from anyio import Path as AsyncPath
+
+from kreuzberg._string import normalize_spaces
+from kreuzberg._sync import run_sync
+from kreuzberg.exceptions import MissingDependencyError, ParsingError, ValidationError
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+    from os import PathLike
+
+try:  # pragma: no cover
+    from typing import NotRequired  # type: ignore[attr-defined]
+except ImportError:  # pragma: no cover
+    from typing_extensions import NotRequired
+
+version_ref: Final[dict[str, bool]] = {"checked": False}
+
+
+# Block-level node types in Pandoc AST
+BLOCK_HEADER: Final = "Header"  # Header with level, attributes and inline content
+BLOCK_PARA: Final = "Para"  # Paragraph containing inline content
+BLOCK_CODE: Final = "CodeBlock"  # Code block with attributes and string content
+BLOCK_QUOTE: Final = "BlockQuote"  # Block quote containing blocks
+BLOCK_LIST: Final = "BulletList"  # Bullet list containing items (blocks)
+BLOCK_ORDERED: Final = "OrderedList"  # Numbered list with attrs and items
+
+# Inline-level node types in Pandoc AST
+INLINE_STR: Final = "Str"  # Plain text string
+INLINE_SPACE: Final = "Space"  # Single space
+INLINE_EMPH: Final = "Emph"  # Emphasized text (contains inlines)
+INLINE_STRONG: Final = "Strong"  # Strong/bold text (contains inlines)
+INLINE_LINK: Final = "Link"  # Link with text and target
+INLINE_IMAGE: Final = "Image"  # Image with alt text and source
+INLINE_CODE: Final = "Code"  # Inline code span
+INLINE_MATH: Final = "Math"  # Math expression
+
+# Metadata node types in Pandoc AST
+META_MAP: Final = "MetaMap"  # Key-value mapping of metadata
+META_LIST: Final = "MetaList"  # List of metadata values
+META_INLINES: Final = "MetaInlines"  # Inline content in metadata
+META_STRING: Final = "MetaString"  # Plain string in metadata
+META_BLOCKS: Final = "MetaBlocks"  # Block content in metadata
+
+# Node content field name
+CONTENT_FIELD: Final = "c"
+TYPE_FIELD: Final = "t"
+
+# Valid node types
+NodeType = Literal[
+    # Block types
+    "Header",
+    "Para",
+    "CodeBlock",
+    "BlockQuote",
+    "BulletList",
+    "OrderedList",
+    # Inline types
+    "Str",
+    "Space",
+    "Emph",
+    "Strong",
+    "Link",
+    "Image",
+    "Code",
+    "Math",
+    # Meta types
+    "MetaMap",
+    "MetaList",
+    "MetaInlines",
+    "MetaString",
+    "MetaBlocks",
+]
+
+PANDOC_MIMETYPE_TO_FORMAT_MAPPING: Final[Mapping[str, str]] = {
+    "application/csl+json": "csljson",
+    "application/docbook+xml": "docbook",
+    "application/epub+zip": "epub",
+    "application/rtf": "rtf",
+    "application/vnd.oasis.opendocument.text": "odt",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
+    "application/x-biblatex": "biblatex",
+    "application/x-bibtex": "bibtex",
+    "application/x-endnote+xml": "endnotexml",
+    "application/x-fictionbook+xml": "fb2",
+    "application/x-ipynb+json": "ipynb",
+    "application/x-jats+xml": "jats",
+    "application/x-latex": "latex",
+    "application/x-opml+xml": "opml",
+    "application/x-research-info-systems": "ris",
+    "application/x-typst": "typst",
+    "text/csv": "csv",
+    "text/tab-separated-values": "tsv",
+    "text/troff": "man",
+    "text/x-commonmark": "commonmark",
+    "text/x-dokuwiki": "dokuwiki",
+    "text/x-gfm": "gfm",
+    "text/x-markdown": "markdown",
+    "text/x-markdown-extra": "markdown_phpextra",
+    "text/x-mdoc": "mdoc",
+    "text/x-multimarkdown": "markdown_mmd",
+    "text/x-org": "org",
+    "text/x-pod": "pod",
+    "text/x-rst": "rst",
+}
+
+
+class Metadata(TypedDict, total=False):
+    """Document metadata extracted from Pandoc document.
+
+    All fields are optional but will only be included if they contain non-empty values.
+    Any field that would be empty or None is omitted from the dictionary.
+    """
+
+    title: NotRequired[str]
+    """Document title."""
+    subtitle: NotRequired[str]
+    """Document subtitle."""
+    abstract: NotRequired[str | list[str]]
+    """Document abstract, summary or description."""
+    authors: NotRequired[list[str]]
+    """List of document authors."""
+    date: NotRequired[str]
+    """Document date as string to preserve original format."""
+    subject: NotRequired[str]
+    """Document subject or topic."""
+    description: NotRequired[str]
+    """Extended description."""
+    keywords: NotRequired[list[str]]
+    """Keywords or tags."""
+    categories: NotRequired[list[str]]
+    """Categories or classifications."""
+    version: NotRequired[str]
+    """Version identifier."""
+    language: NotRequired[str]
+    """Document language code."""
+    references: NotRequired[list[str]]
+    """Reference entries."""
+    citations: NotRequired[list[str]]
+    """Citation identifiers."""
+    copyright: NotRequired[str]
+    """Copyright information."""
+    license: NotRequired[str]
+    """License information."""
+    identifier: NotRequired[str]
+    """Document identifier."""
+    publisher: NotRequired[str]
+    """Publisher name."""
+    contributors: NotRequired[list[str]]
+    """Additional contributors."""
+    creator: NotRequired[str]
+    """Document creator."""
+    institute: NotRequired[str | list[str]]
+    """Institute or organization."""
+
+
+@dataclass
+class PandocResult:
+    """Result of a pandoc conversion including content and metadata."""
+
+    content: str
+    """The processed markdown content."""
+    metadata: Metadata
+    """Document metadata extracted from the source."""
+
+
+def _extract_inline_text(node: dict[str, Any]) -> str | None:
+    if node_type := node.get(TYPE_FIELD):
+        if node_type == INLINE_STR:
+            return node.get(CONTENT_FIELD)
+        if node_type == INLINE_SPACE:
+            return " "
+        if node_type in (INLINE_EMPH, INLINE_STRONG):
+            return _extract_inlines(node.get(CONTENT_FIELD, []))
+    return None  # pragma: no cover
+
+
+def _extract_inlines(nodes: list[dict[str, Any]]) -> str | None:
+    texts = [text for node in nodes if (text := _extract_inline_text(node))]
+    result = "".join(texts).strip()
+    return result if result else None
+
+
+def _extract_meta_value(node: Any) -> str | list[str] | None:
+    if not isinstance(node, dict) or CONTENT_FIELD not in node or TYPE_FIELD not in node:
+        return None
+
+    content = node[CONTENT_FIELD]
+    node_type = node[TYPE_FIELD]
+
+    if not content or node_type not in {
+        META_STRING,
+        META_INLINES,
+        META_LIST,
+        META_BLOCKS,
+    }:
+        return None
+
+    if node_type == META_STRING and isinstance(content, str):
+        return content
+
+    if isinstance(content, list) and (content := [v for v in content if isinstance(v, dict)]):
+        if node_type == META_INLINES:
+            return _extract_inlines(cast(list[dict[str, Any]], content))
+
+        if node_type == META_LIST:
+            results = []
+            for value in [value for item in content if (value := _extract_meta_value(item))]:
+                if isinstance(value, list):
+                    results.extend(value)  # pragma: no cover
+                else:
+                    results.append(value)
+            return results
+
+        if blocks := [block for block in content if block.get(TYPE_FIELD) == BLOCK_PARA]:
+            block_texts = []
+            for block in blocks:
+                block_content = block.get(CONTENT_FIELD, [])
+                if isinstance(block_content, list) and (text := _extract_inlines(block_content)):
+                    block_texts.append(text)
+            return block_texts if block_texts else None
+
+    return None
+
+
+def _extract_metadata(raw_meta: dict[str, Any]) -> Metadata:
+    """Extract all non-empty metadata values from Pandoc AST metadata."""
+    meta: Metadata = {}
+
+    for key, value in raw_meta.items():
+        if extracted := _extract_meta_value(value):
+            meta[key] = extracted  # type: ignore[literal-required]
+
+    citations = [
+        cite["citationId"]
+        for block in raw_meta.get("blocks", [])
+        if block.get(TYPE_FIELD) == "Cite"
+        for cite in block.get(CONTENT_FIELD, [[{}]])[0]
+        if isinstance(cite, dict)
+    ]
+    if citations:
+        meta["citations"] = citations
+
+    return meta
+
+
+def _get_extension_from_mime_type(mime_type: str) -> str:
+    if mime_type not in PANDOC_MIMETYPE_TO_FORMAT_MAPPING or not any(
+        mime_type.startswith(value) for value in PANDOC_MIMETYPE_TO_FORMAT_MAPPING
+    ):
+        raise ValidationError(
+            f"Unsupported mime type: {mime_type}",
+            context={
+                "mime_type": mime_type,
+                "supported_mimetypes": ",".join(sorted(PANDOC_MIMETYPE_TO_FORMAT_MAPPING)),
+            },
+        )
+
+    return PANDOC_MIMETYPE_TO_FORMAT_MAPPING.get(mime_type) or next(
+        PANDOC_MIMETYPE_TO_FORMAT_MAPPING[k] for k in PANDOC_MIMETYPE_TO_FORMAT_MAPPING if k.startswith(mime_type)
+    )
+
+
+async def validate_pandoc_version() -> None:
+    """Validate that Pandoc is installed and is version 3 or above.
+
+    Raises:
+        MissingDependencyError: If Pandoc is not installed or is below version 3.
+    """
+    try:
+        if version_ref["checked"]:
+            return
+
+        result = await run_sync(subprocess.run, ["pandoc", "--version"], capture_output=True)
+        version = result.stdout.decode().split("\n")[0].split()[1]
+        if not version.startswith("3."):
+            raise MissingDependencyError("Pandoc version 3 or above is required.")
+
+        version_ref["checked"] = True
+
+    except FileNotFoundError as e:
+        raise MissingDependencyError("Pandoc is not installed.") from e
+
+
+async def extract_metadata(input_file: str | PathLike[str], *, mime_type: str) -> Metadata:
+    """Extract metadata from a document using pandoc.
+
+    Args:
+        input_file: The path to the file to process.
+        mime_type: The mime type of the file.
+
+    Raises:
+        ParsingError: If Pandoc fails to extract metadata.
+
+    Returns:
+        Dictionary containing document metadata.
+    """
+    extension = _get_extension_from_mime_type(mime_type)
+
+    with NamedTemporaryFile(suffix=".json") as metadata_file:
+        try:
+            command = [
+                "pandoc",
+                str(input_file),
+                f"--from={extension}",
+                "--to=json",
+                "--standalone",
+                "--quiet",
+                "--output",
+                metadata_file.name,
+            ]
+
+            result = await run_sync(
+                subprocess.run,
+                command,
+                capture_output=True,
+            )
+
+            if result.returncode != 0:
+                raise ParsingError(
+                    "Failed to extract file data", context={"file": str(input_file), "error": result.stderr.decode()}
+                )
+
+            json_data = json.loads(await AsyncPath(metadata_file.name).read_text())
+            return _extract_metadata(json_data)
+
+        except (RuntimeError, OSError, json.JSONDecodeError) as e:
+            raise ParsingError("Failed to extract file data", context={"file": str(input_file)}) from e
+
+
+async def _extract_file(input_file: str | PathLike[str], *, mime_type: str, extra_args: list[str] | None = None) -> str:
+    extension = _get_extension_from_mime_type(mime_type)
+
+    with NamedTemporaryFile(suffix=".md") as output_file:
+        command = [
+            "pandoc",
+            str(input_file),
+            f"--from={extension}",
+            "--to=markdown",
+            "--standalone",
+            "--wrap=preserve",
+            "--quiet",
+            "--output",
+            output_file.name,
+        ]
+
+        if extra_args:
+            command.extend(extra_args)
+
+        result = await run_sync(
+            subprocess.run,
+            command,
+            capture_output=True,
+        )
+
+        if result.returncode != 0:
+            raise ParsingError(
+                "Failed to extract file data", context={"file": str(input_file), "error": result.stderr.decode()}
+            )
+
+        text = await AsyncPath(output_file.name).read_text()
+
+        return normalize_spaces(text)
+
+
+async def process_file(
+    input_file: str | PathLike[str], *, mime_type: str, extra_args: list[str] | None = None
+) -> PandocResult:
+    """Process a single file using Pandoc and convert to markdown.
+
+    Args:
+        input_file: The path to the file to process.
+        mime_type: The mime type of the file.
+        extra_args: Additional Pandoc command line arguments.
+
+    Returns:
+        PandocResult containing processed content and metadata.
+    """
+    await validate_pandoc_version()
+
+    metadata, content = await gather(
+        *[
+            extract_metadata(input_file, mime_type=mime_type),
+            _extract_file(input_file, mime_type=mime_type, extra_args=extra_args),
+        ]
+    )
+    return PandocResult(
+        content=content,  # type: ignore[arg-type]
+        metadata=metadata,  # type: ignore[arg-type]
+    )
+
+
+async def process_content(content: bytes, *, mime_type: str, extra_args: list[str] | None = None) -> PandocResult:
+    """Process content using Pandoc and convert to markdown.
+
+    Args:
+        content: The content to process.
+        mime_type: The mime type of the content.
+        extra_args: Additional Pandoc command line arguments.
+
+    Returns:
+        PandocResult containing processed content and metadata.
+    """
+    extension = _get_extension_from_mime_type(mime_type)
+
+    with NamedTemporaryFile(suffix=f".{extension}") as input_file:
+        await AsyncPath(input_file.name).write_bytes(content)
+        return await process_file(input_file.name, mime_type=mime_type, extra_args=extra_args)

kreuzberg-1.5.0.dist-info/METADATA

@@ -0,0 +1,318 @@
+Metadata-Version: 2.2
+Name: kreuzberg
+Version: 1.5.0
+Summary: A text extraction library supporting PDFs, images, office documents and more
+Author-email: Na'aman Hirschfeld <nhirschfed@gmail.com>
+License: MIT
+Project-URL: homepage, https://github.com/Goldziher/kreuzberg
+Keywords: document-processing,image-to-text,ocr,pandoc,pdf-extraction,rag,tesseract,text-extraction,text-processing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: General
+Classifier: Topic :: Utilities
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: anyio>=4.8.0
+Requires-Dist: charset-normalizer>=3.4.1
+Requires-Dist: html-to-markdown>=1.2.0
+Requires-Dist: pypdfium2>=4.30.1
+Requires-Dist: python-pptx>=1.0.2
+Requires-Dist: typing-extensions>=4.12.2; python_version < "3.10"
+
+# Kreuzberg
+
+Kreuzberg is a modern Python library for text extraction from documents, designed for simplicity and efficiency. It provides a unified async interface for extracting text from a wide range of file formats including PDFs, images, office documents, and more.
+
+## Why Kreuzberg?
+
+- **Simple and Hassle-Free**: Clean API that just works, without complex configuration
+- **Local Processing**: No external API calls or cloud dependencies required
+- **Resource Efficient**: Lightweight processing without GPU requirements
+- **Format Support**: Comprehensive support for documents, images, and text formats
+- **Modern Python**: Built with async/await, type hints, and current best practices
+
+Kreuzberg was created to solve text extraction needs in RAG (Retrieval Augmented Generation) applications, but it's suitable for any text extraction use case. Unlike many commercial solutions that require API calls or complex setups, Kreuzberg focuses on local processing with minimal dependencies.
+
+## Features
+
+- **Universal Text Extraction**: Extract text from PDFs (both searchable and scanned), images, office documents, and more
+- **Smart Processing**: Automatic OCR for scanned documents, encoding detection for text files
+- **Modern Python Design**:
+  - Async-first API using `anyio`
+  - Comprehensive type hints for better IDE support
+  - Detailed error handling with context information
+- **Production Ready**:
+  - Robust error handling
+  - Detailed debugging information
+  - Memory efficient processing
+
+## Installation
+
+### 1. Install the Python Package
+
+```shell
+pip install kreuzberg
+```
+
+### 2. Install System Dependencies
+
+Kreuzberg requires two open-source tools:
+
+- [Pandoc](https://pandoc.org/installing.html) - For document format conversion
+
+  - GPL v2.0 licensed (used via CLI only)
+  - Handles office documents and markup formats
+
+- [Tesseract OCR](https://tesseract-ocr.github.io/) - For image and PDF OCR
+  - Apache License
+  - Required for scanned documents and images
+
+## Architecture
+
+Kreuzberg is designed as a high-level async abstraction over established open-source tools. It integrates:
+
+- **PDF Processing**:
+  - `pdfium2` for searchable PDFs
+  - Tesseract OCR for scanned content
+- **Document Conversion**:
+  - Pandoc for office documents and markup
+  - `python-pptx` for PowerPoint files
+  - `html-to-markdown` for HTML content
+- **Text Processing**:
+  - Smart encoding detection
+  - Markdown and plain text handling
+
+### Supported Formats
+
+#### Document Formats
+
+- PDF (`.pdf`, both searchable and scanned documents)
+- Microsoft Word (`.docx`, `.doc`)
+- PowerPoint presentations (`.pptx`)
+- OpenDocument Text (`.odt`)
+- Rich Text Format (`.rtf`)
+- EPUB (`.epub`)
+- DocBook XML (`.dbk`, `.xml`)
+- FictionBook (`.fb2`)
+- LaTeX (`.tex`, `.latex`)
+- Typst (`.typ`)
+
+#### Markup and Text Formats
+
+- HTML (`.html`, `.htm`)
+- Plain text (`.txt`) and Markdown (`.md`, `.markdown`)
+- reStructuredText (`.rst`)
+- Org-mode (`.org`)
+- DokuWiki (`.txt`)
+- Pod (`.pod`)
+- Man pages (`.1`, `.2`, etc.)
+
+#### Data and Research Formats
+
+- CSV (`.csv`) and TSV (`.tsv`) files
+- Jupyter Notebooks (`.ipynb`)
+- BibTeX (`.bib`) and BibLaTeX (`.bib`)
+- CSL-JSON (`.json`)
+- EndNote XML (`.xml`)
+- RIS (`.ris`)
+- JATS XML (`.xml`)
+
+#### Image Formats
+
+- JPEG (`.jpg`, `.jpeg`, `.pjpeg`)
+- PNG (`.png`)
+- TIFF (`.tiff`, `.tif`)
+- BMP (`.bmp`)
+- GIF (`.gif`)
+- WebP (`.webp`)
+- JPEG 2000 (`.jp2`, `.jpx`, `.jpm`, `.mj2`)
+- Portable Anymap (`.pnm`)
+- Portable Bitmap (`.pbm`)
+- Portable Graymap (`.pgm`)
+- Portable Pixmap (`.ppm`)
+
+## Usage
+
+Kreuzberg provides a simple, async-first API for text extraction. The library exports two main functions:
+
+- `extract_file()`: Extract text from a file (accepts string path or `pathlib.Path`)
+- `extract_bytes()`: Extract text from bytes (accepts a byte string)
+
+### Quick Start
+
+```python
+from pathlib import Path
+from kreuzberg import extract_file, extract_bytes
+
+# Basic file extraction
+async def extract_document():
+    # Extract from a PDF file
+    pdf_result = await extract_file("document.pdf")
+    print(f"PDF text: {pdf_result.content}")
+
+    # Extract from an image
+    img_result = await extract_file("scan.png")
+    print(f"Image text: {img_result.content}")
+
+    # Extract from Word document
+    docx_result = await extract_file(Path("document.docx"))
+    print(f"Word text: {docx_result.content}")
+```
+
+### Processing Uploaded Files
+
+```python
+from kreuzberg import extract_bytes
+
+async def process_upload(file_content: bytes, mime_type: str):
+    """Process uploaded file content with known MIME type."""
+    result = await extract_bytes(file_content, mime_type=mime_type)
+    return result.content
+
+# Example usage with different file types
+async def handle_uploads():
+    # Process PDF upload
+    pdf_result = await extract_bytes(pdf_bytes, mime_type="application/pdf")
+
+    # Process image upload
+    img_result = await extract_bytes(image_bytes, mime_type="image/jpeg")
+
+    # Process Word document upload
+    docx_result = await extract_bytes(docx_bytes,
+        mime_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document")
+```
+
+### Advanced Features
+
+#### PDF Processing Options
+
+```python
+from kreuzberg import extract_file
+
+async def process_pdf():
+    # Force OCR for PDFs with embedded images or scanned content
+    result = await extract_file("document.pdf", force_ocr=True)
+
+    # Process a scanned PDF (automatically uses OCR)
+    scanned = await extract_file("scanned.pdf")
+```
+
+#### ExtractionResult Object
+
+All extraction functions return an `ExtractionResult` containing:
+
+- `content`: The extracted text (str)
+- `mime_type`: Output format ("text/plain" or "text/markdown" for Pandoc conversions)
+
+```python
+from kreuzberg import ExtractionResult
+
+async def process_document(path: str) -> tuple[str, str]:
+    # Access as a named tuple
+    result: ExtractionResult = await extract_file(path)
+    print(f"Content: {result.content}")
+    print(f"Format: {result.mime_type}")
+
+    # Or unpack as a tuple
+    content, mime_type = await extract_file(path)
+    return content, mime_type
+```
+
+### Error Handling
+
+Kreuzberg provides detailed error handling with two main exception types:
+
+```python
+from kreuzberg import extract_file
+from kreuzberg.exceptions import ValidationError, ParsingError
+
+async def safe_extract(path: str) -> str:
+    try:
+        result = await extract_file(path)
+        return result.content
+
+    except ValidationError as e:
+        # Handles input validation issues:
+        # - Unsupported file types
+        # - Missing files
+        # - Invalid MIME types
+        print(f"Invalid input: {e.message}")
+        print(f"Details: {e.context}")
+
+    except ParsingError as e:
+        # Handles processing errors:
+        # - PDF parsing failures
+        # - OCR errors
+        # - Format conversion issues
+        print(f"Processing failed: {e.message}")
+        print(f"Details: {e.context}")
+
+    return ""
+
+# Example error contexts
+try:
+    result = await extract_file("document.xyz")
+except ValidationError as e:
+    # e.context might contain:
+    # {
+    #    "file_path": "document.xyz",
+    #    "error": "Unsupported file type",
+    #    "supported_types": ["pdf", "docx", ...]
+    # }
+
+try:
+    result = await extract_file("scan.pdf")
+except ParsingError as e:
+    # e.context might contain:
+    # {
+    #    "file_path": "scan.pdf",
+    #    "error": "OCR processing failed",
+    #    "details": "Tesseract error: Unable to process image"
+    # }
+```
+
+## Roadmap
+
+V1:
+
+- [x] - html file text extraction
+- [ ] - better PDF table extraction
+- [ ] - batch APIs
+- [ ] - sync APIs
+
+V2:
+
+- [ ] - metadata extraction (breaking change)
+- [ ] - TBD
+
+## Contribution
+
+This library is open to contribution. Feel free to open issues or submit PRs. Its better to discuss issues before
+submitting PRs to avoid disappointment.
+
+### Local Development
+
+1. Clone the repo
+2. Install the system dependencies
+3. Install the full dependencies with `uv sync`
+4. Install the pre-commit hooks with:
+   ```shell
+   pre-commit install && pre-commit install --hook-type commit-msg
+   ```
+5. Make your changes and submit a PR
+
+## License
+
+This library uses the MIT license.

{kreuzberg-1.4.0.dist-info → kreuzberg-1.5.0.dist-info}/RECORD

@@ -1,14 +1,15 @@
 kreuzberg/__init__.py,sha256=5IBPjPsZ7faK15gFB9ZEROHhkEX7KKQmrHPCZuGnhb0,285
-kreuzberg/_extractors.py,sha256=Z6fxNMODsiNGPBv8gYpZ0jrc2hPbX-56xdrVPJ-6SQ4,7658
-kreuzberg/_mime_types.py,sha256=hR6LFXWn8dtCDB05PkADYk2l__HpmETNyf4YFixhecE,2918
+kreuzberg/_extractors.py,sha256=k6xO_2ItaftPmlqzfXyxTn8rdaWdwrJHGziBbo7gCio,6599
+kreuzberg/_mime_types.py,sha256=0ZYtRrMAaKpCMDkhpTbWAXHCsVob5MFRMGlbni8iYSA,2573
+kreuzberg/_pandoc.py,sha256=DC6y_NN_CG9dF6fhAj3WumXqKIJLjYmnql2H53_KHnE,13766
 kreuzberg/_string.py,sha256=4txRDnkdR12oO6G8V-jXEMlA9ivgmw8E8EbjyhfL-W4,1106
 kreuzberg/_sync.py,sha256=ovsFHFdkcczz7gNEUJsbZzY8KHG0_GAOOYipQNE4hIY,874
 kreuzberg/_tesseract.py,sha256=nnhkjRIS0BSoovjMIqOlBEXlzngE0QJeFDe7BIqUik8,7872
 kreuzberg/exceptions.py,sha256=pxoEPS0T9e5QSgxsfXn1VmxsY_EGXvTwY0gETPiNn8E,945
 kreuzberg/extraction.py,sha256=gux3fkPIs8IbIKtRGuPFWJBLB5jO6Y9JsBfhHRcpQ0k,6160
 kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kreuzberg-1.4.0.dist-info/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
-kreuzberg-1.4.0.dist-info/METADATA,sha256=ul0iSWSu_1i029aq8X4T4ZboOzWpKK8wZRuvvLVqAoQ,8503
-kreuzberg-1.4.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-kreuzberg-1.4.0.dist-info/top_level.txt,sha256=rbGkygffkZiyKhL8UN41ZOjLfem0jJPA1Whtndne0rE,10
-kreuzberg-1.4.0.dist-info/RECORD,,
+kreuzberg-1.5.0.dist-info/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
+kreuzberg-1.5.0.dist-info/METADATA,sha256=O462ss7M6Cb8cO6fJXwqsOdzkzaZekqa1oGwb7Vrgx8,9641
+kreuzberg-1.5.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+kreuzberg-1.5.0.dist-info/top_level.txt,sha256=rbGkygffkZiyKhL8UN41ZOjLfem0jJPA1Whtndne0rE,10
+kreuzberg-1.5.0.dist-info/RECORD,,

kreuzberg-1.4.0.dist-info/METADATA

@@ -1,304 +0,0 @@
-Metadata-Version: 2.2
-Name: kreuzberg
-Version: 1.4.0
-Summary: A text extraction library supporting PDFs, images, office documents and more
-Author-email: Na'aman Hirschfeld <nhirschfed@gmail.com>
-License: MIT
-Project-URL: homepage, https://github.com/Goldziher/kreuzberg
-Keywords: document-processing,docx,image-to-text,latex,markdown,ocr,odt,office-documents,pandoc,pdf,pdf-extraction,rag,text-extraction,text-processing
-Classifier: Development Status :: 4 - Beta
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Classifier: Topic :: Text Processing :: General
-Classifier: Topic :: Utilities
-Classifier: Typing :: Typed
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: anyio>=4.8.0
-Requires-Dist: charset-normalizer>=3.4.1
-Requires-Dist: html-to-markdown>=1.2.0
-Requires-Dist: pypandoc>=1.15
-Requires-Dist: pypdfium2>=4.30.1
-Requires-Dist: python-pptx>=1.0.2
-
-# Kreuzberg
-
-Kreuzberg is a library for simplified text extraction from PDF files. It's meant to offer simple, hassle free text
-extraction.
-
-Why?
-
-I am building, like many do now, a RAG focused service (checkout https://grantflow.ai). I have text extraction needs.
-There are quite a lot of commercial options out there, and several open-source + paid options.
-But I wanted something simple, which does not require expansive round-trips to an external API.
-Furthermore, I wanted something that is easy to run locally and isn't very heavy / requires a GPU.
-
-Hence, this library.
-
-## Features
-
-- Extract text from PDFs, images, office documents and more (see supported formats below)
-- Use modern Python with async (via `anyio`) and proper type hints
-- Extensive error handling for easy debugging
-
-## Installation
-
-1. Begin by installing the python package:
-
-   ```shell
-
-   pip install kreuzberg
-
-   ```
-
-2. Install the system dependencies:
-
-- [pandoc](https://pandoc.org/installing.html) (non-pdf text extraction, GPL v2.0 licensed but used via CLI only)
-- [tesseract-ocr](https://tesseract-ocr.github.io/) (for image/PDF OCR, Apache License)
-
-## Dependencies and Philosophy
-
-This library is built to be minimalist and simple. It also aims to utilize OSS tools for the job. Its fundamentally a
-high order async abstraction on top of other tools, think of it like the library you would bake in your code base, but
-polished and well maintained.
-
-### Dependencies
-
-- PDFs are processed using pdfium2 for searchable PDFs + Tesseract OCR for scanned documents
-- Images are processed using Tesseract OCR
-- Office documents and other formats are processed using Pandoc
-- PPTX files are converted using python-pptx
-- HTML files are converted using html-to-markdown
-- Plain text files are read directly with appropriate encoding detection
-
-### Roadmap
-
-V1:
-
-- [x] - html file text extraction
-- [ ] - better PDF table extraction
-- [ ] - TBD
-
-V2:
-
-- [ ] - extra install groups (to make dependencies optional)
-- [ ] - metadata extraction (possible breaking change)
-- [ ] - TBD
-
-### Feature Requests
-
-Feel free to open a discussion in GitHub or an issue if you have any feature requests
-
-### Contribution
-
-Is welcome! Read guidelines below.
-
-## Supported File Types
-
-Kreuzberg supports a wide range of file formats:
-
-### Document Formats
-
-- PDF (`.pdf`) - both searchable and scanned documents
-- Word Documents (`.docx`, `.doc`)
-- Power Point Presentations (`.pptx`)
-- OpenDocument Text (`.odt`)
-- Rich Text Format (`.rtf`)
-
-### Image Formats
-
-- JPEG, JPG (`.jpg`, `.jpeg`, `.pjpeg`)
-- PNG (`.png`)
-- TIFF (`.tiff`, `.tif`)
-- BMP (`.bmp`)
-- GIF (`.gif`)
-- WebP (`.webp`)
-- JPEG 2000 (`.jp2`, `.jpx`, `.jpm`, `.mj2`)
-- Portable Anymap (`.pnm`)
-- Portable Bitmap (`.pbm`)
-- Portable Graymap (`.pgm`)
-- Portable Pixmap (`.ppm`)
-
-#### Text and Markup Formats
-
-- HTML (`.html`, `.htm`)
-- Plain Text (`.txt`)
-- Markdown (`.md`)
-- reStructuredText (`.rst`)
-- LaTeX (`.tex`)
-
-#### Data Formats
-
-- Comma-Separated Values (`.csv`)
-- Tab-Separated Values (`.tsv`)
-
-## Usage
-
-Kreuzberg exports two async functions:
-
-- Extract text from a file (string path or `pathlib.Path`) using `extract_file()`
-- Extract text from a byte-string using `extract_bytes()`
-
-### Extract from File
-
-```python
-from pathlib import Path
-from kreuzberg import extract_file
-
-
-# Extract text from a PDF file
-async def extract_pdf():
-    result = await extract_file("document.pdf")
-    print(f"Extracted text: {result.content}")
-    print(f"Output mime type: {result.mime_type}")
-
-
-# Extract text from an image
-async def extract_image():
-    result = await extract_file("scan.png")
-    print(f"Extracted text: {result.content}")
-
-
-# or use Path
-
-async def extract_pdf():
-    result = await extract_file(Path("document.pdf"))
-    print(f"Extracted text: {result.content}")
-    print(f"Output mime type: {result.mime_type}")
-```
-
-### Extract from Bytes
-
-```python
-from kreuzberg import extract_bytes
-
-
-# Extract text from PDF bytes
-async def process_uploaded_pdf(pdf_content: bytes):
-    result = await extract_bytes(pdf_content, mime_type="application/pdf")
-    return result.content
-
-
-# Extract text from image bytes
-async def process_uploaded_image(image_content: bytes):
-    result = await extract_bytes(image_content, mime_type="image/jpeg")
-    return result.content
-```
-
-### Forcing OCR
-
-When extracting a PDF file or bytes, you might want to force OCR - for example, if the PDF includes images that have text that should be extracted etc.
-You can do this by passing `force_ocr=True`:
-
-```python
-from kreuzberg import extract_bytes
-
-
-# Extract text from PDF bytes and force OCR
-async def process_uploaded_pdf(pdf_content: bytes):
-    result = await extract_bytes(pdf_content, mime_type="application/pdf", force_ocr=True)
-    return result.content
-```
-
-### Error Handling
-
-Kreuzberg raises two exception types:
-
-#### ValidationError
-
-Raised when there are issues with input validation:
-
-- Unsupported mime types
-- Undetectable mime types
-- Path doesn't point at an exist file
-
-#### ParsingError
-
-Raised when there are issues during the text extraction process:
-
-- PDF parsing failures
-- OCR errors
-- Pandoc conversion errors
-
-```python
-from kreuzberg import extract_file
-from kreuzberg.exceptions import ValidationError, ParsingError
-
-
-async def safe_extract():
-    try:
-        result = await extract_file("document.doc")
-        return result.content
-    except ValidationError as e:
-        print(f"Validation error: {e.message}")
-        print(f"Context: {e.context}")
-    except ParsingError as e:
-        print(f"Parsing error: {e.message}")
-        print(f"Context: {e.context}")  # Contains detailed error information
-```
-
-Both error types include helpful context information for debugging:
-
-```python
-try:
-    result = await extract_file("scanned.pdf")
-except ParsingError as e:
-# e.context might contain:
-# {
-#    "file_path": "scanned.pdf",
-#    "error": "Tesseract OCR failed: Unable to process image"
-# }
-```
-
-### ExtractionResult
-
-All extraction functions return an ExtractionResult named tuple containing:
-
-- `content`: The extracted text as a string
-- `mime_type`: The mime type of the output (either "text/plain" or, if pandoc is used- "text/markdown")
-
-```python
-from kreuzberg import ExtractionResult
-
-
-async def process_document(path: str) -> str:
-    result: ExtractionResult = await extract_file(path)
-    return result.content
-
-
-# or access the result as tuple
-
-async def process_document(path: str) -> str:
-    content, mime_type = await extract_file(path)
-    # do something with mime_type
-    return content
-```
-
-## Contribution
-
-This library is open to contribution. Feel free to open issues or submit PRs. Its better to discuss issues before
-submitting PRs to avoid disappointment.
-
-### Local Development
-
-1. Clone the repo
-2. Install the system dependencies
-3. Install the full dependencies with `uv sync`
-4. Install the pre-commit hooks with:
-   ```shell
-   pre-commit install && pre-commit install --hook-type commit-msg
-   ```
-5. Make your changes and submit a PR
-
-## License
-
-This library uses the MIT license.