kreuzberg 3.0.0__tar.gz → 3.1.0__tar.gz

{kreuzberg-3.0.0 → kreuzberg-3.1.0}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.0.0
+Version: 3.1.0
 Summary: A text extraction library supporting PDFs, images, office documents and more
 Author-email: Na'aman Hirschfeld <nhirschfed@gmail.com>
 License: MIT
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
-Keywords: document-processing,image-to-text,ocr,pandoc,pdf-extraction,rag,tesseract,text-extraction,text-processing
+Keywords: document-processing,image-to-text,ocr,pandoc,pdf-extraction,rag,table-extraction,tesseract,text-extraction,text-processing
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -27,7 +27,7 @@ License-File: LICENSE
 Requires-Dist: anyio>=4.9.0
 Requires-Dist: charset-normalizer>=3.4.1
 Requires-Dist: exceptiongroup>=1.2.2; python_version < "3.11"
-Requires-Dist: html-to-markdown>=1.2.0
+Requires-Dist: html-to-markdown>=1.2.1
 Requires-Dist: playa-pdf>=0.4.1
 Requires-Dist: pypdfium2==4.30.0
 Requires-Dist: python-calamine>=0.3.1
@@ -35,19 +35,20 @@ Requires-Dist: python-pptx>=1.0.2
 Requires-Dist: typing-extensions>=4.12.2; python_version < "3.12"
 Provides-Extra: all
 Requires-Dist: easyocr>=1.7.2; extra == "all"
-Requires-Dist: numpy>=2.0.2; extra == "all"
+Requires-Dist: gmft>=0.4.1; extra == "all"
 Requires-Dist: paddleocr>=2.10.0; extra == "all"
-Requires-Dist: paddlepaddle>=2.6.2; python_version < "3.13" and extra == "all"
-Requires-Dist: semantic-text-splitter>=0.24.1; extra == "all"
+Requires-Dist: paddlepaddle>=3.0.0; extra == "all"
+Requires-Dist: semantic-text-splitter>=0.25.1; extra == "all"
 Requires-Dist: setuptools>=76.0.0; extra == "all"
 Provides-Extra: chunking
-Requires-Dist: semantic-text-splitter>=0.24.1; extra == "chunking"
+Requires-Dist: semantic-text-splitter>=0.25.1; extra == "chunking"
 Provides-Extra: easyocr
 Requires-Dist: easyocr>=1.7.2; extra == "easyocr"
+Provides-Extra: gmft
+Requires-Dist: gmft>=0.4.1; extra == "gmft"
 Provides-Extra: paddleocr
-Requires-Dist: numpy>=2.0.2; extra == "paddleocr"
 Requires-Dist: paddleocr>=2.10.0; extra == "paddleocr"
-Requires-Dist: paddlepaddle>=2.6.2; python_version < "3.13" and extra == "paddleocr"
+Requires-Dist: paddlepaddle>=3.0.0; extra == "paddleocr"
 Requires-Dist: setuptools>=76.0.0; extra == "paddleocr"
 Dynamic: license-file
 
@@ -66,6 +67,8 @@ Kreuzberg is a Python library for text extraction from documents. It provides a
 - **Resource Efficient**: Lightweight processing without GPU requirements
 - **Format Support**: Comprehensive support for documents, images, and text formats
 - **Multiple OCR Engines**: Support for Tesseract, EasyOCR, and PaddleOCR
+- **Metadata Extraction**: Get document metadata alongside text content
+- **Table Extraction**: Extract tables from documents using the excellent GMFT library
 - **Modern Python**: Built with async/await, type hints, and a functional-first approach
 - **Permissive OSS**: MIT licensed with permissively licensed dependencies
 
@@ -151,7 +154,7 @@ Kreuzberg supports multiple OCR engines:
 - **EasyOCR**: Good for many languages, pure Python, but downloads models on first use
 - **PaddleOCR**: Excellent for Asian languages, pure Python, but downloads models on first use
 
-For comparison and selection guidance, see the [OCR Backends](https://example.com/ocr-backends) documentation.
+For comparison and selection guidance, see the [OCR Backends](https://goldziher.github.io/kreuzberg/user-guide/ocr-backends/) documentation.
 
 ## Contribution
 
@@ -160,17 +163,9 @@ This library is open to contribution. Feel free to open issues or submit PRs. It
 ### Local Development
 
 1. Clone the repo
-
 1. Install the system dependencies
-
 1. Install the full dependencies with `uv sync`
-
-1. Install the pre-commit hooks with:
-
-    ```shell
-    pre-commit install && pre-commit install --hook-type commit-msg
-    ```
-
+1. Install the pre-commit hooks with: `pre-commit install && pre-commit install --hook-type commit-msg`
 1. Make your changes and submit a PR
 
 ## License

{kreuzberg-3.0.0 → kreuzberg-3.1.0}/README.md

@@ -13,6 +13,8 @@ Kreuzberg is a Python library for text extraction from documents. It provides a
 - **Resource Efficient**: Lightweight processing without GPU requirements
 - **Format Support**: Comprehensive support for documents, images, and text formats
 - **Multiple OCR Engines**: Support for Tesseract, EasyOCR, and PaddleOCR
+- **Metadata Extraction**: Get document metadata alongside text content
+- **Table Extraction**: Extract tables from documents using the excellent GMFT library
 - **Modern Python**: Built with async/await, type hints, and a functional-first approach
 - **Permissive OSS**: MIT licensed with permissively licensed dependencies
 
@@ -98,7 +100,7 @@ Kreuzberg supports multiple OCR engines:
 - **EasyOCR**: Good for many languages, pure Python, but downloads models on first use
 - **PaddleOCR**: Excellent for Asian languages, pure Python, but downloads models on first use
 
-For comparison and selection guidance, see the [OCR Backends](https://example.com/ocr-backends) documentation.
+For comparison and selection guidance, see the [OCR Backends](https://goldziher.github.io/kreuzberg/user-guide/ocr-backends/) documentation.
 
 ## Contribution
 
@@ -107,17 +109,9 @@ This library is open to contribution. Feel free to open issues or submit PRs. It
 ### Local Development
 
 1. Clone the repo
-
 1. Install the system dependencies
-
 1. Install the full dependencies with `uv sync`
-
-1. Install the pre-commit hooks with:
-
-    ```shell
-    pre-commit install && pre-commit install --hook-type commit-msg
-    ```
-
+1. Install the pre-commit hooks with: `pre-commit install && pre-commit install --hook-type commit-msg`
 1. Make your changes and submit a PR
 
 ## License

{kreuzberg-3.0.0 → kreuzberg-3.1.0}/kreuzberg/__init__.py

@@ -1,10 +1,11 @@
+from kreuzberg._gmft import GMFTConfig
 from kreuzberg._ocr._easyocr import EasyOCRConfig
 from kreuzberg._ocr._paddleocr import PaddleOCRConfig
 from kreuzberg._ocr._tesseract import TesseractConfig
 
 from ._ocr._tesseract import PSMMode
 from ._registry import ExtractorRegistry
-from ._types import ExtractionConfig, ExtractionResult, Metadata
+from ._types import ExtractionConfig, ExtractionResult, Metadata, TableData
 from .exceptions import KreuzbergError, MissingDependencyError, OCRError, ParsingError, ValidationError
 from .extraction import (
     batch_extract_bytes,
@@ -22,6 +23,7 @@ __all__ = [
     "ExtractionConfig",
     "ExtractionResult",
     "ExtractorRegistry",
+    "GMFTConfig",
     "KreuzbergError",
     "Metadata",
     "MissingDependencyError",
@@ -29,6 +31,7 @@ __all__ = [
     "PSMMode",
     "PaddleOCRConfig",
     "ParsingError",
+    "TableData",
     "TesseractConfig",
     "ValidationError",
     "batch_extract_bytes",

kreuzberg-3.1.0/kreuzberg/_extractors/_base.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, ClassVar
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from kreuzberg import ExtractionResult
+    from kreuzberg._types import ExtractionConfig
+
+
+class Extractor(ABC):
+    """Abstract base class for file content extraction.
+
+    This class provides the interface for different types of content extractors.
+    Subclasses are expected to implement the methods for extracting content
+    either asynchronously or synchronously and determining the supported MIME types.
+
+    Attributes:
+        SUPPORTED_MIME_TYPES: The set of supported mime types - all none abstract extractors must implement this.
+
+    Args:
+        mime_type: The MIME type that this extractor handles (e.g., "application/pdf").
+        config: Configuration options for the extraction process.
+    """
+
+    __slots__ = ("config", "mime_type")
+
+    SUPPORTED_MIME_TYPES: ClassVar[set[str]]
+
+    def __init__(self, mime_type: str, config: ExtractionConfig) -> None:
+        self.mime_type = mime_type
+        self.config = config
+
+    @abstractmethod
+    async def extract_bytes_async(self, content: bytes) -> ExtractionResult:
+        """Asynchronously extract content from a byte stream.
+
+        Args:
+            content: The byte content to extract.
+
+        Returns:
+            ExtractionResult: The extracted content along with metadata about the extraction.
+        """
+
+    @abstractmethod
+    async def extract_path_async(self, path: Path) -> ExtractionResult:
+        """Asynchronously extract content from a file located at the specified path.
+
+        Args:
+            path: The path to the file to process.
+
+        Returns:
+            ExtractionResult: The extracted content along with metadata about the extraction.
+        """
+
+    @abstractmethod
+    def extract_bytes_sync(self, content: bytes) -> ExtractionResult:
+        """Synchronously extract content from a byte stream.
+
+        Args:
+            content: The byte content to extract.
+
+        Returns:
+            ExtractionResult: The extracted content along with metadata about the extraction.
+        """
+
+    @abstractmethod
+    def extract_path_sync(self, path: Path) -> ExtractionResult:
+        """Synchronously extract content from a file located at the specified path.
+
+        Args:
+            path: The path to the file to process.
+
+        Returns:
+            ExtractionResult: The extracted content along with metadata about the extraction.
+        """
+
+    @classmethod
+    def supports_mimetype(cls, mime_type: str) -> bool:
+        """Verify whether the extractor supports the given MIME type.
+
+        Args:
+            mime_type: The MIME type to check (e.g., "application/pdf").
+
+        Returns:
+            bool: True if the MIME type is supported, False otherwise.
+        """
+        return mime_type in cls.SUPPORTED_MIME_TYPES or any(
+            mime_type.startswith(supported_type) for supported_type in cls.SUPPORTED_MIME_TYPES
+        )

kreuzberg-3.1.0/kreuzberg/_extractors/_html.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+import html_to_markdown
+from anyio import Path as AsyncPath
+
+from kreuzberg._extractors._base import Extractor
+from kreuzberg._mime_types import HTML_MIME_TYPE, MARKDOWN_MIME_TYPE
+from kreuzberg._types import ExtractionResult
+from kreuzberg._utils._string import normalize_spaces, safe_decode
+from kreuzberg._utils._sync import run_sync
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+class HTMLExtractor(Extractor):
+    SUPPORTED_MIME_TYPES: ClassVar[set[str]] = {HTML_MIME_TYPE}
+
+    async def extract_bytes_async(self, content: bytes) -> ExtractionResult:
+        return await run_sync(self.extract_bytes_sync, content)
+
+    async def extract_path_async(self, path: Path) -> ExtractionResult:
+        content = await AsyncPath(path).read_bytes()
+        return await run_sync(self.extract_bytes_sync, content)
+
+    def extract_bytes_sync(self, content: bytes) -> ExtractionResult:
+        result = html_to_markdown.convert_to_markdown(safe_decode(content))
+        return ExtractionResult(content=normalize_spaces(result), mime_type=MARKDOWN_MIME_TYPE, metadata={}, chunks=[])
+
+    def extract_path_sync(self, path: Path) -> ExtractionResult:
+        content = path.read_bytes()
+        return self.extract_bytes_sync(content)

kreuzberg-3.1.0/kreuzberg/_extractors/_image.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+import anyio
+from anyio import Path as AsyncPath
+
+from kreuzberg._extractors._base import Extractor
+from kreuzberg._mime_types import IMAGE_MIME_TYPES
+from kreuzberg._ocr import get_ocr_backend
+from kreuzberg._utils._tmp import create_temp_file
+from kreuzberg.exceptions import ValidationError
+
+if TYPE_CHECKING:  # pragma: no cover
+    from collections.abc import Mapping
+    from pathlib import Path
+
+    from kreuzberg._types import ExtractionResult
+
+
+class ImageExtractor(Extractor):
+    SUPPORTED_MIME_TYPES: ClassVar[set[str]] = IMAGE_MIME_TYPES
+
+    IMAGE_MIME_TYPE_EXT_MAP: ClassVar[Mapping[str, str]] = {
+        "image/bmp": "bmp",
+        "image/x-bmp": "bmp",
+        "image/x-ms-bmp": "bmp",
+        "image/gif": "gif",
+        "image/jpeg": "jpg",
+        "image/pjpeg": "jpg",
+        "image/png": "png",
+        "image/tiff": "tiff",
+        "image/x-tiff": "tiff",
+        "image/jp2": "jp2",
+        "image/jpx": "jpx",
+        "image/jpm": "jpm",
+        "image/mj2": "mj2",
+        "image/webp": "webp",
+        "image/x-portable-anymap": "pnm",
+        "image/x-portable-bitmap": "pbm",
+        "image/x-portable-graymap": "pgm",
+        "image/x-portable-pixmap": "ppm",
+    }
+
+    async def extract_bytes_async(self, content: bytes) -> ExtractionResult:
+        extension = self._get_extension_from_mime_type(self.mime_type)
+        file_path, unlink = await create_temp_file(f".{extension}")
+        await AsyncPath(file_path).write_bytes(content)
+        try:
+            return await self.extract_path_async(file_path)
+        finally:
+            await unlink()
+
+    async def extract_path_async(self, path: Path) -> ExtractionResult:
+        if self.config.ocr_backend is None:
+            raise ValidationError("ocr_backend is None, cannot perform OCR")
+
+        return await get_ocr_backend(self.config.ocr_backend).process_file(path, **self.config.get_config_dict())
+
+    def extract_bytes_sync(self, content: bytes) -> ExtractionResult:
+        return anyio.run(self.extract_bytes_async, content)
+
+    def extract_path_sync(self, path: Path) -> ExtractionResult:
+        return anyio.run(self.extract_path_async, path)
+
+    def _get_extension_from_mime_type(self, mime_type: str) -> str:
+        if mime_type in self.IMAGE_MIME_TYPE_EXT_MAP:
+            return self.IMAGE_MIME_TYPE_EXT_MAP[mime_type]
+
+        for k, v in self.IMAGE_MIME_TYPE_EXT_MAP.items():
+            if k.startswith(mime_type):
+                return v
+
+        raise ValidationError("unsupported mimetype", context={"mime_type": mime_type})