kreuzberg 2.1.1__tar.gz → 3.0.0__tar.gz

kreuzberg-3.0.0/PKG-INFO

@@ -0,0 +1,178 @@
+Metadata-Version: 2.4
+Name: kreuzberg
+Version: 3.0.0
+Summary: A text extraction library supporting PDFs, images, office documents and more
+Author-email: Na'aman Hirschfeld <nhirschfed@gmail.com>
+License: MIT
+Project-URL: homepage, https://github.com/Goldziher/kreuzberg
+Keywords: document-processing,image-to-text,ocr,pandoc,pdf-extraction,rag,tesseract,text-extraction,text-processing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: General
+Classifier: Topic :: Utilities
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: anyio>=4.9.0
+Requires-Dist: charset-normalizer>=3.4.1
+Requires-Dist: exceptiongroup>=1.2.2; python_version < "3.11"
+Requires-Dist: html-to-markdown>=1.2.0
+Requires-Dist: playa-pdf>=0.4.1
+Requires-Dist: pypdfium2==4.30.0
+Requires-Dist: python-calamine>=0.3.1
+Requires-Dist: python-pptx>=1.0.2
+Requires-Dist: typing-extensions>=4.12.2; python_version < "3.12"
+Provides-Extra: all
+Requires-Dist: easyocr>=1.7.2; extra == "all"
+Requires-Dist: numpy>=2.0.2; extra == "all"
+Requires-Dist: paddleocr>=2.10.0; extra == "all"
+Requires-Dist: paddlepaddle>=2.6.2; python_version < "3.13" and extra == "all"
+Requires-Dist: semantic-text-splitter>=0.24.1; extra == "all"
+Requires-Dist: setuptools>=76.0.0; extra == "all"
+Provides-Extra: chunking
+Requires-Dist: semantic-text-splitter>=0.24.1; extra == "chunking"
+Provides-Extra: easyocr
+Requires-Dist: easyocr>=1.7.2; extra == "easyocr"
+Provides-Extra: paddleocr
+Requires-Dist: numpy>=2.0.2; extra == "paddleocr"
+Requires-Dist: paddleocr>=2.10.0; extra == "paddleocr"
+Requires-Dist: paddlepaddle>=2.6.2; python_version < "3.13" and extra == "paddleocr"
+Requires-Dist: setuptools>=76.0.0; extra == "paddleocr"
+Dynamic: license-file
+
+# Kreuzberg
+
+[![PyPI version](https://badge.fury.io/py/kreuzberg.svg)](https://badge.fury.io/py/kreuzberg)
+[![Documentation](https://img.shields.io/badge/docs-GitHub_Pages-blue)](https://goldziher.github.io/kreuzberg/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+Kreuzberg is a Python library for text extraction from documents. It provides a unified interface for extracting text from PDFs, images, office documents, and more, with both async and sync APIs.
+
+## Why Kreuzberg?
+
+- **Simple and Hassle-Free**: Clean API that just works, without complex configuration
+- **Local Processing**: No external API calls or cloud dependencies required
+- **Resource Efficient**: Lightweight processing without GPU requirements
+- **Format Support**: Comprehensive support for documents, images, and text formats
+- **Multiple OCR Engines**: Support for Tesseract, EasyOCR, and PaddleOCR
+- **Modern Python**: Built with async/await, type hints, and a functional-first approach
+- **Permissive OSS**: MIT licensed with permissively licensed dependencies
+
+## Quick Start
+
+```bash
+pip install kreuzberg
+```
+
+Install pandoc:
+
+```bash
+# Ubuntu/Debian
+sudo apt-get install tesseract-ocr pandoc
+
+# macOS
+brew install tesseract pandoc
+
+# Windows
+choco install -y tesseract pandoc
+```
+
+The tesseract OCR engine is the default OCR engine. You can decide not to use it - and then either use one of the two alternative OCR engines, or have no OCR at all.
+
+### Alternative OCR engines
+
+```bash
+# Install with EasyOCR support
+pip install "kreuzberg[easyocr]"
+
+# Install with PaddleOCR support
+pip install "kreuzberg[paddleocr]"
+```
+
+## Quick Example
+
+```python
+import asyncio
+from kreuzberg import extract_file
+
+async def main():
+    # Extract text from a PDF
+    result = await extract_file("document.pdf")
+    print(result.content)
+
+    # Extract text from an image
+    result = await extract_file("scan.jpg")
+    print(result.content)
+
+    # Extract text from a Word document
+    result = await extract_file("report.docx")
+    print(result.content)
+
+asyncio.run(main())
+```
+
+## Documentation
+
+For comprehensive documentation, visit our [GitHub Pages](https://goldziher.github.io/kreuzberg/):
+
+- [Getting Started](https://goldziher.github.io/kreuzberg/getting-started/) - Installation and basic usage
+- [User Guide](https://goldziher.github.io/kreuzberg/user-guide/) - In-depth usage information
+- [API Reference](https://goldziher.github.io/kreuzberg/api-reference/) - Detailed API documentation
+- [Examples](https://goldziher.github.io/kreuzberg/examples/) - Code examples for common use cases
+- [OCR Configuration](https://goldziher.github.io/kreuzberg/user-guide/ocr-configuration/) - Configure OCR engines
+- [OCR Backends](https://goldziher.github.io/kreuzberg/user-guide/ocr-backends/) - Choose the right OCR engine
+
+## Supported Formats
+
+Kreuzberg supports a wide range of document formats:
+
+- **Documents**: PDF, DOCX, DOC, RTF, TXT, EPUB, etc.
+- **Images**: JPG, PNG, TIFF, BMP, GIF, etc.
+- **Spreadsheets**: XLSX, XLS, CSV, etc.
+- **Presentations**: PPTX, PPT, etc.
+- **Web Content**: HTML, XML, etc.
+
+## OCR Engines
+
+Kreuzberg supports multiple OCR engines:
+
+- **Tesseract** (Default): Lightweight, fast startup, requires system installation
+- **EasyOCR**: Good for many languages, pure Python, but downloads models on first use
+- **PaddleOCR**: Excellent for Asian languages, pure Python, but downloads models on first use
+
+For comparison and selection guidance, see the [OCR Backends](https://example.com/ocr-backends) documentation.
+
+## Contribution
+
+This library is open to contribution. Feel free to open issues or submit PRs. It's better to discuss issues before submitting PRs to avoid disappointment.
+
+### Local Development
+
+1. Clone the repo
+
+1. Install the system dependencies
+
+1. Install the full dependencies with `uv sync`
+
+1. Install the pre-commit hooks with:
+
+    ```shell
+    pre-commit install && pre-commit install --hook-type commit-msg
+    ```
+
+1. Make your changes and submit a PR
+
+## License
+
+This library is released under the MIT license.

kreuzberg-3.0.0/README.md

@@ -0,0 +1,125 @@
+# Kreuzberg
+
+[![PyPI version](https://badge.fury.io/py/kreuzberg.svg)](https://badge.fury.io/py/kreuzberg)
+[![Documentation](https://img.shields.io/badge/docs-GitHub_Pages-blue)](https://goldziher.github.io/kreuzberg/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+Kreuzberg is a Python library for text extraction from documents. It provides a unified interface for extracting text from PDFs, images, office documents, and more, with both async and sync APIs.
+
+## Why Kreuzberg?
+
+- **Simple and Hassle-Free**: Clean API that just works, without complex configuration
+- **Local Processing**: No external API calls or cloud dependencies required
+- **Resource Efficient**: Lightweight processing without GPU requirements
+- **Format Support**: Comprehensive support for documents, images, and text formats
+- **Multiple OCR Engines**: Support for Tesseract, EasyOCR, and PaddleOCR
+- **Modern Python**: Built with async/await, type hints, and a functional-first approach
+- **Permissive OSS**: MIT licensed with permissively licensed dependencies
+
+## Quick Start
+
+```bash
+pip install kreuzberg
+```
+
+Install pandoc:
+
+```bash
+# Ubuntu/Debian
+sudo apt-get install tesseract-ocr pandoc
+
+# macOS
+brew install tesseract pandoc
+
+# Windows
+choco install -y tesseract pandoc
+```
+
+The tesseract OCR engine is the default OCR engine. You can decide not to use it - and then either use one of the two alternative OCR engines, or have no OCR at all.
+
+### Alternative OCR engines
+
+```bash
+# Install with EasyOCR support
+pip install "kreuzberg[easyocr]"
+
+# Install with PaddleOCR support
+pip install "kreuzberg[paddleocr]"
+```
+
+## Quick Example
+
+```python
+import asyncio
+from kreuzberg import extract_file
+
+async def main():
+    # Extract text from a PDF
+    result = await extract_file("document.pdf")
+    print(result.content)
+
+    # Extract text from an image
+    result = await extract_file("scan.jpg")
+    print(result.content)
+
+    # Extract text from a Word document
+    result = await extract_file("report.docx")
+    print(result.content)
+
+asyncio.run(main())
+```
+
+## Documentation
+
+For comprehensive documentation, visit our [GitHub Pages](https://goldziher.github.io/kreuzberg/):
+
+- [Getting Started](https://goldziher.github.io/kreuzberg/getting-started/) - Installation and basic usage
+- [User Guide](https://goldziher.github.io/kreuzberg/user-guide/) - In-depth usage information
+- [API Reference](https://goldziher.github.io/kreuzberg/api-reference/) - Detailed API documentation
+- [Examples](https://goldziher.github.io/kreuzberg/examples/) - Code examples for common use cases
+- [OCR Configuration](https://goldziher.github.io/kreuzberg/user-guide/ocr-configuration/) - Configure OCR engines
+- [OCR Backends](https://goldziher.github.io/kreuzberg/user-guide/ocr-backends/) - Choose the right OCR engine
+
+## Supported Formats
+
+Kreuzberg supports a wide range of document formats:
+
+- **Documents**: PDF, DOCX, DOC, RTF, TXT, EPUB, etc.
+- **Images**: JPG, PNG, TIFF, BMP, GIF, etc.
+- **Spreadsheets**: XLSX, XLS, CSV, etc.
+- **Presentations**: PPTX, PPT, etc.
+- **Web Content**: HTML, XML, etc.
+
+## OCR Engines
+
+Kreuzberg supports multiple OCR engines:
+
+- **Tesseract** (Default): Lightweight, fast startup, requires system installation
+- **EasyOCR**: Good for many languages, pure Python, but downloads models on first use
+- **PaddleOCR**: Excellent for Asian languages, pure Python, but downloads models on first use
+
+For comparison and selection guidance, see the [OCR Backends](https://example.com/ocr-backends) documentation.
+
+## Contribution
+
+This library is open to contribution. Feel free to open issues or submit PRs. It's better to discuss issues before submitting PRs to avoid disappointment.
+
+### Local Development
+
+1. Clone the repo
+
+1. Install the system dependencies
+
+1. Install the full dependencies with `uv sync`
+
+1. Install the pre-commit hooks with:
+
+    ```shell
+    pre-commit install && pre-commit install --hook-type commit-msg
+    ```
+
+1. Make your changes and submit a PR
+
+## License
+
+This library is released under the MIT license.

{kreuzberg-2.1.1 → kreuzberg-3.0.0}/kreuzberg/__init__.py

@@ -1,5 +1,10 @@
-from ._tesseract import PSMMode
-from ._types import ExtractionResult, Metadata
+from kreuzberg._ocr._easyocr import EasyOCRConfig
+from kreuzberg._ocr._paddleocr import PaddleOCRConfig
+from kreuzberg._ocr._tesseract import TesseractConfig
+
+from ._ocr._tesseract import PSMMode
+from ._registry import ExtractorRegistry
+from ._types import ExtractionConfig, ExtractionResult, Metadata
 from .exceptions import KreuzbergError, MissingDependencyError, OCRError, ParsingError, ValidationError
 from .extraction import (
     batch_extract_bytes,
@@ -7,22 +12,31 @@ from .extraction import (
     batch_extract_file,
     batch_extract_file_sync,
     extract_bytes,
+    extract_bytes_sync,
     extract_file,
+    extract_file_sync,
 )
 
 __all__ = [
+    "EasyOCRConfig",
+    "ExtractionConfig",
     "ExtractionResult",
+    "ExtractorRegistry",
     "KreuzbergError",
     "Metadata",
     "MissingDependencyError",
     "OCRError",
     "PSMMode",
+    "PaddleOCRConfig",
     "ParsingError",
+    "TesseractConfig",
     "ValidationError",
     "batch_extract_bytes",
     "batch_extract_bytes_sync",
     "batch_extract_file",
     "batch_extract_file_sync",
     "extract_bytes",
+    "extract_bytes_sync",
     "extract_file",
+    "extract_file_sync",
 ]

kreuzberg-3.0.0/kreuzberg/_chunker.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from kreuzberg import MissingDependencyError
+from kreuzberg._constants import DEFAULT_MAX_CHARACTERS, DEFAULT_MAX_OVERLAP
+from kreuzberg._mime_types import MARKDOWN_MIME_TYPE
+
+if TYPE_CHECKING:
+    from semantic_text_splitter import MarkdownSplitter, TextSplitter
+
+_chunkers: dict[tuple[int, int, str], MarkdownSplitter | TextSplitter] = {}
+
+
+def get_chunker(
+    mime_type: str,
+    max_characters: int = DEFAULT_MAX_CHARACTERS,
+    overlap_characters: int = DEFAULT_MAX_OVERLAP,
+) -> MarkdownSplitter | TextSplitter:
+    """Creates and returns a Chunker object configured with the given maximum
+    characters per chunk and overlap between chunks.
+
+    Args:
+        mime_type: The mime type of the content.
+        max_characters: Maximum number of characters allowed in each chunk.
+        overlap_characters: Number of characters overlapping between two consecutive chunks.
+
+    Raises:
+        MissingDependencyError: if semantic-text-splitter is not installed.
+
+    Returns:
+        Chunker: A Chunker object configured with the specified maximum
+            characters and overlap.
+    """
+    key = (max_characters, overlap_characters, mime_type)
+    if key not in _chunkers:
+        try:
+            if mime_type == MARKDOWN_MIME_TYPE:
+                from semantic_text_splitter import MarkdownSplitter
+
+                _chunkers[key] = MarkdownSplitter(max_characters, overlap_characters)
+            else:
+                from semantic_text_splitter import TextSplitter
+
+                _chunkers[key] = TextSplitter(max_characters, overlap_characters)
+        except ImportError as e:
+            raise MissingDependencyError.create_for_package(
+                dependency_group="chunking", functionality="chunking", package_name="semantic-text-splitter"
+            ) from e
+
+    return _chunkers[key]

kreuzberg-3.0.0/kreuzberg/_constants.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from typing import Final
+
+MINIMAL_SUPPORTED_PANDOC_VERSION: Final[int] = 2
+DEFAULT_MAX_CHARACTERS: Final[int] = 2000
+DEFAULT_MAX_OVERLAP: Final[int] = 100

{kreuzberg-2.1.1 → kreuzberg-3.0.0}/kreuzberg/_mime_types.py

@@ -16,7 +16,7 @@ PDF_MIME_TYPE: Final = "application/pdf"
 PLAIN_TEXT_MIME_TYPE: Final = "text/plain"
 POWER_POINT_MIME_TYPE: Final = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
 DOCX_MIME_TYPE: Final = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
-# Excel formats
+
 EXCEL_MIME_TYPE: Final = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
 EXCEL_BINARY_MIME_TYPE: Final = "application/vnd.ms-excel"
 EXCEL_MACRO_MIME_TYPE: Final = "application/vnd.ms-excel.sheet.macroEnabled.12"
@@ -24,8 +24,8 @@ EXCEL_BINARY_2007_MIME_TYPE: Final = "application/vnd.ms-excel.sheet.binary.macr
 EXCEL_ADDON_MIME_TYPE: Final = "application/vnd.ms-excel.addin.macroEnabled.12"
 EXCEL_TEMPLATE_MIME_TYPE: Final = "application/vnd.ms-excel.template.macroEnabled.12"
 
-# OpenDocument spreadsheet format
-OPENDOC_SPREADSHEET_MIME_TYPE: Final = "application/vnd.oasis.opendocument.spreadsheet"  # ods
+
+OPENDOC_SPREADSHEET_MIME_TYPE: Final = "application/vnd.oasis.opendocument.spreadsheet"
 PLAIN_TEXT_MIME_TYPES: Final[set[str]] = {PLAIN_TEXT_MIME_TYPE, MARKDOWN_MIME_TYPE}
 
 IMAGE_MIME_TYPES: Final[set[str]] = {
@@ -48,26 +48,7 @@ IMAGE_MIME_TYPES: Final[set[str]] = {
     "image/x-portable-pixmap",
     "image/x-tiff",
 }
-IMAGE_MIME_TYPE_EXT_MAP: Final[Mapping[str, str]] = {
-    "image/bmp": "bmp",
-    "image/x-bmp": "bmp",
-    "image/x-ms-bmp": "bmp",
-    "image/gif": "gif",
-    "image/jpeg": "jpg",
-    "image/pjpeg": "jpg",
-    "image/png": "png",
-    "image/tiff": "tiff",
-    "image/x-tiff": "tiff",
-    "image/jp2": "jp2",
-    "image/jpx": "jpx",
-    "image/jpm": "jpm",
-    "image/mj2": "mj2",
-    "image/webp": "webp",
-    "image/x-portable-anymap": "pnm",
-    "image/x-portable-bitmap": "pbm",
-    "image/x-portable-graymap": "pgm",
-    "image/x-portable-pixmap": "ppm",
-}
+
 PANDOC_SUPPORTED_MIME_TYPES: Final[set[str]] = {
     "application/csl+json",
     "application/docbook+xml",
@@ -162,13 +143,17 @@ SUPPORTED_MIME_TYPES: Final[set[str]] = (
 )
 
 
-def validate_mime_type(file_path: PathLike[str] | str, mime_type: str | None = None) -> str:
+def validate_mime_type(
+    *, file_path: PathLike[str] | str | None = None, mime_type: str | None = None, check_file_exists: bool = True
+) -> str:
     """Validate and detect the MIME type for a given file.
 
     Args:
         file_path: The path to the file.
         mime_type: Optional explicit MIME type. If provided, this will be validated.
             If not provided, the function will attempt to detect the MIME type.
+        check_file_exists: Whether to check if the file exists. Default is True.
+            Set to False in tests where you want to validate a mime type without an actual file.
 
     Raises:
         ValidationError: If the MIME type is not supported or cannot be determined.
@@ -176,10 +161,18 @@ def validate_mime_type(file_path: PathLike[str] | str, mime_type: str | None = N
     Returns:
         The validated MIME type.
     """
-    path = Path(file_path)
+    if file_path and check_file_exists:
+        path = Path(file_path)
+        if not path.exists():
+            raise ValidationError("The file does not exist", context={"file_path": str(path)})
 
     if not mime_type:
-        # Try to determine MIME type from file extension first
+        if not file_path:
+            raise ValidationError(
+                "Could not determine mime type.",
+            )
+        path = Path(file_path)
+
         ext = path.suffix.lower()
         mime_type = EXT_TO_MIME_TYPE.get(ext) or guess_type(path.name)[0]