kreuzberg 3.0.0__py3-none-any.whl → 3.1.0__py3-none-any.whl

kreuzberg/_utils/_string.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from contextlib import suppress
+
+from charset_normalizer import detect
+
+
+def safe_decode(byte_data: bytes, encoding: str | None = None) -> str:
+    """Decode a byte string safely, removing invalid sequences.
+
+    Args:
+        byte_data: The byte string to decode.
+        encoding: The encoding to use when decoding the byte string.
+
+    Returns:
+        The decoded string.
+    """
+    if not byte_data:
+        return ""
+
+    encodings = [encoding, detect(byte_data).get("encoding", ""), "utf-8"]
+
+    for enc in [e for e in encodings if e]:  # pragma: no cover
+        with suppress(UnicodeDecodeError, LookupError):
+            return byte_data.decode(enc)
+
+    return byte_data.decode("latin-1", errors="replace")
+
+
+def normalize_spaces(text: str) -> str:
+    """Normalize the spaces in a string.
+
+    Args:
+        text: The text to sanitize.
+
+    Returns:
+        The sanitized text.
+    """
+    return " ".join(text.strip().split())

kreuzberg/_utils/_sync.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+import sys
+from functools import partial
+from inspect import isawaitable, iscoroutinefunction
+from typing import TYPE_CHECKING, Any, TypeVar, cast
+
+import anyio
+from anyio import create_task_group
+from anyio.to_thread import run_sync as any_io_run_sync
+
+if TYPE_CHECKING:  # pragma: no cover
+    from collections.abc import Awaitable, Callable
+
+if sys.version_info >= (3, 10):
+    from typing import ParamSpec
+else:  # pragma: no cover
+    from typing_extensions import ParamSpec
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+
+async def run_sync(sync_fn: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T:
+    """Run a synchronous function in an asynchronous context.
+
+    Args:
+        sync_fn: The synchronous function to run.
+        *args: The positional arguments to pass to the function.
+        **kwargs: The keyword arguments to pass to the function.
+
+    Returns:
+        The result of the synchronous function.
+    """
+    handler = partial(sync_fn, **kwargs)
+    return cast("T", await any_io_run_sync(handler, *args, abandon_on_cancel=True))  # pyright: ignore [reportCallIssue]
+
+
+async def run_taskgroup(*async_tasks: Awaitable[Any]) -> list[Any]:
+    """Run a list of coroutines concurrently.
+
+    Args:
+        *async_tasks: The list of coroutines to run.
+
+    Returns:
+        The results of the coroutines.
+    """
+    results: list[Any] = [None] * len(async_tasks)
+
+    async def run_task(index: int, task: Awaitable[T]) -> None:
+        results[index] = await task
+
+    async with create_task_group() as tg:
+        for i, t in enumerate(async_tasks):
+            tg.start_soon(run_task, i, t)
+
+    return results
+
+
+async def run_taskgroup_batched(*async_tasks: Awaitable[Any], batch_size: int) -> list[Any]:
+    """Run a list of coroutines concurrently in batches.
+
+    Args:
+        *async_tasks: The list of coroutines to run.
+        batch_size: The size of each batch.
+
+    Returns:
+        The results of the coroutines.
+    """
+    results: list[Any] = []
+
+    for i in range(0, len(async_tasks), batch_size):
+        batch = async_tasks[i : i + batch_size]
+        results.extend(await run_taskgroup(*batch))
+
+    return results
+
+
+async def run_maybe_sync(fn: Callable[P, T | Awaitable[T]], *args: P.args, **kwargs: P.kwargs) -> T:
+    """Executes a callable function and handles both synchronous and asynchronous
+    results.
+
+    This function invokes the provided callable `sync_fn` with the given
+    arguments and keyword arguments. If the result of `sync_fn` is awaitable,
+    it awaits the result before returning it. Otherwise, the result is returned
+    directly.
+
+    Args:
+        fn: The callable to be executed. It can produce either a
+            synchronous or asynchronous result.
+        *args: Positional arguments to pass to `sync_fn`.
+        **kwargs: Keyword arguments to pass to `sync_fn`.
+
+    Returns:
+        The result of `sync_fn` invocation. If the result is awaitable, the
+        awaited value is returned. Otherwise, the synchronous result is
+        returned.
+    """
+    result = fn(*args, **kwargs)
+    if isawaitable(result):
+        return cast("T", await result)
+    return result
+
+
+def run_maybe_async(fn: Callable[P, T | Awaitable[T]], *args: P.args, **kwargs: P.kwargs) -> T:
+    """Runs a synchronous or asynchronous function, resolving the output.
+
+    Determines if the provided function is synchronous or asynchronous. If synchronous,
+    executes it directly. If asynchronous, it runs the function within the event loop
+    using anyio. The return value is resolved regardless of the function type.
+
+    Args:
+        fn: The function to be executed, which can
+            either be synchronous or asynchronous.
+        *args: Positional arguments to be passed to the function.
+        **kwargs: Keyword arguments to be passed to the function.
+
+    Returns:
+        T: The return value of the executed function, resolved if asynchronous.
+    """
+    return cast("T", fn(*args, **kwargs) if not iscoroutinefunction(fn) else anyio.run(partial(fn, **kwargs), *args))

kreuzberg/_utils/_tmp.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from contextlib import suppress
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import TYPE_CHECKING, Callable
+
+from anyio import Path as AsyncPath
+
+from kreuzberg._utils._sync import run_sync
+
+if TYPE_CHECKING:  # pragma: no cover
+    from collections.abc import Coroutine
+
+
+async def create_temp_file(
+    extension: str, content: bytes | None = None
+) -> tuple[Path, Callable[[], Coroutine[None, None, None]]]:
+    """Create a temporary file that is closed.
+
+    Args:
+        extension: The file extension.
+        content: The content to write to the file.
+
+    Returns:
+        The temporary file path.
+    """
+    file = await run_sync(NamedTemporaryFile, suffix=extension, delete=False)
+    if content:
+        await AsyncPath(file.name).write_bytes(content)
+    await run_sync(file.close)
+
+    async def unlink() -> None:
+        with suppress(OSError, PermissionError):
+            await AsyncPath(file.name).unlink(missing_ok=True)
+
+    return Path(file.name), unlink

{kreuzberg-3.0.0.dist-info → kreuzberg-3.1.0.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.0.0
+Version: 3.1.0
 Summary: A text extraction library supporting PDFs, images, office documents and more
 Author-email: Na'aman Hirschfeld <nhirschfed@gmail.com>
 License: MIT
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
-Keywords: document-processing,image-to-text,ocr,pandoc,pdf-extraction,rag,tesseract,text-extraction,text-processing
+Keywords: document-processing,image-to-text,ocr,pandoc,pdf-extraction,rag,table-extraction,tesseract,text-extraction,text-processing
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -27,7 +27,7 @@ License-File: LICENSE
 Requires-Dist: anyio>=4.9.0
 Requires-Dist: charset-normalizer>=3.4.1
 Requires-Dist: exceptiongroup>=1.2.2; python_version < "3.11"
-Requires-Dist: html-to-markdown>=1.2.0
+Requires-Dist: html-to-markdown>=1.2.1
 Requires-Dist: playa-pdf>=0.4.1
 Requires-Dist: pypdfium2==4.30.0
 Requires-Dist: python-calamine>=0.3.1
@@ -35,19 +35,20 @@ Requires-Dist: python-pptx>=1.0.2
 Requires-Dist: typing-extensions>=4.12.2; python_version < "3.12"
 Provides-Extra: all
 Requires-Dist: easyocr>=1.7.2; extra == "all"
-Requires-Dist: numpy>=2.0.2; extra == "all"
+Requires-Dist: gmft>=0.4.1; extra == "all"
 Requires-Dist: paddleocr>=2.10.0; extra == "all"
-Requires-Dist: paddlepaddle>=2.6.2; python_version < "3.13" and extra == "all"
-Requires-Dist: semantic-text-splitter>=0.24.1; extra == "all"
+Requires-Dist: paddlepaddle>=3.0.0; extra == "all"
+Requires-Dist: semantic-text-splitter>=0.25.1; extra == "all"
 Requires-Dist: setuptools>=76.0.0; extra == "all"
 Provides-Extra: chunking
-Requires-Dist: semantic-text-splitter>=0.24.1; extra == "chunking"
+Requires-Dist: semantic-text-splitter>=0.25.1; extra == "chunking"
 Provides-Extra: easyocr
 Requires-Dist: easyocr>=1.7.2; extra == "easyocr"
+Provides-Extra: gmft
+Requires-Dist: gmft>=0.4.1; extra == "gmft"
 Provides-Extra: paddleocr
-Requires-Dist: numpy>=2.0.2; extra == "paddleocr"
 Requires-Dist: paddleocr>=2.10.0; extra == "paddleocr"
-Requires-Dist: paddlepaddle>=2.6.2; python_version < "3.13" and extra == "paddleocr"
+Requires-Dist: paddlepaddle>=3.0.0; extra == "paddleocr"
 Requires-Dist: setuptools>=76.0.0; extra == "paddleocr"
 Dynamic: license-file
 
@@ -66,6 +67,8 @@ Kreuzberg is a Python library for text extraction from documents. It provides a
 - **Resource Efficient**: Lightweight processing without GPU requirements
 - **Format Support**: Comprehensive support for documents, images, and text formats
 - **Multiple OCR Engines**: Support for Tesseract, EasyOCR, and PaddleOCR
+- **Metadata Extraction**: Get document metadata alongside text content
+- **Table Extraction**: Extract tables from documents using the excellent GMFT library
 - **Modern Python**: Built with async/await, type hints, and a functional-first approach
 - **Permissive OSS**: MIT licensed with permissively licensed dependencies
 
@@ -151,7 +154,7 @@ Kreuzberg supports multiple OCR engines:
 - **EasyOCR**: Good for many languages, pure Python, but downloads models on first use
 - **PaddleOCR**: Excellent for Asian languages, pure Python, but downloads models on first use
 
-For comparison and selection guidance, see the [OCR Backends](https://example.com/ocr-backends) documentation.
+For comparison and selection guidance, see the [OCR Backends](https://goldziher.github.io/kreuzberg/user-guide/ocr-backends/) documentation.
 
 ## Contribution
 
@@ -160,17 +163,9 @@ This library is open to contribution. Feel free to open issues or submit PRs. It
 ### Local Development
 
 1. Clone the repo
-
 1. Install the system dependencies
-
 1. Install the full dependencies with `uv sync`
-
-1. Install the pre-commit hooks with:
-
-    ```shell
-    pre-commit install && pre-commit install --hook-type commit-msg
-    ```
-
+1. Install the pre-commit hooks with: `pre-commit install && pre-commit install --hook-type commit-msg`
 1. Make your changes and submit a PR
 
 ## License

kreuzberg-3.1.0.dist-info/RECORD

@@ -0,0 +1,33 @@
+kreuzberg/__init__.py,sha256=lT9OwIdf5CEhSX7IVmtSFPgRhz6B2z2A-RE8Zdm0PH4,1216
+kreuzberg/_chunker.py,sha256=2eHSRHcZdJ2ZjR3in49y3o9tPl5HMO3vkbnMqaVCbHI,1887
+kreuzberg/_constants.py,sha256=Bxc8oiN-wHwnWXT9bEiJhTUcu1ygPpra5qHirAif3b4,191
+kreuzberg/_gmft.py,sha256=qLhfepQuaROjPOdI-tDRqqqnOcqDY1D411ZXzoywnpg,7229
+kreuzberg/_mime_types.py,sha256=pKtxBPDoye2knyou_VODDMPIt3eXotP-ak4MAKFI2SU,6310
+kreuzberg/_playa.py,sha256=agHdhKfKLNtiP37XdNncbCP65v3Qv3m1Gn2KTRUkVx8,10396
+kreuzberg/_registry.py,sha256=c2B_PJbaL0q3ab2eNmj_0jldeyMaqgvRwkZqUU4MM5Q,3290
+kreuzberg/_types.py,sha256=G7UQ5ZUWcpgwHoasexW7f2te3gKe3PHHi_3Fm1cju-w,7503
+kreuzberg/exceptions.py,sha256=xRaiJh11i8E6Nc-gAQPgNW5xvhiiFBhRS-CBbCEbHQM,2881
+kreuzberg/extraction.py,sha256=0sjvbunx5srbR5lzjOAQjGK5JY3bCUHw-dRFmHjFz7o,8671
+kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kreuzberg/_extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kreuzberg/_extractors/_base.py,sha256=YUr6A2n34LlFzbYQkiKqhXAphL9RYrvAls5SlkoQqNg,3028
+kreuzberg/_extractors/_html.py,sha256=L_vcKyv1ObI6irPaD3-WTKqxeRfZA4Rhsl3zUiAe_ws,1312
+kreuzberg/_extractors/_image.py,sha256=VQgSFSzXIMX3A52-DyvuKgfTRXUJIjYn6IX4-sQWWdg,2626
+kreuzberg/_extractors/_pandoc.py,sha256=a6cYQxoh5G9EMrDWVcQhrTkE4Mar24sNiGCY0zOOzw4,20121
+kreuzberg/_extractors/_pdf.py,sha256=eNFws_UxLgWSTC_VC_zJmVojpyQvioOXgNjSHQzBq5c,6607
+kreuzberg/_extractors/_presentation.py,sha256=K4ALrpmZ0EWyp2O-3oEmTRCS7yAET9xjinrzo13rpWo,8764
+kreuzberg/_extractors/_spread_sheet.py,sha256=1ejRZk8AE1dXS1tRIdg2S0J9Vo0wG81iKkW2IF6PjlE,4445
+kreuzberg/_ocr/__init__.py,sha256=VTqwKDlIRbjve71Y11Ztygyhv5aWG9LWTj8iX66ANxE,533
+kreuzberg/_ocr/_base.py,sha256=lNT0Tin4hzbmaamqqySxvYEwNtrJB5gGlStrANQQcyc,1637
+kreuzberg/_ocr/_easyocr.py,sha256=VfYW66SkB2Bigbrtd7WEeJ6QZ_1Y5d8Z_rZYBPMsuk0,11037
+kreuzberg/_ocr/_paddleocr.py,sha256=NDKXiMtHjIy-Uq4hXe4qm5oUWwOrhjJaibyC708Cw5E,10422
+kreuzberg/_ocr/_tesseract.py,sha256=cdnVxNpaKjxtBN4xy0Timz-uYtPA9wq9kc6kyYVeDug,9779
+kreuzberg/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kreuzberg/_utils/_string.py,sha256=oNO0cmwjVNG0jAzaqNCjYtzvM_nxH5TW2KV-Uh3oEUU,978
+kreuzberg/_utils/_sync.py,sha256=lycobEMXk0tBMWLwkuMdOuNMStDwPKMC0V1Qgp_oi6k,4071
+kreuzberg/_utils/_tmp.py,sha256=5rqG_Nlb9xweaLqJA8Kc5csHDase9_eY_Fq93rNQGWc,1044
+kreuzberg-3.1.0.dist-info/licenses/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
+kreuzberg-3.1.0.dist-info/METADATA,sha256=YemIJR6aygDxNgz9aoeg2oIGRHJjm897jD8sHuJYdMY,6651
+kreuzberg-3.1.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+kreuzberg-3.1.0.dist-info/top_level.txt,sha256=rbGkygffkZiyKhL8UN41ZOjLfem0jJPA1Whtndne0rE,10
+kreuzberg-3.1.0.dist-info/RECORD,,

{kreuzberg-3.0.0.dist-info → kreuzberg-3.1.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (77.0.3)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

kreuzberg-3.0.0.dist-info/RECORD

@@ -1,15 +0,0 @@
-kreuzberg/__init__.py,sha256=KZ_y21m64cafWL7goGeG3EIDutM184st28n4UGajADs,1131
-kreuzberg/_chunker.py,sha256=2eHSRHcZdJ2ZjR3in49y3o9tPl5HMO3vkbnMqaVCbHI,1887
-kreuzberg/_constants.py,sha256=Bxc8oiN-wHwnWXT9bEiJhTUcu1ygPpra5qHirAif3b4,191
-kreuzberg/_mime_types.py,sha256=pKtxBPDoye2knyou_VODDMPIt3eXotP-ak4MAKFI2SU,6310
-kreuzberg/_playa.py,sha256=agHdhKfKLNtiP37XdNncbCP65v3Qv3m1Gn2KTRUkVx8,10396
-kreuzberg/_registry.py,sha256=c2B_PJbaL0q3ab2eNmj_0jldeyMaqgvRwkZqUU4MM5Q,3290
-kreuzberg/_types.py,sha256=sZMxjRZQ1c_MzxdumhYSWghW6yXBwohTUIBa5eR-FKA,6582
-kreuzberg/exceptions.py,sha256=xRaiJh11i8E6Nc-gAQPgNW5xvhiiFBhRS-CBbCEbHQM,2881
-kreuzberg/extraction.py,sha256=0sjvbunx5srbR5lzjOAQjGK5JY3bCUHw-dRFmHjFz7o,8671
-kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kreuzberg-3.0.0.dist-info/licenses/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
-kreuzberg-3.0.0.dist-info/METADATA,sha256=wlO9VCvZQy_gJJTmhGzH9j8BlPQPFQdmMZQxJOcQAUg,6515
-kreuzberg-3.0.0.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-kreuzberg-3.0.0.dist-info/top_level.txt,sha256=rbGkygffkZiyKhL8UN41ZOjLfem0jJPA1Whtndne0rE,10
-kreuzberg-3.0.0.dist-info/RECORD,,