kreuzberg 2.1.1__py3-none-any.whl → 3.0.0__py3-none-any.whl

kreuzberg/_string.py

@@ -1,41 +0,0 @@
-from __future__ import annotations
-
-from contextlib import suppress
-
-from charset_normalizer import detect
-
-
-def safe_decode(byte_data: bytes, encoding: str | None = None) -> str:
-    """Decode a byte string safely, removing invalid sequences.
-
-    Args:
-        byte_data: The byte string to decode.
-        encoding: The encoding to use when decoding the byte string.
-
-    Returns:
-        The decoded string.
-    """
-    if not byte_data:
-        return ""
-
-    # We try each encoding in order until one works
-    encodings = [encoding, detect(byte_data).get("encoding", ""), "utf-8"]
-
-    for enc in [e for e in encodings if e]:  # pragma: no cover
-        with suppress(UnicodeDecodeError, LookupError):
-            return byte_data.decode(enc)
-
-    # If all encodings fail, fall back to latin-1 which can handle any byte
-    return byte_data.decode("latin-1", errors="replace")
-
-
-def normalize_spaces(text: str) -> str:
-    """Normalize the spaces in a string.
-
-    Args:
-        text: The text to sanitize.
-
-    Returns:
-        The sanitized text.
-    """
-    return " ".join(text.strip().split())

kreuzberg/_sync.py

@@ -1,74 +0,0 @@
-from __future__ import annotations
-
-import sys
-from functools import partial
-from typing import TYPE_CHECKING, Any, TypeVar, cast
-
-from anyio import create_task_group
-from anyio.to_thread import run_sync as any_io_run_sync
-
-if TYPE_CHECKING:  # pragma: no cover
-    from collections.abc import Awaitable, Callable
-
-if sys.version_info >= (3, 10):
-    from typing import ParamSpec
-else:  # pragma: no cover
-    from typing_extensions import ParamSpec
-
-T = TypeVar("T")
-P = ParamSpec("P")
-
-
-async def run_sync(sync_fn: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T:
-    """Run a synchronous function in an asynchronous context.
-
-    Args:
-        sync_fn: The synchronous function to run.
-        *args: The positional arguments to pass to the function.
-        **kwargs: The keyword arguments to pass to the function.
-
-    Returns:
-        The result of the synchronous function.
-    """
-    handler = partial(sync_fn, **kwargs)
-    return cast(T, await any_io_run_sync(handler, *args, abandon_on_cancel=True))  # pyright: ignore [reportCallIssue]
-
-
-async def run_taskgroup(*async_tasks: Awaitable[Any]) -> list[Any]:
-    """Run a list of coroutines concurrently.
-
-    Args:
-        *async_tasks: The list of coroutines to run.
-
-    Returns:
-        The results of the coroutines.
-    """
-    results: list[Any] = [None] * len(async_tasks)
-
-    async def run_task(index: int, task: Awaitable[T]) -> None:
-        results[index] = await task
-
-    async with create_task_group() as tg:
-        for i, t in enumerate(async_tasks):
-            tg.start_soon(run_task, i, t)
-
-    return results
-
-
-async def run_taskgroup_batched(*async_tasks: Awaitable[Any], batch_size: int) -> list[Any]:
-    """Run a list of coroutines concurrently in batches.
-
-    Args:
-        *async_tasks: The list of coroutines to run.
-        batch_size: The size of each batch.
-
-    Returns:
-        The results of the coroutines.
-    """
-    results: list[Any] = []
-
-    for i in range(0, len(async_tasks), batch_size):
-        batch = async_tasks[i : i + batch_size]
-        results.extend(await run_taskgroup(*batch))
-
-    return results

kreuzberg/_tesseract.py

@@ -1,231 +0,0 @@
-from __future__ import annotations
-
-import re
-import sys
-from enum import Enum
-from os import PathLike
-from typing import Any, TypeVar, Union
-
-from anyio import Path as AsyncPath
-from anyio import run_process
-from PIL.Image import Image
-
-from kreuzberg._constants import DEFAULT_MAX_PROCESSES, MINIMAL_SUPPORTED_TESSERACT_VERSION
-from kreuzberg._mime_types import PLAIN_TEXT_MIME_TYPE
-from kreuzberg._string import normalize_spaces
-from kreuzberg._sync import run_sync, run_taskgroup_batched
-from kreuzberg._tmp import create_temp_file
-from kreuzberg._types import ExtractionResult
-from kreuzberg.exceptions import MissingDependencyError, OCRError, ParsingError
-
-if sys.version_info < (3, 11):  # pragma: no cover
-    from exceptiongroup import ExceptionGroup  # type: ignore[import-not-found]
-
-version_ref = {"checked": False}
-
-T = TypeVar("T", bound=Union[Image, PathLike[str], str])
-
-
-class PSMMode(Enum):
-    """Enum for Tesseract Page Segmentation Modes (PSM) with human-readable values."""
-
-    OSD_ONLY = 0
-    """Orientation and script detection only."""
-    AUTO_OSD = 1
-    """Automatic page segmentation with orientation and script detection."""
-    AUTO_ONLY = 2
-    """Automatic page segmentation without OSD."""
-    AUTO = 3
-    """Fully automatic page segmentation (default)."""
-    SINGLE_COLUMN = 4
-    """Assume a single column of text."""
-    SINGLE_BLOCK_VERTICAL = 5
-    """Assume a single uniform block of vertically aligned text."""
-    SINGLE_BLOCK = 6
-    """Assume a single uniform block of text."""
-    SINGLE_LINE = 7
-    """Treat the image as a single text line."""
-    SINGLE_WORD = 8
-    """Treat the image as a single word."""
-    CIRCLE_WORD = 9
-    """Treat the image as a single word in a circle."""
-    SINGLE_CHAR = 10
-    """Treat the image as a single character."""
-
-
-async def validate_tesseract_version() -> None:
-    """Validate that Tesseract is installed and is version 5 or above.
-
-    Raises:
-        MissingDependencyError: If Tesseract is not installed or is below version 5.
-    """
-    try:
-        if version_ref["checked"]:
-            return
-
-        command = ["tesseract", "--version"]
-        result = await run_process(command)
-        version_match = re.search(r"tesseract\s+v?(\d+)\.\d+\.\d+", result.stdout.decode())
-        if not version_match or int(version_match.group(1)) < MINIMAL_SUPPORTED_TESSERACT_VERSION:
-            raise MissingDependencyError("Tesseract version 5 or above is required.")
-
-        version_ref["checked"] = True
-    except FileNotFoundError as e:
-        raise MissingDependencyError(
-            "Tesseract is not installed or not in path. Please install tesseract 5 and above on your system."
-        ) from e
-
-
-async def process_file(
-    input_file: str | PathLike[str],
-    *,
-    language: str,
-    psm: PSMMode,
-) -> ExtractionResult:
-    """Process a single image file using Tesseract OCR.
-
-    Args:
-        input_file: The path to the image file to process.
-        language: The language code for OCR.
-        psm: Page segmentation mode.
-
-    Raises:
-        OCRError: If OCR fails to extract text from the image.
-
-    Returns:
-        ExtractionResult: The extracted text from the image.
-    """
-    output_path, unlink = await create_temp_file(".txt")
-    try:
-        output_base = str(output_path).replace(".txt", "")
-
-        command = [
-            "tesseract",
-            str(input_file),
-            output_base,
-            "-l",
-            language,
-            "--psm",
-            str(psm.value),
-            "--oem",
-            "1",
-            "--loglevel",
-            "OFF",
-            "-c",
-            "thresholding_method=1",
-            "-c",
-            "tessedit_enable_dict_correction=1",
-            "-c",
-            "language_model_ngram_on=1",
-            "-c",
-            "textord_space_size_is_variable=1",
-            "-c",
-            "classify_use_pre_adapted_templates=1",
-            "-c",
-            "tessedit_dont_blkrej_good_wds=1",
-            "-c",
-            "tessedit_dont_rowrej_good_wds=1",
-            "-c",
-            "tessedit_use_primary_params_model=1",
-        ]
-
-        env: dict[str, Any] | None = None
-        if sys.platform.startswith("linux"):
-            env = {"OMP_THREAD_LIMIT": "1"}
-
-        result = await run_process(command, env=env)
-
-        if not result.returncode == 0:
-            raise OCRError(
-                "OCR failed with a non-0 return code.",
-                context={"error": result.stderr.decode() if isinstance(result.stderr, bytes) else result.stderr},
-            )
-
-        output = await AsyncPath(output_path).read_text("utf-8")
-        return ExtractionResult(content=normalize_spaces(output), mime_type=PLAIN_TEXT_MIME_TYPE, metadata={})
-    except (RuntimeError, OSError) as e:
-        raise OCRError(f"Failed to OCR using tesseract: {e}") from e
-    finally:
-        await unlink()
-
-
-async def process_image(
-    image: Image,
-    *,
-    language: str,
-    psm: PSMMode,
-) -> ExtractionResult:
-    """Process a single Pillow Image using Tesseract OCR.
-
-    Args:
-        image: The Pillow Image to process.
-        language: The language code for OCR.
-        psm: Page segmentation mode.
-
-    Returns:
-        ExtractionResult: The extracted text from the image.
-    """
-    image_path, unlink = await create_temp_file(".png")
-    await run_sync(image.save, str(image_path), format="PNG")
-    result = await process_file(image_path, language=language, psm=psm)
-    await unlink()
-    return result
-
-
-async def process_image_with_tesseract(
-    image: Image | PathLike[str] | str,
-    *,
-    language: str = "eng",
-    psm: PSMMode = PSMMode.AUTO,
-) -> ExtractionResult:
-    """Run Tesseract OCR asynchronously on a single Pillow Image or a list of Pillow Images.
-
-    Args:
-        image: A single Pillow Image, a pathlike or a string or a list of Pillow Images to process.
-        language: The language code for OCR (default: "eng").
-        psm: Page segmentation mode (default: PSMMode.AUTO).
-
-    Raises:
-        ValueError: If the input is not a Pillow Image or a list of Pillow Images.
-
-    Returns:
-        Extracted text as a string
-    """
-    await validate_tesseract_version()
-
-    if isinstance(image, Image):
-        return await process_image(image, language=language, psm=psm)
-
-    if isinstance(image, (PathLike, str)):
-        return await process_file(image, language=language, psm=psm)
-
-    raise ValueError("Input must be one of: str, Pathlike or Pillow Image.")
-
-
-async def batch_process_images(
-    images: list[T],
-    *,
-    language: str = "eng",
-    psm: PSMMode = PSMMode.AUTO,
-    max_processes: int = DEFAULT_MAX_PROCESSES,
-) -> list[ExtractionResult]:
-    """Run Tesseract OCR asynchronously on multiple images with controlled concurrency.
-
-    Args:
-        images: A list of Pillow Images, paths or strings to process.
-        language: The language code for OCR (default: "eng").
-        psm: Page segmentation mode (default: PSMMode.AUTO).
-        max_processes: Maximum number of concurrent processes (default: CPU count / 2).
-
-    Raises:
-        ParsingError: If OCR fails to extract text from any of the images.
-
-    Returns:
-        List of ExtractionResult objects, one per input image.
-    """
-    await validate_tesseract_version()
-    try:
-        tasks = [process_image_with_tesseract(image, language=language, psm=psm) for image in images]
-        return await run_taskgroup_batched(*tasks, batch_size=max_processes)
-    except ExceptionGroup as eg:
-        raise ParsingError("Failed to process images with Tesseract", context={"errors": eg.exceptions}) from eg

kreuzberg/_tmp.py

@@ -1,37 +0,0 @@
-from __future__ import annotations
-
-from contextlib import suppress
-from pathlib import Path
-from tempfile import NamedTemporaryFile
-from typing import TYPE_CHECKING, Callable
-
-from anyio import Path as AsyncPath
-
-from kreuzberg._sync import run_sync
-
-if TYPE_CHECKING:  # pragma: no cover
-    from collections.abc import Coroutine
-
-
-async def create_temp_file(
-    extension: str, content: bytes | None = None
-) -> tuple[Path, Callable[[], Coroutine[None, None, None]]]:
-    """Create a temporary file that is closed.
-
-    Args:
-        extension: The file extension.
-        content: The content to write to the file.
-
-    Returns:
-        The temporary file path.
-    """
-    file = await run_sync(NamedTemporaryFile, suffix=extension, delete=False)
-    if content:
-        await AsyncPath(file.name).write_bytes(content)
-    await run_sync(file.close)
-
-    async def unlink() -> None:
-        with suppress(OSError, PermissionError):
-            await AsyncPath(file.name).unlink(missing_ok=True)
-
-    return Path(file.name), unlink

kreuzberg/_xlsx.py

@@ -1,88 +0,0 @@
-from __future__ import annotations
-
-import csv
-import sys
-from io import StringIO
-from typing import TYPE_CHECKING
-
-from anyio import Path as AsyncPath
-from python_calamine import CalamineWorkbook
-
-from kreuzberg import ExtractionResult, ParsingError
-from kreuzberg._mime_types import MARKDOWN_MIME_TYPE
-from kreuzberg._pandoc import process_file_with_pandoc
-from kreuzberg._string import normalize_spaces
-from kreuzberg._sync import run_sync, run_taskgroup
-from kreuzberg._tmp import create_temp_file
-
-if TYPE_CHECKING:  # pragma: no cover
-    from pathlib import Path
-
-if sys.version_info < (3, 11):  # pragma: no cover
-    from exceptiongroup import ExceptionGroup  # type: ignore[import-not-found]
-
-
-async def convert_sheet_to_text(workbook: CalamineWorkbook, sheet_name: str) -> str:
-    values = workbook.get_sheet_by_name(sheet_name).to_python()
-
-    csv_buffer = StringIO()
-    writer = csv.writer(csv_buffer)
-
-    for row in values:
-        writer.writerow(row)
-
-    csv_data = csv_buffer.getvalue()
-    csv_buffer.close()
-
-    csv_path, unlink = await create_temp_file(".csv")
-    await AsyncPath(csv_path).write_text(csv_data)
-
-    result = await process_file_with_pandoc(csv_path, mime_type="text/csv")
-    await unlink()
-    return f"## {sheet_name}\n\n{normalize_spaces(result.content)}"
-
-
-async def extract_xlsx_file(input_file: Path) -> ExtractionResult:
-    """Extract text from an XLSX file by converting it to CSV and then to markdown.
-
-    Args:
-        input_file: The path to the XLSX file.
-
-    Returns:
-        The extracted text content.
-
-    Raises:
-        ParsingError: If the XLSX file could not be parsed.
-    """
-    try:
-        workbook: CalamineWorkbook = await run_sync(CalamineWorkbook.from_path, str(input_file))
-        tasks = [convert_sheet_to_text(workbook, sheet_name) for sheet_name in workbook.sheet_names]
-        results: list[str] = await run_taskgroup(*tasks)
-
-        return ExtractionResult(
-            content="\n\n".join(results),
-            mime_type=MARKDOWN_MIME_TYPE,
-            metadata={},
-        )
-    except ExceptionGroup as eg:
-        raise ParsingError(
-            "Failed to extract file data",
-            context={"file": str(input_file), "errors": eg.exceptions},
-        ) from eg
-
-
-async def extract_xlsx_content(content: bytes) -> ExtractionResult:
-    """Extract text from an XLSX file content.
-
-    Args:
-        content: The XLSX file content.
-
-    Returns:
-        The extracted text content.
-    """
-    xlsx_path, unlink = await create_temp_file(".xlsx")
-
-    await AsyncPath(xlsx_path).write_bytes(content)
-    result = await extract_xlsx_file(xlsx_path)
-    await unlink()
-    return result