kreuzberg 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

kreuzberg/_extractors.py

@@ -6,50 +6,61 @@ from html import escape
 from io import BytesIO
 from typing import TYPE_CHECKING, cast
 
+import html_to_markdown
+import pptx
+import pypandoc
+import pypdfium2
 from anyio import Path as AsyncPath
 from charset_normalizer import detect
-from pptx import Presentation
-from pptx.enum.shapes import MSO_SHAPE_TYPE
-from pypandoc import convert_file, convert_text
-from pypdfium2 import PdfDocument, PdfiumError
-from pytesseract import TesseractError, image_to_string
 
 from kreuzberg._mime_types import PANDOC_MIME_TYPE_EXT_MAP
-from kreuzberg._string import normalize_spaces
+from kreuzberg._string import normalize_spaces, safe_decode
 from kreuzberg._sync import run_sync
+from kreuzberg._tesseract import batch_process_images
 from kreuzberg.exceptions import ParsingError
 
 if TYPE_CHECKING:  # pragma: no cover
     from pathlib import Path
 
+    from PIL.Image import Image
 
-def _extract_pdf_with_tesseract(file_path: Path) -> str:
-    """Extract text from a scanned PDF file using pytesseract.
+
+async def convert_pdf_to_images(file_path: Path) -> list[Image]:
+    """Convert a PDF file to images.
 
     Args:
         file_path: The path to the PDF file.
 
     Raises:
-        ParsingError: If the text could not be extracted from the PDF file.
+        ParsingError: If the PDF file could not be converted to images.
 
     Returns:
-        The extracted text.
+        A list of Pillow Images.
     """
     try:
-        # make it into an image here:
-        pdf = PdfDocument(str(file_path))
-        images = [page.render(scale=2.0).to_pil() for page in pdf]
-
-        text = "\n".join(image_to_string(img) for img in images)
-        return normalize_spaces(text)
-    except (PdfiumError, TesseractError) as e:
-        # TODO: add test case
+        pdf = await run_sync(pypdfium2.PdfDocument, str(file_path))
+        return [page.render(scale=2.0).to_pil() for page in pdf]
+    except pypdfium2.PdfiumError as e:
         raise ParsingError(
-            "Could not extract text from PDF file", context={"file_path": str(file_path), "error": str(e)}
+            "Could not convert PDF to images", context={"file_path": str(file_path), "error": str(e)}
         ) from e
 
 
-def _extract_pdf_with_pdfium2(file_path: Path) -> str:
+async def extract_pdf_with_tesseract(file_path: Path) -> str:
+    """Extract text from a scanned PDF file using pytesseract.
+
+    Args:
+        file_path: The path to the PDF file.
+
+    Returns:
+        The extracted text.
+    """
+    images = await convert_pdf_to_images(file_path)
+    ocr_results = await batch_process_images(images)
+    return normalize_spaces("\n".join(ocr_results))
+
+
+async def extract_pdf_with_pdfium2(file_path: Path) -> str:
     """Extract text from a searchable PDF file using pypdfium2.
 
     Args:
@@ -62,17 +73,16 @@ def _extract_pdf_with_pdfium2(file_path: Path) -> str:
         The extracted text.
     """
     try:
-        document = PdfDocument(file_path)
+        document = await run_sync(pypdfium2.PdfDocument, file_path)
         text = "\n".join(page.get_textpage().get_text_range() for page in document)
         return normalize_spaces(text)
-    except PdfiumError as e:
-        # TODO: add test case
+    except pypdfium2.PdfiumError as e:
         raise ParsingError(
             "Could not extract text from PDF file", context={"file_path": str(file_path), "error": str(e)}
         ) from e
 
 
-async def _extract_pdf_file(file_path: Path, force_ocr: bool = False) -> str:
+async def extract_pdf_file(file_path: Path, force_ocr: bool = False) -> str:
     """Extract text from a PDF file.
 
     Args:
@@ -82,13 +92,13 @@ async def _extract_pdf_file(file_path: Path, force_ocr: bool = False) -> str:
     Returns:
         The extracted text.
     """
-    if not force_ocr and (content := await run_sync(_extract_pdf_with_pdfium2, file_path)):
+    if not force_ocr and (content := await extract_pdf_with_pdfium2(file_path)):
         return normalize_spaces(content)
 
-    return normalize_spaces(await run_sync(_extract_pdf_with_tesseract, file_path))
+    return await extract_pdf_with_tesseract(file_path)
 
 
-async def _extract_content_with_pandoc(file_data: bytes, mime_type: str, encoding: str | None = None) -> str:
+async def extract_content_with_pandoc(file_data: bytes, mime_type: str, encoding: str | None = None) -> str:
     """Extract text using pandoc.
 
     Args:
@@ -106,7 +116,7 @@ async def _extract_content_with_pandoc(file_data: bytes, mime_type: str, encodin
     encoding = encoding or detect(file_data)["encoding"] or "utf-8"
     try:
         return normalize_spaces(
-            cast(str, await run_sync(convert_text, file_data, to="md", format=ext, encoding=encoding))
+            cast(str, await run_sync(pypandoc.convert_text, file_data, to="md", format=ext, encoding=encoding))
         )
     except RuntimeError as e:
         # TODO: add test case
@@ -116,7 +126,7 @@ async def _extract_content_with_pandoc(file_data: bytes, mime_type: str, encodin
         ) from e
 
 
-async def _extract_file_with_pandoc(file_path: Path | str, mime_type: str) -> str:
+async def extract_file_with_pandoc(file_path: Path | str, mime_type: str) -> str:
     """Extract text using pandoc.
 
     Args:
@@ -131,7 +141,7 @@ async def _extract_file_with_pandoc(file_path: Path | str, mime_type: str) -> st
     """
     ext = PANDOC_MIME_TYPE_EXT_MAP[mime_type]
     try:
-        return normalize_spaces(cast(str, await run_sync(convert_file, file_path, to="md", format=ext)))
+        return normalize_spaces(cast(str, await run_sync(pypandoc.convert_file, file_path, to="md", format=ext)))
     except RuntimeError as e:
         raise ParsingError(
             f"Could not extract text from {PANDOC_MIME_TYPE_EXT_MAP[mime_type]} file",
@@ -139,27 +149,7 @@ async def _extract_file_with_pandoc(file_path: Path | str, mime_type: str) -> st
         ) from e
 
 
-async def _extract_image_with_tesseract(file_path: Path | str) -> str:
-    """Extract text from an image file.
-
-    Args:
-        file_path: The path to the image file.
-
-    Raises:
-        ParsingError: If the text could not be extracted from the image file.
-
-    Returns:
-        The extracted content.
-    """
-    try:
-        return normalize_spaces(cast(str, image_to_string(str(file_path))))
-    except TesseractError as e:
-        raise ParsingError(
-            "Could not extract text from image file", context={"file_path": str(file_path), "error": str(e)}
-        ) from e
-
-
-async def _extract_pptx_file(file_path_or_contents: Path | bytes) -> str:
+async def extract_pptx_file(file_path_or_contents: Path | bytes) -> str:
     """Extract text from a PPTX file.
 
     Notes:
@@ -171,13 +161,15 @@ async def _extract_pptx_file(file_path_or_contents: Path | bytes) -> str:
     Returns:
         The extracted text content
     """
+    from pptx.enum.shapes import MSO_SHAPE_TYPE
+
     md_content = ""
     file_contents = (
         file_path_or_contents
         if isinstance(file_path_or_contents, bytes)
         else await AsyncPath(file_path_or_contents).read_bytes()
     )
-    presentation = Presentation(BytesIO(file_contents))
+    presentation = pptx.Presentation(BytesIO(file_contents))
 
     for index, slide in enumerate(presentation.slides):
         md_content += f"\n\n<!-- Slide number: {index + 1} -->\n"
@@ -227,3 +219,20 @@ async def _extract_pptx_file(file_path_or_contents: Path | bytes) -> str:
             md_content = md_content.strip()
 
     return normalize_spaces(md_content)
+
+
+async def extract_html_string(file_path_or_contents: Path | bytes) -> str:
+    """Extract text from an HTML string.
+
+    Args:
+        file_path_or_contents: The HTML content.
+
+    Returns:
+        The extracted text content.
+    """
+    content = (
+        safe_decode(file_path_or_contents)
+        if isinstance(file_path_or_contents, bytes)
+        else await AsyncPath(file_path_or_contents).read_text()
+    )
+    return normalize_spaces(await run_sync(html_to_markdown.convert_to_markdown, content))

kreuzberg/_mime_types.py

@@ -5,10 +5,12 @@ from typing import TYPE_CHECKING, Final
 if TYPE_CHECKING:  # pragma: no cover
     from collections.abc import Mapping
 
-MARKDOWN_MIME_TYPE: Final[str] = "text/markdown"
-PLAIN_TEXT_MIME_TYPE: Final[str] = "text/plain"
-PDF_MIME_TYPE: Final[str] = "application/pdf"
-POWER_POINT_MIME_TYPE: Final[str] = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+HTML_MIME_TYPE: Final = "text/html"
+MARKDOWN_MIME_TYPE: Final = "text/markdown"
+PDF_MIME_TYPE: Final = "application/pdf"
+PLAIN_TEXT_MIME_TYPE: Final = "text/plain"
+POWER_POINT_MIME_TYPE: Final = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+
 PLAIN_TEXT_MIME_TYPES: Final[set[str]] = {PLAIN_TEXT_MIME_TYPE, MARKDOWN_MIME_TYPE}
 
 IMAGE_MIME_TYPES: Final[set[str]] = {
@@ -93,5 +95,8 @@ PANDOC_MIME_TYPE_EXT_MAP: Final[Mapping[str, str]] = {
 }
 
 SUPPORTED_MIME_TYPES: Final[set[str]] = (
-    PLAIN_TEXT_MIME_TYPES | IMAGE_MIME_TYPES | PANDOC_SUPPORTED_MIME_TYPES | {PDF_MIME_TYPE, POWER_POINT_MIME_TYPE}
+    PLAIN_TEXT_MIME_TYPES
+    | IMAGE_MIME_TYPES
+    | PANDOC_SUPPORTED_MIME_TYPES
+    | {PDF_MIME_TYPE, POWER_POINT_MIME_TYPE, HTML_MIME_TYPE}
 )

kreuzberg/_string.py

@@ -4,6 +4,8 @@ from contextlib import suppress
 
 from charset_normalizer import detect
 
+from kreuzberg.exceptions import ParsingError
+
 
 def safe_decode(byte_data: bytes, encoding: str | None = None) -> str:
     """Decode a byte string safely, removing invalid sequences.
@@ -12,27 +14,22 @@ def safe_decode(byte_data: bytes, encoding: str | None = None) -> str:
         byte_data: The byte string to decode.
         encoding: The encoding to use when decoding the byte string.
 
+    Raises:
+        ParsingError: If the byte string could not be decoded.
+
     Returns:
         The decoded string.
     """
     if not byte_data:
         return ""
 
-    encodings = ["utf-8", "latin-1"]
-
-    if encoding:
-        with suppress(UnicodeDecodeError):
-            return byte_data.decode(encoding, errors="ignore")
-
-    if encoding := detect(byte_data).get("encoding"):
-        encodings.append(encoding)
+    encodings = [encoding, detect(byte_data).get("encoding", ""), "utf-8", "latin-1"]
 
-    for encoding in encodings:
+    for enc in [e for e in encodings if e]:
         with suppress(UnicodeDecodeError):
-            return byte_data.decode(encoding, errors="ignore")
+            return byte_data.decode(enc)
 
-    # TODO: add test case
-    return byte_data.decode("latin-1", errors="replace")
+    raise ParsingError("Could not decode byte string. Please provide an encoding.")
 
 
 def normalize_spaces(text: str) -> str:

kreuzberg/_tesseract.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import re
+import subprocess
+from asyncio import gather
+from enum import Enum
+from os import PathLike
+from tempfile import NamedTemporaryFile
+from typing import Any, Literal, TypeVar, Union
+
+from anyio import Path as AsyncPath
+from PIL.Image import Image
+
+from kreuzberg._sync import run_sync
+from kreuzberg.exceptions import MissingDependencyError, OCRError
+
+version_ref = {"checked": False}
+
+T = TypeVar("T", bound=Union[Image, PathLike[str], str])
+
+SupportedLanguages = Literal[
+    "afr",
+    "amh",
+    "ara",
+    "asm",
+    "aze",
+    "aze_cyrl",
+    "bel",
+    "ben",
+    "bod",
+    "bos",
+    "bre",
+    "bul",
+    "cat",
+    "ceb",
+    "ces",
+    "chi_sim",
+    "chi_tra",
+    "chr",
+    "cos",
+    "cym",
+    "dan",
+    "dan_frak",
+    "deu",
+    "deu_frak",
+    "deu_latf",
+    "dzo",
+    "ell",
+    "eng",
+    "enm",
+    "epo",
+    "equ",
+    "est",
+    "eus",
+    "fao",
+    "fas",
+    "fil",
+    "fin",
+    "fra",
+    "frk",
+    "frm",
+    "fry",
+    "gla",
+    "gle",
+    "glg",
+    "grc",
+    "guj",
+    "hat",
+    "heb",
+    "hin",
+    "hrv",
+    "hun",
+    "hye",
+    "iku",
+    "ind",
+    "isl",
+    "ita",
+    "ita_old",
+    "jav",
+    "jpn",
+    "kan",
+    "kat",
+    "kat_old",
+    "kaz",
+    "khm",
+    "kir",
+    "kmr",
+    "kor",
+    "kor_vert",
+    "kur",
+    "lao",
+    "lat",
+    "lav",
+    "lit",
+    "ltz",
+    "mal",
+    "mar",
+    "mkd",
+    "mlt",
+    "mon",
+    "mri",
+    "msa",
+    "mya",
+    "nep",
+    "nld",
+    "nor",
+    "oci",
+    "ori",
+    "osd",
+    "pan",
+    "pol",
+    "por",
+    "pus",
+    "que",
+    "ron",
+    "rus",
+    "san",
+    "sin",
+    "slk",
+    "slk_frak",
+    "slv",
+    "snd",
+    "spa",
+    "spa_old",
+    "sqi",
+    "srp",
+    "srp_latn",
+    "sun",
+    "swa",
+    "swe",
+    "syr",
+    "tam",
+    "tat",
+    "tel",
+    "tgk",
+    "tgl",
+    "tha",
+    "tir",
+    "ton",
+    "tur",
+    "uig",
+    "ukr",
+    "urd",
+    "uzb",
+    "uzb_cyrl",
+    "vie",
+    "yid",
+    "yor",
+]
+
+
+class PSMMode(Enum):
+    """Enum for Tesseract Page Segmentation Modes (PSM) with human-readable values."""
+
+    OSD_ONLY = 0
+    """Orientation and script detection only."""
+    AUTO_OSD = 1
+    """Automatic page segmentation with orientation and script detection."""
+    AUTO_ONLY = 2
+    """Automatic page segmentation without OSD."""
+    AUTO = 3
+    """Fully automatic page segmentation (default)."""
+    SINGLE_COLUMN = 4
+    """Assume a single column of text."""
+    SINGLE_BLOCK_VERTICAL = 5
+    """Assume a single uniform block of vertically aligned text."""
+    SINGLE_BLOCK = 6
+    """Assume a single uniform block of text."""
+    SINGLE_LINE = 7
+    """Treat the image as a single text line."""
+    SINGLE_WORD = 8
+    """Treat the image as a single word."""
+    CIRCLE_WORD = 9
+    """Treat the image as a single word in a circle."""
+    SINGLE_CHAR = 10
+    """Treat the image as a single character."""
+
+
+async def validate_tesseract_version() -> None:
+    """Validate that Tesseract is installed and is version 5 or above.
+
+    Raises:
+        MissingDependencyError: If Tesseract is not installed or is below version 5.
+    """
+    try:
+        if version_ref["checked"]:
+            return
+
+        result = await run_sync(subprocess.run, ["tesseract", "--version"], capture_output=True)
+        version_match = re.search(r"tesseract\s+(\d+)", result.stdout.decode())
+        if not version_match or int(version_match.group(1)) < 5:
+            raise MissingDependencyError("Tesseract version 5 or above is required.")
+
+        version_ref["checked"] = True
+    except FileNotFoundError as e:
+        raise MissingDependencyError("Tesseract is not installed.") from e
+
+
+async def process_file(
+    input_file: str | PathLike[str], *, language: SupportedLanguages, psm: PSMMode, **kwargs: Any
+) -> str:
+    """Process a single image file using Tesseract OCR.
+
+    Args:
+        input_file: The path to the image file to process.
+        language: The language code for OCR.
+        psm: Page segmentation mode.
+        **kwargs: Additional Tesseract configuration options as key-value pairs.
+
+    Raises:
+        OCRError: If OCR fails to extract text from the image.
+
+    Returns:
+        str: Extracted text from the image.
+    """
+    with NamedTemporaryFile(suffix=".txt") as output_file:
+        # this is needed because tesseract adds .txt to the output file
+        output_file_name = output_file.name.replace(".txt", "")
+        try:
+            command = [
+                "tesseract",
+                str(input_file),
+                output_file_name,
+                "-l",
+                language,
+                "--psm",
+                str(psm.value),
+            ]
+
+            for key, value in kwargs.items():
+                command.extend(["-c", f"{key}={value}"])
+
+            result = await run_sync(
+                subprocess.run,
+                command,
+                capture_output=True,
+            )
+
+            if not result.returncode == 0:
+                raise OCRError("OCR failed with a non-0 return code.")
+
+            output = await AsyncPath(output_file.name).read_text()
+            return output.strip()
+        except (RuntimeError, OSError) as e:
+            raise OCRError("Failed to OCR using tesseract") from e
+
+
+async def process_image(image: Image, *, language: SupportedLanguages, psm: PSMMode, **kwargs: Any) -> str:
+    """Process a single Pillow Image using Tesseract OCR.
+
+    Args:
+        image: The Pillow Image to process.
+        language: The language code for OCR.
+        psm: Page segmentation mode.
+        **kwargs: Additional Tesseract configuration options as key-value pairs.
+
+    Returns:
+        str: Extracted text from the image.
+    """
+    with NamedTemporaryFile(suffix=".png") as image_file:
+        await run_sync(image.save, image_file.name, format="PNG")
+        return await process_file(image_file.name, language=language, psm=psm, **kwargs)
+
+
+async def process_image_with_tesseract(
+    image: Image | PathLike[str] | str,
+    *,
+    language: SupportedLanguages = "eng",
+    psm: PSMMode = PSMMode.AUTO,
+    **kwargs: Any,
+) -> str:
+    """Run Tesseract OCR asynchronously on a single Pillow Image or a list of Pillow Images.
+
+    Args:
+        image: A single Pillow Image, a pathlike or a string or a list of Pillow Images to process.
+        language: The language code for OCR (default: "eng").
+        psm: Page segmentation mode (default: PSMMode.AUTO).
+        **kwargs: Additional Tesseract configuration options as key-value pairs.
+
+    Raises:
+        ValueError: If the input is not a Pillow Image or a list of Pillow Images.
+
+    Returns:
+        Extracted text as a string
+    """
+    await validate_tesseract_version()
+
+    if isinstance(image, Image):
+        return await process_image(image, language=language, psm=psm, **kwargs)
+
+    if isinstance(image, (PathLike, str)):
+        return await process_file(image, language=language, psm=psm, **kwargs)
+
+    raise ValueError("Input must be one of: str, Pathlike or Pillow Image.")
+
+
+async def batch_process_images(
+    images: list[T],
+    *,
+    language: SupportedLanguages = "eng",
+    psm: PSMMode = PSMMode.AUTO,
+    **kwargs: Any,
+) -> list[str]:
+    """Run Tesseract OCR asynchronously on a single Pillow Image or a list of Pillow Images.
+
+    Args:
+        images: A list of Pillow Images, paths or strings to process.
+        language: The language code for OCR (default: "eng").
+        psm: Page segmentation mode (default: PSMMode.AUTO).
+        **kwargs: Additional Tesseract configuration options as key-value pairs.
+
+    Returns:
+        Extracted text as a string (for single image) or a list of strings (for multiple images).
+    """
+    await validate_tesseract_version()
+    return await gather(
+        *[process_image_with_tesseract(image, language=language, psm=psm, **kwargs) for image in images]
+    )

kreuzberg/exceptions.py

@@ -10,7 +10,7 @@ class KreuzbergError(Exception):
     context: Any
     """The context of the error."""
 
-    def __init__(self, message: str, context: Any = None) -> None:
+    def __init__(self, message: str, *, context: Any = None) -> None:
         self.context = context
         super().__init__(message)
 
@@ -27,3 +27,11 @@ class ParsingError(KreuzbergError):
 
 class ValidationError(KreuzbergError):
     """Raised when a validation error occurs."""
+
+
+class MissingDependencyError(KreuzbergError):
+    """Raised when a dependency is missing."""
+
+
+class OCRError(KreuzbergError):
+    """Raised when an OCR error occurs."""

kreuzberg/extraction.py

@@ -17,13 +17,14 @@ from typing import NamedTuple
 from anyio import Path as AsyncPath
 
 from kreuzberg._extractors import (
-    _extract_content_with_pandoc,
-    _extract_file_with_pandoc,
-    _extract_image_with_tesseract,
-    _extract_pdf_file,
-    _extract_pptx_file,
+    extract_content_with_pandoc,
+    extract_file_with_pandoc,
+    extract_html_string,
+    extract_pdf_file,
+    extract_pptx_file,
 )
 from kreuzberg._mime_types import (
+    HTML_MIME_TYPE,
     IMAGE_MIME_TYPE_EXT_MAP,
     IMAGE_MIME_TYPES,
     MARKDOWN_MIME_TYPE,
@@ -34,6 +35,7 @@ from kreuzberg._mime_types import (
     SUPPORTED_MIME_TYPES,
 )
 from kreuzberg._string import safe_decode
+from kreuzberg._tesseract import process_image_with_tesseract
 from kreuzberg.exceptions import ValidationError
 
 
@@ -70,25 +72,28 @@ async def extract_bytes(content: bytes, mime_type: str, force_ocr: bool = False)
         with NamedTemporaryFile(suffix=".pdf") as temp_file:
             temp_file.write(content)
             return ExtractionResult(
-                content=await _extract_pdf_file(Path(temp_file.name), force_ocr), mime_type=PLAIN_TEXT_MIME_TYPE
+                content=await extract_pdf_file(Path(temp_file.name), force_ocr), mime_type=PLAIN_TEXT_MIME_TYPE
             )
 
     if mime_type in IMAGE_MIME_TYPES or any(mime_type.startswith(value) for value in IMAGE_MIME_TYPES):
         with NamedTemporaryFile(suffix=IMAGE_MIME_TYPE_EXT_MAP[mime_type]) as temp_file:
             temp_file.write(content)
             return ExtractionResult(
-                content=await _extract_image_with_tesseract(temp_file.name), mime_type=PLAIN_TEXT_MIME_TYPE
+                content=await process_image_with_tesseract(temp_file.name), mime_type=PLAIN_TEXT_MIME_TYPE
             )
 
     if mime_type in PANDOC_SUPPORTED_MIME_TYPES or any(
         mime_type.startswith(value) for value in PANDOC_SUPPORTED_MIME_TYPES
     ):
         return ExtractionResult(
-            content=await _extract_content_with_pandoc(content, mime_type), mime_type=MARKDOWN_MIME_TYPE
+            content=await extract_content_with_pandoc(content, mime_type), mime_type=MARKDOWN_MIME_TYPE
         )
 
     if mime_type == POWER_POINT_MIME_TYPE or mime_type.startswith(POWER_POINT_MIME_TYPE):
-        return ExtractionResult(content=await _extract_pptx_file(content), mime_type=MARKDOWN_MIME_TYPE)
+        return ExtractionResult(content=await extract_pptx_file(content), mime_type=MARKDOWN_MIME_TYPE)
+
+    if mime_type == HTML_MIME_TYPE or mime_type.startswith(HTML_MIME_TYPE):
+        return ExtractionResult(content=await extract_html_string(content), mime_type=MARKDOWN_MIME_TYPE)
 
     return ExtractionResult(
         content=safe_decode(content),
@@ -127,19 +132,22 @@ async def extract_file(
         raise ValidationError("The file does not exist.", context={"file_path": str(file_path)})
 
     if mime_type == PDF_MIME_TYPE or mime_type.startswith(PDF_MIME_TYPE):
-        return ExtractionResult(content=await _extract_pdf_file(file_path, force_ocr), mime_type=PLAIN_TEXT_MIME_TYPE)
+        return ExtractionResult(content=await extract_pdf_file(file_path, force_ocr), mime_type=PLAIN_TEXT_MIME_TYPE)
 
     if mime_type in IMAGE_MIME_TYPES or any(mime_type.startswith(value) for value in IMAGE_MIME_TYPES):
-        return ExtractionResult(content=await _extract_image_with_tesseract(file_path), mime_type=PLAIN_TEXT_MIME_TYPE)
+        return ExtractionResult(content=await process_image_with_tesseract(file_path), mime_type=PLAIN_TEXT_MIME_TYPE)
 
     if mime_type in PANDOC_SUPPORTED_MIME_TYPES or any(
         mime_type.startswith(value) for value in PANDOC_SUPPORTED_MIME_TYPES
     ):
         return ExtractionResult(
-            content=await _extract_file_with_pandoc(file_path, mime_type), mime_type=MARKDOWN_MIME_TYPE
+            content=await extract_file_with_pandoc(file_path, mime_type), mime_type=MARKDOWN_MIME_TYPE
         )
 
     if mime_type == POWER_POINT_MIME_TYPE or mime_type.startswith(POWER_POINT_MIME_TYPE):
-        return ExtractionResult(content=await _extract_pptx_file(file_path), mime_type=MARKDOWN_MIME_TYPE)
+        return ExtractionResult(content=await extract_pptx_file(file_path), mime_type=MARKDOWN_MIME_TYPE)
+
+    if mime_type == HTML_MIME_TYPE or mime_type.startswith(HTML_MIME_TYPE):
+        return ExtractionResult(content=await extract_html_string(file_path), mime_type=MARKDOWN_MIME_TYPE)
 
     return ExtractionResult(content=await AsyncPath(file_path).read_text(), mime_type=mime_type)

{kreuzberg-1.2.0.dist-info → kreuzberg-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: kreuzberg
-Version: 1.2.0
+Version: 1.4.0
 Summary: A text extraction library supporting PDFs, images, office documents and more
 Author-email: Na'aman Hirschfeld <nhirschfed@gmail.com>
 License: MIT
@@ -25,11 +25,10 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: anyio>=4.8.0
 Requires-Dist: charset-normalizer>=3.4.1
+Requires-Dist: html-to-markdown>=1.2.0
 Requires-Dist: pypandoc>=1.15
 Requires-Dist: pypdfium2>=4.30.1
-Requires-Dist: pytesseract>=0.3.13
 Requires-Dist: python-pptx>=1.0.2
-Requires-Dist: typing-extensions>=4.12.2
 
 # Kreuzberg
 
@@ -38,7 +37,7 @@ extraction.
 
 Why?
 
-I am building, like many do now, a RAG focused service. I have text extraction needs.
+I am building, like many do now, a RAG focused service (checkout https://grantflow.ai). I have text extraction needs.
 There are quite a lot of commercial options out there, and several open-source + paid options.
 But I wanted something simple, which does not require expansive round-trips to an external API.
 Furthermore, I wanted something that is easy to run locally and isn't very heavy / requires a GPU.
@@ -76,17 +75,32 @@ polished and well maintained.
 
 - PDFs are processed using pdfium2 for searchable PDFs + Tesseract OCR for scanned documents
 - Images are processed using Tesseract OCR
-- Office documents and other formats are processed using Pandoc, or python-pptx for PPTX files
+- Office documents and other formats are processed using Pandoc
+- PPTX files are converted using python-pptx
+- HTML files are converted using html-to-markdown
 - Plain text files are read directly with appropriate encoding detection
 
 ### Roadmap
 
-[] - extra install groups (to make dependencies optional and offer alternatives)
-[] - html file text extraction
-[] - better PDF table extraction
-[] - metadata extraction
+V1:
 
-Feel free to open a discussion in GitHub or an issue if you have any feature requests, but keep the philosophy part in mind
+- [x] - html file text extraction
+- [ ] - better PDF table extraction
+- [ ] - TBD
+
+V2:
+
+- [ ] - extra install groups (to make dependencies optional)
+- [ ] - metadata extraction (possible breaking change)
+- [ ] - TBD
+
+### Feature Requests
+
+Feel free to open a discussion in GitHub or an issue if you have any feature requests
+
+### Contribution
+
+Is welcome! Read guidelines below.
 
 ## Supported File Types
 
@@ -116,6 +130,7 @@ Kreuzberg supports a wide range of file formats:
 
 #### Text and Markup Formats
 
+- HTML (`.html`, `.htm`)
 - Plain Text (`.txt`)
 - Markdown (`.md`)
 - reStructuredText (`.rst`)

kreuzberg-1.4.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+kreuzberg/__init__.py,sha256=5IBPjPsZ7faK15gFB9ZEROHhkEX7KKQmrHPCZuGnhb0,285
+kreuzberg/_extractors.py,sha256=Z6fxNMODsiNGPBv8gYpZ0jrc2hPbX-56xdrVPJ-6SQ4,7658
+kreuzberg/_mime_types.py,sha256=hR6LFXWn8dtCDB05PkADYk2l__HpmETNyf4YFixhecE,2918
+kreuzberg/_string.py,sha256=4txRDnkdR12oO6G8V-jXEMlA9ivgmw8E8EbjyhfL-W4,1106
+kreuzberg/_sync.py,sha256=ovsFHFdkcczz7gNEUJsbZzY8KHG0_GAOOYipQNE4hIY,874
+kreuzberg/_tesseract.py,sha256=nnhkjRIS0BSoovjMIqOlBEXlzngE0QJeFDe7BIqUik8,7872
+kreuzberg/exceptions.py,sha256=pxoEPS0T9e5QSgxsfXn1VmxsY_EGXvTwY0gETPiNn8E,945
+kreuzberg/extraction.py,sha256=gux3fkPIs8IbIKtRGuPFWJBLB5jO6Y9JsBfhHRcpQ0k,6160
+kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kreuzberg-1.4.0.dist-info/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
+kreuzberg-1.4.0.dist-info/METADATA,sha256=ul0iSWSu_1i029aq8X4T4ZboOzWpKK8wZRuvvLVqAoQ,8503
+kreuzberg-1.4.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+kreuzberg-1.4.0.dist-info/top_level.txt,sha256=rbGkygffkZiyKhL8UN41ZOjLfem0jJPA1Whtndne0rE,10
+kreuzberg-1.4.0.dist-info/RECORD,,

kreuzberg-1.2.0.dist-info/RECORD

@@ -1,13 +0,0 @@
-kreuzberg/__init__.py,sha256=5IBPjPsZ7faK15gFB9ZEROHhkEX7KKQmrHPCZuGnhb0,285
-kreuzberg/_extractors.py,sha256=eZ12O7Ii2NRba-dDPIino_eKApCihfdxSPZP121D3xA,7541
-kreuzberg/_mime_types.py,sha256=oJc4Qc2RkfZAYoCmxuuJ4S_Mo9-QQ0c4wwy0ZBqMRoA,2873
-kreuzberg/_string.py,sha256=O023sxdYoC4DhFCU1z430UBdbxqwXKmyymUDDx3J_i8,1156
-kreuzberg/_sync.py,sha256=ovsFHFdkcczz7gNEUJsbZzY8KHG0_GAOOYipQNE4hIY,874
-kreuzberg/exceptions.py,sha256=jrXyvcuSU-694OEtXPZfHYcUbpoRZzNKw9Lo3wIZwL0,770
-kreuzberg/extraction.py,sha256=j5p-JCnyGMouRp4qD-1qKSV_cw8DQ9QSo1H2ocwbbqA,5732
-kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kreuzberg-1.2.0.dist-info/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
-kreuzberg-1.2.0.dist-info/METADATA,sha256=rTuoCAAk9mYh7f55bLSQ9CfEPhh3BnqNHwxdruth1P8,8330
-kreuzberg-1.2.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-kreuzberg-1.2.0.dist-info/top_level.txt,sha256=rbGkygffkZiyKhL8UN41ZOjLfem0jJPA1Whtndne0rE,10
-kreuzberg-1.2.0.dist-info/RECORD,,