kreuzberg 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

kreuzberg/_extractors.py

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, cast
 
 from anyio import Path as AsyncPath
 from charset_normalizer import detect
+from html_to_markdown import convert_to_markdown
 from pptx import Presentation
 from pptx.enum.shapes import MSO_SHAPE_TYPE
 from pypandoc import convert_file, convert_text
@@ -15,7 +16,7 @@ from pypdfium2 import PdfDocument, PdfiumError
 from pytesseract import TesseractError, image_to_string
 
 from kreuzberg._mime_types import PANDOC_MIME_TYPE_EXT_MAP
-from kreuzberg._string import normalize_spaces
+from kreuzberg._string import normalize_spaces, safe_decode
 from kreuzberg._sync import run_sync
 from kreuzberg.exceptions import ParsingError
 
@@ -227,3 +228,20 @@ async def _extract_pptx_file(file_path_or_contents: Path | bytes) -> str:
             md_content = md_content.strip()
 
     return normalize_spaces(md_content)
+
+
+async def _extract_html_string(file_path_or_contents: Path | bytes) -> str:
+    """Extract text from an HTML string.
+
+    Args:
+        file_path_or_contents: The HTML content.
+
+    Returns:
+        The extracted text content.
+    """
+    content = (
+        safe_decode(file_path_or_contents)
+        if isinstance(file_path_or_contents, bytes)
+        else await AsyncPath(file_path_or_contents).read_text()
+    )
+    return normalize_spaces(await run_sync(convert_to_markdown, content))

kreuzberg/_mime_types.py

@@ -5,10 +5,12 @@ from typing import TYPE_CHECKING, Final
 if TYPE_CHECKING:  # pragma: no cover
     from collections.abc import Mapping
 
-MARKDOWN_MIME_TYPE: Final[str] = "text/markdown"
-PLAIN_TEXT_MIME_TYPE: Final[str] = "text/plain"
-PDF_MIME_TYPE: Final[str] = "application/pdf"
-POWER_POINT_MIME_TYPE: Final[str] = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+HTML_MIME_TYPE: Final = "text/html"
+MARKDOWN_MIME_TYPE: Final = "text/markdown"
+PDF_MIME_TYPE: Final = "application/pdf"
+PLAIN_TEXT_MIME_TYPE: Final = "text/plain"
+POWER_POINT_MIME_TYPE: Final = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+
 PLAIN_TEXT_MIME_TYPES: Final[set[str]] = {PLAIN_TEXT_MIME_TYPE, MARKDOWN_MIME_TYPE}
 
 IMAGE_MIME_TYPES: Final[set[str]] = {
@@ -93,5 +95,8 @@ PANDOC_MIME_TYPE_EXT_MAP: Final[Mapping[str, str]] = {
 }
 
 SUPPORTED_MIME_TYPES: Final[set[str]] = (
-    PLAIN_TEXT_MIME_TYPES | IMAGE_MIME_TYPES | PANDOC_SUPPORTED_MIME_TYPES | {PDF_MIME_TYPE, POWER_POINT_MIME_TYPE}
+    PLAIN_TEXT_MIME_TYPES
+    | IMAGE_MIME_TYPES
+    | PANDOC_SUPPORTED_MIME_TYPES
+    | {PDF_MIME_TYPE, POWER_POINT_MIME_TYPE, HTML_MIME_TYPE}
 )

kreuzberg/extraction.py

@@ -19,11 +19,13 @@ from anyio import Path as AsyncPath
 from kreuzberg._extractors import (
     _extract_content_with_pandoc,
     _extract_file_with_pandoc,
+    _extract_html_string,
     _extract_image_with_tesseract,
     _extract_pdf_file,
     _extract_pptx_file,
 )
 from kreuzberg._mime_types import (
+    HTML_MIME_TYPE,
     IMAGE_MIME_TYPE_EXT_MAP,
     IMAGE_MIME_TYPES,
     MARKDOWN_MIME_TYPE,
@@ -90,6 +92,9 @@ async def extract_bytes(content: bytes, mime_type: str, force_ocr: bool = False)
     if mime_type == POWER_POINT_MIME_TYPE or mime_type.startswith(POWER_POINT_MIME_TYPE):
         return ExtractionResult(content=await _extract_pptx_file(content), mime_type=MARKDOWN_MIME_TYPE)
 
+    if mime_type == HTML_MIME_TYPE or mime_type.startswith(HTML_MIME_TYPE):
+        return ExtractionResult(content=await _extract_html_string(content), mime_type=MARKDOWN_MIME_TYPE)
+
     return ExtractionResult(
         content=safe_decode(content),
         mime_type=mime_type,
@@ -142,4 +147,7 @@ async def extract_file(
     if mime_type == POWER_POINT_MIME_TYPE or mime_type.startswith(POWER_POINT_MIME_TYPE):
         return ExtractionResult(content=await _extract_pptx_file(file_path), mime_type=MARKDOWN_MIME_TYPE)
 
+    if mime_type == HTML_MIME_TYPE or mime_type.startswith(HTML_MIME_TYPE):
+        return ExtractionResult(content=await _extract_html_string(file_path), mime_type=MARKDOWN_MIME_TYPE)
+
     return ExtractionResult(content=await AsyncPath(file_path).read_text(), mime_type=mime_type)

{kreuzberg-1.2.0.dist-info → kreuzberg-1.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: kreuzberg
-Version: 1.2.0
+Version: 1.3.0
 Summary: A text extraction library supporting PDFs, images, office documents and more
 Author-email: Na'aman Hirschfeld <nhirschfed@gmail.com>
 License: MIT
@@ -25,6 +25,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: anyio>=4.8.0
 Requires-Dist: charset-normalizer>=3.4.1
+Requires-Dist: html-to-markdown>=1.2.0
 Requires-Dist: pypandoc>=1.15
 Requires-Dist: pypdfium2>=4.30.1
 Requires-Dist: pytesseract>=0.3.13
@@ -38,7 +39,7 @@ extraction.
 
 Why?
 
-I am building, like many do now, a RAG focused service. I have text extraction needs.
+I am building, like many do now, a RAG focused service (checkout https://grantflow.ai). I have text extraction needs.
 There are quite a lot of commercial options out there, and several open-source + paid options.
 But I wanted something simple, which does not require expansive round-trips to an external API.
 Furthermore, I wanted something that is easy to run locally and isn't very heavy / requires a GPU.
@@ -76,17 +77,32 @@ polished and well maintained.
 
 - PDFs are processed using pdfium2 for searchable PDFs + Tesseract OCR for scanned documents
 - Images are processed using Tesseract OCR
-- Office documents and other formats are processed using Pandoc, or python-pptx for PPTX files
+- Office documents and other formats are processed using Pandoc
+- PPTX files are converted using python-pptx
+- HTML files are converted using html-to-markdown
 - Plain text files are read directly with appropriate encoding detection
 
 ### Roadmap
 
-[] - extra install groups (to make dependencies optional and offer alternatives)
-[] - html file text extraction
-[] - better PDF table extraction
-[] - metadata extraction
+V1:
 
-Feel free to open a discussion in GitHub or an issue if you have any feature requests, but keep the philosophy part in mind
+- [x] - html file text extraction
+- [ ] - better PDF table extraction
+- [ ] - TBD
+
+V2:
+
+- [ ] - extra install groups (to make dependencies optional)
+- [ ] - metadata extraction (possible breaking change)
+- [ ] - TBD
+
+### Feature Requests
+
+Feel free to open a discussion in GitHub or an issue if you have any feature requests
+
+### Contribution
+
+Is welcome! Read guidelines below.
 
 ## Supported File Types
 
@@ -116,6 +132,7 @@ Kreuzberg supports a wide range of file formats:
 
 #### Text and Markup Formats
 
+- HTML (`.html`, `.htm`)
 - Plain Text (`.txt`)
 - Markdown (`.md`)
 - reStructuredText (`.rst`)

kreuzberg-1.3.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+kreuzberg/__init__.py,sha256=5IBPjPsZ7faK15gFB9ZEROHhkEX7KKQmrHPCZuGnhb0,285
+kreuzberg/_extractors.py,sha256=eiWPpjnZOZFDwlQL4XsgavJEWqxGtzLVvS8YU28RBAo,8095
+kreuzberg/_mime_types.py,sha256=hR6LFXWn8dtCDB05PkADYk2l__HpmETNyf4YFixhecE,2918
+kreuzberg/_string.py,sha256=O023sxdYoC4DhFCU1z430UBdbxqwXKmyymUDDx3J_i8,1156
+kreuzberg/_sync.py,sha256=ovsFHFdkcczz7gNEUJsbZzY8KHG0_GAOOYipQNE4hIY,874
+kreuzberg/exceptions.py,sha256=jrXyvcuSU-694OEtXPZfHYcUbpoRZzNKw9Lo3wIZwL0,770
+kreuzberg/extraction.py,sha256=cgX8uoCVXf-Va30g8T8DwrZUqsSPHIzmPfDgnWOqNNU,6148
+kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kreuzberg-1.3.0.dist-info/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
+kreuzberg-1.3.0.dist-info/METADATA,sha256=3wiaAuaiA865lg5oCjwlAKaZqRQn1w8VqaQXeoEdip4,8579
+kreuzberg-1.3.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+kreuzberg-1.3.0.dist-info/top_level.txt,sha256=rbGkygffkZiyKhL8UN41ZOjLfem0jJPA1Whtndne0rE,10
+kreuzberg-1.3.0.dist-info/RECORD,,

kreuzberg-1.2.0.dist-info/RECORD

@@ -1,13 +0,0 @@
-kreuzberg/__init__.py,sha256=5IBPjPsZ7faK15gFB9ZEROHhkEX7KKQmrHPCZuGnhb0,285
-kreuzberg/_extractors.py,sha256=eZ12O7Ii2NRba-dDPIino_eKApCihfdxSPZP121D3xA,7541
-kreuzberg/_mime_types.py,sha256=oJc4Qc2RkfZAYoCmxuuJ4S_Mo9-QQ0c4wwy0ZBqMRoA,2873
-kreuzberg/_string.py,sha256=O023sxdYoC4DhFCU1z430UBdbxqwXKmyymUDDx3J_i8,1156
-kreuzberg/_sync.py,sha256=ovsFHFdkcczz7gNEUJsbZzY8KHG0_GAOOYipQNE4hIY,874
-kreuzberg/exceptions.py,sha256=jrXyvcuSU-694OEtXPZfHYcUbpoRZzNKw9Lo3wIZwL0,770
-kreuzberg/extraction.py,sha256=j5p-JCnyGMouRp4qD-1qKSV_cw8DQ9QSo1H2ocwbbqA,5732
-kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kreuzberg-1.2.0.dist-info/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
-kreuzberg-1.2.0.dist-info/METADATA,sha256=rTuoCAAk9mYh7f55bLSQ9CfEPhh3BnqNHwxdruth1P8,8330
-kreuzberg-1.2.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-kreuzberg-1.2.0.dist-info/top_level.txt,sha256=rbGkygffkZiyKhL8UN41ZOjLfem0jJPA1Whtndne0rE,10
-kreuzberg-1.2.0.dist-info/RECORD,,