kreuzberg 2.1.0__py3-none-any.whl → 2.1.2__py3-none-any.whl

kreuzberg/_mime_types.py

@@ -15,6 +15,7 @@ MARKDOWN_MIME_TYPE: Final = "text/markdown"
 PDF_MIME_TYPE: Final = "application/pdf"
 PLAIN_TEXT_MIME_TYPE: Final = "text/plain"
 POWER_POINT_MIME_TYPE: Final = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+DOCX_MIME_TYPE: Final = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
 # Excel formats
 EXCEL_MIME_TYPE: Final = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
 EXCEL_BINARY_MIME_TYPE: Final = "application/vnd.ms-excel"
@@ -73,7 +74,7 @@ PANDOC_SUPPORTED_MIME_TYPES: Final[set[str]] = {
     "application/epub+zip",
     "application/rtf",
     "application/vnd.oasis.opendocument.text",
-    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    DOCX_MIME_TYPE,
     "application/x-biblatex",
     "application/x-bibtex",
     "application/x-endnote+xml",
@@ -146,7 +147,7 @@ EXT_TO_MIME_TYPE: Final[Mapping[str, str]] = {
     ".epub": "application/epub+zip",
     ".rtf": "application/rtf",
     ".odt": "application/vnd.oasis.opendocument.text",
-    ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    ".docx": DOCX_MIME_TYPE,
     ".bib": "application/x-bibtex",
     ".ipynb": "application/x-ipynb+json",
     ".tex": "application/x-latex",

kreuzberg/_pandoc.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import re
 import sys
-from functools import partial
 from json import JSONDecodeError, loads
 from typing import TYPE_CHECKING, Any, Final, Literal, cast
 
@@ -333,14 +332,14 @@ async def process_file_with_pandoc(input_file: str | PathLike[str], *, mime_type
     _get_pandoc_type_from_mime_type(mime_type)
 
     try:
-        metadata, content = await run_taskgroup(
-            partial(_handle_extract_metadata, input_file, mime_type=mime_type),
-            partial(_handle_extract_file, input_file, mime_type=mime_type),
-        )
+        metadata_task = _handle_extract_metadata(input_file, mime_type=mime_type)
+        content_task = _handle_extract_file(input_file, mime_type=mime_type)
+        results = await run_taskgroup(metadata_task, content_task)
+        metadata, content = cast(tuple[Metadata, str], results)
 
         return ExtractionResult(
-            content=normalize_spaces(cast(str, content)),
-            metadata=cast(Metadata, metadata),
+            content=normalize_spaces(content),
+            metadata=metadata,
             mime_type=MARKDOWN_MIME_TYPE,
         )
     except ExceptionGroup as eg:

kreuzberg/_pdf.py

@@ -24,32 +24,36 @@ if TYPE_CHECKING:  # pragma: no cover
 # - Control and non-printable characters
 # - Unicode replacement and invalid characters
 # - Zero-width spaces and other invisible characters
-CORRUPTED_PATTERN: Final[Pattern[str]] = compile_regex(
-    r"[\x00-\x08\x0B-\x1F\x7F-\x9F]|\uFFFD|[\u200B-\u200F\u2028-\u202F]"
-)
+CORRUPTED_PATTERN: Final[Pattern[str]] = compile_regex(r"[\x00-\x08\x0B-\x0C\x0E-\x1F]|\uFFFD")
+SHORT_TEXT_THRESHOLD: Final[int] = 50
+MINIMUM_CORRUPTED_RESULTS: Final[int] = 2
 
 
-def _validate_extracted_text(text: str) -> bool:
+def _validate_extracted_text(text: str, corruption_threshold: float = 0.05) -> bool:
     """Check if text extracted from PDF is valid or corrupted.
 
-    This checks for common indicators of corrupted PDF text extraction:
+    This checks for indicators of corrupted PDF text extraction:
     1. Empty or whitespace-only text
-    2. Control characters and other non-printable characters
-    3. Unicode replacement characters
-    4. Zero-width spaces and other invisible characters
+    2. High concentration of control characters and null bytes
+    3. High concentration of Unicode replacement characters
 
     Args:
         text: The extracted text to validate
+        corruption_threshold: Maximum allowed percentage (0.0-1.0) of corrupted
+            characters (default: 0.05 or 5%)
 
     Returns:
         True if the text appears valid, False if it seems corrupted
     """
-    # Check for empty or whitespace-only text
     if not text or not text.strip():
         return False
 
-    # Check for corruption indicators
-    return not bool(CORRUPTED_PATTERN.search(text))
+    corruption_matches = CORRUPTED_PATTERN.findall(text)
+
+    if len(text) < SHORT_TEXT_THRESHOLD:
+        return len(corruption_matches) <= MINIMUM_CORRUPTED_RESULTS
+
+    return (len(corruption_matches) / len(text)) < corruption_threshold
 
 
 async def _convert_pdf_to_images(input_file: Path) -> list[Image]:
@@ -148,12 +152,10 @@ async def extract_pdf_file(
     Returns:
         The extracted text.
     """
-    if (
-        not force_ocr
-        and (content := await _extract_pdf_searchable_text(input_file))
-        and _validate_extracted_text(content)
-    ):
-        return ExtractionResult(content=content, mime_type=PLAIN_TEXT_MIME_TYPE, metadata={})
+    if not force_ocr:
+        content = await _extract_pdf_searchable_text(input_file)
+        if _validate_extracted_text(content):
+            return ExtractionResult(content=content, mime_type=PLAIN_TEXT_MIME_TYPE, metadata={})
     return await _extract_pdf_text_with_ocr(input_file, max_processes=max_processes, language=language, psm=psm)
 
 

kreuzberg/_sync.py

@@ -2,13 +2,13 @@ from __future__ import annotations
 
 import sys
 from functools import partial
-from typing import TYPE_CHECKING, TypeVar, cast
+from typing import TYPE_CHECKING, Any, TypeVar, cast
 
 from anyio import create_task_group
 from anyio.to_thread import run_sync as any_io_run_sync
 
 if TYPE_CHECKING:  # pragma: no cover
-    from collections.abc import Callable, Coroutine
+    from collections.abc import Awaitable, Callable
 
 if sys.version_info >= (3, 10):
     from typing import ParamSpec
@@ -34,7 +34,7 @@ async def run_sync(sync_fn: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -
     return cast(T, await any_io_run_sync(handler, *args, abandon_on_cancel=True))  # pyright: ignore [reportCallIssue]
 
 
-async def run_taskgroup(*async_tasks: Callable[[], Coroutine[None, None, T]]) -> list[T]:
+async def run_taskgroup(*async_tasks: Awaitable[Any]) -> list[Any]:
     """Run a list of coroutines concurrently.
 
     Args:
@@ -43,10 +43,10 @@ async def run_taskgroup(*async_tasks: Callable[[], Coroutine[None, None, T]]) ->
     Returns:
         The results of the coroutines.
     """
-    results = cast(list[T], [None] * len(async_tasks))
+    results: list[Any] = [None] * len(async_tasks)
 
-    async def run_task(index: int, task: Callable[[], Coroutine[None, None, T]]) -> None:
-        results[index] = await task()
+    async def run_task(index: int, task: Awaitable[T]) -> None:
+        results[index] = await task
 
     async with create_task_group() as tg:
         for i, t in enumerate(async_tasks):
@@ -55,7 +55,7 @@ async def run_taskgroup(*async_tasks: Callable[[], Coroutine[None, None, T]]) ->
     return results
 
 
-async def run_taskgroup_batched(*async_tasks: Callable[[], Coroutine[None, None, T]], batch_size: int) -> list[T]:
+async def run_taskgroup_batched(*async_tasks: Awaitable[Any], batch_size: int) -> list[Any]:
     """Run a list of coroutines concurrently in batches.
 
     Args:
@@ -65,7 +65,7 @@ async def run_taskgroup_batched(*async_tasks: Callable[[], Coroutine[None, None,
     Returns:
         The results of the coroutines.
     """
-    results: list[T] = []
+    results: list[Any] = []
 
     for i in range(0, len(async_tasks), batch_size):
         batch = async_tasks[i : i + batch_size]

kreuzberg/_tesseract.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import re
 import sys
 from enum import Enum
-from functools import partial
 from os import PathLike
 from typing import Any, TypeVar, Union
 
@@ -226,9 +225,7 @@ async def batch_process_images(
     """
     await validate_tesseract_version()
     try:
-        return await run_taskgroup_batched(
-            *[partial(process_image_with_tesseract, image, language=language, psm=psm) for image in images],
-            batch_size=max_processes,
-        )
+        tasks = [process_image_with_tesseract(image, language=language, psm=psm) for image in images]
+        return await run_taskgroup_batched(*tasks, batch_size=max_processes)
     except ExceptionGroup as eg:
         raise ParsingError("Failed to process images with Tesseract", context={"errors": eg.exceptions}) from eg

kreuzberg/_xlsx.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import csv
 import sys
-from functools import partial
 from io import StringIO
 from typing import TYPE_CHECKING
 
@@ -57,9 +56,8 @@ async def extract_xlsx_file(input_file: Path) -> ExtractionResult:
     """
     try:
         workbook: CalamineWorkbook = await run_sync(CalamineWorkbook.from_path, str(input_file))
-        results = await run_taskgroup(
-            *[partial(convert_sheet_to_text, workbook, sheet_name) for sheet_name in workbook.sheet_names]
-        )
+        tasks = [convert_sheet_to_text(workbook, sheet_name) for sheet_name in workbook.sheet_names]
+        results: list[str] = await run_taskgroup(*tasks)
 
         return ExtractionResult(
             content="\n\n".join(results),

{kreuzberg-2.1.0.dist-info → kreuzberg-2.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: kreuzberg
-Version: 2.1.0
+Version: 2.1.2
 Summary: A text extraction library supporting PDFs, images, office documents and more
 Author-email: Na'aman Hirschfeld <nhirschfed@gmail.com>
 License: MIT
@@ -28,10 +28,10 @@ Requires-Dist: anyio>=4.8.0
 Requires-Dist: charset-normalizer>=3.4.1
 Requires-Dist: exceptiongroup>=1.2.2; python_version < "3.11"
 Requires-Dist: html-to-markdown>=1.2.0
-Requires-Dist: pypdfium2>=4.30.1
+Requires-Dist: pypdfium2==4.30.0
 Requires-Dist: python-calamine>=0.3.1
 Requires-Dist: python-pptx>=1.0.2
-Requires-Dist: typing-extensions>=4.12.2; python_version < "3.11"
+Requires-Dist: typing-extensions>=4.12.2; python_version < "3.10"
 
 # Kreuzberg
 
@@ -62,7 +62,7 @@ pip install kreuzberg
 Kreuzberg requires two system level dependencies:
 
 - [Pandoc](https://pandoc.org/installing.html) - For document format conversion. Minimum required version is Pandoc 2.
-- [Tesseract OCR](https://tesseract-ocr.github.io/) - For image and PDF OCR. Minimum required version is Tesseract 4.
+- [Tesseract OCR](https://tesseract-ocr.github.io/) - For image and PDF OCR. Minimum required version is Tesseract 5.
 
 You can install these with:
 
@@ -75,7 +75,6 @@ sudo apt-get install pandoc tesseract-ocr
 #### MacOS
 
 ```shell
-#
 brew install tesseract pandoc
 ```
 

{kreuzberg-2.1.0.dist-info → kreuzberg-2.1.2.dist-info}/RECORD

@@ -1,21 +1,21 @@
 kreuzberg/__init__.py,sha256=WgGo3x09JKCk89htZuodbnYysu0ZYpkAP29dcRl5Sg0,694
 kreuzberg/_constants.py,sha256=N61ZF8xuEso8GzRGiVpqIv5yfMkQmLeH_EN9fVARYV0,249
 kreuzberg/_html.py,sha256=yM78bPjyKRaXqMp5QW9xOYe0CBd9uUhDZfjnFB1tZOY,925
-kreuzberg/_mime_types.py,sha256=VFaOozh8o7Xv1d3pa9mObkz7DM8tVsZC_W8hxsMUeb4,6451
-kreuzberg/_pandoc.py,sha256=lUqG1GQqezz011fLn12AUKJ_xw9gElj-S7xRO5g-Rlw,12513
-kreuzberg/_pdf.py,sha256=BI7ooYvvLPEX3y7lKyri4r0k6bW4pj_cmBQW1UqZiF8,6227
+kreuzberg/_mime_types.py,sha256=Kuu0yWY4p0Eck8b_vdp9oamqRZc1RJaS_ZKikVD2Z2o,6431
+kreuzberg/_pandoc.py,sha256=YIXaFC11N2tgVHjBd3JD_21GZ6OOVQ0UY3aKrWNfK-I,12531
+kreuzberg/_pdf.py,sha256=AIwxlydZkJOU4878SaeF9cKUmzSN7o3X40Hye7z017U,6479
 kreuzberg/_pptx.py,sha256=oX1WYabKQ02Hla2jYnkEBjJXCPvrcRnzLi3MeY86TN0,3028
 kreuzberg/_string.py,sha256=pE92BF2E7BXrQ5if3uATM2enwH82ntViBpshxK-797E,1106
-kreuzberg/_sync.py,sha256=DepezWTfsyyeEq7VYjhWD6XFRiaEz-uCvXFUYkQMswQ,2191
-kreuzberg/_tesseract.py,sha256=gKGyZpa_MLLsMTpzi_VvSXFAmLxagRE-sfqH2oKFmPM,7662
+kreuzberg/_sync.py,sha256=sDVH4GrpYW9SOnmu3BqKPL76xl0hxzHjTAC78aovbQA,2122
+kreuzberg/_tesseract.py,sha256=0BkguZJIKlOFHkrN2mjVgaycWwolmuEv6DwpQY7n7Os,7610
 kreuzberg/_tmp.py,sha256=y0PxKJXsRsDCwpFqtJAMl05lMNu3N_E2yaUVL93h7g0,1037
 kreuzberg/_types.py,sha256=Qxlk6qfdtvEsCfjsXU57qgZiONfwF7wUgbCJK8QXNZ4,2195
-kreuzberg/_xlsx.py,sha256=JcQTdV38uiNdyRmHQ1DI6khN8ng4W38tIRaxonIoaHs,2703
+kreuzberg/_xlsx.py,sha256=kSH7PJ33vdLgoh5LmL_bqbc4I0VgZlZUeF4ckKl6NJM,2675
 kreuzberg/exceptions.py,sha256=syDCjy8PNqVMGhD-zAuhkurLMg9bk1j1yJtvJN8cN9A,1679
 kreuzberg/extraction.py,sha256=7oc2C1_bIxrLx2r4NEyGrL9Jt6YpPxfQKMRJm6QQayo,13076
 kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kreuzberg-2.1.0.dist-info/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
-kreuzberg-2.1.0.dist-info/METADATA,sha256=t1NeglNqJFjWpr6WeIp-d33OikT_HIrS8FrEMGSk1hA,14844
-kreuzberg-2.1.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-kreuzberg-2.1.0.dist-info/top_level.txt,sha256=rbGkygffkZiyKhL8UN41ZOjLfem0jJPA1Whtndne0rE,10
-kreuzberg-2.1.0.dist-info/RECORD,,
+kreuzberg-2.1.2.dist-info/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
+kreuzberg-2.1.2.dist-info/METADATA,sha256=0MEegHP8F5ur-wafeprL9UEN6Utipml1SuCF_xF6daA,14842
+kreuzberg-2.1.2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+kreuzberg-2.1.2.dist-info/top_level.txt,sha256=rbGkygffkZiyKhL8UN41ZOjLfem0jJPA1Whtndne0rE,10
+kreuzberg-2.1.2.dist-info/RECORD,,

{kreuzberg-2.1.0.dist-info → kreuzberg-2.1.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any