kreuzberg 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
kreuzberg/__init__.py ADDED
@@ -0,0 +1,11 @@
1
+ from .exceptions import KreuzbergError, ParsingError, ValidationError
2
+ from .extraction import ExtractionResult, extract_bytes, extract_file
3
+
4
+ __all__ = [
5
+ "ExtractionResult",
6
+ "KreuzbergError",
7
+ "ParsingError",
8
+ "ValidationError",
9
+ "extract_bytes",
10
+ "extract_file",
11
+ ]
@@ -0,0 +1,145 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING, cast
4
+
5
+ from charset_normalizer import detect
6
+ from pypandoc import convert_file, convert_text
7
+ from pypdfium2 import PdfDocument, PdfiumError
8
+ from pytesseract import TesseractError, image_to_string
9
+
10
+ from kreuzberg._mime_types import PANDOC_MIME_TYPE_EXT_MAP
11
+ from kreuzberg._sync import run_sync
12
+ from kreuzberg.exceptions import ParsingError
13
+
14
+ if TYPE_CHECKING:
15
+ from pathlib import Path
16
+
17
+
18
+ def _extract_pdf_with_tesseract(file_path: Path) -> str:
19
+ """Extract text from a scanned PDF file using pytesseract.
20
+
21
+ Args:
22
+ file_path: The path to the PDF file.
23
+
24
+ Raises:
25
+ ParsingError: If the text could not be extracted from the PDF file.
26
+
27
+ Returns:
28
+ The extracted text.
29
+ """
30
+ try:
31
+ # make it into an image here:
32
+ pdf = PdfDocument(str(file_path))
33
+ images = [page.render(scale=2.0).to_pil() for page in pdf]
34
+
35
+ text = "\n".join(image_to_string(img) for img in images)
36
+ return text.strip()
37
+ except (PdfiumError, TesseractError) as e:
38
+ raise ParsingError(
39
+ "Could not extract text from PDF file", context={"file_path": str(file_path), "error": str(e)}
40
+ ) from e
41
+
42
+
43
+ def _extract_pdf_with_pdfium2(file_path: Path) -> str:
44
+ """Extract text from a searchable PDF file using pypdfium2.
45
+
46
+ Args:
47
+ file_path: The path to the PDF file.
48
+
49
+ Raises:
50
+ ParsingError: If the text could not be extracted from the PDF file.
51
+
52
+ Returns:
53
+ The extracted text.
54
+ """
55
+ try:
56
+ document = PdfDocument(file_path)
57
+ text = "\n".join(page.get_textpage().get_text_range() for page in document)
58
+ return text.strip()
59
+ except PdfiumError as e:
60
+ raise ParsingError(
61
+ "Could not extract text from PDF file", context={"file_path": str(file_path), "error": str(e)}
62
+ ) from e
63
+
64
+
65
+ async def _extract_pdf_file(file_path: Path) -> str:
66
+ """Extract text from a PDF file.
67
+
68
+ Args:
69
+ file_path: The path to the PDF file.
70
+
71
+ Returns:
72
+ The extracted text.
73
+ """
74
+ if content := await run_sync(_extract_pdf_with_pdfium2, file_path):
75
+ return content
76
+
77
+ return await run_sync(_extract_pdf_with_tesseract, file_path)
78
+
79
+
80
+ async def _extract_content_with_pandoc(file_data: bytes, mime_type: str, encoding: str | None = None) -> str:
81
+ """Extract text using pandoc.
82
+
83
+ Args:
84
+ file_data: The content of the file.
85
+ mime_type: The mime type of the file.
86
+ encoding: An optional encoding to use when decoding the string.
87
+
88
+ Raises:
89
+ ParsingError: If the text could not be extracted from the file using pandoc.
90
+
91
+ Returns:
92
+ The extracted text.
93
+ """
94
+ ext = PANDOC_MIME_TYPE_EXT_MAP[mime_type]
95
+ encoding = encoding or detect(file_data)["encoding"] or "utf-8"
96
+ try:
97
+ return cast(str, await run_sync(convert_text, file_data, to="md", format=ext, encoding=encoding))
98
+ except RuntimeError as e:
99
+ raise ParsingError(
100
+ f"Could not extract text from {PANDOC_MIME_TYPE_EXT_MAP[mime_type]} file contents",
101
+ context={"error": str(e)},
102
+ ) from e
103
+
104
+
105
+ async def _extract_file_with_pandoc(file_path: Path | str, mime_type: str) -> str:
106
+ """Extract text using pandoc.
107
+
108
+ Args:
109
+ file_path: The path to the file.
110
+ mime_type: The mime type of the file.
111
+
112
+ Raises:
113
+ ParsingError: If the text could not be extracted from the file using pandoc.
114
+
115
+ Returns:
116
+ The extracted text.
117
+ """
118
+ ext = PANDOC_MIME_TYPE_EXT_MAP[mime_type]
119
+ try:
120
+ return cast(str, await run_sync(convert_file, file_path, to="md", format=ext))
121
+ except RuntimeError as e:
122
+ raise ParsingError(
123
+ f"Could not extract text from {PANDOC_MIME_TYPE_EXT_MAP[mime_type]} file",
124
+ context={"file_path": str(file_path), "error": str(e)},
125
+ ) from e
126
+
127
+
128
+ async def _extract_image_with_tesseract(file_path: Path | str) -> str:
129
+ """Extract text from an image file.
130
+
131
+ Args:
132
+ file_path: The path to the image file.
133
+
134
+ Raises:
135
+ ParsingError: If the text could not be extracted from the image file.
136
+
137
+ Returns:
138
+ The extracted content.
139
+ """
140
+ try:
141
+ return cast(str, image_to_string(str(file_path)).strip())
142
+ except TesseractError as e:
143
+ raise ParsingError(
144
+ "Could not extract text from image file", context={"file_path": str(file_path), "error": str(e)}
145
+ ) from e
@@ -0,0 +1,97 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING, Final
4
+
5
+ if TYPE_CHECKING:
6
+ from collections.abc import Mapping
7
+
8
+ MARKDOWN_MIME_TYPE: Final[str] = "text/markdown"
9
+ PLAIN_TEXT_MIME_TYPE: Final[str] = "text/plain"
10
+ PDF_MIME_TYPE: Final[str] = "application/pdf"
11
+
12
+ PLAIN_TEXT_MIME_TYPES: Final[set[str]] = {PLAIN_TEXT_MIME_TYPE, MARKDOWN_MIME_TYPE}
13
+
14
+ IMAGE_MIME_TYPES: Final[set[str]] = {
15
+ "image/bmp",
16
+ "image/gif",
17
+ "image/jp2",
18
+ "image/jpeg",
19
+ "image/jpm",
20
+ "image/jpx",
21
+ "image/mj2",
22
+ "image/pjpeg",
23
+ "image/png",
24
+ "image/tiff",
25
+ "image/webp",
26
+ "image/x-bmp",
27
+ "image/x-ms-bmp",
28
+ "image/x-portable-anymap",
29
+ "image/x-portable-bitmap",
30
+ "image/x-portable-graymap",
31
+ "image/x-portable-pixmap",
32
+ "image/x-tiff",
33
+ }
34
+ IMAGE_MIME_TYPE_EXT_MAP: Final[Mapping[str, str]] = {
35
+ "image/bmp": "bmp",
36
+ "image/x-bmp": "bmp",
37
+ "image/x-ms-bmp": "bmp",
38
+ "image/gif": "gif",
39
+ "image/jpeg": "jpg",
40
+ "image/pjpeg": "jpg",
41
+ "image/png": "png",
42
+ "image/tiff": "tiff",
43
+ "image/x-tiff": "tiff",
44
+ "image/jp2": "jp2",
45
+ "image/jpx": "jpx",
46
+ "image/jpm": "jpm",
47
+ "image/mj2": "mj2",
48
+ "image/webp": "webp",
49
+ "image/x-portable-anymap": "pnm",
50
+ "image/x-portable-bitmap": "pbm",
51
+ "image/x-portable-graymap": "pgm",
52
+ "image/x-portable-pixmap": "ppm",
53
+ }
54
+ PANDOC_SUPPORTED_MIME_TYPES: Final[set[str]] = {
55
+ "application/csv",
56
+ "application/latex",
57
+ "application/rtf",
58
+ "application/vnd.oasis.opendocument.text",
59
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
60
+ "application/x-csv",
61
+ "application/x-latex",
62
+ "application/x-rtf",
63
+ "application/x-vnd.oasis.opendocument.text",
64
+ "text/csv",
65
+ "text/latex",
66
+ "text/rst",
67
+ "text/rtf",
68
+ "text/tab-separated-values",
69
+ "text/x-csv",
70
+ "text/x-latex",
71
+ "text/x-rst",
72
+ "text/x-tsv",
73
+ }
74
+ PANDOC_MIME_TYPE_EXT_MAP: Final[Mapping[str, str]] = {
75
+ "application/csv": "csv",
76
+ "application/latex": "latex",
77
+ "application/rtf": "rtf",
78
+ "application/vnd.oasis.opendocument.text": "odt",
79
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
80
+ "application/x-csv": "csv",
81
+ "application/x-latex": "latex",
82
+ "application/x-rtf": "rtf",
83
+ "application/x-vnd.oasis.opendocument.text": "odt",
84
+ "text/csv": "csv",
85
+ "text/latex": "latex",
86
+ "text/rst": "rst",
87
+ "text/rtf": "rtf",
88
+ "text/tab-separated-values": "tsv",
89
+ "text/x-csv": "csv",
90
+ "text/x-latex": "latex",
91
+ "text/x-rst": "rst",
92
+ "text/x-tsv": "tsv",
93
+ }
94
+
95
+ SUPPORTED_MIME_TYPES: Final[set[str]] = (
96
+ PLAIN_TEXT_MIME_TYPES | IMAGE_MIME_TYPES | PANDOC_SUPPORTED_MIME_TYPES | {PDF_MIME_TYPE}
97
+ )
kreuzberg/_string.py ADDED
@@ -0,0 +1,35 @@
1
+ from __future__ import annotations
2
+
3
+ from charset_normalizer import detect
4
+
5
+
6
+ def safe_decode(byte_data: bytes, encoding: str | None = None) -> str:
7
+ """Decode a byte string safely, removing invalid sequences.
8
+
9
+ Args:
10
+ byte_data: The byte string to decode.
11
+ encoding: The encoding to use when decoding the byte string.
12
+
13
+ Returns:
14
+ The decoded string.
15
+ """
16
+ if not byte_data:
17
+ return ""
18
+
19
+ if encoding:
20
+ try:
21
+ return byte_data.decode(encoding, errors="ignore")
22
+ except UnicodeDecodeError: # pragma: no cover
23
+ pass
24
+
25
+ encodings = ["utf-8", "latin-1"]
26
+ if encoding := detect(byte_data).get("encoding"):
27
+ encodings.append(encoding)
28
+
29
+ for encoding in encodings:
30
+ try:
31
+ return byte_data.decode(encoding, errors="ignore")
32
+ except UnicodeDecodeError: # pragma: no cover # noqa: PERF203
33
+ pass
34
+
35
+ return byte_data.decode("latin-1", errors="replace")
kreuzberg/_sync.py ADDED
@@ -0,0 +1,28 @@
1
+ from __future__ import annotations
2
+
3
+ from functools import partial
4
+ from typing import TYPE_CHECKING, TypeVar, cast
5
+
6
+ from anyio.to_thread import run_sync as any_io_run_sync
7
+ from typing_extensions import ParamSpec
8
+
9
+ if TYPE_CHECKING:
10
+ from collections.abc import Callable
11
+
12
+ T = TypeVar("T")
13
+ P = ParamSpec("P")
14
+
15
+
16
+ async def run_sync(sync_fn: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T:
17
+ """Run a synchronous function in an asynchronous context.
18
+
19
+ Args:
20
+ sync_fn: The synchronous function to run.
21
+ *args: The positional arguments to pass to the function.
22
+ **kwargs: The keyword arguments to pass to the function.
23
+
24
+ Returns:
25
+ The result of the synchronous function.
26
+ """
27
+ handler = partial(sync_fn, **kwargs)
28
+ return cast(T, await any_io_run_sync(handler, *args)) # pyright: ignore [reportCallIssue]
@@ -0,0 +1,29 @@
1
+ from __future__ import annotations
2
+
3
+ from json import dumps
4
+ from typing import Any
5
+
6
+
7
+ class KreuzbergError(Exception):
8
+ """Base exception for all Kreuzberg errors."""
9
+
10
+ context: Any
11
+ """The context of the error."""
12
+
13
+ def __init__(self, message: str, context: Any = None) -> None:
14
+ self.context = context
15
+ super().__init__(message)
16
+
17
+ def __str__(self) -> str:
18
+ """Return a string representation of the exception."""
19
+ ctx = f"\n\nContext: {dumps(self.context)}" if self.context else ""
20
+
21
+ return f"{self.__class__.__name__}: {super().__str__()}{ctx}"
22
+
23
+
24
+ class ParsingError(KreuzbergError):
25
+ """Raised when a parsing error occurs."""
26
+
27
+
28
+ class ValidationError(KreuzbergError):
29
+ """Raised when a validation error occurs."""
@@ -0,0 +1,124 @@
1
+ from __future__ import annotations
2
+
3
+ from mimetypes import guess_type
4
+ from pathlib import Path
5
+ from tempfile import NamedTemporaryFile
6
+ from typing import NamedTuple
7
+
8
+ from anyio import Path as AsyncPath
9
+
10
+ from kreuzberg._extractors import (
11
+ _extract_content_with_pandoc,
12
+ _extract_file_with_pandoc,
13
+ _extract_image_with_tesseract,
14
+ _extract_pdf_file,
15
+ )
16
+ from kreuzberg._mime_types import (
17
+ IMAGE_MIME_TYPE_EXT_MAP,
18
+ IMAGE_MIME_TYPES,
19
+ MARKDOWN_MIME_TYPE,
20
+ PANDOC_SUPPORTED_MIME_TYPES,
21
+ PDF_MIME_TYPE,
22
+ PLAIN_TEXT_MIME_TYPE,
23
+ SUPPORTED_MIME_TYPES,
24
+ )
25
+ from kreuzberg._string import safe_decode
26
+ from kreuzberg.exceptions import ValidationError
27
+
28
+
29
+ class ExtractionResult(NamedTuple):
30
+ """The result of a file extraction."""
31
+
32
+ content: str
33
+ """The extracted content."""
34
+ mime_type: str
35
+ """The mime type of the content."""
36
+
37
+
38
+ async def extract_bytes(content: bytes, mime_type: str) -> ExtractionResult:
39
+ """Extract the textual content from a given byte string representing a file's contents.
40
+
41
+ Args:
42
+ content: The content to extract.
43
+ mime_type: The mime type of the content.
44
+
45
+ Raises:
46
+ ValidationError: If the mime type is not supported.
47
+
48
+ Returns:
49
+ The extracted content and the mime type of the content.
50
+ """
51
+ if mime_type not in SUPPORTED_MIME_TYPES or not any(mime_type.startswith(value) for value in SUPPORTED_MIME_TYPES):
52
+ raise ValidationError(
53
+ f"Unsupported mime type: {mime_type}",
54
+ context={"mime_type": mime_type, "supported_mimetypes": ",".join(sorted(SUPPORTED_MIME_TYPES))},
55
+ )
56
+
57
+ if mime_type == PDF_MIME_TYPE or mime_type.startswith(PDF_MIME_TYPE):
58
+ with NamedTemporaryFile(suffix=".pdf") as temp_file:
59
+ temp_file.write(content)
60
+ return ExtractionResult(
61
+ content=await _extract_pdf_file(Path(temp_file.name)), mime_type=PLAIN_TEXT_MIME_TYPE
62
+ )
63
+
64
+ if mime_type in IMAGE_MIME_TYPES or any(mime_type.startswith(value) for value in IMAGE_MIME_TYPES):
65
+ with NamedTemporaryFile(suffix=IMAGE_MIME_TYPE_EXT_MAP[mime_type]) as temp_file:
66
+ temp_file.write(content)
67
+ return ExtractionResult(
68
+ content=await _extract_image_with_tesseract(temp_file.name), mime_type=PLAIN_TEXT_MIME_TYPE
69
+ )
70
+
71
+ if mime_type in PANDOC_SUPPORTED_MIME_TYPES or any(
72
+ mime_type.startswith(value) for value in PANDOC_SUPPORTED_MIME_TYPES
73
+ ):
74
+ return ExtractionResult(
75
+ content=await _extract_content_with_pandoc(content, mime_type), mime_type=MARKDOWN_MIME_TYPE
76
+ )
77
+
78
+ return ExtractionResult(
79
+ content=safe_decode(content),
80
+ mime_type=mime_type,
81
+ )
82
+
83
+
84
+ async def extract_file(file_path: Path | str, mime_type: str | None = None) -> ExtractionResult:
85
+ """Extract the textual content from a given file.
86
+
87
+ Args:
88
+ file_path: The path to the file.
89
+ mime_type: The mime type of the file.
90
+
91
+ Raises:
92
+ ValidationError: If the mime type is not supported.
93
+
94
+ Returns:
95
+ The extracted content and the mime type of the content.
96
+ """
97
+ file_path = Path(file_path)
98
+ mime_type = mime_type or guess_type(file_path.name)[0]
99
+ if not mime_type: # pragma: no cover
100
+ raise ValidationError("Could not determine the mime type of the file.", context={"file_path": str(file_path)})
101
+
102
+ if mime_type not in SUPPORTED_MIME_TYPES or not any(mime_type.startswith(value) for value in SUPPORTED_MIME_TYPES):
103
+ raise ValidationError(
104
+ f"Unsupported mime type: {mime_type}",
105
+ context={"mime_type": mime_type, "supported_mimetypes": ",".join(sorted(SUPPORTED_MIME_TYPES))},
106
+ )
107
+
108
+ if not await AsyncPath(file_path).exists():
109
+ raise ValidationError("The file does not exist.", context={"file_path": str(file_path)})
110
+
111
+ if mime_type == PDF_MIME_TYPE or mime_type.startswith(PDF_MIME_TYPE):
112
+ return ExtractionResult(content=await _extract_pdf_file(file_path), mime_type=PLAIN_TEXT_MIME_TYPE)
113
+
114
+ if mime_type in IMAGE_MIME_TYPES or any(mime_type.startswith(value) for value in IMAGE_MIME_TYPES):
115
+ return ExtractionResult(content=await _extract_image_with_tesseract(file_path), mime_type=PLAIN_TEXT_MIME_TYPE)
116
+
117
+ if mime_type in PANDOC_SUPPORTED_MIME_TYPES or any(
118
+ mime_type.startswith(value) for value in PANDOC_SUPPORTED_MIME_TYPES
119
+ ):
120
+ return ExtractionResult(
121
+ content=await _extract_file_with_pandoc(file_path, mime_type), mime_type=MARKDOWN_MIME_TYPE
122
+ )
123
+
124
+ return ExtractionResult(content=await AsyncPath(file_path).read_text(), mime_type=mime_type)
kreuzberg/py.typed ADDED
File without changes
@@ -0,0 +1,7 @@
1
+ Copyright 2025 Na'aman Hirschfeld
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
4
+
5
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
6
+
7
+ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,259 @@
1
+ Metadata-Version: 2.2
2
+ Name: kreuzberg
3
+ Version: 1.0.0
4
+ Summary: A text extraction library supporting PDFs, images, office documents and more
5
+ Author-email: Na'aman Hirschfeld <nhirschfed@gmail.com>
6
+ License: MIT
7
+ Project-URL: homepage, https://github.com/Goldziher/kreuzberg
8
+ Keywords: async,document-processing,docx,image-to-text,latex,markdown,ocr,odt,office-documents,pandoc,pdf,pdf-extraction,rag,tesseract,text-extraction,text-processing
9
+ Classifier: Development Status :: 4 - Beta
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python :: 3 :: Only
14
+ Classifier: Programming Language :: Python :: 3.9
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
20
+ Classifier: Topic :: Text Processing :: General
21
+ Classifier: Topic :: Utilities
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.9
24
+ Description-Content-Type: text/markdown
25
+ License-File: LICENSE
26
+ Requires-Dist: anyio>=4.8.0
27
+ Requires-Dist: charset-normalizer>=3.4.1
28
+ Requires-Dist: pypandoc>=1.15
29
+ Requires-Dist: pypdfium2>=4.30.1
30
+ Requires-Dist: pytesseract>=0.3.13
31
+ Requires-Dist: typing-extensions>=4.12.2
32
+
33
+ # Kreuzberg
34
+
35
+ Kreuzberg is a library for simplified text extraction from PDF files. It's meant to offer simple, hassle free text
36
+ extraction.
37
+
38
+ Why?
39
+
40
+ I am building, like many do now, a RAG focused service. I have text extraction needs.
41
+ There are quite a lot of commercial options out there, and several open-source + paid options.
42
+ But I wanted something simple, which does not require expansive round-trips to an external API.
43
+ Furthermore, I wanted something that is easy to run locally and isn't very heavy / requires a GPU.
44
+
45
+ Hence, this library.
46
+
47
+ ## Features
48
+
49
+ - Extract text from PDFs, images, and office documents
50
+ - Use modern Python with async (via `anyio`) and proper type hints
51
+ - Extensive error handling for easy debugging
52
+
53
+ ## Installation
54
+
55
+ 1. Begin by installing the python package:
56
+
57
+ ```shell
58
+
59
+ pip install kreuzberg
60
+
61
+ ```
62
+
63
+ 2. Install the system dependencies:
64
+
65
+ - [pandoc](https://pandoc.org/installing.html) (non-pdf text extraction, GPL v2.0 licensed but used via CLI only)
66
+ - [tesseract-ocr](https://tesseract-ocr.github.io/) (for image/PDF OCR, Apache License)
67
+
68
+ ## Supported File Types
69
+
70
+ Kreuzberg supports a wide range of file formats:
71
+
72
+ ### Document Formats
73
+
74
+ - PDF (`.pdf`) - both searchable and scanned documents
75
+ - Word Documents (`.docx`)
76
+ - OpenDocument Text (`.odt`)
77
+ - Rich Text Format (`.rtf`)
78
+
79
+ ### Image Formats
80
+
81
+ - JPEG, JPG (`.jpg`, `.jpeg`, `.pjpeg`)
82
+ - PNG (`.png`)
83
+ - TIFF (`.tiff`, `.tif`)
84
+ - BMP (`.bmp`)
85
+ - GIF (`.gif`)
86
+ - WebP (`.webp`)
87
+ - JPEG 2000 (`.jp2`, `.jpx`, `.jpm`, `.mj2`)
88
+ - Portable Anymap (`.pnm`)
89
+ - Portable Bitmap (`.pbm`)
90
+ - Portable Graymap (`.pgm`)
91
+ - Portable Pixmap (`.ppm`)
92
+
93
+ #### Text and Markup Formats
94
+
95
+ - Plain Text (`.txt`)
96
+ - Markdown (`.md`)
97
+ - reStructuredText (`.rst`)
98
+ - LaTeX (`.tex`)
99
+
100
+ #### Data Formats
101
+
102
+ - Comma-Separated Values (`.csv`)
103
+ - Tab-Separated Values (`.tsv`)
104
+
105
+ All formats support text extraction, with different processing methods:
106
+
107
+ - PDFs are processed using pdfium2 for searchable PDFs and Tesseract OCR for scanned documents
108
+ - Images are processed using Tesseract OCR
109
+ - Office documents and other formats are processed using Pandoc
110
+ - Plain text files are read directly with appropriate encoding detection
111
+
112
+ ## Usage
113
+
114
+ Kreuzberg exports two async functions:
115
+
116
+ - Extract text from a file (string path or `pathlib.Path`) using `extract_file()`
117
+ - Extract text from a byte-string using `extract_bytes()`
118
+
119
+ Note - both of these functions are async and therefore should be used in an async context.
120
+
121
+ ### Extract from File
122
+
123
+ ```python
124
+ from pathlib import Path
125
+ from kreuzberg import extract_file
126
+
127
+
128
+ # Extract text from a PDF file
129
+ async def extract_pdf():
130
+ result = await extract_file("document.pdf")
131
+ print(f"Extracted text: {result.content}")
132
+ print(f"Output mime type: {result.mime_type}")
133
+
134
+
135
+ # Extract text from an image
136
+ async def extract_image():
137
+ result = await extract_file("scan.png")
138
+ print(f"Extracted text: {result.content}")
139
+
140
+
141
+ # or use Path
142
+
143
+ async def extract_pdf():
144
+ result = await extract_file(Path("document.pdf"))
145
+ print(f"Extracted text: {result.content}")
146
+ print(f"Output mime type: {result.mime_type}")
147
+ ```
148
+
149
+ ### Extract from Bytes
150
+
151
+ ```python
152
+ from kreuzberg import extract_bytes
153
+
154
+
155
+ # Extract text from PDF bytes
156
+ async def process_uploaded_pdf(pdf_content: bytes):
157
+ result = await extract_bytes(pdf_content, mime_type="application/pdf")
158
+ return result.content
159
+
160
+
161
+ # Extract text from image bytes
162
+ async def process_uploaded_image(image_content: bytes):
163
+ result = await extract_bytes(image_content, mime_type="image/jpeg")
164
+ return result.content
165
+ ```
166
+
167
+ ### Error Handling
168
+
169
+ Kreuzberg raises two exception types:
170
+
171
+ #### ValidationError
172
+
173
+ Raised when there are issues with input validation:
174
+
175
+ - Unsupported mime types
176
+ - Non-existent files
177
+ - Undetectable mime types
178
+
179
+ #### ParsingError
180
+
181
+ Raised when there are issues during the text extraction process:
182
+
183
+ - PDF parsing failures
184
+ - OCR errors
185
+ - Pandoc conversion errors
186
+
187
+ ```python
188
+ from kreuzberg import extract_file
189
+ from kreuzberg.exceptions import ValidationError, ParsingError
190
+
191
+
192
+ async def safe_extract():
193
+ try:
194
+ result = await extract_file("document.doc")
195
+ return result.content
196
+ except ValidationError as e:
197
+ print(f"Validation error: {e.message}")
198
+ print(f"Context: {e.context}")
199
+ except ParsingError as e:
200
+ print(f"Parsing error: {e.message}")
201
+ print(f"Context: {e.context}") # Contains detailed error information
202
+ ```
203
+
204
+ Both error types include helpful context information for debugging:
205
+
206
+ ```python
207
+ try:
208
+ result = await extract_file("scanned.pdf")
209
+ except ParsingError as e:
210
+ # e.context might contain:
211
+ # {
212
+ # "file_path": "scanned.pdf",
213
+ # "error": "Tesseract OCR failed: Unable to process image"
214
+ # }
215
+ ```
216
+
217
+ ### ExtractionResult
218
+
219
+ All extraction functions return an ExtractionResult named tuple containing:
220
+
221
+ - content: The extracted text as a string
222
+ - mime_type: The mime type of the output (either "text/plain" or, if pandoc is used- "text/markdown")
223
+
224
+ ```python
225
+ from kreuzberg import ExtractionResult
226
+
227
+
228
+ async def process_document(path: str) -> str:
229
+ result: ExtractionResult = await extract_file(path)
230
+ return result.content
231
+
232
+
233
+ # or access the result as tuple
234
+
235
+ async def process_document(path: str) -> str:
236
+ content, mime_type = await extract_file(path)
237
+ # do something with mime_type
238
+ return content
239
+ ```
240
+
241
+ ## Contribution
242
+
243
+ This library is open to contribution. Feel free to open issues or submit PRs. Its better to discuss issues before
244
+ submitting PRs to avoid disappointment.
245
+
246
+ ### Local Development
247
+
248
+ 1. Clone the repo
249
+ 2. Install the system dependencies
250
+ 3. Install the full dependencies with `uv sync`
251
+ 4. Install the pre-commit hooks with:
252
+ ```shell
253
+ pre-commit install && pre-commit install --hook-type commit-msg
254
+ ```
255
+ 5. Make your changes and submit a PR
256
+
257
+ ## License
258
+
259
+ This library uses the MIT license.
@@ -0,0 +1,13 @@
1
+ kreuzberg/__init__.py,sha256=5IBPjPsZ7faK15gFB9ZEROHhkEX7KKQmrHPCZuGnhb0,285
2
+ kreuzberg/_extractors.py,sha256=tmOgzhKw8J21R-NKWSgu7yf5epGleoxC9nKQacUDdms,4461
3
+ kreuzberg/_mime_types.py,sha256=VI3bWm7NBF0Vs2PXpxnJxTlt0pRSE59raVO_KTDJCVQ,2719
4
+ kreuzberg/_string.py,sha256=8YezUPhTGEMk08yGrBxVu4CwhUdCQwOvyC6EGB7wxLk,975
5
+ kreuzberg/_sync.py,sha256=OQZTSKUOaSMkxAb4ynq-BDrx1JLAYP9uc_zFZaAN_fk,854
6
+ kreuzberg/exceptions.py,sha256=jrXyvcuSU-694OEtXPZfHYcUbpoRZzNKw9Lo3wIZwL0,770
7
+ kreuzberg/extraction.py,sha256=utxr9HM8K2aDU0LXHVKCNPXqTu7fGDeNCNpamGr6hAQ,4646
8
+ kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
+ kreuzberg-1.0.0.dist-info/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
10
+ kreuzberg-1.0.0.dist-info/METADATA,sha256=fQszunogmstxhdJMMD5ieXLRqjBojXpb0pXJAZZO8fQ,7238
11
+ kreuzberg-1.0.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
12
+ kreuzberg-1.0.0.dist-info/top_level.txt,sha256=rbGkygffkZiyKhL8UN41ZOjLfem0jJPA1Whtndne0rE,10
13
+ kreuzberg-1.0.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (75.8.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ kreuzberg