kreuzberg 3.9.1__py3-none-any.whl → 3.10.1__py3-none-any.whl

kreuzberg/_api/main.py

@@ -30,7 +30,7 @@ try:
         HTTP_422_UNPROCESSABLE_ENTITY,
         HTTP_500_INTERNAL_SERVER_ERROR,
     )
-except ImportError as e:
+except ImportError as e:  # pragma: no cover
     raise MissingDependencyError.create_for_package(
         dependency_group="litestar",
         functionality="Litestar API and docker container",

kreuzberg/_chunker.py

@@ -43,7 +43,7 @@ def get_chunker(
                 from semantic_text_splitter import TextSplitter  # noqa: PLC0415
 
                 _chunkers[key] = TextSplitter(max_characters, overlap_characters)
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="chunking", functionality="chunking", package_name="semantic-text-splitter"
             ) from e

kreuzberg/_config.py

@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any
 
 if sys.version_info >= (3, 11):
     import tomllib
-else:
+else:  # pragma: no cover
     import tomli as tomllib  # type: ignore[import-not-found]
 
 from kreuzberg._gmft import GMFTConfig
@@ -50,7 +50,13 @@ def load_config_from_file(config_path: Path) -> dict[str, Any]:
     # Handle both kreuzberg.toml (root level) and pyproject.toml ([tool.kreuzberg])
     if config_path.name == "kreuzberg.toml":
         return data  # type: ignore[no-any-return]
-    return data.get("tool", {}).get("kreuzberg", {})  # type: ignore[no-any-return]
+
+    # For other files, check if they have [tool.kreuzberg] section
+    if config_path.name == "pyproject.toml" or ("tool" in data and "kreuzberg" in data.get("tool", {})):
+        return data.get("tool", {}).get("kreuzberg", {})  # type: ignore[no-any-return]
+
+    # Otherwise assume root-level configuration
+    return data  # type: ignore[no-any-return]
 
 
 def merge_configs(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
@@ -129,12 +135,23 @@ def build_extraction_config_from_dict(config_dict: dict[str, Any]) -> Extraction
         "extract_keywords",
         "auto_detect_language",
         "enable_quality_processing",
+        "auto_detect_document_type",
+        "document_type_confidence_threshold",
+        "document_classification_mode",
+        "keyword_count",
     }
     extraction_config.update({field: config_dict[field] for field in basic_fields if field in config_dict})
 
     # Handle OCR backend configuration
     ocr_backend = extraction_config.get("ocr_backend")
     if ocr_backend and ocr_backend != "none":
+        # Validate OCR backend
+        valid_backends = {"tesseract", "easyocr", "paddleocr"}
+        if ocr_backend not in valid_backends:
+            raise ValidationError(
+                f"Invalid OCR backend: {ocr_backend}. Must be one of: {', '.join(sorted(valid_backends))} or 'none'",
+                context={"provided": ocr_backend, "valid": sorted(valid_backends)},
+            )
         ocr_config = parse_ocr_backend_config(config_dict, ocr_backend)
         if ocr_config:
             extraction_config["ocr_config"] = ocr_config
@@ -286,6 +303,10 @@ _CONFIG_FIELDS = [
     "extract_keywords",
     "auto_detect_language",
     "enable_quality_processing",
+    "auto_detect_document_type",
+    "document_type_confidence_threshold",
+    "document_classification_mode",
+    "keyword_count",
 ]
 
 

kreuzberg/_document_classification.py

@@ -4,13 +4,12 @@ import re
 from typing import TYPE_CHECKING
 
 from kreuzberg._ocr import get_ocr_backend
+from kreuzberg._types import ExtractionConfig, ExtractionResult  # noqa: TC001
 from kreuzberg.exceptions import MissingDependencyError
 
 if TYPE_CHECKING:
     from pathlib import Path
 
-    from kreuzberg._types import ExtractionConfig, ExtractionResult
-
 
 DOCUMENT_CLASSIFIERS = {
     "invoice": [
@@ -52,14 +51,25 @@ def _get_translated_text(result: ExtractionResult) -> str:
     Raises:
         MissingDependencyError: If the deep-translator package is not installed
     """
+    # Combine content with metadata for classification
+    text_to_classify = result.content
+    if result.metadata:
+        # Add metadata values to the text for classification
+        metadata_text = " ".join(str(value) for value in result.metadata.values() if value)
+        text_to_classify = f"{text_to_classify} {metadata_text}"
+
     try:
         from deep_translator import GoogleTranslator  # noqa: PLC0415
-    except ImportError as e:
+    except ImportError as e:  # pragma: no cover
         raise MissingDependencyError(
             "The 'deep-translator' library is not installed. Please install it with: pip install 'kreuzberg[auto-classify-document-type]'"
         ) from e
 
-    return str(GoogleTranslator(source="auto", target="en").translate(result.content).lower())
+    try:
+        return str(GoogleTranslator(source="auto", target="en").translate(text_to_classify).lower())
+    except Exception:  # noqa: BLE001
+        # Fall back to original content in lowercase if translation fails
+        return text_to_classify.lower()
 
 
 def classify_document(result: ExtractionResult, config: ExtractionConfig) -> tuple[str | None, float | None]:
@@ -73,6 +83,9 @@ def classify_document(result: ExtractionResult, config: ExtractionConfig) -> tup
         A tuple containing the detected document type and the confidence score,
         or (None, None) if no type is detected with sufficient confidence.
     """
+    if not config.auto_detect_document_type:
+        return None, None
+
     translated_text = _get_translated_text(result)
     scores = dict.fromkeys(DOCUMENT_CLASSIFIERS, 0)
 
@@ -108,7 +121,8 @@ def classify_document_from_layout(
         A tuple containing the detected document type and the confidence score,
         or (None, None) if no type is detected with sufficient confidence.
     """
-    translated_text = _get_translated_text(result)
+    if not config.auto_detect_document_type:
+        return None, None
 
     if result.layout is None or result.layout.empty:
         return None, None
@@ -117,6 +131,24 @@ def classify_document_from_layout(
     if not all(col in layout_df.columns for col in ["text", "top", "height"]):
         return None, None
 
+    # Use layout text for classification, not the content
+    layout_text = " ".join(layout_df["text"].astype(str).tolist())
+
+    # Translate layout text directly for classification
+    text_to_classify = layout_text
+    if result.metadata:
+        # Add metadata values to the text for classification
+        metadata_text = " ".join(str(value) for value in result.metadata.values() if value)
+        text_to_classify = f"{text_to_classify} {metadata_text}"
+
+    try:
+        from deep_translator import GoogleTranslator  # noqa: PLC0415
+
+        translated_text = str(GoogleTranslator(source="auto", target="en").translate(text_to_classify).lower())
+    except Exception:  # noqa: BLE001
+        # Fall back to original content in lowercase if translation fails
+        translated_text = text_to_classify.lower()
+
     layout_df["translated_text"] = translated_text
 
     page_height = layout_df["top"].max() + layout_df["height"].max()
@@ -151,6 +183,9 @@ def auto_detect_document_type(
     if config.document_classification_mode == "vision" and file_path:
         layout_result = get_ocr_backend("tesseract").process_file_sync(file_path, **config.get_config_dict())
         result.document_type, result.document_type_confidence = classify_document_from_layout(layout_result, config)
+    elif result.layout is not None and not result.layout.empty:
+        # Use layout-based classification if layout data is available
+        result.document_type, result.document_type_confidence = classify_document_from_layout(result, config)
     else:
         result.document_type, result.document_type_confidence = classify_document(result, config)
     return result

kreuzberg/_entity_extraction.py

@@ -139,7 +139,7 @@ def extract_entities(
 
     try:
         import spacy  # noqa: F401, PLC0415
-    except ImportError as e:
+    except ImportError as e:  # pragma: no cover
         raise MissingDependencyError.create_for_package(
             package_name="spacy",
             dependency_group="entity-extraction",
@@ -230,7 +230,7 @@ def extract_keywords(
         return [(kw, float(score)) for kw, score in keywords]
     except (RuntimeError, OSError, ValueError):
         return []
-    except ImportError as e:
+    except ImportError as e:  # pragma: no cover
         raise MissingDependencyError.create_for_package(
             package_name="keybert",
             dependency_group="entity-extraction",

kreuzberg/_extractors/_email.py

@@ -19,12 +19,12 @@ if TYPE_CHECKING:
 # Import optional dependencies at module level with proper error handling
 try:
     import mailparse
-except ImportError:
+except ImportError:  # pragma: no cover
     mailparse = None
 
 try:
     import html2text  # type: ignore[import-not-found]
-except ImportError:
+except ImportError:  # pragma: no cover
     html2text = None
 
 # Compile regex pattern once at module level
@@ -59,14 +59,19 @@ class EmailExtractor(Extractor):
 
         to_info = parsed_email.get("to")
         if to_info:
+            # Store the raw value in metadata (could be string, dict, or list)
             if isinstance(to_info, list) and to_info:
+                # For metadata, use first recipient's email if it's a list
                 to_email = to_info[0].get("email", "") if isinstance(to_info[0], dict) else str(to_info[0])
+                metadata["email_to"] = to_email
             elif isinstance(to_info, dict):
-                to_email = to_info.get("email", "")
+                metadata["email_to"] = to_info.get("email", "")
             else:
-                to_email = str(to_info)
-            metadata["email_to"] = to_email
-            text_parts.append(f"To: {to_email}")
+                metadata["email_to"] = str(to_info)
+
+            # For display, format all recipients
+            to_formatted = self._format_email_field(to_info)
+            text_parts.append(f"To: {to_formatted}")
 
         date = parsed_email.get("date")
         if date:
@@ -76,12 +81,30 @@ class EmailExtractor(Extractor):
         cc = parsed_email.get("cc")
         if cc:
             metadata["email_cc"] = cc
-            text_parts.append(f"CC: {cc}")
+            cc_formatted = self._format_email_field(cc)
+            text_parts.append(f"CC: {cc_formatted}")
 
         bcc = parsed_email.get("bcc")
         if bcc:
             metadata["email_bcc"] = bcc
-            text_parts.append(f"BCC: {bcc}")
+            bcc_formatted = self._format_email_field(bcc)
+            text_parts.append(f"BCC: {bcc_formatted}")
+
+    def _format_email_field(self, field: Any) -> str:
+        """Format email field (to, cc, bcc) for display."""
+        if isinstance(field, list):
+            emails = []
+            for item in field:
+                if isinstance(item, dict):
+                    email = item.get("email", "")
+                    if email:
+                        emails.append(email)
+                else:
+                    emails.append(str(item))
+            return ", ".join(emails)
+        if isinstance(field, dict):
+            return str(field.get("email", ""))
+        return str(field)
 
     def _extract_email_body(self, parsed_email: dict[str, Any], text_parts: list[str]) -> None:
         """Extract and process email body content."""

kreuzberg/_extractors/_pdf.py

@@ -22,7 +22,7 @@ from kreuzberg._ocr._easyocr import EasyOCRConfig
 from kreuzberg._ocr._paddleocr import PaddleOCRConfig
 from kreuzberg._ocr._tesseract import TesseractConfig
 from kreuzberg._playa import extract_pdf_metadata, extract_pdf_metadata_sync
-from kreuzberg._types import ExtractionResult, OcrBackendType
+from kreuzberg._types import ExtractionResult, Metadata, OcrBackendType
 from kreuzberg._utils._errors import create_error_context, should_retry
 from kreuzberg._utils._pdf_lock import pypdfium_file_lock
 from kreuzberg._utils._string import normalize_spaces
@@ -33,6 +33,7 @@ from kreuzberg.exceptions import ParsingError
 
 if TYPE_CHECKING:  # pragma: no cover
     from PIL.Image import Image
+    from playa.document import Document
 
 
 class PDFExtractor(Extractor):
@@ -45,7 +46,7 @@ class PDFExtractor(Extractor):
         file_path, unlink = await create_temp_file(".pdf")
         await AsyncPath(file_path).write_bytes(content)
         try:
-            metadata = await extract_pdf_metadata(content)
+            metadata = await self._extract_metadata_with_password_attempts(content)
             result = await self.extract_path_async(file_path)
 
             result.metadata = metadata
@@ -73,7 +74,7 @@ class PDFExtractor(Extractor):
         if not result:
             result = ExtractionResult(content="", mime_type=PLAIN_TEXT_MIME_TYPE, metadata={}, chunks=[])
 
-        result.metadata = await extract_pdf_metadata(content_bytes)
+        result.metadata = await self._extract_metadata_with_password_attempts(content_bytes)
 
         if self.config.extract_tables:
             # GMFT is optional dependency
@@ -81,7 +82,7 @@ class PDFExtractor(Extractor):
                 from kreuzberg._gmft import extract_tables  # noqa: PLC0415
 
                 result.tables = await extract_tables(path, self.config.gmft_config)
-            except ImportError:
+            except ImportError:  # pragma: no cover
                 result.tables = []
 
             # Enhance metadata with table information
@@ -107,7 +108,7 @@ class PDFExtractor(Extractor):
 
             result = self.extract_path_sync(Path(temp_path))
 
-            metadata = extract_pdf_metadata_sync(content)
+            metadata = self._extract_metadata_with_password_attempts_sync(content)
             result.metadata = metadata
 
             return result
@@ -406,11 +407,81 @@ class PDFExtractor(Extractor):
         # Use list comprehension and join for efficient string building
         return "\n\n".join(result.content for result in results)
 
+    def _parse_with_password_attempts(self, content: bytes) -> Document:
+        """Parse PDF with password attempts."""
+        # Normalize password to list
+        if isinstance(self.config.pdf_password, str):
+            passwords = [self.config.pdf_password] if self.config.pdf_password else [""]
+        else:
+            passwords = list(self.config.pdf_password)
+
+        # Try each password in sequence
+        last_exception = None
+        for password in passwords:
+            try:
+                return parse(content, max_workers=1, password=password)
+            except Exception as e:  # noqa: PERF203, BLE001
+                last_exception = e
+                continue
+
+        # If all passwords failed, raise the last exception
+        if last_exception:
+            raise last_exception from None
+
+        # Fallback to no password
+        return parse(content, max_workers=1, password="")
+
+    def _get_passwords_to_try(self) -> list[str]:
+        """Get list of passwords to try in sequence."""
+        if isinstance(self.config.pdf_password, str):
+            return [self.config.pdf_password] if self.config.pdf_password else [""]
+        return list(self.config.pdf_password) if self.config.pdf_password else [""]
+
+    async def _extract_metadata_with_password_attempts(self, content: bytes) -> Metadata:
+        """Extract PDF metadata with password attempts."""
+        passwords = self._get_passwords_to_try()
+
+        last_exception = None
+        for password in passwords:
+            try:
+                return await extract_pdf_metadata(content, password=password)
+            except Exception as e:  # noqa: PERF203, BLE001
+                last_exception = e
+                continue
+
+        # If all passwords failed, try with empty password as fallback
+        try:
+            return await extract_pdf_metadata(content, password="")
+        except Exception:
+            if last_exception:
+                raise last_exception from None
+            raise
+
+    def _extract_metadata_with_password_attempts_sync(self, content: bytes) -> Metadata:
+        """Extract PDF metadata with password attempts (sync version)."""
+        passwords = self._get_passwords_to_try()
+
+        last_exception = None
+        for password in passwords:
+            try:
+                return extract_pdf_metadata_sync(content, password=password)
+            except Exception as e:  # noqa: PERF203, BLE001
+                last_exception = e
+                continue
+
+        # If all passwords failed, try with empty password as fallback
+        try:
+            return extract_pdf_metadata_sync(content, password="")
+        except Exception:
+            if last_exception:
+                raise last_exception from None
+            raise
+
     def _extract_with_playa_sync(self, path: Path, fallback_text: str) -> str:
         """Extract text using playa for better structure preservation."""
         with contextlib.suppress(Exception):
             content = path.read_bytes()
-            document = parse(content, max_workers=1)
+            document = self._parse_with_password_attempts(content)
 
             # Extract text while preserving structure
             pages_text = []

kreuzberg/_extractors/_structured.py

@@ -6,15 +6,15 @@ from typing import TYPE_CHECKING, Any, ClassVar
 
 if sys.version_info >= (3, 11):
     import tomllib
-else:
+else:  # pragma: no cover
     try:
         import tomli as tomllib  # type: ignore[import-not-found]
-    except ImportError:
+    except ImportError:  # pragma: no cover
         tomllib = None
 
 try:
     import yaml
-except ImportError:
+except ImportError:  # pragma: no cover
     yaml = None
 
 from anyio import Path as AsyncPath

kreuzberg/_gmft.py

@@ -265,7 +265,7 @@ async def extract_tables(
             finally:
                 await run_sync(doc.close)
 
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="gmft", functionality="table extraction", package_name="gmft"
             ) from e
@@ -379,7 +379,7 @@ def extract_tables_sync(
         finally:
             doc.close()  # type: ignore[no-untyped-call]
 
-    except ImportError as e:
+    except ImportError as e:  # pragma: no cover
         raise MissingDependencyError.create_for_package(
             dependency_group="gmft", functionality="table extraction", package_name="gmft"
         ) from e

kreuzberg/_language_detection.py

@@ -14,7 +14,7 @@ try:
     from fast_langdetect import detect, detect_multilingual
 
     HAS_FAST_LANGDETECT = True
-except ImportError:
+except ImportError:  # pragma: no cover
     HAS_FAST_LANGDETECT = False
     detect = None
     detect_multilingual = None

kreuzberg/_mcp/server.py

@@ -268,7 +268,7 @@ def extract_structured(file_path: str) -> list[TextContent]:
     return [TextContent(type="text", text=content)]
 
 
-def main() -> None:
+def main() -> None:  # pragma: no cover
     """Main entry point for the MCP server."""
     mcp.run()
 

kreuzberg/_ocr/_base.py

@@ -88,7 +88,7 @@ class OCRBackend(ABC, Generic[T]):
         Returns:
             List of extraction result objects in the same order as input paths
         """
-        return [self.process_file_sync(path, **kwargs) for path in paths]
+        return [self.process_file_sync(path, **kwargs) for path in paths]  # pragma: no cover
 
     async def process_batch(self, paths: list[Path], **kwargs: Unpack[T]) -> list[ExtractionResult]:
         """Asynchronously process a batch of files and extract their text and metadata.
@@ -106,8 +106,8 @@ class OCRBackend(ABC, Generic[T]):
         from kreuzberg._utils._sync import run_taskgroup  # noqa: PLC0415
 
         tasks = [self.process_file(path, **kwargs) for path in paths]
-        return await run_taskgroup(*tasks)
+        return await run_taskgroup(*tasks)  # pragma: no cover
 
     def __hash__(self) -> int:
         """Hash function for allowing caching."""
-        return hash(type(self).__name__)
+        return hash(type(self).__name__)  # pragma: no cover

kreuzberg/_ocr/_easyocr.py

@@ -321,7 +321,7 @@ class EasyOCRBackend(OCRBackend[EasyOCRConfig]):
             import torch  # noqa: PLC0415
 
             return bool(torch.cuda.is_available())
-        except ImportError:
+        except ImportError:  # pragma: no cover
             return False
 
     @classmethod
@@ -340,7 +340,7 @@ class EasyOCRBackend(OCRBackend[EasyOCRConfig]):
 
         try:
             import easyocr  # noqa: PLC0415
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="easyocr", functionality="EasyOCR as an OCR backend", package_name="easyocr"
             ) from e
@@ -508,7 +508,7 @@ class EasyOCRBackend(OCRBackend[EasyOCRConfig]):
 
         try:
             import easyocr  # noqa: PLC0415
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="easyocr", functionality="EasyOCR as an OCR backend", package_name="easyocr"
             ) from e

kreuzberg/_ocr/_paddleocr.py

@@ -261,7 +261,7 @@ class PaddleBackend(OCRBackend[PaddleOCRConfig]):
 
         try:
             from paddleocr import PaddleOCR  # noqa: PLC0415
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="paddleocr", functionality="PaddleOCR as an OCR backend", package_name="paddleocr"
             ) from e
@@ -428,7 +428,7 @@ class PaddleBackend(OCRBackend[PaddleOCRConfig]):
 
         try:
             from paddleocr import PaddleOCR  # noqa: PLC0415
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="paddleocr", functionality="PaddleOCR as an OCR backend", package_name="paddleocr"
             ) from e

kreuzberg/_playa.py

@@ -24,11 +24,12 @@ FULL_DATE_LENGTH = 14
 BOM_CHAR = "\ufeff"
 
 
-async def extract_pdf_metadata(pdf_content: bytes) -> Metadata:
+async def extract_pdf_metadata(pdf_content: bytes, password: str = "") -> Metadata:
     """Extract metadata from a PDF document.
 
     Args:
         pdf_content: The bytes of the PDF document.
+        password: Password for encrypted PDF files.
 
     Raises:
         ParsingError: If the PDF metadata could not be extracted.
@@ -37,7 +38,7 @@ async def extract_pdf_metadata(pdf_content: bytes) -> Metadata:
         A dictionary of metadata extracted from the PDF.
     """
     try:
-        document = parse(pdf_content, max_workers=1)
+        document = parse(pdf_content, max_workers=1, password=password)
         metadata: Metadata = {}
 
         for raw_info in document.info:
@@ -142,7 +143,9 @@ def _parse_date_string(date_str: str) -> str:
             minute = date_str[10:12]
             second = date_str[12:14]
             time_part = f"T{hour}:{minute}:{second}"
-        return datetime.strptime(f"{year}-{month}-{day}{time_part}", "%Y%m%d%H%M%S").isoformat()  # noqa: DTZ007
+        if time_part:
+            return datetime.strptime(f"{year}-{month}-{day}{time_part}", "%Y-%m-%dT%H:%M:%S").isoformat()  # noqa: DTZ007
+        return datetime.strptime(f"{year}-{month}-{day}", "%Y-%m-%d").isoformat()  # noqa: DTZ007
     return date_str
 
 
@@ -275,13 +278,14 @@ def _extract_structure_information(document: Document, result: Metadata) -> None
             result["subtitle"] = subtitle
 
 
-def extract_pdf_metadata_sync(pdf_content: bytes) -> Metadata:
+def extract_pdf_metadata_sync(pdf_content: bytes, password: str = "") -> Metadata:
     """Synchronous version of extract_pdf_metadata.
 
     Extract metadata from a PDF document without using async/await.
 
     Args:
         pdf_content: The bytes of the PDF document.
+        password: Password for encrypted PDF files.
 
     Raises:
         ParsingError: If the PDF metadata could not be extracted.
@@ -290,7 +294,7 @@ def extract_pdf_metadata_sync(pdf_content: bytes) -> Metadata:
         A dictionary of metadata extracted from the PDF.
     """
     try:
-        document = parse(pdf_content, max_workers=1)
+        document = parse(pdf_content, max_workers=1, password=password)
         metadata: Metadata = {}
 
         for raw_info in document.info:

kreuzberg/_types.py

@@ -269,7 +269,7 @@ class ExtractionResult:
         Returns:
             List of CSV strings, one per table
         """
-        if not self.tables:
+        if not self.tables:  # pragma: no cover
             return []
 
         return [export_table_to_csv(table) for table in self.tables]
@@ -280,7 +280,7 @@ class ExtractionResult:
         Returns:
             List of TSV strings, one per table
         """
-        if not self.tables:
+        if not self.tables:  # pragma: no cover
             return []
 
         return [export_table_to_tsv(table) for table in self.tables]
@@ -291,7 +291,7 @@ class ExtractionResult:
         Returns:
             List of table structure dictionaries
         """
-        if not self.tables:
+        if not self.tables:  # pragma: no cover
             return []
 
         return [extract_table_structure_info(table) for table in self.tables]
@@ -349,14 +349,16 @@ class ExtractionConfig:
     """Configuration for language detection. If None, uses default settings."""
     spacy_entity_extraction_config: SpacyEntityExtractionConfig | None = None
     """Configuration for spaCy entity extraction. If None, uses default settings."""
-    auto_detect_document_type: bool = False
+    auto_detect_document_type: bool = True
     """Whether to automatically detect the document type."""
-    document_type_confidence_threshold: float = 0.7
+    document_type_confidence_threshold: float = 0.5
     """Confidence threshold for document type detection."""
     document_classification_mode: Literal["text", "vision"] = "text"
     """The mode to use for document classification."""
     enable_quality_processing: bool = True
     """Whether to apply quality post-processing to improve extraction results."""
+    pdf_password: str | list[str] = ""
+    """Password(s) for encrypted PDF files. Can be a single password or list of passwords to try in sequence. Only used when crypto extra is installed."""
 
     def __post_init__(self) -> None:
         if self.custom_entity_patterns is not None and isinstance(self.custom_entity_patterns, dict):

kreuzberg/_utils/_device.py

@@ -144,7 +144,7 @@ def _is_cuda_available() -> bool:
         import torch  # type: ignore[import-not-found,unused-ignore]  # noqa: PLC0415
 
         return bool(torch.cuda.is_available())
-    except ImportError:
+    except ImportError:  # pragma: no cover
         return False
 
 
@@ -154,7 +154,7 @@ def _is_mps_available() -> bool:
         import torch  # type: ignore[import-not-found,unused-ignore]  # noqa: PLC0415
 
         return bool(torch.backends.mps.is_available())
-    except ImportError:
+    except ImportError:  # pragma: no cover
         return False
 
 
@@ -190,7 +190,7 @@ def _get_cuda_devices() -> list[DeviceInfo]:
                 )
             )
 
-    except ImportError:
+    except ImportError:  # pragma: no cover
         pass
 
     return devices
@@ -209,7 +209,7 @@ def _get_mps_device() -> DeviceInfo | None:
             name="Apple Silicon GPU (MPS)",
         )
 
-    except ImportError:
+    except ImportError:  # pragma: no cover
         return None
 
 
@@ -232,7 +232,7 @@ def _get_cuda_memory_info(device_id: int) -> tuple[float | None, float | None]:
 
         return total_memory, available_memory
 
-    except ImportError:
+    except ImportError:  # pragma: no cover
         return None, None
 
 
@@ -333,7 +333,7 @@ def cleanup_device_memory(device: DeviceInfo) -> None:
 
             if torch.cuda.is_available():
                 torch.cuda.empty_cache()
-        except ImportError:
+        except ImportError:  # pragma: no cover  # pragma: no cover
             pass
 
     elif device.device_type == "mps":

kreuzberg/_utils/_document_cache.py

@@ -58,6 +58,7 @@ class DocumentCache:
                 "chunk_content": config.chunk_content,
                 "max_chars": config.max_chars,
                 "max_overlap": config.max_overlap,
+                "auto_detect_document_type": config.auto_detect_document_type,
             }
 
         cache_data = {**file_info, **config_info}

kreuzberg/cli.py

@@ -12,7 +12,7 @@ try:
     import click
     from rich.console import Console
     from rich.progress import Progress, SpinnerColumn, TextColumn
-except ImportError as e:
+except ImportError as e:  # pragma: no cover
     raise ImportError(
         "CLI dependencies are not installed. Please install kreuzberg with the 'cli' extra: pip install kreuzberg[cli]"
     ) from e
@@ -163,7 +163,7 @@ def _perform_extraction(file: Path | None, extraction_config: ExtractionConfig,
                 import magic  # type: ignore[import-not-found]  # noqa: PLC0415
 
                 mime_type = magic.from_buffer(input_bytes, mime=True)
-            except ImportError:
+            except ImportError:  # pragma: no cover
                 content_str = input_bytes.decode("utf-8", errors="ignore").lower()
                 mime_type = "text/html" if "<html" in content_str or "<body" in content_str else "text/plain"
 
@@ -193,7 +193,7 @@ def _write_output(
         click.echo(formatted_output)
 
 
-def handle_error(error: Exception, verbose: bool) -> None:
+def handle_error(error: Exception, verbose: bool) -> None:  # pragma: no cover
     """Handle and display errors.
 
     Args:
@@ -202,19 +202,19 @@ def handle_error(error: Exception, verbose: bool) -> None:
     """
     if isinstance(error, MissingDependencyError):
         console.print(f"[red]Missing dependency:[/red] {error}", style="bold")
-        sys.exit(2)
+        sys.exit(2)  # pragma: no cover
     elif isinstance(error, KreuzbergError):
         console.print(f"[red]Error:[/red] {error}", style="bold")
         if verbose and error.context:
             console.print("\n[dim]Context:[/dim]")
             console.print(json.dumps(error.context, indent=2))
-        sys.exit(1)
+        sys.exit(1)  # pragma: no cover
     else:
         console.print(f"[red]Unexpected error:[/red] {type(error).__name__}: {error}", style="bold")
         if verbose:
             console.print("\n[dim]Traceback:[/dim]")
             traceback.print_exc()
-        sys.exit(1)
+        sys.exit(1)  # pragma: no cover
 
 
 @click.group(invoke_without_command=True)

{kreuzberg-3.9.1.dist-info → kreuzberg-3.10.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.9.1
+Version: 3.10.1
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
 Project-URL: documentation, https://kreuzberg.dev
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
@@ -53,6 +53,7 @@ Requires-Dist: litestar[opentelemetry,standard,structlog]>=2.16.0; extra == 'all
 Requires-Dist: mailparse>=1.0.15; extra == 'all'
 Requires-Dist: paddleocr>=3.1.0; extra == 'all'
 Requires-Dist: paddlepaddle>=3.1.0; extra == 'all'
+Requires-Dist: playa-pdf[crypto]>=0.6.4; extra == 'all'
 Requires-Dist: rich>=14.1.0; extra == 'all'
 Requires-Dist: semantic-text-splitter>=0.27.0; extra == 'all'
 Requires-Dist: setuptools>=80.9.0; extra == 'all'
@@ -69,6 +70,8 @@ Provides-Extra: cli
 Requires-Dist: click>=8.2.1; extra == 'cli'
 Requires-Dist: rich>=14.1.0; extra == 'cli'
 Requires-Dist: tomli>=2.0.0; (python_version < '3.11') and extra == 'cli'
+Provides-Extra: crypto
+Requires-Dist: playa-pdf[crypto]>=0.6.4; extra == 'crypto'
 Provides-Extra: easyocr
 Requires-Dist: easyocr>=1.7.2; extra == 'easyocr'
 Provides-Extra: entity-extraction

{kreuzberg-3.9.1.dist-info → kreuzberg-3.10.1.dist-info}/RECORD

@@ -1,43 +1,43 @@
 kreuzberg/__init__.py,sha256=0OJ_jNKbS6GxzWC5-EfRCiE80as_ya0-wwyNsTYbxzY,1721
 kreuzberg/__main__.py,sha256=s2qM1nPEkRHAQP-G3P7sf5l6qA_KJeIEHS5LpPz04lg,183
-kreuzberg/_chunker.py,sha256=QmYbPHPE36ztMT70xPwg_Y4NIftCDl0wyufg5X9lmTo,1932
-kreuzberg/_config.py,sha256=EvrBFAawjfKgXu49tACi4CuMmmoIRt_EzbHayZqM_jU,12983
+kreuzberg/_chunker.py,sha256=y4-dX6ILjjBkkC1gkCzXb7v7vbi8844m7vz1gIzbmv4,1952
+kreuzberg/_config.py,sha256=Yptj-wFXsQHxIdG7xd_HYA7bqjaRLq-1Os4nPQwJtl8,13947
 kreuzberg/_constants.py,sha256=Bxc8oiN-wHwnWXT9bEiJhTUcu1ygPpra5qHirAif3b4,191
-kreuzberg/_document_classification.py,sha256=8XVTKh8ohsb4mbKw2gPFr5OB6v4dWuzXhFE_63vHLrw,5189
-kreuzberg/_entity_extraction.py,sha256=43VxtCPbuSrfi-XGSop-VRoZadJcNe02qRGwjGSaR0c,7862
-kreuzberg/_gmft.py,sha256=JWDeBYWSDkh3tLUhwTmW5CbIBEig2A3o712BLshk7hE,25533
-kreuzberg/_language_detection.py,sha256=eEfj4tsh91SfB2_zQIdY-qD7TlPcppaFm0SqQmETS6Y,3295
+kreuzberg/_document_classification.py,sha256=SErB5Ji6AyEDhgHYPUQr1xcj8FYo0cWOHqb-utRjgJE,6874
+kreuzberg/_entity_extraction.py,sha256=Oa1T-9mptimpOHtcda-GtrVYH9PFy7DSJj3thJZUD7k,7902
+kreuzberg/_gmft.py,sha256=HdQ7Xpuixxl2Y0jY8C3KfyQEU0mN4yQdqErWCv4TnFY,25573
+kreuzberg/_language_detection.py,sha256=_Ng2aHgPxOHFgd507gVNiIGVmnxxbpgYwsO0bD0yTzg,3315
 kreuzberg/_mime_types.py,sha256=2warRVqfBUNIg8JBg8yP4pRqaMPvwINosHMkJwtH_Fc,8488
-kreuzberg/_playa.py,sha256=9z4If0WHxbYQxfb8xT7T96L9Du2Fj3Ar5-rF0OHHiMM,11877
+kreuzberg/_playa.py,sha256=_IPrUSWwSfDQlWXOpKlauV0D9MhGrujGP5kmQ0U3L0g,12188
 kreuzberg/_registry.py,sha256=wGSlkS0U1zqruWQCLE95vj4a2mw1yyvf0j6rgz80sJg,3473
-kreuzberg/_types.py,sha256=Si-Kb58HgE4ckGyZnJFqbWRbCNbdyC_Y0-p75aQP838,15065
-kreuzberg/cli.py,sha256=lmthZa0x8pz7SQfCoPPdTaUF9aQZ8W4w5FlnPcsGr9k,12438
+kreuzberg/_types.py,sha256=OiB-W_V3zg5rHCfVOr4x39TZJIsd5-sE_SgPZzokW9k,15318
+kreuzberg/cli.py,sha256=rJMdHg7FhUxefCrx-sf4c2qVGRXr8Xrpjgfx_DQSKMg,12558
 kreuzberg/exceptions.py,sha256=PTiAZgQwcG9hXbgYg2W7sfxksFhq5_wzOFgZGnTJAoc,2991
 kreuzberg/extraction.py,sha256=Kt1mOxdlOb35yVOdpdhiRPuTgA9BW_TTG9qwCkSxSkc,17332
 kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kreuzberg/_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kreuzberg/_api/main.py,sha256=g3kqXUfSie2pcw3-EWOM4TAoJUqM7yj2e-cBQJ_bmYc,3253
+kreuzberg/_api/main.py,sha256=8VwxRlIXwnPs7ZYm0saUZsNOjevEAWJQpNreG-X7ZpE,3273
 kreuzberg/_extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kreuzberg/_extractors/_base.py,sha256=yNVQSECFad-8_MjqpQZ4q0jQoNdzP6-tqw6l3TfgsMc,4418
-kreuzberg/_extractors/_email.py,sha256=6-Mk1TRXPyy9ylWKCpgdrogyzhiFnJOTuTRld1ghO8I,5695
+kreuzberg/_extractors/_email.py,sha256=Jpr4NFef640uVgNFkR1or-omy8RVt-NOHUYgWRDjyBo,6753
 kreuzberg/_extractors/_html.py,sha256=lOM1Tgrrvd7vpEeFAxC1dp0Tibr6N2FEHCjgFx0FK64,1745
 kreuzberg/_extractors/_image.py,sha256=OmkqR5Cd6bTM_qHEdNHeXa5eK-3KvtLgHX-JE5oZWec,4483
 kreuzberg/_extractors/_pandoc.py,sha256=51k7XISfKaPorhapG7aIeQb94KGsfozxKyT2rwhk9Bk,26553
-kreuzberg/_extractors/_pdf.py,sha256=UlliWggWHuVwwJE-bRa7H9-_cieSa8kdrQP3x_GOxxY,17018
+kreuzberg/_extractors/_pdf.py,sha256=kvbB9Brz7brUpqMZN1G-CRAFgXn5hlWa6eGKXSAe4Xk,19938
 kreuzberg/_extractors/_presentation.py,sha256=CUlqZl_QCdJdumsZh0BpROkFbvi9uq7yMoIt3bRTUeE,10859
 kreuzberg/_extractors/_spread_sheet.py,sha256=iagiyJsnl-89OP1eqmEv8jWl7gZBJm2x0YOyqBgLasA,13733
-kreuzberg/_extractors/_structured.py,sha256=J7op5ZZ663a_CkE2SB5eKjPQPXKHdqPst-GFCJMNGqw,5727
+kreuzberg/_extractors/_structured.py,sha256=PbNaXd-_PUPsE0yZkISod_vLBokbWdVTKEPpEmqaEMM,5787
 kreuzberg/_mcp/__init__.py,sha256=8PYV-omC8Rln7Cove8C3rHu3d7sR1FuiwSBG1O7vkAE,92
-kreuzberg/_mcp/server.py,sha256=Ab0w7kR3m7_L1cfhYHiC8HqDL282vt4uBYwYc9w9E08,8703
+kreuzberg/_mcp/server.py,sha256=aAnkRDvr-tDIw-G9CAThVmvR3qOQOowcnWE6BLJg2jg,8723
 kreuzberg/_ocr/__init__.py,sha256=grshVFwVQl2rMvH1hg1JNlYXjy5-Tdb_rusLD1Cselk,706
-kreuzberg/_ocr/_base.py,sha256=urvsLRgOmVYHjxil_IsSL69FmMnboklC4CHAjdBQLKQ,3893
-kreuzberg/_ocr/_easyocr.py,sha256=pw2uDmULuMQ9T1Gl4axP_ev7-qwjLt1mJHHyZ34P_FI,17178
-kreuzberg/_ocr/_paddleocr.py,sha256=s75aQJILXm1ZbacyZiLPXh6jEAg9tk2NYnwPnfSDrRU,17543
+kreuzberg/_ocr/_base.py,sha256=IkONqwG6zxZoVMni1JlYugBoyONahlRny7J2_7Dy69c,3953
+kreuzberg/_ocr/_easyocr.py,sha256=dWfoj5fPIGqJPGTVeZ0W59TrW3DpNwF0bcfgt6FwQUw,17238
+kreuzberg/_ocr/_paddleocr.py,sha256=Is_iJQaSUeCMfCvg5RnuG_pmBRjBt0b3dCBPY1IAc3A,17583
 kreuzberg/_ocr/_tesseract.py,sha256=teLMH1pBhpcmEXDcyZlv56hYINLGMuaKZ0CQtcu_czQ,31510
 kreuzberg/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kreuzberg/_utils/_cache.py,sha256=hYd_a5Ni5VJBE1XU_eN9gvQ5gg0FRsdbRgmJe-OIJHM,15253
-kreuzberg/_utils/_device.py,sha256=arVrJOSp_2LbbN6lu_rMEUOezzRogdWdkF8d5q5Bg8U,10345
-kreuzberg/_utils/_document_cache.py,sha256=z8irioKsOu8xve1YgHatm__wIFvs9I1gDK3tLNsNyqM,6926
+kreuzberg/_utils/_device.py,sha256=JI9p9TGSfQHEi2SL-ovOXMr9RUnVq-RrEly89OvmQ5w,10485
+kreuzberg/_utils/_document_cache.py,sha256=ka90JIT-FXUMOv8z2u3fztQgZZb2XQDHTMnBi32mySA,7005
 kreuzberg/_utils/_errors.py,sha256=UsktQ_p7eOj9crPsFDg8HgRSE5-IpuFC7y1e6dDI_fY,6503
 kreuzberg/_utils/_pdf_lock.py,sha256=nqxAYCNlfWDrJtP4ZNu57st1YnkDl-gYXdr0q8nv0kA,1961
 kreuzberg/_utils/_process_pool.py,sha256=4BqhmRspwMyPT2EBfTu_rrn7v722wlMLD8qlYvYsc00,8621
@@ -47,8 +47,8 @@ kreuzberg/_utils/_string.py,sha256=bCzO3UO6nXupxvtMWvHqfp1Vd9CTzEH9jmpJXQ7upAU,6
 kreuzberg/_utils/_sync.py,sha256=7LSavBmxVKQUzdjfx9fYRAI9IbJtRw8iGf_Q8B7RX9g,4923
 kreuzberg/_utils/_table.py,sha256=IomrfQBP85DZI8RmQjOVs2Siq7VP9FUTYPaZR4t3yRw,8199
 kreuzberg/_utils/_tmp.py,sha256=hVn-VVijIg2FM7EZJ899gc7wZg-TGoJZoeAcxMX-Cxg,1044
-kreuzberg-3.9.1.dist-info/METADATA,sha256=rBzP4yLvNuodmSrOUNXeYnUZCEPocULKhSjykSlPBeU,11908
-kreuzberg-3.9.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kreuzberg-3.9.1.dist-info/entry_points.txt,sha256=GplGhFryCP7kyAG_k-Mdahznvo2fwi73qLFg5yQfH_A,91
-kreuzberg-3.9.1.dist-info/licenses/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
-kreuzberg-3.9.1.dist-info/RECORD,,
+kreuzberg-3.10.1.dist-info/METADATA,sha256=2idgsCPgaT174AZ7HA8yXZEjPLdiLkEH9-e-IMmvFck,12047
+kreuzberg-3.10.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kreuzberg-3.10.1.dist-info/entry_points.txt,sha256=GplGhFryCP7kyAG_k-Mdahznvo2fwi73qLFg5yQfH_A,91
+kreuzberg-3.10.1.dist-info/licenses/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
+kreuzberg-3.10.1.dist-info/RECORD,,