kreuzberg 3.10.0__py3-none-any.whl → 3.11.0__py3-none-any.whl

kreuzberg/_api/main.py

@@ -30,7 +30,7 @@ try:
         HTTP_422_UNPROCESSABLE_ENTITY,
         HTTP_500_INTERNAL_SERVER_ERROR,
     )
-except ImportError as e:
+except ImportError as e:  # pragma: no cover
     raise MissingDependencyError.create_for_package(
         dependency_group="litestar",
         functionality="Litestar API and docker container",

kreuzberg/_chunker.py

@@ -43,7 +43,7 @@ def get_chunker(
                 from semantic_text_splitter import TextSplitter  # noqa: PLC0415
 
                 _chunkers[key] = TextSplitter(max_characters, overlap_characters)
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="chunking", functionality="chunking", package_name="semantic-text-splitter"
             ) from e

kreuzberg/_config.py

@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any
 
 if sys.version_info >= (3, 11):
     import tomllib
-else:
+else:  # pragma: no cover
     import tomli as tomllib  # type: ignore[import-not-found]
 
 from kreuzberg._gmft import GMFTConfig
@@ -50,7 +50,13 @@ def load_config_from_file(config_path: Path) -> dict[str, Any]:
     # Handle both kreuzberg.toml (root level) and pyproject.toml ([tool.kreuzberg])
     if config_path.name == "kreuzberg.toml":
         return data  # type: ignore[no-any-return]
-    return data.get("tool", {}).get("kreuzberg", {})  # type: ignore[no-any-return]
+
+    # For other files, check if they have [tool.kreuzberg] section
+    if config_path.name == "pyproject.toml" or ("tool" in data and "kreuzberg" in data.get("tool", {})):
+        return data.get("tool", {}).get("kreuzberg", {})  # type: ignore[no-any-return]
+
+    # Otherwise assume root-level configuration
+    return data  # type: ignore[no-any-return]
 
 
 def merge_configs(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
@@ -91,19 +97,21 @@ def parse_ocr_backend_config(
     if not isinstance(backend_config, dict):
         return None
 
-    if backend == "tesseract":
-        # Convert psm integer to PSMMode enum if needed
-        processed_config = backend_config.copy()
-        if "psm" in processed_config and isinstance(processed_config["psm"], int):
-            from kreuzberg._ocr._tesseract import PSMMode  # noqa: PLC0415
-
-            processed_config["psm"] = PSMMode(processed_config["psm"])
-        return TesseractConfig(**processed_config)
-    if backend == "easyocr":
-        return EasyOCRConfig(**backend_config)
-    if backend == "paddleocr":
-        return PaddleOCRConfig(**backend_config)
-    return None
+    match backend:
+        case "tesseract":
+            # Convert psm integer to PSMMode enum if needed
+            processed_config = backend_config.copy()
+            if "psm" in processed_config and isinstance(processed_config["psm"], int):
+                from kreuzberg._ocr._tesseract import PSMMode  # noqa: PLC0415
+
+                processed_config["psm"] = PSMMode(processed_config["psm"])
+            return TesseractConfig(**processed_config)
+        case "easyocr":
+            return EasyOCRConfig(**backend_config)
+        case "paddleocr":
+            return PaddleOCRConfig(**backend_config)
+        case _:
+            return None
 
 
 def build_extraction_config_from_dict(config_dict: dict[str, Any]) -> ExtractionConfig:
@@ -129,12 +137,25 @@ def build_extraction_config_from_dict(config_dict: dict[str, Any]) -> Extraction
         "extract_keywords",
         "auto_detect_language",
         "enable_quality_processing",
+        "auto_detect_document_type",
+        "document_type_confidence_threshold",
+        "document_classification_mode",
+        "keyword_count",
+    }
+    extraction_config = extraction_config | {
+        field: config_dict[field] for field in basic_fields if field in config_dict
     }
-    extraction_config.update({field: config_dict[field] for field in basic_fields if field in config_dict})
 
     # Handle OCR backend configuration
     ocr_backend = extraction_config.get("ocr_backend")
     if ocr_backend and ocr_backend != "none":
+        # Validate OCR backend
+        valid_backends = {"tesseract", "easyocr", "paddleocr"}
+        if ocr_backend not in valid_backends:
+            raise ValidationError(
+                f"Invalid OCR backend: {ocr_backend}. Must be one of: {', '.join(sorted(valid_backends))} or 'none'",
+                context={"provided": ocr_backend, "valid": sorted(valid_backends)},
+            )
         ocr_config = parse_ocr_backend_config(config_dict, ocr_backend)
         if ocr_config:
             extraction_config["ocr_config"] = ocr_config
@@ -286,6 +307,10 @@ _CONFIG_FIELDS = [
     "extract_keywords",
     "auto_detect_language",
     "enable_quality_processing",
+    "auto_detect_document_type",
+    "document_type_confidence_threshold",
+    "document_classification_mode",
+    "keyword_count",
 ]
 
 

kreuzberg/_document_classification.py

@@ -4,13 +4,12 @@ import re
 from typing import TYPE_CHECKING
 
 from kreuzberg._ocr import get_ocr_backend
+from kreuzberg._types import ExtractionConfig, ExtractionResult  # noqa: TC001
 from kreuzberg.exceptions import MissingDependencyError
 
 if TYPE_CHECKING:
     from pathlib import Path
 
-    from kreuzberg._types import ExtractionConfig, ExtractionResult
-
 
 DOCUMENT_CLASSIFIERS = {
     "invoice": [
@@ -52,14 +51,25 @@ def _get_translated_text(result: ExtractionResult) -> str:
     Raises:
         MissingDependencyError: If the deep-translator package is not installed
     """
+    # Combine content with metadata for classification
+    text_to_classify = result.content
+    if result.metadata:
+        # Add metadata values to the text for classification
+        metadata_text = " ".join(str(value) for value in result.metadata.values() if value)
+        text_to_classify = f"{text_to_classify} {metadata_text}"
+
     try:
         from deep_translator import GoogleTranslator  # noqa: PLC0415
-    except ImportError as e:
+    except ImportError as e:  # pragma: no cover
         raise MissingDependencyError(
-            "The 'deep-translator' library is not installed. Please install it with: pip install 'kreuzberg[auto-classify-document-type]'"
+            "The 'deep-translator' library is not installed. Please install it with: pip install 'kreuzberg[document-classification]'"
         ) from e
 
-    return str(GoogleTranslator(source="auto", target="en").translate(result.content).lower())
+    try:
+        return str(GoogleTranslator(source="auto", target="en").translate(text_to_classify).lower())
+    except Exception:  # noqa: BLE001
+        # Fall back to original content in lowercase if translation fails
+        return text_to_classify.lower()
 
 
 def classify_document(result: ExtractionResult, config: ExtractionConfig) -> tuple[str | None, float | None]:
@@ -73,6 +83,9 @@ def classify_document(result: ExtractionResult, config: ExtractionConfig) -> tup
         A tuple containing the detected document type and the confidence score,
         or (None, None) if no type is detected with sufficient confidence.
     """
+    if not config.auto_detect_document_type:
+        return None, None
+
     translated_text = _get_translated_text(result)
     scores = dict.fromkeys(DOCUMENT_CLASSIFIERS, 0)
 
@@ -108,7 +121,8 @@ def classify_document_from_layout(
         A tuple containing the detected document type and the confidence score,
         or (None, None) if no type is detected with sufficient confidence.
     """
-    translated_text = _get_translated_text(result)
+    if not config.auto_detect_document_type:
+        return None, None
 
     if result.layout is None or result.layout.empty:
         return None, None
@@ -117,6 +131,24 @@ def classify_document_from_layout(
     if not all(col in layout_df.columns for col in ["text", "top", "height"]):
         return None, None
 
+    # Use layout text for classification, not the content
+    layout_text = " ".join(layout_df["text"].astype(str).tolist())
+
+    # Translate layout text directly for classification
+    text_to_classify = layout_text
+    if result.metadata:
+        # Add metadata values to the text for classification
+        metadata_text = " ".join(str(value) for value in result.metadata.values() if value)
+        text_to_classify = f"{text_to_classify} {metadata_text}"
+
+    try:
+        from deep_translator import GoogleTranslator  # noqa: PLC0415
+
+        translated_text = str(GoogleTranslator(source="auto", target="en").translate(text_to_classify).lower())
+    except Exception:  # noqa: BLE001
+        # Fall back to original content in lowercase if translation fails
+        translated_text = text_to_classify.lower()
+
     layout_df["translated_text"] = translated_text
 
     page_height = layout_df["top"].max() + layout_df["height"].max()
@@ -151,6 +183,9 @@ def auto_detect_document_type(
     if config.document_classification_mode == "vision" and file_path:
         layout_result = get_ocr_backend("tesseract").process_file_sync(file_path, **config.get_config_dict())
         result.document_type, result.document_type_confidence = classify_document_from_layout(layout_result, config)
+    elif result.layout is not None and not result.layout.empty:
+        # Use layout-based classification if layout data is available
+        result.document_type, result.document_type_confidence = classify_document_from_layout(result, config)
     else:
         result.document_type, result.document_type_confidence = classify_document(result, config)
     return result

kreuzberg/_entity_extraction.py

@@ -139,7 +139,7 @@ def extract_entities(
 
     try:
         import spacy  # noqa: F401, PLC0415
-    except ImportError as e:
+    except ImportError as e:  # pragma: no cover
         raise MissingDependencyError.create_for_package(
             package_name="spacy",
             dependency_group="entity-extraction",
@@ -230,7 +230,7 @@ def extract_keywords(
         return [(kw, float(score)) for kw, score in keywords]
     except (RuntimeError, OSError, ValueError):
         return []
-    except ImportError as e:
+    except ImportError as e:  # pragma: no cover
         raise MissingDependencyError.create_for_package(
             package_name="keybert",
             dependency_group="entity-extraction",

kreuzberg/_extractors/_base.py

@@ -116,8 +116,7 @@ class Extractor(ABC):
         quality_score = calculate_quality_score(cleaned_content, dict(result.metadata) if result.metadata else None)
 
         # Add quality metadata
-        enhanced_metadata = dict(result.metadata) if result.metadata else {}
-        enhanced_metadata["quality_score"] = quality_score
+        enhanced_metadata = (dict(result.metadata) if result.metadata else {}) | {"quality_score": quality_score}
 
         # Return enhanced result
         return ExtractionResult(

kreuzberg/_extractors/_email.py

@@ -19,12 +19,12 @@ if TYPE_CHECKING:
 # Import optional dependencies at module level with proper error handling
 try:
     import mailparse
-except ImportError:
+except ImportError:  # pragma: no cover
     mailparse = None
 
 try:
     import html2text  # type: ignore[import-not-found]
-except ImportError:
+except ImportError:  # pragma: no cover
     html2text = None
 
 # Compile regex pattern once at module level
@@ -59,14 +59,19 @@ class EmailExtractor(Extractor):
 
         to_info = parsed_email.get("to")
         if to_info:
+            # Store the raw value in metadata (could be string, dict, or list)
             if isinstance(to_info, list) and to_info:
+                # For metadata, use first recipient's email if it's a list
                 to_email = to_info[0].get("email", "") if isinstance(to_info[0], dict) else str(to_info[0])
+                metadata["email_to"] = to_email
             elif isinstance(to_info, dict):
-                to_email = to_info.get("email", "")
+                metadata["email_to"] = to_info.get("email", "")
             else:
-                to_email = str(to_info)
-            metadata["email_to"] = to_email
-            text_parts.append(f"To: {to_email}")
+                metadata["email_to"] = str(to_info)
+
+            # For display, format all recipients
+            to_formatted = self._format_email_field(to_info)
+            text_parts.append(f"To: {to_formatted}")
 
         date = parsed_email.get("date")
         if date:
@@ -76,12 +81,30 @@ class EmailExtractor(Extractor):
         cc = parsed_email.get("cc")
         if cc:
             metadata["email_cc"] = cc
-            text_parts.append(f"CC: {cc}")
+            cc_formatted = self._format_email_field(cc)
+            text_parts.append(f"CC: {cc_formatted}")
 
         bcc = parsed_email.get("bcc")
         if bcc:
             metadata["email_bcc"] = bcc
-            text_parts.append(f"BCC: {bcc}")
+            bcc_formatted = self._format_email_field(bcc)
+            text_parts.append(f"BCC: {bcc_formatted}")
+
+    def _format_email_field(self, field: Any) -> str:
+        """Format email field (to, cc, bcc) for display."""
+        if isinstance(field, list):
+            emails = []
+            for item in field:
+                if isinstance(item, dict):
+                    email = item.get("email", "")
+                    if email:
+                        emails.append(email)
+                else:
+                    emails.append(str(item))
+            return ", ".join(emails)
+        if isinstance(field, dict):
+            return str(field.get("email", ""))
+        return str(field)
 
     def _extract_email_body(self, parsed_email: dict[str, Any], text_parts: list[str]) -> None:
         """Extract and process email body content."""

kreuzberg/_extractors/_image.py

@@ -85,23 +85,24 @@ class ImageExtractor(Extractor):
 
         backend = get_ocr_backend(self.config.ocr_backend)
 
-        if self.config.ocr_backend == "tesseract":
-            config = (
-                self.config.ocr_config if isinstance(self.config.ocr_config, TesseractConfig) else TesseractConfig()
-            )
-            result = backend.process_file_sync(path, **asdict(config))
-        elif self.config.ocr_backend == "paddleocr":
-            paddle_config = (
-                self.config.ocr_config if isinstance(self.config.ocr_config, PaddleOCRConfig) else PaddleOCRConfig()
-            )
-            result = backend.process_file_sync(path, **asdict(paddle_config))
-        elif self.config.ocr_backend == "easyocr":
-            easy_config = (
-                self.config.ocr_config if isinstance(self.config.ocr_config, EasyOCRConfig) else EasyOCRConfig()
-            )
-            result = backend.process_file_sync(path, **asdict(easy_config))
-        else:
-            raise NotImplementedError(f"Sync OCR not implemented for {self.config.ocr_backend}")
+        match self.config.ocr_backend:
+            case "tesseract":
+                config = (
+                    self.config.ocr_config if isinstance(self.config.ocr_config, TesseractConfig) else TesseractConfig()
+                )
+                result = backend.process_file_sync(path, **asdict(config))
+            case "paddleocr":
+                paddle_config = (
+                    self.config.ocr_config if isinstance(self.config.ocr_config, PaddleOCRConfig) else PaddleOCRConfig()
+                )
+                result = backend.process_file_sync(path, **asdict(paddle_config))
+            case "easyocr":
+                easy_config = (
+                    self.config.ocr_config if isinstance(self.config.ocr_config, EasyOCRConfig) else EasyOCRConfig()
+                )
+                result = backend.process_file_sync(path, **asdict(easy_config))
+            case _:
+                raise NotImplementedError(f"Sync OCR not implemented for {self.config.ocr_backend}")
         return self._apply_quality_processing(result)
 
     def _get_extension_from_mime_type(self, mime_type: str) -> str:

kreuzberg/_extractors/_pdf.py

@@ -82,20 +82,18 @@ class PDFExtractor(Extractor):
                 from kreuzberg._gmft import extract_tables  # noqa: PLC0415
 
                 result.tables = await extract_tables(path, self.config.gmft_config)
-            except ImportError:
+            except ImportError:  # pragma: no cover
                 result.tables = []
 
             # Enhance metadata with table information
             if result.tables:
                 table_summary = generate_table_summary(result.tables)
-                result.metadata.update(
-                    {
-                        "table_count": table_summary["table_count"],
-                        "tables_summary": f"Document contains {table_summary['table_count']} tables "
-                        f"across {table_summary['pages_with_tables']} pages with "
-                        f"{table_summary['total_rows']} total rows",
-                    }
-                )
+                result.metadata = result.metadata | {
+                    "table_count": table_summary["table_count"],
+                    "tables_summary": f"Document contains {table_summary['table_count']} tables "
+                    f"across {table_summary['pages_with_tables']} pages with "
+                    f"{table_summary['total_rows']} total rows",
+                }
 
         return self._apply_quality_processing(result)
 
@@ -153,14 +151,12 @@ class PDFExtractor(Extractor):
         # Enhance metadata with table information
         if tables:
             table_summary = generate_table_summary(tables)
-            result.metadata.update(
-                {
-                    "table_count": table_summary["table_count"],
-                    "tables_summary": f"Document contains {table_summary['table_count']} tables "
-                    f"across {table_summary['pages_with_tables']} pages with "
-                    f"{table_summary['total_rows']} total rows",
-                }
-            )
+            result.metadata = result.metadata | {
+                "table_count": table_summary["table_count"],
+                "tables_summary": f"Document contains {table_summary['table_count']} tables "
+                f"across {table_summary['pages_with_tables']} pages with "
+                f"{table_summary['total_rows']} total rows",
+            }
 
         # Apply quality processing
         return self._apply_quality_processing(result)
@@ -386,23 +382,24 @@ class PDFExtractor(Extractor):
         backend = get_ocr_backend(self.config.ocr_backend)
         paths = [Path(p) for p in image_paths]
 
-        if self.config.ocr_backend == "tesseract":
-            config = (
-                self.config.ocr_config if isinstance(self.config.ocr_config, TesseractConfig) else TesseractConfig()
-            )
-            results = backend.process_batch_sync(paths, **asdict(config))
-        elif self.config.ocr_backend == "paddleocr":
-            paddle_config = (
-                self.config.ocr_config if isinstance(self.config.ocr_config, PaddleOCRConfig) else PaddleOCRConfig()
-            )
-            results = backend.process_batch_sync(paths, **asdict(paddle_config))
-        elif self.config.ocr_backend == "easyocr":
-            easy_config = (
-                self.config.ocr_config if isinstance(self.config.ocr_config, EasyOCRConfig) else EasyOCRConfig()
-            )
-            results = backend.process_batch_sync(paths, **asdict(easy_config))
-        else:
-            raise NotImplementedError(f"Sync OCR not implemented for {self.config.ocr_backend}")
+        match self.config.ocr_backend:
+            case "tesseract":
+                config = (
+                    self.config.ocr_config if isinstance(self.config.ocr_config, TesseractConfig) else TesseractConfig()
+                )
+                results = backend.process_batch_sync(paths, **asdict(config))
+            case "paddleocr":
+                paddle_config = (
+                    self.config.ocr_config if isinstance(self.config.ocr_config, PaddleOCRConfig) else PaddleOCRConfig()
+                )
+                results = backend.process_batch_sync(paths, **asdict(paddle_config))
+            case "easyocr":
+                easy_config = (
+                    self.config.ocr_config if isinstance(self.config.ocr_config, EasyOCRConfig) else EasyOCRConfig()
+                )
+                results = backend.process_batch_sync(paths, **asdict(easy_config))
+            case _:
+                raise NotImplementedError(f"Sync OCR not implemented for {self.config.ocr_backend}")
 
         # Use list comprehension and join for efficient string building
         return "\n\n".join(result.content for result in results)

kreuzberg/_extractors/_structured.py

@@ -6,15 +6,15 @@ from typing import TYPE_CHECKING, Any, ClassVar
 
 if sys.version_info >= (3, 11):
     import tomllib
-else:
+else:  # pragma: no cover
     try:
         import tomli as tomllib  # type: ignore[import-not-found]
-    except ImportError:
+    except ImportError:  # pragma: no cover
         tomllib = None
 
 try:
     import yaml
-except ImportError:
+except ImportError:  # pragma: no cover
     yaml = None
 
 from anyio import Path as AsyncPath

kreuzberg/_gmft.py

@@ -265,7 +265,7 @@ async def extract_tables(
             finally:
                 await run_sync(doc.close)
 
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="gmft", functionality="table extraction", package_name="gmft"
             ) from e
@@ -379,7 +379,7 @@ def extract_tables_sync(
         finally:
             doc.close()  # type: ignore[no-untyped-call]
 
-    except ImportError as e:
+    except ImportError as e:  # pragma: no cover
         raise MissingDependencyError.create_for_package(
             dependency_group="gmft", functionality="table extraction", package_name="gmft"
         ) from e

kreuzberg/_language_detection.py

@@ -14,7 +14,7 @@ try:
     from fast_langdetect import detect, detect_multilingual
 
     HAS_FAST_LANGDETECT = True
-except ImportError:
+except ImportError:  # pragma: no cover
     HAS_FAST_LANGDETECT = False
     detect = None
     detect_multilingual = None

kreuzberg/_mcp/server.py

@@ -51,7 +51,7 @@ def _create_config_with_overrides(**kwargs: Any) -> ExtractionConfig:
     }
 
     # Override with provided parameters
-    config_dict.update(kwargs)
+    config_dict = config_dict | kwargs
 
     return ExtractionConfig(**config_dict)
 
@@ -268,7 +268,7 @@ def extract_structured(file_path: str) -> list[TextContent]:
     return [TextContent(type="text", text=content)]
 
 
-def main() -> None:
+def main() -> None:  # pragma: no cover
     """Main entry point for the MCP server."""
     mcp.run()
 

kreuzberg/_ocr/_base.py

@@ -88,7 +88,7 @@ class OCRBackend(ABC, Generic[T]):
         Returns:
             List of extraction result objects in the same order as input paths
         """
-        return [self.process_file_sync(path, **kwargs) for path in paths]
+        return [self.process_file_sync(path, **kwargs) for path in paths]  # pragma: no cover
 
     async def process_batch(self, paths: list[Path], **kwargs: Unpack[T]) -> list[ExtractionResult]:
         """Asynchronously process a batch of files and extract their text and metadata.
@@ -106,8 +106,8 @@ class OCRBackend(ABC, Generic[T]):
         from kreuzberg._utils._sync import run_taskgroup  # noqa: PLC0415
 
         tasks = [self.process_file(path, **kwargs) for path in paths]
-        return await run_taskgroup(*tasks)
+        return await run_taskgroup(*tasks)  # pragma: no cover
 
     def __hash__(self) -> int:
         """Hash function for allowing caching."""
-        return hash(type(self).__name__)
+        return hash(type(self).__name__)  # pragma: no cover

kreuzberg/_ocr/_easyocr.py

@@ -321,7 +321,7 @@ class EasyOCRBackend(OCRBackend[EasyOCRConfig]):
             import torch  # noqa: PLC0415
 
             return bool(torch.cuda.is_available())
-        except ImportError:
+        except ImportError:  # pragma: no cover
             return False
 
     @classmethod
@@ -340,7 +340,7 @@ class EasyOCRBackend(OCRBackend[EasyOCRConfig]):
 
         try:
             import easyocr  # noqa: PLC0415
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="easyocr", functionality="EasyOCR as an OCR backend", package_name="easyocr"
             ) from e
@@ -508,7 +508,7 @@ class EasyOCRBackend(OCRBackend[EasyOCRConfig]):
 
         try:
             import easyocr  # noqa: PLC0415
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="easyocr", functionality="EasyOCR as an OCR backend", package_name="easyocr"
             ) from e

kreuzberg/_ocr/_paddleocr.py

@@ -261,7 +261,7 @@ class PaddleBackend(OCRBackend[PaddleOCRConfig]):
 
         try:
             from paddleocr import PaddleOCR  # noqa: PLC0415
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="paddleocr", functionality="PaddleOCR as an OCR backend", package_name="paddleocr"
             ) from e
@@ -428,7 +428,7 @@ class PaddleBackend(OCRBackend[PaddleOCRConfig]):
 
         try:
             from paddleocr import PaddleOCR  # noqa: PLC0415
-        except ImportError as e:
+        except ImportError as e:  # pragma: no cover
             raise MissingDependencyError.create_for_package(
                 dependency_group="paddleocr", functionality="PaddleOCR as an OCR backend", package_name="paddleocr"
             ) from e

kreuzberg/_playa.py

@@ -143,7 +143,9 @@ def _parse_date_string(date_str: str) -> str:
             minute = date_str[10:12]
             second = date_str[12:14]
             time_part = f"T{hour}:{minute}:{second}"
-        return datetime.strptime(f"{year}-{month}-{day}{time_part}", "%Y%m%d%H%M%S").isoformat()  # noqa: DTZ007
+        if time_part:
+            return datetime.strptime(f"{year}-{month}-{day}{time_part}", "%Y-%m-%dT%H:%M:%S").isoformat()  # noqa: DTZ007
+        return datetime.strptime(f"{year}-{month}-{day}", "%Y-%m-%d").isoformat()  # noqa: DTZ007
     return date_str
 
 

kreuzberg/_types.py

@@ -269,7 +269,7 @@ class ExtractionResult:
         Returns:
             List of CSV strings, one per table
         """
-        if not self.tables:
+        if not self.tables:  # pragma: no cover
             return []
 
         return [export_table_to_csv(table) for table in self.tables]
@@ -280,7 +280,7 @@ class ExtractionResult:
         Returns:
             List of TSV strings, one per table
         """
-        if not self.tables:
+        if not self.tables:  # pragma: no cover
             return []
 
         return [export_table_to_tsv(table) for table in self.tables]
@@ -291,7 +291,7 @@ class ExtractionResult:
         Returns:
             List of table structure dictionaries
         """
-        if not self.tables:
+        if not self.tables:  # pragma: no cover
             return []
 
         return [extract_table_structure_info(table) for table in self.tables]
@@ -351,7 +351,7 @@ class ExtractionConfig:
     """Configuration for spaCy entity extraction. If None, uses default settings."""
     auto_detect_document_type: bool = False
     """Whether to automatically detect the document type."""
-    document_type_confidence_threshold: float = 0.7
+    document_type_confidence_threshold: float = 0.5
     """Confidence threshold for document type detection."""
     document_classification_mode: Literal["text", "vision"] = "text"
     """The mode to use for document classification."""
@@ -398,15 +398,16 @@ class ExtractionConfig:
             return asdict(self.ocr_config)
 
         # Lazy load and cache default configs instead of creating new instances
-        if self.ocr_backend == "tesseract":
-            from kreuzberg._ocr._tesseract import TesseractConfig  # noqa: PLC0415
+        match self.ocr_backend:
+            case "tesseract":
+                from kreuzberg._ocr._tesseract import TesseractConfig  # noqa: PLC0415
 
-            return asdict(TesseractConfig())
-        if self.ocr_backend == "easyocr":
-            from kreuzberg._ocr._easyocr import EasyOCRConfig  # noqa: PLC0415
+                return asdict(TesseractConfig())
+            case "easyocr":
+                from kreuzberg._ocr._easyocr import EasyOCRConfig  # noqa: PLC0415
 
-            return asdict(EasyOCRConfig())
-        # paddleocr
-        from kreuzberg._ocr._paddleocr import PaddleOCRConfig  # noqa: PLC0415
+                return asdict(EasyOCRConfig())
+            case _:  # paddleocr or any other backend
+                from kreuzberg._ocr._paddleocr import PaddleOCRConfig  # noqa: PLC0415
 
-        return asdict(PaddleOCRConfig())
+                return asdict(PaddleOCRConfig())

kreuzberg/_utils/_device.py

@@ -144,7 +144,7 @@ def _is_cuda_available() -> bool:
         import torch  # type: ignore[import-not-found,unused-ignore]  # noqa: PLC0415
 
         return bool(torch.cuda.is_available())
-    except ImportError:
+    except ImportError:  # pragma: no cover
         return False
 
 
@@ -154,7 +154,7 @@ def _is_mps_available() -> bool:
         import torch  # type: ignore[import-not-found,unused-ignore]  # noqa: PLC0415
 
         return bool(torch.backends.mps.is_available())
-    except ImportError:
+    except ImportError:  # pragma: no cover
         return False
 
 
@@ -190,7 +190,7 @@ def _get_cuda_devices() -> list[DeviceInfo]:
                 )
             )
 
-    except ImportError:
+    except ImportError:  # pragma: no cover
         pass
 
     return devices
@@ -209,7 +209,7 @@ def _get_mps_device() -> DeviceInfo | None:
             name="Apple Silicon GPU (MPS)",
         )
 
-    except ImportError:
+    except ImportError:  # pragma: no cover
         return None
 
 
@@ -232,7 +232,7 @@ def _get_cuda_memory_info(device_id: int) -> tuple[float | None, float | None]:
 
         return total_memory, available_memory
 
-    except ImportError:
+    except ImportError:  # pragma: no cover
         return None, None
 
 
@@ -333,7 +333,7 @@ def cleanup_device_memory(device: DeviceInfo) -> None:
 
             if torch.cuda.is_available():
                 torch.cuda.empty_cache()
-        except ImportError:
+        except ImportError:  # pragma: no cover  # pragma: no cover
             pass
 
     elif device.device_type == "mps":

kreuzberg/_utils/_document_cache.py

@@ -58,6 +58,7 @@ class DocumentCache:
                 "chunk_content": config.chunk_content,
                 "max_chars": config.max_chars,
                 "max_overlap": config.max_overlap,
+                "auto_detect_document_type": config.auto_detect_document_type,
             }
 
         cache_data = {**file_info, **config_info}

kreuzberg/cli.py

@@ -12,7 +12,7 @@ try:
     import click
     from rich.console import Console
     from rich.progress import Progress, SpinnerColumn, TextColumn
-except ImportError as e:
+except ImportError as e:  # pragma: no cover
     raise ImportError(
         "CLI dependencies are not installed. Please install kreuzberg with the 'cli' extra: pip install kreuzberg[cli]"
     ) from e
@@ -163,7 +163,7 @@ def _perform_extraction(file: Path | None, extraction_config: ExtractionConfig,
                 import magic  # type: ignore[import-not-found]  # noqa: PLC0415
 
                 mime_type = magic.from_buffer(input_bytes, mime=True)
-            except ImportError:
+            except ImportError:  # pragma: no cover
                 content_str = input_bytes.decode("utf-8", errors="ignore").lower()
                 mime_type = "text/html" if "<html" in content_str or "<body" in content_str else "text/plain"
 
@@ -193,7 +193,7 @@ def _write_output(
         click.echo(formatted_output)
 
 
-def handle_error(error: Exception, verbose: bool) -> None:
+def handle_error(error: Exception, verbose: bool) -> None:  # pragma: no cover
     """Handle and display errors.
 
     Args:
@@ -202,19 +202,19 @@ def handle_error(error: Exception, verbose: bool) -> None:
     """
     if isinstance(error, MissingDependencyError):
         console.print(f"[red]Missing dependency:[/red] {error}", style="bold")
-        sys.exit(2)
+        sys.exit(2)  # pragma: no cover
     elif isinstance(error, KreuzbergError):
         console.print(f"[red]Error:[/red] {error}", style="bold")
         if verbose and error.context:
             console.print("\n[dim]Context:[/dim]")
             console.print(json.dumps(error.context, indent=2))
-        sys.exit(1)
+        sys.exit(1)  # pragma: no cover
     else:
         console.print(f"[red]Unexpected error:[/red] {type(error).__name__}: {error}", style="bold")
         if verbose:
             console.print("\n[dim]Traceback:[/dim]")
             traceback.print_exc()
-        sys.exit(1)
+        sys.exit(1)  # pragma: no cover
 
 
 @click.group(invoke_without_command=True)

{kreuzberg-3.10.0.dist-info → kreuzberg-3.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kreuzberg
-Version: 3.10.0
+Version: 3.11.0
 Summary: Document intelligence framework for Python - Extract text, metadata, and structured data from diverse file formats
 Project-URL: documentation, https://kreuzberg.dev
 Project-URL: homepage, https://github.com/Goldziher/kreuzberg
@@ -32,7 +32,7 @@ Requires-Dist: anyio>=4.9.0
 Requires-Dist: chardetng-py>=0.3.5
 Requires-Dist: exceptiongroup>=1.2.2; python_version < '3.11'
 Requires-Dist: html-to-markdown[lxml]>=1.9.0
-Requires-Dist: mcp>=1.12.2
+Requires-Dist: mcp>=1.12.3
 Requires-Dist: msgspec>=0.18.0
 Requires-Dist: playa-pdf>=0.6.4
 Requires-Dist: psutil>=7.0.0
@@ -45,6 +45,7 @@ Requires-Dist: mailparse>=1.0.15; extra == 'additional-extensions'
 Requires-Dist: tomli>=2.0.0; (python_version < '3.11') and extra == 'additional-extensions'
 Provides-Extra: all
 Requires-Dist: click>=8.2.1; extra == 'all'
+Requires-Dist: deep-translator>=1.11.4; extra == 'all'
 Requires-Dist: easyocr>=1.7.2; extra == 'all'
 Requires-Dist: fast-langdetect>=0.3.2; extra == 'all'
 Requires-Dist: gmft>=0.4.2; extra == 'all'
@@ -53,6 +54,7 @@ Requires-Dist: litestar[opentelemetry,standard,structlog]>=2.16.0; extra == 'all
 Requires-Dist: mailparse>=1.0.15; extra == 'all'
 Requires-Dist: paddleocr>=3.1.0; extra == 'all'
 Requires-Dist: paddlepaddle>=3.1.0; extra == 'all'
+Requires-Dist: pandas>=2.3.1; extra == 'all'
 Requires-Dist: playa-pdf[crypto]>=0.6.4; extra == 'all'
 Requires-Dist: rich>=14.1.0; extra == 'all'
 Requires-Dist: semantic-text-splitter>=0.27.0; extra == 'all'
@@ -61,9 +63,6 @@ Requires-Dist: spacy>=3.8.7; extra == 'all'
 Requires-Dist: tomli>=2.0.0; (python_version < '3.11') and extra == 'all'
 Provides-Extra: api
 Requires-Dist: litestar[opentelemetry,standard,structlog]>=2.16.0; extra == 'api'
-Provides-Extra: auto-classify-document-type
-Requires-Dist: deep-translator>=1.11.4; extra == 'auto-classify-document-type'
-Requires-Dist: pandas>=2.3.1; extra == 'auto-classify-document-type'
 Provides-Extra: chunking
 Requires-Dist: semantic-text-splitter>=0.27.0; extra == 'chunking'
 Provides-Extra: cli
@@ -72,6 +71,9 @@ Requires-Dist: rich>=14.1.0; extra == 'cli'
 Requires-Dist: tomli>=2.0.0; (python_version < '3.11') and extra == 'cli'
 Provides-Extra: crypto
 Requires-Dist: playa-pdf[crypto]>=0.6.4; extra == 'crypto'
+Provides-Extra: document-classification
+Requires-Dist: deep-translator>=1.11.4; extra == 'document-classification'
+Requires-Dist: pandas>=2.3.1; extra == 'document-classification'
 Provides-Extra: easyocr
 Requires-Dist: easyocr>=1.7.2; extra == 'easyocr'
 Provides-Extra: entity-extraction

{kreuzberg-3.10.0.dist-info → kreuzberg-3.11.0.dist-info}/RECORD

@@ -1,43 +1,43 @@
 kreuzberg/__init__.py,sha256=0OJ_jNKbS6GxzWC5-EfRCiE80as_ya0-wwyNsTYbxzY,1721
 kreuzberg/__main__.py,sha256=s2qM1nPEkRHAQP-G3P7sf5l6qA_KJeIEHS5LpPz04lg,183
-kreuzberg/_chunker.py,sha256=QmYbPHPE36ztMT70xPwg_Y4NIftCDl0wyufg5X9lmTo,1932
-kreuzberg/_config.py,sha256=EvrBFAawjfKgXu49tACi4CuMmmoIRt_EzbHayZqM_jU,12983
+kreuzberg/_chunker.py,sha256=y4-dX6ILjjBkkC1gkCzXb7v7vbi8844m7vz1gIzbmv4,1952
+kreuzberg/_config.py,sha256=Au521UiR7vcQs_8_hhoWIfmDDMJIrDM3XZUB_qHfCmo,14035
 kreuzberg/_constants.py,sha256=Bxc8oiN-wHwnWXT9bEiJhTUcu1ygPpra5qHirAif3b4,191
-kreuzberg/_document_classification.py,sha256=8XVTKh8ohsb4mbKw2gPFr5OB6v4dWuzXhFE_63vHLrw,5189
-kreuzberg/_entity_extraction.py,sha256=43VxtCPbuSrfi-XGSop-VRoZadJcNe02qRGwjGSaR0c,7862
-kreuzberg/_gmft.py,sha256=JWDeBYWSDkh3tLUhwTmW5CbIBEig2A3o712BLshk7hE,25533
-kreuzberg/_language_detection.py,sha256=eEfj4tsh91SfB2_zQIdY-qD7TlPcppaFm0SqQmETS6Y,3295
+kreuzberg/_document_classification.py,sha256=qFGmwvUMhnNAvNNJO7E-huPx-Ps-_DWxdNxsozIzgaw,6870
+kreuzberg/_entity_extraction.py,sha256=Oa1T-9mptimpOHtcda-GtrVYH9PFy7DSJj3thJZUD7k,7902
+kreuzberg/_gmft.py,sha256=HdQ7Xpuixxl2Y0jY8C3KfyQEU0mN4yQdqErWCv4TnFY,25573
+kreuzberg/_language_detection.py,sha256=_Ng2aHgPxOHFgd507gVNiIGVmnxxbpgYwsO0bD0yTzg,3315
 kreuzberg/_mime_types.py,sha256=2warRVqfBUNIg8JBg8yP4pRqaMPvwINosHMkJwtH_Fc,8488
-kreuzberg/_playa.py,sha256=cJ000ZPHRhbpbP7odRuzMKn38teR6RbodoHgksbfjGE,12059
+kreuzberg/_playa.py,sha256=_IPrUSWwSfDQlWXOpKlauV0D9MhGrujGP5kmQ0U3L0g,12188
 kreuzberg/_registry.py,sha256=wGSlkS0U1zqruWQCLE95vj4a2mw1yyvf0j6rgz80sJg,3473
-kreuzberg/_types.py,sha256=ecT2dRg7dr06p7Dxv23YJ7Ur2m4FUCt6xGtuoS7MQaI,15259
-kreuzberg/cli.py,sha256=lmthZa0x8pz7SQfCoPPdTaUF9aQZ8W4w5FlnPcsGr9k,12438
+kreuzberg/_types.py,sha256=bMaU6VuoqwOpW6ufshA-DWpNw6t9EokjEDEfFsznvdo,15389
+kreuzberg/cli.py,sha256=rJMdHg7FhUxefCrx-sf4c2qVGRXr8Xrpjgfx_DQSKMg,12558
 kreuzberg/exceptions.py,sha256=PTiAZgQwcG9hXbgYg2W7sfxksFhq5_wzOFgZGnTJAoc,2991
 kreuzberg/extraction.py,sha256=Kt1mOxdlOb35yVOdpdhiRPuTgA9BW_TTG9qwCkSxSkc,17332
 kreuzberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kreuzberg/_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kreuzberg/_api/main.py,sha256=g3kqXUfSie2pcw3-EWOM4TAoJUqM7yj2e-cBQJ_bmYc,3253
+kreuzberg/_api/main.py,sha256=8VwxRlIXwnPs7ZYm0saUZsNOjevEAWJQpNreG-X7ZpE,3273
 kreuzberg/_extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kreuzberg/_extractors/_base.py,sha256=yNVQSECFad-8_MjqpQZ4q0jQoNdzP6-tqw6l3TfgsMc,4418
-kreuzberg/_extractors/_email.py,sha256=6-Mk1TRXPyy9ylWKCpgdrogyzhiFnJOTuTRld1ghO8I,5695
+kreuzberg/_extractors/_base.py,sha256=H_nwynBX3fozncVjV13c329x5eCLl5r7nyVTLQyDAzI,4396
+kreuzberg/_extractors/_email.py,sha256=Jpr4NFef640uVgNFkR1or-omy8RVt-NOHUYgWRDjyBo,6753
 kreuzberg/_extractors/_html.py,sha256=lOM1Tgrrvd7vpEeFAxC1dp0Tibr6N2FEHCjgFx0FK64,1745
-kreuzberg/_extractors/_image.py,sha256=OmkqR5Cd6bTM_qHEdNHeXa5eK-3KvtLgHX-JE5oZWec,4483
+kreuzberg/_extractors/_image.py,sha256=Iz1JpvGqcYyh9g4zO_bMZG3E9S39KNHFu8PrXDRXeOk,4513
 kreuzberg/_extractors/_pandoc.py,sha256=51k7XISfKaPorhapG7aIeQb94KGsfozxKyT2rwhk9Bk,26553
-kreuzberg/_extractors/_pdf.py,sha256=pn45qKYkMcmG-PzeeF5jRjrw1NwaKU3589dhpn7HvE8,19918
+kreuzberg/_extractors/_pdf.py,sha256=OflyvwEkuFLmw8E3si35MCGH31fvd5o50VdMmu5QRVs,19884
 kreuzberg/_extractors/_presentation.py,sha256=CUlqZl_QCdJdumsZh0BpROkFbvi9uq7yMoIt3bRTUeE,10859
 kreuzberg/_extractors/_spread_sheet.py,sha256=iagiyJsnl-89OP1eqmEv8jWl7gZBJm2x0YOyqBgLasA,13733
-kreuzberg/_extractors/_structured.py,sha256=J7op5ZZ663a_CkE2SB5eKjPQPXKHdqPst-GFCJMNGqw,5727
+kreuzberg/_extractors/_structured.py,sha256=PbNaXd-_PUPsE0yZkISod_vLBokbWdVTKEPpEmqaEMM,5787
 kreuzberg/_mcp/__init__.py,sha256=8PYV-omC8Rln7Cove8C3rHu3d7sR1FuiwSBG1O7vkAE,92
-kreuzberg/_mcp/server.py,sha256=Ab0w7kR3m7_L1cfhYHiC8HqDL282vt4uBYwYc9w9E08,8703
+kreuzberg/_mcp/server.py,sha256=Dxed80MqZsYCFyYo0QdArpKE4H8DhpKY34fijdzV5uw,8731
 kreuzberg/_ocr/__init__.py,sha256=grshVFwVQl2rMvH1hg1JNlYXjy5-Tdb_rusLD1Cselk,706
-kreuzberg/_ocr/_base.py,sha256=urvsLRgOmVYHjxil_IsSL69FmMnboklC4CHAjdBQLKQ,3893
-kreuzberg/_ocr/_easyocr.py,sha256=pw2uDmULuMQ9T1Gl4axP_ev7-qwjLt1mJHHyZ34P_FI,17178
-kreuzberg/_ocr/_paddleocr.py,sha256=s75aQJILXm1ZbacyZiLPXh6jEAg9tk2NYnwPnfSDrRU,17543
+kreuzberg/_ocr/_base.py,sha256=IkONqwG6zxZoVMni1JlYugBoyONahlRny7J2_7Dy69c,3953
+kreuzberg/_ocr/_easyocr.py,sha256=dWfoj5fPIGqJPGTVeZ0W59TrW3DpNwF0bcfgt6FwQUw,17238
+kreuzberg/_ocr/_paddleocr.py,sha256=Is_iJQaSUeCMfCvg5RnuG_pmBRjBt0b3dCBPY1IAc3A,17583
 kreuzberg/_ocr/_tesseract.py,sha256=teLMH1pBhpcmEXDcyZlv56hYINLGMuaKZ0CQtcu_czQ,31510
 kreuzberg/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kreuzberg/_utils/_cache.py,sha256=hYd_a5Ni5VJBE1XU_eN9gvQ5gg0FRsdbRgmJe-OIJHM,15253
-kreuzberg/_utils/_device.py,sha256=arVrJOSp_2LbbN6lu_rMEUOezzRogdWdkF8d5q5Bg8U,10345
-kreuzberg/_utils/_document_cache.py,sha256=z8irioKsOu8xve1YgHatm__wIFvs9I1gDK3tLNsNyqM,6926
+kreuzberg/_utils/_device.py,sha256=JI9p9TGSfQHEi2SL-ovOXMr9RUnVq-RrEly89OvmQ5w,10485
+kreuzberg/_utils/_document_cache.py,sha256=ka90JIT-FXUMOv8z2u3fztQgZZb2XQDHTMnBi32mySA,7005
 kreuzberg/_utils/_errors.py,sha256=UsktQ_p7eOj9crPsFDg8HgRSE5-IpuFC7y1e6dDI_fY,6503
 kreuzberg/_utils/_pdf_lock.py,sha256=nqxAYCNlfWDrJtP4ZNu57st1YnkDl-gYXdr0q8nv0kA,1961
 kreuzberg/_utils/_process_pool.py,sha256=4BqhmRspwMyPT2EBfTu_rrn7v722wlMLD8qlYvYsc00,8621
@@ -47,8 +47,8 @@ kreuzberg/_utils/_string.py,sha256=bCzO3UO6nXupxvtMWvHqfp1Vd9CTzEH9jmpJXQ7upAU,6
 kreuzberg/_utils/_sync.py,sha256=7LSavBmxVKQUzdjfx9fYRAI9IbJtRw8iGf_Q8B7RX9g,4923
 kreuzberg/_utils/_table.py,sha256=IomrfQBP85DZI8RmQjOVs2Siq7VP9FUTYPaZR4t3yRw,8199
 kreuzberg/_utils/_tmp.py,sha256=hVn-VVijIg2FM7EZJ899gc7wZg-TGoJZoeAcxMX-Cxg,1044
-kreuzberg-3.10.0.dist-info/METADATA,sha256=4U1mSEAbT3zRir--SPZmYy09LfEfu5vUz6CUhQL8uzA,12047
-kreuzberg-3.10.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kreuzberg-3.10.0.dist-info/entry_points.txt,sha256=GplGhFryCP7kyAG_k-Mdahznvo2fwi73qLFg5yQfH_A,91
-kreuzberg-3.10.0.dist-info/licenses/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
-kreuzberg-3.10.0.dist-info/RECORD,,
+kreuzberg-3.11.0.dist-info/METADATA,sha256=pvyRM3TAmXE3TnYaNOZ1chD_IQTgWn254wxnqDsy6EM,12135
+kreuzberg-3.11.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kreuzberg-3.11.0.dist-info/entry_points.txt,sha256=GplGhFryCP7kyAG_k-Mdahznvo2fwi73qLFg5yQfH_A,91
+kreuzberg-3.11.0.dist-info/licenses/LICENSE,sha256=-8caMvpCK8SgZ5LlRKhGCMtYDEXqTKH9X8pFEhl91_4,1066
+kreuzberg-3.11.0.dist-info/RECORD,,