biblicus 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

biblicus/extractors/rapidocr_text.py

@@ -109,6 +109,7 @@ class RapidOcrExtractor(TextExtractor):
             return ExtractedText(text="", producer_extractor_id=self.extractor_id)
 
         lines: list[str] = []
+        confidences: list[float] = []
         for entry in result:
             if not isinstance(entry, list) or len(entry) < 3:
                 continue
@@ -124,6 +125,12 @@ class RapidOcrExtractor(TextExtractor):
             cleaned = text_value.strip()
             if cleaned:
                 lines.append(cleaned)
+                confidences.append(confidence)
 
         text = parsed_config.joiner.join(lines).strip()
-        return ExtractedText(text=text, producer_extractor_id=self.extractor_id)
+        avg_confidence = sum(confidences) / len(confidences) if confidences else None
+        return ExtractedText(
+            text=text,
+            producer_extractor_id=self.extractor_id,
+            confidence=avg_confidence,
+        )

biblicus/extractors/select_override.py

@@ -0,0 +1,121 @@
+"""
+Simple override selection extractor that always uses the last extraction for matching types.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class SelectOverrideConfig(BaseModel):
+    """
+    Configuration for simple override selection.
+
+    :ivar media_type_patterns: List of media type patterns where override applies.
+    :vartype media_type_patterns: list[str]
+    :ivar fallback_to_first: If True, fall back to first extraction when no override match.
+    :vartype fallback_to_first: bool
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    media_type_patterns: List[str] = Field(default_factory=lambda: ["*/*"])
+    fallback_to_first: bool = Field(default=False)
+
+
+class SelectOverrideExtractor(TextExtractor):
+    """
+    Simple override selector that uses the last extraction for matching media types.
+
+    For items matching the configured patterns, always use the last extraction.
+    For non-matching items, use the first extraction (if fallback_to_first) or last.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "select-override"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate selection extractor configuration.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration.
+        :rtype: SelectOverrideConfig
+        """
+        import json
+
+        # Parse JSON values from CLI string format
+        parsed_config = {}
+        for key, value in config.items():
+            if isinstance(value, str) and value.startswith("["):
+                try:
+                    parsed_config[key] = json.loads(value)
+                except json.JSONDecodeError:
+                    parsed_config[key] = value
+            else:
+                parsed_config[key] = value
+
+        return SelectOverrideConfig.model_validate(parsed_config)
+
+    def extract_text(
+        self,
+        *,
+        corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Select extracted text using simple override logic.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: SelectOverrideConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Selected extracted text payload or None when no prior outputs exist.
+        :rtype: ExtractedText or None
+        """
+        _ = corpus
+        parsed_config = (
+            config
+            if isinstance(config, SelectOverrideConfig)
+            else SelectOverrideConfig.model_validate(config)
+        )
+
+        extracted_candidates = [e for e in previous_extractions if e.text is not None]
+
+        if not extracted_candidates:
+            return None
+
+        matches_pattern = any(
+            fnmatch.fnmatch(item.media_type, pattern)
+            for pattern in parsed_config.media_type_patterns
+        )
+
+        if matches_pattern:
+            candidate = extracted_candidates[-1]
+        elif parsed_config.fallback_to_first:
+            candidate = extracted_candidates[0]
+        else:
+            candidate = extracted_candidates[-1]
+
+        producer = candidate.producer_extractor_id or candidate.extractor_id
+        return ExtractedText(
+            text=candidate.text or "",
+            producer_extractor_id=producer,
+            source_step_index=candidate.step_index,
+            confidence=candidate.confidence,
+        )

biblicus/extractors/select_smart_override.py

@@ -0,0 +1,187 @@
+"""
+Smart override selection extractor that intelligently chooses between extraction results.
+
+This extractor implements the smart override behavior where it compares the most recent
+extraction against previous ones and makes an intelligent choice based on content quality
+and confidence scores.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class SelectSmartOverrideConfig(BaseModel):
+    """
+    Configuration for smart override selection.
+
+    :ivar media_type_patterns: List of media type patterns to consider (e.g., image/*).
+    :vartype media_type_patterns: list[str]
+    :ivar min_confidence_threshold: Minimum confidence to consider an extraction good.
+    :vartype min_confidence_threshold: float
+    :ivar min_text_length: Minimum text length to consider an extraction meaningful.
+    :vartype min_text_length: int
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    media_type_patterns: List[str] = Field(default_factory=lambda: ["*/*"])
+    min_confidence_threshold: float = Field(default=0.7, ge=0.0, le=1.0)
+    min_text_length: int = Field(default=10, ge=0)
+
+
+class SelectSmartOverrideExtractor(TextExtractor):
+    """
+    Smart override selector that intelligently chooses between extraction results.
+
+    This extractor applies smart override logic for items matching the configured media
+    type patterns. The selection rules are:
+
+    1. If the item's media type doesn't match any configured patterns, use last extraction.
+    2. If the last extraction has meaningful content, use it.
+    3. If the last extraction is empty or low-confidence but a previous extraction has
+       good content with confidence, use the previous one.
+    4. Otherwise, use the last extraction.
+
+    Meaningful content is defined as text length >= min_text_length AND (confidence
+    >= min_confidence_threshold OR confidence is not available).
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "select-smart-override"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate selection extractor configuration.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration.
+        :rtype: SelectSmartOverrideConfig
+        """
+        import json
+
+        # Parse JSON values from CLI string format
+        parsed_config = {}
+        for key, value in config.items():
+            if isinstance(value, str) and value.startswith("["):
+                try:
+                    parsed_config[key] = json.loads(value)
+                except json.JSONDecodeError:
+                    parsed_config[key] = value
+            else:
+                parsed_config[key] = value
+
+        return SelectSmartOverrideConfig.model_validate(parsed_config)
+
+    def extract_text(
+        self,
+        *,
+        corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Select extracted text using smart override logic.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: SelectSmartOverrideConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Selected extracted text payload or None when no prior outputs exist.
+        :rtype: ExtractedText or None
+        """
+        _ = corpus
+        parsed_config = (
+            config
+            if isinstance(config, SelectSmartOverrideConfig)
+            else SelectSmartOverrideConfig.model_validate(config)
+        )
+
+        matches_pattern = any(
+            fnmatch.fnmatch(item.media_type, pattern)
+            for pattern in parsed_config.media_type_patterns
+        )
+
+        extracted_candidates = [e for e in previous_extractions if e.text is not None]
+
+        if not extracted_candidates:
+            return None
+
+        if not matches_pattern:
+            return self._extraction_to_result(extracted_candidates[-1])
+
+        last_extraction = extracted_candidates[-1]
+        previous_candidates = extracted_candidates[:-1]
+
+        last_is_meaningful = self._is_meaningful(last_extraction, parsed_config)
+
+        if last_is_meaningful:
+            return self._extraction_to_result(last_extraction)
+
+        best_candidate = None
+        best_confidence = -1.0
+        for prev in previous_candidates:
+            if self._is_meaningful(prev, parsed_config):
+                prev_confidence = prev.confidence if prev.confidence is not None else 0.0
+                if prev_confidence > best_confidence:
+                    best_candidate = prev
+                    best_confidence = prev_confidence
+
+        if best_candidate is not None:
+            return self._extraction_to_result(best_candidate)
+
+        return self._extraction_to_result(last_extraction)
+
+    def _is_meaningful(
+        self, extraction: ExtractionStepOutput, config: SelectSmartOverrideConfig
+    ) -> bool:
+        """
+        Check if an extraction has meaningful content.
+
+        :param extraction: Extraction step output to check.
+        :type extraction: ExtractionStepOutput
+        :param config: Parsed configuration.
+        :type config: SelectSmartOverrideConfig
+        :return: True if the extraction has meaningful content.
+        :rtype: bool
+        """
+        text = (extraction.text or "").strip()
+        if len(text) < config.min_text_length:
+            return False
+
+        confidence = extraction.confidence
+        if confidence is not None and confidence < config.min_confidence_threshold:
+            return False
+
+        return True
+
+    def _extraction_to_result(self, extraction: ExtractionStepOutput) -> ExtractedText:
+        """
+        Convert an ExtractionStepOutput to ExtractedText.
+
+        :param extraction: Extraction step output to convert.
+        :type extraction: ExtractionStepOutput
+        :return: Extracted text result.
+        :rtype: ExtractedText
+        """
+        producer = extraction.producer_extractor_id or extraction.extractor_id
+        return ExtractedText(
+            text=extraction.text or "",
+            producer_extractor_id=producer,
+            source_step_index=extraction.step_index,
+            confidence=extraction.confidence,
+        )

biblicus/inference.py

@@ -0,0 +1,104 @@
+"""
+Inference backend abstraction for machine learning powered components.
+
+This module provides reusable configuration and credential resolution patterns for components
+that can execute locally or via API providers.
+"""
+
+from __future__ import annotations
+
+import os
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class InferenceBackendMode(str, Enum):
+    """Execution mode for inference backends."""
+
+    LOCAL = "local"
+    API = "api"
+
+
+class ApiProvider(str, Enum):
+    """Supported application programming interface providers for inference."""
+
+    HUGGINGFACE = "huggingface"
+    OPENAI = "openai"
+
+
+class InferenceBackendConfig(BaseModel):
+    """
+    Composable configuration for inference backends.
+
+    This config can be embedded in extractor or transformer configurations to provide
+    a uniform interface for local versus application programming interface execution.
+
+    :ivar mode: Execution mode, local or application programming interface.
+    :vartype mode: InferenceBackendMode
+    :ivar api_provider: Application programming interface provider when mode is application programming interface.
+    :vartype api_provider: ApiProvider or None
+    :ivar api_key: Optional per-config application programming interface key override.
+    :vartype api_key: str or None
+    :ivar model_id: Optional model identifier for application programming interface requests.
+    :vartype model_id: str or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    mode: InferenceBackendMode = Field(default=InferenceBackendMode.LOCAL)
+    api_provider: Optional[ApiProvider] = Field(default=None)
+    api_key: Optional[str] = Field(default=None)
+    model_id: Optional[str] = Field(default=None)
+
+    @model_validator(mode="after")
+    def _validate_api_provider_required(self) -> "InferenceBackendConfig":
+        if self.mode == InferenceBackendMode.API and self.api_provider is None:
+            raise ValueError("api_provider is required when mode is 'api'")
+        return self
+
+
+def resolve_api_key(
+    provider: ApiProvider,
+    *,
+    config_override: Optional[str] = None,
+) -> Optional[str]:
+    """
+    Resolve an application programming interface key with precedence rules.
+
+    Precedence order (highest to lowest):
+    1. Explicit config override parameter
+    2. Environment variable for the provider
+    3. User configuration file
+
+    :param provider: Application programming interface provider to resolve key for.
+    :type provider: ApiProvider
+    :param config_override: Optional explicit key from configuration.
+    :type config_override: str or None
+    :return: Resolved application programming interface key or None if unavailable.
+    :rtype: str or None
+    """
+    if config_override is not None:
+        return config_override
+
+    from .user_config import load_user_config
+
+    if provider == ApiProvider.HUGGINGFACE:
+        env_key = os.environ.get("HUGGINGFACE_API_KEY")
+        if env_key:
+            return env_key
+        user_config = load_user_config()
+        if user_config.huggingface is not None:
+            return user_config.huggingface.api_key
+        return None
+    elif provider == ApiProvider.OPENAI:
+        env_key = os.environ.get("OPENAI_API_KEY")
+        if env_key:
+            return env_key
+        user_config = load_user_config()
+        if user_config.openai is not None:
+            return user_config.openai.api_key
+        return None
+    else:
+        return None

biblicus/models.py

@@ -399,6 +399,8 @@ class ExtractedText(BaseModel):
     :vartype producer_extractor_id: str
     :ivar source_step_index: Optional pipeline step index where this text originated.
     :vartype source_step_index: int or None
+    :ivar confidence: Optional confidence score from 0.0 to 1.0.
+    :vartype confidence: float or None
     """
 
     model_config = ConfigDict(extra="forbid")
@@ -406,6 +408,7 @@ class ExtractedText(BaseModel):
     text: str
     producer_extractor_id: str = Field(min_length=1)
     source_step_index: Optional[int] = Field(default=None, ge=1)
+    confidence: Optional[float] = Field(default=None, ge=0.0, le=1.0)
 
 
 class ExtractionStepOutput(BaseModel):
@@ -426,6 +429,8 @@ class ExtractionStepOutput(BaseModel):
     :vartype producer_extractor_id: str or None
     :ivar source_step_index: Optional step index that supplied the text for selection-style extractors.
     :vartype source_step_index: int or None
+    :ivar confidence: Optional confidence score from 0.0 to 1.0.
+    :vartype confidence: float or None
     :ivar error_type: Optional error type name for errored steps.
     :vartype error_type: str or None
     :ivar error_message: Optional error message for errored steps.
@@ -441,5 +446,6 @@ class ExtractionStepOutput(BaseModel):
     text_characters: int = Field(default=0, ge=0)
     producer_extractor_id: Optional[str] = None
     source_step_index: Optional[int] = Field(default=None, ge=1)
+    confidence: Optional[float] = Field(default=None, ge=0.0, le=1.0)
     error_type: Optional[str] = None
     error_message: Optional[str] = None

biblicus/user_config.py

@@ -29,17 +29,49 @@ class OpenAiUserConfig(BaseModel):
     api_key: str = Field(min_length=1)
 
 
+class HuggingFaceUserConfig(BaseModel):
+    """
+    Configuration for HuggingFace integrations.
+
+    :ivar api_key: HuggingFace API key used for authenticated requests.
+    :vartype api_key: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    api_key: str = Field(min_length=1)
+
+
+class DeepgramUserConfig(BaseModel):
+    """
+    Configuration for Deepgram integrations.
+
+    :ivar api_key: Deepgram API key used for authenticated requests.
+    :vartype api_key: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    api_key: str = Field(min_length=1)
+
+
 class BiblicusUserConfig(BaseModel):
     """
     Parsed user configuration for Biblicus.
 
     :ivar openai: Optional OpenAI configuration.
     :vartype openai: OpenAiUserConfig or None
+    :ivar huggingface: Optional HuggingFace configuration.
+    :vartype huggingface: HuggingFaceUserConfig or None
+    :ivar deepgram: Optional Deepgram configuration.
+    :vartype deepgram: DeepgramUserConfig or None
     """
 
     model_config = ConfigDict(extra="forbid")
 
     openai: Optional[OpenAiUserConfig] = None
+    huggingface: Optional[HuggingFaceUserConfig] = None
+    deepgram: Optional[DeepgramUserConfig] = None
 
 
 def default_user_config_paths(
@@ -136,3 +168,47 @@ def resolve_openai_api_key(*, config: Optional[BiblicusUserConfig] = None) -> Op
     if loaded.openai is None:
         return None
     return loaded.openai.api_key
+
+
+def resolve_huggingface_api_key(
+    *, config: Optional[BiblicusUserConfig] = None
+) -> Optional[str]:
+    """
+    Resolve a HuggingFace API key from environment or user configuration.
+
+    Environment takes precedence over configuration.
+
+    :param config: Optional pre-loaded user configuration.
+    :type config: BiblicusUserConfig or None
+    :return: API key string, or None when no key is available.
+    :rtype: str or None
+    """
+    env_key = os.environ.get("HUGGINGFACE_API_KEY")
+    if env_key:
+        return env_key
+    loaded = config or load_user_config()
+    if loaded.huggingface is None:
+        return None
+    return loaded.huggingface.api_key
+
+
+def resolve_deepgram_api_key(
+    *, config: Optional[BiblicusUserConfig] = None
+) -> Optional[str]:
+    """
+    Resolve a Deepgram API key from environment or user configuration.
+
+    Environment takes precedence over configuration.
+
+    :param config: Optional pre-loaded user configuration.
+    :type config: BiblicusUserConfig or None
+    :return: API key string, or None when no key is available.
+    :rtype: str or None
+    """
+    env_key = os.environ.get("DEEPGRAM_API_KEY")
+    if env_key:
+        return env_key
+    loaded = config or load_user_config()
+    if loaded.deepgram is None:
+        return None
+    return loaded.deepgram.api_key