biblicus 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

biblicus/extractors/openai_stt.py

@@ -0,0 +1,180 @@
+"""
+OpenAI-backed speech to text extractor plugin.
+
+This extractor is implemented as an optional dependency so the core installation stays small.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from ..corpus import Corpus
+from ..errors import ExtractionRunFatalError
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from ..user_config import resolve_openai_api_key
+from .base import TextExtractor
+
+
+class OpenAiSpeechToTextExtractorConfig(BaseModel):
+    """
+    Configuration for OpenAI speech to text extraction.
+
+    :ivar model: OpenAI transcription model identifier.
+    :vartype model: str
+    :ivar response_format: OpenAI transcription response format.
+    :vartype response_format: str
+    :ivar language: Optional language code hint for transcription.
+    :vartype language: str or None
+    :ivar prompt: Optional prompt text to guide transcription.
+    :vartype prompt: str or None
+    :ivar no_speech_probability_threshold: Optional threshold for suppressing hallucinated transcripts.
+    :vartype no_speech_probability_threshold: float or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    model: str = Field(default="whisper-1", min_length=1)
+    response_format: str = Field(default="json", min_length=1)
+    language: Optional[str] = Field(default=None, min_length=1)
+    prompt: Optional[str] = Field(default=None, min_length=1)
+    no_speech_probability_threshold: Optional[float] = Field(default=None, ge=0.0, le=1.0)
+
+    @model_validator(mode="after")
+    def _validate_no_speech_threshold(self) -> "OpenAiSpeechToTextExtractorConfig":
+        if self.no_speech_probability_threshold is None:
+            return self
+        if self.response_format != "verbose_json":
+            raise ValueError(
+                "no_speech_probability_threshold requires response_format='verbose_json' "
+                "so the transcription API returns per-segment no-speech probabilities"
+            )
+        return self
+
+
+class OpenAiSpeechToTextExtractor(TextExtractor):
+    """
+    Extractor plugin that transcribes audio items using the OpenAI API.
+
+    This extractor is intended as a practical, hosted speech to text implementation.
+    It skips non-audio items.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "stt-openai"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate extractor configuration and ensure prerequisites are available.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration model.
+        :rtype: OpenAiSpeechToTextExtractorConfig
+        :raises ExtractionRunFatalError: If the optional dependency or required environment is missing.
+        """
+        try:
+            from openai import OpenAI  # noqa: F401
+        except ImportError as import_error:
+            raise ExtractionRunFatalError(
+                "OpenAI speech to text extractor requires an optional dependency. "
+                'Install it with pip install "biblicus[openai]".'
+            ) from import_error
+
+        api_key = resolve_openai_api_key()
+        if api_key is None:
+            raise ExtractionRunFatalError(
+                "OpenAI speech to text extractor requires an OpenAI API key. "
+                "Set OPENAI_API_KEY or configure it in ~/.biblicus/config.yml or ./.biblicus/config.yml under "
+                "openai.api_key."
+            )
+
+        return OpenAiSpeechToTextExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus: Corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Transcribe an audio item.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: OpenAiSpeechToTextExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Extracted text payload, or None when the item is not audio.
+        :rtype: ExtractedText or None
+        :raises ExtractionRunFatalError: If the optional dependency or required configuration is missing.
+        """
+        _ = previous_extractions
+        if not item.media_type.startswith("audio/"):
+            return None
+
+        parsed_config = (
+            config
+            if isinstance(config, OpenAiSpeechToTextExtractorConfig)
+            else OpenAiSpeechToTextExtractorConfig.model_validate(config)
+        )
+
+        api_key = resolve_openai_api_key()
+        if api_key is None:
+            raise ExtractionRunFatalError(
+                "OpenAI speech to text extractor requires an OpenAI API key. "
+                "Set OPENAI_API_KEY or configure it in ~/.biblicus/config.yml or ./.biblicus/config.yml under "
+                "openai.api_key."
+            )
+
+        try:
+            from openai import OpenAI
+        except ImportError as import_error:
+            raise ExtractionRunFatalError(
+                "OpenAI speech to text extractor requires an optional dependency. "
+                'Install it with pip install "biblicus[openai]".'
+            ) from import_error
+
+        client = OpenAI(api_key=api_key)
+        source_path = corpus.root / item.relpath
+        with source_path.open("rb") as audio_handle:
+            result = client.audio.transcriptions.create(
+                file=audio_handle,
+                model=parsed_config.model,
+                response_format=parsed_config.response_format,
+                language=parsed_config.language,
+                prompt=parsed_config.prompt,
+            )
+
+        transcript_text: str
+        no_speech_probability_threshold = parsed_config.no_speech_probability_threshold
+
+        if isinstance(result, dict):
+            transcript_text = str(result.get("text") or "")
+            segments = result.get("segments")
+            if (
+                no_speech_probability_threshold is not None
+                and isinstance(segments, list)
+                and segments
+            ):
+                probabilities: list[float] = []
+                for entry in segments:
+                    if not isinstance(entry, dict):
+                        continue
+                    value = entry.get("no_speech_prob", entry.get("no_speech_probability"))
+                    if isinstance(value, (int, float)):
+                        probabilities.append(float(value))
+                if probabilities and max(probabilities) >= no_speech_probability_threshold:
+                    transcript_text = ""
+        else:
+            transcript_text = str(getattr(result, "text", "") or "")
+
+        return ExtractedText(text=transcript_text.strip(), producer_extractor_id=self.extractor_id)

biblicus/extractors/pass_through_text.py

@@ -4,13 +4,13 @@ Pass-through extractor for text items.
 
 from __future__ import annotations
 
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel, ConfigDict
 
 from ..corpus import Corpus
 from ..frontmatter import parse_front_matter
-from ..models import CatalogItem, ExtractedText
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
 from .base import TextExtractor
 
 
@@ -45,10 +45,16 @@ class PassThroughTextExtractor(TextExtractor):
         :return: Parsed config.
         :rtype: PassThroughTextExtractorConfig
         """
-
         return PassThroughTextExtractorConfig.model_validate(config)
 
-    def extract_text(self, *, corpus: Corpus, item: CatalogItem, config: BaseModel) -> Optional[ExtractedText]:
+    def extract_text(
+        self,
+        *,
+        corpus: Corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
         """
         Extract text by reading the raw item content from the corpus.
 
@@ -58,11 +64,13 @@ class PassThroughTextExtractor(TextExtractor):
         :type item: CatalogItem
         :param config: Parsed configuration model.
         :type config: PassThroughTextExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
         :return: Extracted text payload, or None if the item is not text.
         :rtype: ExtractedText or None
         """
-
         _ = config
+        _ = previous_extractions
         media_type = item.media_type
         if media_type != "text/markdown" and not media_type.startswith("text/"):
             return None
@@ -71,4 +79,6 @@ class PassThroughTextExtractor(TextExtractor):
             markdown_text = raw_bytes.decode("utf-8")
             parsed_document = parse_front_matter(markdown_text)
             return ExtractedText(text=parsed_document.body, producer_extractor_id=self.extractor_id)
-        return ExtractedText(text=raw_bytes.decode("utf-8"), producer_extractor_id=self.extractor_id)
+        return ExtractedText(
+            text=raw_bytes.decode("utf-8"), producer_extractor_id=self.extractor_id
+        )

biblicus/extractors/pdf_text.py

@@ -0,0 +1,100 @@
+"""
+Portable Document Format text extractor plugin.
+"""
+
+from __future__ import annotations
+
+from io import BytesIO
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+from pypdf import PdfReader
+
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class PortableDocumentFormatTextExtractorConfig(BaseModel):
+    """
+    Configuration for Portable Document Format text extraction.
+
+    :ivar max_pages: Optional maximum number of pages to process.
+    :vartype max_pages: int or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    max_pages: Optional[int] = Field(default=None, ge=1)
+
+
+class PortableDocumentFormatTextExtractor(TextExtractor):
+    """
+    Extractor plugin that attempts to extract text from Portable Document Format items.
+
+    This extractor only handles items whose media type is `application/pdf`.
+    Items of other media types are skipped.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "pdf-text"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate extractor configuration.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration.
+        :rtype: PortableDocumentFormatTextExtractorConfig
+        """
+        return PortableDocumentFormatTextExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Extract text for a Portable Document Format item.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: PortableDocumentFormatTextExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Extracted text payload, or None when the item is not a Portable Document Format item.
+        :rtype: ExtractedText or None
+        """
+        if item.media_type != "application/pdf":
+            return None
+
+        _ = previous_extractions
+        parsed_config = (
+            config
+            if isinstance(config, PortableDocumentFormatTextExtractorConfig)
+            else PortableDocumentFormatTextExtractorConfig.model_validate(config)
+        )
+
+        pdf_path = corpus.root / item.relpath
+        pdf_bytes = pdf_path.read_bytes()
+        reader = PdfReader(BytesIO(pdf_bytes))
+
+        texts: list[str] = []
+        pages = list(reader.pages)
+        if parsed_config.max_pages is not None:
+            pages = pages[: int(parsed_config.max_pages)]
+
+        for page in pages:
+            page_text = page.extract_text() or ""
+            texts.append(page_text)
+
+        combined_text = "\n".join(texts).strip()
+        return ExtractedText(text=combined_text, producer_extractor_id=self.extractor_id)

biblicus/extractors/pipeline.py

@@ -0,0 +1,105 @@
+"""
+Pipeline extractor configuration and validation.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from ..corpus import Corpus
+from ..errors import ExtractionRunFatalError
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class PipelineStepSpec(BaseModel):
+    """
+    Single extractor step within a pipeline.
+
+    :ivar extractor_id: Extractor plugin identifier.
+    :vartype extractor_id: str
+    :ivar config: Extractor configuration mapping.
+    :vartype config: dict[str, Any]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    extractor_id: str = Field(min_length=1)
+    config: Dict[str, Any] = Field(default_factory=dict)
+
+
+class PipelineExtractorConfig(BaseModel):
+    """
+    Configuration for the pipeline extractor.
+
+    :ivar steps: Ordered list of extractor steps to run.
+    :vartype steps: list[PipelineStepSpec]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    steps: List[PipelineStepSpec] = Field(min_length=1)
+
+    @model_validator(mode="after")
+    def _forbid_pipeline_step(self) -> "PipelineExtractorConfig":
+        if any(step.extractor_id == "pipeline" for step in self.steps):
+            raise ValueError("Pipeline steps cannot include the pipeline extractor itself")
+        return self
+
+
+class PipelineExtractor(TextExtractor):
+    """
+    Pipeline extractor configuration shim.
+
+    The pipeline extractor is executed by the extraction engine so it can persist
+    per-step artifacts. This class only validates configuration.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "pipeline"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate pipeline configuration.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration.
+        :rtype: PipelineExtractorConfig
+        """
+        return PipelineExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus: Corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Reject direct execution of the pipeline extractor.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: PipelineExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :raises ExtractionRunFatalError: Always, because the pipeline is executed by the runner.
+        :return: None.
+        :rtype: None
+        """
+        _ = corpus
+        _ = item
+        _ = config
+        _ = previous_extractions
+        raise ExtractionRunFatalError(
+            "Pipeline extractor must be executed by the extraction runner."
+        )

biblicus/extractors/rapidocr_text.py

@@ -0,0 +1,129 @@
+"""
+RapidOCR-backed optical character recognition extractor plugin.
+
+This extractor is an optional dependency. It exists as a practical default for extracting text
+from image items without requiring a separate daemon.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..corpus import Corpus
+from ..errors import ExtractionRunFatalError
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class RapidOcrExtractorConfig(BaseModel):
+    """
+    Configuration for the RapidOCR extractor.
+
+    :ivar min_confidence: Minimum per-line confidence to include in output.
+    :vartype min_confidence: float
+    :ivar joiner: Joiner used to combine recognized lines.
+    :vartype joiner: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    min_confidence: float = Field(default=0.5, ge=0.0, le=1.0)
+    joiner: str = Field(default="\n")
+
+
+class RapidOcrExtractor(TextExtractor):
+    """
+    Extractor plugin that performs optical character recognition on image items using RapidOCR.
+
+    This extractor handles common image media types such as Portable Network Graphics and Joint Photographic Experts Group.
+    It returns an empty extracted text artifact when the image is handled but no text is recognized.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "ocr-rapidocr"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate extractor configuration and ensure prerequisites are available.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration model.
+        :rtype: RapidOcrExtractorConfig
+        :raises ExtractionRunFatalError: If the optional dependency is missing.
+        """
+        try:
+            from rapidocr_onnxruntime import RapidOCR  # noqa: F401
+        except ImportError as import_error:
+            raise ExtractionRunFatalError(
+                "RapidOCR extractor requires an optional dependency. "
+                'Install it with pip install "biblicus[ocr]".'
+            ) from import_error
+
+        return RapidOcrExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus: Corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Extract text from an image item using optical character recognition.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: RapidOcrExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Extracted text payload, or None when the item is not an image.
+        :rtype: ExtractedText or None
+        """
+        _ = previous_extractions
+        media_type = item.media_type
+        if not media_type.startswith("image/"):
+            return None
+
+        parsed_config = (
+            config
+            if isinstance(config, RapidOcrExtractorConfig)
+            else RapidOcrExtractorConfig.model_validate(config)
+        )
+
+        from rapidocr_onnxruntime import RapidOCR
+
+        source_path = corpus.root / item.relpath
+        ocr = RapidOCR()
+        result, _elapsed = ocr(str(source_path))
+
+        if result is None:
+            return ExtractedText(text="", producer_extractor_id=self.extractor_id)
+
+        lines: list[str] = []
+        for entry in result:
+            if not isinstance(entry, list) or len(entry) < 3:
+                continue
+            text_value = entry[1]
+            confidence_value = entry[2]
+            if not isinstance(text_value, str):
+                continue
+            if not isinstance(confidence_value, (int, float)):
+                continue
+            confidence = float(confidence_value)
+            if confidence < parsed_config.min_confidence:
+                continue
+            cleaned = text_value.strip()
+            if cleaned:
+                lines.append(cleaned)
+
+        text = parsed_config.joiner.join(lines).strip()
+        return ExtractedText(text=text, producer_extractor_id=self.extractor_id)

biblicus/extractors/select_longest_text.py

@@ -0,0 +1,105 @@
+"""
+Selection extractor that chooses the longest available text from previous pipeline outputs.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class SelectLongestTextExtractorConfig(BaseModel):
+    """
+    Configuration for the longest text selection extractor.
+
+    Version zero does not expose configuration for this extractor.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SelectLongestTextExtractor(TextExtractor):
+    """
+    Extractor plugin that selects the longest text from previous pipeline outputs.
+
+    This extractor does not attempt to score semantic quality. It is a deterministic
+    selection policy for cases where multiple steps can produce usable text for the
+    same item.
+
+    The selection rules are:
+
+    - If any prior extracted texts are non-empty after stripping whitespace, choose the one
+      with the greatest stripped character count.
+    - Ties are broken by earliest pipeline step index.
+    - If no prior extracted texts are usable but prior extracted texts exist, select the
+      earliest extracted text even if it is empty.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "select-longest-text"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate selection extractor configuration.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration.
+        :rtype: SelectLongestTextExtractorConfig
+        """
+        return SelectLongestTextExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Select the longest extracted text from previous pipeline outputs.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: SelectLongestTextExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Selected extracted text payload or None when no prior outputs exist.
+        :rtype: ExtractedText or None
+        """
+        _ = corpus
+        _ = item
+        _ = config
+
+        extracted_candidates = [entry for entry in previous_extractions if entry.text is not None]
+        if not extracted_candidates:
+            return None
+
+        usable_candidates = [entry for entry in extracted_candidates if entry.text.strip()]
+        if usable_candidates:
+            candidate = max(usable_candidates, key=lambda entry: len(entry.text.strip()))
+            ties = [
+                entry
+                for entry in usable_candidates
+                if len(entry.text.strip()) == len(candidate.text.strip())
+            ]
+            candidate = min(ties, key=lambda entry: int(entry.step_index))
+        else:
+            candidate = min(extracted_candidates, key=lambda entry: int(entry.step_index))
+
+        producer = candidate.producer_extractor_id or candidate.extractor_id
+        return ExtractedText(
+            text=candidate.text or "",
+            producer_extractor_id=producer,
+            source_step_index=candidate.step_index,
+        )