biblicus 0.6.0__py3-none-any.whl

biblicus/extractors/metadata_text.py

@@ -0,0 +1,106 @@
+"""
+Metadata-based text extractor plugin.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class MetadataTextExtractorConfig(BaseModel):
+    """
+    Configuration for the metadata text extractor.
+
+    The metadata text extractor is intentionally minimal and deterministic.
+    It emits a plain text representation derived only from an item's catalog metadata.
+
+    :ivar include_title: Whether to include the item title as the first line, if present.
+    :vartype include_title: bool
+    :ivar include_tags: Whether to include a ``tags: ...`` line, if tags are present.
+    :vartype include_tags: bool
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    include_title: bool = Field(default=True)
+    include_tags: bool = Field(default=True)
+
+
+class MetadataTextExtractor(TextExtractor):
+    """
+    Extractor plugin that emits a small, searchable text representation of item metadata.
+
+    The output is intended to be stable and human-readable:
+
+    - If a title exists, the first line is the title.
+    - If tags exist, the next line is ``tags: <comma separated tags>``.
+
+    This extractor is useful for:
+
+    - Retrieval over non-text items that carry meaningful metadata.
+    - Comparing downstream retrieval backends while holding extraction stable.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "metadata-text"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate extractor configuration.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed config.
+        :rtype: MetadataTextExtractorConfig
+        """
+        return MetadataTextExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Extract a metadata-based text payload for the item.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: MetadataTextExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Extracted text payload, or ``None`` if no metadata is available.
+        :rtype: ExtractedText or None
+        """
+        parsed_config = (
+            config
+            if isinstance(config, MetadataTextExtractorConfig)
+            else MetadataTextExtractorConfig.model_validate(config)
+        )
+        _ = corpus
+        _ = previous_extractions
+        lines: list[str] = []
+
+        if parsed_config.include_title and isinstance(item.title, str) and item.title.strip():
+            lines.append(item.title.strip())
+
+        tags = [tag.strip() for tag in item.tags if isinstance(tag, str) and tag.strip()]
+        if parsed_config.include_tags and tags:
+            lines.append(f"tags: {', '.join(tags)}")
+
+        if not lines:
+            return None
+
+        return ExtractedText(text="\n".join(lines), producer_extractor_id=self.extractor_id)

biblicus/extractors/openai_stt.py

@@ -0,0 +1,180 @@
+"""
+OpenAI-backed speech to text extractor plugin.
+
+This extractor is implemented as an optional dependency so the core installation stays small.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from ..corpus import Corpus
+from ..errors import ExtractionRunFatalError
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from ..user_config import resolve_openai_api_key
+from .base import TextExtractor
+
+
+class OpenAiSpeechToTextExtractorConfig(BaseModel):
+    """
+    Configuration for OpenAI speech to text extraction.
+
+    :ivar model: OpenAI transcription model identifier.
+    :vartype model: str
+    :ivar response_format: OpenAI transcription response format.
+    :vartype response_format: str
+    :ivar language: Optional language code hint for transcription.
+    :vartype language: str or None
+    :ivar prompt: Optional prompt text to guide transcription.
+    :vartype prompt: str or None
+    :ivar no_speech_probability_threshold: Optional threshold for suppressing hallucinated transcripts.
+    :vartype no_speech_probability_threshold: float or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    model: str = Field(default="whisper-1", min_length=1)
+    response_format: str = Field(default="json", min_length=1)
+    language: Optional[str] = Field(default=None, min_length=1)
+    prompt: Optional[str] = Field(default=None, min_length=1)
+    no_speech_probability_threshold: Optional[float] = Field(default=None, ge=0.0, le=1.0)
+
+    @model_validator(mode="after")
+    def _validate_no_speech_threshold(self) -> "OpenAiSpeechToTextExtractorConfig":
+        if self.no_speech_probability_threshold is None:
+            return self
+        if self.response_format != "verbose_json":
+            raise ValueError(
+                "no_speech_probability_threshold requires response_format='verbose_json' "
+                "so the transcription API returns per-segment no-speech probabilities"
+            )
+        return self
+
+
+class OpenAiSpeechToTextExtractor(TextExtractor):
+    """
+    Extractor plugin that transcribes audio items using the OpenAI API.
+
+    This extractor is intended as a practical, hosted speech to text implementation.
+    It skips non-audio items.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "stt-openai"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate extractor configuration and ensure prerequisites are available.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration model.
+        :rtype: OpenAiSpeechToTextExtractorConfig
+        :raises ExtractionRunFatalError: If the optional dependency or required environment is missing.
+        """
+        try:
+            from openai import OpenAI  # noqa: F401
+        except ImportError as import_error:
+            raise ExtractionRunFatalError(
+                "OpenAI speech to text extractor requires an optional dependency. "
+                'Install it with pip install "biblicus[openai]".'
+            ) from import_error
+
+        api_key = resolve_openai_api_key()
+        if api_key is None:
+            raise ExtractionRunFatalError(
+                "OpenAI speech to text extractor requires an OpenAI API key. "
+                "Set OPENAI_API_KEY or configure it in ~/.biblicus/config.yml or ./.biblicus/config.yml under "
+                "openai.api_key."
+            )
+
+        return OpenAiSpeechToTextExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus: Corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Transcribe an audio item.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: OpenAiSpeechToTextExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Extracted text payload, or None when the item is not audio.
+        :rtype: ExtractedText or None
+        :raises ExtractionRunFatalError: If the optional dependency or required configuration is missing.
+        """
+        _ = previous_extractions
+        if not item.media_type.startswith("audio/"):
+            return None
+
+        parsed_config = (
+            config
+            if isinstance(config, OpenAiSpeechToTextExtractorConfig)
+            else OpenAiSpeechToTextExtractorConfig.model_validate(config)
+        )
+
+        api_key = resolve_openai_api_key()
+        if api_key is None:
+            raise ExtractionRunFatalError(
+                "OpenAI speech to text extractor requires an OpenAI API key. "
+                "Set OPENAI_API_KEY or configure it in ~/.biblicus/config.yml or ./.biblicus/config.yml under "
+                "openai.api_key."
+            )
+
+        try:
+            from openai import OpenAI
+        except ImportError as import_error:
+            raise ExtractionRunFatalError(
+                "OpenAI speech to text extractor requires an optional dependency. "
+                'Install it with pip install "biblicus[openai]".'
+            ) from import_error
+
+        client = OpenAI(api_key=api_key)
+        source_path = corpus.root / item.relpath
+        with source_path.open("rb") as audio_handle:
+            result = client.audio.transcriptions.create(
+                file=audio_handle,
+                model=parsed_config.model,
+                response_format=parsed_config.response_format,
+                language=parsed_config.language,
+                prompt=parsed_config.prompt,
+            )
+
+        transcript_text: str
+        no_speech_probability_threshold = parsed_config.no_speech_probability_threshold
+
+        if isinstance(result, dict):
+            transcript_text = str(result.get("text") or "")
+            segments = result.get("segments")
+            if (
+                no_speech_probability_threshold is not None
+                and isinstance(segments, list)
+                and segments
+            ):
+                probabilities: list[float] = []
+                for entry in segments:
+                    if not isinstance(entry, dict):
+                        continue
+                    value = entry.get("no_speech_prob", entry.get("no_speech_probability"))
+                    if isinstance(value, (int, float)):
+                        probabilities.append(float(value))
+                if probabilities and max(probabilities) >= no_speech_probability_threshold:
+                    transcript_text = ""
+        else:
+            transcript_text = str(getattr(result, "text", "") or "")
+
+        return ExtractedText(text=transcript_text.strip(), producer_extractor_id=self.extractor_id)

biblicus/extractors/pass_through_text.py

@@ -0,0 +1,84 @@
+"""
+Pass-through extractor for text items.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+from ..corpus import Corpus
+from ..frontmatter import parse_front_matter
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class PassThroughTextExtractorConfig(BaseModel):
+    """
+    Configuration for the pass-through text extractor.
+
+    This extractor is intentionally minimal and requires no configuration.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class PassThroughTextExtractor(TextExtractor):
+    """
+    Extractor plugin that reads text items from the corpus and returns their text content.
+
+    Non-text items are skipped.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "pass-through-text"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate extractor configuration.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed config.
+        :rtype: PassThroughTextExtractorConfig
+        """
+        return PassThroughTextExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus: Corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Extract text by reading the raw item content from the corpus.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: PassThroughTextExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Extracted text payload, or None if the item is not text.
+        :rtype: ExtractedText or None
+        """
+        _ = config
+        _ = previous_extractions
+        media_type = item.media_type
+        if media_type != "text/markdown" and not media_type.startswith("text/"):
+            return None
+        raw_bytes = (corpus.root / item.relpath).read_bytes()
+        if media_type == "text/markdown":
+            markdown_text = raw_bytes.decode("utf-8")
+            parsed_document = parse_front_matter(markdown_text)
+            return ExtractedText(text=parsed_document.body, producer_extractor_id=self.extractor_id)
+        return ExtractedText(
+            text=raw_bytes.decode("utf-8"), producer_extractor_id=self.extractor_id
+        )

biblicus/extractors/pdf_text.py

@@ -0,0 +1,100 @@
+"""
+Portable Document Format text extractor plugin.
+"""
+
+from __future__ import annotations
+
+from io import BytesIO
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+from pypdf import PdfReader
+
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class PortableDocumentFormatTextExtractorConfig(BaseModel):
+    """
+    Configuration for Portable Document Format text extraction.
+
+    :ivar max_pages: Optional maximum number of pages to process.
+    :vartype max_pages: int or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    max_pages: Optional[int] = Field(default=None, ge=1)
+
+
+class PortableDocumentFormatTextExtractor(TextExtractor):
+    """
+    Extractor plugin that attempts to extract text from Portable Document Format items.
+
+    This extractor only handles items whose media type is `application/pdf`.
+    Items of other media types are skipped.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "pdf-text"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate extractor configuration.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration.
+        :rtype: PortableDocumentFormatTextExtractorConfig
+        """
+        return PortableDocumentFormatTextExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Extract text for a Portable Document Format item.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: PortableDocumentFormatTextExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Extracted text payload, or None when the item is not a Portable Document Format item.
+        :rtype: ExtractedText or None
+        """
+        if item.media_type != "application/pdf":
+            return None
+
+        _ = previous_extractions
+        parsed_config = (
+            config
+            if isinstance(config, PortableDocumentFormatTextExtractorConfig)
+            else PortableDocumentFormatTextExtractorConfig.model_validate(config)
+        )
+
+        pdf_path = corpus.root / item.relpath
+        pdf_bytes = pdf_path.read_bytes()
+        reader = PdfReader(BytesIO(pdf_bytes))
+
+        texts: list[str] = []
+        pages = list(reader.pages)
+        if parsed_config.max_pages is not None:
+            pages = pages[: int(parsed_config.max_pages)]
+
+        for page in pages:
+            page_text = page.extract_text() or ""
+            texts.append(page_text)
+
+        combined_text = "\n".join(texts).strip()
+        return ExtractedText(text=combined_text, producer_extractor_id=self.extractor_id)

biblicus/extractors/pipeline.py

@@ -0,0 +1,105 @@
+"""
+Pipeline extractor configuration and validation.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from ..corpus import Corpus
+from ..errors import ExtractionRunFatalError
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class PipelineStepSpec(BaseModel):
+    """
+    Single extractor step within a pipeline.
+
+    :ivar extractor_id: Extractor plugin identifier.
+    :vartype extractor_id: str
+    :ivar config: Extractor configuration mapping.
+    :vartype config: dict[str, Any]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    extractor_id: str = Field(min_length=1)
+    config: Dict[str, Any] = Field(default_factory=dict)
+
+
+class PipelineExtractorConfig(BaseModel):
+    """
+    Configuration for the pipeline extractor.
+
+    :ivar steps: Ordered list of extractor steps to run.
+    :vartype steps: list[PipelineStepSpec]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    steps: List[PipelineStepSpec] = Field(min_length=1)
+
+    @model_validator(mode="after")
+    def _forbid_pipeline_step(self) -> "PipelineExtractorConfig":
+        if any(step.extractor_id == "pipeline" for step in self.steps):
+            raise ValueError("Pipeline steps cannot include the pipeline extractor itself")
+        return self
+
+
+class PipelineExtractor(TextExtractor):
+    """
+    Pipeline extractor configuration shim.
+
+    The pipeline extractor is executed by the extraction engine so it can persist
+    per-step artifacts. This class only validates configuration.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "pipeline"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate pipeline configuration.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration.
+        :rtype: PipelineExtractorConfig
+        """
+        return PipelineExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus: Corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Reject direct execution of the pipeline extractor.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: PipelineExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :raises ExtractionRunFatalError: Always, because the pipeline is executed by the runner.
+        :return: None.
+        :rtype: None
+        """
+        _ = corpus
+        _ = item
+        _ = config
+        _ = previous_extractions
+        raise ExtractionRunFatalError(
+            "Pipeline extractor must be executed by the extraction runner."
+        )