biblicus 0.6.0__py3-none-any.whl

biblicus/extractors/rapidocr_text.py

@@ -0,0 +1,129 @@
+"""
+RapidOCR-backed optical character recognition extractor plugin.
+
+This extractor is an optional dependency. It exists as a practical default for extracting text
+from image items without requiring a separate daemon.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..corpus import Corpus
+from ..errors import ExtractionRunFatalError
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class RapidOcrExtractorConfig(BaseModel):
+    """
+    Configuration for the RapidOCR extractor.
+
+    :ivar min_confidence: Minimum per-line confidence to include in output.
+    :vartype min_confidence: float
+    :ivar joiner: Joiner used to combine recognized lines.
+    :vartype joiner: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    min_confidence: float = Field(default=0.5, ge=0.0, le=1.0)
+    joiner: str = Field(default="\n")
+
+
+class RapidOcrExtractor(TextExtractor):
+    """
+    Extractor plugin that performs optical character recognition on image items using RapidOCR.
+
+    This extractor handles common image media types such as Portable Network Graphics and Joint Photographic Experts Group.
+    It returns an empty extracted text artifact when the image is handled but no text is recognized.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "ocr-rapidocr"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate extractor configuration and ensure prerequisites are available.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration model.
+        :rtype: RapidOcrExtractorConfig
+        :raises ExtractionRunFatalError: If the optional dependency is missing.
+        """
+        try:
+            from rapidocr_onnxruntime import RapidOCR  # noqa: F401
+        except ImportError as import_error:
+            raise ExtractionRunFatalError(
+                "RapidOCR extractor requires an optional dependency. "
+                'Install it with pip install "biblicus[ocr]".'
+            ) from import_error
+
+        return RapidOcrExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus: Corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Extract text from an image item using optical character recognition.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: RapidOcrExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Extracted text payload, or None when the item is not an image.
+        :rtype: ExtractedText or None
+        """
+        _ = previous_extractions
+        media_type = item.media_type
+        if not media_type.startswith("image/"):
+            return None
+
+        parsed_config = (
+            config
+            if isinstance(config, RapidOcrExtractorConfig)
+            else RapidOcrExtractorConfig.model_validate(config)
+        )
+
+        from rapidocr_onnxruntime import RapidOCR
+
+        source_path = corpus.root / item.relpath
+        ocr = RapidOCR()
+        result, _elapsed = ocr(str(source_path))
+
+        if result is None:
+            return ExtractedText(text="", producer_extractor_id=self.extractor_id)
+
+        lines: list[str] = []
+        for entry in result:
+            if not isinstance(entry, list) or len(entry) < 3:
+                continue
+            text_value = entry[1]
+            confidence_value = entry[2]
+            if not isinstance(text_value, str):
+                continue
+            if not isinstance(confidence_value, (int, float)):
+                continue
+            confidence = float(confidence_value)
+            if confidence < parsed_config.min_confidence:
+                continue
+            cleaned = text_value.strip()
+            if cleaned:
+                lines.append(cleaned)
+
+        text = parsed_config.joiner.join(lines).strip()
+        return ExtractedText(text=text, producer_extractor_id=self.extractor_id)

biblicus/extractors/select_longest_text.py

@@ -0,0 +1,105 @@
+"""
+Selection extractor that chooses the longest available text from previous pipeline outputs.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class SelectLongestTextExtractorConfig(BaseModel):
+    """
+    Configuration for the longest text selection extractor.
+
+    Version zero does not expose configuration for this extractor.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SelectLongestTextExtractor(TextExtractor):
+    """
+    Extractor plugin that selects the longest text from previous pipeline outputs.
+
+    This extractor does not attempt to score semantic quality. It is a deterministic
+    selection policy for cases where multiple steps can produce usable text for the
+    same item.
+
+    The selection rules are:
+
+    - If any prior extracted texts are non-empty after stripping whitespace, choose the one
+      with the greatest stripped character count.
+    - Ties are broken by earliest pipeline step index.
+    - If no prior extracted texts are usable but prior extracted texts exist, select the
+      earliest extracted text even if it is empty.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "select-longest-text"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate selection extractor configuration.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration.
+        :rtype: SelectLongestTextExtractorConfig
+        """
+        return SelectLongestTextExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Select the longest extracted text from previous pipeline outputs.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: SelectLongestTextExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Selected extracted text payload or None when no prior outputs exist.
+        :rtype: ExtractedText or None
+        """
+        _ = corpus
+        _ = item
+        _ = config
+
+        extracted_candidates = [entry for entry in previous_extractions if entry.text is not None]
+        if not extracted_candidates:
+            return None
+
+        usable_candidates = [entry for entry in extracted_candidates if entry.text.strip()]
+        if usable_candidates:
+            candidate = max(usable_candidates, key=lambda entry: len(entry.text.strip()))
+            ties = [
+                entry
+                for entry in usable_candidates
+                if len(entry.text.strip()) == len(candidate.text.strip())
+            ]
+            candidate = min(ties, key=lambda entry: int(entry.step_index))
+        else:
+            candidate = min(extracted_candidates, key=lambda entry: int(entry.step_index))
+
+        producer = candidate.producer_extractor_id or candidate.extractor_id
+        return ExtractedText(
+            text=candidate.text or "",
+            producer_extractor_id=producer,
+            source_step_index=candidate.step_index,
+        )

biblicus/extractors/select_text.py

@@ -0,0 +1,100 @@
+"""
+Selection extractor that chooses text from previous pipeline outputs.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class SelectTextExtractorConfig(BaseModel):
+    """
+    Configuration for the selection extractor.
+
+    The selection extractor is intentionally minimal and requires no configuration.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class SelectTextExtractor(TextExtractor):
+    """
+    Extractor plugin that selects from previous pipeline outputs.
+
+    This extractor is used as a final step when you want to make an explicit choice among
+    multiple extraction outputs in the same pipeline.
+
+    It selects the first usable extracted text in pipeline order. Usable means the text is
+    non-empty after stripping whitespace. If no usable text exists but prior extracted text
+    exists, it selects the first extracted text even if it is empty.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "select-text"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate selection extractor configuration.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed configuration.
+        :rtype: SelectTextExtractorConfig
+        """
+        return SelectTextExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Select extracted text from previous pipeline outputs.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: SelectTextExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Selected extracted text payload or None when no prior outputs exist.
+        :rtype: ExtractedText or None
+        """
+        _ = corpus
+        _ = item
+        _ = config
+
+        extracted_candidates = [entry for entry in previous_extractions if entry.text is not None]
+        usable_candidates = [entry for entry in extracted_candidates if entry.text.strip()]
+
+        if usable_candidates:
+            candidate = usable_candidates[0]
+            producer = candidate.producer_extractor_id or candidate.extractor_id
+            return ExtractedText(
+                text=candidate.text or "",
+                producer_extractor_id=producer,
+                source_step_index=candidate.step_index,
+            )
+
+        if extracted_candidates:
+            candidate = extracted_candidates[0]
+            producer = candidate.producer_extractor_id or candidate.extractor_id
+            return ExtractedText(
+                text=candidate.text or "",
+                producer_extractor_id=producer,
+                source_step_index=candidate.step_index,
+            )
+
+        return None

biblicus/extractors/unstructured_text.py

@@ -0,0 +1,100 @@
+"""
+Unstructured-based text extraction plugin.
+
+This extractor is implemented as an optional dependency so the core installation stays small.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+from ..corpus import Corpus
+from ..errors import ExtractionRunFatalError
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
+from .base import TextExtractor
+
+
+class UnstructuredExtractorConfig(BaseModel):
+    """
+    Configuration for the Unstructured extractor.
+
+    Version zero does not expose any configuration for this extractor.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class UnstructuredExtractor(TextExtractor):
+    """
+    Extractor plugin backed by the `unstructured` library.
+
+    The intent is broad format coverage as a last-resort extractor. This extractor skips items
+    that are already text so the pass-through extractor remains the canonical choice for text
+    items and Markdown front matter handling.
+
+    :ivar extractor_id: Extractor identifier.
+    :vartype extractor_id: str
+    """
+
+    extractor_id = "unstructured"
+
+    def validate_config(self, config: Dict[str, Any]) -> BaseModel:
+        """
+        Validate extractor configuration and ensure the dependency is installed.
+
+        :param config: Configuration mapping.
+        :type config: dict[str, Any]
+        :return: Parsed config.
+        :rtype: UnstructuredExtractorConfig
+        :raises ExtractionRunFatalError: If the optional dependency is not installed.
+        """
+        try:
+            from unstructured.partition.auto import partition  # noqa: F401
+        except ImportError as import_error:
+            raise ExtractionRunFatalError(
+                "Unstructured extractor requires an optional dependency. "
+                'Install it with pip install "biblicus[unstructured]".'
+            ) from import_error
+        return UnstructuredExtractorConfig.model_validate(config)
+
+    def extract_text(
+        self,
+        *,
+        corpus: Corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
+        """
+        Extract text for a non-text item using Unstructured.
+
+        :param corpus: Corpus containing the item bytes.
+        :type corpus: Corpus
+        :param item: Catalog item being processed.
+        :type item: CatalogItem
+        :param config: Parsed configuration model.
+        :type config: UnstructuredExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
+        :return: Extracted text payload, or None when the item is already text.
+        :rtype: ExtractedText or None
+        """
+        _ = config
+        _ = previous_extractions
+        media_type = item.media_type
+        if media_type == "text/markdown" or media_type.startswith("text/"):
+            return None
+
+        from unstructured.partition.auto import partition
+
+        source_path = corpus.root / item.relpath
+        elements = partition(filename=str(source_path))
+        lines: list[str] = []
+        for element in elements or []:
+            text = getattr(element, "text", None)
+            if isinstance(text, str) and text.strip():
+                lines.append(text.strip())
+        combined_text = "\n".join(lines).strip()
+        return ExtractedText(text=combined_text, producer_extractor_id=self.extractor_id)

biblicus/frontmatter.py

@@ -0,0 +1,89 @@
+"""
+Markdown front matter helpers.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, Tuple
+
+import yaml
+
+
+@dataclass(frozen=True)
+class FrontMatterDocument:
+    """
+    Parsed front matter and markdown body.
+
+    :ivar metadata: Front matter metadata mapping.
+    :vartype metadata: dict[str, Any]
+    :ivar body: Markdown body text.
+    :vartype body: str
+    """
+
+    metadata: Dict[str, Any]
+    body: str
+
+
+def parse_front_matter(text: str) -> FrontMatterDocument:
+    """
+    Parse Yet Another Markup Language front matter from a Markdown document.
+
+    :param text: Markdown content with optional front matter.
+    :type text: str
+    :return: Parsed front matter and body.
+    :rtype: FrontMatterDocument
+    :raises ValueError: If front matter is present but not a mapping.
+    """
+    if not text.startswith("---\n"):
+        return FrontMatterDocument(metadata={}, body=text)
+
+    front_matter_end = text.find("\n---\n", 4)
+    if front_matter_end == -1:
+        return FrontMatterDocument(metadata={}, body=text)
+
+    raw_yaml = text[4:front_matter_end]
+    body = text[front_matter_end + len("\n---\n") :]
+
+    metadata = yaml.safe_load(raw_yaml) or {}
+    if not isinstance(metadata, dict):
+        raise ValueError("Yet Another Markup Language front matter must be a mapping object")
+
+    return FrontMatterDocument(metadata=dict(metadata), body=body)
+
+
+def render_front_matter(metadata: Dict[str, Any], body: str) -> str:
+    """
+    Render Yet Another Markup Language front matter with a Markdown body.
+
+    :param metadata: Front matter metadata mapping.
+    :type metadata: dict[str, Any]
+    :param body: Markdown body text.
+    :type body: str
+    :return: Markdown with Yet Another Markup Language front matter.
+    :rtype: str
+    """
+    if not metadata:
+        return body
+
+    yaml_text = yaml.safe_dump(
+        metadata,
+        sort_keys=False,
+        allow_unicode=True,
+        default_flow_style=False,
+    ).strip()
+
+    return f"---\n{yaml_text}\n---\n{body}"
+
+
+def split_markdown_front_matter(path_text: str) -> Tuple[Dict[str, Any], str]:
+    """
+    Split Markdown into front matter metadata and body.
+
+    :param path_text: Markdown content.
+    :type path_text: str
+    :return: Metadata mapping and body text.
+    :rtype: tuple[dict[str, Any], str]
+    """
+    parsed_document = parse_front_matter(path_text)
+    return parsed_document.metadata, parsed_document.body

biblicus/hook_logging.py

@@ -0,0 +1,180 @@
+"""
+Structured hook execution logging.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from pathlib import Path
+from typing import Any, Dict, Optional
+from urllib.parse import urlparse, urlunparse
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .hooks import HookPoint
+from .time import utc_now_iso
+
+
+def new_operation_id() -> str:
+    """
+    Create a new operation identifier for hook log grouping.
+
+    :return: Operation identifier.
+    :rtype: str
+    """
+    return str(uuid.uuid4())
+
+
+def redact_source_uri(source_uri: str) -> str:
+    """
+    Redact sensitive components from a source uniform resource identifier.
+
+    :param source_uri: Source uniform resource identifier.
+    :type source_uri: str
+    :return: Redacted source uniform resource identifier.
+    :rtype: str
+    """
+    parsed = urlparse(source_uri)
+
+    if not parsed.scheme:
+        return source_uri
+
+    netloc = parsed.netloc
+    if "@" in netloc:
+        netloc = netloc.split("@", 1)[-1]
+
+    return urlunparse(
+        (
+            parsed.scheme,
+            netloc,
+            parsed.path,
+            parsed.params,
+            parsed.query,
+            parsed.fragment,
+        )
+    )
+
+
+class HookLogEntry(BaseModel):
+    """
+    Single structured log record for hook execution.
+
+    :ivar operation_id: Identifier for the enclosing command or call.
+    :vartype operation_id: str
+    :ivar hook_point: Hook point that executed.
+    :vartype hook_point: HookPoint
+    :ivar hook_id: Hook implementation identifier.
+    :vartype hook_id: str
+    :ivar recorded_at: International Organization for Standardization 8601 timestamp for log record creation.
+    :vartype recorded_at: str
+    :ivar status: Execution status string.
+    :vartype status: str
+    :ivar message: Optional message describing execution results.
+    :vartype message: str or None
+    :ivar item_id: Optional item identifier.
+    :vartype item_id: str or None
+    :ivar relpath: Optional relative path associated with an item.
+    :vartype relpath: str or None
+    :ivar source_uri: Optional redacted source uniform resource identifier.
+    :vartype source_uri: str or None
+    :ivar details: Optional structured details about changes.
+    :vartype details: dict[str, Any]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    operation_id: str
+    hook_point: HookPoint
+    hook_id: str
+    recorded_at: str
+    status: str = Field(min_length=1)
+    message: Optional[str] = None
+    item_id: Optional[str] = None
+    relpath: Optional[str] = None
+    source_uri: Optional[str] = None
+    details: Dict[str, Any] = Field(default_factory=dict)
+
+
+class HookLogger:
+    """
+    Hook logger that writes JSON lines records to a corpus log directory.
+
+    :ivar log_dir: Directory where log files are written.
+    :vartype log_dir: Path
+    :ivar operation_id: Operation identifier for grouping records.
+    :vartype operation_id: str
+    """
+
+    def __init__(self, *, log_dir: Path, operation_id: str):
+        """
+        Initialize a hook logger.
+
+        :param log_dir: Log directory to write into.
+        :type log_dir: Path
+        :param operation_id: Operation identifier for grouping records.
+        :type operation_id: str
+        """
+        self.log_dir = log_dir
+        self.operation_id = operation_id
+
+    @property
+    def path(self) -> Path:
+        """
+        Return the log file path for this operation.
+
+        :return: Log file path.
+        :rtype: Path
+        """
+        return self.log_dir / f"{self.operation_id}.jsonl"
+
+    def record(
+        self,
+        *,
+        hook_point: HookPoint,
+        hook_id: str,
+        status: str,
+        message: Optional[str] = None,
+        item_id: Optional[str] = None,
+        relpath: Optional[str] = None,
+        source_uri: Optional[str] = None,
+        details: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """
+        Append a structured hook log record.
+
+        :param hook_point: Hook point that executed.
+        :type hook_point: HookPoint
+        :param hook_id: Hook identifier.
+        :type hook_id: str
+        :param status: Status string such as ok, denied, or error.
+        :type status: str
+        :param message: Optional message describing results.
+        :type message: str or None
+        :param item_id: Optional item identifier.
+        :type item_id: str or None
+        :param relpath: Optional relative path for the item.
+        :type relpath: str or None
+        :param source_uri: Optional source uniform resource identifier.
+        :type source_uri: str or None
+        :param details: Optional structured details.
+        :type details: dict[str, Any] or None
+        :return: None.
+        :rtype: None
+        """
+        self.log_dir.mkdir(parents=True, exist_ok=True)
+        entry = HookLogEntry(
+            operation_id=self.operation_id,
+            hook_point=hook_point,
+            hook_id=hook_id,
+            recorded_at=utc_now_iso(),
+            status=status,
+            message=message,
+            item_id=item_id,
+            relpath=relpath,
+            source_uri=redact_source_uri(source_uri) if source_uri else None,
+            details=dict(details or {}),
+        )
+        line = json.dumps(entry.model_dump(), sort_keys=False)
+        with self.path.open("a", encoding="utf-8") as handle:
+            handle.write(line + "\n")