biblicus 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

biblicus/extraction.py

@@ -6,65 +6,20 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
-from typing import Any, Dict, List, Optional
-from uuid import uuid4
+from typing import Any, Dict, List, Optional, Tuple
 
 from pydantic import BaseModel, ConfigDict, Field
 
 from .corpus import Corpus
+from .errors import ExtractionRunFatalError
 from .extractors import get_extractor
-from .models import CatalogItem
+from .extractors.base import TextExtractor
+from .extractors.pipeline import PipelineExtractorConfig, PipelineStepSpec
+from .models import CatalogItem, ExtractionStepOutput
 from .retrieval import hash_text
 from .time import utc_now_iso
 
 
-class ExtractionRunReference(BaseModel):
-    """
-    Reference to an extraction run.
-
-    :ivar extractor_id: Extractor plugin identifier.
-    :vartype extractor_id: str
-    :ivar run_id: Extraction run identifier.
-    :vartype run_id: str
-    """
-
-    model_config = ConfigDict(extra="forbid")
-
-    extractor_id: str = Field(min_length=1)
-    run_id: str = Field(min_length=1)
-
-    def as_string(self) -> str:
-        """
-        Serialize the reference as a single string.
-
-        :return: Reference in the form extractor_id:run_id.
-        :rtype: str
-        """
-
-        return f"{self.extractor_id}:{self.run_id}"
-
-
-def parse_extraction_run_reference(value: str) -> ExtractionRunReference:
-    """
-    Parse an extraction run reference in the form extractor_id:run_id.
-
-    :param value: Raw reference string.
-    :type value: str
-    :return: Parsed extraction run reference.
-    :rtype: ExtractionRunReference
-    :raises ValueError: If the reference is not well formed.
-    """
-
-    if ":" not in value:
-        raise ValueError("Extraction run reference must be extractor_id:run_id")
-    extractor_id, run_id = value.split(":", 1)
-    extractor_id = extractor_id.strip()
-    run_id = run_id.strip()
-    if not extractor_id or not run_id:
-        raise ValueError("Extraction run reference must be extractor_id:run_id with non-empty parts")
-    return ExtractionRunReference(extractor_id=extractor_id, run_id=run_id)
-
-
 class ExtractionRecipeManifest(BaseModel):
     """
     Reproducible configuration for an extraction plugin run.
@@ -90,26 +45,81 @@ class ExtractionRecipeManifest(BaseModel):
     config: Dict[str, Any] = Field(default_factory=dict)
 
 
+class ExtractionStepResult(BaseModel):
+    """
+    Per-item result record for a single pipeline step.
+
+    :ivar step_index: One-based pipeline step index.
+    :vartype step_index: int
+    :ivar extractor_id: Extractor identifier for the step.
+    :vartype extractor_id: str
+    :ivar status: Step status, extracted, skipped, or errored.
+    :vartype status: str
+    :ivar text_relpath: Relative path to the step text artifact, when extracted.
+    :vartype text_relpath: str or None
+    :ivar text_characters: Character count of the extracted text.
+    :vartype text_characters: int
+    :ivar producer_extractor_id: Extractor identifier that produced the text content.
+    :vartype producer_extractor_id: str or None
+    :ivar source_step_index: Optional step index that supplied the text for selection-style extractors.
+    :vartype source_step_index: int or None
+    :ivar error_type: Optional error type name for errored steps.
+    :vartype error_type: str or None
+    :ivar error_message: Optional error message for errored steps.
+    :vartype error_message: str or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    step_index: int = Field(ge=1)
+    extractor_id: str
+    status: str
+    text_relpath: Optional[str] = None
+    text_characters: int = Field(default=0, ge=0)
+    producer_extractor_id: Optional[str] = None
+    source_step_index: Optional[int] = Field(default=None, ge=1)
+    error_type: Optional[str] = None
+    error_message: Optional[str] = None
+
+
 class ExtractionItemResult(BaseModel):
     """
     Per-item result record for an extraction run.
 
     :ivar item_id: Item identifier.
     :vartype item_id: str
-    :ivar status: Result status, extracted or skipped.
+    :ivar status: Final result status, extracted, skipped, or errored.
     :vartype status: str
-    :ivar text_relpath: Relative path to the extracted text artifact, when extracted.
-    :vartype text_relpath: str or None
-    :ivar producer_extractor_id: Extractor identifier that produced the extracted text.
-    :vartype producer_extractor_id: str or None
+    :ivar final_text_relpath: Relative path to the final extracted text artifact, when extracted.
+    :vartype final_text_relpath: str or None
+    :ivar final_step_index: Pipeline step index that produced the final text.
+    :vartype final_step_index: int or None
+    :ivar final_step_extractor_id: Extractor identifier of the step that produced the final text.
+    :vartype final_step_extractor_id: str or None
+    :ivar final_producer_extractor_id: Extractor identifier that produced the final text content.
+    :vartype final_producer_extractor_id: str or None
+    :ivar final_source_step_index: Optional step index that supplied the final text for selection-style extractors.
+    :vartype final_source_step_index: int or None
+    :ivar error_type: Optional error type name when no extracted text was produced.
+    :vartype error_type: str or None
+    :ivar error_message: Optional error message when no extracted text was produced.
+    :vartype error_message: str or None
+    :ivar step_results: Per-step results recorded for this item.
+    :vartype step_results: list[ExtractionStepResult]
     """
 
     model_config = ConfigDict(extra="forbid")
 
     item_id: str
     status: str
-    text_relpath: Optional[str] = None
-    producer_extractor_id: Optional[str] = None
+    final_text_relpath: Optional[str] = None
+    final_step_index: Optional[int] = Field(default=None, ge=1)
+    final_step_extractor_id: Optional[str] = None
+    final_producer_extractor_id: Optional[str] = None
+    final_source_step_index: Optional[int] = Field(default=None, ge=1)
+    error_type: Optional[str] = None
+    error_message: Optional[str] = None
+    step_results: List[ExtractionStepResult] = Field(default_factory=list)
 
 
 class ExtractionRunManifest(BaseModel):
@@ -143,7 +153,9 @@ class ExtractionRunManifest(BaseModel):
     stats: Dict[str, Any] = Field(default_factory=dict)
 
 
-def create_extraction_recipe_manifest(*, extractor_id: str, name: str, config: Dict[str, Any]) -> ExtractionRecipeManifest:
+def create_extraction_recipe_manifest(
+    *, extractor_id: str, name: str, config: Dict[str, Any]
+) -> ExtractionRecipeManifest:
     """
     Create a deterministic extraction recipe manifest.
 
@@ -156,8 +168,9 @@ def create_extraction_recipe_manifest(*, extractor_id: str, name: str, config: D
     :return: Recipe manifest.
     :rtype: ExtractionRecipeManifest
     """
-
-    recipe_payload = json.dumps({"extractor_id": extractor_id, "name": name, "config": config}, sort_keys=True)
+    recipe_payload = json.dumps(
+        {"extractor_id": extractor_id, "name": name, "config": config}, sort_keys=True
+    )
     recipe_id = hash_text(recipe_payload)
     return ExtractionRecipeManifest(
         recipe_id=recipe_id,
@@ -168,7 +181,9 @@ def create_extraction_recipe_manifest(*, extractor_id: str, name: str, config: D
     )
 
 
-def create_extraction_run_manifest(corpus: Corpus, *, recipe: ExtractionRecipeManifest) -> ExtractionRunManifest:
+def create_extraction_run_manifest(
+    corpus: Corpus, *, recipe: ExtractionRecipeManifest
+) -> ExtractionRunManifest:
     """
     Create a new extraction run manifest for a corpus.
 
@@ -179,10 +194,10 @@ def create_extraction_run_manifest(corpus: Corpus, *, recipe: ExtractionRecipeMa
     :return: Run manifest.
     :rtype: ExtractionRunManifest
     """
-
     catalog = corpus.load_catalog()
+    run_id = hash_text(f"{recipe.recipe_id}:{catalog.generated_at}")
     return ExtractionRunManifest(
-        run_id=str(uuid4()),
+        run_id=run_id,
         recipe=recipe,
         corpus_uri=corpus.uri,
         catalog_generated_at=catalog.generated_at,
@@ -203,7 +218,6 @@ def write_extraction_run_manifest(*, run_dir: Path, manifest: ExtractionRunManif
     :return: None.
     :rtype: None
     """
-
     manifest_path = run_dir / "manifest.json"
     manifest_path.write_text(manifest.model_dump_json(indent=2) + "\n", encoding="utf-8")
 
@@ -221,7 +235,6 @@ def write_extracted_text_artifact(*, run_dir: Path, item: CatalogItem, text: str
     :return: Relative path to the stored text artifact.
     :rtype: str
     """
-
     text_dir = run_dir / "text"
     text_dir.mkdir(parents=True, exist_ok=True)
     relpath = str(Path("text") / f"{item.id}.txt")
@@ -230,6 +243,70 @@ def write_extracted_text_artifact(*, run_dir: Path, item: CatalogItem, text: str
     return relpath
 
 
+def _pipeline_step_dir_name(*, step_index: int, extractor_id: str) -> str:
+    """
+    Build a stable directory name for a pipeline step.
+
+    :param step_index: One-based pipeline step index.
+    :type step_index: int
+    :param extractor_id: Extractor identifier for the step.
+    :type extractor_id: str
+    :return: Directory name for the step.
+    :rtype: str
+    """
+    return f"{step_index:02d}-{extractor_id}"
+
+
+def write_pipeline_step_text_artifact(
+    *,
+    run_dir: Path,
+    step_index: int,
+    extractor_id: str,
+    item: CatalogItem,
+    text: str,
+) -> str:
+    """
+    Write a pipeline step text artifact for an item.
+
+    :param run_dir: Extraction run directory.
+    :type run_dir: Path
+    :param step_index: One-based pipeline step index.
+    :type step_index: int
+    :param extractor_id: Extractor identifier for the step.
+    :type extractor_id: str
+    :param item: Catalog item being extracted.
+    :type item: CatalogItem
+    :param text: Extracted text content.
+    :type text: str
+    :return: Relative path to the stored step text artifact.
+    :rtype: str
+    """
+    step_dir_name = _pipeline_step_dir_name(step_index=step_index, extractor_id=extractor_id)
+    text_dir = run_dir / "steps" / step_dir_name / "text"
+    text_dir.mkdir(parents=True, exist_ok=True)
+    relpath = str(Path("steps") / step_dir_name / "text" / f"{item.id}.txt")
+    (run_dir / relpath).write_text(text, encoding="utf-8")
+    return relpath
+
+
+def _final_output_from_steps(
+    step_outputs: List[ExtractionStepOutput],
+) -> Optional[ExtractionStepOutput]:
+    """
+    Select the final pipeline output for an item.
+
+    The final output is the last extracted step output in pipeline order.
+
+    :param step_outputs: Extracted outputs produced by pipeline steps.
+    :type step_outputs: list[biblicus.models.ExtractionStepOutput]
+    :return: Final step output or None when no steps produced extracted text.
+    :rtype: biblicus.models.ExtractionStepOutput or None
+    """
+    if not step_outputs:
+        return None
+    return step_outputs[-1]
+
+
 def build_extraction_run(
     corpus: Corpus,
     *,
@@ -238,11 +315,11 @@ def build_extraction_run(
     config: Dict[str, Any],
 ) -> ExtractionRunManifest:
     """
-    Build an extraction run for a corpus using a named extractor plugin.
+    Build an extraction run for a corpus using the pipeline extractor.
 
     :param corpus: Corpus to extract from.
     :type corpus: Corpus
-    :param extractor_id: Extractor plugin identifier.
+    :param extractor_id: Extractor plugin identifier (must be ``pipeline``).
     :type extractor_id: str
     :param recipe_name: Human-readable recipe name.
     :type recipe_name: str
@@ -253,8 +330,8 @@ def build_extraction_run(
     :raises KeyError: If the extractor identifier is unknown.
     :raises ValueError: If the extractor configuration is invalid.
     :raises OSError: If the run directory or artifacts cannot be written.
+    :raises ExtractionRunFatalError: If the extractor is not the pipeline.
     """
-
     extractor = get_extractor(extractor_id)
     parsed_config = extractor.validate_config(config)
     recipe = create_extraction_recipe_manifest(
@@ -264,17 +341,36 @@ def build_extraction_run(
     )
     manifest = create_extraction_run_manifest(corpus, recipe=recipe)
     run_dir = corpus.extraction_run_dir(extractor_id=extractor_id, run_id=manifest.run_id)
+    if run_dir.exists():
+        return corpus.load_extraction_run_manifest(extractor_id=extractor_id, run_id=manifest.run_id)
     run_dir.mkdir(parents=True, exist_ok=False)
 
     catalog = corpus.load_catalog()
+    if extractor_id != "pipeline":
+        raise ExtractionRunFatalError("Extraction runs must use the pipeline extractor")
+
+    pipeline_config = (
+        parsed_config
+        if isinstance(parsed_config, PipelineExtractorConfig)
+        else PipelineExtractorConfig.model_validate(parsed_config)
+    )
+
+    validated_steps: List[Tuple[PipelineStepSpec, TextExtractor, BaseModel]] = []
+    for step in pipeline_config.steps:
+        step_extractor = get_extractor(step.extractor_id)
+        parsed_step_config = step_extractor.validate_config(step.config)
+        validated_steps.append((step, step_extractor, parsed_step_config))
+
     extracted_items: List[ExtractionItemResult] = []
     extracted_count = 0
     skipped_count = 0
+    errored_count = 0
     extracted_nonempty_count = 0
     extracted_empty_count = 0
     already_text_item_count = 0
     needs_extraction_item_count = 0
     converted_item_count = 0
+
     for item in catalog.items.values():
         media_type = item.media_type
         item_is_text = media_type == "text/markdown" or media_type.startswith("text/")
@@ -283,35 +379,139 @@ def build_extraction_run(
         else:
             needs_extraction_item_count += 1
 
-        extracted_text = extractor.extract_text(corpus=corpus, item=item, config=parsed_config)
-        if extracted_text is None:
-            skipped_count += 1
+        step_results: List[ExtractionStepResult] = []
+        step_outputs: List[ExtractionStepOutput] = []
+        last_error_type: Optional[str] = None
+        last_error_message: Optional[str] = None
+
+        for step_index, (step, step_extractor, parsed_step_config) in enumerate(
+            validated_steps, start=1
+        ):
+            try:
+                extracted_text = step_extractor.extract_text(
+                    corpus=corpus,
+                    item=item,
+                    config=parsed_step_config,
+                    previous_extractions=step_outputs,
+                )
+            except Exception as extraction_error:
+                if isinstance(extraction_error, ExtractionRunFatalError):
+                    raise
+                last_error_type = extraction_error.__class__.__name__
+                last_error_message = str(extraction_error)
+                step_results.append(
+                    ExtractionStepResult(
+                        step_index=step_index,
+                        extractor_id=step.extractor_id,
+                        status="errored",
+                        text_relpath=None,
+                        text_characters=0,
+                        producer_extractor_id=None,
+                        source_step_index=None,
+                        error_type=last_error_type,
+                        error_message=last_error_message,
+                    )
+                )
+                continue
+
+            if extracted_text is None:
+                step_results.append(
+                    ExtractionStepResult(
+                        step_index=step_index,
+                        extractor_id=step.extractor_id,
+                        status="skipped",
+                        text_relpath=None,
+                        text_characters=0,
+                        producer_extractor_id=None,
+                        source_step_index=None,
+                        error_type=None,
+                        error_message=None,
+                    )
+                )
+                continue
+
+            relpath = write_pipeline_step_text_artifact(
+                run_dir=run_dir,
+                step_index=step_index,
+                extractor_id=step.extractor_id,
+                item=item,
+                text=extracted_text.text,
+            )
+            text_characters = len(extracted_text.text)
+            step_results.append(
+                ExtractionStepResult(
+                    step_index=step_index,
+                    extractor_id=step.extractor_id,
+                    status="extracted",
+                    text_relpath=relpath,
+                    text_characters=text_characters,
+                    producer_extractor_id=extracted_text.producer_extractor_id,
+                    source_step_index=extracted_text.source_step_index,
+                    error_type=None,
+                    error_message=None,
+                )
+            )
+            step_outputs.append(
+                ExtractionStepOutput(
+                    step_index=step_index,
+                    extractor_id=step.extractor_id,
+                    status="extracted",
+                    text=extracted_text.text,
+                    text_characters=text_characters,
+                    producer_extractor_id=extracted_text.producer_extractor_id,
+                    source_step_index=extracted_text.source_step_index,
+                    error_type=None,
+                    error_message=None,
+                )
+            )
+
+        final_output = _final_output_from_steps(step_outputs)
+        if final_output is None:
+            status = "errored" if last_error_type else "skipped"
+            if status == "errored":
+                errored_count += 1
+            else:
+                skipped_count += 1
             extracted_items.append(
                 ExtractionItemResult(
                     item_id=item.id,
-                    status="skipped",
-                    text_relpath=None,
-                    producer_extractor_id=None,
+                    status=status,
+                    final_text_relpath=None,
+                    final_step_index=None,
+                    final_step_extractor_id=None,
+                    final_producer_extractor_id=None,
+                    final_source_step_index=None,
+                    error_type=last_error_type if status == "errored" else None,
+                    error_message=last_error_message if status == "errored" else None,
+                    step_results=step_results,
                 )
             )
             continue
 
+        final_text = final_output.text or ""
+        final_text_relpath = write_extracted_text_artifact(
+            run_dir=run_dir, item=item, text=final_text
+        )
         extracted_count += 1
-        stripped_text = extracted_text.text.strip()
-        if stripped_text:
+        if final_text.strip():
             extracted_nonempty_count += 1
             if not item_is_text:
                 converted_item_count += 1
         else:
             extracted_empty_count += 1
 
-        relpath = write_extracted_text_artifact(run_dir=run_dir, item=item, text=extracted_text.text)
         extracted_items.append(
             ExtractionItemResult(
                 item_id=item.id,
                 status="extracted",
-                text_relpath=relpath,
-                producer_extractor_id=extracted_text.producer_extractor_id,
+                final_text_relpath=final_text_relpath,
+                final_step_index=final_output.step_index,
+                final_step_extractor_id=final_output.extractor_id,
+                final_producer_extractor_id=final_output.producer_extractor_id,
+                final_source_step_index=final_output.source_step_index,
+                error_type=None,
+                error_message=None,
+                step_results=step_results,
             )
         )
 
@@ -323,6 +523,7 @@ def build_extraction_run(
         "extracted_nonempty_items": extracted_nonempty_count,
         "extracted_empty_items": extracted_empty_count,
         "skipped_items": skipped_count,
+        "errored_items": errored_count,
         "converted_items": converted_item_count,
     }
     manifest = manifest.model_copy(update={"items": extracted_items, "stats": stats})

biblicus/extractors/__init__.py

@@ -7,9 +7,15 @@ from __future__ import annotations
 from typing import Dict
 
 from .base import TextExtractor
-from .cascade import CascadeExtractor
 from .metadata_text import MetadataTextExtractor
+from .openai_stt import OpenAiSpeechToTextExtractor
 from .pass_through_text import PassThroughTextExtractor
+from .pdf_text import PortableDocumentFormatTextExtractor
+from .pipeline import PipelineExtractor
+from .rapidocr_text import RapidOcrExtractor
+from .select_longest_text import SelectLongestTextExtractor
+from .select_text import SelectTextExtractor
+from .unstructured_text import UnstructuredExtractor
 
 
 def get_extractor(extractor_id: str) -> TextExtractor:
@@ -22,11 +28,16 @@ def get_extractor(extractor_id: str) -> TextExtractor:
     :rtype: TextExtractor
     :raises KeyError: If the extractor identifier is not known.
     """
-
     extractors: Dict[str, TextExtractor] = {
-        CascadeExtractor.extractor_id: CascadeExtractor(),
         MetadataTextExtractor.extractor_id: MetadataTextExtractor(),
         PassThroughTextExtractor.extractor_id: PassThroughTextExtractor(),
+        PipelineExtractor.extractor_id: PipelineExtractor(),
+        PortableDocumentFormatTextExtractor.extractor_id: PortableDocumentFormatTextExtractor(),
+        OpenAiSpeechToTextExtractor.extractor_id: OpenAiSpeechToTextExtractor(),
+        RapidOcrExtractor.extractor_id: RapidOcrExtractor(),
+        SelectTextExtractor.extractor_id: SelectTextExtractor(),
+        SelectLongestTextExtractor.extractor_id: SelectLongestTextExtractor(),
+        UnstructuredExtractor.extractor_id: UnstructuredExtractor(),
     }
     if extractor_id not in extractors:
         raise KeyError(f"Unknown extractor: {extractor_id!r}")

biblicus/extractors/base.py

@@ -5,12 +5,12 @@ Base interfaces for text extraction plugins.
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel
 
 from ..corpus import Corpus
-from ..models import CatalogItem, ExtractedText
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
 
 
 class TextExtractor(ABC):
@@ -38,11 +38,17 @@ class TextExtractor(ABC):
         :rtype: pydantic.BaseModel
         :raises ValueError: If the configuration is invalid.
         """
-
         raise NotImplementedError
 
     @abstractmethod
-    def extract_text(self, *, corpus: Corpus, item: CatalogItem, config: BaseModel) -> Optional[ExtractedText]:
+    def extract_text(
+        self,
+        *,
+        corpus: Corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
         """
         Derive text for a catalog item.
 
@@ -54,8 +60,9 @@ class TextExtractor(ABC):
         :type item: CatalogItem
         :param config: Parsed extractor configuration.
         :type config: pydantic.BaseModel
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
         :return: Extracted text payload or None when skipped.
         :rtype: ExtractedText or None
         """
-
         raise NotImplementedError

biblicus/extractors/metadata_text.py

@@ -4,11 +4,11 @@ Metadata-based text extractor plugin.
 
 from __future__ import annotations
 
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
-from ..models import CatalogItem, ExtractedText
+from ..models import CatalogItem, ExtractedText, ExtractionStepOutput
 from .base import TextExtractor
 
 
@@ -60,10 +60,16 @@ class MetadataTextExtractor(TextExtractor):
         :return: Parsed config.
         :rtype: MetadataTextExtractorConfig
         """
-
         return MetadataTextExtractorConfig.model_validate(config)
 
-    def extract_text(self, *, corpus, item: CatalogItem, config: BaseModel) -> Optional[ExtractedText]:
+    def extract_text(
+        self,
+        *,
+        corpus,
+        item: CatalogItem,
+        config: BaseModel,
+        previous_extractions: List[ExtractionStepOutput],
+    ) -> Optional[ExtractedText]:
         """
         Extract a metadata-based text payload for the item.
 
@@ -73,16 +79,18 @@ class MetadataTextExtractor(TextExtractor):
         :type item: CatalogItem
         :param config: Parsed configuration model.
         :type config: MetadataTextExtractorConfig
+        :param previous_extractions: Prior step outputs for this item within the pipeline.
+        :type previous_extractions: list[biblicus.models.ExtractionStepOutput]
         :return: Extracted text payload, or ``None`` if no metadata is available.
         :rtype: ExtractedText or None
         """
-
         parsed_config = (
             config
             if isinstance(config, MetadataTextExtractorConfig)
             else MetadataTextExtractorConfig.model_validate(config)
         )
         _ = corpus
+        _ = previous_extractions
         lines: list[str] = []
 
         if parsed_config.include_title and isinstance(item.title, str) and item.title.strip():