document-extraction-tools 0.0.1rc1__py3-none-any.whl

document_extraction_tools/config/config_loader.py

@@ -0,0 +1,201 @@
+"""Configuration Loader."""
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from document_extraction_tools.config.base_converter_config import BaseConverterConfig
+from document_extraction_tools.config.base_evaluation_exporter_config import (
+    BaseEvaluationExporterConfig,
+)
+from document_extraction_tools.config.base_evaluator_config import BaseEvaluatorConfig
+from document_extraction_tools.config.base_extraction_exporter_config import (
+    BaseExtractionExporterConfig,
+)
+from document_extraction_tools.config.base_extractor_config import BaseExtractorConfig
+from document_extraction_tools.config.base_file_lister_config import (
+    BaseFileListerConfig,
+)
+from document_extraction_tools.config.base_reader_config import BaseReaderConfig
+from document_extraction_tools.config.base_test_data_loader_config import (
+    BaseTestDataLoaderConfig,
+)
+from document_extraction_tools.config.evaluation_orchestrator_config import (
+    EvaluationOrchestratorConfig,
+)
+from document_extraction_tools.config.evaluation_pipeline_config import (
+    EvaluationPipelineConfig,
+)
+from document_extraction_tools.config.extraction_orchestrator_config import (
+    ExtractionOrchestratorConfig,
+)
+from document_extraction_tools.config.extraction_pipeline_config import (
+    ExtractionPipelineConfig,
+)
+
+
+def _load_yaml(path: Path) -> dict[str, Any]:
+    """Helper to load a YAML file into a dictionary.
+
+    Args:
+        path (Path): Path to the .yaml file.
+
+    Returns:
+        dict[str, Any]: The parsed YAML data. Returns an empty dict if the file
+        does not exist or is empty.
+
+    Raises:
+        FileNotFoundError: If the file does not exist.
+    """
+    if not path.exists():
+        raise FileNotFoundError(f"Config file not found: {path.absolute()}")
+
+    with open(path) as f:
+        return yaml.safe_load(f) or {}
+
+
+def load_config(
+    lister_config_cls: type[BaseFileListerConfig],
+    reader_config_cls: type[BaseReaderConfig],
+    converter_config_cls: type[BaseConverterConfig],
+    extractor_config_cls: type[BaseExtractorConfig],
+    exporter_config_cls: type[BaseExtractionExporterConfig],
+    orchestrator_config_cls: type[
+        ExtractionOrchestratorConfig
+    ] = ExtractionOrchestratorConfig,
+    config_dir: Path = Path("config/yaml"),
+) -> ExtractionPipelineConfig:
+    """Loads configuration based on a mapping file.
+
+    Args:
+        lister_config_cls (type[BaseFileListerConfig]): The FileListerConfig subclass to use.
+        reader_config_cls (type[BaseReaderConfig]): The ReaderConfig subclass to use.
+        converter_config_cls (type[BaseConverterConfig]): The ConverterConfig subclass to use.
+        extractor_config_cls (type[BaseExtractorConfig]): The ExtractorConfig subclass to use.
+        exporter_config_cls (type[BaseExtractionExporterConfig]): The ExporterConfig subclass to use.
+        orchestrator_config_cls (type[ExtractionOrchestratorConfig]): The ExtractionOrchestratorConfig class to use.
+        config_dir (Path): Directory containing the configs.
+
+    Returns:
+        ExtractionPipelineConfig: The fully validated configuration.
+
+    Raises:
+        FileNotFoundError: If the config directory or mapping file is missing.
+    """
+    if not config_dir.exists():
+        raise FileNotFoundError(f"Config directory not found: {config_dir.absolute()}")
+
+    return ExtractionPipelineConfig(
+        orchestrator=orchestrator_config_cls(
+            **_load_yaml(config_dir / orchestrator_config_cls.filename)
+        ),
+        file_lister=lister_config_cls(
+            **_load_yaml(config_dir / lister_config_cls.filename)
+        ),
+        reader=reader_config_cls(**_load_yaml(config_dir / reader_config_cls.filename)),
+        converter=converter_config_cls(
+            **_load_yaml(config_dir / converter_config_cls.filename)
+        ),
+        extractor=extractor_config_cls(
+            **_load_yaml(config_dir / extractor_config_cls.filename)
+        ),
+        exporter=exporter_config_cls(
+            **_load_yaml(config_dir / exporter_config_cls.filename)
+        ),
+    )
+
+
+def load_evaluation_config(
+    test_data_loader_config_cls: type[BaseTestDataLoaderConfig],
+    evaluator_config_classes: list[type[BaseEvaluatorConfig]],
+    reader_config_cls: type[BaseReaderConfig],
+    converter_config_cls: type[BaseConverterConfig],
+    extractor_config_cls: type[BaseExtractorConfig],
+    evaluation_exporter_config_cls: type[BaseEvaluationExporterConfig],
+    orchestrator_config_cls: type[
+        EvaluationOrchestratorConfig
+    ] = EvaluationOrchestratorConfig,
+    config_dir: Path = Path("config/yaml"),
+) -> EvaluationPipelineConfig:
+    """Loads evaluation configuration based on default filenames.
+
+    Args:
+        test_data_loader_config_cls (type[BaseTestDataLoaderConfig]): The TestDataLoaderConfig subclass to use.
+        evaluator_config_classes (list[type[BaseEvaluatorConfig]]): EvaluatorConfig
+            subclasses to load using the top-level keys in evaluator.yaml.
+        reader_config_cls (type[BaseReaderConfig]): The ReaderConfig subclass to use.
+        converter_config_cls (type[BaseConverterConfig]): The ConverterConfig subclass to use.
+        extractor_config_cls (type[BaseExtractorConfig]): The ExtractorConfig subclass to use.
+        evaluation_exporter_config_cls (type[BaseEvaluationExporterConfig]): The EvaluationExporterConfig
+            subclass to use.
+        orchestrator_config_cls (type[EvaluationOrchestratorConfig]): The EvaluationOrchestratorConfig class to use.
+        config_dir (Path): Directory containing the configs.
+
+    Returns:
+        EvaluationPipelineConfig: The fully validated configuration.
+
+    Raises:
+        FileNotFoundError: If the config directory or mapping file is missing.
+    """
+    if not config_dir.exists():
+        raise FileNotFoundError(f"Config directory not found: {config_dir.absolute()}")
+
+    return EvaluationPipelineConfig(
+        orchestrator=orchestrator_config_cls(
+            **_load_yaml(config_dir / orchestrator_config_cls.filename)
+        ),
+        test_data_loader=test_data_loader_config_cls(
+            **_load_yaml(config_dir / test_data_loader_config_cls.filename)
+        ),
+        evaluators=_load_evaluator_configs(config_dir, evaluator_config_classes),
+        reader=reader_config_cls(**_load_yaml(config_dir / reader_config_cls.filename)),
+        converter=converter_config_cls(
+            **_load_yaml(config_dir / converter_config_cls.filename)
+        ),
+        extractor=extractor_config_cls(
+            **_load_yaml(config_dir / extractor_config_cls.filename)
+        ),
+        evaluation_exporter=evaluation_exporter_config_cls(
+            **_load_yaml(config_dir / evaluation_exporter_config_cls.filename)
+        ),
+    )
+
+
+def _load_evaluator_configs(
+    config_dir: Path, evaluator_config_classes: list[type[BaseEvaluatorConfig]]
+) -> list[BaseEvaluatorConfig]:
+    """Helper to load multiple evaluator configs from evaluator.yaml.
+
+    Args:
+        config_dir (Path): Directory containing the configs.
+        evaluator_config_classes (list[type[BaseEvaluatorConfig]]): EvaluatorConfig
+            subclasses keyed by their class names.
+
+    Returns:
+        list[BaseEvaluatorConfig]: The loaded evaluator configurations.
+    """
+    evaluator_lookup = {cls.__name__: cls for cls in evaluator_config_classes}
+    evaluator_yaml = _load_yaml(config_dir / BaseEvaluatorConfig.filename)
+    if not evaluator_yaml:
+        raise ValueError("No evaluator configuration found in evaluator.yaml.")
+    if not isinstance(evaluator_yaml, dict):
+        raise ValueError(
+            "Expected evaluator.yaml to contain a mapping of config class names."
+        )
+
+    evaluators: list[BaseEvaluatorConfig] = []
+    for evaluator_key, evaluator_data in evaluator_yaml.items():
+        evaluator_cls = evaluator_lookup.get(evaluator_key)
+        if evaluator_cls is None:
+            raise ValueError(
+                f"Unknown evaluator config class '{evaluator_key}' in evaluator.yaml."
+            )
+        if evaluator_data is None:
+            evaluator_data = {}
+        if not isinstance(evaluator_data, dict):
+            raise ValueError(
+                f"Expected evaluator data for '{evaluator_key}' to be a mapping."
+            )
+        evaluators.append(evaluator_cls(**evaluator_data))
+    return evaluators

document_extraction_tools/config/evaluation_orchestrator_config.py

@@ -0,0 +1,20 @@
+"""Configuration for the Evaluation Orchestrator component."""
+
+from typing import ClassVar
+
+from pydantic import BaseModel, Field
+
+
+class EvaluationOrchestratorConfig(BaseModel):
+    """Configuration for the Evaluation Orchestrator."""
+
+    filename: ClassVar[str] = "evaluation_orchestrator.yaml"
+
+    max_workers: int = Field(
+        default=4,
+        description="Number of processes to use for CPU-bound tasks.",
+    )
+    max_concurrency: int = Field(
+        default=10,
+        description="Maximum number of concurrent I/O requests allowed.",
+    )

document_extraction_tools/config/evaluation_pipeline_config.py

@@ -0,0 +1,32 @@
+"""Master Evaluation Pipeline Configuration."""
+
+from pydantic import BaseModel
+
+from document_extraction_tools.config.base_converter_config import BaseConverterConfig
+from document_extraction_tools.config.base_evaluation_exporter_config import (
+    BaseEvaluationExporterConfig,
+)
+from document_extraction_tools.config.base_evaluator_config import BaseEvaluatorConfig
+from document_extraction_tools.config.base_extractor_config import BaseExtractorConfig
+from document_extraction_tools.config.base_reader_config import BaseReaderConfig
+from document_extraction_tools.config.base_test_data_loader_config import (
+    BaseTestDataLoaderConfig,
+)
+from document_extraction_tools.config.evaluation_orchestrator_config import (
+    EvaluationOrchestratorConfig,
+)
+
+
+class EvaluationPipelineConfig(BaseModel):
+    """Master container for evaluation pipeline component configurations.
+
+    This class aggregates the configurations for all evaluation pipeline components.
+    """
+
+    orchestrator: EvaluationOrchestratorConfig
+    test_data_loader: BaseTestDataLoaderConfig
+    evaluators: list[BaseEvaluatorConfig]
+    reader: BaseReaderConfig
+    converter: BaseConverterConfig
+    extractor: BaseExtractorConfig
+    evaluation_exporter: BaseEvaluationExporterConfig

document_extraction_tools/config/extraction_orchestrator_config.py

@@ -0,0 +1,20 @@
+"""Configuration for the Extraction Orchestrator component."""
+
+from typing import ClassVar
+
+from pydantic import BaseModel, Field
+
+
+class ExtractionOrchestratorConfig(BaseModel):
+    """Configuration for the Pipeline Orchestrator."""
+
+    filename: ClassVar[str] = "extraction_orchestrator.yaml"
+
+    max_workers: int = Field(
+        default=4,
+        description="Number of processes to use for CPU-bound tasks.",
+    )
+    max_concurrency: int = Field(
+        default=10,
+        description="Maximum number of concurrent I/O requests allowed.",
+    )

document_extraction_tools/config/extraction_pipeline_config.py

@@ -0,0 +1,30 @@
+"""Master Extraction Pipeline Configuration."""
+
+from pydantic import BaseModel
+
+from document_extraction_tools.config.base_converter_config import BaseConverterConfig
+from document_extraction_tools.config.base_extraction_exporter_config import (
+    BaseExtractionExporterConfig,
+)
+from document_extraction_tools.config.base_extractor_config import BaseExtractorConfig
+from document_extraction_tools.config.base_file_lister_config import (
+    BaseFileListerConfig,
+)
+from document_extraction_tools.config.base_reader_config import BaseReaderConfig
+from document_extraction_tools.config.extraction_orchestrator_config import (
+    ExtractionOrchestratorConfig,
+)
+
+
+class ExtractionPipelineConfig(BaseModel):
+    """Master container for extraction pipeline component configurations.
+
+    This class aggregates the configurations for all pipeline components.
+    """
+
+    orchestrator: ExtractionOrchestratorConfig
+    file_lister: BaseFileListerConfig
+    reader: BaseReaderConfig
+    converter: BaseConverterConfig
+    extractor: BaseExtractorConfig
+    exporter: BaseExtractionExporterConfig

document_extraction_tools/runners/__init__.py

@@ -0,0 +1,10 @@
+"""Pipeline orchestrators."""
+
+from document_extraction_tools.runners.evaluation.evaluation_orchestrator import (
+    EvaluationOrchestrator,
+)
+from document_extraction_tools.runners.extraction.extraction_orchestrator import (
+    ExtractionOrchestrator,
+)
+
+__all__ = ["EvaluationOrchestrator", "ExtractionOrchestrator"]

document_extraction_tools/runners/evaluation/evaluation_orchestrator.py

@@ -0,0 +1,260 @@
+"""Evaluation orchestrator.
+
+This module defines the EvaluationOrchestrator class, which coordinates
+the evaluation of extraction models against ground-truth data using multiple
+evaluators. It handles loading evaluation examples, reading and converting documents,
+running extraction, applying evaluators, and exporting results.
+"""
+
+import asyncio
+import contextvars
+import logging
+from collections.abc import Callable, Iterable
+from concurrent.futures import ThreadPoolExecutor
+from typing import Generic, TypeVar
+
+from document_extraction_tools.base.converter.base_converter import BaseConverter
+from document_extraction_tools.base.evaluator.base_evaluator import BaseEvaluator
+from document_extraction_tools.base.exporter.base_evaluation_exporter import (
+    BaseEvaluationExporter,
+)
+from document_extraction_tools.base.extractor.base_extractor import BaseExtractor
+from document_extraction_tools.base.reader.base_reader import BaseReader
+from document_extraction_tools.base.test_data_loader.base_test_data_loader import (
+    BaseTestDataLoader,
+)
+from document_extraction_tools.config.evaluation_orchestrator_config import (
+    EvaluationOrchestratorConfig,
+)
+from document_extraction_tools.config.evaluation_pipeline_config import (
+    EvaluationPipelineConfig,
+)
+from document_extraction_tools.types.document import Document
+from document_extraction_tools.types.document_bytes import DocumentBytes
+from document_extraction_tools.types.evaluation_example import EvaluationExample
+from document_extraction_tools.types.evaluation_result import EvaluationResult
+from document_extraction_tools.types.path_identifier import PathIdentifier
+from document_extraction_tools.types.schema import ExtractionSchema
+
+logger = logging.getLogger(__name__)
+T = TypeVar("T")
+
+
+class EvaluationOrchestrator(Generic[ExtractionSchema]):
+    """Coordinates evaluation across multiple evaluators."""
+
+    def __init__(
+        self,
+        config: EvaluationOrchestratorConfig,
+        test_data_loader: BaseTestDataLoader[ExtractionSchema],
+        reader: BaseReader,
+        converter: BaseConverter,
+        extractor: BaseExtractor,
+        evaluators: Iterable[BaseEvaluator[ExtractionSchema]],
+        exporter: BaseEvaluationExporter,
+        schema: type[ExtractionSchema],
+    ) -> None:
+        """Initialize the evaluation orchestrator with pipeline components.
+
+        Args:
+            config (EvaluationOrchestratorConfig): Configuration for evaluation orchestration.
+            test_data_loader (BaseTestDataLoader[ExtractionSchema]): Component to load evaluation examples.
+            reader (BaseReader): Component to read raw file bytes.
+            converter (BaseConverter): Component to transform bytes into Document objects.
+            extractor (BaseExtractor): Component to generate predictions.
+            evaluators (Iterable[BaseEvaluator[ExtractionSchema]]): Metrics to apply to each example.
+            exporter (BaseEvaluationExporter): Component to persist evaluation results.
+            schema (type[ExtractionSchema]): The target Pydantic model definition for extraction.
+        """
+        self.config = config
+        self.test_data_loader = test_data_loader
+        self.reader = reader
+        self.converter = converter
+        self.extractor = extractor
+        self.evaluators = list(evaluators)
+        self.exporter = exporter
+        self.schema = schema
+
+    @classmethod
+    def from_config(
+        cls,
+        config: EvaluationPipelineConfig,
+        schema: type[ExtractionSchema],
+        reader_cls: type[BaseReader],
+        converter_cls: type[BaseConverter],
+        extractor_cls: type[BaseExtractor],
+        test_data_loader_cls: type[BaseTestDataLoader[ExtractionSchema]],
+        evaluator_classes: list[type[BaseEvaluator[ExtractionSchema]]],
+        evaluation_exporter_cls: type[BaseEvaluationExporter],
+    ) -> "EvaluationOrchestrator[ExtractionSchema]":
+        """Factory method to create an EvaluationOrchestrator from config.
+
+        Args:
+            config (EvaluationPipelineConfig): The full evaluation pipeline configuration.
+            schema (type[ExtractionSchema]): The target Pydantic model definition for extraction.
+            reader_cls (type[BaseReader]): The concrete Reader class to instantiate.
+            converter_cls (type[BaseConverter]): The concrete Converter class to instantiate.
+            extractor_cls (type[BaseExtractor]): The concrete Extractor class to instantiate.
+            test_data_loader_cls (type[BaseTestDataLoader[ExtractionSchema]]): The
+                concrete TestDataLoader class to instantiate.
+            evaluator_classes (list[type[BaseEvaluator[ExtractionSchema]]]): The
+                evaluator classes available for instantiation.
+            evaluation_exporter_cls (type[BaseEvaluationExporter]): The concrete
+                EvaluationExporter class to instantiate.
+
+        Returns:
+            EvaluationOrchestrator[ExtractionSchema]: The configured orchestrator.
+        """
+        reader_instance = reader_cls(config.reader)
+        converter_instance = converter_cls(config.converter)
+        extractor_instance = extractor_cls(config.extractor)
+        test_data_loader_instance = test_data_loader_cls(config.test_data_loader)
+        evaluation_exporter_instance = evaluation_exporter_cls(
+            config.evaluation_exporter
+        )
+
+        config_lookup = {
+            item.__class__.__name__.replace("Config", ""): item
+            for item in config.evaluators
+        }
+
+        evaluators = []
+        for evaluator_cls in evaluator_classes:
+            evaluator_key = evaluator_cls.__name__
+            evaluator_config = config_lookup.get(evaluator_key)
+
+            if evaluator_config is not None:
+                evaluators.append(evaluator_cls(evaluator_config))
+            else:
+                raise ValueError(
+                    f"No configuration found for evaluator '{evaluator_key}'."
+                )
+        if not evaluators:
+            raise ValueError("No valid evaluators configured.")
+
+        return cls(
+            config=config.orchestrator,
+            test_data_loader=test_data_loader_instance,
+            reader=reader_instance,
+            converter=converter_instance,
+            extractor=extractor_instance,
+            evaluators=evaluators,
+            exporter=evaluation_exporter_instance,
+            schema=schema,
+        )
+
+    @staticmethod
+    def _ingest(
+        path_identifier: PathIdentifier,
+        reader: BaseReader,
+        converter: BaseConverter,
+    ) -> Document:
+        """Performs the CPU-bound ingestion phase.
+
+        Args:
+            path_identifier (PathIdentifier): The path identifier to the source file.
+            reader (BaseReader): The reader instance to use.
+            converter (BaseConverter): The converter instance to use.
+
+        Returns:
+            Document: The fully parsed document object.
+        """
+        doc_bytes: DocumentBytes = reader.read(path_identifier)
+        return converter.convert(doc_bytes)
+
+    @staticmethod
+    async def _run_in_executor_with_context(
+        loop: asyncio.AbstractEventLoop,
+        pool: ThreadPoolExecutor,
+        func: Callable[..., T],
+        *args: object,
+    ) -> T:
+        """Run a function in an executor while preserving contextvars.
+
+        Args:
+            loop (asyncio.AbstractEventLoop): The event loop to use.
+            pool (ThreadPoolExecutor): The thread pool to run the function in.
+            func (Callable[..., T]): The function to execute.
+            *args (object): Arguments to pass to the function.
+
+        Returns:
+            The result of the function execution.
+        """
+        ctx = contextvars.copy_context()
+        return await loop.run_in_executor(pool, ctx.run, func, *args)
+
+    async def process_example(
+        self,
+        example: EvaluationExample[ExtractionSchema],
+        pool: ThreadPoolExecutor,
+        semaphore: asyncio.Semaphore,
+    ) -> tuple[Document, list[EvaluationResult]]:
+        """Runs extraction, evaluation, and export for a single example.
+
+        Args:
+            example (EvaluationExample[ExtractionSchema]): The evaluation example to process.
+            pool (ThreadPoolExecutor): The thread pool for CPU-bound tasks.
+            semaphore (asyncio.Semaphore): Semaphore to limit concurrency.
+
+        Returns:
+            tuple[Document, list[EvaluationResult]]: The document and its evaluation results.
+        """
+        loop = asyncio.get_running_loop()
+
+        document: Document = await self._run_in_executor_with_context(
+            loop,
+            pool,
+            self._ingest,
+            example.path_identifier,
+            self.reader,
+            self.converter,
+        )
+
+        async with semaphore:
+            pred: ExtractionSchema = await self.extractor.extract(document, self.schema)
+
+            evaluation_tasks = [
+                self._run_in_executor_with_context(
+                    loop, pool, evaluator.evaluate, example.true, pred
+                )
+                for evaluator in self.evaluators
+            ]
+            results: list[EvaluationResult] = list(
+                await asyncio.gather(*evaluation_tasks)
+            )
+
+            logger.info("Completed evaluation for %s", document.id)
+            return document, results
+
+    async def run(
+        self,
+        examples: list[EvaluationExample[ExtractionSchema]],
+    ) -> None:
+        """Run all evaluators and export results for the provided examples.
+
+        Args:
+            examples (list[EvaluationExample[ExtractionSchema]]): The evaluation examples to evaluate.
+        """
+        semaphore = asyncio.Semaphore(self.config.max_concurrency)
+
+        with ThreadPoolExecutor(max_workers=self.config.max_workers) as pool:
+            tasks = [
+                self.process_example(example, pool, semaphore) for example in examples
+            ]
+
+            results_or_exceptions = await asyncio.gather(*tasks, return_exceptions=True)
+
+            valid_results: list[tuple[Document, list[EvaluationResult]]] = []
+
+            for example, result in zip(examples, results_or_exceptions, strict=True):
+                if isinstance(result, BaseException):
+                    logger.error(
+                        "Evaluation pipeline failed for %s",
+                        example.path_identifier,
+                        exc_info=result,
+                    )
+                else:
+                    valid_results.append(result)
+
+            if valid_results:
+                await self.exporter.export(valid_results)