extractfold 0.1.1__py3-none-any.whl

extractfold/__init__.py

@@ -0,0 +1,40 @@
+"""Schema-first structured data extraction for documents."""
+
+from __future__ import annotations
+
+from extractfold.chunking import TextChunk, chunk_json_array, chunk_rows, chunk_sections, chunk_text
+from extractfold.engines.base import (
+    BatchExtractionResult,
+    ExtractionCapabilities,
+    ExtractionEngine,
+    ExtractionResult,
+    ExtractionRouter,
+    SchemaValidationResult,
+    load_schema,
+    validate_data,
+)
+from extractfold.schema import SchemaConversionResult, infer_schema, template_to_schema
+from extractfold.text import extract_rows_chunked
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "BatchExtractionResult",
+    "ExtractionCapabilities",
+    "ExtractionEngine",
+    "ExtractionResult",
+    "ExtractionRouter",
+    "SchemaConversionResult",
+    "SchemaValidationResult",
+    "TextChunk",
+    "__version__",
+    "chunk_json_array",
+    "chunk_rows",
+    "chunk_sections",
+    "chunk_text",
+    "extract_rows_chunked",
+    "infer_schema",
+    "load_schema",
+    "template_to_schema",
+    "validate_data",
+]

extractfold/chunking.py

@@ -0,0 +1,187 @@
+"""Deterministic chunk planning utilities."""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class TextChunk:
+    """A planned extraction chunk with stable source coordinates."""
+
+    text: str
+    index: int
+    start: int
+    end: int
+    strategy: str
+    overlap: int = 0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the chunk to a plain dictionary."""
+        return {
+            "text": self.text,
+            "index": self.index,
+            "start": self.start,
+            "end": self.end,
+            "strategy": self.strategy,
+            "overlap": self.overlap,
+            "metadata": dict(self.metadata),
+        }
+
+
+def chunk_text(text: str, *, max_chars: int, overlap: int = 0) -> list[TextChunk]:
+    """Plan character-window chunks for plain text."""
+    _validate_window(max_chars, overlap)
+    if not text:
+        return []
+
+    chunks: list[TextChunk] = []
+    start = 0
+    while start < len(text):
+        end = min(start + max_chars, len(text))
+        chunks.append(
+            TextChunk(
+                text=text[start:end],
+                index=len(chunks),
+                start=start,
+                end=end,
+                strategy="text",
+                overlap=0 if not chunks else overlap,
+                metadata={"unit": "char"},
+            )
+        )
+        if end == len(text):
+            break
+        start = end - overlap
+    return chunks
+
+
+def chunk_json_array(json_text: str, *, max_rows: int, overlap: int = 0) -> list[TextChunk]:
+    """Plan row-safe chunks from a JSON array string."""
+    rows = json.loads(json_text)
+    if not isinstance(rows, list):
+        raise ValueError("chunk_json_array expects a JSON array")
+    return chunk_rows(rows, max_rows=max_rows, overlap=overlap, strategy="json_array")
+
+
+def chunk_rows(
+    rows: Sequence[Any],
+    *,
+    max_rows: int,
+    overlap: int = 0,
+    strategy: str = "rows",
+) -> list[TextChunk]:
+    """Plan row-window chunks from already parsed rows."""
+    _validate_window(max_rows, overlap)
+    if not rows:
+        return []
+
+    chunks: list[TextChunk] = []
+    start = 0
+    while start < len(rows):
+        end = min(start + max_rows, len(rows))
+        row_slice = list(rows[start:end])
+        chunks.append(
+            TextChunk(
+                text=json.dumps(row_slice, ensure_ascii=False),
+                index=len(chunks),
+                start=start,
+                end=end,
+                strategy=strategy,
+                overlap=0 if not chunks else overlap,
+                metadata={"unit": "row", "row_start": start, "row_end": end},
+            )
+        )
+        if end == len(rows):
+            break
+        start = end - overlap
+    return chunks
+
+
+def chunk_sections(text: str, *, max_chars: int, overlap: int = 0) -> list[TextChunk]:
+    """Plan section-aware chunks for Markdown or HTML-like text."""
+    _validate_window(max_chars, overlap)
+    if not text:
+        return []
+
+    sections = _find_sections(text)
+    if not sections:
+        return chunk_text(text, max_chars=max_chars, overlap=overlap)
+
+    chunks: list[TextChunk] = []
+    current_start: int | None = None
+    current_end: int | None = None
+    current_text = ""
+
+    for start, end in sections:
+        section_text = text[start:end]
+        if current_text and len(current_text) + len(section_text) > max_chars:
+            chunks.append(
+                _section_chunk(
+                    current_text,
+                    len(chunks),
+                    current_start or 0,
+                    current_end or 0,
+                )
+            )
+            current_start = start
+            current_text = section_text
+            current_end = end
+        else:
+            if current_start is None:
+                current_start = start
+            current_text += section_text
+            current_end = end
+
+    if current_text:
+        chunks.append(
+            _section_chunk(
+                current_text,
+                len(chunks),
+                current_start or 0,
+                current_end or 0,
+            )
+        )
+    return chunks
+
+
+def _validate_window(size: int, overlap: int) -> None:
+    if size <= 0:
+        raise ValueError("chunk size must be positive")
+    if overlap < 0:
+        raise ValueError("overlap must be non-negative")
+    if overlap >= size:
+        raise ValueError("overlap must be smaller than chunk size")
+
+
+def _find_sections(text: str) -> list[tuple[int, int]]:
+    starts = _markdown_heading_starts(text) or _html_heading_starts(text)
+    if not starts:
+        return []
+    ends = starts[1:] + [len(text)]
+    return [(start, end) for start, end in zip(starts, ends) if end > start]
+
+
+def _markdown_heading_starts(text: str) -> list[int]:
+    return [match.start() for match in re.finditer(r"(?m)^#{1,6}\s+", text)]
+
+
+def _html_heading_starts(text: str) -> list[int]:
+    return [match.start() for match in re.finditer(r"(?i)<h[1-6][^>]*>", text)]
+
+
+def _section_chunk(text: str, index: int, start: int, end: int) -> TextChunk:
+    return TextChunk(
+        text=text,
+        index=index,
+        start=start,
+        end=end,
+        strategy="section",
+        overlap=0,
+        metadata={"unit": "char"},
+    )

extractfold/cli.py

@@ -0,0 +1,159 @@
+"""extractfold CLI entry point."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from extractfold.engines.base import ExtractionResult, ExtractionRouter, load_schema
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(
+        prog="extractfold",
+        description="Extract schema-conformant dictionaries from documents.",
+    )
+    sub = parser.add_subparsers(dest="command")
+
+    extract_p = sub.add_parser("extract", help="Extract structured data from a document")
+    extract_p.add_argument("file")
+    extract_p.add_argument("--schema", required=True)
+    extract_p.add_argument("--engine")
+    extract_p.add_argument("--out")
+    extract_p.add_argument("--format", choices=["json"], default="json")
+
+    compare_p = sub.add_parser("compare", help="Compare engines on a document")
+    compare_p.add_argument("file")
+    compare_p.add_argument("--schema", required=True)
+    compare_p.add_argument("--engines")
+
+    sub.add_parser("list-engines", help="List registered engines")
+
+    benchmark_p = sub.add_parser("benchmark", help="Run the benchmark harness")
+    benchmark_p.add_argument("dataset")
+    benchmark_p.add_argument("--engines")
+    benchmark_p.add_argument("--out")
+
+    args = parser.parse_args(argv)
+    if args.command is None:
+        parser.print_help()
+        raise SystemExit(0)
+    if args.command == "extract":
+        asyncio.run(_cmd_extract(args))
+    elif args.command == "compare":
+        asyncio.run(_cmd_compare(args))
+    elif args.command == "list-engines":
+        _cmd_list_engines()
+    elif args.command == "benchmark":
+        _run_benchmark_cli(args)
+
+
+def _build_router() -> ExtractionRouter:
+    from extractfold.engines.azure_docint_engine import AzureDocIntEngine
+    from extractfold.engines.docfold_llm_engine import DocfoldLLMEngine
+    from extractfold.engines.fenic_engine import FenicEngine
+    from extractfold.engines.google_docai_engine import GoogleDocAIEngine
+    from extractfold.engines.instructor_engine import InstructorEngine
+    from extractfold.engines.lift_engine import LiftEngine
+    from extractfold.engines.llamaextract_engine import LlamaExtractEngine
+    from extractfold.engines.llm_structured_engine import LLMStructuredEngine
+    from extractfold.engines.nuextract_engine import NuExtractEngine
+    from extractfold.engines.provider_router_engine import ProviderRouterEngine
+    from extractfold.engines.textract_engine import TextractEngine
+
+    return ExtractionRouter(
+        [
+            LiftEngine(),
+            NuExtractEngine(),
+            ProviderRouterEngine(),
+            LLMStructuredEngine(),
+            InstructorEngine(),
+            FenicEngine(),
+            LlamaExtractEngine(),
+            AzureDocIntEngine(),
+            GoogleDocAIEngine(),
+            TextractEngine(),
+            DocfoldLLMEngine(),
+        ]
+    )
+
+
+async def _cmd_extract(args: argparse.Namespace) -> None:
+    router = _build_router()
+    schema = load_schema(args.schema)
+    result = await router.extract(args.file, schema, engine_hint=args.engine)
+    payload = json.dumps(result.to_dict(), ensure_ascii=False, indent=2)
+    if args.out:
+        Path(args.out).write_text(payload, encoding="utf-8")
+    else:
+        print(payload)
+
+
+async def _cmd_compare(args: argparse.Namespace) -> None:
+    router = _build_router()
+    schema = load_schema(args.schema)
+    engines = args.engines.split(",") if args.engines else None
+    results = await router.compare(args.file, schema, engines=engines)
+    print("Engine results")
+    print("--------------")
+    for name, result in results.items():
+        print(f"{name}: valid={result.valid} data={json.dumps(result.data, ensure_ascii=False)}")
+    print()
+    print("Field agreement")
+    print("---------------")
+    for field, values in _field_agreement(results).items():
+        print(f"{field}: {values}")
+
+
+def _cmd_list_engines() -> None:
+    engines = _build_router().list_engines()
+    print(
+        f"{'Engine':<16} {'Avail':<6} {'Conf':<4} {'Prov':<4} "
+        f"{'Nest':<4} {'Local':<5} {'Remote':<6} Extensions"
+    )
+    print("-" * 86)
+    for engine in engines:
+        caps = engine["capabilities"]
+        print(
+            f"{engine['name']:<16} {str(engine['available']):<6} "
+            f"{_mark(caps['field_confidence']):<4} {_mark(caps['provenance']):<4} "
+            f"{_mark(caps['nested_schemas']):<4} {_mark(caps['local']):<5} "
+            f"{_mark(caps['remote']):<6} {', '.join(engine['extensions'][:8])}"
+        )
+
+
+def _run_benchmark_cli(args: argparse.Namespace) -> None:
+    import benchmark
+
+    engines = args.engines.split(",") if args.engines else None
+    report = asyncio.run(
+        benchmark.run_benchmark(
+            args.dataset,
+            engines=engines,
+            router=_build_router(),
+            output_path=args.out,
+        )
+    )
+    print(report.to_table())
+
+
+def _field_agreement(results: dict[str, ExtractionResult]) -> dict[str, dict[str, Any]]:
+    fields = sorted({field for result in results.values() for field in result.data})
+    agreement: dict[str, dict[str, Any]] = {}
+    for field in fields:
+        values = {engine: result.data.get(field) for engine, result in results.items()}
+        unique = {json.dumps(value, sort_keys=True) for value in values.values()}
+        agreement[field] = {"agree": len(unique) <= 1, "values": values}
+    return agreement
+
+
+def _mark(value: bool) -> str:
+    return "+" if value else "-"
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

extractfold/engines/__init__.py

@@ -0,0 +1,33 @@
+"""Extraction engine adapters."""
+
+from __future__ import annotations
+
+from extractfold.engines.azure_docint_engine import AzureDocIntEngine
+from extractfold.engines.base import ExtractionEngine, ExtractionResult, ExtractionRouter
+from extractfold.engines.docfold_llm_engine import DocfoldLLMEngine
+from extractfold.engines.fenic_engine import FenicEngine
+from extractfold.engines.google_docai_engine import GoogleDocAIEngine
+from extractfold.engines.instructor_engine import InstructorEngine
+from extractfold.engines.lift_engine import LiftEngine
+from extractfold.engines.llamaextract_engine import LlamaExtractEngine
+from extractfold.engines.llm_structured_engine import LLMStructuredEngine
+from extractfold.engines.nuextract_engine import NuExtractEngine
+from extractfold.engines.provider_router_engine import ProviderRouterEngine
+from extractfold.engines.textract_engine import TextractEngine
+
+__all__ = [
+    "AzureDocIntEngine",
+    "DocfoldLLMEngine",
+    "ExtractionEngine",
+    "ExtractionResult",
+    "ExtractionRouter",
+    "FenicEngine",
+    "GoogleDocAIEngine",
+    "InstructorEngine",
+    "LiftEngine",
+    "LLMStructuredEngine",
+    "LlamaExtractEngine",
+    "NuExtractEngine",
+    "ProviderRouterEngine",
+    "TextractEngine",
+]

extractfold/engines/_common.py

@@ -0,0 +1,143 @@
+"""Shared helpers for extraction engine adapters."""
+
+from __future__ import annotations
+
+import inspect
+import re
+import time
+from pathlib import Path
+from typing import Any
+
+from extractfold.engines.base import ExtractionResult, JsonSchema, validate_data
+from extractfold.parsing import normalize_data_payload
+from extractfold.parsing import parse_json_object as _parse_json_object
+
+
+async def maybe_await(value: Any) -> Any:
+    if inspect.isawaitable(value):
+        return await value
+    return value
+
+
+def read_document_text(file_path: str) -> str:
+    path = Path(file_path)
+    try:
+        return path.read_text(encoding="utf-8")
+    except UnicodeDecodeError:
+        return path.read_bytes().decode("utf-8", errors="ignore")
+
+
+def elapsed_ms(start: float) -> int:
+    return int((time.perf_counter() - start) * 1000)
+
+
+def extract_data_payload(payload: Any) -> dict[str, Any]:
+    if isinstance(payload, ExtractionResult):
+        return payload.data
+    if hasattr(payload, "model_dump"):
+        dumped = payload.model_dump()
+        if isinstance(dumped, dict):
+            payload = dumped
+    return normalize_data_payload(payload)
+
+
+def parse_json_object(text: str) -> dict[str, Any]:
+    """Backward-compatible object parser for engine adapters."""
+    return _parse_json_object(text)
+
+
+def coerce_data_to_schema(data: dict[str, Any], schema: JsonSchema) -> dict[str, Any]:
+    properties = schema.get("properties", {})
+    if not isinstance(properties, dict):
+        return data
+    coerced = dict(data)
+    for key, field_schema in properties.items():
+        if key not in coerced or not isinstance(field_schema, dict):
+            continue
+        expected = field_schema.get("type")
+        if expected in {"number", "integer"} and isinstance(coerced[key], str):
+            coerced[key] = _coerce_number(coerced[key], integer=expected == "integer")
+        elif expected == "boolean" and isinstance(coerced[key], str):
+            lowered = coerced[key].strip().lower()
+            if lowered in {"true", "yes", "1"}:
+                coerced[key] = True
+            elif lowered in {"false", "no", "0"}:
+                coerced[key] = False
+    return coerced
+
+
+def _coerce_number(value: str, *, integer: bool) -> float | int | str:
+    cleaned = re.sub(r"[^0-9.\-]", "", value)
+    if cleaned in {"", "-", "."}:
+        return value
+    try:
+        number = float(cleaned)
+    except ValueError:
+        return value
+    return int(number) if integer else number
+
+
+def confidence_from_fields(payload: Any) -> dict[str, float]:
+    if not isinstance(payload, dict):
+        return {}
+    if isinstance(payload.get("field_confidence"), dict):
+        return {
+            str(key): float(value)
+            for key, value in payload["field_confidence"].items()
+            if isinstance(value, (int, float))
+        }
+    fields = payload.get("fields")
+    if isinstance(fields, dict):
+        out: dict[str, float] = {}
+        for name, item in fields.items():
+            if isinstance(item, dict) and isinstance(item.get("confidence"), (int, float)):
+                out[str(name)] = float(item["confidence"])
+        return out
+    return {}
+
+
+def provenance_from_fields(payload: Any) -> dict[str, Any]:
+    if not isinstance(payload, dict):
+        return {}
+    if isinstance(payload.get("provenance"), dict):
+        return payload["provenance"]
+    fields = payload.get("fields")
+    if isinstance(fields, dict):
+        out: dict[str, Any] = {}
+        for name, item in fields.items():
+            if isinstance(item, dict):
+                out[str(name)] = {
+                    key: value
+                    for key, value in item.items()
+                    if key not in {"value", "confidence"}
+                }
+        return {key: value for key, value in out.items() if value}
+    return {}
+
+
+def build_result(
+    *,
+    data: dict[str, Any],
+    engine_name: str,
+    schema: JsonSchema,
+    start: float,
+    raw: Any,
+    field_confidence: dict[str, float] | None = None,
+    provenance: dict[str, Any] | None = None,
+    metadata: dict[str, Any] | None = None,
+    pages: int | None = None,
+) -> ExtractionResult:
+    data = coerce_data_to_schema(data, schema)
+    validation = validate_data(data, schema)
+    return ExtractionResult(
+        data=data,
+        engine_name=engine_name,
+        schema=schema,
+        field_confidence=field_confidence or {},
+        provenance=provenance or {},
+        valid=validation.valid,
+        raw=raw,
+        metadata=metadata or {},
+        pages=pages,
+        processing_time_ms=elapsed_ms(start),
+    )

extractfold/engines/azure_docint_engine.py

@@ -0,0 +1,135 @@
+"""Azure Document Intelligence extraction adapter."""
+
+from __future__ import annotations
+
+import os
+import time
+from typing import Any
+
+from extractfold.engines._common import build_result, maybe_await
+from extractfold.engines.base import (
+    ExtractionCapabilities,
+    ExtractionEngine,
+    ExtractionResult,
+    load_schema,
+)
+
+_SUPPORTED_EXTENSIONS = {"pdf", "png", "jpg", "jpeg", "tiff", "tif", "bmp", "docx"}
+
+
+class AzureDocIntEngine(ExtractionEngine):
+    """Adapter for Azure Document Intelligence prebuilt/query-field extraction."""
+
+    def __init__(
+        self,
+        endpoint: str | None = None,
+        key: str | None = None,
+        model_id: str = "prebuilt-layout",
+        client: Any | None = None,
+    ) -> None:
+        self._endpoint = endpoint or os.getenv("AZURE_DOCINT_ENDPOINT")
+        self._key = key or os.getenv("AZURE_DOCINT_KEY")
+        self._model_id = model_id
+        self._client = client
+
+    @property
+    def name(self) -> str:
+        return "azure_docint"
+
+    @property
+    def supported_extensions(self) -> set[str]:
+        return _SUPPORTED_EXTENSIONS
+
+    @property
+    def capabilities(self) -> ExtractionCapabilities:
+        return ExtractionCapabilities(
+            field_confidence=True,
+            provenance=True,
+            nested_schemas=False,
+            batch=False,
+            local=False,
+            remote=True,
+        )
+
+    def is_available(self) -> bool:
+        if self._client is not None:
+            return True
+        try:
+            import azure.ai.documentintelligence  # noqa: F401
+        except ImportError:
+            return False
+        return bool(self._endpoint and self._key)
+
+    async def extract(self, file_path: str, schema, **kwargs: Any) -> ExtractionResult:
+        schema_obj = load_schema(schema)
+        start = time.perf_counter()
+        if self._client is not None:
+            raw = await maybe_await(
+                self._client(
+                    file_path=file_path,
+                    schema=schema_obj,
+                    model_id=self._model_id,
+                    **kwargs,
+                )
+            )
+        else:
+            raw = await maybe_await(self._call_azure(file_path, schema_obj, **kwargs))
+        data, confidence, provenance = self._parse_fields(raw)
+        pages = _pages_from_raw(raw)
+        return build_result(
+            data=data,
+            engine_name=self.name,
+            schema=schema_obj,
+            start=start,
+            raw=raw,
+            field_confidence=confidence,
+            provenance=provenance,
+            metadata={"model_id": self._model_id},
+            pages=pages,
+        )
+
+    def _call_azure(self, file_path: str, schema: dict[str, Any], **kwargs: Any) -> dict[str, Any]:
+        from azure.ai.documentintelligence import DocumentIntelligenceClient
+        from azure.ai.documentintelligence.models import AnalyzeDocumentRequest
+        from azure.core.credentials import AzureKeyCredential
+
+        if not self._endpoint or not self._key:
+            raise RuntimeError("AZURE_DOCINT_ENDPOINT and AZURE_DOCINT_KEY are required")
+        client = DocumentIntelligenceClient(
+            endpoint=self._endpoint,
+            credential=AzureKeyCredential(self._key),
+        )
+        with open(file_path, "rb") as file_obj:
+            poller = client.begin_analyze_document(
+                self._model_id,
+                AnalyzeDocumentRequest(bytes_source=file_obj.read()),
+            )
+        result = poller.result()
+        fields: dict[str, Any] = {}
+        for name in schema.get("properties", {}):
+            fields[name] = {"value": getattr(result, "content", ""), "confidence": None}
+        return {"fields": fields, "pages": len(getattr(result, "pages", []) or [])}
+
+    @staticmethod
+    def _parse_fields(raw: Any) -> tuple[dict[str, Any], dict[str, float], dict[str, Any]]:
+        fields = raw.get("fields", {}) if isinstance(raw, dict) else {}
+        data: dict[str, Any] = {}
+        confidence: dict[str, float] = {}
+        provenance: dict[str, Any] = {}
+        for name, item in fields.items():
+            if isinstance(item, dict):
+                data[name] = item.get("value")
+                if isinstance(item.get("confidence"), (int, float)):
+                    confidence[name] = float(item["confidence"])
+                provenance[name] = {
+                    key: value
+                    for key, value in item.items()
+                    if key not in {"value", "confidence"}
+                }
+            else:
+                data[name] = item
+        return data, confidence, {key: value for key, value in provenance.items() if value}
+
+
+def _pages_from_raw(raw: Any) -> int | None:
+    return raw.get("pages") if isinstance(raw, dict) and isinstance(raw.get("pages"), int) else None