folioiq 0.1.0__py3-none-any.whl

folioiq/__init__.py

@@ -0,0 +1,20 @@
+"""FolioIQ — schema-driven document extraction."""
+
+from folioiq.client import DocumentExtractor
+from folioiq.exceptions import FolioIQError, NotImplementedPhaseError, SpecError
+from folioiq.models import ExtractionResult
+from folioiq.specs import ExtractionSpec, FieldSpec, build_model, load_spec
+
+__all__ = [
+    "DocumentExtractor",
+    "ExtractionResult",
+    "ExtractionSpec",
+    "FieldSpec",
+    "FolioIQError",
+    "NotImplementedPhaseError",
+    "SpecError",
+    "build_model",
+    "load_spec",
+]
+
+__version__ = "0.1.0"

folioiq/azure_di.py

@@ -0,0 +1,61 @@
+"""Azure Document Intelligence OCR (requires folioiq[azure])."""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+from folioiq.config import FolioIQSettings
+from folioiq.exceptions import FolioIQError
+from folioiq.io import DocumentInput, read_document_bytes
+
+
+class OCRClient(Protocol):
+    def extract_text(self, document: DocumentInput) -> str:
+        ...
+
+
+class AzureDIClient:
+    """OCR via Azure Document Intelligence prebuilt-read."""
+
+    def __init__(self, settings: FolioIQSettings, model_id: str = "prebuilt-read") -> None:
+        self.settings = settings
+        self.model_id = model_id
+        self._client = self._build_client()
+
+    def _build_client(self):
+        endpoint = self.settings.azure_doc_intelligence_endpoint
+        key = self.settings.azure_doc_intelligence_key
+        if not endpoint or not key:
+            raise FolioIQError(
+                "Azure Document Intelligence is not configured. "
+                "Set AZURE_DOC_INTELLIGENCE_ENDPOINT and AZURE_DOC_INTELLIGENCE_KEY."
+            )
+        try:
+            from azure.ai.documentintelligence import DocumentIntelligenceClient
+            from azure.core.credentials import AzureKeyCredential
+        except ImportError as exc:
+            raise FolioIQError(
+                "Azure DI requires: pip install 'folioiq[azure]'"
+            ) from exc
+
+        return DocumentIntelligenceClient(
+            endpoint=endpoint,
+            credential=AzureKeyCredential(key),
+        )
+
+    def extract_text(self, document: DocumentInput) -> str:
+        data, _ = read_document_bytes(document)
+        poller = self._client.begin_analyze_document(
+            self.model_id,
+            body=data,
+            content_type="application/octet-stream",
+        )
+        result = poller.result()
+        text = (result.content or "").strip()
+        if not text:
+            raise FolioIQError("Azure DI returned empty OCR text")
+        return text
+
+
+def get_default_ocr(settings: FolioIQSettings) -> OCRClient:
+    return AzureDIClient(settings)

folioiq/client.py

@@ -0,0 +1,183 @@
+"""Main entrypoint — schema-driven extraction."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import BinaryIO
+
+from pydantic import BaseModel
+
+from folioiq.azure_di import OCRClient, get_default_ocr
+from folioiq.config import FolioIQSettings
+from folioiq.io import is_image_suffix, read_document_bytes, suffix_of
+from folioiq.llm import LLMClient, get_default_llm
+from folioiq.models import ExtractionResult
+from folioiq.pdf_text import extract_pdf_text
+from folioiq.prompt import build_extraction_prompt
+from folioiq.router import ParserName, choose_parser
+from folioiq.specs import ExtractionSpec, build_model, load_spec
+from folioiq.validation import validate_extraction
+
+DocumentInput = str | Path | bytes | BinaryIO
+
+_SYSTEM = (
+    "You are a document extraction engine. "
+    "Extract fields exactly as present in the document text. "
+    "Never invent values. Use null for missing fields."
+)
+
+
+class DocumentExtractor:
+    """Extract structured fields from a document using a YAML spec."""
+
+    def __init__(
+        self,
+        settings: FolioIQSettings | None = None,
+        llm: LLMClient | None = None,
+        ocr: OCRClient | None = None,
+    ) -> None:
+        self.settings = settings or FolioIQSettings()
+        self._llm = llm
+        self._ocr = ocr
+
+    @classmethod
+    def from_env(cls) -> DocumentExtractor:
+        """Build from environment variables / .env."""
+        return cls(settings=FolioIQSettings())
+
+    def load_spec(
+        self,
+        spec: str | Path,
+        *,
+        instructions: str | Path | None = None,
+    ) -> ExtractionSpec:
+        """Load YAML spec (+ optional / sibling Markdown instructions)."""
+        return load_spec(spec, instructions=instructions)
+
+    def build_model(self, spec: ExtractionSpec) -> type[BaseModel]:
+        """Build a Pydantic model from the loaded spec."""
+        return build_model(spec)
+
+    def extract(
+        self,
+        document: DocumentInput,
+        spec: str | Path,
+        *,
+        instructions: str | Path | None = None,
+    ) -> ExtractionResult:
+        """Extract structured data from a PDF or image.
+
+        On validation failure (or parse failure), retries once with the
+        alternate parser when that makes sense.
+        """
+        loaded = self.load_spec(spec, instructions=instructions)
+        model_cls = self.build_model(loaded)
+        data, filename = read_document_bytes(document)
+        decision = choose_parser(data, filename, loaded)
+
+        primary = self._run(
+            data=data,
+            filename=filename,
+            spec=loaded,
+            model_cls=model_cls,
+            parser=decision.parser,
+            route_reason=decision.reason,
+            fallback_used=False,
+        )
+        if primary.is_valid:
+            return primary
+
+        alt = _alternate_parser(decision.parser, filename)
+        if alt is None:
+            return primary
+
+        secondary = self._run(
+            data=data,
+            filename=filename,
+            spec=loaded,
+            model_cls=model_cls,
+            parser=alt,
+            route_reason=f"fallback after {decision.parser} failed",
+            fallback_used=True,
+            primary_parser=decision.parser,
+            primary_errors=primary.validation_errors,
+        )
+        # Prefer a valid fallback; otherwise keep the richer primary errors
+        if secondary.is_valid:
+            return secondary
+        return primary
+
+    def _run(
+        self,
+        *,
+        data: bytes,
+        filename: str,
+        spec: ExtractionSpec,
+        model_cls: type[BaseModel],
+        parser: ParserName,
+        route_reason: str,
+        fallback_used: bool,
+        primary_parser: str | None = None,
+        primary_errors: list[str] | None = None,
+    ) -> ExtractionResult:
+        meta: dict = {
+            "spec": spec.name,
+            "spec_version": spec.version,
+            "parser": parser,
+            "route_reason": route_reason,
+            "fallback_used": fallback_used,
+        }
+        if primary_parser:
+            meta["primary_parser"] = primary_parser
+        if primary_errors:
+            meta["primary_errors"] = primary_errors
+
+        try:
+            text = self._read_text(data, parser)
+        except Exception as exc:
+            return ExtractionResult(
+                data={},
+                is_valid=False,
+                validation_errors=[str(exc)],
+                metadata=meta,
+            )
+
+        llm = self._llm or get_default_llm(self.settings)
+        meta["llm"] = type(llm).__name__
+        meta["chars"] = len(text)
+        prompt = build_extraction_prompt(spec)
+        user = f"{prompt}\n\n--- DOCUMENT TEXT ---\n{text}"
+
+        try:
+            parsed = llm.extract(system=_SYSTEM, user=user, model=model_cls)
+            payload = parsed.model_dump(mode="json")
+        except Exception as exc:
+            return ExtractionResult(
+                data={},
+                is_valid=False,
+                validation_errors=[str(exc)],
+                metadata=meta,
+            )
+
+        errors = validate_extraction(spec, payload)
+        return ExtractionResult(
+            data=payload,
+            is_valid=not errors,
+            validation_errors=errors,
+            metadata=meta,
+        )
+
+    def _read_text(self, data: bytes, parser: str) -> str:
+        if parser == "pymupdf":
+            return extract_pdf_text(data)
+        ocr = self._ocr or get_default_ocr(self.settings)
+        return ocr.extract_text(data)
+
+
+def _alternate_parser(parser: ParserName, filename: str) -> ParserName | None:
+    """One flip: pymupdf ↔ azure_di. Images cannot use pymupdf."""
+    if parser == "pymupdf":
+        return "azure_di"
+    if is_image_suffix(suffix_of(filename)):
+        return None
+    return "pymupdf"

folioiq/config.py

@@ -0,0 +1,36 @@
+"""Environment / settings."""
+
+from __future__ import annotations
+
+from pydantic import AliasChoices, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class FolioIQSettings(BaseSettings):
+    """Loads from env / .env."""
+
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+        populate_by_name=True,
+    )
+
+    pipeline_mode: str = "auto"
+
+    # Azure OpenAI — matches common AZURE_* names used by billing projects
+    azure_api_key: str | None = None
+    azure_api_base: str | None = None
+    azure_api_version: str | None = None
+    azure_deployment: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices(
+            "FOLIOIQ_AZURE_MODEL",
+            "AZURE_OPENAI_DEPLOYMENT",
+            "model",
+        ),
+    )
+
+    # Azure Document Intelligence (Phase 3)
+    azure_doc_intelligence_endpoint: str | None = None
+    azure_doc_intelligence_key: str | None = None

folioiq/eval.py

@@ -0,0 +1,187 @@
+"""Simple fixture-pack evaluation."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from folioiq.client import DocumentExtractor
+from folioiq.exceptions import FolioIQError
+
+
+@dataclass
+class FieldScore:
+    name: str
+    expected: Any
+    actual: Any
+    match: bool
+
+
+@dataclass
+class PackReport:
+    pack: str
+    field_scores: list[FieldScore] = field(default_factory=list)
+    is_valid: bool = False
+    validation_errors: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+    skipped: bool = False
+    skip_reason: str = ""
+
+    @property
+    def scored(self) -> list[FieldScore]:
+        return [f for f in self.field_scores if f.expected is not None]
+
+    @property
+    def accuracy(self) -> float:
+        scored = self.scored
+        if not scored:
+            return 0.0
+        return sum(1 for f in scored if f.match) / len(scored)
+
+
+def normalize_value(value: Any) -> str:
+    """Loose compare for eval (case/whitespace/punctuation tolerant)."""
+    if value is None:
+        return ""
+    if isinstance(value, float):
+        return f"{value:.4f}".rstrip("0").rstrip(".")
+    if isinstance(value, int):
+        return str(value)
+    text = str(value).strip().lower()
+    text = re.sub(r"\s+", " ", text)
+    text = re.sub(r"[,\s]+", " ", text)
+    return text.strip()
+
+
+def values_match(expected: Any, actual: Any) -> bool:
+    if expected is None:
+        return True
+    if isinstance(expected, (int, float)) and isinstance(actual, (int, float)):
+        return abs(float(expected) - float(actual)) < 0.01
+    exp = normalize_value(expected)
+    act = normalize_value(actual)
+    if not exp:
+        return True
+    return exp == act or exp in act or act in exp
+
+
+def score_fields(expected: dict[str, Any], actual: dict[str, Any]) -> list[FieldScore]:
+    scores: list[FieldScore] = []
+    for key, exp in expected.items():
+        if key.startswith("_"):
+            continue
+        act = actual.get(key)
+        scores.append(
+            FieldScore(
+                name=key,
+                expected=exp,
+                actual=act,
+                match=values_match(exp, act),
+            )
+        )
+    return scores
+
+
+def find_document(pack_dir: Path) -> Path:
+    for name in ("document.pdf", "document.jpg", "document.jpeg", "document.png"):
+        path = pack_dir / name
+        if path.is_file():
+            return path
+    raise FolioIQError(f"No document.* found in {pack_dir}")
+
+
+def run_pack(
+    pack_dir: str | Path,
+    extractor: DocumentExtractor | None = None,
+) -> PackReport:
+    """Run one fixture pack and score against expected.json."""
+    pack = Path(pack_dir).resolve()
+    report = PackReport(pack=pack.name)
+
+    expected_path = pack / "expected.json"
+    spec_path = pack / "spec.yaml"
+    if not expected_path.is_file() or not spec_path.is_file():
+        report.skipped = True
+        report.skip_reason = "missing spec.yaml or expected.json"
+        return report
+
+    expected = json.loads(expected_path.read_text(encoding="utf-8"))
+    # Pack with only null expectations is a placeholder
+    if all(v is None for k, v in expected.items() if not k.startswith("_")):
+        report.skipped = True
+        report.skip_reason = "expected.json has no labeled values yet"
+        return report
+
+    document = find_document(pack)
+    instructions = pack / "instructions.md"
+    extractor = extractor or DocumentExtractor.from_env()
+    result = extractor.extract(
+        document,
+        spec=spec_path,
+        instructions=instructions if instructions.is_file() else None,
+    )
+
+    report.is_valid = result.is_valid
+    report.validation_errors = list(result.validation_errors)
+    report.metadata = dict(result.metadata)
+    report.field_scores = score_fields(expected, result.data)
+    return report
+
+
+def format_report(report: PackReport) -> str:
+    lines = [f"pack: {report.pack}"]
+    if report.skipped:
+        lines.append(f"  skipped: {report.skip_reason}")
+        return "\n".join(lines)
+    lines.append(f"  is_valid: {report.is_valid}")
+    lines.append(f"  accuracy: {report.accuracy:.1%} ({sum(1 for f in report.scored if f.match)}/{len(report.scored)})")
+    lines.append(f"  parser: {report.metadata.get('parser')}")
+    lines.append(f"  fallback_used: {report.metadata.get('fallback_used')}")
+    if report.validation_errors:
+        lines.append(f"  validation_errors: {report.validation_errors}")
+    for fs in report.field_scores:
+        if fs.expected is None:
+            continue
+        mark = "OK" if fs.match else "MISS"
+        lines.append(f"  [{mark}] {fs.name}: expected={fs.expected!r} actual={fs.actual!r}")
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Evaluate FolioIQ fixture packs")
+    parser.add_argument(
+        "pack",
+        nargs="?",
+        default=None,
+        help="Pack directory (default: all under tests/fixtures/packs)",
+    )
+    args = parser.parse_args(argv)
+
+    if args.pack:
+        packs = [Path(args.pack)]
+    else:
+        root = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "packs"
+        if not root.is_dir():
+            # installed package — look relative to cwd
+            root = Path("tests/fixtures/packs")
+        packs = sorted(p for p in root.iterdir() if p.is_dir())
+
+    extractor = DocumentExtractor.from_env()
+    exit_code = 0
+    for pack in packs:
+        report = run_pack(pack, extractor=extractor)
+        print(format_report(report))
+        print()
+        if report.skipped:
+            continue
+        if report.accuracy < 1.0 or not report.is_valid:
+            exit_code = 1
+    return exit_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

folioiq/exceptions.py

@@ -0,0 +1,13 @@
+"""Shared exceptions."""
+
+
+class FolioIQError(Exception):
+    """Base error for FolioIQ."""
+
+
+class SpecError(FolioIQError):
+    """Invalid or missing extraction spec."""
+
+
+class NotImplementedPhaseError(FolioIQError):
+    """Feature lands in a later phase — see PLAN.md."""

folioiq/io.py

@@ -0,0 +1,49 @@
+"""Shared document IO helpers."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import BinaryIO
+
+from folioiq.exceptions import FolioIQError
+
+DocumentInput = str | Path | bytes | BinaryIO
+
+_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp", ".gif"}
+_PDF_EXTS = {".pdf"}
+
+
+def read_document_bytes(document: DocumentInput) -> tuple[bytes, str]:
+    """Return (bytes, filename_hint) for routing / OCR."""
+    if isinstance(document, (bytes, bytearray)):
+        return bytes(document), "document.pdf"
+
+    if hasattr(document, "read"):
+        data = document.read()
+        if isinstance(data, str):
+            data = data.encode("utf-8")
+        name = getattr(document, "name", "document.bin")
+        return bytes(data), Path(str(name)).name
+
+    path = Path(document).expanduser().resolve()
+    if not path.is_file():
+        raise FolioIQError(f"Document not found: {path}")
+    return path.read_bytes(), path.name
+
+
+def suffix_of(document: DocumentInput, filename_hint: str | None = None) -> str:
+    if filename_hint:
+        return Path(filename_hint).suffix.lower()
+    if isinstance(document, (str, Path)):
+        return Path(document).suffix.lower()
+    if hasattr(document, "name"):
+        return Path(str(document.name)).suffix.lower()
+    return ""
+
+
+def is_image_suffix(suffix: str) -> bool:
+    return suffix in _IMAGE_EXTS
+
+
+def is_pdf_suffix(suffix: str) -> bool:
+    return suffix in _PDF_EXTS or suffix == ""

folioiq/llm.py

@@ -0,0 +1,80 @@
+"""Azure OpenAI structured extraction (requires folioiq[azure])."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+from pydantic import BaseModel
+
+from folioiq.config import FolioIQSettings
+from folioiq.exceptions import FolioIQError
+
+
+class LLMClient(Protocol):
+    def extract(self, *, system: str, user: str, model: type[BaseModel]) -> BaseModel:
+        ...
+
+
+class AzureOpenAIClient:
+    """Thin wrapper around LangChain AzureChatOpenAI + structured output."""
+
+    def __init__(self, settings: FolioIQSettings) -> None:
+        self.settings = settings
+        self._llm = self._build_llm()
+
+    def _build_llm(self) -> Any:
+        missing = [
+            name
+            for name, value in [
+                ("AZURE_API_KEY", self.settings.azure_api_key),
+                ("AZURE_API_BASE", self.settings.azure_api_base),
+                ("AZURE_API_VERSION", self.settings.azure_api_version),
+                ("FOLIOIQ_AZURE_MODEL / model", self.settings.azure_deployment),
+            ]
+            if not value
+        ]
+        if missing:
+            raise FolioIQError(
+                "Azure OpenAI is not configured. Missing: "
+                + ", ".join(missing)
+                + ". Set env vars or install/configure folioiq[azure]."
+            )
+
+        try:
+            from langchain_openai import AzureChatOpenAI
+        except ImportError as exc:
+            raise FolioIQError(
+                "Azure OpenAI client requires: pip install 'folioiq[azure]'"
+            ) from exc
+
+        deployment = self.settings.azure_deployment or ""
+        deployment = deployment.replace("azure/", "")
+
+        return AzureChatOpenAI(
+            api_key=self.settings.azure_api_key,
+            api_version=self.settings.azure_api_version,
+            azure_endpoint=self.settings.azure_api_base,
+            azure_deployment=deployment,
+            model=deployment,
+        )
+
+    def extract(self, *, system: str, user: str, model: type[BaseModel]) -> BaseModel:
+        from langchain_core.messages import HumanMessage, SystemMessage
+
+        structured = self._llm.with_structured_output(model)
+        result = structured.invoke(
+            [
+                SystemMessage(content=system),
+                HumanMessage(content=user),
+            ]
+        )
+        if result is None:
+            raise FolioIQError("LLM returned empty structured output")
+        if isinstance(result, model):
+            return result
+        # Some versions return a dict
+        return model.model_validate(result)
+
+
+def get_default_llm(settings: FolioIQSettings) -> LLMClient:
+    return AzureOpenAIClient(settings)

folioiq/models.py

@@ -0,0 +1,16 @@
+"""Public result types (minimal for Phase 0)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class ExtractionResult(BaseModel):
+    """Outcome of one extract() call."""
+
+    data: dict[str, Any] = Field(default_factory=dict)
+    is_valid: bool = False
+    validation_errors: list[str] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)

folioiq/pdf_text.py

@@ -0,0 +1,48 @@
+"""Pull plain text from digital PDFs."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import BinaryIO
+
+import fitz
+
+from folioiq.exceptions import FolioIQError
+from folioiq.io import DocumentInput, read_document_bytes, suffix_of
+
+MIN_TEXT_CHARS = 20
+
+
+def peek_pdf_text(document: DocumentInput) -> str:
+    """Return embedded PDF text (may be empty). Does not raise on short text."""
+    data, name = read_document_bytes(document)
+    suffix = suffix_of(document, name)
+    if suffix and suffix != ".pdf":
+        return ""
+
+    doc = fitz.open(stream=data, filetype="pdf")
+    try:
+        parts = [page.get_text("text") for page in doc]
+        return "\n".join(parts).strip()
+    finally:
+        doc.close()
+
+
+def extract_pdf_text(document: DocumentInput) -> str:
+    """Extract embedded text from a digital PDF. Raises if too little text."""
+    # Prefer path open when we have a path (clearer errors)
+    if isinstance(document, (str, Path)):
+        path = Path(document).expanduser().resolve()
+        if not path.is_file():
+            raise FolioIQError(f"Document not found: {path}")
+        if path.suffix.lower() not in ("", ".pdf"):
+            raise FolioIQError(
+                f"pymupdf parser expects PDF (got {path.suffix or 'no extension'})."
+            )
+
+    text = peek_pdf_text(document)
+    if len(text) < MIN_TEXT_CHARS:
+        raise FolioIQError(
+            "PDF has little or no embedded text. Use parser azure_di (or pipeline auto)."
+        )
+    return text

folioiq/prompt.py

@@ -0,0 +1,42 @@
+"""Build extraction prompts from a loaded spec."""
+
+from __future__ import annotations
+
+from folioiq.specs import ExtractionSpec, FieldSpec
+
+
+def build_extraction_prompt(spec: ExtractionSpec) -> str:
+    """Human prompt: instructions + field list. Schema is enforced by the LLM client."""
+    lines: list[str] = []
+
+    if spec.instructions:
+        lines.append(spec.instructions.strip())
+        lines.append("")
+
+    lines.append(f"Extract fields for document type: {spec.name} (v{spec.version}).")
+    lines.append("Return only values found in the document. Use null when missing.")
+    lines.append("")
+    lines.append("Fields:")
+    for name, field in spec.extraction.fields.items():
+        lines.extend(_describe_field(name, field, indent=0))
+
+    return "\n".join(lines)
+
+
+def _describe_field(name: str, field: FieldSpec, indent: int) -> list[str]:
+    pad = "  " * indent
+    req = "required" if field.required else "optional"
+    lines = [f"{pad}- {name} ({field.type}, {req})"]
+    if field.description:
+        lines.append(f"{pad}  {field.description.strip()}")
+    if field.type == "object" and field.fields:
+        for child_name, child in field.fields.items():
+            lines.extend(_describe_field(child_name, child, indent + 1))
+    if field.type == "array" and field.items is not None:
+        lines.append(f"{pad}  items:")
+        if field.items.type == "object" and field.items.fields:
+            for child_name, child in field.items.fields.items():
+                lines.extend(_describe_field(child_name, child, indent + 2))
+        else:
+            lines.extend(_describe_field("item", field.items, indent + 2))
+    return lines

folioiq/router.py

@@ -0,0 +1,68 @@
+"""Choose pymupdf vs azure_di."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+import fitz
+
+from folioiq.io import is_image_suffix, suffix_of
+from folioiq.pdf_text import MIN_TEXT_CHARS
+from folioiq.specs import ExtractionSpec
+
+ParserName = Literal["pymupdf", "azure_di"]
+
+
+@dataclass(frozen=True)
+class RouteDecision:
+    parser: ParserName
+    reason: str
+    text_chars: int = 0
+
+
+def choose_parser(
+    data: bytes,
+    filename: str,
+    spec: ExtractionSpec,
+) -> RouteDecision:
+    """Pick a text parser for this document + spec."""
+    explicit = (
+        (spec.pipeline.parser or "").strip().lower()
+        if spec.pipeline.mode == "explicit"
+        else ""
+    )
+    if explicit in ("pymupdf", "azure_di"):
+        return RouteDecision(
+            parser=explicit,  # type: ignore[arg-type]
+            reason=f"explicit pipeline.parser={explicit}",
+        )
+
+    suffix = suffix_of(filename)
+    if is_image_suffix(suffix):
+        return RouteDecision(parser="azure_di", reason="image file", text_chars=0)
+
+    text = _pdf_text_from_bytes(data)
+    if len(text) >= MIN_TEXT_CHARS:
+        return RouteDecision(
+            parser="pymupdf",
+            reason="pdf has embedded text",
+            text_chars=len(text),
+        )
+
+    return RouteDecision(
+        parser="azure_di",
+        reason="pdf has little/no text layer",
+        text_chars=len(text),
+    )
+
+
+def _pdf_text_from_bytes(data: bytes) -> str:
+    try:
+        doc = fitz.open(stream=data, filetype="pdf")
+    except Exception:
+        return ""
+    try:
+        return "\n".join(page.get_text("text") for page in doc).strip()
+    finally:
+        doc.close()

folioiq/specs.py

@@ -0,0 +1,181 @@
+"""Load YAML extraction specs and build Pydantic models."""
+
+from __future__ import annotations
+
+from datetime import date, datetime
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseModel, Field, create_model
+
+from folioiq.exceptions import SpecError
+
+FieldType = Literal[
+    "string",
+    "number",
+    "integer",
+    "boolean",
+    "date",
+    "datetime",
+    "array",
+    "object",
+]
+
+
+class FieldSpec(BaseModel):
+    """One field in the extraction schema."""
+
+    type: FieldType = "string"
+    required: bool = False
+    description: str = ""
+    items: FieldSpec | None = None
+    fields: dict[str, FieldSpec] | None = None
+
+
+class PipelineSpec(BaseModel):
+    mode: Literal["auto", "explicit"] = "auto"
+    parser: str | None = None
+    llm: str | None = None
+    model: str | None = None
+
+
+class DocumentSpec(BaseModel):
+    allowed_types: list[str] = Field(default_factory=lambda: ["pdf", "png", "jpg", "jpeg"])
+
+
+class ExtractionBlock(BaseModel):
+    fields: dict[str, FieldSpec]
+
+
+class ExtractionSpec(BaseModel):
+    """Machine-readable extraction contract (from YAML)."""
+
+    name: str
+    version: str = "1.0"
+    document: DocumentSpec = Field(default_factory=DocumentSpec)
+    pipeline: PipelineSpec = Field(default_factory=PipelineSpec)
+    extraction: ExtractionBlock
+    # Filled by loader — not from YAML
+    path: Path | None = None
+    instructions: str | None = None
+
+
+# Allow nested FieldSpec references
+FieldSpec.model_rebuild()
+
+
+_TYPE_MAP: dict[str, type] = {
+    "string": str,
+    "number": float,
+    "integer": int,
+    "boolean": bool,
+    "date": date,
+    "datetime": datetime,
+}
+
+
+def load_spec(
+    path: str | Path,
+    *,
+    instructions: str | Path | None = None,
+) -> ExtractionSpec:
+    """Load a YAML spec and optional Markdown instructions.
+
+    If ``instructions`` is omitted, uses a sibling ``.md`` with the same stem
+    when it exists (e.g. ``invoice.yaml`` → ``invoice.md``).
+    """
+    spec_path = Path(path).expanduser().resolve()
+    if not spec_path.is_file():
+        raise SpecError(f"Spec not found: {spec_path}")
+
+    try:
+        raw = yaml.safe_load(spec_path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        raise SpecError(f"Invalid YAML in {spec_path}: {exc}") from exc
+
+    if not isinstance(raw, dict):
+        raise SpecError(f"Spec must be a mapping: {spec_path}")
+
+    try:
+        spec = ExtractionSpec.model_validate(raw)
+    except Exception as exc:
+        raise SpecError(f"Invalid spec structure in {spec_path}: {exc}") from exc
+
+    if not spec.extraction.fields:
+        raise SpecError(f"Spec has no extraction.fields: {spec_path}")
+
+    spec.path = spec_path
+    spec.instructions = _resolve_instructions(spec_path, instructions)
+    return spec
+
+
+def _resolve_instructions(
+    spec_path: Path,
+    instructions: str | Path | None,
+) -> str | None:
+    if instructions is None:
+        sibling = spec_path.with_suffix(".md")
+        if sibling.is_file():
+            return sibling.read_text(encoding="utf-8")
+        return None
+
+    instr_path = Path(instructions)
+    if instr_path.is_file():
+        return instr_path.read_text(encoding="utf-8")
+    # Treat as inline markdown text
+    return str(instructions)
+
+
+def build_model(spec: ExtractionSpec) -> type[BaseModel]:
+    """Build a dynamic Pydantic model from ``extraction.fields``."""
+    return _build_object_model(
+        name=_safe_model_name(spec.name),
+        fields=spec.extraction.fields,
+    )
+
+
+def _safe_model_name(name: str) -> str:
+    cleaned = "".join(ch if ch.isalnum() else "_" for ch in name.strip())
+    return cleaned[:50] or "Document"
+
+
+def _build_object_model(name: str, fields: dict[str, FieldSpec]) -> type[BaseModel]:
+    definitions: dict[str, Any] = {}
+    for field_name, field in fields.items():
+        py_type, default = _python_type(field, field_name)
+        definitions[field_name] = (
+            py_type,
+            Field(default=default, description=field.description or None),
+        )
+    return create_model(name, **definitions)  # type: ignore[call-overload]
+
+
+def _python_type(field: FieldSpec, name: str) -> tuple[Any, Any]:
+    """Return (annotation, default) for one field."""
+    required = field.required
+    default: Any = ... if required else None
+
+    if field.type == "object":
+        if not field.fields:
+            raise SpecError(f"Field '{name}' type=object needs nested fields")
+        nested = _build_object_model(f"{_safe_model_name(name)}Item", field.fields)
+        annotation: Any = nested if required else nested | None
+        return annotation, default
+
+    if field.type == "array":
+        if field.items is None:
+            raise SpecError(f"Field '{name}' type=array needs items")
+        # List elements are concrete types (not Optional).
+        item_spec = field.items.model_copy(update={"required": True})
+        item_type, _ = _python_type(item_spec, f"{name}_item")
+        list_type: Any = list[item_type]  # type: ignore[valid-type]
+        annotation = list_type if required else list_type | None
+        return annotation, default
+
+    base = _TYPE_MAP.get(field.type)
+    if base is None:
+        raise SpecError(f"Unsupported field type '{field.type}' on '{name}'")
+
+    annotation = base if required else base | None
+    return annotation, default

folioiq/validation.py

@@ -0,0 +1,80 @@
+"""Post-extraction checks against the YAML spec."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from folioiq.specs import ExtractionSpec, FieldSpec
+
+
+def validate_extraction(spec: ExtractionSpec, data: dict[str, Any]) -> list[str]:
+    """Return human-readable errors. Empty list means OK."""
+    errors: list[str] = []
+    for name, field in spec.extraction.fields.items():
+        errors.extend(_check_field(name, field, data.get(name)))
+    return errors
+
+
+def _check_field(name: str, field: FieldSpec, value: Any) -> list[str]:
+    errors: list[str] = []
+
+    if field.required and _is_empty(value):
+        errors.append(f"missing required field: {name}")
+        return errors
+
+    if _is_empty(value):
+        return errors
+
+    if field.type == "array":
+        if not isinstance(value, list):
+            errors.append(f"field '{name}' should be an array")
+            return errors
+        if field.items is not None:
+            for i, item in enumerate(value):
+                if field.items.type == "object" and field.items.fields:
+                    if not isinstance(item, dict):
+                        errors.append(f"field '{name}[{i}]' should be an object")
+                        continue
+                    for child_name, child in field.items.fields.items():
+                        errors.extend(
+                            _check_field(f"{name}[{i}].{child_name}", child, item.get(child_name))
+                        )
+        return errors
+
+    if field.type == "object":
+        if not isinstance(value, dict):
+            errors.append(f"field '{name}' should be an object")
+            return errors
+        if field.fields:
+            for child_name, child in field.fields.items():
+                errors.extend(
+                    _check_field(f"{name}.{child_name}", child, value.get(child_name))
+                )
+        return errors
+
+    expected = _simple_type(field.type)
+    if expected is not None and not isinstance(value, expected):
+        # JSON numbers may arrive as int for number fields — accept both
+        if field.type == "number" and isinstance(value, (int, float)):
+            return errors
+        errors.append(
+            f"field '{name}' expected {field.type}, got {type(value).__name__}"
+        )
+    return errors
+
+
+def _is_empty(value: Any) -> bool:
+    if value is None:
+        return True
+    if isinstance(value, str) and not value.strip():
+        return True
+    return False
+
+
+def _simple_type(field_type: str) -> type | tuple[type, ...] | None:
+    return {
+        "string": str,
+        "integer": int,
+        "boolean": bool,
+        "number": (int, float),
+    }.get(field_type)

folioiq-0.1.0.dist-info/METADATA

@@ -0,0 +1,98 @@
+Metadata-Version: 2.5
+Name: folioiq
+Version: 0.1.0
+Summary: Schema-driven document extraction with pluggable providers
+Project-URL: Homepage, https://github.com/auspicious123/folioiq-sdk
+Project-URL: Repository, https://github.com/auspicious123/folioiq-sdk
+Project-URL: Issues, https://github.com/auspicious123/folioiq-sdk/issues
+Author-email: auspicious123 <auspicious123@users.noreply.github.com>
+License: MIT
+License-File: LICENSE
+Keywords: azure,document-extraction,llm,ocr,pdf,pydantic
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.12
+Requires-Dist: pydantic-settings>=2.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pymupdf>=1.24.0
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: azure
+Requires-Dist: azure-ai-documentintelligence>=1.0.0; extra == 'azure'
+Requires-Dist: langchain-core>=0.3.0; extra == 'azure'
+Requires-Dist: langchain-openai>=0.2.0; extra == 'azure'
+Provides-Extra: dev
+Requires-Dist: build>=1.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: twine>=5.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# FolioIQ
+
+[![PyPI](https://img.shields.io/badge/pypi-folioiq-blue)](https://pypi.org/project/folioiq/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+
+Schema-driven document extraction. YAML spec (+ optional MD) in → structured data out.
+
+```python
+from folioiq import DocumentExtractor
+
+extractor = DocumentExtractor.from_env()
+result = extractor.extract("invoice.pdf", spec="examples/invoice.yaml")
+
+if result.is_valid:
+    print(result.data)
+else:
+    print(result.validation_errors)
+```
+
+## Install
+
+```bash
+pip install folioiq
+pip install "folioiq[azure]"   # Azure OpenAI + Document Intelligence
+```
+
+Or from source:
+
+```bash
+git clone https://github.com/auspicious123/folioiq-sdk.git
+cd folioiq-sdk
+python -m venv .venv && source .venv/bin/activate
+pip install -e ".[azure,dev]"
+cp .env.example .env   # fill Azure vars
+```
+
+## Spec only (no LLM)
+
+```python
+spec = extractor.load_spec("examples/invoice.yaml")
+Model = extractor.build_model(spec)
+```
+
+## Pipeline override (YAML)
+
+```yaml
+pipeline:
+  mode: explicit
+  parser: azure_di   # or pymupdf
+```
+
+## Eval
+
+```bash
+pip install -e ".[azure,dev]"
+folioiq-eval                         # all fixture packs
+folioiq-eval tests/fixtures/packs/po
+```
+
+## License
+
+MIT — see [LICENSE](LICENSE).
+
+See [PLAN.md](PLAN.md) for design phases.

folioiq-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+folioiq/__init__.py,sha256=5URI84rXr4oIYvEbtH8ZIhzte9lFxO85PgDQBL-Ootk,531
+folioiq/azure_di.py,sha256=smdpgCTrtp_GtAYGt_atFM_FYeLH_XveeVCNWX8KWwE,2058
+folioiq/client.py,sha256=u9sCjJBloZ7INvR-evY48O2sHtCLjLzMXzKqgvfLPn0,5818
+folioiq/config.py,sha256=yAm_4SpzC7z9T3n8dpv-QhWren8KRRYFjLeEJ_EaUkY,996
+folioiq/eval.py,sha256=PDvoNPRHSIi5JTKZeSeoS2pGDADP_FB_U8q9HEUAntM,5875
+folioiq/exceptions.py,sha256=osKS17RU9Ljaguu2ghr89R1CmtkE4d3l8f2t1vgKAJ4,277
+folioiq/io.py,sha256=LslTFhR5JYYhkrdnqjgPo0QXJ9Uh-rz9JT7rV7cMgdI,1480
+folioiq/llm.py,sha256=zDomhoHhDdNUcagC3FRF6J63ig9cfG0EwKsKAbtr8ew,2653
+folioiq/models.py,sha256=x9BdT_SIw-7ZZT3FqrDssa7LrRY5z5rrAOaPTPyCJqE,431
+folioiq/pdf_text.py,sha256=ZpRSQyfHe_eVdcPnXHQ80m-v5vTOCKnBUZKzEVDdK8g,1522
+folioiq/prompt.py,sha256=7pJ8bv95SSq-scu3S_oATbVdhH50Evnk4I-4ok4SjAg,1662
+folioiq/router.py,sha256=hxIz8rCH5uyzmelVPL2NDUIevsWWAEefqt3s13einnE,1701
+folioiq/specs.py,sha256=PJgc7eWj4qLg1PWs5SUvWDbkIWjAcfXO1ggT9CAZyHo,5350
+folioiq/validation.py,sha256=HtkoJ3Wwqrt1afZ1_AFAIl6nQNFThq6L12IFs_rDJmU,2680
+folioiq-0.1.0.dist-info/METADATA,sha256=j-bz1NsyH9pL44-Fz-smIZco8ca8Qn6qiW4F78cKoSI,2758
+folioiq-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+folioiq-0.1.0.dist-info/entry_points.txt,sha256=Ar2I0Y11v71Z0bTQwtn7969hJ6UwzUKDisypp4rtsZg,51
+folioiq-0.1.0.dist-info/licenses/LICENSE,sha256=aLJXfZyz4RfMsiBPap0jm6YZtCn0nT85T2C_9tXDPW4,1077
+folioiq-0.1.0.dist-info/RECORD,,

folioiq-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

folioiq-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+folioiq-eval = folioiq.eval:main

folioiq-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 FolioIQ contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.