marvisx-cli 0.1.0__py3-none-any.whl

core/api/services/ingest/parsers/image_parser.py

@@ -0,0 +1,251 @@
+"""Standalone image parser for Universal Ingestion phase 1."""
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from PIL import Image
+
+from core.api.services.ingest.parsers.docparse_gateway import parse_docparse_with_gateway
+from core.api.services.ingest.parsers.gateway_aux import MissingGatewayConfig
+from core.api.services.ingest.parsers.gateway_aux import settings as gateway_settings
+from core.api.services.ingest.parsers.ocr_gateway import parse_ocr_with_gateway
+
+logger = logging.getLogger(__name__)
+
+SUPPORTED_IMAGE_SUFFIXES = frozenset({".png", ".jpg", ".jpeg", ".webp"})
+UNSUPPORTED_PHASE1_SUFFIXES = frozenset({".avif", ".heic", ".heif"})
+EXIF_GPS_TAG = 0x8825
+EXIF_ORIENTATION_TAG = 0x0112
+IMAGE_MIME_BY_SUFFIX = {
+    ".jpeg": "image/jpeg",
+    ".jpg": "image/jpeg",
+    ".png": "image/png",
+    ".webp": "image/webp",
+}
+_HTML_TAG_RE = re.compile(r"<[^>]+>")
+_IMAGE_MARKUP_RE = re.compile(r"(<\s*img\b|!\[[^\]]*\]\()", re.IGNORECASE)
+_MARKDOWN_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]*\)")
+
+
+def _validate_image_suffix(image_path: Path) -> None:
+    suffix = image_path.suffix.lower()
+    if suffix in UNSUPPORTED_PHASE1_SUFFIXES:
+        raise ValueError(f"unsupported_phase1_image_ext: {suffix}")
+    if suffix not in SUPPORTED_IMAGE_SUFFIXES:
+        raise ValueError(f"unsupported_image_ext: {suffix}")
+
+
+def _image_result(
+    image_path: Path,
+    *,
+    exif_result: dict[str, Any],
+    ocr_result: dict[str, Any],
+) -> dict[str, Any]:
+    post_redact_sha = hashlib.sha256(image_path.read_bytes()).hexdigest()
+
+    structure = {
+        "kind": "image",
+        "format": exif_result["format"],
+        "width": exif_result["width"],
+        "height": exif_result["height"],
+        "mode": exif_result["mode"],
+        "exif_redacted": exif_result["exif_redacted"],
+        "exif_removed": exif_result["exif_removed"],
+        "sha256_post_redact": post_redact_sha,
+        "ocr_confidence_avg": ocr_result.get("confidence_avg", 0.0),
+        "ocr_lines": ocr_result.get("lines") or [],
+    }
+    if ocr_result.get("raw") is not None:
+        structure["ocr_raw"] = ocr_result["raw"]
+    return {
+        "frontmatter": {},
+        "text": str(ocr_result.get("extracted_text") or ""),
+        "structure": structure,
+        "parser_used": str(ocr_result.get("parser_used") or "none"),
+        "extracted_text": str(ocr_result.get("extracted_text") or ""),
+        "exif_redacted": exif_result["exif_redacted"],
+        "exif_removed": exif_result["exif_removed"],
+        "sha256_post_redact": post_redact_sha,
+        "ocr_confidence_avg": ocr_result.get("confidence_avg", 0.0),
+    }
+
+
+def parse_image(path: str | Path) -> dict[str, Any]:
+    """Parse a standalone image and redact privacy-sensitive EXIF metadata."""
+    image_path = Path(path)
+    _validate_image_suffix(image_path)
+    exif_result = _strip_exif_privacy(image_path)
+    return _image_result(
+        image_path,
+        exif_result=exif_result,
+        ocr_result=_ocr_with_fallback(image_path),
+    )
+
+
+async def parse_image_with_gateway(
+    path: str | Path,
+    mime_type: str | None = None,
+    *,
+    prefer_docparse: bool | None = None,
+    docparse_mode: str | None = None,
+) -> dict[str, Any]:
+    """Parse an image with the Mac Gateway OCR endpoint when configured."""
+    image_path = Path(path)
+    _validate_image_suffix(image_path)
+    exif_result = _strip_exif_privacy(image_path)
+    upload_mime = mime_type or IMAGE_MIME_BY_SUFFIX[image_path.suffix.lower()]
+    cfg = gateway_settings()
+    docparse_enabled = getattr(cfg, "ingest_docparse_enabled", False) and getattr(
+        cfg, "ingest_docparse_images_enabled", True
+    )
+    if prefer_docparse is not None:
+        docparse_enabled = bool(prefer_docparse)
+    if docparse_enabled:
+        try:
+            docparse_result = await parse_docparse_with_gateway(
+                image_path,
+                upload_mime,
+                mode=docparse_mode,
+            )
+            if _docparse_result_is_image_only(docparse_result):
+                logger.warning(
+                    "tier-docparse returned image-only markup; falling back to OCR: path=%s",
+                    image_path,
+                )
+            else:
+                return _image_docparse_result(
+                    image_path,
+                    exif_result=exif_result,
+                    docparse_result=docparse_result,
+                )
+        except MissingGatewayConfig:
+            logger.warning("tier-docparse not configured; falling back to OCR")
+
+    try:
+        ocr_result = await parse_ocr_with_gateway(
+            image_path,
+            upload_mime,
+        )
+    except MissingGatewayConfig:
+        ocr_result = _ocr_with_fallback(image_path)
+    return _image_result(
+        image_path,
+        exif_result=exif_result,
+        ocr_result=ocr_result,
+    )
+
+
+def _docparse_result_is_image_only(docparse_result: dict[str, Any]) -> bool:
+    text = str(docparse_result.get("text") or "")
+    markdown = str(docparse_result.get("markdown") or "")
+    raw = f"{text}\n{markdown}"
+    if not _IMAGE_MARKUP_RE.search(raw):
+        return False
+    meaningful = _meaningful_docparse_text(raw)
+    if len(meaningful) >= 40:
+        return False
+    return True
+
+
+def _meaningful_docparse_text(raw: str) -> str:
+    without_markdown_images = _MARKDOWN_IMAGE_RE.sub(" ", raw)
+    without_html = _HTML_TAG_RE.sub(" ", without_markdown_images)
+    tokens = re.findall(r"\b[^\W\d_]{2,}\b", without_html, flags=re.UNICODE)
+    return " ".join(tokens)
+
+
+def _image_docparse_result(
+    image_path: Path,
+    *,
+    exif_result: dict[str, Any],
+    docparse_result: dict[str, Any],
+) -> dict[str, Any]:
+    text = str(docparse_result.get("text") or docparse_result.get("markdown") or "")
+    structure = {
+        "kind": "image",
+        "format": exif_result["format"],
+        "width": exif_result["width"],
+        "height": exif_result["height"],
+        "mode": exif_result["mode"],
+        "exif_redacted": exif_result["exif_redacted"],
+        "exif_removed": exif_result["exif_removed"],
+        "sha256_post_redact": hashlib.sha256(image_path.read_bytes()).hexdigest(),
+        "docparse_backend": "tier_docparse",
+        "page_count": docparse_result.get("page_count"),
+        "elements_count": len(docparse_result.get("elements") or []),
+        "elements": docparse_result.get("elements") or [],
+        "pages": docparse_result.get("pages") or [],
+        "metadata": docparse_result.get("metadata") or {},
+    }
+    return {
+        "frontmatter": {},
+        "text": text,
+        "structure": structure,
+        "parser_used": "tier_docparse",
+        "extracted_text": text,
+        "exif_redacted": exif_result["exif_redacted"],
+        "exif_removed": exif_result["exif_removed"],
+        "sha256_post_redact": structure["sha256_post_redact"],
+    }
+
+
+def _strip_exif_privacy(path: Path) -> dict[str, Any]:
+    """Remove GPS and non-orientation EXIF while preserving display orientation."""
+    try:
+        with Image.open(path) as image:
+            exif = image.getexif()
+            removed = False
+            if exif:
+                orientation = exif.get(EXIF_ORIENTATION_TAG)
+                sanitized = Image.Exif()
+                if orientation is not None:
+                    sanitized[EXIF_ORIENTATION_TAG] = orientation
+                removed = set(exif.keys()) != set(sanitized.keys()) or any(
+                    exif.get(key) != sanitized.get(key) for key in sanitized.keys()
+                )
+                if EXIF_GPS_TAG in exif:
+                    removed = True
+                if removed:
+                    image.save(path, exif=sanitized.tobytes() if sanitized else b"")
+            return {
+                "format": image.format,
+                "width": image.width,
+                "height": image.height,
+                "mode": image.mode,
+                "exif_redacted": True,
+                "exif_removed": removed,
+            }
+    except Exception as exc:
+        raise ValueError(f"invalid_image: {exc}") from exc
+
+
+def _ocr_with_fallback(path: Path) -> dict[str, Any]:
+    """Run Tesseract OCR locally (PaddleOCR-VL via Mac Gateway tier-ocr upstream)."""
+    try:
+        import pytesseract
+
+        with Image.open(path) as image:
+            try:
+                text = pytesseract.image_to_string(image, lang="ita+eng")
+            except Exception:
+                logger.warning("Tesseract ita+eng failed, retrying default language", exc_info=True)
+                text = pytesseract.image_to_string(image)
+        return {
+            "extracted_text": str(text),
+            "parser_used": "tesseract",
+            "confidence_avg": 0.0,
+            "lines": [],
+        }
+    except Exception as exc:
+        logger.warning("No OCR backend available for image parser: %s", exc)
+        return {
+            "extracted_text": "",
+            "parser_used": "none",
+            "confidence_avg": 0.0,
+            "lines": [],
+        }

core/api/services/ingest/parsers/internal_markdown.py

@@ -0,0 +1,89 @@
+"""Internal Markdown parser used by phase-1 ingestion."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, Field, ValidationError, field_validator
+
+
+MAX_MARKDOWN_BYTES = 5 * 1024 * 1024
+_FRONTMATTER_RE = re.compile(r"\A---\s*\n(.*?)\n---\s*(?:\n|\Z)", re.DOTALL)
+_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$", re.MULTILINE)
+
+
+class MarkdownFrontmatter(BaseModel):
+    type: str | None = Field(default=None, max_length=60)
+    title: str | None = Field(default=None, max_length=300)
+    tags: list[str] = Field(default_factory=list, max_length=20)
+
+    @field_validator("tags", mode="before")
+    @classmethod
+    def normalize_tags(cls, value: Any) -> list[str]:
+        if value is None:
+            return []
+        if isinstance(value, str):
+            return [value]
+        if isinstance(value, list):
+            return [str(v) for v in value if v is not None]
+        return []
+
+
+@dataclass(frozen=True)
+class MarkdownParseResult:
+    frontmatter: dict[str, Any]
+    text: str
+    structure: dict[str, Any]
+
+
+def _split_frontmatter(raw: str) -> tuple[dict[str, Any], str]:
+    match = _FRONTMATTER_RE.match(raw)
+    if not match:
+        return {}, raw
+
+    frontmatter_raw = match.group(1)
+    body = raw[match.end() :]
+    loaded = yaml.safe_load(frontmatter_raw) or {}
+    if not isinstance(loaded, dict):
+        loaded = {}
+
+    try:
+        model = MarkdownFrontmatter.model_validate(loaded)
+    except ValidationError:
+        model = MarkdownFrontmatter()
+    normalized = model.model_dump()
+    extra = {k: v for k, v in loaded.items() if k not in normalized}
+    return {**extra, **normalized}, body
+
+
+def _extract_headings(text: str) -> list[dict[str, Any]]:
+    headings: list[dict[str, Any]] = []
+    for match in _HEADING_RE.finditer(text):
+        headings.append(
+            {
+                "level": len(match.group(1)),
+                "text": match.group(2).strip()[:300],
+            }
+        )
+    return headings[:100]
+
+
+def parse_markdown_file(path: Path) -> MarkdownParseResult:
+    size = path.stat().st_size
+    if size > MAX_MARKDOWN_BYTES:
+        raise ValueError(f"Markdown file too large: {size} bytes")
+
+    raw = path.read_text(encoding="utf-8", errors="replace")
+    frontmatter, body = _split_frontmatter(raw)
+    return MarkdownParseResult(
+        frontmatter=frontmatter,
+        text=body,
+        structure={
+            "headings": _extract_headings(body),
+            "bytes": size,
+            "line_count": body.count("\n") + (1 if body else 0),
+        },
+    )

core/api/services/ingest/parsers/ocr_gateway.py

@@ -0,0 +1,117 @@
+"""OCR parser backed by the Mac Gateway AUX /v1/ocr endpoint."""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from core.api.services.ingest.parsers.gateway_aux import (
+    auth_headers,
+    aux_base_url,
+    request_gateway_with_retries,
+    settings,
+)
+
+
+def _normalize_lines(raw: Any) -> list[dict[str, Any]]:
+    if not isinstance(raw, list):
+        return []
+    lines: list[dict[str, Any]] = []
+    for item in raw:
+        if not isinstance(item, dict):
+            continue
+        text = str(item.get("text") or item.get("content") or "").strip()
+        if not text:
+            continue
+        line: dict[str, Any] = {"text": text}
+        confidence = item.get("confidence", item.get("score"))
+        if confidence is not None:
+            try:
+                line["confidence"] = float(confidence)
+            except (TypeError, ValueError):
+                pass
+        if item.get("bbox") is not None:
+            line["bbox"] = item["bbox"]
+        if item.get("page") is not None:
+            line["page"] = item["page"]
+        lines.append(line)
+    return lines
+
+
+def _confidence_avg(lines: list[dict[str, Any]], payload: dict[str, Any]) -> float:
+    explicit = payload.get("confidence_avg", payload.get("avg_confidence"))
+    if explicit is not None:
+        try:
+            return float(explicit)
+        except (TypeError, ValueError):
+            pass
+    values = [
+        float(line["confidence"])
+        for line in lines
+        if isinstance(line.get("confidence"), (int, float))
+    ]
+    return sum(values) / len(values) if values else 0.0
+
+
+def _text_from_payload(payload: dict[str, Any], lines: list[dict[str, Any]]) -> str:
+    direct = payload.get("text") or payload.get("extracted_text")
+    if direct:
+        return str(direct).strip()
+    return "\n".join(str(line["text"]) for line in lines).strip()
+
+
+async def parse_ocr_with_gateway(path: Path, mime_type: str) -> dict[str, Any]:
+    cfg = settings()
+    size = path.stat().st_size
+    if size > int(cfg.ingest_ocr_max_bytes):
+        raise ValueError(f"File too large for OCR: {size} bytes")
+
+    timeout = httpx.Timeout(
+        connect=5.0,
+        read=float(cfg.ingest_ocr_timeout_seconds),
+        write=60.0,
+        pool=5.0,
+    )
+    async with httpx.AsyncClient(
+        base_url=f"{aux_base_url()}/",
+        timeout=timeout,
+        limits=httpx.Limits(max_connections=4, max_keepalive_connections=2),
+    ) as client:
+        async def post_ocr() -> httpx.Response:
+            with path.open("rb") as handle:
+                return await client.post(
+                    "ocr",
+                    headers=auth_headers(),
+                    files={"file": (path.name, handle, mime_type)},
+                )
+
+        response = await request_gateway_with_retries(
+            post_ocr,
+            service_name="tier-ocr",
+            max_attempts=5,
+        )
+
+    if response.status_code in {401, 403}:
+        raise RuntimeError("tier-ocr authorization failed")
+    if response.status_code == 429:
+        raise RuntimeError("tier-ocr rate limited")
+    if response.status_code >= 500:
+        raise RuntimeError(f"tier-ocr unavailable: HTTP {response.status_code}")
+    response.raise_for_status()
+    payload = response.json()
+    if not isinstance(payload, dict):
+        raise RuntimeError("tier-ocr returned invalid payload")
+
+    lines = _normalize_lines(payload.get("lines"))
+    text = _text_from_payload(payload, lines)
+    if not text:
+        raise RuntimeError("tier-ocr returned empty text")
+
+    return {
+        "extracted_text": text,
+        "parser_used": "tier_ocr",
+        "confidence_avg": _confidence_avg(lines, payload),
+        "lines": lines,
+        "raw": payload,
+    }

core/api/services/ingest/parsers/ocr_pdf_parser.py

@@ -0,0 +1,112 @@
+"""OCR fallback for scanned PDFs."""
+from __future__ import annotations
+
+import asyncio
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from core.api.services.ingest.parsers.gateway_aux import MissingGatewayConfig
+from core.api.services.ingest.parsers.ocr_gateway import parse_ocr_with_gateway
+from core.api.services.ingest.parsers.pdf_types import PdfParseResult
+
+logger = logging.getLogger(__name__)
+
+OCR_TIMEOUT_SECONDS = 180
+OCR_WORKERS = 2
+
+
+@lru_cache(maxsize=1)
+def _executor() -> ThreadPoolExecutor:
+    return ThreadPoolExecutor(max_workers=OCR_WORKERS, thread_name_prefix="pdf-ocr")
+
+
+@lru_cache(maxsize=1)
+def _rapidocr_converter() -> Any:
+    try:
+        from docling.datamodel.base_models import InputFormat
+        from docling.datamodel.pipeline_options import (
+            PdfPipelineOptions,
+            RapidOcrOptions,
+        )
+        from docling.document_converter import DocumentConverter, PdfFormatOption
+    except ImportError as exc:
+        raise RuntimeError("Docling with RapidOCR is required for PDF OCR fallback") from exc
+
+    pipeline_options = PdfPipelineOptions(
+        do_ocr=True,
+        ocr_options=RapidOcrOptions(
+            backend="onnxruntime",
+            force_full_page_ocr=True,
+            lang=["english"],
+        ),
+    )
+    return DocumentConverter(
+        allowed_formats=[InputFormat.PDF],
+        format_options={
+            InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options),
+        },
+    )
+
+
+def _ocr_sync(file_path: Path) -> PdfParseResult:
+    result = _rapidocr_converter().convert(str(file_path))
+    document = result.document
+    markdown = str(document.export_to_markdown() or "").strip()
+    structure = _safe_doc_dict(document)
+    pages = getattr(document, "pages", None)
+    page_count = len(pages) if pages is not None else None
+    if page_count is not None:
+        structure.setdefault("page_count", page_count)
+    structure.update(
+        {
+            "ocr_backend": "rapidocr",
+            "ocr_engine": "docling_rapidocr",
+        }
+    )
+    return PdfParseResult(
+        frontmatter={},
+        text=markdown,
+        structure=structure,
+        parser_used="rapidocr_fallback",
+    )
+
+
+def _safe_doc_dict(document: Any) -> dict[str, Any]:
+    if hasattr(document, "export_to_dict"):
+        value = document.export_to_dict()
+        return value if isinstance(value, dict) else {"document": value}
+    if hasattr(document, "dict"):
+        value = document.dict()
+        return value if isinstance(value, dict) else {"document": value}
+    return {}
+
+
+async def parse_pdf_ocr(file_path: Path) -> PdfParseResult:
+    try:
+        ocr = await parse_ocr_with_gateway(file_path, "application/pdf")
+        raw = ocr.get("raw") or {}
+        return PdfParseResult(
+            frontmatter={},
+            text=str(ocr["extracted_text"]),
+            structure={
+                "pages": raw.get("pages"),
+                "ocr_backend": "tier_ocr",
+                "ocr_confidence_avg": ocr.get("confidence_avg", 0.0),
+                "ocr_lines": ocr.get("lines") or [],
+                "ocr_raw": raw,
+            },
+            parser_used="tier_ocr",
+        )
+    except MissingGatewayConfig:
+        pass
+
+    loop = asyncio.get_running_loop()
+    future = loop.run_in_executor(_executor(), _ocr_sync, file_path)
+    try:
+        return await asyncio.wait_for(future, timeout=OCR_TIMEOUT_SECONDS)
+    except asyncio.TimeoutError as exc:
+        future.cancel()
+        raise TimeoutError(f"OCR timeout {OCR_TIMEOUT_SECONDS}s") from exc

core/api/services/ingest/parsers/pdf_types.py

@@ -0,0 +1,13 @@
+"""Shared PDF parser result types."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class PdfParseResult:
+    frontmatter: dict[str, Any]
+    text: str
+    structure: dict[str, Any]
+    parser_used: str