marvisx-cli 0.1.0__py3-none-any.whl

core/api/services/ingest/parsers/__init__.py

@@ -0,0 +1 @@
+"""File parsers for universal ingestion."""

core/api/services/ingest/parsers/docling_parser.py

@@ -0,0 +1,142 @@
+"""PDF parser using Docling as the primary extractor."""
+from __future__ import annotations
+
+import asyncio
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from core.api.services.ingest.parsers.ocr_pdf_parser import parse_pdf_ocr
+from core.api.services.ingest.parsers.docparse_gateway import parse_pdf_docparse
+from core.api.services.ingest.parsers.gateway_aux import MissingGatewayConfig, settings
+from core.api.services.ingest.parsers.pdf_types import PdfParseResult
+
+logger = logging.getLogger(__name__)
+
+DOCLING_TIMEOUT_SECONDS = 120
+DOCLING_WORKERS = 3
+MIN_TEXT_FOR_OCR_FALLBACK = 100
+
+
+@lru_cache(maxsize=1)
+def _executor() -> ThreadPoolExecutor:
+    return ThreadPoolExecutor(max_workers=DOCLING_WORKERS, thread_name_prefix="docling")
+
+
+@lru_cache(maxsize=1)
+def _converter():
+    from docling.document_converter import DocumentConverter
+
+    return DocumentConverter()
+
+
+def _is_encrypted(file_path: Path) -> bool:
+    try:
+        from pypdf import PdfReader
+
+        return bool(PdfReader(str(file_path)).is_encrypted)
+    except ImportError:
+        logger.warning("pypdf not installed; skipping PDF encryption preflight")
+        return False
+    except Exception:
+        logger.warning("PDF encryption preflight failed for %s", file_path, exc_info=True)
+        return False
+
+
+def _safe_doc_dict(document: Any) -> dict[str, Any]:
+    if hasattr(document, "export_to_dict"):
+        value = document.export_to_dict()
+        return value if isinstance(value, dict) else {"document": value}
+    if hasattr(document, "dict"):
+        value = document.dict()
+        return value if isinstance(value, dict) else {"document": value}
+    return {}
+
+
+def _count_tables(document: Any, structure: dict[str, Any]) -> int:
+    tables = getattr(document, "tables", None)
+    if isinstance(tables, list):
+        return len(tables)
+    if hasattr(document, "iterate_items"):
+        try:
+            return sum(
+                1
+                for item in document.iterate_items()
+                if str(getattr(item, "label", "")).lower() == "table"
+            )
+        except Exception:
+            logger.debug("Docling table iteration failed", exc_info=True)
+    return _count_table_like_entries(structure)
+
+
+def _count_table_like_entries(value: Any) -> int:
+    if isinstance(value, dict):
+        count = 1 if str(value.get("label", "")).lower() == "table" else 0
+        return count + sum(_count_table_like_entries(item) for item in value.values())
+    if isinstance(value, list):
+        return sum(_count_table_like_entries(item) for item in value)
+    return 0
+
+
+def _convert_with_docling(file_path: Path) -> PdfParseResult:
+    result = _converter().convert(str(file_path))
+    document = result.document
+    markdown = document.export_to_markdown()
+    structure = _safe_doc_dict(document)
+    structure.setdefault("tables_count", _count_tables(document, structure))
+    pages = getattr(document, "pages", None)
+    if pages is not None:
+        structure.setdefault("page_count", len(pages))
+    return PdfParseResult(
+        frontmatter={},
+        text=markdown,
+        structure=structure,
+        parser_used="docling",
+    )
+
+
+def _convert_sync(file_path: Path) -> PdfParseResult:
+    return _convert_with_docling(file_path)
+
+
+async def parse_pdf_file(
+    file_path: Path,
+    *,
+    allow_docparse: bool = True,
+    docparse_mode: str | None = None,
+) -> PdfParseResult:
+    """Parse a PDF and run OCR fallback when Docling finds too little text."""
+    if _is_encrypted(file_path):
+        raise ValueError("PDF encrypted, requires password")
+
+    cfg = settings()
+    if allow_docparse and getattr(cfg, "ingest_docparse_enabled", False) and getattr(
+        cfg, "ingest_docparse_pdfs_enabled", True
+    ):
+        try:
+            return await parse_pdf_docparse(file_path, mode=docparse_mode)
+        except MissingGatewayConfig:
+            logger.warning("tier-docparse not configured; falling back to Docling")
+
+    loop = asyncio.get_running_loop()
+    future = loop.run_in_executor(_executor(), _convert_sync, file_path)
+    try:
+        parsed = await asyncio.wait_for(future, timeout=DOCLING_TIMEOUT_SECONDS)
+    except asyncio.TimeoutError as exc:
+        future.cancel()
+        raise TimeoutError(f"parse timeout {DOCLING_TIMEOUT_SECONDS}s") from exc
+    except Exception:
+        logger.warning(
+            "Docling parse failed; trying OCR fallback: %s",
+            file_path,
+            exc_info=True,
+        )
+        return await parse_pdf_ocr(file_path)
+
+    if len(parsed.text.strip()) >= MIN_TEXT_FOR_OCR_FALLBACK:
+        return parsed
+
+    logger.info("Docling extracted <100 chars, trying OCR fallback: %s", file_path)
+    return await parse_pdf_ocr(file_path)

core/api/services/ingest/parsers/docparse_gateway.py

@@ -0,0 +1,178 @@
+"""Document parser backed by the Mac Gateway AUX /v1/docparse endpoint."""
+from __future__ import annotations
+
+import asyncio
+import time
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from core.api.services.ingest.parsers.gateway_aux import (
+    auth_headers,
+    aux_base_url,
+    request_gateway_with_retries,
+    settings,
+)
+from core.api.services.ingest.parsers.pdf_types import PdfParseResult
+
+
+async def parse_docparse_with_gateway(
+    path: Path,
+    mime_type: str,
+    *,
+    mode: str | None = None,
+) -> dict[str, Any]:
+    cfg = settings()
+    size = path.stat().st_size
+    if size > int(cfg.ingest_docparse_max_bytes):
+        raise ValueError(f"File too large for docparse: {size} bytes")
+
+    timeout_seconds = float(cfg.ingest_docparse_timeout_seconds)
+    timeout = httpx.Timeout(
+        connect=5.0,
+        read=timeout_seconds,
+        write=60.0,
+        pool=5.0,
+    )
+    async with httpx.AsyncClient(
+        base_url=f"{aux_base_url()}/",
+        timeout=timeout,
+        limits=httpx.Limits(max_connections=2, max_keepalive_connections=1),
+    ) as client:
+        async def post_docparse() -> httpx.Response:
+            with path.open("rb") as handle:
+                return await client.post(
+                    "docparse",
+                    headers=auth_headers(),
+                    data={
+                        "output_format": "markdown",
+                        "mode": mode or cfg.ingest_docparse_mode,
+                    },
+                    files={"file": (path.name, handle, mime_type)},
+                )
+
+        response = await request_gateway_with_retries(
+            post_docparse,
+            service_name="tier-docparse",
+        )
+        payload = await _resolve_docparse_response(
+            client,
+            response,
+            timeout_seconds=timeout_seconds,
+        )
+
+    text = str(payload.get("text") or "").strip()
+    markdown = str(payload.get("markdown") or "").strip()
+    if not text and markdown:
+        text = markdown
+    if not text:
+        raise RuntimeError("tier-docparse returned empty text")
+
+    return {
+        "text": text,
+        "markdown": markdown,
+        "page_count": payload.get("page_count"),
+        "elements": payload.get("elements") or [],
+        "pages": payload.get("pages") or [],
+        "metadata": payload.get("metadata") or {},
+        "raw": payload,
+    }
+
+
+async def parse_pdf_docparse(path: Path, *, mode: str | None = None) -> PdfParseResult:
+    parsed = await parse_docparse_with_gateway(path, "application/pdf", mode=mode)
+    return PdfParseResult(
+        frontmatter={},
+        text=parsed["markdown"] or parsed["text"],
+        structure={
+            "page_count": parsed.get("page_count"),
+            "elements_count": len(parsed.get("elements") or []),
+            "elements": parsed.get("elements") or [],
+            "pages": parsed.get("pages") or [],
+            "metadata": parsed.get("metadata") or {},
+            "docparse_backend": "tier_docparse",
+        },
+        parser_used="tier_docparse",
+    )
+
+
+async def _resolve_docparse_response(
+    client: httpx.AsyncClient,
+    response: httpx.Response,
+    *,
+    timeout_seconds: float,
+) -> dict[str, Any]:
+    if response.status_code in {401, 403}:
+        raise RuntimeError("tier-docparse authorization failed")
+    if response.status_code == 429:
+        raise RuntimeError("tier-docparse rate limited")
+    if response.status_code >= 500:
+        raise RuntimeError(f"tier-docparse unavailable: HTTP {response.status_code}")
+    response.raise_for_status()
+    payload = response.json()
+    if not isinstance(payload, dict):
+        raise RuntimeError("tier-docparse returned invalid payload")
+    if response.status_code != 202:
+        return payload
+
+    status_url = str(payload.get("status_url") or "").strip()
+    if not status_url:
+        raise RuntimeError("tier-docparse async response missing status_url")
+    return await _poll_docparse_job(client, status_url, timeout_seconds=timeout_seconds)
+
+
+async def _poll_docparse_job(
+    client: httpx.AsyncClient,
+    status_url: str,
+    *,
+    timeout_seconds: float,
+) -> dict[str, Any]:
+    deadline = time.monotonic() + timeout_seconds
+    request_url = _status_request_url(client, status_url)
+    while time.monotonic() < deadline:
+        async def get_status() -> httpx.Response:
+            return await client.get(request_url, headers=auth_headers())
+
+        response = await request_gateway_with_retries(
+            get_status,
+            service_name="tier-docparse-poll",
+        )
+        if response.status_code in {401, 403}:
+            raise RuntimeError("tier-docparse poll authorization failed")
+        if response.status_code == 404:
+            raise RuntimeError("tier-docparse job not found")
+        if response.status_code >= 500:
+            raise RuntimeError(f"tier-docparse poll unavailable: HTTP {response.status_code}")
+        response.raise_for_status()
+        status_payload = response.json()
+        if not isinstance(status_payload, dict):
+            raise RuntimeError("tier-docparse poll returned invalid payload")
+
+        status = status_payload.get("status")
+        if status == "done":
+            result = status_payload.get("result")
+            if not isinstance(result, dict):
+                raise RuntimeError("tier-docparse completed without result")
+            return result
+        if status == "failed":
+            error = status_payload.get("error") or {}
+            message = error.get("message") if isinstance(error, dict) else None
+            raise RuntimeError(f"tier-docparse job failed: {message or 'unknown error'}")
+
+        retry_after = response.headers.get("Retry-After")
+        try:
+            sleep_for = float(retry_after) if retry_after else 2.0
+        except ValueError:
+            sleep_for = 2.0
+        await asyncio.sleep(max(0.2, min(sleep_for, 5.0)))
+
+    raise TimeoutError(f"tier-docparse job timed out after {timeout_seconds:g}s")
+
+
+def _status_request_url(client: httpx.AsyncClient, status_url: str) -> str:
+    if status_url.startswith(("http://", "https://")):
+        return status_url
+    if status_url.startswith("/"):
+        return str(client.base_url.copy_with(path=status_url, query=None, fragment=None))
+    return status_url

core/api/services/ingest/parsers/docx_parser.py

@@ -0,0 +1,127 @@
+"""Minimal local DOCX parser for Ingestor 2.0."""
+from __future__ import annotations
+
+import zipfile
+from pathlib import Path
+from typing import Any
+from xml.etree import ElementTree
+
+from core.api.config import settings
+from core.api.services.ingest.parsers.internal_markdown import MarkdownParseResult
+
+DOCX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+WORD_NS = {"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"}
+
+
+def parse_docx(path: Path) -> MarkdownParseResult:
+    """Extract paragraphs and tables from an Office Open XML document.
+
+    This deliberately avoids a broad conversion dependency. It is enough for
+    classification evidence and triage preview, while legacy binary `.doc`
+    remains unsupported.
+    """
+    size = path.stat().st_size
+    if size > int(settings.ingest_docx_max_bytes):
+        raise ValueError(f"DOCX file too large: {size} bytes")
+    if path.suffix.lower() != ".docx":
+        raise ValueError(f"Unsupported Word extension: {path.suffix or '<none>'}")
+
+    try:
+        with zipfile.ZipFile(path) as archive:
+            document_xml = archive.read("word/document.xml")
+    except KeyError as exc:
+        raise ValueError("Invalid DOCX: word/document.xml missing") from exc
+    except zipfile.BadZipFile as exc:
+        raise ValueError("Invalid DOCX container") from exc
+
+    root = ElementTree.fromstring(document_xml)
+    paragraphs: list[str] = []
+    tables: list[list[list[str]]] = []
+
+    for child in root.findall(".//w:body/*", WORD_NS):
+        tag = _local_name(child.tag)
+        if tag == "p":
+            text = _paragraph_text(child)
+            if text:
+                paragraphs.append(text)
+        elif tag == "tbl":
+            table = _table_rows(child)
+            if table:
+                tables.append(table)
+
+    markdown = _to_markdown(path, paragraphs, tables)
+    return MarkdownParseResult(
+        frontmatter=_frontmatter(path, markdown),
+        text=markdown,
+        structure={
+            "kind": "docx",
+            "bytes": size,
+            "paragraph_count": len(paragraphs),
+            "table_count": len(tables),
+            "table_cell_count": sum(len(row) for table in tables for row in table),
+        },
+    )
+
+
+def _paragraph_text(node: ElementTree.Element) -> str:
+    return "".join(
+        text_node.text or ""
+        for text_node in node.findall(".//w:t", WORD_NS)
+        if text_node.text
+    ).strip()
+
+
+def _table_rows(node: ElementTree.Element) -> list[list[str]]:
+    rows: list[list[str]] = []
+    for row in node.findall(".//w:tr", WORD_NS):
+        cells = [_paragraph_text(cell) for cell in row.findall("./w:tc", WORD_NS)]
+        if any(cells):
+            rows.append(cells)
+    return rows
+
+
+def _to_markdown(path: Path, paragraphs: list[str], tables: list[list[list[str]]]) -> str:
+    parts = [f"# {path.stem}"]
+    for paragraph in paragraphs:
+        parts.extend(["", paragraph])
+    for index, table in enumerate(tables, start=1):
+        parts.extend(["", f"## Table {index}", "", *_table_markdown(table)])
+    return "\n".join(parts).strip() + "\n"
+
+
+def _table_markdown(rows: list[list[str]]) -> list[str]:
+    if not rows:
+        return []
+    width = max(len(row) for row in rows)
+    normalized = [row + [""] * (width - len(row)) for row in rows]
+    header = normalized[0]
+    body = normalized[1:] or [[""] * width]
+    out = [
+        "| " + " | ".join(_cell(cell) for cell in header) + " |",
+        "| " + " | ".join("---" for _ in header) + " |",
+    ]
+    for row in body:
+        out.append("| " + " | ".join(_cell(cell) for cell in row) + " |")
+    return out
+
+
+def _cell(value: str) -> str:
+    return (value or "").replace("|", "\\|").strip()
+
+
+def _frontmatter(path: Path, markdown: str) -> dict[str, Any]:
+    lowered = markdown.lower()
+    tags = ["docx"]
+    doc_type = "file"
+    if any(term in lowered for term in {"contratto", "agreement", "clausola", "firma"}):
+        doc_type = "contract"
+        tags.append("contract")
+    return {
+        "type": doc_type,
+        "title": path.stem,
+        "tags": tags,
+    }
+
+
+def _local_name(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1]

core/api/services/ingest/parsers/folder_unpacker.py

@@ -0,0 +1,85 @@
+"""Folder traversal helpers for Universal Ingestion containers."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+
+from core.api.services.ingest.ignore_patterns import (
+    is_ignored_directory_name,
+    should_ignore,
+)
+
+
+@dataclass(frozen=True)
+class FolderFile:
+    path: Path
+    relative_path: PurePosixPath
+    size_bytes: int
+
+
+@dataclass(frozen=True)
+class FolderSkipped:
+    path: str
+    reason: str
+
+
+@dataclass(frozen=True)
+class FolderScanResult:
+    files: list[FolderFile]
+    skipped: list[FolderSkipped]
+
+
+def scan_folder_files(root: Path, *, max_depth: int = 5) -> FolderScanResult:
+    root = root.resolve()
+    files: list[FolderFile] = []
+    skipped: list[FolderSkipped] = []
+    visited_dirs: set[Path] = set()
+
+    def walk(directory: Path, rel_dir: PurePosixPath, depth: int) -> None:
+        if depth > max_depth:
+            skipped.append(FolderSkipped(str(rel_dir), "max-depth"))
+            return
+        try:
+            resolved_dir = directory.resolve()
+        except OSError:
+            skipped.append(FolderSkipped(str(rel_dir), "unreadable"))
+            return
+        if resolved_dir in visited_dirs:
+            skipped.append(FolderSkipped(str(rel_dir), "symlink-loop"))
+            return
+        visited_dirs.add(resolved_dir)
+
+        try:
+            children = sorted(directory.iterdir(), key=lambda item: item.name.casefold())
+        except OSError:
+            skipped.append(FolderSkipped(str(rel_dir), "unreadable"))
+            return
+
+        for child in children:
+            rel_path = rel_dir / child.name
+            if child.is_symlink():
+                skipped.append(FolderSkipped(str(rel_path), "symlink"))
+                continue
+            if child.is_dir():
+                if is_ignored_directory_name(child.name):
+                    skipped.append(FolderSkipped(str(rel_path), "ignored-directory"))
+                    continue
+                walk(child, rel_path, depth + 1)
+                continue
+            if not child.is_file():
+                skipped.append(FolderSkipped(str(rel_path), "not-file"))
+                continue
+
+            try:
+                size_bytes = child.stat().st_size
+            except OSError:
+                skipped.append(FolderSkipped(str(rel_path), "unreadable"))
+                continue
+            ignore_reason = should_ignore(rel_path, file_size_bytes=size_bytes)
+            if ignore_reason:
+                skipped.append(FolderSkipped(str(rel_path), ignore_reason))
+                continue
+            files.append(FolderFile(child, rel_path, size_bytes))
+
+    walk(root, PurePosixPath("."), 0)
+    return FolderScanResult(files=files, skipped=skipped)

core/api/services/ingest/parsers/gateway_aux.py

@@ -0,0 +1,147 @@
+"""Shared helpers for Mac Gateway AUX endpoints used by ingest parsers."""
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+from typing import Any
+
+import httpx
+
+from core.api import config as _config
+
+logger = logging.getLogger(__name__)
+
+RETRYABLE_STATUS_CODES = {408, 409, 425, 429, 500, 502, 503, 504}
+RETRY_AFTER_CAP_SECONDS = 30.0
+
+
+class MissingGatewayConfig(RuntimeError):
+    """Raised when AUX endpoints are not configured in this environment."""
+
+
+def settings():
+    return _config.settings
+
+
+def secret_value(value: Any) -> str:
+    if hasattr(value, "get_secret_value"):
+        return str(value.get_secret_value())
+    return str(value or "")
+
+
+def gateway_api_key() -> str:
+    cfg = settings()
+    if getattr(cfg, "pir_env", "") == "test":
+        raise MissingGatewayConfig("AUX gateway is disabled under PIR_ENV=test")
+    api_key = secret_value(getattr(cfg, "ingest_llm_gateway_api_key", None))
+    if not api_key:
+        raise MissingGatewayConfig("INGEST_LLM_GATEWAY_API_KEY is not configured")
+    return api_key
+
+
+def gateway_agent_name() -> str:
+    cfg = settings()
+    return str(getattr(cfg, "ingest_llm_gateway_agent_name", "") or "marvisx-ingester")
+
+
+def gateway_priority() -> str:
+    cfg = settings()
+    priority = str(getattr(cfg, "ingest_llm_gateway_priority", "") or "batch").strip()
+    return priority if priority in {"interactive", "batch", "background"} else "batch"
+
+
+def aux_base_url() -> str:
+    cfg = settings()
+    if getattr(cfg, "pir_env", "") == "test":
+        raise MissingGatewayConfig("AUX gateway is disabled under PIR_ENV=test")
+    base_url = cfg.llm_gateway_aux_base_url or cfg.llm_gateway_base_url
+    if not base_url:
+        raise MissingGatewayConfig("LLM_GATEWAY_BASE_URL is not configured")
+
+    base_url = base_url.rstrip("/")
+    if cfg.llm_gateway_aux_base_url:
+        return base_url
+
+    # Production chat calls use LiteLLM on :4000. The ASR/OCR AUX proxy lives on
+    # :9100 on the same Mac; public CF URLs already route by path and stay as-is.
+    if base_url.startswith("http://100.103.221.55:4000"):
+        return base_url.replace(":4000", ":9100", 1)
+    return base_url
+
+
+def auth_headers() -> dict[str, str]:
+    return {
+        "Authorization": f"Bearer {gateway_api_key()}",
+        "X-Agent-Name": gateway_agent_name(),
+        "X-Priority": gateway_priority(),
+    }
+
+
+def is_retryable_gateway_response(response: httpx.Response) -> bool:
+    return response.status_code in RETRYABLE_STATUS_CODES
+
+
+def retry_delay_seconds(response: httpx.Response | None, attempt: int) -> float:
+    retry_after = response.headers.get("Retry-After") if response is not None else None
+    if retry_after:
+        try:
+            return min(max(float(retry_after), 0.0), RETRY_AFTER_CAP_SECONDS)
+        except ValueError:
+            try:
+                parsed = parsedate_to_datetime(retry_after)
+                if parsed.tzinfo is None:
+                    parsed = parsed.replace(tzinfo=timezone.utc)
+                delta = (parsed - datetime.now(timezone.utc)).total_seconds()
+                return min(max(delta, 0.0), RETRY_AFTER_CAP_SECONDS)
+            except (TypeError, ValueError):
+                pass
+    return min(0.5 * attempt, 2.0)
+
+
+async def request_gateway_with_retries(
+    operation: Callable[[], Awaitable[httpx.Response]],
+    *,
+    service_name: str,
+    max_attempts: int = 3,
+) -> httpx.Response:
+    for attempt in range(1, max_attempts + 1):
+        response: httpx.Response | None = None
+        try:
+            response = await operation()
+        except (
+            httpx.ConnectError,
+            httpx.ConnectTimeout,
+            httpx.PoolTimeout,
+            httpx.ReadTimeout,
+            httpx.RemoteProtocolError,
+            httpx.WriteError,
+            httpx.WriteTimeout,
+        ) as exc:
+            if attempt == max_attempts:
+                raise RuntimeError(
+                    f"{service_name} unavailable after {max_attempts} attempts"
+                ) from exc
+            logger.warning(
+                "%s transient request error on attempt %s/%s: %s",
+                service_name,
+                attempt,
+                max_attempts,
+                exc.__class__.__name__,
+            )
+        else:
+            if not is_retryable_gateway_response(response) or attempt == max_attempts:
+                return response
+            logger.warning(
+                "%s transient HTTP %s on attempt %s/%s",
+                service_name,
+                response.status_code,
+                attempt,
+                max_attempts,
+            )
+
+        await asyncio.sleep(retry_delay_seconds(response, attempt))
+
+    raise RuntimeError(f"{service_name} retry loop exited unexpectedly")