memuron 0.1.1__py3-none-any.whl

memuron/documents/parser.py

@@ -0,0 +1,1152 @@
+"""Document parsing helpers for Memuron rich nodes.
+
+This module deliberately stops at normalized markdown and deterministic chunks.
+The Memuron recipe layer decides how those chunks become semantic events.
+"""
+
+from __future__ import annotations
+
+import base64
+import datetime as dt
+import io
+import json
+import re
+import struct
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import requests
+
+
+MAX_DOCUMENT_UPLOAD_BYTES = 8 * 1024 * 1024
+MAX_SPREADSHEET_CELLS = 100_000
+DEFAULT_TARGET_CHARS = 1_600
+DEFAULT_MAX_CHARS = 2_800
+OPENROUTER_CHAT_URL = "https://openrouter.ai/api/v1/chat/completions"
+IMAGE_DESCRIPTION_PROMPT = (
+    "Analyze this image for a semantic memory graph. Return strict JSON only with keys: "
+    "description, include_in_graph, image_kind, reason. The description should be concrete and "
+    "capture visible text, diagrams, entities, relationships, layout, and likely intent. "
+    "Set include_in_graph=true for images with durable semantic value, such as diagrams, "
+    "mindmaps, charts, screenshots, document pages, UI states, photos of meaningful objects, "
+    "or figures that materially explain the document. Set include_in_graph=false for logos, "
+    "letterheads, watermarks, decorative graphics, repeated branding, tiny icons, separators, "
+    "background textures, and images that add no retrievable meaning. image_kind should be one "
+    "of diagram, screenshot, chart, document_page, photo, figure, logo, letterhead, watermark, "
+    "decorative, icon, unknown."
+)
+
+
+class DocumentParseError(ValueError):
+    """Raised when an uploaded source cannot be parsed into useful markdown."""
+
+
+@dataclass(frozen=True)
+class DocumentBlock:
+    text: str
+    page_number: int
+    kind: str = "text"
+    heading_level: int | None = None
+    char_start: int = 0
+    char_end: int = 0
+
+
+@dataclass(frozen=True)
+class ParsedChunk:
+    index: int
+    text: str
+    page_range: tuple[int, int]
+    element_range: tuple[int, int]
+    char_range: tuple[int, int]
+    bboxes: list[dict[str, Any]] = field(default_factory=list)
+
+    def to_location(self) -> dict[str, Any]:
+        return {
+            "page_range": list(self.page_range),
+            "element_range": list(self.element_range),
+            "char_range": list(self.char_range),
+            "bboxes": list(self.bboxes),
+        }
+
+
+@dataclass(frozen=True)
+class ImagePerception:
+    description: str
+    include_in_graph: bool = True
+    image_kind: str = "unknown"
+    reason: str = ""
+
+
+@dataclass(frozen=True)
+class ParsedImage:
+    index: int
+    file_name: str
+    media_type: str
+    description: str
+    include_in_graph: bool = True
+    image_kind: str = "unknown"
+    reason: str = ""
+    page_number: int = 0
+    size_bytes: int = 0
+    source: str = "upload"
+    metadata: dict[str, Any] = field(default_factory=dict)
+    raw_bytes: bytes | None = None
+
+
+@dataclass(frozen=True)
+class ParsedDocument:
+    file_name: str
+    media_type: str
+    source_type: str
+    markdown: str
+    chunks: list[ParsedChunk]
+    page_count: int = 0
+    unreadable_pages: list[int] = field(default_factory=list)
+    source_metadata: dict[str, Any] = field(default_factory=dict)
+    images: list[ParsedImage] = field(default_factory=list)
+
+
+_HEADING_RE = re.compile(r"^(#{1,6})\s+(.*\S)\s*$")
+_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".gif"}
+_MARKDOWN_EXTS = {".md", ".markdown"}
+_TEXT_EXTS = {
+    ".txt",
+    ".text",
+    ".csv",
+    ".tsv",
+    ".json",
+    ".jsonl",
+    ".yaml",
+    ".yml",
+    ".html",
+    ".htm",
+    ".xml",
+    ".rtf",
+}
+_DOCX_EXTS = {".docx"}
+_EXCEL_EXTS = {".xlsx", ".xlsm"}
+_LEGACY_EXCEL_EXTS = {".xls"}
+_POWERPOINT_EXTS = {".pptx"}
+_LEGACY_POWERPOINT_EXTS = {".ppt"}
+
+
+def parse_source(
+    *,
+    file_name: str,
+    content_type: str | None,
+    file_bytes: bytes,
+    describe_images: bool = False,
+    vlm_api_key: str | None = None,
+    vlm_model: str = "perceptron/perceptron-mk1",
+    vlm_timeout_seconds: int = 60,
+) -> ParsedDocument:
+    """Parse an uploaded source into markdown and retrievable chunks."""
+
+    if not file_bytes:
+        raise DocumentParseError("Uploaded file is empty")
+    if len(file_bytes) > MAX_DOCUMENT_UPLOAD_BYTES:
+        raise DocumentParseError(
+            f"Uploaded file is too large; max is {MAX_DOCUMENT_UPLOAD_BYTES // (1024 * 1024)} MB"
+        )
+
+    safe_name = Path(file_name or "document").name
+    ext = Path(safe_name).suffix.lower()
+    media_type = (content_type or _guess_media_type(ext)).split(";", 1)[0].strip()
+    normalized_media_type = media_type.lower()
+    image_describer = ImageDescriber(
+        api_key=vlm_api_key,
+        model=vlm_model,
+        timeout_seconds=vlm_timeout_seconds,
+    )
+
+    if ext == ".pdf" or normalized_media_type == "application/pdf":
+        return _parse_pdf(
+            safe_name,
+            media_type,
+            file_bytes,
+            describe_images=describe_images,
+            image_describer=image_describer,
+        )
+    if ext in _DOCX_EXTS or normalized_media_type == (
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+    ):
+        return _parse_docx(safe_name, media_type, file_bytes)
+    if ext in _EXCEL_EXTS or normalized_media_type in {
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "application/vnd.ms-excel.sheet.macroenabled.12",
+    }:
+        return _parse_excel(safe_name, media_type, file_bytes)
+    if ext in _LEGACY_EXCEL_EXTS or normalized_media_type == "application/vnd.ms-excel":
+        return _parse_legacy_excel(safe_name, media_type, file_bytes)
+    if ext in _POWERPOINT_EXTS or normalized_media_type == (
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+    ):
+        return _parse_pptx(safe_name, media_type, file_bytes)
+    if ext in _LEGACY_POWERPOINT_EXTS or normalized_media_type == "application/vnd.ms-powerpoint":
+        return _parse_legacy_ppt(safe_name, media_type, file_bytes)
+    if ext in _MARKDOWN_EXTS or normalized_media_type in {"text/markdown", "text/x-markdown"}:
+        markdown = _decode_text(file_bytes, safe_name).strip() + "\n"
+        return _parsed_from_markdown(
+            file_name=safe_name,
+            media_type="text/markdown",
+            source_type="markdown",
+            markdown=markdown,
+            metadata={"char_count": len(markdown)},
+        )
+    if ext in _TEXT_EXTS or normalized_media_type.startswith("text/"):
+        text = _decode_text(file_bytes, safe_name).strip()
+        markdown = f"# {Path(safe_name).stem}\n\n{text}\n"
+        return _parsed_from_markdown(
+            file_name=safe_name,
+            media_type=media_type or "text/plain",
+            source_type="text",
+            markdown=markdown,
+            metadata={"char_count": len(text)},
+        )
+    if ext in _IMAGE_EXTS or normalized_media_type.startswith("image/"):
+        return _parse_image(
+            safe_name,
+            media_type,
+            file_bytes,
+            describe_images=describe_images,
+            image_describer=image_describer,
+        )
+
+    raise DocumentParseError(
+        "Unsupported file type. Supported: PDF, DOCX, XLS/XLSX/XLSM, PPT/PPTX, Markdown, "
+        "common text formats, and PNG/JPEG/WebP/GIF image metadata."
+    )
+
+
+class ImageDescriber:
+    """OpenRouter VLM image description client."""
+
+    def __init__(
+        self,
+        *,
+        api_key: str | None,
+        model: str,
+        timeout_seconds: int,
+    ) -> None:
+        self.api_key = (api_key or "").strip()
+        self.model = model
+        self.timeout_seconds = timeout_seconds
+
+    def describe(
+        self,
+        image_bytes: bytes,
+        *,
+        media_type: str,
+        file_name: str,
+        default_include: bool = True,
+    ) -> ImagePerception | None:
+        if not self.api_key:
+            return None
+        data_url = _image_data_url(image_bytes, media_type)
+        payload: dict[str, Any] = {
+            "model": self.model,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": IMAGE_DESCRIPTION_PROMPT},
+                        {"type": "image_url", "image_url": {"url": data_url}},
+                    ],
+                }
+            ],
+        }
+        try:
+            response = requests.post(
+                OPENROUTER_CHAT_URL,
+                headers={
+                    "Authorization": f"Bearer {self.api_key}",
+                    "Content-Type": "application/json",
+                },
+                json=payload,
+                timeout=self.timeout_seconds,
+            )
+        except requests.RequestException as exc:
+            raise DocumentParseError(f"Image VLM request failed for {file_name}: {exc}") from exc
+        if response.status_code >= 400:
+            raise DocumentParseError(
+                f"Image VLM returned HTTP {response.status_code} for {file_name}: {response.text[:500]}"
+            )
+        try:
+            data = response.json()
+            content = data["choices"][0]["message"]["content"]
+        except (KeyError, IndexError, TypeError, ValueError) as exc:
+            raise DocumentParseError(f"Image VLM returned invalid response for {file_name}") from exc
+        if isinstance(content, list):
+            text_parts = [
+                str(item.get("text", "")).strip()
+                for item in content
+                if isinstance(item, dict) and item.get("type") == "text"
+            ]
+            content = "\n".join(part for part in text_parts if part)
+        return _parse_image_perception(
+            str(content or "").strip(),
+            default_include=default_include,
+        )
+
+
+def _parse_pdf(
+    file_name: str,
+    media_type: str,
+    file_bytes: bytes,
+    *,
+    describe_images: bool,
+    image_describer: ImageDescriber,
+) -> ParsedDocument:
+    try:
+        from pypdf import PdfReader
+    except Exception as exc:  # pragma: no cover - dependency sanity guard
+        raise DocumentParseError("PDF parsing requires pypdf to be installed") from exc
+
+    try:
+        reader = PdfReader(io.BytesIO(file_bytes))
+    except Exception as exc:
+        raise DocumentParseError(f"Could not open PDF: {exc}") from exc
+
+    if not reader.pages:
+        raise DocumentParseError("PDF has no pages")
+
+    parts: list[str] = []
+    unreadable: list[int] = []
+    for page_index, page in enumerate(reader.pages, start=1):
+        try:
+            text = (page.extract_text() or "").strip()
+        except Exception:
+            text = ""
+        if not text:
+            unreadable.append(page_index)
+            continue
+        parts.append(f"# Page {page_index}\n\n{text}")
+
+    images = _extract_pdf_images(
+        file_name,
+        file_bytes,
+        unreadable_pages=unreadable,
+        describe_images=describe_images,
+        image_describer=image_describer,
+    )
+    graph_images = [image for image in images if image.include_in_graph]
+    for image in graph_images:
+        parts.append(f"# Image {image.index + 1} on page {image.page_number}\n\n{image.description}")
+
+    markdown = "\n\n".join(parts).strip() + "\n"
+    if not markdown.strip():
+        raise DocumentParseError("No extractable text found in PDF")
+
+    return _parsed_from_markdown(
+        file_name=file_name,
+        media_type=media_type or "application/pdf",
+        source_type="pdf",
+        markdown=markdown,
+        page_count=len(reader.pages),
+        unreadable_pages=unreadable,
+        metadata={
+            "page_count": len(reader.pages),
+            "unreadable_pages": unreadable,
+            "parser": "pypdf",
+            "image_count": len(images),
+            "graph_image_count": len(graph_images),
+            "skipped_image_count": len(images) - len(graph_images),
+        },
+        images=images,
+    )
+
+
+def _parse_docx(file_name: str, media_type: str, file_bytes: bytes) -> ParsedDocument:
+    try:
+        from docx import Document
+        from docx.table import Table
+        from docx.text.paragraph import Paragraph
+    except Exception as exc:  # pragma: no cover - dependency sanity guard
+        raise DocumentParseError("DOCX parsing requires python-docx to be installed") from exc
+
+    try:
+        document = Document(io.BytesIO(file_bytes))
+    except Exception as exc:
+        raise DocumentParseError(f"Could not open DOCX: {exc}") from exc
+
+    parts: list[str] = [f"# {Path(file_name).stem}"]
+    page_number = 1
+    paragraph_count = 0
+    table_count = 0
+
+    for child in document.element.body.iterchildren():
+        if child.tag.endswith("}p"):
+            paragraph = Paragraph(child, document)
+            text = paragraph.text.strip()
+            if text:
+                paragraph_count += 1
+                style_name = str(getattr(paragraph.style, "name", "") or "")
+                heading_match = re.match(r"Heading\s+([1-6])$", style_name, flags=re.IGNORECASE)
+                if heading_match:
+                    parts.append(f"{'#' * int(heading_match.group(1))} {text}")
+                else:
+                    parts.append(text)
+            breaks = _docx_page_break_count(paragraph)
+            for _ in range(breaks):
+                page_number += 1
+                parts.append(f"# Page {page_number}")
+            continue
+
+        if child.tag.endswith("}tbl"):
+            table = Table(child, document)
+            rows = [
+                [cell.text.strip() for cell in row.cells]
+                for row in table.rows
+            ]
+            rendered = _rows_to_markdown_table(rows)
+            if rendered:
+                table_count += 1
+                parts.append(rendered)
+
+    markdown = "\n\n".join(part for part in parts if part.strip()).strip() + "\n"
+    if paragraph_count == 0 and table_count == 0:
+        raise DocumentParseError("No extractable text or tables found in DOCX")
+
+    return _parsed_from_markdown(
+        file_name=file_name,
+        media_type=media_type
+        or "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        source_type="docx",
+        markdown=markdown,
+        page_count=page_number,
+        metadata={
+            "parser": "python-docx",
+            "paragraph_count": paragraph_count,
+            "table_count": table_count,
+            "page_count": page_number,
+        },
+    )
+
+
+def _docx_page_break_count(paragraph: Any) -> int:
+    namespace = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
+    count = 1 if bool(paragraph.paragraph_format.page_break_before) else 0
+    for run in paragraph.runs:
+        count += sum(
+            1
+            for break_element in run._r.findall(f".//{namespace}br")
+            if break_element.get(f"{namespace}type") == "page"
+        )
+    return count
+
+
+def _parse_excel(file_name: str, media_type: str, file_bytes: bytes) -> ParsedDocument:
+    try:
+        from openpyxl import load_workbook
+    except Exception as exc:  # pragma: no cover - dependency sanity guard
+        raise DocumentParseError("Excel parsing requires openpyxl to be installed") from exc
+
+    try:
+        workbook = load_workbook(
+            io.BytesIO(file_bytes),
+            read_only=True,
+            data_only=True,
+            keep_links=False,
+        )
+    except Exception as exc:
+        raise DocumentParseError(f"Could not open Excel workbook: {exc}") from exc
+
+    sheets: list[tuple[str, list[list[str]]]] = []
+    try:
+        for worksheet in workbook.worksheets:
+            rows: list[list[str]] = []
+            for row in worksheet.iter_rows(values_only=True):
+                values = [_excel_cell_text(value) for value in row]
+                while values and not values[-1]:
+                    values.pop()
+                if not any(values):
+                    continue
+                rows.append(values)
+            sheets.append((worksheet.title, rows))
+    finally:
+        workbook.close()
+
+    return _parsed_from_spreadsheet(
+        file_name=file_name,
+        media_type=media_type
+        or "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        sheets=sheets,
+        parser="openpyxl",
+    )
+
+
+def _parse_legacy_excel(file_name: str, media_type: str, file_bytes: bytes) -> ParsedDocument:
+    try:
+        import xlrd
+    except Exception as exc:  # pragma: no cover - dependency sanity guard
+        raise DocumentParseError("Legacy Excel parsing requires xlrd to be installed") from exc
+
+    try:
+        workbook = xlrd.open_workbook(file_contents=file_bytes, on_demand=True)
+    except Exception as exc:
+        raise DocumentParseError(f"Could not open legacy Excel workbook: {exc}") from exc
+
+    sheets: list[tuple[str, list[list[str]]]] = []
+    try:
+        for worksheet in workbook.sheets():
+            rows = [
+                [_excel_cell_text(worksheet.cell_value(row, column)) for column in range(worksheet.ncols)]
+                for row in range(worksheet.nrows)
+            ]
+            sheets.append((worksheet.name, rows))
+    finally:
+        workbook.release_resources()
+
+    return _parsed_from_spreadsheet(
+        file_name=file_name,
+        media_type=media_type or "application/vnd.ms-excel",
+        sheets=sheets,
+        parser="xlrd",
+    )
+
+
+def _parsed_from_spreadsheet(
+    *,
+    file_name: str,
+    media_type: str,
+    sheets: list[tuple[str, list[list[str]]]],
+    parser: str,
+) -> ParsedDocument:
+    parts: list[str] = [f"# {Path(file_name).stem}"]
+    sheet_metadata: list[dict[str, Any]] = []
+    total_cells = 0
+    for sheet_name, raw_rows in sheets:
+        rows: list[list[str]] = []
+        sheet_cells = 0
+        for raw_row in raw_rows:
+            row = list(raw_row)
+            while row and not row[-1]:
+                row.pop()
+            if not any(row):
+                continue
+            non_empty = sum(1 for value in row if value)
+            sheet_cells += non_empty
+            total_cells += non_empty
+            if total_cells > MAX_SPREADSHEET_CELLS:
+                raise DocumentParseError(
+                    f"Excel workbook exceeds the {MAX_SPREADSHEET_CELLS:,} non-empty cell limit"
+                )
+            rows.append(row)
+        if not rows:
+            continue
+        width = max(len(row) for row in rows)
+        normalized_rows = [row + [""] * (width - len(row)) for row in rows]
+        parts.extend([f"## Sheet: {sheet_name}", _rows_to_markdown_table(normalized_rows)])
+        sheet_metadata.append(
+            {
+                "name": sheet_name,
+                "row_count": len(normalized_rows),
+                "column_count": width,
+                "non_empty_cell_count": sheet_cells,
+            }
+        )
+
+    if not sheet_metadata:
+        raise DocumentParseError("No non-empty cells found in Excel workbook")
+
+    return _parsed_from_markdown(
+        file_name=file_name,
+        media_type=media_type,
+        source_type="excel",
+        markdown="\n\n".join(parts).strip() + "\n",
+        page_count=len(sheet_metadata),
+        metadata={
+            "parser": parser,
+            "sheet_count": len(sheet_metadata),
+            "non_empty_cell_count": total_cells,
+            "sheets": sheet_metadata,
+        },
+    )
+
+
+def _parse_pptx(file_name: str, media_type: str, file_bytes: bytes) -> ParsedDocument:
+    try:
+        from pptx import Presentation
+    except Exception as exc:  # pragma: no cover - dependency sanity guard
+        raise DocumentParseError("PPTX parsing requires python-pptx to be installed") from exc
+
+    try:
+        presentation = Presentation(io.BytesIO(file_bytes))
+    except Exception as exc:
+        raise DocumentParseError(f"Could not open PPTX: {exc}") from exc
+
+    parts: list[str] = [f"# {Path(file_name).stem}"]
+    slide_metadata: list[dict[str, Any]] = []
+    for slide_number, slide in enumerate(presentation.slides, start=1):
+        slide_parts: list[str] = []
+        table_count = 0
+        for shape in slide.shapes:
+            if getattr(shape, "has_text_frame", False):
+                text = "\n".join(
+                    paragraph.text.strip()
+                    for paragraph in shape.text_frame.paragraphs
+                    if paragraph.text.strip()
+                )
+                if text:
+                    slide_parts.append(text)
+            if getattr(shape, "has_table", False):
+                rows = [[cell.text.strip() for cell in row.cells] for row in shape.table.rows]
+                rendered = _rows_to_markdown_table(rows)
+                if rendered:
+                    table_count += 1
+                    slide_parts.append(rendered)
+        notes_text = ""
+        if slide.has_notes_slide:
+            notes_text = "\n".join(
+                shape.text.strip()
+                for shape in slide.notes_slide.notes_text_frame.paragraphs
+                if shape.text.strip()
+            )
+            if notes_text:
+                slide_parts.append(f"### Speaker notes\n\n{notes_text}")
+        if slide_parts:
+            parts.append(f"# Slide {slide_number}\n\n" + "\n\n".join(slide_parts))
+        slide_metadata.append(
+            {
+                "slide_number": slide_number,
+                "table_count": table_count,
+                "has_notes": bool(notes_text),
+            }
+        )
+
+    if len(parts) == 1:
+        raise DocumentParseError("No extractable text or tables found in PPTX")
+
+    return _parsed_from_markdown(
+        file_name=file_name,
+        media_type=media_type
+        or "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        source_type="powerpoint",
+        markdown="\n\n".join(parts).strip() + "\n",
+        page_count=len(presentation.slides),
+        metadata={
+            "parser": "python-pptx",
+            "slide_count": len(presentation.slides),
+            "slides": slide_metadata,
+        },
+    )
+
+
+def _parse_legacy_ppt(file_name: str, media_type: str, file_bytes: bytes) -> ParsedDocument:
+    try:
+        import olefile
+    except Exception as exc:  # pragma: no cover - dependency sanity guard
+        raise DocumentParseError("Legacy PPT parsing requires olefile to be installed") from exc
+
+    try:
+        ole = olefile.OleFileIO(io.BytesIO(file_bytes))
+    except Exception as exc:
+        raise DocumentParseError(f"Could not open legacy PPT: {exc}") from exc
+    try:
+        if not ole.exists("PowerPoint Document"):
+            raise DocumentParseError("Legacy PPT does not contain a PowerPoint Document stream")
+        stream = ole.openstream("PowerPoint Document").read()
+    finally:
+        ole.close()
+
+    fragments = _legacy_ppt_text_fragments(stream)
+    if not fragments:
+        raise DocumentParseError("No extractable text found in legacy PPT")
+    markdown = f"# {Path(file_name).stem}\n\n" + "\n\n".join(fragments) + "\n"
+    return _parsed_from_markdown(
+        file_name=file_name,
+        media_type=media_type or "application/vnd.ms-powerpoint",
+        source_type="powerpoint",
+        markdown=markdown,
+        metadata={"parser": "olefile", "text_fragment_count": len(fragments)},
+    )
+
+
+def _legacy_ppt_text_fragments(stream: bytes) -> list[str]:
+    fragments: list[str] = []
+
+    def walk(start: int, end: int) -> None:
+        offset = start
+        while offset + 8 <= end:
+            options, record_type, length = struct.unpack_from("<HHI", stream, offset)
+            payload_start = offset + 8
+            payload_end = payload_start + length
+            if payload_end > end:
+                break
+            record_version = options & 0xF
+            if record_version == 0xF:
+                walk(payload_start, payload_end)
+            elif record_type == 4000:
+                _append_text_fragment(fragments, stream[payload_start:payload_end].decode("utf-16le", errors="ignore"))
+            elif record_type == 4008:
+                _append_text_fragment(fragments, stream[payload_start:payload_end].decode("cp1252", errors="ignore"))
+            offset = payload_end
+
+    walk(0, len(stream))
+    return fragments
+
+
+def _append_text_fragment(fragments: list[str], text: str) -> None:
+    normalized = re.sub(r"\s+", " ", text).strip("\x00 \t\r\n")
+    if normalized and (not fragments or fragments[-1] != normalized):
+        fragments.append(normalized)
+
+
+def _excel_cell_text(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, (dt.datetime, dt.date, dt.time)):
+        return value.isoformat()
+    if isinstance(value, bool):
+        return "TRUE" if value else "FALSE"
+    return str(value).strip()
+
+
+def _rows_to_markdown_table(rows: list[list[str]]) -> str:
+    if not rows:
+        return ""
+    width = max((len(row) for row in rows), default=0)
+    if width == 0:
+        return ""
+    normalized = [row + [""] * (width - len(row)) for row in rows]
+    header = normalized[0]
+    body = normalized[1:]
+
+    def render(row: list[str]) -> str:
+        cells = [
+            str(value).replace("\\", "\\\\").replace("|", "\\|").replace("\r", " ").replace("\n", "<br>")
+            for value in row
+        ]
+        return f"| {' | '.join(cells)} |"
+
+    return "\n".join(
+        [
+            render(header),
+            f"| {' | '.join(['---'] * width)} |",
+            *(render(row) for row in body),
+        ]
+    )
+
+
+def _parse_image(
+    file_name: str,
+    media_type: str,
+    file_bytes: bytes,
+    *,
+    describe_images: bool,
+    image_describer: ImageDescriber,
+) -> ParsedDocument:
+    description = None
+    perception = None
+    parser = "image_metadata"
+    if describe_images:
+        perception = image_describer.describe(
+            file_bytes,
+            media_type=media_type,
+            file_name=file_name,
+            default_include=True,
+        )
+        if perception:
+            description = perception.description
+            parser = "openrouter_vlm"
+    description = description or _image_description_fallback(
+        describe_images=describe_images,
+        has_api_key=bool(image_describer.api_key),
+    )
+    markdown = (
+        f"# {Path(file_name).stem}\n\n"
+        f"Image upload: `{file_name}`.\n\n"
+        f"{description}\n"
+    )
+    image = ParsedImage(
+        index=0,
+        file_name=file_name,
+        media_type=media_type or _guess_media_type(Path(file_name).suffix.lower()),
+        description=description,
+        include_in_graph=True,
+        image_kind=perception.image_kind if perception else "unknown",
+        reason=perception.reason if perception else "",
+        size_bytes=len(file_bytes),
+        source="upload",
+        metadata={
+            "parser": parser,
+            "vlm_include_in_graph": perception.include_in_graph if perception else True,
+            "vlm_image_kind": perception.image_kind if perception else "unknown",
+            "vlm_reason": perception.reason if perception else "",
+        },
+        raw_bytes=file_bytes,
+    )
+    return _parsed_from_markdown(
+        file_name=file_name,
+        media_type=media_type or _guess_media_type(Path(file_name).suffix.lower()),
+        source_type="image",
+        markdown=markdown,
+        metadata={
+            "size_bytes": len(file_bytes),
+            "parser": parser,
+            "visual_description": description,
+            "image_count": 1,
+            "graph_image_count": 1,
+            "skipped_image_count": 0,
+            "vlm_include_in_graph": perception.include_in_graph if perception else True,
+            "vlm_image_kind": perception.image_kind if perception else "unknown",
+            "vlm_reason": perception.reason if perception else "",
+        },
+        images=[image],
+    )
+
+
+def _parsed_from_markdown(
+    *,
+    file_name: str,
+    media_type: str,
+    source_type: str,
+    markdown: str,
+    page_count: int = 0,
+    unreadable_pages: list[int] | None = None,
+    metadata: dict[str, Any] | None = None,
+    images: list[ParsedImage] | None = None,
+) -> ParsedDocument:
+    blocks = _markdown_blocks(markdown)
+    chunks = _chunk_blocks(blocks)
+    if not chunks:
+        raise DocumentParseError("No semantic chunks were produced from uploaded file")
+    return ParsedDocument(
+        file_name=file_name,
+        media_type=media_type,
+        source_type=source_type,
+        markdown=markdown,
+        chunks=chunks,
+        page_count=page_count,
+        unreadable_pages=unreadable_pages or [],
+        source_metadata=metadata or {},
+        images=images or [],
+    )
+
+
+def _markdown_blocks(markdown: str) -> list[DocumentBlock]:
+    lines = markdown.splitlines()
+    blocks: list[DocumentBlock] = []
+    paragraph: list[str] = []
+    offset = 0
+    paragraph_start = 0
+    current_page = 0
+
+    def flush_paragraph(end_offset: int) -> None:
+        nonlocal paragraph, paragraph_start
+        if not paragraph:
+            return
+        text = " ".join(part.strip() for part in paragraph if part.strip()).strip()
+        if text:
+            blocks.append(
+                DocumentBlock(
+                    text=text,
+                    page_number=current_page,
+                    char_start=paragraph_start,
+                    char_end=end_offset,
+                )
+            )
+        paragraph = []
+        paragraph_start = end_offset
+
+    for raw_line in lines:
+        line_start = offset
+        stripped = raw_line.strip()
+        offset += len(raw_line) + 1
+
+        if not stripped:
+            flush_paragraph(line_start)
+            paragraph_start = offset
+            continue
+
+        heading = _HEADING_RE.match(stripped)
+        if heading:
+            flush_paragraph(line_start)
+            level = len(heading.group(1))
+            text = heading.group(2).strip()
+            page_match = re.match(r"Page\s+(\d+)$", text, flags=re.IGNORECASE)
+            if page_match:
+                current_page = int(page_match.group(1))
+            blocks.append(
+                DocumentBlock(
+                    text=f"{'#' * level} {text}",
+                    page_number=current_page,
+                    kind="heading",
+                    heading_level=level,
+                    char_start=line_start,
+                    char_end=offset,
+                )
+            )
+            paragraph_start = offset
+            continue
+
+        if not paragraph:
+            paragraph_start = line_start
+        paragraph.append(raw_line)
+
+    flush_paragraph(offset)
+    return blocks
+
+
+def _chunk_blocks(
+    blocks: list[DocumentBlock],
+    *,
+    target_chars: int = DEFAULT_TARGET_CHARS,
+    max_chars: int = DEFAULT_MAX_CHARS,
+) -> list[ParsedChunk]:
+    chunks: list[ParsedChunk] = []
+    current: list[tuple[int, DocumentBlock]] = []
+
+    def current_len() -> int:
+        return sum(len(block.text) for _index, block in current) + max(0, len(current) - 1) * 2
+
+    def flush() -> None:
+        nonlocal current
+        if not current:
+            return
+        indices = [index for index, _block in current]
+        texts = [block.text for _index, block in current]
+        pages = [block.page_number for _index, block in current if block.page_number > 0]
+        page_range = (min(pages), max(pages)) if pages else (0, 0)
+        chunks.append(
+            ParsedChunk(
+                index=len(chunks),
+                text="\n\n".join(texts).strip(),
+                page_range=page_range,
+                element_range=(indices[0], indices[-1]),
+                char_range=(current[0][1].char_start, current[-1][1].char_end),
+            )
+        )
+        current = []
+
+    for index, block in enumerate(blocks):
+        if block.kind == "heading" and current and current_len() >= target_chars // 3:
+            flush()
+        if current and current_len() + len(block.text) + 2 > max_chars:
+            flush()
+        current.append((index, block))
+        if current_len() >= target_chars and block.kind != "heading":
+            flush()
+
+    flush()
+    return chunks
+
+
+def _extract_pdf_images(
+    file_name: str,
+    file_bytes: bytes,
+    *,
+    unreadable_pages: list[int],
+    describe_images: bool,
+    image_describer: ImageDescriber,
+) -> list[ParsedImage]:
+    try:
+        import fitz
+    except Exception:
+        return []
+
+    images: list[ParsedImage] = []
+    seen_xrefs: set[int] = set()
+    pages_with_images: set[int] = set()
+    try:
+        document = fitz.open(stream=file_bytes, filetype="pdf")
+    except Exception:
+        return []
+
+    try:
+        for page_index, page in enumerate(document, start=1):
+            for image_info in page.get_images(full=True):
+                xref = int(image_info[0])
+                if xref in seen_xrefs:
+                    continue
+                seen_xrefs.add(xref)
+                extracted = document.extract_image(xref)
+                image_bytes = extracted.get("image")
+                if not isinstance(image_bytes, bytes) or not image_bytes:
+                    continue
+                ext = str(extracted.get("ext") or "png").lower()
+                image_media_type = _guess_media_type(f".{ext}")
+                image_name = f"{Path(file_name).stem}-page-{page_index}-image-{len(images) + 1}.{ext}"
+                description = None
+                perception = None
+                parser = "pdf_image_metadata"
+                if describe_images:
+                    perception = image_describer.describe(
+                        image_bytes,
+                        media_type=image_media_type,
+                        file_name=image_name,
+                        default_include=False,
+                    )
+                    if perception:
+                        description = perception.description
+                        parser = "openrouter_vlm"
+                description = description or (
+                    f"Extracted image from `{file_name}` on page {page_index}. "
+                    f"{_image_description_fallback(describe_images=describe_images, has_api_key=bool(image_describer.api_key))}"
+                )
+                pages_with_images.add(page_index)
+                images.append(
+                    ParsedImage(
+                        index=len(images),
+                        file_name=image_name,
+                        media_type=image_media_type,
+                        description=description,
+                        include_in_graph=perception.include_in_graph if perception else not describe_images,
+                        image_kind=perception.image_kind if perception else "unknown",
+                        reason=perception.reason if perception else "",
+                        page_number=page_index,
+                        size_bytes=len(image_bytes),
+                        source="pdf",
+                        metadata={
+                            "parser": parser,
+                            "pdf_file_name": file_name,
+                            "xref": xref,
+                            "vlm_include_in_graph": perception.include_in_graph if perception else None,
+                            "vlm_image_kind": perception.image_kind if perception else "unknown",
+                            "vlm_reason": perception.reason if perception else "",
+                        },
+                        raw_bytes=image_bytes,
+                    )
+                )
+        for page_number in unreadable_pages:
+            if page_number in pages_with_images or page_number < 1 or page_number > document.page_count:
+                continue
+            page = document[page_number - 1]
+            pixmap = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
+            image_bytes = pixmap.tobytes("png")
+            image_name = f"{Path(file_name).stem}-page-{page_number}.png"
+            description = None
+            perception = None
+            parser = "pdf_page_render"
+            if describe_images:
+                perception = image_describer.describe(
+                    image_bytes,
+                    media_type="image/png",
+                    file_name=image_name,
+                    default_include=True,
+                )
+                if perception:
+                    description = perception.description
+                    parser = "openrouter_vlm"
+            description = description or (
+                f"Rendered image-only page from `{file_name}` page {page_number}. "
+                f"{_image_description_fallback(describe_images=describe_images, has_api_key=bool(image_describer.api_key))}"
+            )
+            images.append(
+                ParsedImage(
+                    index=len(images),
+                    file_name=image_name,
+                    media_type="image/png",
+                    description=description,
+                    include_in_graph=perception.include_in_graph if perception else True,
+                    image_kind=perception.image_kind if perception else "document_page",
+                    reason=perception.reason if perception else "",
+                    page_number=page_number,
+                    size_bytes=len(image_bytes),
+                    source="pdf_page_render",
+                    metadata={
+                        "parser": parser,
+                        "pdf_file_name": file_name,
+                        "vlm_include_in_graph": perception.include_in_graph if perception else None,
+                        "vlm_image_kind": perception.image_kind if perception else "document_page",
+                        "vlm_reason": perception.reason if perception else "",
+                    },
+                    raw_bytes=image_bytes,
+                )
+            )
+    finally:
+        document.close()
+    return images
+
+
+def _image_data_url(image_bytes: bytes, media_type: str) -> str:
+    encoded = base64.b64encode(image_bytes).decode("ascii")
+    return f"data:{media_type or 'image/png'};base64,{encoded}"
+
+
+def _image_description_fallback(*, describe_images: bool, has_api_key: bool) -> str:
+    if not describe_images:
+        return "Visual description was skipped because image VLM description is disabled."
+    if not has_api_key:
+        return "Visual description was skipped because OPENROUTER_API_KEY is not configured."
+    return "Visual description was skipped because the image VLM did not return text."
+
+
+def _parse_image_perception(content: str, *, default_include: bool) -> ImagePerception | None:
+    if not content:
+        return None
+    parsed = _loads_json_object(content)
+    if not isinstance(parsed, dict):
+        return ImagePerception(description=content, include_in_graph=default_include)
+    description = str(parsed.get("description") or "").strip()
+    if not description:
+        description = content
+    include_raw = parsed.get("include_in_graph", default_include)
+    include = include_raw if isinstance(include_raw, bool) else default_include
+    image_kind = str(parsed.get("image_kind") or "unknown").strip() or "unknown"
+    reason = str(parsed.get("reason") or "").strip()
+    return ImagePerception(
+        description=description,
+        include_in_graph=include,
+        image_kind=image_kind,
+        reason=reason,
+    )
+
+
+def _loads_json_object(content: str) -> dict[str, Any] | None:
+    text = content.strip()
+    fenced = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, flags=re.DOTALL)
+    if fenced:
+        text = fenced.group(1).strip()
+    if not text.startswith("{"):
+        start = text.find("{")
+        end = text.rfind("}")
+        if start >= 0 and end > start:
+            text = text[start : end + 1]
+    try:
+        data = json.loads(text)
+    except ValueError:
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def _decode_text(file_bytes: bytes, file_name: str) -> str:
+    try:
+        return file_bytes.decode("utf-8")
+    except UnicodeDecodeError:
+        decoded = file_bytes.decode("utf-8", errors="replace")
+        if not decoded.strip():
+            raise DocumentParseError(f"Could not decode text file {file_name}")
+        return decoded
+
+
+def _guess_media_type(ext: str) -> str:
+    return {
+        ".pdf": "application/pdf",
+        ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        ".xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12",
+        ".xls": "application/vnd.ms-excel",
+        ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        ".ppt": "application/vnd.ms-powerpoint",
+        ".md": "text/markdown",
+        ".markdown": "text/markdown",
+        ".txt": "text/plain",
+        ".text": "text/plain",
+        ".csv": "text/csv",
+        ".tsv": "text/tab-separated-values",
+        ".json": "application/json",
+        ".jsonl": "application/x-ndjson",
+        ".yaml": "application/yaml",
+        ".yml": "application/yaml",
+        ".html": "text/html",
+        ".htm": "text/html",
+        ".xml": "application/xml",
+        ".rtf": "application/rtf",
+        ".png": "image/png",
+        ".jpg": "image/jpeg",
+        ".jpeg": "image/jpeg",
+        ".webp": "image/webp",
+        ".gif": "image/gif",
+    }.get(ext.lower(), "application/octet-stream")