markitdown-gc 0.2.0__py3-none-any.whl

markitdown/_media.py

@@ -0,0 +1,67 @@
+"""Original image extraction. Names and diagnostics never use document text."""
+
+import hashlib
+import html
+import mimetypes
+import os
+import re
+from pathlib import Path
+from urllib.parse import quote
+
+
+def check_media_options(options):
+    if options.get("extract_media") and options.get("keep_data_uris"):
+        raise ValueError("extract_media and keep_data_uris are mutually exclusive")
+
+
+def save_media(data, ext, **options):
+    check_media_options(options)
+    ext = ext.lower().lstrip(".")
+    if not re.fullmatch(r"[a-z0-9]{1,12}", ext):
+        raise ValueError("Unsafe media extension")
+    directory = Path(options["extract_media"]).absolute()
+    directory.mkdir(parents=True, exist_ok=True)
+    name = hashlib.sha256(data).hexdigest()[:12] + "." + ext
+    prefix = name.split(".", 1)[0]
+    # Equal content may have extension aliases (jpg/jpeg). Keep one evidence file.
+    existing_files = sorted(directory.glob(prefix + ".*"))
+    for existing in existing_files:
+        if (
+            existing.is_symlink()
+            or not existing.is_file()
+            or existing.read_bytes() != data
+        ):
+            raise ValueError("Media hash collision or unsafe destination")
+    if existing_files:
+        name = existing_files[0].name
+    target = directory / name
+    if target.is_symlink() or target.resolve().parent != directory.resolve():
+        raise ValueError("Unsafe media destination")
+    try:
+        with target.open("xb") as output:
+            output.write(data)
+    except FileExistsError:
+        if target.read_bytes() != data:
+            raise ValueError("Media hash collision")
+    base = options.get("media_base_dir") or os.getcwd()
+    return quote(os.path.relpath(target, base).replace(os.sep, "/"), safe="/.-_")
+
+
+def image_html(stream, info, **options):
+    ext = info.extension or mimetypes.guess_extension(info.mimetype or "") or ".bin"
+    path = save_media(stream.read(), ext, **options)
+    return '<img src="' + html.escape(path, quote=True) + '" alt="Image">'
+
+
+def validate_members(archive):
+    seen = set()
+    for name in archive.namelist():
+        parts = name.replace("\\", "/").split("/")
+        if (
+            name.startswith(("/", "\\"))
+            or ".." in parts
+            or ":" in name
+            or name.casefold() in seen
+        ):
+            raise ValueError("Unsafe or colliding package member")
+        seen.add(name.casefold())

markitdown/_stream_info.py

@@ -0,0 +1,32 @@
+from dataclasses import dataclass, asdict
+from typing import Optional
+
+
+@dataclass(kw_only=True, frozen=True)
+class StreamInfo:
+    """The StreamInfo class is used to store information about a file stream.
+    All fields can be None, and will depend on how the stream was opened.
+    """
+
+    mimetype: Optional[str] = None
+    extension: Optional[str] = None
+    charset: Optional[str] = None
+    filename: Optional[str] = (
+        None  # From local path, url, or Content-Disposition header
+    )
+    local_path: Optional[str] = None  # If read from disk
+    url: Optional[str] = None  # If read from url
+
+    def copy_and_update(self, *args, **kwargs):
+        """Copy the StreamInfo object and update it with the given StreamInfo
+        instance and/or other keyword arguments."""
+        new_info = asdict(self)
+
+        for si in args:
+            assert isinstance(si, StreamInfo)
+            new_info.update({k: v for k, v in asdict(si).items() if v is not None})
+
+        if len(kwargs) > 0:
+            new_info.update(kwargs)
+
+        return StreamInfo(**new_info)

markitdown/_uri_utils.py

@@ -0,0 +1,75 @@
+import base64
+import ntpath
+import os
+from typing import Tuple, Dict
+from urllib.request import url2pathname
+from urllib.parse import urlparse, unquote_to_bytes
+
+
+def _is_unc_or_device_path(path: str) -> bool:
+    """Recognize Windows UNC and device namespace prefixes on any platform."""
+    drive, _ = ntpath.splitdrive(path)
+    return drive.replace("\\", "/").startswith("//")
+
+
+def file_uri_to_path(file_uri: str) -> Tuple[str | None, str]:
+    """Convert a file URI to a path, rejecting UNC and Windows device paths."""
+    parsed = urlparse(file_uri)
+    if parsed.scheme != "file":
+        raise ValueError(f"Not a file URL: {file_uri}")
+
+    decoded_path = unquote_to_bytes(parsed.path).replace(b"\\", b"/")
+    if decoded_path.startswith(b"//"):
+        raise ValueError(
+            f"Unsupported file URI: {file_uri}. "
+            "UNC and Windows device paths are not supported."
+        )
+
+    netloc = parsed.netloc if parsed.netloc else None
+    path = url2pathname(parsed.path)
+    if os.name == "nt" and path[:1] in "/\\" and path[2:3] == ":":
+        path = path[1:]
+    path = os.path.abspath(path)
+
+    if _is_unc_or_device_path(path):
+        raise ValueError(
+            f"Unsupported file URI: {file_uri}. "
+            "UNC and Windows device paths are not supported."
+        )
+    return netloc, path
+
+
+def parse_data_uri(uri: str) -> Tuple[str | None, Dict[str, str], bytes]:
+    if uri[:5].lower() != "data:":
+        raise ValueError("Not a data URI")
+
+    header, _, data = uri.partition(",")
+    if not _:
+        raise ValueError("Malformed data URI, missing ',' separator")
+
+    meta = header[5:]  # Strip 'data:'
+    parts = meta.split(";")
+
+    is_base64 = False
+    # Ends with base64?
+    if parts[-1].lower() == "base64":
+        parts.pop()
+        is_base64 = True
+
+    mime_type = None  # Normally this would default to text/plain but we won't assume
+    if len(parts) and len(parts[0]) > 0:
+        # First part is the mime type
+        mime_type = parts.pop(0)
+
+    attributes: Dict[str, str] = {}
+    for part in parts:
+        # Handle key=value pairs in the middle
+        if "=" in part:
+            key, value = part.split("=", 1)
+            attributes[key.lower()] = value
+        elif len(part) > 0:
+            attributes[part.lower()] = ""
+
+    content = base64.b64decode(data) if is_base64 else unquote_to_bytes(data)
+
+    return mime_type, attributes, content

markitdown/converter_utils/_image.py

@@ -0,0 +1,19 @@
+"""Shared validation for Office image-rendering hooks."""
+
+from typing import Optional
+
+from bs4 import BeautifulSoup, Doctype
+
+
+def _parse_image_html(fragment: Optional[str]) -> Optional[BeautifulSoup]:
+    if fragment is not None and not isinstance(fragment, str):
+        raise TypeError("_image_to_html must return an HTML string or None")
+    if fragment is None or not fragment.strip():
+        return None
+
+    soup = BeautifulSoup(fragment, "html.parser")
+    if soup.find(["html", "head", "body"]) or any(
+        isinstance(node, Doctype) for node in soup.descendants
+    ):
+        raise ValueError("_image_to_html must return a fragment, not a document")
+    return soup

markitdown/converter_utils/_xlsx_images.py

@@ -0,0 +1,114 @@
+"""Read spreadsheet drawing images without reloading or resaving the workbook."""
+
+import io
+import mimetypes
+import posixpath
+import zipfile
+from typing import Any, BinaryIO, Callable, Optional
+from urllib.parse import unquote
+
+from defusedxml import ElementTree as ET
+
+from .._stream_info import StreamInfo
+from ._image import _parse_image_html
+
+_NS = {
+    "s": "http://schemas.openxmlformats.org/spreadsheetml/2006/main",
+    "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships",
+    "xdr": "http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing",
+    "a": "http://schemas.openxmlformats.org/drawingml/2006/main",
+}
+_REL_ID = "{" + _NS["r"] + "}id"
+_EMBED = "{" + _NS["r"] + "}embed"
+
+
+def _relationships(
+    archive: zipfile.ZipFile, part: str, kind: Optional[str] = None
+) -> dict[str, str]:
+    directory, filename = posixpath.split(part)
+    rels = posixpath.join(directory, "_rels", filename + ".rels")
+    if rels not in archive.namelist():
+        return {}
+    root = ET.fromstring(archive.read(rels))
+    return {
+        rel.attrib["Id"]: posixpath.normpath(
+            posixpath.join(directory, unquote(rel.attrib["Target"]))
+        ).lstrip("/")
+        for rel in root
+        if rel.get("TargetMode") != "External"
+        and (kind is None or rel.attrib["Type"].endswith("/" + kind))
+    }
+
+
+class _XlsxImages:
+    def __init__(self, file_stream: BinaryIO):
+        self._sheets: dict[str, list[tuple[bytes, StreamInfo]]] = {}
+        with zipfile.ZipFile(file_stream) as archive:
+            workbook = _relationships(archive, "", "officeDocument")
+            workbook_part = next(iter(workbook.values()))
+            sheets = ET.fromstring(archive.read(workbook_part))
+            sheet_parts = _relationships(archive, workbook_part)
+            content_types = ET.fromstring(archive.read("[Content_Types].xml"))
+            defaults = {
+                item.attrib["Extension"].lower(): item.attrib["ContentType"]
+                for item in content_types
+                if "Extension" in item.attrib
+            }
+            overrides = {
+                unquote(item.attrib["PartName"]).lstrip("/"): item.attrib["ContentType"]
+                for item in content_types
+                if "PartName" in item.attrib
+            }
+            for sheet in sheets.findall("s:sheets/s:sheet", _NS):
+                sheet_part = sheet_parts[sheet.attrib[_REL_ID]]
+                drawings = ET.fromstring(archive.read(sheet_part)).findall(
+                    "s:drawing", _NS
+                )
+                if not drawings:
+                    continue
+                drawing_parts = _relationships(archive, sheet_part, "drawing")
+                images = self._sheets.setdefault(sheet.attrib["name"], [])
+                for drawing in drawings:
+                    drawing_part = drawing_parts[drawing.attrib[_REL_ID]]
+                    image_parts = _relationships(archive, drawing_part, "image")
+                    drawing_root = ET.fromstring(archive.read(drawing_part))
+                    # Match openpyxl's image traversal, not its XML serialization order.
+                    anchors = [
+                        anchor
+                        for kind in ("absoluteAnchor", "oneCellAnchor", "twoCellAnchor")
+                        for anchor in drawing_root.findall(f"xdr:{kind}", _NS)
+                    ]
+                    for anchor in anchors:
+                        for blip in anchor.findall(".//a:blip", _NS):
+                            relationship = blip.get(_EMBED)
+                            if relationship is None:
+                                # Linked images are not embedded package content.
+                                continue
+                            image_part = image_parts[relationship]
+                            extension = posixpath.splitext(image_part)[1].lower()
+                            info = StreamInfo(
+                                mimetype=overrides.get(image_part)
+                                or defaults.get(extension.lstrip("."))
+                                or mimetypes.guess_type(image_part)[0],
+                                extension=extension,
+                                filename=posixpath.basename(image_part),
+                            )
+                            images.append((archive.read(image_part), info))
+
+    def to_html(
+        self,
+        sheet_name: str,
+        render: Callable[..., Optional[str]],
+        options: dict[str, Any],
+    ) -> str:
+        fragments = []
+        for data, info in self._sheets.get(sheet_name, []):
+            with io.BytesIO(data) as image_stream:
+                fragment = render(image_stream, info, **options)
+            soup = _parse_image_html(fragment)
+            if soup is None:
+                continue
+            fragments.append(f"<div>{soup}</div>")
+        if not fragments:
+            return ""
+        return "<h3>Images in this sheet:</h3>" + "".join(fragments)

markitdown/converter_utils/docx/_images.py

@@ -0,0 +1,118 @@
+"""Internal bridge from the DOCX image hook to document HTML."""
+
+import mimetypes
+from typing import Any, Callable, Optional
+from uuid import uuid4
+
+from bs4 import BeautifulSoup, Tag
+from bs4.builder import HTMLTreeBuilder
+from mammoth import html, images
+from mammoth.docx.files import InvalidFileReferenceError
+
+from ..._stream_info import StreamInfo
+from .._image import _parse_image_html
+
+_BLOCK_ELEMENTS = HTMLTreeBuilder.DEFAULT_BLOCK_ELEMENTS | {
+    "details",
+    "dialog",
+    "hgroup",
+    "menu",
+    "search",
+    "summary",
+}
+_PHRASING_CONTAINERS = set(
+    "a abbr b bdi bdo cite code data del dfn em i ins kbd label mark q s samp "
+    "small span strong sub sup time u var p h1 h2 h3 h4 h5 h6 pre".split()
+)
+
+
+def _lift_out_of_parent(node: Tag, soup: BeautifulSoup) -> None:
+    """Split a paragraph/inline wrapper around generated block content."""
+    parent = node.parent
+    assert isinstance(parent, Tag)
+    before = soup.new_tag(parent.name)
+    before.attrs.update(parent.attrs)
+    for sibling in list(parent.contents):
+        if sibling is node:
+            break
+        before.append(sibling.extract())
+    if before.contents:
+        parent.insert_before(before)
+        parent.attrs.pop("id", None)
+    parent.insert_before(node.extract())
+    if not parent.contents:
+        if parent.has_attr("id") and not node.has_attr("id"):
+            node["id"] = parent["id"]
+        parent.decompose()
+
+
+class _DocxImages:
+    def __init__(
+        self,
+        render: Callable[..., Optional[str]],
+        options: dict[str, Any],
+    ):
+        self.omitted = False
+        self._render = render
+        self._options = options
+        self._attribute = "data-markitdown-image-" + uuid4().hex
+        self._fragments: dict[str, BeautifulSoup] = {}
+
+    def convert_image(self, image: Any) -> list[Any]:
+        stream_info = StreamInfo(
+            mimetype=image.content_type,
+            extension=(
+                mimetypes.guess_extension(image.content_type)
+                if image.content_type
+                else None
+            ),
+        )
+        try:
+            image_stream = image.open()
+        except (KeyError, InvalidFileReferenceError):
+            if not self._options.get("extract_media"):
+                raise
+            self.omitted = True
+            return []
+        with image_stream as image_stream:
+            try:
+                fragment = self._render(image_stream, stream_info, **self._options)
+            except InvalidFileReferenceError as exc:
+                # Mammoth swallows this exception for missing document images,
+                # but an error raised by the override must reach the dispatcher.
+                raise RuntimeError("_image_to_html failed") from exc
+        soup = _parse_image_html(fragment)
+        if soup is None:
+            return images.data_uri(image)
+
+        key = str(len(self._fragments))
+        self._fragments[key] = soup
+        return [html.element("img", {self._attribute: key})]
+
+    def replace_images(self, html_content: str) -> str:
+        if not self._fragments:
+            return html_content
+
+        soup = BeautifulSoup(html_content, "html.parser")
+        for image in soup.find_all("img", attrs={self._attribute: True}):
+            key = image[self._attribute]
+            assert isinstance(key, str)
+            fragment = self._fragments[key]
+            blocks = fragment.find_all(_BLOCK_ELEMENTS)
+            links = fragment.find_all("a")
+            image.replace_with(*list(fragment.contents))
+
+            # Only generated content is lifted, never its table cell or list item.
+            for block in blocks:
+                while (
+                    isinstance(block.parent, Tag)
+                    and block.parent.name in _PHRASING_CONTAINERS
+                ):
+                    _lift_out_of_parent(block, soup)
+            for link in links:
+                outer_link = link.find_parent("a")
+                if outer_link is not None:
+                    while link.parent is not outer_link:
+                        _lift_out_of_parent(link, soup)
+                    _lift_out_of_parent(link, soup)
+        return str(soup)