multixtract 0.1.1__py3-none-any.whl

multixtract/extractors/html.py

@@ -0,0 +1,127 @@
+"""HTML extractor (.html / .htm).
+
+Requires beautifulsoup4 (optional extra [html]).
+"""
+from __future__ import annotations
+
+import logging
+import os
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+log = logging.getLogger("multixtract")
+
+_REMOVE_TAGS = {"script", "style", "nav", "header", "footer"}
+_CHARSET_RE = re.compile(rb'charset=["\']?([\w-]+)', re.IGNORECASE)
+
+
+def _detect_encoding(raw: bytes) -> str:
+    m = _CHARSET_RE.search(raw[:4096])
+    if m:
+        return m.group(1).decode("ascii", errors="replace")
+    return "utf-8"
+
+
+def _parse_table(table_tag) -> List[List[str]]:
+    rows = []
+    for tr in table_tag.find_all("tr"):
+        cells = [td.get_text(strip=True) for td in tr.find_all(["th", "td"])]
+        if cells:
+            rows.append(cells)
+    return rows
+
+
+def _section_text_and_tables(tags, soup_cls):
+    """Given a list of tags forming a section, return (text, tables)."""
+    # Build a temporary wrapper so get_text works across the tag list.
+    wrapper = soup_cls.new_tag("div")
+    for tag in tags:
+        wrapper.append(tag.__copy__())
+    for bad in wrapper.find_all(_REMOVE_TAGS):
+        bad.decompose()
+    text = wrapper.get_text(separator="\n", strip=True)
+    tables = [_parse_table(tbl) for tbl in wrapper.find_all("table")]
+    tables = [t for t in tables if t]
+    return text, tables
+
+
+class HtmlExtractor:
+    """DocumentExtractor for HTML files."""
+
+    extensions: Tuple[str, ...] = (".html", ".htm")
+
+    def extract(
+        self,
+        path: str,
+        image_filter: Optional[Any] = None,
+    ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
+        base_name = os.path.splitext(os.path.basename(path))[0]
+        empty = {"_base_name": base_name, "metadata": {}, "pgs": []}
+        try:
+            from bs4 import BeautifulSoup
+        except ImportError as exc:
+            raise ImportError(
+                "HTML support requires beautifulsoup4: pip install 'multixtract[html]'"
+            ) from exc
+        try:
+            with open(path, "rb") as fh:
+                raw = fh.read()
+
+            encoding = _detect_encoding(raw)
+            html = raw.decode(encoding, errors="replace")
+            soup = BeautifulSoup(html, "html.parser")
+
+            title = ""
+            title_tag = soup.find("title")
+            if title_tag:
+                title = title_tag.get_text(strip=True)
+
+            body = soup.body or soup
+
+            # Collect top-level nodes grouped by H1 boundaries.
+            # Each group: list of Tag objects that belong to one page.
+            groups: List[List[Any]] = []
+            current: List[Any] = []
+            for child in body.children:
+                if getattr(child, "name", None) == "h1":
+                    if current:
+                        groups.append(current)
+                    current = [child]
+                else:
+                    current.append(child)
+            if current:
+                groups.append(current)
+
+            # Drop groups that are pure whitespace (e.g. between tags)
+            groups = [g for g in groups if any(
+                getattr(t, "name", None) or str(t).strip() for t in g
+            )]
+
+            if not groups:
+                groups = [[body]]
+
+            pages = []
+            for pg_num, group in enumerate(groups, start=1):
+                text, tables = _section_text_and_tables(group, soup)
+                pages.append({
+                    "pg_num": pg_num,
+                    "txt": text,
+                    "tables": tables,
+                    "imgs": [],
+                })
+
+            document = {
+                "_base_name": base_name,
+                "metadata": {
+                    "page_count": len(pages),
+                    "format": "html",
+                    "title": title,
+                },
+                "pgs": pages,
+            }
+            return document, []
+        except ImportError:
+            raise
+        except Exception:
+            log.warning("HtmlExtractor failed for %s", path, exc_info=True)
+            return empty, []

multixtract/extractors/image.py

@@ -0,0 +1,149 @@
+"""Image file extractor (.png / .jpg / .jpeg / .tiff / .tif / .webp / .bmp).
+
+Each image file becomes a one-page document whose single image is passed
+directly to the vision pipeline. Multi-page TIFF files produce one page per
+frame.
+
+``image_filter`` is accepted in the signature to satisfy the
+``DocumentExtractor`` protocol but is intentionally ignored — a standalone
+image file is an explicit user choice, not embedded noise to filter.
+
+Pillow is a mandatory core dependency so no optional extra is needed.
+"""
+from __future__ import annotations
+
+import io
+import logging
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+log = logging.getLogger("multixtract")
+
+
+class ImageExtractor:
+    """DocumentExtractor for standalone image files."""
+
+    extensions: Tuple[str, ...] = (
+        ".png", ".jpg", ".jpeg", ".tiff", ".tif", ".webp", ".bmp",
+    )
+
+    def extract(
+        self,
+        path: str,
+        image_filter: Optional[Any] = None,
+    ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
+        """Return (document, prepared_images) for one image file.
+
+        ``image_filter`` is ignored — see module docstring.
+        Never raises; returns an empty document on failure.
+        """
+        from PIL import Image
+
+        base_name = os.path.splitext(os.path.basename(path))[0]
+        raw_ext = os.path.splitext(path)[1].lstrip(".").lower()
+        # Normalise .tif -> tiff, .jpg -> jpeg for consistency.
+        ext_map = {"tif": "tiff", "jpg": "jpeg"}
+        ext = ext_map.get(raw_ext, raw_ext)
+
+        empty_doc = {"_base_name": base_name, "metadata": {}, "pgs": []}
+
+        try:
+            with open(path, "rb") as fh:
+                file_bytes = fh.read()
+
+            with Image.open(io.BytesIO(file_bytes)) as img:
+                is_multipage = hasattr(img, "n_frames") and img.n_frames > 1
+
+                if is_multipage:
+                    return self._extract_multipage_tiff(
+                        img, file_bytes, base_name, ext
+                    )
+
+                width, height = img.size
+
+            document = {
+                "_base_name": base_name,
+                "metadata": {"page_count": 1, "format": ext},
+                "pgs": [
+                    {
+                        "pg_num": 1,
+                        "txt": "",
+                        "tables": [],
+                        "imgs": [],
+                    }
+                ],
+            }
+            prepared = [
+                {
+                    "image_id":   f"{base_name}__p1_img0",
+                    "page_number": 1,
+                    "img_idx":    0,
+                    "image_bytes": file_bytes,
+                    "ext":        ext,
+                    "width":      width,
+                    "height":     height,
+                    "img_path":   f"pg1_img0.{ext}",
+                }
+            ]
+            return document, prepared
+
+        except Exception:
+            log.warning("ImageExtractor failed for %s", path, exc_info=True)
+            return empty_doc, []
+
+    def _extract_multipage_tiff(
+        self,
+        img,
+        file_bytes: bytes,
+        base_name: str,
+        ext: str,
+    ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
+        pages = []
+        prepared = []
+
+        for frame_idx in range(img.n_frames):
+            pg_num = frame_idx + 1
+            try:
+                img.seek(frame_idx)
+                width, height = img.size
+
+                buf = io.BytesIO()
+                frame = img.convert("RGB")
+                try:
+                    frame.save(buf, format="PNG")
+                finally:
+                    frame.close()
+                frame_bytes = buf.getvalue()
+
+                pages.append(
+                    {
+                        "pg_num": pg_num,
+                        "txt": "",
+                        "tables": [],
+                        "imgs": [],
+                    }
+                )
+                prepared.append(
+                    {
+                        "image_id":    f"{base_name}__p{pg_num}_img0",
+                        "page_number": pg_num,
+                        "img_idx":     0,
+                        "image_bytes": frame_bytes,
+                        "ext":         "png",
+                        "width":       width,
+                        "height":      height,
+                        "img_path":    f"pg{pg_num}_img0.png",
+                    }
+                )
+            except Exception:
+                log.warning(
+                    "ImageExtractor: failed to read frame %d of %s",
+                    frame_idx, base_name, exc_info=True,
+                )
+
+        document = {
+            "_base_name": base_name,
+            "metadata": {"page_count": len(pages), "format": ext},
+            "pgs": pages,
+        }
+        return document, prepared

multixtract/extractors/legacy.py

@@ -0,0 +1,148 @@
+"""Legacy Office format extractors (.doc, .ppt, .xls) via LibreOffice conversion.
+
+The old binary formats (.doc, .ppt) have no reliable native Python parser, so
+this module converts them to a modern format with a headless LibreOffice call,
+then delegates extraction to the registered extractor for the target format.
+
+Strategy (per the project's design decision):
+  * Prefer converting to the modern *native* equivalent (.doc -> .docx,
+    .ppt -> .pptx) for the best fidelity and a consistent output schema.
+  * Fall back to PDF when no native extractor is registered (the bundled
+    PdfExtractor always works), so legacy files are handled out of the box.
+  * To force PDF, construct ``ConvertingExtractor(..., targets=(".pdf",))``.
+
+LibreOffice is a SYSTEM dependency (NOT pip-installable). If it is missing,
+:meth:`ConvertingExtractor.extract` raises a clear, actionable error telling the
+user to install LibreOffice or pre-convert the file to .docx/.pptx themselves.
+
+    apt-get install libreoffice
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import tempfile
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+from .registry import ExtractorRegistry, default_registry
+
+
+def find_libreoffice() -> Optional[str]:
+    """Return the path to the LibreOffice/soffice binary, or None if absent."""
+    for name in ("libreoffice", "soffice"):
+        path = shutil.which(name)
+        if path:
+            return path
+    return None
+
+
+def convert_with_libreoffice(
+    src_path: str,
+    target_ext: str,
+    out_dir: str,
+    timeout: int = 120,
+) -> str:
+    """Convert *src_path* to *target_ext* (e.g. ``".docx"`` / ``".pdf"``) inside
+    *out_dir* using headless LibreOffice. Returns the converted file path.
+    """
+    soffice = find_libreoffice()
+    if soffice is None:
+        raise RuntimeError(
+            "LibreOffice is required to process legacy .doc/.ppt files but was "
+            "not found. Install it (apt-get install libreoffice) or convert the "
+            "file to a modern format (.docx/.pptx) yourself first."
+        )
+    fmt = target_ext.lstrip(".")
+    proc = subprocess.run(
+        [soffice, "--headless", "--convert-to", fmt, "--outdir", out_dir, src_path],
+        capture_output=True, text=True, timeout=timeout,
+    )
+    if proc.returncode != 0:
+        raise RuntimeError(
+            f"LibreOffice failed to convert {os.path.basename(src_path)} -> {fmt} "
+            f"(exit {proc.returncode}): {(proc.stderr or '').strip()[:300]}"
+        )
+    # LibreOffice normalizes the output filename (spaces → underscores, special
+    # chars dropped/replaced), so we can't predict the exact stem. Scan out_dir
+    # for any file with the right extension instead of checking a fixed path.
+    matches = [
+        f for f in os.listdir(out_dir)
+        if f.lower().endswith(f".{fmt}")
+    ]
+    if not matches:
+        raise RuntimeError(
+            f"LibreOffice exited 0 but produced no .{fmt} file in {out_dir!r} "
+            f"(converting {os.path.basename(src_path)})"
+        )
+    return os.path.join(out_dir, matches[0])
+
+
+class ConvertingExtractor:
+    """DocumentExtractor that converts a legacy file, then delegates.
+
+    It tries each extension in *targets* in order, using the first one that has
+    a registered extractor, converts the input to that format via LibreOffice,
+    and delegates extraction to that target extractor. This means it transparently
+    upgrades to native extraction (.docx/.pptx) once those extractors exist,
+    while working today via the PDF fallback.
+    """
+
+    def __init__(
+        self,
+        source_exts: Iterable[str],
+        targets: Iterable[str] = (".pdf",),
+        registry: Optional[ExtractorRegistry] = None,
+        timeout: int = 120,
+    ) -> None:
+        self.extensions: Tuple[str, ...] = tuple(
+            e if e.startswith(".") else f".{e}" for e in source_exts
+        )
+        self.targets: Tuple[str, ...] = tuple(
+            t if t.startswith(".") else f".{t}" for t in targets
+        )
+        self._registry = registry
+        self.timeout = timeout
+
+    def _resolve_target(self, reg: ExtractorRegistry):
+        """Return ``(target_ext, target_extractor)`` for the first target that
+        has a registered extractor."""
+        for ext in self.targets:
+            try:
+                return ext, reg.get(f"_probe{ext}")
+            except ValueError:
+                continue
+        raise RuntimeError(
+            f"No extractor registered for any conversion target {self.targets} "
+            f"of {self.extensions}. Register a target extractor (e.g. PdfExtractor) first."
+        )
+
+    def extract(
+        self,
+        path: str,
+        image_filter: Optional[Any] = None,
+    ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
+        reg = self._registry or default_registry
+        target_ext, delegate = self._resolve_target(reg)  # fail fast before converting
+        with tempfile.TemporaryDirectory(prefix="docextract_legacy_") as tmp:
+            converted = convert_with_libreoffice(path, target_ext, tmp, timeout=self.timeout)
+            document, prepared = delegate.extract(converted, image_filter=image_filter)
+        # Preserve the ORIGINAL file's base name, not the temp converted name.
+        document["_base_name"] = os.path.splitext(os.path.basename(path))[0]
+        document.setdefault("metadata", {})["converted_from"] = os.path.splitext(path)[1].lower()
+        return document, prepared
+
+
+# Built-in legacy extractors.
+#   .doc -> prefer .docx (native), fall back to .pdf
+#   .ppt -> prefer .pptx (native), fall back to .pdf
+#   .xls -> prefer .xlsx (native), fall back to .pdf
+# DocxExtractor, PptxExtractor, and ExcelExtractor are registered in
+# extractors/__init__.py, so legacy formats automatically get native high-fidelity
+# extraction via LibreOffice conversion. PDF remains the fallback.
+legacy_doc_extractor = ConvertingExtractor((".doc",), targets=(".docx", ".pdf"))
+legacy_ppt_extractor = ConvertingExtractor((".ppt",), targets=(".pptx", ".pdf"))
+legacy_odt_extractor = ConvertingExtractor((".odt",), targets=(".docx", ".pdf"))
+legacy_odp_extractor = ConvertingExtractor((".odp",), targets=(".pptx", ".pdf"))
+legacy_ods_extractor = ConvertingExtractor((".ods",), targets=(".xlsx", ".pdf"))
+legacy_xls_extractor = ConvertingExtractor((".xls",), targets=(".xlsx", ".pdf"))

multixtract/extractors/markdown.py

@@ -0,0 +1,104 @@
+"""Markdown extractor (.md).
+
+Splits on H1 headings into logical pages. Registered after TextExtractor so
+it wins for .md — later registration overwrites earlier in the registry.
+"""
+from __future__ import annotations
+
+import logging
+import os
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+log = logging.getLogger("multixtract")
+
+_H1_RE = re.compile(r"^# .+", re.MULTILINE)
+_H2_RE = re.compile(r"^## .+", re.MULTILINE)
+# A pipe-table row: starts and ends with |, contains at least one |
+_TABLE_ROW_RE = re.compile(r"^\|.+\|$")
+# Separator row: only |, -, :, and spaces
+_TABLE_SEP_RE = re.compile(r"^\|[\s\-:|]+\|$")
+
+
+def _parse_tables(text: str) -> List[List[List[str]]]:
+    """Return all GFM pipe tables found in text as list-of-list-of-strings."""
+    tables = []
+    lines = text.splitlines()
+    i = 0
+    while i < len(lines):
+        if _TABLE_ROW_RE.match(lines[i].strip()):
+            block = []
+            while i < len(lines) and _TABLE_ROW_RE.match(lines[i].strip()):
+                row = lines[i].strip()
+                if _TABLE_SEP_RE.match(row):
+                    i += 1
+                    continue
+                cells = [c.strip() for c in row.strip("|").split("|")]
+                block.append(cells)
+                i += 1
+            if block:
+                tables.append(block)
+        else:
+            i += 1
+    return tables
+
+
+def _split_on_h1(text: str) -> List[str]:
+    """Split text into sections on H1 boundaries, preserving the heading."""
+    positions = [m.start() for m in _H1_RE.finditer(text)]
+    if not positions:
+        return [text]
+    sections = []
+    for idx, start in enumerate(positions):
+        end = positions[idx + 1] if idx + 1 < len(positions) else len(text)
+        sections.append(text[start:end])
+    # Content before the first H1 (preamble), if any
+    if positions[0] > 0:
+        sections.insert(0, text[: positions[0]])
+    return [s for s in sections if s.strip()]
+
+
+class MarkdownExtractor:
+    """DocumentExtractor for Markdown files."""
+
+    extensions: Tuple[str, ...] = (".md",)
+
+    def extract(
+        self,
+        path: str,
+        image_filter: Optional[Any] = None,
+    ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
+        base_name = os.path.splitext(os.path.basename(path))[0]
+        empty = {"_base_name": base_name, "metadata": {}, "pgs": []}
+        try:
+            try:
+                with open(path, encoding="utf-8") as fh:
+                    text = fh.read()
+            except UnicodeDecodeError:
+                with open(path, encoding="latin-1", errors="replace") as fh:
+                    text = fh.read()
+
+            sections = _split_on_h1(text)
+            pages = []
+            for pg_num, section in enumerate(sections, start=1):
+                pages.append({
+                    "pg_num": pg_num,
+                    "txt": section,
+                    "tables": _parse_tables(section),
+                    "imgs": [],
+                })
+
+            document = {
+                "_base_name": base_name,
+                "metadata": {
+                    "page_count": len(pages),
+                    "format": "md",
+                    "h1_count": len(_H1_RE.findall(text)),
+                    "h2_count": len(_H2_RE.findall(text)),
+                },
+                "pgs": pages,
+            }
+            return document, []
+        except Exception:
+            log.warning("MarkdownExtractor failed for %s", path, exc_info=True)
+            return empty, []