fylepy 0.1.0__py3-none-any.whl

fyle/_readers/image/stdlib.py

@@ -0,0 +1,136 @@
+"""Image reader implementation backed by the Python standard library.
+
+Strategy:
+- Detect the concrete image subtype (png / jpeg / webp / ...) from magic
+  bytes or the file extension.
+- Wrap the raw bytes as a ``data:<mime>;base64,<payload>`` URL.
+- Expose the image both as an ``Image`` element and as a Markdown image
+  token in ``Page.text`` so the document can be fed directly into a
+  multimodal LLM prompt.
+- Optionally record pixel dimensions in ``meta.warnings`` if Pillow is
+  available; failure to read dimensions never fails the parse.
+
+Why no OCR here: fyle's contract is "open anything, return Markdown".
+OCR / VLM calls are an application-level choice (network, cost, model
+selection) and are deliberately kept out of the reader layer.
+"""
+from __future__ import annotations
+
+import base64
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from ..base import Reader
+from ..._core.document import Document, Image, Meta, Page
+from ...errors import ParseError
+
+
+# Magic-byte prefixes that pin down the concrete image subtype. Order
+# matters: WEBP also starts with ``RIFF``, so it is checked after the
+# RIFF-aware case below.
+_MAGIC_MIME: list[tuple[bytes, str]] = [
+    (b"\x89PNG\r\n\x1a\n", "image/png"),
+    (b"\xff\xd8\xff", "image/jpeg"),
+    (b"GIF87a", "image/gif"),
+    (b"GIF89a", "image/gif"),
+    (b"BM", "image/bmp"),
+    # TIFF little / big endian
+    (b"II*\x00", "image/tiff"),
+    (b"MM\x00*", "image/tiff"),
+]
+
+# Fallback extension -> MIME if magic-byte detection missed.
+_EXT_MIME: dict[str, str] = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".gif": "image/gif",
+    ".bmp": "image/bmp",
+    ".tif": "image/tiff",
+    ".tiff": "image/tiff",
+    ".webp": "image/webp",
+}
+
+
+def _detect_mime(data: bytes, source_name: Optional[str]) -> str:
+    """Return the best-guess ``image/*`` MIME for ``data``.
+
+    Falls back to ``application/octet-stream`` only if nothing matches;
+    callers receive a usable data URL either way.
+    """
+    # WEBP: ``RIFF....WEBP`` (4-byte size between RIFF and WEBP).
+    if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
+        return "image/webp"
+    for prefix, mime in _MAGIC_MIME:
+        if data.startswith(prefix):
+            return mime
+    if source_name:
+        ext = Path(source_name).suffix.lower()
+        if ext in _EXT_MIME:
+            return _EXT_MIME[ext]
+    return "application/octet-stream"
+
+
+def _try_dimensions(data: bytes, warnings: list[str]) -> Optional[tuple[int, int]]:
+    """Best-effort (width, height) via Pillow. Never raises on failure."""
+    try:
+        from PIL import Image as _PILImage
+    except ImportError:
+        return None
+    try:
+        import io as _io
+        with _PILImage.open(_io.BytesIO(data)) as im:
+            return int(im.width), int(im.height)
+    except Exception as e:
+        warnings.append(f"image dimension read failed: {e}")
+        return None
+
+
+class ImageReader(Reader):
+    name = "image-stdlib"
+    formats = ("image",)
+    is_default = True
+
+    def read(self, data: bytes, *, source_name: Optional[str] = None, **_) -> Document:
+        if not data:
+            raise ParseError("image-stdlib reader: input is empty")
+
+        warnings: list[str] = []
+
+        mime = _detect_mime(data, source_name)
+        b64 = base64.b64encode(data).decode("ascii")
+        data_url = f"data:{mime};base64,{b64}"
+
+        # Caption / title: filename stem if available, else the format name.
+        stem: Optional[str] = None
+        if source_name:
+            try:
+                stem = Path(source_name).stem or None
+            except (TypeError, ValueError):
+                stem = None
+        caption = stem or mime.split("/", 1)[-1]
+
+        dims = _try_dimensions(data, warnings)
+        if dims is not None:
+            warnings.append(f"image dimensions: {dims[0]}x{dims[1]}")
+
+        # Markdown image token. LLM prompts can consume this verbatim.
+        page_text = f"![{caption}]({data_url})"
+
+        image = Image(data_url=data_url, data=data, caption=caption, page=1)
+
+        try:
+            page = Page(text=page_text, number=1, images=[image])
+            meta = Meta(
+                format="image",
+                pages=1,
+                size=len(data),
+                title=stem,
+                reader=self.name,
+                created_at=datetime.now(timezone.utc),
+                warnings=warnings,
+            )
+            return Document(pages=[page], meta=meta)
+        except Exception as e:  # pragma: no cover - defensive
+            raise ParseError(f"image reader failed to build Document: {e}") from e

fyle/_readers/markdown/__init__.py

@@ -0,0 +1,6 @@
+"""Markdown reader — passthrough of ``.md`` / ``.markdown`` files.
+
+File name inside the subpackage is the *core driver library* (the Python
+standard library — no external library is used for passthrough).
+"""
+from . import stdlib  # noqa: F401

fyle/_readers/markdown/stdlib.py

@@ -0,0 +1,61 @@
+"""Markdown reader backed by the Python standard library.
+
+For ``.md`` / ``.markdown`` inputs the content *is already* our target
+representation, so ``Page.text`` is a **byte-preserving passthrough** of
+the decoded source. On top of that passthrough we populate ``doc.tables``
+and ``doc.images`` via the shared Markdown structure extractor
+(``_md_structure``), which delegates to ``markdown-it-py`` + BeautifulSoup.
+
+File naming rule: ``stdlib.py`` — the *core* dependency is the Python
+standard library (decoding). ``markdown-it-py`` and BeautifulSoup are
+ancillary parsers used for structural extraction and do not determine the
+file name.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from ..base import Reader
+from ..._core.document import Document, Meta, Page
+from ...errors import ParseError
+from .._md_structure import extract_images, extract_tables
+from ..text.stdlib import _decode_text
+
+
+class MarkdownReader(Reader):
+    name = "markdown-stdlib"
+    formats = ("markdown",)
+    is_default = True
+
+    def read(self, data: bytes, *, source_name: Optional[str] = None, **_) -> Document:
+        warnings: list[str] = []
+        text, decode_warning = _decode_text(data)
+        if decode_warning:
+            warnings.append(decode_warning)
+
+        title: Optional[str] = None
+        if source_name:
+            try:
+                title = Path(source_name).stem or None
+            except (TypeError, ValueError):
+                title = None
+
+        tables = extract_tables(text, page=1, warnings=warnings)
+        images = extract_images(text, page=1, warnings=warnings, include_html_img=True)
+
+        try:
+            page = Page(text=text, number=1, tables=tables, images=images)
+            meta = Meta(
+                format="markdown",
+                pages=1,
+                size=len(data),
+                title=title,
+                reader=self.name,
+                created_at=datetime.now(timezone.utc),
+                warnings=warnings,
+            )
+            return Document(pages=[page], meta=meta)
+        except Exception as e:  # pragma: no cover - defensive
+            raise ParseError(f"markdown-stdlib reader failed: {e}") from e

fyle/_readers/pdf/__init__.py

@@ -0,0 +1,2 @@
+"""PDF readers — default ``pymupdf4llm``; ``pdfplumber`` / ``pypdf`` as alternatives."""
+from . import pymupdf4llm  # noqa: F401

fyle/_readers/pdf/pymupdf4llm.py

@@ -0,0 +1,202 @@
+"""PDF reader powered by ``pymupdf4llm`` — the default PDF reader.
+
+Rationale: ``pymupdf4llm`` (maintained by the PyMuPDF team) already produces
+LLM-friendly Markdown from PDFs and handles headings, lists, tables, and
+embedded images. It is substantially less work than rebuilding those heuristics
+on top of raw PyMuPDF.
+
+Strategy:
+- Open the PDF once with ``pymupdf`` to read document-level metadata.
+- Call ``pymupdf4llm.to_markdown(doc, page_chunks=True, embed_images=True)`` to
+  get per-page Markdown with images inlined as base64 data URLs.
+- Populate ``Page.text`` from the returned per-page Markdown. ``Page.tables``
+  and ``Page.images`` are back-filled via the shared ``_md_structure``
+  extractor (GFM table tokens + native / HTML image references) so the
+  surface stays consistent with the Markdown / DOCX / HTML readers.
+"""
+from __future__ import annotations
+
+import contextlib
+import os
+import sys
+from datetime import datetime
+from typing import Iterator, Optional
+
+from ..._core.document import Document, Meta, Page
+from ...errors import ParseError
+from ..base import Reader
+from .._md_structure import extract_images, extract_tables
+
+
+class PyMuPdf4LlmReader(Reader):
+    name = "pymupdf4llm"
+    formats = ("pdf",)
+    is_default = True
+
+    def read(self, data: bytes, *, source_name: Optional[str] = None, **_) -> Document:
+        try:
+            import pymupdf as fitz  # PyMuPDF >= 1.24
+        except ImportError:
+            try:
+                import fitz  # type: ignore[no-redef]
+            except ImportError as e:
+                raise ParseError(
+                    "PyMuPDF (pymupdf) is required: pip install pymupdf"
+                ) from e
+
+        try:
+            import pymupdf4llm  # noqa: F401
+        except ImportError as e:
+            raise ParseError(
+                "pymupdf4llm is required for PDF parsing: pip install pymupdf4llm"
+            ) from e
+
+        warnings: list[str] = []
+
+        try:
+            pdf = fitz.open(stream=data, filetype="pdf")
+        except Exception as e:
+            raise ParseError(f"Failed to open PDF: {e}") from e
+
+        try:
+            try:
+                # pymupdf4llm + MuPDF emit a ``=== Document parser
+                # messages ===`` banner (e.g. "Using Tesseract for OCR
+                # processing.", "OCR on page.number=0/1.") during
+                # ``to_markdown``. Some lines come from Python
+                # ``print`` calls, but the per-page OCR progress and
+                # MuPDF warnings are emitted by the C extension
+                # directly via ``fprintf(stderr, ...)`` / ``stdout``,
+                # which ``contextlib.redirect_stdout`` cannot touch
+                # because it only rebinds ``sys.stdout`` at the Python
+                # level. We therefore redirect file descriptors 1 and 2
+                # at the OS level — the only way to silence a C
+                # extension's writes — for the duration of the call.
+                # ``show_progress=False`` still helps but does not
+                # cover the banner or the OCR lines.
+                with _silence_fds():
+                    chunks = pymupdf4llm.to_markdown(
+                        pdf,
+                        page_chunks=True,
+                        embed_images=True,
+                        write_images=False,
+                        show_progress=False,
+                    )
+            except Exception as e:
+                raise ParseError(f"pymupdf4llm.to_markdown failed: {e}") from e
+
+            pages: list[Page] = []
+            for idx, chunk in enumerate(chunks, start=1):
+                text = (chunk.get("text") or "").strip()
+                tables = extract_tables(text, page=idx, warnings=warnings)
+                images = extract_images(
+                    text, page=idx, warnings=warnings, include_html_img=True
+                )
+                pages.append(Page(text=text, number=idx, tables=tables, images=images))
+
+            if not pages:
+                # Degenerate case: pymupdf4llm returned nothing. Keep at least
+                # one empty page so ``doc.pages`` is never an empty list.
+                pages.append(Page(text="", number=1))
+                warnings.append("pymupdf4llm returned no page chunks")
+
+            md = pdf.metadata or {}
+            title = _clean(md.get("title"))
+            author = _clean(md.get("author"))
+            created_at = _parse_pdf_date(md.get("creationDate"))
+
+            if not title and source_name:
+                title = source_name
+
+            meta = Meta(
+                format="pdf",
+                pages=len(pages),
+                size=len(data),
+                title=title,
+                author=author,
+                created_at=created_at,
+                reader=self.name,
+                warnings=warnings,
+            )
+        finally:
+            try:
+                pdf.close()
+            except Exception:
+                pass
+
+        return Document(pages=pages, meta=meta)
+
+
+def _clean(value) -> Optional[str]:
+    if not value:
+        return None
+    if isinstance(value, bytes):
+        try:
+            value = value.decode("utf-8", errors="ignore")
+        except Exception:
+            return None
+    s = str(value).strip()
+    return s or None
+
+
+@contextlib.contextmanager
+def _silence_fds() -> Iterator[None]:
+    """Redirect fds 1 and 2 to ``/dev/null`` at the OS level.
+
+    ``contextlib.redirect_stdout`` only rebinds ``sys.stdout`` at the
+    Python level, so any C extension that calls ``fprintf(stdout, ...)``
+    or ``fprintf(stderr, ...)`` directly (MuPDF warnings, the Tesseract
+    OCR progress banner, etc.) bypasses it. Duplicating ``/dev/null``
+    onto file descriptors 1 and 2 is the only way to silence those
+    writes without a third-party dependency. fds are restored on exit
+    even if the wrapped call raises.
+    """
+    # Flush Python-level buffers so nothing we care about ends up
+    # arriving at the silenced fds.
+    sys.stdout.flush()
+    sys.stderr.flush()
+    saved_stdout_fd = os.dup(1)
+    saved_stderr_fd = os.dup(2)
+    devnull_fd = os.open(os.devnull, os.O_WRONLY)
+    try:
+        os.dup2(devnull_fd, 1)
+        os.dup2(devnull_fd, 2)
+        try:
+            yield
+        finally:
+            sys.stdout.flush()
+            sys.stderr.flush()
+            os.dup2(saved_stdout_fd, 1)
+            os.dup2(saved_stderr_fd, 2)
+    finally:
+        os.close(devnull_fd)
+        os.close(saved_stdout_fd)
+        os.close(saved_stderr_fd)
+
+
+def _parse_pdf_date(s) -> Optional[datetime]:
+    """Parse the PDF standard date format ``D:YYYYMMDDHHmmSS+OFFSET``.
+
+    Returns ``None`` on any parse failure.
+    """
+    if not s:
+        return None
+    if isinstance(s, bytes):
+        try:
+            s = s.decode("utf-8", errors="ignore")
+        except Exception:
+            return None
+    s = str(s).strip()
+    if s.startswith("D:"):
+        s = s[2:]
+    if len(s) >= 14:
+        try:
+            return datetime.strptime(s[:14], "%Y%m%d%H%M%S")
+        except ValueError:
+            pass
+    if len(s) >= 8:
+        try:
+            return datetime.strptime(s[:8], "%Y%m%d")
+        except ValueError:
+            pass
+    return None

fyle/_readers/pptx/__init__.py

@@ -0,0 +1,7 @@
+"""PPTX reader.
+
+File naming rule: ``python_pptx.py`` — the core driver is ``python-pptx``.
+"""
+from __future__ import annotations
+
+from . import python_pptx  # noqa: F401