fylepy 0.1.0__py3-none-any.whl

fyle/_readers/audio/faster_whisper.py

@@ -0,0 +1,162 @@
+"""Audio reader backed by ``faster-whisper``.
+
+Produces a ``Document`` whose ``doc.text`` is the time-stamped Markdown
+transcript of the input audio. One ``Page`` per file; per-segment
+timestamps (``[MM:SS]``) are inlined so the LLM can cite moments.
+
+Why this file exists
+--------------------
+Transcription is the only credible way to turn audio into something an
+LLM "sees". Whisper is the only open-source ASR model with broad
+multilingual coverage that is practical on a CPU; ``faster-whisper``
+(CTranslate2 int8) is the fastest Python binding and avoids the
+PyTorch dependency chain. See ``.._whisper`` for the shared model cache
+and transcript formatter.
+
+Source-path vs bytes
+--------------------
+faster-whisper decodes audio via PyAV, which is happiest with a real
+filesystem path. We write the bytes to a NamedTemporaryFile with the
+right suffix (important: PyAV sometimes sniffs the container from the
+extension), transcribe, then delete. The temp file is **not** the
+input file itself even when a local ``source_path`` is available —
+keeping a single code path simplifies URL / bytes / file-like inputs.
+
+File naming rule: ``faster_whisper.py`` — the core driver library.
+"""
+from __future__ import annotations
+
+import os
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from .. import _whisper
+from ..base import Reader
+from ..._core.document import Document, Meta, Page
+from ...errors import ParseError
+
+
+class FasterWhisperAudioReader(Reader):
+    name = "faster-whisper"
+    formats = ("audio",)
+    is_default = True
+
+    def read(
+        self,
+        data: bytes,
+        *,
+        source_name: Optional[str] = None,
+        source_path: Optional[str] = None,
+        **_,
+    ) -> Document:
+        if not data:
+            raise ParseError("faster-whisper reader: input is empty")
+
+        # Preserve the suffix so PyAV can sniff the container correctly.
+        # ``.mp3`` vs ``.m4a`` share different demuxers; a mislabelled
+        # suffix occasionally trips PyAV's probing.
+        suffix = _pick_suffix(source_name)
+        warnings: list[str] = []
+
+        tmp = tempfile.NamedTemporaryFile(
+            prefix="fyle-audio-",
+            suffix=suffix,
+            delete=False,
+        )
+        try:
+            tmp.write(data)
+            tmp.close()
+            try:
+                segments, info = _whisper.transcribe(tmp.name)
+            except Exception as e:
+                raise ParseError(
+                    f"faster-whisper reader: transcription failed: {e}"
+                ) from e
+        finally:
+            try:
+                os.unlink(tmp.name)
+            except OSError:
+                pass
+
+        text = _render_page_text(
+            source_name=source_name,
+            duration_sec=float(info.get("duration", 0.0) or 0.0),
+            language=info.get("language"),
+            segments=segments,
+        )
+
+        # Record what the model heard: language + duration + model size.
+        # These go into meta.warnings so they surface in the example's
+        # summary without polluting the content surface.
+        if info.get("language"):
+            warnings.append(
+                f"detected language: {info['language']} "
+                f"(p={info.get('language_probability', 0):.2f})"
+            )
+        if info.get("duration"):
+            warnings.append(
+                f"audio duration: {_whisper.format_timestamp(info['duration'])}"
+            )
+        warnings.append(f"whisper model: {_whisper.model_size()}")
+
+        title = Path(source_name).stem if source_name else None
+        page = Page(text=text, number=1)
+        meta = Meta(
+            format="audio",
+            pages=1,
+            size=len(data),
+            title=title,
+            reader=self.name,
+            created_at=datetime.now(timezone.utc),
+            warnings=warnings,
+        )
+        return Document(pages=[page], meta=meta)
+
+
+_KNOWN_AUDIO_SUFFIXES = {".mp3", ".m4a", ".wav", ".flac", ".ogg", ".opus", ".aac"}
+
+
+def _render_page_text(
+    *,
+    source_name: Optional[str],
+    duration_sec: float,
+    language: Optional[str],
+    segments: list[dict],
+) -> str:
+    """Render the audio page as Markdown with a lightweight header.
+
+    Header mirrors the video reader (Duration + Language) so callers that
+    feed ``doc.text`` into an LLM without ``doc.meta`` still see the core
+    context up front. Kept minimal on purpose: model size / confidence go
+    into ``meta.warnings`` rather than the content surface.
+    """
+    title = source_name or "(unnamed audio)"
+    dur = _whisper.format_timestamp(duration_sec) if duration_sec else "-"
+    lang = language or "-"
+    transcript = _whisper.format_transcript(segments)
+    return (
+        f"# Audio: {title}\n"
+        "\n"
+        f"- Duration: `{dur}`\n"
+        f"- Language: `{lang}`\n"
+        "\n"
+        "## Transcript\n"
+        "\n"
+        f"{transcript}"
+    )
+
+
+def _pick_suffix(source_name: Optional[str]) -> str:
+    """Return a file suffix that helps PyAV sniff the container.
+
+    Falls back to ``.wav`` (safest generic guess) when the caller cannot
+    supply a name; PyAV will still probe magic bytes but a plausible
+    extension keeps some demuxers honest.
+    """
+    if source_name:
+        ext = Path(source_name).suffix.lower()
+        if ext in _KNOWN_AUDIO_SUFFIXES:
+            return ext
+    return ".wav"

fyle/_readers/base.py

@@ -0,0 +1,70 @@
+"""Reader abstract base class — every reader subclasses this.
+
+Subclasses are auto-registered via ``__init_subclass__`` at definition time;
+no manual registration call is needed.
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from .._core.document import Document
+from .._core.registry import _register
+
+
+class Reader(ABC):
+    """Abstract base class for all readers.
+
+    Subclasses must define the following class attributes:
+      name: str                   — globally unique reader name (e.g. ``"pymupdf4llm"``).
+      formats: tuple[str, ...]    — supported format names (e.g. ``("pdf",)``).
+      is_default: bool = False    — whether this is the default reader for each
+                                    format listed in ``formats``.
+
+    Subclasses must implement ``read(self, data, *, source_name=None, source_path=None) -> Document``.
+    """
+
+    name: str = ""
+    formats: tuple[str, ...] = ()
+    is_default: bool = False
+
+    def __init_subclass__(cls, **kwargs) -> None:
+        super().__init_subclass__(**kwargs)
+        # Only auto-register concrete subclasses; skip intermediate abstract bases
+        # that still carry @abstractmethod definitions.
+        if getattr(cls, "__abstractmethods__", None):
+            return
+        if not cls.name or not cls.formats:
+            # Subclasses used purely as organisational layers (no name/formats)
+            # are allowed and are skipped by the registry.
+            return
+        _register(cls)
+
+    @abstractmethod
+    def read(
+        self,
+        data: bytes,
+        *,
+        source_name: Optional[str] = None,
+        source_path: Optional[str] = None,
+    ) -> Document:
+        """Parse raw bytes and return a ``Document``.
+
+        The dispatcher (``_core.api._normalize``) is responsible for unifying
+        path / bytes / file-like / URL inputs into bytes before the reader
+        runs; readers never handle polymorphic inputs themselves.
+
+        Args:
+            data: In-memory file contents as ``bytes``.
+            source_name: Original file name (from a local path or URL path),
+                used as a fallback for ``meta.title``.
+            source_path: Absolute filesystem path of a local file source, or
+                ``None`` for URL / bytes / file-like inputs. Most readers
+                ignore this; the archive reader uses it to extract into the
+                same directory as the source file.
+
+        Returns:
+            A ``Document``. Fatal failures raise ``fyle.ParseError``; partial
+            failures are recorded in ``doc.meta.warnings``.
+        """
+        raise NotImplementedError

fyle/_readers/csv/__init__.py

@@ -0,0 +1,6 @@
+"""CSV reader — renders a CSV as a single Markdown table via ``tabulate``.
+
+File name inside the subpackage is the *core driver library* (the Python
+standard library's ``csv`` module); ``tabulate`` is a post-processor.
+"""
+from . import stdlib  # noqa: F401

fyle/_readers/csv/stdlib.py

@@ -0,0 +1,119 @@
+"""CSV reader backed by the Python standard library (``csv`` + ``tabulate``).
+
+Strategy:
+- Decode bytes (UTF-8 with BOM; latin-1 fallback).
+- Use the stdlib ``csv`` module with dialect sniffing to tolerate common
+  separators (``,`` / ``;`` / tab / ``|``).
+- Emit a single ``Table`` and a single ``Page`` whose ``.text`` is a
+  Markdown table produced by ``tabulate`` — never hand-assembled.
+
+Assumption: first row is the header. For header-less CSV a warning is
+recorded and the first row is still used as header (downstream can inspect
+``table.rows`` for the raw cells if needed).
+
+File naming rule: ``stdlib.py`` — the driver is the stdlib ``csv`` module;
+``tabulate`` is a post-processor and does not determine the file name.
+``Reader.name`` is ``csv-stdlib`` to keep the registry key unique.
+"""
+from __future__ import annotations
+
+import csv as _csv
+import io
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from tabulate import tabulate
+
+from ..base import Reader
+from ..._core.document import Document, Meta, Page, Table
+from ...errors import ParseError
+from ..text.stdlib import _decode_text
+
+
+# Dialect sniffer only looks at this many bytes; enough for a realistic CSV header.
+_SNIFF_SAMPLE = 4096
+
+
+def _escape_md_cell(cell: str) -> str:
+    """Escape a single cell so it is safe inside a Markdown pipe table.
+
+    Markdown tables cannot carry literal ``|`` or newlines inside a cell.
+    We replace ``|`` with ``\\|`` and convert line breaks to ``<br>`` (widely
+    supported by Markdown renderers, including GitHub).
+    """
+    if not isinstance(cell, str):
+        cell = "" if cell is None else str(cell)
+    return cell.replace("|", "\\|").replace("\r\n", "<br>").replace("\n", "<br>").replace("\r", "<br>")
+
+
+class CsvReader(Reader):
+    name = "csv-stdlib"
+    formats = ("csv",)
+    is_default = True
+
+    def read(self, data: bytes, *, source_name: Optional[str] = None, **_) -> Document:
+        warnings: list[str] = []
+        text, decode_warning = _decode_text(data)
+        if decode_warning:
+            warnings.append(decode_warning)
+
+        # Sniff the dialect; fall back to the default ``excel`` (comma) dialect
+        # if the sample is too ambiguous.
+        sample = text[:_SNIFF_SAMPLE]
+        try:
+            dialect = _csv.Sniffer().sniff(sample, delimiters=",;\t|")
+        except _csv.Error:
+            dialect = _csv.excel
+
+        try:
+            rows = list(_csv.reader(io.StringIO(text), dialect=dialect))
+        except _csv.Error as e:
+            raise ParseError(f"csv parse failed: {e}") from e
+
+        if not rows:
+            headers: list[str] = []
+            body: list[list[str]] = []
+            md_table = ""
+        else:
+            headers = rows[0]
+            body = rows[1:]
+            # Pre-escape cells for Markdown-table safety before handing off to
+            # ``tabulate``. Three hazards that ``tabulate`` itself does *not*
+            # handle:
+            #   1. a literal ``|`` inside a cell breaks the column structure.
+            #   2. a newline inside a cell is not legal in a Markdown table.
+            #   3. numeric-looking strings get right-aligned and re-formatted,
+            #      which silently drops leading zeros (zip codes, phone
+            #      numbers). ``disable_numparse=True`` fixes that.
+            safe_headers = [_escape_md_cell(h) for h in headers]
+            safe_body = [[_escape_md_cell(c) for c in row] for row in body]
+            md_table = tabulate(
+                safe_body,
+                headers=safe_headers,
+                tablefmt="pipe",
+                disable_numparse=True,
+            )
+
+        title: Optional[str] = None
+        if source_name:
+            try:
+                title = Path(source_name).stem or None
+            except (TypeError, ValueError):
+                title = None
+
+        try:
+            table = Table(text=md_table, rows=body, headers=headers, page=1)
+            page = Page(text=md_table, number=1, tables=[table])
+            meta = Meta(
+                format="csv",
+                pages=1,
+                size=len(data),
+                title=title,
+                reader=self.name,
+                created_at=datetime.now(timezone.utc),
+                warnings=warnings,
+            )
+            return Document(pages=[page], meta=meta)
+        except Exception as e:  # pragma: no cover - defensive
+            raise ParseError(f"csv-stdlib reader failed to build Document: {e}") from e

fyle/_readers/docx/__init__.py

@@ -0,0 +1,6 @@
+"""DOCX reader — uses ``mammoth`` for Word → HTML → Markdown conversion.
+
+File name inside the subpackage is the *core driver library* (``mammoth``);
+``markdownify`` is a post-processor in the DOCX → HTML → Markdown chain.
+"""
+from . import mammoth  # noqa: F401

fyle/_readers/docx/mammoth.py

@@ -0,0 +1,130 @@
+"""DOCX reader backed by ``mammoth``.
+
+Pipeline: DOCX → HTML (via ``mammoth.convert_to_html``) → Markdown (via
+``markdownify``).
+
+Why the two-stage pipeline rather than ``mammoth.convert_to_markdown``?
+``mammoth`` 's Markdown target is known to be lossy on tables: it emits
+each cell as a separate paragraph instead of a Markdown pipe table. Its
+HTML target, by contrast, is faithful — tables render as real ``<table>``
+elements. ``markdownify`` then converts HTML tables into Markdown pipe
+tables cleanly. Each library does what it is good at.
+
+Inline images survive the pipeline: ``mammoth`` embeds them as
+``data:image/...;base64,...`` URLs in the HTML, and ``markdownify``
+preserves those in the Markdown output. Structural extraction (tables,
+images) is delegated to the shared ``_md_structure`` module so DOCX,
+HTML and Markdown readers all present a consistent ``doc.tables`` /
+``doc.images`` surface.
+
+All content lives in a single ``Page``. DOCX has no native pagination
+(page breaks in Word are rendering artifacts, not data), so v0.2 returns
+``pages=[one_page]`` rather than forging a page number. This is noted in
+``meta.warnings`` for transparency.
+
+``python-docx`` is used only for core properties (title / author /
+created_at) — ``mammoth`` does not expose those.
+"""
+from __future__ import annotations
+
+import io
+from datetime import timezone
+from pathlib import Path
+from typing import Optional
+
+from ..base import Reader
+from ..._core.document import Document, Meta, Page
+from ...errors import ParseError
+from .._md_structure import extract_images, extract_tables
+
+
+class DocxReader(Reader):
+    name = "mammoth"
+    formats = ("docx",)
+    is_default = True
+
+    def read(self, data: bytes, *, source_name: Optional[str] = None, **_) -> Document:
+        try:
+            import mammoth
+        except ImportError as e:
+            raise ParseError(
+                "mammoth is required for DOCX parsing: pip install mammoth"
+            ) from e
+        try:
+            from markdownify import markdownify as _to_md
+        except ImportError as e:
+            raise ParseError(
+                "markdownify is required for DOCX parsing: pip install markdownify"
+            ) from e
+
+        warnings: list[str] = []
+
+        # Stage 1: DOCX -> HTML via mammoth (tables, images, headings intact).
+        try:
+            result = mammoth.convert_to_html(io.BytesIO(data))
+        except Exception as e:
+            raise ParseError(f"mammoth.convert_to_html failed: {e}") from e
+        html = result.value or ""
+        for msg in getattr(result, "messages", []) or []:
+            m_type = getattr(msg, "type", "warning")
+            m_text = getattr(msg, "message", str(msg))
+            if m_type in ("warning", "error"):
+                warnings.append(f"mammoth: {m_text}")
+
+        # Stage 2: HTML -> Markdown via markdownify. ``heading_style="ATX"``
+        # selects ``# Heading`` (the dialect we normalise on everywhere).
+        try:
+            md = _to_md(html, heading_style="ATX")
+        except Exception as e:
+            raise ParseError(f"markdownify failed on DOCX HTML: {e}") from e
+
+        # 2b. Core metadata via python-docx (best-effort).
+        title, author, created_at = _read_core_props(data, warnings)
+
+        if not title and source_name:
+            title = Path(source_name).stem or None
+
+        # 3. Structural extraction from the produced Markdown. The DOCX
+        # pipeline yields standard GFM pipe tables (from markdownify) and
+        # ``![alt](data:...)`` image references, both handled by the
+        # shared extractor.
+        tables = extract_tables(md, page=1, warnings=warnings)
+        images = extract_images(md, page=1, warnings=warnings, include_html_img=True)
+
+        try:
+            page = Page(text=md, number=1, tables=tables, images=images)
+            meta = Meta(
+                format="docx",
+                pages=1,
+                size=len(data),
+                title=title,
+                author=author,
+                created_at=created_at,
+                reader=self.name,
+                warnings=warnings,
+            )
+            return Document(pages=[page], meta=meta)
+        except Exception as e:  # pragma: no cover - defensive
+            raise ParseError(f"docx reader failed to build Document: {e}") from e
+
+
+def _read_core_props(data: bytes, warnings: list[str]):
+    """Best-effort read of DOCX core properties via python-docx."""
+    try:
+        from docx import Document as _DocxDoc
+    except ImportError:
+        warnings.append("python-docx not installed; skipping DOCX metadata")
+        return None, None, None
+    try:
+        d = _DocxDoc(io.BytesIO(data))
+        props = d.core_properties
+        title = (props.title or None) or None
+        author = (props.author or None) or None
+        created_at = props.created
+        # Naive datetimes: stamp as UTC so Meta.created_at is always tz-aware.
+        if created_at is not None and created_at.tzinfo is None:
+            created_at = created_at.replace(tzinfo=timezone.utc)
+        return title, author, created_at
+    except Exception as e:
+        warnings.append(f"DOCX metadata read failed: {e}")
+        return None, None, None

fyle/_readers/html/__init__.py

@@ -0,0 +1,6 @@
+"""HTML reader — uses ``markdownify`` for HTML → Markdown conversion.
+
+File name inside the subpackage is the *core driver library* (``markdownify``);
+``beautifulsoup4`` is a pre-processor (strip ``<head>``, read ``<title>``).
+"""
+from . import markdownify  # noqa: F401

fyle/_readers/html/markdownify.py

@@ -0,0 +1,113 @@
+"""HTML reader backed by ``markdownify``.
+
+Strategy:
+- Decode bytes (UTF-8 BOM-aware; latin-1 fallback via the plaintext helper).
+- Strip ``<head>`` before conversion. ``markdownify`` otherwise inlines
+  ``<title>`` / ``<meta>`` text into the body output, which pollutes the
+  Markdown. We use BeautifulSoup to remove the head cleanly (and to pull
+  ``<title>`` out separately for ``meta.title``).
+- Convert the body to Markdown via ``markdownify`` with ``heading_style="ATX"``
+  (the ``# Heading`` form, which is the dialect we normalise on everywhere).
+- Structural extraction (tables, images) is delegated to the shared
+  ``_md_structure`` module, so HTML, DOCX and Markdown readers all present
+  a consistent ``doc.tables`` / ``doc.images`` surface.
+
+File naming rule: ``markdownify.py`` — the core driver for HTML → Markdown.
+BeautifulSoup is a pre-processor (strip head, read title) and therefore
+does not determine the file name.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from ..base import Reader
+from ..._core.document import Document, Meta, Page
+from ...errors import ParseError
+from .._md_structure import extract_images, extract_tables
+from ..text.stdlib import _decode_text
+
+
+class HtmlReader(Reader):
+    name = "markdownify"
+    formats = ("html",)
+    is_default = True
+
+    def read(self, data: bytes, *, source_name: Optional[str] = None, **_) -> Document:
+        try:
+            from markdownify import markdownify as _to_md
+        except ImportError as e:
+            raise ParseError(
+                "markdownify is required for HTML parsing: pip install markdownify"
+            ) from e
+
+        warnings: list[str] = []
+        html, decode_warning = _decode_text(data)
+        if decode_warning:
+            warnings.append(decode_warning)
+
+        # Extract <title> and strip <head> so markdownify does not leak head
+        # contents into the body.
+        title, body_html = _prepare_html(html, warnings)
+
+        try:
+            md = _to_md(body_html, heading_style="ATX")
+        except Exception as e:
+            raise ParseError(f"markdownify failed: {e}") from e
+
+        if not title and source_name:
+            title = Path(source_name).stem or None
+
+        tables = extract_tables(md, page=1, warnings=warnings)
+        images = extract_images(md, page=1, warnings=warnings, include_html_img=True)
+
+        try:
+            page = Page(text=md, number=1, tables=tables, images=images)
+            meta = Meta(
+                format="html",
+                pages=1,
+                size=len(data),
+                title=title,
+                reader=self.name,
+                created_at=datetime.now(timezone.utc),
+                warnings=warnings,
+            )
+            return Document(pages=[page], meta=meta)
+        except Exception as e:  # pragma: no cover - defensive
+            raise ParseError(f"html reader failed to build Document: {e}") from e
+
+
+def _prepare_html(html: str, warnings: list[str]) -> tuple[Optional[str], str]:
+    """Return ``(title, body_html)`` — ``title`` may be ``None``.
+
+    Uses BeautifulSoup if available, otherwise degrades to returning the raw
+    HTML and no title (with a warning). Never raises.
+    """
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        warnings.append("beautifulsoup4 not installed; skipping <head> removal")
+        return None, html
+    try:
+        soup = BeautifulSoup(html, "html.parser")
+        # Pull the title before removing <head>.
+        title: Optional[str] = None
+        if soup.title and soup.title.string:
+            s = soup.title.string.strip()
+            title = s or None
+        # Remove <head> entirely so its contents don't end up in the body
+        # Markdown. Remove stray <script> / <style> too, since markdownify
+        # will otherwise emit their raw contents as text.
+        if soup.head is not None:
+            soup.head.decompose()
+        for tag in soup.find_all(["script", "style", "noscript"]):
+            tag.decompose()
+        # Prefer the body subtree if present; otherwise fall back to the
+        # whole (now head-less) document.
+        body = soup.body
+        body_html = str(body) if body is not None else str(soup)
+        return title, body_html
+    except Exception as e:
+        warnings.append(f"HTML pre-processing failed: {e}")
+        return None, html

fyle/_readers/image/__init__.py

@@ -0,0 +1,18 @@
+"""Image reader.
+
+Multimodal LLMs (GPT-4o, Claude 3.5, Gemini, etc.) consume images via
+base64 ``data:`` URLs or direct URLs. fyle therefore does the minimum
+useful transformation: wrap the raw bytes into a ``data:`` URL and expose
+them through both ``doc.text`` (as a Markdown image token) and
+``doc.images`` so the caller can feed the document into a prompt directly.
+
+fyle does not perform OCR. If you want text extracted from an image, run
+an OCR / VLM pipeline on ``doc.images[0].data`` yourself.
+
+File naming rule: ``stdlib.py`` \u2014 the core driver is Python's standard
+library (``base64`` + ``mimetypes``). Pillow is optional and only used
+for non-fatal metadata (dimensions), not for decoding.
+"""
+from __future__ import annotations
+
+from . import stdlib  # noqa: F401