fylepy 0.1.0__py3-none-any.whl

fyle/__init__.py

@@ -0,0 +1,46 @@
+"""fyle — open anything, get clean Markdown for LLMs.
+
+Public surface: three entry points (``open`` / ``read`` / ``readers``), the
+data model (``Document`` / ``Page`` / ``Table`` / ``Image`` / ``Meta`` /
+``Chunk``), and four exception types.
+"""
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version as _pkg_version
+
+from ._core.api import open, read, readers
+from ._core.document import Chunk, Document, Image, Meta, Page, Table
+from .errors import (
+    DownloadError,
+    ParseError,
+    ReaderNotFoundError,
+    UnsupportedFormatError,
+)
+
+__all__ = [
+    # Entry points
+    "open",
+    "read",
+    "readers",
+    # Data model (exposed for type hints / isinstance checks)
+    "Document",
+    "Page",
+    "Table",
+    "Image",
+    "Meta",
+    "Chunk",
+    # Exceptions
+    "UnsupportedFormatError",
+    "ParseError",
+    "ReaderNotFoundError",
+    "DownloadError",
+]
+
+# ``pyproject.toml`` is the single source of truth for the version string.
+# Read it from the installed package metadata at runtime; fall back to a
+# clearly-fake value when running from an uninstalled source tree (e.g.
+# ``PYTHONPATH=src python -c 'import fyle'``).
+try:
+    __version__ = _pkg_version("fyle")
+except PackageNotFoundError:  # pragma: no cover - only hit without install
+    __version__ = "0.0.0+unknown"

fyle/_core/__init__.py

@@ -0,0 +1,5 @@
+"""Internal core: data models, sniffer, registry, fetcher, chunking.
+
+Not part of the public API. Everything users need is re-exported from the
+top-level ``fyle`` namespace; names under ``fyle._core`` may change at any time.
+"""

fyle/_core/api.py

@@ -0,0 +1,164 @@
+"""Top-level API — ``fyle.open`` / ``fyle.read`` / ``fyle.readers``."""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import IO, Any, Optional, Union
+from urllib.parse import urlparse
+
+from . import fetcher, registry, sniffer
+from .document import Document
+
+Src = Union[str, Path, bytes, bytearray, IO[Any]]
+
+_readers_loaded: bool = False
+
+
+def _ensure_readers() -> None:
+    """Trigger reader registration and startup validation on first use."""
+    global _readers_loaded
+    if _readers_loaded:
+        return
+    # Imported lazily to avoid a circular import during ``fyle`` package init.
+    from .. import _readers  # noqa: F401
+
+    registry.validate()
+    _readers_loaded = True
+
+
+def _normalize(src: Src) -> tuple[bytes, Optional[str], Optional[str], Optional[str]]:
+    """Normalise ``src`` to ``(bytes, source_name, content_type, source_path)``.
+
+    Dispatcher responsibility: every reader receives plain ``bytes``, so
+    individual readers never have to handle polymorphic inputs.
+
+    ``source_path`` is the absolute filesystem path of a local file source,
+    or ``None`` for URL / bytes / file-like inputs. Most readers ignore it;
+    the archive reader needs it to decide where to extract.
+    """
+    # URL.
+    if isinstance(src, str) and (src.startswith("http://") or src.startswith("https://")):
+        data, ct = fetcher.fetch(src)
+        parsed_path = urlparse(src).path
+        name = parsed_path.rsplit("/", 1)[-1] if parsed_path else None
+        return data, (name or None), ct, None
+
+    # Local filesystem path.
+    if isinstance(src, (str, Path)):
+        p = Path(src)
+        try:
+            resolved = str(p.resolve())
+        except OSError:
+            resolved = str(p)
+        return p.read_bytes(), p.name, None, resolved
+
+    # bytes / bytearray.
+    if isinstance(src, (bytes, bytearray)):
+        return bytes(src), None, None, None
+
+    # File-like object.
+    if hasattr(src, "read"):
+        try:
+            seekable = bool(src.seekable()) if hasattr(src, "seekable") else False
+        except Exception:
+            seekable = False
+        if seekable:
+            try:
+                src.seek(0)
+            except Exception:
+                pass
+        data = src.read()
+        if isinstance(data, str):
+            data = data.encode("utf-8")
+        raw_name = getattr(src, "name", None)
+        if isinstance(raw_name, bytes):
+            try:
+                raw_name = raw_name.decode("utf-8", errors="ignore")
+            except Exception:
+                raw_name = None
+        if isinstance(raw_name, str):
+            name = Path(raw_name).name
+            # A file-like object whose ``name`` is a real filesystem path
+            # lets us surface an absolute ``source_path`` too.
+            try:
+                candidate = Path(raw_name)
+                source_path = str(candidate.resolve()) if candidate.exists() else None
+            except OSError:
+                source_path = None
+        else:
+            name = None
+            source_path = None
+        return bytes(data), name, None, source_path
+
+    raise TypeError(f"Unsupported src type: {type(src).__name__}")
+
+
+def open(src: Src, *, reader: Optional[str] = None) -> Document:
+    """Open a document and return a ``Document``.
+
+    ``src`` accepts: a local path (``str`` / ``Path``), ``bytes``, a file-like
+    object, or an ``http(s)://`` URL. Pass ``reader=<name>`` to force a
+    specific reader (see ``fyle.readers()`` for the list of available names).
+    """
+    _ensure_readers()
+    data, source_name, content_type, source_path = _normalize(src)
+    fmt = sniffer.detect(data, source_name=source_name, content_type=content_type)
+    reader_cls = registry.resolve(fmt, reader)
+    doc = reader_cls().read(data, source_name=source_name, source_path=source_path)
+    # Fill in the final meta fields that only the dispatcher can know.
+    doc.meta.reader = reader_cls.name
+    if not doc.meta.format:
+        doc.meta.format = fmt
+    if not doc.meta.size:
+        doc.meta.size = len(data)
+    # Fine-grained subtype. ``format`` is the reader family (e.g. ``image``,
+    # ``text``); ``ext`` pins down the concrete subtype (``png`` vs ``jpeg``,
+    # ``py`` vs ``json``). Filled centrally so every reader gets it for free.
+    if doc.meta.ext is None and source_name:
+        suffix = Path(source_name).suffix.lower().lstrip(".")
+        doc.meta.ext = suffix or None
+    # Normalise ``title`` to a filename stem. Two independent sources can
+    # leave ``title`` already ending in ``.ext``:
+    #   1. A reader falls back to ``source_name`` (full filename) when the
+    #      document has no embedded title field.
+    #   2. Some producers embed a title string that itself includes the
+    #      filename extension (common for PDFs generated from ``save as``).
+    # Either way, pairing such a ``title`` with the separately-stored
+    # ``ext`` would double the suffix in the file-level header
+    # (``report.pdf.pdf``). Strip the redundant suffix centrally, case-
+    # insensitively, while preserving the title's original casing.
+    if doc.meta.title and doc.meta.ext:
+        suffix_with_dot = "." + doc.meta.ext.lower()
+        if doc.meta.title.lower().endswith(suffix_with_dot):
+            doc.meta.title = doc.meta.title[: -len(suffix_with_dot)] or doc.meta.title
+    # Surface the original URL for remote sources so the LLM-ready header
+    # can tell the model *where* the file came from — the domain alone
+    # (arxiv.org / github.com / a vendor's docs site) is a strong
+    # semantic signal. Local filesystem paths are intentionally not
+    # surfaced here (privacy: avoids leaking ``/Users/<user>/...`` into
+    # any payload the user later shares, logs, or forwards to a hosted
+    # model API). bytes / file-like inputs have no URL to surface.
+    if isinstance(src, str) and (
+        src.startswith("http://") or src.startswith("https://")
+    ):
+        doc.meta.source = src
+    return doc
+
+
+def read(src: Src, *, reader: Optional[str] = None) -> str:
+    """Sugar: equivalent to ``str(open(src, reader=reader))``.
+
+    Returns the LLM-ready payload (file-level header + Markdown content),
+    which is what most callers of a one-liner convenience actually want.
+    For the raw content without the header, use ``open(src).text``.
+    """
+    return str(open(src, reader=reader))
+
+
+def readers() -> dict[str, list[str]]:
+    """Return the readers available in the current environment.
+
+    The default reader for each format is suffixed with ``*``. Example:
+    ``{"pdf": ["pymupdf4llm*", "pdfplumber", "pypdf"], ...}``.
+    """
+    _ensure_readers()
+    return registry.list_all()

fyle/_core/chunking.py

@@ -0,0 +1,107 @@
+"""Token estimation and paragraph-boundary chunking.
+
+- Token estimation: prefer ``tiktoken`` with the ``cl100k_base`` encoding;
+  fall back to ~4 chars/token when tiktoken is unavailable.
+- Chunking: aggregate paragraphs (split on ``\n\n``) under a ``max_tokens``
+  soft limit; fill ``overlap`` by back-filling whole trailing paragraphs.
+"""
+from __future__ import annotations
+
+from typing import Iterator, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .document import Chunk, Document
+
+_ENCODING: object = None  # tiktoken Encoding instance or the string "fallback".
+
+
+def _get_encoding():
+    global _ENCODING
+    if _ENCODING is None:
+        try:
+            import tiktoken
+
+            _ENCODING = tiktoken.get_encoding("cl100k_base")
+        except Exception:
+            _ENCODING = "fallback"
+    return _ENCODING
+
+
+def estimate_tokens(text: str) -> int:
+    """Estimate the token count of ``text``."""
+    if not text:
+        return 0
+    enc = _get_encoding()
+    if enc == "fallback":
+        return max(1, len(text) // 4)
+    return len(enc.encode(text))
+
+
+def chunk_document(
+    doc: "Document", *, max_tokens: int = 4000, overlap: int = 200
+) -> Iterator["Chunk"]:
+    """Split a ``Document`` on paragraph boundaries.
+
+    - No hard cuts: if adding the next paragraph would overflow ``max_tokens``,
+      yield the current chunk first.
+    - ``overlap``: back-fill trailing paragraphs of the just-yielded chunk
+      until the accumulated overlap reaches roughly ``overlap`` tokens.
+    - ``page_range``: derived from the source page numbers of the paragraphs
+      in the chunk; ``None`` for formats without native pagination.
+    """
+    from .document import Chunk
+
+    if max_tokens <= 0:
+        raise ValueError("max_tokens must be positive")
+    if overlap < 0:
+        raise ValueError("overlap must be non-negative")
+    if overlap >= max_tokens:
+        raise ValueError("overlap must be smaller than max_tokens")
+
+    # 1. Split every page.text into paragraphs, tagged with their source page number.
+    paragraphs: list[tuple[str, Optional[int]]] = []
+    for page in doc.pages:
+        page_num = page.number
+        for para in page.text.split("\n\n"):
+            para = para.strip()
+            if para:
+                paragraphs.append((para, page_num))
+
+    if not paragraphs:
+        return
+
+    buf: list[str] = []
+    buf_pages: list[Optional[int]] = []
+    buf_tokens: int = 0
+
+    def make_chunk() -> Chunk:
+        text = "\n\n".join(buf)
+        real_pages = [p for p in buf_pages if p is not None]
+        page_range: Optional[tuple[int, int]] = (
+            (min(real_pages), max(real_pages)) if real_pages else None
+        )
+        return Chunk(text=text, tokens=estimate_tokens(text), page_range=page_range)
+
+    for para, page_num in paragraphs:
+        p_tokens = estimate_tokens(para)
+        if buf and buf_tokens + p_tokens > max_tokens:
+            yield make_chunk()
+            # Back-fill overlap from the tail of the previous buffer.
+            carry: list[str] = []
+            carry_pages: list[Optional[int]] = []
+            carry_tokens = 0
+            if overlap > 0:
+                for prev_para, prev_page in zip(reversed(buf), reversed(buf_pages)):
+                    t = estimate_tokens(prev_para)
+                    if carry_tokens + t > overlap:
+                        break
+                    carry.insert(0, prev_para)
+                    carry_pages.insert(0, prev_page)
+                    carry_tokens += t
+            buf, buf_pages, buf_tokens = carry, carry_pages, carry_tokens
+        buf.append(para)
+        buf_pages.append(page_num)
+        buf_tokens += p_tokens
+
+    if buf:
+        yield make_chunk()

fyle/_core/document.py

@@ -0,0 +1,345 @@
+"""Data model — Document / Page / Table / Image / Meta / Chunk.
+
+Naming rule: ``.text`` is always a Markdown string, on every level
+(``doc.text``, ``page.text``, ``table.text``, ``image.text``).
+
+The element types (``Meta`` / ``Image`` / ``Table`` / ``Page`` / ``Chunk``)
+are ``pydantic.BaseModel`` subclasses, so they get runtime validation, a
+proper ``.model_dump()`` / ``.model_dump_json()`` surface, and a consistent
+construction contract. ``Document`` itself is intentionally a plain class
+with ``__slots__`` because it caches derived views and exposes properties.
+"""
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+from typing import Iterator, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class _Element(BaseModel):
+    """Shared base for every element type in the data model."""
+
+    # Element instances are treated as value objects. ``extra="forbid"``
+    # prevents silent typos at construction time; ``ser_json_bytes="base64"``
+    # makes ``.model_dump_json()`` work on image payloads.
+    model_config = ConfigDict(
+        extra="forbid",
+        arbitrary_types_allowed=False,
+        ser_json_bytes="base64",
+    )
+
+
+class Meta(_Element):
+    """Document-level metadata."""
+
+    format: str = ""
+    # Fine-grained subtype. ``format`` is the *reader family* (e.g. ``image``,
+    # ``text``, ``docx``) and intentionally coarse. ``ext`` records the concrete
+    # subtype so callers can distinguish ``.png`` vs ``.jpeg``, ``.py`` vs
+    # ``.json``, etc., without having to re-parse ``source_name`` or the
+    # ``data_url`` MIME of each image.
+    # Filled by the dispatcher from the source name's suffix (lower-cased, no
+    # leading dot). ``None`` when the input has no name (e.g. raw bytes with no
+    # ``source_name`` and a URL whose path has no suffix).
+    ext: Optional[str] = None
+    # Formats without native pagination always report ``pages=1``.
+    pages: int = 1
+    size: int = 0
+    title: Optional[str] = None
+    author: Optional[str] = None
+    created_at: Optional[datetime] = None
+    # Original source URL for inputs fetched from ``http(s)://``. Stays
+    # ``None`` for local paths, bytes, and file-like inputs: we
+    # deliberately never surface local filesystem paths here to avoid
+    # leaking ``/Users/<user>/...`` (or equivalent) into the LLM-ready
+    # payload when it gets shared, logged, or forwarded to a hosted
+    # model API. The filename in ``title`` + ``ext`` is enough.
+    source: Optional[str] = None
+    reader: str = ""
+    warnings: list[str] = Field(default_factory=list)
+
+    def as_dict(self) -> dict:
+        """Return a JSON-friendly dict (``created_at`` as ISO-8601 string)."""
+        return {
+            "format": self.format,
+            "ext": self.ext,
+            "pages": self.pages,
+            "size": self.size,
+            "title": self.title,
+            "author": self.author,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "source": self.source,
+            "reader": self.reader,
+            "warnings": list(self.warnings),
+        }
+
+
+class Image(_Element):
+    """Image element. fyle does not perform OCR."""
+
+    data_url: str
+    # Raw image bytes. Named ``data`` rather than ``bytes`` to avoid shadowing
+    # the builtin ``bytes`` type in annotations.
+    data: bytes = b""
+    caption: Optional[str] = None
+    page: Optional[int] = None
+
+    @property
+    def text(self) -> str:
+        """Return Markdown image syntax: ``![caption](data:image/...;base64,...)``.
+
+        Keeps the ``.text`` contract consistent with ``doc.text`` /
+        ``page.text`` / ``table.text``: ``.text`` is always Markdown.
+        """
+        alt = self.caption or ""
+        return f"![{alt}]({self.data_url})"
+
+    def save(self, path: Union[str, Path]) -> None:
+        Path(path).write_bytes(self.data)
+
+
+class Table(_Element):
+    """Table element."""
+
+    # Markdown table string; name aligned with ``doc.text`` / ``page.text``.
+    text: str
+    rows: list[list[str]] = Field(default_factory=list)
+    headers: list[str] = Field(default_factory=list)
+    page: Optional[int] = None
+
+
+class Page(_Element):
+    """Page element.
+
+    For formats without native pagination, ``pages`` contains a single ``Page``
+    with ``number=1``.
+
+    ``name`` is an optional human-meaningful label for this page. It is used
+    by formats where the page has a natural identity beyond a page number:
+
+    - XLSX: sheet name (``ws.title``).
+    - PPTX (future): slide title.
+
+    For PDF / DOCX / HTML / plain text / Markdown / CSV it stays ``None``.
+    We deliberately keep this on ``Page`` rather than introducing separate
+    ``Sheet`` / ``Slide`` models: the data shape is identical and the content
+    surface (``doc.text`` / ``doc.pages`` / ``doc.tables`` / ``doc.images`` /
+    ``doc.meta``) must stay at exactly five attributes.
+    """
+
+    # Markdown content of this page.
+    text: str
+    number: int = 1
+    name: Optional[str] = None
+    tables: list[Table] = Field(default_factory=list)
+    images: list[Image] = Field(default_factory=list)
+
+
+class Chunk(_Element):
+    """LLM-oriented chunk produced by ``Document.chunks()``."""
+
+    text: str
+    tokens: int
+    # ``None`` for formats without native pagination.
+    page_range: Optional[tuple[int, int]] = None
+
+
+class Document:
+    """Top-level document object returned by ``fyle.open``.
+
+    Five content attributes (``text`` / ``pages`` / ``tables`` / ``images`` /
+    ``meta``) plus three LLM helpers (``tokens`` / ``tokens_for`` / ``chunks``).
+    Eight in total; the surface is frozen.
+    """
+
+    __slots__ = (
+        "_pages",
+        "meta",
+        "_text_cache",
+        "_tables_cache",
+        "_images_cache",
+    )
+
+    def __init__(self, *, pages: list[Page], meta: Meta) -> None:
+        self._pages = pages
+        self.meta = meta
+        self._text_cache: Optional[str] = None
+        self._tables_cache: Optional[list[Table]] = None
+        self._images_cache: Optional[list[Image]] = None
+
+    # ------------------------------------------------------------------
+    # Content attributes (5)
+    # ------------------------------------------------------------------
+    @property
+    def text(self) -> str:
+        if self._text_cache is None:
+            self._text_cache = "\n\n".join(p.text for p in self._pages if p.text)
+        return self._text_cache
+
+    @property
+    def pages(self) -> list[Page]:
+        return self._pages
+
+    @property
+    def tables(self) -> list[Table]:
+        if self._tables_cache is None:
+            self._tables_cache = [t for p in self._pages for t in p.tables]
+        return self._tables_cache
+
+    @property
+    def images(self) -> list[Image]:
+        if self._images_cache is None:
+            self._images_cache = [img for p in self._pages for img in p.images]
+        return self._images_cache
+
+    # ------------------------------------------------------------------
+    # LLM helpers (3)
+    # ------------------------------------------------------------------
+    @property
+    def tokens(self) -> int:
+        from .chunking import estimate_tokens
+
+        return estimate_tokens(self.text)
+
+    def tokens_for(self, obj) -> int:
+        from .chunking import estimate_tokens
+
+        text = getattr(obj, "text", None)
+        if text is None:
+            raise TypeError(f"tokens_for expected an object with .text, got {type(obj).__name__}")
+        return estimate_tokens(text)
+
+    def chunks(self, max_tokens: int = 4000, overlap: int = 200) -> Iterator[Chunk]:
+        from .chunking import chunk_document
+
+        yield from chunk_document(self, max_tokens=max_tokens, overlap=overlap)
+
+    # ------------------------------------------------------------------
+    # Optional context manager
+    # ------------------------------------------------------------------
+    def __enter__(self) -> "Document":
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> bool:
+        return False
+
+    def __repr__(self) -> str:
+        return (
+            f"Document(format={self.meta.format!r}, pages={len(self._pages)}, "
+            f"reader={self.meta.reader!r})"
+        )
+
+    def __str__(self) -> str:
+        """Return an LLM-ready payload: file-level header + ``doc.text``.
+
+        Intended as the one-liner you hand to a model:
+        ``llm.complete(str(doc))``. The header surfaces filename, format,
+        and size so the model still knows what it is looking at when
+        only the string is passed in (no ``doc.meta`` alongside). The
+        filename in particular carries real semantic signal that
+        ``doc.text`` alone would discard.
+
+        For the raw content without the wrapper, use ``doc.text``.
+        """
+        header = self._file_level_header()
+        if not header:
+            return self.text
+        return f"{header}\n\n---\n\n{self.text}"
+
+    def _file_level_header(self) -> str:
+        """Compose the outer Markdown header surfaced by ``__str__``.
+
+        Surfaces the metadata fields that carry real semantic signal for
+        an LLM reading the payload: filename, format, size, page count
+        (when >1), author, creation time, and any parse warnings. The
+        ``reader`` field is deliberately omitted — it's an internal
+        implementation detail (which library parsed the file) with no
+        value to the model; developers who need it can read
+        ``doc.meta.reader`` directly.
+
+        Content-specific metadata (audio duration, video keyframes,
+        detected language, ...) belongs to the reader's own inline
+        header inside ``doc.text`` and stays there.
+
+        The core fields are rendered as a two-column ``field | value``
+        Markdown table — one row per attribute. This shape stays
+        compact regardless of how many fields are present and matches
+        how LLMs naturally parse labeled key/value pairs. Warnings are
+        rendered as a separate bullet list because they are a
+        variable-length ``list[str]`` and don't fit the single-value
+        row shape.
+        """
+        # Collect present (name, value) pairs. Every field is optional
+        # so missing ones simply do not produce a row.
+        fields: list[tuple[str, str]] = []
+        if self.meta.title:
+            # ``meta.title`` is the filename stem; ``meta.ext`` is the
+            # dot-less suffix — together they reconstruct the source
+            # filename. The filename is just another attribute here,
+            # not a separate heading.
+            filename = (
+                f"{self.meta.title}.{self.meta.ext}"
+                if self.meta.ext
+                else self.meta.title
+            )
+            fields.append(("filename", filename))
+        if self.meta.source:
+            # Only populated for ``http(s)://`` inputs — see ``Meta.source``
+            # for why local paths are excluded. The URL carries real
+            # semantic signal for the LLM (arxiv.org / github.com /
+            # a vendor's docs site all imply different content types).
+            fields.append(("source", self.meta.source))
+        if self.meta.format:
+            fields.append(("format", self.meta.format))
+        if self.meta.size:
+            fields.append(("size", _human_size(self.meta.size)))
+        if len(self._pages) > 1:
+            fields.append(("pages", str(len(self._pages))))
+        if self.meta.author:
+            fields.append(("author", self.meta.author))
+        if self.meta.created_at:
+            fields.append(
+                ("created", self.meta.created_at.isoformat(timespec="seconds"))
+            )
+
+        lines: list[str] = []
+        if fields:
+            lines.append("| field | value |")
+            lines.append("| --- | --- |")
+            for name, value in fields:
+                lines.append(
+                    f"| {_escape_table_cell(name)} | {_escape_table_cell(value)} |"
+                )
+        # Warnings are variable-length; give them their own bullet list
+        # so they never distort the table's two-column shape.
+        if self.meta.warnings:
+            if lines:
+                lines.append("")
+            lines.append("**Warnings:**")
+            for w in self.meta.warnings:
+                lines.append(f"- {w}")
+        return "\n".join(lines)
+
+
+def _human_size(n: int) -> str:
+    """Render a byte count as ``1.2 MB`` / ``938.8 KB`` / ``420 B``."""
+    if n < 1024:
+        return f"{n} B"
+    size = float(n)
+    for unit in ("KB", "MB", "GB", "TB"):
+        size /= 1024
+        if size < 1024 or unit == "TB":
+            return f"{size:.1f} {unit}"
+    return f"{size:.1f} TB"
+
+
+def _escape_table_cell(v: str) -> str:
+    """Escape a value for safe inclusion in a single Markdown table cell.
+
+    Replaces ``|`` with ``\\|`` (table column separator) and collapses
+    embedded newlines to spaces so a rogue multi-line ``author`` or
+    ``title`` value never breaks the table's single-row shape.
+    """
+    return v.replace("|", "\\|").replace("\n", " ").replace("\r", " ")