fylepy 0.1.0__py3-none-any.whl

fyle/_readers/pptx/python_pptx.py

@@ -0,0 +1,306 @@
+"""PowerPoint (.pptx) reader backed by ``python-pptx``.
+
+Each slide becomes one ``Page``:
+- ``page.number`` is the slide's 1-based index.
+- ``page.name`` holds the slide title (from the title placeholder) when
+  present. Same field as the sheet name for XLSX \u2014 kept consistent
+  rather than inventing a ``Slide`` type.
+- ``page.text`` is Markdown assembled from the slide's text frames,
+  bullet lists, tables and images, in slide order.
+- ``page.tables`` and ``page.images`` are populated in parallel so
+  downstream code can access structural data without re-parsing.
+
+Tables go through ``tabulate`` (GFM pipe format); images are embedded
+as ``data:image/...;base64,...`` URLs so the document is self-contained
+and can be fed directly to a multimodal LLM.
+
+File naming rule: ``python_pptx.py`` \u2014 the core driver is ``python-pptx``.
+(Module name uses ``_`` rather than ``-`` because Python identifiers
+cannot contain hyphens.)
+"""
+from __future__ import annotations
+
+import base64
+import io
+from datetime import timezone
+from pathlib import Path
+from typing import Any, Optional
+
+from ..base import Reader
+from ..._core.document import Document, Image, Meta, Page, Table
+from ...errors import ParseError
+
+
+class PptxReader(Reader):
+    name = "python-pptx"
+    formats = ("pptx",)
+    is_default = True
+
+    def read(self, data: bytes, *, source_name: Optional[str] = None, **_) -> Document:
+        try:
+            from pptx import Presentation
+        except ImportError as e:
+            raise ParseError(
+                "python-pptx is required for PPTX parsing: pip install python-pptx"
+            ) from e
+        try:
+            from pptx.enum.shapes import MSO_SHAPE_TYPE
+        except ImportError as e:  # pragma: no cover
+            raise ParseError(f"python-pptx import failed: {e}") from e
+
+        warnings: list[str] = []
+
+        try:
+            prs = Presentation(io.BytesIO(data))
+        except Exception as e:
+            raise ParseError(f"python-pptx failed to open PPTX: {e}") from e
+
+        pages: list[Page] = []
+        for idx, slide in enumerate(prs.slides, start=1):
+            page = _render_slide(slide, idx, MSO_SHAPE_TYPE, warnings)
+            pages.append(page)
+
+        if not pages:
+            pages.append(Page(text="", number=1))
+            warnings.append("pptx: no slides found")
+
+        title, author, created_at = _read_core_props(prs, warnings)
+
+        if not title and source_name:
+            try:
+                title = Path(source_name).stem or None
+            except (TypeError, ValueError):
+                title = None
+
+        try:
+            meta = Meta(
+                format="pptx",
+                pages=len(pages),
+                size=len(data),
+                title=title,
+                author=author,
+                created_at=created_at,
+                reader=self.name,
+                warnings=warnings,
+            )
+            return Document(pages=pages, meta=meta)
+        except Exception as e:  # pragma: no cover - defensive
+            raise ParseError(f"pptx reader failed to build Document: {e}") from e
+
+
+def _read_core_props(prs, warnings: list[str]):
+    """Best-effort read of PPTX core properties. Never raises."""
+    try:
+        cp = prs.core_properties
+        title = (cp.title or None) or None
+        author = (cp.author or None) or None
+        created_at = cp.created
+        if created_at is not None and created_at.tzinfo is None:
+            created_at = created_at.replace(tzinfo=timezone.utc)
+        return title, author, created_at
+    except Exception as e:
+        warnings.append(f"PPTX metadata read failed: {e}")
+        return None, None, None
+
+
+def _render_slide(slide: Any, slide_no: int, MSO_SHAPE_TYPE: Any, warnings: list[str]) -> Page:
+    """Walk the slide's shapes and assemble Markdown + structural elements."""
+    md_parts: list[str] = []
+    tables: list[Table] = []
+    images: list[Image] = []
+
+    slide_title = _slide_title(slide)
+    if slide_title:
+        md_parts.append(f"# {slide_title}")
+
+    for shape in slide.shapes:
+        # Skip the title placeholder we already rendered as H1.
+        if _is_title_placeholder(shape) and slide_title:
+            continue
+
+        try:
+            if shape.has_text_frame:
+                block = _render_text_frame(shape.text_frame)
+                if block:
+                    md_parts.append(block)
+                continue
+            if shape.has_table:
+                md, tbl = _render_table(shape.table, slide_no, warnings)
+                if tbl is not None:
+                    tables.append(tbl)
+                if md:
+                    md_parts.append(md)
+                continue
+            if getattr(shape, "shape_type", None) == MSO_SHAPE_TYPE.PICTURE:
+                md, img = _render_picture(shape, slide_no, warnings)
+                if img is not None:
+                    images.append(img)
+                if md:
+                    md_parts.append(md)
+                continue
+        except Exception as e:
+            warnings.append(f"pptx slide {slide_no}: shape render failed: {e}")
+            continue
+
+    # Notes are optional; include them at the bottom under a subheading.
+    notes = _slide_notes(slide)
+    if notes:
+        md_parts.append("## Speaker notes")
+        md_parts.append(notes)
+
+    page_text = "\n\n".join(p for p in md_parts if p)
+
+    return Page(
+        text=page_text,
+        number=slide_no,
+        name=slide_title or None,
+        tables=tables,
+        images=images,
+    )
+
+
+def _sanitize(text: str, line_sep: str = "\n") -> str:
+    """Normalise PowerPoint control characters in extracted strings.
+
+    PowerPoint stores soft line breaks (``Shift+Enter``) as the vertical tab
+    byte ``\\x0b`` (VT) inside a single paragraph. ``python-pptx`` returns
+    those verbatim via ``paragraph.text`` / ``text_frame.text``, which leaks
+    the raw control byte into our Markdown and into ``Page.name`` / titles.
+
+    We collapse those to ``line_sep``:
+    - ``"\n"`` for body / paragraph text — preserves the "line break within
+      paragraph" semantics.
+    - ``" "`` for titles — a slide title is rendered on one line in Markdown
+      anyway, and embedding a newline would break the heading.
+    """
+    if not text:
+        return text
+    return text.replace("\x0b", line_sep).replace("\x0c", line_sep)
+
+
+def _slide_title(slide: Any) -> Optional[str]:
+    """Extract the slide title placeholder text, or ``None``."""
+    try:
+        title_shape = slide.shapes.title
+    except Exception:
+        return None
+    if title_shape is None:
+        return None
+    try:
+        text = (title_shape.text_frame.text or "").strip()
+    except Exception:
+        return None
+    text = _sanitize(text, line_sep=" ").strip()
+    return text or None
+
+
+def _is_title_placeholder(shape: Any) -> bool:
+    try:
+        if not shape.has_text_frame:
+            return False
+        ph = getattr(shape, "placeholder_format", None)
+        if ph is None:
+            return False
+        # ``idx == 0`` is the title placeholder across layouts.
+        return getattr(ph, "idx", None) == 0
+    except Exception:
+        return False
+
+
+def _render_text_frame(tf: Any) -> str:
+    """Render a text frame as Markdown paragraphs / bullets.
+
+    Heuristic: paragraphs with ``level > 0`` or short indented content
+    become bullet items; others become paragraphs.
+    """
+    lines: list[str] = []
+    for para in tf.paragraphs:
+        text = _sanitize((para.text or ""), line_sep="\n").strip()
+        if not text:
+            continue
+        level = getattr(para, "level", 0) or 0
+        if level > 0:
+            indent = "  " * (level - 0)
+            lines.append(f"{indent}- {text}")
+        else:
+            # Treat a leading dash / bullet glyph as a bullet too.
+            if text.startswith(("\u2022", "-", "*")):
+                lines.append(f"- {text.lstrip('\u2022-* ').strip()}")
+            else:
+                lines.append(text)
+    return "\n".join(lines)
+
+
+def _render_table(tbl: Any, slide_no: int, warnings: list[str]) -> tuple[str, Optional[Table]]:
+    """Render a python-pptx table as a GFM pipe table (via tabulate)."""
+    try:
+        from tabulate import tabulate as _tabulate
+    except ImportError:
+        warnings.append("tabulate not installed; skipping pptx table render")
+        return "", None
+
+    rows_raw: list[list[str]] = []
+    for row in tbl.rows:
+        row_cells: list[str] = []
+        for cell in row.cells:
+            try:
+                cell_text = _sanitize((cell.text or ""), line_sep="\n").strip()
+            except Exception:
+                cell_text = ""
+            row_cells.append(cell_text)
+        rows_raw.append(row_cells)
+
+    if not rows_raw:
+        return "", None
+
+    headers = rows_raw[0]
+    body = rows_raw[1:]
+    try:
+        md = _tabulate(body, headers=headers, tablefmt="github")
+    except Exception as e:
+        warnings.append(f"pptx slide {slide_no}: table render failed: {e}")
+        return "", None
+
+    return md, Table(text=md, rows=body, headers=headers, page=slide_no)
+
+
+def _render_picture(shape: Any, slide_no: int, warnings: list[str]) -> tuple[str, Optional[Image]]:
+    """Extract a picture shape's bytes into a ``data:`` URL and Markdown token."""
+    try:
+        image = shape.image
+    except Exception as e:
+        warnings.append(f"pptx slide {slide_no}: picture access failed: {e}")
+        return "", None
+
+    try:
+        blob: bytes = image.blob or b""
+        content_type: str = image.content_type or "application/octet-stream"
+    except Exception as e:
+        warnings.append(f"pptx slide {slide_no}: picture read failed: {e}")
+        return "", None
+
+    if not blob:
+        return "", None
+
+    b64 = base64.b64encode(blob).decode("ascii")
+    data_url = f"data:{content_type};base64,{b64}"
+
+    caption: Optional[str] = None
+    name = getattr(shape, "name", "") or ""
+    if name:
+        caption = name
+
+    md_token = f"![{caption or ''}]({data_url})"
+    img = Image(data_url=data_url, data=blob, caption=caption, page=slide_no)
+    return md_token, img
+
+
+def _slide_notes(slide: Any) -> str:
+    """Return the slide's speaker notes as plain text, or ``""``."""
+    try:
+        if not slide.has_notes_slide:
+            return ""
+        notes_tf = slide.notes_slide.notes_text_frame
+        return _sanitize((notes_tf.text or ""), line_sep="\n").strip()
+    except Exception:
+        return ""

fyle/_readers/sqlite/__init__.py

@@ -0,0 +1,8 @@
+"""SQLite reader.
+
+File naming rule: ``stdlib.py`` — the core driver is Python's built-in
+``sqlite3`` module.
+"""
+from __future__ import annotations
+
+from . import stdlib  # noqa: F401

fyle/_readers/sqlite/stdlib.py

@@ -0,0 +1,366 @@
+"""SQLite reader backed by Python's standard library (``sqlite3``).
+
+Strategy:
+- SQLite ``.db`` / ``.sqlite`` / ``.sqlite3`` files are a native fyle input:
+  a single-file database is still a *file*, so it fits ``fyle.open``.
+- We produce a **read-only schema overview** with a tiny sample of rows per
+  table/view, so an LLM can understand what the database holds without
+  pulling the entire dataset. Actual ad-hoc querying is offered through
+  ``fyle.sqlite`` (``tables`` / ``schema`` / ``query``), which is a thin
+  wrapper around ``sqlite3`` intended for tool-call use.
+
+Per-table Markdown layout:
+
+    # {table_name}
+
+    **Schema**
+
+    | column | type | nullable | default | pk |
+    |---|---|---|---|---|
+    ...
+
+    **Sample rows** (10 of N)
+
+    | col1 | col2 | ... |
+    |---|---|---|
+    ...
+
+- ``page.number`` is the table's position in alphabetical order.
+- ``page.name`` is the table or view name (reuses the ``Page.name`` slot
+  we already use for XLSX sheets and PPTX slide titles).
+- ``page.tables`` always carries two ``Table`` objects: ``[schema, sample]``
+  so callers can consume the data without re-parsing Markdown.
+- ``page.text`` embeds both tables plus the row-count summary.
+
+Sample cap is **10 rows** by design — the reader exists to produce a
+compact prompt context, not to dump data. Full scans belong to the
+``fyle.sqlite`` query helper (or an external tool) where the LLM
+itself decides the row count via ``LIMIT``.
+
+File naming rule: ``stdlib.py`` — the core driver is the Python standard
+library's ``sqlite3`` module.
+"""
+from __future__ import annotations
+
+import os
+import sqlite3
+import tempfile
+import weakref
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from ..base import Reader
+from ..._core.document import Document, Meta, Page, Table
+from ...errors import ParseError
+
+
+# Number of rows previewed per table/view inside the Document. Kept small
+# on purpose: this reader's job is to give an LLM enough shape to reason
+# about schema. Callers who want more data should use ``fyle.sqlite.query``
+# or the ``doc.table(name).query(sql)`` fluent API.
+_SAMPLE_ROWS = 10
+
+
+class SqliteTable:
+    """Interactive handle for one table/view inside a :class:`SqliteDocument`.
+
+    Returned by ``doc.table(name)``. Deliberately *not* a
+    :class:`fyle._core.document.Table`: that one is a Pydantic value object
+    used across every reader for schema + data rows. This class instead
+    bundles a table name with its parent database path so an LLM can chain
+    ``.query(sql)`` / ``.schema()`` off a single reference.
+
+    The SQL passed to :meth:`query` is free-form — it can JOIN other tables
+    in the same database. ``table(name)`` is a starting point for the LLM's
+    train of thought, not a scope restriction.
+    """
+
+    __slots__ = ("name", "_db_path")
+
+    def __init__(self, name: str, db_path: str) -> None:
+        self.name = name
+        self._db_path = db_path
+
+    def query(self, sql: str, params: Optional[list] = None) -> str:
+        """Run read-only SQL against the parent database. Returns Markdown."""
+        from ... import sqlite as _fyle_sqlite
+
+        return _fyle_sqlite.query(self._db_path, sql, params)
+
+    def schema(self) -> str:
+        """Return this table's column schema as a Markdown pipe table."""
+        from ... import sqlite as _fyle_sqlite
+
+        return _fyle_sqlite.schema(self._db_path, self.name)
+
+    def __repr__(self) -> str:  # pragma: no cover - debug aid
+        return f"SqliteTable(name={self.name!r})"
+
+
+class SqliteDocument(Document):
+    """Document subclass returned by :class:`SqliteReader`.
+
+    Adds :meth:`table` for chained SQL access and manages the lifetime of
+    the backing temporary database file. ``sqlite3`` only opens databases
+    from a filesystem path, so fyle spills the input bytes to a temp file
+    and keeps it alive for the Document's lifetime; cleanup is triggered
+    by garbage collection, an explicit :meth:`close`, or a ``with`` block.
+    """
+
+    # ``__weakref__`` is required so ``weakref.finalize`` can attach to
+    # an instance whose parent class already uses ``__slots__``.
+    __slots__ = ("_db_path", "_table_names", "_finalizer", "__weakref__")
+
+    def __init__(
+        self,
+        *,
+        pages: list,
+        meta: Meta,
+        db_path: str,
+        table_names: list[str],
+    ) -> None:
+        super().__init__(pages=pages, meta=meta)
+        self._db_path = db_path
+        self._table_names = frozenset(table_names)
+        self._finalizer = weakref.finalize(self, _unlink_quiet, db_path)
+
+    def table(self, name: str) -> SqliteTable:
+        """Return a handle to ``name`` so the caller can run SQL against it.
+
+        Raises ``KeyError`` if ``name`` is not a known table or view in
+        this database. The available names are exactly the ones present
+        as ``page.name`` on ``doc.pages``.
+        """
+        if name not in self._table_names:
+            raise KeyError(
+                f"Unknown table or view: {name!r}. "
+                f"Available: {sorted(self._table_names)}"
+            )
+        return SqliteTable(name, self._db_path)
+
+    def close(self) -> None:
+        """Eagerly delete the backing temp db file.
+
+        Safe to call multiple times. Called automatically on ``__exit__``
+        and (as a fallback) by the weakref finalizer at GC time.
+        """
+        self._finalizer()
+
+    def __exit__(self, exc_type, exc, tb) -> bool:
+        self.close()
+        return False
+
+
+def _unlink_quiet(path: str) -> None:
+    """Delete ``path`` if it still exists, swallowing OSError."""
+    try:
+        os.unlink(path)
+    except OSError:
+        pass
+
+
+class SqliteReader(Reader):
+    name = "sqlite-stdlib"
+    formats = ("sqlite",)
+    is_default = True
+
+    def read(self, data: bytes, *, source_name: Optional[str] = None, **_) -> Document:
+        if not data:
+            raise ParseError("sqlite-stdlib reader: input is empty")
+        if not data.startswith(b"SQLite format 3\x00"):
+            raise ParseError(
+                "sqlite-stdlib reader: not a valid SQLite 3 database "
+                "(missing 'SQLite format 3' header)"
+            )
+
+        warnings: list[str] = []
+
+        # sqlite3 only opens from a filesystem path, so spill bytes to a
+        # temp file and keep it alive for the Document's lifetime. The
+        # SqliteDocument takes ownership of cleanup.
+        tmp = tempfile.NamedTemporaryFile(
+            prefix="fyle-sqlite-", suffix=".db", delete=False
+        )
+        tmp.write(data)
+        tmp.flush()
+        tmp.close()
+        tmp_path = tmp.name
+
+        handed_off = False
+        try:
+            uri = f"file:{tmp_path}?mode=ro&immutable=1"
+            try:
+                conn = sqlite3.connect(uri, uri=True)
+            except sqlite3.Error as e:
+                raise ParseError(f"sqlite3 failed to open database: {e}") from e
+            try:
+                pages = _build_pages(conn, warnings)
+            finally:
+                conn.close()
+
+            if not pages:
+                pages.append(
+                    Page(text="(database contains no user tables or views)", number=1)
+                )
+                warnings.append("sqlite: no user tables or views found")
+
+            title: Optional[str] = None
+            if source_name:
+                try:
+                    title = Path(source_name).stem or None
+                except (TypeError, ValueError):
+                    title = None
+
+            try:
+                meta = Meta(
+                    format="sqlite",
+                    pages=len(pages),
+                    size=len(data),
+                    title=title,
+                    reader=self.name,
+                    created_at=datetime.now(timezone.utc),
+                    warnings=warnings,
+                )
+                doc = SqliteDocument(
+                    pages=pages,
+                    meta=meta,
+                    db_path=tmp_path,
+                    table_names=[p.name for p in pages if p.name],
+                )
+            except Exception as e:  # pragma: no cover - defensive
+                raise ParseError(f"sqlite reader failed to build Document: {e}") from e
+
+            handed_off = True
+            return doc
+        finally:
+            if not handed_off:
+                _unlink_quiet(tmp_path)
+
+
+def _build_pages(conn: sqlite3.Connection, warnings: list[str]) -> list[Page]:
+    """Walk every user table and view, render each as a ``Page``."""
+    cur = conn.cursor()
+    cur.execute(
+        "SELECT name, type FROM sqlite_master "
+        "WHERE type IN ('table', 'view') "
+        "AND name NOT LIKE 'sqlite_%' "
+        "ORDER BY name"
+    )
+    entries = cur.fetchall()
+
+    pages: list[Page] = []
+    for idx, (obj_name, obj_type) in enumerate(entries, start=1):
+        try:
+            pages.append(_render_object(conn, obj_name, obj_type, idx, warnings))
+        except Exception as e:
+            warnings.append(f"sqlite: failed to render {obj_type} {obj_name!r}: {e}")
+            pages.append(
+                Page(text=f"# {obj_name}\n\n(render failed: {e})", number=idx, name=obj_name)
+            )
+    return pages
+
+
+def _render_object(
+    conn: sqlite3.Connection,
+    obj_name: str,
+    obj_type: str,
+    page_no: int,
+    warnings: list[str],
+) -> Page:
+    """Render one table/view into schema + sample Markdown + structural tables."""
+    cur = conn.cursor()
+
+    # Schema via PRAGMA table_info (works for views too — reports column order
+    # and types as SQLite sees them).
+    schema_headers = ["column", "type", "nullable", "default", "pk"]
+    schema_rows: list[list[str]] = []
+    try:
+        cur.execute(f'PRAGMA table_info("{obj_name}")')
+        for _cid, col_name, col_type, notnull, dflt, pk in cur.fetchall():
+            schema_rows.append([
+                str(col_name),
+                str(col_type or ""),
+                "NO" if notnull else "YES",
+                "" if dflt is None else str(dflt),
+                str(pk) if pk else "",
+            ])
+    except sqlite3.Error as e:
+        warnings.append(f"sqlite: PRAGMA table_info({obj_name!r}) failed: {e}")
+
+    # Row count (views report count of their materialised result).
+    total_rows: Optional[int] = None
+    try:
+        cur.execute(f'SELECT COUNT(*) FROM "{obj_name}"')
+        total_rows = int(cur.fetchone()[0])
+    except sqlite3.Error as e:
+        warnings.append(f"sqlite: COUNT(*) on {obj_name!r} failed: {e}")
+
+    # Sample rows.
+    sample_headers: list[str] = []
+    sample_rows: list[list[str]] = []
+    try:
+        cur.execute(f'SELECT * FROM "{obj_name}" LIMIT {_SAMPLE_ROWS}')
+        sample_headers = [d[0] for d in (cur.description or [])]
+        for row in cur.fetchall():
+            sample_rows.append(["NULL" if v is None else str(v) for v in row])
+    except sqlite3.Error as e:
+        warnings.append(f"sqlite: SELECT sample on {obj_name!r} failed: {e}")
+
+    # Markdown assembly.
+    md_parts: list[str] = [f"# {obj_name}"]
+    kind_hint = "view" if obj_type == "view" else "table"
+    if total_rows is not None:
+        md_parts.append(f"_type: {kind_hint}, rows: {total_rows}_")
+    else:
+        md_parts.append(f"_type: {kind_hint}_")
+
+    md_parts.append("**Schema**")
+    md_parts.append(_render_md_table(schema_headers, schema_rows))
+
+    if sample_headers:
+        shown = len(sample_rows)
+        if total_rows is not None and total_rows > shown:
+            md_parts.append(f"**Sample rows** ({shown} of {total_rows})")
+        else:
+            md_parts.append(f"**Sample rows** ({shown})")
+        md_parts.append(_render_md_table(sample_headers, sample_rows))
+
+    page_text = "\n\n".join(md_parts)
+
+    schema_table = Table(
+        text=_render_md_table(schema_headers, schema_rows),
+        rows=schema_rows,
+        headers=schema_headers,
+        page=page_no,
+    )
+    tables: list[Table] = [schema_table]
+    if sample_headers:
+        tables.append(
+            Table(
+                text=_render_md_table(sample_headers, sample_rows),
+                rows=sample_rows,
+                headers=sample_headers,
+                page=page_no,
+            )
+        )
+
+    return Page(text=page_text, number=page_no, name=obj_name, tables=tables)
+
+
+def _render_md_table(headers: list[str], rows: list[list[str]]) -> str:
+    """Render a GFM pipe table. Uses ``tabulate`` when available, else manual."""
+    if not headers:
+        return "(no columns)"
+    try:
+        from tabulate import tabulate as _tabulate
+        return _tabulate(rows, headers=headers, tablefmt="github")
+    except ImportError:
+        # Minimal manual fallback; kept simple because ``tabulate`` is a
+        # standard fyle dependency so this path is rarely exercised.
+        esc = lambda s: str(s).replace("|", "\\|")
+        out = ["| " + " | ".join(esc(h) for h in headers) + " |"]
+        out.append("|" + "|".join(["---"] * len(headers)) + "|")
+        for row in rows:
+            out.append("| " + " | ".join(esc(c) for c in row) + " |")
+        return "\n".join(out)

fyle/_readers/text/__init__.py

@@ -0,0 +1,7 @@
+"""Plain-text reader — passthrough of ``.txt`` / ``.log`` files.
+
+File name inside the subpackage is the *core driver library* of the
+reader implementation (here: the Python standard library), so any future
+alternative implementation can live alongside under its own library name.
+"""
+from . import stdlib  # noqa: F401