smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/io/progress.py

@@ -0,0 +1,140 @@
+"""Progress feedback — always stderr, always TTY-gated, gone on completion.
+
+Spec §6.1: a single spinner line overwritten in place, with count, percent, and
+an ETA that appears only after a few completions. When stderr is not a terminal
+(a cron job, a pipe), progress is suppressed entirely — stdout stays sacred and
+the log stays clean. The render functions are pure; ``Spinner`` adds the clock,
+throttling, and the stderr writes.
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from smartpipe.io import tty
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from typing import TextIO
+
+__all__ = ["Spinner", "format_eta", "make_stderr_spinner", "render_known", "render_unknown"]
+
+_BRAILLE = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
+_ASCII = "-\\|/"
+_ETA_WARMUP = 5  # completions before an ETA is trustworthy enough to show
+_MIN_REDRAW_S = 0.1  # ≤ 10 fps
+_CLEAR_LINE = "\x1b[K"
+
+
+def format_eta(seconds: float) -> str:
+    total = int(seconds)
+    hours, remainder = divmod(total, 3600)
+    minutes, secs = divmod(remainder, 60)
+    if hours:
+        return f"{hours}h{minutes}m"
+    if minutes:
+        return f"{minutes}m{secs}s"
+    return f"{secs}s"
+
+
+def render_known(frame: str, *, done: int, total: int, eta_seconds: float | None) -> str:
+    percent = int(done / total * 100) if total else 100
+    line = f"{frame} Processing {total} items [{done}/{total}] {percent}%"
+    if eta_seconds is not None:
+        line += f"  ~{format_eta(eta_seconds)} remaining"
+    return line
+
+
+def render_unknown(
+    frame: str, *, done: int, rate: float, matched: int | None = None, extra: str | None = None
+) -> str:
+    line = f"{frame} Processing [{done}] {rate:.1f}/s"
+    if matched is not None:
+        line += f" · {matched} matched"
+    if extra:
+        line += f" · {extra}"
+    return line
+
+
+@dataclass(slots=True)
+class Spinner:
+    stream: TextIO
+    enabled: bool
+    ascii_only: bool
+    clock: Callable[[], float]
+    total: int | None = None
+    matched: int | None = None  # filter's status-line segment
+    extra: str | None = None  # map's live --tally segment
+    _done: int = 0
+    _start: float = 0.0
+    _last_draw: float = field(default=-1.0)
+    _frame: int = 0
+    _drew: bool = False
+
+    def start(self, total: int | None) -> None:
+        self.total = total
+        self._done = 0
+        self._start = self.clock()
+        self._last_draw = -1.0
+
+    def advance(self) -> None:
+        self._done += 1
+        if not self.enabled:
+            return
+        now = self.clock()
+        is_last = self.total is not None and self._done >= self.total
+        if not is_last and now - self._last_draw < _MIN_REDRAW_S:
+            return
+        self._last_draw = now
+        self._draw(now)
+
+    def finish(self) -> None:
+        if self.enabled and self._drew:
+            self.stream.write(f"\r{_CLEAR_LINE}")
+            self.stream.flush()
+
+    def _color(self) -> bool:
+        import os
+
+        return self.enabled and not os.environ.get("NO_COLOR")
+
+    def _draw(self, now: float) -> None:
+        frames = _ASCII if self.ascii_only else _BRAILLE
+        frame = frames[self._frame % len(frames)]
+        self._frame += 1
+        elapsed = max(now - self._start, 1e-9)
+        rate = self._done / elapsed
+        if self.total is None:
+            line = render_unknown(
+                frame, done=self._done, rate=rate, matched=self.matched, extra=self.extra
+            )
+        else:
+            eta = (self.total - self._done) / rate if self._done >= _ETA_WARMUP and rate else None
+            line = render_known(frame, done=self._done, total=self.total, eta_seconds=eta)
+        from smartpipe.io import metering
+
+        consumed = metering.status_segment()  # D40: live observed units
+        if self._color():
+            line = f"\x1b[36m{frame}\x1b[0m{line[len(frame) :]}"
+            if consumed:
+                line += f"   \x1b[2m{consumed}\x1b[0m"
+        elif consumed:
+            line += f"   {consumed}"
+        self.stream.write(f"\r{line}{_CLEAR_LINE}")
+        self.stream.flush()
+        self._drew = True
+
+
+def make_stderr_spinner() -> Spinner:
+    """A spinner wired to the real stderr — enabled only when stderr is a TTY,
+    with a Braille or ASCII frame set depending on the encoding."""
+    encoding = (sys.stderr.encoding or "").lower()
+    return Spinner(
+        stream=sys.stderr,
+        enabled=tty.stderr_is_tty(),
+        ascii_only="utf" not in encoding,
+        clock=time.monotonic,
+    )

smartpipe/io/readers.py

@@ -0,0 +1,455 @@
+"""Item sources — all with the same shape: ``AsyncIterator[Item]``.
+
+Stdin is read **incrementally** (stage-08 as amended): a daemon pump thread does the
+blocking ``readline`` and hands lines to a bounded asyncio queue, so items flow as
+they arrive (``tail -f`` works), backpressure is real (the pump stalls when the queue
+fills), and shutdown can never hang on a blocked read (the async side is cancellable;
+the daemon flag is the last-resort guarantee). ``--in`` file lists stay finite.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import contextlib
+import os
+import threading
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from smartpipe.core.errors import ItemError, SetupFault, UsageFault
+from smartpipe.io import diagnostics
+from smartpipe.io.items import Item, item_from_file, item_from_line
+from smartpipe.models.base import AudioData, ImageData, VideoData
+from smartpipe.parsing.detect import FileKind, detect_kind, route
+from smartpipe.parsing.extract import MissingExtra, extract
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Sequence
+    from typing import TextIO
+
+    from smartpipe.io.inputs import InputSpec
+
+__all__ = [
+    "ensure_not_a_tty",
+    "figure_note",
+    "file_items",
+    "from_files_items",
+    "resolve_items",
+    "stdin_items",
+]
+
+_HEAD_BYTES = 8192
+_QUEUE_MAX = 1024  # lines buffered ahead of consumption — memory stays bounded
+
+
+def resolve_items(
+    spec: InputSpec, stdin: TextIO, *, stop: asyncio.Event | None = None
+) -> tuple[AsyncIterator[Item], int | None]:
+    """The single entry point every verb uses: dispatch on the input flags.
+
+    Returns ``(items, total)`` — total is known only for ``--in`` file lists;
+    stdin is a stream (``tail -f`` works), so its total is ``None`` and the
+    spinner shows count+rate instead of an ETA. Only the stdin paths guard
+    against a bare terminal."""
+    from smartpipe.io.inputs import expand_globs
+
+    if spec.patterns and spec.from_files:
+        raise UsageFault(
+            "--in and --from-files are both file sources — use one\n"
+            "  --in takes globs; --from-files reads filenames from stdin."
+        )
+    if spec.patterns:
+        loaded = file_items(expand_globs(spec.patterns))  # UsageFault if no match
+        if stdin.isatty():  # files only — no pipe to chain
+            return _iter_list(loaded), len(loaded)
+        # spec §8: mixed input is files first (glob-sorted), then stdin lines
+        return _chain_files_then_stdin(loaded, stdin, stop), None
+    ensure_not_a_tty(stdin)
+    if spec.from_files:
+        return from_files_items(stdin, stop=stop), None
+    return stdin_items(stdin, stop=stop), None
+
+
+async def _chain_files_then_stdin(
+    loaded: Sequence[Item], stdin: TextIO, stop: asyncio.Event | None
+) -> AsyncIterator[Item]:
+    for item in loaded:
+        yield item
+    async for item in stdin_items(stdin, stop=stop):
+        yield item
+
+
+async def _iter_list(items: Sequence[Item]) -> AsyncIterator[Item]:
+    for item in items:
+        yield item
+
+
+@dataclass(frozen=True, slots=True)
+class _StdinDocument:
+    tmp_name: str
+    kind: FileKind
+
+
+# A queue message: ("line", text) · ("document", _StdinDocument) ·
+# ("image", ImageData) · ("fatal", screen) · None = EOF.
+_Message = tuple[str, object] | None
+
+_KIND_SUFFIX: dict[FileKind, str] = {
+    FileKind.PDF: ".pdf",
+    FileKind.DOCX: ".docx",
+    FileKind.XLSX: ".xlsx",
+    FileKind.PPTX: ".pptx",
+    FileKind.HTML: ".html",
+    FileKind.EPUB: ".epub",
+    FileKind.AUDIO: ".mp3",
+}
+
+
+async def _messages(stdin: TextIO, stop: asyncio.Event | None) -> AsyncIterator[tuple[str, object]]:
+    """Incremental stdin source: daemon pump thread → bounded queue → cancellable get.
+
+    Real streams are read with ``os.read`` on the raw fd, NOT ``stdin.readline()``:
+    a thread blocked in ``readline`` holds the TextIOWrapper's lock, and CPython's
+    interpreter-shutdown finalization then deadlocks trying to close the stream —
+    the exact hang the streaming e2e caught. On the fd path the FIRST read also
+    sniffs (stage-07 task 4): a binary document redirected to stdin becomes one
+    spooled document message; text proceeds as lines with the sniffed bytes as the
+    carry. Objects without a usable fd (StringIO in tests) fall back to
+    ``readline`` — text-only by construction, never blocking, never sniffed.
+    """
+    queue: asyncio.Queue[_Message] = asyncio.Queue(_QUEUE_MAX)
+    loop = asyncio.get_running_loop()
+
+    def put(message: _Message) -> None:
+        asyncio.run_coroutine_threadsafe(queue.put(message), loop).result()
+
+    def pump_text_fd(fd: int, carry: bytes) -> None:
+        buffer = bytearray(carry)
+        while True:
+            while (newline := buffer.find(0x0A)) != -1:
+                line = bytes(buffer[: newline + 1])
+                del buffer[: newline + 1]
+                put(("line", line.decode("utf-8", errors="replace")))
+            chunk = os.read(fd, 65536)  # blocks WITHOUT holding any io lock
+            if not chunk:
+                if buffer:  # final line without a trailing newline
+                    put(("line", bytes(buffer).decode("utf-8", errors="replace")))
+                return
+            buffer += chunk
+
+    def spool_document(fd: int, head: bytes, kind: FileKind) -> None:
+        import tempfile
+
+        suffix = _KIND_SUFFIX.get(kind, "")
+        with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as handle:
+            handle.write(head)
+            while chunk := os.read(fd, 65536):
+                handle.write(chunk)
+        put(("document", _StdinDocument(handle.name, kind)))
+
+    def collect_image(fd: int, head: bytes) -> None:
+        data = bytearray(head)
+        while chunk := os.read(fd, 65536):
+            data += chunk
+        mime = _magic_image_mime(bytes(data[:16]))
+        put(("image", ImageData(data=bytes(data), mime=mime)))
+
+    def collect_audio(fd: int, head: bytes, kind: FileKind) -> None:
+        data = bytearray(head)
+        while chunk := os.read(fd, 65536):
+            data += chunk
+        from smartpipe.parsing.detect import audio_mime
+
+        suffix = _KIND_SUFFIX.get(kind, ".mp3")
+        put(("audio", AudioData(data=bytes(data), mime=audio_mime(Path(f"x{suffix}")))))
+
+    def collect_video(fd: int, head: bytes) -> None:
+        data = bytearray(head)
+        while chunk := os.read(fd, 65536):
+            data += chunk
+        put(("video", VideoData(data=bytes(data), mime="video/mp4")))
+
+    def pump_fd(fd: int) -> None:
+        head = os.read(fd, _HEAD_BYTES)  # one read — a live stream must not stall here
+        if not head:
+            return  # empty stdin
+        kind = detect_kind(Path("<stdin>"), head)
+        match route(kind):
+            case "text":
+                pump_text_fd(fd, head)
+            case "doc":
+                spool_document(fd, head, kind)
+            case "audio":
+                collect_audio(fd, head, kind)
+            case "video":
+                collect_video(fd, head)
+            case "image":
+                collect_image(fd, head)
+            case "skip":
+                from smartpipe.cli import screens
+
+                put(("fatal", screens.BINARY_STDIN_UNPARSEABLE))
+
+    def pump_readline() -> None:
+        while True:
+            line = stdin.readline()  # non-blocking sources only (StringIO et al.)
+            if not line:
+                return
+            put(("line", line))
+
+    def pump() -> None:
+        try:
+            fd: int | None
+            try:
+                fd = stdin.fileno()
+            except (OSError, ValueError, AttributeError):
+                fd = None
+            if fd is None:
+                pump_readline()
+            else:
+                pump_fd(fd)
+        except (RuntimeError, ValueError, OSError, concurrent.futures.CancelledError):
+            # loop closed, consumer gone, or the stream was closed under us —
+            # any of these means "stop pumping", never a crash or a stderr trace
+            return
+        finally:
+            sentinel = queue.put(None)  # blocking put — a full queue can't swallow EOF
+            try:
+                asyncio.run_coroutine_threadsafe(sentinel, loop).result()
+            except (RuntimeError, concurrent.futures.CancelledError):
+                sentinel.close()  # loop gone — don't leave a never-awaited coroutine
+
+    threading.Thread(target=pump, name="smartpipe-stdin-pump", daemon=True).start()
+    while True:
+        message = await _next_or_stop(queue, stop)
+        if message is None:
+            return
+        yield message
+
+
+def _magic_image_mime(head: bytes) -> str:
+    if head.startswith(b"\x89PNG"):
+        return "image/png"
+    if head.startswith(b"\xff\xd8\xff"):
+        return "image/jpeg"
+    if head.startswith((b"GIF87a", b"GIF89a")):
+        return "image/gif"
+    return "image/webp" if head[8:12] == b"WEBP" else "image/png"
+
+
+async def _next_or_stop(queue: asyncio.Queue[_Message], stop: asyncio.Event | None) -> _Message:
+    if stop is None:
+        return await queue.get()
+    if stop.is_set():
+        return None
+    get_task = asyncio.ensure_future(queue.get())
+    stop_task = asyncio.ensure_future(stop.wait())
+    done, _pending = await asyncio.wait({get_task, stop_task}, return_when=asyncio.FIRST_COMPLETED)
+    if get_task in done:
+        stop_task.cancel()
+        return get_task.result()
+    get_task.cancel()
+    with contextlib.suppress(asyncio.CancelledError):
+        await get_task  # reap it — no strays, no destroyed-pending warnings
+    return None
+
+
+async def _lines(stdin: TextIO, stop: asyncio.Event | None) -> AsyncIterator[str]:
+    """Text-only view of the message stream (``--from-files`` wants filenames)."""
+    async for message in _messages(stdin, stop):
+        tag, payload = message
+        if tag != "line":
+            raise SetupFault(
+                "error: --from-files expects filenames on stdin, got a binary document\n"
+                "  Pipe a list of paths in, e.g.:\n"
+                "  find . -name '*.md' | smartpipe map … --from-files"
+            )
+        assert isinstance(payload, str)
+        yield payload
+
+
+async def stdin_items(stdin: TextIO, *, stop: asyncio.Event | None = None) -> AsyncIterator[Item]:
+    """Each stdin line is one Item, yielded as it arrives (never waits for EOF).
+
+    A redirected binary document (``smartpipe map … < report.pdf``) is ONE item:
+    spooled, extracted, source ``<stdin>`` (stage-07 task 4). A final line without
+    a trailing newline is still an item; empty input yields nothing; CRLF and the
+    line-0 BOM are handled per-item by ``item_from_line``.
+    """
+
+    index = 0
+    async for message in _messages(stdin, stop):
+        tag, payload = message
+        if tag == "line":
+            assert isinstance(payload, str)
+            yield item_from_line(payload, index)
+            index += 1
+        elif tag == "document":
+            assert isinstance(payload, _StdinDocument)
+            yield await asyncio.to_thread(_extract_stdin_document, payload.tmp_name, payload.kind)
+        elif tag == "image":
+            assert isinstance(payload, ImageData)
+            item = item_from_file("", "<stdin>", 0)
+            yield replace(item, media=(payload,))
+        elif tag == "audio":
+            assert isinstance(payload, AudioData)
+            item = item_from_file("", "<stdin>", 0)
+            yield replace(item, media=(payload,))
+        elif tag == "video":
+            assert isinstance(payload, VideoData)
+            item = item_from_file("", "<stdin>", 0)
+            yield replace(item, media=(payload,))
+        else:  # "fatal"
+            assert isinstance(payload, str)
+            raise SetupFault(payload)
+
+
+def _extract_stdin_document(tmp_name: str, kind: FileKind) -> Item:
+    from smartpipe.cli import screens
+
+    path = Path(tmp_name)
+    try:
+        extracted = extract(path, kind)
+    except MissingExtra as exc:
+        raise SetupFault(screens.stdin_document_failed(exc.guidance.splitlines()[0])) from exc
+    except ItemError as exc:
+        raise SetupFault(screens.stdin_document_failed(str(exc))) from exc
+    finally:
+        with contextlib.suppress(OSError):
+            path.unlink()  # statelessness: the spool never outlives the run
+    if extracted.warning is not None:
+        diagnostics.warn(f"<stdin>: {extracted.warning}")
+    return item_from_file(extracted.text, "<stdin>", 0)
+
+
+async def from_files_items(
+    stdin: TextIO, *, stop: asyncio.Event | None = None
+) -> AsyncIterator[Item]:
+    """``--from-files``: each non-blank stdin line names a file to read — also
+    incremental, so ``find … | smartpipe … --from-files`` processes as names arrive."""
+    from pathlib import Path
+
+    warned_extras: set[str] = set()
+    index = 0
+    async for line in _lines(stdin, stop):
+        name = line.strip()
+        if not name:
+            continue
+        item = _load_file(Path(name), index, warned_extras)
+        if item is not None:
+            yield item
+            index += 1
+
+
+def file_items(paths: Sequence[Path]) -> list[Item]:
+    """Each file is one item. Unreadable, unparseable, or missing-dependency files
+    are skipped with a warning (spec §6.3) — the run never crashes on a bad file."""
+    warned_extras: set[str] = set()
+    items: list[Item] = []
+    for index, path in enumerate(paths):
+        item = _load_file(path, index, warned_extras)
+        if item is not None:
+            items.append(item)
+    return items
+
+
+def _load_file(path: Path, index: int, warned_extras: set[str]) -> Item | None:
+    try:
+        with path.open("rb") as handle:
+            head = handle.read(_HEAD_BYTES)
+    except OSError as exc:
+        diagnostics.warn(f"skipped: {path} (cannot read: {exc.strerror or exc})")
+        return None
+    kind = detect_kind(path, head)
+    if route(kind) in ("audio", "video"):
+        # D20/D27: media carries its BYTES — conversion is lazy and per-verb
+        # (map tries the native wire first; text verbs convert on demand)
+        from smartpipe.parsing.detect import audio_mime, video_mime
+
+        try:
+            data = path.read_bytes()
+        except OSError as exc:
+            diagnostics.warn(f"skipped: {path} (cannot read: {exc.strerror or exc})")
+            return None
+        item = item_from_file("", str(path), index)
+        media = (
+            AudioData(data=data, mime=audio_mime(path))
+            if route(kind) == "audio"
+            else VideoData(data=data, mime=video_mime(path))
+        )
+        return replace(item, media=(media,))
+    try:
+        extracted = extract(path, kind)
+    except MissingExtra as exc:
+        if exc.extra not in warned_extras:
+            diagnostics.warn(exc.guidance)
+            warned_extras.add(exc.extra)
+        return None
+    except ItemError as exc:
+        diagnostics.warn(f"skipped: {path} ({exc})")
+        return None
+    if extracted.warning is not None:
+        diagnostics.warn(f"{path}: {extracted.warning}")
+    item = item_from_file(extracted.text, str(path), index)
+    if extracted.image is not None:
+        return replace(item, media=(extracted.image,))  # map sends it to a vision model
+    figures = _document_figures(path, kind, extracted.text)
+    if figures:
+        return replace(item, media=figures)
+    return item
+
+
+_FIGURE_CAP = 8  # request-size and cost sanity per document item (D32)
+_FIGURE_KINDS = {FileKind.PDF, FileKind.DOCX, FileKind.PPTX, FileKind.XLSX}
+
+
+_THIN_TEXT = 64  # under this many chars, a figure-bearing document reads as a scan
+
+
+def _document_figures(path: Path, kind: FileKind, text: str) -> tuple[ImageData, ...]:
+    """D32: a document item carries its embedded figures by default — capped,
+    icon-floored, announced once per file. D39/03: when the text layer is
+    THIN, the announcement says so — a scanned document routed to the vision
+    path must never look like silent emptiness."""
+    if kind not in _FIGURE_KINDS:
+        return ()
+    from smartpipe.parsing.extract import MissingExtra, embedded_images
+
+    try:
+        media = embedded_images(path)
+    except (MissingExtra, ItemError):
+        return ()  # text still flows; --media names scan problems loudly
+    total = len(media.images)
+    if total == 0:
+        return ()
+    kept = media.images[:_FIGURE_CAP]
+    capped = total - len(kept)
+    diagnostics.note(figure_note(path.name, len(text.strip()), len(kept), capped))
+    return tuple(found.image for found in kept)
+
+
+def figure_note(name: str, text_length: int, kept: int, capped: int) -> str:
+    if text_length < _THIN_TEXT:
+        hint = (
+            f" ({capped} more capped — split --by pages --media processes every page)"
+            if capped
+            else ""
+        )
+        return (
+            f"{name}: thin text layer ({text_length} chars) — scanned? "
+            f"routed {kept} page image(s) to the vision path{hint}"
+        )
+    suffix = f" ({capped} more capped)" if capped else ""
+    plural = "s" if kept != 1 else ""
+    return f"{name}: {kept} figure{plural} attached{suffix}"
+
+
+def ensure_not_a_tty(stdin: TextIO) -> None:
+    """A kind guardrail: bare `smartpipe map ...` at a terminal would silently wait."""
+    if stdin.isatty():
+        raise UsageFault(
+            "reading from a terminal — pipe some input in, e.g.:\n"
+            '  cat notes.txt | smartpipe map "..."'
+        )

smartpipe/io/text.py

@@ -0,0 +1,40 @@
+"""Terminal display width — cells, not code points (DEFER-2).
+
+Stdlib-only by design: ``wcwidth`` stays outside the dependency budget. The
+rule: combining marks and zero-width format characters (ZWJ et al.) count 0,
+East-Asian Wide/Fullwidth count 2, everything else counts 1. Emoji-ZWJ
+sequences therefore measure as the *sum of their parts* — approximate on
+purpose, and documented as such wherever alignment matters.
+"""
+
+from __future__ import annotations
+
+import unicodedata
+
+__all__ = ["clip_to_width", "display_width"]
+
+_WIDE = frozenset({"W", "F"})
+
+
+def _char_width(char: str) -> int:
+    if unicodedata.combining(char):
+        return 0
+    if unicodedata.category(char) == "Cf":  # ZWJ, ZWNJ, direction marks — zero cells
+        return 0
+    return 2 if unicodedata.east_asian_width(char) in _WIDE else 1
+
+
+def display_width(text: str) -> int:
+    """How many terminal cells ``text`` occupies (approximate for emoji-ZWJ)."""
+    return sum(_char_width(char) for char in text)
+
+
+def clip_to_width(text: str, budget: int) -> str:
+    """The longest prefix of ``text`` that fits in ``budget`` cells — a wide
+    character that would straddle the boundary is dropped, never split."""
+    used = 0
+    for position, char in enumerate(text):
+        used += _char_width(char)
+        if used > budget:
+            return text[:position]
+    return text