smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/engine/windows.py

@@ -0,0 +1,78 @@
+"""Window bookkeeping for streaming ``reduce`` (stage-08, spec §4.2) — pure.
+
+Semantics (pinned): the first window emits at item ``size`` (a full window);
+thereafter one window per ``every`` new items, each containing the last ``size``
+items. ``flush()`` returns whatever arrived after the last emission as a
+``partial=True`` window — Ctrl+C never silently discards buffered lines. With
+``every == size`` this is tumbling; smaller ``every`` slides. ``every > size``
+would skip items, so the policy rejects it.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass, field
+from typing import Generic, TypeVar
+
+__all__ = ["Window", "WindowBuffer", "WindowPolicy"]
+
+T = TypeVar("T")
+
+
+@dataclass(frozen=True, slots=True)
+class WindowPolicy:
+    size: int
+    every: int  # == size → tumbling; < size → sliding
+
+    def __post_init__(self) -> None:
+        if self.size < 1:
+            raise ValueError(f"window size must be >= 1, got {self.size}")
+        if not 1 <= self.every <= self.size:
+            raise ValueError(
+                f"--every must be between 1 and the window size ({self.size}), got {self.every}"
+            )
+
+
+@dataclass(frozen=True, slots=True)
+class Window(Generic[T]):
+    items: tuple[T, ...]
+    end_index: int  # 1-based ordinal of the last item, over the whole stream
+    partial: bool
+
+
+@dataclass(slots=True)
+class WindowBuffer(Generic[T]):
+    """Tiny mutable state machine (documented Spinner-style exception to the
+    frozen-by-default rule): push items, get a Window when a boundary lands."""
+
+    policy: WindowPolicy
+    _buffer: deque[T] = field(init=False)
+    _seen: int = 0
+    _since_emit: int = 0
+    _emitted_any: bool = False
+
+    def __post_init__(self) -> None:
+        self._buffer = deque(maxlen=self.policy.size)
+
+    def push(self, item: T) -> Window[T] | None:
+        self._buffer.append(item)
+        self._seen += 1
+        self._since_emit += 1
+        boundary = (
+            self._seen >= self.policy.size and self._since_emit >= self.policy.every
+            if self._emitted_any
+            else self._seen == self.policy.size
+        )
+        if not boundary:
+            return None
+        self._emitted_any = True
+        self._since_emit = 0
+        return Window(items=tuple(self._buffer), end_index=self._seen, partial=False)
+
+    def flush(self) -> Window[T] | None:
+        """The trailing partial window: everything that arrived after the last
+        emission (or the whole short stream, if nothing ever emitted)."""
+        if self._since_emit == 0 or self._seen == 0:
+            return None
+        tail = list(self._buffer)[-self._since_emit :] if self._emitted_any else list(self._buffer)
+        return Window(items=tuple(tail), end_index=self._seen, partial=True)

smartpipe/io/__init__.py

@@ -0,0 +1,9 @@
+"""I/O adapters: stdin/file readers, TTY-adaptive writers, stderr diagnostics.
+
+Boundary rule (plan/architecture.md): only ``writers`` touches stdout; only
+``diagnostics`` and ``progress`` touch stderr.
+"""
+
+from __future__ import annotations
+
+__all__: list[str] = []

smartpipe/io/diagnostics.py

@@ -0,0 +1,148 @@
+"""Every user-facing stderr message flows through here (stdout stays sacred).
+
+Message style contract: plan/ux.md "Error message style" — one-line what, short
+why, copy-pasteable fix. Screens arrive pre-formatted with their own ``error:``
+prefix; bare fault messages get the prefix added.
+"""
+
+from __future__ import annotations
+
+import sys
+import traceback
+from typing import NoReturn
+
+from smartpipe.core.errors import (
+    ExitCode,
+    SempipeError,
+    SetupFault,
+    TooManyFailures,
+    UsageFault,
+)
+from smartpipe.io import tty
+
+__all__ = [
+    "DegradationLog",
+    "die",
+    "drain_timed_out",
+    "internal_error",
+    "interrupted_summary",
+    "note",
+    "preview",
+    "report_error",
+    "warn",
+]
+
+_RED = "\x1b[31m"
+_RESET = "\x1b[0m"
+_ISSUES_URL = "https://github.com/prabal-rje/smartpipe/issues/new"
+
+
+def _paint(text: str, code: str) -> str:
+    """stderr color, honestly gated: TTY only, NO_COLOR wins (D42)."""
+    import os
+
+    if not tty.stderr_is_tty() or os.environ.get("NO_COLOR"):
+        return text
+    return f"\x1b[{code}m{text}{_RESET}"
+
+
+def warn(message: str) -> None:
+    sys.stderr.write(_paint(f"⚠ {message}", "33") + "\n")  # yellow — worth a glance
+    sys.stderr.flush()
+
+
+def preview(message: str) -> None:
+    """Informational cost/awareness lines (D18/D21): TTY-only, never in pipes/cron."""
+    if tty.stderr_is_tty():
+        sys.stderr.write(f"{message}\n")
+        sys.stderr.flush()
+
+
+def note(message: str) -> None:
+    sys.stderr.write(_paint(f"note: {message}", "2") + "\n")  # dim — informative, calm
+    sys.stderr.flush()
+
+
+def interrupted_summary(*, processed: int, skipped: int) -> None:
+    """The ux.md §12 drain summary — exact wording is contract."""
+    sys.stderr.write(
+        _paint(f"done: interrupted — {processed} processed · {skipped} skipped", "33") + "\n"
+    )
+    sys.stderr.flush()
+
+
+def drain_timed_out() -> None:
+    sys.stderr.write("done: interrupted — drain timed out\n")
+    sys.stderr.flush()
+
+
+def _emit_error(text: str) -> None:
+    if tty.stderr_supports_color() and text.startswith("error:"):
+        text = f"{_RED}error:{_RESET}{text.removeprefix('error:')}"
+    sys.stderr.write(f"{text}\n")
+    sys.stderr.flush()
+
+
+_DEGRADE_CAP = 5  # per conversion kind: first rows verbatim, then the rollup
+
+
+class DegradationLog:
+    """Per-run ledger of poor-man's conversions (D27): every degraded row is
+    announced (capped per kind), and one rollup line closes the run."""
+
+    def __init__(self) -> None:
+        self.counts: dict[str, int] = {}
+
+    def note(self, where: str, kind: str, detail: str) -> None:
+        count = self.counts.get(kind, 0) + 1
+        self.counts[kind] = count
+        if count <= _DEGRADE_CAP:
+            warn(f"degraded: {where} {kind} ({detail})")
+        elif count == _DEGRADE_CAP + 1:
+            warn(f"more {kind} rows follow (suppressed; the rollup lands at the end)")
+
+    def finish(self) -> None:
+        if not self.counts:
+            return
+        ranked = sorted(self.counts.items(), key=lambda pair: -pair[1])
+        marks = " · ".join(f"{kind} ×{count:,}" for kind, count in ranked)  # noqa: RUF001 — the pinned rollup mark
+        note(f"degraded: {marks}")
+
+
+def report_error(screen: str) -> None:
+    """Emit a full error screen without exiting — for commands that own their
+    exit code after cleanup (e.g. ``smartpipe schema``'s empty-stdout guarantee)."""
+    _emit_error(screen if screen.startswith("error:") else f"error: {screen}")
+
+
+def die(fault: SempipeError, *, debug: bool = False) -> NoReturn:
+    message = str(fault)
+    _emit_error(message if message.startswith("error:") else f"error: {message}")
+    if debug:
+        sys.stderr.write("".join(traceback.format_exception(fault)))
+        sys.stderr.flush()
+    match fault:
+        case UsageFault():
+            raise SystemExit(int(ExitCode.USAGE))
+        case SetupFault():
+            raise SystemExit(int(ExitCode.SETUP))
+        case TooManyFailures():
+            raise SystemExit(int(ExitCode.ALL_FAILED))
+        case _:
+            # ItemError (or the bare base) reaching die() is a programming error:
+            # per the taxonomy those are handled by the runner, not fatal paths.
+            raise SystemExit(int(ExitCode.BUG))
+
+
+def internal_error(exc: BaseException, *, debug: bool) -> NoReturn:
+    summary = f"{type(exc).__name__}: {exc}".splitlines()[0]
+    _emit_error("error: internal error — this is a bug in smartpipe, not in your usage")
+    sys.stderr.write(f"  {summary}\n")
+    if debug:
+        sys.stderr.write("".join(traceback.format_exception(exc)))
+        sys.stderr.write(f"  Please report it: {_ISSUES_URL}\n")
+    else:
+        sys.stderr.write("  Rerun with --debug for the full traceback, and please report it:\n")
+        sys.stderr.write(f"  {_ISSUES_URL}\n")
+    sys.stderr.flush()
+    raise SystemExit(int(ExitCode.BUG))

smartpipe/io/inputs.py

@@ -0,0 +1,44 @@
+"""Where items come from (spec §8): stdin lines, a glob of files, or a list of
+filenames on stdin. ``InputSpec`` captures the flags; ``expand_globs`` resolves
+``--in`` patterns to a sorted, de-duplicated file list.
+"""
+
+from __future__ import annotations
+
+import glob
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from smartpipe.core.errors import UsageFault
+
+__all__ = ["STDIN", "InputSpec", "expand_globs"]
+
+
+@dataclass(frozen=True, slots=True)
+class InputSpec:
+    patterns: tuple[str, ...]  # --in globs (may be empty)
+    from_files: bool  # --from-files: each stdin line names a file
+
+
+# The default: no file flags → read stdin lines. Shared because it's immutable.
+STDIN = InputSpec(patterns=(), from_files=False)
+
+
+def expand_globs(patterns: tuple[str, ...]) -> list[Path]:
+    """Resolve ``--in`` patterns to existing files, sorted, first-seen deduped.
+    An all-empty match is a usage error (exit 64) — silence would look like success."""
+    seen: dict[str, Path] = {}
+    for pattern in patterns:
+        expanded = os.path.expanduser(pattern)
+        for match in sorted(glob.glob(expanded, recursive=True)):
+            path = Path(match)
+            if path.is_file():
+                seen.setdefault(str(path), path)
+    if not seen:
+        joined = " ".join(patterns)
+        raise UsageFault(
+            f"no files matched: {joined}\n"
+            "  check the pattern, and quote it so the shell doesn't expand it first: --in '*.pdf'"
+        )
+    return list(seen.values())

smartpipe/io/items.py

@@ -0,0 +1,149 @@
+"""The Item model: the unit every verb operates on.
+
+Contract (plan/architecture.md "Core types"): ``raw`` preserves the input line
+byte-for-byte (minus the trailing newline) so ``filter``/``top_k`` can honor the
+passthrough-fidelity guarantee; ``data`` is set only when the line is a JSON
+*object* (an NDJSON record) — scalars and arrays are just text.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Literal, TypeGuard
+
+if TYPE_CHECKING:
+    from smartpipe.models.base import MediaData
+
+__all__ = ["Item", "ItemSource", "describe_source", "item_from_file", "item_from_line"]
+
+_BOM = ""
+
+
+@dataclass(frozen=True, slots=True)
+class ItemSource:
+    kind: Literal["stdin", "file"]
+    name: str  # "-" for stdin, else the path as given
+    index: int  # 0-based line number (stdin) or file ordinal
+
+
+@dataclass(frozen=True, slots=True)
+class Item:
+    raw: str  # the line/file EXACTLY as read (newline stripped)
+    text: str  # model-facing content (== raw for lines; extracted text for files)
+    data: Mapping[str, object] | None  # parsed object if the line was a JSON object
+    source: ItemSource
+    media: tuple[MediaData, ...] = ()  # media parts (D32) — text plus any number of figures/clips
+
+
+def item_from_line(line: str, index: int) -> Item:
+    raw = line.removesuffix("\n").removesuffix("\r")
+    if index == 0:
+        raw = raw.removeprefix(_BOM)
+    data = _sniff_json_object(raw)
+    media = _sniff_media(data)
+    text = raw
+    if media and data is not None:
+        # a media-carrying record's text is its "text" field, not the raw JSON
+        carried = data.get("text")
+        text = carried if isinstance(carried, str) else ""
+    return Item(
+        raw=raw,
+        text=text,
+        data=data,
+        source=_named_source(data, index),
+        media=media,
+    )
+
+
+def item_from_file(text: str, path: str, index: int) -> Item:
+    """A whole file is one item: its extracted text, with no JSON sniffing (a
+    document's text isn't an NDJSON line). ``filter``/``top_k`` emit its path."""
+    return Item(
+        raw=text,
+        text=text,
+        data=None,
+        source=ItemSource(kind="file", name=path, index=index),
+    )
+
+
+def _named_source(data: Mapping[str, object] | None, index: int) -> ItemSource:
+    # split emits {"source": "call.mp3 §00:10-00:20"} — keep that provenance
+    name = data.get("source") if data is not None else None
+    return ItemSource(kind="stdin", name=name if isinstance(name, str) else "-", index=index)
+
+
+def _sniff_media(data: Mapping[str, object] | None) -> tuple[MediaData, ...]:
+    """``split`` ships media as base64 NDJSON (audio/video slices, figures, and
+    multi-part page items) — rebuild the bytes so the next verb can hear or see
+    them (D27/D32)."""
+    if data is None:
+        return ()
+    from smartpipe.core.jsontools import as_items, as_record
+
+    entries = as_items(data.get("parts"))
+    if entries is not None:
+        return tuple(
+            part for entry in entries if (part := _one_media(as_record(entry))) is not None
+        )
+    single = _one_media(data)
+    return (single,) if single is not None else ()
+
+
+def _one_media(data: Mapping[str, object] | None) -> MediaData | None:
+    if data is None:
+        return None
+    mime = data.get("mime")
+    if not isinstance(mime, str):
+        return None
+    import base64
+    import binascii
+
+    from smartpipe.models.base import (  # runtime construction
+        AudioData,
+        ImageData,
+        VideoData,
+    )
+
+    for key, build in (
+        ("audio_b64", AudioData),
+        ("image_b64", ImageData),
+        ("video_b64", VideoData),
+    ):
+        encoded = data.get(key)
+        if not isinstance(encoded, str):
+            continue
+        try:
+            return build(base64.b64decode(encoded, validate=True), mime)
+        except (binascii.Error, ValueError):
+            return None  # not ours — treat as a plain JSON line
+    return None
+
+
+def describe_source(source: ItemSource) -> str:
+    """Human wording for warnings — 1-based lines, plain filenames; a split
+    stage's provenance (``call.wav §00:10-00:20``) survives the pipe."""
+    if source.kind == "stdin" and source.name == "-":
+        return f"line {source.index + 1}"
+    if source.kind == "stdin":
+        return source.name
+    return source.name
+
+
+def _sniff_json_object(raw: str) -> Mapping[str, object] | None:
+    candidate = raw.lstrip()
+    if not candidate.startswith("{"):
+        return None
+    try:
+        parsed: object = json.loads(candidate)
+    except json.JSONDecodeError:
+        return None
+    if not _is_json_object(parsed):  # pragma: no cover — a parsed "{…}" is always an object
+        return None
+    return dict(parsed)
+
+
+def _is_json_object(value: object) -> TypeGuard[Mapping[str, object]]:
+    """Sound claim: ``json.loads`` produces ``str`` keys by contract."""
+    return isinstance(value, dict)

smartpipe/io/leaderboard.py

@@ -0,0 +1,52 @@
+"""The live top_k leaderboard (stage-08 §4.3): a K-line block repainted in place.
+
+``render_frame`` is pure (goldens pin it); ``LiveBoard`` adds the clock, the
+≤4-repaints/s throttle, and the ANSI cursor-up block rewrite. TTY only — pipe
+mode uses NDJSON snapshots through the ordinary writer instead.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Sequence
+    from typing import TextIO
+
+__all__ = ["LiveBoard", "render_frame"]
+
+_CLEAR_LINE = "\x1b[K"
+_MIN_REPAINT_S = 0.25  # ≤ 4 repaints/s (spec) — dumb full-block rewrite, no partials
+
+
+def render_frame(rows: Sequence[tuple[float, str]], width: int) -> list[str]:
+    """``(score, text)`` rows → display lines, truncated to the terminal width."""
+    lines: list[str] = []
+    for score, text in rows:
+        prefix = f"{score:0.2f}  "
+        budget = max(width - len(prefix), 8)
+        body = text if len(text) <= budget else text[: budget - 1] + "…"
+        lines.append(prefix + body)
+    return lines
+
+
+@dataclass(slots=True)
+class LiveBoard:
+    stream: TextIO
+    width: int
+    clock: Callable[[], float]
+    _painted: int = 0  # lines currently on screen
+    _last: float = field(default=-1.0)
+
+    def paint(self, rows: Sequence[tuple[float, str]], *, force: bool = False) -> None:
+        now = self.clock()
+        if not force and now - self._last < _MIN_REPAINT_S:
+            return
+        self._last = now
+        if self._painted:
+            self.stream.write(f"\x1b[{self._painted}A")  # cursor up over the old block
+        for line in render_frame(rows, self.width):
+            self.stream.write(f"\r{line}{_CLEAR_LINE}\n")
+        self._painted = len(rows)
+        self.stream.flush()

smartpipe/io/metering.py

@@ -0,0 +1,180 @@
+"""Run telemetry (D40): observed units, never estimated dollars.
+
+A module-level, run-scoped meter — the documented diagnostics-style exception
+to no-globals (one verb per process; ``reset()`` at container build and in
+tests). Numbers come from provider usage fields and real byte counts; when a
+wire omits usage, the meter under-counts rather than lies.
+"""
+
+from __future__ import annotations
+
+import io
+import wave
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from smartpipe.models.base import MediaData
+
+__all__ = [
+    "add_conversion",
+    "add_request_media",
+    "add_tokens",
+    "count",
+    "duration",
+    "megabytes",
+    "receipt",
+    "reset",
+    "snapshot",
+    "status_segment",
+]
+
+
+@dataclass(slots=True)
+class _Meter:
+    tokens_in: int = 0
+    tokens_out: int = 0
+    media_bytes: dict[str, int] = field(default_factory=dict[str, int])
+    media_count: dict[str, int] = field(default_factory=dict[str, int])
+    audio_seconds: float = 0.0
+    conversions: int = 0
+
+
+_state = _Meter()
+
+
+def reset() -> None:
+    global _state
+    _state = _Meter()
+
+
+def add_tokens(*, tokens_in: int = 0, tokens_out: int = 0) -> None:
+    _state.tokens_in += max(0, tokens_in)
+    _state.tokens_out += max(0, tokens_out)
+
+
+def add_conversion() -> None:
+    """One PAID conversion (cloud caption/hear/watch/STT) — local whisper is
+    free and uncounted."""
+    _state.conversions += 1
+
+
+def add_request_media(parts: Sequence[MediaData]) -> None:
+    """Meter media actually being SENT (call after pre-send refusals — a
+    refused part costs nothing)."""
+    from smartpipe.models.base import AudioData, ImageData, VideoData
+
+    for part in parts:
+        match part:
+            case ImageData():
+                kind = "image"
+            case AudioData():
+                kind = "audio"
+                _state.audio_seconds += _wav_seconds(part.data, part.mime) or 0.0
+            case VideoData():
+                kind = "video"
+            case _ as unreachable:  # pragma: no cover — the union is closed
+                from typing import assert_never
+
+                assert_never(unreachable)
+        _state.media_bytes[kind] = _state.media_bytes.get(kind, 0) + len(part.data)
+        _state.media_count[kind] = _state.media_count.get(kind, 0) + 1
+
+
+def _wav_seconds(data: bytes, mime: str) -> float | None:
+    if mime not in ("audio/wav", "audio/x-wav"):
+        return None  # other containers need ffprobe; bytes-only is honest enough
+    try:
+        with wave.open(io.BytesIO(data)) as clip:
+            rate = clip.getframerate()
+            return clip.getnframes() / rate if rate else None
+    except (wave.Error, EOFError, OSError, ValueError, RuntimeError):
+        return None  # malformed RIFF — bytes-only is honest enough
+
+
+@dataclass(frozen=True, slots=True)
+class Snapshot:
+    tokens_in: int
+    tokens_out: int
+    media_bytes: dict[str, int]
+    media_count: dict[str, int]
+    audio_seconds: float
+    conversions: int
+
+    @property
+    def empty(self) -> bool:
+        return not (self.tokens_in or self.tokens_out or self.media_bytes or self.conversions)
+
+
+def snapshot() -> Snapshot:
+    return Snapshot(
+        tokens_in=_state.tokens_in,
+        tokens_out=_state.tokens_out,
+        media_bytes=dict(_state.media_bytes),
+        media_count=dict(_state.media_count),
+        audio_seconds=_state.audio_seconds,
+        conversions=_state.conversions,
+    )
+
+
+# --- formatting --------------------------------------------------------------------
+
+
+def count(value: int) -> str:
+    if value >= 1_000_000:
+        return f"{value / 1_000_000:.1f}M"
+    if value >= 1_000:
+        return f"{value / 1_000:.1f}k"
+    return str(value)
+
+
+def megabytes(size: int) -> str:
+    return f"{size / 1_048_576:.1f} MB"
+
+
+def duration(seconds: float) -> str:
+    whole = int(seconds)
+    if whole >= 60:
+        return f"{whole // 60}m{whole % 60:02d}s"
+    return f"{whole}s"
+
+
+def status_segment() -> str:
+    """The live status-line segment; empty string when nothing was consumed."""
+    view = snapshot()
+    if view.empty:
+        return ""
+    pieces = [f"↑{count(view.tokens_in)} ↓{count(view.tokens_out)} tok"]
+    for kind, label in (("image", "img"), ("video", "vid")):
+        size = view.media_bytes.get(kind)
+        if size:
+            pieces.append(f"{megabytes(size)} {label}")
+    if view.media_bytes.get("audio"):
+        held = (
+            duration(view.audio_seconds)
+            if view.audio_seconds
+            else megabytes(view.media_bytes["audio"])
+        )
+        pieces.append(f"{held} audio")
+    return " · ".join(pieces)
+
+
+def receipt() -> str | None:
+    """The end-of-run totals line — the number that goes in the report."""
+    view = snapshot()
+    if view.empty:
+        return None
+    pieces = [f"{count(view.tokens_in)} in · {count(view.tokens_out)} out tokens"]
+    for kind, plural in (("image", "images"), ("video", "video")):
+        size = view.media_bytes.get(kind)
+        if size:
+            pieces.append(f"{megabytes(size)} {plural} ({view.media_count[kind]})")
+    audio_size = view.media_bytes.get("audio")
+    if audio_size:
+        timed = f" · {duration(view.audio_seconds)}" if view.audio_seconds else ""
+        pieces.append(f"{megabytes(audio_size)} audio ({view.media_count['audio']}){timed}")
+    if view.conversions:
+        pieces.append(f"{view.conversions} paid conversions")
+    return "run: " + " · ".join(pieces)