smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/io/tty.py

@@ -0,0 +1,88 @@
+"""Terminal detection and the color decision.
+
+``supports_color`` is pure — the environment is a parameter — so the whole truth
+table is testable. The thin wrappers read real process state at call time.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+from enum import StrEnum
+from typing import TYPE_CHECKING, assert_never
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+__all__ = [
+    "ColorMode",
+    "enable_windows_vt",
+    "stderr_is_tty",
+    "stderr_supports_color",
+    "stdout_is_tty",
+    "supports_color",
+    "terminal_width",
+]
+
+
+class ColorMode(StrEnum):
+    AUTO = "auto"
+    ALWAYS = "always"
+    NEVER = "never"
+
+
+def stdout_is_tty() -> bool:
+    return sys.stdout.isatty()
+
+
+def stderr_is_tty() -> bool:
+    return sys.stderr.isatty()
+
+
+def supports_color(stream_is_tty: bool, *, mode: ColorMode, env: Mapping[str, str]) -> bool:
+    match mode:
+        case ColorMode.ALWAYS:
+            return True
+        case ColorMode.NEVER:
+            return False
+        case ColorMode.AUTO:
+            if not stream_is_tty:
+                return False
+            if "NO_COLOR" in env:  # any value disables — https://no-color.org
+                return False
+            return env.get("TERM") != "dumb"
+        case _ as unreachable:  # pragma: no cover — pyright proves exhaustiveness
+            assert_never(unreachable)
+
+
+def stderr_supports_color(mode: ColorMode = ColorMode.AUTO) -> bool:
+    return supports_color(stderr_is_tty(), mode=mode, env=os.environ)
+
+
+def stdout_supports_color(mode: ColorMode = ColorMode.AUTO) -> bool:
+    return supports_color(stdout_is_tty(), mode=mode, env=os.environ)
+
+
+def terminal_width(default: int = 80) -> int:
+    return shutil.get_terminal_size((default, 24)).columns
+
+
+def enable_windows_vt() -> bool:
+    """Best-effort ANSI enablement on Windows consoles; True on other platforms."""
+    if sys.platform == "win32":
+        import ctypes
+
+        enable_vt = 0x0004
+        kernel32 = ctypes.windll.kernel32
+        succeeded = True
+        for std_handle in (-11, -12):  # stdout, stderr
+            handle = kernel32.GetStdHandle(std_handle)
+            mode = ctypes.c_ulong()
+            if not kernel32.GetConsoleMode(handle, ctypes.byref(mode)):
+                succeeded = False
+                continue
+            if not kernel32.SetConsoleMode(handle, mode.value | enable_vt):
+                succeeded = False
+        return succeeded
+    return True

smartpipe/io/usage.py

@@ -0,0 +1,214 @@
+"""The usage ledger (D41): what the meter observed, remembered over time.
+
+One event per model-touching run, persisted at container exit; windows are
+computed at read time; ``lifetime`` accumulates separately so pruning old
+events loses nothing. Telemetry must never fail a run — every filesystem
+error here is swallowed.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from smartpipe.core.jsontools import as_items, as_record
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from smartpipe.io.metering import Snapshot
+
+__all__ = ["Totals", "read_ledger", "record_run", "reset_ledger", "usage_path"]
+
+_EVENT_HORIZON_DAYS = 32  # windows top out at 30 days; older events prune
+_FIELDS = ("runs", "tokens_in", "tokens_out", "media_bytes", "audio_seconds", "conversions")
+
+
+@dataclass(frozen=True, slots=True)
+class Totals:
+    runs: int = 0
+    tokens_in: int = 0
+    tokens_out: int = 0
+    media_bytes: int = 0
+    audio_seconds: float = 0.0
+    conversions: int = 0
+
+    def plus(self, other: Totals) -> Totals:
+        return Totals(
+            runs=self.runs + other.runs,
+            tokens_in=self.tokens_in + other.tokens_in,
+            tokens_out=self.tokens_out + other.tokens_out,
+            media_bytes=self.media_bytes + other.media_bytes,
+            audio_seconds=self.audio_seconds + other.audio_seconds,
+            conversions=self.conversions + other.conversions,
+        )
+
+
+def usage_path(env: Mapping[str, str]) -> Path:
+    base = env.get("XDG_STATE_HOME", "").strip()
+    root = Path(base) if base else Path.home() / ".local" / "state"
+    return root / "smartpipe" / "usage.json"
+
+
+def record_run(snapshot: Snapshot, env: Mapping[str, str], *, now: float | None = None) -> None:
+    """Persist one run's observed usage. Best-effort: never raises."""
+    if snapshot.empty:
+        return
+    moment = time.time() if now is None else now
+    try:
+        path = usage_path(env)
+        document = _load(path)
+        event = {
+            "ts": moment,
+            "runs": 1,
+            "tokens_in": snapshot.tokens_in,
+            "tokens_out": snapshot.tokens_out,
+            "media_bytes": sum(snapshot.media_bytes.values()),
+            "audio_seconds": snapshot.audio_seconds,
+            "conversions": snapshot.conversions,
+        }
+        horizon = moment - _EVENT_HORIZON_DAYS * 86_400
+        kept = [entry for entry in _document_events(document) if _event_ts(entry) >= horizon]
+        document["events"] = [*kept, event]
+        document["lifetime"] = _totals_dict(
+            _totals_from(_document_lifetime(document)).plus(_totals_from(event))
+        )
+        if document.get("first_seen") is None:
+            document["first_seen"] = moment
+        _store(path, document)
+    except OSError:
+        return
+
+
+def read_ledger(
+    env: Mapping[str, str], *, now: float | None = None
+) -> tuple[dict[str, Totals], float | None, float | None]:
+    """(windows, first_seen, last_reset) — windows keyed hour/day/week/month/lifetime."""
+    moment = time.time() if now is None else now
+    document = _load(usage_path(env))
+    events = _document_events(document)
+    windows: dict[str, Totals] = {}
+    for name, seconds in (
+        ("past hour", 3_600),
+        ("past day", 86_400),
+        ("past week", 7 * 86_400),
+        ("past month", 30 * 86_400),
+    ):
+        recent = [entry for entry in events if _event_ts(entry) >= moment - seconds]
+        totals = Totals()
+        for entry in recent:
+            totals = totals.plus(_totals_from(entry))
+        windows[name] = totals
+    windows["lifetime"] = _totals_from(_document_lifetime(document))
+    first_seen = document.get("first_seen")
+    last_reset = document.get("last_reset")
+    return (
+        windows,
+        first_seen if isinstance(first_seen, (int, float)) else None,
+        last_reset if isinstance(last_reset, (int, float)) else None,
+    )
+
+
+def reset_ledger(env: Mapping[str, str], *, now: float | None = None) -> Totals:
+    """Zero everything, stamp the reset, return the PREVIOUS lifetime."""
+    moment = time.time() if now is None else now
+    path = usage_path(env)
+    document = _load(path)
+    previous = _totals_from(_document_lifetime(document))
+    fresh = _empty_document()
+    fresh["first_seen"] = document.get("first_seen") or moment
+    fresh["last_reset"] = moment
+    _store(path, fresh)
+    return previous
+
+
+def stamp(moment: float) -> str:
+    return datetime.fromtimestamp(moment, tz=UTC).strftime("%Y-%m-%d %H:%M UTC")
+
+
+# --- the file ----------------------------------------------------------------------
+
+
+def _empty_document() -> dict[str, object]:
+    return {
+        "version": 1,
+        "first_seen": None,
+        "last_reset": None,
+        "lifetime": _totals_dict(Totals()),
+        "events": [],
+    }
+
+
+def _load(path: Path) -> dict[str, object]:
+    try:
+        parsed: object = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return _empty_document()
+    record = as_record(parsed)
+    if record is None:
+        return _empty_document()
+    document = _empty_document()
+    document["first_seen"] = record.get("first_seen")
+    document["last_reset"] = record.get("last_reset")
+    lifetime = as_record(record.get("lifetime"))
+    if lifetime is not None:
+        document["lifetime"] = dict(lifetime)
+    events = as_items(record.get("events"))
+    if events is not None:
+        document["events"] = [dict(entry) for item in events if (entry := as_record(item))]
+    return document
+
+
+def _store(path: Path, document: dict[str, object]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    scratch = path.with_suffix(".tmp")
+    scratch.write_text(json.dumps(document), encoding="utf-8")
+    os.replace(scratch, path)
+
+
+def _document_events(document: dict[str, object]) -> list[dict[str, object]]:
+    held = as_items(document.get("events"))
+    if held is None:
+        return []
+    return [dict(entry) for item in held if (entry := as_record(item)) is not None]
+
+
+def _document_lifetime(document: dict[str, object]) -> Mapping[str, object]:
+    held = as_record(document.get("lifetime"))
+    return held if held is not None else {}
+
+
+def _event_ts(entry: Mapping[str, object]) -> float:
+    value = entry.get("ts")
+    return float(value) if isinstance(value, (int, float)) else 0.0
+
+
+def _totals_from(entry: Mapping[str, object]) -> Totals:
+    def _number(key: str) -> float:
+        value = entry.get(key)
+        return float(value) if isinstance(value, (int, float)) else 0.0
+
+    return Totals(
+        runs=int(_number("runs")),
+        tokens_in=int(_number("tokens_in")),
+        tokens_out=int(_number("tokens_out")),
+        media_bytes=int(_number("media_bytes")),
+        audio_seconds=_number("audio_seconds"),
+        conversions=int(_number("conversions")),
+    )
+
+
+def _totals_dict(totals: Totals) -> dict[str, object]:
+    return {
+        "runs": totals.runs,
+        "tokens_in": totals.tokens_in,
+        "tokens_out": totals.tokens_out,
+        "media_bytes": totals.media_bytes,
+        "audio_seconds": totals.audio_seconds,
+        "conversions": totals.conversions,
+    }

smartpipe/io/writers.py

@@ -0,0 +1,340 @@
+"""TTY-adaptive result writers — the only module that writes to stdout.
+
+Two vocabularies on purpose: ``OutputFormat`` is what users say (``--output`` /
+``SMARTPIPE_OUTPUT``); ``RenderMode`` is what a writer does. ``resolve_format``
+maps one to the other using the TTY matrix in plan/ux.md — notably, AUTO on a
+terminal renders structured results as a human view, while an *explicit*
+``--output json`` forces NDJSON even there (spec §5.2).
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import TYPE_CHECKING, Protocol, assert_never
+
+from smartpipe.core.errors import UsageFault
+from smartpipe.core.jsontools import as_record
+from smartpipe.io import diagnostics
+from smartpipe.io.text import clip_to_width, display_width
+
+if TYPE_CHECKING:
+    from typing import TextIO
+
+    from smartpipe.io.items import Item
+
+_TRAILING_COLUMNS = ("_score", "_rank")  # ranking metadata sorts to the right of the sheet
+
+__all__ = [
+    "OutputFormat",
+    "RenderMode",
+    "ResultWriter",
+    "WriterConfig",
+    "make_writer",
+    "resolve_format",
+]
+
+_DIM = "\x1b[2m"
+_RESET = "\x1b[0m"
+_ELLIPSIS = "…"
+
+
+class OutputFormat(StrEnum):
+    AUTO = "auto"
+    TEXT = "text"
+    JSON = "json"
+    CSV = "csv"
+    TSV = "tsv"
+
+
+class RenderMode(StrEnum):
+    TEXT = "text"
+    NDJSON = "ndjson"
+    HUMAN = "human"
+    CSV = "csv"
+    TSV = "tsv"
+
+
+@dataclass(frozen=True, slots=True)
+class WriterConfig:
+    mode: RenderMode
+    color: bool
+    width: int
+    fields: tuple[str, ...] | None = None  # honored from stage 9
+
+
+class ResultWriter(Protocol):
+    def write_text(self, line: str) -> None: ...
+    def write_record(self, record: Mapping[str, object]) -> None: ...
+    def write_passthrough(self, item: Item) -> None: ...
+    def flush(self) -> None: ...
+
+
+def resolve_format(
+    flag: OutputFormat,
+    env: Mapping[str, str],
+    *,
+    stdout_tty: bool,
+    structured: bool,
+    fields: tuple[str, ...] | None = None,
+) -> RenderMode:
+    if fields is not None and not structured:
+        raise UsageFault(
+            "--fields selects columns from structured output\n"
+            "  This run produces plain text — there are no named fields to pick from.\n"
+            '  Add braces to the prompt (e.g. "Extract {name, email}") or pass --schema.'
+        )
+    requested = flag
+    if requested is OutputFormat.AUTO:
+        env_value = env.get("SMARTPIPE_OUTPUT", "")
+        if env_value:
+            try:
+                requested = OutputFormat(env_value)
+            except ValueError:
+                raise UsageFault(
+                    f"SMARTPIPE_OUTPUT={env_value!r} isn't a format smartpipe knows\n"
+                    "  valid values: auto, text, json, csv, tsv"
+                ) from None
+    match requested:
+        case OutputFormat.AUTO:
+            if structured:
+                return RenderMode.HUMAN if stdout_tty else RenderMode.NDJSON
+            return RenderMode.TEXT
+        case OutputFormat.TEXT:
+            return RenderMode.TEXT
+        case OutputFormat.JSON:
+            return RenderMode.NDJSON
+        case OutputFormat.CSV:
+            _require_structured(requested, structured=structured)
+            return RenderMode.CSV
+        case OutputFormat.TSV:
+            _require_structured(requested, structured=structured)
+            return RenderMode.TSV
+        case _ as unreachable:  # pragma: no cover — pyright proves exhaustiveness
+            assert_never(unreachable)
+
+
+def _require_structured(fmt: OutputFormat, *, structured: bool) -> None:
+    if not structured:
+        raise UsageFault(
+            f"--output {fmt.value} needs structured output — a table needs named columns\n"
+            '  add braces to the prompt (e.g. "Extract {name, email}") or pass --schema'
+        )
+
+
+def make_writer(config: WriterConfig, stdout: TextIO) -> ResultWriter:
+    match config.mode:
+        case RenderMode.TEXT:
+            return _TextWriter(stream=stdout, fields=config.fields)
+        case RenderMode.NDJSON:
+            return _NdjsonWriter(stream=stdout, fields=config.fields)
+        case RenderMode.HUMAN:
+            return _HumanWriter(
+                stream=stdout, color=config.color, width=config.width, fields=config.fields
+            )
+        case RenderMode.CSV:
+            return _TableWriter(stream=stdout, delimiter=",", fields=config.fields)
+        case RenderMode.TSV:
+            return _TableWriter(stream=stdout, delimiter="\t", fields=config.fields)
+        case _ as unreachable:  # pragma: no cover — pyright proves exhaustiveness
+            assert_never(unreachable)
+
+
+def _compact_json(value: object) -> str:
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+
+
+_ABSENT = object()  # sentinel: "no such field", distinct from a genuine null
+
+
+def _lookup(record: Mapping[str, object], name: str) -> object:
+    """Exact key first; else a dotted walk into nested objects (join's left.x/right.x)."""
+    if name in record:
+        return record[name]
+    current: object = record
+    for part in name.split("."):
+        narrowed = as_record(current)
+        if narrowed is None or part not in narrowed:
+            return _ABSENT
+        current = narrowed[part]
+    return current
+
+
+def _warn_missing(record: Mapping[str, object], fields: tuple[str, ...], warned: set[str]) -> None:
+    """The one-time heads-up per requested-but-absent field (plan/ux.md, --fields)."""
+    for name in fields:
+        if _lookup(record, name) is _ABSENT and name not in warned:
+            diagnostics.warn(f"--fields: no field {name!r} in the results; emitting null")
+            warned.add(name)
+
+
+def _project(
+    record: Mapping[str, object], fields: tuple[str, ...], warned: set[str]
+) -> dict[str, object]:
+    """Select + order the requested columns; absent ones become null (shape stays stable)."""
+    _warn_missing(record, fields, warned)
+    projected = {name: _lookup(record, name) for name in fields}
+    return {name: (None if value is _ABSENT else value) for name, value in projected.items()}
+
+
+@dataclass(frozen=True, slots=True)
+class _TextWriter:
+    stream: TextIO
+    fields: tuple[str, ...] | None = None  # top_k routes structured records through TEXT
+    warned: set[str] = field(default_factory=set[str])
+
+    def write_text(self, line: str) -> None:
+        self.stream.write(f"{line}\n")
+        self.stream.flush()
+
+    def write_record(self, record: Mapping[str, object]) -> None:
+        if self.fields is not None:
+            record = _project(record, self.fields, self.warned)
+        self.write_text(_compact_json(dict(record)))
+
+    def write_passthrough(self, item: Item) -> None:
+        self.write_text(item.raw)
+
+    def flush(self) -> None:
+        self.stream.flush()
+
+
+@dataclass(frozen=True, slots=True)
+class _NdjsonWriter:
+    stream: TextIO
+    fields: tuple[str, ...] | None = None
+    warned: set[str] = field(default_factory=set[str])
+
+    def write_text(self, line: str) -> None:
+        self.write_record({"result": line})
+
+    def write_record(self, record: Mapping[str, object]) -> None:
+        if self.fields is not None:
+            record = _project(record, self.fields, self.warned)
+        self.stream.write(f"{_compact_json(dict(record))}\n")
+        self.stream.flush()
+
+    def write_passthrough(self, item: Item) -> None:
+        self.stream.write(f"{item.raw}\n")
+        self.stream.flush()
+
+    def flush(self) -> None:
+        self.stream.flush()
+
+
+class _TableWriter:
+    """CSV/TSV — a rectangle is the contract. Columns are fixed by ``--fields`` or the
+    first record; later records fill missing cells empty and drop surprise keys with a
+    one-time warning. Nested values become compact JSON; TSV strips tabs/newlines."""
+
+    def __init__(self, *, stream: TextIO, delimiter: str, fields: tuple[str, ...] | None) -> None:
+        self.stream = stream
+        self.delimiter = delimiter
+        self.fields = fields
+        self.columns: tuple[str, ...] | None = None
+        self.warned: set[str] = set()
+        self.tsv_cleaned = False
+        # excel dialect gives RFC 4180 quoting + CRLF; TSV mirrors it with a tab delimiter
+        self.csv = csv.writer(stream, dialect="excel", delimiter=delimiter)
+
+    def write_text(self, line: str) -> None:
+        # csv is guarded to structured output, but stay valid if a plain result slips in
+        self.write_record({"result": line})
+
+    def write_record(self, record: Mapping[str, object]) -> None:
+        if self.columns is None:
+            self.columns = self._header(record)
+            self.csv.writerow(self.columns)
+        if self.fields is not None:
+            # explicit projection: dropping extras is the point, absence gets the shared warning
+            _warn_missing(record, self.fields, self.warned)
+        else:
+            for key in record:
+                if key not in self.columns and key not in self.warned:
+                    diagnostics.warn(
+                        f"column {key!r} appeared after the header was fixed; "
+                        "use --fields to pin columns"
+                    )
+                    self.warned.add(key)
+        cells = (_lookup(record, column) for column in self.columns)
+        self.csv.writerow([self._cell(None if value is _ABSENT else value) for value in cells])
+        self.stream.flush()
+
+    def write_passthrough(self, item: Item) -> None:  # pragma: no cover — csv is structured-only
+        self.write_text(item.raw)
+
+    def flush(self) -> None:
+        self.stream.flush()
+
+    def _header(self, record: Mapping[str, object]) -> tuple[str, ...]:
+        if self.fields is not None:
+            return self.fields
+        body = [key for key in record if key not in _TRAILING_COLUMNS]
+        trailing = [key for key in _TRAILING_COLUMNS if key in record]
+        return (*body, *trailing)
+
+    def _cell(self, value: object) -> str:
+        text = _scalar(value)
+        if self.delimiter == "\t" and any(ch in text for ch in "\t\n\r"):
+            if not self.tsv_cleaned:
+                diagnostics.warn("replaced tabs/newlines in TSV cells with spaces")
+                self.tsv_cleaned = True
+            text = text.replace("\t", " ").replace("\n", " ").replace("\r", " ")
+        return text
+
+
+def _scalar(value: object) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, int | float):
+        return str(value)
+    return _compact_json(value)  # objects/arrays → compact JSON in one cell
+
+
+@dataclass(frozen=True, slots=True)
+class _HumanWriter:
+    """Structured results as aligned key/value blocks — TTY reading, never parsing.
+
+    Truncation to the terminal width happens here and only here: piped output
+    (the other writers) is never truncated (spec §5.1).
+    """
+
+    stream: TextIO
+    color: bool
+    width: int
+    fields: tuple[str, ...] | None = None
+    warned: set[str] = field(default_factory=set[str])
+
+    def write_text(self, line: str) -> None:
+        self.stream.write(f"{line}\n")
+        self.stream.flush()
+
+    def write_record(self, record: Mapping[str, object]) -> None:
+        if self.fields is not None:
+            record = _project(record, self.fields, self.warned)
+        for key, value in record.items():
+            # null shows as nothing — for humans an absent value reads best as blank
+            rendered = (
+                "" if value is None else value if isinstance(value, str) else _compact_json(value)
+            )
+            # budget in terminal cells, not code points (DEFER-2) — a Wide char is 2
+            available = self.width - display_width(key) - 2
+            if available >= 2 and display_width(rendered) > available:
+                rendered = clip_to_width(rendered, available - 1) + _ELLIPSIS
+            label = f"{_DIM}{key}:{_RESET}" if self.color else f"{key}:"
+            self.stream.write(f"{label} {rendered}\n")
+        self.stream.write("\n")
+        self.stream.flush()
+
+    def write_passthrough(self, item: Item) -> None:
+        self.write_text(item.raw)
+
+    def flush(self) -> None:
+        self.stream.flush()

smartpipe/models/__init__.py

@@ -0,0 +1,5 @@
+"""Model providers: the ChatModel/EmbeddingModel protocols and their adapters."""
+
+from __future__ import annotations
+
+__all__: list[str] = []