smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/cli/outliers_cmd.py

@@ -0,0 +1,82 @@
+"""``smartpipe outliers`` — the items least like the rest."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+
+import click
+
+from smartpipe.cli.completions import complete_embed_models
+from smartpipe.cli.input_options import input_options, input_spec
+from smartpipe.cli.interrupts import graceful_interrupts, settle_budget
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.outliers import OutliersRequest, run_outliers
+
+__all__ = ["outliers_command"]
+
+
+@click.command(name="outliers")
+@click.argument("count", type=int, default=5)
+@click.option(
+    "--embed-model",
+    "model_flag",
+    shell_complete=complete_embed_models,
+    help="Embedding model for this run.",
+)
+@click.option("--concurrency", "concurrency_flag", type=int, help="Max parallel model calls.")
+@click.option("--max-calls", "max_calls", type=int, help="Stop after N model calls (cost cap).")
+@click.option(
+    "--allow-captions",
+    "allow_captions",
+    is_flag=True,
+    help="Let a CLOUD model convert images/audio to text (paid; local models do it free).",
+)
+@input_options
+def outliers_command(
+    count: int,
+    model_flag: str | None,
+    concurrency_flag: int | None,
+    max_calls: int | None,
+    allow_captions: bool,
+    in_patterns: tuple[str, ...],
+    from_files: bool,
+) -> None:
+    """Rank the N items least like the rest — novelty, surfaced.
+
+    \b
+    Examples:
+      cat today.log | smartpipe outliers 5        # the failure shapes you HAVEN'T seen
+      cat train.jsonl | smartpipe outliers 20     # label noise and template glitches
+
+    top_k's mirror: farthest from everything instead of nearest to a query.
+    Embeddings only — no chat calls. Each row carries _distance, and the
+    stderr line anchors it ("median neighbor distance 0.21 — these are
+    3.1x-3.9x out") so the score means something.
+    """
+    request = OutliersRequest(
+        count=count,
+        model_flag=model_flag,
+        concurrency_flag=concurrency_flag,
+        allow_captions=allow_captions,
+        input=input_spec(in_patterns, from_files=from_files),
+    )
+    code = asyncio.run(_run(request, max_calls))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(request: OutliersRequest, max_calls: int | None) -> ExitCode:
+    from dataclasses import replace
+
+    from smartpipe.container import build_container
+
+    async with (
+        graceful_interrupts() as stop,
+        build_container(os.environ, max_calls=max_calls, stop=stop) as container,
+    ):
+        if not request.allow_captions and container.config.allow_captions:
+            request = replace(request, allow_captions=True)  # profile consent (D35)
+        code = await run_outliers(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)
+        return settle_budget(container.budget, code)

smartpipe/cli/probe_cmd.py

@@ -0,0 +1,223 @@
+"""``doctor --probe`` (D31/D42): the modality matrix, with real tiny calls.
+
+``doctor`` alone never spends a cent (D18); this flag is the explicit opt-in
+that answers what the docs can only claim: which modalities *actually* reach
+your configured models. Marks: check = native; dash+star = works via a
+footnote names it); cross = no path. Four tiny paid calls, announced first.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from importlib import resources
+from typing import TYPE_CHECKING
+
+from smartpipe.core.errors import SempipeError
+from smartpipe.io import diagnostics
+from smartpipe.models.base import (
+    AudioData,
+    CompletionRequest,
+    ImageData,
+    supports_media_embedding,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from smartpipe.models.base import ChatModel, EmbeddingModel
+
+__all__ = ["render_matrix", "run_probe"]
+
+
+@dataclass(frozen=True, slots=True)
+class Cell:
+    verdict: str  # "ok" | "no" | "via" (fallback) | "na"
+    detail: str
+    footnote: str | None = None  # what the * means, when verdict == "via"
+
+
+def _asset(name: str) -> bytes:
+    return (resources.files("smartpipe.assets") / name).read_bytes()
+
+
+async def run_probe(env: Mapping[str, str]) -> str:
+    """Build the matrix against the configured chat + embed models."""
+    import os
+
+    from smartpipe.container import build_container
+
+    del env  # the container reads the process environment itself
+    async with build_container(os.environ) as container:
+        chat = await container.chat_model()
+        embed = await container.embedding_model()
+        diagnostics.note(
+            "probing modalities with 4 tiny calls "
+            f"(chat: {chat.ref.name} · embed: {embed.ref.name})"
+        )
+        stt = _stt_path(os.environ, container.config.stt_model)
+        chat_image = await _chat_image(chat)
+        chat_audio = await _chat_audio(chat, stt)
+        rows = {
+            "text": (await _chat_text(chat), await _embed_text(embed)),
+            "image": (chat_image, _embed_image(embed, chat_image)),
+            "audio": (chat_audio, _embed_audio(chat_audio, stt)),
+            "video": (_chat_video(chat), _embed_video()),
+            "document": (
+                Cell("ok", "parsed locally (no call)"),
+                Cell("ok", "as extracted text"),
+            ),
+        }
+    return render_matrix(rows)
+
+
+def _stt_path(env: Mapping[str, str], configured: str | None) -> str | None:
+    """The transcription path the ladder would take, if any (D39/05)."""
+    named = env.get("SMARTPIPE_STT_MODEL", "").strip() or (configured or "")
+    if named:
+        return named
+    if env.get("OPENAI_API_KEY", "").strip():
+        return "openai/whisper-1 (auto)"
+    from importlib.util import find_spec
+
+    if find_spec("faster_whisper") is not None:
+        return "local whisper"
+    return None
+
+
+async def _chat_text(chat: ChatModel) -> Cell:
+    try:
+        reply = await chat.complete(
+            CompletionRequest(system=None, user="Reply with exactly: OK", max_tokens=8)
+        )
+        return Cell("ok", f"replied {reply.strip()[:16]!r}")
+    except SempipeError as exc:
+        return Cell("no", _first_line(exc))
+
+
+async def _chat_image(chat: ChatModel) -> Cell:
+    try:
+        reply = await chat.complete(
+            CompletionRequest(
+                system=None,
+                user="One word: what color dominates this image?",
+                media=(ImageData(_asset("probe.png"), "image/png"),),
+                max_tokens=8,
+            )
+        )
+        return Cell("ok", f"saw it — {reply.strip()[:16]!r}")
+    except SempipeError:
+        return Cell("no", "this model can't see images")
+
+
+async def _chat_audio(chat: ChatModel, stt: str | None) -> Cell:
+    try:
+        reply = await chat.complete(
+            CompletionRequest(
+                system=None,
+                user="One word: is this sound a tone or speech?",
+                media=(AudioData(_asset("probe.wav"), "audio/wav"),),
+                max_tokens=8,
+            )
+        )
+        return Cell("ok", f"heard it — {reply.strip()[:16]!r}")
+    except SempipeError:
+        if stt is not None:
+            return Cell("via", "transcribed, then chat", footnote=f"audio → {stt}")
+        return Cell("no", "no transcription path — reinstall smartpipe")
+
+
+async def _embed_text(embed: EmbeddingModel) -> Cell:
+    try:
+        vectors = await embed.embed([_asset("probe.txt").decode("utf-8").strip()])
+        return Cell("ok", f"{len(vectors[0])}-dim vector")
+    except SempipeError as exc:
+        return Cell("no", _first_line(exc))
+
+
+def _embed_image(embed: EmbeddingModel, chat_image: Cell) -> Cell:
+    if supports_media_embedding(embed):
+        return Cell("ok", "embedded as pixels")
+    if chat_image.verdict == "ok":
+        return Cell("via", "caption, then embed", footnote="image → caption pivot (D33)")
+    return Cell("no", "needs a vision chat model to caption")
+
+
+def _embed_audio(chat_audio: Cell, stt: str | None) -> Cell:
+    if chat_audio.verdict == "ok" or stt is not None:
+        return Cell("via", "transcript, then embed", footnote="audio → transcript pivot")
+    return Cell("no", "no transcription path")
+
+
+def _chat_video(chat: ChatModel) -> Cell:
+    if chat.ref.provider == "gemini":
+        return Cell("ok", "watched natively")
+    try:
+        from smartpipe.parsing.extract import ffmpeg_exe
+
+        ffmpeg_exe()
+        return Cell("via", "frames + audio track", footnote="video → 1 fps frames + track")
+    except SempipeError:
+        return Cell("no", "ffmpeg unavailable — reinstall smartpipe")
+
+
+def _embed_video() -> Cell:
+    return Cell("via", "halves, then embed", footnote="video → visual+speech halves (D36)")
+
+
+def _first_line(exc: SempipeError) -> str:
+    return str(exc).splitlines()[0].removeprefix("error: ")
+
+
+_MARKS = {"ok": "✓ ", "no": "✗ ", "via": "–*", "na": "– "}  # noqa: RUF001 — pinned marks
+
+
+def render_matrix(rows: Mapping[str, tuple[Cell, Cell]]) -> str:
+    """Aligned by VISIBLE width, cells truncated to the grid — overflow never
+    smashes columns (D42; live-caught by the owner's screenshot)."""
+    from smartpipe.cli.screens import bad, good, heading, tint
+
+    label_width = max(len(name) for name in rows) + 2
+    cap = 30
+
+    def cell_text(cell: Cell) -> str:
+        detail = cell.detail if len(cell.detail) <= cap else cell.detail[: cap - 1] + "…"
+        return f"{_MARKS[cell.verdict]} {detail}"
+
+    def paint(cell: Cell, text: str) -> str:
+        mark_len = 2
+        mark, rest = text[:mark_len], text[mark_len:]
+        match cell.verdict:
+            case "ok":
+                return good(mark) + rest
+            case "no":
+                return bad(mark) + rest
+            case _:
+                return tint(mark, "2") + rest
+
+    chat_width = max(len(cell_text(chat)) for chat, _embed in rows.values()) + 3
+    header = f"  {' ' * label_width}{heading(_pad_plain('chat', chat_width))}{heading('embed')}"
+    lines = [header]
+    footnotes: list[str] = []
+    for modality, (chat_cell, embed_cell) in rows.items():
+        left = paint(chat_cell, cell_text(chat_cell))
+        right = paint(embed_cell, cell_text(embed_cell))
+        label = tint(modality.ljust(label_width), "2")
+        lines.append(f"  {label}{_pad_ansi(left, chat_width)}{right}")
+        for cell in (chat_cell, embed_cell):
+            if cell.footnote and cell.footnote not in footnotes:
+                footnotes.append(cell.footnote)
+    if footnotes:
+        lines.append(tint("  * fallback paths: " + " · ".join(footnotes), "2"))
+    return "\n".join(lines)
+
+
+def _pad_plain(text: str, width: int) -> str:
+    return f"{text:{width}s}"
+
+
+def _pad_ansi(text: str, width: int) -> str:
+    """Pad by VISIBLE length — ANSI escapes are zero-width."""
+    import re
+
+    visible = len(re.sub(r"\x1b\[[0-9;]*m", "", text))
+    return text + " " * max(0, width - visible)

smartpipe/cli/reduce_cmd.py

@@ -0,0 +1,129 @@
+"""``smartpipe reduce`` — synthesize many items into one."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+import click
+
+from smartpipe.cli.completions import complete_chat_models
+from smartpipe.cli.input_options import (
+    fields_option,
+    input_options,
+    input_spec,
+    resolve_prompt,
+)
+from smartpipe.cli.interrupts import graceful_interrupts, settle_budget
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.reduce import ReduceRequest, run_reduce
+
+__all__ = ["reduce_command"]
+
+
+@click.command(name="reduce")
+@click.argument("prompt", required=False)
+@click.option(
+    "--prompt-file",
+    "prompt_file",
+    type=click.Path(path_type=Path),
+    help="Read the prompt from a file (the @file shorthand does the same).",
+)
+@click.option(
+    "--schema-from",
+    "schema_dsl",
+    metavar="DSL",
+    help='Build the schema from a short DSL: "vendor string; total number >= 0".',
+)
+@click.option(
+    "--schema",
+    "schema_path",
+    type=click.Path(path_type=Path),
+    help="Shape the final result with a JSON Schema.",
+)
+@click.option("--group-by", "group_by", help="Reduce per group (by an input JSON field).")
+@click.option(
+    "--model", "model_flag", shell_complete=complete_chat_models, help="Model for this run."
+)
+@click.option("--concurrency", "concurrency_flag", type=int, help="Max parallel model calls.")
+@click.option("--max-calls", "max_calls", type=int, help="Stop after N model calls (cost cap).")
+@click.option("--verbose", is_flag=True, help="Show the chunking tree on stderr.")
+@click.option("--window", type=int, help="Stream mode: reduce every N lines (tumbling).")
+@click.option("--every", type=int, help="With --window: slide, reducing after every M lines.")
+@fields_option
+@click.option(
+    "--allow-captions",
+    "allow_captions",
+    is_flag=True,
+    help="Let a CLOUD model convert images/audio to text (paid; local models do it free).",
+)
+@input_options
+def reduce_command(
+    prompt: str | None,
+    prompt_file: Path | None,
+    schema_path: Path | None,
+    schema_dsl: str | None,
+    group_by: str | None,
+    model_flag: str | None,
+    concurrency_flag: int | None,
+    max_calls: int | None,
+    allow_captions: bool,
+    verbose: bool,
+    window: int | None,
+    every: int | None,
+    fields: tuple[str, ...] | None,
+    in_patterns: tuple[str, ...],
+    from_files: bool,
+) -> None:
+    """Synthesize all input items into a single result.
+
+    \b
+    Examples:
+      smartpipe reduce "Write a one-page executive summary" --in 'notes/*.md'
+      cat reports.jsonl | smartpipe reduce "Write a root-cause analysis" --schema rca.json
+      cat feedback.jsonl | smartpipe reduce "Summarize sentiment" --group-by product
+
+    When the input is too large for the model, smartpipe chunks it and recursively
+    summarizes — automatically. Add --verbose to see the chunking tree.
+    """
+    request = ReduceRequest(
+        allow_captions=allow_captions,
+        prompt=resolve_prompt(prompt, prompt_file),
+        schema_path=schema_path,
+        schema_dsl=schema_dsl,
+        group_by=group_by,
+        model_flag=model_flag,
+        concurrency_flag=concurrency_flag,
+        verbose=verbose,
+        window=window,
+        every=every,
+        input=input_spec(in_patterns, from_files=from_files),
+        fields=fields,
+    )
+    code = asyncio.run(_run(request, max_calls))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(request: ReduceRequest, max_calls: int | None) -> ExitCode:
+    from smartpipe.container import build_container
+
+    if request.window is None:  # whole-set mode: ^C exits immediately; budget is fatal (D18)
+        async with build_container(os.environ, max_calls=max_calls) as container:
+            if not request.allow_captions and container.config.allow_captions:
+                from dataclasses import replace as _replace
+
+                request = _replace(request, allow_captions=True)  # profile consent (D35)
+            return await run_reduce(request, container, stdin=sys.stdin, stdout=sys.stdout)
+    async with (  # stream mode drains + flushes partial
+        graceful_interrupts() as stop,
+        build_container(os.environ, max_calls=max_calls, stop=stop) as container,
+    ):
+        if not request.allow_captions and container.config.allow_captions:
+            from dataclasses import replace as _replace
+
+            request = _replace(request, allow_captions=True)  # profile consent (D35)
+        code = await run_reduce(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)
+        return settle_budget(container.budget, code)