smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/cli/root.py

@@ -0,0 +1,281 @@
+"""The smartpipe entry point: root command group and exit-code mapping.
+
+click's built-in usage-error exit code (2) collides with the spec's "no model
+configured" (plan/decisions.md D12), so ``main`` maps click exceptions onto the
+``ExitCode`` contract itself instead of using click's standalone mode.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import os
+import signal
+import sys
+from typing import TYPE_CHECKING
+
+import click
+
+from smartpipe import __version__
+from smartpipe.cli.auth_cmd import auth_command
+from smartpipe.cli.cache_cmd import cache_command
+from smartpipe.cli.chart_cmd import chart_command
+from smartpipe.cli.cite_cmd import cite_command
+from smartpipe.cli.cluster_cmd import cluster_command
+from smartpipe.cli.config_cmd import config_command
+from smartpipe.cli.diff_cmd import diff_command
+from smartpipe.cli.distinct_cmd import distinct_command
+from smartpipe.cli.doctor_cmd import doctor_command
+from smartpipe.cli.echo_cmd import echo_command
+from smartpipe.cli.embed_cmd import embed_command
+from smartpipe.cli.extend_cmd import extend_command
+from smartpipe.cli.filter_cmd import filter_command
+from smartpipe.cli.getschema_cmd import getschema_command
+from smartpipe.cli.join_cmd import join_command
+from smartpipe.cli.map_cmd import map_command
+from smartpipe.cli.outliers_cmd import outliers_command
+from smartpipe.cli.reduce_cmd import reduce_command
+from smartpipe.cli.run_cmd import run_command
+from smartpipe.cli.sample_cmd import sample_command
+from smartpipe.cli.schema_cmd import schema_command
+from smartpipe.cli.screens import WELCOME
+from smartpipe.cli.sort_cmd import sort_command
+from smartpipe.cli.split_cmd import split_command
+from smartpipe.cli.summarize_cmd import summarize_command
+from smartpipe.cli.top_k_cmd import top_k_command
+from smartpipe.cli.usage_cmd import usage_command
+from smartpipe.cli.where_cmd import where_command
+from smartpipe.core.errors import ExitCode, SempipeError, UsageFault
+
+if True:  # typing-only import kept runtime-cheap
+    from collections.abc import Iterable
+from smartpipe.io import diagnostics
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+__all__ = ["cli", "main"]
+
+
+_ALIASES = {"top-k": "top_k", "topk": "top_k"}
+
+
+class _StyledHelpFormatter(click.HelpFormatter):
+    """Color in --help (D42): headings cyan, commands/options green. ANSI is
+    stripped by click when piped; NO_COLOR turns it off entirely."""
+
+    @staticmethod
+    def _on() -> bool:
+        return sys.stdout.isatty() and not os.environ.get("NO_COLOR")
+
+    def write_usage(self, prog: str, args: str = "", prefix: str | None = None) -> None:
+        if self._on():
+            prog = click.style(prog, bold=True)
+            prefix = click.style(prefix if prefix is not None else "Usage: ", fg="cyan", bold=True)
+        super().write_usage(prog, args, prefix)
+
+    def write_heading(self, heading: str) -> None:
+        if self._on():
+            heading = click.style(heading, fg="cyan", bold=True)
+        super().write_heading(heading)
+
+    def write_dl(
+        self,
+        rows: Iterable[tuple[str, str]],
+        col_max: int = 30,
+        col_spacing: int = 2,
+    ) -> None:
+        if self._on():
+            rows = [(click.style(term, fg="green"), body) for term, body in rows]
+        super().write_dl(rows, col_max, col_spacing)
+
+
+class _StyledContext(click.Context):
+    formatter_class = _StyledHelpFormatter
+
+
+class _RootGroup(click.Group):
+    """Print the welcome screen when invoked bare (plan/ux.md, spec §14)."""
+
+    def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
+        if not args:
+            plain = bool(os.environ.get("NO_COLOR"))
+            click.echo(WELCOME, nl=False, color=False if plain else None)
+            ctx.exit(int(ExitCode.OK))
+        return super().parse_args(ctx, args)
+
+    def get_command(self, ctx: click.Context, cmd_name: str) -> click.Command | None:
+        # Muscle-memory forgiveness for top_k's spelling; not shown in help.
+        built_in = super().get_command(ctx, _ALIASES.get(cmd_name, cmd_name))
+        if built_in is not None:
+            return built_in  # built-ins always win — custom verbs never shadow
+        return _user_sem_command(cmd_name) or _entry_point_command(cmd_name)
+
+    def list_commands(self, ctx: click.Context) -> list[str]:
+        names = set(super().list_commands(ctx))
+        names.update(_user_sem_names())
+        names.update(_entry_point_names())
+        return sorted(names)
+
+
+def _verbs_dir() -> Path:
+    from smartpipe.config.paths import config_path
+
+    return config_path(os.environ).parent / "verbs"
+
+
+def _user_sem_names() -> list[str]:
+    try:
+        return [path.stem for path in _verbs_dir().glob("*.sem")]
+    except OSError:
+        return []
+
+
+def _user_sem_command(name: str) -> click.Command | None:
+    """Leg 1 of the custom-verb contract (D39/06): ~/.config/smartpipe/verbs/
+    NAME.sem becomes `smartpipe NAME` — full key validation, zero machinery."""
+    script = _verbs_dir() / f"{name}.sem"
+    if not script.is_file():
+        return None
+
+    @click.command(name=name, help=f"Custom verb from {script} (.sem file).")
+    def sem_verb() -> None:
+        from smartpipe.cli.run_cmd import execute_script
+
+        execute_script(script)
+
+    return sem_verb
+
+
+def _entry_point_names() -> list[str]:
+    from importlib.metadata import entry_points
+
+    try:
+        return [point.name for point in entry_points(group="smartpipe.verbs")]
+    except Exception:
+        return []
+
+
+def _entry_point_command(name: str) -> click.Command | None:
+    """Leg 2 (D39/06): a package's entry point in group `smartpipe.verbs` naming
+    a click.Command. Broken plugins warn once and are skipped."""
+    from importlib.metadata import entry_points
+
+    try:
+        matches = [point for point in entry_points(group="smartpipe.verbs") if point.name == name]
+    except Exception:
+        return None
+    for point in matches:
+        try:
+            loaded: object = point.load()
+        except Exception as exc:
+            diagnostics.warn(f"custom verb {name!r} failed to load ({exc}) — skipped")
+            return None
+        if isinstance(loaded, click.Command):
+            return loaded
+        diagnostics.warn(f"custom verb {name!r} is not a click.Command — skipped")
+    return None
+
+
+@click.group(cls=_RootGroup)
+@click.version_option(__version__, prog_name="smartpipe", message="%(prog)s %(version)s")
+def cli() -> None:
+    """smartpipe — semantic pipes for your terminal: documents, images, audio, video, text."""
+
+
+cli.add_command(map_command)
+cli.add_command(extend_command)
+cli.add_command(filter_command)
+cli.add_command(embed_command)
+cli.add_command(top_k_command)
+cli.add_command(reduce_command)
+cli.add_command(join_command)
+cli.add_command(cluster_command)
+cli.add_command(diff_command)
+cli.add_command(distinct_command)
+cli.add_command(outliers_command)
+cli.add_command(run_command)
+cli.add_command(config_command)
+cli.add_command(doctor_command)
+cli.add_command(schema_command)
+cli.add_command(split_command)
+cli.add_command(chart_command)
+cli.add_command(where_command)
+cli.add_command(summarize_command)
+cli.add_command(sample_command)
+cli.add_command(getschema_command)
+cli.add_command(sort_command)
+cli.add_command(auth_command)
+cli.add_command(cache_command)
+cli.add_command(usage_command)
+cli.add_command(cite_command)
+cli.add_command(echo_command)
+
+
+def _stylize(command: click.Command) -> None:
+    command.context_class = _StyledContext
+    if isinstance(command, click.Group):
+        for sub in command.commands.values():
+            _stylize(sub)
+
+
+_stylize(cli)
+
+
+def _stdout_is_real() -> bool:
+    """True when stdout is an OS-level stream (tty or pipe), not a capture."""
+    try:
+        return sys.stdout.fileno() >= 0
+    except (OSError, ValueError):  # capture objects raise UnsupportedOperation (a ValueError)
+        return False
+
+
+def main() -> None:
+    # standalone_mode=False so *we* own exit codes. In this mode click does not
+    # sys.exit(): a ctx.exit(n) / --version / --help comes back as a plain int
+    # return value (verified against click 8.4), and UsageError is raised.
+    # --debug becomes a real global flag with the first verb (stage 3); until then
+    # the env var keeps tracebacks reachable for development.
+    if hasattr(signal, "SIGPIPE") and _stdout_is_real():
+        # POSIX: when downstream closes, die exactly like grep. Guarded to
+        # REAL stdout streams — under a test harness's capture there is no
+        # pipe to honor, and SIG_DFL process-wide lets any stray SIGPIPE
+        # (worker threads, GC'd pipes) kill the harness itself (seen only
+        # on the GitHub runner: exit 141 at a moving test).
+        signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+    debug = "SMARTPIPE_DEBUG" in os.environ
+    try:
+        result = cli.main(
+            standalone_mode=False,
+            prog_name="smartpipe",
+            # click glob-expands wildcard args itself on windows - that would
+            # pre-explode a quoted --in '*.txt' into positionals and break the
+            # "we expand our own globs" contract (D43-era; caught by CI round 10)
+            windows_expand_args=False,
+        )
+    except BrokenPipeError:  # Windows / buffered-flush edge; POSIX rarely reaches here
+        with contextlib.suppress(OSError, ValueError):  # silence the shutdown flush
+            os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
+        raise SystemExit(int(ExitCode.PIPE_CLOSED)) from None
+    except click.UsageError as exc:
+        prefix = (
+            click.style("error:", fg="red")
+            if sys.stderr.isatty() and not os.environ.get("NO_COLOR")
+            else "error:"
+        )
+        click.echo(f"{prefix} {exc.format_message()}", err=True)
+        command_path = exc.ctx.command_path if exc.ctx is not None else "smartpipe"
+        click.echo(f"  try: {command_path} --help", err=True)
+        raise SystemExit(int(ExitCode.USAGE)) from exc
+    except SempipeError as exc:
+        diagnostics.die(exc, debug=debug)
+    except KeyboardInterrupt:
+        raise SystemExit(int(ExitCode.INTERRUPTED)) from None
+    except asyncio.CancelledError:  # the drain watchdog cancelled the run (ux.md §12)
+        raise SystemExit(int(ExitCode.INTERRUPTED)) from None
+    except click.ClickException as exc:  # click-internal faults (e.g. click.FileError)
+        diagnostics.die(UsageFault(exc.format_message()), debug=debug)
+    except Exception as exc:  # the last-resort BUG screen (exit 70) — never a raw traceback
+        diagnostics.internal_error(exc, debug=debug)
+    if isinstance(result, int) and result != 0:
+        raise SystemExit(result)

smartpipe/cli/run_cmd.py

@@ -0,0 +1,136 @@
+"""``smartpipe run`` — execute a ``.sem`` file: one stage, or a pipeline (D17/D38-14).
+
+Single-stage files trampoline into their verb unchanged. Pipeline files
+([stage.NAME] tables) run stages sequentially in-process: each stage's stdout
+feeds the next (or a named earlier stage), the last stage writes the real
+stdout, and every stage's stderr is prefixed with its name. ``--dry-run``
+prints the resolved graph with each stage's cost posture and runs nothing —
+D18 at pipeline scale.
+"""
+
+from __future__ import annotations
+
+import io
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import click
+
+from smartpipe.cli.sem_file import parse_pipeline, parse_sem
+
+if TYPE_CHECKING:
+    from smartpipe.cli.sem_file import Stage
+
+__all__ = ["execute_script", "run_command"]
+
+_FREE_VERBS = frozenset({"where", "split", "chart", "sort", "sample", "summarize", "getschema"})
+_EMBED_VERBS = frozenset({"embed", "distinct", "outliers"})
+
+
+def _posture(verb: str) -> str:
+    if verb in _FREE_VERBS:
+        return "free"
+    if verb in _EMBED_VERBS:
+        return "embeddings"
+    return "model calls"
+
+
+class _PrefixedStderr(io.TextIOBase):
+    """Line-prefixes a stage's stderr so interleaved receipts stay readable."""
+
+    def __init__(self, prefix: str, target: object) -> None:
+        self.prefix = prefix
+        self.target = target
+        self._at_line_start = True
+
+    def write(self, text: str) -> int:
+        for line in text.splitlines(keepends=True):
+            if self._at_line_start and line.strip():
+                self.target.write(self.prefix)  # type: ignore[attr-defined]
+            self.target.write(line)  # type: ignore[attr-defined]
+            self._at_line_start = line.endswith("\n")
+        return len(text)
+
+    def flush(self) -> None:
+        self.target.flush()  # type: ignore[attr-defined]
+
+    def isatty(self) -> bool:
+        return False  # stage receipts never animate spinners
+
+
+@click.command(name="run", context_settings={"ignore_unknown_options": True})
+@click.argument("script", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option("--dry-run", "dry_run", is_flag=True, help="Print the pipeline graph; run nothing.")
+@click.argument("extra", nargs=-1, type=click.UNPROCESSED)
+def run_command(script: Path, dry_run: bool, extra: tuple[str, ...]) -> None:
+    """Execute a .sem file — one stage, or a whole pipeline.
+
+    \b
+    Examples:
+      smartpipe run extract.sem < cards.txt
+      smartpipe run triage.sem --dry-run        # the graph + cost posture, zero calls
+      cat tickets.log | smartpipe run triage.sem > report.txt
+
+    A single-stage file is TOML pinning one verb invocation. A pipeline file
+    holds [stage.NAME] tables run in order — each stage reads the previous
+    stage's output (or 'input = "name"' picks an earlier one); the first
+    reads stdin, the last writes stdout. Extra flags apply to single-stage
+    files only.
+    """
+    execute_script(script, extra=extra, dry_run=dry_run)
+
+
+def execute_script(script: Path, *, extra: tuple[str, ...] = (), dry_run: bool = False) -> None:
+    """Run a .sem file (single stage or pipeline) — shared by ``smartpipe run``
+    and user-named custom verbs (D39/06)."""
+    stages = parse_pipeline(script)
+    if stages is None:
+        if dry_run:
+            argv = parse_sem(script)
+            click.echo(f"{script.name}: {' '.join(argv)}   [{_posture(argv[0])}]")
+            return
+        _invoke(parse_sem(script) + list(extra))
+        return
+    if extra:
+        raise click.UsageError("extra flags apply to single-stage files only")
+    if dry_run:
+        for stage in stages:
+            click.echo(
+                f"stage {stage.name:<12} {' '.join(stage.argv)}   [{_posture(stage.argv[0])}]"
+            )
+        return
+    _run_pipeline(stages)
+
+
+def _invoke(argv: list[str]) -> None:
+    context = click.get_current_context()
+    root = context.find_root().command
+    assert isinstance(root, click.Group)  # run is only ever registered on the group
+    verb = root.get_command(context, argv[0])
+    assert verb is not None  # the translator only emits real verbs
+    sub = verb.make_context(argv[0], list(argv[1:]), parent=context, ignore_unknown_options=False)
+    with sub:
+        verb.invoke(sub)
+
+
+def _run_pipeline(stages: tuple[Stage, ...]) -> None:
+    outputs: dict[str, str] = {}
+    previous: str | None = None
+    real_stdin, real_stdout, real_stderr = sys.stdin, sys.stdout, sys.stderr
+    for position, stage in enumerate(stages):
+        last = position == len(stages) - 1
+        source = stage.input_name if stage.input_name is not None else previous
+        stage_in = real_stdin if source is None else io.StringIO(outputs[source])
+        stage_out = real_stdout if last else io.StringIO()
+        sys.stdin = stage_in  # type: ignore[assignment]
+        sys.stdout = stage_out  # type: ignore[assignment]
+        sys.stderr = _PrefixedStderr(f"[{stage.name}] ", real_stderr)  # type: ignore[assignment]
+        try:
+            _invoke(list(stage.argv))
+        finally:
+            sys.stdin, sys.stdout, sys.stderr = real_stdin, real_stdout, real_stderr
+        if not last:
+            assert isinstance(stage_out, io.StringIO)
+            outputs[stage.name] = stage_out.getvalue()
+        previous = stage.name

smartpipe/cli/sample_cmd.py

@@ -0,0 +1,35 @@
+"""``smartpipe sample`` — the same N random rows, every run."""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.sample import SampleRequest, run_sample
+
+__all__ = ["sample_command"]
+
+
+@click.command(name="sample")
+@click.argument("count", type=int)
+@click.option(
+    "--seed", type=int, default=0, show_default=True, help="Vary the (still reproducible) sample."
+)
+def sample_command(count: int, seed: int) -> None:
+    """Keep N random rows — seeded, reproducible. Free — never calls a model.
+
+    \b
+    Examples:
+      cat huge.jsonl | smartpipe sample 20 | smartpipe map "Extract {label}"
+      cat evals.jsonl | smartpipe sample 50 --seed 7 > eval-subset.jsonl
+
+    Deterministic BY DEFAULT: the same input gives the same sample with no
+    flags, so prompt comparisons compare prompts (and the sample is citable).
+    Unlike --max-calls (which takes the head of the stream), sample is
+    representative. Output keeps input order.
+    """
+    code = run_sample(SampleRequest(count=count, seed=seed), stdin=sys.stdin, stdout=sys.stdout)
+    if code is not ExitCode.OK:  # pragma: no cover — sample always OKs
+        raise SystemExit(int(code))

smartpipe/cli/schema_cmd.py

@@ -0,0 +1,75 @@
+"""``smartpipe schema`` — rung 4 of the ladder (D22): English → a validated schema file.
+
+Exactly one drafting call plus at most one repair. The draft is validated against
+the JSON-Schema meta-schema before stdout sees a byte: an invalid draft is exit 3
+with the attempt on stderr and **stdout empty** — a broken schema silently piped
+into the next run is the disaster case this command exists to prevent.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+
+import click
+
+from smartpipe.cli.completions import complete_chat_models
+from smartpipe.core.errors import ExitCode, ItemError
+from smartpipe.engine.prompts import build_repair_request, build_schema_request
+from smartpipe.engine.schema import parse_schema_draft
+from smartpipe.io import diagnostics
+
+__all__ = ["schema_command"]
+
+_FAILED_SCREEN = (
+    "error: the model couldn't produce a valid JSON Schema (after one repair)\n"
+    "  Its last attempt is above on stderr. Try rephrasing, or write the file by hand:\n"
+    "  docs/concepts/structured-output.md"
+)
+
+
+@click.command(name="schema")
+@click.argument("description")
+@click.option(
+    "--model",
+    "model_flag",
+    shell_complete=complete_chat_models,
+    help="Model to draft with (one call + at most one repair).",
+)
+def schema_command(description: str, model_flag: str | None) -> None:
+    """Draft a JSON Schema from an English description, validated before output.
+
+    \b
+    Examples:
+      smartpipe schema "invoice with vendor string, total number, status paid/unpaid" > invoice.json
+      cat receipts.txt | smartpipe map "Extract the fields" --schema invoice.json
+
+    The draft is checked against the JSON-Schema meta-schema; a failed draft
+    exits 3 with NOTHING on stdout, so a broken schema can never slip into a pipe.
+    """
+    code = asyncio.run(_run(description, model_flag))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(description: str, model_flag: str | None) -> ExitCode:
+    from smartpipe.container import build_container
+
+    async with build_container(os.environ) as container:
+        model = await container.chat_model(model_flag)
+        request = build_schema_request(description)
+        reply = await model.complete(request)
+        try:
+            draft = parse_schema_draft(reply)
+        except ItemError as first_error:
+            repair = build_repair_request(request, bad_reply=reply, error=str(first_error))
+            reply = await model.complete(repair)
+            try:
+                draft = parse_schema_draft(reply)
+            except ItemError:
+                diagnostics.warn(f"the model's last draft attempt:\n{reply}")
+                diagnostics.report_error(_FAILED_SCREEN)
+                return ExitCode.ALL_FAILED
+    click.echo(json.dumps(draft, indent=2, ensure_ascii=False))
+    return ExitCode.OK