smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/cli/filter_cmd.py

@@ -0,0 +1,87 @@
+"""``smartpipe filter`` — keep items matching a semantic condition."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+import click
+
+from smartpipe.cli.input_options import input_options, input_spec, resolve_prompt
+from smartpipe.cli.interrupts import graceful_interrupts, settle_budget
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.filter import FilterRequest, run_filter
+
+__all__ = ["filter_command"]
+
+
+@click.command(name="filter")
+@click.argument("condition", required=False)
+@click.option(
+    "--prompt-file",
+    "prompt_file",
+    type=click.Path(path_type=Path),
+    help="Read the condition from a file (the @file shorthand does the same).",
+)
+@click.option("--not", "invert", is_flag=True, help="Keep items that do NOT match (like grep -v).")
+@click.option("--model", "model_flag", help="Model for this run.")
+@click.option("--concurrency", "concurrency_flag", type=int, help="Max parallel model calls.")
+@click.option("--max-calls", "max_calls", type=int, help="Stop after N model calls (cost cap).")
+@click.option(
+    "--allow-captions",
+    "allow_captions",
+    is_flag=True,
+    help="Let a CLOUD model convert images/audio to text (paid; local models do it free).",
+)
+@input_options
+def filter_command(
+    condition: str | None,
+    prompt_file: Path | None,
+    invert: bool,
+    model_flag: str | None,
+    concurrency_flag: int | None,
+    max_calls: int | None,
+    allow_captions: bool,
+    in_patterns: tuple[str, ...],
+    from_files: bool,
+) -> None:
+    """Keep items matching a plain-English condition. Semantic grep.
+
+    \b
+    Examples:
+      cat reviews.txt | smartpipe filter "the reviewer is sarcastic"
+      cat tickets.jsonl | smartpipe filter "{priority} is wrong given {description}"
+      smartpipe filter "mentions a security issue" --in 'logs/*.txt'
+
+    Output is the matching input items, unchanged and in order (in file mode, the
+    matching filenames). Zero matches is a successful (exit 0) empty result.
+    Deterministic condition (field == value, text has "word")? where is free.
+    """
+    request = FilterRequest(
+        allow_captions=allow_captions,
+        condition=resolve_prompt(condition, prompt_file),
+        invert=invert,
+        model_flag=model_flag,
+        concurrency_flag=concurrency_flag,
+        input=input_spec(in_patterns, from_files=from_files),
+    )
+    code = asyncio.run(_run(request, max_calls))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(request: FilterRequest, max_calls: int | None) -> ExitCode:
+    from smartpipe.container import build_container
+
+    async with (
+        graceful_interrupts() as stop,
+        build_container(os.environ, max_calls=max_calls, stop=stop) as container,
+    ):
+        if not request.allow_captions and container.config.allow_captions:
+            from dataclasses import replace as _replace
+
+            request = _replace(request, allow_captions=True)  # profile consent (D35)
+        code = await run_filter(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)
+        return settle_budget(container.budget, code)

smartpipe/cli/getschema_cmd.py

@@ -0,0 +1,31 @@
+"""``smartpipe getschema`` — what fields does this stream even have?"""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.getschema import GetSchemaRequest, run_getschema
+
+__all__ = ["getschema_command"]
+
+
+@click.command(name="getschema")
+@click.option("--all", "scan_all", is_flag=True, help="Scan every row (default: first 10,000).")
+def getschema_command(scan_all: bool) -> None:
+    """Report the stream's fields, types, and coverage. Free — never calls a model.
+
+    \b
+    Examples:
+      cat data.jsonl | smartpipe getschema
+      smartpipe getschema --all < big.jsonl
+
+    A table on a terminal, NDJSON when piped. Mixed types show as unions
+    (string|number) — that's the dirt worth seeing. Plain-text input gets a
+    one-line answer instead of an error. The footer suggests the next move.
+    """
+    code = run_getschema(GetSchemaRequest(scan_all=scan_all), stdin=sys.stdin, stdout=sys.stdout)
+    if code is not ExitCode.OK:  # pragma: no cover — getschema always OKs
+        raise SystemExit(int(code))

smartpipe/cli/input_options.py

@@ -0,0 +1,113 @@
+"""Shared verb options — file inputs (``--in``/``--from-files``) and ``--fields``."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, TypeVar
+
+import click
+
+from smartpipe.core.errors import UsageFault
+from smartpipe.io.inputs import InputSpec
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from pathlib import Path
+
+__all__ = ["fields_option", "input_options", "input_spec", "parse_fields", "resolve_prompt"]
+
+_Command = TypeVar("_Command", bound="Callable[..., object]")
+
+_FIELDS_HINT = (
+    "  --fields is a comma-separated list of columns, each named once.\n"
+    "  Example: --fields name,email"
+)
+
+
+def input_options(command: _Command) -> _Command:
+    """Attach ``--in`` (glob of files as items) and ``--from-files`` (stdin names files)."""
+    command = click.option(
+        "--in",
+        "in_patterns",
+        multiple=True,
+        metavar="GLOB",
+        help="Read each matching file as one item (repeatable). e.g. --in 'docs/*.pdf'",
+    )(command)
+    command = click.option(
+        "--from-files",
+        "from_files",
+        is_flag=True,
+        help="Treat each stdin line as a filename; read each file as one item.",
+    )(command)
+    return command
+
+
+def input_spec(in_patterns: tuple[str, ...], *, from_files: bool) -> InputSpec:
+    return InputSpec(patterns=tuple(in_patterns), from_files=from_files)
+
+
+def fields_option(command: _Command) -> _Command:
+    """Attach ``--fields a,b`` — select + order the columns of structured output."""
+    return click.option(
+        "--fields",
+        "fields",
+        metavar="A,B,...",
+        callback=_fields_callback,
+        help="Select and order output columns (structured output only). e.g. --fields name,email",
+    )(command)
+
+
+def parse_fields(raw: str) -> tuple[str, ...]:
+    """``" a , b "`` → ``("a", "b")``; empty or duplicate names are usage errors."""
+    names = tuple(name.strip() for name in raw.split(","))
+    if any(not name for name in names):
+        raise UsageFault(f"--fields got an empty field name\n{_FIELDS_HINT}")
+    seen: set[str] = set()
+    for name in names:
+        if name in seen:
+            raise UsageFault(f"--fields names {name!r} more than once\n{_FIELDS_HINT}")
+        seen.add(name)
+    return names
+
+
+def _fields_callback(
+    ctx: click.Context, param: click.Parameter, value: str | None
+) -> tuple[str, ...] | None:
+    del ctx, param  # click's callback signature; the parse needs neither
+    return None if value is None else parse_fields(value)
+
+
+def resolve_prompt(argument: str | None, file_flag: Path | None) -> str:
+    """D23: ``@file`` shorthand + ``--prompt-file`` — both spellings, one resolver.
+
+    Only a LEADING ``@`` is special; ``@@x`` escapes a literal ``@``. Missing and
+    empty files fail free at argv time (D18), before anything could cost money.
+    """
+    if argument is not None and file_flag is not None:
+        raise UsageFault("a prompt argument and --prompt-file both given — use one")
+    if file_flag is not None:
+        return _read_prompt_file(file_flag)
+    if argument is None:
+        raise UsageFault("no prompt given — write one, or point at a file: --prompt-file FILE")
+    if argument.startswith("@@"):
+        return argument[1:]  # the escape: a literal leading @
+    if argument.startswith("@"):
+        from pathlib import Path as _Path
+
+        return _read_prompt_file(_Path(argument[1:]))
+    return argument
+
+
+def _read_prompt_file(path: Path) -> str:
+    if not path.exists():
+        raise UsageFault(
+            f"prompt file not found: {path}\n"
+            "  @file reads the prompt from a file; --prompt-file FILE is the explicit form.\n"
+            "  A literal leading @ escapes as @@."
+        )
+    text = path.read_text(encoding="utf-8").removesuffix("\n")
+    if not text.strip():
+        raise UsageFault(
+            f"prompt file is empty: {path}\n"
+            "  An empty prompt is never intended — write the prompt, or drop the @."
+        )
+    return text

smartpipe/cli/interrupts.py

@@ -0,0 +1,92 @@
+"""Graceful Ctrl-C for per-item verbs (ux.md §12).
+
+First SIGINT sets the ``stop`` event — the runner stops intake and drains in-flight
+work; a watchdog caps the drain. Second SIGINT hard-exits 130 immediately (the user
+means it). Whole-set verbs don't enter this context and keep the immediate
+KeyboardInterrupt → 130 path — no partial result exists for them.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import os
+import signal
+import sys
+from contextlib import asynccontextmanager
+from typing import TYPE_CHECKING
+
+from smartpipe.core.errors import ExitCode
+from smartpipe.io import diagnostics
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator
+
+    from smartpipe.models.budget import CallBudget
+
+__all__ = [
+    "drain_cap",
+    "graceful_interrupts",
+    "settle_budget",
+]
+
+_DEFAULT_DRAIN_SECONDS = 10.0
+
+
+def drain_cap() -> float:
+    # SMARTPIPE_DRAIN_SECONDS is a test seam (the drain-timeout e2e would otherwise
+    # take the full 10 s); not documented as public surface.
+    raw = os.environ.get("SMARTPIPE_DRAIN_SECONDS", "").strip()
+    try:
+        return float(raw) if raw else _DEFAULT_DRAIN_SECONDS
+    except ValueError:
+        return _DEFAULT_DRAIN_SECONDS
+
+
+async def _watchdog(cap: float, task: asyncio.Task[object]) -> None:
+    await asyncio.sleep(cap)
+    diagnostics.drain_timed_out()
+    task.cancel()
+
+
+@asynccontextmanager
+async def graceful_interrupts() -> AsyncGenerator[asyncio.Event]:
+    stop = asyncio.Event()
+    loop = asyncio.get_running_loop()
+    task = asyncio.current_task()
+    assert task is not None  # always inside asyncio.run here
+    sigints = 0
+    watchdog: asyncio.Task[None] | None = None
+
+    def on_sigint() -> None:
+        nonlocal sigints, watchdog
+        sigints += 1
+        if sigints == 1:
+            stop.set()
+            watchdog = loop.create_task(_watchdog(drain_cap(), task))
+        else:  # the user means it — flush what we can and go
+            with contextlib.suppress(Exception):
+                sys.stdout.flush()
+                sys.stderr.flush()
+            os._exit(int(ExitCode.INTERRUPTED))
+
+    try:
+        loop.add_signal_handler(signal.SIGINT, on_sigint)
+    except (NotImplementedError, RuntimeError):  # pragma: no cover — Windows loop
+        yield stop  # never set: falls back to KeyboardInterrupt → 130 (documented)
+        return
+    try:
+        yield stop
+    finally:
+        loop.remove_signal_handler(signal.SIGINT)
+        if watchdog is not None:
+            watchdog.cancel()
+
+
+def settle_budget(budget: CallBudget | None, code: ExitCode) -> ExitCode:
+    """D18: a run whose --max-calls budget fired never exits 0 — completeness
+    can't be trusted; the note names the cause after the drain summary."""
+    if budget is None or not budget.exhausted:
+        return code
+    diagnostics.note(f"stopped by --max-calls ({budget.calls} calls made)")
+    return ExitCode.PARTIAL if code is ExitCode.OK else code

smartpipe/cli/join_cmd.py

@@ -0,0 +1,162 @@
+"""``smartpipe join`` — the CLI surface: flags in, verb out (D21)."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+import click
+
+from smartpipe.cli.completions import complete_chat_models, complete_embed_models
+from smartpipe.cli.input_options import (
+    fields_option,
+    input_options,
+    input_spec,
+    resolve_prompt,
+)
+from smartpipe.cli.interrupts import graceful_interrupts, settle_budget
+from smartpipe.core.errors import ExitCode
+from smartpipe.io.writers import OutputFormat
+from smartpipe.verbs.join import JoinRequest, run_join
+
+__all__ = ["join_command"]
+
+
+@click.command(name="join")
+@click.argument("predicate", required=False)
+@click.option(
+    "--prompt-file",
+    "prompt_file",
+    type=click.Path(path_type=Path),
+    help="Read the predicate from a file (the @file shorthand does the same).",
+)
+@click.option(
+    "--right",
+    "right",
+    required=True,
+    type=click.Path(dir_okay=False, path_type=Path, allow_dash=True),
+    help="The finite side to index (JSONL or plain lines). stdin is the left side.",
+)
+@click.option(
+    "--k",
+    "k",
+    type=int,
+    default=5,
+    show_default=True,
+    help="Candidates judged per left item (the recall knob — see the docs).",
+)
+@click.option(
+    "--threshold",
+    type=float,
+    help="Similarity floor (0-1) a candidate must clear before judging.",
+)
+@click.option(
+    "--kind",
+    type=click.Choice(["inner", "leftouter", "anti"]),
+    default="inner",
+    show_default=True,
+    help="inner: matched pairs · leftouter: all left rows (null right) · "
+    "anti: only UNMATCHED left rows, verbatim.",
+)
+@click.option(
+    "--unmatched",
+    "unmatched",
+    type=click.Path(path_type=Path),
+    help="Write left items with zero matches to FILE, verbatim (inner only).",
+)
+@click.option(
+    "--model",
+    "model_flag",
+    shell_complete=complete_chat_models,
+    help="Chat model for the judge calls.",
+)
+@click.option(
+    "--embed-model",
+    "embed_model_flag",
+    shell_complete=complete_embed_models,
+    help="Embedding model for both sides.",
+)
+@click.option(
+    "--output",
+    type=click.Choice([fmt.value for fmt in OutputFormat]),
+    default=OutputFormat.AUTO.value,
+    show_default=True,
+    help="Output format.",
+)
+@click.option("--concurrency", "concurrency_flag", type=int, help="Max parallel left items.")
+@click.option("--max-calls", "max_calls", type=int, help="Stop after N model calls (cost cap).")
+@fields_option
+@click.option(
+    "--allow-captions",
+    "allow_captions",
+    is_flag=True,
+    help="Let a CLOUD model convert images/audio to text (paid; local models do it free).",
+)
+@input_options
+def join_command(
+    predicate: str | None,
+    prompt_file: Path | None,
+    right: Path,
+    k: int,
+    threshold: float | None,
+    unmatched: Path | None,
+    kind: str,
+    model_flag: str | None,
+    embed_model_flag: str | None,
+    output: str,
+    concurrency_flag: int | None,
+    max_calls: int | None,
+    allow_captions: bool,
+    fields: tuple[str, ...] | None,
+    in_patterns: tuple[str, ...],
+    from_files: bool,
+) -> None:
+    """Match stdin against a second input, semantically. Emits matched pairs.
+
+    \b
+    Examples:
+      cat tickets.jsonl | smartpipe join "{left.text} concerns {right.name}" --right products.jsonl
+      tail -f events.log | smartpipe join "{left.text} involves {right.name}" --right people.jsonl
+      cat orders.jsonl | smartpipe join "the same purchase" --right invoices.jsonl --kind anti
+
+    Each brace names a side's field ({left.x} / {right.x}; .text is the whole
+    item). The right side is embedded once and indexed; each left item is
+    compared to its --k nearest candidates and only those pairs are judged by
+    the chat model — so cost is lines x k, never lines x right-size.
+    Output: {"left": {...}, "right": {...}, "_score": ...} per matched pair.
+    """
+    request = JoinRequest(
+        allow_captions=allow_captions,
+        predicate=resolve_prompt(predicate, prompt_file),
+        right=right,
+        k=k,
+        threshold=threshold,
+        unmatched=unmatched,
+        kind=kind,
+        model_flag=model_flag,
+        embed_model_flag=embed_model_flag,
+        concurrency_flag=concurrency_flag,
+        output=OutputFormat(output),
+        input=input_spec(in_patterns, from_files=from_files),
+        fields=fields,
+    )
+    code = asyncio.run(_run(request, max_calls))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(request: JoinRequest, max_calls: int | None) -> ExitCode:
+    from smartpipe.container import build_container
+
+    async with (
+        graceful_interrupts() as stop,
+        build_container(os.environ, max_calls=max_calls, stop=stop) as container,
+    ):
+        if not request.allow_captions and container.config.allow_captions:
+            from dataclasses import replace as _replace
+
+            request = _replace(request, allow_captions=True)  # profile consent (D35)
+        code = await run_join(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)
+        return settle_budget(container.budget, code)

smartpipe/cli/map_cmd.py

@@ -0,0 +1,150 @@
+"""``smartpipe map`` — the CLI surface: flags in, verb out."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+import click
+
+from smartpipe.cli.completions import complete_chat_models
+from smartpipe.cli.input_options import (
+    fields_option,
+    input_options,
+    input_spec,
+    resolve_prompt,
+)
+from smartpipe.cli.interrupts import graceful_interrupts, settle_budget
+from smartpipe.core.errors import ExitCode
+from smartpipe.io.writers import OutputFormat
+from smartpipe.verbs.map import MapRequest, run_map
+
+__all__ = ["map_command"]
+
+
+@click.command(name="map")
+@click.argument("prompt", required=False)
+@click.option(
+    "--prompt-file",
+    "prompt_file",
+    type=click.Path(path_type=Path),
+    help="Prompt from a file (@file works too).",
+)
+@click.option(
+    "--explode",
+    "explode_field",
+    metavar="FIELD",
+    help="One row per element of a list-valued FIELD.",
+)
+@click.option(
+    "--tally",
+    "tally_field",
+    metavar="FIELD",
+    help="Count FIELD's values (live tally on stderr).",
+)
+@click.option(
+    "--schema-from",
+    "schema_dsl",
+    metavar="DSL",
+    help='Schema from a mini-DSL: "vendor string; total number >= 0".',
+)
+@click.option(
+    "--schema",
+    "schema_path",
+    type=click.Path(path_type=Path),
+    help="Enforce a JSON Schema on the output.",
+)
+@click.option(
+    "--model",
+    "model_flag",
+    shell_complete=complete_chat_models,
+    help="Model for this run (e.g. ollama/qwen3:8b, claude-opus-4-8).",
+)
+@click.option(
+    "--output",
+    type=click.Choice([fmt.value for fmt in OutputFormat]),
+    default=OutputFormat.AUTO.value,
+    show_default=True,
+    help="Output format.",
+)
+@click.option(
+    "--frame-every",
+    "frame_every",
+    type=float,
+    metavar="SECONDS",
+    help="Video density guarantee: one frame per period (lifts the 24-frame cap).",
+)
+@click.option(
+    "--max-frames",
+    "max_frames",
+    type=int,
+    help="Video frame budget (default 24; the smaller of the two flags wins).",
+)
+@click.option("--concurrency", "concurrency_flag", type=int, help="Max parallel model calls.")
+@click.option("--max-calls", "max_calls", type=int, help="Stop after N model calls (cost cap).")
+@fields_option
+@input_options
+def map_command(
+    prompt: str | None,
+    frame_every: float | None,
+    max_frames: int | None,
+    prompt_file: Path | None,
+    schema_path: Path | None,
+    schema_dsl: str | None,
+    tally_field: str | None,
+    explode_field: str | None,
+    model_flag: str | None,
+    output: str,
+    concurrency_flag: int | None,
+    max_calls: int | None,
+    fields: tuple[str, ...] | None,
+    in_patterns: tuple[str, ...],
+    from_files: bool,
+) -> None:
+    """Transform each input item with a prompt. One item in, one result out.
+
+    \b
+    Examples:
+      echo "hello" | smartpipe map "translate to Spanish"
+      cat reviews.jsonl | smartpipe map "Extract {product, sentiment}"
+      smartpipe map "Summarize this document" --in 'reports/*.pdf'
+      smartpipe map "What does the caller want?" --in 'calls/*.mp3'
+
+    You usually need NO flags: braces in the prompt name the JSON fields you
+    want back; plain prompts return plain text; and media is first-class —
+    images, audio, video, and the figures inside PDFs go to the model natively
+    when it supports them, converted (and disclosed) when it doesn't.
+
+    Everything else is opt-in refinement: schemas when braces aren't enough,
+    --tally/--explode/--fields to shape output, --max-calls to cap spend.
+    """
+    request = MapRequest(
+        prompt=resolve_prompt(prompt, prompt_file),
+        schema_path=schema_path,
+        schema_dsl=schema_dsl,
+        tally_field=tally_field,
+        explode_field=explode_field,
+        frame_every=frame_every,
+        max_frames=max_frames,
+        model_flag=model_flag,
+        output=OutputFormat(output),
+        concurrency_flag=concurrency_flag,
+        input=input_spec(in_patterns, from_files=from_files),
+        fields=fields,
+    )
+    code = asyncio.run(_run(request, max_calls))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(request: MapRequest, max_calls: int | None) -> ExitCode:
+    from smartpipe.container import build_container
+
+    async with (
+        graceful_interrupts() as stop,
+        build_container(os.environ, max_calls=max_calls, stop=stop) as container,
+    ):
+        code = await run_map(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)
+        return settle_budget(container.budget, code)