smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/cli/sort_cmd.py

@@ -0,0 +1,33 @@
+"""``smartpipe sort`` — order records by a field, without the jq incantation."""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.sortverb import SortRequest, run_sort
+
+__all__ = ["sort_command"]
+
+
+@click.command(name="sort")
+@click.option("--by", "by", required=True, metavar="FIELD", help="The field to order by.")
+@click.option("--desc", "descending", is_flag=True, help="Largest / Z-first.")
+def sort_command(by: str, descending: bool) -> None:
+    """Order records by a field. Free — never calls a model. Reads the whole input.
+
+    \b
+    Examples:
+      cat scored.ndjson | smartpipe sort --by _score --desc | head -5
+      smartpipe map "…{confidence number}" --in 'docs/*.pdf' | smartpipe sort --by confidence --desc
+
+    Numbers sort numerically, strings lexically (numbers first when mixed);
+    rows missing the field always land LAST, in both directions, with a
+    note. Stable: ties keep input order. Rows pass through byte-for-byte.
+    (There is no `take` — `head` already counts NDJSON rows.)
+    """
+    code = run_sort(SortRequest(by=by, descending=descending), stdin=sys.stdin, stdout=sys.stdout)
+    if code is not ExitCode.OK:  # pragma: no cover — sort always OKs
+        raise SystemExit(int(code))

smartpipe/cli/split_cmd.py

@@ -0,0 +1,76 @@
+"""``smartpipe split`` — break oversized items into chunk items. No model calls."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+
+import click
+
+from smartpipe.cli.input_options import input_options, input_spec
+from smartpipe.cli.interrupts import graceful_interrupts
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.split import SplitRequest, run_split
+
+__all__ = ["split_command"]
+
+
+@click.command(name="split")
+@click.option(
+    "--by",
+    "by_flag",
+    metavar="UNIT[:N]",
+    help="Split unit: tokens, pages, minutes, seconds. e.g. --by pages, --by minutes:10",
+)
+@click.option(
+    "--media",
+    "media",
+    is_flag=True,
+    help="Extract images embedded in PDFs/DOCX/PPTX/XLSX as items (icons dropped).",
+)
+@click.option(
+    "--max-tokens",
+    "max_tokens",
+    type=int,
+    help="Shorthand for --by tokens:N (default 2000).",
+)
+@input_options
+def split_command(
+    by_flag: str | None,
+    media: bool,
+    max_tokens: int | None,
+    in_patterns: tuple[str, ...],
+    from_files: bool,
+) -> None:
+    """Break oversized items into budget-sized chunks. Free — no model calls.
+
+    \b
+    Examples:
+      smartpipe split --in '10k-filings/*.pdf' | smartpipe map "list the risk factors {risk}"
+      smartpipe split --by pages:5 --in report.pdf | smartpipe map "summarize these pages"
+      smartpipe split --by minutes:10 --in call.mp3 | smartpipe map "what was agreed?"
+
+    Each chunk is a JSON record: {"text": …, "source": "report.pdf §3/12"} —
+    paragraph-boundary aware, and the chunks of a document concatenate back to
+    its exact text. Recombine downstream with reduce.
+    """
+    request = SplitRequest(
+        max_tokens_flag=max_tokens,
+        by_flag=by_flag,
+        media=media,
+        input=input_spec(in_patterns, from_files=from_files),
+    )
+    code = asyncio.run(_run(request))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(request: SplitRequest) -> ExitCode:
+    from smartpipe.container import build_container
+
+    async with (
+        graceful_interrupts() as stop,
+        build_container(os.environ) as container,
+    ):
+        return await run_split(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)

smartpipe/cli/summarize_cmd.py

@@ -0,0 +1,37 @@
+"""``smartpipe summarize`` — the numbers, without leaving for awk."""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.summarize import SummarizeRequest, run_summarize
+
+__all__ = ["summarize_command"]
+
+
+@click.command(name="summarize")
+@click.argument("expression")
+def summarize_command(expression: str) -> None:
+    """Aggregate records deterministically. Free — never calls a model.
+
+    \b
+    Examples:
+      cat orders.jsonl | smartpipe summarize 'count(), avg(total), p95(total) by region'
+      cat evals.jsonl  | smartpipe summarize 'count() by pass'
+      ... | smartpipe summarize 'dcount(user) by day'
+
+    \b
+    Aggregations: count() · sum(f) · avg(f) · min(f) · max(f)
+                  p50(f) · p90(f) · p95(f) · p99(f) · dcount(f)
+
+    KQL's own grammar and output naming (count, avg_total, p95_total).
+    Groups sort largest first; a missing group field groups under null,
+    visibly. Non-numeric values in numeric aggregations are skipped and
+    counted on stderr — never a mid-stream crash.
+    """
+    code = run_summarize(SummarizeRequest(expression), stdin=sys.stdin, stdout=sys.stdout)
+    if code is not ExitCode.OK:  # pragma: no cover — summarize always OKs
+        raise SystemExit(int(code))

smartpipe/cli/top_k_cmd.py

@@ -0,0 +1,97 @@
+"""``smartpipe top_k`` — rank items by similarity to a query."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+
+import click
+
+from smartpipe.cli.completions import complete_embed_models
+from smartpipe.cli.input_options import fields_option, input_options, input_spec
+from smartpipe.cli.interrupts import graceful_interrupts, settle_budget
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.top_k import TopKRequest, run_top_k
+
+__all__ = ["top_k_command"]
+
+
+@click.command(name="top_k")
+@click.argument("k", type=int, required=False)
+@click.option("--near", required=True, help="Rank items by similarity to this query.")
+@click.option("--threshold", type=float, help="Keep everything at or above this similarity (0-1).")
+@click.option(
+    "--embed-model", "model_flag", shell_complete=complete_embed_models, help="Embedding model."
+)
+@click.option("--concurrency", "concurrency_flag", type=int, help="Max parallel model calls.")
+@click.option("--max-calls", "max_calls", type=int, help="Stop after N model calls (cost cap).")
+@click.option("--stream", "stream", is_flag=True, help="Live leaderboard over a stream.")
+@fields_option
+@click.option(
+    "--allow-captions",
+    "allow_captions",
+    is_flag=True,
+    help="Let a CLOUD model convert images/audio to text (paid; local models do it free).",
+)
+@input_options
+def top_k_command(
+    k: int | None,
+    near: str,
+    threshold: float | None,
+    model_flag: str | None,
+    concurrency_flag: int | None,
+    max_calls: int | None,
+    allow_captions: bool,
+    stream: bool,
+    fields: tuple[str, ...] | None,
+    in_patterns: tuple[str, ...],
+    from_files: bool,
+) -> None:
+    """Rank items by similarity to a query and return the top K.
+
+    \b
+    Examples:
+      smartpipe top_k 5 --near "distributed systems engineer" --in 'resumes/*.pdf'
+      cat corpus.embeddings | smartpipe top_k 10 --near "Q3 revenue strategy"
+      cat articles.jsonl | smartpipe top_k --near "climate policy" --threshold 0.8
+
+    Give a number (K), a --threshold, or both. Each result gains a _score (0-1).
+    In file mode, each result is a filename and its score.
+    """
+    request = TopKRequest(
+        allow_captions=allow_captions,
+        near=near,
+        k=k,
+        threshold=threshold,
+        model_flag=model_flag,
+        concurrency_flag=concurrency_flag,
+        stream=stream,
+        input=input_spec(in_patterns, from_files=from_files),
+        fields=fields,
+    )
+    code = asyncio.run(_run(request, max_calls))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(request: TopKRequest, max_calls: int | None) -> ExitCode:
+    from smartpipe.container import build_container
+
+    if not request.stream:  # whole-set mode: ^C exits immediately; budget is fatal (D18)
+        async with build_container(os.environ, max_calls=max_calls) as container:
+            if not request.allow_captions and container.config.allow_captions:
+                from dataclasses import replace as _replace
+
+                request = _replace(request, allow_captions=True)  # profile consent (D35)
+            return await run_top_k(request, container, stdin=sys.stdin, stdout=sys.stdout)
+    async with (
+        graceful_interrupts() as stop,
+        build_container(os.environ, max_calls=max_calls, stop=stop) as container,
+    ):
+        if not request.allow_captions and container.config.allow_captions:
+            from dataclasses import replace as _replace
+
+            request = _replace(request, allow_captions=True)  # profile consent (D35)
+        code = await run_top_k(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)
+        return settle_budget(container.budget, code)

smartpipe/cli/usage_cmd.py

@@ -0,0 +1,66 @@
+"""``smartpipe usage`` — the ledger of what the meter observed (D41)."""
+
+from __future__ import annotations
+
+import os
+
+import click
+
+from smartpipe.io.metering import count as count_fmt
+from smartpipe.io.metering import duration as duration_fmt
+from smartpipe.io.metering import megabytes as megabytes_fmt
+from smartpipe.io.usage import Totals, read_ledger, reset_ledger, stamp
+
+__all__ = ["usage_command"]
+
+
+@click.group(name="usage", invoke_without_command=True)
+@click.pass_context
+def usage_command(ctx: click.Context) -> None:
+    """Model usage over time: hour, day, week, month, lifetime. Resettable.
+
+    Counts model-touching runs only (free verbs never call a model). The
+    numbers are the meter's observed units — tokens, media, conversions.
+    """
+    if ctx.invoked_subcommand is not None:
+        return
+    windows, first_seen, last_reset = read_ledger(os.environ)
+    if windows["lifetime"].runs == 0:
+        click.echo("no model usage recorded yet")
+        return
+    from smartpipe.cli.screens import heading, tint
+
+    header = (
+        f"{'':<12}{'runs':>6}{'tokens in':>12}{'tokens out':>12}"
+        f"{'media':>10}{'audio':>9}{'conv':>6}"
+    )
+    click.echo(heading(header))
+    for name in ("past hour", "past day", "past week", "past month", "lifetime"):
+        row = _row(name, windows[name])
+        click.echo(tint(row, "1") if name == "lifetime" else row)
+    notes: list[str] = []
+    if last_reset is not None:
+        notes.append(f"last reset {stamp(last_reset)}")
+    if first_seen is not None:
+        notes.append(f"first use {stamp(first_seen)}")
+    if notes:
+        click.echo(tint("since: " + " · ".join(notes), "2"))
+
+
+def _row(name: str, totals: Totals) -> str:
+    audio = duration_fmt(totals.audio_seconds) if totals.audio_seconds else "-"
+    media = megabytes_fmt(totals.media_bytes) if totals.media_bytes else "-"
+    return (
+        f"{name:<12}{totals.runs:>6}{count_fmt(totals.tokens_in):>12}"
+        f"{count_fmt(totals.tokens_out):>12}{media:>10}{audio:>9}{totals.conversions:>6}"
+    )
+
+
+@usage_command.command(name="reset")
+def usage_reset() -> None:
+    """Zero the ledger; the reset time is remembered and shown."""
+    previous = reset_ledger(os.environ)
+    click.echo(
+        f"usage reset (previous lifetime: {count_fmt(previous.tokens_in)} in · "
+        f"{count_fmt(previous.tokens_out)} out tokens across {previous.runs} runs)"
+    )

smartpipe/cli/where_cmd.py

@@ -0,0 +1,36 @@
+"""``smartpipe where`` — the free filter. Runs before anything paid."""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.where import WhereRequest, run_where
+
+__all__ = ["where_command"]
+
+
+@click.command(name="where")
+@click.argument("predicate")
+def where_command(predicate: str) -> None:
+    """Keep rows matching a deterministic predicate. Free — never calls a model.
+
+    \b
+    Examples:
+      tail -f app.log | smartpipe where 'text has "ERROR"'
+      cat orders.jsonl | smartpipe where 'total > 1000'
+      smartpipe where 'level == "error" and not text contains "retry"'
+
+    \b
+    Operators: has (word, case-insensitive) · contains · matches /re/
+               == != > >= < <=   combined with and · or · not · ( )
+    FIELD is a record field, or text for the whole line.
+
+    Put where BEFORE filter/map: it cuts the corpus for free, so the paid
+    stages only see what matters. Semantic condition? Use filter.
+    """
+    code = run_where(WhereRequest(predicate), stdin=sys.stdin, stdout=sys.stdout)
+    if code is not ExitCode.OK:  # pragma: no cover — where currently always OKs
+        raise SystemExit(int(code))

smartpipe/config/__init__.py

@@ -0,0 +1,5 @@
+"""Configuration: file location, load/store, and value precedence."""
+
+from __future__ import annotations
+
+__all__: list[str] = []

smartpipe/config/credentials.py

@@ -0,0 +1,118 @@
+"""The OAuth token store (plan/decisions.md D19) — ``auth.json`` beside the config.
+
+The narrowest amendment to "nothing stored": one file, mode 0600, one purpose
+(login tokens that must persist to refresh), removable with ``smartpipe auth
+logout``. API keys never live here. Unknown provider entries are preserved on
+rewrite (forward compatibility, same spirit as the config store).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from smartpipe.config.paths import human_path
+from smartpipe.core.errors import SetupFault
+from smartpipe.core.jsontools import as_record, as_str
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+__all__ = ["OAuthCredential", "credentials_path", "load_oauth", "remove_oauth", "save_oauth"]
+
+_MODE = 0o600
+
+
+@dataclass(frozen=True, slots=True)
+class OAuthCredential:
+    access: str
+    refresh: str
+    expires_ms: int  # epoch milliseconds, matching the wire's convention
+    account_id: str | None = None
+
+
+def credentials_path(env: Mapping[str, str] | None = None, platform: str | None = None) -> Path:
+    from smartpipe.config.paths import config_path
+
+    return config_path(env, platform).parent / "auth.json"
+
+
+def load_oauth(path: Path, provider: str) -> OAuthCredential | None:
+    record = _read_raw(path).get(provider)
+    entry = as_record(record)
+    if entry is None or entry.get("type") != "oauth":
+        return None
+    access = as_str(entry.get("access"))
+    refresh = as_str(entry.get("refresh"))
+    expires = entry.get("expires")
+    if access is None or refresh is None or not isinstance(expires, int):
+        return None
+    return OAuthCredential(
+        access=access,
+        refresh=refresh,
+        expires_ms=expires,
+        account_id=as_str(entry.get("account_id")),
+    )
+
+
+def save_oauth(path: Path, provider: str, credential: OAuthCredential) -> None:
+    data = _read_raw(path)
+    entry: dict[str, object] = {
+        "type": "oauth",
+        "access": credential.access,
+        "refresh": credential.refresh,
+        "expires": credential.expires_ms,
+    }
+    if credential.account_id is not None:
+        entry["account_id"] = credential.account_id
+    data[provider] = entry
+    _write_raw(path, data)
+
+
+def remove_oauth(path: Path, provider: str) -> bool:
+    """Delete one provider's entry; True if something was removed."""
+    data = _read_raw(path)
+    if provider not in data:
+        return False
+    del data[provider]
+    _write_raw(path, data)
+    return True
+
+
+def _read_raw(path: Path) -> dict[str, object]:
+    if not path.exists():
+        return {}
+    try:
+        parsed: object = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError) as exc:
+        raise SetupFault(
+            f"error: the login store is unreadable\n"
+            f"  {human_path(path)}: {exc}\n"
+            "  Fix the file, or remove it and log in again: smartpipe auth login"
+        ) from exc
+    record = as_record(parsed)
+    return dict(record) if record is not None else {}
+
+
+def _write_raw(path: Path, data: dict[str, object]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
+    try:
+        if hasattr(os, "fchmod"):  # POSIX: 0600 from the first byte
+            os.fchmod(fd, _MODE)
+        else:  # Windows has no fd-chmod; ACLs scope the profile dir instead
+            os.chmod(tmp, _MODE)
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            json.dump(data, handle, indent=2)
+        os.replace(tmp, path)  # atomic on POSIX and Windows (same volume)
+    except BaseException:
+        with contextlib.suppress(OSError):
+            os.unlink(tmp)
+        raise
+    with contextlib.suppress(OSError):  # replace preserves the temp's mode, but be sure
+        os.chmod(path, _MODE)

smartpipe/config/display.py

@@ -0,0 +1,70 @@
+"""Rendering ``config show`` — pure, so the exact layout is golden-testable.
+
+The point of the origin column (plan/decisions.md D09): precedence is never a
+mystery. Each effective value is shown with where it came from — env var,
+config file, or built-in default.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from smartpipe.io.text import display_width
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+
+    from smartpipe.config.store import Config
+
+__all__ = ["Setting", "render_show", "settings_with_origin"]
+
+_DEFAULTS = {
+    "model": "(auto-detect)",
+    "embed-model": "nomic-embed-text",
+    "concurrency": "4",
+    "output": "auto",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class Setting:
+    key: str
+    value: str
+    origin: str  # "env" | "config file" | "default"
+
+
+def settings_with_origin(env: Mapping[str, str], config: Config) -> tuple[Setting, ...]:
+    return (
+        _resolve("model", env.get("SMARTPIPE_MODEL"), config.model),
+        _resolve("embed-model", env.get("SMARTPIPE_EMBED_MODEL"), config.embed_model),
+        _resolve("concurrency", env.get("SMARTPIPE_CONCURRENCY"), config.concurrency),
+        _resolve("output", env.get("SMARTPIPE_OUTPUT"), config.output),
+    )
+
+
+def render_show(settings: Sequence[Setting], config_file: str) -> str:
+    key_width = max(display_width(s.key) for s in (*settings, Setting("config file", "", ""))) + 2
+    value_width = max(display_width(s.value) for s in settings) + 2
+    from smartpipe.cli.screens import tint
+
+    lines = [
+        f"{tint(_pad(s.key, key_width), '2')}{_pad(s.value, value_width)}"
+        f"{tint(f'({s.origin})', '2')}"
+        for s in settings
+    ]
+    lines.append(f"{tint(_pad('config file', key_width), '2')}{config_file}")
+    return "\n".join(lines)
+
+
+def _pad(text: str, width: int) -> str:
+    """Pad to ``width`` terminal cells (DEFER-2) — f-string ``<`` pads code points."""
+    return text + " " * max(0, width - display_width(text))
+
+
+def _resolve(key: str, env_value: str | None, config_value: object) -> Setting:
+    if env_value is not None and env_value.strip():
+        return Setting(key, env_value.strip(), "env")
+    if config_value is not None:
+        return Setting(key, str(config_value), "config file")
+    return Setting(key, _DEFAULTS[key], "default")

smartpipe/config/doctor.py

@@ -0,0 +1,58 @@
+"""``smartpipe doctor`` report — the pure half (D18).
+
+Check *gathering* is the CLI's job (it touches env, disk, and the local Ollama
+probe); this module owns the result type, the rendering, and the exit rule so the
+exact screen is golden-testable. Doctor never makes a paid model call — its whole
+point is telling you the run will work *before* anything costs money.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Literal
+
+from smartpipe.core.errors import ExitCode
+from smartpipe.io.text import display_width
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Sequence
+
+__all__ = ["CheckResult", "doctor_exit_code", "render_report"]
+
+_MARKS: dict[str, str] = {"ok": "✓", "fail": "✗", "skip": "–"}  # noqa: RUF001 — the en-dash skip mark is the ux.md pin
+
+
+@dataclass(frozen=True, slots=True)
+class CheckResult:
+    section: str  # "config", "ollama", "chat", "embed", "keys", "login", "extras", "terminal"
+    status: Literal["ok", "fail", "skip"]
+    detail: str  # the rest of the line, including any "— fix: …"
+
+
+def render_report(results: Sequence[CheckResult]) -> str:
+    """One line per check: padded section, mark, detail (the ux.md doctor screen)."""
+    from smartpipe.cli.screens import bad, good, tint
+
+    width = max(display_width(result.section) for result in results) + 2
+
+    def dim_mark(mark: str) -> str:
+        return tint(mark, "2")
+
+    paint: dict[str, Callable[[str], str]] = {"ok": good, "fail": bad, "skip": dim_mark}
+    lines = (
+        f"{tint(_pad(result.section, width), '2')}"
+        f"{paint[result.status](_MARKS[result.status])} {result.detail}"
+        for result in results
+    )
+    return "\n".join(lines)
+
+
+def doctor_exit_code(results: Sequence[CheckResult]) -> ExitCode:
+    """0 all green (skips are fine); 1 if anything needs fixing."""
+    if any(result.status == "fail" for result in results):
+        return ExitCode.PARTIAL
+    return ExitCode.OK
+
+
+def _pad(text: str, width: int) -> str:
+    return text + " " * max(0, width - display_width(text))

smartpipe/config/paths.py

@@ -0,0 +1,38 @@
+"""Where the config file lives (plan/decisions.md D09).
+
+``~/.config/smartpipe/config.toml`` on every Unix including macOS — the spec
+names this path and terminal users expect it — and ``%APPDATA%\\smartpipe`` on
+Windows. Environment and platform are parameters so the logic is pure.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+__all__ = ["config_path", "human_path"]
+
+
+def config_path(env: Mapping[str, str] | None = None, platform: str | None = None) -> Path:
+    resolved_env = os.environ if env is None else env
+    resolved_platform = sys.platform if platform is None else platform
+    if resolved_platform == "win32":
+        appdata = resolved_env.get("APPDATA")
+        base = Path(appdata) if appdata else Path.home() / "AppData" / "Roaming"
+    else:
+        xdg = resolved_env.get("XDG_CONFIG_HOME")
+        base = Path(xdg) if xdg else Path.home() / ".config"
+    return base / "smartpipe" / "config.toml"
+
+
+def human_path(path: Path) -> str:
+    """``~/.config/smartpipe/config.toml`` beats an absolute path in messages."""
+    try:
+        return f"~/{path.relative_to(Path.home()).as_posix()}"
+    except ValueError:
+        return str(path)