smartpipe-cli 1.3.0__py3-none-any.whl

smartpipe/cli/diff_cmd.py

@@ -0,0 +1,100 @@
+"""``smartpipe diff`` — what distinguishes two sets of items."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+import click
+
+from smartpipe.cli.completions import complete_chat_models, complete_embed_models
+from smartpipe.cli.interrupts import graceful_interrupts, settle_budget
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.diff import DiffRequest, run_diff
+
+__all__ = ["diff_command"]
+
+
+@click.command(name="diff")
+@click.option(
+    "--right",
+    "right",
+    type=click.Path(path_type=Path),
+    required=True,
+    help="The comparison set (JSONL or plain lines). Left is stdin.",
+)
+@click.option("--top", type=int, help="Show at most N discriminating themes.")
+@click.option("--all", "show_all", is_flag=True, help="Also show themes shared by both sides.")
+@click.option(
+    "--model",
+    "model_flag",
+    shell_complete=complete_chat_models,
+    help="Chat model for theme labels.",
+)
+@click.option(
+    "--embed-model",
+    "embed_flag",
+    shell_complete=complete_embed_models,
+    help="Embedding model for grouping.",
+)
+@click.option("--concurrency", "concurrency_flag", type=int, help="Max parallel model calls.")
+@click.option("--max-calls", "max_calls", type=int, help="Stop after N model calls (cost cap).")
+@click.option(
+    "--allow-captions",
+    "allow_captions",
+    is_flag=True,
+    help="Let a CLOUD model convert images/audio to text (paid; local models do it free).",
+)
+def diff_command(
+    right: Path,
+    top: int | None,
+    show_all: bool,
+    model_flag: str | None,
+    embed_flag: str | None,
+    concurrency_flag: int | None,
+    max_calls: int | None,
+    allow_captions: bool,
+) -> None:
+    """Semantic diff of two item SETS — not a line diff.
+
+    \b
+    Examples:
+      smartpipe diff --right errors-before.log < errors-during.log
+      smartpipe diff --right outputs-v1.jsonl < outputs-v2.jsonl
+      smartpipe diff --right v1-train.jsonl < v2-train.jsonl   # dataset drift
+
+    Embeds both sides, groups the union by meaning, and reports the themes
+    that over-index on one side — with both shares shown as evidence and
+    examples from the dominant side. Balanced themes are omitted (a note
+    says how many; --all shows them): the answer to "what's different"
+    shouldn't bury you in what's the same.
+    """
+    request = DiffRequest(
+        right=right,
+        top=top,
+        show_all=show_all,
+        model_flag=model_flag,
+        embed_flag=embed_flag,
+        concurrency_flag=concurrency_flag,
+        allow_captions=allow_captions,
+    )
+    code = asyncio.run(_run(request, max_calls))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(request: DiffRequest, max_calls: int | None) -> ExitCode:
+    from dataclasses import replace
+
+    from smartpipe.container import build_container
+
+    async with (
+        graceful_interrupts() as stop,
+        build_container(os.environ, max_calls=max_calls, stop=stop) as container,
+    ):
+        if not request.allow_captions and container.config.allow_captions:
+            request = replace(request, allow_captions=True)  # profile consent (D35)
+        code = await run_diff(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)
+        return settle_budget(container.budget, code)

smartpipe/cli/distinct_cmd.py

@@ -0,0 +1,94 @@
+"""``smartpipe distinct`` — fold near-duplicates; keep the first of each."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+
+import click
+
+from smartpipe.cli.completions import complete_embed_models
+from smartpipe.cli.input_options import input_options, input_spec
+from smartpipe.cli.interrupts import graceful_interrupts, settle_budget
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.distinct import DistinctRequest, run_distinct
+
+__all__ = ["distinct_command"]
+
+
+@click.command(name="distinct")
+@click.option("--show-groups", is_flag=True, help="Emit group records instead (audit the folds).")
+@click.option(
+    "--threshold",
+    type=float,
+    default=0.90,
+    show_default=True,
+    help="Cosine similarity at which two items are the same thing.",
+)
+@click.option(
+    "--embed-model",
+    "model_flag",
+    shell_complete=complete_embed_models,
+    help="Embedding model for this run.",
+)
+@click.option("--concurrency", "concurrency_flag", type=int, help="Max parallel model calls.")
+@click.option("--max-calls", "max_calls", type=int, help="Stop after N model calls (cost cap).")
+@click.option(
+    "--allow-captions",
+    "allow_captions",
+    is_flag=True,
+    help="Let a CLOUD model convert images/audio to text (paid; local models do it free).",
+)
+@input_options
+def distinct_command(
+    show_groups: bool,
+    threshold: float,
+    model_flag: str | None,
+    concurrency_flag: int | None,
+    max_calls: int | None,
+    allow_captions: bool,
+    in_patterns: tuple[str, ...],
+    from_files: bool,
+) -> None:
+    """Fold near-duplicate items — the same thing worded differently is one item.
+
+    \b
+    Examples:
+      cat tickets.txt | smartpipe distinct > unique.txt
+      cat alerts.jsonl | smartpipe distinct --show-groups   # audit what folded
+      smartpipe distinct < candidates.jsonl > train-clean.jsonl
+
+    Exact duplicates fold for free (no model calls); the rest are embedded
+    once and grouped by meaning. First occurrence wins; output keeps input
+    order and bytes. The receipt on stderr says exactly what was folded.
+
+    Run distinct BEFORE map/filter: every duplicate you fold is a model call
+    you don't pay for downstream.
+    """
+    request = DistinctRequest(
+        show_groups=show_groups,
+        threshold=threshold,
+        model_flag=model_flag,
+        concurrency_flag=concurrency_flag,
+        allow_captions=allow_captions,
+        input=input_spec(in_patterns, from_files=from_files),
+    )
+    code = asyncio.run(_run(request, max_calls))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(request: DistinctRequest, max_calls: int | None) -> ExitCode:
+    from dataclasses import replace
+
+    from smartpipe.container import build_container
+
+    async with (
+        graceful_interrupts() as stop,
+        build_container(os.environ, max_calls=max_calls, stop=stop) as container,
+    ):
+        if not request.allow_captions and container.config.allow_captions:
+            request = replace(request, allow_captions=True)  # profile consent (D35)
+        code = await run_distinct(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)
+        return settle_budget(container.budget, code)

smartpipe/cli/doctor_cmd.py

@@ -0,0 +1,207 @@
+"""``smartpipe doctor`` — every no-cost setup check, one screen, exit 0/1 (D18).
+
+The report is the result, so it goes to stdout. Never a paid model call: the only
+network touch is the existing free 2 s Ollama probe. Key lines report presence,
+never values (validating a key costs a call — that's the live runbook's job).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import click
+
+from smartpipe.config.doctor import CheckResult, doctor_exit_code, render_report
+from smartpipe.config.paths import config_path, human_path
+from smartpipe.core.errors import ExitCode, SempipeError
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from smartpipe.config.store import Config
+
+__all__ = ["doctor_command"]
+
+_KEY_VARS = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "MISTRAL_API_KEY")
+# D46: nothing is optional — these ship in core; a missing one = broken install
+_BUNDLED = (
+    ("documents", "markitdown"),
+    ("whisper", "faster_whisper"),
+    ("embeddings", "fastembed"),
+    ("anthropic", "anthropic"),
+    ("charts", "svgwrite"),
+    ("ffmpeg", "imageio_ffmpeg"),
+)
+
+# absent-by-necessity on 3.14 until onnxruntime/av publish wheels (D46)
+_WAITING_ON_314_WHEELS = {"documents", "whisper", "embeddings"}
+_RC_FILES = {"zsh": "~/.zshrc", "bash": "~/.bashrc"}
+
+
+@click.command(name="doctor")
+@click.option(
+    "--probe",
+    is_flag=True,
+    help="Also send 4 tiny PAID calls to chart which modalities really work.",
+)
+def doctor_command(probe: bool) -> None:
+    """Check that smartpipe is set up and ready — without spending a model call.
+
+    \b
+    Verifies: config parses · Ollama reachable · configured models installed ·
+    API keys present (never printed) · ChatGPT login · optional extras ·
+    shell completions. Exit 0 when everything is green.
+    --probe adds the modality matrix: real (tiny) calls, announced first.
+    """
+    results = asyncio.run(_gather(os.environ))
+    click.echo(render_report(results))
+    if not probe:
+        click.secho(
+            "\n⚠ these checks verify SETUP, not ABILITY — run `smartpipe doctor --probe`\n"
+            "  to test what your models can actually see and hear (4 tiny paid calls)",
+            fg="yellow",
+            bold=True,
+        )
+    if probe:
+        from smartpipe.cli.probe_cmd import run_probe
+
+        click.echo("")
+        click.echo(asyncio.run(run_probe(os.environ)))
+    code = doctor_exit_code(results)
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _gather(env: Mapping[str, str]) -> list[CheckResult]:
+    config, config_result = _check_config(env)
+    results = [config_result]
+    names = await _probe(env)
+    results.append(_check_ollama(env, names))
+    results.append(_check_model("chat", _effective_chat(env, config), names))
+    results.append(_check_model("embed", _effective_embed(env, config), names))
+    results.append(_check_keys(env))
+    results.append(_check_login(env))
+    results.append(_check_extras())
+    results.append(_check_completions(env))
+    return results
+
+
+def _check_config(env: Mapping[str, str]) -> tuple[Config | None, CheckResult]:
+    from smartpipe.config.store import load_config
+
+    path = config_path(env)
+    try:
+        config = load_config(path)
+    except SempipeError as exc:  # doctor reports sickness, it doesn't die of it
+        summary = str(exc).splitlines()[0].removeprefix("error: ")
+        return None, CheckResult("config", "fail", f"{human_path(path)}: {summary}")
+    if not path.exists():
+        return config, CheckResult("config", "skip", "no config file (defaults apply)")
+    described = config.model or "no default model"
+    return config, CheckResult("config", "ok", f"{human_path(path)} parses (model: {described})")
+
+
+async def _probe(env: Mapping[str, str]) -> tuple[str, ...] | None:
+    from smartpipe.models.http_support import make_client
+    from smartpipe.models.ollama import ollama_model_names, resolve_host
+
+    async with make_client() as client:
+        return await ollama_model_names(client, resolve_host(env))
+
+
+def _check_ollama(env: Mapping[str, str], names: tuple[str, ...] | None) -> CheckResult:
+    from smartpipe.models.ollama import resolve_host
+
+    host = resolve_host(env)
+    if names is None:
+        return CheckResult(
+            "ollama", "fail", f"not reachable at {host} — fix: ollama serve (or set OLLAMA_HOST)"
+        )
+    return CheckResult("ollama", "ok", f"reachable at {host} ({len(names)} models)")
+
+
+def _effective_chat(env: Mapping[str, str], config: Config | None) -> str | None:
+    return env.get("SMARTPIPE_MODEL", "").strip() or (config.model if config else None)
+
+
+def _effective_embed(env: Mapping[str, str], config: Config | None) -> str | None:
+    configured = env.get("SMARTPIPE_EMBED_MODEL", "").strip() or (
+        config.embed_model if config else None
+    )
+    return configured or "nomic-embed-text"  # the documented default
+
+
+def _check_model(
+    section: str, configured: str | None, names: tuple[str, ...] | None
+) -> CheckResult:
+    from smartpipe.core.errors import UsageFault
+    from smartpipe.models.base import parse_model_ref
+
+    if configured is None:
+        return CheckResult(
+            section, "fail", "no model configured — fix: smartpipe config model ollama/qwen3:8b"
+        )
+    try:
+        ref = parse_model_ref(configured)
+    except UsageFault as exc:
+        return CheckResult(section, "fail", str(exc).splitlines()[0])
+    if ref.provider != "ollama":
+        return CheckResult(section, "skip", f"{ref} is a cloud model (key presence below)")
+    if names is None:
+        return CheckResult(section, "skip", f"{ref.name} — can't verify, Ollama unreachable")
+    if ref.name in names:
+        return CheckResult(section, "ok", f"{configured} is installed")
+    return CheckResult(
+        section, "fail", f"{ref.name} not in ollama list — fix: ollama pull {ref.name}"
+    )
+
+
+def _check_keys(env: Mapping[str, str]) -> CheckResult:
+    parts = (f"{var} {'set' if env.get(var, '').strip() else 'not set'}" for var in _KEY_VARS)
+    return CheckResult("keys", "skip", " · ".join(parts))
+
+
+def _check_login(env: Mapping[str, str]) -> CheckResult:
+    from smartpipe.config.credentials import credentials_path, load_oauth
+
+    if load_oauth(credentials_path(env), "openai") is not None:
+        return CheckResult("login", "ok", "ChatGPT login present (refreshes automatically)")
+    return CheckResult("login", "skip", "no ChatGPT login — optional: smartpipe auth login")
+
+
+def _check_extras() -> CheckResult:
+    import sys
+    from importlib.util import find_spec
+
+    installed = {name: find_spec(module) is not None for name, module in _BUNDLED}
+    if all(installed.values()):
+        return CheckResult("extras", "ok", "everything ships in the box: " + " · ".join(installed))
+    marks = " · ".join(f"{name} {'✓' if present else '✗'}" for name, present in installed.items())
+    missing = {name for name, present in installed.items() if not present}
+    if sys.version_info >= (3, 14) and missing <= _WAITING_ON_314_WHEELS:
+        return CheckResult("extras", "skip", f"{marks} — waiting on upstream Python 3.14 wheels")
+    return CheckResult("extras", "fail", f"{marks} — broken install; reinstall smartpipe")
+
+
+def _check_completions(env: Mapping[str, str]) -> CheckResult:
+    shell = Path(env.get("SHELL", "")).name
+    if shell == "fish":
+        candidate = Path("~/.config/fish/completions/smartpipe.fish").expanduser()
+        if candidate.exists():
+            return CheckResult("terminal", "ok", "completions installed for fish")
+        return CheckResult("terminal", "skip", "no fish completions — optional: see install docs")
+    rc_name = _RC_FILES.get(shell)
+    if rc_name is None:
+        return CheckResult("terminal", "skip", "unknown shell — completions not checked")
+    rc_path = Path(rc_name).expanduser()
+    try:
+        rc_text = rc_path.read_text(encoding="utf-8")
+        installed = "_SMARTPIPE_COMPLETE" in rc_text or "_SMARTPIPE_COMPLETE" in rc_text
+    except OSError:
+        installed = False
+    if installed:
+        return CheckResult("terminal", "ok", f"completions installed for {shell}")
+    return CheckResult("terminal", "skip", f"no {shell} completions — optional: see install docs")

smartpipe/cli/echo_cmd.py

@@ -0,0 +1,44 @@
+"""Hidden debug verb: pass input through the io spine untouched.
+
+Not listed in help on purpose — it exists so anyone (including future us,
+debugging a support question) can see exactly how smartpipe itemizes and
+serializes a given input: ``cat mystery.jsonl | smartpipe echo --output json``.
+It is also the standing integration test of the whole io layer.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+
+import click
+
+from smartpipe.io import readers, tty
+from smartpipe.io.writers import OutputFormat, WriterConfig, make_writer, resolve_format
+
+__all__ = ["echo_command"]
+
+
+@click.command(name="echo", hidden=True)
+@click.option(
+    "--output",
+    "output_format",
+    type=click.Choice([fmt.value for fmt in OutputFormat]),
+    default=OutputFormat.AUTO.value,
+    show_default=True,
+    help="Output format.",
+)
+def echo_command(output_format: str) -> None:
+    """Pass stdin through smartpipe's item pipeline unchanged (debugging aid)."""
+    asyncio.run(_run(OutputFormat(output_format)))
+
+
+async def _run(flag: OutputFormat) -> None:
+    readers.ensure_not_a_tty(sys.stdin)
+    mode = resolve_format(flag, os.environ, stdout_tty=tty.stdout_is_tty(), structured=False)
+    config = WriterConfig(mode=mode, color=tty.stdout_supports_color(), width=tty.terminal_width())
+    writer = make_writer(config, sys.stdout)
+    async for item in readers.stdin_items(sys.stdin):
+        writer.write_passthrough(item)
+    writer.flush()

smartpipe/cli/embed_cmd.py

@@ -0,0 +1,80 @@
+"""``smartpipe embed`` — convert items to vector embeddings."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+
+import click
+
+from smartpipe.cli.completions import complete_embed_models
+from smartpipe.cli.input_options import fields_option, input_options, input_spec
+from smartpipe.cli.interrupts import graceful_interrupts, settle_budget
+from smartpipe.core.errors import ExitCode
+from smartpipe.verbs.embed import EmbedRequest, run_embed
+
+__all__ = ["embed_command"]
+
+
+@click.command(name="embed")
+@click.option(
+    "--embed-model",
+    "model_flag",
+    shell_complete=complete_embed_models,
+    help="Embedding model (e.g. nomic-embed-text).",
+)
+@click.option("--concurrency", "concurrency_flag", type=int, help="Max parallel model calls.")
+@click.option("--max-calls", "max_calls", type=int, help="Stop after N model calls (cost cap).")
+@fields_option
+@click.option(
+    "--allow-captions",
+    "allow_captions",
+    is_flag=True,
+    help="Let a CLOUD model convert images/audio to text (paid; local models do it free).",
+)
+@input_options
+def embed_command(
+    model_flag: str | None,
+    concurrency_flag: int | None,
+    max_calls: int | None,
+    allow_captions: bool,
+    fields: tuple[str, ...] | None,
+    in_patterns: tuple[str, ...],
+    from_files: bool,
+) -> None:
+    """Convert each item to a vector embedding (NDJSON out).
+
+    \b
+    Examples:
+      cat docs/*.md | smartpipe embed > corpus.embeddings
+      smartpipe embed --in 'docs/*.pdf' > corpus.embeddings
+
+    This is the only command that never touches a chat model — it uses the
+    embedding model, and exists to feed 'top_k'.
+    """
+    request = EmbedRequest(
+        allow_captions=allow_captions,
+        model_flag=model_flag,
+        concurrency_flag=concurrency_flag,
+        input=input_spec(in_patterns, from_files=from_files),
+        fields=fields,
+    )
+    code = asyncio.run(_run(request, max_calls))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(request: EmbedRequest, max_calls: int | None) -> ExitCode:
+    from smartpipe.container import build_container
+
+    async with (
+        graceful_interrupts() as stop,
+        build_container(os.environ, max_calls=max_calls, stop=stop) as container,
+    ):
+        if not request.allow_captions and container.config.allow_captions:
+            from dataclasses import replace as _replace
+
+            request = _replace(request, allow_captions=True)  # profile consent (D35)
+        code = await run_embed(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)
+        return settle_budget(container.budget, code)

smartpipe/cli/extend_cmd.py

@@ -0,0 +1,138 @@
+"""``smartpipe extend`` — your record, plus columns."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+import click
+
+from smartpipe.cli.completions import complete_chat_models
+from smartpipe.cli.input_options import (
+    fields_option,
+    input_options,
+    input_spec,
+    resolve_prompt,
+)
+from smartpipe.cli.interrupts import graceful_interrupts, settle_budget
+from smartpipe.core.errors import ExitCode
+from smartpipe.io.writers import OutputFormat
+from smartpipe.verbs.extend import ExtendRequest, run_extend
+
+__all__ = ["extend_command"]
+
+
+@click.command(name="extend")
+@click.argument("prompt", required=False)
+@click.option(
+    "--prompt-file",
+    "prompt_file",
+    type=click.Path(path_type=Path),
+    help="Prompt from a file (@file works too).",
+)
+@click.option(
+    "--schema",
+    "schema_path",
+    type=click.Path(path_type=Path),
+    help="Shape the ADDED fields with a JSON Schema.",
+)
+@click.option(
+    "--schema-from",
+    "schema_dsl",
+    metavar="DSL",
+    help='Added fields from a mini-DSL: "vendor string; total number >= 0".',
+)
+@click.option("--tally", "tally_field", metavar="FIELD", help="Count FIELD's values (stderr).")
+@click.option(
+    "--explode",
+    "explode_field",
+    metavar="FIELD",
+    help="One row per element of a list-valued FIELD (original fields ride along).",
+)
+@click.option(
+    "--model", "model_flag", shell_complete=complete_chat_models, help="Model for this run."
+)
+@click.option(
+    "--output",
+    type=click.Choice([fmt.value for fmt in OutputFormat]),
+    default=OutputFormat.AUTO.value,
+    show_default=True,
+    help="Output format.",
+)
+@click.option(
+    "--frame-every",
+    "frame_every",
+    type=float,
+    metavar="SECONDS",
+    help="Video density guarantee: one frame per period (lifts the 24-frame cap).",
+)
+@click.option(
+    "--max-frames",
+    "max_frames",
+    type=int,
+    help="Video frame budget (default 24; the smaller of the two flags wins).",
+)
+@click.option("--concurrency", "concurrency_flag", type=int, help="Max parallel model calls.")
+@click.option("--max-calls", "max_calls", type=int, help="Stop after N model calls (cost cap).")
+@fields_option
+@input_options
+def extend_command(
+    prompt: str | None,
+    frame_every: float | None,
+    max_frames: int | None,
+    prompt_file: Path | None,
+    schema_path: Path | None,
+    schema_dsl: str | None,
+    tally_field: str | None,
+    explode_field: str | None,
+    model_flag: str | None,
+    output: str,
+    concurrency_flag: int | None,
+    max_calls: int | None,
+    fields: tuple[str, ...] | None,
+    in_patterns: tuple[str, ...],
+    from_files: bool,
+) -> None:
+    """Add extracted fields to each record — everything it had survives.
+
+    \b
+    Before:  {"id": 812, "body": "app crashes when saving"}
+    $ cat tickets.jsonl | smartpipe extend "Add {sentiment enum(pos, neg, neutral), product string}"
+    After:   {"id": 812, "body": "app crashes when saving", "sentiment": "neg", "product": "app"}
+
+    Same prompt language as map (typed braces, --schema, --schema-from), but
+    the output is your record PLUS the new columns — drop it into the middle
+    of an existing pipeline. Plain text lines become {"text": ..., ...}.
+    Existing fields with the same name are overwritten (noted on stderr) so
+    re-running enrichment stays idempotent.
+    """
+    request = ExtendRequest(
+        prompt=resolve_prompt(prompt, prompt_file),
+        schema_path=schema_path,
+        schema_dsl=schema_dsl,
+        tally_field=tally_field,
+        explode_field=explode_field,
+        frame_every=frame_every,
+        max_frames=max_frames,
+        model_flag=model_flag,
+        output=OutputFormat(output),
+        concurrency_flag=concurrency_flag,
+        fields=fields,
+        input=input_spec(in_patterns, from_files=from_files),
+    )
+    code = asyncio.run(_run(request, max_calls))
+    if code is not ExitCode.OK:
+        raise SystemExit(int(code))
+
+
+async def _run(request: ExtendRequest, max_calls: int | None) -> ExitCode:
+    from smartpipe.container import build_container
+
+    async with (
+        graceful_interrupts() as stop,
+        build_container(os.environ, max_calls=max_calls, stop=stop) as container,
+    ):
+        code = await run_extend(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)
+        return settle_budget(container.budget, code)