smartpipe-cli 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- smartpipe/__init__.py +6 -0
- smartpipe/__main__.py +8 -0
- smartpipe/assets/probe.png +0 -0
- smartpipe/assets/probe.txt +1 -0
- smartpipe/assets/probe.wav +0 -0
- smartpipe/cli/__init__.py +5 -0
- smartpipe/cli/auth_cmd.py +78 -0
- smartpipe/cli/cache_cmd.py +60 -0
- smartpipe/cli/chart_cmd.py +60 -0
- smartpipe/cli/cite_cmd.py +26 -0
- smartpipe/cli/cluster_cmd.py +102 -0
- smartpipe/cli/completions.py +91 -0
- smartpipe/cli/config_cmd.py +234 -0
- smartpipe/cli/diff_cmd.py +100 -0
- smartpipe/cli/distinct_cmd.py +94 -0
- smartpipe/cli/doctor_cmd.py +207 -0
- smartpipe/cli/echo_cmd.py +44 -0
- smartpipe/cli/embed_cmd.py +80 -0
- smartpipe/cli/extend_cmd.py +138 -0
- smartpipe/cli/filter_cmd.py +87 -0
- smartpipe/cli/getschema_cmd.py +31 -0
- smartpipe/cli/input_options.py +113 -0
- smartpipe/cli/interrupts.py +92 -0
- smartpipe/cli/join_cmd.py +162 -0
- smartpipe/cli/map_cmd.py +150 -0
- smartpipe/cli/outliers_cmd.py +82 -0
- smartpipe/cli/probe_cmd.py +223 -0
- smartpipe/cli/reduce_cmd.py +129 -0
- smartpipe/cli/root.py +281 -0
- smartpipe/cli/run_cmd.py +136 -0
- smartpipe/cli/sample_cmd.py +35 -0
- smartpipe/cli/schema_cmd.py +75 -0
- smartpipe/cli/screens.py +231 -0
- smartpipe/cli/sem_file.py +453 -0
- smartpipe/cli/sort_cmd.py +33 -0
- smartpipe/cli/split_cmd.py +76 -0
- smartpipe/cli/summarize_cmd.py +37 -0
- smartpipe/cli/top_k_cmd.py +97 -0
- smartpipe/cli/usage_cmd.py +66 -0
- smartpipe/cli/where_cmd.py +36 -0
- smartpipe/config/__init__.py +5 -0
- smartpipe/config/credentials.py +118 -0
- smartpipe/config/display.py +70 -0
- smartpipe/config/doctor.py +58 -0
- smartpipe/config/paths.py +38 -0
- smartpipe/config/store.py +252 -0
- smartpipe/container.py +439 -0
- smartpipe/core/__init__.py +5 -0
- smartpipe/core/errors.py +57 -0
- smartpipe/core/jsontools.py +56 -0
- smartpipe/engine/__init__.py +9 -0
- smartpipe/engine/aggregate.py +234 -0
- smartpipe/engine/blocking.py +44 -0
- smartpipe/engine/chart.py +143 -0
- smartpipe/engine/chunking.py +161 -0
- smartpipe/engine/clustering.py +94 -0
- smartpipe/engine/predicate.py +330 -0
- smartpipe/engine/prompts.py +601 -0
- smartpipe/engine/ranking.py +62 -0
- smartpipe/engine/runner.py +175 -0
- smartpipe/engine/schema.py +208 -0
- smartpipe/engine/schema_dsl.py +144 -0
- smartpipe/engine/tally.py +53 -0
- smartpipe/engine/timebin.py +67 -0
- smartpipe/engine/units.py +43 -0
- smartpipe/engine/windows.py +78 -0
- smartpipe/io/__init__.py +9 -0
- smartpipe/io/diagnostics.py +148 -0
- smartpipe/io/inputs.py +44 -0
- smartpipe/io/items.py +149 -0
- smartpipe/io/leaderboard.py +52 -0
- smartpipe/io/metering.py +180 -0
- smartpipe/io/progress.py +140 -0
- smartpipe/io/readers.py +455 -0
- smartpipe/io/text.py +40 -0
- smartpipe/io/tty.py +88 -0
- smartpipe/io/usage.py +214 -0
- smartpipe/io/writers.py +340 -0
- smartpipe/models/__init__.py +5 -0
- smartpipe/models/anthropic_adapter.py +149 -0
- smartpipe/models/base.py +170 -0
- smartpipe/models/budget.py +94 -0
- smartpipe/models/cache.py +132 -0
- smartpipe/models/gemini_native.py +196 -0
- smartpipe/models/http_support.py +77 -0
- smartpipe/models/jina.py +98 -0
- smartpipe/models/local_embed.py +76 -0
- smartpipe/models/ollama.py +204 -0
- smartpipe/models/openai_codex.py +237 -0
- smartpipe/models/openai_compat.py +328 -0
- smartpipe/models/openai_oauth.py +366 -0
- smartpipe/models/resolve.py +78 -0
- smartpipe/models/retry.py +69 -0
- smartpipe/models/stt.py +80 -0
- smartpipe/models/windows.py +116 -0
- smartpipe/parsing/__init__.py +10 -0
- smartpipe/parsing/detect.py +178 -0
- smartpipe/parsing/extract.py +582 -0
- smartpipe/py.typed +0 -0
- smartpipe/verbs/__init__.py +5 -0
- smartpipe/verbs/chart.py +153 -0
- smartpipe/verbs/cluster.py +220 -0
- smartpipe/verbs/common.py +468 -0
- smartpipe/verbs/convert.py +251 -0
- smartpipe/verbs/diff.py +206 -0
- smartpipe/verbs/distinct.py +164 -0
- smartpipe/verbs/embed.py +166 -0
- smartpipe/verbs/extend.py +180 -0
- smartpipe/verbs/filter.py +191 -0
- smartpipe/verbs/getschema.py +135 -0
- smartpipe/verbs/join.py +413 -0
- smartpipe/verbs/map.py +315 -0
- smartpipe/verbs/outliers.py +119 -0
- smartpipe/verbs/reduce.py +428 -0
- smartpipe/verbs/sample.py +52 -0
- smartpipe/verbs/sortverb.py +63 -0
- smartpipe/verbs/split.py +333 -0
- smartpipe/verbs/summarize.py +60 -0
- smartpipe/verbs/top_k.py +318 -0
- smartpipe/verbs/where.py +47 -0
- smartpipe_cli-1.3.0.dist-info/METADATA +192 -0
- smartpipe_cli-1.3.0.dist-info/RECORD +126 -0
- smartpipe_cli-1.3.0.dist-info/WHEEL +4 -0
- smartpipe_cli-1.3.0.dist-info/entry_points.txt +3 -0
- smartpipe_cli-1.3.0.dist-info/licenses/LICENSE +201 -0
- smartpipe_cli-1.3.0.dist-info/licenses/NOTICE +5 -0
smartpipe/__init__.py
ADDED
smartpipe/__main__.py
ADDED
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
The quick brown fox jumps over the lazy dog.
|
|
Binary file
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""``smartpipe auth`` — log in with ChatGPT, check the login, log out (D19)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
|
|
9
|
+
from smartpipe.core.errors import ExitCode
|
|
10
|
+
|
|
11
|
+
__all__ = ["auth_command"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@click.group(name="auth")
|
|
15
|
+
def auth_command() -> None:
|
|
16
|
+
"""Log in with your ChatGPT Plus/Pro account (OpenAI models, no API key)."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@auth_command.command(name="login")
|
|
20
|
+
@click.option("--headless", is_flag=True, help="Device-code flow for machines without a browser.")
|
|
21
|
+
def login(headless: bool) -> None:
|
|
22
|
+
"""Log in with ChatGPT. Opens your browser; --headless prints a code instead."""
|
|
23
|
+
asyncio.run(_login(headless=headless))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@auth_command.command(name="status")
|
|
27
|
+
def status() -> None:
|
|
28
|
+
"""Show the login state (the status line is the result — it goes to stdout)."""
|
|
29
|
+
import os
|
|
30
|
+
|
|
31
|
+
from smartpipe.config.credentials import credentials_path, load_oauth
|
|
32
|
+
|
|
33
|
+
credential = load_oauth(credentials_path(os.environ), "openai")
|
|
34
|
+
if credential is None:
|
|
35
|
+
click.echo("openai: not logged in — run: smartpipe auth login")
|
|
36
|
+
raise SystemExit(int(ExitCode.OK))
|
|
37
|
+
account = credential.account_id or "unknown account"
|
|
38
|
+
click.echo(
|
|
39
|
+
f"openai: logged in with ChatGPT (account {account}) — token refreshes automatically"
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@auth_command.command(name="logout")
|
|
44
|
+
def logout() -> None:
|
|
45
|
+
"""Remove the stored ChatGPT login."""
|
|
46
|
+
import os
|
|
47
|
+
|
|
48
|
+
from smartpipe.config.credentials import credentials_path, remove_oauth
|
|
49
|
+
from smartpipe.io import diagnostics
|
|
50
|
+
|
|
51
|
+
if remove_oauth(credentials_path(os.environ), "openai"):
|
|
52
|
+
diagnostics.note("logged out — the stored ChatGPT tokens were removed")
|
|
53
|
+
else:
|
|
54
|
+
diagnostics.note("nothing to remove — you weren't logged in")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
async def _login(*, headless: bool) -> None:
|
|
58
|
+
import os
|
|
59
|
+
|
|
60
|
+
from smartpipe.config.credentials import credentials_path, save_oauth
|
|
61
|
+
from smartpipe.io import diagnostics
|
|
62
|
+
from smartpipe.models.http_support import make_client
|
|
63
|
+
from smartpipe.models.openai_oauth import login_via_browser, poll_device, start_device_flow
|
|
64
|
+
|
|
65
|
+
async with make_client() as client:
|
|
66
|
+
if headless:
|
|
67
|
+
start = await start_device_flow(client)
|
|
68
|
+
diagnostics.note(f"open {start.verify_url} and enter code: {start.user_code}")
|
|
69
|
+
credential = await poll_device(client, start)
|
|
70
|
+
else:
|
|
71
|
+
|
|
72
|
+
def announce(url: str) -> None:
|
|
73
|
+
diagnostics.note(f"complete the login in your browser (opening): {url}")
|
|
74
|
+
|
|
75
|
+
credential = await login_via_browser(client, announce=announce)
|
|
76
|
+
save_oauth(credentials_path(os.environ), "openai", credential)
|
|
77
|
+
account = credential.account_id or "unknown account"
|
|
78
|
+
diagnostics.note(f"logged in with ChatGPT (account {account})")
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""``smartpipe cache`` — maintenance for the result cache (D38/15)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import click
|
|
10
|
+
|
|
11
|
+
__all__ = ["cache_command"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _cache_dir() -> Path:
|
|
15
|
+
base = os.environ.get("XDG_CACHE_HOME", "").strip()
|
|
16
|
+
root = Path(base) if base else Path.home() / ".cache"
|
|
17
|
+
return root / "smartpipe" / "results"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@click.group(name="cache")
|
|
21
|
+
def cache_command() -> None:
|
|
22
|
+
"""Inspect or clear the result cache (enable with: smartpipe config cache on)."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@cache_command.command(name="clear")
|
|
26
|
+
def cache_clear() -> None:
|
|
27
|
+
"""Delete every cached reply and report the space freed."""
|
|
28
|
+
directory = _cache_dir()
|
|
29
|
+
if not directory.exists():
|
|
30
|
+
click.echo("cache is empty")
|
|
31
|
+
return
|
|
32
|
+
size = sum(entry.stat().st_size for entry in directory.rglob("*") if entry.is_file())
|
|
33
|
+
shutil.rmtree(directory)
|
|
34
|
+
click.echo(f"cache cleared: {size / 1_048_576:.1f} MB")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@cache_command.command(name="stats")
|
|
38
|
+
def cache_stats() -> None:
|
|
39
|
+
"""Show the cache's size, entry count, and oldest entry."""
|
|
40
|
+
import time
|
|
41
|
+
|
|
42
|
+
directory = _cache_dir()
|
|
43
|
+
entries = list(directory.rglob("*.json")) if directory.exists() else []
|
|
44
|
+
if not entries:
|
|
45
|
+
click.echo("cache is empty")
|
|
46
|
+
return
|
|
47
|
+
total = sum(p.stat().st_size for p in entries)
|
|
48
|
+
oldest = min(p.stat().st_mtime for p in entries)
|
|
49
|
+
days = (time.time() - oldest) / 86_400
|
|
50
|
+
from smartpipe.cli.screens import tint
|
|
51
|
+
|
|
52
|
+
click.echo(f"{tint('entries', '2')} {len(entries):,}")
|
|
53
|
+
click.echo(
|
|
54
|
+
f"{tint('size', '2')} {total / 1_048_576:.1f} MB "
|
|
55
|
+
f"{tint('(cap 500 MB default — cache-max-mb)', '2')}"
|
|
56
|
+
)
|
|
57
|
+
click.echo(
|
|
58
|
+
f"{tint('oldest', '2')} {days:.0f} days "
|
|
59
|
+
f"{tint('(ttl 30 days default — cache-days)', '2')}"
|
|
60
|
+
)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""``smartpipe chart`` — bars in the terminal, SVG on disk. No model calls."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import click
|
|
9
|
+
|
|
10
|
+
from smartpipe.core.errors import ExitCode
|
|
11
|
+
from smartpipe.verbs.chart import ChartRequest, run_chart
|
|
12
|
+
|
|
13
|
+
__all__ = ["chart_command"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@click.command(name="chart")
|
|
17
|
+
@click.argument("field", required=False)
|
|
18
|
+
@click.option("--top", type=int, help="How many bars (default 20).")
|
|
19
|
+
@click.option(
|
|
20
|
+
"--save",
|
|
21
|
+
type=click.Path(path_type=Path),
|
|
22
|
+
help="Also write the chart as an SVG file (no extra dependencies).",
|
|
23
|
+
)
|
|
24
|
+
@click.option("--title", help="Title for the saved SVG.")
|
|
25
|
+
@click.option("--facet", "facet", help="Several distributions in one pass: --facet label,severity.")
|
|
26
|
+
@click.option(
|
|
27
|
+
"--by-time",
|
|
28
|
+
"by_time",
|
|
29
|
+
metavar="FIELD:BUCKET",
|
|
30
|
+
help="Chronological buckets: --by-time ts:1h (ISO-8601 or epoch).",
|
|
31
|
+
)
|
|
32
|
+
def chart_command(
|
|
33
|
+
field: str | None,
|
|
34
|
+
top: int | None,
|
|
35
|
+
save: Path | None,
|
|
36
|
+
title: str | None,
|
|
37
|
+
facet: str | None,
|
|
38
|
+
by_time: str | None,
|
|
39
|
+
) -> None:
|
|
40
|
+
"""Draw a bar chart of a field's values (or of whole lines). Free.
|
|
41
|
+
|
|
42
|
+
\b
|
|
43
|
+
Examples:
|
|
44
|
+
cat tickets.txt | smartpipe map "Extract {label}" | smartpipe chart label
|
|
45
|
+
jq -r .status data.jsonl | smartpipe chart
|
|
46
|
+
… | smartpipe chart label --save labels.svg --title "Ticket labels"
|
|
47
|
+
cat tickets.jsonl | smartpipe chart --facet label,severity,region
|
|
48
|
+
cat app.jsonl | smartpipe where 'level == "error"' | smartpipe chart --by-time ts:1h
|
|
49
|
+
|
|
50
|
+
Reads NDJSON records (counts FIELD) or plain lines (counts each line).
|
|
51
|
+
The chart is the result — it goes to stdout; --save adds an SVG.
|
|
52
|
+
"""
|
|
53
|
+
facets = tuple(name.strip() for name in facet.split(",") if name.strip()) if facet else ()
|
|
54
|
+
code = run_chart(
|
|
55
|
+
ChartRequest(field=field, top=top, save=save, title=title, facets=facets, by_time=by_time),
|
|
56
|
+
stdin=sys.stdin,
|
|
57
|
+
stdout=sys.stdout,
|
|
58
|
+
)
|
|
59
|
+
if code is not ExitCode.OK:
|
|
60
|
+
raise SystemExit(int(code))
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""``smartpipe cite`` — print the BibTeX entry (the output IS the result, like --version)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import click
|
|
6
|
+
|
|
7
|
+
from smartpipe import __version__
|
|
8
|
+
|
|
9
|
+
__all__ = ["cite_command"]
|
|
10
|
+
|
|
11
|
+
_BIBTEX = """\
|
|
12
|
+
@software{{gupta_smartpipe_2026,
|
|
13
|
+
author = {{Gupta, Prabal}},
|
|
14
|
+
title = {{smartpipe: semantic pipes for your terminal}},
|
|
15
|
+
year = {{2026}},
|
|
16
|
+
version = {{{version}}},
|
|
17
|
+
license = {{Apache-2.0}},
|
|
18
|
+
url = {{https://github.com/prabal-rje/smartpipe}}
|
|
19
|
+
}}
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@click.command(name="cite")
|
|
24
|
+
def cite_command() -> None:
|
|
25
|
+
"""Print a BibTeX entry for citing smartpipe."""
|
|
26
|
+
click.echo(_BIBTEX.format(version=__version__), nl=False)
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""``smartpipe cluster`` — themes with sizes and quotes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
import click
|
|
10
|
+
|
|
11
|
+
from smartpipe.cli.completions import complete_chat_models, complete_embed_models
|
|
12
|
+
from smartpipe.cli.input_options import input_options, input_spec
|
|
13
|
+
from smartpipe.cli.interrupts import graceful_interrupts, settle_budget
|
|
14
|
+
from smartpipe.core.errors import ExitCode
|
|
15
|
+
from smartpipe.verbs.cluster import ClusterRequest, run_cluster
|
|
16
|
+
|
|
17
|
+
__all__ = ["cluster_command"]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@click.command(name="cluster")
|
|
21
|
+
@click.option("--k", type=int, help="Force exactly K clusters (merge smallest).")
|
|
22
|
+
@click.option("--top", type=int, help="Show N clusters; fold the rest into (other).")
|
|
23
|
+
@click.option(
|
|
24
|
+
"--explode",
|
|
25
|
+
metavar="members",
|
|
26
|
+
help="One row per input item, labeled with its cluster.",
|
|
27
|
+
)
|
|
28
|
+
@click.option(
|
|
29
|
+
"--model",
|
|
30
|
+
"model_flag",
|
|
31
|
+
shell_complete=complete_chat_models,
|
|
32
|
+
help="Chat model for cluster labels.",
|
|
33
|
+
)
|
|
34
|
+
@click.option(
|
|
35
|
+
"--embed-model",
|
|
36
|
+
"embed_flag",
|
|
37
|
+
shell_complete=complete_embed_models,
|
|
38
|
+
help="Embedding model for grouping.",
|
|
39
|
+
)
|
|
40
|
+
@click.option("--concurrency", "concurrency_flag", type=int, help="Max parallel model calls.")
|
|
41
|
+
@click.option("--max-calls", "max_calls", type=int, help="Stop after N model calls (cost cap).")
|
|
42
|
+
@click.option(
|
|
43
|
+
"--allow-captions",
|
|
44
|
+
"allow_captions",
|
|
45
|
+
is_flag=True,
|
|
46
|
+
help="Let a CLOUD model convert images/audio to text (paid; local models do it free).",
|
|
47
|
+
)
|
|
48
|
+
@input_options
|
|
49
|
+
def cluster_command(
|
|
50
|
+
k: int | None,
|
|
51
|
+
top: int | None,
|
|
52
|
+
explode: str | None,
|
|
53
|
+
model_flag: str | None,
|
|
54
|
+
embed_flag: str | None,
|
|
55
|
+
concurrency_flag: int | None,
|
|
56
|
+
max_calls: int | None,
|
|
57
|
+
allow_captions: bool,
|
|
58
|
+
in_patterns: tuple[str, ...],
|
|
59
|
+
from_files: bool,
|
|
60
|
+
) -> None:
|
|
61
|
+
"""Group items by meaning and label each group — themes, sized, with quotes.
|
|
62
|
+
|
|
63
|
+
\b
|
|
64
|
+
Examples:
|
|
65
|
+
cat feedback.txt | smartpipe cluster
|
|
66
|
+
cat tickets.jsonl | smartpipe cluster --top 8 | smartpipe chart cluster
|
|
67
|
+
cat snippets.txt | smartpipe cluster --explode members > coded.jsonl
|
|
68
|
+
|
|
69
|
+
One row per cluster, largest first: {"cluster", "size", "share",
|
|
70
|
+
"examples"} — the examples are the most representative quotes. The cost
|
|
71
|
+
shape is the point: N embeddings + one label call per cluster, never N
|
|
72
|
+
chat calls. Labels are deterministic (temperature 0): re-runs don't
|
|
73
|
+
change your slide.
|
|
74
|
+
"""
|
|
75
|
+
request = ClusterRequest(
|
|
76
|
+
k=k,
|
|
77
|
+
top=top,
|
|
78
|
+
explode=explode,
|
|
79
|
+
model_flag=model_flag,
|
|
80
|
+
embed_flag=embed_flag,
|
|
81
|
+
concurrency_flag=concurrency_flag,
|
|
82
|
+
allow_captions=allow_captions,
|
|
83
|
+
input=input_spec(in_patterns, from_files=from_files),
|
|
84
|
+
)
|
|
85
|
+
code = asyncio.run(_run(request, max_calls))
|
|
86
|
+
if code is not ExitCode.OK:
|
|
87
|
+
raise SystemExit(int(code))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
async def _run(request: ClusterRequest, max_calls: int | None) -> ExitCode:
|
|
91
|
+
from dataclasses import replace
|
|
92
|
+
|
|
93
|
+
from smartpipe.container import build_container
|
|
94
|
+
|
|
95
|
+
async with (
|
|
96
|
+
graceful_interrupts() as stop,
|
|
97
|
+
build_container(os.environ, max_calls=max_calls, stop=stop) as container,
|
|
98
|
+
):
|
|
99
|
+
if not request.allow_captions and container.config.allow_captions:
|
|
100
|
+
request = replace(request, allow_captions=True) # profile consent (D35)
|
|
101
|
+
code = await run_cluster(request, container, stdin=sys.stdin, stdout=sys.stdout, stop=stop)
|
|
102
|
+
return settle_budget(container.budget, code)
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Shell-completion callbacks — instant model-name suggestions, never a hang.
|
|
2
|
+
|
|
3
|
+
Completion runs on every ``<TAB>``, so the budget is hard: the configured model
|
|
4
|
+
comes from the config file, the installed ones from a single Ollama probe capped
|
|
5
|
+
at 150 ms — and *any* failure (no daemon, slow daemon, broken config) collapses
|
|
6
|
+
to "no suggestions", never an error and never a wait. httpx stays a
|
|
7
|
+
function-local import: this module loads with the CLI, the probe runs only
|
|
8
|
+
inside a completion request.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from typing import TYPE_CHECKING
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from collections.abc import Mapping
|
|
17
|
+
|
|
18
|
+
import click
|
|
19
|
+
from click.shell_completion import CompletionItem
|
|
20
|
+
|
|
21
|
+
__all__ = ["complete_chat_models", "complete_embed_models", "suggest_models"]
|
|
22
|
+
|
|
23
|
+
_PROBE_TIMEOUT_SECONDS = 0.15 # a <TAB> must feel instant; a slow probe is a missing probe
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def suggest_models(incomplete: str, env: Mapping[str, str], *, embed: bool) -> tuple[str, ...]:
|
|
27
|
+
"""Configured model first, then ``ollama/<name>`` per installed model.
|
|
28
|
+
|
|
29
|
+
Every failure path degrades to fewer suggestions — completion must never
|
|
30
|
+
crash, hang, or print (the shell would splice the noise into the command line).
|
|
31
|
+
"""
|
|
32
|
+
try:
|
|
33
|
+
configured = _configured(env, embed=embed)
|
|
34
|
+
except Exception: # a broken config file must not break <TAB>
|
|
35
|
+
configured = ()
|
|
36
|
+
try:
|
|
37
|
+
local = tuple(f"ollama/{name}" for name in _ollama_names(env))
|
|
38
|
+
except Exception: # no daemon / slow daemon / bad payload — suggest nothing extra
|
|
39
|
+
local = ()
|
|
40
|
+
merged = dict.fromkeys(configured + local) # dedupe, configured first
|
|
41
|
+
return tuple(name for name in merged if name.startswith(incomplete))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _configured(env: Mapping[str, str], *, embed: bool) -> tuple[str, ...]:
|
|
45
|
+
from smartpipe.config.paths import config_path
|
|
46
|
+
from smartpipe.config.store import load_config
|
|
47
|
+
|
|
48
|
+
config = load_config(config_path(env))
|
|
49
|
+
env_value = env.get("SMARTPIPE_EMBED_MODEL" if embed else "SMARTPIPE_MODEL", "").strip()
|
|
50
|
+
stored = config.embed_model if embed else config.model
|
|
51
|
+
return tuple(name for name in (env_value, stored) if name)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _ollama_names(env: Mapping[str, str]) -> tuple[str, ...]:
|
|
55
|
+
import httpx
|
|
56
|
+
|
|
57
|
+
from smartpipe.core.jsontools import as_items, as_record
|
|
58
|
+
from smartpipe.models.ollama import resolve_host
|
|
59
|
+
|
|
60
|
+
with httpx.Client(timeout=_PROBE_TIMEOUT_SECONDS) as client:
|
|
61
|
+
response = client.get(f"{resolve_host(env)}/api/tags")
|
|
62
|
+
response.raise_for_status()
|
|
63
|
+
payload = as_record(response.json())
|
|
64
|
+
entries = as_items(payload.get("models")) if payload is not None else None
|
|
65
|
+
if entries is None:
|
|
66
|
+
return ()
|
|
67
|
+
records = (as_record(entry) for entry in entries)
|
|
68
|
+
names = (record.get("name") for record in records if record is not None)
|
|
69
|
+
return tuple(name for name in names if isinstance(name, str))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def complete_chat_models(
|
|
73
|
+
ctx: click.Context, param: click.Parameter, incomplete: str
|
|
74
|
+
) -> list[CompletionItem]:
|
|
75
|
+
del ctx, param # click's callback shape; the suggestions need neither
|
|
76
|
+
return _items(incomplete, embed=False)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def complete_embed_models(
|
|
80
|
+
ctx: click.Context, param: click.Parameter, incomplete: str
|
|
81
|
+
) -> list[CompletionItem]:
|
|
82
|
+
del ctx, param
|
|
83
|
+
return _items(incomplete, embed=True)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _items(incomplete: str, *, embed: bool) -> list[CompletionItem]:
|
|
87
|
+
import os
|
|
88
|
+
|
|
89
|
+
from click.shell_completion import CompletionItem
|
|
90
|
+
|
|
91
|
+
return [CompletionItem(name) for name in suggest_models(incomplete, os.environ, embed=embed)]
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""The ``config`` verb: show settings, set a model, or run interactive setup.
|
|
2
|
+
|
|
3
|
+
The interactive flow (``run_interactive_setup``) takes its I/O as injected
|
|
4
|
+
callables (ask/confirm/say/save) so it is unit-testable without a real terminal
|
|
5
|
+
— the click wiring supplies the real prompts. This is the first-class-function
|
|
6
|
+
DI the design template favors.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import os
|
|
13
|
+
from dataclasses import replace
|
|
14
|
+
from typing import TYPE_CHECKING
|
|
15
|
+
|
|
16
|
+
import click
|
|
17
|
+
|
|
18
|
+
from smartpipe.cli.completions import complete_chat_models, complete_embed_models
|
|
19
|
+
from smartpipe.config.display import render_show, settings_with_origin
|
|
20
|
+
from smartpipe.config.paths import config_path, human_path
|
|
21
|
+
from smartpipe.config.store import Config, load_config, save_config
|
|
22
|
+
from smartpipe.core.errors import SetupFault
|
|
23
|
+
from smartpipe.io import diagnostics
|
|
24
|
+
from smartpipe.models.base import parse_model_ref
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from collections.abc import Awaitable, Callable
|
|
28
|
+
|
|
29
|
+
__all__ = ["config_command", "run_interactive_setup"]
|
|
30
|
+
|
|
31
|
+
_TRY_IT = 'echo "hello world" | smartpipe map "translate to Spanish"'
|
|
32
|
+
_NON_TTY = (
|
|
33
|
+
"error: 'smartpipe config' is interactive and needs a terminal\n"
|
|
34
|
+
" Set a model without prompts:\n"
|
|
35
|
+
" smartpipe config model ollama/qwen3:8b (local, free)\n"
|
|
36
|
+
" smartpipe config model gpt-5.4-mini (cloud)"
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@click.group(name="config", invoke_without_command=True)
|
|
41
|
+
@click.pass_context
|
|
42
|
+
def config_command(ctx: click.Context) -> None:
|
|
43
|
+
"""Configure models and settings."""
|
|
44
|
+
if ctx.invoked_subcommand is not None:
|
|
45
|
+
return
|
|
46
|
+
import sys
|
|
47
|
+
|
|
48
|
+
if not (sys.stdin.isatty() and sys.stdout.isatty()):
|
|
49
|
+
raise SetupFault(_NON_TTY)
|
|
50
|
+
asyncio.run(_interactive_entry())
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@config_command.command(name="profile")
|
|
54
|
+
@click.argument("name", required=False)
|
|
55
|
+
@click.option("--unset", is_flag=True, help="Clear the active profile.")
|
|
56
|
+
def config_profile(name: str | None, unset: bool) -> None:
|
|
57
|
+
"""Show, switch, or clear the active profile.
|
|
58
|
+
|
|
59
|
+
\b
|
|
60
|
+
smartpipe config profile list profiles (the active one marked)
|
|
61
|
+
smartpipe config profile local switch — local runs ollama/gemma-4-e2b
|
|
62
|
+
SMARTPIPE_PROFILE=gemini smartpipe … one-off, no file change
|
|
63
|
+
"""
|
|
64
|
+
from smartpipe.config.store import profile_names, set_active_profile
|
|
65
|
+
|
|
66
|
+
path = config_path(os.environ)
|
|
67
|
+
if unset:
|
|
68
|
+
set_active_profile(path, None)
|
|
69
|
+
diagnostics.note("profile cleared — flat config keys stand alone")
|
|
70
|
+
return
|
|
71
|
+
if name is None:
|
|
72
|
+
config = load_config(path, os.environ)
|
|
73
|
+
for known in profile_names(path):
|
|
74
|
+
marker = "* " if known == config.profile else " "
|
|
75
|
+
click.echo(f"{marker}{known}")
|
|
76
|
+
return
|
|
77
|
+
known = profile_names(path)
|
|
78
|
+
if name not in known:
|
|
79
|
+
raise SetupFault(
|
|
80
|
+
f"error: profile {name!r} doesn't exist\n Known profiles: {', '.join(known)}\n"
|
|
81
|
+
f" Define [profiles.{name}] in {human_path(path)} to create it."
|
|
82
|
+
)
|
|
83
|
+
set_active_profile(path, name)
|
|
84
|
+
effective = load_config(path)
|
|
85
|
+
summary = ", ".join(
|
|
86
|
+
f"{label} {value}"
|
|
87
|
+
for label, value in (("model:", effective.model), ("embed:", effective.embed_model))
|
|
88
|
+
if value is not None
|
|
89
|
+
)
|
|
90
|
+
diagnostics.note(f"profile '{name}' active — {summary}")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@config_command.command(name="show")
|
|
94
|
+
def config_show() -> None:
|
|
95
|
+
"""Show the effective settings and where each comes from."""
|
|
96
|
+
env = os.environ
|
|
97
|
+
path = config_path(env)
|
|
98
|
+
config = load_config(path, env)
|
|
99
|
+
if config.profile is not None:
|
|
100
|
+
origin = "SMARTPIPE_PROFILE" if env.get("SMARTPIPE_PROFILE", "").strip() else "config file"
|
|
101
|
+
click.echo(f"profile {config.profile} ({origin})")
|
|
102
|
+
click.echo(render_show(settings_with_origin(env, config), human_path(path)))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@config_command.command(name="model")
|
|
106
|
+
@click.argument("model_string", shell_complete=complete_chat_models)
|
|
107
|
+
def config_set_model(model_string: str) -> None:
|
|
108
|
+
"""Set the default chat model (e.g. ollama/qwen3:8b, gpt-5.4-mini)."""
|
|
109
|
+
ref = parse_model_ref(model_string)
|
|
110
|
+
_update(lambda c: replace(c, model=str(ref)))
|
|
111
|
+
click.echo(f"model set to {ref}")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@config_command.command(name="embed-model")
|
|
115
|
+
@click.argument("model_string", shell_complete=complete_embed_models)
|
|
116
|
+
def config_set_embed_model(model_string: str) -> None:
|
|
117
|
+
"""Set the embedding model used by embed and top_k."""
|
|
118
|
+
ref = parse_model_ref(model_string)
|
|
119
|
+
_update(lambda c: replace(c, embed_model=str(ref)))
|
|
120
|
+
click.echo(f"embed-model set to {ref}")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@config_command.command(name="stt-model")
|
|
124
|
+
@click.argument("model_string")
|
|
125
|
+
def config_set_stt(model_string: str) -> None:
|
|
126
|
+
"""Set the remote transcription model (e.g. openai/whisper-1) — verbatim STT."""
|
|
127
|
+
ref = parse_model_ref(model_string)
|
|
128
|
+
_update(lambda c: replace(c, stt_model=str(ref)))
|
|
129
|
+
click.echo(f"stt-model set to {ref} (runs first in the audio ladder; consent rules apply)")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@config_command.command(name="cache")
|
|
133
|
+
@click.argument("state", type=click.Choice(["on", "off"]))
|
|
134
|
+
def config_set_cache(state: str) -> None:
|
|
135
|
+
"""Turn result caching on or off (identical calls reuse stored replies)."""
|
|
136
|
+
_update(lambda c: replace(c, cache=state == "on"))
|
|
137
|
+
click.echo(f"cache {state} — stored replies live in ~/.cache/smartpipe (smartpipe cache clear)")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _update(change: Callable[[Config], Config]) -> None:
|
|
141
|
+
path = config_path(os.environ)
|
|
142
|
+
save_config(path, change(load_config(path)))
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
async def _interactive_entry() -> None:
|
|
146
|
+
from smartpipe.container import build_container
|
|
147
|
+
|
|
148
|
+
async with build_container(os.environ) as container:
|
|
149
|
+
path = config_path(os.environ)
|
|
150
|
+
await run_interactive_setup(
|
|
151
|
+
current=container.config,
|
|
152
|
+
probe=container.probe_ollama,
|
|
153
|
+
ask=lambda question, default: click.prompt(question, default=default),
|
|
154
|
+
confirm=lambda question: click.confirm(question, default=True),
|
|
155
|
+
say=click.echo,
|
|
156
|
+
save=lambda config: save_config(path, config),
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
async def run_interactive_setup(
|
|
161
|
+
*,
|
|
162
|
+
current: Config,
|
|
163
|
+
probe: Callable[[], Awaitable[tuple[str, ...] | None]],
|
|
164
|
+
ask: Callable[[str, str], str],
|
|
165
|
+
confirm: Callable[[str], bool],
|
|
166
|
+
say: Callable[[str], None],
|
|
167
|
+
save: Callable[[Config], None],
|
|
168
|
+
) -> Config:
|
|
169
|
+
say("smartpipe setup — one minute, three questions\n")
|
|
170
|
+
if current.profile is None and current.model is None:
|
|
171
|
+
say("Pick a starting profile (a named bundle you can switch any time):")
|
|
172
|
+
say(
|
|
173
|
+
" 1. openai — gpt-5.4-mini + text-embedding-3-small "
|
|
174
|
+
"(key or ChatGPT login; no audio input)"
|
|
175
|
+
)
|
|
176
|
+
say(" 2. gemini — gemini-3.1-flash-lite, the most multimodal wire (needs GEMINI_API_KEY)")
|
|
177
|
+
say(" 3. local — ollama/gemma-4-e2b, multimodal, nothing leaves this machine")
|
|
178
|
+
say(" 4. custom — answer the questions instead")
|
|
179
|
+
choice = ask("Profile [1-4]?", "4").strip()
|
|
180
|
+
picked = {"1": "openai", "2": "gemini", "3": "local"}.get(choice)
|
|
181
|
+
if picked is not None:
|
|
182
|
+
from smartpipe.config.store import BUILTIN_PROFILES
|
|
183
|
+
|
|
184
|
+
chosen = replace(current, profile=picked)
|
|
185
|
+
save(chosen) # a fresh setup has no flat keys to materialize
|
|
186
|
+
bundle = ", ".join(f"{k} = {v}" for k, v in BUILTIN_PROFILES[picked].items())
|
|
187
|
+
say(f"\n ✓ profile '{picked}' active ({bundle})")
|
|
188
|
+
if BUILTIN_PROFILES[picked].get("allow-captions"):
|
|
189
|
+
say(
|
|
190
|
+
" note: this profile converts images/audio to text through its"
|
|
191
|
+
" model when needed (fractions of a cent each, disclosed per row)"
|
|
192
|
+
)
|
|
193
|
+
say(" Check the setup end to end: smartpipe doctor\n")
|
|
194
|
+
return chosen
|
|
195
|
+
names = await probe() or ()
|
|
196
|
+
chat = _first_chat(names)
|
|
197
|
+
say(" Tip: pick a model that can SEE images (gpt-5.4-mini, gemini-3.1-flash-lite,")
|
|
198
|
+
say(" ollama/llava) — smartpipe is multimodal; text-only models refuse image rows.")
|
|
199
|
+
if chat is not None:
|
|
200
|
+
say(f" ✓ found Ollama ({len(names)} models)\n")
|
|
201
|
+
model_answer = ask("Default model?", f"ollama/{chat}")
|
|
202
|
+
else:
|
|
203
|
+
# No Ollama, or Ollama has only embedding models — offer a cloud chat model.
|
|
204
|
+
say(
|
|
205
|
+
" no local chat model found — install one at https://ollama.com, "
|
|
206
|
+
"or use a cloud model.\n"
|
|
207
|
+
)
|
|
208
|
+
model_answer = ask(
|
|
209
|
+
"Default model (e.g. gpt-5.4-mini, needs OPENAI_API_KEY)", "gpt-5.4-mini"
|
|
210
|
+
)
|
|
211
|
+
embed_answer = ask("Embedding model?", f"ollama/{_first_embed(names)}")
|
|
212
|
+
|
|
213
|
+
updated = replace(
|
|
214
|
+
current,
|
|
215
|
+
model=str(parse_model_ref(model_answer)),
|
|
216
|
+
embed_model=str(parse_model_ref(embed_answer)),
|
|
217
|
+
)
|
|
218
|
+
if confirm("Save to config?"):
|
|
219
|
+
save(updated)
|
|
220
|
+
say("\n Saved. Try it:")
|
|
221
|
+
say(f" {_TRY_IT}")
|
|
222
|
+
else:
|
|
223
|
+
say("\n Not saved. Set a model any time with: smartpipe config model <name>")
|
|
224
|
+
return updated
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _first_chat(names: tuple[str, ...]) -> str | None:
|
|
228
|
+
"""The first non-embedding model, or None — never propose an embedding
|
|
229
|
+
model as the chat default just because it's the only thing installed."""
|
|
230
|
+
return next((name for name in names if "embed" not in name.lower()), None)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _first_embed(names: tuple[str, ...]) -> str:
|
|
234
|
+
return next((name for name in names if "embed" in name.lower()), "nomic-embed-text")
|