corethink 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- corethink/__init__.py +17 -0
- corethink/__main__.py +3 -0
- corethink/_version.py +3 -0
- corethink/account.py +470 -0
- corethink/cli.py +1058 -0
- corethink/client.py +253 -0
- corethink/commands/__init__.py +0 -0
- corethink/commands/admin.py +703 -0
- corethink/commands/api.py +59 -0
- corethink/commands/archive.py +433 -0
- corethink/commands/auth.py +375 -0
- corethink/commands/billing.py +314 -0
- corethink/commands/capture.py +564 -0
- corethink/commands/connection.py +1266 -0
- corethink/commands/endpoint.py +1124 -0
- corethink/commands/env.py +1966 -0
- corethink/commands/hub.py +269 -0
- corethink/commands/library.py +347 -0
- corethink/commands/models.py +258 -0
- corethink/commands/train.py +2655 -0
- corethink/commands/workspace.py +622 -0
- corethink/config.py +181 -0
- corethink/context.py +46 -0
- corethink/env_template.py +117 -0
- corethink/environment_http_store.py +96 -0
- corethink/experiment_cli.py +2187 -0
- corethink/hosted_profiles.py +44 -0
- corethink/hosted_rl.py +688 -0
- corethink/hosted_rl_archive.py +74 -0
- corethink/hosted_rl_attribution.py +44 -0
- corethink/hosted_rl_billing.py +103 -0
- corethink/hosted_rl_state.py +182 -0
- corethink/jobs_cli.py +839 -0
- corethink/main.py +204 -0
- corethink/output.py +201 -0
- corethink/py.typed +0 -0
- corethink/rl_cli.py +997 -0
- corethink/rl_inspect.py +521 -0
- corethink/rl_playground.py +129 -0
- corethink/rl_reassessment.py +214 -0
- corethink/rl_settings.py +43 -0
- corethink/sdk.py +331 -0
- corethink/settings.py +162 -0
- corethink-1.0.0.dist-info/METADATA +120 -0
- corethink-1.0.0.dist-info/RECORD +48 -0
- corethink-1.0.0.dist-info/WHEEL +5 -0
- corethink-1.0.0.dist-info/entry_points.txt +2 -0
- corethink-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1124 @@
|
|
|
1
|
+
"""`corethink endpoint`: every environment's endpoint on the gateway (staging and production), releases, API keys and
|
|
2
|
+
their usage, and calling a program.
|
|
3
|
+
|
|
4
|
+
The gateway is OpenAI-compatible: base URL https://gateway.corethink.ai/v1, model `<workspace>/<environment>@staging`
|
|
5
|
+
or `@production` (or a version, `@harness-<hash>`). A key calls only its own environment's endpoint.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import sys
|
|
12
|
+
import time
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
from urllib.parse import quote, urlsplit
|
|
16
|
+
|
|
17
|
+
import httpx
|
|
18
|
+
import typer
|
|
19
|
+
from rich.text import Text
|
|
20
|
+
|
|
21
|
+
from corethink import output as ui
|
|
22
|
+
from corethink.client import USER_AGENT, ApiError
|
|
23
|
+
from corethink.context import client, state
|
|
24
|
+
|
|
25
|
+
app = typer.Typer(
|
|
26
|
+
help="Endpoints: releases to staging and production, API keys, usage, and calling a program.",
|
|
27
|
+
no_args_is_help=True,
|
|
28
|
+
)
|
|
29
|
+
keys_app = typer.Typer(
|
|
30
|
+
help="API keys of your endpoints: create, rename, limit, rotate, revoke, usage.",
|
|
31
|
+
no_args_is_help=True,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
DEFAULT_GATEWAY = "https://gateway.corethink.ai/v1"
|
|
35
|
+
ALIASES = ("production", "staging")
|
|
36
|
+
POLL_SECONDS = 3.0
|
|
37
|
+
POLL_LIMIT = 60
|
|
38
|
+
|
|
39
|
+
YES_OPTION = typer.Option(False, "--yes", "-y", help="Do not ask.")
|
|
40
|
+
GATEWAY_OPTION = typer.Option(
|
|
41
|
+
None,
|
|
42
|
+
"--gateway-url",
|
|
43
|
+
envvar="CORETHINK_GATEWAY_URL",
|
|
44
|
+
help="The gateway's base URL (default https://gateway.corethink.ai/v1).",
|
|
45
|
+
show_default=False,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ---- helpers ---------------------------------------------------------------------------------------------------------
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _environments() -> list[dict[str, Any]]:
|
|
53
|
+
return (client().get("/api/v1/harnesses/usecases") or {}).get("usecases") or []
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _pick(matches: list[dict[str, Any]], given: str, what: str, label: Any) -> dict[str, Any]:
|
|
57
|
+
if len(matches) == 1:
|
|
58
|
+
return matches[0]
|
|
59
|
+
if not matches:
|
|
60
|
+
ui.error(f"No {what} matches {given!r}.")
|
|
61
|
+
raise typer.Exit(5)
|
|
62
|
+
ui.error(f"{given!r} matches more than one {what} ({', '.join(label(m) for m in matches[:6])}); give more of it.")
|
|
63
|
+
raise typer.Exit(2)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _resolve_env(given: str, rows: list[dict[str, Any]] | None = None) -> dict[str, Any]:
|
|
67
|
+
"""An environment by ID or name, or the start of either."""
|
|
68
|
+
rows = _environments() if rows is None else rows
|
|
69
|
+
folded = given.strip().casefold()
|
|
70
|
+
for test in (
|
|
71
|
+
lambda r: str(r.get("id", "")).casefold() == folded,
|
|
72
|
+
lambda r: str(r.get("name", "")).casefold() == folded,
|
|
73
|
+
lambda r: str(r.get("id", "")).casefold().startswith(folded),
|
|
74
|
+
lambda r: str(r.get("name", "")).casefold().startswith(folded),
|
|
75
|
+
):
|
|
76
|
+
matches = [row for row in rows if test(row)]
|
|
77
|
+
if matches:
|
|
78
|
+
return _pick(matches, given, "environment", lambda r: str(r.get("id")))
|
|
79
|
+
ui.error(f"No environment matches {given!r}. See `corethink endpoint list`.")
|
|
80
|
+
raise typer.Exit(5)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _env_path(env_id: str, tail: str = "") -> str:
|
|
84
|
+
return f"/api/v1/harnesses/usecases/{quote(env_id, safe='')}{tail}"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _endpoint(env_id: str) -> dict[str, Any]:
|
|
88
|
+
return client().get(_env_path(env_id, "/endpoint")) or {}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _served(endpoint: dict[str, Any] | None) -> dict[str, dict[str, Any]]:
|
|
92
|
+
"""What each alias serves now: {alias: catalogue entry's corethink block}."""
|
|
93
|
+
live: dict[str, dict[str, Any]] = {}
|
|
94
|
+
for entry in (endpoint or {}).get("catalogue") or []:
|
|
95
|
+
block = entry.get("corethink") or {}
|
|
96
|
+
alias = block.get("alias")
|
|
97
|
+
if alias and alias not in live:
|
|
98
|
+
live[alias] = block
|
|
99
|
+
return live
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _label(version: Any) -> str:
|
|
103
|
+
"""A version as people read it: harness-e972315b3a96 → v-e972315b3a96."""
|
|
104
|
+
return str(version).replace("harness-", "v-", 1) if version else "—"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _vs_base(gate: dict[str, Any] | None) -> str:
|
|
108
|
+
"""The final check's gain over the base model, in points."""
|
|
109
|
+
if not gate:
|
|
110
|
+
return "—"
|
|
111
|
+
systems = gate.get("systems") or {}
|
|
112
|
+
raw = (systems.get("raw") or {}).get("mean")
|
|
113
|
+
base = raw if raw is not None else (systems.get("base") or {}).get("mean")
|
|
114
|
+
trained = (systems.get("candidate") or {}).get("mean")
|
|
115
|
+
comparisons = gate.get("comparisons") or {}
|
|
116
|
+
comparison = comparisons.get("vs_raw") if raw is not None else comparisons.get("vs_base")
|
|
117
|
+
delta = (comparison or {}).get("delta")
|
|
118
|
+
if delta is None and base is not None and trained is not None:
|
|
119
|
+
delta = float(trained) - float(base)
|
|
120
|
+
return ui.points(delta) if delta is not None else "—"
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _gateway_base(given: str | None, endpoint: dict[str, Any] | None = None) -> str:
|
|
124
|
+
"""The gateway's base URL (…/v1): the option or CORETHINK_GATEWAY_URL, else the endpoint's, else the public one."""
|
|
125
|
+
value = given or ((endpoint or {}).get("gateway")) or DEFAULT_GATEWAY
|
|
126
|
+
value = value.rstrip("/")
|
|
127
|
+
if urlsplit(value).path in ("", "/"):
|
|
128
|
+
value += "/v1"
|
|
129
|
+
return value
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _program_of(env: dict[str, Any]) -> str:
|
|
133
|
+
org = env.get("org")
|
|
134
|
+
if org:
|
|
135
|
+
return f"{org}/{env['id']}"
|
|
136
|
+
return str(_endpoint(str(env["id"])).get("program") or env["id"])
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _target(given: str) -> tuple[str, str | None, dict[str, Any] | None]:
|
|
140
|
+
"""ENV as a program: `<workspace>/<env>[@alias]` is taken as it is (no sign-in needed); anything else is resolved."""
|
|
141
|
+
if "/" in given:
|
|
142
|
+
program, _, alias = given.partition("@")
|
|
143
|
+
return program, alias or None, None
|
|
144
|
+
env = _resolve_env(given)
|
|
145
|
+
return _program_of(env), None, env
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _find_version(endpoint: dict[str, Any], given: str) -> tuple[str, str]:
|
|
149
|
+
"""A version by label (v-…, harness-…), candidate hash (or its start), or the alias serving it: (hash, label)."""
|
|
150
|
+
served = _served(endpoint)
|
|
151
|
+
if given in served:
|
|
152
|
+
block = served[given]
|
|
153
|
+
return str(block.get("candidate_hash")), _label(block.get("version"))
|
|
154
|
+
wanted = given.strip().lower()
|
|
155
|
+
if wanted.startswith("v-"):
|
|
156
|
+
wanted = "harness-" + wanted[2:]
|
|
157
|
+
known: dict[str, str] = {}
|
|
158
|
+
for entry in endpoint.get("versions") or []:
|
|
159
|
+
known[str(entry.get("candidate_hash"))] = str(entry.get("version") or "")
|
|
160
|
+
for block in served.values():
|
|
161
|
+
known.setdefault(str(block.get("candidate_hash")), str(block.get("version") or ""))
|
|
162
|
+
matches = [
|
|
163
|
+
(candidate, version)
|
|
164
|
+
for candidate, version in known.items()
|
|
165
|
+
if version.lower() == wanted
|
|
166
|
+
or (len(wanted) >= 6 and (candidate.startswith(wanted) or version.lower().startswith(wanted)))
|
|
167
|
+
]
|
|
168
|
+
if len(matches) == 1:
|
|
169
|
+
return matches[0][0], _label(matches[0][1] or matches[0][0][:12])
|
|
170
|
+
if not matches:
|
|
171
|
+
ui.error(f"No recent version of this endpoint matches {given!r}. See `corethink endpoint show`.")
|
|
172
|
+
raise typer.Exit(5)
|
|
173
|
+
ui.error(f"{given!r} matches more than one version; give more of it.")
|
|
174
|
+
raise typer.Exit(2)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _resolve_run(given: str) -> dict[str, Any]:
|
|
178
|
+
given = given.strip().lower()
|
|
179
|
+
if len(given) != 36:
|
|
180
|
+
jobs = (client().get("/api/v1/harnesses/training-jobs") or {}).get("jobs") or []
|
|
181
|
+
matches = [job for job in jobs if str(job.get("job_id", "")).startswith(given)]
|
|
182
|
+
given = str(_pick(matches, given, "run", lambda j: str(j.get("job_id"))[:8]).get("job_id"))
|
|
183
|
+
return client().get(f"/api/v1/harnesses/training-jobs/{given}") or {}
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _wait_release(release_id: str) -> dict[str, Any]:
|
|
187
|
+
"""Poll a release until the registrar writes its outcome (about three minutes at most, as the console does)."""
|
|
188
|
+
status: dict[str, Any] = {"state": "pending"}
|
|
189
|
+
with ui.err.status("Releasing… (the release job applies every release rule)", spinner="dots"):
|
|
190
|
+
for _ in range(POLL_LIMIT):
|
|
191
|
+
time.sleep(POLL_SECONDS)
|
|
192
|
+
status = client().get(f"/api/v1/harnesses/releases/{release_id}") or {"state": "pending"}
|
|
193
|
+
if status.get("state") == "done":
|
|
194
|
+
break
|
|
195
|
+
return status
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _release(body: dict[str, Any], *, wait: bool, done_text: str) -> None:
|
|
199
|
+
started = client().post("/api/v1/harnesses/releases", body) or {}
|
|
200
|
+
release_id = str(started.get("releaseId"))
|
|
201
|
+
if not wait:
|
|
202
|
+
data = {"releaseId": release_id, "state": "pending"}
|
|
203
|
+
ui.emit(
|
|
204
|
+
data,
|
|
205
|
+
lambda: ui.info(
|
|
206
|
+
f"Started release {release_id}. Follow it: corethink endpoint release-status {release_id} --wait"
|
|
207
|
+
),
|
|
208
|
+
)
|
|
209
|
+
return
|
|
210
|
+
status = _wait_release(release_id)
|
|
211
|
+
_finish(release_id, status, body, done_text)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _finish(release_id: str, status: dict[str, Any], body: dict[str, Any] | None, done_text: str | None) -> None:
|
|
215
|
+
data = {"releaseId": release_id, **status}
|
|
216
|
+
outcome = status.get("outcome") or {}
|
|
217
|
+
if status.get("state") != "done":
|
|
218
|
+
ui.emit(
|
|
219
|
+
data,
|
|
220
|
+
lambda: ui.warn(
|
|
221
|
+
f"The release is still running; check back in a minute: corethink endpoint release-status {release_id}"
|
|
222
|
+
),
|
|
223
|
+
)
|
|
224
|
+
return
|
|
225
|
+
if not outcome.get("ok"):
|
|
226
|
+
ui.emit(data, lambda: ui.error(str(outcome.get("error") or "Refused.")))
|
|
227
|
+
raise typer.Exit(1)
|
|
228
|
+
body = body or {}
|
|
229
|
+
if body.get("rollback"):
|
|
230
|
+
text = f"{body.get('alias')} rolled back."
|
|
231
|
+
elif outcome.get("canary"):
|
|
232
|
+
text = (
|
|
233
|
+
f"Canary started: {body.get('canaryPercent'):g}% of new conversations for {body.get('canaryMinutes')} minutes. "
|
|
234
|
+
"The judge decides every 5 minutes."
|
|
235
|
+
)
|
|
236
|
+
else:
|
|
237
|
+
text = done_text or "Released."
|
|
238
|
+
ui.emit(data, lambda: ui.success(text))
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _reason(given: str | None, fallback: str) -> str:
|
|
242
|
+
reason = (given or fallback).strip()
|
|
243
|
+
if not 3 <= len(reason) <= 500:
|
|
244
|
+
ui.error("Give a reason of 3 to 500 characters (it is kept in the release history).")
|
|
245
|
+
raise typer.Exit(2)
|
|
246
|
+
return reason
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _canary(percent: float | None, minutes: int | None) -> dict[str, Any]:
|
|
250
|
+
if percent is None:
|
|
251
|
+
if minutes is not None:
|
|
252
|
+
ui.error("--minutes goes with --canary PERCENT.")
|
|
253
|
+
raise typer.Exit(2)
|
|
254
|
+
return {}
|
|
255
|
+
if not 0 < percent <= 100:
|
|
256
|
+
ui.error("--canary is a share of new conversations, above 0 and at most 100 (%).")
|
|
257
|
+
raise typer.Exit(2)
|
|
258
|
+
return {"canaryPercent": percent, "canaryMinutes": minutes or 60}
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
# ---- list and show ---------------------------------------------------------------------------------------------------
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
@app.command("list")
|
|
265
|
+
def list_() -> None:
|
|
266
|
+
"""Every environment's endpoint: what staging and production serve now.
|
|
267
|
+
|
|
268
|
+
Example: corethink endpoint list
|
|
269
|
+
"""
|
|
270
|
+
envs = _environments()
|
|
271
|
+
served_rows = (client().get("/api/v1/harnesses/endpoints") or {}).get("endpoints") or []
|
|
272
|
+
by_id = {row.get("id"): row for row in served_rows}
|
|
273
|
+
rows: list[dict[str, Any]] = []
|
|
274
|
+
gateway = None
|
|
275
|
+
for env in envs:
|
|
276
|
+
row = by_id.get(env.get("id")) or {}
|
|
277
|
+
endpoint = row.get("endpoint") or {}
|
|
278
|
+
gateway = gateway or endpoint.get("gateway")
|
|
279
|
+
live = _served(endpoint)
|
|
280
|
+
rows.append(
|
|
281
|
+
{
|
|
282
|
+
"id": env.get("id"),
|
|
283
|
+
"name": env.get("name"),
|
|
284
|
+
"program": endpoint.get("program") or (f"{env['org']}/{env['id']}" if env.get("org") else None),
|
|
285
|
+
"production": (live.get("production") or {}).get("version"),
|
|
286
|
+
"staging": (live.get("staging") or {}).get("version"),
|
|
287
|
+
"error": row.get("error"),
|
|
288
|
+
}
|
|
289
|
+
)
|
|
290
|
+
base = _gateway_base(None, {"gateway": gateway})
|
|
291
|
+
|
|
292
|
+
def human() -> None:
|
|
293
|
+
ui.table(
|
|
294
|
+
rows,
|
|
295
|
+
[
|
|
296
|
+
("Environment", "name"),
|
|
297
|
+
("ID", "id"),
|
|
298
|
+
(
|
|
299
|
+
"Production",
|
|
300
|
+
lambda r: (
|
|
301
|
+
Text(_label(r["production"]), style="green") if r["production"] else Text("—", style="dim")
|
|
302
|
+
),
|
|
303
|
+
),
|
|
304
|
+
(
|
|
305
|
+
"Staging",
|
|
306
|
+
lambda r: (
|
|
307
|
+
_label(r["staging"])
|
|
308
|
+
if r["staging"]
|
|
309
|
+
else Text("unavailable" if r["error"] else "—", style="dim")
|
|
310
|
+
),
|
|
311
|
+
),
|
|
312
|
+
(
|
|
313
|
+
"Model",
|
|
314
|
+
lambda r: (
|
|
315
|
+
f"{r['program']}@{'production' if r['production'] else 'staging'}"
|
|
316
|
+
if r["program"] and (r["production"] or r["staging"])
|
|
317
|
+
else None
|
|
318
|
+
),
|
|
319
|
+
),
|
|
320
|
+
],
|
|
321
|
+
empty="No environments yet. Each environment gets its own endpoint and keys.",
|
|
322
|
+
)
|
|
323
|
+
if rows:
|
|
324
|
+
live = sum(1 for r in rows if r["production"] or r["staging"])
|
|
325
|
+
ui.info(f"{live} of {len(rows)} serving a program · base URL {base} · model <program>@<alias>")
|
|
326
|
+
|
|
327
|
+
ui.emit({"gateway": base, "endpoints": rows}, human)
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
@app.command("show")
|
|
331
|
+
def show(
|
|
332
|
+
env: str = typer.Argument(..., help="The environment (ID or name, or the start of either)."),
|
|
333
|
+
) -> None:
|
|
334
|
+
"""An endpoint: base URL, model names, the versions on staging and production, recent versions and releases.
|
|
335
|
+
|
|
336
|
+
Example: corethink endpoint show refunds
|
|
337
|
+
"""
|
|
338
|
+
found = _resolve_env(env)
|
|
339
|
+
endpoint = _endpoint(str(found["id"]))
|
|
340
|
+
program = str(endpoint.get("program") or _program_of(found))
|
|
341
|
+
base = _gateway_base(None, endpoint)
|
|
342
|
+
live = _served(endpoint)
|
|
343
|
+
versions = endpoint.get("versions") or []
|
|
344
|
+
labels = {str(v.get("candidate_hash")): _label(v.get("version")) for v in versions}
|
|
345
|
+
history = [
|
|
346
|
+
{"alias": alias, **move} for alias, moves in (endpoint.get("history") or {}).items() for move in (moves or [])
|
|
347
|
+
]
|
|
348
|
+
history.sort(key=lambda move: str(move.get("updated_at") or ""), reverse=True)
|
|
349
|
+
|
|
350
|
+
def human() -> None:
|
|
351
|
+
ui.fields(
|
|
352
|
+
[
|
|
353
|
+
("Environment", f"{found.get('name')} ({found.get('id')})"),
|
|
354
|
+
("Program", program),
|
|
355
|
+
("Base URL", base),
|
|
356
|
+
*[
|
|
357
|
+
(
|
|
358
|
+
alias.capitalize(),
|
|
359
|
+
f"{program}@{alias} · {_label(live[alias].get('version'))} · {_vs_base(live[alias].get('gate'))} vs base"
|
|
360
|
+
if alias in live
|
|
361
|
+
else ("not released yet" if alias == "production" else "nothing on staging"),
|
|
362
|
+
)
|
|
363
|
+
for alias in ALIASES
|
|
364
|
+
],
|
|
365
|
+
(
|
|
366
|
+
"Pinned",
|
|
367
|
+
f"{program}@<version>, e.g. {program}@{next(iter(live.values())).get('version')}" if live else None,
|
|
368
|
+
),
|
|
369
|
+
(
|
|
370
|
+
"Not served",
|
|
371
|
+
"the gateway is not configured here" if endpoint.get("configured") is False else None,
|
|
372
|
+
),
|
|
373
|
+
]
|
|
374
|
+
)
|
|
375
|
+
if versions:
|
|
376
|
+
ui.out.print()
|
|
377
|
+
ui.table(
|
|
378
|
+
versions,
|
|
379
|
+
[
|
|
380
|
+
(
|
|
381
|
+
"Version",
|
|
382
|
+
lambda v: _label(v.get("version") or str(v.get("candidate_hash"))[:12]),
|
|
383
|
+
),
|
|
384
|
+
("On", lambda v: ", ".join(v.get("aliases") or []) or None),
|
|
385
|
+
(
|
|
386
|
+
"Final check",
|
|
387
|
+
lambda v: (
|
|
388
|
+
Text("—", style="dim")
|
|
389
|
+
if not v.get("gate")
|
|
390
|
+
else Text("passed", style="green")
|
|
391
|
+
if (v.get("gate") or {}).get("passed")
|
|
392
|
+
else Text("failed", style="red")
|
|
393
|
+
),
|
|
394
|
+
),
|
|
395
|
+
("vs base", lambda v: _vs_base(v.get("gate"))),
|
|
396
|
+
("Base model", "base_model"),
|
|
397
|
+
("Made", lambda v: ui.when(v.get("created_at"))),
|
|
398
|
+
],
|
|
399
|
+
title="Recent versions",
|
|
400
|
+
)
|
|
401
|
+
if history:
|
|
402
|
+
ui.out.print()
|
|
403
|
+
ui.table(
|
|
404
|
+
history[:8],
|
|
405
|
+
[
|
|
406
|
+
("When", lambda m: ui.when(m.get("updated_at"))),
|
|
407
|
+
("Alias", "alias"),
|
|
408
|
+
(
|
|
409
|
+
"Version",
|
|
410
|
+
lambda m: labels.get(str(m.get("candidate_hash"))) or str(m.get("candidate_hash"))[:12],
|
|
411
|
+
),
|
|
412
|
+
("By", "approved_by"),
|
|
413
|
+
("Reason", "reason"),
|
|
414
|
+
],
|
|
415
|
+
title="Releases",
|
|
416
|
+
)
|
|
417
|
+
usage = endpoint.get("usage") or {}
|
|
418
|
+
if usage.get("requests") is not None:
|
|
419
|
+
ui.info(
|
|
420
|
+
f"Usage: {usage.get('requests'):,} requests · {usage.get('tokens') or 0:,} tokens · {ui.usd(usage.get('cost'))}"
|
|
421
|
+
+ (f" over {usage['window_hours']} h" if usage.get("window_hours") else "")
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
ui.emit({**endpoint, "program": program, "base_url": base, "environment": found.get("id")}, human)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
# ---- releases --------------------------------------------------------------------------------------------------------
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
@app.command("release")
|
|
431
|
+
def release(
|
|
432
|
+
env: str = typer.Argument(..., help="The environment."),
|
|
433
|
+
to: str = typer.Option(..., "--to", help="staging or production."),
|
|
434
|
+
version: str = typer.Option(
|
|
435
|
+
None,
|
|
436
|
+
"--version",
|
|
437
|
+
help="The version: v-…, its hash, or the alias serving it (default for production: staging's).",
|
|
438
|
+
),
|
|
439
|
+
run: str = typer.Option(None, "--run", help="Release the program a run trained (its ID, or the start of it)."),
|
|
440
|
+
canary: float = typer.Option(None, "--canary", help="Canary first: this % of new conversations get it."),
|
|
441
|
+
minutes: int = typer.Option(None, "--minutes", min=1, max=1440, help="How long the canary runs (default 60)."),
|
|
442
|
+
reason: str = typer.Option(None, "--reason", "-m", help="Why (kept in the release history)."),
|
|
443
|
+
wait: bool = typer.Option(True, "--wait/--no-wait", help="Wait for the outcome (default) or return at once."),
|
|
444
|
+
yes: bool = YES_OPTION,
|
|
445
|
+
) -> None:
|
|
446
|
+
"""Release a version to staging or production.
|
|
447
|
+
|
|
448
|
+
Production serves your users: admins and owners only, and only a version with a passing final check (the release job refuses others, and says why). Without --version, production gets staging's version.
|
|
449
|
+
|
|
450
|
+
With --canary, the version first gets a share of new conversations; a judge compares it with the version it replaces every 5 minutes, then promotes it or rolls it back.
|
|
451
|
+
|
|
452
|
+
Examples:
|
|
453
|
+
corethink endpoint release refunds --to production --yes
|
|
454
|
+
corethink endpoint release refunds --to production --canary 10 --minutes 60 -m "new refund policy" --yes
|
|
455
|
+
corethink endpoint release refunds --to staging --run 3f2a1c
|
|
456
|
+
"""
|
|
457
|
+
alias = to.strip().lower()
|
|
458
|
+
if alias not in ALIASES:
|
|
459
|
+
ui.error("--to is staging or production.")
|
|
460
|
+
raise typer.Exit(2)
|
|
461
|
+
if version and run:
|
|
462
|
+
ui.error("Give --version or --run, not both.")
|
|
463
|
+
raise typer.Exit(2)
|
|
464
|
+
found = _resolve_env(env)
|
|
465
|
+
endpoint = _endpoint(str(found["id"]))
|
|
466
|
+
program = str(endpoint.get("program") or _program_of(found))
|
|
467
|
+
body: dict[str, Any] = {"alias": alias, "program": program}
|
|
468
|
+
if run:
|
|
469
|
+
job = _resolve_run(run)
|
|
470
|
+
package = job.get("package_hash")
|
|
471
|
+
if not package:
|
|
472
|
+
ui.error("This run has no trained program to release (it has not finished, or did not pass).")
|
|
473
|
+
raise typer.Exit(2)
|
|
474
|
+
body["packageHash"] = package
|
|
475
|
+
label = f"the program run {str(job.get('job_id'))[:8]} trained"
|
|
476
|
+
fallback = f"trained by job {str(job.get('job_id'))[:8]}"
|
|
477
|
+
else:
|
|
478
|
+
if not version:
|
|
479
|
+
if alias == "staging":
|
|
480
|
+
ui.error("Name what to release: --version (see `corethink endpoint show`) or --run.")
|
|
481
|
+
raise typer.Exit(2)
|
|
482
|
+
if "staging" not in _served(endpoint):
|
|
483
|
+
ui.error("Nothing is on staging to promote. Name a --version, or release a run to staging first.")
|
|
484
|
+
raise typer.Exit(5)
|
|
485
|
+
version = "staging"
|
|
486
|
+
candidate, label = _find_version(endpoint, version)
|
|
487
|
+
current = _served(endpoint).get(alias) or {}
|
|
488
|
+
if current.get("candidate_hash") == candidate:
|
|
489
|
+
ui.info(f"{alias} already serves {label}.")
|
|
490
|
+
return
|
|
491
|
+
if alias == "production" and version == "staging":
|
|
492
|
+
gate = (_served(endpoint).get("staging") or {}).get("gate") or {}
|
|
493
|
+
if not gate.get("passed"):
|
|
494
|
+
ui.warn("Staging's version has no passing final check, so production will likely refuse it.")
|
|
495
|
+
body["candidateHash"] = candidate
|
|
496
|
+
fallback = f"promoted from staging ({label})" if version == "staging" else f"released {label} from the CLI"
|
|
497
|
+
body["reason"] = _reason(reason, fallback)
|
|
498
|
+
body.update(_canary(canary, minutes))
|
|
499
|
+
if alias == "production":
|
|
500
|
+
rollout = f" as a canary for {body['canaryPercent']}% of new conversations" if canary is not None else ""
|
|
501
|
+
ui.confirm(f"Release {label} to {program}@production{rollout}? It serves your users.", yes=yes)
|
|
502
|
+
_release(body, wait=wait, done_text=f"{alias} now serves {label}.")
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
@app.command("promote")
|
|
506
|
+
def promote(
|
|
507
|
+
env: str = typer.Argument(..., help="The environment."),
|
|
508
|
+
canary: float = typer.Option(None, "--canary", help="Canary first: this % of new conversations get it."),
|
|
509
|
+
minutes: int = typer.Option(None, "--minutes", min=1, max=1440, help="How long the canary runs (default 60)."),
|
|
510
|
+
reason: str = typer.Option(None, "--reason", "-m", help="Why (kept in the release history)."),
|
|
511
|
+
wait: bool = typer.Option(True, "--wait/--no-wait", help="Wait for the outcome (default) or return at once."),
|
|
512
|
+
yes: bool = YES_OPTION,
|
|
513
|
+
) -> None:
|
|
514
|
+
"""Release staging's version to production (admins): the same as `release ENV --to production`."""
|
|
515
|
+
release(
|
|
516
|
+
env=env,
|
|
517
|
+
to="production",
|
|
518
|
+
version=None,
|
|
519
|
+
run=None,
|
|
520
|
+
canary=canary,
|
|
521
|
+
minutes=minutes,
|
|
522
|
+
reason=reason,
|
|
523
|
+
wait=wait,
|
|
524
|
+
yes=yes,
|
|
525
|
+
)
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
@app.command("rollback")
|
|
529
|
+
def rollback(
|
|
530
|
+
env: str = typer.Argument(..., help="The environment."),
|
|
531
|
+
alias: str = typer.Option("production", "--alias", help="production (default) or staging."),
|
|
532
|
+
reason: str = typer.Option(None, "--reason", "-m", help="Why (kept in the release history)."),
|
|
533
|
+
wait: bool = typer.Option(True, "--wait/--no-wait", help="Wait for the outcome (default) or return at once."),
|
|
534
|
+
yes: bool = YES_OPTION,
|
|
535
|
+
) -> None:
|
|
536
|
+
"""Roll an alias back to the version it served before. Takes effect within seconds.
|
|
537
|
+
|
|
538
|
+
Example: corethink endpoint rollback refunds -m "answers got worse" --yes
|
|
539
|
+
"""
|
|
540
|
+
alias = alias.strip().lower()
|
|
541
|
+
if alias not in ALIASES:
|
|
542
|
+
ui.error("--alias is staging or production.")
|
|
543
|
+
raise typer.Exit(2)
|
|
544
|
+
found = _resolve_env(env)
|
|
545
|
+
endpoint = _endpoint(str(found["id"]))
|
|
546
|
+
program = str(endpoint.get("program") or _program_of(found))
|
|
547
|
+
current = _served(endpoint).get(alias)
|
|
548
|
+
if not current:
|
|
549
|
+
ui.error(f"Nothing is on {alias} to roll back.")
|
|
550
|
+
raise typer.Exit(5)
|
|
551
|
+
body = {
|
|
552
|
+
"candidateHash": current.get("candidate_hash"),
|
|
553
|
+
"alias": alias,
|
|
554
|
+
"program": program,
|
|
555
|
+
"rollback": True,
|
|
556
|
+
"reason": _reason(reason, f"rolled back {_label(current.get('version'))} from the CLI"),
|
|
557
|
+
}
|
|
558
|
+
ui.confirm(
|
|
559
|
+
f"Roll {program}@{alias} back from {_label(current.get('version'))} to the version it served before?",
|
|
560
|
+
yes=yes,
|
|
561
|
+
)
|
|
562
|
+
_release(body, wait=wait, done_text=f"{alias} rolled back.")
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
@app.command("release-status")
|
|
566
|
+
def release_status(
|
|
567
|
+
release_id: str = typer.Argument(..., help="The release's ID (from `release --no-wait`)."),
|
|
568
|
+
wait: bool = typer.Option(False, "--wait", help="Wait until it is done."),
|
|
569
|
+
) -> None:
|
|
570
|
+
"""Where a release is: pending, done, or refused (and why)."""
|
|
571
|
+
status = client().get(f"/api/v1/harnesses/releases/{quote(release_id, safe='')}") or {}
|
|
572
|
+
if wait and status.get("state") != "done":
|
|
573
|
+
status = _wait_release(release_id)
|
|
574
|
+
if status.get("state") != "done":
|
|
575
|
+
ui.emit({"releaseId": release_id, **status}, lambda: ui.info("Still running."))
|
|
576
|
+
return
|
|
577
|
+
_finish(release_id, status, None, "Done: the alias now serves the version.")
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
# ---- keys ------------------------------------------------------------------------------------------------------------
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def _keys() -> list[dict[str, Any]]:
|
|
584
|
+
return (client().get("/api/v1/harnesses/keys") or {}).get("keys") or []
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def _resolve_key(given: str, keys: list[dict[str, Any]] | None = None) -> dict[str, Any]:
|
|
588
|
+
"""A key by its ID (or the start of it, from `keys list`) or its name."""
|
|
589
|
+
keys = _keys() if keys is None else keys
|
|
590
|
+
folded = given.strip().casefold()
|
|
591
|
+
for test in (
|
|
592
|
+
lambda k: str(k.get("id", "")) == folded,
|
|
593
|
+
lambda k: len(folded) >= 4 and str(k.get("id", "")).startswith(folded),
|
|
594
|
+
lambda k: str(k.get("name", "")).casefold() == folded,
|
|
595
|
+
lambda k: str(k.get("name", "")).casefold().startswith(folded),
|
|
596
|
+
):
|
|
597
|
+
matches = [key for key in keys if test(key)]
|
|
598
|
+
active = [key for key in matches if not key.get("revoked")]
|
|
599
|
+
if len(matches) > 1 and len(active) == 1:
|
|
600
|
+
matches = active
|
|
601
|
+
if matches:
|
|
602
|
+
return _pick(matches, given, "key", lambda k: f"{k.get('name')} {str(k.get('id'))[:10]}")
|
|
603
|
+
ui.error(f"No key matches {given!r}. See `corethink endpoint keys list`.")
|
|
604
|
+
raise typer.Exit(5)
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
def _limits_text(key: dict[str, Any]) -> str:
|
|
608
|
+
limits = key.get("limits") or {}
|
|
609
|
+
period = key.get("period") or {}
|
|
610
|
+
parts = []
|
|
611
|
+
if limits.get("requests_per_day") is not None:
|
|
612
|
+
used = f"{period.get('requests_today', 0):,}/" if period else ""
|
|
613
|
+
parts.append(f"{used}{limits['requests_per_day']:,} req/day")
|
|
614
|
+
if limits.get("usd_per_month") is not None:
|
|
615
|
+
used = f"{ui.usd(period.get('cost_this_month', 0))}/" if period else ""
|
|
616
|
+
parts.append(f"{used}{ui.usd(limits['usd_per_month'])}/month")
|
|
617
|
+
return " · ".join(parts) or "none"
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _status(key: dict[str, Any]) -> Text:
|
|
621
|
+
if key.get("revoked"):
|
|
622
|
+
return Text("revoked", style="dim")
|
|
623
|
+
usage = key.get("usage")
|
|
624
|
+
if usage is not None and not usage.get("requests"):
|
|
625
|
+
return Text("never used", style="yellow")
|
|
626
|
+
return Text("active", style="green")
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def _check_limits(requests: int | None, dollars: float | None) -> None:
|
|
630
|
+
if requests is not None and requests < 1:
|
|
631
|
+
ui.error("Requests per day must be a whole number from 1.")
|
|
632
|
+
raise typer.Exit(2)
|
|
633
|
+
if dollars is not None and dollars < 0.01:
|
|
634
|
+
ui.error("Spend per month must be at least $0.01.")
|
|
635
|
+
raise typer.Exit(2)
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def _show_new_key(made: dict[str, Any], env_name: str) -> None:
|
|
639
|
+
ui.out.print(str(made.get("key")), soft_wrap=True, markup=False, highlight=False)
|
|
640
|
+
ui.info(
|
|
641
|
+
f"Your API key for {env_name}. Copy it now: it is shown only once. Use it as CORETHINK_API_KEY; "
|
|
642
|
+
f"it calls {made.get('program')}@staging and @production only."
|
|
643
|
+
)
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
@keys_app.command("list")
|
|
647
|
+
def keys_list(
|
|
648
|
+
env: str = typer.Argument(None, help="Only this environment's keys."),
|
|
649
|
+
show_all: bool = typer.Option(False, "--all", "-a", help="Revoked keys too."),
|
|
650
|
+
) -> None:
|
|
651
|
+
"""Your workspace's endpoint keys, masked, with what each has done and its limits."""
|
|
652
|
+
keys = _keys()
|
|
653
|
+
if env:
|
|
654
|
+
env_id = _resolve_env(env).get("id")
|
|
655
|
+
keys = [key for key in keys if key.get("environment") == env_id]
|
|
656
|
+
shown = [key for key in keys if show_all or not key.get("revoked")]
|
|
657
|
+
|
|
658
|
+
def human() -> None:
|
|
659
|
+
ui.table(
|
|
660
|
+
shown,
|
|
661
|
+
[
|
|
662
|
+
("ID", lambda k: ui.short_id(k.get("id"), 10)),
|
|
663
|
+
("Name", "name"),
|
|
664
|
+
("Environment", "environment"),
|
|
665
|
+
("Key", "masked"),
|
|
666
|
+
("Status", _status),
|
|
667
|
+
(
|
|
668
|
+
"Requests",
|
|
669
|
+
lambda k: f"{(k.get('usage') or {}).get('requests', 0):,}" if k.get("usage") else None,
|
|
670
|
+
),
|
|
671
|
+
(
|
|
672
|
+
"Cost",
|
|
673
|
+
lambda k: ui.usd((k.get("usage") or {}).get("cost")) if k.get("usage") else None,
|
|
674
|
+
),
|
|
675
|
+
("Last used", lambda k: ui.ago((k.get("usage") or {}).get("last_used_at"))),
|
|
676
|
+
("Limits", _limits_text),
|
|
677
|
+
],
|
|
678
|
+
empty="No keys yet. Make one: corethink endpoint keys create ENV",
|
|
679
|
+
)
|
|
680
|
+
hidden = len(keys) - len(shown)
|
|
681
|
+
if hidden:
|
|
682
|
+
ui.info(f"{hidden} revoked {'key' if hidden == 1 else 'keys'} hidden (--all shows them).")
|
|
683
|
+
|
|
684
|
+
ui.emit(shown, human)
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
@keys_app.command("create")
|
|
688
|
+
def keys_create(
|
|
689
|
+
env: str = typer.Argument(..., help="The environment whose endpoint the key calls."),
|
|
690
|
+
name: str = typer.Option(None, "--name", "-n", help="What it is for, e.g. production-app."),
|
|
691
|
+
requests_per_day: int = typer.Option(None, "--requests-per-day", help="At most this many requests per UTC day."),
|
|
692
|
+
usd_per_month: float = typer.Option(None, "--usd-per-month", help="At most this much spend per UTC month (USD)."),
|
|
693
|
+
) -> None:
|
|
694
|
+
"""Make an API key for an environment's endpoint (admins).
|
|
695
|
+
|
|
696
|
+
The key is printed once: store it as CORETHINK_API_KEY. It calls only this environment's endpoint.
|
|
697
|
+
|
|
698
|
+
Example: corethink endpoint keys create refunds --name web-app --requests-per-day 5000 --usd-per-month 50
|
|
699
|
+
"""
|
|
700
|
+
_check_limits(requests_per_day, usd_per_month)
|
|
701
|
+
found = _resolve_env(env)
|
|
702
|
+
body: dict[str, Any] = {}
|
|
703
|
+
if name:
|
|
704
|
+
body["name"] = name
|
|
705
|
+
if requests_per_day is not None or usd_per_month is not None:
|
|
706
|
+
body["limits"] = {"requests_per_day": requests_per_day, "usd_per_month": usd_per_month}
|
|
707
|
+
made = client().post(_env_path(str(found["id"]), "/keys"), body) or {}
|
|
708
|
+
ui.emit(made, lambda: _show_new_key(made, str(found.get("name") or found.get("id"))))
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
@keys_app.command("rename")
|
|
712
|
+
def keys_rename(
|
|
713
|
+
key: str = typer.Argument(..., help="The key (its ID from `keys list`, or its name)."),
|
|
714
|
+
name: str = typer.Argument(..., help="Its new name."),
|
|
715
|
+
) -> None:
|
|
716
|
+
"""Rename a key (admins)."""
|
|
717
|
+
if not name.strip():
|
|
718
|
+
ui.error("Give the key a name.")
|
|
719
|
+
raise typer.Exit(2)
|
|
720
|
+
target = _resolve_key(key)
|
|
721
|
+
client().post(f"/api/v1/harnesses/keys/{target['id']}:rename", {"name": name.strip()})
|
|
722
|
+
ui.emit(
|
|
723
|
+
{"id": target["id"], "name": name.strip()},
|
|
724
|
+
lambda: ui.success(f"Renamed {target.get('name')} to {name.strip()}."),
|
|
725
|
+
)
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
@keys_app.command("limits")
|
|
729
|
+
def keys_limits(
|
|
730
|
+
key: str = typer.Argument(..., help="The key."),
|
|
731
|
+
requests_per_day: int = typer.Option(None, "--requests-per-day", help="At most this many requests per UTC day."),
|
|
732
|
+
usd_per_month: float = typer.Option(None, "--usd-per-month", help="At most this much spend per UTC month (USD)."),
|
|
733
|
+
no_requests_limit: bool = typer.Option(False, "--no-requests-limit", help="Remove the requests-per-day limit."),
|
|
734
|
+
no_usd_limit: bool = typer.Option(False, "--no-usd-limit", help="Remove the spend-per-month limit."),
|
|
735
|
+
) -> None:
|
|
736
|
+
"""Show or set a key's limits (admins).
|
|
737
|
+
|
|
738
|
+
Requests per UTC day and spend per UTC month. The gateway applies a change within a minute; a key over a limit gets 429.
|
|
739
|
+
|
|
740
|
+
Examples:
|
|
741
|
+
corethink endpoint keys limits web-app
|
|
742
|
+
corethink endpoint keys limits web-app --requests-per-day 10000 --no-usd-limit
|
|
743
|
+
"""
|
|
744
|
+
_check_limits(requests_per_day, usd_per_month)
|
|
745
|
+
if (requests_per_day is not None and no_requests_limit) or (usd_per_month is not None and no_usd_limit):
|
|
746
|
+
ui.error("Set a limit or remove it, not both.")
|
|
747
|
+
raise typer.Exit(2)
|
|
748
|
+
target = _resolve_key(key)
|
|
749
|
+
current = target.get("limits") or {}
|
|
750
|
+
if requests_per_day is None and usd_per_month is None and not no_requests_limit and not no_usd_limit:
|
|
751
|
+
ui.emit(
|
|
752
|
+
{"id": target["id"], "limits": current, "period": target.get("period")},
|
|
753
|
+
lambda: ui.fields(
|
|
754
|
+
[
|
|
755
|
+
("Key", f"{target.get('name')} ({target.get('masked')})"),
|
|
756
|
+
("Limits", _limits_text(target)),
|
|
757
|
+
]
|
|
758
|
+
),
|
|
759
|
+
)
|
|
760
|
+
return
|
|
761
|
+
if target.get("revoked"):
|
|
762
|
+
ui.error("This key is revoked: its limits no longer matter.")
|
|
763
|
+
raise typer.Exit(2)
|
|
764
|
+
limits = {
|
|
765
|
+
"requests_per_day": None
|
|
766
|
+
if no_requests_limit
|
|
767
|
+
else requests_per_day
|
|
768
|
+
if requests_per_day is not None
|
|
769
|
+
else current.get("requests_per_day"),
|
|
770
|
+
"usd_per_month": None
|
|
771
|
+
if no_usd_limit
|
|
772
|
+
else usd_per_month
|
|
773
|
+
if usd_per_month is not None
|
|
774
|
+
else current.get("usd_per_month"),
|
|
775
|
+
}
|
|
776
|
+
client().post(f"/api/v1/harnesses/keys/{target['id']}:limits", limits)
|
|
777
|
+
shown = {**target, "limits": limits, "period": None}
|
|
778
|
+
ui.emit(
|
|
779
|
+
{"id": target["id"], "limits": limits},
|
|
780
|
+
lambda: ui.success(f"{target.get('name')}: {_limits_text(shown)}."),
|
|
781
|
+
)
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
@keys_app.command("revoke")
|
|
785
|
+
def keys_revoke(
|
|
786
|
+
key: str = typer.Argument(..., help="The key."),
|
|
787
|
+
yes: bool = YES_OPTION,
|
|
788
|
+
) -> None:
|
|
789
|
+
"""Revoke a key (admins): the gateway refuses it within a minute. It cannot be turned back on."""
|
|
790
|
+
target = _resolve_key(key)
|
|
791
|
+
if target.get("revoked"):
|
|
792
|
+
ui.info(f"{target.get('name')} is already revoked.")
|
|
793
|
+
return
|
|
794
|
+
ui.confirm(
|
|
795
|
+
f"Revoke {target.get('name')} ({target.get('masked')})? Anything using it stops working within a minute.",
|
|
796
|
+
yes=yes,
|
|
797
|
+
)
|
|
798
|
+
client().post(f"/api/v1/harnesses/keys/{target['id']}:revoke", {})
|
|
799
|
+
ui.emit({"revoked": target["id"]}, lambda: ui.success(f"Revoked {target.get('name')}."))
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
@keys_app.command("rotate")
|
|
803
|
+
def keys_rotate(
|
|
804
|
+
key: str = typer.Argument(..., help="The key to replace."),
|
|
805
|
+
revoke_old: bool = typer.Option(False, "--revoke-old", help="Revoke the old key now (default: later, yourself)."),
|
|
806
|
+
yes: bool = YES_OPTION,
|
|
807
|
+
) -> None:
|
|
808
|
+
"""A new key for the same environment, with the same limits; the old one revoked now (--revoke-old) or later."""
|
|
809
|
+
target = _resolve_key(key)
|
|
810
|
+
if target.get("revoked"):
|
|
811
|
+
ui.error("This key is revoked already: make a new one with `keys create`.")
|
|
812
|
+
raise typer.Exit(2)
|
|
813
|
+
if revoke_old:
|
|
814
|
+
ui.confirm(
|
|
815
|
+
f"Replace {target.get('name')} and revoke it now? Anything using it stops within a minute.",
|
|
816
|
+
yes=yes,
|
|
817
|
+
)
|
|
818
|
+
base = str(target.get("name") or "key").removesuffix(" (rotated)")
|
|
819
|
+
new_name = f"{base[: 60 - len(' (rotated)')].rstrip()} (rotated)"
|
|
820
|
+
body: dict[str, Any] = {"name": new_name}
|
|
821
|
+
limits = target.get("limits") or {}
|
|
822
|
+
if limits.get("requests_per_day") is not None or limits.get("usd_per_month") is not None:
|
|
823
|
+
body["limits"] = {
|
|
824
|
+
"requests_per_day": limits.get("requests_per_day"),
|
|
825
|
+
"usd_per_month": limits.get("usd_per_month"),
|
|
826
|
+
}
|
|
827
|
+
made = client().post(_env_path(str(target.get("environment")), "/keys"), body) or {}
|
|
828
|
+
if revoke_old:
|
|
829
|
+
client().post(f"/api/v1/harnesses/keys/{target['id']}:revoke", {})
|
|
830
|
+
|
|
831
|
+
def human() -> None:
|
|
832
|
+
_show_new_key(made, str(target.get("environment")))
|
|
833
|
+
if revoke_old:
|
|
834
|
+
ui.success(f"Revoked {target.get('name')}.")
|
|
835
|
+
else:
|
|
836
|
+
ui.info(
|
|
837
|
+
f"{target.get('name')} still works: revoke it once nothing uses it (`keys revoke {str(target['id'])[:10]}`)."
|
|
838
|
+
)
|
|
839
|
+
|
|
840
|
+
ui.emit({**made, "replaced": target["id"], "revoked_old": revoke_old}, human)
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
@keys_app.command("usage")
|
|
844
|
+
def keys_usage(
|
|
845
|
+
key: str = typer.Argument(None, help="One key (default: every key of the workspace)."),
|
|
846
|
+
env: str = typer.Option(None, "--env", "-e", help="Only this environment's keys."),
|
|
847
|
+
days: int = typer.Option(30, "--days", "-d", min=1, max=30, help="How many of the last 30 days."),
|
|
848
|
+
) -> None:
|
|
849
|
+
"""Requests, tokens and cost per day over the last 30 days: every key's, one environment's, or one key's.
|
|
850
|
+
|
|
851
|
+
Example: corethink endpoint keys usage --env refunds --days 7
|
|
852
|
+
"""
|
|
853
|
+
if key:
|
|
854
|
+
target = _resolve_key(key)
|
|
855
|
+
data = client().get(f"/api/v1/harnesses/keys/{target['id']}/usage") or {}
|
|
856
|
+
scope = f"{target.get('name')}"
|
|
857
|
+
elif env:
|
|
858
|
+
env_id = str(_resolve_env(env).get("id"))
|
|
859
|
+
data = client().get("/api/v1/harnesses/keys/usage", environment=env_id) or {}
|
|
860
|
+
scope = f"{env_id}'s keys"
|
|
861
|
+
else:
|
|
862
|
+
data = client().get("/api/v1/harnesses/keys/usage") or {}
|
|
863
|
+
scope = "every key"
|
|
864
|
+
daily = data.get("daily")
|
|
865
|
+
if daily is None:
|
|
866
|
+
ui.emit({"daily": None}, lambda: ui.warn("Usage cannot be read here."))
|
|
867
|
+
return
|
|
868
|
+
rows = list(daily)[-days:]
|
|
869
|
+
total = {
|
|
870
|
+
"requests": sum(int(r.get("requests") or 0) for r in rows),
|
|
871
|
+
"tokens": sum(int(r.get("tokens") or 0) for r in rows),
|
|
872
|
+
"cost": sum(float(r.get("cost") or 0) for r in rows),
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
def human() -> None:
|
|
876
|
+
ui.table(
|
|
877
|
+
rows,
|
|
878
|
+
[
|
|
879
|
+
("Day", lambda r: str(r.get("at") or "")[:10]),
|
|
880
|
+
("Requests", lambda r: f"{int(r.get('requests') or 0):,}"),
|
|
881
|
+
("Tokens", lambda r: f"{int(r.get('tokens') or 0):,}"),
|
|
882
|
+
("Cost", lambda r: ui.usd(r.get("cost"), 4)),
|
|
883
|
+
],
|
|
884
|
+
empty="No usage yet.",
|
|
885
|
+
)
|
|
886
|
+
ui.info(
|
|
887
|
+
f"{scope}, last {days} days: {total['requests']:,} requests · {total['tokens']:,} tokens · {ui.usd(total['cost'])}"
|
|
888
|
+
)
|
|
889
|
+
|
|
890
|
+
ui.emit({"daily": rows, "total": total}, human)
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
# ---- calling a program -----------------------------------------------------------------------------------------------
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def _tools_from(path: Path) -> list[dict[str, Any]]:
|
|
897
|
+
"""Tools for the request: an OpenAI tools list, or usecase.json's `tools` (wrapped as functions)."""
|
|
898
|
+
try:
|
|
899
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
900
|
+
except (OSError, ValueError) as error:
|
|
901
|
+
ui.error(f"Cannot read tools from {path}: {error}")
|
|
902
|
+
raise typer.Exit(2) from None
|
|
903
|
+
tools = data.get("tools") if isinstance(data, dict) else data
|
|
904
|
+
if not isinstance(tools, list):
|
|
905
|
+
ui.error(f"{path} has no tools list (a JSON list, or usecase.json with `tools`).")
|
|
906
|
+
raise typer.Exit(2)
|
|
907
|
+
out = []
|
|
908
|
+
for tool in tools:
|
|
909
|
+
if isinstance(tool, dict) and tool.get("type") == "function":
|
|
910
|
+
out.append(tool)
|
|
911
|
+
elif isinstance(tool, dict) and tool.get("name"):
|
|
912
|
+
function = {k: v for k, v in tool.items() if k in ("name", "description", "parameters")}
|
|
913
|
+
out.append({"type": "function", "function": function})
|
|
914
|
+
return out
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
def _gateway_error(response: httpx.Response) -> ApiError:
|
|
918
|
+
try:
|
|
919
|
+
payload = response.json()
|
|
920
|
+
except ValueError:
|
|
921
|
+
payload = None
|
|
922
|
+
message = None
|
|
923
|
+
if isinstance(payload, dict):
|
|
924
|
+
error = payload.get("error")
|
|
925
|
+
if isinstance(error, dict):
|
|
926
|
+
message = error.get("message")
|
|
927
|
+
elif isinstance(error, str):
|
|
928
|
+
message = error
|
|
929
|
+
if not message and payload.get("detail"):
|
|
930
|
+
message = payload["detail"]
|
|
931
|
+
message = str(message or response.text[:200] or "no details")
|
|
932
|
+
lead = {
|
|
933
|
+
401: "The gateway did not accept the key",
|
|
934
|
+
403: "This key cannot call that endpoint",
|
|
935
|
+
404: "The gateway does not know that model",
|
|
936
|
+
429: "Over a limit; wait and try again",
|
|
937
|
+
}.get(response.status_code, "The gateway refused the call")
|
|
938
|
+
return ApiError(
|
|
939
|
+
f"{lead} (HTTP {response.status_code}): {message}",
|
|
940
|
+
status=response.status_code,
|
|
941
|
+
payload=payload,
|
|
942
|
+
)
|
|
943
|
+
|
|
944
|
+
|
|
945
|
+
@app.command("call")
|
|
946
|
+
def call(
|
|
947
|
+
env: str = typer.Argument(..., help="The environment, or a program as <workspace>/<environment>[@alias]."),
|
|
948
|
+
message: str = typer.Argument(None, help="The user's message (- reads it from stdin)."),
|
|
949
|
+
alias: str = typer.Option("staging", "--alias", help="staging (default), production, or a version (harness-…)."),
|
|
950
|
+
key: str = typer.Option(
|
|
951
|
+
None,
|
|
952
|
+
"--key",
|
|
953
|
+
envvar="CORETHINK_API_KEY",
|
|
954
|
+
help="The endpoint key (default CORETHINK_API_KEY; - reads it from stdin). Prefer the variable: argv is visible to others.",
|
|
955
|
+
show_default=False,
|
|
956
|
+
),
|
|
957
|
+
system: str = typer.Option(None, "--system", help="A system message first."),
|
|
958
|
+
messages_file: Path = typer.Option(None, "--messages", help="Earlier turns: a JSON list of chat messages."),
|
|
959
|
+
tools_file: Path = typer.Option(None, "--tools", help="The tools: a JSON list, or your usecase.json."),
|
|
960
|
+
gateway_url: str = GATEWAY_OPTION,
|
|
961
|
+
timeout: float = typer.Option(120.0, "--timeout", help="Seconds to wait for the answer."),
|
|
962
|
+
) -> None:
|
|
963
|
+
"""Send one chat completion to an endpoint and print the answer.
|
|
964
|
+
|
|
965
|
+
With --json, the whole response, including which program version answered (`corethink_program`). The program asks for tool calls the way any OpenAI model does; your code runs them. Calling needs only a key: give the program as <workspace>/<environment> to skip the Workbench lookup.
|
|
966
|
+
|
|
967
|
+
Examples:
|
|
968
|
+
export CORETHINK_API_KEY=ctk_…
|
|
969
|
+
corethink endpoint call refunds "Where is my refund for order A-1001?"
|
|
970
|
+
corethink endpoint call acme/refunds-22982c@production "Hi" --tools usecase.json --json
|
|
971
|
+
"""
|
|
972
|
+
if key == "-":
|
|
973
|
+
key = sys.stdin.readline().strip()
|
|
974
|
+
if message == "-":
|
|
975
|
+
ui.error("Read the key or the message from stdin, not both.")
|
|
976
|
+
raise typer.Exit(2)
|
|
977
|
+
if not key:
|
|
978
|
+
ui.error("Give an endpoint key: set CORETHINK_API_KEY (make one with `corethink endpoint keys create ENV`).")
|
|
979
|
+
raise typer.Exit(2)
|
|
980
|
+
if message == "-":
|
|
981
|
+
message = sys.stdin.read().strip()
|
|
982
|
+
chat: list[dict[str, Any]] = []
|
|
983
|
+
if system:
|
|
984
|
+
chat.append({"role": "system", "content": system})
|
|
985
|
+
if messages_file:
|
|
986
|
+
try:
|
|
987
|
+
earlier = json.loads(messages_file.read_text(encoding="utf-8"))
|
|
988
|
+
except (OSError, ValueError) as error:
|
|
989
|
+
ui.error(f"Cannot read messages from {messages_file}: {error}")
|
|
990
|
+
raise typer.Exit(2) from None
|
|
991
|
+
if not isinstance(earlier, list):
|
|
992
|
+
ui.error(f"{messages_file} must hold a JSON list of messages.")
|
|
993
|
+
raise typer.Exit(2)
|
|
994
|
+
chat.extend(earlier)
|
|
995
|
+
if message:
|
|
996
|
+
chat.append({"role": "user", "content": message})
|
|
997
|
+
if not chat:
|
|
998
|
+
ui.error('Give a message: corethink endpoint call ENV "your message".')
|
|
999
|
+
raise typer.Exit(2)
|
|
1000
|
+
|
|
1001
|
+
program, given_alias, _env = _target(env)
|
|
1002
|
+
model = f"{program}@{given_alias or alias}"
|
|
1003
|
+
base = _gateway_base(gateway_url)
|
|
1004
|
+
body: dict[str, Any] = {"model": model, "messages": chat}
|
|
1005
|
+
if tools_file:
|
|
1006
|
+
body["tools"] = _tools_from(tools_file)
|
|
1007
|
+
try:
|
|
1008
|
+
with (
|
|
1009
|
+
httpx.Client(
|
|
1010
|
+
transport=state.transport, # type: ignore[arg-type]
|
|
1011
|
+
timeout=httpx.Timeout(timeout, connect=15.0),
|
|
1012
|
+
headers={"User-Agent": USER_AGENT},
|
|
1013
|
+
) as http,
|
|
1014
|
+
ui.err.status(f"Calling {model}…", spinner="dots"),
|
|
1015
|
+
):
|
|
1016
|
+
response = http.post(
|
|
1017
|
+
f"{base}/chat/completions",
|
|
1018
|
+
json=body,
|
|
1019
|
+
headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
|
|
1020
|
+
)
|
|
1021
|
+
except httpx.TransportError as error:
|
|
1022
|
+
raise ApiError(f"Could not reach the gateway at {base} ({error.__class__.__name__}).") from error
|
|
1023
|
+
if response.status_code >= 400:
|
|
1024
|
+
raise _gateway_error(response)
|
|
1025
|
+
try:
|
|
1026
|
+
answer = response.json()
|
|
1027
|
+
except ValueError:
|
|
1028
|
+
raise ApiError(f"The gateway answered something that is not JSON (HTTP {response.status_code}).") from None
|
|
1029
|
+
if isinstance(answer, dict) and "corethink_program" not in answer:
|
|
1030
|
+
served = {
|
|
1031
|
+
name.removeprefix("x-corethink-").replace("-", "_"): value
|
|
1032
|
+
for name, value in response.headers.items()
|
|
1033
|
+
if name.lower().startswith("x-corethink-")
|
|
1034
|
+
}
|
|
1035
|
+
if served:
|
|
1036
|
+
answer["corethink_program"] = served
|
|
1037
|
+
|
|
1038
|
+
def human() -> None:
|
|
1039
|
+
choice = ((answer.get("choices") or [{}])[0]) if isinstance(answer, dict) else {}
|
|
1040
|
+
said = choice.get("message") or {}
|
|
1041
|
+
content = said.get("content")
|
|
1042
|
+
if content:
|
|
1043
|
+
sys.stdout.write(str(content).rstrip("\n") + "\n")
|
|
1044
|
+
for tool_call in said.get("tool_calls") or []:
|
|
1045
|
+
function = tool_call.get("function") or {}
|
|
1046
|
+
sys.stdout.write(f"→ {function.get('name')}({function.get('arguments') or ''})\n")
|
|
1047
|
+
sys.stdout.flush()
|
|
1048
|
+
if not content and not said.get("tool_calls"):
|
|
1049
|
+
ui.warn("The program answered with no text and no tool calls.")
|
|
1050
|
+
identity = answer.get("corethink_program") or {}
|
|
1051
|
+
usage = answer.get("usage") or {}
|
|
1052
|
+
who = identity.get("program_id") or program
|
|
1053
|
+
version = identity.get("version") or identity.get("program_version")
|
|
1054
|
+
tail = f" · {usage.get('total_tokens'):,} tokens" if usage.get("total_tokens") is not None else ""
|
|
1055
|
+
via = identity.get("alias") or given_alias or alias
|
|
1056
|
+
ui.info(f"Answered by {who}@{via}" + (f" · {_label(version)}" if version else "") + tail)
|
|
1057
|
+
|
|
1058
|
+
ui.emit(answer, human)
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def _snippets(base: str, model: str) -> dict[str, str]:
|
|
1062
|
+
"""The console's examples (Account.tsx apiSnippets), for this endpoint."""
|
|
1063
|
+
curl = (
|
|
1064
|
+
f"curl {base}/chat/completions \\\n"
|
|
1065
|
+
' -H "Authorization: Bearer $CORETHINK_API_KEY" \\\n'
|
|
1066
|
+
' -H "Content-Type: application/json" \\\n'
|
|
1067
|
+
f' -d \'{{"model": "{model}",\n'
|
|
1068
|
+
' "messages": [{"role": "user", "content": "..."}],\n'
|
|
1069
|
+
' "tools": [...]}\''
|
|
1070
|
+
)
|
|
1071
|
+
python = (
|
|
1072
|
+
"import os\n"
|
|
1073
|
+
"from openai import OpenAI\n"
|
|
1074
|
+
"\n"
|
|
1075
|
+
"client = OpenAI(\n"
|
|
1076
|
+
f' base_url="{base}",\n'
|
|
1077
|
+
' api_key=os.environ["CORETHINK_API_KEY"],\n'
|
|
1078
|
+
")\n"
|
|
1079
|
+
"\n"
|
|
1080
|
+
"client.chat.completions.create(\n"
|
|
1081
|
+
f' model="{model}",\n'
|
|
1082
|
+
' messages=[{"role": "user", "content": "..."}],\n'
|
|
1083
|
+
" tools=[...], # the tools from usecase.json; your code runs the calls\n"
|
|
1084
|
+
")"
|
|
1085
|
+
)
|
|
1086
|
+
return {"curl": curl, "python": python}
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
@app.command("snippet")
|
|
1090
|
+
def snippet(
|
|
1091
|
+
env: str = typer.Argument(..., help="The environment, or a program as <workspace>/<environment>[@alias]."),
|
|
1092
|
+
lang: str = typer.Option("curl", "--lang", "-l", help="curl or python."),
|
|
1093
|
+
alias: str = typer.Option("staging", "--alias", help="staging (default), production, or a version."),
|
|
1094
|
+
gateway_url: str = GATEWAY_OPTION,
|
|
1095
|
+
) -> None:
|
|
1096
|
+
"""A ready-to-paste example that calls an endpoint (curl, or Python with the OpenAI client).
|
|
1097
|
+
|
|
1098
|
+
Example: corethink endpoint snippet refunds --lang python > call.py
|
|
1099
|
+
"""
|
|
1100
|
+
lang = lang.strip().lower()
|
|
1101
|
+
if lang not in ("curl", "python"):
|
|
1102
|
+
ui.error("--lang is curl or python.")
|
|
1103
|
+
raise typer.Exit(2)
|
|
1104
|
+
program, given_alias, _env = _target(env)
|
|
1105
|
+
model = f"{program}@{given_alias or alias}"
|
|
1106
|
+
base = _gateway_base(gateway_url)
|
|
1107
|
+
snippets = _snippets(base, model)
|
|
1108
|
+
data = {"model": model, "base_url": base, **snippets}
|
|
1109
|
+
|
|
1110
|
+
def human() -> None:
|
|
1111
|
+
sys.stdout.write(snippets[lang] + "\n")
|
|
1112
|
+
sys.stdout.flush()
|
|
1113
|
+
ui.info(
|
|
1114
|
+
"Set CORETHINK_API_KEY to a key of this endpoint (`corethink endpoint keys create`). Streaming is not supported yet."
|
|
1115
|
+
)
|
|
1116
|
+
|
|
1117
|
+
ui.emit(data, human)
|
|
1118
|
+
|
|
1119
|
+
|
|
1120
|
+
app.add_typer(keys_app, name="keys")
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
def register(root: typer.Typer) -> None:
|
|
1124
|
+
root.add_typer(app, name="endpoint", rich_help_panel="Ship")
|