lighthouse-cli 0.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lighthouse/__init__.py +6 -0
- lighthouse/__main__.py +9 -0
- lighthouse/commands/__init__.py +0 -0
- lighthouse/commands/agents.py +310 -0
- lighthouse/commands/builds.py +85 -0
- lighthouse/commands/collectors.py +147 -0
- lighthouse/commands/connect.py +717 -0
- lighthouse/commands/connections.py +201 -0
- lighthouse/commands/evals.py +198 -0
- lighthouse/commands/kb.py +224 -0
- lighthouse/commands/language.py +56 -0
- lighthouse/commands/limits.py +61 -0
- lighthouse/commands/login.py +207 -0
- lighthouse/commands/models.py +61 -0
- lighthouse/commands/runs.py +157 -0
- lighthouse/commands/spend.py +84 -0
- lighthouse/commands/stack.py +103 -0
- lighthouse/commands/terms.py +50 -0
- lighthouse/commands/whoami.py +80 -0
- lighthouse/config.py +141 -0
- lighthouse/credentials.py +233 -0
- lighthouse/device.py +124 -0
- lighthouse/exits.py +78 -0
- lighthouse/flags.py +61 -0
- lighthouse/http.py +355 -0
- lighthouse/i18n.py +250 -0
- lighthouse/locales/ar.json +343 -0
- lighthouse/locales/de.json +343 -0
- lighthouse/locales/en.json +343 -0
- lighthouse/locales/es.json +343 -0
- lighthouse/locales/fr.json +343 -0
- lighthouse/locales/ga.json +343 -0
- lighthouse/locales/it.json +343 -0
- lighthouse/locales/ja.json +343 -0
- lighthouse/locales/nl.json +343 -0
- lighthouse/locales/pt.json +343 -0
- lighthouse/locales/zh-Hans.json +343 -0
- lighthouse/locales/zh-Hant.json +343 -0
- lighthouse/main.py +229 -0
- lighthouse/out.py +169 -0
- lighthouse/py.typed +0 -0
- lighthouse/sealing.py +448 -0
- lighthouse_cli-0.16.0.dist-info/METADATA +23 -0
- lighthouse_cli-0.16.0.dist-info/RECORD +49 -0
- lighthouse_cli-0.16.0.dist-info/WHEEL +5 -0
- lighthouse_cli-0.16.0.dist-info/entry_points.txt +2 -0
- lighthouse_cli-0.16.0.dist-info/licenses/LICENSE +202 -0
- lighthouse_cli-0.16.0.dist-info/licenses/NOTICE +2 -0
- lighthouse_cli-0.16.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
# Copyright 2026 Atlantic Agentic Labs. All Rights Reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""`lighthouse connections`: what is connected, how often each syncs, and Sync now.
|
|
4
|
+
|
|
5
|
+
lighthouse connections # every connection, its schedule, next sync
|
|
6
|
+
lighthouse connections schedule cx-abcdef123456 --mode hourly
|
|
7
|
+
lighthouse connections schedule cx-abcdef123456 --mode daily --at 06:30 --tz Europe/London
|
|
8
|
+
lighthouse connections schedule cx-abcdef123456 --mode off
|
|
9
|
+
lighthouse connections sync cx-abcdef123456 --reason "a page was fixed this morning"
|
|
10
|
+
|
|
11
|
+
**The listing** reads `GET /api/v1/connectors`, the same connections the
|
|
12
|
+
Connect page shows, with each one's schedule as the deployment reports it
|
|
13
|
+
(contract 1.17): the mode in words, the last and next sync, and a change that
|
|
14
|
+
was asked for but not yet confirmed, shown as asked for rather than taken for
|
|
15
|
+
what runs.
|
|
16
|
+
|
|
17
|
+
**`schedule` and `sync` act**, so they are the person, not the CLI's token:
|
|
18
|
+
`PUT /api/v1/connections/<id>/schedule` and `POST /api/v1/connections/<id>/sync`
|
|
19
|
+
with the person's own Cloudflare Access session for the tenant host, from
|
|
20
|
+
`cloudflared access token`, exactly as `connect --credential-stdin` does it.
|
|
21
|
+
A delegated token administers nothing (`docs/adr/0012-delegated-credentials.md`),
|
|
22
|
+
and the service refuses a machine key and AAL staff here as it does on the
|
|
23
|
+
card. Each is a signed `connector.schedule` or `connector.sync` command to the
|
|
24
|
+
deployment, audited against the person.
|
|
25
|
+
|
|
26
|
+
**The rules are the service's.** `--mode`, `--at` and `--tz` are sent as
|
|
27
|
+
given and checked by the same function that checks the card
|
|
28
|
+
(`commands.schedule_params`): off, every_15m, hourly, working_hours (with
|
|
29
|
+
`--tz`) or daily (with `--at` HH:MM and `--tz`). A refusal names the field
|
|
30
|
+
and exits 2. Asking for the schedule already asked for sends nothing and says
|
|
31
|
+
so; so does Sync now while one is still in hand.
|
|
32
|
+
"""
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import argparse
|
|
36
|
+
from typing import Any
|
|
37
|
+
|
|
38
|
+
from lighthouse.commands import connect as _connect
|
|
39
|
+
from lighthouse.exits import USAGE, CliError
|
|
40
|
+
from lighthouse.flags import inherited
|
|
41
|
+
from lighthouse.http import Refused
|
|
42
|
+
from lighthouse.main import Command, Context
|
|
43
|
+
from lighthouse.out import Out
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _configure(ap: argparse.ArgumentParser) -> None:
|
|
47
|
+
sub = ap.add_subparsers(dest="connections_command", metavar="<subcommand>")
|
|
48
|
+
sub.add_parser("list", parents=[inherited()],
|
|
49
|
+
help="every connection, its sync schedule and next sync (the default)")
|
|
50
|
+
sched = sub.add_parser("schedule", parents=[inherited()],
|
|
51
|
+
help="set how often a connection syncs (client_admin, as yourself)")
|
|
52
|
+
sched.add_argument("connection", help="the connection's id, cx-...")
|
|
53
|
+
sched.add_argument("--mode", required=True,
|
|
54
|
+
help="off, every_15m, hourly, working_hours or daily")
|
|
55
|
+
sched.add_argument("--at", default=None, help="with --mode daily: the time, 24-hour HH:MM")
|
|
56
|
+
sched.add_argument("--tz", default=None,
|
|
57
|
+
help="with working_hours or daily: a time zone, e.g. Europe/London")
|
|
58
|
+
_source_args(sched)
|
|
59
|
+
sync = sub.add_parser("sync", parents=[inherited()],
|
|
60
|
+
help="sync a connection now (client_admin, as yourself)")
|
|
61
|
+
sync.add_argument("connection", help="the connection's id, cx-...")
|
|
62
|
+
_source_args(sync)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _source_args(ap: argparse.ArgumentParser) -> None:
|
|
66
|
+
ap.add_argument("--card", default=None,
|
|
67
|
+
help="the source, <pack>:<card>, where the deployment reports more than one")
|
|
68
|
+
ap.add_argument("--deployment", default=None,
|
|
69
|
+
help="the deployment, where the source runs in more than one")
|
|
70
|
+
ap.add_argument("--reason", default=None, help="why, for the audit log")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _row(c: dict[str, Any]) -> dict[str, Any]:
|
|
74
|
+
sched = c.get("schedule") or {}
|
|
75
|
+
requested = sched.get("requested") or {}
|
|
76
|
+
return {"id": c.get("id"), "connector": c.get("connector"), "label": c.get("label"),
|
|
77
|
+
"state": c.get("state"), "schedule": sched.get("words") or sched.get("mode"),
|
|
78
|
+
"mode": sched.get("mode"), "pending": bool(sched.get("pending")),
|
|
79
|
+
"requested": requested.get("words") if sched.get("pending") else None,
|
|
80
|
+
"syncing": bool(sched.get("syncing")),
|
|
81
|
+
"last_sync_at": c.get("last_sync_at"), "next_sync_at": c.get("next_sync_at"),
|
|
82
|
+
# 1.27: the deployment's last check, and what to do about it.
|
|
83
|
+
**check_of(c)}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# The deployment's check codes that need the client to act (contract 1.18,
|
|
87
|
+
# and 1.27's two for the finance wave), in plain words. The service's own
|
|
88
|
+
# `health.fix` says what to do where it gives one.
|
|
89
|
+
CHECK_WORDS = {
|
|
90
|
+
"write_capable": "the key can move money: refused, nothing is read",
|
|
91
|
+
"unverified": "the provider does not state the key's permissions: confirm it is view-only",
|
|
92
|
+
"scope_missing": "it signs in but lacks a scope, or grants one that writes",
|
|
93
|
+
"auth_failed": "the provider refused it",
|
|
94
|
+
"absent": "nothing is stored under it",
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def check_of(c: dict[str, Any]) -> dict[str, Any]:
|
|
99
|
+
"""The last check's code, its words, and the fix, for a failing card."""
|
|
100
|
+
health = c.get("health") or {}
|
|
101
|
+
code = str(health.get("code") or "")
|
|
102
|
+
if c.get("state") != "failing" or code not in CHECK_WORDS:
|
|
103
|
+
return {"check": None, "check_words": None, "fix": None}
|
|
104
|
+
return {"check": code, "fix": str(health.get("fix") or "") or None,
|
|
105
|
+
"check_words": CHECK_WORDS[code]}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _list(ctx: Context) -> dict[str, Any]:
|
|
109
|
+
body = ctx.client().get("/api/v1/connectors") or {}
|
|
110
|
+
rows = [_row(c) for c in body.get("connections") or [] if c.get("state") != "disconnected"]
|
|
111
|
+
return {"ok": True, "connections": rows,
|
|
112
|
+
"schedule_modes": [m.get("mode") for m in body.get("schedule_modes") or []]}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _act(ctx: Context, method: str, suffix: str, body: dict[str, Any]) -> dict[str, Any]:
|
|
116
|
+
"""The request, as the person, with the service's refusal said in the
|
|
117
|
+
command line's terms."""
|
|
118
|
+
cid = str(ctx.args.connection)
|
|
119
|
+
host = ctx.host
|
|
120
|
+
client = ctx.new_client(host, None, session=_connect.person_session(host))
|
|
121
|
+
try:
|
|
122
|
+
got = client.request(method, f"/api/v1/connections/{cid}/{suffix}",
|
|
123
|
+
json_body={k: v for k, v in body.items() if v is not None}) or {}
|
|
124
|
+
except Refused as e:
|
|
125
|
+
if e.error == "invalid":
|
|
126
|
+
raise CliError(e.message, code=USAGE, error="invalid",
|
|
127
|
+
hint="see `lighthouse connections schedule --help`") from None
|
|
128
|
+
if e.error == "commands_disabled":
|
|
129
|
+
raise CliError(e.message, code=e.code, error=e.error,
|
|
130
|
+
hint=f"a client administrator switches connector.schedule and "
|
|
131
|
+
f"connector.sync on at https://{host}/command-settings") from None
|
|
132
|
+
raise
|
|
133
|
+
finally:
|
|
134
|
+
client.close()
|
|
135
|
+
command = got.get("command") or {}
|
|
136
|
+
return {"ok": True, "connection": got.get("id") or cid, "connector": got.get("connector"),
|
|
137
|
+
"label": got.get("label"), "unchanged": bool(got.get("unchanged")),
|
|
138
|
+
"command": command.get("command_id"), "command_status": command.get("status"),
|
|
139
|
+
**{k: v for k, v in _row(got).items()
|
|
140
|
+
if k not in ("id", "connector", "label", "state", "check", "check_words", "fix")}}
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _schedule(ctx: Context) -> dict[str, Any]:
|
|
144
|
+
a = ctx.args
|
|
145
|
+
out = _act(ctx, "PUT", "schedule", {"mode": a.mode, "at": a.at, "tz": a.tz, "card": a.card,
|
|
146
|
+
"deployment": a.deployment, "reason": a.reason})
|
|
147
|
+
return {**out, "action": "schedule"}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _sync(ctx: Context) -> dict[str, Any]:
|
|
151
|
+
a = ctx.args
|
|
152
|
+
out = _act(ctx, "POST", "sync", {"card": a.card, "deployment": a.deployment,
|
|
153
|
+
"reason": a.reason})
|
|
154
|
+
return {**out, "action": "sync"}
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def connections(ctx: Context) -> dict[str, Any]:
|
|
158
|
+
which = getattr(ctx.args, "connections_command", None) or "list"
|
|
159
|
+
if which == "schedule":
|
|
160
|
+
return _schedule(ctx)
|
|
161
|
+
if which == "sync":
|
|
162
|
+
return _sync(ctx)
|
|
163
|
+
return _list(ctx)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _when(value: Any) -> str:
|
|
167
|
+
return str(value) if value else "-"
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _show(out: Out, body: dict[str, Any]) -> None:
|
|
171
|
+
if "action" not in body:
|
|
172
|
+
out.table(["connection", "connector", "what", "state", "schedule", "last sync",
|
|
173
|
+
"next sync"],
|
|
174
|
+
[[c["id"], c["connector"], c["label"], c["state"],
|
|
175
|
+
c["schedule"] + (f" (asked: {c['requested']})" if c["requested"] else "")
|
|
176
|
+
+ (" · syncing" if c["syncing"] else ""),
|
|
177
|
+
_when(c["last_sync_at"]), _when(c["next_sync_at"])]
|
|
178
|
+
for c in body["connections"]])
|
|
179
|
+
for c in body["connections"]:
|
|
180
|
+
if c.get("check"):
|
|
181
|
+
out.note(f"{c['id']} ({c['connector']}) is failing: {c['check_words']} "
|
|
182
|
+
f"({c['check']}).{' ' + c['fix'] if c.get('fix') else ''}")
|
|
183
|
+
out.note("`lighthouse connections schedule <id> --mode ...` sets one; "
|
|
184
|
+
"`lighthouse connections sync <id>` syncs one now")
|
|
185
|
+
return
|
|
186
|
+
if body["action"] == "schedule":
|
|
187
|
+
out.pairs([("connection", body["connection"]), ("schedule", body["schedule"]),
|
|
188
|
+
("asked for", body["requested"] or body["schedule"]),
|
|
189
|
+
("next sync", _when(body["next_sync_at"])),
|
|
190
|
+
("command", body["command"] or "")])
|
|
191
|
+
out.note("already asked for: nothing new was sent" if body["unchanged"] else
|
|
192
|
+
"sent to your deployment; the card shows it once the deployment confirms")
|
|
193
|
+
return
|
|
194
|
+
out.pairs([("connection", body["connection"]), ("syncing", "yes"),
|
|
195
|
+
("last sync", _when(body["last_sync_at"])), ("command", body["command"] or "")])
|
|
196
|
+
out.note("a sync is already in hand: nothing new was sent" if body["unchanged"] else
|
|
197
|
+
"sent to your deployment; it syncs at its scheduler's next tick")
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
COMMANDS = (Command("connections", "your connections' sync schedules, set one, or sync now",
|
|
201
|
+
_configure, connections, _show),)
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""`lighthouse evals`: evaluation results, and running a set.
|
|
2
|
+
|
|
3
|
+
lighthouse evals # the latest results
|
|
4
|
+
lighthouse evals list --pack research --json
|
|
5
|
+
lighthouse evals list --fail-below 0.8 # a release gate on metrics.f1
|
|
6
|
+
lighthouse evals run research:summary --cap 5 --wait --fail-below 0.8
|
|
7
|
+
|
|
8
|
+
`evals` reads `GET /api/v1/evals`, every page of it: each row is one measurement
|
|
9
|
+
of one suite for one agent version, with `passed` (did it meet its own bar) and
|
|
10
|
+
`metrics`, a set of named numbers that depends on the suite.
|
|
11
|
+
|
|
12
|
+
**There is no single `score` on a row.** A suite reports whatever measures it
|
|
13
|
+
(`f1` is the one the API documents). So `--fail-below N` gates on one named
|
|
14
|
+
metric, `--metric` (default `f1`), and **fails closed**: a row with no number
|
|
15
|
+
under that name cannot be shown to meet the bar, so the command exits 1 and says
|
|
16
|
+
which rows. A release gate that passed on what it could not read would be worse
|
|
17
|
+
than none.
|
|
18
|
+
|
|
19
|
+
**`evals run` asks your deployment to run the agent's scheduled evaluation
|
|
20
|
+
once** (contract 1.19, `POST /api/v1/agents/{agent}/evals`), under the spend
|
|
21
|
+
cap you give (`--cap`, required: nothing runs without one), optionally only
|
|
22
|
+
one eval set (`--set`, the eval card's id). It needs `eval.run` switched on
|
|
23
|
+
by your organisation and by your deployment's own policy, and a person's
|
|
24
|
+
token with the `deploy` scope (a machine key cannot spend). The deployment's
|
|
25
|
+
scheduler runs it at its next tick, every 15 minutes by default.
|
|
26
|
+
|
|
27
|
+
`--wait` follows it: the command until your deployment accepts it, then the
|
|
28
|
+
evaluation rows it produces for that agent. The run **fails** (exit 1) when a
|
|
29
|
+
new row did not pass, and, with `--fail-below N`, when a new row's metric is
|
|
30
|
+
below N or missing (the same closed gate as `evals list --fail-below`). In a
|
|
31
|
+
pipeline:
|
|
32
|
+
|
|
33
|
+
lighthouse evals run research:summary --cap 5 --wait --fail-below 0.8 \
|
|
34
|
+
&& lighthouse agents deploy research:summary --version "$VERSION" --wait
|
|
35
|
+
"""
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import argparse
|
|
39
|
+
from typing import Any
|
|
40
|
+
|
|
41
|
+
from lighthouse.exits import FAILED, USAGE, CliError
|
|
42
|
+
from lighthouse.flags import inherited
|
|
43
|
+
from lighthouse.main import Command, Context
|
|
44
|
+
from lighthouse.out import Out
|
|
45
|
+
|
|
46
|
+
DEFAULT_METRIC = "f1"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _configure(ap: argparse.ArgumentParser) -> None:
|
|
50
|
+
sub = ap.add_subparsers(dest="evals_command", metavar="<subcommand>")
|
|
51
|
+
listing = sub.add_parser("list", parents=[inherited()], help="the latest results (the default)")
|
|
52
|
+
listing.add_argument("--pack")
|
|
53
|
+
listing.add_argument("--family")
|
|
54
|
+
listing.add_argument("--fail-below", type=float, dest="fail_below",
|
|
55
|
+
help="exit 1 if any row's metric is below this, or has no such metric")
|
|
56
|
+
listing.add_argument("--metric", default=DEFAULT_METRIC,
|
|
57
|
+
help="the metric --fail-below reads from each row's `metrics` "
|
|
58
|
+
f"(default: {DEFAULT_METRIC})")
|
|
59
|
+
run = sub.add_parser("run", parents=[inherited()],
|
|
60
|
+
help="run an agent's evaluation once, under a spend cap")
|
|
61
|
+
run.add_argument("agent", help="pack:card, e.g. research:summary")
|
|
62
|
+
run.add_argument("--cap", type=float, required=True, dest="cap",
|
|
63
|
+
help="the most this run may spend, in US dollars")
|
|
64
|
+
run.add_argument("--set", dest="eval_set", help="one eval set (its eval card's id)")
|
|
65
|
+
run.add_argument("--deployment", help="the deployment, where the tenant has more than one")
|
|
66
|
+
run.add_argument("--reason", default=None, help="why, for the audit log")
|
|
67
|
+
run.add_argument("--wait", action="store_true",
|
|
68
|
+
help="wait for the evaluation's results, and gate on them")
|
|
69
|
+
run.add_argument("--wait-timeout", type=float, default=1800.0, dest="wait_timeout",
|
|
70
|
+
help="seconds to wait at most (default 1800); then exit 5")
|
|
71
|
+
run.add_argument("--fail-below", type=float, dest="fail_below",
|
|
72
|
+
help="with --wait: exit 1 if a new row's metric is below this, or missing")
|
|
73
|
+
run.add_argument("--metric", default=DEFAULT_METRIC,
|
|
74
|
+
help=f"the metric --fail-below reads (default: {DEFAULT_METRIC})")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def evals(ctx: Context) -> dict[str, Any]:
|
|
78
|
+
if getattr(ctx.args, "evals_command", None) == "run":
|
|
79
|
+
return _run(ctx)
|
|
80
|
+
body = ctx.client().get_all("/api/v1/evals", "evals", pack=getattr(ctx.args, "pack", None),
|
|
81
|
+
family=getattr(ctx.args, "family", None))
|
|
82
|
+
floor = getattr(ctx.args, "fail_below", None)
|
|
83
|
+
if floor is None:
|
|
84
|
+
return {"ok": True, **body}
|
|
85
|
+
return gate(body, floor, getattr(ctx.args, "metric", None) or DEFAULT_METRIC)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _run(ctx: Context) -> dict[str, Any]:
|
|
89
|
+
from lighthouse.commands.agents import wait_for
|
|
90
|
+
a = ctx.args
|
|
91
|
+
agent = str(a.agent)
|
|
92
|
+
pack, sep, card = agent.partition(":")
|
|
93
|
+
if not sep or not pack or not card:
|
|
94
|
+
raise CliError("the agent is pack:card, e.g. research:summary", code=USAGE,
|
|
95
|
+
error="usage")
|
|
96
|
+
if a.cap <= 0:
|
|
97
|
+
raise CliError("--cap is above zero: nothing runs without a spend cap", code=USAGE,
|
|
98
|
+
error="usage")
|
|
99
|
+
if a.fail_below is not None and not a.wait:
|
|
100
|
+
raise CliError("--fail-below gates on the results, so it needs --wait", code=USAGE,
|
|
101
|
+
error="usage")
|
|
102
|
+
body: dict[str, Any] = {"cap_usd": f"{a.cap:.2f}",
|
|
103
|
+
"reason": a.reason or "evaluation run from the lighthouse CLI"}
|
|
104
|
+
if a.eval_set:
|
|
105
|
+
body["set"] = a.eval_set
|
|
106
|
+
if a.deployment:
|
|
107
|
+
body["deployment"] = a.deployment
|
|
108
|
+
client = ctx.client()
|
|
109
|
+
cmd = client.post(f"/api/v1/agents/{agent}/evals", body)
|
|
110
|
+
out: dict[str, Any] = {"ok": True, "action": "run", "agent": agent,
|
|
111
|
+
"command": cmd.get("command_id"), "status": cmd.get("status"),
|
|
112
|
+
"cap_usd": (cmd.get("params") or {}).get("cap_usd", body["cap_usd"])}
|
|
113
|
+
if not a.wait:
|
|
114
|
+
return out
|
|
115
|
+
cid = cmd["command_id"]
|
|
116
|
+
got = wait_for(ctx, lambda: client.get(f"/api/v1/agents/{agent}/commands/{cid}"),
|
|
117
|
+
lambda d: d.get("status") in ("applied", "refused", "failed", "expired"),
|
|
118
|
+
"your deployment to accept the evaluation")
|
|
119
|
+
if got.get("status") != "applied":
|
|
120
|
+
raise CliError(f"your deployment did not take the evaluation: {got.get('status')}"
|
|
121
|
+
+ (f" ({got.get('reason')})" if got.get("reason") else ""),
|
|
122
|
+
code=FAILED, error=f"command_{got.get('status')}", command=cid,
|
|
123
|
+
reason=got.get("reason"))
|
|
124
|
+
since = _when(got.get("updated_at") or cmd.get("issued_at"))
|
|
125
|
+
|
|
126
|
+
def fresh() -> list[dict[str, Any]]:
|
|
127
|
+
rows = client.get_all("/api/v1/evals", "evals", pack=pack).get("evals") or []
|
|
128
|
+
return [r for r in rows if str(r.get("agent")).split(":")[-1] == card
|
|
129
|
+
and since is not None and (_when(r.get("ts")) or since) >= since]
|
|
130
|
+
rows = wait_for(ctx, lambda: {"rows": fresh()}, lambda d: bool(d["rows"]),
|
|
131
|
+
"the evaluation's results (it runs at the scheduler's next tick)")["rows"]
|
|
132
|
+
out.update({"status": "measured", "evals": rows})
|
|
133
|
+
failed = [_name(r) for r in rows if r.get("passed") is False]
|
|
134
|
+
if a.fail_below is not None:
|
|
135
|
+
gate({"evals": rows}, a.fail_below, a.metric or DEFAULT_METRIC)
|
|
136
|
+
if failed:
|
|
137
|
+
raise CliError(f"{len(failed)} evaluation row(s) did not pass", code=FAILED,
|
|
138
|
+
error="evaluation_failed", failed=failed)
|
|
139
|
+
return out
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def gate(body: dict[str, Any], floor: float, metric: str) -> dict[str, Any]:
|
|
143
|
+
"""`--fail-below`: every row's `metric` at or above `floor`, failing
|
|
144
|
+
closed on a row that has none (B6)."""
|
|
145
|
+
rows = body.get("evals") or []
|
|
146
|
+
if not rows:
|
|
147
|
+
raise CliError(f"cannot apply --fail-below {floor}: there are no evaluation results to "
|
|
148
|
+
f"read, and a gate with nothing to check fails closed",
|
|
149
|
+
code=FAILED, error="no_results")
|
|
150
|
+
unreadable = [r for r in rows if not _number((r.get("metrics") or {}).get(metric))]
|
|
151
|
+
below = [r for r in rows if r not in unreadable
|
|
152
|
+
and float(r["metrics"][metric]) < floor]
|
|
153
|
+
if unreadable:
|
|
154
|
+
raise CliError(
|
|
155
|
+
f"cannot apply --fail-below {floor}: {len(unreadable)} of {len(rows)} evaluation "
|
|
156
|
+
f"row(s) have no numeric `{metric}` in their metrics. The API gives a row "
|
|
157
|
+
f"`passed` and a suite-specific `metrics` object, not one score, so the gate "
|
|
158
|
+
f"fails closed", code=FAILED, error="no_score",
|
|
159
|
+
hint="pass --metric <name> for a metric these suites report, or filter with "
|
|
160
|
+
"--pack/--family",
|
|
161
|
+
unreadable=[_name(r) for r in unreadable])
|
|
162
|
+
if below:
|
|
163
|
+
raise CliError(f"{len(below)} evaluation row(s) have {metric} below {floor}",
|
|
164
|
+
code=FAILED, error="below_threshold", below=[_name(r) for r in below])
|
|
165
|
+
return {"ok": True, **body, "below": []}
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _when(value: Any) -> Any:
|
|
169
|
+
from datetime import datetime
|
|
170
|
+
try:
|
|
171
|
+
return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
172
|
+
except ValueError:
|
|
173
|
+
return None
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _number(value: Any) -> bool:
|
|
177
|
+
return isinstance(value, (int, float)) and not isinstance(value, bool)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _name(row: dict[str, Any]) -> str:
|
|
181
|
+
return ":".join(str(row.get(k)) for k in ("agent", "family", "suite") if row.get(k))
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _show(out: Out, body: dict[str, Any]) -> None:
|
|
185
|
+
if body.get("action") == "run" and "evals" not in body:
|
|
186
|
+
out.pairs([("agent", body["agent"]), ("cap", f"${body['cap_usd']}"),
|
|
187
|
+
("command", body.get("command") or ""), ("status", body.get("status") or "")])
|
|
188
|
+
out.note("your deployment runs it at its scheduler's next tick; `--wait` follows it")
|
|
189
|
+
return
|
|
190
|
+
out.table(["agent", "pack", "family", "suite", "version", "passed", "metrics", "measured"],
|
|
191
|
+
[[r.get("agent"), r.get("pack"), r.get("family"), r.get("suite"), r.get("version"),
|
|
192
|
+
"yes" if r.get("passed") else "no",
|
|
193
|
+
", ".join(f"{k}={v}" for k, v in sorted((r.get("metrics") or {}).items())),
|
|
194
|
+
r.get("ts")] for r in body["evals"]])
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
COMMANDS = (Command("evals", "evaluation results, and running a set",
|
|
198
|
+
_configure, evals, _show),)
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
# Copyright 2026 Atlantic Agentic Labs. All Rights Reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""`lighthouse kb`: knowledge bases, their freshness, and ingesting into them.
|
|
4
|
+
|
|
5
|
+
lighthouse kb # every knowledge base, with its freshness
|
|
6
|
+
lighthouse kb show <index>
|
|
7
|
+
lighthouse kb ingest ops --source confluence --wait
|
|
8
|
+
lighthouse kb ingest ops --source cx-abcdef123456
|
|
9
|
+
lighthouse kb upload <file> [--deployment NAME] (contract 1.21)
|
|
10
|
+
|
|
11
|
+
`kb` and `kb show` read `GET /api/v1/knowledge-bases` and
|
|
12
|
+
`/api/v1/knowledge-bases/{index}` (contract 1.14): the indexes, what is in
|
|
13
|
+
them, when each was last refreshed and where it is held — which, for every
|
|
14
|
+
client, is the client's own account. Nothing is copied to AAL
|
|
15
|
+
(https://docs.atlanticagenticlabs.com/).
|
|
16
|
+
|
|
17
|
+
**`kb ingest` is `connections sync` by another name**, kept because the docs
|
|
18
|
+
and people's scripts use it. `--source` is a connection id (`cx-...`) or a
|
|
19
|
+
connector (`confluence`, `github`...); a connector with more than one live
|
|
20
|
+
connection is narrowed by `--repo` (matched against the connection's label
|
|
21
|
+
and target) or refused with the ids to choose from. It then sends
|
|
22
|
+
`POST /api/v1/connections/<id>/sync` as you, with your own Cloudflare Access
|
|
23
|
+
session (`cloudflared access login https://<host>`), exactly as
|
|
24
|
+
`connections sync` does, to the knowledge base's own deployment.
|
|
25
|
+
|
|
26
|
+
`--wait` polls the connection's schedule record, which the deployment
|
|
27
|
+
reports, until a sync newer than the request has finished: exit 0 when it
|
|
28
|
+
reports `ok`, exit 1 when it reports `failed`, exit 5 at `--wait-timeout`.
|
|
29
|
+
|
|
30
|
+
`kb upload` sends a file to the deployment's own uploads store without it
|
|
31
|
+
passing through LightHouse: it asks `POST /api/v1/uploads` for a link (the
|
|
32
|
+
deployment mints it, `upload.prepare`), waits for the link, and PUTs the file
|
|
33
|
+
straight to it. The deployment's uploads source indexes it at its next sync.
|
|
34
|
+
"""
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import argparse
|
|
38
|
+
from typing import Any
|
|
39
|
+
|
|
40
|
+
from lighthouse.commands import connect as _connect
|
|
41
|
+
from lighthouse.exits import FAILED, USAGE, CliError
|
|
42
|
+
from lighthouse.flags import inherited
|
|
43
|
+
from lighthouse.main import Command, Context
|
|
44
|
+
from lighthouse.out import Out
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _configure(ap: argparse.ArgumentParser) -> None:
|
|
49
|
+
sub = ap.add_subparsers(dest="kb_command", metavar="<subcommand>")
|
|
50
|
+
sub.add_parser("list", parents=[inherited()], help="every knowledge base (the default)")
|
|
51
|
+
show = sub.add_parser("show", parents=[inherited()], help="one knowledge base")
|
|
52
|
+
show.add_argument("index", help="the index's name")
|
|
53
|
+
ingest = sub.add_parser("ingest", parents=[inherited()],
|
|
54
|
+
help="sync a connection into a knowledge base now (as yourself)")
|
|
55
|
+
ingest.add_argument("index")
|
|
56
|
+
ingest.add_argument("--source", required=True,
|
|
57
|
+
help="a connection id (cx-...) or a connector: github, gitlab, "
|
|
58
|
+
"confluence, jira, gdrive…")
|
|
59
|
+
ingest.add_argument("--repo", help="narrow a connector to one connection by its label or "
|
|
60
|
+
"target, e.g. owner/name")
|
|
61
|
+
ingest.add_argument("--path", help=argparse.SUPPRESS)
|
|
62
|
+
ingest.add_argument("--reason", default=None, help="why, for the audit log")
|
|
63
|
+
ingest.add_argument("--wait", action="store_true", help="wait for the sync to finish")
|
|
64
|
+
ingest.add_argument("--wait-timeout", type=float, default=1800.0, dest="wait_timeout",
|
|
65
|
+
help="seconds to wait at most (default 1800); then exit 5")
|
|
66
|
+
upload = sub.add_parser("upload", parents=[inherited()],
|
|
67
|
+
help="upload a file straight to your deployment's own store")
|
|
68
|
+
upload.add_argument("file", help="the file: .md, .txt, .csv, .pdf, .docx, .pptx, .xlsx…")
|
|
69
|
+
upload.add_argument("--deployment", help="which deployment, where you have more than one")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
LINK_WAIT_S = 120
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _upload(ctx: Context) -> dict[str, Any]:
|
|
76
|
+
"""Ask for a link, wait for the deployment to mint it, PUT the file to it.
|
|
77
|
+
The file's bytes go only to the link: never to LightHouse."""
|
|
78
|
+
import time
|
|
79
|
+
from pathlib import Path
|
|
80
|
+
|
|
81
|
+
import httpx
|
|
82
|
+
path = Path(ctx.args.file)
|
|
83
|
+
if not path.is_file():
|
|
84
|
+
raise CliError(f"{path} is not a file", code=USAGE, error="usage")
|
|
85
|
+
data = path.read_bytes()
|
|
86
|
+
client = ctx.client()
|
|
87
|
+
body: dict[str, Any] = {"filename": path.name, "size": len(data)}
|
|
88
|
+
if ctx.args.deployment:
|
|
89
|
+
body["deployment"] = ctx.args.deployment
|
|
90
|
+
up = client.post("/api/v1/uploads", body)
|
|
91
|
+
sleep = ctx.sleep or time.sleep
|
|
92
|
+
for _ in range(LINK_WAIT_S):
|
|
93
|
+
if up.get("link") or up.get("status") in ("refused", "failed", "expired"):
|
|
94
|
+
break
|
|
95
|
+
sleep(1)
|
|
96
|
+
up = client.get(f"/api/v1/uploads/{up['upload_id']}")
|
|
97
|
+
link = up.get("link")
|
|
98
|
+
if not link:
|
|
99
|
+
raise CliError(f"no upload link: {up.get('about') or up.get('status')}",
|
|
100
|
+
error=str(up.get("reason") or up.get("status") or "no_link"),
|
|
101
|
+
hint="switch upload.prepare on in LightHouse's command settings and in "
|
|
102
|
+
"your deployment's own policy")
|
|
103
|
+
try:
|
|
104
|
+
with httpx.Client(timeout=300.0, transport=ctx.transport,
|
|
105
|
+
follow_redirects=False) as raw:
|
|
106
|
+
got = raw.request(link["method"], link["url"], content=data,
|
|
107
|
+
headers=dict(link.get("headers") or {}))
|
|
108
|
+
except httpx.HTTPError as e:
|
|
109
|
+
raise CliError(f"the upload to your store did not complete: {type(e).__name__}",
|
|
110
|
+
error="upload_failed") from None
|
|
111
|
+
if not 200 <= got.status_code < 300:
|
|
112
|
+
raise CliError(f"your store refused the upload (HTTP {got.status_code})",
|
|
113
|
+
error="upload_refused")
|
|
114
|
+
return {"ok": True, "upload_id": up["upload_id"], "key": up.get("key"),
|
|
115
|
+
"deployment": up.get("deployment"), "bytes": len(data),
|
|
116
|
+
"indexed": "at your deployment's next sync of its uploads source"}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def kb(ctx: Context) -> dict[str, Any]:
|
|
120
|
+
which = getattr(ctx.args, "kb_command", None) or "list"
|
|
121
|
+
if which == "upload":
|
|
122
|
+
return _upload(ctx)
|
|
123
|
+
if which == "ingest":
|
|
124
|
+
return _ingest(ctx)
|
|
125
|
+
if which == "show":
|
|
126
|
+
index = ctx.args.index
|
|
127
|
+
if not index:
|
|
128
|
+
raise CliError("which knowledge base?", code=USAGE, error="usage")
|
|
129
|
+
return {"ok": True, **ctx.client().get(f"/api/v1/knowledge-bases/{index}")}
|
|
130
|
+
return {"ok": True, **ctx.client().get("/api/v1/knowledge-bases")}
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _resolve(ctx: Context, connections: list[dict[str, Any]]) -> dict[str, Any]:
|
|
134
|
+
a = ctx.args
|
|
135
|
+
live = [c for c in connections if c.get("state") != "disconnected"]
|
|
136
|
+
hit = [c for c in live if c.get("id") == a.source]
|
|
137
|
+
if not hit:
|
|
138
|
+
hit = [c for c in live if c.get("connector") == str(a.source).lower()]
|
|
139
|
+
if a.repo:
|
|
140
|
+
want = str(a.repo).lower()
|
|
141
|
+
hit = [c for c in hit if want in str(c.get("label") or "").lower()
|
|
142
|
+
or want in " ".join(str(v) for v in (c.get("target") or {}).values()).lower()]
|
|
143
|
+
if not hit:
|
|
144
|
+
raise CliError(f"no live connection matches {a.source!r}"
|
|
145
|
+
+ (f" and {a.repo!r}" if a.repo else ""), code=USAGE,
|
|
146
|
+
error="no_connection", hint="`lighthouse connections` lists them")
|
|
147
|
+
if len(hit) > 1:
|
|
148
|
+
raise CliError(f"{len(hit)} connections match {a.source!r}: name one", code=USAGE,
|
|
149
|
+
error="ambiguous_connection",
|
|
150
|
+
hint="pass its id as --source, or --repo; one of: "
|
|
151
|
+
+ ", ".join(sorted(str(c.get("id")) for c in hit)))
|
|
152
|
+
return hit[0]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _ingest(ctx: Context) -> dict[str, Any]:
|
|
156
|
+
from lighthouse.commands.agents import wait_for
|
|
157
|
+
from lighthouse.commands.connections import _row
|
|
158
|
+
a = ctx.args
|
|
159
|
+
base = ctx.client().get(f"/api/v1/knowledge-bases/{a.index}")
|
|
160
|
+
conn = _resolve(ctx, ctx.client().get("/api/v1/connectors").get("connections") or [])
|
|
161
|
+
cid = str(conn["id"])
|
|
162
|
+
before = (conn.get("schedule") or {}).get("last_sync_at")
|
|
163
|
+
host = ctx.host
|
|
164
|
+
person = ctx.new_client(host, None, session=_connect.person_session(host))
|
|
165
|
+
try:
|
|
166
|
+
got = person.post(f"/api/v1/connections/{cid}/sync", {k: v for k, v in {
|
|
167
|
+
"deployment": base.get("deployment"),
|
|
168
|
+
"reason": a.reason or f"ingest into {a.index} from the lighthouse CLI"}.items()
|
|
169
|
+
if v is not None}) or {}
|
|
170
|
+
finally:
|
|
171
|
+
person.close()
|
|
172
|
+
out: dict[str, Any] = {"ok": True, "action": "ingest", "index": a.index,
|
|
173
|
+
"connection": cid, "connector": conn.get("connector"),
|
|
174
|
+
"unchanged": bool(got.get("unchanged")),
|
|
175
|
+
"command": (got.get("command") or {}).get("command_id"),
|
|
176
|
+
"status": "requested"}
|
|
177
|
+
if not a.wait:
|
|
178
|
+
return out
|
|
179
|
+
|
|
180
|
+
def read() -> dict[str, Any]:
|
|
181
|
+
rows = ctx.client().get("/api/v1/connectors").get("connections") or []
|
|
182
|
+
return next((c for c in rows if c.get("id") == cid), {})
|
|
183
|
+
|
|
184
|
+
def finished(c: dict[str, Any]) -> bool:
|
|
185
|
+
s = c.get("schedule") or {}
|
|
186
|
+
return not s.get("syncing") and bool(s.get("last_sync_at")) \
|
|
187
|
+
and s.get("last_sync_at") != before
|
|
188
|
+
done = wait_for(ctx, read, finished, "the sync to finish (it runs at the scheduler's next "
|
|
189
|
+
"tick)")
|
|
190
|
+
sched = done.get("schedule") or {}
|
|
191
|
+
out.update({"status": sched.get("last_status") or "unknown",
|
|
192
|
+
"last_sync_at": sched.get("last_sync_at"), **{
|
|
193
|
+
k: v for k, v in _row(done).items() if k == "next_sync_at"}})
|
|
194
|
+
if sched.get("last_status") == "failed":
|
|
195
|
+
raise CliError(f"the sync of {cid} failed in your deployment", code=FAILED,
|
|
196
|
+
error="sync_failed", connection=cid)
|
|
197
|
+
return out
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _show(out: Out, body: dict[str, Any]) -> None:
|
|
201
|
+
if body.get("action") == "ingest":
|
|
202
|
+
out.pairs([("knowledge base", body["index"]), ("connection", body["connection"]),
|
|
203
|
+
("status", body["status"]), ("command", body.get("command") or "")])
|
|
204
|
+
out.note("a sync was already in hand: nothing new was sent" if body["unchanged"] else
|
|
205
|
+
"sent to your deployment; it syncs at its scheduler's next tick")
|
|
206
|
+
return
|
|
207
|
+
bases = body.get("items")
|
|
208
|
+
if "upload_id" in body:
|
|
209
|
+
out.pairs([("uploaded", body.get("key")), ("bytes", body.get("bytes")),
|
|
210
|
+
("deployment", body.get("deployment")), ("indexed", body.get("indexed"))])
|
|
211
|
+
out.note("the file went straight to your own store; LightHouse never saw it")
|
|
212
|
+
return
|
|
213
|
+
if bases is None:
|
|
214
|
+
out.pairs([(k, v) for k, v in body.items()
|
|
215
|
+
if k != "ok" and not isinstance(v, (list, dict))])
|
|
216
|
+
return
|
|
217
|
+
out.table(["index", "documents", "chunks", "last refreshed", "health", "backend"],
|
|
218
|
+
[[b.get("index"), b.get("documents"), b.get("chunks"), b.get("last_refreshed"),
|
|
219
|
+
(b.get("health") or {}).get("grade"), b.get("backend")] for b in bases])
|
|
220
|
+
out.note("every index is in the client's own account; nothing is copied to AAL")
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
COMMANDS = (Command("kb", "knowledge bases, their freshness, and ingesting into them",
|
|
224
|
+
_configure, kb, _show),)
|