kctl-agent 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kctl_agent/__init__.py +5 -0
- kctl_agent/__main__.py +4 -0
- kctl_agent/cli.py +95 -0
- kctl_agent/commands/__init__.py +0 -0
- kctl_agent/commands/_shared.py +122 -0
- kctl_agent/commands/doctor_cmd.py +146 -0
- kctl_agent/commands/eval_cmd.py +116 -0
- kctl_agent/commands/knowledge_cmd.py +93 -0
- kctl_agent/commands/lint_cmd.py +77 -0
- kctl_agent/commands/manifest_cmd.py +226 -0
- kctl_agent/commands/reconcile_cmd.py +117 -0
- kctl_agent/commands/vendor_cmd.py +79 -0
- kctl_agent/core/__init__.py +0 -0
- kctl_agent/core/callbacks.py +20 -0
- kctl_agent/core/config.py +19 -0
- kctl_agent/core/exceptions.py +19 -0
- kctl_agent/evals.py +135 -0
- kctl_agent/health.py +196 -0
- kctl_agent/hoist.py +55 -0
- kctl_agent/hooks.py +72 -0
- kctl_agent/importer.py +341 -0
- kctl_agent/knowledge.py +90 -0
- kctl_agent/lint.py +47 -0
- kctl_agent/lockfile.py +122 -0
- kctl_agent/manifest.py +271 -0
- kctl_agent/model.py +258 -0
- kctl_agent/parser.py +117 -0
- kctl_agent/reconcile.py +189 -0
- kctl_agent/registry.py +127 -0
- kctl_agent/renderers/__init__.py +76 -0
- kctl_agent/renderers/copytree.py +113 -0
- kctl_agent/renderers/jsonmerge.py +277 -0
- kctl_agent/renderers/plugin.py +129 -0
- kctl_agent/renderers/symlink.py +180 -0
- kctl_agent/rules/__init__.py +12 -0
- kctl_agent/rules/_shared.py +88 -0
- kctl_agent/rules/commands.py +126 -0
- kctl_agent/rules/evals.py +85 -0
- kctl_agent/rules/external.py +158 -0
- kctl_agent/rules/hooks.py +141 -0
- kctl_agent/rules/knowledge.py +113 -0
- kctl_agent/rules/mcp.py +162 -0
- kctl_agent/rules/registry.py +58 -0
- kctl_agent/rules/rules.py +110 -0
- kctl_agent/rules/subagents.py +176 -0
- kctl_agent/selection.py +37 -0
- kctl_agent/state.py +142 -0
- kctl_agent/vendor.py +139 -0
- kctl_agent-0.2.0.dist-info/METADATA +11 -0
- kctl_agent-0.2.0.dist-info/RECORD +52 -0
- kctl_agent-0.2.0.dist-info/WHEEL +4 -0
- kctl_agent-0.2.0.dist-info/entry_points.txt +2 -0
kctl_agent/__init__.py
ADDED
kctl_agent/__main__.py
ADDED
kctl_agent/cli.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""kctl-agent — the manifest, registry and reconciler for every agent asset."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Annotated
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
from kctl_lib import cli_entrypoint, register_introspection_commands
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
from .commands import (
|
|
14
|
+
doctor_cmd,
|
|
15
|
+
eval_cmd,
|
|
16
|
+
knowledge_cmd,
|
|
17
|
+
lint_cmd,
|
|
18
|
+
manifest_cmd,
|
|
19
|
+
reconcile_cmd,
|
|
20
|
+
vendor_cmd,
|
|
21
|
+
)
|
|
22
|
+
from .core.callbacks import AppContext
|
|
23
|
+
|
|
24
|
+
PROFILE_ENV = "KCTL_AGENT_PROFILE"
|
|
25
|
+
HOME_ENV = "KCTL_AGENT_HOME"
|
|
26
|
+
|
|
27
|
+
app = typer.Typer(
|
|
28
|
+
name="kctl-agent",
|
|
29
|
+
help="Declare, govern and reconcile every agent asset in this workspace.",
|
|
30
|
+
no_args_is_help=True,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _version_callback(value: bool) -> None:
|
|
35
|
+
if value:
|
|
36
|
+
typer.echo(f"kctl-agent {__version__}")
|
|
37
|
+
raise typer.Exit()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@app.callback()
|
|
41
|
+
def main(
|
|
42
|
+
ctx: typer.Context,
|
|
43
|
+
profile: Annotated[
|
|
44
|
+
str,
|
|
45
|
+
typer.Option("--profile", "-p", help="Manifest profile: which assets this environment gets."),
|
|
46
|
+
] = "",
|
|
47
|
+
json_output: Annotated[bool, typer.Option("--json", help="Emit JSON.")] = False,
|
|
48
|
+
quiet: Annotated[bool, typer.Option("--quiet", "-q", help="Suppress non-essential output.")] = False,
|
|
49
|
+
output_format: Annotated[str, typer.Option("--format", "-f", help="pretty | json | csv | yaml")] = "pretty",
|
|
50
|
+
no_header: Annotated[bool, typer.Option("--no-header", help="Omit table headers.")] = False,
|
|
51
|
+
home: Annotated[str, typer.Option("--home", help="Override the home directory (testing, containers).")] = "",
|
|
52
|
+
version: Annotated[
|
|
53
|
+
bool,
|
|
54
|
+
typer.Option("--version", "-V", callback=_version_callback, is_eager=True, help="Show version."),
|
|
55
|
+
] = False,
|
|
56
|
+
) -> None:
|
|
57
|
+
"""Root callback: build the app context every command reads."""
|
|
58
|
+
home_raw = home or os.environ.get(HOME_ENV, "")
|
|
59
|
+
ctx.obj = AppContext(
|
|
60
|
+
json_mode=json_output,
|
|
61
|
+
quiet=quiet,
|
|
62
|
+
format="json" if json_output else output_format,
|
|
63
|
+
no_header=no_header,
|
|
64
|
+
profile=profile or os.environ.get(PROFILE_ENV, ""),
|
|
65
|
+
home=Path(home_raw).expanduser() if home_raw else Path.home(),
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
app.add_typer(manifest_cmd.app, name="manifest")
|
|
70
|
+
app.add_typer(reconcile_cmd.app, name="reconcile")
|
|
71
|
+
app.add_typer(knowledge_cmd.app, name="knowledge")
|
|
72
|
+
app.add_typer(doctor_cmd.app, name="doctor")
|
|
73
|
+
app.add_typer(lint_cmd.app, name="lint")
|
|
74
|
+
app.add_typer(vendor_cmd.app, name="vendor")
|
|
75
|
+
app.add_typer(eval_cmd.app, name="eval")
|
|
76
|
+
|
|
77
|
+
# The three verbs used constantly are also top-level, because
|
|
78
|
+
# `kctl-agent plan` reads better than `kctl-agent reconcile plan`.
|
|
79
|
+
app.command("plan")(reconcile_cmd.plan_cmd)
|
|
80
|
+
app.command("apply")(reconcile_cmd.apply_cmd)
|
|
81
|
+
app.command("diff")(reconcile_cmd.diff_cmd)
|
|
82
|
+
app.command("import")(manifest_cmd.import_cmd)
|
|
83
|
+
app.command("lock")(manifest_cmd.lock_cmd)
|
|
84
|
+
app.command("registry")(manifest_cmd.registry_cmd)
|
|
85
|
+
app.command("hoist")(manifest_cmd.hoist_cmd)
|
|
86
|
+
|
|
87
|
+
# register_introspection_commands and cli_entrypoint must remain the last two
|
|
88
|
+
# statements: introspection needs every command registered first, and
|
|
89
|
+
# cli_entrypoint wraps the fully-assembled app for error handling.
|
|
90
|
+
register_introspection_commands(app)
|
|
91
|
+
app = cli_entrypoint(app)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _run() -> None:
|
|
95
|
+
app()
|
|
File without changes
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Helpers shared by the command groups: context access and consistent output."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
|
|
10
|
+
from ..core.callbacks import AppContext
|
|
11
|
+
from ..core.config import find_repo_root
|
|
12
|
+
from ..manifest import load_manifest
|
|
13
|
+
from ..model import ACTION_CONFLICT, ACTION_OK, Manifest, Plan
|
|
14
|
+
from ..reconcile import ReconcileRequest, resolve_profile
|
|
15
|
+
|
|
16
|
+
PROFILE_ENV = "KCTL_AGENT_PROFILE"
|
|
17
|
+
|
|
18
|
+
#: Console glyphs per action, so a plan reads at a glance.
|
|
19
|
+
_GLYPH = {
|
|
20
|
+
"create": "+",
|
|
21
|
+
"update": "~",
|
|
22
|
+
"remove": "-",
|
|
23
|
+
"ok": "=",
|
|
24
|
+
"conflict": "!",
|
|
25
|
+
"unmanaged": "?",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def context(ctx: typer.Context) -> AppContext:
|
|
30
|
+
actx: AppContext = ctx.obj
|
|
31
|
+
if actx.root is None:
|
|
32
|
+
actx.root = find_repo_root()
|
|
33
|
+
return actx
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def manifest_of(actx: AppContext) -> Manifest:
|
|
37
|
+
assert actx.root is not None
|
|
38
|
+
return load_manifest(actx.root)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def request(actx: AppContext, manifest: Manifest, prune: bool = False) -> ReconcileRequest:
|
|
42
|
+
assert actx.root is not None
|
|
43
|
+
name = actx.profile or os.environ.get(PROFILE_ENV, "")
|
|
44
|
+
return ReconcileRequest(
|
|
45
|
+
root=actx.root,
|
|
46
|
+
home=actx.home,
|
|
47
|
+
manifest=manifest,
|
|
48
|
+
profile=resolve_profile(manifest, name),
|
|
49
|
+
prune=prune,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def note(actx: AppContext, message: str, level: str = "info") -> None:
|
|
54
|
+
"""Human-readable line, suppressed under --json.
|
|
55
|
+
|
|
56
|
+
kctl_lib.Output.info/success/warn print to stdout regardless of json_mode, so
|
|
57
|
+
an unguarded call corrupts the JSON document a caller is trying to parse.
|
|
58
|
+
"""
|
|
59
|
+
if actx.json_mode:
|
|
60
|
+
return
|
|
61
|
+
getattr(actx.output, level)(message)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def print_plan(actx: AppContext, plan: Plan, show_ok: bool = False) -> None:
|
|
65
|
+
"""Render a plan. Unchanged items are hidden unless asked for -- the signal is what moves."""
|
|
66
|
+
if actx.json_mode:
|
|
67
|
+
return
|
|
68
|
+
out = actx.output
|
|
69
|
+
rows = [a for a in plan.actions if show_ok or a.action != ACTION_OK]
|
|
70
|
+
for action in rows:
|
|
71
|
+
glyph = _GLYPH.get(action.action, " ")
|
|
72
|
+
out.info(f"{glyph} {action.action:<9} {action.target:<12} {action.path}")
|
|
73
|
+
out.info(f" {action.detail}")
|
|
74
|
+
if not rows:
|
|
75
|
+
out.success("nothing to do — every target already matches the manifest")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def plan_payload(plan: Plan) -> dict[str, object]:
|
|
79
|
+
return {
|
|
80
|
+
"counts": plan.counts(),
|
|
81
|
+
"actions": [
|
|
82
|
+
{
|
|
83
|
+
"action": a.action,
|
|
84
|
+
"target": a.target,
|
|
85
|
+
"renderer": a.renderer,
|
|
86
|
+
"path": str(a.path),
|
|
87
|
+
"detail": a.detail,
|
|
88
|
+
"state_key": a.state_key,
|
|
89
|
+
}
|
|
90
|
+
for a in plan.actions
|
|
91
|
+
],
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def conflict_hint(plan: Plan) -> str | None:
|
|
96
|
+
count = sum(1 for a in plan.actions if a.action == ACTION_CONFLICT)
|
|
97
|
+
if not count:
|
|
98
|
+
return None
|
|
99
|
+
return (
|
|
100
|
+
f"{count} conflict(s): items changed outside kctl-agent. "
|
|
101
|
+
"Run `kctl-agent apply --adopt` to keep the machine's version, "
|
|
102
|
+
"or `--force` to overwrite."
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def rel(path: Path, root: Path) -> str:
|
|
107
|
+
try:
|
|
108
|
+
return path.relative_to(root).as_posix()
|
|
109
|
+
except ValueError:
|
|
110
|
+
return str(path)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _note(actx: AppContext, message: str) -> None:
|
|
114
|
+
note(actx, message, "info")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _ok(actx: AppContext, message: str) -> None:
|
|
118
|
+
note(actx, message, "success")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _warn(actx: AppContext, message: str) -> None:
|
|
122
|
+
note(actx, message, "warn")
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""`kctl-agent doctor` — is this workspace's agent tooling actually working?"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Annotated
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from ..health import check_all, machine_env, missing_env
|
|
10
|
+
from ..lockfile import load_lock
|
|
11
|
+
from ..model import ACTION_CONFLICT, ACTION_OK, AssetKind
|
|
12
|
+
from ..reconcile import build_plan
|
|
13
|
+
from ..registry import discover
|
|
14
|
+
from ._shared import _note, _ok, _warn, context, manifest_of, request
|
|
15
|
+
|
|
16
|
+
app = typer.Typer(help="Diagnose the agent platform.", no_args_is_help=False, invoke_without_command=True)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@app.callback(invoke_without_command=True)
|
|
20
|
+
def doctor(
|
|
21
|
+
ctx: typer.Context,
|
|
22
|
+
timeout: Annotated[float, typer.Option("--timeout", help="Per-server health timeout, seconds.")] = 5.0,
|
|
23
|
+
skip_network: Annotated[bool, typer.Option("--skip-network", help="Do not probe MCP servers.")] = False,
|
|
24
|
+
deep: Annotated[
|
|
25
|
+
bool,
|
|
26
|
+
typer.Option("--deep", help="Launch each stdio server and complete an MCP handshake."),
|
|
27
|
+
] = False,
|
|
28
|
+
) -> None:
|
|
29
|
+
"""Report asset counts, MCP reachability, lock coverage, and drift.
|
|
30
|
+
|
|
31
|
+
Without --deep a stdio server is only checked for a resolvable command, which
|
|
32
|
+
says nothing about whether it answers. --deep spawns each one, so it is opt-in.
|
|
33
|
+
"""
|
|
34
|
+
if ctx.invoked_subcommand:
|
|
35
|
+
return
|
|
36
|
+
actx = context(ctx)
|
|
37
|
+
out = actx.output
|
|
38
|
+
assert actx.root is not None
|
|
39
|
+
|
|
40
|
+
manifest = manifest_of(actx)
|
|
41
|
+
assets = discover(actx.root, manifest)
|
|
42
|
+
lock = load_lock(actx.root)
|
|
43
|
+
|
|
44
|
+
counts: dict[str, int] = {}
|
|
45
|
+
for asset in assets:
|
|
46
|
+
counts[asset.kind.value] = counts.get(asset.kind.value, 0) + 1
|
|
47
|
+
|
|
48
|
+
health = (
|
|
49
|
+
[]
|
|
50
|
+
if skip_network
|
|
51
|
+
else check_all(manifest.mcp, timeout=timeout, deep=deep, env_by_server=machine_env(actx.home))
|
|
52
|
+
)
|
|
53
|
+
failing = [h for h in health if h.failing]
|
|
54
|
+
warning = [h for h in health if h.warning]
|
|
55
|
+
|
|
56
|
+
absent_env = missing_env(manifest.mcp)
|
|
57
|
+
declared = len(manifest.marketplaces) + len(manifest.plugins) + len(manifest.skills) + len(manifest.mcp)
|
|
58
|
+
unlocked = max(declared - len(lock.entries), 0)
|
|
59
|
+
|
|
60
|
+
drift = 0
|
|
61
|
+
conflicts = 0
|
|
62
|
+
if actx.profile:
|
|
63
|
+
plan = build_plan(request(actx, manifest))
|
|
64
|
+
drift = sum(1 for a in plan.actions if a.action != ACTION_OK)
|
|
65
|
+
conflicts = sum(1 for a in plan.actions if a.action == ACTION_CONFLICT)
|
|
66
|
+
|
|
67
|
+
if actx.json_mode:
|
|
68
|
+
out.raw_json(
|
|
69
|
+
{
|
|
70
|
+
"assets": counts,
|
|
71
|
+
"assets_total": len(assets),
|
|
72
|
+
"external_declared": declared,
|
|
73
|
+
"lock_entries": len(lock.entries),
|
|
74
|
+
"unlocked": unlocked,
|
|
75
|
+
"mcp": [{"name": h.name, "status": h.status, "health": h.health, "detail": h.detail} for h in health],
|
|
76
|
+
"mcp_failing": len(failing),
|
|
77
|
+
"mcp_warning": len(warning),
|
|
78
|
+
"mcp_missing_env": absent_env,
|
|
79
|
+
"drift": drift,
|
|
80
|
+
"conflicts": conflicts,
|
|
81
|
+
"ok": not failing,
|
|
82
|
+
}
|
|
83
|
+
)
|
|
84
|
+
else:
|
|
85
|
+
columns = [("kind", "cyan"), ("count", "white")]
|
|
86
|
+
rows = [[kind.value, str(counts.get(kind.value, 0))] for kind in AssetKind]
|
|
87
|
+
out.table("Assets", columns, rows)
|
|
88
|
+
|
|
89
|
+
if health:
|
|
90
|
+
hcols = [("server", "cyan"), ("status", "white"), ("class", "dim"), ("detail", "dim")]
|
|
91
|
+
hrows = [[h.name, h.status, h.health, h.detail] for h in health]
|
|
92
|
+
out.table("MCP servers", hcols, hrows)
|
|
93
|
+
|
|
94
|
+
_note(actx, f"external declared: {declared} lock entries: {len(lock.entries)} unlocked: {unlocked}")
|
|
95
|
+
if actx.profile:
|
|
96
|
+
_note(actx, f"profile {actx.profile}: {drift} item(s) drifted, {conflicts} conflict(s)")
|
|
97
|
+
else:
|
|
98
|
+
_note(actx, "pass -p/--profile to include drift in this report")
|
|
99
|
+
|
|
100
|
+
for name, keys in sorted(absent_env.items()):
|
|
101
|
+
_warn(actx, f"MCP {name}: env not set — {', '.join(keys)}")
|
|
102
|
+
if failing:
|
|
103
|
+
out.error(f"{len(failing)} required MCP server(s) unreachable")
|
|
104
|
+
if warning:
|
|
105
|
+
_warn(actx, f"{len(warning)} optional MCP server(s) unreachable")
|
|
106
|
+
if not failing and not warning:
|
|
107
|
+
_ok(actx, "all declared MCP servers reachable")
|
|
108
|
+
|
|
109
|
+
if failing:
|
|
110
|
+
raise typer.Exit(code=1)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@app.command("ai-summary")
|
|
114
|
+
def ai_summary(ctx: typer.Context) -> None:
|
|
115
|
+
"""One-call machine-readable health summary for agents."""
|
|
116
|
+
actx = context(ctx)
|
|
117
|
+
out = actx.output
|
|
118
|
+
assert actx.root is not None
|
|
119
|
+
|
|
120
|
+
manifest = manifest_of(actx)
|
|
121
|
+
assets = discover(actx.root, manifest)
|
|
122
|
+
health = check_all(manifest.mcp)
|
|
123
|
+
failing = [h.name for h in health if h.failing]
|
|
124
|
+
warning = [h.name for h in health if h.warning]
|
|
125
|
+
|
|
126
|
+
counts: dict[str, int] = {}
|
|
127
|
+
for asset in assets:
|
|
128
|
+
counts[asset.kind.value] = counts.get(asset.kind.value, 0) + 1
|
|
129
|
+
|
|
130
|
+
out.raw_json(
|
|
131
|
+
{
|
|
132
|
+
"workspace": manifest.workspace,
|
|
133
|
+
"assets": counts,
|
|
134
|
+
"assets_total": len(assets),
|
|
135
|
+
"profiles": sorted(manifest.profiles),
|
|
136
|
+
"targets": sorted(manifest.targets),
|
|
137
|
+
"mcp_failing": failing,
|
|
138
|
+
"mcp_warning": warning,
|
|
139
|
+
"healthy": not failing,
|
|
140
|
+
"next_commands": [
|
|
141
|
+
"kctl-agent registry --kind skills",
|
|
142
|
+
"kctl-agent -p laptop plan",
|
|
143
|
+
"kctl-agent doctor",
|
|
144
|
+
],
|
|
145
|
+
}
|
|
146
|
+
)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""`kctl-agent eval` — inspect and validate eval scenarios.
|
|
2
|
+
|
|
3
|
+
Deliberately model-free. Running a routing eval needs a model; *validating* that
|
|
4
|
+
scenarios are well-formed, assert something, and reference commands that actually
|
|
5
|
+
exist does not — and that is the part CI can enforce on every push.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import shutil
|
|
11
|
+
from typing import Annotated
|
|
12
|
+
|
|
13
|
+
import typer
|
|
14
|
+
|
|
15
|
+
from ..evals import MIN_SCENARIOS, cli_of, load_scenarios, referenced_commands, results_are_stale
|
|
16
|
+
from ..model import AssetKind
|
|
17
|
+
from ..registry import discover
|
|
18
|
+
from ._shared import _note, _ok, _warn, context, manifest_of
|
|
19
|
+
|
|
20
|
+
app = typer.Typer(help="Inspect and validate eval scenarios.", no_args_is_help=True)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _skills(actx): # type: ignore[no-untyped-def]
|
|
24
|
+
manifest = manifest_of(actx)
|
|
25
|
+
assert actx.root is not None
|
|
26
|
+
return [
|
|
27
|
+
a
|
|
28
|
+
for a in discover(actx.root, manifest, include_external=False)
|
|
29
|
+
if a.kind == AssetKind.SKILLS and a.path.is_dir()
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@app.command("list")
|
|
34
|
+
def list_cmd(ctx: typer.Context) -> None:
|
|
35
|
+
"""Eval coverage per skill."""
|
|
36
|
+
actx = context(ctx)
|
|
37
|
+
out = actx.output
|
|
38
|
+
assert actx.root is not None
|
|
39
|
+
|
|
40
|
+
rows = []
|
|
41
|
+
for asset in _skills(actx):
|
|
42
|
+
scenarios = load_scenarios(asset.path)
|
|
43
|
+
stale = results_are_stale(actx.root, asset.name, asset.path)
|
|
44
|
+
rows.append(
|
|
45
|
+
{
|
|
46
|
+
"skill": asset.name,
|
|
47
|
+
"scenarios": len(scenarios),
|
|
48
|
+
"meets_minimum": len(scenarios) >= MIN_SCENARIOS,
|
|
49
|
+
"results": "stale" if stale else ("recorded" if stale is False else "none"),
|
|
50
|
+
}
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
if actx.json_mode:
|
|
54
|
+
out.raw_json(rows)
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
columns = [("skill", "cyan"), ("scenarios", "white"), ("min met", "dim"), ("results", "dim")]
|
|
58
|
+
out.table(
|
|
59
|
+
"Eval coverage",
|
|
60
|
+
columns,
|
|
61
|
+
[[r["skill"], str(r["scenarios"]), "yes" if r["meets_minimum"] else "no", r["results"]] for r in rows],
|
|
62
|
+
)
|
|
63
|
+
covered = sum(1 for r in rows if r["meets_minimum"])
|
|
64
|
+
_note(actx, f"{covered} of {len(rows)} skill(s) meet the {MIN_SCENARIOS}-scenario minimum")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@app.command("validate")
|
|
68
|
+
def validate_cmd(
|
|
69
|
+
ctx: typer.Context,
|
|
70
|
+
names: Annotated[list[str] | None, typer.Argument(help="Skill names. Omit for all.")] = None,
|
|
71
|
+
) -> None:
|
|
72
|
+
"""Check scenarios are well-formed and reference commands that exist.
|
|
73
|
+
|
|
74
|
+
Exits 1 on a broken scenario. This is the half of eval that needs no model, so
|
|
75
|
+
it is the half CI can run.
|
|
76
|
+
"""
|
|
77
|
+
actx = context(ctx)
|
|
78
|
+
out = actx.output
|
|
79
|
+
wanted = set(names) if names else None
|
|
80
|
+
problems: list[dict[str, str]] = []
|
|
81
|
+
checked = 0
|
|
82
|
+
|
|
83
|
+
for asset in _skills(actx):
|
|
84
|
+
if wanted is not None and asset.name not in wanted:
|
|
85
|
+
continue
|
|
86
|
+
for scenario in load_scenarios(asset.path):
|
|
87
|
+
checked += 1
|
|
88
|
+
label = f"{asset.name}/{scenario.path.name}"
|
|
89
|
+
if scenario.error:
|
|
90
|
+
problems.append({"scenario": label, "problem": scenario.error})
|
|
91
|
+
continue
|
|
92
|
+
if not scenario.prompt.strip():
|
|
93
|
+
problems.append({"scenario": label, "problem": "no prompt"})
|
|
94
|
+
if not scenario.has_expectation:
|
|
95
|
+
problems.append({"scenario": label, "problem": "asserts nothing"})
|
|
96
|
+
if scenario.expect_skill and scenario.expect_skill != asset.name:
|
|
97
|
+
problems.append(
|
|
98
|
+
{"scenario": label, "problem": f"expect_skill {scenario.expect_skill!r} is not this skill"}
|
|
99
|
+
)
|
|
100
|
+
for command in referenced_commands(scenario):
|
|
101
|
+
binary = cli_of(command)
|
|
102
|
+
if binary and shutil.which(binary) is None:
|
|
103
|
+
problems.append({"scenario": label, "problem": f"expects {binary!r}, which is not installed"})
|
|
104
|
+
|
|
105
|
+
if actx.json_mode:
|
|
106
|
+
out.raw_json({"checked": checked, "problems": problems})
|
|
107
|
+
else:
|
|
108
|
+
for problem in problems:
|
|
109
|
+
_warn(actx, f"{problem['scenario']}: {problem['problem']}")
|
|
110
|
+
if problems:
|
|
111
|
+
_note(actx, f"\n{len(problems)} problem(s) across {checked} scenario(s)")
|
|
112
|
+
else:
|
|
113
|
+
_ok(actx, f"{checked} scenario(s) valid")
|
|
114
|
+
|
|
115
|
+
if problems:
|
|
116
|
+
raise typer.Exit(code=1)
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""`kctl-agent knowledge` — discover cross-cutting facts in one call."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Annotated
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from ..knowledge import DEFAULT_MAX_AGE_DAYS, load_all, search
|
|
10
|
+
from ..model import AssetKind
|
|
11
|
+
from ._shared import _note, _warn, context, manifest_of, rel
|
|
12
|
+
|
|
13
|
+
app = typer.Typer(help="Search and audit the knowledge layer.", no_args_is_help=True)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _base(actx): # type: ignore[no-untyped-def]
|
|
17
|
+
manifest = manifest_of(actx)
|
|
18
|
+
assert actx.root is not None
|
|
19
|
+
return actx.root / manifest.root_for(AssetKind.KNOWLEDGE)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@app.command("list")
|
|
23
|
+
def list_cmd(ctx: typer.Context) -> None:
|
|
24
|
+
"""List every knowledge entry with its freshness."""
|
|
25
|
+
actx = context(ctx)
|
|
26
|
+
out = actx.output
|
|
27
|
+
assert actx.root is not None
|
|
28
|
+
entries = load_all(_base(actx))
|
|
29
|
+
|
|
30
|
+
if actx.json_mode:
|
|
31
|
+
out.raw_json(
|
|
32
|
+
[
|
|
33
|
+
{
|
|
34
|
+
"name": e.name,
|
|
35
|
+
"title": e.title,
|
|
36
|
+
"updated": e.updated,
|
|
37
|
+
"source": e.source,
|
|
38
|
+
"age_days": e.age_days(),
|
|
39
|
+
"stale": e.stale(),
|
|
40
|
+
"path": str(e.path),
|
|
41
|
+
}
|
|
42
|
+
for e in entries
|
|
43
|
+
]
|
|
44
|
+
)
|
|
45
|
+
return
|
|
46
|
+
|
|
47
|
+
columns = [("name", "cyan"), ("title", "white"), ("updated", "dim"), ("age", "dim"), ("source", "dim")]
|
|
48
|
+
rows = [
|
|
49
|
+
[e.name, e.title, e.updated or "-", str(e.age_days() if e.age_days() is not None else "-"), e.source or "-"]
|
|
50
|
+
for e in entries
|
|
51
|
+
]
|
|
52
|
+
out.table("Knowledge", columns, rows)
|
|
53
|
+
stale = [e for e in entries if e.stale()]
|
|
54
|
+
if stale:
|
|
55
|
+
_warn(actx, f"{len(stale)} entry/entries older than {DEFAULT_MAX_AGE_DAYS} days")
|
|
56
|
+
_note(actx, f"{len(entries)} entry/entries")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@app.command("search")
|
|
60
|
+
def search_cmd(
|
|
61
|
+
ctx: typer.Context,
|
|
62
|
+
query: Annotated[str, typer.Argument(help="Substring to look for.")],
|
|
63
|
+
limit: Annotated[int, typer.Option("--limit", help="Maximum hits.")] = 10,
|
|
64
|
+
) -> None:
|
|
65
|
+
"""Find the entry that answers a question, without reading the whole tree."""
|
|
66
|
+
actx = context(ctx)
|
|
67
|
+
out = actx.output
|
|
68
|
+
assert actx.root is not None
|
|
69
|
+
hits = search(load_all(_base(actx)), query, limit=limit)
|
|
70
|
+
|
|
71
|
+
if actx.json_mode:
|
|
72
|
+
out.raw_json([{"name": e.name, "path": str(e.path), "context": ctxline} for e, ctxline in hits])
|
|
73
|
+
return
|
|
74
|
+
|
|
75
|
+
for entry, line in hits:
|
|
76
|
+
_note(actx, f"{entry.name:<28} {rel(entry.path, actx.root)}")
|
|
77
|
+
_note(actx, f" {line}")
|
|
78
|
+
_note(actx, f"\n{len(hits)} hit(s)")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@app.command("show")
|
|
82
|
+
def show_cmd(
|
|
83
|
+
ctx: typer.Context,
|
|
84
|
+
name: Annotated[str, typer.Argument(help="Entry name (filename without .md).")],
|
|
85
|
+
) -> None:
|
|
86
|
+
"""Print one knowledge entry."""
|
|
87
|
+
actx = context(ctx)
|
|
88
|
+
out = actx.output
|
|
89
|
+
entry = next((e for e in load_all(_base(actx)) if e.name == name), None)
|
|
90
|
+
if entry is None:
|
|
91
|
+
out.error(f"no knowledge entry named {name!r}")
|
|
92
|
+
raise typer.Exit(code=1)
|
|
93
|
+
typer.echo(entry.path.read_text(encoding="utf-8"))
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""`kctl-agent lint` — evaluate every asset rule family."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Annotated
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from ..lint import has_errors, lint_all
|
|
10
|
+
from ..rules.registry import SEVERITY_ERROR, all_rules
|
|
11
|
+
from ._shared import _note, _ok, context, manifest_of
|
|
12
|
+
|
|
13
|
+
app = typer.Typer(
|
|
14
|
+
help="Lint agent assets against the rule families.", no_args_is_help=False, invoke_without_command=True
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@app.callback(invoke_without_command=True)
|
|
19
|
+
def lint(
|
|
20
|
+
ctx: typer.Context,
|
|
21
|
+
names: Annotated[list[str] | None, typer.Argument(help="Asset names to lint. Omit for all.")] = None,
|
|
22
|
+
warnings_as_errors: Annotated[bool, typer.Option("--warnings-as-errors", help="Exit 1 on warnings too.")] = False,
|
|
23
|
+
) -> None:
|
|
24
|
+
"""Lint every asset. Exits 1 when any error-severity rule fires."""
|
|
25
|
+
if ctx.invoked_subcommand:
|
|
26
|
+
return
|
|
27
|
+
actx = context(ctx)
|
|
28
|
+
out = actx.output
|
|
29
|
+
assert actx.root is not None
|
|
30
|
+
|
|
31
|
+
findings = lint_all(actx.root, manifest_of(actx), names=list(names) if names else None)
|
|
32
|
+
|
|
33
|
+
if actx.json_mode:
|
|
34
|
+
out.raw_json(
|
|
35
|
+
[{"rule": f.rule_id, "location": f.location, "detail": f.detail, "severity": f.severity} for f in findings]
|
|
36
|
+
)
|
|
37
|
+
else:
|
|
38
|
+
for finding in findings:
|
|
39
|
+
line = f"{finding.severity:<5} {finding.rule_id:<10} {finding.location} — {finding.detail}"
|
|
40
|
+
_note(actx, line)
|
|
41
|
+
errors = sum(1 for f in findings if f.severity == SEVERITY_ERROR)
|
|
42
|
+
warns = len(findings) - errors
|
|
43
|
+
if findings:
|
|
44
|
+
_note(actx, f"\n{errors} error(s), {warns} warning(s)")
|
|
45
|
+
else:
|
|
46
|
+
_ok(actx, "no findings")
|
|
47
|
+
|
|
48
|
+
if has_errors(findings) or (warnings_as_errors and findings):
|
|
49
|
+
raise typer.Exit(code=1)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@app.command("rules")
|
|
53
|
+
def rules_cmd(ctx: typer.Context) -> None:
|
|
54
|
+
"""List every registered rule."""
|
|
55
|
+
actx = context(ctx)
|
|
56
|
+
out = actx.output
|
|
57
|
+
registered = all_rules()
|
|
58
|
+
|
|
59
|
+
if actx.json_mode:
|
|
60
|
+
out.raw_json(
|
|
61
|
+
[
|
|
62
|
+
{
|
|
63
|
+
"id": r.id,
|
|
64
|
+
"severity": r.severity,
|
|
65
|
+
"title": r.title,
|
|
66
|
+
"rationale": r.rationale,
|
|
67
|
+
"kinds": sorted(k.value for k in r.kinds) or ["repo"],
|
|
68
|
+
}
|
|
69
|
+
for r in registered
|
|
70
|
+
]
|
|
71
|
+
)
|
|
72
|
+
return
|
|
73
|
+
|
|
74
|
+
columns = [("rule", "cyan"), ("sev", "white"), ("applies to", "dim"), ("title", "white")]
|
|
75
|
+
rows = [[r.id, r.severity, ", ".join(sorted(k.value for k in r.kinds)) or "repo", r.title] for r in registered]
|
|
76
|
+
out.table("Asset rules", columns, rows)
|
|
77
|
+
_note(actx, f"{len(registered)} rule(s)")
|