aisoc-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aisoc_cli/__init__.py +3 -0
- aisoc_cli/main.py +1372 -0
- aisoc_cli/templates/connector/README.md.tmpl +37 -0
- aisoc_cli/templates/connector/connector.py.tmpl +106 -0
- aisoc_cli/templates/connector/plugin.yaml.tmpl +42 -0
- aisoc_cli/templates/detection/README.md.tmpl +30 -0
- aisoc_cli/templates/detection/plugin.yaml.tmpl +17 -0
- aisoc_cli/templates/detection/rules/example.yaml.tmpl +28 -0
- aisoc_cli/templates/enricher/README.md.tmpl +29 -0
- aisoc_cli/templates/enricher/plugin.py.tmpl +42 -0
- aisoc_cli/templates/enricher/plugin.yaml.tmpl +29 -0
- aisoc_cli/templates/responder/README.md.tmpl +25 -0
- aisoc_cli/templates/responder/plugin.py.tmpl +66 -0
- aisoc_cli/templates/responder/plugin.yaml.tmpl +32 -0
- aisoc_cli/templates/widget/README.md.tmpl +40 -0
- aisoc_cli/templates/widget/plugin.yaml.tmpl +29 -0
- aisoc_cli/templates/widget/widget.py.tmpl +50 -0
- aisoc_cli-0.1.0.dist-info/METADATA +90 -0
- aisoc_cli-0.1.0.dist-info/RECORD +21 -0
- aisoc_cli-0.1.0.dist-info/WHEEL +4 -0
- aisoc_cli-0.1.0.dist-info/entry_points.txt +2 -0
aisoc_cli/main.py
ADDED
|
@@ -0,0 +1,1372 @@
|
|
|
1
|
+
"""
|
|
2
|
+
AiSOC CLI — scaffold, validate, publish, and operate.
|
|
3
|
+
|
|
4
|
+
Commands:
|
|
5
|
+
aisoc plugin new <name> Scaffold a typed plugin from disk templates
|
|
6
|
+
aisoc plugin scaffold <name> Alias for `plugin new` (backwards compat)
|
|
7
|
+
aisoc plugin validate [path] Validate plugin.yaml against JSON Schema
|
|
8
|
+
aisoc plugin publish [path] Sign with Ed25519 key + POST to AiSOC API
|
|
9
|
+
aisoc detection validate <file> Validate Sigma rule syntax
|
|
10
|
+
aisoc keygen Generate an Ed25519 signing key pair
|
|
11
|
+
aisoc serve [--detach] Start the dev stack via docker compose
|
|
12
|
+
aisoc db upgrade Run database migrations against the dev stack
|
|
13
|
+
aisoc mcp serve [--transport] Launch the MCP server for IDE assistants
|
|
14
|
+
aisoc mcp install --host <h> Wire AiSOC into Claude / Cursor / Continue
|
|
15
|
+
aisoc submit <file> POST an alert/event JSON payload to the AiSOC API
|
|
16
|
+
aisoc replay --connector-id <id> Measure triage against your own closed findings
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import base64
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import shutil
|
|
25
|
+
import subprocess
|
|
26
|
+
import sys
|
|
27
|
+
import time
|
|
28
|
+
from importlib import resources
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from string import Template
|
|
31
|
+
from typing import Any
|
|
32
|
+
|
|
33
|
+
import click
|
|
34
|
+
import httpx
|
|
35
|
+
import yaml
|
|
36
|
+
from cryptography.hazmat.primitives import serialization
|
|
37
|
+
from cryptography.hazmat.primitives.asymmetric.ed25519 import (
|
|
38
|
+
Ed25519PrivateKey,
|
|
39
|
+
)
|
|
40
|
+
from jsonschema import ValidationError, validate
|
|
41
|
+
from rich.console import Console
|
|
42
|
+
from rich.panel import Panel
|
|
43
|
+
from rich.table import Table
|
|
44
|
+
|
|
45
|
+
console = Console()
|
|
46
|
+
|
|
47
|
+
#: Progress and diagnostics go here, never to stdout.
|
|
48
|
+
#:
|
|
49
|
+
#: ``aisoc replay`` writes the report itself to stdout when no ``--output`` is
|
|
50
|
+
#: given, so ``aisoc replay ... > report.md`` has to produce the report and
|
|
51
|
+
#: nothing else. A progress line on stdout would land in the middle of the
|
|
52
|
+
#: file, and this phase's acceptance bar is a report that reproduces byte for
|
|
53
|
+
#: byte.
|
|
54
|
+
err_console = Console(stderr=True)
|
|
55
|
+
|
|
56
|
+
PLUGIN_TYPES = ("enricher", "connector", "responder", "detection", "widget")
|
|
57
|
+
DEFAULT_AUTHOR = "Your Name <you@example.com>"
|
|
58
|
+
|
|
59
|
+
# ── Schemas ───────────────────────────────────────────────────────────────────
|
|
60
|
+
|
|
61
|
+
PLUGIN_MANIFEST_SCHEMA: dict[str, Any] = {
|
|
62
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
63
|
+
"type": "object",
|
|
64
|
+
"required": ["id", "name", "version", "plugin_type", "description", "author"],
|
|
65
|
+
"properties": {
|
|
66
|
+
"id": {"type": "string", "pattern": "^[a-z0-9_-]+$"},
|
|
67
|
+
"name": {"type": "string"},
|
|
68
|
+
"version": {"type": "string", "pattern": r"^\d+\.\d+\.\d+$"},
|
|
69
|
+
"plugin_type": {
|
|
70
|
+
"type": "string",
|
|
71
|
+
"enum": ["enricher", "connector", "responder", "detection", "widget"],
|
|
72
|
+
},
|
|
73
|
+
"description": {"type": "string"},
|
|
74
|
+
"author": {"type": "string"},
|
|
75
|
+
"tags": {"type": "array", "items": {"type": "string"}},
|
|
76
|
+
"min_aisoc_version": {"type": "string"},
|
|
77
|
+
"config_schema": {"type": "object"},
|
|
78
|
+
"entry_point": {"type": "string"},
|
|
79
|
+
"runtime": {"type": "string", "enum": ["python", "oci"]},
|
|
80
|
+
},
|
|
81
|
+
"additionalProperties": True,
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
# ── Template loading ──────────────────────────────────────────────────────────
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _templates_root() -> Path:
|
|
88
|
+
"""Return the on-disk root of the bundled templates directory.
|
|
89
|
+
|
|
90
|
+
Uses ``importlib.resources`` so this works whether the package is run from
|
|
91
|
+
a source checkout or installed into site-packages.
|
|
92
|
+
"""
|
|
93
|
+
pkg_files = resources.files("aisoc_cli") / "templates"
|
|
94
|
+
return Path(str(pkg_files))
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _render_templates(plugin_type: str, target: Path, substitutions: dict[str, str]) -> list[Path]:
|
|
98
|
+
"""Render every ``*.tmpl`` file under ``templates/<plugin_type>/`` into ``target``.
|
|
99
|
+
|
|
100
|
+
The directory layout under the template root is preserved verbatim. Each
|
|
101
|
+
``foo.ext.tmpl`` becomes ``foo.ext`` in the output, with ``$slug``,
|
|
102
|
+
``$name``, and ``$author`` substituted via ``string.Template`` (so we never
|
|
103
|
+
fight Python ``.format()`` over real curly braces in YAML/JSON).
|
|
104
|
+
"""
|
|
105
|
+
root = _templates_root() / plugin_type
|
|
106
|
+
if not root.is_dir():
|
|
107
|
+
raise click.ClickException(
|
|
108
|
+
f"No templates bundled for plugin_type='{plugin_type}'. Expected directory: {root}"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
written: list[Path] = []
|
|
112
|
+
for src in sorted(root.rglob("*.tmpl")):
|
|
113
|
+
rel = src.relative_to(root)
|
|
114
|
+
# Strip the trailing ".tmpl" suffix from the filename only.
|
|
115
|
+
out_rel = rel.with_name(rel.name[: -len(".tmpl")])
|
|
116
|
+
out_path = target / out_rel
|
|
117
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
118
|
+
rendered = Template(src.read_text()).safe_substitute(substitutions)
|
|
119
|
+
out_path.write_text(rendered)
|
|
120
|
+
written.append(out_path)
|
|
121
|
+
return written
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# ── CLI root ──────────────────────────────────────────────────────────────────
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@click.group()
|
|
128
|
+
@click.version_option(package_name="aisoc-cli")
|
|
129
|
+
def cli() -> None:
|
|
130
|
+
"""AiSOC Developer CLI — build, validate, and publish plugins & detections."""
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
# ── plugin group ──────────────────────────────────────────────────────────────
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@cli.group()
|
|
137
|
+
def plugin() -> None:
|
|
138
|
+
"""Plugin management commands."""
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _scaffold_plugin(name: str, output_dir: str, plugin_type: str, author: str) -> Path:
|
|
142
|
+
"""Shared implementation for ``plugin new`` and ``plugin scaffold``."""
|
|
143
|
+
if plugin_type not in PLUGIN_TYPES:
|
|
144
|
+
raise click.ClickException(
|
|
145
|
+
f"Unknown plugin_type='{plugin_type}'. Expected one of: {', '.join(PLUGIN_TYPES)}"
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
slug = name.lower().replace(" ", "-").replace("_", "-")
|
|
149
|
+
out = Path(output_dir) / slug
|
|
150
|
+
if out.exists():
|
|
151
|
+
console.print(f"[red]Directory already exists: {out}[/red]")
|
|
152
|
+
sys.exit(1)
|
|
153
|
+
out.mkdir(parents=True)
|
|
154
|
+
|
|
155
|
+
substitutions = {"slug": slug, "name": name, "author": author}
|
|
156
|
+
written = _render_templates(plugin_type, out, substitutions)
|
|
157
|
+
|
|
158
|
+
files_listing = "\n".join(f" • {p}" for p in written)
|
|
159
|
+
console.print(
|
|
160
|
+
Panel(
|
|
161
|
+
f"[green]{plugin_type.capitalize()} plugin scaffolded at[/green] [bold]{out}[/bold]\n\n"
|
|
162
|
+
f"Files created:\n{files_listing}\n\n"
|
|
163
|
+
f"Next steps:\n"
|
|
164
|
+
f" 1. Edit [bold]{out}/plugin.yaml[/bold] to fill in metadata\n"
|
|
165
|
+
f" 2. Implement the entry point referenced by [bold]plugin.yaml[/bold]\n"
|
|
166
|
+
f" 3. Run [bold]aisoc plugin validate {out}[/bold] to check",
|
|
167
|
+
title="[bold green]Scaffold complete[/bold green]",
|
|
168
|
+
)
|
|
169
|
+
)
|
|
170
|
+
return out
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@plugin.command("new")
|
|
174
|
+
@click.argument("name")
|
|
175
|
+
@click.option("--output-dir", "-o", default=".", help="Directory to create plugin in")
|
|
176
|
+
@click.option(
|
|
177
|
+
"--type",
|
|
178
|
+
"plugin_type",
|
|
179
|
+
default="enricher",
|
|
180
|
+
type=click.Choice(list(PLUGIN_TYPES)),
|
|
181
|
+
help="Plugin type",
|
|
182
|
+
)
|
|
183
|
+
@click.option(
|
|
184
|
+
"--author",
|
|
185
|
+
default=DEFAULT_AUTHOR,
|
|
186
|
+
show_default=True,
|
|
187
|
+
help="Author string written into plugin.yaml",
|
|
188
|
+
)
|
|
189
|
+
def plugin_new(name: str, output_dir: str, plugin_type: str, author: str) -> None:
|
|
190
|
+
"""Scaffold a new plugin from the bundled templates for its type."""
|
|
191
|
+
_scaffold_plugin(name, output_dir, plugin_type, author)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
@plugin.command("scaffold")
|
|
195
|
+
@click.argument("name")
|
|
196
|
+
@click.option("--output-dir", "-o", default=".", help="Directory to create plugin in")
|
|
197
|
+
@click.option(
|
|
198
|
+
"--type",
|
|
199
|
+
"plugin_type",
|
|
200
|
+
default="enricher",
|
|
201
|
+
type=click.Choice(list(PLUGIN_TYPES)),
|
|
202
|
+
help="Plugin type",
|
|
203
|
+
)
|
|
204
|
+
@click.option(
|
|
205
|
+
"--author",
|
|
206
|
+
default=DEFAULT_AUTHOR,
|
|
207
|
+
show_default=True,
|
|
208
|
+
help="Author string written into plugin.yaml",
|
|
209
|
+
)
|
|
210
|
+
def plugin_scaffold(name: str, output_dir: str, plugin_type: str, author: str) -> None:
|
|
211
|
+
"""Alias for ``aisoc plugin new`` (kept for backwards compatibility)."""
|
|
212
|
+
_scaffold_plugin(name, output_dir, plugin_type, author)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@plugin.command("validate")
|
|
216
|
+
@click.argument("path", default=".", type=click.Path(exists=True))
|
|
217
|
+
def plugin_validate(path: str) -> None:
|
|
218
|
+
"""Validate plugin.yaml against the AiSOC manifest schema."""
|
|
219
|
+
plugin_dir = Path(path)
|
|
220
|
+
manifest_file = plugin_dir / "plugin.yaml" if plugin_dir.is_dir() else plugin_dir
|
|
221
|
+
|
|
222
|
+
if not manifest_file.exists():
|
|
223
|
+
console.print(f"[red]plugin.yaml not found at {manifest_file}[/red]")
|
|
224
|
+
sys.exit(1)
|
|
225
|
+
|
|
226
|
+
with manifest_file.open() as f:
|
|
227
|
+
manifest = yaml.safe_load(f)
|
|
228
|
+
|
|
229
|
+
errors: list[str] = []
|
|
230
|
+
try:
|
|
231
|
+
validate(manifest, PLUGIN_MANIFEST_SCHEMA)
|
|
232
|
+
except ValidationError as exc:
|
|
233
|
+
errors.append(str(exc.message))
|
|
234
|
+
|
|
235
|
+
# Check entry point exists
|
|
236
|
+
if plugin_dir.is_dir():
|
|
237
|
+
entry = manifest.get("entry_point", "plugin.py")
|
|
238
|
+
if not (plugin_dir / entry).exists():
|
|
239
|
+
errors.append(f"entry_point '{entry}' not found in plugin directory")
|
|
240
|
+
|
|
241
|
+
if errors:
|
|
242
|
+
console.print("[red bold]Validation FAILED[/red bold]")
|
|
243
|
+
for err in errors:
|
|
244
|
+
console.print(f" [red]-[/red] {err}")
|
|
245
|
+
sys.exit(1)
|
|
246
|
+
else:
|
|
247
|
+
console.print(f"[green bold]Validation passed[/green bold] — {manifest_file}")
|
|
248
|
+
_print_manifest_table(manifest)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
@plugin.command("publish")
|
|
252
|
+
@click.argument("path", default=".", type=click.Path(exists=True))
|
|
253
|
+
@click.option(
|
|
254
|
+
"--api-url",
|
|
255
|
+
envvar="AISOC_API_URL",
|
|
256
|
+
default="http://localhost:8000",
|
|
257
|
+
show_default=True,
|
|
258
|
+
help="AiSOC API base URL",
|
|
259
|
+
)
|
|
260
|
+
@click.option(
|
|
261
|
+
"--api-key",
|
|
262
|
+
envvar="AISOC_API_KEY",
|
|
263
|
+
required=True,
|
|
264
|
+
help="AiSOC API key (or set AISOC_API_KEY env var)",
|
|
265
|
+
)
|
|
266
|
+
@click.option(
|
|
267
|
+
"--private-key",
|
|
268
|
+
envvar="AISOC_SIGNING_KEY",
|
|
269
|
+
default="~/.aisoc/signing.key",
|
|
270
|
+
show_default=True,
|
|
271
|
+
help="Path to Ed25519 private key PEM file",
|
|
272
|
+
)
|
|
273
|
+
def plugin_publish(path: str, api_url: str, api_key: str, private_key: str) -> None:
|
|
274
|
+
"""Sign and publish a plugin to the AiSOC community marketplace."""
|
|
275
|
+
import io
|
|
276
|
+
import tarfile
|
|
277
|
+
|
|
278
|
+
plugin_dir = Path(path)
|
|
279
|
+
manifest_file = plugin_dir / "plugin.yaml"
|
|
280
|
+
if not manifest_file.exists():
|
|
281
|
+
console.print(f"[red]plugin.yaml not found in {plugin_dir}[/red]")
|
|
282
|
+
sys.exit(1)
|
|
283
|
+
|
|
284
|
+
# Validate first
|
|
285
|
+
with manifest_file.open() as f:
|
|
286
|
+
manifest = yaml.safe_load(f)
|
|
287
|
+
try:
|
|
288
|
+
validate(manifest, PLUGIN_MANIFEST_SCHEMA)
|
|
289
|
+
except ValidationError as exc:
|
|
290
|
+
console.print(f"[red]Manifest validation failed:[/red] {exc.message}")
|
|
291
|
+
sys.exit(1)
|
|
292
|
+
|
|
293
|
+
# Load signing key
|
|
294
|
+
key_path = Path(private_key).expanduser()
|
|
295
|
+
if not key_path.exists():
|
|
296
|
+
console.print(f"[red]Signing key not found: {key_path}[/red]")
|
|
297
|
+
console.print("Run [bold]aisoc keygen[/bold] to generate a key pair.")
|
|
298
|
+
sys.exit(1)
|
|
299
|
+
|
|
300
|
+
private_key_obj = _load_private_key(key_path)
|
|
301
|
+
|
|
302
|
+
# Create tarball in memory
|
|
303
|
+
console.print(f"Packaging [bold]{plugin_dir}[/bold]...")
|
|
304
|
+
buf = io.BytesIO()
|
|
305
|
+
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
|
|
306
|
+
tar.add(str(plugin_dir), arcname=manifest["id"])
|
|
307
|
+
tarball = buf.getvalue()
|
|
308
|
+
|
|
309
|
+
# Sign the tarball
|
|
310
|
+
signature = private_key_obj.sign(tarball)
|
|
311
|
+
sig_b64 = base64.b64encode(signature).decode()
|
|
312
|
+
|
|
313
|
+
# POST to API
|
|
314
|
+
console.print(f"Publishing to [bold]{api_url}[/bold]...")
|
|
315
|
+
with httpx.Client(base_url=api_url, headers={"Authorization": f"Bearer {api_key}"}) as client:
|
|
316
|
+
resp = client.post(
|
|
317
|
+
# `/api/v1/plugins/publish` for as long as this command has
|
|
318
|
+
# existed. No router ever served that path — the real one is
|
|
319
|
+
# under the community prefix — so `aisoc plugin publish` was
|
|
320
|
+
# broken as shipped and failed with a 404 nobody could act on.
|
|
321
|
+
"/api/v1/community/plugins/publish",
|
|
322
|
+
content=tarball,
|
|
323
|
+
headers={
|
|
324
|
+
"Content-Type": "application/octet-stream",
|
|
325
|
+
"X-Plugin-Signature": sig_b64,
|
|
326
|
+
"X-Plugin-Manifest": json.dumps(manifest),
|
|
327
|
+
},
|
|
328
|
+
timeout=60,
|
|
329
|
+
)
|
|
330
|
+
if resp.status_code not in (200, 201):
|
|
331
|
+
console.print(f"[red]Publish failed ({resp.status_code}):[/red] {resp.text}")
|
|
332
|
+
sys.exit(1)
|
|
333
|
+
data = resp.json()
|
|
334
|
+
|
|
335
|
+
console.print(
|
|
336
|
+
Panel(
|
|
337
|
+
f"[green]Plugin submitted for review[/green]\n\n"
|
|
338
|
+
f" ID: {data.get('id', 'unknown')}\n"
|
|
339
|
+
f" Status: {data.get('status', 'pending')}\n\n"
|
|
340
|
+
f"An admin will review your submission. You'll be notified when approved.",
|
|
341
|
+
title="[bold green]Published[/bold green]",
|
|
342
|
+
)
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
# ── detection group ───────────────────────────────────────────────────────────
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
@cli.group()
|
|
350
|
+
def detection() -> None:
|
|
351
|
+
"""Detection rule management commands."""
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
@detection.command("validate")
|
|
355
|
+
@click.argument("file", type=click.Path(exists=True))
|
|
356
|
+
@click.option(
|
|
357
|
+
"--sigma-cli",
|
|
358
|
+
default="sigma",
|
|
359
|
+
help="Path to sigma-cli binary (default: sigma on PATH)",
|
|
360
|
+
)
|
|
361
|
+
def detection_validate(file: str, sigma_cli: str) -> None:
|
|
362
|
+
"""Validate a Sigma detection rule file using sigma-cli."""
|
|
363
|
+
import subprocess
|
|
364
|
+
|
|
365
|
+
rule_path = Path(file)
|
|
366
|
+
console.print(f"Validating [bold]{rule_path}[/bold]...")
|
|
367
|
+
|
|
368
|
+
# Try sigma-cli first
|
|
369
|
+
try:
|
|
370
|
+
result = subprocess.run(
|
|
371
|
+
[sigma_cli, "check", str(rule_path)],
|
|
372
|
+
capture_output=True,
|
|
373
|
+
text=True,
|
|
374
|
+
timeout=30,
|
|
375
|
+
)
|
|
376
|
+
if result.returncode == 0:
|
|
377
|
+
console.print(f"[green bold]Valid Sigma rule[/green bold] — {rule_path}")
|
|
378
|
+
if result.stdout:
|
|
379
|
+
console.print(result.stdout)
|
|
380
|
+
else:
|
|
381
|
+
console.print("[red bold]Invalid Sigma rule[/red bold]")
|
|
382
|
+
if result.stderr:
|
|
383
|
+
console.print(result.stderr)
|
|
384
|
+
if result.stdout:
|
|
385
|
+
console.print(result.stdout)
|
|
386
|
+
sys.exit(1)
|
|
387
|
+
except FileNotFoundError:
|
|
388
|
+
# Fall back to basic YAML + field validation
|
|
389
|
+
console.print(
|
|
390
|
+
f"[yellow]sigma-cli not found ('{sigma_cli}'), falling back to basic YAML validation[/yellow]"
|
|
391
|
+
)
|
|
392
|
+
_basic_sigma_validate(rule_path)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
# ── keygen command ────────────────────────────────────────────────────────────
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
@cli.command()
|
|
399
|
+
@click.option(
|
|
400
|
+
"--output-dir",
|
|
401
|
+
default="~/.aisoc",
|
|
402
|
+
show_default=True,
|
|
403
|
+
help="Directory to store generated key files",
|
|
404
|
+
)
|
|
405
|
+
def keygen(output_dir: str) -> None:
|
|
406
|
+
"""Generate an Ed25519 signing key pair for plugin publishing."""
|
|
407
|
+
out = Path(output_dir).expanduser()
|
|
408
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
409
|
+
|
|
410
|
+
priv_path = out / "signing.key"
|
|
411
|
+
pub_path = out / "signing.pub"
|
|
412
|
+
|
|
413
|
+
if priv_path.exists():
|
|
414
|
+
if not click.confirm(f"Key already exists at {priv_path}. Overwrite?"):
|
|
415
|
+
console.print("Aborted.")
|
|
416
|
+
return
|
|
417
|
+
|
|
418
|
+
private_key = Ed25519PrivateKey.generate()
|
|
419
|
+
public_key = private_key.public_key()
|
|
420
|
+
|
|
421
|
+
priv_pem = private_key.private_bytes(
|
|
422
|
+
encoding=serialization.Encoding.PEM,
|
|
423
|
+
format=serialization.PrivateFormat.PKCS8,
|
|
424
|
+
encryption_algorithm=serialization.NoEncryption(),
|
|
425
|
+
)
|
|
426
|
+
pub_pem = public_key.public_bytes(
|
|
427
|
+
encoding=serialization.Encoding.PEM,
|
|
428
|
+
format=serialization.PublicFormat.SubjectPublicKeyInfo,
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
priv_path.write_bytes(priv_pem)
|
|
432
|
+
priv_path.chmod(0o600)
|
|
433
|
+
pub_path.write_bytes(pub_pem)
|
|
434
|
+
|
|
435
|
+
console.print(
|
|
436
|
+
Panel(
|
|
437
|
+
f"[green]Key pair generated[/green]\n\n"
|
|
438
|
+
f" Private key: [bold]{priv_path}[/bold] (keep secret!)\n"
|
|
439
|
+
f" Public key: [bold]{pub_path}[/bold]\n\n"
|
|
440
|
+
f"Register your public key with the AiSOC marketplace before publishing:\n"
|
|
441
|
+
f" [bold]aisoc plugin publish --private-key {priv_path} <plugin-dir>[/bold]",
|
|
442
|
+
title="[bold green]Key Generation Complete[/bold green]",
|
|
443
|
+
)
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
# ── Helpers ───────────────────────────────────────────────────────────────────
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _load_private_key(path: Path) -> Ed25519PrivateKey:
|
|
451
|
+
pem_data = path.read_bytes()
|
|
452
|
+
key = serialization.load_pem_private_key(pem_data, password=None)
|
|
453
|
+
if not isinstance(key, Ed25519PrivateKey):
|
|
454
|
+
console.print("[red]Key must be an Ed25519 private key[/red]")
|
|
455
|
+
sys.exit(1)
|
|
456
|
+
return key
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _print_manifest_table(manifest: dict[str, Any]) -> None:
|
|
460
|
+
table = Table(show_header=False, box=None)
|
|
461
|
+
table.add_column("Field", style="cyan")
|
|
462
|
+
table.add_column("Value")
|
|
463
|
+
for field in ["id", "name", "version", "plugin_type", "description", "author"]:
|
|
464
|
+
table.add_row(field, str(manifest.get(field, "")))
|
|
465
|
+
console.print(table)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _basic_sigma_validate(rule_path: Path) -> None:
|
|
469
|
+
"""Minimal Sigma rule validation without sigma-cli."""
|
|
470
|
+
required_fields = ["title", "id", "status", "description", "logsource", "detection"]
|
|
471
|
+
with rule_path.open() as f:
|
|
472
|
+
rule = yaml.safe_load(f)
|
|
473
|
+
|
|
474
|
+
missing = [f for f in required_fields if f not in rule]
|
|
475
|
+
if missing:
|
|
476
|
+
console.print(f"[red bold]Missing required Sigma fields:[/red bold] {', '.join(missing)}")
|
|
477
|
+
sys.exit(1)
|
|
478
|
+
|
|
479
|
+
if "condition" not in rule.get("detection", {}):
|
|
480
|
+
console.print("[red bold]detection.condition is required[/red bold]")
|
|
481
|
+
sys.exit(1)
|
|
482
|
+
|
|
483
|
+
console.print(f"[green bold]Basic Sigma validation passed[/green bold] — {rule_path}")
|
|
484
|
+
console.print("[yellow]Install sigma-cli for full validation: pip install sigma-cli[/yellow]")
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
# ── Repo / compose helpers ────────────────────────────────────────────────────
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _find_repo_root(start: Path | None = None) -> Path:
|
|
491
|
+
"""Walk up from ``start`` (default: cwd) looking for a docker-compose root.
|
|
492
|
+
|
|
493
|
+
A repo root here is any directory containing either ``docker-compose.yml``
|
|
494
|
+
or ``infra/compose/docker-compose.dev.yml``. Falls back to the current
|
|
495
|
+
working directory if no match is found, so the underlying
|
|
496
|
+
``docker compose`` invocation can still produce a sensible error message.
|
|
497
|
+
"""
|
|
498
|
+
current = (start or Path.cwd()).resolve()
|
|
499
|
+
for candidate in [current, *current.parents]:
|
|
500
|
+
if (
|
|
501
|
+
(candidate / "docker-compose.yml").exists()
|
|
502
|
+
or (candidate / "infra" / "compose" / "docker-compose.dev.yml").exists()
|
|
503
|
+
or (candidate / "docker-compose.dev.yml").exists()
|
|
504
|
+
):
|
|
505
|
+
return candidate
|
|
506
|
+
return current
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _compose_file_arg(repo_root: Path) -> list[str]:
|
|
510
|
+
"""Pick the right ``-f`` arg for docker compose.
|
|
511
|
+
|
|
512
|
+
Prefers ``infra/compose/docker-compose.dev.yml`` (the dev-aliased entry
|
|
513
|
+
point in the post-tidy layout) when present, then the legacy root path,
|
|
514
|
+
otherwise falls back to the base ``docker-compose.yml``.
|
|
515
|
+
"""
|
|
516
|
+
new_dev = repo_root / "infra" / "compose" / "docker-compose.dev.yml"
|
|
517
|
+
if new_dev.exists():
|
|
518
|
+
return ["-f", str(new_dev)]
|
|
519
|
+
legacy_dev = repo_root / "docker-compose.dev.yml"
|
|
520
|
+
if legacy_dev.exists():
|
|
521
|
+
return ["-f", str(legacy_dev)]
|
|
522
|
+
return ["-f", str(repo_root / "docker-compose.yml")]
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def _require_docker() -> None:
|
|
526
|
+
"""Hard-fail with a friendly message if docker is not on PATH."""
|
|
527
|
+
if shutil.which("docker") is None:
|
|
528
|
+
console.print(
|
|
529
|
+
"[red bold]docker not found on PATH[/red bold]\n\n"
|
|
530
|
+
"Install Docker Desktop or Docker Engine, then retry. See:\n"
|
|
531
|
+
" https://docs.docker.com/engine/install/"
|
|
532
|
+
)
|
|
533
|
+
sys.exit(1)
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
# ── serve command ─────────────────────────────────────────────────────────────
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
@cli.command()
|
|
540
|
+
@click.option(
|
|
541
|
+
"--detach/--no-detach",
|
|
542
|
+
"detach",
|
|
543
|
+
default=True,
|
|
544
|
+
show_default=True,
|
|
545
|
+
help="Run docker compose up in detached mode (default: detached).",
|
|
546
|
+
)
|
|
547
|
+
@click.option(
|
|
548
|
+
"--build/--no-build",
|
|
549
|
+
"build",
|
|
550
|
+
default=False,
|
|
551
|
+
show_default=True,
|
|
552
|
+
help="Force a rebuild of changed images before starting.",
|
|
553
|
+
)
|
|
554
|
+
def serve(detach: bool, build: bool) -> None:
|
|
555
|
+
"""Start the AiSOC dev stack via docker compose.
|
|
556
|
+
|
|
557
|
+
Resolves the closest docker-compose root walking up from the cwd, prefers
|
|
558
|
+
``infra/compose/docker-compose.dev.yml`` if it exists (legacy root path is
|
|
559
|
+
still honoured), and shells out to ``docker compose -f <file> up``. Treat
|
|
560
|
+
this as the founder-style one-liner equivalent of the documented
|
|
561
|
+
``docker compose up -d``.
|
|
562
|
+
"""
|
|
563
|
+
_require_docker()
|
|
564
|
+
repo_root = _find_repo_root()
|
|
565
|
+
cmd = ["docker", "compose", *_compose_file_arg(repo_root), "up"]
|
|
566
|
+
if detach:
|
|
567
|
+
cmd.append("-d")
|
|
568
|
+
if build:
|
|
569
|
+
cmd.append("--build")
|
|
570
|
+
|
|
571
|
+
console.print(
|
|
572
|
+
Panel(
|
|
573
|
+
f"[bold]cwd:[/bold] {repo_root}\n[bold]cmd:[/bold] {' '.join(cmd)}",
|
|
574
|
+
title="[bold green]aisoc serve[/bold green]",
|
|
575
|
+
)
|
|
576
|
+
)
|
|
577
|
+
result = subprocess.run(cmd, cwd=str(repo_root))
|
|
578
|
+
if result.returncode != 0:
|
|
579
|
+
sys.exit(result.returncode)
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
# ── db group ──────────────────────────────────────────────────────────────────
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
@cli.group()
|
|
586
|
+
def db() -> None:
|
|
587
|
+
"""Database lifecycle commands."""
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
@db.command("upgrade")
|
|
591
|
+
@click.option(
|
|
592
|
+
"--service",
|
|
593
|
+
default="api",
|
|
594
|
+
show_default=True,
|
|
595
|
+
help="docker compose service that owns the migrations.",
|
|
596
|
+
)
|
|
597
|
+
def db_upgrade(service: str) -> None:
|
|
598
|
+
"""Apply pending SQL migrations against the running dev stack.
|
|
599
|
+
|
|
600
|
+
Delegates to ``docker compose exec <service> python -m
|
|
601
|
+
app.scripts.run_migrations``, which is the custom forward-only migration
|
|
602
|
+
runner under ``services/api``. Requires the stack to already be up
|
|
603
|
+
(``aisoc serve``).
|
|
604
|
+
"""
|
|
605
|
+
_require_docker()
|
|
606
|
+
repo_root = _find_repo_root()
|
|
607
|
+
cmd = [
|
|
608
|
+
"docker",
|
|
609
|
+
"compose",
|
|
610
|
+
*_compose_file_arg(repo_root),
|
|
611
|
+
"exec",
|
|
612
|
+
"-T",
|
|
613
|
+
service,
|
|
614
|
+
"python",
|
|
615
|
+
"-m",
|
|
616
|
+
"app.scripts.run_migrations",
|
|
617
|
+
]
|
|
618
|
+
console.print(
|
|
619
|
+
Panel(
|
|
620
|
+
f"[bold]cwd:[/bold] {repo_root}\n[bold]cmd:[/bold] {' '.join(cmd)}",
|
|
621
|
+
title="[bold green]aisoc db upgrade[/bold green]",
|
|
622
|
+
)
|
|
623
|
+
)
|
|
624
|
+
result = subprocess.run(cmd, cwd=str(repo_root))
|
|
625
|
+
if result.returncode != 0:
|
|
626
|
+
console.print(
|
|
627
|
+
"[yellow]If the stack is not up yet, run [bold]aisoc serve[/bold] "
|
|
628
|
+
"first, then re-run this command.[/yellow]"
|
|
629
|
+
)
|
|
630
|
+
sys.exit(result.returncode)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
# ── mcp group ─────────────────────────────────────────────────────────────────
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
@cli.group()
|
|
637
|
+
def mcp() -> None:
|
|
638
|
+
"""Model Context Protocol (MCP) commands."""
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def _resolve_mcp_entry(repo_root: Path) -> tuple[list[str], str] | None:
|
|
642
|
+
"""Return (argv, label) for invoking the local MCP build, or None.
|
|
643
|
+
|
|
644
|
+
Looks for a built ``services/mcp/dist/index.js`` next to ``package.json``.
|
|
645
|
+
Returns None if the build artifact is missing, so callers can fall back to
|
|
646
|
+
``npx @aisoc/mcp``.
|
|
647
|
+
"""
|
|
648
|
+
dist = repo_root / "services" / "mcp" / "dist" / "index.js"
|
|
649
|
+
if dist.exists():
|
|
650
|
+
node = shutil.which("node") or "node"
|
|
651
|
+
return [node, str(dist)], f"node {dist.relative_to(repo_root)}"
|
|
652
|
+
return None
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def _mcp_argv(repo_root: Path, subcommand: str, extra: list[str]) -> tuple[list[str], str]:
|
|
656
|
+
"""Build the argv for an MCP subcommand, preferring local dist over npx."""
|
|
657
|
+
local = _resolve_mcp_entry(repo_root)
|
|
658
|
+
if local is not None:
|
|
659
|
+
argv, label = local
|
|
660
|
+
return [*argv, subcommand, *extra], f"{label} {subcommand} {' '.join(extra)}".strip()
|
|
661
|
+
|
|
662
|
+
npx = shutil.which("npx") or "npx"
|
|
663
|
+
return (
|
|
664
|
+
[npx, "@aisoc/mcp", subcommand, *extra],
|
|
665
|
+
f"npx @aisoc/mcp {subcommand} {' '.join(extra)}".strip(),
|
|
666
|
+
)
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
@mcp.command("serve")
|
|
670
|
+
@click.option(
|
|
671
|
+
"--transport",
|
|
672
|
+
type=click.Choice(["stdio", "http"]),
|
|
673
|
+
default="stdio",
|
|
674
|
+
show_default=True,
|
|
675
|
+
help="MCP transport (stdio for IDE assistants, http for remote agents).",
|
|
676
|
+
)
|
|
677
|
+
@click.option(
|
|
678
|
+
"--port",
|
|
679
|
+
type=int,
|
|
680
|
+
default=None,
|
|
681
|
+
help="Port for the http transport (ignored when --transport=stdio).",
|
|
682
|
+
)
|
|
683
|
+
def mcp_serve(transport: str, port: int | None) -> None:
|
|
684
|
+
"""Launch the MCP server that exposes AiSOC to IDE assistants.
|
|
685
|
+
|
|
686
|
+
Prefers the locally built ``services/mcp/dist/index.js`` (from
|
|
687
|
+
``pnpm --filter @aisoc/mcp build``). Falls back to ``npx @aisoc/mcp``
|
|
688
|
+
when no local build is available.
|
|
689
|
+
"""
|
|
690
|
+
repo_root = _find_repo_root()
|
|
691
|
+
extra: list[str] = ["--transport", transport]
|
|
692
|
+
if transport == "http" and port is not None:
|
|
693
|
+
extra.extend(["--port", str(port)])
|
|
694
|
+
|
|
695
|
+
argv, label = _mcp_argv(repo_root, "serve", extra)
|
|
696
|
+
console.print(
|
|
697
|
+
Panel(
|
|
698
|
+
f"[bold]cwd:[/bold] {repo_root}\n[bold]cmd:[/bold] {label}",
|
|
699
|
+
title="[bold green]aisoc mcp serve[/bold green]",
|
|
700
|
+
)
|
|
701
|
+
)
|
|
702
|
+
# Replace this process with node so stdio is fully transparent — required
|
|
703
|
+
# for IDE assistants that pipe MCP frames over stdin/stdout.
|
|
704
|
+
os.execvp(argv[0], argv)
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
@mcp.command("install")
|
|
708
|
+
@click.option(
|
|
709
|
+
"--host",
|
|
710
|
+
type=click.Choice(["claude", "cursor", "continue", "cody"]),
|
|
711
|
+
required=True,
|
|
712
|
+
help="IDE assistant to wire AiSOC into.",
|
|
713
|
+
)
|
|
714
|
+
def mcp_install(host: str) -> None:
|
|
715
|
+
"""Register the AiSOC MCP server with the given IDE assistant.
|
|
716
|
+
|
|
717
|
+
Thin wrapper over ``aisoc-mcp install --host <host>`` that picks the local
|
|
718
|
+
dist build when available and falls back to ``npx @aisoc/mcp`` otherwise.
|
|
719
|
+
"""
|
|
720
|
+
repo_root = _find_repo_root()
|
|
721
|
+
argv, label = _mcp_argv(repo_root, "install", ["--host", host])
|
|
722
|
+
|
|
723
|
+
console.print(
|
|
724
|
+
Panel(
|
|
725
|
+
f"[bold]cwd:[/bold] {repo_root}\n[bold]cmd:[/bold] {label}",
|
|
726
|
+
title="[bold green]aisoc mcp install[/bold green]",
|
|
727
|
+
)
|
|
728
|
+
)
|
|
729
|
+
result = subprocess.run(argv, cwd=str(repo_root))
|
|
730
|
+
if result.returncode != 0:
|
|
731
|
+
sys.exit(result.returncode)
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
# ── submit command ───────────────────────────────────────────────────────────
|
|
735
|
+
#
|
|
736
|
+
# `aisoc submit <file>` lets the founder-flow demo and any operator land a
|
|
737
|
+
# canned OCSF/Okta-shaped payload on the local dev stack in one command.
|
|
738
|
+
# It POSTs to the API's direct-write submit endpoint, which synthesises an
|
|
739
|
+
# Alert row from the event batch and writes it straight to the database — so
|
|
740
|
+
# the result is visible in the web console within the same second.
|
|
741
|
+
#
|
|
742
|
+
# POST {api_url}/api/v1/alerts/submit
|
|
743
|
+
# Authorization: Bearer <token> # optional in dev mode
|
|
744
|
+
# { connector_id, connector_type, source_format, events: [...] }
|
|
745
|
+
#
|
|
746
|
+
# This deliberately bypasses the Kafka detect/correlate/fuse pipeline (which
|
|
747
|
+
# fresh clones don't run by default) so the documented "alert in seconds"
|
|
748
|
+
# quickstart promise actually holds.
|
|
749
|
+
#
|
|
750
|
+
# The fixture format is intentionally forgiving — the file may be:
|
|
751
|
+
# * { "events": [...], "connector_id": ..., "connector_type": ..., "source_format": ... }
|
|
752
|
+
# * a bare JSON list of event dicts
|
|
753
|
+
# * a single event dict (auto-wrapped)
|
|
754
|
+
# Any keys present in a dict-shaped fixture override the matching CLI flags.
|
|
755
|
+
#
|
|
756
|
+
# Backward compatibility: pre-W3 versions of this command POSTed to the Go
|
|
757
|
+
# ingest service at port 8081 with an ``X-Tenant-ID`` header. We keep the
|
|
758
|
+
# ``--ingest-url`` flag and ``AISOC_INGEST_URL`` env var as deprecated aliases
|
|
759
|
+
# for ``--api-url`` / ``AISOC_API_URL`` so existing demo scripts and the v1.0
|
|
760
|
+
# video voiceover keep working. The tenant id, when not derivable from the
|
|
761
|
+
# bearer token (dev mode bypass), comes from ``--tenant-id`` /
|
|
762
|
+
# ``AISOC_TENANT_ID`` and is still echoed in the request panel.
|
|
763
|
+
|
|
764
|
+
_DEFAULT_API_URL = "http://127.0.0.1:8000"
|
|
765
|
+
_DEFAULT_TENANT_ID = "00000000-0000-0000-0000-000000000001"
|
|
766
|
+
_DEFAULT_CONNECTOR_ID = "aisoc-cli-submit"
|
|
767
|
+
_DEFAULT_CONNECTOR_TYPE = "okta_system_log"
|
|
768
|
+
_DEFAULT_SOURCE_FORMAT = "json"
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
def _coerce_events(payload: Any) -> tuple[list[dict[str, Any]], dict[str, str]]:
|
|
772
|
+
"""Normalize a parsed JSON payload into (events, fixture_overrides).
|
|
773
|
+
|
|
774
|
+
Returns the event list plus any connector_id / connector_type /
|
|
775
|
+
source_format overrides that a dict-shaped fixture wants to pin.
|
|
776
|
+
"""
|
|
777
|
+
overrides: dict[str, str] = {}
|
|
778
|
+
if isinstance(payload, list):
|
|
779
|
+
events = payload
|
|
780
|
+
elif isinstance(payload, dict):
|
|
781
|
+
if "events" in payload and isinstance(payload["events"], list):
|
|
782
|
+
events = payload["events"]
|
|
783
|
+
for key in ("connector_id", "connector_type", "source_format"):
|
|
784
|
+
value = payload.get(key)
|
|
785
|
+
if isinstance(value, str) and value:
|
|
786
|
+
overrides[key] = value
|
|
787
|
+
else:
|
|
788
|
+
events = [payload]
|
|
789
|
+
else:
|
|
790
|
+
raise click.ClickException(
|
|
791
|
+
"Payload must be a JSON object, a list of events, or an object with an 'events' list."
|
|
792
|
+
)
|
|
793
|
+
|
|
794
|
+
cleaned: list[dict[str, Any]] = []
|
|
795
|
+
for idx, event in enumerate(events):
|
|
796
|
+
if not isinstance(event, dict):
|
|
797
|
+
raise click.ClickException(
|
|
798
|
+
f"Event at index {idx} is not a JSON object (got {type(event).__name__})."
|
|
799
|
+
)
|
|
800
|
+
cleaned.append(event)
|
|
801
|
+
|
|
802
|
+
if not cleaned:
|
|
803
|
+
raise click.ClickException("No events to submit — the payload is empty.")
|
|
804
|
+
|
|
805
|
+
return cleaned, overrides
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def _resolve_api_url(api_url: str, ingest_url: str | None) -> str:
|
|
809
|
+
"""Pick the effective API base URL, honouring deprecated `--ingest-url`.
|
|
810
|
+
|
|
811
|
+
Precedence (highest first):
|
|
812
|
+
1. ``--ingest-url`` / ``AISOC_INGEST_URL`` (deprecated, prints a warning)
|
|
813
|
+
2. ``--api-url`` / ``AISOC_API_URL``
|
|
814
|
+
3. default ``http://127.0.0.1:8000``
|
|
815
|
+
"""
|
|
816
|
+
if ingest_url:
|
|
817
|
+
console.print(
|
|
818
|
+
"[yellow]--ingest-url / AISOC_INGEST_URL is deprecated; "
|
|
819
|
+
"use --api-url / AISOC_API_URL instead.[/yellow]"
|
|
820
|
+
)
|
|
821
|
+
return ingest_url
|
|
822
|
+
return api_url
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
@cli.command()
|
|
826
|
+
@click.argument(
|
|
827
|
+
"file",
|
|
828
|
+
type=click.Path(exists=True, dir_okay=False, readable=True, path_type=Path),
|
|
829
|
+
)
|
|
830
|
+
@click.option(
|
|
831
|
+
"--api-url",
|
|
832
|
+
envvar="AISOC_API_URL",
|
|
833
|
+
default=_DEFAULT_API_URL,
|
|
834
|
+
show_default=True,
|
|
835
|
+
help="Base URL for the AiSOC API (e.g. http://127.0.0.1:8000).",
|
|
836
|
+
)
|
|
837
|
+
@click.option(
|
|
838
|
+
"--ingest-url",
|
|
839
|
+
envvar="AISOC_INGEST_URL",
|
|
840
|
+
default=None,
|
|
841
|
+
show_default=False,
|
|
842
|
+
help="DEPRECATED alias for --api-url; overrides --api-url when set.",
|
|
843
|
+
)
|
|
844
|
+
@click.option(
|
|
845
|
+
"--api-key",
|
|
846
|
+
envvar="AISOC_API_KEY",
|
|
847
|
+
default=None,
|
|
848
|
+
show_default=False,
|
|
849
|
+
help=(
|
|
850
|
+
"AiSOC bearer token. Optional in dev mode (the API resolves an "
|
|
851
|
+
"unauthenticated request to the demo tenant)."
|
|
852
|
+
),
|
|
853
|
+
)
|
|
854
|
+
@click.option(
|
|
855
|
+
"--tenant-id",
|
|
856
|
+
envvar="AISOC_TENANT_ID",
|
|
857
|
+
default=_DEFAULT_TENANT_ID,
|
|
858
|
+
show_default=True,
|
|
859
|
+
help="Tenant UUID — informational; the API derives tenant from the bearer token.",
|
|
860
|
+
)
|
|
861
|
+
@click.option(
|
|
862
|
+
"--connector-id",
|
|
863
|
+
default=_DEFAULT_CONNECTOR_ID,
|
|
864
|
+
show_default=True,
|
|
865
|
+
help="connector_id field on the synthesised alert.",
|
|
866
|
+
)
|
|
867
|
+
@click.option(
|
|
868
|
+
"--connector-type",
|
|
869
|
+
default=_DEFAULT_CONNECTOR_TYPE,
|
|
870
|
+
show_default=True,
|
|
871
|
+
help="connector_type field — used for routing and tag inference (e.g. okta_system_log).",
|
|
872
|
+
)
|
|
873
|
+
@click.option(
|
|
874
|
+
"--source-format",
|
|
875
|
+
default=_DEFAULT_SOURCE_FORMAT,
|
|
876
|
+
show_default=True,
|
|
877
|
+
help="source_format hint, echoed onto the alert for downstream parsers.",
|
|
878
|
+
)
|
|
879
|
+
@click.option(
|
|
880
|
+
"--timeout",
|
|
881
|
+
type=float,
|
|
882
|
+
default=30.0,
|
|
883
|
+
show_default=True,
|
|
884
|
+
help="HTTP timeout in seconds.",
|
|
885
|
+
)
|
|
886
|
+
def submit(
|
|
887
|
+
file: Path,
|
|
888
|
+
api_url: str,
|
|
889
|
+
ingest_url: str | None,
|
|
890
|
+
api_key: str | None,
|
|
891
|
+
tenant_id: str,
|
|
892
|
+
connector_id: str,
|
|
893
|
+
connector_type: str,
|
|
894
|
+
source_format: str,
|
|
895
|
+
timeout: float,
|
|
896
|
+
) -> None:
|
|
897
|
+
"""Submit a JSON alert/event payload to the AiSOC API.
|
|
898
|
+
|
|
899
|
+
Reads ``FILE``, normalises it into the submit envelope, and POSTs to
|
|
900
|
+
``{api_url}/api/v1/alerts/submit``. The API synthesises one ``Alert`` row
|
|
901
|
+
from the batch and writes it directly to the database, so the result is
|
|
902
|
+
visible in ``GET /api/v1/alerts`` and the web console within the same
|
|
903
|
+
second — bypassing the Kafka pipeline that fresh clones don't run.
|
|
904
|
+
|
|
905
|
+
Used by the quickstart video script to drop a lateral-movement alert
|
|
906
|
+
into the stack in one command.
|
|
907
|
+
"""
|
|
908
|
+
try:
|
|
909
|
+
raw = file.read_text(encoding="utf-8")
|
|
910
|
+
except OSError as exc: # pragma: no cover - click handles `exists=True`
|
|
911
|
+
raise click.ClickException(f"Could not read {file}: {exc}") from exc
|
|
912
|
+
|
|
913
|
+
# Strip a leading BOM, which `json.loads` otherwise refuses.
|
|
914
|
+
if raw.startswith("\ufeff"):
|
|
915
|
+
raw = raw.lstrip("\ufeff")
|
|
916
|
+
|
|
917
|
+
try:
|
|
918
|
+
payload = json.loads(raw)
|
|
919
|
+
except json.JSONDecodeError as exc:
|
|
920
|
+
raise click.ClickException(f"{file} is not valid JSON: {exc}") from exc
|
|
921
|
+
|
|
922
|
+
events, overrides = _coerce_events(payload)
|
|
923
|
+
connector_id = overrides.get("connector_id", connector_id)
|
|
924
|
+
connector_type = overrides.get("connector_type", connector_type)
|
|
925
|
+
source_format = overrides.get("source_format", source_format)
|
|
926
|
+
|
|
927
|
+
effective_url = _resolve_api_url(api_url, ingest_url)
|
|
928
|
+
url = f"{effective_url.rstrip('/')}/api/v1/alerts/submit"
|
|
929
|
+
body = {
|
|
930
|
+
"connector_id": connector_id,
|
|
931
|
+
"connector_type": connector_type,
|
|
932
|
+
"source_format": source_format,
|
|
933
|
+
"events": events,
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
console.print(
|
|
937
|
+
Panel(
|
|
938
|
+
f"[bold]file:[/bold] {file}\n"
|
|
939
|
+
f"[bold]url:[/bold] {url}\n"
|
|
940
|
+
f"[bold]tenant:[/bold] {tenant_id}\n"
|
|
941
|
+
f"[bold]connector_id:[/bold] {connector_id}\n"
|
|
942
|
+
f"[bold]connector_type:[/bold] {connector_type}\n"
|
|
943
|
+
f"[bold]source_format:[/bold] {source_format}\n"
|
|
944
|
+
f"[bold]events:[/bold] {len(events)}",
|
|
945
|
+
title="[bold green]aisoc submit[/bold green]",
|
|
946
|
+
)
|
|
947
|
+
)
|
|
948
|
+
|
|
949
|
+
headers: dict[str, str] = {"Content-Type": "application/json"}
|
|
950
|
+
if api_key:
|
|
951
|
+
headers["Authorization"] = f"Bearer {api_key}"
|
|
952
|
+
|
|
953
|
+
try:
|
|
954
|
+
with httpx.Client(timeout=timeout) as client:
|
|
955
|
+
resp = client.post(url, headers=headers, json=body)
|
|
956
|
+
except httpx.HTTPError as exc:
|
|
957
|
+
console.print(
|
|
958
|
+
f"[red]AiSOC API unreachable at {url}:[/red] {exc}\n"
|
|
959
|
+
"Is the dev stack running? Try [bold]aisoc serve[/bold] first."
|
|
960
|
+
)
|
|
961
|
+
sys.exit(1)
|
|
962
|
+
|
|
963
|
+
if resp.status_code >= 400:
|
|
964
|
+
console.print(f"[red]AiSOC API returned {resp.status_code}:[/red] {resp.text[:500]}")
|
|
965
|
+
sys.exit(1)
|
|
966
|
+
|
|
967
|
+
try:
|
|
968
|
+
data = resp.json()
|
|
969
|
+
except ValueError:
|
|
970
|
+
console.print(
|
|
971
|
+
f"[red]AiSOC API returned non-JSON ({resp.status_code}):[/red] {resp.text[:200]}"
|
|
972
|
+
)
|
|
973
|
+
sys.exit(1)
|
|
974
|
+
|
|
975
|
+
alert_id = data.get("id", "-")
|
|
976
|
+
alert_severity = data.get("severity", "-")
|
|
977
|
+
alert_title = data.get("title", "-")
|
|
978
|
+
alert_tenant = data.get("tenant_id", "-")
|
|
979
|
+
|
|
980
|
+
console.print(
|
|
981
|
+
Panel(
|
|
982
|
+
f"[bold]status:[/bold] {resp.status_code}\n"
|
|
983
|
+
f"[bold]alert_id:[/bold] {alert_id}\n"
|
|
984
|
+
f"[bold]tenant_id:[/bold] {alert_tenant}\n"
|
|
985
|
+
f"[bold]severity:[/bold] {alert_severity}\n"
|
|
986
|
+
f"[bold]title:[/bold] {alert_title}",
|
|
987
|
+
title="[bold green]alert created[/bold green]",
|
|
988
|
+
)
|
|
989
|
+
)
|
|
990
|
+
|
|
991
|
+
|
|
992
|
+
# ── replay command ───────────────────────────────────────────────────────────
|
|
993
|
+
#
|
|
994
|
+
# `aisoc replay` measures AiSOC triage against a tenant's own analysts, on
|
|
995
|
+
# their own closed findings, before anyone is asked to trust it.
|
|
996
|
+
#
|
|
997
|
+
# The CLI drives the API and does no evaluation of its own. That is forced
|
|
998
|
+
# rather than chosen: the history readers live in `services/actions`, triage in
|
|
999
|
+
# `services/agents` and `normalize()` in `services/connectors`, and all three
|
|
1000
|
+
# package their code as top-level `app`, so one Python process can hold exactly
|
|
1001
|
+
# one of them. The API is the service that already orchestrates across all
|
|
1002
|
+
# three, holds the vault and owns the tenant session.
|
|
1003
|
+
#
|
|
1004
|
+
# POST {api_url}/api/v1/evaluations/replay -> 202 with an id
|
|
1005
|
+
# GET {api_url}/api/v1/evaluations/replay/{id} -> poll to a terminal state
|
|
1006
|
+
# GET .../export?format=markdown|json|pdf -> the stored artefact
|
|
1007
|
+
#
|
|
1008
|
+
# Determinism, which is this phase's acceptance bar
|
|
1009
|
+
# -------------------------------------------------
|
|
1010
|
+
# Two runs over one history must produce the same bytes. Three things here
|
|
1011
|
+
# could break that and each is handled rather than hoped about:
|
|
1012
|
+
#
|
|
1013
|
+
# * the window. `--since` / `--until` are passed through exactly as given,
|
|
1014
|
+
# and when they are omitted the API dates the window from now — which is a
|
|
1015
|
+
# different window on a second run, and therefore a different report. The
|
|
1016
|
+
# help text says so rather than leaving it to be discovered.
|
|
1017
|
+
# * the artefact. Nothing here renders anything. The file written is the
|
|
1018
|
+
# bytes the API returned, which are the bytes stored when the run
|
|
1019
|
+
# completed.
|
|
1020
|
+
# * wall-clock latency. `--exclude-latency` asks the API for the report with
|
|
1021
|
+
# its two latency figures replaced by a note. Everything else is a
|
|
1022
|
+
# property of the input and the code.
|
|
1023
|
+
|
|
1024
|
+
_DEFAULT_POLL_SECONDS = 3.0
|
|
1025
|
+
_DEFAULT_TIMEOUT_SECONDS = 3600.0
|
|
1026
|
+
|
|
1027
|
+
#: Statuses the API will never move a run out of.
|
|
1028
|
+
_TERMINAL = frozenset({"completed", "failed"})
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
def _replay_headers(api_key: str | None) -> dict[str, str]:
|
|
1032
|
+
headers = {"Accept": "application/json"}
|
|
1033
|
+
if api_key:
|
|
1034
|
+
headers["Authorization"] = f"Bearer {api_key}"
|
|
1035
|
+
return headers
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
def _api_error(response: httpx.Response) -> str:
|
|
1039
|
+
"""The server's own reason, or the status when it did not give one."""
|
|
1040
|
+
try:
|
|
1041
|
+
body = response.json()
|
|
1042
|
+
except ValueError:
|
|
1043
|
+
return f"HTTP {response.status_code}: {response.text[:300]}"
|
|
1044
|
+
detail = body.get("detail") if isinstance(body, dict) else None
|
|
1045
|
+
return f"HTTP {response.status_code}: {detail or response.text[:300]}"
|
|
1046
|
+
|
|
1047
|
+
|
|
1048
|
+
def _poll_until_terminal(
|
|
1049
|
+
client: httpx.Client,
|
|
1050
|
+
url: str,
|
|
1051
|
+
headers: dict[str, str],
|
|
1052
|
+
*,
|
|
1053
|
+
poll_seconds: float,
|
|
1054
|
+
timeout_seconds: float,
|
|
1055
|
+
) -> dict[str, Any]:
|
|
1056
|
+
"""Poll one evaluation until it stops moving, or give up and say so.
|
|
1057
|
+
|
|
1058
|
+
A timeout here is reported as a timeout, never as a failed evaluation:
|
|
1059
|
+
the run is still going server-side and the id is printed so it can be
|
|
1060
|
+
collected later.
|
|
1061
|
+
"""
|
|
1062
|
+
deadline = time.monotonic() + timeout_seconds
|
|
1063
|
+
last_status = ""
|
|
1064
|
+
while True:
|
|
1065
|
+
response = client.get(url, headers=headers)
|
|
1066
|
+
if response.status_code >= 400:
|
|
1067
|
+
raise click.ClickException(f"Could not read the evaluation: {_api_error(response)}")
|
|
1068
|
+
body: dict[str, Any] = response.json()
|
|
1069
|
+
status = str(body.get("status") or "")
|
|
1070
|
+
if status != last_status:
|
|
1071
|
+
err_console.print(f"[dim]status:[/dim] {status}")
|
|
1072
|
+
last_status = status
|
|
1073
|
+
if status in _TERMINAL:
|
|
1074
|
+
return body
|
|
1075
|
+
if time.monotonic() >= deadline:
|
|
1076
|
+
raise click.ClickException(
|
|
1077
|
+
f"Timed out after {timeout_seconds:.0f}s with the run still '{status}'. "
|
|
1078
|
+
f"It is still running on the server; collect it with "
|
|
1079
|
+
f"`aisoc replay --collect {body.get('id')}`."
|
|
1080
|
+
)
|
|
1081
|
+
time.sleep(poll_seconds)
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
def _write_report(
|
|
1085
|
+
client: httpx.Client,
|
|
1086
|
+
base_url: str,
|
|
1087
|
+
evaluation_id: str,
|
|
1088
|
+
headers: dict[str, str],
|
|
1089
|
+
*,
|
|
1090
|
+
export_format: str,
|
|
1091
|
+
exclude_latency: bool,
|
|
1092
|
+
output: Path | None,
|
|
1093
|
+
) -> None:
|
|
1094
|
+
"""Fetch the stored artefact and write it out, unmodified.
|
|
1095
|
+
|
|
1096
|
+
Nothing is rendered, reformatted or annotated on the way through. The
|
|
1097
|
+
bytes written are the bytes the API returned.
|
|
1098
|
+
"""
|
|
1099
|
+
response = client.get(
|
|
1100
|
+
f"{base_url}/api/v1/evaluations/replay/{evaluation_id}/export",
|
|
1101
|
+
headers=headers,
|
|
1102
|
+
params={"format": export_format, "exclude_latency": str(exclude_latency).lower()},
|
|
1103
|
+
)
|
|
1104
|
+
if response.status_code >= 400:
|
|
1105
|
+
raise click.ClickException(f"Export failed: {_api_error(response)}")
|
|
1106
|
+
|
|
1107
|
+
if output is not None:
|
|
1108
|
+
output.write_bytes(response.content)
|
|
1109
|
+
err_console.print(f"[green]report written[/green] {output} ({len(response.content)} bytes)")
|
|
1110
|
+
return
|
|
1111
|
+
|
|
1112
|
+
if export_format == "pdf":
|
|
1113
|
+
raise click.ClickException("PDF is binary; pass --output to write it to a file.")
|
|
1114
|
+
# stdout, so `aisoc replay ... > report.md` produces the report and
|
|
1115
|
+
# nothing else. Every progress line above went to stderr.
|
|
1116
|
+
sys.stdout.write(response.text)
|
|
1117
|
+
|
|
1118
|
+
|
|
1119
|
+
def _print_summary(body: dict[str, Any]) -> None:
|
|
1120
|
+
"""The sample sizes beside the headline, on stderr.
|
|
1121
|
+
|
|
1122
|
+
Both are shown together on purpose. A headline accuracy with no sample
|
|
1123
|
+
size behind it is the single most misleading thing this surface could
|
|
1124
|
+
print, and a withheld headline has to say why it was withheld rather
|
|
1125
|
+
than showing a blank.
|
|
1126
|
+
"""
|
|
1127
|
+
headline = body.get("headline_accuracy")
|
|
1128
|
+
withheld = body.get("headline_withheld_reason")
|
|
1129
|
+
if headline is None:
|
|
1130
|
+
headline_line = "[yellow]withheld[/yellow]" + (f"\n {withheld}" if withheld else "")
|
|
1131
|
+
else:
|
|
1132
|
+
headline_line = (
|
|
1133
|
+
f"{float(headline) * 100:.1f}% over {body.get('graded', 0)} answered decisions"
|
|
1134
|
+
)
|
|
1135
|
+
|
|
1136
|
+
recall = body.get("malicious_recall")
|
|
1137
|
+
recall_line = "not measured" if recall is None else f"{float(recall) * 100:.1f}%"
|
|
1138
|
+
|
|
1139
|
+
err_console.print(
|
|
1140
|
+
Panel(
|
|
1141
|
+
f"[bold]evaluation:[/bold] {body.get('id')}\n"
|
|
1142
|
+
f"[bold]source:[/bold] {body.get('vendor')} ({body.get('connector_id')})\n"
|
|
1143
|
+
f"[bold]window:[/bold] {body.get('window_start')} to {body.get('window_end')}\n"
|
|
1144
|
+
f"[bold]findings read:[/bold] {body.get('findings_read', 0)} "
|
|
1145
|
+
f"({body.get('findings_labelled', 0)} carried an analyst label)\n"
|
|
1146
|
+
f"[bold]replayed:[/bold] {body.get('decisions_recorded', 0)}, "
|
|
1147
|
+
f"answered {body.get('graded', 0)}, malicious cases {body.get('malicious_support', 0)}\n"
|
|
1148
|
+
f"[bold]malicious recall:[/bold] {recall_line}\n"
|
|
1149
|
+
f"[bold]headline accuracy:[/bold] {headline_line}",
|
|
1150
|
+
title="[bold green]replay evaluation[/bold green]",
|
|
1151
|
+
)
|
|
1152
|
+
)
|
|
1153
|
+
|
|
1154
|
+
|
|
1155
|
+
@cli.command()
|
|
1156
|
+
@click.option(
|
|
1157
|
+
"--api-url",
|
|
1158
|
+
envvar="AISOC_API_URL",
|
|
1159
|
+
default=_DEFAULT_API_URL,
|
|
1160
|
+
show_default=True,
|
|
1161
|
+
help="Base URL for the AiSOC API.",
|
|
1162
|
+
)
|
|
1163
|
+
@click.option(
|
|
1164
|
+
"--api-key",
|
|
1165
|
+
envvar="AISOC_API_KEY",
|
|
1166
|
+
default=None,
|
|
1167
|
+
help="AiSOC bearer token. The tenant is derived from it; there is no tenant flag.",
|
|
1168
|
+
)
|
|
1169
|
+
@click.option(
|
|
1170
|
+
"--connector-id",
|
|
1171
|
+
default=None,
|
|
1172
|
+
help="UUID of the saved connector whose SIEM history to replay.",
|
|
1173
|
+
)
|
|
1174
|
+
@click.option(
|
|
1175
|
+
"--collect",
|
|
1176
|
+
"collect_id",
|
|
1177
|
+
default=None,
|
|
1178
|
+
help="Skip starting a run and collect an existing evaluation by id.",
|
|
1179
|
+
)
|
|
1180
|
+
@click.option(
|
|
1181
|
+
"--since",
|
|
1182
|
+
default=None,
|
|
1183
|
+
help=(
|
|
1184
|
+
"Window start, ISO-8601. Omitting it dates the window from now, which means a second "
|
|
1185
|
+
"run covers a different window and produces a different report. Pin both ends to compare runs."
|
|
1186
|
+
),
|
|
1187
|
+
)
|
|
1188
|
+
@click.option("--until", default=None, help="Window end, ISO-8601. See --since.")
|
|
1189
|
+
@click.option(
|
|
1190
|
+
"--train-fraction",
|
|
1191
|
+
type=float,
|
|
1192
|
+
default=0.7,
|
|
1193
|
+
show_default=True,
|
|
1194
|
+
help="Time split. The earlier fraction fixes the frozen context; only the later period is graded.",
|
|
1195
|
+
)
|
|
1196
|
+
@click.option(
|
|
1197
|
+
"--limit",
|
|
1198
|
+
type=int,
|
|
1199
|
+
default=1000,
|
|
1200
|
+
show_default=True,
|
|
1201
|
+
help="Maximum findings to read from the window.",
|
|
1202
|
+
)
|
|
1203
|
+
@click.option(
|
|
1204
|
+
"--seed",
|
|
1205
|
+
type=int,
|
|
1206
|
+
default=None,
|
|
1207
|
+
help="Bootstrap seed. Defaults to the platform's, and travels into the report either way.",
|
|
1208
|
+
)
|
|
1209
|
+
@click.option(
|
|
1210
|
+
"--resamples",
|
|
1211
|
+
type=int,
|
|
1212
|
+
default=None,
|
|
1213
|
+
help="Bootstrap resample count. Defaults to the platform's, and travels into the report.",
|
|
1214
|
+
)
|
|
1215
|
+
@click.option(
|
|
1216
|
+
"--format",
|
|
1217
|
+
"export_format",
|
|
1218
|
+
type=click.Choice(["markdown", "json", "pdf"]),
|
|
1219
|
+
default="markdown",
|
|
1220
|
+
show_default=True,
|
|
1221
|
+
help="Export format. All three come from the artefact stored when the run completed.",
|
|
1222
|
+
)
|
|
1223
|
+
@click.option(
|
|
1224
|
+
"--output",
|
|
1225
|
+
"-o",
|
|
1226
|
+
type=click.Path(dir_okay=False, writable=True, path_type=Path),
|
|
1227
|
+
default=None,
|
|
1228
|
+
help="Write the report here. Without it, markdown and JSON go to stdout.",
|
|
1229
|
+
)
|
|
1230
|
+
@click.option(
|
|
1231
|
+
"--exclude-latency",
|
|
1232
|
+
is_flag=True,
|
|
1233
|
+
default=False,
|
|
1234
|
+
help=(
|
|
1235
|
+
"Replace the two wall-clock latency figures with a note, so two runs over one pinned "
|
|
1236
|
+
"window are byte-identical. Latency measures the host and never reproduces."
|
|
1237
|
+
),
|
|
1238
|
+
)
|
|
1239
|
+
@click.option(
|
|
1240
|
+
"--wait/--no-wait",
|
|
1241
|
+
default=True,
|
|
1242
|
+
show_default=True,
|
|
1243
|
+
help="Poll to completion. With --no-wait the id is printed and the run continues server-side.",
|
|
1244
|
+
)
|
|
1245
|
+
@click.option(
|
|
1246
|
+
"--poll-interval",
|
|
1247
|
+
type=float,
|
|
1248
|
+
default=_DEFAULT_POLL_SECONDS,
|
|
1249
|
+
show_default=True,
|
|
1250
|
+
help="Seconds between polls.",
|
|
1251
|
+
)
|
|
1252
|
+
@click.option(
|
|
1253
|
+
"--timeout",
|
|
1254
|
+
type=float,
|
|
1255
|
+
default=_DEFAULT_TIMEOUT_SECONDS,
|
|
1256
|
+
show_default=True,
|
|
1257
|
+
help="Give up waiting after this many seconds. The run continues server-side.",
|
|
1258
|
+
)
|
|
1259
|
+
def replay(
|
|
1260
|
+
api_url: str,
|
|
1261
|
+
api_key: str | None,
|
|
1262
|
+
connector_id: str | None,
|
|
1263
|
+
collect_id: str | None,
|
|
1264
|
+
since: str | None,
|
|
1265
|
+
until: str | None,
|
|
1266
|
+
train_fraction: float,
|
|
1267
|
+
limit: int,
|
|
1268
|
+
seed: int | None,
|
|
1269
|
+
resamples: int | None,
|
|
1270
|
+
export_format: str,
|
|
1271
|
+
output: Path | None,
|
|
1272
|
+
exclude_latency: bool,
|
|
1273
|
+
wait: bool,
|
|
1274
|
+
poll_interval: float,
|
|
1275
|
+
timeout: float,
|
|
1276
|
+
) -> None:
|
|
1277
|
+
"""Measure AiSOC triage against your own analysts' past decisions.
|
|
1278
|
+
|
|
1279
|
+
Reads the findings your analysts already closed in a window, replays them
|
|
1280
|
+
through the same triage path production runs (writing nothing, reading a
|
|
1281
|
+
frozen world), and grades the result against the labels they chose.
|
|
1282
|
+
|
|
1283
|
+
The report leads with recall on malicious, prints every rate beside the
|
|
1284
|
+
count it was computed over, and withholds a headline accuracy when the
|
|
1285
|
+
window holds too few malicious cases for one to mean anything.
|
|
1286
|
+
|
|
1287
|
+
The tenant comes from the credential. There is no tenant flag, because a
|
|
1288
|
+
flag would be a value you chose that nothing checked.
|
|
1289
|
+
"""
|
|
1290
|
+
if not collect_id and not connector_id:
|
|
1291
|
+
raise click.ClickException(
|
|
1292
|
+
"Pass --connector-id to start a run, or --collect <id> to fetch one."
|
|
1293
|
+
)
|
|
1294
|
+
|
|
1295
|
+
base_url = api_url.rstrip("/")
|
|
1296
|
+
headers = _replay_headers(api_key)
|
|
1297
|
+
|
|
1298
|
+
with httpx.Client(timeout=60.0) as client:
|
|
1299
|
+
if collect_id:
|
|
1300
|
+
evaluation_id = collect_id
|
|
1301
|
+
err_console.print(f"[dim]collecting[/dim] {evaluation_id}")
|
|
1302
|
+
else:
|
|
1303
|
+
payload: dict[str, Any] = {
|
|
1304
|
+
"connector_id": connector_id,
|
|
1305
|
+
"train_fraction": train_fraction,
|
|
1306
|
+
"limit": limit,
|
|
1307
|
+
}
|
|
1308
|
+
# Only sent when given. An omitted bound lets the API date the
|
|
1309
|
+
# window, and sending a locally-computed "now" instead would put
|
|
1310
|
+
# the CLI's clock into a report the API is responsible for.
|
|
1311
|
+
if since:
|
|
1312
|
+
payload["since"] = since
|
|
1313
|
+
if until:
|
|
1314
|
+
payload["until"] = until
|
|
1315
|
+
if seed is not None:
|
|
1316
|
+
payload["bootstrap_seed"] = seed
|
|
1317
|
+
if resamples is not None:
|
|
1318
|
+
payload["bootstrap_resamples"] = resamples
|
|
1319
|
+
|
|
1320
|
+
try:
|
|
1321
|
+
response = client.post(
|
|
1322
|
+
f"{base_url}/api/v1/evaluations/replay",
|
|
1323
|
+
headers=headers,
|
|
1324
|
+
json=payload,
|
|
1325
|
+
)
|
|
1326
|
+
except httpx.HTTPError as exc:
|
|
1327
|
+
raise click.ClickException(
|
|
1328
|
+
f"AiSOC API unreachable at {base_url}: {exc}\nIs the stack running? Try `aisoc serve`."
|
|
1329
|
+
) from exc
|
|
1330
|
+
if response.status_code >= 400:
|
|
1331
|
+
raise click.ClickException(
|
|
1332
|
+
f"Could not start the evaluation: {_api_error(response)}"
|
|
1333
|
+
)
|
|
1334
|
+
started = response.json()
|
|
1335
|
+
evaluation_id = str(started.get("id"))
|
|
1336
|
+
err_console.print(f"[green]queued[/green] {evaluation_id}")
|
|
1337
|
+
|
|
1338
|
+
if not wait:
|
|
1339
|
+
err_console.print(
|
|
1340
|
+
f"[dim]not waiting. Collect it with:[/dim] aisoc replay --collect {evaluation_id}"
|
|
1341
|
+
)
|
|
1342
|
+
return
|
|
1343
|
+
|
|
1344
|
+
body = _poll_until_terminal(
|
|
1345
|
+
client,
|
|
1346
|
+
f"{base_url}/api/v1/evaluations/replay/{evaluation_id}",
|
|
1347
|
+
headers,
|
|
1348
|
+
poll_seconds=poll_interval,
|
|
1349
|
+
timeout_seconds=timeout,
|
|
1350
|
+
)
|
|
1351
|
+
|
|
1352
|
+
if body.get("status") == "failed":
|
|
1353
|
+
# The server's reason, verbatim. A replay that could not read a
|
|
1354
|
+
# window has measured nothing, and saying so is the useful answer.
|
|
1355
|
+
raise click.ClickException(
|
|
1356
|
+
f"The evaluation failed: {body.get('error') or 'no reason recorded'}"
|
|
1357
|
+
)
|
|
1358
|
+
|
|
1359
|
+
_print_summary(body)
|
|
1360
|
+
_write_report(
|
|
1361
|
+
client,
|
|
1362
|
+
base_url,
|
|
1363
|
+
evaluation_id,
|
|
1364
|
+
headers,
|
|
1365
|
+
export_format=export_format,
|
|
1366
|
+
exclude_latency=exclude_latency,
|
|
1367
|
+
output=output,
|
|
1368
|
+
)
|
|
1369
|
+
|
|
1370
|
+
|
|
1371
|
+
if __name__ == "__main__":
|
|
1372
|
+
cli()
|