argospy 0.2.2__py3-none-any.whl

argos/case.py

@@ -0,0 +1,166 @@
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable
+
+ONCE = "once"
+SOAK = "soak"
+
+
+def case_slug(case_id: str) -> str:
+    return case_id.replace(":", "-")
+
+
+def run_slug(ids: list[str]) -> str:
+    if not ids:
+        return "empty"
+    if len(ids) == 1:
+        return case_slug(ids[0])
+    groups = {part.split(":", 1)[0] if ":" in part else part for part in ids}
+    if len(groups) == 1:
+        return next(iter(groups))
+    if len(ids) <= 3:
+        return "+".join(case_slug(i) for i in ids)
+    return f"{len(ids)}-cases"
+
+
+@dataclass(frozen=True)
+class Spec:
+    id: str
+    title: str
+    group: str
+    tags: tuple[str, ...] = ()
+    modes: tuple[str, ...] = (ONCE,)
+    pack: str = ""
+
+    @property
+    def slug(self) -> str:
+        return case_slug(self.id)
+
+
+@dataclass
+class Step:
+    name: str
+    status: str
+    detail: str = ""
+
+
+@dataclass
+class Result:
+    spec: Spec
+    status: str
+    elapsed_s: float
+    error: str = ""
+    metrics: dict[str, Any] = field(default_factory=dict)
+    steps: list[Step] = field(default_factory=list)
+    notes: list[str] = field(default_factory=list)
+    dest: str = ""
+    iteration: int = 0
+
+
+class Skip(Exception):
+    pass
+
+
+class Fail(Exception):
+    pass
+
+
+class Context:
+    def __init__(
+        self,
+        spec: Spec,
+        dest: Path,
+        emit: Callable[[dict], None],
+        *,
+        iteration: int = 0,
+        run_dest: Path | None = None,
+        mode: str = ONCE,
+    ):
+        self.spec = spec
+        self.dest = dest
+        self.dest.mkdir(parents=True, exist_ok=True)
+        self.iteration = iteration
+        self.run_dest = run_dest or dest.parent
+        self.mode = mode
+        self.metrics: dict[str, Any] = {}
+        self.steps: list[Step] = []
+        self.notes: list[str] = []
+        self._emit = emit
+        self.write(
+            "case.json",
+            {
+                "id": spec.id,
+                "slug": spec.slug,
+                "title": spec.title,
+                "group": spec.group,
+                "pack": spec.pack,
+                "tags": list(spec.tags),
+                "modes": list(spec.modes),
+            },
+        )
+
+    def metric(self, key: str, value: Any) -> None:
+        self.metrics[key] = value
+        self._dump()
+        self._emit({"type": "metric", "id": self.spec.id, "key": key, "value": value})
+
+    def note(self, message: str) -> None:
+        self.notes.append(message)
+        self._emit({"type": "note", "id": self.spec.id, "message": message})
+
+    def step(self, name: str, status: str = "ok", detail: str = "") -> None:
+        row = Step(name=name, status=status, detail=detail)
+        for i, existing in enumerate(self.steps):
+            if existing.name == name:
+                self.steps[i] = row
+                break
+        else:
+            self.steps.append(row)
+        self._dump()
+        self._emit({"type": "step", "id": self.spec.id, "name": name, "status": status, "detail": detail})
+
+    def check(self, ok: bool, message: str) -> None:
+        if not ok:
+            raise Fail(message)
+
+    def skip(self, message: str) -> None:
+        raise Skip(message)
+
+    def fail(self, message: str) -> None:
+        raise Fail(message)
+
+    def own(self, resource_id: str, catalog_id: str = "") -> None:
+        path = self.run_dest / "owned.json"
+        rows: list[dict[str, str]] = []
+        if path.exists():
+            raw = json.loads(path.read_text())
+            if isinstance(raw, list):
+                rows = [row for row in raw if isinstance(row, dict)]
+        if any(row.get("id") == resource_id for row in rows):
+            return
+        rows.append({"id": resource_id, "catalog_id": catalog_id})
+        path.write_text(json.dumps(rows, indent=2, ensure_ascii=False) + "\n")
+
+    def write(self, name: str, obj: Any) -> Path:
+        path = self.dest / name
+        if isinstance(obj, (bytes, bytearray)):
+            path.write_bytes(obj)
+        elif isinstance(obj, str):
+            path.write_text(obj)
+        else:
+            path.write_text(json.dumps(obj, indent=2, ensure_ascii=False) + "\n")
+        return path
+
+    def _dump(self) -> None:
+        self.write(
+            "progress.json",
+            {
+                "id": self.spec.id,
+                "slug": self.spec.slug,
+                "title": self.spec.title,
+                "metrics": self.metrics,
+                "steps": [s.__dict__ for s in self.steps],
+                "notes": self.notes,
+            },
+        )

argos/cli.py

@@ -0,0 +1,351 @@
+import argparse
+import json
+import socket
+import sys
+import threading
+import time
+import webbrowser
+from pathlib import Path
+
+from argos.case import ONCE, SOAK, run_slug
+from argos.client import Client, DashError, dash_token, dash_url, push_dir
+from argos.duration import parse_duration
+from argos.paths import out_root
+from argos.report import write_reports
+from argos.runner import write_events
+from argos.secrets import load_secrets
+from argos.select import select
+from argos.suite import all_cases, all_packs, apply_env, env_names, execute, pack_named, packs_of
+from argos.term import DIM, RESET, Progress, fmt_dur
+
+
+def list_packs() -> int:
+    packs = all_packs()
+    print(f"{'ID':<16} {'ENVS':<24} TITLE")
+    for pack in packs:
+        print(f"{pack.id:<16} {','.join(pack.envs):<24} {pack.title}")
+    print(f"\n{len(packs)} packs")
+    return 0
+
+
+def list_cases(queries: list[str], mode: str | None) -> int:
+    catalog = all_cases()
+    specs = select([c.spec for c in catalog], queries)
+    if mode:
+        specs = [s for s in specs if mode in s.modes]
+    print(f"{'ID':<32} {'PACK':<10} {'SLUG':<28} {'GROUP':<10} {'MODES':<12} {'TAGS':<22} TITLE")
+    for spec in specs:
+        tags = ",".join(spec.tags)
+        modes = ",".join(spec.modes)
+        print(f"{spec.id:<32} {spec.pack:<10} {spec.slug:<28} {spec.group:<10} {modes:<12} {tags:<22} {spec.title}")
+    print(f"\n{len(specs)} cases")
+    print()
+    print("once   argos run <id>")
+    print("soak   argos run <id> --soak --for 8h")
+    print("env    argos run <id> --env <name>")
+    print("pack   argos list pack:<id>")
+    print("push   argos run <id> --push")
+    return 0
+
+
+def _env_help() -> str:
+    names = env_names()
+    if not names:
+        return "named environment from the selected packs"
+    return "e2e target: " + " / ".join(names)
+
+
+def _resolve_stack_pack(pack_id: str | None):
+    candidates = [pack for pack in all_packs() if pack.stack_up]
+    if pack_id:
+        pack = pack_named(pack_id)
+        if not pack:
+            raise RuntimeError(f"unknown pack {pack_id}")
+        if not pack.stack_up:
+            raise RuntimeError(f"pack {pack_id} has no stack")
+        return pack
+    if len(candidates) == 1:
+        return candidates[0]
+    if not candidates:
+        raise RuntimeError("no pack defines a stack")
+    names = " ".join(pack.id for pack in candidates)
+    raise RuntimeError(f"choose a pack: argos up <{names}>")
+
+
+def _public_event(event: dict) -> dict:
+    row = {k: v for k, v in event.items() if k != "result"}
+    if "result" in event:
+        result = event["result"]
+        row["status"] = result.status
+        row["elapsed_s"] = result.elapsed_s
+        row["error"] = result.error
+        row["iteration"] = result.iteration
+    return row
+
+
+def run_cases(
+    queries: list[str],
+    *,
+    soak: bool,
+    duration: str,
+    pause: str,
+    fail_fast: bool,
+    env: str | None,
+    push: bool,
+    dash: str,
+) -> int:
+    if not queries:
+        print("select at least one case. examples:", file=sys.stderr)
+        print("  argos list", file=sys.stderr)
+        print("  argos list pack:<id>", file=sys.stderr)
+        print("  argos run unit", file=sys.stderr)
+        return 2
+    catalog = all_cases()
+    picked = select([c.spec for c in catalog], queries)
+    if not picked:
+        print("no cases matched", " ".join(queries), file=sys.stderr)
+        return 2
+    by_id = {c.spec.id: c for c in catalog}
+    chosen = [by_id[s.id] for s in picked]
+    mode = SOAK if soak else ONCE
+    unsupported = [c.spec.id for c in chosen if mode not in c.spec.modes]
+    if unsupported:
+        print(f"{mode} not supported:", ", ".join(unsupported), file=sys.stderr)
+        print("see: argos list", file=sys.stderr)
+        return 2
+    involved = packs_of(chosen)
+    has_e2e = any("e2e" in c.spec.tags for c in chosen)
+    if has_e2e and not env:
+        names = [name for pack in involved for name in pack.envs]
+        seen: list[str] = []
+        for name in names:
+            if name not in seen:
+                seen.append(name)
+        print("e2e cases require --env " + (" or ".join(seen) if seen else "<pack env>"), file=sys.stderr)
+        return 2
+    if env:
+        try:
+            applied = apply_env(env, involved)
+        except ValueError as exc:
+            print(exc, file=sys.stderr)
+            return 2
+    else:
+        applied = ""
+
+    stamp = time.strftime("%Y%m%d-%H%M%S")
+    slug = run_slug([c.spec.id for c in chosen])
+    dest = out_root() / f"{stamp}__{slug}"
+    dest.mkdir(parents=True, exist_ok=True)
+    run_meta = {
+        "started": stamp,
+        "slug": slug,
+        "mode": mode,
+        "env": applied,
+        "packs": [pack.id for pack in involved],
+        "queries": queries,
+        "runner": socket.gethostname(),
+        "cases": [
+            {
+                "id": c.spec.id,
+                "slug": c.spec.slug,
+                "title": c.spec.title,
+                "group": c.spec.group,
+                "pack": c.spec.pack,
+            }
+            for c in chosen
+        ],
+    }
+    dest.joinpath("run.json").write_text(json.dumps(run_meta, indent=2, ensure_ascii=False) + "\n")
+    events = dest / "events.jsonl"
+    progress = Progress([c.spec for c in chosen], str(dest))
+    lock = threading.Lock()
+    remote: Client | None = None
+    remote_id = ""
+    pending: list[dict] = []
+
+    if push:
+        token = dash_token()
+        if not token:
+            print("ARGOS_TOKEN is required for --push", file=sys.stderr)
+            return 2
+        remote = Client(dash_url(dash), token)
+        try:
+            created = remote.create_run(run_meta)
+        except DashError as exc:
+            print(exc, file=sys.stderr)
+            return 2
+        remote_id = str(created["id"])
+        print(f"dash  {created.get('url') or remote.browse_url(remote_id)}")
+
+    def flush_remote() -> None:
+        if not remote or not remote_id or not pending:
+            return
+        batch = pending[:]
+        pending.clear()
+        remote.post_events(remote_id, batch)
+
+    def emit(event: dict) -> None:
+        with lock:
+            write_events(events, event)
+            progress.on_event(event)
+            if remote and remote_id:
+                pending.append(_public_event(event))
+                if len(pending) >= 20:
+                    try:
+                        flush_remote()
+                    except DashError as exc:
+                        print(exc, file=sys.stderr)
+
+    print(f"run {dest}")
+    print(f"env {applied or '-'}  mode {mode}  selected {len(chosen)}: " + " ".join(c.spec.id for c in chosen))
+    for pack in involved:
+        pack_cases = [c for c in chosen if c.spec.pack == pack.id]
+        if pack.needs_stack and pack.needs_stack(pack_cases):
+            try:
+                pack.stack_up()
+            except RuntimeError as exc:
+                print(exc, file=sys.stderr)
+                return 2
+    started = time.time()
+    results = []
+    try:
+        if not soak:
+            results = execute(chosen, dest, emit, iteration=0, mode=mode)
+        else:
+            budget = parse_duration(duration)
+            gap = parse_duration(pause) if pause else 0.0
+            deadline = started + budget
+            print(f"soak {fmt_dur(budget).strip()}  pause {fmt_dur(gap).strip()}  fail_fast={fail_fast}")
+            n = 0
+            while time.time() < deadline:
+                n += 1
+                left = deadline - time.time()
+                emit({"type": "note", "id": chosen[0].spec.id, "message": f"soak iter {n}  left {fmt_dur(left).strip()}"})
+                batch = execute(chosen, dest, emit, iteration=n, mode=mode)
+                results.extend(batch)
+                if fail_fast and any(r.status == "fail" for r in batch):
+                    break
+                if time.time() + gap >= deadline:
+                    break
+                if gap > 0:
+                    time.sleep(gap)
+    except KeyboardInterrupt:
+        print("interrupted", file=sys.stderr)
+    if soak:
+        for pack in involved:
+            if pack.soak_teardown:
+                pack.soak_teardown(dest, emit, [c for c in chosen if c.spec.pack == pack.id])
+    write_reports(dest, results, stamp, wall_s=time.time() - started)
+    progress.summary()
+    print(f"report  {dest / 'report.html'}")
+    print(f"json    {dest / 'report.json'}")
+    print(f"md      {dest / 'report.md'}")
+    if remote and remote_id:
+        try:
+            with lock:
+                flush_remote()
+            report = json.loads((dest / "report.json").read_text())
+            status = "fail" if any(r.status == "fail" for r in results) else "pass"
+            if not results:
+                status = "interrupted"
+            remote.finish(remote_id, report, status)
+            print(f"dash  {remote.browse_url(remote_id)}")
+        except DashError as exc:
+            print(exc, file=sys.stderr)
+    if sys.stdout.isatty():
+        print(f"{DIM}open the HTML report for the visual summary{RESET}")
+    return 0 if results and all(r.status != "fail" for r in results) else 1
+
+
+def cmd_push(path: str, dash: str) -> int:
+    dest = Path(path)
+    if not dest.is_dir():
+        print(f"not a run directory: {dest}", file=sys.stderr)
+        return 2
+    token = dash_token()
+    if not token:
+        print("ARGOS_TOKEN is required", file=sys.stderr)
+        return 2
+    client = Client(dash_url(dash), token)
+    try:
+        run_id = push_dir(client, dest)
+    except DashError as exc:
+        print(exc, file=sys.stderr)
+        return 2
+    print(client.browse_url(run_id))
+    return 0
+
+
+def cmd_dash(dash: str, no_open: bool) -> int:
+    url = dash_url(dash)
+    print(url)
+    if not no_open:
+        webbrowser.open(url)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    load_secrets()
+    parser = argparse.ArgumentParser(prog="argos", description="once / soak test framework")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("packs", help="list installed packs")
+    p_list = sub.add_parser("list", help="list cases and how to run them")
+    p_list.add_argument("query", nargs="*", help="id, pack:, group:, tag:, mode:once|soak, or glob")
+    p_list.add_argument("--mode", choices=(ONCE, SOAK), help="only cases that support this mode")
+    p_run = sub.add_parser("run", help="run selected cases once, or soak until --for")
+    p_run.add_argument("query", nargs="*", help="id, pack:, group:, tag:, mode:, or glob")
+    p_run.add_argument("--soak", action="store_true", help="repeat until --for (k6-style constant duration)")
+    p_run.add_argument("--for", dest="duration", default="8h", help="soak budget (8h, 90m, 1h30m). default 8h")
+    p_run.add_argument("--pause", default="0s", help="sleep between soak iterations")
+    p_run.add_argument("--fail-fast", action="store_true", help="stop soak on first failed iteration")
+    p_run.add_argument("--env", choices=env_names() or None, help=_env_help())
+    p_run.add_argument("--push", action="store_true", help="stream this run to dash")
+    p_run.add_argument("--dash", default="", help="dash base URL (default ARGOS_DASH_URL or https://argos.saidc.ai)")
+    p_up = sub.add_parser("up", help="start a pack's local stack")
+    p_up.add_argument("pack", nargs="?", help="pack id (default: the only pack that has a stack)")
+    p_down = sub.add_parser("down", help="stop a pack's local stack")
+    p_down.add_argument("pack", nargs="?", help="pack id (default: the only pack that has a stack)")
+    p_push = sub.add_parser("push", help="upload a finished local run directory")
+    p_push.add_argument("path", help="out/<stamp>__<slug> directory")
+    p_push.add_argument("--dash", default="", help="dash base URL")
+    p_dash = sub.add_parser("dash", help="open the configured dash")
+    p_dash.add_argument("--dash", default="", help="dash base URL")
+    p_dash.add_argument("--no-open", action="store_true", help="print the URL only")
+    args = parser.parse_args(argv)
+    if args.cmd == "packs":
+        return list_packs()
+    if args.cmd == "list":
+        return list_cases(args.query, args.mode)
+    if args.cmd == "push":
+        return cmd_push(args.path, args.dash)
+    if args.cmd == "dash":
+        return cmd_dash(args.dash, args.no_open)
+    if args.cmd in {"up", "down"}:
+        try:
+            pack = _resolve_stack_pack(args.pack)
+            if pack.envs:
+                apply_env(pack.envs[0], [pack])
+            if args.cmd == "up":
+                pack.stack_up()
+            else:
+                if not pack.stack_down:
+                    raise RuntimeError(f"pack {pack.id} has no stack down")
+                pack.stack_down()
+        except RuntimeError as exc:
+            print(exc, file=sys.stderr)
+            return 2
+        return 0
+    return run_cases(
+        args.query,
+        soak=args.soak,
+        duration=args.duration,
+        pause=args.pause,
+        fail_fast=args.fail_fast,
+        env=args.env,
+        push=args.push,
+        dash=args.dash,
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

argos/client.py

@@ -0,0 +1,111 @@
+import json
+import os
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any
+from urllib.parse import urljoin
+
+DEFAULT_DASH = "https://argos.saidc.ai"
+
+
+def dash_url(override: str = "") -> str:
+    raw = (override or os.environ.get("ARGOS_DASH_URL") or DEFAULT_DASH).strip()
+    return raw.rstrip("/")
+
+
+def dash_token() -> str:
+    return (os.environ.get("ARGOS_TOKEN") or "").strip()
+
+
+class DashError(RuntimeError):
+    pass
+
+
+class Client:
+    def __init__(self, base: str, token: str) -> None:
+        self.base = base.rstrip("/")
+        self.token = token
+
+    def create_run(self, body: dict[str, Any]) -> dict[str, Any]:
+        return self._json("POST", "/api/runs", body)
+
+    def post_events(self, run_id: str, events: list[dict[str, Any]]) -> dict[str, Any]:
+        return self._json("POST", f"/api/runs/{run_id}/events", {"events": events})
+
+    def finish(self, run_id: str, report: dict[str, Any], status: str) -> dict[str, Any]:
+        return self._json("POST", f"/api/runs/{run_id}/finish", {"report": report, "status": status})
+
+    def upload_file(self, run_id: str, rel: str, text: str) -> dict[str, Any]:
+        return self._json("POST", f"/api/runs/{run_id}/files", {"path": rel, "text": text})
+
+    def browse_url(self, run_id: str) -> str:
+        return f"{self.base}/runs/{run_id}"
+
+    def _json(self, method: str, path: str, body: dict[str, Any] | None = None) -> dict[str, Any]:
+        data = None if body is None else json.dumps(body, ensure_ascii=False, default=str).encode("utf-8")
+        req = urllib.request.Request(
+            urljoin(self.base + "/", path.lstrip("/")),
+            data=data,
+            method=method,
+            headers={
+                "Accept": "application/json",
+                "Content-Type": "application/json; charset=utf-8",
+                "Authorization": f"Bearer {self.token}",
+            },
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                raw = resp.read().decode("utf-8")
+        except urllib.error.HTTPError as exc:
+            detail = exc.read().decode("utf-8", errors="replace")
+            raise DashError(f"{method} {path} -> {exc.code}: {detail}") from exc
+        except urllib.error.URLError as exc:
+            raise DashError(f"{method} {path}: {exc.reason}") from exc
+        if not raw:
+            return {}
+        parsed = json.loads(raw)
+        if not isinstance(parsed, dict):
+            raise DashError(f"{method} {path}: expected object")
+        return parsed
+
+
+def push_dir(client: Client, dest: Path) -> str:
+    run_meta = _read_json(dest / "run.json")
+    report = _read_json(dest / "report.json")
+    created = client.create_run({**run_meta, "status": "done"})
+    run_id = str(created["id"])
+    events_path = dest / "events.jsonl"
+    if events_path.is_file():
+        rows = [_parse_event(line) for line in events_path.read_text().splitlines() if line.strip()]
+        if rows:
+            client.post_events(run_id, rows)
+    status = "fail" if report.get("failed") else "pass"
+    if report:
+        client.finish(run_id, report, status)
+    for path in dest.rglob("*"):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(dest).as_posix()
+        if rel in {"events.jsonl"}:
+            continue
+        if path.stat().st_size > 256_000:
+            continue
+        try:
+            text = path.read_text()
+        except UnicodeDecodeError:
+            continue
+        client.upload_file(run_id, rel, text)
+    return run_id
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    if not path.is_file():
+        return {}
+    raw = json.loads(path.read_text())
+    return raw if isinstance(raw, dict) else {}
+
+
+def _parse_event(line: str) -> dict[str, Any]:
+    row = json.loads(line)
+    return row if isinstance(row, dict) else {"raw": line}

argos/duration.py

@@ -0,0 +1,32 @@
+import re
+
+_TOKEN = re.compile(r"(\d+(?:\.\d+)?)([hms])", re.I)
+
+
+def parse_duration(raw: str) -> float:
+    text = raw.strip().lower()
+    if not text:
+        raise ValueError("empty duration")
+    if text.isdigit():
+        return float(text)
+    if _TOKEN.fullmatch(text) or _TOKEN.match(text):
+        total = 0.0
+        pos = 0
+        for match in _TOKEN.finditer(text):
+            if match.start() != pos:
+                raise ValueError(f"invalid duration {raw!r}")
+            value = float(match.group(1))
+            unit = match.group(2)
+            if unit == "h":
+                total += value * 3600
+            elif unit == "m":
+                total += value * 60
+            else:
+                total += value
+            pos = match.end()
+        if pos != len(text):
+            raise ValueError(f"invalid duration {raw!r}")
+        if total <= 0:
+            raise ValueError(f"duration must be > 0: {raw!r}")
+        return total
+    raise ValueError(f"invalid duration {raw!r} (use 8h, 90m, 45s, 1h30m)")