flintai-cli 1.0.0__py3-none-any.whl

flintai/cli/init_cli.py

@@ -0,0 +1,144 @@
+"""Subcommand for `flintai init`."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import stat
+
+from rich.prompt import Prompt
+
+from flintai.cli.console import console, select
+from flintai.cli.utils import get_flintai_config_path, get_flintai_env_path, get_flintai_dir
+
+_PROVIDER_DEFAULTS: dict[str, dict[str, str | None]] = {
+    "gemini": {
+        "model": "gemini-3.5-flash",
+        "api_key_var": "GEMINI_API_KEY",
+    },
+    "openai": {
+        "model": "gpt-5.4",
+        "api_key_var": "OPENAI_API_KEY",
+    },
+    "anthropic": {
+        "model": "claude-sonnet-4-6",
+        "api_key_var": "ANTHROPIC_API_KEY",
+    },
+    "litellm": {
+        "model": "gpt-5.4",
+        "api_key_var": None,
+    },
+}
+
+_LITELLM_BACKENDS: dict[str, str] = {
+    "openai": "OPENAI_API_KEY",
+    "gemini": "GEMINI_API_KEY",
+    "anthropic": "ANTHROPIC_API_KEY",
+}
+
+
+def register(
+    subparsers: argparse._SubParsersAction,
+) -> None:
+    subparsers.add_parser(
+        "init",
+        help="Initialize flintai configuration",
+    )
+
+
+def run_init() -> None:
+    flintai_dir = get_flintai_dir()
+
+    console.print("[bold cyan]Flint AI Setup[/bold cyan]")
+    console.print()
+
+    if flintai_dir.exists():
+        proceed = select(
+            "Existing configuration will be overwritten."
+            " Do you want to proceed?",
+            options=["yes", "no"],
+            default_index=0,
+        )
+        if proceed.lower() == "no":
+            console.print("[dim]Aborted.[/dim]")
+            return
+        console.print()
+
+    provider = select(
+        "Flint AI CLI needs an LLM for certain functionality. Select your model provider:",
+        options=["gemini", "openai", "anthropic", "litellm"],
+        default_index=0,
+    )
+    console.print()
+
+    defaults = _PROVIDER_DEFAULTS[provider]
+
+    if provider == "litellm":
+        model_name = Prompt.ask(
+            "LiteLLM model string (e.g. openai/gpt-5.4, gemini/gemini-2.5-flash)",
+            default="openai/gpt-5.4",
+        )
+        console.print()
+
+        backend = model_name.split("/")[0] if "/" in model_name else None
+        api_key_var = _LITELLM_BACKENDS.get(backend, None) if backend else None
+        if not api_key_var:
+            api_key_var = Prompt.ask("API key env var name")
+            console.print()
+
+        generator_model = f"litellm:{model_name}"
+    else:
+        model_name = Prompt.ask(
+            "Model name",
+            default=defaults["model"],
+        )
+        console.print()
+        api_key_var = defaults["api_key_var"]
+        generator_model = f"{provider}:{model_name}"
+
+    assert api_key_var is not None
+    api_key = Prompt.ask(
+        f"Enter your {api_key_var}",
+        password=True,
+    )
+    console.print()
+    env_lines = [
+        f"GENERATOR_MODEL={generator_model}",
+        f"{api_key_var}={api_key}",
+        "EXECUTOR_MAX_WORKERS=20"
+    ]
+
+    default_config = {
+        "models": [],
+        "evaluations": [],
+        "detectors": [],
+        "message_collections": [],
+        "model_evaluations": [],
+    }
+
+    old_umask = os.umask(0o077)
+    try:
+        flintai_dir.mkdir(parents=True, exist_ok=True)
+
+        env_path = get_flintai_env_path()
+        env_path.write_text("\n".join(env_lines) + "\n")
+        env_path.chmod(stat.S_IRUSR | stat.S_IWUSR)
+
+        config_path = get_flintai_config_path()
+        config_path.write_text(
+            json.dumps(default_config, indent=2) + "\n",
+        )
+    finally:
+        os.umask(old_umask)
+
+    console.print(f"[green]Created {env_path}[/green]")
+    console.print(
+        f"[green]Created {config_path}[/green]",
+    )
+    console.print()
+    console.print(
+        "[bold green]Flint AI initialized"
+        " successfully![/bold green]",
+    )
+    console.print()

flintai/cli/main.py

@@ -0,0 +1,262 @@
+"""
+CLI entry point for Flint AI CLI.
+
+Usage:
+    python -m flintai.cli init
+    python -m flintai.cli eval models list
+    python -m flintai.cli eval run <model-evaluation-id>
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+import os
+import sys
+import time
+import traceback
+from datetime import datetime
+
+from dotenv import load_dotenv
+from rich.panel import Panel
+from rich.text import Text
+
+from flintai.cli import eval_cli, scan_cli, init_cli
+from flintai.cli.console import CLI_WIDTH, console
+from flintai.cli.version import VERSION
+from flintai.cli.utils import is_ci
+from flintai.eval.common.log import setup_file_logging
+from flintai.eval.db.json.repository_json import JsonRepository
+
+logger = logging.getLogger(__name__)
+
+_LOGO_LINES = [
+    "███████╗██╗     ██╗███╗   ██╗████████╗     █████╗ ██╗",
+    "██╔════╝██║     ██║████╗  ██║╚══██╔══╝    ██╔══██╗██║",
+    "█████╗  ██║     ██║██╔██╗ ██║   ██║       ███████║██║",
+    "██╔══╝  ██║     ██║██║╚██╗██║   ██║       ██╔══██║██║",
+    "██║     ███████╗██║██║ ╚████║   ██║       ██║  ██║██║",
+    "╚═╝     ╚══════╝╚═╝╚═╝  ╚═══╝   ╚═╝       ╚═╝  ╚═╝╚═╝",
+]
+
+
+_TIMESTAMP = datetime.now().strftime("%Y%m%dT%H%M%S")
+
+_BUILTIN_CONFIG = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    "builtin_config.json",
+)
+
+
+
+def _count(repo: JsonRepository) -> dict[str, int]:
+    return {
+        "models": len(repo.models.list()),
+        "evaluations": len(repo.evaluations.list()),
+        "detectors": len(repo.detectors.list()),
+        "collections": len(repo.message_collections.list()),
+        "assignments": len(repo.model_evaluations.list_all()),
+    }
+
+
+def _fmt_count(total: int, builtin: int, user: int) -> str:
+    return f"{total} [dim]({builtin} builtin, {user} user)[/dim]"
+
+
+def _print_logo() -> None:
+    console.print()
+    for line in _LOGO_LINES:
+        pad = max(0, (CLI_WIDTH - len(line)) // 2)
+        console.print(" " * pad + f"[bold cyan]{line}[/bold cyan]")
+    version_text = f"v{VERSION}"
+    pad = max(0, (CLI_WIDTH - len(version_text)) // 2)
+    console.print(" " * pad + f"[dim][bold cyan]{version_text}[/bold cyan][/dim]")
+    console.print()
+
+
+def _print_config_info(
+    merged: JsonRepository,
+    builtin: JsonRepository,
+    user: JsonRepository,
+    config_path: str,
+) -> None:
+    m = _count(merged)
+    b = _count(builtin)
+    u = _count(user)
+
+    console.print(f"[dim]Config:       {config_path}[/dim]")
+    console.print(f"[dim]Models:       {_fmt_count(m['models'], b['models'], u['models'])}[/dim]")
+    console.print(f"[dim]Evaluations:  {_fmt_count(m['evaluations'], b['evaluations'], u['evaluations'])}[/dim]")
+    console.print(f"[dim]Detectors:    {_fmt_count(m['detectors'], b['detectors'], u['detectors'])}[/dim]")
+    console.print(f"[dim]Collections:  {_fmt_count(m['collections'], b['collections'], u['collections'])}[/dim]")
+    console.print(f"[dim]Assignments:  {_fmt_count(m['assignments'], b['assignments'], u['assignments'])}[/dim]")
+    console.print()
+
+
+def _print_path(
+    path: str,
+) -> None:
+    console.print(f"[dim]Path:         {path}[/dim]")
+    console.print()
+
+
+def _print_shutdown(
+    elapsed: float,
+    output_path: str | None,
+    log_path: str | None,
+) -> None:
+    console.print()
+    parts = [f"[dim]Completed in [bold cyan]{elapsed:.1f}s[/bold cyan][/dim]"]
+    if output_path:
+        parts.append(f"[dim]Results: [bold cyan]{output_path}[/bold cyan][/dim]")
+    if log_path:
+        parts.append(f"[dim]Logs: [bold cyan]{log_path}[/bold cyan][/dim]")
+    console.print("  ".join(parts))
+    console.print()
+
+
+def _dispatch(args: argparse.Namespace) -> str | None:
+    logger.info("Command: %s", " ".join(sys.argv))
+
+    if args.command == "eval":
+        return _dispatch_eval(args)
+    elif args.command == "scan":
+        return _dispatch_scan(args)
+    elif args.command == "init":
+        return _dispatch_init(args)
+    else:
+        console.print(f"[red]Unknown command: {args.command}[/red]")
+        sys.exit(1)
+
+
+def _dispatch_init(args: argparse.Namespace) -> str | None:
+    init_cli.run_init()
+    return None
+
+
+def _dispatch_scan(args: argparse.Namespace) -> None | None:
+    logger.info("flintai v%s | scan mode", VERSION)
+    _print_path(args.path)
+    return scan_cli.handle_scan(args)
+
+
+def _dispatch_eval(args: argparse.Namespace) -> str | None:
+    """Run the requested command. Returns output file path if any."""
+    if args.command != "eval":
+        console.print(f"[red]Unknown command: {args.command}[/red]")
+        sys.exit(1)
+
+    config_path = getattr(args, "config", None) or str(
+        init_cli.get_flintai_config_path(),
+    )
+    builtin = JsonRepository(_BUILTIN_CONFIG)
+    user = JsonRepository(config_path)
+    store = user.merge(builtin)
+
+    _print_config_info(store, builtin, user, config_path)
+
+    m = _count(store)
+    logger.info(
+        "flintai v%s | models=%d, evaluations=%d, assignments=%d",
+        VERSION, m["models"], m["evaluations"], m["assignments"],
+    )
+
+    cmd = args.eval_cmd
+    if cmd == "models":
+        eval_cli.handle_models(args, store)
+    elif cmd == "evaluations":
+        eval_cli.handle_evaluations(args, store)
+    elif cmd == "model-evaluations":
+        eval_cli.handle_model_evaluations(
+            args, store, user,
+        )
+    elif cmd == "run":
+        return asyncio.run(eval_cli.handle_run(args, store))
+    else:
+        console.print(f"[red]Unknown eval subcommand: {cmd}[/red]")
+        sys.exit(1)
+    return None
+
+
+def _print_error(error: BaseException) -> None:
+    error_type = type(error).__name__
+    error_msg = str(error) or "(no message)"
+
+    body = Text()
+    body.append(f"{error_type}: ", style="bold red")
+    body.append(error_msg)
+
+    panel = Panel(
+        body,
+        title="[bold red]Error[/bold red]",
+        border_style="red",
+        width=min(CLI_WIDTH, max(60, len(error_msg) + 20)),
+        padding=(1, 2),
+    )
+    console.print()
+    console.print(panel)
+
+
+def main(argv: list[str] | None = None) -> None:
+    load_dotenv()
+
+    parser = argparse.ArgumentParser(
+        prog="flintai",
+        description="Flint AI CLI — AI Agent Evaluation Framework",
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"%(prog)s {VERSION}",
+    )
+
+    subparsers = parser.add_subparsers(dest="command")
+    subparsers.required = True
+    eval_cli.register(subparsers)
+    scan_cli.register(subparsers)
+    init_cli.register(subparsers)
+
+    args = parser.parse_args(argv)
+
+    ci = is_ci()
+    flintai_env = init_cli.get_flintai_env_path()
+
+    if not flintai_env.exists() and args.command != "init":
+        if ci:
+            console.print(
+                "[dim]CI environment detected —"
+                " skipping interactive init.[/dim]",
+            )
+        else:
+            console.print(
+                "[yellow]First-time setup required."
+                " Running flintai init...[/yellow]",
+            )
+            console.print()
+            init_cli.run_init()
+
+    if flintai_env.exists():
+        load_dotenv(flintai_env)
+
+    log_path = getattr(args, "log", None) or f"flintai_{_TIMESTAMP}.log"
+    setup_file_logging(log_path)
+    _print_logo()
+
+    t0 = time.monotonic()
+    output_path: str | None = None
+    try:
+        output_path = _dispatch(args)
+    except KeyboardInterrupt:
+        logger.info("Interrupted by user")
+        console.print("\n[dim]Interrupted.[/dim]")
+        sys.exit(130)
+    except Exception as e:
+        logger.critical(
+            "Fatal error: %s: %s\n%s",
+            type(e).__name__, e, traceback.format_exc(),
+        )
+        _print_error(e)
+        elapsed = time.monotonic() - t0
+        _print_shutdown(elapsed, output_path, log_path)
+        sys.exit(1)
+    elapsed = time.monotonic() - t0
+    _print_shutdown(elapsed, output_path, log_path)

flintai/cli/rich_observer.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from rich.progress import (
+    BarColumn,
+    Progress,
+    ProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TaskID,
+)
+from rich.spinner import Spinner
+from rich.text import Text
+
+from flintai.cli.console import console, score_style
+from flintai.eval.core.eval.evaluation import Evaluation
+
+
+class StatusColumn(ProgressColumn):
+    """Spinner while running, green check when done, red X on error."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._spinner = Spinner("dots")
+
+    def render(self, task: object) -> Text:
+        if task.finished:
+            status = task.fields.get("final_status", "finished")
+            if status == "error":
+                return Text("✗", style="bold red")
+            return Text("✔", style="bold green")
+        return self._spinner.render(task.get_time())
+
+
+class ProgressOrScoreColumn(ProgressColumn):
+    """Shows M/N while running, score percentage when finished."""
+
+    def render(self, task: object) -> Text:
+        if task.finished:
+            score = task.fields.get("score")
+            if score is not None:
+                pct = score * 100
+                return Text(f"{pct:.0f}%", style=score_style(score))
+            return Text("—", style="dim")
+        completed = int(task.completed)
+        if task.total is not None:
+            return Text(f"{completed}/{int(task.total)}")
+        return Text(f"{completed}/?")
+
+
+def create_progress() -> Progress:
+    return Progress(
+        StatusColumn(),
+        TextColumn("{task.description}"),
+        BarColumn(),
+        ProgressOrScoreColumn(),
+        TextColumn("[dim]|[/dim]"),
+        TimeElapsedColumn(),
+        console=console,
+    )
+
+
+def pad_description(name: str, width: int) -> str:
+    if len(name) > width:
+        return name[: width - 1] + "…"
+    return name.ljust(width)
+
+
+class RichObserver:
+    """Evaluation observer that updates a task in a shared Progress."""
+
+    def __init__(self, progress: Progress, task_id: TaskID):
+        self._progress = progress
+        self._task_id = task_id
+
+    def __call__(self, evaluation: Evaluation) -> None:
+        summary = evaluation.get_summary()
+        done = summary.finished_evaluations + summary.error_evaluations
+        self._progress.update(self._task_id, completed=done)
+
+    def finish(self, evaluation: Evaluation) -> None:
+        summary = evaluation.get_summary()
+        total = summary.total_evaluations or 1
+        self._progress.update(
+            self._task_id,
+            completed=total,
+            total=total,
+            final_status=summary.status.value,
+            score=summary.score,
+        )