humanbound-cli 0.4.0__tar.gz → 0.4.1__tar.gz

{humanbound_cli-0.4.0 → humanbound_cli-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: humanbound-cli
-Version: 0.4.0
+Version: 0.4.1
 Summary: Humanbound CLI - command line interface for AI agent security testing.
 Author-email: Kostas Siabanis <hello@humanbound.ai>, Demetris Gerogiannis <hello@humanbound.ai>
 License: Apache-2.0

{humanbound_cli-0.4.0 → humanbound_cli-0.4.1}/humanbound_cli/client.py

@@ -6,13 +6,11 @@ import time
 import webbrowser
 import http.server
 import socketserver
-import threading
 import secrets
 import hashlib
 import base64
 import urllib.parse
 from typing import Optional, Dict, Any, List
-from pathlib import Path
 
 import requests
 
@@ -906,6 +904,47 @@ class HumanboundClient:
             include_project=True,
         )
 
+    def get_project_logs(
+        self,
+        page: int = 1,
+        size: int = 50,
+        result: Optional[str] = None,
+        from_date: Optional[str] = None,
+        until_date: Optional[str] = None,
+        test_category: Optional[str] = None,
+        last: Optional[int] = None,
+    ) -> dict:
+        """Get logs for the current project with optional filters.
+
+        Args:
+            page: Page number.
+            size: Items per page.
+            result: Filter by result (pass/fail).
+            from_date: Start date (ISO 8601).
+            until_date: End date (ISO 8601).
+            test_category: Filter by test category (substring match).
+            last: Limit to last N experiments.
+
+        Returns:
+            Paginated response with logs.
+        """
+        if not self._project_id:
+            raise ValidationError("No project selected. Use set_project() first.")
+
+        params: Dict[str, Any] = {"page": page, "size": size}
+        if result:
+            params["result"] = result
+        if from_date:
+            params["from"] = from_date
+        if until_date:
+            params["until"] = until_date
+        if test_category:
+            params["test_category"] = test_category
+        if last:
+            params["last"] = last
+
+        return self.get("logs", params=params, include_project=True)
+
     # -------------------------------------------------------------------------
     # Provider Methods
     # -------------------------------------------------------------------------

humanbound_cli-0.4.1/humanbound_cli/commands/logs.py

@@ -0,0 +1,479 @@
+"""Logs command for retrieving and exporting experiment results."""
+
+import click
+from rich.console import Console
+from rich.table import Table
+import json
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from ..client import HumanboundClient
+from ..exceptions import NotAuthenticatedError, APIError
+
+console = Console()
+console_err = Console(stderr=True)
+
+
+@click.command("logs")
+@click.argument("experiment_id", required=False)
+@click.option(
+    "--format", "-f", "output_format",
+    type=click.Choice(["table", "json", "html"]),
+    default="table",
+    help="Output format"
+)
+@click.option(
+    "--output", "-o",
+    type=click.Path(),
+    help="Output file path (prints to stdout if not specified)"
+)
+@click.option(
+    "--verdict", "-v",
+    type=click.Choice(["pass", "fail", "all"]),
+    default="all",
+    help="Filter by verdict"
+)
+@click.option(
+    "--page", default=1, help="Page number (for table format)"
+)
+@click.option(
+    "--size", default=50, help="Items per page (for table format)"
+)
+@click.option(
+    "--all", "fetch_all", is_flag=True, help="Fetch all logs (for json format)"
+)
+@click.option(
+    "--last", "last_n", type=int, help="Logs from last N experiments"
+)
+@click.option(
+    "--category", "test_category", help="Filter by test category (substring match)"
+)
+@click.option(
+    "--from", "from_date", help="Start date (ISO 8601, e.g. 2026-01-01)"
+)
+@click.option(
+    "--until", "until_date", help="End date (ISO 8601)"
+)
+@click.option(
+    "--days", type=int, help="Last N days (shortcut for --from)"
+)
+def logs_command(experiment_id, output_format, output, verdict, page, size, fetch_all, last_n, test_category, from_date, until_date, days):
+    """Get logs from an experiment or across a project.
+
+    If no experiment_id or scope flags are provided, uses the most recent experiment.
+    Use scope flags (--last, --category, --from, --until, --days) for project-wide logs.
+
+    \b
+    Examples:
+      hb logs                                    # Latest experiment logs
+      hb logs abc123                             # Specific experiment
+      hb logs --last 5                           # Last 5 experiments
+      hb logs --last 3 --verdict fail            # Failed logs from last 3
+      hb logs --category owasp_multi_turn        # All multi-turn logs
+      hb logs --days 7 --format json -o week.json
+      hb logs --from 2026-01-01 --until 2026-02-01 --format html -o jan.html
+    """
+    client = HumanboundClient()
+
+    if not client.is_authenticated():
+        console_err.print("[red]Not authenticated.[/red] Run 'hb login' first.")
+        raise SystemExit(1)
+
+    if not client.project_id:
+        console_err.print("[yellow]No project selected.[/yellow]")
+        console_err.print("Use 'hb projects use <id>' to select a project first.")
+        raise SystemExit(1)
+
+    # Validation
+    scope_flags = any([last_n, test_category, from_date, until_date, days])
+    if experiment_id and scope_flags:
+        console_err.print("[red]Cannot combine experiment ID with scope flags.[/red]")
+        console_err.print("Use either an experiment ID OR scope flags (--last, --category, --from, --until, --days).")
+        raise SystemExit(1)
+    if days and from_date:
+        console_err.print("[red]Cannot combine --days with --from.[/red]")
+        raise SystemExit(1)
+
+    # --days → --from
+    if days:
+        from_date = (datetime.utcnow() - timedelta(days=days)).strftime("%Y-%m-%dT00:00:00")
+
+    try:
+        if scope_flags:
+            _project_level_logs(
+                client, output_format, output, verdict, page, size, fetch_all,
+                last_n, test_category, from_date, until_date,
+            )
+        elif experiment_id:
+            # Resolve partial experiment ID
+            experiment_id = _resolve_experiment_id(client, experiment_id)
+
+            if output_format == "html":
+                _export_html(client, experiment_id, output)
+            elif output_format == "json":
+                _export_json(client, experiment_id, output, verdict, fetch_all, page, size)
+            else:
+                _show_table(client, experiment_id, verdict, page, size)
+        else:
+            # No args → most recent experiment (existing behavior)
+            response = client.list_experiments(page=1, size=1)
+            exps = response.get("data", [])
+            if not exps:
+                console_err.print("[yellow]No experiments found.[/yellow]")
+                raise SystemExit(1)
+            experiment_id = exps[0].get("id")
+            console_err.print(f"[dim]Using most recent experiment: {experiment_id}[/dim]")
+
+            if output_format == "html":
+                _export_html(client, experiment_id, output)
+            elif output_format == "json":
+                _export_json(client, experiment_id, output, verdict, fetch_all, page, size)
+            else:
+                _show_table(client, experiment_id, verdict, page, size)
+
+    except NotAuthenticatedError:
+        console_err.print("[red]Not authenticated.[/red] Run 'hb login' first.")
+        raise SystemExit(1)
+    except APIError as e:
+        console_err.print(f"[red]Error:[/red] {e}")
+        raise SystemExit(1)
+
+
+# ---------------------------------------------------------------------------
+# Project-level logs
+# ---------------------------------------------------------------------------
+
+def _build_experiment_lookup(client):
+    """Fetch all experiments and build {id: {name, test_category}} lookup."""
+    lookup = {}
+    current_page = 1
+    while True:
+        response = client.list_experiments(page=current_page, size=100)
+        for exp in response.get("data", []):
+            lookup[exp.get("id")] = {
+                "name": exp.get("name", ""),
+                "test_category": exp.get("test_category", ""),
+            }
+        if not response.get("has_next_page"):
+            break
+        current_page += 1
+    return lookup
+
+
+def _enrich_log(log, exp_lookup):
+    """Add experiment_name and test_category to a log entry from lookup."""
+    exp_id = log.get("experiment_id", "")
+    info = exp_lookup.get(exp_id, {})
+    log["experiment_name"] = info.get("name", "")
+    log["test_category"] = info.get("test_category", "")
+    return log
+
+
+def _project_level_logs(client, output_format, output, verdict, page, size, fetch_all,
+                        last_n, test_category, from_date, until_date):
+    """Fetch and display project-level logs with scope filters."""
+    result_filter = None if verdict == "all" else verdict
+
+    # Build experiment lookup for enriching logs
+    exp_lookup = _build_experiment_lookup(client)
+
+    if output_format == "html":
+        _project_export_html(client, output, result_filter, last_n, test_category, from_date, until_date, exp_lookup)
+    elif output_format == "json":
+        _project_export_json(client, output, result_filter, fetch_all, page, size, last_n, test_category, from_date, until_date, exp_lookup)
+    else:
+        _project_show_table(client, result_filter, page, size, last_n, test_category, from_date, until_date, exp_lookup)
+
+
+def _project_show_table(client, result_filter, page, size, last_n, test_category, from_date, until_date, exp_lookup):
+    """Show project-level logs in table format."""
+    response = client.get_project_logs(
+        page=page, size=size, result=result_filter,
+        from_date=from_date, until_date=until_date,
+        test_category=test_category, last=last_n,
+    )
+    logs = response.get("data", [])
+
+    if not logs:
+        console.print("[yellow]No logs found.[/yellow]")
+        return
+
+    table = Table(title=f"Project Logs (page {page})")
+    table.add_column("Experiment", width=20)
+    table.add_column("Test Category", width=20)
+    table.add_column("Verdict", width=6)
+    table.add_column("Severity", width=8)
+    table.add_column("Category", width=15)
+    table.add_column("Prompt", max_width=40)
+
+    for log in logs:
+        _enrich_log(log, exp_lookup)
+
+        result_val = log.get("result", "")
+        result_style = "[green]pass[/green]" if result_val == "pass" else "[red]fail[/red]"
+
+        severity = log.get("severity", "")
+        severity_style = {
+            "critical": "[red bold]critical[/red bold]",
+            "high": "[red]high[/red]",
+            "medium": "[yellow]medium[/yellow]",
+            "low": "[blue]low[/blue]",
+        }.get(str(severity).lower(), str(severity))
+
+        # Shorten test_category for display
+        tc = log.get("test_category", "")
+        tc_short = tc.split("/")[-1] if "/" in tc else tc
+
+        table.add_row(
+            (log.get("experiment_name", "") or "")[:20],
+            tc_short[:20],
+            result_style,
+            severity_style if result_val == "fail" else "",
+            log.get("fail_category") or log.get("gen_category") or "",
+            (log.get("prompt", "") or "")[:40],
+        )
+
+    console.print(table)
+
+    if response.get("has_next_page"):
+        console.print(f"\n[dim]Showing {len(logs)} logs. Use --page to see more.[/dim]")
+
+
+def _project_export_json(client, output, result_filter, fetch_all, page, size, last_n, test_category, from_date, until_date, exp_lookup):
+    """Export project-level logs as JSON."""
+    all_logs = []
+
+    if fetch_all:
+        current_page = 1
+        while True:
+            response = client.get_project_logs(
+                page=current_page, size=100, result=result_filter,
+                from_date=from_date, until_date=until_date,
+                test_category=test_category, last=last_n,
+            )
+            logs = response.get("data", [])
+            all_logs.extend(logs)
+            if not response.get("has_next_page"):
+                break
+            current_page += 1
+    else:
+        response = client.get_project_logs(
+            page=page, size=size, result=result_filter,
+            from_date=from_date, until_date=until_date,
+            test_category=test_category, last=last_n,
+        )
+        all_logs = response.get("data", [])
+
+    # Enrich each log with experiment name and test_category
+    for log in all_logs:
+        _enrich_log(log, exp_lookup)
+
+    export_data = {
+        "project_id": client.project_id,
+        "filters": {
+            "last": last_n,
+            "test_category": test_category,
+            "from": from_date,
+            "until": until_date,
+            "result": result_filter,
+        },
+        "logs": all_logs,
+        "total_logs": len(all_logs),
+    }
+
+    json_output = json.dumps(export_data, indent=2, default=str)
+
+    if output:
+        Path(output).write_text(json_output)
+        console.print(f"[green]JSON exported to:[/green] {output}")
+    else:
+        print(json_output)
+
+
+def _project_export_html(client, output, result_filter, last_n, test_category, from_date, until_date, exp_lookup):
+    """Export project-level logs as HTML report."""
+    with console.status("Generating HTML report...", spinner="dots"):
+        # Fetch all matching logs
+        all_logs = []
+        current_page = 1
+        while True:
+            response = client.get_project_logs(
+                page=current_page, size=100, result=result_filter,
+                from_date=from_date, until_date=until_date,
+                test_category=test_category, last=last_n,
+            )
+            all_logs.extend(response.get("data", []))
+            if not response.get("has_next_page"):
+                break
+            current_page += 1
+
+        # Enrich each log with experiment name and test_category
+        for log in all_logs:
+            _enrich_log(log, exp_lookup)
+
+        # Build pseudo-experiment for the report template
+        pseudo_experiment = {
+            "id": f"project-{client.project_id[:8]}",
+            "name": "Project Logs",
+            "test_category": test_category or "Project-wide",
+            "testing_level": "",
+            "status": "completed",
+            "results": {},
+            "created_at": from_date or "",
+        }
+
+        from ..report import generate_html_report
+        report_html = generate_html_report(pseudo_experiment, all_logs)
+
+    filename = output or f"project_{client.project_id[:8]}_logs.html"
+    Path(filename).write_text(report_html)
+    console.print(f"[green]HTML report exported to:[/green] {filename}")
+
+
+# ---------------------------------------------------------------------------
+# Experiment-level helpers (unchanged)
+# ---------------------------------------------------------------------------
+
+def _resolve_experiment_id(client: HumanboundClient, partial_id: str) -> str:
+    """Resolve a partial experiment ID to full ID."""
+    if len(partial_id) >= 32:
+        return partial_id
+
+    # Search recent experiments for match
+    response = client.list_experiments(page=1, size=50)
+    for exp in response.get("data", []):
+        if exp.get("id", "").startswith(partial_id):
+            return exp.get("id")
+
+    # Not found, return as-is and let API handle error
+    return partial_id
+
+
+def _show_table(client: HumanboundClient, experiment_id: str, verdict: str, page: int, size: int):
+    """Show logs in table format."""
+    result_filter = None if verdict == "all" else verdict
+
+    response = client.get_experiment_logs(
+        experiment_id,
+        page=page,
+        size=size,
+        result=result_filter,
+    )
+    logs = response.get("data", [])
+
+    if not logs:
+        console.print("[yellow]No logs found.[/yellow]")
+        return
+
+    table = Table(title=f"Experiment Logs (page {page})")
+    table.add_column("ID", style="dim")
+    table.add_column("Verdict", width=6)
+    table.add_column("Severity", width=8)
+    table.add_column("Category", width=15)
+    table.add_column("Prompt", max_width=50)
+
+    for log in logs:
+        result_val = log.get("result", "")
+        result_style = "[green]pass[/green]" if result_val == "pass" else "[red]fail[/red]"
+
+        severity = log.get("severity", "")
+        severity_style = {
+            "critical": "[red bold]critical[/red bold]",
+            "high": "[red]high[/red]",
+            "medium": "[yellow]medium[/yellow]",
+            "low": "[blue]low[/blue]",
+        }.get(str(severity).lower(), str(severity))
+
+        table.add_row(
+            log.get("id", ""),
+            result_style,
+            severity_style if result_val == "fail" else "",
+            log.get("fail_category") or log.get("gen_category") or "",
+            (log.get("prompt", "") or "")[:50],
+        )
+
+    console.print(table)
+
+    total = response.get("total", 0)
+    if response.get("has_next_page"):
+        console.print(f"\n[dim]Showing {len(logs)} of {total}. Use --page to see more.[/dim]")
+
+
+def _export_json(client: HumanboundClient, experiment_id: str, output: str, verdict: str, fetch_all: bool, page: int, size: int):
+    """Export logs as JSON."""
+    result_filter = None if verdict == "all" else verdict
+
+    all_logs = []
+
+    if fetch_all:
+        # Fetch all pages
+        current_page = 1
+        while True:
+            response = client.get_experiment_logs(
+                experiment_id,
+                page=current_page,
+                size=100,
+                result=result_filter,
+            )
+            logs = response.get("data", [])
+            all_logs.extend(logs)
+
+            if not response.get("has_next_page"):
+                break
+            current_page += 1
+    else:
+        response = client.get_experiment_logs(
+            experiment_id,
+            page=page,
+            size=size,
+            result=result_filter,
+        )
+        all_logs = response.get("data", [])
+
+    # Get experiment info for context
+    experiment = client.get_experiment(experiment_id)
+
+    export_data = {
+        "experiment": {
+            "id": experiment.get("id"),
+            "name": experiment.get("name"),
+            "status": experiment.get("status"),
+            "test_category": experiment.get("test_category"),
+            "testing_level": experiment.get("testing_level"),
+            "created_at": experiment.get("created_at"),
+        },
+        "results": experiment.get("results", {}),
+        "logs": all_logs,
+        "total_logs": len(all_logs),
+    }
+
+    json_output = json.dumps(export_data, indent=2, default=str)
+
+    if output:
+        Path(output).write_text(json_output)
+        console.print(f"[green]JSON exported to:[/green] {output}")
+    else:
+        print(json_output)
+
+
+def _export_html(client: HumanboundClient, experiment_id: str, output: str):
+    """Export logs as HTML report."""
+    with console.status("Generating HTML report...", spinner="dots"):
+        experiment = client.get_experiment(experiment_id)
+
+        # Fetch all logs
+        all_logs = []
+        page = 1
+        while True:
+            resp = client.get_experiment_logs(experiment_id, page=page, size=100)
+            all_logs.extend(resp.get("data", []))
+            if not resp.get("has_next_page"):
+                break
+            page += 1
+
+        from ..report import generate_html_report
+        report_html = generate_html_report(experiment, all_logs)
+
+    filename = output or f"experiment_{experiment_id[:8]}_report.html"
+    Path(filename).write_text(report_html)
+    console.print(f"[green]HTML report exported to:[/green] {filename}")