humanbound-cli 0.7.0__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/PKG-INFO +65 -1
  2. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/README.md +64 -0
  3. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/__init__.py +2 -1
  4. humanbound_cli-0.9.0/humanbound_cli/commands/firewall.py +270 -0
  5. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/main.py +2 -0
  6. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/PKG-INFO +65 -1
  7. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/SOURCES.txt +1 -0
  8. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/top_level.txt +0 -1
  9. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/pyproject.toml +1 -1
  10. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/LICENSE +0 -0
  11. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/__init__.py +0 -0
  12. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/client.py +0 -0
  13. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/api_keys.py +0 -0
  14. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/assessments.py +0 -0
  15. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/auth.py +0 -0
  16. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/campaigns.py +0 -0
  17. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/completion.py +0 -0
  18. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/connect.py +0 -0
  19. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/connectors.py +0 -0
  20. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/coverage.py +0 -0
  21. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/discover.py +0 -0
  22. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/docs.py +0 -0
  23. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/experiments.py +0 -0
  24. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/findings.py +0 -0
  25. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/guardrails.py +0 -0
  26. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/init.py +0 -0
  27. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/inventory.py +0 -0
  28. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/logs.py +0 -0
  29. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/mcp.py +0 -0
  30. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/members.py +0 -0
  31. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/monitor.py +0 -0
  32. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/orgs.py +0 -0
  33. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/posture.py +0 -0
  34. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/projects.py +0 -0
  35. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/providers.py +0 -0
  36. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/report.py +0 -0
  37. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/scan.py +0 -0
  38. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/sentinel.py +0 -0
  39. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/test.py +0 -0
  40. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/upload_logs.py +0 -0
  41. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/webhooks.py +0 -0
  42. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/config.py +0 -0
  43. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/connectors/__init__.py +0 -0
  44. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/connectors/microsoft.py +0 -0
  45. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/exceptions.py +0 -0
  46. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/extractors/__init__.py +0 -0
  47. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/extractors/openapi.py +0 -0
  48. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/extractors/repo.py +0 -0
  49. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/mcp_server.py +0 -0
  50. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/pytest_plugin/__init__.py +0 -0
  51. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/pytest_plugin/fixtures.py +0 -0
  52. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/pytest_plugin/report.py +0 -0
  53. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/report.py +0 -0
  54. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/report_builder.py +0 -0
  55. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/serve/__init__.py +0 -0
  56. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/serve/config_builder.py +0 -0
  57. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/serve/local_server.py +0 -0
  58. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/serve/runtime_detector.py +0 -0
  59. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/serve/tunnel_client.py +0 -0
  60. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/dependency_links.txt +0 -0
  61. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/entry_points.txt +0 -0
  62. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/requires.txt +0 -0
  63. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/relay/relay.py +0 -0
  64. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/setup.cfg +0 -0
  65. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/tests/__init__.py +0 -0
  66. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/tests/cli_integration_test.py +0 -0
  67. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/tests/conftest.py +0 -0
  68. {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/tests/test_cli_commands.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: humanbound-cli
3
- Version: 0.7.0
3
+ Version: 0.9.0
4
4
  Summary: Humanbound CLI - command line interface for AI agent security testing.
5
5
  Author-email: Kostas Siabanis <hello@humanbound.ai>, Demetris Gerogiannis <hello@humanbound.ai>
6
6
  License: Apache-2.0
@@ -60,6 +60,7 @@ Humanbound runs automated adversarial attacks against your bot's live endpoint,
60
60
  | **Posture Scoring** | Quantified 0-100 security score with breakdown by findings, coverage, and resilience. Track over time. |
61
61
  | **Shadow AI Discovery** | Scan cloud tenants for AI services, assess risk with 15 SAI threat classes, and govern your AI inventory. |
62
62
  | **Guardrails Export** | Generate protection rules from test findings. Export to OpenAI or Humanbound format. |
63
+ | **Firewall Training** | Train agent-specific Tier 2 classifiers from adversarial + QA test data. Pluggable model architecture via AgentClassifier scripts. |
63
64
  | **MCP Server** | Model Context Protocol server exposing all CLI capabilities as tools for AI assistants (Claude Code, Cursor, Gemini CLI, etc.). |
64
65
 
65
66
  ### Why Humanbound?
@@ -557,6 +558,30 @@ hb findings [--status open] [--severity high] [--json]
557
558
  hb guardrails [--vendor humanbound|openai] [--format json|yaml] [-o FILE]
558
559
  ```
559
560
 
561
+ ### Firewall
562
+
563
+ Train agent-specific classifiers for [hb-firewall](https://github.com/humanbound/firewall).
564
+
565
+ ```bash
566
+ # Train from adversarial + QA test data
567
+ hb firewall train --model detectors/setfit_classifier.py
568
+
569
+ # Show model info
570
+ hb firewall show firewall.hbfw
571
+ ```
572
+
573
+ | Flag | Default | Description |
574
+ |------|---------|-------------|
575
+ | `--model PATH` | — | Path to AgentClassifier script (required) |
576
+ | `--last N` | 10 | Last N finished experiments |
577
+ | `--from DATE` | — | Start date filter |
578
+ | `--until DATE` | — | End date filter |
579
+ | `--min-samples` | 30 | Minimum conversations required |
580
+ | `--output` | firewall_\<project\>.hbfw | Output file path |
581
+ | `--benign-dataset` | — | HuggingFace dataset for benign benchmarking |
582
+
583
+ See [hb-firewall docs](https://github.com/humanbound/firewall) for the AgentClassifier interface and full integration guide.
584
+
560
585
  ### Shell Completion
561
586
 
562
587
  ```bash
@@ -723,6 +748,22 @@ hb connect -e ./bot-config.json
723
748
  hb test -e ./bot-config.json
724
749
  ```
725
750
 
751
+ ### Whitebox testing with telemetry
752
+
753
+ Add a `telemetry` block to your agent config to enable whitebox testing. Humanbound fetches tool calls, memory operations, and resource usage from your observability platform (LangFuse, LangSmith, OpenAI Assistants, W&B, Helicone, AgentOps, or custom).
754
+
755
+ ```json
756
+ {
757
+ "telemetry": {
758
+ "format": "langfuse",
759
+ "endpoint": "https://cloud.langfuse.com/api/public/sessions/$session_id",
760
+ "headers": { "Authorization": "Basic <base64(pk:sk)>" }
761
+ }
762
+ }
763
+ ```
764
+
765
+ See the full [Telemetry Integration Guide](https://docs.humanbound.ai/integrations/telemetry/) for vendor-specific setup and the custom extraction map reference.
766
+
726
767
  ### Shadow AI discovery & governance
727
768
 
728
769
  ```bash
@@ -759,6 +800,29 @@ claude mcp add humanbound -- hb mcp
759
800
  hb guardrails --vendor openai --format json -o guardrails.json
760
801
  ```
761
802
 
803
+ ### Train and deploy firewall
804
+
805
+ ```bash
806
+ # Run adversarial tests
807
+ hb test
808
+
809
+ # Train Tier 1 classifier from results
810
+ hb firewall train -o model.hbfw
811
+
812
+ # Verify quality
813
+ hb firewall show model.hbfw
814
+ # F1=0.95, Precision=0.97, Tier 1 coverage=92%
815
+
816
+ # Test before deploying
817
+ # Deploy in your app
818
+ python -c "
819
+ from hb_firewall import Firewall
820
+ fw = Firewall.from_config('agent.yaml', model_path='model.hbfw')
821
+ result = fw.evaluate('What is my balance?')
822
+ print(result.verdict, result.tier) # Verdict.PASS 1
823
+ "
824
+ ```
825
+
762
826
  ---
763
827
 
764
828
  ### On-Premises
@@ -27,6 +27,7 @@ Humanbound runs automated adversarial attacks against your bot's live endpoint,
27
27
  | **Posture Scoring** | Quantified 0-100 security score with breakdown by findings, coverage, and resilience. Track over time. |
28
28
  | **Shadow AI Discovery** | Scan cloud tenants for AI services, assess risk with 15 SAI threat classes, and govern your AI inventory. |
29
29
  | **Guardrails Export** | Generate protection rules from test findings. Export to OpenAI or Humanbound format. |
30
+ | **Firewall Training** | Train agent-specific Tier 2 classifiers from adversarial + QA test data. Pluggable model architecture via AgentClassifier scripts. |
30
31
  | **MCP Server** | Model Context Protocol server exposing all CLI capabilities as tools for AI assistants (Claude Code, Cursor, Gemini CLI, etc.). |
31
32
 
32
33
  ### Why Humanbound?
@@ -524,6 +525,30 @@ hb findings [--status open] [--severity high] [--json]
524
525
  hb guardrails [--vendor humanbound|openai] [--format json|yaml] [-o FILE]
525
526
  ```
526
527
 
528
+ ### Firewall
529
+
530
+ Train agent-specific classifiers for [hb-firewall](https://github.com/humanbound/firewall).
531
+
532
+ ```bash
533
+ # Train from adversarial + QA test data
534
+ hb firewall train --model detectors/setfit_classifier.py
535
+
536
+ # Show model info
537
+ hb firewall show firewall.hbfw
538
+ ```
539
+
540
+ | Flag | Default | Description |
541
+ |------|---------|-------------|
542
+ | `--model PATH` | — | Path to AgentClassifier script (required) |
543
+ | `--last N` | 10 | Last N finished experiments |
544
+ | `--from DATE` | — | Start date filter |
545
+ | `--until DATE` | — | End date filter |
546
+ | `--min-samples` | 30 | Minimum conversations required |
547
+ | `--output` | firewall_\<project\>.hbfw | Output file path |
548
+ | `--benign-dataset` | — | HuggingFace dataset for benign benchmarking |
549
+
550
+ See [hb-firewall docs](https://github.com/humanbound/firewall) for the AgentClassifier interface and full integration guide.
551
+
527
552
  ### Shell Completion
528
553
 
529
554
  ```bash
@@ -690,6 +715,22 @@ hb connect -e ./bot-config.json
690
715
  hb test -e ./bot-config.json
691
716
  ```
692
717
 
718
+ ### Whitebox testing with telemetry
719
+
720
+ Add a `telemetry` block to your agent config to enable whitebox testing. Humanbound fetches tool calls, memory operations, and resource usage from your observability platform (LangFuse, LangSmith, OpenAI Assistants, W&B, Helicone, AgentOps, or custom).
721
+
722
+ ```json
723
+ {
724
+ "telemetry": {
725
+ "format": "langfuse",
726
+ "endpoint": "https://cloud.langfuse.com/api/public/sessions/$session_id",
727
+ "headers": { "Authorization": "Basic <base64(pk:sk)>" }
728
+ }
729
+ }
730
+ ```
731
+
732
+ See the full [Telemetry Integration Guide](https://docs.humanbound.ai/integrations/telemetry/) for vendor-specific setup and the custom extraction map reference.
733
+
693
734
  ### Shadow AI discovery & governance
694
735
 
695
736
  ```bash
@@ -726,6 +767,29 @@ claude mcp add humanbound -- hb mcp
726
767
  hb guardrails --vendor openai --format json -o guardrails.json
727
768
  ```
728
769
 
770
+ ### Train and deploy firewall
771
+
772
+ ```bash
773
+ # Run adversarial tests
774
+ hb test
775
+
776
+ # Train Tier 1 classifier from results
777
+ hb firewall train -o model.hbfw
778
+
779
+ # Verify quality
780
+ hb firewall show model.hbfw
781
+ # F1=0.95, Precision=0.97, Tier 1 coverage=92%
782
+
783
+ # Test before deploying
784
+ # Deploy in your app
785
+ python -c "
786
+ from hb_firewall import Firewall
787
+ fw = Firewall.from_config('agent.yaml', model_path='model.hbfw')
788
+ result = fw.evaluate('What is my balance?')
789
+ print(result.verdict, result.tier) # Verdict.PASS 1
790
+ "
791
+ ```
792
+
729
793
  ---
730
794
 
731
795
  ### On-Premises
@@ -5,7 +5,7 @@ from . import (
5
5
  guardrails, docs, providers, findings, api_keys, members,
6
6
  coverage, campaigns, upload_logs, sentinel, discover,
7
7
  connectors, inventory, completion, connect, report, monitor,
8
- webhooks, assessments,
8
+ webhooks, assessments, firewall,
9
9
  )
10
10
 
11
11
  # MCP command is optional — only available when mcp SDK is installed
@@ -42,5 +42,6 @@ __all__ = [
42
42
  "monitor",
43
43
  "webhooks",
44
44
  "assessments",
45
+ "firewall",
45
46
  "mcp",
46
47
  ]
@@ -0,0 +1,270 @@
1
+ """Firewall commands — train and manage Tier 2 classifiers for hb-firewall."""
2
+
3
+ import sys
4
+ import time
5
+ from pathlib import Path
6
+
7
+ import click
8
+ from rich.console import Console
9
+ from rich.progress import Progress
10
+
11
+ from ..client import HumanboundClient
12
+ from ..exceptions import NotAuthenticatedError, APIError
13
+
14
+ console = Console()
15
+
16
+
17
+ @click.group("firewall")
18
+ def firewall_group():
19
+ """Train and manage Tier 2 classifiers for hb-firewall.
20
+
21
+ \b
22
+ Workflow:
23
+ 1. Run adversarial tests: hb test
24
+ 2. Train classifiers: hb firewall train --model detectors/my_model.py
25
+ 3. Load in your app: Firewall.from_config("agent.yaml", model_path="fw.hbfw")
26
+ """
27
+
28
+
29
+ @firewall_group.command("train")
30
+ @click.option("--model", "model_path", type=str, required=False, default=None,
31
+ help="Path to AgentClassifier script (e.g. detectors/one_class_svm.py)")
32
+ @click.option("--last", "last_n", type=int, default=10,
33
+ help="Last N finished experiments (default: 10)")
34
+ @click.option("--from", "from_date", type=str, default=None)
35
+ @click.option("--until", "until_date", type=str, default=None)
36
+ @click.option("--min-samples", type=int, default=30)
37
+ @click.option("--output", "-o", type=click.Path(), default=None)
38
+ def train_command(model_path, last_n, from_date, until_date, min_samples,
39
+ output):
40
+ """Train Tier 2 classifiers from adversarial + QA test logs."""
41
+ if not model_path:
42
+ # Default to SetFit classifier shipped with hb-firewall
43
+ import importlib.resources
44
+ try:
45
+ # Try to find setfit_classifier.py relative to hb_firewall package
46
+ import hb_firewall
47
+ pkg_dir = Path(hb_firewall.__file__).parent.parent.parent
48
+ default = pkg_dir / "detectors" / "setfit_classifier.py"
49
+ if default.exists():
50
+ model_path = str(default)
51
+ else:
52
+ console.print("[red]Default SetFit classifier not found.[/red]")
53
+ console.print(" Provide a path to an AgentClassifier script:")
54
+ console.print(" hb firewall train --model detectors/setfit_classifier.py")
55
+ sys.exit(1)
56
+ except Exception:
57
+ console.print("[red]Provide --model flag.[/red]")
58
+ console.print(" hb firewall train --model detectors/setfit_classifier.py")
59
+ sys.exit(1)
60
+
61
+ try:
62
+ client = HumanboundClient()
63
+ project_id = client.project_id
64
+ if not project_id:
65
+ console.print("[red]No active project.[/red] Run: hb projects use <id>")
66
+ sys.exit(1)
67
+
68
+ # Load detector
69
+ try:
70
+ from hb_firewall.hbfw import HBFW, load_model_class, save_hbfw
71
+ except ImportError:
72
+ console.print("[red]Install: pip install hb-firewall[/red]")
73
+ sys.exit(1)
74
+
75
+ try:
76
+ detector_cls = load_model_class(model_path)
77
+ except ValueError as e:
78
+ console.print(f"[red]{e}[/red]")
79
+ sys.exit(1)
80
+
81
+ console.print(f"\n[bold]Training firewall classifiers[/bold]")
82
+ console.print(f" Project: {project_id[:8]}...")
83
+ console.print(f" Model: {model_path}")
84
+
85
+ # Step 1: Fetch experiments
86
+ console.print(f"\n[bold]Step 1:[/bold] Fetching experiments...")
87
+ adv_exps = _fetch_experiments(client, "adversarial", last_n=last_n,
88
+ from_date=from_date, until_date=until_date)
89
+ qa_exps = _fetch_experiments(client, "adversarial", exclude=True,
90
+ last_n=last_n, from_date=from_date, until_date=until_date)
91
+ if not adv_exps:
92
+ console.print("[red]No finished adversarial experiments.[/red] Run: hb test")
93
+ sys.exit(1)
94
+ console.print(f" Found {len(adv_exps)} adversarial + {len(qa_exps)} QA experiments")
95
+
96
+ # Step 2: Fetch logs
97
+ console.print(f"\n[bold]Step 2:[/bold] Pulling conversation logs...")
98
+ logs = _fetch_logs(client, adv_exps + qa_exps)
99
+ if len(logs) < min_samples:
100
+ console.print(f"[red]Only {len(logs)} conversations (min: {min_samples}).[/red]")
101
+ sys.exit(1)
102
+
103
+ n_pass = sum(1 for l in logs if l.get("result") == "pass")
104
+ n_fail = len(logs) - n_pass
105
+ adv_fail = sum(1 for l in logs
106
+ if "adversarial" in (l.get("test_category") or "")
107
+ and l.get("result") == "fail")
108
+ console.print(f" Collected {len(logs)} conversations ({n_pass} pass, {n_fail} fail)")
109
+ console.print(f" Training on {adv_fail} failed adversarial conversations")
110
+
111
+ permitted, restricted = _fetch_intents(client, project_id)
112
+ if permitted or restricted:
113
+ console.print(f" Intents: {len(permitted or [])} permitted, {len(restricted or [])} restricted")
114
+
115
+ # Step 3: Prepare
116
+ console.print(f"\n[bold]Step 3:[/bold] Preparing training data...")
117
+ hbfw = HBFW(attack_detector=detector_cls("attack"),
118
+ benign_detector=detector_cls("benign"))
119
+ data = hbfw.prepare(logs, restricted_intents=restricted, permitted_intents=permitted)
120
+
121
+ stats = data.get("stats", {})
122
+ console.print(f" Attack samples: {stats.get('attack_samples', 0)} (curated: {stats.get('curated_attack', 0)})")
123
+ console.print(f" Benign samples: {stats.get('benign_samples', 0)} (curated: {stats.get('curated_benign', 0)})")
124
+ if not data.get("has_qa"):
125
+ console.print(f" [yellow]No QA tests. Using permitted intents as benign data.[/yellow]")
126
+
127
+ # Step 4: Train
128
+ console.print(f"\n[bold]Step 4:[/bold] Training...")
129
+ performance = hbfw.train(data, permitted_intents=permitted,
130
+ restricted_intents=restricted)
131
+
132
+ val = performance.get("validation")
133
+ if val:
134
+ af = val.get("adversarial_fail", {})
135
+ ap = val.get("adversarial_pass", {})
136
+ bn = val.get("benign", {})
137
+ console.print(f"\n [bold]Validation (conversation replay)[/bold]")
138
+ if af.get("total"):
139
+ r = af.get("rate", 0)
140
+ s = "green" if r >= 0.8 else "yellow" if r >= 0.5 else "red"
141
+ console.print(f" Failed adversarial caught: [{s}]{af['caught']}/{af['total']} ({r:.1%})[/{s}]")
142
+ if ap.get("total"):
143
+ r = ap.get("rate", 0)
144
+ s = "green" if r >= 0.5 else "dim"
145
+ console.print(f" Passed adversarial caught: [{s}]{ap['caught']}/{ap['total']} ({r:.1%})[/{s}]")
146
+ if bn.get("total"):
147
+ r = bn.get("rate", 0)
148
+ s = "green" if r >= 0.8 else "yellow" if r >= 0.5 else "red"
149
+ console.print(f" Benign allowed: [{s}]{bn['correct']}/{bn['total']} ({r:.1%})[/{s}]")
150
+ if bn.get("blocked"):
151
+ console.print(f" [red]Benign blocked: {bn['blocked']}[/red]")
152
+
153
+ console.print(f" Training complete.")
154
+
155
+ # Save
156
+ if output is None:
157
+ output = f"firewall_{project_id[:8]}.hbfw"
158
+ model_data = hbfw.export()
159
+ model_data["config"]["project_id"] = project_id
160
+ model_data["config"]["created_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ")
161
+ model_data["config"]["n_conversations"] = len(logs)
162
+ model_data["config"]["detector"] = Path(model_path).stem
163
+ save_hbfw(model_data, output)
164
+
165
+ file_size = Path(output).stat().st_size / 1024
166
+ console.print(f"\n[green]Model saved:[/green] {output} ({file_size:.0f} KB)")
167
+
168
+ except NotAuthenticatedError:
169
+ console.print("[red]Not authenticated.[/red] Run: hb login")
170
+ sys.exit(1)
171
+ except APIError as e:
172
+ console.print(f"[red]API error:[/red] {e}")
173
+ sys.exit(1)
174
+
175
+
176
+ @firewall_group.command("show")
177
+ @click.argument("model_path", type=click.Path(exists=True))
178
+ def show_command(model_path):
179
+ """Show model info from a trained .hbfw file."""
180
+ try:
181
+ from hb_firewall.hbfw import load_hbfw
182
+ except ImportError:
183
+ console.print("[red]hb-firewall not installed.[/red]")
184
+ sys.exit(1)
185
+
186
+ config, _ = load_hbfw(model_path)
187
+ console.print(f"\n[bold]Firewall Model: {model_path}[/bold]")
188
+ console.print(f" Created: {config.get('created_at', '?')}")
189
+ console.print(f" Project: {config.get('project_id', '?')}")
190
+ console.print(f" Detector: {config.get('detector', '?')}")
191
+ perf = config.get("performance", {})
192
+ stats = perf.get("stats", {})
193
+ if stats:
194
+ console.print(f" Attack samples: {stats.get('attack_samples', '?')}")
195
+ console.print(f" Benign samples: {stats.get('benign_samples', '?')}")
196
+ val = perf.get("validation")
197
+ if val:
198
+ af = val.get("adversarial_fail", {})
199
+ ap = val.get("adversarial_pass", {})
200
+ bn = val.get("benign", {})
201
+ console.print(f" Validation:")
202
+ if af.get("total"):
203
+ console.print(f" Failed adversarial caught: {af['caught']}/{af['total']} ({af.get('rate',0):.1%})")
204
+ if ap.get("total"):
205
+ console.print(f" Passed adversarial caught: {ap['caught']}/{ap['total']} ({ap.get('rate',0):.1%})")
206
+ if bn.get("total"):
207
+ console.print(f" Benign allowed: {bn['correct']}/{bn['total']} ({bn.get('rate',0):.1%})")
208
+
209
+
210
+
211
+
212
+ # ---------------------------------------------------------------------------
213
+ # Data fetching
214
+ # ---------------------------------------------------------------------------
215
+
216
+ def _fetch_experiments(client, category, exclude=False, last_n=10,
217
+ from_date=None, until_date=None):
218
+ experiments = []
219
+ page = 1
220
+ while True:
221
+ result = client.list_experiments(page=page, size=50)
222
+ for exp in result.get("data", []):
223
+ if exp.get("status") != "Finished":
224
+ continue
225
+ has = category in exp.get("test_category", "")
226
+ if exclude and has:
227
+ continue
228
+ if not exclude and not has:
229
+ continue
230
+ created = exp.get("created_at", "")
231
+ if from_date and created < from_date:
232
+ continue
233
+ if until_date and created > until_date:
234
+ continue
235
+ experiments.append(exp)
236
+ if not result.get("has_next_page"):
237
+ break
238
+ page += 1
239
+ experiments.sort(key=lambda e: e.get("created_at", ""), reverse=True)
240
+ return experiments[:last_n]
241
+
242
+
243
+ def _fetch_logs(client, experiments):
244
+ all_logs = []
245
+ with Progress(console=console) as progress:
246
+ task = progress.add_task(" Fetching...", total=len(experiments))
247
+ for exp in experiments:
248
+ cat = exp.get("test_category", "")
249
+ page = 1
250
+ while True:
251
+ result = client.get_experiment_logs(exp["id"], page=page, size=100)
252
+ for log in result.get("data", []):
253
+ log["test_category"] = cat
254
+ all_logs.extend(result.get("data", []))
255
+ if not result.get("has_next_page"):
256
+ break
257
+ page += 1
258
+ progress.advance(task)
259
+ return all_logs
260
+
261
+
262
+ def _fetch_intents(client, project_id):
263
+ try:
264
+ data = client.get(f"projects/{project_id}")
265
+ intents = data.get("scope", {}).get("intents", {})
266
+ return intents.get("permitted", []), intents.get("restricted", [])
267
+ except Exception:
268
+ return None, None
269
+
270
+
@@ -35,6 +35,7 @@ from .commands import (
35
35
  monitor,
36
36
  webhooks,
37
37
  assessments,
38
+ firewall,
38
39
  mcp,
39
40
  )
40
41
 
@@ -96,6 +97,7 @@ cli.add_command(webhooks.webhooks_group)
96
97
  cli.add_command(connectors.connectors_group)
97
98
  cli.add_command(inventory.inventory_group)
98
99
  cli.add_command(assessments.assessments_group)
100
+ cli.add_command(firewall.firewall_group)
99
101
  cli.add_command(auth.auth_group)
100
102
  cli.add_command(orgs.orgs_group)
101
103
  cli.add_command(completion.completion_command)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: humanbound-cli
3
- Version: 0.7.0
3
+ Version: 0.9.0
4
4
  Summary: Humanbound CLI - command line interface for AI agent security testing.
5
5
  Author-email: Kostas Siabanis <hello@humanbound.ai>, Demetris Gerogiannis <hello@humanbound.ai>
6
6
  License: Apache-2.0
@@ -60,6 +60,7 @@ Humanbound runs automated adversarial attacks against your bot's live endpoint,
60
60
  | **Posture Scoring** | Quantified 0-100 security score with breakdown by findings, coverage, and resilience. Track over time. |
61
61
  | **Shadow AI Discovery** | Scan cloud tenants for AI services, assess risk with 15 SAI threat classes, and govern your AI inventory. |
62
62
  | **Guardrails Export** | Generate protection rules from test findings. Export to OpenAI or Humanbound format. |
63
+ | **Firewall Training** | Train agent-specific Tier 2 classifiers from adversarial + QA test data. Pluggable model architecture via AgentClassifier scripts. |
63
64
  | **MCP Server** | Model Context Protocol server exposing all CLI capabilities as tools for AI assistants (Claude Code, Cursor, Gemini CLI, etc.). |
64
65
 
65
66
  ### Why Humanbound?
@@ -557,6 +558,30 @@ hb findings [--status open] [--severity high] [--json]
557
558
  hb guardrails [--vendor humanbound|openai] [--format json|yaml] [-o FILE]
558
559
  ```
559
560
 
561
+ ### Firewall
562
+
563
+ Train agent-specific classifiers for [hb-firewall](https://github.com/humanbound/firewall).
564
+
565
+ ```bash
566
+ # Train from adversarial + QA test data
567
+ hb firewall train --model detectors/setfit_classifier.py
568
+
569
+ # Show model info
570
+ hb firewall show firewall.hbfw
571
+ ```
572
+
573
+ | Flag | Default | Description |
574
+ |------|---------|-------------|
575
+ | `--model PATH` | — | Path to AgentClassifier script (required) |
576
+ | `--last N` | 10 | Last N finished experiments |
577
+ | `--from DATE` | — | Start date filter |
578
+ | `--until DATE` | — | End date filter |
579
+ | `--min-samples` | 30 | Minimum conversations required |
580
+ | `--output` | firewall_\<project\>.hbfw | Output file path |
581
+ | `--benign-dataset` | — | HuggingFace dataset for benign benchmarking |
582
+
583
+ See [hb-firewall docs](https://github.com/humanbound/firewall) for the AgentClassifier interface and full integration guide.
584
+
560
585
  ### Shell Completion
561
586
 
562
587
  ```bash
@@ -723,6 +748,22 @@ hb connect -e ./bot-config.json
723
748
  hb test -e ./bot-config.json
724
749
  ```
725
750
 
751
+ ### Whitebox testing with telemetry
752
+
753
+ Add a `telemetry` block to your agent config to enable whitebox testing. Humanbound fetches tool calls, memory operations, and resource usage from your observability platform (LangFuse, LangSmith, OpenAI Assistants, W&B, Helicone, AgentOps, or custom).
754
+
755
+ ```json
756
+ {
757
+ "telemetry": {
758
+ "format": "langfuse",
759
+ "endpoint": "https://cloud.langfuse.com/api/public/sessions/$session_id",
760
+ "headers": { "Authorization": "Basic <base64(pk:sk)>" }
761
+ }
762
+ }
763
+ ```
764
+
765
+ See the full [Telemetry Integration Guide](https://docs.humanbound.ai/integrations/telemetry/) for vendor-specific setup and the custom extraction map reference.
766
+
726
767
  ### Shadow AI discovery & governance
727
768
 
728
769
  ```bash
@@ -759,6 +800,29 @@ claude mcp add humanbound -- hb mcp
759
800
  hb guardrails --vendor openai --format json -o guardrails.json
760
801
  ```
761
802
 
803
+ ### Train and deploy firewall
804
+
805
+ ```bash
806
+ # Run adversarial tests
807
+ hb test
808
+
809
+ # Train Tier 1 classifier from results
810
+ hb firewall train -o model.hbfw
811
+
812
+ # Verify quality
813
+ hb firewall show model.hbfw
814
+ # F1=0.95, Precision=0.97, Tier 1 coverage=92%
815
+
816
+ # Test before deploying
817
+ # Deploy in your app
818
+ python -c "
819
+ from hb_firewall import Firewall
820
+ fw = Firewall.from_config('agent.yaml', model_path='model.hbfw')
821
+ result = fw.evaluate('What is my balance?')
822
+ print(result.verdict, result.tier) # Verdict.PASS 1
823
+ "
824
+ ```
825
+
762
826
  ---
763
827
 
764
828
  ### On-Premises
@@ -28,6 +28,7 @@ humanbound_cli/commands/discover.py
28
28
  humanbound_cli/commands/docs.py
29
29
  humanbound_cli/commands/experiments.py
30
30
  humanbound_cli/commands/findings.py
31
+ humanbound_cli/commands/firewall.py
31
32
  humanbound_cli/commands/guardrails.py
32
33
  humanbound_cli/commands/init.py
33
34
  humanbound_cli/commands/inventory.py
@@ -1,4 +1,3 @@
1
- build
2
1
  dist
3
2
  docs
4
3
  humanbound_cli
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "humanbound-cli"
7
- version = "0.7.0"
7
+ version = "0.9.0"
8
8
  authors = [
9
9
  { name="Kostas Siabanis", email="hello@humanbound.ai" },
10
10
  { name="Demetris Gerogiannis", email="hello@humanbound.ai" },
File without changes
File without changes