humanbound-cli 0.7.0__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/PKG-INFO +65 -1
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/README.md +64 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/__init__.py +2 -1
- humanbound_cli-0.9.0/humanbound_cli/commands/firewall.py +270 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/main.py +2 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/PKG-INFO +65 -1
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/SOURCES.txt +1 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/top_level.txt +0 -1
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/pyproject.toml +1 -1
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/LICENSE +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/__init__.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/client.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/api_keys.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/assessments.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/auth.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/campaigns.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/completion.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/connect.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/connectors.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/coverage.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/discover.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/docs.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/experiments.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/findings.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/guardrails.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/init.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/inventory.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/logs.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/mcp.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/members.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/monitor.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/orgs.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/posture.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/projects.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/providers.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/report.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/scan.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/sentinel.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/test.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/upload_logs.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/commands/webhooks.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/config.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/connectors/__init__.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/connectors/microsoft.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/exceptions.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/extractors/__init__.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/extractors/openapi.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/extractors/repo.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/mcp_server.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/pytest_plugin/__init__.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/pytest_plugin/fixtures.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/pytest_plugin/report.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/report.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/report_builder.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/serve/__init__.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/serve/config_builder.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/serve/local_server.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/serve/runtime_detector.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli/serve/tunnel_client.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/dependency_links.txt +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/entry_points.txt +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/humanbound_cli.egg-info/requires.txt +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/relay/relay.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/setup.cfg +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/tests/__init__.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/tests/cli_integration_test.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/tests/conftest.py +0 -0
- {humanbound_cli-0.7.0 → humanbound_cli-0.9.0}/tests/test_cli_commands.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: humanbound-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Humanbound CLI - command line interface for AI agent security testing.
|
|
5
5
|
Author-email: Kostas Siabanis <hello@humanbound.ai>, Demetris Gerogiannis <hello@humanbound.ai>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -60,6 +60,7 @@ Humanbound runs automated adversarial attacks against your bot's live endpoint,
|
|
|
60
60
|
| **Posture Scoring** | Quantified 0-100 security score with breakdown by findings, coverage, and resilience. Track over time. |
|
|
61
61
|
| **Shadow AI Discovery** | Scan cloud tenants for AI services, assess risk with 15 SAI threat classes, and govern your AI inventory. |
|
|
62
62
|
| **Guardrails Export** | Generate protection rules from test findings. Export to OpenAI or Humanbound format. |
|
|
63
|
+
| **Firewall Training** | Train agent-specific Tier 2 classifiers from adversarial + QA test data. Pluggable model architecture via AgentClassifier scripts. |
|
|
63
64
|
| **MCP Server** | Model Context Protocol server exposing all CLI capabilities as tools for AI assistants (Claude Code, Cursor, Gemini CLI, etc.). |
|
|
64
65
|
|
|
65
66
|
### Why Humanbound?
|
|
@@ -557,6 +558,30 @@ hb findings [--status open] [--severity high] [--json]
|
|
|
557
558
|
hb guardrails [--vendor humanbound|openai] [--format json|yaml] [-o FILE]
|
|
558
559
|
```
|
|
559
560
|
|
|
561
|
+
### Firewall
|
|
562
|
+
|
|
563
|
+
Train agent-specific classifiers for [hb-firewall](https://github.com/humanbound/firewall).
|
|
564
|
+
|
|
565
|
+
```bash
|
|
566
|
+
# Train from adversarial + QA test data
|
|
567
|
+
hb firewall train --model detectors/setfit_classifier.py
|
|
568
|
+
|
|
569
|
+
# Show model info
|
|
570
|
+
hb firewall show firewall.hbfw
|
|
571
|
+
```
|
|
572
|
+
|
|
573
|
+
| Flag | Default | Description |
|
|
574
|
+
|------|---------|-------------|
|
|
575
|
+
| `--model PATH` | — | Path to AgentClassifier script (required) |
|
|
576
|
+
| `--last N` | 10 | Last N finished experiments |
|
|
577
|
+
| `--from DATE` | — | Start date filter |
|
|
578
|
+
| `--until DATE` | — | End date filter |
|
|
579
|
+
| `--min-samples` | 30 | Minimum conversations required |
|
|
580
|
+
| `--output` | firewall_\<project\>.hbfw | Output file path |
|
|
581
|
+
| `--benign-dataset` | — | HuggingFace dataset for benign benchmarking |
|
|
582
|
+
|
|
583
|
+
See [hb-firewall docs](https://github.com/humanbound/firewall) for the AgentClassifier interface and full integration guide.
|
|
584
|
+
|
|
560
585
|
### Shell Completion
|
|
561
586
|
|
|
562
587
|
```bash
|
|
@@ -723,6 +748,22 @@ hb connect -e ./bot-config.json
|
|
|
723
748
|
hb test -e ./bot-config.json
|
|
724
749
|
```
|
|
725
750
|
|
|
751
|
+
### Whitebox testing with telemetry
|
|
752
|
+
|
|
753
|
+
Add a `telemetry` block to your agent config to enable whitebox testing. Humanbound fetches tool calls, memory operations, and resource usage from your observability platform (LangFuse, LangSmith, OpenAI Assistants, W&B, Helicone, AgentOps, or custom).
|
|
754
|
+
|
|
755
|
+
```json
|
|
756
|
+
{
|
|
757
|
+
"telemetry": {
|
|
758
|
+
"format": "langfuse",
|
|
759
|
+
"endpoint": "https://cloud.langfuse.com/api/public/sessions/$session_id",
|
|
760
|
+
"headers": { "Authorization": "Basic <base64(pk:sk)>" }
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
```
|
|
764
|
+
|
|
765
|
+
See the full [Telemetry Integration Guide](https://docs.humanbound.ai/integrations/telemetry/) for vendor-specific setup and the custom extraction map reference.
|
|
766
|
+
|
|
726
767
|
### Shadow AI discovery & governance
|
|
727
768
|
|
|
728
769
|
```bash
|
|
@@ -759,6 +800,29 @@ claude mcp add humanbound -- hb mcp
|
|
|
759
800
|
hb guardrails --vendor openai --format json -o guardrails.json
|
|
760
801
|
```
|
|
761
802
|
|
|
803
|
+
### Train and deploy firewall
|
|
804
|
+
|
|
805
|
+
```bash
|
|
806
|
+
# Run adversarial tests
|
|
807
|
+
hb test
|
|
808
|
+
|
|
809
|
+
# Train Tier 1 classifier from results
|
|
810
|
+
hb firewall train -o model.hbfw
|
|
811
|
+
|
|
812
|
+
# Verify quality
|
|
813
|
+
hb firewall show model.hbfw
|
|
814
|
+
# F1=0.95, Precision=0.97, Tier 1 coverage=92%
|
|
815
|
+
|
|
816
|
+
# Test before deploying
|
|
817
|
+
# Deploy in your app
|
|
818
|
+
python -c "
|
|
819
|
+
from hb_firewall import Firewall
|
|
820
|
+
fw = Firewall.from_config('agent.yaml', model_path='model.hbfw')
|
|
821
|
+
result = fw.evaluate('What is my balance?')
|
|
822
|
+
print(result.verdict, result.tier) # Verdict.PASS 1
|
|
823
|
+
"
|
|
824
|
+
```
|
|
825
|
+
|
|
762
826
|
---
|
|
763
827
|
|
|
764
828
|
### On-Premises
|
|
@@ -27,6 +27,7 @@ Humanbound runs automated adversarial attacks against your bot's live endpoint,
|
|
|
27
27
|
| **Posture Scoring** | Quantified 0-100 security score with breakdown by findings, coverage, and resilience. Track over time. |
|
|
28
28
|
| **Shadow AI Discovery** | Scan cloud tenants for AI services, assess risk with 15 SAI threat classes, and govern your AI inventory. |
|
|
29
29
|
| **Guardrails Export** | Generate protection rules from test findings. Export to OpenAI or Humanbound format. |
|
|
30
|
+
| **Firewall Training** | Train agent-specific Tier 2 classifiers from adversarial + QA test data. Pluggable model architecture via AgentClassifier scripts. |
|
|
30
31
|
| **MCP Server** | Model Context Protocol server exposing all CLI capabilities as tools for AI assistants (Claude Code, Cursor, Gemini CLI, etc.). |
|
|
31
32
|
|
|
32
33
|
### Why Humanbound?
|
|
@@ -524,6 +525,30 @@ hb findings [--status open] [--severity high] [--json]
|
|
|
524
525
|
hb guardrails [--vendor humanbound|openai] [--format json|yaml] [-o FILE]
|
|
525
526
|
```
|
|
526
527
|
|
|
528
|
+
### Firewall
|
|
529
|
+
|
|
530
|
+
Train agent-specific classifiers for [hb-firewall](https://github.com/humanbound/firewall).
|
|
531
|
+
|
|
532
|
+
```bash
|
|
533
|
+
# Train from adversarial + QA test data
|
|
534
|
+
hb firewall train --model detectors/setfit_classifier.py
|
|
535
|
+
|
|
536
|
+
# Show model info
|
|
537
|
+
hb firewall show firewall.hbfw
|
|
538
|
+
```
|
|
539
|
+
|
|
540
|
+
| Flag | Default | Description |
|
|
541
|
+
|------|---------|-------------|
|
|
542
|
+
| `--model PATH` | — | Path to AgentClassifier script (required) |
|
|
543
|
+
| `--last N` | 10 | Last N finished experiments |
|
|
544
|
+
| `--from DATE` | — | Start date filter |
|
|
545
|
+
| `--until DATE` | — | End date filter |
|
|
546
|
+
| `--min-samples` | 30 | Minimum conversations required |
|
|
547
|
+
| `--output` | firewall_\<project\>.hbfw | Output file path |
|
|
548
|
+
| `--benign-dataset` | — | HuggingFace dataset for benign benchmarking |
|
|
549
|
+
|
|
550
|
+
See [hb-firewall docs](https://github.com/humanbound/firewall) for the AgentClassifier interface and full integration guide.
|
|
551
|
+
|
|
527
552
|
### Shell Completion
|
|
528
553
|
|
|
529
554
|
```bash
|
|
@@ -690,6 +715,22 @@ hb connect -e ./bot-config.json
|
|
|
690
715
|
hb test -e ./bot-config.json
|
|
691
716
|
```
|
|
692
717
|
|
|
718
|
+
### Whitebox testing with telemetry
|
|
719
|
+
|
|
720
|
+
Add a `telemetry` block to your agent config to enable whitebox testing. Humanbound fetches tool calls, memory operations, and resource usage from your observability platform (LangFuse, LangSmith, OpenAI Assistants, W&B, Helicone, AgentOps, or custom).
|
|
721
|
+
|
|
722
|
+
```json
|
|
723
|
+
{
|
|
724
|
+
"telemetry": {
|
|
725
|
+
"format": "langfuse",
|
|
726
|
+
"endpoint": "https://cloud.langfuse.com/api/public/sessions/$session_id",
|
|
727
|
+
"headers": { "Authorization": "Basic <base64(pk:sk)>" }
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
```
|
|
731
|
+
|
|
732
|
+
See the full [Telemetry Integration Guide](https://docs.humanbound.ai/integrations/telemetry/) for vendor-specific setup and the custom extraction map reference.
|
|
733
|
+
|
|
693
734
|
### Shadow AI discovery & governance
|
|
694
735
|
|
|
695
736
|
```bash
|
|
@@ -726,6 +767,29 @@ claude mcp add humanbound -- hb mcp
|
|
|
726
767
|
hb guardrails --vendor openai --format json -o guardrails.json
|
|
727
768
|
```
|
|
728
769
|
|
|
770
|
+
### Train and deploy firewall
|
|
771
|
+
|
|
772
|
+
```bash
|
|
773
|
+
# Run adversarial tests
|
|
774
|
+
hb test
|
|
775
|
+
|
|
776
|
+
# Train Tier 1 classifier from results
|
|
777
|
+
hb firewall train -o model.hbfw
|
|
778
|
+
|
|
779
|
+
# Verify quality
|
|
780
|
+
hb firewall show model.hbfw
|
|
781
|
+
# F1=0.95, Precision=0.97, Tier 1 coverage=92%
|
|
782
|
+
|
|
783
|
+
# Test before deploying
|
|
784
|
+
# Deploy in your app
|
|
785
|
+
python -c "
|
|
786
|
+
from hb_firewall import Firewall
|
|
787
|
+
fw = Firewall.from_config('agent.yaml', model_path='model.hbfw')
|
|
788
|
+
result = fw.evaluate('What is my balance?')
|
|
789
|
+
print(result.verdict, result.tier) # Verdict.PASS 1
|
|
790
|
+
"
|
|
791
|
+
```
|
|
792
|
+
|
|
729
793
|
---
|
|
730
794
|
|
|
731
795
|
### On-Premises
|
|
@@ -5,7 +5,7 @@ from . import (
|
|
|
5
5
|
guardrails, docs, providers, findings, api_keys, members,
|
|
6
6
|
coverage, campaigns, upload_logs, sentinel, discover,
|
|
7
7
|
connectors, inventory, completion, connect, report, monitor,
|
|
8
|
-
webhooks, assessments,
|
|
8
|
+
webhooks, assessments, firewall,
|
|
9
9
|
)
|
|
10
10
|
|
|
11
11
|
# MCP command is optional — only available when mcp SDK is installed
|
|
@@ -42,5 +42,6 @@ __all__ = [
|
|
|
42
42
|
"monitor",
|
|
43
43
|
"webhooks",
|
|
44
44
|
"assessments",
|
|
45
|
+
"firewall",
|
|
45
46
|
"mcp",
|
|
46
47
|
]
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"""Firewall commands — train and manage Tier 2 classifiers for hb-firewall."""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
import time
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.progress import Progress
|
|
10
|
+
|
|
11
|
+
from ..client import HumanboundClient
|
|
12
|
+
from ..exceptions import NotAuthenticatedError, APIError
|
|
13
|
+
|
|
14
|
+
console = Console()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@click.group("firewall")
|
|
18
|
+
def firewall_group():
|
|
19
|
+
"""Train and manage Tier 2 classifiers for hb-firewall.
|
|
20
|
+
|
|
21
|
+
\b
|
|
22
|
+
Workflow:
|
|
23
|
+
1. Run adversarial tests: hb test
|
|
24
|
+
2. Train classifiers: hb firewall train --model detectors/my_model.py
|
|
25
|
+
3. Load in your app: Firewall.from_config("agent.yaml", model_path="fw.hbfw")
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@firewall_group.command("train")
|
|
30
|
+
@click.option("--model", "model_path", type=str, required=False, default=None,
|
|
31
|
+
help="Path to AgentClassifier script (e.g. detectors/one_class_svm.py)")
|
|
32
|
+
@click.option("--last", "last_n", type=int, default=10,
|
|
33
|
+
help="Last N finished experiments (default: 10)")
|
|
34
|
+
@click.option("--from", "from_date", type=str, default=None)
|
|
35
|
+
@click.option("--until", "until_date", type=str, default=None)
|
|
36
|
+
@click.option("--min-samples", type=int, default=30)
|
|
37
|
+
@click.option("--output", "-o", type=click.Path(), default=None)
|
|
38
|
+
def train_command(model_path, last_n, from_date, until_date, min_samples,
|
|
39
|
+
output):
|
|
40
|
+
"""Train Tier 2 classifiers from adversarial + QA test logs."""
|
|
41
|
+
if not model_path:
|
|
42
|
+
# Default to SetFit classifier shipped with hb-firewall
|
|
43
|
+
import importlib.resources
|
|
44
|
+
try:
|
|
45
|
+
# Try to find setfit_classifier.py relative to hb_firewall package
|
|
46
|
+
import hb_firewall
|
|
47
|
+
pkg_dir = Path(hb_firewall.__file__).parent.parent.parent
|
|
48
|
+
default = pkg_dir / "detectors" / "setfit_classifier.py"
|
|
49
|
+
if default.exists():
|
|
50
|
+
model_path = str(default)
|
|
51
|
+
else:
|
|
52
|
+
console.print("[red]Default SetFit classifier not found.[/red]")
|
|
53
|
+
console.print(" Provide a path to an AgentClassifier script:")
|
|
54
|
+
console.print(" hb firewall train --model detectors/setfit_classifier.py")
|
|
55
|
+
sys.exit(1)
|
|
56
|
+
except Exception:
|
|
57
|
+
console.print("[red]Provide --model flag.[/red]")
|
|
58
|
+
console.print(" hb firewall train --model detectors/setfit_classifier.py")
|
|
59
|
+
sys.exit(1)
|
|
60
|
+
|
|
61
|
+
try:
|
|
62
|
+
client = HumanboundClient()
|
|
63
|
+
project_id = client.project_id
|
|
64
|
+
if not project_id:
|
|
65
|
+
console.print("[red]No active project.[/red] Run: hb projects use <id>")
|
|
66
|
+
sys.exit(1)
|
|
67
|
+
|
|
68
|
+
# Load detector
|
|
69
|
+
try:
|
|
70
|
+
from hb_firewall.hbfw import HBFW, load_model_class, save_hbfw
|
|
71
|
+
except ImportError:
|
|
72
|
+
console.print("[red]Install: pip install hb-firewall[/red]")
|
|
73
|
+
sys.exit(1)
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
detector_cls = load_model_class(model_path)
|
|
77
|
+
except ValueError as e:
|
|
78
|
+
console.print(f"[red]{e}[/red]")
|
|
79
|
+
sys.exit(1)
|
|
80
|
+
|
|
81
|
+
console.print(f"\n[bold]Training firewall classifiers[/bold]")
|
|
82
|
+
console.print(f" Project: {project_id[:8]}...")
|
|
83
|
+
console.print(f" Model: {model_path}")
|
|
84
|
+
|
|
85
|
+
# Step 1: Fetch experiments
|
|
86
|
+
console.print(f"\n[bold]Step 1:[/bold] Fetching experiments...")
|
|
87
|
+
adv_exps = _fetch_experiments(client, "adversarial", last_n=last_n,
|
|
88
|
+
from_date=from_date, until_date=until_date)
|
|
89
|
+
qa_exps = _fetch_experiments(client, "adversarial", exclude=True,
|
|
90
|
+
last_n=last_n, from_date=from_date, until_date=until_date)
|
|
91
|
+
if not adv_exps:
|
|
92
|
+
console.print("[red]No finished adversarial experiments.[/red] Run: hb test")
|
|
93
|
+
sys.exit(1)
|
|
94
|
+
console.print(f" Found {len(adv_exps)} adversarial + {len(qa_exps)} QA experiments")
|
|
95
|
+
|
|
96
|
+
# Step 2: Fetch logs
|
|
97
|
+
console.print(f"\n[bold]Step 2:[/bold] Pulling conversation logs...")
|
|
98
|
+
logs = _fetch_logs(client, adv_exps + qa_exps)
|
|
99
|
+
if len(logs) < min_samples:
|
|
100
|
+
console.print(f"[red]Only {len(logs)} conversations (min: {min_samples}).[/red]")
|
|
101
|
+
sys.exit(1)
|
|
102
|
+
|
|
103
|
+
n_pass = sum(1 for l in logs if l.get("result") == "pass")
|
|
104
|
+
n_fail = len(logs) - n_pass
|
|
105
|
+
adv_fail = sum(1 for l in logs
|
|
106
|
+
if "adversarial" in (l.get("test_category") or "")
|
|
107
|
+
and l.get("result") == "fail")
|
|
108
|
+
console.print(f" Collected {len(logs)} conversations ({n_pass} pass, {n_fail} fail)")
|
|
109
|
+
console.print(f" Training on {adv_fail} failed adversarial conversations")
|
|
110
|
+
|
|
111
|
+
permitted, restricted = _fetch_intents(client, project_id)
|
|
112
|
+
if permitted or restricted:
|
|
113
|
+
console.print(f" Intents: {len(permitted or [])} permitted, {len(restricted or [])} restricted")
|
|
114
|
+
|
|
115
|
+
# Step 3: Prepare
|
|
116
|
+
console.print(f"\n[bold]Step 3:[/bold] Preparing training data...")
|
|
117
|
+
hbfw = HBFW(attack_detector=detector_cls("attack"),
|
|
118
|
+
benign_detector=detector_cls("benign"))
|
|
119
|
+
data = hbfw.prepare(logs, restricted_intents=restricted, permitted_intents=permitted)
|
|
120
|
+
|
|
121
|
+
stats = data.get("stats", {})
|
|
122
|
+
console.print(f" Attack samples: {stats.get('attack_samples', 0)} (curated: {stats.get('curated_attack', 0)})")
|
|
123
|
+
console.print(f" Benign samples: {stats.get('benign_samples', 0)} (curated: {stats.get('curated_benign', 0)})")
|
|
124
|
+
if not data.get("has_qa"):
|
|
125
|
+
console.print(f" [yellow]No QA tests. Using permitted intents as benign data.[/yellow]")
|
|
126
|
+
|
|
127
|
+
# Step 4: Train
|
|
128
|
+
console.print(f"\n[bold]Step 4:[/bold] Training...")
|
|
129
|
+
performance = hbfw.train(data, permitted_intents=permitted,
|
|
130
|
+
restricted_intents=restricted)
|
|
131
|
+
|
|
132
|
+
val = performance.get("validation")
|
|
133
|
+
if val:
|
|
134
|
+
af = val.get("adversarial_fail", {})
|
|
135
|
+
ap = val.get("adversarial_pass", {})
|
|
136
|
+
bn = val.get("benign", {})
|
|
137
|
+
console.print(f"\n [bold]Validation (conversation replay)[/bold]")
|
|
138
|
+
if af.get("total"):
|
|
139
|
+
r = af.get("rate", 0)
|
|
140
|
+
s = "green" if r >= 0.8 else "yellow" if r >= 0.5 else "red"
|
|
141
|
+
console.print(f" Failed adversarial caught: [{s}]{af['caught']}/{af['total']} ({r:.1%})[/{s}]")
|
|
142
|
+
if ap.get("total"):
|
|
143
|
+
r = ap.get("rate", 0)
|
|
144
|
+
s = "green" if r >= 0.5 else "dim"
|
|
145
|
+
console.print(f" Passed adversarial caught: [{s}]{ap['caught']}/{ap['total']} ({r:.1%})[/{s}]")
|
|
146
|
+
if bn.get("total"):
|
|
147
|
+
r = bn.get("rate", 0)
|
|
148
|
+
s = "green" if r >= 0.8 else "yellow" if r >= 0.5 else "red"
|
|
149
|
+
console.print(f" Benign allowed: [{s}]{bn['correct']}/{bn['total']} ({r:.1%})[/{s}]")
|
|
150
|
+
if bn.get("blocked"):
|
|
151
|
+
console.print(f" [red]Benign blocked: {bn['blocked']}[/red]")
|
|
152
|
+
|
|
153
|
+
console.print(f" Training complete.")
|
|
154
|
+
|
|
155
|
+
# Save
|
|
156
|
+
if output is None:
|
|
157
|
+
output = f"firewall_{project_id[:8]}.hbfw"
|
|
158
|
+
model_data = hbfw.export()
|
|
159
|
+
model_data["config"]["project_id"] = project_id
|
|
160
|
+
model_data["config"]["created_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
161
|
+
model_data["config"]["n_conversations"] = len(logs)
|
|
162
|
+
model_data["config"]["detector"] = Path(model_path).stem
|
|
163
|
+
save_hbfw(model_data, output)
|
|
164
|
+
|
|
165
|
+
file_size = Path(output).stat().st_size / 1024
|
|
166
|
+
console.print(f"\n[green]Model saved:[/green] {output} ({file_size:.0f} KB)")
|
|
167
|
+
|
|
168
|
+
except NotAuthenticatedError:
|
|
169
|
+
console.print("[red]Not authenticated.[/red] Run: hb login")
|
|
170
|
+
sys.exit(1)
|
|
171
|
+
except APIError as e:
|
|
172
|
+
console.print(f"[red]API error:[/red] {e}")
|
|
173
|
+
sys.exit(1)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@firewall_group.command("show")
|
|
177
|
+
@click.argument("model_path", type=click.Path(exists=True))
|
|
178
|
+
def show_command(model_path):
|
|
179
|
+
"""Show model info from a trained .hbfw file."""
|
|
180
|
+
try:
|
|
181
|
+
from hb_firewall.hbfw import load_hbfw
|
|
182
|
+
except ImportError:
|
|
183
|
+
console.print("[red]hb-firewall not installed.[/red]")
|
|
184
|
+
sys.exit(1)
|
|
185
|
+
|
|
186
|
+
config, _ = load_hbfw(model_path)
|
|
187
|
+
console.print(f"\n[bold]Firewall Model: {model_path}[/bold]")
|
|
188
|
+
console.print(f" Created: {config.get('created_at', '?')}")
|
|
189
|
+
console.print(f" Project: {config.get('project_id', '?')}")
|
|
190
|
+
console.print(f" Detector: {config.get('detector', '?')}")
|
|
191
|
+
perf = config.get("performance", {})
|
|
192
|
+
stats = perf.get("stats", {})
|
|
193
|
+
if stats:
|
|
194
|
+
console.print(f" Attack samples: {stats.get('attack_samples', '?')}")
|
|
195
|
+
console.print(f" Benign samples: {stats.get('benign_samples', '?')}")
|
|
196
|
+
val = perf.get("validation")
|
|
197
|
+
if val:
|
|
198
|
+
af = val.get("adversarial_fail", {})
|
|
199
|
+
ap = val.get("adversarial_pass", {})
|
|
200
|
+
bn = val.get("benign", {})
|
|
201
|
+
console.print(f" Validation:")
|
|
202
|
+
if af.get("total"):
|
|
203
|
+
console.print(f" Failed adversarial caught: {af['caught']}/{af['total']} ({af.get('rate',0):.1%})")
|
|
204
|
+
if ap.get("total"):
|
|
205
|
+
console.print(f" Passed adversarial caught: {ap['caught']}/{ap['total']} ({ap.get('rate',0):.1%})")
|
|
206
|
+
if bn.get("total"):
|
|
207
|
+
console.print(f" Benign allowed: {bn['correct']}/{bn['total']} ({bn.get('rate',0):.1%})")
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# ---------------------------------------------------------------------------
|
|
213
|
+
# Data fetching
|
|
214
|
+
# ---------------------------------------------------------------------------
|
|
215
|
+
|
|
216
|
+
def _fetch_experiments(client, category, exclude=False, last_n=10,
|
|
217
|
+
from_date=None, until_date=None):
|
|
218
|
+
experiments = []
|
|
219
|
+
page = 1
|
|
220
|
+
while True:
|
|
221
|
+
result = client.list_experiments(page=page, size=50)
|
|
222
|
+
for exp in result.get("data", []):
|
|
223
|
+
if exp.get("status") != "Finished":
|
|
224
|
+
continue
|
|
225
|
+
has = category in exp.get("test_category", "")
|
|
226
|
+
if exclude and has:
|
|
227
|
+
continue
|
|
228
|
+
if not exclude and not has:
|
|
229
|
+
continue
|
|
230
|
+
created = exp.get("created_at", "")
|
|
231
|
+
if from_date and created < from_date:
|
|
232
|
+
continue
|
|
233
|
+
if until_date and created > until_date:
|
|
234
|
+
continue
|
|
235
|
+
experiments.append(exp)
|
|
236
|
+
if not result.get("has_next_page"):
|
|
237
|
+
break
|
|
238
|
+
page += 1
|
|
239
|
+
experiments.sort(key=lambda e: e.get("created_at", ""), reverse=True)
|
|
240
|
+
return experiments[:last_n]
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _fetch_logs(client, experiments):
|
|
244
|
+
all_logs = []
|
|
245
|
+
with Progress(console=console) as progress:
|
|
246
|
+
task = progress.add_task(" Fetching...", total=len(experiments))
|
|
247
|
+
for exp in experiments:
|
|
248
|
+
cat = exp.get("test_category", "")
|
|
249
|
+
page = 1
|
|
250
|
+
while True:
|
|
251
|
+
result = client.get_experiment_logs(exp["id"], page=page, size=100)
|
|
252
|
+
for log in result.get("data", []):
|
|
253
|
+
log["test_category"] = cat
|
|
254
|
+
all_logs.extend(result.get("data", []))
|
|
255
|
+
if not result.get("has_next_page"):
|
|
256
|
+
break
|
|
257
|
+
page += 1
|
|
258
|
+
progress.advance(task)
|
|
259
|
+
return all_logs
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _fetch_intents(client, project_id):
|
|
263
|
+
try:
|
|
264
|
+
data = client.get(f"projects/{project_id}")
|
|
265
|
+
intents = data.get("scope", {}).get("intents", {})
|
|
266
|
+
return intents.get("permitted", []), intents.get("restricted", [])
|
|
267
|
+
except Exception:
|
|
268
|
+
return None, None
|
|
269
|
+
|
|
270
|
+
|
|
@@ -35,6 +35,7 @@ from .commands import (
|
|
|
35
35
|
monitor,
|
|
36
36
|
webhooks,
|
|
37
37
|
assessments,
|
|
38
|
+
firewall,
|
|
38
39
|
mcp,
|
|
39
40
|
)
|
|
40
41
|
|
|
@@ -96,6 +97,7 @@ cli.add_command(webhooks.webhooks_group)
|
|
|
96
97
|
cli.add_command(connectors.connectors_group)
|
|
97
98
|
cli.add_command(inventory.inventory_group)
|
|
98
99
|
cli.add_command(assessments.assessments_group)
|
|
100
|
+
cli.add_command(firewall.firewall_group)
|
|
99
101
|
cli.add_command(auth.auth_group)
|
|
100
102
|
cli.add_command(orgs.orgs_group)
|
|
101
103
|
cli.add_command(completion.completion_command)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: humanbound-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Humanbound CLI - command line interface for AI agent security testing.
|
|
5
5
|
Author-email: Kostas Siabanis <hello@humanbound.ai>, Demetris Gerogiannis <hello@humanbound.ai>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -60,6 +60,7 @@ Humanbound runs automated adversarial attacks against your bot's live endpoint,
|
|
|
60
60
|
| **Posture Scoring** | Quantified 0-100 security score with breakdown by findings, coverage, and resilience. Track over time. |
|
|
61
61
|
| **Shadow AI Discovery** | Scan cloud tenants for AI services, assess risk with 15 SAI threat classes, and govern your AI inventory. |
|
|
62
62
|
| **Guardrails Export** | Generate protection rules from test findings. Export to OpenAI or Humanbound format. |
|
|
63
|
+
| **Firewall Training** | Train agent-specific Tier 2 classifiers from adversarial + QA test data. Pluggable model architecture via AgentClassifier scripts. |
|
|
63
64
|
| **MCP Server** | Model Context Protocol server exposing all CLI capabilities as tools for AI assistants (Claude Code, Cursor, Gemini CLI, etc.). |
|
|
64
65
|
|
|
65
66
|
### Why Humanbound?
|
|
@@ -557,6 +558,30 @@ hb findings [--status open] [--severity high] [--json]
|
|
|
557
558
|
hb guardrails [--vendor humanbound|openai] [--format json|yaml] [-o FILE]
|
|
558
559
|
```
|
|
559
560
|
|
|
561
|
+
### Firewall
|
|
562
|
+
|
|
563
|
+
Train agent-specific classifiers for [hb-firewall](https://github.com/humanbound/firewall).
|
|
564
|
+
|
|
565
|
+
```bash
|
|
566
|
+
# Train from adversarial + QA test data
|
|
567
|
+
hb firewall train --model detectors/setfit_classifier.py
|
|
568
|
+
|
|
569
|
+
# Show model info
|
|
570
|
+
hb firewall show firewall.hbfw
|
|
571
|
+
```
|
|
572
|
+
|
|
573
|
+
| Flag | Default | Description |
|
|
574
|
+
|------|---------|-------------|
|
|
575
|
+
| `--model PATH` | — | Path to AgentClassifier script (required) |
|
|
576
|
+
| `--last N` | 10 | Last N finished experiments |
|
|
577
|
+
| `--from DATE` | — | Start date filter |
|
|
578
|
+
| `--until DATE` | — | End date filter |
|
|
579
|
+
| `--min-samples` | 30 | Minimum conversations required |
|
|
580
|
+
| `--output` | firewall_\<project\>.hbfw | Output file path |
|
|
581
|
+
| `--benign-dataset` | — | HuggingFace dataset for benign benchmarking |
|
|
582
|
+
|
|
583
|
+
See [hb-firewall docs](https://github.com/humanbound/firewall) for the AgentClassifier interface and full integration guide.
|
|
584
|
+
|
|
560
585
|
### Shell Completion
|
|
561
586
|
|
|
562
587
|
```bash
|
|
@@ -723,6 +748,22 @@ hb connect -e ./bot-config.json
|
|
|
723
748
|
hb test -e ./bot-config.json
|
|
724
749
|
```
|
|
725
750
|
|
|
751
|
+
### Whitebox testing with telemetry
|
|
752
|
+
|
|
753
|
+
Add a `telemetry` block to your agent config to enable whitebox testing. Humanbound fetches tool calls, memory operations, and resource usage from your observability platform (LangFuse, LangSmith, OpenAI Assistants, W&B, Helicone, AgentOps, or custom).
|
|
754
|
+
|
|
755
|
+
```json
|
|
756
|
+
{
|
|
757
|
+
"telemetry": {
|
|
758
|
+
"format": "langfuse",
|
|
759
|
+
"endpoint": "https://cloud.langfuse.com/api/public/sessions/$session_id",
|
|
760
|
+
"headers": { "Authorization": "Basic <base64(pk:sk)>" }
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
```
|
|
764
|
+
|
|
765
|
+
See the full [Telemetry Integration Guide](https://docs.humanbound.ai/integrations/telemetry/) for vendor-specific setup and the custom extraction map reference.
|
|
766
|
+
|
|
726
767
|
### Shadow AI discovery & governance
|
|
727
768
|
|
|
728
769
|
```bash
|
|
@@ -759,6 +800,29 @@ claude mcp add humanbound -- hb mcp
|
|
|
759
800
|
hb guardrails --vendor openai --format json -o guardrails.json
|
|
760
801
|
```
|
|
761
802
|
|
|
803
|
+
### Train and deploy firewall
|
|
804
|
+
|
|
805
|
+
```bash
|
|
806
|
+
# Run adversarial tests
|
|
807
|
+
hb test
|
|
808
|
+
|
|
809
|
+
# Train Tier 1 classifier from results
|
|
810
|
+
hb firewall train -o model.hbfw
|
|
811
|
+
|
|
812
|
+
# Verify quality
|
|
813
|
+
hb firewall show model.hbfw
|
|
814
|
+
# F1=0.95, Precision=0.97, Tier 1 coverage=92%
|
|
815
|
+
|
|
816
|
+
# Test before deploying
|
|
817
|
+
# Deploy in your app
|
|
818
|
+
python -c "
|
|
819
|
+
from hb_firewall import Firewall
|
|
820
|
+
fw = Firewall.from_config('agent.yaml', model_path='model.hbfw')
|
|
821
|
+
result = fw.evaluate('What is my balance?')
|
|
822
|
+
print(result.verdict, result.tier) # Verdict.PASS 1
|
|
823
|
+
"
|
|
824
|
+
```
|
|
825
|
+
|
|
762
826
|
---
|
|
763
827
|
|
|
764
828
|
### On-Premises
|
|
@@ -28,6 +28,7 @@ humanbound_cli/commands/discover.py
|
|
|
28
28
|
humanbound_cli/commands/docs.py
|
|
29
29
|
humanbound_cli/commands/experiments.py
|
|
30
30
|
humanbound_cli/commands/findings.py
|
|
31
|
+
humanbound_cli/commands/firewall.py
|
|
31
32
|
humanbound_cli/commands/guardrails.py
|
|
32
33
|
humanbound_cli/commands/init.py
|
|
33
34
|
humanbound_cli/commands/inventory.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|