nat-engine 1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mannf/__init__.py +33 -0
- mannf/__main__.py +10 -0
- mannf/_version.py +8 -0
- mannf/agents/__init__.py +7 -0
- mannf/agents/analyzer_agent.py +9 -0
- mannf/agents/base.py +9 -0
- mannf/agents/bdi_agent.py +9 -0
- mannf/agents/belief_state.py +9 -0
- mannf/agents/coordinator_agent.py +9 -0
- mannf/agents/executor_agent.py +9 -0
- mannf/agents/monitor_agent.py +9 -0
- mannf/agents/oracle_agent.py +9 -0
- mannf/agents/planner_agent.py +9 -0
- mannf/agents/test_agent.py +9 -0
- mannf/anomaly/__init__.py +7 -0
- mannf/anomaly/enhanced_detector.py +9 -0
- mannf/cli.py +9 -0
- mannf/core/__init__.py +26 -0
- mannf/core/agents/__init__.py +52 -0
- mannf/core/agents/accessibility_scanner_agent.py +245 -0
- mannf/core/agents/analyzer_agent.py +224 -0
- mannf/core/agents/autonomous_loop_agent.py +1086 -0
- mannf/core/agents/autonomous_loop_models.py +62 -0
- mannf/core/agents/autonomous_run_differ.py +427 -0
- mannf/core/agents/base.py +128 -0
- mannf/core/agents/bdi_agent.py +330 -0
- mannf/core/agents/belief_state.py +202 -0
- mannf/core/agents/browser_coordinator_agent.py +224 -0
- mannf/core/agents/browser_executor_agent.py +410 -0
- mannf/core/agents/coordinator_agent.py +262 -0
- mannf/core/agents/executor_agent.py +222 -0
- mannf/core/agents/monitor_agent.py +188 -0
- mannf/core/agents/oracle_agent.py +150 -0
- mannf/core/agents/performance_testing_agent.py +279 -0
- mannf/core/agents/planner_agent.py +128 -0
- mannf/core/agents/test_agent.py +249 -0
- mannf/core/agents/visual_regression_agent.py +311 -0
- mannf/core/agents/web_crawler_agent.py +510 -0
- mannf/core/agents/worker_pool.py +366 -0
- mannf/core/anomaly/__init__.py +14 -0
- mannf/core/anomaly/enhanced_detector.py +541 -0
- mannf/core/browser/__init__.py +63 -0
- mannf/core/browser/accessibility_scanner.py +424 -0
- mannf/core/browser/discovery_model.py +178 -0
- mannf/core/browser/dom_snapshot.py +349 -0
- mannf/core/browser/ingestor_bridge.py +371 -0
- mannf/core/browser/performance_metrics.py +217 -0
- mannf/core/browser/reflection_analyzer.py +442 -0
- mannf/core/browser/scenario_generator.py +1100 -0
- mannf/core/browser/security_scenario_generator.py +695 -0
- mannf/core/browser/visual_comparer.py +159 -0
- mannf/core/diagnostics/__init__.py +28 -0
- mannf/core/diagnostics/failure_clusterer.py +211 -0
- mannf/core/diagnostics/flake_detector.py +233 -0
- mannf/core/diagnostics/root_cause_analyzer.py +273 -0
- mannf/core/distributed/__init__.py +16 -0
- mannf/core/distributed/endpoint.py +139 -0
- mannf/core/distributed/system_under_test.py +207 -0
- mannf/core/functional_orchestrator.py +428 -0
- mannf/core/messaging/__init__.py +11 -0
- mannf/core/messaging/bus.py +113 -0
- mannf/core/messaging/messages.py +89 -0
- mannf/core/nat_orchestrator.py +342 -0
- mannf/core/neural/__init__.py +183 -0
- mannf/core/orchestrator.py +272 -0
- mannf/core/prioritization/__init__.py +17 -0
- mannf/core/prioritization/adaptive_controller.py +509 -0
- mannf/core/prioritization/belief_prioritizer.py +231 -0
- mannf/core/prioritization/risk_scorer.py +430 -0
- mannf/core/reporting/__init__.py +12 -0
- mannf/core/reporting/unified_report.py +664 -0
- mannf/core/testing/__init__.py +17 -0
- mannf/core/testing/adaptive_controller.py +149 -0
- mannf/core/testing/models.py +179 -0
- mannf/core/validation/__init__.py +10 -0
- mannf/core/validation/self_validation_runner.py +180 -0
- mannf/dashboard/__init__.py +7 -0
- mannf/dashboard/app.py +9 -0
- mannf/dashboard/models.py +9 -0
- mannf/dashboard/static/index.html +2538 -0
- mannf/dashboard/telemetry.py +9 -0
- mannf/distributed/__init__.py +7 -0
- mannf/distributed/endpoint.py +9 -0
- mannf/distributed/system_under_test.py +9 -0
- mannf/healing/__init__.py +7 -0
- mannf/healing/graphql_schema_diff.py +9 -0
- mannf/healing/healer.py +9 -0
- mannf/healing/models.py +9 -0
- mannf/healing/schema_diff.py +9 -0
- mannf/integrations/__init__.py +7 -0
- mannf/integrations/auth.py +9 -0
- mannf/integrations/graphql_parser.py +9 -0
- mannf/integrations/graphql_sut.py +9 -0
- mannf/integrations/http_sut.py +9 -0
- mannf/integrations/openapi_parser.py +9 -0
- mannf/integrations/postman_parser.py +9 -0
- mannf/llm/__init__.py +7 -0
- mannf/llm/anthropic_provider.py +9 -0
- mannf/llm/base.py +9 -0
- mannf/llm/config.py +9 -0
- mannf/llm/factory.py +9 -0
- mannf/llm/openai_provider.py +9 -0
- mannf/llm/prompts.py +9 -0
- mannf/messaging/__init__.py +7 -0
- mannf/messaging/bus.py +9 -0
- mannf/messaging/messages.py +9 -0
- mannf/nat_orchestrator.py +9 -0
- mannf/neural/__init__.py +7 -0
- mannf/orchestrator.py +9 -0
- mannf/prioritization/__init__.py +7 -0
- mannf/prioritization/adaptive_controller.py +9 -0
- mannf/prioritization/belief_prioritizer.py +9 -0
- mannf/prioritization/risk_scorer.py +9 -0
- mannf/product/__init__.py +29 -0
- mannf/product/admin/__init__.py +3 -0
- mannf/product/admin/routes.py +514 -0
- mannf/product/auth/__init__.py +5 -0
- mannf/product/auth/saml.py +212 -0
- mannf/product/billing/__init__.py +5 -0
- mannf/product/billing/audit.py +160 -0
- mannf/product/billing/feature_gates.py +180 -0
- mannf/product/billing/metering.py +179 -0
- mannf/product/billing/notifications.py +181 -0
- mannf/product/billing/plans.py +133 -0
- mannf/product/billing/rate_limits.py +35 -0
- mannf/product/billing/stripe_billing.py +906 -0
- mannf/product/billing/tenant_auth.py +233 -0
- mannf/product/billing/tenant_manager.py +873 -0
- mannf/product/cli.py +3900 -0
- mannf/product/cli_admin.py +408 -0
- mannf/product/dashboard/__init__.py +61 -0
- mannf/product/dashboard/app.py +3567 -0
- mannf/product/dashboard/models.py +460 -0
- mannf/product/dashboard/static/index.html +6347 -0
- mannf/product/dashboard/static/manifest.json +25 -0
- mannf/product/dashboard/static/pwa-icon-192.png +0 -0
- mannf/product/dashboard/static/pwa-icon-512.png +0 -0
- mannf/product/dashboard/static/sw.js +64 -0
- mannf/product/dashboard/telemetry.py +547 -0
- mannf/product/database.py +145 -0
- mannf/product/demo.py +844 -0
- mannf/product/doctor.py +509 -0
- mannf/product/exporters/__init__.py +65 -0
- mannf/product/exporters/azuredevops_exporter.py +257 -0
- mannf/product/exporters/base.py +307 -0
- mannf/product/exporters/bugzilla_exporter.py +200 -0
- mannf/product/exporters/dedup.py +275 -0
- mannf/product/exporters/finding_adapter.py +216 -0
- mannf/product/exporters/github_exporter.py +197 -0
- mannf/product/exporters/gitlab_exporter.py +215 -0
- mannf/product/exporters/jira_exporter.py +180 -0
- mannf/product/exporters/linear_exporter.py +195 -0
- mannf/product/exporters/loader.py +233 -0
- mannf/product/exporters/pagerduty_exporter.py +363 -0
- mannf/product/exporters/sentry_exporter.py +322 -0
- mannf/product/exporters/servicenow_exporter.py +240 -0
- mannf/product/exporters/shortcut_exporter.py +231 -0
- mannf/product/exporters/webhook_exporter.py +383 -0
- mannf/product/formatters/__init__.py +18 -0
- mannf/product/formatters/allure_formatter.py +161 -0
- mannf/product/formatters/ctrf_formatter.py +149 -0
- mannf/product/healing/__init__.py +30 -0
- mannf/product/healing/graphql_schema_diff.py +152 -0
- mannf/product/healing/healer.py +141 -0
- mannf/product/healing/models.py +175 -0
- mannf/product/healing/schema_diff.py +251 -0
- mannf/product/ingestors/__init__.py +77 -0
- mannf/product/ingestors/base.py +256 -0
- mannf/product/ingestors/bgstm_ingestor.py +764 -0
- mannf/product/ingestors/curl_ingestor.py +1019 -0
- mannf/product/ingestors/cypress_ingestor.py +487 -0
- mannf/product/ingestors/gherkin_ingestor.py +967 -0
- mannf/product/ingestors/graphql_ingestor.py +845 -0
- mannf/product/ingestors/grpc_ingestor.py +591 -0
- mannf/product/ingestors/har_ingestor.py +976 -0
- mannf/product/ingestors/loader.py +284 -0
- mannf/product/ingestors/models.py +146 -0
- mannf/product/ingestors/openapi_ingestor.py +606 -0
- mannf/product/ingestors/playwright_ingestor.py +449 -0
- mannf/product/ingestors/postman_ingestor.py +631 -0
- mannf/product/ingestors/traffic_ingestor.py +679 -0
- mannf/product/ingestors/websocket_ingestor.py +526 -0
- mannf/product/integrations/__init__.py +21 -0
- mannf/product/integrations/auth.py +190 -0
- mannf/product/integrations/graphql_parser.py +436 -0
- mannf/product/integrations/graphql_sut.py +247 -0
- mannf/product/integrations/grpc_sut.py +469 -0
- mannf/product/integrations/http_sut.py +237 -0
- mannf/product/integrations/kafka_adapter.py +342 -0
- mannf/product/integrations/openapi_parser.py +513 -0
- mannf/product/integrations/postman_parser.py +467 -0
- mannf/product/integrations/webhook_receiver.py +344 -0
- mannf/product/integrations/websocket_sut.py +434 -0
- mannf/product/llm/__init__.py +25 -0
- mannf/product/llm/anthropic_provider.py +94 -0
- mannf/product/llm/base.py +267 -0
- mannf/product/llm/config.py +48 -0
- mannf/product/llm/factory.py +42 -0
- mannf/product/llm/openai_provider.py +93 -0
- mannf/product/llm/prompts.py +403 -0
- mannf/product/llm/root_cause_service.py +311 -0
- mannf/product/llm/test_plan_models.py +78 -0
- mannf/product/metrics.py +149 -0
- mannf/product/middleware/__init__.py +3 -0
- mannf/product/middleware/audit_middleware.py +112 -0
- mannf/product/middleware/tenant_isolation.py +114 -0
- mannf/product/models.py +347 -0
- mannf/product/notifications/__init__.py +24 -0
- mannf/product/notifications/dispatcher.py +411 -0
- mannf/product/onboarding.py +190 -0
- mannf/product/orchestration/__init__.py +39 -0
- mannf/product/orchestration/ingest_scan_orchestrator.py +339 -0
- mannf/product/orchestration/pipeline.py +401 -0
- mannf/product/orchestrator.py +987 -0
- mannf/product/orchestrator_models.py +269 -0
- mannf/product/regression/__init__.py +36 -0
- mannf/product/regression/differ.py +172 -0
- mannf/product/regression/masking.py +100 -0
- mannf/product/regression/models.py +232 -0
- mannf/product/regression/recorder.py +124 -0
- mannf/product/regression/replayer.py +168 -0
- mannf/product/reports/__init__.py +10 -0
- mannf/product/reports/pdf.py +132 -0
- mannf/product/scheduling/__init__.py +57 -0
- mannf/product/scheduling/cron_utils.py +251 -0
- mannf/product/scheduling/engine.py +473 -0
- mannf/product/scheduling/models.py +86 -0
- mannf/product/scheduling/queue.py +894 -0
- mannf/product/scheduling/store.py +235 -0
- mannf/product/security/__init__.py +21 -0
- mannf/product/security/belief_guided.py +143 -0
- mannf/product/security/checks/__init__.py +55 -0
- mannf/product/security/checks/base.py +69 -0
- mannf/product/security/checks/bfla.py +77 -0
- mannf/product/security/checks/bola.py +77 -0
- mannf/product/security/checks/bopla.py +80 -0
- mannf/product/security/checks/broken_auth.py +86 -0
- mannf/product/security/checks/graphql_security.py +299 -0
- mannf/product/security/checks/inventory.py +70 -0
- mannf/product/security/checks/misconfig.py +158 -0
- mannf/product/security/checks/resource_consumption.py +70 -0
- mannf/product/security/checks/sensitive_flows.py +80 -0
- mannf/product/security/checks/ssrf.py +101 -0
- mannf/product/security/checks/unsafe_consumption.py +120 -0
- mannf/product/security/models.py +92 -0
- mannf/product/security/plugin_loader.py +182 -0
- mannf/product/security/reporter.py +92 -0
- mannf/product/security/scanner.py +183 -0
- mannf/product/server.py +6220 -0
- mannf/product/setup_wizard.py +873 -0
- mannf/product/status.py +404 -0
- mannf/product/storage/__init__.py +10 -0
- mannf/product/storage/artifact_store.py +343 -0
- mannf/product/telemetry.py +300 -0
- mannf/product/uninstall.py +169 -0
- mannf/product/upgrade.py +139 -0
- mannf/product/weights/__init__.py +13 -0
- mannf/product/weights/blob_store.py +299 -0
- mannf/product/weights/factory.py +42 -0
- mannf/product/weights/registry.py +159 -0
- mannf/product/weights/store.py +210 -0
- mannf/regression/__init__.py +7 -0
- mannf/regression/differ.py +9 -0
- mannf/regression/masking.py +9 -0
- mannf/regression/models.py +9 -0
- mannf/regression/recorder.py +9 -0
- mannf/regression/replayer.py +9 -0
- mannf/security/__init__.py +7 -0
- mannf/security/belief_guided.py +9 -0
- mannf/security/checks/__init__.py +7 -0
- mannf/security/checks/base.py +9 -0
- mannf/security/checks/bfla.py +9 -0
- mannf/security/checks/bola.py +9 -0
- mannf/security/checks/bopla.py +9 -0
- mannf/security/checks/broken_auth.py +9 -0
- mannf/security/checks/graphql_security.py +9 -0
- mannf/security/checks/inventory.py +9 -0
- mannf/security/checks/misconfig.py +9 -0
- mannf/security/checks/resource_consumption.py +9 -0
- mannf/security/checks/sensitive_flows.py +9 -0
- mannf/security/checks/ssrf.py +9 -0
- mannf/security/checks/unsafe_consumption.py +9 -0
- mannf/security/models.py +9 -0
- mannf/security/reporter.py +9 -0
- mannf/security/scanner.py +9 -0
- mannf/server.py +9 -0
- mannf/testing/__init__.py +7 -0
- mannf/testing/adaptive_controller.py +9 -0
- mannf/testing/models.py +9 -0
- mannf/weights/__init__.py +7 -0
- mannf/weights/registry.py +9 -0
- mannf/weights/store.py +9 -0
- nat_engine-1.dist-info/METADATA +555 -0
- nat_engine-1.dist-info/RECORD +299 -0
- nat_engine-1.dist-info/WHEEL +5 -0
- nat_engine-1.dist-info/entry_points.txt +4 -0
- nat_engine-1.dist-info/licenses/LICENSE +651 -0
- nat_engine-1.dist-info/licenses/NOTICE +178 -0
- nat_engine-1.dist-info/top_level.txt +1 -0
mannf/product/cli.py
ADDED
|
@@ -0,0 +1,3900 @@
|
|
|
1
|
+
# Copyright (C) 2026 Brad Guider
|
|
2
|
+
# This file is part of NAT (Neural Agent Testing Framework).
|
|
3
|
+
# Licensed under the AGPL-3.0. See LICENSE for details.
|
|
4
|
+
# Commercial licensing available — see COMMERCIAL_LICENSE.md.
|
|
5
|
+
|
|
6
|
+
"""CLI entry point for the NeuroAgentTest (NAT) framework.
|
|
7
|
+
|
|
8
|
+
Usage::
|
|
9
|
+
|
|
10
|
+
nat scan --spec path/to/openapi.yaml --base-url http://localhost:8080
|
|
11
|
+
python -m mannf scan --spec path/to/openapi.yaml --base-url http://localhost:8080
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import asyncio
|
|
18
|
+
import importlib
|
|
19
|
+
import json
|
|
20
|
+
import logging
|
|
21
|
+
import os
|
|
22
|
+
import sys
|
|
23
|
+
import xml.etree.ElementTree as ET
|
|
24
|
+
from datetime import datetime, timezone
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
27
|
+
|
|
28
|
+
from mannf._version import __version__
|
|
29
|
+
|
|
30
|
+
if TYPE_CHECKING:
|
|
31
|
+
from mannf.core.testing.models import TestSuite
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
|
|
35
|
+
# ---------------------------------------------------------------------------
|
|
36
|
+
# Exit code constants
|
|
37
|
+
# ---------------------------------------------------------------------------
|
|
38
|
+
EXIT_OK = 0 # all tests passed
|
|
39
|
+
EXIT_FAILURES = 1 # one or more test failures detected
|
|
40
|
+
EXIT_SCAN_ERROR = 2 # scan error (connection failure, spec parse error, runtime exception)
|
|
41
|
+
EXIT_CONFIG_ERROR = 3 # configuration error (invalid flags, missing required args, bad config file)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# ---------------------------------------------------------------------------
|
|
45
|
+
# Config file support (.natrc)
|
|
46
|
+
# ---------------------------------------------------------------------------
|
|
47
|
+
|
|
48
|
+
# Severity icons used in security scan output
|
|
49
|
+
_SEVERITY_ICONS: dict[str, str] = {
|
|
50
|
+
"critical": "🔴",
|
|
51
|
+
"high": "🟠",
|
|
52
|
+
"medium": "🟡",
|
|
53
|
+
"low": "🔵",
|
|
54
|
+
"info": "ℹ️",
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
def _load_natrc(config_path: Optional[str] = None) -> dict:
|
|
58
|
+
"""Load a YAML ``.natrc`` config file and return its contents as a dict.
|
|
59
|
+
|
|
60
|
+
Search order (unless *config_path* is explicitly given):
|
|
61
|
+
1. ``--config PATH`` value
|
|
62
|
+
2. ``.natrc`` in the current working directory
|
|
63
|
+
3. ``~/.natrc`` in the user's home directory
|
|
64
|
+
|
|
65
|
+
Returns an empty dict if no file is found.
|
|
66
|
+
Raises :class:`ValueError` if the file exists but cannot be parsed.
|
|
67
|
+
"""
|
|
68
|
+
import yaml # pyyaml is a listed dependency
|
|
69
|
+
|
|
70
|
+
candidates: list[str] = []
|
|
71
|
+
if config_path:
|
|
72
|
+
candidates = [config_path]
|
|
73
|
+
else:
|
|
74
|
+
candidates = [
|
|
75
|
+
str(Path.cwd() / ".natrc"),
|
|
76
|
+
str(Path.home() / ".natrc"),
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
for path in candidates:
|
|
80
|
+
if os.path.isfile(path):
|
|
81
|
+
try:
|
|
82
|
+
with open(path, encoding="utf-8") as fh:
|
|
83
|
+
data = yaml.safe_load(fh)
|
|
84
|
+
if data is None:
|
|
85
|
+
return {}
|
|
86
|
+
if not isinstance(data, dict):
|
|
87
|
+
raise ValueError(f"Config file {path!r} must be a YAML mapping, got {type(data).__name__}")
|
|
88
|
+
logger.debug("Loaded config from %s", path)
|
|
89
|
+
return data
|
|
90
|
+
except Exception as exc:
|
|
91
|
+
raise ValueError(f"Failed to parse config file {path!r}: {exc}") from exc
|
|
92
|
+
|
|
93
|
+
if config_path:
|
|
94
|
+
raise OSError(f"Config file not found: {config_path!r}")
|
|
95
|
+
|
|
96
|
+
return {}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _apply_config_defaults(parser: argparse.ArgumentParser, config: dict) -> None:
|
|
100
|
+
"""Set *config* values as defaults on every sub-command parser.
|
|
101
|
+
|
|
102
|
+
CLI flags override these defaults because argparse processes command-line
|
|
103
|
+
arguments after defaults are set.
|
|
104
|
+
"""
|
|
105
|
+
defaults = {k.replace("-", "_"): v for k, v in config.items()}
|
|
106
|
+
if not defaults:
|
|
107
|
+
return
|
|
108
|
+
for action in parser._subparsers._group_actions: # type: ignore[attr-defined]
|
|
109
|
+
for subparser in action.choices.values():
|
|
110
|
+
subparser.set_defaults(**defaults)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
114
|
+
parser = argparse.ArgumentParser(
|
|
115
|
+
prog="nat",
|
|
116
|
+
description=(
|
|
117
|
+
"NeuroAgentTest (NAT) — Adaptive API Testing Framework\n\n"
|
|
118
|
+
"Exit codes:\n"
|
|
119
|
+
" 0 All tests passed\n"
|
|
120
|
+
" 1 One or more test failures detected\n"
|
|
121
|
+
" 2 Scan error (connection failure, spec parse error, runtime exception)\n"
|
|
122
|
+
" 3 Configuration error (invalid flags, missing required args, bad config file)"
|
|
123
|
+
),
|
|
124
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
125
|
+
)
|
|
126
|
+
parser.add_argument(
|
|
127
|
+
"--config",
|
|
128
|
+
metavar="PATH",
|
|
129
|
+
default=None,
|
|
130
|
+
help="Path to a YAML config file (default: .natrc in cwd or ~/.natrc).",
|
|
131
|
+
)
|
|
132
|
+
parser.add_argument(
|
|
133
|
+
"--version", "-V",
|
|
134
|
+
action="version",
|
|
135
|
+
version=f"nat {__version__}",
|
|
136
|
+
)
|
|
137
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
138
|
+
|
|
139
|
+
scan = subparsers.add_parser(
|
|
140
|
+
"scan",
|
|
141
|
+
help="Parse an OpenAPI spec and run adaptive tests against a live API.",
|
|
142
|
+
)
|
|
143
|
+
scan.add_argument(
|
|
144
|
+
"--spec",
|
|
145
|
+
required=False,
|
|
146
|
+
default=None,
|
|
147
|
+
metavar="PATH",
|
|
148
|
+
help="Path to an OpenAPI 3.x or Swagger 2.x spec file (JSON or YAML), or a Postman Collection v2.1 JSON file.",
|
|
149
|
+
)
|
|
150
|
+
scan.add_argument(
|
|
151
|
+
"--format",
|
|
152
|
+
choices=["openapi", "postman"],
|
|
153
|
+
default="openapi",
|
|
154
|
+
help="Spec format: openapi (default) or postman (Postman Collection v2.1).",
|
|
155
|
+
)
|
|
156
|
+
scan.add_argument(
|
|
157
|
+
"--base-url",
|
|
158
|
+
required=False,
|
|
159
|
+
default=None,
|
|
160
|
+
metavar="URL",
|
|
161
|
+
help="Base URL of the API under test (e.g. http://localhost:8080).",
|
|
162
|
+
)
|
|
163
|
+
scan.add_argument(
|
|
164
|
+
"--type",
|
|
165
|
+
choices=["rest", "graphql", "grpc", "websocket"],
|
|
166
|
+
default="rest",
|
|
167
|
+
help="API type: rest (default), graphql, grpc, or websocket.",
|
|
168
|
+
)
|
|
169
|
+
scan.add_argument(
|
|
170
|
+
"--graphql-introspection",
|
|
171
|
+
action="store_true",
|
|
172
|
+
help="Auto-discover GraphQL schema via introspection (only used with --type graphql).",
|
|
173
|
+
)
|
|
174
|
+
scan.add_argument(
|
|
175
|
+
"--proto",
|
|
176
|
+
metavar="PATH",
|
|
177
|
+
default=None,
|
|
178
|
+
help="Path to a .proto file (used with --type grpc).",
|
|
179
|
+
)
|
|
180
|
+
scan.add_argument(
|
|
181
|
+
"--grpc-endpoint",
|
|
182
|
+
metavar="HOST:PORT",
|
|
183
|
+
default=None,
|
|
184
|
+
dest="grpc_endpoint",
|
|
185
|
+
help="gRPC server address in host:port format (used with --type grpc).",
|
|
186
|
+
)
|
|
187
|
+
scan.add_argument(
|
|
188
|
+
"--grpc-use-tls",
|
|
189
|
+
action="store_true",
|
|
190
|
+
default=False,
|
|
191
|
+
dest="grpc_use_tls",
|
|
192
|
+
help="Use TLS for gRPC connections (used with --type grpc).",
|
|
193
|
+
)
|
|
194
|
+
scan.add_argument(
|
|
195
|
+
"--asyncapi",
|
|
196
|
+
metavar="PATH",
|
|
197
|
+
default=None,
|
|
198
|
+
help="Path to an AsyncAPI spec file (used with --type websocket).",
|
|
199
|
+
)
|
|
200
|
+
scan.add_argument(
|
|
201
|
+
"--ws-endpoint",
|
|
202
|
+
metavar="URL",
|
|
203
|
+
default=None,
|
|
204
|
+
dest="ws_endpoint",
|
|
205
|
+
help="WebSocket base URL (e.g. ws://localhost:8765) (used with --type websocket).",
|
|
206
|
+
)
|
|
207
|
+
scan.add_argument(
|
|
208
|
+
"--auth-token",
|
|
209
|
+
metavar="TOKEN",
|
|
210
|
+
default=None,
|
|
211
|
+
help="Bearer token for Authorization header.",
|
|
212
|
+
)
|
|
213
|
+
scan.add_argument(
|
|
214
|
+
"--api-key",
|
|
215
|
+
metavar="KEY",
|
|
216
|
+
default=None,
|
|
217
|
+
help="API key value sent in X-API-Key header.",
|
|
218
|
+
)
|
|
219
|
+
scan.add_argument(
|
|
220
|
+
"--api-key-header",
|
|
221
|
+
metavar="HEADER",
|
|
222
|
+
default="X-API-Key",
|
|
223
|
+
help="Header name for the API key (default: X-API-Key).",
|
|
224
|
+
)
|
|
225
|
+
# OAuth2 args
|
|
226
|
+
scan.add_argument("--oauth2-token-url", metavar="URL", default=None, help="OAuth2 token endpoint URL.")
|
|
227
|
+
scan.add_argument("--oauth2-client-id", metavar="ID", default=None, help="OAuth2 client ID.")
|
|
228
|
+
scan.add_argument("--oauth2-client-secret", metavar="SECRET", default=None, help="OAuth2 client secret.")
|
|
229
|
+
scan.add_argument("--oauth2-scopes", metavar="SCOPES", default=None, help="Comma-separated OAuth2 scopes.")
|
|
230
|
+
scan.add_argument(
|
|
231
|
+
"--oauth2-grant-type",
|
|
232
|
+
metavar="TYPE",
|
|
233
|
+
default="client_credentials",
|
|
234
|
+
help="OAuth2 grant type (default: client_credentials).",
|
|
235
|
+
)
|
|
236
|
+
scan.add_argument(
|
|
237
|
+
"--max-tests",
|
|
238
|
+
type=int,
|
|
239
|
+
default=None,
|
|
240
|
+
metavar="N",
|
|
241
|
+
help="Maximum number of test cases to run.",
|
|
242
|
+
)
|
|
243
|
+
scan.add_argument(
|
|
244
|
+
"--executors",
|
|
245
|
+
type=int,
|
|
246
|
+
default=3,
|
|
247
|
+
metavar="N",
|
|
248
|
+
help="Number of executor agents (default: 3).",
|
|
249
|
+
)
|
|
250
|
+
scan.add_argument(
|
|
251
|
+
"--timeout",
|
|
252
|
+
type=float,
|
|
253
|
+
default=10.0,
|
|
254
|
+
metavar="SECONDS",
|
|
255
|
+
help="HTTP request timeout in seconds (default: 10).",
|
|
256
|
+
)
|
|
257
|
+
scan.add_argument(
|
|
258
|
+
"--latency-sla",
|
|
259
|
+
type=float,
|
|
260
|
+
default=None,
|
|
261
|
+
metavar="MS",
|
|
262
|
+
help="Latency SLA in milliseconds — responses slower than this fail.",
|
|
263
|
+
)
|
|
264
|
+
scan.add_argument(
|
|
265
|
+
"--output",
|
|
266
|
+
"-o",
|
|
267
|
+
choices=["text", "json", "html", "junit", "allure", "ctrf", "pdf"],
|
|
268
|
+
default="text",
|
|
269
|
+
help="Output format: text (default), json, html, junit, allure, ctrf, pdf.",
|
|
270
|
+
)
|
|
271
|
+
scan.add_argument(
|
|
272
|
+
"--output-file",
|
|
273
|
+
metavar="PATH",
|
|
274
|
+
default=None,
|
|
275
|
+
help="Write output to this file instead of stdout (default: stdout for text/json; "
|
|
276
|
+
"auto-named file for html/junit/ctrf; output directory for allure).",
|
|
277
|
+
)
|
|
278
|
+
scan.add_argument(
|
|
279
|
+
"--trace-dir",
|
|
280
|
+
metavar="DIR",
|
|
281
|
+
default=None,
|
|
282
|
+
help="Directory to write Playwright trace .zip files when running browser-based "
|
|
283
|
+
"autonomous scenarios. Traces can be replayed in Playwright Trace Viewer.",
|
|
284
|
+
)
|
|
285
|
+
scan.add_argument(
|
|
286
|
+
"--verbose",
|
|
287
|
+
action="store_true",
|
|
288
|
+
help="Enable verbose logging.",
|
|
289
|
+
)
|
|
290
|
+
# Regression flags
|
|
291
|
+
scan.add_argument(
|
|
292
|
+
"--record-regression",
|
|
293
|
+
metavar="OUTPUT_FILE",
|
|
294
|
+
default=None,
|
|
295
|
+
help="Record a regression baseline to the specified JSON file.",
|
|
296
|
+
)
|
|
297
|
+
scan.add_argument(
|
|
298
|
+
"--replay",
|
|
299
|
+
metavar="BASELINE_FILE",
|
|
300
|
+
default=None,
|
|
301
|
+
help="Replay a prior regression baseline and compare results.",
|
|
302
|
+
)
|
|
303
|
+
scan.add_argument(
|
|
304
|
+
"--regression-diff-only",
|
|
305
|
+
action="store_true",
|
|
306
|
+
help="Only show differences when replaying (suppress unchanged endpoints).",
|
|
307
|
+
)
|
|
308
|
+
scan.add_argument(
|
|
309
|
+
"--mask-sensitive",
|
|
310
|
+
action="store_true",
|
|
311
|
+
help="Mask sensitive data (auth headers, tokens) in regression recordings.",
|
|
312
|
+
)
|
|
313
|
+
# Weight persistence flags
|
|
314
|
+
scan.add_argument(
|
|
315
|
+
"--save-weights",
|
|
316
|
+
metavar="PATH",
|
|
317
|
+
default=None,
|
|
318
|
+
help="After scan completes, save all agent weights to this JSON file.",
|
|
319
|
+
)
|
|
320
|
+
scan.add_argument(
|
|
321
|
+
"--load-weights",
|
|
322
|
+
metavar="PATH",
|
|
323
|
+
default=None,
|
|
324
|
+
help="Before scan starts, load pretrained weights from this JSON file.",
|
|
325
|
+
)
|
|
326
|
+
scan.add_argument(
|
|
327
|
+
"--weights-dir",
|
|
328
|
+
metavar="DIR",
|
|
329
|
+
default=None,
|
|
330
|
+
help="Directory for auto-save/auto-load of weights (saves after each scan, "
|
|
331
|
+
"loads the latest on the next scan for the same API).",
|
|
332
|
+
)
|
|
333
|
+
# Healing flags
|
|
334
|
+
scan.add_argument(
|
|
335
|
+
"--previous-spec",
|
|
336
|
+
metavar="PATH",
|
|
337
|
+
default=None,
|
|
338
|
+
help="Path to the previous spec. When provided, NAT diffs it against --spec "
|
|
339
|
+
"and runs self-healing before executing the scan.",
|
|
340
|
+
)
|
|
341
|
+
scan.add_argument(
|
|
342
|
+
"--healing-report",
|
|
343
|
+
metavar="PATH",
|
|
344
|
+
default=None,
|
|
345
|
+
help="Save the healing report to this JSON file (used with --previous-spec).",
|
|
346
|
+
)
|
|
347
|
+
# LLM augmentation flags
|
|
348
|
+
scan.add_argument(
|
|
349
|
+
"--use-llm",
|
|
350
|
+
action="store_true",
|
|
351
|
+
default=False,
|
|
352
|
+
help="Enable LLM-augmented test case generation (default: off).",
|
|
353
|
+
)
|
|
354
|
+
scan.add_argument(
|
|
355
|
+
"--llm-provider",
|
|
356
|
+
choices=["openai", "anthropic"],
|
|
357
|
+
default="openai",
|
|
358
|
+
metavar="PROVIDER",
|
|
359
|
+
help="LLM provider to use: openai (default) or anthropic.",
|
|
360
|
+
)
|
|
361
|
+
scan.add_argument(
|
|
362
|
+
"--llm-model",
|
|
363
|
+
default=None,
|
|
364
|
+
metavar="MODEL",
|
|
365
|
+
help="LLM model name override (e.g. gpt-4o, claude-3-5-sonnet-20241022).",
|
|
366
|
+
)
|
|
367
|
+
scan.add_argument(
|
|
368
|
+
"--llm-api-key",
|
|
369
|
+
default=None,
|
|
370
|
+
metavar="KEY",
|
|
371
|
+
help="LLM API key (alternative to OPENAI_API_KEY / ANTHROPIC_API_KEY env vars).",
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
security = subparsers.add_parser(
|
|
375
|
+
"security-scan",
|
|
376
|
+
help="Run OWASP API Security Top 10 checks against an API.",
|
|
377
|
+
)
|
|
378
|
+
security.add_argument("--spec", required=False, default=None, metavar="PATH", help="Path to OpenAPI spec file or Postman Collection v2.1 JSON file.")
|
|
379
|
+
security.add_argument(
|
|
380
|
+
"--format",
|
|
381
|
+
choices=["openapi", "postman"],
|
|
382
|
+
default="openapi",
|
|
383
|
+
help="Spec format: openapi (default) or postman (Postman Collection v2.1).",
|
|
384
|
+
)
|
|
385
|
+
security.add_argument("--base-url", required=False, default=None, metavar="URL", help="Base URL of the API to test.")
|
|
386
|
+
security.add_argument(
|
|
387
|
+
"--type",
|
|
388
|
+
choices=["rest", "graphql"],
|
|
389
|
+
default="rest",
|
|
390
|
+
help="API type: rest (default) or graphql.",
|
|
391
|
+
)
|
|
392
|
+
security.add_argument("--checks", metavar="CHECKS", default=None,
|
|
393
|
+
help="Comma-separated OWASP check IDs (e.g. API1,API2,API8). Default: all.")
|
|
394
|
+
security.add_argument("--severity-threshold", metavar="LEVEL", default="low",
|
|
395
|
+
choices=["critical", "high", "medium", "low", "info"],
|
|
396
|
+
help="Minimum severity to report (default: low).")
|
|
397
|
+
security.add_argument("--output", "-o", choices=["text", "json", "markdown"], default="text",
|
|
398
|
+
help="Output format: text (default), json, markdown.")
|
|
399
|
+
security.add_argument(
|
|
400
|
+
"--output-file",
|
|
401
|
+
metavar="PATH",
|
|
402
|
+
default=None,
|
|
403
|
+
help="Write output to this file instead of stdout.",
|
|
404
|
+
)
|
|
405
|
+
security.add_argument("--belief-guided", action="store_true", default=True,
|
|
406
|
+
help="Use neural network risk scores to prioritize (default: true).")
|
|
407
|
+
security.add_argument(
|
|
408
|
+
"--prioritization",
|
|
409
|
+
choices=["adaptive", "static", "off"],
|
|
410
|
+
default="adaptive",
|
|
411
|
+
help=(
|
|
412
|
+
"Prioritization mode: 'adaptive' (default) uses live BDI belief states, "
|
|
413
|
+
"'static' uses keyword/verb heuristics only, 'off' scans in spec order."
|
|
414
|
+
),
|
|
415
|
+
)
|
|
416
|
+
security.add_argument("--verbose", action="store_true", help="Enable verbose logging.")
|
|
417
|
+
# OAuth2 args for security-scan
|
|
418
|
+
security.add_argument("--oauth2-token-url", metavar="URL", default=None, help="OAuth2 token endpoint URL.")
|
|
419
|
+
security.add_argument("--oauth2-client-id", metavar="ID", default=None, help="OAuth2 client ID.")
|
|
420
|
+
security.add_argument("--oauth2-client-secret", metavar="SECRET", default=None, help="OAuth2 client secret.")
|
|
421
|
+
security.add_argument("--oauth2-scopes", metavar="SCOPES", default=None, help="Comma-separated OAuth2 scopes.")
|
|
422
|
+
security.add_argument(
|
|
423
|
+
"--oauth2-grant-type",
|
|
424
|
+
metavar="TYPE",
|
|
425
|
+
default="client_credentials",
|
|
426
|
+
help="OAuth2 grant type (default: client_credentials).",
|
|
427
|
+
)
|
|
428
|
+
# LLM augmentation flags
|
|
429
|
+
security.add_argument("--use-llm", action="store_true", default=False,
|
|
430
|
+
help="Enable LLM-augmented test generation (default: off).")
|
|
431
|
+
security.add_argument("--llm-provider", choices=["openai", "anthropic"], default="openai",
|
|
432
|
+
metavar="PROVIDER", help="LLM provider: openai (default) or anthropic.")
|
|
433
|
+
security.add_argument("--llm-model", default=None, metavar="MODEL", help="LLM model name override.")
|
|
434
|
+
security.add_argument("--llm-api-key", default=None, metavar="KEY", help="LLM API key.")
|
|
435
|
+
# Plugin flags
|
|
436
|
+
security.add_argument(
|
|
437
|
+
"--plugins-dir",
|
|
438
|
+
metavar="PATH",
|
|
439
|
+
default=None,
|
|
440
|
+
help="Path to a directory containing custom check plugin .py files.",
|
|
441
|
+
)
|
|
442
|
+
security.add_argument(
|
|
443
|
+
"--plugin",
|
|
444
|
+
metavar="MODULE.ClassName",
|
|
445
|
+
action="append",
|
|
446
|
+
dest="plugins",
|
|
447
|
+
default=None,
|
|
448
|
+
help=(
|
|
449
|
+
"Fully-qualified Python class name to load as a plugin "
|
|
450
|
+
"(e.g. mypackage.checks.MyCheck). Can be specified multiple times."
|
|
451
|
+
),
|
|
452
|
+
)
|
|
453
|
+
# Exporter flags
|
|
454
|
+
security.add_argument(
|
|
455
|
+
"--export",
|
|
456
|
+
metavar="NAME",
|
|
457
|
+
default=None,
|
|
458
|
+
help="Name of the exporter plugin to use (e.g. jira, linear, github-issues, console).",
|
|
459
|
+
)
|
|
460
|
+
security.add_argument(
|
|
461
|
+
"--exporters-dir",
|
|
462
|
+
metavar="PATH",
|
|
463
|
+
default=None,
|
|
464
|
+
help="Path to a directory containing custom exporter plugin .py files.",
|
|
465
|
+
)
|
|
466
|
+
security.add_argument(
|
|
467
|
+
"--export-config",
|
|
468
|
+
metavar="KEY=VALUE",
|
|
469
|
+
action="append",
|
|
470
|
+
dest="export_config",
|
|
471
|
+
default=None,
|
|
472
|
+
help=(
|
|
473
|
+
"Exporter-specific config as key=value pairs "
|
|
474
|
+
"(e.g. --export-config jira_url=https://myco.atlassian.net). "
|
|
475
|
+
"Can be specified multiple times."
|
|
476
|
+
),
|
|
477
|
+
)
|
|
478
|
+
security.add_argument(
|
|
479
|
+
"--export-min-severity",
|
|
480
|
+
metavar="LEVEL",
|
|
481
|
+
choices=["critical", "high", "medium", "low", "info"],
|
|
482
|
+
default="low",
|
|
483
|
+
help="Minimum severity level to export (default: low).",
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
# ----------------------------------------------------------------
|
|
487
|
+
# weights sub-command group
|
|
488
|
+
# ----------------------------------------------------------------
|
|
489
|
+
weights = subparsers.add_parser(
|
|
490
|
+
"weights",
|
|
491
|
+
help="Manage saved weight snapshots.",
|
|
492
|
+
)
|
|
493
|
+
weights_sub = weights.add_subparsers(dest="weights_command", required=True)
|
|
494
|
+
|
|
495
|
+
# nat weights list
|
|
496
|
+
weights_sub.add_parser(
|
|
497
|
+
"list",
|
|
498
|
+
help="List all registered weight snapshots.",
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
# nat weights save
|
|
502
|
+
weights_save = weights_sub.add_parser(
|
|
503
|
+
"save",
|
|
504
|
+
help="Save weights from a completed scan to a file and register the snapshot.",
|
|
505
|
+
)
|
|
506
|
+
weights_save.add_argument(
|
|
507
|
+
"--scan-id",
|
|
508
|
+
metavar="ID",
|
|
509
|
+
required=True,
|
|
510
|
+
help="ID of the completed scan whose agent weights to save.",
|
|
511
|
+
)
|
|
512
|
+
weights_save.add_argument(
|
|
513
|
+
"--output",
|
|
514
|
+
metavar="PATH",
|
|
515
|
+
required=True,
|
|
516
|
+
help="Output file path for the weight JSON file.",
|
|
517
|
+
)
|
|
518
|
+
weights_save.add_argument(
|
|
519
|
+
"--name",
|
|
520
|
+
metavar="NAME",
|
|
521
|
+
default=None,
|
|
522
|
+
help="Human-readable name to register the snapshot under (defaults to the scan ID).",
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
# nat weights info
|
|
526
|
+
weights_info = weights_sub.add_parser(
|
|
527
|
+
"info",
|
|
528
|
+
help="Show metadata about a weight file.",
|
|
529
|
+
)
|
|
530
|
+
weights_info.add_argument(
|
|
531
|
+
"path",
|
|
532
|
+
metavar="PATH",
|
|
533
|
+
help="Path to the weight JSON file (or a registered snapshot name).",
|
|
534
|
+
)
|
|
535
|
+
|
|
536
|
+
# ----------------------------------------------------------------
|
|
537
|
+
# export sub-command group
|
|
538
|
+
# ----------------------------------------------------------------
|
|
539
|
+
export = subparsers.add_parser(
|
|
540
|
+
"export",
|
|
541
|
+
help="Manage and inspect exporter plugins.",
|
|
542
|
+
)
|
|
543
|
+
export_sub = export.add_subparsers(dest="export_command", required=True)
|
|
544
|
+
|
|
545
|
+
# nat export list
|
|
546
|
+
export_list = export_sub.add_parser(
|
|
547
|
+
"list",
|
|
548
|
+
help="List all available exporter plugins (built-in + discovered).",
|
|
549
|
+
)
|
|
550
|
+
export_list.add_argument(
|
|
551
|
+
"--json",
|
|
552
|
+
action="store_true",
|
|
553
|
+
default=False,
|
|
554
|
+
dest="json_output",
|
|
555
|
+
help="Output as JSON (for CI/automation).",
|
|
556
|
+
)
|
|
557
|
+
|
|
558
|
+
# ----------------------------------------------------------------
|
|
559
|
+
# heal sub-command
|
|
560
|
+
# ----------------------------------------------------------------
|
|
561
|
+
heal = subparsers.add_parser(
|
|
562
|
+
"heal",
|
|
563
|
+
help="Detect API schema drift and show what would be healed (dry-run).",
|
|
564
|
+
)
|
|
565
|
+
heal.add_argument(
|
|
566
|
+
"--old-spec",
|
|
567
|
+
metavar="PATH",
|
|
568
|
+
required=True,
|
|
569
|
+
help="Path to the previous OpenAPI spec file.",
|
|
570
|
+
)
|
|
571
|
+
heal.add_argument(
|
|
572
|
+
"--new-spec",
|
|
573
|
+
metavar="PATH",
|
|
574
|
+
required=True,
|
|
575
|
+
help="Path to the current (updated) OpenAPI spec file.",
|
|
576
|
+
)
|
|
577
|
+
heal.add_argument(
|
|
578
|
+
"--output",
|
|
579
|
+
choices=["text", "json", "markdown"],
|
|
580
|
+
default="text",
|
|
581
|
+
help="Output format for the healing report (default: text).",
|
|
582
|
+
)
|
|
583
|
+
heal.add_argument(
|
|
584
|
+
"--healing-report",
|
|
585
|
+
metavar="PATH",
|
|
586
|
+
default=None,
|
|
587
|
+
help="Save healing report to this file path.",
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
# ----------------------------------------------------------------
|
|
591
|
+
# test-gen sub-command
|
|
592
|
+
# ----------------------------------------------------------------
|
|
593
|
+
test_gen = subparsers.add_parser(
|
|
594
|
+
"test-gen",
|
|
595
|
+
help="Generate test cases from a spec and print them (without running a scan).",
|
|
596
|
+
)
|
|
597
|
+
test_gen.add_argument(
|
|
598
|
+
"--spec",
|
|
599
|
+
required=True,
|
|
600
|
+
metavar="PATH",
|
|
601
|
+
help="Path to an OpenAPI 3.x or Swagger 2.x spec file.",
|
|
602
|
+
)
|
|
603
|
+
test_gen.add_argument(
|
|
604
|
+
"--base-url",
|
|
605
|
+
default="http://localhost:8080",
|
|
606
|
+
metavar="URL",
|
|
607
|
+
help="Base URL used for test case targets (default: http://localhost:8080).",
|
|
608
|
+
)
|
|
609
|
+
test_gen.add_argument(
|
|
610
|
+
"--output",
|
|
611
|
+
choices=["text", "json"],
|
|
612
|
+
default="text",
|
|
613
|
+
help="Output format (default: text).",
|
|
614
|
+
)
|
|
615
|
+
test_gen.add_argument(
|
|
616
|
+
"--use-llm",
|
|
617
|
+
action="store_true",
|
|
618
|
+
default=False,
|
|
619
|
+
help="Augment generated cases with LLM-generated edge cases.",
|
|
620
|
+
)
|
|
621
|
+
test_gen.add_argument(
|
|
622
|
+
"--llm-provider",
|
|
623
|
+
choices=["openai", "anthropic"],
|
|
624
|
+
default="openai",
|
|
625
|
+
metavar="PROVIDER",
|
|
626
|
+
help="LLM provider: openai (default) or anthropic.",
|
|
627
|
+
)
|
|
628
|
+
test_gen.add_argument(
|
|
629
|
+
"--llm-model",
|
|
630
|
+
default=None,
|
|
631
|
+
metavar="MODEL",
|
|
632
|
+
help="LLM model name override.",
|
|
633
|
+
)
|
|
634
|
+
test_gen.add_argument(
|
|
635
|
+
"--llm-api-key",
|
|
636
|
+
default=None,
|
|
637
|
+
metavar="KEY",
|
|
638
|
+
help="LLM API key.",
|
|
639
|
+
)
|
|
640
|
+
test_gen.add_argument(
|
|
641
|
+
"--max-tests",
|
|
642
|
+
type=int,
|
|
643
|
+
default=None,
|
|
644
|
+
metavar="N",
|
|
645
|
+
help="Maximum number of test cases to output.",
|
|
646
|
+
)
|
|
647
|
+
|
|
648
|
+
# ----------------------------------------------------------------
|
|
649
|
+
# completions sub-command
|
|
650
|
+
# ----------------------------------------------------------------
|
|
651
|
+
completions = subparsers.add_parser(
|
|
652
|
+
"completions",
|
|
653
|
+
help="Output shell completion script for bash, zsh, or fish.",
|
|
654
|
+
)
|
|
655
|
+
completions.add_argument(
|
|
656
|
+
"shell",
|
|
657
|
+
choices=["bash", "zsh", "fish"],
|
|
658
|
+
help="Shell to generate completions for: bash, zsh, or fish.",
|
|
659
|
+
)
|
|
660
|
+
|
|
661
|
+
# ----------------------------------------------------------------
|
|
662
|
+
# demo sub-command
|
|
663
|
+
# ----------------------------------------------------------------
|
|
664
|
+
demo = subparsers.add_parser(
|
|
665
|
+
"demo",
|
|
666
|
+
help=(
|
|
667
|
+
"Start the NAT server, open the dashboard, and run a curated "
|
|
668
|
+
"demo with live agent telemetry, risk heatmap, anomaly "
|
|
669
|
+
"detection, and pre-seeded security findings. Press Ctrl+C to stop."
|
|
670
|
+
),
|
|
671
|
+
)
|
|
672
|
+
demo.add_argument(
|
|
673
|
+
"--port",
|
|
674
|
+
type=int,
|
|
675
|
+
default=8080,
|
|
676
|
+
metavar="PORT",
|
|
677
|
+
help="Port to serve the demo on (default: 8080).",
|
|
678
|
+
)
|
|
679
|
+
demo.add_argument(
|
|
680
|
+
"--no-browser",
|
|
681
|
+
action="store_true",
|
|
682
|
+
default=False,
|
|
683
|
+
help="Skip auto-launching the browser (useful for headless environments).",
|
|
684
|
+
)
|
|
685
|
+
demo.add_argument(
|
|
686
|
+
"--target",
|
|
687
|
+
default=None,
|
|
688
|
+
metavar="TARGET",
|
|
689
|
+
help=(
|
|
690
|
+
"Demo target to run against. Choices: 'orangehrm' (OrangeHRM live demo), "
|
|
691
|
+
"'vulnapi' (VulnAPI testbed), or a custom URL (e.g. https://example.com). "
|
|
692
|
+
"When omitted, the NAT server + dashboard starts with synthetic data (default behavior)."
|
|
693
|
+
),
|
|
694
|
+
)
|
|
695
|
+
demo.add_argument(
|
|
696
|
+
"--headed",
|
|
697
|
+
action="store_true",
|
|
698
|
+
default=False,
|
|
699
|
+
help="Run the browser in visible/headed mode (for screen recordings and live demos). Default is headless.",
|
|
700
|
+
)
|
|
701
|
+
demo.add_argument(
|
|
702
|
+
"--output-dir",
|
|
703
|
+
default="demo_output",
|
|
704
|
+
metavar="DIR",
|
|
705
|
+
dest="output_dir",
|
|
706
|
+
help="Directory to write HTML + JSON reports into (default: demo_output/).",
|
|
707
|
+
)
|
|
708
|
+
demo.add_argument(
|
|
709
|
+
"--json-summary",
|
|
710
|
+
default="",
|
|
711
|
+
metavar="PATH",
|
|
712
|
+
dest="json_summary",
|
|
713
|
+
help="Optional path to write a JSON summary file.",
|
|
714
|
+
)
|
|
715
|
+
|
|
716
|
+
# ----------------------------------------------------------------
|
|
717
|
+
# setup sub-command
|
|
718
|
+
# ----------------------------------------------------------------
|
|
719
|
+
setup = subparsers.add_parser(
|
|
720
|
+
"setup",
|
|
721
|
+
help="Run the interactive NAT setup wizard.",
|
|
722
|
+
)
|
|
723
|
+
setup.add_argument(
|
|
724
|
+
"--non-interactive",
|
|
725
|
+
action="store_true",
|
|
726
|
+
default=False,
|
|
727
|
+
dest="non_interactive",
|
|
728
|
+
help="Skip all prompts and read configuration from environment variables.",
|
|
729
|
+
)
|
|
730
|
+
setup.add_argument(
|
|
731
|
+
"--demo",
|
|
732
|
+
action="store_true",
|
|
733
|
+
default=False,
|
|
734
|
+
help="Launch `nat demo` after setup completes.",
|
|
735
|
+
)
|
|
736
|
+
|
|
737
|
+
# ----------------------------------------------------------------
|
|
738
|
+
# upgrade sub-command
|
|
739
|
+
# ----------------------------------------------------------------
|
|
740
|
+
upgrade = subparsers.add_parser(
|
|
741
|
+
"upgrade",
|
|
742
|
+
help="Check for updates and upgrade nat-engine.",
|
|
743
|
+
)
|
|
744
|
+
upgrade.add_argument(
|
|
745
|
+
"--check",
|
|
746
|
+
action="store_true",
|
|
747
|
+
default=False,
|
|
748
|
+
help="Only check if an update is available; do not install.",
|
|
749
|
+
)
|
|
750
|
+
upgrade.add_argument(
|
|
751
|
+
"--migrate",
|
|
752
|
+
action="store_true",
|
|
753
|
+
default=False,
|
|
754
|
+
help="Run `alembic upgrade head` after upgrading (for self-hosted deployments).",
|
|
755
|
+
)
|
|
756
|
+
upgrade.add_argument(
|
|
757
|
+
"--pre",
|
|
758
|
+
action="store_true",
|
|
759
|
+
default=False,
|
|
760
|
+
help="Allow pre-release versions when checking or installing.",
|
|
761
|
+
)
|
|
762
|
+
|
|
763
|
+
# ----------------------------------------------------------------
|
|
764
|
+
# uninstall sub-command
|
|
765
|
+
# ----------------------------------------------------------------
|
|
766
|
+
uninstall = subparsers.add_parser(
|
|
767
|
+
"uninstall",
|
|
768
|
+
help="Remove NAT configuration, data directory, and shell completions.",
|
|
769
|
+
)
|
|
770
|
+
uninstall.add_argument(
|
|
771
|
+
"--keep-config",
|
|
772
|
+
action="store_true",
|
|
773
|
+
default=False,
|
|
774
|
+
dest="keep_config",
|
|
775
|
+
help="Preserve .natrc files during uninstall.",
|
|
776
|
+
)
|
|
777
|
+
uninstall.add_argument(
|
|
778
|
+
"--yes",
|
|
779
|
+
"-y",
|
|
780
|
+
action="store_true",
|
|
781
|
+
default=False,
|
|
782
|
+
help="Skip the confirmation prompt.",
|
|
783
|
+
)
|
|
784
|
+
uninstall.add_argument(
|
|
785
|
+
"--purge",
|
|
786
|
+
action="store_true",
|
|
787
|
+
default=False,
|
|
788
|
+
help="Also remove database tables via `alembic downgrade base` and Docker volumes.",
|
|
789
|
+
)
|
|
790
|
+
|
|
791
|
+
# ----------------------------------------------------------------
|
|
792
|
+
# doctor sub-command
|
|
793
|
+
# ----------------------------------------------------------------
|
|
794
|
+
doctor = subparsers.add_parser(
|
|
795
|
+
"doctor",
|
|
796
|
+
help="Run pre-flight checks to validate your NAT environment.",
|
|
797
|
+
)
|
|
798
|
+
doctor.add_argument(
|
|
799
|
+
"--json",
|
|
800
|
+
action="store_true",
|
|
801
|
+
default=False,
|
|
802
|
+
dest="json_output",
|
|
803
|
+
help="Output results as JSON (for CI/automation).",
|
|
804
|
+
)
|
|
805
|
+
doctor.add_argument(
|
|
806
|
+
"--verbose",
|
|
807
|
+
action="store_true",
|
|
808
|
+
default=False,
|
|
809
|
+
help="Show detailed diagnostics for each check.",
|
|
810
|
+
)
|
|
811
|
+
|
|
812
|
+
# ----------------------------------------------------------------
|
|
813
|
+
# point-and-fire sub-command
|
|
814
|
+
# ----------------------------------------------------------------
|
|
815
|
+
paf = subparsers.add_parser(
|
|
816
|
+
"point-and-fire",
|
|
817
|
+
help="Autonomous end-to-end test loop: crawl → generate → execute → feedback → repeat.",
|
|
818
|
+
)
|
|
819
|
+
paf.add_argument(
|
|
820
|
+
"url",
|
|
821
|
+
help="Root URL of the web application to test.",
|
|
822
|
+
)
|
|
823
|
+
paf.add_argument(
|
|
824
|
+
"--strategy",
|
|
825
|
+
default="all",
|
|
826
|
+
choices=["all", "high_risk", "new_changed", "belief_targeted"],
|
|
827
|
+
help="Scenario selection strategy (default: all).",
|
|
828
|
+
)
|
|
829
|
+
paf.add_argument(
|
|
830
|
+
"--max-iterations",
|
|
831
|
+
type=int,
|
|
832
|
+
default=5,
|
|
833
|
+
dest="max_iterations",
|
|
834
|
+
help="Maximum number of loop iterations (default: 5).",
|
|
835
|
+
)
|
|
836
|
+
paf.add_argument(
|
|
837
|
+
"--max-duration",
|
|
838
|
+
type=float,
|
|
839
|
+
default=600.0,
|
|
840
|
+
dest="max_duration_s",
|
|
841
|
+
help="Maximum total run time in seconds (default: 600).",
|
|
842
|
+
)
|
|
843
|
+
paf.add_argument(
|
|
844
|
+
"--stability-threshold",
|
|
845
|
+
type=int,
|
|
846
|
+
default=2,
|
|
847
|
+
dest="stability_threshold",
|
|
848
|
+
help="Stop after N consecutive iterations with no new failures (default: 2).",
|
|
849
|
+
)
|
|
850
|
+
paf.add_argument(
|
|
851
|
+
"--risk-threshold",
|
|
852
|
+
type=float,
|
|
853
|
+
default=0.6,
|
|
854
|
+
dest="risk_threshold",
|
|
855
|
+
help="Minimum risk_score for high_risk strategy (default: 0.6).",
|
|
856
|
+
)
|
|
857
|
+
paf.add_argument(
|
|
858
|
+
"--max-pages",
|
|
859
|
+
type=int,
|
|
860
|
+
default=50,
|
|
861
|
+
dest="max_pages",
|
|
862
|
+
help="Maximum pages to crawl per iteration (default: 50).",
|
|
863
|
+
)
|
|
864
|
+
paf.add_argument(
|
|
865
|
+
"--max-depth",
|
|
866
|
+
type=int,
|
|
867
|
+
default=5,
|
|
868
|
+
dest="max_depth",
|
|
869
|
+
help="Maximum BFS crawl depth (default: 5).",
|
|
870
|
+
)
|
|
871
|
+
paf.add_argument(
|
|
872
|
+
"--no-headless",
|
|
873
|
+
action="store_true",
|
|
874
|
+
default=False,
|
|
875
|
+
dest="no_headless",
|
|
876
|
+
help="Show browser windows (default: headless).",
|
|
877
|
+
)
|
|
878
|
+
paf.add_argument(
|
|
879
|
+
"--recrawl-every",
|
|
880
|
+
type=int,
|
|
881
|
+
default=3,
|
|
882
|
+
dest="recrawl_every",
|
|
883
|
+
help="Re-crawl every N iterations (default: 3).",
|
|
884
|
+
)
|
|
885
|
+
paf.add_argument(
|
|
886
|
+
"--use-llm",
|
|
887
|
+
action="store_true",
|
|
888
|
+
default=False,
|
|
889
|
+
dest="use_llm",
|
|
890
|
+
help="Enable LLM-augmented scenario generation.",
|
|
891
|
+
)
|
|
892
|
+
paf.add_argument("--llm-provider", default=None, dest="llm_provider", help="LLM provider name.")
|
|
893
|
+
paf.add_argument("--llm-model", default=None, dest="llm_model", help="LLM model name.")
|
|
894
|
+
paf.add_argument("--llm-api-key", default=None, dest="llm_api_key", help="LLM API key.")
|
|
895
|
+
paf.add_argument(
|
|
896
|
+
"--output",
|
|
897
|
+
"-o",
|
|
898
|
+
default="text",
|
|
899
|
+
choices=["text", "json"],
|
|
900
|
+
help="Output format (default: text).",
|
|
901
|
+
)
|
|
902
|
+
paf.add_argument(
|
|
903
|
+
"--schedule",
|
|
904
|
+
default=None,
|
|
905
|
+
dest="schedule",
|
|
906
|
+
metavar="CRON",
|
|
907
|
+
help=(
|
|
908
|
+
"Register the run as a recurring scheduled sweep using this cron expression "
|
|
909
|
+
"(e.g. '0 2 * * *'). Requires the NAT server to be running with a database "
|
|
910
|
+
"configured. When supplied, the command registers the schedule and returns "
|
|
911
|
+
"without executing the run immediately."
|
|
912
|
+
),
|
|
913
|
+
)
|
|
914
|
+
paf.add_argument(
|
|
915
|
+
"--schedule-name",
|
|
916
|
+
default=None,
|
|
917
|
+
dest="schedule_name",
|
|
918
|
+
help="Human-readable name for the scheduled sweep (default: auto-generated).",
|
|
919
|
+
)
|
|
920
|
+
paf.add_argument(
|
|
921
|
+
"--tenant-id",
|
|
922
|
+
default=None,
|
|
923
|
+
dest="tenant_id",
|
|
924
|
+
help="Tenant UUID to associate with the scheduled sweep.",
|
|
925
|
+
)
|
|
926
|
+
|
|
927
|
+
# ----------------------------------------------------------------
|
|
928
|
+
# import sub-command
|
|
929
|
+
# ----------------------------------------------------------------
|
|
930
|
+
imp = subparsers.add_parser(
|
|
931
|
+
"import",
|
|
932
|
+
help=(
|
|
933
|
+
"Batch import test artifacts (HAR, Playwright, Cypress, mitmproxy) "
|
|
934
|
+
"and convert them into scenario seeds."
|
|
935
|
+
),
|
|
936
|
+
)
|
|
937
|
+
imp.add_argument(
|
|
938
|
+
"--source",
|
|
939
|
+
required=True,
|
|
940
|
+
dest="source",
|
|
941
|
+
metavar="PATH",
|
|
942
|
+
help=(
|
|
943
|
+
"File or directory to import. When a directory is given, all "
|
|
944
|
+
"matching files are discovered recursively."
|
|
945
|
+
),
|
|
946
|
+
)
|
|
947
|
+
imp.add_argument(
|
|
948
|
+
"--format",
|
|
949
|
+
default="auto",
|
|
950
|
+
dest="import_format",
|
|
951
|
+
metavar="FORMAT",
|
|
952
|
+
help=(
|
|
953
|
+
"Artifact format: auto, har, playwright, cypress, traffic, postman, "
|
|
954
|
+
"openapi, graphql, gherkin, curl (default: auto)."
|
|
955
|
+
),
|
|
956
|
+
)
|
|
957
|
+
imp.add_argument(
|
|
958
|
+
"--base-url",
|
|
959
|
+
default="",
|
|
960
|
+
dest="base_url",
|
|
961
|
+
metavar="URL",
|
|
962
|
+
help="Base URL used to resolve relative paths in imported artifacts.",
|
|
963
|
+
)
|
|
964
|
+
imp.add_argument(
|
|
965
|
+
"--output",
|
|
966
|
+
"-o",
|
|
967
|
+
default="text",
|
|
968
|
+
choices=["text", "json"],
|
|
969
|
+
help="Output format (default: text).",
|
|
970
|
+
)
|
|
971
|
+
imp.add_argument(
|
|
972
|
+
"--replay-mode",
|
|
973
|
+
action="store_true",
|
|
974
|
+
default=False,
|
|
975
|
+
dest="replay_mode",
|
|
976
|
+
help="Enable HAR replay mode — preserve request ordering and timing.",
|
|
977
|
+
)
|
|
978
|
+
imp.add_argument(
|
|
979
|
+
"--out-file",
|
|
980
|
+
default=None,
|
|
981
|
+
dest="out_file",
|
|
982
|
+
metavar="FILE",
|
|
983
|
+
help="Write scenario seeds to this JSON file instead of stdout.",
|
|
984
|
+
)
|
|
985
|
+
|
|
986
|
+
# ----------------------------------------------------------------
|
|
987
|
+
# worker sub-command group
|
|
988
|
+
# ----------------------------------------------------------------
|
|
989
|
+
worker = subparsers.add_parser(
|
|
990
|
+
"worker",
|
|
991
|
+
help="Manage NAT worker pools for distributed test execution.",
|
|
992
|
+
)
|
|
993
|
+
worker_sub = worker.add_subparsers(dest="worker_command", required=True)
|
|
994
|
+
|
|
995
|
+
# nat worker launch
|
|
996
|
+
worker_launch = worker_sub.add_parser(
|
|
997
|
+
"launch",
|
|
998
|
+
help="Launch a local worker pool that consumes jobs from the queue.",
|
|
999
|
+
)
|
|
1000
|
+
worker_launch.add_argument(
|
|
1001
|
+
"--count",
|
|
1002
|
+
type=int,
|
|
1003
|
+
default=4,
|
|
1004
|
+
metavar="N",
|
|
1005
|
+
help="Number of concurrent workers to launch (default: 4).",
|
|
1006
|
+
)
|
|
1007
|
+
worker_launch.add_argument(
|
|
1008
|
+
"--queue",
|
|
1009
|
+
default="memory",
|
|
1010
|
+
metavar="BACKEND",
|
|
1011
|
+
help=(
|
|
1012
|
+
"Queue backend: 'memory' (default), 'sqlite://<path>', "
|
|
1013
|
+
"or 'redis://<host>:<port>[/<db>]'."
|
|
1014
|
+
),
|
|
1015
|
+
)
|
|
1016
|
+
worker_launch.add_argument(
|
|
1017
|
+
"--poll-interval",
|
|
1018
|
+
type=float,
|
|
1019
|
+
default=1.0,
|
|
1020
|
+
metavar="SECONDS",
|
|
1021
|
+
help="Seconds to wait between polling for new jobs (default: 1.0).",
|
|
1022
|
+
)
|
|
1023
|
+
worker_launch.add_argument(
|
|
1024
|
+
"--max-attempts",
|
|
1025
|
+
type=int,
|
|
1026
|
+
default=3,
|
|
1027
|
+
metavar="N",
|
|
1028
|
+
help="Maximum retry attempts per job before marking it failed (default: 3).",
|
|
1029
|
+
)
|
|
1030
|
+
|
|
1031
|
+
# nat worker status
|
|
1032
|
+
worker_status = worker_sub.add_parser(
|
|
1033
|
+
"status",
|
|
1034
|
+
help="Show current worker pool and queue status.",
|
|
1035
|
+
)
|
|
1036
|
+
worker_status.add_argument(
|
|
1037
|
+
"--queue",
|
|
1038
|
+
default="memory",
|
|
1039
|
+
metavar="BACKEND",
|
|
1040
|
+
help="Queue backend to query (default: memory).",
|
|
1041
|
+
)
|
|
1042
|
+
worker_status.add_argument(
|
|
1043
|
+
"--output",
|
|
1044
|
+
choices=["text", "json"],
|
|
1045
|
+
default="text",
|
|
1046
|
+
help="Output format (default: text).",
|
|
1047
|
+
)
|
|
1048
|
+
|
|
1049
|
+
# ----------------------------------------------------------------
|
|
1050
|
+
# plan sub-command
|
|
1051
|
+
# ----------------------------------------------------------------
|
|
1052
|
+
plan = subparsers.add_parser(
|
|
1053
|
+
"plan",
|
|
1054
|
+
help=(
|
|
1055
|
+
"Generate an AI-powered test plan from an API spec. "
|
|
1056
|
+
"Uses the LLM to produce a priority-ranked, risk-assessed plan "
|
|
1057
|
+
"of endpoint groups, attack vectors, and edge cases."
|
|
1058
|
+
),
|
|
1059
|
+
)
|
|
1060
|
+
plan.add_argument(
|
|
1061
|
+
"--spec",
|
|
1062
|
+
required=True,
|
|
1063
|
+
metavar="PATH",
|
|
1064
|
+
help="Path to an OpenAPI 3.x or Swagger 2.x spec file.",
|
|
1065
|
+
)
|
|
1066
|
+
plan.add_argument(
|
|
1067
|
+
"--base-url",
|
|
1068
|
+
default="http://localhost:8080",
|
|
1069
|
+
metavar="URL",
|
|
1070
|
+
help="Base URL of the target API (for context; default: http://localhost:8080).",
|
|
1071
|
+
)
|
|
1072
|
+
plan.add_argument(
|
|
1073
|
+
"--output",
|
|
1074
|
+
choices=["text", "json"],
|
|
1075
|
+
default="text",
|
|
1076
|
+
help="Output format (default: text).",
|
|
1077
|
+
)
|
|
1078
|
+
plan.add_argument(
|
|
1079
|
+
"--execute",
|
|
1080
|
+
action="store_true",
|
|
1081
|
+
default=False,
|
|
1082
|
+
help="Auto-approve and execute the plan immediately after generation.",
|
|
1083
|
+
)
|
|
1084
|
+
plan.add_argument(
|
|
1085
|
+
"--llm-provider",
|
|
1086
|
+
choices=["openai", "anthropic"],
|
|
1087
|
+
default="openai",
|
|
1088
|
+
metavar="PROVIDER",
|
|
1089
|
+
help="LLM provider: openai (default) or anthropic.",
|
|
1090
|
+
)
|
|
1091
|
+
plan.add_argument(
|
|
1092
|
+
"--llm-model",
|
|
1093
|
+
default=None,
|
|
1094
|
+
metavar="MODEL",
|
|
1095
|
+
help="LLM model name override.",
|
|
1096
|
+
)
|
|
1097
|
+
plan.add_argument(
|
|
1098
|
+
"--llm-api-key",
|
|
1099
|
+
default=None,
|
|
1100
|
+
metavar="KEY",
|
|
1101
|
+
help="LLM API key.",
|
|
1102
|
+
)
|
|
1103
|
+
|
|
1104
|
+
nat_test = subparsers.add_parser(
|
|
1105
|
+
"test",
|
|
1106
|
+
help=(
|
|
1107
|
+
"Generate executable test scenarios from a plain-English description. "
|
|
1108
|
+
"Uses the LLM to translate natural language into API and browser test scenarios "
|
|
1109
|
+
"that can be reviewed and optionally executed immediately."
|
|
1110
|
+
),
|
|
1111
|
+
)
|
|
1112
|
+
nat_test.add_argument(
|
|
1113
|
+
"description",
|
|
1114
|
+
metavar="DESCRIPTION",
|
|
1115
|
+
help='Natural-language description of what to test (e.g. "Verify admin cannot access other tenants data").',
|
|
1116
|
+
)
|
|
1117
|
+
nat_test.add_argument(
|
|
1118
|
+
"--base-url",
|
|
1119
|
+
default="",
|
|
1120
|
+
metavar="URL",
|
|
1121
|
+
help="Base URL of the target API or application.",
|
|
1122
|
+
)
|
|
1123
|
+
nat_test.add_argument(
|
|
1124
|
+
"--auth-type",
|
|
1125
|
+
default="none",
|
|
1126
|
+
metavar="AUTH",
|
|
1127
|
+
help="Auth type context hint (e.g. bearer, api-key, cookie; default: none).",
|
|
1128
|
+
)
|
|
1129
|
+
nat_test.add_argument(
|
|
1130
|
+
"--known-endpoints",
|
|
1131
|
+
default="",
|
|
1132
|
+
metavar="ENDPOINTS",
|
|
1133
|
+
help="Comma-separated list of known endpoints in METHOD /path format.",
|
|
1134
|
+
)
|
|
1135
|
+
nat_test.add_argument(
|
|
1136
|
+
"--execute",
|
|
1137
|
+
action="store_true",
|
|
1138
|
+
default=False,
|
|
1139
|
+
help="Generate and immediately execute the scenarios.",
|
|
1140
|
+
)
|
|
1141
|
+
nat_test.add_argument(
|
|
1142
|
+
"--headed",
|
|
1143
|
+
action="store_true",
|
|
1144
|
+
default=False,
|
|
1145
|
+
help="Run browser scenarios in headed mode (not headless).",
|
|
1146
|
+
)
|
|
1147
|
+
nat_test.add_argument(
|
|
1148
|
+
"--output",
|
|
1149
|
+
choices=["text", "json"],
|
|
1150
|
+
default="text",
|
|
1151
|
+
help="Output format (default: text).",
|
|
1152
|
+
)
|
|
1153
|
+
nat_test.add_argument(
|
|
1154
|
+
"--llm-provider",
|
|
1155
|
+
choices=["openai", "anthropic"],
|
|
1156
|
+
default="openai",
|
|
1157
|
+
metavar="PROVIDER",
|
|
1158
|
+
help="LLM provider: openai (default) or anthropic.",
|
|
1159
|
+
)
|
|
1160
|
+
nat_test.add_argument(
|
|
1161
|
+
"--llm-model",
|
|
1162
|
+
default=None,
|
|
1163
|
+
metavar="MODEL",
|
|
1164
|
+
help="LLM model name override.",
|
|
1165
|
+
)
|
|
1166
|
+
nat_test.add_argument(
|
|
1167
|
+
"--llm-api-key",
|
|
1168
|
+
default=None,
|
|
1169
|
+
metavar="KEY",
|
|
1170
|
+
help="LLM API key.",
|
|
1171
|
+
)
|
|
1172
|
+
|
|
1173
|
+
return parser
|
|
1174
|
+
|
|
1175
|
+
|
|
1176
|
+
def _build_llm_provider(args: argparse.Namespace) -> Optional[Any]:
|
|
1177
|
+
"""Build an :class:`~mannf.product.llm.base.LLMProvider` from *args*, or ``None`` if disabled."""
|
|
1178
|
+
if not getattr(args, "use_llm", False):
|
|
1179
|
+
return None
|
|
1180
|
+
try:
|
|
1181
|
+
from mannf.product.llm.config import LLMConfig
|
|
1182
|
+
from mannf.product.llm.factory import get_provider
|
|
1183
|
+
config = LLMConfig(
|
|
1184
|
+
provider=getattr(args, "llm_provider", "openai") or "openai",
|
|
1185
|
+
model=getattr(args, "llm_model", None) or "",
|
|
1186
|
+
api_key=getattr(args, "llm_api_key", None),
|
|
1187
|
+
)
|
|
1188
|
+
provider = get_provider(config)
|
|
1189
|
+
if not provider.is_available():
|
|
1190
|
+
logger.warning(
|
|
1191
|
+
"LLM provider %r is not available (no API key). "
|
|
1192
|
+
"Proceeding without LLM augmentation.",
|
|
1193
|
+
config.provider,
|
|
1194
|
+
)
|
|
1195
|
+
return None
|
|
1196
|
+
return provider
|
|
1197
|
+
except Exception as exc: # noqa: BLE001
|
|
1198
|
+
logger.warning("Could not initialise LLM provider: %s", exc)
|
|
1199
|
+
return None
|
|
1200
|
+
|
|
1201
|
+
|
|
1202
|
+
def _build_auth(args: argparse.Namespace):
|
|
1203
|
+
"""Return the appropriate :class:`~mannf.product.integrations.auth.AuthStrategy` from *args*."""
|
|
1204
|
+
from mannf.product.integrations.auth import ApiKeyAuth, BearerTokenAuth, NoAuth, OAuth2Auth
|
|
1205
|
+
|
|
1206
|
+
if getattr(args, "oauth2_token_url", None) and getattr(args, "oauth2_client_id", None):
|
|
1207
|
+
scopes = None
|
|
1208
|
+
if getattr(args, "oauth2_scopes", None):
|
|
1209
|
+
scopes = [s.strip() for s in args.oauth2_scopes.split(",") if s.strip()]
|
|
1210
|
+
return OAuth2Auth(
|
|
1211
|
+
token_url=args.oauth2_token_url,
|
|
1212
|
+
client_id=args.oauth2_client_id,
|
|
1213
|
+
client_secret=args.oauth2_client_secret or "",
|
|
1214
|
+
scopes=scopes,
|
|
1215
|
+
grant_type=getattr(args, "oauth2_grant_type", "client_credentials"),
|
|
1216
|
+
)
|
|
1217
|
+
if getattr(args, "auth_token", None):
|
|
1218
|
+
return BearerTokenAuth(args.auth_token)
|
|
1219
|
+
if getattr(args, "api_key", None):
|
|
1220
|
+
return ApiKeyAuth(args.api_key, getattr(args, "api_key_header", "X-API-Key"))
|
|
1221
|
+
return NoAuth()
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
def _resolve_weights_dir_arg(args: argparse.Namespace) -> None:
|
|
1225
|
+
"""If ``--weights-dir`` is set, auto-populate ``--load-weights`` and ``--save-weights``.
|
|
1226
|
+
|
|
1227
|
+
The latest weight file for the given API (keyed by base URL) is loaded
|
|
1228
|
+
automatically, and a new file is saved after the scan completes.
|
|
1229
|
+
"""
|
|
1230
|
+
weights_dir = getattr(args, "weights_dir", None)
|
|
1231
|
+
if not weights_dir:
|
|
1232
|
+
return
|
|
1233
|
+
|
|
1234
|
+
import re
|
|
1235
|
+
safe_filename_prefix = re.sub(r"[^a-zA-Z0-9_-]", "_", getattr(args, "base_url", "api"))
|
|
1236
|
+
latest = _find_latest_weights_file(weights_dir, safe_filename_prefix)
|
|
1237
|
+
|
|
1238
|
+
if latest and not getattr(args, "load_weights", None):
|
|
1239
|
+
args.load_weights = latest
|
|
1240
|
+
logger.info("Auto-loading weights from %s", latest)
|
|
1241
|
+
|
|
1242
|
+
if not getattr(args, "save_weights", None):
|
|
1243
|
+
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
1244
|
+
args.save_weights = os.path.join(weights_dir, f"{safe_filename_prefix}_{ts}.json")
|
|
1245
|
+
|
|
1246
|
+
|
|
1247
|
+
def _find_latest_weights_file(directory: str, prefix: str) -> Optional[str]:
|
|
1248
|
+
"""Return the most recently modified weight file matching *prefix* in *directory*."""
|
|
1249
|
+
if not os.path.isdir(directory):
|
|
1250
|
+
return None
|
|
1251
|
+
candidates = [
|
|
1252
|
+
os.path.join(directory, f)
|
|
1253
|
+
for f in os.listdir(directory)
|
|
1254
|
+
if f.startswith(prefix) and f.endswith(".json")
|
|
1255
|
+
]
|
|
1256
|
+
if not candidates:
|
|
1257
|
+
return None
|
|
1258
|
+
return max(candidates, key=os.path.getmtime)
|
|
1259
|
+
|
|
1260
|
+
|
|
1261
|
+
def _handle_post_scan_weights(args: argparse.Namespace) -> None:
|
|
1262
|
+
"""Save agent weights after a scan if ``--save-weights`` is set.
|
|
1263
|
+
|
|
1264
|
+
Because the CLI scan runs through ``HttpApiSUT`` directly (without
|
|
1265
|
+
instantiating BDI agents), this function saves an empty-agents weight file
|
|
1266
|
+
as a valid placeholder so downstream tools can detect that a scan ran.
|
|
1267
|
+
When the full BDI orchestrator is used the agents dict will be populated.
|
|
1268
|
+
"""
|
|
1269
|
+
save_path = getattr(args, "save_weights", None)
|
|
1270
|
+
if not save_path:
|
|
1271
|
+
return
|
|
1272
|
+
|
|
1273
|
+
from mannf.product.weights.store import WeightStore
|
|
1274
|
+
|
|
1275
|
+
try:
|
|
1276
|
+
WeightStore.save(save_path, {})
|
|
1277
|
+
logger.info("Agent weights saved to %s", save_path)
|
|
1278
|
+
print(f" Agent weights saved to: {save_path}")
|
|
1279
|
+
except OSError as exc:
|
|
1280
|
+
logger.warning("Could not save weights to %s: %s", save_path, exc)
|
|
1281
|
+
|
|
1282
|
+
|
|
1283
|
+
# ---------------------------------------------------------------------------
|
|
1284
|
+
# Output formatters
|
|
1285
|
+
# ---------------------------------------------------------------------------
|
|
1286
|
+
|
|
1287
|
+
def _format_html_report(report: dict, suite: "TestSuite", args: argparse.Namespace) -> str:
|
|
1288
|
+
"""Render a standalone HTML report for a scan result."""
|
|
1289
|
+
s = report.get("suite", {})
|
|
1290
|
+
spec = report.get("spec") or report.get("endpoint", "")
|
|
1291
|
+
base_url = report.get("base_url", getattr(args, "base_url", ""))
|
|
1292
|
+
total = s.get("total", 0)
|
|
1293
|
+
passed = s.get("passed", 0)
|
|
1294
|
+
failed = s.get("failed", 0)
|
|
1295
|
+
errored = s.get("errored", 0)
|
|
1296
|
+
pass_rate = s.get("pass_rate", 0.0) * 100
|
|
1297
|
+
avg_latency = s.get("avg_execution_time_ms", 0.0)
|
|
1298
|
+
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
|
1299
|
+
|
|
1300
|
+
# Build rows for individual results
|
|
1301
|
+
rows_html = ""
|
|
1302
|
+
for result in getattr(suite, "results", []):
|
|
1303
|
+
status = "PASS" if result.passed else ("ERROR" if result.error else "FAIL")
|
|
1304
|
+
css_class = "pass" if result.passed else ("error" if result.error else "fail")
|
|
1305
|
+
tc_id = result.test_case_id or ""
|
|
1306
|
+
latency = f"{result.execution_time_ms:.1f}" if result.execution_time_ms is not None else "—"
|
|
1307
|
+
error_msg = str(result.error) if result.error else ""
|
|
1308
|
+
rows_html += (
|
|
1309
|
+
f"<tr class='{css_class}'>"
|
|
1310
|
+
f"<td>{tc_id}</td>"
|
|
1311
|
+
f"<td class='status'>{status}</td>"
|
|
1312
|
+
f"<td>{latency} ms</td>"
|
|
1313
|
+
f"<td>{error_msg}</td>"
|
|
1314
|
+
f"</tr>\n"
|
|
1315
|
+
)
|
|
1316
|
+
|
|
1317
|
+
return f"""<!DOCTYPE html>
|
|
1318
|
+
<html lang="en">
|
|
1319
|
+
<head>
|
|
1320
|
+
<meta charset="UTF-8">
|
|
1321
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
1322
|
+
<title>NAT Scan Report</title>
|
|
1323
|
+
<style>
|
|
1324
|
+
body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; margin: 0; background: #f5f7fa; color: #333; }}
|
|
1325
|
+
.header {{ background: #1a1a2e; color: white; padding: 24px 32px; }}
|
|
1326
|
+
.header h1 {{ margin: 0 0 4px; font-size: 1.6rem; }}
|
|
1327
|
+
.header p {{ margin: 0; opacity: 0.7; font-size: 0.9rem; }}
|
|
1328
|
+
.container {{ max-width: 1100px; margin: 32px auto; padding: 0 24px; }}
|
|
1329
|
+
.summary {{ display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 16px; margin-bottom: 32px; }}
|
|
1330
|
+
.card {{ background: white; border-radius: 8px; padding: 20px; box-shadow: 0 1px 4px rgba(0,0,0,.08); text-align: center; }}
|
|
1331
|
+
.card .value {{ font-size: 2rem; font-weight: bold; }}
|
|
1332
|
+
.card .label {{ font-size: 0.8rem; color: #888; margin-top: 4px; text-transform: uppercase; }}
|
|
1333
|
+
.pass .value {{ color: #22c55e; }}
|
|
1334
|
+
.fail .value {{ color: #ef4444; }}
|
|
1335
|
+
.error .value {{ color: #f97316; }}
|
|
1336
|
+
.rate .value {{ color: #3b82f6; }}
|
|
1337
|
+
table {{ width: 100%; border-collapse: collapse; background: white; border-radius: 8px; overflow: hidden; box-shadow: 0 1px 4px rgba(0,0,0,.08); }}
|
|
1338
|
+
th {{ background: #f1f5f9; padding: 12px 16px; text-align: left; font-size: 0.85rem; color: #555; border-bottom: 1px solid #e2e8f0; }}
|
|
1339
|
+
td {{ padding: 10px 16px; border-bottom: 1px solid #f1f5f9; font-size: 0.9rem; }}
|
|
1340
|
+
tr:last-child td {{ border-bottom: none; }}
|
|
1341
|
+
tr.pass td.status {{ color: #22c55e; font-weight: bold; }}
|
|
1342
|
+
tr.fail td.status {{ color: #ef4444; font-weight: bold; }}
|
|
1343
|
+
tr.error td.status {{ color: #f97316; font-weight: bold; }}
|
|
1344
|
+
.meta {{ font-size: 0.8rem; color: #888; margin-bottom: 16px; }}
|
|
1345
|
+
h2 {{ font-size: 1.1rem; margin-bottom: 12px; color: #444; }}
|
|
1346
|
+
</style>
|
|
1347
|
+
</head>
|
|
1348
|
+
<body>
|
|
1349
|
+
<div class="header">
|
|
1350
|
+
<h1>NAT Scan Report</h1>
|
|
1351
|
+
<p>Generated: {ts}</p>
|
|
1352
|
+
</div>
|
|
1353
|
+
<div class="container">
|
|
1354
|
+
<div class="meta">
|
|
1355
|
+
<strong>Spec:</strong> {spec} |
|
|
1356
|
+
<strong>Base URL:</strong> {base_url}
|
|
1357
|
+
</div>
|
|
1358
|
+
<div class="summary">
|
|
1359
|
+
<div class="card"><div class="value">{total}</div><div class="label">Total</div></div>
|
|
1360
|
+
<div class="card pass"><div class="value">{passed}</div><div class="label">Passed</div></div>
|
|
1361
|
+
<div class="card fail"><div class="value">{failed}</div><div class="label">Failed</div></div>
|
|
1362
|
+
<div class="card error"><div class="value">{errored}</div><div class="label">Errored</div></div>
|
|
1363
|
+
<div class="card rate"><div class="value">{pass_rate:.1f}%</div><div class="label">Pass Rate</div></div>
|
|
1364
|
+
<div class="card"><div class="value">{avg_latency:.0f}</div><div class="label">Avg Latency (ms)</div></div>
|
|
1365
|
+
</div>
|
|
1366
|
+
<h2>Test Results</h2>
|
|
1367
|
+
<table>
|
|
1368
|
+
<thead><tr><th>Test Case</th><th>Status</th><th>Latency</th><th>Details</th></tr></thead>
|
|
1369
|
+
<tbody>
|
|
1370
|
+
{rows_html} </tbody>
|
|
1371
|
+
</table>
|
|
1372
|
+
</div>
|
|
1373
|
+
</body>
|
|
1374
|
+
</html>"""
|
|
1375
|
+
|
|
1376
|
+
|
|
1377
|
+
def _format_junit_report(report: dict, suite: "TestSuite", args: argparse.Namespace) -> str:
|
|
1378
|
+
"""Render a JUnit XML report for a scan result.
|
|
1379
|
+
|
|
1380
|
+
Compatible with Jenkins, GitLab CI, and Azure DevOps test result import.
|
|
1381
|
+
"""
|
|
1382
|
+
s = report.get("suite", {})
|
|
1383
|
+
suite_name = getattr(suite, "name", report.get("spec", "NAT Scan"))
|
|
1384
|
+
total = s.get("total", 0)
|
|
1385
|
+
failures = s.get("failed", 0)
|
|
1386
|
+
errors = s.get("errored", 0)
|
|
1387
|
+
elapsed = s.get("total_execution_time_ms", 0.0) / 1000.0
|
|
1388
|
+
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
|
|
1389
|
+
|
|
1390
|
+
testsuites = ET.Element("testsuites", {
|
|
1391
|
+
"name": "NAT",
|
|
1392
|
+
"tests": str(total),
|
|
1393
|
+
"failures": str(failures),
|
|
1394
|
+
"errors": str(errors),
|
|
1395
|
+
"time": f"{elapsed:.3f}",
|
|
1396
|
+
})
|
|
1397
|
+
testsuite = ET.SubElement(testsuites, "testsuite", {
|
|
1398
|
+
"name": suite_name,
|
|
1399
|
+
"tests": str(total),
|
|
1400
|
+
"failures": str(failures),
|
|
1401
|
+
"errors": str(errors),
|
|
1402
|
+
"time": f"{elapsed:.3f}",
|
|
1403
|
+
"timestamp": ts,
|
|
1404
|
+
})
|
|
1405
|
+
|
|
1406
|
+
for result in getattr(suite, "results", []):
|
|
1407
|
+
tc_id = result.test_case_id or "unknown"
|
|
1408
|
+
tc_time = f"{(result.execution_time_ms or 0.0) / 1000.0:.3f}"
|
|
1409
|
+
tc_elem = ET.SubElement(testsuite, "testcase", {
|
|
1410
|
+
"name": tc_id,
|
|
1411
|
+
"classname": "NAT",
|
|
1412
|
+
"time": tc_time,
|
|
1413
|
+
})
|
|
1414
|
+
if result.error:
|
|
1415
|
+
err_elem = ET.SubElement(tc_elem, "error", {"message": str(result.error)})
|
|
1416
|
+
err_elem.text = str(result.error)
|
|
1417
|
+
elif not result.passed:
|
|
1418
|
+
reason = str(result.error) if result.error else "Test case failed"
|
|
1419
|
+
fail_elem = ET.SubElement(tc_elem, "failure", {"message": reason})
|
|
1420
|
+
fail_elem.text = reason
|
|
1421
|
+
|
|
1422
|
+
ET.indent(testsuites, space=" ")
|
|
1423
|
+
return '<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(testsuites, encoding="unicode")
|
|
1424
|
+
|
|
1425
|
+
|
|
1426
|
+
def _format_allure_report(
|
|
1427
|
+
report: dict, suite: "TestSuite", output_dir: str
|
|
1428
|
+
) -> list[str]:
|
|
1429
|
+
"""Write Allure result JSON files to *output_dir* and return paths written.
|
|
1430
|
+
|
|
1431
|
+
One ``*-result.json`` file is written per test case. The directory is
|
|
1432
|
+
created if it does not already exist.
|
|
1433
|
+
"""
|
|
1434
|
+
from mannf.product.formatters.allure_formatter import write_allure_results
|
|
1435
|
+
return write_allure_results(report, suite, output_dir)
|
|
1436
|
+
|
|
1437
|
+
|
|
1438
|
+
def _format_ctrf_report(report: dict, suite: "TestSuite") -> str:
|
|
1439
|
+
"""Return a CTRF-schema JSON string for the scan result."""
|
|
1440
|
+
from mannf.product.formatters.ctrf_formatter import build_ctrf_report
|
|
1441
|
+
return build_ctrf_report(report, suite)
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
def _write_output(content: str, output_file: Optional[str]) -> None:
|
|
1445
|
+
"""Write *content* to *output_file* or stdout."""
|
|
1446
|
+
if output_file:
|
|
1447
|
+
Path(output_file).parent.mkdir(parents=True, exist_ok=True)
|
|
1448
|
+
with open(output_file, "w", encoding="utf-8") as fh:
|
|
1449
|
+
fh.write(content)
|
|
1450
|
+
print(f" Output written to: {output_file}", file=sys.stderr)
|
|
1451
|
+
else:
|
|
1452
|
+
print(content)
|
|
1453
|
+
|
|
1454
|
+
|
|
1455
|
+
def _auto_output_filename(fmt: str, base: str = "nat-report") -> str:
|
|
1456
|
+
"""Return an auto-generated output filename for *fmt*."""
|
|
1457
|
+
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
1458
|
+
ext = {"html": "html", "junit": "xml", "ctrf": "json", "allure": "", "pdf": "pdf"}.get(fmt, "txt")
|
|
1459
|
+
if fmt == "allure":
|
|
1460
|
+
# allure writes a directory, not a single file
|
|
1461
|
+
return f"{base}-allure-results-{ts}"
|
|
1462
|
+
return f"{base}-{ts}.{ext}"
|
|
1463
|
+
|
|
1464
|
+
|
|
1465
|
+
async def _generate_pdf_from_html(html: str, output_file: str) -> None:
|
|
1466
|
+
"""Convert *html* to a PDF file written to *output_file*.
|
|
1467
|
+
|
|
1468
|
+
Uses :class:`~mannf.product.reports.pdf.PDFReportGenerator` which
|
|
1469
|
+
requires Playwright and an installed Chromium browser binary.
|
|
1470
|
+
|
|
1471
|
+
Raises:
|
|
1472
|
+
RuntimeError: If Playwright or its browser binaries are not installed.
|
|
1473
|
+
"""
|
|
1474
|
+
from mannf.product.reports.pdf import PDFReportGenerator # noqa: PLC0415
|
|
1475
|
+
|
|
1476
|
+
generator = PDFReportGenerator()
|
|
1477
|
+
pdf_bytes = await generator.generate(html)
|
|
1478
|
+
Path(output_file).parent.mkdir(parents=True, exist_ok=True)
|
|
1479
|
+
with open(output_file, "wb") as fh:
|
|
1480
|
+
fh.write(pdf_bytes)
|
|
1481
|
+
print(f" PDF report written to: {output_file}", file=sys.stderr)
|
|
1482
|
+
|
|
1483
|
+
|
|
1484
|
+
async def _run_scan(args: argparse.Namespace) -> int:
|
|
1485
|
+
"""Execute the *scan* sub-command. Returns exit code."""
|
|
1486
|
+
from mannf.product.integrations.auth import ApiKeyAuth, BearerTokenAuth, NoAuth
|
|
1487
|
+
from mannf.product.integrations.http_sut import HttpApiSUT
|
|
1488
|
+
from mannf.product.integrations.openapi_parser import OpenApiParser
|
|
1489
|
+
from mannf.core.testing.models import TestSuite
|
|
1490
|
+
|
|
1491
|
+
api_type = getattr(args, "type", "rest")
|
|
1492
|
+
|
|
1493
|
+
if api_type == "graphql":
|
|
1494
|
+
return await _run_graphql_scan(args)
|
|
1495
|
+
|
|
1496
|
+
if api_type == "grpc":
|
|
1497
|
+
return await _run_grpc_scan(args)
|
|
1498
|
+
|
|
1499
|
+
if api_type == "websocket":
|
|
1500
|
+
return await _run_websocket_scan(args)
|
|
1501
|
+
|
|
1502
|
+
# --- Regression replay mode ------------------------------------------
|
|
1503
|
+
replay_file = getattr(args, "replay", None)
|
|
1504
|
+
if replay_file:
|
|
1505
|
+
return await _run_regression_replay(args)
|
|
1506
|
+
|
|
1507
|
+
# --- Weight pre-scan: resolve auto-load path -------------------------
|
|
1508
|
+
_resolve_weights_dir_arg(args)
|
|
1509
|
+
|
|
1510
|
+
# --- Auth -----------------------------------------------------------
|
|
1511
|
+
auth = _build_auth(args)
|
|
1512
|
+
|
|
1513
|
+
# --- Regression recorder --------------------------------------------
|
|
1514
|
+
record_file = getattr(args, "record_regression", None)
|
|
1515
|
+
recorder = None
|
|
1516
|
+
if record_file:
|
|
1517
|
+
from mannf.product.regression.recorder import RegressionRecorder
|
|
1518
|
+
recorder = RegressionRecorder(
|
|
1519
|
+
spec=args.spec,
|
|
1520
|
+
base_url=args.base_url,
|
|
1521
|
+
mask_sensitive_data=getattr(args, "mask_sensitive", False),
|
|
1522
|
+
)
|
|
1523
|
+
|
|
1524
|
+
# --- SUT ------------------------------------------------------------
|
|
1525
|
+
sut = HttpApiSUT(
|
|
1526
|
+
base_url=args.base_url,
|
|
1527
|
+
timeout=args.timeout,
|
|
1528
|
+
auth=auth,
|
|
1529
|
+
latency_sla_ms=args.latency_sla,
|
|
1530
|
+
recorder=recorder,
|
|
1531
|
+
)
|
|
1532
|
+
|
|
1533
|
+
if not args.spec:
|
|
1534
|
+
print("--spec is required for REST scans.", file=sys.stderr)
|
|
1535
|
+
return EXIT_CONFIG_ERROR
|
|
1536
|
+
|
|
1537
|
+
# --- Parse spec ------------------------------------------------------
|
|
1538
|
+
llm_provider = _build_llm_provider(args)
|
|
1539
|
+
spec_format = getattr(args, "format", "openapi")
|
|
1540
|
+
try:
|
|
1541
|
+
if spec_format == "postman":
|
|
1542
|
+
from mannf.product.integrations.postman_parser import PostmanParser
|
|
1543
|
+
parser = PostmanParser(
|
|
1544
|
+
collection_path=args.spec,
|
|
1545
|
+
base_url=args.base_url,
|
|
1546
|
+
registry=sut.registry,
|
|
1547
|
+
max_tests=args.max_tests,
|
|
1548
|
+
)
|
|
1549
|
+
test_cases = parser.parse()
|
|
1550
|
+
else:
|
|
1551
|
+
parser = OpenApiParser(
|
|
1552
|
+
spec_path=args.spec,
|
|
1553
|
+
base_url=args.base_url,
|
|
1554
|
+
registry=sut.registry,
|
|
1555
|
+
max_tests=args.max_tests,
|
|
1556
|
+
llm_provider=llm_provider,
|
|
1557
|
+
)
|
|
1558
|
+
if llm_provider is not None:
|
|
1559
|
+
test_cases = await parser.parse_with_llm()
|
|
1560
|
+
else:
|
|
1561
|
+
test_cases = parser.parse()
|
|
1562
|
+
except (FileNotFoundError, ValueError, OSError) as exc:
|
|
1563
|
+
print(f"Spec parse error: {exc}", file=sys.stderr)
|
|
1564
|
+
return EXIT_SCAN_ERROR
|
|
1565
|
+
|
|
1566
|
+
if not test_cases:
|
|
1567
|
+
print("No test cases generated from spec.", file=sys.stderr)
|
|
1568
|
+
return EXIT_SCAN_ERROR
|
|
1569
|
+
|
|
1570
|
+
# --- Execute test cases directly through the SUT --------------------
|
|
1571
|
+
# The pre-generated TestCase objects from the OpenAPI parser carry
|
|
1572
|
+
# precise HTTP method, headers, body, and expected_status_codes that
|
|
1573
|
+
# must be preserved. Running them through HttpApiSUT directly (rather
|
|
1574
|
+
# than the adaptive NATOrchestrator BDI pipeline, which generates its own
|
|
1575
|
+
# generic test cases) ensures spec-accurate coverage.
|
|
1576
|
+
suite = TestSuite(name=f"NAT scan: {args.spec}")
|
|
1577
|
+
try:
|
|
1578
|
+
for tc in test_cases:
|
|
1579
|
+
suite.add_case(tc)
|
|
1580
|
+
result = await sut.execute(tc)
|
|
1581
|
+
suite.record_result(result)
|
|
1582
|
+
logger.debug(
|
|
1583
|
+
" %s %s → %s",
|
|
1584
|
+
tc.inputs.get("method", "GET"),
|
|
1585
|
+
tc.target,
|
|
1586
|
+
"PASS" if result.passed else "FAIL",
|
|
1587
|
+
)
|
|
1588
|
+
except Exception as exc:
|
|
1589
|
+
print(f"Scan error during test execution: {exc}", file=sys.stderr)
|
|
1590
|
+
return EXIT_SCAN_ERROR
|
|
1591
|
+
|
|
1592
|
+
# --- Save regression baseline ----------------------------------------
|
|
1593
|
+
if record_file and recorder is not None:
|
|
1594
|
+
recorder.save(record_file)
|
|
1595
|
+
logger.info("Regression baseline saved to: %s", record_file)
|
|
1596
|
+
if args.output not in ("json", "junit", "html", "allure", "ctrf"):
|
|
1597
|
+
print(f" Regression baseline saved to: {record_file}")
|
|
1598
|
+
|
|
1599
|
+
# --- Weight persistence (post-scan) ----------------------------------
|
|
1600
|
+
_handle_post_scan_weights(args)
|
|
1601
|
+
|
|
1602
|
+
# --- Build report dict -----------------------------------------------
|
|
1603
|
+
s = suite.summary()
|
|
1604
|
+
llm_cases = [tc for tc in test_cases if tc.metadata.get("generator") == "llm"]
|
|
1605
|
+
report = {
|
|
1606
|
+
"suite": s,
|
|
1607
|
+
"spec": args.spec,
|
|
1608
|
+
"base_url": args.base_url,
|
|
1609
|
+
"llm_cases_generated": len(llm_cases),
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
# --- Output ----------------------------------------------------------
|
|
1613
|
+
output_fmt = getattr(args, "output", "text")
|
|
1614
|
+
output_file = getattr(args, "output_file", None)
|
|
1615
|
+
|
|
1616
|
+
if output_fmt == "json":
|
|
1617
|
+
_write_output(json.dumps(report, indent=2), output_file)
|
|
1618
|
+
elif output_fmt == "html":
|
|
1619
|
+
html_path = output_file or _auto_output_filename("html", "nat-scan-report")
|
|
1620
|
+
_write_output(_format_html_report(report, suite, args), html_path)
|
|
1621
|
+
elif output_fmt == "pdf":
|
|
1622
|
+
pdf_path = output_file or _auto_output_filename("pdf", "nat-scan-report")
|
|
1623
|
+
await _generate_pdf_from_html(_format_html_report(report, suite, args), pdf_path)
|
|
1624
|
+
elif output_fmt == "junit":
|
|
1625
|
+
junit_path = output_file or _auto_output_filename("junit", "nat-scan-results")
|
|
1626
|
+
_write_output(_format_junit_report(report, suite, args), junit_path)
|
|
1627
|
+
elif output_fmt == "allure":
|
|
1628
|
+
allure_dir = output_file or _auto_output_filename("allure", "nat-scan")
|
|
1629
|
+
paths = _format_allure_report(report, suite, allure_dir)
|
|
1630
|
+
print(f" Allure results written to: {allure_dir} ({len(paths)} file(s))", file=sys.stderr)
|
|
1631
|
+
elif output_fmt == "ctrf":
|
|
1632
|
+
ctrf_path = output_file or _auto_output_filename("ctrf", "nat-scan-report")
|
|
1633
|
+
_write_output(_format_ctrf_report(report, suite), ctrf_path)
|
|
1634
|
+
else:
|
|
1635
|
+
text = (
|
|
1636
|
+
f"\n{'=' * 60}\n"
|
|
1637
|
+
f" NAT Scan Results: {args.spec}\n"
|
|
1638
|
+
f" Base URL: {args.base_url}\n"
|
|
1639
|
+
f"{'=' * 60}\n"
|
|
1640
|
+
f" Total: {s['total']}\n"
|
|
1641
|
+
f" Passed: {s['passed']}\n"
|
|
1642
|
+
f" Failed: {s['failed']}\n"
|
|
1643
|
+
f" Errored: {s['errored']}\n"
|
|
1644
|
+
f" Pass rate: {s['pass_rate'] * 100:.1f}%\n"
|
|
1645
|
+
f" Avg latency: {s['avg_execution_time_ms']:.1f} ms\n"
|
|
1646
|
+
f"{'=' * 60}\n"
|
|
1647
|
+
)
|
|
1648
|
+
_write_output(text, output_file)
|
|
1649
|
+
|
|
1650
|
+
return EXIT_OK if suite.failed == 0 and suite.errored == 0 else EXIT_FAILURES
|
|
1651
|
+
|
|
1652
|
+
|
|
1653
|
+
async def _run_graphql_scan(args: argparse.Namespace) -> int:
|
|
1654
|
+
"""Execute the *scan* sub-command for a GraphQL API."""
|
|
1655
|
+
from mannf.product.integrations.graphql_sut import GraphQLSUT
|
|
1656
|
+
from mannf.core.testing.models import TestSuite
|
|
1657
|
+
|
|
1658
|
+
auth = _build_auth(args)
|
|
1659
|
+
|
|
1660
|
+
# --- Regression recorder --------------------------------------------
|
|
1661
|
+
record_file = getattr(args, "record_regression", None)
|
|
1662
|
+
recorder = None
|
|
1663
|
+
if record_file:
|
|
1664
|
+
from mannf.product.regression.recorder import RegressionRecorder
|
|
1665
|
+
recorder = RegressionRecorder(
|
|
1666
|
+
spec=getattr(args, "spec", None),
|
|
1667
|
+
base_url=args.base_url,
|
|
1668
|
+
mask_sensitive_data=getattr(args, "mask_sensitive", False),
|
|
1669
|
+
)
|
|
1670
|
+
|
|
1671
|
+
sut = GraphQLSUT(
|
|
1672
|
+
endpoint_url=args.base_url,
|
|
1673
|
+
auth=auth,
|
|
1674
|
+
timeout=getattr(args, "timeout", 10.0),
|
|
1675
|
+
recorder=recorder,
|
|
1676
|
+
)
|
|
1677
|
+
|
|
1678
|
+
use_introspection = getattr(args, "graphql_introspection", False)
|
|
1679
|
+
spec = getattr(args, "spec", None)
|
|
1680
|
+
|
|
1681
|
+
if use_introspection or not spec:
|
|
1682
|
+
test_cases = await sut.generate_test_cases()
|
|
1683
|
+
else:
|
|
1684
|
+
# Spec-based test case generation (falls back to introspection for GraphQL)
|
|
1685
|
+
test_cases = await sut.generate_test_cases()
|
|
1686
|
+
|
|
1687
|
+
if not test_cases:
|
|
1688
|
+
print("No test cases generated from GraphQL schema.", file=sys.stderr)
|
|
1689
|
+
return EXIT_SCAN_ERROR
|
|
1690
|
+
|
|
1691
|
+
max_tests = getattr(args, "max_tests", None)
|
|
1692
|
+
if max_tests:
|
|
1693
|
+
test_cases = test_cases[:max_tests]
|
|
1694
|
+
|
|
1695
|
+
suite = TestSuite(name=f"NAT GraphQL scan: {args.base_url}")
|
|
1696
|
+
for tc in test_cases:
|
|
1697
|
+
suite.add_case(tc)
|
|
1698
|
+
result = await sut.run_test(tc)
|
|
1699
|
+
suite.record_result(result)
|
|
1700
|
+
logger.debug(
|
|
1701
|
+
" GraphQL %s → %s",
|
|
1702
|
+
tc.metadata.get("case_type", "?"),
|
|
1703
|
+
"PASS" if result.passed else "FAIL",
|
|
1704
|
+
)
|
|
1705
|
+
|
|
1706
|
+
# --- Save regression baseline ----------------------------------------
|
|
1707
|
+
if record_file and recorder is not None:
|
|
1708
|
+
recorder.save(record_file)
|
|
1709
|
+
logger.info("Regression baseline saved to: %s", record_file)
|
|
1710
|
+
if args.output not in ("json", "junit", "html", "allure", "ctrf"):
|
|
1711
|
+
print(f" Regression baseline saved to: {record_file}")
|
|
1712
|
+
|
|
1713
|
+
s = suite.summary()
|
|
1714
|
+
report = {
|
|
1715
|
+
"suite": s,
|
|
1716
|
+
"endpoint": args.base_url,
|
|
1717
|
+
"type": "graphql",
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
output_fmt = getattr(args, "output", "text")
|
|
1721
|
+
output_file = getattr(args, "output_file", None)
|
|
1722
|
+
|
|
1723
|
+
if output_fmt == "json":
|
|
1724
|
+
_write_output(json.dumps(report, indent=2), output_file)
|
|
1725
|
+
elif output_fmt == "html":
|
|
1726
|
+
html_path = output_file or _auto_output_filename("html", "nat-graphql-report")
|
|
1727
|
+
_write_output(_format_html_report(report, suite, args), html_path)
|
|
1728
|
+
elif output_fmt == "pdf":
|
|
1729
|
+
pdf_path = output_file or _auto_output_filename("pdf", "nat-graphql-report")
|
|
1730
|
+
await _generate_pdf_from_html(_format_html_report(report, suite, args), pdf_path)
|
|
1731
|
+
elif output_fmt == "junit":
|
|
1732
|
+
junit_path = output_file or _auto_output_filename("junit", "nat-graphql-results")
|
|
1733
|
+
_write_output(_format_junit_report(report, suite, args), junit_path)
|
|
1734
|
+
elif output_fmt == "allure":
|
|
1735
|
+
allure_dir = output_file or _auto_output_filename("allure", "nat-graphql")
|
|
1736
|
+
paths = _format_allure_report(report, suite, allure_dir)
|
|
1737
|
+
print(f" Allure results written to: {allure_dir} ({len(paths)} file(s))", file=sys.stderr)
|
|
1738
|
+
elif output_fmt == "ctrf":
|
|
1739
|
+
ctrf_path = output_file or _auto_output_filename("ctrf", "nat-graphql-report")
|
|
1740
|
+
_write_output(_format_ctrf_report(report, suite), ctrf_path)
|
|
1741
|
+
else:
|
|
1742
|
+
text = (
|
|
1743
|
+
f"\n{'=' * 60}\n"
|
|
1744
|
+
f" NAT GraphQL Scan Results\n"
|
|
1745
|
+
f" Endpoint: {args.base_url}\n"
|
|
1746
|
+
f"{'=' * 60}\n"
|
|
1747
|
+
f" Total: {s['total']}\n"
|
|
1748
|
+
f" Passed: {s['passed']}\n"
|
|
1749
|
+
f" Failed: {s['failed']}\n"
|
|
1750
|
+
f" Errored: {s['errored']}\n"
|
|
1751
|
+
f" Pass rate: {s['pass_rate'] * 100:.1f}%\n"
|
|
1752
|
+
f" Avg latency: {s['avg_execution_time_ms']:.1f} ms\n"
|
|
1753
|
+
f"{'=' * 60}\n"
|
|
1754
|
+
)
|
|
1755
|
+
_write_output(text, output_file)
|
|
1756
|
+
|
|
1757
|
+
return EXIT_OK if suite.failed == 0 and suite.errored == 0 else EXIT_FAILURES
|
|
1758
|
+
|
|
1759
|
+
|
|
1760
|
+
async def _run_grpc_scan(args: argparse.Namespace) -> int:
|
|
1761
|
+
"""Execute the *scan* sub-command for a gRPC service."""
|
|
1762
|
+
from mannf.product.integrations.grpc_sut import GrpcSUT
|
|
1763
|
+
from mannf.core.testing.models import TestSuite
|
|
1764
|
+
|
|
1765
|
+
grpc_endpoint = getattr(args, "grpc_endpoint", None) or getattr(args, "base_url", None)
|
|
1766
|
+
proto_path = getattr(args, "proto", None)
|
|
1767
|
+
use_tls = getattr(args, "grpc_use_tls", False)
|
|
1768
|
+
|
|
1769
|
+
if not grpc_endpoint:
|
|
1770
|
+
print(
|
|
1771
|
+
"--grpc-endpoint (or --base-url) is required for gRPC scans.",
|
|
1772
|
+
file=sys.stderr,
|
|
1773
|
+
)
|
|
1774
|
+
return EXIT_CONFIG_ERROR
|
|
1775
|
+
|
|
1776
|
+
sut = GrpcSUT(
|
|
1777
|
+
endpoint=grpc_endpoint,
|
|
1778
|
+
timeout=getattr(args, "timeout", 10.0),
|
|
1779
|
+
use_tls=use_tls,
|
|
1780
|
+
)
|
|
1781
|
+
|
|
1782
|
+
# Parse .proto file with GrpcIngestor if provided
|
|
1783
|
+
test_cases = []
|
|
1784
|
+
if proto_path:
|
|
1785
|
+
from mannf.product.ingestors.grpc_ingestor import GrpcIngestor
|
|
1786
|
+
from mannf.product.ingestors.models import IngestorSource
|
|
1787
|
+
|
|
1788
|
+
ingestor = GrpcIngestor()
|
|
1789
|
+
source = IngestorSource(format="grpc", path=proto_path)
|
|
1790
|
+
result = await ingestor.ingest(source, config={"grpc_endpoint": grpc_endpoint})
|
|
1791
|
+
|
|
1792
|
+
if not result.success:
|
|
1793
|
+
print(f"Proto parse error: {'; '.join(result.errors)}", file=sys.stderr)
|
|
1794
|
+
return EXIT_SCAN_ERROR
|
|
1795
|
+
|
|
1796
|
+
# Convert IngestedEndpoints to TestCase objects
|
|
1797
|
+
from mannf.core.testing.models import TestCase
|
|
1798
|
+
|
|
1799
|
+
for ep in result.endpoints:
|
|
1800
|
+
meta = ep.metadata
|
|
1801
|
+
req_body = ep.request_body or {}
|
|
1802
|
+
rpc_name = meta.get("rpc_name", meta.get("full_method", ep.path))
|
|
1803
|
+
tc = TestCase(
|
|
1804
|
+
target=meta.get("full_method", ep.path),
|
|
1805
|
+
inputs={
|
|
1806
|
+
"rpc_type": meta.get("rpc_type", "unary"),
|
|
1807
|
+
"request": req_body.get("example", {}),
|
|
1808
|
+
},
|
|
1809
|
+
expected_behavior=f"gRPC {meta.get('rpc_type', 'unary')} call to {rpc_name}",
|
|
1810
|
+
metadata={
|
|
1811
|
+
**meta,
|
|
1812
|
+
"case_type": "happy_path",
|
|
1813
|
+
"generator": "grpc_ingestor",
|
|
1814
|
+
},
|
|
1815
|
+
)
|
|
1816
|
+
test_cases.append(tc)
|
|
1817
|
+
|
|
1818
|
+
max_tests = getattr(args, "max_tests", None)
|
|
1819
|
+
if max_tests:
|
|
1820
|
+
test_cases = test_cases[:max_tests]
|
|
1821
|
+
|
|
1822
|
+
if not test_cases:
|
|
1823
|
+
print(
|
|
1824
|
+
"No test cases generated. Provide --proto to parse a .proto file.",
|
|
1825
|
+
file=sys.stderr,
|
|
1826
|
+
)
|
|
1827
|
+
return EXIT_SCAN_ERROR
|
|
1828
|
+
|
|
1829
|
+
suite = TestSuite(name=f"NAT gRPC scan: {grpc_endpoint}")
|
|
1830
|
+
for tc in test_cases:
|
|
1831
|
+
suite.add_case(tc)
|
|
1832
|
+
result = await sut.execute(tc)
|
|
1833
|
+
suite.record_result(result)
|
|
1834
|
+
logger.debug(
|
|
1835
|
+
" gRPC %s → %s",
|
|
1836
|
+
tc.target,
|
|
1837
|
+
"PASS" if result.passed else "FAIL",
|
|
1838
|
+
)
|
|
1839
|
+
|
|
1840
|
+
s = suite.summary()
|
|
1841
|
+
report = {
|
|
1842
|
+
"suite": s,
|
|
1843
|
+
"endpoint": grpc_endpoint,
|
|
1844
|
+
"type": "grpc",
|
|
1845
|
+
"proto": proto_path,
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
output_fmt = getattr(args, "output", "text")
|
|
1849
|
+
output_file = getattr(args, "output_file", None)
|
|
1850
|
+
|
|
1851
|
+
if output_fmt == "json":
|
|
1852
|
+
_write_output(json.dumps(report, indent=2), output_file)
|
|
1853
|
+
elif output_fmt == "html":
|
|
1854
|
+
html_path = output_file or _auto_output_filename("html", "nat-grpc-report")
|
|
1855
|
+
_write_output(_format_html_report(report, suite, args), html_path)
|
|
1856
|
+
elif output_fmt == "pdf":
|
|
1857
|
+
pdf_path = output_file or _auto_output_filename("pdf", "nat-grpc-report")
|
|
1858
|
+
await _generate_pdf_from_html(_format_html_report(report, suite, args), pdf_path)
|
|
1859
|
+
elif output_fmt == "junit":
|
|
1860
|
+
junit_path = output_file or _auto_output_filename("junit", "nat-grpc-results")
|
|
1861
|
+
_write_output(_format_junit_report(report, suite, args), junit_path)
|
|
1862
|
+
elif output_fmt == "allure":
|
|
1863
|
+
allure_dir = output_file or _auto_output_filename("allure", "nat-grpc")
|
|
1864
|
+
paths = _format_allure_report(report, suite, allure_dir)
|
|
1865
|
+
print(f" Allure results written to: {allure_dir} ({len(paths)} file(s))", file=sys.stderr)
|
|
1866
|
+
elif output_fmt == "ctrf":
|
|
1867
|
+
ctrf_path = output_file or _auto_output_filename("ctrf", "nat-grpc-report")
|
|
1868
|
+
_write_output(_format_ctrf_report(report, suite), ctrf_path)
|
|
1869
|
+
else:
|
|
1870
|
+
text = (
|
|
1871
|
+
f"\n{'=' * 60}\n"
|
|
1872
|
+
f" NAT gRPC Scan Results\n"
|
|
1873
|
+
f" Endpoint: {grpc_endpoint}\n"
|
|
1874
|
+
f" Proto: {proto_path or '(none)'}\n"
|
|
1875
|
+
f"{'=' * 60}\n"
|
|
1876
|
+
f" Total: {s['total']}\n"
|
|
1877
|
+
f" Passed: {s['passed']}\n"
|
|
1878
|
+
f" Failed: {s['failed']}\n"
|
|
1879
|
+
f" Errored: {s['errored']}\n"
|
|
1880
|
+
f" Pass rate: {s['pass_rate'] * 100:.1f}%\n"
|
|
1881
|
+
f" Avg latency: {s['avg_execution_time_ms']:.1f} ms\n"
|
|
1882
|
+
f"{'=' * 60}\n"
|
|
1883
|
+
)
|
|
1884
|
+
_write_output(text, output_file)
|
|
1885
|
+
|
|
1886
|
+
return EXIT_OK if suite.failed == 0 and suite.errored == 0 else EXIT_FAILURES
|
|
1887
|
+
|
|
1888
|
+
|
|
1889
|
+
async def _run_websocket_scan(args: argparse.Namespace) -> int:
|
|
1890
|
+
"""Execute the *scan* sub-command for a WebSocket API."""
|
|
1891
|
+
from mannf.product.integrations.websocket_sut import WebSocketSUT
|
|
1892
|
+
from mannf.core.testing.models import TestSuite
|
|
1893
|
+
|
|
1894
|
+
ws_endpoint = getattr(args, "ws_endpoint", None) or getattr(args, "base_url", None)
|
|
1895
|
+
asyncapi_path = getattr(args, "asyncapi", None) or getattr(args, "spec", None)
|
|
1896
|
+
|
|
1897
|
+
if not ws_endpoint:
|
|
1898
|
+
print(
|
|
1899
|
+
"--ws-endpoint (or --base-url) is required for WebSocket scans.",
|
|
1900
|
+
file=sys.stderr,
|
|
1901
|
+
)
|
|
1902
|
+
return EXIT_CONFIG_ERROR
|
|
1903
|
+
|
|
1904
|
+
sut = WebSocketSUT(
|
|
1905
|
+
ws_url=ws_endpoint,
|
|
1906
|
+
timeout=getattr(args, "timeout", 10.0),
|
|
1907
|
+
)
|
|
1908
|
+
|
|
1909
|
+
# Parse AsyncAPI spec with WebSocketIngestor if provided
|
|
1910
|
+
test_cases = []
|
|
1911
|
+
if asyncapi_path:
|
|
1912
|
+
from mannf.product.ingestors.websocket_ingestor import WebSocketIngestor
|
|
1913
|
+
from mannf.product.ingestors.models import IngestorSource
|
|
1914
|
+
|
|
1915
|
+
ingestor = WebSocketIngestor()
|
|
1916
|
+
source = IngestorSource(format="websocket", path=asyncapi_path)
|
|
1917
|
+
ingest_result = await ingestor.ingest(
|
|
1918
|
+
source, config={"ws_endpoint": ws_endpoint}
|
|
1919
|
+
)
|
|
1920
|
+
|
|
1921
|
+
if not ingest_result.success:
|
|
1922
|
+
print(
|
|
1923
|
+
f"AsyncAPI parse error: {'; '.join(ingest_result.errors)}",
|
|
1924
|
+
file=sys.stderr,
|
|
1925
|
+
)
|
|
1926
|
+
return EXIT_SCAN_ERROR
|
|
1927
|
+
|
|
1928
|
+
from mannf.core.testing.models import TestCase
|
|
1929
|
+
|
|
1930
|
+
for ep in ingest_result.endpoints:
|
|
1931
|
+
meta = ep.metadata
|
|
1932
|
+
req_body = ep.request_body or {}
|
|
1933
|
+
tc = TestCase(
|
|
1934
|
+
target=meta.get("channel", ep.path),
|
|
1935
|
+
inputs={
|
|
1936
|
+
"action": "send_receive" if meta.get("operation") == "publish" else "receive_only",
|
|
1937
|
+
"payload": req_body.get("example", {}),
|
|
1938
|
+
},
|
|
1939
|
+
metadata={
|
|
1940
|
+
**meta,
|
|
1941
|
+
"case_type": "happy_path",
|
|
1942
|
+
"generator": "websocket_ingestor",
|
|
1943
|
+
},
|
|
1944
|
+
)
|
|
1945
|
+
test_cases.append(tc)
|
|
1946
|
+
|
|
1947
|
+
max_tests = getattr(args, "max_tests", None)
|
|
1948
|
+
if max_tests:
|
|
1949
|
+
test_cases = test_cases[:max_tests]
|
|
1950
|
+
|
|
1951
|
+
if not test_cases:
|
|
1952
|
+
print(
|
|
1953
|
+
"No test cases generated. Provide --asyncapi to parse an AsyncAPI spec.",
|
|
1954
|
+
file=sys.stderr,
|
|
1955
|
+
)
|
|
1956
|
+
return EXIT_SCAN_ERROR
|
|
1957
|
+
|
|
1958
|
+
suite = TestSuite(name=f"NAT WebSocket scan: {ws_endpoint}")
|
|
1959
|
+
for tc in test_cases:
|
|
1960
|
+
suite.add_case(tc)
|
|
1961
|
+
result = await sut.execute(tc)
|
|
1962
|
+
suite.record_result(result)
|
|
1963
|
+
logger.debug(
|
|
1964
|
+
" WS %s → %s",
|
|
1965
|
+
tc.target,
|
|
1966
|
+
"PASS" if result.passed else "FAIL",
|
|
1967
|
+
)
|
|
1968
|
+
|
|
1969
|
+
s = suite.summary()
|
|
1970
|
+
report = {
|
|
1971
|
+
"suite": s,
|
|
1972
|
+
"endpoint": ws_endpoint,
|
|
1973
|
+
"type": "websocket",
|
|
1974
|
+
"asyncapi": asyncapi_path,
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
output_fmt = getattr(args, "output", "text")
|
|
1978
|
+
output_file = getattr(args, "output_file", None)
|
|
1979
|
+
|
|
1980
|
+
if output_fmt == "json":
|
|
1981
|
+
_write_output(json.dumps(report, indent=2), output_file)
|
|
1982
|
+
elif output_fmt == "html":
|
|
1983
|
+
html_path = output_file or _auto_output_filename("html", "nat-ws-report")
|
|
1984
|
+
_write_output(_format_html_report(report, suite, args), html_path)
|
|
1985
|
+
elif output_fmt == "pdf":
|
|
1986
|
+
pdf_path = output_file or _auto_output_filename("pdf", "nat-ws-report")
|
|
1987
|
+
await _generate_pdf_from_html(_format_html_report(report, suite, args), pdf_path)
|
|
1988
|
+
elif output_fmt == "junit":
|
|
1989
|
+
junit_path = output_file or _auto_output_filename("junit", "nat-ws-results")
|
|
1990
|
+
_write_output(_format_junit_report(report, suite, args), junit_path)
|
|
1991
|
+
elif output_fmt == "allure":
|
|
1992
|
+
allure_dir = output_file or _auto_output_filename("allure", "nat-ws")
|
|
1993
|
+
paths = _format_allure_report(report, suite, allure_dir)
|
|
1994
|
+
print(f" Allure results written to: {allure_dir} ({len(paths)} file(s))", file=sys.stderr)
|
|
1995
|
+
elif output_fmt == "ctrf":
|
|
1996
|
+
ctrf_path = output_file or _auto_output_filename("ctrf", "nat-ws-report")
|
|
1997
|
+
_write_output(_format_ctrf_report(report, suite), ctrf_path)
|
|
1998
|
+
else:
|
|
1999
|
+
text = (
|
|
2000
|
+
f"\n{'=' * 60}\n"
|
|
2001
|
+
f" NAT WebSocket Scan Results\n"
|
|
2002
|
+
f" Endpoint: {ws_endpoint}\n"
|
|
2003
|
+
f" AsyncAPI: {asyncapi_path or '(none)'}\n"
|
|
2004
|
+
f"{'=' * 60}\n"
|
|
2005
|
+
f" Total: {s['total']}\n"
|
|
2006
|
+
f" Passed: {s['passed']}\n"
|
|
2007
|
+
f" Failed: {s['failed']}\n"
|
|
2008
|
+
f" Errored: {s['errored']}\n"
|
|
2009
|
+
f" Pass rate: {s['pass_rate'] * 100:.1f}%\n"
|
|
2010
|
+
f" Avg latency: {s['avg_execution_time_ms']:.1f} ms\n"
|
|
2011
|
+
f"{'=' * 60}\n"
|
|
2012
|
+
)
|
|
2013
|
+
_write_output(text, output_file)
|
|
2014
|
+
|
|
2015
|
+
return EXIT_OK if suite.failed == 0 and suite.errored == 0 else EXIT_FAILURES
|
|
2016
|
+
|
|
2017
|
+
|
|
2018
|
+
async def _run_regression_replay(args: argparse.Namespace) -> int:
|
|
2019
|
+
"""Execute a regression replay against a previously recorded baseline."""
|
|
2020
|
+
from mannf.product.regression.models import RegressionBaseline
|
|
2021
|
+
from mannf.product.regression.replayer import RegressionReplayer
|
|
2022
|
+
from mannf.product.regression.differ import RegressionDiffer
|
|
2023
|
+
|
|
2024
|
+
baseline_file = args.replay
|
|
2025
|
+
try:
|
|
2026
|
+
baseline = RegressionBaseline.load(baseline_file)
|
|
2027
|
+
except (OSError, ValueError, KeyError) as exc:
|
|
2028
|
+
print(f"Failed to load baseline file {baseline_file!r}: {exc}", file=sys.stderr)
|
|
2029
|
+
return EXIT_SCAN_ERROR
|
|
2030
|
+
|
|
2031
|
+
if not baseline.records:
|
|
2032
|
+
print("Baseline file contains no records.", file=sys.stderr)
|
|
2033
|
+
return EXIT_SCAN_ERROR
|
|
2034
|
+
|
|
2035
|
+
auth = _build_auth(args)
|
|
2036
|
+
mask = getattr(args, "mask_sensitive", False)
|
|
2037
|
+
|
|
2038
|
+
replayer = RegressionReplayer(
|
|
2039
|
+
baseline=baseline,
|
|
2040
|
+
auth=auth,
|
|
2041
|
+
timeout=getattr(args, "timeout", 10.0),
|
|
2042
|
+
mask_sensitive_data=mask,
|
|
2043
|
+
)
|
|
2044
|
+
|
|
2045
|
+
replay_baseline = await replayer.replay()
|
|
2046
|
+
|
|
2047
|
+
differ = RegressionDiffer()
|
|
2048
|
+
diff = differ.diff(baseline, replay_baseline)
|
|
2049
|
+
|
|
2050
|
+
diff_only = getattr(args, "regression_diff_only", False)
|
|
2051
|
+
|
|
2052
|
+
if args.output == "json":
|
|
2053
|
+
diff_dict = diff.to_dict()
|
|
2054
|
+
if diff_only:
|
|
2055
|
+
# Only include entries with actual changes
|
|
2056
|
+
diff_dict["changes"] = [c for c in diff_dict["changes"] if c["changes"]]
|
|
2057
|
+
print(json.dumps(diff_dict, indent=2))
|
|
2058
|
+
else:
|
|
2059
|
+
print(diff.human_readable())
|
|
2060
|
+
if diff_only and not diff.changes:
|
|
2061
|
+
print(" (no differences found)")
|
|
2062
|
+
|
|
2063
|
+
# Exit 1 if there are any regressions
|
|
2064
|
+
return EXIT_FAILURES if diff.changes else EXIT_OK
|
|
2065
|
+
|
|
2066
|
+
|
|
2067
|
+
async def _run_security_scan(args: argparse.Namespace) -> int:
|
|
2068
|
+
"""Execute the security-scan sub-command."""
|
|
2069
|
+
from mannf.product.integrations.http_sut import HttpApiSUT
|
|
2070
|
+
from mannf.product.integrations.openapi_parser import OpenApiParser
|
|
2071
|
+
from mannf.product.security.scanner import SecurityScanner
|
|
2072
|
+
from mannf.product.security.checks.base import EndpointInfo
|
|
2073
|
+
from mannf.product.security.reporter import SecurityReporter
|
|
2074
|
+
|
|
2075
|
+
api_type = getattr(args, "type", "rest")
|
|
2076
|
+
|
|
2077
|
+
if api_type == "graphql":
|
|
2078
|
+
# For GraphQL, the "endpoint" is the base URL itself
|
|
2079
|
+
endpoints = [
|
|
2080
|
+
EndpointInfo(
|
|
2081
|
+
method="POST",
|
|
2082
|
+
path="",
|
|
2083
|
+
base_url=args.base_url,
|
|
2084
|
+
)
|
|
2085
|
+
]
|
|
2086
|
+
else:
|
|
2087
|
+
if not getattr(args, "spec", None):
|
|
2088
|
+
print("--spec is required for REST security scans.", file=sys.stderr)
|
|
2089
|
+
return EXIT_CONFIG_ERROR
|
|
2090
|
+
|
|
2091
|
+
sut = HttpApiSUT(base_url=args.base_url)
|
|
2092
|
+
spec_format = getattr(args, "format", "openapi")
|
|
2093
|
+
if spec_format == "postman":
|
|
2094
|
+
from mannf.product.integrations.postman_parser import PostmanParser
|
|
2095
|
+
parser = PostmanParser(
|
|
2096
|
+
collection_path=args.spec,
|
|
2097
|
+
base_url=args.base_url,
|
|
2098
|
+
registry=sut.registry,
|
|
2099
|
+
)
|
|
2100
|
+
else:
|
|
2101
|
+
parser = OpenApiParser(spec_path=args.spec, base_url=args.base_url, registry=sut.registry)
|
|
2102
|
+
test_cases = parser.parse()
|
|
2103
|
+
|
|
2104
|
+
endpoints = []
|
|
2105
|
+
seen: set = set()
|
|
2106
|
+
for tc in test_cases:
|
|
2107
|
+
method = tc.inputs.get("method", "GET").upper()
|
|
2108
|
+
path = tc.target
|
|
2109
|
+
key = f"{method} {path}"
|
|
2110
|
+
if key not in seen:
|
|
2111
|
+
seen.add(key)
|
|
2112
|
+
endpoints.append(EndpointInfo(
|
|
2113
|
+
method=method,
|
|
2114
|
+
path=path,
|
|
2115
|
+
base_url=args.base_url,
|
|
2116
|
+
parameters=tc.inputs.get("parameters", []),
|
|
2117
|
+
request_body=tc.inputs.get("body"),
|
|
2118
|
+
))
|
|
2119
|
+
|
|
2120
|
+
if not endpoints:
|
|
2121
|
+
print("No endpoints discovered from spec.", file=sys.stderr)
|
|
2122
|
+
return EXIT_SCAN_ERROR
|
|
2123
|
+
|
|
2124
|
+
checks = [c.strip() for c in args.checks.split(",")] if args.checks else None
|
|
2125
|
+
|
|
2126
|
+
# Resolve prioritization mode: --prioritization takes precedence over legacy --belief-guided
|
|
2127
|
+
prioritization_mode = getattr(args, "prioritization", "adaptive")
|
|
2128
|
+
belief_guided = prioritization_mode != "off"
|
|
2129
|
+
|
|
2130
|
+
# Collect extra check plugins from --plugins-dir and --plugin flags
|
|
2131
|
+
extra_checks: list = []
|
|
2132
|
+
plugins_dir = getattr(args, "plugins_dir", None)
|
|
2133
|
+
if plugins_dir:
|
|
2134
|
+
from mannf.product.security.plugin_loader import load_plugins_from_directory
|
|
2135
|
+
extra_checks.extend(load_plugins_from_directory(plugins_dir))
|
|
2136
|
+
|
|
2137
|
+
plugin_classes = getattr(args, "plugins", None) or []
|
|
2138
|
+
for fqn in plugin_classes:
|
|
2139
|
+
try:
|
|
2140
|
+
module_name, class_name = fqn.rsplit(".", 1)
|
|
2141
|
+
except ValueError:
|
|
2142
|
+
print(f"Invalid --plugin value '{fqn}': expected 'module.ClassName'.", file=sys.stderr)
|
|
2143
|
+
continue
|
|
2144
|
+
try:
|
|
2145
|
+
mod = importlib.import_module(module_name)
|
|
2146
|
+
cls = getattr(mod, class_name)
|
|
2147
|
+
instance = cls()
|
|
2148
|
+
extra_checks.append(instance)
|
|
2149
|
+
except Exception as exc: # noqa: BLE001
|
|
2150
|
+
print(
|
|
2151
|
+
f"Failed to load plugin '{fqn}': {exc} "
|
|
2152
|
+
"— ensure the module is installed and the class name is correct.",
|
|
2153
|
+
file=sys.stderr,
|
|
2154
|
+
)
|
|
2155
|
+
|
|
2156
|
+
scanner = SecurityScanner(
|
|
2157
|
+
base_url=args.base_url,
|
|
2158
|
+
enabled_checks=checks,
|
|
2159
|
+
severity_threshold=args.severity_threshold,
|
|
2160
|
+
belief_guided=belief_guided,
|
|
2161
|
+
adaptive_checks=(prioritization_mode == "adaptive"),
|
|
2162
|
+
extra_checks=extra_checks or None,
|
|
2163
|
+
)
|
|
2164
|
+
|
|
2165
|
+
# For 'adaptive' mode, attempt to load belief states from a prior scan if available
|
|
2166
|
+
belief_states = None
|
|
2167
|
+
risk_scorer = None
|
|
2168
|
+
if prioritization_mode == "adaptive":
|
|
2169
|
+
try:
|
|
2170
|
+
from mannf.core.prioritization.risk_scorer import RiskScorer
|
|
2171
|
+
risk_scorer = RiskScorer()
|
|
2172
|
+
except Exception:
|
|
2173
|
+
pass
|
|
2174
|
+
|
|
2175
|
+
report = await scanner.scan(endpoints, belief_states=belief_states, risk_scorer=risk_scorer)
|
|
2176
|
+
|
|
2177
|
+
reporter = SecurityReporter()
|
|
2178
|
+
output_fmt = getattr(args, "output", "text")
|
|
2179
|
+
output_file = getattr(args, "output_file", None)
|
|
2180
|
+
|
|
2181
|
+
if output_fmt == "json":
|
|
2182
|
+
_write_output(reporter.to_json(report), output_file)
|
|
2183
|
+
elif output_fmt == "markdown":
|
|
2184
|
+
_write_output(reporter.to_markdown(report), output_file)
|
|
2185
|
+
else:
|
|
2186
|
+
lines = [
|
|
2187
|
+
f"\n{'=' * 60}",
|
|
2188
|
+
f" NAT Security Scan Results",
|
|
2189
|
+
f" Target: {args.base_url}",
|
|
2190
|
+
f"{'=' * 60}",
|
|
2191
|
+
]
|
|
2192
|
+
total = report.total_endpoints_scanned
|
|
2193
|
+
lines.append(f" Endpoints scanned: {total}")
|
|
2194
|
+
lines.append(f" Findings: {len(report.findings)}")
|
|
2195
|
+
sev_counts = report.summary
|
|
2196
|
+
for sev in ["critical", "high", "medium", "low", "info"]:
|
|
2197
|
+
count = sev_counts.get(sev, 0)
|
|
2198
|
+
if count > 0:
|
|
2199
|
+
lines.append(f" {_SEVERITY_ICONS.get(sev, ' ')} {sev.capitalize()}: {count}")
|
|
2200
|
+
lines.append(f"{'=' * 60}\n")
|
|
2201
|
+
for finding in report.findings:
|
|
2202
|
+
icon = _SEVERITY_ICONS.get(finding.severity, "•")
|
|
2203
|
+
lines.append(f"{icon} [{finding.check_id}] {finding.title}")
|
|
2204
|
+
lines.append(f" Endpoint: {finding.endpoint}")
|
|
2205
|
+
lines.append(f" {finding.description}")
|
|
2206
|
+
lines.append(f" Fix: {finding.remediation}")
|
|
2207
|
+
lines.append("")
|
|
2208
|
+
_write_output("\n".join(lines), output_file)
|
|
2209
|
+
|
|
2210
|
+
high_severity = [f for f in report.findings if f.severity in ("critical", "high")]
|
|
2211
|
+
|
|
2212
|
+
# ------------------------------------------------------------------
|
|
2213
|
+
# Exporter integration
|
|
2214
|
+
# ------------------------------------------------------------------
|
|
2215
|
+
export_name = getattr(args, "export", None)
|
|
2216
|
+
if export_name:
|
|
2217
|
+
from mannf.product.exporters.loader import (
|
|
2218
|
+
get_exporter_by_name,
|
|
2219
|
+
load_all_exporters,
|
|
2220
|
+
)
|
|
2221
|
+
|
|
2222
|
+
exporters_dir = getattr(args, "exporters_dir", None)
|
|
2223
|
+
exporters = load_all_exporters(exporters_dir=exporters_dir)
|
|
2224
|
+
exporter = get_exporter_by_name(export_name, exporters)
|
|
2225
|
+
|
|
2226
|
+
if exporter is None:
|
|
2227
|
+
print(
|
|
2228
|
+
f"Exporter '{export_name}' not found. "
|
|
2229
|
+
f"Available: {[e.name for e in exporters] or 'none'}",
|
|
2230
|
+
file=sys.stderr,
|
|
2231
|
+
)
|
|
2232
|
+
return EXIT_CONFIG_ERROR
|
|
2233
|
+
|
|
2234
|
+
# Parse --export-config key=value pairs
|
|
2235
|
+
export_config: dict[str, str] = {}
|
|
2236
|
+
for pair in (getattr(args, "export_config", None) or []):
|
|
2237
|
+
if "=" not in pair:
|
|
2238
|
+
print(
|
|
2239
|
+
f"Invalid --export-config value '{pair}': expected KEY=VALUE.",
|
|
2240
|
+
file=sys.stderr,
|
|
2241
|
+
)
|
|
2242
|
+
return EXIT_CONFIG_ERROR
|
|
2243
|
+
key, _, value = pair.partition("=")
|
|
2244
|
+
export_config[key.strip()] = value.strip()
|
|
2245
|
+
|
|
2246
|
+
config_errors = exporter.validate_config(export_config)
|
|
2247
|
+
if config_errors:
|
|
2248
|
+
print("Exporter config errors:", file=sys.stderr)
|
|
2249
|
+
for err in config_errors:
|
|
2250
|
+
print(f" - {err}", file=sys.stderr)
|
|
2251
|
+
return EXIT_CONFIG_ERROR
|
|
2252
|
+
|
|
2253
|
+
min_severity = getattr(args, "export_min_severity", "low")
|
|
2254
|
+
summary = await exporter.export_findings(
|
|
2255
|
+
report.findings, report, export_config, min_severity
|
|
2256
|
+
)
|
|
2257
|
+
|
|
2258
|
+
print(
|
|
2259
|
+
f"\nExport summary ({exporter.display_name}): "
|
|
2260
|
+
f"{summary.succeeded} exported, "
|
|
2261
|
+
f"{summary.failed} failed, "
|
|
2262
|
+
f"{summary.skipped} skipped."
|
|
2263
|
+
)
|
|
2264
|
+
if summary.failed:
|
|
2265
|
+
for result in summary.results:
|
|
2266
|
+
if not result.success:
|
|
2267
|
+
print(
|
|
2268
|
+
f" ✗ [{result.finding.check_id}] {result.finding.title}: {result.error}",
|
|
2269
|
+
file=sys.stderr,
|
|
2270
|
+
)
|
|
2271
|
+
|
|
2272
|
+
return EXIT_FAILURES if high_severity else EXIT_OK
|
|
2273
|
+
|
|
2274
|
+
|
|
2275
|
+
def _run_weights(args: argparse.Namespace) -> int:
|
|
2276
|
+
"""Execute the *weights* sub-command group."""
|
|
2277
|
+
from mannf.product.weights.registry import WeightRegistry
|
|
2278
|
+
from mannf.product.weights.store import WeightStore
|
|
2279
|
+
|
|
2280
|
+
registry = WeightRegistry()
|
|
2281
|
+
sub = args.weights_command
|
|
2282
|
+
|
|
2283
|
+
if sub == "list":
|
|
2284
|
+
snapshots = registry.list()
|
|
2285
|
+
if not snapshots:
|
|
2286
|
+
print("No weight snapshots registered.")
|
|
2287
|
+
return 0
|
|
2288
|
+
print(f"{'NAME':<30} {'SAVED AT':<28} PATH")
|
|
2289
|
+
print("-" * 90)
|
|
2290
|
+
for snap in snapshots:
|
|
2291
|
+
print(f"{snap['name']:<30} {snap['registered_at']:<28} {snap['path']}")
|
|
2292
|
+
return 0
|
|
2293
|
+
|
|
2294
|
+
if sub == "save":
|
|
2295
|
+
output_path = args.output
|
|
2296
|
+
name = args.name if args.name is not None else args.scan_id
|
|
2297
|
+
# weights save is a placeholder for CLI-driven save; the actual agent
|
|
2298
|
+
# weight extraction happens through the server or --save-weights flag.
|
|
2299
|
+
# Here we just register the path if the file already exists.
|
|
2300
|
+
if not os.path.isfile(output_path):
|
|
2301
|
+
print(
|
|
2302
|
+
f"Weight file {output_path!r} not found. "
|
|
2303
|
+
"Use --save-weights during a scan to produce it.",
|
|
2304
|
+
file=sys.stderr,
|
|
2305
|
+
)
|
|
2306
|
+
return 1
|
|
2307
|
+
registry.register(name, output_path, metadata={"scan_id": args.scan_id})
|
|
2308
|
+
print(f"Registered weight snapshot {name!r} → {output_path}")
|
|
2309
|
+
return 0
|
|
2310
|
+
|
|
2311
|
+
if sub == "info":
|
|
2312
|
+
path = args.path
|
|
2313
|
+
# Allow a registered name as well as a raw path
|
|
2314
|
+
if not os.path.isfile(path):
|
|
2315
|
+
try:
|
|
2316
|
+
path = registry.get(path)
|
|
2317
|
+
except KeyError:
|
|
2318
|
+
print(f"No weight file or registered snapshot found for {args.path!r}", file=sys.stderr)
|
|
2319
|
+
return 1
|
|
2320
|
+
try:
|
|
2321
|
+
data = WeightStore.load(path)
|
|
2322
|
+
except (FileNotFoundError, ValueError) as exc:
|
|
2323
|
+
print(f"Failed to read weight file: {exc}", file=sys.stderr)
|
|
2324
|
+
return 1
|
|
2325
|
+
|
|
2326
|
+
print(f"Weight file: {path}")
|
|
2327
|
+
print(f" Version: {data['version']}")
|
|
2328
|
+
print(f" Saved at: {data['saved_at']}")
|
|
2329
|
+
print(f" Agents: {len(data['agents'])}")
|
|
2330
|
+
for agent_id, info in data["agents"].items():
|
|
2331
|
+
layers = info["layers"]
|
|
2332
|
+
print(f" [{agent_id}] role={info['role']} layers={len(layers)}", end="")
|
|
2333
|
+
if layers:
|
|
2334
|
+
shapes = " → ".join(
|
|
2335
|
+
f"{l['W'].shape[0]}×{l['W'].shape[1]}" for l in layers
|
|
2336
|
+
)
|
|
2337
|
+
print(f" shapes={shapes}", end="")
|
|
2338
|
+
print()
|
|
2339
|
+
return 0
|
|
2340
|
+
|
|
2341
|
+
print(f"Unknown weights sub-command: {sub!r}", file=sys.stderr)
|
|
2342
|
+
return 1
|
|
2343
|
+
|
|
2344
|
+
|
|
2345
|
+
def _run_export(args: argparse.Namespace) -> int:
|
|
2346
|
+
"""Execute the *export* sub-command group."""
|
|
2347
|
+
import json as _json # noqa: PLC0415
|
|
2348
|
+
|
|
2349
|
+
from mannf.product.exporters import BUILTIN_EXPORTERS, load_all_exporters # noqa: PLC0415
|
|
2350
|
+
|
|
2351
|
+
sub = args.export_command
|
|
2352
|
+
|
|
2353
|
+
if sub == "list":
|
|
2354
|
+
exporters = load_all_exporters()
|
|
2355
|
+
# Merge built-ins with any discovered extras, deduplicating by name.
|
|
2356
|
+
all_names = {e.name for e in exporters}
|
|
2357
|
+
merged = list(exporters)
|
|
2358
|
+
for builtin in BUILTIN_EXPORTERS:
|
|
2359
|
+
if builtin.name not in all_names:
|
|
2360
|
+
merged.append(builtin)
|
|
2361
|
+
all_names.add(builtin.name)
|
|
2362
|
+
merged.sort(key=lambda e: e.name)
|
|
2363
|
+
|
|
2364
|
+
if not merged:
|
|
2365
|
+
if getattr(args, "json_output", False):
|
|
2366
|
+
print(_json.dumps([]))
|
|
2367
|
+
else:
|
|
2368
|
+
print("No exporter plugins found.")
|
|
2369
|
+
return 0
|
|
2370
|
+
|
|
2371
|
+
if getattr(args, "json_output", False):
|
|
2372
|
+
data = [
|
|
2373
|
+
{"name": e.name, "display_name": e.display_name, "description": e.description}
|
|
2374
|
+
for e in merged
|
|
2375
|
+
]
|
|
2376
|
+
print(_json.dumps(data, indent=2))
|
|
2377
|
+
else:
|
|
2378
|
+
print(f"Available exporters ({len(merged)} total):")
|
|
2379
|
+
print(f" {'NAME':<20} {'DISPLAY NAME':<24} DESCRIPTION")
|
|
2380
|
+
print(" " + "-" * 80)
|
|
2381
|
+
for exp in merged:
|
|
2382
|
+
print(f" {exp.name:<20} {exp.display_name:<24} {exp.description}")
|
|
2383
|
+
return 0
|
|
2384
|
+
|
|
2385
|
+
print(f"Unknown export sub-command: {sub!r}", file=sys.stderr)
|
|
2386
|
+
return 1
|
|
2387
|
+
|
|
2388
|
+
|
|
2389
|
+
def _parse_queue_backend(queue_arg: str) -> "Any":
|
|
2390
|
+
"""Parse the ``--queue`` CLI argument and return a configured :class:`JobQueue`."""
|
|
2391
|
+
from mannf.product.scheduling.queue import create_job_queue # noqa: PLC0415
|
|
2392
|
+
|
|
2393
|
+
q = queue_arg.strip()
|
|
2394
|
+
if q == "memory" or not q:
|
|
2395
|
+
return create_job_queue("memory")
|
|
2396
|
+
if q.startswith("sqlite://"):
|
|
2397
|
+
db_path = q[len("sqlite://"):]
|
|
2398
|
+
return create_job_queue("sqlite", db_path=db_path)
|
|
2399
|
+
if q.startswith("redis://"):
|
|
2400
|
+
return create_job_queue("redis", redis_url=q)
|
|
2401
|
+
# Plain path — treat as SQLite
|
|
2402
|
+
return create_job_queue("sqlite", db_path=q)
|
|
2403
|
+
|
|
2404
|
+
|
|
2405
|
+
def _run_worker(args: argparse.Namespace) -> int:
|
|
2406
|
+
"""Execute the *worker* sub-command group."""
|
|
2407
|
+
sub = args.worker_command
|
|
2408
|
+
|
|
2409
|
+
if sub == "launch":
|
|
2410
|
+
return asyncio.run(_run_worker_launch(args))
|
|
2411
|
+
|
|
2412
|
+
if sub == "status":
|
|
2413
|
+
return asyncio.run(_run_worker_status(args))
|
|
2414
|
+
|
|
2415
|
+
print(f"Unknown worker sub-command: {sub!r}", file=sys.stderr)
|
|
2416
|
+
return 1
|
|
2417
|
+
|
|
2418
|
+
|
|
2419
|
+
async def _run_worker_launch(args: argparse.Namespace) -> int:
|
|
2420
|
+
"""Launch a local worker pool and block until interrupted."""
|
|
2421
|
+
from mannf.core.agents.worker_pool import WorkerPool # noqa: PLC0415
|
|
2422
|
+
|
|
2423
|
+
queue = _parse_queue_backend(args.queue)
|
|
2424
|
+
pool = WorkerPool(
|
|
2425
|
+
queue=queue,
|
|
2426
|
+
worker_count=args.count,
|
|
2427
|
+
poll_interval=args.poll_interval,
|
|
2428
|
+
max_attempts=args.max_attempts,
|
|
2429
|
+
)
|
|
2430
|
+
await pool.start()
|
|
2431
|
+
|
|
2432
|
+
print(
|
|
2433
|
+
f"NAT worker pool started — {args.count} worker(s), queue={args.queue!r}"
|
|
2434
|
+
)
|
|
2435
|
+
print("Press Ctrl+C to stop.")
|
|
2436
|
+
|
|
2437
|
+
try:
|
|
2438
|
+
while True:
|
|
2439
|
+
await asyncio.sleep(5)
|
|
2440
|
+
info = await pool.full_status()
|
|
2441
|
+
active = sum(1 for w in info["workers"] if w["status"] == "running")
|
|
2442
|
+
idle = sum(1 for w in info["workers"] if w["status"] == "idle")
|
|
2443
|
+
q = info["queue"]
|
|
2444
|
+
print(
|
|
2445
|
+
f" workers: {active} active / {idle} idle | "
|
|
2446
|
+
f"queue: {q['pending']} pending / {q['running']} running / "
|
|
2447
|
+
f"{q['completed']} completed / {q['failed']} failed"
|
|
2448
|
+
)
|
|
2449
|
+
except (KeyboardInterrupt, asyncio.CancelledError):
|
|
2450
|
+
print("\nShutting down worker pool…")
|
|
2451
|
+
await pool.stop()
|
|
2452
|
+
print("Worker pool stopped.")
|
|
2453
|
+
return 0
|
|
2454
|
+
|
|
2455
|
+
|
|
2456
|
+
async def _run_worker_status(args: argparse.Namespace) -> int:
|
|
2457
|
+
"""Show worker pool and queue status."""
|
|
2458
|
+
queue = _parse_queue_backend(args.queue)
|
|
2459
|
+
stats = await queue.status()
|
|
2460
|
+
|
|
2461
|
+
if args.output == "json":
|
|
2462
|
+
print(json.dumps(stats.to_dict(), indent=2))
|
|
2463
|
+
else:
|
|
2464
|
+
print("Queue Status")
|
|
2465
|
+
print("------------")
|
|
2466
|
+
print(f" Pending: {stats.pending}")
|
|
2467
|
+
print(f" Running: {stats.running}")
|
|
2468
|
+
print(f" Completed: {stats.completed}")
|
|
2469
|
+
print(f" Failed: {stats.failed}")
|
|
2470
|
+
print(f" Retrying: {stats.retrying}")
|
|
2471
|
+
print(f" Total: {stats.total}")
|
|
2472
|
+
return 0
|
|
2473
|
+
|
|
2474
|
+
|
|
2475
|
+
def _run_heal(args: argparse.Namespace) -> int:
|
|
2476
|
+
"""Execute the *heal* sub-command (schema drift detection + dry-run report)."""
|
|
2477
|
+
from mannf.product.healing.healer import TestHealer
|
|
2478
|
+
|
|
2479
|
+
healer = TestHealer()
|
|
2480
|
+
try:
|
|
2481
|
+
result = healer.heal(args.old_spec, args.new_spec, [])
|
|
2482
|
+
except (FileNotFoundError, ValueError, OSError) as exc:
|
|
2483
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
2484
|
+
return EXIT_SCAN_ERROR
|
|
2485
|
+
|
|
2486
|
+
report = result.healing_report
|
|
2487
|
+
|
|
2488
|
+
if args.output == "json":
|
|
2489
|
+
print(json.dumps(report.to_dict(), indent=2))
|
|
2490
|
+
elif args.output == "markdown":
|
|
2491
|
+
print(report.markdown())
|
|
2492
|
+
else:
|
|
2493
|
+
print(report.human_readable())
|
|
2494
|
+
|
|
2495
|
+
if getattr(args, "healing_report", None):
|
|
2496
|
+
report.save(args.healing_report)
|
|
2497
|
+
print(f"\n Healing report saved to: {args.healing_report}", file=sys.stderr)
|
|
2498
|
+
|
|
2499
|
+
return 1 if report.schema_diff and report.schema_diff.has_changes else 0
|
|
2500
|
+
|
|
2501
|
+
|
|
2502
|
+
# ---------------------------------------------------------------------------
|
|
2503
|
+
# Shell completions
|
|
2504
|
+
# ---------------------------------------------------------------------------
|
|
2505
|
+
|
|
2506
|
+
_BASH_COMPLETION = """\
|
|
2507
|
+
# bash completion for nat
|
|
2508
|
+
# Install: source <(nat completions bash)
|
|
2509
|
+
|
|
2510
|
+
_nat_completions() {
|
|
2511
|
+
local cur prev words cword
|
|
2512
|
+
_init_completion || return
|
|
2513
|
+
|
|
2514
|
+
local commands="scan security-scan weights heal test-gen completions demo setup upgrade uninstall doctor"
|
|
2515
|
+
local global_opts="--config --help"
|
|
2516
|
+
|
|
2517
|
+
if [[ ${cword} -eq 1 ]]; then
|
|
2518
|
+
COMPREPLY=( $(compgen -W "${commands} ${global_opts}" -- "${cur}") )
|
|
2519
|
+
return
|
|
2520
|
+
fi
|
|
2521
|
+
|
|
2522
|
+
local cmd="${words[1]}"
|
|
2523
|
+
case "${cmd}" in
|
|
2524
|
+
scan)
|
|
2525
|
+
local opts="--spec --base-url --type --graphql-introspection --auth-token --api-key
|
|
2526
|
+
--api-key-header --max-tests --executors --timeout --latency-sla
|
|
2527
|
+
--output -o --output-file --verbose --record-regression --replay
|
|
2528
|
+
--regression-diff-only --mask-sensitive --save-weights --load-weights
|
|
2529
|
+
--weights-dir --previous-spec --healing-report --use-llm --llm-provider
|
|
2530
|
+
--llm-model --llm-api-key --oauth2-token-url --oauth2-client-id
|
|
2531
|
+
--oauth2-client-secret --oauth2-scopes --oauth2-grant-type"
|
|
2532
|
+
case "${prev}" in
|
|
2533
|
+
--output|-o)
|
|
2534
|
+
COMPREPLY=( $(compgen -W "text json html junit" -- "${cur}") ) ;;
|
|
2535
|
+
--type)
|
|
2536
|
+
COMPREPLY=( $(compgen -W "rest graphql" -- "${cur}") ) ;;
|
|
2537
|
+
--spec|--save-weights|--load-weights|--healing-report|--previous-spec|--record-regression|--output-file)
|
|
2538
|
+
COMPREPLY=( $(compgen -f -- "${cur}") ) ;;
|
|
2539
|
+
--base-url|--oauth2-token-url)
|
|
2540
|
+
COMPREPLY=() ;;
|
|
2541
|
+
*)
|
|
2542
|
+
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) ;;
|
|
2543
|
+
esac ;;
|
|
2544
|
+
security-scan)
|
|
2545
|
+
local opts="--spec --base-url --type --checks --severity-threshold --output -o
|
|
2546
|
+
--output-file --belief-guided --prioritization --verbose --use-llm
|
|
2547
|
+
--llm-provider --llm-model --llm-api-key --oauth2-token-url
|
|
2548
|
+
--oauth2-client-id --oauth2-client-secret --oauth2-scopes --oauth2-grant-type"
|
|
2549
|
+
case "${prev}" in
|
|
2550
|
+
--output|-o)
|
|
2551
|
+
COMPREPLY=( $(compgen -W "text json markdown" -- "${cur}") ) ;;
|
|
2552
|
+
--type)
|
|
2553
|
+
COMPREPLY=( $(compgen -W "rest graphql" -- "${cur}") ) ;;
|
|
2554
|
+
--severity-threshold)
|
|
2555
|
+
COMPREPLY=( $(compgen -W "critical high medium low info" -- "${cur}") ) ;;
|
|
2556
|
+
--prioritization)
|
|
2557
|
+
COMPREPLY=( $(compgen -W "adaptive static off" -- "${cur}") ) ;;
|
|
2558
|
+
--spec|--output-file)
|
|
2559
|
+
COMPREPLY=( $(compgen -f -- "${cur}") ) ;;
|
|
2560
|
+
*)
|
|
2561
|
+
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) ;;
|
|
2562
|
+
esac ;;
|
|
2563
|
+
completions)
|
|
2564
|
+
COMPREPLY=( $(compgen -W "bash zsh fish" -- "${cur}") ) ;;
|
|
2565
|
+
setup)
|
|
2566
|
+
COMPREPLY=( $(compgen -W "--non-interactive --demo" -- "${cur}") ) ;;
|
|
2567
|
+
upgrade)
|
|
2568
|
+
COMPREPLY=( $(compgen -W "--check --migrate --pre" -- "${cur}") ) ;;
|
|
2569
|
+
uninstall)
|
|
2570
|
+
COMPREPLY=( $(compgen -W "--keep-config --yes -y --purge" -- "${cur}") ) ;;
|
|
2571
|
+
doctor)
|
|
2572
|
+
COMPREPLY=( $(compgen -W "--json --verbose" -- "${cur}") ) ;;
|
|
2573
|
+
*)
|
|
2574
|
+
COMPREPLY=( $(compgen -W "${commands}" -- "${cur}") ) ;;
|
|
2575
|
+
esac
|
|
2576
|
+
}
|
|
2577
|
+
|
|
2578
|
+
complete -F _nat_completions nat
|
|
2579
|
+
"""
|
|
2580
|
+
|
|
2581
|
+
_ZSH_COMPLETION = """\
|
|
2582
|
+
#compdef nat
|
|
2583
|
+
# zsh completion for nat
|
|
2584
|
+
# Install: nat completions zsh > "${fpath[1]}/_nat"
|
|
2585
|
+
# or: source <(nat completions zsh)
|
|
2586
|
+
|
|
2587
|
+
_nat() {
|
|
2588
|
+
local -a commands
|
|
2589
|
+
commands=(
|
|
2590
|
+
'scan:Parse an OpenAPI spec and run adaptive tests'
|
|
2591
|
+
'security-scan:Run OWASP API Security Top 10 checks'
|
|
2592
|
+
'weights:Manage saved weight snapshots'
|
|
2593
|
+
'heal:Detect API schema drift'
|
|
2594
|
+
'test-gen:Generate test cases from a spec'
|
|
2595
|
+
'completions:Output shell completion script'
|
|
2596
|
+
'demo:Run a curated interactive demo'
|
|
2597
|
+
'setup:Interactive setup wizard'
|
|
2598
|
+
'upgrade:Check for updates and upgrade NAT'
|
|
2599
|
+
'uninstall:Remove NAT config, data, and completions'
|
|
2600
|
+
'doctor:Run pre-flight environment checks'
|
|
2601
|
+
)
|
|
2602
|
+
|
|
2603
|
+
_arguments -C \\
|
|
2604
|
+
'--config[Path to YAML config file]:config file:_files' \\
|
|
2605
|
+
'(-h --help)'{-h,--help}'[Show help]' \\
|
|
2606
|
+
'1:command:->command' \\
|
|
2607
|
+
'*::args:->args'
|
|
2608
|
+
|
|
2609
|
+
case "${state}" in
|
|
2610
|
+
command)
|
|
2611
|
+
_describe 'nat command' commands ;;
|
|
2612
|
+
args)
|
|
2613
|
+
case "${words[1]}" in
|
|
2614
|
+
scan)
|
|
2615
|
+
_arguments \\
|
|
2616
|
+
'--spec[OpenAPI spec file]:spec:_files' \\
|
|
2617
|
+
'--base-url[Base URL]:url:' \\
|
|
2618
|
+
'--type[API type]:type:(rest graphql)' \\
|
|
2619
|
+
'--graphql-introspection[Enable introspection]' \\
|
|
2620
|
+
'--auth-token[Bearer token]:token:' \\
|
|
2621
|
+
'--api-key[API key]:key:' \\
|
|
2622
|
+
'--api-key-header[API key header]:header:' \\
|
|
2623
|
+
'--max-tests[Max test cases]:N:' \\
|
|
2624
|
+
'--executors[Executor count]:N:' \\
|
|
2625
|
+
'--timeout[HTTP timeout seconds]:seconds:' \\
|
|
2626
|
+
'--latency-sla[Latency SLA ms]:ms:' \\
|
|
2627
|
+
'(-o --output)'{-o,--output}'[Output format]:format:(text json html junit)' \\
|
|
2628
|
+
'--output-file[Output file path]:file:_files' \\
|
|
2629
|
+
'--verbose[Verbose logging]' ;;
|
|
2630
|
+
security-scan)
|
|
2631
|
+
_arguments \\
|
|
2632
|
+
'--spec[OpenAPI spec file]:spec:_files' \\
|
|
2633
|
+
'--base-url[Base URL]:url:' \\
|
|
2634
|
+
'--type[API type]:type:(rest graphql)' \\
|
|
2635
|
+
'--checks[Check IDs]:checks:' \\
|
|
2636
|
+
'--severity-threshold[Min severity]:level:(critical high medium low info)' \\
|
|
2637
|
+
'(-o --output)'{-o,--output}'[Output format]:format:(text json markdown)' \\
|
|
2638
|
+
'--output-file[Output file path]:file:_files' \\
|
|
2639
|
+
'--prioritization[Mode]:mode:(adaptive static off)' \\
|
|
2640
|
+
'--verbose[Verbose logging]' ;;
|
|
2641
|
+
completions)
|
|
2642
|
+
_arguments '1:shell:(bash zsh fish)' ;;
|
|
2643
|
+
setup)
|
|
2644
|
+
_arguments \
|
|
2645
|
+
'--non-interactive[Skip prompts]' \
|
|
2646
|
+
'--demo[Launch demo after setup]' ;;
|
|
2647
|
+
upgrade)
|
|
2648
|
+
_arguments \
|
|
2649
|
+
'--check[Only check for updates]' \
|
|
2650
|
+
'--migrate[Run DB migrations after upgrade]' \
|
|
2651
|
+
'--pre[Allow pre-release versions]' ;;
|
|
2652
|
+
uninstall)
|
|
2653
|
+
_arguments \
|
|
2654
|
+
'--keep-config[Preserve .natrc files]' \
|
|
2655
|
+
'(-y --yes)'{-y,--yes}'[Skip confirmation]' \
|
|
2656
|
+
'--purge[Remove DB tables and Docker volumes]' ;;
|
|
2657
|
+
doctor)
|
|
2658
|
+
_arguments \
|
|
2659
|
+
'--json[Output as JSON]' \
|
|
2660
|
+
'--verbose[Show detailed diagnostics]' ;;
|
|
2661
|
+
esac ;;
|
|
2662
|
+
esac
|
|
2663
|
+
}
|
|
2664
|
+
|
|
2665
|
+
_nat "$@"
|
|
2666
|
+
"""
|
|
2667
|
+
|
|
2668
|
+
_FISH_COMPLETION = """\
|
|
2669
|
+
# fish completion for nat
|
|
2670
|
+
# Install: nat completions fish > ~/.config/fish/completions/nat.fish
|
|
2671
|
+
|
|
2672
|
+
set -l commands scan security-scan weights heal test-gen completions demo setup upgrade uninstall doctor
|
|
2673
|
+
|
|
2674
|
+
complete -c nat -f -n "not __fish_seen_subcommand_from $commands" -a scan -d 'Run functional test scan'
|
|
2675
|
+
complete -c nat -f -n "not __fish_seen_subcommand_from $commands" -a security-scan -d 'Run OWASP security checks'
|
|
2676
|
+
complete -c nat -f -n "not __fish_seen_subcommand_from $commands" -a weights -d 'Manage weight snapshots'
|
|
2677
|
+
complete -c nat -f -n "not __fish_seen_subcommand_from $commands" -a heal -d 'Detect schema drift'
|
|
2678
|
+
complete -c nat -f -n "not __fish_seen_subcommand_from $commands" -a test-gen -d 'Generate test cases'
|
|
2679
|
+
complete -c nat -f -n "not __fish_seen_subcommand_from $commands" -a completions -d 'Shell completion script'
|
|
2680
|
+
complete -c nat -f -n "not __fish_seen_subcommand_from $commands" -a demo -d 'Run curated demo'
|
|
2681
|
+
complete -c nat -f -n "not __fish_seen_subcommand_from $commands" -a setup -d 'Interactive setup wizard'
|
|
2682
|
+
complete -c nat -f -n "not __fish_seen_subcommand_from $commands" -a upgrade -d 'Check for updates and upgrade'
|
|
2683
|
+
complete -c nat -f -n "not __fish_seen_subcommand_from $commands" -a uninstall -d 'Remove NAT config and data'
|
|
2684
|
+
complete -c nat -f -n "not __fish_seen_subcommand_from $commands" -a doctor -d 'Pre-flight environment checks'
|
|
2685
|
+
|
|
2686
|
+
# Global
|
|
2687
|
+
complete -c nat -l config -d 'YAML config file path' -r
|
|
2688
|
+
|
|
2689
|
+
# scan
|
|
2690
|
+
complete -c nat -n "__fish_seen_subcommand_from scan" -l spec -d 'OpenAPI spec file' -r -F
|
|
2691
|
+
complete -c nat -n "__fish_seen_subcommand_from scan" -l base-url -d 'Base URL' -r
|
|
2692
|
+
complete -c nat -n "__fish_seen_subcommand_from scan" -l type -d 'API type' -r -a 'rest graphql'
|
|
2693
|
+
complete -c nat -n "__fish_seen_subcommand_from scan" -l graphql-introspection -d 'Enable introspection'
|
|
2694
|
+
complete -c nat -n "__fish_seen_subcommand_from scan" -l auth-token -d 'Bearer token' -r
|
|
2695
|
+
complete -c nat -n "__fish_seen_subcommand_from scan" -l api-key -d 'API key' -r
|
|
2696
|
+
complete -c nat -n "__fish_seen_subcommand_from scan" -l max-tests -d 'Max test cases' -r
|
|
2697
|
+
complete -c nat -n "__fish_seen_subcommand_from scan" -l output -s o -d 'Output format' -r -a 'text json html junit'
|
|
2698
|
+
complete -c nat -n "__fish_seen_subcommand_from scan" -l output-file -d 'Output file path' -r -F
|
|
2699
|
+
complete -c nat -n "__fish_seen_subcommand_from scan" -l verbose -d 'Verbose logging'
|
|
2700
|
+
|
|
2701
|
+
# security-scan
|
|
2702
|
+
complete -c nat -n "__fish_seen_subcommand_from security-scan" -l spec -d 'OpenAPI spec file' -r -F
|
|
2703
|
+
complete -c nat -n "__fish_seen_subcommand_from security-scan" -l base-url -d 'Base URL' -r
|
|
2704
|
+
complete -c nat -n "__fish_seen_subcommand_from security-scan" -l type -d 'API type' -r -a 'rest graphql'
|
|
2705
|
+
complete -c nat -n "__fish_seen_subcommand_from security-scan" -l checks -d 'Check IDs' -r
|
|
2706
|
+
complete -c nat -n "__fish_seen_subcommand_from security-scan" -l severity-threshold -d 'Min severity' -r -a 'critical high medium low info'
|
|
2707
|
+
complete -c nat -n "__fish_seen_subcommand_from security-scan" -l output -s o -d 'Output format' -r -a 'text json markdown'
|
|
2708
|
+
complete -c nat -n "__fish_seen_subcommand_from security-scan" -l output-file -d 'Output file path' -r -F
|
|
2709
|
+
complete -c nat -n "__fish_seen_subcommand_from security-scan" -l prioritization -d 'Prioritization mode' -r -a 'adaptive static off'
|
|
2710
|
+
complete -c nat -n "__fish_seen_subcommand_from security-scan" -l verbose -d 'Verbose logging'
|
|
2711
|
+
|
|
2712
|
+
# completions
|
|
2713
|
+
complete -c nat -n "__fish_seen_subcommand_from completions" -a 'bash zsh fish' -d 'Shell type'
|
|
2714
|
+
|
|
2715
|
+
# setup
|
|
2716
|
+
complete -c nat -n "__fish_seen_subcommand_from setup" -l non-interactive -d 'Skip prompts'
|
|
2717
|
+
complete -c nat -n "__fish_seen_subcommand_from setup" -l demo -d 'Launch demo after setup'
|
|
2718
|
+
|
|
2719
|
+
# upgrade
|
|
2720
|
+
complete -c nat -n "__fish_seen_subcommand_from upgrade" -l check -d 'Only check for updates'
|
|
2721
|
+
complete -c nat -n "__fish_seen_subcommand_from upgrade" -l migrate -d 'Run DB migrations'
|
|
2722
|
+
complete -c nat -n "__fish_seen_subcommand_from upgrade" -l pre -d 'Allow pre-release'
|
|
2723
|
+
|
|
2724
|
+
# uninstall
|
|
2725
|
+
complete -c nat -n "__fish_seen_subcommand_from uninstall" -l keep-config -d 'Preserve .natrc'
|
|
2726
|
+
complete -c nat -n "__fish_seen_subcommand_from uninstall" -l yes -s y -d 'Skip confirmation'
|
|
2727
|
+
complete -c nat -n "__fish_seen_subcommand_from uninstall" -l purge -d 'Remove DB and Docker volumes'
|
|
2728
|
+
|
|
2729
|
+
# doctor
|
|
2730
|
+
complete -c nat -n "__fish_seen_subcommand_from doctor" -l json -d 'Output as JSON'
|
|
2731
|
+
complete -c nat -n "__fish_seen_subcommand_from doctor" -l verbose -d 'Show detailed diagnostics'
|
|
2732
|
+
"""
|
|
2733
|
+
|
|
2734
|
+
|
|
2735
|
+
def _run_completions(args: argparse.Namespace) -> int:
|
|
2736
|
+
"""Output a shell completion script for the requested shell."""
|
|
2737
|
+
shell = args.shell
|
|
2738
|
+
scripts = {
|
|
2739
|
+
"bash": _BASH_COMPLETION,
|
|
2740
|
+
"zsh": _ZSH_COMPLETION,
|
|
2741
|
+
"fish": _FISH_COMPLETION,
|
|
2742
|
+
}
|
|
2743
|
+
print(scripts[shell], end="")
|
|
2744
|
+
return EXIT_OK
|
|
2745
|
+
|
|
2746
|
+
|
|
2747
|
+
async def _run_test_gen(args: argparse.Namespace) -> int:
|
|
2748
|
+
"""Execute the *test-gen* sub-command. Generates and prints test cases."""
|
|
2749
|
+
from mannf.product.integrations.openapi_parser import OpenApiParser
|
|
2750
|
+
|
|
2751
|
+
llm_provider = _build_llm_provider(args)
|
|
2752
|
+
|
|
2753
|
+
parser = OpenApiParser(
|
|
2754
|
+
spec_path=args.spec,
|
|
2755
|
+
base_url=args.base_url,
|
|
2756
|
+
max_tests=getattr(args, "max_tests", None),
|
|
2757
|
+
llm_provider=llm_provider,
|
|
2758
|
+
)
|
|
2759
|
+
|
|
2760
|
+
if llm_provider is not None:
|
|
2761
|
+
test_cases = await parser.parse_with_llm()
|
|
2762
|
+
else:
|
|
2763
|
+
test_cases = parser.parse()
|
|
2764
|
+
|
|
2765
|
+
if not test_cases:
|
|
2766
|
+
print("No test cases generated.", file=sys.stderr)
|
|
2767
|
+
return 1
|
|
2768
|
+
|
|
2769
|
+
llm_count = sum(1 for tc in test_cases if tc.metadata.get("generator") == "llm")
|
|
2770
|
+
|
|
2771
|
+
if args.output == "json":
|
|
2772
|
+
output = [
|
|
2773
|
+
{
|
|
2774
|
+
"id": tc.id,
|
|
2775
|
+
"target": tc.target,
|
|
2776
|
+
"inputs": tc.inputs,
|
|
2777
|
+
"expected_behavior": tc.expected_behavior,
|
|
2778
|
+
"priority": tc.priority,
|
|
2779
|
+
"metadata": tc.metadata,
|
|
2780
|
+
}
|
|
2781
|
+
for tc in test_cases
|
|
2782
|
+
]
|
|
2783
|
+
print(json.dumps(output, indent=2))
|
|
2784
|
+
else:
|
|
2785
|
+
print(f"\n{'=' * 60}")
|
|
2786
|
+
print(f" NAT Test Generation: {args.spec}")
|
|
2787
|
+
print(f"{'=' * 60}")
|
|
2788
|
+
print(f" Total cases: {len(test_cases)}")
|
|
2789
|
+
if llm_count:
|
|
2790
|
+
print(f" LLM-generated: {llm_count} 🤖")
|
|
2791
|
+
print(f"{'=' * 60}")
|
|
2792
|
+
for tc in test_cases:
|
|
2793
|
+
llm_tag = " 🤖" if tc.metadata.get("generator") == "llm" else ""
|
|
2794
|
+
print(f" [{tc.metadata.get('method', '?')}] {tc.target}{llm_tag}")
|
|
2795
|
+
print(f" {tc.expected_behavior}")
|
|
2796
|
+
print()
|
|
2797
|
+
|
|
2798
|
+
return 0
|
|
2799
|
+
|
|
2800
|
+
|
|
2801
|
+
async def _run_plan(args: argparse.Namespace) -> int:
|
|
2802
|
+
"""Execute the *plan* sub-command. Generates and prints an LLM test plan."""
|
|
2803
|
+
import yaml # noqa: PLC0415
|
|
2804
|
+
|
|
2805
|
+
from mannf.product.llm.config import LLMConfig # noqa: PLC0415
|
|
2806
|
+
from mannf.product.llm.factory import get_provider # noqa: PLC0415
|
|
2807
|
+
|
|
2808
|
+
spec_path = args.spec
|
|
2809
|
+
if not os.path.isfile(spec_path):
|
|
2810
|
+
print(f"error: spec file not found: {spec_path!r}", file=sys.stderr)
|
|
2811
|
+
return EXIT_SCAN_ERROR
|
|
2812
|
+
|
|
2813
|
+
# Parse spec → compact summary
|
|
2814
|
+
try:
|
|
2815
|
+
with open(spec_path, encoding="utf-8") as fh:
|
|
2816
|
+
raw = fh.read()
|
|
2817
|
+
if spec_path.endswith(".json"):
|
|
2818
|
+
spec = json.loads(raw)
|
|
2819
|
+
else:
|
|
2820
|
+
spec = yaml.safe_load(raw) or {}
|
|
2821
|
+
except Exception as exc: # noqa: BLE001
|
|
2822
|
+
print(f"error: could not parse spec: {exc}", file=sys.stderr)
|
|
2823
|
+
return EXIT_SCAN_ERROR
|
|
2824
|
+
|
|
2825
|
+
if not isinstance(spec, dict):
|
|
2826
|
+
print("error: spec must be a YAML/JSON object", file=sys.stderr)
|
|
2827
|
+
return EXIT_SCAN_ERROR
|
|
2828
|
+
|
|
2829
|
+
endpoints = []
|
|
2830
|
+
for path, path_item in spec.get("paths", {}).items():
|
|
2831
|
+
if not isinstance(path_item, dict):
|
|
2832
|
+
continue
|
|
2833
|
+
for method, operation in path_item.items():
|
|
2834
|
+
if method.lower() not in ("get", "post", "put", "patch", "delete", "head", "options"):
|
|
2835
|
+
continue
|
|
2836
|
+
if not isinstance(operation, dict):
|
|
2837
|
+
continue
|
|
2838
|
+
endpoints.append({
|
|
2839
|
+
"method": method.upper(),
|
|
2840
|
+
"path": path,
|
|
2841
|
+
"summary": operation.get("summary") or operation.get("operationId") or "",
|
|
2842
|
+
"description": operation.get("description", ""),
|
|
2843
|
+
"tags": operation.get("tags", []),
|
|
2844
|
+
"requires_auth": bool(operation.get("security")),
|
|
2845
|
+
})
|
|
2846
|
+
|
|
2847
|
+
info = spec.get("info", {})
|
|
2848
|
+
auth_schemes = list(spec.get("components", {}).get("securitySchemes", {}).keys())
|
|
2849
|
+
api_summary = {
|
|
2850
|
+
"spec_source": os.path.basename(spec_path),
|
|
2851
|
+
"base_url": args.base_url,
|
|
2852
|
+
"title": info.get("title", "Unknown API"),
|
|
2853
|
+
"version": info.get("version", ""),
|
|
2854
|
+
"auth_schemes": auth_schemes,
|
|
2855
|
+
"total_endpoints": len(endpoints),
|
|
2856
|
+
"endpoints": endpoints,
|
|
2857
|
+
}
|
|
2858
|
+
|
|
2859
|
+
# Build LLM provider
|
|
2860
|
+
try:
|
|
2861
|
+
config = LLMConfig(
|
|
2862
|
+
provider=getattr(args, "llm_provider", "openai") or "openai",
|
|
2863
|
+
model=getattr(args, "llm_model", None) or "",
|
|
2864
|
+
api_key=getattr(args, "llm_api_key", None),
|
|
2865
|
+
)
|
|
2866
|
+
provider = get_provider(config)
|
|
2867
|
+
except Exception as exc: # noqa: BLE001
|
|
2868
|
+
print(f"error: could not initialise LLM provider: {exc}", file=sys.stderr)
|
|
2869
|
+
return EXIT_CONFIG_ERROR
|
|
2870
|
+
|
|
2871
|
+
if not provider.is_available():
|
|
2872
|
+
print(
|
|
2873
|
+
"error: LLM provider is not available — set OPENAI_API_KEY or ANTHROPIC_API_KEY, "
|
|
2874
|
+
"or pass --llm-api-key.",
|
|
2875
|
+
file=sys.stderr,
|
|
2876
|
+
)
|
|
2877
|
+
return EXIT_CONFIG_ERROR
|
|
2878
|
+
|
|
2879
|
+
print(f"\n 🧠 Generating test plan for {api_summary['title']} …\n")
|
|
2880
|
+
|
|
2881
|
+
test_plan = await provider.generate_test_plan(api_summary)
|
|
2882
|
+
|
|
2883
|
+
if test_plan is None:
|
|
2884
|
+
print("error: LLM did not return a valid test plan.", file=sys.stderr)
|
|
2885
|
+
return EXIT_SCAN_ERROR
|
|
2886
|
+
|
|
2887
|
+
output_format = getattr(args, "output", "text")
|
|
2888
|
+
|
|
2889
|
+
if output_format == "json":
|
|
2890
|
+
print(json.dumps(test_plan.model_dump(), indent=2))
|
|
2891
|
+
return EXIT_OK
|
|
2892
|
+
|
|
2893
|
+
# Text output
|
|
2894
|
+
_PRIORITY_ICONS = {"critical": "🔴", "high": "🟠", "medium": "🟡", "low": "🔵"}
|
|
2895
|
+
print(f"{'=' * 64}")
|
|
2896
|
+
print(f" NAT Test Plan — {api_summary['title']} {api_summary.get('version', '')}")
|
|
2897
|
+
print(f" Spec: {os.path.basename(spec_path)} | Endpoints: {len(endpoints)}")
|
|
2898
|
+
if test_plan.estimated_coverage is not None:
|
|
2899
|
+
print(f" Estimated coverage: {test_plan.estimated_coverage * 100:.0f}%")
|
|
2900
|
+
print(f"{'=' * 64}")
|
|
2901
|
+
for group in test_plan.endpoint_groups:
|
|
2902
|
+
icon = _PRIORITY_ICONS.get(group.priority.value if hasattr(group.priority, 'value') else group.priority, "⚪")
|
|
2903
|
+
print(f"\n {icon} {group.name} [{group.priority if isinstance(group.priority, str) else group.priority.value}]")
|
|
2904
|
+
if group.risk_assessment:
|
|
2905
|
+
print(f" Risk: {group.risk_assessment}")
|
|
2906
|
+
for ep in group.endpoints:
|
|
2907
|
+
print(f" • {ep}")
|
|
2908
|
+
if group.suggested_strategies:
|
|
2909
|
+
strats = ", ".join(group.suggested_strategies)
|
|
2910
|
+
print(f" Strategies: {strats}")
|
|
2911
|
+
if group.edge_cases:
|
|
2912
|
+
for ec in group.edge_cases:
|
|
2913
|
+
print(f" ⚠ {ec}")
|
|
2914
|
+
print(f"\n{'=' * 64}\n")
|
|
2915
|
+
|
|
2916
|
+
if getattr(args, "execute", False):
|
|
2917
|
+
print(" ▶ Auto-execute requested — plan approved and executed locally.")
|
|
2918
|
+
print(f" Groups scheduled: {len(test_plan.endpoint_groups)}")
|
|
2919
|
+
print()
|
|
2920
|
+
|
|
2921
|
+
return EXIT_OK
|
|
2922
|
+
|
|
2923
|
+
|
|
2924
|
+
async def _run_nl_test(args: argparse.Namespace) -> int:
|
|
2925
|
+
"""Execute the *test* sub-command.
|
|
2926
|
+
|
|
2927
|
+
Generates executable test scenarios from a plain-English description using
|
|
2928
|
+
the LLM, prints them for review, and optionally executes them immediately.
|
|
2929
|
+
"""
|
|
2930
|
+
from mannf.product.llm.config import LLMConfig # noqa: PLC0415
|
|
2931
|
+
from mannf.product.llm.factory import get_provider # noqa: PLC0415
|
|
2932
|
+
|
|
2933
|
+
description = args.description
|
|
2934
|
+
base_url = getattr(args, "base_url", "")
|
|
2935
|
+
auth_type = getattr(args, "auth_type", "none")
|
|
2936
|
+
known_endpoints_raw = getattr(args, "known_endpoints", "")
|
|
2937
|
+
known_endpoints = [e.strip() for e in known_endpoints_raw.split(",") if e.strip()]
|
|
2938
|
+
|
|
2939
|
+
context = {
|
|
2940
|
+
"base_url": base_url,
|
|
2941
|
+
"auth_type": auth_type,
|
|
2942
|
+
"known_endpoints": known_endpoints,
|
|
2943
|
+
}
|
|
2944
|
+
|
|
2945
|
+
# Build LLM provider
|
|
2946
|
+
try:
|
|
2947
|
+
config = LLMConfig(
|
|
2948
|
+
provider=getattr(args, "llm_provider", "openai") or "openai",
|
|
2949
|
+
model=getattr(args, "llm_model", None) or "",
|
|
2950
|
+
api_key=getattr(args, "llm_api_key", None),
|
|
2951
|
+
)
|
|
2952
|
+
provider = get_provider(config)
|
|
2953
|
+
except Exception as exc: # noqa: BLE001
|
|
2954
|
+
print(f"error: could not initialise LLM provider: {exc}", file=sys.stderr)
|
|
2955
|
+
return EXIT_CONFIG_ERROR
|
|
2956
|
+
|
|
2957
|
+
if not provider.is_available():
|
|
2958
|
+
print(
|
|
2959
|
+
"error: LLM provider is not available — set OPENAI_API_KEY or ANTHROPIC_API_KEY, "
|
|
2960
|
+
"or pass --llm-api-key.",
|
|
2961
|
+
file=sys.stderr,
|
|
2962
|
+
)
|
|
2963
|
+
return EXIT_CONFIG_ERROR
|
|
2964
|
+
|
|
2965
|
+
print(f"\n 🧪 Generating test scenarios for: {description!r} …\n")
|
|
2966
|
+
|
|
2967
|
+
scenarios = await provider.generate_from_natural_language(description, context)
|
|
2968
|
+
|
|
2969
|
+
if not scenarios:
|
|
2970
|
+
print("error: LLM did not return any test scenarios.", file=sys.stderr)
|
|
2971
|
+
return EXIT_SCAN_ERROR
|
|
2972
|
+
|
|
2973
|
+
output_format = getattr(args, "output", "text")
|
|
2974
|
+
|
|
2975
|
+
if output_format == "json":
|
|
2976
|
+
print(json.dumps(scenarios, indent=2))
|
|
2977
|
+
return EXIT_OK
|
|
2978
|
+
|
|
2979
|
+
# Text output
|
|
2980
|
+
_TYPE_ICONS = {"api": "🌐", "browser": "🖥️"}
|
|
2981
|
+
_PRIORITY_ICONS = {"critical": "🔴", "high": "🟠", "medium": "🟡", "low": "🔵"}
|
|
2982
|
+
print(f"{'=' * 64}")
|
|
2983
|
+
print(f" NAT Generated Test Scenarios")
|
|
2984
|
+
print(f" Description: {description}")
|
|
2985
|
+
if base_url:
|
|
2986
|
+
print(f" Base URL: {base_url}")
|
|
2987
|
+
print(f" Scenarios: {len(scenarios)}")
|
|
2988
|
+
print(f"{'=' * 64}")
|
|
2989
|
+
for i, scenario in enumerate(scenarios, 1):
|
|
2990
|
+
s_type = scenario.get("type", "api")
|
|
2991
|
+
priority = scenario.get("priority", "medium")
|
|
2992
|
+
type_icon = _TYPE_ICONS.get(s_type, "🔷")
|
|
2993
|
+
priority_icon = _PRIORITY_ICONS.get(priority, "⚪")
|
|
2994
|
+
print(f"\n {type_icon} [{i}] {scenario.get('name', 'Unnamed')} {priority_icon} [{priority}]")
|
|
2995
|
+
if scenario.get("description"):
|
|
2996
|
+
print(f" {scenario['description']}")
|
|
2997
|
+
if s_type == "api":
|
|
2998
|
+
method = scenario.get("method", "GET")
|
|
2999
|
+
path = scenario.get("path", "/")
|
|
3000
|
+
print(f" Endpoint: {method} {path}")
|
|
3001
|
+
if scenario.get("body"):
|
|
3002
|
+
print(f" Body: {json.dumps(scenario['body'])}")
|
|
3003
|
+
elif s_type == "browser":
|
|
3004
|
+
steps = scenario.get("steps", [])
|
|
3005
|
+
for step in steps:
|
|
3006
|
+
action = step.get("action", "")
|
|
3007
|
+
selector = step.get("selector", "")
|
|
3008
|
+
value = step.get("value", "")
|
|
3009
|
+
if value:
|
|
3010
|
+
print(f" • {action}({selector!r}, {value!r})")
|
|
3011
|
+
else:
|
|
3012
|
+
print(f" • {action}({selector!r})")
|
|
3013
|
+
if scenario.get("assertions"):
|
|
3014
|
+
for assertion in scenario["assertions"]:
|
|
3015
|
+
print(f" ✓ {assertion}")
|
|
3016
|
+
print(f"\n{'=' * 64}\n")
|
|
3017
|
+
|
|
3018
|
+
if getattr(args, "execute", False):
|
|
3019
|
+
print(" ▶ --execute flag detected — running scenarios now …\n")
|
|
3020
|
+
_execute_nl_scenarios(scenarios, base_url, getattr(args, "headed", False))
|
|
3021
|
+
|
|
3022
|
+
return EXIT_OK
|
|
3023
|
+
|
|
3024
|
+
|
|
3025
|
+
def _execute_nl_scenarios(
|
|
3026
|
+
scenarios: list,
|
|
3027
|
+
base_url: str,
|
|
3028
|
+
headed: bool,
|
|
3029
|
+
) -> None:
|
|
3030
|
+
"""Execute generated NL test scenarios locally.
|
|
3031
|
+
|
|
3032
|
+
API scenarios are sent as HTTP requests; browser scenarios are logged
|
|
3033
|
+
for execution by the functional orchestrator (full browser runner not
|
|
3034
|
+
invoked inline to avoid heavyweight dependencies from the CLI path).
|
|
3035
|
+
"""
|
|
3036
|
+
import urllib.request # noqa: PLC0415
|
|
3037
|
+
|
|
3038
|
+
api_scenarios = [s for s in scenarios if s.get("type") == "api"]
|
|
3039
|
+
browser_scenarios = [s for s in scenarios if s.get("type") == "browser"]
|
|
3040
|
+
|
|
3041
|
+
for scenario in api_scenarios:
|
|
3042
|
+
method = scenario.get("method", "GET").upper()
|
|
3043
|
+
path = scenario.get("path", "/")
|
|
3044
|
+
url = f"{base_url.rstrip('/')}{path}"
|
|
3045
|
+
body_data = scenario.get("body")
|
|
3046
|
+
print(f" 🌐 {method} {url}", end=" … ")
|
|
3047
|
+
try:
|
|
3048
|
+
body_bytes = json.dumps(body_data).encode() if body_data else None
|
|
3049
|
+
headers = {"Content-Type": "application/json"} if body_bytes else {}
|
|
3050
|
+
headers.update(scenario.get("headers") or {})
|
|
3051
|
+
req = urllib.request.Request(url, data=body_bytes, headers=headers, method=method)
|
|
3052
|
+
with urllib.request.urlopen(req, timeout=10) as resp: # noqa: S310
|
|
3053
|
+
status = resp.getcode()
|
|
3054
|
+
print(f"HTTP {status}")
|
|
3055
|
+
except Exception as exc: # noqa: BLE001
|
|
3056
|
+
print(f"error: {exc}")
|
|
3057
|
+
|
|
3058
|
+
if browser_scenarios:
|
|
3059
|
+
mode = "headed" if headed else "headless"
|
|
3060
|
+
print(f"\n 🖥️ {len(browser_scenarios)} browser scenario(s) queued for {mode} execution.")
|
|
3061
|
+
print(" Use `nat scan --base-url <url>` or the dashboard to run browser scenarios.")
|
|
3062
|
+
|
|
3063
|
+
|
|
3064
|
+
def _run_demo(args: argparse.Namespace) -> int:
|
|
3065
|
+
"""Execute the ``demo`` sub-command.
|
|
3066
|
+
|
|
3067
|
+
When ``--target`` is provided, delegates to the appropriate demo runner:
|
|
3068
|
+
|
|
3069
|
+
* ``orangehrm`` — runs the OrangeHRM live demo via
|
|
3070
|
+
:func:`scripts.demo_orangehrm.run_orangehrm_demo`.
|
|
3071
|
+
* ``vulnapi`` — starts the VulnAPI testbed (if not already running) and
|
|
3072
|
+
runs a security scan against it.
|
|
3073
|
+
* Any URL (starts with ``http://`` or ``https://``) — runs the autonomous
|
|
3074
|
+
browser pipeline via :class:`~mannf.core.functional_orchestrator.FunctionalTestOrchestrator`.
|
|
3075
|
+
|
|
3076
|
+
When ``--target`` is **not** provided, the original behavior is preserved:
|
|
3077
|
+
starts the NAT FastAPI server in-process, waits for it to be healthy,
|
|
3078
|
+
opens the browser, runs the curated demo scan, then blocks until the user
|
|
3079
|
+
presses Ctrl+C.
|
|
3080
|
+
"""
|
|
3081
|
+
import threading
|
|
3082
|
+
import time
|
|
3083
|
+
import webbrowser
|
|
3084
|
+
|
|
3085
|
+
import httpx
|
|
3086
|
+
import uvicorn
|
|
3087
|
+
|
|
3088
|
+
from mannf.product.server import app
|
|
3089
|
+
|
|
3090
|
+
port: int = getattr(args, "port", 8080)
|
|
3091
|
+
no_browser: bool = getattr(args, "no_browser", False)
|
|
3092
|
+
target: str | None = getattr(args, "target", None)
|
|
3093
|
+
headed: bool = getattr(args, "headed", False)
|
|
3094
|
+
output_dir: str = getattr(args, "output_dir", "demo_output")
|
|
3095
|
+
json_summary: str = getattr(args, "json_summary", "")
|
|
3096
|
+
|
|
3097
|
+
# ------------------------------------------------------------------
|
|
3098
|
+
# Route to the appropriate demo runner based on --target
|
|
3099
|
+
# ------------------------------------------------------------------
|
|
3100
|
+
|
|
3101
|
+
if target is not None:
|
|
3102
|
+
return _run_demo_target(
|
|
3103
|
+
target=target,
|
|
3104
|
+
headed=headed,
|
|
3105
|
+
output_dir=output_dir,
|
|
3106
|
+
json_summary=json_summary,
|
|
3107
|
+
)
|
|
3108
|
+
|
|
3109
|
+
dashboard_url = f"http://localhost:{port}/dashboard"
|
|
3110
|
+
|
|
3111
|
+
print(f"\n 🚀 NAT Demo Mode — starting server on port {port} …\n")
|
|
3112
|
+
|
|
3113
|
+
# ------------------------------------------------------------------
|
|
3114
|
+
# Run the server as a co-task inside the demo event loop so that
|
|
3115
|
+
# WebSocket telemetry broadcasts are delivered to browser clients.
|
|
3116
|
+
# ------------------------------------------------------------------
|
|
3117
|
+
|
|
3118
|
+
async def _demo_main() -> None:
|
|
3119
|
+
from mannf.product.demo import run_demo_scan
|
|
3120
|
+
|
|
3121
|
+
config = uvicorn.Config(
|
|
3122
|
+
app,
|
|
3123
|
+
host="0.0.0.0",
|
|
3124
|
+
port=port,
|
|
3125
|
+
log_level="warning",
|
|
3126
|
+
)
|
|
3127
|
+
server = uvicorn.Server(config)
|
|
3128
|
+
|
|
3129
|
+
# Start the uvicorn server as a background task
|
|
3130
|
+
server_task = asyncio.create_task(server.serve())
|
|
3131
|
+
|
|
3132
|
+
# Wait until the server is accepting requests
|
|
3133
|
+
_healthy = False
|
|
3134
|
+
for _ in range(30):
|
|
3135
|
+
await asyncio.sleep(0.5)
|
|
3136
|
+
try:
|
|
3137
|
+
async with httpx.AsyncClient() as client:
|
|
3138
|
+
resp = await client.get(
|
|
3139
|
+
f"http://localhost:{port}/api/v1/health",
|
|
3140
|
+
timeout=2.0,
|
|
3141
|
+
)
|
|
3142
|
+
if resp.status_code == 200:
|
|
3143
|
+
_healthy = True
|
|
3144
|
+
break
|
|
3145
|
+
except Exception: # noqa: BLE001
|
|
3146
|
+
pass
|
|
3147
|
+
|
|
3148
|
+
if not _healthy:
|
|
3149
|
+
print(
|
|
3150
|
+
f" ⚠️ Server did not become healthy in time. "
|
|
3151
|
+
f"Check that port {port} is not already in use.",
|
|
3152
|
+
file=sys.stderr,
|
|
3153
|
+
)
|
|
3154
|
+
server.should_exit = True
|
|
3155
|
+
await server_task
|
|
3156
|
+
return
|
|
3157
|
+
|
|
3158
|
+
print(f" ✅ Server ready at http://localhost:{port}")
|
|
3159
|
+
print(f" 📊 Dashboard: {dashboard_url}")
|
|
3160
|
+
print("\n Running demo scan … (press Ctrl+C to stop)\n")
|
|
3161
|
+
|
|
3162
|
+
if not no_browser:
|
|
3163
|
+
# Open browser in a thread to avoid blocking the event loop
|
|
3164
|
+
threading.Thread(
|
|
3165
|
+
target=lambda: (time.sleep(0.5), webbrowser.open(dashboard_url)),
|
|
3166
|
+
daemon=True,
|
|
3167
|
+
).start()
|
|
3168
|
+
|
|
3169
|
+
# Run the demo scan (populates dashboard with live data)
|
|
3170
|
+
try:
|
|
3171
|
+
await run_demo_scan(port=port)
|
|
3172
|
+
except Exception as exc: # noqa: BLE001
|
|
3173
|
+
logger.warning("Demo scan error: %s", exc)
|
|
3174
|
+
|
|
3175
|
+
print("\n ✅ Demo data loaded — dashboard is ready!")
|
|
3176
|
+
print(f" 🌐 Open: {dashboard_url}")
|
|
3177
|
+
print("\n Press Ctrl+C to stop the server.\n")
|
|
3178
|
+
|
|
3179
|
+
# Block until Ctrl+C
|
|
3180
|
+
try:
|
|
3181
|
+
await asyncio.Event().wait()
|
|
3182
|
+
except asyncio.CancelledError:
|
|
3183
|
+
pass
|
|
3184
|
+
finally:
|
|
3185
|
+
server.should_exit = True
|
|
3186
|
+
await server_task
|
|
3187
|
+
|
|
3188
|
+
try:
|
|
3189
|
+
asyncio.run(_demo_main())
|
|
3190
|
+
except KeyboardInterrupt:
|
|
3191
|
+
print("\n\n 👋 Demo stopped. Goodbye!\n")
|
|
3192
|
+
|
|
3193
|
+
return EXIT_OK
|
|
3194
|
+
|
|
3195
|
+
|
|
3196
|
+
def _run_demo_target(
|
|
3197
|
+
target: str,
|
|
3198
|
+
headed: bool,
|
|
3199
|
+
output_dir: str,
|
|
3200
|
+
json_summary: str,
|
|
3201
|
+
) -> int:
|
|
3202
|
+
"""Route ``nat demo --target <target>`` to the correct demo runner.
|
|
3203
|
+
|
|
3204
|
+
Parameters
|
|
3205
|
+
----------
|
|
3206
|
+
target:
|
|
3207
|
+
``"orangehrm"`` — OrangeHRM live demo.
|
|
3208
|
+
``"vulnapi"`` — VulnAPI testbed demo.
|
|
3209
|
+
A URL string (starts with ``http://`` or ``https://``) — autonomous
|
|
3210
|
+
browser pipeline against that URL.
|
|
3211
|
+
headed:
|
|
3212
|
+
When ``True``, open a visible browser window.
|
|
3213
|
+
output_dir:
|
|
3214
|
+
Directory for HTML + JSON reports.
|
|
3215
|
+
json_summary:
|
|
3216
|
+
Optional path for a JSON summary file.
|
|
3217
|
+
"""
|
|
3218
|
+
headless = not headed
|
|
3219
|
+
|
|
3220
|
+
if target == "orangehrm":
|
|
3221
|
+
return _run_demo_orangehrm(
|
|
3222
|
+
headless=headless,
|
|
3223
|
+
output_dir=output_dir,
|
|
3224
|
+
json_summary=json_summary,
|
|
3225
|
+
)
|
|
3226
|
+
|
|
3227
|
+
if target == "vulnapi":
|
|
3228
|
+
return _run_demo_vulnapi(
|
|
3229
|
+
headless=headless,
|
|
3230
|
+
output_dir=output_dir,
|
|
3231
|
+
json_summary=json_summary,
|
|
3232
|
+
)
|
|
3233
|
+
|
|
3234
|
+
if target.startswith("http://") or target.startswith("https://"):
|
|
3235
|
+
return _run_demo_url(
|
|
3236
|
+
url=target,
|
|
3237
|
+
headless=headless,
|
|
3238
|
+
output_dir=output_dir,
|
|
3239
|
+
json_summary=json_summary,
|
|
3240
|
+
)
|
|
3241
|
+
|
|
3242
|
+
print(
|
|
3243
|
+
f" ❌ Unknown --target value: {target!r}\n"
|
|
3244
|
+
" Valid choices: 'orangehrm', 'vulnapi', or a URL starting with http:// or https://",
|
|
3245
|
+
file=sys.stderr,
|
|
3246
|
+
)
|
|
3247
|
+
return EXIT_FAILURES
|
|
3248
|
+
|
|
3249
|
+
|
|
3250
|
+
def _run_demo_orangehrm(
|
|
3251
|
+
headless: bool,
|
|
3252
|
+
output_dir: str,
|
|
3253
|
+
json_summary: str,
|
|
3254
|
+
) -> int:
|
|
3255
|
+
"""Run the OrangeHRM live demo via :func:`scripts.demo_orangehrm.run_orangehrm_demo`."""
|
|
3256
|
+
import json as _json
|
|
3257
|
+
|
|
3258
|
+
try:
|
|
3259
|
+
import pathlib as _pathlib
|
|
3260
|
+
import sys as _sys
|
|
3261
|
+
|
|
3262
|
+
# Make the scripts/ directory importable so that demo_orangehrm can be
|
|
3263
|
+
# imported as a module regardless of the current working directory.
|
|
3264
|
+
_scripts_dir = str(_pathlib.Path(__file__).parents[3] / "scripts")
|
|
3265
|
+
if _scripts_dir not in _sys.path:
|
|
3266
|
+
_sys.path.insert(0, _scripts_dir)
|
|
3267
|
+
|
|
3268
|
+
from demo_orangehrm import run_orangehrm_demo # type: ignore[import]
|
|
3269
|
+
except ImportError as exc:
|
|
3270
|
+
print(
|
|
3271
|
+
f" ❌ Could not import the OrangeHRM demo runner: {exc}\n"
|
|
3272
|
+
" Make sure 'mannf[browser]' is installed and you are running from "
|
|
3273
|
+
"the repository root.",
|
|
3274
|
+
file=sys.stderr,
|
|
3275
|
+
)
|
|
3276
|
+
return EXIT_FAILURES
|
|
3277
|
+
|
|
3278
|
+
try:
|
|
3279
|
+
report = asyncio.run(
|
|
3280
|
+
run_orangehrm_demo(
|
|
3281
|
+
headless=headless,
|
|
3282
|
+
output_dir=output_dir,
|
|
3283
|
+
)
|
|
3284
|
+
)
|
|
3285
|
+
except KeyboardInterrupt:
|
|
3286
|
+
print("\nDemo interrupted.", file=sys.stderr)
|
|
3287
|
+
return 130
|
|
3288
|
+
except Exception as exc: # noqa: BLE001
|
|
3289
|
+
logger.error("OrangeHRM demo failed: %s", exc)
|
|
3290
|
+
return EXIT_FAILURES
|
|
3291
|
+
|
|
3292
|
+
if json_summary:
|
|
3293
|
+
try:
|
|
3294
|
+
import pathlib
|
|
3295
|
+
|
|
3296
|
+
pathlib.Path(json_summary).parent.mkdir(parents=True, exist_ok=True)
|
|
3297
|
+
pathlib.Path(json_summary).write_text(_json.dumps(report, indent=2), encoding="utf-8")
|
|
3298
|
+
print(f" 📄 JSON summary written to: {json_summary}")
|
|
3299
|
+
except Exception as exc: # noqa: BLE001
|
|
3300
|
+
logger.warning("Could not write JSON summary to %s: %s", json_summary, exc)
|
|
3301
|
+
|
|
3302
|
+
failed = report.get("failed", 0)
|
|
3303
|
+
return EXIT_OK if failed == 0 else EXIT_FAILURES
|
|
3304
|
+
|
|
3305
|
+
|
|
3306
|
+
def _run_demo_vulnapi(
|
|
3307
|
+
headless: bool,
|
|
3308
|
+
output_dir: str,
|
|
3309
|
+
json_summary: str,
|
|
3310
|
+
) -> int:
|
|
3311
|
+
"""Start the VulnAPI testbed (if needed) and run a security scan against it."""
|
|
3312
|
+
import json as _json
|
|
3313
|
+
import socket
|
|
3314
|
+
import subprocess
|
|
3315
|
+
import time
|
|
3316
|
+
|
|
3317
|
+
vulnapi_port = 9181
|
|
3318
|
+
vulnapi_url = f"http://localhost:{vulnapi_port}"
|
|
3319
|
+
|
|
3320
|
+
# Check if the testbed is already running
|
|
3321
|
+
_already_running = False
|
|
3322
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as _sock:
|
|
3323
|
+
_already_running = _sock.connect_ex(("localhost", vulnapi_port)) == 0
|
|
3324
|
+
|
|
3325
|
+
_server_proc = None
|
|
3326
|
+
if not _already_running:
|
|
3327
|
+
print(f" 🚀 Starting VulnAPI testbed on port {vulnapi_port} …")
|
|
3328
|
+
try:
|
|
3329
|
+
_server_proc = subprocess.Popen( # noqa: S603
|
|
3330
|
+
[sys.executable, "-m", "uvicorn", "testbed.app:app", f"--port={vulnapi_port}", "--log-level=warning"],
|
|
3331
|
+
stdout=subprocess.DEVNULL,
|
|
3332
|
+
stderr=subprocess.DEVNULL,
|
|
3333
|
+
)
|
|
3334
|
+
# Wait for the testbed to become healthy
|
|
3335
|
+
for _ in range(20):
|
|
3336
|
+
time.sleep(0.5)
|
|
3337
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as _s:
|
|
3338
|
+
if _s.connect_ex(("localhost", vulnapi_port)) == 0:
|
|
3339
|
+
break
|
|
3340
|
+
else:
|
|
3341
|
+
print(" ⚠️ VulnAPI testbed did not start in time.", file=sys.stderr)
|
|
3342
|
+
if _server_proc:
|
|
3343
|
+
_server_proc.terminate()
|
|
3344
|
+
return EXIT_FAILURES
|
|
3345
|
+
print(f" ✅ VulnAPI testbed ready at {vulnapi_url}")
|
|
3346
|
+
except Exception as exc: # noqa: BLE001
|
|
3347
|
+
logger.error("Failed to start VulnAPI testbed: %s", exc)
|
|
3348
|
+
return EXIT_FAILURES
|
|
3349
|
+
else:
|
|
3350
|
+
print(f" ✅ VulnAPI testbed already running at {vulnapi_url}")
|
|
3351
|
+
|
|
3352
|
+
try:
|
|
3353
|
+
return _run_demo_url(
|
|
3354
|
+
url=vulnapi_url,
|
|
3355
|
+
headless=headless,
|
|
3356
|
+
output_dir=output_dir,
|
|
3357
|
+
json_summary=json_summary,
|
|
3358
|
+
)
|
|
3359
|
+
finally:
|
|
3360
|
+
if _server_proc is not None:
|
|
3361
|
+
_server_proc.terminate()
|
|
3362
|
+
print(" 🛑 VulnAPI testbed stopped.")
|
|
3363
|
+
|
|
3364
|
+
|
|
3365
|
+
def _run_demo_url(
|
|
3366
|
+
url: str,
|
|
3367
|
+
headless: bool,
|
|
3368
|
+
output_dir: str,
|
|
3369
|
+
json_summary: str,
|
|
3370
|
+
) -> int:
|
|
3371
|
+
"""Run the autonomous browser pipeline against *url*."""
|
|
3372
|
+
import json as _json
|
|
3373
|
+
|
|
3374
|
+
from mannf.core.functional_orchestrator import FunctionalTestOrchestrator
|
|
3375
|
+
|
|
3376
|
+
print(f"\n 🚀 NAT Demo Mode — autonomous browser pipeline")
|
|
3377
|
+
print(f" 🎯 Target: {url}")
|
|
3378
|
+
print(f" 🖥️ Browser: {'headed' if not headless else 'headless'}\n")
|
|
3379
|
+
|
|
3380
|
+
tasks = [
|
|
3381
|
+
{
|
|
3382
|
+
"url": url,
|
|
3383
|
+
"steps": [
|
|
3384
|
+
{"action": "navigate", "url": url},
|
|
3385
|
+
],
|
|
3386
|
+
}
|
|
3387
|
+
]
|
|
3388
|
+
|
|
3389
|
+
orch = FunctionalTestOrchestrator(
|
|
3390
|
+
num_browser_agents=1,
|
|
3391
|
+
browser_type="chromium",
|
|
3392
|
+
headless=headless,
|
|
3393
|
+
max_tasks=len(tasks),
|
|
3394
|
+
)
|
|
3395
|
+
|
|
3396
|
+
try:
|
|
3397
|
+
report = asyncio.run(orch.run(tasks))
|
|
3398
|
+
except KeyboardInterrupt:
|
|
3399
|
+
print("\nDemo interrupted.", file=sys.stderr)
|
|
3400
|
+
return 130
|
|
3401
|
+
except Exception as exc: # noqa: BLE001
|
|
3402
|
+
logger.error("Browser pipeline failed: %s", exc)
|
|
3403
|
+
return EXIT_FAILURES
|
|
3404
|
+
|
|
3405
|
+
report_paths = orch.generate_report(output_dir=output_dir)
|
|
3406
|
+
report["report_paths"] = report_paths
|
|
3407
|
+
|
|
3408
|
+
passed = report.get("passed", 0)
|
|
3409
|
+
failed = report.get("failed", 0)
|
|
3410
|
+
total = report.get("total", 0)
|
|
3411
|
+
|
|
3412
|
+
print(f"\n ✅ Demo complete — {passed}/{total} tasks passed, {failed} failed")
|
|
3413
|
+
if report_paths:
|
|
3414
|
+
print(" Reports written to:")
|
|
3415
|
+
for p in report_paths:
|
|
3416
|
+
print(f" 📄 {p}")
|
|
3417
|
+
|
|
3418
|
+
if json_summary:
|
|
3419
|
+
try:
|
|
3420
|
+
import pathlib
|
|
3421
|
+
|
|
3422
|
+
pathlib.Path(json_summary).parent.mkdir(parents=True, exist_ok=True)
|
|
3423
|
+
pathlib.Path(json_summary).write_text(_json.dumps(report, indent=2), encoding="utf-8")
|
|
3424
|
+
print(f" 📄 JSON summary written to: {json_summary}")
|
|
3425
|
+
except Exception as exc: # noqa: BLE001
|
|
3426
|
+
logger.warning("Could not write JSON summary to %s: %s", json_summary, exc)
|
|
3427
|
+
|
|
3428
|
+
return EXIT_OK if failed == 0 else EXIT_FAILURES
|
|
3429
|
+
|
|
3430
|
+
|
|
3431
|
+
def _register_autonomous_schedule(args: argparse.Namespace, cron_expression: str) -> int:
|
|
3432
|
+
"""Register an autonomous sweep schedule in the ScheduleStore and exit.
|
|
3433
|
+
|
|
3434
|
+
This uses the in-process store singleton when available (i.e. when the
|
|
3435
|
+
server is embedded in the same process) or prints a summary for manual
|
|
3436
|
+
registration.
|
|
3437
|
+
"""
|
|
3438
|
+
import uuid as _uuid # noqa: PLC0415
|
|
3439
|
+
|
|
3440
|
+
from mannf.product.scheduling.cron_utils import validate_cron # noqa: PLC0415
|
|
3441
|
+
|
|
3442
|
+
try:
|
|
3443
|
+
validate_cron(cron_expression)
|
|
3444
|
+
except Exception as exc: # noqa: BLE001
|
|
3445
|
+
print(f"\n ❌ Invalid cron expression '{cron_expression}': {exc}", file=sys.stderr)
|
|
3446
|
+
return EXIT_CONFIG_ERROR
|
|
3447
|
+
|
|
3448
|
+
schedule_id = str(_uuid.uuid4())
|
|
3449
|
+
tenant_id = getattr(args, "tenant_id", None) or "00000000-0000-0000-0000-000000000000"
|
|
3450
|
+
name = (
|
|
3451
|
+
getattr(args, "schedule_name", None)
|
|
3452
|
+
or f"Autonomous sweep — {args.url}"
|
|
3453
|
+
)
|
|
3454
|
+
|
|
3455
|
+
schedule_dict = {
|
|
3456
|
+
"schedule_id": schedule_id,
|
|
3457
|
+
"name": name,
|
|
3458
|
+
"schedule_type": "autonomous",
|
|
3459
|
+
"cron_expression": cron_expression,
|
|
3460
|
+
"timezone": "UTC",
|
|
3461
|
+
"tenant_id": tenant_id,
|
|
3462
|
+
"spec_path": "",
|
|
3463
|
+
"spec_format": "openapi",
|
|
3464
|
+
"base_url": args.url,
|
|
3465
|
+
"enabled": True,
|
|
3466
|
+
"scan_config": {
|
|
3467
|
+
"strategy": args.strategy,
|
|
3468
|
+
"max_iterations": args.max_iterations,
|
|
3469
|
+
"max_duration_s": args.max_duration_s,
|
|
3470
|
+
"max_pages": args.max_pages,
|
|
3471
|
+
"max_depth": args.max_depth,
|
|
3472
|
+
"recrawl_every": args.recrawl_every,
|
|
3473
|
+
},
|
|
3474
|
+
}
|
|
3475
|
+
|
|
3476
|
+
# Attempt to register with the live store
|
|
3477
|
+
registered = False
|
|
3478
|
+
try:
|
|
3479
|
+
from mannf.product.scheduling.store import get_store # noqa: PLC0415
|
|
3480
|
+
from mannf.product.scheduling.models import ScanSchedule # noqa: PLC0415
|
|
3481
|
+
from datetime import datetime as _dt, timezone as _tz # noqa: PLC0415
|
|
3482
|
+
|
|
3483
|
+
store = get_store()
|
|
3484
|
+
schedule = ScanSchedule(
|
|
3485
|
+
schedule_id=schedule_id,
|
|
3486
|
+
tenant_id=tenant_id,
|
|
3487
|
+
name=name,
|
|
3488
|
+
schedule_type="autonomous",
|
|
3489
|
+
cron_expression=cron_expression,
|
|
3490
|
+
timezone="UTC",
|
|
3491
|
+
spec_path="",
|
|
3492
|
+
spec_format="openapi",
|
|
3493
|
+
base_url=args.url,
|
|
3494
|
+
enabled=True,
|
|
3495
|
+
scan_config=schedule_dict["scan_config"],
|
|
3496
|
+
created_at=_dt.now(_tz.utc).isoformat(),
|
|
3497
|
+
updated_at=_dt.now(_tz.utc).isoformat(),
|
|
3498
|
+
)
|
|
3499
|
+
store.create(schedule, plan_tier="enterprise")
|
|
3500
|
+
registered = True
|
|
3501
|
+
except Exception: # noqa: BLE001
|
|
3502
|
+
pass
|
|
3503
|
+
|
|
3504
|
+
output_fmt: str = getattr(args, "output", "text")
|
|
3505
|
+
if output_fmt == "json":
|
|
3506
|
+
print(json.dumps({"registered": registered, "schedule": schedule_dict}, indent=2))
|
|
3507
|
+
else:
|
|
3508
|
+
print(f"\n 📅 Autonomous sweep scheduled")
|
|
3509
|
+
print(f" Schedule ID: {schedule_id}")
|
|
3510
|
+
print(f" Name: {name}")
|
|
3511
|
+
print(f" Target: {args.url}")
|
|
3512
|
+
print(f" Cron: {cron_expression}")
|
|
3513
|
+
print(f" Strategy: {args.strategy}")
|
|
3514
|
+
if registered:
|
|
3515
|
+
print(f" ✅ Registered with in-process scheduler")
|
|
3516
|
+
else:
|
|
3517
|
+
print(f" ℹ️ Use the NAT API to register this schedule on a running server")
|
|
3518
|
+
print()
|
|
3519
|
+
|
|
3520
|
+
return EXIT_OK
|
|
3521
|
+
|
|
3522
|
+
|
|
3523
|
+
async def _run_point_and_fire(args: argparse.Namespace) -> int:
|
|
3524
|
+
"""Execute the *point-and-fire* sub-command (autonomous test loop)."""
|
|
3525
|
+
schedule_cron: Optional[str] = getattr(args, "schedule", None)
|
|
3526
|
+
|
|
3527
|
+
# --schedule mode: register a recurring sweep and exit
|
|
3528
|
+
if schedule_cron:
|
|
3529
|
+
return _register_autonomous_schedule(args, schedule_cron)
|
|
3530
|
+
|
|
3531
|
+
from mannf.core.agents.autonomous_loop_agent import AutonomousTestLoopAgent # noqa: PLC0415
|
|
3532
|
+
|
|
3533
|
+
llm_provider = _build_llm_provider(args)
|
|
3534
|
+
|
|
3535
|
+
agent = AutonomousTestLoopAgent(
|
|
3536
|
+
target_url=args.url,
|
|
3537
|
+
max_iterations=args.max_iterations,
|
|
3538
|
+
max_duration_s=args.max_duration_s,
|
|
3539
|
+
stability_threshold=args.stability_threshold,
|
|
3540
|
+
strategy=args.strategy,
|
|
3541
|
+
risk_threshold=args.risk_threshold,
|
|
3542
|
+
max_pages=args.max_pages,
|
|
3543
|
+
max_depth=args.max_depth,
|
|
3544
|
+
headless=not args.no_headless,
|
|
3545
|
+
llm_provider=llm_provider,
|
|
3546
|
+
recrawl_every=args.recrawl_every,
|
|
3547
|
+
)
|
|
3548
|
+
|
|
3549
|
+
output_fmt: str = getattr(args, "output", "text")
|
|
3550
|
+
|
|
3551
|
+
print(f"\n 🤖 NAT Autonomous Test Loop")
|
|
3552
|
+
print(f" Target: {args.url}")
|
|
3553
|
+
print(f" Strategy: {args.strategy}")
|
|
3554
|
+
print(f" Max iters: {args.max_iterations}")
|
|
3555
|
+
print(f" Budget: {args.max_duration_s:.0f}s")
|
|
3556
|
+
print()
|
|
3557
|
+
|
|
3558
|
+
try:
|
|
3559
|
+
report = await agent.run_loop()
|
|
3560
|
+
except KeyboardInterrupt:
|
|
3561
|
+
print("\n\n 👋 Autonomous run interrupted.\n")
|
|
3562
|
+
return EXIT_SCAN_ERROR
|
|
3563
|
+
except Exception as exc: # noqa: BLE001
|
|
3564
|
+
print(f"\n ❌ Autonomous run failed: {exc}", file=sys.stderr)
|
|
3565
|
+
return EXIT_SCAN_ERROR
|
|
3566
|
+
|
|
3567
|
+
if output_fmt == "json":
|
|
3568
|
+
print(json.dumps(report.model_dump(), indent=2))
|
|
3569
|
+
return EXIT_OK
|
|
3570
|
+
|
|
3571
|
+
# Human-readable summary
|
|
3572
|
+
print(f" ✅ Run complete — stop reason: {report.stop_reason}")
|
|
3573
|
+
print(f" Iterations: {report.total_iterations}")
|
|
3574
|
+
print(f" Scenarios: {report.total_scenarios_executed} executed")
|
|
3575
|
+
print(f" Passed: {report.total_passed}")
|
|
3576
|
+
print(f" Failed: {report.total_failed}")
|
|
3577
|
+
print(f" Flaky: {report.total_flaky}")
|
|
3578
|
+
print(f" Duration: {report.duration_s:.1f}s")
|
|
3579
|
+
print()
|
|
3580
|
+
|
|
3581
|
+
if report.iterations:
|
|
3582
|
+
print(" Iteration summary:")
|
|
3583
|
+
for it in report.iterations:
|
|
3584
|
+
status = "✅" if it.failed == 0 else "❌"
|
|
3585
|
+
print(
|
|
3586
|
+
f" {status} [{it.iteration:2d}] "
|
|
3587
|
+
f"exec={it.scenarios_executed:3d} "
|
|
3588
|
+
f"pass={it.passed:3d} "
|
|
3589
|
+
f"fail={it.failed:3d} "
|
|
3590
|
+
f"new={it.new_failures:2d} "
|
|
3591
|
+
f"({it.duration_s:.1f}s)"
|
|
3592
|
+
)
|
|
3593
|
+
print()
|
|
3594
|
+
|
|
3595
|
+
if report.unique_failures:
|
|
3596
|
+
print(f" Unique failures ({len(report.unique_failures)}):")
|
|
3597
|
+
for f in report.unique_failures[:10]:
|
|
3598
|
+
print(f" • {f.get('page', f.get('url', '?'))}: {f.get('error', '')[:80]}")
|
|
3599
|
+
if len(report.unique_failures) > 10:
|
|
3600
|
+
print(f" … and {len(report.unique_failures) - 10} more")
|
|
3601
|
+
print()
|
|
3602
|
+
|
|
3603
|
+
return EXIT_FAILURES if report.total_failed > 0 else EXIT_OK
|
|
3604
|
+
|
|
3605
|
+
|
|
3606
|
+
async def _run_import(args: argparse.Namespace) -> int:
|
|
3607
|
+
"""Execute the *import* sub-command (batch artifact ingestion)."""
|
|
3608
|
+
import pathlib # noqa: PLC0415
|
|
3609
|
+
from mannf.product.ingestors.loader import ( # noqa: PLC0415
|
|
3610
|
+
get_ingestor_by_name,
|
|
3611
|
+
get_ingestor_for_source,
|
|
3612
|
+
load_all_ingestors,
|
|
3613
|
+
)
|
|
3614
|
+
from mannf.product.ingestors.models import IngestorSource # noqa: PLC0415
|
|
3615
|
+
from mannf.core.browser.ingestor_bridge import IngestorBridge # noqa: PLC0415
|
|
3616
|
+
from mannf.product.orchestrator import _guess_format # noqa: PLC0415
|
|
3617
|
+
|
|
3618
|
+
source_path = pathlib.Path(args.source)
|
|
3619
|
+
import_format: str = args.import_format
|
|
3620
|
+
base_url: str = getattr(args, "base_url", "") or ""
|
|
3621
|
+
output_fmt: str = getattr(args, "output", "text")
|
|
3622
|
+
replay_mode: bool = getattr(args, "replay_mode", False)
|
|
3623
|
+
out_file: Optional[str] = getattr(args, "out_file", None)
|
|
3624
|
+
|
|
3625
|
+
# --- Collect files to ingest ---
|
|
3626
|
+
files_to_ingest: list[pathlib.Path] = []
|
|
3627
|
+
if source_path.is_dir():
|
|
3628
|
+
# Recursively discover all files; filter later by format
|
|
3629
|
+
for f in sorted(source_path.rglob("*")):
|
|
3630
|
+
if f.is_file():
|
|
3631
|
+
files_to_ingest.append(f)
|
|
3632
|
+
elif source_path.is_file():
|
|
3633
|
+
files_to_ingest.append(source_path)
|
|
3634
|
+
else:
|
|
3635
|
+
print(
|
|
3636
|
+
f"\n ❌ Source path not found: {source_path}\n",
|
|
3637
|
+
file=sys.stderr,
|
|
3638
|
+
)
|
|
3639
|
+
return EXIT_CONFIG_ERROR
|
|
3640
|
+
|
|
3641
|
+
if not files_to_ingest:
|
|
3642
|
+
print(f"\n ❌ No files found in {source_path}\n", file=sys.stderr)
|
|
3643
|
+
return EXIT_CONFIG_ERROR
|
|
3644
|
+
|
|
3645
|
+
all_ingestors = load_all_ingestors()
|
|
3646
|
+
|
|
3647
|
+
results_summary: list[dict] = []
|
|
3648
|
+
all_scenarios: list[dict] = []
|
|
3649
|
+
total_files = 0
|
|
3650
|
+
total_scenarios = 0
|
|
3651
|
+
skipped_files = 0
|
|
3652
|
+
|
|
3653
|
+
if output_fmt == "text":
|
|
3654
|
+
print(f"\n 📥 NAT Batch Import")
|
|
3655
|
+
print(f" Source: {source_path}")
|
|
3656
|
+
print(f" Format: {import_format}")
|
|
3657
|
+
if base_url:
|
|
3658
|
+
print(f" Base URL: {base_url}")
|
|
3659
|
+
print()
|
|
3660
|
+
|
|
3661
|
+
for file_path in files_to_ingest:
|
|
3662
|
+
# Determine format
|
|
3663
|
+
if import_format == "auto":
|
|
3664
|
+
guessed = _guess_format(str(file_path))
|
|
3665
|
+
if guessed is None:
|
|
3666
|
+
# Try by filename pattern for playwright/cypress
|
|
3667
|
+
fn = file_path.name.lower()
|
|
3668
|
+
if ".spec." in fn or ".test." in fn:
|
|
3669
|
+
if fn.endswith(".ts") or fn.endswith(".js"):
|
|
3670
|
+
guessed = "playwright"
|
|
3671
|
+
elif fn.endswith(".cy.js") or fn.endswith(".cy.ts"):
|
|
3672
|
+
guessed = "cypress"
|
|
3673
|
+
if guessed is None:
|
|
3674
|
+
skipped_files += 1
|
|
3675
|
+
if output_fmt == "text":
|
|
3676
|
+
print(f" ⚠️ Skipped (unknown format): {file_path.name}")
|
|
3677
|
+
continue
|
|
3678
|
+
fmt = guessed
|
|
3679
|
+
else:
|
|
3680
|
+
fmt = import_format
|
|
3681
|
+
|
|
3682
|
+
# Build source
|
|
3683
|
+
source = IngestorSource(
|
|
3684
|
+
format=fmt,
|
|
3685
|
+
path=str(file_path),
|
|
3686
|
+
metadata={"filename": file_path.name},
|
|
3687
|
+
)
|
|
3688
|
+
|
|
3689
|
+
# Find ingestor
|
|
3690
|
+
ingestor = get_ingestor_for_source(source, all_ingestors)
|
|
3691
|
+
if ingestor is None:
|
|
3692
|
+
skipped_files += 1
|
|
3693
|
+
if output_fmt == "text":
|
|
3694
|
+
print(f" ⚠️ Skipped (no ingestor for format={fmt!r}): {file_path.name}")
|
|
3695
|
+
continue
|
|
3696
|
+
|
|
3697
|
+
# Run ingestor
|
|
3698
|
+
ingest_config: dict = {}
|
|
3699
|
+
if replay_mode and ingestor.name == "har":
|
|
3700
|
+
ingest_config["replay_mode"] = True
|
|
3701
|
+
|
|
3702
|
+
try:
|
|
3703
|
+
ingest_result = await ingestor.ingest(source, ingest_config)
|
|
3704
|
+
except Exception as exc: # noqa: BLE001
|
|
3705
|
+
skipped_files += 1
|
|
3706
|
+
if output_fmt == "text":
|
|
3707
|
+
print(f" ❌ Failed ({ingestor.name}): {file_path.name} — {exc}")
|
|
3708
|
+
continue
|
|
3709
|
+
|
|
3710
|
+
if not ingest_result.success and not ingest_result.test_cases and not ingest_result.endpoints:
|
|
3711
|
+
skipped_files += 1
|
|
3712
|
+
if output_fmt == "text":
|
|
3713
|
+
errors_str = "; ".join(ingest_result.errors) if ingest_result.errors else "no results"
|
|
3714
|
+
print(f" ⚠️ Empty result: {file_path.name} — {errors_str}")
|
|
3715
|
+
continue
|
|
3716
|
+
|
|
3717
|
+
# Convert to scenario dicts via IngestorBridge
|
|
3718
|
+
bridge = IngestorBridge(base_url=base_url or "https://example.com", source_tag=ingestor.name)
|
|
3719
|
+
scenarios = bridge.convert(ingest_result)
|
|
3720
|
+
|
|
3721
|
+
all_scenarios.extend(scenarios)
|
|
3722
|
+
total_files += 1
|
|
3723
|
+
total_scenarios += len(scenarios)
|
|
3724
|
+
|
|
3725
|
+
file_summary = {
|
|
3726
|
+
"file": str(file_path),
|
|
3727
|
+
"format": fmt,
|
|
3728
|
+
"ingestor": ingestor.name,
|
|
3729
|
+
"endpoints": len(ingest_result.endpoints),
|
|
3730
|
+
"test_cases": len(ingest_result.test_cases),
|
|
3731
|
+
"scenarios": len(scenarios),
|
|
3732
|
+
}
|
|
3733
|
+
results_summary.append(file_summary)
|
|
3734
|
+
|
|
3735
|
+
if output_fmt == "text":
|
|
3736
|
+
print(
|
|
3737
|
+
f" ✅ {file_path.name} [{ingestor.name}] "
|
|
3738
|
+
f"→ {len(scenarios)} scenario(s)"
|
|
3739
|
+
)
|
|
3740
|
+
|
|
3741
|
+
if output_fmt == "text":
|
|
3742
|
+
print()
|
|
3743
|
+
print(f" Summary: {total_files} file(s) imported, {total_scenarios} scenario(s) generated")
|
|
3744
|
+
if skipped_files:
|
|
3745
|
+
print(f" {skipped_files} file(s) skipped (unrecognized or empty)")
|
|
3746
|
+
print()
|
|
3747
|
+
|
|
3748
|
+
# Write results
|
|
3749
|
+
output_data = {
|
|
3750
|
+
"total_files": total_files,
|
|
3751
|
+
"total_scenarios": total_scenarios,
|
|
3752
|
+
"skipped_files": skipped_files,
|
|
3753
|
+
"files": results_summary,
|
|
3754
|
+
"scenarios": all_scenarios,
|
|
3755
|
+
}
|
|
3756
|
+
|
|
3757
|
+
if out_file:
|
|
3758
|
+
out_path = pathlib.Path(out_file)
|
|
3759
|
+
out_path.write_text(json.dumps(output_data, indent=2), encoding="utf-8")
|
|
3760
|
+
if output_fmt == "text":
|
|
3761
|
+
print(f" 💾 Scenarios written to: {out_path}\n")
|
|
3762
|
+
elif output_fmt == "json":
|
|
3763
|
+
print(json.dumps(output_data, indent=2))
|
|
3764
|
+
|
|
3765
|
+
return EXIT_OK if total_files > 0 else EXIT_CONFIG_ERROR
|
|
3766
|
+
|
|
3767
|
+
|
|
3768
|
+
def main(argv: Optional[list] = None) -> None:
|
|
3769
|
+
"""Entry point for ``nat`` CLI and ``python -m mannf``."""
|
|
3770
|
+
# ------------------------------------------------------------------
|
|
3771
|
+
# Step 1: Pre-parse to find --config before the full parser runs.
|
|
3772
|
+
# This allows the config file to set defaults that CLI flags override.
|
|
3773
|
+
# ------------------------------------------------------------------
|
|
3774
|
+
if argv is None:
|
|
3775
|
+
argv = sys.argv[1:]
|
|
3776
|
+
|
|
3777
|
+
pre_parser = argparse.ArgumentParser(add_help=False)
|
|
3778
|
+
pre_parser.add_argument("--config", default=None)
|
|
3779
|
+
pre_parser.add_argument("--version", "-V", action="version", version=f"nat {__version__}")
|
|
3780
|
+
pre_args, _ = pre_parser.parse_known_args(argv)
|
|
3781
|
+
|
|
3782
|
+
# ------------------------------------------------------------------
|
|
3783
|
+
# Step 2: Load config file and apply as defaults.
|
|
3784
|
+
# ------------------------------------------------------------------
|
|
3785
|
+
try:
|
|
3786
|
+
config = _load_natrc(pre_args.config)
|
|
3787
|
+
except (OSError, ValueError) as exc:
|
|
3788
|
+
print(f"Configuration error: {exc}", file=sys.stderr)
|
|
3789
|
+
sys.exit(EXIT_CONFIG_ERROR)
|
|
3790
|
+
|
|
3791
|
+
parser = _build_parser()
|
|
3792
|
+
|
|
3793
|
+
if config:
|
|
3794
|
+
_apply_config_defaults(parser, config)
|
|
3795
|
+
|
|
3796
|
+
# ------------------------------------------------------------------
|
|
3797
|
+
# Step 3: Parse argv. Map argparse's exit-2 (bad usage) → exit 3.
|
|
3798
|
+
# ------------------------------------------------------------------
|
|
3799
|
+
# Strip --config from argv since it's a top-level flag handled above;
|
|
3800
|
+
# individual subcommand parsers don't know about it.
|
|
3801
|
+
clean_argv = []
|
|
3802
|
+
skip_next = False
|
|
3803
|
+
for arg in argv:
|
|
3804
|
+
if skip_next:
|
|
3805
|
+
skip_next = False
|
|
3806
|
+
continue
|
|
3807
|
+
if arg == "--config":
|
|
3808
|
+
skip_next = True
|
|
3809
|
+
continue
|
|
3810
|
+
if arg.startswith("--config="):
|
|
3811
|
+
continue
|
|
3812
|
+
clean_argv.append(arg)
|
|
3813
|
+
|
|
3814
|
+
try:
|
|
3815
|
+
args = parser.parse_args(clean_argv)
|
|
3816
|
+
except SystemExit as exc:
|
|
3817
|
+
sys.exit(EXIT_CONFIG_ERROR if exc.code == 2 else (exc.code or 0))
|
|
3818
|
+
|
|
3819
|
+
# Propagate config path for downstream use
|
|
3820
|
+
args.config = pre_args.config
|
|
3821
|
+
|
|
3822
|
+
# ------------------------------------------------------------------
|
|
3823
|
+
# Step 4: Validate required fields that may be satisfied by config.
|
|
3824
|
+
# ------------------------------------------------------------------
|
|
3825
|
+
if args.command in ("scan", "security-scan"):
|
|
3826
|
+
if not getattr(args, "base_url", None):
|
|
3827
|
+
print(
|
|
3828
|
+
"error: --base-url is required (or set base_url in .natrc)",
|
|
3829
|
+
file=sys.stderr,
|
|
3830
|
+
)
|
|
3831
|
+
sys.exit(EXIT_CONFIG_ERROR)
|
|
3832
|
+
|
|
3833
|
+
# ------------------------------------------------------------------
|
|
3834
|
+
# Step 5: Dispatch.
|
|
3835
|
+
# ------------------------------------------------------------------
|
|
3836
|
+
log_level = logging.INFO if getattr(args, "verbose", False) else logging.WARNING
|
|
3837
|
+
logging.basicConfig(level=log_level, format="%(levelname)s %(name)s: %(message)s")
|
|
3838
|
+
|
|
3839
|
+
if args.command == "scan":
|
|
3840
|
+
exit_code = asyncio.run(_run_scan(args))
|
|
3841
|
+
sys.exit(exit_code)
|
|
3842
|
+
elif args.command == "security-scan":
|
|
3843
|
+
exit_code = asyncio.run(_run_security_scan(args))
|
|
3844
|
+
sys.exit(exit_code)
|
|
3845
|
+
elif args.command == "weights":
|
|
3846
|
+
exit_code = _run_weights(args)
|
|
3847
|
+
sys.exit(exit_code)
|
|
3848
|
+
elif args.command == "export":
|
|
3849
|
+
exit_code = _run_export(args)
|
|
3850
|
+
sys.exit(exit_code)
|
|
3851
|
+
elif args.command == "heal":
|
|
3852
|
+
exit_code = _run_heal(args)
|
|
3853
|
+
sys.exit(exit_code)
|
|
3854
|
+
elif args.command == "test-gen":
|
|
3855
|
+
exit_code = asyncio.run(_run_test_gen(args))
|
|
3856
|
+
sys.exit(exit_code)
|
|
3857
|
+
elif args.command == "completions":
|
|
3858
|
+
exit_code = _run_completions(args)
|
|
3859
|
+
sys.exit(exit_code)
|
|
3860
|
+
elif args.command == "demo":
|
|
3861
|
+
exit_code = _run_demo(args)
|
|
3862
|
+
sys.exit(exit_code)
|
|
3863
|
+
elif args.command == "setup":
|
|
3864
|
+
from mannf.product.setup_wizard import run_setup_wizard # noqa: PLC0415
|
|
3865
|
+
exit_code = asyncio.run(run_setup_wizard(args))
|
|
3866
|
+
sys.exit(exit_code)
|
|
3867
|
+
elif args.command == "upgrade":
|
|
3868
|
+
from mannf.product.upgrade import run_upgrade # noqa: PLC0415
|
|
3869
|
+
exit_code = run_upgrade(args)
|
|
3870
|
+
sys.exit(exit_code)
|
|
3871
|
+
elif args.command == "uninstall":
|
|
3872
|
+
from mannf.product.uninstall import run_uninstall # noqa: PLC0415
|
|
3873
|
+
exit_code = run_uninstall(args)
|
|
3874
|
+
sys.exit(exit_code)
|
|
3875
|
+
elif args.command == "doctor":
|
|
3876
|
+
from mannf.product.doctor import run_doctor # noqa: PLC0415
|
|
3877
|
+
exit_code = asyncio.run(run_doctor(args))
|
|
3878
|
+
sys.exit(exit_code)
|
|
3879
|
+
elif args.command == "point-and-fire":
|
|
3880
|
+
exit_code = asyncio.run(_run_point_and_fire(args))
|
|
3881
|
+
sys.exit(exit_code)
|
|
3882
|
+
elif args.command == "import":
|
|
3883
|
+
exit_code = asyncio.run(_run_import(args))
|
|
3884
|
+
sys.exit(exit_code)
|
|
3885
|
+
elif args.command == "worker":
|
|
3886
|
+
exit_code = _run_worker(args)
|
|
3887
|
+
sys.exit(exit_code)
|
|
3888
|
+
elif args.command == "plan":
|
|
3889
|
+
exit_code = asyncio.run(_run_plan(args))
|
|
3890
|
+
sys.exit(exit_code)
|
|
3891
|
+
elif args.command == "test":
|
|
3892
|
+
exit_code = asyncio.run(_run_nl_test(args))
|
|
3893
|
+
sys.exit(exit_code)
|
|
3894
|
+
else:
|
|
3895
|
+
parser.print_help()
|
|
3896
|
+
sys.exit(EXIT_CONFIG_ERROR)
|
|
3897
|
+
|
|
3898
|
+
|
|
3899
|
+
if __name__ == "__main__":
|
|
3900
|
+
main()
|