hackagent 0.7.0__tar.gz → 0.8.0__tar.gz

{hackagent-0.7.0 → hackagent-0.8.0}/.gitignore

@@ -134,3 +134,4 @@ dmypy.json
 
 
 .copilotignore
+tests/e2e/attacks/

hackagent-0.8.0/PKG-INFO

@@ -0,0 +1,154 @@
+Metadata-Version: 2.4
+Name: hackagent
+Version: 0.8.0
+Summary: HackAgent is an open-source security toolkit to detect vulnerabilities of your AI Agents.
+Author-email: AI Security Lab <ais@ai4i.it>
+License: Apache-2.0
+License-File: LICENSE
+Keywords: agents,ai,security,testing,vulnerabilities
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1.0
+Requires-Dist: datasets>=2.14.0
+Requires-Dist: faiss-cpu>=1.13.2
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: litellm>=1.69.2
+Requires-Dist: nicegui>=2.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: pydantic[email]>=2.0
+Requires-Dist: python-dateutil>=2.8.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: rich>=14.0.0
+Requires-Dist: textual>=1.0.0
+Description-Content-Type: text/markdown
+
+<div align="center">
+<p align="center">
+  <img src="https://docs.hackagent.dev/img/banner.svg" alt="HackAgent - AI Agent Security Testing Toolkit" width="800">
+</p>
+
+  <strong>AI Security Red-Team Toolkit</strong>
+
+<br>
+
+[App](https://app.hackagent.dev/) -- [Docs](https://docs.hackagent.dev/) -- [API](https://api.hackagent.dev/schema/redoc)
+
+<br>
+
+![Python Version](https://img.shields.io/badge/python-3.10%2B-blue)
+![License](https://img.shields.io/badge/license-Apache%202.0-green)
+![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)
+[![Commitizen](https://img.shields.io/badge/commitizen-friendly-brightgreen.svg)](http://commitizen.github.io/cz-cli/)
+![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)
+![Test Coverage](https://img.shields.io/codecov/c/github/AISecurityLab/hackagent)
+![CI Status](https://img.shields.io/github/actions/workflow/status/AISecurityLab/hackagent/ci.yml)
+</div>
+
+## What is HackAgent?
+
+HackAgent is a comprehensive Python SDK and CLI designed to help security researchers, developers, and AI safety practitioners evaluate and strengthen the security of AI agents.
+
+
+
+As AI agents become more powerful and autonomous, they face security challenges that traditional testing tools cannot address:
+
+| Threat | Description |
+|--------|-------------|
+| **Prompt Injection** | Malicious inputs that hijack agent behavior |
+| **Jailbreaking** | Bypassing safety guardrails and content filters |
+| **Goal Hijacking** | Manipulating agents to pursue unintended objectives |
+| **Tool Misuse** | Exploiting agent capabilities for unauthorized actions |
+
+HackAgent automates testing for these vulnerabilities using research-backed attack techniques, helping you identify and fix security issues before they are exploited.
+
+<div align="center">
+  <img src="docs/static/gifs/terminal.gif" alt="HackAgent CLI Demo" width="100%" />
+  <p><em>Interactive TUI with real-time attack progress and visual reporting.</em></p>
+</div>
+
+## Get Started Now
+
+### Quick Install
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install hackagent
+```
+
+No API key required: HackAgent works locally out of the box.
+
+
+Questions? Join [community discussions](https://github.com/AISecurityLab/hackagent/discussions) or email ais@ai4i.it.
+
+## Architecture
+
+HackAgent uses a modular pipeline to test agent robustness end-to-end.
+
+| Component | Description |
+|-----------|-------------|
+| **Attack Engine** | Orchestrates attacks using AdvPrefix, AutoDAN-Turbo, PAIR, TAP, FlipAttack, BoN, h4rm3l, CipherChat, PAP, and Baseline |
+| **Generator** | LLM role that creates adversarial prompts to test the target agent |
+| **Judge** | LLM role that evaluates whether attacks bypass safety measures |
+| **Target Agent** | Your AI agent under test across supported frameworks |
+| **Datasets** | Pre-built benchmark presets plus custom HuggingFace/file datasets |
+
+## Supported Frameworks
+
+[![Google ADK](https://img.shields.io/badge/Google-ADK-green?style=for-the-badge&logo=google)](https://google.github.io/adk-docs/)
+[![OpenAI SDK](https://img.shields.io/badge/OpenAI-SDK-412991?style=for-the-badge&logo=openai)](https://platform.openai.com/docs)
+[![LiteLLM](https://img.shields.io/badge/LiteLLM-blue?style=for-the-badge&logo=github)](https://github.com/BerriAI/litellm)
+[![LangChain](https://img.shields.io/badge/LangChain-1C3C3C?style=for-the-badge)](https://python.langchain.com)
+
+## Reporting
+
+HackAgent supports both local and remote reporting.
+
+- Local mode stores test results in SQLite and includes a built-in dashboard.
+- Cloud mode syncs runs to the HackAgent remote platform when an API key is configured.
+
+```bash
+hackagent web
+```
+
+Access cloud reporting at [https://app.hackagent.dev](https://app.hackagent.dev).
+
+## Responsible Use
+
+HackAgent is designed for authorized security testing only. Always obtain explicit permission before testing any AI system.
+
+### Do
+
+- Test your own agents
+- Conduct authorized pentesting
+- Follow coordinated disclosure
+- Share security knowledge responsibly
+
+### Don't
+
+- Test systems without permission
+- Exploit vulnerabilities maliciously
+- Violate terms of service
+- Share harmful exploit instructions irresponsibly
+
+Read the full guidelines: [Responsible Disclosure](docs/docs/security/responsible-disclosure.md)
+
+## Contributing
+
+Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) and [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md).
+
+## License
+
+Licensed under Apache-2.0. See [LICENSE](LICENSE).
+
+## Disclaimer
+
+HackAgent is intended for security research and AI safety improvement. The authors are not responsible for misuse.

hackagent-0.8.0/README.md

@@ -0,0 +1,122 @@
+<div align="center">
+<p align="center">
+  <img src="https://docs.hackagent.dev/img/banner.svg" alt="HackAgent - AI Agent Security Testing Toolkit" width="800">
+</p>
+
+  <strong>AI Security Red-Team Toolkit</strong>
+
+<br>
+
+[App](https://app.hackagent.dev/) -- [Docs](https://docs.hackagent.dev/) -- [API](https://api.hackagent.dev/schema/redoc)
+
+<br>
+
+![Python Version](https://img.shields.io/badge/python-3.10%2B-blue)
+![License](https://img.shields.io/badge/license-Apache%202.0-green)
+![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)
+[![Commitizen](https://img.shields.io/badge/commitizen-friendly-brightgreen.svg)](http://commitizen.github.io/cz-cli/)
+![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)
+![Test Coverage](https://img.shields.io/codecov/c/github/AISecurityLab/hackagent)
+![CI Status](https://img.shields.io/github/actions/workflow/status/AISecurityLab/hackagent/ci.yml)
+</div>
+
+## What is HackAgent?
+
+HackAgent is a comprehensive Python SDK and CLI designed to help security researchers, developers, and AI safety practitioners evaluate and strengthen the security of AI agents.
+
+
+
+As AI agents become more powerful and autonomous, they face security challenges that traditional testing tools cannot address:
+
+| Threat | Description |
+|--------|-------------|
+| **Prompt Injection** | Malicious inputs that hijack agent behavior |
+| **Jailbreaking** | Bypassing safety guardrails and content filters |
+| **Goal Hijacking** | Manipulating agents to pursue unintended objectives |
+| **Tool Misuse** | Exploiting agent capabilities for unauthorized actions |
+
+HackAgent automates testing for these vulnerabilities using research-backed attack techniques, helping you identify and fix security issues before they are exploited.
+
+<div align="center">
+  <img src="docs/static/gifs/terminal.gif" alt="HackAgent CLI Demo" width="100%" />
+  <p><em>Interactive TUI with real-time attack progress and visual reporting.</em></p>
+</div>
+
+## Get Started Now
+
+### Quick Install
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install hackagent
+```
+
+No API key required: HackAgent works locally out of the box.
+
+
+Questions? Join [community discussions](https://github.com/AISecurityLab/hackagent/discussions) or email ais@ai4i.it.
+
+## Architecture
+
+HackAgent uses a modular pipeline to test agent robustness end-to-end.
+
+| Component | Description |
+|-----------|-------------|
+| **Attack Engine** | Orchestrates attacks using AdvPrefix, AutoDAN-Turbo, PAIR, TAP, FlipAttack, BoN, h4rm3l, CipherChat, PAP, and Baseline |
+| **Generator** | LLM role that creates adversarial prompts to test the target agent |
+| **Judge** | LLM role that evaluates whether attacks bypass safety measures |
+| **Target Agent** | Your AI agent under test across supported frameworks |
+| **Datasets** | Pre-built benchmark presets plus custom HuggingFace/file datasets |
+
+## Supported Frameworks
+
+[![Google ADK](https://img.shields.io/badge/Google-ADK-green?style=for-the-badge&logo=google)](https://google.github.io/adk-docs/)
+[![OpenAI SDK](https://img.shields.io/badge/OpenAI-SDK-412991?style=for-the-badge&logo=openai)](https://platform.openai.com/docs)
+[![LiteLLM](https://img.shields.io/badge/LiteLLM-blue?style=for-the-badge&logo=github)](https://github.com/BerriAI/litellm)
+[![LangChain](https://img.shields.io/badge/LangChain-1C3C3C?style=for-the-badge)](https://python.langchain.com)
+
+## Reporting
+
+HackAgent supports both local and remote reporting.
+
+- Local mode stores test results in SQLite and includes a built-in dashboard.
+- Cloud mode syncs runs to the HackAgent remote platform when an API key is configured.
+
+```bash
+hackagent web
+```
+
+Access cloud reporting at [https://app.hackagent.dev](https://app.hackagent.dev).
+
+## Responsible Use
+
+HackAgent is designed for authorized security testing only. Always obtain explicit permission before testing any AI system.
+
+### Do
+
+- Test your own agents
+- Conduct authorized pentesting
+- Follow coordinated disclosure
+- Share security knowledge responsibly
+
+### Don't
+
+- Test systems without permission
+- Exploit vulnerabilities maliciously
+- Violate terms of service
+- Share harmful exploit instructions irresponsibly
+
+Read the full guidelines: [Responsible Disclosure](docs/docs/security/responsible-disclosure.md)
+
+## Contributing
+
+Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) and [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md).
+
+## License
+
+Licensed under Apache-2.0. See [LICENSE](LICENSE).
+
+## Disclaimer
+
+HackAgent is intended for security research and AI safety improvement. The authors are not responsible for misuse.

{hackagent-0.7.0 → hackagent-0.8.0}/hackagent/__init__.py

@@ -1,7 +1,7 @@
 # Copyright 2026 - AI4I. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-"""A client library for accessing HackAgent API"""
+"""A client library for HackAgent — AI Agent Security Testing"""
 
 from .agent import HackAgent
 from .server.client import AuthenticatedClient, Client

{hackagent-0.7.0 → hackagent-0.8.0}/hackagent/agent.py

@@ -8,7 +8,6 @@ from hackagent import utils
 from hackagent.errors import HackAgentError
 from hackagent.router import AgentRouter
 from hackagent.router.types import AgentTypeEnum
-from hackagent.server.storage.base import StorageBackend
 
 # Lazy import for attack orchestrators to avoid ~0.5s startup delay
 if TYPE_CHECKING:
@@ -44,12 +43,11 @@ class HackAgent:
     - Executing automated security tests against the configured agents.
     - Retrieving and handling test results.
 
-    It encapsulates complexities such as API authentication, agent registration
-    with the backend (via `AgentRouter`), and the dynamic dispatch of various
+    It encapsulates complexities such as agent registration
+    with the local backend (via `AgentRouter`), and the dynamic dispatch of various
     attack methodologies.
 
     Attributes:
-        client: An `AuthenticatedClient` instance for API communication.
         router: An `AgentRouter` instance managing the agent's representation
             in the HackAgent backend.
         attack_strategies: A dictionary mapping strategy names to their
@@ -72,7 +70,7 @@ class HackAgent:
         """
         Initializes the HackAgent client and prepares it for interaction.
 
-        This constructor sets up the authenticated API client, loads default
+        This constructor sets up the local storage backend, loads default
         prompts, resolves the agent type, and initializes the agent router
         to ensure the agent is known to the backend. It also prepares available
         attack strategies.
@@ -90,10 +88,6 @@ class HackAgent:
                 String values are automatically converted to the corresponding
                 `AgentTypeEnum` member. Defaults to `AgentTypeEnum.UNKNOWN` if
                 not specified or if an invalid string is provided.
-            base_url: The base URL for the HackAgent API service.
-            api_key: The API key for authenticating with the HackAgent API.
-                If omitted, the client will attempt to retrieve it from the
-                config file (~/.config/hackagent/config.json).
             raise_on_unexpected_status: If set to `True`, the API client will
                 raise an exception for any HTTP status codes that are not typically
                 expected for a successful operation. Defaults to `False`.
@@ -122,7 +116,7 @@ class HackAgent:
                 raise_on_unexpected_status=raise_on_unexpected_status,
                 timeout=timeout,
             )
-            self.backend: StorageBackend = RemoteBackend(_client)
+            self.backend = RemoteBackend(_client)
             logger.info("HackAgent using remote backend → %s", _base_url)
         else:
             from hackagent.server.storage.local import LocalBackend
@@ -132,8 +126,7 @@ class HackAgent:
                 "HackAgent using local backend → ~/.local/share/hackagent/hackagent.db"
             )
 
-        # Keep self.client as the raw HTTP client for backward compat
-        # (adapters that need it can access it via backend.get_api_key())
+        # Backward compatible raw HTTP client reference.
         self.client = getattr(self.backend, "_client", None)
 
         processed_agent_type = utils.resolve_agent_type(agent_type)
@@ -189,16 +182,16 @@ class HackAgent:
             )
 
             self._attack_strategies = {
-                "advprefix": AdvPrefixOrchestrator(hack_agent=self),
-                "autodan_turbo": AutoDANTurboOrchestrator(hack_agent=self),
-                "baseline": BaselineOrchestrator(hack_agent=self),
-                "bon": BoNOrchestrator(hack_agent=self),
-                "cipherchat": CipherChatOrchestrator(hack_agent=self),
-                "pair": PAIROrchestrator(hack_agent=self),
-                "flipattack": FlipAttackOrchestrator(hack_agent=self),
-                "tap": TAPOrchestrator(hack_agent=self),
-                "h4rm3l": H4rm3lOrchestrator(hack_agent=self),
-                "pap": PAPOrchestrator(hack_agent=self),
+                "advprefix": AdvPrefixOrchestrator(hackagent_agent=self),
+                "autodan_turbo": AutoDANTurboOrchestrator(hackagent_agent=self),
+                "baseline": BaselineOrchestrator(hackagent_agent=self),
+                "bon": BoNOrchestrator(hackagent_agent=self),
+                "cipherchat": CipherChatOrchestrator(hackagent_agent=self),
+                "pair": PAIROrchestrator(hackagent_agent=self),
+                "flipattack": FlipAttackOrchestrator(hackagent_agent=self),
+                "tap": TAPOrchestrator(hackagent_agent=self),
+                "h4rm3l": H4rm3lOrchestrator(hackagent_agent=self),
+                "pap": PAPOrchestrator(hackagent_agent=self),
             }
         return self._attack_strategies
 
@@ -238,7 +231,7 @@ class HackAgent:
             ValueError: If the 'attack_type' is missing from `attack_config` or
                 if the specified 'attack_type' is not a supported/registered
                 strategy.
-            HackAgentError: For issues during API interaction, problems with backend
+            HackAgentError: For issues during backend
                 agent operations, or other unexpected errors during the attack process.
         """
         try:

{hackagent-0.7.0 → hackagent-0.8.0}/hackagent/attacks/evaluator/base.py

@@ -255,18 +255,36 @@ class BaseJudgeEvaluator(ABC):
         for idx, row in enumerate(data):
             row["_original_index"] = idx
 
+        # Preserve upstream execution errors and skip judge calls for those rows.
+        rows_error = [
+            row
+            for row in data
+            if row.get("is_error") is True
+            or (
+                (row.get("error") or row.get("error_message"))
+                and not str(row.get("completion") or "").strip()
+            )
+        ]
+
+        non_error_rows = [row for row in data if row not in rows_error]
+
         # Split into filtered and processable
         rows_trivial = [
             row
-            for row in data
+            for row in non_error_rows
             if self._is_trivial_completion(row.get("completion", ""))
         ]
         rows_non_trivial = [
             row
-            for row in data
+            for row in non_error_rows
             if not self._is_trivial_completion(row.get("completion", ""))
         ]
 
+        for row in rows_error:
+            row[self.eval_column] = 0
+            err = row.get("error") or row.get("error_message") or "Unknown error"
+            row[self.explanation_column] = f"execution/adapter error: {err}"
+
         if self.skip_length_filter:
             rows_to_filter: List[Dict[str, Any]] = []
             rows_to_process = rows_non_trivial
@@ -296,7 +314,7 @@ class BaseJudgeEvaluator(ABC):
 
         self.logger.info(
             f"Evaluation split: "
-            f"total={len(data)}  filtered_trivial={len(rows_trivial)}  "
+            f"total={len(data)}  errors={len(rows_error)}  filtered_trivial={len(rows_trivial)}  "
             f"filtered_short={len(rows_to_filter)}  "
             f"to_process={len(rows_to_process)}"
         )