hackagent 0.11.0__tar.gz → 0.12.0__tar.gz

{hackagent-0.11.0 → hackagent-0.12.0}/.gitignore

@@ -7,6 +7,7 @@ reports/
 slurm/
 tools/
 docs/docs/attacks/walkthroughs/
+.worktrees/
 
 # Editors
 .vscode/
@@ -54,6 +55,8 @@ MANIFEST
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
+# The release build spec is checked in, unlike generated ones.
+!packaging/hackagent.spec
 
 # Installer logs
 pip-log.txt
@@ -139,3 +142,6 @@ dmypy.json
 tests/e2e/attacks/
 
 .tmp
+
+# pytest-textual-snapshot report artifact
+snapshot_report.html

{hackagent-0.11.0 → hackagent-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hackagent
-Version: 0.11.0
+Version: 0.12.0
 Summary: HackAgent is an open-source security toolkit to detect vulnerabilities of your AI Agents.
 Author-email: AI Security Lab <ais@ai4i.it>
 License: Apache-2.0
@@ -18,6 +18,7 @@ Requires-Python: >=3.10
 Requires-Dist: click>=8.1.0
 Requires-Dist: datasets>=2.14.0
 Requires-Dist: faiss-cpu>=1.13.2
+Requires-Dist: flask>=3.1.2
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: litellm>=1.69.2
 Requires-Dist: nicegui>=2.0
@@ -91,6 +92,27 @@ pip install hackagent
 
 No API key required: HackAgent works locally out of the box.
 
+### Standalone binary (no Python required)
+
+Every [GitHub Release](https://github.com/AISecurityLab/hackagent/releases) ships a
+self-contained archive per platform:
+
+| Platform | Asset |
+|----------|-------|
+| Linux x86_64 | `hackagent-<version>-linux-x86_64.tar.gz` |
+| macOS arm64 | `hackagent-<version>-macos-arm64.tar.gz` |
+| macOS x86_64 | `hackagent-<version>-macos-x86_64.tar.gz` |
+| Windows x86_64 | `hackagent-<version>-windows-x86_64.zip` |
+
+Extract the archive and run the `hackagent` launcher inside it — no Python, `pip`
+or `uv` install needed. Keep the extracted folder intact; the launcher loads the
+libraries next to it.
+
+> **Note:** the `WEB` provider drives a real browser through Playwright, whose
+> browser binaries cannot be embedded in the archive (or in the PyPI package).
+> Before using `hackagent web`-based targets, install them once with
+> `playwright install`.
+
 
 Questions? Join [community discussions](https://github.com/AISecurityLab/hackagent/discussions) or email ais@ai4i.it.
 
@@ -104,7 +126,7 @@ HackAgent uses a modular pipeline to test agent robustness end-to-end.
 | **Generator** | LLM role that creates adversarial prompts to test the target agent |
 | **Judge** | LLM role that evaluates whether attacks bypass safety measures |
 | **Target Agent** | Your AI agent under test across supported frameworks |
-| **Datasets** | Pre-built benchmark presets plus custom HuggingFace/file datasets |
+| **Datasets** | Pre-built benchmark presets plus custom HuggingFace/file/URL JSON datasets |
 
 ## Supported Frameworks
 

{hackagent-0.11.0 → hackagent-0.12.0}/README.md

@@ -58,6 +58,27 @@ pip install hackagent
 
 No API key required: HackAgent works locally out of the box.
 
+### Standalone binary (no Python required)
+
+Every [GitHub Release](https://github.com/AISecurityLab/hackagent/releases) ships a
+self-contained archive per platform:
+
+| Platform | Asset |
+|----------|-------|
+| Linux x86_64 | `hackagent-<version>-linux-x86_64.tar.gz` |
+| macOS arm64 | `hackagent-<version>-macos-arm64.tar.gz` |
+| macOS x86_64 | `hackagent-<version>-macos-x86_64.tar.gz` |
+| Windows x86_64 | `hackagent-<version>-windows-x86_64.zip` |
+
+Extract the archive and run the `hackagent` launcher inside it — no Python, `pip`
+or `uv` install needed. Keep the extracted folder intact; the launcher loads the
+libraries next to it.
+
+> **Note:** the `WEB` provider drives a real browser through Playwright, whose
+> browser binaries cannot be embedded in the archive (or in the PyPI package).
+> Before using `hackagent web`-based targets, install them once with
+> `playwright install`.
+
 
 Questions? Join [community discussions](https://github.com/AISecurityLab/hackagent/discussions) or email ais@ai4i.it.
 
@@ -71,7 +92,7 @@ HackAgent uses a modular pipeline to test agent robustness end-to-end.
 | **Generator** | LLM role that creates adversarial prompts to test the target agent |
 | **Judge** | LLM role that evaluates whether attacks bypass safety measures |
 | **Target Agent** | Your AI agent under test across supported frameworks |
-| **Datasets** | Pre-built benchmark presets plus custom HuggingFace/file datasets |
+| **Datasets** | Pre-built benchmark presets plus custom HuggingFace/file/URL JSON datasets |
 
 ## Supported Frameworks
 

hackagent-0.12.0/hackagent/_version.py

@@ -0,0 +1,21 @@
+# Copyright 2026 - AI4I. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Version lookup that also works from a frozen (PyInstaller) binary."""
+
+import os
+from importlib.metadata import PackageNotFoundError, version as _distribution_version
+
+UNKNOWN_VERSION = "unknown"
+
+
+def get_version() -> str:
+    """Return the installed ``hackagent`` version.
+
+    Frozen builds may ship without distribution metadata, so fall back to the
+    ``HACKAGENT_BUILD_VERSION`` value baked in at packaging time.
+    """
+    try:
+        return _distribution_version("hackagent")
+    except PackageNotFoundError:
+        return os.environ.get("HACKAGENT_BUILD_VERSION") or UNKNOWN_VERSION

{hackagent-0.11.0 → hackagent-0.12.0}/hackagent/agent.py

@@ -62,13 +62,14 @@ class HackAgent:
         base_url: Optional[str] = None,
         api_key: Optional[str] = None,
         raise_on_unexpected_status: bool = False,
-        timeout: Optional[float] = None,
+        timeout: Optional[float] = 120.0,
         metadata: Optional[Dict[str, Any]] = None,
         target_config: Optional[Dict[str, Any]] = None,
         adapter_operational_config: Optional[Dict[str, Any]] = None,
         thinking: Optional[bool] = None,
         before_guardrail: Optional[Dict[str, Any]] = None,
         after_guardrail: Optional[Dict[str, Any]] = None,
+        backend: Optional[Any] = None,
     ):
         """
         Initializes the HackAgent client and prepares it for interaction.
@@ -95,8 +96,11 @@ class HackAgent:
                 raise an exception for any HTTP status codes that are not typically
                 expected for a successful operation. Defaults to `False`.
             timeout: The timeout duration in seconds for API requests made by the
-                authenticated client. Defaults to `None` (which might mean a
-                default timeout from the underlying HTTP library is used).
+                authenticated (remote) HackAgent backend client. Defaults to
+                `120.0` seconds so requests to a misbehaving/unreachable backend
+                fail predictably instead of hanging indefinitely. Pass `None`
+                explicitly to opt out and disable the timeout (unbounded wait,
+                the previous default behavior).
             metadata: Optional dictionary containing agent-specific metadata.
             target_config: Optional default request settings for the configured
                 victim model. This is the preferred place to define target-side
@@ -107,11 +111,21 @@ class HackAgent:
                 When set to `False`, requests sent through the target OLLAMA adapter
                 include `think: false` to disable thinking output. Ignored for
                 non-OLLAMA target agent types.
+            backend: Optional pre-built ``StorageBackend`` to persist runs and
+                results through. When omitted, a backend is selected from the
+                resolved API key (remote) or a default local SQLite database.
+                Supplying one lets an embedding host — e.g. the local dashboard
+                — reuse its own already-open backend.
         """
 
         resolved_auth_token = utils.resolve_api_token(direct_api_key_param=api_key)
 
-        if resolved_auth_token:
+        if backend is not None:
+            self.backend = backend
+            logger.info(
+                "HackAgent using caller-provided backend %s", type(backend).__name__
+            )
+        elif resolved_auth_token:
             from hackagent.server.client import AuthenticatedClient
             from hackagent.server.storage.remote import RemoteBackend
 
@@ -320,3 +334,231 @@ class HackAgent:
             raise HackAgentError(
                 f"An unexpected error occurred during attack: {e}"
             ) from e
+
+    def hack_chain(
+        self,
+        attacks: Optional[list] = None,
+        goals: Optional[list] = None,
+        run_config_override: Optional[Dict[str, Any]] = None,
+        fail_on_run_error: bool = True,
+        escalate_only_mitigated: bool = True,
+        _tui_event_bus: Optional[Any] = None,
+    ) -> list:
+        """
+        Runs a sequence of attack strategies against a shared pool of goals.
+
+        By default (``escalate_only_mitigated=True``) this implements a
+        "fallback ladder": every goal starts at ``attacks[0]``. Any goal for
+        which the victim's response is judged successful (a jailbreak/
+        violation) is considered resolved and is dropped from the chain — it
+        is never retried. Any goal that is mitigated (the victim's response
+        is judged safe) is carried over and retried with ``attacks[1]``, then
+        ``attacks[2]``, and so on, until either the goal succeeds or the
+        chain is exhausted.
+
+        With ``escalate_only_mitigated=False``, every goal is instead sent to
+        *every* attack in the chain regardless of outcome — useful for
+        running several attacks against the same goal set and collecting all
+        of their results in one call, rather than escalating only failures.
+
+        Success/mitigation is determined per goal from the evaluated result
+        rows returned by each step (see
+        ``hackagent.attacks.evaluator.metrics.is_successful_result``): a goal
+        is considered successful for a step if *any* of its result rows for
+        that step are judged successful.
+
+        Args:
+            attacks: Ordered list of ``attack_config`` dicts, one per chain
+                step, using the same shape accepted by :meth:`hack` (each
+                must include its own ``attack_type`` and any attack-specific
+                settings). Only the *first* entry needs to specify how goals
+                are sourced (``goals``, ``dataset`` or ``intents``) unless
+                the ``goals`` parameter below is provided; subsequent steps
+                automatically receive only the goals still mitigated by the
+                previous step (or all goals, see ``escalate_only_mitigated``).
+                Defaults to ``None``, which resolves to the Jailbreak
+                evaluation campaign's primary attacks, in order — ``h4rm3l``
+                → ``TAP`` → ``PAIR`` (see
+                ``hackagent.risks.jailbreak.JAILBREAK_PROFILE``). A goal
+                source is still required either way, via ``goals`` or a
+                ``dataset``/``goals``/``intents`` key on the first step.
+            goals: Optional explicit list of goal strings to use for the
+                whole chain. When provided, it takes precedence over any
+                ``goals``/``dataset``/``intents`` set on ``attacks[0]``.
+            run_config_override: Optional run configuration overrides applied
+                to every step, forwarded to :meth:`hack`.
+            fail_on_run_error: Forwarded to :meth:`hack` for every step.
+            escalate_only_mitigated: When ``True`` (default), a goal only
+                moves on to the next attack if it was mitigated at the
+                current step — goals that already succeeded are dropped, and
+                each goal's final result is either its first success or its
+                last (final) attempt. When ``False``, every goal is sent to
+                every attack regardless of outcome, and results from *all*
+                steps are kept for every goal (nothing is dropped or
+                overwritten).
+
+        Returns:
+            A flat list of result rows (same row shape as :meth:`hack`),
+            grouped by original goal, in first-seen order. Each row is
+            tagged with ``chain_step`` (0-based index into ``attacks``) and
+            ``chain_attack_type`` identifying which attack produced it.
+
+        Raises:
+            HackAgentError: If ``attacks`` is empty, or a step is missing
+                ``attack_type``.
+        """
+        if attacks is None:
+            # Default chain: the Jailbreak evaluation campaign's primary
+            # attacks, in campaign order. `technique` strings in the profile
+            # (e.g. "TAP", "PAIR") use display casing; `attack_strategies`
+            # keys are lowercase, so normalize before use.
+            from hackagent.risks.jailbreak import JAILBREAK_PROFILE
+
+            attacks = [
+                {"attack_type": rec.technique.strip().lower()}
+                for rec in JAILBREAK_PROFILE.primary_attacks
+            ]
+
+        if not attacks:
+            raise HackAgentError(
+                "'attacks' must be a non-empty list of attack_config dicts."
+            )
+
+        from hackagent.attacks.evaluator.metrics import is_successful_result
+
+        n_steps = len(attacks)
+        remaining_goals: Optional[list] = list(goals) if goals is not None else None
+        goal_order: list = list(goals) if goals is not None else []
+        final_rows_by_goal: Dict[str, list] = {}
+
+        for step_index, step_config in enumerate(attacks):
+            if remaining_goals is not None and not remaining_goals:
+                logger.info(
+                    "All goals resolved before chain step %d/%d; skipping remaining attack(s).",
+                    step_index + 1,
+                    n_steps,
+                )
+                break
+
+            attack_type = step_config.get("attack_type")
+            if not attack_type:
+                raise HackAgentError(
+                    f"hack_chain step {step_index} is missing 'attack_type'."
+                )
+
+            step_attack_config = dict(step_config)
+            if remaining_goals is not None:
+                step_attack_config["goals"] = remaining_goals
+                step_attack_config.pop("dataset", None)
+                step_attack_config.pop("intents", None)
+
+            logger.info(
+                "hack_chain step %d/%d: running '%s' against %s goal(s)",
+                step_index + 1,
+                n_steps,
+                attack_type,
+                len(remaining_goals) if remaining_goals is not None else "all",
+            )
+
+            # Each step is executed through the same public `hack()` entry
+            # point used everywhere else (CLI/TUI/SDK) — hack_chain adds no
+            # parallel execution path, it only decides *which* goals each
+            # step receives and tags/regroups the returned rows afterwards.
+            step_results = self.hack(
+                attack_config=step_attack_config,
+                run_config_override=run_config_override,
+                fail_on_run_error=fail_on_run_error,
+                _tui_event_bus=_tui_event_bus,
+            )
+            step_rows = (
+                step_results
+                if isinstance(step_results, list)
+                else list(step_results or [])
+            )
+
+            # Group returned rows by their `goal` text — the same lookup
+            # convention used throughout the codebase to associate a result
+            # back to its goal (e.g. Tracker.get_goal_context_by_goal,
+            # TrackingCoordinator, BaseEvaluationStep's MERGE_KEYS). Every
+            # technique preserves the original goal string verbatim in its
+            # output rows, so this is a plain, reliable lookup — no need to
+            # reconstruct or second-guess it.
+            #
+            # Grouping (rather than checking each row independently) matters
+            # because a single goal can have several rows in one step — not
+            # from multiple judges (those are already merged into per-judge
+            # columns, e.g. eval_hb/eval_jb, on the *same* row, with the
+            # majority vote across them resolved by is_successful_result()
+            # before we ever see it here) but from multiple distinct attempts
+            # at the same goal (e.g. AdvPrefix's several generated prefixes,
+            # BoN's N augmented variants, TAP/PAIR's tree branches), each its
+            # own (goal, prefix, completion) row. "Mitigated" means *all* of
+            # that goal's attempts failed; "succeeded" means *any* attempt
+            # did. A flat `for row in step_rows: if not successful: ...`
+            # can't express that OR-across-attempts semantics, and would
+            # also append the same goal multiple times (once per failing
+            # row) without an explicit dedup step.
+            rows_by_goal: Dict[str, list] = {}
+            for row in step_rows:
+                if not isinstance(row, dict):
+                    continue
+                row_goal = row.get("goal", "unknown")
+                rows_by_goal.setdefault(row_goal, []).append(
+                    {**row, "chain_step": step_index, "chain_attack_type": attack_type}
+                )
+
+            if remaining_goals is None:
+                # First step resolved its own goals internally (via 'goals',
+                # 'dataset' or 'intents' on attacks[0]) — dict insertion
+                # order already matches first-seen order, so reuse it
+                # directly instead of tracking a parallel order list.
+                remaining_goals = list(rows_by_goal.keys())
+                goal_order = list(remaining_goals)
+
+            next_remaining: list = []
+            # Iterate the goals we *sent* this step (`remaining_goals`), not
+            # `rows_by_goal`/`step_rows` directly: a goal that comes back
+            # with zero rows (e.g. the attack errored on it) must still be
+            # looked up here — `rows_by_goal.get(goal, [])` defaults to `[]`,
+            # and `not any([])` correctly keeps it mitigated/in-chain. If we
+            # only walked what actually came back in `step_rows`, a goal
+            # with no response would simply never appear and would silently
+            # fall out of the chain, as if it had succeeded.
+            for goal in remaining_goals:
+                goal_rows = rows_by_goal.get(goal, [])
+
+                if not escalate_only_mitigated:
+                    # Every goal proceeds to every attack regardless of
+                    # outcome; keep rows from all steps instead of only the
+                    # latest attempt.
+                    final_rows_by_goal.setdefault(goal, []).extend(goal_rows)
+                    next_remaining.append(goal)
+                    continue
+
+                if goal_rows:
+                    final_rows_by_goal[goal] = goal_rows
+                mitigated = not any(is_successful_result(row) for row in goal_rows)
+                if mitigated:
+                    next_remaining.append(goal)
+                else:
+                    logger.info(
+                        "Goal succeeded at chain step %d ('%s'); dropped from chain: %s",
+                        step_index + 1,
+                        attack_type,
+                        str(goal)[:80],
+                    )
+
+            remaining_goals = next_remaining
+
+        if remaining_goals and escalate_only_mitigated:
+            logger.info(
+                "%d goal(s) mitigated through the entire chain (%d step(s)): %s",
+                len(remaining_goals),
+                n_steps,
+                ", ".join(str(g)[:60] for g in remaining_goals),
+            )
+
+        final_results: list = []
+        for goal in goal_order:
+            final_results.extend(final_rows_by_goal.get(goal, []))
+        return final_results

{hackagent-0.11.0 → hackagent-0.12.0}/hackagent/attacks/base.py

@@ -2,7 +2,9 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import abc
-from typing import Any, Dict
+from typing import Any, Dict, List
+
+from hackagent.attacks.types import AttackResult
 
 
 class BaseAttack(abc.ABC):
@@ -64,7 +66,7 @@ class BaseAttack(abc.ABC):
         pass
 
     @abc.abstractmethod
-    def run(self, **kwargs: Any) -> Any:
+    def run(self, **kwargs: Any) -> List[AttackResult]:
         """
         Executes the attack logic.
 
@@ -80,12 +82,7 @@ class BaseAttack(abc.ABC):
                 - target_model: The model to attack
 
         Returns:
-            Attack-specific results. The format varies by implementation but
-            typically includes:
-            - adversarial_examples: Generated adversarial inputs
-            - success_metrics: Attack success rates and statistics
-            - detailed_results: Comprehensive result data (e.g., pandas DataFrame)
-            - attack_report: Summary of attack performance
+            A list of :class:`~hackagent.attacks.types.AttackResult` instances.
 
         Raises:
             NotImplementedError: If the method is not implemented by a subclass.

{hackagent-0.11.0 → hackagent-0.12.0}/hackagent/attacks/evaluator/__init__.py

@@ -38,6 +38,7 @@ Usage:
         calculate_majority_vote_asr,
         calculate_fleiss_kappa,
         calculate_per_judge_strictness,
+        is_successful_result,
         # Server sync
         sync_evaluation_to_server,
         update_single_result,
@@ -53,6 +54,7 @@ from hackagent.attacks.evaluator.judge_evaluators import (
     JailbreakBenchEvaluator,
     NuancedEvaluator,
     OnTopicEvaluator,
+    ScorerEvaluator,
 )
 from hackagent.attacks.evaluator.metrics import (
     calculate_confidence_score,
@@ -60,6 +62,7 @@ from hackagent.attacks.evaluator.metrics import (
     calculate_success_rate,
     generate_summary_report,
     group_by_goal,
+    is_successful_result,
     calculate_majority_vote_asr,
     calculate_fleiss_kappa,
     calculate_per_judge_strictness,
@@ -85,6 +88,7 @@ __all__ = [
     "HarmBenchEvaluator",
     "HarmBenchVariantEvaluator",
     "OnTopicEvaluator",
+    "ScorerEvaluator",
     "EVALUATOR_MAP",
     # Pattern evaluators
     "PatternEvaluator",
@@ -96,6 +100,7 @@ __all__ = [
     "calculate_per_goal_metrics",
     "generate_summary_report",
     "group_by_goal",
+    "is_successful_result",
     "calculate_majority_vote_asr",
     "calculate_fleiss_kappa",
     "calculate_per_judge_strictness",

{hackagent-0.11.0 → hackagent-0.12.0}/hackagent/attacks/evaluator/base.py

@@ -186,6 +186,19 @@ class BaseJudgeEvaluator(ABC):
             f"Registration key: {self.agent_registration_key}"
         )
 
+    def _build_messages(self, user_content: str) -> List[Dict[str, str]]:
+        """Build the judge chat messages, honouring a user-defined system prompt.
+
+        ``EvaluatorConfig.system_prompt`` lets users prepend their own system
+        message (e.g. a custom rubric or persona) to every judge request.
+        """
+        messages: List[Dict[str, str]] = []
+        system_prompt = getattr(self.config, "system_prompt", None)
+        if isinstance(system_prompt, str) and system_prompt.strip():
+            messages.append({"role": "system", "content": system_prompt.strip()})
+        messages.append({"role": "user", "content": user_content})
+        return messages
+
     def _verify_keys(self, data: list[dict], required_keys: list) -> None:
         """Verify that all required keys exist in each dict."""
         if not data:
@@ -853,7 +866,7 @@ Answer (yes or no):"""
         )
 
         return {
-            "messages": [{"role": "user", "content": feedback_prompt}],
+            "messages": self._build_messages(feedback_prompt),
             "max_tokens": self.config.max_tokens_eval,
             "temperature": 0.0,  # Deterministic for retry
         }