hackagent 0.2.3__tar.gz → 0.2.5__tar.gz

{hackagent-0.2.3 → hackagent-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hackagent
-Version: 0.2.3
+Version: 0.2.5
 Summary: HackAgent is an open-source security toolkit to detect vulnerabilities of your AI Agents.
 Author: Nicola Franco
 Author-email: nicola@vista-labs.ai
@@ -9,17 +9,19 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: click (>=8.1.0,<9.0.0)
 Requires-Dist: litellm (>=1.69.2,<2.0.0)
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: pydantic (>=2.0,<3.0)
 Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
+Requires-Dist: pyyaml (>=6.0.0,<7.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: rich (>=14.0.0,<15.0.0)
 Description-Content-Type: text/markdown
 
 <div align="center">
 
-<img src="./assets/banner.png" alt="Hack Agent" width=400></img>
+<img src="https://docs.hackagent.dev/img/banner.png" alt="Hack Agent" width=400></img>
 
 
   ⚔️
@@ -28,10 +30,10 @@ Description-Content-Type: text/markdown
 
 <br>
 
-![ico](./assets/favicon.ico) [Web App][Web App] -- [Docs][Docs] ![ico](./assets/favicon.ico)
+![ico](https://docs.hackagent.dev/img/favicon.ico) [Web App][Web App] -- [Docs][Docs] ![ico](https://docs.hackagent.dev/img/favicon.ico)
 
 [Web App]: https://hackagent.dev/
-[Docs]: https://hackagent.dev/docs/
+[Docs]: https://docs.hackagent.dev/
 
 <br>
 

{hackagent-0.2.3 → hackagent-0.2.5}/README.md

@@ -1,6 +1,6 @@
 <div align="center">
 
-<img src="./assets/banner.png" alt="Hack Agent" width=400></img>
+<img src="https://docs.hackagent.dev/img/banner.png" alt="Hack Agent" width=400></img>
 
 
   ⚔️
@@ -9,10 +9,10 @@
 
 <br>
 
-![ico](./assets/favicon.ico) [Web App][Web App] -- [Docs][Docs] ![ico](./assets/favicon.ico)
+![ico](https://docs.hackagent.dev/img/favicon.ico) [Web App][Web App] -- [Docs][Docs] ![ico](https://docs.hackagent.dev/img/favicon.ico)
 
 [Web App]: https://hackagent.dev/
-[Docs]: https://hackagent.dev/docs/
+[Docs]: https://docs.hackagent.dev/
 
 <br>
 

{hackagent-0.2.3 → hackagent-0.2.5}/hackagent/agent.py

@@ -100,7 +100,6 @@ class HackAgent:
                 variables (such as `HACKAGENT_API_KEY`) will be loaded from this
                 file if not already present in the environment.
         """
-        utils.display_hackagent_splash()
 
         resolved_auth_token = utils.resolve_api_token(
             direct_api_key_param=api_key, env_file_path=env_file_path

hackagent-0.2.5/hackagent/attacks/AdvPrefix/__init__.py

@@ -0,0 +1,13 @@
+"""
+AdvPrefix Attack Module
+
+Suppress pandas warnings for cleaner attack execution output.
+"""
+
+import warnings
+
+# Suppress pandas FutureWarnings specifically for groupby operations
+# This addresses warnings from preprocessing operations in the AdvPrefix pipeline
+warnings.filterwarnings(
+    "ignore", category=FutureWarning, message=".*include_groups.*", module="pandas.*"
+)

{hackagent-0.2.3 → hackagent-0.2.5}/hackagent/attacks/AdvPrefix/completions.py

@@ -6,6 +6,16 @@ from typing import Dict, Any, Optional, List  # Added List
 # --- Import AgentRouter and related components ---
 from hackagent.router.router import AgentRouter, AgentTypeEnum
 
+# --- Import Rich progress bar components ---
+from rich.progress import (
+    Progress,
+    BarColumn,
+    TextColumn,
+    TimeRemainingColumn,
+    MofNCompleteColumn,
+    SpinnerColumn,
+)
+
 
 # Constants for surrogate prompts
 SURROGATE_ATTACK_PROMPTS = {
@@ -247,50 +257,67 @@ def execute(
     completion_results_list: List[Dict[str, Any]] = []
     logger.info(f"Executing {len(input_df)} completion requests sequentially...")
 
-    for index, row in input_df.iterrows():
-        prefix_text = row["prefix"]
-        # 'goal' might not be directly used if surrogate_prompt_template is complex or prefix_text is already combined
-        # goal_text = row.get("goal", "") # Ensure goal is available if needed by prompt construction
+    # Create progress bar for agent interactions
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        MofNCompleteColumn(),
+        TextColumn("[progress.percentage]{task.percentage:>3.1f}%"),
+        TimeRemainingColumn(),
+    ) as progress_bar:
+        task = progress_bar.add_task(
+            f"[green]Step 6: Getting completions from {victim_agent_type.value} agent...",
+            total=len(input_df),
+        )
 
-        try:
-            # n_samples handling: If n_samples_per_prefix > 1, the _get_completion_via_router (and adapter) needs to support it.
-            # Currently, it makes one call per row in input_df. If input_df is already expanded for samples, this is fine.
-            # If input_df has one row per unique prefix, and n_samples_per_prefix > 1, this loop needs to run n_samples_per_prefix times
-            # or _get_completion_via_router must handle requesting n_samples from the adapter.
-            # Assuming input_df might be pre-expanded or n_samples=1 for this synchronous version for simplicity.
-            # If n_samples > 1 and not pre-expanded, this will only get 1 sample per prefix.
-            result = _get_completion_via_router(
-                agent_router=agent_router,
-                agent_reg_key=victim_agent_reg_key,
-                prefix_text=prefix_text,
-                surrogate_prompt_template=actual_surrogate_prompt_str,
-                user_id=step_user_id_adk,
-                session_id=step_session_id_adk,
-                request_timeout=request_timeout,
-                max_new_tokens=max_new_tokens,
-                temperature=temperature,
-                n_samples=1,  # Forcing 1 for this simple loop; adapter might take n_samples_per_prefix
-                logger_instance=logger,
-                original_index=index,
-            )
-            completion_results_list.append(result)
-        except Exception as e:
-            logger.error(
-                f"Exception during synchronous completion for original index {index}: {e}",
-                exc_info=e,
-            )
-            completion_results_list.append(
-                {
-                    "completion": None,
-                    "raw_request_payload": None,
-                    "raw_response_status": None,
-                    "raw_response_headers": None,
-                    "raw_response_body": None,
-                    "adapter_specific_events": None,
-                    "error_message": f"Sync Task Exception: {type(e).__name__} - {str(e)}",
-                    "log_message": None,
-                }
-            )
+        for index, row in input_df.iterrows():
+            prefix_text = row["prefix"]
+            # 'goal' might not be directly used if surrogate_prompt_template is complex or prefix_text is already combined
+            # goal_text = row.get("goal", "") # Ensure goal is available if needed by prompt construction
+
+            try:
+                # n_samples handling: If n_samples_per_prefix > 1, the _get_completion_via_router (and adapter) needs to support it.
+                # Currently, it makes one call per row in input_df. If input_df is already expanded for samples, this is fine.
+                # If input_df has one row per unique prefix, and n_samples_per_prefix > 1, this loop needs to run n_samples_per_prefix times
+                # or _get_completion_via_router must handle requesting n_samples from the adapter.
+                # Assuming input_df might be pre-expanded or n_samples=1 for this synchronous version for simplicity.
+                # If n_samples > 1 and not pre-expanded, this will only get 1 sample per prefix.
+                result = _get_completion_via_router(
+                    agent_router=agent_router,
+                    agent_reg_key=victim_agent_reg_key,
+                    prefix_text=prefix_text,
+                    surrogate_prompt_template=actual_surrogate_prompt_str,
+                    user_id=step_user_id_adk,
+                    session_id=step_session_id_adk,
+                    request_timeout=request_timeout,
+                    max_new_tokens=max_new_tokens,
+                    temperature=temperature,
+                    n_samples=1,  # Forcing 1 for this simple loop; adapter might take n_samples_per_prefix
+                    logger_instance=logger,
+                    original_index=index,
+                )
+                completion_results_list.append(result)
+            except Exception as e:
+                logger.error(
+                    f"Exception during synchronous completion for original index {index}: {e}",
+                    exc_info=e,
+                )
+                completion_results_list.append(
+                    {
+                        "completion": None,
+                        "raw_request_payload": None,
+                        "raw_response_status": None,
+                        "raw_response_headers": None,
+                        "raw_response_body": None,
+                        "adapter_specific_events": None,
+                        "error_message": f"Sync Task Exception: {type(e).__name__} - {str(e)}",
+                        "log_message": None,
+                    }
+                )
+
+            # Update progress bar after each completion
+            progress_bar.update(task, advance=1)
 
     logger.info("All completion requests processed.")
 

{hackagent-0.2.3 → hackagent-0.2.5}/hackagent/attacks/AdvPrefix/compute_ce.py

@@ -7,6 +7,16 @@ import uuid
 from hackagent.client import AuthenticatedClient
 from hackagent.router.router import AgentRouter, AgentTypeEnum
 
+# --- Import Rich progress bar components ---
+from rich.progress import (
+    Progress,
+    BarColumn,
+    TextColumn,
+    TimeRemainingColumn,
+    MofNCompleteColumn,
+    SpinnerColumn,
+)
+
 # --- Remove old ADK utility imports and ADK_REFUSAL_KEYWORDS import ---
 # from hackagent.api.utils import ADK_REFUSAL_KEYWORDS # Removed this import
 
@@ -112,38 +122,55 @@ def execute(
         f"Executing {len(input_df)} ADK acceptability scoring requests sequentially..."
     )
 
-    # Synchronous loop instead of asyncio.gather
-    for index, row in input_df.iterrows():
-        prefix = row["prefix"]
-        try:
-            result = _get_adk_acceptability_via_router(
-                router=agent_router,
-                agent_reg_key=victim_agent_reg_key,
-                prefix_text=prefix,
-                user_id=step_user_id,
-                session_id=step_session_id,
-                request_timeout=request_timeout,
-                logger_instance=logger,
-                original_index=index,
-            )
-            interaction_results_list.append(result)
-        except Exception as e:
-            logger.error(
-                f"Exception during synchronous ADK acceptability scoring for original index {index}: {e}",
-                exc_info=e,
-            )
-            interaction_results_list.append(
-                {
-                    "score": float("inf"),
-                    "request_payload": None,
-                    "response_status_code": None,
-                    "response_headers": None,
-                    "response_body_raw": None,
-                    "adk_events_list": None,
-                    "error_message": f"Sync Task Exception: {type(e).__name__} - {str(e)}",
-                    "log_message": None,
-                }
-            )
+    # Create progress bar for ADK acceptability scoring
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        MofNCompleteColumn(),
+        TextColumn("[progress.percentage]{task.percentage:>3.1f}%"),
+        TimeRemainingColumn(),
+    ) as progress_bar:
+        task = progress_bar.add_task(
+            f"[blue]Step 4: Computing cross-entropy via {agent_router.backend_agent.agent_type.value} agent...",
+            total=len(input_df),
+        )
+
+        # Synchronous loop instead of asyncio.gather
+        for index, row in input_df.iterrows():
+            prefix = row["prefix"]
+            try:
+                result = _get_adk_acceptability_via_router(
+                    router=agent_router,
+                    agent_reg_key=victim_agent_reg_key,
+                    prefix_text=prefix,
+                    user_id=step_user_id,
+                    session_id=step_session_id,
+                    request_timeout=request_timeout,
+                    logger_instance=logger,
+                    original_index=index,
+                )
+                interaction_results_list.append(result)
+            except Exception as e:
+                logger.error(
+                    f"Exception during synchronous ADK acceptability scoring for original index {index}: {e}",
+                    exc_info=e,
+                )
+                interaction_results_list.append(
+                    {
+                        "score": float("inf"),
+                        "request_payload": None,
+                        "response_status_code": None,
+                        "response_headers": None,
+                        "response_body_raw": None,
+                        "adk_events_list": None,
+                        "error_message": f"Sync Task Exception: {type(e).__name__} - {str(e)}",
+                        "log_message": None,
+                    }
+                )
+
+            # Update progress bar after each scoring request
+            progress_bar.update(task, advance=1)
 
     logger.info("All ADK acceptability scoring requests processed.")
 

{hackagent-0.2.3 → hackagent-0.2.5}/hackagent/attacks/AdvPrefix/config.py

@@ -6,7 +6,7 @@ DEFAULT_PREFIX_GENERATION_CONFIG: Dict[str, Any] = {
     "output_dir": "./logs/runs",
     # --- Model Configurations ---
     "generator": {
-        "identifier": "ollama/llama2-uncensored",
+        "identifier": "hackagent/generate",
         "endpoint": "https://hackagent.dev/api/generate",
         "batch_size": 2,
         "max_new_tokens": 50,
@@ -15,14 +15,14 @@ DEFAULT_PREFIX_GENERATION_CONFIG: Dict[str, Any] = {
     },
     "judges": [
         {
-            "identifier": "ollama/llama3",
+            "identifier": "hackagent/judge",
             "endpoint": "https://hackagent.dev/api/judge",
             "type": "harmbench",
         }
     ],
     "selection_judges": [
         {
-            "identifier": "ollama/llama3",
+            "identifier": "hackagent/judge",
             "endpoint": "https://hackagent.dev/api/judge",
             "type": "harmbench",
         }

{hackagent-0.2.3 → hackagent-0.2.5}/hackagent/attacks/AdvPrefix/generate.py

@@ -55,30 +55,7 @@ def _construct_prompts(
             if n_samples <= 0:
                 continue
 
-            # chat = [{"role": "user", "content": goal}] # Not directly used for router prompt format
             try:
-                # The prompt for the router will be the fully constructed context.
-                # Custom chat templating needs to happen before sending to router.
-                # This templating logic might be simplified if direct calls are made,
-                # as the local proxy expects a more direct LiteLLM-like payload.
-
-                # For direct calls, the "prompt" is often just the user message content.
-                # For AgentRouter, the current logic constructs a more complex prompt string.
-                # We will adapt this based on whether we're calling directly or via router.
-
-                # The `final_prompt` here is what's sent to LiteLLM or the router.
-                # For direct local proxy, `messages` will be constructed later.
-                # For AgentRouter, this `final_prompt` is used.
-
-                # Let's keep final_prompt simple for now, it's the content for the "user" role
-                # and meta_prefix will be added to the generated part.
-                # This part of the logic might need to be revisited based on how CustomChatTemplates are meant to work
-                # with local proxy vs router.
-
-                # The current _construct_prompts prepares a `final_prompt` string.
-                # Let's assume this `final_prompt` is the "content" for the "user" message
-                # when making direct calls.
-
                 if meta_prefix in CUSTOM_CHAT_TEMPLATES:
                     prompt_content_for_template = CUSTOM_CHAT_TEMPLATES[
                         meta_prefix

{hackagent-0.2.3 → hackagent-0.2.5}/hackagent/attacks/AdvPrefix/scorer_parser.py

@@ -77,46 +77,34 @@ class BaseEvaluator(ABC):
         self.underlying_httpx_client = self.client.get_httpx_client()
 
         self.is_local_judge_proxy_defined = False
-        self.actual_api_key: Optional[str] = None
+        self.actual_api_key: str = client.token
 
-        if self.config.agent_endpoint and (
-            "localhost:8888/api/judge" in self.config.agent_endpoint
-            or "127.0.0.1:8888/api/judge" in self.config.agent_endpoint
-        ):
+        api_key_config_value = self.config.agent_metadata.get("api_key")
+
+        if api_key_config_value:
+            env_key_value = os.environ.get(api_key_config_value)
+            if env_key_value:
+                self.actual_api_key = env_key_value
+                self.logger.info(
+                    f"Loaded API key for generator from environment variable: {api_key_config_value}"
+                )
+            else:
+                self.actual_api_key = api_key_config_value
+                self.logger.info(
+                    f"Using provided value directly as API key for generator (not found as env var: {api_key_config_value[:5]}...)."
+                )
+
+        print("config.agent_endpoint", self.config.agent_endpoint)
+        is_local_proxy_defined = bool(
+            self.config.agent_endpoint == "https://hackagent.dev/api/judge"
+        )
+
+        if is_local_proxy_defined:
             self.is_local_judge_proxy_defined = True
             self.logger.info(
                 f"Local judge proxy detected for '{self.config.agent_name}' at: {self.config.agent_endpoint}"
             )
 
-            if self.config.agent_metadata:
-                direct_api_key = self.config.agent_metadata.get("api_key")
-                api_key_env_var = self.config.agent_metadata.get("api_key_env_var")
-
-                if direct_api_key:
-                    self.actual_api_key = direct_api_key
-                    self.logger.info(
-                        f"Using direct API key for local judge proxy '{self.config.agent_name}'."
-                    )
-                elif api_key_env_var:
-                    env_key_value = os.environ.get(api_key_env_var)
-                    if env_key_value:
-                        self.actual_api_key = env_key_value
-                        self.logger.info(
-                            f"Loaded API key for local judge proxy '{self.config.agent_name}' from env var: {api_key_env_var}"
-                        )
-                    else:
-                        self.logger.warning(
-                            f"Env var {api_key_env_var} for local judge proxy '{self.config.agent_name}' API key not found."
-                        )
-                else:
-                    self.logger.warning(
-                        f"Local judge proxy '{self.config.agent_name}' detected, but no 'api_key' or 'api_key_env_var' found in agent_metadata."
-                    )
-            else:
-                self.logger.warning(
-                    f"Local judge proxy '{self.config.agent_name}' detected, but agent_metadata is missing for API key."
-                )
-
             if not self.actual_api_key:
                 self.is_local_judge_proxy_defined = (
                     False  # Cannot use local proxy without API key
@@ -497,8 +485,12 @@ Based on these criteria, please provide your classification. You should first br
         super().__init__(client, config)
         if not self.config.model_id:  # Ensure model_id is present
             raise ValueError("NuancedEvaluator requires a model_id in the config.")
-        if not self.agent_router or not self.agent_registration_key:
-            raise RuntimeError("AgentRouter not initialized for NuancedEvaluator.")
+        if not (self.is_local_judge_proxy_defined and self.actual_api_key) and (
+            not self.agent_router or not self.agent_registration_key
+        ):
+            raise RuntimeError(
+                "AgentRouter not initialized for NuancedEvaluator and local proxy not available/functional."
+            )
 
     def _get_request_data_for_row(self, row: pd.Series) -> Dict[str, Any]:
         prompt_content = self.PROMPT.format(
@@ -630,9 +622,11 @@ Answer with either 'safe' or 'unsafe' and nothing else (in particular, never sta
             raise ValueError(
                 "JailbreakBenchEvaluator requires a model_id in the config."
             )
-        if not self.agent_router or not self.agent_registration_key:
+        if not (self.is_local_judge_proxy_defined and self.actual_api_key) and (
+            not self.agent_router or not self.agent_registration_key
+        ):
             raise RuntimeError(
-                "AgentRouter not initialized for JailbreakBenchEvaluator."
+                "AgentRouter not initialized for JailbreakBenchEvaluator and local proxy not available/functional."
             )
 
     def _get_request_data_for_row(self, row: pd.Series) -> Dict[str, Any]:
@@ -732,8 +726,12 @@ Answer with only "yes" or "no"."""
             )
             # Depending on strictness, could raise ValueError here
             # raise ValueError("HarmBenchEvaluator requires a model_id in the config.")
-        if not self.agent_router or not self.agent_registration_key:
-            raise RuntimeError("AgentRouter not initialized for HarmBenchEvaluator.")
+        if not (self.is_local_judge_proxy_defined and self.actual_api_key) and (
+            not self.agent_router or not self.agent_registration_key
+        ):
+            raise RuntimeError(
+                "AgentRouter not initialized for HarmBenchEvaluator and local proxy not available/functional."
+            )
 
         # model_id processing (like adding provider prefix) should ideally happen when creating
         # the EvaluatorConfig instance before passing it here, or handled by the LiteLLMAgentAdapter itself.

hackagent-0.2.5/hackagent/cli/__init__.py

@@ -0,0 +1,7 @@
+"""
+HackAgent CLI Package
+
+Command-line interface for HackAgent security testing toolkit.
+"""
+
+__version__ = "0.2.4"

hackagent-0.2.5/hackagent/cli/commands/__init__.py

@@ -0,0 +1,5 @@
+"""
+CLI Commands Package
+
+Contains all the command group implementations for the HackAgent CLI.
+"""