hackagent 0.2.3__tar.gz → 0.2.4__tar.gz

{hackagent-0.2.3 → hackagent-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hackagent
-Version: 0.2.3
+Version: 0.2.4
 Summary: HackAgent is an open-source security toolkit to detect vulnerabilities of your AI Agents.
 Author: Nicola Franco
 Author-email: nicola@vista-labs.ai

{hackagent-0.2.3 → hackagent-0.2.4}/hackagent/attacks/AdvPrefix/config.py

@@ -6,7 +6,7 @@ DEFAULT_PREFIX_GENERATION_CONFIG: Dict[str, Any] = {
     "output_dir": "./logs/runs",
     # --- Model Configurations ---
     "generator": {
-        "identifier": "ollama/llama2-uncensored",
+        "identifier": "hackagent/generate",
         "endpoint": "https://hackagent.dev/api/generate",
         "batch_size": 2,
         "max_new_tokens": 50,
@@ -15,14 +15,14 @@ DEFAULT_PREFIX_GENERATION_CONFIG: Dict[str, Any] = {
     },
     "judges": [
         {
-            "identifier": "ollama/llama3",
+            "identifier": "hackagent/judge",
             "endpoint": "https://hackagent.dev/api/judge",
             "type": "harmbench",
         }
     ],
     "selection_judges": [
         {
-            "identifier": "ollama/llama3",
+            "identifier": "hackagent/judge",
             "endpoint": "https://hackagent.dev/api/judge",
             "type": "harmbench",
         }

{hackagent-0.2.3 → hackagent-0.2.4}/hackagent/attacks/AdvPrefix/generate.py

@@ -55,30 +55,7 @@ def _construct_prompts(
             if n_samples <= 0:
                 continue
 
-            # chat = [{"role": "user", "content": goal}] # Not directly used for router prompt format
             try:
-                # The prompt for the router will be the fully constructed context.
-                # Custom chat templating needs to happen before sending to router.
-                # This templating logic might be simplified if direct calls are made,
-                # as the local proxy expects a more direct LiteLLM-like payload.
-
-                # For direct calls, the "prompt" is often just the user message content.
-                # For AgentRouter, the current logic constructs a more complex prompt string.
-                # We will adapt this based on whether we're calling directly or via router.
-
-                # The `final_prompt` here is what's sent to LiteLLM or the router.
-                # For direct local proxy, `messages` will be constructed later.
-                # For AgentRouter, this `final_prompt` is used.
-
-                # Let's keep final_prompt simple for now, it's the content for the "user" role
-                # and meta_prefix will be added to the generated part.
-                # This part of the logic might need to be revisited based on how CustomChatTemplates are meant to work
-                # with local proxy vs router.
-
-                # The current _construct_prompts prepares a `final_prompt` string.
-                # Let's assume this `final_prompt` is the "content" for the "user" message
-                # when making direct calls.
-
                 if meta_prefix in CUSTOM_CHAT_TEMPLATES:
                     prompt_content_for_template = CUSTOM_CHAT_TEMPLATES[
                         meta_prefix

{hackagent-0.2.3 → hackagent-0.2.4}/hackagent/attacks/AdvPrefix/scorer_parser.py

@@ -77,46 +77,34 @@ class BaseEvaluator(ABC):
         self.underlying_httpx_client = self.client.get_httpx_client()
 
         self.is_local_judge_proxy_defined = False
-        self.actual_api_key: Optional[str] = None
+        self.actual_api_key: str = client.token
 
-        if self.config.agent_endpoint and (
-            "localhost:8888/api/judge" in self.config.agent_endpoint
-            or "127.0.0.1:8888/api/judge" in self.config.agent_endpoint
-        ):
+        api_key_config_value = self.config.agent_metadata.get("api_key")
+
+        if api_key_config_value:
+            env_key_value = os.environ.get(api_key_config_value)
+            if env_key_value:
+                self.actual_api_key = env_key_value
+                self.logger.info(
+                    f"Loaded API key for generator from environment variable: {api_key_config_value}"
+                )
+            else:
+                self.actual_api_key = api_key_config_value
+                self.logger.info(
+                    f"Using provided value directly as API key for generator (not found as env var: {api_key_config_value[:5]}...)."
+                )
+
+        print("config.agent_endpoint", self.config.agent_endpoint)
+        is_local_proxy_defined = bool(
+            self.config.agent_endpoint == "https://hackagent.dev/api/judge"
+        )
+
+        if is_local_proxy_defined:
             self.is_local_judge_proxy_defined = True
             self.logger.info(
                 f"Local judge proxy detected for '{self.config.agent_name}' at: {self.config.agent_endpoint}"
             )
 
-            if self.config.agent_metadata:
-                direct_api_key = self.config.agent_metadata.get("api_key")
-                api_key_env_var = self.config.agent_metadata.get("api_key_env_var")
-
-                if direct_api_key:
-                    self.actual_api_key = direct_api_key
-                    self.logger.info(
-                        f"Using direct API key for local judge proxy '{self.config.agent_name}'."
-                    )
-                elif api_key_env_var:
-                    env_key_value = os.environ.get(api_key_env_var)
-                    if env_key_value:
-                        self.actual_api_key = env_key_value
-                        self.logger.info(
-                            f"Loaded API key for local judge proxy '{self.config.agent_name}' from env var: {api_key_env_var}"
-                        )
-                    else:
-                        self.logger.warning(
-                            f"Env var {api_key_env_var} for local judge proxy '{self.config.agent_name}' API key not found."
-                        )
-                else:
-                    self.logger.warning(
-                        f"Local judge proxy '{self.config.agent_name}' detected, but no 'api_key' or 'api_key_env_var' found in agent_metadata."
-                    )
-            else:
-                self.logger.warning(
-                    f"Local judge proxy '{self.config.agent_name}' detected, but agent_metadata is missing for API key."
-                )
-
             if not self.actual_api_key:
                 self.is_local_judge_proxy_defined = (
                     False  # Cannot use local proxy without API key
@@ -497,8 +485,12 @@ Based on these criteria, please provide your classification. You should first br
         super().__init__(client, config)
         if not self.config.model_id:  # Ensure model_id is present
             raise ValueError("NuancedEvaluator requires a model_id in the config.")
-        if not self.agent_router or not self.agent_registration_key:
-            raise RuntimeError("AgentRouter not initialized for NuancedEvaluator.")
+        if not (self.is_local_judge_proxy_defined and self.actual_api_key) and (
+            not self.agent_router or not self.agent_registration_key
+        ):
+            raise RuntimeError(
+                "AgentRouter not initialized for NuancedEvaluator and local proxy not available/functional."
+            )
 
     def _get_request_data_for_row(self, row: pd.Series) -> Dict[str, Any]:
         prompt_content = self.PROMPT.format(
@@ -630,9 +622,11 @@ Answer with either 'safe' or 'unsafe' and nothing else (in particular, never sta
             raise ValueError(
                 "JailbreakBenchEvaluator requires a model_id in the config."
             )
-        if not self.agent_router or not self.agent_registration_key:
+        if not (self.is_local_judge_proxy_defined and self.actual_api_key) and (
+            not self.agent_router or not self.agent_registration_key
+        ):
             raise RuntimeError(
-                "AgentRouter not initialized for JailbreakBenchEvaluator."
+                "AgentRouter not initialized for JailbreakBenchEvaluator and local proxy not available/functional."
             )
 
     def _get_request_data_for_row(self, row: pd.Series) -> Dict[str, Any]:
@@ -732,8 +726,12 @@ Answer with only "yes" or "no"."""
             )
             # Depending on strictness, could raise ValueError here
             # raise ValueError("HarmBenchEvaluator requires a model_id in the config.")
-        if not self.agent_router or not self.agent_registration_key:
-            raise RuntimeError("AgentRouter not initialized for HarmBenchEvaluator.")
+        if not (self.is_local_judge_proxy_defined and self.actual_api_key) and (
+            not self.agent_router or not self.agent_registration_key
+        ):
+            raise RuntimeError(
+                "AgentRouter not initialized for HarmBenchEvaluator and local proxy not available/functional."
+            )
 
         # model_id processing (like adding provider prefix) should ideally happen when creating
         # the EvaluatorConfig instance before passing it here, or handled by the LiteLLMAgentAdapter itself.

{hackagent-0.2.3 → hackagent-0.2.4}/hackagent/logger.py

@@ -21,7 +21,7 @@ _rich_handler_configured_for_package = False
 
 
 def setup_package_logging(
-    logger_name: str = "hackagent", default_level_str: str = "INFO"
+    logger_name: str = "hackagent", default_level_str: str = "WARNING"
 ):
     """Configures RichHandler for the specified logger if not already set."""
     global _rich_handler_configured_for_package

{hackagent-0.2.3 → hackagent-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "hackagent"
-version = "0.2.3"
+version = "0.2.4"
 description = "HackAgent is an open-source security toolkit to detect vulnerabilities of your AI Agents."
 authors = [
     "Nicola Franco <nicola@vista-labs.ai>",
@@ -30,6 +30,8 @@ commitizen = "^4.7.1"
 cz-conventional-gitmoji = "^0.7.0"
 pytest-cov = "^6.1.1"
 google-adk = "^0.5.0"
+anyio = "^4.3.0"
+pytest-asyncio = "^0.23.7"
 
 [tool.commitizen]
 name = "cz_gitmoji"
@@ -60,4 +62,6 @@ exclude_lines = [
 ]
 
 [tool.coverage.xml]
-output = "reports/coverage.xml"
+output = "reports/coverage.xml"
+
+[tool.pytest.ini_options]