hyperplane-eval 0.1.8__tar.gz → 0.1.10__tar.gz

{hyperplane_eval-0.1.8 → hyperplane_eval-0.1.10}/MANIFEST.in

@@ -3,4 +3,4 @@ include README.md
 include LICENSE
 recursive-include hyperplane/prompts *.txt
 recursive-include hyperplane/framework/domain *.json
-recursive-include hyperplane/reporting/templates *.html
+recursive-include hyperplane/framework/reporting/templates *.html

{hyperplane_eval-0.1.8/hyperplane_eval.egg-info → hyperplane_eval-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperplane-eval
-Version: 0.1.8
+Version: 0.1.10
 Summary: Local tool to evaluate AI agents and find their weak points. 
 Author: Marten Panchev
 Author-email: marten@aquithm.com
@@ -110,7 +110,8 @@ evaluator = Evaluator(
     agent_desc="A helpful AI assistant",
     param_desc={"prompt": "The user input prompt"},
     target_callable=my_agent,
-    llm_client=router
+    llm_client=router,
+    model_name="gpt-4o"
 )
 
 # 4. Add constraints and run

{hyperplane_eval-0.1.8 → hyperplane_eval-0.1.10}/README.md

@@ -76,7 +76,8 @@ evaluator = Evaluator(
     agent_desc="A helpful AI assistant",
     param_desc={"prompt": "The user input prompt"},
     target_callable=my_agent,
-    llm_client=router
+    llm_client=router,
+    model_name="gpt-4o"
 )
 
 # 4. Add constraints and run

{hyperplane_eval-0.1.8 → hyperplane_eval-0.1.10}/hyperplane/evaluator.py

@@ -21,6 +21,7 @@ class Evaluator:
         param_desc: dict,
         target_callable: Callable,
         llm_client: litellm.Router,
+        model_name: str,
     ):
         """
         Initialize the Evaluator.
@@ -35,6 +36,7 @@ class Evaluator:
         self.param_desc = param_desc
         self.target_callable = target_callable
         self.llm_client = llm_client
+        self.model_name = model_name
 
     def run(
         self,
@@ -83,9 +85,9 @@ class Evaluator:
 
         # 3. Resolve LLM configuration
         class RouterWrapper(LLMClient):
-            def __init__(self, router):
+            def __init__(self, router, model_name):
                 self.router = router
-                self.model = "router"
+                self.model = model_name
                 self._semaphore = asyncio.Semaphore(10)
 
             async def generate(
@@ -97,7 +99,7 @@ class Evaluator:
                 prompt += "\n\n" + load_prompt("llms/schema_prompt", schema=schema_str)
 
                 kwargs = {
-                    "model": "",
+                    "model": self.model,
                     "messages": [{"role": "user", "content": prompt}],
                     "temperature": temperature,
                     "response_format": {"type": "json_object"},
@@ -108,7 +110,7 @@ class Evaluator:
                 except Exception as e:
                     raise RuntimeError(f"LLM Server Error: {e}")
 
-        llm_client_resolved = RouterWrapper(self.llm_client)
+        llm_client_resolved = RouterWrapper(self.llm_client, self.model_name)
 
         # 4. Setup Custom Execution Environment
         selected_func = {
@@ -122,7 +124,8 @@ class Evaluator:
                 if asyncio.iscoroutinefunction(self.target_callable):
                     res = await self.target_callable(**params)
                 else:
-                    res = self.target_callable(**params)
+                    import functools
+                    res = await asyncio.to_thread(functools.partial(self.target_callable, **params))
 
                 if not isinstance(res, str):
                     res_str = json.dumps(res)

{hyperplane_eval-0.1.8 → hyperplane_eval-0.1.10}/hyperplane/framework/domain/dimensions/__init__.py

@@ -1,3 +1,3 @@
 from .prompt_feature import PromptFeature
 
-__all__ = ["PromptFeature"]
+__all__ = ["PromptFeature"]

{hyperplane_eval-0.1.8 → hyperplane_eval-0.1.10}/hyperplane/framework/plane_evaluator.py

@@ -94,6 +94,13 @@ class PlaneEvaluator:
                     if vector.id in self.active_scenarios:
                         del self.active_scenarios[vector.id]
                         self.update_dashboard()
+                    
+                    if self.generator.discard_count > max(10, self.target):
+                        print(f"\n[Fatal Error] Target agent has crashed {self.generator.discard_count} times. Aborting to prevent infinite stall. Last error: {agent_output}", file=sys.stderr)
+                        self.stop_event.set()
+                        self.synth_queue.task_done()
+                        break
+
                     await self.coord_queue.put(self.navigator.get_next_point())
                     self.synth_queue.task_done()
                     continue