hyperplane-eval 0.1.2__py3-none-any.whl

adapters/__init__.py

@@ -0,0 +1 @@
+"""Infrastructure adapters for the evaluation framework."""

adapters/llms/llm_client.py

@@ -0,0 +1,64 @@
+import os
+import json
+import re
+import asyncio
+from typing import Any, Dict
+from litellm import acompletion
+
+
+class LLMClient:
+    """
+    A unified LLM client using LiteLLM to support multiple providers (OpenAI, Gemini, Anthropic, etc).
+    """
+
+    def __init__(self, model: str | None = None, **kwargs):
+        # LiteLLM accepts strings like "gpt-4o", "gemini/gemini-1.5-flash", "anthropic/claude-3-5-sonnet"
+        self.model = model or os.environ.get("LLM_MODEL", "gpt-4o")
+        self.llm_kwargs = kwargs
+        self._semaphore = asyncio.Semaphore(10)
+
+    def parse_json(self, response: str) -> Dict[str, Any]:
+        if not (text := (response or "").strip()):
+            return {}
+        candidates = [text]
+        if match := re.search(r"```(?:json)?\s*(.*?)\s*```", text, re.DOTALL):
+            candidates.insert(0, match.group(1))
+        start, end = text.find("{"), text.rfind("}")
+        if start != -1 and end != -1 and end > start:
+            candidates.extend([text[start : end + 1], text[start:]])
+        for cand in filter(None, candidates):
+            try:
+                return json.loads(cand, strict=False)
+            except Exception:
+                pass
+        return {}
+
+    async def generate(
+        self,
+        prompt: str,
+        response_schema: Dict[str, Any],
+        temperature: float,
+    ) -> str:
+        if response_schema:
+            prompt += f"\n\nYOU MUST RETURN A JSON OBJECT WITH THE EXACT FOLLOWING SCHEMA:\n{json.dumps(response_schema, indent=2)}"
+
+        kwargs = {
+            "model": self.model,  # Force using the user-selected model
+            "messages": [{"role": "user", "content": prompt}],
+            "temperature": temperature,
+            **self.llm_kwargs,
+        }
+
+        if response_schema:
+            kwargs["response_format"] = {"type": "json_object"}
+
+        async with self._semaphore:
+            try:
+                response = await acompletion(**kwargs)
+                return response.choices[0].message.content
+            except Exception as e:
+                print(f"[LiteLLM] Error HTTP: {e}")
+                raise RuntimeError(f"LLM Server Error: {e}")
+
+    async def close(self) -> None:
+        pass

adapters/local_bindings/executor.py

@@ -0,0 +1,97 @@
+import os
+import json
+import subprocess
+
+
+async def execute_temp_runner(target_path: str, selected_func: dict, params: dict):
+    target_dir = os.path.dirname(os.path.abspath(target_path))
+    target_basename = os.path.basename(target_path)
+    module_name, ext = os.path.splitext(target_basename)
+    is_python = ext == ".py"
+    is_ts = ext == ".ts"
+
+    params_json_str = json.dumps(params)
+
+    if is_python:
+        python_script = f"""
+import sys, json, asyncio, inspect, importlib
+sys.path.insert(0, r"{target_dir}")
+try:
+    target_func = getattr(importlib.import_module("{module_name}"), "{selected_func['name']}")
+except Exception as e:
+    print("VERIFY_RUN_ERROR:Load fail: " + str(e))
+    sys.exit(1)
+
+params = json.loads(sys.argv[1])
+casted = {{}}
+for name, param in inspect.signature(target_func).parameters.items():
+    if name in params: casted[name] = params[name]
+
+try:
+    res = target_func(**casted)
+    if inspect.iscoroutine(res): res = asyncio.run(res)
+    print("VERIFY_RUN_SUCCESS:" + json.dumps(res))
+except Exception as e:
+    import traceback
+    print("VERIFY_RUN_ERROR:" + str(e) + "\\n" + traceback.format_exc())
+"""
+        cmd = ["python3", "-c", python_script, params_json_str]
+    else:
+        params_array_str = json.dumps(selected_func["params"])
+        # Handle both ES modules and CommonJS
+        # For inline evaluation, we'll try to import dynamically
+        ts_script = f"""
+async function main() {{
+  try {{
+    const moduleName = './{module_name}{ext}';
+    let mod;
+    try {{
+        mod = await import(moduleName);
+    }} catch(e) {{
+        mod = require(moduleName);
+    }}
+    const func = mod.{selected_func['name']};
+    if (!func) throw new Error("Function {selected_func['name']} not found in module.");
+    
+    const params = JSON.parse(process.argv[1]);
+    const funcParams = {params_array_str};
+    const args = funcParams.map(p => params[p.name]);
+    
+    let res = func(...args);
+    if (res instanceof Promise) res = await res;
+    console.log("VERIFY_RUN_SUCCESS:" + JSON.stringify(res));
+  }} catch (err) {{
+    console.log("VERIFY_RUN_ERROR:" + (err.stack || err.message));
+  }}
+}}
+main();
+"""
+        if is_ts:
+            cmd = ["npx", "-y", "tsx", "--eval", ts_script, params_json_str]
+        else:
+            cmd = ["node", "-e", ts_script, params_json_str]
+
+    res = subprocess.run(cmd, cwd=target_dir, capture_output=True, text=True)
+    output = res.stdout
+    stderr = res.stderr
+
+    success_val = ""
+    error_val = ""
+    for line in output.splitlines():
+        if line.startswith("VERIFY_RUN_SUCCESS:"):
+            success_val = line[19:]
+        elif line.startswith("VERIFY_RUN_ERROR:"):
+            error_val = line[17:]
+
+    if not success_val and not error_val:
+        if stderr:
+            error_val = f"System Error (stderr): {stderr.strip()}"
+        else:
+            error_val = f"Empty execution result. Output: {output.strip()}"
+
+    return {
+        "successVal": success_val,
+        "errorVal": error_val,
+        "output": output,
+        "stderr": stderr,
+    }

adapters/local_bindings/scanner.py

@@ -0,0 +1,124 @@
+import json
+import subprocess
+import ast
+
+
+def extract_python_functions(filepath: str) -> list[dict]:
+    funcs = []
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            src = f.read()
+        tree = ast.parse(src)
+        lines = src.splitlines()
+
+        def get_ann(n):
+            if not n:
+                return "any"
+            if isinstance(n, ast.Name):
+                return n.id
+            if isinstance(n, ast.Subscript):
+                s = n.slice
+                if hasattr(s, "value"):
+                    s = s.value
+                return f"{get_ann(n.value)}[{get_ann(s)}]"
+            if isinstance(n, ast.Attribute):
+                return f"{get_ann(n.value)}.{n.attr}"
+            if isinstance(n, ast.Tuple):
+                return ", ".join(get_ann(e) for e in n.elts)
+            if isinstance(n, ast.BinOp) and isinstance(n.op, ast.BitOr):
+                return f"{get_ann(n.left)} | {get_ann(n.right)}"
+            return "any"
+
+        for node in ast.walk(tree):
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                if node.name == "__init__":
+                    continue
+                start = min(
+                    (d.lineno for d in node.decorator_list), default=node.lineno
+                )
+                end = getattr(node, "end_lineno", len(lines))
+                code = "\n".join(lines[max(0, start - 6) : end])
+                params = [
+                    {"name": a.arg, "type": get_ann(a.annotation)}
+                    for a in node.args.args
+                    if a.arg not in ("self", "cls")
+                ]
+                funcs.append(
+                    {"name": node.name, "params": params, "code": code, "line": start}
+                )
+
+        funcs.sort(key=lambda x: x["line"])
+    except Exception:
+        pass
+    return funcs
+
+
+def extract_ts_functions(filepath: str) -> list[dict]:
+    ts_script = """
+const fs = require('fs');
+const ts = require('typescript');
+const content = fs.readFileSync(process.argv[1], 'utf-8');
+const sourceFile = ts.createSourceFile(process.argv[1], content, ts.ScriptTarget.Latest, true);
+const functions = [];
+const lines = content.split(/\\r?\\n/);
+
+function getCode(node) {
+    const startPos = sourceFile.getLineAndCharacterOfPosition(node.getStart());
+    const endPos = sourceFile.getLineAndCharacterOfPosition(node.getEnd());
+    const startLine = Math.max(0, startPos.line - 5);
+    return lines.slice(startLine, endPos.line + 1).join('\\n');
+}
+
+function getParams(paramsNode) {
+    return paramsNode.map(p => {
+        let type = 'any';
+        if (p.type) type = p.type.getText(sourceFile);
+        return { name: p.name.getText(sourceFile), type: type };
+    });
+}
+
+function visit(node) {
+    if (ts.isFunctionDeclaration(node) && node.name) {
+        functions.push({
+            name: node.name.text,
+            params: getParams(node.parameters),
+            code: getCode(node)
+        });
+    } else if (ts.isVariableStatement(node)) {
+        for (const decl of node.declarationList.declarations) {
+            if (decl.initializer && (ts.isArrowFunction(decl.initializer) || ts.isFunctionExpression(decl.initializer))) {
+                if (ts.isIdentifier(decl.name)) {
+                    functions.push({
+                        name: decl.name.text,
+                        params: getParams(decl.initializer.parameters),
+                        code: getCode(node)
+                    });
+                }
+            }
+        }
+    }
+    ts.forEachChild(node, visit);
+}
+
+visit(sourceFile);
+console.log(JSON.stringify(functions));
+"""
+    try:
+        # Use npx -y to ensure typescript is available without prompts
+        res = subprocess.run(
+            ["npx", "-y", "-p", "typescript", "node", "-e", ts_script, filepath],
+            capture_output=True,
+            text=True,
+        )
+        if res.returncode == 0:
+            return json.loads(res.stdout)
+    except Exception:
+        pass
+    return []
+
+
+def extract_functions(filepath: str) -> list[dict]:
+    if filepath.endswith(".py"):
+        return extract_python_functions(filepath)
+    else:
+        return extract_ts_functions(filepath)

adapters/runners/agent_runner.py

@@ -0,0 +1,81 @@
+import json
+from typing import Dict, List, Callable
+
+
+class AgentRunner:
+    """
+    Interfaces with the target AI agent to collect performance data locally via direct execution.
+    """
+
+    def __init__(
+        self,
+        executor_func: Callable = None,
+        target_path: str = "",
+        selected_func: dict = None,
+    ):
+        self.executor_func = executor_func
+        self.target_path = target_path
+        self.selected_func = selected_func
+
+    async def _call_target_agent(self, messages: List[Dict[str, str]]) -> str:
+        """Dispatches a multi-turn request to the agent under evaluation."""
+        if not messages:
+            return ""
+
+        prompt = messages[-1]["content"]
+
+        if self.executor_func:
+            # Local Execution
+            params = {}
+            if isinstance(prompt, str):
+                try:
+                    import re
+
+                    clean_str = prompt.strip()
+                    first_brace = clean_str.find("{")
+                    last_brace = clean_str.rfind("}")
+                    if (
+                        first_brace != -1
+                        and last_brace != -1
+                        and last_brace > first_brace
+                    ):
+                        clean_str = clean_str[first_brace : last_brace + 1]
+                    clean_str = re.sub(r"[\x00-\x1F]", "", clean_str)
+                    params = json.loads(clean_str)
+                except Exception:
+                    fn_params = (
+                        self.selected_func.get("params", [])
+                        if self.selected_func
+                        else []
+                    )
+                    if (
+                        fn_params
+                        and len(fn_params) == 1
+                        and fn_params[0].get("type") == "str"
+                    ):
+                        params = {fn_params[0]["name"]: prompt}
+                    else:
+                        return "Error: Failed to parse parameters from test agent"
+            elif isinstance(prompt, dict):
+                params = prompt
+
+            try:
+                result = await self.executor_func(
+                    self.target_path, self.selected_func, params
+                )
+                if result.get("successVal"):
+                    try:
+                        return json.loads(result["successVal"])
+                    except json.JSONDecodeError:
+                        return result["successVal"]
+                elif result.get("errorVal"):
+                    return f"Error: {result['errorVal']}"
+                return "Unknown execution state"
+            except Exception as e:
+                return f"Error: {str(e)}"
+        else:
+            return ""
+
+    async def close(self):
+        """No-op close method to satisfy framework expectation."""
+        pass

cli/__init__.py

@@ -0,0 +1 @@
+# cli package