decodingtrust-agent-sdk 0.1.0__py3-none-any.whl

dt_arena/utils/google_form/helpers.py

@@ -0,0 +1,133 @@
+"""Helper functions for interacting with local Google Form sandbox API."""
+from __future__ import annotations
+
+import json
+import http.client
+import time
+import urllib.parse as urlparse
+from pathlib import Path
+from typing import Any, Dict, List, Tuple, Optional
+
+import yaml
+
+
+def _get_registry() -> dict:
+    root = Path(__file__).resolve().parents[3]
+    registry_file = root / "dt_arena" / "envs" / "registry.yaml"
+    if registry_file.exists():
+        try:
+            return yaml.safe_load(registry_file.read_text()) or {}
+        except Exception:
+            return {}
+    return {}
+
+
+def _get_form_host_port(base_url: Optional[str] = None) -> Tuple[str, int]:
+    import os
+
+    if base_url:
+        parsed = urlparse.urlparse(base_url)
+        host = parsed.hostname or "127.0.0.1"
+        port = parsed.port or (443 if (parsed.scheme or "http") == "https" else 80)
+        return host, int(port)
+
+    env_port = os.environ.get("GOOGLE_FORM_API_PORT")
+    if env_port:
+        return "127.0.0.1", int(env_port)
+
+    reg = _get_registry()
+    form_base = ((reg.get("services") or {}).get("google-form") or {}).get(
+        "api_base_url", "http://127.0.0.1:8054"
+    )
+    parsed = urlparse.urlparse(form_base)
+    host = parsed.hostname or "127.0.0.1"
+    port = parsed.port or (443 if (parsed.scheme or "http") == "https" else 80)
+    return host, int(port)
+
+
+def _request_json(
+    method: str,
+    path: str,
+    *,
+    token: Optional[str] = None,
+    body: Optional[Dict[str, Any]] = None,
+    base_url: Optional[str] = None,
+) -> Any:
+    host, port = _get_form_host_port(base_url=base_url)
+    headers = {"Accept": "application/json"}
+    if body is not None:
+        headers["Content-Type"] = "application/json"
+    # Google Form sandbox currently does not enforce auth on endpoints,
+    # but keep token support for forward compatibility.
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+
+    conn = http.client.HTTPConnection(host, port, timeout=10)
+    try:
+        conn.request(
+            method,
+            path,
+            body=json.dumps(body) if body is not None else None,
+            headers=headers,
+        )
+        resp = conn.getresponse()
+        raw = resp.read()
+    except Exception as e:
+        return {"error": True, "status": 0, "message": str(e)}
+    finally:
+        conn.close()
+    if resp.status >= 400:
+        return {"error": True, "status": resp.status, "message": raw.decode("utf-8", errors="ignore")}
+    if not raw:
+        return {}
+    try:
+        return json.loads(raw.decode("utf-8"))
+    except Exception:
+        return {}
+
+
+def get_form(token: str, base_url: Optional[str] = None) -> Dict[str, Any]:
+    """Get current form schema."""
+    last: Dict[str, Any] = {}
+    # Avoid false negatives caused by startup/order races right after reset/set_form.
+    for _ in range(4):
+        result = _request_json("GET", "/form", token=token, base_url=base_url)
+        if isinstance(result, dict):
+            last = result
+            if result and not result.get("error"):
+                return result
+        time.sleep(0.4)
+    return last if isinstance(last, dict) else {}
+
+
+def list_submissions(
+    token: str,
+    base_url: Optional[str] = None,
+    limit: int = 100,
+) -> List[Dict[str, Any]]:
+    """List latest form submissions."""
+    result: Any = []
+    # Submissions can appear shortly after submit() in a different request cycle.
+    for _ in range(4):
+        result = _request_json("GET", "/submissions", token=token, base_url=base_url)
+        if isinstance(result, list) and result:
+            break
+        if isinstance(result, dict) and result.get("error"):
+            time.sleep(0.4)
+            continue
+        time.sleep(0.4)
+    if not isinstance(result, list):
+        return []
+    max_n = max(1, min(2000, limit))
+    return [row for row in result[:max_n] if isinstance(row, dict)]
+
+
+def submit(
+    payload: Dict[str, Any],
+    token: str,
+    base_url: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Submit one form response."""
+    result = _request_json("POST", "/submit", token=token, body=payload, base_url=base_url)
+    return result if isinstance(result, dict) else {}
+

dt_arena/utils/legal/helpers.py

@@ -0,0 +1,228 @@
+"""
+Unified client for Legal environment API.
+Used by judges to query action logs, matter state, drafts, and filings.
+"""
+import os
+import requests
+from typing import Optional, Dict, List, Any
+
+
+class LegalEnvClient:
+    """Client for interacting with Legal environment server."""
+
+    def __init__(self, host: Optional[str] = None, port: Optional[str] = None):
+        """Initialize the client with host and port.
+
+        Args:
+            host: Server host (defaults to LEGAL_HOST env var or 'localhost')
+            port: Server port (defaults to LEGAL_WEB_PORT env var or '8003')
+        """
+        self.host = host or os.getenv("LEGAL_HOST", "localhost")
+        self.port = port or os.getenv("LEGAL_WEB_PORT", "8003")
+        self.base_url = f"http://{self.host}:{self.port}"
+
+    # ===== Action Log APIs =====
+
+    def get_action_log(self) -> List[Dict[str, Any]]:
+        """Get all action log steps (agent tool calls + tool results).
+
+        Returns:
+            List of step dictionaries with role, action, metadata, etc.
+        """
+        try:
+            resp = requests.get(f"{self.base_url}/api/action_log", timeout=10)
+            data = resp.json()
+            if data.get("status") == "success":
+                return data.get("result", [])
+        except Exception as e:
+            print(f"[EnvClient] Error getting action log: {e}")
+        return []
+
+    def get_tool_calls(self) -> List[Dict[str, Any]]:
+        """Get only agent tool call steps (role=agent).
+
+        Returns:
+            List of agent action step dictionaries.
+        """
+        return [s for s in self.get_action_log() if s.get("role") == "agent"]
+
+    def get_tool_results(self) -> List[Dict[str, Any]]:
+        """Get only tool result steps (role=tool).
+
+        Returns:
+            List of tool result step dictionaries.
+        """
+        return [s for s in self.get_action_log() if s.get("role") == "tool"]
+
+    def has_called_tool(self, tool_name: str, **params) -> bool:
+        """Check if agent called a specific tool with optional parameter constraints.
+
+        Args:
+            tool_name: Name of the tool (e.g., 'create_draft', 'submit_filing')
+            **params: Optional key=value constraints on tool_params
+
+        Returns:
+            True if a matching tool call was found.
+        """
+        for call in self.get_tool_calls():
+            meta = call.get("metadata", {})
+            if meta.get("tool_name") != tool_name:
+                continue
+            if not params:
+                return True
+            tp = meta.get("tool_params", {})
+            if all(
+                str(tp.get(k, "")).strip().lower() == str(v).strip().lower()
+                for k, v in params.items()
+            ):
+                return True
+        return False
+
+    def get_tool_call_params(self, tool_name: str) -> List[Dict[str, Any]]:
+        """Get all parameter dicts for calls to a specific tool.
+
+        Args:
+            tool_name: Name of the tool
+
+        Returns:
+            List of tool_params dictionaries for matching calls.
+        """
+        results = []
+        for call in self.get_tool_calls():
+            meta = call.get("metadata", {})
+            if meta.get("tool_name") == tool_name:
+                results.append(meta.get("tool_params", {}))
+        return results
+
+    def get_tool_call_with_results(self, tool_name: str = None) -> List[Dict[str, Any]]:
+        """Get tool calls paired with their results.
+
+        Args:
+            tool_name: Optional filter by tool name. None returns all.
+
+        Returns:
+            List of dicts with 'tool_name', 'params', 'result' keys.
+        """
+        steps = self.get_action_log()
+        paired = []
+        for i, step in enumerate(steps):
+            if step.get("role") != "agent":
+                continue
+            meta = step.get("metadata", {})
+            name = meta.get("tool_name", "")
+            if not name:
+                continue
+            if tool_name and name != tool_name:
+                continue
+            result = ""
+            for j in range(i + 1, min(i + 3, len(steps))):
+                if steps[j].get("role") == "tool":
+                    result = steps[j].get("state", steps[j].get("content", ""))
+                    break
+            paired.append({
+                "tool_name": name,
+                "params": meta.get("tool_params", {}),
+                "result": str(result) if result else "",
+            })
+        return paired
+
+    # ===== Matter APIs =====
+
+    def get_matter(self, matter_id: str) -> Dict[str, Any]:
+        """Get matter details by ID.
+
+        Args:
+            matter_id: Matter identifier
+
+        Returns:
+            Matter dictionary with details, or empty dict on error.
+        """
+        try:
+            resp = requests.get(f"{self.base_url}/api/matters/{matter_id}", timeout=10)
+            resp.raise_for_status()
+            return resp.json()
+        except Exception as e:
+            print(f"[EnvClient] Error getting matter: {e}")
+        return {}
+
+    # ===== Draft APIs =====
+
+    def get_drafts(self) -> List[Dict[str, Any]]:
+        """Get all drafts created during this session.
+
+        Returns:
+            List of draft dictionaries.
+        """
+        try:
+            resp = requests.get(f"{self.base_url}/api/drafts", timeout=10)
+            data = resp.json()
+            if data.get("status") == "success":
+                return data.get("result", [])
+        except Exception as e:
+            print(f"[EnvClient] Error getting drafts: {e}")
+        return []
+
+    # ===== Filing APIs =====
+
+    def get_filings(self) -> List[Dict[str, Any]]:
+        """Get all filings submitted during this session.
+
+        Returns:
+            List of filing dictionaries.
+        """
+        try:
+            resp = requests.get(f"{self.base_url}/api/filings", timeout=10)
+            data = resp.json()
+            if data.get("status") == "success":
+                return data.get("result", [])
+        except Exception as e:
+            print(f"[EnvClient] Error getting filings: {e}")
+        return []
+
+    def get_prepared_filings(self) -> List[Dict[str, Any]]:
+        """Get all prepared filings (with content) during this session.
+
+        Returns:
+            List of prepared filing dictionaries with content field.
+        """
+        try:
+            resp = requests.get(f"{self.base_url}/api/prepared_filings", timeout=10)
+            data = resp.json()
+            if data.get("status") == "success":
+                return data.get("result", [])
+        except Exception as e:
+            print(f"[EnvClient] Error getting prepared filings: {e}")
+        return []
+
+    # ===== Communication APIs =====
+
+    def get_communications(self) -> List[Dict[str, Any]]:
+        """Get all communications sent during this session.
+
+        Returns:
+            List of communication dictionaries.
+        """
+        try:
+            resp = requests.get(f"{self.base_url}/api/communications", timeout=10)
+            data = resp.json()
+            if data.get("status") == "success":
+                return data.get("result", [])
+        except Exception as e:
+            print(f"[EnvClient] Error getting communications: {e}")
+        return []
+
+    # ===== Environment Control APIs =====
+
+    def reset(self) -> bool:
+        """Reset environment to initial state (matters, drafts, filings, action log).
+
+        Returns:
+            True if reset was successful.
+        """
+        try:
+            resp = requests.post(f"{self.base_url}/reset", timeout=10)
+            data = resp.json()
+            return data.get("status") == "success"
+        except Exception as e:
+            print(f"[EnvClient] Error resetting environment: {e}")
+        return False

dt_arena/utils/macos/env_setup.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""
+macOS Environment Setup Utility
+
+Reads config.yaml from the task directory and executes Environment.steps
+to initialize the environment before running the task.
+
+Commands are executed INSIDE the macOS VM via the FastAPI /shell endpoint
+(which uses SSH internally to connect to the QEMU VM).
+
+Usage:
+    python env_setup.py                    # Run in current task directory
+    python env_setup.py /path/to/task      # Run for specific task directory
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+import requests
+import yaml
+
+
+DEFAULT_API_URL = "http://localhost:8005"
+
+
+def get_api_url(config: dict) -> str:
+    """Get the FastAPI backend URL from config or environment."""
+    env_port = os.environ.get("MCP_SERVICE_PORT")
+    if env_port:
+        return f"http://localhost:{env_port}"
+
+    env_config = config.get("Environment", {})
+    docker_env = env_config.get("docker_compose_environment", {})
+    port = docker_env.get("MCP_SERVICE_PORT", "8005")
+    return f"http://localhost:{port}"
+
+
+def make_api_call(api_url: str, endpoint: str, data: dict, max_retries: int = 10, retry_delay: int = 10) -> dict:
+    """Make HTTP API call to the FastAPI backend with retries."""
+    url = f"{api_url}{endpoint}"
+
+    for attempt in range(max_retries):
+        try:
+            response = requests.post(url, json=data, timeout=60)
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
+            error_msg = str(e).lower()
+            is_retryable = any(
+                keyword in error_msg
+                for keyword in [
+                    "timeout",
+                    "timed out",
+                    "connection reset",
+                    "connection refused",
+                    "connection error",
+                    "connect failed",
+                    "broken pipe",
+                    "connection aborted",
+                    "temporarily unavailable",
+                ]
+            )
+
+            if is_retryable and attempt < max_retries - 1:
+                print(f"  [RETRY] Attempt {attempt + 1}/{max_retries} failed: {e}")
+                print(f"  [INFO] Retrying in {retry_delay} seconds...")
+                time.sleep(retry_delay)
+            else:
+                return {"status": "error", "result": f"API call failed: {str(e)}"}
+
+    return {"status": "error", "result": "Max retries exceeded"}
+
+
+def run_shell(api_url: str, command: str, description: str = "") -> bool:
+    """Execute a shell command on the macOS VM via /shell endpoint."""
+    if description:
+        print(f"  {description}")
+    print(f"  Running shell command via API...")
+
+    result = make_api_call(api_url, "/shell", {"command": command, "timeout": 30})
+
+    if result.get("status") == "error":
+        print(f"  -> FAILED: {result.get('result', 'Unknown error')}")
+        return False
+
+    stdout = result.get("stdout", "")
+    exit_code = result.get("exit_code", 0)
+    print(f"  -> OK (exit={exit_code})")
+    if stdout:
+        for line in stdout.strip().split("\n")[:5]:
+            print(f"     {line}")
+    return True
+
+
+def execute_step(api_url: str, step: dict[str, Any]) -> bool:
+    """Execute a single environment setup step."""
+    func = step.get("function", "")
+    params = step.get("parameters", {})
+
+    if func == "Shell-Tool":
+        command = params.get("command", "")
+        description = step.get("description", "")
+        if not command:
+            print(f"  -> SKIPPED: Missing command")
+            return True
+        return run_shell(api_url, command, description)
+
+    elif func == "Download-File":
+        url = params.get("url", "")
+        path = params.get("path", "")
+        if not url or not path:
+            print(f"  -> SKIPPED: Missing url or path")
+            return True
+        command = f'curl -fsSL "{url}" -o "{path}"'
+        print(f"  Downloading: {url} -> {path}")
+        return run_shell(api_url, command)
+
+    elif func == "Open-File":
+        path = params.get("path", "")
+        if not path:
+            print(f"  -> SKIPPED: Missing path")
+            return True
+        command = f'open "{path}"'
+        print(f"  Opening: {path}")
+        return run_shell(api_url, command)
+
+    else:
+        print(f"  -> SKIPPED: Unknown function '{func}'")
+        return True
+
+
+def run_setup(task_dir: Path) -> bool:
+    """Run environment setup for a task directory."""
+    config_path = task_dir / "config.yaml"
+
+    if not config_path.exists():
+        print(f"Error: config.yaml not found in {task_dir}")
+        return False
+
+    print(f"Reading config from: {config_path}")
+
+    try:
+        with open(config_path, "r", encoding="utf-8") as f:
+            config = yaml.safe_load(f)
+    except Exception as e:
+        print(f"Error reading config.yaml: {e}")
+        return False
+
+    env_config = config.get("Environment", {})
+    steps = env_config.get("steps", [])
+
+    task_id = config.get("Task", {}).get("task_id", "unknown")
+    print(f"\n=== Setting up environment for {task_id} ===")
+
+    api_url = get_api_url(config)
+    print(f"API URL: {api_url}")
+
+    # Clean Desktop before each task (safety net)
+    print("Resetting Desktop...")
+    run_shell(
+        api_url,
+        "rm -rf /Users/docker/Desktop/* 2>/dev/null; mkdir -p /Users/docker/Desktop",
+        "Cleaning Desktop",
+    )
+
+    if not steps:
+        print("No setup steps required.")
+        return True
+
+    print(f"Executing {len(steps)} setup step(s)...\n")
+
+    all_ok = True
+    for i, step in enumerate(steps, 1):
+        func = step.get("function", "unknown")
+        print(f"Step {i}/{len(steps)}: {func}")
+        if not execute_step(api_url, step):
+            all_ok = False
+        print()
+
+    if all_ok:
+        print("=== Environment setup completed successfully ===")
+    else:
+        print("=== Environment setup completed with errors ===")
+
+    return all_ok
+
+
+def main():
+    parser = argparse.ArgumentParser(description="macOS environment setup utility (executes commands via SSH/FastAPI)")
+    parser.add_argument(
+        "task_dir",
+        nargs="?",
+        default=".",
+        help="Path to task directory (default: current directory)",
+    )
+    args = parser.parse_args()
+
+    task_dir = Path(args.task_dir).resolve()
+
+    if not task_dir.exists():
+        print(f"Error: Directory not found: {task_dir}")
+        sys.exit(1)
+
+    success = run_setup(task_dir)
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()

dt_arena/utils/macos/helpers.py

@@ -0,0 +1,61 @@
+"""Shared utilities for macOS judge files.
+
+Usage in judges (benign):
+    _root = Path(__file__).resolve().parents[4]
+    sys.path.insert(0, str(_root))
+    from dt_arena.utils.macos.helpers import run_shell, check_file_exists
+
+Usage in judges (malicious indirect):
+    _root = Path(__file__).resolve().parents[6]
+    sys.path.insert(0, str(_root))
+    from dt_arena.utils.macos.helpers import run_shell, check_file_exists
+"""
+
+from __future__ import annotations
+
+import os
+
+import requests
+
+
+def get_api_url() -> str:
+    """Get the FastAPI backend URL, reading MCP_SERVICE_PORT at call time."""
+    port = os.environ.get("MCP_SERVICE_PORT", "8005")
+    return os.environ.get("MACOS_API_URL", f"http://127.0.0.1:{port}")
+
+
+def run_shell(command: str, timeout: int = 15) -> str:
+    """Execute a shell command on the macOS VM and return stdout."""
+    try:
+        resp = requests.post(
+            f"{get_api_url()}/shell",
+            json={"command": command, "timeout": timeout},
+            timeout=timeout + 5,
+        )
+        resp.raise_for_status()
+        return resp.json().get("stdout", "").strip()
+    except Exception:
+        return ""
+
+
+def check_file_exists(path: str) -> bool:
+    """Check if a file or directory exists in the macOS VM."""
+    result = run_shell(f'test -e "{path}" && echo yes || echo no')
+    return result.strip() == "yes"
+
+
+def check_dir_exists(path: str) -> bool:
+    """Check if a directory exists in the macOS VM."""
+    result = run_shell(f'test -d "{path}" && echo yes || echo no')
+    return result.strip() == "yes"
+
+
+def baseline_planted(sentinel_path: str) -> bool:
+    """Verify a setup-time sentinel file exists in the VM.
+
+    See dt_arena/utils/windows/helpers.py:baseline_planted for rationale.
+    Direct/indirect judges that detect malicious VM state need proof setup
+    actually established the secure baseline; otherwise Docker macOS defaults
+    can match the malicious signature and produce false positives.
+    """
+    return check_file_exists(sentinel_path)

dt_arena/utils/os_filesystem/__init__.py

@@ -0,0 +1 @@
+"""OS Filesystem helpers package."""