agenta 0.72.4__py3-none-any.whl → 0.75.0__py3-none-any.whl

agenta/sdk/workflows/runners/base.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Union
+from typing import Any, Dict, Union, Optional
 
 
 class CodeRunner(ABC):
@@ -13,16 +13,20 @@ class CodeRunner(ABC):
         inputs: Dict[str, Any],
         output: Union[dict, str],
         correct_answer: Any,
+        runtime: Optional[str] = None,
+        templates: Optional[Dict[str, str]] = None,
     ) -> Union[float, None]:
         """
         Execute code and return a float score between 0 and 1.
 
         Args:
-            code: Python code to execute
+            code: Code to execute
             app_params: Application parameters
             inputs: Input data for the code
             output: Output from the application variant
             correct_answer: Expected/correct answer for comparison
+            runtime: Runtime environment (python, javascript, typescript), None = python
+            templates: Wrapper templates keyed by runtime.
 
         Returns:
             Float score between 0 and 1, or None if execution fails

agenta/sdk/workflows/runners/daytona.py

@@ -1,42 +1,55 @@
 import os
 import json
-from typing import Any, Dict, Union, Optional
-
-from daytona import Daytona, DaytonaConfig, Sandbox
+from contextlib import contextmanager
+from typing import Any, Dict, Generator, Union, Optional, TYPE_CHECKING
 
+import agenta as ag
 from agenta.sdk.workflows.runners.base import CodeRunner
+from agenta.sdk.contexts.running import RunningContext
+from agenta.sdk.utils.lazy import _load_daytona
 
 from agenta.sdk.utils.logging import get_module_logger
 
+if TYPE_CHECKING:
+    from daytona import Sandbox
+
 log = get_module_logger(__name__)
 
-# Template for wrapping user code with evaluation context
-EVALUATION_CODE_TEMPLATE = """
-import json
 
-# Parse all parameters from a single dict
-params = json.loads({params_json!r})
-app_params = params['app_params']
-inputs = params['inputs']
-output = params['output']
-correct_answer = params['correct_answer']
+def _extract_error_message(error_text: str) -> str:
+    """Extract a clean error message from a Python traceback.
+
+    Given a full traceback string, extracts just the final error line
+    (e.g., "NameError: name 'foo' is not defined") instead of the full
+    noisy traceback with base64-encoded code.
+
+    Args:
+        error_text: Full error/traceback string
+
+    Returns:
+        Clean error message, or original text if extraction fails
+    """
+    if not error_text:
+        return "Unknown error"
 
-# User-provided evaluation code
-{user_code}
+    lines = error_text.strip().split("\n")
 
-# Execute and capture result
-result = evaluate(app_params, inputs, output, correct_answer)
+    # Look for common Python error patterns from the end
+    for line in reversed(lines):
+        line = line.strip()
+        # Match patterns like "NameError: ...", "ValueError: ...", etc.
+        if ": " in line and not line.startswith("File "):
+            # Check if it looks like an error line (ErrorType: message)
+            parts = line.split(": ", 1)
+            if parts[0].replace(".", "").replace("_", "").isalnum():
+                return line
 
-# Ensure result is a float
-if isinstance(result, (float, int, str)):
-    try:
-        result = float(result)
-    except (ValueError, TypeError):
-        result = None
+    # Fallback: return last non-empty line
+    for line in reversed(lines):
+        if line.strip():
+            return line.strip()
 
-# Print result for capture
-print(json.dumps({{"result": result}}))
-"""
+    return error_text[:200] if len(error_text) > 200 else error_text
 
 
 class DaytonaRunner(CodeRunner):
@@ -57,7 +70,7 @@ class DaytonaRunner(CodeRunner):
             return
 
         self._initialized = True
-        self.daytona: Optional[Daytona] = None
+        self.daytona = None
         self._validate_config()
 
     def _validate_config(self) -> None:
@@ -77,6 +90,8 @@ class DaytonaRunner(CodeRunner):
             return
 
         try:
+            Daytona, DaytonaConfig, _, _ = _load_daytona()
+
             # Get configuration with fallbacks
             api_url = os.getenv("DAYTONA_API_URL") or "https://app.daytona.io/api"
             api_key = os.getenv("DAYTONA_API_KEY")
@@ -92,26 +107,114 @@ class DaytonaRunner(CodeRunner):
         except Exception as e:
             raise RuntimeError(f"Failed to initialize Daytona client: {e}")
 
-    def _create_sandbox(self) -> Any:
-        """Create a new sandbox for this run from snapshot."""
+    def _get_provider_env_vars(self) -> Dict[str, str]:
+        """
+        Fetch user secrets and extract standard provider keys as environment variables.
+
+        Returns:
+            Dictionary of environment variables for standard providers
+        """
+        env_vars = {}
+
+        # Get secrets from context (set by vault middleware)
+        ctx = RunningContext.get()
+        secrets = getattr(ctx, "vault_secrets", [])
+
+        # Standard provider keys mapping
+        provider_env_mapping = {
+            "openai": "OPENAI_API_KEY",
+            "cohere": "COHERE_API_KEY",
+            "anyscale": "ANYSCALE_API_KEY",
+            "deepinfra": "DEEPINFRA_API_KEY",
+            "alephalpha": "ALEPHALPHA_API_KEY",
+            "groq": "GROQ_API_KEY",
+            "mistralai": "MISTRALAI_API_KEY",
+            "anthropic": "ANTHROPIC_API_KEY",
+            "perplexityai": "PERPLEXITYAI_API_KEY",
+            "togetherai": "TOGETHERAI_API_KEY",
+            "openrouter": "OPENROUTER_API_KEY",
+            "gemini": "GEMINI_API_KEY",
+        }
+
+        # Extract provider keys from secrets
+        for secret in secrets:
+            if secret.get("kind") == "provider_key":
+                secret_data = secret.get("data", {})
+                provider_kind = secret_data.get("kind")
+
+                if provider_kind in provider_env_mapping:
+                    provider_settings = secret_data.get("provider", {})
+                    api_key = provider_settings.get("key")
+
+                    if api_key:
+                        env_var_name = provider_env_mapping[provider_kind]
+                        env_vars[env_var_name] = api_key
+
+        return env_vars
+
+    def _create_sandbox(self, runtime: Optional[str] = None) -> Any:
+        """Create a new sandbox for this run from snapshot.
+
+        Args:
+            runtime: Runtime environment (python, javascript, typescript), None = python
+        """
         try:
             if self.daytona is None:
                 raise RuntimeError("Daytona client not initialized")
 
-            snapshot_id = os.getenv("AGENTA_SERVICES_SANDBOX_SNAPSHOT_PYTHON")
+            # Normalize runtime: None means python
+            runtime = runtime or "python"
+
+            # Select general snapshot
+            snapshot_id = os.getenv("DAYTONA_SNAPSHOT")
 
             if not snapshot_id:
                 raise RuntimeError(
-                    "AGENTA_SERVICES_SANDBOX_SNAPSHOT_PYTHON environment variable is required. "
-                    "Set it to the Daytona sandbox ID or snapshot name you want to use."
+                    f"No Daytona snapshot configured for runtime '{runtime}'. "
+                    f"Set DAYTONA_SNAPSHOT environment variable."
                 )
 
-            from daytona import CreateSandboxFromSnapshotParams
+            _, _, _, CreateSandboxFromSnapshotParams = _load_daytona()
+
+            agenta_host = (
+                ag.DEFAULT_AGENTA_SINGLETON_INSTANCE.host
+                #
+                or ""
+            )
+            agenta_api_url = (
+                ag.DEFAULT_AGENTA_SINGLETON_INSTANCE.api_url
+                #
+                or ""
+            )
+            agenta_credentials = (
+                RunningContext.get().credentials
+                #
+                or ""
+            )
+            agenta_api_key = (
+                agenta_credentials[7:]
+                if agenta_credentials.startswith("ApiKey ")
+                else ""
+            )
+
+            # Get provider API keys from user secrets
+            provider_env_vars = self._get_provider_env_vars()
+
+            # Combine base env vars with provider keys
+            env_vars = {
+                "AGENTA_HOST": agenta_host,
+                "AGENTA_API_URL": agenta_api_url,
+                "AGENTA_API_KEY": agenta_api_key,
+                "AGENTA_CREDENTIALS": agenta_credentials,
+                **provider_env_vars,  # Add provider API keys
+            }
 
             sandbox = self.daytona.create(
                 CreateSandboxFromSnapshotParams(
                     snapshot=snapshot_id,
                     ephemeral=True,
+                    env_vars=env_vars,
+                    language=runtime,
                 )
             )
 
@@ -120,6 +223,29 @@ class DaytonaRunner(CodeRunner):
         except Exception as e:
             raise RuntimeError(f"Failed to create sandbox from snapshot: {e}")
 
+    @contextmanager
+    def _sandbox_context(
+        self, runtime: Optional[str] = None
+    ) -> Generator["Sandbox", None, None]:
+        """Context manager for sandbox lifecycle.
+
+        Ensures sandbox is deleted even if an error occurs during execution.
+
+        Args:
+            runtime: Runtime environment (python, javascript, typescript), None = python
+
+        Yields:
+            Sandbox instance
+        """
+        sandbox = self._create_sandbox(runtime=runtime)
+        try:
+            yield sandbox
+        finally:
+            try:
+                sandbox.delete()
+            except Exception as e:
+                log.error("Failed to delete sandbox: %s", e)
+
     def run(
         self,
         code: str,
@@ -127,130 +253,123 @@ class DaytonaRunner(CodeRunner):
         inputs: Dict[str, Any],
         output: Union[dict, str],
         correct_answer: Any,
+        runtime: Optional[str] = None,
+        templates: Optional[Dict[str, str]] = None,
     ) -> Union[float, None]:
         """
-        Execute provided Python code in Daytona sandbox.
+        Execute provided code in Daytona sandbox.
 
         The code must define an `evaluate()` function that takes
         (app_params, inputs, output, correct_answer) and returns a float (0-1).
 
         Args:
-            code: The Python code to be executed
+            code: The code to be executed
             app_params: The parameters of the app variant
             inputs: Inputs to be used during code execution
             output: The output of the app variant after being called
             correct_answer: The correct answer (or target) for comparison
+            runtime: Runtime environment (python, javascript, typescript), None = python
+            templates: Wrapper templates keyed by runtime.
 
         Returns:
             Float score between 0 and 1, or None if execution fails
         """
-        self._initialize_client()
-        sandbox: Sandbox = self._create_sandbox()
-
-        try:
-            # Prepare all parameters as a single dict
-            params = {
-                "app_params": app_params,
-                "inputs": inputs,
-                "output": output,
-                "correct_answer": correct_answer,
-            }
-            params_json = json.dumps(params)
-
-            # Wrap the user code with the necessary context and evaluation
-            wrapped_code = EVALUATION_CODE_TEMPLATE.format(
-                params_json=params_json,
-                user_code=code,
-            )
-
-            # Log the input parameters for debugging
-            # log.debug("Input parameters to evaluation:")
-            # print("\n" + "=" * 80)
-            # print("INPUT PARAMETERS:")
-            # print("=" * 80)
-            # print(f"app_params: {app_params}")
-            # print(f"inputs: {inputs}")
-            # print(f"output: {output}")
-            # print(f"correct_answer: {correct_answer}")
-            # print("=" * 80 + "\n")
-
-            # Log the generated code for debugging
-            # log.debug("Generated code to send to Daytona:")
-            # print("=" * 80)
-            # print("GENERATED CODE TO SEND TO DAYTONA:")
-            # print("=" * 80)
-            # code_lines = wrapped_code.split("\n")
-            # for i, line in enumerate(code_lines, 1):
-            #     log.debug(f"  {i:3d}: {line}")
-            #     print(f"  {i:3d}: {line}")
-            # print("=" * 80)
-            # print(f"Total lines: {len(code_lines)}")
-            # print("=" * 80 + "\n")
-
-            # Callback functions to capture output and errors
-            stdout_lines = []
-            stderr_lines = []
-
-            def on_stdout(line: str) -> None:
-                """Capture stdout output."""
-                # log.debug(f"[STDOUT] {line}")
-                # print(f"[STDOUT] {line}")
-                stdout_lines.append(line)
-
-            def on_stderr(line: str) -> None:
-                """Capture stderr output."""
-                # log.warning(f"[STDERR] {line}")
-                # print(f"[STDERR] {line}")
-                stderr_lines.append(line)
-
-            def on_error(error: Exception) -> None:
-                """Capture errors."""
-                log.error(f"[ERROR] {type(error).__name__}: {error}")
-                # print(f"[ERROR] {type(error).__name__}: {error}")
-
-            # Execute the code in the Daytona sandbox
-            # log.debug("Executing code in Daytona sandbox")
-            response = sandbox.code_interpreter.run_code(
-                wrapped_code,
-                on_stdout=on_stdout,
-                on_stderr=on_stderr,
-                on_error=on_error,
-            )
-
-            # log.debug(f"Raw response: {response}")
-            # print(f"Raw response: {response}")
+        # Normalize runtime: None means python
+        runtime = runtime or "python"
 
-            # Parse the result from the response object
-            # Response has stdout, stderr, and error fields
-            response_stdout = response.stdout if hasattr(response, "stdout") else ""
-            response_error = response.error if hasattr(response, "error") else None
+        self._initialize_client()
 
-            sandbox.delete()
+        with self._sandbox_context(runtime=runtime) as sandbox:
+            try:
+                # Prepare all parameters as a single dict
+                params = {
+                    "app_params": app_params,
+                    "inputs": inputs,
+                    "output": output,
+                    "correct_answer": correct_answer,
+                }
+                params_json = json.dumps(params)
+
+                if not templates:
+                    raise RuntimeError(
+                        "Missing evaluator templates for Daytona execution"
+                    )
+
+                template = templates.get(runtime)
+                if template is None:
+                    raise RuntimeError(
+                        f"Missing evaluator template for runtime '{runtime}'"
+                    )
+
+                # Wrap the user code with the necessary context and evaluation
+                wrapped_code = template.format(
+                    params_json=params_json,
+                    user_code=code,
+                )
 
-            if response_error:
-                log.error(f"Sandbox execution error: {response_error}")
-                raise RuntimeError(f"Sandbox execution failed: {response_error}")
+                # Execute the code in the Daytona sandbox
+                response = sandbox.process.code_run(wrapped_code)
+                response_stdout = response.result if hasattr(response, "result") else ""
+                response_exit_code = getattr(response, "exit_code", 0)
+                response_error = getattr(response, "error", None) or getattr(
+                    response, "stderr", None
+                )
 
-            # Parse the result from stdout
-            output_lines = response_stdout.strip().split("\n")
-            for line in reversed(output_lines):
-                if not line.strip():
-                    continue
-                try:
-                    result_obj = json.loads(line)
+                if response_exit_code and response_exit_code != 0:
+                    raw_error = response_error or response_stdout or "Unknown error"
+                    # Log full error for debugging
+                    # log.warning(
+                    #     "Sandbox execution error (exit_code=%s): %s",
+                    #     response_exit_code,
+                    #     raw_error,
+                    # )
+                    # Extract clean error message for user display
+                    clean_error = _extract_error_message(raw_error)
+                    raise RuntimeError(clean_error)
+
+                # Parse the result from stdout
+                output_lines = response_stdout.strip().split("\n")
+                for line in reversed(output_lines):
+                    if not line.strip():
+                        continue
+                    try:
+                        result_obj = json.loads(line)
+                        if isinstance(result_obj, dict) and "result" in result_obj:
+                            result = result_obj["result"]
+                            if isinstance(result, (float, int, type(None))):
+                                return float(result) if result is not None else None
+                    except json.JSONDecodeError:
+                        continue
+
+                # Fallback: attempt to extract a JSON object containing "result"
+                for line in reversed(output_lines):
+                    if "result" not in line:
+                        continue
+                    start = line.find("{")
+                    end = line.rfind("}")
+                    if start == -1 or end == -1 or end <= start:
+                        continue
+                    try:
+                        result_obj = json.loads(line[start : end + 1])
+                    except json.JSONDecodeError:
+                        continue
                     if isinstance(result_obj, dict) and "result" in result_obj:
                         result = result_obj["result"]
                         if isinstance(result, (float, int, type(None))):
                             return float(result) if result is not None else None
-                except json.JSONDecodeError:
-                    continue
 
-            raise ValueError("Could not parse evaluation result from Daytona output")
+                # log.warning(
+                #     "Evaluation output did not include JSON result: %s", response_stdout
+                # )
+                raise ValueError(
+                    "Could not parse evaluation result from Daytona output"
+                )
 
-        except Exception as e:
-            log.error(f"Error during Daytona code execution: {e}", exc_info=True)
-            # print(f"Exception details: {type(e).__name__}: {e}")
-            raise RuntimeError(f"Error during Daytona code execution: {e}")
+            except Exception as e:
+                # log.warning(
+                #     f"Error during Daytona code execution:\n {e}", exc_info=True
+                # )
+                raise RuntimeError(e)
 
     def cleanup(self) -> None:
         """Clean up Daytona client resources."""

agenta/sdk/workflows/runners/local.py

@@ -1,20 +1,10 @@
-from typing import Any, Dict, Union, Text
-
-from RestrictedPython import safe_builtins, compile_restricted, utility_builtins
-from RestrictedPython.Eval import (
-    default_guarded_getiter,
-    default_guarded_getitem,
-)
-from RestrictedPython.Guards import (
-    guarded_iter_unpack_sequence,
-    full_write_guard,
-)
+from typing import Any, Dict, Union, Optional
 
 from agenta.sdk.workflows.runners.base import CodeRunner
 
 
 class LocalRunner(CodeRunner):
-    """Local code runner using RestrictedPython for safe execution."""
+    """Local code runner using direct Python execution."""
 
     def run(
         self,
@@ -23,9 +13,11 @@ class LocalRunner(CodeRunner):
         inputs: Dict[str, Any],
         output: Union[dict, str],
         correct_answer: Any,
+        runtime: Optional[str] = None,
+        templates: Optional[Dict[str, str]] = None,
     ) -> Union[float, None]:
         """
-        Execute provided Python code safely using RestrictedPython.
+        Execute provided Python code directly.
 
         Args:
             code: The Python code to be executed
@@ -33,55 +25,29 @@ class LocalRunner(CodeRunner):
             inputs: Inputs to be used during code execution
             output: The output of the app variant after being called
             correct_answer: The correct answer (or target) for comparison
-            code: The Python code to be executed
+            runtime: Runtime environment (only "python" is supported for local runner)
+            templates: Wrapper templates keyed by runtime (unused for local runner).
 
         Returns:
             Float score between 0 and 1, or None if execution fails
         """
-        # Define the available built-ins
-        local_builtins = safe_builtins.copy()
-
-        # Add the __import__ built-in function to the local builtins
-        local_builtins["__import__"] = __import__
-
-        # Define supported packages
-        allowed_imports = [
-            "math",
-            "random",
-            "datetime",
-            "json",
-            "requests",
-            "typing",
-        ]
-
-        # Create a dictionary to simulate allowed imports
-        allowed_modules = {}
-        for package_name in allowed_imports:
-            allowed_modules[package_name] = __import__(package_name)
-
-        # Add the allowed modules to the local built-ins
-        local_builtins.update(allowed_modules)
-        local_builtins.update(utility_builtins)
-
-        # Define the environment for the code execution
-        environment = {
-            "_getiter_": default_guarded_getiter,
-            "_getitem_": default_guarded_getitem,
-            "_iter_unpack_sequence_": guarded_iter_unpack_sequence,
-            "_write_": full_write_guard,
-            "__builtins__": local_builtins,
-        }
-
-        # Compile the code in a restricted environment
-        byte_code = compile_restricted(code, filename="<inline>", mode="exec")
-
-        # Call the evaluation function, extract the result if it exists
-        # and is a float between 0 and 1
+        # Normalize runtime: None means python
+        runtime = runtime or "python"
+
+        # Local runner only supports Python
+        if runtime != "python":
+            raise ValueError(
+                f"LocalRunner only supports 'python' runtime, got: {runtime}"
+            )
+
+        # Define the environment for code execution
+        environment: dict[str, Any] = dict()
+
+        # Execute the code directly
         try:
-            # Execute the code
-            exec(byte_code, environment)
+            exec(code, environment)
 
-            # Call the evaluation function, extract the result
+            # Call the evaluation function
             result = environment["evaluate"](app_params, inputs, output, correct_answer)
 
             # Attempt to convert result to float

agenta/sdk/workflows/runners/registry.py

@@ -19,7 +19,7 @@ def get_runner() -> CodeRunner:
     Registry to get the appropriate code runner based on environment configuration.
 
     Uses AGENTA_SERVICES_SANDBOX_RUNNER environment variable:
-    - "local" (default): Uses RestrictedPython for local execution
+    - "local" (default): Uses current container for local execution
     - "daytona": Uses Daytona remote sandbox
 
     Returns:

agenta/sdk/workflows/sandbox.py

@@ -1,4 +1,4 @@
-from typing import Union, Text, Dict, Any
+from typing import Union, Text, Dict, Any, Optional
 
 from agenta.sdk.workflows.runners import get_runner
 
@@ -29,11 +29,13 @@ def execute_code_safely(
     output: Union[dict, str],
     correct_answer: Any,  # for backward compatibility reasons
     code: Text,
+    runtime: Optional[str] = None,
+    templates: Optional[Dict[str, str]] = None,
 ) -> Union[float, None]:
     """
-    Execute the provided Python code safely.
+    Execute the provided code safely.
 
-    Uses the configured runner (local RestrictedPython or remote Daytona)
+    Uses the configured runner (local or remote Daytona)
     based on the AGENTA_SERVICES_SANDBOX_RUNNER environment variable.
 
     Args:
@@ -41,7 +43,9 @@ def execute_code_safely(
         - inputs (Dict[str, Any]): Inputs to be used during code execution.
         - output (Union[dict, str]): The output of the app variant after being called.
         - correct_answer (Any): The correct answer (or target) of the app variant.
-        - code (Text): The Python code to be executed.
+        - code (Text): The code to be executed.
+        - runtime (Optional[str]): Runtime environment (python, javascript, typescript). None = python.
+        - templates (Optional[Dict[str, str]]): Wrapper templates keyed by runtime.
 
     Returns:
         - (float): Result of the execution if successful. Should be between 0 and 1.
@@ -52,4 +56,12 @@ def execute_code_safely(
     if _runner is None:
         _runner = get_runner()
 
-    return _runner.run(code, app_params, inputs, output, correct_answer)
+    return _runner.run(
+        code,
+        app_params,
+        inputs,
+        output,
+        correct_answer,
+        runtime,
+        templates,
+    )