sandboxy 0.0.1__py3-none-any.whl

sandboxy/core/safe_eval.py

@@ -0,0 +1,165 @@
+"""Safe expression evaluation module.
+
+Provides sandboxed expression evaluation using simpleeval, replacing raw eval()
+calls throughout the codebase with a secure, consistent implementation.
+
+Security features:
+- Only whitelisted functions/operators are allowed
+- No access to __builtins__, __import__, or dunder methods
+- Consistent evaluation context across all callers
+- Proper error handling with typed exceptions
+"""
+
+import logging
+from typing import Any
+
+from simpleeval import EvalWithCompoundTypes, InvalidExpression
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluationError(Exception):
+    """Error during expression evaluation."""
+
+    def __init__(self, message: str, expression: str | None = None) -> None:
+        """Initialize evaluation error with message and optional expression.
+
+        Args:
+            message: Error message describing what went wrong.
+            expression: The expression that caused the error, if applicable.
+        """
+        super().__init__(message)
+        self.expression = expression
+
+
+# Safe functions available in all evaluations
+SAFE_FUNCTIONS = {
+    # Type conversions
+    "str": str,
+    "int": int,
+    "float": float,
+    "bool": bool,
+    "list": list,
+    "dict": dict,
+    # Math
+    "abs": abs,
+    "round": round,
+    "min": min,
+    "max": max,
+    "sum": sum,
+    # Collections
+    "len": len,
+    "any": any,
+    "all": all,
+    "sorted": sorted,
+    "reversed": lambda x: list(reversed(x)),
+    # Constants
+    "True": True,
+    "False": False,
+    "None": None,
+}
+
+
+def safe_eval(
+    expr: str,
+    context: dict[str, Any] | None = None,
+) -> Any:
+    """Safely evaluate an expression with restricted scope.
+
+    Uses simpleeval to provide a sandboxed evaluation environment.
+    Only whitelisted functions are available.
+
+    Args:
+        expr: Expression to evaluate.
+        context: Additional variables available in the expression.
+
+    Returns:
+        Result of the expression evaluation.
+
+    Raises:
+        EvaluationError: If the expression is invalid or evaluation fails.
+
+    """
+    if not expr or not expr.strip():
+        raise EvaluationError("Empty expression", expr)
+
+    # Build evaluation context
+    names = dict(SAFE_FUNCTIONS)
+    if context:
+        names.update(context)
+
+    try:
+        evaluator = EvalWithCompoundTypes(names=names)
+        return evaluator.eval(expr)
+    except InvalidExpression as e:
+        logger.debug("Invalid expression '%s': %s", expr, e)
+        msg = f"Invalid expression: {e}"
+        raise EvaluationError(msg, expr) from e
+    except (TypeError, ValueError, KeyError, AttributeError) as e:
+        logger.debug("Evaluation error for '%s': %s", expr, e)
+        msg = f"Evaluation failed: {e}"
+        raise EvaluationError(msg, expr) from e
+    except Exception as e:
+        # Catch any other errors from simpleeval
+        logger.warning("Unexpected evaluation error for '%s': %s", expr, e)
+        msg = f"Evaluation failed: {e}"
+        raise EvaluationError(msg, expr) from e
+
+
+def safe_eval_condition(
+    condition: str,
+    variables: dict[str, Any] | None = None,
+) -> bool:
+    """Safely evaluate a boolean condition.
+
+    Args:
+        condition: Condition expression (e.g., "x > 5", "name == 'test'").
+        variables: Variables available in the condition.
+
+    Returns:
+        Boolean result of the condition.
+
+    Note:
+        Returns False if evaluation fails (defensive default for conditions).
+
+    """
+    if not condition or not condition.strip():
+        return False
+
+    try:
+        result = safe_eval(condition, variables)
+        return bool(result)
+    except EvaluationError:
+        return False
+
+
+def safe_eval_formula(
+    formula: str,
+    check_values: dict[str, float],
+    env_state: dict[str, Any] | None = None,
+) -> float:
+    """Safely evaluate a score formula.
+
+    Args:
+        formula: Formula expression (e.g., "score * 0.5 + bonus").
+        check_values: Check result values available as variables.
+        env_state: Optional environment state for advanced formulas.
+
+    Returns:
+        Numeric result of the formula.
+
+    Raises:
+        EvaluationError: If the formula is invalid or doesn't return a number.
+
+    """
+    context = dict(check_values)
+    if env_state:
+        context["env_state"] = env_state
+
+    result = safe_eval(formula, context)
+
+    try:
+        return float(result)
+    except (TypeError, ValueError) as e:
+        msg = f"Formula must return a number, got {type(result).__name__}"
+        raise EvaluationError(msg, formula) from e

sandboxy/core/state.py

@@ -0,0 +1,234 @@
+"""Core state models for Sandboxy MDL and runtime."""
+
+from enum import Enum
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+Role = Literal["system", "user", "assistant", "tool"]
+
+
+class SessionState(str, Enum):
+    """State of an interactive session."""
+
+    IDLE = "idle"  # Created but not started
+    RUNNING = "running"  # Executing steps
+    AWAITING_USER = "awaiting_user"  # Paused waiting for user input
+    AWAITING_AGENT = "awaiting_agent"  # Waiting for LLM response
+    PAUSED = "paused"  # Manually paused
+    COMPLETED = "completed"  # All steps done
+    ERROR = "error"  # Execution failed
+
+
+class StepAction(str, Enum):
+    """Valid step actions in MDL."""
+
+    INJECT_USER = "inject_user"  # Add scripted user message
+    AWAIT_USER = "await_user"  # Wait for real user input (interactive)
+    AWAIT_AGENT = "await_agent"  # Wait for agent response
+    BRANCH = "branch"  # Conditional branching
+    TOOL_CALL = "tool_call"  # Direct tool invocation (not via agent)
+
+
+class ToolCall(BaseModel):
+    """A tool call made by the assistant."""
+
+    id: str
+    name: str
+    arguments: str  # JSON string
+
+
+class Message(BaseModel):
+    """A message in the conversation history."""
+
+    role: Role
+    content: str
+    tool_name: str | None = None
+    tool_call_id: str | None = None
+    tool_calls: list[ToolCall] | None = None
+
+
+class ToolRef(BaseModel):
+    """Reference to a tool in a module's environment."""
+
+    name: str
+    type: str
+    description: str = ""
+    config: dict[str, Any] = Field(default_factory=dict)
+
+
+class EnvConfig(BaseModel):
+    """Environment configuration for a module."""
+
+    sandbox_type: str = "local"
+    tools: list[ToolRef] = Field(default_factory=list)
+    initial_state: dict[str, Any] = Field(default_factory=dict)
+
+
+class Step(BaseModel):
+    """A step in the module's execution flow.
+
+    Actions:
+        inject_user: Add a scripted user message
+            params: {content: str}
+        await_user: Wait for real user input (interactive sessions only)
+            params: {prompt?: str, timeout?: int}
+        await_agent: Wait for agent response
+            params: {}
+        branch: Conditional branching
+            params: {branch_name: str}
+        tool_call: Direct tool invocation
+            params: {tool: str, action: str, args: dict}
+    """
+
+    id: str
+    action: str  # See StepAction enum
+    params: dict[str, Any] = Field(default_factory=dict)
+    condition: str | None = None  # Optional condition expression for conditional steps
+
+
+class BranchCondition(BaseModel):
+    """Condition for branching in the execution flow."""
+
+    expr: str
+    next_step: str
+
+
+class CheckKind(str, Enum):
+    """Types of evaluation checks."""
+
+    CONTAINS = "contains"  # Check if target contains/doesn't contain a value
+    REGEX = "regex"  # Check if target matches a regex pattern
+    COUNT = "count"  # Check count of items (min/max)
+    TOOL_CALLED = "tool_called"  # Check if a tool was called
+    EQUALS = "equals"  # Check if target equals a value
+    ENV_STATE = "env_state"  # Check environment state value
+    # Legacy support
+    DETERMINISTIC = "deterministic"  # Raw Python expression (deprecated)
+    LLM = "llm"  # LLM-based evaluation (not implemented)
+
+
+class CheckTarget(str, Enum):
+    """Valid targets for evaluation checks."""
+
+    AGENT_MESSAGES = "agent_messages"  # All agent message content
+    USER_MESSAGES = "user_messages"  # All user message content
+    ALL_MESSAGES = "all_messages"  # All message content
+    TOOL_CALLS = "tool_calls"  # List of tool calls
+    LAST_AGENT_MESSAGE = "last_agent_message"  # Most recent agent message
+    LAST_USER_MESSAGE = "last_user_message"  # Most recent user message
+
+
+class EvaluationCheck(BaseModel):
+    """An evaluation check to run after module execution.
+
+    Predefined check types:
+        contains: Check if target contains a string
+            - target: what to search (e.g., "agent_messages")
+            - value: string to look for
+            - expected: True if should contain, False if should not (default: True)
+            - case_sensitive: whether to do case-sensitive match (default: False)
+
+        regex: Check if target matches a regex pattern
+            - target: what to search
+            - pattern: regex pattern
+            - expected: True if should match, False if should not (default: True)
+
+        count: Check count of items
+            - target: what to count (e.g., "agent_messages", "tool_calls")
+            - min: minimum count (optional)
+            - max: maximum count (optional)
+
+        tool_called: Check if a specific tool was called
+            - tool: tool name
+            - action: action name (optional)
+            - expected: True if should be called, False if should not (default: True)
+
+        equals: Check if a value equals expected
+            - target: what to check (e.g., "env.order_status")
+            - value: expected value
+
+        env_state: Check environment state
+            - key: state key to check
+            - value: expected value
+
+        deterministic: (deprecated) Raw Python expression
+            - expr: Python expression string
+    """
+
+    name: str
+    kind: str  # See CheckKind enum
+    # Common fields
+    target: str | None = None  # What to evaluate (see CheckTarget)
+    value: Any = None  # Value to check against
+    expected: bool = True  # Expected result (True = should match/contain)
+    # Type-specific fields
+    pattern: str | None = None  # For regex
+    case_sensitive: bool = False  # For contains
+    min: int | None = None  # For count
+    max: int | None = None  # For count
+    tool: str | None = None  # For tool_called
+    action: str | None = None  # For tool_called
+    key: str | None = None  # For env_state
+    # Legacy support
+    config: dict[str, Any] = Field(default_factory=dict)  # For deterministic/llm
+
+
+class VariableOption(BaseModel):
+    """An option for a select/dropdown variable."""
+
+    value: str
+    label: str
+
+
+class ModuleVariable(BaseModel):
+    """A configurable variable for a module."""
+
+    name: str
+    label: str
+    description: str = ""
+    type: str = "string"  # "string" | "number" | "boolean" | "select" | "slider"
+    default: Any = None
+    options: list[VariableOption] | None = None  # For select type
+    min: float | None = None  # For slider type
+    max: float | None = None  # For slider type
+    step: float | None = None  # For slider type
+
+
+class ScoringConfig(BaseModel):
+    """Configuration for how the final score is computed."""
+
+    # Score formula using check names as variables, e.g.:
+    # "Profit * 2 + Reputation + CustomersServed * 5 - CustomersLost * 10"
+    formula: str | None = None
+
+    # If no formula, use weighted average. Default weight is 1.0.
+    weights: dict[str, float] = Field(default_factory=dict)
+
+    # Normalization settings
+    normalize: bool = False  # Normalize score to 0-1 range
+    min_score: float = 0.0  # Expected minimum for normalization
+    max_score: float = 100.0  # Expected maximum for normalization
+
+
+class ModuleSpec(BaseModel):
+    """Complete specification for an MDL module."""
+
+    id: str
+    description: str = ""
+    variables: list[ModuleVariable] = Field(default_factory=list)
+    agent_config: dict[str, Any] = Field(default_factory=dict)  # Override agent settings
+    environment: EnvConfig
+    steps: list[Step] = Field(default_factory=list)
+    branches: dict[str, list[Step]] = Field(default_factory=dict)
+    evaluation: list[EvaluationCheck] = Field(default_factory=list)
+    scoring: ScoringConfig = Field(default_factory=ScoringConfig)  # Score computation config
+
+
+class EvaluationResult(BaseModel):
+    """Result of running evaluation checks."""
+
+    checks: dict[str, Any] = Field(default_factory=dict)
+    score: float = 0.0
+    num_events: int = 0
+    status: str = "ok"

sandboxy/datasets/__init__.py

@@ -0,0 +1,20 @@
+"""Dataset support for multi-case benchmarking."""
+
+from sandboxy.datasets.loader import Dataset, TestCase, load_dataset, load_multiple_datasets
+from sandboxy.datasets.runner import (
+    CaseResult,
+    DatasetResult,
+    run_dataset,
+    run_dataset_parallel,
+)
+
+__all__ = [
+    "Dataset",
+    "TestCase",
+    "load_dataset",
+    "load_multiple_datasets",
+    "CaseResult",
+    "DatasetResult",
+    "run_dataset",
+    "run_dataset_parallel",
+]

sandboxy/datasets/loader.py

@@ -0,0 +1,193 @@
+"""Dataset loader for multi-case benchmarking."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from itertools import product
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+@dataclass
+class TestCase:
+    """Single test case from a dataset."""
+
+    id: str
+    expected: list[str] = field(default_factory=list)
+    variables: dict[str, Any] = field(default_factory=dict)
+    tool_responses: dict[str, Any] = field(default_factory=dict)
+    tags: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Dataset:
+    """Collection of test cases."""
+
+    id: str
+    name: str = ""
+    description: str = ""
+    scenario_id: str | None = None
+    cases: list[TestCase] = field(default_factory=list)
+    source_path: Path | None = None
+
+
+def load_dataset(path: Path) -> Dataset:
+    """Load dataset from YAML file.
+
+    Supports both manual case definitions and generators.
+
+    Args:
+        path: Path to the dataset YAML file
+
+    Returns:
+        Dataset with loaded/generated test cases
+
+    """
+    with open(path) as f:
+        data = yaml.safe_load(f)
+
+    dataset_id = data.get("id", path.stem)
+    dataset = Dataset(
+        id=dataset_id,
+        name=data.get("name", dataset_id),
+        description=data.get("description", ""),
+        scenario_id=data.get("scenario_id"),
+        source_path=path,
+    )
+
+    if "cases" in data:
+        for case_data in data["cases"]:
+            expected_raw = case_data.get("expected")
+            if expected_raw is None:
+                expected = []
+            elif isinstance(expected_raw, list):
+                expected = expected_raw
+            else:
+                expected = [expected_raw]
+
+            case = TestCase(
+                id=case_data.get("id", f"case_{len(dataset.cases)}"),
+                expected=expected,
+                variables=case_data.get("variables", {}),
+                tool_responses=case_data.get("tool_responses", {}),
+                tags=case_data.get("tags", []),
+                metadata=case_data.get("metadata", {}),
+            )
+            dataset.cases.append(case)
+
+    if "generator" in data:
+        generated = _generate_cases(data["generator"])
+        dataset.cases.extend(generated)
+
+    return dataset
+
+
+def load_multiple_datasets(paths: list[Path]) -> Dataset:
+    """Load and merge multiple datasets."""
+    if not paths:
+        return Dataset(id="empty", name="Empty Dataset")
+
+    merged = load_dataset(paths[0])
+
+    for path in paths[1:]:
+        ds = load_dataset(path)
+        merged.cases.extend(ds.cases)
+        merged.id = f"{merged.id}+{ds.id}"
+        merged.name = f"{merged.name} + {ds.name}"
+
+    return merged
+
+
+def _generate_cases(config: dict[str, Any]) -> list[TestCase]:
+    """Generate cases from dimension combinations."""
+    dimensions = config.get("dimensions", {})
+    rules = config.get("rules", [])
+    tool_mapping = config.get("tool_mapping", {})
+
+    if not dimensions:
+        return []
+
+    dim_names = list(dimensions.keys())
+    dim_values = [dimensions[name] for name in dim_names]
+
+    cases = []
+    for i, combo in enumerate(product(*dim_values)):
+        case_data = dict(zip(dim_names, combo, strict=True))
+        expected = _find_expected(case_data, rules)
+        tool_responses = _build_tool_responses(case_data, tool_mapping)
+
+        cases.append(
+            TestCase(
+                id=f"gen_{i:04d}",
+                expected=expected,
+                variables=case_data.copy(),
+                tool_responses=tool_responses,
+            )
+        )
+
+    return cases
+
+
+def _find_expected(case_data: dict[str, Any], rules: list[dict]) -> list[str]:
+    """Find expected outcome(s) from rules."""
+    for rule in rules:
+        if "when" in rule:
+            if _matches_rule(case_data, rule["when"]):
+                expected = rule.get("expected")
+                if expected is None:
+                    return []
+                return expected if isinstance(expected, list) else [expected]
+        elif "otherwise" in rule or "expected" in rule:
+            expected = rule.get("expected")
+            if expected is None:
+                return []
+            return expected if isinstance(expected, list) else [expected]
+
+    return []
+
+
+def _matches_rule(data: dict[str, Any], conditions: dict[str, Any]) -> bool:
+    """Check if case data matches rule conditions."""
+    for key, condition in conditions.items():
+        value = data.get(key)
+
+        if isinstance(condition, dict):
+            if "gte" in condition and value < condition["gte"]:
+                return False
+            if "lte" in condition and value > condition["lte"]:
+                return False
+            if "gt" in condition and value <= condition["gt"]:
+                return False
+            if "lt" in condition and value >= condition["lt"]:
+                return False
+            if "eq" in condition and value != condition["eq"]:
+                return False
+            if "ne" in condition and value == condition["ne"]:
+                return False
+            if "in" in condition and value not in condition["in"]:
+                return False
+        elif value != condition:
+            return False
+
+    return True
+
+
+def _build_tool_responses(
+    case_data: dict[str, Any], tool_mapping: dict[str, Any]
+) -> dict[str, Any]:
+    """Build tool response overrides from case data and mapping."""
+    tool_responses: dict[str, Any] = {}
+
+    for tool_name, mapping in tool_mapping.items():
+        response: dict[str, Any] = {}
+        for response_key, source_key in mapping.items():
+            if isinstance(source_key, str) and source_key in case_data:
+                response[response_key] = case_data[source_key]
+            else:
+                response[response_key] = source_key
+        tool_responses[tool_name] = response
+
+    return tool_responses