isage-benchmark-agent 0.1.0.1__cp311-none-any.whl

sage/benchmark/benchmark_agent/config/config_loader.py

@@ -0,0 +1,176 @@
+"""
+Configuration Loader for Agent Benchmark Experiments
+
+Provides utilities for loading and parsing YAML configuration files
+with environment variable substitution.
+"""
+
+import os
+import re
+from pathlib import Path
+from typing import Any, Optional
+
+import yaml
+
+from sage.benchmark.benchmark_agent.experiments.base_experiment import (
+    ExperimentConfig,
+    create_config,
+)
+
+
+class ConfigLoader:
+    """
+    Loader for experiment YAML configurations.
+
+    Supports:
+    - Environment variable substitution (${VAR} or $VAR)
+    - Special ${PROJECT_ROOT} variable
+    - Type-safe config object creation
+    """
+
+    @staticmethod
+    def _find_project_root() -> Path:
+        """Find project root directory (where .git exists)."""
+        current = Path.cwd()
+        while current.parent != current:
+            if (current / ".git").exists():
+                return current
+            current = current.parent
+        return Path.cwd()
+
+    @staticmethod
+    def _expand_vars(value: Any, context: Optional[dict[str, str]] = None) -> Any:
+        """
+        Recursively expand environment variables in config values.
+
+        Args:
+            value: Config value (can be str, dict, list, etc.)
+            context: Additional variable context
+
+        Returns:
+            Value with expanded variables
+        """
+        if context is None:
+            context = {}
+
+        # Add PROJECT_ROOT to context
+        if "PROJECT_ROOT" not in context:
+            context["PROJECT_ROOT"] = str(ConfigLoader._find_project_root())
+
+        if isinstance(value, str):
+            # Pattern matches ${VAR} or $VAR
+            pattern = r"\$\{([^}]+)\}|\$([A-Za-z_][A-Za-z0-9_]*)"
+
+            def replacer(match):
+                var_name = match.group(1) or match.group(2)
+                # Check context first, then environment
+                return context.get(var_name, os.environ.get(var_name, match.group(0)))
+
+            return re.sub(pattern, replacer, value)
+
+        elif isinstance(value, dict):
+            return {k: ConfigLoader._expand_vars(v, context) for k, v in value.items()}
+
+        elif isinstance(value, list):
+            return [ConfigLoader._expand_vars(item, context) for item in value]
+
+        return value
+
+    @classmethod
+    def load_yaml(cls, config_path) -> dict[str, Any]:
+        """
+        Load and parse YAML config file.
+
+        Args:
+            config_path: Path to YAML config file (str or Path)
+
+        Returns:
+            Parsed config dictionary with expanded variables
+        """
+        config_path = Path(config_path)
+        if not config_path.exists():
+            raise FileNotFoundError(f"Config file not found: {config_path}")
+
+        with open(config_path, encoding="utf-8") as f:
+            raw_config = yaml.safe_load(f)
+
+        # Expand environment variables
+        expanded = cls._expand_vars(raw_config)
+
+        return expanded
+
+    @classmethod
+    def load_config(cls, config_path) -> ExperimentConfig:
+        """
+        Load YAML config and create typed config object.
+
+        Args:
+            config_path: Path to YAML config file (str or Path)
+
+        Returns:
+            ExperimentConfig subclass instance
+        """
+        config_dict = cls.load_yaml(config_path)
+        return create_config(config_dict)
+
+    @classmethod
+    def load_default_config(cls) -> dict[str, Any]:
+        """
+        Load default configuration.
+
+        Returns:
+            Default config dictionary
+        """
+        default_path = Path(__file__).parent.parent / "config" / "default_config.yaml"
+        if default_path.exists():
+            return cls.load_yaml(default_path)
+        return {}
+
+    @classmethod
+    def merge_configs(
+        cls, base_config: dict[str, Any], override_config: dict[str, Any]
+    ) -> dict[str, Any]:
+        """
+        Merge two configuration dictionaries.
+
+        Args:
+            base_config: Base configuration
+            override_config: Override configuration
+
+        Returns:
+            Merged configuration (override takes precedence)
+        """
+        merged = base_config.copy()
+
+        for key, value in override_config.items():
+            if key in merged and isinstance(merged[key], dict) and isinstance(value, dict):
+                merged[key] = cls.merge_configs(merged[key], value)
+            else:
+                merged[key] = value
+
+        return merged
+
+
+def load_config_with_defaults(config_path: Path) -> ExperimentConfig:
+    """
+    Load config with default values merged in.
+
+    Args:
+        config_path: Path to experiment config file
+
+    Returns:
+        ExperimentConfig with defaults applied
+    """
+    loader = ConfigLoader()
+
+    # Load default config
+    defaults = loader.load_default_config()
+
+    # Load experiment config
+    exp_config = loader.load_yaml(config_path)
+
+    # Merge
+    merged = loader.merge_configs(defaults, exp_config)
+
+    # Create typed config
+    return create_config(merged)

sage/benchmark/benchmark_agent/config/default_config.yaml

@@ -0,0 +1,24 @@
+# Default Configuration for Agent Benchmark Experiments
+
+# Data settings
+profile: "quick_eval"  # agent_eval usage profile
+split: "dev"           # Data split: train/dev/test
+max_samples:           # Limit samples (null = all)
+
+# Randomness
+seed: 42
+
+# Output and reporting
+report:
+  format: ["json", "markdown"]
+  include_breakdowns: true
+  path: "${PROJECT_ROOT}/outputs/agent_benchmark"
+  markdown_template:
+
+# Metrics (common defaults, overridden per experiment)
+metrics:
+- "accuracy"
+
+# Logging
+verbose: true
+log_level: "INFO"

sage/benchmark/benchmark_agent/config/planning_exp.yaml

@@ -0,0 +1,34 @@
+# Planning Experiment Configuration
+
+experiment: planning
+
+# Data configuration
+profile: "full_eval"
+split: "dev"
+max_samples:
+
+# Strategy configuration
+planner: "baseline.template"
+min_steps: 5
+max_steps: 10
+planner_params:
+  allow_tool_reuse: true
+  enforce_sequence: true
+
+# Metrics to evaluate
+metrics:
+- "plan_success_rate"
+- "step_accuracy"
+- "tool_sequence_match"
+- "average_plan_length"
+
+# Report configuration
+report:
+  format: ["json", "markdown"]
+  include_breakdowns: true
+  path: "${PROJECT_ROOT}/outputs/agent_benchmark/planning"
+  markdown_template:
+
+# Reproducibility
+seed: 42
+verbose: true

sage/benchmark/benchmark_agent/config/timing_detection_exp.yaml

@@ -0,0 +1,34 @@
+# Timing Detection Experiment Configuration
+
+experiment: timing_detection
+
+# Data configuration
+profile: "full_eval"
+split: "dev"
+max_samples:
+
+# Strategy configuration
+detector: "baseline.threshold"
+threshold: 0.5
+detector_params:
+  use_context: true
+  confidence_threshold: 0.7
+
+# Metrics to evaluate
+metrics:
+- "f1_score"
+- "precision"
+- "recall"
+- "accuracy"
+- "confusion_matrix"
+
+# Report configuration
+report:
+  format: ["json", "markdown"]
+  include_breakdowns: true
+  path: "${PROJECT_ROOT}/outputs/agent_benchmark/timing_detection"
+  markdown_template:
+
+# Reproducibility
+seed: 42
+verbose: true

sage/benchmark/benchmark_agent/config/tool_selection_exp.yaml

@@ -0,0 +1,32 @@
+# Tool Selection Experiment Configuration
+
+experiment: tool_selection
+
+# Data configuration
+profile: "quick_eval"
+split: "dev"
+max_samples:
+
+# Strategy configuration
+selector: "baseline.keyword"
+top_k: 5
+selector_params:
+  min_score: 0.1
+
+# Metrics to evaluate
+metrics:
+- "top_k_accuracy"
+- "recall@5"
+- "precision@5"
+- "mrr"
+
+# Report configuration
+report:
+  format: ["json", "markdown"]
+  include_breakdowns: true
+  path: "${PROJECT_ROOT}/outputs/agent_benchmark/tool_selection"
+  markdown_template:
+
+# Reproducibility
+seed: 42
+verbose: true

sage/benchmark/benchmark_agent/data_paths.py

@@ -0,0 +1,332 @@
+"""
+Data Paths Configuration for Agent Benchmark
+
+This module provides centralized management of data paths for agent benchmark.
+It follows SAGE's two-layer data architecture:
+1. Source Layer: Original data accessed via DataManager
+2. Runtime Layer: Generated data for specific experiments
+
+Usage:
+    from sage.benchmark.benchmark_agent.data_paths import (
+        get_source_paths,
+        get_runtime_paths,
+        ensure_runtime_dirs,
+        DataPathsConfig,
+    )
+
+    # Get source data paths (read-only)
+    source = get_source_paths()
+    tools_file = source.tools_catalog
+
+    # Get runtime data paths
+    runtime = get_runtime_paths()
+    output_dir = runtime.tool_selection_dir
+
+    # Ensure runtime directories exist
+    ensure_runtime_dirs()
+"""
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+
+def _find_sage_root() -> Path:
+    """
+    Find SAGE project root directory.
+
+    Looks for .git directory or SAGE_ROOT environment variable.
+    """
+    # Check environment variable first
+    if "SAGE_ROOT" in os.environ:
+        return Path(os.environ["SAGE_ROOT"])
+
+    # Walk up from current file to find project root
+    current = Path(__file__).resolve()
+    while current.parent != current:
+        if (current / ".git").exists() and (current / "packages").exists():
+            return current
+        current = current.parent
+
+    # Fallback to current working directory
+    return Path.cwd()
+
+
+def _find_package_root() -> Path:
+    """Find sage-benchmark package root."""
+    current = Path(__file__).resolve()
+    # Navigate up to find sage-benchmark/src
+    while current.parent != current:
+        if current.name == "sage-benchmark":
+            return current
+        current = current.parent
+
+    # Fallback
+    sage_root = _find_sage_root()
+    return sage_root / "packages" / "sage-benchmark"
+
+
+@dataclass
+class SourcePaths:
+    """
+    Paths to source data (read-only, via DataManager).
+
+    These are the original data files that should not be modified directly.
+    Use DataManager for standard access.
+    """
+
+    # Root directories
+    data_root: Path
+
+    # Agent benchmark data
+    benchmark_dir: Path
+    benchmark_splits_dir: Path
+    benchmark_metadata_dir: Path
+
+    # Agent tools data
+    tools_dir: Path
+    tools_data_dir: Path
+
+    # Agent SFT data
+    sft_dir: Path
+    sft_data_dir: Path
+
+    @property
+    def tool_selection_file(self) -> Path:
+        """Tool selection benchmark data file."""
+        return self.benchmark_splits_dir / "tool_selection.jsonl"
+
+    @property
+    def task_planning_file(self) -> Path:
+        """Task planning benchmark data file."""
+        return self.benchmark_splits_dir / "task_planning.jsonl"
+
+    @property
+    def timing_judgment_file(self) -> Path:
+        """Timing judgment benchmark data file."""
+        return self.benchmark_splits_dir / "timing_judgment.jsonl"
+
+    @property
+    def tools_catalog(self) -> Path:
+        """Tools catalog file."""
+        return self.tools_data_dir / "tool_catalog.jsonl"
+
+    @property
+    def tools_categories(self) -> Path:
+        """Tools categories file."""
+        return self.tools_data_dir / "categories.json"
+
+    @property
+    def sft_conversations(self) -> Path:
+        """SFT conversations file."""
+        return self.sft_data_dir / "sft_conversations.jsonl"
+
+
+@dataclass
+class RuntimePaths:
+    """
+    Paths to runtime/generated data.
+
+    These are generated by prepare_*.py scripts for specific experiments.
+    Stored in .sage/benchmark/data/ (gitignored).
+    """
+
+    # Root directories
+    data_root: Path
+    results_root: Path
+
+    # Task-specific data directories
+    tool_selection_dir: Path
+    task_planning_dir: Path
+    timing_judgment_dir: Path
+
+    # Results directories
+    tool_selection_results: Path
+    task_planning_results: Path
+    timing_judgment_results: Path
+
+    @property
+    def tool_selection_base(self) -> Path:
+        """Base tool selection data file."""
+        return self.tool_selection_dir / "tool_selection.jsonl"
+
+    def tool_selection_with_candidates(self, num_candidates: int) -> Path:
+        """Tool selection data file with specific candidate pool size."""
+        return self.tool_selection_dir / f"tool_selection_{num_candidates}.jsonl"
+
+    @property
+    def task_planning_base(self) -> Path:
+        """Base task planning data file."""
+        return self.task_planning_dir / "task_planning.jsonl"
+
+    def timing_split_file(self, split: str) -> Path:
+        """Timing judgment data file for specific split."""
+        return self.timing_judgment_dir / f"{split}.jsonl"
+
+
+@dataclass
+class DataPathsConfig:
+    """
+    Complete data paths configuration.
+
+    Provides access to both source and runtime paths.
+    """
+
+    source: SourcePaths
+    runtime: RuntimePaths
+
+    @property
+    def sage_root(self) -> Path:
+        """SAGE project root."""
+        return _find_sage_root()
+
+    @property
+    def package_root(self) -> Path:
+        """sage-benchmark package root."""
+        return _find_package_root()
+
+
+# Module-level cached config
+_config: Optional[DataPathsConfig] = None
+
+
+def get_source_paths() -> SourcePaths:
+    """
+    Get source data paths.
+
+    Returns:
+        SourcePaths object with paths to original data files.
+    """
+    package_root = _find_package_root()
+    data_root = package_root / "src" / "sage" / "data" / "sources"
+
+    return SourcePaths(
+        data_root=data_root,
+        # Agent benchmark
+        benchmark_dir=data_root / "agent_benchmark",
+        benchmark_splits_dir=data_root / "agent_benchmark" / "splits",
+        benchmark_metadata_dir=data_root / "agent_benchmark" / "metadata",
+        # Agent tools
+        tools_dir=data_root / "agent_tools",
+        tools_data_dir=data_root / "agent_tools" / "data",
+        # Agent SFT
+        sft_dir=data_root / "agent_sft",
+        sft_data_dir=data_root / "agent_sft" / "data",
+    )
+
+
+def get_runtime_paths() -> RuntimePaths:
+    """
+    Get runtime/generated data paths.
+
+    Returns:
+        RuntimePaths object with paths to generated data files.
+    """
+    sage_root = _find_sage_root()
+
+    # Use .sage directory for runtime data
+    data_root = sage_root / ".sage" / "benchmark" / "data"
+    results_root = sage_root / ".sage" / "benchmark" / "results"
+
+    return RuntimePaths(
+        data_root=data_root,
+        results_root=results_root,
+        # Data directories
+        tool_selection_dir=data_root / "tool_selection",
+        task_planning_dir=data_root / "task_planning",
+        timing_judgment_dir=data_root / "timing_judgment",
+        # Results directories
+        tool_selection_results=results_root / "tool_selection",
+        task_planning_results=results_root / "task_planning",
+        timing_judgment_results=results_root / "timing_judgment",
+    )
+
+
+def get_data_paths_config() -> DataPathsConfig:
+    """
+    Get complete data paths configuration.
+
+    Returns cached config object.
+    """
+    global _config
+    if _config is None:
+        _config = DataPathsConfig(
+            source=get_source_paths(),
+            runtime=get_runtime_paths(),
+        )
+    return _config
+
+
+def ensure_runtime_dirs() -> None:
+    """
+    Ensure all runtime directories exist.
+
+    Call this before writing generated data.
+    """
+    runtime = get_runtime_paths()
+
+    # Create data directories
+    runtime.tool_selection_dir.mkdir(parents=True, exist_ok=True)
+    runtime.task_planning_dir.mkdir(parents=True, exist_ok=True)
+    runtime.timing_judgment_dir.mkdir(parents=True, exist_ok=True)
+
+    # Create results directories
+    runtime.tool_selection_results.mkdir(parents=True, exist_ok=True)
+    runtime.task_planning_results.mkdir(parents=True, exist_ok=True)
+    runtime.timing_judgment_results.mkdir(parents=True, exist_ok=True)
+
+
+def print_data_paths_summary() -> None:
+    """Print summary of all data paths for debugging."""
+    config = get_data_paths_config()
+
+    print("\n" + "=" * 60)
+    print("SAGE Agent Benchmark Data Paths")
+    print("=" * 60)
+
+    print("\n📂 Source Data (read-only, via DataManager):")
+    print(f"   Root: {config.source.data_root}")
+    print(f"   Tool Selection: {config.source.tool_selection_file}")
+    print(f"   Task Planning: {config.source.task_planning_file}")
+    print(f"   Timing Judgment: {config.source.timing_judgment_file}")
+    print(f"   Tools Catalog: {config.source.tools_catalog}")
+    print(f"   SFT Data: {config.source.sft_conversations}")
+
+    print("\n📂 Runtime Data (generated, in .sage/):")
+    print(f"   Root: {config.runtime.data_root}")
+    print(f"   Tool Selection: {config.runtime.tool_selection_dir}")
+    print(f"   Task Planning: {config.runtime.task_planning_dir}")
+    print(f"   Timing Judgment: {config.runtime.timing_judgment_dir}")
+
+    print("\n📂 Results:")
+    print(f"   Root: {config.runtime.results_root}")
+
+    # Check existence
+    print("\n📋 Status:")
+    source_exists = config.source.tool_selection_file.exists()
+    runtime_exists = config.runtime.tool_selection_dir.exists()
+    print(f"   Source data exists: {'✅' if source_exists else '❌'}")
+    print(f"   Runtime data exists: {'✅' if runtime_exists else '❌'}")
+    print("=" * 60)
+
+
+# Backwards compatibility aliases
+def get_data_paths() -> dict:
+    """
+    Legacy function for backwards compatibility.
+
+    Returns dict with source data paths.
+    Prefer using get_source_paths() or get_runtime_paths() directly.
+    """
+    source = get_source_paths()
+    return {
+        "tools_dir": source.tools_data_dir,
+        "benchmark_dir": source.benchmark_splits_dir,
+        "output_dir": get_runtime_paths().tool_selection_dir,
+    }
+
+
+if __name__ == "__main__":
+    # Print summary when run directly
+    print_data_paths_summary()