isage-benchmark-agent 0.1.0.1__cp311-none-any.whl

isage_benchmark_agent-0.1.0.1.dist-info/METADATA

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: isage-benchmark-agent
+Version: 0.1.0.1
+Summary: SAGE Benchmark Agent - Agent capability evaluation framework
+Author-email: IntelliStream Team <shuhao_zhang@hust.edu.cn>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/intellistream/sage-agent-benchmark
+Project-URL: Documentation, https://github.com/intellistream/sage-agent-benchmark#readme
+Project-URL: Repository, https://github.com/intellistream/sage-agent-benchmark
+Project-URL: Issues, https://github.com/intellistream/sage-agent-benchmark/issues
+Keywords: sage,benchmark,agent,tool-selection,planning,evaluation,intellistream
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: isage-common
+Requires-Dist: isage-libs
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: numpy<2.3.0,>=1.26.0
+Requires-Dist: typer<1.0.0,>=0.15.0
+Requires-Dist: rich<14.0.0,>=13.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff==0.14.6; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: types-PyYAML>=6.0.0; extra == "dev"
+Provides-Extra: all
+Requires-Dist: isage-benchmark-agent[dev]; extra == "all"
+Dynamic: license-file
+
+# SAGE Benchmark Agent
+
+Configuration-driven experiment framework for evaluating agent capabilities.
+
+## Features
+
+- **Tool Selection Evaluation**: Tool retrieval and ranking benchmarks
+- **Planning Evaluation**: Multi-step planning with tool composition
+- **Timing Detection**: Timing judgment for tool invocation decisions
+
+## Quick Start
+
+```bash
+# Install
+pip install isage-benchmark-agent
+
+# Run tool selection experiment
+sage-agent-bench tool-selection --config config/tool_selection_exp.yaml
+
+# Run planning experiment
+sage-agent-bench planning --config config/planning_exp.yaml
+```
+
+## Documentation
+
+See [benchmark_agent/README.md](src/sage/benchmark/benchmark_agent/README.md) for detailed documentation.
+
+## Development
+
+```bash
+# Clone
+git clone https://github.com/intellistream/sage-agent-benchmark.git
+cd sage-agent-benchmark
+
+# Setup virtual environment
+python -m venv .venv
+source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+
+# Install in development mode
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+```
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.

isage_benchmark_agent-0.1.0.1.dist-info/RECORD

@@ -0,0 +1,51 @@
+isage_benchmark_agent-0.1.0.1.dist-info/licenses/LICENSE,sha256=vBNVIGkYYZY0B8f0Ui1ITYwRu7WNtSwyxvIAVGYS6jU,1075
+sage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sage/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sage/benchmark/benchmark_agent/__init__.py,sha256=Bnlx9jxIxIq0xo1Q-3Hrr2VRJZq0g6nHhXcmffotBYc,2807
+sage/benchmark/benchmark_agent/__main__.py,sha256=3yDJ34Wuk9TaNnOoKWAsZL-74H8xiw6xFTeaobN77ew,5464
+sage/benchmark/benchmark_agent/acebench_loader.py,sha256=NHCuCoJXczv2OhjaSsQmSk2l8f4wc04Nyga79Wn26VQ,11571
+sage/benchmark/benchmark_agent/adapter_registry.py,sha256=MjliEzacQHNiHMohpYe8AHdxIR06e2h25vDHX2yLdYo,123890
+sage/benchmark/benchmark_agent/data_paths.py,sha256=qpVxRNkFe7B2ktuFvGFp30JhVCfL4YBWiLHZ3c4FjZs,9761
+sage/benchmark/benchmark_agent/tools_loader.py,sha256=25OT8_8F2NyuydrmCcRkqaOWc2FneAg3mqcoOHFMlOs,6324
+sage/benchmark/benchmark_agent/config/config_loader.py,sha256=a5Du-HIIMjpa9rzWdlObksrwVU3cDtLAK4QfepnbmI4,4954
+sage/benchmark/benchmark_agent/config/default_config.yaml,sha256=iHNVAkou8UxcIOMBEuJheNeVI0fAlfe-BV9inzxw9_s,532
+sage/benchmark/benchmark_agent/config/planning_exp.yaml,sha256=nyimmpQ725skpvEF1eNJpYudkCxYuh2hz6u2e1velNE,607
+sage/benchmark/benchmark_agent/config/timing_detection_exp.yaml,sha256=058TwEeeP8RkzjV7hUbYWFBctrzpcbAXKAlZzw6fJdI,604
+sage/benchmark/benchmark_agent/config/tool_selection_exp.yaml,sha256=dfar6m9XJ1AHsoqVqEN3iN5x8Ka2M3UpvzhpXlCkc6o,544
+sage/benchmark/benchmark_agent/evaluation/__init__.py,sha256=Ypoer04sin7kTryIJWOqIImifonmXvhHWhOAwrhHK10,6615
+sage/benchmark/benchmark_agent/evaluation/evaluator.py,sha256=kvve7bWrkDLfDNub-bL4EYoCRCHGkjVhNIS10_aMAGM,8003
+sage/benchmark/benchmark_agent/evaluation/metrics.py,sha256=6SkPkFexcRpoUvpF7F9AWgGVVhSvazdKTZ6NlADULcU,19060
+sage/benchmark/benchmark_agent/evaluation/report_builder.py,sha256=OYh2JdxsgyE9ovWS0o-DKBeoJWuPV8qx5fIVjrcwFdg,7029
+sage/benchmark/benchmark_agent/evaluation/unified_tool_selection.py,sha256=7UqR6RwyO440oO3Dto6XUnzGO2zr0l07A2cY7fQrYl8,19665
+sage/benchmark/benchmark_agent/evaluation/analyzers/__init__.py,sha256=e_kzn4nWKe6l4L0OtSd4-V7Tt064bzwvlz3BsfQXBZI,282
+sage/benchmark/benchmark_agent/evaluation/analyzers/planning_analyzer.py,sha256=4RHWITAYgm4oefaPHFJaoeGW5v95_VgtLwKUP4DcBlQ,3931
+sage/benchmark/benchmark_agent/evaluation/analyzers/timing_analyzer.py,sha256=pgID5v_lcHNQi1U0NH2XJhXFkCBuAMUOITzTdLmvgl8,4808
+sage/benchmark/benchmark_agent/evaluation/analyzers/tool_selection_analyzer.py,sha256=nWFxeGMwJO7ZOR_KB4KJAVZ5voQ_c1QFlM7PoFP1QHI,4086
+sage/benchmark/benchmark_agent/experiments/__init__.py,sha256=w3aKDv8nfgXceZX81KFtOwxuRXhbETLSxUeQDD7HONY,1574
+sage/benchmark/benchmark_agent/experiments/base_experiment.py,sha256=6u5kVNIDysGIck6qAbs9ihSn_bBBqiLWPn_p-EBLqEE,8519
+sage/benchmark/benchmark_agent/experiments/method_comparison.py,sha256=3LP8x7PjnhssTnKabQiBK29TxkCQ0ZIqz6zJ8i33rLI,27680
+sage/benchmark/benchmark_agent/experiments/planning_exp.py,sha256=-qrC6urBQTaS5YCOQjn2uWzt6KBwSRZwAmPJQ1zzwA8,9751
+sage/benchmark/benchmark_agent/experiments/timing_detection_exp.py,sha256=ifkAdd7Nym21HrrbxriY3QNtRnprtft73WrlGiMdAwE,6992
+sage/benchmark/benchmark_agent/experiments/tool_selection_exp.py,sha256=9-BNYZMZ_FIujuAo1cAAH6j8llwxeeThBffG9fCMtRc,9320
+sage/benchmark/benchmark_agent/scripts/__init__.py,sha256=C8YqglL5eDKIyB8fKg7mC5NZJZ_Fn7LlmEFl2e8RVRw,668
+sage/benchmark/benchmark_agent/scripts/experiments/__init__.py,sha256=d3VxS2Qfuz7WujRnREz25IUUOVOOatokoD5rhAExy14,1125
+sage/benchmark/benchmark_agent/scripts/experiments/exp_analysis_ablation.py,sha256=78aNKL0dhw5rDU9aiIQ80O0j7CAl0JPHYxWEFINFcOs,13238
+sage/benchmark/benchmark_agent/scripts/experiments/exp_analysis_error.py,sha256=kXp2e3hteVvil1nMhjBLxvJvtiV00XbgmmhqZzS5yfk,13422
+sage/benchmark/benchmark_agent/scripts/experiments/exp_analysis_robustness.py,sha256=vMmeXskW-kmWjWGr6sPGEw_Y45x9cLchKk5LMUtjyd0,14445
+sage/benchmark/benchmark_agent/scripts/experiments/exp_analysis_scaling.py,sha256=uXfzkT8HV1OQahyDJ_ns6hYJj7YxlvqjuGenmcpIIyM,17725
+sage/benchmark/benchmark_agent/scripts/experiments/exp_cross_dataset.py,sha256=v6Ku0hgKmeUo0CcYD2UK-kjDZi0K6TmS0FwUDB2ZcHw,12179
+sage/benchmark/benchmark_agent/scripts/experiments/exp_main_planning.py,sha256=uXPCFcnn1FC8DMGPvcylVNwT6tjDfEU7CJc7950pWSs,10845
+sage/benchmark/benchmark_agent/scripts/experiments/exp_main_selection.py,sha256=yM1Dkxh6iguHzBKG1saABCDkr5QqslNmfPM9CxctYa4,11423
+sage/benchmark/benchmark_agent/scripts/experiments/exp_main_timing.py,sha256=Q5ZvPf-MumLCIsV7Dtvb7wt1R9GA39rzcqMjhWQ9dLc,8674
+sage/benchmark/benchmark_agent/scripts/experiments/exp_training_comparison.py,sha256=BWgkS37KM0lUcIEs1iofQbfvrIIyzq7iQ9g2gx5vyFs,20490
+sage/benchmark/benchmark_agent/scripts/experiments/exp_utils.py,sha256=5F8F4UvS1eTdNJ7YpDk0crT2pYzqAehX4eqEshZHxrI,12875
+sage/benchmark/benchmark_agent/scripts/experiments/figure_generator.py,sha256=hIcGCGQ28TVDb90SVLbO1MqBaZyi9RsJfpnm0kFZMV0,20136
+sage/benchmark/benchmark_agent/scripts/experiments/llm_service.py,sha256=na_bR0Gvj0088GEHQzfg0xakDRv5cWX-UlLE3odbc1k,8893
+sage/benchmark/benchmark_agent/scripts/experiments/run_paper1_experiments.py,sha256=cZ1z8fYULZ2ezIg56ITBOXAGWHmwpL_qiY0CgAYs53E,20854
+sage/benchmark/benchmark_agent/scripts/experiments/sage_bench_cli.py,sha256=m_MqNJIfC1motfodbjd1WPsv37wCTIafP5lwJUKsHrg,14950
+sage/benchmark/benchmark_agent/scripts/experiments/table_generator.py,sha256=Mxqo3lmcQsEpTuaiwklQVEo3xn6G-KRVsCizbsMeJM0,12885
+isage_benchmark_agent-0.1.0.1.dist-info/METADATA,sha256=GQNMGxPXYmBtSRcJ9TOu5XznKYVbQLPHTZzWmceCWPg,3016
+isage_benchmark_agent-0.1.0.1.dist-info/WHEEL,sha256=yk-B4c9kYsinhQ_MzhPAVcDm9mhkAVmdo0rg0jgFCmo,94
+isage_benchmark_agent-0.1.0.1.dist-info/entry_points.txt,sha256=g34HO224bwnCvIklVp2JQw4wTNDgNW7u61F-cL02pSA,82
+isage_benchmark_agent-0.1.0.1.dist-info/top_level.txt,sha256=hibFyzQHiLOMK68qL1OWsNKaXOmSXqZjeLTBem6Yy7I,5
+isage_benchmark_agent-0.1.0.1.dist-info/RECORD,,

isage_benchmark_agent-0.1.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: cp311-none-any
+

isage_benchmark_agent-0.1.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sage-agent-bench = sage.benchmark.benchmark_agent.__main__:main

isage_benchmark_agent-0.1.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 IntelliStream Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

isage_benchmark_agent-0.1.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+sage

sage/benchmark/benchmark_agent/__init__.py

@@ -0,0 +1,108 @@
+"""
+Agent Capability Benchmark Module
+
+This module provides infrastructure for evaluating agent capabilities including:
+- Tool selection
+- Task planning
+- Timing detection
+
+Architecture:
+    config/         Configuration files and loaders
+    experiments/    Experiment runners and base classes
+    adapter_registry.py  Strategy adapter registry
+
+Usage:
+    # Via CLI
+    python -m sage.benchmark.benchmark_agent --config config/tool_selection_exp.yaml
+
+    # Programmatic
+    from sage.benchmark.benchmark_agent import ToolSelectionExperiment
+    from sage.benchmark.benchmark_agent.config import ConfigLoader
+    from sage.benchmark.benchmark_agent.adapter_registry import get_adapter_registry
+    from sage.data import DataManager
+
+    loader = ConfigLoader()
+    config = loader.load_config("config/tool_selection_exp.yaml")
+
+    dm = DataManager.get_instance()
+    registry = get_adapter_registry()
+
+    exp = ToolSelectionExperiment(config, data_manager=dm, adapter_registry=registry)
+    exp.prepare()
+    result = exp.run()
+    exp.finalize()
+"""
+
+from sage.benchmark.benchmark_agent.adapter_registry import (
+    AdapterRegistry,
+    PlannerAdapter,
+    SelectorAdapter,
+    TimingAdapter,
+    get_adapter_registry,
+    register_strategy,
+)
+
+# Data paths management
+from sage.benchmark.benchmark_agent.data_paths import (
+    DataPathsConfig,
+    RuntimePaths,
+    SourcePaths,
+    ensure_runtime_dirs,
+    get_data_paths_config,
+    get_runtime_paths,
+    get_source_paths,
+)
+from sage.benchmark.benchmark_agent.experiments import (  # Base classes; Configs; Experiments
+    BaseExperiment,
+    ExperimentConfig,
+    ExperimentResult,
+    PlanningConfig,
+    PlanningExperiment,
+    TimingDetectionConfig,
+    TimingDetectionExperiment,
+    ToolSelectionConfig,
+    ToolSelectionExperiment,
+)
+from sage.benchmark.benchmark_agent.experiments.method_comparison import (
+    ExperimentResult as ComparisonResult,
+)
+from sage.benchmark.benchmark_agent.experiments.method_comparison import (
+    MethodComparisonExperiment,
+    MethodConfig,
+    MethodRegistry,
+    run_full_comparison,
+    run_quick_comparison,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    # Experiments
+    "ToolSelectionExperiment",
+    "PlanningExperiment",
+    "TimingDetectionExperiment",
+    # Base
+    "BaseExperiment",
+    "ExperimentConfig",
+    "ExperimentResult",
+    # Configs
+    "ToolSelectionConfig",
+    "PlanningConfig",
+    "TimingDetectionConfig",
+    # Adapter Registry
+    "AdapterRegistry",
+    "SelectorAdapter",
+    "PlannerAdapter",
+    "TimingAdapter",
+    "get_adapter_registry",
+    "register_strategy",
+    # Data Paths
+    "get_source_paths",
+    "get_runtime_paths",
+    "get_data_paths_config",
+    "ensure_runtime_dirs",
+    "SourcePaths",
+    "RuntimePaths",
+    "DataPathsConfig",
+]

sage/benchmark/benchmark_agent/__main__.py

@@ -0,0 +1,177 @@
+"""
+CLI entry point for running agent benchmark experiments.
+
+Usage:
+    python -m sage.benchmark.benchmark_agent --config config/tool_selection_exp.yaml
+    python -m sage.benchmark.benchmark_agent --config config/planning_exp.yaml
+    python -m sage.benchmark.benchmark_agent --config config/timing_detection_exp.yaml
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+from sage.benchmark.benchmark_agent.adapter_registry import get_adapter_registry
+from sage.benchmark.benchmark_agent.config.config_loader import ConfigLoader
+from sage.benchmark.benchmark_agent.experiments import (
+    PlanningExperiment,
+    TimingDetectionExperiment,
+    ToolSelectionExperiment,
+)
+from sage.data import DataManager
+
+
+def create_experiment(config, data_manager, adapter_registry):
+    """
+    Create appropriate experiment instance based on config type.
+
+    Args:
+        config: Experiment configuration
+        data_manager: DataManager instance
+        adapter_registry: AdapterRegistry instance
+
+    Returns:
+        Experiment instance
+
+    Raises:
+        ValueError: If experiment type not recognized
+    """
+    experiment_type = config.experiment
+
+    if experiment_type == "tool_selection":
+        return ToolSelectionExperiment(
+            config, data_manager=data_manager, adapter_registry=adapter_registry
+        )
+    elif experiment_type == "planning":
+        return PlanningExperiment(
+            config, data_manager=data_manager, adapter_registry=adapter_registry
+        )
+    elif experiment_type == "timing_detection":
+        return TimingDetectionExperiment(
+            config, data_manager=data_manager, adapter_registry=adapter_registry
+        )
+    else:
+        raise ValueError(f"Unknown experiment type: {experiment_type}")
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Run agent benchmark experiments",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run tool selection experiment
+  python -m sage.benchmark.benchmark_agent --config config/tool_selection_exp.yaml
+
+  # Run planning experiment with custom output
+  python -m sage.benchmark.benchmark_agent \\
+      --config config/planning_exp.yaml \\
+      --output results/planning_results.json
+
+  # Run timing detection with verbose output
+  python -m sage.benchmark.benchmark_agent \\
+      --config config/timing_detection_exp.yaml \\
+      --verbose
+        """,
+    )
+
+    parser.add_argument(
+        "--config", type=str, required=True, help="Path to experiment configuration YAML file"
+    )
+
+    parser.add_argument("--output", type=str, help="Override output path for results (optional)")
+
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose output")
+
+    parser.add_argument(
+        "--dry-run", action="store_true", help="Validate config without running experiment"
+    )
+
+    args = parser.parse_args()
+
+    # Load configuration
+    try:
+        config_path = Path(args.config)
+        if not config_path.exists():
+            print(f"Error: Config file not found: {config_path}")
+            sys.exit(1)
+
+        loader = ConfigLoader()
+        config = loader.load_config(str(config_path))
+
+        # Override from CLI args
+        if args.output:
+            config.report.path = args.output
+        if args.verbose:
+            config.verbose = getattr(config, "verbose", True)
+
+        print(f"Loaded config: {config.experiment}")
+
+        if args.dry_run:
+            print("Config validation successful (dry-run mode)")
+            print(f"  Experiment type: {config.experiment}")
+            print(f"  Profile: {config.profile}")
+            print(f"  Split: {config.split}")
+            print(f"  Output: {config.report.path}")
+            return 0
+
+    except Exception as e:
+        print(f"Error loading config: {e}")
+        sys.exit(1)
+
+    # Initialize DataManager and AdapterRegistry
+    try:
+        print("Initializing DataManager...")
+        data_manager = DataManager.get_instance()
+
+        print("Initializing AdapterRegistry...")
+        adapter_registry = get_adapter_registry()
+
+        print(f"  Available strategies: {adapter_registry.list_strategies()}")
+
+    except Exception as e:
+        print(f"Error initializing managers: {e}")
+        sys.exit(1)
+
+    # Create experiment
+    try:
+        experiment = create_experiment(config, data_manager, adapter_registry)
+        print(f"Created experiment: {experiment.__class__.__name__}")
+    except Exception as e:
+        print(f"Error creating experiment: {e}")
+        sys.exit(1)
+
+    # Run experiment
+    try:
+        print("\n" + "=" * 60)
+        print("Starting experiment...")
+        print("=" * 60 + "\n")
+
+        experiment.prepare()
+        result = experiment.run()
+        experiment.finalize()
+
+        print("\n" + "=" * 60)
+        print("Experiment completed successfully!")
+        print("=" * 60)
+        print(f"Results saved to: {config.report.path}")
+        print(f"Total samples: {result.metadata.get('total_samples', 0)}")
+        print(f"Failed samples: {result.metadata.get('failed_samples', 0)}")
+
+        return 0
+
+    except KeyboardInterrupt:
+        print("\n\nExperiment interrupted by user")
+        return 130
+
+    except Exception as e:
+        print(f"\nError running experiment: {e}")
+        import traceback
+
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    sys.exit(main())