isage-tooluse-benchmark 0.1.0.1__tar.gz

isage_tooluse_benchmark-0.1.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 IntelliStream Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

isage_tooluse_benchmark-0.1.0.1/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: isage-tooluse-benchmark
+Version: 0.1.0.1
+Summary: SAGE Tool Use Benchmark - Tool selection and use evaluation framework
+Author-email: IntelliStream Team <shuhao_zhang@hust.edu.cn>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/intellistream/sage-agent-benchmark
+Project-URL: Documentation, https://github.com/intellistream/sage-agent-benchmark#readme
+Project-URL: Repository, https://github.com/intellistream/sage-agent-benchmark
+Project-URL: Issues, https://github.com/intellistream/sage-agent-benchmark/issues
+Keywords: sage,benchmark,tool-selection,tool-use,planning,timing-detection,evaluation,intellistream
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: isage-common
+Requires-Dist: isage-libs
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: numpy<2.3.0,>=1.26.0
+Requires-Dist: typer<1.0.0,>=0.15.0
+Requires-Dist: rich<14.0.0,>=13.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff==0.14.6; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: types-PyYAML>=6.0.0; extra == "dev"
+Provides-Extra: all
+Requires-Dist: isage-benchmark-agent[dev]; extra == "all"
+Dynamic: license-file
+
+# SAGE Tool Use Benchmark
+
+Configuration-driven experiment framework for evaluating tool selection and use capabilities.
+
+## Features
+
+- **Tool Selection Evaluation**: Tool retrieval and ranking benchmarks
+- **Planning Evaluation**: Multi-step planning with tool composition
+- **Timing Detection**: Timing judgment for tool invocation decisions
+
+## Quick Start
+
+```bash
+# Install
+pip install isage-tooluse-benchmark
+
+# Run tool selection experiment
+sage-tooluse-bench tool-selection --config config/tool_selection_exp.yaml
+
+# Run planning experiment
+sage-tooluse-bench planning --config config/planning_exp.yaml
+```
+
+## Documentation
+
+See [benchmark_agent/README.md](src/sage/benchmark/benchmark_agent/README.md) for detailed documentation.
+
+## Development
+
+```bash
+# Clone
+git clone https://github.com/intellistream/sage-tooluse-benchmark.git
+cd sage-tooluse-benchmark
+
+# Setup virtual environment
+python -m venv .venv
+source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+
+# Install in development mode
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+```
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.

isage_tooluse_benchmark-0.1.0.1/README.md

@@ -0,0 +1,48 @@
+# SAGE Tool Use Benchmark
+
+Configuration-driven experiment framework for evaluating tool selection and use capabilities.
+
+## Features
+
+- **Tool Selection Evaluation**: Tool retrieval and ranking benchmarks
+- **Planning Evaluation**: Multi-step planning with tool composition
+- **Timing Detection**: Timing judgment for tool invocation decisions
+
+## Quick Start
+
+```bash
+# Install
+pip install isage-tooluse-benchmark
+
+# Run tool selection experiment
+sage-tooluse-bench tool-selection --config config/tool_selection_exp.yaml
+
+# Run planning experiment
+sage-tooluse-bench planning --config config/planning_exp.yaml
+```
+
+## Documentation
+
+See [benchmark_agent/README.md](src/sage/benchmark/benchmark_agent/README.md) for detailed documentation.
+
+## Development
+
+```bash
+# Clone
+git clone https://github.com/intellistream/sage-tooluse-benchmark.git
+cd sage-tooluse-benchmark
+
+# Setup virtual environment
+python -m venv .venv
+source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+
+# Install in development mode
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+```
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.

isage_tooluse_benchmark-0.1.0.1/pyproject.toml

@@ -0,0 +1,139 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "isage-tooluse-benchmark"
+version = "0.1.0.1"
+description = "SAGE Tool Use Benchmark - Tool selection and use evaluation framework"
+readme = "README.md"
+requires-python = ">=3.11"
+authors = [
+    { name = "IntelliStream Team", email = "shuhao_zhang@hust.edu.cn" },
+]
+keywords = [
+    "sage",
+    "benchmark",
+    "tool-selection",
+    "tool-use",
+    "planning",
+    "timing-detection",
+    "evaluation",
+    "intellistream",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+dependencies = [
+    # SAGE dependencies from PyPI
+    "isage-common",
+    "isage-libs",  # For agent interfaces if needed
+    
+    # Core dependencies
+    "pyyaml>=6.0",
+    "pandas>=2.0.0",
+    "numpy>=1.26.0,<2.3.0",
+    
+    # CLI dependencies
+    "typer>=0.15.0,<1.0.0",
+    "rich>=13.0.0,<14.0.0",
+]
+
+license = "MIT"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-asyncio>=0.21.0",
+    "pytest-cov>=4.0.0",
+    "black>=23.0.0",
+    "ruff==0.14.6",
+    "pre-commit>=3.0.0",
+    "mypy>=1.0.0",
+    "types-PyYAML>=6.0.0",
+]
+
+all = [
+    "isage-benchmark-agent[dev]",
+]
+
+[project.urls]
+Homepage = "https://github.com/intellistream/sage-agent-benchmark"
+Documentation = "https://github.com/intellistream/sage-agent-benchmark#readme"
+Repository = "https://github.com/intellistream/sage-agent-benchmark"
+Issues = "https://github.com/intellistream/sage-agent-benchmark/issues"
+
+[project.scripts]
+sage-agent-bench = "sage.benchmark.benchmark_agent.__main__:main"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["sage*"]
+namespaces = true
+
+[tool.setuptools.package-data]
+"sage.benchmark.benchmark_agent" = [
+    "config/*.yaml",
+    "data/**/*",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W"]
+ignore = [
+    "E501",   # Line too long
+    "F401",   # Unused imports
+    "F841",   # Unused local variable
+]
+
+[tool.mypy]
+ignore_missing_imports = true
+python_version = "3.11"
+warn_return_any = true
+warn_unused_configs = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+addopts = [
+    "--strict-markers",
+    "--verbose",
+    "-ra",
+]
+markers = [
+    "slow: marks tests as slow",
+    "integration: marks tests as integration tests",
+    "unit: marks tests as unit tests",
+]
+
+[tool.coverage.run]
+source = ["src/sage"]
+omit = ["*/tests/*", "*/test_*.py"]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+]
+
+[tool.black]
+line-length = 100
+target-version = ['py311']

isage_tooluse_benchmark-0.1.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

isage_tooluse_benchmark-0.1.0.1/src/isage_tooluse_benchmark.egg-info/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: isage-tooluse-benchmark
+Version: 0.1.0.1
+Summary: SAGE Tool Use Benchmark - Tool selection and use evaluation framework
+Author-email: IntelliStream Team <shuhao_zhang@hust.edu.cn>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/intellistream/sage-agent-benchmark
+Project-URL: Documentation, https://github.com/intellistream/sage-agent-benchmark#readme
+Project-URL: Repository, https://github.com/intellistream/sage-agent-benchmark
+Project-URL: Issues, https://github.com/intellistream/sage-agent-benchmark/issues
+Keywords: sage,benchmark,tool-selection,tool-use,planning,timing-detection,evaluation,intellistream
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: isage-common
+Requires-Dist: isage-libs
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: numpy<2.3.0,>=1.26.0
+Requires-Dist: typer<1.0.0,>=0.15.0
+Requires-Dist: rich<14.0.0,>=13.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff==0.14.6; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: types-PyYAML>=6.0.0; extra == "dev"
+Provides-Extra: all
+Requires-Dist: isage-benchmark-agent[dev]; extra == "all"
+Dynamic: license-file
+
+# SAGE Tool Use Benchmark
+
+Configuration-driven experiment framework for evaluating tool selection and use capabilities.
+
+## Features
+
+- **Tool Selection Evaluation**: Tool retrieval and ranking benchmarks
+- **Planning Evaluation**: Multi-step planning with tool composition
+- **Timing Detection**: Timing judgment for tool invocation decisions
+
+## Quick Start
+
+```bash
+# Install
+pip install isage-tooluse-benchmark
+
+# Run tool selection experiment
+sage-tooluse-bench tool-selection --config config/tool_selection_exp.yaml
+
+# Run planning experiment
+sage-tooluse-bench planning --config config/planning_exp.yaml
+```
+
+## Documentation
+
+See [benchmark_agent/README.md](src/sage/benchmark/benchmark_agent/README.md) for detailed documentation.
+
+## Development
+
+```bash
+# Clone
+git clone https://github.com/intellistream/sage-tooluse-benchmark.git
+cd sage-tooluse-benchmark
+
+# Setup virtual environment
+python -m venv .venv
+source .venv/bin/activate  # On Windows: .venv\Scripts\activate
+
+# Install in development mode
+pip install -e ".[dev]"
+
+# Run tests
+pytest
+```
+
+## License
+
+MIT License - see [LICENSE](LICENSE) for details.

isage_tooluse_benchmark-0.1.0.1/src/isage_tooluse_benchmark.egg-info/SOURCES.txt

@@ -0,0 +1,59 @@
+LICENSE
+README.md
+pyproject.toml
+src/isage_tooluse_benchmark.egg-info/PKG-INFO
+src/isage_tooluse_benchmark.egg-info/SOURCES.txt
+src/isage_tooluse_benchmark.egg-info/dependency_links.txt
+src/isage_tooluse_benchmark.egg-info/entry_points.txt
+src/isage_tooluse_benchmark.egg-info/requires.txt
+src/isage_tooluse_benchmark.egg-info/top_level.txt
+src/sage/__init__.py
+src/sage/benchmark/__init__.py
+src/sage/benchmark/benchmark_agent/__init__.py
+src/sage/benchmark/benchmark_agent/__main__.py
+src/sage/benchmark/benchmark_agent/acebench_loader.py
+src/sage/benchmark/benchmark_agent/adapter_registry.py
+src/sage/benchmark/benchmark_agent/data_paths.py
+src/sage/benchmark/benchmark_agent/tools_loader.py
+src/sage/benchmark/benchmark_agent/config/config_loader.py
+src/sage/benchmark/benchmark_agent/config/default_config.yaml
+src/sage/benchmark/benchmark_agent/config/planning_exp.yaml
+src/sage/benchmark/benchmark_agent/config/timing_detection_exp.yaml
+src/sage/benchmark/benchmark_agent/config/tool_selection_exp.yaml
+src/sage/benchmark/benchmark_agent/evaluation/__init__.py
+src/sage/benchmark/benchmark_agent/evaluation/evaluator.py
+src/sage/benchmark/benchmark_agent/evaluation/metrics.py
+src/sage/benchmark/benchmark_agent/evaluation/report_builder.py
+src/sage/benchmark/benchmark_agent/evaluation/unified_tool_selection.py
+src/sage/benchmark/benchmark_agent/evaluation/analyzers/__init__.py
+src/sage/benchmark/benchmark_agent/evaluation/analyzers/planning_analyzer.py
+src/sage/benchmark/benchmark_agent/evaluation/analyzers/timing_analyzer.py
+src/sage/benchmark/benchmark_agent/evaluation/analyzers/tool_selection_analyzer.py
+src/sage/benchmark/benchmark_agent/experiments/__init__.py
+src/sage/benchmark/benchmark_agent/experiments/base_experiment.py
+src/sage/benchmark/benchmark_agent/experiments/method_comparison.py
+src/sage/benchmark/benchmark_agent/experiments/planning_exp.py
+src/sage/benchmark/benchmark_agent/experiments/timing_detection_exp.py
+src/sage/benchmark/benchmark_agent/experiments/tool_selection_exp.py
+src/sage/benchmark/benchmark_agent/scripts/__init__.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/__init__.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/exp_analysis_ablation.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/exp_analysis_error.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/exp_analysis_robustness.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/exp_analysis_scaling.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/exp_cross_dataset.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/exp_main_planning.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/exp_main_selection.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/exp_main_timing.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/exp_training_comparison.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/exp_utils.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/figure_generator.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/llm_service.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/run_paper1_experiments.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/sage_bench_cli.py
+src/sage/benchmark/benchmark_agent/scripts/experiments/table_generator.py
+tests/test_evaluation.py
+tests/test_experiments.py
+tests/test_react_planner.py
+tests/test_rule_based_decider.py
+tests/test_toolalpaca_loader.py

isage_tooluse_benchmark-0.1.0.1/src/isage_tooluse_benchmark.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

isage_tooluse_benchmark-0.1.0.1/src/isage_tooluse_benchmark.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sage-agent-bench = sage.benchmark.benchmark_agent.__main__:main

isage_tooluse_benchmark-0.1.0.1/src/isage_tooluse_benchmark.egg-info/requires.txt

@@ -0,0 +1,20 @@
+isage-common
+isage-libs
+pyyaml>=6.0
+pandas>=2.0.0
+numpy<2.3.0,>=1.26.0
+typer<1.0.0,>=0.15.0
+rich<14.0.0,>=13.0.0
+
+[all]
+isage-benchmark-agent[dev]
+
+[dev]
+pytest>=7.0.0
+pytest-asyncio>=0.21.0
+pytest-cov>=4.0.0
+black>=23.0.0
+ruff==0.14.6
+pre-commit>=3.0.0
+mypy>=1.0.0
+types-PyYAML>=6.0.0

isage_tooluse_benchmark-0.1.0.1/src/isage_tooluse_benchmark.egg-info/top_level.txt

@@ -0,0 +1 @@
+sage

isage_tooluse_benchmark-0.1.0.1/src/sage/benchmark/benchmark_agent/__init__.py

@@ -0,0 +1,108 @@
+"""
+Agent Capability Benchmark Module
+
+This module provides infrastructure for evaluating agent capabilities including:
+- Tool selection
+- Task planning
+- Timing detection
+
+Architecture:
+    config/         Configuration files and loaders
+    experiments/    Experiment runners and base classes
+    adapter_registry.py  Strategy adapter registry
+
+Usage:
+    # Via CLI
+    python -m sage.benchmark.benchmark_agent --config config/tool_selection_exp.yaml
+
+    # Programmatic
+    from sage.benchmark.benchmark_agent import ToolSelectionExperiment
+    from sage.benchmark.benchmark_agent.config import ConfigLoader
+    from sage.benchmark.benchmark_agent.adapter_registry import get_adapter_registry
+    from sage.data import DataManager
+
+    loader = ConfigLoader()
+    config = loader.load_config("config/tool_selection_exp.yaml")
+
+    dm = DataManager.get_instance()
+    registry = get_adapter_registry()
+
+    exp = ToolSelectionExperiment(config, data_manager=dm, adapter_registry=registry)
+    exp.prepare()
+    result = exp.run()
+    exp.finalize()
+"""
+
+from sage.benchmark.benchmark_agent.adapter_registry import (
+    AdapterRegistry,
+    PlannerAdapter,
+    SelectorAdapter,
+    TimingAdapter,
+    get_adapter_registry,
+    register_strategy,
+)
+
+# Data paths management
+from sage.benchmark.benchmark_agent.data_paths import (
+    DataPathsConfig,
+    RuntimePaths,
+    SourcePaths,
+    ensure_runtime_dirs,
+    get_data_paths_config,
+    get_runtime_paths,
+    get_source_paths,
+)
+from sage.benchmark.benchmark_agent.experiments import (  # Base classes; Configs; Experiments
+    BaseExperiment,
+    ExperimentConfig,
+    ExperimentResult,
+    PlanningConfig,
+    PlanningExperiment,
+    TimingDetectionConfig,
+    TimingDetectionExperiment,
+    ToolSelectionConfig,
+    ToolSelectionExperiment,
+)
+from sage.benchmark.benchmark_agent.experiments.method_comparison import (
+    ExperimentResult as ComparisonResult,
+)
+from sage.benchmark.benchmark_agent.experiments.method_comparison import (
+    MethodComparisonExperiment,
+    MethodConfig,
+    MethodRegistry,
+    run_full_comparison,
+    run_quick_comparison,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    # Experiments
+    "ToolSelectionExperiment",
+    "PlanningExperiment",
+    "TimingDetectionExperiment",
+    # Base
+    "BaseExperiment",
+    "ExperimentConfig",
+    "ExperimentResult",
+    # Configs
+    "ToolSelectionConfig",
+    "PlanningConfig",
+    "TimingDetectionConfig",
+    # Adapter Registry
+    "AdapterRegistry",
+    "SelectorAdapter",
+    "PlannerAdapter",
+    "TimingAdapter",
+    "get_adapter_registry",
+    "register_strategy",
+    # Data Paths
+    "get_source_paths",
+    "get_runtime_paths",
+    "get_data_paths_config",
+    "ensure_runtime_dirs",
+    "SourcePaths",
+    "RuntimePaths",
+    "DataPathsConfig",
+]