eval-protocol 0.2.5.dev1__tar.gz → 0.2.6.dev1__tar.gz

{eval_protocol-0.2.5.dev1/eval_protocol.egg-info → eval_protocol-0.2.6.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.5.dev1
+Version: 0.2.6.dev1
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -41,10 +41,10 @@ Requires-Dist: pandas>=1.5.0
 Requires-Dist: watchdog>=2.1.0
 Requires-Dist: websockets>=15.0.1
 Requires-Dist: fastapi>=0.116.1
+Requires-Dist: pytest>=6.0.0
 Provides-Extra: dev
 Requires-Dist: build; extra == "dev"
 Requires-Dist: twine; extra == "dev"
-Requires-Dist: pytest>=6.0.0; extra == "dev"
 Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: pytest-httpserver; extra == "dev"
 Requires-Dist: werkzeug>=2.0.0; extra == "dev"
@@ -109,7 +109,7 @@ markdown generation tasks to customer service agents with tool calling
 capabilities.
 
 <p align="center">
-	<img src="./assets/ui.png" alt="UI" />
+	<img src="https://raw.githubusercontent.com/eval-protocol/python-sdk/refs/heads/main/assets/ui.png" alt="UI" />
 	<br>
 	<sub><b>Log Viewer: Monitor your evaluation rollouts in real time.</b></sub>
 </p>

{eval_protocol-0.2.5.dev1 → eval_protocol-0.2.6.dev1}/README.md

@@ -13,7 +13,7 @@ markdown generation tasks to customer service agents with tool calling
 capabilities.
 
 <p align="center">
-	<img src="./assets/ui.png" alt="UI" />
+	<img src="https://raw.githubusercontent.com/eval-protocol/python-sdk/refs/heads/main/assets/ui.png" alt="UI" />
 	<br>
 	<sub><b>Log Viewer: Monitor your evaluation rollouts in real time.</b></sub>
 </p>

{eval_protocol-0.2.5.dev1 → eval_protocol-0.2.6.dev1}/eval_protocol/__init__.py

@@ -10,15 +10,16 @@ tool-augmented models using self-contained task bundles.
 
 import warnings
 
-from .adapters.braintrust import reward_fn_to_scorer, scorer_to_reward_fn
+from eval_protocol.adapters.braintrust import reward_fn_to_scorer, scorer_to_reward_fn
+
 from .auth import get_fireworks_account_id, get_fireworks_api_key
 from .common_utils import load_jsonl
 from .config import RewardKitConfig, get_config, load_config
 from .mcp_env import (
     AnthropicPolicy,
-    OpenAIPolicy,
-    LiteLLMPolicy,
     FireworksPolicy,
+    LiteLLMPolicy,
+    OpenAIPolicy,
     make,
     rollout,
     test_mcp,

{eval_protocol-0.2.5.dev1 → eval_protocol-0.2.6.dev1}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-08-06T17:51:29-0700",
+ "date": "2025-08-08T10:51:54-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "a807140937b9002c71ee42a6afef594ea6377c2d",
- "version": "0.2.5-dev1"
+ "full-revisionid": "986452fd04442f9a0d1ba902753a83480e413d43",
+ "version": "0.2.6-dev1"
 }
 '''  # END VERSION_JSON
 

eval_protocol-0.2.6.dev1/eval_protocol/common_utils.py

@@ -0,0 +1,30 @@
+import json
+import re
+from typing import Any, Dict, List
+
+
+def load_jsonl(file_path: str) -> List[Dict[str, Any]]:
+    """
+    Reads a JSONL file where each line is a valid JSON object and returns a list of these objects.
+
+    Args:
+        file_path: Path to the JSONL file.
+
+    Returns:
+        A list of dictionaries, where each dictionary is a parsed JSON object from a line.
+        Returns an empty list if the file is not found or if errors occur during parsing.
+    """
+    data: List[Dict[str, Any]] = []
+    with open(file_path, "r", encoding="utf-8") as f:
+        for line_number, line in enumerate(f):
+            try:
+                data.append(json.loads(line.strip()))
+            except json.JSONDecodeError as e:
+                print(f"Error parsing JSON line for file {file_path} at line {line_number}")
+                # attempt to find "row_id" in the line by finding index of "row_id" and performing regex of `"row_id": (.*),`
+                row_id_index = line.find("row_id")
+                if row_id_index != -1:
+                    row_id = re.search(r'"row_id": (.*),', line[row_id_index:])
+                    raise ValueError(f"{e.msg} at line {line_number}: {line} ({row_id})")
+                raise e
+    return data

eval_protocol-0.2.6.dev1/eval_protocol/dataset_logger/directory_utils.py

@@ -0,0 +1,55 @@
+import os
+from typing import Optional
+
+# Shared constants for directory discovery
+EVAL_PROTOCOL_DIR = ".eval_protocol"
+PYTHON_FILES = ["pyproject.toml", "requirements.txt"]
+DATASETS_DIR = "datasets"
+
+
+def find_eval_protocol_dir() -> str:
+    """
+    Find the .eval_protocol directory by looking up the directory tree.
+
+    Returns:
+        Path to the .eval_protocol directory
+    """
+    # recursively look up for a .eval_protocol directory
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    while current_dir != "/":
+        if os.path.exists(os.path.join(current_dir, EVAL_PROTOCOL_DIR)):
+            log_dir = os.path.join(current_dir, EVAL_PROTOCOL_DIR)
+            break
+        current_dir = os.path.dirname(current_dir)
+    else:
+        # if not found, recursively look up until a pyproject.toml or requirements.txt is found
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        while current_dir != "/":
+            if any(os.path.exists(os.path.join(current_dir, f)) for f in PYTHON_FILES):
+                log_dir = os.path.join(current_dir, EVAL_PROTOCOL_DIR)
+                break
+            current_dir = os.path.dirname(current_dir)
+        else:
+            # get the PWD that this python process is running in
+            log_dir = os.path.join(os.getcwd(), EVAL_PROTOCOL_DIR)
+
+    # create the .eval_protocol directory if it doesn't exist
+    os.makedirs(log_dir, exist_ok=True)
+
+    return log_dir
+
+
+def find_eval_protocol_datasets_dir() -> str:
+    """
+    Find the .eval_protocol/datasets directory by looking up the directory tree.
+
+    Returns:
+        Path to the .eval_protocol/datasets directory
+    """
+    log_dir = find_eval_protocol_dir()
+
+    # create the datasets subdirectory
+    datasets_dir = os.path.join(log_dir, DATASETS_DIR)
+    os.makedirs(datasets_dir, exist_ok=True)
+
+    return datasets_dir

eval_protocol-0.2.6.dev1/eval_protocol/dataset_logger/local_fs_dataset_logger_adapter.py

@@ -0,0 +1,98 @@
+import json
+import os
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TYPE_CHECKING, List, Optional
+
+from eval_protocol.common_utils import load_jsonl
+from eval_protocol.dataset_logger.dataset_logger import DatasetLogger
+from eval_protocol.dataset_logger.directory_utils import find_eval_protocol_datasets_dir
+
+if TYPE_CHECKING:
+    from eval_protocol.models import EvaluationRow
+
+
+class LocalFSDatasetLoggerAdapter(DatasetLogger):
+    """
+    Logger that stores logs in the local filesystem with file locking to prevent race conditions.
+    """
+
+    def __init__(self):
+        self.log_dir = os.path.dirname(find_eval_protocol_datasets_dir())
+        self.datasets_dir = find_eval_protocol_datasets_dir()
+
+        # ensure that log file exists
+        if not os.path.exists(self.current_jsonl_path):
+            with open(self.current_jsonl_path, "w") as f:
+                f.write("")
+
+    @property
+    def current_date(self) -> str:
+        # Use UTC timezone to be consistent across local device/locations/CI
+        return datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+    @property
+    def current_jsonl_path(self) -> str:
+        """
+        The current JSONL file path. Based on the current date.
+        """
+        return os.path.join(self.datasets_dir, f"{self.current_date}.jsonl")
+
+    def log(self, row: "EvaluationRow") -> None:
+        """Log a row, updating existing row with same ID or appending new row."""
+        row_id = row.input_metadata.row_id
+
+        # Check if row with this ID already exists in any JSONL file
+        if os.path.exists(self.datasets_dir):
+            for filename in os.listdir(self.datasets_dir):
+                if filename.endswith(".jsonl"):
+                    file_path = os.path.join(self.datasets_dir, filename)
+                    if os.path.exists(file_path):
+                        with open(file_path, "r") as f:
+                            lines = f.readlines()
+
+                        # Find the line with matching ID
+                        for i, line in enumerate(lines):
+                            try:
+                                line_data = json.loads(line.strip())
+                                if line_data["input_metadata"]["row_id"] == row_id:
+                                    # Update existing row
+                                    lines[i] = row.model_dump_json(exclude_none=True) + os.linesep
+                                    with open(file_path, "w") as f:
+                                        f.writelines(lines)
+                                    return
+                            except json.JSONDecodeError:
+                                continue
+
+        # If no existing row found, append new row to current file
+        with open(self.current_jsonl_path, "a") as f:
+            f.write(row.model_dump_json(exclude_none=True) + os.linesep)
+
+    def read(self, row_id: Optional[str] = None) -> List["EvaluationRow"]:
+        """Read rows from all JSONL files in the datasets directory. Also
+        ensures that there are no duplicate row IDs."""
+        from eval_protocol.models import EvaluationRow
+
+        if not os.path.exists(self.datasets_dir):
+            return []
+
+        all_rows = []
+        existing_row_ids = set()
+        for filename in os.listdir(self.datasets_dir):
+            if filename.endswith(".jsonl"):
+                file_path = os.path.join(self.datasets_dir, filename)
+                data = load_jsonl(file_path)
+                for r in data:
+                    row = EvaluationRow(**r)
+                    if row.input_metadata.row_id not in existing_row_ids:
+                        existing_row_ids.add(row.input_metadata.row_id)
+                    else:
+                        raise ValueError(f"Duplicate Row ID {row.input_metadata.row_id} already exists")
+                    all_rows.append(row)
+
+        if row_id:
+            # Filter by row_id if specified
+            return [row for row in all_rows if getattr(row.input_metadata, "row_id", None) == row_id]
+        else:
+            return all_rows

eval_protocol-0.2.6.dev1/eval_protocol/get_pep440_version.py

@@ -0,0 +1,133 @@
+# Cache for PEP 440 version string
+import subprocess
+
+_version_cache = {"version": None, "base_version": None}
+
+
+def get_pep440_version(base_version=None):
+    """
+    Generate a PEP 440 compliant version string based on git information.
+
+    This function is inspired by versioneer but doesn't require the full versioneer
+    setup, making it easier for downstream users to adopt without additional dependencies.
+
+    The result is cached statically to avoid repeated git calls.
+
+    Args:
+        base_version: The base version string (e.g., "1.0.0"). If None, will try to
+                     find the most recent version tag in git.
+
+    Returns:
+        A PEP 440 compliant version string that includes:
+        - Development release number (devN) based on commit count since base_version
+        - Local version identifier with git commit hash
+        - Dirty indicator if there are uncommitted changes
+
+    Examples:
+        >>> get_pep440_version("1.0.0")
+        "1.0.0.dev42+g1234567"  # 42 commits since 1.0.0, commit hash 1234567
+        >>> get_pep440_version("1.0.0")  # with uncommitted changes
+        "1.0.0.dev42+g1234567.dirty"  # indicates dirty working directory
+        >>> get_pep440_version("1.0.0")  # no git available
+        "1.0.0+unknown"  # indicates git info not available
+    """
+    # Check if we have a cached version for this base_version
+    if _version_cache["version"] is not None and _version_cache["base_version"] == base_version:
+        return _version_cache["version"]
+    try:
+        # Check if we're in a git repository
+        subprocess.run(
+            ["git", "rev-parse", "--git-dir"],
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        )
+
+        # If base_version is None, try to find the most recent version tag
+        if base_version is None:
+            try:
+                base_version = subprocess.check_output(
+                    ["git", "describe", "--tags", "--abbrev=0"], universal_newlines=True, stderr=subprocess.DEVNULL
+                ).strip()
+            except subprocess.CalledProcessError:
+                # No tags found, we'll handle this case specially
+                base_version = None
+
+        # Get commit count since base_version
+        if base_version is None:
+            # No base version (no tags), just count all commits
+            count = subprocess.check_output(
+                ["git", "rev-list", "--count", "HEAD"], universal_newlines=True, stderr=subprocess.DEVNULL
+            ).strip()
+            base_version = "0.0.0"  # Use this for the final version string
+        else:
+            try:
+                count = subprocess.check_output(
+                    ["git", "rev-list", "--count", f"{base_version}..HEAD"],
+                    universal_newlines=True,
+                    stderr=subprocess.DEVNULL,
+                ).strip()
+                # If no commits found, try counting from the beginning
+                if count == "0" or not count:
+                    count = subprocess.check_output(
+                        ["git", "rev-list", "--count", "HEAD"], universal_newlines=True, stderr=subprocess.DEVNULL
+                    ).strip()
+            except subprocess.CalledProcessError:
+                # If base_version tag doesn't exist, count all commits
+                count = subprocess.check_output(
+                    ["git", "rev-list", "--count", "HEAD"], universal_newlines=True, stderr=subprocess.DEVNULL
+                ).strip()
+
+        # Get short commit hash
+        commit_hash = subprocess.check_output(
+            ["git", "rev-parse", "--short", "HEAD"], universal_newlines=True, stderr=subprocess.DEVNULL
+        ).strip()
+
+        # Check for uncommitted changes (dirty working directory)
+        try:
+            subprocess.run(
+                ["git", "diff-index", "--quiet", "HEAD", "--"],
+                check=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            dirty_suffix = ""
+        except subprocess.CalledProcessError:
+            dirty_suffix = ".dirty"
+
+        # Ensure count is a valid integer
+        try:
+            dev_count = int(count)
+        except (ValueError, TypeError):
+            dev_count = 0
+
+        # Build PEP 440 compliant version string
+        # Format: <base_version>.dev<count>+g<hash>[.dirty]
+        version_parts = [base_version]
+
+        if dev_count > 0:
+            version_parts.append(f".dev{dev_count}")
+
+        version_parts.append(f"+g{commit_hash}")
+
+        if dirty_suffix:
+            version_parts.append(dirty_suffix)
+
+        result = "".join(version_parts)
+
+        # Cache the result
+        _version_cache["version"] = result
+        _version_cache["base_version"] = base_version
+
+        return result
+
+    except (subprocess.CalledProcessError, FileNotFoundError, OSError):
+        # Git is not available or not a git repository
+        result = f"{base_version}+unknown"
+
+        # Cache the result
+        _version_cache["version"] = result
+        _version_cache["base_version"] = base_version
+
+        return result

{eval_protocol-0.2.5.dev1 → eval_protocol-0.2.6.dev1}/eval_protocol/human_id/__init__.py

@@ -1,6 +1,7 @@
-import random
 import itertools
+import random
 from typing import Hashable
+
 from . import dictionary
 
 __all__ = ["generate_id"]
@@ -8,7 +9,7 @@ __all__ = ["generate_id"]
 system_random = random.SystemRandom()
 
 
-def generate_id(separator="-", seed: int | float | str | bytes | bytearray | None = None, word_count=4) -> str:
+def generate_id(separator="-", seed: int | float | str | bytes | bytearray | None = None, word_count=5) -> str:
     """
     Generate a human readable ID
 

eval_protocol-0.2.6.dev1/eval_protocol/logging_utils.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+"""
+Logging utilities for the eval_protocol package.
+
+This module provides centralized logging configuration and utilities
+for consistent logging across the eval_protocol package.
+"""
+
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import Optional
+
+from eval_protocol.dataset_logger.directory_utils import find_eval_protocol_dir
+
+
+def setup_logger(
+    name: str,
+    log_file: Optional[str] = None,
+    level: int = logging.INFO,
+    console_level: int = logging.INFO,
+    file_level: int = logging.DEBUG,
+) -> logging.Logger:
+    """
+    Set up a logger with both console and file handlers.
+
+    Args:
+        name: Logger name
+        log_file: Optional log file name (will be created in logs directory)
+        level: Overall logger level
+        console_level: Console handler level
+        file_level: File handler level
+
+    Returns:
+        Configured logger instance
+    """
+    # Create logs directory under eval_protocol
+    eval_protocol_dir = Path(find_eval_protocol_dir())
+    logs_dir = eval_protocol_dir / "logs"
+    logs_dir.mkdir(exist_ok=True)
+
+    # Create logger
+    logger = logging.getLogger(name)
+
+    # Only configure if not already configured (has handlers and proper level)
+    if logger.handlers and logger.level != logging.NOTSET:
+        return logger
+
+    logger.setLevel(level)
+
+    # Clear existing handlers to avoid duplicates
+    logger.handlers.clear()
+
+    # Create formatters
+    file_formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    console_formatter = logging.Formatter("%(levelname)s - %(message)s")
+
+    # Console handler - explicitly write to sys.stdout
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(console_level)
+    console_handler.setFormatter(console_formatter)
+    logger.addHandler(console_handler)
+
+    # File handler (if log_file specified) - explicitly write to file only
+    if log_file:
+        log_file_path = logs_dir / log_file
+        file_handler = logging.FileHandler(log_file_path)
+        file_handler.setLevel(file_level)
+        file_handler.setFormatter(file_formatter)
+        logger.addHandler(file_handler)
+
+    # Prevent propagation to avoid duplicate logging
+    logger.propagate = False
+
+    return logger
+
+
+def get_logger(name: str) -> logging.Logger:
+    """
+    Get a logger instance. If it doesn't exist, create it with default settings.
+
+    Args:
+        name: Logger name
+
+    Returns:
+        Logger instance
+    """
+    logger = logging.getLogger(name)
+
+    # If logger doesn't have handlers, set it up with defaults
+    if not logger.handlers:
+        # For eval_watcher, check if running in daemon mode
+        if name == "eval_watcher":
+            import sys
+
+            # Check if running in daemon mode (subprocess)
+            if "--daemon" in sys.argv:
+                # Subprocess: log to file only
+                logger = setup_logger(name, f"{name}.log", console_level=logging.CRITICAL)
+            else:
+                # Top-level: log to console only
+                logger = setup_logger(name, None)
+        else:
+            logger = setup_logger(name, f"{name}.log")
+
+    return logger
+
+
+def log_evaluation_event(
+    event_type: str, evaluation_id: str, message: str, level: int = logging.INFO, **kwargs
+) -> None:
+    """
+    Log evaluation-specific events to a dedicated evaluation log file.
+
+    Args:
+        event_type: Type of event (e.g., 'start', 'complete', 'error')
+        evaluation_id: Evaluation identifier
+        message: Log message
+        level: Log level
+        **kwargs: Additional context to include in log
+    """
+    logger = get_logger("evaluation_events")
+
+    # Create structured log entry
+    log_entry = {"event_type": event_type, "evaluation_id": evaluation_id, "message": message, **kwargs}
+
+    if level == logging.DEBUG:
+        logger.debug(f"EVENT: {log_entry}")
+    elif level == logging.INFO:
+        logger.info(f"EVENT: {log_entry}")
+    elif level == logging.WARNING:
+        logger.warning(f"EVENT: {log_entry}")
+    elif level == logging.ERROR:
+        logger.error(f"EVENT: {log_entry}")
+    elif level == logging.CRITICAL:
+        logger.critical(f"EVENT: {log_entry}")
+
+
+def log_performance_metric(metric_name: str, value: float, unit: str = "", context: Optional[dict] = None) -> None:
+    """
+    Log performance metrics to a dedicated metrics log file.
+
+    Args:
+        metric_name: Name of the metric
+        value: Metric value
+        unit: Unit of measurement
+        context: Additional context information
+    """
+    logger = get_logger("performance_metrics")
+
+    metric_entry = {"metric": metric_name, "value": value, "unit": unit, "context": context or {}}
+
+    logger.info(f"METRIC: {metric_entry}")
+
+
+def log_error_with_context(error: Exception, context: str, additional_info: Optional[dict] = None) -> None:
+    """
+    Log errors with additional context information.
+
+    Args:
+        error: The exception that occurred
+        context: Context where the error occurred
+        additional_info: Additional information about the error
+    """
+    logger = get_logger("errors")
+
+    error_entry = {
+        "error_type": type(error).__name__,
+        "error_message": str(error),
+        "context": context,
+        "additional_info": additional_info or {},
+    }
+
+    logger.error(f"ERROR: {error_entry}", exc_info=True)

{eval_protocol-0.2.5.dev1 → eval_protocol-0.2.6.dev1}/eval_protocol/mcp/client/connection.py

@@ -16,6 +16,7 @@ from mcp.client.session import ClientSession
 from mcp.client.streamable_http import streamablehttp_client
 
 from ...types import MCPSession
+from mcp.types import Implementation
 
 logger = logging.getLogger(__name__)
 
@@ -50,19 +51,16 @@ class MCPConnectionManager:
 
         exit_stack = AsyncExitStack()
 
-        client_info = None
-        if session.seed is not None or (session.dataset_row and session.dataset_row.environment_context):
-            from mcp.types import Implementation
-
-            client_info = Implementation(name="reward-kit", version="1.0.0", _extra={})
-            if session.seed is not None:
-                client_info._extra["seed"] = session.seed
-            if session.dataset_row and session.dataset_row.environment_context:
-                client_info._extra["config"] = session.dataset_row.environment_context
-            if session.dataset_row and session.dataset_row.id:
-                client_info._extra["dataset_row_id"] = session.dataset_row.id
-            if session.model_id:
-                client_info._extra["model_id"] = session.model_id
+        client_info = Implementation(name="reward-kit", version="1.0.0", _extra={})
+        client_info._extra["session_id"] = session.session_id
+        if session.seed is not None:
+            client_info._extra["seed"] = session.seed
+        if session.dataset_row and session.dataset_row.environment_context:
+            client_info._extra["config"] = session.dataset_row.environment_context
+        if session.dataset_row and session.dataset_row.id:
+            client_info._extra["dataset_row_id"] = session.dataset_row.id
+        if session.model_id:
+            client_info._extra["model_id"] = session.model_id
 
         read_stream, write_stream, _ = await exit_stack.enter_async_context(
             streamablehttp_client(session.base_url, terminate_on_close=True)
@@ -77,32 +75,6 @@ class MCPConnectionManager:
         session._mcp_session = mcp_session
         session._exit_stack = exit_stack
 
-        # Update session ID to match server's calculation (for control plane sync)
-        if client_info and hasattr(client_info, "_extra"):
-            extra_data = client_info._extra
-            if extra_data and isinstance(extra_data, dict):
-
-                seed_value = extra_data.get("seed")
-                config_value = extra_data.get("config", {})
-                dataset_row_id_value = extra_data.get("dataset_row_id")
-                model_id_value = extra_data.get("model_id")
-
-                stable_data = {
-                    "seed": seed_value,
-                    "config": config_value,
-                    "dataset_row_id": dataset_row_id_value,
-                    "model_id": model_id_value,
-                    "name": client_info.name,
-                    "version": client_info.version,
-                }
-
-                stable_str = json.dumps(stable_data, sort_keys=True)
-                server_session_id = hashlib.md5(stable_str.encode()).hexdigest()
-
-                # Update the session ID to match what the server generated
-                session.session_id = server_session_id
-                logger.info(f"Updated session ID to match server: {server_session_id}")
-
         # PRE-WARM: Discover and cache tools immediately after session initialization
         # This prevents concurrent list_tools() calls later
         await self._prewarm_tools_cache(session)