eval-protocol 0.2.98.dev1__tar.gz → 0.3.10.dev2__tar.gz

{eval_protocol-0.2.98.dev1/eval_protocol.egg-info → eval_protocol-0.3.10.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.98.dev1
+Version: 0.3.10.dev2
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -29,6 +29,7 @@ Requires-Dist: pytest>=6.0.0
 Requires-Dist: pytest-asyncio>=0.21.0
 Requires-Dist: peewee>=3.18.2
 Requires-Dist: backoff>=2.2.0
+Requires-Dist: fireworks-ai==1.0.0a22
 Requires-Dist: questionary>=2.0.0
 Requires-Dist: toml>=0.10.0
 Requires-Dist: loguru>=0.6.0
@@ -72,8 +73,6 @@ Requires-Dist: transformers>=4.0.0; extra == "trl"
 Requires-Dist: accelerate>=0.28.0; extra == "trl"
 Provides-Extra: openevals
 Requires-Dist: openevals>=0.1.0; extra == "openevals"
-Provides-Extra: fireworks
-Requires-Dist: fireworks-ai>=0.19.19; extra == "fireworks"
 Provides-Extra: box2d
 Requires-Dist: swig; extra == "box2d"
 Requires-Dist: gymnasium[box2d]>=0.29.0; extra == "box2d"
@@ -102,6 +101,10 @@ Provides-Extra: braintrust
 Requires-Dist: braintrust[otel]; extra == "braintrust"
 Provides-Extra: openenv
 Requires-Dist: openenv-core; extra == "openenv"
+Provides-Extra: dspy
+Requires-Dist: dspy>=3.0.0; extra == "dspy"
+Provides-Extra: klavis
+Requires-Dist: klavis>=2.18.0; extra == "klavis"
 Provides-Extra: langgraph
 Requires-Dist: langgraph>=0.6.7; extra == "langgraph"
 Requires-Dist: langchain-core>=0.3.75; extra == "langgraph"

{eval_protocol-0.2.98.dev1 → eval_protocol-0.3.10.dev2}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-12-15T16:40:32-0800",
+ "date": "2026-01-13T16:25:00-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "438a49431d16626a8e883cfb04afecfb188eb9dc",
- "version": "0.2.98.dev.1"
+ "full-revisionid": "66f191a09db5364b9cd9bb21230e1f48e50be724",
+ "version": "0.3.10.dev.2"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.98.dev1 → eval_protocol-0.3.10.dev2}/eval_protocol/adapters/__init__.py

@@ -99,3 +99,19 @@ try:
     __all__.extend(["WeaveAdapter"])
 except ImportError:
     pass
+
+# DataFrame adapter (pandas integration for Lilac, etc.)
+try:
+    from .dataframe import (
+        evaluation_rows_to_dataframe,
+        dataframe_to_evaluation_rows,
+    )
+
+    __all__.extend(
+        [
+            "evaluation_rows_to_dataframe",
+            "dataframe_to_evaluation_rows",
+        ]
+    )
+except ImportError:
+    pass

eval_protocol-0.3.10.dev2/eval_protocol/adapters/dataframe.py

@@ -0,0 +1,66 @@
+"""
+Pandas DataFrame adapter for Eval Protocol.
+
+This module provides utilities for converting between EvaluationRow format
+and pandas DataFrame format, enabling integration with data curation tools
+such as Lilac, Great Expectations, or any pandas-based workflow.
+
+Example usage:
+    >>> from eval_protocol.adapters.dataframe import (
+    ...     evaluation_rows_to_dataframe,
+    ...     dataframe_to_evaluation_rows,
+    ... )
+    >>>
+    >>> # Convert EvaluationRows to DataFrame
+    >>> df = evaluation_rows_to_dataframe(rows)
+    >>>
+    >>> # Convert back to EvaluationRows
+    >>> rows = dataframe_to_evaluation_rows(df)
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pandas as pd
+
+from ..models import EvaluationRow
+
+logger = logging.getLogger(__name__)
+
+
+def evaluation_rows_to_dataframe(rows: list[EvaluationRow]) -> pd.DataFrame:
+    """Convert EvaluationRows to a pandas DataFrame.
+
+    Uses EvaluationRow.to_dict() for serialization.
+
+    Args:
+        rows: List of EvaluationRow objects
+
+    Returns:
+        DataFrame with 'data_json' containing serialized rows plus convenience fields
+    """
+    records = [row.to_dict() for row in rows]
+    return pd.DataFrame(records)
+
+
+def dataframe_to_evaluation_rows(df: pd.DataFrame) -> list[EvaluationRow]:
+    """Convert a pandas DataFrame back to EvaluationRows.
+
+    Uses EvaluationRow.from_dict() for deserialization.
+
+    Args:
+        df: DataFrame with 'data_json' column containing serialized EvaluationRows
+
+    Returns:
+        List of EvaluationRow objects
+    """
+    rows = []
+    for _, row_data in df.iterrows():
+        try:
+            row = EvaluationRow.from_dict(row_data.to_dict())
+            rows.append(row)
+        except Exception as e:
+            logger.warning(f"Failed to convert row: {e}")
+            continue
+    return rows

{eval_protocol-0.2.98.dev1 → eval_protocol-0.3.10.dev2}/eval_protocol/adapters/fireworks_tracing.py

@@ -253,6 +253,7 @@ class FireworksTracingAdapter(BaseAdapter):
         project_id: Optional[str] = None,
         base_url: str = "https://tracing.fireworks.ai",
         timeout: int = 300,
+        api_key: Optional[str] = None,
     ):
         """Initialize the Fireworks Tracing adapter.
 
@@ -260,15 +261,21 @@ class FireworksTracingAdapter(BaseAdapter):
             project_id: Optional project ID. If not provided, uses the default project configured on the server.
             base_url: The base URL of the tracing proxy (default: https://tracing.fireworks.ai)
             timeout: Request timeout in seconds (default: 300)
+            api_key: Optional API key. If not provided, falls back to FIREWORKS_API_KEY environment variable.
         """
         self.project_id = project_id
         self.base_url = base_url.rstrip("/")
         self.timeout = timeout
+        self._api_key = api_key
+
+    def _get_api_key(self) -> Optional[str]:
+        """Get the API key, preferring instance-level key over environment variable."""
+        return self._api_key or os.environ.get("FIREWORKS_API_KEY")
 
     def search_logs(self, tags: List[str], limit: int = 100, hours_back: int = 24) -> List[Dict[str, Any]]:
         """Fetch logs from Fireworks tracing gateway /logs endpoint.
 
-        Returns entries with keys: timestamp, message, severity, tags.
+        Returns entries with keys: timestamp, message, severity, tags, status, extras.
         """
         if not tags:
             raise ValueError("At least one tag is required to fetch logs")
@@ -276,7 +283,7 @@ class FireworksTracingAdapter(BaseAdapter):
         from ..common_utils import get_user_agent
 
         headers = {
-            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "Authorization": f"Bearer {self._get_api_key()}",
             "User-Agent": get_user_agent(),
         }
         params: Dict[str, Any] = {"tags": tags, "limit": limit, "hours_back": hours_back, "program": "eval_protocol"}
@@ -315,6 +322,7 @@ class FireworksTracingAdapter(BaseAdapter):
                     "severity": e.get("severity", "INFO"),
                     "tags": e.get("tags", []),
                     "status": e.get("status"),
+                    "extras": e.get("extras"),
                 }
             )
         return results
@@ -406,7 +414,7 @@ class FireworksTracingAdapter(BaseAdapter):
         from ..common_utils import get_user_agent
 
         headers = {
-            "Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}",
+            "Authorization": f"Bearer {self._get_api_key()}",
             "User-Agent": get_user_agent(),
         }
 

{eval_protocol-0.2.98.dev1 → eval_protocol-0.3.10.dev2}/eval_protocol/agent/resources/sql_resource.py

@@ -12,6 +12,50 @@ from typing import Any, Dict, List, Optional
 from ..resource_abc import ForkableResource
 
 
+# SQLite connection settings for hardened concurrency safety
+SQLITE_CONNECTION_TIMEOUT = 30  # 30 seconds
+
+
+def _apply_hardened_pragmas(conn: sqlite3.Connection) -> None:
+    """Apply hardened SQLite pragmas for concurrency safety."""
+    conn.execute("PRAGMA journal_mode=WAL")  # Write-Ahead Logging
+    conn.execute("PRAGMA synchronous=NORMAL")  # Balance safety and performance
+    conn.execute("PRAGMA busy_timeout=30000")  # 30 second timeout
+    conn.execute("PRAGMA wal_autocheckpoint=1000")  # Checkpoint every 1000 pages
+    conn.execute("PRAGMA cache_size=-64000")  # 64MB cache
+    conn.execute("PRAGMA foreign_keys=ON")  # Enable foreign key constraints
+    conn.execute("PRAGMA temp_store=MEMORY")  # Store temp tables in memory
+
+
+def _checkpoint_and_copy_database(
+    source_path: Path, dest_path: Path, timeout: int = SQLITE_CONNECTION_TIMEOUT
+) -> None:
+    """
+    Safely copy a SQLite database by checkpointing WAL first.
+
+    In WAL mode, data may exist in the -wal file that hasn't been written
+    to the main database file. This function performs a TRUNCATE checkpoint
+    to flush all WAL data to the main file before copying, ensuring a
+    complete and consistent copy.
+
+    Args:
+        source_path: Path to the source database file.
+        dest_path: Path where the copy should be created.
+        timeout: Connection timeout in seconds.
+    """
+    # First, checkpoint the WAL to ensure all data is in the main file
+    conn = sqlite3.connect(str(source_path), timeout=timeout)
+    try:
+        # TRUNCATE mode: checkpoint and truncate the WAL file to zero bytes
+        # This ensures all data is flushed to the main database file
+        conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+    finally:
+        conn.close()
+
+    # Now safely copy just the main database file
+    shutil.copyfile(str(source_path), str(dest_path))
+
+
 class SQLResource(ForkableResource):
     """
     A ForkableResource for managing SQL database states, primarily SQLite.
@@ -20,6 +64,8 @@ class SQLResource(ForkableResource):
     and seed data, forked (by copying the DB file), checkpointed (by copying),
     and restored.
 
+    Uses hardened SQLite settings for concurrency safety.
+
     Attributes:
         _config (Dict[str, Any]): Configuration for the resource.
         _db_path (Optional[Path]): Path to the current SQLite database file.
@@ -38,8 +84,14 @@ class SQLResource(ForkableResource):
     def _get_db_connection(self) -> sqlite3.Connection:
         if not self._db_path:
             raise ConnectionError("Database path not set. Call setup() or fork() first.")
-        # Set timeout to prevent indefinite hangs
-        return sqlite3.connect(str(self._db_path), timeout=10)
+        # Set timeout to prevent indefinite hangs with hardened settings
+        conn = sqlite3.connect(
+            str(self._db_path),
+            timeout=SQLITE_CONNECTION_TIMEOUT,
+            isolation_level="DEFERRED",  # Better for concurrent access
+        )
+        _apply_hardened_pragmas(conn)
+        return conn
 
     async def setup(self, config: Dict[str, Any]) -> None:
         """
@@ -111,7 +163,8 @@ class SQLResource(ForkableResource):
         forked_db_name = f"fork_{uuid.uuid4().hex}.sqlite"
         forked_resource._db_path = self._temp_dir / forked_db_name
 
-        shutil.copyfile(str(self._db_path), str(forked_resource._db_path))
+        # Use checkpoint-and-copy to ensure WAL data is flushed before copying
+        _checkpoint_and_copy_database(self._db_path, forked_resource._db_path)
         return forked_resource
 
     async def checkpoint(self) -> Dict[str, Any]:
@@ -125,7 +178,8 @@ class SQLResource(ForkableResource):
 
         checkpoint_name = f"checkpoint_{self._db_path.stem}_{uuid.uuid4().hex}.sqlite"
         checkpoint_path = self._temp_dir / checkpoint_name
-        shutil.copyfile(str(self._db_path), str(checkpoint_path))
+        # Use checkpoint-and-copy to ensure WAL data is flushed before copying
+        _checkpoint_and_copy_database(self._db_path, checkpoint_path)
         return {"db_type": "sqlite", "checkpoint_path": str(checkpoint_path)}
 
     async def restore(self, state_data: Dict[str, Any]) -> None:
@@ -147,7 +201,8 @@ class SQLResource(ForkableResource):
         if not self._db_path:
             self._db_path = self._temp_dir / f"restored_{uuid.uuid4().hex}.sqlite"
 
-        shutil.copyfile(str(checkpoint_path), str(self._db_path))
+        # Use checkpoint-and-copy to ensure WAL data is flushed before copying
+        _checkpoint_and_copy_database(checkpoint_path, self._db_path)
         self._base_db_path = self._db_path  # The restored state becomes the new base for future forks
 
     async def step(self, action_name: str, action_params: Dict[str, Any]) -> Any:

eval_protocol-0.3.10.dev2/eval_protocol/auth.py

@@ -0,0 +1,176 @@
+import logging
+import os
+from typing import Dict, Optional
+
+import requests
+from dotenv import dotenv_values, find_dotenv, load_dotenv
+
+logger = logging.getLogger(__name__)
+
+
+def find_dotenv_path(search_path: Optional[str] = None) -> Optional[str]:
+    """
+    Find the .env file path, searching .env.dev first, then .env.
+
+    Args:
+        search_path: Directory to search from. If None, uses current working directory.
+
+    Returns:
+        Path to the .env file if found, otherwise None.
+    """
+    # If a specific search path is provided, look there first
+    if search_path:
+        env_dev_path = os.path.join(search_path, ".env.dev")
+        if os.path.isfile(env_dev_path):
+            return env_dev_path
+        env_path = os.path.join(search_path, ".env")
+        if os.path.isfile(env_path):
+            return env_path
+        return None
+
+    # Otherwise use find_dotenv to search up the directory tree
+    env_dev_path = find_dotenv(filename=".env.dev", raise_error_if_not_found=False, usecwd=True)
+    if env_dev_path:
+        return env_dev_path
+    env_path = find_dotenv(filename=".env", raise_error_if_not_found=False, usecwd=True)
+    if env_path:
+        return env_path
+    return None
+
+
+def get_dotenv_values(search_path: Optional[str] = None) -> Dict[str, Optional[str]]:
+    """
+    Get all key-value pairs from the .env file.
+
+    Args:
+        search_path: Directory to search from. If None, uses current working directory.
+
+    Returns:
+        Dictionary of environment variable names to values.
+    """
+    dotenv_path = find_dotenv_path(search_path)
+    if dotenv_path:
+        return dotenv_values(dotenv_path)
+    return {}
+
+
+# --- Load .env files ---
+# Attempt to load .env.dev first, then .env as a fallback.
+# This happens when the module is imported.
+# We use override=False (default) so that existing environment variables
+# (e.g., set in the shell) are NOT overridden by .env files.
+_DOTENV_PATH = find_dotenv_path()
+if _DOTENV_PATH:
+    load_dotenv(dotenv_path=_DOTENV_PATH, override=False)
+    logger.debug(f"eval_protocol.auth: Loaded environment variables from: {_DOTENV_PATH}")
+else:
+    logger.debug(
+        "eval_protocol.auth: No .env.dev or .env file found. Relying on shell/existing environment variables."
+    )
+# --- End .env loading ---
+
+
+def get_fireworks_api_key() -> Optional[str]:
+    """
+    Retrieves the Fireworks API key.
+
+    Returns:
+        The API key if found, otherwise None.
+    """
+    api_key = os.environ.get("FIREWORKS_API_KEY")
+    if api_key and api_key.strip():
+        logger.debug("Using FIREWORKS_API_KEY from environment variable.")
+        return api_key.strip()
+    logger.debug("Fireworks API key not found in environment variables.")
+    return None
+
+
+def get_fireworks_account_id() -> Optional[str]:
+    """
+    Retrieves the Fireworks Account ID.
+
+    Returns:
+        The Account ID if found, otherwise None.
+    """
+    # Account id is derived from the API key (single source of truth).
+    try:
+        api_key_for_verify = get_fireworks_api_key()
+        if api_key_for_verify:
+            resolved = verify_api_key_and_get_account_id(api_key=api_key_for_verify, api_base=get_fireworks_api_base())
+            if resolved:
+                logger.debug("Resolved account id via verifyApiKey: %s", resolved)
+                return resolved
+    except Exception as e:
+        logger.debug("Failed to resolve account id via verifyApiKey: %s", e)
+
+    logger.debug("Fireworks Account ID not found via verifyApiKey.")
+    return None
+
+
+def get_fireworks_api_base() -> str:
+    """
+    Retrieves the Fireworks API base URL.
+
+    The base URL is sourced from the FIREWORKS_API_BASE environment variable.
+    If not set, it defaults to "https://api.fireworks.ai".
+
+    Returns:
+        The API base URL.
+    """
+    api_base = os.environ.get("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+    if os.environ.get("FIREWORKS_API_BASE"):
+        logger.debug("Using FIREWORKS_API_BASE from environment variable.")
+    else:
+        logger.debug("FIREWORKS_API_BASE not set in environment, defaulting to %s.", api_base)
+    return api_base
+
+
+def verify_api_key_and_get_account_id(
+    api_key: Optional[str] = None,
+    api_base: Optional[str] = None,
+) -> Optional[str]:
+    """
+    Calls the Fireworks API verify endpoint to validate the API key and returns the
+    account id from response headers when available.
+
+    Args:
+        api_key: Optional explicit API key. When None, resolves via get_fireworks_api_key().
+        api_base: Optional explicit API base. When None, resolves via get_fireworks_api_base().
+            If api_base is api.fireworks.ai, it is used directly. Otherwise, defaults to
+            dev.api.fireworks.ai for the verification call.
+
+    Returns:
+        The resolved account id if verification succeeds and the header is present; otherwise None.
+    """
+    try:
+        resolved_key = api_key or get_fireworks_api_key()
+        if not resolved_key:
+            return None
+        provided_base = api_base or get_fireworks_api_base()
+        # Use api.fireworks.ai if explicitly provided, otherwise fall back to dev
+        if "api.fireworks.ai" in provided_base:
+            resolved_base = provided_base
+        else:
+            resolved_base = "https://dev.api.fireworks.ai"
+
+        from .common_utils import get_user_agent
+
+        url = f"{resolved_base.rstrip('/')}/verifyApiKey"
+        headers = {
+            "Authorization": f"Bearer {resolved_key}",
+            "User-Agent": get_user_agent(),
+        }
+        resp = requests.get(url, headers=headers, timeout=10)
+
+        if resp.status_code != 200:
+            logger.debug("verifyApiKey returned status %s", resp.status_code)
+            return None
+        # Header keys could vary in case; requests provides case-insensitive dict
+        account_id = resp.headers.get("x-fireworks-account-id") or resp.headers.get("X-Fireworks-Account-Id")
+        if account_id and account_id.strip():
+            logger.debug("Resolved account id via verifyApiKey: %s", account_id)
+            return account_id.strip()
+        return None
+    except Exception as e:
+        logger.debug("Failed to verify API key for account id resolution: %s", e)
+        return None