eval-protocol 0.2.98.dev1__tar.gz → 0.2.99.dev1__tar.gz

{eval_protocol-0.2.98.dev1/eval_protocol.egg-info → eval_protocol-0.2.99.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eval-protocol
-Version: 0.2.98.dev1
+Version: 0.2.99.dev1
 Summary: The official Python SDK for Eval Protocol (EP.) EP is an open protocol that standardizes how developers author evals for large language model (LLM) applications.
 Author-email: Fireworks AI <info@fireworks.ai>
 License-Expression: MIT
@@ -39,6 +39,7 @@ Requires-Dist: addict>=2.4.0
 Requires-Dist: deepdiff>=6.0.0
 Requires-Dist: websockets>=15.0.1
 Requires-Dist: fastapi>=0.116.1
+Requires-Dist: dspy>=3.0.0
 Provides-Extra: dev
 Requires-Dist: build; extra == "dev"
 Requires-Dist: twine; extra == "dev"

{eval_protocol-0.2.98.dev1 → eval_protocol-0.2.99.dev1}/eval_protocol/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-12-15T16:40:32-0800",
+ "date": "2025-12-16T22:20:08-0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "438a49431d16626a8e883cfb04afecfb188eb9dc",
- "version": "0.2.98.dev.1"
+ "full-revisionid": "2e19cf73b707d21f779e17809ab02ad5bb14b4d7",
+ "version": "0.2.99.dev.1"
 }
 '''  # END VERSION_JSON
 

{eval_protocol-0.2.98.dev1 → eval_protocol-0.2.99.dev1}/eval_protocol/agent/resources/sql_resource.py

@@ -12,6 +12,50 @@ from typing import Any, Dict, List, Optional
 from ..resource_abc import ForkableResource
 
 
+# SQLite connection settings for hardened concurrency safety
+SQLITE_CONNECTION_TIMEOUT = 30  # 30 seconds
+
+
+def _apply_hardened_pragmas(conn: sqlite3.Connection) -> None:
+    """Apply hardened SQLite pragmas for concurrency safety."""
+    conn.execute("PRAGMA journal_mode=WAL")  # Write-Ahead Logging
+    conn.execute("PRAGMA synchronous=NORMAL")  # Balance safety and performance
+    conn.execute("PRAGMA busy_timeout=30000")  # 30 second timeout
+    conn.execute("PRAGMA wal_autocheckpoint=1000")  # Checkpoint every 1000 pages
+    conn.execute("PRAGMA cache_size=-64000")  # 64MB cache
+    conn.execute("PRAGMA foreign_keys=ON")  # Enable foreign key constraints
+    conn.execute("PRAGMA temp_store=MEMORY")  # Store temp tables in memory
+
+
+def _checkpoint_and_copy_database(
+    source_path: Path, dest_path: Path, timeout: int = SQLITE_CONNECTION_TIMEOUT
+) -> None:
+    """
+    Safely copy a SQLite database by checkpointing WAL first.
+
+    In WAL mode, data may exist in the -wal file that hasn't been written
+    to the main database file. This function performs a TRUNCATE checkpoint
+    to flush all WAL data to the main file before copying, ensuring a
+    complete and consistent copy.
+
+    Args:
+        source_path: Path to the source database file.
+        dest_path: Path where the copy should be created.
+        timeout: Connection timeout in seconds.
+    """
+    # First, checkpoint the WAL to ensure all data is in the main file
+    conn = sqlite3.connect(str(source_path), timeout=timeout)
+    try:
+        # TRUNCATE mode: checkpoint and truncate the WAL file to zero bytes
+        # This ensures all data is flushed to the main database file
+        conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+    finally:
+        conn.close()
+
+    # Now safely copy just the main database file
+    shutil.copyfile(str(source_path), str(dest_path))
+
+
 class SQLResource(ForkableResource):
     """
     A ForkableResource for managing SQL database states, primarily SQLite.
@@ -20,6 +64,8 @@ class SQLResource(ForkableResource):
     and seed data, forked (by copying the DB file), checkpointed (by copying),
     and restored.
 
+    Uses hardened SQLite settings for concurrency safety.
+
     Attributes:
         _config (Dict[str, Any]): Configuration for the resource.
         _db_path (Optional[Path]): Path to the current SQLite database file.
@@ -38,8 +84,14 @@ class SQLResource(ForkableResource):
     def _get_db_connection(self) -> sqlite3.Connection:
         if not self._db_path:
             raise ConnectionError("Database path not set. Call setup() or fork() first.")
-        # Set timeout to prevent indefinite hangs
-        return sqlite3.connect(str(self._db_path), timeout=10)
+        # Set timeout to prevent indefinite hangs with hardened settings
+        conn = sqlite3.connect(
+            str(self._db_path),
+            timeout=SQLITE_CONNECTION_TIMEOUT,
+            isolation_level="DEFERRED",  # Better for concurrent access
+        )
+        _apply_hardened_pragmas(conn)
+        return conn
 
     async def setup(self, config: Dict[str, Any]) -> None:
         """
@@ -111,7 +163,8 @@ class SQLResource(ForkableResource):
         forked_db_name = f"fork_{uuid.uuid4().hex}.sqlite"
         forked_resource._db_path = self._temp_dir / forked_db_name
 
-        shutil.copyfile(str(self._db_path), str(forked_resource._db_path))
+        # Use checkpoint-and-copy to ensure WAL data is flushed before copying
+        _checkpoint_and_copy_database(self._db_path, forked_resource._db_path)
         return forked_resource
 
     async def checkpoint(self) -> Dict[str, Any]:
@@ -125,7 +178,8 @@ class SQLResource(ForkableResource):
 
         checkpoint_name = f"checkpoint_{self._db_path.stem}_{uuid.uuid4().hex}.sqlite"
         checkpoint_path = self._temp_dir / checkpoint_name
-        shutil.copyfile(str(self._db_path), str(checkpoint_path))
+        # Use checkpoint-and-copy to ensure WAL data is flushed before copying
+        _checkpoint_and_copy_database(self._db_path, checkpoint_path)
         return {"db_type": "sqlite", "checkpoint_path": str(checkpoint_path)}
 
     async def restore(self, state_data: Dict[str, Any]) -> None:
@@ -147,7 +201,8 @@ class SQLResource(ForkableResource):
         if not self._db_path:
             self._db_path = self._temp_dir / f"restored_{uuid.uuid4().hex}.sqlite"
 
-        shutil.copyfile(str(checkpoint_path), str(self._db_path))
+        # Use checkpoint-and-copy to ensure WAL data is flushed before copying
+        _checkpoint_and_copy_database(checkpoint_path, self._db_path)
         self._base_db_path = self._db_path  # The restored state becomes the new base for future forks
 
     async def step(self, action_name: str, action_params: Dict[str, Any]) -> Any:

eval_protocol-0.2.99.dev1/eval_protocol/auth.py

@@ -0,0 +1,106 @@
+import logging
+import os
+from typing import Optional
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def get_fireworks_api_key() -> Optional[str]:
+    """
+    Retrieves the Fireworks API key.
+
+    Returns:
+        The API key if found, otherwise None.
+    """
+    api_key = os.environ.get("FIREWORKS_API_KEY")
+    if api_key and api_key.strip():
+        logger.debug("Using FIREWORKS_API_KEY from environment variable.")
+        return api_key.strip()
+    logger.debug("Fireworks API key not found in environment variables.")
+    return None
+
+
+def get_fireworks_account_id() -> Optional[str]:
+    """
+    Retrieves the Fireworks Account ID.
+
+    Returns:
+        The Account ID if found, otherwise None.
+    """
+    # Account id is derived from the API key (single source of truth).
+    try:
+        api_key_for_verify = get_fireworks_api_key()
+        if api_key_for_verify:
+            resolved = verify_api_key_and_get_account_id(api_key=api_key_for_verify, api_base=get_fireworks_api_base())
+            if resolved:
+                logger.debug("Resolved account id via verifyApiKey: %s", resolved)
+                return resolved
+    except Exception as e:
+        logger.debug("Failed to resolve account id via verifyApiKey: %s", e)
+
+    logger.debug("Fireworks Account ID not found via verifyApiKey.")
+    return None
+
+
+def get_fireworks_api_base() -> str:
+    """
+    Retrieves the Fireworks API base URL.
+
+    The base URL is sourced from the FIREWORKS_API_BASE environment variable.
+    If not set, it defaults to "https://api.fireworks.ai".
+
+    Returns:
+        The API base URL.
+    """
+    api_base = os.environ.get("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+    if os.environ.get("FIREWORKS_API_BASE"):
+        logger.debug("Using FIREWORKS_API_BASE from environment variable.")
+    else:
+        logger.debug("FIREWORKS_API_BASE not set in environment, defaulting to %s.", api_base)
+    return api_base
+
+
+def verify_api_key_and_get_account_id(
+    api_key: Optional[str] = None,
+    api_base: Optional[str] = None,
+) -> Optional[str]:
+    """
+    Calls the Fireworks API verify endpoint to validate the API key and returns the
+    account id from response headers when available.
+
+    Args:
+        api_key: Optional explicit API key. When None, resolves via get_fireworks_api_key().
+        api_base: Optional explicit API base. When None, resolves via get_fireworks_api_base().
+
+    Returns:
+        The resolved account id if verification succeeds and the header is present; otherwise None.
+    """
+    try:
+        resolved_key = api_key or get_fireworks_api_key()
+        if not resolved_key:
+            return None
+        resolved_base = api_base or get_fireworks_api_base()
+
+        from .common_utils import get_user_agent
+
+        url = f"{resolved_base.rstrip('/')}/verifyApiKey"
+        headers = {
+            "Authorization": f"Bearer {resolved_key}",
+            "User-Agent": get_user_agent(),
+        }
+        resp = requests.get(url, headers=headers, timeout=10)
+
+        if resp.status_code != 200:
+            logger.debug("verifyApiKey returned status %s", resp.status_code)
+            return None
+        # Header keys could vary in case; requests provides case-insensitive dict
+        account_id = resp.headers.get("x-fireworks-account-id") or resp.headers.get("X-Fireworks-Account-Id")
+        if account_id and account_id.strip():
+            logger.debug("Resolved account id via verifyApiKey: %s", account_id)
+            return account_id.strip()
+        return None
+    except Exception as e:
+        logger.debug("Failed to verify API key for account id resolution: %s", e)
+        return None