fiddler-evals 0.1.1.dev2__tar.gz → 0.1.1.dev9__tar.gz

{fiddler_evals-0.1.1.dev2/fiddler_evals.egg-info → fiddler_evals-0.1.1.dev9}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: fiddler-evals
-Version: 0.1.1.dev2
+Version: 0.1.1.dev9
 Summary: Python SDK for evaluating LLM Applications
 Author-email: Fiddler AI <support@fiddler.ai>
+Maintainer-email: Fiddler AI <support@fiddler.ai>
 Project-URL: Homepage, https://fiddler.ai
-Project-URL: Repository, https://github.com/fiddler-labs/fiddler-evals-sdk
 Project-URL: Documentation, https://docs.fiddler.ai/
 Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent

fiddler_evals-0.1.1.dev9/fiddler_evals/VERSION

@@ -0,0 +1 @@
+0.1.1.dev9

{fiddler_evals-0.1.1.dev2 → fiddler_evals-0.1.1.dev9}/fiddler_evals/runner/evaluation.py

@@ -5,7 +5,7 @@ from datetime import datetime
 from typing import Callable
 
 from fiddler_evals.entities.dataset import Dataset
-from fiddler_evals.entities.experiment import Experiment, ExperimentStatus
+from fiddler_evals.entities.experiment import Experiment
 from fiddler_evals.evaluators.base import Evaluator
 from fiddler_evals.evaluators.eval_fn import EvalFn
 from fiddler_evals.runner.experiment_runner import (
@@ -161,17 +161,7 @@ def evaluate(
         max_workers=max_workers,
     )
 
-    try:
-        return runner()
-    except KeyboardInterrupt:
-        # Set experiment status to cancelled if interrupted by the user
-        experiment.update(
-            status=ExperimentStatus.CANCELLED,
-            error_reason="KeyboardInterrupt",
-            error_message="Experiment cancelled by the user",
-            traceback=None,
-        )
-        raise
+    return runner()
 
 
 # Alias

{fiddler_evals-0.1.1.dev2 → fiddler_evals-0.1.1.dev9}/fiddler_evals/runner/experiment_runner.py

@@ -2,9 +2,10 @@ from __future__ import annotations
 
 import inspect
 import logging
+import signal
 import time
 from dataclasses import dataclass
-from functools import cached_property
+from functools import cached_property, wraps
 from typing import Any, Callable, Dict, Union
 
 from tqdm.autonotebook import tqdm
@@ -32,6 +33,37 @@ from fiddler_evals.utils.tqdm import thread_map
 
 logger = logging.getLogger(__name__)
 
+
+def with_signal_handlers(func: Callable) -> Callable:
+    """Decorator to setup and cleanup signal handlers for experiment execution.
+
+    This decorator automatically:
+    1. Sets up SIGINT and SIGTERM handlers before execution
+    2. Cleans up signal handlers after execution (success or failure)
+    3. Ensures proper resource cleanup
+
+    Args:
+        func: The method to decorate (typically the run method)
+
+    Returns:
+        The decorated method with signal handling
+    """
+
+    @wraps(func)
+    def wrapper(self: ExperimentRunner, *args: Any, **kwargs: Any) -> Any:
+        # Setup signal handlers
+        self._setup_signal_handlers()
+
+        try:
+            # Execute the original method
+            return func(self, *args, **kwargs)
+        finally:
+            # Always cleanup signal handlers
+            self._cleanup_signal_handlers()
+
+    return wrapper
+
+
 # Function(inputs, extras, metadata) -> outputs
 EvalTaskType = Callable[
     [Dict[str, Any], Dict[str, Any], Dict[str, Any]], Dict[str, Any]
@@ -145,6 +177,43 @@ class ExperimentRunner:
         """Get the HTTP client for making API requests."""
         return get_connection().client
 
+    def _setup_signal_handlers(self) -> None:
+        """Register signal handlers for graceful shutdown on SIGTERM/SIGINT."""
+        signal.signal(signal.SIGTERM, self._handle_shutdown)
+        signal.signal(signal.SIGINT, self._handle_shutdown)
+
+    def _cleanup_signal_handlers(self) -> None:
+        """Restore default signal handlers after experiment completion."""
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
+        signal.signal(signal.SIGTERM, signal.SIG_DFL)
+
+    def _handle_shutdown(self, signum: int, _frame: Any) -> None:
+        """Handle shutdown signal by cancelling the running experiment.
+
+        :param signum: Signal number received
+        :param _frame: Signal frame (unused)
+        """
+        logger.info(
+            "Received shutdown signal %s(%d), cancelling experiment - %s",
+            self._experiment.name,
+            signal.Signals(signum).name,
+            signum,
+        )
+
+        self._result_publisher.flush()
+
+        self._experiment.update(
+            status=ExperimentStatus.CANCELLED,
+            error_reason=f"Received shutdown signal {signal.Signals(signum).name}({signum})",
+            error_message="Experiment cancelled due to user interruption or system termination",
+            traceback=None,
+        )
+
+        # Use KeyboardInterrupt for both signals (consistent)
+        raise KeyboardInterrupt(
+            f"Experiment cancelled due to {signal.Signals(signum).name}({signum})"
+        )
+
     def _run_eval_task(
         self,
         dataset_item: DatasetItem,
@@ -521,6 +590,7 @@ class ExperimentRunner:
                 f"Expected: {expected_params}. Original error: {str(e)}"
             ) from e
 
+    @with_signal_handlers
     def run(
         self,
     ) -> ExperimentResult:

{fiddler_evals-0.1.1.dev2 → fiddler_evals-0.1.1.dev9/fiddler_evals.egg-info}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: fiddler-evals
-Version: 0.1.1.dev2
+Version: 0.1.1.dev9
 Summary: Python SDK for evaluating LLM Applications
 Author-email: Fiddler AI <support@fiddler.ai>
+Maintainer-email: Fiddler AI <support@fiddler.ai>
 Project-URL: Homepage, https://fiddler.ai
-Project-URL: Repository, https://github.com/fiddler-labs/fiddler-evals-sdk
 Project-URL: Documentation, https://docs.fiddler.ai/
 Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent

{fiddler_evals-0.1.1.dev2 → fiddler_evals-0.1.1.dev9}/pyproject.toml

@@ -1,9 +1,12 @@
 [project]
 name = "fiddler-evals"
-version = "0.1.1.dev2"
+version = "0.1.1.dev9"
 authors = [
   { name = "Fiddler AI", email = "support@fiddler.ai" },
 ]
+maintainers = [
+  { name = "Fiddler AI", email = "support@fiddler.ai" },
+]
 classifiers = [
   "Programming Language :: Python :: 3",
   "Operating System :: OS Independent",
@@ -26,7 +29,6 @@ pandas = ["pandas>=1.2.5"]
 
 [project.urls]
 Homepage = "https://fiddler.ai"
-Repository = "https://github.com/fiddler-labs/fiddler-evals-sdk"
 Documentation = "https://docs.fiddler.ai/"
 
 [build-system]

fiddler_evals-0.1.1.dev2/fiddler_evals/VERSION

@@ -1 +0,0 @@
-0.1.1.dev2