judgeval 0.0.44__py3-none-any.whl → 0.0.46__py3-none-any.whl

judgeval/__init__.py

@@ -2,11 +2,12 @@
 from judgeval.clients import client, together_client
 from judgeval.judgment_client import JudgmentClient
 from judgeval.version_check import check_latest_version
+
 check_latest_version()
 
 __all__ = [
     # Clients
-    'client',
-    'together_client',
-    'JudgmentClient',
-]
+    "client",
+    "together_client",
+    "JudgmentClient",
+]

judgeval/clients.py

@@ -9,18 +9,19 @@ load_dotenv(dotenv_path=PATH_TO_DOTENV)
 
 
 # Initialize optional OpenAI client
-client: Optional['OpenAI'] = None
+client: Optional["OpenAI"] = None
 if os.getenv("OPENAI_API_KEY"):
     try:
         from openai import OpenAI
+
         client = OpenAI()
     except ImportError:
         # openai package not installed
         pass
-    
+
 # Initialize optional Together clients
-together_client: Optional['Together'] = None
-async_together_client: Optional['AsyncTogether'] = None
+together_client: Optional["Together"] = None
+async_together_client: Optional["AsyncTogether"] = None
 
 # Only initialize Together clients if API key is available
 
@@ -29,6 +30,5 @@ if together_api_key:
     try:
         together_client = Together(api_key=together_api_key)
         async_together_client = AsyncTogether(api_key=together_api_key)
-    except Exception as e:
+    except Exception:
         pass
-

judgeval/common/__init__.py

@@ -2,7 +2,12 @@ from judgeval.common.utils import (
     get_chat_completion,
     aget_chat_completion,
     get_completion_multiple_models,
-    aget_completion_multiple_models
+    aget_completion_multiple_models,
 )
 
-__all__ = ["get_chat_completion", "aget_chat_completion", "get_completion_multiple_models", "aget_completion_multiple_models"]
+__all__ = [
+    "get_chat_completion",
+    "aget_chat_completion",
+    "get_completion_multiple_models",
+    "aget_completion_multiple_models",
+]

judgeval/common/exceptions.py

@@ -11,7 +11,7 @@ class JudgmentAPIError(Exception):
     """
     Exception raised when an error occurs while executing a Judgment API request
     """
-    
+
     def __init__(self, message: str):
         super().__init__(message)
         self.message = message
@@ -21,8 +21,7 @@ class InvalidJudgeModelError(Exception):
     """
     Exception raised when an invalid judge model is provided
     """
-    
+
     def __init__(self, message: str):
         super().__init__(message)
         self.message = message
-

judgeval/common/logger.py

@@ -6,9 +6,12 @@ from contextlib import contextmanager
 
 # Global variables
 logger = None
+
+
 class LoggingState:
-    enabled = False
-    path = None
+    enabled: bool = False
+    path: str | None = None
+
 
 LOGGING_STATE = LoggingState()
 
@@ -18,7 +21,12 @@ current_timestamp = None
 
 
 @contextmanager
-def enable_logging(name: str = "judgeval", path: str = "./logs", max_bytes: int = 1024 * 1024, backup_count: int = 5):
+def enable_logging(
+    name: str = "judgeval",
+    path: str = "./logs",
+    max_bytes: int = 1024 * 1024,
+    backup_count: int = 5,
+):
     """
     Context manager to temporarily enable logging for a specific block of code.
     """
@@ -27,7 +35,9 @@ def enable_logging(name: str = "judgeval", path: str = "./logs", max_bytes: int
     LOGGING_STATE.path = path
     # Initialize logger if not already initialized
     if logger is None:
-        logger = _initialize_logger(name=name, path=path, max_bytes=max_bytes, backup_count=backup_count)
+        logger = _initialize_logger(
+            name=name, path=path, max_bytes=max_bytes, backup_count=backup_count
+        )
     try:
         logger.info("Logging enabled")
         yield
@@ -36,36 +46,31 @@ def enable_logging(name: str = "judgeval", path: str = "./logs", max_bytes: int
         LOGGING_STATE.enabled = False
         LOGGING_STATE.path = None
 
+
 def _initialize_logger(
     name: str = "judgeval",
     max_bytes: int = 1024 * 1024,  # 1MB
     backup_count: int = 5,
-    path: str = "./logs"  # Added path parameter with default
+    path: str = "./logs",  # Added path parameter with default
 ) -> logging.Logger:
     """
     Initialize the global logger instance if it doesn't exist.
     Returns the global logger instance.
     """
     global logger
-    
+
     log_dir = Path(path)
     log_dir.mkdir(exist_ok=True, parents=True)
     log_file = log_dir / f"{name}.log"
     if log_file.exists():
         log_file.unlink()  # Delete existing log file
-    
+
     if logger is not None:
         return logger
-        
+
     # Create logs directory if it doesn't exist
     log_dir = Path(path)
     log_dir.mkdir(exist_ok=True)
-    
-    # Create formatter
-    formatter = logging.Formatter(
-        fmt='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S'
-    )
 
     # Create a custom formatter that includes example info when available
     class ExampleFormatter(logging.Formatter):
@@ -73,22 +78,23 @@ def _initialize_logger(
             if current_example_id is not None and current_timestamp is not None:
                 record.example_id = current_example_id
                 record.timestamp = current_timestamp
-                return logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - [Example_%(example_id)s][%(timestamp)s] %(message)s', 
-                                      datefmt='%Y-%m-%d %H:%M:%S').format(record)
-            return logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-                                  datefmt='%Y-%m-%d %H:%M:%S').format(record)
-    
+                return logging.Formatter(
+                    "%(asctime)s - %(name)s - %(levelname)s - [Example_%(example_id)s][%(timestamp)s] %(message)s",
+                    datefmt="%Y-%m-%d %H:%M:%S",
+                ).format(record)
+            return logging.Formatter(
+                "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+                datefmt="%Y-%m-%d %H:%M:%S",
+            ).format(record)
+
     # Use the custom formatter
     console_handler = logging.StreamHandler(sys.stdout)
     console_handler.setFormatter(ExampleFormatter())
     console_handler.setLevel(logging.DEBUG)
-    
+
     log_filename = f"{name}.log"
     file_handler = RotatingFileHandler(
-        log_dir / log_filename,
-        maxBytes=max_bytes,
-        backupCount=backup_count,
-        mode='a'
+        log_dir / log_filename, maxBytes=max_bytes, backupCount=backup_count, mode="a"
     )
     file_handler.setFormatter(ExampleFormatter())
     file_handler.setLevel(logging.DEBUG)
@@ -96,93 +102,112 @@ def _initialize_logger(
     # Get logger
     logger = logging.getLogger(name)
     logger.setLevel(logging.DEBUG)
-    
+
     # Prevent adding handlers multiple times
     if not logger.handlers:
         logger.addHandler(console_handler)
         logger.addHandler(file_handler)
-    
+
     return logger
 
+
 # Initialize the global logger when module is imported
 # logger = _initialize_logger()
 
+
 def log_if_enabled(func):
     """Decorator to check if logging is enabled before executing logging statements"""
+
     def wrapper(*args, **kwargs):
         if LOGGING_STATE.enabled:
             return func(*args, **kwargs)
+
     return wrapper
 
+
 @log_if_enabled
-def debug(msg: str, example_idx: int = None):
+def debug(msg: str, example_idx: int | None = None):
     """Log debug message if logging is enabled"""
-    logger.debug(msg)
+    if logger:
+        logger.debug(msg)
+
 
 @log_if_enabled
-def info(msg: str, example_idx: int = None):
+def info(msg: str, example_idx: int | None = None):
     """Log info message if logging is enabled"""
-    logger.info(msg)
+    if logger:
+        logger.info(msg)
+
 
 @log_if_enabled
-def warning(msg: str, example_idx: int = None):
+def warning(msg: str, example_idx: int | None = None):
     """Log warning message if logging is enabled"""
-    logger.warning(msg)
+    if logger:
+        logger.warning(msg)
+
 
 @log_if_enabled
-def error(msg: str, example_idx: int = None):
+def error(msg: str, example_idx: int | None = None):
     """Log error message if logging is enabled"""
-    logger.error(msg)
+    if logger:
+        logger.error(msg)
+
 
 def create_example_handler(
-    timestamp: str, 
+    timestamp: str,
     example_idx: int,
-    path: str = "./logs"  # Added path parameter with default
+    path: str = "./logs",  # Added path parameter with default
 ) -> RotatingFileHandler:
     """Creates a file handler for a specific example"""
-    debug(f"Creating example handler for timestamp={timestamp}, example_idx={example_idx}")
+    debug(
+        f"Creating example handler for timestamp={timestamp}, example_idx={example_idx}"
+    )
     log_dir = Path(path) / "examples"
     log_dir.mkdir(exist_ok=True, parents=True)
-    
+
     formatter = logging.Formatter(
-        fmt='%(asctime)s - %(name)s - %(levelname)s - [Example_%(example_id)s][%(timestamp)s] %(message)s',
-        datefmt='%Y-%m-%d %H:%M:%S'
+        fmt="%(asctime)s - %(name)s - %(levelname)s - [Example_%(example_id)s][%(timestamp)s] %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
     )
-    
+
     # Create a unique file for each example
     file_handler = RotatingFileHandler(
         log_dir / f"{timestamp}_example_{example_idx}.log",
         maxBytes=1024 * 1024,  # 1MB
         backupCount=5,
-        mode='a'
+        mode="a",
     )
     file_handler.setFormatter(formatter)
     file_handler.setLevel(logging.DEBUG)
     info(f"Created example handler for example {example_idx}")
     return file_handler
 
+
 @contextmanager
 def example_logging_context(timestamp: str, example_idx: int):
     """Context manager for example-specific logging"""
     if not LOGGING_STATE.enabled:
         yield
         return
-        
+
     global current_example_id, current_timestamp
-    
+
     debug(f"Entering example logging context for example {example_idx}")
     current_example_id = example_idx
     current_timestamp = timestamp
-    
-    handler = create_example_handler(timestamp, example_idx, path=LOGGING_STATE.path)
-    if handler:
+
+    if LOGGING_STATE.path:
+        handler = create_example_handler(
+            timestamp, example_idx, path=LOGGING_STATE.path
+        )
+    if handler and logger:
         logger.addHandler(handler)
     try:
         yield
     finally:
         current_example_id = None
         current_timestamp = None
-        if handler:
+        if handler and logger:
             logger.removeHandler(handler)
             handler.close()
-            debug(f"Closed example handler for example {example_idx}")
+            debug(f"Closed example handler for example {example_idx}")

judgeval/common/s3_storage.py

@@ -6,18 +6,19 @@ from datetime import datetime, UTC
 from botocore.exceptions import ClientError
 from judgeval.common.logger import warning, info
 
+
 class S3Storage:
     """Utility class for storing and retrieving trace data from S3."""
-    
+
     def __init__(
         self,
         bucket_name: str,
         aws_access_key_id: Optional[str] = None,
         aws_secret_access_key: Optional[str] = None,
-        region_name: Optional[str] = None
+        region_name: Optional[str] = None,
     ):
         """Initialize S3 storage with credentials and bucket name.
-        
+
         Args:
             bucket_name: Name of the S3 bucket to store traces in
             aws_access_key_id: AWS access key ID (optional, will use environment variables if not provided)
@@ -26,70 +27,76 @@ class S3Storage:
         """
         self.bucket_name = bucket_name
         self.s3_client = boto3.client(
-            's3',
-            aws_access_key_id=aws_access_key_id or os.getenv('AWS_ACCESS_KEY_ID'),
-            aws_secret_access_key=aws_secret_access_key or os.getenv('AWS_SECRET_ACCESS_KEY'),
-            region_name=region_name or os.getenv('AWS_REGION', 'us-west-1')
+            "s3",
+            aws_access_key_id=aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
+            aws_secret_access_key=aws_secret_access_key
+            or os.getenv("AWS_SECRET_ACCESS_KEY"),
+            region_name=region_name or os.getenv("AWS_REGION", "us-west-1"),
         )
-        
+
     def _ensure_bucket_exists(self):
         """Ensure the S3 bucket exists, creating it if necessary."""
         try:
             self.s3_client.head_bucket(Bucket=self.bucket_name)
         except ClientError as e:
-            error_code = e.response['Error']['Code']
-            if error_code == '404':
+            error_code = e.response["Error"]["Code"]
+            if error_code == "404":
                 # Bucket doesn't exist, create it
                 info(f"Bucket {self.bucket_name} doesn't exist, creating it ...")
                 try:
                     self.s3_client.create_bucket(
                         Bucket=self.bucket_name,
                         CreateBucketConfiguration={
-                            'LocationConstraint': self.s3_client.meta.region_name
-                        }               
+                            "LocationConstraint": self.s3_client.meta.region_name
+                        },
                     ) if self.s3_client.meta.region_name != "us-east-1" else self.s3_client.create_bucket(
                         Bucket=self.bucket_name
                     )
                     info(f"Created S3 bucket: {self.bucket_name}")
                 except ClientError as create_error:
-                    if create_error.response['Error']['Code'] == 'BucketAlreadyOwnedByYou':
+                    if (
+                        create_error.response["Error"]["Code"]
+                        == "BucketAlreadyOwnedByYou"
+                    ):
                         # Bucket was just created by another process
-                        warning(f"Bucket {self.bucket_name} was just created by another process")
+                        warning(
+                            f"Bucket {self.bucket_name} was just created by another process"
+                        )
                         pass
                     else:
                         raise create_error
             else:
                 # Some other error occurred
                 raise e
-        
+
     def save_trace(self, trace_data: dict, trace_id: str, project_name: str) -> str:
         """Save trace data to S3.
-        
+
         Args:
             trace_data: The trace data to save
             trace_id: Unique identifier for the trace
             project_name: Name of the project the trace belongs to
-            
+
         Returns:
             str: S3 key where the trace was saved
         """
         # Ensure bucket exists before saving
         self._ensure_bucket_exists()
-        
+
         # Create a timestamped key for the trace
-        timestamp = datetime.now(UTC).strftime('%Y%m%d_%H%M%S')
+        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
         s3_key = f"traces/{project_name}/{trace_id}_{timestamp}.json"
-        
+
         # Convert trace data to JSON string
         trace_json = json.dumps(trace_data)
-        
+
         # Upload to S3
         info(f"Uploading trace to S3 at key {s3_key}, in bucket {self.bucket_name} ...")
         self.s3_client.put_object(
             Bucket=self.bucket_name,
             Key=s3_key,
             Body=trace_json,
-            ContentType='application/json'
+            ContentType="application/json",
         )
-        
+
         return s3_key