deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.6.6"
+__version__: str = "3.6.8"

deepeval/benchmarks/equity_med_qa/equity_med_qa.py

@@ -121,6 +121,7 @@ class EquityMedQA(DeepEvalBaseBenchmark):
         score = metric.measure(
             LLMTestCase(input=golden.input, actual_output=prediction),
             _show_indicator=False,
+            _log_metric_to_confident=False,
         )
         flipped_score = (
             1 - metric.score if metric.score in [0, 1] else metric.score

deepeval/cli/main.py

@@ -328,6 +328,31 @@ def set_debug(
         "--trace-flush/--no-trace-flush",
         help="Enable / disable  CONFIDENT_TRACE_FLUSH.",
     ),
+    trace_sample_rate: Optional[float] = typer.Option(
+        None,
+        "--trace-sample-rate",
+        help="Set CONFIDENT_TRACE_SAMPLE_RATE.",
+    ),
+    metric_logging_verbose: Optional[bool] = typer.Option(
+        None,
+        "--metric-logging-verbose/--no-metric-logging-verbose",
+        help="Enable / disable CONFIDENT_METRIC_LOGGING_VERBOSE.",
+    ),
+    metric_logging_flush: Optional[bool] = typer.Option(
+        None,
+        "--metric-logging-flush/--no-metric-logging-flush",
+        help="Enable / disable CONFIDENT_METRIC_LOGGING_FLUSH.",
+    ),
+    metric_logging_sample_rate: Optional[float] = typer.Option(
+        None,
+        "--metric-logging-sample-rate",
+        help="Set CONFIDENT_METRIC_LOGGING_SAMPLE_RATE.",
+    ),
+    metric_logging_enabled: Optional[bool] = typer.Option(
+        None,
+        "--metric-logging-enabled/--no-metric-logging-enabled",
+        help="Enable / disable CONFIDENT_METRIC_LOGGING_ENABLED.",
+    ),
     # Advanced / potentially surprising
     error_reporting: Optional[bool] = typer.Option(
         None,
@@ -387,6 +412,20 @@ def set_debug(
             settings.CONFIDENT_TRACE_ENVIRONMENT = trace_env
         if trace_flush is not None:
             settings.CONFIDENT_TRACE_FLUSH = trace_flush
+        if trace_sample_rate is not None:
+            settings.CONFIDENT_TRACE_SAMPLE_RATE = trace_sample_rate
+
+        # Confident metrics
+        if metric_logging_verbose is not None:
+            settings.CONFIDENT_METRIC_LOGGING_VERBOSE = metric_logging_verbose
+        if metric_logging_flush is not None:
+            settings.CONFIDENT_METRIC_LOGGING_FLUSH = metric_logging_flush
+        if metric_logging_sample_rate is not None:
+            settings.CONFIDENT_METRIC_LOGGING_SAMPLE_RATE = (
+                metric_logging_sample_rate
+            )
+        if metric_logging_enabled is not None:
+            settings.CONFIDENT_METRIC_LOGGING_ENABLED = metric_logging_enabled
 
         # Advanced
         if error_reporting is not None:
@@ -438,6 +477,8 @@ def unset_debug(
         settings.LOG_LEVEL = "info"
         settings.CONFIDENT_TRACE_ENVIRONMENT = "development"
         settings.CONFIDENT_TRACE_VERBOSE = True
+        settings.CONFIDENT_METRIC_LOGGING_VERBOSE = True
+        settings.CONFIDENT_METRIC_LOGGING_ENABLED = True
 
         # Clear optional toggles/overrides
         settings.DEEPEVAL_VERBOSE_MODE = None
@@ -449,6 +490,7 @@ def unset_debug(
         settings.GRPC_TRACE = None
 
         settings.CONFIDENT_TRACE_FLUSH = None
+        settings.CONFIDENT_METRIC_LOGGING_FLUSH = None
 
         settings.ERROR_REPORTING = None
         settings.IGNORE_DEEPEVAL_ERRORS = None

deepeval/confident/api.py

@@ -87,6 +87,7 @@ class Endpoints(Enum):
     DATASET_ALIAS_QUEUE_ENDPOINT = "/v1/datasets/:alias/queue"
 
     TEST_RUN_ENDPOINT = "/v1/test-run"
+    METRIC_DATA_ENDPOINT = "/v1/metric-data"
     TRACES_ENDPOINT = "/v1/traces"
     ANNOTATIONS_ENDPOINT = "/v1/annotations"
     PROMPTS_VERSION_ID_ENDPOINT = "/v1/prompts/:alias/versions/:versionId"

deepeval/config/settings.py

@@ -337,10 +337,17 @@ class Settings(BaseSettings):
     SKIP_DEEPEVAL_MISSING_PARAMS: Optional[bool] = None
     DEEPEVAL_VERBOSE_MODE: Optional[bool] = None
     ENABLE_DEEPEVAL_CACHE: Optional[bool] = None
+
     CONFIDENT_TRACE_FLUSH: Optional[bool] = None
     CONFIDENT_TRACE_ENVIRONMENT: Optional[str] = "development"
     CONFIDENT_TRACE_VERBOSE: Optional[bool] = True
-    CONFIDENT_SAMPLE_RATE: Optional[float] = 1.0
+    CONFIDENT_TRACE_SAMPLE_RATE: Optional[float] = 1.0
+
+    CONFIDENT_METRIC_LOGGING_FLUSH: Optional[bool] = None
+    CONFIDENT_METRIC_LOGGING_VERBOSE: Optional[bool] = True
+    CONFIDENT_METRIC_LOGGING_SAMPLE_RATE: Optional[float] = 1.0
+    CONFIDENT_METRIC_LOGGING_ENABLED: Optional[bool] = True
+
     OTEL_EXPORTER_OTLP_ENDPOINT: Optional[AnyUrl] = None
 
     #
@@ -355,6 +362,12 @@ class Settings(BaseSettings):
         None  # per-attempt timeout. Set 0/None to disable
     )
 
+    #
+    # Async Document Pipelines
+    #
+
+    DEEPEVAL_MAX_CONCURRENT_DOC_PROCESSING: conint(ge=1) = 2
+
     #
     # Async Task Configuration
     #
@@ -484,7 +497,8 @@ class Settings(BaseSettings):
         "OPENAI_COST_PER_INPUT_TOKEN",
         "OPENAI_COST_PER_OUTPUT_TOKEN",
         "TEMPERATURE",
-        "CONFIDENT_SAMPLE_RATE",
+        "CONFIDENT_TRACE_SAMPLE_RATE",
+        "CONFIDENT_METRIC_LOGGING_SAMPLE_RATE",
         mode="before",
     )
     @classmethod
@@ -496,13 +510,17 @@ class Settings(BaseSettings):
             return None
         return float(v)
 
-    @field_validator("CONFIDENT_SAMPLE_RATE")
+    @field_validator(
+        "CONFIDENT_TRACE_SAMPLE_RATE", "CONFIDENT_METRIC_LOGGING_SAMPLE_RATE"
+    )
     @classmethod
     def _validate_sample_rate(cls, v):
         if v is None:
             return None
         if not (0.0 <= float(v) <= 1.0):
-            raise ValueError("CONFIDENT_SAMPLE_RATE must be between 0 and 1")
+            raise ValueError(
+                "CONFIDENT_TRACE_SAMPLE_RATE or CONFIDENT_METRIC_LOGGING_SAMPLE_RATE must be between 0 and 1"
+            )
         return float(v)
 
     @field_validator("DEEPEVAL_DEFAULT_SAVE", mode="before")

deepeval/constants.py

@@ -9,9 +9,16 @@ LOGIN_PROMPT = "\n✨👀 Looking for a place for your LLM test data to live
 
 CONFIDENT_TRACE_VERBOSE = "CONFIDENT_TRACE_VERBOSE"
 CONFIDENT_TRACE_FLUSH = "CONFIDENT_TRACE_FLUSH"
-CONFIDENT_SAMPLE_RATE = "CONFIDENT_SAMPLE_RATE"
+CONFIDENT_TRACE_SAMPLE_RATE = "CONFIDENT_TRACE_SAMPLE_RATE"
 CONFIDENT_TRACE_ENVIRONMENT = "CONFIDENT_TRACE_ENVIRONMENT"
 CONFIDENT_TRACING_ENABLED = "CONFIDENT_TRACING_ENABLED"
+
+CONFIDENT_METRIC_LOGGING_VERBOSE = "CONFIDENT_METRIC_LOGGING_VERBOSE"
+CONFIDENT_METRIC_LOGGING_FLUSH = "CONFIDENT_METRIC_LOGGING_FLUSH"
+CONFIDENT_METRIC_LOGGING_SAMPLE_RATE = "CONFIDENT_METRIC_LOGGING_SAMPLE_RATE"
+CONFIDENT_METRIC_LOGGING_ENABLED = "CONFIDENT_METRIC_LOGGING_ENABLED"
+
+
 CONFIDENT_OPEN_BROWSER = "CONFIDENT_OPEN_BROWSER"
 CONFIDENT_TEST_CASE_BATCH_SIZE = "CONFIDENT_TEST_CASE_BATCH_SIZE"
 

deepeval/dataset/dataset.py

@@ -49,7 +49,7 @@ from deepeval.utils import (
 from deepeval.test_run import (
     global_test_run_manager,
 )
-from deepeval.openai.utils import openai_test_case_pairs
+
 from deepeval.tracing import trace_manager
 from deepeval.tracing.tracing import EVAL_DUMMY_SPAN_NAME
 
@@ -1248,16 +1248,7 @@ class EvaluationDataset:
                         display_config.file_output_dir,
                     )
 
-            # update hyperparameters
-            test_run = global_test_run_manager.get_test_run()
-            if len(openai_test_case_pairs) > 0:
-                raw_hyperparameters = openai_test_case_pairs[-1].hyperparameters
-                test_run.hyperparameters = process_hyperparameters(
-                    raw_hyperparameters
-                )
-
-            # clean up
-            openai_test_case_pairs.clear()
+            # save test run
             global_test_run_manager.save_test_run(TEMP_FILE_PATH)
 
             # sandwich end trace for OTEL

deepeval/dataset/utils.py

@@ -120,7 +120,7 @@ def format_turns(turns: List[Turn]) -> str:
         }
         res.append(cur_turn)
     try:
-        return json.dumps(res)
+        return json.dumps(res, ensure_ascii=False)
     except Exception as e:
         raise ValueError(f"Error serializing turns: {e}")
 

deepeval/errors.py

@@ -1,6 +1,24 @@
-class MissingTestCaseParamsError(Exception):
+class DeepEvalError(Exception):
+    """Base class for framework-originated errors.
+    If raised and not handled, it will abort the current operation.
+    We may also stringify instances of this class and attach them to traces or spans to surface
+    non-fatal diagnostics while allowing the run to continue.
+    """
+
+
+class UserAppError(Exception):
+    """Represents exceptions thrown by user LLM apps/tools.
+    We record these on traces or spans and keep the overall evaluation run alive.
+    """
+
+
+class MissingTestCaseParamsError(DeepEvalError):
+    """Required test case fields are missing."""
+
     pass
 
 
-class MismatchedTestCaseInputsError(Exception):
+class MismatchedTestCaseInputsError(DeepEvalError):
+    """Inputs provided to a metric or test case are inconsistent or invalid."""
+
     pass

deepeval/evaluate/evaluate.py

@@ -28,7 +28,10 @@ from deepeval.evaluate.utils import (
 from deepeval.dataset import Golden
 from deepeval.prompt import Prompt
 from deepeval.test_case.utils import check_valid_test_cases_type
-from deepeval.test_run.hyperparameters import process_hyperparameters
+from deepeval.test_run.hyperparameters import (
+    process_hyperparameters,
+    process_prompts,
+)
 from deepeval.test_run.test_run import TEMP_FILE_PATH
 from deepeval.utils import (
     get_or_create_event_loop,
@@ -267,6 +270,7 @@ def evaluate(
 
         test_run = global_test_run_manager.get_test_run()
         test_run.hyperparameters = process_hyperparameters(hyperparameters)
+        test_run.prompts = process_prompts(hyperparameters)
         global_test_run_manager.save_test_run(TEMP_FILE_PATH)
         res = global_test_run_manager.wrap_up_test_run(
             run_duration, display_table=False