deepeval 3.4.7__py3-none-any.whl → 3.4.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. deepeval/__init__.py +8 -7
  2. deepeval/_version.py +1 -1
  3. deepeval/cli/dotenv_handler.py +71 -0
  4. deepeval/cli/main.py +1021 -280
  5. deepeval/cli/utils.py +116 -2
  6. deepeval/confident/api.py +29 -14
  7. deepeval/config/__init__.py +0 -0
  8. deepeval/config/settings.py +565 -0
  9. deepeval/config/settings_manager.py +133 -0
  10. deepeval/config/utils.py +86 -0
  11. deepeval/dataset/__init__.py +1 -0
  12. deepeval/dataset/dataset.py +70 -10
  13. deepeval/dataset/test_run_tracer.py +82 -0
  14. deepeval/dataset/utils.py +23 -0
  15. deepeval/key_handler.py +64 -2
  16. deepeval/metrics/__init__.py +4 -1
  17. deepeval/metrics/answer_relevancy/template.py +7 -2
  18. deepeval/metrics/conversational_dag/__init__.py +7 -0
  19. deepeval/metrics/conversational_dag/conversational_dag.py +139 -0
  20. deepeval/metrics/conversational_dag/nodes.py +931 -0
  21. deepeval/metrics/conversational_dag/templates.py +117 -0
  22. deepeval/metrics/dag/dag.py +13 -4
  23. deepeval/metrics/dag/graph.py +47 -15
  24. deepeval/metrics/dag/utils.py +103 -38
  25. deepeval/metrics/faithfulness/template.py +11 -8
  26. deepeval/metrics/multimodal_metrics/multimodal_answer_relevancy/template.py +6 -4
  27. deepeval/metrics/multimodal_metrics/multimodal_faithfulness/template.py +6 -4
  28. deepeval/metrics/tool_correctness/tool_correctness.py +7 -3
  29. deepeval/models/llms/amazon_bedrock_model.py +24 -3
  30. deepeval/models/llms/openai_model.py +37 -41
  31. deepeval/models/retry_policy.py +280 -0
  32. deepeval/openai_agents/agent.py +4 -2
  33. deepeval/synthesizer/chunking/doc_chunker.py +87 -51
  34. deepeval/test_run/api.py +1 -0
  35. deepeval/tracing/otel/exporter.py +20 -8
  36. deepeval/tracing/otel/utils.py +57 -0
  37. deepeval/tracing/tracing.py +37 -16
  38. deepeval/tracing/utils.py +98 -1
  39. deepeval/utils.py +111 -70
  40. {deepeval-3.4.7.dist-info → deepeval-3.4.9.dist-info}/METADATA +3 -1
  41. {deepeval-3.4.7.dist-info → deepeval-3.4.9.dist-info}/RECORD +44 -34
  42. deepeval/env.py +0 -35
  43. {deepeval-3.4.7.dist-info → deepeval-3.4.9.dist-info}/LICENSE.md +0 -0
  44. {deepeval-3.4.7.dist-info → deepeval-3.4.9.dist-info}/WHEEL +0 -0
  45. {deepeval-3.4.7.dist-info → deepeval-3.4.9.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,565 @@
1
+ """
2
+ Central config for DeepEval.
3
+
4
+ - Autoloads dotenv files into os.environ without overwriting existing vars
5
+ (order: .env -> .env.{APP_ENV} -> .env.local).
6
+ - Defines the Pydantic `Settings` model and `get_settings()` singleton.
7
+ - Exposes an `edit()` context manager that diffs changes and persists them to
8
+ dotenv and the legacy JSON keystore (non-secret keys only), with validators and
9
+ type coercion.
10
+ """
11
+
12
+ import os
13
+ import re
14
+
15
+ from dotenv import dotenv_values
16
+ from pathlib import Path
17
+ from pydantic import AnyUrl, SecretStr, field_validator, confloat
18
+ from pydantic_settings import BaseSettings, SettingsConfigDict
19
+ from typing import Any, Dict, Optional, NamedTuple
20
+
21
+ from deepeval.config.utils import parse_bool
22
+
23
+
24
+ _SAVE_RE = re.compile(r"^(?P<scheme>dotenv)(?::(?P<path>.+))?$")
25
+
26
+
27
+ def _find_legacy_enum(env_key: str):
28
+ from deepeval.key_handler import (
29
+ ModelKeyValues,
30
+ EmbeddingKeyValues,
31
+ KeyValues,
32
+ )
33
+
34
+ enums = (ModelKeyValues, EmbeddingKeyValues, KeyValues)
35
+
36
+ for enum in enums:
37
+ try:
38
+ return getattr(enum, env_key)
39
+ except AttributeError:
40
+ pass
41
+
42
+ for enum in enums:
43
+ for member in enum:
44
+ if member.value == env_key:
45
+ return member
46
+ return None
47
+
48
+
49
+ def _is_secret_key(settings: "Settings", env_key: str) -> bool:
50
+ field = type(settings).model_fields.get(env_key)
51
+ if not field:
52
+ return False
53
+ if field.annotation is SecretStr:
54
+ return True
55
+ # Optional[SecretStr] etc.
56
+ from typing import get_origin, get_args, Union
57
+
58
+ origin = get_origin(field.annotation)
59
+ if origin is Union:
60
+ return any(arg is SecretStr for arg in get_args(field.annotation))
61
+ return False
62
+
63
+
64
+ def _read_env_file(path: Path) -> Dict[str, str]:
65
+ if not path.exists():
66
+ return {}
67
+ try:
68
+ # filter out None to avoid writing "None" later
69
+ return {
70
+ k: v for k, v in dotenv_values(str(path)).items() if v is not None
71
+ }
72
+ except Exception:
73
+ return {}
74
+
75
+
76
+ def _discover_app_env_from_files(env_dir: Path) -> Optional[str]:
77
+ # prefer base .env.local, then .env for APP_ENV discovery
78
+ for name in (".env.local", ".env"):
79
+ v = _read_env_file(env_dir / name).get("APP_ENV")
80
+ if v:
81
+ v = str(v).strip()
82
+ if v:
83
+ return v
84
+ return None
85
+
86
+
87
+ def autoload_dotenv() -> None:
88
+ """
89
+ Load env vars from .env files without overriding existing process env.
90
+
91
+ Precedence (lowest -> highest): .env -> .env.{APP_ENV} -> .env.local
92
+ Process env always wins over file values.
93
+
94
+ Controls:
95
+ - DEEPEVAL_DISABLE_DOTENV=1 -> skip
96
+ - ENV_DIR_PATH -> directory containing .env files (default: CWD)
97
+ """
98
+ if parse_bool(os.getenv("DEEPEVAL_DISABLE_DOTENV"), default=False):
99
+ return
100
+
101
+ raw_dir = os.getenv("ENV_DIR_PATH")
102
+ if raw_dir:
103
+ env_dir = Path(os.path.expanduser(os.path.expandvars(raw_dir)))
104
+ else:
105
+ env_dir = Path(os.getcwd())
106
+
107
+ # merge files in precedence order
108
+ base = _read_env_file(env_dir / ".env")
109
+ local = _read_env_file(env_dir / ".env.local")
110
+
111
+ # Pick APP_ENV (process -> .env.local -> .env -> default)
112
+ app_env = (
113
+ os.getenv("APP_ENV") or _discover_app_env_from_files(env_dir) or None
114
+ )
115
+ merged: Dict[str, str] = {}
116
+ env_specific: Dict[str, str] = {}
117
+ if app_env is not None:
118
+ app_env = app_env.strip()
119
+ if app_env:
120
+ env_specific = _read_env_file(env_dir / f".env.{app_env}")
121
+ merged.setdefault("APP_ENV", app_env)
122
+
123
+ merged.update(base)
124
+ merged.update(env_specific)
125
+ merged.update(local)
126
+
127
+ # Write only keys that aren’t already in process env
128
+ for k, v in merged.items():
129
+ if k not in os.environ:
130
+ os.environ[k] = v
131
+
132
+
133
+ class PersistResult(NamedTuple):
134
+ handled: bool
135
+ path: Optional[Path]
136
+ updated: Dict[str, Any] # typed, validated and changed
137
+
138
+
139
+ class Settings(BaseSettings):
140
+ model_config = SettingsConfigDict(
141
+ extra="ignore",
142
+ case_sensitive=True,
143
+ validate_assignment=True,
144
+ )
145
+
146
+ #
147
+ # General
148
+ #
149
+
150
+ APP_ENV: str = "dev"
151
+ LOG_LEVEL: str = "info"
152
+ PYTHONPATH: str = "."
153
+ CONFIDENT_REGION: Optional[str] = None
154
+ CONFIDENT_OPEN_BROWSER: Optional[bool] = True
155
+
156
+ #
157
+ # CLI
158
+ #
159
+
160
+ DEEPEVAL_DEFAULT_SAVE: Optional[str] = None
161
+ DEEPEVAL_DISABLE_DOTENV: Optional[bool] = None
162
+ ENV_DIR_PATH: Optional[Path] = (
163
+ None # where .env files live (CWD if not set)
164
+ )
165
+ DEEPEVAL_FILE_SYSTEM: Optional[str] = None
166
+ DEEPEVAL_IDENTIFIER: Optional[str] = None
167
+
168
+ #
169
+ # Storage & Output
170
+ #
171
+
172
+ # When set, DeepEval will export a timestamped JSON of the latest test run
173
+ # into this directory. The directory will be created on demand.
174
+ DEEPEVAL_RESULTS_FOLDER: Optional[Path] = None
175
+
176
+ #
177
+ # GPU and perf toggles
178
+ #
179
+
180
+ CUDA_LAUNCH_BLOCKING: Optional[bool] = None
181
+ CUDA_VISIBLE_DEVICES: Optional[str] = None
182
+ TOKENIZERS_PARALLELISM: Optional[bool] = None
183
+ TRANSFORMERS_NO_ADVISORY_WARNINGS: Optional[bool] = None
184
+
185
+ #
186
+ # Model Keys
187
+ #
188
+
189
+ API_KEY: Optional[SecretStr] = None
190
+ CONFIDENT_API_KEY: Optional[SecretStr] = None
191
+
192
+ # General
193
+ TEMPERATURE: Optional[confloat(ge=0, le=2)] = None
194
+
195
+ # Anthropic
196
+ ANTHROPIC_API_KEY: Optional[SecretStr] = None
197
+ # Azure Open AI
198
+ AZURE_OPENAI_API_KEY: Optional[SecretStr] = None
199
+ AZURE_OPENAI_ENDPOINT: Optional[AnyUrl] = None
200
+ OPENAI_API_VERSION: Optional[str] = None
201
+ AZURE_DEPLOYMENT_NAME: Optional[str] = None
202
+ AZURE_MODEL_NAME: Optional[str] = None
203
+ AZURE_MODEL_VERSION: Optional[str] = None
204
+ USE_AZURE_OPENAI: Optional[bool] = None
205
+ # DeepSeek
206
+ USE_DEEPSEEK_MODEL: Optional[bool] = None
207
+ DEEPSEEK_API_KEY: Optional[SecretStr] = None
208
+ DEEPSEEK_MODEL_NAME: Optional[str] = None
209
+ # Gemini
210
+ USE_GEMINI_MODEL: Optional[bool] = None
211
+ GOOGLE_API_KEY: Optional[SecretStr] = None
212
+ GEMINI_MODEL_NAME: Optional[str] = None
213
+ GOOGLE_GENAI_USE_VERTEXAI: Optional[bool] = None
214
+ GOOGLE_CLOUD_PROJECT: Optional[str] = None
215
+ GOOGLE_CLOUD_LOCATION: Optional[str] = None
216
+ # Grok
217
+ USE_GROK_MODEL: Optional[bool] = None
218
+ GROK_API_KEY: Optional[SecretStr] = None
219
+ GROK_MODEL_NAME: Optional[str] = None
220
+ # LiteLLM
221
+ USE_LITELLM: Optional[bool] = None
222
+ LITELLM_API_KEY: Optional[SecretStr] = None
223
+ LITELLM_MODEL_NAME: Optional[str] = None
224
+ LITELLM_API_BASE: Optional[AnyUrl] = None
225
+ LITELLM_PROXY_API_BASE: Optional[AnyUrl] = None
226
+ LITELLM_PROXY_API_KEY: Optional[SecretStr] = None
227
+ # LM Studio
228
+ LM_STUDIO_API_KEY: Optional[SecretStr] = None
229
+ LM_STUDIO_MODEL_NAME: Optional[str] = None
230
+ # Local Model
231
+ USE_LOCAL_MODEL: Optional[bool] = None
232
+ LOCAL_MODEL_API_KEY: Optional[SecretStr] = None
233
+ LOCAL_EMBEDDING_API_KEY: Optional[SecretStr] = None
234
+ LOCAL_MODEL_NAME: Optional[str] = None
235
+ LOCAL_MODEL_BASE_URL: Optional[AnyUrl] = None
236
+ LOCAL_MODEL_FORMAT: Optional[str] = None
237
+ # Moonshot
238
+ USE_MOONSHOT_MODEL: Optional[bool] = None
239
+ MOONSHOT_API_KEY: Optional[SecretStr] = None
240
+ MOONSHOT_MODEL_NAME: Optional[str] = None
241
+ # Ollama
242
+ OLLAMA_MODEL_NAME: Optional[str] = None
243
+ # OpenAI
244
+ USE_OPENAI_MODEL: Optional[bool] = None
245
+ OPENAI_API_KEY: Optional[SecretStr] = None
246
+ OPENAI_MODEL_NAME: Optional[str] = None
247
+ OPENAI_COST_PER_INPUT_TOKEN: Optional[float] = None
248
+ OPENAI_COST_PER_OUTPUT_TOKEN: Optional[float] = None
249
+ # Vertex AI
250
+ VERTEX_AI_MODEL_NAME: Optional[str] = None
251
+ # VLLM
252
+ VLLM_API_KEY: Optional[SecretStr] = None
253
+ VLLM_MODEL_NAME: Optional[str] = None
254
+
255
+ #
256
+ # Embedding Keys
257
+ #
258
+
259
+ # Azure OpenAI
260
+ USE_AZURE_OPENAI_EMBEDDING: Optional[bool] = None
261
+ AZURE_EMBEDDING_DEPLOYMENT_NAME: Optional[str] = None
262
+ # Local
263
+ USE_LOCAL_EMBEDDINGS: Optional[bool] = None
264
+ LOCAL_EMBEDDING_MODEL_NAME: Optional[str] = None
265
+ LOCAL_EMBEDDING_BASE_URL: Optional[AnyUrl] = None
266
+
267
+ #
268
+ # Telemetry and Debug
269
+ #
270
+ DEEPEVAL_TELEMETRY_OPT_OUT: Optional[bool] = None
271
+ DEEPEVAL_UPDATE_WARNING_OPT_IN: Optional[bool] = None
272
+ DEEPEVAL_GRPC_LOGGING: Optional[bool] = None
273
+ GRPC_VERBOSITY: Optional[str] = None
274
+ GRPC_TRACE: Optional[str] = None
275
+ ERROR_REPORTING: Optional[bool] = None
276
+ IGNORE_DEEPEVAL_ERRORS: Optional[bool] = None
277
+ SKIP_DEEPEVAL_MISSING_PARAMS: Optional[bool] = None
278
+ DEEPEVAL_VERBOSE_MODE: Optional[bool] = None
279
+ ENABLE_DEEPEVAL_CACHE: Optional[bool] = None
280
+ CONFIDENT_TRACE_FLUSH: Optional[bool] = None
281
+ CONFIDENT_TRACE_ENVIRONMENT: Optional[str] = "development"
282
+ CONFIDENT_TRACE_VERBOSE: Optional[bool] = True
283
+ CONFIDENT_SAMPLE_RATE: Optional[float] = 1.0
284
+ OTEL_EXPORTER_OTLP_ENDPOINT: Optional[AnyUrl] = None
285
+
286
+ ##############
287
+ # Validators #
288
+ ##############
289
+
290
+ @field_validator(
291
+ "CONFIDENT_OPEN_BROWSER",
292
+ "CONFIDENT_TRACE_FLUSH",
293
+ "CONFIDENT_TRACE_VERBOSE",
294
+ "USE_OPENAI_MODEL",
295
+ "USE_AZURE_OPENAI",
296
+ "USE_LOCAL_MODEL",
297
+ "USE_GEMINI_MODEL",
298
+ "GOOGLE_GENAI_USE_VERTEXAI",
299
+ "USE_MOONSHOT_MODEL",
300
+ "USE_GROK_MODEL",
301
+ "USE_DEEPSEEK_MODEL",
302
+ "USE_LITELLM",
303
+ "USE_AZURE_OPENAI_EMBEDDING",
304
+ "USE_LOCAL_EMBEDDINGS",
305
+ "DEEPEVAL_GRPC_LOGGING",
306
+ "DEEPEVAL_DISABLE_DOTENV",
307
+ "DEEPEVAL_TELEMETRY_OPT_OUT",
308
+ "DEEPEVAL_UPDATE_WARNING_OPT_IN",
309
+ "TOKENIZERS_PARALLELISM",
310
+ "TRANSFORMERS_NO_ADVISORY_WARNINGS",
311
+ "CUDA_LAUNCH_BLOCKING",
312
+ "ERROR_REPORTING",
313
+ "IGNORE_DEEPEVAL_ERRORS",
314
+ "SKIP_DEEPEVAL_MISSING_PARAMS",
315
+ "DEEPEVAL_VERBOSE_MODE",
316
+ "ENABLE_DEEPEVAL_CACHE",
317
+ mode="before",
318
+ )
319
+ @classmethod
320
+ def _coerce_yes_no(cls, v):
321
+ return None if v is None else parse_bool(v, default=False)
322
+
323
+ @field_validator("DEEPEVAL_RESULTS_FOLDER", "ENV_DIR_PATH", mode="before")
324
+ @classmethod
325
+ def _coerce_path(cls, v):
326
+ if v is None:
327
+ return None
328
+ s = str(v).strip()
329
+ if not s:
330
+ return None
331
+ # expand ~ and env vars;
332
+ # but don't resolve to avoid failing on non-existent paths
333
+ return Path(os.path.expandvars(os.path.expanduser(s)))
334
+
335
+ # Treat "", "none", "null" as None for numeric overrides
336
+ @field_validator(
337
+ "OPENAI_COST_PER_INPUT_TOKEN",
338
+ "OPENAI_COST_PER_OUTPUT_TOKEN",
339
+ "TEMPERATURE",
340
+ "CONFIDENT_SAMPLE_RATE",
341
+ mode="before",
342
+ )
343
+ @classmethod
344
+ def _none_or_float(cls, v):
345
+ if v is None:
346
+ return None
347
+ s = str(v).strip().lower()
348
+ if s in {"", "none", "null"}:
349
+ return None
350
+ return float(v)
351
+
352
+ @field_validator("CONFIDENT_SAMPLE_RATE")
353
+ @classmethod
354
+ def _validate_sample_rate(cls, v):
355
+ if v is None:
356
+ return None
357
+ if not (0.0 <= float(v) <= 1.0):
358
+ raise ValueError("CONFIDENT_SAMPLE_RATE must be between 0 and 1")
359
+ return float(v)
360
+
361
+ @field_validator("DEEPEVAL_DEFAULT_SAVE", mode="before")
362
+ @classmethod
363
+ def _validate_default_save(cls, v):
364
+ if v is None:
365
+ return None
366
+ s = str(v).strip()
367
+ if not s:
368
+ return None
369
+ m = _SAVE_RE.match(s)
370
+ if not m:
371
+ raise ValueError(
372
+ "DEEPEVAL_DEFAULT_SAVE must be 'dotenv' or 'dotenv:<path>'"
373
+ )
374
+ path = m.group("path")
375
+ if path is None:
376
+ return "dotenv"
377
+ path = os.path.expanduser(os.path.expandvars(path))
378
+ return f"dotenv:{path}"
379
+
380
+ @field_validator("DEEPEVAL_FILE_SYSTEM", mode="before")
381
+ @classmethod
382
+ def _normalize_fs(cls, v):
383
+ if v is None:
384
+ return None
385
+ s = str(v).strip().upper()
386
+
387
+ # adds friendly aliases
388
+ if s in {"READ_ONLY", "READ-ONLY", "READONLY", "RO"}:
389
+ return "READ_ONLY"
390
+ raise ValueError(
391
+ "DEEPEVAL_FILE_SYSTEM must be READ_ONLY (case-insensitive)."
392
+ )
393
+
394
+ @field_validator("CONFIDENT_REGION", mode="before")
395
+ @classmethod
396
+ def _normalize_upper(cls, v):
397
+ if v is None:
398
+ return None
399
+ s = str(v).strip()
400
+ if not s:
401
+ return None
402
+ return s.upper()
403
+
404
+ #######################
405
+ # Persistence support #
406
+ #######################
407
+ class _SettingsEditCtx:
408
+ def __init__(
409
+ self,
410
+ settings: "Settings",
411
+ save: Optional[str],
412
+ persist: Optional[bool],
413
+ ):
414
+ self._s = settings
415
+ self._save = save
416
+ self._persist = persist
417
+ self._before: Dict[str, Any] = {}
418
+ self.result: Optional[PersistResult] = None
419
+
420
+ @property
421
+ def s(self) -> "Settings":
422
+ return self._s
423
+
424
+ def __enter__(self) -> "Settings._SettingsEditCtx":
425
+ # snapshot current state
426
+ self._before = {
427
+ k: getattr(self._s, k) for k in type(self._s).model_fields
428
+ }
429
+ return self
430
+
431
+ def __exit__(self, exc_type, exc, tb):
432
+ if exc_type is not None:
433
+ return False # don’t persist on error
434
+
435
+ from deepeval.config.settings_manager import (
436
+ update_settings_and_persist,
437
+ _normalize_for_env,
438
+ )
439
+
440
+ # lazy import legacy JSON store deps
441
+ from deepeval.key_handler import KEY_FILE_HANDLER
442
+
443
+ # compute diff of changed fields
444
+ after = {k: getattr(self._s, k) for k in type(self._s).model_fields}
445
+
446
+ before_norm = {
447
+ k: _normalize_for_env(v) for k, v in self._before.items()
448
+ }
449
+ after_norm = {k: _normalize_for_env(v) for k, v in after.items()}
450
+
451
+ changed_keys = {
452
+ k for k in after_norm if after_norm[k] != before_norm.get(k)
453
+ }
454
+ if not changed_keys:
455
+ self.result = PersistResult(False, None, {})
456
+ return False
457
+
458
+ updates = {k: after[k] for k in changed_keys}
459
+
460
+ #
461
+ # .deepeval JSON support
462
+ #
463
+
464
+ if self._persist is not False:
465
+ for k in changed_keys:
466
+ legacy_member = _find_legacy_enum(k)
467
+ if legacy_member is None:
468
+ continue # skip if not a defined as legacy field
469
+
470
+ val = updates[k]
471
+ # Remove from JSON if unset
472
+ if val is None:
473
+ KEY_FILE_HANDLER.remove_key(legacy_member)
474
+ continue
475
+
476
+ # Never store secrets in the JSON keystore
477
+ if _is_secret_key(self._s, k):
478
+ continue
479
+
480
+ # For booleans, the legacy store expects "YES"/"NO"
481
+ if isinstance(val, bool):
482
+ KEY_FILE_HANDLER.write_key(
483
+ legacy_member, "YES" if val else "NO"
484
+ )
485
+ else:
486
+ # store as string
487
+ KEY_FILE_HANDLER.write_key(legacy_member, str(val))
488
+
489
+ #
490
+ # dotenv store
491
+ #
492
+
493
+ # defer import to avoid cyclics
494
+ handled, path = update_settings_and_persist(
495
+ updates,
496
+ save=self._save,
497
+ persist_dotenv=(False if self._persist is False else True),
498
+ )
499
+ self.result = PersistResult(handled, path, updates)
500
+ return False
501
+
502
+ def switch_model_provider(self, target) -> None:
503
+ """
504
+ Flip all USE_* toggles so that the one matching the target is True and the rest are False.
505
+ Also, mirror this change into the legacy JSON keystore as "YES"/"NO".
506
+
507
+ `target` may be an Enum with `.value`, such as ModelKeyValues.USE_OPENAI_MODEL
508
+ or a plain string like "USE_OPENAI_MODEL".
509
+ """
510
+ from deepeval.key_handler import KEY_FILE_HANDLER
511
+
512
+ # Target key is the env style string, such as "USE_OPENAI_MODEL"
513
+ target_key = getattr(target, "value", str(target))
514
+
515
+ use_fields = [
516
+ k for k in type(self._s).model_fields if k.startswith("USE_")
517
+ ]
518
+ if target_key not in use_fields:
519
+ raise ValueError(
520
+ f"{target_key} is not a recognized USE_* field"
521
+ )
522
+
523
+ for k in use_fields:
524
+ on = k == target_key
525
+ # dotenv persistence will serialize to "1"/"0"
526
+ setattr(self._s, k, on)
527
+ if self._persist is not False:
528
+ # legacy json persistence will serialize to "YES"/"NO"
529
+ legacy_member = _find_legacy_enum(k)
530
+ if legacy_member is not None:
531
+ KEY_FILE_HANDLER.write_key(
532
+ legacy_member, "YES" if on else "NO"
533
+ )
534
+
535
+ def edit(
536
+ self, *, save: Optional[str] = None, persist: Optional[bool] = None
537
+ ):
538
+ """Context manager for atomic, persisted updates.
539
+
540
+ Args:
541
+ save: 'dotenv[:path]' to explicitly write to a dotenv file.
542
+ None (default) respects DEEPEVAL_DEFAULT_SAVE if set.
543
+ persist: If False, do not write (dotenv, JSON), update runtime only.
544
+ If True or None, normal persistence rules apply.
545
+ """
546
+ return self._SettingsEditCtx(self, save, persist)
547
+
548
+ def set_model_provider(self, target, *, save: Optional[str] = None):
549
+ """
550
+ Convenience wrapper to switch providers outside of an existing edit() block.
551
+ Returns the PersistResult.
552
+ """
553
+ with self.edit(save=save) as ctx:
554
+ ctx.switch_model_provider(target)
555
+ return ctx.result
556
+
557
+
558
+ _settings_singleton: Optional[Settings] = None
559
+
560
+
561
+ def get_settings() -> Settings:
562
+ global _settings_singleton
563
+ if _settings_singleton is None:
564
+ _settings_singleton = Settings()
565
+ return _settings_singleton
@@ -0,0 +1,133 @@
1
+ """
2
+ Applies CLI driven updates to the live Settings and optionally persists them to a
3
+ dotenv file. Also syncs os.environ, handles unsets, and warns on unknown fields.
4
+ Primary entrypoint: update_settings_and_persist.
5
+ """
6
+
7
+ import logging
8
+ import os
9
+
10
+ from difflib import get_close_matches
11
+ from pathlib import Path
12
+ from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Union
13
+ from enum import Enum
14
+
15
+ from pydantic import SecretStr
16
+ from deepeval.config.settings import get_settings, _SAVE_RE
17
+ from deepeval.cli.dotenv_handler import DotenvHandler
18
+ from deepeval.utils import bool_to_env_str
19
+
20
+ logger = logging.getLogger(__name__)
21
+ StrOrEnum = Union[str, Enum]
22
+
23
+
24
+ def _env_key(k: StrOrEnum) -> str:
25
+ return k.value if isinstance(k, Enum) else str(k)
26
+
27
+
28
+ def _normalize_for_env(val: Any) -> Optional[str]:
29
+ """Convert typed value to string for dotenv + os.environ; None -> unset."""
30
+ if val is None:
31
+ return None
32
+ if isinstance(val, SecretStr):
33
+ return val.get_secret_value()
34
+ if isinstance(val, bool):
35
+ return bool_to_env_str(val)
36
+ return str(val)
37
+
38
+
39
+ def _resolve_save_path(save_opt: Optional[str]) -> Tuple[bool, Optional[Path]]:
40
+ """
41
+ Returns (ok, path).
42
+ - ok=False -> invalid save option format
43
+ - ok=True, path=None -> no persistence requested
44
+ - ok=True, path=Path -> persist to that file
45
+ """
46
+ raw = (
47
+ save_opt if save_opt is not None else os.getenv("DEEPEVAL_DEFAULT_SAVE")
48
+ )
49
+ if not raw:
50
+ return True, None
51
+ m = _SAVE_RE.match(raw.strip())
52
+ if not m:
53
+ return False, None
54
+ path = m.group("path") or ".env.local"
55
+ path = Path(os.path.expanduser(os.path.expandvars(path)))
56
+ return True, path
57
+
58
+
59
+ def update_settings_and_persist(
60
+ updates: Mapping[StrOrEnum, Any],
61
+ *,
62
+ save: Optional[str] = None,
63
+ unset: Iterable[StrOrEnum] = (),
64
+ persist_dotenv: bool = True,
65
+ ) -> Tuple[bool, Optional[Path]]:
66
+ """
67
+ Write and update:
68
+ - validate + assign into live Settings()
69
+ - update os.environ
70
+ - persist to dotenv, if `save` or DEEPEVAL_DEFAULT_SAVE provided
71
+ - unset keys where value is None or explicitly in `unset`
72
+ Returns (handled, path_to_dotenv_if_any).
73
+ """
74
+ settings = get_settings()
75
+
76
+ # validate + assign into settings.
77
+ # validation is handled in Settings as long as validate_assignment=True
78
+ typed: Dict[str, Any] = {}
79
+ for key, value in updates.items():
80
+ k = _env_key(key)
81
+ if k not in type(settings).model_fields:
82
+ suggestion = get_close_matches(
83
+ k, type(settings).model_fields.keys(), n=1
84
+ )
85
+ if suggestion:
86
+ logger.warning(
87
+ "Unknown settings field '%s'; did you mean '%s'? Ignoring.",
88
+ k,
89
+ suggestion[0],
90
+ stacklevel=2,
91
+ )
92
+ else:
93
+ logger.warning(
94
+ "Unknown settings field '%s'; ignoring.", k, stacklevel=2
95
+ )
96
+ continue
97
+
98
+ setattr(settings, k, value)
99
+ # coercion is handled in Settings
100
+ typed[k] = getattr(settings, k)
101
+
102
+ # build env maps
103
+ to_write: Dict[str, str] = {}
104
+ to_unset: set[str] = set(_env_key(k) for k in unset)
105
+
106
+ for k, v in typed.items():
107
+ env_val = _normalize_for_env(v)
108
+ if env_val is None:
109
+ to_unset.add(k)
110
+ else:
111
+ to_write[k] = env_val
112
+
113
+ # update process env so that it is effective immediately
114
+ for k, v in to_write.items():
115
+ os.environ[k] = v
116
+ for k in to_unset:
117
+ os.environ.pop(k, None)
118
+
119
+ if not persist_dotenv:
120
+ return True, None
121
+
122
+ # persist to dotenv if save is ok
123
+ ok, path = _resolve_save_path(save)
124
+ if not ok:
125
+ return False, None # unsupported --save
126
+ if path:
127
+ h = DotenvHandler(path)
128
+ if to_write:
129
+ h.upsert(to_write)
130
+ if to_unset:
131
+ h.unset(to_unset)
132
+ return True, path
133
+ return True, None