deepeval 3.4.7__py3-none-any.whl → 3.4.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deepeval/__init__.py +8 -7
- deepeval/_version.py +1 -1
- deepeval/cli/dotenv_handler.py +71 -0
- deepeval/cli/main.py +1021 -280
- deepeval/cli/utils.py +116 -2
- deepeval/confident/api.py +29 -14
- deepeval/config/__init__.py +0 -0
- deepeval/config/settings.py +565 -0
- deepeval/config/settings_manager.py +133 -0
- deepeval/config/utils.py +86 -0
- deepeval/dataset/__init__.py +1 -0
- deepeval/dataset/dataset.py +70 -10
- deepeval/dataset/test_run_tracer.py +82 -0
- deepeval/dataset/utils.py +23 -0
- deepeval/key_handler.py +64 -2
- deepeval/metrics/__init__.py +4 -1
- deepeval/metrics/answer_relevancy/template.py +7 -2
- deepeval/metrics/conversational_dag/__init__.py +7 -0
- deepeval/metrics/conversational_dag/conversational_dag.py +139 -0
- deepeval/metrics/conversational_dag/nodes.py +931 -0
- deepeval/metrics/conversational_dag/templates.py +117 -0
- deepeval/metrics/dag/dag.py +13 -4
- deepeval/metrics/dag/graph.py +47 -15
- deepeval/metrics/dag/utils.py +103 -38
- deepeval/metrics/faithfulness/template.py +11 -8
- deepeval/metrics/multimodal_metrics/multimodal_answer_relevancy/template.py +6 -4
- deepeval/metrics/multimodal_metrics/multimodal_faithfulness/template.py +6 -4
- deepeval/metrics/tool_correctness/tool_correctness.py +7 -3
- deepeval/models/llms/amazon_bedrock_model.py +24 -3
- deepeval/models/llms/openai_model.py +37 -41
- deepeval/models/retry_policy.py +280 -0
- deepeval/openai_agents/agent.py +4 -2
- deepeval/synthesizer/chunking/doc_chunker.py +87 -51
- deepeval/test_run/api.py +1 -0
- deepeval/tracing/otel/exporter.py +20 -8
- deepeval/tracing/otel/utils.py +57 -0
- deepeval/tracing/tracing.py +37 -16
- deepeval/tracing/utils.py +98 -1
- deepeval/utils.py +111 -70
- {deepeval-3.4.7.dist-info → deepeval-3.4.9.dist-info}/METADATA +3 -1
- {deepeval-3.4.7.dist-info → deepeval-3.4.9.dist-info}/RECORD +44 -34
- deepeval/env.py +0 -35
- {deepeval-3.4.7.dist-info → deepeval-3.4.9.dist-info}/LICENSE.md +0 -0
- {deepeval-3.4.7.dist-info → deepeval-3.4.9.dist-info}/WHEEL +0 -0
- {deepeval-3.4.7.dist-info → deepeval-3.4.9.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,565 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Central config for DeepEval.
|
|
3
|
+
|
|
4
|
+
- Autoloads dotenv files into os.environ without overwriting existing vars
|
|
5
|
+
(order: .env -> .env.{APP_ENV} -> .env.local).
|
|
6
|
+
- Defines the Pydantic `Settings` model and `get_settings()` singleton.
|
|
7
|
+
- Exposes an `edit()` context manager that diffs changes and persists them to
|
|
8
|
+
dotenv and the legacy JSON keystore (non-secret keys only), with validators and
|
|
9
|
+
type coercion.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
|
|
15
|
+
from dotenv import dotenv_values
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from pydantic import AnyUrl, SecretStr, field_validator, confloat
|
|
18
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
19
|
+
from typing import Any, Dict, Optional, NamedTuple
|
|
20
|
+
|
|
21
|
+
from deepeval.config.utils import parse_bool
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
_SAVE_RE = re.compile(r"^(?P<scheme>dotenv)(?::(?P<path>.+))?$")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _find_legacy_enum(env_key: str):
|
|
28
|
+
from deepeval.key_handler import (
|
|
29
|
+
ModelKeyValues,
|
|
30
|
+
EmbeddingKeyValues,
|
|
31
|
+
KeyValues,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
enums = (ModelKeyValues, EmbeddingKeyValues, KeyValues)
|
|
35
|
+
|
|
36
|
+
for enum in enums:
|
|
37
|
+
try:
|
|
38
|
+
return getattr(enum, env_key)
|
|
39
|
+
except AttributeError:
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
for enum in enums:
|
|
43
|
+
for member in enum:
|
|
44
|
+
if member.value == env_key:
|
|
45
|
+
return member
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _is_secret_key(settings: "Settings", env_key: str) -> bool:
|
|
50
|
+
field = type(settings).model_fields.get(env_key)
|
|
51
|
+
if not field:
|
|
52
|
+
return False
|
|
53
|
+
if field.annotation is SecretStr:
|
|
54
|
+
return True
|
|
55
|
+
# Optional[SecretStr] etc.
|
|
56
|
+
from typing import get_origin, get_args, Union
|
|
57
|
+
|
|
58
|
+
origin = get_origin(field.annotation)
|
|
59
|
+
if origin is Union:
|
|
60
|
+
return any(arg is SecretStr for arg in get_args(field.annotation))
|
|
61
|
+
return False
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _read_env_file(path: Path) -> Dict[str, str]:
|
|
65
|
+
if not path.exists():
|
|
66
|
+
return {}
|
|
67
|
+
try:
|
|
68
|
+
# filter out None to avoid writing "None" later
|
|
69
|
+
return {
|
|
70
|
+
k: v for k, v in dotenv_values(str(path)).items() if v is not None
|
|
71
|
+
}
|
|
72
|
+
except Exception:
|
|
73
|
+
return {}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _discover_app_env_from_files(env_dir: Path) -> Optional[str]:
|
|
77
|
+
# prefer base .env.local, then .env for APP_ENV discovery
|
|
78
|
+
for name in (".env.local", ".env"):
|
|
79
|
+
v = _read_env_file(env_dir / name).get("APP_ENV")
|
|
80
|
+
if v:
|
|
81
|
+
v = str(v).strip()
|
|
82
|
+
if v:
|
|
83
|
+
return v
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def autoload_dotenv() -> None:
|
|
88
|
+
"""
|
|
89
|
+
Load env vars from .env files without overriding existing process env.
|
|
90
|
+
|
|
91
|
+
Precedence (lowest -> highest): .env -> .env.{APP_ENV} -> .env.local
|
|
92
|
+
Process env always wins over file values.
|
|
93
|
+
|
|
94
|
+
Controls:
|
|
95
|
+
- DEEPEVAL_DISABLE_DOTENV=1 -> skip
|
|
96
|
+
- ENV_DIR_PATH -> directory containing .env files (default: CWD)
|
|
97
|
+
"""
|
|
98
|
+
if parse_bool(os.getenv("DEEPEVAL_DISABLE_DOTENV"), default=False):
|
|
99
|
+
return
|
|
100
|
+
|
|
101
|
+
raw_dir = os.getenv("ENV_DIR_PATH")
|
|
102
|
+
if raw_dir:
|
|
103
|
+
env_dir = Path(os.path.expanduser(os.path.expandvars(raw_dir)))
|
|
104
|
+
else:
|
|
105
|
+
env_dir = Path(os.getcwd())
|
|
106
|
+
|
|
107
|
+
# merge files in precedence order
|
|
108
|
+
base = _read_env_file(env_dir / ".env")
|
|
109
|
+
local = _read_env_file(env_dir / ".env.local")
|
|
110
|
+
|
|
111
|
+
# Pick APP_ENV (process -> .env.local -> .env -> default)
|
|
112
|
+
app_env = (
|
|
113
|
+
os.getenv("APP_ENV") or _discover_app_env_from_files(env_dir) or None
|
|
114
|
+
)
|
|
115
|
+
merged: Dict[str, str] = {}
|
|
116
|
+
env_specific: Dict[str, str] = {}
|
|
117
|
+
if app_env is not None:
|
|
118
|
+
app_env = app_env.strip()
|
|
119
|
+
if app_env:
|
|
120
|
+
env_specific = _read_env_file(env_dir / f".env.{app_env}")
|
|
121
|
+
merged.setdefault("APP_ENV", app_env)
|
|
122
|
+
|
|
123
|
+
merged.update(base)
|
|
124
|
+
merged.update(env_specific)
|
|
125
|
+
merged.update(local)
|
|
126
|
+
|
|
127
|
+
# Write only keys that aren’t already in process env
|
|
128
|
+
for k, v in merged.items():
|
|
129
|
+
if k not in os.environ:
|
|
130
|
+
os.environ[k] = v
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class PersistResult(NamedTuple):
|
|
134
|
+
handled: bool
|
|
135
|
+
path: Optional[Path]
|
|
136
|
+
updated: Dict[str, Any] # typed, validated and changed
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class Settings(BaseSettings):
|
|
140
|
+
model_config = SettingsConfigDict(
|
|
141
|
+
extra="ignore",
|
|
142
|
+
case_sensitive=True,
|
|
143
|
+
validate_assignment=True,
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
#
|
|
147
|
+
# General
|
|
148
|
+
#
|
|
149
|
+
|
|
150
|
+
APP_ENV: str = "dev"
|
|
151
|
+
LOG_LEVEL: str = "info"
|
|
152
|
+
PYTHONPATH: str = "."
|
|
153
|
+
CONFIDENT_REGION: Optional[str] = None
|
|
154
|
+
CONFIDENT_OPEN_BROWSER: Optional[bool] = True
|
|
155
|
+
|
|
156
|
+
#
|
|
157
|
+
# CLI
|
|
158
|
+
#
|
|
159
|
+
|
|
160
|
+
DEEPEVAL_DEFAULT_SAVE: Optional[str] = None
|
|
161
|
+
DEEPEVAL_DISABLE_DOTENV: Optional[bool] = None
|
|
162
|
+
ENV_DIR_PATH: Optional[Path] = (
|
|
163
|
+
None # where .env files live (CWD if not set)
|
|
164
|
+
)
|
|
165
|
+
DEEPEVAL_FILE_SYSTEM: Optional[str] = None
|
|
166
|
+
DEEPEVAL_IDENTIFIER: Optional[str] = None
|
|
167
|
+
|
|
168
|
+
#
|
|
169
|
+
# Storage & Output
|
|
170
|
+
#
|
|
171
|
+
|
|
172
|
+
# When set, DeepEval will export a timestamped JSON of the latest test run
|
|
173
|
+
# into this directory. The directory will be created on demand.
|
|
174
|
+
DEEPEVAL_RESULTS_FOLDER: Optional[Path] = None
|
|
175
|
+
|
|
176
|
+
#
|
|
177
|
+
# GPU and perf toggles
|
|
178
|
+
#
|
|
179
|
+
|
|
180
|
+
CUDA_LAUNCH_BLOCKING: Optional[bool] = None
|
|
181
|
+
CUDA_VISIBLE_DEVICES: Optional[str] = None
|
|
182
|
+
TOKENIZERS_PARALLELISM: Optional[bool] = None
|
|
183
|
+
TRANSFORMERS_NO_ADVISORY_WARNINGS: Optional[bool] = None
|
|
184
|
+
|
|
185
|
+
#
|
|
186
|
+
# Model Keys
|
|
187
|
+
#
|
|
188
|
+
|
|
189
|
+
API_KEY: Optional[SecretStr] = None
|
|
190
|
+
CONFIDENT_API_KEY: Optional[SecretStr] = None
|
|
191
|
+
|
|
192
|
+
# General
|
|
193
|
+
TEMPERATURE: Optional[confloat(ge=0, le=2)] = None
|
|
194
|
+
|
|
195
|
+
# Anthropic
|
|
196
|
+
ANTHROPIC_API_KEY: Optional[SecretStr] = None
|
|
197
|
+
# Azure Open AI
|
|
198
|
+
AZURE_OPENAI_API_KEY: Optional[SecretStr] = None
|
|
199
|
+
AZURE_OPENAI_ENDPOINT: Optional[AnyUrl] = None
|
|
200
|
+
OPENAI_API_VERSION: Optional[str] = None
|
|
201
|
+
AZURE_DEPLOYMENT_NAME: Optional[str] = None
|
|
202
|
+
AZURE_MODEL_NAME: Optional[str] = None
|
|
203
|
+
AZURE_MODEL_VERSION: Optional[str] = None
|
|
204
|
+
USE_AZURE_OPENAI: Optional[bool] = None
|
|
205
|
+
# DeepSeek
|
|
206
|
+
USE_DEEPSEEK_MODEL: Optional[bool] = None
|
|
207
|
+
DEEPSEEK_API_KEY: Optional[SecretStr] = None
|
|
208
|
+
DEEPSEEK_MODEL_NAME: Optional[str] = None
|
|
209
|
+
# Gemini
|
|
210
|
+
USE_GEMINI_MODEL: Optional[bool] = None
|
|
211
|
+
GOOGLE_API_KEY: Optional[SecretStr] = None
|
|
212
|
+
GEMINI_MODEL_NAME: Optional[str] = None
|
|
213
|
+
GOOGLE_GENAI_USE_VERTEXAI: Optional[bool] = None
|
|
214
|
+
GOOGLE_CLOUD_PROJECT: Optional[str] = None
|
|
215
|
+
GOOGLE_CLOUD_LOCATION: Optional[str] = None
|
|
216
|
+
# Grok
|
|
217
|
+
USE_GROK_MODEL: Optional[bool] = None
|
|
218
|
+
GROK_API_KEY: Optional[SecretStr] = None
|
|
219
|
+
GROK_MODEL_NAME: Optional[str] = None
|
|
220
|
+
# LiteLLM
|
|
221
|
+
USE_LITELLM: Optional[bool] = None
|
|
222
|
+
LITELLM_API_KEY: Optional[SecretStr] = None
|
|
223
|
+
LITELLM_MODEL_NAME: Optional[str] = None
|
|
224
|
+
LITELLM_API_BASE: Optional[AnyUrl] = None
|
|
225
|
+
LITELLM_PROXY_API_BASE: Optional[AnyUrl] = None
|
|
226
|
+
LITELLM_PROXY_API_KEY: Optional[SecretStr] = None
|
|
227
|
+
# LM Studio
|
|
228
|
+
LM_STUDIO_API_KEY: Optional[SecretStr] = None
|
|
229
|
+
LM_STUDIO_MODEL_NAME: Optional[str] = None
|
|
230
|
+
# Local Model
|
|
231
|
+
USE_LOCAL_MODEL: Optional[bool] = None
|
|
232
|
+
LOCAL_MODEL_API_KEY: Optional[SecretStr] = None
|
|
233
|
+
LOCAL_EMBEDDING_API_KEY: Optional[SecretStr] = None
|
|
234
|
+
LOCAL_MODEL_NAME: Optional[str] = None
|
|
235
|
+
LOCAL_MODEL_BASE_URL: Optional[AnyUrl] = None
|
|
236
|
+
LOCAL_MODEL_FORMAT: Optional[str] = None
|
|
237
|
+
# Moonshot
|
|
238
|
+
USE_MOONSHOT_MODEL: Optional[bool] = None
|
|
239
|
+
MOONSHOT_API_KEY: Optional[SecretStr] = None
|
|
240
|
+
MOONSHOT_MODEL_NAME: Optional[str] = None
|
|
241
|
+
# Ollama
|
|
242
|
+
OLLAMA_MODEL_NAME: Optional[str] = None
|
|
243
|
+
# OpenAI
|
|
244
|
+
USE_OPENAI_MODEL: Optional[bool] = None
|
|
245
|
+
OPENAI_API_KEY: Optional[SecretStr] = None
|
|
246
|
+
OPENAI_MODEL_NAME: Optional[str] = None
|
|
247
|
+
OPENAI_COST_PER_INPUT_TOKEN: Optional[float] = None
|
|
248
|
+
OPENAI_COST_PER_OUTPUT_TOKEN: Optional[float] = None
|
|
249
|
+
# Vertex AI
|
|
250
|
+
VERTEX_AI_MODEL_NAME: Optional[str] = None
|
|
251
|
+
# VLLM
|
|
252
|
+
VLLM_API_KEY: Optional[SecretStr] = None
|
|
253
|
+
VLLM_MODEL_NAME: Optional[str] = None
|
|
254
|
+
|
|
255
|
+
#
|
|
256
|
+
# Embedding Keys
|
|
257
|
+
#
|
|
258
|
+
|
|
259
|
+
# Azure OpenAI
|
|
260
|
+
USE_AZURE_OPENAI_EMBEDDING: Optional[bool] = None
|
|
261
|
+
AZURE_EMBEDDING_DEPLOYMENT_NAME: Optional[str] = None
|
|
262
|
+
# Local
|
|
263
|
+
USE_LOCAL_EMBEDDINGS: Optional[bool] = None
|
|
264
|
+
LOCAL_EMBEDDING_MODEL_NAME: Optional[str] = None
|
|
265
|
+
LOCAL_EMBEDDING_BASE_URL: Optional[AnyUrl] = None
|
|
266
|
+
|
|
267
|
+
#
|
|
268
|
+
# Telemetry and Debug
|
|
269
|
+
#
|
|
270
|
+
DEEPEVAL_TELEMETRY_OPT_OUT: Optional[bool] = None
|
|
271
|
+
DEEPEVAL_UPDATE_WARNING_OPT_IN: Optional[bool] = None
|
|
272
|
+
DEEPEVAL_GRPC_LOGGING: Optional[bool] = None
|
|
273
|
+
GRPC_VERBOSITY: Optional[str] = None
|
|
274
|
+
GRPC_TRACE: Optional[str] = None
|
|
275
|
+
ERROR_REPORTING: Optional[bool] = None
|
|
276
|
+
IGNORE_DEEPEVAL_ERRORS: Optional[bool] = None
|
|
277
|
+
SKIP_DEEPEVAL_MISSING_PARAMS: Optional[bool] = None
|
|
278
|
+
DEEPEVAL_VERBOSE_MODE: Optional[bool] = None
|
|
279
|
+
ENABLE_DEEPEVAL_CACHE: Optional[bool] = None
|
|
280
|
+
CONFIDENT_TRACE_FLUSH: Optional[bool] = None
|
|
281
|
+
CONFIDENT_TRACE_ENVIRONMENT: Optional[str] = "development"
|
|
282
|
+
CONFIDENT_TRACE_VERBOSE: Optional[bool] = True
|
|
283
|
+
CONFIDENT_SAMPLE_RATE: Optional[float] = 1.0
|
|
284
|
+
OTEL_EXPORTER_OTLP_ENDPOINT: Optional[AnyUrl] = None
|
|
285
|
+
|
|
286
|
+
##############
|
|
287
|
+
# Validators #
|
|
288
|
+
##############
|
|
289
|
+
|
|
290
|
+
@field_validator(
|
|
291
|
+
"CONFIDENT_OPEN_BROWSER",
|
|
292
|
+
"CONFIDENT_TRACE_FLUSH",
|
|
293
|
+
"CONFIDENT_TRACE_VERBOSE",
|
|
294
|
+
"USE_OPENAI_MODEL",
|
|
295
|
+
"USE_AZURE_OPENAI",
|
|
296
|
+
"USE_LOCAL_MODEL",
|
|
297
|
+
"USE_GEMINI_MODEL",
|
|
298
|
+
"GOOGLE_GENAI_USE_VERTEXAI",
|
|
299
|
+
"USE_MOONSHOT_MODEL",
|
|
300
|
+
"USE_GROK_MODEL",
|
|
301
|
+
"USE_DEEPSEEK_MODEL",
|
|
302
|
+
"USE_LITELLM",
|
|
303
|
+
"USE_AZURE_OPENAI_EMBEDDING",
|
|
304
|
+
"USE_LOCAL_EMBEDDINGS",
|
|
305
|
+
"DEEPEVAL_GRPC_LOGGING",
|
|
306
|
+
"DEEPEVAL_DISABLE_DOTENV",
|
|
307
|
+
"DEEPEVAL_TELEMETRY_OPT_OUT",
|
|
308
|
+
"DEEPEVAL_UPDATE_WARNING_OPT_IN",
|
|
309
|
+
"TOKENIZERS_PARALLELISM",
|
|
310
|
+
"TRANSFORMERS_NO_ADVISORY_WARNINGS",
|
|
311
|
+
"CUDA_LAUNCH_BLOCKING",
|
|
312
|
+
"ERROR_REPORTING",
|
|
313
|
+
"IGNORE_DEEPEVAL_ERRORS",
|
|
314
|
+
"SKIP_DEEPEVAL_MISSING_PARAMS",
|
|
315
|
+
"DEEPEVAL_VERBOSE_MODE",
|
|
316
|
+
"ENABLE_DEEPEVAL_CACHE",
|
|
317
|
+
mode="before",
|
|
318
|
+
)
|
|
319
|
+
@classmethod
|
|
320
|
+
def _coerce_yes_no(cls, v):
|
|
321
|
+
return None if v is None else parse_bool(v, default=False)
|
|
322
|
+
|
|
323
|
+
@field_validator("DEEPEVAL_RESULTS_FOLDER", "ENV_DIR_PATH", mode="before")
|
|
324
|
+
@classmethod
|
|
325
|
+
def _coerce_path(cls, v):
|
|
326
|
+
if v is None:
|
|
327
|
+
return None
|
|
328
|
+
s = str(v).strip()
|
|
329
|
+
if not s:
|
|
330
|
+
return None
|
|
331
|
+
# expand ~ and env vars;
|
|
332
|
+
# but don't resolve to avoid failing on non-existent paths
|
|
333
|
+
return Path(os.path.expandvars(os.path.expanduser(s)))
|
|
334
|
+
|
|
335
|
+
# Treat "", "none", "null" as None for numeric overrides
|
|
336
|
+
@field_validator(
|
|
337
|
+
"OPENAI_COST_PER_INPUT_TOKEN",
|
|
338
|
+
"OPENAI_COST_PER_OUTPUT_TOKEN",
|
|
339
|
+
"TEMPERATURE",
|
|
340
|
+
"CONFIDENT_SAMPLE_RATE",
|
|
341
|
+
mode="before",
|
|
342
|
+
)
|
|
343
|
+
@classmethod
|
|
344
|
+
def _none_or_float(cls, v):
|
|
345
|
+
if v is None:
|
|
346
|
+
return None
|
|
347
|
+
s = str(v).strip().lower()
|
|
348
|
+
if s in {"", "none", "null"}:
|
|
349
|
+
return None
|
|
350
|
+
return float(v)
|
|
351
|
+
|
|
352
|
+
@field_validator("CONFIDENT_SAMPLE_RATE")
|
|
353
|
+
@classmethod
|
|
354
|
+
def _validate_sample_rate(cls, v):
|
|
355
|
+
if v is None:
|
|
356
|
+
return None
|
|
357
|
+
if not (0.0 <= float(v) <= 1.0):
|
|
358
|
+
raise ValueError("CONFIDENT_SAMPLE_RATE must be between 0 and 1")
|
|
359
|
+
return float(v)
|
|
360
|
+
|
|
361
|
+
@field_validator("DEEPEVAL_DEFAULT_SAVE", mode="before")
|
|
362
|
+
@classmethod
|
|
363
|
+
def _validate_default_save(cls, v):
|
|
364
|
+
if v is None:
|
|
365
|
+
return None
|
|
366
|
+
s = str(v).strip()
|
|
367
|
+
if not s:
|
|
368
|
+
return None
|
|
369
|
+
m = _SAVE_RE.match(s)
|
|
370
|
+
if not m:
|
|
371
|
+
raise ValueError(
|
|
372
|
+
"DEEPEVAL_DEFAULT_SAVE must be 'dotenv' or 'dotenv:<path>'"
|
|
373
|
+
)
|
|
374
|
+
path = m.group("path")
|
|
375
|
+
if path is None:
|
|
376
|
+
return "dotenv"
|
|
377
|
+
path = os.path.expanduser(os.path.expandvars(path))
|
|
378
|
+
return f"dotenv:{path}"
|
|
379
|
+
|
|
380
|
+
@field_validator("DEEPEVAL_FILE_SYSTEM", mode="before")
|
|
381
|
+
@classmethod
|
|
382
|
+
def _normalize_fs(cls, v):
|
|
383
|
+
if v is None:
|
|
384
|
+
return None
|
|
385
|
+
s = str(v).strip().upper()
|
|
386
|
+
|
|
387
|
+
# adds friendly aliases
|
|
388
|
+
if s in {"READ_ONLY", "READ-ONLY", "READONLY", "RO"}:
|
|
389
|
+
return "READ_ONLY"
|
|
390
|
+
raise ValueError(
|
|
391
|
+
"DEEPEVAL_FILE_SYSTEM must be READ_ONLY (case-insensitive)."
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
@field_validator("CONFIDENT_REGION", mode="before")
|
|
395
|
+
@classmethod
|
|
396
|
+
def _normalize_upper(cls, v):
|
|
397
|
+
if v is None:
|
|
398
|
+
return None
|
|
399
|
+
s = str(v).strip()
|
|
400
|
+
if not s:
|
|
401
|
+
return None
|
|
402
|
+
return s.upper()
|
|
403
|
+
|
|
404
|
+
#######################
|
|
405
|
+
# Persistence support #
|
|
406
|
+
#######################
|
|
407
|
+
class _SettingsEditCtx:
|
|
408
|
+
def __init__(
|
|
409
|
+
self,
|
|
410
|
+
settings: "Settings",
|
|
411
|
+
save: Optional[str],
|
|
412
|
+
persist: Optional[bool],
|
|
413
|
+
):
|
|
414
|
+
self._s = settings
|
|
415
|
+
self._save = save
|
|
416
|
+
self._persist = persist
|
|
417
|
+
self._before: Dict[str, Any] = {}
|
|
418
|
+
self.result: Optional[PersistResult] = None
|
|
419
|
+
|
|
420
|
+
@property
|
|
421
|
+
def s(self) -> "Settings":
|
|
422
|
+
return self._s
|
|
423
|
+
|
|
424
|
+
def __enter__(self) -> "Settings._SettingsEditCtx":
|
|
425
|
+
# snapshot current state
|
|
426
|
+
self._before = {
|
|
427
|
+
k: getattr(self._s, k) for k in type(self._s).model_fields
|
|
428
|
+
}
|
|
429
|
+
return self
|
|
430
|
+
|
|
431
|
+
def __exit__(self, exc_type, exc, tb):
|
|
432
|
+
if exc_type is not None:
|
|
433
|
+
return False # don’t persist on error
|
|
434
|
+
|
|
435
|
+
from deepeval.config.settings_manager import (
|
|
436
|
+
update_settings_and_persist,
|
|
437
|
+
_normalize_for_env,
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
# lazy import legacy JSON store deps
|
|
441
|
+
from deepeval.key_handler import KEY_FILE_HANDLER
|
|
442
|
+
|
|
443
|
+
# compute diff of changed fields
|
|
444
|
+
after = {k: getattr(self._s, k) for k in type(self._s).model_fields}
|
|
445
|
+
|
|
446
|
+
before_norm = {
|
|
447
|
+
k: _normalize_for_env(v) for k, v in self._before.items()
|
|
448
|
+
}
|
|
449
|
+
after_norm = {k: _normalize_for_env(v) for k, v in after.items()}
|
|
450
|
+
|
|
451
|
+
changed_keys = {
|
|
452
|
+
k for k in after_norm if after_norm[k] != before_norm.get(k)
|
|
453
|
+
}
|
|
454
|
+
if not changed_keys:
|
|
455
|
+
self.result = PersistResult(False, None, {})
|
|
456
|
+
return False
|
|
457
|
+
|
|
458
|
+
updates = {k: after[k] for k in changed_keys}
|
|
459
|
+
|
|
460
|
+
#
|
|
461
|
+
# .deepeval JSON support
|
|
462
|
+
#
|
|
463
|
+
|
|
464
|
+
if self._persist is not False:
|
|
465
|
+
for k in changed_keys:
|
|
466
|
+
legacy_member = _find_legacy_enum(k)
|
|
467
|
+
if legacy_member is None:
|
|
468
|
+
continue # skip if not a defined as legacy field
|
|
469
|
+
|
|
470
|
+
val = updates[k]
|
|
471
|
+
# Remove from JSON if unset
|
|
472
|
+
if val is None:
|
|
473
|
+
KEY_FILE_HANDLER.remove_key(legacy_member)
|
|
474
|
+
continue
|
|
475
|
+
|
|
476
|
+
# Never store secrets in the JSON keystore
|
|
477
|
+
if _is_secret_key(self._s, k):
|
|
478
|
+
continue
|
|
479
|
+
|
|
480
|
+
# For booleans, the legacy store expects "YES"/"NO"
|
|
481
|
+
if isinstance(val, bool):
|
|
482
|
+
KEY_FILE_HANDLER.write_key(
|
|
483
|
+
legacy_member, "YES" if val else "NO"
|
|
484
|
+
)
|
|
485
|
+
else:
|
|
486
|
+
# store as string
|
|
487
|
+
KEY_FILE_HANDLER.write_key(legacy_member, str(val))
|
|
488
|
+
|
|
489
|
+
#
|
|
490
|
+
# dotenv store
|
|
491
|
+
#
|
|
492
|
+
|
|
493
|
+
# defer import to avoid cyclics
|
|
494
|
+
handled, path = update_settings_and_persist(
|
|
495
|
+
updates,
|
|
496
|
+
save=self._save,
|
|
497
|
+
persist_dotenv=(False if self._persist is False else True),
|
|
498
|
+
)
|
|
499
|
+
self.result = PersistResult(handled, path, updates)
|
|
500
|
+
return False
|
|
501
|
+
|
|
502
|
+
def switch_model_provider(self, target) -> None:
|
|
503
|
+
"""
|
|
504
|
+
Flip all USE_* toggles so that the one matching the target is True and the rest are False.
|
|
505
|
+
Also, mirror this change into the legacy JSON keystore as "YES"/"NO".
|
|
506
|
+
|
|
507
|
+
`target` may be an Enum with `.value`, such as ModelKeyValues.USE_OPENAI_MODEL
|
|
508
|
+
or a plain string like "USE_OPENAI_MODEL".
|
|
509
|
+
"""
|
|
510
|
+
from deepeval.key_handler import KEY_FILE_HANDLER
|
|
511
|
+
|
|
512
|
+
# Target key is the env style string, such as "USE_OPENAI_MODEL"
|
|
513
|
+
target_key = getattr(target, "value", str(target))
|
|
514
|
+
|
|
515
|
+
use_fields = [
|
|
516
|
+
k for k in type(self._s).model_fields if k.startswith("USE_")
|
|
517
|
+
]
|
|
518
|
+
if target_key not in use_fields:
|
|
519
|
+
raise ValueError(
|
|
520
|
+
f"{target_key} is not a recognized USE_* field"
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
for k in use_fields:
|
|
524
|
+
on = k == target_key
|
|
525
|
+
# dotenv persistence will serialize to "1"/"0"
|
|
526
|
+
setattr(self._s, k, on)
|
|
527
|
+
if self._persist is not False:
|
|
528
|
+
# legacy json persistence will serialize to "YES"/"NO"
|
|
529
|
+
legacy_member = _find_legacy_enum(k)
|
|
530
|
+
if legacy_member is not None:
|
|
531
|
+
KEY_FILE_HANDLER.write_key(
|
|
532
|
+
legacy_member, "YES" if on else "NO"
|
|
533
|
+
)
|
|
534
|
+
|
|
535
|
+
def edit(
|
|
536
|
+
self, *, save: Optional[str] = None, persist: Optional[bool] = None
|
|
537
|
+
):
|
|
538
|
+
"""Context manager for atomic, persisted updates.
|
|
539
|
+
|
|
540
|
+
Args:
|
|
541
|
+
save: 'dotenv[:path]' to explicitly write to a dotenv file.
|
|
542
|
+
None (default) respects DEEPEVAL_DEFAULT_SAVE if set.
|
|
543
|
+
persist: If False, do not write (dotenv, JSON), update runtime only.
|
|
544
|
+
If True or None, normal persistence rules apply.
|
|
545
|
+
"""
|
|
546
|
+
return self._SettingsEditCtx(self, save, persist)
|
|
547
|
+
|
|
548
|
+
def set_model_provider(self, target, *, save: Optional[str] = None):
|
|
549
|
+
"""
|
|
550
|
+
Convenience wrapper to switch providers outside of an existing edit() block.
|
|
551
|
+
Returns the PersistResult.
|
|
552
|
+
"""
|
|
553
|
+
with self.edit(save=save) as ctx:
|
|
554
|
+
ctx.switch_model_provider(target)
|
|
555
|
+
return ctx.result
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
_settings_singleton: Optional[Settings] = None
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def get_settings() -> Settings:
|
|
562
|
+
global _settings_singleton
|
|
563
|
+
if _settings_singleton is None:
|
|
564
|
+
_settings_singleton = Settings()
|
|
565
|
+
return _settings_singleton
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Applies CLI driven updates to the live Settings and optionally persists them to a
|
|
3
|
+
dotenv file. Also syncs os.environ, handles unsets, and warns on unknown fields.
|
|
4
|
+
Primary entrypoint: update_settings_and_persist.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
import os
|
|
9
|
+
|
|
10
|
+
from difflib import get_close_matches
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Union
|
|
13
|
+
from enum import Enum
|
|
14
|
+
|
|
15
|
+
from pydantic import SecretStr
|
|
16
|
+
from deepeval.config.settings import get_settings, _SAVE_RE
|
|
17
|
+
from deepeval.cli.dotenv_handler import DotenvHandler
|
|
18
|
+
from deepeval.utils import bool_to_env_str
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger(__name__)
|
|
21
|
+
StrOrEnum = Union[str, Enum]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _env_key(k: StrOrEnum) -> str:
|
|
25
|
+
return k.value if isinstance(k, Enum) else str(k)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _normalize_for_env(val: Any) -> Optional[str]:
|
|
29
|
+
"""Convert typed value to string for dotenv + os.environ; None -> unset."""
|
|
30
|
+
if val is None:
|
|
31
|
+
return None
|
|
32
|
+
if isinstance(val, SecretStr):
|
|
33
|
+
return val.get_secret_value()
|
|
34
|
+
if isinstance(val, bool):
|
|
35
|
+
return bool_to_env_str(val)
|
|
36
|
+
return str(val)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _resolve_save_path(save_opt: Optional[str]) -> Tuple[bool, Optional[Path]]:
|
|
40
|
+
"""
|
|
41
|
+
Returns (ok, path).
|
|
42
|
+
- ok=False -> invalid save option format
|
|
43
|
+
- ok=True, path=None -> no persistence requested
|
|
44
|
+
- ok=True, path=Path -> persist to that file
|
|
45
|
+
"""
|
|
46
|
+
raw = (
|
|
47
|
+
save_opt if save_opt is not None else os.getenv("DEEPEVAL_DEFAULT_SAVE")
|
|
48
|
+
)
|
|
49
|
+
if not raw:
|
|
50
|
+
return True, None
|
|
51
|
+
m = _SAVE_RE.match(raw.strip())
|
|
52
|
+
if not m:
|
|
53
|
+
return False, None
|
|
54
|
+
path = m.group("path") or ".env.local"
|
|
55
|
+
path = Path(os.path.expanduser(os.path.expandvars(path)))
|
|
56
|
+
return True, path
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def update_settings_and_persist(
|
|
60
|
+
updates: Mapping[StrOrEnum, Any],
|
|
61
|
+
*,
|
|
62
|
+
save: Optional[str] = None,
|
|
63
|
+
unset: Iterable[StrOrEnum] = (),
|
|
64
|
+
persist_dotenv: bool = True,
|
|
65
|
+
) -> Tuple[bool, Optional[Path]]:
|
|
66
|
+
"""
|
|
67
|
+
Write and update:
|
|
68
|
+
- validate + assign into live Settings()
|
|
69
|
+
- update os.environ
|
|
70
|
+
- persist to dotenv, if `save` or DEEPEVAL_DEFAULT_SAVE provided
|
|
71
|
+
- unset keys where value is None or explicitly in `unset`
|
|
72
|
+
Returns (handled, path_to_dotenv_if_any).
|
|
73
|
+
"""
|
|
74
|
+
settings = get_settings()
|
|
75
|
+
|
|
76
|
+
# validate + assign into settings.
|
|
77
|
+
# validation is handled in Settings as long as validate_assignment=True
|
|
78
|
+
typed: Dict[str, Any] = {}
|
|
79
|
+
for key, value in updates.items():
|
|
80
|
+
k = _env_key(key)
|
|
81
|
+
if k not in type(settings).model_fields:
|
|
82
|
+
suggestion = get_close_matches(
|
|
83
|
+
k, type(settings).model_fields.keys(), n=1
|
|
84
|
+
)
|
|
85
|
+
if suggestion:
|
|
86
|
+
logger.warning(
|
|
87
|
+
"Unknown settings field '%s'; did you mean '%s'? Ignoring.",
|
|
88
|
+
k,
|
|
89
|
+
suggestion[0],
|
|
90
|
+
stacklevel=2,
|
|
91
|
+
)
|
|
92
|
+
else:
|
|
93
|
+
logger.warning(
|
|
94
|
+
"Unknown settings field '%s'; ignoring.", k, stacklevel=2
|
|
95
|
+
)
|
|
96
|
+
continue
|
|
97
|
+
|
|
98
|
+
setattr(settings, k, value)
|
|
99
|
+
# coercion is handled in Settings
|
|
100
|
+
typed[k] = getattr(settings, k)
|
|
101
|
+
|
|
102
|
+
# build env maps
|
|
103
|
+
to_write: Dict[str, str] = {}
|
|
104
|
+
to_unset: set[str] = set(_env_key(k) for k in unset)
|
|
105
|
+
|
|
106
|
+
for k, v in typed.items():
|
|
107
|
+
env_val = _normalize_for_env(v)
|
|
108
|
+
if env_val is None:
|
|
109
|
+
to_unset.add(k)
|
|
110
|
+
else:
|
|
111
|
+
to_write[k] = env_val
|
|
112
|
+
|
|
113
|
+
# update process env so that it is effective immediately
|
|
114
|
+
for k, v in to_write.items():
|
|
115
|
+
os.environ[k] = v
|
|
116
|
+
for k in to_unset:
|
|
117
|
+
os.environ.pop(k, None)
|
|
118
|
+
|
|
119
|
+
if not persist_dotenv:
|
|
120
|
+
return True, None
|
|
121
|
+
|
|
122
|
+
# persist to dotenv if save is ok
|
|
123
|
+
ok, path = _resolve_save_path(save)
|
|
124
|
+
if not ok:
|
|
125
|
+
return False, None # unsupported --save
|
|
126
|
+
if path:
|
|
127
|
+
h = DotenvHandler(path)
|
|
128
|
+
if to_write:
|
|
129
|
+
h.upsert(to_write)
|
|
130
|
+
if to_unset:
|
|
131
|
+
h.unset(to_unset)
|
|
132
|
+
return True, path
|
|
133
|
+
return True, None
|