deepeval 3.8.3__tar.gz → 3.8.4__tar.gz

{deepeval-3.8.3 → deepeval-3.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.8.3
+Version: 3.8.4
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

deepeval-3.8.4/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.8.4"

{deepeval-3.8.3 → deepeval-3.8.4}/deepeval/config/settings.py

@@ -316,6 +316,12 @@ class Settings(BaseSettings):
         description="If set, export a timestamped JSON of the latest test run into this folder (created if missing).",
     )
 
+    # When set, overrides the default DeepEval cache directory
+    DEEPEVAL_CACHE_FOLDER: Optional[Path] = Field(
+        ".deepeval",
+        description="Path to the directory used by DeepEval to store cache files. If set, this overrides the default cache location. The directory will be created if it does not exist.",
+    )
+
     # Display / Truncation
     DEEPEVAL_MAXLEN_TINY: Optional[int] = Field(
         40,
@@ -1015,7 +1021,12 @@ class Settings(BaseSettings):
     def _coerce_yes_no(cls, v):
         return None if v is None else parse_bool(v, default=False)
 
-    @field_validator("DEEPEVAL_RESULTS_FOLDER", "ENV_DIR_PATH", mode="before")
+    @field_validator(
+        "DEEPEVAL_RESULTS_FOLDER",
+        "ENV_DIR_PATH",
+        "DEEPEVAL_CACHE_FOLDER",
+        mode="before",
+    )
     @classmethod
     def _coerce_path(cls, v):
         if v is None:

{deepeval-3.8.3 → deepeval-3.8.4}/deepeval/constants.py

@@ -1,8 +1,9 @@
 from enum import Enum
 from typing import Union
+import os
 
 KEY_FILE: str = ".deepeval"
-HIDDEN_DIR: str = ".deepeval"
+HIDDEN_DIR: str = os.getenv("DEEPEVAL_CACHE_FOLDER", ".deepeval")
 PYTEST_RUN_TEST_NAME: str = "CONFIDENT_AI_RUN_TEST_NAME"
 LOGIN_PROMPT = "\n✨👀 Looking for a place for your LLM test data to live 🏡❤️ ? Use [rgb(106,0,255)]Confident AI[/rgb(106,0,255)] to get & share testing reports, experiment with models/prompts, and catch regressions for your LLM system. Just run [cyan]'deepeval login'[/cyan] in the CLI."
 

{deepeval-3.8.3 → deepeval-3.8.4}/deepeval/prompt/__init__.py

@@ -7,6 +7,7 @@ from .api import (
     ReasoningEffort,
     OutputType,
     PromptInterpolationType,
+    Tool,
 )
 
 __all__ = [
@@ -18,4 +19,5 @@ __all__ = [
     "ReasoningEffort",
     "OutputType",
     "PromptInterpolationType",
+    "Tool",
 ]

{deepeval-3.8.3 → deepeval-3.8.4}/deepeval/prompt/api.py

@@ -1,6 +1,14 @@
-from pydantic import BaseModel, Field, AliasChoices, ConfigDict
+from pydantic import (
+    BaseModel,
+    Field,
+    AliasChoices,
+    ConfigDict,
+    model_validator,
+    model_serializer,
+)
 from enum import Enum
-from typing import List, Optional
+import uuid
+from typing import List, Optional, Dict, Any, Union, Type
 from pydantic import TypeAdapter
 
 from deepeval.utils import make_model_config
@@ -33,6 +41,12 @@ class ModelProvider(Enum):
     OPENROUTER = "OPENROUTER"
 
 
+class ToolMode(str, Enum):
+    ALLOW_ADDITIONAL = "ALLOW_ADDITIONAL"
+    NO_ADDITIONAL = "NO_ADDITIONAL"
+    STRICT = "STRICT"
+
+
 class ModelSettings(BaseModel):
     provider: Optional[ModelProvider] = None
     name: Optional[str] = None
@@ -100,6 +114,7 @@ class OutputSchemaField(BaseModel):
     id: str
     type: SchemaDataType
     name: str
+    description: Optional[str] = None
     required: Optional[bool] = False
     parent_id: Optional[str] = Field(
         default=None,
@@ -109,8 +124,36 @@ class OutputSchemaField(BaseModel):
 
 
 class OutputSchema(BaseModel):
+    id: Optional[str] = None
     fields: Optional[List[OutputSchemaField]] = None
+    name: Optional[str] = None
+
+
+class Tool(BaseModel):
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
     name: str
+    description: Optional[str] = None
+    mode: ToolMode
+    structured_schema: Optional[Union[Type[BaseModel], OutputSchema]] = Field(
+        serialization_alias="structuredSchema",
+        validation_alias=AliasChoices("structured_schema", "structuredSchema"),
+    )
+
+    @model_validator(mode="after")
+    def update_schema(self):
+        if not isinstance(self.structured_schema, OutputSchema):
+            from deepeval.prompt.utils import construct_output_schema
+
+            self.structured_schema = construct_output_schema(
+                self.structured_schema
+            )
+        return self
+
+    @property
+    def input_schema(self) -> Dict[str, Any]:
+        from deepeval.prompt.utils import output_schema_to_json_schema
+
+        return output_schema_to_json_schema(self.structured_schema)
 
 
 ###################################
@@ -186,6 +229,7 @@ class PromptHttpResponse(BaseModel):
         serialization_alias="outputSchema",
         validation_alias=AliasChoices("output_schema", "outputSchema"),
     )
+    tools: Optional[List[Tool]] = None
 
 
 class PromptPushRequest(BaseModel):
@@ -196,6 +240,7 @@ class PromptPushRequest(BaseModel):
     alias: str
     text: Optional[str] = None
     messages: Optional[List[PromptMessage]] = None
+    tools: Optional[List[Tool]] = None
     interpolation_type: PromptInterpolationType = Field(
         serialization_alias="interpolationType"
     )
@@ -215,6 +260,7 @@ class PromptUpdateRequest(BaseModel):
 
     text: Optional[str] = None
     messages: Optional[List[PromptMessage]] = None
+    tools: Optional[List[Tool]] = None
     interpolation_type: PromptInterpolationType = Field(
         serialization_alias="interpolationType"
     )

{deepeval-3.8.3 → deepeval-3.8.4}/deepeval/prompt/prompt.py

@@ -25,6 +25,7 @@ from deepeval.prompt.api import (
     ModelSettings,
     OutputSchema,
     OutputType,
+    Tool,
 )
 from deepeval.prompt.utils import (
     interpolate_text,
@@ -101,6 +102,7 @@ class CachedPrompt(BaseModel):
     model_settings: Optional[ModelSettings]
     output_type: Optional[OutputType]
     output_schema: Optional[OutputSchema]
+    tools: Optional[List[Tool]] = None
 
 
 class Prompt:
@@ -131,6 +133,7 @@ class Prompt:
             interpolation_type or PromptInterpolationType.FSTRING
         )
         self.confident_api_key = confident_api_key
+        self.tools: Optional[List[Tool]] = None
 
         self._version = None
         self._prompt_version_id: Optional[str] = None
@@ -308,6 +311,7 @@ class Prompt:
         model_settings: Optional[ModelSettings] = None,
         output_type: Optional[OutputType] = None,
         output_schema: Optional[OutputSchema] = None,
+        tools: Optional[List[Tool]] = None,
     ):
         if portalocker is None or not self.alias:
             return
@@ -354,6 +358,7 @@ class Prompt:
                     "model_settings": model_settings,
                     "output_type": output_type,
                     "output_schema": output_schema,
+                    "tools": tools,
                 }
 
                 if cache_key == VERSION_CACHE_KEY:
@@ -415,6 +420,7 @@ class Prompt:
             self.output_schema = construct_base_model(
                 cached_prompt.output_schema
             )
+            self.tools = cached_prompt.tools
 
         end_time = time.perf_counter()
         time_taken = format(end_time - start_time, ".2f")
@@ -494,6 +500,7 @@ class Prompt:
                         self.output_schema = construct_base_model(
                             cached_prompt.output_schema
                         )
+                        self.tools = cached_prompt.tools
                     return
             except Exception:
                 pass
@@ -547,6 +554,7 @@ class Prompt:
                     model_settings=data.get("modelSettings", None),
                     output_type=data.get("outputType", None),
                     output_schema=data.get("outputSchema", None),
+                    tools=data.get("tools", None),
                 )
             except Exception:
                 if fallback_to_cache:
@@ -573,6 +581,7 @@ class Prompt:
                 self.output_schema = construct_base_model(
                     response.output_schema
                 )
+                self.tools = response.tools
 
             end_time = time.perf_counter()
             time_taken = format(end_time - start_time, ".2f")
@@ -594,6 +603,7 @@ class Prompt:
                     model_settings=response.model_settings,
                     output_type=response.output_type,
                     output_schema=response.output_schema,
+                    tools=response.tools,
                 )
 
     def push(
@@ -606,6 +616,7 @@ class Prompt:
         model_settings: Optional[ModelSettings] = None,
         output_type: Optional[OutputType] = None,
         output_schema: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Tool]] = None,
         _verbose: Optional[bool] = True,
     ):
         if self.alias is None:
@@ -628,6 +639,7 @@ class Prompt:
             output_type=output_type or self.output_type,
             output_schema=construct_output_schema(output_schema)
             or construct_output_schema(self.output_schema),
+            tools=tools or self.tools,
         )
         try:
             body = body.model_dump(
@@ -655,6 +667,7 @@ class Prompt:
             self.model_settings = model_settings or self.model_settings
             self.output_type = output_type or self.output_type
             self.output_schema = output_schema or self.output_schema
+            self.tools = tools or self.tools
             self.type = PromptType.TEXT if text_template else PromptType.LIST
             if _verbose:
                 console = Console()
@@ -674,6 +687,7 @@ class Prompt:
         model_settings: Optional[ModelSettings] = None,
         output_type: Optional[OutputType] = None,
         output_schema: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Tool]] = None,
     ):
         if self.alias is None:
             raise ValueError(
@@ -687,6 +701,7 @@ class Prompt:
             model_settings=model_settings,
             output_type=output_type,
             output_schema=construct_output_schema(output_schema),
+            tools=tools,
         )
         try:
             body = body.model_dump(
@@ -712,6 +727,7 @@ class Prompt:
             self.model_settings = model_settings
             self.output_type = output_type
             self.output_schema = output_schema
+            self.tools = tools
             self.type = PromptType.TEXT if text else PromptType.LIST
             console = Console()
             console.print("✅ Prompt successfully updated on Confident AI!")
@@ -796,6 +812,10 @@ class Prompt:
                     messages=data.get("messages", None),
                     type=data["type"],
                     interpolation_type=data["interpolationType"],
+                    model_settings=data.get("modelSettings", None),
+                    output_type=data.get("outputType", None),
+                    output_schema=data.get("outputSchema", None),
+                    tools=data.get("tools", None),
                 )
 
                 # Update the cache with fresh data from server
@@ -808,6 +828,10 @@ class Prompt:
                     prompt_version_id=response.id,
                     type=response.type,
                     interpolation_type=response.interpolation_type,
+                    model_settings=response.model_settings,
+                    output_type=response.output_type,
+                    output_schema=response.output_schema,
+                    tools=response.tools,
                 )
 
                 # Update in-memory properties with fresh data (thread-safe)
@@ -819,6 +843,12 @@ class Prompt:
                     self._prompt_version_id = response.id
                     self.type = response.type
                     self.interpolation_type = response.interpolation_type
+                    self.model_settings = response.model_settings
+                    self.output_type = response.output_type
+                    self.output_schema = construct_base_model(
+                        response.output_schema
+                    )
+                    self.tools = response.tools
 
             except Exception:
                 pass

{deepeval-3.8.3 → deepeval-3.8.4}/deepeval/prompt/utils.py

@@ -130,7 +130,7 @@ def construct_base_model(
     if not schema:
         return None
     if not schema.fields:
-        return create_model(schema.name)
+        return create_model(schema.name or "EmptySchema")
 
     parent_id_map: Dict[Optional[str], List[OutputSchemaField]] = {}
     for field in schema.fields:
@@ -153,7 +153,7 @@ def construct_base_model(
         default = ... if field.required else None
         root_fields[field.name] = (python_type, default)
 
-    return create_model(schema.name, **root_fields)
+    return create_model(schema.name or "Schema", **root_fields)
 
 
 ###################################
@@ -219,3 +219,73 @@ def construct_output_schema(
         return None
     all_fields = _process_model(base_model_class)
     return OutputSchema(fields=all_fields, name=base_model_class.__name__)
+
+
+def output_schema_to_json_schema(
+    schema: Optional[OutputSchema] = None,
+) -> Dict[str, Any]:
+    if not schema or not schema.fields:
+        return {
+            "type": "object",
+            "properties": {},
+            "additionalProperties": False,
+        }
+
+    # Build parent-child mapping
+    children_map: Dict[Optional[str], List[OutputSchemaField]] = {}
+    for field in schema.fields:
+        parent_id = field.parent_id
+        children_map.setdefault(parent_id, []).append(field)
+
+    # Map SchemaDataType to JSON Schema types
+    def map_type(dtype: SchemaDataType) -> str:
+        return {
+            SchemaDataType.STRING: "string",
+            SchemaDataType.INTEGER: "integer",
+            SchemaDataType.FLOAT: "number",
+            SchemaDataType.BOOLEAN: "boolean",
+            SchemaDataType.OBJECT: "object",
+            SchemaDataType.NULL: "null",
+        }.get(dtype, "string")
+
+    def build_node(field_list: List[OutputSchemaField]) -> Dict[str, Any]:
+        properties = {}
+        required_fields = []
+
+        for field in field_list:
+            field_type = (
+                field.type.value if hasattr(field.type, "value") else field.type
+            )
+            field_schema = {"type": map_type(field.type)}
+
+            # Add description if available
+            if field.description:
+                field_schema["description"] = field.description
+
+            # Handle nested objects
+            if field_type == SchemaDataType.OBJECT.value:
+                children = children_map.get(field.id, [])
+                if children:
+                    nested = build_node(children)
+                    field_schema.update(nested)
+                else:
+                    field_schema["properties"] = {}
+                    field_schema["additionalProperties"] = False
+
+            properties[field.name] = field_schema
+            if field.required:
+                required_fields.append(field.name)
+
+        schema_dict = {
+            "type": "object",
+            "properties": properties,
+            "additionalProperties": False,
+        }
+
+        if required_fields:
+            schema_dict["required"] = required_fields
+
+        return schema_dict
+
+    root_fields = children_map.get(None, [])
+    return build_node(root_fields)

{deepeval-3.8.3 → deepeval-3.8.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deepeval"
-version = "3.8.3"
+version = "3.8.4"
 description = "The LLM Evaluation Framework"
 authors = ["Jeffrey Ip <jeffreyip@confident-ai.com>"]
 license = "Apache-2.0"

deepeval-3.8.3/deepeval/_version.py

@@ -1 +0,0 @@
-__version__: str = "3.8.3"