deepeval 3.4.7__py3-none-any.whl → 3.4.8__py3-none-any.whl

deepeval/metrics/conversational_dag/templates.py

@@ -0,0 +1,117 @@
+from typing import List
+from textwrap import dedent
+
+
+class ConversationalVerdictNodeTemplate:
+    @staticmethod
+    def generate_reason(verbose_steps: List[str], score: float, name: str):
+        return dedent(
+            f"""You are given a metric name, its score, and a traversal path through a conversational evaluation DAG (Directed Acyclic Graph).
+                This DAG reflects step-by-step reasoning over a dialogue to arrive at the final verdict.
+
+                Each step in the DAG represents a judgment based on parts of the conversation — including roles and the contents they spoke of.
+
+                Your task is to explain **why the score was assigned**, using the traversal steps to justify the reasoning.
+
+                Metric Name:
+                {name}
+
+                Score:
+                {score}
+
+                DAG Traversal:
+                {verbose_steps}
+
+                **
+                IMPORTANT: Only return JSON with a 'reason' key.
+                Example:
+                {{
+                "reason": "The score is {score} because the assistant repeatedly failed to clarify the user's ambiguous statements, as shown in the DAG traversal path."
+                }}
+                **
+                JSON:
+            """
+        )
+
+
+class ConversationalTaskNodeTemplate:
+    @staticmethod
+    def generate_task_output(instructions: str, text: str):
+        return dedent(
+            f"""You are given a set of task instructions and a full conversation between a user and an assistant.
+
+                Instructions:
+                {instructions}
+
+                {text}
+
+                ===END OF INPUT===
+
+                **
+                IMPORTANT: Only return a JSON with the 'output' key containing the result of applying the instructions to the conversation.
+                Example:
+                {{
+                "output": "..."
+                }}
+                **
+                JSON:
+            """
+        )
+
+
+class ConversationalBinaryJudgementTemplate:
+    @staticmethod
+    def generate_binary_verdict(criteria: str, text: str):
+        return dedent(
+            f"""{criteria}
+
+                Below is the full conversation you should evaluate. Consider dialogue context, speaker roles, and how responses were handled.
+
+                Full Conversation:
+                {text}
+
+                **
+                IMPORTANT: Only return JSON with two keys:
+                - 'verdict': true or false
+                - 'reason': justification based on specific parts of the conversation
+
+                Example:
+                {{
+                "verdict": true,
+                "reason": "The assistant provided a clear and direct answer in response to every user query."
+                }}
+                **
+                JSON:
+            """
+        )
+
+
+class ConversationalNonBinaryJudgementTemplate:
+    @staticmethod
+    def generate_non_binary_verdict(
+        criteria: str, text: str, options: List[str]
+    ):
+        return dedent(
+            f"""{criteria}
+
+                You are evaluating the following conversation. Choose one of the options that best reflects the assistant's behavior.
+
+                Options: {options}
+
+                Full Conversation:
+                {text}
+
+                **
+                IMPORTANT: Only return JSON with two keys:
+                - 'verdict': one of the listed options
+                - 'reason': explanation referencing specific conversation points
+
+                Example:
+                {{
+                "verdict": "{options[1]}",
+                "reason": "The assistant partially addressed the user’s issue but missed clarifying their follow-up question."
+                }}
+                **
+                JSON:
+            """
+        )

deepeval/metrics/dag/dag.py

@@ -13,8 +13,8 @@ from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.g_eval.schema import *
 from deepeval.metrics.dag.graph import DeepAcyclicGraph
-from deepeval.metrics.dag.utils import copy_graph
 from deepeval.metrics.dag.utils import (
+    copy_graph,
     is_valid_dag_from_roots,
     extract_required_params,
 )
@@ -34,7 +34,12 @@ class DAGMetric(BaseMetric):
         verbose_mode: bool = False,
         _include_dag_suffix: bool = True,
     ):
-        if is_valid_dag_from_roots(dag.root_nodes) == False:
+        if (
+            is_valid_dag_from_roots(
+                root_nodes=dag.root_nodes, multiturn=dag.multiturn
+            )
+            == False
+        ):
             raise ValueError("Cycle detected in DAG graph.")
 
         self._verbose_steps: List[str] = []
@@ -56,7 +61,9 @@ class DAGMetric(BaseMetric):
         _in_component: bool = False,
     ) -> float:
         check_llm_test_case_params(
-            test_case, extract_required_params(self.dag.root_nodes), self
+            test_case,
+            extract_required_params(self.dag.root_nodes, self.dag.multiturn),
+            self,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -91,7 +98,9 @@ class DAGMetric(BaseMetric):
         _in_component: bool = False,
     ) -> float:
         check_llm_test_case_params(
-            test_case, extract_required_params(self.dag.root_nodes), self
+            test_case,
+            extract_required_params(self.dag.root_nodes, self.dag.multiturn),
+            self,
         )
 
         self.evaluation_cost = 0 if self.using_native_model else None

deepeval/metrics/dag/graph.py

@@ -1,39 +1,71 @@
 import asyncio
-from typing import List
+from typing import List, Union
 
 from deepeval.metrics.dag import (
     BaseNode,
     NonBinaryJudgementNode,
     BinaryJudgementNode,
 )
-from deepeval.test_case import LLMTestCase
-from deepeval.metrics import BaseMetric
+from deepeval.metrics.conversational_dag import (
+    ConversationalBaseNode,
+    ConversationalBinaryJudgementNode,
+    ConversationalNonBinaryJudgementNode,
+)
+from deepeval.test_case import LLMTestCase, ConversationalTestCase
+from deepeval.metrics import BaseMetric, BaseConversationalMetric
+
+
+def validate_root_nodes(
+    root_nodes: Union[List[BaseNode], List[ConversationalBaseNode]],
+):
+    # see if all root nodes are of the same type, more verbose error message, actualy we should say we cannot mix multi and single turn nodes
+    if not all(isinstance(node, type(root_nodes[0])) for node in root_nodes):
+        raise ValueError("You cannot mix multi and single turn nodes")
+    return True
 
 
 class DeepAcyclicGraph:
+    multiturn: bool
+
     def __init__(
         self,
-        root_nodes: List[BaseNode],
+        root_nodes: Union[List[BaseNode], List[ConversationalBaseNode]],
     ):
-        for root_node in root_nodes:
-            if isinstance(root_node, NonBinaryJudgementNode) or isinstance(
-                root_node, BinaryJudgementNode
-            ):
-                if len(root_nodes) > 1:
-                    raise ValueError(
-                        "You cannot provide more than one root node when using 'BinaryJudgementNode' or 'NonBinaryJudgementNode' in root_nodes."
-                    )
+        validate_root_nodes(root_nodes)
+        self.multiturn = isinstance(root_nodes[0], ConversationalBaseNode)
 
+        if not self.multiturn:
+            for root_node in root_nodes:
+                if isinstance(root_node, NonBinaryJudgementNode) or isinstance(
+                    root_node, BinaryJudgementNode
+                ):
+                    if len(root_nodes) > 1:
+                        raise ValueError(
+                            "You cannot provide more than one root node when using 'BinaryJudgementNode' or 'NonBinaryJudgementNode' in root_nodes."
+                        )
+        else:
+            for root_node in root_nodes:
+                if isinstance(
+                    root_node, ConversationalNonBinaryJudgementNode
+                ) or isinstance(root_node, ConversationalBinaryJudgementNode):
+                    if len(root_nodes) > 1:
+                        raise ValueError(
+                            "You cannot provide more than one root node when using 'ConversationalBinaryJudgementNode' or 'ConversationalNonBinaryJudgementNode' in root_nodes."
+                        )
         self.root_nodes = root_nodes
 
-    def _execute(self, metric: BaseMetric, test_case: LLMTestCase) -> None:
+    def _execute(
+        self,
+        metric: Union[BaseMetric, BaseConversationalMetric],
+        test_case: Union[LLMTestCase, ConversationalTestCase],
+    ) -> None:
         for root_node in self.root_nodes:
             root_node._execute(metric=metric, test_case=test_case, depth=0)
 
     async def _a_execute(
         self,
-        metric: BaseMetric,
-        test_case: LLMTestCase,
+        metric: Union[BaseMetric, BaseConversationalMetric],
+        test_case: Union[LLMTestCase, ConversationalTestCase],
     ) -> None:
         await asyncio.gather(
             *(

deepeval/metrics/dag/utils.py

@@ -1,4 +1,4 @@
-from typing import Set, Dict, Optional
+from typing import Set, Dict, Optional, Union
 import inspect
 
 from deepeval.metrics.dag import (
@@ -9,18 +9,33 @@ from deepeval.metrics.dag import (
     TaskNode,
     DeepAcyclicGraph,
 )
-from deepeval.test_case import LLMTestCaseParams
+from deepeval.metrics.conversational_dag import (
+    ConversationalBaseNode,
+    ConversationalBinaryJudgementNode,
+    ConversationalNonBinaryJudgementNode,
+    ConversationalTaskNode,
+    ConversationalVerdictNode,
+)
+from deepeval.test_case import LLMTestCaseParams, TurnParams
 
 
-def is_valid_dag_from_roots(root_nodes: list[BaseNode]) -> bool:
+def is_valid_dag_from_roots(
+    root_nodes: Union[list[BaseNode], list[ConversationalBaseNode]],
+    multiturn: bool,
+) -> bool:
     visited = set()
     for root in root_nodes:
-        if not is_valid_dag(root, visited, set()):
+        if not is_valid_dag(root, multiturn, visited, set()):
             return False
     return True
 
 
-def is_valid_dag(node: BaseNode, visited=None, stack=None) -> bool:
+def is_valid_dag(
+    node: Union[BaseNode, ConversationalBaseNode],
+    multiturn: bool,
+    visited=None,
+    stack=None,
+) -> bool:
     if visited is None:
         visited = set()
     if stack is None:
@@ -33,14 +48,24 @@ def is_valid_dag(node: BaseNode, visited=None, stack=None) -> bool:
 
     visited.add(node)
     stack.add(node)
-    if (
-        isinstance(node, TaskNode)
-        or isinstance(node, BinaryJudgementNode)
-        or isinstance(node, NonBinaryJudgementNode)
-    ):
-        for child in node.children:
-            if not is_valid_dag(child, visited, stack):
-                return False
+    if not multiturn:
+        if (
+            isinstance(node, TaskNode)
+            or isinstance(node, BinaryJudgementNode)
+            or isinstance(node, NonBinaryJudgementNode)
+        ):
+            for child in node.children:
+                if not is_valid_dag(child, multiturn, visited, stack):
+                    return False
+    else:
+        if (
+            isinstance(node, ConversationalTaskNode)
+            or isinstance(node, ConversationalBinaryJudgementNode)
+            or isinstance(node, ConversationalNonBinaryJudgementNode)
+        ):
+            for child in node.children:
+                if not is_valid_dag(child, multiturn, visited, stack):
+                    return False
 
     stack.remove(node)
     return True
@@ -48,29 +73,51 @@ def is_valid_dag(node: BaseNode, visited=None, stack=None) -> bool:
 
 def extract_required_params(
     nodes: list[BaseNode],
-    required_params: Optional[Set[LLMTestCaseParams]] = None,
-) -> Set[LLMTestCaseParams]:
+    multiturn: bool,
+    required_params: Optional[
+        Union[Set[LLMTestCaseParams], Set[TurnParams]]
+    ] = None,
+) -> Union[Set[LLMTestCaseParams], Set[TurnParams]]:
     if required_params is None:
         required_params = set()
 
     for node in nodes:
-        if (
-            isinstance(node, TaskNode)
-            or isinstance(node, BinaryJudgementNode)
-            or isinstance(node, NonBinaryJudgementNode)
-        ):
-            if node.evaluation_params is not None:
-                required_params.update(node.evaluation_params)
-            extract_required_params(node.children, required_params)
+        if not multiturn:
+            if (
+                isinstance(node, TaskNode)
+                or isinstance(node, BinaryJudgementNode)
+                or isinstance(node, NonBinaryJudgementNode)
+            ):
+                if node.evaluation_params is not None:
+                    required_params.update(node.evaluation_params)
+                extract_required_params(
+                    node.children, multiturn, required_params
+                )
+        else:
+            if (
+                isinstance(node, ConversationalTaskNode)
+                or isinstance(node, ConversationalBinaryJudgementNode)
+                or isinstance(node, ConversationalNonBinaryJudgementNode)
+            ):
+                if node.evaluation_params is not None:
+                    required_params.update(node.evaluation_params)
+                extract_required_params(
+                    node.children, multiturn, required_params
+                )
 
     return required_params
 
 
 def copy_graph(original_dag: DeepAcyclicGraph) -> DeepAcyclicGraph:
     # This mapping avoids re-copying nodes that appear in multiple places.
-    visited: Dict[BaseNode, BaseNode] = {}
-
-    def copy_node(node: BaseNode) -> BaseNode:
+    visited: Union[
+        Dict[BaseNode, BaseNode],
+        Dict[ConversationalBaseNode, ConversationalBaseNode],
+    ] = {}
+
+    def copy_node(
+        node: Union[BaseNode, ConversationalBaseNode],
+    ) -> Union[BaseNode, ConversationalBaseNode]:
         if node in visited:
             return visited[node]
 
@@ -98,22 +145,40 @@ def copy_graph(original_dag: DeepAcyclicGraph) -> DeepAcyclicGraph:
                 "_depth",
             ]
         }
-        if (
-            isinstance(node, TaskNode)
-            or isinstance(node, BinaryJudgementNode)
-            or isinstance(node, NonBinaryJudgementNode)
-        ):
-            copied_node = node_class(
-                **valid_args,
-                children=[copy_node(child) for child in node.children]
-            )
+        if not original_dag.multiturn:
+            if (
+                isinstance(node, TaskNode)
+                or isinstance(node, BinaryJudgementNode)
+                or isinstance(node, NonBinaryJudgementNode)
+            ):
+                copied_node = node_class(
+                    **valid_args,
+                    children=[copy_node(child) for child in node.children]
+                )
+            else:
+                if isinstance(node, VerdictNode) and node.child:
+                    copied_node = node_class(
+                        **valid_args, child=copy_node(node.child)
+                    )
+                else:
+                    copied_node = node_class(**valid_args)
         else:
-            if isinstance(node, VerdictNode) and node.child:
+            if (
+                isinstance(node, ConversationalTaskNode)
+                or isinstance(node, ConversationalBinaryJudgementNode)
+                or isinstance(node, ConversationalNonBinaryJudgementNode)
+            ):
                 copied_node = node_class(
-                    **valid_args, child=copy_node(node.child)
+                    **valid_args,
+                    children=[copy_node(child) for child in node.children]
                 )
             else:
-                copied_node = node_class(**valid_args)
+                if isinstance(node, ConversationalVerdictNode) and node.child:
+                    copied_node = node_class(
+                        **valid_args, child=copy_node(node.child)
+                    )
+                else:
+                    copied_node = node_class(**valid_args)
 
         visited[node] = copied_node
         return copied_node

deepeval/synthesizer/chunking/doc_chunker.py

@@ -1,47 +1,72 @@
-from typing import Optional, List, Dict, Union, Type
 import os
 
+from typing import Dict, List, Optional, Type, TYPE_CHECKING
+from types import SimpleNamespace
+
 from deepeval.models.base_model import DeepEvalBaseEmbeddingModel
 
-# check langchain availability
-try:
+
+if TYPE_CHECKING:
+    from chromadb.api.models.Collection import Collection
     from langchain_core.documents import Document as LCDocument
-    from langchain_text_splitters import TokenTextSplitter
     from langchain_text_splitters.base import TextSplitter
-    from langchain_community.document_loaders import (
-        PyPDFLoader,
-        TextLoader,
-        Docx2txtLoader,
-    )
     from langchain_community.document_loaders.base import BaseLoader
 
-    langchain_available = True
-except ImportError:
-    langchain_available = False
-
-# check chromadb availability
-try:
-    import chromadb
-    from chromadb import Metadata
-    from chromadb.api.models.Collection import Collection
-
-    chroma_db_available = True
-except ImportError:
-    chroma_db_available = False
 
-
-# Define a helper function to check availability
-def _check_chromadb_available():
-    if not chroma_db_available:
+# Lazy import caches
+_langchain_ns = None
+_chroma_mod = None
+_langchain_import_error = None
+_chroma_import_error = None
+
+
+def _get_langchain():
+    """Return a namespace of langchain classes, or raise ImportError with root cause."""
+    global _langchain_ns, _langchain_import_error
+    if _langchain_ns is not None:
+        return _langchain_ns
+    try:
+        from langchain_core.documents import Document as LCDocument  # type: ignore
+        from langchain_text_splitters import TokenTextSplitter  # type: ignore
+        from langchain_text_splitters.base import TextSplitter  # type: ignore
+        from langchain_community.document_loaders import (  # type: ignore
+            PyPDFLoader,
+            TextLoader,
+            Docx2txtLoader,
+        )
+        from langchain_community.document_loaders.base import BaseLoader  # type: ignore
+
+        _langchain_ns = SimpleNamespace(
+            LCDocument=LCDocument,
+            TokenTextSplitter=TokenTextSplitter,
+            TextSplitter=TextSplitter,
+            PyPDFLoader=PyPDFLoader,
+            TextLoader=TextLoader,
+            Docx2txtLoader=Docx2txtLoader,
+            BaseLoader=BaseLoader,
+        )
+        return _langchain_ns
+    except Exception as e:
+        _langchain_import_error = e
         raise ImportError(
-            "chromadb is required for this functionality. Install it via your package manager"
+            f"langchain, langchain_community, and langchain_text_splitters are required. Root cause: {e}"
         )
 
 
-def _check_langchain_available():
-    if not langchain_available:
+def _get_chromadb():
+    """Return the chromadb module, or raise ImportError with root cause."""
+    global _chroma_mod, _chroma_import_error
+    if _chroma_mod is not None:
+        return _chroma_mod
+    try:
+        import chromadb
+
+        _chroma_mod = chromadb
+        return _chroma_mod
+    except Exception as e:
+        _chroma_import_error = e
         raise ImportError(
-            "langchain, langchain_community, and langchain_text_splitters are required for this functionality. Install it via your package manager"
+            f"chromadb is required for this functionality. Root cause: {e}"
         )
 
 
@@ -50,22 +75,16 @@ class DocumentChunker:
         self,
         embedder: DeepEvalBaseEmbeddingModel,
     ):
-        _check_chromadb_available()
-        _check_langchain_available()
         self.text_token_count: Optional[int] = None  # set later
 
         self.source_file: Optional[str] = None
         self.chunks: Optional["Collection"] = None
-        self.sections: Optional[List[LCDocument]] = None
+        self.sections: Optional[List["LCDocument"]] = None
         self.embedder: DeepEvalBaseEmbeddingModel = embedder
         self.mean_embedding: Optional[float] = None
 
         # Mapping of file extensions to their respective loader classes
-        self.loader_mapping: Dict[str, Type[BaseLoader]] = {
-            ".pdf": PyPDFLoader,
-            ".txt": TextLoader,
-            ".docx": Docx2txtLoader,
-        }
+        self.loader_mapping: Dict[str, "Type[BaseLoader]"] = {}
 
     #########################################################
     ### Chunking Docs #######################################
@@ -74,7 +93,8 @@ class DocumentChunker:
     async def a_chunk_doc(
         self, chunk_size: int = 1024, chunk_overlap: int = 0
     ) -> "Collection":
-        _check_chromadb_available()
+        lc = _get_langchain()
+        chroma = _get_chromadb()
 
         # Raise error if chunk_doc is called before load_doc
         if self.sections is None or self.source_file is None:
@@ -85,13 +105,13 @@ class DocumentChunker:
         # Create ChromaDB client
         full_document_path, _ = os.path.splitext(self.source_file)
         document_name = os.path.basename(full_document_path)
-        client = chromadb.PersistentClient(path=f".vector_db/{document_name}")
+        client = chroma.PersistentClient(path=f".vector_db/{document_name}")
 
         collection_name = f"processed_chunks_{chunk_size}_{chunk_overlap}"
         try:
             collection = client.get_collection(name=collection_name)
         except Exception:
-            text_splitter: TextSplitter = TokenTextSplitter(
+            text_splitter: "TextSplitter" = lc.TokenTextSplitter(
                 chunk_size=chunk_size, chunk_overlap=chunk_overlap
             )
             # Collection doesn't exist, so create it and then add documents
@@ -108,7 +128,7 @@ class DocumentChunker:
                 batch_contents = contents[i:batch_end]
                 batch_embeddings = embeddings[i:batch_end]
                 batch_ids = ids[i:batch_end]
-                batch_metadatas: List["Metadata"] = [
+                batch_metadatas: List[dict] = [
                     {"source_file": self.source_file} for _ in batch_contents
                 ]
 
@@ -121,7 +141,8 @@ class DocumentChunker:
         return collection
 
     def chunk_doc(self, chunk_size: int = 1024, chunk_overlap: int = 0):
-        _check_chromadb_available()
+        lc = _get_langchain()
+        chroma = _get_chromadb()
 
         # Raise error if chunk_doc is called before load_doc
         if self.sections is None or self.source_file is None:
@@ -132,13 +153,13 @@ class DocumentChunker:
         # Create ChromaDB client
         full_document_path, _ = os.path.splitext(self.source_file)
         document_name = os.path.basename(full_document_path)
-        client = chromadb.PersistentClient(path=f".vector_db/{document_name}")
+        client = chroma.PersistentClient(path=f".vector_db/{document_name}")
 
         collection_name = f"processed_chunks_{chunk_size}_{chunk_overlap}"
         try:
             collection = client.get_collection(name=collection_name)
         except Exception:
-            text_splitter: TextSplitter = TokenTextSplitter(
+            text_splitter: "TextSplitter" = lc.TokenTextSplitter(
                 chunk_size=chunk_size, chunk_overlap=chunk_overlap
             )
             # Collection doesn't exist, so create it and then add documents
@@ -155,7 +176,7 @@ class DocumentChunker:
                 batch_contents = contents[i:batch_end]
                 batch_embeddings = embeddings[i:batch_end]
                 batch_ids = ids[i:batch_end]
-                batch_metadatas: List["Metadata"] = [
+                batch_metadatas: List[dict] = [
                     {"source_file": self.source_file} for _ in batch_contents
                 ]
 
@@ -172,17 +193,31 @@ class DocumentChunker:
     #########################################################
 
     def get_loader(self, path: str, encoding: Optional[str]) -> "BaseLoader":
+        lc = _get_langchain()
+        # set mapping lazily now that langchain classes exist
+        if not self.loader_mapping:
+            self.loader_mapping = {
+                ".pdf": lc.PyPDFLoader,
+                ".txt": lc.TextLoader,
+                ".docx": lc.Docx2txtLoader,
+                ".md": lc.TextLoader,
+                ".markdown": lc.TextLoader,
+                ".mdx": lc.TextLoader,
+            }
+
         # Find appropriate doc loader
         _, extension = os.path.splitext(path)
         extension = extension.lower()
-        loader: Optional[type[BaseLoader]] = self.loader_mapping.get(extension)
+        loader: Optional["Type[BaseLoader]"] = self.loader_mapping.get(
+            extension
+        )
         if loader is None:
             raise ValueError(f"Unsupported file format: {extension}")
 
-        # Load doc into sections and calculate total character count
-        if loader is TextLoader:
+        # Load doc into sections and calculate total token count
+        if loader is lc.TextLoader:
             return loader(path, encoding=encoding, autodetect_encoding=True)
-        elif loader is PyPDFLoader or loader is Docx2txtLoader:
+        elif loader in (lc.PyPDFLoader, lc.Docx2txtLoader):
             return loader(path)
         else:
             raise ValueError(f"Unsupported file format: {extension}")
@@ -200,5 +235,6 @@ class DocumentChunker:
         self.source_file = path
 
     def count_tokens(self, chunks: List["LCDocument"]):
-        counter = TokenTextSplitter(chunk_size=1, chunk_overlap=0)
+        lc = _get_langchain()
+        counter = lc.TokenTextSplitter(chunk_size=1, chunk_overlap=0)
         return len(counter.split_documents(chunks))