judgeval 0.0.30__py3-none-any.whl → 0.0.32__py3-none-any.whl

judgeval/integrations/langgraph.py

@@ -22,9 +22,8 @@ from langchain_core.documents import Document
 class JudgevalCallbackHandler(BaseCallbackHandler):
     def __init__(self, tracer: Tracer):
         self.tracer = tracer
-        self.trace_client = tracer.get_current_trace() if tracer.get_current_trace() else None
         self.previous_spans = [] # stack of previous spans
-        self.finished = False
+        self.created_trace = False
 
         # Attributes for users to access
         self.previous_node = None
@@ -33,43 +32,58 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
         self.executed_tools = []
 
     def start_span(self, name: str, span_type: SpanType = "span"):
+        current_trace = self.tracer.get_current_trace()
         start_time = time.time()
+
+        # Generate a unique ID for *this specific span invocation*
+        span_id = str(uuid.uuid4())
+        
+        parent_span_id = current_trace.get_current_span()
+        token = current_trace.set_current_span(span_id) # Set *this* span's ID as the current one
+        
+        current_depth = 0
+        if parent_span_id and parent_span_id in current_trace._span_depths:
+            current_depth = current_trace._span_depths[parent_span_id] + 1
         
+        current_trace._span_depths[span_id] = current_depth # Store depth by span_id
         # Record span entry
-        self.trace_client.add_entry(TraceEntry(
+        current_trace.add_entry(TraceEntry(
             type="enter",
+            span_id=span_id,
+            trace_id=current_trace.trace_id,
+            parent_span_id=parent_span_id,
             function=name,
-            depth=self.trace_client.tracer.depth,
+            depth=current_depth,
             message=name,
-            timestamp=start_time,
+            created_at=start_time,
             span_type=span_type
         ))
 
-        self.trace_client.tracer.depth += 1
-        self.previous_spans.append(self.trace_client._current_span)
-        self.trace_client._current_span = name
+        self.previous_spans.append(token)
         self._start_time = start_time
     
-    def end_span(self, name: str, span_type: SpanType = "span"):
-        self.trace_client.tracer.depth -= 1
+    def end_span(self, span_type: SpanType = "span"):
+        current_trace = self.tracer.get_current_trace()
         duration = time.time() - self._start_time
+        span_id = current_trace.get_current_span()
+        exit_depth = current_trace._span_depths.get(span_id, 0) # Get depth using this span's ID
         
         # Record span exit
-        self.trace_client.add_entry(TraceEntry(
+        current_trace.add_entry(TraceEntry(
             type="exit",
-            function=name,
-            depth=self.trace_client.tracer.depth,
-            message=f"{name}",
-            timestamp=time.time(),
+            span_id=span_id,
+            trace_id=current_trace.trace_id,
+            depth=exit_depth,
+            created_at=time.time(),
             duration=duration,
             span_type=span_type
         ))
-        self.trace_client._current_span = self.previous_spans.pop()
-
-        if self.trace_client.tracer.depth == 0:
+        current_trace.reset_current_span(self.previous_spans.pop())
+        if exit_depth == 0:
             # Save the trace if we are the root, this is when users dont use any @observe decorators
-            self.trace_client.save(empty_save=False, overwrite=True)
-            self.trace_client._current_trace = None
+            trace_id, trace_data = current_trace.save(overwrite=True)
+            self._trace_id = trace_id
+            current_trace = None
                  
     def on_retriever_start(
         self,
@@ -85,9 +99,9 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
         name = "RETRIEVER_CALL"
         if serialized and "name" in serialized:
             name = f"RETRIEVER_{serialized['name'].upper()}"
-        
+        current_trace = self.tracer.get_current_trace()
         self.start_span(name, span_type="retriever")
-        self.trace_client.record_input({
+        current_trace.record_input({
             'query': query,
             'tags': tags,
             'metadata': metadata,
@@ -103,6 +117,7 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
         **kwargs: Any
     ) -> Any:
         # Process the retrieved documents into a format suitable for logging
+        current_trace = self.tracer.get_current_trace()
         doc_summary = []
         for i, doc in enumerate(documents):
             # Extract key information from each document
@@ -114,13 +129,13 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
             doc_summary.append(doc_data)
         
         # Record the document data
-        self.trace_client.record_output({
+        current_trace.record_output({
             "document_count": len(documents),
             "documents": doc_summary
         })
         
         # End the retriever span
-        self.end_span(self.trace_client._current_span, span_type="retriever")
+        self.end_span(span_type="retriever")
 
     def on_chain_start(
         self,
@@ -134,29 +149,26 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
         **kwargs: Any
     ) -> None:
         # If the user doesnt use any @observe decorators, the first action in LangGraph workflows seems tohave this attribute, so we intialize our trace client here
+        current_trace = self.tracer.get_current_trace()
         if kwargs.get('name') == 'LangGraph':
-            if not self.trace_client:
+            if not current_trace:
+                self.created_trace = True
                 trace_id = str(uuid.uuid4())
                 project = self.tracer.project_name
-                trace = TraceClient(self.tracer, trace_id, trace_id, project_name=project, overwrite=False, rules=self.tracer.rules, enable_monitoring=self.tracer.enable_monitoring, enable_evaluations=self.tracer.enable_evaluations)
-                self.trace_client = trace
-                self.tracer._current_trace = trace # set the trace in the original tracer object
-                # Only save empty trace for the root call
-                self.trace_client.save(empty_save=True, overwrite=False)
-            
-            self.start_span("LangGraph", span_type="Main Function")
+                trace = TraceClient(self.tracer, trace_id, "Langgraph", project_name=project, overwrite=False, rules=self.tracer.rules, enable_monitoring=self.tracer.enable_monitoring, enable_evaluations=self.tracer.enable_evaluations)
+                self.tracer.set_current_trace(trace)
+                self.start_span("LangGraph", span_type="Main Function")
             
-        metadata = kwargs.get("metadata", {})
-        if node := metadata.get("langgraph_node"):
-            if node != self.previous_node:
-                # Track node execution
-                self.trace_client.visited_nodes.append(node)
-                self.trace_client.executed_node_tools.append(node)
-                self.trace_client.record_input({
-                    'args': inputs,
-                    'kwargs': kwargs
-                })
-            self.previous_node = node
+        node = metadata.get("langgraph_node")
+        if node != None and node != self.previous_node:
+             self.start_span(node, span_type="node")
+             self.executed_node_tools.append(node)
+             self.executed_nodes.append(node)
+             current_trace.record_input({
+                 'args': inputs,
+                 'kwargs': kwargs
+             })
+        self.previous_node = node
     
     def on_chain_end(
         self,
@@ -167,14 +179,13 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
         tags: Optional[List[str]] = None,
         **kwargs: Any,
     ) -> Any:   
-        if outputs == "__end__":
-            self.finished = True
+        current_trace = self.tracer.get_current_trace()
         if tags is not None and any("graph:step" in tag for tag in tags):
-            self.trace_client.record_output(outputs)
-            self.end_span(self.trace_client._current_span, span_type="node")
+            current_trace.record_output(outputs)
+            self.end_span(span_type="node")
 
-            if self.finished:
-                self.end_span(self.trace_client._current_span, span_type="Main Function")
+        if self.created_trace and (outputs == "__end__" or (not kwargs and not tags)):
+            self.end_span(span_type="Main Function")
     
     def on_chain_error(
         self,
@@ -184,9 +195,9 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,
     ) -> Any:
-        print(f"Chain error: {error}")
-        self.trace_client.record_output(error)
-        self.end_span(self.trace_client._current_span, span_type="node")
+        current_trace = self.tracer.get_current_trace()
+        current_trace.record_output(error)
+        self.end_span(span_type="node")
 
     def on_tool_start(
         self,
@@ -199,19 +210,21 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
     ):
         name = serialized["name"]
         self.start_span(name, span_type="tool")
+        current_trace = self.tracer.get_current_trace()
         if name:
             # Track tool execution
-            self.trace_client.executed_tools.append(name)
+            current_trace.executed_tools.append(name)
             node_tool = f"{self.previous_node}:{name}" if self.previous_node else name
-            self.trace_client.executed_node_tools.append(node_tool)
-        self.trace_client.record_input({
-            'args': input_str,
-            'kwargs': kwargs
-        })
+            current_trace.executed_node_tools.append(node_tool)
+            current_trace.record_input({
+                'args': input_str,
+                'kwargs': kwargs
+            })
 
     def on_tool_end(self, output: Any, *, run_id: UUID, parent_run_id: Optional[UUID] = None, **kwargs: Any) -> Any:
-        self.trace_client.record_output(output)
-        self.end_span(self.trace_client._current_span, span_type="tool")
+        current_trace = self.tracer.get_current_trace()
+        current_trace.record_output(output)
+        self.end_span(span_type="tool")
 
     def on_tool_error(
         self,
@@ -221,9 +234,9 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,
     ) -> Any:
-        print(f"Tool error: {error}")
-        self.trace_client.record_output(error)
-        self.end_span(self.trace_client._current_span, span_type="tool")
+        current_trace = self.tracer.get_current_trace()
+        current_trace.record_output(error)
+        self.end_span(span_type="tool")
 
     def on_agent_action(
         self,
@@ -233,7 +246,7 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,
     ) -> Any:
-        print(f"Agent action: {action}")
+        pass
 
     def on_agent_finish(
         self,
@@ -243,7 +256,8 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,
     ) -> Any:
-        print(f"Agent finish: {finish}")
+        
+        pass
 
     def on_llm_start(
         self,
@@ -256,14 +270,16 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
     ) -> Any:    
         name = "LLM call"
         self.start_span(name, span_type="llm")
-        self.trace_client.record_input({
+        current_trace = self.tracer.get_current_trace()
+        current_trace.record_input({
             'args': prompts,
             'kwargs': kwargs
         })
 
     def on_llm_end(self, response: LLMResult, *, run_id: UUID, parent_run_id: Optional[UUID] = None, **kwargs: Any):
-        self.trace_client.record_output(response.generations[0][0].text)
-        self.end_span(self.trace_client._current_span, span_type="llm")
+        current_trace = self.tracer.get_current_trace()
+        current_trace.record_output(response.generations[0][0].text)
+        self.end_span(span_type="llm")
 
     def on_llm_error(
         self,
@@ -273,9 +289,9 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,
     ) -> Any:
-        print(f"LLM error: {error}")
-        self.trace_client.record_output(error)
-        self.end_span(self.trace_client._current_span, span_type="llm")
+        current_trace = self.tracer.get_current_trace()
+        current_trace.record_output(error)
+        self.end_span(span_type="llm")
 
     def on_chat_model_start(
         self,
@@ -297,7 +313,8 @@ class JudgevalCallbackHandler(BaseCallbackHandler):
             name = "LLM call"
 
         self.start_span(name, span_type="llm")
-        self.trace_client.record_input({
+        current_trace = self.tracer.get_current_trace()
+        current_trace.record_input({
             'args': str(messages),
             'kwargs': kwargs
         })

judgeval/judgment_client.py

@@ -10,6 +10,8 @@ from judgeval.data.datasets import EvalDataset, EvalDatasetClient
 from judgeval.data import (
     ScoringResult, 
     Example,
+    CustomExample,
+    Sequence,
 )
 from judgeval.scorers import (
     APIJudgmentScorer, 
@@ -20,8 +22,10 @@ from judgeval.scorers import (
 from judgeval.evaluation_run import EvaluationRun
 from judgeval.run_evaluation import (
     run_eval, 
-    assert_test
+    assert_test,
+    run_sequence_eval
 )
+from judgeval.data.sequence_run import SequenceRun
 from judgeval.judges import JudgevalJudge
 from judgeval.constants import (
     JUDGMENT_EVAL_FETCH_API_URL, 
@@ -78,66 +82,58 @@ class JudgmentClient(metaclass=SingletonMeta):
         project_name: str = "default_project",
         eval_run_name: str = "default_eval_run",
         override: bool = False,
+        append: bool = False,
         use_judgment: bool = True,
         ignore_errors: bool = True,
         rules: Optional[List[Rule]] = None
     ) -> List[ScoringResult]:
-        return self.run_evaluation(examples, scorers, model, aggregator, metadata, log_results, project_name, eval_run_name, override, use_judgment, ignore_errors, True, rules)
+        return self.run_evaluation(examples, scorers, model, aggregator, metadata, log_results, project_name, eval_run_name, override, append, use_judgment, ignore_errors, True, rules)
 
-    def run_evaluation(
-        self, 
-        examples: List[Example],
-        scorers: List[Union[ScorerWrapper, JudgevalScorer]],
+    def run_sequence_evaluation(
+        self,
+        sequences: List[Sequence],
         model: Union[str, List[str], JudgevalJudge],
+        scorers: List[Union[ScorerWrapper, JudgevalScorer]],
         aggregator: Optional[str] = None,
-        metadata: Optional[Dict[str, Any]] = None,
-        log_results: bool = True,
         project_name: str = "default_project",
-        eval_run_name: str = "default_eval_run",
-        override: bool = False,
+        eval_run_name: str = "default_eval_sequence",
         use_judgment: bool = True,
+        log_results: bool = True,
+        append: bool = False,
+        override: bool = False,
         ignore_errors: bool = True,
-        async_execution: bool = False,
         rules: Optional[List[Rule]] = None
     ) -> List[ScoringResult]:
-        """
-        Executes an evaluation of `Example`s using one or more `Scorer`s
-        
-        Args:
-            examples (List[Example]): The examples to evaluate
-            scorers (List[Union[ScorerWrapper, JudgevalScorer]]): A list of scorers to use for evaluation
-            model (Union[str, List[str], JudgevalJudge]): The model used as a judge when using LLM as a Judge
-            aggregator (Optional[str]): The aggregator to use for evaluation if using Mixture of Judges
-            metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run
-            log_results (bool): Whether to log the results to the Judgment API
-            project_name (str): The name of the project the evaluation results belong to
-            eval_run_name (str): A name for this evaluation run
-            override (bool): Whether to override an existing evaluation run with the same name
-            use_judgment (bool): Whether to use Judgment API for evaluation
-            ignore_errors (bool): Whether to ignore errors during evaluation (safely handled)
-            rules (Optional[List[Rule]]): Rules to evaluate against scoring results
-            
-        Returns:
-            List[ScoringResult]: The results of the evaluation
-        """
         try:
-            # Load appropriate implementations for all scorers
-            loaded_scorers: List[Union[JudgevalScorer, APIJudgmentScorer]] = []
+            loaded_scorers = []
             for scorer in scorers:
                 try:
                     if isinstance(scorer, ScorerWrapper):
-                        loaded_scorers.append(scorer.load_implementation(use_judgment=use_judgment))
+                        loaded_scorers.append(scorer.load_implementation())
                     else:
                         loaded_scorers.append(scorer)
                 except Exception as e:
                     raise ValueError(f"Failed to load implementation for scorer {scorer}: {str(e)}")
 
-            # Prevent using JudgevalScorer with rules - only APIJudgmentScorer allowed with rules
-            if rules and any(isinstance(scorer, JudgevalScorer) for scorer in loaded_scorers):
-                raise ValueError("Cannot use Judgeval scorers (only API scorers) when using rules. Please either remove rules or use only APIJudgmentScorer types.")
+            def get_all_sequences(root: Sequence) -> List[Sequence]:
+                all_sequences = [root]
+
+                for item in root.items:
+                    if isinstance(item, Sequence):
+                        all_sequences.extend(get_all_sequences(item))
+
+                return all_sequences
+
+            def flatten_sequence_list(sequences: List[Sequence]) -> List[Sequence]:
+                flattened = []
+                for seq in sequences:
+                    flattened.extend(get_all_sequences(seq))
+                return flattened
+            
+            flattened_sequences = flatten_sequence_list(sequences)
+            for sequence in flattened_sequences:
+                sequence.scorers = loaded_scorers
 
-            # Convert ScorerWrapper in rules to their implementations
-            loaded_rules = None
             if rules:
                 loaded_rules = []
                 for rule in rules:
@@ -161,57 +157,64 @@ class JudgmentClient(metaclass=SingletonMeta):
                         loaded_rules.append(new_rule)
                     except Exception as e:
                         raise ValueError(f"Failed to process rule '{rule.name}': {str(e)}")
-
-            eval = EvaluationRun(
-                log_results=log_results,
+                    
+            sequence_run = SequenceRun(
                 project_name=project_name,
                 eval_name=eval_run_name,
-                examples=examples,
-                scorers=loaded_scorers,
+                sequences=sequences,
                 model=model,
                 aggregator=aggregator,
-                metadata=metadata,
+                log_results=log_results,
+                append=append,
                 judgment_api_key=self.judgment_api_key,
-                rules=loaded_rules,
                 organization_id=self.organization_id
             )
-            return run_eval(eval, override, ignore_errors=ignore_errors, async_execution=async_execution)
+            return run_sequence_eval(sequence_run, override, ignore_errors, use_judgment)
         except ValueError as e:
-            raise ValueError(f"Please check your EvaluationRun object, one or more fields are invalid: \n{str(e)}")
+            raise ValueError(f"Please check your SequenceRun object, one or more fields are invalid: \n{str(e)}")
         except Exception as e:
             raise Exception(f"An unexpected error occurred during evaluation: {str(e)}")
-    
-    def evaluate_dataset(
+
+    def run_evaluation(
         self, 
-        dataset: EvalDataset,
+        examples: Union[List[Example], List[CustomExample]],
         scorers: List[Union[ScorerWrapper, JudgevalScorer]],
         model: Union[str, List[str], JudgevalJudge],
         aggregator: Optional[str] = None,
         metadata: Optional[Dict[str, Any]] = None,
-        project_name: str = "",
-        eval_run_name: str = "",
         log_results: bool = True,
+        project_name: str = "default_project",
+        eval_run_name: str = "default_eval_run",
+        override: bool = False,
+        append: bool = False,
         use_judgment: bool = True,
+        ignore_errors: bool = True,
+        async_execution: bool = False,
         rules: Optional[List[Rule]] = None
     ) -> List[ScoringResult]:
         """
-        Executes an evaluation of a `EvalDataset` using one or more `Scorer`s
+        Executes an evaluation of `Example`s using one or more `Scorer`s
         
         Args:
-            dataset (EvalDataset): The dataset containing examples to evaluate
+            examples (Union[List[Example], List[CustomExample]]): The examples to evaluate
             scorers (List[Union[ScorerWrapper, JudgevalScorer]]): A list of scorers to use for evaluation
             model (Union[str, List[str], JudgevalJudge]): The model used as a judge when using LLM as a Judge
             aggregator (Optional[str]): The aggregator to use for evaluation if using Mixture of Judges
             metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run
+            log_results (bool): Whether to log the results to the Judgment API
             project_name (str): The name of the project the evaluation results belong to
             eval_run_name (str): A name for this evaluation run
-            log_results (bool): Whether to log the results to the Judgment API
+            override (bool): Whether to override an existing evaluation run with the same name
             use_judgment (bool): Whether to use Judgment API for evaluation
+            ignore_errors (bool): Whether to ignore errors during evaluation (safely handled)
             rules (Optional[List[Rule]]): Rules to evaluate against scoring results
             
         Returns:
             List[ScoringResult]: The results of the evaluation
         """
+        if override and append:
+            raise ValueError("Cannot set both override and append to True. Please choose one.")
+
         try:
             # Load appropriate implementations for all scorers
             loaded_scorers: List[Union[JudgevalScorer, APIJudgmentScorer]] = []
@@ -253,12 +256,12 @@ class JudgmentClient(metaclass=SingletonMeta):
                         loaded_rules.append(new_rule)
                     except Exception as e:
                         raise ValueError(f"Failed to process rule '{rule.name}': {str(e)}")
-
-            evaluation_run = EvaluationRun(
+            eval = EvaluationRun(
                 log_results=log_results,
+                append=append,
                 project_name=project_name,
                 eval_name=eval_run_name,
-                examples=dataset.examples,
+                examples=examples,
                 scorers=loaded_scorers,
                 model=model,
                 aggregator=aggregator,
@@ -267,7 +270,7 @@ class JudgmentClient(metaclass=SingletonMeta):
                 rules=loaded_rules,
                 organization_id=self.organization_id
             )
-            return run_eval(evaluation_run)
+            return run_eval(eval, override, ignore_errors=ignore_errors, async_execution=async_execution)
         except ValueError as e:
             raise ValueError(f"Please check your EvaluationRun object, one or more fields are invalid: \n{str(e)}")
         except Exception as e:
@@ -292,6 +295,12 @@ class JudgmentClient(metaclass=SingletonMeta):
         dataset.judgment_api_key = self.judgment_api_key
         return self.eval_dataset_client.push(dataset, alias, project_name, overwrite)
     
+    def append_dataset(self, alias: str, examples: List[Example], project_name: str) -> bool:
+        """
+        Appends an `EvalDataset` to the Judgment platform for storage.
+        """
+        return self.eval_dataset_client.append(alias, examples, project_name)
+    
     def pull_dataset(self, alias: str, project_name: str) -> EvalDataset:
         """
         Retrieves a saved `EvalDataset` from the Judgment platform.
@@ -355,14 +364,7 @@ class JudgmentClient(metaclass=SingletonMeta):
         if eval_run.status_code != requests.codes.ok:
             raise ValueError(f"Error fetching eval results: {eval_run.json()}")
 
-        eval_run_result = [{}]
-        for result in eval_run.json():
-            result_id = result.get("id", "")
-            result_data = result.get("result", dict())
-            filtered_result = {k: v for k, v in result_data.items() if k in ScoringResult.__annotations__}
-            eval_run_result[0]["id"] = result_id
-            eval_run_result[0]["results"] = [ScoringResult(**filtered_result)]
-        return eval_run_result
+        return eval_run.json()
     
     def delete_eval(self, project_name: str, eval_run_names: List[str]) -> bool:
         """