ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/main.py

@@ -1,270 +1,152 @@
-from wxo_agentic_evaluation.service_provider import get_provider
-from wxo_agentic_evaluation.resource_map import ResourceMap
-from wxo_agentic_evaluation.llm_user import LLMUser
-from wxo_agentic_evaluation.prompt.template_render import LlamaUserTemplateRenderer
-from wxo_agentic_evaluation.inference_backend import (
-    EvaluationController,
-    get_wxo_client,
-    WXOInferenceBackend
-)
-from typing import List
-from wxo_agentic_evaluation.evaluation_package import EvaluationPackage
-from wxo_agentic_evaluation.type import EvaluationData
+import dataclasses
+import json
+import os
+import pathlib
+from datetime import datetime
+from pathlib import Path
+
+import yaml
+from jsonargparse import CLI
 
 from wxo_agentic_evaluation.arg_configs import TestConfig
+from wxo_agentic_evaluation.clients import bootstrap_clients
+from wxo_agentic_evaluation.metrics.metrics import (
+    extract_metrics,
+    format_metrics_for_display,
+)
+from wxo_agentic_evaluation.runner import process_test_case
+from wxo_agentic_evaluation.scheduler import (
+    discover_tests,
+    enumerate_jobs,
+    run_jobs,
+)
 from wxo_agentic_evaluation.utils.utils import (
-    create_table,
     SummaryPanel,
-    safe_divide
+    create_table,
+    csv_dump,
 )
-from wxo_agentic_evaluation.utils import json_dump
-from wxo_agentic_evaluation.metrics.metrics import KnowledgeBaseMetricSummary, ToolCallAndRoutingMetrics, TextMatchType
-import os
-import json
-import traceback
-import yaml
-import dataclasses
-import glob
-import rich
-import csv
-from rich.progress import Progress
-from pathlib import Path
-from concurrent.futures import ThreadPoolExecutor
-from jsonargparse import CLI
-
+from wxo_agentic_evaluation.langfuse_collection import LangfuseCollection
+from wxo_agentic_evaluation.metrics.journey_success import JourneySuccessMetric
+from wxo_agentic_evaluation.metrics.tool_calling import ToolCalling
+from wxo_agentic_evaluation.langfuse_evaluation_package import EvaluationRunner, sample_aggregator
 
-def process_test_case(task_n, test_case, config, inference_backend, resource_map, llm_user):
-    summary_results_for_path = []
-    tc_name = os.path.basename(test_case).replace(".json", "")
-    with open(test_case, "r") as f:
-        test_case: EvaluationData = EvaluationData.model_validate(json.load(f))
 
-    evaluation_controller = EvaluationController(
-        wxo_inference_backend=inference_backend, llm_user=llm_user, config=config
-    )
-    rich.print(f"[bold magenta]Running test case: {tc_name}[/bold magenta]")
-    history, call_tracker, conversational_search_data = evaluation_controller.run(
-        task_n,
-        test_case.story,
-        agent_name=test_case.agent,
-        starting_user_input=test_case.starting_sentence,
-    )
-    result = list()
-    for message in history:
-        result.append(message.model_dump())
-
-    json_dump(
-        os.path.join(config.output_dir, "messages", tc_name + ".messages.json"), result
-    )
-
-    if len(conversational_search_data) > 0:
-        fn = tc_name + ".retrieval_context.json"
-        out_folder = Path(config.output_dir) / "knowledge_base_metrics"
-        out_folder.mkdir(exist_ok=True)
-        rc = [context.model_dump() for context in conversational_search_data]
-        json_dump(out_folder / fn, rc)
-
-    # If data annotation run, skip summary generation
-    if config.data_annotation_run:
-        return summary_results_for_path  # empty result set, skip summary
-
-    evaluation_package = EvaluationPackage(
-        test_case_name=tc_name,
-        messages=history,
-        ground_truth=test_case,
-        conversational_search_data=conversational_search_data,
-        resource_map=resource_map
-    )
-    (
-        keyword_semantic_matches,
-        knowledge_base_metrics,
-        messages_with_reason,
-        metrics,
-    ) = evaluation_package.generate_summary()
-    temp = []
-    for message in messages_with_reason:
-        temp.append(message.model_dump())
-    json_dump(
-        os.path.join(config.output_dir, "messages", tc_name + ".messages.analyze.json"),
-        temp,
-    )
+def main(config: TestConfig):
+    # setup
+    clients = bootstrap_clients(config)
+    if not getattr(config, "skip_available_results", False):
+        ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        config.output_dir = os.path.join(config.output_dir, ts)
+
+    if not config.skip_legacy_evaluation:
+        knowledge_base_output_folder = (
+            Path(config.output_dir) / "knowledge_base_metrics"
+        )
+        knowledge_base_output_folder.mkdir(exist_ok=True, parents=True)
+        detailed_rag_output_file = (
+            knowledge_base_output_folder / "knowledge_base_detailed_metrics.json"
+        )
+        summary_rag_output_file = (
+            Path(config.output_dir) / "knowledge_base_summary_metrics.json"
+        )
+        os.makedirs(os.path.join(config.output_dir, "messages"), exist_ok=True)
 
-    json_dump(
-        os.path.join(config.output_dir, "messages", tc_name + ".metrics.json"),
-        metrics.model_dump(),
+    # discover & schedule tests
+    test_cases = discover_tests(
+        config.test_paths, config.enable_recursive_search
     )
-
-    metrics.dataset_name = tc_name
-    metrics.avg_resp_time = (
-        sum(call_tracker.generic) + sum(call_tracker.tool_call)
-    ) / (len(call_tracker.generic) + len(call_tracker.tool_call))
-    metrics.avg_resp_time = round(metrics.avg_resp_time, 2)
-
-    summary_results_for_path.append((metrics, knowledge_base_metrics))
-
-    return summary_results_for_path
-
-
-def main(config: TestConfig):
-    executor = ThreadPoolExecutor(max_workers=config.num_workers)
-    wxo_client = get_wxo_client(
-        config.auth_config.url, config.auth_config.tenant_name, config.auth_config.token
+    jobs = enumerate_jobs(
+        test_cases,
+        config.n_runs,
+        config.skip_available_results,
+        config.output_dir,
     )
-    resource_map = ResourceMap(wxo_client)
-    inference_backend = WXOInferenceBackend(wxo_client=wxo_client)
-    llm_user = LLMUser(
-        wai_client=get_provider(config=config.provider_config, model_id=config.llm_user_config.model_id),
-        template=LlamaUserTemplateRenderer(config.llm_user_config.prompt_config),
-        user_response_style=config.llm_user_config.user_response_style,
+    results = run_jobs(
+        jobs, config, clients, process_test_case, config.num_workers
     )
 
-    print(f"Running evaluation with tenant {config.auth_config.tenant_name}")
+    # extract
+    tool_metrics, kb_summary, custom_metrics = extract_metrics(results)
 
-    results_list = []
-
-    knowledge_base_output_folder = Path(config.output_dir) / "knowledge_base_metrics"
-    knowledge_base_output_folder.mkdir(exist_ok=True, parents=True)
-    detailed_rag_output_file = (
-        knowledge_base_output_folder / "knowledge_base_detailed_metrics.json"
-    )
-    summary_rag_output_file = (
-        Path(config.output_dir) / "knowledge_base_summary_metrics.json"
-    )
-
-    os.makedirs(os.path.join(config.output_dir, "messages"), exist_ok=True)
-    available_res = set()
-    if config.skip_available_results:
-        available_res = set(
-            [
-                os.path.basename(f).replace(".messages", "")
-                for f in glob.glob(
-                    os.path.join(config.output_dir, "messages", "*.messages.json")
-                )
-            ]
+    if not config.skip_legacy_evaluation:
+        # write results
+        csv_dump(
+            pathlib.Path(config.output_dir) / "summary_metrics.csv",
+            rows=[metric.model_dump() for metric in tool_metrics],
         )
-
-    test_cases = []
-    for test_path in config.test_paths:
-        if os.path.isdir(test_path):
-            test_path = os.path.join(test_path, "*.json")
-        test_cases.extend(sorted(glob.glob(test_path)))
-    
-    futures = []
-    task_n = 0
-    for test_case in test_cases:
-        if not test_case.endswith(".json") or test_case.endswith("agent.json"):
-            continue
-        if config.skip_available_results:
-            if test_case in available_res:
-                print(f"Skipping test case {test_case} as results already exist.")
-                continue
-
-        future = executor.submit(
-            process_test_case,
-            task_n,
-            test_case,
-            config,
-            inference_backend,
-            resource_map,
-            llm_user,
+        for file_path, key in [
+            (detailed_rag_output_file, "detailed"),
+            (summary_rag_output_file, "summary"),
+        ]:
+            with open(file_path, "w+", encoding="utf-8") as f:
+                json.dump(kb_summary.model_dump(by_alias=True)[key], f, indent=4)
+
+        # print results
+        SummaryPanel(kb_summary).print()
+        tool_table = create_table(
+            format_metrics_for_display(tool_metrics), title="Agent Metrics"
+        )
+        if tool_table:
+            tool_table.print()
+        if any(cm.custom_metrics for cm in custom_metrics):
+            rows = []
+            for cm in custom_metrics:
+                row = {"dataset_name": cm.dataset_name}
+                for m in cm.custom_metrics:
+                    row[m.eval_name] = str(
+                        m.value
+                    )  # Convert to string to avoid type issues
+                rows.append(row)
+            custom_metrics_table = create_table(rows, title="Custom Metrics")
+            if custom_metrics_table:
+                custom_metrics_table.print()
+    else:
+        collection_name = os.path.basename(config.output_dir) + "_collection"
+        collection = LangfuseCollection(
+            name=collection_name,
+            description="",
+        )
+        dataset_paths = []
+        session_ids = []
+        for test_case in test_cases:
+            name = os.path.basename(test_case).replace(".json", "")
+            with open(os.path.join(config.output_dir, f"{name}.metadata.json"), "r") as f:
+                metadata = json.load(f)
+            session_id = metadata["thread_id"]
+            dataset_paths.append(test_case)
+            session_ids.append(session_id)
+        
+        collection.upload(paths=dataset_paths)
+        
+        langfuse_collection = LangfuseCollection(name=collection_name)
+
+        journey_sucess_metric = JourneySuccessMetric()
+        tool_calling = ToolCalling()
+
+        run = EvaluationRunner(
+            evaluation_name=os.path.basename(config.output_dir) + "_evaluation",
+            run_name=os.path.basename(config.output_dir) + "_run",
+            session_ids=session_ids,
+            collection=langfuse_collection,
+            metrics=[journey_sucess_metric, tool_calling],
+            aggregator=sample_aggregator
         )
 
-        futures.append((test_case, future))
-        task_n += 1
-
-    if futures:
-        with Progress() as progress:
-            task1 = progress.add_task(
-                f"[purple]Evaluating {len(futures)} tasks...", total=len(futures)
-            )
-            for test_case, future in futures:
-                try:
-                    results_list.extend(future.result())
-                except Exception as e:
-                    rich.print(f"test case {test_case} fails with {e}")
-                    traceback.print_exc()
-                finally:
-                    progress.update(task1, advance=1)
-
-    tool_call_metrics = [metric[0] for metric in results_list]
-    knowledge_base_metrics = [metric[1] for metric in results_list]
-
-    rag_metric_summary = KnowledgeBaseMetricSummary(
-        knowledge_base_metrics=knowledge_base_metrics
-    )
-    SummaryPanel(rag_metric_summary).print()
-
-    with open(detailed_rag_output_file, "w+", encoding="utf-8") as f:
-        json.dump(rag_metric_summary.model_dump(by_alias=True)["detailed"], f, indent=4)
-
-    with open(summary_rag_output_file, "w+", encoding="utf-8") as f:
-        json.dump(rag_metric_summary.model_dump(by_alias=True)["summary"], f, indent=4)
-
-    if len(tool_call_metrics) > 0:
-        # remove the average row if exist
-        tool_call_metrics = [
-            row for row in tool_call_metrics if row.dataset_name != "Summary (Average)"
-        ]
-
-        def filter_display_only_values(tool_call_metric: ToolCallAndRoutingMetrics):
-            row = {"Dataset": tool_call_metric.dataset_name, "Total Steps": tool_call_metric.total_steps,
-                   "LLM Steps": tool_call_metric.llm_step, "Total Tool Calls":tool_call_metric.total_tool_calls, "Tool Call Precision": tool_call_metric.tool_call_precision, "Tool Call Recall": tool_call_metric.tool_call_recall,
-                   "Agent Routing Accuracy": tool_call_metric.agent_routing_accuracy, "Text Match": tool_call_metric.text_match, "Journey Success": tool_call_metric.is_success, "Avg Resp Time (sec)": tool_call_metric.avg_resp_time}
-            return row
-
-        def create_avg_row(metrics: List[dict]):
-            avg_row = {"Dataset": "Summary (Average)", "Total Steps": 0,
-                   "LLM Steps": 0, "Total Tool Calls":0,  "Tool Call Precision": 0, "Tool Call Recall": 0, "Agent Routing Accuracy": 0,
-                   "Text Match": 0, "Journey Success": 0, "Avg Resp Time (sec)": 0}
-            if metrics:
-                for row in metrics:
-                    avg_row["Total Steps"] += row["Total Steps"]
-                    avg_row["LLM Steps"] += row["LLM Steps"]
-                    avg_row["Total Tool Calls"] += row["Total Tool Calls"]
-                    avg_row["Tool Call Precision"] += row["Tool Call Precision"]
-                    avg_row["Tool Call Recall"] += row["Tool Call Recall"]
-                    avg_row["Agent Routing Accuracy"] += row["Agent Routing Accuracy"]
-                    avg_row["Text Match"] += row["Text Match"] == TextMatchType.text_match.value
-                    avg_row["Journey Success"] += row["Journey Success"]
-                    avg_row["Avg Resp Time (sec)"] += row["Avg Resp Time (sec)"]
-
-                avg_row["Total Steps"] = round(safe_divide(avg_row["Total Steps"], len(metrics)), 2)
-                avg_row["LLM Steps"] = round(safe_divide(avg_row["LLM Steps"], len(metrics)), 2)
-                avg_row["Total Tool Calls"] = round(safe_divide(avg_row["Total Tool Calls"], len(metrics)), 2)
-                avg_row["Tool Call Precision"] = round(safe_divide(avg_row["Tool Call Precision"], len(metrics)), 2)
-                avg_row["Tool Call Recall"] = round(safe_divide(avg_row["Tool Call Recall"], len(metrics)), 2)
-                avg_row["Agent Routing Accuracy"] = round(safe_divide(avg_row["Agent Routing Accuracy"], len(metrics)), 2)
-                avg_row["Text Match"] = round(safe_divide(avg_row["Text Match"], len([row for row in metrics if row["Text Match"] != TextMatchType.text_match.na])), 2)
-                avg_row["Journey Success"] = round(safe_divide(avg_row["Journey Success"], len(metrics)), 2)
-                avg_row["Avg Resp Time (sec)"] = round(safe_divide(avg_row["Avg Resp Time (sec)"], len(metrics)), 2)
-            return avg_row
-
-        tool_call_metrics_for_display = []
-        for row in tool_call_metrics:
-            tool_call_metrics_for_display.append(filter_display_only_values(row))
-        tool_call_metrics_for_display.append(create_avg_row(tool_call_metrics_for_display))
-        tool_call_table_for_display = create_table(tool_call_metrics_for_display)
-    
-        if tool_call_table_for_display:
-            tool_call_table_for_display.print()
-
-    if len(tool_call_metrics) > 0:
-        tool_call_metrics = [metric.model_dump() for metric in tool_call_metrics]
-        output_file = os.path.join(config.output_dir, "summary_metrics.csv")
-        header = list(tool_call_metrics[0].keys())
-
-        with open(output_file, "w") as file:
-            csv_writer = csv.writer(file)
-            csv_writer.writerow(header)
-            for entry in tool_call_metrics:
-                csv_writer.writerow([entry[name] for name in header])
+        run.evaluate()
 
+    # persist config
     with open(
-        os.path.join(config.output_dir, "config.yml"), "w", encoding="utf-8"
+        pathlib.Path(config.output_dir) / "config.yml", "w", encoding="utf-8"
     ) as f:
         yaml.safe_dump(dataclasses.asdict(config), f)
 
-    print(f"Results saved to {config.output_dir}")
+    if not config.skip_legacy_evaluation:
+        print(f"Results saved to {config.output_dir}")
+    else:
+        print(f"Config and metadata saved to {config.output_dir}")
+        print(f"Langfuse Evaluation run completed for collection {collection_name}:")
+        for session_id in session_ids:
+            print(f" - http://localhost:3010/project/orchestrate-lite/sessions/{session_id}")
 
 
 if __name__ == "__main__":

wxo_agentic_evaluation/metrics/__init__.py

@@ -0,0 +1,15 @@
+from wxo_agentic_evaluation.metrics.evaluations import Evaluation
+from wxo_agentic_evaluation.metrics.metrics import (
+    Annotation,
+    FailedSemanticTestCases,
+    FailedStaticTestCases,
+)
+
+def argument_matching(expected, actual):
+    if actual is None:
+        return False
+    for field in actual:
+        if field not in expected:
+            return False
+
+    return True

wxo_agentic_evaluation/metrics/dummy_metric.py

@@ -0,0 +1,16 @@
+from langfuse.api.resources.commons.types.score_data_type import ScoreDataType
+
+from wxo_agentic_evaluation.metrics import Evaluation
+from wxo_agentic_evaluation.metrics.metrics import LangfuseMetric
+
+class DummyMetric(Evaluation):
+    def __init__(self, llm_client = None):
+        super().__init__(llm_client)
+    
+    def evaluate(self, messages, ground_truth, extracted_context, metadata = ..., **kwargs):
+        return LangfuseMetric(
+            eval_name="dummy_metric",
+            value=True,
+            metadata=metadata,
+            data_type="BOOLEAN",
+        )

wxo_agentic_evaluation/metrics/evaluations.py

@@ -0,0 +1,107 @@
+import os
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Optional
+
+from wxo_agentic_evaluation.metrics.metrics import Metric
+from wxo_agentic_evaluation.prompt.template_render import LLMaaJTemplateRenderer
+from wxo_agentic_evaluation.service_provider.provider import Provider
+from wxo_agentic_evaluation.type import Message, OrchestrateDataset
+from wxo_agentic_evaluation.utils.messages_parser import ParsedMessages
+
+root_dir: str = os.path.dirname(os.path.dirname(__file__))
+LLMAAJ_PROMPT_PATH = os.path.join(root_dir, "prompt", "llmaaj_prompt.jinja2")
+
+
+class Evaluation(ABC):
+    """Abstract base class for all evaluations."""
+
+    def __init__(self, llm_client: Optional[Provider] = None) -> None:
+        self._llm_client = llm_client
+
+    @property
+    def llm_client(self) -> Any:
+        """Access client, require it if used."""
+        if self._llm_client is None:
+            raise RuntimeError(
+                f"{self.__class__.__name__} requires a client, but none was provided"
+            )
+        return self._llm_client
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Unique name for the evaluator."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def evaluate(
+        self,
+        messages: list[Message],
+        ground_truth: OrchestrateDataset,
+        extracted_context: Dict[str, Any],
+    ) -> Optional[Metric]:
+        """
+        Evaluation method.
+
+        Args:
+            messages: agent and user conversational messages (includes tool calls)
+            ground_truth: ground truth data
+            extracted_context: dictionary containing data derived from the messages
+
+        Returns:
+            Metic
+        """
+        raise NotImplementedError
+
+
+class LLMaaJEvaluation(Evaluation, ABC):
+    """Evaluation metric for LLMaaJ."""
+
+    @property
+    @abstractmethod
+    def llmaaj_instructions(self) -> str:
+        """LLMaaJ instructions for the evaluator."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def format_llm_output(self, string: str) -> int | float | bool | str:
+        """Format the output of the LLMaaJ query."""
+        raise NotImplementedError
+
+    @property
+    def selected_context_keys(self) -> set[str]:
+        """Override to implement context keys to pass to the prompt."""
+        return set()
+
+    def select_context(
+        self, extracted_context: Dict[str, Any]
+    ) -> dict[str, Any]:
+        """Additional context to be added to the prompt."""
+        selected_context = {
+            key: value
+            for key, value in extracted_context.items()
+            if key in self.selected_context_keys
+        }
+
+        return selected_context
+
+    def evaluate(
+        self,
+        messages: list[Message],
+        ground_truth: OrchestrateDataset,
+        extracted_context: Dict[str, Any],
+    ) -> Optional[Metric]:
+        renderer = LLMaaJTemplateRenderer(LLMAAJ_PROMPT_PATH)
+        parsed = ParsedMessages(messages=messages)
+        if parsed.user_input is None or parsed.agent_response is None:
+            return None
+        context = str(self.select_context(extracted_context))
+        prompt = renderer.render(
+            user_input=parsed.user_input,
+            agent_answer=parsed.agent_response,
+            llmaaj_instructions=self.llmaaj_instructions,
+            context=context,
+        )
+        score_str = self.llm_client.query(prompt)
+        value = self.format_llm_output(score_str)
+        return Metric(eval_name=self.name, value=value)

wxo_agentic_evaluation/metrics/journey_success.py

@@ -0,0 +1,137 @@
+import json
+from collections import defaultdict
+
+from langfuse.api.resources.commons.types.score_data_type import ScoreDataType
+
+from wxo_agentic_evaluation.metrics import Evaluation, argument_matching
+from wxo_agentic_evaluation.metrics.metrics import LangfuseMetric
+
+## fix later
+from wxo_agentic_evaluation.otel_parser.parser_types import (
+    Message as OtelMessage,
+)
+
+"""
+    - hyphens are not allowed in python function names, so it is safe to use as a dummy function name
+    - purpose behind `DUMMY_GRAPH_NODE_NAME` is to append
+    a dummy node to the ground truth and the labelled messages to take into account 
+    single, summary step goals.
+"""
+DUMMY_GRAPH_NODE_NAME = "dummy-goal"
+
+
+class JourneySuccessMetric(Evaluation):
+    def __init__(self, llm_client=None):
+        super().__init__(llm_client)
+        self.is_strict = True
+
+    @property
+    def name(self):
+        return "Journey Success"
+
+    def find_terminal_nodes(self, graph: dict[str, list[str]]) -> set[str]:
+        """Finds terminal nodes (nodes with no outgoing edges).
+
+        Args:
+            graph: the input graph
+
+        Returns:
+            a set of the terminal nodes
+        """
+
+        seen_nodes = set()  # track seen nodes
+        non_terminal_nodes = set()  # track nodes with children
+
+        for node in graph:
+            seen_nodes.add(node)
+            if graph[node]:
+                non_terminal_nodes.add(node)
+                for n in graph[node]:
+                    seen_nodes.add(n)
+        return seen_nodes - non_terminal_nodes
+
+    def is_topological_sort(
+        self,
+        graph: dict[str, list[str]],
+        ordering: list[str],
+        is_strict: bool = True,
+    ) -> bool:
+        """Graph traversal to check if every node in `graph` is visited in `ordering` only after all its dependencies are visited.
+
+        Args:
+            graph: the graph representing the ground truth, where keys represent nodes and values represent its dependent nodes
+            ordering: the nodes visited, in order
+
+        Returns:
+            Boolean representing if `ordering` visits all nodes in a valid order based on the dependencies in graph.
+        """
+        # No keyword match or goal details were achieved
+        if not ordering:
+            return False
+
+        if is_strict:
+            # strict matching: only consider most recent tool call
+            position = {node: [i] for i, node in enumerate(ordering)}
+        else:
+            # lenient matching: consider all tool calls (account for all indexes of the node)
+            position = defaultdict(list)
+            for i, node in enumerate(ordering):
+                position[node].append(i)
+
+        terminal_nodes = self.find_terminal_nodes(graph)
+        # adds a dummy node for each terminal node
+        next_idx = (
+            max(val for values in position.values() for val in values) + 1
+        )
+
+        for n in terminal_nodes:
+            graph[n] = [DUMMY_GRAPH_NODE_NAME]
+            graph[DUMMY_GRAPH_NODE_NAME] = []
+            position[DUMMY_GRAPH_NODE_NAME] = [next_idx]
+            next_idx += 1
+
+        for node in graph:
+            for child_nodes in graph[node]:
+                # Current node/children doesn't show up in made calls
+                if node not in position or child_nodes not in position:
+                    return False
+                # Current node doesn't show up before any of its child
+                # all index in current nodes are larger than every child nodes' index
+                if all(
+                    curr >= max(position[child_nodes])
+                    for curr in position[node]
+                ):
+                    return False
+        return True
+
+    def evaluate(
+        self, messages, ground_truth, extracted_context, metadata, **kwargs
+    ):
+        labeled_messages = extracted_context.get("labeled_messages")
+        correct_tool_calls = []
+
+        for message_idx, matching_goal_details in labeled_messages.items():
+            msg_tool_call = messages[message_idx]
+            msg_tool_call = msg_tool_call.tool_calls[0].function
+            for goal_detail in matching_goal_details:
+                args_match = argument_matching(
+                    expected=goal_detail.args,
+                    actual=None if len(msg_tool_call.arguments) == 0 else json.loads(msg_tool_call.arguments),
+                )
+
+                if args_match:
+                    correct_tool_calls.append(goal_detail.name)
+
+        is_topological_sort = self.is_topological_sort(
+            graph=ground_truth.goals,
+            ordering=correct_tool_calls,
+            is_strict=self.is_strict,
+        )
+
+        return LangfuseMetric(
+            eval_name=self.name,
+            comment="",
+            value=is_topological_sort,
+            data_type="NUMERIC",
+            metadata=metadata,
+        )