ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/utils/open_ai_tool_extractor.py

@@ -0,0 +1,178 @@
+import ast
+import re
+from pathlib import Path
+from typing import Any, List, Mapping, Union
+
+
+class PythonTypeToJsonType:
+    OPTIONAL_PARAM_EXTRACT = re.compile(r"[Oo]ptional\[(\w+)\]")
+
+    @staticmethod
+    def python_to_json_type(python_annotation: str):
+        if not python_annotation:
+            return "string"
+        python_annotation = python_annotation.lower().strip()
+        if "str" == python_annotation:
+            return "string"
+        if "int" == python_annotation:
+            return "integer"
+        if "float" == python_annotation:
+            return "number"
+        if "bool" == python_annotation:
+            return "boolean"
+        if python_annotation.startswith("list"):
+            return "array"
+        if python_annotation.startswith("dict"):
+            return "object"
+        if python_annotation.startswith("optional"):
+            # extract the type within Optional[T]
+            inner_type = PythonTypeToJsonType.OPTIONAL_PARAM_EXTRACT.search(
+                python_annotation
+            ).group(1)
+            return PythonTypeToJsonType.python_to_json_type(inner_type)
+
+        return "string"
+
+
+class ToolExtractionOpenAIFormat:
+    @staticmethod
+    def get_default_arguments(node):
+        """Returns the default arguments (if any)
+
+        The default arguments are stored in args.default array.
+        Since, in Python, the default arguments only come after positional arguments,
+        we can index the argument array starting from the last `n` arguments, where n is
+        the length of the default arguments.
+
+        ex.
+        def add(a, b=5):
+           pass
+
+        Then we have,
+        args = [a, b]
+        defaults = [Constant(value=5)]
+
+        args[-len(defaults):] = [b]
+
+        (
+        "FunctionDef(
+            name='add',
+            args=arguments(
+                posonlyargs=[],
+                args=[
+                    arg(arg='a'), "
+                    "arg(arg='b')
+                ],
+                kwonlyargs=[],
+                kw_defaults=[],
+                defaults=[Constant(value=5)]), "
+            "body=[Return(value=BinOp(left=Name(id='a', ctx=Load()), op=Add(), "
+            "right=Name(id='b', ctx=Load())))], decorator_list=[], type_params=[])")
+        """
+        default_arguments = set()
+        num_defaults = len(node.args.defaults)
+        if num_defaults > 0:
+            for arg in node.args.args[-num_defaults:]:
+                default_arguments.add(arg)
+
+        return default_arguments
+
+    @staticmethod
+    def from_file(tools_path: Union[str, Path]) -> Mapping[str, Any]:
+        """Uses `extract_tool_signatures` function, but converts the response
+            to open-ai format
+
+            ```
+            function_spec = {
+                    "type": "function",
+                    "function": {
+                        "name": func_name,
+                        "description": description,
+                        "parameters": parameters,
+                    },
+                }
+        ```
+
+        """
+        tool_data = []
+        tools_path = Path(tools_path)
+
+        with tools_path.open("r", encoding="utf-8") as f:
+            code = f.read()
+
+        try:
+            parsed_code = ast.parse(code)
+            for node in parsed_code.body:
+                if isinstance(node, ast.FunctionDef):
+                    parameters = {
+                        "type": "object",
+                        "properties": {},
+                        "required": [],
+                    }
+                    function_name = node.name
+                    for arg in node.args.args:
+                        type_annotation = None
+                        if arg.arg == "self":
+                            continue
+                        if arg.annotation:
+                            type_annotation = ast.unparse(arg.annotation)
+
+                        parameter_type = (
+                            PythonTypeToJsonType.python_to_json_type(
+                                type_annotation
+                            )
+                        )
+                        parameters["properties"][arg.arg] = {
+                            "type": parameter_type,
+                            "description": "",  # todo
+                        }
+
+                        if (
+                            type_annotation
+                            and "Optional" not in type_annotation
+                        ):
+                            parameters["required"].append(arg.arg)
+
+                    default_arguments = (
+                        ToolExtractionOpenAIFormat.get_default_arguments(node)
+                    )
+                    for arg_name in parameters["required"]:
+                        if arg_name in default_arguments:
+                            parameters.remove(arg_name)
+
+                    open_ai_format_fn = {
+                        "type": "function",
+                        "function": {
+                            "name": function_name,
+                            "parameters": parameters,
+                            "description": ast.get_docstring(
+                                node
+                            ),  # fix (does not do :params)
+                        },
+                    }
+                    tool_data.append(open_ai_format_fn)
+
+        except Exception as e:
+            print(f"Warning: Failed to parse {tools_path}: {str(e)}")
+
+        return tool_data
+
+    @staticmethod
+    def from_path(tools_path: Union[str, Path]) -> List[Mapping[str, Any]]:
+        tools_path = Path(tools_path)
+        files_to_parse = []
+        all_tools = []
+
+        if tools_path.is_file():
+            files_to_parse.append(tools_path)
+        elif tools_path.is_dir():
+            files_to_parse.extend(tools_path.glob("**/*.py"))
+        else:
+            raise ValueError(
+                f"Tools path {tools_path} is neither a file nor directory"
+            )
+
+        for file_path in files_to_parse:
+            all_tools.extend(ToolExtractionOpenAIFormat.from_file(file_path))
+
+        return all_tools

wxo_agentic_evaluation/utils/parsers.py

@@ -0,0 +1,71 @@
+from typing import Any, List, Mapping, Optional
+
+from wxo_agentic_evaluation.metrics.metrics import (
+    Annotation,
+    FailedSemanticTestCases,
+    FailedStaticTestCases,
+)
+
+
+class ReferencelessEvalParser:
+    @staticmethod
+    def static_parser(
+        static_metrics: Mapping[str, Mapping[str, Any]],
+    ) -> List[FailedStaticTestCases]:
+        """
+        static.metrics
+        """
+
+        failed_test_cases = []
+
+        for metric, metric_data in static_metrics.items():
+            if not metric_data.get("valid", False):
+                fail = FailedStaticTestCases(
+                    metric_name=metric,
+                    description=metric_data.get("description"),
+                    explanation=metric_data.get("explanation"),
+                )
+
+                failed_test_cases.append(fail)
+
+        return failed_test_cases
+
+    @staticmethod
+    def parse_annotations(
+        actionable_reccomendations, filters: List[str]
+    ) -> Optional[List[Annotation]]:
+        annotations = [
+            Annotation(
+                parameter_name=recc.get("parameter_name"),
+                recommendation=recc.get("recommendation"),
+                details=recc.get("details"),
+                quote=recc.get("quote"),
+            )
+            for recc in actionable_reccomendations
+            if recc.get("recommendation") in filters
+        ]
+
+        annotations = annotations if annotations else None
+
+        return annotations
+
+    @staticmethod
+    def semantic_parser(
+        metric_name, data, annotation_filters: Optional[List[str]]
+    ):
+        semantic_metric = FailedSemanticTestCases(
+            metric_name=metric_name,
+            evidence=data.get("evidence"),
+            explanation=data.get("explanation"),
+            output=data.get("output"),
+            confidence=data.get("confidence"),
+        )
+
+        if annotation_filters and (
+            annotations := ReferencelessEvalParser.parse_annotations(
+                data.get("actionable_recommendations"), annotation_filters
+            )
+        ):
+            semantic_metric.annotations = annotations
+
+        return semantic_metric

wxo_agentic_evaluation/utils/rich_utils.py

@@ -0,0 +1,188 @@
+from typing import Any, List, Optional
+
+import rich
+from rich.text import Text
+
+
+def pretty_print(content: Any, style: Optional[str] = None):
+    """
+    Utility function for stylized prints.
+    Please refer to: https://rich.readthedocs.io/en/stable/appendix/colors.html for valid  `style` strings.
+    NOTE:
+        Rich allows for nested [style][/style] tags within a string.
+        This utility only applies an outermost style wrapper using the passed `style` (ONLY for a string `content`).
+
+    :param content: The content to be printed
+    :param style: a valid `rich` colour.
+    """
+    if isinstance(content, str):
+        if style:
+            rich.print(f"[{style}]{content}[/{style}]")
+        else:
+            rich.print(content)
+    else:
+        rich.print(content)
+
+
+def warn(
+    message: str,
+    style: Optional[str] = "bold yellow",
+    prompt: Optional[str] = "WARNING ⚠️ :",
+) -> Text:
+    """Utility function for formatting a warning message."""
+    return Text(f"{prompt}{message}\n\n", style=style)
+
+
+def is_ok(
+    message: str,
+    style: Optional[str] = "bold green",
+    prompt: Optional[str] = "OK ✅ :",
+) -> Text:
+    """Utility function for formatting an OK message."""
+    return Text(f"{prompt}{message}\n\n", style=style)
+
+
+def print_done(
+    prompt: Optional[str] = "Done ✅", style: Optional[str] = "bold cyan"
+):
+    """
+    Prints a prompt indicating completion of a process/routine.
+    :param prompt: default is `"Done ✅"`
+    :param style: The style for the text (default is bold cyan).
+    """
+    pretty_print(content=prompt, style=style)
+
+
+def print_success(
+    message: str,
+    style: Optional[str] = "bold green",
+    prompt: Optional[str] = "✅ PASSED",
+):
+    """
+    Prints a success message.
+    :param message: a statement that is printed alongside a PASSED outcome.
+    :param style: The style for the text (default is bold green).
+    :param prompt: The prompt to display before the message (default is "✅ PASSED").
+    """
+    pretty_print(content=f"{prompt} - {message}", style=style)
+
+
+def print_failure(
+    message: str,
+    style: Optional[str] = "bold red",
+    prompt: Optional[str] = "❌ FAILED",
+):
+    """
+    Prints a failure message.
+    :param message: a statement that is printed alongside a FAILED outcome.
+    :param style: The style for the text (default is bold red).
+    :param prompt: The prompt to display before the message (default is "❌ FAILED").
+    """
+    pretty_print(content=f"{prompt} - {message}", style=style)
+
+
+class IncorrectParameterUtils:
+    """
+    Utility functions for handling warning and suggestion messages related to bad parameters in tool descriptions.
+    These are primarily used for providing feedback on incorrect parameter usage by the assistant in `analyze_run`.
+    """
+
+    @staticmethod
+    def suggest(message: str, style: Optional[str] = "green") -> Text:
+        """
+        Used for formatting a suggestion message for improving agent behaviour relating to bad parameter usage.
+        :param message: The suggestion message to display.
+        :param style: The style for the text (default is green).
+        :return: A rich Text object styled as a suggestion.
+        """
+        return Text(
+            f"💡 {message}\n✅ A good description is insightful of the tool's purpose, and clarifies parameter usage to the assistant.\n\n",
+            style=style,
+        )
+
+    @staticmethod
+    def format_missing_description_message(
+        tool_definition_path: str, tool_name: str
+    ) -> List[Text]:
+
+        return [
+            warn(
+                f"Tool description for '{tool_name}' not found in file: '{tool_definition_path}'"
+            ),
+            IncorrectParameterUtils.suggest(
+                f"Please consider adding a description for '{tool_name}'."
+            ),
+        ]
+
+    @staticmethod
+    def format_bad_description_message(
+        tool_name: str, tool_desc: str
+    ) -> List[Text]:
+
+        return [
+            warn(
+                f"Tool description for '{tool_name}' may be incomplete or unclear: '{tool_desc.strip()}'."
+            ),
+            IncorrectParameterUtils.suggest(
+                f"Please consider making the description for '{tool_name}' more informative on parameter usage."
+            ),
+        ]
+
+
+class TestingUtils:
+    """
+    Provides a collection of formatted messages that can be used in testing workflows.
+    """
+
+    @staticmethod
+    def print_test_header(
+        test_case_count: int,
+        test_description: str,
+        style: Optional[str] = "bold cyan",
+        prompt: Optional[str] = "\n⚙️ Testing",
+    ):
+        """
+        Print formatted test suite header.
+        :param test_case_count: # of test-cases.
+        :param test_description: a short statement explaining what is being examined.
+        For example, this can be read as: `"{\n⚙️ Testing} {20} {good tool descriptions}"`.
+        """
+        pretty_print(
+            content=f"{prompt} {test_case_count} {test_description}",
+            style=style,
+        )
+
+    @staticmethod
+    def print_error_details(
+        expected: List[str],
+        detected: List[str],
+        style: Optional[str] = "bold red",
+    ):
+        """
+        Print detailed error information.
+        An error in this context can be an assertion mis-match.
+        Use this function to display the delta.
+        :param expected: the expected outcome.
+        :param detected: the actual/observed outcome.
+        """
+        pretty_print(content=f"   Expected: {expected}", style=style)
+        pretty_print(content=f"   Detected: {detected}", style=style)
+
+    @staticmethod
+    def print_failure_summary(
+        failed_cases: List[str],
+        prompt: Optional[str] = "Failed cases",
+        style: Optional[str] = "bold red",
+    ):
+        """
+        Print summary of all failures.
+        List out the specific cases that failed the test.
+        :param failed_cases: List of failed case names, this list is iterated over to print/list all failures.
+        :param style: The style for the text (default is bold red).
+        """
+        if failed_cases:
+            pretty_print(
+                content=f"{prompt} ({len(failed_cases)}):", style=style
+            )
+            for case in failed_cases:
+                pretty_print(content=f"  - {case}", style=style)

wxo_agentic_evaluation/utils/rouge_score.py

@@ -0,0 +1,23 @@
+def lcs_length(x, y):
+    """Compute the length of the Longest Common Subsequence (LCS)."""
+    m, n = len(x), len(y)
+    dp = [[0] * (n + 1) for _ in range(m + 1)]
+    for i in range(m):
+        for j in range(n):
+            if x[i] == y[j]:
+                dp[i + 1][j + 1] = dp[i][j] + 1
+            else:
+                dp[i + 1][j + 1] = max(dp[i][j + 1], dp[i + 1][j])
+    return dp[m][n]
+
+
+def rouge_l_recall(prediction, reference):
+    """Compute ROUGE-L recall. No stemming."""
+    pred_tokens = prediction.split()
+    ref_tokens = reference.split()
+
+    lcs = lcs_length(pred_tokens, ref_tokens)
+    if len(pred_tokens) == 0:
+        return 0.0
+
+    return lcs / len(pred_tokens)